diff --git a/docs/models/index_tts.md b/docs/models/index_tts.md index 7e76a6fcf..4b654b87e 100644 --- a/docs/models/index_tts.md +++ b/docs/models/index_tts.md @@ -100,8 +100,8 @@ audiocpp_cli --task tts --family index_tts2 \ | `index_tts2.speaker_cache_slots` | integer slots | `1` | Prepared speaker-reference cache slots; set `0` to disable reuse. | | `index_tts2.emotion_cache_slots` | integer slots | `1` | Prepared emotion-reference cache slots; set `0` to disable reuse. | | `index_tts2.emotion_text_cache_slots` | integer slots | `1` | Emotion-text weight cache slots; set `0` to disable reuse. | -| `index_tts2.gpt_graph_arena_mb` | MB | `2048` | GPT graph arena size. | -| `index_tts2.s2mel_graph_arena_mb` | MB | `2048` | S2Mel graph arena size. | +| `index_tts2.gpt_graph_arena_mb` | MB | `32` | GPT graph arena size. | +| `index_tts2.s2mel_graph_arena_mb` | MB | `32` | S2Mel graph arena size. | | `index_tts2.reference_graph_arena_mb` | MB | `512` | Reference encoder and codec graph arena size. | | `index_tts2.emotion_text_prefill_graph_arena_mb` | MB | `2048` | Emotion-text prefill graph arena size. | | `index_tts2.emotion_text_decode_graph_arena_mb` | MB | `512` | Emotion-text cached-step graph arena size. | @@ -251,8 +251,8 @@ or commercial use. | `index_tts2.speaker_cache_slots` | integer slots | `1` | Prepared speaker-reference cache slots; set `0` to disable reuse. | | `index_tts2.emotion_cache_slots` | integer slots | `1` | Prepared emotion-reference cache slots; set `0` to disable reuse. | | `index_tts2.emotion_text_cache_slots` | integer slots | `1` | Emotion-text weight cache slots; set `0` to disable reuse. | -| `index_tts2.gpt_graph_arena_mb` | MB | `2048` | GPT graph arena size. | -| `index_tts2.s2mel_graph_arena_mb` | MB | `2048` | S2Mel graph arena size. | +| `index_tts2.gpt_graph_arena_mb` | MB | `32` | GPT graph arena size. | +| `index_tts2.s2mel_graph_arena_mb` | MB | `32` | S2Mel graph arena size. | | `index_tts2.reference_graph_arena_mb` | MB | `512` | Reference encoder and codec graph arena size. | | `index_tts2.emotion_text_prefill_graph_arena_mb` | MB | `2048` | Emotion-text prefill graph arena size. | | `index_tts2.emotion_text_decode_graph_arena_mb` | MB | `512` | Emotion-text cached-step graph arena size. | diff --git a/docs/models/qwen3.md b/docs/models/qwen3.md index 74dd04fd5..1ac6eb3b7 100644 --- a/docs/models/qwen3.md +++ b/docs/models/qwen3.md @@ -126,13 +126,13 @@ constant-context sizes are in MiB. | `qwen3_tts.speech_encoder_weight_type` | `native`, `f32`, `f16`, `bf16`, `q8_0` | `native` | Reference speech encoder matmul weight storage. | | `qwen3_tts.speech_decoder_weight_type` | `native`, `f32`, `f16`, `bf16`, `q8_0` | `native` | Speech decoder matmul weight storage. | | `qwen3_tts.conv_weight_type` | `native`, `f32`, `f16` | `f32` | Speech tokenizer and speaker encoder convolution weight storage. | -| `qwen3_tts.talker_graph_arena_mb` | positive integer | `256` | Talker graph arena. | +| `qwen3_tts.talker_graph_arena_mb` | positive integer | `32` | Talker graph arena. | | `qwen3_tts.speech_encoder_graph_arena_mb` | positive integer | `32` | Reference speech encoder graph arena. | | `qwen3_tts.speech_decoder_graph_arena_mb` | positive integer | `32` | Speech decoder graph arena. | | `qwen3_tts.speaker_encoder_graph_arena_mb` | positive integer | `32` | Speaker encoder graph arena. | -| `qwen3_tts.talker_constant_context_mb` | positive integer | `4096` | Talker constant tensor context. | -| `qwen3_tts.code_predictor_constant_context_mb` | positive integer | `1536` | Code predictor constant tensor context. | -| `qwen3_tts.speech_decoder_constant_context_mb` | positive integer | `1536` | Speech decoder constant tensor context. | +| `qwen3_tts.talker_constant_context_mb` | positive integer | `4` | Talker constant tensor context. | +| `qwen3_tts.code_predictor_constant_context_mb` | positive integer | `4` | Code predictor constant tensor context. | +| `qwen3_tts.speech_decoder_constant_context_mb` | positive integer | `4` | Speech decoder constant tensor context. | ## Qwen3 ASR diff --git a/docs/reports/qwen3_index_breeze_tts_optimizations.md b/docs/reports/qwen3_index_breeze_tts_optimizations.md new file mode 100644 index 000000000..e2442679c --- /dev/null +++ b/docs/reports/qwen3_index_breeze_tts_optimizations.md @@ -0,0 +1,41 @@ +# Qwen3-TTS and IndexTTS Optimization Results + +RTX 5090, CUDA Debug build, Q8 weights, 8 CPU threads, fixed seed. Each build/model ran sequentially in a fresh server with the same mixed-length request sequence and logging enabled. Peak VRAM was sampled continuously across the sequence (5 ms target interval); warm RTF is the median of four identical long-request repeats. Positive RTF changes mean slower. + +| Model/path | Before peak (MiB) | Current peak (MiB) | Peak change (MiB) | Before warm RTF | Current warm RTF | RTF change | +| --- | ---: | ---: | ---: | ---: | ---: | ---: | +| Qwen3-TTS Base clone | 6,376 | 5,070 | -1,306 | 0.10238 | 0.10417 | +1.75% | +| Qwen3-TTS CustomVoice | 6,022 | 5,068 | -954 | 0.10207 | 0.10307 | +0.98% | +| Qwen3-TTS VoiceDesign | 6,180 | 5,110 | -1,070 | 0.10334 | 0.10347 | +0.12% | +| IndexTTS2, including emotion request | 6,288 | 6,290 | +2 | 0.14590 | 0.14676 | +0.60% | +| IndexTTS2.5, including emotion request | 5,918 | 5,908 | -10 | 0.08688 | 0.08623 | -0.75% | + +Index warm RTF above measures ordinary synthesis, not the emotion request. Before the first emotion request, the measured ordinary-only peaks were: + +| Path | Before peak (MiB) | Current peak (MiB) | Peak change (MiB) | +| --- | ---: | ---: | ---: | +| IndexTTS2 ordinary synthesis | 6,288 | 5,556 | -732 | +| IndexTTS2.5 ordinary synthesis | 5,918 | 5,188 | -730 | + +## Qwen3 timing cross-check + +The same saved binaries were retested in reversed order (current, then before), without rebuilding or changing code. Four warmed repeats per path: + +| Path | Before warm RTF | Current warm RTF | RTF change | Initial A/B change | +| --- | ---: | ---: | ---: | ---: | +| Base clone | 0.10309 | 0.10196 | -1.09% | +1.75% | +| CustomVoice | 0.10215 | 0.10232 | +0.17% | +0.98% | +| VoiceDesign | 0.10190 | 0.10295 | +1.03% | +0.12% | + +Measured component medians in the reversed run: + +| Path/component | Before (ms) | Current (ms) | +| --- | ---: | ---: | +| Base cached-step compute | 489.660 | 481.306 | +| CustomVoice cached-step compute | 482.797 | 481.691 | +| VoiceDesign cached-step compute | 478.612 | 478.666 | +| VoiceDesign code-predictor compute (unchanged implementation) | 675.911 | 684.647 | + +The initial slowdown did not reproduce consistently: Base reversed direction, CustomVoice narrowed to 0.17%, and VoiceDesign's changed cached-step compute differed by only 0.054 ms. Its total slowdown was mainly in the unchanged code predictor. These results support run-to-run timing variation rather than a demonstrated cached-step regression; a specific clock/thermal/scheduling cause was not established. They do not prove that every sub-percent effect is noise. + +**Parity-safe in the tested coverage:** all tested Qwen3-TTS and IndexTTS mixed-length A/B output WAV pairs and all 18 reversed-order Qwen3 pairs were byte-identical. This is before/current C++ parity, not a new Python parity claim. diff --git a/include/engine/models/index_tts2/session.h b/include/engine/models/index_tts2/session.h index bbcd394c1..d59dc97eb 100644 --- a/include/engine/models/index_tts2/session.h +++ b/include/engine/models/index_tts2/session.h @@ -88,8 +88,8 @@ class IndexTTS2Session final runtime::TaskSpec task_; std::shared_ptr assets_; - size_t gpt_graph_arena_bytes_ = 2048ull * 1024ull * 1024ull; - size_t s2mel_graph_arena_bytes_ = 2048ull * 1024ull * 1024ull; + size_t gpt_graph_arena_bytes_ = 32ull * 1024ull * 1024ull; + size_t s2mel_graph_arena_bytes_ = 32ull * 1024ull * 1024ull; size_t reference_graph_arena_bytes_ = 512ull * 1024ull * 1024ull; size_t emotion_text_prefill_graph_arena_bytes_ = 2048ull * 1024ull * 1024ull; size_t emotion_text_decode_graph_arena_bytes_ = 512ull * 1024ull * 1024ull; diff --git a/include/engine/models/qwen3_tts/session.h b/include/engine/models/qwen3_tts/session.h index b39742da3..e010ed3f4 100644 --- a/include/engine/models/qwen3_tts/session.h +++ b/include/engine/models/qwen3_tts/session.h @@ -58,16 +58,16 @@ class Qwen3TTSSession final runtime::TaskSpec task_; std::shared_ptr assets_; - size_t talker_graph_arena_bytes_ = 256ull * 1024ull * 1024ull; + size_t talker_graph_arena_bytes_ = 32ull * 1024ull * 1024ull; size_t speech_encoder_graph_arena_bytes_ = 32ull * 1024ull * 1024ull; size_t speech_decoder_graph_arena_bytes_ = 32ull * 1024ull * 1024ull; size_t speaker_encoder_graph_arena_bytes_ = 32ull * 1024ull * 1024ull; // No-alloc GGML context capacities for reusable constant tensor descriptors. // Generous on purpose; ConstantTensorCache fits them to the host when it has // to, so a small machine is not asked to reserve what it does not have. - size_t talker_constant_context_bytes_ = 4ull * 1024ull * 1024ull * 1024ull; - size_t code_predictor_constant_context_bytes_ = 1536ull * 1024ull * 1024ull; - size_t speech_decoder_constant_context_bytes_ = 1536ull * 1024ull * 1024ull; + size_t talker_constant_context_bytes_ = 4ull * 1024ull * 1024ull; + size_t code_predictor_constant_context_bytes_ = 4ull * 1024ull * 1024ull; + size_t speech_decoder_constant_context_bytes_ = 4ull * 1024ull * 1024ull; engine::assets::TensorStorageType talker_weight_storage_type_ = engine::assets::TensorStorageType::Native; engine::assets::TensorStorageType speech_encoder_weight_storage_type_ = engine::assets::TensorStorageType::Native; engine::assets::TensorStorageType speech_decoder_weight_storage_type_ = engine::assets::TensorStorageType::Native; diff --git a/model_specs/breeze_tts.json b/model_specs/breeze_tts.json index 7a9e9df4e..88a844cae 100644 --- a/model_specs/breeze_tts.json +++ b/model_specs/breeze_tts.json @@ -157,18 +157,18 @@ { "name": "graph_arena_mb", "type": "int", - "description": "Reusable runtime graph arena size in MiB; default 1024.", + "description": "Reusable runtime graph arena size in MiB; default 32.", "required": false, "min": 1, - "default": 1024 + "default": 32 }, { "name": "weight_context_mb", "type": "int", - "description": "Weight loading context size in MiB; default 2048.", + "description": "Weight loading context size in MiB; default 4.", "required": false, "min": 1, - "default": 2048 + "default": 4 }, { "name": "reference_cache_slots", diff --git a/model_specs/index_tts2.json b/model_specs/index_tts2.json index 45c03c917..12e3667cb 100644 --- a/model_specs/index_tts2.json +++ b/model_specs/index_tts2.json @@ -222,7 +222,7 @@ "description": "Context or graph arena size in MiB.", "required": false, "min": 1, - "default": 2048 + "default": 32 }, { "name": "s2mel_graph_arena_mb", @@ -230,7 +230,7 @@ "description": "Context or graph arena size in MiB.", "required": false, "min": 1, - "default": 2048 + "default": 32 }, { "name": "reference_graph_arena_mb", diff --git a/model_specs/qwen3_tts.json b/model_specs/qwen3_tts.json index 21fb8bd48..627bfd7f9 100644 --- a/model_specs/qwen3_tts.json +++ b/model_specs/qwen3_tts.json @@ -228,10 +228,10 @@ { "name": "talker_graph_arena_mb", "type": "int", - "description": "Talker graph arena size in MiB; default 256.", + "description": "Talker graph arena size in MiB; default 32.", "required": false, "min": 1, - "default": 256 + "default": 32 }, { "name": "speech_encoder_graph_arena_mb", @@ -260,26 +260,26 @@ { "name": "talker_constant_context_mb", "type": "int", - "description": "Talker constant tensor context size in MiB; default 4096.", + "description": "Talker constant tensor context size in MiB; default 4.", "required": false, "min": 1, - "default": 4096 + "default": 4 }, { "name": "code_predictor_constant_context_mb", "type": "int", - "description": "Code predictor constant tensor context size in MiB; default 1536.", + "description": "Code predictor constant tensor context size in MiB; default 4.", "required": false, "min": 1, - "default": 1536 + "default": 4 }, { "name": "speech_decoder_constant_context_mb", "type": "int", - "description": "Speech decoder constant tensor context size in MiB; default 1536.", + "description": "Speech decoder constant tensor context size in MiB; default 4.", "required": false, "min": 1, - "default": 1536 + "default": 4 } ], "load": [] diff --git a/src/framework/modules/vocoders/bigvgan_vocoder.cpp b/src/framework/modules/vocoders/bigvgan_vocoder.cpp index ea744483a..7b7185880 100644 --- a/src/framework/modules/vocoders/bigvgan_vocoder.cpp +++ b/src/framework/modules/vocoders/bigvgan_vocoder.cpp @@ -793,7 +793,7 @@ class BigVganRunner { } release_graph(); ggml_init_params params{ - 1024ull * 1024ull * 1024ull, + 32ull * 1024ull * 1024ull, nullptr, true, }; @@ -1055,7 +1055,7 @@ BigVganVocoderComponent BigVganVocoderComponent::load_from_tensor_source( weights->execution_context->backend(), weights->execution_context->backend_type(), "framework.bigvgan.weights", - 1024ull * 1024ull * 1024ull); + 4ull * 1024ull * 1024ull); for (const auto & tensor : source->tensors()) { weights->parameter_count += tensor_elements(tensor.shape); diff --git a/src/models/breeze_tts/session.cpp b/src/models/breeze_tts/session.cpp index c13688fc0..5fba436a6 100644 --- a/src/models/breeze_tts/session.cpp +++ b/src/models/breeze_tts/session.cpp @@ -185,11 +185,11 @@ BreezeTTSSession::BreezeTTSSession( const auto graph_arena_bytes = runtime::parse_size_mb_option( options.options, {"graph_arena_mb"}, - 1024ull * 1024ull * 1024ull); + 32ull * 1024ull * 1024ull); const auto weight_context_bytes = runtime::parse_size_mb_option( options.options, {"weight_context_mb"}, - 2048ull * 1024ull * 1024ull); + 4ull * 1024ull * 1024ull); const auto attention_preference = attention_preference_from_options(options); trace_attention_preference(attention_preference); const auto bf16_activations = bf16_activation_mode_from_options(options); diff --git a/src/models/index_tts2/gpt.cpp b/src/models/index_tts2/gpt.cpp index 958ad1cfb..69fb71caf 100644 --- a/src/models/index_tts2/gpt.cpp +++ b/src/models/index_tts2/gpt.cpp @@ -1146,7 +1146,7 @@ class IndexTTS2GptRuntime::ConditioningGraph { output_dims_ = out.shape.dims[2]; ggml_set_output(output_); - graph_ = ggml_new_graph_custom(ctx_.get(), static_cast(std::max(65536, frames_ * 4096 + 8192)), false); + graph_ = ggml_new_graph_custom(ctx_.get(), 65536, false); ggml_build_forward_expand(graph_, output_); input_buffer_ = ggml_backend_alloc_ctx_tensors(input_ctx_.get(), execution_.backend()); if (input_buffer_ == nullptr) { @@ -1438,7 +1438,7 @@ class IndexTTS2GptRuntime::PrefillGraph { mel = core::reshape_tensor(ctx, core::ensure_backend_addressable_layout(ctx, mel), core::TensorShape::from_dims({1, 1, kModelDim})); auto x = modules::ConcatModule({1}).build(ctx, conds, text); x = modules::ConcatModule({1}).build(ctx, x, mel); - graph_ = ggml_new_graph_custom(ctx_.get(), static_cast(std::max(65536, prompt_steps_ * 8192)), false); + graph_ = ggml_new_graph_custom(ctx_.get(), 65536, false); for (const auto & layer : weights_->gpt_layers) { auto out = gpt2_layer_full(ctx, x, layer); x = out.output; @@ -1681,10 +1681,7 @@ class IndexTTS2GptRuntime::ForwardGraph { output_ = core::ensure_backend_addressable_layout(ctx, x).tensor; ggml_set_output(output_); - graph_ = ggml_new_graph_custom( - ctx_.get(), - static_cast(std::max(65536, (kV2ConditionTokens + text_steps_ + mel_steps_) * 8192)), - false); + graph_ = ggml_new_graph_custom(ctx_.get(), 65536, false); ggml_build_forward_expand(graph_, output_); input_buffer_ = ggml_backend_alloc_ctx_tensors(input_ctx_.get(), execution_.backend()); if (input_buffer_ == nullptr) { diff --git a/src/models/index_tts2/s2mel.cpp b/src/models/index_tts2/s2mel.cpp index a6461faa5..ae0eec10c 100644 --- a/src/models/index_tts2/s2mel.cpp +++ b/src/models/index_tts2/s2mel.cpp @@ -659,7 +659,7 @@ class IndexTTS2S2MelRuntime::GptLayerGraph { x = modules::LinearModule({128, kContentDim, true}).build(ctx, x, weights_->gpt_layer.linear2); output_ = core::ensure_backend_addressable_layout(ctx, x).tensor; ggml_set_output(output_); - graph_ = ggml_new_graph_custom(ctx_.get(), static_cast(std::max(8192, frames_ * 128)), false); + graph_ = ggml_new_graph_custom(ctx_.get(), 8192, false); ggml_build_forward_expand(graph_, output_); input_buffer_ = ggml_backend_alloc_ctx_tensors(input_ctx_.get(), execution_.backend()); if (input_buffer_ == nullptr) { @@ -795,7 +795,7 @@ class IndexTTS2S2MelRuntime::LengthRegulatorGraph { auto out = modules::MulModule{}.build(ctx, x, mask); output_ = core::ensure_backend_addressable_layout(ctx, out).tensor; ggml_set_output(output_); - graph_ = ggml_new_graph_custom(ctx_.get(), static_cast(std::max(32768, output_frames_ * 512)), false); + graph_ = ggml_new_graph_custom(ctx_.get(), 32768, false); ggml_build_forward_expand(graph_, output_); input_buffer_ = ggml_backend_alloc_ctx_tensors(input_ctx_.get(), execution_.backend()); if (input_buffer_ == nullptr) { @@ -937,7 +937,7 @@ class IndexTTS2S2MelRuntime::CfmGraph { weights_->cfm); output_ = core::ensure_backend_addressable_layout(ctx, output).tensor; ggml_set_output(output_); - graph_ = ggml_new_graph_custom(ctx_.get(), static_cast(std::max(131072, frames_ * 4096)), false); + graph_ = ggml_new_graph_custom(ctx_.get(), 65536, false); ggml_build_forward_expand(graph_, output_); input_buffer_ = ggml_backend_alloc_ctx_tensors(input_ctx_.get(), execution_.backend()); if (input_buffer_ == nullptr) { diff --git a/src/models/index_tts2/session.cpp b/src/models/index_tts2/session.cpp index c2177de3d..449d990d8 100644 --- a/src/models/index_tts2/session.cpp +++ b/src/models/index_tts2/session.cpp @@ -305,13 +305,6 @@ IndexTTS2Session::IndexTTS2Session( assets_, options.backend, conv_weight_storage_type_); - qwen_emotion_ = std::make_unique( - assets_, - execution_context(), - emotion_text_prefill_graph_arena_bytes_, - emotion_text_decode_graph_arena_bytes_, - weight_context_bytes_, - matmul_weight_storage_type_); int64_t matrix_rows = 0; for (const int64_t count : assets_->config.emo_num) { matrix_rows += count; @@ -582,6 +575,11 @@ std::vector IndexTTS2Session::resolve_emotion_vector( const bool will_evict = emotion_text_weights_cache_.capacity() > 0 && emotion_text_weights_cache_.size() >= emotion_text_weights_cache_.capacity(); + if (!qwen_emotion_) { + qwen_emotion_ = std::make_unique( + assets_, execution_context(), emotion_text_prefill_graph_arena_bytes_, + emotion_text_decode_graph_arena_bytes_, weight_context_bytes_, matmul_weight_storage_type_); + } explicit_weights = qwen_emotion_->infer(emotion_text, emotion_text_max_tokens_).values; if (mem_saver_) { qwen_emotion_->release_graphs(); diff --git a/src/models/qwen3_tts/talker.cpp b/src/models/qwen3_tts/talker.cpp index 889ac8867..4f8bf0480 100644 --- a/src/models/qwen3_tts/talker.cpp +++ b/src/models/qwen3_tts/talker.cpp @@ -16,6 +16,7 @@ #include "engine/framework/core/constant_tensor_cache.h" +#include #include #include @@ -955,13 +956,31 @@ class TalkerPrefillGraph { } hidden_output_ = decoder_out.hidden.tensor; logits_output_ = decoder_out.logits.tensor; + ggml_set_input(input_); + ggml_set_input(positions_); + ggml_set_output(positions_); + if (attention_mask_ != nullptr) { + ggml_set_input(attention_mask_); + ggml_set_output(attention_mask_); + } + ggml_set_output(hidden_output_); + if (hidden_output_->view_src != nullptr) ggml_set_output(hidden_output_->view_src); ggml_set_output(logits_output_); graph_ = ggml_new_graph_custom(ctx_.get(), 65536, false); ggml_build_forward_expand(graph_, logits_output_); + // The CPU exports every layer's K/V after execution, not just logits. + for (size_t layer = 0; layer < keys_.size(); ++layer) { + for (auto * output : {keys_[layer], values_[layer]}) { + ggml_set_output(output); + if (output->view_src != nullptr) ggml_set_output(output->view_src); + ggml_build_forward_expand(graph_, output); + } + } constants.finish_graph(); constants.ensure_uploaded(); - buffer_ = ggml_backend_alloc_ctx_tensors(ctx_.get(), weights_->backend()); - if (buffer_ == nullptr) { + graph_allocator_.reset(ggml_gallocr_new(ggml_backend_get_default_buffer_type(weights_->backend()))); + if (!graph_allocator_ || !ggml_gallocr_reserve(graph_allocator_.get(), graph_) || + !ggml_gallocr_alloc_graph(graph_allocator_.get(), graph_)) { throw std::runtime_error("failed to allocate Qwen3 talker prefill graph"); } std::vector positions(static_cast(prompt_capacity_), 0); @@ -977,9 +996,7 @@ class TalkerPrefillGraph { ~TalkerPrefillGraph() { engine::core::release_backend_graph_resources(weights_->backend(), graph_); - if (buffer_ != nullptr) { - ggml_backend_buffer_free(buffer_); - } + graph_allocator_.reset(); } bool matches(const Qwen3TalkerWeightsRuntime & weights, int64_t prompt_capacity) const { @@ -1038,7 +1055,7 @@ class TalkerPrefillGraph { std::vector keys_; std::vector values_; ggml_cgraph * graph_ = nullptr; - ggml_backend_buffer_t buffer_ = nullptr; + std::unique_ptr graph_allocator_{nullptr, ggml_gallocr_free}; }; class TalkerCachedStepGraph { @@ -1086,12 +1103,23 @@ class TalkerCachedStepGraph { step_cache_ = std::move(decoder_out.cache); hidden_output_ = decoder_out.hidden.tensor; logits_output_ = decoder_out.logits.tensor; + for (auto * input : {input_, positions_, cache_slot_, attention_mask_}) ggml_set_input(input); + ggml_set_output(hidden_output_); + if (hidden_output_->view_src != nullptr) ggml_set_output(hidden_output_->view_src); + // Persistent state must not be recycled between autoregressive steps. + for (int64_t layer = 0; layer < config.num_hidden_layers; ++layer) { + for (const auto & state : {step_cache_.key_tensor(layer), step_cache_.value_tensor(layer)}) { + ggml_set_input(state.tensor); + ggml_set_output(state.tensor); + } + } ggml_set_output(logits_output_); ggml_build_forward_expand(graph_, logits_output_); constants.finish_graph(); constants.ensure_uploaded(); - buffer_ = ggml_backend_alloc_ctx_tensors(ctx_.get(), weights_->backend()); - if (buffer_ == nullptr) { + graph_allocator_.reset(ggml_gallocr_new(ggml_backend_get_default_buffer_type(weights_->backend()))); + if (!graph_allocator_ || !ggml_gallocr_reserve(graph_allocator_.get(), graph_) || + !ggml_gallocr_alloc_graph(graph_allocator_.get(), graph_)) { throw std::runtime_error("failed to allocate Qwen3 talker cached step graph"); } attention_mask_buffer_.assign(static_cast(cache_steps_), ggml_fp32_to_fp16(-INFINITY)); @@ -1099,9 +1127,7 @@ class TalkerCachedStepGraph { ~TalkerCachedStepGraph() { engine::core::release_backend_graph_resources(weights_->backend(), graph_); - if (buffer_ != nullptr) { - ggml_backend_buffer_free(buffer_); - } + graph_allocator_.reset(); } bool can_run(const Qwen3TalkerWeightsRuntime & weights, int64_t required_capacity) const { @@ -1185,7 +1211,7 @@ class TalkerCachedStepGraph { std::vector attention_mask_buffer_; runtime::TransformerKVCache step_cache_; ggml_cgraph * graph_ = nullptr; - ggml_backend_buffer_t buffer_ = nullptr; + std::unique_ptr graph_allocator_{nullptr, ggml_gallocr_free}; CachedStepTiming last_timing_; }; diff --git a/src/models/qwen3_tts/tokenizer_speech_decoder.cpp b/src/models/qwen3_tts/tokenizer_speech_decoder.cpp index 3a1b89478..990109aaf 100644 --- a/src/models/qwen3_tts/tokenizer_speech_decoder.cpp +++ b/src/models/qwen3_tts/tokenizer_speech_decoder.cpp @@ -1226,6 +1226,7 @@ runtime::AudioBuffer Qwen3SpeechTokenizerDecoderRuntime::decode(const Qwen3Speec graph == nullptr || !graph->matches(*weights_, chunk_frames, execution_context_->backend(), threads); if (graph_rebuilt) { const auto build_start = Clock::now(); + graph.reset(); auto replacement = std::make_unique( weights_, chunk_frames,