From 2fc346c7d18f1e858f88e6de08c2d0d836b09ec6 Mon Sep 17 00:00:00 2001 From: Chris Thompson Date: Tue, 15 Sep 2026 16:36:27 -0600 Subject: [PATCH 1/3] kokoro_tts: accept a supplied phoneme stream, bypassing the built-in G2P MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds a `phonemes` request option, the first user of the list-valued options this branch's base introduces. When set, its entries are synthesized as-is instead of running eSpeak-ng over the text. The built-in G2P is one opinion about pronunciation, and a caller may have a better one for its material: a lexicon the engine does not carry, a language it does not cover, a domain vocabulary, or a pronunciation the application has already shown its user and must now speak the same way. Today there is no way to express any of that -- phonemize_text() is unconditional and the option validator rejects anything undeclared, so the door is bolted rather than shut. ⚠ THE LIST IS THE POINT, not a convenience. Chunking splits the TEXT, and nothing here knows where the matching cut points in someone else's phoneme stream are -- only their G2P does. So the caller supplies the chunks, one per entry, rendered in order and merged exactly as text chunks are. A caller whose document exceeds the 510-symbol limit still makes ONE call and gets ONE buffer back rather than stitching audio itself. ⚠ AND A CALLER'S STREAM IS VALIDATED, where our own G2P's output is not. The skip added for our own output is right there and wrong here, because you can only demand a correction from someone able to make one. Canonical IPA writes a diphthong as two symbols where Kokoro writes one, so dropping the off-glide renders "like" as "lack" -- byte-identical to having written it that way, with no error. A caller can fix that if told; eSpeak's syllabic mark for "button" nobody can. Two details worth review: - The run cache is keyed on the entry. It keys on text today, and with a list every chunk shares one text, so that key is the only thing telling them apart. - prepare() sizes the graph on the largest entry, since with a caller's chunking the text boundaries are not the ones run() will use. ⚠ The declared option set is embedded in the GGUF at conversion time, so an existing model file needs --model-spec-override before it will accept the option. A package converted with prepare_kokoro_gguf.py after this change declares it natively. --- include/engine/models/kokoro_tts/frontend.h | 11 ++- model_specs/kokoro_tts.json | 6 ++ src/framework/model_spec/options.cpp | 1 + src/models/kokoro_tts/frontend.cpp | 48 +++++++++++-- src/models/kokoro_tts/session.cpp | 74 ++++++++++++++++++--- 5 files changed, 121 insertions(+), 19 deletions(-) diff --git a/include/engine/models/kokoro_tts/frontend.h b/include/engine/models/kokoro_tts/frontend.h index 7b66789bb..f1ed03533 100644 --- a/include/engine/models/kokoro_tts/frontend.h +++ b/include/engine/models/kokoro_tts/frontend.h @@ -31,14 +31,21 @@ KokoroFrontendSessionState resolve_kokoro_frontend_session_state( const std::optional & voice, const KokoroAssets & assets); +// `phoneme_override`, when non-empty, is synthesized as-is INSTEAD of running the built-in +// G2P over `text`. It lets a caller with its own grapheme-to-phoneme stage — a lexicon the +// engine does not carry, a language it does not cover, a pronunciation the application has +// already shown its user — drive the model directly. `text` is still required and its +// language must still agree with the voice; only the phonemization is replaced. KokoroSynthesisInput build_kokoro_synthesis_input( const runtime::Transcript & text, const KokoroFrontendSessionState & state, - const KokoroAssets & assets); + const KokoroAssets & assets, + const std::string & phoneme_override = std::string()); int64_t estimate_kokoro_request_tokens( const runtime::SessionPreparationRequest & request, const KokoroFrontendSessionState & state, - const KokoroAssets & assets); + const KokoroAssets & assets, + const std::string & phoneme_override = std::string()); } // namespace engine::models::kokoro_tts diff --git a/model_specs/kokoro_tts.json b/model_specs/kokoro_tts.json index a4b4faff3..ee5d3cc5d 100644 --- a/model_specs/kokoro_tts.json +++ b/model_specs/kokoro_tts.json @@ -29,6 +29,12 @@ "required": false, "min": 0 }, + { + "name": "phonemes", + "type": "string_list", + "description": "Kokoro-vocabulary phonemes to synthesize directly, bypassing the built-in eSpeak-ng G2P. One entry per chunk: entries are rendered in order and merged into one result, so the caller chooses the split points that only its own G2P knows. Each entry must be at most 510 symbols. Text is still required and its language must still match the voice.", + "required": false + }, { "name": "text_chunk_size", "type": "int", diff --git a/src/framework/model_spec/options.cpp b/src/framework/model_spec/options.cpp index cf0324d47..7fb518837 100644 --- a/src/framework/model_spec/options.cpp +++ b/src/framework/model_spec/options.cpp @@ -50,6 +50,7 @@ const std::unordered_map> & shared_ {"min_p", {"float"}}, {"matmul_weight_type", {"enum"}}, {"negative_prompt", {"string"}}, + {"phonemes", {"string_list"}}, {"num_beams", {"int"}}, {"num_inference_steps", {"int"}}, {"output_sample_rate", {"int"}}, diff --git a/src/models/kokoro_tts/frontend.cpp b/src/models/kokoro_tts/frontend.cpp index 6bdfa82ac..a4c26e991 100644 --- a/src/models/kokoro_tts/frontend.cpp +++ b/src/models/kokoro_tts/frontend.cpp @@ -117,9 +117,22 @@ struct EncodedInputIds { size_t phoneme_count = 0; }; +/// `reject_unknown` decides what an out-of-vocabulary symbol means, and the answer depends +/// entirely on who produced the stream. +/// +/// ⚠ FALSE for our own G2P, TRUE for a caller's. The reference implementation (hexgrad/Kokoro's +/// KModel, `filter(None, map(vocab.get, phonemes))`) drops what it cannot tokenize, and for our +/// own output that is the only sane answer: eSpeak emits a syllabic mark for "button" that this +/// vocabulary has no id for, and nobody downstream can do anything about it. +/// +/// A CALLER'S stream is the opposite case. They can fix it, so telling them is strictly more +/// useful than guessing -- and guessing is not harmless here. Canonical IPA writes a diphthong +/// as two symbols, and Kokoro writes it as one, so dropping the off-glide silently turns +/// `lˈaᶦk` into `lˈak`: "like" becomes "lack", with no error and audio that sounds deliberate. EncodedInputIds encode_input_ids_and_count( const std::string & phonemes, - const KokoroAssets & assets) { + const KokoroAssets & assets, + bool reject_unknown) { EncodedInputIds encoded; encoded.ids.reserve(phonemes.size() + 2); encoded.ids.push_back(0); @@ -149,6 +162,13 @@ EncodedInputIds encode_input_ids_and_count( const std::string symbol = phonemes.substr(i, width); const auto it = assets.vocab.find(symbol); if (it == assets.vocab.end()) { + if (reject_unknown) { + throw std::runtime_error( + "Kokoro vocab is missing phoneme symbol: " + symbol + + "; supplied phonemes must be in Kokoro's own " + std::to_string(assets.vocab.size()) + + "-symbol vocabulary, which is not canonical IPA -- a diphthong is one symbol there" + " and two in IPA, so an off-glide is a common cause"); + } // Skipped, not fatal, matching the reference implementation: hexgrad/Kokoro's KModel // tokenizes with `filter(None, map(vocab.get, phonemes))`, which drops any phoneme the // 114-entry vocab has no id for. @@ -231,15 +251,28 @@ KokoroFrontendSessionState resolve_kokoro_frontend_session_state( KokoroSynthesisInput build_kokoro_synthesis_input( const runtime::Transcript & text, const KokoroFrontendSessionState & state, - const KokoroAssets & assets) { + const KokoroAssets & assets, + const std::string & phoneme_override) { if (state.voice_pack == nullptr) { throw std::runtime_error("Kokoro frontend session voice pack was not prepared"); } - const std::string phonemes = phonemize_text(text, state.language_code, assets); - const EncodedInputIds encoded = encode_input_ids_and_count(phonemes, assets); + const bool supplied = !phoneme_override.empty(); + const std::string phonemes = + supplied ? phoneme_override : phonemize_text(text, state.language_code, assets); + // Strict for a caller's stream, lenient for our own -- see encode_input_ids_and_count. + const EncodedInputIds encoded = encode_input_ids_and_count(phonemes, assets, supplied); if (encoded.phoneme_count > 510) { + // Two different failures wearing one message helps nobody: the caller who supplied the + // phonemes can fix this by sending less, and is told so; the caller who supplied text + // is hitting an engine limitation and is told that instead. throw std::runtime_error( - "Kokoro phoneme string exceeds 510 symbols; segmenting is not implemented in the framework path yet"); + supplied + ? "Kokoro phoneme chunk exceeds 510 symbols; split the supplied phonemes across " + "more list entries (each entry is rendered separately and the audio is merged)" + : "Kokoro phoneme string exceeds 510 symbols; segmenting is not implemented in the framework path yet"); + } + if (supplied) { + engine::debug::trace_log_scalar("kokoro.supplied_phoneme_count", static_cast(encoded.phoneme_count)); } KokoroSynthesisInput input; input.voice_id = state.voice_id; @@ -257,11 +290,12 @@ KokoroSynthesisInput build_kokoro_synthesis_input( int64_t estimate_kokoro_request_tokens( const runtime::SessionPreparationRequest & request, const KokoroFrontendSessionState & state, - const KokoroAssets & assets) { + const KokoroAssets & assets, + const std::string & phoneme_override) { if (!request.text.has_value()) { return 0; } - const auto input = build_kokoro_synthesis_input(*request.text, state, assets); + const auto input = build_kokoro_synthesis_input(*request.text, state, assets, phoneme_override); return static_cast(input.input_ids.size()); } diff --git a/src/models/kokoro_tts/session.cpp b/src/models/kokoro_tts/session.cpp index 0547d50c7..4f37c581a 100644 --- a/src/models/kokoro_tts/session.cpp +++ b/src/models/kokoro_tts/session.cpp @@ -236,8 +236,25 @@ void KokoroTTSSession::prepare_decoder_graph_capacity(int64_t capacity) { engine::debug::timing_log_scalar("kokoro.prepare.decoder_runtime_build_ms", build_ms); } +namespace { + +/// The "no override, use the built-in G2P" sentinel, so the chunk loop can bind a reference. +const std::string kNoSuppliedPhonemes; + +/// A list-valued request option, or empty when the caller did not set one. +const std::vector & find_option_array( + const std::unordered_map> & option_arrays, + const std::string & key) { + static const std::vector none; + const auto it = option_arrays.find(key); + return it == option_arrays.end() ? none : it->second; +} + +} // namespace + void KokoroTTSSession::prepare(const runtime::SessionPreparationRequest & request) { - runtime::validate_spec_backed_request_options(request.options, *contract_, kModelName); + runtime::validate_spec_backed_request_options( + request.options, request.option_arrays, *contract_, kModelName); if (const auto seed = runtime::parse_u64_option(request.options, {"seed"})) { if (rng_seed_ != *seed) { rng_seed_ = *seed; @@ -248,18 +265,32 @@ void KokoroTTSSession::prepare(const runtime::SessionPreparationRequest & reques } auto adapter = make_graph_capacity_adapter(); int64_t request_size = 0; + const auto prepare_phonemes = find_option_array(request.option_arrays, "phonemes"); if (request.text.has_value()) { const int64_t text_chunk_size = engine::text::parse_text_chunk_size_override(request.options).value_or(kDefaultTextChunkSize); - const auto text_chunks = engine::text::split_text_chunks(request.text->text, text_chunk_size); + // The graph is sized for the LARGEST chunk either way. With supplied phonemes the + // caller's own chunking decides that, so the text is not split -- splitting it would + // size the graph against a boundary run() is never going to use. + const auto text_chunks = prepare_phonemes.empty() + ? engine::text::split_text_chunks(request.text->text, text_chunk_size) + : std::vector{request.text->text}; for (const auto & chunk : text_chunks) { runtime::SessionPreparationRequest chunk_request = request; chunk_request.text = runtime::Transcript{chunk, request.text->language}; const auto frontend_state = resolve_kokoro_frontend_session_state(chunk_request.text, chunk_request.voice, *assets_); - request_size = std::max( - request_size, - estimate_kokoro_request_tokens(chunk_request, frontend_state, *assets_)); + if (prepare_phonemes.empty()) { + request_size = std::max( + request_size, + estimate_kokoro_request_tokens(chunk_request, frontend_state, *assets_)); + } else { + for (const auto & supplied : prepare_phonemes) { + request_size = std::max( + request_size, + estimate_kokoro_request_tokens(chunk_request, frontend_state, *assets_, supplied)); + } + } } } graph_capacity_controller_.ensure_prepared(adapter, request_size); @@ -271,19 +302,36 @@ runtime::TaskResult KokoroTTSSession::run(const runtime::TaskRequest & request) if (!request.text_input.has_value()) { throw std::runtime_error("Kokoro TTS run requires text_input"); } - runtime::validate_spec_backed_request_options(request.options, *contract_, kModelName); + runtime::validate_spec_backed_request_options( + request.options, request.option_arrays, *contract_, kModelName); const int64_t text_chunk_size = engine::text::parse_text_chunk_size_override(request.options).value_or(kDefaultTextChunkSize); - const auto chunk_requests = runtime::chunk_text_request(request, text_chunk_size); + // ⚠ WHEN PHONEMES ARE SUPPLIED THE CALLER OWNS THE CHUNKING. Chunking splits the TEXT, and + // nothing here knows where the matching cut points in someone else's phoneme stream are -- + // only their G2P does. So the option is a LIST: one entry per chunk, rendered in order and + // merged into one result exactly as text chunks are. A caller whose document exceeds the + // 510-symbol limit therefore still makes ONE call and gets ONE buffer back, instead of + // having to stitch the audio itself. + const auto supplied_phonemes = find_option_array(request.option_arrays, "phonemes"); + const auto chunk_requests = supplied_phonemes.empty() + ? runtime::chunk_text_request(request, text_chunk_size) + : std::vector(supplied_phonemes.size(), request); engine::debug::trace_log_scalar("kokoro.text_chunk_size", text_chunk_size); engine::debug::trace_log_scalar("kokoro.text_chunk_count", static_cast(chunk_requests.size())); + if (!supplied_phonemes.empty()) { + engine::debug::trace_log_scalar( + "kokoro.supplied_phoneme_chunks", static_cast(supplied_phonemes.size())); + } double frontend_ms = 0.0; double inference_ms = 0.0; double predictor_ms = 0.0; double decoder_ms = 0.0; runtime::AudioBuffer merged_audio; - for (const auto & chunk_request : chunk_requests) { + for (size_t chunk_index = 0; chunk_index < chunk_requests.size(); ++chunk_index) { + const auto & chunk_request = chunk_requests[chunk_index]; + const std::string & chunk_phonemes = + supplied_phonemes.empty() ? kNoSuppliedPhonemes : supplied_phonemes[chunk_index]; const auto frontend_state = resolve_kokoro_frontend_session_state(chunk_request.text_input, chunk_request.voice, *assets_); const std::string cache_key = @@ -291,14 +339,20 @@ runtime::TaskResult KokoroTTSSession::run(const runtime::TaskRequest & request) frontend_state.language_code + ":" + std::to_string(frontend_state.speaking_rate) + ":" + std::to_string(chunk_request.text_input->text.size()) + ":" + - chunk_request.text_input->text; + chunk_request.text_input->text + ":" + + // Without this, two requests with the same text and different supplied phonemes + // would hit the same cache entry and the second would be spoken as the first -- + // and with a list, every chunk shares one text, so this is the ONLY thing telling + // them apart. + chunk_phonemes; KokoroSynthesisInput input; frontend_ms += measure_ms([&]() { if (!cache_key.empty() && cached_input_ && cache_key == cached_request_key_) { input = *cached_input_; return; } - input = build_kokoro_synthesis_input(*chunk_request.text_input, frontend_state, *assets_); + input = build_kokoro_synthesis_input( + *chunk_request.text_input, frontend_state, *assets_, chunk_phonemes); if (!cache_key.empty()) { cached_request_key_ = cache_key; cached_input_ = std::make_unique(input); From 2b22e7d2a0c91d57f58808b3fdcb92f8ce430644 Mon Sep 17 00:00:00 2001 From: Chris Thompson Date: Wed, 16 Sep 2026 12:38:23 -0600 Subject: [PATCH 2/3] kokoro_tts: let a published package take phonemes its spec predates Review on #577: follow irodori_tts.codec_backend rather than waiting for the package to be regenerated. A GGUF's embedded schema-v1 contract is authoritative, so the shipped kokoro-82m-q8_0.gguf -- written before this option existed -- rejected `phonemes` as an unknown request option, and a published package cannot be edited in place. The engine serves the request either way; only the validation disagreed. So the key is dropped from the validation COPY when the contract does not declare it, the same shape as irodori_tts.codec_backend and the Parakeet TDT VAD controls. Unrelated unknown options still fail, and a contract that does declare `phonemes` validates it as before. Driven through the C ABI against the shipped package with no spec override, and against the same package with model_specs/kokoro_tts.json as an override, so both sides of the branch ran: package declares a 'phonemes' request option: no (and: yes) == supplied phonemes, one chunk == ok 1.52s == supplied phonemes, three chunks == ok 5.10s == no phonemes, built-in G2P == ok 1.52s supplied 'R' -> Kokoro vocab is missing phoneme symbol: R supplied 'not_a_kokoro_option' -> unknown Kokoro TTS request option Identical on both, including the two rejections. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_014X4jSwJu1ScZVgUspcA9HC --- src/models/kokoro_tts/session.cpp | 33 +++++++++++++++++++++++++------ 1 file changed, 27 insertions(+), 6 deletions(-) diff --git a/src/models/kokoro_tts/session.cpp b/src/models/kokoro_tts/session.cpp index 4f37c581a..7e4e4f1d3 100644 --- a/src/models/kokoro_tts/session.cpp +++ b/src/models/kokoro_tts/session.cpp @@ -241,6 +241,29 @@ namespace { /// The "no override, use the built-in G2P" sentinel, so the chunk loop can bind a reference. const std::string kNoSuppliedPhonemes; +constexpr const char * kPhonemesOption = "phonemes"; + +/// Request options, validated against the package's own contract. +/// +/// Older standalone GGUF packages embed a schema-v1 contract written before +/// `phonemes` existed, and a published package cannot be edited in place. Drop +/// the key from the VALIDATION COPY so those packages can still be given +/// phonemes -- the engine serves the request either way -- while every unrelated +/// unknown option is still rejected. Same shape as irodori_tts.codec_backend and +/// the Parakeet TDT VAD controls, which are older options in the same position. +void validate_request_options( + const std::unordered_map & options, + const std::unordered_map> & option_arrays, + const engine::model_spec::ModelContract & contract) { + if (contract.request_option_keys.find(kPhonemesOption) != contract.request_option_keys.end()) { + runtime::validate_spec_backed_request_options(options, option_arrays, contract, kModelName); + return; + } + auto validation_arrays = option_arrays; + validation_arrays.erase(kPhonemesOption); + runtime::validate_spec_backed_request_options(options, validation_arrays, contract, kModelName); +} + /// A list-valued request option, or empty when the caller did not set one. const std::vector & find_option_array( const std::unordered_map> & option_arrays, @@ -253,8 +276,7 @@ const std::vector & find_option_array( } // namespace void KokoroTTSSession::prepare(const runtime::SessionPreparationRequest & request) { - runtime::validate_spec_backed_request_options( - request.options, request.option_arrays, *contract_, kModelName); + validate_request_options(request.options, request.option_arrays, *contract_); if (const auto seed = runtime::parse_u64_option(request.options, {"seed"})) { if (rng_seed_ != *seed) { rng_seed_ = *seed; @@ -265,7 +287,7 @@ void KokoroTTSSession::prepare(const runtime::SessionPreparationRequest & reques } auto adapter = make_graph_capacity_adapter(); int64_t request_size = 0; - const auto prepare_phonemes = find_option_array(request.option_arrays, "phonemes"); + const auto prepare_phonemes = find_option_array(request.option_arrays, kPhonemesOption); if (request.text.has_value()) { const int64_t text_chunk_size = engine::text::parse_text_chunk_size_override(request.options).value_or(kDefaultTextChunkSize); @@ -302,8 +324,7 @@ runtime::TaskResult KokoroTTSSession::run(const runtime::TaskRequest & request) if (!request.text_input.has_value()) { throw std::runtime_error("Kokoro TTS run requires text_input"); } - runtime::validate_spec_backed_request_options( - request.options, request.option_arrays, *contract_, kModelName); + validate_request_options(request.options, request.option_arrays, *contract_); const int64_t text_chunk_size = engine::text::parse_text_chunk_size_override(request.options).value_or(kDefaultTextChunkSize); @@ -313,7 +334,7 @@ runtime::TaskResult KokoroTTSSession::run(const runtime::TaskRequest & request) // merged into one result exactly as text chunks are. A caller whose document exceeds the // 510-symbol limit therefore still makes ONE call and gets ONE buffer back, instead of // having to stitch the audio itself. - const auto supplied_phonemes = find_option_array(request.option_arrays, "phonemes"); + const auto supplied_phonemes = find_option_array(request.option_arrays, kPhonemesOption); const auto chunk_requests = supplied_phonemes.empty() ? runtime::chunk_text_request(request, text_chunk_size) : std::vector(supplied_phonemes.size(), request); From 567f84e26f5cf819e743b6b120b2a9163f5eac47 Mon Sep 17 00:00:00 2001 From: Chris Thompson Date: Wed, 16 Sep 2026 18:37:24 -0600 Subject: [PATCH 3/3] kokoro_tts: an empty phoneme entry is a caller error, not a fallback MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review on #577, both points, plus what reviewing the fix turned up. A blank entry spoke the source text. `build_kokoro_synthesis_input` tested `!phoneme_override.empty()` to decide whether an override had been given, so an empty entry WAS the "no override" sentinel and that chunk fell through to the built-in G2P -- ["həlˈO", ""] rendered the phonemes and then read "placeholder" aloud. The override is now std::optional, so absent and empty are different things, and an empty entry is refused by index. The same hole existed one level up: an empty LIST left supplied_phonemes.empty() true, and the whole request was spoken from text. The C ABI accepts a zero-length array, so a caller whose G2P stage produced nothing got the source text read out with no error. Set-but-empty is now refused as well. The chunk loop no longer copies the request. It built one TaskRequest per chunk, each holding the entire phoneme list, so the caller's input was copied once per entry in it; with supplied phonemes every chunk runs the same request, so it is used directly. Two further copies in the same shape: the cache key concatenated the whole document per chunk, and the option-array validation copied every phoneme string to drop one key from the map. The key's text half is now built once, and validation copies key names only. A supplied-entry failure names its entry, matching the empty-entry message: a bad symbol in entry 200 of 201 used to report only the symbol. Sizing in prepare() covers every entry, so it is still reported before anything is rendered -- 0.00s for that case. Driven through the C ABI against the shipped package and against the same package with model_specs/kokoro_tts.json as an override; identical on both: ["həlˈO"] ok 1.40s ["həlˈO", ""] entry 1 is empty; ... [""] entry 0 is empty; ... count=0 'phonemes' was set but holds no entries; ... 201 entries, last one bad entry 200: vocab is missing phoneme symbol: R (0.00s) not_a_kokoro_option unknown Kokoro TTS request option One chunk 1.52s, three chunks 5.10s and the built-in G2P path are unchanged. kokoro_cpu_kernel_test passes. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_014X4jSwJu1ScZVgUspcA9HC --- include/engine/models/kokoro_tts/frontend.h | 11 +- model_specs/kokoro_tts.json | 2 +- src/models/kokoro_tts/frontend.cpp | 13 +- src/models/kokoro_tts/session.cpp | 158 +++++++++++++++----- 4 files changed, 139 insertions(+), 45 deletions(-) diff --git a/include/engine/models/kokoro_tts/frontend.h b/include/engine/models/kokoro_tts/frontend.h index f1ed03533..8e7e6f673 100644 --- a/include/engine/models/kokoro_tts/frontend.h +++ b/include/engine/models/kokoro_tts/frontend.h @@ -6,6 +6,7 @@ #include #include #include +#include #include namespace engine::models::kokoro_tts { @@ -31,21 +32,25 @@ KokoroFrontendSessionState resolve_kokoro_frontend_session_state( const std::optional & voice, const KokoroAssets & assets); -// `phoneme_override`, when non-empty, is synthesized as-is INSTEAD of running the built-in +// `phoneme_override`, when present, is synthesized as-is INSTEAD of running the built-in // G2P over `text`. It lets a caller with its own grapheme-to-phoneme stage — a lexicon the // engine does not carry, a language it does not cover, a pronunciation the application has // already shown its user — drive the model directly. `text` is still required and its // language must still agree with the voice; only the phonemization is replaced. +// +// ⚠ OPTIONAL, NOT "EMPTY MEANS NO". An empty string is a caller that asked for an override +// and gave nothing, and is rejected; it must not quietly fall back to speaking `text`, which +// is what an empty-string sentinel did here before. KokoroSynthesisInput build_kokoro_synthesis_input( const runtime::Transcript & text, const KokoroFrontendSessionState & state, const KokoroAssets & assets, - const std::string & phoneme_override = std::string()); + std::optional phoneme_override = std::nullopt); int64_t estimate_kokoro_request_tokens( const runtime::SessionPreparationRequest & request, const KokoroFrontendSessionState & state, const KokoroAssets & assets, - const std::string & phoneme_override = std::string()); + std::optional phoneme_override = std::nullopt); } // namespace engine::models::kokoro_tts diff --git a/model_specs/kokoro_tts.json b/model_specs/kokoro_tts.json index ee5d3cc5d..f5b49c1bb 100644 --- a/model_specs/kokoro_tts.json +++ b/model_specs/kokoro_tts.json @@ -32,7 +32,7 @@ { "name": "phonemes", "type": "string_list", - "description": "Kokoro-vocabulary phonemes to synthesize directly, bypassing the built-in eSpeak-ng G2P. One entry per chunk: entries are rendered in order and merged into one result, so the caller chooses the split points that only its own G2P knows. Each entry must be at most 510 symbols. Text is still required and its language must still match the voice.", + "description": "Kokoro-vocabulary phonemes to synthesize directly, bypassing the built-in eSpeak-ng G2P. One entry per chunk: entries are rendered in order and merged into one result, so the caller chooses the split points that only its own G2P knows. Each entry must be at most 510 symbols and must not be empty -- an entry is a chunk to speak, so drop it rather than leaving it blank. Text is still required and its language must still match the voice.", "required": false }, { diff --git a/src/models/kokoro_tts/frontend.cpp b/src/models/kokoro_tts/frontend.cpp index a4c26e991..4174c483d 100644 --- a/src/models/kokoro_tts/frontend.cpp +++ b/src/models/kokoro_tts/frontend.cpp @@ -252,13 +252,18 @@ KokoroSynthesisInput build_kokoro_synthesis_input( const runtime::Transcript & text, const KokoroFrontendSessionState & state, const KokoroAssets & assets, - const std::string & phoneme_override) { + std::optional phoneme_override) { if (state.voice_pack == nullptr) { throw std::runtime_error("Kokoro frontend session voice pack was not prepared"); } - const bool supplied = !phoneme_override.empty(); + const bool supplied = phoneme_override.has_value(); + if (supplied && phoneme_override->empty()) { + // Falling back to the built-in G2P here would speak `text` for a chunk the caller + // asked to be spoken from phonemes, which reads as the engine ignoring the option. + throw std::runtime_error("Kokoro supplied phonemes are empty; an override must carry symbols"); + } const std::string phonemes = - supplied ? phoneme_override : phonemize_text(text, state.language_code, assets); + supplied ? std::string(*phoneme_override) : phonemize_text(text, state.language_code, assets); // Strict for a caller's stream, lenient for our own -- see encode_input_ids_and_count. const EncodedInputIds encoded = encode_input_ids_and_count(phonemes, assets, supplied); if (encoded.phoneme_count > 510) { @@ -291,7 +296,7 @@ int64_t estimate_kokoro_request_tokens( const runtime::SessionPreparationRequest & request, const KokoroFrontendSessionState & state, const KokoroAssets & assets, - const std::string & phoneme_override) { + std::optional phoneme_override) { if (!request.text.has_value()) { return 0; } diff --git a/src/models/kokoro_tts/session.cpp b/src/models/kokoro_tts/session.cpp index 7e4e4f1d3..a21577a86 100644 --- a/src/models/kokoro_tts/session.cpp +++ b/src/models/kokoro_tts/session.cpp @@ -13,7 +13,11 @@ #include #include #include +#include #include +#include +#include +#include namespace engine::models::kokoro_tts { @@ -238,11 +242,76 @@ void KokoroTTSSession::prepare_decoder_graph_capacity(int64_t capacity) { namespace { -/// The "no override, use the built-in G2P" sentinel, so the chunk loop can bind a reference. -const std::string kNoSuppliedPhonemes; - constexpr const char * kPhonemesOption = "phonemes"; +/// The supplied phoneme chunks, or empty when the caller set no such option. +/// +/// ⚠ SET-BUT-EMPTY IS NOT THE SAME AS UNSET, at either level. An empty list, and an empty entry +/// within a list, both used to read as "no override" -- the first because the vector is empty, +/// the second because the frontend took an empty string as its sentinel -- and the chunk was +/// then spoken from `text`. A caller that asked for phonemes and supplied none got the source +/// text read aloud, which is the one thing this option exists to avoid. Both are refused here, +/// where the offending entry can still be named. The C ABI accepts a zero-length array +/// (audiocpp_request_set_option_array with count == 0), so the empty list is reachable. +const std::vector & require_phoneme_chunks( + const std::unordered_map> & option_arrays) { + static const std::vector none; + const auto it = option_arrays.find(kPhonemesOption); + if (it == option_arrays.end()) return none; + if (it->second.empty()) { + throw std::runtime_error( + "Kokoro 'phonemes' was set but holds no entries; pass the phonemes to synthesize, or " + "leave the option unset to use the built-in G2P"); + } + for (size_t index = 0; index < it->second.size(); ++index) { + if (it->second[index].empty()) { + throw std::runtime_error( + "Kokoro supplied phoneme entry " + std::to_string(index) + + " is empty; every entry is rendered as its own chunk, so each one must carry " + "symbols -- drop the entry instead of leaving it blank"); + } + } + return it->second; +} + +/// Builds one chunk's input, naming the entry a supplied-phoneme failure came from. +/// +/// The frontend's own rejections -- an out-of-vocab symbol, a chunk over 510 symbols -- say what +/// is wrong but not which entry it is in, and a list can be long. The empty-entry check above +/// names its entry, so these should too. +KokoroSynthesisInput build_supplied_or_text_input( + const runtime::TaskRequest & chunk_request, + const KokoroFrontendSessionState & state, + const KokoroAssets & assets, + std::optional chunk_phonemes, + size_t chunk_index, + bool supplied) { + if (!supplied) { + return build_kokoro_synthesis_input(*chunk_request.text_input, state, assets, chunk_phonemes); + } + try { + return build_kokoro_synthesis_input(*chunk_request.text_input, state, assets, chunk_phonemes); + } catch (const std::exception & error) { + throw std::runtime_error( + "Kokoro supplied phoneme entry " + std::to_string(chunk_index) + ": " + error.what()); + } +} + +/// The text-derived half of the synthesis cache key. +/// +/// Held apart from the phoneme half because in the supplied path every chunk shares one request: +/// this is the same string for all of them, and it carries the WHOLE document, so building it +/// per chunk would copy the caller's input once per entry. +std::string cache_key_prefix( + const KokoroFrontendSessionState & state, + const runtime::Transcript & text) { + return state.voice_id + ":" + + state.language_code + ":" + + std::to_string(state.speaking_rate) + ":" + + std::to_string(text.text.size()) + ":" + + text.text + ":"; +} + /// Request options, validated against the package's own contract. /// /// Older standalone GGUF packages embed a schema-v1 contract written before @@ -259,20 +328,15 @@ void validate_request_options( runtime::validate_spec_backed_request_options(options, option_arrays, contract, kModelName); return; } - auto validation_arrays = option_arrays; - validation_arrays.erase(kPhonemesOption); + // Keys only: the validator reads names, and copying the map wholesale would copy every + // supplied phoneme string to drop one entry from it. + std::unordered_map> validation_arrays; + for (const auto & [key, _] : option_arrays) { + if (key != kPhonemesOption) validation_arrays.emplace(key, std::vector{}); + } runtime::validate_spec_backed_request_options(options, validation_arrays, contract, kModelName); } -/// A list-valued request option, or empty when the caller did not set one. -const std::vector & find_option_array( - const std::unordered_map> & option_arrays, - const std::string & key) { - static const std::vector none; - const auto it = option_arrays.find(key); - return it == option_arrays.end() ? none : it->second; -} - } // namespace void KokoroTTSSession::prepare(const runtime::SessionPreparationRequest & request) { @@ -287,7 +351,7 @@ void KokoroTTSSession::prepare(const runtime::SessionPreparationRequest & reques } auto adapter = make_graph_capacity_adapter(); int64_t request_size = 0; - const auto prepare_phonemes = find_option_array(request.option_arrays, kPhonemesOption); + const auto & prepare_phonemes = require_phoneme_chunks(request.option_arrays); if (request.text.has_value()) { const int64_t text_chunk_size = engine::text::parse_text_chunk_size_override(request.options).value_or(kDefaultTextChunkSize); @@ -307,10 +371,18 @@ void KokoroTTSSession::prepare(const runtime::SessionPreparationRequest & reques request_size, estimate_kokoro_request_tokens(chunk_request, frontend_state, *assets_)); } else { - for (const auto & supplied : prepare_phonemes) { - request_size = std::max( - request_size, - estimate_kokoro_request_tokens(chunk_request, frontend_state, *assets_, supplied)); + // Every entry is sized here, before anything is synthesized, so a bad entry + // near the end of a long list is reported without rendering the ones before it. + for (size_t index = 0; index < prepare_phonemes.size(); ++index) { + try { + request_size = std::max( + request_size, + estimate_kokoro_request_tokens( + chunk_request, frontend_state, *assets_, prepare_phonemes[index])); + } catch (const std::exception & error) { + throw std::runtime_error( + "Kokoro supplied phoneme entry " + std::to_string(index) + ": " + error.what()); + } } } } @@ -334,12 +406,17 @@ runtime::TaskResult KokoroTTSSession::run(const runtime::TaskRequest & request) // merged into one result exactly as text chunks are. A caller whose document exceeds the // 510-symbol limit therefore still makes ONE call and gets ONE buffer back, instead of // having to stitch the audio itself. - const auto supplied_phonemes = find_option_array(request.option_arrays, kPhonemesOption); - const auto chunk_requests = supplied_phonemes.empty() + const auto & supplied_phonemes = require_phoneme_chunks(request.option_arrays); + // ⚠ NOT one TaskRequest per chunk. With supplied phonemes every chunk runs the SAME request + // -- only the phoneme entry differs -- and the request carries the whole phoneme list, so a + // copy per chunk would copy the caller's own input once for every entry in it. + const auto text_chunk_requests = supplied_phonemes.empty() ? runtime::chunk_text_request(request, text_chunk_size) - : std::vector(supplied_phonemes.size(), request); + : std::vector{}; + const size_t chunk_count = + supplied_phonemes.empty() ? text_chunk_requests.size() : supplied_phonemes.size(); engine::debug::trace_log_scalar("kokoro.text_chunk_size", text_chunk_size); - engine::debug::trace_log_scalar("kokoro.text_chunk_count", static_cast(chunk_requests.size())); + engine::debug::trace_log_scalar("kokoro.text_chunk_count", static_cast(chunk_count)); if (!supplied_phonemes.empty()) { engine::debug::trace_log_scalar( "kokoro.supplied_phoneme_chunks", static_cast(supplied_phonemes.size())); @@ -349,31 +426,38 @@ runtime::TaskResult KokoroTTSSession::run(const runtime::TaskRequest & request) double predictor_ms = 0.0; double decoder_ms = 0.0; runtime::AudioBuffer merged_audio; - for (size_t chunk_index = 0; chunk_index < chunk_requests.size(); ++chunk_index) { - const auto & chunk_request = chunk_requests[chunk_index]; - const std::string & chunk_phonemes = - supplied_phonemes.empty() ? kNoSuppliedPhonemes : supplied_phonemes[chunk_index]; - const auto frontend_state = - resolve_kokoro_frontend_session_state(chunk_request.text_input, chunk_request.voice, *assets_); + // Resolved once in the supplied path, where every chunk runs the same request: the state + // and the text half of the key cannot differ between chunks there. + const bool supplied = !supplied_phonemes.empty(); + std::optional shared_state; + std::string shared_key_prefix; + if (supplied) { + shared_state = resolve_kokoro_frontend_session_state(request.text_input, request.voice, *assets_); + shared_key_prefix = cache_key_prefix(*shared_state, *request.text_input); + } + for (size_t chunk_index = 0; chunk_index < chunk_count; ++chunk_index) { + const auto & chunk_request = supplied ? request : text_chunk_requests[chunk_index]; + const std::optional chunk_phonemes = + supplied ? std::optional{supplied_phonemes[chunk_index]} + : std::optional{}; + const auto frontend_state = supplied + ? *shared_state + : resolve_kokoro_frontend_session_state(chunk_request.text_input, chunk_request.voice, *assets_); const std::string cache_key = - frontend_state.voice_id + ":" + - frontend_state.language_code + ":" + - std::to_string(frontend_state.speaking_rate) + ":" + - std::to_string(chunk_request.text_input->text.size()) + ":" + - chunk_request.text_input->text + ":" + + (supplied ? shared_key_prefix : cache_key_prefix(frontend_state, *chunk_request.text_input)) + // Without this, two requests with the same text and different supplied phonemes // would hit the same cache entry and the second would be spoken as the first -- // and with a list, every chunk shares one text, so this is the ONLY thing telling // them apart. - chunk_phonemes; + std::string(chunk_phonemes.value_or(std::string_view{})); KokoroSynthesisInput input; frontend_ms += measure_ms([&]() { if (!cache_key.empty() && cached_input_ && cache_key == cached_request_key_) { input = *cached_input_; return; } - input = build_kokoro_synthesis_input( - *chunk_request.text_input, frontend_state, *assets_, chunk_phonemes); + input = build_supplied_or_text_input( + chunk_request, frontend_state, *assets_, chunk_phonemes, chunk_index, supplied); if (!cache_key.empty()) { cached_request_key_ = cache_key; cached_input_ = std::make_unique(input);