diff --git a/CMakeLists.txt b/CMakeLists.txt index e83f9df96..b9684bd5f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -586,16 +586,13 @@ audiocpp_add_model(demucs audiocpp_add_model(kokoro_tts SOURCES src/models/kokoro_tts/assets.cpp - src/models/kokoro_tts/package.cpp src/models/kokoro_tts/g2p_multilingual.cpp - src/models/kokoro_tts/loader.cpp src/models/kokoro_tts/frontend.cpp - src/models/kokoro_tts/g2p_en.cpp src/models/kokoro_tts/predictor.cpp src/models/kokoro_tts/decoder.cpp src/models/kokoro_tts/plbert.cpp src/models/kokoro_tts/session.cpp - INCLUDES engine/models/kokoro_tts/loader.h + INCLUDES engine/models/kokoro_tts/session.h LOADERS engine::models::kokoro_tts::make_kokoro_tts_loader ) if(MSVC) diff --git a/docs/models/kokoro_tts.md b/docs/models/kokoro_tts.md new file mode 100644 index 000000000..c6227c5ec --- /dev/null +++ b/docs/models/kokoro_tts.md @@ -0,0 +1,136 @@ +# Kokoro 82M + +Kokoro 82M is a compact multilingual text-to-speech model exposed as +`--family kokoro_tts`. audio.cpp packages the model as standalone GGUF files +with all 54 upstream voice packs. + +## Install + +```bash +python3 tools/model_manager_v2.py install kokoro_82m_q8_0 +``` + +The default package installs: + +```text +models/Kokoro-82M-GGUF/kokoro-82m-q8_0.gguf +``` + +BF16 is also available: + +```bash +python3 tools/model_manager_v2.py install kokoro_82m_bf16 +``` + +## Quick Start + +```bash +audiocpp_cli --task tts --family kokoro_tts \ + --model models/Kokoro-82M-GGUF/kokoro-82m-q8_0.gguf \ + --backend cpu \ + --language en-us \ + --voice-id af_heart \ + --text "Hello from Kokoro." \ + --out out.wav +``` + +Chinese example: + +```bash +audiocpp_cli --task tts --family kokoro_tts \ + --model models/Kokoro-82M-GGUF/kokoro-82m-q8_0.gguf \ + --backend cpu \ + --language zh \ + --voice-id zf_xiaobei \ + --text "你好，这是中文语音测试。" \ + --out out_zh.wav +``` + +## Model + +| Field | Value | +|---|---| +| Family | `kokoro_tts` | +| Task | `tts` | +| Modes | `offline` | +| Default package | `kokoro_82m_q8_0` | +| Other package | `kokoro_82m_bf16` | +| Model file | `models/Kokoro-82M-GGUF/kokoro-82m-q8_0.gguf` | + +## Voices and Languages + +Use `--voice-id ` to select one of the packaged voices. The voice prefix +selects the language family: + +| Prefix | Language | +|---|---| +| `af`, `am` | American English | +| `bf`, `bm` | British English | +| `ef`, `em` | Spanish | +| `ff` | French | +| `hf`, `hm` | Hindi | +| `if`, `im` | Italian | +| `jf`, `jm` | Japanese | +| `pf`, `pm` | Brazilian Portuguese | +| `zf`, `zm` | Mandarin Chinese | + +The request language must match the selected voice. For example, use +`--language zh` with `zf_*` or `zm_*` voices. + +## Runtime Resources + +The release GGUF includes model weights, config, vocabulary, all voice packs, +and the generated `g2p/ja.json` and `g2p/zh.json` tables. + +English, Spanish, French, Hindi, Italian, and Portuguese use the shared eSpeak +runtime. Install or package eSpeak data as described in +[`docs/espeak_phonemizer.md`](../espeak_phonemizer.md). + +Chinese works from the release GGUF because `g2p/zh.json` is bundled. Avoid +mixed Latin words inside Chinese text unless they are known to map to Kokoro's +vocabulary. + +Japanese also needs MeCab and UniDic. The small release GGUF does not include +UniDic because it is large. Export a local full multilingual GGUF with +`--embed-multilingual-resources` if you need Japanese to work from a bundled +package. + +## Options + +| Option | Values | Default | Meaning | +|---|---|---:|---| +| `--language ` / `--request-option language=` | language code | voice prefix | Text frontend language. | +| `--voice-id ` | packaged voice id | `af_heart` | Built-in voice pack. | +| `--seed ` / `--request-option seed=` | integer | random | Decoder noise seed. | +| `--text-chunk-size ` / `--request-option text_chunk_size=` | integer chars | `240` | Long-form chunk size. | +| `--session-option kokoro_tts.weight_type=` | `native`, `f32`, `f16`, `bf16`, `q8_0` | `native` | Matmul weight storage type. | +| `--session-option kokoro_tts.conv_weight_type=` | `native`, `f32`, `f16` | `native` | Convolution weight storage type. | + +For GGUF packages, leave weight options at `native` unless you are testing a +conversion or storage policy. + +## Conversion + +Convert from the official `hexgrad/Kokoro-82M` source checkout: + +```bash +python tools/prepare_kokoro_gguf.py \ + --source /path/to/Kokoro-82M \ + --output-dir /path/to/Kokoro-82M-GGUF \ + --type both \ + --overwrite +``` + +For a fully bundled local multilingual package with eSpeak data and UniDic: + +```bash +python tools/prepare_kokoro_gguf.py \ + --source /path/to/Kokoro-82M \ + --output-dir /path/to/Kokoro-82M-GGUF \ + --type both \ + --embed-multilingual-resources \ + --overwrite +``` + +The detailed validation notes live in +[`tests/kokoro_tts/MULTILINGUAL_GGUF.md`](../../tests/kokoro_tts/MULTILINGUAL_GGUF.md). diff --git a/docs/tts.md b/docs/tts.md index a9f147d41..4787500bb 100644 --- a/docs/tts.md +++ b/docs/tts.md @@ -28,6 +28,7 @@ | IndexTTS2 | `index_tts2` | `tts` | [IndexTTS](models/index_tts.md) | | IndexTTS2.5 | `index_tts2` (variant `2.5`) | `tts` | [IndexTTS](models/index_tts.md) | | Irodori-TTS | `irodori_tts` | `tts`, `vdes` | [Irodori-TTS](#irodori-tts) | +| Kokoro 82M | `kokoro_tts` | `tts` | [Kokoro 82M](models/kokoro_tts.md) | | GLM-TTS | `glm_tts` | `tts`, `clon` | [GLM-TTS](#glm-tts) | | Inflect Micro v2 | `inflect_v2` | `tts` | [Inflect v2](#inflect-v2) | | OuteTTS | `outetts` | `tts`, `clon` | [OuteTTS](#outetts) | diff --git a/include/engine/models/kokoro_tts/assets.h b/include/engine/models/kokoro_tts/assets.h index 8251820cd..3adfe83ef 100644 --- a/include/engine/models/kokoro_tts/assets.h +++ b/include/engine/models/kokoro_tts/assets.h @@ -1,6 +1,7 @@ #pragma once #include "engine/framework/assets/tensor_source.h" +#include "engine/framework/assets/resource_bundle.h" #include "engine/framework/core/backend_weight_store.h" #include "engine/framework/core/module.h" #include "engine/framework/io/json.h" @@ -243,9 +244,6 @@ struct KokoroWeights { std::shared_ptr store; }; -namespace g2p_en { -class EnglishG2P; -} } namespace engine::models::kokoro_tts { @@ -259,20 +257,17 @@ struct KokoroVoicePack { }; struct KokoroAssets { - std::shared_ptr package; std::shared_ptr multilingual_g2p; std::filesystem::path model_root; + engine::assets::ResourceBundle resources; engine::io::json::Value config; std::shared_ptr model_weights; int64_t context_length = 512; std::unordered_map vocab; std::unordered_map voices; - std::filesystem::path english_lexicon_dir; - std::shared_ptr english_g2p_us; - std::shared_ptr english_g2p_gb; }; -std::shared_ptr load_kokoro_assets(const std::filesystem::path & model_root); +std::shared_ptr load_kokoro_assets(const std::filesystem::path & model_path); std::shared_ptr load_kokoro_backend_weights( const KokoroAssets & assets, diff --git a/include/engine/models/kokoro_tts/frontend.h b/include/engine/models/kokoro_tts/frontend.h index d092f207f..7b66789bb 100644 --- a/include/engine/models/kokoro_tts/frontend.h +++ b/include/engine/models/kokoro_tts/frontend.h @@ -31,12 +31,6 @@ KokoroFrontendSessionState resolve_kokoro_frontend_session_state( const std::optional & voice, const KokoroAssets & assets); -void validate_kokoro_frontend_session_state( - const runtime::Transcript & text, - const std::optional & voice, - const KokoroFrontendSessionState & state, - const KokoroAssets & assets); - KokoroSynthesisInput build_kokoro_synthesis_input( const runtime::Transcript & text, const KokoroFrontendSessionState & state, diff --git a/include/engine/models/kokoro_tts/g2p_en.h b/include/engine/models/kokoro_tts/g2p_en.h deleted file mode 100644 index c24777759..000000000 --- a/include/engine/models/kokoro_tts/g2p_en.h +++ /dev/null @@ -1,159 +0,0 @@ -#pragma once - -#include -#include -#include -#include -#include -#include -#include -#include - -namespace kokoro_ggml::g2p_en { - -struct TokenContext { - std::optional future_vowel; - bool future_to = false; -}; - -struct LexiconResult { - std::string phonemes; - int rating = 0; -}; - -struct LexiconEntry { - bool has_default = false; - std::string default_phonemes; - std::unordered_map> by_tag; -}; - -struct TokenMeta { - bool is_head = true; - bool prespace = false; - std::optional stress; - std::optional alias; - std::optional currency; - std::string num_flags; - std::optional rating; -}; - -struct MToken { - std::string text; - std::string tag; - std::string whitespace; - std::optional phonemes; - TokenMeta meta; -}; - -using FeatureValue = std::variant; - -struct PreprocessResult { - std::string text; - std::unordered_map features; -}; - -class Lexicon { -public: - explicit Lexicon(bool british = false); - Lexicon(std::filesystem::path gold_tsv, std::filesystem::path silver_tsv, bool british = false); - - std::optional lookup( - const std::string & word, - const std::optional & tag, - const std::optional & stress, - const TokenContext & ctx) const; - - std::optional operator()(const MToken & token, const TokenContext & ctx) const; - static std::string apply_stress(const std::string & phonemes, const std::optional & stress); - static bool is_number_token(const std::string & word, bool is_head); - -private: - std::optional get_special_case( - const std::string & word, - const std::optional & tag, - const std::optional & stress, - const TokenContext & ctx) const; - - std::optional get_nnp(const std::string & word) const; - std::optional lookup_raw( - const std::string & word, - const std::optional & tag, - const std::optional & stress) const; - - std::optional stem_s( - const std::string & word, - const std::optional & tag, - const std::optional & stress, - const TokenContext & ctx) const; - std::optional stem_ed( - const std::string & word, - const std::optional & tag, - const std::optional & stress, - const TokenContext & ctx) const; - std::optional stem_ing( - const std::string & word, - const std::optional & tag, - const std::optional & stress, - const TokenContext & ctx) const; - - std::optional get_word( - const std::string & word, - const std::optional & tag, - const std::optional & stress, - const TokenContext & ctx) const; - - std::optional get_number( - const std::string & word, - const std::optional & currency, - bool is_head) const; - - bool is_known(const std::string & word) const; - - static std::optional parent_tag(const std::optional & tag); - static bool is_currency_shape(const std::string & word); - - std::optional s_suffix(const std::string & stem) const; - std::optional ed_suffix(const std::string & stem) const; - std::optional ing_suffix(const std::string & stem) const; - - void load_tsv( - const std::filesystem::path & path, - std::unordered_map & target); - - bool british_ = false; - std::unordered_map golds_; - std::unordered_map silvers_; -}; - -class EnglishG2P { -public: - explicit EnglishG2P(bool british = false); - EnglishG2P(std::filesystem::path lexicon_dir, bool british = false); - - static PreprocessResult preprocess(const std::string & text); - std::vector tokenize( - const std::string & text, - const std::unordered_map & features) const; - - std::pair> operator()(const std::string & text, bool enable_preprocess = true) const; - -private: - static std::vector fold_left(const std::vector & tokens); - static std::vector>> retokenize(const std::vector & tokens); - static void resolve_tokens(std::vector & tokens); - static TokenContext token_context(const TokenContext & ctx, const std::optional & phonemes, const MToken & token); - static std::string tokens_to_phonemes(const std::vector & tokens); - static MToken merge_token_pair( - const MToken & left, - const MToken & right, - const std::optional & unk); - static MToken merge_tokens( - const std::vector & tokens, - size_t begin, - size_t end, - const std::optional & unk); - - Lexicon lexicon_; -}; - -} // namespace kokoro_ggml::g2p_en diff --git a/include/engine/models/kokoro_tts/loader.h b/include/engine/models/kokoro_tts/loader.h deleted file mode 100644 index a4d39328f..000000000 --- a/include/engine/models/kokoro_tts/loader.h +++ /dev/null @@ -1,33 +0,0 @@ -#pragma once - -#include "engine/framework/runtime/model.h" -#include "engine/models/kokoro_tts/assets.h" - -#include -#include - -namespace engine::models::kokoro_tts { - -class KokoroTTSLoadedModel final : public runtime::ILoadedVoiceModel { -public: - KokoroTTSLoadedModel( - runtime::ModelMetadata metadata, - runtime::CapabilitySet capabilities, - std::shared_ptr assets); - - const runtime::ModelMetadata & metadata() const noexcept override; - const runtime::CapabilitySet & capabilities() const noexcept override; - std::unique_ptr create_task_session( - const runtime::TaskSpec & task, - const runtime::SessionOptions & options) const override; - -private: - runtime::ModelMetadata metadata_; - runtime::CapabilitySet capabilities_; - std::shared_ptr assets_; -}; - -std::unique_ptr load_kokoro_tts_model(const std::filesystem::path & model_path); -std::shared_ptr make_kokoro_tts_loader(); - -} // namespace engine::models::kokoro_tts diff --git a/include/engine/models/kokoro_tts/package.h b/include/engine/models/kokoro_tts/package.h deleted file mode 100644 index 22c6d4a0d..000000000 --- a/include/engine/models/kokoro_tts/package.h +++ /dev/null @@ -1,17 +0,0 @@ -#pragma once - -#include "engine/framework/assets/tensor_source.h" -#include -#include - -namespace engine::models::kokoro_tts { -// Owns materialized resources for the lifetime of a standalone model. -struct KokoroPackage { - std::filesystem::path root; - std::shared_ptr weights; - bool temporary = false; - ~KokoroPackage(); -}; -std::shared_ptr open_kokoro_package(const std::filesystem::path & path); -bool is_kokoro_gguf(const std::filesystem::path & path) noexcept; -} diff --git a/include/engine/models/kokoro_tts/session.h b/include/engine/models/kokoro_tts/session.h index 1ae330f4a..ef8d48f6b 100644 --- a/include/engine/models/kokoro_tts/session.h +++ b/include/engine/models/kokoro_tts/session.h @@ -1,5 +1,6 @@ #pragma once +#include "engine/framework/model_spec/metadata.h" #include "engine/framework/runtime/session_base.h" #include "engine/models/kokoro_tts/assets.h" @@ -8,13 +9,14 @@ namespace kokoro_ggml { class KokoroDecoderRuntime; +class KokoroPredictorRuntime; } namespace engine::models::kokoro_tts { -struct KokoroSynthesisInput; -struct KokoroFrontendSessionState; +std::shared_ptr make_kokoro_tts_loader(); +struct KokoroSynthesisInput; class KokoroTTSSession final : public runtime::RuntimeSessionBase , public runtime::IOfflineVoiceTaskSession { @@ -22,7 +24,8 @@ class KokoroTTSSession final KokoroTTSSession( runtime::TaskSpec task, runtime::SessionOptions options, - std::shared_ptr assets); + std::shared_ptr assets, + std::shared_ptr contract); ~KokoroTTSSession() override; std::string family() const override; @@ -38,10 +41,7 @@ class KokoroTTSSession final int64_t conditioning_frame_capacity = 0; }; - struct PreparedRuntime; - runtime::MappedGraphCapacityAdapter make_graph_capacity_adapter(); - int64_t base_graph_capacity_tokens() const; std::vector prepared_graph_capacities() const; DecoderCapacityContract make_decoder_capacity_contract(int64_t decoder_frame_capacity) const; void prepare_graph_capacity(int64_t capacity); @@ -49,6 +49,7 @@ class KokoroTTSSession final runtime::TaskSpec task_; std::shared_ptr assets_; + std::shared_ptr contract_; std::shared_ptr weights_; runtime::GraphCapacityController graph_capacity_controller_; int64_t fixed_token_capacity_ = 0; @@ -62,12 +63,11 @@ class KokoroTTSSession final size_t predictor_tail_graph_bytes_ = 640ull * 1024ull * 1024ull; std::string cached_request_key_; std::unique_ptr cached_input_; - std::unique_ptr frontend_session_state_; int64_t prepared_decoder_capacity_ = 0; std::unique_ptr prepared_decoder_; DecoderCapacityContract prepared_decoder_context_ = {}; int64_t prepared_session_capacity_ = 0; - std::unique_ptr prepared_session_; + std::unique_ptr prepared_predictor_; }; } // namespace engine::models::kokoro_tts diff --git a/model_specs/kokoro_tts.json b/model_specs/kokoro_tts.json index 305f4885c..a4b4faff3 100644 --- a/model_specs/kokoro_tts.json +++ b/model_specs/kokoro_tts.json @@ -16,18 +16,31 @@ }, "options": { "request": [ + { + "name": "language", + "type": "string", + "description": "Text language code; defaults to the selected voice prefix.", + "required": false + }, { "name": "seed", "type": "int", "description": "Decoder noise seed; omitted requests choose a random seed.", "required": false, "min": 0 + }, + { + "name": "text_chunk_size", + "type": "int", + "description": "Maximum codepoints per text chunk before synthesis. Default 240.", + "required": false, + "min": 32, + "default": 240 } ], - "session": [], - "load": [ + "session": [ { - "name": "matmul_weight_type", + "name": "weight_type", "type": "enum", "preset": "weight_type_full", "required": false, @@ -39,15 +52,113 @@ "type": "enum", "preset": "weight_type_conv", "required": false, - "default": "f32", + "default": "native", "description": "Storage type for convolution weights." + }, + { + "name": "weight_context_mb", + "type": "int", + "description": "Weight loading context size in MiB; default 512.", + "required": false, + "min": 1, + "default": 512 + }, + { + "name": "predictor_duration_graph_mb", + "type": "int", + "description": "Predictor duration graph arena size in MiB; default 384.", + "required": false, + "min": 1, + "default": 384 + }, + { + "name": "predictor_text_graph_mb", + "type": "int", + "description": "Predictor text graph arena size in MiB; default 256.", + "required": false, + "min": 1, + "default": 256 + }, + { + "name": "predictor_tail_graph_mb", + "type": "int", + "description": "Predictor tail graph arena size in MiB; default 640.", + "required": false, + "min": 1, + "default": 640 + }, + { + "name": "graph_capacity_mode", + "type": "enum", + "description": "Offline graph capacity policy; default fixed.", + "values": ["fixed", "tiered", "grow", "double"], + "required": false, + "default": "fixed" + }, + { + "name": "max_input_tokens", + "type": "int", + "description": "Predictor graph input token capacity; default min(512, model context length).", + "required": false, + "min": 1 + }, + { + "name": "pre_tail_tokens", + "type": "int", + "description": "Optional prebuilt predictor tail token capacity; default 0.", + "required": false, + "min": 0, + "default": 0 } - ] + ], + "load": [] + }, + "package_defaults": { + "download": { + "kind": "huggingface_snapshot", + "repo": "audio-cpp/audio.cpp-gguf", + "revision": "main", + "gated": false + } }, - "packages": [], + "packages": [ + { + "id": "kokoro_82m_q8_0", + "display_name": "Kokoro 82M Q8_0 GGUF", + "default": true, + "format": "gguf", + "precision": "q8_0", + "target_directory": "Kokoro-82M-GGUF", + "files": [ + "Kokoro-82M-GGUF/kokoro-82m-q8_0.gguf" + ], + "strip_prefix": "Kokoro-82M-GGUF" + }, + { + "id": "kokoro_82m_bf16", + "display_name": "Kokoro 82M BF16 GGUF", + "format": "gguf", + "precision": "bf16", + "target_directory": "Kokoro-82M-GGUF", + "files": [ + "Kokoro-82M-GGUF/kokoro-82m-bf16.gguf" + ], + "strip_prefix": "Kokoro-82M-GGUF" + } + ], "dependencies": [], "ui": { - "tags": ["TTS", "Preset voices"], + "recommended_package": "kokoro_82m_q8_0", + "tags": ["TTS", "GGUF"], + "builtin_voices": [ + "af_alloy", "af_aoede", "af_bella", "af_heart", "af_jessica", "af_kore", "af_nicole", "af_nova", "af_river", "af_sarah", "af_sky", + "am_adam", "am_echo", "am_eric", "am_fenrir", "am_liam", "am_michael", "am_onyx", "am_puck", "am_santa", + "bf_alice", "bf_emma", "bf_isabella", "bf_lily", "bm_daniel", "bm_fable", "bm_george", "bm_lewis", + "ef_dora", "em_alex", "em_santa", "ff_siwis", "hf_alpha", "hf_beta", "hm_omega", "hm_psi", "if_sara", "im_nicola", + "jf_alpha", "jf_gongitsune", "jf_nezumi", "jf_tebukuro", "jm_kumo", "pf_dora", "pm_alex", "pm_santa", + "zf_xiaobei", "zf_xiaoni", "zf_xiaoxiao", "zf_xiaoyi", "zm_yunjian", "zm_yunxi", "zm_yunxia", "zm_yunyang" + ], + "default_voice": "af_heart", "docs": ["tests/kokoro_tts/MULTILINGUAL_GGUF.md"] }, "sources": [ @@ -68,20 +179,6 @@ "prefix": "kokoro" } } - }, - { - "format": "safetensors", - "roots": { - "model": "." - }, - "files": { - "config": "model:config.json", - "voices": "model:voices.json", - "vocabulary": "model:vocab.tsv" - }, - "tensors": { - "weights": "model:kokoro-v1_0.safetensors" - } } ] } diff --git a/src/models/kokoro_tts/assets.cpp b/src/models/kokoro_tts/assets.cpp index 19fa28282..fe8a88227 100644 --- a/src/models/kokoro_tts/assets.cpp +++ b/src/models/kokoro_tts/assets.cpp @@ -2,12 +2,10 @@ #include "engine/framework/core/backend_weight_store.h" #include "engine/framework/io/binary.h" #include "engine/framework/io/json.h" +#include "engine/framework/model_spec/package.h" #include "engine/models/kokoro_tts/assets.h" -#include "engine/models/kokoro_tts/package.h" #include "engine/models/kokoro_tts/g2p_multilingual.h" -#include "engine/models/kokoro_tts/g2p_en.h" - #include #include #include @@ -716,34 +714,6 @@ namespace engine::models::kokoro_tts { namespace { -struct KokoroAssetResources { - std::shared_ptr package; - assets::ResourceBundle bundle; - io::json::Value config; - io::json::Value voices; - std::shared_ptr weights; -}; - -KokoroAssetResources load_asset_resources(const std::filesystem::path & model_root) { - KokoroAssetResources resources; - resources.package = open_kokoro_package(model_root); - resources.bundle = assets::ResourceBundle(resources.package->root); - resources.bundle.add_model_files({ - {"config", "config.json"}, - {"voices", "voices.json"}, - }); - resources.config = resources.bundle.parse_json("config"); - if (!resources.config.is_object()) { - throw std::runtime_error("Kokoro config root must be an object"); - } - resources.voices = resources.bundle.parse_json("voices"); - if (!resources.voices.is_object()) { - throw std::runtime_error("Kokoro voices.json root must be an object"); - } - resources.weights = resources.package->weights; - return resources; -} - std::string language_code_from_voice_id(const std::string & voice_id) { if (voice_id.size() < 2 || (voice_id[1] != 'f' && voice_id[1] != 'm')) { throw std::runtime_error("invalid Kokoro voice id: " + voice_id); @@ -767,20 +737,26 @@ std::vector read_f32_file_exact(const std::filesystem::path & path, size_ } // namespace -std::shared_ptr load_kokoro_assets(const std::filesystem::path & model_root) { - auto resources = load_asset_resources(model_root); - const auto & root = resources.bundle.model_root(); +std::shared_ptr load_kokoro_assets(const std::filesystem::path & model_path) { + auto resources = engine::model_spec::load_resource_bundle_for_family(model_path, "kokoro_tts"); + auto config = resources.parse_json("config"); + if (!config.is_object()) { + throw std::runtime_error("Kokoro config root must be an object"); + } + auto voices = resources.parse_json("voices"); + if (!voices.is_object()) { + throw std::runtime_error("Kokoro voices.json root must be an object"); + } + auto weights = resources.open_tensor_source("weights"); + const auto & root = resources.model_root(); auto assets = std::make_shared(); - assets->package = resources.package; + assets->resources = resources; assets->multilingual_g2p = std::make_shared(root); assets->model_root = root; - assets->config = std::move(resources.config); - assets->model_weights = std::move(resources.weights); + assets->config = std::move(config); + assets->model_weights = std::move(weights); assets->context_length = kokoro_ggml::parse_kokoro_config_metadata(assets->config).plbert_max_position_embeddings; - assets->english_lexicon_dir = root / "misaki_en"; - assets->english_g2p_us = std::make_shared(assets->english_lexicon_dir, false); - assets->english_g2p_gb = std::make_shared(assets->english_lexicon_dir, true); const auto * vocab = assets->config.find("vocab"); if (vocab == nullptr || !vocab->is_object()) { @@ -790,7 +766,7 @@ std::shared_ptr load_kokoro_assets(const std::filesystem::pa assets->vocab[symbol] = static_cast(value.as_i64()); } - for (const auto & [voice_id, value] : resources.voices.as_object()) { + for (const auto & [voice_id, value] : voices.as_object()) { if (!value.is_object()) { throw std::runtime_error("Kokoro voice entry must be an object: " + voice_id); } diff --git a/src/models/kokoro_tts/frontend.cpp b/src/models/kokoro_tts/frontend.cpp index b59fb398c..98f183e62 100644 --- a/src/models/kokoro_tts/frontend.cpp +++ b/src/models/kokoro_tts/frontend.cpp @@ -1,6 +1,5 @@ #include "engine/models/kokoro_tts/frontend.h" -#include "engine/models/kokoro_tts/g2p_en.h" #include "engine/models/kokoro_tts/g2p_multilingual.h" #include @@ -107,13 +106,6 @@ std::string phonemize_text( if (text.text.empty()) { throw std::runtime_error("Kokoro TTS requires non-empty text"); } - if (language_code == "a" || language_code == "b") { - const auto & g2p = language_code == "b" ? assets.english_g2p_gb : assets.english_g2p_us; - if (!g2p) { - throw std::runtime_error("Kokoro English G2P assets were not prepared"); - } - return (*g2p)(text.text).first; - } if (!assets.multilingual_g2p) throw std::runtime_error("Kokoro multilingual resources were not prepared"); return assets.multilingual_g2p->phonemize(text.text, language_code); } @@ -220,33 +212,6 @@ KokoroFrontendSessionState resolve_kokoro_frontend_session_state( return state; } -void validate_kokoro_frontend_session_state( - const runtime::Transcript & text, - const std::optional & voice, - const KokoroFrontendSessionState & state, - const KokoroAssets & assets) { - const std::string resolved_voice_id = resolve_voice_id(voice, assets); - if (resolved_voice_id != state.voice_id) { - throw std::runtime_error( - "Kokoro session voice_id changed after launch: " + - state.voice_id + " -> " + resolved_voice_id); - } - const std::string resolved_language_code = resolve_language_code(text, voice, state.voice_id); - if (resolved_language_code != state.language_code) { - throw std::runtime_error( - "Kokoro session language_code changed after launch: " + - state.language_code + " -> " + resolved_language_code); - } - const auto voice_it = assets.voices.find(state.voice_id); - if (voice_it == assets.voices.end() || &voice_it->second != state.voice_pack) { - throw std::runtime_error("Kokoro session voice pack changed after launch"); - } - const float resolved_speaking_rate = resolve_speaking_rate(voice); - if (resolved_speaking_rate != state.speaking_rate) { - throw std::runtime_error("Kokoro session speaking_rate changed after launch"); - } -} - KokoroSynthesisInput build_kokoro_synthesis_input( const runtime::Transcript & text, const KokoroFrontendSessionState & state, diff --git a/src/models/kokoro_tts/g2p_en.cpp b/src/models/kokoro_tts/g2p_en.cpp deleted file mode 100644 index 2ba430bb6..000000000 --- a/src/models/kokoro_tts/g2p_en.cpp +++ /dev/null @@ -1,1866 +0,0 @@ -#include "engine/models/kokoro_tts/g2p_en.h" - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace kokoro_ggml::g2p_en { -namespace { - -constexpr char kPrimaryStress[] = "ˈ"; -constexpr char kSecondaryStress[] = "ˌ"; - -const std::string kDiphthongs = "AIOQWYʤʧ"; -const std::string kVowels = "AIOQWYaiuæɑɒɔəɛɜɪʊʌᵻ"; -const std::string kConsonants = "bdfhjklmnpstvwzðŋɡɹɾʃʒʤʧθ"; -const std::string kUsTaus = "AIOWYiuæɑəɛɪɹʊʌ"; - -const std::unordered_set kPunctTags = {".", ",", "-LRB-", "-RRB-", "``", "\"\"", "''", ":", "$", "#", "NFP"}; -const std::unordered_map kPunctTagPhonemes = { - {"-LRB-", "("}, {"-RRB-", ")"}, {"``", "\xE2\x80\x9C"}, {"\"\"", "\xE2\x80\x9D"}, {"''", "\xE2\x80\x9D"}, -}; -const std::unordered_map kAddSymbols = {{'.', "dot"}, {'/', "slash"}}; -const std::unordered_map kSymbols = {{'%', "percent"}, {'&', "and"}, {'+', "plus"}, {'@', "at"}}; -const std::unordered_map> kCurrencies = { - {"$", {"dollar", "cent"}}, - {"£", {"pound", "pence"}}, - {"€", {"euro", "cent"}}, -}; -const std::unordered_set kOrdinals = {"st", "nd", "rd", "th"}; -const std::unordered_set kSubtokenJunk = {'\'', ',', '-', '.', '_', '/', static_cast(0xe2)}; -const std::string kPuncts = ";:,.!?—…\"“”"; -const std::string kNonQuotePuncts = ";:,.!?—…"; - -const std::unordered_set kDeterminers = {"a", "an", "the", "this", "that", "these", "those"}; -const std::unordered_set kPronouns = {"i", "you", "he", "she", "it", "we", "they", "me", "him", "her", "us", "them"}; -const std::unordered_set kPrepositions = { - "to", "in", "on", "at", "by", "for", "from", "with", "without", "within", "into", "onto", "over", - "under", "before", "after", "through", "across", "between", "among", "versus", "vs", "of" -}; -const std::unordered_set kConjunctions = {"and", "or", "but", "nor", "so", "yet"}; -const std::unordered_set kAdverbs = {"not", "never", "always", "often", "rarely", "here", "there", "why", "how", "when"}; - -struct NumberWords { - std::string cardinal; - std::string ordinal; -}; - -template -bool contains_key(const Container & container, const Key & key) { - return container.find(key) != container.end(); -} - -bool contains_codepoint(const std::string & text, const std::string & symbol) { - return text.find(symbol) != std::string::npos; -} - -bool starts_with(const std::string & text, const std::string & prefix) { - return text.size() >= prefix.size() && text.compare(0, prefix.size(), prefix) == 0; -} - -bool ends_with(const std::string & text, const std::string & suffix) { - return text.size() >= suffix.size() && text.compare(text.size() - suffix.size(), suffix.size(), suffix) == 0; -} - -bool ends_with_any(const std::string & text, const std::initializer_list & suffixes) { - for (const char * suffix : suffixes) { - if (ends_with(text, suffix)) { - return true; - } - } - return false; -} - -bool is_ascii_alpha(char ch) { - return std::isalpha(static_cast(ch)) != 0; -} - -bool is_all_ascii_alpha(const std::string & text) { - return !text.empty() && std::all_of(text.begin(), text.end(), is_ascii_alpha); -} - -bool is_all_upper_ascii(const std::string & text) { - return !text.empty() && std::all_of(text.begin(), text.end(), [](char ch) { - return !std::isalpha(static_cast(ch)) || std::isupper(static_cast(ch)) != 0; - }); -} - -bool is_capitalized_ascii(const std::string & text) { - return !text.empty() && - std::isupper(static_cast(text.front())) != 0 && - std::all_of(text.begin() + 1, text.end(), [](char ch) { - return !std::isalpha(static_cast(ch)) || std::islower(static_cast(ch)) != 0; - }); -} - -std::string lower_ascii(std::string text) { - std::transform(text.begin(), text.end(), text.begin(), [](unsigned char ch) { - return static_cast(std::tolower(ch)); - }); - return text; -} - -std::string upper_ascii(std::string text) { - std::transform(text.begin(), text.end(), text.begin(), [](unsigned char ch) { - return static_cast(std::toupper(ch)); - }); - return text; -} - -std::string capitalize_ascii(std::string text) { - if (!text.empty()) { - text[0] = static_cast(std::toupper(static_cast(text[0]))); - for (size_t i = 1; i < text.size(); ++i) { - text[i] = static_cast(std::tolower(static_cast(text[i]))); - } - } - return text; -} - -std::vector split_tab(const std::string & line) { - std::vector fields; - size_t begin = 0; - while (true) { - const size_t tab = line.find('\t', begin); - if (tab == std::string::npos) { - fields.push_back(line.substr(begin)); - return fields; - } - fields.push_back(line.substr(begin, tab - begin)); - begin = tab + 1; - } -} - -std::optional normalize_stress(const std::optional & stress, const std::string & phonemes) { - if (!stress.has_value()) { - return stress; - } - if (*stress == 0.0f && contains_codepoint(phonemes, kPrimaryStress)) { - return -1.0f; - } - return stress; -} - -int stress_weight(const std::string & phonemes) { - int sum = 0; - for (char ch : phonemes) { - sum += kDiphthongs.find(ch) != std::string::npos ? 2 : 1; - } - return sum; -} - -bool is_digit_text(const std::string & text) { - return !text.empty() && std::all_of(text.begin(), text.end(), [](char ch) { - return std::isdigit(static_cast(ch)) != 0; - }); -} - -bool is_number_token_impl(const std::string & word, bool is_head) { - if (std::none_of(word.begin(), word.end(), [](char ch) { return std::isdigit(static_cast(ch)) != 0; })) { - return false; - } - std::string trimmed = word; - for (const char * suffix : {"ing", "'d", "ed", "'s", "st", "nd", "rd", "th", "s"}) { - if (ends_with(trimmed, suffix)) { - trimmed = trimmed.substr(0, trimmed.size() - std::char_traits::length(suffix)); - break; - } - } - for (size_t i = 0; i < trimmed.size(); ++i) { - const char ch = trimmed[i]; - if (std::isdigit(static_cast(ch)) != 0 || ch == ',' || ch == '.') { - continue; - } - if (is_head && i == 0 && ch == '-') { - continue; - } - return false; - } - return !trimmed.empty(); -} - -bool contains_whitespace(const std::string & text) { - return std::any_of(text.begin(), text.end(), [](unsigned char ch) { - return std::isspace(ch) != 0; - }); -} - -int english_g2p_char_class(char ch) { - if (std::isalpha(static_cast(ch)) != 0) { - return 0; - } - if (is_digit_text(std::string(1, ch))) { - return 1; - } - return 2; -} - -size_t count_words(const std::string & text) { - size_t count = 0; - bool in_word = false; - for (unsigned char ch : text) { - if (std::isspace(ch) != 0) { - in_word = false; - continue; - } - if (!in_word) { - ++count; - in_word = true; - } - } - return count; -} - -bool all_subtoken_junk(const std::string & text) { - for (char ch : text) { - if (ch == '\'' || ch == ',' || ch == '-' || ch == '.' || ch == '_' || ch == '/') { - continue; - } - return false; - } - return !text.empty(); -} - -size_t utf8_codepoint_width(const std::string & text, size_t offset) { - const unsigned char lead = static_cast(text[offset]); - size_t width = 0; - if ((lead & 0x80u) == 0) { - width = 1; - } else if ((lead & 0xE0u) == 0xC0u) { - width = 2; - } else if ((lead & 0xF0u) == 0xE0u) { - width = 3; - } else if ((lead & 0xF8u) == 0xF0u) { - width = 4; - } else { - throw std::runtime_error("invalid UTF-8 lead byte in English G2P phoneme string"); - } - if (offset + width > text.size()) { - throw std::runtime_error("truncated UTF-8 codepoint in English G2P phoneme string"); - } - for (size_t j = 1; j < width; ++j) { - const unsigned char byte = static_cast(text[offset + j]); - if ((byte & 0xC0u) != 0x80u) { - throw std::runtime_error("invalid UTF-8 continuation byte in English G2P phoneme string"); - } - } - return width; -} - -bool utf8_inventory_contains( - const std::string & inventory, - const char * symbol, - size_t symbol_width) { - for (size_t i = 0; i < inventory.size();) { - const size_t width = utf8_codepoint_width(inventory, i); - if (width == symbol_width && std::memcmp(inventory.data() + i, symbol, symbol_width) == 0) { - return true; - } - i += width; - } - return false; -} - -std::string join_phoneme_pieces(const std::vector & pieces) { - std::string out; - for (size_t i = 0; i < pieces.size(); ++i) { - if (i > 0) { - out.push_back(' '); - } - out += pieces[i]; - } - return out; -} - -std::optional compound_fallback_impl( - const Lexicon & lexicon, - const std::string & word, - std::unordered_map> & memo) { - if (const auto found = memo.find(word); found != memo.end()) { - return found->second; - } - if (word.size() < 4 || !is_all_ascii_alpha(word)) { - memo[word] = std::nullopt; - return std::nullopt; - } - - const TokenContext ctx{}; - if (auto direct = lexicon.lookup(word, std::nullopt, std::nullopt, ctx)) { - memo[word] = direct; - return direct; - } - - for (size_t split = 2; split + 2 <= word.size(); ++split) { - const std::string left = word.substr(0, split); - const std::string right = word.substr(split); - auto left_result = lexicon.lookup(left, std::nullopt, std::nullopt, ctx); - if (!left_result.has_value()) { - continue; - } - auto right_result = compound_fallback_impl(lexicon, right, memo); - if (!right_result.has_value()) { - right_result = lexicon.lookup(right, std::nullopt, std::nullopt, ctx); - } - if (!right_result.has_value()) { - continue; - } - const int rating = std::min(left_result->rating, right_result->rating); - memo[word] = LexiconResult{left_result->phonemes + " " + right_result->phonemes, rating}; - return memo[word]; - } - - memo[word] = std::nullopt; - return std::nullopt; -} - -std::optional compound_fallback( - const Lexicon & lexicon, - const std::string & text, - const std::optional & stress) { - const std::string word = lower_ascii(text); - std::unordered_map> memo; - auto result = compound_fallback_impl(lexicon, word, memo); - if (!result.has_value()) { - return std::nullopt; - } - result->phonemes = Lexicon::apply_stress(result->phonemes, stress); - return result; -} - -std::optional spell_out_fallback( - const Lexicon & lexicon, - const std::string & text, - const std::optional & stress) { - std::vector pieces; - for (char ch : text) { - if (std::isspace(static_cast(ch)) != 0) { - continue; - } - MToken token; - token.text = std::string(1, is_ascii_alpha(ch) ? static_cast(std::toupper(static_cast(ch))) : ch); - token.tag = std::isdigit(static_cast(ch)) != 0 ? "CD" : "NNP"; - token.meta.is_head = true; - if (auto result = lexicon(token, TokenContext{})) { - pieces.push_back(result->phonemes); - } - } - if (pieces.empty()) { - return std::nullopt; - } - return LexiconResult{Lexicon::apply_stress(join_phoneme_pieces(pieces), stress), 1}; -} - -LexiconResult resolve_with_safe_fallback( - const Lexicon & lexicon, - const MToken & token) { - if (auto compound = compound_fallback(lexicon, token.text, token.meta.stress)) { - return *compound; - } - if (auto spelled = spell_out_fallback(lexicon, token.text, token.meta.stress)) { - return *spelled; - } - return LexiconResult{Lexicon::apply_stress("ə", token.meta.stress), 1}; -} - -std::string trim_copy(const std::string & text) { - size_t begin = 0; - while (begin < text.size() && std::isspace(static_cast(text[begin])) != 0) { - ++begin; - } - size_t end = text.size(); - while (end > begin && std::isspace(static_cast(text[end - 1])) != 0) { - --end; - } - return text.substr(begin, end - begin); -} - -void replace_all(std::string & text, const std::string & from, const std::string & to) { - size_t pos = 0; - while ((pos = text.find(from, pos)) != std::string::npos) { - text.replace(pos, from.size(), to); - pos += to.size(); - } -} - -std::vector split_words(const std::string & text) { - std::vector words; - std::string current; - for (char ch : text) { - if (std::isspace(static_cast(ch)) != 0) { - if (!current.empty()) { - words.push_back(current); - current.clear(); - } - continue; - } - current.push_back(ch); - } - if (!current.empty()) { - words.push_back(current); - } - return words; -} - -std::vector split_non_alpha(const std::string & text) { - std::vector out; - std::string current; - for (char ch : text) { - if (std::isalpha(static_cast(ch)) != 0) { - current.push_back(ch); - continue; - } - if (!current.empty()) { - out.push_back(current); - current.clear(); - } - } - if (!current.empty()) { - out.push_back(current); - } - return out; -} - -std::filesystem::path resolve_default_lexicon_dir() { - const std::array candidates = { - std::filesystem::path("models/kokoro-82m-v1_0-ggml/misaki_en"), - std::filesystem::path("models/misaki_en"), - std::filesystem::current_path() / "models/kokoro-82m-v1_0-ggml/misaki_en", - std::filesystem::current_path() / "models/misaki_en", - }; - for (const auto & candidate : candidates) { - if (std::filesystem::exists(candidate / "us_gold.tsv") || std::filesystem::exists(candidate / "gb_gold.tsv")) { - return candidate; - } - } - throw std::runtime_error("failed to locate Kokoro misaki_en lexicon assets"); -} - -std::vector subtokenize(const std::string & word) { - std::vector tokens; - size_t i = 0; - while (i < word.size()) { - const unsigned char ch = static_cast(word[i]); - if (word[i] == '\'') { - size_t j = i + 1; - while (j < word.size() && word[j] == '\'') { - ++j; - } - tokens.push_back(word.substr(i, j - i)); - i = j; - continue; - } - if (std::isdigit(ch) != 0 || ((word[i] == '-' || word[i] == '+') && i + 1 < word.size() && std::isdigit(static_cast(word[i + 1])) != 0)) { - size_t j = i + 1; - while (j < word.size()) { - const unsigned char cj = static_cast(word[j]); - if (std::isdigit(cj) != 0) { - ++j; - continue; - } - if ((word[j] == ',' || word[j] == '.') && - j + 1 < word.size() && - std::isdigit(static_cast(word[j + 1])) != 0) { - ++j; - continue; - } - break; - } - tokens.push_back(word.substr(i, j - i)); - i = j; - continue; - } - if (std::isalpha(ch) != 0) { - size_t j = i + 1; - while (j < word.size()) { - const unsigned char pj = static_cast(word[j]); - if (std::isalpha(pj) != 0 || word[j] == '\'') { - if (j + 1 < word.size() && std::islower(pj) != 0 && std::isupper(static_cast(word[j + 1])) != 0) { - ++j; - break; - } - ++j; - continue; - } - break; - } - tokens.push_back(word.substr(i, j - i)); - i = j; - continue; - } - if (word[i] == '-' || word[i] == '_') { - size_t j = i + 1; - while (j < word.size() && (word[j] == '-' || word[j] == '_')) { - ++j; - } - tokens.push_back(word.substr(i, j - i)); - i = j; - continue; - } - tokens.push_back(word.substr(i, 1)); - ++i; - } - return tokens; -} - -std::string guess_tag(const std::string & token) { - if (token.empty()) { - return "NN"; - } - if (token == "$" || token == "£" || token == "€") { - return "$"; - } - if (token == "(") return "-LRB-"; - if (token == ")") return "-RRB-"; - if (token == "\"" || token == "“") return "``"; - if (token == "”") return "''"; - if (token == ":" || token == ";" || token == "-" || token == "–" || token == "—") return ":"; - if (token == "," || token == "." || token == "!" || token == "?" || token == "…") return std::string(1, token[0]); - - const std::string lowered = lower_ascii(token); - if (contains_key(kDeterminers, lowered)) return "DT"; - if (contains_key(kPronouns, lowered)) return "PRP"; - if (contains_key(kPrepositions, lowered)) return lowered == "to" ? "TO" : "IN"; - if (contains_key(kConjunctions, lowered)) return "CC"; - if (contains_key(kAdverbs, lowered) || ends_with(lowered, "ly")) return "RB"; - if (is_number_token_impl(token, true)) return "CD"; - if (is_all_upper_ascii(token) || is_capitalized_ascii(token)) return "NNP"; - return "NN"; -} - -std::string join_words(const std::vector & words, const std::string & delimiter) { - std::ostringstream out; - for (size_t i = 0; i < words.size(); ++i) { - if (i > 0) { - out << delimiter; - } - out << words[i]; - } - return out.str(); -} - -std::string two_digit_cardinal(int value) { - static const std::array below_20 = { - "zero", "one", "two", "three", "four", "five", "six", "seven", "eight", "nine", - "ten", "eleven", "twelve", "thirteen", "fourteen", "fifteen", "sixteen", "seventeen", "eighteen", "nineteen" - }; - static const std::array tens = { - "", "", "twenty", "thirty", "forty", "fifty", "sixty", "seventy", "eighty", "ninety" - }; - if (value < 20) { - return below_20[static_cast(value)]; - } - const int ten = value / 10; - const int one = value % 10; - if (one == 0) { - return tens[static_cast(ten)]; - } - return std::string(tens[static_cast(ten)]) + "-" + below_20[static_cast(one)]; -} - -std::string integer_to_cardinal(int64_t value) { - if (value == 0) { - return "zero"; - } - if (value < 0) { - return "minus " + integer_to_cardinal(-value); - } - static const std::array scales = { - "", "thousand", "million", "billion", "trillion", "quadrillion", "quintillion" - }; - std::vector chunks; - size_t scale = 0; - while (value > 0) { - const int part = static_cast(value % 1000); - if (part != 0) { - std::string chunk; - const int hundreds = part / 100; - const int rest = part % 100; - if (hundreds > 0) { - chunk += two_digit_cardinal(hundreds) + " hundred"; - if (rest != 0) { - chunk += " "; - } - } - if (rest != 0) { - chunk += two_digit_cardinal(rest); - } - if (scale > 0) { - chunk += " "; - chunk += scales[scale]; - } - chunks.push_back(chunk); - } - value /= 1000; - ++scale; - } - std::reverse(chunks.begin(), chunks.end()); - return join_words(chunks, " "); -} - -std::string integer_to_ordinal(int64_t value) { - static const std::unordered_map replacements = { - {"one", "first"}, - {"two", "second"}, - {"three", "third"}, - {"five", "fifth"}, - {"eight", "eighth"}, - {"nine", "ninth"}, - {"twelve", "twelfth"}, - {"twenty", "twentieth"}, - {"thirty", "thirtieth"}, - {"forty", "fortieth"}, - {"fifty", "fiftieth"}, - {"sixty", "sixtieth"}, - {"seventy", "seventieth"}, - {"eighty", "eightieth"}, - {"ninety", "ninetieth"}, - {"hundred", "hundredth"}, - {"thousand", "thousandth"}, - {"million", "millionth"}, - {"billion", "billionth"}, - {"trillion", "trillionth"}, - }; - std::string cardinal = integer_to_cardinal(value); - size_t pos = cardinal.find_last_of(" -"); - const size_t begin = pos == std::string::npos ? 0 : pos + 1; - std::string tail = cardinal.substr(begin); - const auto it = replacements.find(tail); - if (it != replacements.end()) { - return cardinal.substr(0, begin) + it->second; - } - if (ends_with(tail, "y")) { - return cardinal.substr(0, cardinal.size() - 1) + "ieth"; - } - return cardinal + "th"; -} - -std::string integer_to_year_words(int64_t value) { - if (value < 1000 || value > 9999) { - return integer_to_cardinal(value); - } - if (value >= 2000 && value <= 2009) { - return "two thousand " + integer_to_cardinal(value % 1000); - } - const int first = static_cast(value / 100); - const int second = static_cast(value % 100); - if (second == 0) { - return integer_to_cardinal(first) + " hundred"; - } - if (second < 10) { - return integer_to_cardinal(first) + " oh " + integer_to_cardinal(second); - } - return integer_to_cardinal(first) + " " + integer_to_cardinal(second); -} - -std::vector spell_digit_string(const std::string & digits) { - static const std::array digit_words = { - "zero", "one", "two", "three", "four", "five", "six", "seven", "eight", "nine" - }; - std::vector out; - for (char ch : digits) { - if (std::isdigit(static_cast(ch)) != 0) { - out.emplace_back(digit_words[static_cast(ch - '0')]); - } - } - return out; -} - -} // namespace - -Lexicon::Lexicon(bool british) - : Lexicon( - resolve_default_lexicon_dir() / (british ? "gb_gold.tsv" : "us_gold.tsv"), - resolve_default_lexicon_dir() / (british ? "gb_silver.tsv" : "us_silver.tsv"), - british) {} - -Lexicon::Lexicon(std::filesystem::path gold_tsv, std::filesystem::path silver_tsv, bool british) - : british_(british) { - load_tsv(gold_tsv, golds_); - load_tsv(silver_tsv, silvers_); -} - -void Lexicon::load_tsv( - const std::filesystem::path & path, - std::unordered_map & target) { - std::ifstream input(path); - if (!input) { - throw std::runtime_error("failed to open misaki lexicon TSV: " + path.string()); - } - std::string line; - while (std::getline(input, line)) { - if (line.empty()) { - continue; - } - const auto fields = split_tab(line); - if (fields.size() != 3) { - throw std::runtime_error("invalid misaki lexicon TSV line: " + line); - } - auto & entry = target[fields[0]]; - if (fields[1] == "*") { - entry.has_default = true; - entry.default_phonemes = fields[2]; - } else { - entry.by_tag[fields[1]] = fields[2] == "~" ? std::optional() : std::optional(fields[2]); - } - if (fields[0].size() >= 2) { - if (fields[0] == lower_ascii(fields[0])) { - const auto capitalized = capitalize_ascii(fields[0]); - if (capitalized != fields[0] && target.find(capitalized) == target.end()) { - target[capitalized] = entry; - } - } else if (fields[0] == capitalize_ascii(fields[0])) { - const auto lowered = lower_ascii(fields[0]); - if (target.find(lowered) == target.end()) { - target[lowered] = entry; - } - } - } - } -} - -std::optional Lexicon::parent_tag(const std::optional & tag) { - if (!tag.has_value()) { - return std::nullopt; - } - if (tag->rfind("VB", 0) == 0) return std::string("VERB"); - if (tag->rfind("NN", 0) == 0) return std::string("NOUN"); - if (tag->rfind("ADV", 0) == 0 || tag->rfind("RB", 0) == 0) return std::string("ADV"); - if (tag->rfind("ADJ", 0) == 0 || tag->rfind("JJ", 0) == 0) return std::string("ADJ"); - return tag; -} - -std::string Lexicon::apply_stress(const std::string & phonemes, const std::optional & stress) { - if (phonemes.empty() || !stress.has_value()) { - return phonemes; - } - std::string out = phonemes; - if (*stress < -1.0f) { - while (true) { - size_t pos = out.find(kPrimaryStress); - if (pos == std::string::npos) break; - out.erase(pos, std::char_traits::length(kPrimaryStress)); - } - while (true) { - size_t pos = out.find(kSecondaryStress); - if (pos == std::string::npos) break; - out.erase(pos, std::char_traits::length(kSecondaryStress)); - } - return out; - } - if (*stress == -1.0f || ((*stress == 0.0f || *stress == -0.5f) && contains_codepoint(out, kPrimaryStress))) { - replace_all(out, kSecondaryStress, ""); - replace_all(out, kPrimaryStress, kSecondaryStress); - return out; - } - if ((*stress == 0.0f || *stress == 0.5f || *stress == 1.0f) && - !contains_codepoint(out, kPrimaryStress) && !contains_codepoint(out, kSecondaryStress)) { - for (char ch : out) { - if (kVowels.find(ch) != std::string::npos) { - return std::string(kSecondaryStress) + out; - } - } - return out; - } - if (*stress >= 1.0f && !contains_codepoint(out, kPrimaryStress) && contains_codepoint(out, kSecondaryStress)) { - replace_all(out, kSecondaryStress, kPrimaryStress); - return out; - } - if (*stress > 1.0f && !contains_codepoint(out, kPrimaryStress) && !contains_codepoint(out, kSecondaryStress)) { - for (char ch : out) { - if (kVowels.find(ch) != std::string::npos) { - return std::string(kPrimaryStress) + out; - } - } - } - return out; -} - -std::optional Lexicon::get_nnp(const std::string & word) const { - std::string phonemes; - for (char ch : word) { - if (!is_ascii_alpha(ch)) { - continue; - } - const std::string key(1, static_cast(std::toupper(static_cast(ch)))); - const auto it = golds_.find(key); - if (it == golds_.end() || !it->second.has_default) { - return std::nullopt; - } - phonemes += it->second.default_phonemes; - } - if (phonemes.empty()) { - return std::nullopt; - } - phonemes = apply_stress(phonemes, 0.0f); - const size_t split = phonemes.rfind(kSecondaryStress); - if (split != std::string::npos) { - phonemes.replace(split, std::char_traits::length(kSecondaryStress), kPrimaryStress); - } - return LexiconResult{phonemes, 3}; -} - -bool Lexicon::is_known(const std::string & word) const { - if (contains_key(golds_, word) || contains_key(silvers_, word) || (word.size() == 1 && contains_key(kSymbols, word[0]))) { - return true; - } - if (!is_all_ascii_alpha(word)) { - return false; - } - if (word.size() == 1) { - return true; - } - if (is_all_upper_ascii(word) && contains_key(golds_, lower_ascii(word))) { - return true; - } - return word.size() > 1 && std::all_of(word.begin() + 1, word.end(), [](char ch) { - return std::isupper(static_cast(ch)) != 0; - }); -} - -std::optional Lexicon::lookup_raw( - const std::string & word, - const std::optional & tag, - const std::optional & stress) const { - std::optional selected; - int rating = 4; - auto it = golds_.find(word); - if (it != golds_.end()) { - const auto & entry = it->second; - if (!entry.by_tag.empty()) { - std::optional query = tag; - if (!query.has_value()) { - query = "DEFAULT"; - } else if (!contains_key(entry.by_tag, *query)) { - query = parent_tag(query); - } - auto jt = entry.by_tag.find(query.value_or("DEFAULT")); - if (jt == entry.by_tag.end()) { - jt = entry.by_tag.find("DEFAULT"); - } - if (jt != entry.by_tag.end()) { - if (!jt->second.has_value()) { - return std::nullopt; - } - selected = *jt->second; - } - } else if (entry.has_default) { - selected = entry.default_phonemes; - } - } - if (!selected.has_value()) { - auto st = silvers_.find(word); - if (st != silvers_.end() && st->second.has_default) { - selected = st->second.default_phonemes; - rating = 3; - } - } - if (!selected.has_value() && is_all_upper_ascii(word)) { - return get_nnp(word); - } - if (!selected.has_value()) { - return std::nullopt; - } - return LexiconResult{apply_stress(*selected, normalize_stress(stress, *selected)), rating}; -} - -std::optional Lexicon::s_suffix(const std::string & stem) const { - if (stem.empty()) return std::nullopt; - if (ends_with_any(stem, {"p", "t", "k", "f", "θ"})) return stem + "s"; - if (ends_with_any(stem, {"s", "z", "ʃ", "ʒ", "ʧ", "ʤ"})) { - return stem + (british_ ? "ɪz" : "ᵻz"); - } - return stem + "z"; -} - -std::optional Lexicon::ed_suffix(const std::string & stem) const { - if (stem.empty()) return std::nullopt; - if (ends_with_any(stem, {"p", "k", "f", "θ", "ʃ", "s", "ʧ"})) return stem + "t"; - if (ends_with(stem, "d")) return stem + (british_ ? "ɪd" : "ᵻd"); - if (!ends_with(stem, "t")) return stem + "d"; - if (british_ || stem.size() < 2) return stem + "ɪd"; - if (kUsTaus.find(stem[stem.size() - 2]) != std::string::npos) return stem.substr(0, stem.size() - 1) + "ɾᵻd"; - return stem + "ᵻd"; -} - -std::optional Lexicon::ing_suffix(const std::string & stem) const { - if (stem.empty()) return std::nullopt; - if (british_) { - if (ends_with_any(stem, {"ə", "ː"})) return std::nullopt; - } else if (stem.size() > 1 && stem.back() == 't' && kUsTaus.find(stem[stem.size() - 2]) != std::string::npos) { - return stem.substr(0, stem.size() - 1) + "ɾɪŋ"; - } - return stem + "ɪŋ"; -} - -std::optional Lexicon::stem_s( - const std::string & word, - const std::optional & tag, - const std::optional & stress, - const TokenContext & ctx) const { - if (word.size() < 3 || word.back() != 's') return std::nullopt; - std::string stem; - if (!(word.size() >= 2 && word.substr(word.size() - 2) == "ss") && is_known(word.substr(0, word.size() - 1))) { - stem = word.substr(0, word.size() - 1); - } else if (((word.size() >= 2 && word.substr(word.size() - 2) == "'s") || - (word.size() > 4 && word.size() >= 2 && word.substr(word.size() - 2) == "es" && - word.substr(word.size() - 3) != "ies")) && - is_known(word.substr(0, word.size() - 2))) { - stem = word.substr(0, word.size() - 2); - } else if (word.size() > 4 && word.substr(word.size() - 3) == "ies" && is_known(word.substr(0, word.size() - 3) + "y")) { - stem = word.substr(0, word.size() - 3) + "y"; - } else { - return std::nullopt; - } - auto base = lookup(stem, tag, stress, ctx); - if (!base.has_value()) return std::nullopt; - auto phonemes = s_suffix(base->phonemes); - if (!phonemes.has_value()) return std::nullopt; - return LexiconResult{*phonemes, base->rating}; -} - -std::optional Lexicon::stem_ed( - const std::string & word, - const std::optional & tag, - const std::optional & stress, - const TokenContext & ctx) const { - if (word.size() < 4 || word.back() != 'd') return std::nullopt; - std::string stem; - if (!(word.size() >= 2 && word.substr(word.size() - 2) == "dd") && is_known(word.substr(0, word.size() - 1))) { - stem = word.substr(0, word.size() - 1); - } else if (word.size() > 4 && word.substr(word.size() - 2) == "ed" && - word.substr(word.size() - 3) != "eed" && is_known(word.substr(0, word.size() - 2))) { - stem = word.substr(0, word.size() - 2); - } else { - return std::nullopt; - } - auto base = lookup(stem, tag, stress, ctx); - if (!base.has_value()) return std::nullopt; - auto phonemes = ed_suffix(base->phonemes); - if (!phonemes.has_value()) return std::nullopt; - return LexiconResult{*phonemes, base->rating}; -} - -std::optional Lexicon::stem_ing( - const std::string & word, - const std::optional & tag, - const std::optional & stress, - const TokenContext & ctx) const { - if (word.size() < 5 || word.substr(word.size() - 3) != "ing") return std::nullopt; - std::string stem; - if (word.size() > 5 && is_known(word.substr(0, word.size() - 3))) { - stem = word.substr(0, word.size() - 3); - } else if (is_known(word.substr(0, word.size() - 3) + "e")) { - stem = word.substr(0, word.size() - 3) + "e"; - } else if (word.size() > 5) { - const std::string doubled = word.substr(word.size() - 6); - if ((doubled.size() >= 6 && - ((doubled[0] == doubled[1] && std::string("bcdgklmnprstvxz").find(doubled[0]) != std::string::npos) || - doubled.rfind("cking") != std::string::npos)) && - is_known(word.substr(0, word.size() - 4))) { - stem = word.substr(0, word.size() - 4); - } else { - return std::nullopt; - } - } else { - return std::nullopt; - } - auto base = lookup(stem, tag, stress.has_value() ? stress : std::optional(0.5f), ctx); - if (!base.has_value()) return std::nullopt; - auto phonemes = ing_suffix(base->phonemes); - if (!phonemes.has_value()) return std::nullopt; - return LexiconResult{*phonemes, base->rating}; -} - -std::optional Lexicon::get_special_case( - const std::string & word, - const std::optional & tag, - const std::optional & stress, - const TokenContext & ctx) const { - (void) stress; - if (tag.has_value() && *tag == "ADD" && word.size() == 1 && contains_key(kAddSymbols, word[0])) { - return lookup(kAddSymbols.at(word[0]), std::nullopt, -0.5f, ctx); - } - if (word.size() == 1 && contains_key(kSymbols, word[0])) { - return lookup(kSymbols.at(word[0]), std::nullopt, std::nullopt, ctx); - } - if ((word == "a" || word == "A")) { - if (tag.has_value() && *tag == "DT") return LexiconResult{"ɐ", 4}; - return LexiconResult{"ˈA", 4}; - } - if (word == "am" || word == "Am" || word == "AM") { - if (tag.has_value() && tag->rfind("NN", 0) == 0) { - return get_nnp(word); - } - if (!ctx.future_vowel.has_value() || word != "am" || (stress.has_value() && *stress > 0.0f)) { - return lookup_raw("am", std::nullopt, stress); - } - return LexiconResult{"ɐm", 4}; - } - if (word == "an" || word == "An" || word == "AN") { - if (word == "AN" && tag.has_value() && tag->rfind("NN", 0) == 0) { - return get_nnp(word); - } - return LexiconResult{"ɐn", 4}; - } - if (word == "I" && tag.has_value() && *tag == "PRP") { - return LexiconResult{std::string(kSecondaryStress) + "I", 4}; - } - if ((word == "by" || word == "By" || word == "BY") && parent_tag(tag) == std::optional("ADV")) { - return LexiconResult{"bˈI", 4}; - } - if ((word == "to" || word == "To") || (word == "TO" && tag.has_value() && (*tag == "TO" || *tag == "IN"))) { - if (!ctx.future_vowel.has_value()) return lookup_raw("to", std::nullopt, stress); - return LexiconResult{*ctx.future_vowel ? "tʊ" : "tə", 4}; - } - if ((word == "in" || word == "In") || (word == "IN" && (!tag.has_value() || *tag != "NNP"))) { - return LexiconResult{(ctx.future_vowel.has_value() && tag == std::optional("IN")) ? "ɪn" : std::string(kPrimaryStress) + "ɪn", 4}; - } - if ((word == "the" || word == "The") || (word == "THE" && tag.has_value() && *tag == "DT")) { - return LexiconResult{ctx.future_vowel == true ? "ði" : "ðə", 4}; - } - if (tag == std::optional("IN")) { - const std::string lowered = lower_ascii(word); - if (lowered == "vs" || lowered == "vs.") { - return lookup("versus", std::nullopt, std::nullopt, ctx); - } - } - if (word == "used" || word == "Used" || word == "USED") { - if (tag == std::optional("VBD") && ctx.future_to) { - return lookup_raw("used", std::optional("VBD"), stress); - } - return lookup_raw("used", std::nullopt, stress); - } - if (word.find('.') != std::string::npos) { - std::string stripped = word; - stripped.erase(std::remove(stripped.begin(), stripped.end(), '.'), stripped.end()); - if (!stripped.empty() && is_all_ascii_alpha(stripped)) { - const auto parts = split_non_alpha(word); - size_t longest = 0; - for (const auto & part : parts) longest = std::max(longest, part.size()); - if (longest < 3) { - return get_nnp(word); - } - } - } - return std::nullopt; -} - -std::optional Lexicon::lookup( - const std::string & word, - const std::optional & tag, - const std::optional & stress, - const TokenContext & ctx) const { - if (const auto special = get_special_case(word, tag, stress, ctx)) { - return special; - } - std::string lookup_word = word; - const std::string lowered = lower_ascii(word); - if (word.size() > 1 && - std::all_of(word.begin(), word.end(), [](unsigned char ch) { return std::isalpha(ch) != 0 || ch == '\''; }) && - word != lowered && - (!tag.has_value() || *tag != "NNP" || word.size() > 7) && - !contains_key(golds_, word) && !contains_key(silvers_, word) && - (is_all_upper_ascii(word) || is_capitalized_ascii(word)) && - (contains_key(golds_, lowered) || contains_key(silvers_, lowered))) { - lookup_word = lowered; - } - if (is_known(lookup_word)) { - if (const auto direct = lookup_raw(lookup_word, tag, stress)) { - return direct; - } - } - if (lookup_word.size() >= 2 && lookup_word.substr(lookup_word.size() - 2) == "s'") { - if (const auto direct = lookup_raw(lookup_word.substr(0, lookup_word.size() - 2) + "'s", tag, stress)) { - return direct; - } - } - if (!lookup_word.empty() && lookup_word.back() == '\'') { - if (const auto direct = lookup_raw(lookup_word.substr(0, lookup_word.size() - 1), tag, stress)) { - return direct; - } - } - if (const auto stem = stem_s(lookup_word, tag, stress, ctx)) return stem; - if (const auto stem = stem_ed(lookup_word, tag, stress, ctx)) return stem; - if (const auto stem = stem_ing(lookup_word, tag, stress, ctx)) return stem; - return std::nullopt; -} - -std::optional Lexicon::get_word( - const std::string & word, - const std::optional & tag, - const std::optional & stress, - const TokenContext & ctx) const { - std::string maybe_lower = word; - const std::string lowered = lower_ascii(word); - if (word.size() > 1 && word.find_first_not_of("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz'") == std::string::npos && - word != lowered && - (!tag.has_value() || *tag != "NNP" || word.size() > 7) && - !contains_key(golds_, word) && !contains_key(silvers_, word) && - (is_all_upper_ascii(word) || is_capitalized_ascii(word)) && - (contains_key(golds_, lowered) || contains_key(silvers_, lowered) || - stem_s(lowered, tag, stress, ctx).has_value() || - stem_ed(lowered, tag, stress, ctx).has_value() || - stem_ing(lowered, tag, stress, ctx).has_value())) { - maybe_lower = lowered; - } - if (is_known(maybe_lower)) { - return lookup(maybe_lower, tag, stress, ctx); - } - if (ends_with(maybe_lower, "s'") && is_known(maybe_lower.substr(0, maybe_lower.size() - 2) + "'s")) { - return lookup(maybe_lower.substr(0, maybe_lower.size() - 2) + "'s", tag, stress, ctx); - } - if (ends_with(maybe_lower, "'") && is_known(maybe_lower.substr(0, maybe_lower.size() - 1))) { - return lookup(maybe_lower.substr(0, maybe_lower.size() - 1), tag, stress, ctx); - } - if (const auto stem = stem_s(maybe_lower, tag, stress, ctx)) return stem; - if (const auto stem = stem_ed(maybe_lower, tag, stress, ctx)) return stem; - if (const auto stem = stem_ing(maybe_lower, tag, stress.has_value() ? stress : std::optional(0.5f), ctx)) return stem; - return std::nullopt; -} - -bool Lexicon::is_currency_shape(const std::string & word) { - const size_t dot_count = static_cast(std::count(word.begin(), word.end(), '.')); - if (dot_count == 0) { - return true; - } - if (dot_count > 1) { - return false; - } - const size_t dot = word.find('.'); - const std::string cents = word.substr(dot + 1); - if (cents.size() < 3) { - return true; - } - return std::all_of(cents.begin(), cents.end(), [](char ch) { return ch == '0'; }); -} - -bool Lexicon::is_number_token(const std::string & word, bool is_head) { - return is_number_token_impl(word, is_head); -} - -std::optional Lexicon::get_number( - const std::string & word, - const std::optional & currency, - bool is_head) const { - std::string suffix; - std::string raw = word; - size_t suffix_begin = raw.size(); - while (suffix_begin > 0 && std::isalpha(static_cast(raw[suffix_begin - 1])) != 0) { - --suffix_begin; - } - suffix = raw.substr(suffix_begin); - raw = raw.substr(0, suffix_begin); - - std::vector> pieces; - auto append_lookup = [&](const std::string & token, std::optional stress = std::nullopt) { - auto found = lookup(token, std::nullopt, stress, TokenContext{}); - if (!found.has_value()) { - throw std::runtime_error("english g2p number expansion missing lexicon entry for: " + token); - } - pieces.emplace_back(found->phonemes, found->rating); - }; - auto append_phrase = [&](const std::string & phrase) { - for (const std::string & chunk : split_words(phrase)) { - const auto parts = split_non_alpha(chunk); - if (parts.empty()) { - continue; - } - for (const std::string & part : parts) { - if (part.empty()) continue; - append_lookup(part, part == "point" ? std::optional(-2.0f) : std::nullopt); - } - } - }; - - if (!raw.empty() && raw.front() == '-') { - append_lookup("minus"); - raw.erase(raw.begin()); - } - - const std::string no_commas = [&]() { - std::string s = raw; - s.erase(std::remove(s.begin(), s.end(), ','), s.end()); - return s; - }(); - - if (currency.has_value() && contains_key(kCurrencies, *currency) && is_currency_shape(no_commas)) { - const auto [major_name, minor_name] = kCurrencies.at(*currency); - const size_t dot = no_commas.find('.'); - const int64_t major = dot == std::string::npos || no_commas.substr(0, dot).empty() ? 0 : std::stoll(no_commas.substr(0, dot)); - std::string minor_text = dot == std::string::npos ? "" : no_commas.substr(dot + 1); - if (minor_text.size() > 2) { - minor_text = minor_text.substr(0, 2); - } - while (!minor_text.empty() && minor_text.size() < 2) { - minor_text.push_back('0'); - } - const int64_t minor = minor_text.empty() ? 0 : std::stoll(minor_text); - if (!(major == 0 && minor > 0)) { - append_phrase(integer_to_cardinal(major)); - append_phrase(major == 1 ? major_name : major_name + "s"); - } - if (minor > 0) { - if (major > 0) { - append_lookup("and"); - } - append_phrase(integer_to_cardinal(minor)); - append_phrase((minor == 1 || minor_name == "pence") ? minor_name : minor_name + "s"); - } - } else if (is_digit_text(no_commas) && contains_key(kOrdinals, lower_ascii(suffix))) { - append_phrase(integer_to_ordinal(std::stoll(no_commas))); - } else if (currency.has_value() == false && no_commas.find('.') == std::string::npos && no_commas.size() == 4 && is_digit_text(no_commas)) { - append_phrase(integer_to_year_words(std::stoll(no_commas))); - } else if (!is_head && no_commas.find('.') == std::string::npos) { - if ((no_commas.size() > 1 && no_commas.front() == '0') || no_commas.size() > 3) { - for (const std::string & digit_word : spell_digit_string(no_commas)) { - append_lookup(digit_word); - } - } else if (no_commas.size() == 3 && no_commas.substr(1) != "00") { - append_phrase(integer_to_cardinal(no_commas[0] - '0')); - if (no_commas[1] == '0') { - append_lookup("O", -2.0f); - append_phrase(integer_to_cardinal(no_commas[2] - '0')); - } else { - append_phrase(integer_to_cardinal(std::stoi(no_commas.substr(1)))); - } - } else { - append_phrase(integer_to_cardinal(std::stoll(no_commas))); - } - } else if (std::count(no_commas.begin(), no_commas.end(), '.') > 1 || !is_head) { - std::stringstream ss(no_commas); - std::string segment; - while (std::getline(ss, segment, '.')) { - if (segment.empty()) continue; - if ((segment.size() > 1 && segment.front() == '0') || (segment.size() != 2 && std::any_of(segment.begin() + 1, segment.end(), [](char ch) { return ch != '0'; }))) { - for (const std::string & digit_word : spell_digit_string(segment)) { - append_lookup(digit_word); - } - } else { - append_phrase(integer_to_cardinal(std::stoll(segment))); - } - } - } else if (no_commas.find('.') != std::string::npos) { - const size_t dot = no_commas.find('.'); - const std::string left = no_commas.substr(0, dot); - const std::string right = no_commas.substr(dot + 1); - if (left.empty()) { - append_lookup("point"); - } else { - append_phrase(integer_to_cardinal(std::stoll(left))); - append_lookup("point"); - } - for (const std::string & digit_word : spell_digit_string(right)) { - append_lookup(digit_word); - } - } else { - append_phrase(integer_to_cardinal(std::stoll(no_commas))); - } - - if (pieces.empty()) { - return std::nullopt; - } - - std::string phonemes; - int rating = pieces.front().second; - for (size_t i = 0; i < pieces.size(); ++i) { - if (i > 0) phonemes.push_back(' '); - phonemes += pieces[i].first; - rating = std::min(rating, pieces[i].second); - } - - const std::string lowered_suffix = lower_ascii(suffix); - if (lowered_suffix == "s" || lowered_suffix == "'s") { - auto updated = s_suffix(phonemes); - if (updated.has_value()) phonemes = *updated; - } else if (lowered_suffix == "ed" || lowered_suffix == "'d") { - auto updated = ed_suffix(phonemes); - if (updated.has_value()) phonemes = *updated; - } else if (lowered_suffix == "ing") { - auto updated = ing_suffix(phonemes); - if (updated.has_value()) phonemes = *updated; - } - return LexiconResult{phonemes, rating}; -} - -std::optional Lexicon::operator()(const MToken & token, const TokenContext & ctx) const { - std::string word = token.meta.alias.value_or(token.text); - std::replace(word.begin(), word.end(), static_cast(0x91), '\''); - std::replace(word.begin(), word.end(), static_cast(0x92), '\''); - - std::optional stress; - if (word != lower_ascii(word)) { - stress = word == upper_ascii(word) ? 2.0f : 0.5f; - } - if (const auto result = get_word(word, token.tag, stress, ctx)) { - std::string phonemes = result->phonemes; - if (token.meta.currency.has_value()) { - const auto it = kCurrencies.find(*token.meta.currency); - if (it != kCurrencies.end()) { - auto unit = get_word(it->second.first + "s", std::nullopt, std::nullopt, TokenContext{}); - if (unit.has_value()) { - phonemes += " " + unit->phonemes; - } - } - } - return LexiconResult{apply_stress(phonemes, token.meta.stress), result->rating}; - } - if (is_number_token(word, token.meta.is_head)) { - if (const auto result = get_number(word, token.meta.currency, token.meta.is_head)) { - return LexiconResult{apply_stress(result->phonemes, token.meta.stress), result->rating}; - } - } - if (!std::all_of(word.begin(), word.end(), [](unsigned char ch) { return std::isalpha(ch) != 0; })) { - return std::nullopt; - } - return std::nullopt; -} - -EnglishG2P::EnglishG2P(bool british) - : lexicon_(british) {} - -EnglishG2P::EnglishG2P(std::filesystem::path lexicon_dir, bool british) - : lexicon_( - lexicon_dir / (british ? "gb_gold.tsv" : "us_gold.tsv"), - lexicon_dir / (british ? "gb_silver.tsv" : "us_silver.tsv"), - british) {} - -PreprocessResult EnglishG2P::preprocess(const std::string & text) { - PreprocessResult result; - result.text = trim_copy(text); - std::string rebuilt; - size_t token_index = 0; - for (size_t i = 0; i < result.text.size();) { - if (result.text[i] == '[') { - const size_t close_text = result.text.find(']', i + 1); - const size_t open_feat = close_text == std::string::npos ? std::string::npos : result.text.find('(', close_text + 1); - const size_t close_feat = open_feat == std::string::npos ? std::string::npos : result.text.find(')', open_feat + 1); - if (close_text != std::string::npos && open_feat == close_text + 1 && close_feat != std::string::npos) { - const std::string visible = result.text.substr(i + 1, close_text - i - 1); - const std::string feature = result.text.substr(open_feat + 1, close_feat - open_feat - 1); - rebuilt += visible; - const size_t emitted_count = count_words(visible); - FeatureValue value; - if (!feature.empty()) { - int64_t as_int = 0; - const auto int_result = std::from_chars(feature.data(), feature.data() + feature.size(), as_int); - if (int_result.ec == std::errc() && int_result.ptr == feature.data() + feature.size()) { - value = as_int; - } else if (feature == "0.5" || feature == "+0.5") { - value = 0.5f; - } else if (feature == "-0.5") { - value = -0.5f; - } else if (feature.size() > 1 && feature.front() == '/' && feature.back() == '/') { - value = feature.substr(0, feature.size() - 1); - } else if (feature.size() > 1 && feature.front() == '#' && feature.back() == '#') { - value = feature.substr(0, feature.size() - 1); - } else { - value = std::string(); - } - if (emitted_count != 0) { - result.features[token_index] = value; - } - } - token_index += emitted_count; - i = close_feat + 1; - continue; - } - } - rebuilt.push_back(result.text[i]); - ++i; - } - result.text = rebuilt; - return result; -} - -std::vector EnglishG2P::tokenize( - const std::string & text, - const std::unordered_map & features) const { - std::vector tokens; - size_t i = 0; - while (i < text.size()) { - while (i < text.size() && std::isspace(static_cast(text[i])) != 0) { - ++i; - } - if (i >= text.size()) { - break; - } - size_t begin = i; - if (std::isalpha(static_cast(text[i])) != 0 || std::isdigit(static_cast(text[i])) != 0) { - while (i < text.size() && (std::isalnum(static_cast(text[i])) != 0 || text[i] == '\'' || text[i] == '.' || text[i] == ',' || text[i] == '-' || text[i] == '/' || text[i] == '_' || static_cast(text[i]) >= 0x80)) { - ++i; - } - } else { - ++i; - } - size_t end = i; - size_t ws_begin = i; - while (i < text.size() && std::isspace(static_cast(text[i])) != 0) { - ++i; - } - MToken token; - token.text = text.substr(begin, end - begin); - token.tag = guess_tag(token.text); - token.whitespace = text.substr(ws_begin, i - ws_begin); - token.meta.num_flags = ""; - tokens.push_back(std::move(token)); - } - for (const auto & [index, value] : features) { - if (index >= tokens.size()) { - continue; - } - auto & token = tokens[index]; - if (const auto * as_int = std::get_if(&value)) { - token.meta.stress = static_cast(*as_int); - } else if (const auto * as_float = std::get_if(&value)) { - token.meta.stress = *as_float; - } else if (const auto * as_string = std::get_if(&value)) { - if (starts_with(*as_string, "/")) { - token.phonemes = as_string->substr(1); - token.meta.rating = 5; - } else if (starts_with(*as_string, "#")) { - token.meta.num_flags = as_string->substr(1); - } - } - } - for (size_t idx = 0; idx < tokens.size(); ++idx) { - auto & token = tokens[idx]; - const std::string lowered = lower_ascii(token.text); - const std::string next = idx + 1 < tokens.size() ? lower_ascii(tokens[idx + 1].text) : std::string(); - if (lowered == "used" && next == "to") { - token.tag = "VBD"; - } else if (lowered == "by" && next != "the" && next != "way") { - token.tag = "RB"; - } else if (lowered == "read" && idx > 0 && lower_ascii(tokens[idx - 1].text) == "to") { - token.tag = "VB"; - } - } - return tokens; -} - -MToken EnglishG2P::merge_token_pair( - const MToken & left, - const MToken & right, - const std::optional & unk) { - MToken merged; - merged.text.reserve(left.text.size() + left.whitespace.size() + right.text.size()); - merged.text += left.text; - merged.text += left.whitespace; - merged.text += right.text; - merged.tag = left.tag; - merged.whitespace = right.whitespace; - merged.meta.is_head = left.meta.is_head; - merged.meta.prespace = left.meta.prespace; - merged.meta.num_flags = left.meta.num_flags; - if (unk.has_value()) { - std::string phonemes; - phonemes += left.phonemes.value_or(*unk); - if (right.meta.prespace && !phonemes.empty() && - !std::isspace(static_cast(phonemes.back())) && - right.phonemes.has_value()) { - phonemes.push_back(' '); - } - phonemes += right.phonemes.value_or(*unk); - merged.phonemes = std::move(phonemes); - } - if (left.meta.rating.has_value() && right.meta.rating.has_value()) { - merged.meta.rating = std::min(*left.meta.rating, *right.meta.rating); - } else if (left.meta.rating.has_value()) { - merged.meta.rating = left.meta.rating; - } else { - merged.meta.rating = right.meta.rating; - } - if (left.meta.stress.has_value() && right.meta.stress.has_value()) { - if (*left.meta.stress == *right.meta.stress) { - merged.meta.stress = left.meta.stress; - } - } else if (left.meta.stress.has_value()) { - merged.meta.stress = left.meta.stress; - } else { - merged.meta.stress = right.meta.stress; - } - merged.meta.currency = right.meta.currency.has_value() ? right.meta.currency : left.meta.currency; - return merged; -} - -MToken EnglishG2P::merge_tokens( - const std::vector & tokens, - size_t begin, - size_t end, - const std::optional & unk) { - MToken merged; - merged.text.reserve((end - begin) * 4); - for (size_t i = begin; i < end; ++i) { - merged.text += tokens[i].text; - if (i + 1 != end) { - merged.text += tokens[i].whitespace; - } - } - merged.tag = tokens[begin].tag; - merged.whitespace = tokens[end - 1].whitespace; - merged.meta.is_head = tokens[begin].meta.is_head; - merged.meta.prespace = tokens[begin].meta.prespace; - merged.meta.num_flags = tokens[begin].meta.num_flags; - std::optional rating; - std::optional stress; - std::optional currency; - if (unk.has_value()) { - std::string phonemes; - for (size_t i = begin; i < end; ++i) { - const auto & token = tokens[i]; - if (token.meta.prespace && !phonemes.empty() && !std::isspace(static_cast(phonemes.back())) && token.phonemes.has_value()) { - phonemes.push_back(' '); - } - phonemes += token.phonemes.value_or(*unk); - } - merged.phonemes = phonemes; - } - for (size_t i = begin; i < end; ++i) { - const auto & token = tokens[i]; - if (!rating.has_value()) { - rating = token.meta.rating; - } else if (token.meta.rating.has_value()) { - rating = std::min(*rating, *token.meta.rating); - } - if (token.meta.stress.has_value()) { - if (stress.has_value() && *stress != *token.meta.stress) { - stress = std::nullopt; - } else if (!stress.has_value()) { - stress = token.meta.stress; - } - } - if (token.meta.currency.has_value()) { - currency = token.meta.currency; - } - } - merged.meta.rating = rating; - merged.meta.stress = stress; - merged.meta.currency = currency; - return merged; -} - -std::vector EnglishG2P::fold_left(const std::vector & tokens) { - std::vector result; - result.reserve(tokens.size()); - for (const auto & token : tokens) { - if (!result.empty() && !token.meta.is_head) { - result.back() = merge_token_pair(result.back(), token, std::string()); - } else { - result.push_back(token); - } - } - return result; -} - -std::vector>> EnglishG2P::retokenize(const std::vector & tokens) { - std::vector>> words; - std::optional currency; - for (size_t i = 0; i < tokens.size(); ++i) { - const auto & token = tokens[i]; - std::vector subtokens; - if (!token.meta.alias.has_value() && !token.phonemes.has_value()) { - const auto parts = subtokenize(token.text); - subtokens.reserve(parts.size()); - for (const auto & part : parts) { - MToken sub = token; - sub.text = part; - sub.tag = part == token.text ? token.tag : guess_tag(part); - sub.whitespace.clear(); - sub.meta.is_head = true; - sub.meta.prespace = false; - subtokens.push_back(std::move(sub)); - } - } else { - subtokens.push_back(token); - } - if (!subtokens.empty()) { - subtokens.back().whitespace = token.whitespace; - } - for (size_t j = 0; j < subtokens.size(); ++j) { - auto & sub = subtokens[j]; - if (sub.meta.alias.has_value() || sub.phonemes.has_value()) { - } else if (sub.tag == "$" && contains_key(kCurrencies, sub.text)) { - currency = sub.text; - sub.phonemes = ""; - sub.meta.rating = 4; - } else if (sub.tag == ":" && (sub.text == "-" || sub.text == "–")) { - sub.phonemes = "—"; - sub.meta.rating = 3; - } else if (contains_key(kPunctTags, sub.tag) && - !std::all_of(sub.text.begin(), sub.text.end(), [](unsigned char ch) { return std::isalpha(ch) != 0; })) { - const auto it = kPunctTagPhonemes.find(sub.tag); - if (it != kPunctTagPhonemes.end()) { - sub.phonemes = it->second; - } else { - std::string kept; - for (char ch : sub.text) { - if (kPuncts.find(ch) != std::string::npos) { - kept.push_back(ch); - } - } - sub.phonemes = kept; - } - sub.meta.rating = 4; - } else if (currency.has_value()) { - if (sub.tag != "CD") { - currency.reset(); - } else if (j + 1 == subtokens.size() && (i + 1 == tokens.size() || tokens[i + 1].tag != "CD")) { - sub.meta.currency = currency; - } - } else if (0 < j && j + 1 < subtokens.size() && sub.text == "2" && - !subtokens[j - 1].text.empty() && !subtokens[j + 1].text.empty() && - std::isalpha(static_cast(subtokens[j - 1].text.back())) != 0 && - std::isalpha(static_cast(subtokens[j + 1].text.front())) != 0) { - sub.meta.alias = "to"; - } - - if (sub.meta.alias.has_value() || sub.phonemes.has_value()) { - words.push_back(sub); - } else if (!words.empty() && std::holds_alternative>(words.back()) && - std::get>(words.back()).back().whitespace.empty()) { - sub.meta.is_head = false; - std::get>(words.back()).push_back(sub); - } else { - if (sub.whitespace.empty()) { - words.push_back(std::vector{sub}); - } else { - words.push_back(sub); - } - } - } - } - for (auto & word : words) { - if (std::holds_alternative>(word)) { - auto & group = std::get>(word); - if (group.size() == 1) { - word = group.front(); - } - } - } - return words; -} - -void EnglishG2P::resolve_tokens(std::vector & tokens) { - std::string text; - for (size_t i = 0; i < tokens.size(); ++i) { - text += tokens[i].text; - if (i + 1 != tokens.size()) { - text += tokens[i].whitespace; - } - } - bool saw_alpha = false; - bool saw_digit = false; - bool saw_other = false; - for (char ch : text) { - if (contains_key(kSubtokenJunk, ch)) { - continue; - } - const int klass = english_g2p_char_class(ch); - saw_alpha = saw_alpha || klass == 0; - saw_digit = saw_digit || klass == 1; - saw_other = saw_other || klass == 2; - } - const bool prespace = - contains_whitespace(text) || - text.find('/') != std::string::npos || - static_cast(saw_alpha) + static_cast(saw_digit) + static_cast(saw_other) > 1; - for (size_t i = 0; i < tokens.size(); ++i) { - auto & token = tokens[i]; - if (!token.phonemes.has_value()) { - if (i + 1 == tokens.size() && token.text.size() == 1 && kNonQuotePuncts.find(token.text[0]) != std::string::npos) { - token.phonemes = token.text; - token.meta.rating = 3; - } else if (all_subtoken_junk(token.text)) { - token.phonemes = ""; - token.meta.rating = 3; - } - } else if (i > 0) { - token.meta.prespace = prespace; - } - } - if (prespace) { - return; - } - struct WeightedIndex { - bool primary = false; - int weight = 0; - size_t index = 0; - }; - std::vector indices; - for (size_t i = 0; i < tokens.size(); ++i) { - if (tokens[i].phonemes.has_value() && !tokens[i].phonemes->empty()) { - indices.push_back({tokens[i].phonemes->find(kPrimaryStress) != std::string::npos, stress_weight(*tokens[i].phonemes), i}); - } - } - int primary_count = 0; - for (const auto & entry : indices) primary_count += entry.primary ? 1 : 0; - if (indices.size() == 2 && tokens[indices[0].index].text.size() == 1) { - tokens[indices[1].index].phonemes = Lexicon::apply_stress(*tokens[indices[1].index].phonemes, -0.5f); - return; - } - if (indices.size() < 2 || primary_count <= static_cast((indices.size() + 1) / 2)) { - return; - } - std::sort(indices.begin(), indices.end(), [](const WeightedIndex & lhs, const WeightedIndex & rhs) { - if (lhs.primary != rhs.primary) return lhs.primary < rhs.primary; - if (lhs.weight != rhs.weight) return lhs.weight < rhs.weight; - return lhs.index < rhs.index; - }); - indices.resize(indices.size() / 2); - for (const auto & entry : indices) { - tokens[entry.index].phonemes = Lexicon::apply_stress(*tokens[entry.index].phonemes, -0.5f); - } -} - -TokenContext EnglishG2P::token_context(const TokenContext & ctx, const std::optional & phonemes, const MToken & token) { - std::optional vowel = ctx.future_vowel; - if (phonemes.has_value()) { - for (size_t i = 0; i < phonemes->size();) { - const size_t width = utf8_codepoint_width(*phonemes, i); - const char * symbol = phonemes->data() + i; - if (utf8_inventory_contains(kVowels, symbol, width)) { - vowel = true; - break; - } - if (utf8_inventory_contains(kConsonants, symbol, width) || - (width == 1 && kNonQuotePuncts.find(*symbol) != std::string::npos)) { - vowel = false; - break; - } - i += width; - } - } - const bool future_to = token.text == "to" || token.text == "To" || (token.text == "TO" && (token.tag == "TO" || token.tag == "IN")); - return TokenContext{vowel, future_to}; -} - -std::string EnglishG2P::tokens_to_phonemes(const std::vector & tokens) { - std::string result; - size_t reserve = 0; - for (const auto & token : tokens) { - if (token.phonemes.has_value()) { - reserve += token.phonemes->size(); - if (!token.whitespace.empty()) { - ++reserve; - } - } - } - result.reserve(reserve); - bool pending_space = false; - for (const auto & token : tokens) { - if (!token.phonemes.has_value()) { - continue; - } - if (pending_space && !result.empty()) { - result.push_back(' '); - } - result += *token.phonemes; - pending_space = !token.whitespace.empty(); - } - return result; -} - -std::pair> EnglishG2P::operator()(const std::string & text, bool enable_preprocess) const { - const PreprocessResult pre = enable_preprocess ? preprocess(text) : PreprocessResult{text, {}}; - auto tokens = tokenize(pre.text, pre.features); - tokens = fold_left(tokens); - auto words = retokenize(tokens); - TokenContext ctx; - std::vector resolved; - resolved.reserve(words.size()); - for (auto it = words.rbegin(); it != words.rend(); ++it) { - if (std::holds_alternative(*it)) { - MToken token = std::move(std::get(*it)); - if (!token.phonemes.has_value()) { - auto result = lexicon_(token, ctx); - if (!result.has_value()) { - result = resolve_with_safe_fallback(lexicon_, token); - } - token.phonemes = result->phonemes; - token.meta.rating = result->rating; - } - ctx = token_context(ctx, token.phonemes, token); - resolved.push_back(std::move(token)); - continue; - } - - auto group = std::move(std::get>(*it)); - bool should_fallback = false; - size_t left = 0; - size_t right = group.size(); - while (left < right) { - bool blocked = false; - for (size_t i = left; i < right; ++i) { - if (group[i].meta.alias.has_value() || group[i].phonemes.has_value()) { - blocked = true; - break; - } - } - if (!blocked) { - MToken merged = merge_tokens(group, left, right, std::nullopt); - if (auto result = lexicon_(merged, ctx)) { - group[left].phonemes = result->phonemes; - group[left].meta.rating = result->rating; - for (size_t i = left + 1; i < right; ++i) { - group[i].phonemes = ""; - group[i].meta.rating = result->rating; - } - ctx = token_context(ctx, group[left].phonemes, merged); - right = left; - left = 0; - continue; - } - } - if (left + 1 < right) { - ++left; - continue; - } - --right; - if (!group[right].phonemes.has_value()) { - if (all_subtoken_junk(group[right].text)) { - group[right].phonemes = ""; - group[right].meta.rating = 3; - } else { - should_fallback = true; - break; - } - } - left = 0; - } - if (should_fallback) { - MToken merged = merge_tokens(group, 0, group.size(), std::nullopt); - const LexiconResult fallback_result = resolve_with_safe_fallback(lexicon_, merged); - group[0].phonemes = fallback_result.phonemes; - group[0].meta.rating = fallback_result.rating; - for (size_t i = 1; i < group.size(); ++i) { - group[i].phonemes = ""; - group[i].meta.rating = fallback_result.rating; - } - } else { - resolve_tokens(group); - } - for (auto rit = group.rbegin(); rit != group.rend(); ++rit) { - ctx = token_context(ctx, rit->phonemes, *rit); - } - std::reverse(group.begin(), group.end()); - for (auto & token : group) { - resolved.push_back(std::move(token)); - } - } - std::reverse(resolved.begin(), resolved.end()); - for (auto & token : resolved) { - if (token.phonemes.has_value()) { - replace_all(*token.phonemes, "ɾ", "T"); - replace_all(*token.phonemes, "ʔ", "t"); - } - } - return {tokens_to_phonemes(resolved), resolved}; -} - -} // namespace kokoro_ggml::g2p_en diff --git a/src/models/kokoro_tts/g2p_multilingual.cpp b/src/models/kokoro_tts/g2p_multilingual.cpp index cefca2f96..cc624a116 100644 --- a/src/models/kokoro_tts/g2p_multilingual.cpp +++ b/src/models/kokoro_tts/g2p_multilingual.cpp @@ -90,9 +90,9 @@ std::string espeak_text(const std::string & text, const std::string & language, std::filesystem::path data; if (data_override && *data_override) data = std::filesystem::u8path(data_override); #ifdef AUDIOCPP_STATIC_ESPEAK - else if (!library.empty()) data = root / "espeak-ng-data"; + else if (!library.empty() && std::filesystem::is_regular_file(root / "espeak-ng-data" / "phontab")) data = root / "espeak-ng-data"; #else - else data = root / "espeak-ng-data"; + else if (std::filesystem::is_regular_file(root / "espeak-ng-data" / "phontab")) data = root / "espeak-ng-data"; #endif engine::audio::EspeakPhonemizer phonemizer(library, data, {language == "fr-fr" ? "fr" : language}); // Preserve punctuation ourselves: TextToPhonemes consumes clause punctuation. @@ -174,7 +174,19 @@ struct MultilingualG2P::Impl { explicit Impl(std::filesystem::path path) : root(std::move(path)) {} void load_ja() const { std::call_once(ja_once, [&] { - auto value = engine::io::json::parse_file(root / "g2p/ja.json"); + const auto g2p_path = root / "g2p/ja.json"; + if (!std::filesystem::is_regular_file(g2p_path)) { + throw std::runtime_error( + "Kokoro Japanese G2P resources are not bundled in this GGUF; " + "re-export the model with --embed-multilingual-resources to use Japanese voices"); + } + const auto unidic_path = root / "unidic"; + if (!std::filesystem::is_regular_file(unidic_path / "dicrc")) { + throw std::runtime_error( + "Kokoro UniDic resources are not bundled in this GGUF; " + "re-export the model with --embed-multilingual-resources to use Japanese voices"); + } + auto value = engine::io::json::parse_file(g2p_path); for (const auto & [k, v] : value.require("kana").as_object()) kana.emplace(decode(k), decode(v.as_string())); for (const auto & v : value.require("words").as_array()) ja_words.insert(v.as_string()); }); @@ -275,7 +287,15 @@ struct MultilingualG2P::Impl { } std::string chinese(const std::string & text) const { - std::call_once(zh_once, [&] { zh = engine::io::json::parse_file(root / "g2p/zh.json"); }); + std::call_once(zh_once, [&] { + const auto g2p_path = root / "g2p/zh.json"; + if (!std::filesystem::is_regular_file(g2p_path)) { + throw std::runtime_error( + "Kokoro Chinese G2P resources are not bundled in this GGUF; " + "re-export the model with --embed-multilingual-resources to use Chinese voices"); + } + zh = engine::io::json::parse_file(g2p_path); + }); const auto & frequencies = zh.require("frequency").as_object(); const auto & characters = zh.require("chars").as_object(); const auto & phrases = zh.require("phrases").as_object(); @@ -386,8 +406,8 @@ MultilingualG2P::~MultilingualG2P() = default; std::string MultilingualG2P::phonemize(const std::string & text, const std::string & language) const { if (language == "j") return impl_->japanese(text); if (language == "z") return impl_->chinese(text); - static const std::map langs = {{"e", "es"}, {"f", "fr-fr"}, - {"h", "hi"}, {"i", "it"}, {"p", "pt-br"}}; + static const std::map langs = {{"a", "en-us"}, {"b", "en"}, + {"e", "es"}, {"f", "fr-fr"}, {"h", "hi"}, {"i", "it"}, {"p", "pt-br"}}; auto it = langs.find(language); if (it == langs.end()) throw std::runtime_error("Unsupported Kokoro language: " + language); return espeak_text(text, it->second, impl_->root); diff --git a/src/models/kokoro_tts/loader.cpp b/src/models/kokoro_tts/loader.cpp deleted file mode 100644 index e00387e76..000000000 --- a/src/models/kokoro_tts/loader.cpp +++ /dev/null @@ -1,126 +0,0 @@ -#include "engine/models/kokoro_tts/loader.h" -#include "engine/models/kokoro_tts/package.h" -#include "engine/models/kokoro_tts/session.h" - -#include "engine/framework/io/filesystem.h" - -#include - -namespace engine::models::kokoro_tts { - -namespace { - -std::filesystem::path resolve_model_root(const std::filesystem::path & model_path) { - if (engine::io::is_existing_directory(model_path)) { - return std::filesystem::weakly_canonical(model_path); - } - if (engine::io::is_existing_file(model_path) && model_path.extension() == ".gguf") - return std::filesystem::weakly_canonical(model_path); - throw std::runtime_error("Kokoro TTS expects a model directory or GGUF: " + model_path.string()); -} - -std::vector discover_config_assets(const runtime::ModelLoadRequest & request) { - return runtime::discover_named_assets(resolve_model_root(request.model_path), {"config.json", "voices.json", "vocab.tsv"}); -} - -std::vector discover_weight_assets(const runtime::ModelLoadRequest & request) { - return runtime::discover_named_assets(resolve_model_root(request.model_path), {"kokoro-v1_0.safetensors"}); -} - -class KokoroTTSLoader final : public runtime::IVoiceModelLoader { -public: - std::string family() const override { - return "kokoro_tts"; - } - - bool can_load(const runtime::ModelLoadRequest & request) const override { - try { - const auto root = resolve_model_root(request.model_path); - if (root.extension() == ".gguf" && engine::io::is_existing_file(root)) - return (!request.family_hint.has_value() || *request.family_hint == family()) && is_kokoro_gguf(root); - return engine::io::is_existing_file(root / "config.json") - && engine::io::is_existing_file(root / "voices.json") - && engine::io::is_existing_file(root / "kokoro-v1_0.safetensors") - && (!request.family_hint.has_value() || *request.family_hint == family()); - } catch (...) { - return false; - } - } - - runtime::ModelInspection inspect(const runtime::ModelLoadRequest & request) const override { - const auto root = resolve_model_root(request.model_path); - runtime::ModelInspection inspection; - inspection.model_root = root; - inspection.metadata.family = family(); - inspection.metadata.variant = root.filename().string(); - inspection.metadata.description = "Kokoro TTS loaded from local extracted assets."; - inspection.metadata.config_candidates = {"config.json", "voices.json", "vocab.tsv"}; - inspection.metadata.weight_candidates = {"kokoro-v1_0.safetensors"}; - inspection.capabilities.supported_tasks = { - {runtime::VoiceTaskKind::Tts, {runtime::RunMode::Offline}}, - }; - inspection.capabilities.languages = {"a", "b", "e", "f", "h", "i", "j", "p", "z"}; - inspection.capabilities.supports_style_condition = true; - inspection.discovered_configs = discover_config_assets(request); - inspection.discovered_weights = discover_weight_assets(request); - return inspection; - } - - std::unique_ptr load(const runtime::ModelLoadRequest & request) const override { - return load_kokoro_tts_model(resolve_model_root(request.model_path)); - } -}; - -} // namespace - -KokoroTTSLoadedModel::KokoroTTSLoadedModel( - runtime::ModelMetadata metadata, - runtime::CapabilitySet capabilities, - std::shared_ptr assets) - : metadata_(std::move(metadata)), - capabilities_(std::move(capabilities)), - assets_(std::move(assets)) {} - -const runtime::ModelMetadata & KokoroTTSLoadedModel::metadata() const noexcept { - return metadata_; -} - -const runtime::CapabilitySet & KokoroTTSLoadedModel::capabilities() const noexcept { - return capabilities_; -} - -std::unique_ptr KokoroTTSLoadedModel::create_task_session( - const runtime::TaskSpec & task, - const runtime::SessionOptions & options) const { - if (task.task != runtime::VoiceTaskKind::Tts) { - throw std::runtime_error("Kokoro TTS only supports VoiceTaskKind::Tts"); - } - return std::make_unique(task, options, assets_); -} - -std::unique_ptr load_kokoro_tts_model(const std::filesystem::path & model_path) { - const auto root = resolve_model_root(model_path); - auto assets = load_kokoro_assets(root); - - runtime::ModelMetadata metadata; - metadata.family = "kokoro_tts"; - metadata.variant = root.filename().string(); - metadata.description = "Kokoro TTS loaded from local extracted assets."; - metadata.config_candidates = {"config.json", "voices.json", "vocab.tsv"}; - metadata.weight_candidates = {"kokoro-v1_0.safetensors"}; - - runtime::CapabilitySet capabilities; - capabilities.supported_tasks = { - {runtime::VoiceTaskKind::Tts, {runtime::RunMode::Offline}}, - }; - capabilities.languages = {"a", "b", "e", "f", "h", "i", "j", "p", "z"}; - capabilities.supports_style_condition = true; - - return std::make_unique(std::move(metadata), std::move(capabilities), std::move(assets)); -} - -std::shared_ptr make_kokoro_tts_loader() { - return std::make_shared(); -} - -} // namespace engine::models::kokoro_tts diff --git a/src/models/kokoro_tts/package.cpp b/src/models/kokoro_tts/package.cpp deleted file mode 100644 index 4c61c3fea..000000000 --- a/src/models/kokoro_tts/package.cpp +++ /dev/null @@ -1,178 +0,0 @@ -#include "engine/models/kokoro_tts/package.h" -#include "engine/framework/io/binary.h" -#include -#include -#include -#include -#include -#include -#include -#include - -namespace engine::models::kokoro_tts { -namespace { -using namespace engine::assets; -class GgufSource final : public TensorSource { - std::filesystem::path path_; - std::unique_ptr gguf_{nullptr, gguf_free}; - std::unique_ptr tensors_{nullptr, ggml_free}; - std::map names_; -public: - explicit GgufSource(const std::filesystem::path & path) : path_(path) { - ggml_context * tensors = nullptr; - gguf_.reset(gguf_init_from_file(path.string().c_str(), {true, &tensors})); - tensors_.reset(tensors); - if (!gguf_ || !tensors_) throw std::runtime_error("Invalid Kokoro GGUF: " + path.string()); - const auto architecture = gguf_find_key(gguf_.get(), "general.architecture"); - if (architecture < 0 || gguf_get_kv_type(gguf_.get(), architecture) != GGUF_TYPE_STRING || - std::string(gguf_get_val_str(gguf_.get(), architecture)) != "kokoro_tts") - throw std::runtime_error("Expected a kokoro_tts GGUF architecture"); - const auto names = gguf_find_key(gguf_.get(), "kokoro.tensor_names"); - if (names < 0 || gguf_get_kv_type(gguf_.get(), names) != GGUF_TYPE_ARRAY || - gguf_get_arr_type(gguf_.get(), names) != GGUF_TYPE_STRING || - gguf_get_arr_n(gguf_.get(), names) != static_cast(gguf_get_n_tensors(gguf_.get()))) - throw std::runtime_error("Missing Kokoro tensor name mapping"); - for (int64_t i = 0; i < gguf_get_n_tensors(gguf_.get()); ++i) - if (!names_.emplace(gguf_get_arr_str(gguf_.get(), names, i), gguf_get_tensor_name(gguf_.get(), i)).second) - throw std::runtime_error("Duplicate Kokoro tensor mapping"); - for (auto * t = ggml_get_first_tensor(tensors_.get()); t; t = ggml_get_next_tensor(tensors_.get(), t)) { - if (t->type != GGML_TYPE_F32 && t->type != GGML_TYPE_F16 && - t->type != GGML_TYPE_BF16 && t->type != GGML_TYPE_Q8_0) - throw std::runtime_error("Unsupported Kokoro GGUF tensor type: " + std::string(t->name)); - } - } - const gguf_context * metadata() const { return gguf_.get(); } - const std::filesystem::path & source_path() const noexcept override { return path_; } - bool has_tensor(std::string_view name) const noexcept override { - return names_.find(std::string(name)) != names_.end(); - } - TensorMetadata require_metadata(std::string_view name) const override { - auto found = names_.find(std::string(name)); - auto * t = found == names_.end() ? nullptr : ggml_get_tensor(tensors_.get(), found->second.c_str()); - if (!t) throw std::runtime_error("Missing Kokoro tensor: " + std::string(name)); - TensorMetadata m{std::string(name), ggml_type_name(t->type), {}}; - // GGUF dimensions use the reverse of the source checkpoint order. - for (int i = ggml_n_dims(t) - 1; i >= 0; --i) m.shape.push_back(t->ne[i]); - // GGUF elides trailing singleton dimensions; retain the source shape metadata. - auto key = "kokoro.tensor_shape." + std::string(name); - auto id = gguf_find_key(gguf_.get(), key.c_str()); - if (id >= 0) { - if (gguf_get_kv_type(gguf_.get(), id) != GGUF_TYPE_ARRAY || - gguf_get_arr_type(gguf_.get(), id) != GGUF_TYPE_INT64) - throw std::runtime_error("Invalid Kokoro tensor shape metadata"); - const auto n = gguf_get_arr_n(gguf_.get(), id); - const auto * dims = static_cast(gguf_get_arr_data(gguf_.get(), id)); - if (!n || n > 4) throw std::runtime_error("Invalid Kokoro tensor rank"); - m.shape.assign(dims, dims + n); - for (size_t i = 0; i < n; ++i) - if (dims[i] <= 0 || dims[i] != t->ne[n - i - 1]) - throw std::runtime_error("Inconsistent Kokoro tensor shape"); - } - return m; - } - std::vector tensors() const override { - std::vector out; - for (const auto & item : names_) out.push_back(require_metadata(item.first)); - return out; - } - RawTensorData require_tensor_data(std::string_view name) const override { - RawTensorData out{require_metadata(name), {}}; - const auto id = gguf_find_tensor(gguf_.get(), names_.at(std::string(name)).c_str()); - out.bytes.resize(gguf_get_tensor_size(gguf_.get(), id)); - std::ifstream in(path_, std::ios::binary); - in.seekg(gguf_get_data_offset(gguf_.get()) + gguf_get_tensor_offset(gguf_.get(), id)); - if (!in.read(reinterpret_cast(out.bytes.data()), out.bytes.size())) - throw std::runtime_error("Truncated Kokoro GGUF tensor: " + std::string(name)); - return out; - } - std::vector require_f32(std::string_view name, - const std::optional> & expected = std::nullopt) const override { - auto raw = require_tensor_data(name); - if (expected && *expected != raw.metadata.shape) throw std::runtime_error("Kokoro tensor shape mismatch"); - core::TensorShape shape; - shape.rank = static_cast(raw.metadata.shape.size()); - for (size_t i = 0; i < raw.metadata.shape.size(); ++i) shape.dims[i] = raw.metadata.shape[i]; - return tensor_data_to_f32(name, {shape, - ggml_type_for_tensor_storage(tensor_storage_type_for_dtype(raw.metadata.dtype)), std::move(raw.bytes)}); - } - std::optional> optional_f32(std::string_view name, - const std::optional> & expected = std::nullopt) const override { - if (!has_tensor(name)) return std::nullopt; - return require_f32(name, expected); - } - int64_t require_i64_scalar(std::string_view) const override { - throw std::runtime_error("Kokoro GGUF does not contain integer scalars"); - } -}; - -void extract_resources(const gguf_context * g, const std::filesystem::path & root) { - auto names = gguf_find_key(g, "audiocpp.embedded_files.names"); - auto offsets = gguf_find_key(g, "audiocpp.embedded_files.offsets"); - auto data = gguf_find_key(g, "audiocpp.embedded_files.data"); - if (names < 0 || offsets < 0 || data < 0 || - gguf_get_kv_type(g, names) != GGUF_TYPE_ARRAY || gguf_get_arr_type(g, names) != GGUF_TYPE_STRING || - gguf_get_kv_type(g, offsets) != GGUF_TYPE_ARRAY || gguf_get_arr_type(g, offsets) != GGUF_TYPE_UINT64 || - gguf_get_kv_type(g, data) != GGUF_TYPE_ARRAY || gguf_get_arr_type(g, data) != GGUF_TYPE_UINT8 || - gguf_get_arr_n(g, offsets) != gguf_get_arr_n(g, names) + 1) - throw std::runtime_error("Kokoro GGUF is missing valid embedded resources"); - const auto * begin = static_cast(gguf_get_arr_data(g, data)); - const auto * off = static_cast(gguf_get_arr_data(g, offsets)); - const auto size = gguf_get_arr_n(g, data); - std::set seen; - for (size_t i = 0; i < gguf_get_arr_n(g, names); ++i) { - std::string name = gguf_get_arr_str(g, names, i); - std::replace(name.begin(), name.end(), '\\', '/'); - const auto relative = std::filesystem::u8path(name).lexically_normal(); - if (name.empty() || name.find(':') != std::string::npos || relative.has_root_path() || - relative.empty() || *relative.begin() == ".." || !seen.insert(relative).second || - off[i] > off[i + 1] || off[i + 1] > size) - throw std::runtime_error("Unsafe Kokoro embedded resource"); - auto path = root / relative; - std::filesystem::create_directories(path.parent_path()); - std::ofstream out(path, std::ios::binary); - if (!out.write(begin + off[i], off[i + 1] - off[i])) - throw std::runtime_error("Cannot extract Kokoro resource: " + name); - } -} -} - -bool is_kokoro_gguf(const std::filesystem::path & path) noexcept { - try { - gguf_init_params params{true, nullptr}; - std::unique_ptr metadata( - gguf_init_from_file(path.string().c_str(), params), gguf_free); - if (!metadata) return false; - const auto architecture = gguf_find_key(metadata.get(), "general.architecture"); - return architecture >= 0 && gguf_get_kv_type(metadata.get(), architecture) == GGUF_TYPE_STRING && - std::string(gguf_get_val_str(metadata.get(), architecture)) == "kokoro_tts"; - } catch (...) { - return false; - } -} - -KokoroPackage::~KokoroPackage() { - if (temporary) { - std::error_code ec; - std::filesystem::remove_all(root, ec); - } -} -std::shared_ptr open_kokoro_package(const std::filesystem::path & path) { - auto package = std::make_shared(); - if (std::filesystem::is_directory(path)) { - package->root = std::filesystem::canonical(path); - package->weights = assets::open_tensor_source(package->root / "kokoro-v1_0.safetensors"); - return package; - } - auto source = std::make_shared(std::filesystem::canonical(path)); - std::random_device rng; - for (int tries = 0; tries < 20; ++tries) { - package->root = std::filesystem::temp_directory_path() / - ("audiocpp-kokoro-" + std::to_string(rng()) + "-" + std::to_string(rng())); - if (std::filesystem::create_directory(package->root)) { package->temporary = true; break; } - } - if (!package->temporary) throw std::runtime_error("Cannot create Kokoro resource directory"); - extract_resources(source->metadata(), package->root); - package->weights = source; - return package; -} -} diff --git a/src/models/kokoro_tts/session.cpp b/src/models/kokoro_tts/session.cpp index 98279685e..0547d50c7 100644 --- a/src/models/kokoro_tts/session.cpp +++ b/src/models/kokoro_tts/session.cpp @@ -4,6 +4,7 @@ #include "engine/framework/text/chunking.h" #include "engine/framework/debug/trace.h" #include "engine/framework/runtime/options.h" +#include "engine/framework/runtime/spec_backed_model.h" #include "engine/models/kokoro_tts/decoder.h" #include "engine/models/kokoro_tts/frontend.h" @@ -11,8 +12,7 @@ #include #include -#include -#include +#include #include namespace engine::models::kokoro_tts { @@ -20,142 +20,60 @@ namespace engine::models::kokoro_tts { namespace { using engine::debug::measure_ms; constexpr int64_t kDefaultTextChunkSize = 240; - -int64_t parse_positive_i64_option( - const runtime::SessionOptions & options, - std::initializer_list keys, - int64_t fallback) { - for (const char * key : keys) { - const auto it = options.options.find(key); - if (it == options.options.end() || it->second.empty()) { - continue; - } - const int64_t value = std::stoll(it->second); - if (value <= 0) { - throw std::runtime_error(std::string(key) + " must be positive"); - } - return value; - } - return fallback; -} - -uint64_t parse_u64_option( - const runtime::SessionOptions & options, - std::initializer_list keys, - uint64_t fallback) { - for (const char * key : keys) { - const auto it = options.options.find(key); - if (it == options.options.end() || it->second.empty()) { - continue; - } - return static_cast(std::stoull(it->second)); - } - return fallback; -} - -size_t parse_size_mb_option( - const runtime::SessionOptions & options, - std::initializer_list keys, - size_t fallback) { - for (const char * key : keys) { - const auto it = options.options.find(key); - if (it == options.options.end() || it->second.empty()) { - continue; - } - const auto mb = std::stoull(it->second); - if (mb == 0 || mb > std::numeric_limits::max() / (1024ull * 1024ull)) { - throw std::runtime_error(std::string(key) + " is out of range"); - } - return static_cast(mb) * 1024ull * 1024ull; - } - return fallback; -} - -engine::assets::TensorStorageType parse_storage_option( - const runtime::SessionOptions & options, - std::initializer_list keys, - engine::assets::TensorStorageType fallback) { - for (const char * key : keys) { - const auto it = options.options.find(key); - if (it == options.options.end() || it->second.empty()) { - continue; - } - return engine::assets::parse_tensor_storage_type(it->second); - } - return fallback; -} - -void validate_matmul_storage(engine::assets::TensorStorageType storage_type, const char * option_name) { - if (storage_type == engine::assets::TensorStorageType::Native || - storage_type == engine::assets::TensorStorageType::F32 || - storage_type == engine::assets::TensorStorageType::F16 || - storage_type == engine::assets::TensorStorageType::BF16 || - storage_type == engine::assets::TensorStorageType::Q8_0) { - return; - } - throw std::runtime_error(std::string(option_name) + " supports only native, f32, f16, bf16, and q8_0"); -} - -void validate_conv_storage(engine::assets::TensorStorageType storage_type, const char * option_name) { - if (storage_type == engine::assets::TensorStorageType::Native || - storage_type == engine::assets::TensorStorageType::F32 || - storage_type == engine::assets::TensorStorageType::F16) { - return; - } - throw std::runtime_error(std::string(option_name) + " supports only native, f32, and f16"); -} - -std::string request_cache_key(const runtime::Transcript & text) { - std::ostringstream out; - out << text.text.size() << ":" << text.text; - return out.str(); -} +constexpr const char * kFamily = "kokoro_tts"; +constexpr const char * kModelName = "Kokoro TTS"; } // namespace -struct KokoroTTSSession::PreparedRuntime { - std::unique_ptr predictor; -}; - KokoroTTSSession::KokoroTTSSession( runtime::TaskSpec task, runtime::SessionOptions options, - std::shared_ptr assets) + std::shared_ptr assets, + std::shared_ptr contract) : RuntimeSessionBase(options), task_(std::move(task)), - assets_(std::move(assets)) { + assets_(std::move(assets)), + contract_(std::move(contract)) { if (!assets_ || !assets_->model_weights) { throw std::runtime_error("Kokoro TTS session requires loaded assets"); } - matmul_weight_storage_type_ = parse_storage_option( - RuntimeSessionBase::options(), - {"kokoro_tts.weight_type", "kokoro.weight_type"}, - matmul_weight_storage_type_); - conv_weight_storage_type_ = parse_storage_option( - RuntimeSessionBase::options(), - {"kokoro_tts.conv_weight_type", "kokoro.conv_weight_type"}, - conv_weight_storage_type_); - matmul_weight_storage_type_ = parse_storage_option( - RuntimeSessionBase::options(), - {"kokoro_tts.matmul_weight_type", "kokoro.matmul_weight_type"}, - matmul_weight_storage_type_); - validate_matmul_storage(matmul_weight_storage_type_, "kokoro_tts.weight_type"); - validate_conv_storage(conv_weight_storage_type_, "kokoro_tts.conv_weight_type"); - weight_context_bytes_ = parse_size_mb_option( - RuntimeSessionBase::options(), - {"kokoro_tts.weight_context_mb", "kokoro_weight_context_mb"}, + if (contract_ == nullptr) { + throw std::runtime_error("Kokoro TTS session requires a model contract"); + } + runtime::validate_spec_backed_session_options(RuntimeSessionBase::options(), *contract_, kFamily, kModelName); + if (task_.task != runtime::VoiceTaskKind::Tts) { + throw std::runtime_error("Kokoro TTS only supports tts tasks"); + } + if (task_.mode != runtime::RunMode::Offline) { + throw std::runtime_error("Kokoro TTS only supports offline sessions"); + } + const auto & session_options = RuntimeSessionBase::options().options; + using T = engine::assets::TensorStorageType; + matmul_weight_storage_type_ = runtime::parse_tensor_storage_option( + session_options, + "kokoro_tts.weight_type", + matmul_weight_storage_type_, + {T::Native, T::F32, T::F16, T::BF16, T::Q8_0}); + conv_weight_storage_type_ = runtime::parse_tensor_storage_option( + session_options, + "kokoro_tts.conv_weight_type", + conv_weight_storage_type_, + {T::Native, T::F32, T::F16}); + weight_context_bytes_ = runtime::parse_size_mb_option( + session_options, + {"kokoro_tts.weight_context_mb"}, weight_context_bytes_); - predictor_duration_graph_bytes_ = parse_size_mb_option( - RuntimeSessionBase::options(), - {"kokoro_tts.predictor_duration_graph_mb", "kokoro_predictor_duration_graph_mb"}, + predictor_duration_graph_bytes_ = runtime::parse_size_mb_option( + session_options, + {"kokoro_tts.predictor_duration_graph_mb"}, predictor_duration_graph_bytes_); - predictor_text_graph_bytes_ = parse_size_mb_option( - RuntimeSessionBase::options(), - {"kokoro_tts.predictor_text_graph_mb", "kokoro_predictor_text_graph_mb"}, + predictor_text_graph_bytes_ = runtime::parse_size_mb_option( + session_options, + {"kokoro_tts.predictor_text_graph_mb"}, predictor_text_graph_bytes_); - predictor_tail_graph_bytes_ = parse_size_mb_option( - RuntimeSessionBase::options(), - {"kokoro_tts.predictor_tail_graph_mb", "kokoro_predictor_tail_graph_mb"}, + predictor_tail_graph_bytes_ = runtime::parse_size_mb_option( + session_options, + {"kokoro_tts.predictor_tail_graph_mb"}, predictor_tail_graph_bytes_); weights_ = load_kokoro_backend_weights( *assets_, @@ -167,23 +85,22 @@ KokoroTTSSession::KokoroTTSSession( const auto graph_capacity_mode = runtime::resolve_graph_capacity_mode( RuntimeSessionBase::options(), runtime::GraphCapacityMode::Fixed, - {"offline_graph_capacity_mode", "graph_capacity_mode"}); + {"kokoro_tts.graph_capacity_mode"}); if (graph_capacity_mode == runtime::GraphCapacityMode::Unsupported) { throw std::runtime_error("Kokoro TTS graph_capacity_mode=unsupported is not implemented"); } graph_capacity_controller_ = runtime::GraphCapacityController(graph_capacity_mode); - fixed_token_capacity_ = parse_positive_i64_option( - RuntimeSessionBase::options(), - {"max_input_tokens", "offline_max_input_tokens", "kokoro_max_input_tokens"}, + fixed_token_capacity_ = runtime::parse_positive_i64_option( + session_options, + {"kokoro_tts.max_input_tokens"}, std::min(512, weights_->context_length)); - pre_tail_token_capacity_ = parse_positive_i64_option( - RuntimeSessionBase::options(), - {"kokoro_pretail_tokens", "pre_tail_tokens"}, - 0); - rng_seed_ = parse_u64_option( - RuntimeSessionBase::options(), - {"kokoro_rng_seed", "rng_seed"}, - runtime::random_u64_seed()); + pre_tail_token_capacity_ = runtime::parse_i64_option( + session_options, + {"kokoro_tts.pre_tail_tokens"}).value_or(0); + if (pre_tail_token_capacity_ < 0) { + throw std::runtime_error("kokoro_tts.pre_tail_tokens must be non-negative"); + } + rng_seed_ = runtime::random_u64_seed(); if (fixed_token_capacity_ > weights_->context_length) { throw std::runtime_error("Kokoro fixed token capacity exceeds model context length"); } @@ -195,7 +112,7 @@ KokoroTTSSession::KokoroTTSSession( KokoroTTSSession::~KokoroTTSSession() = default; std::string KokoroTTSSession::family() const { - return "kokoro_tts"; + return kFamily; } runtime::VoiceTaskKind KokoroTTSSession::task_kind() const { @@ -206,14 +123,10 @@ runtime::RunMode KokoroTTSSession::run_mode() const { return task_.mode; } -int64_t KokoroTTSSession::base_graph_capacity_tokens() const { - return fixed_token_capacity_; -} - runtime::MappedGraphCapacityAdapter KokoroTTSSession::make_graph_capacity_adapter() { return runtime::MappedGraphCapacityAdapter( - base_graph_capacity_tokens(), - base_graph_capacity_tokens(), + fixed_token_capacity_, + fixed_token_capacity_, [this](int64_t request_size) { if (request_size <= 0) { throw std::runtime_error("Kokoro graph capacity request size must be positive"); @@ -229,7 +142,7 @@ runtime::MappedGraphCapacityAdapter KokoroTTSSession::make_graph_capacity_adapte std::vector KokoroTTSSession::prepared_graph_capacities() const { std::vector capacities; - if (prepared_session_ && prepared_session_->predictor && prepared_session_capacity_ > 0) { + if (prepared_predictor_ && prepared_session_capacity_ > 0) { capacities.push_back(prepared_session_capacity_); } return capacities; @@ -259,7 +172,7 @@ void KokoroTTSSession::prepare_graph_capacity(int64_t capacity) { if (capacity > weights_->context_length) { throw std::runtime_error("Kokoro graph capacity exceeds model context length"); } - if (prepared_session_ && prepared_session_capacity_ >= capacity) { + if (prepared_predictor_ && prepared_session_capacity_ >= capacity) { return; } const int threads = std::max(1, execution_context().config().threads); @@ -267,16 +180,15 @@ void KokoroTTSSession::prepare_graph_capacity(int64_t capacity) { ggml_backend_t backend = execution_context().backend(); const int64_t plbert_fixed_token_capacity = 0; const int64_t predictor_pre_tail_capacity = pre_tail_token_capacity_; - prepared_session_.reset(); + prepared_predictor_.reset(); prepared_session_capacity_ = 0; - auto prepared = std::make_unique(); double build_ms = 0.0; kokoro_ggml::KokoroPredictorGraphConfig predictor_graph_config; predictor_graph_config.duration_graph_bytes = predictor_duration_graph_bytes_; predictor_graph_config.text_graph_bytes = predictor_text_graph_bytes_; predictor_graph_config.tail_graph_bytes = predictor_tail_graph_bytes_; build_ms = measure_ms([&]() { - prepared->predictor = std::make_unique( + prepared_predictor_ = std::make_unique( weights_, backend, threads, @@ -285,7 +197,6 @@ void KokoroTTSSession::prepare_graph_capacity(int64_t capacity) { predictor_pre_tail_capacity, predictor_graph_config); }); - prepared_session_ = std::move(prepared); prepared_session_capacity_ = capacity; engine::debug::timing_log_scalar("kokoro.prepare.predictor.graph.build_ms", build_ms); } @@ -326,13 +237,8 @@ void KokoroTTSSession::prepare_decoder_graph_capacity(int64_t capacity) { } void KokoroTTSSession::prepare(const runtime::SessionPreparationRequest & request) { - if (task_.task != runtime::VoiceTaskKind::Tts) { - throw std::runtime_error("Kokoro TTS session only supports VoiceTaskKind::Tts"); - } - if (task_.mode != runtime::RunMode::Offline) { - throw std::runtime_error("Kokoro TTS session only supports offline mode"); - } - if (const auto seed = runtime::parse_u64_option(request.options, {"seed", "kokoro_rng_seed", "rng_seed"})) { + runtime::validate_spec_backed_request_options(request.options, *contract_, kModelName); + if (const auto seed = runtime::parse_u64_option(request.options, {"seed"})) { if (rng_seed_ != *seed) { rng_seed_ = *seed; prepared_decoder_.reset(); @@ -340,21 +246,6 @@ void KokoroTTSSession::prepare(const runtime::SessionPreparationRequest & reques prepared_decoder_context_ = {}; } } - if (!frontend_session_state_) { - frontend_session_state_ = - std::make_unique( - resolve_kokoro_frontend_session_state(request.text, request.voice, *assets_)); - } else if (request.text.has_value() || request.voice.has_value()) { - runtime::Transcript transcript; - if (request.text.has_value()) { - transcript = *request.text; - } - validate_kokoro_frontend_session_state( - transcript, - request.voice, - *frontend_session_state_, - *assets_); - } auto adapter = make_graph_capacity_adapter(); int64_t request_size = 0; if (request.text.has_value()) { @@ -364,9 +255,11 @@ void KokoroTTSSession::prepare(const runtime::SessionPreparationRequest & reques for (const auto & chunk : text_chunks) { runtime::SessionPreparationRequest chunk_request = request; chunk_request.text = runtime::Transcript{chunk, request.text->language}; + const auto frontend_state = + resolve_kokoro_frontend_session_state(chunk_request.text, chunk_request.voice, *assets_); request_size = std::max( request_size, - estimate_kokoro_request_tokens(chunk_request, *frontend_session_state_, *assets_)); + estimate_kokoro_request_tokens(chunk_request, frontend_state, *assets_)); } } graph_capacity_controller_.ensure_prepared(adapter, request_size); @@ -375,15 +268,10 @@ void KokoroTTSSession::prepare(const runtime::SessionPreparationRequest & reques runtime::TaskResult KokoroTTSSession::run(const runtime::TaskRequest & request) { require_prepared("Kokoro TTS run()"); - if (task_.task != runtime::VoiceTaskKind::Tts) { - throw std::runtime_error("Kokoro TTS session only supports VoiceTaskKind::Tts"); - } - if (task_.mode != runtime::RunMode::Offline) { - throw std::runtime_error("Kokoro TTS session only supports offline mode"); - } if (!request.text_input.has_value()) { throw std::runtime_error("Kokoro TTS run requires text_input"); } + runtime::validate_spec_backed_request_options(request.options, *contract_, kModelName); const int64_t text_chunk_size = engine::text::parse_text_chunk_size_override(request.options).value_or(kDefaultTextChunkSize); @@ -396,75 +284,70 @@ runtime::TaskResult KokoroTTSSession::run(const runtime::TaskRequest & request) double decoder_ms = 0.0; runtime::AudioBuffer merged_audio; for (const auto & chunk_request : chunk_requests) { - if (!frontend_session_state_) { - frontend_session_state_ = - std::make_unique( - resolve_kokoro_frontend_session_state(chunk_request.text_input, chunk_request.voice, *assets_)); - } - validate_kokoro_frontend_session_state( - *chunk_request.text_input, - chunk_request.voice, - *frontend_session_state_, - *assets_); - const std::string cache_key = request_cache_key(*chunk_request.text_input); - KokoroSynthesisInput input; - frontend_ms += measure_ms([&]() { - if (!cache_key.empty() && cached_input_ && cache_key == cached_request_key_) { - input = *cached_input_; - return; - } - input = build_kokoro_synthesis_input(*chunk_request.text_input, *frontend_session_state_, *assets_); - if (!cache_key.empty()) { - cached_request_key_ = cache_key; - cached_input_ = std::make_unique(input); - } - }); + const auto frontend_state = + resolve_kokoro_frontend_session_state(chunk_request.text_input, chunk_request.voice, *assets_); + const std::string cache_key = + frontend_state.voice_id + ":" + + frontend_state.language_code + ":" + + std::to_string(frontend_state.speaking_rate) + ":" + + std::to_string(chunk_request.text_input->text.size()) + ":" + + chunk_request.text_input->text; + KokoroSynthesisInput input; + frontend_ms += measure_ms([&]() { + if (!cache_key.empty() && cached_input_ && cache_key == cached_request_key_) { + input = *cached_input_; + return; + } + input = build_kokoro_synthesis_input(*chunk_request.text_input, frontend_state, *assets_); + if (!cache_key.empty()) { + cached_request_key_ = cache_key; + cached_input_ = std::make_unique(input); + } + }); - const auto inference_started = std::chrono::steady_clock::now(); - auto adapter = make_graph_capacity_adapter(); - const int64_t request_size = static_cast(input.input_ids.size()); - graph_capacity_controller_.ensure_prepared(adapter, request_size); - const int64_t selected_capacity = graph_capacity_controller_.select_capacity_for_run(adapter, request_size); - if (!prepared_session_ || - prepared_session_capacity_ != selected_capacity || - !prepared_session_->predictor) { - throw std::runtime_error("Kokoro selected graph capacity was not prepared"); - } + const auto inference_started = std::chrono::steady_clock::now(); + auto adapter = make_graph_capacity_adapter(); + const int64_t request_size = static_cast(input.input_ids.size()); + graph_capacity_controller_.ensure_prepared(adapter, request_size); + const int64_t selected_capacity = graph_capacity_controller_.select_capacity_for_run(adapter, request_size); + if (!prepared_predictor_ || prepared_session_capacity_ != selected_capacity) { + throw std::runtime_error("Kokoro selected graph capacity was not prepared"); + } - kokoro_ggml::PredictorOutputs predictor; - predictor_ms += measure_ms([&]() { - predictor = prepared_session_->predictor->predict( - input.input_ids, - input.style, - input.speaking_rate); - }); - const int64_t decoder_request_size = predictor.decoder_x_cols; - if (predictor.decoder_x_on_backend && predictor.decoder_x_tensor == nullptr) { - throw std::runtime_error("Kokoro predictor reported backend decoder features without a tensor"); - } - if (decoder_request_size <= 0) { - throw std::runtime_error("Kokoro predictor produced invalid decoder request size"); - } - if (static_cast(predictor.f0_curve.size()) != decoder_request_size) { - throw std::runtime_error("Kokoro predictor decoder and f0 frame counts diverged"); - } - if (!prepared_decoder_ || prepared_decoder_capacity_ != decoder_request_size) { - prepare_decoder_graph_capacity(decoder_request_size); - } - if (!prepared_decoder_ || - prepared_decoder_context_.decoder_frame_capacity <= 0 || - decoder_request_size != prepared_decoder_context_.decoder_frame_capacity) { - throw std::runtime_error("Kokoro decoder runtime was not prepared"); - } - std::vector audio; - decoder_ms += measure_ms([&]() { - audio = prepared_decoder_->decode( - predictor, - input.style); - }); - const auto inference_ended = std::chrono::steady_clock::now(); - inference_ms += std::chrono::duration(inference_ended - inference_started).count(); - runtime::append_audio_buffer(merged_audio, runtime::AudioBuffer{24000, 1, std::move(audio)}); + kokoro_ggml::PredictorOutputs predictor; + predictor_ms += measure_ms([&]() { + predictor = prepared_predictor_->predict( + input.input_ids, + input.style, + input.speaking_rate); + }); + const int64_t decoder_request_size = predictor.decoder_x_cols; + if (predictor.decoder_x_on_backend && predictor.decoder_x_tensor == nullptr) { + throw std::runtime_error("Kokoro predictor reported backend decoder features without a tensor"); + } + if (decoder_request_size <= 0) { + throw std::runtime_error("Kokoro predictor produced invalid decoder request size"); + } + if (static_cast(predictor.f0_curve.size()) != decoder_request_size) { + throw std::runtime_error("Kokoro predictor decoder and f0 frame counts diverged"); + } + if (!prepared_decoder_ || prepared_decoder_capacity_ != decoder_request_size) { + prepare_decoder_graph_capacity(decoder_request_size); + } + if (!prepared_decoder_ || + prepared_decoder_context_.decoder_frame_capacity <= 0 || + decoder_request_size != prepared_decoder_context_.decoder_frame_capacity) { + throw std::runtime_error("Kokoro decoder runtime was not prepared"); + } + std::vector audio; + decoder_ms += measure_ms([&]() { + audio = prepared_decoder_->decode( + predictor, + input.style); + }); + const auto inference_ended = std::chrono::steady_clock::now(); + inference_ms += std::chrono::duration(inference_ended - inference_started).count(); + runtime::append_audio_buffer(merged_audio, runtime::AudioBuffer{24000, 1, std::move(audio)}); } runtime::TaskResult result; @@ -478,4 +361,20 @@ runtime::TaskResult KokoroTTSSession::run(const runtime::TaskRequest & request) return result; } +std::shared_ptr make_kokoro_tts_loader() { + runtime::SpecBackedVoiceModelConfig config; + config.family = kFamily; + config.load_assets = [](const std::filesystem::path & model_path) { + return load_kokoro_assets(model_path); + }; + config.create_session = []( + const runtime::TaskSpec & task, + const runtime::SessionOptions & options, + std::shared_ptr assets, + std::shared_ptr contract) { + return std::make_unique(task, options, std::move(assets), std::move(contract)); + }; + return runtime::make_spec_backed_voice_loader(std::move(config)); +} + } // namespace engine::models::kokoro_tts diff --git a/tests/kokoro_tts/MULTILINGUAL_GGUF.md b/tests/kokoro_tts/MULTILINGUAL_GGUF.md index 91901683d..d535ef2fc 100644 --- a/tests/kokoro_tts/MULTILINGUAL_GGUF.md +++ b/tests/kokoro_tts/MULTILINGUAL_GGUF.md @@ -5,9 +5,11 @@ GGUF produced by `tools/prepare_kokoro_gguf.py`. Specify `--family kokoro_tts` f GGUF loading. This is a Kokoro-specific container; arbitrary third-party Kokoro GGUF layouts are not supported. -Each package includes all 54 source voices, configuration, English pronunciation -resources, eSpeak data, the full UniDic dictionary, and Chinese pronunciation and -segmentation dictionaries. Executable libraries are not embedded in the model. +Release packages include all 54 source voices, configuration, vocabulary, +weights, and the generated `g2p/ja.json` / `g2p/zh.json` tables. Heavy frontend +runtime resources such as eSpeak data and UniDic are not included by default to +keep the GGUFs small. Users who want a fully bundled local package can pass +`--embed-multilingual-resources` during conversion. | Language | CLI language | Example voice | | --- | --- | --- | @@ -23,9 +25,10 @@ segmentation dictionaries. Executable libraries are not embedded in the model. ## Runtime dependencies -English retains the existing native frontend. Spanish, French, Hindi, Italian, -and Portuguese use the native eSpeak library. Japanese uses native MeCab with -embedded UniDic data. Chinese uses native dictionary/DAG/HMM processing. +English, Spanish, French, Hindi, Italian, and Portuguese use the shared native +eSpeak library and runtime eSpeak data. Chinese uses the bundled `g2p/zh.json` +dictionary/DAG/HMM data. Japanese uses the bundled `g2p/ja.json` plus native +MeCab and UniDic data when a fully bundled package is exported. Python is used only for conversion and upstream comparison, never inference. Kokoro uses the shared `engine::audio::EspeakPhonemizer` introduced in PR #502. @@ -46,32 +49,39 @@ explicit library and retains the existing model-local eSpeak data default. `AUDIOCPP_ESPEAK_DATA` overrides the data location with a directory or a `.bin`/`.gguf` data package in either build mode. -Japanese still requires `libmecab.dll` on Windows or an installed MeCab library. -`AUDIOCPP_MECAB_LIBRARY` selects its absolute path. -Embedded data is extracted to a temporary directory for the loaded model's -lifetime and removed when its assets are released. +Japanese still requires `libmecab.dll` on Windows or an installed MeCab library; +`AUDIOCPP_MECAB_LIBRARY` selects its absolute path. The small release GGUF does +not include UniDic. Export a local GGUF with `--embed-multilingual-resources` +for the full multilingual resource bundle. Bundled data is extracted to a +temporary directory for the loaded model's lifetime and removed when its assets +are released. ## Conversion -Use a preparation environment containing numpy, safetensors, gguf (tested with -0.19), misaki[ja,zh] (tested with 0.9.4), espeakng-loader, unidic, and Jieba. -Run `python -m unidic download` first. Start from the original extracted Kokoro -directory containing weights, config, all voices, and English resources. +Use a preparation environment containing numpy, gguf, and torch. Start from the +official hexgrad/Kokoro-82M source checkout containing `config.json`, +`kokoro-v1_0.pth`, and `voices/*.pt`. ```powershell -python tools/prepare_kokoro_gguf.py --source ../models_v3_test/kokoro-82m-v1_0-ggml --resources ../models_v3_test/Kokoro-multilingual-resources --output-dir ../models_v3_test/Kokoro-GGUF +python tools/prepare_kokoro_gguf.py --source ../models_v3_test/Kokoro-82M --output-dir ../models_v3_test/Kokoro-GGUF ``` -Outputs are `kokoro-v1.0-q8_0.gguf` and `kokoro-v1.0-bf16.gguf`. Q8 quantizes +Outputs are `kokoro-82m-q8_0.gguf` and `kokoro-82m-bf16.gguf`. Q8 quantizes eligible matrices; unsupported weight layouts remain BF16, while sensitive small tensors remain F32 in both packages. Q8 does not mean every tensor is Q8. -The full embedded dictionaries dominate file size: approximately 943 MB for Q8 -and 965 MB for BF16. Language resources are identical in both. + +To build a large fully bundled multilingual package for local redistribution or +offline deployment, also install misaki[ja,zh] (tested with 0.9.4), +espeakng-loader, unidic, and Jieba, then run `python -m unidic download` first: + +```powershell +python tools/prepare_kokoro_gguf.py --source ../models_v3_test/Kokoro-82M --output-dir ../models_v3_test/Kokoro-GGUF --embed-multilingual-resources +``` ## Synthesis ```powershell -.\build\windows-cpu-release\bin\audiocpp_cli.exe --task tts --family kokoro_tts --model ..\models_v3_test\Kokoro-GGUF\kokoro-v1.0-q8_0.gguf --backend cpu --threads 8 --language en-us --voice-id af_heart --text "Hello, this is a native Kokoro TTS test." --out kokoro-q8.wav +.\build\windows-cpu-release\bin\audiocpp_cli.exe --task tts --family kokoro_tts --model ..\models_v3_test\Kokoro-GGUF\kokoro-82m-q8_0.gguf --backend cpu --threads 8 --language en-us --voice-id af_heart --text "Hello, this is a native Kokoro TTS test." --out kokoro-q8.wav ``` For non-Latin text on Windows, use a UTF-8 text file with @@ -90,9 +100,11 @@ shared dynamic and static eSpeak modes. All 12 multilingual pronunciation cases match exactly in both modes on Windows, including punctuation and number cases. The script exits with failure on any mismatch or frontend error. -`compare_multilingual_g2p.py` compares `kokoro_g2p_probe` against installed Misaki. -`validate_multilingual_packages.py` synthesizes each of the nine language variants -with both precisions and saves WAV files, command logs, and `validation.json`. +`compare_multilingual_g2p.py` compares `kokoro_g2p_probe` against installed +eSpeak and Misaki-compatible Japanese/Chinese frontends. +`validate_multilingual_packages.py` synthesizes each of the nine language +variants with both precisions from fully bundled packages and saves WAV files, +command logs, and `validation.json`. Initial Windows CPU results: 12/12 pronunciation cases matched upstream exactly; 18/18 synthesis cases produced non-silent 24 kHz audio. These are smoke tests, diff --git a/tests/kokoro_tts/compare_multilingual_g2p.py b/tests/kokoro_tts/compare_multilingual_g2p.py index 629004d1f..7b2bbed6a 100644 --- a/tests/kokoro_tts/compare_multilingual_g2p.py +++ b/tests/kokoro_tts/compare_multilingual_g2p.py @@ -1,4 +1,4 @@ -"""Compare native pronunciation with the upstream Misaki frontends.""" +"""Compare native pronunciation with upstream eSpeak and Misaki frontends.""" import argparse import json import os diff --git a/tests/kokoro_tts/validate_multilingual_packages.py b/tests/kokoro_tts/validate_multilingual_packages.py index 40b13b034..d35ccfc34 100644 --- a/tests/kokoro_tts/validate_multilingual_packages.py +++ b/tests/kokoro_tts/validate_multilingual_packages.py @@ -22,7 +22,7 @@ a.output.mkdir(parents=True, exist_ok=True) report = [] for precision in a.types: - model = a.models.resolve() / ('kokoro-v1.0-' + precision + '.gguf') + model = a.models.resolve() / ('kokoro-82m-' + precision + '.gguf') for case in cases: lang = codes.get(case['language'], case['language']) dest = a.output.resolve() / (precision + '-' + lang) @@ -33,7 +33,8 @@ command = [str(a.cli.resolve()), '--task', 'tts', '--family', 'kokoro_tts', '--model', str(model), '--backend', 'cpu', '--threads', '8', '--seed', '1234', '--voice-id', case['voice'], '--language', lang, '--batch-text-file', str(textfile), - '--batch-merge-audio', 'concat', '--out', str(wav)] + '--batch-merge-audio', 'concat', '--out', str(wav), + '--log', '--log-file', str(dest / 'inference.log')] start = time.perf_counter() run = subprocess.run(command, capture_output=True, encoding='utf-8', errors='replace', timeout=180) elapsed = time.perf_counter() - start diff --git a/tools/model_manager_deprecated.py b/tools/model_manager_deprecated.py index c4d431cf1..5187cda7c 100644 --- a/tools/model_manager_deprecated.py +++ b/tools/model_manager_deprecated.py @@ -249,10 +249,6 @@ def package_usage_examples(package: ModelPackage) -> list[str]: "config.json", "kokoro-v1_0.safetensors", "voices.json", - "misaki_en/gb_gold.tsv", - "misaki_en/gb_silver.tsv", - "misaki_en/us_gold.tsv", - "misaki_en/us_silver.tsv", "voices/af_heart.f32", ), ), diff --git a/tools/prepare_kokoro_gguf.py b/tools/prepare_kokoro_gguf.py index cb341fd2d..90ec05e5c 100644 --- a/tools/prepare_kokoro_gguf.py +++ b/tools/prepare_kokoro_gguf.py @@ -1,17 +1,16 @@ -"""Create standalone Kokoro GGUFs, preserving every voice and pronunciation resource. +"""Create Kokoro GGUFs from the official hexgrad/Kokoro-82M source checkout. -Conversion dependencies: numpy, safetensors, gguf, misaki[ja,zh], espeakng-loader, -unidic (run `python -m unidic download` once). Python is not used for inference. +The default GGUF contains model weights, config, vocab, and voice packs only. +Use --embed-multilingual-resources to build a larger fully bundled package. +Python is not used for inference. """ import argparse import importlib.metadata import json import struct -import shutil from pathlib import Path import numpy as np import gguf -from safetensors import safe_open class ResourceWriter(gguf.GGUFWriter): def _pack_val(self, val, vtype, add_vtype, sub_type=None): @@ -21,29 +20,84 @@ def _pack_val(self, val, vtype, add_vtype, sub_type=None): return super()._pack_val(val, vtype, add_vtype, sub_type) -def dump(path, value): - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(json.dumps(value, ensure_ascii=False, separators=(',', ':')), encoding='utf-8') +def flatten_torch_state(prefix, value, out): + import torch + if isinstance(value, dict): + for key, item in value.items(): + key = str(key) + if key == 'module' and prefix: + name = prefix + else: + if prefix and key.startswith('module.'): + key = key[len('module.'):] + name = key if not prefix else prefix + '.' + key + flatten_torch_state(name, item, out) + return + if not torch.is_tensor(value): + raise RuntimeError(f'unsupported Kokoro checkpoint value at {prefix}: {type(value)!r}') + out[prefix] = value.detach().cpu().contiguous() -def prepare(source, root): - import espeakng_loader - import unidic +def load_official_weights(source): + checkpoint = source / 'kokoro-v1_0.pth' + if not checkpoint.is_file(): + raise RuntimeError('official Kokoro source must contain kokoro-v1_0.pth') + import torch + state = torch.load(checkpoint, map_location='cpu') + flat = {} + flatten_torch_state('', state, flat) + return flat + + +def make_vocab_tsv(config): + vocab = config.get('vocab') + if not isinstance(vocab, dict): + raise RuntimeError('Kokoro config.json is missing vocab object') + lines = [f'{symbol}\t{int(index)}\n' for symbol, index in sorted(vocab.items(), key=lambda item: int(item[1]))] + return ''.join(lines).encode('utf-8') + + +def add_file(files, name, data): + files[name] = data if isinstance(data, bytes) else data.encode('utf-8') + + +def add_tree(files, root, prefix): + for path in sorted(root.rglob('*')): + if path.is_file(): + files[prefix + '/' + path.relative_to(root).as_posix()] = path.read_bytes() + + +def load_official_voices(source): + voice_files = sorted((source / 'voices').glob('*.pt')) + if not voice_files: + raise RuntimeError('official Kokoro source must contain voices/*.pt') + import torch + voices = {} + files = {} + for path in voice_files: + tensor = torch.load(path, map_location='cpu') + if not torch.is_tensor(tensor): + raise RuntimeError(f'Kokoro voice file is not a tensor: {path}') + array = tensor.detach().cpu().to(torch.float32).numpy() + if array.ndim == 3 and array.shape[1] == 1: + array = array[:, 0, :] + if array.ndim != 2: + raise RuntimeError(f'Kokoro voice tensor must be rank 2 or [rows,1,cols]: {path}') + array = np.ascontiguousarray(array, dtype=np.float32) + out_name = path.stem + '.bin' + files['voices/' + out_name] = array.tobytes() + voices[path.stem] = {'rows': int(array.shape[0]), 'cols': int(array.shape[1]), 'path': out_name} + return voices, files + + +def load_g2p_tables(): import jieba from pypinyin import pinyin_dict, phrases_dict, Style, lazy_pinyin from misaki.zh import ZHG2P from misaki.cutlet import HEPBURN, JA_WORDS - if not (Path(unidic.DICDIR) / 'sys.dic').is_file(): - raise RuntimeError('Download the Japanese dictionary with: python -m unidic download') - shutil.copytree(source, root, dirs_exist_ok=True, - ignore=shutil.ignore_patterns('.cache', '.gitattributes', '*.gguf')) - shutil.copytree(espeakng_loader.get_data_path(), root / 'espeak-ng-data', dirs_exist_ok=True) - dic = root / 'unidic' - dic.mkdir(exist_ok=True) - for name in ['sys.dic', 'unk.dic', 'char.bin', 'matrix.bin', 'dicrc']: - shutil.copy2(Path(unidic.DICDIR) / name, dic / name) - dump(root / 'g2p' / 'ja.json', {'kana': HEPBURN, 'words': sorted(JA_WORDS)}) - # Export the complete pypinyin dictionaries and Jieba DAG/HMM data, not sample phrases. + files = {} + add_file(files, 'g2p/ja.json', json.dumps({'kana': HEPBURN, 'words': sorted(JA_WORDS)}, + ensure_ascii=False, separators=(',', ':'))) from jieba.finalseg import start_P, trans_P, emit_P jieba.initialize() syllables = set() @@ -63,37 +117,82 @@ def prepare(source, root): ipa[value] = ZHG2P.py2ipa(value).replace('\u032f', '') except (ValueError, IndexError, KeyError, AssertionError): continue - dump(root / 'g2p' / 'zh.json', {'chars': chars, 'phrases': phrases, 'ipa': ipa, + add_file(files, 'g2p/zh.json', json.dumps({'chars': chars, 'phrases': phrases, 'ipa': ipa, 'frequency': {k: v for k, v in jieba.dt.FREQ.items() if v > 0}, 'total': jieba.dt.total, - 'start': start_P, 'transition': trans_P, 'emission': emit_P}) - # Keep redistribution notices with the resources. - notices = root / 'licenses' - notices.mkdir(exist_ok=True) - shutil.copytree(Path(unidic.DICDIR) / 'licenses', notices / 'unidic-dictionary', dirs_exist_ok=True) - shutil.copy2(Path(unidic.DICDIR) / 'README', notices / 'unidic-dictionary-README') - for dist_name in ['misaki', 'unidic', 'espeakng-loader', 'jieba', 'pypinyin', 'mecab-python3']: + 'start': start_P, 'transition': trans_P, 'emission': emit_P}, + ensure_ascii=False, separators=(',', ':'))) + return files + + +def load_multilingual_resources(): + import espeakng_loader + import unidic + if not (Path(unidic.DICDIR) / 'sys.dic').is_file(): + raise RuntimeError('Download the Japanese dictionary with: python -m unidic download') + files = {} + add_tree(files, Path(espeakng_loader.get_data_path()), 'espeak-ng-data') + for name in ['sys.dic', 'unk.dic', 'char.bin', 'matrix.bin', 'dicrc']: + files['unidic/' + name] = (Path(unidic.DICDIR) / name).read_bytes() + files.update(load_g2p_tables()) + notices = {} + add_tree(notices, Path(unidic.DICDIR) / 'licenses', 'licenses/unidic-dictionary') + readme = Path(unidic.DICDIR) / 'README' + if readme.is_file(): + notices['licenses/unidic-dictionary-README'] = readme.read_bytes() + for dist_name in ['misaki', 'unidic', 'espeakng-loader', 'jieba', 'pypinyin']: dist = importlib.metadata.distribution(dist_name) for file in dist.files or []: if any(x in file.name.lower() for x in ['license', 'copying', 'notice']): src = Path(dist.locate_file(file)) - if src.is_file(): shutil.copy2(src, notices / (dist_name + '-' + file.name)) - dump(root / 'g2p' / 'versions.json', {n: importlib.metadata.version(n) - for n in ['misaki', 'unidic', 'espeakng-loader', 'jieba', 'pypinyin']}) + if src.is_file(): + notices['licenses/' + dist_name + '-' + file.name] = src.read_bytes() + files.update(notices) + add_file(files, 'g2p/versions.json', json.dumps({n: importlib.metadata.version(n) + for n in ['misaki', 'unidic', 'espeakng-loader', 'jieba', 'pypinyin']}, + ensure_ascii=False, separators=(',', ':'))) + return files + + +def load_official_source(source, embed_multilingual_resources): + config_path = source / 'config.json' + if not config_path.is_file(): + raise RuntimeError('official Kokoro source must contain config.json') + config_bytes = config_path.read_bytes() + config = json.loads(config_bytes.decode('utf-8')) + voices, voice_files = load_official_voices(source) + files = { + 'config.json': config_bytes, + 'voices.json': json.dumps(voices, ensure_ascii=False, separators=(',', ':')).encode('utf-8'), + 'vocab.tsv': make_vocab_tsv(config), + } + files.update(voice_files) + files.update(load_g2p_tables()) + if embed_multilingual_resources: + files.update(load_multilingual_resources()) + return config, voices, files, load_official_weights(source) -def convert(root, output, precision, overwrite=False): +def convert(source, output, precision, overwrite=False, model_spec=None, embed_multilingual_resources=False): if output.exists() and not overwrite: raise FileExistsError(output) + _, voices, files, weights = load_official_source(source, embed_multilingual_resources) output.parent.mkdir(parents=True, exist_ok=True) temporary = output.with_suffix('.gguf.partial') writer = ResourceWriter(str(temporary), 'kokoro_tts') writer.add_name('Kokoro v1.0 multilingual ' + precision) writer.add_array('kokoro.languages', ['en-us', 'en-gb', 'es', 'fr-fr', 'hi', 'it', 'ja', 'pt-br', 'zh']) - writer.add_array('kokoro.voices', sorted(json.loads((root / 'voices.json').read_text()))) + writer.add_array('kokoro.voices', sorted(voices)) + if model_spec is not None: + spec_text = model_spec.read_text(encoding='utf-8') + spec_json = json.loads(spec_text) + if spec_json.get('family') != 'kokoro_tts' or spec_json.get('schema_version') != 1: + raise RuntimeError(f'not a Kokoro schema-v1 model spec: {model_spec}') + writer.add_uint32('audiocpp.model_spec.version', 1) + writer.add_string('audiocpp.model_spec.family', 'kokoro_tts') + writer.add_string('audiocpp.model_spec.json', json.dumps(spec_json, ensure_ascii=False, separators=(',', ':'))) names, offsets, data = [], [0], bytearray() - for file in sorted(root.rglob('*')): - if not file.is_file() or file.suffix in ['.safetensors', '.gguf']: continue - names.append(file.relative_to(root).as_posix()) - data.extend(file.read_bytes()) + for name in sorted(files): + names.append(name) + data.extend(files[name]) offsets.append(len(data)) writer.add_array('audiocpp.embedded_files.names', names) writer.add_key_value('audiocpp.embedded_files.offsets', offsets, gguf.GGUFValueType.ARRAY, @@ -101,22 +200,30 @@ def convert(root, output, precision, overwrite=False): writer.add_key_value('audiocpp.embedded_files.data', bytes(data), gguf.GGUFValueType.ARRAY, sub_type=gguf.GGUFValueType.UINT8) counts = {} - with safe_open(root / 'kokoro-v1_0.safetensors', framework='numpy') as model: - writer.add_array('kokoro.tensor_names', list(model.keys())) - for index, name in enumerate(model.keys()): - array = model.get_tensor(name).astype(np.float32) - writer.add_key_value('kokoro.tensor_shape.' + name, list(array.shape), - gguf.GGUFValueType.ARRAY, sub_type=gguf.GGUFValueType.INT64) - # Retain scalar/vector/norm and Snake parameters in F32. Weight-norm - # convolution tensors use BF16 on disk and are reconstructed in F32. - kind = gguf.GGMLQuantizationType.F32 - if array.ndim >= 2 and all(d > 1 for d in array.shape): - kind = gguf.GGMLQuantizationType.BF16 - if precision == 'q8_0' and array.ndim == 2 and array.shape[-1] % 32 == 0: - kind = gguf.GGMLQuantizationType.Q8_0 - encoded = gguf.quants.quantize(array, kind) - writer.add_tensor('kokoro.' + str(index), encoded, raw_dtype=kind) - counts[kind.name] = counts.get(kind.name, 0) + 1 + source_names = sorted(weights) + writer.add_array('kokoro.tensor_names', source_names) + writer.add_array('audiocpp.tensor_names', ['kokoro/' + name for name in source_names]) + ranks, shapes = [], [] + for index, name in enumerate(source_names): + array = weights[name].float().numpy() + ranks.append(len(array.shape)) + shapes.extend(int(dim) for dim in array.shape) + writer.add_key_value('kokoro.tensor_shape.' + name, list(array.shape), + gguf.GGUFValueType.ARRAY, sub_type=gguf.GGUFValueType.INT64) + # Retain scalar/vector/norm and Snake parameters in F32. Weight-norm + # convolution tensors use BF16 on disk and are reconstructed in F32. + kind = gguf.GGMLQuantizationType.F32 + if array.ndim >= 2 and all(d > 1 for d in array.shape): + kind = gguf.GGMLQuantizationType.BF16 + if precision == 'q8_0' and array.ndim == 2 and array.shape[-1] % 32 == 0: + kind = gguf.GGMLQuantizationType.Q8_0 + encoded = gguf.quants.quantize(np.ascontiguousarray(array, dtype=np.float32), kind) + writer.add_tensor('kokoro.' + str(index), encoded, raw_dtype=kind) + counts[kind.name] = counts.get(kind.name, 0) + 1 + writer.add_key_value('audiocpp.tensor_ranks', ranks, gguf.GGUFValueType.ARRAY, + sub_type=gguf.GGUFValueType.INT32) + writer.add_key_value('audiocpp.tensor_shapes', shapes, gguf.GGUFValueType.ARRAY, + sub_type=gguf.GGUFValueType.INT64) output.parent.mkdir(parents=True, exist_ok=True) writer.write_header_to_file() writer.write_kv_data_to_file() @@ -130,12 +237,13 @@ def convert(root, output, precision, overwrite=False): if __name__ == '__main__': p = argparse.ArgumentParser(description=__doc__) p.add_argument('--source', type=Path, required=True) - p.add_argument('--resources', type=Path, required=True) p.add_argument('--output-dir', type=Path, required=True) - p.add_argument('--skip-prepare', action='store_true') + p.add_argument('--model-spec', type=Path, + default=Path(__file__).resolve().parents[1] / 'model_specs' / 'kokoro_tts.json') + p.add_argument('--embed-multilingual-resources', action='store_true') p.add_argument('--overwrite', action='store_true') p.add_argument('--type', choices=['q8_0', 'bf16', 'both'], default='both') args = p.parse_args() - if not args.skip_prepare: prepare(args.source, args.resources) for precision in (['q8_0', 'bf16'] if args.type == 'both' else [args.type]): - convert(args.resources, args.output_dir / ('kokoro-v1.0-' + precision + '.gguf'), precision, args.overwrite) + convert(args.source, args.output_dir / ('kokoro-82m-' + precision + '.gguf'), + precision, args.overwrite, args.model_spec, args.embed_multilingual_resources) diff --git a/webui/configs/models_catalog.json b/webui/configs/models_catalog.json index 62b46c896..f3be949d8 100644 --- a/webui/configs/models_catalog.json +++ b/webui/configs/models_catalog.json @@ -11,6 +11,8 @@ { "id": "dots-tts-soar", "display_name": "DotTTS SOAR (tts + clone)", "family": "dots_tts", "path": "models/DotTTS-SOAR-GGUF", "task": "tts", "mode": "offline", "download_id": "dots_tts_soar_q8_0", "min_vram_gb": 8 }, { "id": "dots-tts-meanflow", "display_name": "DotTTS MeanFlow (tts + clone)", "family": "dots_tts", "path": "models/DotTTS-MF-GGUF", "task": "tts", "mode": "offline", "download_id": "dots_tts_mf_q8_0", "min_vram_gb": 8 }, { "id": "neutts-2e", "display_name": "NeuTTS 2E (tts, preset voices)", "family": "neutts", "path": "models/NeuTTS-2E-GGUF", "task": "tts", "mode": "offline", "download_id": "neutts_2e_orig", "min_vram_gb": 4 }, + { "id": "kokoro-tts", "display_name": "Kokoro 82M (tts, preset voices)", "family": "kokoro_tts", "path": "models/Kokoro-82M-GGUF/kokoro-82m-q8_0.gguf", "task": "tts", "mode": "offline", "download_id": "kokoro_82m_q8_0", "min_vram_gb": 1, + "input_hint_en": "**Kokoro 82M**: multilingual TTS with built-in preset voices. Choose a language and voice; no reference audio is needed." }, { "id": "qwen3-tts", "display_name": "Qwen3-TTS 0.6B (tts)", "family": "qwen3_tts", "path": "models/Qwen3-TTS-12Hz-0.6B-Base", "task": "tts", "mode": "offline", "download_id": "qwen3_tts_0_6b_base", "min_vram_gb": 5 }, { "id": "qwen3-tts-1.7b", "display_name": "Qwen3-TTS 1.7B Base (tts)", "family": "qwen3_tts", "path": "models/Qwen3-TTS-12Hz-1.7B-Base", "task": "tts", "mode": "offline", "download_id": "qwen3_tts_1_7b_base", "min_vram_gb": 8 }, { "id": "qwen3-tts-1.7b-custom", "display_name": "Qwen3-TTS 1.7B CustomVoice (tts)", "family": "qwen3_tts", "path": "models/Qwen3-TTS-12Hz-1.7B-CustomVoice", "task": "tts", "mode": "offline", "download_id": "qwen3_tts_1_7b_custom_voice", "min_vram_gb": 8 }, diff --git a/webui/native/dist/index.html b/webui/native/dist/index.html index 3fc30f041..eb6c023ec 100644 --- a/webui/native/dist/index.html +++ b/webui/native/dist/index.html @@ -31,20 +31,20 @@
diff --git a/webui/native/src/lib/catalog.ts b/webui/native/src/lib/catalog.ts index 491cc7bd7..894350e06 100644 --- a/webui/native/src/lib/catalog.ts +++ b/webui/native/src/lib/catalog.ts @@ -47,6 +47,7 @@ const exposeAllGgufPackageFamilies = new Set([ 'firered_audio', 'fireredtts3', 'irodori_tts', + 'kokoro_tts', 'meanvc2', 'midashenglm_gen', 'sanotts' diff --git a/webui/native/src/routes/+page.svelte b/webui/native/src/routes/+page.svelte index 800322c5a..d82fe100e 100644 --- a/webui/native/src/routes/+page.svelte +++ b/webui/native/src/routes/+page.svelte @@ -150,6 +150,7 @@ 'firered_audio', 'fireredtts3', 'irodori_tts', + 'kokoro_tts', 'meanvc2', 'midashenglm_gen' ]); @@ -414,6 +415,7 @@ (selected?.task === 's2s' && selected?.family === 'personaplex') || (selected?.task === 'tts' && !['supertonic'].includes(selected?.family) && !supportsTextOnlyTts); $: usesVibeVoiceSpeakerFiles = selected?.family === 'vibevoice'; + $: usesBuiltInVoiceSelector = Boolean(selected?.builtin_voices?.length); $: isQwenBase = selected?.task === 'tts' && selected?.family === 'qwen3_tts' && !selected?.id.includes('custom'); $: allowsQuickStartVoice = ['tts', 'clon'].includes(selected?.task); @@ -423,11 +425,16 @@ $: referenceTextRequired = requiresRequestOption(selected, 'reference_text') || (Boolean(voiceFile) && isQwenBase); $: quickStartVoices = server && !server.ui_management - ? configuredVoices + ? Array.from(new Set([ + ...configuredVoices, + ...(usesBuiltInVoiceSelector ? selected?.builtin_voices || [] : []) + ])) + : usesBuiltInVoiceSelector + ? selected?.builtin_voices || [] : Object.entries(demoVoiceSources) .filter(([, source]) => bundledVoices.includes(source)) .map(([voice]) => voice); - $: quickStartVoicePreview = quickStartVoice && server?.ui_management !== false + $: quickStartVoicePreview = quickStartVoice && server?.ui_management !== false && !usesBuiltInVoiceSelector ? voicePreviewUrl(demoVoiceSources[quickStartVoice] || quickStartVoice) : ''; $: showsText = ['tts', 'clon', 'gen', 's2s', 'align', 'vdes'].includes(selected?.task); @@ -1005,6 +1012,9 @@ if (!text.trim() && selected?.default_text) { text = selected.default_text; } + if (selected?.builtin_voices?.length && selected.default_voice && !quickStartVoice) { + quickStartVoice = selected.default_voice; + } advancedJson = '{}'; } @@ -1428,7 +1438,13 @@ } try { configuredVoices = await availableVoices(selectedId); - if (quickStartVoice && !configuredVoices.includes(quickStartVoice)) quickStartVoice = ''; + if (quickStartVoice) { + const allowed = new Set([ + ...configuredVoices, + ...(usesBuiltInVoiceSelector ? selected?.builtin_voices || [] : []) + ]); + if (!allowed.has(quickStartVoice)) quickStartVoice = ''; + } } catch (error) { configuredVoices = []; log(`Configured voices unavailable: ${error instanceof Error ? error.message : error}`); @@ -2315,49 +2331,58 @@
{tr('voice.bundledNote')}
- + {#if quickStartVoicePreview} + + {/if} {/if} {/if} -
-
- - chooseVoiceReference(event.currentTarget.files?.[0] || null)} /> - + {#if !usesBuiltInVoiceSelector} +
+
+ + chooseVoiceReference(event.currentTarget.files?.[0] || null)} /> + +
+
+ + chooseReferenceText(event.currentTarget.files?.[0] || null)} /> + +
-
- - chooseReferenceText(event.currentTarget.files?.[0] || null)} /> - + {/if} + {#if !usesBuiltInVoiceSelector} +
+ {#if recordingTarget === 'voice'} + + {tr('voice.recording')} + {:else} + + + {#if voiceFile}{voiceFile.name}{/if} + {/if}
-
-
- {#if recordingTarget === 'voice'} - - {tr('voice.recording')} - {:else} - - - {#if voiceFile}{voiceFile.name}{/if} - {/if} -
- - - + {/if} + {#if !usesBuiltInVoiceSelector} + + + + {/if} + {#if !usesBuiltInVoiceSelector}
@@ -2377,6 +2402,7 @@ on:click={removeCurrentVoice}>{tr('common.delete')}
+ {/if} {/if} {#if usesVibeVoiceSpeakerFiles}