diff --git a/include/engine/community_models/f5_tts/runtime.h b/include/engine/community_models/f5_tts/runtime.h index 333ddd6ae..90a43a327 100644 --- a/include/engine/community_models/f5_tts/runtime.h +++ b/include/engine/community_models/f5_tts/runtime.h @@ -42,11 +42,6 @@ struct F5ComputeDevice { bool use_cuda = false; int device = 0; // CUDA device index int threads = 0; // CPU threads; 0 = hardware concurrency - // FP16 linear weights: GEMMs get ~3x faster on tensor cores but each - // mul_mat converts the F32 activations to F16 first; at F5's GEMM sizes - // (K=1024/2048, N~1022) the conversion overhead outweighs the gain on an - // RTX 3090 (measured 4.0s -> 5.0s per clip). Off by default. - bool fp16_weights = false; }; // Debug taps for parity testing: when non-null, intermediate stage outputs are diff --git a/src/community_models/f5_tts/runtime.cpp b/src/community_models/f5_tts/runtime.cpp index c914d16a4..b7c55b46a 100644 --- a/src/community_models/f5_tts/runtime.cpp +++ b/src/community_models/f5_tts/runtime.cpp @@ -31,46 +31,12 @@ namespace engine::models::f5_tts { namespace { namespace core = engine::core; -namespace modules = engine::modules; // Column convention throughout: tensors are [features, seq] (ggml ne0 = // features), so ggml_norm == LayerNorm over features and ggml_mul_mat(W, X) // matches torch X @ W^T with torch weights stored [out, in]. -struct F5Linear { - core::TensorValue weight; - core::TensorValue bias; -}; - -struct F5ConvNeXt { - modules::DepthwiseConv1dWeights dwconv; - core::TensorValue norm_w, norm_b; - F5Linear pw1, pw2; - std::vector grn_gamma, grn_beta; -}; - -struct F5Block { - F5Linear attn_norm; // -> 6*dim - F5Linear to_q, to_k, to_v, to_out; - F5Linear ff0, ff2; -}; - -struct F5Weights { - std::shared_ptr store; - core::TensorValue text_embedding; // [512, 2731] - F5Linear input_proj; - modules::Conv1dWeights cpe0, cpe2; // grouped k31 g16 - F5Linear time0, time2; - std::vector inv_freq; - std::vector text_blocks; - std::vector blocks; - F5Linear norm_out; - F5Linear proj_out; -}; - struct LoadedModel { - F5Weights w; - F5Architecture arch; ggml_backend_t backend = nullptr; // backend owning the weight store core::BackendType backend_type = core::BackendType::Cpu; }; @@ -83,78 +49,6 @@ struct BackendOwner { ~BackendOwner() = default; }; -F5Weights load_weights( - const engine::assets::TensorSource & source, - ggml_backend_t backend, - core::BackendType backend_type, - bool fp16_linears) { - F5Weights w; - w.store = std::make_shared( - backend, backend_type, "f5_tts.weights", 2ULL * 1024ULL * 1024ULL * 1024ULL); - const auto f32 = [&](const std::string & n) { - return w.store->load_f32_tensor( - source, n, source.require_metadata(n).shape); - }; - // Linear weights consumed only by ggml_mul_mat: FP16 storage halves GEMM - // time on tensor cores (mul_mat returns F32 regardless of weight dtype). - // Everything else (embeddings consumed by get_rows, dwconv kernels read - // on host, biases/norms broadcast) stays F32. - const auto gemm_w = [&](const std::string & n) { - if (fp16_linears) { - return w.store->load_tensor( - source, n, engine::assets::TensorStorageType::F16, - source.require_metadata(n).shape); - } - return f32(n); - }; - const auto lin_f32bias = [&](const std::string & n) { - return F5Linear{gemm_w(n + ".weight"), f32(n + ".bias")}; - }; - - w.text_embedding = f32("text_embed.text_embed.weight"); // get_rows consumer: keep F32 - w.input_proj = lin_f32bias("input_embed.proj"); - w.cpe0.weight = gemm_w("input_embed.conv_pos_embed.conv1d.0.weight"); - w.cpe0.bias = f32("input_embed.conv_pos_embed.conv1d.0.bias"); - w.cpe2.weight = gemm_w("input_embed.conv_pos_embed.conv1d.2.weight"); - w.cpe2.bias = f32("input_embed.conv_pos_embed.conv1d.2.bias"); - w.time0 = lin_f32bias("time_embed.time_mlp.0"); - w.time2 = lin_f32bias("time_embed.time_mlp.2"); - w.inv_freq = source.require_f32("rotary_embed.inv_freq"); - - w.text_blocks.reserve(4); - for (int i = 0; i < 4; ++i) { - const std::string p = "text_embed.text_blocks." + std::to_string(i); - F5ConvNeXt b; - b.dwconv.weight = f32(p + ".dwconv.weight"); // host-side kernel read - b.dwconv.bias = f32(p + ".dwconv.bias"); - b.norm_w = f32(p + ".norm.weight"); - b.norm_b = f32(p + ".norm.bias"); - b.pw1 = lin_f32bias(p + ".pwconv1"); - b.pw2 = lin_f32bias(p + ".pwconv2"); - b.grn_gamma = source.require_f32(p + ".grn.gamma"); - b.grn_beta = source.require_f32(p + ".grn.beta"); - w.text_blocks.push_back(std::move(b)); - } - w.blocks.reserve(22); - for (int i = 0; i < 22; ++i) { - const std::string p = "transformer_blocks." + std::to_string(i); - F5Block b; - b.attn_norm = lin_f32bias(p + ".attn_norm.linear"); - b.to_q = lin_f32bias(p + ".attn.to_q"); - b.to_k = lin_f32bias(p + ".attn.to_k"); - b.to_v = lin_f32bias(p + ".attn.to_v"); - b.to_out = lin_f32bias(p + ".attn.to_out.0"); - b.ff0 = lin_f32bias(p + ".ff.ff.0.0"); - b.ff2 = lin_f32bias(p + ".ff.ff.2"); - w.blocks.push_back(std::move(b)); - } - w.norm_out = lin_f32bias("norm_out.linear"); - w.proj_out = lin_f32bias("proj_out"); - w.store->upload(); - source.release_storage(); - return w; -} - // write a small view that strips that dotted prefix. class StrippedView final : public engine::assets::TensorSource { public: @@ -273,7 +167,6 @@ const LoadedModel & load_model_once(const std::string & path, const F5ComputeDev if (const auto found = cache->find(key); found != cache->end()) { return found->second; } - auto stripped = open_dit_source(path); auto owner = std::make_unique(); const core::BackendType type = dev.use_cuda ? core::BackendType::Cuda : core::BackendType::Cpu; core::BackendConfig cfg{type, dev.use_cuda ? dev.device : 0, dev.use_cuda ? 1 : std::max(1, dev.threads)}; @@ -282,12 +175,8 @@ const LoadedModel & load_model_once(const std::string & path, const F5ComputeDev core::set_backend_threads(owner->value, std::max(1, dev.threads)); } LoadedModel model; - model.arch = F5Architecture{}; model.backend = owner->value; model.backend_type = type; - // FP16 linear weights on CUDA only (CPU mul_mat with F16 weights is slow - // via the fallback path; CUDA hits tensor cores). - model.w = load_weights(*stripped, owner->value, type, dev.use_cuda && dev.fp16_weights); owners.push_back(std::move(owner)); return cache->emplace(key, std::move(model)).first->second; }