Skip to content

refactor(f5_tts): drop the unused raw DiT weight store #956

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
0xShug0 merged 1 commit into from
Oct 17, 2026
Merged

refactor(f5_tts): drop the unused raw DiT weight store #956

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 0 additions & 5 deletions include/engine/community_models/f5_tts/runtime.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,11 +42,6 @@ struct F5ComputeDevice {
bool use_cuda = false;
int device = 0; // CUDA device index
int threads = 0; // CPU threads; 0 = hardware concurrency
// FP16 linear weights: GEMMs get ~3x faster on tensor cores but each
// mul_mat converts the F32 activations to F16 first; at F5's GEMM sizes
// (K=1024/2048, N~1022) the conversion overhead outweighs the gain on an
// RTX 3090 (measured 4.0s -> 5.0s per clip). Off by default.
bool fp16_weights = false;
};

// Debug taps for parity testing: when non-null, intermediate stage outputs are
Expand Down
111 changes: 0 additions & 111 deletions src/community_models/f5_tts/runtime.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,46 +31,12 @@ namespace engine::models::f5_tts {
namespace {

namespace core = engine::core;
namespace modules = engine::modules;

// Column convention throughout: tensors are [features, seq] (ggml ne0 =
// features), so ggml_norm == LayerNorm over features and ggml_mul_mat(W, X)
// matches torch X @ W^T with torch weights stored [out, in].

struct F5Linear {
core::TensorValue weight;
core::TensorValue bias;
};

struct F5ConvNeXt {
modules::DepthwiseConv1dWeights dwconv;
core::TensorValue norm_w, norm_b;
F5Linear pw1, pw2;
std::vector<float> grn_gamma, grn_beta;
};

struct F5Block {
F5Linear attn_norm; // -> 6*dim
F5Linear to_q, to_k, to_v, to_out;
F5Linear ff0, ff2;
};

struct F5Weights {
std::shared_ptr<core::BackendWeightStore> store;
core::TensorValue text_embedding; // [512, 2731]
F5Linear input_proj;
modules::Conv1dWeights cpe0, cpe2; // grouped k31 g16
F5Linear time0, time2;
std::vector<float> inv_freq;
std::vector<F5ConvNeXt> text_blocks;
std::vector<F5Block> blocks;
F5Linear norm_out;
F5Linear proj_out;
};

struct LoadedModel {
F5Weights w;
F5Architecture arch;
ggml_backend_t backend = nullptr; // backend owning the weight store
core::BackendType backend_type = core::BackendType::Cpu;
};
Expand All @@ -83,78 +49,6 @@ struct BackendOwner {
~BackendOwner() = default;
};

F5Weights load_weights(
const engine::assets::TensorSource & source,
ggml_backend_t backend,
core::BackendType backend_type,
bool fp16_linears) {
F5Weights w;
w.store = std::make_shared<core::BackendWeightStore>(
backend, backend_type, "f5_tts.weights", 2ULL * 1024ULL * 1024ULL * 1024ULL);
const auto f32 = [&](const std::string & n) {
return w.store->load_f32_tensor(
source, n, source.require_metadata(n).shape);
};
// Linear weights consumed only by ggml_mul_mat: FP16 storage halves GEMM
// time on tensor cores (mul_mat returns F32 regardless of weight dtype).
// Everything else (embeddings consumed by get_rows, dwconv kernels read
// on host, biases/norms broadcast) stays F32.
const auto gemm_w = [&](const std::string & n) {
if (fp16_linears) {
return w.store->load_tensor(
source, n, engine::assets::TensorStorageType::F16,
source.require_metadata(n).shape);
}
return f32(n);
};
const auto lin_f32bias = [&](const std::string & n) {
return F5Linear{gemm_w(n + ".weight"), f32(n + ".bias")};
};

w.text_embedding = f32("text_embed.text_embed.weight"); // get_rows consumer: keep F32
w.input_proj = lin_f32bias("input_embed.proj");
w.cpe0.weight = gemm_w("input_embed.conv_pos_embed.conv1d.0.weight");
w.cpe0.bias = f32("input_embed.conv_pos_embed.conv1d.0.bias");
w.cpe2.weight = gemm_w("input_embed.conv_pos_embed.conv1d.2.weight");
w.cpe2.bias = f32("input_embed.conv_pos_embed.conv1d.2.bias");
w.time0 = lin_f32bias("time_embed.time_mlp.0");
w.time2 = lin_f32bias("time_embed.time_mlp.2");
w.inv_freq = source.require_f32("rotary_embed.inv_freq");

w.text_blocks.reserve(4);
for (int i = 0; i < 4; ++i) {
const std::string p = "text_embed.text_blocks." + std::to_string(i);
F5ConvNeXt b;
b.dwconv.weight = f32(p + ".dwconv.weight"); // host-side kernel read
b.dwconv.bias = f32(p + ".dwconv.bias");
b.norm_w = f32(p + ".norm.weight");
b.norm_b = f32(p + ".norm.bias");
b.pw1 = lin_f32bias(p + ".pwconv1");
b.pw2 = lin_f32bias(p + ".pwconv2");
b.grn_gamma = source.require_f32(p + ".grn.gamma");
b.grn_beta = source.require_f32(p + ".grn.beta");
w.text_blocks.push_back(std::move(b));
}
w.blocks.reserve(22);
for (int i = 0; i < 22; ++i) {
const std::string p = "transformer_blocks." + std::to_string(i);
F5Block b;
b.attn_norm = lin_f32bias(p + ".attn_norm.linear");
b.to_q = lin_f32bias(p + ".attn.to_q");
b.to_k = lin_f32bias(p + ".attn.to_k");
b.to_v = lin_f32bias(p + ".attn.to_v");
b.to_out = lin_f32bias(p + ".attn.to_out.0");
b.ff0 = lin_f32bias(p + ".ff.ff.0.0");
b.ff2 = lin_f32bias(p + ".ff.ff.2");
w.blocks.push_back(std::move(b));
}
w.norm_out = lin_f32bias("norm_out.linear");
w.proj_out = lin_f32bias("proj_out");
w.store->upload();
source.release_storage();
return w;
}

// write a small view that strips that dotted prefix.
class StrippedView final : public engine::assets::TensorSource {
public:
Expand Down Expand Up @@ -273,7 +167,6 @@ const LoadedModel & load_model_once(const std::string & path, const F5ComputeDev
if (const auto found = cache->find(key); found != cache->end()) {
return found->second;
}
auto stripped = open_dit_source(path);
auto owner = std::make_unique<BackendOwner>();
const core::BackendType type = dev.use_cuda ? core::BackendType::Cuda : core::BackendType::Cpu;
core::BackendConfig cfg{type, dev.use_cuda ? dev.device : 0, dev.use_cuda ? 1 : std::max(1, dev.threads)};
Expand All @@ -282,12 +175,8 @@ const LoadedModel & load_model_once(const std::string & path, const F5ComputeDev
core::set_backend_threads(owner->value, std::max(1, dev.threads));
}
LoadedModel model;
model.arch = F5Architecture{};
model.backend = owner->value;
model.backend_type = type;
// FP16 linear weights on CUDA only (CPU mul_mat with F16 weights is slow
// via the fallback path; CUDA hits tensor cores).
model.w = load_weights(*stripped, owner->value, type, dev.use_cuda && dev.fp16_weights);
owners.push_back(std::move(owner));
return cache->emplace(key, std::move(model)).first->second;
}
Expand Down
Loading