diff --git a/CMakeLists.txt b/CMakeLists.txt index ddaf9e2f3..d291073b9 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -2473,6 +2473,7 @@ audiocpp_add_model(cosyvoice3 src/models/cosyvoice3/flow.cpp src/models/cosyvoice3/frontend.cpp src/models/cosyvoice3/hift.cpp + src/models/cosyvoice3/rknn_flow.cpp src/models/cosyvoice3/session.cpp src/models/cosyvoice3/tokenizer_text.cpp INCLUDES diff --git a/docs/build/rockchip-npu.md b/docs/build/rockchip-npu.md index c208b7acc..bdbc5944a 100644 --- a/docs/build/rockchip-npu.md +++ b/docs/build/rockchip-npu.md @@ -366,6 +366,7 @@ the configurable components: | Qwen3 `speech_decoder` | `qwen3_tts.exec_plan` | `ggml`, `rknn` | RKNN: required `main`, optional `first` | | sanoTTS `decoder` | `sanotts.exec_plan` | `default`, `ggml`, `neon` | None | | Inflect v2 `decoder` | `inflect_v2.exec_plan` | `ggml`, `rknn` | RKNN: required `main` | +| CosyVoice3 `flow` | `cosyvoice3.exec_plan` | `ggml`, `rknn` | RKNN: required `main` (comma-separated ladder of per-length engines) | These options also work with older GGUFs whose embedded specs do not declare `exec_plan`; the model validates the plan without requiring a spec override. diff --git a/docs/models/cosyvoice3.md b/docs/models/cosyvoice3.md index 049349632..c388c4e9a 100644 --- a/docs/models/cosyvoice3.md +++ b/docs/models/cosyvoice3.md @@ -89,6 +89,72 @@ audiocpp_cli \ |---|---|---:|---| | `cosyvoice3.reference_cache_slots` | integer >= 0 | `4` | Prepared reference-audio cache slots. | | `cosyvoice3.mem_saver` | bool | `false` | Release cached runtime graphs after request phases. | +| `cosyvoice3.exec_plan` | JSON object | `{}` | Component execution plans. The only component is `flow`; see the RKNN section. | + +## Rockchip NPU (RKNN) estimator + +The flow decoder is the expensive half of this family: the DiT estimator runs once per Euler step +(`num_inference_steps`, default 10) over the whole utterance, and on an RK3588 it is ~79% of +synthesis. `cosyvoice3.exec_plan` voices that one stage through `librknnrt` and leaves the AR LLM +and the HiFT vocoder on the host, which is the split the reference RKNN port of this model +measured best: a quantized NPU LLM degrades the speech tokens, and a quantized HiFT adds +high-frequency artifacts. + +RKNN has no runtime graph builder, so each engine is compiled offline for one fixed sequence +length. Build them on an x86_64 host from the same checkpoint the GGUF was converted from: + +```bash +python3 tools/rknn/prepare_cosyvoice3_rknn_flow.py \ + --estimator-onnx flow.decoder.estimator.fp32.onnx \ + --seq-len 200 --seq-len 500 --seq-len 900 --seq-len 1100 \ + --output-dir models/CosyVoice3-RKNN +``` + +Then point the session at the ladder (order does not matter; the smallest engine that fits is +used, and an utterance longer than the largest engine runs the CPU graph). The ladder travels as +the `flow` component's `main` artifact, comma-separated — see +[component execution plans](../build/rockchip-npu.md#component-execution-plans): + +```bash +audiocpp_cli --task clon --family cosyvoice3 --model models/CosyVoice3-GGUF/cosyvoice3-q8_0.gguf \ + --backend cpu --session-option 'cosyvoice3.exec_plan={"flow":{"impl":"rknn","artifacts":{"main":"models/CosyVoice3-RKNN/flow_estimator_seq200.rknn,models/CosyVoice3-RKNN/flow_estimator_seq900.rknn"}}}' \ + --text "..." --voice-ref assets/resources/b.wav --reference-text "..." --out out.wav +``` + +Build with `-DENGINE_ENABLE_RKNN=ON` (plus `ENGINE_RKNN_LIBRARY` / `ENGINE_RKNN_INCLUDE_DIR` +pointing at the board's `librknnrt`); a plan selecting `impl: rknn` throws at load in a build +without it rather than falling back silently. Each engine is ~700 MB and an utterance is padded up +to the engine's +length, so the ladder is a time trade: the estimator costs about `S^2`, and the padding is exact +(measured bit-identical to an unpadded run), not a quality one. + +### What it measures on an RK3588 + +Measured on an Orange Pi 5 Plus (RK3588, 32 GB, `librknnrt` 2.3.2, `--backend cpu --threads 4`, +10 Euler steps, same text/seed/reference on both paths), `dit_ms` for the whole ODE loop: + +| mel frames | engine (pad) | ggml CPU | RKNN | +|---|---|---|---| +| 102 | seq200 (98) | 8.48 s | 11.49 s | +| 212 | seq500 (288) | 19.14 s | 57.10 s | +| 102 | seq110 (8) | 8.48 s | 6.14 s | +| 212 | seq250 (38) | 19.29 s | 18.51 s | +| 402 | seq400 (2) | 38.87 s | 38.84 s | + +The estimator on the NPU is at parity with four A76 cores running the ggml graph, so the padding is +what decides: a coarse ladder loses by 3x, a dense one breaks even above 200 frames and wins 1.38x +only at the shortest lengths. Peak RSS is 3.8-6.8 GB against 2.3-2.6 GB. + +Two ceilings bound the ladder. The flow's frame count includes the prompt mel, so a 4-second +reference puts every utterance above 700 frames. And the driver's IOVA arena is 32-bit on this +kernel: one engine asks 0.7 GB at seq110 and 4.71 GB at seq900, and above about 4 GB `rknn_init` +fails with `-4` (`RKNPU: failed to allocate IOVA: -12`), which is why the ladder is loaded lazily, +one engine resident. Engines above ~700 frames are therefore unusable on this board, and long +utterances — where the estimator is 79% of synthesis — run on the ggml graph. + +The fp16 cost of the graph, measured on the PC simulator against the fp32 ONNX (seq200, 106 real +frames): snr 44.0 dB, tail snr 60.5 dB, `max_abs 0.5544`, mean `-3.83689` against `-3.83806`. There +is no systematic mel bias, so no offset knob is needed. Legacy `cosyvoice3.conv_weight_type` remains accepted but has no consumer and does not change convolution storage. The implementation uses the shared diff --git a/include/engine/models/cosyvoice3/flow.h b/include/engine/models/cosyvoice3/flow.h index 398af05b8..897e73a69 100644 --- a/include/engine/models/cosyvoice3/flow.h +++ b/include/engine/models/cosyvoice3/flow.h @@ -31,7 +31,8 @@ class CosyVoice3DiTFlowRuntime { engine::core::ExecutionContext & execution, size_t graph_arena_bytes, size_t weight_context_bytes, - engine::assets::TensorStorageType storage_type); + engine::assets::TensorStorageType storage_type, + const std::vector & rknn_engine_paths = {}); ~CosyVoice3DiTFlowRuntime(); CosyVoice3DiTFlowRuntime(const CosyVoice3DiTFlowRuntime &) = delete; diff --git a/include/engine/models/cosyvoice3/rknn_flow.h b/include/engine/models/cosyvoice3/rknn_flow.h new file mode 100644 index 000000000..a2727901f --- /dev/null +++ b/include/engine/models/cosyvoice3/rknn_flow.h @@ -0,0 +1,63 @@ +#pragma once + +#include +#include +#include +#include + +namespace engine::models::cosyvoice3 { + +// Rockchip NPU (RKNN) backend for the flow-matching DiT estimator: the ODE step that the ggml +// DiT graph in flow.cpp computes. The estimator is the flow decoder's velocity field, and it is +// the stage the reference RKNN port keeps on the NPU (the AR LLM and the HiFT vocoder stay on +// CPU there, for the same reasons this repository keeps its sidecars on the decoder half). +// +// RKNN has no dynamic shapes, so each engine is compiled offline for one fixed sequence length +// (tools/rknn/prepare_cosyvoice3_rknn_flow.py) and a ladder of them is registered here. An utterance of +// `frames` mel frames runs on the smallest engine that fits it: `x`/`mu`/`cond` zero-padded on +// the right, `mask` 1 on the real frames and 0 in the pad, output cropped back. Measured on the +// host, a padded run is bit-identical to an exact run at the real length (355 dB, max_abs 0.0), +// so the ladder is a time trade, not a quality one: the estimator costs ~S^2, and every frame of +// pad is paid for ten times, once per ODE step. +// +// The engine takes the raw timestep `t` (its time-embedding MLP is inside the graph) and a frame +// `mask`; the ggml graph takes the timestep embedding and has no mask. Both are supplied here +// from the same ODE loop, so the two backends see the same step. +class CosyVoice3RknnFlow { +public: + // Registers the ladder without loading any of it: one engine asks the driver for 0.7-3.3 GB + // of NPU memory (measured, seq200 to seq1100), so a ladder cannot be resident at once. The + // length an engine covers is read from its file name (`flow_estimator_seq900.rknn`, the + // producer's convention) and validated against the engine's own attributes when it loads. + // `mel_channels` is the checkpoint's, checked the same way. + CosyVoice3RknnFlow(const std::vector & engine_paths, int64_t mel_channels); + ~CosyVoice3RknnFlow(); + + CosyVoice3RknnFlow(const CosyVoice3RknnFlow &) = delete; + CosyVoice3RknnFlow & operator=(const CosyVoice3RknnFlow &) = delete; + + // Longest utterance the ladder covers, in mel frames. A longer one has no engine and has to + // run on the ggml graph, which is why the caller keeps it. + int64_t max_frames() const; + int64_t engine_count() const; + // Mel channels the ladder was built for. + int64_t mel_channels() const; + + // One ODE step. `x`, `mu` and `cond` are channel-major [2, mel_channels, frames] and `spks` + // is [2, mel_channels], with the classifier-free-guidance uncond branch zeroed in `mu`, + // `cond` and `spks` exactly as the ggml path builds it. `t` is the raw timestep. Returns + // dphi_dt as [2, mel_channels, frames]: the cond branch first, then uncond. + std::vector run( + const float * x, + const float * mu, + const float * cond, + const float * spks, + float t, + int64_t frames); + +private: + struct Impl; + std::unique_ptr impl_; +}; + +} // namespace engine::models::cosyvoice3 diff --git a/model_specs/cosyvoice3.json b/model_specs/cosyvoice3.json index bb3f5e14a..6908f0947 100644 --- a/model_specs/cosyvoice3.json +++ b/model_specs/cosyvoice3.json @@ -174,6 +174,13 @@ "description": "Release cached runtime graphs after each request to reduce peak VRAM; default false.", "required": false, "default": false + }, + { + "name": "exec_plan", + "type": "string", + "description": "JSON object of component execution plans. Component: flow. Each plan accepts impl, backend, artifacts, and params. impl is default/ggml/rknn. GGML keeps the session backend. RKNN requires artifacts.main, a comma-separated ladder of estimator engines compiled one per sequence length (tools/rknn/prepare_cosyvoice3_rknn_flow.py), and ENGINE_ENABLE_RKNN; omit backend. An utterance longer than the largest engine runs the CPU graph. Neither implementation takes params.", + "required": false, + "default": "{}" } ], "load": [] diff --git a/src/models/cosyvoice3/flow.cpp b/src/models/cosyvoice3/flow.cpp index 0f3cd46ec..468a4d81b 100644 --- a/src/models/cosyvoice3/flow.cpp +++ b/src/models/cosyvoice3/flow.cpp @@ -14,6 +14,7 @@ #include "engine/framework/modules/primitive_modules.h" #include "engine/framework/modules/structural_modules.h" #include "engine/framework/modules/weight_binding.h" +#include "engine/models/cosyvoice3/rknn_flow.h" #include #include @@ -269,6 +270,10 @@ std::shared_ptr load_flow_weights( *weights->store, source, "pre_lookahead_layer.conv1", storage_type, c.flow_hidden_size, c.flow_mel_channels, c.pre_lookahead_len + 1, true); weights->pre_lookahead_conv2 = binding::conv1d_from_source( *weights->store, source, "pre_lookahead_layer.conv2", storage_type, c.flow_mel_channels, c.flow_hidden_size, 3, true); + // The ggml DiT weights are loaded in both paths, deliberately: the RKNN ladder covers a fixed + // set of lengths, and an utterance longer than the largest engine runs the ggml graph, so the + // copy is not spare memory. Skipping it would trade a fallback that works for one that throws + // `weight is invalid` the first time the ladder runs out. weights->input_projection = binding::linear_from_source( *weights->store, source, "decoder.estimator.input_embed.proj", storage_type, c.flow_hidden_size, 320, true); weights->conv_pos_1.weight = weights->store->load_tensor( @@ -514,7 +519,7 @@ class DiTGraph { const std::vector & mu, const std::vector & cond, const std::vector & speaker, - const std::vector & time_embedding, + float t, int64_t frames) { if (frames <= 0) { throw std::runtime_error("CosyVoice3 DiT frames must be positive"); @@ -524,10 +529,15 @@ class DiTGraph { if (static_cast(x.size()) != batch * channels * frames || static_cast(mu.size()) != batch * channels * frames || static_cast(cond.size()) != batch * channels * frames || - static_cast(speaker.size()) != batch * channels || - static_cast(time_embedding.size()) != batch * kTimeEmbeddingSize) { + static_cast(speaker.size()) != batch * channels) { throw std::runtime_error("CosyVoice3 DiT input size mismatch"); } + // The graph takes the timestep embedding, not the timestep: the sinusoidal expansion is + // host-side here and inside the graph in the RKNN engine. Both backends get `t`. + const auto temb = timestep_embedding(t); + std::vector time_embedding(static_cast(batch) * kTimeEmbeddingSize); + std::copy(temb.begin(), temb.end(), time_embedding.begin()); + std::copy(temb.begin(), temb.end(), time_embedding.begin() + kTimeEmbeddingSize); ensure(frames); ggml_backend_tensor_set(x_, x.data(), 0, x.size() * sizeof(float)); ggml_backend_tensor_set(mu_, mu.data(), 0, mu.size() * sizeof(float)); @@ -728,16 +738,29 @@ class CosyVoice3DiTFlowRuntime::Impl { core::ExecutionContext & execution, size_t graph_arena_bytes, size_t weight_context_bytes, - engine::assets::TensorStorageType storage_type) + engine::assets::TensorStorageType storage_type, + const std::vector & rknn_engine_paths) : assets_(std::move(assets)), execution_(execution), - weights_(load_flow_weights(*assets_, execution_, weight_context_bytes, storage_type)), + weights_(load_flow_weights( + *assets_, + execution_, + weight_context_bytes, + storage_type)), condition_(execution_, weights_, assets_->config, graph_arena_bytes), dit_(execution_, weights_, assets_->config, graph_arena_bytes), graph_arena_bytes_(graph_arena_bytes) { if (assets_ == nullptr) { throw std::runtime_error("CosyVoice3 flow runtime requires assets"); } + if (!rknn_engine_paths.empty()) { + // The engines' own channel count is checked against the checkpoint when an engine + // loads, so nothing to compare here: registering the ladder loads none of it. + rknn_ = std::make_unique( + rknn_engine_paths, assets_->config.flow_mel_channels); + debug::trace_log_scalar("cosyvoice3.flow.dit.rknn.engines", rknn_->engine_count()); + debug::trace_log_scalar("cosyvoice3.flow.dit.rknn.max_frames", rknn_->max_frames()); + } } CosyVoice3FlowOutput generate(const CosyVoice3FlowRequest & request) { @@ -788,15 +811,20 @@ class CosyVoice3DiTFlowRuntime::Impl { std::copy(speaker_cond.begin(), speaker_cond.end(), spks.begin()); const auto schedule = cosine_time_schedule(request.num_inference_steps); + // The NPU ladder covers a fixed range of lengths; past its largest engine the ggml graph + // is the only backend that has a shape for the utterance, so the fallback is per call. + const bool on_npu = rknn_ != nullptr && total_frames <= rknn_->max_frames(); + if (rknn_ != nullptr && !on_npu) { + debug::timing_log_scalar( + "cosyvoice3.flow.dit.rknn.fallback_frames", static_cast(total_frames)); + } const auto dit_start = std::chrono::steady_clock::now(); for (int64_t step = 1; step < static_cast(schedule.size()); ++step) { const float t = schedule[static_cast(step - 1)]; const float dt = schedule[static_cast(step)] - t; - auto temb = timestep_embedding(t); - std::vector time_batched(static_cast(2 * kTimeEmbeddingSize)); - std::copy(temb.begin(), temb.end(), time_batched.begin()); - std::copy(temb.begin(), temb.end(), time_batched.begin() + kTimeEmbeddingSize); - auto pred = dit_.run(x_batched, mu, cond, spks, time_batched, total_frames); + auto pred = on_npu + ? rknn_->run(x_batched.data(), mu.data(), cond.data(), spks.data(), t, total_frames) + : dit_.run(x_batched, mu, cond, spks, t, total_frames); const size_t branch = static_cast(c.flow_mel_channels * total_frames); for (size_t i = 0; i < branch; ++i) { const float guided = (1.0F + kInferenceCfgRate) * pred[i] - kInferenceCfgRate * pred[branch + i]; @@ -832,6 +860,7 @@ class CosyVoice3DiTFlowRuntime::Impl { std::shared_ptr weights_; ConditionGraph condition_; DiTGraph dit_; + std::unique_ptr rknn_; size_t graph_arena_bytes_ = 0; }; @@ -840,13 +869,15 @@ CosyVoice3DiTFlowRuntime::CosyVoice3DiTFlowRuntime( engine::core::ExecutionContext & execution, size_t graph_arena_bytes, size_t weight_context_bytes, - engine::assets::TensorStorageType storage_type) + engine::assets::TensorStorageType storage_type, + const std::vector & rknn_engine_paths) : impl_(std::make_unique( std::move(assets), execution, graph_arena_bytes, weight_context_bytes, - storage_type)) {} + storage_type, + rknn_engine_paths)) {} CosyVoice3DiTFlowRuntime::~CosyVoice3DiTFlowRuntime() = default; diff --git a/src/models/cosyvoice3/rknn_flow.cpp b/src/models/cosyvoice3/rknn_flow.cpp new file mode 100644 index 000000000..d5c0b6212 --- /dev/null +++ b/src/models/cosyvoice3/rknn_flow.cpp @@ -0,0 +1,366 @@ +#include "engine/models/cosyvoice3/rknn_flow.h" + +#include "engine/framework/debug/profiler.h" +#include "engine/framework/debug/trace.h" +#include "engine/framework/rknn/engine.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace engine::models::cosyvoice3 { + +namespace { + +#if defined(ENGINE_HAVE_RKNN) +using Clock = std::chrono::steady_clock; + +// Prefix of every message this sidecar raises, shared with the plumbing in engine::rknn so a +// failure names the stage that failed. +const char * const kLabel = "CosyVoice3 flow estimator RKNN"; + +// The classifier-free-guidance pair: branch 0 is the conditioned estimate, branch 1 the +// unconditioned one, and the ODE loop mixes them outside the engine. +constexpr int64_t kCfgBatch = 2; + +// The NPU I/O aperture an engine occupies is not its file size: measured on RK3588 with every +// context of a ladder resident in one process, the estimator engines occupy 0.27 GiB at S=110, +// 0.48 at 200, 0.77 at 250, 1.93 at 400, 3.04 at 500 and 4.39 at 900, while all of their files are +// 661-714 MB. The short rungs are therefore genuinely co-resident (S=200 x3, 250+250+110 and 110 x4 +// all initialize together, at <= 1.81 GiB), and the long ones are not (400 coexists only with +// 110/200, 500 with nothing). 5 MiB per frame over-estimates the short rungs by ~2x (S=110 really +// costs 0.27 GiB, S=200 0.48, S=250 0.77) and is within 5% of the long ones, so the budget is a +// statement about the long rungs with a safety margin on the short ones. 2800 MiB of proxy is the +// largest resident set the measured aperture fits: 110+200+250 = 1.52 GiB real (three engines fit +// 1.81 GiB), 110+400 = 2.20 GiB (two engines fit 2.42), while 200+400 = 2.41 GiB is refused by the +// proxy although it measures just inside the two-engine ceiling -- the proxy pins less than the +// aperture allows, never more. +constexpr int64_t kApertureMiBPerFrame = 5; +constexpr int64_t kAperturePinBudgetMiB = 2800; + +// Graph order, as tools/rknn/prepare_cosyvoice3_rknn_flow.py compiles it. +const char * const kInputNames[] = {"x", "mask", "mu", "t", "spks", "cond"}; +constexpr uint32_t kInputCount = 6; +constexpr uint32_t kOutputCount = 1; +constexpr uint32_t kInputX = 0; +constexpr uint32_t kInputMask = 1; +constexpr uint32_t kInputMu = 2; +constexpr uint32_t kInputT = 3; +constexpr uint32_t kInputSpks = 4; +constexpr uint32_t kInputCond = 5; + +// The sequence length an engine covers, read out of the producer's file name +// (`flow_estimator_seq900.rknn`). Reading it without loading is what keeps the ladder from being +// resident: an engine asks the driver for 0.7-3.3 GB of NPU memory, so a ladder of them does not +// fit, and the length is all the sidecar needs in order to choose. +int64_t name_seq_len(const std::string & path) { + const std::string stem = std::filesystem::path(path).stem().string(); + const auto at = stem.find("seq"); + if (at == std::string::npos) { + return -1; + } + int64_t value = 0; + size_t index = at + 3; + for (; index < stem.size() && stem[index] >= '0' && stem[index] <= '9'; ++index) { + value = value * 10 + (stem[index] - '0'); + } + return index == at + 3 ? -1 : value; +} +#endif + +} // namespace + +struct CosyVoice3RknnFlow::Impl { + // One registered engine of the ladder, plus the scratch the padding needs. The scratch is + // sized on load and reused across calls because the ODE loop runs the same shape ten times. + struct Slot { + std::string path; + int64_t frames = 0; // the sequence length this engine covers + int64_t channels = 0; +#if defined(ENGINE_HAVE_RKNN) + engine::rknn::Engine engine; + bool loaded = false; + std::vector x; + std::vector mask; + std::vector mu; + std::vector cond; + std::vector t; + std::vector spks; + + void load(int64_t expected_channels) { + const auto start = Clock::now(); + engine.load(path, kLabel); + if (engine.input_count() != kInputCount || engine.output_count() != kOutputCount) { + throw std::runtime_error( + std::string(kLabel) + ": expected the 6-input/1-output estimator export, got " + + std::to_string(engine.input_count()) + "/" + std::to_string(engine.output_count())); + } + for (uint32_t index = 0; index < engine.input_count(); ++index) { + const rknn_tensor_attr & attr = engine.input_attr(index); + const std::string name(attr.name); + debug::timing_log_scalar( + "cosyvoice3.flow.dit.rknn.input_attr" + std::to_string(index), + name + " type=" + std::to_string(attr.type) + " size=" + std::to_string(attr.size) + + " elems=" + std::to_string(attr.n_elems) + " fmt=" + std::to_string(attr.fmt) + + " dims=" + std::to_string(attr.n_dims)); + if (name == "x" && attr.n_dims == 3) { + if (attr.dims[0] != kCfgBatch) { + throw std::runtime_error( + std::string(kLabel) + ": " + path + " has batch " + std::to_string(attr.dims[0]) + + ", the classifier-free-guidance pair is " + std::to_string(kCfgBatch)); + } + channels = attr.dims[1]; + if (channels * attr.dims[2] != frames * channels) { + throw std::runtime_error( + std::string(kLabel) + ": " + path + " covers " + std::to_string(attr.dims[2]) + + " frames, its name promises " + std::to_string(frames)); + } + } + } + if (channels <= 0) { + throw std::runtime_error(std::string(kLabel) + ": " + path + " is not the estimator export"); + } + if (channels != expected_channels) { + throw std::runtime_error( + std::string(kLabel) + ": " + path + " carries " + std::to_string(channels) + + " mel channels, the checkpoint has " + std::to_string(expected_channels) + + "; compile the engines with tools/rknn/prepare_cosyvoice3_rknn_flow.py from this checkpoint"); + } + const rknn_tensor_attr & out = engine.output_attr(0); + if (out.n_elems != kCfgBatch * channels * frames) { + throw std::runtime_error( + std::string(kLabel) + ": " + path + " returns " + std::to_string(out.n_elems) + + " values, expected " + std::to_string(kCfgBatch * channels * frames)); + } + // The length-carrying inputs are [2, channels, frames] — the guidance pair is two + // copies of the plane — while `mask` is [2, 1, frames], two rows of `frames`. + const size_t plane = static_cast(kCfgBatch) * channels * static_cast(frames); + x.resize(plane); + mu.resize(plane); + cond.resize(plane); + mask.resize(static_cast(kCfgBatch) * static_cast(frames)); + t.resize(kCfgBatch); + spks.resize(static_cast(kCfgBatch) * static_cast(channels)); + debug::timing_log_scalar("cosyvoice3.flow.dit.rknn.load_ms", debug::elapsed_ms(start)); + debug::trace_log_scalar("cosyvoice3.flow.dit.rknn.loaded_frames", frames); + // Only once the engine is validated and its scratch exists: a slot that threw halfway + // through would otherwise be marked resident, and every later request would be handed + // an engine with no input buffers instead of the same clear failure. + loaded = true; + } + + // One ODE step at the engine's own length. The five host-side tensors arrive at the real + // length `frames_in` and are widened to `frames` here; the result is cut back by the caller. + void run( + const float * x_in, + const float * mu_in, + const float * cond_in, + const float * spks_in, + float t_in, + int64_t frames_in, + std::vector & out) { + // Row strides, not plane sizes: row `b*channels + ch` starts at row*frames_in on the + // host and at row*frames in the engine, because each row is one (branch, channel) strip + // of the frame axis. + const int64_t rows = kCfgBatch * channels; + const auto copy_padded = [this, rows, frames_in]( + std::vector & dst, const float * src) { + std::fill(dst.begin(), dst.end(), 0.0F); + for (int64_t row = 0; row < rows; ++row) { + std::memcpy( + dst.data() + row * frames, + src + row * frames_in, + static_cast(frames_in) * sizeof(float)); + } + }; + copy_padded(x, x_in); + copy_padded(mu, mu_in); + copy_padded(cond, cond_in); + // mask: 1 where there is a real frame, 0 in the pad, which is what switches the pad + // off inside the graph (measured: a padded run is bit-identical to an exact one). + std::fill(mask.begin(), mask.end(), 0.0F); + for (int64_t row = 0; row < kCfgBatch; ++row) { + std::fill(mask.begin() + row * frames, mask.begin() + row * frames + frames_in, 1.0F); + } + std::fill(t.begin(), t.end(), t_in); + std::memcpy(spks.data(), spks_in, spks.size() * sizeof(float)); + + std::vector inputs(kInputCount); + for (uint32_t index = 0; index < inputs.size(); ++index) { + inputs[index].index = index; + inputs[index].type = RKNN_TENSOR_FLOAT32; + } + // With pass_through = 0 the runtime wants the width of the declared type, not + // attr.size, which reports the engine's native fp16 width. + inputs[kInputX].buf = x.data(); + inputs[kInputX].size = static_cast(x.size() * sizeof(float)); + inputs[kInputMask].buf = mask.data(); + inputs[kInputMask].size = static_cast(mask.size() * sizeof(float)); + inputs[kInputMu].buf = mu.data(); + inputs[kInputMu].size = static_cast(mu.size() * sizeof(float)); + inputs[kInputT].buf = t.data(); + inputs[kInputT].size = static_cast(t.size() * sizeof(float)); + inputs[kInputSpks].buf = spks.data(); + inputs[kInputSpks].size = static_cast(spks.size() * sizeof(float)); + inputs[kInputCond].buf = cond.data(); + inputs[kInputCond].size = static_cast(cond.size() * sizeof(float)); + std::vector> outputs; + engine.run(inputs, outputs); + out.swap(outputs[0]); + } +#endif + }; + + // Held by pointer: Engine owns a rknn_context and is neither copyable nor movable, so the + // ladder has to be a vector of slots, not of slots by value. + std::vector> slots; + int64_t channels = 0; + +#if defined(ENGINE_HAVE_RKNN) + void register_ladder(const std::vector & engine_paths, int64_t expected_channels) { + channels = expected_channels; + for (const auto & path : engine_paths) { + if (path.empty()) { + continue; + } + const int64_t frames = name_seq_len(path); + if (frames <= 0) { + throw std::runtime_error( + std::string(kLabel) + ": " + path + + " does not name the length it covers; the producer names engines " + "flow_estimator_seq.rknn and the length is read from the name so a ladder " + "can be chosen without loading all of it"); + } + // Built in place: Slot owns an Engine, so it is neither copyable nor movable. + auto slot = std::make_unique(); + slot->path = path; + slot->frames = frames; + slot->channels = expected_channels; + slots.push_back(std::move(slot)); + } + std::sort(slots.begin(), slots.end(), [](const auto & a, const auto & b) { + return a->frames < b->frames; + }); + if (slots.empty()) { + throw std::runtime_error(std::string(kLabel) + ": no engine paths given"); + } + } + + // The smallest engine that fits `frames`. The long-sequence engines are loaded on demand and + // evicted on the next length change, because the driver allocates the whole engine (weights and + // workspace) per context and the long rungs need gigabytes. The short rungs stay resident: they + // fit together in the NPU aperture (see kAperturePinBudgetMiB), they are the rungs the served + // requests ask for, and each of them costs ~9 s of `rknn_init` per utterance when it has to be + // re-loaded. + Slot & pick(int64_t frames) { + for (auto & holder : slots) { + Slot & slot = *holder; + if (slot.frames < frames) { + continue; + } + if (!slot.loaded) { + int64_t resident = 0; + for (auto & other : slots) { + if (other->loaded) { + resident += other->frames; + } + } + // The rung is kept (and the new one pinned) only while the whole resident set stays + // inside the aperture; a rung above the budget can never share it with anything, so + // it evicts everything and is itself evicted by the next length change. The budget + // is spent on the engine's compiled length, not the request's: a 900-frame engine + // asked for by a 200-frame utterance reserves 900 frames of aperture. + // ponytail: linear aperture proxy; a per-engine measured table would pin ~2x more. + const bool fits = (resident + slot.frames) * kApertureMiBPerFrame <= kAperturePinBudgetMiB; + if (!fits) { + for (auto & other : slots) { + if (other->loaded) { + other->engine.release(); + other->loaded = false; + } + } + } + slot.load(channels); + debug::trace_log_scalar( + "cosyvoice3.flow.dit.rknn.resident_frames", + fits ? resident + slot.frames : slot.frames); + } + return slot; + } + throw std::runtime_error( + std::string(kLabel) + ": no engine covers " + std::to_string(frames) + " mel frames, the largest is " + + std::to_string(slots.back()->frames)); + } +#endif +}; + +CosyVoice3RknnFlow::CosyVoice3RknnFlow( + const std::vector & engine_paths, + int64_t mel_channels) + : impl_(std::make_unique()) { +#if defined(ENGINE_HAVE_RKNN) + impl_->register_ladder(engine_paths, mel_channels); +#else + (void)engine_paths; + (void)mel_channels; + throw std::runtime_error("CosyVoice3 flow estimator RKNN: built without ENGINE_ENABLE_RKNN"); +#endif +} + +CosyVoice3RknnFlow::~CosyVoice3RknnFlow() = default; + +int64_t CosyVoice3RknnFlow::max_frames() const { + return impl_->slots.empty() ? 0 : impl_->slots.back()->frames; +} + +int64_t CosyVoice3RknnFlow::engine_count() const { + return static_cast(impl_->slots.size()); +} + +int64_t CosyVoice3RknnFlow::mel_channels() const { + return impl_->channels; +} + +std::vector CosyVoice3RknnFlow::run( + const float * x, + const float * mu, + const float * cond, + const float * spks, + float t, + int64_t frames) { +#if defined(ENGINE_HAVE_RKNN) + const auto start = Clock::now(); + auto & slot = impl_->pick(frames); + std::vector padded; + slot.run(x, mu, cond, spks, t, frames, padded); + // Cut the pad back out: [2, channels, frames] out of [2, channels, engine_frames]. Same row + // strides as the padding above: `frames` on the host, `engine_frames` in the engine. + std::vector out(static_cast(kCfgBatch * slot.channels * frames), 0.0F); + for (int64_t row = 0; row < kCfgBatch * slot.channels; ++row) { + std::memcpy( + out.data() + row * frames, + padded.data() + row * slot.frames, + static_cast(frames) * sizeof(float)); + } + debug::timing_log_scalar("cosyvoice3.flow.dit.rknn.step_ms", debug::elapsed_ms(start)); + debug::trace_log_scalar("cosyvoice3.flow.dit.rknn.engine_frames", slot.frames); + return out; +#else + (void)x; + (void)mu; + (void)cond; + (void)spks; + (void)t; + (void)frames; + throw std::runtime_error("CosyVoice3 flow estimator RKNN: built without ENGINE_ENABLE_RKNN"); +#endif +} + +} // namespace engine::models::cosyvoice3 diff --git a/src/models/cosyvoice3/session.cpp b/src/models/cosyvoice3/session.cpp index 3e178091a..e14f1128d 100644 --- a/src/models/cosyvoice3/session.cpp +++ b/src/models/cosyvoice3/session.cpp @@ -13,6 +13,7 @@ #include #include #include +#include namespace engine::models::cosyvoice3 { namespace { @@ -49,8 +50,27 @@ std::string instruction_text(const runtime::TaskRequest & request) { return runtime::find_option(request.options, {"instruct", "instruction"}).value_or(""); } -std::vector split_request(const runtime::TaskRequest & request) { - const int64_t text_chunk_size = +// The NPU sidecar takes a ladder of compiled engines in one option, comma-separated: which lengths +// get an engine is a deployment choice (each one is ~700 MB and covers one sequence length), and a +// list-typed session option would be a schema feature for one consumer. +std::vector split_paths(const std::string & value) { + std::vector out; + size_t start = 0; + while (start <= value.size()) { + const auto end = value.find(',', start); + const auto stop = end == std::string::npos ? value.size() : end; + if (stop > start) { + out.push_back(value.substr(start, stop - start)); + } + if (end == std::string::npos) { + break; + } + start = end + 1; + } + return out; +} + +std::vector split_request(const runtime::TaskRequest & request) { const int64_t text_chunk_size = engine::text::parse_text_chunk_size_override(request.options).value_or(kDefaultTextChunkSize); if (text_chunk_size <= 0) { throw std::runtime_error("CosyVoice3 text_chunk_size must be positive"); @@ -87,6 +107,10 @@ CosyVoice3Session::CosyVoice3Session( auto validation_options = options; // This previously advertised option never selected convolution storage. validation_options.options.erase("cosyvoice3.conv_weight_type"); + // Older embedded contracts omit exec_plan; parse_exec_plan validates its value. + if (contract_->session_option_keys.find("cosyvoice3.exec_plan") == contract_->session_option_keys.end()) { + validation_options.options.erase("cosyvoice3.exec_plan"); + } runtime::validate_spec_backed_session_options(validation_options, *contract_, kFamily, kModelName); if (task_.task != runtime::VoiceTaskKind::Tts && task_.task != runtime::VoiceTaskKind::VoiceCloning) { throw std::runtime_error("CosyVoice3 supports tts and clone tasks"); @@ -132,12 +156,27 @@ CosyVoice3Session::CosyVoice3Session( graph_arena_bytes, weight_context_bytes, storage_type); + // The Rockchip NPU sidecar for the flow estimator: one compiled engine per sequence length, + // loaded up front because an engine takes seconds to init and the ODE loop needs it per step. + // The ladder travels as the component's main artifact, comma-separated, because the roles of + // an exec_plan artifact block are a closed vocabulary while the ladder is not. + const auto exec_plan = runtime::parse_exec_plan( + options.options, + "cosyvoice3.exec_plan", + {{"flow", "ggml", + {{"ggml", {options.backend.type}, {}, {}}, + {"rknn", {}, {"main"}, {}}}}}); + const auto & flow_plan = exec_plan.at("flow"); + const std::vector rknn_flow_engines = flow_plan.impl == "rknn" + ? split_paths(flow_plan.artifacts.at("main")) + : std::vector{}; flow_ = std::make_unique( assets_, execution_context(), graph_arena_bytes, weight_context_bytes, - storage_type); + storage_type, + rknn_flow_engines); hift_ = std::make_unique( assets_, execution_context(), diff --git a/tools/rknn/prepare_cosyvoice3_rknn_flow.py b/tools/rknn/prepare_cosyvoice3_rknn_flow.py new file mode 100644 index 000000000..816bae50e --- /dev/null +++ b/tools/rknn/prepare_cosyvoice3_rknn_flow.py @@ -0,0 +1,277 @@ +#!/usr/bin/env python3 +"""Build the fixed-length RKNN Flow DiT estimator for the CosyVoice3 family. + +The runtime side (``src/models/cosyvoice3/rknn_flow.cpp``, session option +``cosyvoice3.rknn_flow``) runs one compiled RKNN estimator step in place of the ggml DiT graph, +inside the same flow-matching ODE loop. Nothing in this repository produces that graph, so this +script is its only producer: it compiles the checkpoint's own +``flow.decoder.estimator.fp32.onnx`` (from ``FunAudioLLM/Fun-CosyVoice3-0.5B-2512``, i.e. the same +weights the GGUF package was converted from) for one fixed sequence length and for a RKNPU2 target. + +The estimator is the flow decoder's ODE step: batch 2 is the classifier-free-guidance pair +``[cond, uncond]`` (the uncond branch carries zeroed ``mu``/``cond``/``spks``), and one call returns +``dphi_dt`` for both branches. Six inputs ``x [2,80,S]`` / ``mask [2,1,S]`` / ``mu [2,80,S]`` / +``t [2]`` (the raw timestep; the time-embedding MLP is inside the graph) / ``spks [2,80]`` / +``cond [2,80,S]``, one output ``estimator_out [2,80,S]``, fp32 in and out. The runtime's ggml graph +has no ``mask`` and takes the timestep embedding instead of ``t``; the sidecar supplies both. + +RKNN has no dynamic shapes, so the runtime compiles a ladder of lengths and pads a shorter +utterance up to the smallest engine that fits: ``x``/``mu``/``cond`` zero-padded on the right, +``mask`` 1 on the real frames and 0 in the pad, output cropped back. That convention is what the +padding check measures on the host: a padded run at ``S`` against an exact run at ``frames``, per +length, before any of it is paid for on a board (``--skip-check`` skips it). ``--verify-sim`` goes +one step further and runs the fp16 graph on the PC simulator against the fp32 ONNX, which is the +engine's own accuracy, measured without a board. + +Requirements (host, x86_64, dedicated venv -- the RKNN toolkit pins its own numpy/onnx): + pip install onnx onnxruntime numpy + pip install rknn-toolkit2 # PC-side converter, x86_64 only +On Python 3.12 the 2.3.2 converter additionally needs ``setuptools<81`` and an ``onnx/mapping.py`` +shim inside the venv's onnx package; see docs/build/rockchip-npu.md. + +Usage: + python3 tools/rknn/prepare_cosyvoice3_rknn_flow.py --estimator-onnx flow.decoder.estimator.fp32.onnx \ + --seq-len 200 --seq-len 500 --seq-len 1000 --output-dir models/CosyVoice3-RKNN + python3 tools/rknn/prepare_cosyvoice3_rknn_flow.py --estimator-onnx --seq-len 200 --skip-rknn + python3 tools/rknn/prepare_cosyvoice3_rknn_flow.py --estimator-onnx --seq-len 200 \ + --check-frames 106 --verify-sim +""" + +from __future__ import annotations + +import argparse +import sys +from pathlib import Path + +import numpy as np +import onnx + +MEL_CHANNELS = 80 +CFG_BATCH = 2 +# Graph order, as exported by the checkpoint. `mask` is the frame mask, `t` the raw timestep. +INPUT_NAMES = ["x", "mask", "mu", "t", "spks", "cond"] + + +def input_shapes(seq_len: int) -> list[list[int]]: + return [ + [CFG_BATCH, MEL_CHANNELS, seq_len], + [CFG_BATCH, 1, seq_len], + [CFG_BATCH, MEL_CHANNELS, seq_len], + [CFG_BATCH], + [CFG_BATCH, MEL_CHANNELS], + [CFG_BATCH, MEL_CHANNELS, seq_len], + ] + + +def graph_input_names(onnx_path: Path) -> list[str]: + model = onnx.load(str(onnx_path), load_external_data=False) + names = [i.name for i in model.graph.input] + # Initializers can appear as graph inputs in older exports; only the real feeders count. + feed = [n for n in names if n not in {i.name for i in model.graph.initializer}] + if feed != INPUT_NAMES: + raise SystemExit( + f"{onnx_path.name}: expected inputs {INPUT_NAMES} in graph order, got {feed}; " + "this is not the checkpoint's flow.decoder.estimator export" + ) + return feed + + +def pad_to(inputs: list[np.ndarray], seq_len: int) -> list[np.ndarray]: + """Right-pad a length-carrying input set to `seq_len`: zeros everywhere, mask 0 in the pad.""" + out = [] + for name, arr in zip(INPUT_NAMES, inputs): + if name in ("t", "spks"): + out.append(arr) + continue + axis = arr.ndim - 1 + pad = seq_len - arr.shape[axis] + if pad < 0: + raise SystemExit(f"{name} is longer ({arr.shape[axis]}) than the engine ({seq_len})") + # Zero pad: for x/mu/cond that is the reference port's `zero` mode, and for mask it is the + # 0 that switches the pad frames off. + out.append(arr if pad == 0 else np.pad(arr, [(0, 0), (0, 0), (0, pad)])) + return out + + +def random_inputs(seq_len: int, rng: np.random.Generator) -> list[np.ndarray]: + inputs = [] + for name, shape in zip(INPUT_NAMES, input_shapes(seq_len)): + if name == "t": + # The ODE schedule evaluates the estimator at the start of each interval, t in [0, 1). + inputs.append(rng.random(shape).astype(np.float32)) + elif name == "mask": + inputs.append(np.ones(shape, dtype=np.float32)) + else: + inputs.append(rng.standard_normal(shape).astype(np.float32)) + return inputs + + +def run(session, inputs: list[np.ndarray]) -> np.ndarray: + feeds = {name: arr for name, arr in zip(INPUT_NAMES, inputs)} + return session.run(None, feeds)[0] + + +def snr_db(reference: np.ndarray, other: np.ndarray) -> float: + power = float((reference * reference).sum()) + err = float(((reference - other) ** 2).sum()) + return 10.0 * np.log10(max(power, 1e-30) / max(err, 1e-30)) + + +def check_padding(onnx_path: Path, seq_len: int, frames: list[int], seed: int) -> None: + """Padded run at `seq_len` vs exact run at `frames`: the padding convention's own gate.""" + import onnxruntime as ort + + opts = ort.SessionOptions() + opts.log_severity_level = 3 + session = ort.InferenceSession(str(onnx_path), opts, providers=["CPUExecutionProvider"]) + rng = np.random.default_rng(seed) + print(f"check {seq_len}: padded run vs exact run (onnxruntime CPU, fp32)") + for length in frames: + if length > seq_len: + continue + inputs = random_inputs(length, rng) + exact = run(session, inputs) + padded = run(session, pad_to(inputs, seq_len))[:, :, :length] + # The frames nearest the pad are the ones the pad can contaminate: the estimator's causal + # conv-pos-embed pads on the left only, so a right pad reaches the tail through attention. + tail = padded[:, :, -min(60, length):] + tail_ref = exact[:, :, -min(60, length):] + print( + f" frames={length:4d} pad={seq_len - length:4d} " + f"snr={snr_db(exact, padded):6.1f} dB tail_snr={snr_db(tail_ref, tail):6.1f} dB " + f"max_abs={np.abs(exact - padded).max():.4f}" + ) + + +def verify_fp16( + onnx_path: Path, + seq_len: int, + frames: int, + target: str, + opt_level: int, + seed: int, +) -> None: + """The fp16 cost of the compiled graph, measured on the PC simulator. + + Built from the ONNX rather than run from the exported `.rknn`: a `load_rknn` engine cannot be + inferred on the simulator at all (`init_runtime: RKNN model that loaded by 'load_rknn' not + support inference on the simulator, please set 'target' first!`), and `load_onnx` + `build` + with the same config produces the same fp16 graph the board runs. + + Worth measuring before a board is asked to judge it: the whole utterance's mel comes out of ten + of these steps, so a per-step bias is what the reference port compensates with a hand-tuned mel + offset. + """ + import onnxruntime as ort + from rknn.api import RKNN + + opts = ort.SessionOptions() + opts.log_severity_level = 3 + session = ort.InferenceSession(str(onnx_path), opts, providers=["CPUExecutionProvider"]) + rng = np.random.default_rng(seed) + inputs = pad_to(random_inputs(frames, rng), seq_len) + reference = run(session, inputs) + + rknn = RKNN(verbose=False) + rknn.config(target_platform=target, optimization_level=opt_level) + if rknn.load_onnx(model=str(onnx_path), inputs=INPUT_NAMES, input_size_list=input_shapes(seq_len)) != 0: + raise SystemExit("load_onnx failed") + if rknn.build(do_quantization=False) != 0: + raise SystemExit("build failed") + if rknn.init_runtime() != 0: + raise SystemExit("init_runtime (PC simulator) failed") + got = rknn.inference(inputs=inputs)[0] + rknn.release() + got = np.asarray(got).reshape(reference.shape) + # The frames nearest the pad are where an fp16 engine's error concentrates, same as the + # padding gate above, and the ODE loop feeds its own output back in ten times. + tail = min(60, frames) + print( + f"verify fp16 (seq {seq_len}) vs fp32 ONNX (PC simulator): frames={frames} pad={seq_len - frames} " + f"snr={snr_db(reference, got):6.1f} dB " + f"tail_snr={snr_db(reference[:, :, -tail:], got[:, :, -tail:]):6.1f} dB " + f"max_abs={np.abs(reference - got).max():.4f} " + f"mean={reference.mean():+.5f}/{got.mean():+.5f}" + ) + + +def convert_rknn( + onnx_path: Path, + output: Path, + target: str, + opt_level: int, + seq_len: int, +) -> None: + from rknn.api import RKNN + + names = graph_input_names(onnx_path) + rknn = RKNN(verbose=False) + # FP16 with no quantization: the flow estimator is the stage the reference port keeps on the NPU, + # and quantizing it (int8) is what pushes the mel off enough to need a hand-tuned offset. + rknn.config(target_platform=target, optimization_level=opt_level) + if rknn.load_onnx(model=str(onnx_path), inputs=names, input_size_list=input_shapes(seq_len)) != 0: + raise SystemExit("load_onnx failed") + sys.path.insert(0, str(Path(__file__).resolve().parent)) + from rknn_build_log import rknn_build_gate + + with rknn_build_gate(f"build {output.name}", output.with_suffix(".build.log")): + built = rknn.build(do_quantization=False) + if built != 0: + raise SystemExit("build failed (see the rknn log above)") + if rknn.export_rknn(str(output)) != 0: + raise SystemExit("export_rknn failed") + rknn.release() + print(f"wrote {output} ({output.stat().st_size / 1e6:.1f} MB, seq_len {seq_len}, " + f"{target}, optimization_level {opt_level})") + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument( + "--estimator-onnx", type=Path, required=True, + help="flow.decoder.estimator.fp32.onnx from the Fun-CosyVoice3 checkpoint") + parser.add_argument( + "--seq-len", type=int, action="append", default=None, + help="sequence length to compile (repeatable); the runtime pads up to the smallest fit") + parser.add_argument("--target", default="rk3588") + parser.add_argument("--opt-level", type=int, default=3) + parser.add_argument("--output-dir", type=Path, default=Path("models/CosyVoice3-RKNN")) + parser.add_argument( + "--check-frames", type=int, action="append", default=None, + help="utterance lengths to check the padding at, per engine (default: a short and a " + "near-full length)") + parser.add_argument("--seed", type=int, default=1986, help="matches the runtime's flow seed") + parser.add_argument("--skip-check", action="store_true") + parser.add_argument("--skip-rknn", action="store_true", help="check only") + parser.add_argument( + "--verify-sim", action="store_true", + help="build the first --seq-len and run it on the PC simulator against the fp32 ONNX at " + "the first --check-frames; slow, the simulator runs the whole estimator") + args = parser.parse_args() + + seq_lens = args.seq_len or [200, 500, 1000] + if args.verify_sim: + seq_len = seq_lens[0] + frames = (args.check_frames or [seq_len // 2])[0] + verify_fp16(args.estimator_onnx, seq_len, frames, args.target, args.opt_level, args.seed) + return 0 + if not args.skip_check: + for seq_len in seq_lens: + frames = args.check_frames or [max(1, seq_len // 2), max(1, seq_len - 1)] + check_padding(args.estimator_onnx, seq_len, frames, args.seed) + if args.skip_rknn: + return 0 + args.output_dir.mkdir(parents=True, exist_ok=True) + for seq_len in seq_lens: + convert_rknn( + args.estimator_onnx, + args.output_dir / f"flow_estimator_seq{seq_len}.rknn", + args.target, + args.opt_level, + seq_len, + ) + return 0 + + +if __name__ == "__main__": + sys.exit(main())