From e5432c845860661a9b78eabd3690e569fded8720 Mon Sep 17 00:00:00 2001 From: 0xShug0 <231717474+0xShug0@users.noreply.github.com> Date: Wed, 7 Oct 2026 17:48:52 -0400 Subject: [PATCH 1/2] Add Hviske v6 Danish ASR support --- CMakeLists.txt | 13 + README.md | 1 + docs/model_licenses.md | 1 + docs/models/hviske_asr_v6.md | 61 +++ include/engine/models/hviske_asr_v6/assets.h | 55 +++ include/engine/models/hviske_asr_v6/decoder.h | 30 ++ include/engine/models/hviske_asr_v6/encoder.h | 24 + .../engine/models/hviske_asr_v6/frontend.h | 26 + include/engine/models/hviske_asr_v6/session.h | 7 + model_specs/hviske_asr_v6.json | 153 ++++++ src/models/hviske_asr_v6/assets.cpp | 53 +++ src/models/hviske_asr_v6/decoder.cpp | 450 ++++++++++++++++++ src/models/hviske_asr_v6/encoder.cpp | 230 +++++++++ src/models/hviske_asr_v6/frontend.cpp | 47 ++ src/models/hviske_asr_v6/session.cpp | 190 ++++++++ .../audiocpp_cli/audiocpp_cli_path_cases.json | 30 ++ 16 files changed, 1371 insertions(+) create mode 100644 docs/models/hviske_asr_v6.md create mode 100644 include/engine/models/hviske_asr_v6/assets.h create mode 100644 include/engine/models/hviske_asr_v6/decoder.h create mode 100644 include/engine/models/hviske_asr_v6/encoder.h create mode 100644 include/engine/models/hviske_asr_v6/frontend.h create mode 100644 include/engine/models/hviske_asr_v6/session.h create mode 100644 model_specs/hviske_asr_v6.json create mode 100644 src/models/hviske_asr_v6/assets.cpp create mode 100644 src/models/hviske_asr_v6/decoder.cpp create mode 100644 src/models/hviske_asr_v6/encoder.cpp create mode 100644 src/models/hviske_asr_v6/frontend.cpp create mode 100644 src/models/hviske_asr_v6/session.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index e32d4addd..e393a0830 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1511,6 +1511,19 @@ audiocpp_add_model(hviske_asr engine::models::hviske_asr::make_hviske_asr_loader ) +audiocpp_add_model(hviske_asr_v6 + SOURCES + src/models/hviske_asr_v6/assets.cpp + src/models/hviske_asr_v6/frontend.cpp + src/models/hviske_asr_v6/encoder.cpp + src/models/hviske_asr_v6/decoder.cpp + src/models/hviske_asr_v6/session.cpp + INCLUDES + engine/models/hviske_asr_v6/session.h + LOADERS + engine::models::hviske_asr_v6::make_hviske_asr_v6_loader +) + audiocpp_add_model(irodori_tts SOURCES src/models/irodori_tts/assets.cpp diff --git a/README.md b/README.md index 8624bf77f..6fdb10471 100644 --- a/README.md +++ b/README.md @@ -129,6 +129,7 @@ Model weights keep the license of their original release, which is separate from | **fun_asr_nano** | ASR | auto, zh, en, ja | Fun-ASR-Nano-2512 | GGUF 16/Q8 | | **higgs_audio_stt** | ASR | en | Higgs Audio v3 STT | GGUF 16/Q8, Stream | | **hviske_asr** | ASR | da | Hviske v5.3 | GGUF Q8 | +| **[hviske_asr_v6](docs/models/hviske_asr_v6.md)** | ASR | da | Hviske v6 | GGUF BF16 / Q8 | | **index_echo** | ASR, Translate | zh → en, es, ja | Index-Echo-S2TT 2B
Index-Echo-S2TT 9B | GGUF original/Q8_0/Q4_K | | **marblenet_vad** | VAD | lang agnostic | MarbleNet VAD | Bundled | | **pulsevad** | VAD | lang agnostic | PulseVAD 2.1K Student / 81K Teacher | GGUF F32 | diff --git a/docs/model_licenses.md b/docs/model_licenses.md index 12a5300c9..128225583 100644 --- a/docs/model_licenses.md +++ b/docs/model_licenses.md @@ -60,6 +60,7 @@ Licenses change. If a row is wrong or out of date, please open a PR. | `htdemucs` | [facebookresearch/demucs](https://github.com/facebookresearch/demucs) | MIT | Yes | | 2026-09-21 | | `htdemucs_6stems` | [facebookresearch/demucs `5c90dfd2`](https://dl.fbaipublicfiles.com/demucs/hybrid_transformer/5c90dfd2-34c22ccb.th) | MIT | Yes | | 2026-09-26 | | `hviske_asr` | [syvai/hviske-v5.3](https://huggingface.co/syvai/hviske-v5.3) | CC-BY-NC-4.0 | No | | 2026-09-21 | +| `hviske_asr_v6` | [syvai/hviske-v6](https://huggingface.co/syvai/hviske-v6) | CC-BY-NC-4.0 | No | Retain upstream NOTICE and third-party license notices. | 2026-10-07 | | `index_echo` | [IndexTeam/Index-Echo-S2TT-2B](https://huggingface.co/IndexTeam/Index-Echo-S2TT-2B)
[IndexTeam/Index-Echo-S2TT-9B](https://huggingface.co/IndexTeam/Index-Echo-S2TT-9B) | Apache-2.0 | Yes | S2TT checkpoints only. | 2026-10-01 | | `index_tts2` | [IndexTeam/IndexTTS-2](https://huggingface.co/IndexTeam/IndexTTS-2)
[IndexTeam/IndexTTS-2.5](https://huggingface.co/IndexTeam/IndexTTS-2.5) | [bilibili Model Use License](https://huggingface.co/IndexTeam/IndexTTS-2.5/blob/main/LICENSE) | Conditional | A separate license is needed above RMB 1 billion yearly revenue or 100 million monthly users. The model and its outputs may not be used to improve other commercial AI models. | 2026-09-21 | | `inflect_v2` | [owensong/Inflect-Micro-v2](https://huggingface.co/owensong/Inflect-Micro-v2)
[owensong/Inflect-Nano-v2](https://huggingface.co/owensong/Inflect-Nano-v2) | Apache-2.0 | Yes | | 2026-09-21 | diff --git a/docs/models/hviske_asr_v6.md b/docs/models/hviske_asr_v6.md new file mode 100644 index 000000000..f253d88be --- /dev/null +++ b/docs/models/hviske_asr_v6.md @@ -0,0 +1,61 @@ +# Hviske v6 + +`hviske_asr_v6` is a separate family from the Conformer-based `hviske_asr`. +It uses a Whisper encoder with partial rotary positions, four-frame stacking, +an RMSNorm/linear projector, and a Qwen3 causal decoder. + +Upstream: [syvai/hviske-v6](https://huggingface.co/syvai/hviske-v6). +The checkpoint is Danish-only and licensed under CC BY-NC 4.0; see the +upstream model card and notices for the underlying models' licenses. + +## Usage + +```bash +audiocpp_cli --task asr --family hviske_asr_v6 \ + --model /path/to/hviske-v6-bf16.gguf --backend cuda \ + --audio danish.wav --log +``` + +The model supports offline transcription, without timestamps, diarization, or +native streaming. Input is mixed to mono and resampled to 16 kHz. Recordings +longer than 30 seconds are split at silence boundaries into chunks of at most +28 seconds, following the upstream transcription wrapper. + +## Request Options + +| Option | Default | Meaning | +|---|---|---| +| `max_tokens` | `440` | Per-chunk generation limit, further capped to eight tokens per second plus twelve. | +| `num_beams` | `2` | Beam-search width; `1` selects greedy decoding. | +| `length_penalty` | `1.0` | Beam-score length-normalization exponent. | +| `cased` | Unset | Request cased or lowercase output. | +| `punctuated` | Unset | Request or suppress punctuation. | + +Set both `cased` and `punctuated` to override automatic formatting. With either +option omitted, the model selects formatting from the audio, matching upstream. +These are model conditioning tokens, not post-processing guarantees. + +```bash +--request-option cased=true --request-option punctuated=true +``` + +## Session Options + +| Option | Default | Meaning | +|---|---|---| +| `hviske_asr_v6.weight_type` | `native` | Preserve checkpoint tensor types, or explicitly override weight storage. | + +## Conversion + +No model-specific checkpoint rewrite is required. Use the standard GGUF tool: + +```bash +audiocpp_gguf --family hviske_asr_v6 \ + --model-spec model_specs/hviske_asr_v6.json \ + --input weights=/path/to/hviske-v6/model.safetensors \ + --root /path/to/hviske-v6 --type orig \ + --output /path/to/hviske-v6-bf16.gguf +``` + +The original checkpoint tensors are BF16. The converter embeds the config, +tokenizer, and model spec for standalone loading. diff --git a/include/engine/models/hviske_asr_v6/assets.h b/include/engine/models/hviske_asr_v6/assets.h new file mode 100644 index 000000000..92838315d --- /dev/null +++ b/include/engine/models/hviske_asr_v6/assets.h @@ -0,0 +1,55 @@ +#pragma once + +#include "engine/framework/assets/resource_bundle.h" + +#include +#include +#include + +namespace engine::models::hviske_asr_v6 { + +struct HviskeV6WhisperConfig { + int64_t mel_bins = 128; + int64_t hidden_size = 1280; + int64_t intermediate_size = 5120; + int64_t layers = 32; + int64_t heads = 20; + int64_t rotary_dim = 56; + float rope_theta = 10000.0f; +}; + +struct HviskeV6Qwen3Config { + int64_t vocab_size = 151936; + int64_t hidden_size = 1024; + int64_t intermediate_size = 3072; + int64_t layers = 28; + int64_t heads = 16; + int64_t kv_heads = 8; + int64_t head_dim = 128; + int64_t max_positions = 32768; + float rms_norm_eps = 1.0e-6f; + float rope_theta = 1000000.0f; +}; + +struct HviskeV6Config { + HviskeV6WhisperConfig encoder; + HviskeV6Qwen3Config decoder; + int64_t frame_stack = 4; + int32_t bos_token_id = 151644; + int32_t eos_token_id = 151645; + int32_t pad_token_id = 151643; + int32_t cased_token_id = 151670; + int32_t nocase_token_id = 151669; + int32_t punctuation_token_id = 151672; + int32_t no_punctuation_token_id = 151671; +}; + +struct HviskeV6Assets { + assets::ResourceBundle resources; + HviskeV6Config config; + std::shared_ptr weights; +}; + +std::shared_ptr load_hviske_v6_assets(const std::filesystem::path & path); + +} // namespace engine::models::hviske_asr_v6 diff --git a/include/engine/models/hviske_asr_v6/decoder.h b/include/engine/models/hviske_asr_v6/decoder.h new file mode 100644 index 000000000..83b6c2cd4 --- /dev/null +++ b/include/engine/models/hviske_asr_v6/decoder.h @@ -0,0 +1,30 @@ +#pragma once + +#include "engine/framework/core/execution_context.h" +#include "engine/models/hviske_asr_v6/assets.h" + +#include +#include + +namespace engine::models::hviske_asr_v6 { + +struct HviskeV6DecodingOptions { + int64_t max_tokens = 440; + int64_t num_beams = 2; + float length_penalty = 1.0f; + std::vector control_tokens; +}; + +class HviskeV6Qwen3DecoderRuntime { +public: + HviskeV6Qwen3DecoderRuntime(std::shared_ptr assets, + core::ExecutionContext & execution, assets::TensorStorageType storage); + ~HviskeV6Qwen3DecoderRuntime(); + std::vector generate(const std::vector & audio, const HviskeV6DecodingOptions & options); + +private: + struct Impl; + std::unique_ptr impl_; +}; + +} // namespace engine::models::hviske_asr_v6 diff --git a/include/engine/models/hviske_asr_v6/encoder.h b/include/engine/models/hviske_asr_v6/encoder.h new file mode 100644 index 000000000..930c7ac25 --- /dev/null +++ b/include/engine/models/hviske_asr_v6/encoder.h @@ -0,0 +1,24 @@ +#pragma once + +#include "engine/framework/core/execution_context.h" +#include "engine/models/hviske_asr_v6/assets.h" +#include "engine/models/hviske_asr_v6/frontend.h" + +#include +#include + +namespace engine::models::hviske_asr_v6 { + +class HviskeV6WhisperRoPEEncoderRuntime { +public: + HviskeV6WhisperRoPEEncoderRuntime(std::shared_ptr assets, + core::ExecutionContext & execution, assets::TensorStorageType storage); + ~HviskeV6WhisperRoPEEncoderRuntime(); + std::vector encode(const HviskeV6Features & features); + +private: + struct Impl; + std::unique_ptr impl_; +}; + +} // namespace engine::models::hviske_asr_v6 diff --git a/include/engine/models/hviske_asr_v6/frontend.h b/include/engine/models/hviske_asr_v6/frontend.h new file mode 100644 index 000000000..defcfcba8 --- /dev/null +++ b/include/engine/models/hviske_asr_v6/frontend.h @@ -0,0 +1,26 @@ +#pragma once + +#include "engine/framework/audio/mel_spectrogram_frontend.h" + +#include +#include +#include + +namespace engine::models::hviske_asr_v6 { + +struct HviskeV6Features { + std::vector values; + int64_t frames = 0; + int64_t valid_frames = 0; +}; + +class HviskeV6WhisperFrontend { +public: + explicit HviskeV6WhisperFrontend(int64_t mel_bins); + HviskeV6Features extract(const std::vector & mono_16k, size_t threads) const; + +private: + std::shared_ptr mel_; +}; + +} // namespace engine::models::hviske_asr_v6 diff --git a/include/engine/models/hviske_asr_v6/session.h b/include/engine/models/hviske_asr_v6/session.h new file mode 100644 index 000000000..15326b408 --- /dev/null +++ b/include/engine/models/hviske_asr_v6/session.h @@ -0,0 +1,7 @@ +#pragma once + +#include "engine/framework/runtime/model.h" + +namespace engine::models::hviske_asr_v6 { +std::shared_ptr make_hviske_asr_v6_loader(); +} diff --git a/model_specs/hviske_asr_v6.json b/model_specs/hviske_asr_v6.json new file mode 100644 index 000000000..efd8679e3 --- /dev/null +++ b/model_specs/hviske_asr_v6.json @@ -0,0 +1,153 @@ +{ + "schema_version": 1, + "family": "hviske_asr_v6", + "display_name": "Hviske v6", + "description": "Danish speech recognition with a rotary Whisper encoder and Qwen3 decoder.", + "category": "asr", + "status": "supported", + "tasks": [ + "asr" + ], + "modes": [ + "offline" + ], + "languages": [ + "da" + ], + "capabilities": {}, + "runtime": { + "tags": [ + "gguf" + ] + }, + "options": { + "request": [ + { + "name": "max_tokens", + "type": "int", + "description": "Maximum generated tokens per chunk, also capped to eight tokens per audio second plus twelve.", + "required": false, + "default": 440, + "min": 1 + }, + { + "name": "num_beams", + "type": "int", + "description": "Beam-search width. One selects greedy decoding.", + "required": false, + "default": 2, + "min": 1 + }, + { + "name": "length_penalty", + "type": "float", + "description": "Exponent used to normalize beam scores by generated length.", + "required": false, + "default": 1.0 + }, + { + "name": "cased", + "type": "bool", + "description": "Request casing. Both cased and punctuated must be provided to override automatic formatting.", + "required": false + }, + { + "name": "punctuated", + "type": "bool", + "description": "Request punctuation. Both cased and punctuated must be provided to override automatic formatting.", + "required": false + } + ], + "session": [ + { + "name": "weight_type", + "type": "enum", + "preset": "weight_type_full", + "description": "Weight storage override; native preserves the supplied checkpoint types.", + "required": false, + "default": "native" + } + ], + "load": [] + }, + "packages": [ + { + "id": "hviske_asr_v6_bf16", + "display_name": "Hviske v6 BF16 GGUF", + "default": true, + "format": "gguf", + "precision": "bf16", + "target_directory": "Hviske-v6-GGUF", + "files": [ + "Hviske-v6-GGUF/hviske-v6-bf16.gguf" + ], + "strip_prefix": "Hviske-v6-GGUF", + "download": { + "kind": "huggingface_snapshot", + "repo": "audio-cpp/audio.cpp-gguf" + } + }, + { + "id": "hviske_asr_v6_q8_0", + "display_name": "Hviske v6 Q8_0 GGUF", + "format": "gguf", + "precision": "q8_0", + "target_directory": "Hviske-v6-GGUF", + "files": [ + "Hviske-v6-GGUF/hviske-v6-q8_0.gguf" + ], + "strip_prefix": "Hviske-v6-GGUF", + "download": { + "kind": "huggingface_snapshot", + "repo": "audio-cpp/audio.cpp-gguf" + } + } + ], + "dependencies": [], + "ui": { + "recommended_package": "hviske_asr_v6_bf16", + "tags": [ + "ASR", + "GGUF" + ], + "docs": [ + "docs/models/hviske_asr_v6.md" + ] + }, + "sources": [ + { + "format": "safetensors", + "roots": { + "model": "." + }, + "files": { + "config": "model:config.json", + "tokenizer_config": "model:tokenizer_config.json", + "tokenizer_json": "model:tokenizer.json" + }, + "tensors": { + "weights": { + "source": "model:model.safetensors" + } + } + }, + { + "format": "gguf", + "roots": { + "model": ".", + "weights": "$gguf" + }, + "files": { + "config": "model:config.json", + "tokenizer_config": "model:tokenizer_config.json", + "tokenizer_json": "model:tokenizer.json" + }, + "tensors": { + "weights": { + "source": "weights:", + "prefix": "weights" + } + } + } + ] +} diff --git a/src/models/hviske_asr_v6/assets.cpp b/src/models/hviske_asr_v6/assets.cpp new file mode 100644 index 000000000..19f17c4ce --- /dev/null +++ b/src/models/hviske_asr_v6/assets.cpp @@ -0,0 +1,53 @@ +#include "engine/models/hviske_asr_v6/assets.h" + +#include "engine/framework/io/json.h" +#include "engine/framework/model_spec/package.h" + +#include +#include + +namespace engine::models::hviske_asr_v6 { + +std::shared_ptr load_hviske_v6_assets(const std::filesystem::path & path) { + auto result = std::make_shared(); + result->resources = model_spec::load_resource_bundle_for_family(path, "hviske_asr_v6"); + const auto root = result->resources.parse_json("config"); + if (root.require("model_type").as_string() != "whisper_qwen") { + throw std::runtime_error("Hviske v6 requires a whisper_qwen checkpoint"); + } + auto & config = result->config; + const auto & whisper = root.require("whisper"); + auto & enc = config.encoder; + enc.mel_bins = whisper.require("num_mel_bins").as_i64(); + enc.hidden_size = whisper.require("d_model").as_i64(); + enc.intermediate_size = whisper.require("encoder_ffn_dim").as_i64(); + enc.layers = whisper.require("encoder_layers").as_i64(); + enc.heads = whisper.require("encoder_attention_heads").as_i64(); + enc.rotary_dim = whisper.require("encoder_rotary_dim").as_i64(); + enc.rope_theta = io::json::require_f32(root, "rope_theta"); + const auto & qwen = root.require("qwen"); + auto & dec = config.decoder; + dec.vocab_size = qwen.require("vocab_size").as_i64(); + dec.hidden_size = qwen.require("hidden_size").as_i64(); + dec.intermediate_size = qwen.require("intermediate_size").as_i64(); + dec.layers = qwen.require("num_hidden_layers").as_i64(); + dec.heads = qwen.require("num_attention_heads").as_i64(); + dec.kv_heads = qwen.require("num_key_value_heads").as_i64(); + dec.head_dim = qwen.require("head_dim").as_i64(); + dec.max_positions = qwen.require("max_position_embeddings").as_i64(); + dec.rms_norm_eps = io::json::require_f32(qwen, "rms_norm_eps"); + dec.rope_theta = io::json::require_f32(qwen, "rope_theta"); + config.frame_stack = root.require("stack").as_i64(); + config.bos_token_id = static_cast(root.require("bos_token_id").as_i64()); + config.eos_token_id = static_cast(root.require("eos_token_id").as_i64()); + config.pad_token_id = static_cast(root.require("pad_token_id").as_i64()); + const auto & style = root.require("style_tokens"); + config.cased_token_id = static_cast(style.require("<|cased|>").as_i64()); + config.nocase_token_id = static_cast(style.require("<|nocase|>").as_i64()); + config.punctuation_token_id = static_cast(style.require("<|punc|>").as_i64()); + config.no_punctuation_token_id = static_cast(style.require("<|nopunc|>").as_i64()); + result->weights = result->resources.open_tensor_source("weights"); + return result; +} + +} // namespace engine::models::hviske_asr_v6 diff --git a/src/models/hviske_asr_v6/decoder.cpp b/src/models/hviske_asr_v6/decoder.cpp new file mode 100644 index 000000000..42d948bb6 --- /dev/null +++ b/src/models/hviske_asr_v6/decoder.cpp @@ -0,0 +1,450 @@ +#include "engine/models/hviske_asr_v6/decoder.h" + +#include "engine/framework/core/backend_weight_store.h" +#include "engine/framework/debug/profiler.h" +#include "engine/framework/modules/lookup_modules.h" +#include "engine/framework/modules/optimizations/fast_kv_modules.h" +#include "engine/framework/modules/packed_linear_weights.h" +#include "engine/framework/modules/primitive_modules.h" +#include "engine/framework/modules/structural_modules.h" +#include "engine/framework/modules/transformers/causal_decoder_runtime.h" +#include "engine/framework/modules/weight_binding.h" + +#include + +#include +#include +#include +#include +#include +#include +#include + +namespace engine::models::hviske_asr_v6 { +namespace { +constexpr size_t kContextBytes = 4 * 1024 * 1024; +struct ContextDeleter { + void operator()(ggml_context * p) const { ggml_free(p); } +}; +struct AllocatorDeleter { + void operator()(ggml_gallocr_t p) const { ggml_gallocr_free(p); } +}; + +class HviskeV6Qwen3BeamGraph { +public: + HviskeV6Qwen3BeamGraph(core::ExecutionContext & execution, const modules::CausalDecoderConfig & config, + const modules::CausalDecoderRuntimeWeights & weights, int64_t capacity, int64_t batch) + : execution_(execution), capacity_(capacity), layers_(config.stack.layers), vocab_(config.logits_size), batch_(batch) { + context_.reset(ggml_init({kContextBytes, nullptr, true})); + core::ModuleBuildContext ctx{context_.get(), "hviske_asr_v6.decoder", execution.backend_type()}; + token_ = core::make_tensor(ctx, GGML_TYPE_I32, core::TensorShape::from_dims({batch, 1})); + position_ = core::make_tensor(ctx, GGML_TYPE_I32, core::TensorShape::from_dims({1})); + slot_ = core::make_tensor(ctx, GGML_TYPE_I32, core::TensorShape::from_dims({batch})); + mask_ = core::make_tensor(ctx, GGML_TYPE_F16, core::TensorShape::from_dims({batch, 1, 1, capacity})); + ggml_set_input(token_.tensor); + ggml_set_input(position_.tensor); + ggml_set_input(slot_.tensor); + ggml_set_input(mask_.tensor); + auto x = modules::EmbeddingModule({vocab_, config.stack.hidden_size}).build(ctx, token_, weights.token_embedding); + graph_ = ggml_new_graph_custom(ctx.ggml, 8192, false); + std::vector> keys(batch), values(batch); + const modules::DecoderLayerModule layer(modules::decoder_layer_config_from_stack(config.stack)); + for (int64_t i = 0; i < layers_; ++i) { + const auto shape = core::TensorShape::from_dims( + {batch, capacity, config.stack.num_key_value_heads, config.stack.head_dim}); + auto key = core::make_tensor(ctx, config.static_cache_type, shape); + auto value = core::make_tensor(ctx, config.static_cache_type, shape); + ggml_set_input(key.tensor); + ggml_set_output(key.tensor); + ggml_set_input(value.tensor); + ggml_set_output(value.tensor); + x = layer.build_with_static_cache_tail_batched(ctx, graph_, x, position_, weights.stack.layers[i], + key, value, slot_, mask_).output; + for (int64_t b = 0; b < batch; ++b) { + keys[b].push_back(modules::SliceModule({0, b, 1}).build(ctx, key)); + values[b].push_back(modules::SliceModule({0, b, 1}).build(ctx, value)); + } + } + runtime::TransformerKVCacheOptions cache_options; + cache_options.allow_f16_storage = config.static_cache_type == GGML_TYPE_F16; + cache_options.allow_bf16_storage = config.static_cache_type == GGML_TYPE_BF16; + cache_options.lazy_import_scratch = true; + for (int64_t b = 0; b < batch; ++b) { + caches_.emplace_back(capacity, config.stack.num_key_value_heads * config.stack.head_dim, + std::move(keys[b]), std::move(values[b]), cache_options); + } + x = modules::RMSNormModule({config.stack.hidden_size, config.stack.rms_norm_eps, true, false}) + .build(ctx, x, weights.final_norm); + output_ = modules::LinearModule({config.stack.hidden_size, vocab_, false}).build(ctx, x, *weights.lm_head); + ggml_set_output(output_.tensor); + ggml_build_forward_expand(graph_, output_.tensor); + allocator_.reset(ggml_gallocr_new(ggml_backend_get_default_buffer_type(execution.backend()))); + if (!ggml_gallocr_alloc_graph(allocator_.get(), graph_)) { + throw std::runtime_error("Hviske v6 beam graph allocation failed"); + } + for (auto & cache : caches_) { + for (int64_t i = 0; i < layers_; ++i) { + if (ggml_backend_view_init(cache.key_tensor(i).tensor) != GGML_STATUS_SUCCESS || + ggml_backend_view_init(cache.value_tensor(i).tensor) != GGML_STATUS_SUCCESS) { + throw std::runtime_error("Hviske v6 beam cache view initialization failed"); + } + } + } + core::prepare_host_graph_plan(execution, graph_, plan_); + } + ~HviskeV6Qwen3BeamGraph() { + core::release_backend_graph_resources(execution_.backend(), prefill_graph_); + prefill_plan_.reset(); + core::release_backend_graph_resources(execution_.backend(), graph_); + plan_.reset(); + } + int64_t capacity() const { return capacity_; } + int64_t batch_size() const { return batch_; } + std::vector prefill(const std::vector & input, int64_t steps, + const modules::CausalDecoderConfig & config, const modules::CausalDecoderRuntimeWeights & weights) { + if (!prefill_graph_ || prefill_steps_ != steps) { + core::release_backend_graph_resources(execution_.backend(), prefill_graph_); + prefill_plan_.reset(); + prefill_graph_ = nullptr; + prefill_allocator_.reset(); + prefill_context_.reset(ggml_init({kContextBytes, nullptr, true})); + core::ModuleBuildContext ctx{prefill_context_.get(), "hviske_asr_v6.decoder.prefill", execution_.backend_type()}; + prefill_input_ = core::make_tensor(ctx, GGML_TYPE_F32, + core::TensorShape::from_dims({1, steps, config.stack.hidden_size})); + prefill_positions_ = core::make_tensor(ctx, GGML_TYPE_I32, core::TensorShape::from_dims({steps})); + prefill_mask_ = core::make_tensor(ctx, GGML_TYPE_F16, core::TensorShape::from_dims({1, 1, steps, steps})); + ggml_set_input(prefill_input_.tensor); + ggml_set_input(prefill_positions_.tensor); + ggml_set_input(prefill_mask_.tensor); + const auto out = modules::CausalDecoderModule(config).build(ctx, prefill_input_, prefill_positions_, + {weights.stack, weights.final_norm, *weights.lm_head}, std::nullopt, prefill_mask_); + prefill_graph_ = ggml_new_graph_custom(ctx.ggml, 8192, false); + const modules::FastKVSetRowsModule write_cache({modules::FastKVSetRowsMode::BackendViewOptimized}); + // Populate the first beam directly; beam selection forks it on the backend. + for (int64_t i = 0; i < layers_; ++i) { + const auto & state = out.state.layers[i]; + auto key = write_cache.build_block(ctx, caches_.front().key_tensor(i), *state.key, prefill_positions_); + auto value = write_cache.build_block(ctx, caches_.front().value_tensor(i), *state.value, prefill_positions_); + ggml_build_forward_expand(prefill_graph_, key.tensor); + ggml_build_forward_expand(prefill_graph_, value.tensor); + } + prefill_output_ = out.logits; + ggml_set_output(prefill_output_.tensor); + ggml_build_forward_expand(prefill_graph_, prefill_output_.tensor); + prefill_allocator_.reset(ggml_gallocr_new(ggml_backend_get_default_buffer_type(execution_.backend()))); + if (!ggml_gallocr_alloc_graph(prefill_allocator_.get(), prefill_graph_)) { + throw std::runtime_error("Hviske v6 prefill graph allocation failed"); + } + core::prepare_host_graph_plan(execution_, prefill_graph_, prefill_plan_); + prefill_steps_ = steps; + } + caches_.front().clear_on_backend(); + std::vector positions(steps); + std::iota(positions.begin(), positions.end(), 0); + const auto mask = modules::causal_prefill_mask_values(1, steps); + ggml_backend_tensor_set(prefill_input_.tensor, input.data(), 0, input.size() * sizeof(float)); + ggml_backend_tensor_set(prefill_positions_.tensor, positions.data(), 0, positions.size() * sizeof(int32_t)); + ggml_backend_tensor_set(prefill_mask_.tensor, mask.data(), 0, mask.size() * sizeof(ggml_fp16_t)); + if (core::compute_graph(execution_, prefill_graph_, prefill_plan_, "hviske_asr_v6.decoder.prefill") != GGML_STATUS_SUCCESS) { + throw std::runtime_error("Hviske v6 prefill execution failed"); + } + std::vector logits(vocab_); + ggml_backend_tensor_get(prefill_output_.tensor, logits.data(), 0, logits.size() * sizeof(float)); + return logits; + } + void copy_from(size_t destination, size_t source) { + for (int64_t i = 0; i < layers_; ++i) { + ggml_backend_tensor_copy(caches_[source].key_tensor(i).tensor, caches_[destination].key_tensor(i).tensor); + ggml_backend_tensor_copy(caches_[source].value_tensor(i).tensor, caches_[destination].value_tensor(i).tensor); + } + } + const std::vector & step(const std::vector & tokens, int32_t position) { + ggml_backend_tensor_set(token_.tensor, tokens.data(), 0, tokens.size() * sizeof(int32_t)); + ggml_backend_tensor_set(position_.tensor, &position, 0, sizeof(position)); + slots_.resize(batch_); + for (int64_t b = 0; b < batch_; ++b) slots_[b] = static_cast(b * capacity_ + position); + ggml_backend_tensor_set(slot_.tensor, slots_.data(), 0, slots_.size() * sizeof(int32_t)); + modules::write_decoder_batched_cached_step_mask(mask_.tensor, mask_values_, batch_, capacity_, position, position); + if (core::compute_graph(execution_, graph_, plan_, "hviske_asr_v6.decoder") != GGML_STATUS_SUCCESS) { + throw std::runtime_error("Hviske v6 decoder execution failed"); + } + logits_.resize(static_cast(vocab_ * batch_)); + ggml_backend_tensor_get(output_.tensor, logits_.data(), 0, logits_.size() * sizeof(float)); + return logits_; + } + +private: + core::ExecutionContext & execution_; + int64_t capacity_, layers_, vocab_, batch_; + std::unique_ptr context_; + std::unique_ptr allocator_; + core::HostGraphPlan plan_; + ggml_cgraph * graph_ = nullptr; + core::TensorValue token_, position_, slot_, mask_, output_; + std::vector caches_; + std::vector slots_; + std::vector mask_values_; + std::vector logits_; + std::unique_ptr prefill_context_; + std::unique_ptr prefill_allocator_; + core::HostGraphPlan prefill_plan_; + ggml_cgraph * prefill_graph_ = nullptr; + core::TensorValue prefill_input_, prefill_positions_, prefill_mask_, prefill_output_; + int64_t prefill_steps_ = 0; +}; +} // namespace + +struct HviskeV6Qwen3DecoderRuntime::Impl { + Impl(std::shared_ptr assets, core::ExecutionContext & execution, + assets::TensorStorageType storage) + : assets_(std::move(assets)), execution_(execution), + store_(execution.backend(), execution.backend_type(), "hviske_asr_v6.decoder.weights", kContextBytes) { + const auto & c = assets_->config.decoder; + const auto & source = *assets_->weights; + bool packed_qkv = true, packed_gate_up = true; + if (storage == assets::TensorStorageType::Native) { + for (int64_t i = 0; i < c.layers; ++i) { + const auto p = "llm.model.layers." + std::to_string(i); + const auto q_type = source.require_metadata(p + ".self_attn.q_proj.weight").dtype; + packed_qkv = packed_qkv && q_type == source.require_metadata(p + ".self_attn.k_proj.weight").dtype && + q_type == source.require_metadata(p + ".self_attn.v_proj.weight").dtype; + packed_gate_up = packed_gate_up && source.require_metadata(p + ".mlp.gate_proj.weight").dtype == + source.require_metadata(p + ".mlp.up_proj.weight").dtype; + } + } + weights_.token_embedding = store_.load_tensor(source, "llm.model.embed_tokens.weight", storage, + {c.vocab_size, c.hidden_size}); + for (int64_t i = 0; i < c.layers; ++i) { + const auto p = "llm.model.layers." + std::to_string(i); + modules::DecoderLayerWeights layer; + layer.input_norm = modules::binding::norm_weight_from_source(store_, source, p + ".input_layernorm", c.hidden_size); + layer.post_norm = modules::binding::norm_weight_from_source(store_, source, p + ".post_attention_layernorm", c.hidden_size); + layer.q_norm = modules::binding::norm_weight_from_source(store_, source, p + ".self_attn.q_norm", c.head_dim); + layer.k_norm = modules::binding::norm_weight_from_source(store_, source, p + ".self_attn.k_norm", c.head_dim); + if (packed_qkv) { + layer.self_attention.qkv_weight = modules::PackedLinearWeightsBuilder({c.hidden_size, { + {p + ".self_attn.q_proj.weight", std::nullopt, c.heads * c.head_dim}, + {p + ".self_attn.k_proj.weight", std::nullopt, c.kv_heads * c.head_dim}, + {p + ".self_attn.v_proj.weight", std::nullopt, c.kv_heads * c.head_dim}}, false}) + .build(store_, source, storage).weight; + } else { + layer.self_attention.q_weight = store_.load_tensor(source, p + ".self_attn.q_proj.weight", storage, + {c.heads * c.head_dim, c.hidden_size}); + layer.self_attention.k_weight = store_.load_tensor(source, p + ".self_attn.k_proj.weight", storage, + {c.kv_heads * c.head_dim, c.hidden_size}); + layer.self_attention.v_weight = store_.load_tensor(source, p + ".self_attn.v_proj.weight", storage, + {c.kv_heads * c.head_dim, c.hidden_size}); + } + layer.self_attention.out_weight = store_.load_tensor(source, p + ".self_attn.o_proj.weight", storage, + {c.hidden_size, c.heads * c.head_dim}); + if (packed_gate_up) { + layer.mlp.gate_up_proj = modules::PackedLinearWeightsBuilder({c.hidden_size, { + {p + ".mlp.gate_proj.weight", std::nullopt, c.intermediate_size}, + {p + ".mlp.up_proj.weight", std::nullopt, c.intermediate_size}}, false}) + .build(store_, source, storage); + } else { + layer.mlp.gate_proj = modules::binding::linear_from_source(store_, source, p + ".mlp.gate_proj", storage, + c.intermediate_size, c.hidden_size, false); + layer.mlp.up_proj = modules::binding::linear_from_source(store_, source, p + ".mlp.up_proj", storage, + c.intermediate_size, c.hidden_size, false); + } + layer.mlp.down_proj = modules::binding::linear_from_source(store_, source, p + ".mlp.down_proj", storage, + c.hidden_size, c.intermediate_size, false); + weights_.stack.layers.push_back(std::move(layer)); + } + weights_.final_norm = modules::binding::norm_weight_from_source(store_, source, "llm.model.norm", c.hidden_size); + weights_.lm_head = modules::LinearWeights{weights_.token_embedding, std::nullopt}; + store_.upload(); + auto & d = config_; + d.logits_size = c.vocab_size; + d.static_cache_type = GGML_TYPE_F16; + auto & s = d.stack; + s.hidden_size = c.hidden_size; + s.num_attention_heads = c.heads; + s.num_key_value_heads = c.kv_heads; + s.head_dim = c.head_dim; + s.intermediate_size = c.intermediate_size; + s.layers = c.layers; + s.rms_norm_eps = c.rms_norm_eps; + s.rope_theta = c.rope_theta; + s.use_qk_norm = true; + s.qkv_layout = packed_qkv ? modules::DecoderQKVLayout::PackedQKV : modules::DecoderQKVLayout::Separate; + s.runtime.attention.prefill_mode = modules::DecoderAttentionMode::FlashGroupedViewKV; + s.runtime.attention.static_mode = modules::DecoderAttentionMode::FlashGroupedViewKV; + s.runtime.static_cache.update_mode = modules::DecoderStaticCacheUpdateMode::DirectSetRows; + s.runtime.static_cache.set_rows_mode = modules::DecoderStaticCacheSetRowsMode::BackendViewOptimized; + s.runtime.mlp.mode = packed_gate_up ? modules::DecoderMLPMode::PackedGateUp : modules::DecoderMLPMode::FusedSwiGLU; + + std::unique_ptr context(ggml_init({kContextBytes, nullptr, true})); + core::ModuleBuildContext ctx{context.get(), "hviske_asr_v6.embedding", execution.backend_type()}; + const auto & mc = assets_->config; + special_ids_ = {mc.bos_token_id, mc.cased_token_id, mc.nocase_token_id, + mc.punctuation_token_id, mc.no_punctuation_token_id}; + auto ids = core::make_tensor(ctx, GGML_TYPE_I32, core::TensorShape::from_dims({5})); + auto embeddings = modules::EmbeddingModule({c.vocab_size, c.hidden_size}).build(ctx, ids, weights_.token_embedding); + auto * graph = ggml_new_graph_custom(ctx.ggml, 32, false); + ggml_build_forward_expand(graph, embeddings.tensor); + std::unique_ptr allocator( + ggml_gallocr_new(ggml_backend_get_default_buffer_type(execution.backend()))); + if (!ggml_gallocr_alloc_graph(allocator.get(), graph)) { + throw std::runtime_error("Hviske v6 control embedding allocation failed"); + } + ggml_backend_tensor_set(ids.tensor, special_ids_.data(), 0, sizeof(special_ids_)); + if (core::compute_backend_graph(execution.backend(), graph) != GGML_STATUS_SUCCESS) { + throw std::runtime_error("Hviske v6 control embedding lookup failed"); + } + special_embeddings_.resize(static_cast(5 * c.hidden_size)); + ggml_backend_tensor_get(embeddings.tensor, special_embeddings_.data(), 0, special_embeddings_.size() * sizeof(float)); + core::release_backend_graph_resources(execution.backend(), graph); + } + + std::vector generate(const std::vector & audio, const HviskeV6DecodingOptions & options) { + const auto & c = assets_->config; + auto input = audio; + auto controls = options.control_tokens; + controls.push_back(c.bos_token_id); + for (const auto id : controls) { + const auto found = std::find(special_ids_.begin(), special_ids_.end(), id); + if (found == special_ids_.end()) throw std::runtime_error("Invalid Hviske v6 control token"); + const auto offset = std::distance(special_ids_.begin(), found) * c.decoder.hidden_size; + input.insert(input.end(), special_embeddings_.begin() + offset, + special_embeddings_.begin() + offset + c.decoder.hidden_size); + } + const int64_t steps = input.size() / c.decoder.hidden_size; + const int64_t capacity = steps + options.max_tokens; + if (capacity > c.decoder.max_positions) throw std::runtime_error("Hviske v6 decoder context exceeded"); + if (!beam_graph_ || beam_graph_->batch_size() != options.num_beams || beam_graph_->capacity() != capacity) { + beam_graph_.reset(); + beam_graph_ = std::make_unique( + execution_, config_, weights_, capacity, options.num_beams); + } + auto started = std::chrono::steady_clock::now(); + auto logits = beam_graph_->prefill(input, steps, config_, weights_); + debug::timing_log_scalar("hviske_asr_v6.decoder.prefill_ms", debug::elapsed_ms(started)); + started = std::chrono::steady_clock::now(); + struct Beam { + std::vector tokens; + float score = 0; + size_t slot = 0; + }; + struct Candidate { float score; size_t parent; int32_t token; }; + struct Finished { float score; std::vector tokens; }; + const std::vector * current_logits = &logits; + std::vector active{{{}, 0.0f, 0}}; + std::vector finished; + double search_ms = 0, cache_copy_ms = 0, graph_ms = 0; + for (int64_t step = 0; step < options.max_tokens; ++step) { + const auto search_started = std::chrono::steady_clock::now(); + std::vector candidates; + const auto candidate_limit = static_cast(2 * options.num_beams); + candidates.reserve(candidate_limit + 1); + for (size_t b = 0; b < active.size(); ++b) { + const auto first = current_logits->begin() + active[b].slot * c.decoder.vocab_size; + const auto maximum = *std::max_element(first, first + c.decoder.vocab_size); + double sum = 0; +#if defined(_OPENMP) +#pragma omp parallel for reduction(+:sum) num_threads(std::max(1, execution_.config().threads)) +#endif + for (int64_t token = 0; token < c.decoder.vocab_size; ++token) { + sum += std::exp(static_cast(first[token] - maximum)); + } + const auto norm = maximum + std::log(sum); + for (int32_t token = 0; token < c.decoder.vocab_size; ++token) { + if (token == c.cased_token_id || token == c.nocase_token_id || + token == c.punctuation_token_id || token == c.no_punctuation_token_id) continue; + const Candidate candidate{active[b].score + static_cast(first[token] - norm), b, token}; + if (candidates.size() == candidate_limit && candidate.score <= candidates.back().score) continue; + const auto at = std::lower_bound(candidates.begin(), candidates.end(), candidate, + [](const auto & a, const auto & b) { return a.score > b.score; }); + candidates.insert(at, candidate); + if (candidates.size() > candidate_limit) candidates.pop_back(); + } + } + const auto top = candidates.size(); + if (options.num_beams == 1 && candidates.front().token == c.eos_token_id) { + finished.push_back({0.0f, active.front().tokens}); + active.clear(); + search_ms += debug::elapsed_ms(search_started); + break; + } + std::vector next; + std::vector parents; + for (size_t rank = 0; rank < top && next.size() < static_cast(options.num_beams); ++rank) { + const auto & item = candidates[rank]; + auto tokens = active[item.parent].tokens; + if (item.token == c.eos_token_id) { + if (rank < static_cast(options.num_beams)) { + finished.push_back({item.score / std::pow(static_cast(step + 1), options.length_penalty), tokens}); + } + continue; + } + tokens.push_back(item.token); + next.push_back({std::move(tokens), item.score, active[item.parent].slot}); + parents.push_back(active[item.parent].slot); + } + std::sort(finished.begin(), finished.end(), [](const auto & a, const auto & b) { return a.score > b.score; }); + if (finished.size() > static_cast(options.num_beams)) finished.resize(options.num_beams); + search_ms += debug::elapsed_ms(search_started); + if (next.empty()) break; + if (finished.size() == static_cast(options.num_beams) && finished.back().score >= + candidates.front().score / std::pow(static_cast(step + 1), options.length_penalty)) { + active.clear(); + break; + } + // Unique parents keep their physical cache; only a fork copies K/V, on the backend. + const auto copy_started = std::chrono::steady_clock::now(); + std::vector used(options.num_beams, false), parent_used(options.num_beams, false); + for (const auto parent : parents) used[parent] = true; + for (size_t i = 0; i < next.size(); ++i) { + if (parent_used[parents[i]]) { + const auto free = std::find(used.begin(), used.end(), false); + const auto slot = static_cast(std::distance(used.begin(), free)); + beam_graph_->copy_from(slot, parents[i]); + used[slot] = true; + next[i].slot = slot; + } + parent_used[parents[i]] = true; + } + cache_copy_ms += debug::elapsed_ms(copy_started); + active = std::move(next); + if (step + 1 == options.max_tokens) break; + const auto graph_started = std::chrono::steady_clock::now(); + std::vector tokens(options.num_beams, c.pad_token_id); + for (const auto & beam : active) tokens[beam.slot] = beam.tokens.back(); + current_logits = &beam_graph_->step(tokens, steps + step); + graph_ms += debug::elapsed_ms(graph_started); + } + for (const auto & beam : active) { + finished.push_back({beam.score / std::pow(static_cast(std::max(1, beam.tokens.size())), + options.length_penalty), beam.tokens}); + } + if (finished.empty()) throw std::runtime_error("Hviske v6 beam search returned no hypothesis"); + const auto best = std::max_element(finished.begin(), finished.end(), + [](const auto & a, const auto & b) { return a.score < b.score; }); + debug::timing_log_scalar("hviske_asr_v6.decoder.decode_ms", debug::elapsed_ms(started)); + debug::timing_log_scalar("hviske_asr_v6.decoder.search_ms", search_ms); + debug::timing_log_scalar("hviske_asr_v6.decoder.cache_copy_ms", cache_copy_ms); + debug::timing_log_scalar("hviske_asr_v6.decoder.graph_ms", graph_ms); + return best->tokens; + } + + std::shared_ptr assets_; + core::ExecutionContext & execution_; + core::BackendWeightStore store_; + modules::CausalDecoderRuntimeWeights weights_; + modules::CausalDecoderConfig config_; + std::unique_ptr beam_graph_; + std::array special_ids_; + std::vector special_embeddings_; +}; + +HviskeV6Qwen3DecoderRuntime::HviskeV6Qwen3DecoderRuntime(std::shared_ptr assets, + core::ExecutionContext & execution, assets::TensorStorageType storage) + : impl_(std::make_unique(std::move(assets), execution, storage)) {} +HviskeV6Qwen3DecoderRuntime::~HviskeV6Qwen3DecoderRuntime() = default; +std::vector HviskeV6Qwen3DecoderRuntime::generate( + const std::vector & audio, const HviskeV6DecodingOptions & options) { + return impl_->generate(audio, options); +} + +} // namespace engine::models::hviske_asr_v6 diff --git a/src/models/hviske_asr_v6/encoder.cpp b/src/models/hviske_asr_v6/encoder.cpp new file mode 100644 index 000000000..a72492f39 --- /dev/null +++ b/src/models/hviske_asr_v6/encoder.cpp @@ -0,0 +1,230 @@ +#include "engine/models/hviske_asr_v6/encoder.h" + +#include "engine/framework/core/backend_weight_store.h" +#include "engine/framework/debug/profiler.h" +#include "engine/framework/modules/activation_modules.h" +#include "engine/framework/modules/attention/scaled_dot_product_attention.h" +#include "engine/framework/modules/packed_linear_weights.h" +#include "engine/framework/modules/positional_modules.h" +#include "engine/framework/modules/primitive_modules.h" +#include "engine/framework/modules/speech_encoders/whisper_embedding.h" +#include "engine/framework/modules/structural_modules.h" +#include "engine/framework/modules/weight_binding.h" + +#include + +#include +#include +#include +#include +#include + +namespace engine::models::hviske_asr_v6 { +namespace { +constexpr size_t kContextBytes = 4 * 1024 * 1024; +struct ContextDeleter { + void operator()(ggml_context * p) const { ggml_free(p); } +}; +struct AllocatorDeleter { + void operator()(ggml_gallocr_t p) const { ggml_gallocr_free(p); } +}; +} // namespace + +struct HviskeV6WhisperRoPEEncoderRuntime::Impl { + Impl(std::shared_ptr assets, core::ExecutionContext & execution, + assets::TensorStorageType storage) + : assets_(std::move(assets)), execution_(execution), + store_(execution.backend(), execution.backend_type(), "hviske_asr_v6.encoder.weights", kContextBytes) { + const auto & c = assets_->config.encoder; + const auto & source = *assets_->weights; + conv1_ = modules::binding::conv1d_from_source(store_, source, "encoder.conv1", storage, + c.hidden_size, c.mel_bins, 3, true); + conv2_ = modules::binding::conv1d_from_source(store_, source, "encoder.conv2", storage, + c.hidden_size, c.hidden_size, 3, true); + for (int64_t i = 0; i < c.layers; ++i) { + const auto p = "encoder.layers." + std::to_string(i); + modules::WhisperEncoderLayerWeights layer; + layer.attention_norm = modules::binding::norm_from_named_source(store_, source, + p + ".self_attn_layer_norm.weight", p + ".self_attn_layer_norm.bias"); + layer.mlp_norm = modules::binding::norm_from_named_source(store_, source, + p + ".final_layer_norm.weight", p + ".final_layer_norm.bias"); + const auto q_type = source.require_metadata(p + ".self_attn.q_proj.weight").dtype; + const bool pack = storage != assets::TensorStorageType::Native || + (q_type == source.require_metadata(p + ".self_attn.k_proj.weight").dtype && + q_type == source.require_metadata(p + ".self_attn.v_proj.weight").dtype); + if (pack) { + packed_qkv_.push_back(modules::PackedLinearWeightsBuilder({c.hidden_size, { + {p + ".self_attn.q_proj.weight", std::nullopt, c.hidden_size}, + {p + ".self_attn.k_proj.weight", std::nullopt, c.hidden_size}, + {p + ".self_attn.v_proj.weight", std::nullopt, c.hidden_size}}, false}) + .build(store_, source, storage)); + auto bias = source.require_f32(p + ".self_attn.q_proj.bias", {c.hidden_size}); + bias.resize(2 * c.hidden_size, 0.0f); + const auto value_bias = source.require_f32(p + ".self_attn.v_proj.bias", {c.hidden_size}); + bias.insert(bias.end(), value_bias.begin(), value_bias.end()); + packed_qkv_.back()->bias = store_.make_f32(core::TensorShape::from_dims({3 * c.hidden_size}), std::move(bias)); + } else { + packed_qkv_.push_back(std::nullopt); + layer.attention.query = modules::binding::linear_from_source(store_, source, + p + ".self_attn.q_proj", storage, c.hidden_size, c.hidden_size, true); + layer.attention.key = modules::binding::linear_from_source(store_, source, + p + ".self_attn.k_proj", storage, c.hidden_size, c.hidden_size, false); + layer.attention.value = modules::binding::linear_from_source(store_, source, + p + ".self_attn.v_proj", storage, c.hidden_size, c.hidden_size, true); + } + layer.attention.out = modules::binding::linear_from_source(store_, source, + p + ".self_attn.out_proj", storage, c.hidden_size, c.hidden_size, true); + const auto fc1 = modules::binding::linear_from_source(store_, source, + p + ".fc1", storage, c.intermediate_size, c.hidden_size, true); + const auto fc2 = modules::binding::linear_from_source(store_, source, + p + ".fc2", storage, c.hidden_size, c.intermediate_size, true); + layer.mlp = {fc1.weight, fc1.bias, fc2.weight, fc2.bias}; + layers_.push_back(std::move(layer)); + } + final_norm_ = modules::binding::norm_from_named_source(store_, source, + "encoder.layer_norm.weight", "encoder.layer_norm.bias"); + const auto stacked = c.hidden_size * assets_->config.frame_stack; + projector_norm_ = modules::binding::norm_weight_from_source(store_, source, "norm", stacked); + projector_ = modules::binding::linear_from_source(store_, source, "proj", storage, + assets_->config.decoder.hidden_size, stacked, false); + store_.upload(); + } + + ~Impl() { release_graph(); } + + void release_graph() { + core::release_backend_graph_resources(execution_.backend(), graph_); + graph_ = nullptr; + plan_.reset(); + allocator_.reset(); + context_.reset(); + } + + void build(int64_t frames) { + release_graph(); + const auto started = std::chrono::steady_clock::now(); + const auto & c = assets_->config.encoder; + frames_ = frames; + const auto encoded_frames = (frames + 1) / 2; + const auto dim = c.hidden_size / c.heads; + context_.reset(ggml_init({kContextBytes, nullptr, true})); + core::ModuleBuildContext ctx{context_.get(), "hviske_asr_v6.encoder", execution_.backend_type()}; + input_ = core::make_tensor(ctx, GGML_TYPE_F32, core::TensorShape::from_dims({1, c.mel_bins, frames})); + positions_ = core::make_tensor(ctx, GGML_TYPE_I32, core::TensorShape::from_dims({encoded_frames})); + ggml_set_input(input_.tensor); + ggml_set_input(positions_.tensor); + ggml_set_output(positions_.tensor); + const modules::GeluModule gelu({modules::GeluApproximation::ExactErf}); + auto x = modules::Conv1dModule({c.mel_bins, c.hidden_size, 3, 1, 1, 1, true}).build(ctx, input_, conv1_); + x = gelu.build(ctx, x); + x = modules::Conv1dModule({c.hidden_size, c.hidden_size, 3, 2, 1, 1, true}).build(ctx, x, conv2_); + x = gelu.build(ctx, x); + x = modules::TransposeModule({{0, 2, 1}, 3}).build(ctx, x); + x = core::ensure_backend_addressable_layout(ctx, x); + const modules::LayerNormModule norm({c.hidden_size, 1.0e-5f, true, true}); + const modules::RoPEModule rope({c.rotary_dim, GGML_ROPE_TYPE_NEOX, c.rope_theta}); + for (size_t i = 0; i < layers_.size(); ++i) { + const auto & layer = layers_[i]; + auto h = norm.build(ctx, x, layer.attention_norm); + core::TensorValue q, k, v; + if (packed_qkv_[i]) { + auto qkv = modules::LinearModule({c.hidden_size, 3 * c.hidden_size, true}).build(ctx, h, *packed_qkv_[i]); + q = modules::SliceModule({2, 0, c.hidden_size}).build(ctx, qkv); + k = modules::SliceModule({2, c.hidden_size, c.hidden_size}).build(ctx, qkv); + v = modules::SliceModule({2, 2 * c.hidden_size, c.hidden_size}).build(ctx, qkv); + q = core::ensure_backend_addressable_layout(ctx, q); + k = core::ensure_backend_addressable_layout(ctx, k); + v = core::ensure_backend_addressable_layout(ctx, v); + } else { + q = modules::LinearModule({c.hidden_size, c.hidden_size, true}).build(ctx, h, layer.attention.query); + k = modules::LinearModule({c.hidden_size, c.hidden_size, false}).build(ctx, h, layer.attention.key); + v = modules::LinearModule({c.hidden_size, c.hidden_size, true}).build(ctx, h, layer.attention.value); + } + const auto heads = core::TensorShape::from_dims({1, encoded_frames, c.heads, dim}); + q = rope.build(ctx, core::reshape_tensor(ctx, q, heads), positions_); + k = rope.build(ctx, core::reshape_tensor(ctx, k, heads), positions_); + v = core::reshape_tensor(ctx, v, heads); + q = modules::TransposeModule({{0, 2, 1, 3}, 4}).build(ctx, q); + k = modules::TransposeModule({{0, 2, 1, 3}, 4}).build(ctx, k); + v = modules::TransposeModule({{0, 2, 1, 3}, 4}).build(ctx, v); + h = modules::ScaledDotProductAttentionModule({dim, modules::ScaledDotProductAttentionLowering::Flash}) + .build(ctx, q, k, v); + h = core::reshape_tensor(ctx, core::ensure_backend_addressable_layout(ctx, h), + core::TensorShape::from_dims({1, encoded_frames, c.hidden_size})); + h = modules::LinearModule({c.hidden_size, c.hidden_size, true}).build(ctx, h, layer.attention.out); + x = modules::AddModule().build(ctx, x, h); + h = norm.build(ctx, x, layer.mlp_norm); + h = modules::FeedForwardModule({c.hidden_size, c.intermediate_size, true, + modules::GeluApproximation::ExactErf}).build(ctx, h, layer.mlp); + x = modules::AddModule().build(ctx, x, h); + } + x = norm.build(ctx, x, final_norm_); + const auto stack = assets_->config.frame_stack; + const auto tokens = encoded_frames / stack; + if (tokens * stack != encoded_frames) { + x = modules::SliceModule({1, 0, tokens * stack}).build(ctx, x); + } + x = core::reshape_tensor(ctx, core::ensure_backend_addressable_layout(ctx, x), + core::TensorShape::from_dims({1, tokens, stack * c.hidden_size})); + x = modules::RMSNormModule({stack * c.hidden_size, 1.0e-6f, true, false}).build(ctx, x, projector_norm_); + output_ = modules::LinearModule({stack * c.hidden_size, assets_->config.decoder.hidden_size, false}) + .build(ctx, x, projector_); + ggml_set_output(output_.tensor); + graph_ = ggml_new_graph_custom(ctx.ggml, 8192, false); + ggml_build_forward_expand(graph_, output_.tensor); + allocator_.reset(ggml_gallocr_new(ggml_backend_get_default_buffer_type(execution_.backend()))); + if (!ggml_gallocr_alloc_graph(allocator_.get(), graph_)) { + throw std::runtime_error("Hviske v6 encoder graph allocation failed"); + } + core::prepare_host_graph_plan(execution_, graph_, plan_); + std::vector positions(static_cast(encoded_frames)); + std::iota(positions.begin(), positions.end(), 0); + ggml_backend_tensor_set(positions_.tensor, positions.data(), 0, positions.size() * sizeof(int32_t)); + debug::timing_log_scalar("hviske_asr_v6.encoder.graph.build_ms", debug::elapsed_ms(started)); + } + + std::vector encode(const HviskeV6Features & features) { + if (!graph_ || frames_ != features.frames) { + build(features.frames); + } + const auto started = std::chrono::steady_clock::now(); + ggml_backend_tensor_set(input_.tensor, features.values.data(), 0, features.values.size() * sizeof(float)); + if (core::compute_graph(execution_, graph_, plan_, "hviske_asr_v6.encoder") != GGML_STATUS_SUCCESS) { + throw std::runtime_error("Hviske v6 encoder execution failed"); + } + const auto & c = assets_->config; + const auto valid_encoded = (features.valid_frames + 1) / 2; + const auto tokens = std::clamp((valid_encoded + c.frame_stack - 1) / c.frame_stack, + 1, ((features.frames + 1) / 2) / c.frame_stack); + std::vector output(static_cast(tokens * c.decoder.hidden_size)); + ggml_backend_tensor_get(output_.tensor, output.data(), 0, output.size() * sizeof(float)); + debug::timing_log_scalar("hviske_asr_v6.encoder.wall_ms", debug::elapsed_ms(started)); + return output; + } + + std::shared_ptr assets_; + core::ExecutionContext & execution_; + core::BackendWeightStore store_; + modules::Conv1dWeights conv1_, conv2_; + std::vector layers_; + std::vector> packed_qkv_; + modules::NormWeights final_norm_, projector_norm_; + modules::LinearWeights projector_; + std::unique_ptr context_; + std::unique_ptr allocator_; + core::HostGraphPlan plan_; + core::TensorValue input_, positions_, output_; + ggml_cgraph * graph_ = nullptr; + int64_t frames_ = 0; +}; + +HviskeV6WhisperRoPEEncoderRuntime::HviskeV6WhisperRoPEEncoderRuntime( + std::shared_ptr assets, core::ExecutionContext & execution, + assets::TensorStorageType storage) + : impl_(std::make_unique(std::move(assets), execution, storage)) {} +HviskeV6WhisperRoPEEncoderRuntime::~HviskeV6WhisperRoPEEncoderRuntime() = default; +std::vector HviskeV6WhisperRoPEEncoderRuntime::encode(const HviskeV6Features & features) { + return impl_->encode(features); +} + +} // namespace engine::models::hviske_asr_v6 diff --git a/src/models/hviske_asr_v6/frontend.cpp b/src/models/hviske_asr_v6/frontend.cpp new file mode 100644 index 000000000..10a9c80f9 --- /dev/null +++ b/src/models/hviske_asr_v6/frontend.cpp @@ -0,0 +1,47 @@ +#include "engine/models/hviske_asr_v6/frontend.h" + +#include +#include +#include + +namespace engine::models::hviske_asr_v6 { + +HviskeV6WhisperFrontend::HviskeV6WhisperFrontend(int64_t mel_bins) { + audio::MelSpectrogramFrontendConfig config; + config.sample_rate = 16000; + config.n_fft = config.win_length = 400; + config.hop_length = 160; + config.n_mels = mel_bins; + config.mel_fmax = 8000.0f; + config.stft_center = true; + config.waveform_padding = audio::MelWaveformPadding::None; + config.spectrum_mode = audio::MelSpectrumMode::PowerBeforeProjection; + config.value_transform = audio::MelValueTransform::Log10; + config.log_floor = 1.0e-10; + config.log_dynamic_range = 8.0f; + config.log_shift = 4.0f; + config.log_divisor = 4.0f; + config.drop_last_frames = 1; + mel_ = audio::get_cached_mel_spectrogram_frontend(config); +} + +HviskeV6Features HviskeV6WhisperFrontend::extract( + const std::vector & mono_16k, size_t threads) const { + if (mono_16k.size() > 30 * 16000) { + throw std::runtime_error("Hviske v6 frontend requires chunks of at most 30 seconds"); + } + HviskeV6Features result; + result.valid_frames = (static_cast(mono_16k.size()) + 159) / 160; + result.frames = std::max(200, ((result.valid_frames + 99) / 100) * 100); + // Upstream pads the waveform before STFT, not the resulting mel features. + std::vector padded(static_cast(result.frames * 160), 0.0f); + std::copy(mono_16k.begin(), mono_16k.end(), padded.begin()); + auto features = mel_->extract_mono(padded, threads); + if (features.frames != result.frames) { + throw std::runtime_error("Hviske v6 frontend frame count mismatch"); + } + result.values = std::move(features.values); + return result; +} + +} // namespace engine::models::hviske_asr_v6 diff --git a/src/models/hviske_asr_v6/session.cpp b/src/models/hviske_asr_v6/session.cpp new file mode 100644 index 000000000..b17b1d68c --- /dev/null +++ b/src/models/hviske_asr_v6/session.cpp @@ -0,0 +1,190 @@ +#include "engine/models/hviske_asr_v6/session.h" + +#include "engine/framework/audio/conversion.h" +#include "engine/framework/audio/resampling.h" +#include "engine/framework/runtime/options.h" +#include "engine/framework/runtime/session_base.h" +#include "engine/framework/runtime/spec_backed_model.h" +#include "engine/framework/tokenizers/qwen_bpe_bundle.h" +#include "engine/models/hviske_asr_v6/decoder.h" +#include "engine/models/hviske_asr_v6/encoder.h" + +#include +#include +#include +#include +#include + +namespace engine::models::hviske_asr_v6 { +namespace { +constexpr const char * kFamily = "hviske_asr_v6"; + +std::vector> plan_chunks(const std::vector & samples) { + const auto n = static_cast(samples.size()); + if (n <= 30 * 16000) return {{0, n}}; + // librosa.effects.split defaults: centered 2048-sample RMS, 512-sample hop, top_db=30. + std::vector energy(static_cast(1 + n / 512)); + double sum = 0; + for (int64_t j = 0; j < std::min(n, 1024); ++j) sum += samples[j] * samples[j]; + for (size_t i = 0; i < energy.size(); ++i) { + energy[i] = sum / 2048; + const auto center = static_cast(i) * 512; + for (int64_t j = center - 1024; j < center - 512; ++j) { + if (j >= 0 && j < n) sum -= samples[j] * samples[j]; + } + for (int64_t j = center + 1024; j < center + 1536; ++j) { + if (j < n) sum += samples[j] * samples[j]; + } + } + const auto threshold = std::max(1.0e-10, *std::max_element(energy.begin(), energy.end())) * 0.001; + std::vector> voiced; + int64_t start = -1; + for (size_t i = 0; i <= energy.size(); ++i) { + const bool active = i < energy.size() && std::max(1.0e-10, energy[i]) > threshold; + if (active && start < 0) start = i * 512; + if (!active && start >= 0) { + voiced.emplace_back(start, std::min(n, i * 512)); + start = -1; + } + } + std::vector cuts{0}; + for (size_t i = 1; i < voiced.size(); ++i) cuts.push_back((voiced[i - 1].second + voiced[i].first) / 2); + cuts.push_back(n); + std::vector> spans; + start = 0; + for (size_t i = 1; i < cuts.size(); ++i) { + while (cuts[i] - start > 28 * 16000) { + const auto limit = start + 28 * 16000; + const auto upper = std::upper_bound(cuts.begin(), cuts.end(), limit); + const auto cut = upper != cuts.begin() && *std::prev(upper) > start ? *std::prev(upper) : limit; + spans.emplace_back(start, cut); + start = cut; + } + } + if (start < n) spans.emplace_back(start, n); + return spans; +} + +std::string collapse_repeats(const std::string & text) { + std::istringstream stream(text); + std::vector words; + for (std::string word; stream >> word;) words.push_back(std::move(word)); + bool changed = true; + while (changed) { + changed = false; + for (size_t size = 1; size <= 4; ++size) { + std::vector out; + for (size_t i = 0; i < words.size();) { + if (i + size > words.size()) { + out.insert(out.end(), words.begin() + i, words.end()); + break; + } + size_t repeats = 1; + while (i + (repeats + 1) * size <= words.size() && + std::equal(words.begin() + i, words.begin() + i + size, words.begin() + i + repeats * size)) ++repeats; + out.insert(out.end(), words.begin() + i, words.begin() + i + std::min(3, repeats) * size); + changed = changed || repeats > 3; + i += repeats * size; + } + words = std::move(out); + } + } + std::string out; + for (const auto & word : words) { + if (!out.empty()) out += ' '; + out += word; + } + return out; +} + +class HviskeV6Session final : public runtime::RuntimeSessionBase, public runtime::IOfflineVoiceTaskSession { +public: + HviskeV6Session(const runtime::SessionOptions & options, std::shared_ptr assets, + std::shared_ptr contract) + : RuntimeSessionBase(options), assets_(std::move(assets)), contract_(std::move(contract)), + tokenizer_(tokenizers::load_qwen_bpe_tokenizer(assets_->resources)), frontend_(assets_->config.encoder.mel_bins) { + runtime::validate_spec_backed_session_options(options, *contract_, kFamily, "Hviske v6"); + const auto storage = assets::parse_tensor_storage_type( + runtime::find_option(options.options, {"hviske_asr_v6.weight_type"}).value_or("native")); + encoder_ = std::make_unique(assets_, execution_context(), storage); + decoder_ = std::make_unique(assets_, execution_context(), storage); + assets_->weights->release_storage(); + } + std::string family() const override { return kFamily; } + runtime::VoiceTaskKind task_kind() const override { return runtime::VoiceTaskKind::Asr; } + runtime::RunMode run_mode() const override { return runtime::RunMode::Offline; } + void prepare(const runtime::SessionPreparationRequest & request) override { + if (!request.audio) throw std::runtime_error("Hviske v6 requires audio preparation"); + mark_prepared(); + } + runtime::TaskResult run(const runtime::TaskRequest & request) override { + require_prepared("Hviske v6 run"); + runtime::validate_spec_backed_request_options(request.options, request.option_arrays, *contract_, "Hviske v6"); + if (!request.audio_input) throw std::runtime_error("Hviske v6 requires audio input"); + const auto started = std::chrono::steady_clock::now(); + const auto & audio = *request.audio_input; + auto mono = audio::mixdown_interleaved_to_mono_average(audio.samples, audio.channels); + if (audio.sample_rate != 16000) { + audio::SoxrResampleOptions options; + options.output_length_policy = audio::SoxrOutputLengthPolicy::ExactExpected; + options.require_full_input = true; + auto resampled = audio::try_resample_mono_soxr(mono, audio.sample_rate, 16000, options); + if (!resampled) throw std::runtime_error("Hviske v6 needs SOXR for non-16-kHz input"); + mono = std::move(*resampled); + } + if (mono.empty()) mono.resize(8000, 0.0f); + HviskeV6DecodingOptions options; + options.max_tokens = runtime::parse_positive_i64_option(request.options, {"max_tokens"}, 440); + options.num_beams = runtime::parse_positive_i64_option(request.options, {"num_beams"}, 2); + options.length_penalty = runtime::parse_finite_float_option(request.options, {"length_penalty"}).value_or(1.0f); + const auto cased = runtime::find_option(request.options, {"cased"}); + const auto punctuation = runtime::find_option(request.options, {"punctuated"}); + if (cased && punctuation) { + const auto & c = assets_->config; + options.control_tokens = { + runtime::parse_bool_option(*cased, "cased") ? c.cased_token_id : c.nocase_token_id, + runtime::parse_bool_option(*punctuation, "punctuated") ? c.punctuation_token_id : c.no_punctuation_token_id}; + } + std::string text; + const auto maximum = options.max_tokens; + for (const auto & [begin, end] : plan_chunks(mono)) { + std::vector chunk(mono.begin() + begin, mono.begin() + end); + options.max_tokens = std::min(maximum, (chunk.size() + 1999) / 2000 + 12); + const auto features = frontend_.extract(chunk, execution_context().config().threads); + const auto encoded = encoder_->encode(features); + const auto ids = decoder_->generate(encoded, options); + const auto part = collapse_repeats(tokenizer_->decode(ids, true)); + if (!part.empty()) { + if (!text.empty()) text += ' '; + text += part; + } + } + runtime::TaskResult result; + result.text_output = runtime::Transcript{text, "da"}; + debug::timing_log_scalar("session.wall_ms", debug::elapsed_ms(started)); + return result; + } +private: + std::shared_ptr assets_; + std::shared_ptr contract_; + std::shared_ptr tokenizer_; + HviskeV6WhisperFrontend frontend_; + std::unique_ptr encoder_; + std::unique_ptr decoder_; +}; +} // namespace + +std::shared_ptr make_hviske_asr_v6_loader() { + runtime::SpecBackedVoiceModelConfig config; + config.family = kFamily; + config.load_assets = load_hviske_v6_assets; + config.create_session = [](const runtime::TaskSpec & task, const runtime::SessionOptions & options, + std::shared_ptr assets, std::shared_ptr contract) { + if (task.task != runtime::VoiceTaskKind::Asr || task.mode != runtime::RunMode::Offline) { + throw std::runtime_error("Hviske v6 supports offline ASR"); + } + return std::make_unique(options, std::move(assets), std::move(contract)); + }; + return runtime::make_spec_backed_voice_loader(std::move(config)); +} +} // namespace engine::models::hviske_asr_v6 diff --git a/tools/audiocpp_cli/audiocpp_cli_path_cases.json b/tools/audiocpp_cli/audiocpp_cli_path_cases.json index d877b15af..0c9b90234 100644 --- a/tools/audiocpp_cli/audiocpp_cli_path_cases.json +++ b/tools/audiocpp_cli/audiocpp_cli_path_cases.json @@ -1,5 +1,35 @@ { "cases": [ + { + "id": "hviske_v6_longform", + "coverage": "Silence-split Danish longform and short request after long input", + "family": "hviske_asr_v6", + "model": "models/Hviske-v6-GGUF/hviske-v6-bf16.gguf", + "task": "asr", + "outputs": ["text"], + "requests": [ + {"id": "warmup", "audio": "resources/hviske_asr/danish_unified_00.wav"}, + {"id": "longform", "audio": "resources/hviske_asr/da_solen_180s.wav"}, + {"id": "short_after_long", "audio": "resources/hviske_asr/danish_unified_00.wav"} + ] + }, + { + "id": "hviske_v6_asr", + "coverage": "Danish ASR, formatting controls, greedy decoding, varying input length and repeated requests", + "family": "hviske_asr_v6", + "model": "models/Hviske-v6-GGUF/hviske-v6-bf16.gguf", + "task": "asr", + "outputs": ["text"], + "requests": [ + {"id": "first", "audio": "resources/hviske_asr/danish_unified_00.wav"}, + {"id": "different", "audio": "resources/hviske_asr/danish_unified_01.wav"}, + {"id": "third", "audio": "resources/hviske_asr/danish_unified_02.wav"}, + {"id": "formatted", "audio": "resources/hviske_asr/danish_unified_00.wav", "options": {"cased": true, "punctuated": true}}, + {"id": "unformatted", "audio": "resources/hviske_asr/danish_unified_00.wav", "options": {"cased": false, "punctuated": false}}, + {"id": "greedy", "audio": "resources/hviske_asr/danish_unified_00.wav", "options": {"num_beams": 1}}, + {"id": "repeat_first", "audio": "resources/hviske_asr/danish_unified_00.wav"} + ] + }, { "id": "owsm_v4_streaming", "coverage": "OWSM v4 committed long-form text deltas", From 70884ff3e57a01b9a5972ddee1264b0ddf069166 Mon Sep 17 00:00:00 2001 From: 0xShug0 <231717474+0xShug0@users.noreply.github.com> Date: Wed, 7 Oct 2026 23:27:04 -0400 Subject: [PATCH 2/2] Expose Hviske v6 chunk controls and shared silence splitting --- docs/models/hviske_asr_v6.md | 2 + include/engine/framework/audio/chunking.h | 14 ++++ model_specs/hviske_asr_v6.json | 22 ++++++ src/framework/audio/chunking.cpp | 56 ++++++++++++++- src/models/hviske_asr_v6/session.cpp | 72 +++++++------------ tests/unittests/test_audio_chunking.cpp | 48 +++++++++++++ .../audiocpp_cli/audiocpp_cli_path_cases.json | 9 ++- 7 files changed, 174 insertions(+), 49 deletions(-) diff --git a/docs/models/hviske_asr_v6.md b/docs/models/hviske_asr_v6.md index f253d88be..a8c7ba2fa 100644 --- a/docs/models/hviske_asr_v6.md +++ b/docs/models/hviske_asr_v6.md @@ -25,6 +25,8 @@ longer than 30 seconds are split at silence boundaries into chunks of at most | Option | Default | Meaning | |---|---|---| +| `audio_chunk_mode` | `auto` | `auto`: silence splitting above 30 seconds; `silence`: silence splitting above the configured duration; `fixed`: fixed-duration cuts; `none`: no splitting, input must be at most 30 seconds. | +| `audio_chunk_duration_sec` | `28` | Maximum chunk duration (0.001-30 seconds). Does not change the 30-second trigger in `auto` mode. | | `max_tokens` | `440` | Per-chunk generation limit, further capped to eight tokens per second plus twelve. | | `num_beams` | `2` | Beam-search width; `1` selects greedy decoding. | | `length_penalty` | `1.0` | Beam-score length-normalization exponent. | diff --git a/include/engine/framework/audio/chunking.h b/include/engine/framework/audio/chunking.h index 5ece8db73..8a4717219 100644 --- a/include/engine/framework/audio/chunking.h +++ b/include/engine/framework/audio/chunking.h @@ -31,6 +31,7 @@ enum class AudioChunkMode { QuietEnergy, Vad, None, + Silence, // Recording-relative RMS threshold and silence-gap midpoints. }; struct AudioChunkSpec { @@ -61,6 +62,19 @@ struct QuietEnergyAudioChunkOptions { int64_t min_energy_window_samples = 0; }; +struct SilenceAudioChunkOptions { + int64_t max_chunk_samples = 0; + // Inputs at or below this length stay whole, even if above max_chunk_samples. + int64_t trigger_samples = 0; +}; + +// Librosa-style silence splitting: centered 2048-sample RMS, hop 512, +// 30 dB below peak RMS. Preserve all audio and cut at silence-gap midpoints, +// falling back to max_chunk_samples when no gap is available. +std::vector plan_silence_audio_chunks( + const std::vector & mono_samples, + const SilenceAudioChunkOptions & options); + std::vector plan_audio_chunks(int64_t input_samples, const AudioChunkSpec & spec); AudioChunkMode parse_audio_chunk_mode( diff --git a/model_specs/hviske_asr_v6.json b/model_specs/hviske_asr_v6.json index efd8679e3..dcbbc15bf 100644 --- a/model_specs/hviske_asr_v6.json +++ b/model_specs/hviske_asr_v6.json @@ -22,6 +22,28 @@ }, "options": { "request": [ + { + "name": "audio_chunk_mode", + "type": "enum", + "description": "auto preserves silence splitting above 30 seconds; silence splits above the configured duration at silence boundaries; fixed splits at the configured duration; none disables splitting and accepts at most 30 seconds.", + "required": false, + "default": "auto", + "values": [ + "auto", + "silence", + "fixed", + "none" + ] + }, + { + "name": "audio_chunk_duration_sec", + "type": "float", + "description": "Maximum chunk duration when splitting. In auto mode the 30-second trigger remains unchanged.", + "required": false, + "default": 28.0, + "min": 0.001, + "max": 30.0 + }, { "name": "max_tokens", "type": "int", diff --git a/src/framework/audio/chunking.cpp b/src/framework/audio/chunking.cpp index b60ba5e37..59570c845 100644 --- a/src/framework/audio/chunking.cpp +++ b/src/framework/audio/chunking.cpp @@ -180,7 +180,10 @@ AudioChunkMode parse_audio_chunk_mode( if (*mode == "none") { return AudioChunkMode::None; } - throw std::runtime_error("audio_chunk_mode must be auto, fixed, quiet_energy, vad, or none"); + if (*mode == "silence") { + return AudioChunkMode::Silence; + } + throw std::runtime_error("audio_chunk_mode must be auto, fixed, quiet_energy, vad, none, or silence"); } std::optional parse_audio_chunk_seconds_override( @@ -329,6 +332,57 @@ std::vector plan_vad_audio_chunks( options); } +std::vector plan_silence_audio_chunks( + const std::vector & samples, + const SilenceAudioChunkOptions & options) { + const auto n = static_cast(samples.size()); + require_positive(n, "silence input samples"); + require_positive(options.max_chunk_samples, "silence max chunk samples"); + require_positive(options.trigger_samples, "silence trigger samples"); + if (n <= options.trigger_samples) return {{0, n}}; + // librosa.effects.split defaults: centered 2048-sample RMS, 512-sample hop, top_db=30. + std::vector energy(static_cast(1 + n / 512)); + double sum = 0; + for (int64_t j = 0; j < std::min(n, 1024); ++j) sum += samples[j] * samples[j]; + for (size_t i = 0; i < energy.size(); ++i) { + energy[i] = sum / 2048; + const auto center = static_cast(i) * 512; + for (int64_t j = center - 1024; j < center - 512; ++j) { + if (j >= 0 && j < n) sum -= samples[j] * samples[j]; + } + for (int64_t j = center + 1024; j < center + 1536; ++j) { + if (j < n) sum += samples[j] * samples[j]; + } + } + const auto threshold = std::max(1.0e-10, *std::max_element(energy.begin(), energy.end())) * 0.001; + std::vector> voiced; + int64_t start = -1; + for (size_t i = 0; i <= energy.size(); ++i) { + const bool active = i < energy.size() && std::max(1.0e-10, energy[i]) > threshold; + if (active && start < 0) start = i * 512; + if (!active && start >= 0) { + voiced.emplace_back(start, std::min(n, i * 512)); + start = -1; + } + } + std::vector cuts{0}; + for (size_t i = 1; i < voiced.size(); ++i) cuts.push_back((voiced[i - 1].second + voiced[i].first) / 2); + cuts.push_back(n); + std::vector spans; + start = 0; + for (size_t i = 1; i < cuts.size(); ++i) { + while (cuts[i] - start > options.max_chunk_samples) { + const auto limit = start + options.max_chunk_samples; + const auto upper = std::upper_bound(cuts.begin(), cuts.end(), limit); + const auto cut = upper != cuts.begin() && *std::prev(upper) > start ? *std::prev(upper) : limit; + spans.push_back({start, cut}); + start = cut; + } + } + if (start < n) spans.push_back({start, n}); + return spans; +} + std::vector plan_quiet_energy_audio_chunks( const std::vector & mono_samples, const QuietEnergyAudioChunkOptions & options) { diff --git a/src/models/hviske_asr_v6/session.cpp b/src/models/hviske_asr_v6/session.cpp index b17b1d68c..457c48992 100644 --- a/src/models/hviske_asr_v6/session.cpp +++ b/src/models/hviske_asr_v6/session.cpp @@ -1,5 +1,6 @@ #include "engine/models/hviske_asr_v6/session.h" +#include "engine/framework/audio/chunking.h" #include "engine/framework/audio/conversion.h" #include "engine/framework/audio/resampling.h" #include "engine/framework/runtime/options.h" @@ -19,52 +20,6 @@ namespace engine::models::hviske_asr_v6 { namespace { constexpr const char * kFamily = "hviske_asr_v6"; -std::vector> plan_chunks(const std::vector & samples) { - const auto n = static_cast(samples.size()); - if (n <= 30 * 16000) return {{0, n}}; - // librosa.effects.split defaults: centered 2048-sample RMS, 512-sample hop, top_db=30. - std::vector energy(static_cast(1 + n / 512)); - double sum = 0; - for (int64_t j = 0; j < std::min(n, 1024); ++j) sum += samples[j] * samples[j]; - for (size_t i = 0; i < energy.size(); ++i) { - energy[i] = sum / 2048; - const auto center = static_cast(i) * 512; - for (int64_t j = center - 1024; j < center - 512; ++j) { - if (j >= 0 && j < n) sum -= samples[j] * samples[j]; - } - for (int64_t j = center + 1024; j < center + 1536; ++j) { - if (j < n) sum += samples[j] * samples[j]; - } - } - const auto threshold = std::max(1.0e-10, *std::max_element(energy.begin(), energy.end())) * 0.001; - std::vector> voiced; - int64_t start = -1; - for (size_t i = 0; i <= energy.size(); ++i) { - const bool active = i < energy.size() && std::max(1.0e-10, energy[i]) > threshold; - if (active && start < 0) start = i * 512; - if (!active && start >= 0) { - voiced.emplace_back(start, std::min(n, i * 512)); - start = -1; - } - } - std::vector cuts{0}; - for (size_t i = 1; i < voiced.size(); ++i) cuts.push_back((voiced[i - 1].second + voiced[i].first) / 2); - cuts.push_back(n); - std::vector> spans; - start = 0; - for (size_t i = 1; i < cuts.size(); ++i) { - while (cuts[i] - start > 28 * 16000) { - const auto limit = start + 28 * 16000; - const auto upper = std::upper_bound(cuts.begin(), cuts.end(), limit); - const auto cut = upper != cuts.begin() && *std::prev(upper) > start ? *std::prev(upper) : limit; - spans.emplace_back(start, cut); - start = cut; - } - } - if (start < n) spans.emplace_back(start, n); - return spans; -} - std::string collapse_repeats(const std::string & text) { std::istringstream stream(text); std::vector words; @@ -147,7 +102,30 @@ class HviskeV6Session final : public runtime::RuntimeSessionBase, public runtime } std::string text; const auto maximum = options.max_tokens; - for (const auto & [begin, end] : plan_chunks(mono)) { + const auto chunk_mode = audio::parse_audio_chunk_mode(request.options); + const double chunk_seconds = runtime::parse_finite_float_option( + request.options, {"audio_chunk_duration_sec"}).value_or(28.0F); + if (chunk_seconds < 0.001 || chunk_seconds > 30.0) { + throw std::runtime_error("Hviske v6 audio_chunk_duration_sec must be between 0.001 and 30 seconds"); + } + const auto chunk_samples = static_cast(std::llround(chunk_seconds * 16000)); + std::vector spans; + if (chunk_mode == audio::AudioChunkMode::Auto || chunk_mode == audio::AudioChunkMode::Silence) { + const auto trigger = chunk_mode == audio::AudioChunkMode::Auto ? 30 * 16000 : chunk_samples; + spans = audio::plan_silence_audio_chunks(mono, {chunk_samples, trigger}); + } else if (chunk_mode == audio::AudioChunkMode::Fixed) { + for (const auto & span : audio::plan_audio_chunks(mono.size(), {chunk_samples, chunk_samples})) { + spans.push_back({span.output_start_sample, span.output_start_sample + span.valid_samples}); + } + } else if (chunk_mode == audio::AudioChunkMode::None) { + if (mono.size() > 30 * 16000) { + throw std::runtime_error("Hviske v6 audio_chunk_mode=none requires audio of at most 30 seconds; use auto or fixed for longform"); + } + spans.push_back({0, static_cast(mono.size())}); + } else { + throw std::runtime_error("Hviske v6 audio_chunk_mode must be auto, silence, fixed, or none"); + } + for (const auto & [begin, end] : spans) { std::vector chunk(mono.begin() + begin, mono.begin() + end); options.max_tokens = std::min(maximum, (chunk.size() + 1999) / 2000 + 12); const auto features = frontend_.extract(chunk, execution_context().config().threads); diff --git a/tests/unittests/test_audio_chunking.cpp b/tests/unittests/test_audio_chunking.cpp index 2d0686ca8..1a7b387fa 100644 --- a/tests/unittests/test_audio_chunking.cpp +++ b/tests/unittests/test_audio_chunking.cpp @@ -640,6 +640,10 @@ void test_audio_chunk_mode_parser() { engine::test::require( engine::audio::parse_audio_chunk_mode(options) == engine::audio::AudioChunkMode::Vad, "vad audio chunk mode"); + options["audio_chunk_mode"] = "silence"; + engine::test::require( + engine::audio::parse_audio_chunk_mode(options) == engine::audio::AudioChunkMode::Silence, + "parse silence mode"); options["audio_chunk_mode"] = "quiet_energy"; engine::test::require( engine::audio::parse_audio_chunk_mode(options) == engine::audio::AudioChunkMode::QuietEnergy, @@ -914,10 +918,54 @@ void test_chunk_speech_metadata_merge_drops_outside_spans() { require_span(merged.speaker_turns[0].span, 1050, 1080, "valid speaker turn span"); } +void test_silence_chunks() { + using engine::audio::plan_silence_audio_chunks; + const engine::audio::SilenceAudioChunkOptions options{28 * 16000, 30 * 16000}; + require_throws([&]() { + (void) plan_silence_audio_chunks({}, options); + }, "empty silence planner input"); + require_throws([]() { + (void) plan_silence_audio_chunks({1.0F}, {0, 1}); + }, "zero silence max chunk size"); + require_throws([]() { + (void) plan_silence_audio_chunks({1.0F}, {1, 0}); + }, "zero silence trigger"); + auto spans = plan_silence_audio_chunks(std::vector(30 * 16000, 1.0F), options); + engine::test::require_eq(spans.size(), size_t{1}, "trigger includes exactly 30 seconds"); + require_span(spans[0], 0, 30 * 16000, "short input unchanged"); + + // No gaps: both continuous sound and complete silence use hard cuts. + for (float value : {0.0F, 1.0F}) { + spans = plan_silence_audio_chunks(std::vector(60 * 16000, value), options); + engine::test::require_eq(spans.size(), size_t{3}, "hard cut count"); + require_span(spans[0], 0, 28 * 16000, "first hard cut"); + require_span(spans[1], 28 * 16000, 56 * 16000, "second hard cut"); + require_span(spans[2], 56 * 16000, 60 * 16000, "remaining audio"); + } + + std::vector samples(8192, 0.0F); + std::fill(samples.begin(), samples.begin() + 2048, 1.0F); + std::fill(samples.begin() + 6144, samples.end(), 1.0F); + const auto below_trigger = plan_silence_audio_chunks(samples, {6000, 9000}); + engine::test::require_eq(below_trigger.size(), size_t{1}, "auto policy can retain input above chunk limit"); + require_span(below_trigger[0], 0, 8192, "auto policy trigger preserves whole input"); + spans = plan_silence_audio_chunks(samples, {6000, 6000}); + engine::test::require_eq(spans.size(), size_t{2}, "silence midpoint count"); + require_span(spans[0], 0, 4352, "centered RMS silence midpoint"); + require_span(spans[1], 4352, 8192, "silence retained without overlap"); + for (auto & sample : samples) sample *= 0.125F; + const auto scaled = plan_silence_audio_chunks(samples, {6000, 6000}); + engine::test::require_eq(scaled.size(), spans.size(), "relative threshold count"); + for (size_t i = 0; i < spans.size(); ++i) { + require_span(scaled[i], spans[i].start_sample, spans[i].end_sample, "relative threshold boundaries"); + } +} + } // namespace int main() { try { + test_silence_chunks(); test_fixed_chunks_plan_start_aligned_tail(); test_fixed_chunks_plan_overlapping_windows(); test_fixed_chunks_plan_centered_tail_and_copy_zero_pad(); diff --git a/tools/audiocpp_cli/audiocpp_cli_path_cases.json b/tools/audiocpp_cli/audiocpp_cli_path_cases.json index 0c9b90234..52f96d124 100644 --- a/tools/audiocpp_cli/audiocpp_cli_path_cases.json +++ b/tools/audiocpp_cli/audiocpp_cli_path_cases.json @@ -2,7 +2,7 @@ "cases": [ { "id": "hviske_v6_longform", - "coverage": "Silence-split Danish longform and short request after long input", + "coverage": "Danish longform, auto versus explicit silence splitting, fixed and unsplit modes, and repeated requests", "family": "hviske_asr_v6", "model": "models/Hviske-v6-GGUF/hviske-v6-bf16.gguf", "task": "asr", @@ -10,6 +10,13 @@ "requests": [ {"id": "warmup", "audio": "resources/hviske_asr/danish_unified_00.wav"}, {"id": "longform", "audio": "resources/hviske_asr/da_solen_180s.wav"}, + {"id": "auto_20", "audio": "resources/hviske_asr/da_solen_180s.wav", "options": {"audio_chunk_mode": "auto", "audio_chunk_duration_sec": 20}}, + {"id": "fixed_28", "audio": "resources/hviske_asr/da_solen_180s.wav", "options": {"audio_chunk_mode": "fixed", "audio_chunk_duration_sec": 28}}, + {"id": "none_short", "audio": "resources/hviske_asr/danish_unified_00.wav", "options": {"audio_chunk_mode": "none"}}, + {"id": "silence_short", "audio": "resources/hviske_asr/danish_unified_00.wav", "options": {"audio_chunk_mode": "silence", "audio_chunk_duration_sec": 8}}, + {"id": "auto_short_8", "audio": "resources/hviske_asr/danish_unified_00.wav", "options": {"audio_chunk_mode": "auto", "audio_chunk_duration_sec": 8}}, + {"id": "silence_long", "audio": "resources/hviske_asr/da_solen_180s.wav", "options": {"audio_chunk_mode": "silence"}}, + {"id": "repeat_default", "audio": "resources/hviske_asr/da_solen_180s.wav"}, {"id": "short_after_long", "audio": "resources/hviske_asr/danish_unified_00.wav"} ] },