diff --git a/CMakeLists.txt b/CMakeLists.txt index 5471ba9d0..72eac022a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -479,6 +479,7 @@ if (ENGINE_ENABLE_NEON) add_library(engine_neon OBJECT src/framework/neon/tensor.cpp src/framework/neon/operators.cpp + src/framework/neon/tensor_operators.cpp src/framework/neon/execution.cpp src/framework/neon/worker_team.cpp src/framework/neon/blocks.cpp @@ -971,6 +972,9 @@ audiocpp_add_model(micro_wake_word LOADERS engine::models::micro_wake_word::make_micro_wake_word_loader ) +if(ENGINE_ENABLE_NEON) + target_sources(engine_model_micro_wake_word PRIVATE src/models/micro_wake_word/neon_network.cpp) +endif() audiocpp_add_model(sherpa_kws SOURCES @@ -3382,6 +3386,9 @@ if (ENGINE_BUILD_TESTS OR ENGINE_BUILD_EXTENDED_TESTS OR ENGINE_BUILD_MODEL_TEST target_include_directories(neon_library_test PRIVATE tests/unittests) target_link_libraries(neon_library_test PRIVATE engine_neon) add_test(NAME neon_library_test COMMAND neon_library_test) + add_executable(neon_tensor_ops_test tests/unittests/test_neon_tensor_ops.cpp) + target_link_libraries(neon_tensor_ops_test PRIVATE engine_neon) + add_test(NAME neon_tensor_ops_test COMMAND neon_tensor_ops_test) endif() if (sanotts IN_LIST AUDIOCPP_LINKED_MODELS) add_engine_unittest(sanotts_neon_decoder_test tests/unittests/test_sanotts_neon_decoder.cpp) diff --git a/docs/models/micro_wake_word.md b/docs/models/micro_wake_word.md index 875ca9b37..5c17a41ab 100644 --- a/docs/models/micro_wake_word.md +++ b/docs/models/micro_wake_word.md @@ -31,3 +31,24 @@ build/debug/bin/audiocpp_cli --family micro_wake_word --task wake \ Detections are returned as speech segments. `text` contains the wake phrase, `confidence` contains the smoothed probability, and the sample span identifies the feature block that triggered the detection. Request options `threshold` and `sliding_window_size` override the values copied from the model manifest. Use `--mode streaming` for incremental audio. + +## Optional NEON network + +The default network remains the existing GGML implementation on the session +backend. On AArch64, build with `-DENGINE_ENABLE_NEON=ON` and select the native +stateful CNN with: + +```bash +--session-option 'micro_wake_word.exec_plan={"network":{"impl":"neon"}}' +``` + +The NEON network runs on CPU, uses the session thread count, and works in both +offline and streaming modes. The fixed-point frontend, input quantization, +activation quantization boundaries, recurrent-state updates, probability +smoothing and detection thresholds retain their existing behavior. The native +network supports batch-one MixedNet manifests with unit spatial width and +depthwise multiplier one. + +`impl: "ggml"` or `impl: "default"` retains the original path. The component +accepts no artifacts or params. Existing GGUFs can use the new session option +without reconversion, including when loading their embedded spec. diff --git a/include/engine/framework/neon/operators.h b/include/engine/framework/neon/operators.h index 8eeab843f..ce19b8165 100644 --- a/include/engine/framework/neon/operators.h +++ b/include/engine/framework/neon/operators.h @@ -86,6 +86,7 @@ class Conv2d : public Operator { class DepthwiseConv1d : public Operator { public: DepthwiseConv1d(int channels, int kernel_size, const std::vector &weight, const std::vector &bias); + DepthwiseConv1d(Conv1dConfig config, const std::vector &weight, const std::vector &bias); }; class LayerNorm : public Operator { @@ -93,7 +94,7 @@ class LayerNorm : public Operator { LayerNorm(const std::vector &weight, const std::vector &bias, float epsilon); }; -enum class ActivationKind { SiLU, GELU, LeakyReLU }; +enum class ActivationKind { SiLU, GELU, LeakyReLU, Sigmoid }; class Activation : public Operator { public: @@ -121,6 +122,39 @@ class ConcatChannels : public Operator { ConcatChannels(); }; +// Logical-axis operations on contiguous channel-first tensors. +class Concat : public Operator { +public: + explicit Concat(int axis); +}; + +class Slice : public Operator { +public: + Slice(int axis, int start, int length); +}; + +class Reshape : public Operator { +public: + explicit Reshape(Shape shape); +}; + +// Matrix transpose: [channels, frames] becomes [frames, channels]. +class Transpose2d : public Operator { +public: + Transpose2d(); +}; + +class Clamp : public Operator { +public: + Clamp(float minimum, float maximum); +}; + +// Round ties away from zero, clamp to integer bounds, then dequantize. +class QuantizeDequantize : public Operator { +public: + QuantizeDequantize(float scale, int zero_point, int minimum, int maximum); +}; + // Spatial dimensions follow the input; values do not depend on its contents. class ConstantChannels : public Operator { public: diff --git a/include/engine/models/micro_wake_word/neon_network.h b/include/engine/models/micro_wake_word/neon_network.h new file mode 100644 index 000000000..2bde78149 --- /dev/null +++ b/include/engine/models/micro_wake_word/neon_network.h @@ -0,0 +1,25 @@ +#pragma once + +#include "engine/models/micro_wake_word/runtime.h" +#include "engine/framework/neon/execution.h" + +namespace engine::models::micro_wake_word { + +class NeonNetwork { +public: + NeonNetwork(const assets::TensorSource &source, const io::json::Value &config, + const std::vector &tensors, int threads); + float infer(const std::vector &input); + void reset(); + +private: + neon::ExecutionPlan plan_; + neon::Shape input_shape_{1}; + int threads_; + float output_scale_ = 1.0F; + std::vector states_; + std::vector state_outputs_; + std::vector inputs_; +}; + +} // namespace engine::models::micro_wake_word diff --git a/include/engine/models/micro_wake_word/runtime.h b/include/engine/models/micro_wake_word/runtime.h index ab0479ccc..c3479afee 100644 --- a/include/engine/models/micro_wake_word/runtime.h +++ b/include/engine/models/micro_wake_word/runtime.h @@ -6,6 +6,7 @@ #include #include +#include #include #include @@ -20,11 +21,19 @@ struct ModelConfig { std::string wake_phrase; }; +struct NetworkTensorInfo { + std::vector tflite_shape; + std::string type; + std::optional scale; + int64_t zero_point = 0; + bool per_channel_quantized = false; +}; + class Runtime { public: Runtime(std::shared_ptr source, const io::json::Value & config, - core::ExecutionContext & execution); + core::ExecutionContext & execution, bool use_neon = false); ~Runtime(); Runtime(const Runtime &) = delete; diff --git a/model_specs/micro_wake_word.json b/model_specs/micro_wake_word.json index d76c84a9f..c5abf28f5 100644 --- a/model_specs/micro_wake_word.json +++ b/model_specs/micro_wake_word.json @@ -45,7 +45,15 @@ "min": 1 } ], - "session": [], + "session": [ + { + "name": "exec_plan", + "type": "string", + "description": "JSON component execution plans. Component network accepts impl ggml/neon (default ggml). GGML retains the session backend; neon runs the stateful CNN on CPU and requires ENGINE_ENABLE_NEON=ON and AArch64 NEON. Frontend and detection logic are unchanged. No artifacts or params are used.", + "required": false, + "default": "{}" + } + ], "load": [] }, "package_defaults": { diff --git a/src/framework/neon/kernels.cpp b/src/framework/neon/kernels.cpp index 4cfb4bce3..8174cfb06 100644 --- a/src/framework/neon/kernels.cpp +++ b/src/framework/neon/kernels.cpp @@ -25,6 +25,36 @@ bool available() noexcept { #endif } +void quantize_dequantize(const float *input, float *output, size_t count, float scale, + float minimum, float maximum) noexcept { + const float inverse = 1.0F / scale; + size_t i = 0; +#if defined(__aarch64__) && defined(__ARM_NEON) + for (; i + 4 <= count; i += 4) { + auto value = vrndaq_f32(vmulq_n_f32(vld1q_f32(input + i), inverse)); + value = vmaxq_f32(vdupq_n_f32(minimum), vminq_f32(vdupq_n_f32(maximum), value)); + vst1q_f32(output + i, vmulq_n_f32(value, scale)); + } +#endif + for (; i < count; ++i) output[i] = std::clamp(std::round(input[i] * inverse), minimum, maximum) * scale; +} + +void depthwise_conv1d(const float *input, const float *weight, const float *bias, float *output, + const Conv1dConfig &config, int frames, int output_frames, WorkerContext &worker) noexcept { + int begin, end; + split_range(config.input_channels, worker.index, worker.count, begin, end); + for (int channel = begin; channel < end; ++channel) { + for (int frame = 0; frame < output_frames; ++frame) { + float value = bias ? bias[channel] : 0.0F; + for (int tap = 0; tap < config.kernel_size; ++tap) { + const int at = frame * config.stride - config.padding_left + tap * config.dilation; + if (at >= 0 && at < frames) value += input[channel * frames + at] * weight[channel * config.kernel_size + tap]; + } + output[channel * output_frames + frame] = value; + } + } +} + PackedConvWeights pack_conv_weights(const float *w, const float *b, int cin, int cout, int kernel) { if (!w || cin <= 0 || cout <= 0 || cout > std::numeric_limits::max() - 3 || kernel <= 0 || cin > std::numeric_limits::max() / kernel) { diff --git a/src/framework/neon/kernels.h b/src/framework/neon/kernels.h index db7d1b9c1..4f87ab293 100644 --- a/src/framework/neon/kernels.h +++ b/src/framework/neon/kernels.h @@ -10,6 +10,11 @@ namespace engine::neon::detail { +void quantize_dequantize(const float *input, float *output, size_t count, float scale, + float minimum, float maximum) noexcept; +void depthwise_conv1d(const float *input, const float *weight, const float *bias, float *output, + const Conv1dConfig &config, int frames, int output_frames, WorkerContext &worker) noexcept; + bool available() noexcept; struct PackedConvWeights { diff --git a/src/framework/neon/operators.cpp b/src/framework/neon/operators.cpp index 2cb8ee1e6..bde9661f7 100644 --- a/src/framework/neon/operators.cpp +++ b/src/framework/neon/operators.cpp @@ -284,6 +284,41 @@ class DepthwiseConv1dKernel final : public detail::Kernel { std::vector weight_, bias_; }; +class ConfiguredDepthwiseConv1dKernel final : public detail::Kernel { +public: + ConfiguredDepthwiseConv1dKernel(Conv1dConfig config, std::vector weight, std::vector bias) + : config_(config), weight_(std::move(weight)), bias_(std::move(bias)) { + if (config.input_channels <= 0 || config.input_channels != config.output_channels || config.kernel_size <= 0 || + config.stride <= 0 || config.dilation <= 0 || config.padding_left < 0 || config.padding_right < 0 || + weight_.size() != size_t(config.input_channels) * config.kernel_size || + (!bias_.empty() && bias_.size() != size_t(config.input_channels))) { + throw std::invalid_argument("NEON depthwise convolution configuration invalid"); + } + } + Shape output_shape(const std::vector &inputs) const override { + auto shape = unary_shape(inputs, 2); + const auto &c = config_; + if (shape.rank != 2 || shape.dimensions[0] != c.input_channels) { + throw std::invalid_argument("NEON depthwise convolution input shape invalid"); + } + const int64_t span = int64_t(shape.dimensions[1]) + c.padding_left + c.padding_right - + int64_t(c.dilation) * (c.kernel_size - 1) - 1; + if (span < 0 || span / c.stride >= std::numeric_limits::max()) { + throw std::invalid_argument("NEON depthwise convolution output extent invalid"); + } + shape.dimensions[1] = int(span / c.stride + 1); + return shape; + } + void compute(const std::vector &inputs, float *output, const Shape &shape, + detail::Workspace &, detail::WorkerContext &worker) const noexcept override { + detail::depthwise_conv1d(inputs[0].data, weight_.data(), bias_.empty() ? nullptr : bias_.data(), output, + config_, inputs[0].shape->dimensions[1], shape.dimensions[1], worker); + } +private: + Conv1dConfig config_; + std::vector weight_, bias_; +}; + class LayerNormKernel final : public detail::Kernel { public: LayerNormKernel(std::vector weight, std::vector bias, float epsilon) @@ -320,7 +355,8 @@ class LayerNormKernel final : public detail::Kernel { class ActivationKernel final : public detail::Kernel { public: ActivationKernel(ActivationKind kind, float slope) : kind_(kind), slope_(slope) { - if ((kind != ActivationKind::SiLU && kind != ActivationKind::GELU && kind != ActivationKind::LeakyReLU) || + if ((kind != ActivationKind::SiLU && kind != ActivationKind::GELU && kind != ActivationKind::LeakyReLU && + kind != ActivationKind::Sigmoid) || !std::isfinite(slope)) { throw std::invalid_argument("ARM activation configuration invalid"); } @@ -347,6 +383,8 @@ class ActivationKernel final : public detail::Kernel { detail::apply_gelu(output, begin, end); } else if (kind_ == ActivationKind::SiLU) { detail::apply_silu(output, begin, end); + } else if (kind_ == ActivationKind::Sigmoid) { + for (int i = begin; i < end; ++i) output[i] = 1.0F / (1.0F + std::exp(-output[i])); } else { detail::apply_leaky_relu_clamp(output, begin, end, slope_, -std::numeric_limits::infinity()); } @@ -678,6 +716,8 @@ DepthwiseConv1d::DepthwiseConv1d(int channels, int kernel, const std::vector(channels, kernel, weight, bias)) {} LayerNorm::LayerNorm(const std::vector &weight, const std::vector &bias, float epsilon) : Operator(std::make_shared(weight, bias, epsilon)) {} +DepthwiseConv1d::DepthwiseConv1d(Conv1dConfig config, const std::vector &weight, const std::vector &bias) + : Operator(std::make_shared(config, weight, bias)) {} Activation::Activation(ActivationKind kind, float slope) : Operator(std::make_shared(kind, slope)) {} ClampMin::ClampMin(float minimum) : Operator(std::make_shared(minimum)) {} MaxPool2d::MaxPool2d(int kh, int kw, int sh, int sw) : Operator(std::make_shared(kh, kw, sh, sw)) {} diff --git a/src/framework/neon/tensor_operators.cpp b/src/framework/neon/tensor_operators.cpp new file mode 100644 index 000000000..76fa6acff --- /dev/null +++ b/src/framework/neon/tensor_operators.cpp @@ -0,0 +1,179 @@ +#include "engine/framework/neon/operators.h" +#include "operator_kernel.h" + +#include +#include +#include + +namespace engine::neon { +namespace { + +const Shape &input_shape(const std::vector &inputs) { + if (inputs.size() != 1 || inputs[0].layout != Layout::ChannelFirst) { + throw std::invalid_argument("NEON tensor operator requires one channel-first input"); + } + return inputs[0]; +} + +size_t inner_size(const Shape &shape, int axis) { + size_t size = 1; + for (size_t i = axis + 1; i < shape.rank; ++i) size *= shape.dimensions[i]; + return size; +} + +class ConcatKernel final : public detail::Kernel { +public: + explicit ConcatKernel(int axis) : axis_(axis) {} + Shape output_shape(const std::vector &inputs) const override { + if (inputs.empty() || axis_ < 0 || size_t(axis_) >= inputs[0].rank) { + throw std::invalid_argument("NEON concat axis or input count invalid"); + } + auto output = inputs[0]; + output.dimensions[axis_] = 0; + for (const auto &shape : inputs) { + if (shape.layout != Layout::ChannelFirst || shape.rank != output.rank) { + throw std::invalid_argument("NEON concat layout or rank mismatch"); + } + for (size_t i = 0; i < shape.rank; ++i) { + if (int(i) != axis_ && shape.dimensions[i] != output.dimensions[i]) { + throw std::invalid_argument("NEON concat non-axis dimension mismatch"); + } + } + if (shape.dimensions[axis_] > std::numeric_limits::max() - output.dimensions[axis_]) { + throw std::invalid_argument("NEON concat axis overflow"); + } + output.dimensions[axis_] += shape.dimensions[axis_]; + } + return output; + } + void compute(const std::vector &inputs, float *output, const Shape &shape, + detail::Workspace &, detail::WorkerContext &worker) const noexcept override { + const size_t inner = inner_size(shape, axis_); + const size_t width = inner * shape.dimensions[axis_]; + const size_t outer = shape.elements() / width; + for (size_t row = outer * worker.index / worker.count; row < outer * (worker.index + 1) / worker.count; ++row) { + size_t offset = row * width; + for (const auto &input : inputs) { + const size_t count = inner * input.shape->dimensions[axis_]; + std::copy_n(input.data + row * count, count, output + offset); + offset += count; + } + } + } +private: + int axis_; +}; + +class SliceKernel final : public detail::Kernel { +public: + SliceKernel(int axis, int start, int length) : axis_(axis), start_(start), length_(length) {} + Shape output_shape(const std::vector &inputs) const override { + auto output = input_shape(inputs); + if (axis_ < 0 || size_t(axis_) >= output.rank || start_ < 0 || length_ <= 0 || + start_ > output.dimensions[axis_] || length_ > output.dimensions[axis_] - start_) { + throw std::invalid_argument("NEON slice bounds invalid"); + } + output.dimensions[axis_] = length_; + return output; + } + void compute(const std::vector &inputs, float *output, const Shape &shape, + detail::Workspace &, detail::WorkerContext &worker) const noexcept override { + const size_t inner = inner_size(shape, axis_); + const size_t count = inner * length_; + const size_t source_width = inner * inputs[0].shape->dimensions[axis_]; + const size_t outer = shape.elements() / count; + for (size_t row = outer * worker.index / worker.count; row < outer * (worker.index + 1) / worker.count; ++row) { + std::copy_n(inputs[0].data + row * source_width + start_ * inner, count, output + row * count); + } + } +private: + int axis_, start_, length_; +}; + +class ReshapeKernel final : public detail::Kernel { +public: + explicit ReshapeKernel(Shape shape) : shape_(shape) { shape_.elements(); } + Shape output_shape(const std::vector &inputs) const override { + if (input_shape(inputs).elements() != shape_.elements() || shape_.layout != Layout::ChannelFirst) { + throw std::invalid_argument("NEON reshape element count or layout mismatch"); + } + return shape_; + } + void compute(const std::vector &inputs, float *output, const Shape &shape, + detail::Workspace &, detail::WorkerContext &worker) const noexcept override { + const auto begin = shape.elements() * worker.index / worker.count; + const auto end = shape.elements() * (worker.index + 1) / worker.count; + std::copy(inputs[0].data + begin, inputs[0].data + end, output + begin); + } +private: + Shape shape_; +}; + +class Transpose2dKernel final : public detail::Kernel { +public: + Shape output_shape(const std::vector &inputs) const override { + const auto &shape = input_shape(inputs); + if (shape.rank != 2) throw std::invalid_argument("NEON matrix transpose requires rank two"); + return Shape({shape.dimensions[1], shape.dimensions[0]}); + } + void compute(const std::vector &inputs, float *output, const Shape &shape, + detail::Workspace &, detail::WorkerContext &worker) const noexcept override { + const int rows = shape.dimensions[0], columns = shape.dimensions[1]; + int begin, end; + detail::split_range(rows, worker.index, worker.count, begin, end); + for (int row = begin; row < end; ++row) { + for (int col = 0; col < columns; ++col) output[row * columns + col] = inputs[0].data[col * rows + row]; + } + } +}; + +class ClampKernel final : public detail::Kernel { +public: + ClampKernel(float minimum, float maximum) : minimum_(minimum), maximum_(maximum) { + if (std::isnan(minimum) || std::isnan(maximum) || minimum > maximum) { + throw std::invalid_argument("NEON clamp bounds invalid"); + } + } + Shape output_shape(const std::vector &inputs) const override { return input_shape(inputs); } + bool inplace() const noexcept override { return true; } + void compute(const std::vector &inputs, float *output, const Shape &shape, + detail::Workspace &, detail::WorkerContext &worker) const noexcept override { + const auto begin = shape.elements() * worker.index / worker.count; + const auto end = shape.elements() * (worker.index + 1) / worker.count; + for (auto i = begin; i < end; ++i) output[i] = std::clamp(inputs[0].data[i], minimum_, maximum_); + } +private: + float minimum_, maximum_; +}; + +class QuantizeDequantizeKernel final : public detail::Kernel { +public: + QuantizeDequantizeKernel(float scale, int zero_point, int minimum, int maximum) + : scale_(scale), minimum_(float(minimum) - float(zero_point)), maximum_(float(maximum) - float(zero_point)) { + if (!std::isfinite(scale) || scale <= 0 || !std::isfinite(1.0F / scale) || minimum > maximum || + zero_point < minimum || zero_point > maximum) { + throw std::invalid_argument("NEON quantization parameters invalid"); + } + } + Shape output_shape(const std::vector &inputs) const override { return input_shape(inputs); } + bool inplace() const noexcept override { return true; } + void compute(const std::vector &inputs, float *output, const Shape &shape, + detail::Workspace &, detail::WorkerContext &worker) const noexcept override { + const auto begin = shape.elements() * worker.index / worker.count; + const auto end = shape.elements() * (worker.index + 1) / worker.count; + detail::quantize_dequantize(inputs[0].data + begin, output + begin, end - begin, scale_, minimum_, maximum_); + } +private: + float scale_, minimum_, maximum_; +}; + +} // namespace + +Concat::Concat(int axis) : Operator(std::make_shared(axis)) {} +Slice::Slice(int axis, int start, int length) : Operator(std::make_shared(axis, start, length)) {} +Reshape::Reshape(Shape shape) : Operator(std::make_shared(shape)) {} +Transpose2d::Transpose2d() : Operator(std::make_shared()) {} +Clamp::Clamp(float minimum, float maximum) : Operator(std::make_shared(minimum, maximum)) {} +QuantizeDequantize::QuantizeDequantize(float scale, int zero_point, int minimum, int maximum) + : Operator(std::make_shared(scale, zero_point, minimum, maximum)) {} +} // namespace engine::neon diff --git a/src/models/micro_wake_word/neon_network.cpp b/src/models/micro_wake_word/neon_network.cpp new file mode 100644 index 000000000..7934fa656 --- /dev/null +++ b/src/models/micro_wake_word/neon_network.cpp @@ -0,0 +1,183 @@ +#include "engine/models/micro_wake_word/neon_network.h" + +#include +#include +#include +#include + +namespace engine::models::micro_wake_word { +namespace { +using Value = neon::ExecutionPlan::Value; + +neon::Shape shape(const NetworkTensorInfo &info) { + const auto &s = info.tflite_shape; + if (s.size() == 4 && s[0] == 1 && s[2] == 1) return neon::Shape({int(s[3]), int(s[1])}); + if (s.size() == 3 && s[0] == 1) return neon::Shape({int(s[2]), int(s[1])}); + if (s.size() == 2 && s[0] == 1) return neon::Shape({int(s[1]), 1}); + if (s.size() == 1) return neon::Shape({int(s[0]), 1}); + throw std::runtime_error("microWakeWord NEON requires batch one and a unit spatial width"); +} + +int axis(int original, size_t rank) { + if (original < 0) original += int(rank); + if (original == 1 && rank >= 3) return 1; + if (original == int(rank) - 1) return 0; + throw std::runtime_error("microWakeWord NEON operator axis is not a channel or time axis"); +} + +bool same_quantization(const NetworkTensorInfo &a, const NetworkTensorInfo &b) { + return a.type == b.type && a.scale == b.scale && a.zero_point == b.zero_point && + a.per_channel_quantized == b.per_channel_quantized; +} +} // namespace + +NeonNetwork::NeonNetwork(const assets::TensorSource &source, const io::json::Value &config, + const std::vector &tensors, int threads) + : threads_(threads) { + namespace json = io::json; + const int input_index = json::require_i32(config, "input_tensor"); + const int output_index = json::require_i32(config, "output_tensor"); + input_shape_ = shape(tensors.at(input_index)); + std::unordered_map values; + values.emplace(input_index, plan_.input()); + std::vector state_assigns; + for (const auto &state : config.require("states").as_array()) { + const auto read = json::require_i32(state, "read"); + const auto state_shape = shape(tensors.at(read)); + states_.push_back({state_shape, std::vector(state_shape.elements(), 0.0F)}); + values.emplace(read, plan_.input()); + state_assigns.push_back(json::require_i32(state, "assign")); + } + inputs_.reserve(1 + states_.size()); + + const auto quantize = [&](Value value, int index) { + const auto &info = tensors.at(index); + if (info.per_channel_quantized) throw std::runtime_error("microWakeWord NEON activation quantization must be per-tensor"); + if (!info.scale) return value; + if (info.type != "i8" && info.type != "u8") throw std::runtime_error("microWakeWord NEON activation must use i8/u8 quantization"); + return plan_.add(neon::QuantizeDequantize(*info.scale, int(info.zero_point), + info.type == "i8" ? -128 : 0, info.type == "i8" ? 127 : 255), {value}); + }; + const auto finish = [&](Value value, int index, const std::string &activation) { + if (activation == "relu") value = plan_.add(neon::Clamp(0, std::numeric_limits::infinity()), {value}); + else if (activation == "relu6") value = plan_.add(neon::Clamp(0, 6), {value}); + else if (activation != "none") throw std::runtime_error("microWakeWord NEON activation is unsupported"); + values.insert_or_assign(index, quantize(value, index)); + }; + const auto passthrough = [&](Value value, int input, int output) { + if (!same_quantization(tensors.at(input), tensors.at(output))) { + throw std::runtime_error("microWakeWord NEON view operator changes quantization"); + } + values.insert_or_assign(output, value); + }; + + for (const auto &op : config.require("operators").as_array()) { + const auto type = json::require_string(op, "type"); + const auto in = json::require_i64_array(op, "inputs"); + const auto out = json::require_i64_array(op, "outputs"); + auto value = values.at(int(in.at(0))); + const int input = int(in[0]), output = int(out.at(0)); + if (type == "reshape" || type == "expand_dims") { + const auto &before = tensors.at(input).tflite_shape; + const auto &after = tensors.at(output).tflite_shape; + // TFLite flattens NHWC storage, whereas the native network uses C,T. + if (before.size() == 4 && after.size() == 2) value = plan_.add(neon::Transpose2d(), {value}); + if (!(shape(tensors.at(input)) == shape(tensors.at(output))) || (before.size() == 4 && after.size() == 2)) { + value = plan_.add(neon::Reshape(shape(tensors.at(output))), {value}); + } + passthrough(value, input, output); + } else if (type == "concatenation") { + if (json::require_string(op, "activation") != "none") throw std::runtime_error("microWakeWord NEON concat activation unsupported"); + const int dimension = axis(json::require_i32(op, "axis"), tensors.at(input).tflite_shape.size()); + for (size_t i = 0; i < in.size(); ++i) { + if (!same_quantization(tensors.at(int(in[i])), tensors.at(output))) { + throw std::runtime_error("microWakeWord NEON concat changes quantization"); + } + if (i) value = plan_.add(neon::Concat(dimension), {value, values.at(int(in[i]))}); + } + values.insert_or_assign(output, value); + } else if (type == "strided_slice") { + const auto begin = json::require_i64_array(op, "begin"), end = json::require_i64_array(op, "end"); + const auto &source_shape = tensors.at(input).tflite_shape; + for (size_t a = 0; a < begin.size(); ++a) { + if (begin[a] || end[a] != source_shape.at(a)) { + value = plan_.add(neon::Slice(axis(int(a), begin.size()), int(begin[a]), int(end[a] - begin[a])), {value}); + } + } + passthrough(value, input, output); + } else if (type == "split_v") { + const auto sizes = json::require_i64_array(op, "sizes"); + if (sizes.size() != out.size()) throw std::runtime_error("microWakeWord NEON split size mismatch"); + const int dimension = axis(json::require_i32(op, "axis"), tensors.at(input).tflite_shape.size()); + int offset = 0; + for (size_t i = 0; i < sizes.size(); ++i) { + passthrough(plan_.add(neon::Slice(dimension, offset, int(sizes[i])), {value}), input, int(out[i])); + offset += int(sizes[i]); + } + } else if (type == "conv_2d" || type == "depthwise_conv_2d") { + const auto name = json::require_string(op, "weight"); + const auto dimensions = source.require_metadata(name).shape; + const auto weights = source.require_f32(name, dimensions); + const auto bias = source.require_f32(json::require_string(op, "bias"), {dimensions.at(0)}); + const auto stride = json::require_i64_array(op, "stride"), dilation = json::require_i64_array(op, "dilation"); + const auto input_shape = shape(tensors.at(input)), output_shape = shape(tensors.at(output)); + if (dimensions.size() != 4 || dimensions[3] != 1 || stride.at(1) != 1 || dilation.at(1) != 1) { + throw std::runtime_error("microWakeWord NEON convolution must have unit spatial width"); + } + const int padding_mode = json::require_i32(op, "padding"); + if (padding_mode != 0 && padding_mode != 1) throw std::runtime_error("microWakeWord NEON convolution padding invalid"); + const int total_padding = padding_mode == 1 ? 0 : std::max(0, + (output_shape.dimensions[1] - 1) * stride[0] + dilation[0] * (dimensions[2] - 1) + 1 - input_shape.dimensions[1]); + if (total_padding % 2) throw std::runtime_error("microWakeWord NEON asymmetric SAME padding unsupported"); + neon::Conv1dConfig c{input_shape.dimensions[0], output_shape.dimensions[0], int(dimensions[2]), + int(stride[0]), int(dilation[0]), total_padding / 2, total_padding / 2}; + if (type == "depthwise_conv_2d") { + value = plan_.add(neon::DepthwiseConv1d(c, weights, bias), {value}); + } else if (!total_padding && dilation[0] == 1 && input_shape.dimensions[1] == c.kernel_size && output_shape.dimensions[1] == 1) { + // A valid convolution with one output is exactly one matrix-vector product. + value = plan_.add(neon::Reshape(neon::Shape({c.input_channels * c.kernel_size, 1})), {value}); + value = plan_.add(neon::Linear(c.input_channels * c.kernel_size, c.output_channels, weights, bias), {value}); + } else { + value = plan_.add(neon::Conv1d(c, weights, bias), {value}); + } + finish(value, output, json::require_string(op, "activation")); + } else if (type == "fully_connected") { + const auto name = json::require_string(op, "weight"); + const auto dimensions = source.require_metadata(name).shape; + value = plan_.add(neon::Linear(int(dimensions.at(1)), int(dimensions.at(0)), source.require_f32(name, dimensions), + source.require_f32(json::require_string(op, "bias"), {dimensions[0]})), {value}); + finish(value, output, json::require_string(op, "activation")); + } else if (type == "logistic") { + finish(plan_.add(neon::Activation(neon::ActivationKind::Sigmoid), {value}), output, "none"); + } else if (type == "quantize") { + finish(value, output, "none"); + } else throw std::runtime_error("microWakeWord NEON operator unsupported: " + type); + } + + for (size_t i = 0; i < state_assigns.size(); ++i) { + const auto value = plan_.add(neon::Reshape(states_[i].shape), {values.at(state_assigns[i])}); + plan_.retain(value); + state_outputs_.push_back(value); + } + plan_.set_output(values.at(output_index)); + if (tensors.at(output_index).type == "u8" && tensors[output_index].scale) output_scale_ = 256.0F / 255.0F; +} + +float NeonNetwork::infer(const std::vector &input) { + inputs_.clear(); + inputs_.emplace_back(input, input_shape_); + for (const auto &state : states_) inputs_.emplace_back(state.values, state.shape); + const auto &output = plan_.run(inputs_, threads_); + if (output.values.size() != 1) throw std::runtime_error("microWakeWord NEON probability shape invalid"); + // Copy all retained new state only after every consumer of the old state has finished. + for (size_t i = 0; i < states_.size(); ++i) { + const auto view = plan_.view(state_outputs_[i]); + std::copy_n(view.data, view.size, states_[i].values.data()); + } + return output.values[0] * output_scale_; +} + +void NeonNetwork::reset() { + for (auto &state : states_) std::fill(state.values.begin(), state.values.end(), 0.0F); +} +} // namespace engine::models::micro_wake_word diff --git a/src/models/micro_wake_word/runtime.cpp b/src/models/micro_wake_word/runtime.cpp index 391e56eea..ecf1ba165 100644 --- a/src/models/micro_wake_word/runtime.cpp +++ b/src/models/micro_wake_word/runtime.cpp @@ -1,4 +1,7 @@ #include "engine/models/micro_wake_word/runtime.h" +#ifdef ENGINE_HAVE_NEON +#include "engine/models/micro_wake_word/neon_network.h" +#endif #include "engine/framework/core/backend_weight_store.h" #include "engine/framework/debug/profiler.h" @@ -20,13 +23,7 @@ namespace engine::models::micro_wake_word { namespace json = engine::io::json; namespace { -struct TensorInfo { - std::vector tflite_shape; - std::string type; - std::optional scale; - int64_t zero_point = 0; - bool per_channel_quantized = false; -}; +using TensorInfo = NetworkTensorInfo; struct StateInfo { std::string name; @@ -125,9 +122,12 @@ struct Runtime::Impl { float input_scale = 0.0F; int64_t input_zero_point = 0; ggml_cgraph * graph = nullptr; +#ifdef ENGINE_HAVE_NEON + std::unique_ptr neon_network; +#endif Impl(const assets::TensorSource & source, const json::Value & root, - core::ExecutionContext & execution_context) + core::ExecutionContext & execution_context, bool use_neon) : execution(execution_context), store(execution.backend(), execution.backend_type(), "micro_wake_word.weights", 1024 * 1024) { if (json::require_string(root, "format") != "micro_wake_word_tflite_v1") { @@ -170,17 +170,26 @@ struct Runtime::Impl { throw std::runtime_error("microWakeWord input/output tensor index is invalid"); } - context.reset(ggml_init({8 * 1024 * 1024, nullptr, true})); - if (!context) throw std::runtime_error("microWakeWord graph context allocation failed"); - core::ModuleBuildContext ctx{context.get(), "micro_wake_word", execution.backend_type()}; - input = core::make_tensor(ctx, GGML_TYPE_F32, native_shape(infos[input_index].tflite_shape)); - ggml_set_input(input.tensor); if (!infos[input_index].scale.has_value() || infos[input_index].per_channel_quantized || infos[input_index].type != "i8") { throw std::runtime_error("microWakeWord input quantization metadata is unsupported"); } input_scale = *infos[input_index].scale; input_zero_point = infos[input_index].zero_point; + if (use_neon) { +#ifdef ENGINE_HAVE_NEON + if (!neon::available()) throw std::runtime_error("microWakeWord neon network requires AArch64 NEON"); + neon_network = std::make_unique(source, root, infos, execution.config().threads); + return; +#else + throw std::runtime_error("microWakeWord neon network requires ENGINE_ENABLE_NEON=ON"); +#endif + } + context.reset(ggml_init({8 * 1024 * 1024, nullptr, true})); + if (!context) throw std::runtime_error("microWakeWord graph context allocation failed"); + core::ModuleBuildContext ctx{context.get(), "micro_wake_word", execution.backend_type()}; + input = core::make_tensor(ctx, GGML_TYPE_F32, native_shape(infos[input_index].tflite_shape)); + ggml_set_input(input.tensor); std::unordered_map values; values.emplace(input_index, input); @@ -360,7 +369,7 @@ struct Runtime::Impl { clear(); } - ~Impl() { core::release_backend_graph_resources(execution.backend(), graph, true); } + ~Impl() { if (graph) core::release_backend_graph_resources(execution.backend(), graph, true); } float run(const std::vector & features) { if (features.size() != static_cast(model.input_frames * model.feature_dim)) { @@ -377,13 +386,21 @@ struct Runtime::Impl { (static_cast(quantized) - static_cast(input_zero_point)) * input_scale; } } - core::write_tensor_f32(input, channel_major); + if (graph) core::write_tensor_f32(input, channel_major); const auto started = std::chrono::steady_clock::now(); - if (core::compute_graph(execution, graph, plan, "micro_wake_word") != GGML_STATUS_SUCCESS) { - throw std::runtime_error("microWakeWord graph compute failed"); + std::vector result; +#ifdef ENGINE_HAVE_NEON + if (neon_network) { + result = {neon_network->infer(channel_major)}; + } else +#endif + { + if (core::compute_graph(execution, graph, plan, "micro_wake_word") != GGML_STATUS_SUCCESS) { + throw std::runtime_error("microWakeWord graph compute failed"); + } } debug::timing_log_scalar("micro_wake_word.network_ms", debug::elapsed_ms(started)); - const auto result = core::read_tensor_f32(output.tensor); + if (graph) result = core::read_tensor_f32(output.tensor); if (result.size() != 1 || !std::isfinite(result[0])) { throw std::runtime_error("microWakeWord produced an invalid probability"); } @@ -392,6 +409,9 @@ struct Runtime::Impl { } void clear() { +#ifdef ENGINE_HAVE_NEON + if (neon_network) { neon_network->reset(); return; } +#endif for (const auto & state : state_inputs) { ggml_backend_tensor_memset(state.tensor, 0, 0, ggml_nbytes(state.tensor)); } @@ -400,8 +420,8 @@ struct Runtime::Impl { Runtime::Runtime(std::shared_ptr source, const io::json::Value & config, - core::ExecutionContext & execution) - : impl_(std::make_unique(*source, config, execution)) {} + core::ExecutionContext & execution, bool use_neon) + : impl_(std::make_unique(*source, config, execution, use_neon)) {} Runtime::~Runtime() = default; const ModelConfig & Runtime::config() const noexcept { return impl_->model; } float Runtime::infer(const std::vector & features) { return impl_->run(features); } diff --git a/src/models/micro_wake_word/session.cpp b/src/models/micro_wake_word/session.cpp index 9cbecbd2d..08bc3ae4d 100644 --- a/src/models/micro_wake_word/session.cpp +++ b/src/models/micro_wake_word/session.cpp @@ -22,6 +22,12 @@ struct Assets { io::json::Value config; }; +bool use_neon_network(const runtime::SessionOptions &options, core::BackendType backend) { + const auto plan = runtime::parse_exec_plan(options.options, "micro_wake_word.exec_plan", + {{"network", "ggml", {{"ggml", {backend}, {}, {}}, {"neon", {core::BackendType::Cpu}, {}, {}}}}}); + return plan.at("network").impl == "neon"; +} + class Session final : public runtime::RuntimeSessionBase, public runtime::IOfflineVoiceTaskSession, public runtime::IStreamingVoiceTaskSession { @@ -32,7 +38,7 @@ class Session final : public runtime::RuntimeSessionBase, : RuntimeSessionBase(options), mode_(mode), assets_(std::move(assets)), contract_(std::move(contract)), network_(assets_->resources.open_tensor_source("weights"), assets_->config, - execution_context()) {} + execution_context(), use_neon_network(options, execution_context().backend_type())) {} std::string family() const override { return "micro_wake_word"; } runtime::VoiceTaskKind task_kind() const override { return runtime::VoiceTaskKind::WakeWord; } @@ -234,8 +240,12 @@ std::shared_ptr make_micro_wake_word_loader() { if (task.task != runtime::VoiceTaskKind::WakeWord) { throw std::runtime_error("microWakeWord supports the wake-word task"); } + auto validation_options = options; + if (!contract->session_option_keys.count("micro_wake_word.exec_plan")) { + validation_options.options.erase("micro_wake_word.exec_plan"); + } runtime::validate_spec_backed_session_options( - options, *contract, "micro_wake_word", "microWakeWord"); + validation_options, *contract, "micro_wake_word", "microWakeWord"); return std::make_unique(options, task.mode, std::move(assets), std::move(contract)); }; return runtime::make_spec_backed_voice_loader(std::move(config)); diff --git a/tests/unittests/test_neon_tensor_ops.cpp b/tests/unittests/test_neon_tensor_ops.cpp new file mode 100644 index 000000000..86a5226ce --- /dev/null +++ b/tests/unittests/test_neon_tensor_ops.cpp @@ -0,0 +1,78 @@ +#include "engine/framework/neon/execution.h" + +#include +#include +#include +#include + +namespace n = engine::neon; + +void expect(const std::vector &actual, const std::vector &expected, float tolerance = 0) { + if (actual.size() != expected.size()) throw std::runtime_error("NEON sequence shape mismatch"); + for (size_t i = 0; i < actual.size(); ++i) { + if (!std::isfinite(actual[i]) || std::abs(actual[i] - expected[i]) > tolerance) { + throw std::runtime_error("NEON sequence scalar parity failed at " + std::to_string(i)); + } + } +} + +int main() try { + for (int threads : {1, 4}) { + n::ExecutionPlan plan; + const auto a = plan.input(), b = plan.input(); + const auto joined = plan.add(n::Concat(1), {a, b}); + const auto sliced = plan.add(n::Slice(1, 1, 3), {joined}); + const auto transposed = plan.add(n::Transpose2d(), {sliced}); + plan.set_output(plan.add(n::Reshape({6, 1}), {transposed})); + expect(plan.run({n::TensorView(std::vector{1,2,3,4,5,6}, {2,3}), + n::TensorView(std::vector{7,8,9,10}, {2,2})}, threads).values, + {2,5,3,6,7,9}); + + for (float scale : {0.25f, 0.07f, 1.0f / 256}) for (int zero_point : {-128, 0, 128}) { + const int minimum = zero_point == 128 ? 0 : -128, maximum = zero_point == 128 ? 255 : 127; + std::vector input{-100, -32.125f, -1.375f, -0.375f, -0.125f, 0, 0.125f, 0.375f, 1.375f, 31.875f, 100}; + input.push_back(-0.5f * scale); + input.push_back(0.5f * scale); + std::vector expected; + for (float value : input) expected.push_back(std::clamp(std::round(value * (1.0f / scale)), float(minimum - zero_point), float(maximum - zero_point)) * scale); + n::ExecutionPlan quantization; + const auto in = quantization.input(); + quantization.set_output(quantization.add(n::QuantizeDequantize(scale, zero_point, minimum, maximum), {in})); + expect(quantization.run({n::TensorView(input, {1,int(input.size())})}, threads).values, expected); + } + + n::ExecutionPlan activation; + const auto in = activation.input(); + const auto value = activation.add(n::Clamp(-2, 2), {in}); + activation.set_output(activation.add(n::Activation(n::ActivationKind::Sigmoid), {value})); + expect(activation.run({n::TensorView(std::vector{-8,-1,0,1,8}, {1,5})}, threads).values, + {1/(1+std::exp(2.0f)), 1/(1+std::exp(1.0f)), 0.5f, 1/(1+std::exp(-1.0f)), 1/(1+std::exp(-2.0f))}); + + for (int kernel : {1, 3, 5}) for (int stride : {1, 2}) for (int dilation : {1, 2}) { + const int channels = 3, frames = 15; + std::vector input(channels * frames), weight(channels * kernel), bias{0.1f, -0.2f, 0.3f}; + for (size_t i = 0; i < input.size(); ++i) input[i] = std::sin(float(i)); + for (size_t i = 0; i < weight.size(); ++i) weight[i] = std::cos(float(i)); + n::Conv1dConfig config{channels, channels, kernel, stride, dilation, 1, 2}; + const int count = (frames + 3 - dilation * (kernel - 1) - 1) / stride + 1; + std::vector expected(channels * count); + for (int channel = 0; channel < channels; ++channel) for (int frame = 0; frame < count; ++frame) { + float sum = bias[channel]; + for (int tap = 0; tap < kernel; ++tap) { + const int at = frame * stride - 1 + tap * dilation; + if (at >= 0 && at < frames) sum += input[channel * frames + at] * weight[channel * kernel + tap]; + } + expected[channel * count + frame] = sum; + } + n::ExecutionPlan convolution; + const auto signal = convolution.input(); + convolution.set_output(convolution.add(n::DepthwiseConv1d(config, weight, bias), {signal})); + expect(convolution.run({n::TensorView(input, {channels, frames})}, threads).values, expected, 2e-6f); + } + } + std::cout << "NEON tensor operators passed\n"; + return 0; +} catch (const std::exception &error) { + std::cerr << error.what() << '\n'; + return 1; +}