diff --git a/modules/dnn/include/opencv2/dnn/all_layers.hpp b/modules/dnn/include/opencv2/dnn/all_layers.hpp index 14e629e8ea..37156a71fa 100644 --- a/modules/dnn/include/opencv2/dnn/all_layers.hpp +++ b/modules/dnn/include/opencv2/dnn/all_layers.hpp @@ -1519,6 +1519,11 @@ CV__DNN_INLINE_NS_BEGIN static Ptr create(const LayerParams ¶ms); }; + class CV_EXPORTS AttentionOnnxAiLayer : public Layer { + public: + static Ptr create(const LayerParams ¶ms); + }; + class CV_EXPORTS GroupNormLayer : public Layer { public: static Ptr create(const LayerParams ¶ms); diff --git a/modules/dnn/src/init.cpp b/modules/dnn/src/init.cpp index f22e538a91..9cd5bd84ce 100644 --- a/modules/dnn/src/init.cpp +++ b/modules/dnn/src/init.cpp @@ -203,6 +203,7 @@ void initializeLayerFactory() CV_DNN_REGISTER_LAYER_CLASS(Expand, ExpandLayer); CV_DNN_REGISTER_LAYER_CLASS(InstanceNormalization, InstanceNormLayer); CV_DNN_REGISTER_LAYER_CLASS(Attention, AttentionLayer); + CV_DNN_REGISTER_LAYER_CLASS(AttentionOnnxAi, AttentionOnnxAiLayer); CV_DNN_REGISTER_LAYER_CLASS(RotaryEmbedding, RotaryEmbeddingLayer); CV_DNN_REGISTER_LAYER_CLASS(GroupNormalization, GroupNormLayer); CV_DNN_REGISTER_LAYER_CLASS(Cast, CastLayer); diff --git a/modules/dnn/src/layers/attention_onnxai_layer.cpp b/modules/dnn/src/layers/attention_onnxai_layer.cpp new file mode 100644 index 0000000000..b9fb62fdce --- /dev/null +++ b/modules/dnn/src/layers/attention_onnxai_layer.cpp @@ -0,0 +1,250 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. + +#include "../precomp.hpp" +#include "cpu_kernels/fast_gemm.hpp" +#include "cpu_kernels/fast_attn.hpp" + +#include "layers_common.hpp" +#include "../net_impl.hpp" + +#include + +namespace cv { namespace dnn { + +// Operator spec: https://onnx.ai/onnx/operators/onnx__Attention.html#attention-23 +class AttentionOnnxAiLayerImpl CV_FINAL : public AttentionOnnxAiLayer { + public: + AttentionOnnxAiLayerImpl(const LayerParams ¶ms) { + setParamsFrom(params); + is_causal = params.get("is_causal", false); + kv_num_heads = params.get("kv_num_heads", 0); + q_num_heads = params.get("q_num_heads", 0); + qk_matmul_output_mode = params.get("qk_matmul_output_mode", 0); + scale = params.get("scale", 1.0f ); + is_scale_set = params.has("scale"); + softcap = params.get("softcap", 0.f); + softmax_precision = params.get("softmax_precision", 0); + } + + virtual bool supportBackend(int backendId) CV_OVERRIDE { + return backendId == DNN_BACKEND_OPENCV; + } + + virtual void getTypes(const std::vector&inputs, + const int requiredOutputs, + const int requiredInternals, + std::vector&outputs, + std::vector&internals) const CV_OVERRIDE { + // type checks + CV_CheckTrue(inputs.size() >= 3, "At least three inputs (query, key, value) are required"); + + for (int i = 0; i < 3; i++) { + CV_CheckType(inputs[i], inputs[i] == CV_16F || inputs[i] == CV_32F, ""); + } + + CV_CheckType(inputs[0], inputs[0] == inputs[1] && inputs[0] == inputs[2], ""); + + if (inputs.size() >= 4) { + CV_CheckType(inputs[3], inputs[3] == CV_8U || inputs[3] == CV_8S || + inputs[3] == CV_16U || inputs[3] == CV_16S || + inputs[3] == CV_32S || inputs[3] == CV_64S || + inputs[3] == CV_64U || inputs[3] == CV_Bool || + inputs[3] == inputs[2], ""); // attention_mask + } + + outputs.assign(requiredOutputs, inputs[0]); + + // internals: + internals.clear(); + + // 1. attention_prob + internals.push_back(inputs[0]); + } + + virtual bool getMemoryShapes(const std::vector &inputs, + const int requiredOutputs, + std::vector &outputs, + std::vector &internals) const CV_OVERRIDE { + CV_CheckTrue(inputs.size() >= 3, "At least three inputs (query, key, value) are required"); + CV_CheckTrue(inputs.size() < 5, "past key and past value are not supported yet"); + CV_CheckTrue(inputs[0].dims == inputs[1].dims && + inputs[0].dims == inputs[2].dims, + "Query, key and value must have the same number of dimensions"); + + const int input_dims = inputs[0].dims; + CV_CheckTrue( + input_dims == 4 || (q_num_heads > 0 && kv_num_heads > 0 && input_dims == 3), + "Input dimensions must be 4D or 3D (in the latter case, q_num_heads and kv_num_heads must be set)" + ); + + const int batch_size = inputs[0][0]; + const int seq_len_q = inputs[0][input_dims - 2]; + const int seq_len_kv = inputs[1][input_dims - 2]; + + const int q_hn = input_dims == 4 ? + inputs[0][1] : q_num_heads; + const int kv_hn =input_dims == 4 ? + inputs[1][1] : kv_num_heads; + + CV_CheckTrue(inputs[2][input_dims - 2] == seq_len_kv, + "Key and query sequence lengths must be equal"); + const int nhq = input_dims == 4 ? inputs[0][1] : q_num_heads; + + CV_CheckTrue(q_hn % kv_hn == 0, + "q_num_heads must be divisible by kv_num_heads"); + + if (input_dims == 3) + { + CV_CheckTrue(kv_hn > 0, + "For 3D input, kv_num_heads must be greater than 0 (this normally means that kv_num_heads is not set)"); + CV_CheckTrue(q_hn > 0, + "For 3D input, q_num_heads must be greater than 0 (this normally means that q_num_heads is not set)"); + + int v_head_size = inputs[2][2] / kv_hn; + MatShape output_shape{batch_size, seq_len_q, v_head_size * q_num_heads}; + outputs.push_back(output_shape); + } + else + { + int v_head_size = inputs[2][3]; + MatShape output_shape{batch_size, nhq, seq_len_q, v_head_size}; + outputs.push_back(output_shape); + } + + MatShape attention_prob_shape{batch_size , nhq, seq_len_q, seq_len_kv}; + internals.push_back(attention_prob_shape); + + return false; + } + + void forward(InputArrayOfArrays inputs_arr, OutputArrayOfArrays outputs_arr, OutputArrayOfArrays internals_arr) CV_OVERRIDE { + opt.init(); + + if (inputs_arr.depth() == CV_16F) + { + forward_fallback(inputs_arr, outputs_arr, internals_arr); + return; + } + + std::vector inputs, outputs, internals; + inputs_arr.getMatVector(inputs); + outputs_arr.getMatVector(outputs); + internals_arr.getMatVector(internals); + + const int input_dims = inputs[0].dims; + const int batch_size = inputs[0].size[0]; + + const int nhq = input_dims == 3 ? + q_num_heads : + inputs[0].size[1]; + const int nhkv = input_dims == 3 ? + kv_num_heads : + inputs[1].size[1]; + + const int qk_head_size = input_dims == 3 ? + inputs[0].size[2] / nhq : + inputs[0].size[3]; + const int v_head_size = input_dims == 3 ? + inputs[2].size[2] / nhkv : + inputs[2].size[3]; + const int num_gq_groups = nhq / nhkv; + const int seq_len_q = input_dims == 3 ? + inputs[0].size[1]: + inputs[0].size[2]; + const int seq_len_kv = input_dims == 3 ? + inputs[1].size[1]: + inputs[1].size[2]; + const auto seq_len_square = seq_len_q * seq_len_kv; + + const auto* Q = inputs[0].ptr(); + const auto* K = inputs[1].ptr(); + const auto* V = inputs[2].ptr(); + + scale = is_scale_set ? scale : 1.0f / std::sqrt(static_cast(qk_head_size)); + + std::vector _q_offsets(nhq * batch_size), + _k_offsets(nhq * batch_size), + _a_offsets(nhq * batch_size), + _v_offsets(nhq * batch_size), + _o_offsets(nhq * batch_size); + + for (int b = 0; b < batch_size; b++) + for (int n = 0; n < nhq; n++){ + _q_offsets[b * nhq + n] = + b * seq_len_q * qk_head_size * nhq + + (input_dims == 3 ? n * qk_head_size : n * qk_head_size * seq_len_q); + _k_offsets[b * nhq + n] = + b * seq_len_kv * qk_head_size * nhkv + + (n / num_gq_groups * qk_head_size) * (input_dims == 3 ? 1 : seq_len_kv); + _v_offsets[b * nhq + n] = + b * seq_len_kv * v_head_size * nhkv + + (n / num_gq_groups * v_head_size) * (input_dims == 3 ? 1 : seq_len_kv); + _a_offsets[b * nhq + n] = + b * seq_len_square * nhq + + n * seq_len_square; + _o_offsets[b * nhq + n] = + b * seq_len_q * v_head_size * nhq + + (input_dims == 3 ? n * v_head_size : n * v_head_size * seq_len_q); + } + + const int ldq0 = input_dims == 3 ? qk_head_size * nhq : qk_head_size; + const int ldk0 = input_dims == 3 ? qk_head_size * nhkv : qk_head_size; + auto &attention_prob = internals[internals.size() - 1]; + + fastGemmBatch( + batch_size * nhq, + _q_offsets.data(), _k_offsets.data(), _a_offsets.data(), + seq_len_q, seq_len_kv, qk_head_size , scale, + Q, ldq0, 1, + K, 1, ldk0, + 0.f, + attention_prob.ptr(), seq_len_kv, + opt + ); + + + fused_softmax_softcap_mask( + attention_prob, + inputs.size() > 3 ? inputs[3] : Mat(), + softcap, softcap > 0.f, + 9.f, + -FLT_MAX, + inputs.size() > 3, + is_causal + ); + + + const int ldv0 = input_dims == 3 ? v_head_size * nhkv : v_head_size; + const int ldout = input_dims == 3 ? v_head_size * nhq : v_head_size; + + fastGemmBatch( + batch_size * nhq, + _a_offsets.data(), _v_offsets.data(), _o_offsets.data(), + seq_len_q, v_head_size, seq_len_kv, 1.f, + attention_prob.ptr(), seq_len_kv, 1, + V, ldv0, 1, + 0.f, + outputs[0].ptr(), ldout, + opt + ); + } + + private: + bool is_causal; + int kv_num_heads; + int q_num_heads; + int qk_matmul_output_mode; + float scale; + bool is_scale_set = false; + float softcap; + int softmax_precision; + FastGemmOpt opt; +}; + +Ptr AttentionOnnxAiLayer::create(const LayerParams ¶ms) { + return makePtr(params); +} + +}} // cv::dnn diff --git a/modules/dnn/src/layers/cpu_kernels/fast_attn.cpp b/modules/dnn/src/layers/cpu_kernels/fast_attn.cpp new file mode 100644 index 0000000000..1fb40f70c9 --- /dev/null +++ b/modules/dnn/src/layers/cpu_kernels/fast_attn.cpp @@ -0,0 +1,256 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. + +#include "../../precomp.hpp" +#include "opencv2/core/hal/intrin.hpp" + +namespace cv { namespace dnn { + +static inline v_float32 load_int_mask_as_f32(const int32_t* ptr) { + v_int32 v = vx_load(ptr); + return v_reinterpret_as_f32(v_ne(v, vx_setall_s32(0))); +} + +static inline v_float32 load_int_mask_as_f32(const uint8_t* ptr) { + v_uint32 v = vx_load_expand_q(ptr); + return v_reinterpret_as_f32(v_ne(v, vx_setall_u32(0))); +} + +static inline v_float32 load_int_mask_as_f32(const int8_t* ptr) { + v_int32 v = vx_load_expand_q(ptr); + return v_reinterpret_as_f32(v_ne(v, vx_setall_s32(0))); +} + +static inline v_float32 load_int_mask_as_f32(const uint16_t* ptr) { + v_uint32 v = vx_load_expand(ptr); + return v_reinterpret_as_f32(v_ne(v, vx_setall_u32(0))); +} + +template +inline v_float32 load_int_mask_as_f32(const T* ptr) { + return vx_setall_f32(0.f); +} + + +template +struct MaskPolicyFloat { + static inline v_float32 load_mask(const MaskT* ptr) { return vx_load((const float*)ptr); } + static inline v_float32 load_mask_scalar(const MaskT* ptr) { return vx_setall_f32((float)*ptr); } + static inline v_float32 apply(v_float32 val, v_float32 mask, v_float32 min_val) { return v_add(val, mask); } + static inline float apply_scalar(float val, MaskT mask, float min_val) { return val + mask; } +}; + +template +struct MaskPolicyInt { + static inline v_float32 load_mask(const MaskT* ptr) { return load_int_mask_as_f32(ptr); } + static inline v_float32 load_mask_scalar(const MaskT* ptr) { + return (*ptr != 0) ? v_reinterpret_as_f32(vx_setall_u32(0xFFFFFFFF)) : vx_setall_f32(0.f); + } + static inline v_float32 apply(v_float32 val, v_float32 mask, v_float32 min_val) { return v_select(mask, val, min_val); } + static inline float apply_scalar(float val, MaskT mask, float min_val) { return (mask != 0) ? val : min_val; } +}; + +struct MaskPolicyNone { + static inline v_float32 load_mask(const void* ptr) { return vx_setzero_f32(); } + static inline v_float32 load_mask_scalar(const void* ptr) { return vx_setzero_f32(); } + static inline v_float32 apply(v_float32 val, v_float32 mask, v_float32 min_val) { return val; } + static inline float apply_scalar(float val, int mask, float min_val) { return val; } +}; + +template +void run_fused_softmax( + Mat &att_weights, const Mat &att_mask, + const float softcap, const bool do_softcap, + const float threshold, + const float min_val, const bool is_causal) +{ + const int batch_size = att_weights.size[0]; + const int n_heads = att_weights.size[1]; + const int seq_len_q = att_weights.size[2]; + const int seq_len_kv = att_weights.size[3]; + + float* data = att_weights.ptr(); + const MaskT* mask_data = att_mask.empty() ? nullptr : att_mask.ptr(); + + size_t mask_step_b = 0, mask_step_h = 0, mask_step_q = 0, mask_step_k = 0; + if (mask_data) { + int dims_m = att_mask.dims; + int offset_dim = 4 - dims_m; + + auto get_size = [&](int i) { return (i < offset_dim) ? 1 : att_mask.size[i - offset_dim]; }; + auto get_step = [&](int i) { return (i < offset_dim) ? 0 : att_mask.step[i - offset_dim] / sizeof(MaskT); }; + + if (get_size(0) > 1) mask_step_b = get_step(0); + if (get_size(1) > 1) mask_step_h = get_step(1); + if (get_size(2) > 1) mask_step_q = get_step(2); + if (get_size(3) > 1) mask_step_k = 1; + } + + size_t total_tasks = (size_t)batch_size * n_heads; + + parallel_for_(Range(0, (int)total_tasks), [&](const Range &range) { + for (int i = range.start; i < range.end; i++) { + int b = i / n_heads; + int h = i % n_heads; + + size_t offset = (size_t)i * seq_len_q * seq_len_kv; + size_t mask_offset_q = b * mask_step_b + h * mask_step_h; + + for (int tq = 0; tq < seq_len_q; tq++){ + const int tmax = is_causal ? std::min(tq + 1, seq_len_kv) : seq_len_kv; + float maxVal = -FLT_MAX; + int tk = 0; +#if CV_SIMD + const int w = VTraits::vlanes(); + v_float32 v_max_val = vx_setall_f32(maxVal); + v_float32 v_softcap = vx_setall_f32(softcap); + v_float32 v_inv_softcap = vx_setall_f32(1.f / softcap); + v_float32 v_threshold = vx_setall_f32(threshold); + v_float32 v_minus_threshold = vx_setall_f32(-threshold); + v_float32 v_one = vx_setall_f32(1.f); + v_float32 v_minus_one = vx_setall_f32(-1.f); + v_float32 v_minus_two = vx_setall_f32(-2.f); + v_float32 v_min_val = vx_setall_f32(min_val); + + for (; tk <= tmax - w; tk += w) { + v_float32 v_val = vx_load(&data[offset + tk]); + + if (mask_data) { + v_float32 v_mask_val; + if (mask_step_k) + v_mask_val = Policy::load_mask(&mask_data[mask_offset_q + tk]); + else + v_mask_val = Policy::load_mask_scalar(&mask_data[mask_offset_q]); + v_val = Policy::apply(v_val, v_mask_val, v_min_val); + } + + if (do_softcap) { + v_float32 v_scaled = v_mul(v_val, v_inv_softcap); + + v_float32 v_mask_pos = v_gt(v_scaled, v_threshold); + v_float32 v_mask_neg = v_lt(v_scaled, v_minus_threshold); + + v_float32 v_scaled_safe = v_select(v_mask_neg, vx_setall_f32(0.f), v_scaled); + + v_float32 v_exp_part = v_exp(v_mul(v_minus_two, v_scaled_safe)); + v_float32 v_tanh = v_div(v_sub(v_one, v_exp_part), v_add(v_one, v_exp_part)); + + v_float32 v_res = v_select(v_mask_pos, v_one, + v_select(v_mask_neg, v_minus_one, v_tanh)); + + v_val = v_mul(v_res, v_softcap); + } + vx_store(&data[offset + tk], v_val); + v_max_val = v_max(v_max_val, v_val); + } + maxVal = v_reduce_max(v_max_val); +#endif + for (; tk < tmax; tk++){ + if (mask_data) { + size_t mask_idx = mask_offset_q + (mask_step_k ? tk : 0); + data[offset + tk] = Policy::apply_scalar(data[offset + tk], mask_data[mask_idx], min_val); + } + + if (do_softcap) { + float softcap_val = data[offset + tk] / softcap; + if (softcap_val > threshold) + data[offset + tk ] = 1.f; + else if(softcap_val < -threshold) + data[offset + tk ] = -1.f; + else + data[offset + tk] = (1.f - expf(-2 *softcap_val)) / (1.f + expf(-2 *softcap_val)); + data[offset + tk] *= softcap; + } + maxVal = std::max(maxVal, data[offset + tk]); + } + + for (; tk < seq_len_kv; tk++) { + data[offset + tk] = min_val; + } + + float sum = 0.f; + tk = 0; +#if CV_SIMD + v_float32 v_sum = vx_setzero_f32(); + v_float32 v_max_val_shift = vx_setall_f32(maxVal); + for (; tk <= seq_len_kv - w; tk += w) { + v_float32 v_val = vx_load(&data[offset + tk]); + v_val = v_sub(v_val, v_max_val_shift); + v_val = v_exp(v_val); + v_sum = v_add(v_sum, v_val); + vx_store(&data[offset + tk], v_val); + } + sum = v_reduce_sum(v_sum); +#endif + for (; tk < seq_len_kv; tk++){ + data[offset + tk] = expf(data[offset + tk] - maxVal); + sum += data[offset + tk]; + } + + float inv_sum = 1.f / sum; + tk = 0; +#if CV_SIMD + v_float32 v_inv_sum = vx_setall_f32(inv_sum); + for (; tk <= seq_len_kv - w; tk += w) { + v_float32 v_val = vx_load(&data[offset + tk]); + v_val = v_mul(v_val, v_inv_sum); + vx_store(&data[offset + tk], v_val); + } +#endif + for (; tk < seq_len_kv; tk++){ + data[offset + tk] *= inv_sum; + } + + offset += seq_len_kv; + mask_offset_q += mask_step_q; + } + } + }); +} + +void fused_softmax_softcap_mask( + Mat &att_weights,const Mat &att_mask, + const float softcap, const bool do_softcap, + const float threshold, + const float min_val, const bool has_mask, const bool is_causal +){ + if (has_mask) { + switch(att_mask.depth()) + { + case CV_32F: + run_fused_softmax>(att_weights, att_mask, softcap, do_softcap, threshold, min_val, is_causal); + break; + case CV_Bool: + case CV_8U: + run_fused_softmax>(att_weights, att_mask, softcap, do_softcap, threshold, min_val, is_causal); + break; + case CV_8S: + run_fused_softmax>(att_weights, att_mask, softcap, do_softcap, threshold, min_val, is_causal); + break; + case CV_16U: + run_fused_softmax>(att_weights, att_mask, softcap, do_softcap, threshold, min_val, is_causal); + break; + case CV_16S: + run_fused_softmax>(att_weights, att_mask, softcap, do_softcap, threshold, min_val, is_causal); + break; + case CV_32U: + run_fused_softmax>(att_weights, att_mask, softcap, do_softcap, threshold, min_val, is_causal); + break; + case CV_32S: + run_fused_softmax>(att_weights, att_mask, softcap, do_softcap, threshold, min_val, is_causal); + break; + case CV_64U: + run_fused_softmax>(att_weights, att_mask, softcap, do_softcap, threshold, min_val, is_causal); + break; + case CV_64S: + run_fused_softmax>(att_weights, att_mask, softcap, do_softcap, threshold, min_val, is_causal); + break; + default: + CV_Error(Error::StsUnsupportedFormat, "Unsupported mask data type in fused_softmax_softcap_mask"); + } + } else { + run_fused_softmax(att_weights, att_mask, softcap, do_softcap, threshold, min_val, is_causal); + } +} +}} diff --git a/modules/dnn/src/layers/cpu_kernels/fast_attn.hpp b/modules/dnn/src/layers/cpu_kernels/fast_attn.hpp new file mode 100644 index 0000000000..d533a91aec --- /dev/null +++ b/modules/dnn/src/layers/cpu_kernels/fast_attn.hpp @@ -0,0 +1,23 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. + + +#ifndef OPENCV_DNN_FAST_ATTN +#define OPENCV_DNN_FAST_ATTN + +#include "opencv2/core/hal/intrin.hpp" +#include + +namespace cv { namespace dnn { + +void fused_softmax_softcap_mask( + Mat &att_weights,const Mat &att_mask, + const float softcap, const bool do_softcap, + const float threshold, + const float min_val, const bool has_mask, const bool is_causal +); + +}} + +#endif diff --git a/modules/dnn/src/net_impl.hpp b/modules/dnn/src/net_impl.hpp index 8eec5dbf3f..e6573604a0 100644 --- a/modules/dnn/src/net_impl.hpp +++ b/modules/dnn/src/net_impl.hpp @@ -107,7 +107,6 @@ struct Net::Impl : public detail::NetImplBase // FIXIT use inheritance virtual Ptr wrap(Mat& host); - virtual void clear(); diff --git a/modules/dnn/src/onnx/onnx_graph_simplifier.cpp b/modules/dnn/src/onnx/onnx_graph_simplifier.cpp index 61c789a2ba..65e05cf45d 100644 --- a/modules/dnn/src/onnx/onnx_graph_simplifier.cpp +++ b/modules/dnn/src/onnx/onnx_graph_simplifier.cpp @@ -501,6 +501,7 @@ class AttentionSubGraph : public Subgraph { std::vector >&) CV_OVERRIDE { // add attrs opencv_onnx::NodeProto* node = fusedNode.dynamicCast()->node; + node->set_domain("com.microsoft"); opencv_onnx::AttributeProto* attr_num_heads = node->add_attribute(); attr_num_heads->set_name("num_heads"); attr_num_heads->set_i(num_heads); @@ -611,6 +612,7 @@ class AttentionSingleHeadSubGraph : public Subgraph { std::vector >&) CV_OVERRIDE { // add attrs opencv_onnx::NodeProto* node = fusedNode.dynamicCast()->node; + node->set_domain("com.microsoft"); opencv_onnx::AttributeProto* attr_num_heads = node->add_attribute(); attr_num_heads->set_name("num_heads"); attr_num_heads->set_i(num_heads); diff --git a/modules/dnn/src/onnx/onnx_importer2.cpp b/modules/dnn/src/onnx/onnx_importer2.cpp index e3b8c09dd0..89a8491a6c 100644 --- a/modules/dnn/src/onnx/onnx_importer2.cpp +++ b/modules/dnn/src/onnx/onnx_importer2.cpp @@ -257,6 +257,7 @@ protected: // Domain: com.microsoft // URL: https://github.com/microsoft/onnxruntime/blob/master/docs/ContribOperators.md void parseAttention (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseAttentionOnnxAi (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); void parseDequantizeLinear (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); void parseQuantizeLinear (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); void parseCustomLayer (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); @@ -2660,6 +2661,26 @@ void ONNXImporter2::parseAttention(LayerParams& params, const opencv_onnx::NodeP addLayer(params, node_proto, n_inputs); } +void ONNXImporter2::parseAttentionOnnxAi(LayerParams& params, const opencv_onnx::NodeProto& node_proto) { + int i, n_inputs = node_proto.input_size(); + + for (i = 1; i < n_inputs; i++) { + if (!net.isConstArg(node_inputs[i])) + break; + } + + if (i == n_inputs) { + for (i = 1; i < n_inputs; i++) { + Mat blob = net.argTensor(node_inputs[i]); + params.blobs.push_back(blob); + } + n_inputs = 1; + } + params.type = "AttentionOnnxAi"; + + addLayer(params, node_proto, n_inputs); +} + // Domain: ai.onnx (default) // URL: https://github.com/onnx/onnx/blob/master/docs/Operators.md void ONNXImporter2::buildDispatchMap_ONNX_AI() @@ -2743,6 +2764,7 @@ void ONNXImporter2::buildDispatchMap_ONNX_AI() dispatch["GroupNormalization"] = &ONNXImporter2::parseInstanceNormalization; dispatch["NegativeLogLikelihoodLoss"] = &ONNXImporter2::parseNegativeLogLikelihoodLoss; dispatch["SoftmaxCrossEntropyLoss"] = &ONNXImporter2::parseSoftmaxCrossEntropyLoss; + // @TODO@ONNX: Add support for SDPA dispatch["Equal"] = dispatch["Greater"] = dispatch["Less"] = dispatch["Pow"] = dispatch["Add"] = dispatch["Sub"] = dispatch["Mul"] = dispatch["Div"] = dispatch["GreaterOrEqual"] = @@ -2777,7 +2799,7 @@ void ONNXImporter2::buildDispatchMap_ONNX_AI() // com.microsft: This operator is added for compatibility via onnx graph simplifier. // Opset domain cannot be modified from onnx_graph_simplifier.cpp so this // operator cannot be parsed if only added in buildDispatchMap_COM_MICROSOFT - dispatch["Attention"] = &ONNXImporter2::parseAttention; + dispatch["Attention"] = &ONNXImporter2::parseAttentionOnnxAi; domain_dispatch_map[str_domain_ai_onnx] = dispatch; } diff --git a/modules/dnn/test/test_onnx_conformance_layer_filter__cuda_denylist.inl.hpp b/modules/dnn/test/test_onnx_conformance_layer_filter__cuda_denylist.inl.hpp index 02636f2ff5..e7a1de0cd3 100644 --- a/modules/dnn/test/test_onnx_conformance_layer_filter__cuda_denylist.inl.hpp +++ b/modules/dnn/test/test_onnx_conformance_layer_filter__cuda_denylist.inl.hpp @@ -102,3 +102,41 @@ "test_quantizelinear", // Issue https://github.com/opencv/opencv/issues/25999 "test_quantizelinear_axis", // Issue https://github.com/opencv/opencv/issues/25999 "test_quantizelinear_blocked", // Issue https://github.com/opencv/opencv/issues/25999 +"test_attention_3d_attn_mask", +"test_attention_3d_causal", +"test_attention_3d_diff_heads_sizes", +"test_attention_3d_diff_heads_sizes_attn_mask", +"test_attention_3d_diff_heads_sizes_causal", +"test_attention_3d_diff_heads_sizes_softcap", +"test_attention_3d_diff_heads_sizes_scaled", +"test_attention_3d_gqa", +"test_attention_3d_gqa_attn_mask", +"test_attention_3d_gqa_causal", +"test_attention_3d_gqa_scaled", +"test_attention_3d_gqa_softcap", +"test_attention_3d_scaled", +"test_attention_3d_softcap", +"test_attention_3d_transpose_verification", +"test_attention_4d", +"test_attention_4d_attn_mask", +"test_attention_4d_attn_mask_3d", +"test_attention_4d_attn_mask_3d_causal", +"test_attention_4d_attn_mask_4d", +"test_attention_4d_attn_mask_4d_causal", +"test_attention_4d_attn_mask_bool", +"test_attention_4d_attn_mask_bool_4d", +"test_attention_4d_causal", +"test_attention_4d_diff_heads_sizes", +"test_attention_4d_diff_heads_sizes_attn_mask", +"test_attention_4d_diff_heads_sizes_causal", +"test_attention_4d_diff_heads_sizes_scaled", +"test_attention_4d_diff_heads_sizes_softcap", +"test_attention_4d_gqa", +"test_attention_4d_gqa_attn_mask", +"test_attention_4d_gqa_causal", +"test_attention_4d_gqa_scaled", +"test_attention_4d_gqa_softcap", +"test_attention_4d_scaled", +"test_attention_4d_softcap", +"test_attention_4d_attn_mask_bool", +"test_attention_4d_attn_mask_bool_4d", \ No newline at end of file diff --git a/modules/dnn/test/test_onnx_conformance_layer_filter__cuda_fp16_denylist.inl.hpp b/modules/dnn/test/test_onnx_conformance_layer_filter__cuda_fp16_denylist.inl.hpp index 4526b51431..cf2ce74922 100644 --- a/modules/dnn/test/test_onnx_conformance_layer_filter__cuda_fp16_denylist.inl.hpp +++ b/modules/dnn/test/test_onnx_conformance_layer_filter__cuda_fp16_denylist.inl.hpp @@ -38,3 +38,41 @@ "test_max_float64", "test_min_float64", "test_mod_mixed_sign_float64", +"test_attention_3d_attn_mask", +"test_attention_3d_causal", +"test_attention_3d_diff_heads_sizes", +"test_attention_3d_diff_heads_sizes_attn_mask", +"test_attention_3d_diff_heads_sizes_causal", +"test_attention_3d_diff_heads_sizes_softcap", +"test_attention_3d_diff_heads_sizes_scaled", +"test_attention_3d_gqa", +"test_attention_3d_gqa_attn_mask", +"test_attention_3d_gqa_causal", +"test_attention_3d_gqa_scaled", +"test_attention_3d_gqa_softcap", +"test_attention_3d_scaled", +"test_attention_3d_softcap", +"test_attention_3d_transpose_verification", +"test_attention_4d", +"test_attention_4d_attn_mask", +"test_attention_4d_attn_mask_3d", +"test_attention_4d_attn_mask_3d_causal", +"test_attention_4d_attn_mask_4d", +"test_attention_4d_attn_mask_4d_causal", +"test_attention_4d_attn_mask_bool", +"test_attention_4d_attn_mask_bool_4d", +"test_attention_4d_causal", +"test_attention_4d_diff_heads_sizes", +"test_attention_4d_diff_heads_sizes_attn_mask", +"test_attention_4d_diff_heads_sizes_causal", +"test_attention_4d_diff_heads_sizes_scaled", +"test_attention_4d_diff_heads_sizes_softcap", +"test_attention_4d_gqa", +"test_attention_4d_gqa_attn_mask", +"test_attention_4d_gqa_causal", +"test_attention_4d_gqa_scaled", +"test_attention_4d_gqa_softcap", +"test_attention_4d_scaled", +"test_attention_4d_softcap", +"test_attention_4d_attn_mask_bool", +"test_attention_4d_attn_mask_bool_4d", \ No newline at end of file diff --git a/modules/dnn/test/test_onnx_conformance_layer_filter__openvino.inl.hpp b/modules/dnn/test/test_onnx_conformance_layer_filter__openvino.inl.hpp index 4ea7c6f1ef..5bd3287845 100644 --- a/modules/dnn/test/test_onnx_conformance_layer_filter__openvino.inl.hpp +++ b/modules/dnn/test/test_onnx_conformance_layer_filter__openvino.inl.hpp @@ -2837,6 +2837,85 @@ CASE(test_xor_bcast4v3d) // no filter CASE(test_xor_bcast4v4d) // no filter +CASE() +CASE(test_attention_3d) + SKIP; +CASE(test_attention_3d_attn_mask) + SKIP; +CASE(test_attention_3d_causal) + SKIP; +CASE(test_attention_3d_diff_heads_sizes) + SKIP; +CASE(test_attention_3d_diff_heads_sizes_attn_mask) + SKIP; +CASE(test_attention_3d_diff_heads_sizes_causal) + SKIP; +CASE(test_attention_3d_diff_heads_sizes_softcap) + SKIP; +CASE(test_attention_3d_diff_heads_sizes_scaled) + SKIP; +CASE(test_attention_3d_gqa) + SKIP; +CASE(test_attention_3d_gqa_attn_mask) + SKIP; +CASE(test_attention_3d_gqa_causal) + SKIP; +CASE(test_attention_3d_gqa_scaled) + SKIP; +CASE(test_attention_3d_gqa_softcap) + SKIP; +CASE(test_attention_3d_scaled) + SKIP; +CASE(test_attention_3d_softcap) + SKIP; +CASE(test_attention_3d_transpose_verification) + SKIP; +CASE(test_attention_4d) + SKIP; +CASE(test_attention_4d_attn_mask) + SKIP; +CASE(test_attention_4d_attn_mask_3d) + SKIP; +CASE(test_attention_4d_attn_mask_3d_causal) + SKIP; +CASE(test_attention_4d_attn_mask_4d) + SKIP; +CASE(test_attention_4d_attn_mask_4d_causal) + SKIP; +CASE(test_attention_4d_attn_mask_bool) + SKIP; +CASE(test_attention_4d_attn_mask_bool_4d) + SKIP; +CASE(test_attention_4d_causal) + SKIP; +CASE(test_attention_4d_diff_heads_sizes) + SKIP; +CASE(test_attention_4d_diff_heads_sizes_attn_mask) + SKIP; +CASE(test_attention_4d_diff_heads_sizes_causal) + SKIP; +CASE(test_attention_4d_diff_heads_sizes_scaled) + SKIP; +CASE(test_attention_4d_diff_heads_sizes_softcap) + SKIP; +CASE(test_attention_4d_gqa) + SKIP; +CASE(test_attention_4d_gqa_attn_mask) + SKIP; +CASE(test_attention_4d_gqa_causal) + SKIP; +CASE(test_attention_4d_gqa_scaled) + SKIP; +CASE(test_attention_4d_gqa_softcap) + SKIP; +CASE(test_attention_4d_scaled) + SKIP; +CASE(test_attention_4d_softcap) + SKIP; +CASE(test_attention_4d_attn_mask_bool) + SKIP; +CASE(test_attention_4d_attn_mask_bool_4d) + SKIP; END_SWITCH() #undef EOF_LABEL #undef BEGIN_SWITCH diff --git a/modules/dnn/test/test_onnx_conformance_layer_filter__vulkan_denylist.inl.hpp b/modules/dnn/test/test_onnx_conformance_layer_filter__vulkan_denylist.inl.hpp index f6aee0dd36..1b82375957 100644 --- a/modules/dnn/test/test_onnx_conformance_layer_filter__vulkan_denylist.inl.hpp +++ b/modules/dnn/test/test_onnx_conformance_layer_filter__vulkan_denylist.inl.hpp @@ -95,3 +95,41 @@ "test_scatternd_max", "test_scatternd_min", "test_scatternd_multiply", +"test_attention_3d_attn_mask", +"test_attention_3d_causal", +"test_attention_3d_diff_heads_sizes", +"test_attention_3d_diff_heads_sizes_attn_mask", +"test_attention_3d_diff_heads_sizes_causal", +"test_attention_3d_diff_heads_sizes_softcap", +"test_attention_3d_diff_heads_sizes_scaled", +"test_attention_3d_gqa", +"test_attention_3d_gqa_attn_mask", +"test_attention_3d_gqa_causal", +"test_attention_3d_gqa_scaled", +"test_attention_3d_gqa_softcap", +"test_attention_3d_scaled", +"test_attention_3d_softcap", +"test_attention_3d_transpose_verification", +"test_attention_4d", +"test_attention_4d_attn_mask", +"test_attention_4d_attn_mask_3d", +"test_attention_4d_attn_mask_3d_causal", +"test_attention_4d_attn_mask_4d", +"test_attention_4d_attn_mask_4d_causal", +"test_attention_4d_attn_mask_bool", +"test_attention_4d_attn_mask_bool_4d", +"test_attention_4d_causal", +"test_attention_4d_diff_heads_sizes", +"test_attention_4d_diff_heads_sizes_attn_mask", +"test_attention_4d_diff_heads_sizes_causal", +"test_attention_4d_diff_heads_sizes_scaled", +"test_attention_4d_diff_heads_sizes_softcap", +"test_attention_4d_gqa", +"test_attention_4d_gqa_attn_mask", +"test_attention_4d_gqa_causal", +"test_attention_4d_gqa_scaled", +"test_attention_4d_gqa_softcap", +"test_attention_4d_scaled", +"test_attention_4d_softcap" +"test_attention_4d_attn_mask_bool", +"test_attention_4d_attn_mask_bool_4d", \ No newline at end of file diff --git a/modules/dnn/test/test_onnx_conformance_layer_filter_opencv_classic_denylist.inl.hpp b/modules/dnn/test/test_onnx_conformance_layer_filter_opencv_classic_denylist.inl.hpp index d149c67ce6..048a8ea8d0 100644 --- a/modules/dnn/test/test_onnx_conformance_layer_filter_opencv_classic_denylist.inl.hpp +++ b/modules/dnn/test/test_onnx_conformance_layer_filter_opencv_classic_denylist.inl.hpp @@ -600,6 +600,45 @@ "test_rotary_embedding_with_interleaved_rotary_dim", "test_rotary_embedding_with_interleaved_rotary_dim_expanded", "test_rotary_embedding_with_rotary_dim", +"test_rotary_embedding_with_rotary_dim_expanded","test_attention_3d", +"test_attention_3d_attn_mask", +"test_attention_3d_causal", +"test_attention_3d_diff_heads_sizes", +"test_attention_3d_diff_heads_sizes_attn_mask", +"test_attention_3d_diff_heads_sizes_causal", +"test_attention_3d_diff_heads_sizes_softcap", +"test_attention_3d_diff_heads_sizes_scaled", +"test_attention_3d_gqa", +"test_attention_3d_gqa_attn_mask", +"test_attention_3d_gqa_causal", +"test_attention_3d_gqa_scaled", +"test_attention_3d_gqa_softcap", +"test_attention_3d_scaled", +"test_attention_3d_softcap", +"test_attention_3d_transpose_verification", +"test_attention_4d", +"test_attention_4d_attn_mask", +"test_attention_4d_attn_mask_3d", +"test_attention_4d_attn_mask_3d_causal", +"test_attention_4d_attn_mask_4d", +"test_attention_4d_attn_mask_4d_causal", +"test_attention_4d_attn_mask_bool", +"test_attention_4d_attn_mask_bool_4d", +"test_attention_4d_causal", +"test_attention_4d_diff_heads_sizes", +"test_attention_4d_diff_heads_sizes_attn_mask", +"test_attention_4d_diff_heads_sizes_causal", +"test_attention_4d_diff_heads_sizes_scaled", +"test_attention_4d_diff_heads_sizes_softcap", +"test_attention_4d_gqa", +"test_attention_4d_gqa_attn_mask", +"test_attention_4d_gqa_causal", +"test_attention_4d_gqa_scaled", +"test_attention_4d_gqa_softcap", +"test_attention_4d_scaled", +"test_attention_4d_softcap", +"test_attention_4d_attn_mask_bool", +"test_attention_4d_attn_mask_bool_4d", "test_rotary_embedding_with_rotary_dim_expanded", "test_hammingwindow", "test_hammingwindow_expanded", diff --git a/modules/dnn/test/test_onnx_conformance_layer_parser_denylist.inl.hpp b/modules/dnn/test/test_onnx_conformance_layer_parser_denylist.inl.hpp index 1f1c055773..3c50616c74 100644 --- a/modules/dnn/test/test_onnx_conformance_layer_parser_denylist.inl.hpp +++ b/modules/dnn/test/test_onnx_conformance_layer_parser_denylist.inl.hpp @@ -16,107 +16,81 @@ "test_ai_onnx_ml_label_encoder_tensor_value_only_mapping", "test_ai_onnx_ml_tree_ensemble_set_membership", "test_ai_onnx_ml_tree_ensemble_single_tree", -"test_attention_3d", -"test_attention_3d_attn_mask", + +// autoSize <= INT_MAX && autoSize*outTotal == inpTotal in function 'getOutShape' @ C++ exception with description +//"OpenCV(5.0.0-pre) opencv/modules/dnn/src/layers/reshape2_layer.cpp:110 +// expanded graphs are not imported correctly +"test_attention_3d_expanded", "test_attention_3d_attn_mask_expanded", -"test_attention_3d_causal", -"test_attention_3d_causal_expanded", -"test_attention_3d_diff_heads_sizes", -"test_attention_3d_diff_heads_sizes_attn_mask", "test_attention_3d_diff_heads_sizes_attn_mask_expanded", -"test_attention_3d_diff_heads_sizes_causal", -"test_attention_3d_diff_heads_sizes_causal_expanded", +"test_attention_3d_scaled_expanded", +"test_attention_3d_gqa_attn_mask_expanded", "test_attention_3d_diff_heads_sizes_expanded", -"test_attention_3d_diff_heads_sizes_scaled", +"test_attention_3d_transpose_verification_expanded", +"test_attention_3d_softcap_expanded", +"test_attention_3d_gqa_scaled_expanded", +"test_attention_3d_with_past_and_present_qk_matmul_softmax_expanded", +"test_attention_3d_with_past_and_present_qk_matmul_softcap_expanded", +"test_attention_3d_with_past_and_present_qk_matmul_bias_expanded", +"test_attention_3d_with_past_and_present_qk_matmul_expanded", +"test_attention_3d_with_past_and_present_expanded", +"test_attention_3d_gqa_with_past_and_present_expanded", +"test_attention_3d_causal_expanded", +"test_attention_3d_diff_heads_sizes_causal_expanded", "test_attention_3d_diff_heads_sizes_scaled_expanded", "test_attention_3d_diff_heads_sizes_softcap", "test_attention_3d_diff_heads_sizes_softcap_expanded", "test_attention_3d_diff_heads_with_past_and_present", "test_attention_3d_diff_heads_with_past_and_present_expanded", -"test_attention_3d_expanded", -"test_attention_3d_gqa", -"test_attention_3d_gqa_attn_mask", -"test_attention_3d_gqa_attn_mask_expanded", -"test_attention_3d_gqa_causal", "test_attention_3d_gqa_causal_expanded", "test_attention_3d_gqa_expanded", -"test_attention_3d_gqa_scaled", -"test_attention_3d_gqa_scaled_expanded", -"test_attention_3d_gqa_softcap", "test_attention_3d_gqa_softcap_expanded", -"test_attention_3d_gqa_with_past_and_present", -"test_attention_3d_gqa_with_past_and_present_expanded", -"test_attention_3d_scaled", -"test_attention_3d_scaled_expanded", -"test_attention_3d_softcap", -"test_attention_3d_softcap_expanded", -"test_attention_3d_transpose_verification", -"test_attention_3d_transpose_verification_expanded", -"test_attention_3d_with_past_and_present", -"test_attention_3d_with_past_and_present_expanded", -"test_attention_3d_with_past_and_present_qk_matmul", -"test_attention_3d_with_past_and_present_qk_matmul_bias", -"test_attention_3d_with_past_and_present_qk_matmul_bias_expanded", -"test_attention_3d_with_past_and_present_qk_matmul_expanded", -"test_attention_3d_with_past_and_present_qk_matmul_softcap", -"test_attention_3d_with_past_and_present_qk_matmul_softcap_expanded", -"test_attention_3d_with_past_and_present_qk_matmul_softmax", -"test_attention_3d_with_past_and_present_qk_matmul_softmax_expanded", -"test_attention_4d", -"test_attention_4d_attn_mask", -"test_attention_4d_attn_mask_3d", -"test_attention_4d_attn_mask_3d_causal", "test_attention_4d_attn_mask_3d_causal_expanded", "test_attention_4d_attn_mask_3d_expanded", -"test_attention_4d_attn_mask_4d", -"test_attention_4d_attn_mask_4d_causal", +"test_attention_4d_causal_expanded", +"test_attention_4d_diff_heads_sizes_attn_mask_expanded", +"test_attention_4d_diff_heads_mask4d_padded_kv_expanded", "test_attention_4d_attn_mask_4d_causal_expanded", "test_attention_4d_attn_mask_4d_expanded", -"test_attention_4d_attn_mask_bool", -"test_attention_4d_attn_mask_bool_4d", + "test_attention_4d_attn_mask_bool_4d_expanded", "test_attention_4d_attn_mask_bool_expanded", "test_attention_4d_attn_mask_expanded", -"test_attention_4d_causal", -"test_attention_4d_causal_expanded", -"test_attention_4d_diff_heads_mask4d_padded_kv", -"test_attention_4d_diff_heads_mask4d_padded_kv_expanded", -"test_attention_4d_diff_heads_sizes", -"test_attention_4d_diff_heads_sizes_attn_mask", -"test_attention_4d_diff_heads_sizes_attn_mask_expanded", -"test_attention_4d_diff_heads_sizes_causal", "test_attention_4d_diff_heads_sizes_causal_expanded", "test_attention_4d_diff_heads_sizes_expanded", -"test_attention_4d_diff_heads_sizes_scaled", "test_attention_4d_diff_heads_sizes_scaled_expanded", -"test_attention_4d_diff_heads_sizes_softcap", "test_attention_4d_diff_heads_sizes_softcap_expanded", +"test_attention_4d_expanded", + +"test_attention_3d_gqa_with_past_and_present", +"test_attention_3d_with_past_and_present", +"test_attention_3d_with_past_and_present_qk_matmul", +"test_attention_3d_with_past_and_present_qk_matmul_bias", +"test_attention_3d_with_past_and_present_qk_matmul_softcap", +"test_attention_3d_with_past_and_present_qk_matmul_softmax", +"test_attention_4d_diff_heads_mask4d_padded_kv", "test_attention_4d_diff_heads_with_past_and_present", "test_attention_4d_diff_heads_with_past_and_present_expanded", "test_attention_4d_diff_heads_with_past_and_present_mask3d", "test_attention_4d_diff_heads_with_past_and_present_mask3d_expanded", "test_attention_4d_diff_heads_with_past_and_present_mask4d", "test_attention_4d_diff_heads_with_past_and_present_mask4d_expanded", -"test_attention_4d_expanded", -"test_attention_4d_fp16", "test_attention_4d_fp16_expanded", -"test_attention_4d_gqa", -"test_attention_4d_gqa_attn_mask", "test_attention_4d_gqa_attn_mask_expanded", -"test_attention_4d_gqa_causal", "test_attention_4d_gqa_causal_expanded", "test_attention_4d_gqa_expanded", -"test_attention_4d_gqa_scaled", "test_attention_4d_gqa_scaled_expanded", -"test_attention_4d_gqa_softcap", "test_attention_4d_gqa_softcap_expanded", + +// fixme +"test_attention_4d_fp16", + "test_attention_4d_gqa_with_past_and_present", "test_attention_4d_gqa_with_past_and_present_expanded", "test_attention_4d_gqa_with_past_and_present_fp16", "test_attention_4d_gqa_with_past_and_present_fp16_expanded", -"test_attention_4d_scaled", + "test_attention_4d_scaled_expanded", -"test_attention_4d_softcap", "test_attention_4d_softcap_expanded", "test_attention_4d_with_past_and_present", "test_attention_4d_with_past_and_present_expanded",