diff --git a/modules/dnn/perf/perf_net.cpp b/modules/dnn/perf/perf_net.cpp index cb9138ce24..7a38ed4f49 100644 --- a/modules/dnn/perf/perf_net.cpp +++ b/modules/dnn/perf/perf_net.cpp @@ -406,6 +406,22 @@ PERF_TEST_P_(DNNTestNetwork, VIT_B_32) processNet("dnn/onnx/models/vit_b_32.onnx", "", cv::Size(224, 224)); } +PERF_TEST_P_(DNNTestNetwork, BERT) +{ + const int seq_len = 9; + int64_t input_ids_data[seq_len] = {101, 1996, 103, 2938, 2006, 1996, 13523, 1012, 102}; + int64_t attention_mask_data[seq_len] = {1, 1, 1, 1, 1, 1, 1, 1, 1}; + int64_t token_type_ids_data[seq_len] = {0, 0, 0, 0, 0, 0, 0, 0, 0}; + int shp[2] = {1, seq_len}; + Mat input_ids(2, shp, CV_64S, input_ids_data); + Mat attention_mask(2, shp, CV_64S, attention_mask_data); + Mat token_type_ids(2, shp, CV_64S, token_type_ids_data); + processNet("dnn/onnx/models/bert.onnx", "", + {std::make_tuple(input_ids, "input_ids"), + std::make_tuple(attention_mask, "attention_mask"), + std::make_tuple(token_type_ids, "token_type_ids")}); +} + PERF_TEST_P_(DNNTestNetwork, VIT_Base_Patch16_224) { applyTestTag(CV_TEST_TAG_MEMORY_512MB); diff --git a/modules/dnn/src/graph_fusion_attention.cpp b/modules/dnn/src/graph_fusion_attention.cpp new file mode 100644 index 0000000000..74cf1e7861 --- /dev/null +++ b/modules/dnn/src/graph_fusion_attention.cpp @@ -0,0 +1,530 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// Copyright (C) 2026, BigVision LLC, all rights reserved. +// Third party copyrights are property of their respective owners. + +// Fuses multi-head attention subgraphs into a single Attention layer. + +#include "precomp.hpp" +#include "net_impl.hpp" + +namespace cv { namespace dnn { +CV__DNN_INLINE_NS_BEGIN + +using std::vector; +using std::string; + +struct ModelFusionAttention +{ + ModelFusionAttention(Net::Impl* netimpl_) : netimpl(netimpl_) {} + + void fuse() + { + fuseGraph(netimpl->mainGraph); + } + + int singleConsumer(Arg a) const + { + auto it = consumers_.find(a.idx); + if (it == consumers_.end() || it->second.size() != 1) + return -1; + return it->second[0]; + } + + bool isReshape(const vector>& prog, int idx) const + { + if (idx < 0 || idx >= (int)prog.size() || !prog[idx]) + return false; + return dynamic_cast(prog[idx].get()) != nullptr; + } + + bool isTranspose(const vector>& prog, int idx) const + { + if (idx < 0 || idx >= (int)prog.size() || !prog[idx]) + return false; + return dynamic_cast(prog[idx].get()) != nullptr; + } + + bool isSoftmax(const vector>& prog, int idx) const + { + if (idx < 0 || idx >= (int)prog.size() || !prog[idx]) + return false; + return prog[idx]->type == "Softmax"; + } + + bool isMatMul(const vector>& prog, int idx) const + { + if (idx < 0 || idx >= (int)prog.size() || !prog[idx]) + return false; + return dynamic_cast(prog[idx].get()) != nullptr; + } + + bool isScalarBinOp(const vector>& prog, int idx, + NaryEltwiseLayer::OPERATION op, float* val) const + { + if (idx < 0 || idx >= (int)prog.size() || !prog[idx]) + return false; + NaryEltwiseLayer* elt = dynamic_cast(prog[idx].get()); + if (!elt || elt->op != op) + return false; + const auto& inputs = prog[idx]->inputs; + if (inputs.size() != 2) return false; + for (int k = 0; k < 2; k++) { + Arg inp = inputs[k]; + if (netimpl->isConstArg(inp)) { + Mat t = netimpl->argTensor(inp); + if (t.total() == 1 && t.type() == CV_32F) { + *val = t.at(0); + return true; + } + } + } + return false; + } + + bool isScalarMul(const vector>& prog, int idx, float* val) const + { + return isScalarBinOp(prog, idx, NaryEltwiseLayer::OPERATION::PROD, val); + } + + bool isScalarDiv(const vector>& prog, int idx, float* val) const + { + return isScalarBinOp(prog, idx, NaryEltwiseLayer::OPERATION::DIV, val); + } + + // Accept Add op with exactly two inputs; identify the non-constant runtime + // input (the mask tensor). Returns false if the Add doesn't match. + bool isMaskAdd(const vector>& prog, int idx, Arg* out_mask) const + { + if (idx < 0 || idx >= (int)prog.size() || !prog[idx]) + return false; + NaryEltwiseLayer* elt = dynamic_cast(prog[idx].get()); + if (!elt || elt->op != NaryEltwiseLayer::OPERATION::ADD) + return false; + const auto& inputs = prog[idx]->inputs; + if (inputs.size() != 2) return false; + *out_mask = inputs[1]; + return true; + } + + // Extract a scalar integer from a const-valued arg, possibly wrapped in an + // Unsqueeze of a scalar const. Returns -1 if extraction fails. + int extractConstInt(const vector>& prog, Arg a) const + { + auto readScalar = [&](Arg x) -> int { + if (!netimpl->isConstArg(x)) return -1; + Mat t = netimpl->argTensor(x); + if (t.total() != 1) return -1; + if (t.type() == CV_64S) return (int)t.at(0); + if (t.type() == CV_32S) return (int)t.at(0); + return -1; + }; + int v = readScalar(a); + if (v > 0) return v; + auto it = producer_.find(a.idx); + if (it == producer_.end()) return -1; + int pidx = it->second; + if (pidx < 0 || pidx >= (int)prog.size() || !prog[pidx]) return -1; + if (prog[pidx]->type == "Unsqueeze" && !prog[pidx]->inputs.empty()) + return readScalar(prog[pidx]->inputs[0]); + return -1; + } + + void collectShapeChain(const vector>& prog, int concat_idx, + std::set& chain) const + { + if (concat_idx < 0 || concat_idx >= (int)prog.size() || !prog[concat_idx]) + return; + chain.insert(concat_idx); + for (Arg ci : prog[concat_idx]->inputs) { + if (netimpl->isConstArg(ci)) continue; + int cur = -1; + auto it = producer_.find(ci.idx); + if (it != producer_.end()) cur = it->second; + while (cur >= 0 && (int)prog.size() > cur && prog[cur]) { + const std::string& t = prog[cur]->type; + if (t != "Unsqueeze" && t != "Gather2" && t != "Shape") break; + Arg cur_out = prog[cur]->outputs[0]; + auto cit = consumers_.find(cur_out.idx); + if (cit == consumers_.end()) break; + bool all_in_chain = true; + for (int c : cit->second) if (!chain.count(c)) { all_in_chain = false; break; } + if (!all_in_chain) break; + chain.insert(cur); + if (prog[cur]->inputs.empty()) break; + Arg prev = prog[cur]->inputs[0]; + auto pit = producer_.find(prev.idx); + if (pit == producer_.end()) break; + cur = pit->second; + } + } + } + + template + int findMatchingConsumer(const vector>& prog, Arg out, + Pred pred, std::set* extra_shape_ops) const + { + auto it = consumers_.find(out.idx); + if (it == consumers_.end()) return -1; + int matched = -1; + for (int c : it->second) { + if (c < 0 || c >= (int)prog.size() || !prog[c]) return -1; + if (pred(prog[c].get())) { + if (matched >= 0) return -1; + matched = c; + } else if (prog[c]->type == "Shape") { + if (extra_shape_ops) extra_shape_ops->insert(c); + } else { + return -1; + } + } + return matched; + } + + int followProjChain(const vector>& prog, + int proj_matmul_idx, + int* out_reshape_idx, + int* out_num_heads, + vector* out_perm, + std::set* extra_ops_to_remove) const + { + if (proj_matmul_idx < 0) return -1; + Arg proj_out = prog[proj_matmul_idx]->outputs[0]; + int reshape_idx = findMatchingConsumer(prog, proj_out, + [](Layer* L){ return dynamic_cast(L) != nullptr; }, + extra_ops_to_remove); + if (!isReshape(prog, reshape_idx)) return -1; + + const auto& reshape_inputs = prog[reshape_idx]->inputs; + if (reshape_inputs.size() < 2) return -1; + Arg shape_arg = reshape_inputs[1]; + + int num_heads = -1; + if (netimpl->isConstArg(shape_arg)) { + Mat shape_mat = netimpl->argTensor(shape_arg); + if (shape_mat.total() != 4) return -1; + const int64_t* shape_data = shape_mat.ptr(); + num_heads = static_cast(shape_data[2]); + } else { + auto it = producer_.find(shape_arg.idx); + if (it == producer_.end()) return -1; + int concat_idx = it->second; + if (concat_idx < 0 || concat_idx >= (int)prog.size() || !prog[concat_idx]) + return -1; + if (!dynamic_cast(prog[concat_idx].get())) + return -1; + const auto& cinputs = prog[concat_idx]->inputs; + if (cinputs.size() != 4) return -1; + num_heads = extractConstInt(prog, cinputs[2]); + if (num_heads <= 0) return -1; + if (extra_ops_to_remove) + collectShapeChain(prog, concat_idx, *extra_ops_to_remove); + } + *out_num_heads = num_heads; + *out_reshape_idx = reshape_idx; + + Arg reshape_out = prog[reshape_idx]->outputs[0]; + int transpose_idx = singleConsumer(reshape_out); + if (!isTranspose(prog, transpose_idx)) return -1; + TransposeLayer* tr = dynamic_cast(prog[transpose_idx].get()); + if (!tr) return -1; + *out_perm = tr->perm; + return transpose_idx; + } + + bool fuseGraph(Ptr& graph) + { + const vector>& prog = graph->prog(); + size_t nops = prog.size(); + + producer_.clear(); + consumers_.clear(); + for (size_t i = 0; i < nops; i++) { + if (!prog[i]) continue; + for (Arg out : prog[i]->outputs) + producer_[out.idx] = (int)i; + for (Arg inp : prog[i]->inputs) + consumers_[inp.idx].push_back((int)i); + } + + std::map> qkv_candidates; + for (size_t i = 0; i < nops; i++) { + if (!prog[i]) continue; + MatMulLayer* mm = dynamic_cast(prog[i].get()); + if (!mm) continue; + if (prog[i]->blobs.empty() || prog[i]->inputs.size() != 1) + continue; + Arg inp = prog[i]->inputs[0]; + qkv_candidates[inp.idx].push_back((int)i); + } + + bool modified = false; + std::set removed_ops; + + for (auto& [inp_idx, matmul_indices] : qkv_candidates) { + if (matmul_indices.size() < 3) continue; + + for (size_t qi = 0; qi + 2 < matmul_indices.size(); qi++) + for (size_t ki = qi + 1; ki + 1 < matmul_indices.size(); ki++) + for (size_t vi = ki + 1; vi < matmul_indices.size(); vi++) + { + int indices[3] = { matmul_indices[qi], matmul_indices[ki], matmul_indices[vi] }; + + bool any_removed = false; + for (int idx : indices) + if (removed_ops.count(idx)) any_removed = true; + if (any_removed) continue; + + bool shapes_ok = true; + for (int k = 0; k < 3; k++) { + const Mat& w = prog[indices[k]]->blobs[0]; + if (w.dims != 2) { shapes_ok = false; break; } + } + if (!shapes_ok) continue; + + int reshape_idx[3], num_heads[3], transpose_idx[3]; + vector perms[3]; + std::set extra_ops; + bool pattern_ok = true; + for (int k = 0; k < 3; k++) { + transpose_idx[k] = followProjChain(prog, indices[k], + &reshape_idx[k], + &num_heads[k], &perms[k], + &extra_ops); + if (transpose_idx[k] < 0) { pattern_ok = false; break; } + } + if (!pattern_ok) continue; + if (num_heads[0] != num_heads[1] || num_heads[1] != num_heads[2]) + continue; + + // K is identified by perm [0,2,3,1] (transposes head_dim to second-last); + // Q and V use perm [0,2,1,3]. + int k_slot = -1; + vector perm_k = {0, 2, 3, 1}; + for (int k = 0; k < 3; k++) + if (perms[k] == perm_k) k_slot = k; + if (k_slot < 0) continue; + + int remaining[2]; + { int j = 0; for (int k = 0; k < 3; k++) if (k != k_slot) remaining[j++] = k; } + + Arg k_tr_out = prog[transpose_idx[k_slot]]->outputs[0]; + int k_next = singleConsumer(k_tr_out); + int k_mul_idx = -1; + float k_scale_val = 1.f; + int qk_matmul_idx = -1; + bool vit_style; + if (isScalarMul(prog, k_next, &k_scale_val)) { + vit_style = true; + k_mul_idx = k_next; + qk_matmul_idx = singleConsumer(prog[k_mul_idx]->outputs[0]); + } else if (isMatMul(prog, k_next)) { + vit_style = false; + qk_matmul_idx = k_next; + } else { + continue; + } + if (!isMatMul(prog, qk_matmul_idx)) continue; + + int q_slot = -1, v_slot = -1; + for (int attempt = 0; attempt < 2; attempt++) { + q_slot = remaining[attempt]; + v_slot = remaining[1 - attempt]; + + int q_mul_idx = -1; + float q_scale_val = 1.f; + Arg q_tr_out = prog[transpose_idx[q_slot]]->outputs[0]; + + if (vit_style) { + int q_next = singleConsumer(q_tr_out); + if (!isScalarMul(prog, q_next, &q_scale_val)) continue; + q_mul_idx = q_next; + Arg q_mul_out = prog[q_mul_idx]->outputs[0]; + int q_dest = singleConsumer(q_mul_out); + if (q_dest != qk_matmul_idx) continue; + } else { + const auto& qk_inputs = prog[qk_matmul_idx]->inputs; + bool q_connected = false; + for (auto& ai : qk_inputs) + if (ai.idx == q_tr_out.idx) q_connected = true; + if (!q_connected) continue; + } + + // qk_matmul → [Div(√d_k)] → [Add(mask)] → Softmax (BERT) + // OR → Softmax (ViT) + Arg cur_out = prog[qk_matmul_idx]->outputs[0]; + int qk_div_idx = -1; + int qk_add_idx = -1; + Arg mask_arg; + bool has_mask = false; + float post_scale_val = 1.f; + + int cur_consumer = singleConsumer(cur_out); + if (!vit_style) { + if (!isScalarDiv(prog, cur_consumer, &post_scale_val)) continue; + qk_div_idx = cur_consumer; + cur_out = prog[qk_div_idx]->outputs[0]; + cur_consumer = singleConsumer(cur_out); + Arg maybe_mask; + if (isMaskAdd(prog, cur_consumer, &maybe_mask)) { + qk_add_idx = cur_consumer; + mask_arg = maybe_mask; + has_mask = true; + cur_out = prog[qk_add_idx]->outputs[0]; + cur_consumer = singleConsumer(cur_out); + } + } + int softmax_idx = cur_consumer; + if (!isSoftmax(prog, softmax_idx)) continue; + + Arg softmax_out = prog[softmax_idx]->outputs[0]; + int attnv_matmul_idx = singleConsumer(softmax_out); + if (!isMatMul(prog, attnv_matmul_idx)) continue; + + Arg v_tr_out = prog[transpose_idx[v_slot]]->outputs[0]; + const auto& av_inputs = prog[attnv_matmul_idx]->inputs; + bool v_connected = false; + for (auto& ai : av_inputs) + if (ai.idx == v_tr_out.idx) v_connected = true; + if (!v_connected) continue; + + Arg attnv_out = prog[attnv_matmul_idx]->outputs[0]; + int out_transpose_idx = singleConsumer(attnv_out); + if (!isTranspose(prog, out_transpose_idx)) continue; + + Arg out_tr_out = prog[out_transpose_idx]->outputs[0]; + int out_reshape_idx = findMatchingConsumer(prog, out_tr_out, + [](Layer* L){ return dynamic_cast(L) != nullptr; }, + &extra_ops); + if (!isReshape(prog, out_reshape_idx)) continue; + + const auto& or_inputs = prog[out_reshape_idx]->inputs; + if (or_inputs.size() >= 2 && !netimpl->isConstArg(or_inputs[1])) { + auto it = producer_.find(or_inputs[1].idx); + if (it != producer_.end()) + collectShapeChain(prog, it->second, extra_ops); + } + + const Mat& Wq = prog[indices[q_slot]]->blobs[0]; + const Mat& Wk = prog[indices[k_slot]]->blobs[0]; + const Mat& Wv = prog[indices[v_slot]]->blobs[0]; + int input_hidden = Wq.size[0]; + int q_hidden = Wq.size[1]; + int k_hidden = Wk.size[1]; + int v_hidden = Wv.size[1]; + int total_hidden = q_hidden + k_hidden + v_hidden; + + int wshape[] = {input_hidden, total_hidden}; + Mat W_qkv(2, wshape, CV_32F); + for (int r = 0; r < input_hidden; r++) { + float* dst = W_qkv.ptr(r); + memcpy(dst, Wq.ptr(r), q_hidden * sizeof(float)); + memcpy(dst + q_hidden, Wk.ptr(r), k_hidden * sizeof(float)); + memcpy(dst + q_hidden + k_hidden, Wv.ptr(r), v_hidden * sizeof(float)); + } + + Mat bias_qkv; + bool has_bias = prog[indices[q_slot]]->blobs.size() >= 2; + if (has_bias) { + const Mat& bq = prog[indices[q_slot]]->blobs[1]; + const Mat& bk = prog[indices[k_slot]]->blobs[1]; + const Mat& bv = prog[indices[v_slot]]->blobs[1]; + int bias_total = (int)(bq.total() + bk.total() + bv.total()); + bias_qkv.create(1, &bias_total, CV_32F); + float* dst = bias_qkv.ptr(); + memcpy(dst, bq.ptr(), bq.total() * sizeof(float)); + memcpy(dst + bq.total(), bk.ptr(), bk.total() * sizeof(float)); + memcpy(dst + bq.total() + bk.total(), bv.ptr(), bv.total() * sizeof(float)); + } + + float param_scale = vit_style ? (1.f / (q_scale_val * k_scale_val)) + : post_scale_val; + + LayerParams attn_params; + attn_params.name = prog[indices[q_slot]]->name + "_fused_attention"; + attn_params.type = "Attention"; + attn_params.set("num_heads", num_heads[0]); + DictValue qkv_sizes_param = DictValue::arrayInt( + std::vector{q_hidden, k_hidden, v_hidden}.data(), 3); + attn_params.set("qkv_hidden_sizes", qkv_sizes_param); + attn_params.set("scale", param_scale); + attn_params.set("output_ndims", 3); + + attn_params.blobs.push_back(W_qkv); + if (has_bias) + attn_params.blobs.push_back(bias_qkv); + + Ptr attn_layer = LayerFactory::createLayerInstance( + attn_params.type, attn_params); + CV_Assert(attn_layer); + + Arg shared_input = prog[indices[0]]->inputs[0]; + if (has_mask) + attn_layer->inputs = { shared_input, mask_arg }; + else + attn_layer->inputs = { shared_input }; + attn_layer->outputs = prog[out_reshape_idx]->outputs; + attn_layer->netimpl = netimpl; + + std::set to_remove = { + indices[0], indices[1], indices[2], + reshape_idx[0], reshape_idx[1], reshape_idx[2], + transpose_idx[0], transpose_idx[1], transpose_idx[2], + qk_matmul_idx, softmax_idx, attnv_matmul_idx, + out_transpose_idx, out_reshape_idx + }; + if (k_mul_idx >= 0) to_remove.insert(k_mul_idx); + if (q_mul_idx >= 0) to_remove.insert(q_mul_idx); + if (qk_div_idx >= 0) to_remove.insert(qk_div_idx); + if (qk_add_idx >= 0) to_remove.insert(qk_add_idx); + for (int op : extra_ops) to_remove.insert(op); + for (int op : to_remove) + removed_ops.insert(op); + + int insert_pos = *std::min_element(to_remove.begin(), to_remove.end()); + attention_replacements_.push_back({insert_pos, attn_layer}); + modified = true; + break; + } + } + } + + if (modified) { + vector> newprog; + std::sort(attention_replacements_.begin(), attention_replacements_.end(), + [](auto& a, auto& b) { return a.first < b.first; }); + + size_t repl_idx = 0; + for (size_t i = 0; i < nops; i++) { + while (repl_idx < attention_replacements_.size() && + attention_replacements_[repl_idx].first == (int)i) { + newprog.push_back(attention_replacements_[repl_idx].second); + repl_idx++; + } + if (removed_ops.count((int)i)) + continue; + newprog.push_back(prog[i]); + } + graph->setProg(newprog); + } + + return modified; + } + + Net::Impl* netimpl; + +private: + std::map producer_; + std::map> consumers_; + vector>> attention_replacements_; +}; + +void Net::Impl::fuseAttention() +{ + ModelFusionAttention attnFusion(this); + attnFusion.fuse(); +} + +CV__DNN_INLINE_NS_END +}} diff --git a/modules/dnn/src/layers/attention_layer.cpp b/modules/dnn/src/layers/attention_layer.cpp index 807b3e2bda..df70dfbdad 100644 --- a/modules/dnn/src/layers/attention_layer.cpp +++ b/modules/dnn/src/layers/attention_layer.cpp @@ -165,7 +165,8 @@ class AttentionLayerImpl CV_FINAL : public AttentionLayer { std::vector &outputs, std::vector &internals) const CV_OVERRIDE { int num_inputs = inputs.size() + blobs.size(); - CV_CheckEQ(num_inputs, 3, "DNN/Attention: three inputs are required"); + CV_CheckGE(num_inputs, 3, "DNN/Attention: at least three inputs are required (data, weight, bias)"); + CV_CheckLE(num_inputs, 4, "DNN/Attention: at most four inputs are supported (data, weight, bias, mask)"); const auto &input_shape = inputs[0]; const auto &weight_shape = blobs.empty() ? inputs[1] : shape(blobs.front()); const auto &bias_shape = blobs.empty() ? inputs[2] : shape(blobs.back()); @@ -392,6 +393,41 @@ class AttentionLayerImpl CV_FINAL : public AttentionLayer { } }, loops * seq_len * qk_head_size * seq_len * (1 / 1024.0)); + // Additive attention mask applied before softmax. Supports BERT-style mask + // shapes [B, 1, 1, S] (per-batch) and [1, 1, 1, S] (global broadcast). + // Only reached when the fusion pass attached an external mask input. + int num_non_blob_inputs = (int)inputs.size(); + bool has_mask = (!blobs.empty() && num_non_blob_inputs >= 2) || + (blobs.empty() && num_non_blob_inputs >= 4); + if (has_mask) { + const Mat &mask_mat = blobs.empty() ? inputs[3] : inputs[1]; + CV_CheckTypeEQ(mask_mat.type(), CV_32F, + "DNN/Attention: mask must be float32"); + const int mask_last_dim = shape(mask_mat).back(); + CV_CheckEQ(mask_last_dim, (int)seq_len, + "DNN/Attention: mask last dim must equal seq_len"); + const size_t mask_total = mask_mat.total(); + int mask_batch_stride; + if (mask_total == batch_size * seq_len) mask_batch_stride = (int)seq_len; + else if (mask_total == seq_len) mask_batch_stride = 0; + else CV_Error(Error::StsNotImplemented, + "DNN/Attention: unsupported mask shape"); + + const float *mask_data = mask_mat.ptr(); + parallel_for_(Range(0, (int)loops), [&](const Range &r) { + for (int i = r.start; i < r.end; i++) { + const int b = i / (int)num_heads; + const float *m = mask_data + b * mask_batch_stride; + float *prob = output + i * seq_len_square; + for (size_t row = 0; row < seq_len; row++) { + float *p = prob + row * seq_len; + for (size_t j = 0; j < seq_len; j++) + p[j] += m[j]; + } + } + }, loops * seq_len * (1 / 1024.0)); + } + // Compute softmax on the last dimension softmax(attention_prob, attention_prob, shape(attention_prob).size() - 1); } diff --git a/modules/dnn/src/layers/cpu_kernels/fast_gemm.cpp b/modules/dnn/src/layers/cpu_kernels/fast_gemm.cpp index 81d9390100..e7e5605cbb 100644 --- a/modules/dnn/src/layers/cpu_kernels/fast_gemm.cpp +++ b/modules/dnn/src/layers/cpu_kernels/fast_gemm.cpp @@ -233,33 +233,208 @@ void fastGemmPackB(bool trans, size_t N, size_t K, const float *B, size_t ldb, f } } +static constexpr int FAST_GEMM_THIN_MAX_M = 12; +#if !CV_SIMD +// Scalar-fallback strip width (unroll factor, not SIMD lanes); 4 matches the narrowest universal-intrinsic float width so layout stays consistent with SIMD builds. +static constexpr int FAST_GEMM_THIN_SCALAR_NR = 4; +#endif + +static inline int fast_gemm_thin_lanes() { +#if (CV_SIMD || CV_SIMD_SCALABLE) + return VTraits::vlanes(); +#else + return FAST_GEMM_THIN_SCALAR_NR; +#endif +} + +static inline void fast_gemm_thin_strip(int M, int K, float alpha, + const float* A, int lda0, int lda1, + const float* b_strip, + float beta, float* c_strip, int ldc) { +#if CV_SIMD + const int NR = VTraits::vlanes(); + v_float32 acc[FAST_GEMM_THIN_MAX_M]; + for (int m = 0; m < M; m++) acc[m] = vx_setzero_f32(); + + for (int k = 0; k < K; k++) { + v_float32 bv = vx_load(b_strip + k * NR); + for (int m = 0; m < M; m++) { + v_float32 am = vx_setall_f32(A[m * lda0 + k * lda1]); + acc[m] = v_fma(bv, am, acc[m]); + } + } + + const v_float32 v_alpha = vx_setall_f32(alpha); + if (beta == 0.f) { + for (int m = 0; m < M; m++) + vx_store(c_strip + m * ldc, v_mul(acc[m], v_alpha)); + } else if (beta == 1.f) { + for (int m = 0; m < M; m++) { + v_float32 cv = vx_load(c_strip + m * ldc); + vx_store(c_strip + m * ldc, v_fma(acc[m], v_alpha, cv)); + } + } else { + const v_float32 v_beta = vx_setall_f32(beta); + for (int m = 0; m < M; m++) { + v_float32 cv = vx_load(c_strip + m * ldc); + cv = v_mul(cv, v_beta); + vx_store(c_strip + m * ldc, v_fma(acc[m], v_alpha, cv)); + } + } +#elif CV_SIMD_SCALABLE + // Scalable vector types (e.g. RVV) are sizeless and cannot form arrays; + // back the per-row accumulators with a scalar scratch buffer. + const int NR = VTraits::vlanes(); + float acc_buf[FAST_GEMM_THIN_MAX_M * VTraits::max_nlanes]; + for (int m = 0; m < M; m++) vx_store(acc_buf + m * NR, vx_setzero_f32()); + + for (int k = 0; k < K; k++) { + v_float32 bv = vx_load(b_strip + k * NR); + for (int m = 0; m < M; m++) { + v_float32 am = vx_setall_f32(A[m * lda0 + k * lda1]); + v_float32 acc_m = vx_load(acc_buf + m * NR); + vx_store(acc_buf + m * NR, v_fma(bv, am, acc_m)); + } + } + + const v_float32 v_alpha = vx_setall_f32(alpha); + if (beta == 0.f) { + for (int m = 0; m < M; m++) + vx_store(c_strip + m * ldc, v_mul(vx_load(acc_buf + m * NR), v_alpha)); + } else if (beta == 1.f) { + for (int m = 0; m < M; m++) { + v_float32 cv = vx_load(c_strip + m * ldc); + vx_store(c_strip + m * ldc, v_fma(vx_load(acc_buf + m * NR), v_alpha, cv)); + } + } else { + const v_float32 v_beta = vx_setall_f32(beta); + for (int m = 0; m < M; m++) { + v_float32 cv = vx_load(c_strip + m * ldc); + cv = v_mul(cv, v_beta); + vx_store(c_strip + m * ldc, v_fma(vx_load(acc_buf + m * NR), v_alpha, cv)); + } + } +#else + const int NR = FAST_GEMM_THIN_SCALAR_NR; + float acc[FAST_GEMM_THIN_MAX_M * FAST_GEMM_THIN_SCALAR_NR] = {0}; + for (int k = 0; k < K; k++) { + for (int m = 0; m < M; m++) { + float a_mk = A[m * lda0 + k * lda1]; + for (int c = 0; c < NR; c++) + acc[m * NR + c] += a_mk * b_strip[k * NR + c]; + } + } + if (beta == 0.f) { + for (int m = 0; m < M; m++) + for (int c = 0; c < NR; c++) + c_strip[m * ldc + c] = alpha * acc[m * NR + c]; + } else { + for (int m = 0; m < M; m++) + for (int c = 0; c < NR; c++) + c_strip[m * ldc + c] = beta * c_strip[m * ldc + c] + alpha * acc[m * NR + c]; + } +#endif +} + +bool fastGemmThinEligible(int M, int N, int K) { + if (M <= 0 || N <= 0 || K <= 0) return false; + if (M > FAST_GEMM_THIN_MAX_M) return false; + const int NR = fast_gemm_thin_lanes(); + if (N < 2 * NR) return false; + return true; +} + +size_t fastGemmThinPackBSize(int N, int K) { + const int NR = fast_gemm_thin_lanes(); + const int n_strips = (N + NR - 1) / NR; + return (size_t)n_strips * (size_t)NR * (size_t)K; +} + +void fastGemmThinPackB(int N, int K, const float* B, size_t ldb_K, size_t ldb_N, float* packed_B) { + const int NR = fast_gemm_thin_lanes(); + const int n_strips = (N + NR - 1) / NR; + for (int s = 0; s < n_strips; s++) { + const int j0 = s * NR; + const int nr = std::min(NR, N - j0); + float* strip = packed_B + (size_t)s * NR * K; + for (int k = 0; k < K; k++) { + float* out = strip + k * NR; + int c = 0; + for (; c < nr; c++) out[c] = B[k * ldb_K + (j0 + c) * ldb_N]; + for (; c < NR; c++) out[c] = 0.f; + } + } +} + +void fastGemmThin(int M, int N, int K, float alpha, + const float* A, int lda0, int lda1, + const float* packed_B, float beta, + float* C, int ldc, bool multi_thread) { + const int NR = fast_gemm_thin_lanes(); + const int n_full_strips = N / NR; + const int n_tail = N - n_full_strips * NR; + + auto fn = [&](const Range& r) { + for (int s = r.start; s < r.end; s++) { + const float* b_strip = packed_B + (size_t)s * NR * K; + float* c_strip = C + s * NR; + fast_gemm_thin_strip(M, K, alpha, A, lda0, lda1, b_strip, beta, c_strip, ldc); + } + }; + if (multi_thread && n_full_strips > 1) { + parallel_for_(Range(0, n_full_strips), fn, (double)n_full_strips * M * K * NR * (1.0 / 1024.0)); + } else { + fn(Range(0, n_full_strips)); + } + + if (n_tail > 0) { + const int j0 = n_full_strips * NR; + const float* b_strip = packed_B + (size_t)n_full_strips * NR * K; + for (int m = 0; m < M; m++) { + for (int c = 0; c < n_tail; c++) { + float acc_s = 0.f; + for (int k = 0; k < K; k++) + acc_s += A[m * lda0 + k * lda1] * b_strip[k * NR + c]; + const float mul = alpha * acc_s; + float* p = C + m * ldc + j0 + c; + *p = (beta == 0.f) ? mul : beta * (*p) + mul; + } + } + } +} + static void fast_gemm_thin(float alpha, float beta, int M, int N, int K, const char *a_, int lda0, int lda1, const char *b_, int ldb, char *c_, int ldc, bool multi_thread) { - const float* a = (const float*)a_; + const float* A = (const float*)a_; + const float* B = (const float*)b_; + float* C = (float*)c_; + + if (fastGemmThinEligible(M, N, K)) { + AutoBuffer packed(fastGemmThinPackBSize(N, K)); + fastGemmThinPackB(N, K, B, (size_t)ldb, 1, packed.data()); + fastGemmThin(M, N, K, alpha, A, lda0, lda1, packed.data(), beta, C, ldc, multi_thread); + return; + } auto fn = [&](const Range &r) { - for(int start = r.start ; start < r.end; start++ ) { - float* c_i = (float*)c_ + start * ldc; + for (int i = r.start; i < r.end; i++) { + float* c_i = C + i * ldc; if (beta == 0.f) - for(int j = 0; j < N; j++ ) c_i[j] = 0.f; + for (int j = 0; j < N; j++) c_i[j] = 0.f; else if (beta != 1.f) - for(int j = 0; j < N; j++ ) c_i[j] *= beta; - for(int k = 0; k < K; k++ ) { - const float* b_k = (const float*)b_ + k * ldb; - float aval = alpha * a[start * lda0 + k * lda1]; - for(int j = 0; j < N; j++ ) + for (int j = 0; j < N; j++) c_i[j] *= beta; + for (int k = 0; k < K; k++) { + const float* b_k = B + k * ldb; + float aval = alpha * A[i * lda0 + k * lda1]; + for (int j = 0; j < N; j++) c_i[j] += aval * b_k[j]; } } }; - if (multi_thread) { - int total = M; // outer loops - int cost_per_thread = static_cast(K * N); // inner loops - double nstripes = (size_t)total * cost_per_thread * (1 / 1024.0); - parallel_for_(Range(0, total), fn, nstripes); + parallel_for_(Range(0, M), fn, (double)M * K * N * (1.0 / 1024.0)); } else { fn(Range(0, M)); } @@ -321,7 +496,7 @@ void fastGemm(bool trans_a, bool trans_b, int ma, int na, int mb, int nb, std::swap(ldb0, ldb1); } - if (!trans_b && ldb1 == 1 && (M <= 4 || (uint64_t)M * N * K <= 10000)) { + if (!trans_b && ldb1 == 1 && (fastGemmThinEligible(M, N, K) || (uint64_t)M * N * K <= 10000)) { return fast_gemm_thin(alpha, beta, M, N, K, a, lda0, lda1, b, ldb0, c, ldc, opt.multi_thread); } @@ -395,6 +570,18 @@ void fastGemmBatch(size_t batch, const size_t *A_offsets, const size_t *B_offset const char *b = (const char *)B; char *c = (char *)C; + // Below ~10K MACs (M*N*K) the blocked SIMD kernel's pack/tile overhead outweighs its speedup, so route tiny problems through the unblocked thin path. + if (ldb1 == 1 && (fastGemmThinEligible(M, N, K) || (uint64_t)M * N * K <= 10000)) { + const size_t esz = sizeof(float); + for (size_t i = 0; i < batch; i++) { + fast_gemm_thin(alpha, beta, M, N, K, + a + A_offsets[i] * esz, lda0, lda1, + b + B_offsets[i] * esz, ldb0, + c + C_offsets[i] * esz, ldc, opt.multi_thread); + } + return; + } + #if CV_TRY_NEON if (opt.use_neon) { opt_NEON::fastGemmBatchKernel(batch, A_offsets, B_offsets, C_offsets, M, N, K, alpha, a, lda0, lda1, b, ldb0, ldb1, beta, c, ldc, sizeof(float)); @@ -486,6 +673,17 @@ void fastGemmBatch(size_t batch, const char *b = (const char *)B.ptr(); char *c = (char *)C.ptr(); + if (ldb1 == 1 && (fastGemmThinEligible(M, N, K) || (uint64_t)M * N * K <= 10000)) { + const size_t esz = sizeof(float); + for (size_t i = 0; i < batch; i++) { + fast_gemm_thin(alpha, beta, M, N, K, + a + A_offsets[i] * esz, lda0, lda1, + b + B_offsets[i] * esz, ldb0, + c + C_offsets[i] * esz, ldc, opt.multi_thread); + } + return; + } + #if CV_TRY_NEON if (opt.use_neon) { opt_NEON::fastGemmBatchKernel(batch, A_offsets.data(), B_offsets.data(), C_offsets.data(), M, N, K, alpha, a, lda0, lda1, b, ldb0, ldb1, beta, c, ldc, sizeof(float)); diff --git a/modules/dnn/src/layers/cpu_kernels/fast_gemm.hpp b/modules/dnn/src/layers/cpu_kernels/fast_gemm.hpp index c9bdfa9ad4..b0f9740074 100644 --- a/modules/dnn/src/layers/cpu_kernels/fast_gemm.hpp +++ b/modules/dnn/src/layers/cpu_kernels/fast_gemm.hpp @@ -186,6 +186,14 @@ void fastGemmBatch(size_t batch, int M, int N, int K, float alpha, const Mat&A, int lda0, int lda1, const Mat&B, int ldb0, int ldb1, float beta, Mat&C, int ldc, FastGemmOpt &opt); +bool fastGemmThinEligible(int M, int N, int K); +size_t fastGemmThinPackBSize(int N, int K); +void fastGemmThinPackB(int N, int K, const float* B, size_t ldb_K, size_t ldb_N, float* packed_B); +void fastGemmThin(int M, int N, int K, float alpha, + const float* A, int lda0, int lda1, + const float* packed_B, float beta, + float* C, int ldc, bool multi_thread); + void pagedAttnQKGemm( const Mat& Q, const std::vector &K, Mat& A, int T_q, int Nq, int N_k, int T_s, int D, diff --git a/modules/dnn/src/layers/matmul_layer.cpp b/modules/dnn/src/layers/matmul_layer.cpp index 10463f1440..384d69bde5 100644 --- a/modules/dnn/src/layers/matmul_layer.cpp +++ b/modules/dnn/src/layers/matmul_layer.cpp @@ -156,6 +156,17 @@ class MatMulLayerImpl CV_FINAL : public MatMulLayer { if (!blobs.empty()) { fastGemmPackB(blobs[0], packed_input_B, trans_b, opt); helper.updatePackedBOffsets(packed_input_B.size()); + + if (helper.batch == 1 && blobs[0].type() == CV_32F && + fastGemmThinEligible(helper.M, helper.N, helper.K)) { + thin_packed_B.resize(fastGemmThinPackBSize(helper.N, helper.K)); + fastGemmThinPackB(helper.N, helper.K, + blobs[0].ptr(), + (size_t)helper.ldb0, (size_t)helper.ldb1, + thin_packed_B.data()); + } else { + thin_packed_B.clear(); + } } // broadcast bias if needed @@ -259,6 +270,11 @@ class MatMulLayerImpl CV_FINAL : public MatMulLayer { fastGemmBatch(helper.batch, helper.A_offsets.data(), helper.B_offsets.data(), helper.C_offsets.data(), helper.M, helper.N, helper.K, alpha, a, helper.lda0, helper.lda1, b, helper.ldb0, helper.ldb1, beta, y, helper.ldc, opt); + } else if (!thin_packed_B.empty()) { + fastGemmThin(helper.M, helper.N, helper.K, alpha, + a, helper.lda0, helper.lda1, + thin_packed_B.data(), beta, + y, helper.ldc, opt.multi_thread); } else { fastGemmBatch(helper.batch, helper.A_offsets.data(), helper.packed_B_offsets.data(), helper.C_offsets.data(), helper.M, helper.N, helper.K, alpha, a, helper.lda0, helper.lda1, @@ -495,6 +511,7 @@ class MatMulLayerImpl CV_FINAL : public MatMulLayer { int real_ndims_C; std::vector packed_input_B; + std::vector thin_packed_B; Mat broadcast_bias; FastGemmOpt opt; diff --git a/modules/dnn/src/net_impl.hpp b/modules/dnn/src/net_impl.hpp index 0b1e2673cb..b8b6c0356c 100644 --- a/modules/dnn/src/net_impl.hpp +++ b/modules/dnn/src/net_impl.hpp @@ -454,6 +454,8 @@ struct Net::Impl : public detail::NetImplBase void assignBuffers(); // fuse batch norm, add bias and activation to convolution void fuseBasic(); + // fuse ViT-style multi-head attention subgraphs + void fuseAttention(); // replace constant sub-expressions with their results void fuseQDQ(); diff --git a/modules/dnn/src/net_impl2.cpp b/modules/dnn/src/net_impl2.cpp index 3e5883c0a4..d8490e59dd 100644 --- a/modules/dnn/src/net_impl2.cpp +++ b/modules/dnn/src/net_impl2.cpp @@ -548,6 +548,7 @@ void Net::Impl::prepareForInference() fuseQDQ(); constFold(); constArgs(); + fuseAttention(); useBlockLayout(); fuseBasic(); assignBuffers(); diff --git a/modules/dnn/src/onnx/onnx_graph_simplifier.cpp b/modules/dnn/src/onnx/onnx_graph_simplifier.cpp index ec06ff6a30..03bb8bf3c4 100644 --- a/modules/dnn/src/onnx/onnx_graph_simplifier.cpp +++ b/modules/dnn/src/onnx/onnx_graph_simplifier.cpp @@ -845,12 +845,28 @@ public: std::vector axes = extractAxis(net, matchedNodesIds[mean]); // check whether it is -1 or last_axis or [axis, ..., last_axis] int64_t input_ndims = static_cast(net.dynamicCast()->getTensorShapeSize(matchedNodesIds[mean], 0)); - if (input_ndims == -1) { - return false; // input shape unknown + + // When axes are all negative (e.g. [-1] or [-2, -1]), we can validate + // the pattern without knowing input_ndims. + bool all_axes_negative = !axes.empty(); + for (size_t i = 0; i < axes.size(); i++) { + if (axes[i] >= 0) { all_axes_negative = false; break; } } - // assume that axes are sorted in ascending order, e.g. [0, 1, 2, 3] or [-3, -2, -1] - if (axes.back() != -1 && axes.back() != (input_ndims - 1)) { - return false; + + if (input_ndims == -1 && !all_axes_negative) { + return false; // input shape unknown and axes are positive + } + + if (input_ndims != -1) { + // assume that axes are sorted in ascending order, e.g. [0, 1, 2, 3] or [-3, -2, -1] + if (axes.back() != -1 && axes.back() != (input_ndims - 1)) { + return false; + } + } else { + // axes are all negative; check that the last axis is -1 + if (axes.back() != -1) { + return false; + } } for (size_t i = 0; i < axes.size() - 1; i++) { if (axes[i] - axes[i + 1] != -1) { @@ -861,9 +877,18 @@ public: std::vector axes1 = extractAxis(net, matchedNodesIds[mean1]); if (axes.size() != axes1.size()) return false; - for (size_t i = 0; i < axes.size(); i++) { - if (((axes[i] + input_ndims) % input_ndims) != ((axes1[i] + input_ndims) % input_ndims)) { - return false; + if (input_ndims != -1) { + for (size_t i = 0; i < axes.size(); i++) { + if (((axes[i] + input_ndims) % input_ndims) != ((axes1[i] + input_ndims) % input_ndims)) { + return false; + } + } + } else { + // both axes sets are negative; just compare directly + for (size_t i = 0; i < axes.size(); i++) { + if (axes[i] != axes1[i]) { + return false; + } } } axis = axes[0]; diff --git a/modules/dnn/test/test_model.cpp b/modules/dnn/test/test_model.cpp index 5fcf9a562c..f5b429032c 100644 --- a/modules/dnn/test/test_model.cpp +++ b/modules/dnn/test/test_model.cpp @@ -948,6 +948,140 @@ TEST_P(Reproducibility_ResNet50_QDQ_ONNX, Accuracy) INSTANTIATE_TEST_CASE_P(/**/, Reproducibility_ResNet50_QDQ_ONNX, testing::ValuesIn(getAvailableTargets(DNN_BACKEND_OPENCV))); +typedef testing::TestWithParam Reproducibility_ViT_ONNX; +TEST_P(Reproducibility_ViT_ONNX, Accuracy) +{ + Target targetId = GetParam(); + applyTestTag(targetId == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_1GB : CV_TEST_TAG_MEMORY_2GB); + ASSERT_TRUE(ocl::useOpenCL() || targetId == DNN_TARGET_CPU || targetId == DNN_TARGET_CPU_FP16); + auto engine_forced = static_cast( + cv::utils::getConfigurationParameterSizeT("OPENCV_FORCE_DNN_ENGINE", ENGINE_AUTO)); + if (engine_forced == ENGINE_CLASSIC) + { + applyTestTag(CV_TEST_TAG_DNN_SKIP_PARSER); + return; + } + + std::string modelname = _tf("vit_base_patch16_224_Opset16.onnx", false); + Net net = readNetFromONNX(modelname); + + net.setPreferableBackend(DNN_BACKEND_OPENCV); + net.setPreferableTarget(targetId); + + if (targetId == DNN_TARGET_CPU_FP16) + net.enableWinograd(false); + + std::string imgname = _tf("sqcat.png"); + Mat image = imread(imgname); + // ViT preprocessing: (pixel/255 - 0.5)/0.5 == pixel/127.5 - 1 + // blobFromImage form: (pixel - 127.5) * (1/127.5) + Mat input = blobFromImage(image, 1.0 / 127.5, Size(224, 224), + Scalar(127.5, 127.5, 127.5), + true, true, CV_32F); + ASSERT_TRUE(!input.empty()); + + net.setInput(input); + Mat out = net.forward(); + + const int K = 5; + std::vector > res; + topK(out, res, K); + ASSERT_EQ(int(res.size()), K); + + // Reference top-5 captured from the ONNX Runtime engine (OPENCV_FORCE_DNN_ENGINE=4). + std::vector > ref = { + {285, 7.683f}, {282, 7.182f}, {281, 6.894f}, {287, 3.623f}, {283, 3.287f} + }; + const float eps = 0.5f; + + std::vector reflabels(K), reslabels(K); + for (int i = 0; i < K; i++) { + reflabels[i] = ref[i].first; + reslabels[i] = res[i].first; + } + ASSERT_EQ(reflabels, reslabels); + + for (int i = 0; i < K; i++) { + EXPECT_NEAR(ref[i].second, res[i].second, eps); + } +} +INSTANTIATE_TEST_CASE_P(/**/, Reproducibility_ViT_ONNX, + testing::ValuesIn(getAvailableTargets(DNN_BACKEND_OPENCV))); + +typedef testing::TestWithParam Reproducibility_BERT_ONNX; +TEST_P(Reproducibility_BERT_ONNX, Accuracy) +{ + Target targetId = GetParam(); + applyTestTag(targetId == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_1GB : CV_TEST_TAG_MEMORY_2GB); + ASSERT_TRUE(ocl::useOpenCL() || targetId == DNN_TARGET_CPU || targetId == DNN_TARGET_CPU_FP16); + auto engine_forced = static_cast( + cv::utils::getConfigurationParameterSizeT("OPENCV_FORCE_DNN_ENGINE", ENGINE_AUTO)); + if (engine_forced == ENGINE_CLASSIC) + { + applyTestTag(CV_TEST_TAG_DNN_SKIP_PARSER); + return; + } + + std::string modelname = _tf("onnx/models/bert.onnx", false); + Net net = readNetFromONNX(modelname); + + net.setPreferableBackend(DNN_BACKEND_OPENCV); + net.setPreferableTarget(targetId); + + // Tokenized with bert-base-uncased: "The [MASK] sat on the mat." + // [CLS]=101 the=1996 [MASK]=103 sat=2938 on=2006 the=1996 mat=13523 .=1012 [SEP]=102 + const int seq_len = 9; + int64_t input_ids_data[seq_len] = {101, 1996, 103, 2938, 2006, 1996, 13523, 1012, 102}; + int64_t attention_mask_data[seq_len] = {1, 1, 1, 1, 1, 1, 1, 1, 1}; + int64_t token_type_ids_data[seq_len] = {0, 0, 0, 0, 0, 0, 0, 0, 0}; + + int shape[2] = {1, seq_len}; + Mat input_ids(2, shape, CV_64S, input_ids_data); + Mat attention_mask(2, shape, CV_64S, attention_mask_data); + Mat token_type_ids(2, shape, CV_64S, token_type_ids_data); + + net.setInput(input_ids, "input_ids"); + net.setInput(attention_mask, "attention_mask"); + net.setInput(token_type_ids, "token_type_ids"); + Mat out = net.forward(); + + // Output shape: [1, 9, 30522] (batch, seq_len, vocab_size) + ASSERT_EQ(out.dims, 3); + ASSERT_EQ(out.size[0], 1); + ASSERT_EQ(out.size[1], seq_len); + ASSERT_EQ(out.size[2], 30522); + + const int vocab = 30522; + const float* mask_logits = out.ptr() + 2 * vocab; + + const int K = 5; + std::vector > res; + std::vector idx(vocab); + for (int i = 0; i < vocab; i++) idx[i] = i; + std::partial_sort(idx.begin(), idx.begin() + K, idx.end(), + [&](int a, int b) { return mask_logits[a] > mask_logits[b]; }); + for (int k = 0; k < K; k++) + res.emplace_back(idx[k], mask_logits[idx[k]]); + + std::vector > ref = { + {2611, 8.222f}, {2158, 8.196f}, {3899, 8.021f}, {2879, 7.972f}, {2450, 7.393f} + }; + const float eps = 0.5f; + + std::vector reflabels(K), reslabels(K); + for (int i = 0; i < K; i++) { + reflabels[i] = ref[i].first; + reslabels[i] = res[i].first; + } + ASSERT_EQ(reflabels, reslabels); + + for (int i = 0; i < K; i++) { + EXPECT_NEAR(ref[i].second, res[i].second, eps); + } +} +INSTANTIATE_TEST_CASE_P(/**/, Reproducibility_BERT_ONNX, + testing::ValuesIn(getAvailableTargets(DNN_BACKEND_OPENCV))); + typedef testing::TestWithParam Reproducibility_MobileNetSSD_ONNX; TEST_P(Reproducibility_MobileNetSSD_ONNX, Accuracy) {