mirror of
https://github.com/opencv/opencv.git
synced 2026-07-21 19:33:03 +04:00
Merge pull request #28859 from abhishek-gola:fuse_attention
Added Attention fusion and thin gemm support #28859 Merge with: https://github.com/opencv/opencv_extra/pull/1350 Performance numbers after these optimizations: For Device: Intel(R) Core(TM) i9-14900KS, x86, 32 Cores, ubuntu 24.04, | Model | `ENGINE_NEW (Before)` | `ENGINE_NEW (After)` | `ENGINE_ORT` | | :--- | :--- | :--- | :--- | | **BERT** |26.3 ms| 9.15 ms| 9.13 ms| | **ViT** | 79.65 ms| 63.23 ms| 32.3 ms| ### Pull Request Readiness Checklist See details at https://github.com/opencv/opencv/wiki/How_to_contribute#making-a-good-pull-request - [x] I agree to contribute to the project under Apache 2 License. - [x] To the best of my knowledge, the proposed patch is not based on a code under GPL or another license that is incompatible with OpenCV - [x] The PR is proposed to the proper branch - [x] There is a reference to the original bug report and related work - [x] There is accuracy test, performance test and test data in opencv_extra repository, if applicable Patch to opencv_extra has the same branch name. - [x] The feature is well documented and sample code can be built with the project CMake
This commit is contained in:
Abhishek Gola
GitHub
@@ -406,6 +406,22 @@ PERF_TEST_P_(DNNTestNetwork, VIT_B_32)
|
||||
processNet("dnn/onnx/models/vit_b_32.onnx", "", cv::Size(224, 224));
|
||||
}
|
||||
|
||||
PERF_TEST_P_(DNNTestNetwork, BERT)
|
||||
{
|
||||
const int seq_len = 9;
|
||||
int64_t input_ids_data[seq_len] = {101, 1996, 103, 2938, 2006, 1996, 13523, 1012, 102};
|
||||
int64_t attention_mask_data[seq_len] = {1, 1, 1, 1, 1, 1, 1, 1, 1};
|
||||
int64_t token_type_ids_data[seq_len] = {0, 0, 0, 0, 0, 0, 0, 0, 0};
|
||||
int shp[2] = {1, seq_len};
|
||||
Mat input_ids(2, shp, CV_64S, input_ids_data);
|
||||
Mat attention_mask(2, shp, CV_64S, attention_mask_data);
|
||||
Mat token_type_ids(2, shp, CV_64S, token_type_ids_data);
|
||||
processNet("dnn/onnx/models/bert.onnx", "",
|
||||
{std::make_tuple(input_ids, "input_ids"),
|
||||
std::make_tuple(attention_mask, "attention_mask"),
|
||||
std::make_tuple(token_type_ids, "token_type_ids")});
|
||||
}
|
||||
|
||||
PERF_TEST_P_(DNNTestNetwork, VIT_Base_Patch16_224)
|
||||
{
|
||||
applyTestTag(CV_TEST_TAG_MEMORY_512MB);
|
||||
|
||||
@@ -0,0 +1,530 @@
|
||||
// This file is part of OpenCV project.
|
||||
// It is subject to the license terms in the LICENSE file found in the top-level directory
|
||||
// of this distribution and at http://opencv.org/license.html.
|
||||
// Copyright (C) 2026, BigVision LLC, all rights reserved.
|
||||
// Third party copyrights are property of their respective owners.
|
||||
|
||||
// Fuses multi-head attention subgraphs into a single Attention layer.
|
||||
|
||||
#include "precomp.hpp"
|
||||
#include "net_impl.hpp"
|
||||
|
||||
namespace cv { namespace dnn {
|
||||
CV__DNN_INLINE_NS_BEGIN
|
||||
|
||||
using std::vector;
|
||||
using std::string;
|
||||
|
||||
struct ModelFusionAttention
|
||||
{
|
||||
ModelFusionAttention(Net::Impl* netimpl_) : netimpl(netimpl_) {}
|
||||
|
||||
void fuse()
|
||||
{
|
||||
fuseGraph(netimpl->mainGraph);
|
||||
}
|
||||
|
||||
int singleConsumer(Arg a) const
|
||||
{
|
||||
auto it = consumers_.find(a.idx);
|
||||
if (it == consumers_.end() || it->second.size() != 1)
|
||||
return -1;
|
||||
return it->second[0];
|
||||
}
|
||||
|
||||
bool isReshape(const vector<Ptr<Layer>>& prog, int idx) const
|
||||
{
|
||||
if (idx < 0 || idx >= (int)prog.size() || !prog[idx])
|
||||
return false;
|
||||
return dynamic_cast<Reshape2Layer*>(prog[idx].get()) != nullptr;
|
||||
}
|
||||
|
||||
bool isTranspose(const vector<Ptr<Layer>>& prog, int idx) const
|
||||
{
|
||||
if (idx < 0 || idx >= (int)prog.size() || !prog[idx])
|
||||
return false;
|
||||
return dynamic_cast<TransposeLayer*>(prog[idx].get()) != nullptr;
|
||||
}
|
||||
|
||||
bool isSoftmax(const vector<Ptr<Layer>>& prog, int idx) const
|
||||
{
|
||||
if (idx < 0 || idx >= (int)prog.size() || !prog[idx])
|
||||
return false;
|
||||
return prog[idx]->type == "Softmax";
|
||||
}
|
||||
|
||||
bool isMatMul(const vector<Ptr<Layer>>& prog, int idx) const
|
||||
{
|
||||
if (idx < 0 || idx >= (int)prog.size() || !prog[idx])
|
||||
return false;
|
||||
return dynamic_cast<MatMulLayer*>(prog[idx].get()) != nullptr;
|
||||
}
|
||||
|
||||
bool isScalarBinOp(const vector<Ptr<Layer>>& prog, int idx,
|
||||
NaryEltwiseLayer::OPERATION op, float* val) const
|
||||
{
|
||||
if (idx < 0 || idx >= (int)prog.size() || !prog[idx])
|
||||
return false;
|
||||
NaryEltwiseLayer* elt = dynamic_cast<NaryEltwiseLayer*>(prog[idx].get());
|
||||
if (!elt || elt->op != op)
|
||||
return false;
|
||||
const auto& inputs = prog[idx]->inputs;
|
||||
if (inputs.size() != 2) return false;
|
||||
for (int k = 0; k < 2; k++) {
|
||||
Arg inp = inputs[k];
|
||||
if (netimpl->isConstArg(inp)) {
|
||||
Mat t = netimpl->argTensor(inp);
|
||||
if (t.total() == 1 && t.type() == CV_32F) {
|
||||
*val = t.at<float>(0);
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
bool isScalarMul(const vector<Ptr<Layer>>& prog, int idx, float* val) const
|
||||
{
|
||||
return isScalarBinOp(prog, idx, NaryEltwiseLayer::OPERATION::PROD, val);
|
||||
}
|
||||
|
||||
bool isScalarDiv(const vector<Ptr<Layer>>& prog, int idx, float* val) const
|
||||
{
|
||||
return isScalarBinOp(prog, idx, NaryEltwiseLayer::OPERATION::DIV, val);
|
||||
}
|
||||
|
||||
// Accept Add op with exactly two inputs; identify the non-constant runtime
|
||||
// input (the mask tensor). Returns false if the Add doesn't match.
|
||||
bool isMaskAdd(const vector<Ptr<Layer>>& prog, int idx, Arg* out_mask) const
|
||||
{
|
||||
if (idx < 0 || idx >= (int)prog.size() || !prog[idx])
|
||||
return false;
|
||||
NaryEltwiseLayer* elt = dynamic_cast<NaryEltwiseLayer*>(prog[idx].get());
|
||||
if (!elt || elt->op != NaryEltwiseLayer::OPERATION::ADD)
|
||||
return false;
|
||||
const auto& inputs = prog[idx]->inputs;
|
||||
if (inputs.size() != 2) return false;
|
||||
*out_mask = inputs[1];
|
||||
return true;
|
||||
}
|
||||
|
||||
// Extract a scalar integer from a const-valued arg, possibly wrapped in an
|
||||
// Unsqueeze of a scalar const. Returns -1 if extraction fails.
|
||||
int extractConstInt(const vector<Ptr<Layer>>& prog, Arg a) const
|
||||
{
|
||||
auto readScalar = [&](Arg x) -> int {
|
||||
if (!netimpl->isConstArg(x)) return -1;
|
||||
Mat t = netimpl->argTensor(x);
|
||||
if (t.total() != 1) return -1;
|
||||
if (t.type() == CV_64S) return (int)t.at<int64_t>(0);
|
||||
if (t.type() == CV_32S) return (int)t.at<int32_t>(0);
|
||||
return -1;
|
||||
};
|
||||
int v = readScalar(a);
|
||||
if (v > 0) return v;
|
||||
auto it = producer_.find(a.idx);
|
||||
if (it == producer_.end()) return -1;
|
||||
int pidx = it->second;
|
||||
if (pidx < 0 || pidx >= (int)prog.size() || !prog[pidx]) return -1;
|
||||
if (prog[pidx]->type == "Unsqueeze" && !prog[pidx]->inputs.empty())
|
||||
return readScalar(prog[pidx]->inputs[0]);
|
||||
return -1;
|
||||
}
|
||||
|
||||
void collectShapeChain(const vector<Ptr<Layer>>& prog, int concat_idx,
|
||||
std::set<int>& chain) const
|
||||
{
|
||||
if (concat_idx < 0 || concat_idx >= (int)prog.size() || !prog[concat_idx])
|
||||
return;
|
||||
chain.insert(concat_idx);
|
||||
for (Arg ci : prog[concat_idx]->inputs) {
|
||||
if (netimpl->isConstArg(ci)) continue;
|
||||
int cur = -1;
|
||||
auto it = producer_.find(ci.idx);
|
||||
if (it != producer_.end()) cur = it->second;
|
||||
while (cur >= 0 && (int)prog.size() > cur && prog[cur]) {
|
||||
const std::string& t = prog[cur]->type;
|
||||
if (t != "Unsqueeze" && t != "Gather2" && t != "Shape") break;
|
||||
Arg cur_out = prog[cur]->outputs[0];
|
||||
auto cit = consumers_.find(cur_out.idx);
|
||||
if (cit == consumers_.end()) break;
|
||||
bool all_in_chain = true;
|
||||
for (int c : cit->second) if (!chain.count(c)) { all_in_chain = false; break; }
|
||||
if (!all_in_chain) break;
|
||||
chain.insert(cur);
|
||||
if (prog[cur]->inputs.empty()) break;
|
||||
Arg prev = prog[cur]->inputs[0];
|
||||
auto pit = producer_.find(prev.idx);
|
||||
if (pit == producer_.end()) break;
|
||||
cur = pit->second;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
template <class Pred>
|
||||
int findMatchingConsumer(const vector<Ptr<Layer>>& prog, Arg out,
|
||||
Pred pred, std::set<int>* extra_shape_ops) const
|
||||
{
|
||||
auto it = consumers_.find(out.idx);
|
||||
if (it == consumers_.end()) return -1;
|
||||
int matched = -1;
|
||||
for (int c : it->second) {
|
||||
if (c < 0 || c >= (int)prog.size() || !prog[c]) return -1;
|
||||
if (pred(prog[c].get())) {
|
||||
if (matched >= 0) return -1;
|
||||
matched = c;
|
||||
} else if (prog[c]->type == "Shape") {
|
||||
if (extra_shape_ops) extra_shape_ops->insert(c);
|
||||
} else {
|
||||
return -1;
|
||||
}
|
||||
}
|
||||
return matched;
|
||||
}
|
||||
|
||||
int followProjChain(const vector<Ptr<Layer>>& prog,
|
||||
int proj_matmul_idx,
|
||||
int* out_reshape_idx,
|
||||
int* out_num_heads,
|
||||
vector<int>* out_perm,
|
||||
std::set<int>* extra_ops_to_remove) const
|
||||
{
|
||||
if (proj_matmul_idx < 0) return -1;
|
||||
Arg proj_out = prog[proj_matmul_idx]->outputs[0];
|
||||
int reshape_idx = findMatchingConsumer(prog, proj_out,
|
||||
[](Layer* L){ return dynamic_cast<Reshape2Layer*>(L) != nullptr; },
|
||||
extra_ops_to_remove);
|
||||
if (!isReshape(prog, reshape_idx)) return -1;
|
||||
|
||||
const auto& reshape_inputs = prog[reshape_idx]->inputs;
|
||||
if (reshape_inputs.size() < 2) return -1;
|
||||
Arg shape_arg = reshape_inputs[1];
|
||||
|
||||
int num_heads = -1;
|
||||
if (netimpl->isConstArg(shape_arg)) {
|
||||
Mat shape_mat = netimpl->argTensor(shape_arg);
|
||||
if (shape_mat.total() != 4) return -1;
|
||||
const int64_t* shape_data = shape_mat.ptr<int64_t>();
|
||||
num_heads = static_cast<int>(shape_data[2]);
|
||||
} else {
|
||||
auto it = producer_.find(shape_arg.idx);
|
||||
if (it == producer_.end()) return -1;
|
||||
int concat_idx = it->second;
|
||||
if (concat_idx < 0 || concat_idx >= (int)prog.size() || !prog[concat_idx])
|
||||
return -1;
|
||||
if (!dynamic_cast<Concat2Layer*>(prog[concat_idx].get()))
|
||||
return -1;
|
||||
const auto& cinputs = prog[concat_idx]->inputs;
|
||||
if (cinputs.size() != 4) return -1;
|
||||
num_heads = extractConstInt(prog, cinputs[2]);
|
||||
if (num_heads <= 0) return -1;
|
||||
if (extra_ops_to_remove)
|
||||
collectShapeChain(prog, concat_idx, *extra_ops_to_remove);
|
||||
}
|
||||
*out_num_heads = num_heads;
|
||||
*out_reshape_idx = reshape_idx;
|
||||
|
||||
Arg reshape_out = prog[reshape_idx]->outputs[0];
|
||||
int transpose_idx = singleConsumer(reshape_out);
|
||||
if (!isTranspose(prog, transpose_idx)) return -1;
|
||||
TransposeLayer* tr = dynamic_cast<TransposeLayer*>(prog[transpose_idx].get());
|
||||
if (!tr) return -1;
|
||||
*out_perm = tr->perm;
|
||||
return transpose_idx;
|
||||
}
|
||||
|
||||
bool fuseGraph(Ptr<Graph>& graph)
|
||||
{
|
||||
const vector<Ptr<Layer>>& prog = graph->prog();
|
||||
size_t nops = prog.size();
|
||||
|
||||
producer_.clear();
|
||||
consumers_.clear();
|
||||
for (size_t i = 0; i < nops; i++) {
|
||||
if (!prog[i]) continue;
|
||||
for (Arg out : prog[i]->outputs)
|
||||
producer_[out.idx] = (int)i;
|
||||
for (Arg inp : prog[i]->inputs)
|
||||
consumers_[inp.idx].push_back((int)i);
|
||||
}
|
||||
|
||||
std::map<int, vector<int>> qkv_candidates;
|
||||
for (size_t i = 0; i < nops; i++) {
|
||||
if (!prog[i]) continue;
|
||||
MatMulLayer* mm = dynamic_cast<MatMulLayer*>(prog[i].get());
|
||||
if (!mm) continue;
|
||||
if (prog[i]->blobs.empty() || prog[i]->inputs.size() != 1)
|
||||
continue;
|
||||
Arg inp = prog[i]->inputs[0];
|
||||
qkv_candidates[inp.idx].push_back((int)i);
|
||||
}
|
||||
|
||||
bool modified = false;
|
||||
std::set<int> removed_ops;
|
||||
|
||||
for (auto& [inp_idx, matmul_indices] : qkv_candidates) {
|
||||
if (matmul_indices.size() < 3) continue;
|
||||
|
||||
for (size_t qi = 0; qi + 2 < matmul_indices.size(); qi++)
|
||||
for (size_t ki = qi + 1; ki + 1 < matmul_indices.size(); ki++)
|
||||
for (size_t vi = ki + 1; vi < matmul_indices.size(); vi++)
|
||||
{
|
||||
int indices[3] = { matmul_indices[qi], matmul_indices[ki], matmul_indices[vi] };
|
||||
|
||||
bool any_removed = false;
|
||||
for (int idx : indices)
|
||||
if (removed_ops.count(idx)) any_removed = true;
|
||||
if (any_removed) continue;
|
||||
|
||||
bool shapes_ok = true;
|
||||
for (int k = 0; k < 3; k++) {
|
||||
const Mat& w = prog[indices[k]]->blobs[0];
|
||||
if (w.dims != 2) { shapes_ok = false; break; }
|
||||
}
|
||||
if (!shapes_ok) continue;
|
||||
|
||||
int reshape_idx[3], num_heads[3], transpose_idx[3];
|
||||
vector<int> perms[3];
|
||||
std::set<int> extra_ops;
|
||||
bool pattern_ok = true;
|
||||
for (int k = 0; k < 3; k++) {
|
||||
transpose_idx[k] = followProjChain(prog, indices[k],
|
||||
&reshape_idx[k],
|
||||
&num_heads[k], &perms[k],
|
||||
&extra_ops);
|
||||
if (transpose_idx[k] < 0) { pattern_ok = false; break; }
|
||||
}
|
||||
if (!pattern_ok) continue;
|
||||
if (num_heads[0] != num_heads[1] || num_heads[1] != num_heads[2])
|
||||
continue;
|
||||
|
||||
// K is identified by perm [0,2,3,1] (transposes head_dim to second-last);
|
||||
// Q and V use perm [0,2,1,3].
|
||||
int k_slot = -1;
|
||||
vector<int> perm_k = {0, 2, 3, 1};
|
||||
for (int k = 0; k < 3; k++)
|
||||
if (perms[k] == perm_k) k_slot = k;
|
||||
if (k_slot < 0) continue;
|
||||
|
||||
int remaining[2];
|
||||
{ int j = 0; for (int k = 0; k < 3; k++) if (k != k_slot) remaining[j++] = k; }
|
||||
|
||||
Arg k_tr_out = prog[transpose_idx[k_slot]]->outputs[0];
|
||||
int k_next = singleConsumer(k_tr_out);
|
||||
int k_mul_idx = -1;
|
||||
float k_scale_val = 1.f;
|
||||
int qk_matmul_idx = -1;
|
||||
bool vit_style;
|
||||
if (isScalarMul(prog, k_next, &k_scale_val)) {
|
||||
vit_style = true;
|
||||
k_mul_idx = k_next;
|
||||
qk_matmul_idx = singleConsumer(prog[k_mul_idx]->outputs[0]);
|
||||
} else if (isMatMul(prog, k_next)) {
|
||||
vit_style = false;
|
||||
qk_matmul_idx = k_next;
|
||||
} else {
|
||||
continue;
|
||||
}
|
||||
if (!isMatMul(prog, qk_matmul_idx)) continue;
|
||||
|
||||
int q_slot = -1, v_slot = -1;
|
||||
for (int attempt = 0; attempt < 2; attempt++) {
|
||||
q_slot = remaining[attempt];
|
||||
v_slot = remaining[1 - attempt];
|
||||
|
||||
int q_mul_idx = -1;
|
||||
float q_scale_val = 1.f;
|
||||
Arg q_tr_out = prog[transpose_idx[q_slot]]->outputs[0];
|
||||
|
||||
if (vit_style) {
|
||||
int q_next = singleConsumer(q_tr_out);
|
||||
if (!isScalarMul(prog, q_next, &q_scale_val)) continue;
|
||||
q_mul_idx = q_next;
|
||||
Arg q_mul_out = prog[q_mul_idx]->outputs[0];
|
||||
int q_dest = singleConsumer(q_mul_out);
|
||||
if (q_dest != qk_matmul_idx) continue;
|
||||
} else {
|
||||
const auto& qk_inputs = prog[qk_matmul_idx]->inputs;
|
||||
bool q_connected = false;
|
||||
for (auto& ai : qk_inputs)
|
||||
if (ai.idx == q_tr_out.idx) q_connected = true;
|
||||
if (!q_connected) continue;
|
||||
}
|
||||
|
||||
// qk_matmul → [Div(√d_k)] → [Add(mask)] → Softmax (BERT)
|
||||
// OR → Softmax (ViT)
|
||||
Arg cur_out = prog[qk_matmul_idx]->outputs[0];
|
||||
int qk_div_idx = -1;
|
||||
int qk_add_idx = -1;
|
||||
Arg mask_arg;
|
||||
bool has_mask = false;
|
||||
float post_scale_val = 1.f;
|
||||
|
||||
int cur_consumer = singleConsumer(cur_out);
|
||||
if (!vit_style) {
|
||||
if (!isScalarDiv(prog, cur_consumer, &post_scale_val)) continue;
|
||||
qk_div_idx = cur_consumer;
|
||||
cur_out = prog[qk_div_idx]->outputs[0];
|
||||
cur_consumer = singleConsumer(cur_out);
|
||||
Arg maybe_mask;
|
||||
if (isMaskAdd(prog, cur_consumer, &maybe_mask)) {
|
||||
qk_add_idx = cur_consumer;
|
||||
mask_arg = maybe_mask;
|
||||
has_mask = true;
|
||||
cur_out = prog[qk_add_idx]->outputs[0];
|
||||
cur_consumer = singleConsumer(cur_out);
|
||||
}
|
||||
}
|
||||
int softmax_idx = cur_consumer;
|
||||
if (!isSoftmax(prog, softmax_idx)) continue;
|
||||
|
||||
Arg softmax_out = prog[softmax_idx]->outputs[0];
|
||||
int attnv_matmul_idx = singleConsumer(softmax_out);
|
||||
if (!isMatMul(prog, attnv_matmul_idx)) continue;
|
||||
|
||||
Arg v_tr_out = prog[transpose_idx[v_slot]]->outputs[0];
|
||||
const auto& av_inputs = prog[attnv_matmul_idx]->inputs;
|
||||
bool v_connected = false;
|
||||
for (auto& ai : av_inputs)
|
||||
if (ai.idx == v_tr_out.idx) v_connected = true;
|
||||
if (!v_connected) continue;
|
||||
|
||||
Arg attnv_out = prog[attnv_matmul_idx]->outputs[0];
|
||||
int out_transpose_idx = singleConsumer(attnv_out);
|
||||
if (!isTranspose(prog, out_transpose_idx)) continue;
|
||||
|
||||
Arg out_tr_out = prog[out_transpose_idx]->outputs[0];
|
||||
int out_reshape_idx = findMatchingConsumer(prog, out_tr_out,
|
||||
[](Layer* L){ return dynamic_cast<Reshape2Layer*>(L) != nullptr; },
|
||||
&extra_ops);
|
||||
if (!isReshape(prog, out_reshape_idx)) continue;
|
||||
|
||||
const auto& or_inputs = prog[out_reshape_idx]->inputs;
|
||||
if (or_inputs.size() >= 2 && !netimpl->isConstArg(or_inputs[1])) {
|
||||
auto it = producer_.find(or_inputs[1].idx);
|
||||
if (it != producer_.end())
|
||||
collectShapeChain(prog, it->second, extra_ops);
|
||||
}
|
||||
|
||||
const Mat& Wq = prog[indices[q_slot]]->blobs[0];
|
||||
const Mat& Wk = prog[indices[k_slot]]->blobs[0];
|
||||
const Mat& Wv = prog[indices[v_slot]]->blobs[0];
|
||||
int input_hidden = Wq.size[0];
|
||||
int q_hidden = Wq.size[1];
|
||||
int k_hidden = Wk.size[1];
|
||||
int v_hidden = Wv.size[1];
|
||||
int total_hidden = q_hidden + k_hidden + v_hidden;
|
||||
|
||||
int wshape[] = {input_hidden, total_hidden};
|
||||
Mat W_qkv(2, wshape, CV_32F);
|
||||
for (int r = 0; r < input_hidden; r++) {
|
||||
float* dst = W_qkv.ptr<float>(r);
|
||||
memcpy(dst, Wq.ptr<float>(r), q_hidden * sizeof(float));
|
||||
memcpy(dst + q_hidden, Wk.ptr<float>(r), k_hidden * sizeof(float));
|
||||
memcpy(dst + q_hidden + k_hidden, Wv.ptr<float>(r), v_hidden * sizeof(float));
|
||||
}
|
||||
|
||||
Mat bias_qkv;
|
||||
bool has_bias = prog[indices[q_slot]]->blobs.size() >= 2;
|
||||
if (has_bias) {
|
||||
const Mat& bq = prog[indices[q_slot]]->blobs[1];
|
||||
const Mat& bk = prog[indices[k_slot]]->blobs[1];
|
||||
const Mat& bv = prog[indices[v_slot]]->blobs[1];
|
||||
int bias_total = (int)(bq.total() + bk.total() + bv.total());
|
||||
bias_qkv.create(1, &bias_total, CV_32F);
|
||||
float* dst = bias_qkv.ptr<float>();
|
||||
memcpy(dst, bq.ptr<float>(), bq.total() * sizeof(float));
|
||||
memcpy(dst + bq.total(), bk.ptr<float>(), bk.total() * sizeof(float));
|
||||
memcpy(dst + bq.total() + bk.total(), bv.ptr<float>(), bv.total() * sizeof(float));
|
||||
}
|
||||
|
||||
float param_scale = vit_style ? (1.f / (q_scale_val * k_scale_val))
|
||||
: post_scale_val;
|
||||
|
||||
LayerParams attn_params;
|
||||
attn_params.name = prog[indices[q_slot]]->name + "_fused_attention";
|
||||
attn_params.type = "Attention";
|
||||
attn_params.set("num_heads", num_heads[0]);
|
||||
DictValue qkv_sizes_param = DictValue::arrayInt(
|
||||
std::vector<int>{q_hidden, k_hidden, v_hidden}.data(), 3);
|
||||
attn_params.set("qkv_hidden_sizes", qkv_sizes_param);
|
||||
attn_params.set("scale", param_scale);
|
||||
attn_params.set("output_ndims", 3);
|
||||
|
||||
attn_params.blobs.push_back(W_qkv);
|
||||
if (has_bias)
|
||||
attn_params.blobs.push_back(bias_qkv);
|
||||
|
||||
Ptr<Layer> attn_layer = LayerFactory::createLayerInstance(
|
||||
attn_params.type, attn_params);
|
||||
CV_Assert(attn_layer);
|
||||
|
||||
Arg shared_input = prog[indices[0]]->inputs[0];
|
||||
if (has_mask)
|
||||
attn_layer->inputs = { shared_input, mask_arg };
|
||||
else
|
||||
attn_layer->inputs = { shared_input };
|
||||
attn_layer->outputs = prog[out_reshape_idx]->outputs;
|
||||
attn_layer->netimpl = netimpl;
|
||||
|
||||
std::set<int> to_remove = {
|
||||
indices[0], indices[1], indices[2],
|
||||
reshape_idx[0], reshape_idx[1], reshape_idx[2],
|
||||
transpose_idx[0], transpose_idx[1], transpose_idx[2],
|
||||
qk_matmul_idx, softmax_idx, attnv_matmul_idx,
|
||||
out_transpose_idx, out_reshape_idx
|
||||
};
|
||||
if (k_mul_idx >= 0) to_remove.insert(k_mul_idx);
|
||||
if (q_mul_idx >= 0) to_remove.insert(q_mul_idx);
|
||||
if (qk_div_idx >= 0) to_remove.insert(qk_div_idx);
|
||||
if (qk_add_idx >= 0) to_remove.insert(qk_add_idx);
|
||||
for (int op : extra_ops) to_remove.insert(op);
|
||||
for (int op : to_remove)
|
||||
removed_ops.insert(op);
|
||||
|
||||
int insert_pos = *std::min_element(to_remove.begin(), to_remove.end());
|
||||
attention_replacements_.push_back({insert_pos, attn_layer});
|
||||
modified = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (modified) {
|
||||
vector<Ptr<Layer>> newprog;
|
||||
std::sort(attention_replacements_.begin(), attention_replacements_.end(),
|
||||
[](auto& a, auto& b) { return a.first < b.first; });
|
||||
|
||||
size_t repl_idx = 0;
|
||||
for (size_t i = 0; i < nops; i++) {
|
||||
while (repl_idx < attention_replacements_.size() &&
|
||||
attention_replacements_[repl_idx].first == (int)i) {
|
||||
newprog.push_back(attention_replacements_[repl_idx].second);
|
||||
repl_idx++;
|
||||
}
|
||||
if (removed_ops.count((int)i))
|
||||
continue;
|
||||
newprog.push_back(prog[i]);
|
||||
}
|
||||
graph->setProg(newprog);
|
||||
}
|
||||
|
||||
return modified;
|
||||
}
|
||||
|
||||
Net::Impl* netimpl;
|
||||
|
||||
private:
|
||||
std::map<int, int> producer_;
|
||||
std::map<int, vector<int>> consumers_;
|
||||
vector<std::pair<int, Ptr<Layer>>> attention_replacements_;
|
||||
};
|
||||
|
||||
void Net::Impl::fuseAttention()
|
||||
{
|
||||
ModelFusionAttention attnFusion(this);
|
||||
attnFusion.fuse();
|
||||
}
|
||||
|
||||
CV__DNN_INLINE_NS_END
|
||||
}}
|
||||
@@ -165,7 +165,8 @@ class AttentionLayerImpl CV_FINAL : public AttentionLayer {
|
||||
std::vector<MatShape> &outputs,
|
||||
std::vector<MatShape> &internals) const CV_OVERRIDE {
|
||||
int num_inputs = inputs.size() + blobs.size();
|
||||
CV_CheckEQ(num_inputs, 3, "DNN/Attention: three inputs are required");
|
||||
CV_CheckGE(num_inputs, 3, "DNN/Attention: at least three inputs are required (data, weight, bias)");
|
||||
CV_CheckLE(num_inputs, 4, "DNN/Attention: at most four inputs are supported (data, weight, bias, mask)");
|
||||
const auto &input_shape = inputs[0];
|
||||
const auto &weight_shape = blobs.empty() ? inputs[1] : shape(blobs.front());
|
||||
const auto &bias_shape = blobs.empty() ? inputs[2] : shape(blobs.back());
|
||||
@@ -392,6 +393,41 @@ class AttentionLayerImpl CV_FINAL : public AttentionLayer {
|
||||
}
|
||||
}, loops * seq_len * qk_head_size * seq_len * (1 / 1024.0));
|
||||
|
||||
// Additive attention mask applied before softmax. Supports BERT-style mask
|
||||
// shapes [B, 1, 1, S] (per-batch) and [1, 1, 1, S] (global broadcast).
|
||||
// Only reached when the fusion pass attached an external mask input.
|
||||
int num_non_blob_inputs = (int)inputs.size();
|
||||
bool has_mask = (!blobs.empty() && num_non_blob_inputs >= 2) ||
|
||||
(blobs.empty() && num_non_blob_inputs >= 4);
|
||||
if (has_mask) {
|
||||
const Mat &mask_mat = blobs.empty() ? inputs[3] : inputs[1];
|
||||
CV_CheckTypeEQ(mask_mat.type(), CV_32F,
|
||||
"DNN/Attention: mask must be float32");
|
||||
const int mask_last_dim = shape(mask_mat).back();
|
||||
CV_CheckEQ(mask_last_dim, (int)seq_len,
|
||||
"DNN/Attention: mask last dim must equal seq_len");
|
||||
const size_t mask_total = mask_mat.total();
|
||||
int mask_batch_stride;
|
||||
if (mask_total == batch_size * seq_len) mask_batch_stride = (int)seq_len;
|
||||
else if (mask_total == seq_len) mask_batch_stride = 0;
|
||||
else CV_Error(Error::StsNotImplemented,
|
||||
"DNN/Attention: unsupported mask shape");
|
||||
|
||||
const float *mask_data = mask_mat.ptr<const float>();
|
||||
parallel_for_(Range(0, (int)loops), [&](const Range &r) {
|
||||
for (int i = r.start; i < r.end; i++) {
|
||||
const int b = i / (int)num_heads;
|
||||
const float *m = mask_data + b * mask_batch_stride;
|
||||
float *prob = output + i * seq_len_square;
|
||||
for (size_t row = 0; row < seq_len; row++) {
|
||||
float *p = prob + row * seq_len;
|
||||
for (size_t j = 0; j < seq_len; j++)
|
||||
p[j] += m[j];
|
||||
}
|
||||
}
|
||||
}, loops * seq_len * (1 / 1024.0));
|
||||
}
|
||||
|
||||
// Compute softmax on the last dimension
|
||||
softmax(attention_prob, attention_prob, shape(attention_prob).size() - 1);
|
||||
}
|
||||
|
||||
@@ -233,33 +233,208 @@ void fastGemmPackB(bool trans, size_t N, size_t K, const float *B, size_t ldb, f
|
||||
}
|
||||
}
|
||||
|
||||
static constexpr int FAST_GEMM_THIN_MAX_M = 12;
|
||||
#if !CV_SIMD
|
||||
// Scalar-fallback strip width (unroll factor, not SIMD lanes); 4 matches the narrowest universal-intrinsic float width so layout stays consistent with SIMD builds.
|
||||
static constexpr int FAST_GEMM_THIN_SCALAR_NR = 4;
|
||||
#endif
|
||||
|
||||
static inline int fast_gemm_thin_lanes() {
|
||||
#if (CV_SIMD || CV_SIMD_SCALABLE)
|
||||
return VTraits<v_float32>::vlanes();
|
||||
#else
|
||||
return FAST_GEMM_THIN_SCALAR_NR;
|
||||
#endif
|
||||
}
|
||||
|
||||
static inline void fast_gemm_thin_strip(int M, int K, float alpha,
|
||||
const float* A, int lda0, int lda1,
|
||||
const float* b_strip,
|
||||
float beta, float* c_strip, int ldc) {
|
||||
#if CV_SIMD
|
||||
const int NR = VTraits<v_float32>::vlanes();
|
||||
v_float32 acc[FAST_GEMM_THIN_MAX_M];
|
||||
for (int m = 0; m < M; m++) acc[m] = vx_setzero_f32();
|
||||
|
||||
for (int k = 0; k < K; k++) {
|
||||
v_float32 bv = vx_load(b_strip + k * NR);
|
||||
for (int m = 0; m < M; m++) {
|
||||
v_float32 am = vx_setall_f32(A[m * lda0 + k * lda1]);
|
||||
acc[m] = v_fma(bv, am, acc[m]);
|
||||
}
|
||||
}
|
||||
|
||||
const v_float32 v_alpha = vx_setall_f32(alpha);
|
||||
if (beta == 0.f) {
|
||||
for (int m = 0; m < M; m++)
|
||||
vx_store(c_strip + m * ldc, v_mul(acc[m], v_alpha));
|
||||
} else if (beta == 1.f) {
|
||||
for (int m = 0; m < M; m++) {
|
||||
v_float32 cv = vx_load(c_strip + m * ldc);
|
||||
vx_store(c_strip + m * ldc, v_fma(acc[m], v_alpha, cv));
|
||||
}
|
||||
} else {
|
||||
const v_float32 v_beta = vx_setall_f32(beta);
|
||||
for (int m = 0; m < M; m++) {
|
||||
v_float32 cv = vx_load(c_strip + m * ldc);
|
||||
cv = v_mul(cv, v_beta);
|
||||
vx_store(c_strip + m * ldc, v_fma(acc[m], v_alpha, cv));
|
||||
}
|
||||
}
|
||||
#elif CV_SIMD_SCALABLE
|
||||
// Scalable vector types (e.g. RVV) are sizeless and cannot form arrays;
|
||||
// back the per-row accumulators with a scalar scratch buffer.
|
||||
const int NR = VTraits<v_float32>::vlanes();
|
||||
float acc_buf[FAST_GEMM_THIN_MAX_M * VTraits<v_float32>::max_nlanes];
|
||||
for (int m = 0; m < M; m++) vx_store(acc_buf + m * NR, vx_setzero_f32());
|
||||
|
||||
for (int k = 0; k < K; k++) {
|
||||
v_float32 bv = vx_load(b_strip + k * NR);
|
||||
for (int m = 0; m < M; m++) {
|
||||
v_float32 am = vx_setall_f32(A[m * lda0 + k * lda1]);
|
||||
v_float32 acc_m = vx_load(acc_buf + m * NR);
|
||||
vx_store(acc_buf + m * NR, v_fma(bv, am, acc_m));
|
||||
}
|
||||
}
|
||||
|
||||
const v_float32 v_alpha = vx_setall_f32(alpha);
|
||||
if (beta == 0.f) {
|
||||
for (int m = 0; m < M; m++)
|
||||
vx_store(c_strip + m * ldc, v_mul(vx_load(acc_buf + m * NR), v_alpha));
|
||||
} else if (beta == 1.f) {
|
||||
for (int m = 0; m < M; m++) {
|
||||
v_float32 cv = vx_load(c_strip + m * ldc);
|
||||
vx_store(c_strip + m * ldc, v_fma(vx_load(acc_buf + m * NR), v_alpha, cv));
|
||||
}
|
||||
} else {
|
||||
const v_float32 v_beta = vx_setall_f32(beta);
|
||||
for (int m = 0; m < M; m++) {
|
||||
v_float32 cv = vx_load(c_strip + m * ldc);
|
||||
cv = v_mul(cv, v_beta);
|
||||
vx_store(c_strip + m * ldc, v_fma(vx_load(acc_buf + m * NR), v_alpha, cv));
|
||||
}
|
||||
}
|
||||
#else
|
||||
const int NR = FAST_GEMM_THIN_SCALAR_NR;
|
||||
float acc[FAST_GEMM_THIN_MAX_M * FAST_GEMM_THIN_SCALAR_NR] = {0};
|
||||
for (int k = 0; k < K; k++) {
|
||||
for (int m = 0; m < M; m++) {
|
||||
float a_mk = A[m * lda0 + k * lda1];
|
||||
for (int c = 0; c < NR; c++)
|
||||
acc[m * NR + c] += a_mk * b_strip[k * NR + c];
|
||||
}
|
||||
}
|
||||
if (beta == 0.f) {
|
||||
for (int m = 0; m < M; m++)
|
||||
for (int c = 0; c < NR; c++)
|
||||
c_strip[m * ldc + c] = alpha * acc[m * NR + c];
|
||||
} else {
|
||||
for (int m = 0; m < M; m++)
|
||||
for (int c = 0; c < NR; c++)
|
||||
c_strip[m * ldc + c] = beta * c_strip[m * ldc + c] + alpha * acc[m * NR + c];
|
||||
}
|
||||
#endif
|
||||
}
|
||||
|
||||
bool fastGemmThinEligible(int M, int N, int K) {
|
||||
if (M <= 0 || N <= 0 || K <= 0) return false;
|
||||
if (M > FAST_GEMM_THIN_MAX_M) return false;
|
||||
const int NR = fast_gemm_thin_lanes();
|
||||
if (N < 2 * NR) return false;
|
||||
return true;
|
||||
}
|
||||
|
||||
size_t fastGemmThinPackBSize(int N, int K) {
|
||||
const int NR = fast_gemm_thin_lanes();
|
||||
const int n_strips = (N + NR - 1) / NR;
|
||||
return (size_t)n_strips * (size_t)NR * (size_t)K;
|
||||
}
|
||||
|
||||
void fastGemmThinPackB(int N, int K, const float* B, size_t ldb_K, size_t ldb_N, float* packed_B) {
|
||||
const int NR = fast_gemm_thin_lanes();
|
||||
const int n_strips = (N + NR - 1) / NR;
|
||||
for (int s = 0; s < n_strips; s++) {
|
||||
const int j0 = s * NR;
|
||||
const int nr = std::min(NR, N - j0);
|
||||
float* strip = packed_B + (size_t)s * NR * K;
|
||||
for (int k = 0; k < K; k++) {
|
||||
float* out = strip + k * NR;
|
||||
int c = 0;
|
||||
for (; c < nr; c++) out[c] = B[k * ldb_K + (j0 + c) * ldb_N];
|
||||
for (; c < NR; c++) out[c] = 0.f;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void fastGemmThin(int M, int N, int K, float alpha,
|
||||
const float* A, int lda0, int lda1,
|
||||
const float* packed_B, float beta,
|
||||
float* C, int ldc, bool multi_thread) {
|
||||
const int NR = fast_gemm_thin_lanes();
|
||||
const int n_full_strips = N / NR;
|
||||
const int n_tail = N - n_full_strips * NR;
|
||||
|
||||
auto fn = [&](const Range& r) {
|
||||
for (int s = r.start; s < r.end; s++) {
|
||||
const float* b_strip = packed_B + (size_t)s * NR * K;
|
||||
float* c_strip = C + s * NR;
|
||||
fast_gemm_thin_strip(M, K, alpha, A, lda0, lda1, b_strip, beta, c_strip, ldc);
|
||||
}
|
||||
};
|
||||
if (multi_thread && n_full_strips > 1) {
|
||||
parallel_for_(Range(0, n_full_strips), fn, (double)n_full_strips * M * K * NR * (1.0 / 1024.0));
|
||||
} else {
|
||||
fn(Range(0, n_full_strips));
|
||||
}
|
||||
|
||||
if (n_tail > 0) {
|
||||
const int j0 = n_full_strips * NR;
|
||||
const float* b_strip = packed_B + (size_t)n_full_strips * NR * K;
|
||||
for (int m = 0; m < M; m++) {
|
||||
for (int c = 0; c < n_tail; c++) {
|
||||
float acc_s = 0.f;
|
||||
for (int k = 0; k < K; k++)
|
||||
acc_s += A[m * lda0 + k * lda1] * b_strip[k * NR + c];
|
||||
const float mul = alpha * acc_s;
|
||||
float* p = C + m * ldc + j0 + c;
|
||||
*p = (beta == 0.f) ? mul : beta * (*p) + mul;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
static void fast_gemm_thin(float alpha, float beta, int M, int N, int K,
|
||||
const char *a_, int lda0, int lda1,
|
||||
const char *b_, int ldb,
|
||||
char *c_, int ldc, bool multi_thread) {
|
||||
const float* a = (const float*)a_;
|
||||
const float* A = (const float*)a_;
|
||||
const float* B = (const float*)b_;
|
||||
float* C = (float*)c_;
|
||||
|
||||
if (fastGemmThinEligible(M, N, K)) {
|
||||
AutoBuffer<float> packed(fastGemmThinPackBSize(N, K));
|
||||
fastGemmThinPackB(N, K, B, (size_t)ldb, 1, packed.data());
|
||||
fastGemmThin(M, N, K, alpha, A, lda0, lda1, packed.data(), beta, C, ldc, multi_thread);
|
||||
return;
|
||||
}
|
||||
|
||||
auto fn = [&](const Range &r) {
|
||||
for(int start = r.start ; start < r.end; start++ ) {
|
||||
float* c_i = (float*)c_ + start * ldc;
|
||||
for (int i = r.start; i < r.end; i++) {
|
||||
float* c_i = C + i * ldc;
|
||||
if (beta == 0.f)
|
||||
for(int j = 0; j < N; j++ ) c_i[j] = 0.f;
|
||||
for (int j = 0; j < N; j++) c_i[j] = 0.f;
|
||||
else if (beta != 1.f)
|
||||
for(int j = 0; j < N; j++ ) c_i[j] *= beta;
|
||||
for(int k = 0; k < K; k++ ) {
|
||||
const float* b_k = (const float*)b_ + k * ldb;
|
||||
float aval = alpha * a[start * lda0 + k * lda1];
|
||||
for(int j = 0; j < N; j++ )
|
||||
for (int j = 0; j < N; j++) c_i[j] *= beta;
|
||||
for (int k = 0; k < K; k++) {
|
||||
const float* b_k = B + k * ldb;
|
||||
float aval = alpha * A[i * lda0 + k * lda1];
|
||||
for (int j = 0; j < N; j++)
|
||||
c_i[j] += aval * b_k[j];
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
if (multi_thread) {
|
||||
int total = M; // outer loops
|
||||
int cost_per_thread = static_cast<int>(K * N); // inner loops
|
||||
double nstripes = (size_t)total * cost_per_thread * (1 / 1024.0);
|
||||
parallel_for_(Range(0, total), fn, nstripes);
|
||||
parallel_for_(Range(0, M), fn, (double)M * K * N * (1.0 / 1024.0));
|
||||
} else {
|
||||
fn(Range(0, M));
|
||||
}
|
||||
@@ -321,7 +496,7 @@ void fastGemm(bool trans_a, bool trans_b, int ma, int na, int mb, int nb,
|
||||
std::swap(ldb0, ldb1);
|
||||
}
|
||||
|
||||
if (!trans_b && ldb1 == 1 && (M <= 4 || (uint64_t)M * N * K <= 10000)) {
|
||||
if (!trans_b && ldb1 == 1 && (fastGemmThinEligible(M, N, K) || (uint64_t)M * N * K <= 10000)) {
|
||||
return fast_gemm_thin(alpha, beta, M, N, K, a, lda0, lda1, b, ldb0, c, ldc, opt.multi_thread);
|
||||
}
|
||||
|
||||
@@ -395,6 +570,18 @@ void fastGemmBatch(size_t batch, const size_t *A_offsets, const size_t *B_offset
|
||||
const char *b = (const char *)B;
|
||||
char *c = (char *)C;
|
||||
|
||||
// Below ~10K MACs (M*N*K) the blocked SIMD kernel's pack/tile overhead outweighs its speedup, so route tiny problems through the unblocked thin path.
|
||||
if (ldb1 == 1 && (fastGemmThinEligible(M, N, K) || (uint64_t)M * N * K <= 10000)) {
|
||||
const size_t esz = sizeof(float);
|
||||
for (size_t i = 0; i < batch; i++) {
|
||||
fast_gemm_thin(alpha, beta, M, N, K,
|
||||
a + A_offsets[i] * esz, lda0, lda1,
|
||||
b + B_offsets[i] * esz, ldb0,
|
||||
c + C_offsets[i] * esz, ldc, opt.multi_thread);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
#if CV_TRY_NEON
|
||||
if (opt.use_neon) {
|
||||
opt_NEON::fastGemmBatchKernel(batch, A_offsets, B_offsets, C_offsets, M, N, K, alpha, a, lda0, lda1, b, ldb0, ldb1, beta, c, ldc, sizeof(float));
|
||||
@@ -486,6 +673,17 @@ void fastGemmBatch(size_t batch,
|
||||
const char *b = (const char *)B.ptr<const float>();
|
||||
char *c = (char *)C.ptr<float>();
|
||||
|
||||
if (ldb1 == 1 && (fastGemmThinEligible(M, N, K) || (uint64_t)M * N * K <= 10000)) {
|
||||
const size_t esz = sizeof(float);
|
||||
for (size_t i = 0; i < batch; i++) {
|
||||
fast_gemm_thin(alpha, beta, M, N, K,
|
||||
a + A_offsets[i] * esz, lda0, lda1,
|
||||
b + B_offsets[i] * esz, ldb0,
|
||||
c + C_offsets[i] * esz, ldc, opt.multi_thread);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
#if CV_TRY_NEON
|
||||
if (opt.use_neon) {
|
||||
opt_NEON::fastGemmBatchKernel(batch, A_offsets.data(), B_offsets.data(), C_offsets.data(), M, N, K, alpha, a, lda0, lda1, b, ldb0, ldb1, beta, c, ldc, sizeof(float));
|
||||
|
||||
@@ -186,6 +186,14 @@ void fastGemmBatch(size_t batch,
|
||||
int M, int N, int K, float alpha, const Mat&A, int lda0, int lda1,
|
||||
const Mat&B, int ldb0, int ldb1, float beta, Mat&C, int ldc, FastGemmOpt &opt);
|
||||
|
||||
bool fastGemmThinEligible(int M, int N, int K);
|
||||
size_t fastGemmThinPackBSize(int N, int K);
|
||||
void fastGemmThinPackB(int N, int K, const float* B, size_t ldb_K, size_t ldb_N, float* packed_B);
|
||||
void fastGemmThin(int M, int N, int K, float alpha,
|
||||
const float* A, int lda0, int lda1,
|
||||
const float* packed_B, float beta,
|
||||
float* C, int ldc, bool multi_thread);
|
||||
|
||||
void pagedAttnQKGemm(
|
||||
const Mat& Q, const std::vector<Mat> &K, Mat& A,
|
||||
int T_q, int Nq, int N_k, int T_s, int D,
|
||||
|
||||
@@ -156,6 +156,17 @@ class MatMulLayerImpl CV_FINAL : public MatMulLayer {
|
||||
if (!blobs.empty()) {
|
||||
fastGemmPackB(blobs[0], packed_input_B, trans_b, opt);
|
||||
helper.updatePackedBOffsets(packed_input_B.size());
|
||||
|
||||
if (helper.batch == 1 && blobs[0].type() == CV_32F &&
|
||||
fastGemmThinEligible(helper.M, helper.N, helper.K)) {
|
||||
thin_packed_B.resize(fastGemmThinPackBSize(helper.N, helper.K));
|
||||
fastGemmThinPackB(helper.N, helper.K,
|
||||
blobs[0].ptr<const float>(),
|
||||
(size_t)helper.ldb0, (size_t)helper.ldb1,
|
||||
thin_packed_B.data());
|
||||
} else {
|
||||
thin_packed_B.clear();
|
||||
}
|
||||
}
|
||||
|
||||
// broadcast bias if needed
|
||||
@@ -259,6 +270,11 @@ class MatMulLayerImpl CV_FINAL : public MatMulLayer {
|
||||
fastGemmBatch(helper.batch, helper.A_offsets.data(), helper.B_offsets.data(), helper.C_offsets.data(),
|
||||
helper.M, helper.N, helper.K, alpha, a, helper.lda0, helper.lda1,
|
||||
b, helper.ldb0, helper.ldb1, beta, y, helper.ldc, opt);
|
||||
} else if (!thin_packed_B.empty()) {
|
||||
fastGemmThin(helper.M, helper.N, helper.K, alpha,
|
||||
a, helper.lda0, helper.lda1,
|
||||
thin_packed_B.data(), beta,
|
||||
y, helper.ldc, opt.multi_thread);
|
||||
} else {
|
||||
fastGemmBatch(helper.batch, helper.A_offsets.data(), helper.packed_B_offsets.data(), helper.C_offsets.data(),
|
||||
helper.M, helper.N, helper.K, alpha, a, helper.lda0, helper.lda1,
|
||||
@@ -495,6 +511,7 @@ class MatMulLayerImpl CV_FINAL : public MatMulLayer {
|
||||
int real_ndims_C;
|
||||
|
||||
std::vector<float> packed_input_B;
|
||||
std::vector<float> thin_packed_B;
|
||||
Mat broadcast_bias;
|
||||
|
||||
FastGemmOpt opt;
|
||||
|
||||
@@ -454,6 +454,8 @@ struct Net::Impl : public detail::NetImplBase
|
||||
void assignBuffers();
|
||||
// fuse batch norm, add bias and activation to convolution
|
||||
void fuseBasic();
|
||||
// fuse ViT-style multi-head attention subgraphs
|
||||
void fuseAttention();
|
||||
// replace constant sub-expressions with their results
|
||||
|
||||
void fuseQDQ();
|
||||
|
||||
@@ -548,6 +548,7 @@ void Net::Impl::prepareForInference()
|
||||
fuseQDQ();
|
||||
constFold();
|
||||
constArgs();
|
||||
fuseAttention();
|
||||
useBlockLayout();
|
||||
fuseBasic();
|
||||
assignBuffers();
|
||||
|
||||
@@ -845,12 +845,28 @@ public:
|
||||
std::vector<int64_t> axes = extractAxis(net, matchedNodesIds[mean]);
|
||||
// check whether it is -1 or last_axis or [axis, ..., last_axis]
|
||||
int64_t input_ndims = static_cast<int64_t>(net.dynamicCast<ONNXGraphWrapper>()->getTensorShapeSize(matchedNodesIds[mean], 0));
|
||||
if (input_ndims == -1) {
|
||||
return false; // input shape unknown
|
||||
|
||||
// When axes are all negative (e.g. [-1] or [-2, -1]), we can validate
|
||||
// the pattern without knowing input_ndims.
|
||||
bool all_axes_negative = !axes.empty();
|
||||
for (size_t i = 0; i < axes.size(); i++) {
|
||||
if (axes[i] >= 0) { all_axes_negative = false; break; }
|
||||
}
|
||||
// assume that axes are sorted in ascending order, e.g. [0, 1, 2, 3] or [-3, -2, -1]
|
||||
if (axes.back() != -1 && axes.back() != (input_ndims - 1)) {
|
||||
return false;
|
||||
|
||||
if (input_ndims == -1 && !all_axes_negative) {
|
||||
return false; // input shape unknown and axes are positive
|
||||
}
|
||||
|
||||
if (input_ndims != -1) {
|
||||
// assume that axes are sorted in ascending order, e.g. [0, 1, 2, 3] or [-3, -2, -1]
|
||||
if (axes.back() != -1 && axes.back() != (input_ndims - 1)) {
|
||||
return false;
|
||||
}
|
||||
} else {
|
||||
// axes are all negative; check that the last axis is -1
|
||||
if (axes.back() != -1) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
for (size_t i = 0; i < axes.size() - 1; i++) {
|
||||
if (axes[i] - axes[i + 1] != -1) {
|
||||
@@ -861,9 +877,18 @@ public:
|
||||
std::vector<int64_t> axes1 = extractAxis(net, matchedNodesIds[mean1]);
|
||||
if (axes.size() != axes1.size())
|
||||
return false;
|
||||
for (size_t i = 0; i < axes.size(); i++) {
|
||||
if (((axes[i] + input_ndims) % input_ndims) != ((axes1[i] + input_ndims) % input_ndims)) {
|
||||
return false;
|
||||
if (input_ndims != -1) {
|
||||
for (size_t i = 0; i < axes.size(); i++) {
|
||||
if (((axes[i] + input_ndims) % input_ndims) != ((axes1[i] + input_ndims) % input_ndims)) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// both axes sets are negative; just compare directly
|
||||
for (size_t i = 0; i < axes.size(); i++) {
|
||||
if (axes[i] != axes1[i]) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
axis = axes[0];
|
||||
|
||||
@@ -948,6 +948,140 @@ TEST_P(Reproducibility_ResNet50_QDQ_ONNX, Accuracy)
|
||||
INSTANTIATE_TEST_CASE_P(/**/, Reproducibility_ResNet50_QDQ_ONNX,
|
||||
testing::ValuesIn(getAvailableTargets(DNN_BACKEND_OPENCV)));
|
||||
|
||||
typedef testing::TestWithParam<Target> Reproducibility_ViT_ONNX;
|
||||
TEST_P(Reproducibility_ViT_ONNX, Accuracy)
|
||||
{
|
||||
Target targetId = GetParam();
|
||||
applyTestTag(targetId == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_1GB : CV_TEST_TAG_MEMORY_2GB);
|
||||
ASSERT_TRUE(ocl::useOpenCL() || targetId == DNN_TARGET_CPU || targetId == DNN_TARGET_CPU_FP16);
|
||||
auto engine_forced = static_cast<EngineType>(
|
||||
cv::utils::getConfigurationParameterSizeT("OPENCV_FORCE_DNN_ENGINE", ENGINE_AUTO));
|
||||
if (engine_forced == ENGINE_CLASSIC)
|
||||
{
|
||||
applyTestTag(CV_TEST_TAG_DNN_SKIP_PARSER);
|
||||
return;
|
||||
}
|
||||
|
||||
std::string modelname = _tf("vit_base_patch16_224_Opset16.onnx", false);
|
||||
Net net = readNetFromONNX(modelname);
|
||||
|
||||
net.setPreferableBackend(DNN_BACKEND_OPENCV);
|
||||
net.setPreferableTarget(targetId);
|
||||
|
||||
if (targetId == DNN_TARGET_CPU_FP16)
|
||||
net.enableWinograd(false);
|
||||
|
||||
std::string imgname = _tf("sqcat.png");
|
||||
Mat image = imread(imgname);
|
||||
// ViT preprocessing: (pixel/255 - 0.5)/0.5 == pixel/127.5 - 1
|
||||
// blobFromImage form: (pixel - 127.5) * (1/127.5)
|
||||
Mat input = blobFromImage(image, 1.0 / 127.5, Size(224, 224),
|
||||
Scalar(127.5, 127.5, 127.5),
|
||||
true, true, CV_32F);
|
||||
ASSERT_TRUE(!input.empty());
|
||||
|
||||
net.setInput(input);
|
||||
Mat out = net.forward();
|
||||
|
||||
const int K = 5;
|
||||
std::vector<std::pair<int, float> > res;
|
||||
topK(out, res, K);
|
||||
ASSERT_EQ(int(res.size()), K);
|
||||
|
||||
// Reference top-5 captured from the ONNX Runtime engine (OPENCV_FORCE_DNN_ENGINE=4).
|
||||
std::vector<std::pair<int, float> > ref = {
|
||||
{285, 7.683f}, {282, 7.182f}, {281, 6.894f}, {287, 3.623f}, {283, 3.287f}
|
||||
};
|
||||
const float eps = 0.5f;
|
||||
|
||||
std::vector<int> reflabels(K), reslabels(K);
|
||||
for (int i = 0; i < K; i++) {
|
||||
reflabels[i] = ref[i].first;
|
||||
reslabels[i] = res[i].first;
|
||||
}
|
||||
ASSERT_EQ(reflabels, reslabels);
|
||||
|
||||
for (int i = 0; i < K; i++) {
|
||||
EXPECT_NEAR(ref[i].second, res[i].second, eps);
|
||||
}
|
||||
}
|
||||
INSTANTIATE_TEST_CASE_P(/**/, Reproducibility_ViT_ONNX,
|
||||
testing::ValuesIn(getAvailableTargets(DNN_BACKEND_OPENCV)));
|
||||
|
||||
typedef testing::TestWithParam<Target> Reproducibility_BERT_ONNX;
|
||||
TEST_P(Reproducibility_BERT_ONNX, Accuracy)
|
||||
{
|
||||
Target targetId = GetParam();
|
||||
applyTestTag(targetId == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_1GB : CV_TEST_TAG_MEMORY_2GB);
|
||||
ASSERT_TRUE(ocl::useOpenCL() || targetId == DNN_TARGET_CPU || targetId == DNN_TARGET_CPU_FP16);
|
||||
auto engine_forced = static_cast<EngineType>(
|
||||
cv::utils::getConfigurationParameterSizeT("OPENCV_FORCE_DNN_ENGINE", ENGINE_AUTO));
|
||||
if (engine_forced == ENGINE_CLASSIC)
|
||||
{
|
||||
applyTestTag(CV_TEST_TAG_DNN_SKIP_PARSER);
|
||||
return;
|
||||
}
|
||||
|
||||
std::string modelname = _tf("onnx/models/bert.onnx", false);
|
||||
Net net = readNetFromONNX(modelname);
|
||||
|
||||
net.setPreferableBackend(DNN_BACKEND_OPENCV);
|
||||
net.setPreferableTarget(targetId);
|
||||
|
||||
// Tokenized with bert-base-uncased: "The [MASK] sat on the mat."
|
||||
// [CLS]=101 the=1996 [MASK]=103 sat=2938 on=2006 the=1996 mat=13523 .=1012 [SEP]=102
|
||||
const int seq_len = 9;
|
||||
int64_t input_ids_data[seq_len] = {101, 1996, 103, 2938, 2006, 1996, 13523, 1012, 102};
|
||||
int64_t attention_mask_data[seq_len] = {1, 1, 1, 1, 1, 1, 1, 1, 1};
|
||||
int64_t token_type_ids_data[seq_len] = {0, 0, 0, 0, 0, 0, 0, 0, 0};
|
||||
|
||||
int shape[2] = {1, seq_len};
|
||||
Mat input_ids(2, shape, CV_64S, input_ids_data);
|
||||
Mat attention_mask(2, shape, CV_64S, attention_mask_data);
|
||||
Mat token_type_ids(2, shape, CV_64S, token_type_ids_data);
|
||||
|
||||
net.setInput(input_ids, "input_ids");
|
||||
net.setInput(attention_mask, "attention_mask");
|
||||
net.setInput(token_type_ids, "token_type_ids");
|
||||
Mat out = net.forward();
|
||||
|
||||
// Output shape: [1, 9, 30522] (batch, seq_len, vocab_size)
|
||||
ASSERT_EQ(out.dims, 3);
|
||||
ASSERT_EQ(out.size[0], 1);
|
||||
ASSERT_EQ(out.size[1], seq_len);
|
||||
ASSERT_EQ(out.size[2], 30522);
|
||||
|
||||
const int vocab = 30522;
|
||||
const float* mask_logits = out.ptr<float>() + 2 * vocab;
|
||||
|
||||
const int K = 5;
|
||||
std::vector<std::pair<int, float> > res;
|
||||
std::vector<int> idx(vocab);
|
||||
for (int i = 0; i < vocab; i++) idx[i] = i;
|
||||
std::partial_sort(idx.begin(), idx.begin() + K, idx.end(),
|
||||
[&](int a, int b) { return mask_logits[a] > mask_logits[b]; });
|
||||
for (int k = 0; k < K; k++)
|
||||
res.emplace_back(idx[k], mask_logits[idx[k]]);
|
||||
|
||||
std::vector<std::pair<int, float> > ref = {
|
||||
{2611, 8.222f}, {2158, 8.196f}, {3899, 8.021f}, {2879, 7.972f}, {2450, 7.393f}
|
||||
};
|
||||
const float eps = 0.5f;
|
||||
|
||||
std::vector<int> reflabels(K), reslabels(K);
|
||||
for (int i = 0; i < K; i++) {
|
||||
reflabels[i] = ref[i].first;
|
||||
reslabels[i] = res[i].first;
|
||||
}
|
||||
ASSERT_EQ(reflabels, reslabels);
|
||||
|
||||
for (int i = 0; i < K; i++) {
|
||||
EXPECT_NEAR(ref[i].second, res[i].second, eps);
|
||||
}
|
||||
}
|
||||
INSTANTIATE_TEST_CASE_P(/**/, Reproducibility_BERT_ONNX,
|
||||
testing::ValuesIn(getAvailableTargets(DNN_BACKEND_OPENCV)));
|
||||
|
||||
typedef testing::TestWithParam<Target> Reproducibility_MobileNetSSD_ONNX;
|
||||
TEST_P(Reproducibility_MobileNetSSD_ONNX, Accuracy)
|
||||
{
|
||||
|
||||
Reference in New Issue
Block a user