mirror of
https://github.com/opencv/opencv.git
synced 2026-07-29 07:13:02 +04:00
Merge pull request #28931 from abhishek-gola:extended_fusions
Extended graph fusion for transformer models #28931 ### Pull Request Readiness Checklist Performance numbers on Intel(R) Core(TM) i9-14900KS: ``` AlexNet::DNNTestNetwork::OCV/CPU 7.143 6.871 1.04 AlexNet::DNNTestNetwork::OCV/OCL 7.471 7.047 1.06 AlexNet::DNNTestNetwork::OCV/OCL_FP16 7.157 6.907 1.04 BERT::DNNTestNetwork::OCV/CPU 9.454 9.154 1.03 BERT::DNNTestNetwork::OCV/OCL 9.657 9.138 1.06 BERT::DNNTestNetwork::OCV/OCL_FP16 9.590 9.271 1.03 CRNN::DNNTestNetwork::OCV/CPU 18.194 17.020 1.07 CRNN::DNNTestNetwork::OCV/OCL 17.855 17.433 1.02 CRNN::DNNTestNetwork::OCV/OCL_FP16 17.629 17.277 1.02 DenseNet_121::DNNTestNetwork::OCV/CPU 24.735 23.863 1.04 DenseNet_121::DNNTestNetwork::OCV/OCL 24.991 25.140 0.99 DenseNet_121::DNNTestNetwork::OCV/OCL_FP16 24.598 24.019 1.02 EAST_text_detection::DNNTestNetwork::OCV/CPU 26.929 27.084 0.99 EAST_text_detection::DNNTestNetwork::OCV/OCL 26.863 27.531 0.98 EAST_text_detection::DNNTestNetwork::OCV/OCL_FP16 26.884 27.082 0.99 EfficientDet::DNNTestNetwork::OCV/CPU 62.667 62.716 1.00 EfficientDet_int8::DNNTestNetwork::OCV/CPU 34.446 34.796 0.99 EfficientNet::DNNTestNetwork::OCV/CPU 11.875 11.786 1.01 EfficientNet::DNNTestNetwork::OCV/OCL 12.434 11.806 1.05 EfficientNet::DNNTestNetwork::OCV/OCL_FP16 11.886 11.899 1.00 FastNeuralStyle_eccv16::DNNTestNetwork::OCV/CPU 15.104 14.327 1.05 FastNeuralStyle_eccv16::DNNTestNetwork::OCV/OCL 15.071 14.670 1.03 FastNeuralStyle_eccv16::DNNTestNetwork::OCV/OCL_FP16 15.262 14.072 1.08 GoogLeNet::DNNTestNetwork::OCV/CPU 5.030 5.031 1.00 GoogLeNet::DNNTestNetwork::OCV/OCL 5.075 5.017 1.01 GoogLeNet::DNNTestNetwork::OCV/OCL_FP16 5.053 5.055 1.00 Inception_5h::DNNTestNetwork::OCV/CPU 6.777 6.430 1.05 Inception_5h::DNNTestNetwork::OCV/OCL 7.309 6.683 1.09 Inception_5h::DNNTestNetwork::OCV/OCL_FP16 6.624 6.549 1.01 Inception_v2_Faster_RCNN::DNNTestNetwork::OCV/CPU 86.676 82.918 1.05 Inception_v2_Faster_RCNN::DNNTestNetwork::OCV/OCL 85.460 84.873 1.01 Inception_v2_SSD_TensorFlow::DNNTestNetwork::OCV/CPU 16.007 15.723 1.02 Inception_v2_SSD_TensorFlow::DNNTestNetwork::OCV/OCL 15.974 16.015 1.00 Inception_v2_SSD_TensorFlow::DNNTestNetwork::OCV/OCL_FP16 16.219 15.593 1.04 MPHand::DNNTestNetwork::OCV/CPU 3.273 3.313 0.99 MPHand::DNNTestNetwork::OCV/OCL 3.466 3.318 1.04 MPHand::DNNTestNetwork::OCV/OCL_FP16 3.314 3.258 1.02 MPPalm::DNNTestNetwork::OCV/CPU 2.839 1.973 1.44 MPPalm::DNNTestNetwork::OCV/OCL 2.773 2.132 1.30 MPPalm::DNNTestNetwork::OCV/OCL_FP16 2.783 2.133 1.30 MPPose::DNNTestNetwork::OCV/CPU 7.644 8.711 0.88 MPPose::DNNTestNetwork::OCV/OCL 7.555 7.210 1.05 MPPose::DNNTestNetwork::OCV/OCL_FP16 7.564 7.559 1.00 MobileNet_SSD_Caffe::DNNTestNetwork::OCV/CPU 9.040 8.813 1.03 MobileNet_SSD_Caffe::DNNTestNetwork::OCV/OCL 8.692 9.481 0.92 MobileNet_SSD_Caffe::DNNTestNetwork::OCV/OCL_FP16 8.792 8.884 0.99 MobileNet_SSD_v1_TensorFlow::DNNTestNetwork::OCV/CPU 7.893 7.735 1.02 MobileNet_SSD_v1_TensorFlow::DNNTestNetwork::OCV/OCL 8.233 7.816 1.05 MobileNet_SSD_v1_TensorFlow::DNNTestNetwork::OCV/OCL_FP16 7.949 7.817 1.02 MobileNet_SSD_v2_TensorFlow::DNNTestNetwork::OCV/CPU 14.373 15.282 0.94 MobileNet_SSD_v2_TensorFlow::DNNTestNetwork::OCV/OCL 14.218 15.788 0.90 MobileNet_SSD_v2_TensorFlow::DNNTestNetwork::OCV/OCL_FP16 14.473 14.853 0.97 MobileNetv2_ONNX::DNNTestNetwork::OCV/CPU 1.507 1.440 1.05 MobileNetv2_ONNX::DNNTestNetwork::OCV/OCL 1.529 1.394 1.10 MobileNetv2_ONNX::DNNTestNetwork::OCV/OCL_FP16 1.448 1.433 1.01 PPHumanSeg::DNNTestNetwork::OCV/CPU 2.302 1.919 1.20 PPHumanSeg::DNNTestNetwork::OCV/OCL 2.097 1.865 1.12 PPHumanSeg::DNNTestNetwork::OCV/OCL_FP16 2.129 1.908 1.12 PPOCRv3::DNNTestNetwork::OCV/CPU 20.612 20.858 0.99 PPOCRv3::DNNTestNetwork::OCV/OCL 21.672 21.444 1.01 PPOCRv3::DNNTestNetwork::OCV/OCL_FP16 22.660 20.857 1.09 ResNet50_QDQ_ONNX::DNNTestNetwork::OCV/CPU 6.443 6.718 0.96 ResNet50_QDQ_ONNX::DNNTestNetwork::OCV/OCL 6.842 6.823 1.00 ResNet50_QDQ_ONNX::DNNTestNetwork::OCV/OCL_FP16 6.895 6.579 1.05 ResNet_50_v1_ONNX::DNNTestNetwork::OCV/CPU 7.669 7.629 1.01 ResNet_50_v1_ONNX::DNNTestNetwork::OCV/OCL 7.514 7.473 1.01 ResNet_50_v1_ONNX::DNNTestNetwork::OCV/OCL_FP16 7.598 7.513 1.01 SFace::DNNTestNetwork::OCV/CPU 3.427 3.334 1.03 SFace::DNNTestNetwork::OCV/OCL 3.420 3.337 1.02 SFace::DNNTestNetwork::OCV/OCL_FP16 3.497 3.375 1.04 SSD::DNNTestNetwork::OCV/CPU 83.165 83.476 1.00 SSD::DNNTestNetwork::OCV/OCL 83.531 84.121 0.99 SSD::DNNTestNetwork::OCV/OCL_FP16 82.447 83.234 0.99 SqueezeNet_v1_1::DNNTestNetwork::OCV/CPU 1.334 1.308 1.02 SqueezeNet_v1_1::DNNTestNetwork::OCV/OCL 1.279 1.313 0.97 SqueezeNet_v1_1::DNNTestNetwork::OCV/OCL_FP16 1.429 1.290 1.11 VIT_Base_Patch16_224::DNNTestNetwork::OCV/CPU 71.211 75.013 0.95 VIT_Base_Patch16_224::DNNTestNetwork::OCV/OCL 71.988 71.218 1.01 VIT_Base_Patch16_224::DNNTestNetwork::OCV/OCL_FP16 73.700 82.285 0.90 VitTrack::DNNTestNetwork::OCV/CPU 4.096 2.091 1.96 VitTrack::DNNTestNetwork::OCV/OCL 3.830 2.056 1.86 VitTrack::DNNTestNetwork::OCV/OCL_FP16 3.796 2.110 1.80 YOLOX::DNNTestNetwork::OCV/CPU 24.346 23.079 1.05 YOLOX::DNNTestNetwork::OCV/OCL 24.442 23.459 1.04 YOLOX::DNNTestNetwork::OCV/OCL_FP16 23.947 23.327 1.03 YOLOv3::DNNTestNetwork::OCV/CPU 59.140 59.140 1.00 YOLOv3::DNNTestNetwork::OCV/OCL 46.725 45.292 1.03 YOLOv3::DNNTestNetwork::OCV/OCL_FP16 46.939 45.382 1.03 YOLOv4::DNNTestNetwork::OCV/CPU 70.062 68.145 1.03 YOLOv4::DNNTestNetwork::OCV/OCL 219.570 214.823 1.02 YOLOv4::DNNTestNetwork::OCV/OCL_FP16 219.276 213.472 1.03 YOLOv4_tiny::DNNTestNetwork::OCV/CPU 8.362 8.221 1.02 YOLOv4_tiny::DNNTestNetwork::OCV/OCL 8.373 8.095 1.03 YOLOv4_tiny::DNNTestNetwork::OCV/OCL_FP16 8.830 8.088 1.09 YOLOv5::DNNTestNetwork::OCV/CPU 9.237 8.484 1.09 YOLOv5::DNNTestNetwork::OCV/OCL 8.776 8.537 1.03 YOLOv5::DNNTestNetwork::OCV/OCL_FP16 8.747 8.536 1.02 YOLOv8::DNNTestNetwork::OCV/CPU 11.019 11.463 0.96 YOLOv8::DNNTestNetwork::OCV/OCL 11.204 10.766 1.04 YOLOv8::DNNTestNetwork::OCV/OCL_FP16 11.355 10.764 1.05 YuNet::DNNTestNetwork::OCV/CPU 3.371 3.466 0.97 YuNet::DNNTestNetwork::OCV/OCL 3.396 3.105 1.09 YuNet::DNNTestNetwork::OCV/OCL_FP16 3.150 3.130 1.01 ``` See details at https://github.com/opencv/opencv/wiki/How_to_contribute#making-a-good-pull-request - [x] I agree to contribute to the project under Apache 2 License. - [x] To the best of my knowledge, the proposed patch is not based on a code under GPL or another license that is incompatible with OpenCV - [x] The PR is proposed to the proper branch - [x] There is a reference to the original bug report and related work - [x] There is accuracy test, performance test and test data in opencv_extra repository, if applicable Patch to opencv_extra has the same branch name. - [x] The feature is well documented and sample code can be built with the project CMake
This commit is contained in:
Abhishek Gola
GitHub
@@ -602,6 +602,7 @@ CV__DNN_INLINE_NS_BEGIN
|
||||
{
|
||||
public:
|
||||
bool logSoftMax;
|
||||
float scale;
|
||||
|
||||
static Ptr<SoftmaxLayer> create(const LayerParams& params);
|
||||
};
|
||||
@@ -1792,6 +1793,9 @@ CV__DNN_INLINE_NS_BEGIN
|
||||
|
||||
class CV_EXPORTS MatMulLayer : public Layer {
|
||||
public:
|
||||
bool trans_a;
|
||||
bool trans_b;
|
||||
|
||||
static Ptr<MatMulLayer> create(const LayerParams ¶ms);
|
||||
};
|
||||
|
||||
|
||||
@@ -60,6 +60,32 @@ struct ModelFusionAttention
|
||||
return dynamic_cast<MatMulLayer*>(prog[idx].get()) != nullptr;
|
||||
}
|
||||
|
||||
static bool isProjCandidate(const Ptr<Layer>& l)
|
||||
{
|
||||
if (l->blobs.empty() || l->inputs.size() != 1) return false;
|
||||
if (dynamic_cast<MatMulLayer*>(l.get()))
|
||||
return true;
|
||||
GemmLayer* g = dynamic_cast<GemmLayer*>(l.get());
|
||||
if (!g) return false;
|
||||
if (g->trans_a || g->alpha != 1.f) return false;
|
||||
if (l->blobs.size() == 2 && g->beta != 1.f) return false;
|
||||
return true;
|
||||
}
|
||||
|
||||
// Returns the projection weight in [K, N] (input_hidden, output_hidden)
|
||||
// layout, transposing if the source is a Gemm with trans_b.
|
||||
static Mat getProjWeight(const Ptr<Layer>& l)
|
||||
{
|
||||
const Mat& W = l->blobs[0];
|
||||
GemmLayer* g = dynamic_cast<GemmLayer*>(l.get());
|
||||
if (g && g->trans_b) {
|
||||
Mat Wt;
|
||||
cv::transpose(W, Wt);
|
||||
return Wt;
|
||||
}
|
||||
return W;
|
||||
}
|
||||
|
||||
bool isScalarBinOp(const vector<Ptr<Layer>>& prog, int idx,
|
||||
NaryEltwiseLayer::OPERATION op, float* val) const
|
||||
{
|
||||
@@ -251,10 +277,7 @@ struct ModelFusionAttention
|
||||
std::map<int, vector<int>> qkv_candidates;
|
||||
for (size_t i = 0; i < nops; i++) {
|
||||
if (!prog[i]) continue;
|
||||
MatMulLayer* mm = dynamic_cast<MatMulLayer*>(prog[i].get());
|
||||
if (!mm) continue;
|
||||
if (prog[i]->blobs.empty() || prog[i]->inputs.size() != 1)
|
||||
continue;
|
||||
if (!isProjCandidate(prog[i])) continue;
|
||||
Arg inp = prog[i]->inputs[0];
|
||||
qkv_candidates[inp.idx].push_back((int)i);
|
||||
}
|
||||
@@ -406,9 +429,11 @@ struct ModelFusionAttention
|
||||
collectShapeChain(prog, it->second, extra_ops);
|
||||
}
|
||||
|
||||
const Mat& Wq = prog[indices[q_slot]]->blobs[0];
|
||||
const Mat& Wk = prog[indices[k_slot]]->blobs[0];
|
||||
const Mat& Wv = prog[indices[v_slot]]->blobs[0];
|
||||
// Normalize each projection weight to [K, N] regardless of
|
||||
// whether the source op is MatMul or Gemm-with-trans_b.
|
||||
Mat Wq = getProjWeight(prog[indices[q_slot]]);
|
||||
Mat Wk = getProjWeight(prog[indices[k_slot]]);
|
||||
Mat Wv = getProjWeight(prog[indices[v_slot]]);
|
||||
int input_hidden = Wq.size[0];
|
||||
int q_hidden = Wq.size[1];
|
||||
int k_hidden = Wk.size[1];
|
||||
|
||||
@@ -0,0 +1,195 @@
|
||||
// This file is part of OpenCV project.
|
||||
// It is subject to the license terms in the LICENSE file found in the top-level directory
|
||||
// of this distribution and at http://opencv.org/license.html.
|
||||
// Copyright (C) 2026, BigVision LLC, all rights reserved.
|
||||
// Third party copyrights are property of their respective owners.
|
||||
|
||||
// Cleans up redundant Reshape/Transpose chains:
|
||||
// - Reshape(A) -> Reshape(B): drop the inner reshape (A's shape is irrelevant
|
||||
// once B reshapes to the final target).
|
||||
// - Transpose(p1) -> Transpose(p2): compose into a single Transpose with
|
||||
// perm[i] = p1[p2[i]].
|
||||
// - Identity Transpose (perm == [0,1,...,n-1]): bypass.
|
||||
// These run iteratively, so a Reshape+Transpose+Reshape that ends up with an
|
||||
// identity transpose collapses to a single Reshape, and adjacent transposes
|
||||
// in attention layouts merge.
|
||||
|
||||
#include "precomp.hpp"
|
||||
#include "net_impl.hpp"
|
||||
|
||||
namespace cv { namespace dnn {
|
||||
CV__DNN_INLINE_NS_BEGIN
|
||||
|
||||
using std::vector;
|
||||
using std::string;
|
||||
|
||||
struct ModelFusionReshapeTranspose
|
||||
{
|
||||
explicit ModelFusionReshapeTranspose(Net::Impl* netimpl_) : netimpl(netimpl_) {}
|
||||
|
||||
void fuse()
|
||||
{
|
||||
for (int iter = 0; iter < 10; iter++) {
|
||||
netimpl->useCounts(usecounts);
|
||||
if (!fuseGraph(netimpl->mainGraph))
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
static bool isIdentityPerm(const vector<int>& perm)
|
||||
{
|
||||
if (perm.empty()) return false;
|
||||
for (size_t i = 0; i < perm.size(); i++)
|
||||
if (perm[i] != (int)i) return false;
|
||||
return true;
|
||||
}
|
||||
|
||||
bool fuseGraph(Ptr<Graph>& graph)
|
||||
{
|
||||
const vector<Ptr<Layer>>& prog = graph->prog();
|
||||
size_t nops = prog.size();
|
||||
bool modified = false;
|
||||
|
||||
for (size_t i = 0; i < nops; i++) {
|
||||
if (!prog[i]) continue;
|
||||
vector<Ptr<Graph>>* subgraphs = prog[i]->subgraphs();
|
||||
if (subgraphs) {
|
||||
for (Ptr<Graph>& g : *subgraphs)
|
||||
if (fuseGraph(g)) modified = true;
|
||||
}
|
||||
}
|
||||
|
||||
std::map<int, int> producer;
|
||||
std::set<int> externalArgs;
|
||||
for (Arg out : graph->outputs())
|
||||
externalArgs.insert(out.idx);
|
||||
for (size_t i = 0; i < nops; i++) {
|
||||
if (!prog[i]) continue;
|
||||
for (Arg out : prog[i]->outputs)
|
||||
producer[out.idx] = (int)i;
|
||||
}
|
||||
|
||||
vector<bool> dropped(nops, false);
|
||||
|
||||
for (size_t i = 0; i < nops; i++) {
|
||||
const Ptr<Layer>& layer = prog[i];
|
||||
if (!layer || dropped[i]) continue;
|
||||
if (layer->inputs.empty() || layer->outputs.empty()) continue;
|
||||
|
||||
// Identity Transpose: bypass.
|
||||
TransposeLayer* tr = dynamic_cast<TransposeLayer*>(layer.get());
|
||||
if (tr && layer->outputs.size() == 1 && isIdentityPerm(tr->perm)) {
|
||||
Arg out = layer->outputs[0];
|
||||
if (externalArgs.count(out.idx) == 0) {
|
||||
redirectConsumers(prog, dropped, i + 1, out, layer->inputs[0]);
|
||||
dropped[i] = true;
|
||||
usecounts[out.idx] = 0;
|
||||
modified = true;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
// Transpose + Transpose -> Transpose (compose perms).
|
||||
if (tr && layer->outputs.size() == 1) {
|
||||
auto it = producer.find(layer->inputs[0].idx);
|
||||
if (it != producer.end()) {
|
||||
int prod_idx = it->second;
|
||||
if (prod_idx >= 0 && !dropped[prod_idx]) {
|
||||
const Ptr<Layer>& pl = prog[prod_idx];
|
||||
TransposeLayer* prevTr = dynamic_cast<TransposeLayer*>(pl.get());
|
||||
Arg prevOut = layer->inputs[0];
|
||||
bool single_consumer = usecounts[prevOut.idx] == 1
|
||||
&& externalArgs.count(prevOut.idx) == 0;
|
||||
if (prevTr && pl->outputs.size() == 1 && single_consumer
|
||||
&& !prevTr->perm.empty()
|
||||
&& prevTr->perm.size() == tr->perm.size())
|
||||
{
|
||||
// composed[i] = prevTr->perm[tr->perm[i]]
|
||||
vector<int> composed(tr->perm.size());
|
||||
bool ok = true;
|
||||
for (size_t k = 0; k < tr->perm.size(); k++) {
|
||||
int idx = tr->perm[k];
|
||||
if (idx < 0 || idx >= (int)prevTr->perm.size()) {
|
||||
ok = false; break;
|
||||
}
|
||||
composed[k] = prevTr->perm[idx];
|
||||
}
|
||||
if (ok) {
|
||||
tr->perm = composed;
|
||||
layer->inputs[0] = pl->inputs[0];
|
||||
dropped[prod_idx] = true;
|
||||
usecounts[prevOut.idx] = 0;
|
||||
producer[layer->outputs[0].idx] = (int)i;
|
||||
modified = true;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
//Reshape + Reshape -> Reshape (drop the inner reshape).
|
||||
Reshape2Layer* rs = dynamic_cast<Reshape2Layer*>(layer.get());
|
||||
if (rs && layer->outputs.size() == 1) {
|
||||
auto it = producer.find(layer->inputs[0].idx);
|
||||
if (it != producer.end()) {
|
||||
int prod_idx = it->second;
|
||||
if (prod_idx >= 0 && !dropped[prod_idx]) {
|
||||
const Ptr<Layer>& pl = prog[prod_idx];
|
||||
Reshape2Layer* prevRs = dynamic_cast<Reshape2Layer*>(pl.get());
|
||||
Arg prevOut = layer->inputs[0];
|
||||
bool single_consumer = usecounts[prevOut.idx] == 1
|
||||
&& externalArgs.count(prevOut.idx) == 0;
|
||||
if (prevRs && pl->outputs.size() == 1 && single_consumer)
|
||||
{
|
||||
layer->inputs[0] = pl->inputs[0];
|
||||
dropped[prod_idx] = true;
|
||||
usecounts[prevOut.idx] = 0;
|
||||
modified = true;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (modified) {
|
||||
vector<Ptr<Layer>> newprog;
|
||||
newprog.reserve(nops);
|
||||
for (size_t i = 0; i < nops; i++) {
|
||||
if (!dropped[i] && prog[i])
|
||||
newprog.push_back(prog[i]);
|
||||
}
|
||||
graph->setProg(newprog);
|
||||
}
|
||||
|
||||
return modified;
|
||||
}
|
||||
|
||||
void redirectConsumers(const vector<Ptr<Layer>>& prog,
|
||||
const vector<bool>& dropped,
|
||||
size_t start_idx, Arg from, Arg to)
|
||||
{
|
||||
for (size_t j = start_idx; j < prog.size(); j++) {
|
||||
if (!prog[j] || dropped[j]) continue;
|
||||
for (Arg& in : prog[j]->inputs) {
|
||||
if (in.idx == from.idx) {
|
||||
in = to;
|
||||
usecounts[to.idx]++;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Net::Impl* netimpl;
|
||||
vector<int> usecounts;
|
||||
};
|
||||
|
||||
void Net::Impl::fuseReshapeTranspose()
|
||||
{
|
||||
ModelFusionReshapeTranspose pass(this);
|
||||
pass.fuse();
|
||||
}
|
||||
|
||||
CV__DNN_INLINE_NS_END
|
||||
}}
|
||||
@@ -0,0 +1,151 @@
|
||||
// This file is part of OpenCV project.
|
||||
// It is subject to the license terms in the LICENSE file found in the top-level directory
|
||||
// of this distribution and at http://opencv.org/license.html.
|
||||
// Copyright (C) 2026, BigVision LLC, all rights reserved.
|
||||
// Third party copyrights are property of their respective owners.
|
||||
|
||||
// Folds a scalar Mul / Div immediately preceding Softmax into the Softmax
|
||||
// layer's `scale` parameter. The fused softmax bakes the scale into the
|
||||
// per-element load before max/exp/sum/divide, removing one full pass over
|
||||
// the (often large) [B, H, S, S] attention-logits tensor.
|
||||
//
|
||||
// Pattern:
|
||||
// X -> Mul(X, c) -> Softmax => X -> Softmax(scale=c)
|
||||
// X -> Div(X, c) -> Softmax => X -> Softmax(scale=1/c)
|
||||
|
||||
#include "precomp.hpp"
|
||||
#include "net_impl.hpp"
|
||||
|
||||
namespace cv { namespace dnn {
|
||||
CV__DNN_INLINE_NS_BEGIN
|
||||
|
||||
using std::vector;
|
||||
using std::string;
|
||||
|
||||
struct ModelFusionScaleSoftmax
|
||||
{
|
||||
explicit ModelFusionScaleSoftmax(Net::Impl* netimpl_) : netimpl(netimpl_) {}
|
||||
|
||||
bool extractScalar(Arg a, float& out) const
|
||||
{
|
||||
if (!netimpl->isConstArg(a)) return false;
|
||||
Mat t = netimpl->argTensor(a);
|
||||
if (t.total() != 1) return false;
|
||||
if (t.type() == CV_32F) { out = t.ptr<float>()[0]; return true; }
|
||||
if (t.type() == CV_64F) { out = (float)t.ptr<double>()[0]; return true; }
|
||||
return false;
|
||||
}
|
||||
|
||||
void fuse() { fuseGraph(netimpl->mainGraph); }
|
||||
|
||||
bool fuseGraph(Ptr<Graph>& graph)
|
||||
{
|
||||
const vector<Ptr<Layer>>& prog = graph->prog();
|
||||
size_t nops = prog.size();
|
||||
bool modified = false;
|
||||
|
||||
for (size_t i = 0; i < nops; i++) {
|
||||
if (!prog[i]) continue;
|
||||
vector<Ptr<Graph>>* subgraphs = prog[i]->subgraphs();
|
||||
if (subgraphs) {
|
||||
for (Ptr<Graph>& g : *subgraphs)
|
||||
if (fuseGraph(g)) modified = true;
|
||||
}
|
||||
}
|
||||
|
||||
vector<int> usecounts;
|
||||
netimpl->useCounts(usecounts);
|
||||
|
||||
std::set<int> externalArgs;
|
||||
for (Arg out : graph->outputs())
|
||||
externalArgs.insert(out.idx);
|
||||
|
||||
std::map<int, int> producer;
|
||||
for (size_t i = 0; i < nops; i++) {
|
||||
if (!prog[i]) continue;
|
||||
for (Arg out : prog[i]->outputs)
|
||||
producer[out.idx] = (int)i;
|
||||
}
|
||||
|
||||
vector<bool> dropped(nops, false);
|
||||
|
||||
for (size_t i = 0; i < nops; i++) {
|
||||
const Ptr<Layer>& layer = prog[i];
|
||||
if (!layer || dropped[i]) continue;
|
||||
|
||||
SoftmaxLayer* sm = dynamic_cast<SoftmaxLayer*>(layer.get());
|
||||
if (!sm || sm->logSoftMax) continue;
|
||||
if (layer->inputs.size() != 1) continue;
|
||||
|
||||
Arg sm_in = layer->inputs[0];
|
||||
auto it = producer.find(sm_in.idx);
|
||||
if (it == producer.end()) continue;
|
||||
int prod_idx = it->second;
|
||||
if (prod_idx < 0 || dropped[prod_idx]) continue;
|
||||
|
||||
const Ptr<Layer>& pl = prog[prod_idx];
|
||||
NaryEltwiseLayer* elt = dynamic_cast<NaryEltwiseLayer*>(pl.get());
|
||||
if (!elt) continue;
|
||||
const auto op = elt->op;
|
||||
const bool is_mul = (op == NaryEltwiseLayer::OPERATION::PROD);
|
||||
const bool is_div = (op == NaryEltwiseLayer::OPERATION::DIV);
|
||||
if (!is_mul && !is_div) continue;
|
||||
if (pl->inputs.size() != 2 || pl->outputs.size() != 1) continue;
|
||||
|
||||
// Identify which operand is the scalar constant.
|
||||
float scalar = 0.f;
|
||||
int x_slot = -1;
|
||||
if (extractScalar(pl->inputs[1], scalar)) {
|
||||
x_slot = 0;
|
||||
} else if (is_mul && extractScalar(pl->inputs[0], scalar)) {
|
||||
// Mul is commutative; Div with the data on the right doesn't
|
||||
// simplify (1 / scale*x ≠ softmax-fold) so only handle Mul here.
|
||||
x_slot = 1;
|
||||
} else {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (is_div) {
|
||||
if (scalar == 0.f) continue;
|
||||
scalar = 1.f / scalar;
|
||||
}
|
||||
if (!std::isfinite(scalar)) continue;
|
||||
|
||||
bool single_consumer = usecounts[sm_in.idx] == 1
|
||||
&& externalArgs.count(sm_in.idx) == 0;
|
||||
if (!single_consumer) continue;
|
||||
|
||||
sm->scale *= scalar;
|
||||
layer->inputs[0] = pl->inputs[x_slot];
|
||||
dropped[prod_idx] = true;
|
||||
usecounts[sm_in.idx] = 0;
|
||||
modified = true;
|
||||
}
|
||||
|
||||
if (modified) {
|
||||
vector<Ptr<Layer>> newprog;
|
||||
newprog.reserve(nops);
|
||||
for (size_t i = 0; i < nops; i++) {
|
||||
if (!dropped[i] && prog[i])
|
||||
newprog.push_back(prog[i]);
|
||||
}
|
||||
graph->setProg(newprog);
|
||||
}
|
||||
|
||||
return modified;
|
||||
}
|
||||
|
||||
Net::Impl* netimpl;
|
||||
};
|
||||
|
||||
void Net::Impl::fuseScaleSoftmax()
|
||||
{
|
||||
if (preferableBackend != DNN_BACKEND_OPENCV ||
|
||||
preferableTarget != DNN_TARGET_CPU)
|
||||
return;
|
||||
ModelFusionScaleSoftmax pass(this);
|
||||
pass.fuse();
|
||||
}
|
||||
|
||||
CV__DNN_INLINE_NS_END
|
||||
}}
|
||||
@@ -0,0 +1,287 @@
|
||||
// This file is part of OpenCV project.
|
||||
// It is subject to the license terms in the LICENSE file found in the top-level directory
|
||||
// of this distribution and at http://opencv.org/license.html.
|
||||
// Copyright (C) 2026, BigVision LLC, all rights reserved.
|
||||
// Third party copyrights are property of their respective owners.
|
||||
|
||||
// Fuses multiple Gemm layers that share the same input into one wider Gemm.
|
||||
//
|
||||
// Pattern:
|
||||
// X --+--> Gemm[W0, b0] -> Y0
|
||||
// +--> Gemm[W1, b1] -> Y1
|
||||
// +--> Gemm[W2, b2] -> Y2
|
||||
// ...
|
||||
//
|
||||
// Replacement:
|
||||
// X -> Gemm[concat(Wi), concat(bi)] -> Y_concat
|
||||
// Y_concat -> Slice2(0, N0) -> Y0
|
||||
// -> Slice2(N0, N0+N1) -> Y1
|
||||
// -> Slice2(N0+N1, ...) -> Y2
|
||||
// ...
|
||||
//
|
||||
// Triggers when:
|
||||
// - At least 2 Gemm layers share the same input arg.
|
||||
// - Each Gemm has a const weight blob and (optionally) const bias.
|
||||
// - All Gemms have identical (alpha, beta, trans_a, trans_b) and matching K.
|
||||
|
||||
#include "precomp.hpp"
|
||||
#include "net_impl.hpp"
|
||||
|
||||
namespace cv { namespace dnn {
|
||||
CV__DNN_INLINE_NS_BEGIN
|
||||
|
||||
using std::vector;
|
||||
using std::string;
|
||||
|
||||
namespace {
|
||||
|
||||
static bool readGemmWeight(const Ptr<Layer>& l, bool trans_b, Mat& W_out)
|
||||
{
|
||||
if (l->blobs.empty()) return false;
|
||||
const Mat& W = l->blobs[0];
|
||||
if (W.dims != 2 || W.type() != CV_32F) return false;
|
||||
if (trans_b) {
|
||||
cv::transpose(W, W_out);
|
||||
} else {
|
||||
W.copyTo(W_out);
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
static bool readGemmBias(const Ptr<Layer>& l, Mat& b_out)
|
||||
{
|
||||
if (l->blobs.size() < 2) { b_out.release(); return true; }
|
||||
const Mat& b = l->blobs[1];
|
||||
if (b.type() != CV_32F) return false;
|
||||
b.copyTo(b_out);
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
struct ModelFusionSharedGemm
|
||||
{
|
||||
explicit ModelFusionSharedGemm(Net::Impl* netimpl_) : netimpl(netimpl_) {}
|
||||
|
||||
struct GemmInfo
|
||||
{
|
||||
int layer_idx = -1;
|
||||
Arg input;
|
||||
int K = 0;
|
||||
int N = 0;
|
||||
bool trans_a = false;
|
||||
bool trans_b = false;
|
||||
float alpha = 1.f;
|
||||
float beta = 1.f;
|
||||
bool has_bias = false;
|
||||
};
|
||||
|
||||
bool inspectGemm(const vector<Ptr<Layer>>& prog, int idx, GemmInfo& info) const
|
||||
{
|
||||
if (idx < 0 || idx >= (int)prog.size() || !prog[idx]) return false;
|
||||
const Ptr<Layer>& l = prog[idx];
|
||||
GemmLayer* g = dynamic_cast<GemmLayer*>(l.get());
|
||||
if (!g) return false;
|
||||
|
||||
if (l->inputs.size() != 1) return false;
|
||||
if (l->blobs.empty() || l->blobs.size() > 2) return false;
|
||||
|
||||
const Mat& W = l->blobs[0];
|
||||
if (W.dims != 2 || W.type() != CV_32F) return false;
|
||||
|
||||
bool trans_a = g->trans_a;
|
||||
bool trans_b = g->trans_b;
|
||||
float alpha = g->alpha;
|
||||
float beta = g->beta;
|
||||
|
||||
if (trans_a) return false;
|
||||
if (alpha != 1.f) return false;
|
||||
if (l->blobs.size() == 2 && beta != 1.f) return false;
|
||||
|
||||
int K = trans_b ? W.size[1] : W.size[0];
|
||||
int N = trans_b ? W.size[0] : W.size[1];
|
||||
if (K <= 0 || N <= 0) return false;
|
||||
|
||||
if (l->blobs.size() == 2) {
|
||||
const Mat& b = l->blobs[1];
|
||||
if (b.type() != CV_32F) return false;
|
||||
if ((int)b.total() != N) return false;
|
||||
}
|
||||
|
||||
info.layer_idx = idx;
|
||||
info.input = l->inputs[0];
|
||||
info.K = K;
|
||||
info.N = N;
|
||||
info.trans_a = trans_a;
|
||||
info.trans_b = trans_b;
|
||||
info.alpha = alpha;
|
||||
info.beta = beta;
|
||||
info.has_bias = (l->blobs.size() == 2);
|
||||
return true;
|
||||
}
|
||||
|
||||
void fuse() { fuseGraph(netimpl->mainGraph); }
|
||||
|
||||
bool fuseGraph(Ptr<Graph>& graph)
|
||||
{
|
||||
const vector<Ptr<Layer>>& prog = graph->prog();
|
||||
size_t nops = prog.size();
|
||||
|
||||
struct Key { int input_idx; int trans_b; int K; };
|
||||
auto keyLess = [](const Key& a, const Key& b) {
|
||||
if (a.input_idx != b.input_idx) return a.input_idx < b.input_idx;
|
||||
if (a.trans_b != b.trans_b) return a.trans_b < b.trans_b;
|
||||
return a.K < b.K;
|
||||
};
|
||||
std::map<Key, vector<GemmInfo>, decltype(keyLess)> groups(keyLess);
|
||||
|
||||
for (size_t i = 0; i < nops; i++) {
|
||||
GemmInfo info;
|
||||
if (!inspectGemm(prog, (int)i, info)) continue;
|
||||
Key k{info.input.idx, info.trans_b ? 1 : 0, info.K};
|
||||
groups[k].push_back(info);
|
||||
}
|
||||
|
||||
bool modified = false;
|
||||
std::set<int> removed_ops;
|
||||
vector<std::pair<int, vector<Ptr<Layer>>>> insertions; // (insert_pos, fused-and-slice layers)
|
||||
|
||||
for (auto& [key, infos] : groups) {
|
||||
if (infos.size() < 2) continue;
|
||||
|
||||
bool all_have_bias = infos[0].has_bias;
|
||||
bool uniform = true;
|
||||
for (auto& info : infos)
|
||||
if (info.has_bias != all_have_bias) { uniform = false; break; }
|
||||
if (!uniform) continue;
|
||||
|
||||
int K = infos[0].K;
|
||||
int total_N = 0;
|
||||
for (auto& info : infos) total_N += info.N;
|
||||
|
||||
int wshape[] = { K, total_N };
|
||||
Mat W_concat(2, wshape, CV_32F);
|
||||
int col_offset = 0;
|
||||
for (auto& info : infos) {
|
||||
Mat W;
|
||||
if (!readGemmWeight(prog[info.layer_idx], info.trans_b, W)) {
|
||||
uniform = false; break;
|
||||
}
|
||||
CV_Assert(W.size[0] == K && W.size[1] == info.N);
|
||||
for (int r = 0; r < K; r++) {
|
||||
float* dst = W_concat.ptr<float>(r) + col_offset;
|
||||
memcpy(dst, W.ptr<float>(r), info.N * sizeof(float));
|
||||
}
|
||||
col_offset += info.N;
|
||||
}
|
||||
if (!uniform) continue;
|
||||
|
||||
Mat b_concat;
|
||||
if (all_have_bias) {
|
||||
int bshape[] = { total_N };
|
||||
b_concat.create(1, bshape, CV_32F);
|
||||
int b_offset = 0;
|
||||
for (auto& info : infos) {
|
||||
Mat b;
|
||||
if (!readGemmBias(prog[info.layer_idx], b)) {
|
||||
uniform = false; break;
|
||||
}
|
||||
CV_Assert((int)b.total() == info.N);
|
||||
memcpy(b_concat.ptr<float>() + b_offset, b.ptr<float>(),
|
||||
info.N * sizeof(float));
|
||||
b_offset += info.N;
|
||||
}
|
||||
if (!uniform) continue;
|
||||
}
|
||||
|
||||
int insert_pos = (int)nops;
|
||||
for (auto& info : infos) insert_pos = std::min(insert_pos, info.layer_idx);
|
||||
|
||||
LayerParams fp;
|
||||
fp.name = prog[infos[0].layer_idx]->name + "_shared_input_fused";
|
||||
fp.type = "Gemm";
|
||||
fp.set("transA", false);
|
||||
fp.set("transB", false);
|
||||
fp.set("alpha", 1.f);
|
||||
fp.set("beta", 1.f);
|
||||
fp.blobs.push_back(W_concat);
|
||||
if (all_have_bias) fp.blobs.push_back(b_concat);
|
||||
|
||||
Ptr<Layer> fused = LayerFactory::createLayerInstance("Gemm", fp);
|
||||
if (!fused) continue;
|
||||
|
||||
string fused_out_name = fp.name + "_out";
|
||||
Arg fused_out_arg = netimpl->getArg(fused_out_name);
|
||||
fused->inputs = { infos[0].input };
|
||||
fused->outputs = { fused_out_arg };
|
||||
fused->netimpl = netimpl;
|
||||
|
||||
vector<Ptr<Layer>> slices;
|
||||
int col_cursor = 0;
|
||||
for (size_t s = 0; s < infos.size(); s++) {
|
||||
int N_s = infos[s].N;
|
||||
int begin = col_cursor;
|
||||
int end = col_cursor + N_s;
|
||||
col_cursor += N_s;
|
||||
|
||||
LayerParams sp;
|
||||
sp.name = prog[infos[s].layer_idx]->name + "_shared_input_slice";
|
||||
sp.type = "Slice2";
|
||||
|
||||
Mat starts(1, 1, CV_32S); starts.at<int>(0) = begin;
|
||||
Mat ends (1, 1, CV_32S); ends.at<int>(0) = end;
|
||||
Mat axes (1, 1, CV_32S); axes.at<int>(0) = -1; // last axis
|
||||
Mat steps (1, 1, CV_32S); steps.at<int>(0) = 1;
|
||||
|
||||
Arg starts_arg = netimpl->newConstArg(sp.name + "_starts", starts);
|
||||
Arg ends_arg = netimpl->newConstArg(sp.name + "_ends", ends);
|
||||
Arg axes_arg = netimpl->newConstArg(sp.name + "_axes", axes);
|
||||
Arg steps_arg = netimpl->newConstArg(sp.name + "_steps", steps);
|
||||
|
||||
Ptr<Layer> slice = LayerFactory::createLayerInstance("Slice2", sp);
|
||||
if (!slice) { uniform = false; break; }
|
||||
slice->inputs = { fused_out_arg, starts_arg, ends_arg, axes_arg, steps_arg };
|
||||
slice->outputs = prog[infos[s].layer_idx]->outputs;
|
||||
slice->netimpl = netimpl;
|
||||
slices.push_back(slice);
|
||||
}
|
||||
if (!uniform) continue;
|
||||
|
||||
for (auto& info : infos) removed_ops.insert(info.layer_idx);
|
||||
vector<Ptr<Layer>> bundle;
|
||||
bundle.push_back(fused);
|
||||
for (auto& s : slices) bundle.push_back(s);
|
||||
insertions.emplace_back(insert_pos, std::move(bundle));
|
||||
modified = true;
|
||||
}
|
||||
|
||||
if (!modified) return false;
|
||||
|
||||
std::sort(insertions.begin(), insertions.end(),
|
||||
[](auto& a, auto& b) { return a.first < b.first; });
|
||||
|
||||
vector<Ptr<Layer>> newprog;
|
||||
size_t ins_idx = 0;
|
||||
for (size_t i = 0; i < nops; i++) {
|
||||
while (ins_idx < insertions.size() &&
|
||||
insertions[ins_idx].first == (int)i) {
|
||||
for (auto& l : insertions[ins_idx].second) newprog.push_back(l);
|
||||
ins_idx++;
|
||||
}
|
||||
if (removed_ops.count((int)i)) continue;
|
||||
newprog.push_back(prog[i]);
|
||||
}
|
||||
graph->setProg(newprog);
|
||||
return true;
|
||||
}
|
||||
|
||||
Net::Impl* netimpl;
|
||||
};
|
||||
|
||||
void Net::Impl::fuseSharedInputGemm()
|
||||
{
|
||||
ModelFusionSharedGemm pass(this);
|
||||
pass.fuse();
|
||||
}
|
||||
|
||||
CV__DNN_INLINE_NS_END
|
||||
}}
|
||||
@@ -0,0 +1,127 @@
|
||||
// This file is part of OpenCV project.
|
||||
// It is subject to the license terms in the LICENSE file found in the top-level directory
|
||||
// of this distribution and at http://opencv.org/license.html.
|
||||
// Copyright (C) 2026, BigVision LLC, all rights reserved.
|
||||
// Third party copyrights are property of their respective owners.
|
||||
|
||||
// Folds a Transpose-of-last-two-dims into the consuming MatMul's trans_a / trans_b.
|
||||
//
|
||||
// Pattern:
|
||||
// X -> Transpose(perm=[0,..,n-3, n-1, n-2]) -> MatMul(., Y)
|
||||
// => X -> MatMul(trans_a=true, ., Y)
|
||||
//
|
||||
// Symmetric on the second operand -> trans_b.
|
||||
|
||||
#include "precomp.hpp"
|
||||
#include "net_impl.hpp"
|
||||
|
||||
namespace cv { namespace dnn {
|
||||
CV__DNN_INLINE_NS_BEGIN
|
||||
|
||||
using std::vector;
|
||||
using std::string;
|
||||
|
||||
struct ModelFusionTransposeMatMul
|
||||
{
|
||||
explicit ModelFusionTransposeMatMul(Net::Impl* netimpl_) : netimpl(netimpl_) {}
|
||||
|
||||
void fuse() { fuseGraph(netimpl->mainGraph); }
|
||||
|
||||
static bool isLastTwoSwap(const vector<int>& perm)
|
||||
{
|
||||
int n = (int)perm.size();
|
||||
if (n < 2) return false;
|
||||
for (int i = 0; i < n - 2; i++)
|
||||
if (perm[i] != i) return false;
|
||||
return perm[n - 2] == n - 1 && perm[n - 1] == n - 2;
|
||||
}
|
||||
|
||||
bool fuseGraph(Ptr<Graph>& graph)
|
||||
{
|
||||
const vector<Ptr<Layer>>& prog = graph->prog();
|
||||
size_t nops = prog.size();
|
||||
bool modified = false;
|
||||
|
||||
for (size_t i = 0; i < nops; i++) {
|
||||
if (!prog[i]) continue;
|
||||
vector<Ptr<Graph>>* subgraphs = prog[i]->subgraphs();
|
||||
if (subgraphs) {
|
||||
for (Ptr<Graph>& g : *subgraphs)
|
||||
if (fuseGraph(g)) modified = true;
|
||||
}
|
||||
}
|
||||
|
||||
vector<int> usecounts;
|
||||
netimpl->useCounts(usecounts);
|
||||
|
||||
std::set<int> externalArgs;
|
||||
for (Arg out : graph->outputs())
|
||||
externalArgs.insert(out.idx);
|
||||
|
||||
std::map<int, int> producer;
|
||||
for (size_t i = 0; i < nops; i++) {
|
||||
if (!prog[i]) continue;
|
||||
for (Arg out : prog[i]->outputs)
|
||||
producer[out.idx] = (int)i;
|
||||
}
|
||||
|
||||
vector<bool> dropped(nops, false);
|
||||
|
||||
for (size_t i = 0; i < nops; i++) {
|
||||
const Ptr<Layer>& layer = prog[i];
|
||||
if (!layer || dropped[i]) continue;
|
||||
|
||||
MatMulLayer* mm = dynamic_cast<MatMulLayer*>(layer.get());
|
||||
if (!mm) continue;
|
||||
|
||||
if (layer->inputs.size() != 2) continue;
|
||||
|
||||
for (int slot = 0; slot < 2; slot++) {
|
||||
Arg in = layer->inputs[slot];
|
||||
auto it = producer.find(in.idx);
|
||||
if (it == producer.end()) continue;
|
||||
int prod_idx = it->second;
|
||||
if (prod_idx < 0 || dropped[prod_idx]) continue;
|
||||
|
||||
const Ptr<Layer>& pl = prog[prod_idx];
|
||||
TransposeLayer* tr = dynamic_cast<TransposeLayer*>(pl.get());
|
||||
if (!tr || pl->outputs.size() != 1) continue;
|
||||
if (!isLastTwoSwap(tr->perm)) continue;
|
||||
|
||||
bool single_consumer = usecounts[in.idx] == 1
|
||||
&& externalArgs.count(in.idx) == 0;
|
||||
if (!single_consumer) continue;
|
||||
|
||||
if (slot == 0) mm->trans_a = !mm->trans_a;
|
||||
else mm->trans_b = !mm->trans_b;
|
||||
|
||||
layer->inputs[slot] = pl->inputs[0];
|
||||
dropped[prod_idx] = true;
|
||||
usecounts[in.idx] = 0;
|
||||
modified = true;
|
||||
}
|
||||
}
|
||||
|
||||
if (modified) {
|
||||
vector<Ptr<Layer>> newprog;
|
||||
newprog.reserve(nops);
|
||||
for (size_t i = 0; i < nops; i++) {
|
||||
if (!dropped[i] && prog[i])
|
||||
newprog.push_back(prog[i]);
|
||||
}
|
||||
graph->setProg(newprog);
|
||||
}
|
||||
return modified;
|
||||
}
|
||||
|
||||
Net::Impl* netimpl;
|
||||
};
|
||||
|
||||
void Net::Impl::fuseTransposeMatMul()
|
||||
{
|
||||
ModelFusionTransposeMatMul pass(this);
|
||||
pass.fuse();
|
||||
}
|
||||
|
||||
CV__DNN_INLINE_NS_END
|
||||
}}
|
||||
@@ -286,6 +286,8 @@ public:
|
||||
desiredInputs[0] = DATA_LAYOUT_BLOCK;
|
||||
for (size_t i = 1; i < ninputs; i++)
|
||||
desiredInputs[i] = DATA_LAYOUT_UNKNOWN;
|
||||
if (addResidual && ninputs >= 2)
|
||||
desiredInputs[ninputs - 1] = DATA_LAYOUT_BLOCK;
|
||||
outputs.assign(requiredOutputs, DATA_LAYOUT_BLOCK);
|
||||
return getNetImpl(this)->defaultC0;
|
||||
}
|
||||
|
||||
@@ -15,11 +15,12 @@
|
||||
|
||||
namespace cv { namespace dnn {
|
||||
|
||||
void softmax(Mat &dst, const Mat &src, int axis, int axisBias, int axisStep){
|
||||
void softmax(Mat &dst, const Mat &src, int axis, int axisBias, int axisStep, float scale){
|
||||
CV_Assert(src.type() == CV_32F);
|
||||
CV_Assert(src.isContinuous() && dst.isContinuous());
|
||||
CV_Assert(src.size == dst.size);
|
||||
axis = normalize_axis(axis, src.dims);
|
||||
const bool scaled = scale != 1.f;
|
||||
|
||||
size_t outerSize = src.total(0, axis),
|
||||
innerSize = src.total(axis + 1);
|
||||
@@ -61,6 +62,22 @@ void softmax(Mat &dst, const Mat &src, int axis, int axisBias, int axisStep){
|
||||
for (; _cnDim < axisStep; _cnDim++)
|
||||
axisBuf[_cnDim] = srcPtr[srcOffset + (_cnDim + axisBias) * cnStep];
|
||||
|
||||
// Bake the fused pre-Softmax scale into the buffer once. axisBuf
|
||||
// already lives in L1 from the load above, so this pass is short.
|
||||
if (scaled) {
|
||||
int sk = 0;
|
||||
#if (CV_SIMD || CV_SIMD_SCALABLE)
|
||||
v_float32 vscale = vx_setall_f32(scale);
|
||||
for (; sk <= axisStep - nlanes; sk += nlanes) {
|
||||
v_float32 val = vx_load(axisBuf + sk);
|
||||
val = v_mul(val, vscale);
|
||||
v_store(axisBuf + sk, val);
|
||||
}
|
||||
#endif
|
||||
for (; sk < axisStep; sk++)
|
||||
axisBuf[sk] *= scale;
|
||||
}
|
||||
|
||||
float maxVal = -FLT_MAX;
|
||||
int cnDim = 0;
|
||||
#if (CV_SIMD || CV_SIMD_SCALABLE)
|
||||
@@ -128,6 +145,10 @@ void softmax(Mat &dst, const Mat &src, int axis) {
|
||||
softmax(dst, src, axis, 0, src.size[axis]);
|
||||
}
|
||||
|
||||
void softmax(Mat &dst, const Mat &src, int axis, float scale) {
|
||||
softmax(dst, src, axis, 0, src.size[axis], scale);
|
||||
}
|
||||
|
||||
void logSoftmax(Mat &dst, const Mat &src, int axis) {
|
||||
softmax(dst, src, axis);
|
||||
log(dst, dst);
|
||||
|
||||
@@ -17,10 +17,12 @@
|
||||
|
||||
namespace cv { namespace dnn {
|
||||
|
||||
void softmax(Mat &dst, const Mat &src, int axis, int axisBias, int axisStep);
|
||||
void softmax(Mat &dst, const Mat &src, int axis, int axisBias, int axisStep, float scale = 1.f);
|
||||
|
||||
void softmax(Mat &dst, const Mat &src, int axis);
|
||||
|
||||
void softmax(Mat &dst, const Mat &src, int axis, float scale);
|
||||
|
||||
void logSoftmax(Mat &dst, const Mat &src, int axis);
|
||||
|
||||
}} // cv::dnn
|
||||
|
||||
@@ -503,8 +503,6 @@ class MatMulLayerImpl CV_FINAL : public MatMulLayer {
|
||||
#endif // HAVE_CANN
|
||||
|
||||
private:
|
||||
bool trans_a;
|
||||
bool trans_b;
|
||||
float alpha;
|
||||
float beta;
|
||||
|
||||
|
||||
@@ -77,6 +77,7 @@ public:
|
||||
{
|
||||
axisRaw = params.get<int>("axis", -1);
|
||||
logSoftMax = params.get<bool>("log_softmax", false);
|
||||
scale = params.get<float>("scale", 1.f);
|
||||
setParamsFrom(params);
|
||||
}
|
||||
|
||||
@@ -231,10 +232,14 @@ public:
|
||||
Mat &dst = outputs[0];
|
||||
int axis = normalize_axis(axisRaw, src.dims);
|
||||
|
||||
if(logSoftMax)
|
||||
if (logSoftMax) {
|
||||
CV_Assert(scale == 1.f);
|
||||
logSoftmax(dst, src, axis);
|
||||
else
|
||||
} else if (scale != 1.f) {
|
||||
softmax(dst, src, axis, scale);
|
||||
} else {
|
||||
softmax(dst, src, axis);
|
||||
}
|
||||
}
|
||||
|
||||
#ifdef HAVE_CUDA
|
||||
|
||||
@@ -456,6 +456,14 @@ struct Net::Impl : public detail::NetImplBase
|
||||
void fuseBasic();
|
||||
// fuse ViT-style multi-head attention subgraphs
|
||||
void fuseAttention();
|
||||
// fuse Gemm layers that share the same input into one wider Gemm
|
||||
void fuseSharedInputGemm();
|
||||
// collapse redundant Reshape/Transpose chains
|
||||
void fuseReshapeTranspose();
|
||||
// absorb a last-two-dims Transpose into the consuming MatMul
|
||||
void fuseTransposeMatMul();
|
||||
// fold a scalar Mul/Div before Softmax into Softmax::scale (CPU only)
|
||||
void fuseScaleSoftmax();
|
||||
// replace constant sub-expressions with their results
|
||||
|
||||
void fuseQDQ();
|
||||
|
||||
@@ -549,8 +549,12 @@ void Net::Impl::prepareForInference()
|
||||
constFold();
|
||||
constArgs();
|
||||
fuseAttention();
|
||||
useBlockLayout();
|
||||
fuseSharedInputGemm();
|
||||
fuseReshapeTranspose();
|
||||
fuseTransposeMatMul();
|
||||
fuseScaleSoftmax();
|
||||
fuseBasic();
|
||||
useBlockLayout();
|
||||
assignBuffers();
|
||||
totalLayers = updateGraphOfs(mainGraph, 0, true);
|
||||
prepared = true;
|
||||
|
||||
Reference in New Issue
Block a user