From 2760c0c08ae384acaa5c8fd5fd66dfb879d90def Mon Sep 17 00:00:00 2001 From: Abhishek Gola Date: Wed, 6 May 2026 14:24:40 +0530 Subject: [PATCH] Merge pull request #28931 from abhishek-gola:extended_fusions Extended graph fusion for transformer models #28931 ### Pull Request Readiness Checklist Performance numbers on Intel(R) Core(TM) i9-14900KS: ``` AlexNet::DNNTestNetwork::OCV/CPU 7.143 6.871 1.04 AlexNet::DNNTestNetwork::OCV/OCL 7.471 7.047 1.06 AlexNet::DNNTestNetwork::OCV/OCL_FP16 7.157 6.907 1.04 BERT::DNNTestNetwork::OCV/CPU 9.454 9.154 1.03 BERT::DNNTestNetwork::OCV/OCL 9.657 9.138 1.06 BERT::DNNTestNetwork::OCV/OCL_FP16 9.590 9.271 1.03 CRNN::DNNTestNetwork::OCV/CPU 18.194 17.020 1.07 CRNN::DNNTestNetwork::OCV/OCL 17.855 17.433 1.02 CRNN::DNNTestNetwork::OCV/OCL_FP16 17.629 17.277 1.02 DenseNet_121::DNNTestNetwork::OCV/CPU 24.735 23.863 1.04 DenseNet_121::DNNTestNetwork::OCV/OCL 24.991 25.140 0.99 DenseNet_121::DNNTestNetwork::OCV/OCL_FP16 24.598 24.019 1.02 EAST_text_detection::DNNTestNetwork::OCV/CPU 26.929 27.084 0.99 EAST_text_detection::DNNTestNetwork::OCV/OCL 26.863 27.531 0.98 EAST_text_detection::DNNTestNetwork::OCV/OCL_FP16 26.884 27.082 0.99 EfficientDet::DNNTestNetwork::OCV/CPU 62.667 62.716 1.00 EfficientDet_int8::DNNTestNetwork::OCV/CPU 34.446 34.796 0.99 EfficientNet::DNNTestNetwork::OCV/CPU 11.875 11.786 1.01 EfficientNet::DNNTestNetwork::OCV/OCL 12.434 11.806 1.05 EfficientNet::DNNTestNetwork::OCV/OCL_FP16 11.886 11.899 1.00 FastNeuralStyle_eccv16::DNNTestNetwork::OCV/CPU 15.104 14.327 1.05 FastNeuralStyle_eccv16::DNNTestNetwork::OCV/OCL 15.071 14.670 1.03 FastNeuralStyle_eccv16::DNNTestNetwork::OCV/OCL_FP16 15.262 14.072 1.08 GoogLeNet::DNNTestNetwork::OCV/CPU 5.030 5.031 1.00 GoogLeNet::DNNTestNetwork::OCV/OCL 5.075 5.017 1.01 GoogLeNet::DNNTestNetwork::OCV/OCL_FP16 5.053 5.055 1.00 Inception_5h::DNNTestNetwork::OCV/CPU 6.777 6.430 1.05 Inception_5h::DNNTestNetwork::OCV/OCL 7.309 6.683 1.09 Inception_5h::DNNTestNetwork::OCV/OCL_FP16 6.624 6.549 1.01 Inception_v2_Faster_RCNN::DNNTestNetwork::OCV/CPU 86.676 82.918 1.05 Inception_v2_Faster_RCNN::DNNTestNetwork::OCV/OCL 85.460 84.873 1.01 Inception_v2_SSD_TensorFlow::DNNTestNetwork::OCV/CPU 16.007 15.723 1.02 Inception_v2_SSD_TensorFlow::DNNTestNetwork::OCV/OCL 15.974 16.015 1.00 Inception_v2_SSD_TensorFlow::DNNTestNetwork::OCV/OCL_FP16 16.219 15.593 1.04 MPHand::DNNTestNetwork::OCV/CPU 3.273 3.313 0.99 MPHand::DNNTestNetwork::OCV/OCL 3.466 3.318 1.04 MPHand::DNNTestNetwork::OCV/OCL_FP16 3.314 3.258 1.02 MPPalm::DNNTestNetwork::OCV/CPU 2.839 1.973 1.44 MPPalm::DNNTestNetwork::OCV/OCL 2.773 2.132 1.30 MPPalm::DNNTestNetwork::OCV/OCL_FP16 2.783 2.133 1.30 MPPose::DNNTestNetwork::OCV/CPU 7.644 8.711 0.88 MPPose::DNNTestNetwork::OCV/OCL 7.555 7.210 1.05 MPPose::DNNTestNetwork::OCV/OCL_FP16 7.564 7.559 1.00 MobileNet_SSD_Caffe::DNNTestNetwork::OCV/CPU 9.040 8.813 1.03 MobileNet_SSD_Caffe::DNNTestNetwork::OCV/OCL 8.692 9.481 0.92 MobileNet_SSD_Caffe::DNNTestNetwork::OCV/OCL_FP16 8.792 8.884 0.99 MobileNet_SSD_v1_TensorFlow::DNNTestNetwork::OCV/CPU 7.893 7.735 1.02 MobileNet_SSD_v1_TensorFlow::DNNTestNetwork::OCV/OCL 8.233 7.816 1.05 MobileNet_SSD_v1_TensorFlow::DNNTestNetwork::OCV/OCL_FP16 7.949 7.817 1.02 MobileNet_SSD_v2_TensorFlow::DNNTestNetwork::OCV/CPU 14.373 15.282 0.94 MobileNet_SSD_v2_TensorFlow::DNNTestNetwork::OCV/OCL 14.218 15.788 0.90 MobileNet_SSD_v2_TensorFlow::DNNTestNetwork::OCV/OCL_FP16 14.473 14.853 0.97 MobileNetv2_ONNX::DNNTestNetwork::OCV/CPU 1.507 1.440 1.05 MobileNetv2_ONNX::DNNTestNetwork::OCV/OCL 1.529 1.394 1.10 MobileNetv2_ONNX::DNNTestNetwork::OCV/OCL_FP16 1.448 1.433 1.01 PPHumanSeg::DNNTestNetwork::OCV/CPU 2.302 1.919 1.20 PPHumanSeg::DNNTestNetwork::OCV/OCL 2.097 1.865 1.12 PPHumanSeg::DNNTestNetwork::OCV/OCL_FP16 2.129 1.908 1.12 PPOCRv3::DNNTestNetwork::OCV/CPU 20.612 20.858 0.99 PPOCRv3::DNNTestNetwork::OCV/OCL 21.672 21.444 1.01 PPOCRv3::DNNTestNetwork::OCV/OCL_FP16 22.660 20.857 1.09 ResNet50_QDQ_ONNX::DNNTestNetwork::OCV/CPU 6.443 6.718 0.96 ResNet50_QDQ_ONNX::DNNTestNetwork::OCV/OCL 6.842 6.823 1.00 ResNet50_QDQ_ONNX::DNNTestNetwork::OCV/OCL_FP16 6.895 6.579 1.05 ResNet_50_v1_ONNX::DNNTestNetwork::OCV/CPU 7.669 7.629 1.01 ResNet_50_v1_ONNX::DNNTestNetwork::OCV/OCL 7.514 7.473 1.01 ResNet_50_v1_ONNX::DNNTestNetwork::OCV/OCL_FP16 7.598 7.513 1.01 SFace::DNNTestNetwork::OCV/CPU 3.427 3.334 1.03 SFace::DNNTestNetwork::OCV/OCL 3.420 3.337 1.02 SFace::DNNTestNetwork::OCV/OCL_FP16 3.497 3.375 1.04 SSD::DNNTestNetwork::OCV/CPU 83.165 83.476 1.00 SSD::DNNTestNetwork::OCV/OCL 83.531 84.121 0.99 SSD::DNNTestNetwork::OCV/OCL_FP16 82.447 83.234 0.99 SqueezeNet_v1_1::DNNTestNetwork::OCV/CPU 1.334 1.308 1.02 SqueezeNet_v1_1::DNNTestNetwork::OCV/OCL 1.279 1.313 0.97 SqueezeNet_v1_1::DNNTestNetwork::OCV/OCL_FP16 1.429 1.290 1.11 VIT_Base_Patch16_224::DNNTestNetwork::OCV/CPU 71.211 75.013 0.95 VIT_Base_Patch16_224::DNNTestNetwork::OCV/OCL 71.988 71.218 1.01 VIT_Base_Patch16_224::DNNTestNetwork::OCV/OCL_FP16 73.700 82.285 0.90 VitTrack::DNNTestNetwork::OCV/CPU 4.096 2.091 1.96 VitTrack::DNNTestNetwork::OCV/OCL 3.830 2.056 1.86 VitTrack::DNNTestNetwork::OCV/OCL_FP16 3.796 2.110 1.80 YOLOX::DNNTestNetwork::OCV/CPU 24.346 23.079 1.05 YOLOX::DNNTestNetwork::OCV/OCL 24.442 23.459 1.04 YOLOX::DNNTestNetwork::OCV/OCL_FP16 23.947 23.327 1.03 YOLOv3::DNNTestNetwork::OCV/CPU 59.140 59.140 1.00 YOLOv3::DNNTestNetwork::OCV/OCL 46.725 45.292 1.03 YOLOv3::DNNTestNetwork::OCV/OCL_FP16 46.939 45.382 1.03 YOLOv4::DNNTestNetwork::OCV/CPU 70.062 68.145 1.03 YOLOv4::DNNTestNetwork::OCV/OCL 219.570 214.823 1.02 YOLOv4::DNNTestNetwork::OCV/OCL_FP16 219.276 213.472 1.03 YOLOv4_tiny::DNNTestNetwork::OCV/CPU 8.362 8.221 1.02 YOLOv4_tiny::DNNTestNetwork::OCV/OCL 8.373 8.095 1.03 YOLOv4_tiny::DNNTestNetwork::OCV/OCL_FP16 8.830 8.088 1.09 YOLOv5::DNNTestNetwork::OCV/CPU 9.237 8.484 1.09 YOLOv5::DNNTestNetwork::OCV/OCL 8.776 8.537 1.03 YOLOv5::DNNTestNetwork::OCV/OCL_FP16 8.747 8.536 1.02 YOLOv8::DNNTestNetwork::OCV/CPU 11.019 11.463 0.96 YOLOv8::DNNTestNetwork::OCV/OCL 11.204 10.766 1.04 YOLOv8::DNNTestNetwork::OCV/OCL_FP16 11.355 10.764 1.05 YuNet::DNNTestNetwork::OCV/CPU 3.371 3.466 0.97 YuNet::DNNTestNetwork::OCV/OCL 3.396 3.105 1.09 YuNet::DNNTestNetwork::OCV/OCL_FP16 3.150 3.130 1.01 ``` See details at https://github.com/opencv/opencv/wiki/How_to_contribute#making-a-good-pull-request - [x] I agree to contribute to the project under Apache 2 License. - [x] To the best of my knowledge, the proposed patch is not based on a code under GPL or another license that is incompatible with OpenCV - [x] The PR is proposed to the proper branch - [x] There is a reference to the original bug report and related work - [x] There is accuracy test, performance test and test data in opencv_extra repository, if applicable Patch to opencv_extra has the same branch name. - [x] The feature is well documented and sample code can be built with the project CMake --- .../dnn/include/opencv2/dnn/all_layers.hpp | 4 + modules/dnn/src/graph_fusion_attention.cpp | 39 ++- .../src/graph_fusion_reshape_transpose.cpp | 195 ++++++++++++ .../dnn/src/graph_fusion_scale_softmax.cpp | 151 +++++++++ modules/dnn/src/graph_fusion_shared_gemm.cpp | 287 ++++++++++++++++++ .../dnn/src/graph_fusion_transpose_matmul.cpp | 127 ++++++++ modules/dnn/src/layers/conv2_layer.cpp | 2 + .../dnn/src/layers/cpu_kernels/softmax.cpp | 23 +- .../dnn/src/layers/cpu_kernels/softmax.hpp | 4 +- modules/dnn/src/layers/matmul_layer.cpp | 2 - modules/dnn/src/layers/softmax_layer.cpp | 9 +- modules/dnn/src/net_impl.hpp | 8 + modules/dnn/src/net_impl2.cpp | 6 +- 13 files changed, 843 insertions(+), 14 deletions(-) create mode 100644 modules/dnn/src/graph_fusion_reshape_transpose.cpp create mode 100644 modules/dnn/src/graph_fusion_scale_softmax.cpp create mode 100644 modules/dnn/src/graph_fusion_shared_gemm.cpp create mode 100644 modules/dnn/src/graph_fusion_transpose_matmul.cpp diff --git a/modules/dnn/include/opencv2/dnn/all_layers.hpp b/modules/dnn/include/opencv2/dnn/all_layers.hpp index 129e9bb365..97c888c3ce 100644 --- a/modules/dnn/include/opencv2/dnn/all_layers.hpp +++ b/modules/dnn/include/opencv2/dnn/all_layers.hpp @@ -602,6 +602,7 @@ CV__DNN_INLINE_NS_BEGIN { public: bool logSoftMax; + float scale; static Ptr create(const LayerParams& params); }; @@ -1792,6 +1793,9 @@ CV__DNN_INLINE_NS_BEGIN class CV_EXPORTS MatMulLayer : public Layer { public: + bool trans_a; + bool trans_b; + static Ptr create(const LayerParams ¶ms); }; diff --git a/modules/dnn/src/graph_fusion_attention.cpp b/modules/dnn/src/graph_fusion_attention.cpp index 74cf1e7861..c9e2e591cc 100644 --- a/modules/dnn/src/graph_fusion_attention.cpp +++ b/modules/dnn/src/graph_fusion_attention.cpp @@ -60,6 +60,32 @@ struct ModelFusionAttention return dynamic_cast(prog[idx].get()) != nullptr; } + static bool isProjCandidate(const Ptr& l) + { + if (l->blobs.empty() || l->inputs.size() != 1) return false; + if (dynamic_cast(l.get())) + return true; + GemmLayer* g = dynamic_cast(l.get()); + if (!g) return false; + if (g->trans_a || g->alpha != 1.f) return false; + if (l->blobs.size() == 2 && g->beta != 1.f) return false; + return true; + } + + // Returns the projection weight in [K, N] (input_hidden, output_hidden) + // layout, transposing if the source is a Gemm with trans_b. + static Mat getProjWeight(const Ptr& l) + { + const Mat& W = l->blobs[0]; + GemmLayer* g = dynamic_cast(l.get()); + if (g && g->trans_b) { + Mat Wt; + cv::transpose(W, Wt); + return Wt; + } + return W; + } + bool isScalarBinOp(const vector>& prog, int idx, NaryEltwiseLayer::OPERATION op, float* val) const { @@ -251,10 +277,7 @@ struct ModelFusionAttention std::map> qkv_candidates; for (size_t i = 0; i < nops; i++) { if (!prog[i]) continue; - MatMulLayer* mm = dynamic_cast(prog[i].get()); - if (!mm) continue; - if (prog[i]->blobs.empty() || prog[i]->inputs.size() != 1) - continue; + if (!isProjCandidate(prog[i])) continue; Arg inp = prog[i]->inputs[0]; qkv_candidates[inp.idx].push_back((int)i); } @@ -406,9 +429,11 @@ struct ModelFusionAttention collectShapeChain(prog, it->second, extra_ops); } - const Mat& Wq = prog[indices[q_slot]]->blobs[0]; - const Mat& Wk = prog[indices[k_slot]]->blobs[0]; - const Mat& Wv = prog[indices[v_slot]]->blobs[0]; + // Normalize each projection weight to [K, N] regardless of + // whether the source op is MatMul or Gemm-with-trans_b. + Mat Wq = getProjWeight(prog[indices[q_slot]]); + Mat Wk = getProjWeight(prog[indices[k_slot]]); + Mat Wv = getProjWeight(prog[indices[v_slot]]); int input_hidden = Wq.size[0]; int q_hidden = Wq.size[1]; int k_hidden = Wk.size[1]; diff --git a/modules/dnn/src/graph_fusion_reshape_transpose.cpp b/modules/dnn/src/graph_fusion_reshape_transpose.cpp new file mode 100644 index 0000000000..a0eb6cf62e --- /dev/null +++ b/modules/dnn/src/graph_fusion_reshape_transpose.cpp @@ -0,0 +1,195 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// Copyright (C) 2026, BigVision LLC, all rights reserved. +// Third party copyrights are property of their respective owners. + +// Cleans up redundant Reshape/Transpose chains: +// - Reshape(A) -> Reshape(B): drop the inner reshape (A's shape is irrelevant +// once B reshapes to the final target). +// - Transpose(p1) -> Transpose(p2): compose into a single Transpose with +// perm[i] = p1[p2[i]]. +// - Identity Transpose (perm == [0,1,...,n-1]): bypass. +// These run iteratively, so a Reshape+Transpose+Reshape that ends up with an +// identity transpose collapses to a single Reshape, and adjacent transposes +// in attention layouts merge. + +#include "precomp.hpp" +#include "net_impl.hpp" + +namespace cv { namespace dnn { +CV__DNN_INLINE_NS_BEGIN + +using std::vector; +using std::string; + +struct ModelFusionReshapeTranspose +{ + explicit ModelFusionReshapeTranspose(Net::Impl* netimpl_) : netimpl(netimpl_) {} + + void fuse() + { + for (int iter = 0; iter < 10; iter++) { + netimpl->useCounts(usecounts); + if (!fuseGraph(netimpl->mainGraph)) + break; + } + } + + static bool isIdentityPerm(const vector& perm) + { + if (perm.empty()) return false; + for (size_t i = 0; i < perm.size(); i++) + if (perm[i] != (int)i) return false; + return true; + } + + bool fuseGraph(Ptr& graph) + { + const vector>& prog = graph->prog(); + size_t nops = prog.size(); + bool modified = false; + + for (size_t i = 0; i < nops; i++) { + if (!prog[i]) continue; + vector>* subgraphs = prog[i]->subgraphs(); + if (subgraphs) { + for (Ptr& g : *subgraphs) + if (fuseGraph(g)) modified = true; + } + } + + std::map producer; + std::set externalArgs; + for (Arg out : graph->outputs()) + externalArgs.insert(out.idx); + for (size_t i = 0; i < nops; i++) { + if (!prog[i]) continue; + for (Arg out : prog[i]->outputs) + producer[out.idx] = (int)i; + } + + vector dropped(nops, false); + + for (size_t i = 0; i < nops; i++) { + const Ptr& layer = prog[i]; + if (!layer || dropped[i]) continue; + if (layer->inputs.empty() || layer->outputs.empty()) continue; + + // Identity Transpose: bypass. + TransposeLayer* tr = dynamic_cast(layer.get()); + if (tr && layer->outputs.size() == 1 && isIdentityPerm(tr->perm)) { + Arg out = layer->outputs[0]; + if (externalArgs.count(out.idx) == 0) { + redirectConsumers(prog, dropped, i + 1, out, layer->inputs[0]); + dropped[i] = true; + usecounts[out.idx] = 0; + modified = true; + continue; + } + } + + // Transpose + Transpose -> Transpose (compose perms). + if (tr && layer->outputs.size() == 1) { + auto it = producer.find(layer->inputs[0].idx); + if (it != producer.end()) { + int prod_idx = it->second; + if (prod_idx >= 0 && !dropped[prod_idx]) { + const Ptr& pl = prog[prod_idx]; + TransposeLayer* prevTr = dynamic_cast(pl.get()); + Arg prevOut = layer->inputs[0]; + bool single_consumer = usecounts[prevOut.idx] == 1 + && externalArgs.count(prevOut.idx) == 0; + if (prevTr && pl->outputs.size() == 1 && single_consumer + && !prevTr->perm.empty() + && prevTr->perm.size() == tr->perm.size()) + { + // composed[i] = prevTr->perm[tr->perm[i]] + vector composed(tr->perm.size()); + bool ok = true; + for (size_t k = 0; k < tr->perm.size(); k++) { + int idx = tr->perm[k]; + if (idx < 0 || idx >= (int)prevTr->perm.size()) { + ok = false; break; + } + composed[k] = prevTr->perm[idx]; + } + if (ok) { + tr->perm = composed; + layer->inputs[0] = pl->inputs[0]; + dropped[prod_idx] = true; + usecounts[prevOut.idx] = 0; + producer[layer->outputs[0].idx] = (int)i; + modified = true; + continue; + } + } + } + } + } + + //Reshape + Reshape -> Reshape (drop the inner reshape). + Reshape2Layer* rs = dynamic_cast(layer.get()); + if (rs && layer->outputs.size() == 1) { + auto it = producer.find(layer->inputs[0].idx); + if (it != producer.end()) { + int prod_idx = it->second; + if (prod_idx >= 0 && !dropped[prod_idx]) { + const Ptr& pl = prog[prod_idx]; + Reshape2Layer* prevRs = dynamic_cast(pl.get()); + Arg prevOut = layer->inputs[0]; + bool single_consumer = usecounts[prevOut.idx] == 1 + && externalArgs.count(prevOut.idx) == 0; + if (prevRs && pl->outputs.size() == 1 && single_consumer) + { + layer->inputs[0] = pl->inputs[0]; + dropped[prod_idx] = true; + usecounts[prevOut.idx] = 0; + modified = true; + continue; + } + } + } + } + } + + if (modified) { + vector> newprog; + newprog.reserve(nops); + for (size_t i = 0; i < nops; i++) { + if (!dropped[i] && prog[i]) + newprog.push_back(prog[i]); + } + graph->setProg(newprog); + } + + return modified; + } + + void redirectConsumers(const vector>& prog, + const vector& dropped, + size_t start_idx, Arg from, Arg to) + { + for (size_t j = start_idx; j < prog.size(); j++) { + if (!prog[j] || dropped[j]) continue; + for (Arg& in : prog[j]->inputs) { + if (in.idx == from.idx) { + in = to; + usecounts[to.idx]++; + } + } + } + } + + Net::Impl* netimpl; + vector usecounts; +}; + +void Net::Impl::fuseReshapeTranspose() +{ + ModelFusionReshapeTranspose pass(this); + pass.fuse(); +} + +CV__DNN_INLINE_NS_END +}} diff --git a/modules/dnn/src/graph_fusion_scale_softmax.cpp b/modules/dnn/src/graph_fusion_scale_softmax.cpp new file mode 100644 index 0000000000..81c841064a --- /dev/null +++ b/modules/dnn/src/graph_fusion_scale_softmax.cpp @@ -0,0 +1,151 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// Copyright (C) 2026, BigVision LLC, all rights reserved. +// Third party copyrights are property of their respective owners. + +// Folds a scalar Mul / Div immediately preceding Softmax into the Softmax +// layer's `scale` parameter. The fused softmax bakes the scale into the +// per-element load before max/exp/sum/divide, removing one full pass over +// the (often large) [B, H, S, S] attention-logits tensor. +// +// Pattern: +// X -> Mul(X, c) -> Softmax => X -> Softmax(scale=c) +// X -> Div(X, c) -> Softmax => X -> Softmax(scale=1/c) + +#include "precomp.hpp" +#include "net_impl.hpp" + +namespace cv { namespace dnn { +CV__DNN_INLINE_NS_BEGIN + +using std::vector; +using std::string; + +struct ModelFusionScaleSoftmax +{ + explicit ModelFusionScaleSoftmax(Net::Impl* netimpl_) : netimpl(netimpl_) {} + + bool extractScalar(Arg a, float& out) const + { + if (!netimpl->isConstArg(a)) return false; + Mat t = netimpl->argTensor(a); + if (t.total() != 1) return false; + if (t.type() == CV_32F) { out = t.ptr()[0]; return true; } + if (t.type() == CV_64F) { out = (float)t.ptr()[0]; return true; } + return false; + } + + void fuse() { fuseGraph(netimpl->mainGraph); } + + bool fuseGraph(Ptr& graph) + { + const vector>& prog = graph->prog(); + size_t nops = prog.size(); + bool modified = false; + + for (size_t i = 0; i < nops; i++) { + if (!prog[i]) continue; + vector>* subgraphs = prog[i]->subgraphs(); + if (subgraphs) { + for (Ptr& g : *subgraphs) + if (fuseGraph(g)) modified = true; + } + } + + vector usecounts; + netimpl->useCounts(usecounts); + + std::set externalArgs; + for (Arg out : graph->outputs()) + externalArgs.insert(out.idx); + + std::map producer; + for (size_t i = 0; i < nops; i++) { + if (!prog[i]) continue; + for (Arg out : prog[i]->outputs) + producer[out.idx] = (int)i; + } + + vector dropped(nops, false); + + for (size_t i = 0; i < nops; i++) { + const Ptr& layer = prog[i]; + if (!layer || dropped[i]) continue; + + SoftmaxLayer* sm = dynamic_cast(layer.get()); + if (!sm || sm->logSoftMax) continue; + if (layer->inputs.size() != 1) continue; + + Arg sm_in = layer->inputs[0]; + auto it = producer.find(sm_in.idx); + if (it == producer.end()) continue; + int prod_idx = it->second; + if (prod_idx < 0 || dropped[prod_idx]) continue; + + const Ptr& pl = prog[prod_idx]; + NaryEltwiseLayer* elt = dynamic_cast(pl.get()); + if (!elt) continue; + const auto op = elt->op; + const bool is_mul = (op == NaryEltwiseLayer::OPERATION::PROD); + const bool is_div = (op == NaryEltwiseLayer::OPERATION::DIV); + if (!is_mul && !is_div) continue; + if (pl->inputs.size() != 2 || pl->outputs.size() != 1) continue; + + // Identify which operand is the scalar constant. + float scalar = 0.f; + int x_slot = -1; + if (extractScalar(pl->inputs[1], scalar)) { + x_slot = 0; + } else if (is_mul && extractScalar(pl->inputs[0], scalar)) { + // Mul is commutative; Div with the data on the right doesn't + // simplify (1 / scale*x ≠ softmax-fold) so only handle Mul here. + x_slot = 1; + } else { + continue; + } + + if (is_div) { + if (scalar == 0.f) continue; + scalar = 1.f / scalar; + } + if (!std::isfinite(scalar)) continue; + + bool single_consumer = usecounts[sm_in.idx] == 1 + && externalArgs.count(sm_in.idx) == 0; + if (!single_consumer) continue; + + sm->scale *= scalar; + layer->inputs[0] = pl->inputs[x_slot]; + dropped[prod_idx] = true; + usecounts[sm_in.idx] = 0; + modified = true; + } + + if (modified) { + vector> newprog; + newprog.reserve(nops); + for (size_t i = 0; i < nops; i++) { + if (!dropped[i] && prog[i]) + newprog.push_back(prog[i]); + } + graph->setProg(newprog); + } + + return modified; + } + + Net::Impl* netimpl; +}; + +void Net::Impl::fuseScaleSoftmax() +{ + if (preferableBackend != DNN_BACKEND_OPENCV || + preferableTarget != DNN_TARGET_CPU) + return; + ModelFusionScaleSoftmax pass(this); + pass.fuse(); +} + +CV__DNN_INLINE_NS_END +}} diff --git a/modules/dnn/src/graph_fusion_shared_gemm.cpp b/modules/dnn/src/graph_fusion_shared_gemm.cpp new file mode 100644 index 0000000000..4f0ab67947 --- /dev/null +++ b/modules/dnn/src/graph_fusion_shared_gemm.cpp @@ -0,0 +1,287 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// Copyright (C) 2026, BigVision LLC, all rights reserved. +// Third party copyrights are property of their respective owners. + +// Fuses multiple Gemm layers that share the same input into one wider Gemm. +// +// Pattern: +// X --+--> Gemm[W0, b0] -> Y0 +// +--> Gemm[W1, b1] -> Y1 +// +--> Gemm[W2, b2] -> Y2 +// ... +// +// Replacement: +// X -> Gemm[concat(Wi), concat(bi)] -> Y_concat +// Y_concat -> Slice2(0, N0) -> Y0 +// -> Slice2(N0, N0+N1) -> Y1 +// -> Slice2(N0+N1, ...) -> Y2 +// ... +// +// Triggers when: +// - At least 2 Gemm layers share the same input arg. +// - Each Gemm has a const weight blob and (optionally) const bias. +// - All Gemms have identical (alpha, beta, trans_a, trans_b) and matching K. + +#include "precomp.hpp" +#include "net_impl.hpp" + +namespace cv { namespace dnn { +CV__DNN_INLINE_NS_BEGIN + +using std::vector; +using std::string; + +namespace { + +static bool readGemmWeight(const Ptr& l, bool trans_b, Mat& W_out) +{ + if (l->blobs.empty()) return false; + const Mat& W = l->blobs[0]; + if (W.dims != 2 || W.type() != CV_32F) return false; + if (trans_b) { + cv::transpose(W, W_out); + } else { + W.copyTo(W_out); + } + return true; +} + +static bool readGemmBias(const Ptr& l, Mat& b_out) +{ + if (l->blobs.size() < 2) { b_out.release(); return true; } + const Mat& b = l->blobs[1]; + if (b.type() != CV_32F) return false; + b.copyTo(b_out); + return true; +} +} + +struct ModelFusionSharedGemm +{ + explicit ModelFusionSharedGemm(Net::Impl* netimpl_) : netimpl(netimpl_) {} + + struct GemmInfo + { + int layer_idx = -1; + Arg input; + int K = 0; + int N = 0; + bool trans_a = false; + bool trans_b = false; + float alpha = 1.f; + float beta = 1.f; + bool has_bias = false; + }; + + bool inspectGemm(const vector>& prog, int idx, GemmInfo& info) const + { + if (idx < 0 || idx >= (int)prog.size() || !prog[idx]) return false; + const Ptr& l = prog[idx]; + GemmLayer* g = dynamic_cast(l.get()); + if (!g) return false; + + if (l->inputs.size() != 1) return false; + if (l->blobs.empty() || l->blobs.size() > 2) return false; + + const Mat& W = l->blobs[0]; + if (W.dims != 2 || W.type() != CV_32F) return false; + + bool trans_a = g->trans_a; + bool trans_b = g->trans_b; + float alpha = g->alpha; + float beta = g->beta; + + if (trans_a) return false; + if (alpha != 1.f) return false; + if (l->blobs.size() == 2 && beta != 1.f) return false; + + int K = trans_b ? W.size[1] : W.size[0]; + int N = trans_b ? W.size[0] : W.size[1]; + if (K <= 0 || N <= 0) return false; + + if (l->blobs.size() == 2) { + const Mat& b = l->blobs[1]; + if (b.type() != CV_32F) return false; + if ((int)b.total() != N) return false; + } + + info.layer_idx = idx; + info.input = l->inputs[0]; + info.K = K; + info.N = N; + info.trans_a = trans_a; + info.trans_b = trans_b; + info.alpha = alpha; + info.beta = beta; + info.has_bias = (l->blobs.size() == 2); + return true; + } + + void fuse() { fuseGraph(netimpl->mainGraph); } + + bool fuseGraph(Ptr& graph) + { + const vector>& prog = graph->prog(); + size_t nops = prog.size(); + + struct Key { int input_idx; int trans_b; int K; }; + auto keyLess = [](const Key& a, const Key& b) { + if (a.input_idx != b.input_idx) return a.input_idx < b.input_idx; + if (a.trans_b != b.trans_b) return a.trans_b < b.trans_b; + return a.K < b.K; + }; + std::map, decltype(keyLess)> groups(keyLess); + + for (size_t i = 0; i < nops; i++) { + GemmInfo info; + if (!inspectGemm(prog, (int)i, info)) continue; + Key k{info.input.idx, info.trans_b ? 1 : 0, info.K}; + groups[k].push_back(info); + } + + bool modified = false; + std::set removed_ops; + vector>>> insertions; // (insert_pos, fused-and-slice layers) + + for (auto& [key, infos] : groups) { + if (infos.size() < 2) continue; + + bool all_have_bias = infos[0].has_bias; + bool uniform = true; + for (auto& info : infos) + if (info.has_bias != all_have_bias) { uniform = false; break; } + if (!uniform) continue; + + int K = infos[0].K; + int total_N = 0; + for (auto& info : infos) total_N += info.N; + + int wshape[] = { K, total_N }; + Mat W_concat(2, wshape, CV_32F); + int col_offset = 0; + for (auto& info : infos) { + Mat W; + if (!readGemmWeight(prog[info.layer_idx], info.trans_b, W)) { + uniform = false; break; + } + CV_Assert(W.size[0] == K && W.size[1] == info.N); + for (int r = 0; r < K; r++) { + float* dst = W_concat.ptr(r) + col_offset; + memcpy(dst, W.ptr(r), info.N * sizeof(float)); + } + col_offset += info.N; + } + if (!uniform) continue; + + Mat b_concat; + if (all_have_bias) { + int bshape[] = { total_N }; + b_concat.create(1, bshape, CV_32F); + int b_offset = 0; + for (auto& info : infos) { + Mat b; + if (!readGemmBias(prog[info.layer_idx], b)) { + uniform = false; break; + } + CV_Assert((int)b.total() == info.N); + memcpy(b_concat.ptr() + b_offset, b.ptr(), + info.N * sizeof(float)); + b_offset += info.N; + } + if (!uniform) continue; + } + + int insert_pos = (int)nops; + for (auto& info : infos) insert_pos = std::min(insert_pos, info.layer_idx); + + LayerParams fp; + fp.name = prog[infos[0].layer_idx]->name + "_shared_input_fused"; + fp.type = "Gemm"; + fp.set("transA", false); + fp.set("transB", false); + fp.set("alpha", 1.f); + fp.set("beta", 1.f); + fp.blobs.push_back(W_concat); + if (all_have_bias) fp.blobs.push_back(b_concat); + + Ptr fused = LayerFactory::createLayerInstance("Gemm", fp); + if (!fused) continue; + + string fused_out_name = fp.name + "_out"; + Arg fused_out_arg = netimpl->getArg(fused_out_name); + fused->inputs = { infos[0].input }; + fused->outputs = { fused_out_arg }; + fused->netimpl = netimpl; + + vector> slices; + int col_cursor = 0; + for (size_t s = 0; s < infos.size(); s++) { + int N_s = infos[s].N; + int begin = col_cursor; + int end = col_cursor + N_s; + col_cursor += N_s; + + LayerParams sp; + sp.name = prog[infos[s].layer_idx]->name + "_shared_input_slice"; + sp.type = "Slice2"; + + Mat starts(1, 1, CV_32S); starts.at(0) = begin; + Mat ends (1, 1, CV_32S); ends.at(0) = end; + Mat axes (1, 1, CV_32S); axes.at(0) = -1; // last axis + Mat steps (1, 1, CV_32S); steps.at(0) = 1; + + Arg starts_arg = netimpl->newConstArg(sp.name + "_starts", starts); + Arg ends_arg = netimpl->newConstArg(sp.name + "_ends", ends); + Arg axes_arg = netimpl->newConstArg(sp.name + "_axes", axes); + Arg steps_arg = netimpl->newConstArg(sp.name + "_steps", steps); + + Ptr slice = LayerFactory::createLayerInstance("Slice2", sp); + if (!slice) { uniform = false; break; } + slice->inputs = { fused_out_arg, starts_arg, ends_arg, axes_arg, steps_arg }; + slice->outputs = prog[infos[s].layer_idx]->outputs; + slice->netimpl = netimpl; + slices.push_back(slice); + } + if (!uniform) continue; + + for (auto& info : infos) removed_ops.insert(info.layer_idx); + vector> bundle; + bundle.push_back(fused); + for (auto& s : slices) bundle.push_back(s); + insertions.emplace_back(insert_pos, std::move(bundle)); + modified = true; + } + + if (!modified) return false; + + std::sort(insertions.begin(), insertions.end(), + [](auto& a, auto& b) { return a.first < b.first; }); + + vector> newprog; + size_t ins_idx = 0; + for (size_t i = 0; i < nops; i++) { + while (ins_idx < insertions.size() && + insertions[ins_idx].first == (int)i) { + for (auto& l : insertions[ins_idx].second) newprog.push_back(l); + ins_idx++; + } + if (removed_ops.count((int)i)) continue; + newprog.push_back(prog[i]); + } + graph->setProg(newprog); + return true; + } + + Net::Impl* netimpl; +}; + +void Net::Impl::fuseSharedInputGemm() +{ + ModelFusionSharedGemm pass(this); + pass.fuse(); +} + +CV__DNN_INLINE_NS_END +}} diff --git a/modules/dnn/src/graph_fusion_transpose_matmul.cpp b/modules/dnn/src/graph_fusion_transpose_matmul.cpp new file mode 100644 index 0000000000..0b13cc99bf --- /dev/null +++ b/modules/dnn/src/graph_fusion_transpose_matmul.cpp @@ -0,0 +1,127 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// Copyright (C) 2026, BigVision LLC, all rights reserved. +// Third party copyrights are property of their respective owners. + +// Folds a Transpose-of-last-two-dims into the consuming MatMul's trans_a / trans_b. +// +// Pattern: +// X -> Transpose(perm=[0,..,n-3, n-1, n-2]) -> MatMul(., Y) +// => X -> MatMul(trans_a=true, ., Y) +// +// Symmetric on the second operand -> trans_b. + +#include "precomp.hpp" +#include "net_impl.hpp" + +namespace cv { namespace dnn { +CV__DNN_INLINE_NS_BEGIN + +using std::vector; +using std::string; + +struct ModelFusionTransposeMatMul +{ + explicit ModelFusionTransposeMatMul(Net::Impl* netimpl_) : netimpl(netimpl_) {} + + void fuse() { fuseGraph(netimpl->mainGraph); } + + static bool isLastTwoSwap(const vector& perm) + { + int n = (int)perm.size(); + if (n < 2) return false; + for (int i = 0; i < n - 2; i++) + if (perm[i] != i) return false; + return perm[n - 2] == n - 1 && perm[n - 1] == n - 2; + } + + bool fuseGraph(Ptr& graph) + { + const vector>& prog = graph->prog(); + size_t nops = prog.size(); + bool modified = false; + + for (size_t i = 0; i < nops; i++) { + if (!prog[i]) continue; + vector>* subgraphs = prog[i]->subgraphs(); + if (subgraphs) { + for (Ptr& g : *subgraphs) + if (fuseGraph(g)) modified = true; + } + } + + vector usecounts; + netimpl->useCounts(usecounts); + + std::set externalArgs; + for (Arg out : graph->outputs()) + externalArgs.insert(out.idx); + + std::map producer; + for (size_t i = 0; i < nops; i++) { + if (!prog[i]) continue; + for (Arg out : prog[i]->outputs) + producer[out.idx] = (int)i; + } + + vector dropped(nops, false); + + for (size_t i = 0; i < nops; i++) { + const Ptr& layer = prog[i]; + if (!layer || dropped[i]) continue; + + MatMulLayer* mm = dynamic_cast(layer.get()); + if (!mm) continue; + + if (layer->inputs.size() != 2) continue; + + for (int slot = 0; slot < 2; slot++) { + Arg in = layer->inputs[slot]; + auto it = producer.find(in.idx); + if (it == producer.end()) continue; + int prod_idx = it->second; + if (prod_idx < 0 || dropped[prod_idx]) continue; + + const Ptr& pl = prog[prod_idx]; + TransposeLayer* tr = dynamic_cast(pl.get()); + if (!tr || pl->outputs.size() != 1) continue; + if (!isLastTwoSwap(tr->perm)) continue; + + bool single_consumer = usecounts[in.idx] == 1 + && externalArgs.count(in.idx) == 0; + if (!single_consumer) continue; + + if (slot == 0) mm->trans_a = !mm->trans_a; + else mm->trans_b = !mm->trans_b; + + layer->inputs[slot] = pl->inputs[0]; + dropped[prod_idx] = true; + usecounts[in.idx] = 0; + modified = true; + } + } + + if (modified) { + vector> newprog; + newprog.reserve(nops); + for (size_t i = 0; i < nops; i++) { + if (!dropped[i] && prog[i]) + newprog.push_back(prog[i]); + } + graph->setProg(newprog); + } + return modified; + } + + Net::Impl* netimpl; +}; + +void Net::Impl::fuseTransposeMatMul() +{ + ModelFusionTransposeMatMul pass(this); + pass.fuse(); +} + +CV__DNN_INLINE_NS_END +}} diff --git a/modules/dnn/src/layers/conv2_layer.cpp b/modules/dnn/src/layers/conv2_layer.cpp index 7c9a3fad46..cde386b5f2 100644 --- a/modules/dnn/src/layers/conv2_layer.cpp +++ b/modules/dnn/src/layers/conv2_layer.cpp @@ -286,6 +286,8 @@ public: desiredInputs[0] = DATA_LAYOUT_BLOCK; for (size_t i = 1; i < ninputs; i++) desiredInputs[i] = DATA_LAYOUT_UNKNOWN; + if (addResidual && ninputs >= 2) + desiredInputs[ninputs - 1] = DATA_LAYOUT_BLOCK; outputs.assign(requiredOutputs, DATA_LAYOUT_BLOCK); return getNetImpl(this)->defaultC0; } diff --git a/modules/dnn/src/layers/cpu_kernels/softmax.cpp b/modules/dnn/src/layers/cpu_kernels/softmax.cpp index e07be85e51..7ff7022810 100644 --- a/modules/dnn/src/layers/cpu_kernels/softmax.cpp +++ b/modules/dnn/src/layers/cpu_kernels/softmax.cpp @@ -15,11 +15,12 @@ namespace cv { namespace dnn { -void softmax(Mat &dst, const Mat &src, int axis, int axisBias, int axisStep){ +void softmax(Mat &dst, const Mat &src, int axis, int axisBias, int axisStep, float scale){ CV_Assert(src.type() == CV_32F); CV_Assert(src.isContinuous() && dst.isContinuous()); CV_Assert(src.size == dst.size); axis = normalize_axis(axis, src.dims); + const bool scaled = scale != 1.f; size_t outerSize = src.total(0, axis), innerSize = src.total(axis + 1); @@ -61,6 +62,22 @@ void softmax(Mat &dst, const Mat &src, int axis, int axisBias, int axisStep){ for (; _cnDim < axisStep; _cnDim++) axisBuf[_cnDim] = srcPtr[srcOffset + (_cnDim + axisBias) * cnStep]; + // Bake the fused pre-Softmax scale into the buffer once. axisBuf + // already lives in L1 from the load above, so this pass is short. + if (scaled) { + int sk = 0; +#if (CV_SIMD || CV_SIMD_SCALABLE) + v_float32 vscale = vx_setall_f32(scale); + for (; sk <= axisStep - nlanes; sk += nlanes) { + v_float32 val = vx_load(axisBuf + sk); + val = v_mul(val, vscale); + v_store(axisBuf + sk, val); + } +#endif + for (; sk < axisStep; sk++) + axisBuf[sk] *= scale; + } + float maxVal = -FLT_MAX; int cnDim = 0; #if (CV_SIMD || CV_SIMD_SCALABLE) @@ -128,6 +145,10 @@ void softmax(Mat &dst, const Mat &src, int axis) { softmax(dst, src, axis, 0, src.size[axis]); } +void softmax(Mat &dst, const Mat &src, int axis, float scale) { + softmax(dst, src, axis, 0, src.size[axis], scale); +} + void logSoftmax(Mat &dst, const Mat &src, int axis) { softmax(dst, src, axis); log(dst, dst); diff --git a/modules/dnn/src/layers/cpu_kernels/softmax.hpp b/modules/dnn/src/layers/cpu_kernels/softmax.hpp index 19a89fa878..8ba8ba0560 100644 --- a/modules/dnn/src/layers/cpu_kernels/softmax.hpp +++ b/modules/dnn/src/layers/cpu_kernels/softmax.hpp @@ -17,10 +17,12 @@ namespace cv { namespace dnn { -void softmax(Mat &dst, const Mat &src, int axis, int axisBias, int axisStep); +void softmax(Mat &dst, const Mat &src, int axis, int axisBias, int axisStep, float scale = 1.f); void softmax(Mat &dst, const Mat &src, int axis); +void softmax(Mat &dst, const Mat &src, int axis, float scale); + void logSoftmax(Mat &dst, const Mat &src, int axis); }} // cv::dnn diff --git a/modules/dnn/src/layers/matmul_layer.cpp b/modules/dnn/src/layers/matmul_layer.cpp index 384d69bde5..e0c70ffc65 100644 --- a/modules/dnn/src/layers/matmul_layer.cpp +++ b/modules/dnn/src/layers/matmul_layer.cpp @@ -503,8 +503,6 @@ class MatMulLayerImpl CV_FINAL : public MatMulLayer { #endif // HAVE_CANN private: - bool trans_a; - bool trans_b; float alpha; float beta; diff --git a/modules/dnn/src/layers/softmax_layer.cpp b/modules/dnn/src/layers/softmax_layer.cpp index 2bce8d60f8..bf0f1ce477 100644 --- a/modules/dnn/src/layers/softmax_layer.cpp +++ b/modules/dnn/src/layers/softmax_layer.cpp @@ -77,6 +77,7 @@ public: { axisRaw = params.get("axis", -1); logSoftMax = params.get("log_softmax", false); + scale = params.get("scale", 1.f); setParamsFrom(params); } @@ -231,10 +232,14 @@ public: Mat &dst = outputs[0]; int axis = normalize_axis(axisRaw, src.dims); - if(logSoftMax) + if (logSoftMax) { + CV_Assert(scale == 1.f); logSoftmax(dst, src, axis); - else + } else if (scale != 1.f) { + softmax(dst, src, axis, scale); + } else { softmax(dst, src, axis); + } } #ifdef HAVE_CUDA diff --git a/modules/dnn/src/net_impl.hpp b/modules/dnn/src/net_impl.hpp index b8b6c0356c..a7b453ebbe 100644 --- a/modules/dnn/src/net_impl.hpp +++ b/modules/dnn/src/net_impl.hpp @@ -456,6 +456,14 @@ struct Net::Impl : public detail::NetImplBase void fuseBasic(); // fuse ViT-style multi-head attention subgraphs void fuseAttention(); + // fuse Gemm layers that share the same input into one wider Gemm + void fuseSharedInputGemm(); + // collapse redundant Reshape/Transpose chains + void fuseReshapeTranspose(); + // absorb a last-two-dims Transpose into the consuming MatMul + void fuseTransposeMatMul(); + // fold a scalar Mul/Div before Softmax into Softmax::scale (CPU only) + void fuseScaleSoftmax(); // replace constant sub-expressions with their results void fuseQDQ(); diff --git a/modules/dnn/src/net_impl2.cpp b/modules/dnn/src/net_impl2.cpp index d8490e59dd..69ad9830bf 100644 --- a/modules/dnn/src/net_impl2.cpp +++ b/modules/dnn/src/net_impl2.cpp @@ -549,8 +549,12 @@ void Net::Impl::prepareForInference() constFold(); constArgs(); fuseAttention(); - useBlockLayout(); + fuseSharedInputGemm(); + fuseReshapeTranspose(); + fuseTransposeMatMul(); + fuseScaleSoftmax(); fuseBasic(); + useBlockLayout(); assignBuffers(); totalLayers = updateGraphOfs(mainGraph, 0, true); prepared = true;