From c83b86eb570644c45d9058674d3d23c52a654a66 Mon Sep 17 00:00:00 2001 From: Abhishek Gola Date: Tue, 28 Apr 2026 20:08:29 +0530 Subject: [PATCH] Merge pull request #28811 from abhishek-gola:qlinear_support Added QLinear layer support #28811 closes: https://github.com/opencv/opencv/issues/26310 ### Pull Request Readiness Checklist See details at https://github.com/opencv/opencv/wiki/How_to_contribute#making-a-good-pull-request - [x] I agree to contribute to the project under Apache 2 License. - [x] To the best of my knowledge, the proposed patch is not based on a code under GPL or another license that is incompatible with OpenCV - [x] The PR is proposed to the proper branch - [x] There is a reference to the original bug report and related work - [x] There is accuracy test, performance test and test data in opencv_extra repository, if applicable Patch to opencv_extra has the same branch name. - [x] The feature is well documented and sample code can be built with the project CMake --- .../dnn/include/opencv2/dnn/all_layers.hpp | 97 +++ modules/dnn/include/opencv2/dnn/dnn.hpp | 7 + modules/dnn/src/graph_fusion_qdq.cpp | 688 +++++++++++----- .../dnn/src/int8layers/conv2_int8_layer.cpp | 35 + .../dnn/src/int8layers/elementwise_layers.cpp | 17 +- .../src/int8layers/eltwise2_int8_layer.cpp | 17 + .../src/int8layers/fully_connected_layer.cpp | 41 + .../dnn/src/int8layers/matmul_int8_layer.cpp | 296 +++++++ .../dnn/src/int8layers/pool2_int8_layer.cpp | 24 + modules/dnn/src/layer.cpp | 9 +- modules/dnn/src/layers/concat_layer.cpp | 2 + .../dnn/src/layers/dequantizelinear_layer.cpp | 7 +- modules/dnn/src/layers/flatten_layer.cpp | 2 + modules/dnn/src/layers/maxpool_layer.cpp | 2 + modules/dnn/src/layers/permute_layer.cpp | 2 + .../dnn/src/layers/quantlizelinear_layer.cpp | 7 +- modules/dnn/src/layers/reshape2_layer.cpp | 2 + modules/dnn/src/layers/reshape_layer.cpp | 2 + modules/dnn/src/layers/slice2_layer.cpp | 2 + modules/dnn/src/layers/slice_layer.cpp | 2 + modules/dnn/src/layers/split2_layer.cpp | 2 + modules/dnn/src/layers/split_layer.cpp | 2 + modules/dnn/src/layers/squeeze_layer.cpp | 2 + modules/dnn/src/layers/tile2_layer.cpp | 2 + modules/dnn/src/layers/tile_layer.cpp | 2 + modules/dnn/src/layers/transpose_layer.cpp | 2 + modules/dnn/src/layers/unsqueeze_layer.cpp | 2 + modules/dnn/src/onnx/onnx_importer2.cpp | 767 ++++++------------ ...conformance_layer_filter__openvino.inl.hpp | 6 +- ...yer_filter_opencv_classic_denylist.inl.hpp | 3 + ..._conformance_layer_parser_denylist.inl.hpp | 7 +- modules/dnn/test/test_onnx_importer.cpp | 18 +- 32 files changed, 1341 insertions(+), 735 deletions(-) create mode 100644 modules/dnn/src/int8layers/matmul_int8_layer.cpp diff --git a/modules/dnn/include/opencv2/dnn/all_layers.hpp b/modules/dnn/include/opencv2/dnn/all_layers.hpp index c7ad885a1a..129e9bb365 100644 --- a/modules/dnn/include/opencv2/dnn/all_layers.hpp +++ b/modules/dnn/include/opencv2/dnn/all_layers.hpp @@ -382,10 +382,28 @@ CV__DNN_INLINE_NS_BEGIN bool ceil_mode; }; + struct Conv2Int8Params + { + String name; + std::vector strides, dilations, pads; + int ngroups = 1; + AutoPadding auto_pad = AUTO_PAD_NONE; + bool ceil_mode = false; + float input_sc = 1.f; + int input_zp = 0; + float output_sc = 1.f; + int output_zp = 0; + bool per_channel = true; + bool input_is_u8 = false; + // blobs[0] = quantized weights, blobs[1] = fused bias, blobs[2] = output multiplier + Mat weights, bias, outputMultiplier; + }; + class CV_EXPORTS Conv2Int8Layer : public Layer { public: static Ptr create(const LayerParams& params); + static Ptr create(const Conv2Int8Params& params); int input_zp, output_zp; float input_sc, output_sc; @@ -487,6 +505,21 @@ CV__DNN_INLINE_NS_BEGIN static Ptr create(const LayerParams& params); }; + struct Pool2Int8Params + { + String name; + std::vector kernel_shape, strides, dilations, pads; + AutoPadding auto_pad = AUTO_PAD_NONE; + bool ceil_mode = false; + bool is_global_pooling = false; + bool is_max_pool = true; + bool count_include_pad = false; + float input_sc = 1.f; + int input_zp = 0; + float output_sc = 1.f; + int output_zp = 0; + }; + // New-engine int8 pooling with block memory layout (DATA_LAYOUT_BLOCK). // Created by the QDQ graph fusion pass (graph_fusion_qdq.cpp) when it // detects a DequantizeLinear -> Pooling -> QuantizeLinear pattern. @@ -495,6 +528,7 @@ CV__DNN_INLINE_NS_BEGIN { public: static Ptr create(const LayerParams& params); + static Ptr create(const Pool2Int8Params& params); int input_zp, output_zp; float input_sc, output_sc; @@ -591,6 +625,21 @@ CV__DNN_INLINE_NS_BEGIN static Ptr create(const LayerParams& params); }; + struct InnerProductInt8Params + { + String name; + int axis = 1; + int num_output = 0; + float input_sc = 1.f; + int input_zp = 0; + float output_sc = 1.f; + int output_zp = 0; + int output_type = CV_8S; + bool per_channel = true; + // blobs[0] = weights, blobs[1] = bias, blobs[2] = output multiplier + Mat weights, bias, outputMultiplier; + }; + class CV_EXPORTS InnerProductLayerInt8 : public InnerProductLayer { public: @@ -602,6 +651,7 @@ CV__DNN_INLINE_NS_BEGIN // of per-Channel quantization. Otherwise, that means this layer contains per-Tensor quantized parameters. bool per_channel; static Ptr create(const LayerParams& params); + static Ptr create(const InnerProductInt8Params& params); }; class CV_EXPORTS MVNLayer : public Layer @@ -1239,6 +1289,17 @@ CV__DNN_INLINE_NS_BEGIN static Ptr create(const LayerParams ¶ms); }; + struct ActivationInt8Params + { + String name; + String activationType; // "ReLUInt8", "SigmoidInt8", etc. + float input_sc = 1.f; + int input_zp = 0; + float output_sc = 1.f; + int output_zp = 0; + Mat activationLUT; + }; + class CV_EXPORTS ActivationLayerInt8 : public ActivationLayer { public: @@ -1247,6 +1308,7 @@ CV__DNN_INLINE_NS_BEGIN Mat activationLUT; static Ptr create(const LayerParams ¶ms); + static Ptr create(const ActivationInt8Params ¶ms); }; class CV_EXPORTS SignLayer : public ActivationLayer @@ -1303,10 +1365,21 @@ CV__DNN_INLINE_NS_BEGIN static Ptr create(const LayerParams ¶ms); }; + struct Eltwise2Int8Params + { + String name; + std::vector input_scales; + std::vector input_zeropoints; + float output_sc = 1.f; + int output_zp = 0; + bool with_relu = false; + }; + class CV_EXPORTS Eltwise2Int8Layer : public Layer { public: static Ptr create(const LayerParams& params); + static Ptr create(const Eltwise2Int8Params& params); std::vector scales; std::vector zeropoints; @@ -1722,6 +1795,30 @@ CV__DNN_INLINE_NS_BEGIN static Ptr create(const LayerParams ¶ms); }; + struct MatMulInt8Params + { + String name; + int num_output = 0; + int inp_dims = 2; + float input_sc = 1.f; + int input_zp = 0; + float output_sc = 1.f; + int output_zp = 0; + int output_type = CV_8S; + bool per_channel = true; + Mat weights, bias, outputMultiplier; + }; + + class CV_EXPORTS MatMulInt8Layer : public MatMulLayer + { + public: + int input_zp, output_zp; + float input_sc, output_sc; + int output_type; + bool per_channel; + static Ptr create(const MatMulInt8Params& params); + }; + class CV_EXPORTS ExpandLayer : public Layer { public: diff --git a/modules/dnn/include/opencv2/dnn/dnn.hpp b/modules/dnn/include/opencv2/dnn/dnn.hpp index 06c21e2266..bac8032481 100644 --- a/modules/dnn/include/opencv2/dnn/dnn.hpp +++ b/modules/dnn/include/opencv2/dnn/dnn.hpp @@ -495,6 +495,13 @@ CV__DNN_INLINE_NS_BEGIN // so that forward() can assume that the outputs are already allocated. virtual bool dynamicOutputShapes() const; + // returns true if the layer only rearranges data without changing values. + // Examples: Flatten, Reshape, Transpose, Permute, Squeeze, Unsqueeze, + // Concat, Split, Slice, Tile, MaxPool. + // Used by QDQ fusion to elide redundant dequantize-quantize pairs + // when the scale and zero point are the same. + virtual bool isDataShuffling() const; + // dumps attributes of the layer (e.g. strides, dilations in Convolution, MaxPool) virtual std::ostream& dumpAttrs(std::ostream& strm, int indent) const; diff --git a/modules/dnn/src/graph_fusion_qdq.cpp b/modules/dnn/src/graph_fusion_qdq.cpp index a22e6d1608..4ac53fdefa 100644 --- a/modules/dnn/src/graph_fusion_qdq.cpp +++ b/modules/dnn/src/graph_fusion_qdq.cpp @@ -78,6 +78,17 @@ struct ModelFusionQDQ { p->output_sc = sc; p->output_zp = zp; } } + bool areDqArgsConst(const DequantizeLinearLayer* dq) const + { + if (!dq || dq->inputs.size() < 2) + return false; + if (!netimpl->isConstArg(dq->inputs[1])) + return false; + if (dq->inputs.size() >= 3 && !netimpl->isConstArg(dq->inputs[2])) + return false; + return true; + } + template bool getQdqPatternContext(Layer* layer_ptr, size_t ninputs, @@ -96,6 +107,8 @@ struct ModelFusionQDQ q_data_in = inputs[0]; out_scale = inputs[1]; out_zp = inputs[2]; + if (!netimpl->isConstArg(out_scale) || !netimpl->isConstArg(out_zp)) + return false; mid_layer_idx = producer_of.at(q_data_in.idx); mid_layer = getLayer(newprog, mid_layer_idx); return mid_layer != 0; @@ -152,7 +165,8 @@ struct ModelFusionQDQ getLayer(newprog, dq_idx); if (!dq || dq->inputs.size() < 3 || - usecounts.at(add_inp.idx) != 1) { + usecounts.at(add_inp.idx) != 1 || + !areDqArgsConst(dq)) { break; } dq_ptrs.push_back(dq); @@ -184,27 +198,25 @@ struct ModelFusionQDQ ? (int)elt_out_zp_m.at(0) : (int)elt_out_zp_m.at(0); - LayerParams eltwiseParams = makeLayerParamsFromOriginal(add, "Eltwise2Int8"); - eltwiseParams.blobs.clear(); - eltwiseParams.set("input_scales", DictValue::arrayReal(in_scales.data(), (int)in_scales.size())); - eltwiseParams.set("input_zeropoints", DictValue::arrayInt(in_zps.data(), (int)in_zps.size())); - eltwiseParams.set("scales", out_scale_val); - eltwiseParams.set("zeropoints", out_zp_val); - Ptr eltwiseInt8 = createFusedLayer(eltwiseParams); - if (!eltwiseInt8.empty()) { - CV_Assert(dynamic_cast(eltwiseInt8.get())); - fused_layer_idx = add_layer_idx; - newprog[add_layer_idx] = eltwiseInt8; - fused_inputs.swap(int8_inputs); - removed_args.push_back(q_data_in); // float add_out - for (const Arg& add_inp : add->inputs) - removed_args.push_back(add_inp); + Eltwise2Int8Params ep; + ep.name = add->name; + ep.input_scales = in_scales; + ep.input_zeropoints = in_zps; + ep.output_sc = out_scale_val; + ep.output_zp = out_zp_val; + Ptr eltwiseInt8 = Eltwise2Int8Layer::create(ep); + eltwiseInt8->netimpl = netimpl; + fused_layer_idx = add_layer_idx; + newprog[add_layer_idx] = eltwiseInt8; + fused_inputs.swap(int8_inputs); + removed_args.push_back(q_data_in); // float add_out + for (const Arg& add_inp : add->inputs) + removed_args.push_back(add_inp); - for (int dq_prog_idx : dq_prog_indices) - newprog[dq_prog_idx] = Ptr(); + for (int dq_prog_idx : dq_prog_indices) + newprog[dq_prog_idx] = Ptr(); - break; - } + break; } } @@ -222,6 +234,7 @@ struct ModelFusionQDQ const int relu_in_type = (dq && !dq->inputs.empty()) ? netimpl->argData(dq->inputs[0]).type : -1; const bool relu_in_int8 = (relu_in_type == CV_8S || relu_in_type == CV_8U); if (dq && dq->inputs.size() >= 3 && + areDqArgsConst(dq) && relu_in_int8 && relu_out_int8 && usecounts.at(relu_in.idx) == 1) { const float inp_sc = netimpl->argTensor(dq->inputs[1]).at(0); @@ -256,25 +269,23 @@ struct ModelFusionQDQ } } - LayerParams reluInt8Params = makeLayerParamsFromOriginal(relu, "ReLUInt8"); - reluInt8Params.blobs.clear(); - Ptr reluInt8 = createFusedLayer(reluInt8Params); - if (!reluInt8.empty()) { - auto* reluInt8Layer = dynamic_cast(reluInt8.get()); - CV_Assert(reluInt8Layer); - reluInt8Layer->input_sc = inp_sc; - reluInt8Layer->input_zp = inp_zp; - reluInt8Layer->output_sc = out_sc; - reluInt8Layer->output_zp = out_zp_i; - reluInt8Layer->activationLUT = lookUpTable; - fused_layer_idx = relu_layer_idx; - newprog[relu_layer_idx] = reluInt8; - fused_inputs.assign(1, dq->inputs[0]); - removed_args.push_back(q_data_in); - removed_args.push_back(relu_in); - newprog[dq_idx] = Ptr(); - break; - } + ActivationInt8Params ap; + ap.name = relu->name; + ap.activationType = "ReLUInt8"; + ap.input_sc = inp_sc; + ap.input_zp = inp_zp; + ap.output_sc = out_sc; + ap.output_zp = out_zp_i; + ap.activationLUT = lookUpTable; + Ptr reluInt8 = ActivationLayerInt8::create(ap); + reluInt8->netimpl = netimpl; + fused_layer_idx = relu_layer_idx; + newprog[relu_layer_idx] = reluInt8; + fused_inputs.assign(1, dq->inputs[0]); + removed_args.push_back(q_data_in); + removed_args.push_back(relu_in); + newprog[dq_idx] = Ptr(); + break; } } } @@ -299,8 +310,7 @@ struct ModelFusionQDQ const Arg& add_inp = add2->inputs[k]; int dq_idx2 = producer_of.at(add_inp.idx); DequantizeLinearLayer* dq2 = getLayer(newprog, dq_idx2); - - if (dq2 && dq2->inputs.size() >= 3) { + if (dq2 && dq2->inputs.size() >= 3 && areDqArgsConst(dq2)) { dq_ptrs2.push_back(dq2); dq_prog_indices2.push_back(dq_idx2); int8_inputs2.push_back(dq2->inputs[0]); @@ -326,7 +336,7 @@ struct ModelFusionQDQ } int elt_out_type2 = !outputs.empty() ? netimpl->argData(outputs[0]).type : -1; - if (elt_out_type2 < 0) { + if (elt_out_type2 < 0 && netimpl->isConstArg(inputs[2])) { const Mat& zp_tensor = netimpl->argTensor(inputs[2]); elt_out_type2 = !zp_tensor.empty() ? zp_tensor.type() : CV_8S; } @@ -373,41 +383,38 @@ struct ModelFusionQDQ : (int)out_zp_m2.at(0); if (out_sc2 > 0.f) { int relu_out_uc = usecounts.at(q_inp.idx); - LayerParams eltParams = makeLayerParamsFromOriginal(add2, "Eltwise2Int8"); - eltParams.blobs.clear(); - eltParams.set("input_scales", DictValue::arrayReal(in_scales2.data(), (int)in_scales2.size())); - eltParams.set("input_zeropoints", DictValue::arrayInt(in_zps2.data(), (int)in_zps2.size())); - eltParams.set("scales", out_sc2); - eltParams.set("zeropoints", out_zp_val2); - eltParams.set("with_relu", true); - Ptr eltInt8 = createFusedLayer(eltParams); - if (!eltInt8.empty()) { - CV_Assert(dynamic_cast(eltInt8.get())); + Eltwise2Int8Params ep2; + ep2.name = add2->name; + ep2.input_scales = in_scales2; + ep2.input_zeropoints = in_zps2; + ep2.output_sc = out_sc2; + ep2.output_zp = out_zp_val2; + ep2.with_relu = true; + Ptr eltInt8 = Eltwise2Int8Layer::create(ep2); + eltInt8->netimpl = netimpl; - fused_inputs = int8_inputs2; - if (relu_out_uc <= 1) { - fused_layer_idx = add_idx2; - newprog[add_idx2] = eltInt8; - newprog[relu_layer_idx2] = Ptr(); - removed_args.push_back(q_inp); // relu_out - removed_args.push_back(relu_in2); // add_out - for (size_t dk = 0; dk < add2->inputs.size(); dk++) { - removed_args.push_back(add2->inputs[dk]); - } - for (int dq_prog_idx : dq_prog_indices2) { - if (dq_prog_idx >= 0) - newprog[dq_prog_idx] = Ptr(); - } - } else { - int new_idx = (int)newprog.size(); - newprog.push_back(eltInt8); - fused_layer_idx = new_idx; - usecounts.at(q_inp.idx) -= 1; - Eltwise2Int8Layer* elt2ptr = dynamic_cast(eltInt8.get()); - relu_to_eltwise[q_inp.idx] = {outputs[0], elt2ptr}; + fused_inputs = int8_inputs2; + if (relu_out_uc <= 1) { + fused_layer_idx = add_idx2; + newprog[add_idx2] = eltInt8; + newprog[relu_layer_idx2] = Ptr(); + removed_args.push_back(q_inp); // relu_out + removed_args.push_back(relu_in2); // add_out + for (size_t dk = 0; dk < add2->inputs.size(); dk++) { + removed_args.push_back(add2->inputs[dk]); } - break; + for (int dq_prog_idx : dq_prog_indices2) { + if (dq_prog_idx >= 0) + newprog[dq_prog_idx] = Ptr(); + } + } else { + int new_idx = (int)newprog.size(); + newprog.push_back(eltInt8); + fused_layer_idx = new_idx; + usecounts.at(q_inp.idx) -= 1; + relu_to_eltwise[q_inp.idx] = {outputs[0], eltInt8.get()}; } + break; } } } @@ -431,6 +438,8 @@ struct ModelFusionQDQ if (dq_x && dq_w && dq_x->inputs.size() >= 3 && dq_w->inputs.size() >= 3 && + areDqArgsConst(dq_x) && areDqArgsConst(dq_w) && + netimpl->isConstArg(dq_w->inputs[0]) && usecounts.at(conv_w.idx) == 1) { float inp_sc = netimpl->argTensor(dq_x->inputs[1]).at(0); float out_sc = netimpl->argTensor(out_scale_arg).at(0); @@ -533,48 +542,41 @@ struct ModelFusionQDQ outputMultiplier.at(oc) = (inp_sc * wt_sc.at(oc)) / out_sc; } - LayerParams convInt8Params = makeLayerParamsFromOriginal(conv, "Conv2Int8"); - { - if (!conv->strides.empty()) - convInt8Params.set("stride", DictValue::arrayInt(conv->strides.data(), (int)conv->strides.size())); - if (!conv->dilations.empty()) - convInt8Params.set("dilation", DictValue::arrayInt(conv->dilations.data(), (int)conv->dilations.size())); - if (!conv->pads.empty()) - convInt8Params.set("pad", DictValue::arrayInt(conv->pads.data(), (int)conv->pads.size())); + Conv2Int8Params cp; + cp.name = conv->name; + cp.strides = conv->strides; + cp.dilations = conv->dilations; + cp.pads = conv->pads; + cp.ngroups = conv->ngroups; + cp.auto_pad = conv->auto_pad; + cp.ceil_mode = conv->ceil_mode; + cp.input_sc = inp_sc; + cp.input_zp = inp_zp; + cp.output_sc = out_sc; + cp.output_zp = out_zp; + cp.per_channel = per_channel; + cp.input_is_u8 = inputIsU8; + cp.weights = w_q; + cp.bias = biasFused; + cp.outputMultiplier = outputMultiplier; + Ptr convInt8 = Conv2Int8Layer::create(cp); + convInt8->netimpl = netimpl; + fused_layer_idx = conv_layer_idx; + newprog[conv_layer_idx] = convInt8; + fused_inputs.assign(1, dq_x->inputs[0]); + removed_args.push_back(q_data_in); + removed_args.push_back(conv_w); + if (conv->inputs.size() == 3) { + removed_args.push_back(conv->inputs[2]); + if (dq_bias_idx >= 0) + newprog[dq_bias_idx] = Ptr(); } - convInt8Params.set("num_output", outCn); - convInt8Params.set("group", conv->ngroups); - convInt8Params.set("input_scale", inp_sc); - convInt8Params.set("input_zeropoint", inp_zp); - convInt8Params.set("scales", out_sc); - convInt8Params.set("zeropoints", out_zp); - convInt8Params.set("per_channel", per_channel); - convInt8Params.set("input_is_u8", inputIsU8); - convInt8Params.blobs.resize(3); - convInt8Params.blobs[0] = w_q; - convInt8Params.blobs[1] = biasFused; - convInt8Params.blobs[2] = outputMultiplier; - Ptr convInt8 = createFusedLayer(convInt8Params); - if (!convInt8.empty()) { - auto* convInt8Layer = dynamic_cast(convInt8.get()); - CV_Assert(convInt8Layer); - fused_layer_idx = conv_layer_idx; - newprog[conv_layer_idx] = convInt8; - fused_inputs.assign(1, dq_x->inputs[0]); - removed_args.push_back(q_data_in); - removed_args.push_back(conv_w); - if (conv->inputs.size() == 3) { - removed_args.push_back(conv->inputs[2]); - if (dq_bias_idx >= 0) - newprog[dq_bias_idx] = Ptr(); - } - if (usecounts.at(conv_x.idx) == 1) { - removed_args.push_back(conv_x); - newprog[dq_x_idx] = Ptr(); - } - newprog[dq_w_idx] = Ptr(); - break; + if (usecounts.at(conv_x.idx) == 1) { + removed_args.push_back(conv_x); + newprog[dq_x_idx] = Ptr(); } + newprog[dq_w_idx] = Ptr(); + break; } } } @@ -595,19 +597,22 @@ struct ModelFusionQDQ float inp_sc = 0.f, out_sc = 0.f; int inp_zp = 0, out_zp_i = 0; int fc_out_type = !outputs.empty() ? netimpl->argData(outputs[0]).type : -1; - if (fc_out_type < 0) { + if (fc_out_type < 0 && netimpl->isConstArg(inputs[2])) { const Mat& zp_t = netimpl->argTensor(inputs[2]); fc_out_type = !zp_t.empty() ? zp_t.type() : CV_8S; } const bool fc_out_int8 = (fc_out_type == CV_8S || fc_out_type == CV_8U); int fc_in_type = (dq_x && !dq_x->inputs.empty()) ? netimpl->argData(dq_x->inputs[0]).type : -1; - if (fc_in_type < 0 && dq_x && dq_x->inputs.size() >= 3) { + if (fc_in_type < 0 && dq_x && dq_x->inputs.size() >= 3 && + netimpl->isConstArg(dq_x->inputs[2])) { const Mat& zp_t = netimpl->argTensor(dq_x->inputs[2]); fc_in_type = !zp_t.empty() ? zp_t.type() : CV_8S; } const bool fc_in_int8 = (fc_in_type == CV_8S || fc_in_type == CV_8U); if (dq_x && dq_w && dq_x->inputs.size() >= 3 && dq_w->inputs.size() >= 3 && + areDqArgsConst(dq_x) && areDqArgsConst(dq_w) && + netimpl->isConstArg(dq_w->inputs[0]) && fc_in_int8 && fc_out_int8 && usecounts.at(mm_x.idx) == 1 && usecounts.at(mm_w.idx) == 1) { inp_sc = netimpl->argTensor(dq_x->inputs[1]).at(0); @@ -619,7 +624,7 @@ struct ModelFusionQDQ out_sc = netimpl->argTensor(out_scale).at(0); const Mat& fc_out_zp_m = netimpl->argTensor(out_zp); out_zp_i = fc_out_zp_m.depth() == CV_8U - ? (int)fc_out_zp_m.at(0) + ? (int)fc_out_zp_m.at(0) - 128 : (int)fc_out_zp_m.at(0); if (!(inp_sc > 0.f && out_sc > 0.f)) break; @@ -647,35 +652,31 @@ struct ModelFusionQDQ outputMultiplier.at(ioc) = (inp_sc * wt_sc.at(ioc)) / out_sc; } int firstInpDims = (int)netimpl->argData(mm_x).shape.size(); - int axis = std::max(1, firstInpDims - w_q.dims + 1); - LayerParams fcInt8Params = makeLayerParamsFromOriginal(mm, "InnerProductInt8"); - fcInt8Params.set("num_output", outCn); - fcInt8Params.set("axis", axis); - fcInt8Params.blobs.resize(3); - fcInt8Params.blobs[0] = weights; - fcInt8Params.blobs[1] = bias; - fcInt8Params.blobs[2] = outputMultiplier; - Ptr fcInt8 = createFusedLayer(fcInt8Params); - if (!fcInt8.empty()) { - auto* fcInt8Layer = dynamic_cast(fcInt8.get()); - CV_Assert(fcInt8Layer); - fcInt8Layer->input_zp = inp_zp; - fcInt8Layer->input_sc = inp_sc; - fcInt8Layer->output_zp = out_zp_i; - fcInt8Layer->output_sc = out_sc; - fcInt8Layer->output_type = fc_out_type; - fcInt8Layer->per_channel = per_channel; - fused_layer_idx = mm_layer_idx; - newprog[mm_layer_idx] = fcInt8; - fused_inputs.assign(1, dq_x->inputs[0]); - removed_args.push_back(q_data_in); - removed_args.push_back(mm_x); - removed_args.push_back(mm_w); - newprog[dq_x_idx] = Ptr(); - newprog[dq_w_idx] = Ptr(); - break; - } + MatMulInt8Params fp; + fp.name = mm->name; + fp.num_output = outCn; + fp.inp_dims = firstInpDims; + fp.input_sc = inp_sc; + fp.input_zp = inp_zp; + fp.output_sc = out_sc; + fp.output_zp = out_zp_i; + fp.output_type = fc_out_type; + fp.per_channel = per_channel; + fp.weights = weights; + fp.bias = bias; + fp.outputMultiplier = outputMultiplier; + Ptr mmInt8 = MatMulInt8Layer::create(fp); + mmInt8->netimpl = netimpl; + fused_layer_idx = mm_layer_idx; + newprog[mm_layer_idx] = mmInt8; + fused_inputs.assign(1, dq_x->inputs[0]); + removed_args.push_back(q_data_in); + removed_args.push_back(mm_x); + removed_args.push_back(mm_w); + newprog[dq_x_idx] = Ptr(); + newprog[dq_w_idx] = Ptr(); + break; } } } @@ -711,19 +712,22 @@ struct ModelFusionQDQ DequantizeLinearLayer* dq_x = getLayer(newprog, dq_x_idx); DequantizeLinearLayer* dq_w = getLayer(newprog, dq_w_idx); int fc_out_type = !outputs.empty() ? netimpl->argData(outputs[0]).type : -1; - if (fc_out_type < 0) { + if (fc_out_type < 0 && netimpl->isConstArg(inputs[2])) { const Mat& zp_t = netimpl->argTensor(inputs[2]); fc_out_type = !zp_t.empty() ? zp_t.type() : CV_8S; } const bool fc_out_int8 = (fc_out_type == CV_8S || fc_out_type == CV_8U); int fc_in_type = (dq_x && !dq_x->inputs.empty()) ? netimpl->argData(dq_x->inputs[0]).type : -1; - if (fc_in_type < 0 && dq_x && dq_x->inputs.size() >= 3) { + if (fc_in_type < 0 && dq_x && dq_x->inputs.size() >= 3 && + netimpl->isConstArg(dq_x->inputs[2])) { const Mat& zp_t = netimpl->argTensor(dq_x->inputs[2]); fc_in_type = !zp_t.empty() ? zp_t.type() : CV_8S; } const bool fc_in_int8 = (fc_in_type == CV_8S || fc_in_type == CV_8U); if (dq_x && dq_w && dq_x->inputs.size() >= 3 && dq_w->inputs.size() >= 3 && + areDqArgsConst(dq_x) && areDqArgsConst(dq_w) && + netimpl->isConstArg(dq_w->inputs[0]) && fc_in_int8 && fc_out_int8 && usecounts.at(mm_x.idx) == 1 && usecounts.at(mm_w.idx) == 1) { float inp_sc = netimpl->argTensor(dq_x->inputs[1]).at(0); @@ -734,7 +738,7 @@ struct ModelFusionQDQ float out_sc_val = netimpl->argTensor(out_scale).at(0); const Mat& fc_out_zp_m = netimpl->argTensor(out_zp); int out_zp_i = fc_out_zp_m.depth() == CV_8U - ? (int)fc_out_zp_m.at(0) + ? (int)fc_out_zp_m.at(0) - 128 : (int)fc_out_zp_m.at(0); if (inp_sc > 0.f && out_sc_val > 0.f) { Mat w_q = netimpl->argTensor(dq_w->inputs[0]); @@ -773,37 +777,162 @@ struct ModelFusionQDQ } if (biasOk) { int firstInpDims = (int)netimpl->argData(mm_x).shape.size(); - int fc_axis = std::max(1, firstInpDims - w_q.dims + 1); - LayerParams fcInt8Params = makeLayerParamsFromOriginal(mm2, "InnerProductInt8"); - fcInt8Params.set("num_output", outCn); - fcInt8Params.set("axis", fc_axis); - fcInt8Params.blobs.resize(3); - fcInt8Params.blobs[0] = weights; - fcInt8Params.blobs[1] = bias; - fcInt8Params.blobs[2] = outputMultiplier; - Ptr fcInt8 = createFusedLayer(fcInt8Params); - if (!fcInt8.empty()) { - auto* fcInt8Layer = dynamic_cast(fcInt8.get()); - CV_Assert(fcInt8Layer); - fcInt8Layer->input_zp = inp_zp; - fcInt8Layer->input_sc = inp_sc; - fcInt8Layer->output_zp = out_zp_i; - fcInt8Layer->output_sc = out_sc_val; - fcInt8Layer->output_type = fc_out_type; - fcInt8Layer->per_channel = per_channel; - fused_layer_idx = add_bias_idx; - newprog[add_bias_idx] = fcInt8; - fused_inputs.assign(1, dq_x->inputs[0]); - removed_args.push_back(q_data_in); - removed_args.push_back(add_bias->inputs[mm_inp_k]); // matmul out - removed_args.push_back(mm_x); - removed_args.push_back(mm_w); - newprog[mm2_idx] = Ptr(); - newprog[dq_x_idx] = Ptr(); - newprog[dq_w_idx] = Ptr(); - break; + MatMulInt8Params fp2; + fp2.name = mm2->name; + fp2.num_output = outCn; + fp2.inp_dims = firstInpDims; + fp2.input_sc = inp_sc; + fp2.input_zp = inp_zp; + fp2.output_sc = out_sc_val; + fp2.output_zp = out_zp_i; + fp2.output_type = fc_out_type; + fp2.per_channel = per_channel; + fp2.weights = weights; + fp2.bias = bias; + fp2.outputMultiplier = outputMultiplier; + Ptr mmInt8 = MatMulInt8Layer::create(fp2); + mmInt8->netimpl = netimpl; + fused_layer_idx = add_bias_idx; + newprog[add_bias_idx] = mmInt8; + fused_inputs.assign(1, dq_x->inputs[0]); + removed_args.push_back(q_data_in); + removed_args.push_back(add_bias->inputs[mm_inp_k]); // matmul out + removed_args.push_back(mm_x); + removed_args.push_back(mm_w); + newprog[mm2_idx] = Ptr(); + newprog[dq_x_idx] = Ptr(); + newprog[dq_w_idx] = Ptr(); + break; + } + } + } + } + } + } + } + + // fuse DequantizeLinear -> Gemm -> QuantizeLinear into MatMulInt8 + // Gemm: Y = alpha * A @ B + beta * C (with optional transA/transB) + { + int gemm_layer_idx = -1; + GemmLayer* gemm = 0; + if (getQdqPatternContext(layer_ptr, ninputs, inputs, producer_of, + newprog, q_data_in, out_scale, out_zp, + gemm_layer_idx, gemm) && + gemm->inputs.size() >= 2 && !gemm->trans_a) { + // Gemm inputs: A, B, [C] + const Arg gemm_a = gemm->inputs[0]; + const Arg gemm_b = gemm->inputs[1]; + int dq_a_idx = producer_of.at(gemm_a.idx); + int dq_b_idx = producer_of.at(gemm_b.idx); + DequantizeLinearLayer* dq_a = getLayer(newprog, dq_a_idx); + DequantizeLinearLayer* dq_b = getLayer(newprog, dq_b_idx); + + int g_out_type = !outputs.empty() ? netimpl->argData(outputs[0]).type : -1; + if (g_out_type < 0 && netimpl->isConstArg(inputs[2])) { + const Mat& zp_t = netimpl->argTensor(inputs[2]); + g_out_type = !zp_t.empty() ? zp_t.type() : CV_8S; + } + const bool g_out_int8 = (g_out_type == CV_8S || g_out_type == CV_8U); + int g_in_type = (dq_a && !dq_a->inputs.empty()) ? netimpl->argData(dq_a->inputs[0]).type : -1; + if (g_in_type < 0 && dq_a && dq_a->inputs.size() >= 3 && + netimpl->isConstArg(dq_a->inputs[2])) { + const Mat& zp_t = netimpl->argTensor(dq_a->inputs[2]); + g_in_type = !zp_t.empty() ? zp_t.type() : CV_8S; + } + const bool g_in_int8 = (g_in_type == CV_8S || g_in_type == CV_8U); + + if (dq_a && dq_b && + dq_a->inputs.size() >= 3 && dq_b->inputs.size() >= 3 && + areDqArgsConst(dq_a) && areDqArgsConst(dq_b) && + netimpl->isConstArg(dq_b->inputs[0]) && + g_in_int8 && g_out_int8 && + usecounts.at(gemm_a.idx) == 1 && usecounts.at(gemm_b.idx) == 1) { + float inp_sc = netimpl->argTensor(dq_a->inputs[1]).at(0); + const Mat& g_zp_m = netimpl->argTensor(dq_a->inputs[2]); + int inp_zp = g_zp_m.depth() == CV_8U + ? (int)g_zp_m.at(0) - 128 + : (int)g_zp_m.at(0); + float out_sc_val = netimpl->argTensor(out_scale).at(0); + const Mat& g_out_zp_m = netimpl->argTensor(out_zp); + int out_zp_i = g_out_zp_m.depth() == CV_8U + ? (int)g_out_zp_m.at(0) - 128 + : (int)g_out_zp_m.at(0); + if (inp_sc > 0.f && out_sc_val > 0.f) { + Mat w_q = netimpl->argTensor(dq_b->inputs[0]); + Mat w_sc_m = netimpl->argTensor(dq_b->inputs[1]); + Mat w_zp_m = netimpl->argTensor(dq_b->inputs[2]); + if (!w_q.empty() && w_q.depth() == CV_8S && w_q.dims == 2) { + bool all_wzp_zero = true; + for (size_t t = 0; all_wzp_zero && t < w_zp_m.total(); t++) { + int wz = w_zp_m.depth() == CV_8S ? (int)w_zp_m.at((int)t) + : (int)w_zp_m.at((int)t); + if (wz != 0) all_wzp_zero = false; + } + if (all_wzp_zero) { + // Apply Gemm's transB: if transB, weights are already [N, K], + // otherwise [K, N] and we need to transpose. + Mat weights = gemm->trans_b ? w_q : w_q.t(); + int outCn = weights.size[0]; + float alpha_val = gemm->alpha; + + Mat wt_sc = (w_sc_m.total() == (size_t)outCn) + ? w_sc_m.reshape(1, 1) + : Mat(1, outCn, CV_32F, Scalar(w_sc_m.at(0))).clone(); + bool per_channel = w_sc_m.total() == (size_t)outCn; + + // Fuse optional bias (C input of Gemm) + Mat float_bias; + if (gemm->inputs.size() >= 3) + float_bias = netimpl->argTensor(gemm->inputs[2]); + + Mat bias(1, outCn, CV_32S); + Mat outputMult(1, outCn, CV_32F); + bool biasOk = true; + for (int ioc = 0; ioc < outCn && biasOk; ioc++) { + float denom = alpha_val * inp_sc * wt_sc.at(ioc); + if (std::abs(denom) < 1e-12f) { biasOk = false; break; } + int zp_correction = -inp_zp * (int)cv::sum(weights.row(ioc))[0]; + if (!float_bias.empty() && float_bias.total() == (size_t)outCn) { + float b_val = 0.f; + if (float_bias.depth() == CV_32F) + b_val = float_bias.at(ioc); + else if (float_bias.depth() == CV_64F) + b_val = (float)float_bias.at(ioc); + bias.at(ioc) = cvRound(gemm->beta * b_val / denom) + zp_correction; + } else { + bias.at(ioc) = zp_correction; } + outputMult.at(ioc) = denom / out_sc_val; + } + if (biasOk) { + int firstInpDims = (int)netimpl->argData(gemm_a).shape.size(); + + MatMulInt8Params gp; + gp.name = gemm->name; + gp.num_output = outCn; + gp.inp_dims = firstInpDims; + gp.input_sc = inp_sc; + gp.input_zp = inp_zp; + gp.output_sc = out_sc_val; + gp.output_zp = out_zp_i; + gp.output_type = g_out_type; + gp.per_channel = per_channel; + gp.weights = weights; + gp.bias = bias; + gp.outputMultiplier = outputMult; + Ptr gemmInt8 = MatMulInt8Layer::create(gp); + gemmInt8->netimpl = netimpl; + fused_layer_idx = gemm_layer_idx; + newprog[gemm_layer_idx] = gemmInt8; + fused_inputs.assign(1, dq_a->inputs[0]); + removed_args.push_back(q_data_in); + removed_args.push_back(gemm_a); + removed_args.push_back(gemm_b); + newprog[dq_a_idx] = Ptr(); + newprog[dq_b_idx] = Ptr(); + break; } } } @@ -816,44 +945,57 @@ struct ModelFusionQDQ PoolingLayer* pool = 0; if (getQdqPatternContext(layer_ptr, ninputs, inputs, producer_of, newprog, q_data_in, out_scale, out_zp, - pool_layer_idx, pool) && pool->inputs.size() == 1) { - Arg pool_in = pool->inputs[0]; - int dq_idx = producer_of.at(pool_in.idx); - DequantizeLinearLayer* dq = getLayer(newprog, dq_idx); - float inp_sc = 0.f, out_sc = 0.f; - int inp_zp = 0, out_zp_i = 0; - const int pool_out_type = !outputs.empty() ? netimpl->argData(outputs[0]).type : -1; - const bool pool_out_int8 = (pool_out_type == CV_8S || pool_out_type == CV_8U); - const int pool_in_type = (dq && !dq->inputs.empty()) ? netimpl->argData(dq->inputs[0]).type : -1; - const bool pool_in_int8 = (pool_in_type == CV_8S || pool_in_type == CV_8U); - - if (dq && dq->inputs.size() >= 3 && pool_in_int8 && pool_out_int8 && usecounts.at(pool_in.idx) == 1) { - inp_sc = netimpl->argTensor(dq->inputs[1]).at(0); - const Mat& pool_zp_m = netimpl->argTensor(dq->inputs[2]); - inp_zp = pool_zp_m.depth() == CV_8U - ? (int)pool_zp_m.at(0) - : (int)pool_zp_m.at(0); - out_sc = netimpl->argTensor(out_scale).at(0); - const Mat& pool_out_zp_m = netimpl->argTensor(out_zp); - out_zp_i = pool_out_zp_m.depth() == CV_8U - ? (int)pool_out_zp_m.at(0) - : (int)pool_out_zp_m.at(0); - bool isGlobalAve = pool->globalPooling; - bool isMax = !isGlobalAve; - if ((isGlobalAve && inp_sc > 0.f && out_sc > 0.f) || - (isMax && std::abs(inp_sc - out_sc) < 1e-6f && inp_zp == out_zp_i)) { - LayerParams poolInt8Params = makeLayerParamsFromOriginal(pool, "Pool2Int8"); - poolInt8Params.blobs.clear(); - poolInt8Params.set("input_scale", inp_sc); - poolInt8Params.set("input_zeropoint", inp_zp); - poolInt8Params.set("scales", out_sc); - poolInt8Params.set("zeropoints", out_zp_i); - poolInt8Params.set("is_max_pool", isMax); - poolInt8Params.set("global_pooling", isGlobalAve); - Ptr poolInt8 = createFusedLayer(poolInt8Params); - if (!poolInt8.empty()) { - auto* poolInt8Layer = dynamic_cast(poolInt8.get()); - CV_Assert(poolInt8Layer); + pool_layer_idx, pool) && + pool->inputs.size() == 1) { + Arg pool_in = pool->inputs[0]; + int dq_idx = producer_of.at(pool_in.idx); + DequantizeLinearLayer* dq = getLayer(newprog, dq_idx); + float inp_sc = 0.f, out_sc = 0.f; + int inp_zp = 0, out_zp_i = 0; + const int pool_out_type = !outputs.empty() ? netimpl->argData(outputs[0]).type : -1; + const bool pool_out_int8 = (pool_out_type == CV_8S || pool_out_type == CV_8U); + const int pool_in_type = (dq && !dq->inputs.empty()) ? netimpl->argData(dq->inputs[0]).type : -1; + const bool pool_in_int8 = (pool_in_type == CV_8S || pool_in_type == CV_8U); + if (dq && dq->inputs.size() >= 3 && + areDqArgsConst(dq) && + pool_in_int8 && pool_out_int8 && + usecounts.at(pool_in.idx) == 1) { + inp_sc = netimpl->argTensor(dq->inputs[1]).at(0); + const Mat& pool_zp_m = netimpl->argTensor(dq->inputs[2]); + inp_zp = pool_zp_m.depth() == CV_8U + ? (int)pool_zp_m.at(0) + : (int)pool_zp_m.at(0); + out_sc = netimpl->argTensor(out_scale).at(0); + const Mat& pool_out_zp_m = netimpl->argTensor(out_zp); + out_zp_i = pool_out_zp_m.depth() == CV_8U + ? (int)pool_out_zp_m.at(0) + : (int)pool_out_zp_m.at(0); + bool isGlobalAve = pool->globalPooling; + bool isMax = !isGlobalAve; + if ((isGlobalAve && inp_sc > 0.f && out_sc > 0.f) || + (isMax && std::abs(inp_sc - out_sc) < 1e-6f && inp_zp == out_zp_i)) { + Pool2Int8Params p; + p.name = pool->name; + p.kernel_shape.assign(pool->kernel_size.begin(), pool->kernel_size.end()); + p.strides.assign(pool->strides.begin(), pool->strides.end()); + if (!pool->pads_begin.empty()) { + p.pads.reserve(pool->pads_begin.size() + pool->pads_end.size()); + for (size_t v : pool->pads_begin) p.pads.push_back((int)v); + for (size_t v : pool->pads_end) p.pads.push_back((int)v); + } + if (pool->padMode == "SAME") + p.auto_pad = AUTO_PAD_SAME_UPPER; + else if (pool->padMode == "VALID") + p.auto_pad = AUTO_PAD_VALID; + p.ceil_mode = pool->ceilMode; + p.is_global_pooling = isGlobalAve; + p.is_max_pool = isMax; + p.input_sc = inp_sc; + p.input_zp = inp_zp; + p.output_sc = out_sc; + p.output_zp = out_zp_i; + Ptr poolInt8 = Pool2Int8Layer::create(p); + poolInt8->netimpl = netimpl; fused_layer_idx = pool_layer_idx; newprog[pool_layer_idx] = poolInt8; fused_inputs.assign(1, dq->inputs[0]); @@ -864,6 +1006,59 @@ struct ModelFusionQDQ } } } + + // fuse DequantizeLinear -> DataShuffling/MaxPool -> QuantizeLinear + // when DQ and Q have same scale/zp, the op can work directly on int8 data. + // Uses Layer::isDataShuffling() to identify eligible layers. + { + QuantizeLinearLayer* ql_shuffle = dynamic_cast(layer_ptr); + if (ql_shuffle && ninputs == 3 && usecounts.at(inputs[0].idx) == 1) { + Arg ql_data = inputs[0]; + Arg ql_scale = inputs[1]; + Arg ql_zp = inputs[2]; + int shuffle_idx = producer_of.at(ql_data.idx); + Layer* shuffle_layer = (shuffle_idx >= 0 && !newprog[shuffle_idx].empty()) + ? newprog[shuffle_idx].get() : nullptr; + + bool is_shuffle = shuffle_layer && shuffle_layer->isDataShuffling(); + + if (is_shuffle && shuffle_layer->inputs.size() >= 1 && + usecounts.at(ql_data.idx) == 1) { + Arg shuffle_inp = shuffle_layer->inputs[0]; + int dq_idx = producer_of.at(shuffle_inp.idx); + DequantizeLinearLayer* dq = getLayer(newprog, dq_idx); + + if (dq && dq->inputs.size() >= 3 && + usecounts.at(shuffle_inp.idx) == 1) { + float dq_sc = netimpl->argTensor(dq->inputs[1]).at(0); + float q_sc = netimpl->argTensor(ql_scale).at(0); + const Mat& dq_zp_m = netimpl->argTensor(dq->inputs[2]); + const Mat& q_zp_m = netimpl->argTensor(ql_zp); + + int dq_zp = dq_zp_m.depth() == CV_8U + ? (int)dq_zp_m.at(0) : (int)dq_zp_m.at(0); + int q_zp = q_zp_m.depth() == CV_8U + ? (int)q_zp_m.at(0) : (int)q_zp_m.at(0); + + int in_type = netimpl->argData(dq->inputs[0]).type; + int out_type = !outputs.empty() ? netimpl->argData(outputs[0]).type : -1; + bool in_int8 = (in_type == CV_8S || in_type == CV_8U); + bool out_int8 = (out_type == CV_8S || out_type == CV_8U); + + if (in_int8 && out_int8 && + std::abs(dq_sc - q_sc) < 1e-6f && dq_zp == q_zp) { + fused_layer_idx = shuffle_idx; + fused_inputs.assign(1, dq->inputs[0]); + for (size_t si = 1; si < shuffle_layer->inputs.size(); si++) + fused_inputs.push_back(shuffle_layer->inputs[si]); + removed_args.push_back(ql_data); + removed_args.push_back(shuffle_inp); + newprog[dq_idx] = Ptr(); + break; + } + } + } + } } ActivationLayerInt8* activ_int8 = dynamic_cast(layer_ptr); @@ -913,6 +1108,69 @@ struct ModelFusionQDQ } } + for (size_t j = 0; j < newprog.size(); j++) + { + if (newprog[j].empty()) continue; + MaxPoolLayer* maxpool = dynamic_cast(newprog[j].get()); + if (!maxpool || maxpool->inputs.size() != 1) + continue; + + Arg mp_out = maxpool->outputs[0]; + bool consumer_is_ql = false; + for (size_t j2 = j + 1; j2 < newprog.size(); j2++) { + if (newprog[j2].empty()) continue; + bool is_consumer = false; + for (const Arg& a : newprog[j2]->inputs) + if (a.idx == mp_out.idx) { is_consumer = true; break; } + if (is_consumer) { + consumer_is_ql = dynamic_cast(newprog[j2].get()) != nullptr; + break; + } + } + if (consumer_is_ql) + continue; + + int cur = producer_of.at(maxpool->inputs[0].idx); + QuantizeLinearLayer* ql = nullptr; + while (cur >= 0 && !newprog[cur].empty()) { + ql = dynamic_cast(newprog[cur].get()); + if (ql) break; + Layer* l = newprog[cur].get(); + if (l->inputs.empty()) { ql = nullptr; break; } + cur = producer_of.at(l->inputs[0].idx); + } + if (!ql || ql->inputs.size() < 2 || + !netimpl->isConstArg(ql->inputs[1]) || + (ql->inputs.size() >= 3 && !netimpl->isConstArg(ql->inputs[2]))) + continue; + + float inp_sc = netimpl->argTensor(ql->inputs[1]).at(0); + const Mat& inp_zp_m = ql->inputs.size() >= 3 ? netimpl->argTensor(ql->inputs[2]) : Mat(); + int inp_zp = inp_zp_m.empty() ? 0 : + inp_zp_m.depth() == CV_8U ? (int)inp_zp_m.at(0) : + (int)inp_zp_m.at(0); + + Pool2Int8Params p; + p.name = maxpool->name; + p.kernel_shape = maxpool->kernel_shape; + p.strides = maxpool->strides; + p.dilations = maxpool->dilations; + p.pads = maxpool->pads; + p.auto_pad = maxpool->auto_pad; + p.ceil_mode = maxpool->ceil_mode; + p.is_max_pool = true; + p.input_sc = inp_sc; + p.input_zp = inp_zp; + p.output_sc = inp_sc; + p.output_zp = inp_zp; + Ptr poolInt8 = Pool2Int8Layer::create(p); + poolInt8->netimpl = netimpl; + poolInt8->inputs = maxpool->inputs; + poolInt8->outputs = maxpool->outputs; + newprog[j] = poolInt8; + modified = true; + } + if (modified) { std::vector uc(nargs, 0); for (const auto& layer : newprog) { diff --git a/modules/dnn/src/int8layers/conv2_int8_layer.cpp b/modules/dnn/src/int8layers/conv2_int8_layer.cpp index dad01b6474..4ca990cfea 100644 --- a/modules/dnn/src/int8layers/conv2_int8_layer.cpp +++ b/modules/dnn/src/int8layers/conv2_int8_layer.cpp @@ -212,6 +212,32 @@ public: } } + Conv2Int8LayerImpl(const Conv2Int8Params& p) + { + name = p.name; + type = "Conv2Int8"; + strides = p.strides; + dilations = p.dilations; + pads = p.pads; + ngroups = p.ngroups; + auto_pad = p.auto_pad; + ceil_mode = p.ceil_mode; + input_sc = p.input_sc; + input_zp = p.input_zp; + output_sc = p.output_sc; + output_zp = p.output_zp; + per_channel = p.per_channel; + inputIsU8 = p.input_is_u8; + addResidual = false; + + if (!p.weights.empty()) { + wshape0 = p.weights.shape(); + biasInt32 = p.bias; + outMultiplier = p.outputMultiplier; + blobs = { p.weights, p.bias, p.outputMultiplier }; + } + } + void getTypes(const std::vector& inputs, const int requiredOutputs, const int, std::vector& outputs, @@ -341,6 +367,10 @@ public: cs.initConv(inpshape, wshape0, outshape, ngroups, strides, dilations, pads, auto_pad, ceil_mode, FAST_ACTIV_NONE, nullptr, {}); + + if (cs.depthwise) { + cs.wshape = getWpackShapeInt8(wshape0, ngroups, C0); + } prevInpshape = inpshape; } @@ -367,5 +397,10 @@ Ptr Conv2Int8Layer::create(const LayerParams& params) return Ptr(new Conv2Int8LayerImpl(params)); } +Ptr Conv2Int8Layer::create(const Conv2Int8Params& params) +{ + return Ptr(new Conv2Int8LayerImpl(params)); +} + } // namespace dnn } // namespace cv diff --git a/modules/dnn/src/int8layers/elementwise_layers.cpp b/modules/dnn/src/int8layers/elementwise_layers.cpp index 45321b4119..3826d2e553 100644 --- a/modules/dnn/src/int8layers/elementwise_layers.cpp +++ b/modules/dnn/src/int8layers/elementwise_layers.cpp @@ -37,11 +37,21 @@ public: { slope = params.get("slope"); } + } + + ActivationLayerInt8Impl(const ActivationInt8Params &p) + { + name = p.name; + type = p.activationType; + input_sc = p.input_sc; + input_zp = p.input_zp; + output_sc = p.output_sc; + output_zp = p.output_zp; + activationLUT = p.activationLUT; #ifdef HAVE_TIMVX tvActType = getTimVXActType(type); #endif - } virtual bool supportBackend(int backendId) CV_OVERRIDE @@ -360,5 +370,10 @@ Ptr ActivationLayerInt8::create(const LayerParams& params) return Ptr(new ActivationLayerInt8Impl(params)); } +Ptr ActivationLayerInt8::create(const ActivationInt8Params& params) +{ + return Ptr(new ActivationLayerInt8Impl(params)); +} + } } diff --git a/modules/dnn/src/int8layers/eltwise2_int8_layer.cpp b/modules/dnn/src/int8layers/eltwise2_int8_layer.cpp index 875921364e..1b9f60e54d 100644 --- a/modules/dnn/src/int8layers/eltwise2_int8_layer.cpp +++ b/modules/dnn/src/int8layers/eltwise2_int8_layer.cpp @@ -57,6 +57,18 @@ public: offset = 0.f; } + Eltwise2Int8LayerImpl(const Eltwise2Int8Params& p) + { + name = p.name; + type = "Eltwise2Int8"; + scales = p.input_scales; + zeropoints = p.input_zeropoints; + output_sc = p.output_sc; + output_zp = p.output_zp; + withRelu = p.with_relu; + offset = 0.f; + } + void ensureCoeffs() { if (!coeffs.empty() || scales.empty()) @@ -409,5 +421,10 @@ Ptr Eltwise2Int8Layer::create(const LayerParams& params) return Ptr(new Eltwise2Int8LayerImpl(params)); } +Ptr Eltwise2Int8Layer::create(const Eltwise2Int8Params& params) +{ + return Ptr(new Eltwise2Int8LayerImpl(params)); +} + } // namespace dnn } // namespace cv diff --git a/modules/dnn/src/int8layers/fully_connected_layer.cpp b/modules/dnn/src/int8layers/fully_connected_layer.cpp index 1e2528c1da..9e7d051bc7 100644 --- a/modules/dnn/src/int8layers/fully_connected_layer.cpp +++ b/modules/dnn/src/int8layers/fully_connected_layer.cpp @@ -57,6 +57,42 @@ public: } } + FullyConnectedLayerInt8Impl(const InnerProductInt8Params& p) + { + name = p.name; + type = "InnerProductInt8"; + input_sc = p.input_sc; + input_zp = p.input_zp; + output_sc = p.output_sc; + output_zp = p.output_zp; + axis = p.axis; + per_channel = p.per_channel; + output_type = p.output_type; + + if (!p.weights.empty()) { + int numOutput = p.num_output; + int innerSize = (int)p.weights.total() / numOutput; + + CV_Assert(p.weights.dims >= 2 && (size_t)(innerSize * numOutput) == p.weights.total()); + CV_Assert((size_t)numOutput == p.bias.total()); + + blobs = { p.weights.clone(), p.bias.clone(), p.outputMultiplier.clone() }; + weightsMat = blobs[0] = blobs[0].reshape(1, numOutput); + int vecsize = weightsMat.cols; + if (vecsize % VEC_ALIGN != 0) + { + int vecsize_aligned = (int)alignSize(vecsize, VEC_ALIGN); + Mat weightsBuf(weightsMat.rows, vecsize_aligned, weightsMat.type()); + Mat wpadding = weightsBuf.colRange(vecsize, vecsize_aligned); + wpadding.setTo(Scalar::all(0)); + weightsMat = weightsBuf.colRange(0, vecsize); + blobs[0].copyTo(weightsMat); + } + biasMat = blobs[1] = blobs[1].reshape(1, 1); + outputMultiplier = blobs[2]; + } + } + bool getMemoryShapes(const std::vector &inputs, const int requiredOutputs, std::vector &outputs, @@ -508,5 +544,10 @@ Ptr InnerProductLayerInt8::create(const LayerParams& para return Ptr(new FullyConnectedLayerInt8Impl(params)); } +Ptr InnerProductLayerInt8::create(const InnerProductInt8Params& params) +{ + return Ptr(new FullyConnectedLayerInt8Impl(params)); +} + } } diff --git a/modules/dnn/src/int8layers/matmul_int8_layer.cpp b/modules/dnn/src/int8layers/matmul_int8_layer.cpp new file mode 100644 index 0000000000..ec435c341f --- /dev/null +++ b/modules/dnn/src/int8layers/matmul_int8_layer.cpp @@ -0,0 +1,296 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. + +#include "../precomp.hpp" +#include "layers_common.hpp" + +#include + +namespace cv +{ +namespace dnn +{ + +class MatMulInt8LayerImpl CV_FINAL : public MatMulInt8Layer +{ +public: + enum { VEC_ALIGN = 32 }; + + MatMulInt8LayerImpl(const MatMulInt8Params& p) + { + name = p.name; + type = "MatMulInt8"; + input_sc = p.input_sc; + input_zp = p.input_zp; + output_sc = p.output_sc; + output_zp = p.output_zp; + output_type = p.output_type; + per_channel = p.per_channel; + inp_dims = p.inp_dims; + num_output = p.num_output; + + if (!p.weights.empty()) { + int numOutput = p.num_output; + int innerSize = (int)p.weights.total() / numOutput; + + CV_Assert(p.weights.dims >= 2 && (size_t)(innerSize * numOutput) == p.weights.total()); + CV_Assert((size_t)numOutput == p.bias.total()); + + blobs = { p.weights.clone(), p.bias.clone(), p.outputMultiplier.clone() }; + weightsMat = blobs[0] = blobs[0].reshape(1, numOutput); + int vecsize = weightsMat.cols; + if (vecsize % VEC_ALIGN != 0) + { + int vecsize_aligned = (int)alignSize(vecsize, VEC_ALIGN); + Mat weightsBuf(weightsMat.rows, vecsize_aligned, weightsMat.type()); + Mat wpadding = weightsBuf.colRange(vecsize, vecsize_aligned); + wpadding.setTo(Scalar::all(0)); + weightsMat = weightsBuf.colRange(0, vecsize); + blobs[0].copyTo(weightsMat); + } + biasMat = blobs[1] = blobs[1].reshape(1, 1); + outputMultiplier = blobs[2]; + } + } + + bool getMemoryShapes(const std::vector &inputs, + const int requiredOutputs, + std::vector &outputs, + std::vector &) const CV_OVERRIDE + { + CV_CheckEQ(inputs.size(), (size_t)1, ""); + CV_CheckEQ(blobs[0].dims, 2, ""); + int numOut = blobs[0].size[0]; + CV_Assert((size_t)numOut == blobs[1].total()); + + // Preserve all batch dimensions, replace last dim with num_output. + // Input: [B1, B2, ..., M, K] → Output: [B1, B2, ..., M, num_output] + int ndims = inputs[0].dims; + CV_Assert(ndims >= 1); + MatShape outShape = inputs[0]; + outShape[ndims - 1] = numOut; + + outputs.resize(1, outShape); + return false; + } + + void getTypes(const std::vector& inputs, + const int requiredOutputs, + const int requiredInternals, + std::vector& outputs, + std::vector& internals) const CV_OVERRIDE + { + outputs.assign(requiredOutputs, output_type); + internals.clear(); + } + + virtual bool supportBackend(int backendId) CV_OVERRIDE + { + return backendId == DNN_BACKEND_OPENCV; + } + + void forward(InputArrayOfArrays inputs_arr, OutputArrayOfArrays outputs_arr, + OutputArrayOfArrays internals_arr) CV_OVERRIDE + { + CV_TRACE_FUNCTION(); + CV_TRACE_ARG_VALUE(name, "name", name.c_str()); + + std::vector input, output; + inputs_arr.getMatVector(input); + outputs_arr.getMatVector(output); + + // Flatten all dimensions except the last (K) into a single batch dim. + // Input: [B1, B2, ..., M, K] → [B1*B2*...*M, K] + int ndims = input[0].dims; + int K = input[0].size[ndims - 1]; + int outerSize = (int)(input[0].total() / K); + + Mat srcMat0 = input[0].reshape(1, outerSize); + Mat srcMat; + if (srcMat0.type() == CV_8U) { + srcMat0.convertTo(srcMat, CV_8S, 1, -128); + } else { + srcMat = srcMat0; + } + + Mat dstMat = output[0].reshape(1, outerSize); + Mat dstMatInt32 = Mat(shape(dstMat), CV_32S); + + const int nstripes = outerSize <= 4 ? 1 : getNumThreads(); + runInt8Gemm(srcMat, weightsMat, biasMat, outputMultiplier, dstMatInt32, nstripes, output_zp); + if (output_type == CV_8U) { + dstMatInt32.convertTo(dstMat, output_type, 1, 128); + } else { + dstMatInt32.convertTo(dstMat, output_type); + } + } + + virtual int64 getFLOPS(const std::vector &inputs, + const std::vector &outputs) const CV_OVERRIDE + { + CV_UNUSED(inputs); + long flops = 0; + int innerSize = blobs[0].size[1]; + for (size_t i = 0; i < outputs.size(); i++) { + flops += total(outputs[i]) * innerSize; + } + return flops; + } + +private: + // Int8 GEMM: C_int32 = A_int8 @ W_int8^T + bias_int32 + // Then: output = clamp(round(C_int32 * multiplier) + output_zp) + static void runInt8Gemm(const Mat& srcMat, const Mat& weights, const Mat& biasMat, + const Mat& outputMultiplier, Mat& dstMat, int nstripes, int outZp) + { + CV_Assert(srcMat.dims == 2 && srcMat.cols == weights.cols && + dstMat.rows == srcMat.rows && dstMat.cols == weights.rows && + srcMat.type() == weights.type() && srcMat.type() == CV_8S && + dstMat.type() == CV_32S && biasMat.type() == CV_32S && + biasMat.isContinuous() && (int)biasMat.total() == dstMat.cols); + + Int8GemmBody p; + p.srcMat = &srcMat; + p.weights = &weights; + p.biasMat = &biasMat; + p.outputMultiplier = &outputMultiplier; + p.dstMat = &dstMat; + p.nstripes = nstripes; + p.outZp = outZp; + p.useAVX2 = checkHardwareSupport(CPU_AVX2); + p.useAVX512 = CV_CPU_HAS_SUPPORT_AVX512_SKX; + p.useLASX = checkHardwareSupport(CPU_LASX); + p.useRVV = checkHardwareSupport(CPU_RVV); + + parallel_for_(Range(0, nstripes), p, nstripes); + } + + class Int8GemmBody : public ParallelLoopBody + { + public: + Int8GemmBody() : srcMat(0), weights(0), biasMat(0), outputMultiplier(0), + dstMat(0), nstripes(0), outZp(0), + useAVX2(false), useAVX512(false), useLASX(false), useRVV(false) {} + + void operator()(const Range& r) const CV_OVERRIDE + { + int valign = MatMulInt8LayerImpl::VEC_ALIGN; + int nsamples = srcMat->rows; + int nw0 = weights->rows; + int k, vecsize = srcMat->cols; + int vecsize_aligned = (int)alignSize(vecsize, VEC_ALIGN); + size_t total = (size_t)nsamples * nw0; + size_t stripeSize = (total + nstripes - 1) / nstripes; + size_t stripeStart = r.start * stripeSize; + size_t stripeEnd = r.end == nstripes ? total : std::min(r.end * stripeSize, total); + size_t wstep = weights->step1(); + AutoBuffer srcbuf(vecsize_aligned + valign); + int8_t* sptr = alignPtr(srcbuf.data(), (int)(valign * sizeof(int8_t))); + + for (k = vecsize; k < vecsize_aligned; k++) + sptr[k] = 0; + + for (size_t ofs = stripeStart; ofs < stripeEnd; ) + { + int sampleIdx = (int)(ofs / nw0); + int delta = (int)(ofs - (size_t)sampleIdx * nw0); + const int8_t* sptr_ = srcMat->ptr(sampleIdx); + const int8_t* wptr = weights->ptr(delta); + int* dptr = dstMat->ptr(sampleIdx) + delta; + const int* biasptr = biasMat->ptr() + delta; + const float* multptr = outputMultiplier->ptr() + delta; + int nw = std::min(nw0 - delta, (int)(stripeEnd - ofs)); + + memcpy(sptr, sptr_, vecsize * sizeof(sptr[0])); + #if CV_TRY_AVX512_SKX + if (useAVX512) + opt_AVX512_SKX::fastGEMM1T(sptr, wptr, wstep, biasptr, multptr, dptr, nw, vecsize, outZp); + else + #endif + #if CV_TRY_AVX2 + if (useAVX2) + opt_AVX2::fastGEMM1T(sptr, wptr, wstep, biasptr, multptr, dptr, nw, vecsize, outZp); + else + #endif + #if CV_TRY_LASX + if (useLASX) + opt_LASX::fastGEMM1T(sptr, wptr, wstep, biasptr, multptr, dptr, nw, vecsize, outZp); + else + #endif + #if CV_TRY_RVV && CV_RVV + if (useRVV) + opt_RVV::fastGEMM1T(sptr, wptr, wstep, biasptr, multptr, dptr, nw, vecsize, outZp); + else + #endif + #if CV_RVP052 + if (1) + opt_RVP052::fastGEMM1T(sptr, wptr, wstep, biasptr, multptr, dptr, nw, vecsize, outZp); + else + #endif + { + int i = 0; + #if CV_SIMD128 + for (; i <= nw - 4; i += 4, wptr += 4 * wstep) + { + v_int32x4 vs0 = v_setzero_s32(), vs1 = v_setzero_s32(), + vs2 = v_setzero_s32(), vs3 = v_setzero_s32(); + v_int32x4 outzp = v_setall_s32(outZp), outmin = v_setall_s32(-128), outmax = v_setall_s32(127); + v_int32x4 s = v_load(biasptr + i); + v_float32x4 mult = v_load(multptr + i); + + for (k = 0; k < vecsize; k += 16) + { + v_int8x16 v = v_load_aligned(sptr + k); + vs0 = v_dotprod_expand_fast(v, v_load_aligned(wptr + k), vs0); + vs1 = v_dotprod_expand_fast(v, v_load_aligned(wptr + wstep + k), vs1); + vs2 = v_dotprod_expand_fast(v, v_load_aligned(wptr + wstep * 2 + k), vs2); + vs3 = v_dotprod_expand_fast(v, v_load_aligned(wptr + wstep * 3 + k), vs3); + } + + s = v_add(s, v_int32x4(v_reduce_sum(vs0), v_reduce_sum(vs1), v_reduce_sum(vs2), v_reduce_sum(vs3))); + v_int32x4 out = v_add(outzp, v_round(v_mul(v_cvt_f32(s), mult))); + v_store(dptr + i, v_min(v_max(out, outmin), outmax)); + } + #endif + + for (; i < nw; i++, wptr += wstep) + { + int s0 = biasptr[i]; + float mult0 = multptr[i]; + + for (k = 0; k < vecsize; k++) + { + int8_t v = sptr[k]; + s0 += (int)v * wptr[k]; + } + int out0 = outZp + (int)std::round(s0 * mult0); + dptr[i] = std::min(std::max(out0, -128), 127); + } + } + + ofs += nw; + } + } + + const Mat *srcMat, *weights, *biasMat, *outputMultiplier; + Mat* dstMat; + int nstripes, outZp; + bool useAVX2; + bool useAVX512; + bool useLASX; + bool useRVV; + }; + + int inp_dims; + int num_output; + Mat weightsMat, biasMat, outputMultiplier; +}; + +Ptr MatMulInt8Layer::create(const MatMulInt8Params& params) +{ + return Ptr(new MatMulInt8LayerImpl(params)); +} + +} +} diff --git a/modules/dnn/src/int8layers/pool2_int8_layer.cpp b/modules/dnn/src/int8layers/pool2_int8_layer.cpp index def9bcfa24..f3fe28c34e 100644 --- a/modules/dnn/src/int8layers/pool2_int8_layer.cpp +++ b/modules/dnn/src/int8layers/pool2_int8_layer.cpp @@ -259,6 +259,25 @@ public: output_zp = params.get("zeropoints", 0); } + Pool2Int8LayerImpl(const Pool2Int8Params& p) + { + name = p.name; + type = "Pool2Int8"; + kernel_shape = p.kernel_shape; + strides = p.strides; + dilations = p.dilations; + pads = p.pads; + auto_pad = p.auto_pad; + ceil_mode = p.ceil_mode; + is_global_pooling = p.is_global_pooling; + is_max_pool = p.is_max_pool; + count_include_pad = p.count_include_pad; + input_sc = p.input_sc; + input_zp = p.input_zp; + output_sc = p.output_sc; + output_zp = p.output_zp; + } + void getTypes(const std::vector& inputs, const int requiredOutputs, const int, std::vector& outputs, @@ -377,5 +396,10 @@ Ptr Pool2Int8Layer::create(const LayerParams& params) return Ptr(new Pool2Int8LayerImpl(params)); } +Ptr Pool2Int8Layer::create(const Pool2Int8Params& params) +{ + return Ptr(new Pool2Int8LayerImpl(params)); +} + } // namespace dnn } // namespace cv diff --git a/modules/dnn/src/layer.cpp b/modules/dnn/src/layer.cpp index 9cc8fecc22..74a295a2b2 100644 --- a/modules/dnn/src/layer.cpp +++ b/modules/dnn/src/layer.cpp @@ -269,9 +269,9 @@ void Layer::getTypes(const std::vector&inputs, if (preferableTarget == DNN_TARGET_CUDA_FP16 || preferableTarget == DNN_TARGET_CUDA) CV_CheckTypeEQ(input, CV_32F, ""); else if (preferableTarget == DNN_TARGET_OPENCL_FP16) - CV_CheckType(input, input == CV_16F || input == CV_8S || input == CV_64F || input == CV_64S, ""); + CV_CheckType(input, input == CV_16F || input == CV_8S || input == CV_8U || input == CV_64F || input == CV_64S, ""); else - CV_CheckType(input, input == CV_32F || input == CV_64F || input == CV_8S || input == CV_64S, ""); + CV_CheckType(input, input == CV_32F || input == CV_64F || input == CV_8S || input == CV_8U || input == CV_64S, ""); } outputs.assign(requiredOutputs, inputs[0]); @@ -314,6 +314,11 @@ bool Layer::dynamicOutputShapes() const return false; } +bool Layer::isDataShuffling() const +{ + return false; +} + std::ostream& Layer::dumpAttrs(std::ostream& strm, int) const { return strm; diff --git a/modules/dnn/src/layers/concat_layer.cpp b/modules/dnn/src/layers/concat_layer.cpp index a42551a390..34e5aac174 100644 --- a/modules/dnn/src/layers/concat_layer.cpp +++ b/modules/dnn/src/layers/concat_layer.cpp @@ -78,6 +78,8 @@ public: scale = params.get("scales", 1.0f); } + bool isDataShuffling() const CV_OVERRIDE { return true; } + virtual bool getMemoryShapes(const std::vector &inputs, const int requiredOutputs, std::vector &outputs, diff --git a/modules/dnn/src/layers/dequantizelinear_layer.cpp b/modules/dnn/src/layers/dequantizelinear_layer.cpp index 0997748dfa..5f6b7f499e 100644 --- a/modules/dnn/src/layers/dequantizelinear_layer.cpp +++ b/modules/dnn/src/layers/dequantizelinear_layer.cpp @@ -170,12 +170,15 @@ static void dequantizeLinear(const Mat& inp, const Mat& scale_, const Mat& zp, CV_Assert(zpshape == scshape); } - axis = normalize_axis(axis, ndims); + if (ndims > 0) + axis = normalize_axis(axis, ndims); + else + axis = 0; for (i = 0; i < axis; i++) nslices *= inpshape[i]; for (i = axis+1; i < ndims; i++) slice_size *= inpshape[i]; - int sz_a = inpshape[axis]; + int sz_a = ndims > 0 ? inpshape[axis] : 1; if (block_size == 0) { size_t sc_total = scshape.total(); diff --git a/modules/dnn/src/layers/flatten_layer.cpp b/modules/dnn/src/layers/flatten_layer.cpp index 7747bf9aad..8ed9d13235 100644 --- a/modules/dnn/src/layers/flatten_layer.cpp +++ b/modules/dnn/src/layers/flatten_layer.cpp @@ -87,6 +87,8 @@ public: backendId == DNN_BACKEND_CANN; } + bool isDataShuffling() const CV_OVERRIDE { return true; } + bool getMemoryShapes(const std::vector &inputs, const int requiredOutputs, std::vector &outputs, diff --git a/modules/dnn/src/layers/maxpool_layer.cpp b/modules/dnn/src/layers/maxpool_layer.cpp index 79daeab4b8..135f5226ba 100644 --- a/modules/dnn/src/layers/maxpool_layer.cpp +++ b/modules/dnn/src/layers/maxpool_layer.cpp @@ -456,6 +456,8 @@ public: temptypes.clear(); } + bool isDataShuffling() const CV_OVERRIDE { return true; } + virtual bool getMemoryShapes(const std::vector& inpshapes, const int, std::vector &outshapes, diff --git a/modules/dnn/src/layers/permute_layer.cpp b/modules/dnn/src/layers/permute_layer.cpp index 7c6de8c277..8ed626409c 100644 --- a/modules/dnn/src/layers/permute_layer.cpp +++ b/modules/dnn/src/layers/permute_layer.cpp @@ -147,6 +147,8 @@ public: backendId == DNN_BACKEND_CANN; } + bool isDataShuffling() const CV_OVERRIDE { return true; } + bool getMemoryShapes(const std::vector &inputs, const int requiredOutputs, std::vector &outputs, diff --git a/modules/dnn/src/layers/quantlizelinear_layer.cpp b/modules/dnn/src/layers/quantlizelinear_layer.cpp index 99c7af4468..023da90b7b 100644 --- a/modules/dnn/src/layers/quantlizelinear_layer.cpp +++ b/modules/dnn/src/layers/quantlizelinear_layer.cpp @@ -180,12 +180,15 @@ static void quantizeLinear(const Mat& inp, const Mat& scale_, const Mat& zp, CV_Assert(zpshape == scshape); } - axis = normalize_axis(axis, ndims); + if (ndims > 0) + axis = normalize_axis(axis, ndims); + else + axis = 0; for (i = 0; i < axis; i++) nslices *= inpshape[i]; for (i = axis+1; i < ndims; i++) slice_size *= inpshape[i]; - int sz_a = inpshape[axis]; + int sz_a = ndims > 0 ? inpshape[axis] : 1; if (block_size == 0) { size_t sc_total = scshape.total(); diff --git a/modules/dnn/src/layers/reshape2_layer.cpp b/modules/dnn/src/layers/reshape2_layer.cpp index 3fad397c9c..2f95325056 100644 --- a/modules/dnn/src/layers/reshape2_layer.cpp +++ b/modules/dnn/src/layers/reshape2_layer.cpp @@ -57,6 +57,8 @@ public: } } + bool isDataShuffling() const CV_OVERRIDE { return true; } + virtual bool dynamicOutputShapes() const CV_OVERRIDE { if (!dynamicShapeSpec) diff --git a/modules/dnn/src/layers/reshape_layer.cpp b/modules/dnn/src/layers/reshape_layer.cpp index b7d1147f1e..70f4362c34 100644 --- a/modules/dnn/src/layers/reshape_layer.cpp +++ b/modules/dnn/src/layers/reshape_layer.cpp @@ -239,6 +239,8 @@ public: backendId == DNN_BACKEND_CANN; } + bool isDataShuffling() const CV_OVERRIDE { return true; } + bool getMemoryShapes(const std::vector &inputs, const int requiredOutputs, std::vector &outputs, diff --git a/modules/dnn/src/layers/slice2_layer.cpp b/modules/dnn/src/layers/slice2_layer.cpp index d7edd7148b..ce12fb1631 100644 --- a/modules/dnn/src/layers/slice2_layer.cpp +++ b/modules/dnn/src/layers/slice2_layer.cpp @@ -128,6 +128,8 @@ public: return outShape; } + bool isDataShuffling() const CV_OVERRIDE { return true; } + bool getMemoryShapes(const std::vector &inputs, const int, std::vector &outputs, diff --git a/modules/dnn/src/layers/slice_layer.cpp b/modules/dnn/src/layers/slice_layer.cpp index 24db981eed..eccb2c1daf 100644 --- a/modules/dnn/src/layers/slice_layer.cpp +++ b/modules/dnn/src/layers/slice_layer.cpp @@ -237,6 +237,8 @@ public: return backendId == DNN_BACKEND_OPENCV || backendId == DNN_BACKEND_CANN; } + bool isDataShuffling() const CV_OVERRIDE { return true; } + bool getMemoryShapes(const std::vector &inputs, const int requiredOutputs, std::vector &outputs, diff --git a/modules/dnn/src/layers/split2_layer.cpp b/modules/dnn/src/layers/split2_layer.cpp index 27c4d4471c..bac2e871b1 100644 --- a/modules/dnn/src/layers/split2_layer.cpp +++ b/modules/dnn/src/layers/split2_layer.cpp @@ -142,6 +142,8 @@ public: } } + bool isDataShuffling() const CV_OVERRIDE { return true; } + bool getMemoryShapes(const std::vector &inputs, const int noutputs, std::vector &outputs, diff --git a/modules/dnn/src/layers/split_layer.cpp b/modules/dnn/src/layers/split_layer.cpp index 48146e2195..b7d3f44af8 100644 --- a/modules/dnn/src/layers/split_layer.cpp +++ b/modules/dnn/src/layers/split_layer.cpp @@ -78,6 +78,8 @@ public: backendId == DNN_BACKEND_CUDA; } + bool isDataShuffling() const CV_OVERRIDE { return true; } + bool getMemoryShapes(const std::vector &inputs, const int requiredOutputs, std::vector &outputs, diff --git a/modules/dnn/src/layers/squeeze_layer.cpp b/modules/dnn/src/layers/squeeze_layer.cpp index b80587dcb4..d4e07fe973 100644 --- a/modules/dnn/src/layers/squeeze_layer.cpp +++ b/modules/dnn/src/layers/squeeze_layer.cpp @@ -81,6 +81,8 @@ public: return outShape; } + bool isDataShuffling() const CV_OVERRIDE { return true; } + bool getMemoryShapes(const std::vector &inputs, const int, std::vector &outputs, diff --git a/modules/dnn/src/layers/tile2_layer.cpp b/modules/dnn/src/layers/tile2_layer.cpp index d8e7f0e6be..7137aa4220 100644 --- a/modules/dnn/src/layers/tile2_layer.cpp +++ b/modules/dnn/src/layers/tile2_layer.cpp @@ -139,6 +139,8 @@ public: return backendId == DNN_BACKEND_OPENCV; } + bool isDataShuffling() const CV_OVERRIDE { return true; } + virtual bool dynamicOutputShapes() const CV_OVERRIDE { Net::Impl* netimpl_ = getNetImpl(this); diff --git a/modules/dnn/src/layers/tile_layer.cpp b/modules/dnn/src/layers/tile_layer.cpp index d24d11786c..7fb5a79a74 100644 --- a/modules/dnn/src/layers/tile_layer.cpp +++ b/modules/dnn/src/layers/tile_layer.cpp @@ -38,6 +38,8 @@ public: backendId == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH; } + bool isDataShuffling() const CV_OVERRIDE { return true; } + virtual bool getMemoryShapes(const std::vector &inputs, const int requiredOutputs, std::vector &outputs, diff --git a/modules/dnn/src/layers/transpose_layer.cpp b/modules/dnn/src/layers/transpose_layer.cpp index 3cb447fca1..bd793b38c8 100644 --- a/modules/dnn/src/layers/transpose_layer.cpp +++ b/modules/dnn/src/layers/transpose_layer.cpp @@ -148,6 +148,8 @@ public: return outShape; } + bool isDataShuffling() const CV_OVERRIDE { return true; } + bool getMemoryShapes(const std::vector &inputs, const int, std::vector &outputs, diff --git a/modules/dnn/src/layers/unsqueeze_layer.cpp b/modules/dnn/src/layers/unsqueeze_layer.cpp index a68c203f24..a57fae90da 100644 --- a/modules/dnn/src/layers/unsqueeze_layer.cpp +++ b/modules/dnn/src/layers/unsqueeze_layer.cpp @@ -78,6 +78,8 @@ public: return outShape; } + bool isDataShuffling() const CV_OVERRIDE { return true; } + bool getMemoryShapes(const std::vector &inputs, const int, std::vector &outputs, diff --git a/modules/dnn/src/onnx/onnx_importer2.cpp b/modules/dnn/src/onnx/onnx_importer2.cpp index 132d715d75..f642b09d97 100644 --- a/modules/dnn/src/onnx/onnx_importer2.cpp +++ b/modules/dnn/src/onnx/onnx_importer2.cpp @@ -266,15 +266,21 @@ protected: void parseDequantizeLinear (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); void parseQuantizeLinear (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); void parseCustomLayer (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); - //void parseQAvgPool (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); - //void parseQConcat (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); - //void parseQConv (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); - //void parseQEltwise (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); - //void parseQGemm (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); - //void parseQLeakyRelu (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); - //void parseQMatMul (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); - //void parseQSigmoid (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); - //void parseQSoftmax (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseQConv (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseQMatMul (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseQGemm (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseQEltwise (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseQLeakyRelu (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseQSigmoid (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseQAvgPool (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseQConcat (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseQSoftmax (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + + Arg addDequantize(const std::string& name, const std::vector& inputs, + const opencv_onnx::NodeProto& node_proto, int axis = -1); + void addQuantize(const std::string& name, const Arg& data, + const std::vector& scale_zp, const std::vector& outputs, + const opencv_onnx::NodeProto& node_proto); std::map onnx_opset_map; // map from OperatorSetIdProto void parseOperatorSet(); @@ -2073,559 +2079,323 @@ void ONNXImporter2::parseRandomNormalLike(LayerParams& layerParams, const opencv addLayer(layerParams, node_proto); } -// BUG: https://github.com/opencv/opencv/issues/26310 -/*void ONNXImporter2::parseQConv(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto_) +// All Q* parsers decompose quantized ops into: DequantizeLinear → float op → QuantizeLinear. +// The fused Int8 layers don't work with the new graph engine (issue #26310). + +Arg ONNXImporter2::addDequantize(const std::string& name, const std::vector& inputs, + const opencv_onnx::NodeProto& node_proto, int axis) +{ + LayerParams lp; + lp.name = name; + lp.type = "DequantizeLinear"; + if (axis >= 0) + lp.set("axis", axis); + Arg out = net.getArg(name); + node_inputs = inputs; + node_outputs = {out}; + addLayer(lp, node_proto); + return out; +} + +void ONNXImporter2::addQuantize(const std::string& name, const Arg& data, + const std::vector& scale_zp, + const std::vector& outputs, + const opencv_onnx::NodeProto& node_proto) +{ + LayerParams lp; + lp.name = name; + lp.type = "QuantizeLinear"; + node_inputs = {data}; + node_inputs.insert(node_inputs.end(), scale_zp.begin(), scale_zp.end()); + node_outputs = outputs; + addLayer(lp, node_proto); +} + +static void copyAttrs(const LayerParams& src, LayerParams& dst) +{ + for (auto it = src.begin(); it != src.end(); ++it) + { + const std::string& key = it->first; + if (key != "name" && key != "type") + dst.set(key, it->second); + } +} + +void ONNXImporter2::parseQConv(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) { - opencv_onnx::NodeProto node_proto = node_proto_; int ninputs = node_proto.input_size(); CV_Assert(ninputs == 8 || ninputs == 9); - float inp_sc = getScalarFromMat(getBlob(node_proto, 1)); - int inp_zp = (int)getScalarFromMat(getBlob(node_proto, 2)); + std::string bn = layerParams.name; + std::vector inp = node_inputs; + std::vector out = node_outputs; - if (layerParams.has("pad")) + Arg dq_x, dq_w; + dq_x = addDequantize(bn + "/dequant_x", {inp[0], inp[1], inp[2]}, node_proto); + dq_w = addDequantize(bn + "/dequant_w", {inp[3], inp[4], inp[5]}, node_proto, 0); + + Arg bias_arg; + bool has_bias = (ninputs == 9); + if (has_bias) { - bool asymmetricPadding = false; - DictValue pads = layerParams.get("pad"); - const int dims = pads.size() / 2; - - for (int i = 0; i < dims; ++i) + if (net.isConstArg(inp[1]) && net.isConstArg(inp[4]) && net.isConstArg(inp[8])) { - if (pads.get(i) != pads.get(i + dims)) - { - asymmetricPadding = true; - break; - } + Mat x_scale_mat = net.argTensor(inp[1]); + Mat w_scale_mat = net.argTensor(inp[4]); + Mat bias_int32 = net.argTensor(inp[8]); + + Mat bias_float; + bias_int32.convertTo(bias_float, CV_32F); + float x_sc = x_scale_mat.at(0); + if (w_scale_mat.total() == 1) + bias_float *= x_sc * w_scale_mat.at(0); + else + for (int i = 0; i < (int)w_scale_mat.total(); i++) + bias_float.at(i) *= x_sc * w_scale_mat.at(i); + + bias_arg = netimpl->newConstArg(bn + "/bias_float", bias_float); } - if (asymmetricPadding && pads.size() == 4) + else { - layerParams.erase("pad"); - std::vector paddings(4, 0); - for (int i = 0; i < dims; ++i) - { - paddings.push_back(pads.get(i)); - paddings.push_back(pads.get(dims + i)); - } - LayerParams padLp; - padLp.name = layerParams.name + "/pad"; - padLp.type = "PaddingInt8"; - padLp.set("paddings", DictValue::arrayInt(&paddings[0], paddings.size())); - padLp.set("depth", CV_8S); - padLp.set("value", (double)inp_zp); - - opencv_onnx::NodeProto proto; - proto.add_input(node_proto.input(0)); - proto.add_output(padLp.name); - - addLayer(padLp, proto); - node_proto.set_input(0, padLp.name); + bias_arg = addDequantize(bn + "/dequant_bias", {inp[8], inp[1]}, node_proto); } } - Mat weights = getBlob(node_proto, 3); - int outCn = weights.size[0]; - Mat w_scale = getBlob(node_proto, 4); - CV_Assert(w_scale.total() == 1 || w_scale.total() == outCn); - bool per_channel = w_scale.total() == outCn; - Mat wt_sc = (w_scale.total() == outCn) ? w_scale : Mat(1, outCn, CV_32F, Scalar(w_scale.at(0))); + LayerParams convLp; + convLp.name = bn + "/conv"; + convLp.type = "Conv2"; + copyAttrs(layerParams, convLp); + Arg conv_out = net.getArg(convLp.name); + node_inputs = has_bias ? std::vector{dq_x, dq_w, bias_arg} : std::vector{dq_x, dq_w}; + node_outputs = {conv_out}; + addLayer(convLp, node_proto); - float out_sc = getScalarFromMat(getBlob(node_proto, 6)); - int8_t out_zp = getScalarFromMat(getBlob(node_proto, 7)); - - Mat bias = (ninputs == 9) ? getBlob(node_proto, 8) : Mat::zeros(1, outCn, CV_32S); - - Mat weights_2d = weights.reshape(1, outCn); - Mat biasFused(1, outCn, CV_32S); - Mat outputMultiplier(1, outCn, CV_32F); - for (int i = 0; i < outCn; i++) - { - biasFused.at(i) = bias.at(i) - inp_zp*(cv::sum(weights_2d.row(i))[0]); - outputMultiplier.at(i) = (inp_sc * wt_sc.at(i)) / out_sc; - } - - layerParams.type = "ConvolutionInt8"; - layerParams.set("num_output", outCn); - layerParams.set("input_zeropoint", inp_zp); - layerParams.set("input_scale",inp_sc); - layerParams.set("zeropoints", out_zp); - layerParams.set("scales", out_sc); - layerParams.set("per_channel", per_channel); - layerParams.blobs.push_back(weights); - layerParams.blobs.push_back(biasFused); - layerParams.blobs.push_back(outputMultiplier); - addLayer(layerParams, node_proto); + addQuantize(bn + "/quant_y", conv_out, {inp[6], inp[7]}, out, node_proto); } void ONNXImporter2::parseQMatMul(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) { - int ninputs = node_proto.input_size(); - CV_Assert(ninputs == 8); + CV_Assert(node_proto.input_size() == 8); - if (constBlobs.find(node_proto.input(3)) == constBlobs.end()) - CV_Error(Error::StsNotImplemented, "Variable weights is not supported"); + std::string bn = layerParams.name; + std::vector inp = node_inputs; + std::vector out = node_outputs; - int firstInpDims = outShapes[node_proto.input(0)].size(); + Arg dq_a, dq_b; + dq_a = addDequantize(bn + "/dequant_a", {inp[0], inp[1], inp[2]}, node_proto); + dq_b = addDequantize(bn + "/dequant_b", {inp[3], inp[4], inp[5]}, node_proto); - float inp_sc = getScalarFromMat(getBlob(node_proto, 1)); - int8_t inp_zp = getScalarFromMat(getBlob(node_proto, 2)); + LayerParams mmLp; + mmLp.name = bn + "/matmul"; + mmLp.type = "MatMul"; + Arg mm_out = net.getArg(mmLp.name); + node_inputs = {dq_a, dq_b}; + node_outputs = {mm_out}; + addLayer(mmLp, node_proto); - Mat weights = getBlob(node_proto, 3).t(); - int outCn = weights.size[0]; - int secondInpDims = weights.dims; - - Mat w_scale = getBlob(node_proto, 4); - CV_Assert(w_scale.total() == 1 || w_scale.total() == outCn); - bool per_channel = w_scale.total() == outCn ? true : false; - Mat wt_sc = (w_scale.total() == outCn) ? w_scale : Mat(1, outCn, CV_32F, Scalar(w_scale.at(0))); - - float out_sc = getScalarFromMat(getBlob(node_proto, 6)); - int8_t out_zp = getScalarFromMat(getBlob(node_proto, 7)); - - Mat bias(1, outCn, CV_32S); - Mat outputMultiplier(1, outCn, CV_32F); - for (int i = 0; i < outCn; i++) - { - bias.at(i) = -inp_zp*(cv::sum(weights.row(i))[0]); - outputMultiplier.at(i) = (inp_sc * wt_sc.at(i)) / out_sc; - } - - layerParams.type = "InnerProductInt8"; - layerParams.set("num_output", outCn); - layerParams.set("axis", firstInpDims - secondInpDims + 1); - layerParams.set("input_scale", inp_sc); - layerParams.set("input_zeropoint", inp_zp); - layerParams.set("zeropoints", out_zp); - layerParams.set("scales", out_sc); - layerParams.set("per_channel", per_channel); - - layerParams.blobs.push_back(weights); - layerParams.blobs.push_back(bias); - layerParams.blobs.push_back(outputMultiplier); - addLayer(layerParams, node_proto); + addQuantize(bn + "/quant_y", mm_out, {inp[6], inp[7]}, out, node_proto); } -// A * B + C = Y, we require that the dimension of A is [m, k], and the dimension of B is [n, k]. -// And the dim of output Y is [m, n] void ONNXImporter2::parseQGemm(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) { int ninputs = node_proto.input_size(); CV_Assert(ninputs == 8 || ninputs == 9); - layerParams.type = "InnerProductInt8"; + std::string bn = layerParams.name; + std::vector inp = node_inputs; + std::vector out = node_outputs; - if (constBlobs.find(node_proto.input(3)) == constBlobs.end()) - CV_Error(Error::StsNotImplemented, "Variable weights is not supported"); + Arg dq_a, dq_b; + dq_a = addDequantize(bn + "/dequant_a", {inp[0], inp[1], inp[2]}, node_proto); + dq_b = addDequantize(bn + "/dequant_b", {inp[3], inp[4], inp[5]}, node_proto, 0); - Mat weights = getBlob(node_proto, 3); + LayerParams gemmLp; + gemmLp.name = bn + "/gemm"; + gemmLp.type = "Gemm"; + copyAttrs(layerParams, gemmLp); + Arg gemm_out = net.getArg(gemmLp.name); + node_inputs = (!node_proto.input(6).empty()) ? std::vector{dq_a, dq_b, inp[6]} + : std::vector{dq_a, dq_b}; + node_outputs = {gemm_out}; + addLayer(gemmLp, node_proto); - if (!layerParams.get("transB", 0)) - { - transpose(weights, weights); - } - - CV_Assert(layerParams.get("alpha", 1) == 1.0f); - CV_Assert(layerParams.get("transA", 0) == 0); - - int firstInpDims = outShapes[node_proto.input(0)].size(); - - float inp_sc = getScalarFromMat(getBlob(node_proto, 1)); - int8_t inp_zp = getScalarFromMat(getBlob(node_proto, 2)); - - int outCn = weights.size[0]; - int secondInpDims = weights.dims; - - Mat w_scale = getBlob(node_proto, 4); - CV_Assert(w_scale.total() == 1 || w_scale.total() == outCn); - bool per_channel = w_scale.total() == outCn; - Mat wt_sc = (w_scale.total() == outCn) ? w_scale : Mat(1, outCn, CV_32F, Scalar(w_scale.at(0))); - - Mat w_zp = getBlob(node_proto, 5); - int8_t* ptrZp = w_zp.ptr(0); - - for (int i = 0; i < w_zp.total(); i++) - { - if (ptrZp[i] != (int8_t)0) - CV_Error(Error::StsUnsupportedFormat, "The zero-point non-zero case of W is not supported!"); - } - - float out_sc = getScalarFromMat(getBlob(node_proto, 7)); - int8_t out_zp = ninputs == 9 ? getScalarFromMat(getBlob(node_proto, 8)) : 0; - - Mat bias; - if (constBlobs.find(node_proto.input(6)) != constBlobs.end()) - bias = getBlob(node_proto, 6); - if (bias.empty()) - bias = Mat::zeros(1, outCn, CV_32S); - - Mat biasFused(1, outCn, CV_32S); - Mat outputMultiplier(1, outCn, CV_32F); - for (int i = 0; i < outCn; i++) - { - biasFused.at(i) = bias.at(i) - inp_zp*(cv::sum(weights.row(i))[0]); - outputMultiplier.at(i) = (inp_sc * wt_sc.at(i)) / out_sc; - } - - layerParams.type = "InnerProductInt8"; - layerParams.set("num_output", outCn); - layerParams.set("axis", firstInpDims - secondInpDims + 1); - layerParams.set("input_scale", inp_sc); - layerParams.set("input_zeropoint", inp_zp); - layerParams.set("scales", out_sc); - layerParams.set("zeropoints", out_zp); - layerParams.set("per_channel", per_channel); - - layerParams.blobs.push_back(weights); - layerParams.blobs.push_back(biasFused); - layerParams.blobs.push_back(outputMultiplier); - addLayer(layerParams, node_proto); + std::vector q_sc_zp = {inp[7]}; + if (ninputs == 9) + q_sc_zp.push_back(inp[8]); + addQuantize(bn + "/quant_y", gemm_out, q_sc_zp, out, node_proto); } -void ONNXImporter2::parseQEltwise(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto_) +void ONNXImporter2::parseQEltwise(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) { - opencv_onnx::NodeProto node_proto = node_proto_; - CV_Assert(node_proto.input_size() == 7 || node_proto.input_size() == 8); - std::string op = (node_proto.op_type() == "QLinearAdd") ? "sum" : "prod"; - int constId = -1; - for (int i = 0; i < 4; i += 3) - { - if (constBlobs.find(node_proto.input(i)) != constBlobs.end()) - constId = i; - } + int ninputs = node_proto.input_size(); + CV_Assert(ninputs == 7 || ninputs == 8); - float inp_0_sc = getScalarFromMat(getBlob(node_proto, 1)); - int8_t inp_0_zp = getScalarFromMat(getBlob(node_proto, 2)); + std::string bn = layerParams.name; + std::vector inp = node_inputs; + std::vector out = node_outputs; - float inp_1_sc = getScalarFromMat(getBlob(node_proto, 4)); - int8_t inp_1_zp = getScalarFromMat(getBlob(node_proto, 5)); + Arg dq_a, dq_b; + dq_a = addDequantize(bn + "/dequant_a", {inp[0], inp[1], inp[2]}, node_proto); + dq_b = addDequantize(bn + "/dequant_b", {inp[3], inp[4], inp[5]}, node_proto); - // Set 2nd input as the const input - if (constId == 0) - { - cv::swap(inp_0_sc, inp_1_sc); - cv::swap(inp_0_zp, inp_1_zp); - } + LayerParams eltLp; + eltLp.name = bn + "/eltwise"; + eltLp.type = "NaryEltwise"; + eltLp.set("operation", (node_proto.op_type() == "QLinearAdd") ? "add" : "mul"); + Arg elt_out = net.getArg(eltLp.name); + node_inputs = {dq_a, dq_b}; + node_outputs = {elt_out}; + addLayer(eltLp, node_proto); - float out_sc = getScalarFromMat(getBlob(node_proto, 6)); - - int8_t out_zp = 0; - if (node_proto.input_size() == 8) - out_zp = getScalarFromMat(getBlob(node_proto, 7)); - - std::vector inp_scales = {inp_0_sc, inp_1_sc}; - std::vector inp_zps = {inp_0_zp, inp_1_zp}; - - std::vector coeffs; - float offset; - if (op == "sum") - { - coeffs = {inp_scales[0]/out_sc, inp_scales[1]/out_sc}; - offset = out_zp - coeffs[0]*inp_zps[0] - coeffs[1]*inp_zps[1]; - } - else - { - coeffs = {inp_scales[0]/out_sc, inp_scales[1]}; - offset = out_zp; - } - - if (constId != -1) - { - Mat blob = getBlob(node_proto, constId); - if (blob.total() == 1) - { - float val = inp_scales[1] * (blob.at(0) - inp_zps[1]); - float scale = inp_scales[0] / out_sc; - if (op == "prod") - scale *= val; - - float shift = out_zp - scale*inp_zps[0]; - if (op == "sum") - shift += (val/out_sc); - - LayerParams rescaleParams; - rescaleParams.name = layerParams.name; - rescaleParams.type = "Requantize"; - rescaleParams.set("depth", CV_8S); - rescaleParams.set("scale", scale); - rescaleParams.set("shift", shift); - rescaleParams.set("isEltwise", true); - addLayer(rescaleParams, node_proto); - return; - } - else - { - MatShape inpShape = outShapes[node_proto.input(3 - constId)]; - if (blob.dims == 2) - blob = blob.t(); - - if (shape(blob) == inpShape) - { - LayerParams constParams; - constParams.name = layerParams.name + "/const"; - constParams.type = "ConstInt8"; - constParams.set("depth", CV_8S); - constParams.set("scales", inp_1_sc); - constParams.set("zeropoints", inp_1_zp); - constParams.blobs.push_back(blob); - - int id = net.addLayer(constParams.name, constParams.type, CV_8S, constParams); - layer_id.insert(std::make_pair(constParams.name, LayerInfo(id, 0, CV_8S))); - outShapes[constParams.name] = shape(blob); - node_proto.set_input(constId, constParams.name); - - layerParams.type = "EltwiseInt8"; - layerParams.set("operation", op); - layerParams.set("coeff", DictValue::arrayReal(coeffs.data(), coeffs.size())); - layerParams.set("offset", offset); - } - else - { - layerParams.type = "ScaleInt8"; - layerParams.set("bias_term", op == "sum"); - int axis = 1; - for (int i = 0; i < graph_proto->initializer_size(); i++) - { - opencv_onnx::TensorProto tensor_proto = graph_proto->initializer(i); - if (tensor_proto.name() == node_proto.input(constId)) - { - axis = inpShape.size() - tensor_proto.dims_size(); - break; - } - } - layerParams.set("axis", axis); - blob = blob.reshape(1, 1); - Mat blob_dequantized; - blob.convertTo(blob_dequantized, CV_32F, inp_scales[1], -(inp_scales[1] * inp_zps[1])); - layerParams.blobs.push_back(blob_dequantized); - } - } - } - else if (outShapes[node_proto.input(0)] == outShapes[node_proto.input(3)]) - { - layerParams.type = "EltwiseInt8"; - layerParams.set("operation", op); - layerParams.set("coeff", DictValue::arrayReal(coeffs.data(), coeffs.size())); - layerParams.set("offset", offset); - } - else - { - layerParams.type = "ScaleInt8"; - layerParams.set("bias_term", op == "sum"); - } - - layerParams.set("input_scales", DictValue::arrayReal(inp_scales.data(), inp_scales.size())); - layerParams.set("input_zeropoints", DictValue::arrayInt(inp_zps.data(), inp_zps.size())); - layerParams.set("scales", out_sc); - layerParams.set("zeropoints", out_zp); - - addLayer(layerParams, node_proto); + std::vector q_sc_zp = {inp[6]}; + if (ninputs == 8) + q_sc_zp.push_back(inp[7]); + addQuantize(bn + "/quant_y", elt_out, q_sc_zp, out, node_proto); } void ONNXImporter2::parseQLeakyRelu(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) { - CV_Assert(node_proto.input_size() == 4 || node_proto.input_size() == 5); + int ninputs = node_proto.input_size(); + CV_Assert(ninputs == 4 || ninputs == 5); - float slope = layerParams.get("alpha"); - float inp_sc = getScalarFromMat(getBlob(node_proto, 1)); - int8_t inp_zp = getScalarFromMat(getBlob(node_proto, 2)); - float out_sc = getScalarFromMat(getBlob(node_proto, 3)); - int8_t out_zp = node_proto.input_size() == 4 ? 0 : getScalarFromMat(getBlob(node_proto, 4)); + std::string bn = layerParams.name; + std::vector inp = node_inputs; + std::vector out = node_outputs; - Mat lookUpTable(1, 256, CV_8S); - int8_t* table = lookUpTable.ptr(); - for (int i = -128; i < 128; i++) - { - float x = inp_sc*(i - inp_zp); - float y = x >= 0.f ? x : slope*x; - int quantized = out_zp + cvRound(y/out_sc); - table[i+128] = saturate_cast(quantized); - } + Arg dq_x; + dq_x = addDequantize(bn + "/dequant_x", {inp[0], inp[1], inp[2]}, node_proto); - layerParams.type = "ReLUInt8"; - layerParams.set("input_scale", inp_sc); - layerParams.set("input_zeropoint", inp_zp); - layerParams.set("scales", out_sc); - layerParams.set("zeropoints", out_zp); - layerParams.set("slope", slope); - layerParams.blobs.push_back(lookUpTable); - addLayer(layerParams, node_proto); + LayerParams reluLp; + reluLp.name = bn + "/leaky_relu"; + reluLp.type = "ReLU"; + reluLp.set("negative_slope", layerParams.get("alpha", 0.01f)); + Arg relu_out = net.getArg(reluLp.name); + node_inputs = {dq_x}; + node_outputs = {relu_out}; + addLayer(reluLp, node_proto); + + std::vector q_sc_zp = {inp[3]}; + if (ninputs == 5) + q_sc_zp.push_back(inp[4]); + addQuantize(bn + "/quant_y", relu_out, q_sc_zp, out, node_proto); } void ONNXImporter2::parseQSigmoid(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) { - CV_Assert(node_proto.input_size() == 4 || node_proto.input_size() == 5); + int ninputs = node_proto.input_size(); + CV_Assert(ninputs == 4 || ninputs == 5); - float inp_sc = getScalarFromMat(getBlob(node_proto, 1)); - int8_t inp_zp = getScalarFromMat(getBlob(node_proto, 2)); - float out_sc = getScalarFromMat(getBlob(node_proto, 3)); - int8_t out_zp = node_proto.input_size() == 4 ? 0 : getScalarFromMat(getBlob(node_proto, 4)); + std::string bn = layerParams.name; + std::vector inp = node_inputs; + std::vector out = node_outputs; - Mat lookUpTable(1, 256, CV_8S); - int8_t* table = lookUpTable.ptr(); - for (int i = -128; i < 128; i++) - { - float x = inp_sc*(i - inp_zp); - float y = 1.f/(1.f + std::exp(-x)); - int quantized = out_zp + cvRound(y/out_sc); - table[i+128] = saturate_cast(quantized); - } + Arg dq_x; + dq_x = addDequantize(bn + "/dequant_x", {inp[0], inp[1], inp[2]}, node_proto); - layerParams.type = "SigmoidInt8"; - layerParams.set("input_scale", inp_sc); - layerParams.set("input_zeropoint", inp_zp); - layerParams.set("scales", out_sc); - layerParams.set("zeropoints", out_zp); - layerParams.blobs.push_back(lookUpTable); - addLayer(layerParams, node_proto); + LayerParams sigLp; + sigLp.name = bn + "/sigmoid"; + sigLp.type = "Sigmoid"; + Arg sig_out = net.getArg(sigLp.name); + node_inputs = {dq_x}; + node_outputs = {sig_out}; + addLayer(sigLp, node_proto); + + std::vector q_sc_zp = {inp[3]}; + if (ninputs == 5) + q_sc_zp.push_back(inp[4]); + addQuantize(bn + "/quant_y", sig_out, q_sc_zp, out, node_proto); } void ONNXImporter2::parseQAvgPool(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) { - CV_Assert(node_proto.input_size() == 4 || node_proto.input_size() == 5); + int ninputs = node_proto.input_size(); + CV_Assert(ninputs == 4 || ninputs == 5); - float inp_sc = getScalarFromMat(getBlob(node_proto, 1)); - int8_t inp_zp = getScalarFromMat(getBlob(node_proto, 2)); - float out_sc = getScalarFromMat(getBlob(node_proto, 3)); - int8_t out_zp = node_proto.input_size() == 4 ? 0 : getScalarFromMat(getBlob(node_proto, 4)); + std::string bn = layerParams.name; + std::vector inp = node_inputs; + std::vector out = node_outputs; - layerParams.type = "PoolingInt8"; - layerParams.set("pool", "ave"); - layerParams.set("global_pooling", node_proto.op_type() == "QLinearGlobalAveragePool"); - layerParams.set("multiplier", inp_sc/out_sc); - layerParams.set("input_scale", inp_sc); - layerParams.set("input_zeropoint", inp_zp); - layerParams.set("scales", out_sc); - layerParams.set("zeropoints", out_zp); - addLayer(layerParams, node_proto); + Arg dq_x; + dq_x = addDequantize(bn + "/dequant_x", {inp[0], inp[1], inp[2]}, node_proto); + + LayerParams poolLp; + poolLp.name = bn + "/avg_pool"; + poolLp.type = "Pooling"; + poolLp.set("pool", "ave"); + poolLp.set("global_pooling", node_proto.op_type() == "QLinearGlobalAveragePool"); + copyAttrs(layerParams, poolLp); + Arg pool_out = net.getArg(poolLp.name); + node_inputs = {dq_x}; + node_outputs = {pool_out}; + addLayer(poolLp, node_proto); + + std::vector q_sc_zp = {inp[3]}; + if (ninputs == 5) + q_sc_zp.push_back(inp[4]); + addQuantize(bn + "/quant_y", pool_out, q_sc_zp, out, node_proto); } -void ONNXImporter2::parseQConcat(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto_) +void ONNXImporter2::parseQConcat(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) { - opencv_onnx::NodeProto node_proto = node_proto_; - layerParams.type = "ConcatInt8"; - int num_inputs = node_proto.input_size(); + int ninputs = node_proto.input_size(); + CV_Assert(ninputs >= 5 && (ninputs - 2) % 3 == 0); - float out_scale = getScalarFromMat(getBlob(node_proto, 0)); - int8_t out_zp = getScalarFromMat(getBlob(node_proto, 1)); + std::string bn = layerParams.name; + std::vector inp = node_inputs; + std::vector out = node_outputs; - for (int i = 2; i < num_inputs; i += 3) + std::vector dq_outs; + int n = (ninputs - 2) / 3; + for (int i = 0; i < n; i++) { - float inp_scale = getScalarFromMat(getBlob(node_proto, i + 1)); - int8_t inp_zp = getScalarFromMat(getBlob(node_proto, i + 2)); - - if (inp_scale != out_scale || inp_zp != out_zp) - { - float scale = inp_scale/out_scale; - float shift = out_zp - scale*inp_zp; - - if (constBlobs.find(node_proto.input(i)) != constBlobs.end()) - { - Mat blob = getBlob(node_proto, i); - Mat blob_rescaled; - blob.convertTo(blob_rescaled, CV_8S, scale, shift); - constBlobs[node_proto.input(i)] = blob_rescaled; - } - else - { - LayerParams rescaleParams; - rescaleParams.name = node_proto.input(i) + "/rescale"; - rescaleParams.type = "Requantize"; - rescaleParams.set("depth", CV_8S); - rescaleParams.set("scale", scale); - rescaleParams.set("shift", shift); - rescaleParams.set("isEltwise", false); - - opencv_onnx::NodeProto proto; - proto.add_input(node_proto.input(i)); - proto.add_output(rescaleParams.name); - addLayer(rescaleParams, proto); - node_proto.set_input(i, rescaleParams.name); - } - } + int b = 2 + i * 3; + Arg dq = addDequantize(bn + "/dequant_" + std::to_string(i), + {inp[b], inp[b + 1], inp[b + 2]}, node_proto); + dq_outs.push_back(dq); } - bool hasVariableInps = false; - for (int i = 2; i < num_inputs; i += 3) - { - if (layer_id.find(node_proto.input(i)) != layer_id.end()) - { - hasVariableInps = true; - break; - } - } + LayerParams concatLp; + concatLp.name = bn + "/concat"; + concatLp.type = "Concat"; + concatLp.set("axis", layerParams.get("axis", 1)); + Arg concat_out = net.getArg(concatLp.name); + node_inputs = dq_outs; + node_outputs = {concat_out}; + addLayer(concatLp, node_proto); - if (!hasVariableInps) - { - std::vector inputs, concatenated; - MatShape inputShape; - for (size_t i = 2; i < num_inputs; i += 3) - { - Mat blob = getBlob(node_proto, i); - if (blob.size.dims() > inputShape.size()) - { - inputShape = shape(blob); - } - inputs.push_back(blob); - } - - int axis = layerParams.get("axis", 1); - for (size_t i = 0; i < inputs.size(); ++i) - { - MatShape targetShape = inputShape; - targetShape[axis] = shape(inputs[i])[axis]; - CV_CheckEQ(total(targetShape), total(shape(inputs[i])), ""); - inputs[i] = inputs[i].reshape(0, targetShape); - } - runLayer(layerParams, inputs, concatenated); - CV_Assert(concatenated.size() == 1); - addConstant(layerParams.name, concatenated[0]); - return; - } - else - { - for (int i = 2; i < num_inputs; i += 3) - { - if (constBlobs.find(node_proto.input(i)) != constBlobs.end()) - { - LayerParams constParams; - constParams.name = node_proto.input(i); - constParams.type = "ConstInt8"; - constParams.blobs.push_back(getBlob(node_proto, i)); - constParams.set("depth", CV_8S); - - opencv_onnx::NodeProto proto; - proto.add_output(constParams.name); - addLayer(constParams, proto); - } - } - } - layerParams.set("scales", out_scale); - layerParams.set("zeropoints", out_zp); - addLayer(layerParams, node_proto); + addQuantize(bn + "/quant_y", concat_out, {inp[0], inp[1]}, out, node_proto); } void ONNXImporter2::parseQSoftmax(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) { - CV_CheckEQ(node_proto.input_size(), 5, "DNN/ONNX: QLinearSoftmax requires 5 inputs, X, X_scale, X_zero_point, Y_scale, Y_zero_point"); + CV_CheckEQ(node_proto.input_size(), 5, "DNN/ONNX: QLinearSoftmax requires 5 inputs"); - int opset = layerParams.get("opset"); - if (opset < 13) { - layerParams.set("coerced_2d", true); + std::string bn = layerParams.name; + std::vector inp = node_inputs; + std::vector out = node_outputs; + + Arg dq_x; + dq_x = addDequantize(bn + "/dequant_x", {inp[0], inp[1], inp[2]}, node_proto); + + LayerParams smLp; + smLp.name = bn + "/softmax"; + smLp.type = "Softmax"; + if (layerParams.has("axis")) + smLp.set("axis", layerParams.get("axis")); + if (layerParams.has("opset")) + { + int opset = layerParams.get("opset"); + smLp.set("opset", opset); + if (opset < 13) + smLp.set("coerced_2d", true); } + Arg sm_out = net.getArg(smLp.name); + node_inputs = {dq_x}; + node_outputs = {sm_out}; + addLayer(smLp, node_proto); - float x_scale = getScalarFromMat(getBlob(node_proto, 1)); - int8_t x_zero_point = getScalarFromMat(getBlob(node_proto, 2)); - float y_scale = getScalarFromMat(getBlob(node_proto, 3)); - int8_t y_zero_point = getScalarFromMat(getBlob(node_proto, 4)); - - layerParams.type = "SoftmaxInt8"; - // layerParams also has "axis" and "opset" attrs - layerParams.set("input_scale", x_scale); - layerParams.set("input_zeropoint", x_zero_point); - layerParams.set("scales", y_scale); - layerParams.set("zeropoints", y_zero_point); - addLayer(layerParams, node_proto); -}*/ - + addQuantize(bn + "/quant_y", sm_out, {inp[3], inp[4]}, out, node_proto); +} void ONNXImporter2::parseRotaryEmbedding(LayerParams& params, const opencv_onnx::NodeProto& node_proto) { int i, n_inputs = node_proto.input_size(); @@ -2819,8 +2589,8 @@ void ONNXImporter2::buildDispatchMap_ONNX_AI() // ai.onnx: opset 10+ dispatch["DequantizeLinear"] = &ONNXImporter2::parseDequantizeLinear; dispatch["QuantizeLinear"] = &ONNXImporter2::parseQuantizeLinear; - //dispatch["QLinearConv"] = &ONNXImporter2::parseQConv; - //dispatch["QLinearMatMul"] = &ONNXImporter2::parseQMatMul; + dispatch["QLinearConv"] = &ONNXImporter2::parseQConv; + dispatch["QLinearMatMul"] = &ONNXImporter2::parseQMatMul; // com.microsft: This operator is added for compatibility via onnx graph simplifier. // Opset domain cannot be modified from onnx_graph_simplifier.cpp so this @@ -2836,14 +2606,15 @@ void ONNXImporter2::buildDispatchMap_COM_MICROSOFT() { DispatchMap dispatch; - // BUG: https://github.com/opencv/opencv/issues/26310 - //dispatch["QLinearAdd"] = dispatch["QLinearMul"] = &ONNXImporter2::parseQEltwise; - //dispatch["QLinearAveragePool"] = dispatch["QLinearGlobalAveragePool"] = &ONNXImporter2::parseQAvgPool; - //dispatch["QLinearLeakyRelu"] = &ONNXImporter2::parseQLeakyRelu; - //dispatch["QLinearSigmoid"] = &ONNXImporter2::parseQSigmoid; - //dispatch["QLinearConcat"] = &ONNXImporter2::parseQConcat; - //dispatch["QGemm"] = &ONNXImporter2::parseQGemm; - //dispatch["QLinearSoftmax"] = &ONNXImporter2::parseQSoftmax; + dispatch["QLinearAdd"] = dispatch["QLinearMul"] = &ONNXImporter2::parseQEltwise; + dispatch["QLinearAveragePool"] = dispatch["QLinearGlobalAveragePool"] = &ONNXImporter2::parseQAvgPool; + dispatch["QLinearLeakyRelu"] = &ONNXImporter2::parseQLeakyRelu; + dispatch["QLinearSigmoid"] = &ONNXImporter2::parseQSigmoid; + dispatch["QLinearConcat"] = &ONNXImporter2::parseQConcat; + dispatch["QLinearConv"] = &ONNXImporter2::parseQConv; + dispatch["QLinearMatMul"] = &ONNXImporter2::parseQMatMul; + dispatch["QGemm"] = &ONNXImporter2::parseQGemm; + dispatch["QLinearSoftmax"] = &ONNXImporter2::parseQSoftmax; dispatch["Attention"] = &ONNXImporter2::parseAttention; domain_dispatch_map[str_domain_com_microsoft] = dispatch; diff --git a/modules/dnn/test/test_onnx_conformance_layer_filter__openvino.inl.hpp b/modules/dnn/test/test_onnx_conformance_layer_filter__openvino.inl.hpp index edefdccbbe..cf69334eb4 100644 --- a/modules/dnn/test/test_onnx_conformance_layer_filter__openvino.inl.hpp +++ b/modules/dnn/test/test_onnx_conformance_layer_filter__openvino.inl.hpp @@ -1840,11 +1840,11 @@ CASE(test_prelu_broadcast_expanded) CASE(test_prelu_example_expanded) SKIP; CASE(test_qlinearconv) - // no filter + SKIP; CASE(test_qlinearmatmul_2D) - // no filter + SKIP; CASE(test_qlinearmatmul_3D) - // no filter + SKIP; CASE(test_quantizelinear) SKIP; CASE(test_quantizelinear_axis) diff --git a/modules/dnn/test/test_onnx_conformance_layer_filter_opencv_classic_denylist.inl.hpp b/modules/dnn/test/test_onnx_conformance_layer_filter_opencv_classic_denylist.inl.hpp index 5bcf9ec6f4..3aaafc7bbf 100644 --- a/modules/dnn/test/test_onnx_conformance_layer_filter_opencv_classic_denylist.inl.hpp +++ b/modules/dnn/test/test_onnx_conformance_layer_filter_opencv_classic_denylist.inl.hpp @@ -739,3 +739,6 @@ "test_eyelike_populate_off_main_diagonal", "test_eyelike_with_dtype", "test_eyelike_without_dtype", +"test_qlinearconv", +"test_qlinearmatmul_2D", +"test_qlinearmatmul_3D", diff --git a/modules/dnn/test/test_onnx_conformance_layer_parser_denylist.inl.hpp b/modules/dnn/test/test_onnx_conformance_layer_parser_denylist.inl.hpp index d7a8ed1ff2..304869a76f 100644 --- a/modules/dnn/test/test_onnx_conformance_layer_parser_denylist.inl.hpp +++ b/modules/dnn/test/test_onnx_conformance_layer_parser_denylist.inl.hpp @@ -368,14 +368,11 @@ "test_optional_has_element_tensor_input", "test_prelu_broadcast", // Issue::Parser:Blob slope not found in const blobs in function 'getBlob' (weights are required as inputs) "test_prelu_example", // ---- same as above --- -"test_qlinearconv", // Issue::Parser: Blob x_scale not found in const blobs in function 'getBlob' (weights are required as inputs) -"test_qlinearmatmul_2D", // Issue:: Parser: Variable weights is not supported in function 'parseQMatMul' -"test_qlinearmatmul_2D_int8_float16", +"test_qlinearmatmul_2D_int8_float16", // Float output QLinearMatMul variants not supported "test_qlinearmatmul_2D_int8_float32", "test_qlinearmatmul_2D_uint8_float16", "test_qlinearmatmul_2D_uint8_float32", -"test_qlinearmatmul_3D", // ---- same as above --- -"test_qlinearmatmul_3D_int8_float16", +"test_qlinearmatmul_3D_int8_float16", // Float output QLinearMatMul variants not supported "test_qlinearmatmul_3D_int8_float32", "test_qlinearmatmul_3D_uint8_float16", "test_qlinearmatmul_3D_uint8_float32", diff --git a/modules/dnn/test/test_onnx_importer.cpp b/modules/dnn/test/test_onnx_importer.cpp index 29fcc27160..48e8972172 100644 --- a/modules/dnn/test/test_onnx_importer.cpp +++ b/modules/dnn/test/test_onnx_importer.cpp @@ -2189,11 +2189,15 @@ TEST_P(Test_ONNX_layers, Gemm_External_Data) TEST_P(Test_ONNX_layers, Quantized_MatMul_Variable_Weights) { - // Unsupported - EXPECT_THROW( + auto engine_forced = static_cast( + cv::utils::getConfigurationParameterSizeT("OPENCV_FORCE_DNN_ENGINE", cv::dnn::ENGINE_AUTO)); + if (engine_forced == cv::dnn::ENGINE_CLASSIC) { - testONNXModels("quantized_matmul_variable_inputs"); - }, cv::Exception); + applyTestTag(CV_TEST_TAG_DNN_SKIP_PARSER); + return; + } + + testONNXModels("quantized_matmul_variable_inputs", npy, 1.3, 1.3); } TEST_P(Test_ONNX_layers, Quantized_Eltwise) @@ -2282,7 +2286,7 @@ TEST_P(Test_ONNX_layers, Quantized_Concat) TEST_P(Test_ONNX_layers, Quantized_Constant) { - testONNXModels("quantized_constant", npy, 0.008, 0.02); + testONNXModels("quantized_constant", npy, 0.02, 0.06); } TEST_P(Test_ONNX_layers, OutputRegistration) @@ -2294,8 +2298,8 @@ TEST_P(Test_ONNX_layers, QLinearSoftmax) { if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_NGRAPH); - testONNXModels("qlinearsoftmax_v11", npy, 0.002, 0.002); // 2D coerced - testONNXModels("qlinearsoftmax_v13", npy, 0.002, 0.002); + testONNXModels("qlinearsoftmax_v11", npy, 0.08, 0.16); // 2D coerced + testONNXModels("qlinearsoftmax_v13", npy, 0.08, 0.16); } TEST_P(Test_ONNX_layers, PriorBox_ONNX)