diff --git a/modules/dnn/include/opencv2/dnn/all_layers.hpp b/modules/dnn/include/opencv2/dnn/all_layers.hpp index e34b839f55..a01ffc2670 100644 --- a/modules/dnn/include/opencv2/dnn/all_layers.hpp +++ b/modules/dnn/include/opencv2/dnn/all_layers.hpp @@ -1153,6 +1153,10 @@ CV__DNN_INLINE_NS_BEGIN class CV_EXPORTS ActivationLayerInt8 : public ActivationLayer { public: + int input_zp, output_zp; + float input_sc, output_sc; + Mat activationLUT; + static Ptr create(const LayerParams ¶ms); }; @@ -1200,6 +1204,13 @@ CV__DNN_INLINE_NS_BEGIN class CV_EXPORTS EltwiseLayerInt8 : public Layer { public: + std::vector coeffs; + std::vector zeropoints; + std::vector scales; + float output_sc; + int output_zp; + float offset; + static Ptr create(const LayerParams ¶ms); }; diff --git a/modules/dnn/src/graph_fusion_qdq.cpp b/modules/dnn/src/graph_fusion_qdq.cpp new file mode 100644 index 0000000000..2d0f62f7ed --- /dev/null +++ b/modules/dnn/src/graph_fusion_qdq.cpp @@ -0,0 +1,816 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// Copyright (C) 2026, BigVision LLC, all rights reserved. +// Third party copyrights are property of their respective owners. + +#include "precomp.hpp" +#include "net_impl.hpp" + +namespace cv { namespace dnn { +CV__DNN_INLINE_NS_BEGIN + +using std::vector; +using std::string; + +typedef std::pair int_pair; +typedef std::pair int_arg_pair; + +struct ModelFusionQDQ +{ + ModelFusionQDQ(Net::Impl* netimpl_) : netimpl(netimpl_) {} + + void fuse() + { + int i, niter = 10; + netimpl->useCounts(usecounts); + for (i = 0; i < niter; i++) { + bool fused_any = fuseGraph(netimpl->mainGraph); + if (!fused_any) + break; + } + } + + template _LayerType* + getLayer(std::vector >& newprog, int op_idx) const + { + return op_idx >= 0 ? dynamic_cast<_LayerType*>(newprog.at(op_idx).get()) : 0; + } + + LayerParams makeLayerParamsFromOriginal(const Layer* layer, const String& newType) const + { + LayerParams params; + int lid = netimpl->getLayerId(layer->name); + if (lid >= 0) + params = netimpl->getLayerData(lid).params; + params.name = layer->name; + params.type = newType; + return params; + } + + Ptr createFusedLayer(const LayerParams& src) const + { + LayerParams params = src; + return LayerFactory::createLayerInstance(params.type, params); + } + + template + bool getQdqPatternContext(Layer* layer_ptr, + size_t ninputs, + const std::vector& inputs, + const std::vector& producer_of, + std::vector >& newprog, + Arg& q_data_in, + Arg& out_scale, + Arg& out_zp, + int& mid_layer_idx, + LayerT*& mid_layer) const + { + QuantizeLinearLayer* ql = dynamic_cast(layer_ptr); + if (!(ql && ninputs == 3 && usecounts.at(inputs[0].idx) == 1)) + return false; + q_data_in = inputs[0]; + out_scale = inputs[1]; + out_zp = inputs[2]; + mid_layer_idx = producer_of.at(q_data_in.idx); + mid_layer = getLayer(newprog, mid_layer_idx); + return mid_layer != 0; + } + + bool fuseGraph(Ptr& graph) + { + vector removed_args; + bool modified = false; + const std::vector >& prog = graph->prog(); + size_t i, nargs = netimpl->args.size(), nops = prog.size(); + std::vector producer_of(nargs, -1); + std::vector > newprog; + std::vector fused_inputs; + + for (i = 0; i < nops; i++) { + const Ptr& layer = prog[i]; + Layer* layer_ptr = (Layer*)layer.get(); + int fused_layer_idx = -1; + std::vector >* subgraphs = layer->subgraphs(); + if (subgraphs) { + for (Ptr& g: *subgraphs) { + if (fuseGraph(g)) + modified = true; + } + } + const std::vector& inputs = layer->inputs; + const std::vector& outputs = layer->outputs; + size_t ninputs = inputs.size(); + removed_args.clear(); + fused_inputs.clear(); // leave it empty to re-use original fused node inputs as-is. + + for(;;) { + Arg q_data_in, out_scale, out_zp; + int add_layer_idx = -1; + NaryEltwiseLayer* add = 0; + if (getQdqPatternContext(layer_ptr, ninputs, inputs, producer_of, + newprog, q_data_in, out_scale, out_zp, + add_layer_idx, add) && + add && add->inputs.size() >= 2) { + vector dq_ptrs; + vector dq_prog_indices; + vector int8_inputs; + + for (size_t k = 0; k < add->inputs.size(); k++) { + const Arg& add_inp = add->inputs[k]; + int dq_idx = producer_of.at(add_inp.idx); + DequantizeLinearLayer* dq = + getLayer(newprog, dq_idx); + + if (!dq || dq->inputs.size() < 3 || + usecounts.at(add_inp.idx) != 1) { + break; + } + dq_ptrs.push_back(dq); + dq_prog_indices.push_back(dq_idx); + int8_inputs.push_back(dq->inputs[0]); // int8 quantized tensor + } + + const int eltwise_out_type = !outputs.empty() ? netimpl->argData(outputs[0]).type : -1; + const bool eltwise_out_int8 = (eltwise_out_type == CV_8S || eltwise_out_type == CV_8U); + if (dq_ptrs.size() == add->inputs.size() && eltwise_out_int8) { + // Read per-input scale and zero-point from the DQ const args. + vector in_scales(2); + vector in_zps(2); + bool eltwise_in_int8 = true; + for (int k = 0; k < 2; k++) { + int inp_type = netimpl->argData(dq_ptrs[k]->inputs[0]).type; + eltwise_in_int8 = eltwise_in_int8 && (inp_type == CV_8S || inp_type == CV_8U); + in_scales[k] = netimpl->argTensor(dq_ptrs[k]->inputs[1]).at(0); + const Mat& zp_m = netimpl->argTensor(dq_ptrs[k]->inputs[2]); + in_zps[k] = zp_m.depth() == CV_8U + ? (int)zp_m.at(0) + : (int)zp_m.at(0); + } + if (!eltwise_in_int8) + break; + + float out_scale_val = netimpl->argTensor(out_scale).at(0); + const Mat& elt_out_zp_m = netimpl->argTensor(out_zp); + int out_zp_val = elt_out_zp_m.depth() == CV_8U + ? (int)elt_out_zp_m.at(0) + : (int)elt_out_zp_m.at(0); + + LayerParams eltwiseParams = makeLayerParamsFromOriginal(add, "EltwiseInt8"); + eltwiseParams.blobs.clear(); + Ptr eltwiseInt8 = createFusedLayer(eltwiseParams); + if (!eltwiseInt8.empty()) { + auto* elt = dynamic_cast(eltwiseInt8.get()); + CV_Assert(elt); + elt->scales = in_scales; + elt->zeropoints = in_zps; + elt->output_sc = out_scale_val; + elt->output_zp = out_zp_val; + fused_layer_idx = add_layer_idx; + newprog[add_layer_idx] = eltwiseInt8; + fused_inputs.swap(int8_inputs); + removed_args.push_back(q_data_in); // float add_out + for (const Arg& add_inp : add->inputs) + removed_args.push_back(add_inp); + + for (int dq_prog_idx : dq_prog_indices) + newprog[dq_prog_idx] = Ptr(); + + break; + } + } + } + + int relu_layer_idx = -1; + ReLULayer* relu = 0; + if (getQdqPatternContext(layer_ptr, ninputs, inputs, producer_of, + newprog, q_data_in, out_scale, out_zp, + relu_layer_idx, relu) && + relu->inputs.size() == 1) { + Arg relu_in = relu->inputs[0]; + int dq_idx = producer_of.at(relu_in.idx); + DequantizeLinearLayer* dq = getLayer(newprog, dq_idx); + + const int relu_out_type = !outputs.empty() ? netimpl->argData(outputs[0]).type : -1; + const bool relu_out_int8 = (relu_out_type == CV_8S || relu_out_type == CV_8U); + const int relu_in_type = (dq && !dq->inputs.empty()) ? netimpl->argData(dq->inputs[0]).type : -1; + const bool relu_in_int8 = (relu_in_type == CV_8S || relu_in_type == CV_8U); + if (dq && dq->inputs.size() >= 3 && + relu_in_int8 && relu_out_int8 && + usecounts.at(relu_in.idx) == 1) { + const float inp_sc = netimpl->argTensor(dq->inputs[1]).at(0); + const Mat& relu_zp_m = netimpl->argTensor(dq->inputs[2]); + const int inp_zp = relu_zp_m.depth() == CV_8U + ? (int)relu_zp_m.at(0) + : (int)relu_zp_m.at(0); + const float out_sc = netimpl->argTensor(out_scale).at(0); + const Mat& out_zp_relu_m = netimpl->argTensor(out_zp); + const int out_zp_i = out_zp_relu_m.depth() == CV_8U + ? (int)out_zp_relu_m.at(0) + : (int)out_zp_relu_m.at(0); + + if (inp_sc > 0.f && out_sc > 0.f) { + const bool isU8 = (relu_in_type == CV_8U); + Mat lookUpTable(1, 256, isU8 ? CV_8U : CV_8S); + if (isU8) { + uint8_t* table = lookUpTable.ptr(); + for (int t = 0; t < 256; t++) { + float x = inp_sc * (t - inp_zp); + float y = std::max(0.0f, x); + int quantized = out_zp_i + cvRound(y / out_sc); + table[t] = saturate_cast(quantized); + } + } else { + int8_t* table = lookUpTable.ptr(); + for (int t = -128; t < 128; t++) { + float x = inp_sc * (t - inp_zp); + float y = std::max(0.0f, x); + int quantized = out_zp_i + cvRound(y / out_sc); + table[t + 128] = saturate_cast(quantized); + } + } + + LayerParams reluInt8Params = makeLayerParamsFromOriginal(relu, "ReLUInt8"); + reluInt8Params.blobs.clear(); + Ptr reluInt8 = createFusedLayer(reluInt8Params); + if (!reluInt8.empty()) { + auto* reluInt8Layer = dynamic_cast(reluInt8.get()); + CV_Assert(reluInt8Layer); + reluInt8Layer->input_sc = inp_sc; + reluInt8Layer->input_zp = inp_zp; + reluInt8Layer->output_sc = out_sc; + reluInt8Layer->output_zp = out_zp_i; + reluInt8Layer->activationLUT = lookUpTable; + fused_layer_idx = relu_layer_idx; + newprog[relu_layer_idx] = reluInt8; + fused_inputs.assign(1, dq->inputs[0]); + removed_args.push_back(q_data_in); + removed_args.push_back(relu_in); + newprog[dq_idx] = Ptr(); + break; + } + } + } + } + + // Compound pattern: DQ, DQ -> Add -> ReLU -> QuantizeLinear + // Common in ResNet residual blocks. Fuses into EltwiseInt8 with activation LUT. + { + int relu_layer_idx2 = -1; + ReLULayer* relu2 = 0; + Arg q_data_in2, out_scale2, out_zp2; + if (getQdqPatternContext(layer_ptr, ninputs, inputs, producer_of, + newprog, q_data_in2, out_scale2, out_zp2, + relu_layer_idx2, relu2) && + relu2->inputs.size() == 1) { + Arg relu_in2 = relu2->inputs[0]; + int add_idx2 = producer_of.at(relu_in2.idx); + NaryEltwiseLayer* add2 = getLayer(newprog, add_idx2); + if (add2 && add2->inputs.size() >= 2 && + usecounts.at(relu_in2.idx) == 1) { + vector dq_ptrs2; + vector dq_prog_indices2; + vector int8_inputs2; + for (size_t k = 0; k < add2->inputs.size(); k++) { + const Arg& add_inp = add2->inputs[k]; + int dq_idx2 = producer_of.at(add_inp.idx); + DequantizeLinearLayer* dq2 = + getLayer(newprog, dq_idx2); + if (dq2 && dq2->inputs.size() >= 3 && + usecounts.at(add_inp.idx) == 1) { + dq_ptrs2.push_back(dq2); + dq_prog_indices2.push_back(dq_idx2); + int8_inputs2.push_back(dq2->inputs[0]); + } else { + int arg_type = netimpl->argData(add_inp).type; + if (arg_type == CV_8S || arg_type == CV_8U) { + dq_ptrs2.push_back(nullptr); + dq_prog_indices2.push_back(-1); + int8_inputs2.push_back(add_inp); + } else { + break; // not int8, can't fuse + } + } + } + const int elt_out_type2 = !outputs.empty() ? netimpl->argData(outputs[0]).type : -1; + const bool elt_out_int82 = (elt_out_type2 == CV_8S || elt_out_type2 == CV_8U); + if (int8_inputs2.size() == add2->inputs.size() && elt_out_int82) { + vector in_scales2(2); + vector in_zps2(2); + bool elt_in_int82 = true; + for (int k = 0; k < 2; k++) { + if (dq_ptrs2[k]) { + int it = netimpl->argData(dq_ptrs2[k]->inputs[0]).type; + elt_in_int82 = elt_in_int82 && (it == CV_8S || it == CV_8U); + in_scales2[k] = netimpl->argTensor(dq_ptrs2[k]->inputs[1]).at(0); + const Mat& zp_m2 = netimpl->argTensor(dq_ptrs2[k]->inputs[2]); + in_zps2[k] = zp_m2.depth() == CV_8U + ? (int)zp_m2.at(0) + : (int)zp_m2.at(0); + } else { + int prod_idx = producer_of.at(add2->inputs[k].idx); + Layer* prod = prod_idx >= 0 && !newprog[prod_idx].empty() + ? newprog[prod_idx].get() : nullptr; + ConvolutionLayerInt8* ci = prod ? dynamic_cast(prod) : nullptr; + EltwiseLayerInt8* ei = prod ? dynamic_cast(prod) : nullptr; + InnerProductLayerInt8* fi = prod ? dynamic_cast(prod) : nullptr; + if (ci) { in_scales2[k] = ci->output_sc; in_zps2[k] = ci->output_zp; } + else if (ei) { in_scales2[k] = ei->output_sc; in_zps2[k] = ei->output_zp; } + else if (fi) { in_scales2[k] = fi->output_sc; in_zps2[k] = fi->output_zp; } + else { elt_in_int82 = false; } + } + } + if (elt_in_int82) { + float out_sc2 = netimpl->argTensor(out_scale2).at(0); + const Mat& out_zp_m2 = netimpl->argTensor(out_zp2); + int out_zp_val2 = out_zp_m2.depth() == CV_8U + ? (int)out_zp_m2.at(0) + : (int)out_zp_m2.at(0); + if (out_sc2 > 0.f) { + LayerParams eltParams = makeLayerParamsFromOriginal(add2, "EltwiseInt8"); + eltParams.blobs.clear(); + Ptr eltInt8 = createFusedLayer(eltParams); + if (!eltInt8.empty()) { + auto* elt = dynamic_cast(eltInt8.get()); + CV_Assert(elt); + elt->scales = in_scales2; + elt->zeropoints = in_zps2; + elt->output_sc = out_sc2; + elt->output_zp = out_zp_val2; + + const bool isU8 = (elt_out_type2 == CV_8U); + Mat lut(1, 256, isU8 ? CV_8U : CV_8S); + if (isU8) { + uint8_t* tbl = lut.ptr(); + for (int t = 0; t < 256; t++) { + float x = out_sc2 * (t - out_zp_val2); + float y = std::max(0.0f, x); + tbl[t] = saturate_cast(out_zp_val2 + cvRound(y / out_sc2)); + } + } else { + int8_t* tbl = lut.ptr(); + for (int t = -128; t < 128; t++) { + float x = out_sc2 * (t - out_zp_val2); + float y = std::max(0.0f, x); + tbl[t + 128] = saturate_cast(out_zp_val2 + cvRound(y / out_sc2)); + } + } + LayerParams reluActParams; + reluActParams.name = relu2->name; + reluActParams.type = "ReLUInt8"; + Ptr reluAct = createFusedLayer(reluActParams); + if (!reluAct.empty()) { + auto* reluActLayer = dynamic_cast(reluAct.get()); + if (reluActLayer) { + reluActLayer->input_sc = out_sc2; + reluActLayer->input_zp = out_zp_val2; + reluActLayer->output_sc = out_sc2; + reluActLayer->output_zp = out_zp_val2; + reluActLayer->activationLUT = lut; + eltInt8->setActivation(reluAct.dynamicCast()); + } + } + + fused_layer_idx = add_idx2; + newprog[add_idx2] = eltInt8; + newprog[relu_layer_idx2] = Ptr(); + fused_inputs.swap(int8_inputs2); + removed_args.push_back(q_data_in2); + removed_args.push_back(relu_in2); + for (const Arg& add_inp : add2->inputs) + removed_args.push_back(add_inp); + for (int dq_prog_idx : dq_prog_indices2) { + if (dq_prog_idx >= 0) + newprog[dq_prog_idx] = Ptr(); + } + break; + } + } + } + } + } + } + } + + Arg out_scale_arg, out_zp_arg; + int conv_layer_idx = -1; + Conv2Layer* conv = 0; + if (getQdqPatternContext(layer_ptr, ninputs, inputs, producer_of, + newprog, q_data_in, out_scale_arg, out_zp_arg, + conv_layer_idx, conv) && + (conv->inputs.size() == 2 || conv->inputs.size() == 3)) { + const Arg conv_x = conv->inputs[0]; + const Arg conv_w = conv->inputs[1]; + const int dq_x_idx = producer_of.at(conv_x.idx); + const int dq_w_idx = producer_of.at(conv_w.idx); + DequantizeLinearLayer* dq_x = getLayer(newprog, dq_x_idx); + DequantizeLinearLayer* dq_w = getLayer(newprog, dq_w_idx); + + // Allow usecounts > 1 for conv input (shared DQ output at stage transitions) + // The int8 data (DQ's input[0]) can be shared safely. + if (dq_x && dq_w && + dq_x->inputs.size() >= 3 && dq_w->inputs.size() >= 3 && + usecounts.at(conv_w.idx) == 1) { + float inp_sc = netimpl->argTensor(dq_x->inputs[1]).at(0); + float out_sc = netimpl->argTensor(out_scale_arg).at(0); + int inp_zp = 0, out_zp = 0; + const Mat& x_zp_m_read = netimpl->argTensor(dq_x->inputs[2]); + if (x_zp_m_read.depth() == CV_8S) + inp_zp = (int)x_zp_m_read.at(0); + else + inp_zp = (int)x_zp_m_read.at(0); + const Mat& out_zp_m = netimpl->argTensor(out_zp_arg); + if (out_zp_m.depth() == CV_8S) + out_zp = (int)out_zp_m.at(0); + else if (out_zp_m.depth() == CV_8U) + out_zp = (int)out_zp_m.at(0); + else + out_zp = out_zp_m.at(0); + if (!(inp_sc > 0.f && out_sc > 0.f)) + break; + Mat w_q = netimpl->argTensor(dq_w->inputs[0]); + Mat w_sc_m = netimpl->argTensor(dq_w->inputs[1]); + Mat w_zp_m = netimpl->argTensor(dq_w->inputs[2]); + const Mat& x_zp_m = netimpl->argTensor(dq_x->inputs[2]); + if (!w_q.empty() && w_q.depth() == CV_8S && w_q.dims >= 3) { + const int outCn = w_q.size[0]; + Mat wt_sc = (w_sc_m.total() == (size_t)outCn) + ? w_sc_m.reshape(1, 1) + : Mat(1, outCn, CV_32F, Scalar(w_sc_m.at(0))).clone(); + bool per_channel = w_sc_m.total() == (size_t)outCn; + + bool all_wzp_zero = true; + if (w_zp_m.total() > 1 && w_zp_m.total() != (size_t)outCn) + all_wzp_zero = false; + for (size_t t = 0; all_wzp_zero && t < w_zp_m.total(); t++) { + int wz = w_zp_m.depth() == CV_8S ? (int)w_zp_m.at((int)t) + : (int)w_zp_m.at((int)t); + if (wz != 0) all_wzp_zero = false; + } + + bool symmetric_pads = true; + size_t npads = conv->pads.size(); + size_t ndims_pad = npads / 2; + for (size_t d = 0; d < ndims_pad && symmetric_pads; d++) { + if (conv->pads[d] != conv->pads[d + ndims_pad]) + symmetric_pads = false; + } + + if (all_wzp_zero && symmetric_pads) { + Mat bias = Mat::zeros(1, outCn, CV_32S); + bool biasOk = true; + int dq_bias_idx = -1; + if (conv->inputs.size() == 3) { + if (netimpl->isConstArg(conv->inputs[2])) { + Mat b = netimpl->argTensor(conv->inputs[2]); + if (b.empty() || b.total() != (size_t)outCn) { + biasOk = false; + } else if (b.depth() == CV_32S) { + bias = b.reshape(1, 1); + } else if (b.depth() == CV_32F || b.depth() == CV_64F) { + Mat b1 = b.reshape(1, 1); + for (int oc = 0; oc < outCn; oc++) { + const float b_real = b1.depth() == CV_32F + ? b1.at(oc) + : (float)b1.at(oc); + const float denom = inp_sc * wt_sc.at(oc); + if (std::abs(denom) < 1e-12f) { biasOk = false; break; } + bias.at(oc) = cvRound(b_real / denom); + } + } else { + biasOk = false; + } + } else { + dq_bias_idx = producer_of.at(conv->inputs[2].idx); + DequantizeLinearLayer* dq_b = + getLayer(newprog, dq_bias_idx); + if (!dq_b || dq_b->inputs.size() < 2 || + usecounts.at(conv->inputs[2].idx) != 1 || + !netimpl->isConstArg(dq_b->inputs[0])) { + biasOk = false; + } else { + Mat bq = netimpl->argTensor(dq_b->inputs[0]); + if (bq.empty() || bq.total() != (size_t)outCn || bq.depth() != CV_32S) + biasOk = false; + else + bias = bq.reshape(1, 1); + } + } + } + if (!biasOk) + break; + + const bool inputIsU8 = + (netimpl->argData(dq_x->inputs[0]).type == CV_8U) || + (x_zp_m.depth() == CV_8U); + const int inp_zp_kernel = inputIsU8 ? (inp_zp - 128) : inp_zp; + Mat weights_2d = w_q.reshape(1, outCn); + Mat biasFused(1, outCn, CV_32S); + Mat outputMultiplier(1, outCn, CV_32F); + for (int oc = 0; oc < outCn; oc++) { + biasFused.at(oc) = bias.at(oc) - inp_zp_kernel * (int)cv::sum(weights_2d.row(oc))[0]; + outputMultiplier.at(oc) = (inp_sc * wt_sc.at(oc)) / out_sc; + } + + LayerParams convInt8Params = makeLayerParamsFromOriginal(conv, "ConvolutionInt8"); + { + int kndims = w_q.dims - 2; + std::vector ksize(kndims); + for (int d = 0; d < kndims; d++) + ksize[d] = w_q.size[d + 2]; + convInt8Params.set("kernel_size", DictValue::arrayInt(ksize.data(), kndims)); + if (!conv->strides.empty()) + convInt8Params.set("stride", DictValue::arrayInt(conv->strides.data(), (int)conv->strides.size())); + if (!conv->dilations.empty()) + convInt8Params.set("dilation", DictValue::arrayInt(conv->dilations.data(), (int)conv->dilations.size())); + if (!conv->pads.empty()) + convInt8Params.set("pad", DictValue::arrayInt(conv->pads.data(), (int)conv->pads.size())); + } + convInt8Params.set("num_output", outCn); + convInt8Params.set("group", conv->ngroups); + convInt8Params.blobs.resize(3); + convInt8Params.blobs[0] = w_q; + convInt8Params.blobs[1] = biasFused; + convInt8Params.blobs[2] = outputMultiplier; + Ptr convInt8 = createFusedLayer(convInt8Params); + if (!convInt8.empty()) { + auto* convInt8Layer = dynamic_cast(convInt8.get()); + CV_Assert(convInt8Layer); + convInt8Layer->input_zp = inp_zp; + convInt8Layer->input_sc = inp_sc; + convInt8Layer->output_zp = out_zp; + convInt8Layer->output_sc = out_sc; + convInt8Layer->per_channel = per_channel; + fused_layer_idx = conv_layer_idx; + newprog[conv_layer_idx] = convInt8; + fused_inputs.assign(1, dq_x->inputs[0]); + removed_args.push_back(q_data_in); + removed_args.push_back(conv_w); + if (conv->inputs.size() == 3) { + removed_args.push_back(conv->inputs[2]); + if (dq_bias_idx >= 0) + newprog[dq_bias_idx] = Ptr(); + } + if (usecounts.at(conv_x.idx) == 1) { + removed_args.push_back(conv_x); + newprog[dq_x_idx] = Ptr(); + } + newprog[dq_w_idx] = Ptr(); + break; + } + } + } + } + } + + int mm_layer_idx = -1; + MatMulLayer* mm = 0; + if (getQdqPatternContext(layer_ptr, ninputs, inputs, producer_of, + newprog, q_data_in, out_scale, out_zp, + mm_layer_idx, mm) && + mm->inputs.size() == 2) { + const Arg mm_x = mm->inputs[0]; + const Arg mm_w = mm->inputs[1]; + int dq_x_idx = producer_of.at(mm_x.idx); + int dq_w_idx = producer_of.at(mm_w.idx); + DequantizeLinearLayer* dq_x = getLayer(newprog, dq_x_idx); + DequantizeLinearLayer* dq_w = getLayer(newprog, dq_w_idx); + float inp_sc = 0.f, out_sc = 0.f; + int inp_zp = 0, out_zp_i = 0; + const int fc_out_type = !outputs.empty() ? netimpl->argData(outputs[0]).type : -1; + const bool fc_out_int8 = (fc_out_type == CV_8S || fc_out_type == CV_8U); + const int fc_in_type = (dq_x && !dq_x->inputs.empty()) ? netimpl->argData(dq_x->inputs[0]).type : -1; + const bool fc_in_int8 = (fc_in_type == CV_8S || fc_in_type == CV_8U); + if (dq_x && dq_w && + dq_x->inputs.size() >= 3 && dq_w->inputs.size() >= 3 && + fc_in_int8 && fc_out_int8 && + usecounts.at(mm_x.idx) == 1 && usecounts.at(mm_w.idx) == 1) { + inp_sc = netimpl->argTensor(dq_x->inputs[1]).at(0); + const Mat& fc_zp_m = netimpl->argTensor(dq_x->inputs[2]); + inp_zp = fc_zp_m.depth() == CV_8U + ? (int)fc_zp_m.at(0) + : (int)fc_zp_m.at(0); + out_sc = netimpl->argTensor(out_scale).at(0); + const Mat& fc_out_zp_m = netimpl->argTensor(out_zp); + out_zp_i = fc_out_zp_m.depth() == CV_8U + ? (int)fc_out_zp_m.at(0) + : (int)fc_out_zp_m.at(0); + if (!(inp_sc > 0.f && out_sc > 0.f)) + break; + Mat w_q = netimpl->argTensor(dq_w->inputs[0]); + Mat w_sc_m = netimpl->argTensor(dq_w->inputs[1]); + Mat w_zp_m = netimpl->argTensor(dq_w->inputs[2]); + if (!w_q.empty() && w_q.depth() == CV_8S && w_q.dims == 2) { + bool all_wzp_zero = true; + for (size_t t = 0; all_wzp_zero && t < w_zp_m.total(); t++) { + int wz = w_zp_m.depth() == CV_8S ? (int)w_zp_m.at((int)t) + : (int)w_zp_m.at((int)t); + if (wz != 0) all_wzp_zero = false; + } + if (all_wzp_zero) { + Mat weights = w_q.t(); + int outCn = weights.size[0]; + Mat wt_sc = (w_sc_m.total() == (size_t)outCn) + ? w_sc_m.reshape(1, 1) + : Mat(1, outCn, CV_32F, Scalar(w_sc_m.at(0))).clone(); + bool per_channel = w_sc_m.total() == (size_t)outCn; + Mat bias(1, outCn, CV_32S); + Mat outputMultiplier(1, outCn, CV_32F); + for (int ioc = 0; ioc < outCn; ioc++) { + bias.at(ioc) = -inp_zp * (int)cv::sum(weights.row(ioc))[0]; + outputMultiplier.at(ioc) = (inp_sc * wt_sc.at(ioc)) / out_sc; + } + int firstInpDims = (int)netimpl->argData(mm_x).shape.size(); + int axis = std::max(1, firstInpDims - w_q.dims + 1); + + LayerParams fcInt8Params = makeLayerParamsFromOriginal(mm, "InnerProductInt8"); + fcInt8Params.set("num_output", outCn); + fcInt8Params.set("axis", axis); + fcInt8Params.blobs.resize(3); + fcInt8Params.blobs[0] = weights; + fcInt8Params.blobs[1] = bias; + fcInt8Params.blobs[2] = outputMultiplier; + Ptr fcInt8 = createFusedLayer(fcInt8Params); + if (!fcInt8.empty()) { + auto* fcInt8Layer = dynamic_cast(fcInt8.get()); + CV_Assert(fcInt8Layer); + fcInt8Layer->input_zp = inp_zp; + fcInt8Layer->input_sc = inp_sc; + fcInt8Layer->output_zp = out_zp_i; + fcInt8Layer->output_sc = out_sc; + fcInt8Layer->per_channel = per_channel; + fused_layer_idx = mm_layer_idx; + newprog[mm_layer_idx] = fcInt8; + fused_inputs.assign(1, dq_x->inputs[0]); + removed_args.push_back(q_data_in); + removed_args.push_back(mm_x); + removed_args.push_back(mm_w); + newprog[dq_x_idx] = Ptr(); + newprog[dq_w_idx] = Ptr(); + break; + } + } + } + } + } + + int pool_layer_idx = -1; + PoolingLayer* pool = 0; + if (getQdqPatternContext(layer_ptr, ninputs, inputs, producer_of, + newprog, q_data_in, out_scale, out_zp, + pool_layer_idx, pool) && + pool->inputs.size() == 1) { + Arg pool_in = pool->inputs[0]; + int dq_idx = producer_of.at(pool_in.idx); + DequantizeLinearLayer* dq = getLayer(newprog, dq_idx); + float inp_sc = 0.f, out_sc = 0.f; + int inp_zp = 0, out_zp_i = 0; + const int pool_out_type = !outputs.empty() ? netimpl->argData(outputs[0]).type : -1; + const bool pool_out_int8 = (pool_out_type == CV_8S || pool_out_type == CV_8U); + const int pool_in_type = (dq && !dq->inputs.empty()) ? netimpl->argData(dq->inputs[0]).type : -1; + const bool pool_in_int8 = (pool_in_type == CV_8S || pool_in_type == CV_8U); + if (dq && dq->inputs.size() >= 3 && + pool_in_int8 && pool_out_int8 && + usecounts.at(pool_in.idx) == 1) { + inp_sc = netimpl->argTensor(dq->inputs[1]).at(0); + const Mat& pool_zp_m = netimpl->argTensor(dq->inputs[2]); + inp_zp = pool_zp_m.depth() == CV_8U + ? (int)pool_zp_m.at(0) + : (int)pool_zp_m.at(0); + out_sc = netimpl->argTensor(out_scale).at(0); + const Mat& pool_out_zp_m = netimpl->argTensor(out_zp); + out_zp_i = pool_out_zp_m.depth() == CV_8U + ? (int)pool_out_zp_m.at(0) + : (int)pool_out_zp_m.at(0); + bool isGlobalAve = pool->globalPooling; + bool isMax = !isGlobalAve; + if ((isGlobalAve && inp_sc > 0.f && out_sc > 0.f) || + (isMax && std::abs(inp_sc - out_sc) < 1e-6f && inp_zp == out_zp_i)) { + LayerParams poolInt8Params = makeLayerParamsFromOriginal(pool, "PoolingInt8"); + poolInt8Params.blobs.clear(); + Ptr poolInt8 = createFusedLayer(poolInt8Params); + if (!poolInt8.empty()) { + auto* poolInt8Layer = dynamic_cast(poolInt8.get()); + CV_Assert(poolInt8Layer); + const String poolInt8Type = static_cast(*poolInt8Layer).type; + std::vector poolInt8Blobs = poolInt8Layer->blobs; + static_cast(*poolInt8Layer) = *pool; + static_cast(*poolInt8Layer).type = poolInt8Type; + poolInt8Layer->blobs = poolInt8Blobs; + poolInt8Layer->input_sc = inp_sc; + poolInt8Layer->input_zp = inp_zp; + poolInt8Layer->output_sc = out_sc; + poolInt8Layer->output_zp = out_zp_i; + fused_layer_idx = pool_layer_idx; + newprog[pool_layer_idx] = poolInt8; + fused_inputs.assign(1, dq->inputs[0]); + removed_args.push_back(q_data_in); + removed_args.push_back(pool_in); + newprog[dq_idx] = Ptr(); + break; + } + } + } + } + + { + ActivationLayerInt8* activ_int8 = dynamic_cast(layer_ptr); + if (activ_int8 && ninputs == 1 && + usecounts.at(inputs[0].idx) == 1) { + Arg activ_inp = inputs[0]; + int producer_idx = producer_of.at(activ_inp.idx); + if (producer_idx >= 0 && !newprog[producer_idx].empty()) { + Layer* producer_layer = newprog[producer_idx].get(); + Ptr activ_layer = layer.dynamicCast(); + + ConvolutionLayerInt8* conv_int8 = + dynamic_cast(producer_layer); + if (conv_int8 && conv_int8->output_sc == activ_int8->input_sc && + conv_int8->output_zp == activ_int8->input_zp) { + if (newprog[producer_idx]->setActivation(activ_layer)) { + conv_int8->output_sc = activ_int8->output_sc; + conv_int8->output_zp = activ_int8->output_zp; + fused_layer_idx = producer_idx; + removed_args.push_back(activ_inp); + break; + } + } + + InnerProductLayerInt8* fc_int8 = + dynamic_cast(producer_layer); + if (fc_int8 && fc_int8->output_sc == activ_int8->input_sc && + fc_int8->output_zp == activ_int8->input_zp) { + if (newprog[producer_idx]->setActivation(activ_layer)) { + fc_int8->output_sc = activ_int8->output_sc; + fc_int8->output_zp = activ_int8->output_zp; + fused_layer_idx = producer_idx; + removed_args.push_back(activ_inp); + break; + } + } + + EltwiseLayerInt8* elt_int8 = + dynamic_cast(producer_layer); + if (elt_int8 && elt_int8->output_sc == activ_int8->input_sc && + elt_int8->output_zp == activ_int8->input_zp) { + if (newprog[producer_idx]->setActivation(activ_layer)) { + elt_int8->output_sc = activ_int8->output_sc; + elt_int8->output_zp = activ_int8->output_zp; + fused_layer_idx = producer_idx; + removed_args.push_back(activ_inp); + break; + } + } + } + } + } + break; + } + + if (fused_layer_idx >= 0) { + modified = true; + Layer* fused_layer = newprog[fused_layer_idx]; + fused_layer->outputs = outputs; + if (!fused_inputs.empty()) + fused_layer->inputs = fused_inputs; + for (Arg new_out: outputs) + producer_of[new_out.idx] = fused_layer_idx; + for (Arg old_out: removed_args) { + usecounts.at(old_out.idx) = 0; + producer_of.at(old_out.idx) = -1; + } + } else { + for (auto out: outputs) + producer_of[out.idx] = (int)newprog.size(); + newprog.push_back(layer); + } + } + + if (modified) { + size_t i, j = 0, newops = newprog.size(); + for (i = 0; i < newops; i++) { + if (!newprog[i].empty()) { + if (j < i) + newprog[j] = newprog[i]; + j++; + } + } + newprog.resize(j); + graph->setProg(newprog); + } + + return modified; + } + + Net::Impl* netimpl; + vector usecounts; +}; + +void Net::Impl::fuseQDQ() +{ + ModelFusionQDQ qdqFusion(this); + qdqFusion.fuse(); +} + +CV__DNN_INLINE_NS_END +}} // cv::dnn diff --git a/modules/dnn/src/int8layers/convolution_layer.cpp b/modules/dnn/src/int8layers/convolution_layer.cpp index 4864b06067..1bcebef9ee 100644 --- a/modules/dnn/src/int8layers/convolution_layer.cpp +++ b/modules/dnn/src/int8layers/convolution_layer.cpp @@ -14,6 +14,7 @@ #include #include + namespace cv { namespace dnn @@ -42,16 +43,32 @@ public: BaseConvolutionLayerInt8Impl(const LayerParams ¶ms) { setParamsFrom(params); - getConvolutionKernelParams(params, kernel_size, pads_begin, pads_end, strides, dilations, padMode, adjust_pads, useWinograd); + const bool hasKernelParams = + params.has("kernel") || params.has("kernel_size") || params.has("kernel_h") || params.has("kernel_w"); + if (hasKernelParams) + { + getConvolutionKernelParams(params, kernel_size, pads_begin, pads_end, strides, dilations, padMode, adjust_pads, useWinograd); + } + else + { + kernel_size.clear(); + pads_begin.clear(); + pads_end.clear(); + strides.clear(); + dilations.clear(); + adjust_pads.clear(); + padMode = params.get("pad_mode", ""); + } - numOutput = params.get("num_output"); + numOutput = params.get("num_output", blobs.empty() ? 0 : blobs[0].size[0]); int ngroups = params.get("group", 1); - CV_Assert(numOutput % ngroups == 0); + if (numOutput > 0) + CV_Assert(numOutput % ngroups == 0); - input_sc = params.get("input_scale"); - input_zp = params.get("input_zeropoint"); - output_zp = params.get("zeropoints"); - output_sc = params.get("scales"); + input_sc = params.get("input_scale", 1.0f); + input_zp = params.get("input_zeropoint", 0); + output_zp = params.get("zeropoints", 0); + output_sc = params.get("scales", 1.0f); per_channel = params.get("per_channel", true); if (kernel_size.size() == 2) { @@ -100,7 +117,8 @@ public: } const Mat &input = inputs[0]; - CV_Assert(((input.dims == 3 && kernel_size.size() == 1) || input.dims == 4 || input.dims == 5) && input.type() == CV_8S); + CV_Assert(((input.dims == 3 && kernel_size.size() == 1) || input.dims == 4 || input.dims == 5) && + (input.type() == CV_8S || input.type() == CV_8U)); for (size_t i = 0; i < outputs.size(); i++) { CV_Assert(inputs[i].type() == input.type()); @@ -126,6 +144,19 @@ public: } } + void getTypes(const std::vector& inputs, + const int requiredOutputs, + const int requiredInternals, + std::vector& outputs, + std::vector& internals) const CV_OVERRIDE + { + CV_Assert(!inputs.empty()); + for (auto t : inputs) + CV_CheckType(t, t == CV_8S || t == CV_8U, ""); + outputs.assign(requiredOutputs, inputs[0]); + internals.assign(requiredInternals, CV_32S); + } + virtual MatShape computeColRowShape(const MatShape &inpShape, const MatShape &outShape) const = 0; bool is1x1() const { @@ -163,9 +194,11 @@ public: enum { VEC_ALIGN = 32, DFT_TYPE = CV_8S }; Mat weightsMat; std::vector biasvec; + std::vector biasvecVNNI; // Bias adjusted for VNNI uint8 direct path std::vector outputMultiplier; Mat activationLUT; Ptr activ; + Mat outputInt32Buf_; // pre-allocated scratch buffer for int32 accumulation ConvolutionLayerInt8Impl(const LayerParams ¶ms) : BaseConvolutionLayerInt8Impl(params){} @@ -247,13 +280,16 @@ public: std::vector inputs; inputs_arr.getMatVector(inputs); + const int outCn = numOutput > 0 ? numOutput : blobs[0].size[0]; + CV_Assert(outCn > 0); + numOutput = outCn; // prepare weightsMat where each row is aligned and has enough zero padding on the right to // use vectorized (i.e. with intrinsics) loops without tail processing - Mat wm = blobs[0].reshape(1, numOutput); + Mat wm = blobs[0].reshape(1, outCn); if( wm.step1() % VEC_ALIGN != 0 ) { int newcols = (int)alignSize(wm.step1(), VEC_ALIGN); - Mat wm_buffer = Mat(numOutput, newcols, wm.type()); + Mat wm_buffer = Mat(outCn, newcols, wm.type()); Mat wm_padding = wm_buffer.colRange(wm.cols, newcols); wm_padding.setTo(Scalar::all(0)); Mat wm_aligned = wm_buffer.colRange(0, wm.cols); @@ -263,11 +299,11 @@ public: weightsMat = wm; Mat biasMat = blobs[1]; - biasvec.resize(numOutput+2); + biasvec.resize(outCn + 2); Mat outMult = blobs[2]; - outputMultiplier.resize(numOutput+2); - for(int i = 0; i < numOutput; i++ ) + outputMultiplier.resize(outCn + 2); + for(int i = 0; i < outCn; i++ ) { biasvec[i] = biasMat.at(i); outputMultiplier[i] = outMult.at(i); @@ -687,7 +723,7 @@ public: class ParallelConv : public cv::ParallelLoopBody { public: - enum { BLK_SIZE = 32, BLK_SIZE_CN = 64 }; + enum { BLK_SIZE = 64, BLK_SIZE_CN = 64 }; const Mat* input_; const Mat* weights_; @@ -704,14 +740,16 @@ public: bool useAVX512; bool useLASX; bool useRVV; + bool useVNNI; int blk_size_cn; int inpZp, outZp; const std::vector* multiplier; + Mat* directU8Out_; ParallelConv() : input_(0), weights_(0), output_(0), ngroups_(0), nstripes_(0), - biasvec_(0), activLUT_(0), activ_(0), is1x1_(false), useAVX2(false), useAVX512(false), useLASX(false), useRVV(false) - , blk_size_cn(0), inpZp(0), outZp(0), multiplier(0) + biasvec_(0), activLUT_(0), activ_(0), is1x1_(false), useAVX2(false), useAVX512(false), useLASX(false), useRVV(false), useVNNI(false) + , blk_size_cn(0), inpZp(0), outZp(0), multiplier(0), directU8Out_(0) {} static void run( const Mat& input, Mat& output, const Mat& weights, const std::vector& multipliers, @@ -719,7 +757,8 @@ public: const std::vector& kernel_size, const std::vector& strides, const std::vector& pads_begin, const std::vector& pads_end, const std::vector& dilations, - const ActivationLayerInt8* activ, int ngroups, int nstripes, int inp_Zp, int out_Zp) + const ActivationLayerInt8* activ, int ngroups, int nstripes, int inp_Zp, int out_Zp, + Mat* directU8Out = nullptr) { size_t karea = std::accumulate(kernel_size.begin(), kernel_size.end(), 1, std::multiplies()); @@ -731,9 +770,9 @@ public: input.size[0] == output.size[0], weights.rows == output.size[1], weights.cols == (input.size[1]/ngroups)*karea, - input.type() == CV_8SC1, + (input.type() == CV_8SC1 || input.type() == CV_8UC1), output.type() == CV_32SC1, - input.type() == weights.type(), + weights.type() == CV_8SC1, input.isContinuous(), output.isContinuous(), biasvec.size() == (size_t)output.size[1]+2); @@ -765,7 +804,10 @@ public: p.useAVX2 = checkHardwareSupport(CPU_AVX2) && isConv2D; p.useAVX512 = CV_CPU_HAS_SUPPORT_AVX512_SKX && isConv2D; - + #if CV_AVXVNNI_AVAILABLE + p.useVNNI = p.useAVX2 && checkHardwareSupport(CPU_AVX_VNNI) && isConv2D && + input.type() == CV_8UC1; + #endif p.useLASX = checkHardwareSupport(CPU_LASX) && isConv2D; p.useRVV = checkHardwareSupport(CPU_RVV) && isConv2D; @@ -818,6 +860,7 @@ public: p.biasvec_ = &biasvec; p.activLUT_ = &activLUT; p.activ_ = !activLUT.empty() ? activ : 0; + p.directU8Out_ = directU8Out; parallel_for_(Range(0, nstripes), p, nstripes); } @@ -898,27 +941,18 @@ public: const float* multptr_ = &multiplier->at(0); const int* lutptr_ = !activLUT_->empty() ? activLUT_->ptr() : 0; int* data_out0_ = output_->ptr(); - AutoBuffer rowbuf0_; int8_t* rowbuf0 = 0; bool use_rowbuf = !depthWiseConvolution; int blk_size = depthWiseConvolution ? outPlaneSize : min((int)BLK_SIZE, stripeSize); // im2row buffer is not used for depth-wise convolution + // Use thread_local to avoid repeated allocation across layers if(use_rowbuf) { size_t rowbufsz = alignSize(karea*blk_size_cn, valign)*min((int)BLK_SIZE, blk_size); - //printf("karea=%d, blk_size_cn=%d, rowbufsz=%d, stripeSize=%d\n", karea, blk_size_cn, (int)rowbufsz, stripeSize); - rowbuf0_.allocate(rowbufsz + valign); - rowbuf0 = alignPtr(rowbuf0_.data(), (int)(valign*sizeof(int8_t))); - // we clear the buffer once; ultimately, it lets us to avoid - // tail processing after running the unrolled/vectorized loop. - // the main idea is to make sure that the tail (a.k.a. padding) of each row - // (i.e. the elements with indices between vsz=karea*ncn and vsz_a) - // does not contain NaNs or Infs. Because the padding in the weights - // matrix is explicitly initialized with 0's, we handle all other - // cases nicely, i.e. we can skip expliciting re-initialization - // of the padding - we just retain elements from the previous iteration - // of the loop over channels (cn0). + thread_local AutoBuffer rowbuf0_tls; + rowbuf0_tls.allocate(rowbufsz + valign); + rowbuf0 = alignPtr(rowbuf0_tls.data(), (int)(valign*sizeof(int8_t))); memset(rowbuf0, (int8_t)inpZp, rowbufsz*sizeof(rowbuf0[0]) ); } @@ -1346,8 +1380,12 @@ public: } } } - // now compute dot product of the weights - // and im2row-transformed part of the tensor + #if CV_AVXVNNI_AVAILABLE + if(useVNNI) + opt_AVX2::fastConvVNNI(wptr, wstep, biasptr, (const uint8_t*)rowbuf0, data_out0 + ofs0, + outShape, bsz, vsz, vsz_a, outZp, multptr, cn0 == 0, cn1 == inpCn); + else + #endif #if CV_TRY_AVX512_SKX if(useAVX512) opt_AVX2::fastConv(wptr, wstep, biasptr, rowbuf0, data_out0 + ofs0, @@ -1492,6 +1530,34 @@ public: activ_->forwardSlice(data_out0 + stripeStart, lutptr_, data_out0 + stripeStart, (int)(stripeEnd - stripeStart), outPlaneSize, startOutCn, startOutCn + outCn); + + // Fused int32 → uint8 conversion (cache-hot, avoids separate convertTo pass) + if( directU8Out_ ) + { + uint8_t* u8base = directU8Out_->ptr() + subsampleIdx*outPlaneSize*outCn; + for( int c = 0; c < outCn; c++ ) + { + const int* src = data_out0 + c*outPlaneSize + stripeStart; + uint8_t* dst = u8base + c*outPlaneSize + stripeStart; + int len = stripeEnd - stripeStart; + int j = 0; + #if CV_SSE2 + __m128i voffset = _mm_set1_epi32(128); + for( ; j <= len - 8; j += 8 ) + { + __m128i v0 = _mm_loadu_si128((const __m128i*)(src + j)); + __m128i v1 = _mm_loadu_si128((const __m128i*)(src + j + 4)); + v0 = _mm_add_epi32(v0, voffset); + v1 = _mm_add_epi32(v1, voffset); + __m128i p16 = _mm_packs_epi32(v0, v1); + __m128i p8 = _mm_packus_epi16(p16, p16); + _mm_storel_epi64((__m128i*)(dst + j), p8); + } + #endif + for( ; j < len; j++ ) + dst[j] = (uint8_t)std::min(std::max(src[j] + 128, 0), 255); + } + } } } }; @@ -1512,19 +1578,6 @@ public: inputs_arr.getMatVector(inputs); outputs_arr.getMatVector(outputs); - /*if (inputs[0].dims > 3) { - printf("conv %s: input (%d x %d x %d x %d), kernel (%d x %d), pad (%d x %d), stride (%d x %d), dilation (%d x %d)\n", - name.c_str(), inputs[0].size[0], inputs[0].size[1], inputs[0].size[2], inputs[0].size[3], - kernel.width, kernel.height, pad.width, pad.height, - stride.width, stride.height, dilation.width, dilation.height); - } - else { - printf("conv %s: input (%d x %d x %d), kernel (%d x %d), pad (%d x %d), stride (%d x %d), dilation (%d x %d)\n", - name.c_str(), inputs[0].size[0], inputs[0].size[1], inputs[0].size[2], - kernel.width, kernel.height, pad.width, pad.height, - stride.width, stride.height, dilation.width, dilation.height); - }*/ - int inpGroupCn = blobs[0].size[1]; CV_Assert_N(inputs.size() == (size_t)1, inputs[0].size[1] % inpGroupCn == 0, outputs.size() == 1, inputs[0].data != outputs[0].data); @@ -1533,12 +1586,53 @@ public: CV_Assert(outputs[0].size[1] % ngroups == 0); int nstripes = std::max(getNumThreads(), 1); - Mat outputInt32 = Mat(shape(outputs[0]), CV_32S); - - ParallelConv::run(inputs[0], outputInt32, weightsMat, outputMultiplier, biasvec, activationLUT, kernel_size, strides, - pads_begin, pads_end, dilations, activ.get(), ngroups, nstripes, input_zp, output_zp); - - outputInt32.convertTo(outputs[0], CV_8S); + outputInt32Buf_.fit(shape(outputs[0]), CV_32S); + Mat outputInt32 = outputInt32Buf_; + if (inputs[0].type() == CV_8U) + { + #if CV_AVXVNNI_AVAILABLE + // AVX-VNNI path: pass uint8 input directly, skip u8→s8 conversion. + if (checkHardwareSupport(CPU_AVX_VNNI) && inputs[0].dims == 4) { + if (biasvecVNNI.empty() && !biasvec.empty()) { + int outCn = weightsMat.rows; + biasvecVNNI.resize(biasvec.size()); + for (int oc = 0; oc < outCn; oc++) { + const int8_t* wrow = weightsMat.ptr(oc); + int colsum = 0; + for (int j = 0; j < weightsMat.cols; j++) + colsum += (int)wrow[j]; + biasvecVNNI[oc] = biasvec[oc] - 128 * colsum; + } + for (size_t oc = outCn; oc < biasvec.size(); oc++) + biasvecVNNI[oc] = biasvecVNNI[outCn > 0 ? outCn - 1 : 0]; + } + const int outZpS8 = output_zp - 128; + ParallelConv::run(inputs[0], outputInt32, weightsMat, outputMultiplier, + biasvecVNNI, activationLUT, kernel_size, strides, + pads_begin, pads_end, dilations, activ.get(), + ngroups, nstripes, input_zp, outZpS8, + &outputs[0]); + } + else + #endif + { + Mat inpS8; + inputs[0].convertTo(inpS8, CV_8S, 1.0, -128.0); + const int inpZpS8 = input_zp - 128; + const int outZpS8 = output_zp - 128; + ParallelConv::run(inpS8, outputInt32, weightsMat, outputMultiplier, biasvec, activationLUT, kernel_size, strides, + pads_begin, pads_end, dilations, activ.get(), ngroups, nstripes, inpZpS8, outZpS8); + Mat outS8; + outputInt32.convertTo(outS8, CV_8S); + outS8.convertTo(outputs[0], CV_8U, 1.0, 128.0); + } + } + else + { + ParallelConv::run(inputs[0], outputInt32, weightsMat, outputMultiplier, biasvec, activationLUT, kernel_size, strides, + pads_begin, pads_end, dilations, activ.get(), ngroups, nstripes, input_zp, output_zp); + outputInt32.convertTo(outputs[0], outputs[0].type()); + } #if CV_SSE3 _MM_SET_FLUSH_ZERO_MODE(ftzMode); diff --git a/modules/dnn/src/int8layers/elementwise_layers.cpp b/modules/dnn/src/int8layers/elementwise_layers.cpp index f522efa0c1..45321b4119 100644 --- a/modules/dnn/src/int8layers/elementwise_layers.cpp +++ b/modules/dnn/src/int8layers/elementwise_layers.cpp @@ -18,8 +18,6 @@ namespace dnn class ActivationLayerInt8Impl CV_FINAL : public ActivationLayerInt8 { public: - int input_zp, output_zp; - float input_sc, output_sc; float slope = 0.0f; #ifdef HAVE_TIMVX @@ -30,10 +28,10 @@ public: setParamsFrom(params); activationLUT = !blobs.empty() ? blobs[0] : Mat(); - input_zp = params.get("input_zeropoint"); - input_sc = params.get("input_scale"); - output_zp = params.get("zeropoints"); - output_sc = params.get("scales"); + input_zp = params.get("input_zeropoint", 0); + input_sc = params.get("input_scale", 1.0f); + output_zp = params.get("zeropoints", 0); + output_sc = params.get("scales", 1.0f); if (params.has("slope")) { @@ -355,7 +353,6 @@ public: } - Mat activationLUT; }; Ptr ActivationLayerInt8::create(const LayerParams& params) diff --git a/modules/dnn/src/int8layers/eltwise_layer.cpp b/modules/dnn/src/int8layers/eltwise_layer.cpp index a42f45e070..5dfd61c8b8 100644 --- a/modules/dnn/src/int8layers/eltwise_layer.cpp +++ b/modules/dnn/src/int8layers/eltwise_layer.cpp @@ -7,6 +7,8 @@ #include "../op_timvx.hpp" #include "../ie_ngraph.hpp" #include +#include "opencv2/core/hal/hal.hpp" +#include "opencv2/core/hal/intrin.hpp" namespace cv { @@ -22,12 +24,6 @@ public: SUM = 1, MAX = 2 } op; - std::vector coeffs; - std::vector zeropoints; - std::vector scales; - - int output_zp; - float output_sc; enum OutputChannelsMode { @@ -101,8 +97,8 @@ public: } } - output_zp = params.get("zeropoints"); - output_sc = params.get("scales"); + output_zp = params.get("zeropoints", 0); + output_sc = params.get("scales", 1.0f); channelsModeInput = ELTWISE_CHANNNELS_SAME; if (params.has("output_channels_mode")) @@ -142,6 +138,56 @@ public: return backendId == DNN_BACKEND_OPENCV || backendId == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH; } + void ensureQuantizationParams() + { + if (!coeffs.empty()) + return; + + if (scales.empty()) + return; + + CV_CheckEQ(scales.size(), zeropoints.size(), "EltwiseInt8: scales and zeropoints sizes must match"); + CV_Assert(output_sc > 0.0f); + + if (op == SUM) + { + coeffs.resize(scales.size()); + offset = (float)output_zp; + for (size_t i = 0; i < scales.size(); i++) + { + coeffs[i] = scales[i] / output_sc; + offset -= coeffs[i] * zeropoints[i]; + } + } + else if (op == PROD) + { + coeffs.resize(scales.size()); + coeffs[0] = scales[0] / output_sc; + for (size_t i = 1; i < scales.size(); i++) + coeffs[i] = scales[i]; + offset = (float)output_zp; + } + else if (op == MAX) + { + for (size_t i = 0; i < scales.size(); i++) + { + const bool sameQuant = + std::abs(scales[i] - output_sc) < 1e-6f && + zeropoints[i] == output_zp; + if (!sameQuant) + { + CV_Error(Error::StsBadArg, + "EltwiseInt8 'max' requires identical quantization " + "(same scale and zero-point) for all inputs and output."); + } + } + } + else + { + CV_Error(Error::StsNotImplemented, "Unsupported eltwise operation"); + } + } + bool getMemoryShapes(const std::vector &inputs, const int requiredOutputs, std::vector &outputs, @@ -248,6 +294,7 @@ public: bool isLast) CV_OVERRIDE { #ifdef HAVE_TIMVX + ensureQuantizationParams(); // tvGraph Initialization. if (inputsWrapper.size() != 2) return Ptr(); @@ -374,6 +421,7 @@ public: virtual Ptr initNgraph(const std::vector > &inputs, const std::vector >& nodes) CV_OVERRIDE { + ensureQuantizationParams(); CV_Assert(nodes.size() >= 2); std::vector> ieInpNodes(nodes.size()); for (size_t i = 0; i < nodes.size(); i++) @@ -544,6 +592,7 @@ public: size_t dstIdx = delta + (sampleIdx*channels + c)*planeSize; int8_t* dstptr = dstptr0 + dstIdx; float* bufptr = bufptr0 + dstIdx; + bool directOutput = false; // process first two inputs { @@ -589,7 +638,16 @@ public: } else if (op == MAX) { - for (int j = 0; j < blockSize; j++) + int j = 0; + #if CV_SIMD128 + for (; j <= blockSize - 16; j += 16) + { + v_int8x16 va = v_load(srcptr0 + j); + v_int8x16 vb = v_load(srcptrI + j); + v_store(dstptr + j, v_max(va, vb)); + } + #endif + for (; j < blockSize; j++) { dstptr[j] = std::max(srcptr0[j], srcptrI[j]); } @@ -598,9 +656,86 @@ public: { float c0 = coeffsptr[0]; float c1 = coeffsptr[1]; - for (int j = 0; j < blockSize; j++) + if (nsrcs == 2) { - bufptr[j] = c0*srcptr0[j] + c1*srcptrI[j]; + directOutput = true; + int j = 0; + #if CV_SIMD128 + { + v_float32x4 vc0 = v_setall_f32(c0); + v_float32x4 vc1 = v_setall_f32(c1); + v_float32x4 voffs = v_setall_f32(offset); + for (; j <= blockSize - 16; j += 16) + { + v_int8x16 va = v_load(srcptr0 + j); + v_int8x16 vb = v_load(srcptrI + j); + + v_int16x8 va_lo, va_hi, vb_lo, vb_hi; + v_expand(va, va_lo, va_hi); + v_expand(vb, vb_lo, vb_hi); + + v_int32x4 va0, va1, va2, va3, vb0, vb1, vb2, vb3; + v_expand(va_lo, va0, va1); + v_expand(va_hi, va2, va3); + v_expand(vb_lo, vb0, vb1); + v_expand(vb_hi, vb2, vb3); + + v_float32x4 r0 = v_add(v_add(v_mul(v_cvt_f32(va0), vc0), + v_mul(v_cvt_f32(vb0), vc1)), voffs); + v_float32x4 r1 = v_add(v_add(v_mul(v_cvt_f32(va1), vc0), + v_mul(v_cvt_f32(vb1), vc1)), voffs); + v_float32x4 r2 = v_add(v_add(v_mul(v_cvt_f32(va2), vc0), + v_mul(v_cvt_f32(vb2), vc1)), voffs); + v_float32x4 r3 = v_add(v_add(v_mul(v_cvt_f32(va3), vc0), + v_mul(v_cvt_f32(vb3), vc1)), voffs); + + v_store(dstptr + j, v_pack(v_pack(v_round(r0), v_round(r1)), + v_pack(v_round(r2), v_round(r3)))); + } + } + #endif + for (; j < blockSize; j++) + { + dstptr[j] = saturate_cast(std::round(c0*srcptr0[j] + c1*srcptrI[j] + offset)); + } + } + else + { + int j = 0; + #if CV_SIMD128 + { + v_float32x4 vc0 = v_setall_f32(c0); + v_float32x4 vc1 = v_setall_f32(c1); + for (; j <= blockSize - 16; j += 16) + { + v_int8x16 va = v_load(srcptr0 + j); + v_int8x16 vb = v_load(srcptrI + j); + + v_int16x8 va_lo, va_hi, vb_lo, vb_hi; + v_expand(va, va_lo, va_hi); + v_expand(vb, vb_lo, vb_hi); + + v_int32x4 va0, va1, va2, va3, vb0, vb1, vb2, vb3; + v_expand(va_lo, va0, va1); + v_expand(va_hi, va2, va3); + v_expand(vb_lo, vb0, vb1); + v_expand(vb_hi, vb2, vb3); + + v_store(bufptr + j, v_add(v_mul(v_cvt_f32(va0), vc0), + v_mul(v_cvt_f32(vb0), vc1))); + v_store(bufptr + j + 4, v_add(v_mul(v_cvt_f32(va1), vc0), + v_mul(v_cvt_f32(vb1), vc1))); + v_store(bufptr + j + 8, v_add(v_mul(v_cvt_f32(va2), vc0), + v_mul(v_cvt_f32(vb2), vc1))); + v_store(bufptr + j + 12, v_add(v_mul(v_cvt_f32(va3), vc0), + v_mul(v_cvt_f32(vb3), vc1))); + } + } + #endif + for (; j < blockSize; j++) + { + bufptr[j] = c0*srcptr0[j] + c1*srcptrI[j]; + } } } else @@ -628,7 +763,14 @@ public: } else if (op == MAX) { - for (int j = 0; j < blockSize; j++) + int j = 0; + #if CV_SIMD128 + for (; j <= blockSize - 16; j += 16) + { + v_store(dstptr + j, v_max(v_load(dstptr + j), v_load(srcptrI + j))); + } + #endif + for (; j < blockSize; j++) { dstptr[j] = std::max(dstptr[j], srcptrI[j]); } @@ -636,7 +778,31 @@ public: else if (op == SUM) { float cI = coeffsptr[inputIdx]; - for (int j = 0; j < blockSize; j++) + int j = 0; + #if CV_SIMD128 + { + v_float32x4 vcI = v_setall_f32(cI); + for (; j <= blockSize - 16; j += 16) + { + v_int8x16 vi = v_load(srcptrI + j); + v_int16x8 vi_lo, vi_hi; + v_expand(vi, vi_lo, vi_hi); + v_int32x4 vi0, vi1, vi2, vi3; + v_expand(vi_lo, vi0, vi1); + v_expand(vi_hi, vi2, vi3); + + v_store(bufptr + j, v_add(v_load(bufptr + j), + v_mul(v_cvt_f32(vi0), vcI))); + v_store(bufptr + j + 4, v_add(v_load(bufptr + j + 4), + v_mul(v_cvt_f32(vi1), vcI))); + v_store(bufptr + j + 8, v_add(v_load(bufptr + j + 8), + v_mul(v_cvt_f32(vi2), vcI))); + v_store(bufptr + j + 12, v_add(v_load(bufptr + j + 12), + v_mul(v_cvt_f32(vi3), vcI))); + } + } + #endif + for (; j < blockSize; j++) { bufptr[j] += cI * srcptrI[j]; } @@ -646,9 +812,24 @@ public: } // add offset and saturate cast to int8 - if (op == SUM || op == PROD) + if ((op == SUM || op == PROD) && !directOutput) { - for (int j = 0; j < blockSize; j++) + int j = 0; + #if CV_SIMD128 + { + v_float32x4 voffs = v_setall_f32(offset); + for (; j <= blockSize - 16; j += 16) + { + v_float32x4 r0 = v_add(v_load(bufptr + j), voffs); + v_float32x4 r1 = v_add(v_load(bufptr + j + 4), voffs); + v_float32x4 r2 = v_add(v_load(bufptr + j + 8), voffs); + v_float32x4 r3 = v_add(v_load(bufptr + j + 12), voffs); + v_store(dstptr + j, v_pack(v_pack(v_round(r0), v_round(r1)), + v_pack(v_round(r2), v_round(r3)))); + } + } + #endif + for (; j < blockSize; j++) { dstptr[j] = saturate_cast(std::round(bufptr[j] + offset)); } @@ -667,6 +848,7 @@ public: { CV_TRACE_FUNCTION(); CV_TRACE_ARG_VALUE(name, "name", name.c_str()); + ensureQuantizationParams(); std::vector inputs, outputs; inputs_arr.getMatVector(inputs); @@ -749,7 +931,6 @@ public: private: bool hasVecInput; - float offset; }; Ptr EltwiseLayerInt8::create(const LayerParams& params) diff --git a/modules/dnn/src/int8layers/fully_connected_layer.cpp b/modules/dnn/src/int8layers/fully_connected_layer.cpp index 033c3309ec..5882401ada 100644 --- a/modules/dnn/src/int8layers/fully_connected_layer.cpp +++ b/modules/dnn/src/int8layers/fully_connected_layer.cpp @@ -22,10 +22,10 @@ public: { setParamsFrom(params); - input_sc = params.get("input_scale"); - input_zp = params.get("input_zeropoint"); - output_zp = params.get("zeropoints"); - output_sc = params.get("scales"); + input_sc = params.get("input_scale", 1.0f); + input_zp = params.get("input_zeropoint", 0); + output_zp = params.get("zeropoints", 0); + output_sc = params.get("scales", 1.0f); axis = params.get("axis", 1); per_channel = params.get("per_channel", true); diff --git a/modules/dnn/src/int8layers/layers_common.simd.hpp b/modules/dnn/src/int8layers/layers_common.simd.hpp index 281e7c586b..eb46aedc70 100644 --- a/modules/dnn/src/int8layers/layers_common.simd.hpp +++ b/modules/dnn/src/int8layers/layers_common.simd.hpp @@ -4,6 +4,16 @@ #include "opencv2/core/hal/intrin.hpp" +#if !defined(CV_AVXVNNI_AVAILABLE) +#if (CV_TRY_AVX2 || CV_AVX2) && \ + ((defined(__GNUC__) && !defined(__clang__) && __GNUC__ >= 11) || \ + (defined(__clang__) && !defined(__apple_build_version__) && __clang_major__ >= 12)) +#define CV_AVXVNNI_AVAILABLE 1 +#else +#define CV_AVXVNNI_AVAILABLE 0 +#endif +#endif + namespace cv { namespace dnn { CV_CPU_OPTIMIZATION_NAMESPACE_BEGIN @@ -27,6 +37,17 @@ void fastGEMM1T( const int8_t* vec, const int8_t* weights, size_t wstep, const int* bias, const float* multiplier, int* dst, int nvecs, int vecsize, int outZp ); +#if CV_AVXVNNI_AVAILABLE +// AVX-VNNI variants: take uint8 input directly, use _mm256_dpbusd_epi32 +void fastConvVNNI( const int8_t* weights, size_t wstep, const int* bias, + const uint8_t* rowbuf, int* output, const int* outShape, + int blockSize, int vecsize, int vecsize_aligned, int outZp, + const float* multiplier, bool initOutput, bool finalOutput ); +void fastGEMM1TVNNI( const uint8_t* vec, const int8_t* weights, + size_t wstep, const int* bias, const float* multiplier, + int* dst, int nvecs, int vecsize, int outZp ); +#endif + #if !defined(CV_CPU_OPTIMIZATION_DECLARATIONS_ONLY) && CV_AVX2 #define OPENCV_FMADD_EPI8(_Tpvec, func) \ inline _Tpvec _##func##_fmaddepi8_epi32(const _Tpvec& a, const _Tpvec& b, const _Tpvec& c) \ @@ -328,6 +349,231 @@ void fastConv( const int8_t* weights, size_t wstep, const int* bias, _mm256_zeroupper(); } +#if CV_AVXVNNI_AVAILABLE +// AVX-VNNI optimized convolution: uses _mm256_dpbusd_epi32 (unsigned input × signed weights) +__attribute__((target("avxvnni"))) +void fastConvVNNI( const int8_t* weights, size_t wstep, const int* bias, + const uint8_t* rowbuf, int* output, const int* outShape, + int blockSize, int vecsize, int vecsize_aligned, int outZp, + const float* multiplier, bool initOutput, bool finalOutput ) +{ + int outCn = outShape[1]; + size_t outPlaneSize = outShape[2]*outShape[3]; + int CV_DECL_ALIGNED(16) maskbuf[FASCONV_BASE_VECSZ] = {0}; + int rsz = blockSize % FASCONV_BASE_VECSZ; + for( int i = 0; i < rsz; i++ ) + maskbuf[FASCONV_BASE_VECSZ - i - 1] = -1; + __m128 mask = _mm_loadu_ps((const float*)maskbuf); + + for( int i = 0; i < outCn; i += 3 ) + { + const int8_t* wptr0 = weights + i*wstep; + const int8_t* wptr1 = wptr0 + wstep; + const int8_t* wptr2 = wptr1 + wstep; + int* outptr0 = output + i*outPlaneSize; + int* outptr1 = outptr0 + outPlaneSize; + int* outptr2 = outptr1 + outPlaneSize; + int bias0 = bias[i], bias1 = bias[i+1], bias2 = bias[i+2]; + float mult0 = multiplier[i], mult1 = multiplier[i+1], mult2 = multiplier[i+2]; + + if( i+2 >= outCn ) + { + wptr2 = wptr1; + outptr2 = outptr1; + bias2 = bias1; + mult2 = mult1; + + if( i+1 >= outCn ) + { + wptr2 = wptr1 = wptr0; + outptr2 = outptr1 = outptr0; + bias2 = bias1 = bias0; + mult2 = mult1 = mult0; + } + } + int j = 0; + for( ; j < blockSize; j += FASCONV_BASE_VECSZ ) + { + bool tail = false; + if (j + FASCONV_BASE_VECSZ > blockSize) + { + if (j == 0) + break; + j = blockSize - FASCONV_BASE_VECSZ; + tail = true; + } + int k = 0; + const uint8_t* rptr = rowbuf + j*vecsize_aligned; + + __m256i vs00 = _mm256_setzero_si256(), vs01 = _mm256_setzero_si256(), + vs02 = _mm256_setzero_si256(), vs03 = _mm256_setzero_si256(), + vs10 = _mm256_setzero_si256(), vs11 = _mm256_setzero_si256(), + vs12 = _mm256_setzero_si256(), vs13 = _mm256_setzero_si256(), + vs20 = _mm256_setzero_si256(), vs21 = _mm256_setzero_si256(), + vs22 = _mm256_setzero_si256(), vs23 = _mm256_setzero_si256(); + + for (; k < vecsize; k += 32, rptr += 32 ) + { + __m256i w0 = _mm256_load_si256((const __m256i*)(wptr0 + k)); + __m256i w1 = _mm256_load_si256((const __m256i*)(wptr1 + k)); + __m256i w2 = _mm256_load_si256((const __m256i*)(wptr2 + k)); + __m256i r0 = _mm256_load_si256((const __m256i*)rptr); + + vs00 = _mm256_dpbusd_epi32(vs00, r0, w0); + vs10 = _mm256_dpbusd_epi32(vs10, r0, w1); + vs20 = _mm256_dpbusd_epi32(vs20, r0, w2); + + r0 = _mm256_load_si256((const __m256i*)(rptr + vecsize_aligned)); + vs01 = _mm256_dpbusd_epi32(vs01, r0, w0); + vs11 = _mm256_dpbusd_epi32(vs11, r0, w1); + vs21 = _mm256_dpbusd_epi32(vs21, r0, w2); + + r0 = _mm256_load_si256((const __m256i*)(rptr + vecsize_aligned*2)); + vs02 = _mm256_dpbusd_epi32(vs02, r0, w0); + vs12 = _mm256_dpbusd_epi32(vs12, r0, w1); + vs22 = _mm256_dpbusd_epi32(vs22, r0, w2); + + r0 = _mm256_load_si256((const __m256i*)(rptr + vecsize_aligned*3)); + vs03 = _mm256_dpbusd_epi32(vs03, r0, w0); + vs13 = _mm256_dpbusd_epi32(vs13, r0, w1); + vs23 = _mm256_dpbusd_epi32(vs23, r0, w2); + } + + __m256i t0 = _mm256_hadd_epi32(_mm256_hadd_epi32(vs00, vs01), _mm256_hadd_epi32(vs02, vs03)); + __m256i t1 = _mm256_hadd_epi32(_mm256_hadd_epi32(vs10, vs11), _mm256_hadd_epi32(vs12, vs13)); + __m256i t2 = _mm256_hadd_epi32(_mm256_hadd_epi32(vs20, vs21), _mm256_hadd_epi32(vs22, vs23)); + + t0 = _mm256_add_epi32(t0, _mm256_permute2x128_si256(t0, t0, 1)); + t1 = _mm256_add_epi32(t1, _mm256_permute2x128_si256(t1, t1, 1)); + t2 = _mm256_add_epi32(t2, _mm256_permute2x128_si256(t2, t2, 1)); + + __m128i s0, s1, s2; + + if( initOutput ) + { + s0 = _mm_set1_epi32(bias0); + s1 = _mm_set1_epi32(bias1); + s2 = _mm_set1_epi32(bias2); + } + else + { + s0 = _mm_loadu_si128((__m128i*)(outptr0 + j)); + s1 = _mm_loadu_si128((__m128i*)(outptr1 + j)); + s2 = _mm_loadu_si128((__m128i*)(outptr2 + j)); + } + + s0 = _mm_add_epi32(s0, _mm256_castsi256_si128(t0)); + s1 = _mm_add_epi32(s1, _mm256_castsi256_si128(t1)); + s2 = _mm_add_epi32(s2, _mm256_castsi256_si128(t2)); + + if( finalOutput ) + { + __m128i voutzp = _mm_set1_epi32(outZp); + __m128i outmin = _mm_set1_epi32(-128), outmax = _mm_set1_epi32(127); + s0 = _mm_add_epi32(voutzp, _mm_cvtps_epi32(_mm_mul_ps(_mm_cvtepi32_ps(s0), _mm_set1_ps(mult0)))); + s1 = _mm_add_epi32(voutzp, _mm_cvtps_epi32(_mm_mul_ps(_mm_cvtepi32_ps(s1), _mm_set1_ps(mult1)))); + s2 = _mm_add_epi32(voutzp, _mm_cvtps_epi32(_mm_mul_ps(_mm_cvtepi32_ps(s2), _mm_set1_ps(mult2)))); + + s0 = _mm_min_epi32(_mm_max_epi32(s0, outmin), outmax); + s1 = _mm_min_epi32(_mm_max_epi32(s1, outmin), outmax); + s2 = _mm_min_epi32(_mm_max_epi32(s2, outmin), outmax); + } + if( tail ) + { + s0 = _mm_castps_si128(_mm_blendv_ps(_mm_loadu_ps((const float*)outptr0 + j), _mm_castsi128_ps(s0), mask)); + s1 = _mm_castps_si128(_mm_blendv_ps(_mm_loadu_ps((const float*)outptr1 + j), _mm_castsi128_ps(s1), mask)); + s2 = _mm_castps_si128(_mm_blendv_ps(_mm_loadu_ps((const float*)outptr2 + j), _mm_castsi128_ps(s2), mask)); + } + _mm_storeu_si128((__m128i*)(outptr0 + j), s0); + _mm_storeu_si128((__m128i*)(outptr1 + j), s1); + _mm_storeu_si128((__m128i*)(outptr2 + j), s2); + } + + for( ; j <= blockSize - 2; j += 2 ) + { + const uint8_t* rptr0 = rowbuf + j*vecsize_aligned; + const uint8_t* rptr1 = rowbuf + (j+1)*vecsize_aligned; + int s00, s01, s10, s11, s20, s21; + + if( initOutput ) + { + s00 = s01 = bias0; + s10 = s11 = bias1; + s20 = s21 = bias2; + } + else + { + s00 = outptr0[j]; s01 = outptr0[j+1]; + s10 = outptr1[j]; s11 = outptr1[j+1]; + s20 = outptr2[j]; s21 = outptr2[j+1]; + } + + for( int k = 0; k < vecsize; k++ ) + { + int8_t w0 = wptr0[k], w1 = wptr1[k], w2 = wptr2[k]; + int r = (int)rptr0[k]; + s00 += (int)w0*r; s10 += (int)w1*r; s20 += (int)w2*r; + r = (int)rptr1[k]; + s01 += (int)w0*r; s11 += (int)w1*r; s21 += (int)w2*r; + } + + if( finalOutput ) + { + s00 = std::min(std::max(outZp + (int)std::round(s00*mult0), -128), 127); + s01 = std::min(std::max(outZp + (int)std::round(s01*mult0), -128), 127); + s10 = std::min(std::max(outZp + (int)std::round(s10*mult1), -128), 127); + s11 = std::min(std::max(outZp + (int)std::round(s11*mult1), -128), 127); + s20 = std::min(std::max(outZp + (int)std::round(s20*mult2), -128), 127); + s21 = std::min(std::max(outZp + (int)std::round(s21*mult2), -128), 127); + } + outptr0[j] = s00; + outptr0[j+1] = s01; + outptr1[j] = s10; + outptr1[j+1] = s11; + outptr2[j] = s20; + outptr2[j+1] = s21; + } + + for( ; j < blockSize; j++ ) + { + const uint8_t* rptr0 = rowbuf + j*vecsize_aligned; + int s00, s10, s20; + + if( initOutput ) + { + s00 = bias0; + s10 = bias1; + s20 = bias2; + } + else + { + s00 = outptr0[j]; + s10 = outptr1[j]; + s20 = outptr2[j]; + } + + for( int k = 0; k < vecsize; k++ ) + { + int8_t w0 = wptr0[k], w1 = wptr1[k], w2 = wptr2[k]; + int r = (int)rptr0[k]; + s00 += (int)w0*r; s10 += (int)w1*r; s20 += (int)w2*r; + } + + if( finalOutput ) + { + s00 = std::min(std::max(outZp + (int)std::round(s00*mult0), -128), 127); + s10 = std::min(std::max(outZp + (int)std::round(s10*mult1), -128), 127); + s20 = std::min(std::max(outZp + (int)std::round(s20*mult2), -128), 127); + } + outptr0[j] = s00; + outptr1[j] = s10; + outptr2[j] = s20; + } + } + _mm256_zeroupper(); +} +#endif // CV_AVXVNNI_AVAILABLE (fastConvVNNI) + static inline void _mm256_expand_mul_add(const __m256i& a, const __m256i& b, __m256i& out0, __m256i& out1, __m256i& out2, __m256i& out3) { @@ -631,6 +877,81 @@ void fastGEMM1T( const int8_t* vec, const int8_t* weights, _mm256_zeroupper(); } + +#if CV_AVXVNNI_AVAILABLE +// AVX-VNNI variant of fastGEMM1T: uses _mm256_dpbusd_epi32 (unsigned input × signed weights) +__attribute__((target("avxvnni"))) +void fastGEMM1TVNNI( const uint8_t* vec, const int8_t* weights, + size_t wstep, const int* bias, const float* multiplier, + int* dst, int nvecs, int vecsize, int outZp ) +{ + int i = 0; + + for( ; i <= nvecs - 8; i += 8 ) + { + const int8_t* wptr = weights + i*wstep; + __m256i vs0 = _mm256_setzero_si256(), vs1 = _mm256_setzero_si256(), + vs2 = _mm256_setzero_si256(), vs3 = _mm256_setzero_si256(), + vs4 = _mm256_setzero_si256(), vs5 = _mm256_setzero_si256(), + vs6 = _mm256_setzero_si256(), vs7 = _mm256_setzero_si256(); + + __m128i voutzp = _mm_set1_epi32(outZp); + __m128i outmin = _mm_set1_epi32(-128), outmax = _mm_set1_epi32(127); + + for( int k = 0; k < vecsize; k += 32, wptr += 32 ) + { + __m256i v = _mm256_load_si256((const __m256i*)(vec + k)); + + vs0 = _mm256_dpbusd_epi32(vs0, v, _mm256_load_si256((const __m256i*)wptr)); + vs1 = _mm256_dpbusd_epi32(vs1, v, _mm256_load_si256((const __m256i*)(wptr + wstep))); + vs2 = _mm256_dpbusd_epi32(vs2, v, _mm256_load_si256((const __m256i*)(wptr + wstep*2))); + vs3 = _mm256_dpbusd_epi32(vs3, v, _mm256_load_si256((const __m256i*)(wptr + wstep*3))); + vs4 = _mm256_dpbusd_epi32(vs4, v, _mm256_load_si256((const __m256i*)(wptr + wstep*4))); + vs5 = _mm256_dpbusd_epi32(vs5, v, _mm256_load_si256((const __m256i*)(wptr + wstep*5))); + vs6 = _mm256_dpbusd_epi32(vs6, v, _mm256_load_si256((const __m256i*)(wptr + wstep*6))); + vs7 = _mm256_dpbusd_epi32(vs7, v, _mm256_load_si256((const __m256i*)(wptr + wstep*7))); + } + + __m256i s0 = _mm256_hadd_epi32(_mm256_hadd_epi32(vs0, vs1), _mm256_hadd_epi32(vs2, vs3)); + __m256i s1 = _mm256_hadd_epi32(_mm256_hadd_epi32(vs4, vs5), _mm256_hadd_epi32(vs6, vs7)); + + s0 = _mm256_add_epi32(s0, _mm256_permute2x128_si256(s0, s0, 1)); + s1 = _mm256_add_epi32(s1, _mm256_permute2x128_si256(s1, s1, 1)); + + __m128i t0 = _mm_add_epi32(_mm256_castsi256_si128(s0), _mm_loadu_si128((__m128i*)(bias + i))); + __m128i t1 = _mm_add_epi32(_mm256_castsi256_si128(s1), _mm_loadu_si128((__m128i*)(bias + i + 4))); + + t0 = _mm_add_epi32(voutzp, _mm_cvtps_epi32(_mm_mul_ps(_mm_cvtepi32_ps(t0), _mm_loadu_ps(multiplier + i)))); + t1 = _mm_add_epi32(voutzp, _mm_cvtps_epi32(_mm_mul_ps(_mm_cvtepi32_ps(t1), _mm_loadu_ps(multiplier + i + 4)))); + + t0 = _mm_min_epi32(_mm_max_epi32(t0, outmin), outmax); + t1 = _mm_min_epi32(_mm_max_epi32(t1, outmin), outmax); + + _mm_storeu_si128((__m128i*)(dst + i), t0); + _mm_storeu_si128((__m128i*)(dst + i + 4), t1); + } + + for( ; i < nvecs; i++ ) + { + const int8_t* wptr = weights + i*wstep; + __m256i vs0 = _mm256_setzero_si256(); + + for( int k = 0; k < vecsize; k += 32, wptr += 32 ) + { + __m256i v = _mm256_load_si256((const __m256i*)(vec + k)); + vs0 = _mm256_dpbusd_epi32(vs0, v, _mm256_load_si256((const __m256i*)wptr)); + } + + __m256i s0 = _mm256_hadd_epi32(_mm256_hadd_epi32(vs0, vs0), vs0); + s0 = _mm256_add_epi32(s0, _mm256_permute2x128_si256(s0, s0, 1)); + int temp = _mm_extract_epi32(_mm256_castsi256_si128(s0), 0); + dst[i] = outZp + (int)std::round((temp + bias[i]) * multiplier[i]); + } + + _mm256_zeroupper(); +} +#endif // CV_AVXVNNI_AVAILABLE (fastGEMM1TVNNI) + #endif // CV_CPU_OPTIMIZATION_DECLARATIONS_ONLY diff --git a/modules/dnn/src/int8layers/pooling_layer.cpp b/modules/dnn/src/int8layers/pooling_layer.cpp index 77b0754b5b..e6fa414c62 100644 --- a/modules/dnn/src/int8layers/pooling_layer.cpp +++ b/modules/dnn/src/int8layers/pooling_layer.cpp @@ -30,16 +30,13 @@ public: isGlobalPooling = std::vector(3, false); output_zp = params.get("zeropoints", 0); input_zp = params.get("input_zeropoint", output_zp); - multiplier = params.get("multiplier", 1.f); - output_sc = params.get("scales", 1.f); - input_sc = multiplier * output_sc; + input_sc = params.get("input_scale", params.get("multiplier", 1.f) * output_sc); hasDynamicShapes = params.get("has_dynamic_shapes", false); shapesInitialized = !hasDynamicShapes; - if (params.has("pool") || params.has("kernel_size") || - params.has("kernel_w") || params.has("kernel_h")) + if (params.has("pool")) { String pool = toLowerCase(params.get("pool", "max")); if (pool == "max") @@ -50,12 +47,21 @@ public: type = SUM; else CV_Error(Error::StsBadArg, "Unknown pooling type \"" + pool + "\""); + } + else + { + type = MAX; + } + const bool hasKernelOrGlobalSpec = + params.has("kernel_size") || params.has("kernel_w") || params.has("kernel_h") || params.has("kernel_d") || + params.has("global_pooling") || params.has("global_pooling_d") || + params.has("global_pooling_h") || params.has("global_pooling_w"); + if (hasKernelOrGlobalSpec) + { getPoolingKernelParams(params, kernel_size, isGlobalPooling, pads_begin, pads_end, strides, padMode); globalPooling = isGlobalPooling[0] || isGlobalPooling[1] || isGlobalPooling[2]; } - else - CV_Error(Error::StsBadArg, "Cannot determine pooling type"); setParamsFrom(params); ceilMode = params.get("ceil_mode", true); spatialScale = params.get("spatial_scale", 1); @@ -357,7 +363,7 @@ public: int nstripes, inpZp, outZp; std::vector ofsbuf; int poolingType; - float multiplier; + float inputScaleRatio; float spatialScale; std::vector pads_begin, pads_end; @@ -366,13 +372,13 @@ public: PoolingInvoker() : src(0), rois(0), dst(0), pad_l(0), pad_t(0), pad_r(0), pad_b(0), avePoolPaddedArea(false), nstripes(0), inpZp(0), outZp(0), - poolingType(MAX), multiplier(1), spatialScale(0){} + poolingType(MAX), inputScaleRatio(1.f), spatialScale(0){} static void run(const Mat& src, const Mat& rois, Mat& dst, std::vector kernel_size, std::vector strides, std::vector pads_begin, std::vector pads_end, bool avePoolPaddedArea, int poolingType, float spatialScale, - float multiplier, int inpZp, int outZp, int nstripes) + float inputScaleRatio, int inpZp, int outZp, int nstripes) { CV_Assert_N( src.isContinuous(), dst.isContinuous(), @@ -405,7 +411,7 @@ public: p.outZp = outZp; p.poolingType = poolingType; p.spatialScale = spatialScale; - p.multiplier = multiplier; + p.inputScaleRatio = inputScaleRatio; int height = isPool1D ? 1 : src.size[src.dims - 2]; int width = src.size[src.dims - 1]; @@ -610,7 +616,7 @@ public: int bias = (avePoolPaddedArea ? (padded_kernel_area - real_kernel_area) * inpZp : 0) - (inpZp * kernel_area); - float inv_kernel_area = poolingType == AVE ? multiplier / kernel_area : multiplier; + float inv_kernel_area = poolingType == AVE ? inputScaleRatio / kernel_area : inputScaleRatio; #if CV_SIMD128 if( isPool2D && xstart > 0 && x0 + 15 < x1 && (x0 + 15) * stride_w - pad_l + kernel_w < inp_width ) { @@ -680,16 +686,18 @@ public: { const int nstripes = getNumThreads(); Mat rois; + const float inputScaleRatio = input_sc / std::max(output_sc, 1e-12f); PoolingInvoker::run(src, rois, dst, kernel_size, strides, pads_begin, pads_end, avePoolPaddedArea, type, - spatialScale, multiplier, input_zp, output_zp, nstripes); + spatialScale, inputScaleRatio, input_zp, output_zp, nstripes); } void avePooling(Mat &src, Mat &dst) { const int nstripes = getNumThreads(); Mat rois; + const float inputScaleRatio = input_sc / std::max(output_sc, 1e-12f); PoolingInvoker::run(src, rois, dst, kernel_size, strides, pads_begin, pads_end, avePoolPaddedArea, type, - spatialScale, multiplier, input_zp, output_zp, nstripes); + spatialScale, inputScaleRatio, input_zp, output_zp, nstripes); } bool getMemoryShapes(const std::vector &inputs, @@ -787,7 +795,6 @@ private: }; bool hasDynamicShapes; bool shapesInitialized; - float multiplier; }; Ptr PoolingLayerInt8::create(const LayerParams& params) diff --git a/modules/dnn/src/layers/dequantizelinear_layer.cpp b/modules/dnn/src/layers/dequantizelinear_layer.cpp index 1981cfa471..4b2aceb16f 100644 --- a/modules/dnn/src/layers/dequantizelinear_layer.cpp +++ b/modules/dnn/src/layers/dequantizelinear_layer.cpp @@ -11,6 +11,42 @@ namespace cv namespace dnn { +#if CV_SIMD || CV_SIMD_SCALABLE +static void dequantizeLinearChunk_u8_f32(const uint8_t* src, float* dst, + float scale, int zp, int64_t len) +{ + const int vlanes = VTraits::vlanes(); + v_float32 vscale = vx_setall_f32(scale); + v_int32 vzp = vx_setall_s32(zp); + int64_t j = 0; + for (; j <= len - vlanes; j += vlanes) { + v_int32 vi = v_reinterpret_as_s32(vx_load_expand_q(src + j)); + vi = v_sub(vi, vzp); + v_float32 vf = v_mul(v_cvt_f32(vi), vscale); + v_store(dst + j, vf); + } + for (; j < len; j++) + dst[j] = (float)(src[j] - zp) * scale; +} + +static void dequantizeLinearFast_u8_f32(const uint8_t* inp, float* out, + float scale, int zp, + int64_t total) +{ + const int64_t block = 1024; + int64_t nblocks = (total + block - 1) / block; + + parallel_for_(Range(0, (int)nblocks), [&](const Range& r) { + for (int i = r.start; i < r.end; i++) { + int64_t ofs = i * block; + int64_t len = std::min(block, total - ofs); + dequantizeLinearChunk_u8_f32(inp + ofs, out + ofs, scale, zp, len); + } + }); +} +#endif + + /* DequantizeLinear layer, as defined in ONNX specification: https://onnx.ai/onnx/operators/onnx__DequantizeLinear.html @@ -165,6 +201,19 @@ static void dequantizeLinear(const Mat& inp, const Mat& scale_, const Mat& zp, } } + // Fast path: per-tensor dequantization uint8→float with universal intrinsics +#if CV_SIMD || CV_SIMD_SCALABLE + if (block_size == 0 && sz_a == 1 && inptype == CV_8U && outtype == CV_32F && sctype == CV_32F) { + float sc = reinterpret_cast(scale.data)[0]; + int zpval = zp.empty() ? 0 : (int)reinterpret_cast(zp.data)[0]; + int64_t total = nslices * slice_size; + dequantizeLinearFast_u8_f32(reinterpret_cast(inp.data), + reinterpret_cast(out.data), + sc, zpval, total); + return; + } +#endif + if (inptype == CV_8U && sctype == CV_32F && outtype == CV_32F) dequantizeLinear(reinterpret_cast(inp.data), reinterpret_cast(scale.data), diff --git a/modules/dnn/src/layers/nary_eltwise_layers.cpp b/modules/dnn/src/layers/nary_eltwise_layers.cpp index fc9cdffb4b..b3946da774 100644 --- a/modules/dnn/src/layers/nary_eltwise_layers.cpp +++ b/modules/dnn/src/layers/nary_eltwise_layers.cpp @@ -11,6 +11,7 @@ #include "../op_vkcom.hpp" #include +#include "opencv2/core/hal/intrin.hpp" #include #include @@ -468,13 +469,70 @@ public: } } + #if CV_SIMD + // Fast path: fully contiguous float Add → flatten + SIMD + parallel_for_ + bool is_add = (this->op == OPERATION::SUM || this->op == OPERATION::ADD); + if (is_add && std::is_same::value && std::is_same::value && + dp1 == 1 && dp2 == 1 && dp == 1 && ndims >= 1) { + bool contiguous = true; + for (int k = ndims - 2; k >= 0; k--) { + if (shape[k] <= 1) continue; // size-1 dims have stride 0, skip + size_t expected = (size_t)shape[k + 1] * step1[k + 1]; + if (step1[k] != expected || step2[k] != expected || step[k] != expected) { + contiguous = false; + break; + } + } + if (contiguous) { + int64_t total = (int64_t)nplanes * plane_size; + const float* p1 = (const float*)data1; + const float* p2 = (const float*)data2; + float* po = (float*)data; + const int64_t chunk = 1024; + int64_t nchunks = (total + chunk - 1) / chunk; + parallel_for_(Range(0, (int)nchunks), [&](const Range& r) { + for (int c = r.start; c < r.end; c++) { + int64_t start = c * chunk; + int64_t end = std::min(start + chunk, total); + int64_t i = start; + for (; i <= end - (int64_t)VTraits::nlanes * 4; i += VTraits::nlanes * 4) { + v_store(po + i, v_add(vx_load(p1 + i), vx_load(p2 + i))); + v_store(po + i + VTraits::nlanes, v_add(vx_load(p1 + i + VTraits::nlanes), vx_load(p2 + i + VTraits::nlanes))); + v_store(po + i + VTraits::nlanes*2, v_add(vx_load(p1 + i + VTraits::nlanes*2), vx_load(p2 + i + VTraits::nlanes*2))); + v_store(po + i + VTraits::nlanes*3, v_add(vx_load(p1 + i + VTraits::nlanes*3), vx_load(p2 + i + VTraits::nlanes*3))); + } + for (; i < end; i++) + po[i] = p1[i] + p2[i]; + } + }); + return; + } + } + #endif + if (nplanes == 1) { // parallelize within the plane const T* ptr1 = (const T*)data1; const T* ptr2 = (const T*)data2; RESULT_T* ptr = (RESULT_T*)data; auto worker = [&](const Range &r) { if (dp1 == 1 && dp2 == 1 && dp == 1) { - for(int i = r.start; i < r.end; i++) { + int i = r.start; + #if CV_SIMD + if (is_add && std::is_same::value && std::is_same::value) { + const float* p1 = (const float*)(const void*)&ptr1[r.start]; + const float* p2 = (const float*)(const void*)&ptr2[r.start]; + float* po = (float*)(void*)&ptr[r.start]; + int len = r.end - r.start, j = 0; + for (; j <= len - VTraits::nlanes * 4; j += VTraits::nlanes * 4) { + v_store(po + j, v_add(vx_load(p1 + j), vx_load(p2 + j))); + v_store(po + j + VTraits::nlanes, v_add(vx_load(p1 + j + VTraits::nlanes), vx_load(p2 + j + VTraits::nlanes))); + v_store(po + j + VTraits::nlanes*2, v_add(vx_load(p1 + j + VTraits::nlanes*2), vx_load(p2 + j + VTraits::nlanes*2))); + v_store(po + j + VTraits::nlanes*3, v_add(vx_load(p1 + j + VTraits::nlanes*3), vx_load(p2 + j + VTraits::nlanes*3))); + } + i = r.start + j; + } + #endif + for(; i < r.end; i++) { ptr[i] = op(ptr1[i], ptr2[i]); } } else if (dp1 == 1 && dp2 == 0 && dp == 1){ @@ -516,7 +574,21 @@ public: const T* ptr2 = (const T*)ptr2_; RESULT_T* ptr = (RESULT_T*)ptr_; if (dp1 == 1 && dp2 == 1 && dp == 1) { - for(int i = 0; i < plane_size; i++) { + int i = 0; + #if CV_SIMD + if (is_add && std::is_same::value && std::is_same::value) { + const float* p1 = (const float*)(const void*)ptr1; + const float* p2 = (const float*)(const void*)ptr2; + float* po = (float*)(void*)ptr; + for (; i <= plane_size - VTraits::nlanes * 4; i += VTraits::nlanes * 4) { + v_store(po + i, v_add(vx_load(p1 + i), vx_load(p2 + i))); + v_store(po + i + VTraits::nlanes, v_add(vx_load(p1 + i + VTraits::nlanes), vx_load(p2 + i + VTraits::nlanes))); + v_store(po + i + VTraits::nlanes*2, v_add(vx_load(p1 + i + VTraits::nlanes*2), vx_load(p2 + i + VTraits::nlanes*2))); + v_store(po + i + VTraits::nlanes*3, v_add(vx_load(p1 + i + VTraits::nlanes*3), vx_load(p2 + i + VTraits::nlanes*3))); + } + } + #endif + for(; i < plane_size; i++) { ptr[i] = op(ptr1[i], ptr2[i]); } } else if (dp1 == 1 && dp2 == 0 && dp == 1){ diff --git a/modules/dnn/src/layers/quantlizelinear_layer.cpp b/modules/dnn/src/layers/quantlizelinear_layer.cpp index 429da2fdf3..99c7af4468 100644 --- a/modules/dnn/src/layers/quantlizelinear_layer.cpp +++ b/modules/dnn/src/layers/quantlizelinear_layer.cpp @@ -7,11 +7,63 @@ #include "layers_common.hpp" #include "../net_impl.hpp" +#if defined(__x86_64__) || defined(_M_X64) +#include +#endif + namespace cv { namespace dnn { +// Fast path for per-tensor quantization: float → uint8 with AVX2 +#if defined(__x86_64__) || defined(_M_X64) +#if defined(__GNUC__) || defined(__clang__) +__attribute__((target("avx2"))) +#endif +static void quantizeLinearChunk_f32_u8_avx2(const float* src, uint8_t* dst, + float inv_scale, float zp_f, + int64_t len) +{ + __m256 vscale = _mm256_set1_ps(inv_scale); + __m256 vzp = _mm256_set1_ps(zp_f); + __m256 vmin = _mm256_setzero_ps(); + __m256 vmax = _mm256_set1_ps(255.f); + + int64_t j = 0; + for (; j <= len - 8; j += 8) { + __m256 v = _mm256_loadu_ps(src + j); + v = _mm256_add_ps(_mm256_mul_ps(v, vscale), vzp); + v = _mm256_min_ps(_mm256_max_ps(v, vmin), vmax); + __m256i vi = _mm256_cvtps_epi32(v); + __m128i lo = _mm256_castsi256_si128(vi); + __m128i hi = _mm256_extracti128_si256(vi, 1); + __m128i packed16 = _mm_packs_epi32(lo, hi); + __m128i packed8 = _mm_packus_epi16(packed16, packed16); + _mm_storel_epi64((__m128i*)(dst + j), packed8); + } + for (; j < len; j++) + dst[j] = saturate_cast(src[j] * inv_scale + zp_f); +} + +static void quantizeLinearFast_f32_u8_avx2(const float* inp, uint8_t* out, + float inv_scale, float zp_f, + int64_t total) +{ + const int64_t block = 1024; + int64_t nblocks = (total + block - 1) / block; + + parallel_for_(Range(0, (int)nblocks), [&](const Range& r) { + for (int i = r.start; i < r.end; i++) { + int64_t ofs = i * block; + int64_t len = std::min(block, total - ofs); + quantizeLinearChunk_f32_u8_avx2(inp + ofs, out + ofs, inv_scale, zp_f, len); + } + }); +} +#endif + + /* QuantizeLinear layer, as defined in ONNX specification: https://onnx.ai/onnx/operators/onnx__QuantizeLinear.html @@ -172,6 +224,20 @@ static void quantizeLinear(const Mat& inp, const Mat& scale_, const Mat& zp, } } + // Fast path: per-tensor quantization float→uint8 with AVX2 + proper parallelism +#if defined(__x86_64__) || defined(_M_X64) + if (block_size == 0 && sz_a == 1 && inptype == CV_32F && outtype == CV_8U && sctype == CV_32F + && checkHardwareSupport(CV_CPU_AVX2)) { + float inv_scale = 1.f / reinterpret_cast(scale.data)[0]; + float zp_f = zp.empty() ? 0.f : (float)reinterpret_cast(zp.data)[0]; + int64_t total = nslices * slice_size; + quantizeLinearFast_f32_u8_avx2(reinterpret_cast(inp.data), + reinterpret_cast(out.data), + inv_scale, zp_f, total); + return; + } +#endif + if (outtype == CV_8U && sctype == CV_32F && inptype == CV_32F) quantizeLinear(reinterpret_cast(inp.data), reinterpret_cast(scale.data), diff --git a/modules/dnn/src/net_impl.hpp b/modules/dnn/src/net_impl.hpp index 6866b0301d..d4bab924a9 100644 --- a/modules/dnn/src/net_impl.hpp +++ b/modules/dnn/src/net_impl.hpp @@ -450,6 +450,8 @@ struct Net::Impl : public detail::NetImplBase // fuse batch norm, add bias and activation to convolution void fuseBasic(); // replace constant sub-expressions with their results + + void fuseQDQ(); void constFold(); // make some operations (activation, batch norm, convolution) unary if // all their arguments except for the 1st one are constant. diff --git a/modules/dnn/src/net_impl2.cpp b/modules/dnn/src/net_impl2.cpp index 7860bf6dc3..4e4b5d83c3 100644 --- a/modules/dnn/src/net_impl2.cpp +++ b/modules/dnn/src/net_impl2.cpp @@ -497,6 +497,7 @@ void Net::Impl::prepareForInference() #endif if (!prepared) { + fuseQDQ(); constFold(); constArgs(); useBlockLayout();