diff --git a/modules/dnn/CMakeLists.txt b/modules/dnn/CMakeLists.txt index 2365e77173..3262b0135c 100644 --- a/modules/dnn/CMakeLists.txt +++ b/modules/dnn/CMakeLists.txt @@ -12,6 +12,7 @@ ocv_add_dispatched_file("layers/cpu_kernels/conv_winograd_f63" AVX AVX2 NEON NEO ocv_add_dispatched_file_force_all("layers/cpu_kernels/fast_gemm_kernels" AVX AVX2 NEON LASX) ocv_add_dispatched_file("layers/cpu_kernels/conv2_depthwise" AVX AVX2 NEON NEON_FP16) ocv_add_dispatched_file("layers/cpu_kernels/conv2_kernels" AVX AVX2 NEON NEON_FP16) +ocv_add_dispatched_file_force_all("int8layers/conv2_int8_kernels" AVX2) ocv_add_module(dnn opencv_core opencv_imgproc WRAP python java objc js) diff --git a/modules/dnn/include/opencv2/dnn/all_layers.hpp b/modules/dnn/include/opencv2/dnn/all_layers.hpp index a01ffc2670..7e00e2c0b7 100644 --- a/modules/dnn/include/opencv2/dnn/all_layers.hpp +++ b/modules/dnn/include/opencv2/dnn/all_layers.hpp @@ -382,6 +382,21 @@ CV__DNN_INLINE_NS_BEGIN bool ceil_mode; }; + class CV_EXPORTS Conv2Int8Layer : public Layer + { + public: + static Ptr create(const LayerParams& params); + + int input_zp, output_zp; + float input_sc, output_sc; + bool per_channel; + + std::vector strides, dilations, pads; + int ngroups; + AutoPadding auto_pad; + bool ceil_mode; + }; + class CV_EXPORTS LRNLayer : public Layer { public: @@ -461,6 +476,9 @@ CV__DNN_INLINE_NS_BEGIN static Ptr create(const LayerParams& params); }; + // Old-engine int8 pooling. Created directly by the ONNX importer for + // QLinearAveragePool / QLinearGlobalAveragePool / int8 MaxPool ops. + // Inherits PoolingLayer so it can delegate to TIMVX / NGRAPH backends. class CV_EXPORTS PoolingLayerInt8 : public PoolingLayer { public: @@ -469,6 +487,25 @@ CV__DNN_INLINE_NS_BEGIN static Ptr create(const LayerParams& params); }; + // New-engine int8 pooling with block memory layout (DATA_LAYOUT_BLOCK). + // Created by the QDQ graph fusion pass (graph_fusion_qdq.cpp) when it + // detects a DequantizeLinear -> Pooling -> QuantizeLinear pattern. + // Uses optimised SIMD kernels; OPENCV (CPU) backend only. + class CV_EXPORTS Pool2Int8Layer : public Layer + { + public: + static Ptr create(const LayerParams& params); + + int input_zp, output_zp; + float input_sc, output_sc; + + std::vector kernel_shape, strides, dilations, pads; + AutoPadding auto_pad; + bool ceil_mode; + bool is_global_pooling; + bool is_max_pool; + }; + class CV_EXPORTS AveragePoolLayer : public Layer { public: @@ -559,6 +596,7 @@ CV__DNN_INLINE_NS_BEGIN public: int input_zp, output_zp; float input_sc, output_sc; + int output_type; // CV_8S or CV_8U // quantization type flag. The perChannel default is true, that means it contains the parameters // of per-Channel quantization. Otherwise, that means this layer contains per-Tensor quantized parameters. @@ -1214,6 +1252,17 @@ CV__DNN_INLINE_NS_BEGIN static Ptr create(const LayerParams ¶ms); }; + class CV_EXPORTS Eltwise2Int8Layer : public Layer + { + public: + static Ptr create(const LayerParams& params); + + std::vector scales; + std::vector zeropoints; + float output_sc; + int output_zp; + }; + class CV_EXPORTS NaryEltwiseLayer : public Layer { public: diff --git a/modules/dnn/src/graph_fusion_qdq.cpp b/modules/dnn/src/graph_fusion_qdq.cpp index 2d0f62f7ed..a22e6d1608 100644 --- a/modules/dnn/src/graph_fusion_qdq.cpp +++ b/modules/dnn/src/graph_fusion_qdq.cpp @@ -51,7 +51,31 @@ struct ModelFusionQDQ Ptr createFusedLayer(const LayerParams& src) const { LayerParams params = src; - return LayerFactory::createLayerInstance(params.type, params); + Ptr layer = LayerFactory::createLayerInstance(params.type, params); + if (!layer.empty()) + layer->netimpl = netimpl; + return layer; + } + + static bool getInt8OutputParams(Layer* layer, float& sc, int& zp) + { + if (auto* p = dynamic_cast(layer)) + { sc = p->output_sc; zp = p->output_zp; return true; } + if (auto* p = dynamic_cast(layer)) + { sc = p->output_sc; zp = p->output_zp; return true; } + if (auto* p = dynamic_cast(layer)) + { sc = p->output_sc; zp = p->output_zp; return true; } + return false; + } + + static void setInt8OutputParams(Layer* layer, float sc, int zp) + { + if (auto* p = dynamic_cast(layer)) + { p->output_sc = sc; p->output_zp = zp; } + else if (auto* p = dynamic_cast(layer)) + { p->output_sc = sc; p->output_zp = zp; } + else if (auto* p = dynamic_cast(layer)) + { p->output_sc = sc; p->output_zp = zp; } } template @@ -86,8 +110,12 @@ struct ModelFusionQDQ std::vector producer_of(nargs, -1); std::vector > newprog; std::vector fused_inputs; + std::set skip_indices; + std::vector override_outputs; + std::map> relu_to_eltwise; for (i = 0; i < nops; i++) { + if (skip_indices.count((int)i)) continue; const Ptr& layer = prog[i]; Layer* layer_ptr = (Layer*)layer.get(); int fused_layer_idx = -1; @@ -102,7 +130,8 @@ struct ModelFusionQDQ const std::vector& outputs = layer->outputs; size_t ninputs = inputs.size(); removed_args.clear(); - fused_inputs.clear(); // leave it empty to re-use original fused node inputs as-is. + fused_inputs.clear(); + override_outputs.clear(); for(;;) { Arg q_data_in, out_scale, out_zp; @@ -128,13 +157,12 @@ struct ModelFusionQDQ } dq_ptrs.push_back(dq); dq_prog_indices.push_back(dq_idx); - int8_inputs.push_back(dq->inputs[0]); // int8 quantized tensor + int8_inputs.push_back(dq->inputs[0]); } const int eltwise_out_type = !outputs.empty() ? netimpl->argData(outputs[0]).type : -1; const bool eltwise_out_int8 = (eltwise_out_type == CV_8S || eltwise_out_type == CV_8U); if (dq_ptrs.size() == add->inputs.size() && eltwise_out_int8) { - // Read per-input scale and zero-point from the DQ const args. vector in_scales(2); vector in_zps(2); bool eltwise_in_int8 = true; @@ -156,16 +184,15 @@ struct ModelFusionQDQ ? (int)elt_out_zp_m.at(0) : (int)elt_out_zp_m.at(0); - LayerParams eltwiseParams = makeLayerParamsFromOriginal(add, "EltwiseInt8"); + LayerParams eltwiseParams = makeLayerParamsFromOriginal(add, "Eltwise2Int8"); eltwiseParams.blobs.clear(); + eltwiseParams.set("input_scales", DictValue::arrayReal(in_scales.data(), (int)in_scales.size())); + eltwiseParams.set("input_zeropoints", DictValue::arrayInt(in_zps.data(), (int)in_zps.size())); + eltwiseParams.set("scales", out_scale_val); + eltwiseParams.set("zeropoints", out_zp_val); Ptr eltwiseInt8 = createFusedLayer(eltwiseParams); if (!eltwiseInt8.empty()) { - auto* elt = dynamic_cast(eltwiseInt8.get()); - CV_Assert(elt); - elt->scales = in_scales; - elt->zeropoints = in_zps; - elt->output_sc = out_scale_val; - elt->output_zp = out_zp_val; + CV_Assert(dynamic_cast(eltwiseInt8.get())); fused_layer_idx = add_layer_idx; newprog[add_layer_idx] = eltwiseInt8; fused_inputs.swap(int8_inputs); @@ -185,114 +212,124 @@ struct ModelFusionQDQ ReLULayer* relu = 0; if (getQdqPatternContext(layer_ptr, ninputs, inputs, producer_of, newprog, q_data_in, out_scale, out_zp, - relu_layer_idx, relu) && - relu->inputs.size() == 1) { - Arg relu_in = relu->inputs[0]; - int dq_idx = producer_of.at(relu_in.idx); - DequantizeLinearLayer* dq = getLayer(newprog, dq_idx); + relu_layer_idx, relu) && relu->inputs.size() == 1) { + Arg relu_in = relu->inputs[0]; + int dq_idx = producer_of.at(relu_in.idx); + DequantizeLinearLayer* dq = getLayer(newprog, dq_idx); - const int relu_out_type = !outputs.empty() ? netimpl->argData(outputs[0]).type : -1; - const bool relu_out_int8 = (relu_out_type == CV_8S || relu_out_type == CV_8U); - const int relu_in_type = (dq && !dq->inputs.empty()) ? netimpl->argData(dq->inputs[0]).type : -1; - const bool relu_in_int8 = (relu_in_type == CV_8S || relu_in_type == CV_8U); - if (dq && dq->inputs.size() >= 3 && - relu_in_int8 && relu_out_int8 && - usecounts.at(relu_in.idx) == 1) { - const float inp_sc = netimpl->argTensor(dq->inputs[1]).at(0); - const Mat& relu_zp_m = netimpl->argTensor(dq->inputs[2]); - const int inp_zp = relu_zp_m.depth() == CV_8U - ? (int)relu_zp_m.at(0) - : (int)relu_zp_m.at(0); - const float out_sc = netimpl->argTensor(out_scale).at(0); - const Mat& out_zp_relu_m = netimpl->argTensor(out_zp); - const int out_zp_i = out_zp_relu_m.depth() == CV_8U - ? (int)out_zp_relu_m.at(0) - : (int)out_zp_relu_m.at(0); + const int relu_out_type = !outputs.empty() ? netimpl->argData(outputs[0]).type : -1; + const bool relu_out_int8 = (relu_out_type == CV_8S || relu_out_type == CV_8U); + const int relu_in_type = (dq && !dq->inputs.empty()) ? netimpl->argData(dq->inputs[0]).type : -1; + const bool relu_in_int8 = (relu_in_type == CV_8S || relu_in_type == CV_8U); + if (dq && dq->inputs.size() >= 3 && + relu_in_int8 && relu_out_int8 && + usecounts.at(relu_in.idx) == 1) { + const float inp_sc = netimpl->argTensor(dq->inputs[1]).at(0); + const Mat& relu_zp_m = netimpl->argTensor(dq->inputs[2]); + const int inp_zp = relu_zp_m.depth() == CV_8U + ? (int)relu_zp_m.at(0) + : (int)relu_zp_m.at(0); + const float out_sc = netimpl->argTensor(out_scale).at(0); + const Mat& out_zp_relu_m = netimpl->argTensor(out_zp); + const int out_zp_i = out_zp_relu_m.depth() == CV_8U + ? (int)out_zp_relu_m.at(0) + : (int)out_zp_relu_m.at(0); - if (inp_sc > 0.f && out_sc > 0.f) { - const bool isU8 = (relu_in_type == CV_8U); - Mat lookUpTable(1, 256, isU8 ? CV_8U : CV_8S); - if (isU8) { - uint8_t* table = lookUpTable.ptr(); - for (int t = 0; t < 256; t++) { - float x = inp_sc * (t - inp_zp); - float y = std::max(0.0f, x); - int quantized = out_zp_i + cvRound(y / out_sc); - table[t] = saturate_cast(quantized); - } - } else { - int8_t* table = lookUpTable.ptr(); - for (int t = -128; t < 128; t++) { - float x = inp_sc * (t - inp_zp); - float y = std::max(0.0f, x); - int quantized = out_zp_i + cvRound(y / out_sc); - table[t + 128] = saturate_cast(quantized); - } + if (inp_sc > 0.f && out_sc > 0.f) { + const bool isU8 = (relu_in_type == CV_8U); + Mat lookUpTable(1, 256, isU8 ? CV_8U : CV_8S); + if (isU8) { + uint8_t* table = lookUpTable.ptr(); + for (int t = 0; t < 256; t++) { + float x = inp_sc * (t - inp_zp); + float y = std::max(0.0f, x); + int quantized = out_zp_i + cvRound(y / out_sc); + table[t] = saturate_cast(quantized); } - - LayerParams reluInt8Params = makeLayerParamsFromOriginal(relu, "ReLUInt8"); - reluInt8Params.blobs.clear(); - Ptr reluInt8 = createFusedLayer(reluInt8Params); - if (!reluInt8.empty()) { - auto* reluInt8Layer = dynamic_cast(reluInt8.get()); - CV_Assert(reluInt8Layer); - reluInt8Layer->input_sc = inp_sc; - reluInt8Layer->input_zp = inp_zp; - reluInt8Layer->output_sc = out_sc; - reluInt8Layer->output_zp = out_zp_i; - reluInt8Layer->activationLUT = lookUpTable; - fused_layer_idx = relu_layer_idx; - newprog[relu_layer_idx] = reluInt8; - fused_inputs.assign(1, dq->inputs[0]); - removed_args.push_back(q_data_in); - removed_args.push_back(relu_in); - newprog[dq_idx] = Ptr(); - break; + } else { + int8_t* table = lookUpTable.ptr(); + for (int t = -128; t < 128; t++) { + float x = inp_sc * (t - inp_zp); + float y = std::max(0.0f, x); + int quantized = out_zp_i + cvRound(y / out_sc); + table[t + 128] = saturate_cast(quantized); } } + + LayerParams reluInt8Params = makeLayerParamsFromOriginal(relu, "ReLUInt8"); + reluInt8Params.blobs.clear(); + Ptr reluInt8 = createFusedLayer(reluInt8Params); + if (!reluInt8.empty()) { + auto* reluInt8Layer = dynamic_cast(reluInt8.get()); + CV_Assert(reluInt8Layer); + reluInt8Layer->input_sc = inp_sc; + reluInt8Layer->input_zp = inp_zp; + reluInt8Layer->output_sc = out_sc; + reluInt8Layer->output_zp = out_zp_i; + reluInt8Layer->activationLUT = lookUpTable; + fused_layer_idx = relu_layer_idx; + newprog[relu_layer_idx] = reluInt8; + fused_inputs.assign(1, dq->inputs[0]); + removed_args.push_back(q_data_in); + removed_args.push_back(relu_in); + newprog[dq_idx] = Ptr(); + break; + } } } + } - // Compound pattern: DQ, DQ -> Add -> ReLU -> QuantizeLinear - // Common in ResNet residual blocks. Fuses into EltwiseInt8 with activation LUT. - { - int relu_layer_idx2 = -1; - ReLULayer* relu2 = 0; - Arg q_data_in2, out_scale2, out_zp2; - if (getQdqPatternContext(layer_ptr, ninputs, inputs, producer_of, - newprog, q_data_in2, out_scale2, out_zp2, - relu_layer_idx2, relu2) && - relu2->inputs.size() == 1) { + QuantizeLinearLayer* ql_compound = dynamic_cast(layer_ptr); + if (ql_compound && ninputs == 3) { + Arg q_inp = inputs[0]; + int relu_layer_idx2 = (q_inp.idx < (int)producer_of.size()) ? producer_of.at(q_inp.idx) : -1; + ReLULayer* relu2 = getLayer(newprog, relu_layer_idx2); + (void)relu_layer_idx2; + if (relu2 && relu2->inputs.size() == 1) { Arg relu_in2 = relu2->inputs[0]; int add_idx2 = producer_of.at(relu_in2.idx); NaryEltwiseLayer* add2 = getLayer(newprog, add_idx2); + int relu_in_uc = usecounts.at(relu_in2.idx); if (add2 && add2->inputs.size() >= 2 && - usecounts.at(relu_in2.idx) == 1) { + relu_in_uc == 1) { vector dq_ptrs2; vector dq_prog_indices2; vector int8_inputs2; for (size_t k = 0; k < add2->inputs.size(); k++) { const Arg& add_inp = add2->inputs[k]; int dq_idx2 = producer_of.at(add_inp.idx); - DequantizeLinearLayer* dq2 = - getLayer(newprog, dq_idx2); - if (dq2 && dq2->inputs.size() >= 3 && - usecounts.at(add_inp.idx) == 1) { + DequantizeLinearLayer* dq2 = getLayer(newprog, dq_idx2); + + if (dq2 && dq2->inputs.size() >= 3) { dq_ptrs2.push_back(dq2); dq_prog_indices2.push_back(dq_idx2); int8_inputs2.push_back(dq2->inputs[0]); } else { int arg_type = netimpl->argData(add_inp).type; - if (arg_type == CV_8S || arg_type == CV_8U) { + bool found_int8 = (arg_type == CV_8S || arg_type == CV_8U); + Arg int8_arg = add_inp; + if (!found_int8) { + auto it = relu_to_eltwise.find(add_inp.idx); + if (it != relu_to_eltwise.end()) { + int8_arg = it->second.first; + found_int8 = true; + } + } + if (found_int8) { dq_ptrs2.push_back(nullptr); dq_prog_indices2.push_back(-1); - int8_inputs2.push_back(add_inp); + int8_inputs2.push_back(int8_arg); } else { - break; // not int8, can't fuse + break; } } } - const int elt_out_type2 = !outputs.empty() ? netimpl->argData(outputs[0]).type : -1; + + int elt_out_type2 = !outputs.empty() ? netimpl->argData(outputs[0]).type : -1; + if (elt_out_type2 < 0) { + const Mat& zp_tensor = netimpl->argTensor(inputs[2]); + elt_out_type2 = !zp_tensor.empty() ? zp_tensor.type() : CV_8S; + } const bool elt_out_int82 = (elt_out_type2 == CV_8S || elt_out_type2 == CV_8U); if (int8_inputs2.size() == add2->inputs.size() && elt_out_int82) { vector in_scales2(2); @@ -301,6 +338,10 @@ struct ModelFusionQDQ for (int k = 0; k < 2; k++) { if (dq_ptrs2[k]) { int it = netimpl->argData(dq_ptrs2[k]->inputs[0]).type; + if (it < 0) { + const Mat& zp_infer = netimpl->argTensor(dq_ptrs2[k]->inputs[2]); + it = !zp_infer.empty() ? zp_infer.type() : CV_8S; + } elt_in_int82 = elt_in_int82 && (it == CV_8S || it == CV_8U); in_scales2[k] = netimpl->argTensor(dq_ptrs2[k]->inputs[1]).at(0); const Mat& zp_m2 = netimpl->argTensor(dq_ptrs2[k]->inputs[2]); @@ -308,80 +349,62 @@ struct ModelFusionQDQ ? (int)zp_m2.at(0) : (int)zp_m2.at(0); } else { + auto relu_it = relu_to_eltwise.find(add2->inputs[k].idx); + if (relu_it != relu_to_eltwise.end()) { + Eltwise2Int8Layer* elt2 = relu_it->second.second; + if (elt2) { + in_scales2[k] = elt2->output_sc; + in_zps2[k] = elt2->output_zp; + continue; + } + } int prod_idx = producer_of.at(add2->inputs[k].idx); Layer* prod = prod_idx >= 0 && !newprog[prod_idx].empty() ? newprog[prod_idx].get() : nullptr; - ConvolutionLayerInt8* ci = prod ? dynamic_cast(prod) : nullptr; - EltwiseLayerInt8* ei = prod ? dynamic_cast(prod) : nullptr; - InnerProductLayerInt8* fi = prod ? dynamic_cast(prod) : nullptr; - if (ci) { in_scales2[k] = ci->output_sc; in_zps2[k] = ci->output_zp; } - else if (ei) { in_scales2[k] = ei->output_sc; in_zps2[k] = ei->output_zp; } - else if (fi) { in_scales2[k] = fi->output_sc; in_zps2[k] = fi->output_zp; } - else { elt_in_int82 = false; } + if (!prod || !getInt8OutputParams(prod, in_scales2[k], in_zps2[k])) + elt_in_int82 = false; } } if (elt_in_int82) { - float out_sc2 = netimpl->argTensor(out_scale2).at(0); - const Mat& out_zp_m2 = netimpl->argTensor(out_zp2); + float out_sc2 = netimpl->argTensor(inputs[1]).at(0); + const Mat& out_zp_m2 = netimpl->argTensor(inputs[2]); int out_zp_val2 = out_zp_m2.depth() == CV_8U ? (int)out_zp_m2.at(0) : (int)out_zp_m2.at(0); if (out_sc2 > 0.f) { - LayerParams eltParams = makeLayerParamsFromOriginal(add2, "EltwiseInt8"); + int relu_out_uc = usecounts.at(q_inp.idx); + LayerParams eltParams = makeLayerParamsFromOriginal(add2, "Eltwise2Int8"); eltParams.blobs.clear(); + eltParams.set("input_scales", DictValue::arrayReal(in_scales2.data(), (int)in_scales2.size())); + eltParams.set("input_zeropoints", DictValue::arrayInt(in_zps2.data(), (int)in_zps2.size())); + eltParams.set("scales", out_sc2); + eltParams.set("zeropoints", out_zp_val2); + eltParams.set("with_relu", true); Ptr eltInt8 = createFusedLayer(eltParams); if (!eltInt8.empty()) { - auto* elt = dynamic_cast(eltInt8.get()); - CV_Assert(elt); - elt->scales = in_scales2; - elt->zeropoints = in_zps2; - elt->output_sc = out_sc2; - elt->output_zp = out_zp_val2; + CV_Assert(dynamic_cast(eltInt8.get())); - const bool isU8 = (elt_out_type2 == CV_8U); - Mat lut(1, 256, isU8 ? CV_8U : CV_8S); - if (isU8) { - uint8_t* tbl = lut.ptr(); - for (int t = 0; t < 256; t++) { - float x = out_sc2 * (t - out_zp_val2); - float y = std::max(0.0f, x); - tbl[t] = saturate_cast(out_zp_val2 + cvRound(y / out_sc2)); + fused_inputs = int8_inputs2; + if (relu_out_uc <= 1) { + fused_layer_idx = add_idx2; + newprog[add_idx2] = eltInt8; + newprog[relu_layer_idx2] = Ptr(); + removed_args.push_back(q_inp); // relu_out + removed_args.push_back(relu_in2); // add_out + for (size_t dk = 0; dk < add2->inputs.size(); dk++) { + removed_args.push_back(add2->inputs[dk]); + } + for (int dq_prog_idx : dq_prog_indices2) { + if (dq_prog_idx >= 0) + newprog[dq_prog_idx] = Ptr(); } } else { - int8_t* tbl = lut.ptr(); - for (int t = -128; t < 128; t++) { - float x = out_sc2 * (t - out_zp_val2); - float y = std::max(0.0f, x); - tbl[t + 128] = saturate_cast(out_zp_val2 + cvRound(y / out_sc2)); - } - } - LayerParams reluActParams; - reluActParams.name = relu2->name; - reluActParams.type = "ReLUInt8"; - Ptr reluAct = createFusedLayer(reluActParams); - if (!reluAct.empty()) { - auto* reluActLayer = dynamic_cast(reluAct.get()); - if (reluActLayer) { - reluActLayer->input_sc = out_sc2; - reluActLayer->input_zp = out_zp_val2; - reluActLayer->output_sc = out_sc2; - reluActLayer->output_zp = out_zp_val2; - reluActLayer->activationLUT = lut; - eltInt8->setActivation(reluAct.dynamicCast()); - } - } - - fused_layer_idx = add_idx2; - newprog[add_idx2] = eltInt8; - newprog[relu_layer_idx2] = Ptr(); - fused_inputs.swap(int8_inputs2); - removed_args.push_back(q_data_in2); - removed_args.push_back(relu_in2); - for (const Arg& add_inp : add2->inputs) - removed_args.push_back(add_inp); - for (int dq_prog_idx : dq_prog_indices2) { - if (dq_prog_idx >= 0) - newprog[dq_prog_idx] = Ptr(); + int new_idx = (int)newprog.size(); + newprog.push_back(eltInt8); + fused_layer_idx = new_idx; + usecounts.at(q_inp.idx) -= 1; + Eltwise2Int8Layer* elt2ptr = dynamic_cast(eltInt8.get()); + relu_to_eltwise[q_inp.idx] = {outputs[0], elt2ptr}; } break; } @@ -406,8 +429,6 @@ struct ModelFusionQDQ DequantizeLinearLayer* dq_x = getLayer(newprog, dq_x_idx); DequantizeLinearLayer* dq_w = getLayer(newprog, dq_w_idx); - // Allow usecounts > 1 for conv input (shared DQ output at stage transitions) - // The int8 data (DQ's input[0]) can be shared safely. if (dq_x && dq_w && dq_x->inputs.size() >= 3 && dq_w->inputs.size() >= 3 && usecounts.at(conv_w.idx) == 1) { @@ -433,133 +454,129 @@ struct ModelFusionQDQ Mat w_zp_m = netimpl->argTensor(dq_w->inputs[2]); const Mat& x_zp_m = netimpl->argTensor(dq_x->inputs[2]); if (!w_q.empty() && w_q.depth() == CV_8S && w_q.dims >= 3) { - const int outCn = w_q.size[0]; - Mat wt_sc = (w_sc_m.total() == (size_t)outCn) - ? w_sc_m.reshape(1, 1) - : Mat(1, outCn, CV_32F, Scalar(w_sc_m.at(0))).clone(); - bool per_channel = w_sc_m.total() == (size_t)outCn; + const int outCn = w_q.size[0]; + Mat wt_sc = (w_sc_m.total() == (size_t)outCn) + ? w_sc_m.reshape(1, 1) + : Mat(1, outCn, CV_32F, Scalar(w_sc_m.at(0))).clone(); + bool per_channel = w_sc_m.total() == (size_t)outCn; - bool all_wzp_zero = true; - if (w_zp_m.total() > 1 && w_zp_m.total() != (size_t)outCn) - all_wzp_zero = false; - for (size_t t = 0; all_wzp_zero && t < w_zp_m.total(); t++) { - int wz = w_zp_m.depth() == CV_8S ? (int)w_zp_m.at((int)t) - : (int)w_zp_m.at((int)t); - if (wz != 0) all_wzp_zero = false; - } + bool all_wzp_zero = true; + if (w_zp_m.total() > 1 && w_zp_m.total() != (size_t)outCn) + all_wzp_zero = false; + for (size_t t = 0; all_wzp_zero && t < w_zp_m.total(); t++) { + int wz = w_zp_m.depth() == CV_8S ? (int)w_zp_m.at((int)t) + : (int)w_zp_m.at((int)t); + if (wz != 0) all_wzp_zero = false; + } - bool symmetric_pads = true; - size_t npads = conv->pads.size(); - size_t ndims_pad = npads / 2; - for (size_t d = 0; d < ndims_pad && symmetric_pads; d++) { - if (conv->pads[d] != conv->pads[d + ndims_pad]) - symmetric_pads = false; - } + bool symmetric_pads = true; + size_t npads = conv->pads.size(); + size_t ndims_pad = npads / 2; + for (size_t d = 0; d < ndims_pad && symmetric_pads; d++) { + if (conv->pads[d] != conv->pads[d + ndims_pad]) + symmetric_pads = false; + } - if (all_wzp_zero && symmetric_pads) { - Mat bias = Mat::zeros(1, outCn, CV_32S); - bool biasOk = true; - int dq_bias_idx = -1; - if (conv->inputs.size() == 3) { - if (netimpl->isConstArg(conv->inputs[2])) { - Mat b = netimpl->argTensor(conv->inputs[2]); - if (b.empty() || b.total() != (size_t)outCn) { - biasOk = false; - } else if (b.depth() == CV_32S) { - bias = b.reshape(1, 1); - } else if (b.depth() == CV_32F || b.depth() == CV_64F) { - Mat b1 = b.reshape(1, 1); - for (int oc = 0; oc < outCn; oc++) { - const float b_real = b1.depth() == CV_32F - ? b1.at(oc) - : (float)b1.at(oc); - const float denom = inp_sc * wt_sc.at(oc); - if (std::abs(denom) < 1e-12f) { biasOk = false; break; } - bias.at(oc) = cvRound(b_real / denom); - } - } else { - biasOk = false; + if (all_wzp_zero && symmetric_pads) { + Mat bias = Mat::zeros(1, outCn, CV_32S); + bool biasOk = true; + int dq_bias_idx = -1; + if (conv->inputs.size() == 3) { + if (netimpl->isConstArg(conv->inputs[2])) { + Mat b = netimpl->argTensor(conv->inputs[2]); + if (b.empty() || b.total() != (size_t)outCn) { + biasOk = false; + } else if (b.depth() == CV_32S) { + bias = b.reshape(1, 1); + } else if (b.depth() == CV_32F || b.depth() == CV_64F) { + Mat b1 = b.reshape(1, 1); + for (int oc = 0; oc < outCn; oc++) { + const float b_real = b1.depth() == CV_32F + ? b1.at(oc) + : (float)b1.at(oc); + const float denom = inp_sc * wt_sc.at(oc); + if (std::abs(denom) < 1e-12f) { biasOk = false; break; } + bias.at(oc) = cvRound(b_real / denom); } } else { - dq_bias_idx = producer_of.at(conv->inputs[2].idx); - DequantizeLinearLayer* dq_b = - getLayer(newprog, dq_bias_idx); - if (!dq_b || dq_b->inputs.size() < 2 || - usecounts.at(conv->inputs[2].idx) != 1 || - !netimpl->isConstArg(dq_b->inputs[0])) { + biasOk = false; + } + } else { + dq_bias_idx = producer_of.at(conv->inputs[2].idx); + DequantizeLinearLayer* dq_b = + getLayer(newprog, dq_bias_idx); + if (!dq_b || dq_b->inputs.size() < 2 || + usecounts.at(conv->inputs[2].idx) != 1 || + !netimpl->isConstArg(dq_b->inputs[0])) { + biasOk = false; + } else { + Mat bq = netimpl->argTensor(dq_b->inputs[0]); + if (bq.empty() || bq.total() != (size_t)outCn || bq.depth() != CV_32S) biasOk = false; - } else { - Mat bq = netimpl->argTensor(dq_b->inputs[0]); - if (bq.empty() || bq.total() != (size_t)outCn || bq.depth() != CV_32S) - biasOk = false; - else - bias = bq.reshape(1, 1); - } + else + bias = bq.reshape(1, 1); } } - if (!biasOk) - break; - - const bool inputIsU8 = - (netimpl->argData(dq_x->inputs[0]).type == CV_8U) || - (x_zp_m.depth() == CV_8U); - const int inp_zp_kernel = inputIsU8 ? (inp_zp - 128) : inp_zp; - Mat weights_2d = w_q.reshape(1, outCn); - Mat biasFused(1, outCn, CV_32S); - Mat outputMultiplier(1, outCn, CV_32F); - for (int oc = 0; oc < outCn; oc++) { - biasFused.at(oc) = bias.at(oc) - inp_zp_kernel * (int)cv::sum(weights_2d.row(oc))[0]; - outputMultiplier.at(oc) = (inp_sc * wt_sc.at(oc)) / out_sc; - } - - LayerParams convInt8Params = makeLayerParamsFromOriginal(conv, "ConvolutionInt8"); - { - int kndims = w_q.dims - 2; - std::vector ksize(kndims); - for (int d = 0; d < kndims; d++) - ksize[d] = w_q.size[d + 2]; - convInt8Params.set("kernel_size", DictValue::arrayInt(ksize.data(), kndims)); - if (!conv->strides.empty()) - convInt8Params.set("stride", DictValue::arrayInt(conv->strides.data(), (int)conv->strides.size())); - if (!conv->dilations.empty()) - convInt8Params.set("dilation", DictValue::arrayInt(conv->dilations.data(), (int)conv->dilations.size())); - if (!conv->pads.empty()) - convInt8Params.set("pad", DictValue::arrayInt(conv->pads.data(), (int)conv->pads.size())); - } - convInt8Params.set("num_output", outCn); - convInt8Params.set("group", conv->ngroups); - convInt8Params.blobs.resize(3); - convInt8Params.blobs[0] = w_q; - convInt8Params.blobs[1] = biasFused; - convInt8Params.blobs[2] = outputMultiplier; - Ptr convInt8 = createFusedLayer(convInt8Params); - if (!convInt8.empty()) { - auto* convInt8Layer = dynamic_cast(convInt8.get()); - CV_Assert(convInt8Layer); - convInt8Layer->input_zp = inp_zp; - convInt8Layer->input_sc = inp_sc; - convInt8Layer->output_zp = out_zp; - convInt8Layer->output_sc = out_sc; - convInt8Layer->per_channel = per_channel; - fused_layer_idx = conv_layer_idx; - newprog[conv_layer_idx] = convInt8; - fused_inputs.assign(1, dq_x->inputs[0]); - removed_args.push_back(q_data_in); - removed_args.push_back(conv_w); - if (conv->inputs.size() == 3) { - removed_args.push_back(conv->inputs[2]); - if (dq_bias_idx >= 0) - newprog[dq_bias_idx] = Ptr(); - } - if (usecounts.at(conv_x.idx) == 1) { - removed_args.push_back(conv_x); - newprog[dq_x_idx] = Ptr(); - } - newprog[dq_w_idx] = Ptr(); - break; - } + } + if (!biasOk) + break; + + const bool inputIsU8 = + (netimpl->argData(dq_x->inputs[0]).type == CV_8U) || + (x_zp_m.depth() == CV_8U); + const int inp_zp_kernel = inputIsU8 ? (inp_zp - 128) : inp_zp; + Mat weights_2d = w_q.reshape(1, outCn); + Mat biasFused(1, outCn, CV_32S); + Mat outputMultiplier(1, outCn, CV_32F); + for (int oc = 0; oc < outCn; oc++) { + biasFused.at(oc) = bias.at(oc) - inp_zp_kernel * (int)cv::sum(weights_2d.row(oc))[0]; + outputMultiplier.at(oc) = (inp_sc * wt_sc.at(oc)) / out_sc; + } + + LayerParams convInt8Params = makeLayerParamsFromOriginal(conv, "Conv2Int8"); + { + if (!conv->strides.empty()) + convInt8Params.set("stride", DictValue::arrayInt(conv->strides.data(), (int)conv->strides.size())); + if (!conv->dilations.empty()) + convInt8Params.set("dilation", DictValue::arrayInt(conv->dilations.data(), (int)conv->dilations.size())); + if (!conv->pads.empty()) + convInt8Params.set("pad", DictValue::arrayInt(conv->pads.data(), (int)conv->pads.size())); + } + convInt8Params.set("num_output", outCn); + convInt8Params.set("group", conv->ngroups); + convInt8Params.set("input_scale", inp_sc); + convInt8Params.set("input_zeropoint", inp_zp); + convInt8Params.set("scales", out_sc); + convInt8Params.set("zeropoints", out_zp); + convInt8Params.set("per_channel", per_channel); + convInt8Params.set("input_is_u8", inputIsU8); + convInt8Params.blobs.resize(3); + convInt8Params.blobs[0] = w_q; + convInt8Params.blobs[1] = biasFused; + convInt8Params.blobs[2] = outputMultiplier; + Ptr convInt8 = createFusedLayer(convInt8Params); + if (!convInt8.empty()) { + auto* convInt8Layer = dynamic_cast(convInt8.get()); + CV_Assert(convInt8Layer); + fused_layer_idx = conv_layer_idx; + newprog[conv_layer_idx] = convInt8; + fused_inputs.assign(1, dq_x->inputs[0]); + removed_args.push_back(q_data_in); + removed_args.push_back(conv_w); + if (conv->inputs.size() == 3) { + removed_args.push_back(conv->inputs[2]); + if (dq_bias_idx >= 0) + newprog[dq_bias_idx] = Ptr(); + } + if (usecounts.at(conv_x.idx) == 1) { + removed_args.push_back(conv_x); + newprog[dq_x_idx] = Ptr(); + } + newprog[dq_w_idx] = Ptr(); + break; } } + } } } @@ -577,9 +594,17 @@ struct ModelFusionQDQ DequantizeLinearLayer* dq_w = getLayer(newprog, dq_w_idx); float inp_sc = 0.f, out_sc = 0.f; int inp_zp = 0, out_zp_i = 0; - const int fc_out_type = !outputs.empty() ? netimpl->argData(outputs[0]).type : -1; + int fc_out_type = !outputs.empty() ? netimpl->argData(outputs[0]).type : -1; + if (fc_out_type < 0) { + const Mat& zp_t = netimpl->argTensor(inputs[2]); + fc_out_type = !zp_t.empty() ? zp_t.type() : CV_8S; + } const bool fc_out_int8 = (fc_out_type == CV_8S || fc_out_type == CV_8U); - const int fc_in_type = (dq_x && !dq_x->inputs.empty()) ? netimpl->argData(dq_x->inputs[0]).type : -1; + int fc_in_type = (dq_x && !dq_x->inputs.empty()) ? netimpl->argData(dq_x->inputs[0]).type : -1; + if (fc_in_type < 0 && dq_x && dq_x->inputs.size() >= 3) { + const Mat& zp_t = netimpl->argTensor(dq_x->inputs[2]); + fc_in_type = !zp_t.empty() ? zp_t.type() : CV_8S; + } const bool fc_in_int8 = (fc_in_type == CV_8S || fc_in_type == CV_8U); if (dq_x && dq_w && dq_x->inputs.size() >= 3 && dq_w->inputs.size() >= 3 && @@ -587,8 +612,9 @@ struct ModelFusionQDQ usecounts.at(mm_x.idx) == 1 && usecounts.at(mm_w.idx) == 1) { inp_sc = netimpl->argTensor(dq_x->inputs[1]).at(0); const Mat& fc_zp_m = netimpl->argTensor(dq_x->inputs[2]); + inp_zp = fc_zp_m.depth() == CV_8U - ? (int)fc_zp_m.at(0) + ? (int)fc_zp_m.at(0) - 128 : (int)fc_zp_m.at(0); out_sc = netimpl->argTensor(out_scale).at(0); const Mat& fc_out_zp_m = netimpl->argTensor(out_zp); @@ -638,6 +664,7 @@ struct ModelFusionQDQ fcInt8Layer->input_sc = inp_sc; fcInt8Layer->output_zp = out_zp_i; fcInt8Layer->output_sc = out_sc; + fcInt8Layer->output_type = fc_out_type; fcInt8Layer->per_channel = per_channel; fused_layer_idx = mm_layer_idx; newprog[mm_layer_idx] = fcInt8; @@ -654,112 +681,211 @@ struct ModelFusionQDQ } } - int pool_layer_idx = -1; - PoolingLayer* pool = 0; - if (getQdqPatternContext(layer_ptr, ninputs, inputs, producer_of, - newprog, q_data_in, out_scale, out_zp, - pool_layer_idx, pool) && - pool->inputs.size() == 1) { - Arg pool_in = pool->inputs[0]; - int dq_idx = producer_of.at(pool_in.idx); - DequantizeLinearLayer* dq = getLayer(newprog, dq_idx); - float inp_sc = 0.f, out_sc = 0.f; - int inp_zp = 0, out_zp_i = 0; - const int pool_out_type = !outputs.empty() ? netimpl->argData(outputs[0]).type : -1; - const bool pool_out_int8 = (pool_out_type == CV_8S || pool_out_type == CV_8U); - const int pool_in_type = (dq && !dq->inputs.empty()) ? netimpl->argData(dq->inputs[0]).type : -1; - const bool pool_in_int8 = (pool_in_type == CV_8S || pool_in_type == CV_8U); - if (dq && dq->inputs.size() >= 3 && - pool_in_int8 && pool_out_int8 && - usecounts.at(pool_in.idx) == 1) { - inp_sc = netimpl->argTensor(dq->inputs[1]).at(0); - const Mat& pool_zp_m = netimpl->argTensor(dq->inputs[2]); - inp_zp = pool_zp_m.depth() == CV_8U - ? (int)pool_zp_m.at(0) - : (int)pool_zp_m.at(0); - out_sc = netimpl->argTensor(out_scale).at(0); - const Mat& pool_out_zp_m = netimpl->argTensor(out_zp); - out_zp_i = pool_out_zp_m.depth() == CV_8U - ? (int)pool_out_zp_m.at(0) - : (int)pool_out_zp_m.at(0); - bool isGlobalAve = pool->globalPooling; - bool isMax = !isGlobalAve; - if ((isGlobalAve && inp_sc > 0.f && out_sc > 0.f) || - (isMax && std::abs(inp_sc - out_sc) < 1e-6f && inp_zp == out_zp_i)) { - LayerParams poolInt8Params = makeLayerParamsFromOriginal(pool, "PoolingInt8"); - poolInt8Params.blobs.clear(); - Ptr poolInt8 = createFusedLayer(poolInt8Params); - if (!poolInt8.empty()) { - auto* poolInt8Layer = dynamic_cast(poolInt8.get()); - CV_Assert(poolInt8Layer); - const String poolInt8Type = static_cast(*poolInt8Layer).type; - std::vector poolInt8Blobs = poolInt8Layer->blobs; - static_cast(*poolInt8Layer) = *pool; - static_cast(*poolInt8Layer).type = poolInt8Type; - poolInt8Layer->blobs = poolInt8Blobs; - poolInt8Layer->input_sc = inp_sc; - poolInt8Layer->input_zp = inp_zp; - poolInt8Layer->output_sc = out_sc; - poolInt8Layer->output_zp = out_zp_i; - fused_layer_idx = pool_layer_idx; - newprog[pool_layer_idx] = poolInt8; - fused_inputs.assign(1, dq->inputs[0]); - removed_args.push_back(q_data_in); - removed_args.push_back(pool_in); - newprog[dq_idx] = Ptr(); - break; + int add_bias_idx = -1; + NaryEltwiseLayer* add_bias = 0; + if (getQdqPatternContext(layer_ptr, ninputs, inputs, producer_of, + newprog, q_data_in, out_scale, out_zp, + add_bias_idx, add_bias) && + add_bias && add_bias->inputs.size() == 2) { + int mm_inp_k = -1; + int bias_inp_k = -1; + MatMulLayer* mm2 = nullptr; + int mm2_idx = -1; + for (int k = 0; k < 2; k++) { + int prod_idx = producer_of.at(add_bias->inputs[k].idx); + MatMulLayer* candidate = getLayer(newprog, prod_idx); + if (candidate && candidate->inputs.size() == 2) { + mm_inp_k = k; + mm2 = candidate; + mm2_idx = prod_idx; + } else if (netimpl->isConstArg(add_bias->inputs[k])) { + bias_inp_k = k; + } + } + if (mm2 && bias_inp_k >= 0 && mm_inp_k >= 0 && + usecounts.at(add_bias->inputs[mm_inp_k].idx) == 1) { + const Arg mm_x = mm2->inputs[0]; + const Arg mm_w = mm2->inputs[1]; + int dq_x_idx = producer_of.at(mm_x.idx); + int dq_w_idx = producer_of.at(mm_w.idx); + DequantizeLinearLayer* dq_x = getLayer(newprog, dq_x_idx); + DequantizeLinearLayer* dq_w = getLayer(newprog, dq_w_idx); + int fc_out_type = !outputs.empty() ? netimpl->argData(outputs[0]).type : -1; + if (fc_out_type < 0) { + const Mat& zp_t = netimpl->argTensor(inputs[2]); + fc_out_type = !zp_t.empty() ? zp_t.type() : CV_8S; + } + const bool fc_out_int8 = (fc_out_type == CV_8S || fc_out_type == CV_8U); + int fc_in_type = (dq_x && !dq_x->inputs.empty()) ? netimpl->argData(dq_x->inputs[0]).type : -1; + if (fc_in_type < 0 && dq_x && dq_x->inputs.size() >= 3) { + const Mat& zp_t = netimpl->argTensor(dq_x->inputs[2]); + fc_in_type = !zp_t.empty() ? zp_t.type() : CV_8S; + } + const bool fc_in_int8 = (fc_in_type == CV_8S || fc_in_type == CV_8U); + if (dq_x && dq_w && + dq_x->inputs.size() >= 3 && dq_w->inputs.size() >= 3 && + fc_in_int8 && fc_out_int8 && + usecounts.at(mm_x.idx) == 1 && usecounts.at(mm_w.idx) == 1) { + float inp_sc = netimpl->argTensor(dq_x->inputs[1]).at(0); + const Mat& fc_zp_m = netimpl->argTensor(dq_x->inputs[2]); + int inp_zp = fc_zp_m.depth() == CV_8U + ? (int)fc_zp_m.at(0) - 128 + : (int)fc_zp_m.at(0); + float out_sc_val = netimpl->argTensor(out_scale).at(0); + const Mat& fc_out_zp_m = netimpl->argTensor(out_zp); + int out_zp_i = fc_out_zp_m.depth() == CV_8U + ? (int)fc_out_zp_m.at(0) + : (int)fc_out_zp_m.at(0); + if (inp_sc > 0.f && out_sc_val > 0.f) { + Mat w_q = netimpl->argTensor(dq_w->inputs[0]); + Mat w_sc_m = netimpl->argTensor(dq_w->inputs[1]); + Mat w_zp_m = netimpl->argTensor(dq_w->inputs[2]); + if (!w_q.empty() && w_q.depth() == CV_8S && w_q.dims == 2) { + bool all_wzp_zero = true; + for (size_t t = 0; all_wzp_zero && t < w_zp_m.total(); t++) { + int wz = w_zp_m.depth() == CV_8S ? (int)w_zp_m.at((int)t) + : (int)w_zp_m.at((int)t); + if (wz != 0) all_wzp_zero = false; + } + if (all_wzp_zero) { + Mat weights = w_q.t(); + int outCn = weights.size[0]; + Mat wt_sc = (w_sc_m.total() == (size_t)outCn) + ? w_sc_m.reshape(1, 1) + : Mat(1, outCn, CV_32F, Scalar(w_sc_m.at(0))).clone(); + bool per_channel = w_sc_m.total() == (size_t)outCn; + + // Fuse the float bias into int32 bias + Mat float_bias = netimpl->argTensor(add_bias->inputs[bias_inp_k]); + Mat bias(1, outCn, CV_32S); + Mat outputMultiplier(1, outCn, CV_32F); + bool biasOk = (float_bias.total() == (size_t)outCn); + for (int ioc = 0; ioc < outCn && biasOk; ioc++) { + float denom = inp_sc * wt_sc.at(ioc); + if (std::abs(denom) < 1e-12f) { biasOk = false; break; } + float b_val = 0.f; + if (float_bias.depth() == CV_32F) + b_val = float_bias.at(ioc); + else if (float_bias.depth() == CV_64F) + b_val = (float)float_bias.at(ioc); + bias.at(ioc) = cvRound(b_val / denom) - inp_zp * (int)cv::sum(weights.row(ioc))[0]; + outputMultiplier.at(ioc) = denom / out_sc_val; + } + if (biasOk) { + int firstInpDims = (int)netimpl->argData(mm_x).shape.size(); + int fc_axis = std::max(1, firstInpDims - w_q.dims + 1); + + LayerParams fcInt8Params = makeLayerParamsFromOriginal(mm2, "InnerProductInt8"); + fcInt8Params.set("num_output", outCn); + fcInt8Params.set("axis", fc_axis); + fcInt8Params.blobs.resize(3); + fcInt8Params.blobs[0] = weights; + fcInt8Params.blobs[1] = bias; + fcInt8Params.blobs[2] = outputMultiplier; + Ptr fcInt8 = createFusedLayer(fcInt8Params); + if (!fcInt8.empty()) { + auto* fcInt8Layer = dynamic_cast(fcInt8.get()); + CV_Assert(fcInt8Layer); + fcInt8Layer->input_zp = inp_zp; + fcInt8Layer->input_sc = inp_sc; + fcInt8Layer->output_zp = out_zp_i; + fcInt8Layer->output_sc = out_sc_val; + fcInt8Layer->output_type = fc_out_type; + fcInt8Layer->per_channel = per_channel; + fused_layer_idx = add_bias_idx; + newprog[add_bias_idx] = fcInt8; + fused_inputs.assign(1, dq_x->inputs[0]); + removed_args.push_back(q_data_in); + removed_args.push_back(add_bias->inputs[mm_inp_k]); // matmul out + removed_args.push_back(mm_x); + removed_args.push_back(mm_w); + newprog[mm2_idx] = Ptr(); + newprog[dq_x_idx] = Ptr(); + newprog[dq_w_idx] = Ptr(); + break; + } + } + } } } } } + } - { - ActivationLayerInt8* activ_int8 = dynamic_cast(layer_ptr); - if (activ_int8 && ninputs == 1 && - usecounts.at(inputs[0].idx) == 1) { - Arg activ_inp = inputs[0]; - int producer_idx = producer_of.at(activ_inp.idx); - if (producer_idx >= 0 && !newprog[producer_idx].empty()) { - Layer* producer_layer = newprog[producer_idx].get(); + int pool_layer_idx = -1; + PoolingLayer* pool = 0; + if (getQdqPatternContext(layer_ptr, ninputs, inputs, producer_of, + newprog, q_data_in, out_scale, out_zp, + pool_layer_idx, pool) && pool->inputs.size() == 1) { + Arg pool_in = pool->inputs[0]; + int dq_idx = producer_of.at(pool_in.idx); + DequantizeLinearLayer* dq = getLayer(newprog, dq_idx); + float inp_sc = 0.f, out_sc = 0.f; + int inp_zp = 0, out_zp_i = 0; + const int pool_out_type = !outputs.empty() ? netimpl->argData(outputs[0]).type : -1; + const bool pool_out_int8 = (pool_out_type == CV_8S || pool_out_type == CV_8U); + const int pool_in_type = (dq && !dq->inputs.empty()) ? netimpl->argData(dq->inputs[0]).type : -1; + const bool pool_in_int8 = (pool_in_type == CV_8S || pool_in_type == CV_8U); + + if (dq && dq->inputs.size() >= 3 && pool_in_int8 && pool_out_int8 && usecounts.at(pool_in.idx) == 1) { + inp_sc = netimpl->argTensor(dq->inputs[1]).at(0); + const Mat& pool_zp_m = netimpl->argTensor(dq->inputs[2]); + inp_zp = pool_zp_m.depth() == CV_8U + ? (int)pool_zp_m.at(0) + : (int)pool_zp_m.at(0); + out_sc = netimpl->argTensor(out_scale).at(0); + const Mat& pool_out_zp_m = netimpl->argTensor(out_zp); + out_zp_i = pool_out_zp_m.depth() == CV_8U + ? (int)pool_out_zp_m.at(0) + : (int)pool_out_zp_m.at(0); + bool isGlobalAve = pool->globalPooling; + bool isMax = !isGlobalAve; + if ((isGlobalAve && inp_sc > 0.f && out_sc > 0.f) || + (isMax && std::abs(inp_sc - out_sc) < 1e-6f && inp_zp == out_zp_i)) { + LayerParams poolInt8Params = makeLayerParamsFromOriginal(pool, "Pool2Int8"); + poolInt8Params.blobs.clear(); + poolInt8Params.set("input_scale", inp_sc); + poolInt8Params.set("input_zeropoint", inp_zp); + poolInt8Params.set("scales", out_sc); + poolInt8Params.set("zeropoints", out_zp_i); + poolInt8Params.set("is_max_pool", isMax); + poolInt8Params.set("global_pooling", isGlobalAve); + Ptr poolInt8 = createFusedLayer(poolInt8Params); + if (!poolInt8.empty()) { + auto* poolInt8Layer = dynamic_cast(poolInt8.get()); + CV_Assert(poolInt8Layer); + fused_layer_idx = pool_layer_idx; + newprog[pool_layer_idx] = poolInt8; + fused_inputs.assign(1, dq->inputs[0]); + removed_args.push_back(q_data_in); + removed_args.push_back(pool_in); + newprog[dq_idx] = Ptr(); + break; + } + } + } + } + + ActivationLayerInt8* activ_int8 = dynamic_cast(layer_ptr); + if (activ_int8 && ninputs == 1 && + usecounts.at(inputs[0].idx) == 1) { + Arg activ_inp = inputs[0]; + int producer_idx = producer_of.at(activ_inp.idx); + if (producer_idx >= 0 && !newprog[producer_idx].empty()) { + Layer* producer_layer = newprog[producer_idx].get(); + float prod_sc = 0.f; + int prod_zp = 0; + if (getInt8OutputParams(producer_layer, prod_sc, prod_zp) && + prod_sc == activ_int8->input_sc && + prod_zp == activ_int8->input_zp) { Ptr activ_layer = layer.dynamicCast(); - - ConvolutionLayerInt8* conv_int8 = - dynamic_cast(producer_layer); - if (conv_int8 && conv_int8->output_sc == activ_int8->input_sc && - conv_int8->output_zp == activ_int8->input_zp) { - if (newprog[producer_idx]->setActivation(activ_layer)) { - conv_int8->output_sc = activ_int8->output_sc; - conv_int8->output_zp = activ_int8->output_zp; - fused_layer_idx = producer_idx; - removed_args.push_back(activ_inp); - break; - } - } - - InnerProductLayerInt8* fc_int8 = - dynamic_cast(producer_layer); - if (fc_int8 && fc_int8->output_sc == activ_int8->input_sc && - fc_int8->output_zp == activ_int8->input_zp) { - if (newprog[producer_idx]->setActivation(activ_layer)) { - fc_int8->output_sc = activ_int8->output_sc; - fc_int8->output_zp = activ_int8->output_zp; - fused_layer_idx = producer_idx; - removed_args.push_back(activ_inp); - break; - } - } - - EltwiseLayerInt8* elt_int8 = - dynamic_cast(producer_layer); - if (elt_int8 && elt_int8->output_sc == activ_int8->input_sc && - elt_int8->output_zp == activ_int8->input_zp) { - if (newprog[producer_idx]->setActivation(activ_layer)) { - elt_int8->output_sc = activ_int8->output_sc; - elt_int8->output_zp = activ_int8->output_zp; - fused_layer_idx = producer_idx; - removed_args.push_back(activ_inp); - break; - } + if (newprog[producer_idx]->setActivation(activ_layer)) { + setInt8OutputParams(producer_layer, + activ_int8->output_sc, + activ_int8->output_zp); + fused_layer_idx = producer_idx; + removed_args.push_back(activ_inp); + break; } } } @@ -770,12 +896,13 @@ struct ModelFusionQDQ if (fused_layer_idx >= 0) { modified = true; Layer* fused_layer = newprog[fused_layer_idx]; - fused_layer->outputs = outputs; + const std::vector& final_outputs = override_outputs.empty() ? outputs : override_outputs; + fused_layer->outputs = final_outputs; if (!fused_inputs.empty()) fused_layer->inputs = fused_inputs; - for (Arg new_out: outputs) + for (const Arg& new_out: final_outputs) producer_of[new_out.idx] = fused_layer_idx; - for (Arg old_out: removed_args) { + for (const Arg& old_out: removed_args) { usecounts.at(old_out.idx) = 0; producer_of.at(old_out.idx) = -1; } @@ -787,6 +914,41 @@ struct ModelFusionQDQ } if (modified) { + std::vector uc(nargs, 0); + for (const auto& layer : newprog) { + if (layer.empty()) continue; + for (const Arg& inp : layer->inputs) { + if (inp.idx >= 0 && inp.idx < (int)nargs) + uc[inp.idx]++; + } + } + const auto& graph_outputs = graph->outputs(); + for (const Arg& out : graph_outputs) { + if (out.idx >= 0 && out.idx < (int)nargs) + uc[out.idx]++; + } + bool changed = true; + while (changed) { + changed = false; + for (auto& layer : newprog) { + if (layer.empty()) continue; + bool all_dead = true; + for (const Arg& out : layer->outputs) { + if (out.idx >= 0 && out.idx < (int)nargs && uc[out.idx] > 0) { + all_dead = false; + break; + } + } + if (all_dead) { + for (const Arg& inp : layer->inputs) { + if (inp.idx >= 0 && inp.idx < (int)nargs) + uc[inp.idx]--; + } + layer = Ptr(); + changed = true; + } + } + } size_t i, j = 0, newops = newprog.size(); for (i = 0; i < newops; i++) { if (!newprog[i].empty()) { diff --git a/modules/dnn/src/init.cpp b/modules/dnn/src/init.cpp index bbf9343699..930b7a4726 100644 --- a/modules/dnn/src/init.cpp +++ b/modules/dnn/src/init.cpp @@ -263,6 +263,9 @@ void initializeLayerFactory() CV_DNN_REGISTER_LAYER_CLASS(InnerProductInt8, InnerProductLayerInt8); CV_DNN_REGISTER_LAYER_CLASS(PoolingInt8, PoolingLayerInt8); CV_DNN_REGISTER_LAYER_CLASS(EltwiseInt8, EltwiseLayerInt8); + CV_DNN_REGISTER_LAYER_CLASS(Conv2Int8, Conv2Int8Layer); + CV_DNN_REGISTER_LAYER_CLASS(Pool2Int8, Pool2Int8Layer); + CV_DNN_REGISTER_LAYER_CLASS(Eltwise2Int8, Eltwise2Int8Layer); CV_DNN_REGISTER_LAYER_CLASS(BatchNormInt8, BatchNormLayerInt8); CV_DNN_REGISTER_LAYER_CLASS(ScaleInt8, ScaleLayerInt8); CV_DNN_REGISTER_LAYER_CLASS(ShiftInt8, ShiftLayerInt8); diff --git a/modules/dnn/src/int8layers/conv2_int8_kernels.simd.hpp b/modules/dnn/src/int8layers/conv2_int8_kernels.simd.hpp new file mode 100644 index 0000000000..a1525ed1bd --- /dev/null +++ b/modules/dnn/src/int8layers/conv2_int8_kernels.simd.hpp @@ -0,0 +1,1221 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// Copyright (C) 2026, BigVision LLC, all rights reserved. +// Third party copyrights are property of their respective owners. + +#include "opencv2/core/hal/intrin.hpp" +#include "../layers/conv2_common.hpp" + +#if !defined(CV_AVXVNNI_AVAILABLE) +#if (CV_TRY_AVX2 || CV_AVX2) && \ + ((defined(__GNUC__) && !defined(__clang__) && __GNUC__ >= 11) || \ + (defined(__clang__) && !defined(__apple_build_version__) && __clang_major__ >= 12)) +#define CV_AVXVNNI_AVAILABLE 1 +#else +#define CV_AVXVNNI_AVAILABLE 0 +#endif +#endif + +namespace cv { +namespace dnn { +CV_CPU_OPTIMIZATION_NAMESPACE_BEGIN + +void convInt8Block(const void* inp_, const void* residual_, + void* out_, const ConvState& cs, + const void* weights_, + const void* weightsVNNI_, + const int* bias, const int* biasVNNI_, + const float* multiplier, + int inp_zp, int out_zp, + const int8_t* activLUT, + bool inputIsU8); + +#if !defined(CV_CPU_OPTIMIZATION_DECLARATIONS_ONLY) + +#if CV_AVXVNNI_AVAILABLE +#if defined(__x86_64__) || defined(_M_X64) +#include +#endif +#ifdef __clang__ +#pragma clang attribute push(__attribute__((target("avx2,avxvnni"))), apply_to = function) +#else +#pragma GCC push_options +#pragma GCC target("avxvnni") +#endif +static void convInt8BlockVNNI(const void* inp_, const void* residual_, + void* out_, const ConvState& cs, + const void* weightsVNNI_, + const int* biasVNNI, const float* multiplier, + int inp_zp, int out_zp, + const int8_t* activLUT, + bool inputIsU8) +{ + constexpr int C0 = 8, K0 = 8; + constexpr int SPAT_BLOCK_SIZE = 8; + constexpr int MAX_CONV_DIMS = ConvState::MAX_CONV_DIMS; + + CV_Assert(cs.inpshape.layout == DATA_LAYOUT_BLOCK); + CV_Assert(cs.outshape.layout == DATA_LAYOUT_BLOCK); + CV_Assert(cs.inpshape.back() == C0); + + const MatShape& inpshape = cs.inpshape; + const MatShape& outshape = cs.outshape; + + int N = outshape[0]; + int ndims = outshape.dims; + int K = outshape.channels(); + int C = inpshape.channels(); + int K1 = (K + K0 - 1) / K0; + int C1 = (C + C0 - 1) / C0; + + int D_ = ndims >= 6 ? outshape[ndims-4] : 1; + int H_ = ndims >= 5 ? outshape[ndims-3] : 1; + int W_ = outshape[ndims-2]; + int planeblocks_ = D_ * H_ * W_; + + int ngroups = cs.ngroups; + int Kg = K / ngroups; + int Cg = C / ngroups; + int Kblk = cs.wshape[1]; + int ksize_ = cs.wshape[2]; + int C1Max = cs.wshape[3]; + + int innerZ0 = cs.inner[0], innerZ1 = cs.inner[MAX_CONV_DIMS]; + int innerY0 = cs.inner[1], innerY1 = cs.inner[MAX_CONV_DIMS+1]; + int innerX0 = cs.inner[2], innerX1 = cs.inner[MAX_CONV_DIMS+2]; + + const int8_t* inp = (const int8_t*)inp_; + const int8_t* residual = (const int8_t*)residual_; + int8_t* out = (int8_t*)out_; + const int8_t* wdata = (const int8_t*)weightsVNNI_; + const int* ofsZYX = cs.coordtab.data(); + + size_t outtotal = outshape.total(); + if ((Kg % K0) != 0) memset(out, 0, outtotal); + + int total_blocks = N * ngroups * Kblk; + const __m256i v_xor = inputIsU8 ? _mm256_setzero_si256() : _mm256_set1_epi8((char)0x80); + + parallel_for_(Range(0, total_blocks), [&](const Range& range) { + constexpr int C0 = 8, K0 = 8; + constexpr int C0shift = 3; + int D = D_, H = H_, W = W_; + int Di = ndims >= 6 ? inpshape[ndims-4] : 1; + int Hi = ndims >= 5 ? inpshape[ndims-3] : 1; + int Wi = inpshape[ndims-2]; + int planeblocks = planeblocks_; + int iplanesize = Di * Hi * Wi * C0; + int planesize = planeblocks * K0; + + int Sz = cs.strides[0], Sy = cs.strides[1], Sx = cs.strides[2]; + int padZ = cs.pads[0], padY = cs.pads[1], padX = cs.pads[2]; + int ksize = ksize_; + int8_t zbuf[C0]; + memset(zbuf, (uint8_t)inp_zp, C0); + + for (int t = range.start; t < range.end; t++) { + int n = t / (ngroups * Kblk); + int rem = t - n * (ngroups * Kblk); + int g = rem / Kblk; + int kblk = rem - g * Kblk; + + int k_base = g * Kg + kblk * K0; + if (k_base >= K) continue; + + int c_start = g * Cg; + int c1_start = c_start >> C0shift; + int c00 = c_start & (C0 - 1); + int cblocks = (c00 + Cg + C0 - 1) >> C0shift; + + const int8_t* inpbaseptr = inp + (size_t)(n * C1 + c1_start) * iplanesize; + const int8_t* wbaseptr = wdata + (size_t)(g * Kblk + kblk) * ksize * C1Max * C0 * K0; + + int k1 = k_base >> C0shift; + int8_t* outptr = out + (size_t)(n * K1 + k1) * planesize; + const int8_t* resptr = residual ? residual + (size_t)(n * K1 + k1) * planesize : nullptr; + + alignas(32) int32_t biasbuf[K0]; + memcpy(biasbuf, biasVNNI + k_base, K0 * sizeof(int32_t)); + + alignas(32) float multbuf[K0]; + memcpy(multbuf, multiplier + k_base, K0 * sizeof(float)); + + int D_l = D, H_l = H, W_l = W; + int Di_l = Di, Hi_l = Hi, Wi_l = Wi; + int iplanesize_l = iplanesize; + int planeblocks_l = planeblocks; + int ksize_l = ksize; + + if (ksize == 1 && Sx == 1 && Sy == 1 && Sz == 1) { + W_l *= D_l * H_l; + Wi_l *= Di_l * Hi_l; + D_l = Di_l = H_l = Hi_l = 1; + iplanesize_l = Wi_l * C0; + planeblocks_l = W_l; + } + + int p = 0; + for (; p < planeblocks_l; p += SPAT_BLOCK_SIZE) { + if (p + SPAT_BLOCK_SIZE > planeblocks_l) { + if (p == 0) break; + p = planeblocks_l - SPAT_BLOCK_SIZE; + } + + Vec3i pt[SPAT_BLOCK_SIZE]; + bool inner[SPAT_BLOCK_SIZE]; + bool all_inner = true; + + if ((p % W_l) + SPAT_BLOCK_SIZE <= W_l) { + int zj = p / (H_l * W_l); + int yxj = p - zj * H_l * W_l; + int yj = yxj / W_l; + int xj0 = yxj - yj * W_l; + bool zy_inner = (zj >= innerZ0 && zj < innerZ1) && + (yj >= innerY0 && yj < innerY1); + for (int j = 0; j < SPAT_BLOCK_SIZE; j++) { + int xj = xj0 + j; + pt[j] = Vec3i(zj * Sz - padZ, yj * Sy - padY, xj * Sx - padX); + inner[j] = zy_inner && (xj >= innerX0 && xj < innerX1); + all_inner &= inner[j]; + } + } else { + for (int j = 0; j < SPAT_BLOCK_SIZE; j++) { + int pj = p + j; + int zj = pj / (H_l * W_l); + int yxj = pj - zj * H_l * W_l; + int yj = yxj / W_l; + int xj = yxj - yj * W_l; + pt[j] = Vec3i(zj * Sz - padZ, yj * Sy - padY, xj * Sx - padX); + inner[j] = (zj >= innerZ0 && zj < innerZ1) && + (yj >= innerY0 && yj < innerY1) && + (xj >= innerX0 && xj < innerX1); + all_inner &= inner[j]; + } + } + + __m256i vbias = _mm256_load_si256((const __m256i*)biasbuf); + __m256i s0 = vbias, s1 = vbias, s2 = vbias, s3 = vbias; + __m256i s4 = vbias, s5 = vbias, s6 = vbias, s7 = vbias; + + #define CONV_VNNI_MAC(j) { \ + __m256i x0 = _mm256_xor_si256(_mm256_set1_epi32(*(const int32_t*)&inptr[(j)][0]), v_xor); \ + __m256i x1 = _mm256_xor_si256(_mm256_set1_epi32(*(const int32_t*)&inptr[(j)][4]), v_xor); \ + s##j = _mm256_dpbusd_epi32(s##j, x0, wg0); \ + s##j = _mm256_dpbusd_epi32(s##j, x1, wg1); \ + } + + if (all_inner) { + for (int i = 0; i < ksize_l; i++) { + const int8_t* inptr[SPAT_BLOCK_SIZE]; + for (int j = 0; j < SPAT_BLOCK_SIZE; j++) { + int zij = pt[j][0] + ofsZYX[i * 3]; + int yij = pt[j][1] + ofsZYX[i * 3 + 1]; + int xij = pt[j][2] + ofsZYX[i * 3 + 2]; + inptr[j] = inpbaseptr + (((zij * Hi_l) + yij) * Wi_l + xij) * C0; + } + + const int8_t* wptr = wbaseptr + (size_t)i * C1Max * K0 * C0; + + for (int c1 = 0; c1 < cblocks; c1++, wptr += C0 * K0) { + __m256i wg0 = _mm256_load_si256((const __m256i*)(wptr)); + __m256i wg1 = _mm256_load_si256((const __m256i*)(wptr + 32)); + + CONV_VNNI_MAC(0); CONV_VNNI_MAC(1); + CONV_VNNI_MAC(2); CONV_VNNI_MAC(3); + CONV_VNNI_MAC(4); CONV_VNNI_MAC(5); + CONV_VNNI_MAC(6); CONV_VNNI_MAC(7); + + for (int j = 0; j < SPAT_BLOCK_SIZE; j++) + inptr[j] += iplanesize_l; + } + } + } else { + for (int i = 0; i < ksize_l; i++) { + const int8_t* inptr[SPAT_BLOCK_SIZE]; + int inpstep[SPAT_BLOCK_SIZE]; + + for (int j = 0; j < SPAT_BLOCK_SIZE; j++) { + int zij = pt[j][0] + ofsZYX[i * 3]; + int yij = pt[j][1] + ofsZYX[i * 3 + 1]; + int xij = pt[j][2] + ofsZYX[i * 3 + 2]; + if (inner[j] || ((((unsigned)zij < (unsigned)Di_l) & + ((unsigned)yij < (unsigned)Hi_l) & + ((unsigned)xij < (unsigned)Wi_l)) != 0)) { + inptr[j] = inpbaseptr + (((zij * Hi_l) + yij) * Wi_l + xij) * C0; + inpstep[j] = iplanesize_l; + } else { + inptr[j] = zbuf; + inpstep[j] = 0; + } + } + + const int8_t* wptr = wbaseptr + (size_t)i * C1Max * K0 * C0; + + for (int c1 = 0; c1 < cblocks; c1++, wptr += C0 * K0) { + __m256i wg0 = _mm256_load_si256((const __m256i*)(wptr)); + __m256i wg1 = _mm256_load_si256((const __m256i*)(wptr + 32)); + + CONV_VNNI_MAC(0); CONV_VNNI_MAC(1); + CONV_VNNI_MAC(2); CONV_VNNI_MAC(3); + CONV_VNNI_MAC(4); CONV_VNNI_MAC(5); + CONV_VNNI_MAC(6); CONV_VNNI_MAC(7); + + for (int j = 0; j < SPAT_BLOCK_SIZE; j++) + inptr[j] += inpstep[j]; + } + } + } + #undef CONV_VNNI_MAC + + __m256 vmult = _mm256_load_ps(multbuf); + __m256 vzp = _mm256_set1_ps((float)out_zp); + + #define CONV_VNNI_STORE(j) { \ + __m256 facc = _mm256_add_ps( \ + _mm256_mul_ps(_mm256_cvtepi32_ps(s##j), vmult), vzp); \ + if (resptr) { \ + __m256i res32 = _mm256_cvtepi8_epi32( \ + _mm_loadl_epi64((const __m128i*)(resptr + (p + (j)) * K0))); \ + facc = _mm256_add_ps(facc, \ + _mm256_sub_ps(_mm256_cvtepi32_ps(res32), vzp)); \ + } \ + __m256i ival = _mm256_cvtps_epi32(facc); \ + __m128i lo = _mm256_castsi256_si128(ival); \ + __m128i hi = _mm256_extracti128_si256(ival, 1); \ + __m128i packed16 = _mm_packs_epi32(lo, hi); \ + __m128i packed8 = inputIsU8 ? _mm_packus_epi16(packed16, packed16) \ + : _mm_packs_epi16(packed16, packed16); \ + int8_t* optr = outptr + (p + (j)) * K0; \ + if (activLUT) { \ + alignas(16) int8_t tmp8[16]; \ + _mm_store_si128((__m128i*)tmp8, packed8); \ + for (int k = 0; k < K0; k++) \ + optr[k] = (int8_t)activLUT[(int)tmp8[k] + 128]; \ + } else { \ + _mm_storel_epi64((__m128i*)optr, packed8); \ + } \ + } + CONV_VNNI_STORE(0); CONV_VNNI_STORE(1); + CONV_VNNI_STORE(2); CONV_VNNI_STORE(3); + CONV_VNNI_STORE(4); CONV_VNNI_STORE(5); + CONV_VNNI_STORE(6); CONV_VNNI_STORE(7); + #undef CONV_VNNI_STORE + } + + // Scalar tail + for (; p < planeblocks_l; p++) { + alignas(32) int32_t acc[K0]; + memcpy(acc, biasbuf, K0 * sizeof(int32_t)); + + int zj = p / (H_l * W_l); + int yxj = p - zj * H_l * W_l; + int yj = yxj / W_l; + int xj = yxj - yj * W_l; + int zi_base = zj * Sz - padZ, yi_base = yj * Sy - padY, xi_base = xj * Sx - padX; + + for (int i = 0; i < ksize_l; i++) { + int zi = zi_base + ofsZYX[i * 3]; + int yi = yi_base + ofsZYX[i * 3 + 1]; + int xi = xi_base + ofsZYX[i * 3 + 2]; + if ((((unsigned)zi >= (unsigned)Di_l) | ((unsigned)yi >= (unsigned)Hi_l) | + ((unsigned)xi >= (unsigned)Wi_l)) != 0) continue; + const int8_t* inptr = inpbaseptr + (((zi * Hi_l) + yi) * Wi_l + xi) * C0; + const int8_t* wptr = wbaseptr + (size_t)i * C1Max * K0 * C0; + for (int c1i = 0; c1i < cblocks; c1i++, inptr += iplanesize_l, wptr += C0 * K0) { + for (int c0 = 0; c0 < C0; c0++) { + int ival = inputIsU8 ? (int)(uint8_t)inptr[c0] + : (int)(uint8_t)((uint8_t)inptr[c0] ^ 0x80u); + const int8_t* wp = wptr + c0 * K0; + for (int k0 = 0; k0 < K0; k0++) + acc[k0] += ival * (int)wp[k0]; + } + } + } + int8_t* optr = outptr + p * K0; + const int8_t* rptr = resptr ? resptr + p * K0 : nullptr; + for (int k0 = 0; k0 < K0; k0++) { + float val = (float)acc[k0] * multbuf[k0] + (float)out_zp; + if (rptr) val += (float)((int)rptr[k0] - out_zp); + int ival = cvRound(val); + if (inputIsU8) { + ival = std::max(0, std::min(255, ival)); + if (activLUT) ival = (int)(uint8_t)activLUT[ival]; + ((uint8_t*)optr)[k0] = (uint8_t)ival; + } else { + ival = std::max(-128, std::min(127, ival)); + if (activLUT) ival = (int)activLUT[ival + 128]; + optr[k0] = (int8_t)ival; + } + } + } + } + }); +} +#ifdef __clang__ +#pragma clang attribute pop +#else +#pragma GCC pop_options +#endif +#endif // CV_AVXVNNI_AVAILABLE + +#if CV_NEON && defined(CV_NEON_DOT) && CV_NEON_DOT +#include + +#define CONV_NEON_STORE(j) { \ + float32x4_t facc_lo = vfmaq_f32(vzp, vcvtq_f32_s32(s##j##_lo), vmult_lo); \ + float32x4_t facc_hi = vfmaq_f32(vzp, vcvtq_f32_s32(s##j##_hi), vmult_hi); \ + if (resptr) { \ + const int8_t* rp = resptr + (p + (j)) * K0; \ + if (inputIsU8) { \ + uint16x8_t r16u = vmovl_u8(vld1_u8((const uint8_t*)rp)); \ + facc_lo = vaddq_f32(facc_lo, vsubq_f32( \ + vcvtq_f32_s32(vreinterpretq_s32_u32(vmovl_u16(vget_low_u16(r16u)))), vzp)); \ + facc_hi = vaddq_f32(facc_hi, vsubq_f32( \ + vcvtq_f32_s32(vreinterpretq_s32_u32(vmovl_u16(vget_high_u16(r16u)))), vzp)); \ + } else { \ + int16x8_t r16 = vmovl_s8(vld1_s8(rp)); \ + facc_lo = vaddq_f32(facc_lo, vsubq_f32( \ + vcvtq_f32_s32(vmovl_s16(vget_low_s16(r16))), vzp)); \ + facc_hi = vaddq_f32(facc_hi, vsubq_f32( \ + vcvtq_f32_s32(vmovl_s16(vget_high_s16(r16))), vzp)); \ + } \ + } \ + int32x4_t ival_lo = vcvtnq_s32_f32(facc_lo); \ + int32x4_t ival_hi = vcvtnq_s32_f32(facc_hi); \ + int16x4_t p16_lo = vqmovn_s32(ival_lo); \ + int16x4_t p16_hi = vqmovn_s32(ival_hi); \ + int16x8_t p16 = vcombine_s16(p16_lo, p16_hi); \ + int8_t* optr = outptr + (p + (j)) * K0; \ + if (inputIsU8) { \ + uint8x8_t p8u = vqmovun_s16(p16); \ + if (activLUT) { \ + uint8_t tmp8[8]; \ + vst1_u8(tmp8, p8u); \ + for (int k = 0; k < K0; k++) \ + ((uint8_t*)optr)[k] = (uint8_t)activLUT[tmp8[k]]; \ + } else { \ + vst1_u8((uint8_t*)optr, p8u); \ + } \ + } else { \ + int8x8_t p8 = vqmovn_s16(p16); \ + if (activLUT) { \ + int8_t tmp8[8]; \ + vst1_s8(tmp8, p8); \ + for (int k = 0; k < K0; k++) \ + optr[k] = (int8_t)activLUT[(int)tmp8[k] + 128]; \ + } else { \ + vst1_s8(optr, p8); \ + } \ + } \ +} + +static void convInt8BlockNEON(const void* inp_, const void* residual_, + void* out_, const ConvState& cs, + const void* weightsVNNI_, + const int* bias, const float* multiplier, + int inp_zp, int out_zp, + const int8_t* activLUT, + bool inputIsU8) +{ + constexpr int C0 = 8, K0 = 8; + constexpr int C0shift = 3; + constexpr int SPAT_BLOCK_SIZE = 8; + constexpr int MAX_CONV_DIMS = ConvState::MAX_CONV_DIMS; + + CV_Assert(cs.inpshape.layout == DATA_LAYOUT_BLOCK); + CV_Assert(cs.outshape.layout == DATA_LAYOUT_BLOCK); + CV_Assert(cs.inpshape.back() == C0); + + const MatShape& inpshape = cs.inpshape; + const MatShape& outshape = cs.outshape; + + int N = outshape[0]; + int ndims = outshape.dims; + int K = outshape.channels(); + int C = inpshape.channels(); + int K1 = (K + K0 - 1) / K0; + int C1 = (C + C0 - 1) / C0; + + int D_ = ndims >= 6 ? outshape[ndims-4] : 1; + int H_ = ndims >= 5 ? outshape[ndims-3] : 1; + int W_ = outshape[ndims-2]; + int planeblocks_ = D_ * H_ * W_; + + int ngroups = cs.ngroups; + int Kg = K / ngroups; + int Cg = C / ngroups; + int Kblk = cs.wshape[1]; + int ksize_ = cs.wshape[2]; + int C1Max = cs.wshape[3]; + + const int8_t* inp = (const int8_t*)inp_; + const int8_t* residual = (const int8_t*)residual_; + int8_t* out = (int8_t*)out_; + const int8_t* wdata = (const int8_t*)weightsVNNI_; + const int* ofsZYX = cs.coordtab.data(); + + size_t outtotal = outshape.total(); + if ((Kg % K0) != 0) memset(out, 0, outtotal); + + const int8x16_t v_xor = inputIsU8 ? vdupq_n_s8((int8_t)0x80) : vdupq_n_s8(0); + + int Sz = cs.strides[0], Sy = cs.strides[1], Sx = cs.strides[2]; + + if (ksize_ == 1 && Sx == 1 && Sy == 1 && Sz == 1) { + int total_spatial = planeblocks_; + int iplanesize = total_spatial * C0; + int cblocks_g = (Cg + C0 - 1) / C0; + + int cblk_bytes = C0 * cblocks_g; + int SPAT_TILE = std::max(SPAT_BLOCK_SIZE, + ((48 * 1024) / std::max(1, cblk_bytes)) & ~(SPAT_BLOCK_SIZE - 1)); + SPAT_TILE = std::min(SPAT_TILE, total_spatial); + SPAT_TILE = std::max(SPAT_TILE, SPAT_BLOCK_SIZE); + + int wblk_bytes = C0 * K0 * cblocks_g; + int KG = std::min(Kblk, std::max(1, (48 * 1024) / std::max(1, wblk_bytes))); + + int ntiles = (total_spatial + SPAT_TILE - 1) / SPAT_TILE; + int nktiles = (Kblk + KG - 1) / KG; + + while (N * ngroups * ntiles * nktiles < 16 && KG > 1) { + KG = std::max(1, KG / 2); + nktiles = (Kblk + KG - 1) / KG; + } + + int total_tasks = N * ngroups * ntiles * nktiles; + + parallel_for_(Range(0, total_tasks), [&](const Range& range) { + for (int task = range.start; task < range.end; task++) { + int temp = task; + int n = temp / (ngroups * ntiles * nktiles); + temp -= n * (ngroups * ntiles * nktiles); + int g = temp / (ntiles * nktiles); + temp -= g * (ntiles * nktiles); + int tile_idx = temp / nktiles; + int kgroup_idx = temp - tile_idx * nktiles; + + int p_start = tile_idx * SPAT_TILE; + int p_end = std::min(p_start + SPAT_TILE, total_spatial); + + int kblk_start = kgroup_idx * KG; + int kblk_end = std::min(kblk_start + KG, Kblk); + + int c_start = g * Cg; + int c1_start = c_start >> C0shift; + int c00 = c_start & (C0 - 1); + int cblocks = (c00 + Cg + C0 - 1) >> C0shift; + + const int8_t* inpbase = inp + (size_t)(n * C1 + c1_start) * iplanesize; + int planesize = total_spatial * K0; + + for (int kblk = kblk_start; kblk < kblk_end; kblk++) { + int k_base = g * Kg + kblk * K0; + if (k_base >= K) continue; + + int k1 = k_base >> C0shift; + int8_t* outptr = out + (size_t)(n * K1 + k1) * planesize; + const int8_t* resptr = residual ? residual + (size_t)(n * K1 + k1) * planesize : nullptr; + const int8_t* wbaseptr = wdata + (size_t)(g * Kblk + kblk) * C1Max * C0 * K0; + + int32x4_t vbias_lo = vld1q_s32(bias + k_base); + int32x4_t vbias_hi = vld1q_s32(bias + k_base + 4); + float32x4_t vmult_lo = vld1q_f32(multiplier + k_base); + float32x4_t vmult_hi = vld1q_f32(multiplier + k_base + 4); + float32x4_t vzp = vdupq_n_f32((float)out_zp); + + int p = p_start; + for (; p <= p_end - SPAT_BLOCK_SIZE; p += SPAT_BLOCK_SIZE) { + int32x4_t s0_lo = vbias_lo, s0_hi = vbias_hi; + int32x4_t s1_lo = vbias_lo, s1_hi = vbias_hi; + int32x4_t s2_lo = vbias_lo, s2_hi = vbias_hi; + int32x4_t s3_lo = vbias_lo, s3_hi = vbias_hi; + int32x4_t s4_lo = vbias_lo, s4_hi = vbias_hi; + int32x4_t s5_lo = vbias_lo, s5_hi = vbias_hi; + int32x4_t s6_lo = vbias_lo, s6_hi = vbias_hi; + int32x4_t s7_lo = vbias_lo, s7_hi = vbias_hi; + + const int8_t* inpptr = inpbase + (size_t)p * C0; + const int8_t* wptr = wbaseptr; + + for (int c1 = 0; c1 < cblocks; c1++, wptr += C0 * K0) { + int8x16_t wg0 = vld1q_s8(wptr); + int8x16_t wg1 = vld1q_s8(wptr + 16); + int8x16_t wg2 = vld1q_s8(wptr + 32); + int8x16_t wg3 = vld1q_s8(wptr + 48); + + const int8_t* ip = inpptr; + #define CONV_NEON_MAC_1x1(j) { \ + int8x16_t x_lo = veorq_s8(vreinterpretq_s8_s32( \ + vdupq_n_s32(*(const int32_t*)&ip[0])), v_xor); \ + int8x16_t x_hi = veorq_s8(vreinterpretq_s8_s32( \ + vdupq_n_s32(*(const int32_t*)&ip[4])), v_xor); \ + s##j##_lo = vdotq_s32(s##j##_lo, x_lo, wg0); \ + s##j##_hi = vdotq_s32(s##j##_hi, x_lo, wg1); \ + s##j##_lo = vdotq_s32(s##j##_lo, x_hi, wg2); \ + s##j##_hi = vdotq_s32(s##j##_hi, x_hi, wg3); \ + ip += C0; \ + } + CONV_NEON_MAC_1x1(0); CONV_NEON_MAC_1x1(1); + CONV_NEON_MAC_1x1(2); CONV_NEON_MAC_1x1(3); + CONV_NEON_MAC_1x1(4); CONV_NEON_MAC_1x1(5); + CONV_NEON_MAC_1x1(6); CONV_NEON_MAC_1x1(7); + #undef CONV_NEON_MAC_1x1 + + inpptr += iplanesize; + } + + CONV_NEON_STORE(0); CONV_NEON_STORE(1); + CONV_NEON_STORE(2); CONV_NEON_STORE(3); + CONV_NEON_STORE(4); CONV_NEON_STORE(5); + CONV_NEON_STORE(6); CONV_NEON_STORE(7); + } + + if (p < p_end && p_end >= SPAT_BLOCK_SIZE) { + p = p_end - SPAT_BLOCK_SIZE; + int32x4_t s0_lo = vbias_lo, s0_hi = vbias_hi; + int32x4_t s1_lo = vbias_lo, s1_hi = vbias_hi; + int32x4_t s2_lo = vbias_lo, s2_hi = vbias_hi; + int32x4_t s3_lo = vbias_lo, s3_hi = vbias_hi; + int32x4_t s4_lo = vbias_lo, s4_hi = vbias_hi; + int32x4_t s5_lo = vbias_lo, s5_hi = vbias_hi; + int32x4_t s6_lo = vbias_lo, s6_hi = vbias_hi; + int32x4_t s7_lo = vbias_lo, s7_hi = vbias_hi; + + const int8_t* inpptr = inpbase + (size_t)p * C0; + const int8_t* wptr = wbaseptr; + for (int c1 = 0; c1 < cblocks; c1++, wptr += C0 * K0) { + int8x16_t wg0 = vld1q_s8(wptr); + int8x16_t wg1 = vld1q_s8(wptr + 16); + int8x16_t wg2 = vld1q_s8(wptr + 32); + int8x16_t wg3 = vld1q_s8(wptr + 48); + const int8_t* ip = inpptr; + #define CONV_NEON_MAC_T(j) { \ + int8x16_t x_lo = veorq_s8(vreinterpretq_s8_s32( \ + vdupq_n_s32(*(const int32_t*)&ip[0])), v_xor); \ + int8x16_t x_hi = veorq_s8(vreinterpretq_s8_s32( \ + vdupq_n_s32(*(const int32_t*)&ip[4])), v_xor); \ + s##j##_lo = vdotq_s32(s##j##_lo, x_lo, wg0); \ + s##j##_hi = vdotq_s32(s##j##_hi, x_lo, wg1); \ + s##j##_lo = vdotq_s32(s##j##_lo, x_hi, wg2); \ + s##j##_hi = vdotq_s32(s##j##_hi, x_hi, wg3); \ + ip += C0; \ + } + CONV_NEON_MAC_T(0); CONV_NEON_MAC_T(1); + CONV_NEON_MAC_T(2); CONV_NEON_MAC_T(3); + CONV_NEON_MAC_T(4); CONV_NEON_MAC_T(5); + CONV_NEON_MAC_T(6); CONV_NEON_MAC_T(7); + #undef CONV_NEON_MAC_T + inpptr += iplanesize; + } + CONV_NEON_STORE(0); CONV_NEON_STORE(1); + CONV_NEON_STORE(2); CONV_NEON_STORE(3); + CONV_NEON_STORE(4); CONV_NEON_STORE(5); + CONV_NEON_STORE(6); CONV_NEON_STORE(7); + } + } + } + }); + return; + } + + int innerZ0 = cs.inner[0], innerZ1 = cs.inner[MAX_CONV_DIMS]; + int innerY0 = cs.inner[1], innerY1 = cs.inner[MAX_CONV_DIMS+1]; + int innerX0 = cs.inner[2], innerX1 = cs.inner[MAX_CONV_DIMS+2]; + + int total_blocks = N * ngroups * Kblk; + + parallel_for_(Range(0, total_blocks), [&](const Range& range) { + int H = H_, W = W_; + int Di = ndims >= 6 ? inpshape[ndims-4] : 1; + int Hi = ndims >= 5 ? inpshape[ndims-3] : 1; + int Wi = inpshape[ndims-2]; + int planeblocks = planeblocks_; + int iplanesize = Di * Hi * Wi * C0; + int planesize = planeblocks * K0; + + int padZ = cs.pads[0], padY = cs.pads[1], padX = cs.pads[2]; + int ksize = ksize_; + int8_t zbuf[C0]; + memset(zbuf, (uint8_t)inp_zp, C0); + + for (int t = range.start; t < range.end; t++) { + int n = t / (ngroups * Kblk); + int rem = t - n * (ngroups * Kblk); + int g = rem / Kblk; + int kblk = rem - g * Kblk; + + int k_base = g * Kg + kblk * K0; + if (k_base >= K) continue; + + int c_start = g * Cg; + int c1_start = c_start >> C0shift; + int c00 = c_start & (C0 - 1); + int cblocks = (c00 + Cg + C0 - 1) >> C0shift; + + const int8_t* inpbaseptr = inp + (size_t)(n * C1 + c1_start) * iplanesize; + const int8_t* wbaseptr = wdata + (size_t)(g * Kblk + kblk) * ksize * C1Max * C0 * K0; + + int k1 = k_base >> C0shift; + int8_t* outptr = out + (size_t)(n * K1 + k1) * planesize; + const int8_t* resptr = residual ? residual + (size_t)(n * K1 + k1) * planesize : nullptr; + + int32x4_t vbias_lo = vld1q_s32(bias + k_base); + int32x4_t vbias_hi = vld1q_s32(bias + k_base + 4); + float32x4_t vmult_lo = vld1q_f32(multiplier + k_base); + float32x4_t vmult_hi = vld1q_f32(multiplier + k_base + 4); + float32x4_t vzp = vdupq_n_f32((float)out_zp); + + int H_l = H, W_l = W; + int Di_l = Di, Hi_l = Hi, Wi_l = Wi; + int iplanesize_l = iplanesize; + int planeblocks_l = planeblocks; + int ksize_l = ksize; + + int p = 0; + for (; p < planeblocks_l; p += SPAT_BLOCK_SIZE) { + if (p + SPAT_BLOCK_SIZE > planeblocks_l) { + if (p == 0) break; + p = planeblocks_l - SPAT_BLOCK_SIZE; + } + + Vec3i pt[SPAT_BLOCK_SIZE]; + bool inner[SPAT_BLOCK_SIZE]; + bool all_inner = true; + + if ((p % W_l) + SPAT_BLOCK_SIZE <= W_l) { + int zj = p / (H_l * W_l); + int yxj = p - zj * H_l * W_l; + int yj = yxj / W_l; + int xj0 = yxj - yj * W_l; + bool zy_inner = (zj >= innerZ0 && zj < innerZ1) && + (yj >= innerY0 && yj < innerY1); + for (int j = 0; j < SPAT_BLOCK_SIZE; j++) { + int xj = xj0 + j; + pt[j] = Vec3i(zj * Sz - padZ, yj * Sy - padY, xj * Sx - padX); + inner[j] = zy_inner && (xj >= innerX0 && xj < innerX1); + all_inner &= inner[j]; + } + } else { + for (int j = 0; j < SPAT_BLOCK_SIZE; j++) { + int pj = p + j; + int zj = pj / (H_l * W_l); + int yxj = pj - zj * H_l * W_l; + int yj = yxj / W_l; + int xj = yxj - yj * W_l; + pt[j] = Vec3i(zj * Sz - padZ, yj * Sy - padY, xj * Sx - padX); + inner[j] = (zj >= innerZ0 && zj < innerZ1) && + (yj >= innerY0 && yj < innerY1) && + (xj >= innerX0 && xj < innerX1); + all_inner &= inner[j]; + } + } + + int32x4_t s0_lo = vbias_lo, s0_hi = vbias_hi; + int32x4_t s1_lo = vbias_lo, s1_hi = vbias_hi; + int32x4_t s2_lo = vbias_lo, s2_hi = vbias_hi; + int32x4_t s3_lo = vbias_lo, s3_hi = vbias_hi; + int32x4_t s4_lo = vbias_lo, s4_hi = vbias_hi; + int32x4_t s5_lo = vbias_lo, s5_hi = vbias_hi; + int32x4_t s6_lo = vbias_lo, s6_hi = vbias_hi; + int32x4_t s7_lo = vbias_lo, s7_hi = vbias_hi; + + #define CONV_NEON_MAC(j) { \ + int8x16_t x_lo = veorq_s8(vreinterpretq_s8_s32( \ + vdupq_n_s32(*(const int32_t*)&inptr[(j)][0])), v_xor); \ + int8x16_t x_hi = veorq_s8(vreinterpretq_s8_s32( \ + vdupq_n_s32(*(const int32_t*)&inptr[(j)][4])), v_xor); \ + s##j##_lo = vdotq_s32(s##j##_lo, x_lo, wg0); \ + s##j##_hi = vdotq_s32(s##j##_hi, x_lo, wg1); \ + s##j##_lo = vdotq_s32(s##j##_lo, x_hi, wg2); \ + s##j##_hi = vdotq_s32(s##j##_hi, x_hi, wg3); \ + } + + if (all_inner) { + for (int i = 0; i < ksize_l; i++) { + const int8_t* inptr[SPAT_BLOCK_SIZE]; + for (int j = 0; j < SPAT_BLOCK_SIZE; j++) { + int zij = pt[j][0] + ofsZYX[i * 3]; + int yij = pt[j][1] + ofsZYX[i * 3 + 1]; + int xij = pt[j][2] + ofsZYX[i * 3 + 2]; + inptr[j] = inpbaseptr + (((zij * Hi_l) + yij) * Wi_l + xij) * C0; + } + + const int8_t* wptr = wbaseptr + (size_t)i * C1Max * K0 * C0; + + for (int c1 = 0; c1 < cblocks; c1++, wptr += C0 * K0) { + int8x16_t wg0 = vld1q_s8(wptr); + int8x16_t wg1 = vld1q_s8(wptr + 16); + int8x16_t wg2 = vld1q_s8(wptr + 32); + int8x16_t wg3 = vld1q_s8(wptr + 48); + + CONV_NEON_MAC(0); CONV_NEON_MAC(1); + CONV_NEON_MAC(2); CONV_NEON_MAC(3); + CONV_NEON_MAC(4); CONV_NEON_MAC(5); + CONV_NEON_MAC(6); CONV_NEON_MAC(7); + + for (int j = 0; j < SPAT_BLOCK_SIZE; j++) + inptr[j] += iplanesize_l; + } + } + } else { + for (int i = 0; i < ksize_l; i++) { + const int8_t* inptr[SPAT_BLOCK_SIZE]; + int inpstep[SPAT_BLOCK_SIZE]; + + for (int j = 0; j < SPAT_BLOCK_SIZE; j++) { + int zij = pt[j][0] + ofsZYX[i * 3]; + int yij = pt[j][1] + ofsZYX[i * 3 + 1]; + int xij = pt[j][2] + ofsZYX[i * 3 + 2]; + if (inner[j] || ((((unsigned)zij < (unsigned)Di_l) & + ((unsigned)yij < (unsigned)Hi_l) & + ((unsigned)xij < (unsigned)Wi_l)) != 0)) { + inptr[j] = inpbaseptr + (((zij * Hi_l) + yij) * Wi_l + xij) * C0; + inpstep[j] = iplanesize_l; + } else { + inptr[j] = zbuf; + inpstep[j] = 0; + } + } + + const int8_t* wptr = wbaseptr + (size_t)i * C1Max * K0 * C0; + + for (int c1 = 0; c1 < cblocks; c1++, wptr += C0 * K0) { + int8x16_t wg0 = vld1q_s8(wptr); + int8x16_t wg1 = vld1q_s8(wptr + 16); + int8x16_t wg2 = vld1q_s8(wptr + 32); + int8x16_t wg3 = vld1q_s8(wptr + 48); + + CONV_NEON_MAC(0); CONV_NEON_MAC(1); + CONV_NEON_MAC(2); CONV_NEON_MAC(3); + CONV_NEON_MAC(4); CONV_NEON_MAC(5); + CONV_NEON_MAC(6); CONV_NEON_MAC(7); + + for (int j = 0; j < SPAT_BLOCK_SIZE; j++) + inptr[j] += inpstep[j]; + } + } + } + #undef CONV_NEON_MAC + + CONV_NEON_STORE(0); CONV_NEON_STORE(1); + CONV_NEON_STORE(2); CONV_NEON_STORE(3); + CONV_NEON_STORE(4); CONV_NEON_STORE(5); + CONV_NEON_STORE(6); CONV_NEON_STORE(7); + } + + for (; p < planeblocks_l; p++) { + alignas(16) int32_t acc[K0]; + memcpy(acc, bias + k_base, K0 * sizeof(int32_t)); + + int zj = p / (H_l * W_l); + int yxj = p - zj * H_l * W_l; + int yj = yxj / W_l; + int xj = yxj - yj * W_l; + int zi_base = zj * Sz - padZ, yi_base = yj * Sy - padY, xi_base = xj * Sx - padX; + + for (int i = 0; i < ksize_l; i++) { + int zi = zi_base + ofsZYX[i * 3]; + int yi = yi_base + ofsZYX[i * 3 + 1]; + int xi = xi_base + ofsZYX[i * 3 + 2]; + if ((((unsigned)zi >= (unsigned)Di_l) | ((unsigned)yi >= (unsigned)Hi_l) | + ((unsigned)xi >= (unsigned)Wi_l)) != 0) continue; + const int8_t* inptr = inpbaseptr + (((zi * Hi_l) + yi) * Wi_l + xi) * C0; + const int8_t* wptr_vnni = wbaseptr + (size_t)i * C1Max * K0 * C0; + for (int c1i = 0; c1i < cblocks; c1i++, inptr += iplanesize_l) { + const int8_t* wp = wptr_vnni + (size_t)c1i * C0 * K0; + for (int c0 = 0; c0 < C0; c0++) { + int ival = inputIsU8 ? ((int)(uint8_t)inptr[c0] - 128) + : (int)inptr[c0]; + int base = (c0 < 4) ? 0 : 32; + int c_in_group = c0 & 3; + for (int k0 = 0; k0 < K0; k0++) + acc[k0] += ival * (int)wp[base + k0 * 4 + c_in_group]; + } + } + } + int8_t* optr = outptr + p * K0; + const int8_t* rptr = resptr ? resptr + p * K0 : nullptr; + for (int k0 = 0; k0 < K0; k0++) { + float val = (float)acc[k0] * multiplier[k_base + k0] + (float)out_zp; + if (rptr) { + int rval = inputIsU8 ? (int)(uint8_t)rptr[k0] : (int)rptr[k0]; + val += (float)(rval - out_zp); + } + int ival = cvRound(val); + if (inputIsU8) { + ival = std::max(0, std::min(255, ival)); + if (activLUT) ival = (int)(uint8_t)activLUT[ival]; + ((uint8_t*)optr)[k0] = (uint8_t)ival; + } else { + ival = std::max(-128, std::min(127, ival)); + if (activLUT) ival = (int)activLUT[ival + 128]; + optr[k0] = (int8_t)ival; + } + } + } + } + }); +} +#undef CONV_NEON_STORE +#endif // CV_NEON && CV_NEON_DOT + +void convInt8Block(const void* inp_, const void* residual_, + void* out_, const ConvState& cs, + const void* weights_, + const void* weightsVNNI_, + const int* bias, const int* biasVNNI_, + const float* multiplier, + int inp_zp, int out_zp, + const int8_t* activLUT, + bool inputIsU8) +{ +#if CV_AVXVNNI_AVAILABLE + if (cv::checkHardwareSupport(CV_CPU_AVX_VNNI) && weightsVNNI_ && biasVNNI_) { + convInt8BlockVNNI(inp_, residual_, out_, cs, weightsVNNI_, + biasVNNI_, multiplier, inp_zp, out_zp, activLUT, inputIsU8); + return; + } +#endif +#if CV_NEON && defined(CV_NEON_DOT) && CV_NEON_DOT + if (weightsVNNI_) { + convInt8BlockNEON(inp_, residual_, out_, cs, weightsVNNI_, + bias, multiplier, inp_zp, out_zp, activLUT, inputIsU8); + return; + } +#endif +#if !CV_AVXVNNI_AVAILABLE && !(CV_NEON && defined(CV_NEON_DOT) && CV_NEON_DOT) + (void)weightsVNNI_; + (void)biasVNNI_; +#else + (void)biasVNNI_; +#endif + + constexpr int C0 = 8, K0 = 8; + + CV_Assert(cs.inpshape.layout == DATA_LAYOUT_BLOCK); + CV_Assert(cs.outshape.layout == DATA_LAYOUT_BLOCK); + CV_Assert(cs.inpshape.back() == C0); + + const MatShape& inpshape = cs.inpshape; + const MatShape& outshape = cs.outshape; + + int N = outshape[0]; + int ndims = outshape.dims; + int K = outshape.channels(); + int C = inpshape.channels(); + int K1 = (K + K0 - 1) / K0; + int C1 = (C + C0 - 1) / C0; + + int D_ = ndims >= 6 ? outshape[ndims-4] : 1; + int H_ = ndims >= 5 ? outshape[ndims-3] : 1; + int W_ = outshape[ndims-2]; + int planeblocks_ = D_ * H_ * W_; + + int ngroups = cs.ngroups; + int Kg = K / ngroups; + int Cg = C / ngroups; + int Kblk = cs.wshape[1]; + int ksize_ = cs.wshape[2]; + int C1Max = cs.wshape[3]; + +#if CV_AVX2 + constexpr int MAX_CONV_DIMS = ConvState::MAX_CONV_DIMS; + int innerZ0 = cs.inner[0], innerZ1 = cs.inner[MAX_CONV_DIMS]; + int innerY0 = cs.inner[1], innerY1 = cs.inner[MAX_CONV_DIMS+1]; + int innerX0 = cs.inner[2], innerX1 = cs.inner[MAX_CONV_DIMS+2]; +#endif + + const int8_t* inp = (const int8_t*)inp_; + const int8_t* residual = (const int8_t*)residual_; + int8_t* out = (int8_t*)out_; + const int8_t* wdata = (const int8_t*)weights_; + const int* ofsZYX = cs.coordtab.data(); + + size_t outtotal = outshape.total(); + if ((Kg % K0) != 0) { + memset(out, 0, outtotal); + } + + int total_blocks = N * ngroups * Kblk; + + parallel_for_(Range(0, total_blocks), [&](const Range& range) { + constexpr int C0 = 8, K0 = 8; + constexpr int C0shift = 3; +#if CV_AVX2 + constexpr int SPAT_BLOCK_SIZE = 8; + alignas(32) int32_t sumbuf[SPAT_BLOCK_SIZE * K0]; +#endif + const uint8_t xor_val = inputIsU8 ? 0x80 : 0; + + int D = D_, H = H_, W = W_; + int Di = ndims >= 6 ? inpshape[ndims-4] : 1; + int Hi = ndims >= 5 ? inpshape[ndims-3] : 1; + int Wi = inpshape[ndims-2]; + int planeblocks = planeblocks_; + int iplanesize = Di * Hi * Wi * C0; + int planesize = planeblocks * K0; + + int Sz = cs.strides[0], Sy = cs.strides[1], Sx = cs.strides[2]; + int padZ = cs.pads[0], padY = cs.pads[1], padX = cs.pads[2]; + int ksize = ksize_; + int8_t zbuf[C0]; + memset(zbuf, (uint8_t)inp_zp, C0); + + for (int t = range.start; t < range.end; t++) { + int n = t / (ngroups * Kblk); + int rem = t - n * (ngroups * Kblk); + int g = rem / Kblk; + int kblk = rem - g * Kblk; + + int k_base = g * Kg + kblk * K0; + if (k_base >= K) continue; + + int c_start = g * Cg; + int c1_start = c_start >> C0shift; + int c00 = c_start & (C0 - 1); + int cblocks = (c00 + Cg + C0 - 1) >> C0shift; + + const int8_t* inpbaseptr = inp + (size_t)(n * C1 + c1_start) * iplanesize; + const int8_t* wbaseptr = wdata + (size_t)(g * Kblk + kblk) * ksize * C1Max * C0 * K0; + + int k1 = k_base >> C0shift; + int8_t* outptr = out + (size_t)(n * K1 + k1) * planesize; + const int8_t* resptr = residual ? residual + (size_t)(n * K1 + k1) * planesize : nullptr; + + int D_l = D, H_l = H, W_l = W; + int Di_l = Di, Hi_l = Hi, Wi_l = Wi; + int iplanesize_l = iplanesize; + int planeblocks_l = planeblocks; + int ksize_l = ksize; + + if (ksize == 1 && Sx == 1 && Sy == 1 && Sz == 1) { + W_l *= D_l * H_l; + Wi_l *= Di_l * Hi_l; + D_l = Di_l = H_l = Hi_l = 1; + iplanesize_l = Wi_l * C0; + planeblocks_l = W_l; + } + + alignas(32) int32_t biasbuf[K0]; + memcpy(biasbuf, bias + k_base, K0 * sizeof(int32_t)); + + alignas(32) float multbuf[K0]; + memcpy(multbuf, multiplier + k_base, K0 * sizeof(float)); + + #if CV_AVX2 + int p = 0; + for (; p < planeblocks_l; p += SPAT_BLOCK_SIZE) { + if (p + SPAT_BLOCK_SIZE > planeblocks_l) { + if (p == 0) break; + p = planeblocks_l - SPAT_BLOCK_SIZE; + } + + Vec3i pt[SPAT_BLOCK_SIZE]; + bool inner[SPAT_BLOCK_SIZE]; + + if ((p % W_l) + SPAT_BLOCK_SIZE <= W_l) { + int zj = p / (H_l * W_l); + int yxj = p - zj * H_l * W_l; + int yj = yxj / W_l; + int xj0 = yxj - yj * W_l; + bool zy_inner = (zj >= innerZ0 && zj < innerZ1) && + (yj >= innerY0 && yj < innerY1); + for (int j = 0; j < SPAT_BLOCK_SIZE; j++) { + int xj = xj0 + j; + pt[j] = Vec3i(zj * Sz - padZ, yj * Sy - padY, xj * Sx - padX); + inner[j] = zy_inner && (xj >= innerX0 && xj < innerX1); + } + } else { + for (int j = 0; j < SPAT_BLOCK_SIZE; j++) { + int pj = p + j; + int zj = pj / (H_l * W_l); + int yxj = pj - zj * H_l * W_l; + int yj = yxj / W_l; + int xj = yxj - yj * W_l; + pt[j] = Vec3i(zj * Sz - padZ, yj * Sy - padY, xj * Sx - padX); + inner[j] = (zj >= innerZ0 && zj < innerZ1) && + (yj >= innerY0 && yj < innerY1) && + (xj >= innerX0 && xj < innerX1); + } + } + + __m256i vbias = _mm256_load_si256((const __m256i*)biasbuf); + __m256i s0 = vbias, s1 = vbias, s2 = vbias, s3 = vbias; + __m256i s4 = vbias, s5 = vbias, s6 = vbias, s7 = vbias; + + for (int i = 0; i < ksize_l; i++) { + const int8_t* inptr[SPAT_BLOCK_SIZE]; + int inpstep[SPAT_BLOCK_SIZE]; + + for (int j = 0; j < SPAT_BLOCK_SIZE; j++) { + int zij = pt[j][0] + ofsZYX[i * 3]; + int yij = pt[j][1] + ofsZYX[i * 3 + 1]; + int xij = pt[j][2] + ofsZYX[i * 3 + 2]; + + if (inner[j] || ((((unsigned)zij < (unsigned)Di_l) & + ((unsigned)yij < (unsigned)Hi_l) & + ((unsigned)xij < (unsigned)Wi_l)) != 0)) { + inptr[j] = inpbaseptr + (((zij * Hi_l) + yij) * Wi_l + xij) * C0; + inpstep[j] = iplanesize_l; + } else { + inptr[j] = zbuf; + inpstep[j] = 0; + } + } + + const int8_t* wptr = wbaseptr + (size_t)i * C1Max * K0 * C0; + + for (int c1 = 0; c1 < cblocks; c1++, wptr += C0 * K0) { + #define CONV_INT8_LOAD(j, c0) \ + _mm256_set1_epi32( \ + (uint16_t)(int16_t)(int8_t)((uint8_t)inptr[j][(c0)] ^ xor_val) | \ + ((uint32_t)(uint16_t)(int16_t)(int8_t)((uint8_t)inptr[j][(c0)+1] ^ xor_val) << 16)) + #define CONV_INT8_MAC_PAIR(c0) { \ + __m128i wraw = _mm_loadu_si128((const __m128i*)(wptr + (c0) * K0)); \ + __m128i w_intlv = _mm_unpacklo_epi8(wraw, _mm_srli_si128(wraw, 8)); \ + __m256i w16 = _mm256_cvtepi8_epi16(w_intlv); \ + __m256i x0 = CONV_INT8_LOAD(0, c0); \ + __m256i x1 = CONV_INT8_LOAD(1, c0); \ + __m256i x2 = CONV_INT8_LOAD(2, c0); \ + __m256i x3 = CONV_INT8_LOAD(3, c0); \ + __m256i x4 = CONV_INT8_LOAD(4, c0); \ + __m256i x5 = CONV_INT8_LOAD(5, c0); \ + __m256i x6 = CONV_INT8_LOAD(6, c0); \ + __m256i x7 = CONV_INT8_LOAD(7, c0); \ + s0 = _mm256_add_epi32(s0, _mm256_madd_epi16(x0, w16)); \ + s1 = _mm256_add_epi32(s1, _mm256_madd_epi16(x1, w16)); \ + s2 = _mm256_add_epi32(s2, _mm256_madd_epi16(x2, w16)); \ + s3 = _mm256_add_epi32(s3, _mm256_madd_epi16(x3, w16)); \ + s4 = _mm256_add_epi32(s4, _mm256_madd_epi16(x4, w16)); \ + s5 = _mm256_add_epi32(s5, _mm256_madd_epi16(x5, w16)); \ + s6 = _mm256_add_epi32(s6, _mm256_madd_epi16(x6, w16)); \ + s7 = _mm256_add_epi32(s7, _mm256_madd_epi16(x7, w16)); \ + } + CONV_INT8_MAC_PAIR(0); + CONV_INT8_MAC_PAIR(2); + CONV_INT8_MAC_PAIR(4); + CONV_INT8_MAC_PAIR(6); + #undef CONV_INT8_MAC_PAIR + #undef CONV_INT8_LOAD + + for (int j = 0; j < SPAT_BLOCK_SIZE; j++) + inptr[j] += inpstep[j]; + } + } + + // Store accumulators + _mm256_store_si256((__m256i*)(sumbuf + 0*K0), s0); + _mm256_store_si256((__m256i*)(sumbuf + 1*K0), s1); + _mm256_store_si256((__m256i*)(sumbuf + 2*K0), s2); + _mm256_store_si256((__m256i*)(sumbuf + 3*K0), s3); + _mm256_store_si256((__m256i*)(sumbuf + 4*K0), s4); + _mm256_store_si256((__m256i*)(sumbuf + 5*K0), s5); + _mm256_store_si256((__m256i*)(sumbuf + 6*K0), s6); + _mm256_store_si256((__m256i*)(sumbuf + 7*K0), s7); + + __m256 vmult = _mm256_load_ps(multbuf); + __m256 vzp = _mm256_set1_ps((float)out_zp); + + for (int j = 0; j < SPAT_BLOCK_SIZE; j++) { + __m256i acc = _mm256_load_si256((const __m256i*)(sumbuf + j * K0)); + __m256 facc = _mm256_cvtepi32_ps(acc); + facc = _mm256_add_ps(_mm256_mul_ps(facc, vmult), vzp); + + if (resptr) { + const int8_t* rptr = resptr + (p + j) * K0; + __m256i res32 = inputIsU8 ? + _mm256_cvtepu8_epi32(_mm_loadl_epi64((const __m128i*)rptr)) : + _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*)rptr)); + facc = _mm256_add_ps(facc, + _mm256_sub_ps(_mm256_cvtepi32_ps(res32), vzp)); + } + + __m256i ival = _mm256_cvtps_epi32(facc); + __m128i lo = _mm256_castsi256_si128(ival); + __m128i hi = _mm256_extracti128_si256(ival, 1); + __m128i packed16 = _mm_packs_epi32(lo, hi); + __m128i packed8 = inputIsU8 ? _mm_packus_epi16(packed16, packed16) + : _mm_packs_epi16(packed16, packed16); + + int8_t* optr = outptr + (p + j) * K0; + if (activLUT) { + alignas(16) int8_t tmp8[16]; + _mm_store_si128((__m128i*)tmp8, packed8); + for (int k = 0; k < K0; k++) + optr[k] = (int8_t)activLUT[(int)tmp8[k] + 128]; + } else { + _mm_storel_epi64((__m128i*)optr, packed8); + } + } + } + + for (; p < planeblocks_l; p++) + #else + for (int p = 0; p < planeblocks_l; p++) + #endif + { + alignas(32) int32_t acc[K0]; + memcpy(acc, biasbuf, K0 * sizeof(int32_t)); + + int zj = p / (H_l * W_l); + int yxj = p - zj * H_l * W_l; + int yj = yxj / W_l; + int xj = yxj - yj * W_l; + int zi_base = zj * Sz - padZ; + int yi_base = yj * Sy - padY; + int xi_base = xj * Sx - padX; + + for (int i = 0; i < ksize_l; i++) { + int zi = zi_base + ofsZYX[i * 3]; + int yi = yi_base + ofsZYX[i * 3 + 1]; + int xi = xi_base + ofsZYX[i * 3 + 2]; + + bool out_of_bounds = (((unsigned)zi >= (unsigned)Di_l) | + ((unsigned)yi >= (unsigned)Hi_l) | + ((unsigned)xi >= (unsigned)Wi_l)) != 0; + + const int8_t* inptr = out_of_bounds ? zbuf : + inpbaseptr + (((zi * Hi_l) + yi) * Wi_l + xi) * C0; + int inpstep = out_of_bounds ? 0 : iplanesize_l; + const int8_t* wptr = wbaseptr + (size_t)i * C1Max * K0 * C0; + + for (int c1 = 0; c1 < cblocks; c1++, inptr += inpstep, wptr += C0 * K0) { + for (int c0 = 0; c0 < C0; c0++) { + int ival = (int)(int8_t)((uint8_t)inptr[c0] ^ xor_val); + const int8_t* wp = wptr + c0 * K0; + for (int k0 = 0; k0 < K0; k0++) + acc[k0] += ival * (int)wp[k0]; + } + } + } + + int8_t* optr = outptr + p * K0; + const int8_t* rptr = resptr ? resptr + p * K0 : nullptr; + for (int k0 = 0; k0 < K0; k0++) { + float val = (float)acc[k0] * multbuf[k0] + (float)out_zp; + if (rptr) { + int rval = inputIsU8 ? (int)(uint8_t)rptr[k0] : (int)rptr[k0]; + val += (float)(rval - out_zp); + } + int ival = cvRound(val); + if (inputIsU8) { + ival = std::max(0, std::min(255, ival)); + if (activLUT) ival = (int)(uint8_t)activLUT[ival]; + ((uint8_t*)optr)[k0] = (uint8_t)ival; + } else { + ival = std::max(-128, std::min(127, ival)); + if (activLUT) ival = (int)activLUT[ival + 128]; + optr[k0] = (int8_t)ival; + } + } + } + } + }); +} + +#endif // !CV_CPU_OPTIMIZATION_DECLARATIONS_ONLY + +CV_CPU_OPTIMIZATION_NAMESPACE_END +}} // namespace cv::dnn diff --git a/modules/dnn/src/int8layers/conv2_int8_layer.cpp b/modules/dnn/src/int8layers/conv2_int8_layer.cpp new file mode 100644 index 0000000000..2f66292294 --- /dev/null +++ b/modules/dnn/src/int8layers/conv2_int8_layer.cpp @@ -0,0 +1,371 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// Copyright (C) 2026, BigVision LLC, all rights reserved. +// Third party copyrights are property of their respective owners. + +#include "../precomp.hpp" +#include "../net_impl.hpp" +#include "../layers/conv2_common.hpp" +#include "opencv2/core/hal/intrin.hpp" + +#include "conv2_int8_kernels.simd.hpp" +#include "int8layers/conv2_int8_kernels.simd_declarations.hpp" + +namespace cv { +namespace dnn { + +static MatShape getWpackShapeInt8(const MatShape& wshape, int ngroups, int C0) +{ + CV_Assert(wshape.dims >= 3); + int K = wshape[0], Cg = wshape[1]; + int ksize = (int)(wshape.total()) / (K * Cg); + CV_Assert(K % ngroups == 0); + int Kg = K / ngroups, K0 = C0; + int Kblk = (Kg + K0 - 1) / K0; + int C1Max = 0; + for (int g = 0; g < ngroups; ++g) { + int c_start = g * Cg; + int c00 = c_start & (C0 - 1); + int cblocks = (c00 + Cg + C0 - 1) / C0; + C1Max = std::max(C1Max, cblocks); + } + return MatShape({ngroups, Kblk, ksize, C1Max, C0 * K0}, DATA_LAYOUT_UNKNOWN); +} + +static void repackConvWeightsInt8(const Mat& weights, Mat& Wpack, int ngroups, int C0_) +{ + CV_Assert(weights.isContinuous()); + CV_Assert(weights.type() == CV_8SC1); + CV_Assert(ngroups > 0); + CV_Assert((C0_ & (C0_ - 1)) == 0 && C0_ >= 4); + + MatShape wshape = weights.shape(); + CV_Assert(wshape.dims >= 3); + + int K = wshape[0]; + CV_Assert(K % ngroups == 0); + + if (!Wpack.isContinuous()) + Wpack.release(); + + MatShape wpackShape = getWpackShapeInt8(wshape, ngroups, C0_); + Wpack.create(wpackShape, CV_8SC1); + Wpack.setZero(); + + parallel_for_(Range(0, K), [&](const Range& range) { + int Cg = wshape[1], Kg = K / ngroups; + int ksize = wpackShape[2], C1Max = wpackShape[3]; + int C0 = C0_, K0 = C0; + const int8_t* wdata = weights.ptr(); + int8_t* Wpackdata = Wpack.ptr(); + + for (int k = range.start; k < range.end; ++k) { + int g = k / Kg; + int kin = k - g * Kg; + int kblk = kin / K0; + int k0 = kin & (K0 - 1); + + int c_start = g * Cg; + int c00 = c_start & (C0 - 1); + + for (int c = 0; c < Cg; ++c) { + int ch = c00 + c; + int c1 = ch / C0; + int c0 = ch & (C0 - 1); + + const int8_t* wptr = wdata + ((k * Cg + c) * ksize); + int8_t* wpackptr = Wpackdata + + (((g * (int64_t)wpackShape[1] + kblk) * ksize * C1Max + c1) * C0 + c0) * K0 + k0; + for (int i = 0; i < ksize; ++i) { + wpackptr[i * (C1Max * C0 * K0)] = wptr[i]; + } + } + } + }); +} + +static void repackWeightsForVNNI(const Mat& wpack, int ngroups, int Kg, int Cg, + const Mat& biasInt32, + Mat& wpackVNNI, Mat& biasVNNI) +{ + constexpr int C0 = 8, K0 = 8; + MatShape ws = wpack.shape(); + int Kblk = ws[1], ksize = ws[2], C1Max = ws[3]; + + wpackVNNI.create(ws, CV_8SC1); + const int8_t* src = wpack.ptr(); + int8_t* dst = wpackVNNI.ptr(); + + int blockSize = C0 * K0; + int totalBlocks = (int)(ws.total() / blockSize); + + parallel_for_(Range(0, totalBlocks), [&](const Range& range) { + for (int b = range.start; b < range.end; b++) { + const int8_t* s = src + (size_t)b * blockSize; + int8_t* d = dst + (size_t)b * blockSize; + for (int k = 0; k < K0; k++) { + d[k*4 + 0] = s[0*K0 + k]; + d[k*4 + 1] = s[1*K0 + k]; + d[k*4 + 2] = s[2*K0 + k]; + d[k*4 + 3] = s[3*K0 + k]; + d[32 + k*4 + 0] = s[4*K0 + k]; + d[32 + k*4 + 1] = s[5*K0 + k]; + d[32 + k*4 + 2] = s[6*K0 + k]; + d[32 + k*4 + 3] = s[7*K0 + k]; + } + } + }); + + int K = ngroups * Kg; + biasVNNI.create({K}, CV_32SC1); + const int32_t* bsrc = biasInt32.ptr(); + int32_t* bdst = biasVNNI.ptr(); + + parallel_for_(Range(0, K), [&](const Range& range) { + for (int k = range.start; k < range.end; k++) { + int g = k / Kg; + int kin = k - g * Kg; + int kblk = kin / K0; + int k0 = kin & (K0 - 1); + int c_start = g * Cg; + int c00 = c_start & (C0 - 1); + int cblocks = (c00 + Cg + C0 - 1) / C0; + + const int8_t* wbase = src + (size_t)((g * Kblk + kblk) * ksize * C1Max) * C0 * K0; + int32_t wsum = 0; + for (int ki = 0; ki < ksize; ki++) + for (int c1 = 0; c1 < cblocks; c1++) + for (int c0 = 0; c0 < C0; c0++) + wsum += (int)wbase[((size_t)ki * C1Max + c1) * C0 * K0 + c0 * K0 + k0]; + bdst[k] = bsrc[k] - 128 * wsum; + } + }); +} + +static void convInt8Block(const void* inp_, const void* residual_, + void* out_, const ConvState& cs, + const void* weights_, + const void* weightsVNNI_, + const int* bias, const int* biasVNNI_, + const float* multiplier, + int inp_zp, int out_zp, + const int8_t* activLUT, + bool inputIsU8) +{ +#if CV_TRY_AVX2 + if (cv::checkHardwareSupport(CV_CPU_AVX2)) { + opt_AVX2::convInt8Block(inp_, residual_, out_, cs, weights_, + weightsVNNI_, bias, biasVNNI_, + multiplier, inp_zp, out_zp, activLUT, inputIsU8); + return; + } +#endif + CV_CPU_CALL_BASELINE(convInt8Block, (inp_, residual_, out_, cs, weights_, + weightsVNNI_, bias, biasVNNI_, + multiplier, inp_zp, out_zp, activLUT, inputIsU8)); +} + + +class Conv2Int8LayerImpl CV_FINAL : public Conv2Int8Layer +{ +public: + Mat weights; // repacked int8 weights in block format + Mat weightsVNNI; // VNNI-transposed weights (pre-computed) + Mat biasInt32; // int32 fused bias + Mat biasVNNI; // int32 VNNI-adjusted bias (pre-computed) + Mat outMultiplier; // float32 per-channel output multiplier + MatShape wshape0; // original weight shape (K x Cg x kH x kW) + MatShape prevInpshape; + ConvState cs; + + Mat activationLUT; + Ptr activ; + + bool addResidual; + bool inputIsU8; + + Conv2Int8LayerImpl(const LayerParams& params) + { + setParamsFrom(params); + auto_pad = getAutoPadding(params); + ceil_mode = params.get("ceil_mode", false); + strides = params.getVector("stride"); + dilations = params.getVector("dilation"); + pads = params.getVector("pad"); + ngroups = params.get("group", 1); + + input_sc = params.get("input_scale", 1.f); + input_zp = params.get("input_zeropoint", 0); + output_sc = params.get("scales", 1.f); + output_zp = params.get("zeropoints", 0); + per_channel = params.get("per_channel", true); + inputIsU8 = params.get("input_is_u8", false); + + addResidual = false; + + if (!blobs.empty()) { + CV_Assert(blobs.size() >= 3); + wshape0 = blobs[0].shape(); + biasInt32 = blobs[1]; + outMultiplier = blobs[2]; + } + } + + void getTypes(const std::vector& inputs, + const int requiredOutputs, const int, + std::vector& outputs, + std::vector& internals) const CV_OVERRIDE + { + // Input can be CV_8SC1 or CV_8UC1; output matches input type + int outtype = !inputs.empty() ? inputs[0] : CV_8SC1; + outputs.assign(requiredOutputs, outtype); + internals.clear(); + } + + bool getMemoryShapes(const std::vector& inpshapes, + const int, + std::vector& outshapes, + std::vector& tempshapes) const CV_OVERRIDE + { + size_t ninputs = inpshapes.size(); + CV_Assert(ninputs >= 1); + + std::vector emptyKernelShape; + outshapes.assign(1, convInferShape(inpshapes[0], wshape0, emptyKernelShape, + ngroups, strides, dilations, + pads, auto_pad, ceil_mode)); + tempshapes.clear(); + return true; + } + + int getLayouts(const std::vector& actualInputs, + std::vector& desiredInputs, + const int requiredOutputs, + std::vector& outputs) const CV_OVERRIDE + { + size_t ninputs = actualInputs.size(); + CV_Assert(ninputs >= 1u && requiredOutputs == 1u); + desiredInputs = actualInputs; + desiredInputs[0] = DATA_LAYOUT_BLOCK; + for (size_t i = 1; i < ninputs; i++) + desiredInputs[i] = DATA_LAYOUT_UNKNOWN; + outputs.assign(requiredOutputs, DATA_LAYOUT_BLOCK); + return getNetImpl(this)->defaultC0; + } + + void setWeightsInt8(const Mat& w_q, const Mat& bias, const Mat& multiplier, int C0) + { + CV_Assert(w_q.type() == CV_8SC1); + wshape0 = w_q.shape(); + biasInt32 = bias; + outMultiplier = multiplier; + + repackConvWeightsInt8(w_q, weights, ngroups, C0); + } + + bool fuseAddResidual(Arg /*residual*/) + { + addResidual = true; + return true; + } + + bool setActivation(const Ptr& layer) CV_OVERRIDE + { + Ptr activ_int8 = layer.dynamicCast(); + if (!activ_int8.empty()) { + activ = activ_int8; + if (!activ_int8->blobs.empty()) + activ_int8->blobs[0].convertTo(activationLUT, CV_8S); + return true; + } + return false; + } + + void forward(InputArrayOfArrays input_arrs, + OutputArrayOfArrays output_arrs, + OutputArrayOfArrays) CV_OVERRIDE + { + int ninputs = (int)input_arrs.total(); + CV_Assert(ninputs >= 1); + + const Mat& inp = input_arrs.getMat(0); + MatShape inpshape = inp.shape(); + CV_Assert(inpshape.layout == DATA_LAYOUT_BLOCK); + CV_Assert(inp.type() == CV_8SC1 || inp.type() == CV_8UC1); + + int C0 = inpshape.back(); + + if (weights.empty() && !blobs.empty()) { + repackConvWeightsInt8(blobs[0], weights, ngroups, C0); + } + + if (weightsVNNI.empty() && !weights.empty()) { + int K = wshape0[0]; + int Cg = wshape0[1]; + int Kg = K / ngroups; + repackWeightsForVNNI(weights, ngroups, Kg, Cg, + biasInt32, weightsVNNI, biasVNNI); + } + + Mat residual; + const void* resptr = nullptr; + if (addResidual) { + residual = input_arrs.getMat(ninputs - 1); + resptr = residual.data; + ninputs--; + } + + std::vector emptyKernelShape; + MatShape outshape = convInferShape(inpshape, wshape0, emptyKernelShape, + ngroups, strides, dilations, + pads, auto_pad, ceil_mode); + int outtype = inp.type(); + + int outkind = output_arrs.kind(); + Mat out; + + if (outkind == _InputArray::STD_VECTOR_MAT) { + std::vector& outs = output_arrs.getMatVecRef(); + outs.resize(1); + outs[0].fit(outshape, outtype); + out = outs[0]; + } else { + std::vector& outs = output_arrs.getUMatVecRef(); + outs.resize(1); + outs[0].fit(outshape, outtype); + out.fit(outshape, outtype); + } + + if (inpshape != prevInpshape) { + cs.initConv(inpshape, wshape0, outshape, ngroups, + strides, dilations, pads, auto_pad, ceil_mode, + FAST_ACTIV_NONE, {}); + prevInpshape = inpshape; + } + + const int8_t* lutptr = !activationLUT.empty() ? activationLUT.ptr() : nullptr; + + convInt8Block(inp.data, resptr, out.data, cs, + weights.data, + weightsVNNI.empty() ? nullptr : weightsVNNI.data, + biasInt32.ptr(), + biasVNNI.empty() ? nullptr : biasVNNI.ptr(), + outMultiplier.ptr(), + input_zp, output_zp, + lutptr, inputIsU8); + + if (outkind == _InputArray::STD_VECTOR_UMAT) { + std::vector& outs = output_arrs.getUMatVecRef(); + out.copyTo(outs[0]); + } + } +}; + +Ptr Conv2Int8Layer::create(const LayerParams& params) +{ + return Ptr(new Conv2Int8LayerImpl(params)); +} + +} // namespace dnn +} // namespace cv diff --git a/modules/dnn/src/int8layers/eltwise2_int8_layer.cpp b/modules/dnn/src/int8layers/eltwise2_int8_layer.cpp new file mode 100644 index 0000000000..875921364e --- /dev/null +++ b/modules/dnn/src/int8layers/eltwise2_int8_layer.cpp @@ -0,0 +1,413 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// Copyright (C) 2026, BigVision LLC, all rights reserved. +// Third party copyrights are property of their respective owners. + +#include "../precomp.hpp" +#include "../net_impl.hpp" +#include "opencv2/core/hal/intrin.hpp" + +#if CV_AVX2 +#include +#endif +#if CV_NEON +#include +#endif + +namespace cv { +namespace dnn { + +class Eltwise2Int8LayerImpl CV_FINAL : public Eltwise2Int8Layer +{ +public: + std::vector coeffs; + float offset; + bool withRelu; + + Mat activationLUT; + Ptr activ; + + Eltwise2Int8LayerImpl(const LayerParams& params) + { + setParamsFrom(params); + + output_zp = params.get("zeropoints", 0); + output_sc = params.get("scales", 1.0f); + withRelu = params.get("with_relu", false); + + if (params.has("input_scales")) + { + DictValue sc = params.get("input_scales"); + int n = sc.size(); + scales.resize(n); + for (int i = 0; i < n; i++) + scales[i] = sc.get(i); + } + + if (params.has("input_zeropoints")) + { + DictValue zp = params.get("input_zeropoints"); + int n = zp.size(); + zeropoints.resize(n); + for (int i = 0; i < n; i++) + zeropoints[i] = zp.get(i); + } + + offset = 0.f; + } + + void ensureCoeffs() + { + if (!coeffs.empty() || scales.empty()) + return; + + CV_CheckEQ(scales.size(), zeropoints.size(), + "Eltwise2Int8: scales and zeropoints sizes must match"); + CV_Assert(output_sc > 0.f); + + coeffs.resize(scales.size()); + offset = (float)output_zp; + for (size_t i = 0; i < scales.size(); i++) + { + coeffs[i] = scales[i] / output_sc; + offset -= coeffs[i] * zeropoints[i]; + } + } + + bool getMemoryShapes(const std::vector& inputs, + const int requiredOutputs, + std::vector& outputs, + std::vector& internals) const CV_OVERRIDE + { + CV_Assert(inputs.size() >= 2); + outputs.assign(1, inputs[0]); + internals.clear(); + return true; + } + + void getTypes(const std::vector& inputs, + const int requiredOutputs, + const int requiredInternals, + std::vector& outputs, + std::vector& internals) const CV_OVERRIDE + { + CV_Assert(!inputs.empty()); + outputs.assign(requiredOutputs, inputs[0]); + internals.clear(); + } + + int getLayouts(const std::vector& actualInputs, + std::vector& desiredInputs, + const int requiredOutputs, + std::vector& outputs) const CV_OVERRIDE + { + auto* netimpl_ = getNetImpl(this); + DataLayout defaultLayout = netimpl_->originalLayout; + size_t ninputs = actualInputs.size(), nblockInputs = 0; + CV_Assert(ninputs >= 1u); + for (size_t i = 0; i < ninputs; i++) + nblockInputs += actualInputs[i] == DATA_LAYOUT_BLOCK; + + desiredInputs = actualInputs; + if (nblockInputs == ninputs) { + outputs.assign(requiredOutputs, DATA_LAYOUT_BLOCK); + } else { + if (nblockInputs < ninputs) { + for (size_t i = 0; i < ninputs; i++) { + DataLayout layout = actualInputs[i]; + desiredInputs[i] = layout == DATA_LAYOUT_BLOCK ? defaultLayout : layout; + } + } + outputs.assign(requiredOutputs, DATA_LAYOUT_UNKNOWN); + } + return outputs[0] == DATA_LAYOUT_BLOCK ? netimpl_->defaultC0 : 0; + } + + void forward(InputArrayOfArrays input_arrs, + OutputArrayOfArrays output_arrs, + OutputArrayOfArrays) CV_OVERRIDE + { + ensureCoeffs(); + + int ninputs = (int)input_arrs.total(); + CV_Assert(ninputs >= 2 && coeffs.size() == (size_t)ninputs); + + const Mat& inp0 = input_arrs.getMat(0); + MatShape outshape = inp0.shape(); + int outtype = inp0.type(); + CV_Assert(outtype == CV_8SC1 || outtype == CV_8UC1); + const bool isU8 = (outtype == CV_8UC1); + + int outkind = output_arrs.kind(); + std::vector* outs = nullptr; + std::vector* uouts = nullptr; + Mat out; + + if (outkind == _InputArray::STD_VECTOR_MAT) { + outs = &output_arrs.getMatVecRef(); + outs->resize(1); + outs->at(0).fit(outshape, outtype); + out = outs->at(0); + } else { + uouts = &output_arrs.getUMatVecRef(); + uouts->resize(1); + uouts->at(0).fit(outshape, outtype); + out.fit(outshape, outtype); + } + + const size_t total_elems = inp0.total(); + const float* cptr = coeffs.data(); + const float off = offset; + const int8_t* lutptr = !activationLUT.empty() ? activationLUT.ptr() : nullptr; + + const float c0 = cptr[0]; + const float c1 = cptr[1]; + + const int out_min = isU8 ? (withRelu ? output_zp : 0) : (withRelu ? output_zp : -128); + const int out_max = isU8 ? 255 : 127; + + const double nstripes = (double)std::max(1, getNumThreads() * 4); + + if (isU8) { + std::vector inptrs(ninputs); + for (int k = 0; k < ninputs; k++) + inptrs[k] = input_arrs.getMat(k).ptr(); + const uint8_t* p0 = inptrs[0]; + const uint8_t* p1 = inptrs[1]; + uint8_t* outptr = out.ptr(); + + parallel_for_(Range(0, (int)total_elems), [&](const Range& r) { + int i = r.start; + #if CV_AVX2 + if (!lutptr && ninputs == 2) { + __m256 vc0 = _mm256_set1_ps(c0); + __m256 vc1 = _mm256_set1_ps(c1); + __m256 voff = _mm256_set1_ps(off); + __m256i vmin = _mm256_set1_epi32(out_min); + __m256i vmax = _mm256_set1_epi32(out_max); + + for (; i <= r.end - 8; i += 8) { + __m128i a8 = _mm_loadl_epi64((const __m128i*)(p0 + i)); + __m128i b8 = _mm_loadl_epi64((const __m128i*)(p1 + i)); + __m256i a32 = _mm256_cvtepu8_epi32(a8); + __m256i b32 = _mm256_cvtepu8_epi32(b8); + + __m256 af = _mm256_cvtepi32_ps(a32); + __m256 bf = _mm256_cvtepi32_ps(b32); + + __m256 val = _mm256_fmadd_ps(vc0, af, _mm256_fmadd_ps(vc1, bf, voff)); + + __m256i ival = _mm256_cvtps_epi32(val); + ival = _mm256_max_epi32(ival, vmin); + ival = _mm256_min_epi32(ival, vmax); + + __m128i lo = _mm256_castsi256_si128(ival); + __m128i hi = _mm256_extracti128_si256(ival, 1); + __m128i packed16 = _mm_packs_epi32(lo, hi); + __m128i packed8 = _mm_packus_epi16(packed16, packed16); + + _mm_storel_epi64((__m128i*)(outptr + i), packed8); + } + } + #elif CV_NEON + if (ninputs == 2) { + float32x4_t vc0 = vdupq_n_f32(c0); + float32x4_t vc1 = vdupq_n_f32(c1); + float32x4_t voff = vdupq_n_f32(off); + int32x4_t vmin = vdupq_n_s32(out_min); + int32x4_t vmax = vdupq_n_s32(out_max); + + for (; i <= r.end - 8; i += 8) { + uint8x8_t a8 = vld1_u8(p0 + i); + uint8x8_t b8 = vld1_u8(p1 + i); + uint16x8_t a16 = vmovl_u8(a8); + uint16x8_t b16 = vmovl_u8(b8); + + int32x4_t a32_lo = vreinterpretq_s32_u32(vmovl_u16(vget_low_u16(a16))); + int32x4_t b32_lo = vreinterpretq_s32_u32(vmovl_u16(vget_low_u16(b16))); + int32x4_t a32_hi = vreinterpretq_s32_u32(vmovl_u16(vget_high_u16(a16))); + int32x4_t b32_hi = vreinterpretq_s32_u32(vmovl_u16(vget_high_u16(b16))); + + float32x4_t val_lo = vaddq_f32(vaddq_f32(voff, + vmulq_f32(vc0, vcvtq_f32_s32(a32_lo))), + vmulq_f32(vc1, vcvtq_f32_s32(b32_lo))); + float32x4_t val_hi = vaddq_f32(vaddq_f32(voff, + vmulq_f32(vc0, vcvtq_f32_s32(a32_hi))), + vmulq_f32(vc1, vcvtq_f32_s32(b32_hi))); + +#if CV_NEON_AARCH64 + int32x4_t ival_lo = vminq_s32(vmaxq_s32(vcvtnq_s32_f32(val_lo), vmin), vmax); + int32x4_t ival_hi = vminq_s32(vmaxq_s32(vcvtnq_s32_f32(val_hi), vmin), vmax); +#else + float32x4_t half = vdupq_n_f32(0.5f); + int32x4_t ival_lo = vminq_s32(vmaxq_s32(vcvtq_s32_f32(vaddq_f32(val_lo, + vbslq_f32(vcgeq_f32(val_lo, vdupq_n_f32(0.f)), half, vnegq_f32(half)))), vmin), vmax); + int32x4_t ival_hi = vminq_s32(vmaxq_s32(vcvtq_s32_f32(vaddq_f32(val_hi, + vbslq_f32(vcgeq_f32(val_hi, vdupq_n_f32(0.f)), half, vnegq_f32(half)))), vmin), vmax); +#endif + + int16x8_t p16 = vcombine_s16(vqmovn_s32(ival_lo), vqmovn_s32(ival_hi)); + uint8x8_t p8u = vqmovun_s16(p16); + if (lutptr) { + uint8_t tmp[8]; + vst1_u8(tmp, p8u); + for (int k = 0; k < 8; k++) + outptr[i + k] = (uint8_t)lutptr[tmp[k]]; + } else { + vst1_u8(outptr + i, p8u); + } + } + } + #endif + for (; i < r.end; i++) + { + float val = c0 * (float)p0[i] + c1 * (float)p1[i] + off; + + for (int k = 2; k < ninputs; k++) + val += cptr[k] * (float)inptrs[k][i]; + + int ival = cvRound(val); + ival = std::max(out_min, std::min(out_max, ival)); + + if (lutptr) + ival = (int)(uint8_t)lutptr[ival]; + + outptr[i] = (uint8_t)ival; + } + }, nstripes); + } else { + std::vector inptrs(ninputs); + for (int k = 0; k < ninputs; k++) + inptrs[k] = input_arrs.getMat(k).ptr(); + const int8_t* p0 = inptrs[0]; + const int8_t* p1 = inptrs[1]; + int8_t* outptr = out.ptr(); + + parallel_for_(Range(0, (int)total_elems), [&](const Range& r) { + int i = r.start; + #if CV_AVX2 + if (!lutptr && ninputs == 2) { + __m256 vc0 = _mm256_set1_ps(c0); + __m256 vc1 = _mm256_set1_ps(c1); + __m256 voff = _mm256_set1_ps(off); + __m256i vmin = _mm256_set1_epi32(out_min); + __m256i vmax = _mm256_set1_epi32(out_max); + + for (; i <= r.end - 8; i += 8) { + __m128i a8 = _mm_loadl_epi64((const __m128i*)(p0 + i)); + __m128i b8 = _mm_loadl_epi64((const __m128i*)(p1 + i)); + __m256i a32 = _mm256_cvtepi8_epi32(a8); + __m256i b32 = _mm256_cvtepi8_epi32(b8); + + __m256 af = _mm256_cvtepi32_ps(a32); + __m256 bf = _mm256_cvtepi32_ps(b32); + + __m256 val = _mm256_fmadd_ps(vc0, af, _mm256_fmadd_ps(vc1, bf, voff)); + + __m256i ival = _mm256_cvtps_epi32(val); + ival = _mm256_max_epi32(ival, vmin); + ival = _mm256_min_epi32(ival, vmax); + + __m128i lo = _mm256_castsi256_si128(ival); + __m128i hi = _mm256_extracti128_si256(ival, 1); + __m128i packed16 = _mm_packs_epi32(lo, hi); + __m128i packed8 = _mm_packs_epi16(packed16, packed16); + + _mm_storel_epi64((__m128i*)(outptr + i), packed8); + } + } + #elif CV_NEON + if (ninputs == 2) { + float32x4_t vc0 = vdupq_n_f32(c0); + float32x4_t vc1 = vdupq_n_f32(c1); + float32x4_t voff = vdupq_n_f32(off); + int32x4_t vmin = vdupq_n_s32(out_min); + int32x4_t vmax = vdupq_n_s32(out_max); + + for (; i <= r.end - 8; i += 8) { + int8x8_t a8 = vld1_s8(p0 + i); + int8x8_t b8 = vld1_s8(p1 + i); + int16x8_t a16 = vmovl_s8(a8); + int16x8_t b16 = vmovl_s8(b8); + + int32x4_t a32_lo = vmovl_s16(vget_low_s16(a16)); + int32x4_t b32_lo = vmovl_s16(vget_low_s16(b16)); + int32x4_t a32_hi = vmovl_s16(vget_high_s16(a16)); + int32x4_t b32_hi = vmovl_s16(vget_high_s16(b16)); + + float32x4_t val_lo = vaddq_f32(vaddq_f32(voff, + vmulq_f32(vc0, vcvtq_f32_s32(a32_lo))), + vmulq_f32(vc1, vcvtq_f32_s32(b32_lo))); + float32x4_t val_hi = vaddq_f32(vaddq_f32(voff, + vmulq_f32(vc0, vcvtq_f32_s32(a32_hi))), + vmulq_f32(vc1, vcvtq_f32_s32(b32_hi))); + +#if CV_NEON_AARCH64 + int32x4_t ival_lo = vminq_s32(vmaxq_s32(vcvtnq_s32_f32(val_lo), vmin), vmax); + int32x4_t ival_hi = vminq_s32(vmaxq_s32(vcvtnq_s32_f32(val_hi), vmin), vmax); +#else + float32x4_t half = vdupq_n_f32(0.5f); + int32x4_t ival_lo = vminq_s32(vmaxq_s32(vcvtq_s32_f32(vaddq_f32(val_lo, + vbslq_f32(vcgeq_f32(val_lo, vdupq_n_f32(0.f)), half, vnegq_f32(half)))), vmin), vmax); + int32x4_t ival_hi = vminq_s32(vmaxq_s32(vcvtq_s32_f32(vaddq_f32(val_hi, + vbslq_f32(vcgeq_f32(val_hi, vdupq_n_f32(0.f)), half, vnegq_f32(half)))), vmin), vmax); +#endif + + int16x8_t p16 = vcombine_s16(vqmovn_s32(ival_lo), vqmovn_s32(ival_hi)); + int8x8_t p8 = vqmovn_s16(p16); + if (lutptr) { + int8_t tmp[8]; + vst1_s8(tmp, p8); + for (int k = 0; k < 8; k++) + outptr[i + k] = (int8_t)lutptr[(int)tmp[k] + 128]; + } else { + vst1_s8(outptr + i, p8); + } + } + } + #endif + for (; i < r.end; i++) + { + float val = c0 * (float)p0[i] + c1 * (float)p1[i] + off; + + for (int k = 2; k < ninputs; k++) + val += cptr[k] * (float)inptrs[k][i]; + + int ival = cvRound(val); + ival = std::max(out_min, std::min(out_max, ival)); + + if (lutptr) + ival = (int)lutptr[ival + 128]; + + outptr[i] = (int8_t)ival; + } + }, nstripes); + } + + if (uouts) { + out.copyTo(uouts->at(0)); + } + } + + bool setActivation(const Ptr& layer) CV_OVERRIDE + { + Ptr activ_int8 = layer.dynamicCast(); + if (!activ_int8.empty()) + { + activ = activ_int8; + if (!activ_int8->blobs.empty()) + activationLUT = activ_int8->blobs[0]; + return true; + } + return false; + } +}; + +Ptr Eltwise2Int8Layer::create(const LayerParams& params) +{ + return Ptr(new Eltwise2Int8LayerImpl(params)); +} + +} // namespace dnn +} // namespace cv diff --git a/modules/dnn/src/int8layers/fully_connected_layer.cpp b/modules/dnn/src/int8layers/fully_connected_layer.cpp index 5882401ada..1e2528c1da 100644 --- a/modules/dnn/src/int8layers/fully_connected_layer.cpp +++ b/modules/dnn/src/int8layers/fully_connected_layer.cpp @@ -28,6 +28,7 @@ public: output_sc = params.get("scales", 1.0f); axis = params.get("axis", 1); per_channel = params.get("per_channel", true); + output_type = CV_8S; // default, may be overridden by fusion code if (blobs.size() == 3) { @@ -77,6 +78,16 @@ public: return false; } + void getTypes(const std::vector& inputs, + const int requiredOutputs, + const int requiredInternals, + std::vector& outputs, + std::vector& internals) const CV_OVERRIDE + { + outputs.assign(requiredOutputs, output_type); + internals.clear(); + } + virtual bool supportBackend(int backendId) CV_OVERRIDE { if (backendId == DNN_BACKEND_TIMVX && haveTimVX()) @@ -383,14 +394,22 @@ public: int axisCan = normalize_axis(axis, input[0].dims); int outerSize = input[0].total(0, axisCan); - Mat srcMat = input[0].reshape(1, outerSize); + Mat srcMat0 = input[0].reshape(1, outerSize); + Mat srcMat; + if (srcMat0.type() == CV_8U) { + // Convert uint8 to int8 by subtracting 128. + // Zero-point was adjusted at fusion time to compensate. + srcMat0.convertTo(srcMat, CV_8S, 1, -128); + } else { + srcMat = srcMat0; + } Mat dstMat = output[0].reshape(1, outerSize); Mat dstMatInt32= Mat(shape(dstMat), CV_32S); - const int nstripes = getNumThreads(); + const int nstripes = outerSize <= 4 ? 1 : getNumThreads(); FullyConnected::run(srcMat, weightsMat, biasMat, outputMultiplier, activationLUT, dstMatInt32, activ.get(), nstripes, output_zp); - dstMatInt32.convertTo(dstMat, CV_8S); + dstMatInt32.convertTo(dstMat, output_type); } virtual int64 getFLOPS(const std::vector &inputs, diff --git a/modules/dnn/src/int8layers/pool2_int8_layer.cpp b/modules/dnn/src/int8layers/pool2_int8_layer.cpp new file mode 100644 index 0000000000..def9bcfa24 --- /dev/null +++ b/modules/dnn/src/int8layers/pool2_int8_layer.cpp @@ -0,0 +1,381 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// Copyright (C) 2026, BigVision LLC, all rights reserved. +// Third party copyrights are property of their respective owners + +#include "../precomp.hpp" +#include "../net_impl.hpp" +#include "../layers/conv2_common.hpp" +#include "opencv2/core/hal/intrin.hpp" + +namespace cv { +namespace dnn { + +template +static void maxPoolImpl(const void* inp_, void* out_, const ConvState& cs) +{ + constexpr int MAX_POOL_DIMS = ConvState::MAX_CONV_DIMS; + int NC1 = cs.inpshape[0] * cs.inpshape[1]; + + CV_Assert(cs.inpshape.layout == DATA_LAYOUT_BLOCK); + CV_Assert(cs.outshape.layout == DATA_LAYOUT_BLOCK); + CV_Assert(cs.inpshape.dims == cs.outshape.dims); + + parallel_for_(Range(0, NC1), [&](const Range& r) { + CV_Assert(cs.nspatialdims <= MAX_POOL_DIMS && MAX_POOL_DIMS == 3); + + int sdims = cs.nspatialdims; + int nc0 = r.start, nc1 = r.end; + int C0 = cs.inpshape.back(); + int Di = sdims > 2 ? cs.inpshape[sdims - 1] : 1; + int Hi = sdims > 1 ? cs.inpshape[sdims] : 1; + int Wi = cs.inpshape[sdims + 1]; + int D = sdims > 2 ? cs.outshape[sdims - 1] : 1; + int H = sdims > 1 ? cs.outshape[sdims] : 1; + int W = cs.outshape[sdims + 1]; + int iplanesize = Di * Hi * Wi * C0; + int planesize = D * H * W * C0; + int SZ = cs.strides[0], SY = cs.strides[1], SX = cs.strides[2]; + int padZ0 = cs.pads[0], padY0 = cs.pads[1], padX0 = cs.pads[2]; + int inner_z0 = cs.inner[0], inner_z1 = cs.inner[MAX_POOL_DIMS]; + int inner_y0 = cs.inner[1], inner_y1 = cs.inner[MAX_POOL_DIMS + 1]; + int inner_x0 = cs.inner[2], inner_x1 = cs.inner[MAX_POOL_DIMS + 2]; + int ksize = (int)cs.ofstab.size(); + const int* zyxtab = cs.coordtab.data(); + const int* ofstab = cs.ofstab.data(); + + const T* inp = (const T*)inp_ + nc0 * iplanesize; + T* out = (T*)out_ + nc0 * planesize; + const T INITVAL = std::numeric_limits::min(); + + for (int nc = nc0; nc < nc1; nc++, inp += iplanesize) { + for (int z0 = 0; z0 < D; z0++) { + int zi_ = z0 * SZ - padZ0; + for (int y0 = 0; y0 < H; y0++, out += W * C0) { + int x0 = 0; + int x1 = z0 >= inner_z0 && z0 < inner_z1 && + y0 >= inner_y0 && y0 < inner_y1 ? inner_x0 : W; + int yi_ = y0 * SY - padY0; + + for (;;) { + for (; x0 < x1; x0++) { + int xi_ = x0 * SX - padX0; + for (int c = 0; c < C0; c++) + out[x0 * C0 + c] = INITVAL; + + for (int k = 0; k < ksize; k++) { + int zi = zi_ + zyxtab[k * MAX_POOL_DIMS]; + int yi = yi_ + zyxtab[k * MAX_POOL_DIMS + 1]; + int xi = xi_ + zyxtab[k * MAX_POOL_DIMS + 2]; + if ((unsigned)zi >= (unsigned)Di || + (unsigned)yi >= (unsigned)Hi || + (unsigned)xi >= (unsigned)Wi) + continue; + const T* inptr = inp + ((zi * Hi + yi) * Wi + xi) * C0; + for (int c = 0; c < C0; c++) + out[x0 * C0 + c] = std::max(out[x0 * C0 + c], inptr[c]); + } + } + + if (x0 == W) + break; + x1 = inner_x1; + + for (; x0 < x1; x0++) { + int xi_ = x0 * SX - padX0; + const T* inp_xi = inp + ((Hi * zi_ + yi_) * Wi + xi_) * C0; + + for (int c = 0; c < C0; c++) { + T s = inp_xi[ofstab[0] + c]; + for (int k = 1; k < ksize; k++) + s = std::max(s, inp_xi[ofstab[k] + c]); + out[x0 * C0 + c] = s; + } + } + + x1 = W; + } + } + } + } + }); +} + +static void maxPoolInt8(const void* inp_, void* out_, const ConvState& cs, bool isU8) +{ + if (isU8) + maxPoolImpl(inp_, out_, cs); + else + maxPoolImpl(inp_, out_, cs); +} + +static void avgPoolInt8(const void* inp_, void* out_, const ConvState& cs, + float inp_sc, int inp_zp, float out_sc, int out_zp, + bool count_include_pad_, bool isU8) +{ + constexpr int MAX_POOL_DIMS = ConvState::MAX_CONV_DIMS; + int NC1 = cs.inpshape[0] * cs.inpshape[1]; + + CV_Assert(cs.inpshape.layout == DATA_LAYOUT_BLOCK); + CV_Assert(cs.outshape.layout == DATA_LAYOUT_BLOCK); + CV_Assert(cs.inpshape.dims == cs.outshape.dims); + + parallel_for_(Range(0, NC1), [&](const Range& r) { + CV_Assert(cs.nspatialdims <= MAX_POOL_DIMS && MAX_POOL_DIMS == 3); + + bool count_include_pad = count_include_pad_; + int sdims = cs.nspatialdims; + int nc0 = r.start, nc1 = r.end; + int C0 = cs.inpshape.back(); + int Di = sdims > 2 ? cs.inpshape[sdims - 1] : 1; + int Hi = sdims > 1 ? cs.inpshape[sdims] : 1; + int Wi = cs.inpshape[sdims + 1]; + int D = sdims > 2 ? cs.outshape[sdims - 1] : 1; + int H = sdims > 1 ? cs.outshape[sdims] : 1; + int W = cs.outshape[sdims + 1]; + int iplanesize = Di * Hi * Wi * C0; + int planesize = D * H * W * C0; + int SZ = cs.strides[0], SY = cs.strides[1], SX = cs.strides[2]; + int padZ0 = cs.pads[0], padY0 = cs.pads[1], padX0 = cs.pads[2]; + int ksize_total = (int)cs.ofstab.size(); + const int* zyxtab = cs.coordtab.data(); + + const uint8_t* inp = (const uint8_t*)inp_ + nc0 * iplanesize; + uint8_t* out = (uint8_t*)out_ + nc0 * planesize; + + float scale_ratio = inp_sc / out_sc; + + std::vector accum(C0); + + for (int nc = nc0; nc < nc1; nc++, inp += iplanesize) { + for (int z0 = 0; z0 < D; z0++) { + int zi_ = z0 * SZ - padZ0; + for (int y0 = 0; y0 < H; y0++, out += W * C0) { + int yi_ = y0 * SY - padY0; + + for (int x0 = 0; x0 < W; x0++) { + int xi_ = x0 * SX - padX0; + + for (int c = 0; c < C0; c++) + accum[c] = 0; + + int count = 0; + for (int k = 0; k < ksize_total; k++) { + int zi = zi_ + zyxtab[k * MAX_POOL_DIMS]; + int yi = yi_ + zyxtab[k * MAX_POOL_DIMS + 1]; + int xi = xi_ + zyxtab[k * MAX_POOL_DIMS + 2]; + + if ((unsigned)zi >= (unsigned)Di || + (unsigned)yi >= (unsigned)Hi || + (unsigned)xi >= (unsigned)Wi) { + if (count_include_pad) + count++; + continue; + } + + count++; + const uint8_t* inptr = inp + ((zi * Hi + yi) * Wi + xi) * C0; + for (int c = 0; c < C0; c++) { + int v = isU8 ? (int)inptr[c] + : (int)(int8_t)inptr[c]; + accum[c] += v - inp_zp; + } + } + + if (count == 0) count = 1; + float inv_count = 1.f / count; + + for (int c = 0; c < C0; c++) { + float val = (float)accum[c] * inv_count * scale_ratio + (float)out_zp; + int ival = cvRound(val); + if (isU8) + out[x0 * C0 + c] = (uint8_t)std::max(0, std::min(255, ival)); + else + out[x0 * C0 + c] = (uint8_t)(int8_t)std::max(-128, std::min(127, ival)); + } + } + } + } + } + }); +} + +static void globalAvgPoolInt8(const void* inp_, void* out_, + int N, int C1, int spatialSize, int C0, + float inp_sc, int inp_zp, float out_sc, int out_zp, + bool isU8) +{ + int NC1 = N * C1; + float scale_ratio = inp_sc / out_sc; + + parallel_for_(Range(0, NC1), [&](const Range& r) { + for (int nc = r.start; nc < r.end; nc++) { + const uint8_t* inptr = (const uint8_t*)inp_ + (size_t)nc * spatialSize * C0; + uint8_t* outptr = (uint8_t*)out_ + (size_t)nc * C0; + + for (int c = 0; c < C0; c++) { + int sum = 0; + for (int s = 0; s < spatialSize; s++) { + int v = isU8 ? (int)inptr[s * C0 + c] + : (int)(int8_t)inptr[s * C0 + c]; + sum += v - inp_zp; + } + + float val = (float)sum / spatialSize * scale_ratio + (float)out_zp; + int ival = cvRound(val); + if (isU8) + outptr[c] = (uint8_t)std::max(0, std::min(255, ival)); + else + outptr[c] = (uint8_t)(int8_t)std::max(-128, std::min(127, ival)); + } + } + }); +} + +class Pool2Int8LayerImpl CV_FINAL : public Pool2Int8Layer +{ +public: + bool count_include_pad; + ConvState cs; + MatShape prevInpshape; + + Pool2Int8LayerImpl(const LayerParams& params) + { + setParamsFrom(params); + auto_pad = getAutoPadding(params); + kernel_shape = params.getVector("kernel_size"); + strides = params.getVector("stride"); + dilations = params.getVector("dilation"); + pads = params.getVector("pad"); + ceil_mode = params.get("ceil_mode", false); + is_global_pooling = params.get("global_pooling", false); + is_max_pool = params.get("is_max_pool", true); + count_include_pad = params.get("count_include_pad", false); + + input_sc = params.get("input_scale", 1.f); + input_zp = params.get("input_zeropoint", 0); + output_sc = params.get("scales", 1.f); + output_zp = params.get("zeropoints", 0); + } + + void getTypes(const std::vector& inputs, + const int requiredOutputs, const int, + std::vector& outputs, + std::vector& internals) const CV_OVERRIDE + { + CV_Assert(!inputs.empty()); + outputs.assign(requiredOutputs, inputs[0]); + internals.clear(); + } + + bool getMemoryShapes(const std::vector& inpshapes, + const int, + std::vector& outshapes, + std::vector& tempshapes) const CV_OVERRIDE + { + CV_Assert(inpshapes.size() == 1); + + if (is_global_pooling) { + MatShape outshape = inpshapes[0]; + for (int d = 2; d < outshape.dims - 1; d++) + outshape[d] = 1; + outshapes.assign(1, outshape); + } else { + outshapes.assign(1, convInferShape(inpshapes[0], MatShape(), + kernel_shape, 0, strides, dilations, + pads, auto_pad, ceil_mode)); + } + tempshapes.clear(); + return true; + } + + int getLayouts(const std::vector& actualInputs, + std::vector& desiredInputs, + const int requiredOutputs, + std::vector& outputs) const CV_OVERRIDE + { + CV_Assert(actualInputs.size() == 1u); + desiredInputs.assign(1, DATA_LAYOUT_BLOCK); + outputs.assign(requiredOutputs, DATA_LAYOUT_BLOCK); + return getNetImpl(this)->defaultC0; + } + + void forward(InputArrayOfArrays inputs_arr, + OutputArrayOfArrays outputs_arr, + OutputArrayOfArrays) CV_OVERRIDE + { + size_t ninputs = inputs_arr.total(); + CV_Assert(ninputs == 1); + + const Mat& inp = inputs_arr.getMat(0); + int inptype = inp.type(); + MatShape inpshape = inp.shape(); + CV_Assert(inpshape.layout == DATA_LAYOUT_BLOCK); + CV_Assert(inptype == CV_8SC1 || inptype == CV_8UC1); + + MatShape outshape; + if (is_global_pooling) { + outshape = inpshape; + for (int d = 2; d < outshape.dims - 1; d++) + outshape[d] = 1; + } else { + outshape = convInferShape(inpshape, MatShape(), + kernel_shape, 0, strides, dilations, + pads, auto_pad, ceil_mode); + } + + int outkind = outputs_arr.kind(); + Mat out; + + if (outkind == _InputArray::STD_VECTOR_MAT) { + std::vector& outs = outputs_arr.getMatVecRef(); + outs.resize(1); + outs[0].fit(outshape, inptype); + out = outs[0]; + } else { + std::vector& outs = outputs_arr.getUMatVecRef(); + outs.resize(1); + outs[0].fit(outshape, inptype); + out.fit(outshape, inptype); + } + + if (is_global_pooling) { + int N = inpshape[0]; + int C1 = inpshape[1]; + int C0 = inpshape.back(); + int spatialSize = 1; + for (int d = 2; d < inpshape.dims - 1; d++) + spatialSize *= inpshape[d]; + + globalAvgPoolInt8(inp.data, out.data, N, C1, spatialSize, C0, + input_sc, input_zp, output_sc, output_zp, + inptype == CV_8UC1); + } else { + ConvState cs_local; + cs_local.initPooling(inpshape, outshape, kernel_shape, strides, + dilations, pads, auto_pad, ceil_mode); + + if (is_max_pool) { + maxPoolInt8(inp.data, out.data, cs_local, inptype == CV_8UC1); + } else { + avgPoolInt8(inp.data, out.data, cs_local, + input_sc, input_zp, output_sc, output_zp, + count_include_pad, inptype == CV_8UC1); + } + } + + if (outkind == _InputArray::STD_VECTOR_UMAT) { + std::vector& outs = outputs_arr.getUMatVecRef(); + out.copyTo(outs[0]); + } + } +}; + +Ptr Pool2Int8Layer::create(const LayerParams& params) +{ + return Ptr(new Pool2Int8LayerImpl(params)); +} + +} // namespace dnn +} // namespace cv diff --git a/modules/dnn/src/layers/dequantizelinear_layer.cpp b/modules/dnn/src/layers/dequantizelinear_layer.cpp index 4b2aceb16f..0997748dfa 100644 --- a/modules/dnn/src/layers/dequantizelinear_layer.cpp +++ b/modules/dnn/src/layers/dequantizelinear_layer.cpp @@ -1,37 +1,54 @@ // This file is part of OpenCV project. // It is subject to the license terms in the LICENSE file found in the top-level directory // of this distribution and at http://opencv.org/license.html. +// Copyright (C) 2026, BigVision LLC, all rights reserved. +// Third party copyrights are property of their respective owners. #include "../precomp.hpp" #include "layers_common.hpp" #include "../net_impl.hpp" +#if defined(__x86_64__) || defined(_M_X64) +#include +#endif + namespace cv { namespace dnn { -#if CV_SIMD || CV_SIMD_SCALABLE -static void dequantizeLinearChunk_u8_f32(const uint8_t* src, float* dst, - float scale, int zp, int64_t len) +/* + DequantizeLinear layer, as defined in ONNX specification: + https://onnx.ai/onnx/operators/onnx__DequantizeLinear.html + + Opset's 10 to 25 are covered. +*/ + +#if defined(__x86_64__) || defined(_M_X64) +#if defined(__GNUC__) || defined(__clang__) +__attribute__((target("avx2"))) +#endif +static void dequantizeLinearChunk_u8_f32_avx2(const uint8_t* src, float* dst, + float scale, int zp, int64_t len) { - const int vlanes = VTraits::vlanes(); - v_float32 vscale = vx_setall_f32(scale); - v_int32 vzp = vx_setall_s32(zp); + __m256 vscale = _mm256_set1_ps(scale); + __m256i vzp = _mm256_set1_epi32(zp); int64_t j = 0; - for (; j <= len - vlanes; j += vlanes) { - v_int32 vi = v_reinterpret_as_s32(vx_load_expand_q(src + j)); - vi = v_sub(vi, vzp); - v_float32 vf = v_mul(v_cvt_f32(vi), vscale); - v_store(dst + j, vf); + for (; j <= len - 8; j += 8) { + __m128i raw = _mm_loadl_epi64((__m128i*)(src + j)); + __m256i vi = _mm256_cvtepu8_epi32(raw); + vi = _mm256_sub_epi32(vi, vzp); + __m256 vf = _mm256_cvtepi32_ps(vi); + vf = _mm256_mul_ps(vf, vscale); + _mm256_storeu_ps(dst + j, vf); } for (; j < len; j++) dst[j] = (float)(src[j] - zp) * scale; } -static void dequantizeLinearFast_u8_f32(const uint8_t* inp, float* out, - float scale, int zp, - int64_t total) +static void dequantizeLinearFast_u8_f32_avx2(const uint8_t* inp, float* out, + float scale, int zp, + int64_t total) { const int64_t block = 1024; int64_t nblocks = (total + block - 1) / block; @@ -40,20 +57,12 @@ static void dequantizeLinearFast_u8_f32(const uint8_t* inp, float* out, for (int i = r.start; i < r.end; i++) { int64_t ofs = i * block; int64_t len = std::min(block, total - ofs); - dequantizeLinearChunk_u8_f32(inp + ofs, out + ofs, scale, zp, len); + dequantizeLinearChunk_u8_f32_avx2(inp + ofs, out + ofs, scale, zp, len); } }); } #endif - -/* - DequantizeLinear layer, as defined in ONNX specification: - https://onnx.ai/onnx/operators/onnx__DequantizeLinear.html - - Opset's 10 to 23 are covered. -*/ - template static void dequantizeLinear(const _InpTp* inp_, const _ScaleTp* scale_, const _InpTp* zp_, _OutTp* out_, @@ -91,7 +100,6 @@ static void dequantizeLinear(const _InpTp* inp_, const _ScaleTp* scale_, const _ScaleTp* sc = scale_ + scale_ofs; _OutTp* out = out_ + block_ofs; - // [TODO] vectorize using intrinsics if (slice_size > 1) { for (int k = 0; k < delta; k++, inp += slice_size, out += slice_size, sc += scale_step, zp += zp_step) { @@ -201,15 +209,16 @@ static void dequantizeLinear(const Mat& inp, const Mat& scale_, const Mat& zp, } } - // Fast path: per-tensor dequantization uint8→float with universal intrinsics -#if CV_SIMD || CV_SIMD_SCALABLE - if (block_size == 0 && sz_a == 1 && inptype == CV_8U && outtype == CV_32F && sctype == CV_32F) { + // Fast path: per-tensor dequantization uint8→float with AVX2 + proper parallelism +#if defined(__x86_64__) || defined(_M_X64) + if (block_size == 0 && sz_a == 1 && inptype == CV_8U && outtype == CV_32F && sctype == CV_32F + && checkHardwareSupport(CV_CPU_AVX2)) { float sc = reinterpret_cast(scale.data)[0]; int zpval = zp.empty() ? 0 : (int)reinterpret_cast(zp.data)[0]; int64_t total = nslices * slice_size; - dequantizeLinearFast_u8_f32(reinterpret_cast(inp.data), - reinterpret_cast(out.data), - sc, zpval, total); + dequantizeLinearFast_u8_f32_avx2(reinterpret_cast(inp.data), + reinterpret_cast(out.data), + sc, zpval, total); return; } #endif diff --git a/modules/dnn/test/test_model.cpp b/modules/dnn/test/test_model.cpp index 60a72377d9..12a0098afb 100644 --- a/modules/dnn/test/test_model.cpp +++ b/modules/dnn/test/test_model.cpp @@ -921,4 +921,61 @@ TEST_P(Reproducibility_ResNet50_ONNX, Accuracy) INSTANTIATE_TEST_CASE_P(/**/, Reproducibility_ResNet50_ONNX, testing::ValuesIn(getAvailableTargets(DNN_BACKEND_OPENCV))); +typedef testing::TestWithParam Reproducibility_ResNet50_QDQ_ONNX; +TEST_P(Reproducibility_ResNet50_QDQ_ONNX, Accuracy) +{ + Target targetId = GetParam(); + applyTestTag(targetId == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_512MB : CV_TEST_TAG_MEMORY_1GB); + ASSERT_TRUE(ocl::useOpenCL() || targetId == DNN_TARGET_CPU || targetId == DNN_TARGET_CPU_FP16); + + std::string modelname = _tf("onnx/models/resnet50-v1-12-qdq.onnx", false); + Net net = readNetFromONNX(modelname); + + net.setPreferableBackend(DNN_BACKEND_OPENCV); + net.setPreferableTarget(targetId); + + if (targetId == DNN_TARGET_CPU_FP16) + net.enableWinograd(false); + + std::string imgname = _tf("sqcat.png"); + Mat image = imread(imgname); + Mat input = blobFromImage(image, 0.017, Size(224,224), + Scalar(103.939, 116.779, 123.68), + false, true, CV_32F); + ASSERT_TRUE(!input.empty()); + + Mat out; + double min_t = 0; + const int niters = +#ifdef _DEBUG + 1; +#else + 30; +#endif + + for (int i = 0; i < niters; i++) { + double t = (double)getTickCount(); + net.setInput(input); + out = net.forward(); + t = (double)getTickCount() - t; + min_t = i == 0 ? t : std::min(min_t, t); + } + printf("run time = %.2fms\n", min_t*1000./getTickFrequency()); + + const int K = 5; + std::vector > res; + topK(out, res, K); + ASSERT_EQ(int(res.size()), K); + + std::vector > ref = {{285, 10.44}, {287, 10.13}, {283, 8.89}, {278, 8.43}}; + const float eps = 0.5f; + + for (int i = 0; i < (int)ref.size(); i++) { + EXPECT_EQ(ref[i].first, res[i].first); + EXPECT_NEAR(ref[i].second, res[i].second, eps); + } +} +INSTANTIATE_TEST_CASE_P(/**/, Reproducibility_ResNet50_QDQ_ONNX, + testing::ValuesIn(getAvailableTargets(DNN_BACKEND_OPENCV))); + }} // namespace