diff --git a/modules/dnn/src/int8layers/quantization_utils.cpp b/modules/dnn/src/int8layers/quantization_utils.cpp index 146ad68257..4690f68e5f 100644 --- a/modules/dnn/src/int8layers/quantization_utils.cpp +++ b/modules/dnn/src/int8layers/quantization_utils.cpp @@ -15,7 +15,10 @@ namespace dnn static void broadcast1D2TargetMat(Mat& data, const MatShape& targetShape, int axis) { // The data is the 1-D scales or zeropoints. - CV_Assert(axis >= 0 && targetShape.size() > axis && data.total() == targetShape[axis]); + CV_CheckGE(axis, 0, "Quantization axis must be non-negative."); + CV_CheckGT((int)targetShape.size(),axis,"Quantization axis must be within the valid range of target shape dimensions."); + CV_CheckEQ((int)data.total(), (int)targetShape[axis], "Data total size must match the size of the specified target dimension."); + std::vector broadcast_axes; for (int i = 0; i < targetShape.size(); i++) { @@ -35,29 +38,98 @@ static void broadcast1D2TargetMat(Mat& data, const MatShape& targetShape, int ax } } +static void block_repeat(InputArray src, const MatShape& srcShape, int axis, int repetitions, OutputArray dst) +{ + CV_Assert(src.getObj() != dst.getObj()); + CV_Check(axis, axis >= 0 && axis < src.dims(), "Axis out of range"); + CV_CheckGT(repetitions, 1, "More than one repetition expected"); + + Mat src_mat = src.getMat(); + Mat dst_mat; + + if (src_mat.depth() != CV_32F) + src_mat.convertTo(src_mat, CV_32F); + + MatShape sshape = srcShape; + MatShape dshape = srcShape; + + size_t dtype_bytes = src_mat.elemSize(); + int chunk_size = dtype_bytes; + int num_chunks = 1; + + dshape[axis] *= repetitions; + + for (int i = axis+1; i < sshape.size(); ++i) + chunk_size*=sshape[i]; + + for (int i = 0; i <= axis; ++i) + num_chunks*=sshape[i]; + + dst.create(dshape.size(), dshape.data(), src_mat.type()); + dst_mat = dst.getMat(); + + CV_Assert(dst_mat.isContinuous()); + CV_Assert(src_mat.isContinuous()); + + for (int i = 0; i < repetitions; ++i) { + size_t src_offset = 0; + size_t dst_offset = i * chunk_size; + + for (int j = 0; j < num_chunks; ++j) { + memcpy(dst_mat.data + dst_offset, src_mat.data + src_offset, chunk_size); + src_offset += chunk_size; + dst_offset += chunk_size * repetitions; + } + } +} + +template +static void copyVecToMat(Mat& mat, const std::vector& data){ + float * matPtr = mat.ptr(0); + const int len = data.size(); + + for (int i = 0; i < len; i++) + matPtr[i] = (float) data[i]; +} + +template +static void broadcastBlockedMatrix(Mat& mat, const std::vector& data, const MatShape& targetShape, int axis, int block_size){ + CV_Check(block_size, targetShape[axis] % block_size == 0 && block_size <= targetShape[axis], "Block size must be a divisor of the target dimension size and not exceed it."); + + MatShape subTargetShape(targetShape); + subTargetShape[axis] = static_cast(subTargetShape[axis] / block_size); + + block_repeat(data, subTargetShape, axis, block_size, mat); +} + +template +static void broadcastStandardMatrix(Mat& mat, const std::vector& data, const MatShape& targetShape, int axis) +{ + MatShape subTargetShape(targetShape.size(), 1); + subTargetShape[axis] = data.size(); + mat.create(subTargetShape.size(), subTargetShape.data(), CV_32FC1); + + copyVecToMat(mat,data); + + broadcast1D2TargetMat(mat, targetShape, axis); +} + + static void broadcastScaleAndZeropoint(Mat& scalesMat, Mat& zeropointsMat, const std::vector& scales, - const std::vector& zeropoints, const MatShape& targetShape, int axis) + const std::vector& zeropoints, const MatShape& targetShape, int axis, int block_size) { // broad cast the scales and zeropoint to the input shape. - MatShape subTargetShape(targetShape.size(), 1); - subTargetShape[axis] = scales.size(); - zeropointsMat.create(subTargetShape.size(), subTargetShape.data(), CV_32FC1); - scalesMat.create(subTargetShape.size(), subTargetShape.data(), CV_32FC1); - - const int len = scales.size(); - // Deep copy the scales and zeropoint data and prevent the original data from being changed. - - float * scalePtr = scalesMat.ptr(0); - for (int i = 0; i < len; i++) - scalePtr[i] = scales[i]; - - float * zpPtr = zeropointsMat.ptr(0); - for (int i = 0; i < len; i++) - zpPtr[i] = (float )zeropoints[i]; - - broadcast1D2TargetMat(scalesMat, targetShape, axis); - broadcast1D2TargetMat(zeropointsMat, targetShape, axis); + if (block_size == 0) + { + broadcastStandardMatrix(zeropointsMat, zeropoints, targetShape, axis); + broadcastStandardMatrix(scalesMat, scales, targetShape, axis); + } + else + { + broadcastBlockedMatrix(zeropointsMat, zeropoints, targetShape, axis, block_size); + broadcastBlockedMatrix(scalesMat, scales, targetShape, axis, block_size); + } } // Quantize FP32/FP16 Inputs to INT8 @@ -65,13 +137,17 @@ class QuantizeLayerImpl CV_FINAL : public QuantizeLayer { public: int axis; + int block_size; bool is1D; - Mat scalesMat, zeropointsMat; // Saving the broadcasetd scales data. + Mat scalesMat, zeropointsMat; // Saving the broadcasted scales data. + bool quantParamExternal = true; // Indicates if the quantization parameters (scale and zero point) are provided as inputs to the node. QuantizeLayerImpl(const LayerParams& params) { is1D = params.get("is1D", false); axis = params.get("axis", 1); + block_size = params.get("block_size", 0); + if (!is1D) { scales.push_back(params.get("scales", 1.0f)); @@ -82,7 +158,7 @@ public: DictValue paramScales = params.get("scales"); int i, n = paramScales.size(); - CV_Assert(n > 0); + CV_CheckGT(n, 0, "Scale missing."); scales.resize(n, 0.); for (i = 0; i < n; i++) scales[i] = paramScales.get(i); @@ -108,7 +184,7 @@ public: std::vector &outputs, std::vector &internals) const CV_OVERRIDE { - CV_Assert(inputs.size() == 1); + CV_Check(inputs.size(), inputs.size() >= 1 && inputs.size() <= 3, "Number of inputs must be between 1 and 3 inclusive."); Layer::getMemoryShapes(inputs, requiredOutputs, outputs, internals); return false; } @@ -124,7 +200,7 @@ public: if (is1D) { MatShape inputShape = shape(inputs[0]); - broadcastScaleAndZeropoint(scalesMat, zeropointsMat, scales, zeropoints, inputShape, axis); + broadcastScaleAndZeropoint(scalesMat, zeropointsMat, scales, zeropoints, inputShape, axis, block_size); } } @@ -146,6 +222,39 @@ public: return true; } #endif + void processInputOutput(std::vector& inputs, std::vector& outputs) + { + CV_Check(inputs.size(), inputs.size() >= 1 && inputs.size() <= 3, "Number of inputs must be between 1 and 3 inclusive."); + quantParamExternal &= inputs.size() > 1; + + // Scale and zeropoint taken as input + if (quantParamExternal) + { + quantParamExternal = false; + scalesMat = inputs[1]; + + scalesMat.reshape(1, 1).copyTo(scales); + + if(scalesMat.total() > 1) is1D = true; + + + if (inputs.size() > 2) + { + zeropointsMat = inputs[2]; + CV_CheckEQ((int)zeropointsMat.total(), (int)scalesMat.total(), "Scale and zero point elements number must match."); + zeropointsMat.reshape(1, 1).copyTo(zeropoints); + } + + if (is1D) + { + MatShape inputShape = shape(inputs[0]); + broadcastScaleAndZeropoint(scalesMat, zeropointsMat, scales, zeropoints, inputShape, axis, block_size); + } + } + + if (outputs[0].depth() != CV_8S) + outputs[0].convertTo(outputs[0], CV_8S); + } void forward(InputArrayOfArrays inputs_arr, OutputArrayOfArrays outputs_arr, OutputArrayOfArrays internals_arr) CV_OVERRIDE { @@ -159,14 +268,13 @@ public: inputs_arr.getMatVector(inputs); outputs_arr.getMatVector(outputs); - if (outputs[0].depth() != CV_8S) - outputs[0].convertTo(outputs[0], CV_8S); + processInputOutput(inputs, outputs); if (is1D) { Mat inputTmp; divide(inputs[0], scalesMat, inputTmp); - subtract(inputTmp, zeropointsMat, inputTmp); + add(inputTmp, zeropointsMat, inputTmp); inputTmp.convertTo(outputs[0], CV_8S); } @@ -190,13 +298,16 @@ class DequantizeLayerImpl CV_FINAL : public DequantizeLayer { public: int axis; + int block_size; bool is1D; Mat scalesMat, zeropointsMat; // Saving the broadcasetd scales data. + bool quantParamExternal = true; DequantizeLayerImpl(const LayerParams& params) { is1D = params.get("is1D", false); axis = params.get("axis", 1); + block_size = params.get("block_size", 0); if (!is1D) { @@ -208,7 +319,7 @@ public: DictValue paramScales = params.get("scales"); int i, n = paramScales.size(); - CV_Assert(n > 0); + CV_CheckGT(n, 0, "Scale missing."); scales.resize(n); for (i = 0; i < n; i++) scales[i] = paramScales.get(i); @@ -234,7 +345,7 @@ public: std::vector &outputs, std::vector &internals) const CV_OVERRIDE { - CV_Assert(inputs.size() == 1); + CV_Check(inputs.size(), inputs.size() >= 1 && inputs.size() <= 3, "Number of inputs must be between 1 and 3 inclusive."); Layer::getMemoryShapes(inputs, requiredOutputs, outputs, internals); return false; } @@ -250,7 +361,7 @@ public: if (is1D) { MatShape inputShape = shape(inputs[0]); - broadcastScaleAndZeropoint(scalesMat, zeropointsMat, scales, zeropoints, inputShape, axis); + broadcastScaleAndZeropoint(scalesMat, zeropointsMat, scales, zeropoints, inputShape, axis, block_size); } } @@ -269,6 +380,39 @@ public: } #endif + void processInputOutput(std::vector& inputs, std::vector& outputs) + { + CV_Check(inputs.size(), inputs.size() >= 1 && inputs.size() <= 3, "Number of inputs must be between 1 and 3 inclusive."); + + quantParamExternal &= inputs.size() > 1; + // Scale and zeropoint taken as input + if (quantParamExternal) + { + quantParamExternal = false; + scalesMat = inputs[1]; + + scalesMat.reshape(1, 1).copyTo(scales); + + if(scalesMat.total() > 1) is1D = true; + + if (inputs.size() > 2) + { + zeropointsMat = inputs[2]; + CV_CheckEQ((int)zeropointsMat.total(), (int)scalesMat.total(), "Scale and zero point elements number must match."); + zeropointsMat.reshape(1, 1).copyTo(zeropoints); + } + + if (is1D) + { + MatShape inputShape = shape(inputs[0]); + broadcastScaleAndZeropoint(scalesMat, zeropointsMat, scales, zeropoints, inputShape, axis, block_size); + } + } + + if (outputs[0].depth() != CV_32F) + outputs[0].convertTo(outputs[0], CV_32F); + } + void forward(InputArrayOfArrays inputs_arr, OutputArrayOfArrays outputs_arr, OutputArrayOfArrays internals_arr) CV_OVERRIDE { CV_TRACE_FUNCTION(); @@ -281,8 +425,7 @@ public: inputs_arr.getMatVector(inputs); outputs_arr.getMatVector(outputs); - if (outputs[0].depth() != CV_32F) - outputs[0].convertTo(outputs[0], CV_32F); + processInputOutput(inputs, outputs); if (is1D) { diff --git a/modules/dnn/src/onnx/onnx_importer.cpp b/modules/dnn/src/onnx/onnx_importer.cpp index 3745d7ed86..e91e2605c5 100644 --- a/modules/dnn/src/onnx/onnx_importer.cpp +++ b/modules/dnn/src/onnx/onnx_importer.cpp @@ -3239,6 +3239,17 @@ void ONNXImporter::parseQuantDequant(LayerParams& layerParams, const opencv_onnx // or 1-D tensor (per-channel quantized). bool is1D = false; + if (layerParams.type == "Quantize") + layerParams.set("depth", CV_8S); + else // Dequantize + layerParams.set("depth", CV_32F); + + // If scale is not defined as a constant blob, it is considered an external input. + if(constBlobs.find(node_proto.input(1)) == constBlobs.end()){ + addLayer(layerParams, node_proto); + return; + } + Mat scaleMat = getBlob(node_proto, 1); if(scaleMat.total() > 1) is1D = true; @@ -3280,11 +3291,6 @@ void ONNXImporter::parseQuantDequant(LayerParams& layerParams, const opencv_onnx layerParams.set("zeropoints", zeropoint); } - if (layerParams.type == "Quantize") - layerParams.set("depth", CV_8S); - else // Dequantize - layerParams.set("depth", CV_32F); - if (constBlobs.find(node_proto.input(0)) != constBlobs.end()) // Variable input. { std::vector inputs, outputs; diff --git a/modules/dnn/test/test_onnx_conformance.cpp b/modules/dnn/test/test_onnx_conformance.cpp index 57969ced87..0199d29548 100644 --- a/modules/dnn/test/test_onnx_conformance.cpp +++ b/modules/dnn/test/test_onnx_conformance.cpp @@ -224,6 +224,7 @@ static const TestCase testConformanceConfig[] = { {"test_depthtospace_example", 1, 1}, {"test_dequantizelinear", 3, 1}, {"test_dequantizelinear_axis", 3, 1}, + {"test_dequantizelinear_blocked", 3, 1}, {"test_det_2d", 1, 1}, {"test_det_nd", 1, 1}, {"test_div", 2, 1}, @@ -569,6 +570,7 @@ static const TestCase testConformanceConfig[] = { {"test_qlinearmatmul_3D", 8, 1}, {"test_quantizelinear", 3, 1}, {"test_quantizelinear_axis", 3, 1}, + {"test_quantizelinear_blocked", 3, 1}, {"test_range_float_type_positive_delta", 3, 1}, {"test_range_float_type_positive_delta_expanded", 3, 1}, {"test_range_int32_type_negative_delta", 3, 1}, diff --git a/modules/dnn/test/test_onnx_conformance_layer_filter__openvino.inl.hpp b/modules/dnn/test/test_onnx_conformance_layer_filter__openvino.inl.hpp index 9b2a2f4f2d..9069a69ff4 100644 --- a/modules/dnn/test/test_onnx_conformance_layer_filter__openvino.inl.hpp +++ b/modules/dnn/test/test_onnx_conformance_layer_filter__openvino.inl.hpp @@ -565,9 +565,11 @@ CASE(test_depthtospace_dcr_mode) CASE(test_depthtospace_example) // no filter CASE(test_dequantizelinear) - // no filter + SKIP; CASE(test_dequantizelinear_axis) - // no filter + SKIP; +CASE(test_dequantizelinear_blocked) + SKIP; CASE(test_det_2d) // no filter CASE(test_det_nd) @@ -1348,9 +1350,11 @@ CASE(test_qlinearmatmul_2D) CASE(test_qlinearmatmul_3D) // no filter CASE(test_quantizelinear) - // no filter + SKIP; CASE(test_quantizelinear_axis) - // no filter + SKIP; +CASE(test_quantizelinear_blocked) + SKIP; CASE(test_range_float_type_positive_delta) // no filter CASE(test_range_float_type_positive_delta_expanded) diff --git a/modules/dnn/test/test_onnx_conformance_layer_filter__vulkan_denylist.inl.hpp b/modules/dnn/test/test_onnx_conformance_layer_filter__vulkan_denylist.inl.hpp index 968dd1e025..f6aee0dd36 100644 --- a/modules/dnn/test/test_onnx_conformance_layer_filter__vulkan_denylist.inl.hpp +++ b/modules/dnn/test/test_onnx_conformance_layer_filter__vulkan_denylist.inl.hpp @@ -48,6 +48,9 @@ "test_cumsum_2d_axis_1", "test_cumsum_2d_negative_axis", "test_concat_1d_axis_negative_1", +"test_dequantizelinear", +"test_dequantizelinear_axis", +"test_dequantizelinear_blocked", "test_div_uint8", "test_flatten_axis0", "test_flatten_axis2", @@ -71,6 +74,9 @@ "test_pow_types_float32_int32", // vulkan backend does not take tensor other than float32 data type "test_pow_types_float32_int64", // vulkan backend does not take tensor other than float32 data type "test_pow_types_int", // vulkan backend does not take tensor other than float32 data type +"test_quantizelinear", +"test_quantizelinear_axis", +"test_quantizelinear_blocked", "test_softmax_default_axis", "test_sub_bcast", "test_sub_uint8", diff --git a/modules/dnn/test/test_onnx_conformance_layer_filter_opencv_ocl_fp16_denylist.inl.hpp b/modules/dnn/test/test_onnx_conformance_layer_filter_opencv_ocl_fp16_denylist.inl.hpp index 7303348d10..8dc970fe1e 100644 --- a/modules/dnn/test/test_onnx_conformance_layer_filter_opencv_ocl_fp16_denylist.inl.hpp +++ b/modules/dnn/test/test_onnx_conformance_layer_filter_opencv_ocl_fp16_denylist.inl.hpp @@ -1,4 +1,7 @@ "test_averagepool_3d_default", +"test_dequantizelinear", +"test_dequantizelinear_axis", +"test_dequantizelinear_blocked", "test_dropout_default_ratio", "test_globalmaxpool", "test_globalmaxpool_precomputed", @@ -14,7 +17,10 @@ "test_maxpool_2d_same_upper", "test_maxpool_2d_strides", "test_maxpool_3d_default", -"test_pow", // fp16 accuracy issue +"test_pow", +"test_quantizelinear", +"test_quantizelinear_axis", +"test_quantizelinear_blocked", "test_softmax_large_number", "test_softmax_large_number_expanded", "test_split_equal_parts_1d", diff --git a/modules/dnn/test/test_onnx_conformance_layer_filter_opencv_ocl_fp32_denylist.inl.hpp b/modules/dnn/test/test_onnx_conformance_layer_filter_opencv_ocl_fp32_denylist.inl.hpp index 7fe58a07fd..2453e2ad9f 100644 --- a/modules/dnn/test/test_onnx_conformance_layer_filter_opencv_ocl_fp32_denylist.inl.hpp +++ b/modules/dnn/test/test_onnx_conformance_layer_filter_opencv_ocl_fp32_denylist.inl.hpp @@ -1,5 +1,11 @@ "test_averagepool_3d_default", +"test_dequantizelinear", +"test_dequantizelinear_axis", +"test_dequantizelinear_blocked", "test_maxpool_3d_default", +"test_quantizelinear", +"test_quantizelinear_axis", +"test_quantizelinear_blocked", "test_scatter_elements_with_axis", "test_scatter_elements_with_duplicate_indices", "test_scatter_elements_with_negative_indices", diff --git a/modules/dnn/test/test_onnx_conformance_layer_parser_denylist.inl.hpp b/modules/dnn/test/test_onnx_conformance_layer_parser_denylist.inl.hpp index 78c26eeea2..7b408619d2 100644 --- a/modules/dnn/test/test_onnx_conformance_layer_parser_denylist.inl.hpp +++ b/modules/dnn/test/test_onnx_conformance_layer_parser_denylist.inl.hpp @@ -89,8 +89,6 @@ "test_convtranspose_pad", "test_convtranspose_pads", "test_convtranspose_with_kernel", -"test_dequantizelinear", -"test_dequantizelinear_axis", "test_det_2d", "test_det_nd", "test_dropout_default_mask", @@ -290,8 +288,6 @@ "test_qlinearconv", "test_qlinearmatmul_2D", "test_qlinearmatmul_3D", -"test_quantizelinear", -"test_quantizelinear_axis", "test_range_float_type_positive_delta", "test_range_float_type_positive_delta_expanded", "test_range_int32_type_negative_delta",