From 411fd2b761c7ad054d1bb9eeb496cdbd881da3fa Mon Sep 17 00:00:00 2001 From: Tiago De Gaspari Date: Mon, 7 Jun 2021 10:55:23 -0300 Subject: [PATCH 001/262] Add Thickness parameter in drawMatches function This commit adds the feature of selecting the thickness of the matches drawn by the drawMatches function. In larger images, the default thickness of 1 pixel creates images that are hard to visualize. --- .../features2d/include/opencv2/features2d.hpp | 7 ++++++ modules/features2d/src/draw.cpp | 25 ++++++++++++++++--- 2 files changed, 28 insertions(+), 4 deletions(-) diff --git a/modules/features2d/include/opencv2/features2d.hpp b/modules/features2d/include/opencv2/features2d.hpp index 86b5e935c8..cff09170c5 100644 --- a/modules/features2d/include/opencv2/features2d.hpp +++ b/modules/features2d/include/opencv2/features2d.hpp @@ -1314,6 +1314,13 @@ CV_EXPORTS_W void drawMatches( InputArray img1, const std::vector& key const std::vector& matchesMask=std::vector(), int flags=DrawMatchesFlags::DEFAULT ); /** @overload */ +CV_EXPORTS_W void drawMatches( InputArray img1, const std::vector& keypoints1, + InputArray img2, const std::vector& keypoints2, + const std::vector& matches1to2, InputOutputArray outImg, + const int matchesThickness, const Scalar& matchColor=Scalar::all(-1), + const Scalar& singlePointColor=Scalar::all(-1), const std::vector& matchesMask=std::vector(), + int flags=DrawMatchesFlags::DEFAULT ); + CV_EXPORTS_AS(drawMatchesKnn) void drawMatches( InputArray img1, const std::vector& keypoints1, InputArray img2, const std::vector& keypoints2, const std::vector >& matches1to2, InputOutputArray outImg, diff --git a/modules/features2d/src/draw.cpp b/modules/features2d/src/draw.cpp index dc74ecb080..e4c75144fb 100644 --- a/modules/features2d/src/draw.cpp +++ b/modules/features2d/src/draw.cpp @@ -183,7 +183,8 @@ static void _prepareImgAndDrawKeypoints( InputArray img1, const std::vector& keypoints1, @@ -207,6 +208,21 @@ void drawMatches( InputArray img1, const std::vector& keypoints1, const std::vector& matches1to2, InputOutputArray outImg, const Scalar& matchColor, const Scalar& singlePointColor, const std::vector& matchesMask, int flags ) +{ + drawMatches( img1, keypoints1, + img2, keypoints2, + matches1to2, outImg, + 1, matchColor, + singlePointColor, matchesMask, + flags); +} + +void drawMatches( InputArray img1, const std::vector& keypoints1, + InputArray img2, const std::vector& keypoints2, + const std::vector& matches1to2, InputOutputArray outImg, + const int matchesThickness, const Scalar& matchColor, + const Scalar& singlePointColor, const std::vector& matchesMask, + int flags ) { if( !matchesMask.empty() && matchesMask.size() != matches1to2.size() ) CV_Error( Error::StsBadSize, "matchesMask must have the same size as matches1to2" ); @@ -226,11 +242,12 @@ void drawMatches( InputArray img1, const std::vector& keypoints1, CV_Assert(i2 >= 0 && i2 < static_cast(keypoints2.size())); const KeyPoint &kp1 = keypoints1[i1], &kp2 = keypoints2[i2]; - _drawMatch( outImg, outImg1, outImg2, kp1, kp2, matchColor, flags ); + _drawMatch( outImg, outImg1, outImg2, kp1, kp2, matchColor, flags, matchesThickness ); } } } + void drawMatches( InputArray img1, const std::vector& keypoints1, InputArray img2, const std::vector& keypoints2, const std::vector >& matches1to2, InputOutputArray outImg, @@ -254,7 +271,7 @@ void drawMatches( InputArray img1, const std::vector& keypoints1, if( matchesMask.empty() || matchesMask[i][j] ) { const KeyPoint &kp1 = keypoints1[i1], &kp2 = keypoints2[i2]; - _drawMatch( outImg, outImg1, outImg2, kp1, kp2, matchColor, flags ); + _drawMatch( outImg, outImg1, outImg2, kp1, kp2, matchColor, flags, 1 ); } } } From 8f4f834ce6ff054b16aeb3c92f6ea1279185b632 Mon Sep 17 00:00:00 2001 From: Smirnov Egor Date: Wed, 9 Jun 2021 18:43:42 +0300 Subject: [PATCH 002/262] applied modifier mask to the state --- modules/highgui/src/window_gtk.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/modules/highgui/src/window_gtk.cpp b/modules/highgui/src/window_gtk.cpp index 17307ea7f9..78e78e12a2 100644 --- a/modules/highgui/src/window_gtk.cpp +++ b/modules/highgui/src/window_gtk.cpp @@ -1881,6 +1881,7 @@ static gboolean icvOnMouse( GtkWidget *widget, GdkEvent *event, gpointer user_da (unsigned)pt.y < (unsigned)(image_widget->original_image->height) )) { + state &= gtk_accelerator_get_default_mod_mask(); flags |= BIT_MAP(state, GDK_SHIFT_MASK, CV_EVENT_FLAG_SHIFTKEY) | BIT_MAP(state, GDK_CONTROL_MASK, CV_EVENT_FLAG_CTRLKEY) | BIT_MAP(state, GDK_MOD1_MASK, CV_EVENT_FLAG_ALTKEY) | From 7ee181661231a949d7998f3caaa9e4cba6250e3e Mon Sep 17 00:00:00 2001 From: rogday Date: Tue, 1 Jun 2021 17:05:27 +0300 Subject: [PATCH 003/262] split if into map of functions --- modules/dnn/src/tensorflow/tf_importer.cpp | 3443 +++++++++++--------- 1 file changed, 1833 insertions(+), 1610 deletions(-) diff --git a/modules/dnn/src/tensorflow/tf_importer.cpp b/modules/dnn/src/tensorflow/tf_importer.cpp index 084e4ac6da..39c2309394 100644 --- a/modules/dnn/src/tensorflow/tf_importer.cpp +++ b/modules/dnn/src/tensorflow/tf_importer.cpp @@ -510,10 +510,1834 @@ protected: private: void addPermuteLayer(const int* order, const std::string& permName, Pin& inpId); + + typedef void (TFImporter::*TFImporterNodeParser)(tensorflow::GraphDef&, const tensorflow::NodeDef&, LayerParams&); + typedef std::map DispatchMap; + + const DispatchMap dispatch; + static const DispatchMap buildDispatchMap(); + + void parseConvolution (tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams); + void parseBias (tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams); + void parseMatMul (tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams); + void parseReshape (tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams); + void parseFlatten (tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams); + void parseTranspose (tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams); + void parseConstant (tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams); + void parseLrn (tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams); + void parseConcat (tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams); + void parseMaxPool (tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams); + void parseAvgPool (tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams); + void parseMaxPoolGrad (tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams); + void parsePlaceholder (tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams); + void parseSplit (tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams); + void parseSlice (tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams); + void parseStridedSlice (tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams); + void parseMul (tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams); + void parseFusedBatchNorm (tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams); + void parseConv2DBackpropInput(tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams); + void parseBlockLSTM (tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams); + void parseResize (tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams); + void parseL2Normalize (tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams); + void parsePriorBox (tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams); + void parseSoftmax (tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams); + void parseCropAndResize (tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams); + void parseMean (tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams); + void parsePack (tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams); + void parseClipByValue (tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams); + void parseLeakyRelu (tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams); + void parseActivation (tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams); + + void parseCustomLayer (tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams); }; +const TFImporter::DispatchMap TFImporter::buildDispatchMap() +{ + static DispatchMap dispatch; + dispatch["Conv2D"] = dispatch["SpaceToBatchND"] = dispatch["DepthwiseConv2dNative"] = + dispatch["Pad"] = dispatch["MirrorPad"] = dispatch["Conv3D"] = &TFImporter::parseConvolution; + dispatch["BiasAdd"] = dispatch["Add"] = dispatch["AddV2"] = dispatch["Sub"] = dispatch["AddN"] = &TFImporter::parseBias; + dispatch["MatMul"] = &TFImporter::parseMatMul; + dispatch["Reshape"] = &TFImporter::parseReshape; + dispatch["Flatten"] = dispatch["Squeeze"] = &TFImporter::parseFlatten; + dispatch["Transpose"] = &TFImporter::parseTranspose; + dispatch["Const"] = &TFImporter::parseConstant; + dispatch["LRN"] = &TFImporter::parseLrn; + dispatch["Concat"] = dispatch["ConcatV2"] = &TFImporter::parseConcat; + dispatch["MaxPool"] = dispatch["MaxPool3D"] = &TFImporter::parseMaxPool; + dispatch["AvgPool"] = dispatch["AvgPool3D"] = &TFImporter::parseAvgPool; + dispatch["MaxPoolGrad"] = &TFImporter::parseMaxPoolGrad; + dispatch["Placeholder"] = &TFImporter::parsePlaceholder; + dispatch["Split"] = &TFImporter::parseSplit; + dispatch["Slice"] = &TFImporter::parseSlice; + dispatch["StridedSlice"] = &TFImporter::parseStridedSlice; + dispatch["Mul"] = dispatch["RealDiv"] = &TFImporter::parseMul; + dispatch["FusedBatchNorm"] = dispatch["FusedBatchNormV3"] = &TFImporter::parseFusedBatchNorm; + dispatch["Conv2DBackpropInput"] = &TFImporter::parseConv2DBackpropInput; + dispatch["BlockLSTM"] = &TFImporter::parseBlockLSTM; + dispatch["ResizeNearestNeighbor"] = dispatch["ResizeBilinear"] = dispatch["FusedResizeAndPadConv2D"] = &TFImporter::parseResize; + dispatch["L2Normalize"] = &TFImporter::parseL2Normalize; + dispatch["PriorBox"] = &TFImporter::parsePriorBox; + dispatch["Softmax"] = &TFImporter::parseSoftmax; + dispatch["CropAndResize"] = &TFImporter::parseCropAndResize; + dispatch["Mean"] = dispatch["Sum"] = &TFImporter::parseMean; + dispatch["Pack"] = &TFImporter::parsePack; + dispatch["ClipByValue"] = &TFImporter::parseClipByValue; + dispatch["LeakyRelu"] = &TFImporter::parseLeakyRelu; + dispatch["Abs"] = dispatch["Tanh"] = dispatch["Sigmoid"] = dispatch["Relu"] = + dispatch["Elu"] = dispatch["Exp"] = dispatch["Identity"] = dispatch["Relu6"] = &TFImporter::parseActivation; + + return dispatch; +} + +void TFImporter::parseConvolution(tensorflow::GraphDef& net, const tensorflow::NodeDef& layer_, LayerParams& layerParams) +{ + tensorflow::NodeDef layer = layer_; + std::string name = layer.name(); + std::string type = layer.op(); + int num_inputs = layer.input_size(); + + CV_CheckGT(num_inputs, 0, ""); + // The first node of dilated convolution subgraph. + // Extract input node, dilation rate and paddings. + std::string input = layer.input(0); + StrIntVector next_layers; + if (type == "SpaceToBatchND" || type == "Pad") + { + next_layers = getNextLayers(net, name, "Conv2D"); + if (next_layers.empty()) + next_layers = getNextLayers(net, name, "DepthwiseConv2dNative"); + } + + if (type == "SpaceToBatchND") + { + // op: "SpaceToBatchND" + // input: "input" + // input: "SpaceToBatchND/block_shape" + // input: "SpaceToBatchND/paddings" + CV_CheckEQ(num_inputs, 3, ""); + + DictValue dilation = parseDims(getConstBlob(layer, value_id, 1)); + CV_Assert(dilation.size() == 2); + layerParams.set("dilation_h", dilation.get(0)); + layerParams.set("dilation_w", dilation.get(1)); + + Mat paddings; + parseTensor(getConstBlob(layer, value_id, 2), paddings); + + // paddings is a 2x2 matrix: [[top, bot], [left, right]] + layerParams.set("pad_h", paddings.at(0)); + layerParams.set("pad_w", paddings.at(2)); + + CV_Assert(next_layers.size() == 1); + layers_to_ignore.insert(next_layers[0].first); + + // FIXIT don't override, rewrite this code + layer = net.node(next_layers[0].second); + name = layer.name(); + type = layer.op(); + num_inputs = layer.input_size(); + CV_LOG_DEBUG(NULL, "DNN/TF: switched to layer " << name << " @ " << type << ") with " << num_inputs << " inputs"); + } + else if (type == "Pad" || type == "MirrorPad") + { + Mat paddings = getTensorContent(getConstBlob(layer, value_id, 1)); + CV_Assert(paddings.type() == CV_32SC1); + if (paddings.total() == 8) + { + // Perhaps, we have NHWC padding dimensions order. + // N H W C + // 0 1 2 3 4 5 6 7 + std::swap(paddings.at(2), paddings.at(6)); + std::swap(paddings.at(3), paddings.at(7)); + // N C W H + // 0 1 2 3 4 5 6 7 + std::swap(paddings.at(4), paddings.at(6)); + std::swap(paddings.at(5), paddings.at(7)); + // N C H W + // 0 1 2 3 4 5 6 7 + } + + if (next_layers.empty() || paddings.total() != 8 || + paddings.at(4) != paddings.at(5) || + paddings.at(6) != paddings.at(7) || type == "MirrorPad") + { + // Just a single padding layer. + layerParams.set("paddings", DictValue::arrayInt((int*)paddings.data, paddings.total())); + if (type == "MirrorPad") + layerParams.set("type", "reflect"); + + int id = dstNet.addLayer(name, "Padding", layerParams); + layer_id[name] = id; + + connect(layer_id, dstNet, parsePin(input), id, 0); + return; + } + else + { + // Merge with subsequent convolutional layer. + CV_Assert(next_layers.size() == 1); + + layerParams.set("pad_h", paddings.at(4)); + layerParams.set("pad_w", paddings.at(6)); + + layers_to_ignore.insert(next_layers[0].first); + + // FIXIT don't override, rewrite this code + layer = net.node(next_layers[0].second); + name = layer.name(); + type = layer.op(); + num_inputs = layer.input_size(); + CV_LOG_DEBUG(NULL, "DNN/TF: switched to layer " << name << " @ " << type << ") with " << num_inputs << " inputs"); + } + } + + // For the object detection networks, TensorFlow Object Detection API + // predicts deltas for bounding boxes in yxYX (ymin, xmin, ymax, xmax) + // order. We can manage it at DetectionOutput layer parsing predictions + // or shuffle last convolution's weights. + bool locPredTransposed = hasLayerAttr(layer, "loc_pred_transposed") && + getLayerAttr(layer, "loc_pred_transposed").b(); + + layerParams.set("bias_term", false); + layerParams.blobs.resize(1); + + next_layers = getNextLayers(net, name, "BiasAdd"); + if (next_layers.size() == 1) { + layerParams.set("bias_term", true); + layerParams.blobs.resize(2); + + int weights_layer_index = next_layers[0].second; + + blobFromTensor(getConstBlob(net.node(weights_layer_index), value_id), layerParams.blobs[1]); + ExcludeLayer(net, weights_layer_index, 0, false); + layers_to_ignore.insert(next_layers[0].first); + + // Shuffle bias from yxYX to xyXY. + if (locPredTransposed) + { + const int numWeights = layerParams.blobs[1].total(); + float* biasData = reinterpret_cast(layerParams.blobs[1].data); + CV_Assert(numWeights % 4 == 0); + for (int i = 0; i < numWeights; i += 2) + { + std::swap(biasData[i], biasData[i + 1]); + } + } + } + + int kernelTensorInpId = -1; + const tensorflow::TensorProto& kernelTensor = getConstBlob(layer, value_id, -1, &kernelTensorInpId); + const String kernelTensorName = layer.input(kernelTensorInpId); + std::map::iterator sharedWeightsIt = sharedWeights.find(kernelTensorName); + if (sharedWeightsIt == sharedWeights.end()) + { + kernelFromTensor(kernelTensor, layerParams.blobs[0]); + releaseTensor(const_cast(&kernelTensor)); + + int* kshape = layerParams.blobs[0].size.p; + const int outCh = kshape[0]; + const int inCh = kshape[1]; + const int height = kshape[2]; + const int width = kshape[3]; + if (type == "DepthwiseConv2dNative") + { + CV_Assert(!locPredTransposed); + const int chMultiplier = kshape[0]; + + Mat copy = layerParams.blobs[0].clone(); + float* src = (float*)copy.data; + float* dst = (float*)layerParams.blobs[0].data; + for (int i = 0; i < chMultiplier; ++i) + for (int j = 0; j < inCh; ++j) + for (int s = 0; s < height * width; ++s) + { + int src_i = (i * inCh + j) * height * width + s; + int dst_i = (j * chMultiplier + i) * height* width + s; + dst[dst_i] = src[src_i]; + } + // TODO Use reshape instead + kshape[0] = inCh * chMultiplier; + kshape[1] = 1; + size_t* kstep = layerParams.blobs[0].step.p; + kstep[0] = kstep[1]; // fix steps too + } + + // Shuffle output channels from yxYX to xyXY. + if (locPredTransposed) + { + const int slice = height * width * inCh; + for (int i = 0; i < outCh; i += 2) + { + cv::Mat src(1, slice, CV_32F, layerParams.blobs[0].ptr(i)); + cv::Mat dst(1, slice, CV_32F, layerParams.blobs[0].ptr(i + 1)); + std::swap_ranges(src.begin(), src.end(), dst.begin()); + } + } + sharedWeights[kernelTensorName] = layerParams.blobs[0]; + } + else + { + layerParams.blobs[0] = sharedWeightsIt->second; + } + Mat weights = layerParams.blobs[0]; + layerParams.set("kernel_size", DictValue::arrayInt(&weights.size[2], weights.dims - 2)); + + layerParams.set("num_output", layerParams.blobs[0].size[0]); + + setStrides(layerParams, layer); + if (!layerParams.has("pad_w") && !layerParams.has("pad_h")) + setPadding(layerParams, layer); + + // The final node of dilated convolution subgraph. + next_layers = getNextLayers(net, name, "BatchToSpaceND"); + if (!next_layers.empty()) + { + CV_Assert(next_layers.size() == 1); + ExcludeLayer(net, next_layers[0].second, 0, false); + layers_to_ignore.insert(next_layers[0].first); + } + + int id = dstNet.addLayer(name, "Convolution", layerParams); + layer_id[name] = id; + + // one input only + connect(layer_id, dstNet, parsePin(input), id, 0); + + + if (getDataLayout(name, data_layouts) == DATA_LAYOUT_UNKNOWN) + data_layouts[name] = DATA_LAYOUT_NHWC; +} + +void TFImporter::parseBias(tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams) +{ + const std::string& name = layer.name(); + const std::string& type = layer.op(); + const int num_inputs = layer.input_size(); + + CV_CheckGT(num_inputs, 0, ""); + bool haveConst = false; + for(int ii = 0; !haveConst && ii < num_inputs; ++ii) + { + Pin input = parsePin(layer.input(ii)); + haveConst = value_id.find(input.name) != value_id.end(); + } + CV_Assert(!haveConst || num_inputs == 2); + + if (haveConst) + { + Mat values = getTensorContent(getConstBlob(layer, value_id)); + CV_Assert(values.type() == CV_32FC1); + if (type == "Sub") + values *= -1.0f; + + int id; + if (values.total() == 1) // is a scalar. + { + layerParams.set("shift", values.at(0)); + id = dstNet.addLayer(name, "Power", layerParams); + } + else // is a vector + { + layerParams.blobs.resize(1, values); + id = dstNet.addLayer(name, "Shift", layerParams); + } + layer_id[name] = id; + + // one input only + connect(layer_id, dstNet, parsePin(layer.input(0)), id, 0); + } + else + { + layerParams.set("operation", "sum"); + if (type == "Sub") + { + static float subCoeffs[] = {1.f, -1.f}; + layerParams.set("coeff", DictValue::arrayReal(subCoeffs, 2)); + } + + int id = dstNet.addLayer(name, "Eltwise", layerParams); + layer_id[name] = id; + + for (int ii = 0; ii < num_inputs; ii++) + { + Pin inp = parsePin(layer.input(ii)); + if (layer_id.find(inp.name) == layer_id.end()) + CV_Error(Error::StsError, "Input layer not found: " + inp.name); + connect(layer_id, dstNet, inp, id, ii); + } + } +} + +void TFImporter::parseMatMul(tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams) +{ + const std::string& name = layer.name(); + const int num_inputs = layer.input_size(); + + CV_CheckEQ(num_inputs, 2, ""); + + // For the object detection networks, TensorFlow Object Detection API + // predicts deltas for bounding boxes in yxYX (ymin, xmin, ymax, xmax) + // order. We can manage it at DetectionOutput layer parsing predictions + // or shuffle last Faster-RCNN's matmul weights. + bool locPredTransposed = hasLayerAttr(layer, "loc_pred_transposed") && + getLayerAttr(layer, "loc_pred_transposed").b(); + + layerParams.set("bias_term", false); + layerParams.blobs.resize(1); + + StrIntVector next_layers = getNextLayers(net, name, "BiasAdd"); // FIXIT Use layers fusion instead + if (next_layers.empty()) + { + next_layers = getNextLayers(net, name, "Add"); + } + if (next_layers.size() == 1) { + layerParams.set("bias_term", true); + layerParams.blobs.resize(2); + + int weights_layer_index = next_layers[0].second; + blobFromTensor(getConstBlob(net.node(weights_layer_index), value_id), layerParams.blobs[1]); + ExcludeLayer(net, weights_layer_index, 0, false); + layers_to_ignore.insert(next_layers[0].first); + + if (locPredTransposed) + { + const int numWeights = layerParams.blobs[1].total(); + float* biasData = reinterpret_cast(layerParams.blobs[1].data); + CV_Assert(numWeights % 4 == 0); + for (int i = 0; i < numWeights; i += 2) + { + std::swap(biasData[i], biasData[i + 1]); + } + } + } + + int kernel_blob_index = -1; + const tensorflow::TensorProto& kernelTensor = getConstBlob(layer, value_id, -1, &kernel_blob_index); + const String kernelTensorName = layer.input(kernel_blob_index); + std::map::iterator sharedWeightsIt = sharedWeights.find(kernelTensorName); + if (sharedWeightsIt == sharedWeights.end()) + { + blobFromTensor(kernelTensor, layerParams.blobs[0]); + releaseTensor(const_cast(&kernelTensor)); + sharedWeights[kernelTensorName] = layerParams.blobs[0]; + } + else + { + layerParams.blobs[0] = sharedWeightsIt->second; + } + + if (kernel_blob_index == 1) { // In this case output is computed by x*W formula - W should be transposed + Mat data = layerParams.blobs[0].t(); + layerParams.blobs[0] = data.clone(); + } + + layerParams.set("num_output", layerParams.blobs[0].size[0]); + if (locPredTransposed) + { + CV_Assert(layerParams.blobs[0].dims == 2); + for (int i = 0; i < layerParams.blobs[0].size[0]; i += 2) + { + cv::Mat src = layerParams.blobs[0].row(i); + cv::Mat dst = layerParams.blobs[0].row(i + 1); + std::swap_ranges(src.begin(), src.end(), dst.begin()); + } + } + + int id = dstNet.addLayer(name, "InnerProduct", layerParams); + layer_id[name] = id; + + // one input only + int input_blob_index = kernel_blob_index == 0 ? 1 : 0; + connect(layer_id, dstNet, parsePin(layer.input(input_blob_index)), id, 0); + data_layouts[name] = DATA_LAYOUT_PLANAR; +} + +void TFImporter::parseReshape(tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams) +{ + const std::string& name = layer.name(); + const int num_inputs = layer.input_size(); + + CV_CheckGT(num_inputs, 0, ""); + Pin inpId = parsePin(layer.input(0)); + DataLayout inpLayout = getDataLayout(layer.input(0), data_layouts); + // There are two possible implementations: reshape an input using + // predefined sizes or use a second input blob as a source of new shape. + if (value_id.find(layer.input(1)) != value_id.end()) + { + Mat newShape = getTensorContent(getConstBlob(layer, value_id, 1)); + int newShapeSize = newShape.total(); + bool hasSwap = false; + if (newShapeSize == 4 && hasAllOnes(newShape, 0, 2)) + { + // NHWC->NCHW + std::swap(*newShape.ptr(0, 2), *newShape.ptr(0, 3)); + std::swap(*newShape.ptr(0, 1), *newShape.ptr(0, 2)); + hasSwap = true; + } + if (inpLayout == DATA_LAYOUT_NHWC) + { + if (newShapeSize >= 2 || newShape.at(1) == 1) + { + int order[] = {0, 2, 3, 1}; // From OpenCV's NCHW to NHWC. + addPermuteLayer(order, name + "/nhwc", inpId); + if (newShapeSize < 4) + { + inpLayout = DATA_LAYOUT_NCHW; + } + else + { + inpLayout = DATA_LAYOUT_NHWC; + } + } + } + layerParams.set("dim", DictValue::arrayInt(newShape.ptr(), newShapeSize)); + + int id = dstNet.addLayer(name, "Reshape", layerParams); + layer_id[name] = id; + + // one input only + connect(layer_id, dstNet, inpId, id, 0); + inpId = Pin(name); + + if ((inpLayout == DATA_LAYOUT_NHWC || inpLayout == DATA_LAYOUT_UNKNOWN || inpLayout == DATA_LAYOUT_PLANAR) && + newShapeSize == 4 && !hasSwap) + { + int order[] = {0, 3, 1, 2}; // Transform back to OpenCV's NCHW. + addPermuteLayer(order, name + "/nchw", inpId); + inpLayout = DATA_LAYOUT_NCHW; + } + + data_layouts[name] = newShapeSize == 2 ? DATA_LAYOUT_PLANAR : inpLayout; + } + else + { + int id = dstNet.addLayer(name, "Reshape", layerParams); + layer_id[name] = id; + connect(layer_id, dstNet, inpId, id, 0); + connect(layer_id, dstNet, parsePin(layer.input(1)), id, 1); + data_layouts[name] = inpLayout; + } +} + +void TFImporter::parseFlatten(tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams) +{ + const std::string& name = layer.name(); + const std::string& type = layer.op(); + const int num_inputs = layer.input_size(); + + CV_CheckGT(num_inputs, 0, ""); + Pin inpId = parsePin(layer.input(0)); + int inpLayout = getDataLayout(layer.input(0), data_layouts); + if (type == "Squeeze") + { + CV_Assert(hasLayerAttr(layer, "squeeze_dims")); + const tensorflow::AttrValue& dims = getLayerAttr(layer, "squeeze_dims"); + std::vector dimsVector(dims.list().i_size()); + for (int i = 0; i < dimsVector.size(); ++i) + dimsVector[i] = dims.list().i(i); + + // Flatten layer can squeeze dimensions range into one. + std::sort(dimsVector.begin(), dimsVector.end()); + for (int i = 1; i < dimsVector.size(); ++i) + { + if (dimsVector[i] != dimsVector[i - 1] + 1) + CV_Error(Error::StsNotImplemented, "Unsupported squeeze configuration"); + } + int start = dimsVector.front() - 1, end = dimsVector.back(); + if (start == -1 && end == 0) // squeeze 0th dimension + { + start = 0; + end = 1; + } + layerParams.set("axis", start); + layerParams.set("end_axis", end); + } + if (inpLayout == DATA_LAYOUT_NHWC) + { + LayerParams permLP; + int order[] = {0, 2, 3, 1}; // From OpenCV's NCHW to NHWC. + permLP.set("order", DictValue::arrayInt(order, 4)); + + std::string permName = name + "/nchw"; + CV_Assert(layer_id.find(permName) == layer_id.end()); + int permId = dstNet.addLayer(permName, "Permute", permLP); + layer_id[permName] = permId; + connect(layer_id, dstNet, inpId, permId, 0); + inpId = Pin(permName); + } + int id = dstNet.addLayer(name, "Flatten", layerParams); + layer_id[name] = id; + connect(layer_id, dstNet, inpId, id, 0); + data_layouts[name] = DATA_LAYOUT_PLANAR; +} + +void TFImporter::parseTranspose(tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams) +{ + const std::string& name = layer.name(); + const int num_inputs = layer.input_size(); + + CV_CheckGT(num_inputs, 0, ""); + Mat perm = getTensorContent(getConstBlob(layer, value_id, 1)); + CV_Assert(perm.type() == CV_32SC1); + int* permData = (int*)perm.data; + if (perm.total() == 4) + { + // Only NHWC <-> NCHW permutations are allowed. OpenCV is always + // keep NCHW layout this way. + int inpLayout = getDataLayout(layer.input(0), data_layouts); + std::string type = "Identity"; + if (inpLayout == DATA_LAYOUT_NHWC) + { + if (permData[0] == 0 && permData[1] == 3 && permData[2] == 1 && permData[3] == 2) + { + // in TensorFlow: NHWC->NCHW + // in OpenCV: NCHW->NCHW + data_layouts[name] = DATA_LAYOUT_NCHW; + } + else if (permData[0] == 0 && permData[1] == 1 && permData[2] == 2 && permData[3] == 3) + { + // in TensorFlow: NHWC->NHWC + // in OpenCV: NCHW->NCHW + data_layouts[name] = DATA_LAYOUT_NHWC; + } + else if (permData[0] == 0 && permData[1] == 3 && permData[2] == 2 && permData[3] == 1) + { + // in TensorFlow: NHWC->NCWH + // in OpenCV: NCHW->NCWH + int permData[] = {0, 1, 3, 2}; + layerParams.set("order", DictValue::arrayInt(permData, perm.total())); + data_layouts[name] = DATA_LAYOUT_NCHW; // we keep track NCHW because channels position only matters + type = "Permute"; + } + else + CV_Error(Error::StsParseError, "Only NHWC <-> NCHW permutations are allowed."); + } + else if (inpLayout == DATA_LAYOUT_NCHW) + { + if (permData[0] == 0 && permData[1] == 2 && permData[2] == 3 && permData[3] == 1) + { + // in TensorFlow: NCHW->NHWC + // in OpenCV: NCHW->NCHW + data_layouts[name] = DATA_LAYOUT_NHWC; + } + else if (permData[0] == 0 && permData[1] == 1 && permData[2] == 2 && permData[3] == 3) + { + // in TensorFlow: NCHW->NCHW + // in OpenCV: NCHW->NCHW + data_layouts[name] = DATA_LAYOUT_NCHW; + } + else + CV_Error(Error::StsParseError, "Only NHWC <-> NCHW permutations are allowed."); + } + int id = dstNet.addLayer(name, type, layerParams); + layer_id[name] = id; + connect(layer_id, dstNet, parsePin(layer.input(0)), id, 0); + } + else + { + layerParams.set("order", DictValue::arrayInt(permData, perm.total())); + + int id = dstNet.addLayer(name, "Permute", layerParams); + layer_id[name] = id; + + // one input only + connect(layer_id, dstNet, parsePin(layer.input(0)), id, 0); + data_layouts[name] = DATA_LAYOUT_UNKNOWN; + } +} + +void TFImporter::parseConstant(tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams) +{ +} + +void TFImporter::parseLrn(tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams) +{ + const std::string& name = layer.name(); + const int num_inputs = layer.input_size(); + + CV_CheckGT(num_inputs, 0, ""); + if(hasLayerAttr(layer, "alpha")) { + layerParams.set("alpha", getLayerAttr(layer, "alpha").f()); + } + if(hasLayerAttr(layer, "beta")) { + layerParams.set("beta", getLayerAttr(layer, "beta").f()); + } + if(hasLayerAttr(layer, "depth_radius")) { + int radius = (int)getLayerAttr(layer, "depth_radius").i(); + layerParams.set("local_size", 2*radius + 1); + } + if(hasLayerAttr(layer, "bias")) { + layerParams.set("bias", getLayerAttr(layer, "bias").f()); + } + layerParams.set("norm_by_size", false); + + int id = dstNet.addLayer(name, "LRN", layerParams); + layer_id[name] = id; + + connectToAllBlobs(layer_id, dstNet, parsePin(layer.input(0)), id, num_inputs); +} + +void TFImporter::parseConcat(tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams) +{ + const std::string& name = layer.name(); + const std::string& type = layer.op(); + const int num_inputs = layer.input_size(); + + CV_CheckGT(num_inputs, 0, ""); + int axisId = (type == "Concat" ? 0 : num_inputs - 1); + int axis = getConstBlob(layer, value_id, axisId).int_val().Get(0); + + if (getDataLayout(name, data_layouts) == DATA_LAYOUT_NHWC) + axis = toNCHW(axis); + else if (getDataLayout(name, data_layouts) == DATA_LAYOUT_NDHWC) + axis = toNCDHW(axis); + layerParams.set("axis", axis); + + // input(0) or input(n-1) is concat_dim + int from = (type == "Concat" ? 1 : 0); + int to = (type == "Concat" ? num_inputs : num_inputs - 1); + + for (int ii = from; ii < to; ii++) + { + Pin inp = parsePin(layer.input(ii)); + if (layer_id.find(inp.name) == layer_id.end()) + { + // There are constant inputs. + LayerParams lp; + lp.name = inp.name; + lp.type = "Const"; + lp.blobs.resize(1); + blobFromTensor(getConstBlob(layer, value_id, ii), lp.blobs.back()); + CV_Assert_N(!lp.blobs[0].empty(), lp.blobs[0].type() == CV_32F); + + int constInpId = dstNet.addLayer(lp.name, lp.type, lp); + layer_id[lp.name] = constInpId; + } + } + + int id = dstNet.addLayer(name, "Concat", layerParams); + layer_id[name] = id; + + for (int ii = from; ii < to; ii++) + { + Pin inp = parsePin(layer.input(ii)); + if (layer_id.find(inp.name) == layer_id.end()) + CV_Error(Error::StsError, "Input layer not found: " + inp.name); + connect(layer_id, dstNet, inp, id, ii - from); + } +} + +void TFImporter::parseMaxPool(tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams) +{ + const std::string& name = layer.name(); + const int num_inputs = layer.input_size(); + + CV_CheckGT(num_inputs, 0, ""); + layerParams.set("pool", "max"); + + setKSize(layerParams, layer); + setStrides(layerParams, layer); + setPadding(layerParams, layer); + // Test_TensorFlow_nets.EAST_text_detection/1, NGRAPH/CPU + layerParams.set("ceil_mode", false); + + int id = dstNet.addLayer(name, "Pooling", layerParams); + layer_id[name] = id; + + connectToAllBlobs(layer_id, dstNet, parsePin(layer.input(0)), id, num_inputs); +} + +void TFImporter::parseAvgPool(tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams) +{ + const std::string& name = layer.name(); + const int num_inputs = layer.input_size(); + + CV_CheckGT(num_inputs, 0, ""); + layerParams.set("pool", "ave"); + layerParams.set("ave_pool_padded_area", false); + setKSize(layerParams, layer); + setStrides(layerParams, layer); + setPadding(layerParams, layer); + + int id = dstNet.addLayer(name, "Pooling", layerParams); + layer_id[name] = id; + + connectToAllBlobs(layer_id, dstNet, parsePin(layer.input(0)), id, num_inputs); +} + +void TFImporter::parseMaxPoolGrad(tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams) +{ + const std::string& name = layer.name(); + const int num_inputs = layer.input_size(); + + CV_CheckEQ(num_inputs, 3, ""); + + layerParams.set("pool_k_h", 0); + layerParams.set("pool_k_w", 0); + layerParams.set("pool_stride_h", 0); + layerParams.set("pool_stride_w", 0); + layerParams.set("pool_pad_h", 0); + layerParams.set("pool_pad_w", 0); + + int id = dstNet.addLayer(name, "MaxUnpool", layerParams); + layer_id[name] = id; + + connect(layer_id, dstNet, parsePin(layer.input(2)), id, 0); + connect(layer_id, dstNet, parsePin(layer.input(1) + ":1"), id, 1); + connect(layer_id, dstNet, parsePin(layer.input(0)), id, 2); +} + +void TFImporter::parsePlaceholder(tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams) +{ + const std::string& name = layer.name(); + + DataLayout predictedLayout = data_layouts[name]; + + if (!hasLayerAttr(layer, "dtype") || + getLayerAttr(layer, "dtype").type() != tensorflow::DT_BOOL) // If input is not a train/test flag. + { + netInputsNames.push_back(name); + layer_id[name] = 0; + } + tensorflow::TensorShapeProto shape; + if (hasLayerAttr(layer, "shape")) + shape = getLayerAttr(layer, "shape").shape(); + else if (hasLayerAttr(layer, "_output_shapes")) + { + tensorflow::AttrValue_ListValue list = getLayerAttr(layer, "_output_shapes").list(); + if (list.shape_size()) + shape = list.shape()[0]; + } + if (shape.dim_size()) + { + MatShape dims(shape.dim_size()); + for (int i = 0; i < dims.size(); ++i) + dims[i] = shape.dim(i).size(); + if (dims.size() == 4 && predictedLayout == DATA_LAYOUT_NHWC) + { + std::swap(dims[1], dims[3]); // NHWC->NCWH + std::swap(dims[2], dims[3]); // NCWH->NCHW + if (dims[0] == -1) // It's OK to have undetermined batch size + dims[0] = 1; + } + bool hasNeg = false; + for (int i = 0; i < dims.size() && !hasNeg; ++i) + { + hasNeg = dims[i] < 0; + } + if (!hasNeg) + netInputShapes.push_back(dims); + } +} + +void TFImporter::parseSplit(tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams) +{ + // TODO: determining axis index remapping by input dimensions order of input blob + // TODO: slicing input may be Const op + // TODO: slicing kernels for convolutions - in current implementation it is impossible + // TODO: add parsing num of slices parameter + const std::string& name = layer.name(); + const int num_inputs = layer.input_size(); + + CV_CheckEQ(num_inputs, 2, ""); + // num_split + // 1st blob is dims tensor + int axis = getConstBlob(layer, value_id, 0).int_val().Get(0); + if (getDataLayout(name, data_layouts) == DATA_LAYOUT_NHWC) + axis = toNCHW(axis); + layerParams.set("axis", axis); + + if (hasLayerAttr(layer, "num_split")) + layerParams.set("num_split", getLayerAttr(layer, "num_split").i()); + + int id = dstNet.addLayer(name, "Slice", layerParams); + layer_id[name] = id; + + // one input only + connect(layer_id, dstNet, parsePin(layer.input(1)), id, 0); +} + +void TFImporter::parseSlice(tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams) +{ + // op: "Slice" + // input: "input_node" + // input: "Slice/begin" + // input: "Slice/size" + const std::string& name = layer.name(); + const int num_inputs = layer.input_size(); + + CV_CheckEQ(num_inputs, 3, ""); + Mat begins = getTensorContent(getConstBlob(layer, value_id, 1)); + Mat sizes = getTensorContent(getConstBlob(layer, value_id, 2)); + CV_Assert_N(!begins.empty(), !sizes.empty()); + CV_CheckTypeEQ(begins.type(), CV_32SC1, ""); + CV_CheckTypeEQ(sizes.type(), CV_32SC1, ""); + + if (begins.total() == 4 && getDataLayout(name, data_layouts) == DATA_LAYOUT_NHWC) + { + // Swap NHWC parameters' order to NCHW. + std::swap(*begins.ptr(0, 2), *begins.ptr(0, 3)); + std::swap(*begins.ptr(0, 1), *begins.ptr(0, 2)); + std::swap(*sizes.ptr(0, 2), *sizes.ptr(0, 3)); + std::swap(*sizes.ptr(0, 1), *sizes.ptr(0, 2)); + } + layerParams.set("begin", DictValue::arrayInt((int*)begins.data, begins.total())); + layerParams.set("size", DictValue::arrayInt((int*)sizes.data, sizes.total())); + + int id = dstNet.addLayer(name, "Slice", layerParams); + layer_id[name] = id; + + connect(layer_id, dstNet, parsePin(layer.input(0)), id, 0); +} + +void TFImporter::parseStridedSlice(tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams) +{ + const std::string& name = layer.name(); + const int num_inputs = layer.input_size(); + + CV_CheckEQ(num_inputs, 4, ""); + Mat begins = getTensorContent(getConstBlob(layer, value_id, 1)); + Mat ends = getTensorContent(getConstBlob(layer, value_id, 2)); + Mat strides = getTensorContent(getConstBlob(layer, value_id, 3)); + CV_CheckTypeEQ(begins.type(), CV_32SC1, ""); + CV_CheckTypeEQ(ends.type(), CV_32SC1, ""); + CV_CheckTypeEQ(strides.type(), CV_32SC1, ""); + const int num = begins.total(); + CV_Assert_N(num == ends.total(), num == strides.total()); + + int end_mask = getLayerAttr(layer, "end_mask").i(); + for (int i = 0; i < num; ++i) + { + if (ends.at(i) < 0) + ends.at(i) -= 1; + if (end_mask & (1 << i)) + ends.at(i) = -1; + if (strides.at(i) != 1) + CV_Error(Error::StsNotImplemented, + format("StridedSlice with stride %d", strides.at(i))); + } + if (begins.total() == 4 && getDataLayout(name, data_layouts) == DATA_LAYOUT_NHWC) + { + // Swap NHWC parameters' order to NCHW. + std::swap(begins.at(2), begins.at(3)); + std::swap(begins.at(1), begins.at(2)); + std::swap(ends.at(2), ends.at(3)); + std::swap(ends.at(1), ends.at(2)); + } + layerParams.set("begin", DictValue::arrayInt((int*)begins.data, begins.total())); + layerParams.set("end", DictValue::arrayInt((int*)ends.data, ends.total())); + + int id = dstNet.addLayer(name, "Slice", layerParams); + layer_id[name] = id; + + connect(layer_id, dstNet, parsePin(layer.input(0)), id, 0); +} + +void TFImporter::parseMul(tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams) +{ + const std::string& name = layer.name(); + const std::string& type = layer.op(); + const int num_inputs = layer.input_size(); + + CV_CheckGT(num_inputs, 0, ""); + int constId = -1; + for(int ii = 0; ii < num_inputs; ++ii) + { + Pin input = parsePin(layer.input(ii)); + if (value_id.find(input.name) != value_id.end()) + { + constId = ii; + break; + } + } + CV_Assert((constId != -1) || (num_inputs == 2)); + + if (constId != -1) + { + // Multiplication by constant. + CV_CheckEQ(num_inputs, 2, ""); + Mat scaleMat = getTensorContent(getConstBlob(layer, value_id)); + CV_Assert(scaleMat.type() == CV_32FC1); + if (type == "RealDiv") + { + if (constId == 0) + CV_Error(Error::StsNotImplemented, "Division of constant over variable"); + scaleMat = 1.0f / scaleMat; + } + + int id; + if (scaleMat.total() == 1) // is a scalar. + { + // Try to match with a LeakyRelu: + // node { + // name: "LeakyRelu/mul" + // op: "Mul" + // input: "LeakyRelu/alpha" + // input: "input" + // } + // node { + // name: "LeakyRelu/Maximum" + // op: "Maximum" + // input: "LeakyRelu/mul" + // input: "input" + // } + StrIntVector next_layers = getNextLayers(net, name, "Maximum"); + if (!next_layers.empty()) + { + int maximumLayerIdx = next_layers[0].second; + + CV_Assert(net.node(maximumLayerIdx).input_size() == 2); + + // The input from the Mul layer can also be at index 1. + int mulInputIdx = (net.node(maximumLayerIdx).input(0) == name) ? 0 : 1; + + ExcludeLayer(net, maximumLayerIdx, mulInputIdx, false); + layers_to_ignore.insert(next_layers[0].first); + + layerParams.set("negative_slope", scaleMat.at(0)); + id = dstNet.addLayer(name, "ReLU", layerParams); + } + else + { + // Just a multiplication. + layerParams.set("scale", scaleMat.at(0)); + id = dstNet.addLayer(name, "Power", layerParams); + } + } + else // is a vector + { + layerParams.blobs.resize(1, scaleMat); + + StrIntVector next_layers = getNextLayers(net, name, "Add"); + if (!next_layers.empty()) + { + layerParams.set("bias_term", true); + layerParams.blobs.resize(2); + + int weights_layer_index = next_layers[0].second; + blobFromTensor(getConstBlob(net.node(weights_layer_index), value_id), layerParams.blobs.back()); + ExcludeLayer(net, weights_layer_index, 0, false); + layers_to_ignore.insert(next_layers[0].first); + } + + if (hasLayerAttr(layer, "axis")) + layerParams.set("axis", getLayerAttr(layer, "axis").i()); + + id = dstNet.addLayer(name, "Scale", layerParams); + } + layer_id[name] = id; + + Pin inp0 = parsePin(layer.input(0)); + if (layer_id.find(inp0.name) != layer_id.end()) + // First operand is a constant. + connect(layer_id, dstNet, parsePin(layer.input(0)), id, 0); + else + connect(layer_id, dstNet, parsePin(layer.input(1)), id, 0); + } + else + { + // Check if all the inputs have the same shape. + bool equalInpShapes = true; + bool isShapeOnes = false; + MatShape outShape0; + for (int ii = 0; ii < num_inputs && !netInputShapes.empty(); ii++) + { + Pin pin = parsePin(layer.input(ii)); + int inpId = layer_id.find(pin.name)->second; + + // Get input shape + MatShape outShape; + std::vector inpShapes, outShapes; + dstNet.getLayerShapes(netInputShapes, inpId, inpShapes, outShapes); + CV_CheckGT(static_cast(outShapes.size()), pin.blobIndex, ""); + outShape = outShapes[pin.blobIndex]; + + if (ii == 0) + { + outShape0 = outShape; + } + else if (outShape != outShape0) + { + equalInpShapes = false; + isShapeOnes = isAllOnes(outShape, 2, outShape.size()) || + isAllOnes(outShape0, 2, outShape0.size()); + break; + } + } + + int id; + if (equalInpShapes || netInputShapes.empty() || (!equalInpShapes && isShapeOnes)) + { + layerParams.set("operation", type == "RealDiv" ? "div" : "prod"); + id = dstNet.addLayer(name, "Eltwise", layerParams); + } + else + { + if (type == "RealDiv") + CV_Error(Error::StsNotImplemented, "Division of non equal tensors"); + id = dstNet.addLayer(name, "Scale", layerParams); + } + + layer_id[name] = id; + + for (int ii = 0; ii < num_inputs; ii++) + { + Pin inp = parsePin(layer.input(ii)); + if (layer_id.find(inp.name) == layer_id.end()) + CV_Error(Error::StsError, "Input layer not found: " + inp.name); + connect(layer_id, dstNet, inp, id, ii); + } + } +} + +void TFImporter::parseFusedBatchNorm(tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams) +{ + // op: "FusedBatchNorm" + // input: "input" + // input: "BatchNorm/gamma" + // input: "BatchNorm/beta" + // input: "BatchNorm/moving_mean" + // input: "BatchNorm/moving_variance" + + const std::string& name = layer.name(); + const int num_inputs = layer.input_size(); + + CV_CheckEQ(num_inputs, 5, "Expected gamma, beta, mean and std"); + Pin inpId = parsePin(layer.input(0)); + + bool isTraining = hasLayerAttr(layer, "is_training") && getLayerAttr(layer, "is_training").b(); + + layerParams.blobs.resize(2); + + const tensorflow::TensorProto& gammaTensor = getConstBlob(layer, value_id, 1); + if (!gammaTensor.tensor_content().empty()) + { + layerParams.blobs.resize(layerParams.blobs.size() + 1); + layerParams.set("has_weight", true); + blobFromTensor(gammaTensor, layerParams.blobs.back()); + } + else + layerParams.set("has_weight", false); + + const tensorflow::TensorProto& betaTensor = getConstBlob(layer, value_id, 2); + if (!betaTensor.tensor_content().empty()) + { + layerParams.blobs.resize(layerParams.blobs.size() + 1); + layerParams.set("has_bias", true); + blobFromTensor(betaTensor, layerParams.blobs.back()); + } + else + layerParams.set("has_bias", false); + + Mat mean, std; + if (isTraining) + { + if (layerParams.blobs.size() == 2) + CV_Error(Error::StsNotImplemented, "Cannot determine number " + "of parameters for batch normalization layer."); + mean = Mat::zeros(1, layerParams.blobs[2].total(), CV_32F); + std = Mat::ones(1, layerParams.blobs[2].total(), CV_32F); + + // Add an extra layer: Mean-Variance normalization + LayerParams mvnParams; + std::string mvnName = name + "/MVN"; + CV_Assert(layer_id.find(mvnName) == layer_id.end()); + int mvnId = dstNet.addLayer(mvnName, "MVN", mvnParams); + layer_id[mvnName] = mvnId; + connect(layer_id, dstNet, inpId, mvnId, 0); + inpId = Pin(mvnName); + } + else + { + blobFromTensor(getConstBlob(layer, value_id, 3), mean); + blobFromTensor(getConstBlob(layer, value_id, 4), std); + } + layerParams.blobs[0] = mean; + layerParams.blobs[1] = std; + + if (hasLayerAttr(layer, "epsilon")) + layerParams.set("eps", getLayerAttr(layer, "epsilon").f()); + + int id = dstNet.addLayer(name, "BatchNorm", layerParams); + layer_id[name] = id; + + // one input only + connect(layer_id, dstNet, inpId, id, 0); +} + +void TFImporter::parseConv2DBackpropInput(tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams) +{ + // op: "Conv2DBackpropInput" + // input: "conv2d_transpose/output_shape" + // input: "weights" + // input: "input" + + const std::string& name = layer.name(); + const int num_inputs = layer.input_size(); + + CV_CheckEQ(num_inputs, 3, "Expected output shape, weights and input nodes"); + + layerParams.set("bias_term", false); + layerParams.blobs.resize(1); + + StrIntVector next_layers = getNextLayers(net, name, "BiasAdd"); + if (next_layers.size() == 1) + { + layerParams.set("bias_term", true); + layerParams.blobs.resize(2); + + int weights_layer_index = next_layers[0].second; + + blobFromTensor(getConstBlob(net.node(weights_layer_index), value_id), layerParams.blobs[1]); + ExcludeLayer(net, weights_layer_index, 0, false); + layers_to_ignore.insert(next_layers[0].first); + } + + kernelFromTensor(getConstBlob(layer, value_id, 1), layerParams.blobs[0]); + + const int* kshape = layerParams.blobs[0].size.p; + const int kernelH = kshape[2]; + const int kernelW = kshape[3]; + layerParams.set("kernel_h", kernelH); + layerParams.set("kernel_w", kernelW); + layerParams.set("num_output", kshape[1]); + + setStrides(layerParams, layer); + setPadding(layerParams, layer); + + // For convolution layer, output shape computes as + // o = 1 + (i - k + 2*p) / s + // i - input size, o - output size, k - kernel size, p - pad, s - stride + // In TensorFlow, p == 0 is padMode == 'VALID' or p == (k - 1) / 2 + // considering that k is odd. + // SAME: o = 1 + (i - 1) / s + // VALID: o = 1 + i / s + // Deconvolution's layer output shape computes as + // SAME: o = 1 + (i - 1)*s + // VALID: o = (i - 1)*s + // If output_shape differs from formulas above then adjust padding is applied. + + const int strideY = layerParams.get("stride_h"); + const int strideX = layerParams.get("stride_w"); + Mat outShape = getTensorContent(getConstBlob(layer, value_id, 0)); + const int outH = outShape.at(1); + const int outW = outShape.at(2); + if (layerParams.get("pad_mode") == "SAME") + { + layerParams.set("adj_w", (outW - 1) % strideX); + layerParams.set("adj_h", (outH - 1) % strideY); + } + else if (layerParams.get("pad_mode") == "VALID") + { + layerParams.set("adj_w", (outW - kernelW) % strideX); + layerParams.set("adj_h", (outH - kernelH) % strideY); + } + int id = dstNet.addLayer(name, "Deconvolution", layerParams); + layer_id[name] = id; + + // one input only + connect(layer_id, dstNet, parsePin(layer.input(2)), id, 0); +} + +void TFImporter::parseBlockLSTM(tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams) +{ + // op: "BlockLSTM" + // input: "lstm_block_wrapper/ToInt64/x" (ignore, number of time stamps) + // input: "input" + // input: "lstm_block_wrapper/zeros" (ignore) + // input: "lstm_block_wrapper/zeros" (ignore) + // input: "lstm_block_wrapper/kernel" + // input: "lstm_block_wrapper/w_i_diag" + // input: "lstm_block_wrapper/w_f_diag" + // input: "lstm_block_wrapper/w_o_diag" + // input: "lstm_block_wrapper/bias" + + const std::string& name = layer.name(); + const int num_inputs = layer.input_size(); + + CV_CheckEQ(num_inputs, 9, "Unexpected number of input nodes"); + + if (hasLayerAttr(layer, "forget_bias")) + layerParams.set("forget_bias", getLayerAttr(layer, "forget_bias").f()); + + if (hasLayerAttr(layer, "forget_bias")) + { + float cellClip = getLayerAttr(layer, "cell_clip").f(); + // Cell clip disabled if it's negative. + if (cellClip >= 0) + { + layerParams.set("use_cell_clip", true); + layerParams.set("cell_clip", cellClip); + } + } + + Mat W, Wh, Wx, b; + blobFromTensor(getConstBlob(layer, value_id, 4), W); + blobFromTensor(getConstBlob(layer, value_id, 8), b); + const int outSize = W.cols / 4; + + // IGFO->IFOG + float* weightData = (float*)W.data; + for (int i = 0; i < W.rows; ++i) + for (int j = 0; j < outSize; ++j) + { + std::swap(weightData[i * W.cols + 1 * outSize + j], + weightData[i * W.cols + 2 * outSize + j]); + std::swap(weightData[i * W.cols + 2 * outSize + j], + weightData[i * W.cols + 3 * outSize + j]); + } + Wx = W.rowRange(0, W.rows - outSize).t(); + Wh = W.rowRange(W.rows - outSize, W.rows).t(); + + layerParams.blobs.resize(3); + layerParams.blobs[0] = Wh; + layerParams.blobs[1] = Wx; + layerParams.blobs[2] = b; + + if (hasLayerAttr(layer, "use_peephole")) + { + bool usePeephole = getLayerAttr(layer, "use_peephole").b(); + if (usePeephole) + { + layerParams.set("use_peephole", true); + layerParams.blobs.resize(6); + for (int i = 0; i < 3; ++i) + { + Mat w; + blobFromTensor(getConstBlob(layer, value_id, 5 + i), w); + w = w.reshape(1, w.total()); // Single column. + w = Mat::diag(w); // Make a diagonal matrix. + layerParams.blobs[3 + i] = w; + } + } + } + + int id = dstNet.addLayer(name, "LSTM", layerParams); + layer_id[name] = id; + + // one input only + connect(layer_id, dstNet, parsePin(layer.input(1)), id, 0); + data_layouts[name] = DATA_LAYOUT_UNKNOWN; +} + +void TFImporter::parseResize(tensorflow::GraphDef& net, const tensorflow::NodeDef& layer_, LayerParams& layerParams) +{ + tensorflow::NodeDef layer = layer_; + std::string name = layer.name(); + const std::string& type = layer.op(); + int num_inputs = layer.input_size(); + + CV_CheckGT(num_inputs, 0, ""); + std::string convWeights = ""; + if (type == "FusedResizeAndPadConv2D") + { + // input: "mul_1" + // input: "decoder/ResizeBilinear/size" + // input: "decoder/decoder_conv0/Conv2D_dummy_paddings" + // input: "decoder/decoder_conv0/weights" + CV_CheckEQ(num_inputs, 4, "Number of input for FusedResizeAndPadConv2D"); + + Mat paddings = getTensorContent(getConstBlob(layer, value_id, 2)); + CV_CheckEQ(countNonZero(paddings), 0, "Unsupported mode"); + + convWeights = layer.input(3); + layer.mutable_input()->DeleteSubrange(2, 2); // FIXIT do NOT modify input model + num_inputs = layer.input_size(); + name = name + "/resize"; + + if (hasLayerAttr(layer, "resize_align_corners")) + { + // FIXIT do NOT modify input model + layer.mutable_attr()->insert( + ::google::protobuf::MapPair("align_corners", + getLayerAttr(layer, "resize_align_corners"))); + } + } + if (num_inputs == 2) + { + Mat outSize = getTensorContent(getConstBlob(layer, value_id, 1)); + CV_CheckTypeEQ(outSize.type(), CV_32SC1, ""); CV_CheckEQ(outSize.total(), (size_t)2, ""); + layerParams.set("height", outSize.at(0, 0)); + layerParams.set("width", outSize.at(0, 1)); + } + else if (num_inputs == 3) + { + Mat factorHeight = getTensorContent(getConstBlob(layer, value_id, 1)); + Mat factorWidth = getTensorContent(getConstBlob(layer, value_id, 2)); + factorHeight.convertTo(factorHeight, CV_32F); + factorWidth.convertTo(factorWidth, CV_32F); + layerParams.set("zoom_factor_x", factorWidth.at(0)); + layerParams.set("zoom_factor_y", factorHeight.at(0)); + } + else + CV_Check(num_inputs, num_inputs == 2 || num_inputs == 3, ""); + + if (type == "ResizeNearestNeighbor") + layerParams.set("interpolation", "nearest"); + else + layerParams.set("interpolation", "bilinear"); + + if (hasLayerAttr(layer, "align_corners")) + layerParams.set("align_corners", getLayerAttr(layer, "align_corners").b()); + + if (hasLayerAttr(layer, "half_pixel_centers")) + layerParams.set("half_pixel_centers", getLayerAttr(layer, "half_pixel_centers").b()); + + int id = dstNet.addLayer(name, "Resize", layerParams); + layer_id[name] = id; + + connect(layer_id, dstNet, parsePin(layer.input(0)), id, 0); + + // Step back to add convolution + if (type == "FusedResizeAndPadConv2D") + { + tensorflow::NodeDef conv = layer_; + conv.clear_input(); + conv.add_input(name); + conv.add_input(convWeights); + conv.set_op("Conv2D"); + parseNode(conv); + } +} + +void TFImporter::parseL2Normalize(tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams) +{ + // op: "L2Normalize" + // input: "input" + // input: "reduction_indices" (axis) + + const std::string& name = layer.name(); + const int num_inputs = layer.input_size(); + + CV_CheckEQ(num_inputs, 2, ""); + Mat reductionIndices = getTensorContent(getConstBlob(layer, value_id, 1)); + CV_Assert(reductionIndices.type() == CV_32SC1); + + const int numAxes = reductionIndices.total(); + if (getDataLayout(name, data_layouts) == DATA_LAYOUT_NHWC) + for (int i = 0; i < numAxes; ++i) + reductionIndices.at(i) = toNCHW(reductionIndices.at(i)); + + cv::sort(reductionIndices, reductionIndices, SORT_ASCENDING); + for (int i = 1; i < numAxes; ++i) + { + CV_Assert(reductionIndices.at(i) == reductionIndices.at(i - 1) + 1); + // Axes have the same sign. + CV_Assert(reductionIndices.at(i) * reductionIndices.at(i - 1) >= 0); + } + layerParams.set("start_axis", reductionIndices.at(0)); + layerParams.set("end_axis", reductionIndices.at(numAxes - 1)); + + int id = dstNet.addLayer(name, "Normalize", layerParams); + layer_id[name] = id; + connect(layer_id, dstNet, parsePin(layer.input(0)), id, 0); +} + +void TFImporter::parsePriorBox(tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams) +{ + const std::string& name = layer.name(); + const int num_inputs = layer.input_size(); + + CV_CheckEQ(num_inputs, 2, ""); + if (hasLayerAttr(layer, "min_size")) + layerParams.set("min_size", getLayerAttr(layer, "min_size").i()); + if (hasLayerAttr(layer, "max_size")) + layerParams.set("max_size", getLayerAttr(layer, "max_size").i()); + if (hasLayerAttr(layer, "flip")) + layerParams.set("flip", getLayerAttr(layer, "flip").b()); + if (hasLayerAttr(layer, "clip")) + layerParams.set("clip", getLayerAttr(layer, "clip").b()); + if (hasLayerAttr(layer, "offset")) + layerParams.set("offset", getLayerAttr(layer, "offset").f()); + if (hasLayerAttr(layer, "step")) + layerParams.set("step", getLayerAttr(layer, "step").f()); + + const std::string paramNames[] = {"variance", "aspect_ratio", "scales", + "width", "height"}; + for (int i = 0; i < 5; ++i) + { + if (hasLayerAttr(layer, paramNames[i])) + { + Mat values = getTensorContent(getLayerAttr(layer, paramNames[i]).tensor()); + layerParams.set(paramNames[i], + DictValue::arrayReal((float*)values.data, values.total())); + } + } + int id = dstNet.addLayer(name, "PriorBox", layerParams); + layer_id[name] = id; + connect(layer_id, dstNet, parsePin(layer.input(0)), id, 0); + connect(layer_id, dstNet, parsePin(layer.input(1)), id, 1); + data_layouts[name] = DATA_LAYOUT_UNKNOWN; +} + +void TFImporter::parseSoftmax(tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams) +{ + const std::string& name = layer.name(); + const int num_inputs = layer.input_size(); + + CV_CheckGT(num_inputs, 0, ""); + if (hasLayerAttr(layer, "axis")) + layerParams.set("axis", getLayerAttr(layer, "axis").i()); + + int id = dstNet.addLayer(name, "Softmax", layerParams); + layer_id[name] = id; + connectToAllBlobs(layer_id, dstNet, parsePin(layer.input(0)), id, num_inputs); +} + +void TFImporter::parseCropAndResize(tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams) +{ + // op: "CropAndResize" + // input: "input" + // input: "boxes" + // input: "sizes" + + const std::string& name = layer.name(); + const int num_inputs = layer.input_size(); + CV_CheckEQ(num_inputs, 3, ""); + + Mat cropSize = getTensorContent(getConstBlob(layer, value_id, 2)); + CV_CheckTypeEQ(cropSize.type(), CV_32SC1, ""); CV_CheckEQ(cropSize.total(), (size_t)2, ""); + + layerParams.set("height", cropSize.at(0)); + layerParams.set("width", cropSize.at(1)); + + int id = dstNet.addLayer(name, "CropAndResize", layerParams); + layer_id[name] = id; + + connect(layer_id, dstNet, parsePin(layer.input(0)), id, 0); + connect(layer_id, dstNet, parsePin(layer.input(1)), id, 1); +} + +void TFImporter::parseMean(tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams) +{ + // Computes the mean of elements across dimensions of a tensor. + // If keepdims is false (default) reduces input_tensor along the dimensions given in axis, + // else the reduced dimensions are retained with length 1. + // if indices = [1, 2] in NHWC layout we use global pooling: NxCxHxW --Pooling--> NxCx1x1 + // if keepdims is false we use Flatten after Pooling: out_shape = NxC + // if indices = [0] we use a global pooling by indices. + // To return correct shape, we use Reshape after Pooling. To determine input shape use Slice for input, + // if keepdims is false we use Flatten after Slice. + // Example: input_shape = NxCxHxW + // determine out shape: NxCxHxW --Slice--> 1xCxHxW + // out_shape = 1xCxHxW if keepDims else (1xCxHxW --Flatten--> CxHxW) + // global pool: NxCxHxW --Flatten--> Nx(C*H*W) --Reshape--> 1x1xNx(C*H*W) --Pooling--> 1x1x1x(C*H*W) --Reshape--> out_shape + + const std::string& name = layer.name(); + const std::string& type = layer.op(); + const int num_inputs = layer.input_size(); + + CV_CheckGT(num_inputs, 0, ""); + + Mat indices = getTensorContent(getConstBlob(layer, value_id, 1)); + CV_Assert(indices.type() == CV_32SC1); + + // There are two attributes, "keepdims" and a deprecated "keep_dims". + bool keepDims = false; + if (hasLayerAttr(layer, "keepdims")) + keepDims = getLayerAttr(layer, "keepdims").b(); + else if (hasLayerAttr(layer, "keep_dims")) + keepDims = getLayerAttr(layer, "keep_dims").b(); + + if (indices.total() == 1 && indices.at(0) == 0) + { + LayerParams flattenLp; + std::string flattenName = name + "/flatten"; + CV_Assert(layer_id.find(flattenName) == layer_id.end()); + int flattenId = dstNet.addLayer(flattenName, "Flatten", flattenLp); + layer_id[flattenName] = flattenId; + connect(layer_id, dstNet, parsePin(layer.input(0)), flattenId, 0); + + LayerParams reshapeLp; + std::string reshapeName = name + "/reshape"; + CV_Assert(layer_id.find(reshapeName) == layer_id.end()); + reshapeLp.set("axis", 0); + reshapeLp.set("num_axes", 1); + int newShape[] = {1, 1, -1}; + reshapeLp.set("dim", DictValue::arrayInt(&newShape[0], 3)); + + int reshapeId = dstNet.addLayer(reshapeName, "Reshape", reshapeLp); + layer_id[reshapeName] = reshapeId; + connect(layer_id, dstNet, Pin(flattenName), reshapeId, 0); + + LayerParams avgLp; + std::string avgName = name + "/avg"; + CV_Assert(layer_id.find(avgName) == layer_id.end()); + avgLp.set("pool", type == "Mean" ? "ave" : "sum"); + // pooling kernel H x 1 + avgLp.set("global_pooling_h", true); + avgLp.set("kernel_w", 1); + int avgId = dstNet.addLayer(avgName, "Pooling", avgLp); + layer_id[avgName] = avgId; + connect(layer_id, dstNet, Pin(reshapeName), avgId, 0); + + LayerParams sliceLp; + std::string layerShapeName = name + "/slice"; + CV_Assert(layer_id.find(layerShapeName) == layer_id.end()); + sliceLp.set("axis", 0); + int begin[] = {0}; + int size[] = {1}; + sliceLp.set("begin", DictValue::arrayInt(&begin[0], 1)); + sliceLp.set("size", DictValue::arrayInt(&size[0], 1)); + int sliceId = dstNet.addLayer(layerShapeName, "Slice", sliceLp); + layer_id[layerShapeName] = sliceId; + connect(layer_id, dstNet, Pin(layer.input(0)), sliceId, 0); + + if (!keepDims) + { + LayerParams squeezeLp; + std::string squeezeName = name + "/squeeze"; + CV_Assert(layer_id.find(squeezeName) == layer_id.end()); + squeezeLp.set("axis", 0); + squeezeLp.set("end_axis", 1); + int squeezeId = dstNet.addLayer(squeezeName, "Flatten", squeezeLp); + layer_id[squeezeName] = squeezeId; + connect(layer_id, dstNet, Pin(layerShapeName), squeezeId, 0); + layerShapeName = squeezeName; + } + + int id = dstNet.addLayer(name, "Reshape", layerParams); + layer_id[name] = id; + connect(layer_id, dstNet, Pin(avgName), id, 0); + connect(layer_id, dstNet, Pin(layerShapeName), id, 1); + } else if (indices.total() == 1) { + int axis = toNCHW(indices.at(0)); + if (axis == 2 || axis == 3) + { + layerParams.set("pool", type == "Mean" ? "ave" : "sum"); + layerParams.set(axis == 2 ? "kernel_w" : "kernel_h", 1); + layerParams.set(axis == 2 ? "global_pooling_h" : "global_pooling_w", true); + int id = dstNet.addLayer(name, "Pooling", layerParams); + layer_id[name] = id; + connect(layer_id, dstNet, parsePin(layer.input(0)), id, 0); + + if (!keepDims) + { + // To keep correct order after squeeze dims we first need to change layout from NCHW to NHWC + LayerParams permLP; + int order[] = {0, 2, 3, 1}; // From OpenCV's NCHW to NHWC. + std::string permName = name + "/nchw"; + Pin inpId = Pin(name); + addPermuteLayer(order, permName, inpId); + + LayerParams squeezeLp; + std::string squeezeName = name + "/squeeze"; + CV_Assert(layer_id.find(squeezeName) == layer_id.end()); + squeezeLp.set("axis", indices.at(0)); + squeezeLp.set("end_axis", indices.at(0) + 1); + int squeezeId = dstNet.addLayer(squeezeName, "Flatten", squeezeLp); + layer_id[squeezeName] = squeezeId; + connect(layer_id, dstNet, Pin(permName), squeezeId, 0); + } + } + else if (axis == 1) + { + int order[] = {0, 2, 3, 1}; // From OpenCV's NCHW to NHWC. + Pin inpId = parsePin(layer.input(0)); + addPermuteLayer(order, name + "/nhwc", inpId); + + layerParams.set("pool", type == "Mean" ? "ave" : "sum"); + layerParams.set("kernel_h", 1); + layerParams.set("global_pooling_w", true); + int id = dstNet.addLayer(name, "Pooling", layerParams); + layer_id[name] = id; + connect(layer_id, dstNet, inpId, id, 0); + + if (!keepDims) + { + LayerParams squeezeLp; + std::string squeezeName = name + "/squeeze"; + CV_Assert(layer_id.find(squeezeName) == layer_id.end()); + int channel_id = 3; // TF NHWC layout + squeezeLp.set("axis", channel_id - 1); + squeezeLp.set("end_axis", channel_id); + int squeezeId = dstNet.addLayer(squeezeName, "Flatten", squeezeLp); + layer_id[squeezeName] = squeezeId; + connect(layer_id, dstNet, Pin(name), squeezeId, 0); + } + else + { + int order[] = {0, 3, 1, 2}; // From NHWC to OpenCV's NCHW. + Pin inpId = parsePin(name); + addPermuteLayer(order, name + "/nchw", inpId); + } + } + } else { + if (indices.total() != 2 || indices.at(0) != 1 || indices.at(1) != 2) + CV_Error(Error::StsNotImplemented, "Unsupported mode of reduce_mean or reduce_sum operation."); + + layerParams.set("pool", type == "Mean" ? "ave" : "sum"); + layerParams.set("global_pooling", true); + int id = dstNet.addLayer(name, "Pooling", layerParams); + layer_id[name] = id; + connect(layer_id, dstNet, parsePin(layer.input(0)), id, 0); + + if (!keepDims) + { + LayerParams flattenLp; + std::string flattenName = name + "/flatten"; + CV_Assert(layer_id.find(flattenName) == layer_id.end()); + int flattenId = dstNet.addLayer(flattenName, "Flatten", flattenLp); + layer_id[flattenName] = flattenId; + connect(layer_id, dstNet, Pin(name), flattenId, 0); + } + } +} + +void TFImporter::parsePack(tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams) +{ + // op: tf.stack(list of tensors, axis=0) + // Join a list of inputs along a new axis. + // The "axis" specifies the index of the new axis in the dimensions of the output. + // Example: given a list with "N" tensors of shape (C, H, W): + // if axis == 0 then the output tensor will have the shape (N, C, H, W), + // if axis == 1 then the output tensor will have the shape (C, N, H, W). + + const std::string& name = layer.name(); + const int num_inputs = layer.input_size(); + + CV_CheckGT(num_inputs, 0, ""); + CV_Assert(hasLayerAttr(layer, "axis")); + int dim = (int)getLayerAttr(layer, "axis").i(); + if (dim != 0) + CV_Error(Error::StsNotImplemented, "Unsupported mode of pack operation."); + + CV_Assert(hasLayerAttr(layer, "N")); + int num = (int)getLayerAttr(layer, "N").i(); + CV_CheckEQ(num_inputs, num, ""); + std::string base_name = name + "/reshape_"; + std::vector reshape_ids; + for (int i = 0; i < num; i++) { + std::ostringstream ss; + ss << i; + std::string reshape_name = base_name + ss.str(); + LayerParams reshapeLP; + reshapeLP.set("axis", dim); + reshapeLP.set("num_axes", 1); + int outShape[] = {1, -1}; + reshapeLP.set("dim", DictValue::arrayInt(&outShape[0], 2)); + int id = dstNet.addLayer(reshape_name, "Reshape", reshapeLP); + layer_id[reshape_name] = id; + reshape_ids.push_back(id); + connect(layer_id, dstNet, parsePin(layer.input(i)), id, 0); + } + + layerParams.set("axis", dim); + int id = dstNet.addLayer(name, "Concat", layerParams); + layer_id[name] = id; + + for (int li = 0; li < num; li++) + dstNet.connect(reshape_ids[li], 0, id, li); +} + +void TFImporter::parseClipByValue(tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams) +{ + // op: "ClipByValue" + // input: "input" + // input: "mix" + // input: "max" + + const std::string& name = layer.name(); + const int num_inputs = layer.input_size(); + + CV_CheckEQ(num_inputs, 3, ""); + + Mat minValue = getTensorContent(getConstBlob(layer, value_id, 1)); + Mat maxValue = getTensorContent(getConstBlob(layer, value_id, 2)); + CV_CheckEQ(minValue.total(), (size_t)1, ""); CV_CheckTypeEQ(minValue.type(), CV_32FC1, ""); + CV_CheckEQ(maxValue.total(), (size_t)1, ""); CV_CheckTypeEQ(maxValue.type(), CV_32FC1, ""); + + layerParams.set("min_value", minValue.at(0)); + layerParams.set("max_value", maxValue.at(0)); + + int id = dstNet.addLayer(name, "ReLU6", layerParams); + layer_id[name] = id; + + connect(layer_id, dstNet, parsePin(layer.input(0)), id, 0); +} + +void TFImporter::parseLeakyRelu(tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams) +{ + const std::string& name = layer.name(); + const int num_inputs = layer.input_size(); + + CV_CheckGT(num_inputs, 0, ""); + CV_Assert(hasLayerAttr(layer, "alpha")); + layerParams.set("negative_slope", getLayerAttr(layer, "alpha").f()); + + int id = dstNet.addLayer(name, "ReLU", layerParams); + layer_id[name] = id; + connectToAllBlobs(layer_id, dstNet, parsePin(layer.input(0)), id, num_inputs); +} + +void TFImporter::parseActivation(tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams) +{ + const std::string& name = layer.name(); + const std::string& type = layer.op(); + const int num_inputs = layer.input_size(); + + CV_CheckGT(num_inputs, 0, ""); + std::string dnnType = type; + if (type == "Abs") dnnType = "AbsVal"; + else if (type == "Tanh") dnnType = "TanH"; + else if (type == "Relu") dnnType = "ReLU"; + else if (type == "Relu6") dnnType = "ReLU6"; + else if (type == "Elu") dnnType = "ELU"; + + int id = dstNet.addLayer(name, dnnType, layerParams); + layer_id[name] = id; + connectToAllBlobs(layer_id, dstNet, parsePin(layer.input(0)), id, num_inputs); +} + +void TFImporter::parseCustomLayer(tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams) +{ + // Importer does not know how to map this TensorFlow's operation onto OpenCV's layer. + // However we create a layer with the same type and rely that user defined a custom layer. + + const std::string& name = layer.name(); + const std::string& type = layer.op(); + const int num_inputs = layer.input_size(); + + // All the attributes are added to LayerParams. + google::protobuf::Map attr = layer.attr(); + for (google::protobuf::Map::const_iterator ai = attr.begin(); + ai != attr.end(); ++ai) + { + if (ai->second.value_case() == tensorflow::AttrValue::kS) // string + layerParams.set(ai->first, ai->second.s()); + if (ai->second.value_case() == tensorflow::AttrValue::kI) // int64 + layerParams.set(ai->first, ai->second.i()); + if (ai->second.value_case() == tensorflow::AttrValue::kF) // float + layerParams.set(ai->first, ai->second.f()); + if (ai->second.value_case() == tensorflow::AttrValue::kB) // bool + layerParams.set(ai->first, ai->second.b()); + } + + // All the Const input nodes are added to layer's blobs. + std::vector inputsNames; + for (int i = 0; i < num_inputs; ++i) + { + // Check if input is a Const node. + if (value_id.find(layer.input(i)) != value_id.end()) + { + Mat blob = getTensorContent(getConstBlob(layer, value_id, i)); + layerParams.blobs.push_back(blob); + } + else + inputsNames.push_back(layer.input(i)); + } + int id = dstNet.addLayer(name, type, layerParams); + layer_id[name] = id; + + for (int i = 0; i < inputsNames.size(); ++i) + { + connect(layer_id, dstNet, parsePin(inputsNames[i]), id, i); + } +} + TFImporter::TFImporter(Net& net, const char *model, const char *config) - : dstNet(net) + : dstNet(net), dispatch(buildDispatchMap()) { if (model && model[0]) { @@ -534,7 +2358,7 @@ TFImporter::TFImporter( const char *dataModel, size_t lenModel, const char *dataConfig, size_t lenConfig ) - : dstNet(net) + : dstNet(net), dispatch(buildDispatchMap()) { if (dataModel != NULL && lenModel > 0) { @@ -926,15 +2750,12 @@ void TFImporter::addPermuteLayer(const int* order, const std::string& permName, inpId = Pin(permName); } -void TFImporter::parseNode(const tensorflow::NodeDef& layer_) +void TFImporter::parseNode(const tensorflow::NodeDef& layer) { - tensorflow::NodeDef layer = layer_; - tensorflow::GraphDef& net = netTxt.ByteSize() != 0 ? netTxt : netBin; - /*const*/ std::string name = layer.name(); - /*const*/ std::string type = layer.op(); - /*const*/ int num_inputs = layer.input_size(); + const std::string& name = layer.name(); + const std::string& type = layer.op(); try { @@ -949,1612 +2770,14 @@ void TFImporter::parseNode(const tensorflow::NodeDef& layer_) DataLayout predictedLayout = predictOutputDataLayout(layer); data_layouts[name] = predictedLayout; - if (type == "Conv2D" || type == "SpaceToBatchND" || type == "DepthwiseConv2dNative" || type == "Pad" || type == "MirrorPad" || type == "Conv3D") + DispatchMap::const_iterator iter = dispatch.find(type); + if (iter != dispatch.end()) { - CV_CheckGT(num_inputs, 0, ""); - // The first node of dilated convolution subgraph. - // Extract input node, dilation rate and paddings. - std::string input = layer.input(0); - StrIntVector next_layers; - if (type == "SpaceToBatchND" || type == "Pad") - { - next_layers = getNextLayers(net, name, "Conv2D"); - if (next_layers.empty()) - next_layers = getNextLayers(net, name, "DepthwiseConv2dNative"); - } - - if (type == "SpaceToBatchND") - { - // op: "SpaceToBatchND" - // input: "input" - // input: "SpaceToBatchND/block_shape" - // input: "SpaceToBatchND/paddings" - CV_CheckEQ(num_inputs, 3, ""); - - DictValue dilation = parseDims(getConstBlob(layer, value_id, 1)); - CV_Assert(dilation.size() == 2); - layerParams.set("dilation_h", dilation.get(0)); - layerParams.set("dilation_w", dilation.get(1)); - - Mat paddings; - parseTensor(getConstBlob(layer, value_id, 2), paddings); - - // paddings is a 2x2 matrix: [[top, bot], [left, right]] - layerParams.set("pad_h", paddings.at(0)); - layerParams.set("pad_w", paddings.at(2)); - - CV_Assert(next_layers.size() == 1); - layers_to_ignore.insert(next_layers[0].first); - - // FIXIT don't override, rewrite this code - layer = net.node(next_layers[0].second); - name = layer.name(); - type = layer.op(); - num_inputs = layer.input_size(); - CV_LOG_DEBUG(NULL, "DNN/TF: switched to layer " << name << " @ " << type << ") with " << num_inputs << " inputs"); - } - else if (type == "Pad" || type == "MirrorPad") - { - Mat paddings = getTensorContent(getConstBlob(layer, value_id, 1)); - CV_Assert(paddings.type() == CV_32SC1); - if (paddings.total() == 8) - { - // Perhaps, we have NHWC padding dimensions order. - // N H W C - // 0 1 2 3 4 5 6 7 - std::swap(paddings.at(2), paddings.at(6)); - std::swap(paddings.at(3), paddings.at(7)); - // N C W H - // 0 1 2 3 4 5 6 7 - std::swap(paddings.at(4), paddings.at(6)); - std::swap(paddings.at(5), paddings.at(7)); - // N C H W - // 0 1 2 3 4 5 6 7 - } - - if (next_layers.empty() || paddings.total() != 8 || - paddings.at(4) != paddings.at(5) || - paddings.at(6) != paddings.at(7) || type == "MirrorPad") - { - // Just a single padding layer. - layerParams.set("paddings", DictValue::arrayInt((int*)paddings.data, paddings.total())); - if (type == "MirrorPad") - layerParams.set("type", "reflect"); - - int id = dstNet.addLayer(name, "Padding", layerParams); - layer_id[name] = id; - - connect(layer_id, dstNet, parsePin(input), id, 0); - return; - } - else - { - // Merge with subsequent convolutional layer. - CV_Assert(next_layers.size() == 1); - - layerParams.set("pad_h", paddings.at(4)); - layerParams.set("pad_w", paddings.at(6)); - - layers_to_ignore.insert(next_layers[0].first); - - // FIXIT don't override, rewrite this code - layer = net.node(next_layers[0].second); - name = layer.name(); - type = layer.op(); - num_inputs = layer.input_size(); - CV_LOG_DEBUG(NULL, "DNN/TF: switched to layer " << name << " @ " << type << ") with " << num_inputs << " inputs"); - } - } - - // For the object detection networks, TensorFlow Object Detection API - // predicts deltas for bounding boxes in yxYX (ymin, xmin, ymax, xmax) - // order. We can manage it at DetectionOutput layer parsing predictions - // or shuffle last convolution's weights. - bool locPredTransposed = hasLayerAttr(layer, "loc_pred_transposed") && - getLayerAttr(layer, "loc_pred_transposed").b(); - - layerParams.set("bias_term", false); - layerParams.blobs.resize(1); - - next_layers = getNextLayers(net, name, "BiasAdd"); - if (next_layers.size() == 1) { - layerParams.set("bias_term", true); - layerParams.blobs.resize(2); - - int weights_layer_index = next_layers[0].second; - - blobFromTensor(getConstBlob(net.node(weights_layer_index), value_id), layerParams.blobs[1]); - ExcludeLayer(net, weights_layer_index, 0, false); - layers_to_ignore.insert(next_layers[0].first); - - // Shuffle bias from yxYX to xyXY. - if (locPredTransposed) - { - const int numWeights = layerParams.blobs[1].total(); - float* biasData = reinterpret_cast(layerParams.blobs[1].data); - CV_Assert(numWeights % 4 == 0); - for (int i = 0; i < numWeights; i += 2) - { - std::swap(biasData[i], biasData[i + 1]); - } - } - } - - int kernelTensorInpId = -1; - const tensorflow::TensorProto& kernelTensor = getConstBlob(layer, value_id, -1, &kernelTensorInpId); - const String kernelTensorName = layer.input(kernelTensorInpId); - std::map::iterator sharedWeightsIt = sharedWeights.find(kernelTensorName); - if (sharedWeightsIt == sharedWeights.end()) - { - kernelFromTensor(kernelTensor, layerParams.blobs[0]); - releaseTensor(const_cast(&kernelTensor)); - - int* kshape = layerParams.blobs[0].size.p; - const int outCh = kshape[0]; - const int inCh = kshape[1]; - const int height = kshape[2]; - const int width = kshape[3]; - if (type == "DepthwiseConv2dNative") - { - CV_Assert(!locPredTransposed); - const int chMultiplier = kshape[0]; - - Mat copy = layerParams.blobs[0].clone(); - float* src = (float*)copy.data; - float* dst = (float*)layerParams.blobs[0].data; - for (int i = 0; i < chMultiplier; ++i) - for (int j = 0; j < inCh; ++j) - for (int s = 0; s < height * width; ++s) - { - int src_i = (i * inCh + j) * height * width + s; - int dst_i = (j * chMultiplier + i) * height* width + s; - dst[dst_i] = src[src_i]; - } - // TODO Use reshape instead - kshape[0] = inCh * chMultiplier; - kshape[1] = 1; - size_t* kstep = layerParams.blobs[0].step.p; - kstep[0] = kstep[1]; // fix steps too - } - - // Shuffle output channels from yxYX to xyXY. - if (locPredTransposed) - { - const int slice = height * width * inCh; - for (int i = 0; i < outCh; i += 2) - { - cv::Mat src(1, slice, CV_32F, layerParams.blobs[0].ptr(i)); - cv::Mat dst(1, slice, CV_32F, layerParams.blobs[0].ptr(i + 1)); - std::swap_ranges(src.begin(), src.end(), dst.begin()); - } - } - sharedWeights[kernelTensorName] = layerParams.blobs[0]; - } - else - { - layerParams.blobs[0] = sharedWeightsIt->second; - } - Mat weights = layerParams.blobs[0]; - layerParams.set("kernel_size", DictValue::arrayInt(&weights.size[2], weights.dims - 2)); - - layerParams.set("num_output", layerParams.blobs[0].size[0]); - - setStrides(layerParams, layer); - if (!layerParams.has("pad_w") && !layerParams.has("pad_h")) - setPadding(layerParams, layer); - - // The final node of dilated convolution subgraph. - next_layers = getNextLayers(net, name, "BatchToSpaceND"); - if (!next_layers.empty()) - { - CV_Assert(next_layers.size() == 1); - ExcludeLayer(net, next_layers[0].second, 0, false); - layers_to_ignore.insert(next_layers[0].first); - } - - int id = dstNet.addLayer(name, "Convolution", layerParams); - layer_id[name] = id; - - // one input only - connect(layer_id, dstNet, parsePin(input), id, 0); - - - if (getDataLayout(name, data_layouts) == DATA_LAYOUT_UNKNOWN) - data_layouts[name] = DATA_LAYOUT_NHWC; - } - else if (type == "BiasAdd" || type == "Add" || type == "AddV2" || type == "Sub" || type=="AddN") - { - CV_CheckGT(num_inputs, 0, ""); - bool haveConst = false; - for(int ii = 0; !haveConst && ii < num_inputs; ++ii) - { - Pin input = parsePin(layer.input(ii)); - haveConst = value_id.find(input.name) != value_id.end(); - } - CV_Assert(!haveConst || num_inputs == 2); - - if (haveConst) - { - Mat values = getTensorContent(getConstBlob(layer, value_id)); - CV_Assert(values.type() == CV_32FC1); - if (type == "Sub") - values *= -1.0f; - - int id; - if (values.total() == 1) // is a scalar. - { - layerParams.set("shift", values.at(0)); - id = dstNet.addLayer(name, "Power", layerParams); - } - else // is a vector - { - layerParams.blobs.resize(1, values); - id = dstNet.addLayer(name, "Shift", layerParams); - } - layer_id[name] = id; - - // one input only - connect(layer_id, dstNet, parsePin(layer.input(0)), id, 0); - } - else - { - layerParams.set("operation", "sum"); - if (type == "Sub") - { - static float subCoeffs[] = {1.f, -1.f}; - layerParams.set("coeff", DictValue::arrayReal(subCoeffs, 2)); - } - - int id = dstNet.addLayer(name, "Eltwise", layerParams); - layer_id[name] = id; - - for (int ii = 0; ii < num_inputs; ii++) - { - Pin inp = parsePin(layer.input(ii)); - if (layer_id.find(inp.name) == layer_id.end()) - CV_Error(Error::StsError, "Input layer not found: " + inp.name); - connect(layer_id, dstNet, inp, id, ii); - } - } - } - else if (type == "MatMul") - { - CV_CheckEQ(num_inputs, 2, ""); - - // For the object detection networks, TensorFlow Object Detection API - // predicts deltas for bounding boxes in yxYX (ymin, xmin, ymax, xmax) - // order. We can manage it at DetectionOutput layer parsing predictions - // or shuffle last Faster-RCNN's matmul weights. - bool locPredTransposed = hasLayerAttr(layer, "loc_pred_transposed") && - getLayerAttr(layer, "loc_pred_transposed").b(); - - layerParams.set("bias_term", false); - layerParams.blobs.resize(1); - - StrIntVector next_layers = getNextLayers(net, name, "BiasAdd"); // FIXIT Use layers fusion instead - if (next_layers.empty()) - { - next_layers = getNextLayers(net, name, "Add"); - } - if (next_layers.size() == 1) { - layerParams.set("bias_term", true); - layerParams.blobs.resize(2); - - int weights_layer_index = next_layers[0].second; - blobFromTensor(getConstBlob(net.node(weights_layer_index), value_id), layerParams.blobs[1]); - ExcludeLayer(net, weights_layer_index, 0, false); - layers_to_ignore.insert(next_layers[0].first); - - if (locPredTransposed) - { - const int numWeights = layerParams.blobs[1].total(); - float* biasData = reinterpret_cast(layerParams.blobs[1].data); - CV_Assert(numWeights % 4 == 0); - for (int i = 0; i < numWeights; i += 2) - { - std::swap(biasData[i], biasData[i + 1]); - } - } - } - - int kernel_blob_index = -1; - const tensorflow::TensorProto& kernelTensor = getConstBlob(layer, value_id, -1, &kernel_blob_index); - const String kernelTensorName = layer.input(kernel_blob_index); - std::map::iterator sharedWeightsIt = sharedWeights.find(kernelTensorName); - if (sharedWeightsIt == sharedWeights.end()) - { - blobFromTensor(kernelTensor, layerParams.blobs[0]); - releaseTensor(const_cast(&kernelTensor)); - sharedWeights[kernelTensorName] = layerParams.blobs[0]; - } - else - { - layerParams.blobs[0] = sharedWeightsIt->second; - } - - if (kernel_blob_index == 1) { // In this case output is computed by x*W formula - W should be transposed - Mat data = layerParams.blobs[0].t(); - layerParams.blobs[0] = data.clone(); - } - - layerParams.set("num_output", layerParams.blobs[0].size[0]); - if (locPredTransposed) - { - CV_Assert(layerParams.blobs[0].dims == 2); - for (int i = 0; i < layerParams.blobs[0].size[0]; i += 2) - { - cv::Mat src = layerParams.blobs[0].row(i); - cv::Mat dst = layerParams.blobs[0].row(i + 1); - std::swap_ranges(src.begin(), src.end(), dst.begin()); - } - } - - int id = dstNet.addLayer(name, "InnerProduct", layerParams); - layer_id[name] = id; - - // one input only - int input_blob_index = kernel_blob_index == 0 ? 1 : 0; - connect(layer_id, dstNet, parsePin(layer.input(input_blob_index)), id, 0); - data_layouts[name] = DATA_LAYOUT_PLANAR; - } - else if (type == "Reshape") - { - CV_CheckGT(num_inputs, 0, ""); - Pin inpId = parsePin(layer.input(0)); - DataLayout inpLayout = getDataLayout(layer.input(0), data_layouts); - // There are two possible implementations: reshape an input using - // predefined sizes or use a second input blob as a source of new shape. - if (value_id.find(layer.input(1)) != value_id.end()) - { - Mat newShape = getTensorContent(getConstBlob(layer, value_id, 1)); - int newShapeSize = newShape.total(); - bool hasSwap = false; - if (newShapeSize == 4 && hasAllOnes(newShape, 0, 2)) - { - // NHWC->NCHW - std::swap(*newShape.ptr(0, 2), *newShape.ptr(0, 3)); - std::swap(*newShape.ptr(0, 1), *newShape.ptr(0, 2)); - hasSwap = true; - } - if (inpLayout == DATA_LAYOUT_NHWC) - { - if (newShapeSize >= 2 || newShape.at(1) == 1) - { - int order[] = {0, 2, 3, 1}; // From OpenCV's NCHW to NHWC. - addPermuteLayer(order, name + "/nhwc", inpId); - if (newShapeSize < 4) - { - inpLayout = DATA_LAYOUT_NCHW; - } - else - { - inpLayout = DATA_LAYOUT_NHWC; - } - } - } - layerParams.set("dim", DictValue::arrayInt(newShape.ptr(), newShapeSize)); - - int id = dstNet.addLayer(name, "Reshape", layerParams); - layer_id[name] = id; - - // one input only - connect(layer_id, dstNet, inpId, id, 0); - inpId = Pin(name); - - if ((inpLayout == DATA_LAYOUT_NHWC || inpLayout == DATA_LAYOUT_UNKNOWN || inpLayout == DATA_LAYOUT_PLANAR) && - newShapeSize == 4 && !hasSwap) - { - int order[] = {0, 3, 1, 2}; // Transform back to OpenCV's NCHW. - addPermuteLayer(order, name + "/nchw", inpId); - inpLayout = DATA_LAYOUT_NCHW; - } - - data_layouts[name] = newShapeSize == 2 ? DATA_LAYOUT_PLANAR : inpLayout; - } - else - { - int id = dstNet.addLayer(name, "Reshape", layerParams); - layer_id[name] = id; - connect(layer_id, dstNet, inpId, id, 0); - connect(layer_id, dstNet, parsePin(layer.input(1)), id, 1); - data_layouts[name] = inpLayout; - } - } - else if (type == "Flatten" || type == "Squeeze") - { - CV_CheckGT(num_inputs, 0, ""); - Pin inpId = parsePin(layer.input(0)); - int inpLayout = getDataLayout(layer.input(0), data_layouts); - if (type == "Squeeze") - { - CV_Assert(hasLayerAttr(layer, "squeeze_dims")); - const tensorflow::AttrValue& dims = getLayerAttr(layer, "squeeze_dims"); - std::vector dimsVector(dims.list().i_size()); - for (int i = 0; i < dimsVector.size(); ++i) - dimsVector[i] = dims.list().i(i); - - // Flatten layer can squeeze dimensions range into one. - std::sort(dimsVector.begin(), dimsVector.end()); - for (int i = 1; i < dimsVector.size(); ++i) - { - if (dimsVector[i] != dimsVector[i - 1] + 1) - CV_Error(Error::StsNotImplemented, "Unsupported squeeze configuration"); - } - int start = dimsVector.front() - 1, end = dimsVector.back(); - if (start == -1 && end == 0) // squeeze 0th dimension - { - start = 0; - end = 1; - } - layerParams.set("axis", start); - layerParams.set("end_axis", end); - } - if (inpLayout == DATA_LAYOUT_NHWC) - { - LayerParams permLP; - int order[] = {0, 2, 3, 1}; // From OpenCV's NCHW to NHWC. - permLP.set("order", DictValue::arrayInt(order, 4)); - - std::string permName = name + "/nchw"; - CV_Assert(layer_id.find(permName) == layer_id.end()); - int permId = dstNet.addLayer(permName, "Permute", permLP); - layer_id[permName] = permId; - connect(layer_id, dstNet, inpId, permId, 0); - inpId = Pin(permName); - } - int id = dstNet.addLayer(name, "Flatten", layerParams); - layer_id[name] = id; - connect(layer_id, dstNet, inpId, id, 0); - data_layouts[name] = DATA_LAYOUT_PLANAR; - } - else if (type == "Transpose") - { - CV_CheckGT(num_inputs, 0, ""); - Mat perm = getTensorContent(getConstBlob(layer, value_id, 1)); - CV_Assert(perm.type() == CV_32SC1); - int* permData = (int*)perm.data; - if (perm.total() == 4) - { - // Only NHWC <-> NCHW permutations are allowed. OpenCV is always - // keep NCHW layout this way. - int inpLayout = getDataLayout(layer.input(0), data_layouts); - std::string type = "Identity"; - if (inpLayout == DATA_LAYOUT_NHWC) - { - if (permData[0] == 0 && permData[1] == 3 && permData[2] == 1 && permData[3] == 2) - { - // in TensorFlow: NHWC->NCHW - // in OpenCV: NCHW->NCHW - data_layouts[name] = DATA_LAYOUT_NCHW; - } - else if (permData[0] == 0 && permData[1] == 1 && permData[2] == 2 && permData[3] == 3) - { - // in TensorFlow: NHWC->NHWC - // in OpenCV: NCHW->NCHW - data_layouts[name] = DATA_LAYOUT_NHWC; - } - else if (permData[0] == 0 && permData[1] == 3 && permData[2] == 2 && permData[3] == 1) - { - // in TensorFlow: NHWC->NCWH - // in OpenCV: NCHW->NCWH - int permData[] = {0, 1, 3, 2}; - layerParams.set("order", DictValue::arrayInt(permData, perm.total())); - data_layouts[name] = DATA_LAYOUT_NCHW; // we keep track NCHW because channels position only matters - type = "Permute"; - } - else - CV_Error(Error::StsParseError, "Only NHWC <-> NCHW permutations are allowed."); - } - else if (inpLayout == DATA_LAYOUT_NCHW) - { - if (permData[0] == 0 && permData[1] == 2 && permData[2] == 3 && permData[3] == 1) - { - // in TensorFlow: NCHW->NHWC - // in OpenCV: NCHW->NCHW - data_layouts[name] = DATA_LAYOUT_NHWC; - } - else if (permData[0] == 0 && permData[1] == 1 && permData[2] == 2 && permData[3] == 3) - { - // in TensorFlow: NCHW->NCHW - // in OpenCV: NCHW->NCHW - data_layouts[name] = DATA_LAYOUT_NCHW; - } - else - CV_Error(Error::StsParseError, "Only NHWC <-> NCHW permutations are allowed."); - } - int id = dstNet.addLayer(name, type, layerParams); - layer_id[name] = id; - connect(layer_id, dstNet, parsePin(layer.input(0)), id, 0); - } - else - { - layerParams.set("order", DictValue::arrayInt(permData, perm.total())); - - int id = dstNet.addLayer(name, "Permute", layerParams); - layer_id[name] = id; - - // one input only - connect(layer_id, dstNet, parsePin(layer.input(0)), id, 0); - data_layouts[name] = DATA_LAYOUT_UNKNOWN; - } - } - else if (type == "Const") - { - } - else if (type == "LRN") - { - CV_CheckGT(num_inputs, 0, ""); - if(hasLayerAttr(layer, "alpha")) { - layerParams.set("alpha", getLayerAttr(layer, "alpha").f()); - } - if(hasLayerAttr(layer, "beta")) { - layerParams.set("beta", getLayerAttr(layer, "beta").f()); - } - if(hasLayerAttr(layer, "depth_radius")) { - int radius = (int)getLayerAttr(layer, "depth_radius").i(); - layerParams.set("local_size", 2*radius + 1); - } - if(hasLayerAttr(layer, "bias")) { - layerParams.set("bias", getLayerAttr(layer, "bias").f()); - } - layerParams.set("norm_by_size", false); - - int id = dstNet.addLayer(name, "LRN", layerParams); - layer_id[name] = id; - - connectToAllBlobs(layer_id, dstNet, parsePin(layer.input(0)), id, num_inputs); - } - else if (type == "Concat" || type == "ConcatV2") - { - CV_CheckGT(num_inputs, 0, ""); - int axisId = (type == "Concat" ? 0 : num_inputs - 1); - int axis = getConstBlob(layer, value_id, axisId).int_val().Get(0); - - if (getDataLayout(name, data_layouts) == DATA_LAYOUT_NHWC) - axis = toNCHW(axis); - else if (getDataLayout(name, data_layouts) == DATA_LAYOUT_NDHWC) - axis = toNCDHW(axis); - layerParams.set("axis", axis); - - // input(0) or input(n-1) is concat_dim - int from = (type == "Concat" ? 1 : 0); - int to = (type == "Concat" ? num_inputs : num_inputs - 1); - - for (int ii = from; ii < to; ii++) - { - Pin inp = parsePin(layer.input(ii)); - if (layer_id.find(inp.name) == layer_id.end()) - { - // There are constant inputs. - LayerParams lp; - lp.name = inp.name; - lp.type = "Const"; - lp.blobs.resize(1); - blobFromTensor(getConstBlob(layer, value_id, ii), lp.blobs.back()); - CV_Assert_N(!lp.blobs[0].empty(), lp.blobs[0].type() == CV_32F); - - int constInpId = dstNet.addLayer(lp.name, lp.type, lp); - layer_id[lp.name] = constInpId; - } - } - - int id = dstNet.addLayer(name, "Concat", layerParams); - layer_id[name] = id; - - for (int ii = from; ii < to; ii++) - { - Pin inp = parsePin(layer.input(ii)); - if (layer_id.find(inp.name) == layer_id.end()) - CV_Error(Error::StsError, "Input layer not found: " + inp.name); - connect(layer_id, dstNet, inp, id, ii - from); - } - } - else if (type == "MaxPool" || type == "MaxPool3D") - { - CV_CheckGT(num_inputs, 0, ""); - layerParams.set("pool", "max"); - - setKSize(layerParams, layer); - setStrides(layerParams, layer); - setPadding(layerParams, layer); - // Test_TensorFlow_nets.EAST_text_detection/1, NGRAPH/CPU - layerParams.set("ceil_mode", false); - - int id = dstNet.addLayer(name, "Pooling", layerParams); - layer_id[name] = id; - - connectToAllBlobs(layer_id, dstNet, parsePin(layer.input(0)), id, num_inputs); - } - else if (type == "AvgPool" || type == "AvgPool3D") - { - CV_CheckGT(num_inputs, 0, ""); - layerParams.set("pool", "ave"); - layerParams.set("ave_pool_padded_area", false); - setKSize(layerParams, layer); - setStrides(layerParams, layer); - setPadding(layerParams, layer); - - int id = dstNet.addLayer(name, "Pooling", layerParams); - layer_id[name] = id; - - connectToAllBlobs(layer_id, dstNet, parsePin(layer.input(0)), id, num_inputs); - } - else if (type == "MaxPoolGrad") - { - CV_CheckEQ(num_inputs, 3, ""); - - layerParams.set("pool_k_h", 0); - layerParams.set("pool_k_w", 0); - layerParams.set("pool_stride_h", 0); - layerParams.set("pool_stride_w", 0); - layerParams.set("pool_pad_h", 0); - layerParams.set("pool_pad_w", 0); - - int id = dstNet.addLayer(name, "MaxUnpool", layerParams); - layer_id[name] = id; - - connect(layer_id, dstNet, parsePin(layer.input(2)), id, 0); - connect(layer_id, dstNet, parsePin(layer.input(1) + ":1"), id, 1); - connect(layer_id, dstNet, parsePin(layer.input(0)), id, 2); - } - else if (type == "Placeholder") - { - if (!hasLayerAttr(layer, "dtype") || - getLayerAttr(layer, "dtype").type() != tensorflow::DT_BOOL) // If input is not a train/test flag. - { - netInputsNames.push_back(name); - layer_id[name] = 0; - } - tensorflow::TensorShapeProto shape; - if (hasLayerAttr(layer, "shape")) - shape = getLayerAttr(layer, "shape").shape(); - else if (hasLayerAttr(layer, "_output_shapes")) - { - tensorflow::AttrValue_ListValue list = getLayerAttr(layer, "_output_shapes").list(); - if (list.shape_size()) - shape = list.shape()[0]; - } - if (shape.dim_size()) - { - MatShape dims(shape.dim_size()); - for (int i = 0; i < dims.size(); ++i) - dims[i] = shape.dim(i).size(); - if (dims.size() == 4 && predictedLayout == DATA_LAYOUT_NHWC) - { - std::swap(dims[1], dims[3]); // NHWC->NCWH - std::swap(dims[2], dims[3]); // NCWH->NCHW - if (dims[0] == -1) // It's OK to have undetermined batch size - dims[0] = 1; - } - bool hasNeg = false; - for (int i = 0; i < dims.size() && !hasNeg; ++i) - { - hasNeg = dims[i] < 0; - } - if (!hasNeg) - netInputShapes.push_back(dims); - } - } - else if (type == "Split") { - // TODO: determining axis index remapping by input dimensions order of input blob - // TODO: slicing input may be Const op - // TODO: slicing kernels for convolutions - in current implementation it is impossible - // TODO: add parsing num of slices parameter - CV_CheckEQ(num_inputs, 2, ""); - // num_split - // 1st blob is dims tensor - int axis = getConstBlob(layer, value_id, 0).int_val().Get(0); - if (getDataLayout(name, data_layouts) == DATA_LAYOUT_NHWC) - axis = toNCHW(axis); - layerParams.set("axis", axis); - - if (hasLayerAttr(layer, "num_split")) - layerParams.set("num_split", getLayerAttr(layer, "num_split").i()); - - int id = dstNet.addLayer(name, "Slice", layerParams); - layer_id[name] = id; - - // one input only - connect(layer_id, dstNet, parsePin(layer.input(1)), id, 0); - } - else if (type == "Slice") - { - // op: "Slice" - // input: "input_node" - // input: "Slice/begin" - // input: "Slice/size" - CV_CheckEQ(num_inputs, 3, ""); - Mat begins = getTensorContent(getConstBlob(layer, value_id, 1)); - Mat sizes = getTensorContent(getConstBlob(layer, value_id, 2)); - CV_Assert_N(!begins.empty(), !sizes.empty()); - CV_CheckTypeEQ(begins.type(), CV_32SC1, ""); - CV_CheckTypeEQ(sizes.type(), CV_32SC1, ""); - - if (begins.total() == 4 && getDataLayout(name, data_layouts) == DATA_LAYOUT_NHWC) - { - // Swap NHWC parameters' order to NCHW. - std::swap(*begins.ptr(0, 2), *begins.ptr(0, 3)); - std::swap(*begins.ptr(0, 1), *begins.ptr(0, 2)); - std::swap(*sizes.ptr(0, 2), *sizes.ptr(0, 3)); - std::swap(*sizes.ptr(0, 1), *sizes.ptr(0, 2)); - } - layerParams.set("begin", DictValue::arrayInt((int*)begins.data, begins.total())); - layerParams.set("size", DictValue::arrayInt((int*)sizes.data, sizes.total())); - - int id = dstNet.addLayer(name, "Slice", layerParams); - layer_id[name] = id; - - connect(layer_id, dstNet, parsePin(layer.input(0)), id, 0); - } - else if (type == "StridedSlice") - { - CV_CheckEQ(num_inputs, 4, ""); - Mat begins = getTensorContent(getConstBlob(layer, value_id, 1)); - Mat ends = getTensorContent(getConstBlob(layer, value_id, 2)); - Mat strides = getTensorContent(getConstBlob(layer, value_id, 3)); - CV_CheckTypeEQ(begins.type(), CV_32SC1, ""); - CV_CheckTypeEQ(ends.type(), CV_32SC1, ""); - CV_CheckTypeEQ(strides.type(), CV_32SC1, ""); - const int num = begins.total(); - CV_Assert_N(num == ends.total(), num == strides.total()); - - int end_mask = getLayerAttr(layer, "end_mask").i(); - for (int i = 0; i < num; ++i) - { - if (ends.at(i) < 0) - ends.at(i) -= 1; - if (end_mask & (1 << i)) - ends.at(i) = -1; - if (strides.at(i) != 1) - CV_Error(Error::StsNotImplemented, - format("StridedSlice with stride %d", strides.at(i))); - } - if (begins.total() == 4 && getDataLayout(name, data_layouts) == DATA_LAYOUT_NHWC) - { - // Swap NHWC parameters' order to NCHW. - std::swap(begins.at(2), begins.at(3)); - std::swap(begins.at(1), begins.at(2)); - std::swap(ends.at(2), ends.at(3)); - std::swap(ends.at(1), ends.at(2)); - } - layerParams.set("begin", DictValue::arrayInt((int*)begins.data, begins.total())); - layerParams.set("end", DictValue::arrayInt((int*)ends.data, ends.total())); - - int id = dstNet.addLayer(name, "Slice", layerParams); - layer_id[name] = id; - - connect(layer_id, dstNet, parsePin(layer.input(0)), id, 0); - } - else if (type == "Mul" || type == "RealDiv") - { - CV_CheckGT(num_inputs, 0, ""); - int constId = -1; - for(int ii = 0; ii < num_inputs; ++ii) - { - Pin input = parsePin(layer.input(ii)); - if (value_id.find(input.name) != value_id.end()) - { - constId = ii; - break; - } - } - CV_Assert((constId != -1) || (num_inputs == 2)); - - if (constId != -1) - { - // Multiplication by constant. - CV_CheckEQ(num_inputs, 2, ""); - Mat scaleMat = getTensorContent(getConstBlob(layer, value_id)); - CV_Assert(scaleMat.type() == CV_32FC1); - if (type == "RealDiv") - { - if (constId == 0) - CV_Error(Error::StsNotImplemented, "Division of constant over variable"); - scaleMat = 1.0f / scaleMat; - } - - int id; - if (scaleMat.total() == 1) // is a scalar. - { - // Try to match with a LeakyRelu: - // node { - // name: "LeakyRelu/mul" - // op: "Mul" - // input: "LeakyRelu/alpha" - // input: "input" - // } - // node { - // name: "LeakyRelu/Maximum" - // op: "Maximum" - // input: "LeakyRelu/mul" - // input: "input" - // } - StrIntVector next_layers = getNextLayers(net, name, "Maximum"); - if (!next_layers.empty()) - { - int maximumLayerIdx = next_layers[0].second; - - CV_Assert(net.node(maximumLayerIdx).input_size() == 2); - - // The input from the Mul layer can also be at index 1. - int mulInputIdx = (net.node(maximumLayerIdx).input(0) == name) ? 0 : 1; - - ExcludeLayer(net, maximumLayerIdx, mulInputIdx, false); - layers_to_ignore.insert(next_layers[0].first); - - layerParams.set("negative_slope", scaleMat.at(0)); - id = dstNet.addLayer(name, "ReLU", layerParams); - } - else - { - // Just a multiplication. - layerParams.set("scale", scaleMat.at(0)); - id = dstNet.addLayer(name, "Power", layerParams); - } - } - else // is a vector - { - layerParams.blobs.resize(1, scaleMat); - - StrIntVector next_layers = getNextLayers(net, name, "Add"); - if (!next_layers.empty()) - { - layerParams.set("bias_term", true); - layerParams.blobs.resize(2); - - int weights_layer_index = next_layers[0].second; - blobFromTensor(getConstBlob(net.node(weights_layer_index), value_id), layerParams.blobs.back()); - ExcludeLayer(net, weights_layer_index, 0, false); - layers_to_ignore.insert(next_layers[0].first); - } - - if (hasLayerAttr(layer, "axis")) - layerParams.set("axis", getLayerAttr(layer, "axis").i()); - - id = dstNet.addLayer(name, "Scale", layerParams); - } - layer_id[name] = id; - - Pin inp0 = parsePin(layer.input(0)); - if (layer_id.find(inp0.name) != layer_id.end()) - // First operand is a constant. - connect(layer_id, dstNet, parsePin(layer.input(0)), id, 0); - else - connect(layer_id, dstNet, parsePin(layer.input(1)), id, 0); - } - else - { - // Check if all the inputs have the same shape. - bool equalInpShapes = true; - bool isShapeOnes = false; - MatShape outShape0; - for (int ii = 0; ii < num_inputs && !netInputShapes.empty(); ii++) - { - Pin pin = parsePin(layer.input(ii)); - int inpId = layer_id.find(pin.name)->second; - - // Get input shape - MatShape outShape; - std::vector inpShapes, outShapes; - dstNet.getLayerShapes(netInputShapes, inpId, inpShapes, outShapes); - CV_CheckGT(static_cast(outShapes.size()), pin.blobIndex, ""); - outShape = outShapes[pin.blobIndex]; - - if (ii == 0) - { - outShape0 = outShape; - } - else if (outShape != outShape0) - { - equalInpShapes = false; - isShapeOnes = isAllOnes(outShape, 2, outShape.size()) || - isAllOnes(outShape0, 2, outShape0.size()); - break; - } - } - - int id; - if (equalInpShapes || netInputShapes.empty() || (!equalInpShapes && isShapeOnes)) - { - layerParams.set("operation", type == "RealDiv" ? "div" : "prod"); - id = dstNet.addLayer(name, "Eltwise", layerParams); - } - else - { - if (type == "RealDiv") - CV_Error(Error::StsNotImplemented, "Division of non equal tensors"); - id = dstNet.addLayer(name, "Scale", layerParams); - } - - layer_id[name] = id; - - for (int ii = 0; ii < num_inputs; ii++) - { - Pin inp = parsePin(layer.input(ii)); - if (layer_id.find(inp.name) == layer_id.end()) - CV_Error(Error::StsError, "Input layer not found: " + inp.name); - connect(layer_id, dstNet, inp, id, ii); - } - } - } - else if (type == "FusedBatchNorm" || type == "FusedBatchNormV3") - { - // op: "FusedBatchNorm" - // input: "input" - // input: "BatchNorm/gamma" - // input: "BatchNorm/beta" - // input: "BatchNorm/moving_mean" - // input: "BatchNorm/moving_variance" - CV_CheckEQ(num_inputs, 5, "Expected gamma, beta, mean and std"); - Pin inpId = parsePin(layer.input(0)); - - bool isTraining = hasLayerAttr(layer, "is_training") && getLayerAttr(layer, "is_training").b(); - - layerParams.blobs.resize(2); - - const tensorflow::TensorProto& gammaTensor = getConstBlob(layer, value_id, 1); - if (!gammaTensor.tensor_content().empty()) - { - layerParams.blobs.resize(layerParams.blobs.size() + 1); - layerParams.set("has_weight", true); - blobFromTensor(gammaTensor, layerParams.blobs.back()); - } - else - layerParams.set("has_weight", false); - - const tensorflow::TensorProto& betaTensor = getConstBlob(layer, value_id, 2); - if (!betaTensor.tensor_content().empty()) - { - layerParams.blobs.resize(layerParams.blobs.size() + 1); - layerParams.set("has_bias", true); - blobFromTensor(betaTensor, layerParams.blobs.back()); - } - else - layerParams.set("has_bias", false); - - Mat mean, std; - if (isTraining) - { - if (layerParams.blobs.size() == 2) - CV_Error(Error::StsNotImplemented, "Cannot determine number " - "of parameters for batch normalization layer."); - mean = Mat::zeros(1, layerParams.blobs[2].total(), CV_32F); - std = Mat::ones(1, layerParams.blobs[2].total(), CV_32F); - - // Add an extra layer: Mean-Variance normalization - LayerParams mvnParams; - std::string mvnName = name + "/MVN"; - CV_Assert(layer_id.find(mvnName) == layer_id.end()); - int mvnId = dstNet.addLayer(mvnName, "MVN", mvnParams); - layer_id[mvnName] = mvnId; - connect(layer_id, dstNet, inpId, mvnId, 0); - inpId = Pin(mvnName); - } - else - { - blobFromTensor(getConstBlob(layer, value_id, 3), mean); - blobFromTensor(getConstBlob(layer, value_id, 4), std); - } - layerParams.blobs[0] = mean; - layerParams.blobs[1] = std; - - if (hasLayerAttr(layer, "epsilon")) - layerParams.set("eps", getLayerAttr(layer, "epsilon").f()); - - int id = dstNet.addLayer(name, "BatchNorm", layerParams); - layer_id[name] = id; - - // one input only - connect(layer_id, dstNet, inpId, id, 0); - } - else if (type == "Conv2DBackpropInput") - { - // op: "Conv2DBackpropInput" - // input: "conv2d_transpose/output_shape" - // input: "weights" - // input: "input" - CV_CheckEQ(num_inputs, 3, "Expected output shape, weights and input nodes"); - - layerParams.set("bias_term", false); - layerParams.blobs.resize(1); - - StrIntVector next_layers = getNextLayers(net, name, "BiasAdd"); - if (next_layers.size() == 1) - { - layerParams.set("bias_term", true); - layerParams.blobs.resize(2); - - int weights_layer_index = next_layers[0].second; - - blobFromTensor(getConstBlob(net.node(weights_layer_index), value_id), layerParams.blobs[1]); - ExcludeLayer(net, weights_layer_index, 0, false); - layers_to_ignore.insert(next_layers[0].first); - } - - kernelFromTensor(getConstBlob(layer, value_id, 1), layerParams.blobs[0]); - - const int* kshape = layerParams.blobs[0].size.p; - const int kernelH = kshape[2]; - const int kernelW = kshape[3]; - layerParams.set("kernel_h", kernelH); - layerParams.set("kernel_w", kernelW); - layerParams.set("num_output", kshape[1]); - - setStrides(layerParams, layer); - setPadding(layerParams, layer); - - // For convolution layer, output shape computes as - // o = 1 + (i - k + 2*p) / s - // i - input size, o - output size, k - kernel size, p - pad, s - stride - // In TensorFlow, p == 0 is padMode == 'VALID' or p == (k - 1) / 2 - // considering that k is odd. - // SAME: o = 1 + (i - 1) / s - // VALID: o = 1 + i / s - // Deconvolution's layer output shape computes as - // SAME: o = 1 + (i - 1)*s - // VALID: o = (i - 1)*s - // If output_shape differs from formulas above then adjust padding is applied. - - const int strideY = layerParams.get("stride_h"); - const int strideX = layerParams.get("stride_w"); - Mat outShape = getTensorContent(getConstBlob(layer, value_id, 0)); - const int outH = outShape.at(1); - const int outW = outShape.at(2); - if (layerParams.get("pad_mode") == "SAME") - { - layerParams.set("adj_w", (outW - 1) % strideX); - layerParams.set("adj_h", (outH - 1) % strideY); - } - else if (layerParams.get("pad_mode") == "VALID") - { - layerParams.set("adj_w", (outW - kernelW) % strideX); - layerParams.set("adj_h", (outH - kernelH) % strideY); - } - int id = dstNet.addLayer(name, "Deconvolution", layerParams); - layer_id[name] = id; - - // one input only - connect(layer_id, dstNet, parsePin(layer.input(2)), id, 0); - } - else if (type == "BlockLSTM") - { - // op: "BlockLSTM" - // input: "lstm_block_wrapper/ToInt64/x" (ignore, number of time stamps) - // input: "input" - // input: "lstm_block_wrapper/zeros" (ignore) - // input: "lstm_block_wrapper/zeros" (ignore) - // input: "lstm_block_wrapper/kernel" - // input: "lstm_block_wrapper/w_i_diag" - // input: "lstm_block_wrapper/w_f_diag" - // input: "lstm_block_wrapper/w_o_diag" - // input: "lstm_block_wrapper/bias" - CV_CheckEQ(num_inputs, 9, "Unexpected number of input nodes"); - - if (hasLayerAttr(layer, "forget_bias")) - layerParams.set("forget_bias", getLayerAttr(layer, "forget_bias").f()); - - if (hasLayerAttr(layer, "forget_bias")) - { - float cellClip = getLayerAttr(layer, "cell_clip").f(); - // Cell clip disabled if it's negative. - if (cellClip >= 0) - { - layerParams.set("use_cell_clip", true); - layerParams.set("cell_clip", cellClip); - } - } - - Mat W, Wh, Wx, b; - blobFromTensor(getConstBlob(layer, value_id, 4), W); - blobFromTensor(getConstBlob(layer, value_id, 8), b); - const int outSize = W.cols / 4; - - // IGFO->IFOG - float* weightData = (float*)W.data; - for (int i = 0; i < W.rows; ++i) - for (int j = 0; j < outSize; ++j) - { - std::swap(weightData[i * W.cols + 1 * outSize + j], - weightData[i * W.cols + 2 * outSize + j]); - std::swap(weightData[i * W.cols + 2 * outSize + j], - weightData[i * W.cols + 3 * outSize + j]); - } - Wx = W.rowRange(0, W.rows - outSize).t(); - Wh = W.rowRange(W.rows - outSize, W.rows).t(); - - layerParams.blobs.resize(3); - layerParams.blobs[0] = Wh; - layerParams.blobs[1] = Wx; - layerParams.blobs[2] = b; - - if (hasLayerAttr(layer, "use_peephole")) - { - bool usePeephole = getLayerAttr(layer, "use_peephole").b(); - if (usePeephole) - { - layerParams.set("use_peephole", true); - layerParams.blobs.resize(6); - for (int i = 0; i < 3; ++i) - { - Mat w; - blobFromTensor(getConstBlob(layer, value_id, 5 + i), w); - w = w.reshape(1, w.total()); // Single column. - w = Mat::diag(w); // Make a diagonal matrix. - layerParams.blobs[3 + i] = w; - } - } - } - - int id = dstNet.addLayer(name, "LSTM", layerParams); - layer_id[name] = id; - - // one input only - connect(layer_id, dstNet, parsePin(layer.input(1)), id, 0); - data_layouts[name] = DATA_LAYOUT_UNKNOWN; - } - else if (type == "ResizeNearestNeighbor" || type == "ResizeBilinear" || type == "FusedResizeAndPadConv2D") - { - CV_CheckGT(num_inputs, 0, ""); - std::string convWeights = ""; - if (type == "FusedResizeAndPadConv2D") - { - // input: "mul_1" - // input: "decoder/ResizeBilinear/size" - // input: "decoder/decoder_conv0/Conv2D_dummy_paddings" - // input: "decoder/decoder_conv0/weights" - CV_CheckEQ(num_inputs, 4, "Number of input for FusedResizeAndPadConv2D"); - - Mat paddings = getTensorContent(getConstBlob(layer, value_id, 2)); - CV_CheckEQ(countNonZero(paddings), 0, "Unsupported mode"); - - convWeights = layer.input(3); - layer.mutable_input()->DeleteSubrange(2, 2); // FIXIT do NOT modify input model - num_inputs = layer.input_size(); - name = name + "/resize"; - - if (hasLayerAttr(layer, "resize_align_corners")) - { - // FIXIT do NOT modify input model - layer.mutable_attr()->insert( - ::google::protobuf::MapPair("align_corners", - getLayerAttr(layer, "resize_align_corners"))); - } - } - if (num_inputs == 2) - { - Mat outSize = getTensorContent(getConstBlob(layer, value_id, 1)); - CV_CheckTypeEQ(outSize.type(), CV_32SC1, ""); CV_CheckEQ(outSize.total(), (size_t)2, ""); - layerParams.set("height", outSize.at(0, 0)); - layerParams.set("width", outSize.at(0, 1)); - } - else if (num_inputs == 3) - { - Mat factorHeight = getTensorContent(getConstBlob(layer, value_id, 1)); - Mat factorWidth = getTensorContent(getConstBlob(layer, value_id, 2)); - factorHeight.convertTo(factorHeight, CV_32F); - factorWidth.convertTo(factorWidth, CV_32F); - layerParams.set("zoom_factor_x", factorWidth.at(0)); - layerParams.set("zoom_factor_y", factorHeight.at(0)); - } - else - CV_Check(num_inputs, num_inputs == 2 || num_inputs == 3, ""); - - if (type == "ResizeNearestNeighbor") - layerParams.set("interpolation", "nearest"); - else - layerParams.set("interpolation", "bilinear"); - - if (hasLayerAttr(layer, "align_corners")) - layerParams.set("align_corners", getLayerAttr(layer, "align_corners").b()); - - if (hasLayerAttr(layer, "half_pixel_centers")) - layerParams.set("half_pixel_centers", getLayerAttr(layer, "half_pixel_centers").b()); - - int id = dstNet.addLayer(name, "Resize", layerParams); - layer_id[name] = id; - - connect(layer_id, dstNet, parsePin(layer.input(0)), id, 0); - - // Step back to add convolution - if (type == "FusedResizeAndPadConv2D") - { - tensorflow::NodeDef conv = layer_; - conv.clear_input(); - conv.add_input(name); - conv.add_input(convWeights); - conv.set_op("Conv2D"); - parseNode(conv); - } - } - else if (type == "L2Normalize") - { - // op: "L2Normalize" - // input: "input" - // input: "reduction_indices" (axis) - CV_CheckEQ(num_inputs, 2, ""); - Mat reductionIndices = getTensorContent(getConstBlob(layer, value_id, 1)); - CV_Assert(reductionIndices.type() == CV_32SC1); - - const int numAxes = reductionIndices.total(); - if (getDataLayout(name, data_layouts) == DATA_LAYOUT_NHWC) - for (int i = 0; i < numAxes; ++i) - reductionIndices.at(i) = toNCHW(reductionIndices.at(i)); - - cv::sort(reductionIndices, reductionIndices, SORT_ASCENDING); - for (int i = 1; i < numAxes; ++i) - { - CV_Assert(reductionIndices.at(i) == reductionIndices.at(i - 1) + 1); - // Axes have the same sign. - CV_Assert(reductionIndices.at(i) * reductionIndices.at(i - 1) >= 0); - } - layerParams.set("start_axis", reductionIndices.at(0)); - layerParams.set("end_axis", reductionIndices.at(numAxes - 1)); - - int id = dstNet.addLayer(name, "Normalize", layerParams); - layer_id[name] = id; - connect(layer_id, dstNet, parsePin(layer.input(0)), id, 0); - } - else if (type == "PriorBox") - { - CV_CheckEQ(num_inputs, 2, ""); - if (hasLayerAttr(layer, "min_size")) - layerParams.set("min_size", getLayerAttr(layer, "min_size").i()); - if (hasLayerAttr(layer, "max_size")) - layerParams.set("max_size", getLayerAttr(layer, "max_size").i()); - if (hasLayerAttr(layer, "flip")) - layerParams.set("flip", getLayerAttr(layer, "flip").b()); - if (hasLayerAttr(layer, "clip")) - layerParams.set("clip", getLayerAttr(layer, "clip").b()); - if (hasLayerAttr(layer, "offset")) - layerParams.set("offset", getLayerAttr(layer, "offset").f()); - if (hasLayerAttr(layer, "step")) - layerParams.set("step", getLayerAttr(layer, "step").f()); - - const std::string paramNames[] = {"variance", "aspect_ratio", "scales", - "width", "height"}; - for (int i = 0; i < 5; ++i) - { - if (hasLayerAttr(layer, paramNames[i])) - { - Mat values = getTensorContent(getLayerAttr(layer, paramNames[i]).tensor()); - layerParams.set(paramNames[i], - DictValue::arrayReal((float*)values.data, values.total())); - } - } - int id = dstNet.addLayer(name, "PriorBox", layerParams); - layer_id[name] = id; - connect(layer_id, dstNet, parsePin(layer.input(0)), id, 0); - connect(layer_id, dstNet, parsePin(layer.input(1)), id, 1); - data_layouts[name] = DATA_LAYOUT_UNKNOWN; - } - else if (type == "Softmax") - { - CV_CheckGT(num_inputs, 0, ""); - if (hasLayerAttr(layer, "axis")) - layerParams.set("axis", getLayerAttr(layer, "axis").i()); - - int id = dstNet.addLayer(name, "Softmax", layerParams); - layer_id[name] = id; - connectToAllBlobs(layer_id, dstNet, parsePin(layer.input(0)), id, num_inputs); - } - else if (type == "CropAndResize") - { - // op: "CropAndResize" - // input: "input" - // input: "boxes" - // input: "sizes" - CV_CheckEQ(num_inputs, 3, ""); - - Mat cropSize = getTensorContent(getConstBlob(layer, value_id, 2)); - CV_CheckTypeEQ(cropSize.type(), CV_32SC1, ""); CV_CheckEQ(cropSize.total(), (size_t)2, ""); - - layerParams.set("height", cropSize.at(0)); - layerParams.set("width", cropSize.at(1)); - - int id = dstNet.addLayer(name, "CropAndResize", layerParams); - layer_id[name] = id; - - connect(layer_id, dstNet, parsePin(layer.input(0)), id, 0); - connect(layer_id, dstNet, parsePin(layer.input(1)), id, 1); - } - else if (type == "Mean" || type == "Sum") - { - // Computes the mean of elements across dimensions of a tensor. - // If keepdims is false (default) reduces input_tensor along the dimensions given in axis, - // else the reduced dimensions are retained with length 1. - // if indices = [1, 2] in NHWC layout we use global pooling: NxCxHxW --Pooling--> NxCx1x1 - // if keepdims is false we use Flatten after Pooling: out_shape = NxC - // if indices = [0] we use a global pooling by indices. - // To return correct shape, we use Reshape after Pooling. To determine input shape use Slice for input, - // if keepdims is false we use Flatten after Slice. - // Example: input_shape = NxCxHxW - // determine out shape: NxCxHxW --Slice--> 1xCxHxW - // out_shape = 1xCxHxW if keepDims else (1xCxHxW --Flatten--> CxHxW) - // global pool: NxCxHxW --Flatten--> Nx(C*H*W) --Reshape--> 1x1xNx(C*H*W) --Pooling--> 1x1x1x(C*H*W) --Reshape--> out_shape - CV_CheckGT(num_inputs, 0, ""); - - Mat indices = getTensorContent(getConstBlob(layer, value_id, 1)); - CV_Assert(indices.type() == CV_32SC1); - - // There are two attributes, "keepdims" and a deprecated "keep_dims". - bool keepDims = false; - if (hasLayerAttr(layer, "keepdims")) - keepDims = getLayerAttr(layer, "keepdims").b(); - else if (hasLayerAttr(layer, "keep_dims")) - keepDims = getLayerAttr(layer, "keep_dims").b(); - - if (indices.total() == 1 && indices.at(0) == 0) - { - LayerParams flattenLp; - std::string flattenName = name + "/flatten"; - CV_Assert(layer_id.find(flattenName) == layer_id.end()); - int flattenId = dstNet.addLayer(flattenName, "Flatten", flattenLp); - layer_id[flattenName] = flattenId; - connect(layer_id, dstNet, parsePin(layer.input(0)), flattenId, 0); - - LayerParams reshapeLp; - std::string reshapeName = name + "/reshape"; - CV_Assert(layer_id.find(reshapeName) == layer_id.end()); - reshapeLp.set("axis", 0); - reshapeLp.set("num_axes", 1); - int newShape[] = {1, 1, -1}; - reshapeLp.set("dim", DictValue::arrayInt(&newShape[0], 3)); - - int reshapeId = dstNet.addLayer(reshapeName, "Reshape", reshapeLp); - layer_id[reshapeName] = reshapeId; - connect(layer_id, dstNet, Pin(flattenName), reshapeId, 0); - - LayerParams avgLp; - std::string avgName = name + "/avg"; - CV_Assert(layer_id.find(avgName) == layer_id.end()); - avgLp.set("pool", type == "Mean" ? "ave" : "sum"); - // pooling kernel H x 1 - avgLp.set("global_pooling_h", true); - avgLp.set("kernel_w", 1); - int avgId = dstNet.addLayer(avgName, "Pooling", avgLp); - layer_id[avgName] = avgId; - connect(layer_id, dstNet, Pin(reshapeName), avgId, 0); - - LayerParams sliceLp; - std::string layerShapeName = name + "/slice"; - CV_Assert(layer_id.find(layerShapeName) == layer_id.end()); - sliceLp.set("axis", 0); - int begin[] = {0}; - int size[] = {1}; - sliceLp.set("begin", DictValue::arrayInt(&begin[0], 1)); - sliceLp.set("size", DictValue::arrayInt(&size[0], 1)); - int sliceId = dstNet.addLayer(layerShapeName, "Slice", sliceLp); - layer_id[layerShapeName] = sliceId; - connect(layer_id, dstNet, Pin(layer.input(0)), sliceId, 0); - - if (!keepDims) - { - LayerParams squeezeLp; - std::string squeezeName = name + "/squeeze"; - CV_Assert(layer_id.find(squeezeName) == layer_id.end()); - squeezeLp.set("axis", 0); - squeezeLp.set("end_axis", 1); - int squeezeId = dstNet.addLayer(squeezeName, "Flatten", squeezeLp); - layer_id[squeezeName] = squeezeId; - connect(layer_id, dstNet, Pin(layerShapeName), squeezeId, 0); - layerShapeName = squeezeName; - } - - int id = dstNet.addLayer(name, "Reshape", layerParams); - layer_id[name] = id; - connect(layer_id, dstNet, Pin(avgName), id, 0); - connect(layer_id, dstNet, Pin(layerShapeName), id, 1); - } else if (indices.total() == 1) { - int axis = toNCHW(indices.at(0)); - if (axis == 2 || axis == 3) - { - layerParams.set("pool", type == "Mean" ? "ave" : "sum"); - layerParams.set(axis == 2 ? "kernel_w" : "kernel_h", 1); - layerParams.set(axis == 2 ? "global_pooling_h" : "global_pooling_w", true); - int id = dstNet.addLayer(name, "Pooling", layerParams); - layer_id[name] = id; - connect(layer_id, dstNet, parsePin(layer.input(0)), id, 0); - - if (!keepDims) - { - // To keep correct order after squeeze dims we first need to change layout from NCHW to NHWC - LayerParams permLP; - int order[] = {0, 2, 3, 1}; // From OpenCV's NCHW to NHWC. - std::string permName = name + "/nchw"; - Pin inpId = Pin(name); - addPermuteLayer(order, permName, inpId); - - LayerParams squeezeLp; - std::string squeezeName = name + "/squeeze"; - CV_Assert(layer_id.find(squeezeName) == layer_id.end()); - squeezeLp.set("axis", indices.at(0)); - squeezeLp.set("end_axis", indices.at(0) + 1); - int squeezeId = dstNet.addLayer(squeezeName, "Flatten", squeezeLp); - layer_id[squeezeName] = squeezeId; - connect(layer_id, dstNet, Pin(permName), squeezeId, 0); - } - } - else if (axis == 1) - { - int order[] = {0, 2, 3, 1}; // From OpenCV's NCHW to NHWC. - Pin inpId = parsePin(layer.input(0)); - addPermuteLayer(order, name + "/nhwc", inpId); - - layerParams.set("pool", type == "Mean" ? "ave" : "sum"); - layerParams.set("kernel_h", 1); - layerParams.set("global_pooling_w", true); - int id = dstNet.addLayer(name, "Pooling", layerParams); - layer_id[name] = id; - connect(layer_id, dstNet, inpId, id, 0); - - if (!keepDims) - { - LayerParams squeezeLp; - std::string squeezeName = name + "/squeeze"; - CV_Assert(layer_id.find(squeezeName) == layer_id.end()); - int channel_id = 3; // TF NHWC layout - squeezeLp.set("axis", channel_id - 1); - squeezeLp.set("end_axis", channel_id); - int squeezeId = dstNet.addLayer(squeezeName, "Flatten", squeezeLp); - layer_id[squeezeName] = squeezeId; - connect(layer_id, dstNet, Pin(name), squeezeId, 0); - } - else - { - int order[] = {0, 3, 1, 2}; // From NHWC to OpenCV's NCHW. - Pin inpId = parsePin(name); - addPermuteLayer(order, name + "/nchw", inpId); - } - } - } else { - if (indices.total() != 2 || indices.at(0) != 1 || indices.at(1) != 2) - CV_Error(Error::StsNotImplemented, "Unsupported mode of reduce_mean or reduce_sum operation."); - - layerParams.set("pool", type == "Mean" ? "ave" : "sum"); - layerParams.set("global_pooling", true); - int id = dstNet.addLayer(name, "Pooling", layerParams); - layer_id[name] = id; - connect(layer_id, dstNet, parsePin(layer.input(0)), id, 0); - - if (!keepDims) - { - LayerParams flattenLp; - std::string flattenName = name + "/flatten"; - CV_Assert(layer_id.find(flattenName) == layer_id.end()); - int flattenId = dstNet.addLayer(flattenName, "Flatten", flattenLp); - layer_id[flattenName] = flattenId; - connect(layer_id, dstNet, Pin(name), flattenId, 0); - } - } - } - else if (type == "Pack") - { - // op: tf.stack(list of tensors, axis=0) - // Join a list of inputs along a new axis. - // The "axis" specifies the index of the new axis in the dimensions of the output. - // Example: given a list with "N" tensors of shape (C, H, W): - // if axis == 0 then the output tensor will have the shape (N, C, H, W), - // if axis == 1 then the output tensor will have the shape (C, N, H, W). - CV_CheckGT(num_inputs, 0, ""); - CV_Assert(hasLayerAttr(layer, "axis")); - int dim = (int)getLayerAttr(layer, "axis").i(); - if (dim != 0) - CV_Error(Error::StsNotImplemented, "Unsupported mode of pack operation."); - - CV_Assert(hasLayerAttr(layer, "N")); - int num = (int)getLayerAttr(layer, "N").i(); - CV_CheckEQ(num_inputs, num, ""); - std::string base_name = name + "/reshape_"; - std::vector reshape_ids; - for (int i = 0; i < num; i++) { - std::ostringstream ss; - ss << i; - std::string reshape_name = base_name + ss.str(); - LayerParams reshapeLP; - reshapeLP.set("axis", dim); - reshapeLP.set("num_axes", 1); - int outShape[] = {1, -1}; - reshapeLP.set("dim", DictValue::arrayInt(&outShape[0], 2)); - int id = dstNet.addLayer(reshape_name, "Reshape", reshapeLP); - layer_id[reshape_name] = id; - reshape_ids.push_back(id); - connect(layer_id, dstNet, parsePin(layer.input(i)), id, 0); - } - - layerParams.set("axis", dim); - int id = dstNet.addLayer(name, "Concat", layerParams); - layer_id[name] = id; - - for (int li = 0; li < num; li++) - dstNet.connect(reshape_ids[li], 0, id, li); - } - else if (type == "ClipByValue") - { - // op: "ClipByValue" - // input: "input" - // input: "mix" - // input: "max" - CV_CheckEQ(num_inputs, 3, ""); - - Mat minValue = getTensorContent(getConstBlob(layer, value_id, 1)); - Mat maxValue = getTensorContent(getConstBlob(layer, value_id, 2)); - CV_CheckEQ(minValue.total(), (size_t)1, ""); CV_CheckTypeEQ(minValue.type(), CV_32FC1, ""); - CV_CheckEQ(maxValue.total(), (size_t)1, ""); CV_CheckTypeEQ(maxValue.type(), CV_32FC1, ""); - - layerParams.set("min_value", minValue.at(0)); - layerParams.set("max_value", maxValue.at(0)); - - int id = dstNet.addLayer(name, "ReLU6", layerParams); - layer_id[name] = id; - - connect(layer_id, dstNet, parsePin(layer.input(0)), id, 0); - } - else if (type == "LeakyRelu") - { - CV_CheckGT(num_inputs, 0, ""); - CV_Assert(hasLayerAttr(layer, "alpha")); - layerParams.set("negative_slope", getLayerAttr(layer, "alpha").f()); - - int id = dstNet.addLayer(name, "ReLU", layerParams); - layer_id[name] = id; - connectToAllBlobs(layer_id, dstNet, parsePin(layer.input(0)), id, num_inputs); - } - else if (type == "Abs" || type == "Tanh" || type == "Sigmoid" || - type == "Relu" || type == "Elu" || type == "Exp" || - type == "Identity" || type == "Relu6") - { - CV_CheckGT(num_inputs, 0, ""); - std::string dnnType = type; - if (type == "Abs") dnnType = "AbsVal"; - else if (type == "Tanh") dnnType = "TanH"; - else if (type == "Relu") dnnType = "ReLU"; - else if (type == "Relu6") dnnType = "ReLU6"; - else if (type == "Elu") dnnType = "ELU"; - - int id = dstNet.addLayer(name, dnnType, layerParams); - layer_id[name] = id; - connectToAllBlobs(layer_id, dstNet, parsePin(layer.input(0)), id, num_inputs); + ((*this).*(iter->second))(net, layer, layerParams); } else { - // Importer does not know how to map this TensorFlow's operation onto OpenCV's layer. - // However we create a layer with the same type and rely that user defined a custom layer. - - // All the attributes are added to LayerParams. - google::protobuf::Map attr = layer.attr(); - for (google::protobuf::Map::const_iterator ai = attr.begin(); - ai != attr.end(); ++ai) - { - if (ai->second.value_case() == tensorflow::AttrValue::kS) // string - layerParams.set(ai->first, ai->second.s()); - if (ai->second.value_case() == tensorflow::AttrValue::kI) // int64 - layerParams.set(ai->first, ai->second.i()); - if (ai->second.value_case() == tensorflow::AttrValue::kF) // float - layerParams.set(ai->first, ai->second.f()); - if (ai->second.value_case() == tensorflow::AttrValue::kB) // bool - layerParams.set(ai->first, ai->second.b()); - } - - // All the Const input nodes are added to layer's blobs. - std::vector inputsNames; - for (int i = 0; i < num_inputs; ++i) - { - // Check if input is a Const node. - if (value_id.find(layer.input(i)) != value_id.end()) - { - Mat blob = getTensorContent(getConstBlob(layer, value_id, i)); - layerParams.blobs.push_back(blob); - } - else - inputsNames.push_back(layer.input(i)); - } - int id = dstNet.addLayer(name, type, layerParams); - layer_id[name] = id; - - for (int i = 0; i < inputsNames.size(); ++i) - { - connect(layer_id, dstNet, parsePin(inputsNames[i]), id, i); - } + parseCustomLayer(net, layer, layerParams); } } catch (const std::exception& e) From 3cf43753876e3c2c2676283d4fd10a6018b5a264 Mon Sep 17 00:00:00 2001 From: Tiago De Gaspari Date: Sat, 12 Jun 2021 17:28:54 -0300 Subject: [PATCH 004/262] Merge pull request #19842 from gasparitiago:3.4 Update rotatedRectangleIntersection function to calculate near to origin * Change type used in points function from RotatedRect In the function that sets the points of a RotatedRect, the types should be double in order to keep the precision when dealing with RotatedRects that are defined far from the origin. This commit solves the problem in some assertions from rotatedRectangleIntersection when dealing with rectangles far from origin. * added proper type casts * Update rotatedRectangleIntersection function to calculate near to origin This commit changes the rotatedRectangleIntersection function in order to calculate the intersection of two rectangles considering that they are shifted near the coordinates origin (0, 0). This commit solves the problem in some assertions from rotatedRectangleIntersection when dealing with rectangles far from origin. * Revert type changes in types.cpp and adequate code to c++98 * Revert unnecessary casts on types.cpp Co-authored-by: Vadim Pisarevsky --- modules/imgproc/src/intersection.cpp | 57 +++++++++++++++++----- modules/imgproc/test/test_intersection.cpp | 17 +++++++ 2 files changed, 62 insertions(+), 12 deletions(-) diff --git a/modules/imgproc/src/intersection.cpp b/modules/imgproc/src/intersection.cpp index 3f749896a4..47d3f3f457 100644 --- a/modules/imgproc/src/intersection.cpp +++ b/modules/imgproc/src/intersection.cpp @@ -47,24 +47,16 @@ namespace cv { -int rotatedRectangleIntersection( const RotatedRect& rect1, const RotatedRect& rect2, OutputArray intersectingRegion ) +static int _rotatedRectangleIntersection( const RotatedRect& rect1, const RotatedRect& rect2, std::vector &intersection ) { CV_INSTRUMENT_REGION(); // L2 metric const float samePointEps = std::max(1e-16f, 1e-6f * (float)std::max(rect1.size.area(), rect2.size.area())); - if (rect1.size.empty() || rect2.size.empty()) - { - intersectingRegion.release(); - return INTERSECT_NONE; - } - Point2f vec1[4], vec2[4]; Point2f pts1[4], pts2[4]; - std::vector intersection; intersection.reserve(24); - rect1.points(pts1); rect2.points(pts2); @@ -92,8 +84,6 @@ int rotatedRectangleIntersection( const RotatedRect& rect1, const RotatedRect& r intersection[i] = pts1[i]; } - Mat(intersection).copyTo(intersectingRegion); - return INTERSECT_FULL; } } @@ -300,7 +290,50 @@ int rotatedRectangleIntersection( const RotatedRect& rect1, const RotatedRect& r } intersection.resize(N); - Mat(intersection).copyTo(intersectingRegion); + + return ret; +} + +int rotatedRectangleIntersection( const RotatedRect& rect1, const RotatedRect& rect2, OutputArray intersectingRegion ) +{ + CV_INSTRUMENT_REGION(); + + if (rect1.size.empty() || rect2.size.empty()) + { + intersectingRegion.release(); + return INTERSECT_NONE; + } + + // Shift rectangles closer to origin (0, 0) to improve the calculation of the intesection region + // To do that, the average center of the rectangles is moved to the origin + const Point2f averageCenter = (rect1.center + rect2.center) / 2.0f; + + RotatedRect shiftedRect1(rect1); + RotatedRect shiftedRect2(rect2); + + // Move rectangles closer to origin + shiftedRect1.center -= averageCenter; + shiftedRect2.center -= averageCenter; + + std::vector intersection; intersection.reserve(24); + + const int ret = _rotatedRectangleIntersection(shiftedRect1, shiftedRect2, intersection); + + // If return is not None, the intersection Points are shifted back to the original position + // and copied to the interesectingRegion + if (ret != INTERSECT_NONE) + { + for (size_t i = 0; i < intersection.size(); ++i) + { + intersection[i] += averageCenter; + } + + Mat(intersection).copyTo(intersectingRegion); + } + else + { + intersectingRegion.release(); + } return ret; } diff --git a/modules/imgproc/test/test_intersection.cpp b/modules/imgproc/test/test_intersection.cpp index 7527dd9a22..c455c439fc 100644 --- a/modules/imgproc/test/test_intersection.cpp +++ b/modules/imgproc/test/test_intersection.cpp @@ -391,4 +391,21 @@ TEST(Imgproc_RotatedRectangleIntersection, regression_18520) } } +TEST(Imgproc_RotatedRectangleIntersection, regression_19824) +{ + RotatedRect r1( + Point2f(246805.033f, 4002326.94f), + Size2f(26.40587f, 6.20026f), + -62.10156f); + RotatedRect r2( + Point2f(246805.122f, 4002326.59f), + Size2f(27.4821f, 8.5361f), + -56.33761f); + + std::vector intersections; + int interType = cv::rotatedRectangleIntersection(r1, r2, intersections); + EXPECT_EQ(INTERSECT_PARTIAL, interType); + EXPECT_LE(intersections.size(), (size_t)7); +} + }} // namespace From c8268e65fd6c3a8fba7a5a1f4b830222b9bc9d66 Mon Sep 17 00:00:00 2001 From: Vincent Rabaud Date: Fri, 11 Jun 2021 22:03:33 +0200 Subject: [PATCH 005/262] Fix potential NaN in cv::norm. There can be an int overflow. cv::norm( InputArray _src, int normType, InputArray _mask ) is fine, not cv::norm( InputArray _src1, InputArray _src2, int normType, InputArray _mask ). --- modules/core/src/norm.cpp | 2 +- modules/core/test/test_arithm.cpp | 9 +++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/modules/core/src/norm.cpp b/modules/core/src/norm.cpp index 601082783e..fad641554d 100644 --- a/modules/core/src/norm.cpp +++ b/modules/core/src/norm.cpp @@ -1171,7 +1171,7 @@ double norm( InputArray _src1, InputArray _src2, int normType, InputArray _mask // special case to handle "integer" overflow in accumulator const size_t esz = src1.elemSize(); const int total = (int)it.size; - const int intSumBlockSize = normType == NORM_L1 && depth <= CV_8S ? (1 << 23) : (1 << 15); + const int intSumBlockSize = (normType == NORM_L1 && depth <= CV_8S ? (1 << 23) : (1 << 15))/cn; const int blockSize = std::min(total, intSumBlockSize); int isum = 0; int count = 0; diff --git a/modules/core/test/test_arithm.cpp b/modules/core/test/test_arithm.cpp index 2746feb2f2..74bf39fbc7 100644 --- a/modules/core/test/test_arithm.cpp +++ b/modules/core/test/test_arithm.cpp @@ -2117,6 +2117,15 @@ TEST(Core_Norm, IPP_regression_NORM_L1_16UC3_small) EXPECT_EQ((double)20*cn, cv::norm(a, b, NORM_L1, mask)); } +TEST(Core_Norm, NORM_L2_8UC4) +{ + // Tests there is no integer overflow in norm computation for multiple channels. + const int kSide = 100; + cv::Mat4b a(kSide, kSide, cv::Scalar(255, 255, 255, 255)); + cv::Mat4b b = cv::Mat4b::zeros(kSide, kSide); + const double kNorm = 2.*kSide*255.; + EXPECT_EQ(kNorm, cv::norm(a, b, NORM_L2)); +} TEST(Core_ConvertTo, regression_12121) { From 651967b95c852ebf4b751746197049ee558fe450 Mon Sep 17 00:00:00 2001 From: Maxim Pashchenkov Date: Tue, 15 Jun 2021 22:02:17 +0300 Subject: [PATCH 006/262] Merge pull request #19341 from mpashchenkov:mp/ocv-gapi-parsessd-fix G-API: Removing ParseSSD overload. * Removed specialization. * Removed united --- .../include/opencv2/gapi/infer/parsers.hpp | 9 ++-- modules/gapi/src/backends/cpu/gcpucore.cpp | 15 +++++- modules/gapi/src/backends/cpu/gnnparsers.cpp | 53 +++++-------------- modules/gapi/src/backends/cpu/gnnparsers.hpp | 13 ++--- 4 files changed, 35 insertions(+), 55 deletions(-) diff --git a/modules/gapi/include/opencv2/gapi/infer/parsers.hpp b/modules/gapi/include/opencv2/gapi/infer/parsers.hpp index 3225c73831..22c8701a6c 100644 --- a/modules/gapi/include/opencv2/gapi/infer/parsers.hpp +++ b/modules/gapi/include/opencv2/gapi/infer/parsers.hpp @@ -69,7 +69,8 @@ GAPI_EXPORTS std::tuple, GArray> parseSSD(const GMat& in, const float confidenceThreshold = 0.5f, const int filterLabel = -1); -/** @overload +/** @brief Parses output of SSD network. + Extracts detection information (box, confidence) from SSD output and filters it by given confidence and by going out of bounds. @@ -87,9 +88,9 @@ the larger side of the rectangle. */ GAPI_EXPORTS_W GArray parseSSD(const GMat& in, const GOpaque& inSz, - const float confidenceThreshold = 0.5f, - const bool alignmentToSquare = false, - const bool filterOutOfBounds = false); + const float confidenceThreshold, + const bool alignmentToSquare, + const bool filterOutOfBounds); /** @brief Parses output of Yolo network. diff --git a/modules/gapi/src/backends/cpu/gcpucore.cpp b/modules/gapi/src/backends/cpu/gcpucore.cpp index 168a2f9833..1f7dfb234f 100644 --- a/modules/gapi/src/backends/cpu/gcpucore.cpp +++ b/modules/gapi/src/backends/cpu/gcpucore.cpp @@ -652,7 +652,12 @@ GAPI_OCV_KERNEL(GCPUParseSSDBL, cv::gapi::nn::parsers::GParseSSDBL) std::vector& out_boxes, std::vector& out_labels) { - cv::parseSSDBL(in_ssd_result, in_size, confidence_threshold, filter_label, out_boxes, out_labels); + cv::ParseSSD(in_ssd_result, in_size, + confidence_threshold, + filter_label, + false, + false, + out_boxes, out_labels); } }; @@ -665,7 +670,13 @@ GAPI_OCV_KERNEL(GOCVParseSSD, cv::gapi::nn::parsers::GParseSSD) const bool filter_out_of_bounds, std::vector& out_boxes) { - cv::parseSSD(in_ssd_result, in_size, confidence_threshold, alignment_to_square, filter_out_of_bounds, out_boxes); + std::vector unused_labels; + cv::ParseSSD(in_ssd_result, in_size, + confidence_threshold, + -1, + alignment_to_square, + filter_out_of_bounds, + out_boxes, unused_labels); } }; diff --git a/modules/gapi/src/backends/cpu/gnnparsers.cpp b/modules/gapi/src/backends/cpu/gnnparsers.cpp index a5e4bf5f85..26de960b7f 100644 --- a/modules/gapi/src/backends/cpu/gnnparsers.cpp +++ b/modules/gapi/src/backends/cpu/gnnparsers.cpp @@ -170,12 +170,14 @@ private: } // namespace nn } // namespace gapi -void parseSSDBL(const cv::Mat& in_ssd_result, - const cv::Size& in_size, - const float confidence_threshold, - const int filter_label, - std::vector& out_boxes, - std::vector& out_labels) +void ParseSSD(const cv::Mat& in_ssd_result, + const cv::Size& in_size, + const float confidence_threshold, + const int filter_label, + const bool alignment_to_square, + const bool filter_out_of_bounds, + std::vector& out_boxes, + std::vector& out_labels) { cv::gapi::nn::SSDParser parser(in_ssd_result.size, in_size, in_ssd_result.ptr()); out_boxes.clear(); @@ -188,38 +190,6 @@ void parseSSDBL(const cv::Mat& in_ssd_result, { std::tie(rc, image_id, confidence, label) = parser.extract(i); - if (image_id < 0.f) - { - break; // marks end-of-detections - } - - if (confidence < confidence_threshold || - (filter_label != -1 && label != filter_label)) - { - continue; // filter out object classes if filter is specified - } // and skip objects with low confidence - out_boxes.emplace_back(rc & parser.getSurface()); - out_labels.emplace_back(label); - } -} - -void parseSSD(const cv::Mat& in_ssd_result, - const cv::Size& in_size, - const float confidence_threshold, - const bool alignment_to_square, - const bool filter_out_of_bounds, - std::vector& out_boxes) -{ - cv::gapi::nn::SSDParser parser(in_ssd_result.size, in_size, in_ssd_result.ptr()); - out_boxes.clear(); - cv::Rect rc; - float image_id, confidence; - int label; - const size_t range = parser.getMaxProposals(); - for (size_t i = 0; i < range; ++i) - { - std::tie(rc, image_id, confidence, label) = parser.extract(i); - if (image_id < 0.f) { break; // marks end-of-detections @@ -228,12 +198,14 @@ void parseSSD(const cv::Mat& in_ssd_result, { continue; // skip objects with low confidence } - + if((filter_label != -1) && (label != filter_label)) + { + continue; // filter out object classes if filter is specified + } if (alignment_to_square) { parser.adjustBoundingBox(rc); } - const auto clipped_rc = rc & parser.getSurface(); if (filter_out_of_bounds) { @@ -243,6 +215,7 @@ void parseSSD(const cv::Mat& in_ssd_result, } } out_boxes.emplace_back(clipped_rc); + out_labels.emplace_back(label); } } diff --git a/modules/gapi/src/backends/cpu/gnnparsers.hpp b/modules/gapi/src/backends/cpu/gnnparsers.hpp index 2ae6693705..28b6128f0c 100644 --- a/modules/gapi/src/backends/cpu/gnnparsers.hpp +++ b/modules/gapi/src/backends/cpu/gnnparsers.hpp @@ -11,19 +11,14 @@ namespace cv { -void parseSSDBL(const cv::Mat& in_ssd_result, - const cv::Size& in_size, - const float confidence_threshold, - const int filter_label, - std::vector& out_boxes, - std::vector& out_labels); - -void parseSSD(const cv::Mat& in_ssd_result, +void ParseSSD(const cv::Mat& in_ssd_result, const cv::Size& in_size, const float confidence_threshold, + const int filter_label, const bool alignment_to_square, const bool filter_out_of_bounds, - std::vector& out_boxes); + std::vector& out_boxes, + std::vector& out_labels); void parseYolo(const cv::Mat& in_yolo_result, const cv::Size& in_size, From 415668ecf04561959aca6a020e4f5c1390120225 Mon Sep 17 00:00:00 2001 From: Dmitry Matveev Date: Mon, 31 May 2021 22:02:05 +0300 Subject: [PATCH 007/262] G-API: Documentation updates 1) Document GFrame/MediaFrame (and also other G-API types) - Added doxygen comments for GMat, GScalar, GArray, GOpaque classes; - Documented GFrame and its host-side counterpart MediaFrame; - Added some more notes to the data type classes. 2) Give @brief descriptions to most of the cv::gapi::* namespaces 3) Make some symbols private - These structures are mainly internal and shouldn't be used directly --- doc/Doxyfile.in | 2 +- modules/gapi/include/opencv2/gapi/core.hpp | 4 + .../include/opencv2/gapi/cpu/gcpukernel.hpp | 6 +- .../opencv2/gapi/fluid/gfluidkernel.hpp | 3 + modules/gapi/include/opencv2/gapi/garray.hpp | 66 +++++++- .../include/opencv2/gapi/gasync_context.hpp | 7 + modules/gapi/include/opencv2/gapi/gframe.hpp | 48 +++++- modules/gapi/include/opencv2/gapi/gkernel.hpp | 2 + modules/gapi/include/opencv2/gapi/gmat.hpp | 38 ++++- modules/gapi/include/opencv2/gapi/gopaque.hpp | 27 ++- modules/gapi/include/opencv2/gapi/gscalar.hpp | 73 +++++++- .../gapi/include/opencv2/gapi/gstreaming.hpp | 8 + modules/gapi/include/opencv2/gapi/imgproc.hpp | 4 + modules/gapi/include/opencv2/gapi/infer.hpp | 3 +- .../gapi/include/opencv2/gapi/infer/ie.hpp | 5 + .../gapi/include/opencv2/gapi/infer/onnx.hpp | 4 + modules/gapi/include/opencv2/gapi/media.hpp | 156 ++++++++++++++++-- .../include/opencv2/gapi/ocl/goclkernel.hpp | 3 + .../gapi/include/opencv2/gapi/own/types.hpp | 5 + .../include/opencv2/gapi/plaidml/plaidml.hpp | 5 + .../include/opencv2/gapi/python/python.hpp | 9 + .../include/opencv2/gapi/render/render.hpp | 4 + modules/gapi/include/opencv2/gapi/rmat.hpp | 4 + .../gapi/include/opencv2/gapi/s11n/base.hpp | 5 + modules/gapi/include/opencv2/gapi/stereo.hpp | 5 + modules/gapi/include/opencv2/gapi/video.hpp | 4 + modules/gapi/samples/api_ref_snippets.cpp | 12 +- modules/gapi/samples/gaze_estimation.cpp | 10 +- 28 files changed, 485 insertions(+), 37 deletions(-) diff --git a/doc/Doxyfile.in b/doc/Doxyfile.in index 26ad42b1e5..ce207d3e31 100644 --- a/doc/Doxyfile.in +++ b/doc/Doxyfile.in @@ -106,7 +106,7 @@ RECURSIVE = YES EXCLUDE = @CMAKE_DOXYGEN_EXCLUDE_LIST@ EXCLUDE_SYMLINKS = NO EXCLUDE_PATTERNS = *.inl.hpp *.impl.hpp *_detail.hpp */cudev/**/detail/*.hpp *.m */opencl/runtime/* */legacy/* *_c.h @DOXYGEN_EXCLUDE_PATTERNS@ -EXCLUDE_SYMBOLS = cv::DataType<*> cv::traits::* int void CV__* T __CV* +EXCLUDE_SYMBOLS = cv::DataType<*> cv::traits::* int void CV__* T __CV* cv::gapi::detail* EXAMPLE_PATH = @CMAKE_DOXYGEN_EXAMPLE_PATH@ EXAMPLE_PATTERNS = * EXAMPLE_RECURSIVE = YES diff --git a/modules/gapi/include/opencv2/gapi/core.hpp b/modules/gapi/include/opencv2/gapi/core.hpp index cb8a6127d7..b7f65b6719 100644 --- a/modules/gapi/include/opencv2/gapi/core.hpp +++ b/modules/gapi/include/opencv2/gapi/core.hpp @@ -29,6 +29,10 @@ */ namespace cv { namespace gapi { +/** + * @brief This namespace contains G-API Operation Types for OpenCV + * Core module functionality. + */ namespace core { using GMat2 = std::tuple; using GMat3 = std::tuple; // FIXME: how to avoid this? diff --git a/modules/gapi/include/opencv2/gapi/cpu/gcpukernel.hpp b/modules/gapi/include/opencv2/gapi/cpu/gcpukernel.hpp index 5539e244ba..009c19688b 100644 --- a/modules/gapi/include/opencv2/gapi/cpu/gcpukernel.hpp +++ b/modules/gapi/include/opencv2/gapi/cpu/gcpukernel.hpp @@ -40,6 +40,10 @@ namespace gimpl namespace gapi { +/** + * @brief This namespace contains G-API CPU backend functions, + * structures, and symbols. + */ namespace cpu { /** @@ -492,7 +496,7 @@ public: #define GAPI_OCV_KERNEL_ST(Name, API, State) \ struct Name: public cv::GCPUStKernelImpl \ - +/// @private class gapi::cpu::GOCVFunctor : public gapi::GFunctor { public: diff --git a/modules/gapi/include/opencv2/gapi/fluid/gfluidkernel.hpp b/modules/gapi/include/opencv2/gapi/fluid/gfluidkernel.hpp index dbb36d801d..53b68c4e21 100644 --- a/modules/gapi/include/opencv2/gapi/fluid/gfluidkernel.hpp +++ b/modules/gapi/include/opencv2/gapi/fluid/gfluidkernel.hpp @@ -25,6 +25,9 @@ namespace cv { namespace gapi { +/** + * @brief This namespace contains G-API Fluid backend functions, structures, and symbols. + */ namespace fluid { /** diff --git a/modules/gapi/include/opencv2/gapi/garray.hpp b/modules/gapi/include/opencv2/gapi/garray.hpp index 32799bc07e..d7b365f7f9 100644 --- a/modules/gapi/include/opencv2/gapi/garray.hpp +++ b/modules/gapi/include/opencv2/gapi/garray.hpp @@ -340,21 +340,79 @@ namespace detail /** \addtogroup gapi_data_objects * @{ */ - +/** + * @brief `cv::GArray` template class represents a list of objects + * of class `T` in the graph. + * + * `cv::GArray` describes a functional relationship between + * operations consuming and producing arrays of objects of class + * `T`. The primary purpose of `cv::GArray` is to represent a + * dynamic list of objects -- where the size of the list is not known + * at the graph construction or compile time. Examples include: corner + * and feature detectors (`cv::GArray`), object detection + * and tracking results (`cv::GArray`). Programmers can use + * their own types with `cv::GArray` in the custom operations. + * + * Similar to `cv::GScalar`, `cv::GArray` may be value-initialized + * -- in this case a graph-constant value is associated with the object. + * + * `GArray` is a virtual counterpart of `std::vector`, which is + * usually used to represent the `GArray` data in G-API during the + * execution. + * + * @sa `cv::GOpaque` + */ template class GArray { public: // Host type (or Flat type) - the type this GArray is actually // specified to. + /// @private using HT = typename detail::flatten_g::type>::type; + /** + * @brief Constructs a value-initialized `cv::GArray` + * + * `cv::GArray` objects may have their values + * be associated at graph construction time. It is useful when + * some operation has a `cv::GArray` input which doesn't change during + * the program execution, and is set only once. In this case, + * there is no need to declare such `cv::GArray` as a graph input. + * + * @note The value of `cv::GArray` may be overwritten by assigning some + * other `cv::GArray` to the object using `operator=` -- on the + * assigment, the old association or value is discarded. + * + * @param v a std::vector to associate with this + * `cv::GArray` object. Vector data is copied into the + * `cv::GArray` (no reference to the passed data is held). + */ explicit GArray(const std::vector& v) // Constant value constructor : m_ref(detail::GArrayU(detail::VectorRef(v))) { putDetails(); } + + /** + * @overload + * @brief Constructs a value-initialized `cv::GArray` + * + * @param v a std::vector to associate with this + * `cv::GArray` object. Vector data is moved into the `cv::GArray`. + */ explicit GArray(std::vector&& v) // Move-constructor : m_ref(detail::GArrayU(detail::VectorRef(std::move(v)))) { putDetails(); } - GArray() { putDetails(); } // Empty constructor - explicit GArray(detail::GArrayU &&ref) // GArrayU-based constructor - : m_ref(ref) { putDetails(); } // (used by GCall, not for users) + + /** + * @brief Constructs an empty `cv::GArray` + * + * Normally, empty G-API data objects denote a starting point of + * the graph. When an empty `cv::GArray` is assigned to a result + * of some operation, it obtains a functional link to this + * operation (and is not empty anymore). + */ + GArray() { putDetails(); } // Empty constructor + + /// @private + explicit GArray(detail::GArrayU &&ref) // GArrayU-based constructor + : m_ref(ref) { putDetails(); } // (used by GCall, not for users) /// @private detail::GArrayU strip() const { diff --git a/modules/gapi/include/opencv2/gapi/gasync_context.hpp b/modules/gapi/include/opencv2/gapi/gasync_context.hpp index 69ce530fc9..f49b59822d 100644 --- a/modules/gapi/include/opencv2/gapi/gasync_context.hpp +++ b/modules/gapi/include/opencv2/gapi/gasync_context.hpp @@ -17,6 +17,13 @@ namespace cv { namespace gapi{ + +/** + * @brief This namespace contains experimental G-API functionality, + * functions or structures in this namespace are subjects to change or + * removal in the future releases. This namespace also contains + * functions which API is not stabilized yet. + */ namespace wip { /** diff --git a/modules/gapi/include/opencv2/gapi/gframe.hpp b/modules/gapi/include/opencv2/gapi/gframe.hpp index 13fd5d6d29..96913dc4cc 100644 --- a/modules/gapi/include/opencv2/gapi/gframe.hpp +++ b/modules/gapi/include/opencv2/gapi/gframe.hpp @@ -28,14 +28,54 @@ struct GOrigin; /** \addtogroup gapi_data_objects * @{ */ +/** + * @brief GFrame class represents an image or media frame in the graph. + * + * GFrame doesn't store any data itself, instead it describes a + * functional relationship between operations consuming and producing + * GFrame objects. + * + * GFrame is introduced to handle various media formats (e.g., NV12 or + * I420) under the same type. Various image formats may differ in the + * number of planes (e.g. two for NV12, three for I420) and the pixel + * layout inside. GFrame type allows to handle these media formats in + * the graph uniformly -- the graph structure will not change if the + * media format changes, e.g. a different camera or decoder is used + * with the same graph. G-API provides a number of operations which + * operate directly on GFrame, like `infer<>()` or + * renderFrame(); these operations are expected to handle different + * media formats inside. There is also a number of accessor + * operations like BGR(), Y(), UV() -- these operations provide + * access to frame's data in the familiar cv::GMat form, which can be + * used with the majority of the existing G-API operations. These + * accessor functions may perform color space converion on the fly if + * the image format of the GFrame they are applied to differs from the + * operation's semantic (e.g. the BGR() accessor is called on an NV12 + * image frame). + * + * GFrame is a virtual counterpart of cv::MediaFrame. + * + * @sa cv::MediaFrame, cv::GFrameDesc, BGR(), Y(), UV(), infer<>(). + */ class GAPI_EXPORTS_W_SIMPLE GFrame { public: - GAPI_WRAP GFrame(); // Empty constructor - GFrame(const GNode &n, std::size_t out); // Operation result constructor + /** + * @brief Constructs an empty GFrame + * + * Normally, empty G-API data objects denote a starting point of + * the graph. When an empty GFrame is assigned to a result of some + * operation, it obtains a functional link to this operation (and + * is not empty anymore). + */ + GAPI_WRAP GFrame(); // Empty constructor - GOrigin& priv(); // Internal use only - const GOrigin& priv() const; // Internal use only + /// @private + GFrame(const GNode &n, std::size_t out); // Operation result constructor + /// @private + GOrigin& priv(); // Internal use only + /// @private + const GOrigin& priv() const; // Internal use only private: std::shared_ptr m_priv; diff --git a/modules/gapi/include/opencv2/gapi/gkernel.hpp b/modules/gapi/include/opencv2/gapi/gkernel.hpp index f70e50253d..cfac552bfe 100644 --- a/modules/gapi/include/opencv2/gapi/gkernel.hpp +++ b/modules/gapi/include/opencv2/gapi/gkernel.hpp @@ -372,6 +372,7 @@ namespace gapi { // Prework: model "Device" API before it gets to G-API headers. // FIXME: Don't mix with internal Backends class! + /// @private class GAPI_EXPORTS GBackend { public: @@ -412,6 +413,7 @@ namespace std namespace cv { namespace gapi { + /// @private class GFunctor { public: diff --git a/modules/gapi/include/opencv2/gapi/gmat.hpp b/modules/gapi/include/opencv2/gapi/gmat.hpp index 11c3071f03..7bea97bbc5 100644 --- a/modules/gapi/include/opencv2/gapi/gmat.hpp +++ b/modules/gapi/include/opencv2/gapi/gmat.hpp @@ -30,29 +30,57 @@ struct GOrigin; * @brief G-API data objects used to build G-API expressions. * * These objects do not own any particular data (except compile-time - * associated values like with cv::GScalar) and are used to construct - * graphs. + * associated values like with cv::GScalar or `cv::GArray`) and are + * used only to construct graphs. * * Every graph in G-API starts and ends with data objects. * * Once constructed and compiled, G-API operates with regular host-side * data instead. Refer to the below table to find the mapping between - * G-API and regular data types. + * G-API and regular data types when passing input and output data + * structures to G-API: * * G-API data type | I/O data type * ------------------ | ------------- - * cv::GMat | cv::Mat + * cv::GMat | cv::Mat, cv::UMat, cv::RMat * cv::GScalar | cv::Scalar * `cv::GArray` | std::vector * `cv::GOpaque` | T + * cv::GFrame | cv::MediaFrame + */ +/** + * @brief GMat class represents image or tensor data in the + * graph. + * + * GMat doesn't store any data itself, instead it describes a + * functional relationship between operations consuming and producing + * GMat objects. + * + * GMat is a virtual counterpart of Mat and UMat, but it + * doesn't mean G-API use Mat or UMat objects internally to represent + * GMat objects -- the internal data representation may be + * backend-specific or optimized out at all. + * + * @sa Mat, GMatDesc */ class GAPI_EXPORTS_W_SIMPLE GMat { public: + /** + * @brief Constructs an empty GMat + * + * Normally, empty G-API data objects denote a starting point of + * the graph. When an empty GMat is assigned to a result of some + * operation, it obtains a functional link to this operation (and + * is not empty anymore). + */ GAPI_WRAP GMat(); // Empty constructor - GMat(const GNode &n, std::size_t out); // Operation result constructor + /// @private + GMat(const GNode &n, std::size_t out); // Operation result constructor + /// @private GOrigin& priv(); // Internal use only + /// @private const GOrigin& priv() const; // Internal use only private: diff --git a/modules/gapi/include/opencv2/gapi/gopaque.hpp b/modules/gapi/include/opencv2/gapi/gopaque.hpp index 00f0718422..979a9db8c4 100644 --- a/modules/gapi/include/opencv2/gapi/gopaque.hpp +++ b/modules/gapi/include/opencv2/gapi/gopaque.hpp @@ -307,15 +307,40 @@ namespace detail /** \addtogroup gapi_data_objects * @{ */ - +/** + * @brief `cv::GOpaque` template class represents an object of + * class `T` in the graph. + * + * `cv::GOpaque` describes a functional relationship between operations + * consuming and producing object of class `T`. `cv::GOpaque` is + * designed to extend G-API with user-defined data types, which are + * often required with user-defined operations. G-API can't apply any + * optimizations to user-defined types since these types are opaque to + * the framework. However, there is a number of G-API operations + * declared with `cv::GOpaque` as a return type, + * e.g. cv::gapi::streaming::timestamp() or cv::gapi::streaming::size(). + * + * @sa `cv::GArray` + */ template class GOpaque { public: // Host type (or Flat type) - the type this GOpaque is actually // specified to. + /// @private using HT = typename detail::flatten_g>::type; + /** + * @brief Constructs an empty `cv::GOpaque` + * + * Normally, empty G-API data objects denote a starting point of + * the graph. When an empty `cv::GOpaque` is assigned to a result + * of some operation, it obtains a functional link to this + * operation (and is not empty anymore). + */ GOpaque() { putDetails(); } // Empty constructor + + /// @private explicit GOpaque(detail::GOpaqueU &&ref) // GOpaqueU-based constructor : m_ref(ref) { putDetails(); } // (used by GCall, not for users) diff --git a/modules/gapi/include/opencv2/gapi/gscalar.hpp b/modules/gapi/include/opencv2/gapi/gscalar.hpp index d4af2cab5d..c6edc33b75 100644 --- a/modules/gapi/include/opencv2/gapi/gscalar.hpp +++ b/modules/gapi/include/opencv2/gapi/gscalar.hpp @@ -25,18 +25,83 @@ struct GOrigin; /** \addtogroup gapi_data_objects * @{ */ - +/** + * @brief GScalar class represents cv::Scalar data in the graph. + * + * GScalar may be associated with a cv::Scalar value, which becomes + * its constant value bound in graph compile time. cv::GScalar describes a + * functional relationship between operations consuming and producing + * GScalar objects. + * + * GScalar is a virtual counterpart of cv::Scalar, which is usually used + * to represent the GScalar data in G-API during the execution. + * + * @sa Scalar + */ class GAPI_EXPORTS_W_SIMPLE GScalar { public: - GAPI_WRAP GScalar(); // Empty constructor - explicit GScalar(const cv::Scalar& s); // Constant value constructor from cv::Scalar + /** + * @brief Constructs an empty GScalar + * + * Normally, empty G-API data objects denote a starting point of + * the graph. When an empty GScalar is assigned to a result of some + * operation, it obtains a functional link to this operation (and + * is not empty anymore). + */ + GAPI_WRAP GScalar(); + + /** + * @brief Constructs a value-initialized GScalar + * + * In contrast with GMat (which can be either an explicit graph input + * or a result of some operation), GScalars may have their values + * be associated at graph construction time. It is useful when + * some operation has a GScalar input which doesn't change during + * the program execution, and is set only once. In this case, + * there is no need to declare such GScalar as a graph input. + * + * @note The value of GScalar may be overwritten by assigning some + * other GScalar to the object using `operator=` -- on the + * assigment, the old GScalar value is discarded. + * + * @param s a cv::Scalar value to associate with this GScalar object. + */ + explicit GScalar(const cv::Scalar& s); + + /** + * @overload + * @brief Constructs a value-initialized GScalar + * + * @param s a cv::Scalar value to associate with this GScalar object. + */ explicit GScalar(cv::Scalar&& s); // Constant value move-constructor from cv::Scalar + /** + * @overload + * @brief Constructs a value-initialized GScalar + * + * @param v0 A `double` value to associate with this GScalar. Note + * that only the first component of a four-component cv::Scalar is + * set to this value, with others remain zeros. + * + * This constructor overload is not marked `explicit` and can be + * used in G-API expression code like this: + * + * @snippet modules/gapi/samples/api_ref_snippets.cpp gscalar_implicit + * + * Here operator+(GMat,GScalar) is used to wrap cv::gapi::addC() + * and a value-initialized GScalar is created on the fly. + * + * @overload + */ GScalar(double v0); // Constant value constructor from double - GScalar(const GNode &n, std::size_t out); // Operation result constructor + /// @private + GScalar(const GNode &n, std::size_t out); // Operation result constructor + /// @private GOrigin& priv(); // Internal use only + /// @private const GOrigin& priv() const; // Internal use only private: diff --git a/modules/gapi/include/opencv2/gapi/gstreaming.hpp b/modules/gapi/include/opencv2/gapi/gstreaming.hpp index 371581345f..5bbed5e12d 100644 --- a/modules/gapi/include/opencv2/gapi/gstreaming.hpp +++ b/modules/gapi/include/opencv2/gapi/gstreaming.hpp @@ -372,6 +372,14 @@ protected: /** @} */ namespace gapi { + +/** + * @brief This namespace contains G-API functions, structures, and + * symbols related to the Streaming execution mode. + * + * Some of the operations defined in this namespace (e.g. size(), + * BGR(), etc.) can be used in the traditional execution mode too. + */ namespace streaming { /** * @brief Specify queue capacity for streaming execution. diff --git a/modules/gapi/include/opencv2/gapi/imgproc.hpp b/modules/gapi/include/opencv2/gapi/imgproc.hpp index 2dbe626ff1..5c4c6f7031 100644 --- a/modules/gapi/include/opencv2/gapi/imgproc.hpp +++ b/modules/gapi/include/opencv2/gapi/imgproc.hpp @@ -47,6 +47,10 @@ void validateFindingContoursMeta(const int depth, const int chan, const int mode namespace cv { namespace gapi { +/** + * @brief This namespace contains G-API Operation Types for OpenCV + * ImgProc module functionality. + */ namespace imgproc { using GMat2 = std::tuple; using GMat3 = std::tuple; // FIXME: how to avoid this? diff --git a/modules/gapi/include/opencv2/gapi/infer.hpp b/modules/gapi/include/opencv2/gapi/infer.hpp index eb9b9768c9..93701856bb 100644 --- a/modules/gapi/include/opencv2/gapi/infer.hpp +++ b/modules/gapi/include/opencv2/gapi/infer.hpp @@ -653,6 +653,7 @@ namespace gapi { // A type-erased form of network parameters. // Similar to how a type-erased GKernel is represented and used. +/// @private struct GAPI_EXPORTS GNetParam { std::string tag; // FIXME: const? GBackend backend; // Specifies the execution model @@ -664,7 +665,7 @@ struct GAPI_EXPORTS GNetParam { */ /** * @brief A container class for network configurations. Similar to - * GKernelPackage.Use cv::gapi::networks() to construct this object. + * GKernelPackage. Use cv::gapi::networks() to construct this object. * * @sa cv::gapi::networks */ diff --git a/modules/gapi/include/opencv2/gapi/infer/ie.hpp b/modules/gapi/include/opencv2/gapi/infer/ie.hpp index 5b614de363..610a2061af 100644 --- a/modules/gapi/include/opencv2/gapi/infer/ie.hpp +++ b/modules/gapi/include/opencv2/gapi/infer/ie.hpp @@ -24,6 +24,11 @@ namespace cv { namespace gapi { // FIXME: introduce a new sub-namespace for NN? + +/** + * @brief This namespace contains G-API OpenVINO backend functions, + * structures, and symbols. + */ namespace ie { GAPI_EXPORTS cv::gapi::GBackend backend(); diff --git a/modules/gapi/include/opencv2/gapi/infer/onnx.hpp b/modules/gapi/include/opencv2/gapi/infer/onnx.hpp index 3a4e35fb09..cfef0cc758 100644 --- a/modules/gapi/include/opencv2/gapi/infer/onnx.hpp +++ b/modules/gapi/include/opencv2/gapi/infer/onnx.hpp @@ -20,6 +20,10 @@ namespace cv { namespace gapi { + +/** + * @brief This namespace contains G-API ONNX Runtime backend functions, structures, and symbols. + */ namespace onnx { GAPI_EXPORTS cv::gapi::GBackend backend(); diff --git a/modules/gapi/include/opencv2/gapi/media.hpp b/modules/gapi/include/opencv2/gapi/media.hpp index 61a7098b55..aa7d6d6a1f 100644 --- a/modules/gapi/include/opencv2/gapi/media.hpp +++ b/modules/gapi/include/opencv2/gapi/media.hpp @@ -17,28 +17,107 @@ namespace cv { +/** \addtogroup gapi_data_structures + * @{ + * + * @brief Extra G-API data structures used to pass input/output data + * to the graph for processing. + */ +/** + * @brief cv::MediaFrame class represents an image/media frame + * obtained from an external source. + * + * cv::MediaFrame represents image data as specified in + * cv::MediaFormat. cv::MediaFrame is designed to be a thin wrapper over some + * external memory of buffer; the class itself provides an uniform + * interface over such types of memory. cv::MediaFrame wraps data from + * a camera driver or from a media codec and provides an abstraction + * layer over this memory to G-API. MediaFrame defines a compact interface + * to access and manage the underlying data; the implementation is + * fully defined by the associated Adapter (which is usually + * user-defined). + * + * @sa cv::RMat + */ class GAPI_EXPORTS MediaFrame { public: - enum class Access { R, W }; + /// This enum defines different types of cv::MediaFrame provided + /// access to the underlying data. Note that different flags can't + /// be combined in this version. + enum class Access { + R, ///< Access data for reading + W, ///< Access data for writing + }; class IAdapter; class View; using AdapterPtr = std::unique_ptr; + /** + * @brief Constructs an empty MediaFrame + * + * The constructed object has no any data associated with it. + */ MediaFrame(); - explicit MediaFrame(AdapterPtr &&); - template static cv::MediaFrame Create(Args&&...); - View access(Access) const; + /** + * @brief Constructs a MediaFrame with the given + * Adapter. MediaFrame takes ownership over the passed adapter. + * + * @param p an unique pointer to instance of IAdapter derived class. + */ + explicit MediaFrame(AdapterPtr &&p); + + /** + * @overload + * @brief Constructs a MediaFrame with the given parameters for + * the Adapter. The adapter of type `T` is costructed on the fly. + * + * @param args list of arguments to construct an adapter of type + * `T`. + */ + template static cv::MediaFrame Create(Args&&... args); + + /** + * @brief Obtain access to the underlying data with the given + * mode. + * + * Depending on the associated Adapter and the data wrapped, this + * method may be cheap (e.g., the underlying memory is local) or + * costly (if the underlying memory is external or device + * memory). + * + * @param mode an access mode flag + * @return a MediaFrame::View object. The views should be handled + * carefully, refer to the MediaFrame::View documentation for details. + */ + View access(Access mode) const; + + /** + * @brief Returns a media frame descriptor -- the information + * about the media format, dimensions, etc. + * @return a cv::GFrameDesc + */ cv::GFrameDesc desc() const; // FIXME: design a better solution // Should be used only if the actual adapter provides implementation + /// @private -- exclude from the OpenCV documentation for now. cv::util::any blobParams() const; - // Cast underlying MediaFrame adapter to the particular adapter type, - // return nullptr if underlying type is different - template T* get() const - { + /** + * @brief Casts and returns the associated MediaFrame adapter to + * the particular adapter type `T`, returns nullptr if the type is + * different. + * + * This method may be useful if the adapter type is known by the + * caller, and some lower level access to the memory is required. + * Depending on the memory type, it may be more efficient than + * access(). + * + * @return a pointer to the adapter object, nullptr if the adapter + * type is different. + */ + template T* get() const { static_assert(std::is_base_of::value, "T is not derived from cv::MediaFrame::IAdapter!"); auto* adapter = getAdapter(); @@ -58,6 +137,43 @@ inline cv::MediaFrame cv::MediaFrame::Create(Args&&... args) { return cv::MediaFrame(std::move(ptr)); } +/** + * @brief Provides access to the MediaFrame's underlying data. + * + * This object contains the necessary information to access the pixel + * data of the associated MediaFrame: arrays of pointers and strides + * (distance between every plane row, in bytes) for every image + * plane, as defined in cv::MediaFormat. + * There may be up to four image planes in MediaFrame. + * + * Depending on the MediaFrame::Access flag passed in + * MediaFrame::access(), a MediaFrame::View may be read- or + * write-only. + * + * Depending on the MediaFrame::IAdapter implementation associated + * with the parent MediaFrame, writing to memory with + * MediaFrame::Access::R flag may have no effect or lead to + * undefined behavior. Same applies to reading the memory with + * MediaFrame::Access::W flag -- again, depending on the IAdapter + * implementation, the host-side buffer the view provides access to + * may have no current data stored in (so in-place editing of the + * buffer contents may not be possible). + * + * MediaFrame::View objects must be handled carefully, as an external + * resource associated with MediaFrame may be locked for the time the + * MediaFrame::View object exists. Obtaining MediaFrame::View should + * be seen as "map" and destroying it as "unmap" in the "map/unmap" + * idiom (applicable to OpenCL, device memory, remote + * memory). + * + * When a MediaFrame buffer is accessed for writing, and the memory + * under MediaFrame::View::Ptrs is altered, the data synchronization + * of a host-side and device/remote buffer is not guaranteed until the + * MediaFrame::View is destroyed. In other words, the real data on the + * device or in a remote target may be updated at the MediaFrame::View + * destruction only -- but it depends on the associated + * MediaFrame::IAdapter implementation. + */ class GAPI_EXPORTS MediaFrame::View final { public: static constexpr const size_t MAX_PLANES = 4; @@ -65,19 +181,38 @@ public: using Strides = std::array; // in bytes using Callback = std::function; + /// @private View(Ptrs&& ptrs, Strides&& strs, Callback &&cb = [](){}); + + /// @private View(const View&) = delete; + + /// @private View(View&&) = default; + + /// @private View& operator = (const View&) = delete; + ~View(); - Ptrs ptr; - Strides stride; + Ptrs ptr; ///< Array of image plane pointers + Strides stride; ///< Array of image plane strides, in bytes. private: Callback m_cb; }; +/** + * @brief An interface class for MediaFrame data adapters. + * + * Implement this interface to wrap media data in the MediaFrame. It + * makes sense to implement this class if there is a custom + * cv::gapi::wip::IStreamSource defined -- in this case, a stream + * source can produce MediaFrame objects with this adapter and the + * media data may be passed to graph without any copy. For example, a + * GStreamer-based stream source can implement an adapter over + * `GstBuffer` and G-API will transparently use it in the graph. + */ class GAPI_EXPORTS MediaFrame::IAdapter { public: virtual ~IAdapter() = 0; @@ -87,6 +222,7 @@ public: // The default implementation does nothing virtual cv::util::any blobParams() const; }; +/** @} */ } //namespace cv diff --git a/modules/gapi/include/opencv2/gapi/ocl/goclkernel.hpp b/modules/gapi/include/opencv2/gapi/ocl/goclkernel.hpp index 8ec388d588..6a2f1df769 100644 --- a/modules/gapi/include/opencv2/gapi/ocl/goclkernel.hpp +++ b/modules/gapi/include/opencv2/gapi/ocl/goclkernel.hpp @@ -29,6 +29,9 @@ namespace gimpl namespace gapi { +/** + * @brief This namespace contains G-API OpenCL backend functions, structures, and symbols. + */ namespace ocl { /** diff --git a/modules/gapi/include/opencv2/gapi/own/types.hpp b/modules/gapi/include/opencv2/gapi/own/types.hpp index c77a62ca53..53bb867e32 100644 --- a/modules/gapi/include/opencv2/gapi/own/types.hpp +++ b/modules/gapi/include/opencv2/gapi/own/types.hpp @@ -15,6 +15,11 @@ namespace cv { namespace gapi { + +/** + * @brief This namespace contains G-API own data structures used in + * its standalone mode build. + */ namespace own { diff --git a/modules/gapi/include/opencv2/gapi/plaidml/plaidml.hpp b/modules/gapi/include/opencv2/gapi/plaidml/plaidml.hpp index bd12d25832..3207a8cb2e 100644 --- a/modules/gapi/include/opencv2/gapi/plaidml/plaidml.hpp +++ b/modules/gapi/include/opencv2/gapi/plaidml/plaidml.hpp @@ -15,6 +15,11 @@ namespace cv { namespace gapi { + +/** + * @brief This namespace contains G-API PlaidML backend functions, + * structures, and symbols. + */ namespace plaidml { diff --git a/modules/gapi/include/opencv2/gapi/python/python.hpp b/modules/gapi/include/opencv2/gapi/python/python.hpp index 1c85d69d9f..0e20bbbb59 100644 --- a/modules/gapi/include/opencv2/gapi/python/python.hpp +++ b/modules/gapi/include/opencv2/gapi/python/python.hpp @@ -13,6 +13,15 @@ namespace cv { namespace gapi { + +/** + * @brief This namespace contains G-API Python backend functions, + * structures, and symbols. + * + * This functionality is required to enable G-API custom operations + * and kernels when using G-API from Python, no need to use it in the + * C++ form. + */ namespace python { GAPI_EXPORTS cv::gapi::GBackend backend(); diff --git a/modules/gapi/include/opencv2/gapi/render/render.hpp b/modules/gapi/include/opencv2/gapi/render/render.hpp index a84c26c810..6bfe92388a 100644 --- a/modules/gapi/include/opencv2/gapi/render/render.hpp +++ b/modules/gapi/include/opencv2/gapi/render/render.hpp @@ -169,6 +169,10 @@ GAPI_EXPORTS GFrame renderFrame(const GFrame& m_frame, } // namespace draw } // namespace wip +/** + * @brief This namespace contains G-API CPU rendering backend functions, + * structures, and symbols. See @ref gapi_draw for details. + */ namespace render { namespace ocv diff --git a/modules/gapi/include/opencv2/gapi/rmat.hpp b/modules/gapi/include/opencv2/gapi/rmat.hpp index f50bd08b65..cc27f48664 100644 --- a/modules/gapi/include/opencv2/gapi/rmat.hpp +++ b/modules/gapi/include/opencv2/gapi/rmat.hpp @@ -42,6 +42,9 @@ namespace cv { // performCalculations(in_view, out_view); // // data from out_view is transferred to the device when out_view is destroyed // } +/** \addtogroup gapi_data_structures + * @{ + */ class GAPI_EXPORTS RMat { public: @@ -146,6 +149,7 @@ private: template RMat make_rmat(Ts&&... args) { return { std::make_shared(std::forward(args)...) }; } +/** @} */ } //namespace cv diff --git a/modules/gapi/include/opencv2/gapi/s11n/base.hpp b/modules/gapi/include/opencv2/gapi/s11n/base.hpp index d9335ee9f7..b8ec8cfaff 100644 --- a/modules/gapi/include/opencv2/gapi/s11n/base.hpp +++ b/modules/gapi/include/opencv2/gapi/s11n/base.hpp @@ -12,6 +12,11 @@ namespace cv { namespace gapi { + +/** + * @brief This namespace contains G-API serialization and + * deserialization functions and data structures. + */ namespace s11n { struct IOStream; struct IIStream; diff --git a/modules/gapi/include/opencv2/gapi/stereo.hpp b/modules/gapi/include/opencv2/gapi/stereo.hpp index 908045d4c7..b573dbf0db 100644 --- a/modules/gapi/include/opencv2/gapi/stereo.hpp +++ b/modules/gapi/include/opencv2/gapi/stereo.hpp @@ -21,6 +21,11 @@ enum class StereoOutputFormat { DISPARITY_FIXED16_12_4 }; + +/** + * @brief This namespace contains G-API Operation Types for Stereo and + * related functionality. + */ namespace calib3d { G_TYPED_KERNEL(GStereo, , "org.opencv.stereo") { diff --git a/modules/gapi/include/opencv2/gapi/video.hpp b/modules/gapi/include/opencv2/gapi/video.hpp index 10965b0aa6..4dcc1d4182 100644 --- a/modules/gapi/include/opencv2/gapi/video.hpp +++ b/modules/gapi/include/opencv2/gapi/video.hpp @@ -42,6 +42,10 @@ struct GAPI_EXPORTS KalmanParams Mat controlMatrix; }; +/** + * @brief This namespace contains G-API Operations and functions for + * video-oriented algorithms, like optical flow and background subtraction. + */ namespace video { using GBuildPyrOutput = std::tuple, GScalar>; diff --git a/modules/gapi/samples/api_ref_snippets.cpp b/modules/gapi/samples/api_ref_snippets.cpp index 9c3e73a17c..6c660fb8fa 100644 --- a/modules/gapi/samples/api_ref_snippets.cpp +++ b/modules/gapi/samples/api_ref_snippets.cpp @@ -9,6 +9,14 @@ #include #include +static void gscalar_example() +{ + //! [gscalar_implicit] + cv::GMat a; + cv::GMat b = a + 1; + //! [gscalar_implicit] +} + static void typed_example() { const cv::Size sz(32, 32); @@ -116,7 +124,9 @@ int main(int argc, char *argv[]) >(); //! [kernels_snippet] - // Just call typed example with no input/output + // Just call typed example with no input/output - avoid warnings about + // unused functions typed_example(); + gscalar_example(); return 0; } diff --git a/modules/gapi/samples/gaze_estimation.cpp b/modules/gapi/samples/gaze_estimation.cpp index 6396d500c1..96b2ebf163 100644 --- a/modules/gapi/samples/gaze_estimation.cpp +++ b/modules/gapi/samples/gaze_estimation.cpp @@ -16,13 +16,13 @@ const std::string keys = "{ h help | | Print this help message }" "{ input | | Path to the input video file }" "{ facem | face-detection-retail-0005.xml | Path to OpenVINO face detection model (.xml) }" - "{ faced | CPU | Target device for the face detection (e.g. CPU, GPU, VPU, ...) }" + "{ faced | CPU | Target device for the face detection (e.g. CPU, GPU, ...) }" "{ landm | facial-landmarks-35-adas-0002.xml | Path to OpenVINO landmarks detector model (.xml) }" - "{ landd | CPU | Target device for the landmarks detector (e.g. CPU, GPU, VPU, ...) }" + "{ landd | CPU | Target device for the landmarks detector (e.g. CPU, GPU, ...) }" "{ headm | head-pose-estimation-adas-0001.xml | Path to OpenVINO head pose estimation model (.xml) }" - "{ headd | CPU | Target device for the head pose estimation inference (e.g. CPU, GPU, VPU, ...) }" + "{ headd | CPU | Target device for the head pose estimation inference (e.g. CPU, GPU, ...) }" "{ gazem | gaze-estimation-adas-0002.xml | Path to OpenVINO gaze vector estimaiton model (.xml) }" - "{ gazed | CPU | Target device for the gaze vector estimation inference (e.g. CPU, GPU, VPU, ...) }" + "{ gazed | CPU | Target device for the gaze vector estimation inference (e.g. CPU, GPU, ...) }" ; namespace { @@ -338,7 +338,7 @@ int main(int argc, char *argv[]) cv::GMat in; cv::GMat faces = cv::gapi::infer(in); - cv::GOpaque sz = custom::Size::on(in); // FIXME + cv::GOpaque sz = cv::gapi::streaming::size(in); cv::GArray faces_rc = custom::ParseSSD::on(faces, sz, true); cv::GArray angles_y, angles_p, angles_r; std::tie(angles_y, angles_p, angles_r) = cv::gapi::infer(faces_rc, in); From 7a276f39fbfe975b971cc5030077fe57faf85ca4 Mon Sep 17 00:00:00 2001 From: Smirnov Egor Date: Wed, 16 Jun 2021 11:36:13 +0300 Subject: [PATCH 008/262] reorder defined checks according to cmake file --- modules/highgui/src/window.cpp | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/modules/highgui/src/window.cpp b/modules/highgui/src/window.cpp index dca6ddbaf8..56c1456a5d 100644 --- a/modules/highgui/src/window.cpp +++ b/modules/highgui/src/window.cpp @@ -630,9 +630,8 @@ int cv::waitKey(int delay) return (code != -1) ? (code & 0xff) : -1; } -#if defined(HAVE_WIN32UI) -// pollKey() implemented in window_w32.cpp -#elif defined(HAVE_GTK) || defined(HAVE_COCOA) || defined(HAVE_QT) || (defined (WINRT) && !defined (WINRT_8_0)) +#if defined(HAVE_QT) || (defined (WINRT) && !defined (WINRT_8_0)) || \ + !defined(HAVE_WIN32UI) && (defined(HAVE_GTK) || defined(HAVE_COCOA)) // pollKey() fallback implementation int cv::pollKey() { @@ -650,6 +649,8 @@ int cv::pollKey() // fallback. please implement a proper polling function return cvWaitKey(1); } +#elif defined(HAVE_WIN32UI) +// pollKey() implemented in window_w32.cpp #endif int cv::createTrackbar(const String& trackbarName, const String& winName, From 464441d8c3943126a5238b34ee34633dcfbf399e Mon Sep 17 00:00:00 2001 From: Ian Maquignaz <9im14@queensu.ca> Date: Thu, 10 Jun 2021 20:13:06 -0400 Subject: [PATCH 009/262] Added new unit test for initInverseRectificationMap() Function is validated. Included an update to DISABLED_Calib3d_InitInverseRectificationMap. Includes updates per input from @alalek and unit test regression # to reflect PR # --- modules/calib3d/test/test_undistort.cpp | 82 +++++++++++++++++++++++-- 1 file changed, 78 insertions(+), 4 deletions(-) diff --git a/modules/calib3d/test/test_undistort.cpp b/modules/calib3d/test/test_undistort.cpp index 9663d36b78..ea1a952079 100644 --- a/modules/calib3d/test/test_undistort.cpp +++ b/modules/calib3d/test/test_undistort.cpp @@ -897,7 +897,7 @@ void CV_InitInverseRectificationMapTest::prepare_to_validation(int/* test_case_i Mat _new_cam0 = zero_new_cam ? test_mat[INPUT][0] : test_mat[INPUT][3]; Mat _mapx(img_size, CV_32F), _mapy(img_size, CV_32F); - double a[9], d[5]={0,0,0,0,0}, R[9]={1, 0, 0, 0, 1, 0, 0, 0, 1}, a1[9]; + double a[9], d[5]={0., 0., 0., 0. , 0.}, R[9]={1., 0., 0., 0., 1., 0., 0., 0., 1.}, a1[9]; Mat _a(3, 3, CV_64F, a), _a1(3, 3, CV_64F, a1); Mat _d(_d0.rows,_d0.cols, CV_MAKETYPE(CV_64F,_d0.channels()),d); Mat _R(3, 3, CV_64F, R); @@ -951,9 +951,9 @@ void CV_InitInverseRectificationMapTest::prepare_to_validation(int/* test_case_i // Undistort double x2 = x*x, y2 = y*y; double r2 = x2 + y2; - double cdist = 1./(1 + (d[0] + (d[1] + d[4]*r2)*r2)*r2); // (1 + (d[5] + (d[6] + d[7]*r2)*r2)*r2) == 1 as d[5-7]=0; - double x_ = x*cdist - d[2]*2*x*y + d[3]*(r2 + 2*x2); - double y_ = y*cdist - d[3]*2*x*y + d[2]*(r2 + 2*y2); + double cdist = 1./(1. + (d[0] + (d[1] + d[4]*r2)*r2)*r2); // (1. + (d[5] + (d[6] + d[7]*r2)*r2)*r2) == 1 as d[5-7]=0; + double x_ = (x - (d[2]*2.*x*y + d[3]*(r2 + 2.*x2)))*cdist; + double y_ = (y - (d[3]*2.*x*y + d[2]*(r2 + 2.*y2)))*cdist; // Rectify double X = R[0]*x_ + R[1]*y_ + R[2]; @@ -1807,4 +1807,78 @@ TEST(Calib3d_initUndistortRectifyMap, regression_14467) EXPECT_LE(cvtest::norm(dst, mesh_uv, NORM_INF), 1e-3); } +TEST(Calib3d_initInverseRectificationMap, regression_20165) +{ + Size size_w_h(1280, 800); + Mat dst(size_w_h, CV_32FC2); // Reference for validation + Mat mapxy; // Output of initInverseRectificationMap() + + // Camera Matrix + double k[9]={ + 1.5393951443032472e+03, 0., 6.7491727003047140e+02, + 0., 1.5400748240626747e+03, 5.1226968329123963e+02, + 0., 0., 1. + }; + Mat _K(3, 3, CV_64F, k); + + // Distortion + // double d[5]={0,0,0,0,0}; // Zero Distortion + double d[5]={ // Non-zero distortion + -3.4134571357400023e-03, 2.9733267766101856e-03, // K1, K2 + 3.6653586399031184e-03, -3.1960714017365702e-03, // P1, P2 + 0. // K3 + }; + Mat _d(1, 5, CV_64F, d); + + // Rotation + //double R[9]={1., 0., 0., 0., 1., 0., 0., 0., 1.}; // Identity transform (none) + double R[9]={ // Random transform + 9.6625486010428052e-01, 1.6055789378989216e-02, 2.5708706103628531e-01, + -8.0300261706161002e-03, 9.9944797497929860e-01, -3.2237617614807819e-02, + -2.5746274294459848e-01, 2.9085338870243265e-02, 9.6585039165403186e-01 + }; + Mat _R(3, 3, CV_64F, R); + + // --- Validation --- // + initInverseRectificationMap(_K, _d, _R, _K, size_w_h, CV_32FC2, mapxy, noArray()); + + // Copy camera matrix + double fx, fy, cx, cy, ifx, ify, cxn, cyn; + fx = k[0]; fy = k[4]; cx = k[2]; cy = k[5]; + + // Copy new camera matrix + ifx = k[0]; ify = k[4]; cxn = k[2]; cyn = k[5]; + + // Distort Points + for( int v = 0; v < size_w_h.height; v++ ) + { + for( int u = 0; u < size_w_h.width; u++ ) + { + // Convert from image to pin-hole coordinates + double x = (u - cx)/fx; + double y = (v - cy)/fy; + + // Undistort + double x2 = x*x, y2 = y*y; + double r2 = x2 + y2; + double cdist = 1./(1. + (d[0] + (d[1] + d[4]*r2)*r2)*r2); // (1. + (d[5] + (d[6] + d[7]*r2)*r2)*r2) == 1 as d[5-7]=0; + double x_ = (x - (d[2]*2.*x*y + d[3]*(r2 + 2.*x2)))*cdist; + double y_ = (y - (d[3]*2.*x*y + d[2]*(r2 + 2.*y2)))*cdist; + + // Rectify + double X = R[0]*x_ + R[1]*y_ + R[2]; + double Y = R[3]*x_ + R[4]*y_ + R[5]; + double Z = R[6]*x_ + R[7]*y_ + R[8]; + double x__ = X/Z; + double y__ = Y/Z; + + // Convert from pin-hole to image coordinates + dst.at(v, u) = Vec2f((float)(x__*ifx + cxn), (float)(y__*ify + cyn)); + } + } + + // Check Result + EXPECT_LE(cvtest::norm(dst, mapxy, NORM_INF), 2e-1); +} + }} // namespace From 9557b9f70f18785d44e3f21155f4f9f8859e7c0f Mon Sep 17 00:00:00 2001 From: Developer-Ecosystem-Engineering <65677710+Developer-Ecosystem-Engineering@users.noreply.github.com> Date: Thu, 17 Jun 2021 10:14:48 -0700 Subject: [PATCH 010/262] Improve SIFT for arm64/Apple silicon - Reduce branch density by collapsing compares. - Fix windows build errors - Use OpenCV universal intrinsics - Use v_check_any and v_signmask as requested --- modules/features2d/src/sift.simd.hpp | 195 ++++++++++++++++++++++++--- 1 file changed, 174 insertions(+), 21 deletions(-) diff --git a/modules/features2d/src/sift.simd.hpp b/modules/features2d/src/sift.simd.hpp index b5033459b9..60129b1535 100644 --- a/modules/features2d/src/sift.simd.hpp +++ b/modules/features2d/src/sift.simd.hpp @@ -450,31 +450,184 @@ public: const sift_wt* currptr = img.ptr(r); const sift_wt* prevptr = prev.ptr(r); const sift_wt* nextptr = next.ptr(r); + int c = SIFT_IMG_BORDER; - for( int c = SIFT_IMG_BORDER; c < cols-SIFT_IMG_BORDER; c++) +#if CV_SIMD && !(DoG_TYPE_SHORT) + const int vecsize = v_float32::nlanes; + for( ; c <= cols-SIFT_IMG_BORDER - vecsize; c += vecsize) + { + v_float32 val = vx_load(&currptr[c]); + v_float32 _00,_01,_02; + v_float32 _10, _12; + v_float32 _20,_21,_22; + + v_float32 vmin,vmax; + + + v_float32 cond = v_abs(val) > vx_setall_f32((float)threshold); + if (!v_check_any(cond)) + { + continue; + } + + _00 = vx_load(&currptr[c-step-1]); _01 = vx_load(&currptr[c-step]); _02 = vx_load(&currptr[c-step+1]); + _10 = vx_load(&currptr[c -1]); _12 = vx_load(&currptr[c +1]); + _20 = vx_load(&currptr[c+step-1]); _21 = vx_load(&currptr[c+step]); _22 = vx_load(&currptr[c+step+1]); + + vmax = v_max(v_max(v_max(_00,_01),v_max(_02,_10)),v_max(v_max(_12,_20),v_max(_21,_22))); + vmin = v_min(v_min(v_min(_00,_01),v_min(_02,_10)),v_min(v_min(_12,_20),v_min(_21,_22))); + + v_float32 condp = cond & (val > vx_setall_f32(0)) & (val >= vmax); + v_float32 condm = cond & (val < vx_setall_f32(0)) & (val <= vmin); + + cond = condp | condm; + if (!v_check_any(cond)) + { + continue; + } + + _00 = vx_load(&prevptr[c-step-1]); _01 = vx_load(&prevptr[c-step]); _02 = vx_load(&prevptr[c-step+1]); + _10 = vx_load(&prevptr[c -1]); _12 = vx_load(&prevptr[c +1]); + _20 = vx_load(&prevptr[c+step-1]); _21 = vx_load(&prevptr[c+step]); _22 = vx_load(&prevptr[c+step+1]); + + vmax = v_max(v_max(v_max(_00,_01),v_max(_02,_10)),v_max(v_max(_12,_20),v_max(_21,_22))); + vmin = v_min(v_min(v_min(_00,_01),v_min(_02,_10)),v_min(v_min(_12,_20),v_min(_21,_22))); + + condp &= (val >= vmax); + condm &= (val <= vmin); + + cond = condp | condm; + if (!v_check_any(cond)) + { + continue; + } + + v_float32 _11p = vx_load(&prevptr[c]); + v_float32 _11n = vx_load(&nextptr[c]); + + v_float32 max_middle = v_max(_11n,_11p); + v_float32 min_middle = v_min(_11n,_11p); + + _00 = vx_load(&nextptr[c-step-1]); _01 = vx_load(&nextptr[c-step]); _02 = vx_load(&nextptr[c-step+1]); + _10 = vx_load(&nextptr[c -1]); _12 = vx_load(&nextptr[c +1]); + _20 = vx_load(&nextptr[c+step-1]); _21 = vx_load(&nextptr[c+step]); _22 = vx_load(&nextptr[c+step+1]); + + vmax = v_max(v_max(v_max(_00,_01),v_max(_02,_10)),v_max(v_max(_12,_20),v_max(_21,_22))); + vmin = v_min(v_min(v_min(_00,_01),v_min(_02,_10)),v_min(v_min(_12,_20),v_min(_21,_22))); + + condp &= (val >= v_max(vmax,max_middle)); + condm &= (val <= v_min(vmin,min_middle)); + + cond = condp | condm; + if (!v_check_any(cond)) + { + continue; + } + + int mask = v_signmask(cond); + for (int k = 0; k 0 ? j - 1 : n - 1; + int r2 = j < n-1 ? j + 1 : 0; + + if( hist[j] > hist[l] && hist[j] > hist[r2] && hist[j] >= mag_thr ) + { + float bin = j + 0.5f * (hist[l]-hist[r2]) / (hist[l] - 2*hist[j] + hist[r2]); + bin = bin < 0 ? n + bin : bin >= n ? bin - n : bin; + kpt.angle = 360.f - (float)((360.f/n) * bin); + if(std::abs(kpt.angle - 360.f) < FLT_EPSILON) + kpt.angle = 0.f; + + kpts_.push_back(kpt); + } + } + } + } + +#endif //CV_SIMD && !(DoG_TYPE_SHORT) + + // vector loop reminder, better predictibility and less branch density + for( ; c < cols-SIFT_IMG_BORDER; c++) { sift_wt val = currptr[c]; + if (std::abs(val) <= threshold) + continue; - // find local extrema with pixel accuracy - if( std::abs(val) > threshold && - ((val > 0 && val >= currptr[c-1] && val >= currptr[c+1] && - val >= currptr[c-step-1] && val >= currptr[c-step] && val >= currptr[c-step+1] && - val >= currptr[c+step-1] && val >= currptr[c+step] && val >= currptr[c+step+1] && - val >= nextptr[c] && val >= nextptr[c-1] && val >= nextptr[c+1] && - val >= nextptr[c-step-1] && val >= nextptr[c-step] && val >= nextptr[c-step+1] && - val >= nextptr[c+step-1] && val >= nextptr[c+step] && val >= nextptr[c+step+1] && - val >= prevptr[c] && val >= prevptr[c-1] && val >= prevptr[c+1] && - val >= prevptr[c-step-1] && val >= prevptr[c-step] && val >= prevptr[c-step+1] && - val >= prevptr[c+step-1] && val >= prevptr[c+step] && val >= prevptr[c+step+1]) || - (val < 0 && val <= currptr[c-1] && val <= currptr[c+1] && - val <= currptr[c-step-1] && val <= currptr[c-step] && val <= currptr[c-step+1] && - val <= currptr[c+step-1] && val <= currptr[c+step] && val <= currptr[c+step+1] && - val <= nextptr[c] && val <= nextptr[c-1] && val <= nextptr[c+1] && - val <= nextptr[c-step-1] && val <= nextptr[c-step] && val <= nextptr[c-step+1] && - val <= nextptr[c+step-1] && val <= nextptr[c+step] && val <= nextptr[c+step+1] && - val <= prevptr[c] && val <= prevptr[c-1] && val <= prevptr[c+1] && - val <= prevptr[c-step-1] && val <= prevptr[c-step] && val <= prevptr[c-step+1] && - val <= prevptr[c+step-1] && val <= prevptr[c+step] && val <= prevptr[c+step+1]))) + sift_wt _00,_01,_02; + sift_wt _10, _12; + sift_wt _20,_21,_22; + _00 = currptr[c-step-1]; _01 = currptr[c-step]; _02 = currptr[c-step+1]; + _10 = currptr[c -1]; _12 = currptr[c +1]; + _20 = currptr[c+step-1]; _21 = currptr[c+step]; _22 = currptr[c+step+1]; + + bool calculate = false; + if (val > 0) + { + sift_wt vmax = std::max(std::max(std::max(_00,_01),std::max(_02,_10)),std::max(std::max(_12,_20),std::max(_21,_22))); + if (val >= vmax) + { + _00 = prevptr[c-step-1]; _01 = prevptr[c-step]; _02 = prevptr[c-step+1]; + _10 = prevptr[c -1]; _12 = prevptr[c +1]; + _20 = prevptr[c+step-1]; _21 = prevptr[c+step]; _22 = prevptr[c+step+1]; + vmax = std::max(std::max(std::max(_00,_01),std::max(_02,_10)),std::max(std::max(_12,_20),std::max(_21,_22))); + if (val >= vmax) + { + _00 = nextptr[c-step-1]; _01 = nextptr[c-step]; _02 = nextptr[c-step+1]; + _10 = nextptr[c -1]; _12 = nextptr[c +1]; + _20 = nextptr[c+step-1]; _21 = nextptr[c+step]; _22 = nextptr[c+step+1]; + vmax = std::max(std::max(std::max(_00,_01),std::max(_02,_10)),std::max(std::max(_12,_20),std::max(_21,_22))); + if (val >= vmax) + { + sift_wt _11p = prevptr[c], _11n = nextptr[c]; + calculate = (val >= std::max(_11p,_11n)); + } + } + } + + } else { // val cant be zero here (first abs took care of zero), must be negative + sift_wt vmin = std::min(std::min(std::min(_00,_01),std::min(_02,_10)),std::min(std::min(_12,_20),std::min(_21,_22))); + if (val <= vmin) + { + _00 = prevptr[c-step-1]; _01 = prevptr[c-step]; _02 = prevptr[c-step+1]; + _10 = prevptr[c -1]; _12 = prevptr[c +1]; + _20 = prevptr[c+step-1]; _21 = prevptr[c+step]; _22 = prevptr[c+step+1]; + vmin = std::min(std::min(std::min(_00,_01),std::min(_02,_10)),std::min(std::min(_12,_20),std::min(_21,_22))); + if (val <= vmin) + { + _00 = nextptr[c-step-1]; _01 = nextptr[c-step]; _02 = nextptr[c-step+1]; + _10 = nextptr[c -1]; _12 = nextptr[c +1]; + _20 = nextptr[c+step-1]; _21 = nextptr[c+step]; _22 = nextptr[c+step+1]; + vmin = std::min(std::min(std::min(_00,_01),std::min(_02,_10)),std::min(std::min(_12,_20),std::min(_21,_22))); + if (val <= vmin) + { + sift_wt _11p = prevptr[c], _11n = nextptr[c]; + calculate = (val <= std::min(_11p,_11n)); + } + } + } + } + + if (calculate) { CV_TRACE_REGION("pixel_candidate"); From 3a15a3821ac147e2715c2a9364bc02c6d789177d Mon Sep 17 00:00:00 2001 From: Zhang Yin Date: Thu, 10 Jun 2021 08:08:20 +0000 Subject: [PATCH 011/262] Update RISC-V back-end to RVV 0.10 --- cmake/checks/cpu_rvv.cpp | 2 +- .../include/opencv2/core/hal/intrin_rvv.hpp | 1272 ++++++++--------- platforms/linux/riscv64-gcc.toolchain.cmake | 4 +- 3 files changed, 572 insertions(+), 706 deletions(-) diff --git a/cmake/checks/cpu_rvv.cpp b/cmake/checks/cpu_rvv.cpp index a3eab2abc4..684b2ecbeb 100644 --- a/cmake/checks/cpu_rvv.cpp +++ b/cmake/checks/cpu_rvv.cpp @@ -9,7 +9,7 @@ int test() { const float src[] = { 0.0f, 0.0f, 0.0f, 0.0f }; - vfloat32m1_t val = vle32_v_f32m1((const float*)(src)); + vfloat32m1_t val = vle32_v_f32m1((const float*)(src), 4); return (int)vfmv_f_s_f32m1_f32(val); } #else diff --git a/modules/core/include/opencv2/core/hal/intrin_rvv.hpp b/modules/core/include/opencv2/core/hal/intrin_rvv.hpp index cb2140df58..4a3455b073 100644 --- a/modules/core/include/opencv2/core/hal/intrin_rvv.hpp +++ b/modules/core/include/opencv2/core/hal/intrin_rvv.hpp @@ -151,12 +151,14 @@ struct vint8mf4_t }; #define OPENCV_HAL_IMPL_RVV_NATIVE_LOADSTORE_MF2(_Tpvec, _Tp, suffix, width, n) \ -inline _Tpvec vle##width##_v_##suffix##mf2(const _Tp* ptr) \ +inline _Tpvec vle##width##_v_##suffix##mf2(const _Tp* ptr, size_t vl) \ { \ + CV_UNUSED(vl); \ return _Tpvec(ptr); \ } \ -inline void vse##width##_v_##suffix##mf2(_Tp* ptr, _Tpvec v) \ +inline void vse##width##_v_##suffix##mf2(_Tp* ptr, _Tpvec v, size_t vl) \ { \ + CV_UNUSED(vl); \ for (int i = 0; i < n; ++i) \ { \ ptr[i] = v.val[i]; \ @@ -176,15 +178,14 @@ OPENCV_HAL_IMPL_RVV_NATIVE_LOADSTORE_MF2(vfloat64mf2_t, float64_t, f64, 64, 1) #define OPENCV_HAL_IMPL_RVV_NATIVE_WCVT(_Tpwvec, _Tpvec, _wTp, wcvt, suffix, width, n) \ -inline _Tpwvec wcvt (_Tpvec v) \ +inline _Tpwvec wcvt (_Tpvec v, size_t vl) \ { \ _wTp tmp[n]; \ for (int i = 0; i < n; ++i) \ { \ tmp[i] = (_wTp)v.val[i]; \ } \ - vsetvlmax_e##width##m1(); \ - return vle##width##_v_##suffix##m1(tmp); \ + return vle##width##_v_##suffix##m1(tmp, vl); \ } OPENCV_HAL_IMPL_RVV_NATIVE_WCVT(vuint16m1_t, vuint8mf2_t, ushort, vwcvtu_x_x_v_u16m1, u16, 16, 8) @@ -194,32 +195,34 @@ OPENCV_HAL_IMPL_RVV_NATIVE_WCVT(vint32m1_t, vint16mf2_t, int, vwcvt_x_x_v_i32m1, OPENCV_HAL_IMPL_RVV_NATIVE_WCVT(vuint64m1_t, vuint32mf2_t, uint64, vwcvtu_x_x_v_u64m1, u64, 64, 2) OPENCV_HAL_IMPL_RVV_NATIVE_WCVT(vint64m1_t, vint32mf2_t, int64, vwcvt_x_x_v_i64m1, i64, 64, 2) -inline vuint8mf4_t vle8_v_u8mf4 (const uint8_t *base) +inline vuint8mf4_t vle8_v_u8mf4 (const uint8_t *base, size_t vl) { + CV_UNUSED(vl); return vuint8mf4_t(base); } -inline vint8mf4_t vle8_v_i8mf4 (const int8_t *base) +inline vint8mf4_t vle8_v_i8mf4 (const int8_t *base, size_t vl) { + CV_UNUSED(vl); return vint8mf4_t(base); } -inline vuint16mf2_t vwcvtu_x_x_v_u16mf2 (vuint8mf4_t src) +inline vuint16mf2_t vwcvtu_x_x_v_u16mf2 (vuint8mf4_t src, size_t vl) { ushort tmp[4]; for (int i = 0; i < 4; ++i) { tmp[i] = (ushort)src.val[i]; } - return vle16_v_u16mf2(tmp); + return vle16_v_u16mf2(tmp, vl); } -inline vint16mf2_t vwcvt_x_x_v_i16mf2 (vint8mf4_t src) +inline vint16mf2_t vwcvt_x_x_v_i16mf2 (vint8mf4_t src, size_t vl) { short tmp[4]; for (int i = 0; i < 4; ++i) { tmp[i] = (short)src.val[i]; } - return vle16_v_i16mf2(tmp); + return vle16_v_i16mf2(tmp, vl); } //////////// Types //////////// @@ -232,8 +235,7 @@ struct v_uint8x16 v_uint8x16() {} explicit v_uint8x16(vuint8m1_t v) { - vsetvlmax_e8m1(); - vse8_v_u8m1(val, v); + vse8_v_u8m1(val, v, nlanes); } v_uint8x16(uchar v0, uchar v1, uchar v2, uchar v3, uchar v4, uchar v5, uchar v6, uchar v7, uchar v8, uchar v9, uchar v10, uchar v11, uchar v12, uchar v13, uchar v14, uchar v15) @@ -246,8 +248,7 @@ struct v_uint8x16 } operator vuint8m1_t() const { - vsetvlmax_e8m1(); - return vle8_v_u8m1(val); + return vle8_v_u8m1(val, nlanes); } uchar get0() const { @@ -265,8 +266,7 @@ struct v_int8x16 v_int8x16() {} explicit v_int8x16(vint8m1_t v) { - vsetvlmax_e8m1(); - vse8_v_i8m1(val, v); + vse8_v_i8m1(val, v, nlanes); } v_int8x16(schar v0, schar v1, schar v2, schar v3, schar v4, schar v5, schar v6, schar v7, schar v8, schar v9, schar v10, schar v11, schar v12, schar v13, schar v14, schar v15) @@ -279,8 +279,7 @@ struct v_int8x16 } operator vint8m1_t() const { - vsetvlmax_e8m1(); - return vle8_v_i8m1(val); + return vle8_v_i8m1(val, nlanes); } schar get0() const { @@ -298,8 +297,7 @@ struct v_uint16x8 v_uint16x8() {} explicit v_uint16x8(vuint16m1_t v) { - vsetvlmax_e16m1(); - vse16_v_u16m1(val, v); + vse16_v_u16m1(val, v, nlanes); } v_uint16x8(ushort v0, ushort v1, ushort v2, ushort v3, ushort v4, ushort v5, ushort v6, ushort v7) { @@ -311,8 +309,7 @@ struct v_uint16x8 } operator vuint16m1_t() const { - vsetvlmax_e16m1(); - return vle16_v_u16m1(val); + return vle16_v_u16m1(val, nlanes); } ushort get0() const { @@ -330,8 +327,7 @@ struct v_int16x8 v_int16x8() {} explicit v_int16x8(vint16m1_t v) { - vsetvlmax_e16m1(); - vse16_v_i16m1(val, v); + vse16_v_i16m1(val, v, nlanes); } v_int16x8(short v0, short v1, short v2, short v3, short v4, short v5, short v6, short v7) { @@ -343,8 +339,7 @@ struct v_int16x8 } operator vint16m1_t() const { - vsetvlmax_e16m1(); - return vle16_v_i16m1(val); + return vle16_v_i16m1(val, nlanes); } short get0() const { @@ -362,8 +357,7 @@ struct v_uint32x4 v_uint32x4() {} explicit v_uint32x4(vuint32m1_t v) { - vsetvlmax_e32m1(); - vse32_v_u32m1(val, v); + vse32_v_u32m1(val, v, nlanes); } v_uint32x4(unsigned v0, unsigned v1, unsigned v2, unsigned v3) { @@ -375,8 +369,7 @@ struct v_uint32x4 } operator vuint32m1_t() const { - vsetvlmax_e32m1(); - return vle32_v_u32m1(val); + return vle32_v_u32m1(val, nlanes); } unsigned get0() const { @@ -394,8 +387,7 @@ struct v_int32x4 v_int32x4() {} explicit v_int32x4(vint32m1_t v) { - vsetvlmax_e32m1(); - vse32_v_i32m1(val, v); + vse32_v_i32m1(val, v, nlanes); } v_int32x4(int v0, int v1, int v2, int v3) { @@ -407,8 +399,7 @@ struct v_int32x4 } operator vint32m1_t() const { - vsetvlmax_e32m1(); - return vle32_v_i32m1(val); + return vle32_v_i32m1(val, nlanes); } int get0() const { @@ -425,8 +416,7 @@ struct v_float32x4 v_float32x4() {} explicit v_float32x4(vfloat32m1_t v) { - vsetvlmax_e32m1(); - vse32_v_f32m1(val, v); + vse32_v_f32m1(val, v, nlanes); } v_float32x4(float v0, float v1, float v2, float v3) { @@ -438,8 +428,7 @@ struct v_float32x4 } operator vfloat32m1_t() const { - vsetvlmax_e32m1(); - return vle32_v_f32m1(val); + return vle32_v_f32m1(val, nlanes); } float get0() const { @@ -456,8 +445,7 @@ struct v_uint64x2 v_uint64x2() {} explicit v_uint64x2(vuint64m1_t v) { - vsetvlmax_e64m1(); - vse64_v_u64m1(val, v); + vse64_v_u64m1(val, v, nlanes); } v_uint64x2(uint64 v0, uint64 v1) { @@ -469,8 +457,7 @@ struct v_uint64x2 } operator vuint64m1_t() const { - vsetvlmax_e64m1(); - return vle64_v_u64m1(val); + return vle64_v_u64m1(val, nlanes); } uint64 get0() const { @@ -488,8 +475,7 @@ struct v_int64x2 v_int64x2() {} explicit v_int64x2(vint64m1_t v) { - vsetvlmax_e64m1(); - vse64_v_i64m1(val, v); + vse64_v_i64m1(val, v, nlanes); } v_int64x2(int64 v0, int64 v1) { @@ -501,8 +487,7 @@ struct v_int64x2 } operator vint64m1_t() const { - vsetvlmax_e64m1(); - return vle64_v_i64m1(val); + return vle64_v_i64m1(val, nlanes); } int64 get0() const { @@ -521,8 +506,7 @@ struct v_float64x2 v_float64x2() {} explicit v_float64x2(vfloat64m1_t v) { - vsetvlmax_e64m1(); - vse64_v_f64m1(val, v); + vse64_v_f64m1(val, v, nlanes); } v_float64x2(double v0, double v1) { @@ -534,8 +518,7 @@ struct v_float64x2 } operator vfloat64m1_t() const { - vsetvlmax_e64m1(); - return vle64_v_f64m1(val); + return vle64_v_f64m1(val, nlanes); } double get0() const { @@ -549,42 +532,38 @@ struct v_float64x2 //////////// Initial //////////// -#define OPENCV_HAL_IMPL_RVV_INIT_INTEGER(_Tpvec, _Tp, width, suffix1, suffix2) \ +#define OPENCV_HAL_IMPL_RVV_INIT_INTEGER(_Tpvec, _Tp, suffix1, suffix2, vl) \ inline v_##_Tpvec v_setzero_##suffix1() \ { \ - vsetvlmax_e##width##m1(); \ - return v_##_Tpvec(vzero_##suffix2##m1()); \ + return v_##_Tpvec(vmv_v_x_##suffix2##m1(0, vl)); \ } \ inline v_##_Tpvec v_setall_##suffix1(_Tp v) \ { \ - vsetvlmax_e##width##m1(); \ - return v_##_Tpvec(vmv_v_x_##suffix2##m1(v)); \ + return v_##_Tpvec(vmv_v_x_##suffix2##m1(v, vl)); \ } -OPENCV_HAL_IMPL_RVV_INIT_INTEGER(uint8x16, uchar, 8, u8, u8) -OPENCV_HAL_IMPL_RVV_INIT_INTEGER(int8x16, schar, 8, s8, i8) -OPENCV_HAL_IMPL_RVV_INIT_INTEGER(uint16x8, ushort, 16, u16, u16) -OPENCV_HAL_IMPL_RVV_INIT_INTEGER(int16x8, short, 16, s16, i16) -OPENCV_HAL_IMPL_RVV_INIT_INTEGER(uint32x4, unsigned, 32, u32, u32) -OPENCV_HAL_IMPL_RVV_INIT_INTEGER(int32x4, int, 32, s32, i32) -OPENCV_HAL_IMPL_RVV_INIT_INTEGER(uint64x2, uint64, 64, u64, u64) -OPENCV_HAL_IMPL_RVV_INIT_INTEGER(int64x2, int64, 64, s64, i64) +OPENCV_HAL_IMPL_RVV_INIT_INTEGER(uint8x16, uchar, u8, u8, 16) +OPENCV_HAL_IMPL_RVV_INIT_INTEGER(int8x16, schar, s8, i8, 16) +OPENCV_HAL_IMPL_RVV_INIT_INTEGER(uint16x8, ushort, u16, u16, 8) +OPENCV_HAL_IMPL_RVV_INIT_INTEGER(int16x8, short, s16, i16, 8) +OPENCV_HAL_IMPL_RVV_INIT_INTEGER(uint32x4, unsigned, u32, u32, 4) +OPENCV_HAL_IMPL_RVV_INIT_INTEGER(int32x4, int, s32, i32, 4) +OPENCV_HAL_IMPL_RVV_INIT_INTEGER(uint64x2, uint64, u64, u64, 2) +OPENCV_HAL_IMPL_RVV_INIT_INTEGER(int64x2, int64, s64, i64, 2) -#define OPENCV_HAL_IMPL_RVV_INIT_FP(_Tpv, _Tp, width, suffix) \ +#define OPENCV_HAL_IMPL_RVV_INIT_FP(_Tpv, _Tp, suffix, vl) \ inline v_##_Tpv v_setzero_##suffix() \ { \ - vsetvlmax_e##width##m1(); \ - return v_##_Tpv(vzero_##suffix##m1()); \ + return v_##_Tpv(vfmv_v_f_##suffix##m1(0, vl)); \ } \ inline v_##_Tpv v_setall_##suffix(_Tp v) \ { \ - vsetvlmax_e##width##m1(); \ - return v_##_Tpv(vfmv_v_f_##suffix##m1(v)); \ + return v_##_Tpv(vfmv_v_f_##suffix##m1(v, vl)); \ } -OPENCV_HAL_IMPL_RVV_INIT_FP(float32x4, float, 32, f32) +OPENCV_HAL_IMPL_RVV_INIT_FP(float32x4, float, f32, 4) #if CV_SIMD128_64F -OPENCV_HAL_IMPL_RVV_INIT_FP(float64x2, double, 64, f64) +OPENCV_HAL_IMPL_RVV_INIT_FP(float64x2, double, f64, 2) #endif //////////// Reinterpret //////////// @@ -605,167 +584,155 @@ OPENCV_HAL_IMPL_RVV_SELF_REINTERPRET(int64x2, s64) OPENCV_HAL_IMPL_RVV_SELF_REINTERPRET(float64x2, f64) #endif -#define OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(_Tpvec1, _Tpvec2, _nTpvec1, _nTpvec2, suffix1, suffix2, nsuffix1, nsuffix2, width1, width2) \ +#define OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(_Tpvec1, _Tpvec2, _nTpvec1, _nTpvec2, suffix1, suffix2, nsuffix1, nsuffix2, width1, width2, vl1, vl2) \ inline v_##_Tpvec1 v_reinterpret_as_##suffix1(const v_##_Tpvec2& v) \ { \ - vsetvlmax_e##width2##m1(); \ - return v_##_Tpvec1((_nTpvec1)vle##width2##_v_##nsuffix2##m1(v.val)); \ + return v_##_Tpvec1((_nTpvec1)vle##width2##_v_##nsuffix2##m1(v.val, vl2)); \ } \ inline v_##_Tpvec2 v_reinterpret_as_##suffix2(const v_##_Tpvec1& v) \ { \ - vsetvlmax_e##width1##m1(); \ - return v_##_Tpvec2((_nTpvec2)vle##width1##_v_##nsuffix1##m1(v.val)); \ + return v_##_Tpvec2((_nTpvec2)vle##width1##_v_##nsuffix1##m1(v.val, vl1)); \ } -OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint8x16, int8x16, vuint8m1_t, vint8m1_t, u8, s8, u8, i8, 8, 8) -OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint16x8, int16x8, vuint16m1_t, vint16m1_t, u16, s16, u16, i16, 16, 16) -OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint32x4, int32x4, vuint32m1_t, vint32m1_t, u32, s32, u32, i32, 32, 32) -OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint32x4, float32x4, vuint32m1_t, vfloat32m1_t, u32, f32, u32, f32, 32, 32) -OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(int32x4, float32x4, vint32m1_t, vfloat32m1_t, s32, f32, i32, f32, 32, 32) -OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint64x2, int64x2, vuint64m1_t, vint64m1_t, u64, s64, u64, i64, 64, 64) -OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint8x16, uint16x8, vuint8m1_t, vuint16m1_t, u8, u16, u8, u16, 8, 16) -OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint8x16, uint32x4, vuint8m1_t, vuint32m1_t, u8, u32, u8, u32, 8, 32) -OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint8x16, uint64x2, vuint8m1_t, vuint64m1_t, u8, u64, u8, u64, 8, 64) -OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint16x8, uint32x4, vuint16m1_t, vuint32m1_t, u16, u32, u16, u32, 16, 32) -OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint16x8, uint64x2, vuint16m1_t, vuint64m1_t, u16, u64, u16, u64, 16, 64) -OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint32x4, uint64x2, vuint32m1_t, vuint64m1_t, u32, u64, u32, u64, 32, 64) -OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(int8x16, int16x8, vint8m1_t, vint16m1_t, s8, s16, i8, i16, 8, 16) -OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(int8x16, int32x4, vint8m1_t, vint32m1_t, s8, s32, i8, i32, 8, 32) -OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(int8x16, int64x2, vint8m1_t, vint64m1_t, s8, s64, i8, i64, 8, 64) -OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(int16x8, int32x4, vint16m1_t, vint32m1_t, s16, s32, i16, i32, 16, 32) -OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(int16x8, int64x2, vint16m1_t, vint64m1_t, s16, s64, i16, i64, 16, 64) -OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(int32x4, int64x2, vint32m1_t, vint64m1_t, s32, s64, i32, i64, 32, 64) -OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint8x16, int16x8, vuint8m1_t, vint16m1_t, u8, s16, u8, i16, 8, 16) -OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint8x16, int32x4, vuint8m1_t, vint32m1_t, u8, s32, u8, i32, 8, 32) -OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint8x16, int64x2, vuint8m1_t, vint64m1_t, u8, s64, u8, i64, 8, 64) -OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint16x8, int8x16, vuint16m1_t, vint8m1_t, u16, s8, u16, i8, 16, 8) -OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint16x8, int32x4, vuint16m1_t, vint32m1_t, u16, s32, u16, i32, 16, 32) -OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint16x8, int64x2, vuint16m1_t, vint64m1_t, u16, s64, u16, i64, 16, 64) -OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint32x4, int8x16, vuint32m1_t, vint8m1_t, u32, s8, u32, i8, 32, 8) -OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint32x4, int16x8, vuint32m1_t, vint16m1_t, u32, s16, u32, i16, 32, 16) -OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint32x4, int64x2, vuint32m1_t, vint64m1_t, u32, s64, u32, i64, 32, 64) -OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint64x2, int8x16, vuint64m1_t, vint8m1_t, u64, s8, u64, i8, 64, 8) -OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint64x2, int16x8, vuint64m1_t, vint16m1_t, u64, s16, u64, i16, 64, 16) -OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint64x2, int32x4, vuint64m1_t, vint32m1_t, u64, s32, u64, i32, 64, 32) -OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint8x16, float32x4, vuint8m1_t, vfloat32m1_t, u8, f32, u8, f32, 8, 32) -OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint16x8, float32x4, vuint16m1_t, vfloat32m1_t, u16, f32, u16, f32, 16, 32) -OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint64x2, float32x4, vuint64m1_t, vfloat32m1_t, u64, f32, u64, f32, 64, 32) -OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(int8x16, float32x4, vint8m1_t, vfloat32m1_t, s8, f32, i8, f32, 8, 32) -OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(int16x8, float32x4, vint16m1_t, vfloat32m1_t, s16, f32, i16, f32, 16, 32) -OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(int64x2, float32x4, vint64m1_t, vfloat32m1_t, s64, f32, i64, f32, 64, 32) +OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint8x16, int8x16, vuint8m1_t, vint8m1_t, u8, s8, u8, i8, 8, 8, 16, 16) +OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint16x8, int16x8, vuint16m1_t, vint16m1_t, u16, s16, u16, i16, 16, 16, 8, 8) +OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint32x4, int32x4, vuint32m1_t, vint32m1_t, u32, s32, u32, i32, 32, 32, 4, 4) +OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint32x4, float32x4, vuint32m1_t, vfloat32m1_t, u32, f32, u32, f32, 32, 32, 4, 4) +OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(int32x4, float32x4, vint32m1_t, vfloat32m1_t, s32, f32, i32, f32, 32, 32, 4, 4) +OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint64x2, int64x2, vuint64m1_t, vint64m1_t, u64, s64, u64, i64, 64, 64, 2, 2) +OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint8x16, uint16x8, vuint8m1_t, vuint16m1_t, u8, u16, u8, u16, 8, 16, 16, 8) +OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint8x16, uint32x4, vuint8m1_t, vuint32m1_t, u8, u32, u8, u32, 8, 32, 16, 4) +OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint8x16, uint64x2, vuint8m1_t, vuint64m1_t, u8, u64, u8, u64, 8, 64, 16, 2) +OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint16x8, uint32x4, vuint16m1_t, vuint32m1_t, u16, u32, u16, u32, 16, 32, 8, 4) +OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint16x8, uint64x2, vuint16m1_t, vuint64m1_t, u16, u64, u16, u64, 16, 64, 8, 2) +OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint32x4, uint64x2, vuint32m1_t, vuint64m1_t, u32, u64, u32, u64, 32, 64, 4, 2) +OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(int8x16, int16x8, vint8m1_t, vint16m1_t, s8, s16, i8, i16, 8, 16, 16, 8) +OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(int8x16, int32x4, vint8m1_t, vint32m1_t, s8, s32, i8, i32, 8, 32, 16, 4) +OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(int8x16, int64x2, vint8m1_t, vint64m1_t, s8, s64, i8, i64, 8, 64, 16, 2) +OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(int16x8, int32x4, vint16m1_t, vint32m1_t, s16, s32, i16, i32, 16, 32, 8, 4) +OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(int16x8, int64x2, vint16m1_t, vint64m1_t, s16, s64, i16, i64, 16, 64, 8, 2) +OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(int32x4, int64x2, vint32m1_t, vint64m1_t, s32, s64, i32, i64, 32, 64, 4, 2) +OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint8x16, int16x8, vuint8m1_t, vint16m1_t, u8, s16, u8, i16, 8, 16, 16, 8) +OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint8x16, int32x4, vuint8m1_t, vint32m1_t, u8, s32, u8, i32, 8, 32, 16, 4) +OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint8x16, int64x2, vuint8m1_t, vint64m1_t, u8, s64, u8, i64, 8, 64, 16, 2) +OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint16x8, int8x16, vuint16m1_t, vint8m1_t, u16, s8, u16, i8, 16, 8, 8, 16) +OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint16x8, int32x4, vuint16m1_t, vint32m1_t, u16, s32, u16, i32, 16, 32, 8, 4) +OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint16x8, int64x2, vuint16m1_t, vint64m1_t, u16, s64, u16, i64, 16, 64, 8, 2) +OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint32x4, int8x16, vuint32m1_t, vint8m1_t, u32, s8, u32, i8, 32, 8, 4, 16) +OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint32x4, int16x8, vuint32m1_t, vint16m1_t, u32, s16, u32, i16, 32, 16, 4, 8) +OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint32x4, int64x2, vuint32m1_t, vint64m1_t, u32, s64, u32, i64, 32, 64, 4, 2) +OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint64x2, int8x16, vuint64m1_t, vint8m1_t, u64, s8, u64, i8, 64, 8, 2, 16) +OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint64x2, int16x8, vuint64m1_t, vint16m1_t, u64, s16, u64, i16, 64, 16, 2, 8) +OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint64x2, int32x4, vuint64m1_t, vint32m1_t, u64, s32, u64, i32, 64, 32, 2, 4) +OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint8x16, float32x4, vuint8m1_t, vfloat32m1_t, u8, f32, u8, f32, 8, 32, 16, 4) +OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint16x8, float32x4, vuint16m1_t, vfloat32m1_t, u16, f32, u16, f32, 16, 32, 8, 4) +OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint64x2, float32x4, vuint64m1_t, vfloat32m1_t, u64, f32, u64, f32, 64, 32, 2, 4) +OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(int8x16, float32x4, vint8m1_t, vfloat32m1_t, s8, f32, i8, f32, 8, 32, 16, 4) +OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(int16x8, float32x4, vint16m1_t, vfloat32m1_t, s16, f32, i16, f32, 16, 32, 8, 4) +OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(int64x2, float32x4, vint64m1_t, vfloat32m1_t, s64, f32, i64, f32, 64, 32, 2, 4) #if CV_SIMD128_64F -OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint64x2, float64x2, vuint64m1_t, vfloat64m1_t, u64, f64, u64, f64, 64, 64) -OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(int64x2, float64x2, vint64m1_t, vfloat64m1_t, s64, f64, i64, f64, 64, 64) -OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint8x16, float64x2, vuint8m1_t, vfloat64m1_t, u8, f64, u8, f64, 8, 64) -OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint16x8, float64x2, vuint16m1_t, vfloat64m1_t, u16, f64, u16, f64, 16, 64) -OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint32x4, float64x2, vuint32m1_t, vfloat64m1_t, u32, f64, u32, f64, 32, 64) -OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(int8x16, float64x2, vint8m1_t, vfloat64m1_t, s8, f64, i8, f64, 8, 64) -OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(int16x8, float64x2, vint16m1_t, vfloat64m1_t, s16, f64, i16, f64, 16, 64) -OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(int32x4, float64x2, vint32m1_t, vfloat64m1_t, s32, f64, i32, f64, 32, 64) -OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(float32x4, float64x2, vfloat32m1_t, vfloat64m1_t, f32, f64, f32, f64, 32, 64) +OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint64x2, float64x2, vuint64m1_t, vfloat64m1_t, u64, f64, u64, f64, 64, 64, 2, 2) +OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(int64x2, float64x2, vint64m1_t, vfloat64m1_t, s64, f64, i64, f64, 64, 64, 2, 2) +OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint8x16, float64x2, vuint8m1_t, vfloat64m1_t, u8, f64, u8, f64, 8, 64, 16, 2) +OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint16x8, float64x2, vuint16m1_t, vfloat64m1_t, u16, f64, u16, f64, 16, 64, 6, 2) +OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint32x4, float64x2, vuint32m1_t, vfloat64m1_t, u32, f64, u32, f64, 32, 64, 4, 2) +OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(int8x16, float64x2, vint8m1_t, vfloat64m1_t, s8, f64, i8, f64, 8, 64, 16, 2) +OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(int16x8, float64x2, vint16m1_t, vfloat64m1_t, s16, f64, i16, f64, 16, 64, 8, 2) +OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(int32x4, float64x2, vint32m1_t, vfloat64m1_t, s32, f64, i32, f64, 32, 64, 4, 2) +OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(float32x4, float64x2, vfloat32m1_t, vfloat64m1_t, f32, f64, f32, f64, 32, 64, 4, 2) #endif ////////////// Extract ////////////// -#define OPENCV_HAL_IMPL_RVV_EXTRACT(_Tpvec, _Tp, suffix, width, vmv) \ +#define OPENCV_HAL_IMPL_RVV_EXTRACT_INTEGER(_Tpvec, _Tp, suffix, vmv, vl) \ template \ inline _Tpvec v_extract(const _Tpvec& a, const _Tpvec& b) \ { \ - vsetvlmax_e##width##m1(); \ - return _Tpvec(vslideup_vx_##suffix##m1(vslidedown_vx_##suffix##m1(vzero_##suffix##m1(), a, s), b, _Tpvec::nlanes - s)); \ + return _Tpvec(vslideup_vx_##suffix##m1(vslidedown_vx_##suffix##m1(vmv_v_x_##suffix##m1(0, vl), a, s, vl), b, _Tpvec::nlanes - s, vl)); \ } \ template inline _Tp v_extract_n(_Tpvec v) \ { \ - vsetvlmax_e##width##m1(); \ - return _Tp(vmv(vslidedown_vx_##suffix##m1(vzero_##suffix##m1(), v, i))); \ + return _Tp(vmv(vslidedown_vx_##suffix##m1(vmv_v_x_##suffix##m1(0, vl), v, i, vl))); \ } -OPENCV_HAL_IMPL_RVV_EXTRACT(v_uint8x16, uchar, u8, 8, vmv_x_s_u8m1_u8) -OPENCV_HAL_IMPL_RVV_EXTRACT(v_int8x16, schar, i8, 8, vmv_x_s_i8m1_i8) -OPENCV_HAL_IMPL_RVV_EXTRACT(v_uint16x8, ushort, u16, 16, vmv_x_s_u16m1_u16) -OPENCV_HAL_IMPL_RVV_EXTRACT(v_int16x8, short, i16, 16, vmv_x_s_i16m1_i16) -OPENCV_HAL_IMPL_RVV_EXTRACT(v_uint32x4, uint, u32, 32, vmv_x_s_u32m1_u32) -OPENCV_HAL_IMPL_RVV_EXTRACT(v_int32x4, int, i32, 32, vmv_x_s_i32m1_i32) -OPENCV_HAL_IMPL_RVV_EXTRACT(v_uint64x2, uint64, u64, 64, vmv_x_s_u64m1_u64) -OPENCV_HAL_IMPL_RVV_EXTRACT(v_int64x2, int64, i64, 64, vmv_x_s_i64m1_i64) -OPENCV_HAL_IMPL_RVV_EXTRACT(v_float32x4, float, f32, 32, vfmv_f_s_f32m1_f32) +OPENCV_HAL_IMPL_RVV_EXTRACT_INTEGER(v_uint8x16, uchar, u8, vmv_x_s_u8m1_u8, 16) +OPENCV_HAL_IMPL_RVV_EXTRACT_INTEGER(v_int8x16, schar, i8, vmv_x_s_i8m1_i8, 16) +OPENCV_HAL_IMPL_RVV_EXTRACT_INTEGER(v_uint16x8, ushort, u16, vmv_x_s_u16m1_u16, 8) +OPENCV_HAL_IMPL_RVV_EXTRACT_INTEGER(v_int16x8, short, i16, vmv_x_s_i16m1_i16, 8) +OPENCV_HAL_IMPL_RVV_EXTRACT_INTEGER(v_uint32x4, uint, u32, vmv_x_s_u32m1_u32, 4) +OPENCV_HAL_IMPL_RVV_EXTRACT_INTEGER(v_int32x4, int, i32, vmv_x_s_i32m1_i32, 4) +OPENCV_HAL_IMPL_RVV_EXTRACT_INTEGER(v_uint64x2, uint64, u64, vmv_x_s_u64m1_u64, 2) +OPENCV_HAL_IMPL_RVV_EXTRACT_INTEGER(v_int64x2, int64, i64, vmv_x_s_i64m1_i64, 2) + +#define OPENCV_HAL_IMPL_RVV_EXTRACT_FP(_Tpvec, _Tp, suffix, vmv, vl) \ +template \ +inline _Tpvec v_extract(const _Tpvec& a, const _Tpvec& b) \ +{ \ + return _Tpvec(vslideup_vx_##suffix##m1(vslidedown_vx_##suffix##m1(vfmv_v_f_##suffix##m1(0, vl), a, s, vl), b, _Tpvec::nlanes - s, vl)); \ +} \ +template inline _Tp v_extract_n(_Tpvec v) \ +{ \ + return _Tp(vmv(vslidedown_vx_##suffix##m1(vfmv_v_f_##suffix##m1(0, vl), v, i, vl))); \ +} + +OPENCV_HAL_IMPL_RVV_EXTRACT_FP(v_float32x4, float, f32, vfmv_f_s_f32m1_f32, 4) #if CV_SIMD128_64F -OPENCV_HAL_IMPL_RVV_EXTRACT(v_float64x2, double, f64, 64, vfmv_f_s_f64m1_f64) +OPENCV_HAL_IMPL_RVV_EXTRACT_FP(v_float64x2, double, f64, vfmv_f_s_f64m1_f64, 2) #endif ////////////// Load/Store ////////////// -#define OPENCV_HAL_IMPL_RVV_LOADSTORE_OP(_Tpvec, _nTpvec, _Tp, hvl, width, suffix) \ +#define OPENCV_HAL_IMPL_RVV_LOADSTORE_OP(_Tpvec, _nTpvec, _Tp, hvl, vl, width, suffix, vmv) \ inline _Tpvec v_load(const _Tp* ptr) \ { \ - vsetvlmax_e8m1(); \ - return _Tpvec((_nTpvec)vle8_v_u8m1((uchar*)ptr)); \ + return _Tpvec((_nTpvec)vle8_v_u8m1((uchar*)ptr, 16)); \ } \ inline _Tpvec v_load_aligned(const _Tp* ptr) \ { \ - vsetvlmax_e##width##m1(); \ - return _Tpvec(vle##width##_v_##suffix##m1(ptr)); \ + return _Tpvec(vle##width##_v_##suffix##m1(ptr, vl)); \ } \ inline _Tpvec v_load_low(const _Tp* ptr) \ { \ - vsetvl_e##width##m1(hvl); \ - _Tpvec res = _Tpvec(vle##width##_v_##suffix##m1(ptr)); \ - vsetvlmax_e##width##m1(); \ + _Tpvec res = _Tpvec(vle##width##_v_##suffix##m1(ptr, hvl)); \ return res; \ } \ inline void v_store(_Tp* ptr, const _Tpvec& a) \ { \ - vsetvlmax_e8m1(); \ - vse8_v_u8m1((uchar*)ptr, vle8_v_u8m1((uchar*)a.val)); \ + vse8_v_u8m1((uchar*)ptr, vle8_v_u8m1((uchar*)a.val, 16), 16); \ } \ inline void v_store_aligned(_Tp* ptr, const _Tpvec& a) \ { \ - vsetvlmax_e##width##m1(); \ - vse##width##_v_##suffix##m1(ptr, a); \ + vse##width##_v_##suffix##m1(ptr, a, vl); \ } \ inline void v_store_aligned_nocache(_Tp* ptr, const _Tpvec& a) \ { \ - vsetvlmax_e##width##m1(); \ - vse##width##_v_##suffix##m1(ptr, a); \ + vse##width##_v_##suffix##m1(ptr, a, vl); \ } \ inline void v_store(_Tp* ptr, const _Tpvec& a, hal::StoreMode /*mode*/) \ { \ - vsetvlmax_e##width##m1(); \ - vse##width##_v_##suffix##m1(ptr, a); \ + vse##width##_v_##suffix##m1(ptr, a, vl); \ } \ inline void v_store_low(_Tp* ptr, const _Tpvec& a) \ { \ - _Tp CV_DECL_ALIGNED(32) tmp_ptr[_Tpvec::nlanes] = {0}; \ - vsetvlmax_e##width##m1(); \ - vse##width##_v_##suffix##m1(tmp_ptr, a); \ - for(int i = 0; i < _Tpvec::nlanes/2; ++i) \ - { \ - ptr[i] = tmp_ptr[i]; \ - } \ + vse##width##_v_##suffix##m1(ptr, a, hvl); \ } \ inline void v_store_high(_Tp* ptr, const _Tpvec& a) \ { \ - _Tp CV_DECL_ALIGNED(32) tmp_ptr[_Tpvec::nlanes] = {0}; \ - vsetvlmax_e##width##m1(); \ - vse##width##_v_##suffix##m1(tmp_ptr, a); \ - for(int i = 0; i < _Tpvec::nlanes/2; ++i) \ - { \ - ptr[i] = tmp_ptr[i+_Tpvec::nlanes/2]; \ - } \ + vse##width##_v_##suffix##m1(ptr, vslidedown_vx_##suffix##m1(vmv(0, vl), a, hvl, vl), hvl); \ } -OPENCV_HAL_IMPL_RVV_LOADSTORE_OP(v_uint8x16, vuint8m1_t, uchar, 8, 8, u8) -OPENCV_HAL_IMPL_RVV_LOADSTORE_OP(v_int8x16, vint8m1_t, schar, 8, 8, i8) -OPENCV_HAL_IMPL_RVV_LOADSTORE_OP(v_uint16x8, vuint16m1_t, ushort, 4, 16, u16) -OPENCV_HAL_IMPL_RVV_LOADSTORE_OP(v_int16x8, vint16m1_t, short, 4, 16, i16) -OPENCV_HAL_IMPL_RVV_LOADSTORE_OP(v_uint32x4, vuint32m1_t, unsigned, 2, 32, u32) -OPENCV_HAL_IMPL_RVV_LOADSTORE_OP(v_int32x4, vint32m1_t, int, 2, 32, i32) -OPENCV_HAL_IMPL_RVV_LOADSTORE_OP(v_uint64x2, vuint64m1_t, uint64, 1, 64, u64) -OPENCV_HAL_IMPL_RVV_LOADSTORE_OP(v_int64x2, vint64m1_t, int64, 1, 64, i64) -OPENCV_HAL_IMPL_RVV_LOADSTORE_OP(v_float32x4, vfloat32m1_t, float, 2, 32, f32) +OPENCV_HAL_IMPL_RVV_LOADSTORE_OP(v_uint8x16, vuint8m1_t, uchar, 8, 16, 8, u8, vmv_v_x_u8m1) +OPENCV_HAL_IMPL_RVV_LOADSTORE_OP(v_int8x16, vint8m1_t, schar, 8, 16, 8, i8, vmv_v_x_i8m1) +OPENCV_HAL_IMPL_RVV_LOADSTORE_OP(v_uint16x8, vuint16m1_t, ushort, 4, 8, 16, u16, vmv_v_x_u16m1) +OPENCV_HAL_IMPL_RVV_LOADSTORE_OP(v_int16x8, vint16m1_t, short, 4, 8, 16, i16, vmv_v_x_i16m1) +OPENCV_HAL_IMPL_RVV_LOADSTORE_OP(v_uint32x4, vuint32m1_t, unsigned, 2, 4, 32, u32, vmv_v_x_u32m1) +OPENCV_HAL_IMPL_RVV_LOADSTORE_OP(v_int32x4, vint32m1_t, int, 2, 4, 32, i32, vmv_v_x_i32m1) +OPENCV_HAL_IMPL_RVV_LOADSTORE_OP(v_uint64x2, vuint64m1_t, uint64, 1, 2, 64, u64, vmv_v_x_u64m1) +OPENCV_HAL_IMPL_RVV_LOADSTORE_OP(v_int64x2, vint64m1_t, int64, 1, 2, 64, i64, vmv_v_x_i64m1) +OPENCV_HAL_IMPL_RVV_LOADSTORE_OP(v_float32x4, vfloat32m1_t, float, 2, 4, 32, f32, vfmv_v_f_f32m1) #if CV_SIMD128_64F -OPENCV_HAL_IMPL_RVV_LOADSTORE_OP(v_float64x2, vfloat64m1_t, double, 1, 64, f64) +OPENCV_HAL_IMPL_RVV_LOADSTORE_OP(v_float64x2, vfloat64m1_t, double, 1, 2, 64, f64, vfmv_v_f_f64m1) #endif inline v_int8x16 v_load_halves(const schar* ptr0, const schar* ptr1) @@ -775,8 +742,7 @@ inline v_int8x16 v_load_halves(const schar* ptr0, const schar* ptr1) ptr0[0], ptr0[1], ptr0[2], ptr0[3], ptr0[4], ptr0[5], ptr0[6], ptr0[7], ptr1[0], ptr1[1], ptr1[2], ptr1[3], ptr1[4], ptr1[5], ptr1[6], ptr1[7] }; - vsetvlmax_e8m1(); - return v_int8x16(vle8_v_i8m1(elems)); + return v_int8x16(vle8_v_i8m1(elems, 16)); } inline v_uint8x16 v_load_halves(const uchar* ptr0, const uchar* ptr1) { return v_reinterpret_as_u8(v_load_halves((schar*)ptr0, (schar*)ptr1)); } @@ -786,8 +752,7 @@ inline v_int16x8 v_load_halves(const short* ptr0, const short* ptr1) { ptr0[0], ptr0[1], ptr0[2], ptr0[3], ptr1[0], ptr1[1], ptr1[2], ptr1[3] }; - vsetvlmax_e16m1(); - return v_int16x8(vle16_v_i16m1(elems)); + return v_int16x8(vle16_v_i16m1(elems, 8)); } inline v_uint16x8 v_load_halves(const ushort* ptr0, const ushort* ptr1) { return v_reinterpret_as_u16(v_load_halves((short*)ptr0, (short*)ptr1)); } @@ -797,8 +762,7 @@ inline v_int32x4 v_load_halves(const int* ptr0, const int* ptr1) { ptr0[0], ptr0[1], ptr1[0], ptr1[1] }; - vsetvlmax_e32m1(); - return v_int32x4(vle32_v_i32m1(elems)); + return v_int32x4(vle32_v_i32m1(elems, 4)); } inline v_float32x4 v_load_halves(const float* ptr0, const float* ptr1) { @@ -806,8 +770,7 @@ inline v_float32x4 v_load_halves(const float* ptr0, const float* ptr1) { ptr0[0], ptr0[1], ptr1[0], ptr1[1] }; - vsetvlmax_e32m1(); - return v_float32x4(vle32_v_f32m1(elems)); + return v_float32x4(vle32_v_f32m1(elems, 4)); } inline v_uint32x4 v_load_halves(const unsigned* ptr0, const unsigned* ptr1) { return v_reinterpret_as_u32(v_load_halves((int*)ptr0, (int*)ptr1)); } @@ -817,8 +780,7 @@ inline v_int64x2 v_load_halves(const int64* ptr0, const int64* ptr1) { ptr0[0], ptr1[0] }; - vsetvlmax_e64m1(); - return v_int64x2(vle64_v_i64m1(elems)); + return v_int64x2(vle64_v_i64m1(elems, 2)); } inline v_uint64x2 v_load_halves(const uint64* ptr0, const uint64* ptr1) { return v_reinterpret_as_u64(v_load_halves((int64*)ptr0, (int64*)ptr1)); } @@ -829,8 +791,7 @@ inline v_float64x2 v_load_halves(const double* ptr0, const double* ptr1) { ptr0[0], ptr1[0] }; - vsetvlmax_e64m1(); - return v_float64x2(vle64_v_f64m1(elems)); + return v_float64x2(vle64_v_f64m1(elems, 2)); } #endif @@ -858,8 +819,7 @@ inline v_int8x16 v_lut(const schar* tab, const int* idx) tab[idx[14]], tab[idx[15]] }; - vsetvlmax_e8m1(); - return v_int8x16(vle8_v_i8m1(elems)); + return v_int8x16(vle8_v_i8m1(elems, 16)); } inline v_int8x16 v_lut_pairs(const schar* tab, const int* idx) { @@ -882,8 +842,7 @@ inline v_int8x16 v_lut_pairs(const schar* tab, const int* idx) tab[idx[7]], tab[idx[7] + 1] }; - vsetvlmax_e8m1(); - return v_int8x16(vle8_v_i8m1(elems)); + return v_int8x16(vle8_v_i8m1(elems, 16)); } inline v_int8x16 v_lut_quads(const schar* tab, const int* idx) { @@ -906,8 +865,7 @@ inline v_int8x16 v_lut_quads(const schar* tab, const int* idx) tab[idx[3] + 2], tab[idx[3] + 3] }; - vsetvlmax_e8m1(); - return v_int8x16(vle8_v_i8m1(elems)); + return v_int8x16(vle8_v_i8m1(elems, 16)); } inline v_uint8x16 v_lut(const uchar* tab, const int* idx) { return v_reinterpret_as_u8(v_lut((schar*)tab, idx)); } inline v_uint8x16 v_lut_pairs(const uchar* tab, const int* idx) { return v_reinterpret_as_u8(v_lut_pairs((schar*)tab, idx)); } @@ -926,8 +884,7 @@ inline v_int16x8 v_lut(const short* tab, const int* idx) tab[idx[6]], tab[idx[7]] }; - vsetvlmax_e16m1(); - return v_int16x8(vle16_v_i16m1(elems)); + return v_int16x8(vle16_v_i16m1(elems, 8)); } inline v_int16x8 v_lut_pairs(const short* tab, const int* idx) { @@ -942,8 +899,7 @@ inline v_int16x8 v_lut_pairs(const short* tab, const int* idx) tab[idx[3]], tab[idx[3] + 1] }; - vsetvlmax_e16m1(); - return v_int16x8(vle16_v_i16m1(elems)); + return v_int16x8(vle16_v_i16m1(elems, 8)); } inline v_int16x8 v_lut_quads(const short* tab, const int* idx) { @@ -958,8 +914,7 @@ inline v_int16x8 v_lut_quads(const short* tab, const int* idx) tab[idx[1] + 2], tab[idx[1] + 3] }; - vsetvlmax_e16m1(); - return v_int16x8(vle16_v_i16m1(elems)); + return v_int16x8(vle16_v_i16m1(elems, 8)); } inline v_uint16x8 v_lut(const ushort* tab, const int* idx) { return v_reinterpret_as_u16(v_lut((short*)tab, idx)); } inline v_uint16x8 v_lut_pairs(const ushort* tab, const int* idx) { return v_reinterpret_as_u16(v_lut_pairs((short*)tab, idx)); } @@ -974,8 +929,7 @@ inline v_int32x4 v_lut(const int* tab, const int* idx) tab[idx[2]], tab[idx[3]] }; - vsetvlmax_e32m1(); - return v_int32x4(vle32_v_i32m1(elems)); + return v_int32x4(vle32_v_i32m1(elems, 4)); } inline v_int32x4 v_lut_pairs(const int* tab, const int* idx) { @@ -986,13 +940,11 @@ inline v_int32x4 v_lut_pairs(const int* tab, const int* idx) tab[idx[1]], tab[idx[1] + 1] }; - vsetvlmax_e32m1(); - return v_int32x4(vle32_v_i32m1(elems)); + return v_int32x4(vle32_v_i32m1(elems, 4)); } inline v_int32x4 v_lut_quads(const int* tab, const int* idx) { - vsetvlmax_e32m1(); - return v_int32x4(vle32_v_i32m1(tab + idx[0])); + return v_int32x4(vle32_v_i32m1(tab + idx[0], 4)); } inline v_uint32x4 v_lut(const unsigned* tab, const int* idx) { return v_reinterpret_as_u32(v_lut((int*)tab, idx)); } @@ -1006,13 +958,11 @@ inline v_int64x2 v_lut(const int64_t* tab, const int* idx) tab[idx[0]], tab[idx[1]] }; - vsetvlmax_e64m1(); - return v_int64x2(vle64_v_i64m1(elems)); + return v_int64x2(vle64_v_i64m1(elems, 2)); } inline v_int64x2 v_lut_pairs(const int64* tab, const int* idx) { - vsetvlmax_e64m1(); - return v_int64x2(vle64_v_i64m1(tab + idx[0])); + return v_int64x2(vle64_v_i64m1(tab + idx[0], 2)); } inline v_uint64x2 v_lut(const uint64* tab, const int* idx) { return v_reinterpret_as_u64(v_lut((const int64_t *)tab, idx)); } inline v_uint64x2 v_lut_pairs(const uint64* tab, const int* idx) { return v_reinterpret_as_u64(v_lut_pairs((const int64_t *)tab, idx)); } @@ -1026,8 +976,7 @@ inline v_float32x4 v_lut(const float* tab, const int* idx) tab[idx[2]], tab[idx[3]] }; - vsetvlmax_e32m1(); - return v_float32x4(vle32_v_f32m1(elems)); + return v_float32x4(vle32_v_f32m1(elems, 4)); } inline v_float32x4 v_lut_pairs(const float* tab, const int* idx) { @@ -1038,13 +987,11 @@ inline v_float32x4 v_lut_pairs(const float* tab, const int* idx) tab[idx[1]], tab[idx[1] + 1] }; - vsetvlmax_e32m1(); - return v_float32x4(vle32_v_f32m1(elems)); + return v_float32x4(vle32_v_f32m1(elems, 4)); } inline v_float32x4 v_lut_quads(const float* tab, const int* idx) { - vsetvlmax_e32m1(); - return v_float32x4(vle32_v_f32m1(tab + idx[0])); + return v_float32x4(vle32_v_f32m1(tab + idx[0], 4)); } inline v_int32x4 v_lut(const int* tab, const v_int32x4& idxvec) @@ -1056,8 +1003,7 @@ inline v_int32x4 v_lut(const int* tab, const v_int32x4& idxvec) tab[v_extract_n<2>(idxvec)], tab[v_extract_n<3>(idxvec)] }; - vsetvlmax_e32m1(); - return v_int32x4(vle32_v_i32m1(elems)); + return v_int32x4(vle32_v_i32m1(elems, 4)); } inline v_uint32x4 v_lut(const unsigned* tab, const v_int32x4& idxvec) @@ -1069,8 +1015,7 @@ inline v_uint32x4 v_lut(const unsigned* tab, const v_int32x4& idxvec) tab[v_extract_n<2>(idxvec)], tab[v_extract_n<3>(idxvec)] }; - vsetvlmax_e32m1(); - return v_uint32x4(vle32_v_u32m1(elems)); + return v_uint32x4(vle32_v_u32m1(elems, 4)); } inline v_float32x4 v_lut(const float* tab, const v_int32x4& idxvec) @@ -1082,8 +1027,7 @@ inline v_float32x4 v_lut(const float* tab, const v_int32x4& idxvec) tab[v_extract_n<2>(idxvec)], tab[v_extract_n<3>(idxvec)] }; - vsetvlmax_e32m1(); - return v_float32x4(vle32_v_f32m1(elems)); + return v_float32x4(vle32_v_f32m1(elems, 4)); } inline void v_lut_deinterleave(const float* tab, const v_int32x4& idxvec, v_float32x4& x, v_float32x4& y) @@ -1103,14 +1047,12 @@ inline v_float64x2 v_lut(const double* tab, const int* idx) tab[idx[0]], tab[idx[1]] }; - vsetvlmax_e64m1(); - return v_float64x2(vle64_v_f64m1(elems)); + return v_float64x2(vle64_v_f64m1(elems, 2)); } inline v_float64x2 v_lut_pairs(const double* tab, const int* idx) { - vsetvlmax_e64m1(); - return v_float64x2(vle64_v_f64m1(tab + idx[0])); + return v_float64x2(vle64_v_f64m1(tab + idx[0], 2)); } inline v_float64x2 v_lut(const double* tab, const v_int32x4& idxvec) @@ -1120,8 +1062,7 @@ inline v_float64x2 v_lut(const double* tab, const v_int32x4& idxvec) tab[v_extract_n<0>(idxvec)], tab[v_extract_n<1>(idxvec)] }; - vsetvlmax_e64m1(); - return v_float64x2(vle64_v_f64m1(elems)); + return v_float64x2(vle64_v_f64m1(elems, 2)); } inline void v_lut_deinterleave(const double* tab, const v_int32x4& idxvec, v_float64x2& x, v_float64x2& y) @@ -1141,8 +1082,7 @@ inline v_uint8x16 v_pack_b(const v_uint16x8& a, const v_uint16x8& b) ushort CV_DECL_ALIGNED(32) ptr[16] = {0}; v_store(ptr, a); v_store(ptr + 8, b); - vsetvlmax_e8m1(); - return v_uint8x16(vnsrl_wx_u8m1(vle16_v_u16m2(ptr), 0)); + return v_uint8x16(vnsrl_wx_u8m1(vle16_v_u16m2(ptr, 16), 0, 16)); } inline v_uint8x16 v_pack_b(const v_uint32x4& a, const v_uint32x4& b, @@ -1153,8 +1093,7 @@ inline v_uint8x16 v_pack_b(const v_uint32x4& a, const v_uint32x4& b, v_store(ptr + 4, b); v_store(ptr + 8, c); v_store(ptr + 12, d); - vsetvlmax_e8m1(); - return v_uint8x16(vnsrl_wx_u8m1(vnsrl_wx_u16m2(vle32_v_u32m4(ptr), 0), 0)); + return v_uint8x16(vnsrl_wx_u8m1(vnsrl_wx_u16m2(vle32_v_u32m4(ptr, 16), 0, 16), 0, 16)); } inline v_uint8x16 v_pack_b(const v_uint64x2& a, const v_uint64x2& b, const v_uint64x2& c, @@ -1170,95 +1109,89 @@ inline v_uint8x16 v_pack_b(const v_uint64x2& a, const v_uint64x2& b, const v_uin v_store(ptr + 10, f); v_store(ptr + 12, g); v_store(ptr + 14, h); - vsetvlmax_e8m1(); - return v_uint8x16(vnsrl_wx_u8m1(vnsrl_wx_u16m2(vnsrl_wx_u32m4(vle64_v_u64m8(ptr), 0), 0), 0)); + return v_uint8x16(vnsrl_wx_u8m1(vnsrl_wx_u16m2(vnsrl_wx_u32m4(vle64_v_u64m8(ptr, 16), 0, 16), 0, 16), 0, 16)); } ////////////// Arithmetics ////////////// -#define OPENCV_HAL_IMPL_RVV_BIN_OP(bin_op, _Tpvec, intrin, width) \ +#define OPENCV_HAL_IMPL_RVV_BIN_OP(bin_op, _Tpvec, intrin, vl) \ inline _Tpvec operator bin_op (const _Tpvec& a, const _Tpvec& b) \ { \ - vsetvlmax_e##width##m1(); \ - return _Tpvec(intrin(a, b)); \ + return _Tpvec(intrin(a, b, vl)); \ } \ inline _Tpvec& operator bin_op##= (_Tpvec& a, const _Tpvec& b) \ { \ - vsetvlmax_e##width##m1(); \ - a = _Tpvec(intrin(a, b)); \ + a = _Tpvec(intrin(a, b, vl)); \ return a; \ } -OPENCV_HAL_IMPL_RVV_BIN_OP(+, v_uint8x16, vsaddu_vv_u8m1, 8) -OPENCV_HAL_IMPL_RVV_BIN_OP(-, v_uint8x16, vssubu_vv_u8m1, 8) -OPENCV_HAL_IMPL_RVV_BIN_OP(/, v_uint8x16, vdivu_vv_u8m1, 8) -OPENCV_HAL_IMPL_RVV_BIN_OP(+, v_int8x16, vsadd_vv_i8m1, 8) -OPENCV_HAL_IMPL_RVV_BIN_OP(-, v_int8x16, vssub_vv_i8m1, 8) -OPENCV_HAL_IMPL_RVV_BIN_OP(/, v_int8x16, vdiv_vv_i8m1, 8) -OPENCV_HAL_IMPL_RVV_BIN_OP(+, v_uint16x8, vsaddu_vv_u16m1, 16) -OPENCV_HAL_IMPL_RVV_BIN_OP(-, v_uint16x8, vssubu_vv_u16m1, 16) -OPENCV_HAL_IMPL_RVV_BIN_OP(/, v_uint16x8, vdivu_vv_u16m1, 16) -OPENCV_HAL_IMPL_RVV_BIN_OP(+, v_int16x8, vsadd_vv_i16m1, 16) -OPENCV_HAL_IMPL_RVV_BIN_OP(-, v_int16x8, vssub_vv_i16m1, 16) -OPENCV_HAL_IMPL_RVV_BIN_OP(/, v_int16x8, vdiv_vv_i16m1, 16) -OPENCV_HAL_IMPL_RVV_BIN_OP(+, v_uint32x4, vadd_vv_u32m1, 32) -OPENCV_HAL_IMPL_RVV_BIN_OP(-, v_uint32x4, vsub_vv_u32m1, 32) -OPENCV_HAL_IMPL_RVV_BIN_OP(*, v_uint32x4, vmul_vv_u32m1, 32) -OPENCV_HAL_IMPL_RVV_BIN_OP(/, v_uint32x4, vdivu_vv_u32m1, 32) -OPENCV_HAL_IMPL_RVV_BIN_OP(+, v_int32x4, vadd_vv_i32m1, 32) -OPENCV_HAL_IMPL_RVV_BIN_OP(-, v_int32x4, vsub_vv_i32m1, 32) -OPENCV_HAL_IMPL_RVV_BIN_OP(*, v_int32x4, vmul_vv_i32m1, 32) -OPENCV_HAL_IMPL_RVV_BIN_OP(/, v_int32x4, vdiv_vv_i32m1, 32) -OPENCV_HAL_IMPL_RVV_BIN_OP(+, v_float32x4, vfadd_vv_f32m1, 32) -OPENCV_HAL_IMPL_RVV_BIN_OP(-, v_float32x4, vfsub_vv_f32m1, 32) -OPENCV_HAL_IMPL_RVV_BIN_OP(*, v_float32x4, vfmul_vv_f32m1, 32) -OPENCV_HAL_IMPL_RVV_BIN_OP(/, v_float32x4, vfdiv_vv_f32m1, 32) -OPENCV_HAL_IMPL_RVV_BIN_OP(+, v_uint64x2, vadd_vv_u64m1, 64) -OPENCV_HAL_IMPL_RVV_BIN_OP(-, v_uint64x2, vsub_vv_u64m1, 64) -OPENCV_HAL_IMPL_RVV_BIN_OP(*, v_uint64x2, vmul_vv_u64m1, 64) -OPENCV_HAL_IMPL_RVV_BIN_OP(/, v_uint64x2, vdivu_vv_u64m1, 64) -OPENCV_HAL_IMPL_RVV_BIN_OP(+, v_int64x2, vadd_vv_i64m1, 64) -OPENCV_HAL_IMPL_RVV_BIN_OP(-, v_int64x2, vsub_vv_i64m1, 64) -OPENCV_HAL_IMPL_RVV_BIN_OP(*, v_int64x2, vmul_vv_i64m1, 64) -OPENCV_HAL_IMPL_RVV_BIN_OP(/, v_int64x2, vdiv_vv_i64m1, 64) +OPENCV_HAL_IMPL_RVV_BIN_OP(+, v_uint8x16, vsaddu_vv_u8m1, 16) +OPENCV_HAL_IMPL_RVV_BIN_OP(-, v_uint8x16, vssubu_vv_u8m1, 16) +OPENCV_HAL_IMPL_RVV_BIN_OP(/, v_uint8x16, vdivu_vv_u8m1, 16) +OPENCV_HAL_IMPL_RVV_BIN_OP(+, v_int8x16, vsadd_vv_i8m1, 16) +OPENCV_HAL_IMPL_RVV_BIN_OP(-, v_int8x16, vssub_vv_i8m1, 16) +OPENCV_HAL_IMPL_RVV_BIN_OP(/, v_int8x16, vdiv_vv_i8m1, 16) +OPENCV_HAL_IMPL_RVV_BIN_OP(+, v_uint16x8, vsaddu_vv_u16m1, 8) +OPENCV_HAL_IMPL_RVV_BIN_OP(-, v_uint16x8, vssubu_vv_u16m1, 8) +OPENCV_HAL_IMPL_RVV_BIN_OP(/, v_uint16x8, vdivu_vv_u16m1, 8) +OPENCV_HAL_IMPL_RVV_BIN_OP(+, v_int16x8, vsadd_vv_i16m1, 8) +OPENCV_HAL_IMPL_RVV_BIN_OP(-, v_int16x8, vssub_vv_i16m1, 8) +OPENCV_HAL_IMPL_RVV_BIN_OP(/, v_int16x8, vdiv_vv_i16m1, 8) +OPENCV_HAL_IMPL_RVV_BIN_OP(+, v_uint32x4, vadd_vv_u32m1, 4) +OPENCV_HAL_IMPL_RVV_BIN_OP(-, v_uint32x4, vsub_vv_u32m1, 4) +OPENCV_HAL_IMPL_RVV_BIN_OP(*, v_uint32x4, vmul_vv_u32m1, 4) +OPENCV_HAL_IMPL_RVV_BIN_OP(/, v_uint32x4, vdivu_vv_u32m1, 4) +OPENCV_HAL_IMPL_RVV_BIN_OP(+, v_int32x4, vadd_vv_i32m1, 4) +OPENCV_HAL_IMPL_RVV_BIN_OP(-, v_int32x4, vsub_vv_i32m1, 4) +OPENCV_HAL_IMPL_RVV_BIN_OP(*, v_int32x4, vmul_vv_i32m1, 4) +OPENCV_HAL_IMPL_RVV_BIN_OP(/, v_int32x4, vdiv_vv_i32m1, 4) +OPENCV_HAL_IMPL_RVV_BIN_OP(+, v_float32x4, vfadd_vv_f32m1, 4) +OPENCV_HAL_IMPL_RVV_BIN_OP(-, v_float32x4, vfsub_vv_f32m1, 4) +OPENCV_HAL_IMPL_RVV_BIN_OP(*, v_float32x4, vfmul_vv_f32m1, 4) +OPENCV_HAL_IMPL_RVV_BIN_OP(/, v_float32x4, vfdiv_vv_f32m1, 4) +OPENCV_HAL_IMPL_RVV_BIN_OP(+, v_uint64x2, vadd_vv_u64m1, 2) +OPENCV_HAL_IMPL_RVV_BIN_OP(-, v_uint64x2, vsub_vv_u64m1, 2) +OPENCV_HAL_IMPL_RVV_BIN_OP(*, v_uint64x2, vmul_vv_u64m1, 2) +OPENCV_HAL_IMPL_RVV_BIN_OP(/, v_uint64x2, vdivu_vv_u64m1, 2) +OPENCV_HAL_IMPL_RVV_BIN_OP(+, v_int64x2, vadd_vv_i64m1, 2) +OPENCV_HAL_IMPL_RVV_BIN_OP(-, v_int64x2, vsub_vv_i64m1, 2) +OPENCV_HAL_IMPL_RVV_BIN_OP(*, v_int64x2, vmul_vv_i64m1, 2) +OPENCV_HAL_IMPL_RVV_BIN_OP(/, v_int64x2, vdiv_vv_i64m1, 2) #if CV_SIMD128_64F -OPENCV_HAL_IMPL_RVV_BIN_OP(+, v_float64x2, vfadd_vv_f64m1, 64) -OPENCV_HAL_IMPL_RVV_BIN_OP(-, v_float64x2, vfsub_vv_f64m1, 64) -OPENCV_HAL_IMPL_RVV_BIN_OP(*, v_float64x2, vfmul_vv_f64m1, 64) -OPENCV_HAL_IMPL_RVV_BIN_OP(/, v_float64x2, vfdiv_vv_f64m1, 64) +OPENCV_HAL_IMPL_RVV_BIN_OP(+, v_float64x2, vfadd_vv_f64m1, 2) +OPENCV_HAL_IMPL_RVV_BIN_OP(-, v_float64x2, vfsub_vv_f64m1, 2) +OPENCV_HAL_IMPL_RVV_BIN_OP(*, v_float64x2, vfmul_vv_f64m1, 2) +OPENCV_HAL_IMPL_RVV_BIN_OP(/, v_float64x2, vfdiv_vv_f64m1, 2) #endif ////////////// Bitwise logic ////////////// -#define OPENCV_HAL_IMPL_RVV_LOGIC_OP(_Tpvec, suffix, width) \ -OPENCV_HAL_IMPL_RVV_BIN_OP(&, _Tpvec, vand_vv_##suffix##m1, width) \ -OPENCV_HAL_IMPL_RVV_BIN_OP(|, _Tpvec, vor_vv_##suffix##m1, width) \ -OPENCV_HAL_IMPL_RVV_BIN_OP(^, _Tpvec, vxor_vv_##suffix##m1, width) \ +#define OPENCV_HAL_IMPL_RVV_LOGIC_OP(_Tpvec, suffix, vl) \ +OPENCV_HAL_IMPL_RVV_BIN_OP(&, _Tpvec, vand_vv_##suffix##m1, vl) \ +OPENCV_HAL_IMPL_RVV_BIN_OP(|, _Tpvec, vor_vv_##suffix##m1, vl) \ +OPENCV_HAL_IMPL_RVV_BIN_OP(^, _Tpvec, vxor_vv_##suffix##m1, vl) \ inline _Tpvec operator ~ (const _Tpvec& a) \ { \ - vsetvlmax_e##width##m1(); \ - return _Tpvec(vnot_v_##suffix##m1(a)); \ + return _Tpvec(vnot_v_##suffix##m1(a, vl)); \ } -OPENCV_HAL_IMPL_RVV_LOGIC_OP(v_uint8x16, u8, 8) -OPENCV_HAL_IMPL_RVV_LOGIC_OP(v_int8x16, i8, 8) -OPENCV_HAL_IMPL_RVV_LOGIC_OP(v_uint16x8, u16, 16) -OPENCV_HAL_IMPL_RVV_LOGIC_OP(v_int16x8, i16, 16) -OPENCV_HAL_IMPL_RVV_LOGIC_OP(v_uint32x4, u32, 32) -OPENCV_HAL_IMPL_RVV_LOGIC_OP(v_int32x4, i32, 32) -OPENCV_HAL_IMPL_RVV_LOGIC_OP(v_uint64x2, u64, 64) -OPENCV_HAL_IMPL_RVV_LOGIC_OP(v_int64x2, i64, 64) +OPENCV_HAL_IMPL_RVV_LOGIC_OP(v_uint8x16, u8, 16) +OPENCV_HAL_IMPL_RVV_LOGIC_OP(v_int8x16, i8, 16) +OPENCV_HAL_IMPL_RVV_LOGIC_OP(v_uint16x8, u16, 8) +OPENCV_HAL_IMPL_RVV_LOGIC_OP(v_int16x8, i16, 8) +OPENCV_HAL_IMPL_RVV_LOGIC_OP(v_uint32x4, u32, 4) +OPENCV_HAL_IMPL_RVV_LOGIC_OP(v_int32x4, i32, 4) +OPENCV_HAL_IMPL_RVV_LOGIC_OP(v_uint64x2, u64, 2) +OPENCV_HAL_IMPL_RVV_LOGIC_OP(v_int64x2, i64, 2) #define OPENCV_HAL_IMPL_RVV_FLT_BIT_OP(bin_op, intrin) \ inline v_float32x4 operator bin_op (const v_float32x4& a, const v_float32x4& b) \ { \ - vsetvlmax_e32m1(); \ - return v_float32x4(vreinterpret_v_i32m1_f32m1(intrin(vreinterpret_v_f32m1_i32m1(a), vreinterpret_v_f32m1_i32m1(b)))); \ + return v_float32x4(vreinterpret_v_i32m1_f32m1(intrin(vreinterpret_v_f32m1_i32m1(a), vreinterpret_v_f32m1_i32m1(b), 4))); \ } \ inline v_float32x4& operator bin_op##= (v_float32x4& a, const v_float32x4& b) \ { \ - vsetvlmax_e32m1(); \ - a = v_float32x4(vreinterpret_v_i32m1_f32m1(intrin(vreinterpret_v_f32m1_i32m1(a), vreinterpret_v_f32m1_i32m1(b)))); \ + a = v_float32x4(vreinterpret_v_i32m1_f32m1(intrin(vreinterpret_v_f32m1_i32m1(a), vreinterpret_v_f32m1_i32m1(b), 4))); \ return a; \ } @@ -1268,21 +1201,18 @@ OPENCV_HAL_IMPL_RVV_FLT_BIT_OP(^, vxor_vv_i32m1) inline v_float32x4 operator ~ (const v_float32x4& a) { - vsetvlmax_e32m1(); - return v_float32x4(vreinterpret_v_i32m1_f32m1(vnot_v_i32m1(vreinterpret_v_f32m1_i32m1(a)))); + return v_float32x4(vreinterpret_v_i32m1_f32m1(vnot_v_i32m1(vreinterpret_v_f32m1_i32m1(a), 4))); } #if CV_SIMD128_64F #define OPENCV_HAL_IMPL_RVV_FLT64_BIT_OP(bin_op, intrin) \ inline v_float64x2 operator bin_op (const v_float64x2& a, const v_float64x2& b) \ { \ - vsetvlmax_e64m1(); \ - return v_float64x2(vreinterpret_v_i64m1_f64m1(intrin(vreinterpret_v_f64m1_i64m1(a), vreinterpret_v_f64m1_i64m1(b)))); \ + return v_float64x2(vreinterpret_v_i64m1_f64m1(intrin(vreinterpret_v_f64m1_i64m1(a), vreinterpret_v_f64m1_i64m1(b), 2))); \ } \ inline v_float64x2& operator bin_op##= (v_float64x2& a, const v_float64x2& b) \ { \ - vsetvlmax_e64m1(); \ - a = v_float64x2(vreinterpret_v_i64m1_f64m1(intrin(vreinterpret_v_f64m1_i64m1(a), vreinterpret_v_f64m1_i64m1(b)))); \ + a = v_float64x2(vreinterpret_v_i64m1_f64m1(intrin(vreinterpret_v_f64m1_i64m1(a), vreinterpret_v_f64m1_i64m1(b), 2))); \ return a; \ } @@ -1292,119 +1222,108 @@ OPENCV_HAL_IMPL_RVV_FLT64_BIT_OP(^, vxor_vv_i64m1) inline v_float64x2 operator ~ (const v_float64x2& a) { - vsetvlmax_e64m1(); - return v_float64x2(vreinterpret_v_i64m1_f64m1(vnot_v_i64m1(vreinterpret_v_f64m1_i64m1(a)))); + return v_float64x2(vreinterpret_v_i64m1_f64m1(vnot_v_i64m1(vreinterpret_v_f64m1_i64m1(a), 2))); } #endif ////////////// Bitwise shifts ////////////// -#define OPENCV_HAL_IMPL_RVV_UNSIGNED_SHIFT_OP(_Tpvec, suffix, width) \ +#define OPENCV_HAL_IMPL_RVV_UNSIGNED_SHIFT_OP(_Tpvec, suffix, vl) \ inline _Tpvec operator << (const _Tpvec& a, int n) \ { \ - vsetvlmax_e##width##m1(); \ - return _Tpvec(vsll_vx_##suffix##m1(a, uint8_t(n))); \ + return _Tpvec(vsll_vx_##suffix##m1(a, uint8_t(n), vl)); \ } \ inline _Tpvec operator >> (const _Tpvec& a, int n) \ { \ - vsetvlmax_e##width##m1(); \ - return _Tpvec(vsrl_vx_##suffix##m1(a, uint8_t(n))); \ + return _Tpvec(vsrl_vx_##suffix##m1(a, uint8_t(n), vl)); \ } \ template inline _Tpvec v_shl(const _Tpvec& a) \ { \ - vsetvlmax_e##width##m1(); \ - return _Tpvec(vsll_vx_##suffix##m1(a, uint8_t(n))); \ + return _Tpvec(vsll_vx_##suffix##m1(a, uint8_t(n), vl)); \ } \ template inline _Tpvec v_shr(const _Tpvec& a) \ { \ - vsetvlmax_e##width##m1(); \ - return _Tpvec(vsrl_vx_##suffix##m1(a, uint8_t(n))); \ + return _Tpvec(vsrl_vx_##suffix##m1(a, uint8_t(n), vl)); \ } -#define OPENCV_HAL_IMPL_RVV_SIGNED_SHIFT_OP(_Tpvec, suffix, width) \ +#define OPENCV_HAL_IMPL_RVV_SIGNED_SHIFT_OP(_Tpvec, suffix, vl) \ inline _Tpvec operator << (const _Tpvec& a, int n) \ { \ - vsetvlmax_e##width##m1(); \ - return _Tpvec(vsll_vx_##suffix##m1(a, uint8_t(n))); \ + return _Tpvec(vsll_vx_##suffix##m1(a, uint8_t(n), vl)); \ } \ inline _Tpvec operator >> (const _Tpvec& a, int n) \ { \ - vsetvlmax_e##width##m1(); \ - return _Tpvec(vsra_vx_##suffix##m1(a, uint8_t(n))); \ + return _Tpvec(vsra_vx_##suffix##m1(a, uint8_t(n), vl)); \ } \ template inline _Tpvec v_shl(const _Tpvec& a) \ { \ - vsetvlmax_e##width##m1(); \ - return _Tpvec(vsll_vx_##suffix##m1(a, uint8_t(n))); \ + return _Tpvec(vsll_vx_##suffix##m1(a, uint8_t(n), vl)); \ } \ template inline _Tpvec v_shr(const _Tpvec& a) \ { \ - vsetvlmax_e##width##m1(); \ - return _Tpvec(vsra_vx_##suffix##m1(a, uint8_t(n))); \ + return _Tpvec(vsra_vx_##suffix##m1(a, uint8_t(n), vl)); \ } -OPENCV_HAL_IMPL_RVV_UNSIGNED_SHIFT_OP(v_uint8x16, u8, 8) -OPENCV_HAL_IMPL_RVV_UNSIGNED_SHIFT_OP(v_uint16x8, u16, 16) -OPENCV_HAL_IMPL_RVV_UNSIGNED_SHIFT_OP(v_uint32x4, u32, 32) -OPENCV_HAL_IMPL_RVV_UNSIGNED_SHIFT_OP(v_uint64x2, u64, 64) -OPENCV_HAL_IMPL_RVV_SIGNED_SHIFT_OP(v_int8x16, i8, 8) -OPENCV_HAL_IMPL_RVV_SIGNED_SHIFT_OP(v_int16x8, i16, 16) -OPENCV_HAL_IMPL_RVV_SIGNED_SHIFT_OP(v_int32x4, i32, 32) -OPENCV_HAL_IMPL_RVV_SIGNED_SHIFT_OP(v_int64x2, i64, 64) +OPENCV_HAL_IMPL_RVV_UNSIGNED_SHIFT_OP(v_uint8x16, u8, 16) +OPENCV_HAL_IMPL_RVV_UNSIGNED_SHIFT_OP(v_uint16x8, u16, 8) +OPENCV_HAL_IMPL_RVV_UNSIGNED_SHIFT_OP(v_uint32x4, u32, 4) +OPENCV_HAL_IMPL_RVV_UNSIGNED_SHIFT_OP(v_uint64x2, u64, 2) +OPENCV_HAL_IMPL_RVV_SIGNED_SHIFT_OP(v_int8x16, i8, 16) +OPENCV_HAL_IMPL_RVV_SIGNED_SHIFT_OP(v_int16x8, i16, 8) +OPENCV_HAL_IMPL_RVV_SIGNED_SHIFT_OP(v_int32x4, i32, 4) +OPENCV_HAL_IMPL_RVV_SIGNED_SHIFT_OP(v_int64x2, i64, 2) ////////////// Comparison ////////////// -#define OPENCV_HAL_IMPL_RVV_INT_CMP_OP(_Tpvec, op, intrin, suffix, width) \ +#define OPENCV_HAL_IMPL_RVV_INT_CMP_OP(_Tpvec, op, intrin, suffix, vl) \ inline _Tpvec operator op (const _Tpvec& a, const _Tpvec& b) \ { \ - vsetvlmax_e##width##m1(); \ - return _Tpvec(vmerge_vxm_##suffix##m1(intrin(a, b), vzero_##suffix##m1(), 1)); \ + return _Tpvec(vmerge_vxm_##suffix##m1(intrin(a, b, vl), vmv_v_x_##suffix##m1(0, vl), 1, vl)); \ } -#define OPENCV_HAL_IMPL_RVV_FLOAT_CMP_OP(_Tpvec, op, intrin, suffix, width) \ +#define OPENCV_HAL_IMPL_RVV_FLOAT_CMP_OP(_Tpvec, op, intrin, suffix, vl) \ inline _Tpvec operator op (const _Tpvec& a, const _Tpvec& b) \ { \ - vsetvlmax_e##width##m1(); \ - return _Tpvec(vfmerge_vfm_##suffix##m1(intrin(a, b), vzero_##suffix##m1(), 1)); \ + return _Tpvec(vfmerge_vfm_##suffix##m1(intrin(a, b, vl), vfmv_v_f_##suffix##m1(0, vl), 1, vl)); \ } -#define OPENCV_HAL_IMPL_RVV_UNSIGNED_CMP(_Tpvec, suffix, width) \ -OPENCV_HAL_IMPL_RVV_INT_CMP_OP(_Tpvec, ==, vmseq_vv_##suffix##m1_b##width, suffix, width) \ -OPENCV_HAL_IMPL_RVV_INT_CMP_OP(_Tpvec, !=, vmsne_vv_##suffix##m1_b##width, suffix, width) \ -OPENCV_HAL_IMPL_RVV_INT_CMP_OP(_Tpvec, <, vmsltu_vv_##suffix##m1_b##width, suffix, width) \ -OPENCV_HAL_IMPL_RVV_INT_CMP_OP(_Tpvec, >, vmsgtu_vv_##suffix##m1_b##width, suffix, width) \ -OPENCV_HAL_IMPL_RVV_INT_CMP_OP(_Tpvec, <=, vmsleu_vv_##suffix##m1_b##width, suffix, width) \ -OPENCV_HAL_IMPL_RVV_INT_CMP_OP(_Tpvec, >=, vmsgeu_vv_##suffix##m1_b##width, suffix, width) +#define OPENCV_HAL_IMPL_RVV_UNSIGNED_CMP(_Tpvec, suffix, width, vl) \ +OPENCV_HAL_IMPL_RVV_INT_CMP_OP(_Tpvec, ==, vmseq_vv_##suffix##m1_b##width, suffix, vl) \ +OPENCV_HAL_IMPL_RVV_INT_CMP_OP(_Tpvec, !=, vmsne_vv_##suffix##m1_b##width, suffix, vl) \ +OPENCV_HAL_IMPL_RVV_INT_CMP_OP(_Tpvec, <, vmsltu_vv_##suffix##m1_b##width, suffix, vl) \ +OPENCV_HAL_IMPL_RVV_INT_CMP_OP(_Tpvec, >, vmsgtu_vv_##suffix##m1_b##width, suffix, vl) \ +OPENCV_HAL_IMPL_RVV_INT_CMP_OP(_Tpvec, <=, vmsleu_vv_##suffix##m1_b##width, suffix, vl) \ +OPENCV_HAL_IMPL_RVV_INT_CMP_OP(_Tpvec, >=, vmsgeu_vv_##suffix##m1_b##width, suffix, vl) -#define OPENCV_HAL_IMPL_RVV_SIGNED_CMP(_Tpvec, suffix, width) \ -OPENCV_HAL_IMPL_RVV_INT_CMP_OP(_Tpvec, ==, vmseq_vv_##suffix##m1_b##width, suffix, width) \ -OPENCV_HAL_IMPL_RVV_INT_CMP_OP(_Tpvec, !=, vmsne_vv_##suffix##m1_b##width, suffix, width) \ -OPENCV_HAL_IMPL_RVV_INT_CMP_OP(_Tpvec, <, vmslt_vv_##suffix##m1_b##width, suffix, width) \ -OPENCV_HAL_IMPL_RVV_INT_CMP_OP(_Tpvec, >, vmsgt_vv_##suffix##m1_b##width, suffix, width) \ -OPENCV_HAL_IMPL_RVV_INT_CMP_OP(_Tpvec, <=, vmsle_vv_##suffix##m1_b##width, suffix, width) \ -OPENCV_HAL_IMPL_RVV_INT_CMP_OP(_Tpvec, >=, vmsge_vv_##suffix##m1_b##width, suffix, width) +#define OPENCV_HAL_IMPL_RVV_SIGNED_CMP(_Tpvec, suffix, width, vl) \ +OPENCV_HAL_IMPL_RVV_INT_CMP_OP(_Tpvec, ==, vmseq_vv_##suffix##m1_b##width, suffix, vl) \ +OPENCV_HAL_IMPL_RVV_INT_CMP_OP(_Tpvec, !=, vmsne_vv_##suffix##m1_b##width, suffix, vl) \ +OPENCV_HAL_IMPL_RVV_INT_CMP_OP(_Tpvec, <, vmslt_vv_##suffix##m1_b##width, suffix, vl) \ +OPENCV_HAL_IMPL_RVV_INT_CMP_OP(_Tpvec, >, vmsgt_vv_##suffix##m1_b##width, suffix, vl) \ +OPENCV_HAL_IMPL_RVV_INT_CMP_OP(_Tpvec, <=, vmsle_vv_##suffix##m1_b##width, suffix, vl) \ +OPENCV_HAL_IMPL_RVV_INT_CMP_OP(_Tpvec, >=, vmsge_vv_##suffix##m1_b##width, suffix, vl) -#define OPENCV_HAL_IMPL_RVV_FLOAT_CMP(_Tpvec, suffix, width) \ -OPENCV_HAL_IMPL_RVV_FLOAT_CMP_OP(_Tpvec, ==, vmfeq_vv_##suffix##m1_b##width, suffix, width) \ -OPENCV_HAL_IMPL_RVV_FLOAT_CMP_OP(_Tpvec, !=, vmfne_vv_##suffix##m1_b##width, suffix, width) \ -OPENCV_HAL_IMPL_RVV_FLOAT_CMP_OP(_Tpvec, <, vmflt_vv_##suffix##m1_b##width, suffix, width) \ -OPENCV_HAL_IMPL_RVV_FLOAT_CMP_OP(_Tpvec, >, vmfgt_vv_##suffix##m1_b##width, suffix, width) \ -OPENCV_HAL_IMPL_RVV_FLOAT_CMP_OP(_Tpvec, <=, vmfle_vv_##suffix##m1_b##width, suffix, width) \ -OPENCV_HAL_IMPL_RVV_FLOAT_CMP_OP(_Tpvec, >=, vmfge_vv_##suffix##m1_b##width, suffix, width) +#define OPENCV_HAL_IMPL_RVV_FLOAT_CMP(_Tpvec, suffix, width, vl) \ +OPENCV_HAL_IMPL_RVV_FLOAT_CMP_OP(_Tpvec, ==, vmfeq_vv_##suffix##m1_b##width, suffix, vl) \ +OPENCV_HAL_IMPL_RVV_FLOAT_CMP_OP(_Tpvec, !=, vmfne_vv_##suffix##m1_b##width, suffix, vl) \ +OPENCV_HAL_IMPL_RVV_FLOAT_CMP_OP(_Tpvec, <, vmflt_vv_##suffix##m1_b##width, suffix, vl) \ +OPENCV_HAL_IMPL_RVV_FLOAT_CMP_OP(_Tpvec, >, vmfgt_vv_##suffix##m1_b##width, suffix, vl) \ +OPENCV_HAL_IMPL_RVV_FLOAT_CMP_OP(_Tpvec, <=, vmfle_vv_##suffix##m1_b##width, suffix, vl) \ +OPENCV_HAL_IMPL_RVV_FLOAT_CMP_OP(_Tpvec, >=, vmfge_vv_##suffix##m1_b##width, suffix, vl) -OPENCV_HAL_IMPL_RVV_UNSIGNED_CMP(v_uint8x16, u8, 8) -OPENCV_HAL_IMPL_RVV_UNSIGNED_CMP(v_uint16x8, u16, 16) -OPENCV_HAL_IMPL_RVV_UNSIGNED_CMP(v_uint32x4, u32, 32) -OPENCV_HAL_IMPL_RVV_UNSIGNED_CMP(v_uint64x2, u64, 64) -OPENCV_HAL_IMPL_RVV_SIGNED_CMP(v_int8x16, i8, 8) -OPENCV_HAL_IMPL_RVV_SIGNED_CMP(v_int16x8, i16, 16) -OPENCV_HAL_IMPL_RVV_SIGNED_CMP(v_int32x4, i32, 32) -OPENCV_HAL_IMPL_RVV_SIGNED_CMP(v_int64x2, i64, 64) -OPENCV_HAL_IMPL_RVV_FLOAT_CMP(v_float32x4, f32, 32) +OPENCV_HAL_IMPL_RVV_UNSIGNED_CMP(v_uint8x16, u8, 8, 16) +OPENCV_HAL_IMPL_RVV_UNSIGNED_CMP(v_uint16x8, u16, 16, 8) +OPENCV_HAL_IMPL_RVV_UNSIGNED_CMP(v_uint32x4, u32, 32, 4) +OPENCV_HAL_IMPL_RVV_UNSIGNED_CMP(v_uint64x2, u64, 64, 2) +OPENCV_HAL_IMPL_RVV_SIGNED_CMP(v_int8x16, i8, 8, 16) +OPENCV_HAL_IMPL_RVV_SIGNED_CMP(v_int16x8, i16, 16, 8) +OPENCV_HAL_IMPL_RVV_SIGNED_CMP(v_int32x4, i32, 32, 4) +OPENCV_HAL_IMPL_RVV_SIGNED_CMP(v_int64x2, i64, 64, 2) +OPENCV_HAL_IMPL_RVV_FLOAT_CMP(v_float32x4, f32, 32, 4) #if CV_SIMD128_64F -OPENCV_HAL_IMPL_RVV_FLOAT_CMP(v_float64x2, f64, 64) +OPENCV_HAL_IMPL_RVV_FLOAT_CMP(v_float64x2, f64, 64, 2) #endif inline v_float32x4 v_not_nan(const v_float32x4& a) @@ -1417,99 +1336,106 @@ inline v_float64x2 v_not_nan(const v_float64x2& a) ////////////// Min/Max ////////////// -#define OPENCV_HAL_IMPL_RVV_BIN_FUNC(_Tpvec, func, intrin, width) \ +#define OPENCV_HAL_IMPL_RVV_BIN_FUNC(_Tpvec, func, intrin, vl) \ inline _Tpvec func(const _Tpvec& a, const _Tpvec& b) \ { \ - vsetvlmax_e##width##m1(); \ - return _Tpvec(intrin(a, b)); \ + return _Tpvec(intrin(a, b, vl)); \ } -OPENCV_HAL_IMPL_RVV_BIN_FUNC(v_uint8x16, v_min, vminu_vv_u8m1, 8) -OPENCV_HAL_IMPL_RVV_BIN_FUNC(v_uint8x16, v_max, vmaxu_vv_u8m1, 8) -OPENCV_HAL_IMPL_RVV_BIN_FUNC(v_int8x16, v_min, vmin_vv_i8m1, 8) -OPENCV_HAL_IMPL_RVV_BIN_FUNC(v_int8x16, v_max, vmax_vv_i8m1, 8) -OPENCV_HAL_IMPL_RVV_BIN_FUNC(v_uint16x8, v_min, vminu_vv_u16m1, 16) -OPENCV_HAL_IMPL_RVV_BIN_FUNC(v_uint16x8, v_max, vmaxu_vv_u16m1, 16) -OPENCV_HAL_IMPL_RVV_BIN_FUNC(v_int16x8, v_min, vmin_vv_i16m1, 16) -OPENCV_HAL_IMPL_RVV_BIN_FUNC(v_int16x8, v_max, vmax_vv_i16m1, 16) -OPENCV_HAL_IMPL_RVV_BIN_FUNC(v_uint32x4, v_min, vminu_vv_u32m1, 32) -OPENCV_HAL_IMPL_RVV_BIN_FUNC(v_uint32x4, v_max, vmaxu_vv_u32m1, 32) -OPENCV_HAL_IMPL_RVV_BIN_FUNC(v_int32x4, v_min, vmin_vv_i32m1, 32) -OPENCV_HAL_IMPL_RVV_BIN_FUNC(v_int32x4, v_max, vmax_vv_i32m1, 32) -OPENCV_HAL_IMPL_RVV_BIN_FUNC(v_float32x4, v_min, vfmin_vv_f32m1, 32) -OPENCV_HAL_IMPL_RVV_BIN_FUNC(v_float32x4, v_max, vfmax_vv_f32m1, 32) -OPENCV_HAL_IMPL_RVV_BIN_FUNC(v_uint64x2, v_min, vminu_vv_u64m1, 64) -OPENCV_HAL_IMPL_RVV_BIN_FUNC(v_uint64x2, v_max, vmaxu_vv_u64m1, 64) -OPENCV_HAL_IMPL_RVV_BIN_FUNC(v_int64x2, v_min, vmin_vv_i64m1, 64) -OPENCV_HAL_IMPL_RVV_BIN_FUNC(v_int64x2, v_max, vmax_vv_i64m1, 64) +OPENCV_HAL_IMPL_RVV_BIN_FUNC(v_uint8x16, v_min, vminu_vv_u8m1, 16) +OPENCV_HAL_IMPL_RVV_BIN_FUNC(v_uint8x16, v_max, vmaxu_vv_u8m1, 16) +OPENCV_HAL_IMPL_RVV_BIN_FUNC(v_int8x16, v_min, vmin_vv_i8m1, 16) +OPENCV_HAL_IMPL_RVV_BIN_FUNC(v_int8x16, v_max, vmax_vv_i8m1, 16) +OPENCV_HAL_IMPL_RVV_BIN_FUNC(v_uint16x8, v_min, vminu_vv_u16m1, 8) +OPENCV_HAL_IMPL_RVV_BIN_FUNC(v_uint16x8, v_max, vmaxu_vv_u16m1, 8) +OPENCV_HAL_IMPL_RVV_BIN_FUNC(v_int16x8, v_min, vmin_vv_i16m1, 8) +OPENCV_HAL_IMPL_RVV_BIN_FUNC(v_int16x8, v_max, vmax_vv_i16m1, 8) +OPENCV_HAL_IMPL_RVV_BIN_FUNC(v_uint32x4, v_min, vminu_vv_u32m1, 4) +OPENCV_HAL_IMPL_RVV_BIN_FUNC(v_uint32x4, v_max, vmaxu_vv_u32m1, 4) +OPENCV_HAL_IMPL_RVV_BIN_FUNC(v_int32x4, v_min, vmin_vv_i32m1, 4) +OPENCV_HAL_IMPL_RVV_BIN_FUNC(v_int32x4, v_max, vmax_vv_i32m1, 4) +OPENCV_HAL_IMPL_RVV_BIN_FUNC(v_float32x4, v_min, vfmin_vv_f32m1, 4) +OPENCV_HAL_IMPL_RVV_BIN_FUNC(v_float32x4, v_max, vfmax_vv_f32m1, 4) +OPENCV_HAL_IMPL_RVV_BIN_FUNC(v_uint64x2, v_min, vminu_vv_u64m1, 2) +OPENCV_HAL_IMPL_RVV_BIN_FUNC(v_uint64x2, v_max, vmaxu_vv_u64m1, 2) +OPENCV_HAL_IMPL_RVV_BIN_FUNC(v_int64x2, v_min, vmin_vv_i64m1, 2) +OPENCV_HAL_IMPL_RVV_BIN_FUNC(v_int64x2, v_max, vmax_vv_i64m1, 2) #if CV_SIMD128_64F -OPENCV_HAL_IMPL_RVV_BIN_FUNC(v_float64x2, v_min, vfmin_vv_f64m1, 64) -OPENCV_HAL_IMPL_RVV_BIN_FUNC(v_float64x2, v_max, vfmax_vv_f64m1, 64) +OPENCV_HAL_IMPL_RVV_BIN_FUNC(v_float64x2, v_min, vfmin_vv_f64m1, 2) +OPENCV_HAL_IMPL_RVV_BIN_FUNC(v_float64x2, v_max, vfmax_vv_f64m1, 2) #endif ////////////// Arithmetics wrap ////////////// -OPENCV_HAL_IMPL_RVV_BIN_FUNC(v_uint8x16, v_add_wrap, vadd_vv_u8m1, 8) -OPENCV_HAL_IMPL_RVV_BIN_FUNC(v_int8x16, v_add_wrap, vadd_vv_i8m1, 8) -OPENCV_HAL_IMPL_RVV_BIN_FUNC(v_uint16x8, v_add_wrap, vadd_vv_u16m1, 16) -OPENCV_HAL_IMPL_RVV_BIN_FUNC(v_int16x8, v_add_wrap, vadd_vv_i16m1, 16) -OPENCV_HAL_IMPL_RVV_BIN_FUNC(v_uint8x16, v_sub_wrap, vsub_vv_u8m1, 8) -OPENCV_HAL_IMPL_RVV_BIN_FUNC(v_int8x16, v_sub_wrap, vsub_vv_i8m1, 8) -OPENCV_HAL_IMPL_RVV_BIN_FUNC(v_uint16x8, v_sub_wrap, vsub_vv_u16m1, 16) -OPENCV_HAL_IMPL_RVV_BIN_FUNC(v_int16x8, v_sub_wrap, vsub_vv_i16m1, 16) -OPENCV_HAL_IMPL_RVV_BIN_FUNC(v_uint8x16, v_mul_wrap, vmul_vv_u8m1, 8) -OPENCV_HAL_IMPL_RVV_BIN_FUNC(v_int8x16, v_mul_wrap, vmul_vv_i8m1, 8) -OPENCV_HAL_IMPL_RVV_BIN_FUNC(v_uint16x8, v_mul_wrap, vmul_vv_u16m1, 16) -OPENCV_HAL_IMPL_RVV_BIN_FUNC(v_int16x8, v_mul_wrap, vmul_vv_i16m1, 16) +OPENCV_HAL_IMPL_RVV_BIN_FUNC(v_uint8x16, v_add_wrap, vadd_vv_u8m1, 16) +OPENCV_HAL_IMPL_RVV_BIN_FUNC(v_int8x16, v_add_wrap, vadd_vv_i8m1, 16) +OPENCV_HAL_IMPL_RVV_BIN_FUNC(v_uint16x8, v_add_wrap, vadd_vv_u16m1, 8) +OPENCV_HAL_IMPL_RVV_BIN_FUNC(v_int16x8, v_add_wrap, vadd_vv_i16m1, 8) +OPENCV_HAL_IMPL_RVV_BIN_FUNC(v_uint8x16, v_sub_wrap, vsub_vv_u8m1, 16) +OPENCV_HAL_IMPL_RVV_BIN_FUNC(v_int8x16, v_sub_wrap, vsub_vv_i8m1, 16) +OPENCV_HAL_IMPL_RVV_BIN_FUNC(v_uint16x8, v_sub_wrap, vsub_vv_u16m1, 8) +OPENCV_HAL_IMPL_RVV_BIN_FUNC(v_int16x8, v_sub_wrap, vsub_vv_i16m1, 8) +OPENCV_HAL_IMPL_RVV_BIN_FUNC(v_uint8x16, v_mul_wrap, vmul_vv_u8m1, 16) +OPENCV_HAL_IMPL_RVV_BIN_FUNC(v_int8x16, v_mul_wrap, vmul_vv_i8m1, 16) +OPENCV_HAL_IMPL_RVV_BIN_FUNC(v_uint16x8, v_mul_wrap, vmul_vv_u16m1, 8) +OPENCV_HAL_IMPL_RVV_BIN_FUNC(v_int16x8, v_mul_wrap, vmul_vv_i16m1, 8) ////////////// Reduce ////////////// -#define OPENCV_HAL_IMPL_RVV_REDUCE_SUM(_Tpvec, _wTpvec, _nwTpvec, scalartype, suffix, wsuffix, wwidth, red) \ +#define OPENCV_HAL_IMPL_RVV_REDUCE_SUM(_Tpvec, _wTpvec, _nwTpvec, scalartype, suffix, wsuffix, vl, red) \ inline scalartype v_reduce_sum(const _Tpvec& a) \ { \ - vsetvlmax_e##wwidth##m1(); \ - _nwTpvec zero = vzero_##wsuffix##m1(); \ - _nwTpvec res = vzero_##wsuffix##m1(); \ - res = v##red##_vs_##suffix##m1_##wsuffix##m1(res, a, zero); \ + _nwTpvec zero = vmv_v_x_##wsuffix##m1(0, vl); \ + _nwTpvec res = vmv_v_x_##wsuffix##m1(0, vl); \ + res = v##red##_vs_##suffix##m1_##wsuffix##m1(res, a, zero, vl); \ return (scalartype)(_wTpvec(res).get0()); \ } OPENCV_HAL_IMPL_RVV_REDUCE_SUM(v_uint8x16, v_uint16x8, vuint16m1_t, unsigned, u8, u16, 16, wredsumu) OPENCV_HAL_IMPL_RVV_REDUCE_SUM(v_int8x16, v_int16x8, vint16m1_t, int, i8, i16, 16, wredsum) -OPENCV_HAL_IMPL_RVV_REDUCE_SUM(v_uint16x8, v_uint32x4, vuint32m1_t, unsigned, u16, u32, 32, wredsumu) -OPENCV_HAL_IMPL_RVV_REDUCE_SUM(v_int16x8, v_int32x4, vint32m1_t, int, i16, i32, 32, wredsum) -OPENCV_HAL_IMPL_RVV_REDUCE_SUM(v_uint32x4, v_uint64x2, vuint64m1_t, unsigned, u32, u64, 64, wredsumu) -OPENCV_HAL_IMPL_RVV_REDUCE_SUM(v_int32x4, v_int64x2, vint64m1_t, int, i32, i64, 64, wredsum) -OPENCV_HAL_IMPL_RVV_REDUCE_SUM(v_float32x4, v_float32x4, vfloat32m1_t, float, f32, f32, 32, fredsum) -OPENCV_HAL_IMPL_RVV_REDUCE_SUM(v_uint64x2, v_uint64x2, vuint64m1_t, uint64, u64, u64, 64, redsum) -OPENCV_HAL_IMPL_RVV_REDUCE_SUM(v_int64x2, v_int64x2, vint64m1_t, int64, i64, i64, 64, redsum) +OPENCV_HAL_IMPL_RVV_REDUCE_SUM(v_uint16x8, v_uint32x4, vuint32m1_t, unsigned, u16, u32, 8, wredsumu) +OPENCV_HAL_IMPL_RVV_REDUCE_SUM(v_int16x8, v_int32x4, vint32m1_t, int, i16, i32, 8, wredsum) +OPENCV_HAL_IMPL_RVV_REDUCE_SUM(v_uint32x4, v_uint64x2, vuint64m1_t, unsigned, u32, u64, 4, wredsumu) +OPENCV_HAL_IMPL_RVV_REDUCE_SUM(v_int32x4, v_int64x2, vint64m1_t, int, i32, i64, 4, wredsum) +OPENCV_HAL_IMPL_RVV_REDUCE_SUM(v_uint64x2, v_uint64x2, vuint64m1_t, uint64, u64, u64, 4, redsum) +OPENCV_HAL_IMPL_RVV_REDUCE_SUM(v_int64x2, v_int64x2, vint64m1_t, int64, i64, i64, 4, redsum) + +#define OPENCV_HAL_IMPL_RVV_REDUCE_SUM_FP(_Tpvec, _wTpvec, _nwTpvec, scalartype, suffix, wsuffix, vl, red) \ +inline scalartype v_reduce_sum(const _Tpvec& a) \ +{ \ + _nwTpvec zero = vfmv_v_f_##wsuffix##m1(0, vl); \ + _nwTpvec res = vfmv_v_f_##wsuffix##m1(0, vl); \ + res = v##red##_vs_##suffix##m1_##wsuffix##m1(res, a, zero, vl); \ + return (scalartype)(_wTpvec(res).get0()); \ +} + +OPENCV_HAL_IMPL_RVV_REDUCE_SUM_FP(v_float32x4, v_float32x4, vfloat32m1_t, float, f32, f32, 8, fredsum) #if CV_SIMD128_64F -OPENCV_HAL_IMPL_RVV_REDUCE_SUM(v_float64x2, v_float64x2, vfloat64m1_t, double, f64, f64, 64, fredsum) +OPENCV_HAL_IMPL_RVV_REDUCE_SUM_FP(v_float64x2, v_float64x2, vfloat64m1_t, double, f64, f64, 4, fredsum) #endif -#define OPENCV_HAL_IMPL_RVV_REDUCE(_Tpvec, func, scalartype, suffix, width, red) \ +#define OPENCV_HAL_IMPL_RVV_REDUCE(_Tpvec, func, scalartype, suffix, vl, red) \ inline scalartype v_reduce_##func(const _Tpvec& a) \ { \ - vsetvlmax_e##width##m1(); \ - _Tpvec res = _Tpvec(v##red##_vs_##suffix##m1_##suffix##m1(a, a, a)); \ + _Tpvec res = _Tpvec(v##red##_vs_##suffix##m1_##suffix##m1(a, a, a, vl)); \ return scalartype(res.get0()); \ } -OPENCV_HAL_IMPL_RVV_REDUCE(v_uint8x16, min, uchar, u8, 8, redminu) -OPENCV_HAL_IMPL_RVV_REDUCE(v_int8x16, min, schar, i8, 8, redmin) -OPENCV_HAL_IMPL_RVV_REDUCE(v_uint16x8, min, ushort, u16, 16, redminu) -OPENCV_HAL_IMPL_RVV_REDUCE(v_int16x8, min, short, i16, 16, redmin) -OPENCV_HAL_IMPL_RVV_REDUCE(v_uint32x4, min, unsigned, u32, 32, redminu) -OPENCV_HAL_IMPL_RVV_REDUCE(v_int32x4, min, int, i32, 32, redmin) -OPENCV_HAL_IMPL_RVV_REDUCE(v_float32x4, min, float, f32, 32, fredmin) -OPENCV_HAL_IMPL_RVV_REDUCE(v_uint8x16, max, uchar, u8, 8, redmaxu) -OPENCV_HAL_IMPL_RVV_REDUCE(v_int8x16, max, schar, i8, 8, redmax) -OPENCV_HAL_IMPL_RVV_REDUCE(v_uint16x8, max, ushort, u16, 16, redmaxu) -OPENCV_HAL_IMPL_RVV_REDUCE(v_int16x8, max, short, i16, 16, redmax) -OPENCV_HAL_IMPL_RVV_REDUCE(v_uint32x4, max, unsigned, u32, 32, redmaxu) -OPENCV_HAL_IMPL_RVV_REDUCE(v_int32x4, max, int, i32, 32, redmax) -OPENCV_HAL_IMPL_RVV_REDUCE(v_float32x4, max, float, f32, 32, fredmax) +OPENCV_HAL_IMPL_RVV_REDUCE(v_uint8x16, min, uchar, u8, 16, redminu) +OPENCV_HAL_IMPL_RVV_REDUCE(v_int8x16, min, schar, i8, 16, redmin) +OPENCV_HAL_IMPL_RVV_REDUCE(v_uint16x8, min, ushort, u16, 8, redminu) +OPENCV_HAL_IMPL_RVV_REDUCE(v_int16x8, min, short, i16, 8, redmin) +OPENCV_HAL_IMPL_RVV_REDUCE(v_uint32x4, min, unsigned, u32, 4, redminu) +OPENCV_HAL_IMPL_RVV_REDUCE(v_int32x4, min, int, i32, 4, redmin) +OPENCV_HAL_IMPL_RVV_REDUCE(v_float32x4, min, float, f32, 4, fredmin) +OPENCV_HAL_IMPL_RVV_REDUCE(v_uint8x16, max, uchar, u8, 16, redmaxu) +OPENCV_HAL_IMPL_RVV_REDUCE(v_int8x16, max, schar, i8, 16, redmax) +OPENCV_HAL_IMPL_RVV_REDUCE(v_uint16x8, max, ushort, u16, 8, redmaxu) +OPENCV_HAL_IMPL_RVV_REDUCE(v_int16x8, max, short, i16, 8, redmax) +OPENCV_HAL_IMPL_RVV_REDUCE(v_uint32x4, max, unsigned, u32, 4, redmaxu) +OPENCV_HAL_IMPL_RVV_REDUCE(v_int32x4, max, int, i32, 4, redmax) +OPENCV_HAL_IMPL_RVV_REDUCE(v_float32x4, max, float, f32, 4, fredmax) inline v_float32x4 v_reduce_sum4(const v_float32x4& a, const v_float32x4& b, @@ -1522,16 +1448,14 @@ inline v_float32x4 v_reduce_sum4(const v_float32x4& a, const v_float32x4& b, v_reduce_sum(c), v_reduce_sum(d) }; - vsetvlmax_e32m1(); - return v_float32x4(vle32_v_f32m1(elems)); + return v_float32x4(vle32_v_f32m1(elems, 4)); } ////////////// Square-Root ////////////// inline v_float32x4 v_sqrt(const v_float32x4& x) { - vsetvlmax_e32m1(); - return v_float32x4(vfsqrt_v_f32m1(x)); + return v_float32x4(vfsqrt_v_f32m1(x, 4)); } inline v_float32x4 v_invsqrt(const v_float32x4& x) @@ -1543,8 +1467,7 @@ inline v_float32x4 v_invsqrt(const v_float32x4& x) #if CV_SIMD128_64F inline v_float64x2 v_sqrt(const v_float64x2& x) { - vsetvlmax_e64m1(); - return v_float64x2(vfsqrt_v_f64m1(x)); + return v_float64x2(vfsqrt_v_f64m1(x, 4)); } inline v_float64x2 v_invsqrt(const v_float64x2& x) @@ -1556,29 +1479,25 @@ inline v_float64x2 v_invsqrt(const v_float64x2& x) inline v_float32x4 v_magnitude(const v_float32x4& a, const v_float32x4& b) { - vsetvlmax_e32m1(); - v_float32x4 x(vfmacc_vv_f32m1(vfmul_vv_f32m1(a, a), b, b)); + v_float32x4 x(vfmacc_vv_f32m1(vfmul_vv_f32m1(a, a, 4), b, b, 4)); return v_sqrt(x); } inline v_float32x4 v_sqr_magnitude(const v_float32x4& a, const v_float32x4& b) { - vsetvlmax_e32m1(); - return v_float32x4(vfmacc_vv_f32m1(vfmul_vv_f32m1(a, a), b, b)); + return v_float32x4(vfmacc_vv_f32m1(vfmul_vv_f32m1(a, a, 4), b, b, 4)); } #if CV_SIMD128_64F inline v_float64x2 v_magnitude(const v_float64x2& a, const v_float64x2& b) { - vsetvlmax_e64m1(); - v_float64x2 x(vfmacc_vv_f64m1(vfmul_vv_f64m1(a, a), b, b)); + v_float64x2 x(vfmacc_vv_f64m1(vfmul_vv_f64m1(a, a, 2), b, b, 2)); return v_sqrt(x); } inline v_float64x2 v_sqr_magnitude(const v_float64x2& a, const v_float64x2& b) { - vsetvlmax_e64m1(); - return v_float64x2(vfmacc_vv_f64m1(vfmul_vv_f64m1(a, a), b, b)); + return v_float64x2(vfmacc_vv_f64m1(vfmul_vv_f64m1(a, a, 2), b, b, 2)); } #endif @@ -1586,13 +1505,11 @@ inline v_float64x2 v_sqr_magnitude(const v_float64x2& a, const v_float64x2& b) inline v_float32x4 v_fma(const v_float32x4& a, const v_float32x4& b, const v_float32x4& c) { - vsetvlmax_e32m1(); - return v_float32x4(vfmacc_vv_f32m1(c, a, b)); + return v_float32x4(vfmacc_vv_f32m1(c, a, b, 4)); } inline v_int32x4 v_fma(const v_int32x4& a, const v_int32x4& b, const v_int32x4& c) { - vsetvlmax_e32m1(); - return v_int32x4(vmacc_vv_i32m1(c, a, b)); + return v_int32x4(vmacc_vv_i32m1(c, a, b, 4)); } inline v_float32x4 v_muladd(const v_float32x4& a, const v_float32x4& b, const v_float32x4& c) @@ -1608,8 +1525,7 @@ inline v_int32x4 v_muladd(const v_int32x4& a, const v_int32x4& b, const v_int32x #if CV_SIMD128_64F inline v_float64x2 v_fma(const v_float64x2& a, const v_float64x2& b, const v_float64x2& c) { - vsetvlmax_e64m1(); - return v_float64x2(vfmacc_vv_f64m1(c, a, b)); + return v_float64x2(vfmacc_vv_f64m1(c, a, b, 2)); } inline v_float64x2 v_muladd(const v_float64x2& a, const v_float64x2& b, const v_float64x2& c) @@ -1620,24 +1536,22 @@ inline v_float64x2 v_muladd(const v_float64x2& a, const v_float64x2& b, const v_ ////////////// Check all/any ////////////// -#define OPENCV_HAL_IMPL_RVV_CHECK_ALLANY(_Tpvec, suffix, shift, width) \ +#define OPENCV_HAL_IMPL_RVV_CHECK_ALLANY(_Tpvec, suffix, shift, vl) \ inline bool v_check_all(const _Tpvec& a) \ { \ - vsetvlmax_e##width##m1(); \ - v_uint64x2 v = v_uint64x2((vuint64m1_t)vsrl_vx_##suffix##m1(vnot_v_##suffix##m1(a), shift)); \ + v_uint64x2 v = v_uint64x2((vuint64m1_t)vsrl_vx_##suffix##m1(vnot_v_##suffix##m1(a, vl), shift, vl)); \ return (v.val[0] | v.val[1]) == 0; \ } \ inline bool v_check_any(const _Tpvec& a) \ { \ - vsetvlmax_e##width##m1(); \ - v_uint64x2 v = v_uint64x2((vuint64m1_t)vsrl_vx_##suffix##m1(a, shift)); \ + v_uint64x2 v = v_uint64x2((vuint64m1_t)vsrl_vx_##suffix##m1(a, shift, vl)); \ return (v.val[0] | v.val[1]) != 0; \ } -OPENCV_HAL_IMPL_RVV_CHECK_ALLANY(v_uint8x16, u8, 7, 8) -OPENCV_HAL_IMPL_RVV_CHECK_ALLANY(v_uint16x8, u16, 15, 16) -OPENCV_HAL_IMPL_RVV_CHECK_ALLANY(v_uint32x4, u32, 31, 32) -OPENCV_HAL_IMPL_RVV_CHECK_ALLANY(v_uint64x2, u64, 63, 64) +OPENCV_HAL_IMPL_RVV_CHECK_ALLANY(v_uint8x16, u8, 7, 16) +OPENCV_HAL_IMPL_RVV_CHECK_ALLANY(v_uint16x8, u16, 15, 8) +OPENCV_HAL_IMPL_RVV_CHECK_ALLANY(v_uint32x4, u32, 31, 4) +OPENCV_HAL_IMPL_RVV_CHECK_ALLANY(v_uint64x2, u64, 63, 2) inline bool v_check_all(const v_int8x16& a) @@ -1690,16 +1604,15 @@ OPENCV_HAL_IMPL_RVV_ABSDIFF(v_float64x2, absdiff) OPENCV_HAL_IMPL_RVV_ABSDIFF(v_int8x16, absdiffs) OPENCV_HAL_IMPL_RVV_ABSDIFF(v_int16x8, absdiffs) -#define OPENCV_HAL_IMPL_RVV_ABSDIFF_S(_Tpvec, _rTpvec, _nwTpvec, sub, rshr, width) \ +#define OPENCV_HAL_IMPL_RVV_ABSDIFF_S(_Tpvec, _rTpvec, _nwTpvec, sub, rshr, vl) \ inline _rTpvec v_absdiff(const _Tpvec& a, const _Tpvec& b) \ { \ - vsetvlmax_e##width##m1(); \ - return _rTpvec(rshr((_nwTpvec)sub(v_max(a, b), v_min(a, b)), 0)); \ + return _rTpvec(rshr((_nwTpvec)sub(v_max(a, b), v_min(a, b), vl), 0, vl)); \ } -OPENCV_HAL_IMPL_RVV_ABSDIFF_S(v_int8x16, v_uint8x16, vuint16m2_t, vwsub_vv_i16m2, vnclipu_wx_u8m1, 8) -OPENCV_HAL_IMPL_RVV_ABSDIFF_S(v_int16x8, v_uint16x8, vuint32m2_t, vwsub_vv_i32m2, vnclipu_wx_u16m1, 16) -OPENCV_HAL_IMPL_RVV_ABSDIFF_S(v_int32x4, v_uint32x4, vuint64m2_t, vwsub_vv_i64m2, vnclipu_wx_u32m1, 32) +OPENCV_HAL_IMPL_RVV_ABSDIFF_S(v_int8x16, v_uint8x16, vuint16m2_t, vwsub_vv_i16m2, vnclipu_wx_u8m1, 16) +OPENCV_HAL_IMPL_RVV_ABSDIFF_S(v_int16x8, v_uint16x8, vuint32m2_t, vwsub_vv_i32m2, vnclipu_wx_u16m1, 8) +OPENCV_HAL_IMPL_RVV_ABSDIFF_S(v_int32x4, v_uint32x4, vuint64m2_t, vwsub_vv_i64m2, vnclipu_wx_u32m1, 4) #define OPENCV_HAL_IMPL_RVV_ABS(_Tprvec, _Tpvec, suffix) \ inline _Tprvec v_abs(const _Tpvec& a) \ @@ -1732,149 +1645,152 @@ OPENCV_HAL_IMPL_RVV_REDUCE_SAD(v_float32x4, float) ////////////// Select ////////////// -#define OPENCV_HAL_IMPL_RVV_SELECT(_Tpvec, merge, ne, width) \ +#define OPENCV_HAL_IMPL_RVV_SELECT(_Tpvec, merge, ne, vl) \ inline _Tpvec v_select(const _Tpvec& mask, const _Tpvec& a, const _Tpvec& b) \ { \ - vsetvlmax_e##width##m1(); \ - return _Tpvec(merge(ne(mask, 0), b, a)); \ + return _Tpvec(merge(ne(mask, 0, vl), b, a, vl)); \ } -OPENCV_HAL_IMPL_RVV_SELECT(v_uint8x16, vmerge_vvm_u8m1, vmsne_vx_u8m1_b8, 8) -OPENCV_HAL_IMPL_RVV_SELECT(v_int8x16, vmerge_vvm_i8m1, vmsne_vx_i8m1_b8, 8) -OPENCV_HAL_IMPL_RVV_SELECT(v_uint16x8, vmerge_vvm_u16m1, vmsne_vx_u16m1_b16, 16) -OPENCV_HAL_IMPL_RVV_SELECT(v_int16x8, vmerge_vvm_i16m1, vmsne_vx_i16m1_b16, 16) -OPENCV_HAL_IMPL_RVV_SELECT(v_uint32x4, vmerge_vvm_u32m1, vmsne_vx_u32m1_b32, 32) -OPENCV_HAL_IMPL_RVV_SELECT(v_int32x4, vmerge_vvm_i32m1, vmsne_vx_i32m1_b32, 32) -OPENCV_HAL_IMPL_RVV_SELECT(v_float32x4, vmerge_vvm_f32m1, vmfne_vf_f32m1_b32, 32) +OPENCV_HAL_IMPL_RVV_SELECT(v_uint8x16, vmerge_vvm_u8m1, vmsne_vx_u8m1_b8, 16) +OPENCV_HAL_IMPL_RVV_SELECT(v_int8x16, vmerge_vvm_i8m1, vmsne_vx_i8m1_b8, 16) +OPENCV_HAL_IMPL_RVV_SELECT(v_uint16x8, vmerge_vvm_u16m1, vmsne_vx_u16m1_b16, 8) +OPENCV_HAL_IMPL_RVV_SELECT(v_int16x8, vmerge_vvm_i16m1, vmsne_vx_i16m1_b16, 8) +OPENCV_HAL_IMPL_RVV_SELECT(v_uint32x4, vmerge_vvm_u32m1, vmsne_vx_u32m1_b32, 4) +OPENCV_HAL_IMPL_RVV_SELECT(v_int32x4, vmerge_vvm_i32m1, vmsne_vx_i32m1_b32, 4) +OPENCV_HAL_IMPL_RVV_SELECT(v_float32x4, vmerge_vvm_f32m1, vmfne_vf_f32m1_b32, 4) #if CV_SIMD128_64F -OPENCV_HAL_IMPL_RVV_SELECT(v_float64x2, vmerge_vvm_f64m1, vmfne_vf_f64m1_b64, 64) +OPENCV_HAL_IMPL_RVV_SELECT(v_float64x2, vmerge_vvm_f64m1, vmfne_vf_f64m1_b64, 2) #endif ////////////// Rotate shift ////////////// -#define OPENCV_HAL_IMPL_RVV_ROTATE_OP(_Tpvec, suffix, width) \ +#define OPENCV_HAL_IMPL_RVV_ROTATE_INTEGER(_Tpvec, suffix, vl) \ template inline _Tpvec v_rotate_right(const _Tpvec& a) \ { \ - vsetvlmax_e##width##m1(); \ - return _Tpvec(vslidedown_vx_##suffix##m1(vzero_##suffix##m1(), a, n)); \ + return _Tpvec(vslidedown_vx_##suffix##m1(vmv_v_x_##suffix##m1(0, vl), a, n, vl)); \ } \ template inline _Tpvec v_rotate_left(const _Tpvec& a) \ { \ - vsetvlmax_e##width##m1(); \ - return _Tpvec(vslideup_vx_##suffix##m1(vzero_##suffix##m1(), a, n)); \ + return _Tpvec(vslideup_vx_##suffix##m1(vmv_v_x_##suffix##m1(0, vl), a, n, vl)); \ } \ template<> inline _Tpvec v_rotate_left<0>(const _Tpvec& a) \ { return a; } \ template inline _Tpvec v_rotate_right(const _Tpvec& a, const _Tpvec& b) \ { \ - vsetvlmax_e##width##m1(); \ - return _Tpvec(vslideup_vx_##suffix##m1(vslidedown_vx_##suffix##m1(vzero_##suffix##m1(), a, n), b, _Tpvec::nlanes - n)); \ + return _Tpvec(vslideup_vx_##suffix##m1(vslidedown_vx_##suffix##m1(vmv_v_x_##suffix##m1(0, vl), a, n, vl), b, _Tpvec::nlanes - n, vl)); \ } \ template inline _Tpvec v_rotate_left(const _Tpvec& a, const _Tpvec& b) \ { \ - vsetvlmax_e##width##m1(); \ - return _Tpvec(vslideup_vx_##suffix##m1(vslidedown_vx_##suffix##m1(vzero_##suffix##m1(), b, _Tpvec::nlanes - n), a, n)); \ + return _Tpvec(vslideup_vx_##suffix##m1(vslidedown_vx_##suffix##m1(vmv_v_x_##suffix##m1(0, vl), b, _Tpvec::nlanes - n, vl), a, n, vl)); \ } \ template<> inline _Tpvec v_rotate_left<0>(const _Tpvec& a, const _Tpvec& b) \ { CV_UNUSED(b); return a; } +OPENCV_HAL_IMPL_RVV_ROTATE_INTEGER(v_uint8x16, u8, 16) +OPENCV_HAL_IMPL_RVV_ROTATE_INTEGER(v_int8x16, i8, 16) +OPENCV_HAL_IMPL_RVV_ROTATE_INTEGER(v_uint16x8, u16, 8) +OPENCV_HAL_IMPL_RVV_ROTATE_INTEGER(v_int16x8, i16, 8) +OPENCV_HAL_IMPL_RVV_ROTATE_INTEGER(v_uint32x4, u32, 4) +OPENCV_HAL_IMPL_RVV_ROTATE_INTEGER(v_int32x4, i32, 4) +OPENCV_HAL_IMPL_RVV_ROTATE_INTEGER(v_uint64x2, u64, 2) +OPENCV_HAL_IMPL_RVV_ROTATE_INTEGER(v_int64x2, i64, 2) -OPENCV_HAL_IMPL_RVV_ROTATE_OP(v_uint8x16, u8, 8) -OPENCV_HAL_IMPL_RVV_ROTATE_OP(v_int8x16, i8, 8) -OPENCV_HAL_IMPL_RVV_ROTATE_OP(v_uint16x8, u16, 16) -OPENCV_HAL_IMPL_RVV_ROTATE_OP(v_int16x8, i16, 16) -OPENCV_HAL_IMPL_RVV_ROTATE_OP(v_uint32x4, u32, 32) -OPENCV_HAL_IMPL_RVV_ROTATE_OP(v_int32x4, i32, 32) -OPENCV_HAL_IMPL_RVV_ROTATE_OP(v_float32x4, f32, 32) -OPENCV_HAL_IMPL_RVV_ROTATE_OP(v_uint64x2, u64, 64) -OPENCV_HAL_IMPL_RVV_ROTATE_OP(v_int64x2, i64, 64) +#define OPENCV_HAL_IMPL_RVV_ROTATE_FP(_Tpvec, suffix, vl) \ +template inline _Tpvec v_rotate_right(const _Tpvec& a) \ +{ \ + return _Tpvec(vslidedown_vx_##suffix##m1(vfmv_v_f_##suffix##m1(0, vl), a, n, vl)); \ +} \ +template inline _Tpvec v_rotate_left(const _Tpvec& a) \ +{ \ + return _Tpvec(vslideup_vx_##suffix##m1(vfmv_v_f_##suffix##m1(0, vl), a, n, vl)); \ +} \ +template<> inline _Tpvec v_rotate_left<0>(const _Tpvec& a) \ +{ return a; } \ +template inline _Tpvec v_rotate_right(const _Tpvec& a, const _Tpvec& b) \ +{ \ + return _Tpvec(vslideup_vx_##suffix##m1(vslidedown_vx_##suffix##m1(vfmv_v_f_##suffix##m1(0, vl), a, n, vl), b, _Tpvec::nlanes - n, vl)); \ +} \ +template inline _Tpvec v_rotate_left(const _Tpvec& a, const _Tpvec& b) \ +{ \ + return _Tpvec(vslideup_vx_##suffix##m1(vslidedown_vx_##suffix##m1(vfmv_v_f_##suffix##m1(0, vl), b, _Tpvec::nlanes - n, vl), a, n, vl)); \ +} \ +template<> inline _Tpvec v_rotate_left<0>(const _Tpvec& a, const _Tpvec& b) \ +{ CV_UNUSED(b); return a; } + +OPENCV_HAL_IMPL_RVV_ROTATE_FP(v_float32x4, f32, 4) #if CV_SIMD128_64F -OPENCV_HAL_IMPL_RVV_ROTATE_OP(v_float64x2, f64, 64) +OPENCV_HAL_IMPL_RVV_ROTATE_FP(v_float64x2, f64, 2) #endif ////////////// Convert to float ////////////// inline v_float32x4 v_cvt_f32(const v_int32x4& a) { - vsetvlmax_e32m1(); - return v_float32x4(vfcvt_f_x_v_f32m1(a)); + return v_float32x4(vfcvt_f_x_v_f32m1(a, 4)); } #if CV_SIMD128_64F inline v_float32x4 v_cvt_f32(const v_float64x2& a) { double arr[4] = {a.val[0], a.val[1], 0, 0}; - vsetvlmax_e64m2(); - vfloat64m2_t tmp = vle64_v_f64m2(arr); - vsetvlmax_e32m1(); - return v_float32x4(vfncvt_f_f_w_f32m1(tmp)); + vfloat64m2_t tmp = vle64_v_f64m2(arr, 4); + return v_float32x4(vfncvt_f_f_w_f32m1(tmp, 4)); } inline v_float32x4 v_cvt_f32(const v_float64x2& a, const v_float64x2& b) { double arr[4] = {a.val[0], a.val[1], b.val[0], b.val[1]}; - vsetvlmax_e64m2(); - vfloat64m2_t tmp = vle64_v_f64m2(arr); - vsetvlmax_e32m1(); - return v_float32x4(vfncvt_f_f_w_f32m1(tmp)); + vfloat64m2_t tmp = vle64_v_f64m2(arr, 4); + return v_float32x4(vfncvt_f_f_w_f32m1(tmp, 4)); } inline v_float64x2 v_cvt_f64(const v_int32x4& a) { double CV_DECL_ALIGNED(32) ptr[4] = {0}; - vsetvlmax_e64m2(); - vse64_v_f64m2(ptr, vfwcvt_f_x_v_f64m2(a)); + vse64_v_f64m2(ptr, vfwcvt_f_x_v_f64m2(a, 4), 4); double CV_DECL_ALIGNED(32) elems[2] = { ptr[0], ptr[1] }; - vsetvlmax_e64m1(); - return v_float64x2(vle64_v_f64m1(elems)); + return v_float64x2(vle64_v_f64m1(elems, 2)); } inline v_float64x2 v_cvt_f64_high(const v_int32x4& a) { double CV_DECL_ALIGNED(32) ptr[4] = {0}; - vsetvlmax_e64m2(); - vse64_v_f64m2(ptr, vfwcvt_f_x_v_f64m2(a)); + vse64_v_f64m2(ptr, vfwcvt_f_x_v_f64m2(a, 4), 4); double CV_DECL_ALIGNED(32) elems[2] = { ptr[2], ptr[3] }; - vsetvlmax_e64m1(); - return v_float64x2(vle64_v_f64m1(elems)); + return v_float64x2(vle64_v_f64m1(elems, 2)); } inline v_float64x2 v_cvt_f64(const v_float32x4& a) { double CV_DECL_ALIGNED(32) ptr[4] = {0}; - vsetvlmax_e64m2(); - vse64_v_f64m2(ptr, vfwcvt_f_f_v_f64m2(a)); + vse64_v_f64m2(ptr, vfwcvt_f_f_v_f64m2(a, 4), 4); double CV_DECL_ALIGNED(32) elems[2] = { ptr[0], ptr[1] }; - vsetvlmax_e64m1(); - return v_float64x2(vle64_v_f64m1(elems)); + return v_float64x2(vle64_v_f64m1(elems, 2)); } inline v_float64x2 v_cvt_f64_high(const v_float32x4& a) { double CV_DECL_ALIGNED(32) ptr[4] = {0}; - vsetvlmax_e64m2(); - vse64_v_f64m2(ptr, vfwcvt_f_f_v_f64m2(a)); + vse64_v_f64m2(ptr, vfwcvt_f_f_v_f64m2(a, 4), 4); double CV_DECL_ALIGNED(32) elems[2] = { ptr[2], ptr[3] }; - vsetvlmax_e64m1(); - return v_float64x2(vle64_v_f64m1(elems)); + return v_float64x2(vle64_v_f64m1(elems, 2)); } inline v_float64x2 v_cvt_f64(const v_int64x2& a) { - vsetvlmax_e64m1(); - return v_float64x2(vfcvt_f_x_v_f64m1(a)); + return v_float64x2(vfcvt_f_x_v_f64m1(a, 2)); } #endif @@ -1947,7 +1863,7 @@ OPENCV_HAL_IMPL_RVV_TRANSPOSE4x4(float32x4, float, f32) ////////////// Reverse ////////////// -#define OPENCV_HAL_IMPL_RVV_REVERSE(_Tpvec, _Tp, width, suffix) \ +#define OPENCV_HAL_IMPL_RVV_REVERSE(_Tpvec, _Tp, suffix) \ inline _Tpvec v_reverse(const _Tpvec& a) \ { \ _Tp CV_DECL_ALIGNED(32) ptr[_Tpvec::nlanes] = {0}; \ @@ -1960,84 +1876,80 @@ inline _Tpvec v_reverse(const _Tpvec& a) \ return v_load(ptr); \ } -OPENCV_HAL_IMPL_RVV_REVERSE(v_uint8x16, uchar, 8, u8) -OPENCV_HAL_IMPL_RVV_REVERSE(v_int8x16, schar, 8, i8) -OPENCV_HAL_IMPL_RVV_REVERSE(v_uint16x8, ushort, 16, u16) -OPENCV_HAL_IMPL_RVV_REVERSE(v_int16x8, short, 16, i16) -OPENCV_HAL_IMPL_RVV_REVERSE(v_uint32x4, unsigned, 32, u32) -OPENCV_HAL_IMPL_RVV_REVERSE(v_int32x4, int, 32, i32) -OPENCV_HAL_IMPL_RVV_REVERSE(v_float32x4, float, 32, f32) -OPENCV_HAL_IMPL_RVV_REVERSE(v_uint64x2, uint64, 64, u64) -OPENCV_HAL_IMPL_RVV_REVERSE(v_int64x2, int64, 64, i64) +OPENCV_HAL_IMPL_RVV_REVERSE(v_uint8x16, uchar, u8) +OPENCV_HAL_IMPL_RVV_REVERSE(v_int8x16, schar, i8) +OPENCV_HAL_IMPL_RVV_REVERSE(v_uint16x8, ushort, u16) +OPENCV_HAL_IMPL_RVV_REVERSE(v_int16x8, short, i16) +OPENCV_HAL_IMPL_RVV_REVERSE(v_uint32x4, unsigned, u32) +OPENCV_HAL_IMPL_RVV_REVERSE(v_int32x4, int, i32) +OPENCV_HAL_IMPL_RVV_REVERSE(v_float32x4, float, f32) +OPENCV_HAL_IMPL_RVV_REVERSE(v_uint64x2, uint64, u64) +OPENCV_HAL_IMPL_RVV_REVERSE(v_int64x2, int64, i64) #if CV_SIMD128_64F -OPENCV_HAL_IMPL_RVV_REVERSE(v_float64x2, double, 64, f64) +OPENCV_HAL_IMPL_RVV_REVERSE(v_float64x2, double, f64) #endif //////////// Value reordering //////////// -#define OPENCV_HAL_IMPL_RVV_EXPAND(_Tpwvec, _Tp, _Tpvec, width, suffix, wcvt) \ +#define OPENCV_HAL_IMPL_RVV_EXPAND(_Tpwvec, _Tp, _Tpvec, width, suffix, wcvt, vl) \ inline void v_expand(const _Tpvec& a, _Tpwvec& b0, _Tpwvec& b1) \ { \ _Tp CV_DECL_ALIGNED(32) lptr[_Tpvec::nlanes/2] = {0}; \ _Tp CV_DECL_ALIGNED(32) hptr[_Tpvec::nlanes/2] = {0}; \ v_store_low(lptr, a); \ v_store_high(hptr, a); \ - b0 = _Tpwvec(wcvt(vle##width##_v_##suffix##mf2(lptr))); \ - b1 = _Tpwvec(wcvt(vle##width##_v_##suffix##mf2(hptr))); \ + b0 = _Tpwvec(wcvt(vle##width##_v_##suffix##mf2(lptr, vl), vl)); \ + b1 = _Tpwvec(wcvt(vle##width##_v_##suffix##mf2(hptr, vl), vl)); \ } \ inline _Tpwvec v_expand_low(const _Tpvec& a) \ { \ _Tp CV_DECL_ALIGNED(32) lptr[_Tpvec::nlanes/2] = {0}; \ v_store_low(lptr, a); \ - return _Tpwvec(wcvt(vle##width##_v_##suffix##mf2(lptr))); \ + return _Tpwvec(wcvt(vle##width##_v_##suffix##mf2(lptr, vl), vl)); \ } \ inline _Tpwvec v_expand_high(const _Tpvec& a) \ { \ _Tp CV_DECL_ALIGNED(32) hptr[_Tpvec::nlanes/2] = {0}; \ v_store_high(hptr, a); \ - return _Tpwvec(wcvt(vle##width##_v_##suffix##mf2(hptr))); \ + return _Tpwvec(wcvt(vle##width##_v_##suffix##mf2(hptr, vl), vl)); \ } \ inline _Tpwvec v_load_expand(const _Tp* ptr) \ { \ - return _Tpwvec(wcvt(vle##width##_v_##suffix##mf2(ptr))); \ + return _Tpwvec(wcvt(vle##width##_v_##suffix##mf2(ptr, vl), vl)); \ } -OPENCV_HAL_IMPL_RVV_EXPAND(v_uint16x8, uchar, v_uint8x16, 8, u8, vwcvtu_x_x_v_u16m1) -OPENCV_HAL_IMPL_RVV_EXPAND(v_int16x8, schar, v_int8x16, 8, i8, vwcvt_x_x_v_i16m1) -OPENCV_HAL_IMPL_RVV_EXPAND(v_uint32x4, ushort, v_uint16x8, 16, u16, vwcvtu_x_x_v_u32m1) -OPENCV_HAL_IMPL_RVV_EXPAND(v_int32x4, short, v_int16x8, 16, i16, vwcvt_x_x_v_i32m1) -OPENCV_HAL_IMPL_RVV_EXPAND(v_uint64x2, uint, v_uint32x4, 32, u32, vwcvtu_x_x_v_u64m1) -OPENCV_HAL_IMPL_RVV_EXPAND(v_int64x2, int, v_int32x4, 32, i32, vwcvt_x_x_v_i64m1) +OPENCV_HAL_IMPL_RVV_EXPAND(v_uint16x8, uchar, v_uint8x16, 8, u8, vwcvtu_x_x_v_u16m1, 8) +OPENCV_HAL_IMPL_RVV_EXPAND(v_int16x8, schar, v_int8x16, 8, i8, vwcvt_x_x_v_i16m1, 8) +OPENCV_HAL_IMPL_RVV_EXPAND(v_uint32x4, ushort, v_uint16x8, 16, u16, vwcvtu_x_x_v_u32m1, 4) +OPENCV_HAL_IMPL_RVV_EXPAND(v_int32x4, short, v_int16x8, 16, i16, vwcvt_x_x_v_i32m1, 4) +OPENCV_HAL_IMPL_RVV_EXPAND(v_uint64x2, uint, v_uint32x4, 32, u32, vwcvtu_x_x_v_u64m1, 2) +OPENCV_HAL_IMPL_RVV_EXPAND(v_int64x2, int, v_int32x4, 32, i32, vwcvt_x_x_v_i64m1, 2) inline v_uint32x4 v_load_expand_q(const uchar* ptr) { - vsetvlmax_e32m1(); - return v_uint32x4(vwcvtu_x_x_v_u32m1(vwcvtu_x_x_v_u16mf2(vle8_v_u8mf4(ptr)))); + return v_uint32x4(vwcvtu_x_x_v_u32m1(vwcvtu_x_x_v_u16mf2(vle8_v_u8mf4(ptr, 4), 4), 4)); } inline v_int32x4 v_load_expand_q(const schar* ptr) { - vsetvlmax_e32m1(); - return v_int32x4(vwcvt_x_x_v_i32m1(vwcvt_x_x_v_i16mf2(vle8_v_i8mf4(ptr)))); + return v_int32x4(vwcvt_x_x_v_i32m1(vwcvt_x_x_v_i16mf2(vle8_v_i8mf4(ptr, 4), 4), 4)); } -#define OPENCV_HAL_IMPL_RVV_PACK(_Tpvec, _Tp, _wTpvec, _wTp, width, suffix, rshr, shr) \ +#define OPENCV_HAL_IMPL_RVV_PACK(_Tpvec, _Tp, _wTpvec, _wTp, width, suffix, rshr, shr, hvl, vl) \ inline _Tpvec v_pack(const _wTpvec& a, const _wTpvec& b) \ { \ _wTp CV_DECL_ALIGNED(32) arr[_Tpvec::nlanes] = {0}; \ v_store(arr, a); \ v_store(arr + _wTpvec::nlanes, b); \ - vsetvlmax_e##width##m2(); \ - return _Tpvec(shr(vle##width##_v_##suffix##m2(arr), 0)); \ + return _Tpvec(shr(vle##width##_v_##suffix##m2(arr, vl), 0, vl)); \ } \ inline void v_pack_store(_Tp* ptr, const _wTpvec& a) \ { \ _wTp CV_DECL_ALIGNED(32) arr[_Tpvec::nlanes] = {0}; \ v_store(arr, a); \ - v_store(arr + _wTpvec::nlanes, _wTpvec(vzero_##suffix##m1())); \ - vsetvlmax_e##width##m2(); \ - v_store(ptr, _Tpvec(shr(vle##width##_v_##suffix##m2(arr), 0))); \ + v_store(arr + _wTpvec::nlanes, _wTpvec(vmv_v_x_##suffix##m1(0, hvl))); \ + v_store(ptr, _Tpvec(shr(vle##width##_v_##suffix##m2(arr, vl), 0, vl))); \ } \ template inline \ _Tpvec v_rshr_pack(const _wTpvec& a, const _wTpvec& b) \ @@ -2045,43 +1957,39 @@ _Tpvec v_rshr_pack(const _wTpvec& a, const _wTpvec& b) \ _wTp CV_DECL_ALIGNED(32) arr[_Tpvec::nlanes] = {0}; \ v_store(arr, a); \ v_store(arr + _wTpvec::nlanes, b); \ - vsetvlmax_e##width##m2(); \ - return _Tpvec(rshr(vle##width##_v_##suffix##m2(arr), n)); \ + return _Tpvec(rshr(vle##width##_v_##suffix##m2(arr, vl), n, vl)); \ } \ template inline \ void v_rshr_pack_store(_Tp* ptr, const _wTpvec& a) \ { \ _wTp CV_DECL_ALIGNED(32) arr[_Tpvec::nlanes] = {0}; \ v_store(arr, a); \ - v_store(arr + _wTpvec::nlanes, _wTpvec(vzero_##suffix##m1())); \ - vsetvlmax_e##width##m2(); \ - v_store(ptr, _Tpvec(rshr(vle##width##_v_##suffix##m2(arr), n))); \ + v_store(arr + _wTpvec::nlanes, _wTpvec(vmv_v_x_##suffix##m1(0, hvl))); \ + v_store(ptr, _Tpvec(rshr(vle##width##_v_##suffix##m2(arr, vl), n, vl))); \ } -OPENCV_HAL_IMPL_RVV_PACK(v_uint8x16, uchar, v_uint16x8, ushort, 16, u16, vnclipu_wx_u8m1, vnclipu_wx_u8m1) -OPENCV_HAL_IMPL_RVV_PACK(v_int8x16, schar, v_int16x8, short, 16, i16, vnclip_wx_i8m1, vnclip_wx_i8m1) -OPENCV_HAL_IMPL_RVV_PACK(v_uint16x8, ushort, v_uint32x4, unsigned, 32, u32, vnclipu_wx_u16m1, vnclipu_wx_u16m1) -OPENCV_HAL_IMPL_RVV_PACK(v_int16x8, short, v_int32x4, int, 32, i32, vnclip_wx_i16m1, vnclip_wx_i16m1) -OPENCV_HAL_IMPL_RVV_PACK(v_uint32x4, unsigned, v_uint64x2, uint64, 64, u64, vnclipu_wx_u32m1, vnsrl_wx_u32m1) -OPENCV_HAL_IMPL_RVV_PACK(v_int32x4, int, v_int64x2, int64, 64, i64, vnclip_wx_i32m1, vnsra_wx_i32m1) +OPENCV_HAL_IMPL_RVV_PACK(v_uint8x16, uchar, v_uint16x8, ushort, 16, u16, vnclipu_wx_u8m1, vnclipu_wx_u8m1, 8, 16) +OPENCV_HAL_IMPL_RVV_PACK(v_int8x16, schar, v_int16x8, short, 16, i16, vnclip_wx_i8m1, vnclip_wx_i8m1, 8, 16) +OPENCV_HAL_IMPL_RVV_PACK(v_uint16x8, ushort, v_uint32x4, unsigned, 32, u32, vnclipu_wx_u16m1, vnclipu_wx_u16m1, 4, 8) +OPENCV_HAL_IMPL_RVV_PACK(v_int16x8, short, v_int32x4, int, 32, i32, vnclip_wx_i16m1, vnclip_wx_i16m1, 4, 8) +OPENCV_HAL_IMPL_RVV_PACK(v_uint32x4, unsigned, v_uint64x2, uint64, 64, u64, vnclipu_wx_u32m1, vnsrl_wx_u32m1, 2, 4) +OPENCV_HAL_IMPL_RVV_PACK(v_int32x4, int, v_int64x2, int64, 64, i64, vnclip_wx_i32m1, vnsra_wx_i32m1, 2, 4) -#define OPENCV_HAL_IMPL_RVV_PACK_U(_Tpvec, _Tp, _wTpvec, _wTp, width, suffix, rshr, cast) \ +#define OPENCV_HAL_IMPL_RVV_PACK_U(_Tpvec, _Tp, _wTpvec, _wTp, width, suffix, rshr, cast, vl) \ inline _Tpvec v_pack_u(const _wTpvec& a, const _wTpvec& b) \ { \ _wTp CV_DECL_ALIGNED(32) arr[_Tpvec::nlanes] = {0}; \ v_store(arr, a); \ v_store(arr + _wTpvec::nlanes, b); \ - vsetvlmax_e##width##m2(); \ - return _Tpvec(rshr(cast(vmax_vx_##suffix##m2(vle##width##_v_##suffix##m2(arr), 0)), 0)); \ + return _Tpvec(rshr(cast(vmax_vx_##suffix##m2(vle##width##_v_##suffix##m2(arr, vl), 0, vl)), 0, vl)); \ } \ inline void v_pack_u_store(_Tp* ptr, const _wTpvec& a) \ { \ _wTp CV_DECL_ALIGNED(32) arr[_Tpvec::nlanes] = {0}; \ v_store(arr, a); \ - v_store(arr + _wTpvec::nlanes, _wTpvec(vzero_##suffix##m1())); \ - vsetvlmax_e##width##m2(); \ - v_store(ptr, _Tpvec(rshr(cast(vmax_vx_##suffix##m2(vle##width##_v_##suffix##m2(arr), 0)), 0))); \ + v_store(arr + _wTpvec::nlanes, _wTpvec(vmv_v_x_##suffix##m1(0, vl))); \ + v_store(ptr, _Tpvec(rshr(cast(vmax_vx_##suffix##m2(vle##width##_v_##suffix##m2(arr, vl), 0, vl)), 0, vl))); \ } \ template inline \ _Tpvec v_rshr_pack_u(const _wTpvec& a, const _wTpvec& b) \ @@ -2089,24 +1997,22 @@ _Tpvec v_rshr_pack_u(const _wTpvec& a, const _wTpvec& b) \ _wTp CV_DECL_ALIGNED(32) arr[_Tpvec::nlanes] = {0}; \ v_store(arr, a); \ v_store(arr + _wTpvec::nlanes, b); \ - vsetvlmax_e##width##m2(); \ - return _Tpvec(rshr(cast(vmax_vx_##suffix##m2(vle##width##_v_##suffix##m2(arr), 0)), n)); \ + return _Tpvec(rshr(cast(vmax_vx_##suffix##m2(vle##width##_v_##suffix##m2(arr, vl), 0, vl)), n, vl)); \ } \ template inline \ void v_rshr_pack_u_store(_Tp* ptr, const _wTpvec& a) \ { \ _wTp CV_DECL_ALIGNED(32) arr[_Tpvec::nlanes] = {0}; \ v_store(arr, a); \ - v_store(arr + _wTpvec::nlanes, _wTpvec(vzero_##suffix##m1())); \ - vsetvlmax_e##width##m2(); \ - v_store(ptr, _Tpvec(rshr(cast(vmax_vx_##suffix##m2(vle##width##_v_##suffix##m2(arr), 0)), n))); \ + v_store(arr + _wTpvec::nlanes, _wTpvec(vmv_v_x_##suffix##m1(0, vl))); \ + v_store(ptr, _Tpvec(rshr(cast(vmax_vx_##suffix##m2(vle##width##_v_##suffix##m2(arr, vl), 0, vl)), n, vl))); \ } -OPENCV_HAL_IMPL_RVV_PACK_U(v_uint8x16, uchar, v_int16x8, short, 16, i16, vnclipu_wx_u8m1, vreinterpret_v_i16m2_u16m2) -OPENCV_HAL_IMPL_RVV_PACK_U(v_uint16x8, ushort, v_int32x4, int, 32, i32, vnclipu_wx_u16m1, vreinterpret_v_i32m2_u32m2) +OPENCV_HAL_IMPL_RVV_PACK_U(v_uint8x16, uchar, v_int16x8, short, 16, i16, vnclipu_wx_u8m1, vreinterpret_v_i16m2_u16m2, 16) +OPENCV_HAL_IMPL_RVV_PACK_U(v_uint16x8, ushort, v_int32x4, int, 32, i32, vnclipu_wx_u16m1, vreinterpret_v_i32m2_u32m2, 8) -#define OPENCV_HAL_IMPL_RVV_UNPACKS(_Tpvec, _Tp, width, suffix) \ +#define OPENCV_HAL_IMPL_RVV_UNPACKS(_Tpvec, _Tp, suffix) \ inline void v_zip(const v_##_Tpvec& a0, const v_##_Tpvec& a1, v_##_Tpvec& b0, v_##_Tpvec& b1) \ { \ _Tp CV_DECL_ALIGNED(32) ptra0[v_##_Tpvec::nlanes] = {0}; \ @@ -2151,19 +2057,19 @@ inline void v_recombine(const v_##_Tpvec& a, const v_##_Tpvec& b, v_##_Tpvec& c, d = v_combine_high(a, b); \ } -OPENCV_HAL_IMPL_RVV_UNPACKS(uint8x16, uchar, 8, u8) -OPENCV_HAL_IMPL_RVV_UNPACKS(int8x16, schar, 8, i8) -OPENCV_HAL_IMPL_RVV_UNPACKS(uint16x8, ushort, 16, u16) -OPENCV_HAL_IMPL_RVV_UNPACKS(int16x8, short, 16, i16) -OPENCV_HAL_IMPL_RVV_UNPACKS(uint32x4, unsigned, 32, u32) -OPENCV_HAL_IMPL_RVV_UNPACKS(int32x4, int, 32, i32) -OPENCV_HAL_IMPL_RVV_UNPACKS(float32x4, float, 32, f32) +OPENCV_HAL_IMPL_RVV_UNPACKS(uint8x16, uchar, u8) +OPENCV_HAL_IMPL_RVV_UNPACKS(int8x16, schar, i8) +OPENCV_HAL_IMPL_RVV_UNPACKS(uint16x8, ushort, u16) +OPENCV_HAL_IMPL_RVV_UNPACKS(int16x8, short, i16) +OPENCV_HAL_IMPL_RVV_UNPACKS(uint32x4, unsigned, u32) +OPENCV_HAL_IMPL_RVV_UNPACKS(int32x4, int, i32) +OPENCV_HAL_IMPL_RVV_UNPACKS(float32x4, float, f32) #if CV_SIMD128_64F -OPENCV_HAL_IMPL_RVV_UNPACKS(float64x2, double, 64, f64) +OPENCV_HAL_IMPL_RVV_UNPACKS(float64x2, double, f64) #endif -#define OPENCV_HAL_IMPL_RVV_INTERLEAVED(_Tpvec, _Tp, suffix, width) \ +#define OPENCV_HAL_IMPL_RVV_INTERLEAVED(_Tpvec, _Tp) \ inline void v_load_deinterleave(const _Tp* ptr, v_##_Tpvec& a, v_##_Tpvec& b) \ { \ _Tp CV_DECL_ALIGNED(32) ptra[v_##_Tpvec::nlanes] = {0}; \ @@ -2298,17 +2204,17 @@ inline v_##_Tpvec v_interleave_quads(const v_##_Tpvec& vec) \ return v_load(ptr); \ } -OPENCV_HAL_IMPL_RVV_INTERLEAVED(uint8x16, uchar, u8, 8) -OPENCV_HAL_IMPL_RVV_INTERLEAVED(int8x16, schar, i8, 8) -OPENCV_HAL_IMPL_RVV_INTERLEAVED(uint16x8, ushort, u16, 16) -OPENCV_HAL_IMPL_RVV_INTERLEAVED(int16x8, short, i16, 16) -OPENCV_HAL_IMPL_RVV_INTERLEAVED(uint32x4, unsigned, u32, 32) -OPENCV_HAL_IMPL_RVV_INTERLEAVED(int32x4, int, i32, 32) -OPENCV_HAL_IMPL_RVV_INTERLEAVED(float32x4, float, f32, 32) -OPENCV_HAL_IMPL_RVV_INTERLEAVED(uint64x2, uint64, u64, 64) -OPENCV_HAL_IMPL_RVV_INTERLEAVED(int64x2, int64, i64, 64) +OPENCV_HAL_IMPL_RVV_INTERLEAVED(uint8x16, uchar) +OPENCV_HAL_IMPL_RVV_INTERLEAVED(int8x16, schar) +OPENCV_HAL_IMPL_RVV_INTERLEAVED(uint16x8, ushort) +OPENCV_HAL_IMPL_RVV_INTERLEAVED(int16x8, short) +OPENCV_HAL_IMPL_RVV_INTERLEAVED(uint32x4, unsigned) +OPENCV_HAL_IMPL_RVV_INTERLEAVED(int32x4, int) +OPENCV_HAL_IMPL_RVV_INTERLEAVED(float32x4, float) +OPENCV_HAL_IMPL_RVV_INTERLEAVED(uint64x2, uint64) +OPENCV_HAL_IMPL_RVV_INTERLEAVED(int64x2, int64) #if CV_SIMD128_64F -OPENCV_HAL_IMPL_RVV_INTERLEAVED(float64x2, double, f64, 64) +OPENCV_HAL_IMPL_RVV_INTERLEAVED(float64x2, double) #endif //////////// PopCount //////////// @@ -2356,21 +2262,20 @@ OPENCV_HAL_IMPL_RVV_POPCOUNT_OP(v_uint64x2, v_int64x2, uint64, int64, u64) //////////// SignMask //////////// -#define OPENCV_HAL_IMPL_RVV_SIGNMASK_OP(_Tpvec, _Tp, suffix, width, shift) \ +#define OPENCV_HAL_IMPL_RVV_SIGNMASK_OP(_Tpvec, _Tp, suffix, vl, shift) \ inline int v_signmask(const _Tpvec& a) \ { \ int mask = 0; \ - vsetvlmax_e##width##m1(); \ - _Tpvec tmp = _Tpvec(vsrl_vx_##suffix##m1(a, shift)); \ + _Tpvec tmp = _Tpvec(vsrl_vx_##suffix##m1(a, shift, vl)); \ for( int i = 0; i < _Tpvec::nlanes; i++ ) \ mask |= (int)(tmp.val[i]) << i; \ return mask; \ } -OPENCV_HAL_IMPL_RVV_SIGNMASK_OP(v_uint8x16, uchar, u8, 8, 7) -OPENCV_HAL_IMPL_RVV_SIGNMASK_OP(v_uint16x8, ushort, u16, 16, 15) -OPENCV_HAL_IMPL_RVV_SIGNMASK_OP(v_uint32x4, unsigned, u32, 32, 31) -OPENCV_HAL_IMPL_RVV_SIGNMASK_OP(v_uint64x2, uint64, u64, 64, 63) +OPENCV_HAL_IMPL_RVV_SIGNMASK_OP(v_uint8x16, uchar, u8, 16, 7) +OPENCV_HAL_IMPL_RVV_SIGNMASK_OP(v_uint16x8, ushort, u16, 8, 15) +OPENCV_HAL_IMPL_RVV_SIGNMASK_OP(v_uint32x4, unsigned, u32, 4, 31) +OPENCV_HAL_IMPL_RVV_SIGNMASK_OP(v_uint64x2, uint64, u64, 2, 63) inline int v_signmask(const v_int8x16& a) { return v_signmask(v_reinterpret_as_u8(a)); } @@ -2445,12 +2350,12 @@ OPENCV_HAL_IMPL_RVV_PACK_TRIPLETS(v_float32x4, float) #if CV_FP16 inline v_float32x4 v_load_expand(const float16_t* ptr) { - return v_float32x4(vfwcvt_f_f_v_f32m1(vle16_v_f16mf2(ptr))); + return v_float32x4(vfwcvt_f_f_v_f32m1(vle16_v_f16mf2(ptr, 4), 4)); } inline void v_pack_store(float16_t* ptr, const v_float32x4& v) { - vse16_v_f16mf2(ptr, vfncvt_f_f_w_f16mf2(v)); + vse16_v_f16mf2(ptr, vfncvt_f_f_w_f16mf2(v, 4), 4); } #else inline v_float32x4 v_load_expand(const float16_t* ptr) @@ -2474,70 +2379,61 @@ inline void v_pack_store(float16_t* ptr, const v_float32x4& v) inline v_int32x4 v_round(const v_float32x4& a) { - vsetvlmax_e32m1(); - return v_int32x4(vfcvt_x_f_v_i32m1(a)); + return v_int32x4(vfcvt_x_f_v_i32m1(a, 4)); } inline v_int32x4 v_floor(const v_float32x4& a) { v_float32x4 ZP5 = v_setall_f32(0.5f); v_float32x4 t = a - ZP5; - vsetvlmax_e32m1(); - return v_int32x4(vfcvt_x_f_v_i32m1(t)); + return v_int32x4(vfcvt_x_f_v_i32m1(t, 4)); } inline v_int32x4 v_ceil(const v_float32x4& a) { v_float32x4 ZP5 = v_setall_f32(0.5f); v_float32x4 t = a + ZP5; - vsetvlmax_e32m1(); - return v_int32x4(vfcvt_x_f_v_i32m1(t)); + return v_int32x4(vfcvt_x_f_v_i32m1(t, 4)); } inline v_int32x4 v_trunc(const v_float32x4& a) { - vsetvlmax_e32m1(); - return v_int32x4(vfcvt_rtz_x_f_v_i32m1(a)); + return v_int32x4(vfcvt_rtz_x_f_v_i32m1(a, 4)); } #if CV_SIMD128_64F inline v_int32x4 v_round(const v_float64x2& a) { double arr[4] = {a.val[0], a.val[1], 0, 0}; - vsetvlmax_e64m2(); - vfloat64m2_t tmp = vle64_v_f64m2(arr); - return v_int32x4(vfncvt_x_f_w_i32m1(tmp)); + vfloat64m2_t tmp = vle64_v_f64m2(arr, 4); + return v_int32x4(vfncvt_x_f_w_i32m1(tmp, 4)); } inline v_int32x4 v_round(const v_float64x2& a, const v_float64x2& b) { double arr[4] = {a.val[0], a.val[1], b.val[0], b.val[1]}; - vsetvlmax_e64m2(); - vfloat64m2_t tmp = vle64_v_f64m2(arr); - return v_int32x4(vfncvt_x_f_w_i32m1(tmp)); + vfloat64m2_t tmp = vle64_v_f64m2(arr, 4); + return v_int32x4(vfncvt_x_f_w_i32m1(tmp, 4)); } inline v_int32x4 v_floor(const v_float64x2& a) { double arr[4] = {a.val[0]-0.5f, a.val[1]-0.5f, 0, 0}; - vsetvlmax_e64m2(); - vfloat64m2_t tmp = vle64_v_f64m2(arr); - return v_int32x4(vfncvt_x_f_w_i32m1(tmp)); + vfloat64m2_t tmp = vle64_v_f64m2(arr, 4); + return v_int32x4(vfncvt_x_f_w_i32m1(tmp, 4)); } inline v_int32x4 v_ceil(const v_float64x2& a) { double arr[4] = {a.val[0]+0.5f, a.val[1]+0.5f, 0, 0}; - vsetvlmax_e64m2(); - vfloat64m2_t tmp = vle64_v_f64m2(arr); - return v_int32x4(vfncvt_x_f_w_i32m1(tmp)); + vfloat64m2_t tmp = vle64_v_f64m2(arr, 4); + return v_int32x4(vfncvt_x_f_w_i32m1(tmp, 4)); } inline v_int32x4 v_trunc(const v_float64x2& a) { double arr[4] = {a.val[0], a.val[1], 0, 0}; - vsetvlmax_e64m2(); - vfloat64m2_t tmp = vle64_v_f64m2(arr); - return v_int32x4(vfncvt_rtz_x_f_w_i32m1(tmp)); + vfloat64m2_t tmp = vle64_v_f64m2(arr, 4); + return v_int32x4(vfncvt_rtz_x_f_w_i32m1(tmp, 4)); } #endif @@ -2549,8 +2445,7 @@ inline v_int32x4 v_dotprod(const v_int16x8& a, const v_int16x8& b) { int CV_DECL_ALIGNED(32) ptr[8] = {0}; v_int32x4 t1, t2; - vsetvlmax_e32m2(); - vse32_v_i32m2(ptr, vwmul_vv_i32m2(a, b)); + vse32_v_i32m2(ptr, vwmul_vv_i32m2(a, b, 8), 8); v_load_deinterleave(ptr, t1, t2); return t1 + t2; } @@ -2558,8 +2453,7 @@ inline v_int32x4 v_dotprod(const v_int16x8& a, const v_int16x8& b, const v_int32 { int CV_DECL_ALIGNED(32) ptr[8] = {0}; v_int32x4 t1, t2; - vsetvlmax_e32m2(); - vse32_v_i32m2(ptr, vwmul_vv_i32m2(a, b)); + vse32_v_i32m2(ptr, vwmul_vv_i32m2(a, b, 8), 8); v_load_deinterleave(ptr, t1, t2); return t1 + t2 + c; } @@ -2569,8 +2463,7 @@ inline v_int64x2 v_dotprod(const v_int32x4& a, const v_int32x4& b) { int64 CV_DECL_ALIGNED(32) ptr[4] = {0}; v_int64x2 t1, t2; - vsetvlmax_e64m2(); - vse64_v_i64m2(ptr, vwmul_vv_i64m2(a, b)); + vse64_v_i64m2(ptr, vwmul_vv_i64m2(a, b, 4), 4); v_load_deinterleave(ptr, t1, t2); return t1 + t2; } @@ -2578,8 +2471,7 @@ inline v_int64x2 v_dotprod(const v_int32x4& a, const v_int32x4& b, const v_int64 { int64 CV_DECL_ALIGNED(32) ptr[4] = {0}; v_int64x2 t1, t2; - vsetvlmax_e64m2(); - vse64_v_i64m2(ptr, vwmul_vv_i64m2(a, b)); + vse64_v_i64m2(ptr, vwmul_vv_i64m2(a, b, 4), 4); v_load_deinterleave(ptr, t1, t2); return t1 + t2 + c; } @@ -2589,8 +2481,7 @@ inline v_uint32x4 v_dotprod_expand(const v_uint8x16& a, const v_uint8x16& b) { unsigned CV_DECL_ALIGNED(32) ptr[16] = {0}; v_uint32x4 t1, t2, t3, t4; - vsetvlmax_e32m4(); - vse32_v_u32m4(ptr, vqmaccu_vv_u32m4(vzero_u32m4(), a, b)); + vse32_v_u32m4(ptr, vwcvtu_x_x_v_u32m4(vwmulu_vv_u16m2(a, b, 16), 16), 16); v_load_deinterleave(ptr, t1, t2, t3, t4); return t1 + t2 + t3 + t4; } @@ -2599,8 +2490,7 @@ inline v_uint32x4 v_dotprod_expand(const v_uint8x16& a, const v_uint8x16& b, { unsigned CV_DECL_ALIGNED(32) ptr[16] = {0}; v_uint32x4 t1, t2, t3, t4; - vsetvlmax_e32m4(); - vse32_v_u32m4(ptr, vqmaccu_vv_u32m4(vzero_u32m4(), a, b)); + vse32_v_u32m4(ptr, vwcvtu_x_x_v_u32m4(vwmulu_vv_u16m2(a, b, 16), 16), 16); v_load_deinterleave(ptr, t1, t2, t3, t4); return t1 + t2 + t3 + t4 + c; } @@ -2609,8 +2499,7 @@ inline v_int32x4 v_dotprod_expand(const v_int8x16& a, const v_int8x16& b) { int CV_DECL_ALIGNED(32) ptr[16] = {0}; v_int32x4 t1, t2, t3, t4; - vsetvlmax_e32m4(); - vse32_v_i32m4(ptr, vqmacc_vv_i32m4(vzero_i32m4(), a, b)); + vse32_v_i32m4(ptr, vwcvt_x_x_v_i32m4(vwmul_vv_i16m2(a, b, 16), 16), 16); v_load_deinterleave(ptr, t1, t2, t3, t4); return t1 + t2 + t3 + t4; } @@ -2619,8 +2508,7 @@ inline v_int32x4 v_dotprod_expand(const v_int8x16& a, const v_int8x16& b, { int CV_DECL_ALIGNED(32) ptr[16] = {0}; v_int32x4 t1, t2, t3, t4; - vsetvlmax_e32m4(); - vse32_v_i32m4(ptr, vqmacc_vv_i32m4(vzero_i32m4(), a, b)); + vse32_v_i32m4(ptr, vwcvt_x_x_v_i32m4(vwmul_vv_i16m2(a, b, 16), 16), 16); v_load_deinterleave(ptr, t1, t2, t3, t4); return t1 + t2 + t3 + t4 + c; } @@ -2630,8 +2518,7 @@ inline v_uint64x2 v_dotprod_expand(const v_uint16x8& a, const v_uint16x8& b) { uint64 CV_DECL_ALIGNED(32) ptr[8] = {0}; v_uint64x2 t1, t2, t3, t4; - vsetvlmax_e64m4(); - vse64_v_u64m4(ptr, vqmaccu_vv_u64m4(vzero_u64m4(), a, b)); + vse64_v_u64m4(ptr, vwcvtu_x_x_v_u64m4(vwmulu_vv_u32m2(a, b, 8), 8), 8); v_load_deinterleave(ptr, t1, t2, t3, t4); return t1 + t2 + t3 + t4; } @@ -2639,8 +2526,7 @@ inline v_uint64x2 v_dotprod_expand(const v_uint16x8& a, const v_uint16x8& b, con { uint64 CV_DECL_ALIGNED(32) ptr[8] = {0}; v_uint64x2 t1, t2, t3, t4; - vsetvlmax_e64m4(); - vse64_v_u64m4(ptr, vqmaccu_vv_u64m4(vzero_u64m4(), a, b)); + vse64_v_u64m4(ptr, vwcvtu_x_x_v_u64m4(vwmulu_vv_u32m2(a, b, 8), 8), 8); v_load_deinterleave(ptr, t1, t2, t3, t4); return t1 + t2 + t3 + t4 + c; } @@ -2649,8 +2535,7 @@ inline v_int64x2 v_dotprod_expand(const v_int16x8& a, const v_int16x8& b) { int64 CV_DECL_ALIGNED(32) ptr[8] = {0}; v_int64x2 t1, t2, t3, t4; - vsetvlmax_e64m4(); - vse64_v_i64m4(ptr, vqmacc_vv_i64m4(vzero_i64m4(), a, b)); + vse64_v_i64m4(ptr, vwcvt_x_x_v_i64m4(vwmul_vv_i32m2(a, b, 8), 8), 8); v_load_deinterleave(ptr, t1, t2, t3, t4); return t1 + t2 + t3 + t4; } @@ -2659,8 +2544,7 @@ inline v_int64x2 v_dotprod_expand(const v_int16x8& a, const v_int16x8& b, { int64 CV_DECL_ALIGNED(32) ptr[8] = {0}; v_int64x2 t1, t2, t3, t4; - vsetvlmax_e64m4(); - vse64_v_i64m4(ptr, vqmacc_vv_i64m4(vzero_i64m4(), a, b)); + vse64_v_i64m4(ptr, vwcvt_x_x_v_i64m4(vwmul_vv_i32m2(a, b, 8), 8), 8); v_load_deinterleave(ptr, t1, t2, t3, t4); return t1 + t2 + t3 + t4 + c; } @@ -2680,8 +2564,7 @@ inline v_float64x2 v_dotprod_expand(const v_int32x4& a, const v_int32x4& b, inline v_int32x4 v_dotprod_fast(const v_int16x8& a, const v_int16x8& b) { int CV_DECL_ALIGNED(32) ptr[8] = {0}; - vsetvlmax_e32m2(); - vse32_v_i32m2(ptr, vwmul_vv_i32m2(a, b)); + vse32_v_i32m2(ptr, vwmul_vv_i32m2(a, b, 8), 8); v_int32x4 t1 = v_load(ptr); v_int32x4 t2 = v_load(ptr+4); return t1 + t2; @@ -2689,8 +2572,7 @@ inline v_int32x4 v_dotprod_fast(const v_int16x8& a, const v_int16x8& b) inline v_int32x4 v_dotprod_fast(const v_int16x8& a, const v_int16x8& b, const v_int32x4& c) { int CV_DECL_ALIGNED(32) ptr[8] = {0}; - vsetvlmax_e32m2(); - vse32_v_i32m2(ptr, vwmul_vv_i32m2(a, b)); + vse32_v_i32m2(ptr, vwmul_vv_i32m2(a, b, 8), 8); v_int32x4 t1 = v_load(ptr); v_int32x4 t2 = v_load(ptr+4); return t1 + t2 + c; @@ -2700,8 +2582,7 @@ inline v_int32x4 v_dotprod_fast(const v_int16x8& a, const v_int16x8& b, const v_ inline v_int64x2 v_dotprod_fast(const v_int32x4& a, const v_int32x4& b) { int64 CV_DECL_ALIGNED(32) ptr[4] = {0}; - vsetvlmax_e64m2(); - vse64_v_i64m2(ptr, vwmul_vv_i64m2(a, b)); + vse64_v_i64m2(ptr, vwmul_vv_i64m2(a, b, 4), 4); v_int64x2 t1 = v_load(ptr); v_int64x2 t2 = v_load(ptr+2); return t1 + t2; @@ -2709,8 +2590,7 @@ inline v_int64x2 v_dotprod_fast(const v_int32x4& a, const v_int32x4& b) inline v_int64x2 v_dotprod_fast(const v_int32x4& a, const v_int32x4& b, const v_int64x2& c) { int64 CV_DECL_ALIGNED(32) ptr[4] = {0}; - vsetvlmax_e64m2(); - vse64_v_i64m2(ptr, vwmul_vv_i64m2(a, b)); + vse64_v_i64m2(ptr, vwmul_vv_i64m2(a, b, 4), 4); v_int64x2 t1 = v_load(ptr); v_int64x2 t2 = v_load(ptr+2); return t1 + t2 + c; @@ -2721,8 +2601,7 @@ inline v_int64x2 v_dotprod_fast(const v_int32x4& a, const v_int32x4& b, const v_ inline v_uint32x4 v_dotprod_expand_fast(const v_uint8x16& a, const v_uint8x16& b) { unsigned CV_DECL_ALIGNED(32) ptr[16] = {0}; - vsetvlmax_e32m4(); - vse32_v_u32m4(ptr, vqmaccu_vv_u32m4(vzero_u32m4(), a, b)); + vse32_v_u32m4(ptr, vwcvtu_x_x_v_u32m4(vwmulu_vv_u16m2(a, b, 16), 16), 16); v_uint32x4 t1 = v_load(ptr); v_uint32x4 t2 = v_load(ptr+4); v_uint32x4 t3 = v_load(ptr+8); @@ -2732,8 +2611,7 @@ inline v_uint32x4 v_dotprod_expand_fast(const v_uint8x16& a, const v_uint8x16& b inline v_uint32x4 v_dotprod_expand_fast(const v_uint8x16& a, const v_uint8x16& b, const v_uint32x4& c) { unsigned CV_DECL_ALIGNED(32) ptr[16] = {0}; - vsetvlmax_e32m4(); - vse32_v_u32m4(ptr, vqmaccu_vv_u32m4(vzero_u32m4(), a, b)); + vse32_v_u32m4(ptr, vwcvtu_x_x_v_u32m4(vwmulu_vv_u16m2(a, b, 16), 16), 16); v_uint32x4 t1 = v_load(ptr); v_uint32x4 t2 = v_load(ptr+4); v_uint32x4 t3 = v_load(ptr+8); @@ -2743,8 +2621,7 @@ inline v_uint32x4 v_dotprod_expand_fast(const v_uint8x16& a, const v_uint8x16& b inline v_int32x4 v_dotprod_expand_fast(const v_int8x16& a, const v_int8x16& b) { int CV_DECL_ALIGNED(32) ptr[16] = {0}; - vsetvlmax_e32m4(); - vse32_v_i32m4(ptr, vqmacc_vv_i32m4(vzero_i32m4(), a, b)); + vse32_v_i32m4(ptr, vwcvt_x_x_v_i32m4(vwmul_vv_i16m2(a, b, 16), 16), 16); v_int32x4 t1 = v_load(ptr); v_int32x4 t2 = v_load(ptr+4); v_int32x4 t3 = v_load(ptr+8); @@ -2754,8 +2631,7 @@ inline v_int32x4 v_dotprod_expand_fast(const v_int8x16& a, const v_int8x16& b) inline v_int32x4 v_dotprod_expand_fast(const v_int8x16& a, const v_int8x16& b, const v_int32x4& c) { int CV_DECL_ALIGNED(32) ptr[16] = {0}; - vsetvlmax_e32m4(); - vse32_v_i32m4(ptr, vqmacc_vv_i32m4(vzero_i32m4(), a, b)); + vse32_v_i32m4(ptr, vwcvt_x_x_v_i32m4(vwmul_vv_i16m2(a, b, 16), 16), 16); v_int32x4 t1 = v_load(ptr); v_int32x4 t2 = v_load(ptr+4); v_int32x4 t3 = v_load(ptr+8); @@ -2767,8 +2643,7 @@ inline v_int32x4 v_dotprod_expand_fast(const v_int8x16& a, const v_int8x16& b, c inline v_uint64x2 v_dotprod_expand_fast(const v_uint16x8& a, const v_uint16x8& b) { uint64 CV_DECL_ALIGNED(32) ptr[8] = {0}; - vsetvlmax_e64m4(); - vse64_v_u64m4(ptr, vqmaccu_vv_u64m4(vzero_u64m4(), a, b)); + vse64_v_u64m4(ptr, vwcvtu_x_x_v_u64m4(vwmulu_vv_u32m2(a, b, 8), 8), 8); v_uint64x2 t1 = v_load(ptr); v_uint64x2 t2 = v_load(ptr+2); v_uint64x2 t3 = v_load(ptr+4); @@ -2778,8 +2653,7 @@ inline v_uint64x2 v_dotprod_expand_fast(const v_uint16x8& a, const v_uint16x8& b inline v_uint64x2 v_dotprod_expand_fast(const v_uint16x8& a, const v_uint16x8& b, const v_uint64x2& c) { uint64 CV_DECL_ALIGNED(32) ptr[8] = {0}; - vsetvlmax_e64m4(); - vse64_v_u64m4(ptr, vqmaccu_vv_u64m4(vzero_u64m4(), a, b)); + vse64_v_u64m4(ptr, vwcvtu_x_x_v_u64m4(vwmulu_vv_u32m2(a, b, 8), 8), 8); v_uint64x2 t1 = v_load(ptr); v_uint64x2 t2 = v_load(ptr+2); v_uint64x2 t3 = v_load(ptr+4); @@ -2789,8 +2663,7 @@ inline v_uint64x2 v_dotprod_expand_fast(const v_uint16x8& a, const v_uint16x8& b inline v_int64x2 v_dotprod_expand_fast(const v_int16x8& a, const v_int16x8& b) { int64 CV_DECL_ALIGNED(32) ptr[8] = {0}; - vsetvlmax_e64m4(); - vse64_v_i64m4(ptr, vqmacc_vv_i64m4(vzero_i64m4(), a, b)); + vse64_v_i64m4(ptr, vwcvt_x_x_v_i64m4(vwmul_vv_i32m2(a, b, 8), 8), 8); v_int64x2 t1 = v_load(ptr); v_int64x2 t2 = v_load(ptr+2); v_int64x2 t3 = v_load(ptr+4); @@ -2800,8 +2673,7 @@ inline v_int64x2 v_dotprod_expand_fast(const v_int16x8& a, const v_int16x8& b) inline v_int64x2 v_dotprod_expand_fast(const v_int16x8& a, const v_int16x8& b, const v_int64x2& c) { int64 CV_DECL_ALIGNED(32) ptr[8] = {0}; - vsetvlmax_e64m4(); - vse64_v_i64m4(ptr, vqmacc_vv_i64m4(vzero_i64m4(), a, b)); + vse64_v_i64m4(ptr, vwcvt_x_x_v_i64m4(vwmul_vv_i32m2(a, b, 8), 8), 8); v_int64x2 t1 = v_load(ptr); v_int64x2 t2 = v_load(ptr+2); v_int64x2 t3 = v_load(ptr+4); @@ -2822,11 +2694,10 @@ inline v_float32x4 v_matmul(const v_float32x4& v, const v_float32x4& m0, const v_float32x4& m1, const v_float32x4& m2, const v_float32x4& m3) { - vsetvlmax_e32m1(); - vfloat32m1_t res = vfmul_vf_f32m1(m0, v_extract_n<0>(v)); - res = vfmacc_vf_f32m1(res, v_extract_n<1>(v), m1); - res = vfmacc_vf_f32m1(res, v_extract_n<2>(v), m2); - res = vfmacc_vf_f32m1(res, v_extract_n<3>(v), m3); + vfloat32m1_t res = vfmul_vf_f32m1(m0, v_extract_n<0>(v), 4); + res = vfmacc_vf_f32m1(res, v_extract_n<1>(v), m1, 4); + res = vfmacc_vf_f32m1(res, v_extract_n<2>(v), m2, 4); + res = vfmacc_vf_f32m1(res, v_extract_n<3>(v), m3, 4); return v_float32x4(res); } @@ -2834,40 +2705,35 @@ inline v_float32x4 v_matmuladd(const v_float32x4& v, const v_float32x4& m0, const v_float32x4& m1, const v_float32x4& m2, const v_float32x4& a) { - vsetvlmax_e32m1(); - vfloat32m1_t res = vfmul_vf_f32m1(m0, v_extract_n<0>(v)); - res = vfmacc_vf_f32m1(res, v_extract_n<1>(v), m1); - res = vfmacc_vf_f32m1(res, v_extract_n<2>(v), m2); + vfloat32m1_t res = vfmul_vf_f32m1(m0, v_extract_n<0>(v), 4); + res = vfmacc_vf_f32m1(res, v_extract_n<1>(v), m1, 4); + res = vfmacc_vf_f32m1(res, v_extract_n<2>(v), m2, 4); return v_float32x4(res) + a; } -#define OPENCV_HAL_IMPL_RVV_MUL_EXPAND(_Tpvec, _Tpwvec, _Tpw, suffix, wmul, width) \ +#define OPENCV_HAL_IMPL_RVV_MUL_EXPAND(_Tpvec, _Tpwvec, _Tpw, suffix, wmul, width, vl, hvl) \ inline void v_mul_expand(const _Tpvec& a, const _Tpvec& b, _Tpwvec& c, _Tpwvec& d) \ { \ _Tpw CV_DECL_ALIGNED(32) ptr[_Tpwvec::nlanes*2] = {0}; \ - vsetvlmax_e##width##m2(); \ - vse##width##_v_##suffix##m2(ptr, wmul(a, b)); \ - vsetvlmax_e##width##m1(); \ - c = _Tpwvec(vle##width##_v_##suffix##m1(ptr)); \ - d = _Tpwvec(vle##width##_v_##suffix##m1(ptr+_Tpwvec::nlanes)); \ + vse##width##_v_##suffix##m2(ptr, wmul(a, b, vl), vl); \ + c = _Tpwvec(vle##width##_v_##suffix##m1(ptr, hvl)); \ + d = _Tpwvec(vle##width##_v_##suffix##m1(ptr+_Tpwvec::nlanes, hvl)); \ } -OPENCV_HAL_IMPL_RVV_MUL_EXPAND(v_uint8x16, v_uint16x8, ushort, u16, vwmulu_vv_u16m2, 16) -OPENCV_HAL_IMPL_RVV_MUL_EXPAND(v_int8x16, v_int16x8, short, i16, vwmul_vv_i16m2, 16) -OPENCV_HAL_IMPL_RVV_MUL_EXPAND(v_uint16x8, v_uint32x4, unsigned, u32, vwmulu_vv_u32m2, 32) -OPENCV_HAL_IMPL_RVV_MUL_EXPAND(v_int16x8, v_int32x4, int, i32, vwmul_vv_i32m2, 32) -OPENCV_HAL_IMPL_RVV_MUL_EXPAND(v_uint32x4, v_uint64x2, uint64, u64, vwmulu_vv_u64m2, 64) +OPENCV_HAL_IMPL_RVV_MUL_EXPAND(v_uint8x16, v_uint16x8, ushort, u16, vwmulu_vv_u16m2, 16, 16, 8) +OPENCV_HAL_IMPL_RVV_MUL_EXPAND(v_int8x16, v_int16x8, short, i16, vwmul_vv_i16m2, 16, 16, 8) +OPENCV_HAL_IMPL_RVV_MUL_EXPAND(v_uint16x8, v_uint32x4, unsigned, u32, vwmulu_vv_u32m2, 32, 8, 4) +OPENCV_HAL_IMPL_RVV_MUL_EXPAND(v_int16x8, v_int32x4, int, i32, vwmul_vv_i32m2, 32, 8, 4) +OPENCV_HAL_IMPL_RVV_MUL_EXPAND(v_uint32x4, v_uint64x2, uint64, u64, vwmulu_vv_u64m2, 64, 4, 2) inline v_int16x8 v_mul_hi(const v_int16x8& a, const v_int16x8& b) { - vsetvlmax_e16m1(); - return v_int16x8(vnsra_wx_i16m1(vwmul_vv_i32m2(a, b), 16)); + return v_int16x8(vnsra_wx_i16m1(vwmul_vv_i32m2(a, b, 8), 16, 8)); } inline v_uint16x8 v_mul_hi(const v_uint16x8& a, const v_uint16x8& b) { - vsetvlmax_e16m1(); - return v_uint16x8(vnsrl_wx_u16m1(vwmulu_vv_u32m2(a, b), 16)); + return v_uint16x8(vnsrl_wx_u16m1(vwmulu_vv_u32m2(a, b, 8), 16, 8)); } diff --git a/platforms/linux/riscv64-gcc.toolchain.cmake b/platforms/linux/riscv64-gcc.toolchain.cmake index c46d62a360..675879f86b 100644 --- a/platforms/linux/riscv64-gcc.toolchain.cmake +++ b/platforms/linux/riscv64-gcc.toolchain.cmake @@ -10,8 +10,8 @@ set(CMAKE_CXX_COMPILER ${RISCV_GCC_INSTALL_ROOT}/bin/riscv64-unknown-linux-gnu-g # Don't run the linker on compiler check set(CMAKE_TRY_COMPILE_TARGET_TYPE STATIC_LIBRARY) -set(CMAKE_C_FLAGS "-march=rv64gcv_zvqmac ${CMAKE_C_FLAGS}") -set(CMAKE_CXX_FLAGS "-march=rv64gcv_zvqmac ${CXX_FLAGS}") +set(CMAKE_C_FLAGS "-march=rv64gcv_zfh ${CMAKE_C_FLAGS}") +set(CMAKE_CXX_FLAGS "-march=rv64gcv_zfh ${CXX_FLAGS}") set(CMAKE_FIND_ROOT_PATH ${CMAKE_SYSROOT}) set(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER) From 53eca2ff5b24a6088647632cb598d732fc582ce6 Mon Sep 17 00:00:00 2001 From: Anatoliy Talamanov Date: Fri, 18 Jun 2021 20:16:07 +0300 Subject: [PATCH 012/262] Merge pull request #20196 from TolyaTalamanov:at/support-vaargs-compile-args G-API: Support vaargs for cv.compile_args * Support cv.compile_args to work with variadic number of inputs * Disable python2.x G-API * Move compile_args to gapi pkg --- .../gapi/misc/python/package/gapi/__init__.py | 5 + modules/gapi/misc/python/shadow_gapi.hpp | 9 +- .../gapi/misc/python/test/test_gapi_core.py | 336 +++++------ .../misc/python/test/test_gapi_imgproc.py | 163 +++--- .../gapi/misc/python/test/test_gapi_infer.py | 528 +++++++++--------- .../python/test/test_gapi_sample_pipelines.py | 36 +- .../misc/python/test/test_gapi_streaming.py | 392 +++++++------ .../gapi/misc/python/test/test_gapi_types.py | 52 +- 8 files changed, 817 insertions(+), 704 deletions(-) diff --git a/modules/gapi/misc/python/package/gapi/__init__.py b/modules/gapi/misc/python/package/gapi/__init__.py index 733c980010..23f5f41846 100644 --- a/modules/gapi/misc/python/package/gapi/__init__.py +++ b/modules/gapi/misc/python/package/gapi/__init__.py @@ -11,6 +11,11 @@ def register(mname): return parameterized +@register('cv2.gapi') +def compile_args(*args): + return list(map(cv.GCompileArg, args)) + + @register('cv2') class GOpaque(): # NB: Inheritance from c++ class cause segfault. diff --git a/modules/gapi/misc/python/shadow_gapi.hpp b/modules/gapi/misc/python/shadow_gapi.hpp index 40dab41581..941250c2fb 100644 --- a/modules/gapi/misc/python/shadow_gapi.hpp +++ b/modules/gapi/misc/python/shadow_gapi.hpp @@ -3,11 +3,10 @@ namespace cv { - struct GAPI_EXPORTS_W_SIMPLE GCompileArg { }; - - GAPI_EXPORTS_W GCompileArgs compile_args(gapi::GKernelPackage pkg); - GAPI_EXPORTS_W GCompileArgs compile_args(gapi::GNetPackage pkg); - GAPI_EXPORTS_W GCompileArgs compile_args(gapi::GKernelPackage kernels, gapi::GNetPackage nets); + struct GAPI_EXPORTS_W_SIMPLE GCompileArg { + GAPI_WRAP GCompileArg(gapi::GKernelPackage pkg); + GAPI_WRAP GCompileArg(gapi::GNetPackage pkg); + }; // NB: This classes doesn't exist in *.so // HACK: Mark them as a class to force python wrapper generate code for this entities diff --git a/modules/gapi/misc/python/test/test_gapi_core.py b/modules/gapi/misc/python/test/test_gapi_core.py index 814d05d7cd..780558d98b 100644 --- a/modules/gapi/misc/python/test/test_gapi_core.py +++ b/modules/gapi/misc/python/test/test_gapi_core.py @@ -3,187 +3,209 @@ import numpy as np import cv2 as cv import os +import sys +import unittest from tests_common import NewOpenCVTests -# Plaidml is an optional backend -pkgs = [ - ('ocl' , cv.gapi.core.ocl.kernels()), - ('cpu' , cv.gapi.core.cpu.kernels()), - ('fluid' , cv.gapi.core.fluid.kernels()) - # ('plaidml', cv.gapi.core.plaidml.kernels()) - ] +try: + + if sys.version_info[:2] < (3, 0): + raise unittest.SkipTest('Python 2.x is not supported') + + # Plaidml is an optional backend + pkgs = [ + ('ocl' , cv.gapi.core.ocl.kernels()), + ('cpu' , cv.gapi.core.cpu.kernels()), + ('fluid' , cv.gapi.core.fluid.kernels()) + # ('plaidml', cv.gapi.core.plaidml.kernels()) + ] -class gapi_core_test(NewOpenCVTests): + class gapi_core_test(NewOpenCVTests): - def test_add(self): - # TODO: Extend to use any type and size here - sz = (720, 1280) - in1 = np.full(sz, 100) - in2 = np.full(sz, 50) + def test_add(self): + # TODO: Extend to use any type and size here + sz = (720, 1280) + in1 = np.full(sz, 100) + in2 = np.full(sz, 50) - # OpenCV - expected = cv.add(in1, in2) + # OpenCV + expected = cv.add(in1, in2) - # G-API - g_in1 = cv.GMat() - g_in2 = cv.GMat() - g_out = cv.gapi.add(g_in1, g_in2) - comp = cv.GComputation(cv.GIn(g_in1, g_in2), cv.GOut(g_out)) + # G-API + g_in1 = cv.GMat() + g_in2 = cv.GMat() + g_out = cv.gapi.add(g_in1, g_in2) + comp = cv.GComputation(cv.GIn(g_in1, g_in2), cv.GOut(g_out)) - for pkg_name, pkg in pkgs: - actual = comp.apply(cv.gin(in1, in2), args=cv.compile_args(pkg)) - # Comparison - self.assertEqual(0.0, cv.norm(expected, actual, cv.NORM_INF), - 'Failed on ' + pkg_name + ' backend') - self.assertEqual(expected.dtype, actual.dtype, 'Failed on ' + pkg_name + ' backend') - - - def test_add_uint8(self): - sz = (720, 1280) - in1 = np.full(sz, 100, dtype=np.uint8) - in2 = np.full(sz, 50 , dtype=np.uint8) - - # OpenCV - expected = cv.add(in1, in2) - - # G-API - g_in1 = cv.GMat() - g_in2 = cv.GMat() - g_out = cv.gapi.add(g_in1, g_in2) - comp = cv.GComputation(cv.GIn(g_in1, g_in2), cv.GOut(g_out)) - - for pkg_name, pkg in pkgs: - actual = comp.apply(cv.gin(in1, in2), args=cv.compile_args(pkg)) - # Comparison - self.assertEqual(0.0, cv.norm(expected, actual, cv.NORM_INF), - 'Failed on ' + pkg_name + ' backend') - self.assertEqual(expected.dtype, actual.dtype, 'Failed on ' + pkg_name + ' backend') - - - def test_mean(self): - img_path = self.find_file('cv/face/david2.jpg', [os.environ.get('OPENCV_TEST_DATA_PATH')]) - in_mat = cv.imread(img_path) - - # OpenCV - expected = cv.mean(in_mat) - - # G-API - g_in = cv.GMat() - g_out = cv.gapi.mean(g_in) - comp = cv.GComputation(g_in, g_out) - - for pkg_name, pkg in pkgs: - actual = comp.apply(cv.gin(in_mat), args=cv.compile_args(pkg)) - # Comparison - self.assertEqual(0.0, cv.norm(expected, actual, cv.NORM_INF), - 'Failed on ' + pkg_name + ' backend') - - - def test_split3(self): - img_path = self.find_file('cv/face/david2.jpg', [os.environ.get('OPENCV_TEST_DATA_PATH')]) - in_mat = cv.imread(img_path) - - # OpenCV - expected = cv.split(in_mat) - - # G-API - g_in = cv.GMat() - b, g, r = cv.gapi.split3(g_in) - comp = cv.GComputation(cv.GIn(g_in), cv.GOut(b, g, r)) - - for pkg_name, pkg in pkgs: - actual = comp.apply(cv.gin(in_mat), args=cv.compile_args(pkg)) - # Comparison - for e, a in zip(expected, actual): - self.assertEqual(0.0, cv.norm(e, a, cv.NORM_INF), + for pkg_name, pkg in pkgs: + actual = comp.apply(cv.gin(in1, in2), args=cv.gapi.compile_args(pkg)) + # Comparison + self.assertEqual(0.0, cv.norm(expected, actual, cv.NORM_INF), 'Failed on ' + pkg_name + ' backend') - self.assertEqual(e.dtype, a.dtype, 'Failed on ' + pkg_name + ' backend') + self.assertEqual(expected.dtype, actual.dtype, 'Failed on ' + pkg_name + ' backend') - def test_threshold(self): - img_path = self.find_file('cv/face/david2.jpg', [os.environ.get('OPENCV_TEST_DATA_PATH')]) - in_mat = cv.cvtColor(cv.imread(img_path), cv.COLOR_RGB2GRAY) - maxv = (30, 30) + def test_add_uint8(self): + sz = (720, 1280) + in1 = np.full(sz, 100, dtype=np.uint8) + in2 = np.full(sz, 50 , dtype=np.uint8) - # OpenCV - expected_thresh, expected_mat = cv.threshold(in_mat, maxv[0], maxv[0], cv.THRESH_TRIANGLE) + # OpenCV + expected = cv.add(in1, in2) - # G-API - g_in = cv.GMat() - g_sc = cv.GScalar() - mat, threshold = cv.gapi.threshold(g_in, g_sc, cv.THRESH_TRIANGLE) - comp = cv.GComputation(cv.GIn(g_in, g_sc), cv.GOut(mat, threshold)) + # G-API + g_in1 = cv.GMat() + g_in2 = cv.GMat() + g_out = cv.gapi.add(g_in1, g_in2) + comp = cv.GComputation(cv.GIn(g_in1, g_in2), cv.GOut(g_out)) - for pkg_name, pkg in pkgs: - actual_mat, actual_thresh = comp.apply(cv.gin(in_mat, maxv), args=cv.compile_args(pkg)) - # Comparison - self.assertEqual(0.0, cv.norm(expected_mat, actual_mat, cv.NORM_INF), - 'Failed on ' + pkg_name + ' backend') - self.assertEqual(expected_mat.dtype, actual_mat.dtype, - 'Failed on ' + pkg_name + ' backend') - self.assertEqual(expected_thresh, actual_thresh[0], - 'Failed on ' + pkg_name + ' backend') - - def test_kmeans(self): - # K-means params - count = 100 - sz = (count, 2) - in_mat = np.random.random(sz).astype(np.float32) - K = 5 - flags = cv.KMEANS_RANDOM_CENTERS - attempts = 1; - criteria = (cv.TERM_CRITERIA_MAX_ITER + cv.TERM_CRITERIA_EPS, 30, 0) - - # G-API - g_in = cv.GMat() - compactness, out_labels, centers = cv.gapi.kmeans(g_in, K, criteria, attempts, flags) - comp = cv.GComputation(cv.GIn(g_in), cv.GOut(compactness, out_labels, centers)) - - compact, labels, centers = comp.apply(cv.gin(in_mat)) - - # Assert - self.assertTrue(compact >= 0) - self.assertEqual(sz[0], labels.shape[0]) - self.assertEqual(1, labels.shape[1]) - self.assertTrue(labels.size != 0) - self.assertEqual(centers.shape[1], sz[1]); - self.assertEqual(centers.shape[0], K); - self.assertTrue(centers.size != 0); + for pkg_name, pkg in pkgs: + actual = comp.apply(cv.gin(in1, in2), args=cv.gapi.compile_args(pkg)) + # Comparison + self.assertEqual(0.0, cv.norm(expected, actual, cv.NORM_INF), + 'Failed on ' + pkg_name + ' backend') + self.assertEqual(expected.dtype, actual.dtype, 'Failed on ' + pkg_name + ' backend') - def generate_random_points(self, sz): - arr = np.random.random(sz).astype(np.float32).T - return list(zip(arr[0], arr[1])) + def test_mean(self): + img_path = self.find_file('cv/face/david2.jpg', [os.environ.get('OPENCV_TEST_DATA_PATH')]) + in_mat = cv.imread(img_path) + + # OpenCV + expected = cv.mean(in_mat) + + # G-API + g_in = cv.GMat() + g_out = cv.gapi.mean(g_in) + comp = cv.GComputation(g_in, g_out) + + for pkg_name, pkg in pkgs: + actual = comp.apply(cv.gin(in_mat), args=cv.gapi.compile_args(pkg)) + # Comparison + self.assertEqual(0.0, cv.norm(expected, actual, cv.NORM_INF), + 'Failed on ' + pkg_name + ' backend') - def test_kmeans_2d(self): - # K-means 2D params - count = 100 - sz = (count, 2) - amount = sz[0] - K = 5 - flags = cv.KMEANS_RANDOM_CENTERS - attempts = 1; - criteria = (cv.TERM_CRITERIA_MAX_ITER + cv.TERM_CRITERIA_EPS, 30, 0); - in_vector = self.generate_random_points(sz) - in_labels = [] + def test_split3(self): + img_path = self.find_file('cv/face/david2.jpg', [os.environ.get('OPENCV_TEST_DATA_PATH')]) + in_mat = cv.imread(img_path) - # G-API - data = cv.GArrayT(cv.gapi.CV_POINT2F) - best_labels = cv.GArrayT(cv.gapi.CV_INT) + # OpenCV + expected = cv.split(in_mat) - compactness, out_labels, centers = cv.gapi.kmeans(data, K, best_labels, criteria, attempts, flags); - comp = cv.GComputation(cv.GIn(data, best_labels), cv.GOut(compactness, out_labels, centers)); + # G-API + g_in = cv.GMat() + b, g, r = cv.gapi.split3(g_in) + comp = cv.GComputation(cv.GIn(g_in), cv.GOut(b, g, r)) - compact, labels, centers = comp.apply(cv.gin(in_vector, in_labels)); + for pkg_name, pkg in pkgs: + actual = comp.apply(cv.gin(in_mat), args=cv.gapi.compile_args(pkg)) + # Comparison + for e, a in zip(expected, actual): + self.assertEqual(0.0, cv.norm(e, a, cv.NORM_INF), + 'Failed on ' + pkg_name + ' backend') + self.assertEqual(e.dtype, a.dtype, 'Failed on ' + pkg_name + ' backend') - # Assert - self.assertTrue(compact >= 0) - self.assertEqual(amount, len(labels)) - self.assertEqual(K, len(centers)) + + def test_threshold(self): + img_path = self.find_file('cv/face/david2.jpg', [os.environ.get('OPENCV_TEST_DATA_PATH')]) + in_mat = cv.cvtColor(cv.imread(img_path), cv.COLOR_RGB2GRAY) + maxv = (30, 30) + + # OpenCV + expected_thresh, expected_mat = cv.threshold(in_mat, maxv[0], maxv[0], cv.THRESH_TRIANGLE) + + # G-API + g_in = cv.GMat() + g_sc = cv.GScalar() + mat, threshold = cv.gapi.threshold(g_in, g_sc, cv.THRESH_TRIANGLE) + comp = cv.GComputation(cv.GIn(g_in, g_sc), cv.GOut(mat, threshold)) + + for pkg_name, pkg in pkgs: + actual_mat, actual_thresh = comp.apply(cv.gin(in_mat, maxv), args=cv.gapi.compile_args(pkg)) + # Comparison + self.assertEqual(0.0, cv.norm(expected_mat, actual_mat, cv.NORM_INF), + 'Failed on ' + pkg_name + ' backend') + self.assertEqual(expected_mat.dtype, actual_mat.dtype, + 'Failed on ' + pkg_name + ' backend') + self.assertEqual(expected_thresh, actual_thresh[0], + 'Failed on ' + pkg_name + ' backend') + + + def test_kmeans(self): + # K-means params + count = 100 + sz = (count, 2) + in_mat = np.random.random(sz).astype(np.float32) + K = 5 + flags = cv.KMEANS_RANDOM_CENTERS + attempts = 1 + criteria = (cv.TERM_CRITERIA_MAX_ITER + cv.TERM_CRITERIA_EPS, 30, 0) + + # G-API + g_in = cv.GMat() + compactness, out_labels, centers = cv.gapi.kmeans(g_in, K, criteria, attempts, flags) + comp = cv.GComputation(cv.GIn(g_in), cv.GOut(compactness, out_labels, centers)) + + compact, labels, centers = comp.apply(cv.gin(in_mat)) + + # Assert + self.assertTrue(compact >= 0) + self.assertEqual(sz[0], labels.shape[0]) + self.assertEqual(1, labels.shape[1]) + self.assertTrue(labels.size != 0) + self.assertEqual(centers.shape[1], sz[1]) + self.assertEqual(centers.shape[0], K) + self.assertTrue(centers.size != 0) + + + def generate_random_points(self, sz): + arr = np.random.random(sz).astype(np.float32).T + return list(zip(arr[0], arr[1])) + + + def test_kmeans_2d(self): + # K-means 2D params + count = 100 + sz = (count, 2) + amount = sz[0] + K = 5 + flags = cv.KMEANS_RANDOM_CENTERS + attempts = 1 + criteria = (cv.TERM_CRITERIA_MAX_ITER + cv.TERM_CRITERIA_EPS, 30, 0) + in_vector = self.generate_random_points(sz) + in_labels = [] + + # G-API + data = cv.GArrayT(cv.gapi.CV_POINT2F) + best_labels = cv.GArrayT(cv.gapi.CV_INT) + + compactness, out_labels, centers = cv.gapi.kmeans(data, K, best_labels, criteria, attempts, flags) + comp = cv.GComputation(cv.GIn(data, best_labels), cv.GOut(compactness, out_labels, centers)) + + compact, labels, centers = comp.apply(cv.gin(in_vector, in_labels)) + + # Assert + self.assertTrue(compact >= 0) + self.assertEqual(amount, len(labels)) + self.assertEqual(K, len(centers)) + + +except unittest.SkipTest as e: + + message = str(e) + + class TestSkip(unittest.TestCase): + def setUp(self): + self.skipTest('Skip tests: ' + message) + + def test_skip(): + pass + + pass if __name__ == '__main__': diff --git a/modules/gapi/misc/python/test/test_gapi_imgproc.py b/modules/gapi/misc/python/test/test_gapi_imgproc.py index ed6f883fe5..365a5a8cca 100644 --- a/modules/gapi/misc/python/test/test_gapi_imgproc.py +++ b/modules/gapi/misc/python/test/test_gapi_imgproc.py @@ -3,103 +3,124 @@ import numpy as np import cv2 as cv import os +import sys +import unittest from tests_common import NewOpenCVTests -# Plaidml is an optional backend -pkgs = [ - ('ocl' , cv.gapi.core.ocl.kernels()), - ('cpu' , cv.gapi.core.cpu.kernels()), - ('fluid' , cv.gapi.core.fluid.kernels()) - # ('plaidml', cv.gapi.core.plaidml.kernels()) - ] +try: + + if sys.version_info[:2] < (3, 0): + raise unittest.SkipTest('Python 2.x is not supported') + + # Plaidml is an optional backend + pkgs = [ + ('ocl' , cv.gapi.core.ocl.kernels()), + ('cpu' , cv.gapi.core.cpu.kernels()), + ('fluid' , cv.gapi.core.fluid.kernels()) + # ('plaidml', cv.gapi.core.plaidml.kernels()) + ] -class gapi_imgproc_test(NewOpenCVTests): + class gapi_imgproc_test(NewOpenCVTests): - def test_good_features_to_track(self): - # TODO: Extend to use any type and size here - img_path = self.find_file('cv/face/david2.jpg', [os.environ.get('OPENCV_TEST_DATA_PATH')]) - in1 = cv.cvtColor(cv.imread(img_path), cv.COLOR_RGB2GRAY) + def test_good_features_to_track(self): + # TODO: Extend to use any type and size here + img_path = self.find_file('cv/face/david2.jpg', [os.environ.get('OPENCV_TEST_DATA_PATH')]) + in1 = cv.cvtColor(cv.imread(img_path), cv.COLOR_RGB2GRAY) - # NB: goodFeaturesToTrack configuration - max_corners = 50 - quality_lvl = 0.01 - min_distance = 10 - block_sz = 3 - use_harris_detector = True - k = 0.04 - mask = None + # NB: goodFeaturesToTrack configuration + max_corners = 50 + quality_lvl = 0.01 + min_distance = 10 + block_sz = 3 + use_harris_detector = True + k = 0.04 + mask = None - # OpenCV - expected = cv.goodFeaturesToTrack(in1, max_corners, quality_lvl, - min_distance, mask=mask, - blockSize=block_sz, useHarrisDetector=use_harris_detector, k=k) + # OpenCV + expected = cv.goodFeaturesToTrack(in1, max_corners, quality_lvl, + min_distance, mask=mask, + blockSize=block_sz, useHarrisDetector=use_harris_detector, k=k) - # G-API - g_in = cv.GMat() - g_out = cv.gapi.goodFeaturesToTrack(g_in, max_corners, quality_lvl, - min_distance, mask, block_sz, use_harris_detector, k) + # G-API + g_in = cv.GMat() + g_out = cv.gapi.goodFeaturesToTrack(g_in, max_corners, quality_lvl, + min_distance, mask, block_sz, use_harris_detector, k) - comp = cv.GComputation(cv.GIn(g_in), cv.GOut(g_out)) + comp = cv.GComputation(cv.GIn(g_in), cv.GOut(g_out)) - for pkg_name, pkg in pkgs: - actual = comp.apply(cv.gin(in1), args=cv.compile_args(pkg)) - # NB: OpenCV & G-API have different output shapes: - # OpenCV - (num_points, 1, 2) - # G-API - (num_points, 2) - # Comparison - self.assertEqual(0.0, cv.norm(expected.flatten(), - np.array(actual, dtype=np.float32).flatten(), - cv.NORM_INF), - 'Failed on ' + pkg_name + ' backend') + for pkg_name, pkg in pkgs: + actual = comp.apply(cv.gin(in1), args=cv.gapi.compile_args(pkg)) + # NB: OpenCV & G-API have different output shapes: + # OpenCV - (num_points, 1, 2) + # G-API - (num_points, 2) + # Comparison + self.assertEqual(0.0, cv.norm(expected.flatten(), + np.array(actual, dtype=np.float32).flatten(), + cv.NORM_INF), + 'Failed on ' + pkg_name + ' backend') - def test_rgb2gray(self): - # TODO: Extend to use any type and size here - img_path = self.find_file('cv/face/david2.jpg', [os.environ.get('OPENCV_TEST_DATA_PATH')]) - in1 = cv.imread(img_path) + def test_rgb2gray(self): + # TODO: Extend to use any type and size here + img_path = self.find_file('cv/face/david2.jpg', [os.environ.get('OPENCV_TEST_DATA_PATH')]) + in1 = cv.imread(img_path) - # OpenCV - expected = cv.cvtColor(in1, cv.COLOR_RGB2GRAY) + # OpenCV + expected = cv.cvtColor(in1, cv.COLOR_RGB2GRAY) - # G-API - g_in = cv.GMat() - g_out = cv.gapi.RGB2Gray(g_in) + # G-API + g_in = cv.GMat() + g_out = cv.gapi.RGB2Gray(g_in) - comp = cv.GComputation(cv.GIn(g_in), cv.GOut(g_out)) + comp = cv.GComputation(cv.GIn(g_in), cv.GOut(g_out)) - for pkg_name, pkg in pkgs: - actual = comp.apply(cv.gin(in1), args=cv.compile_args(pkg)) - # Comparison - self.assertEqual(0.0, cv.norm(expected, actual, cv.NORM_INF), - 'Failed on ' + pkg_name + ' backend') + for pkg_name, pkg in pkgs: + actual = comp.apply(cv.gin(in1), args=cv.gapi.compile_args(pkg)) + # Comparison + self.assertEqual(0.0, cv.norm(expected, actual, cv.NORM_INF), + 'Failed on ' + pkg_name + ' backend') - def test_bounding_rect(self): - sz = 1280 - fscale = 256 + def test_bounding_rect(self): + sz = 1280 + fscale = 256 - def sample_value(fscale): - return np.random.uniform(0, 255 * fscale) / fscale + def sample_value(fscale): + return np.random.uniform(0, 255 * fscale) / fscale - points = np.array([(sample_value(fscale), sample_value(fscale)) for _ in range(1280)], np.float32) + points = np.array([(sample_value(fscale), sample_value(fscale)) for _ in range(1280)], np.float32) - # OpenCV - expected = cv.boundingRect(points) + # OpenCV + expected = cv.boundingRect(points) - # G-API - g_in = cv.GMat() - g_out = cv.gapi.boundingRect(g_in) + # G-API + g_in = cv.GMat() + g_out = cv.gapi.boundingRect(g_in) - comp = cv.GComputation(cv.GIn(g_in), cv.GOut(g_out)) + comp = cv.GComputation(cv.GIn(g_in), cv.GOut(g_out)) - for pkg_name, pkg in pkgs: - actual = comp.apply(cv.gin(points), args=cv.compile_args(pkg)) - # Comparison - self.assertEqual(0.0, cv.norm(expected, actual, cv.NORM_INF), - 'Failed on ' + pkg_name + ' backend') + for pkg_name, pkg in pkgs: + actual = comp.apply(cv.gin(points), args=cv.gapi.compile_args(pkg)) + # Comparison + self.assertEqual(0.0, cv.norm(expected, actual, cv.NORM_INF), + 'Failed on ' + pkg_name + ' backend') + + +except unittest.SkipTest as e: + + message = str(e) + + class TestSkip(unittest.TestCase): + def setUp(self): + self.skipTest('Skip tests: ' + message) + + def test_skip(): + pass + + pass if __name__ == '__main__': diff --git a/modules/gapi/misc/python/test/test_gapi_infer.py b/modules/gapi/misc/python/test/test_gapi_infer.py index db048f5786..8ecc957e41 100644 --- a/modules/gapi/misc/python/test/test_gapi_infer.py +++ b/modules/gapi/misc/python/test/test_gapi_infer.py @@ -3,318 +3,338 @@ import numpy as np import cv2 as cv import os +import sys +import unittest from tests_common import NewOpenCVTests -class test_gapi_infer(NewOpenCVTests): +try: - def infer_reference_network(self, model_path, weights_path, img): - net = cv.dnn.readNetFromModelOptimizer(model_path, weights_path) - net.setPreferableBackend(cv.dnn.DNN_BACKEND_INFERENCE_ENGINE) - net.setPreferableTarget(cv.dnn.DNN_TARGET_CPU) - - blob = cv.dnn.blobFromImage(img) - - net.setInput(blob) - return net.forward(net.getUnconnectedOutLayersNames()) + if sys.version_info[:2] < (3, 0): + raise unittest.SkipTest('Python 2.x is not supported') - def make_roi(self, img, roi): - return img[roi[1]:roi[1] + roi[3], roi[0]:roi[0] + roi[2], ...] + class test_gapi_infer(NewOpenCVTests): + + def infer_reference_network(self, model_path, weights_path, img): + net = cv.dnn.readNetFromModelOptimizer(model_path, weights_path) + net.setPreferableBackend(cv.dnn.DNN_BACKEND_INFERENCE_ENGINE) + net.setPreferableTarget(cv.dnn.DNN_TARGET_CPU) + + blob = cv.dnn.blobFromImage(img) + + net.setInput(blob) + return net.forward(net.getUnconnectedOutLayersNames()) - def test_age_gender_infer(self): - # NB: Check IE - if not cv.dnn.DNN_TARGET_CPU in cv.dnn.getAvailableTargets(cv.dnn.DNN_BACKEND_INFERENCE_ENGINE): - return - - root_path = '/omz_intel_models/intel/age-gender-recognition-retail-0013/FP32/age-gender-recognition-retail-0013' - model_path = self.find_file(root_path + '.xml', [os.environ.get('OPENCV_DNN_TEST_DATA_PATH')]) - weights_path = self.find_file(root_path + '.bin', [os.environ.get('OPENCV_DNN_TEST_DATA_PATH')]) - device_id = 'CPU' - - img_path = self.find_file('cv/face/david2.jpg', [os.environ.get('OPENCV_TEST_DATA_PATH')]) - img = cv.resize(cv.imread(img_path), (62,62)) - - # OpenCV DNN - dnn_age, dnn_gender = self.infer_reference_network(model_path, weights_path, img) - - # OpenCV G-API - g_in = cv.GMat() - inputs = cv.GInferInputs() - inputs.setInput('data', g_in) - - outputs = cv.gapi.infer("net", inputs) - age_g = outputs.at("age_conv3") - gender_g = outputs.at("prob") - - comp = cv.GComputation(cv.GIn(g_in), cv.GOut(age_g, gender_g)) - pp = cv.gapi.ie.params("net", model_path, weights_path, device_id) - - gapi_age, gapi_gender = comp.apply(cv.gin(img), args=cv.compile_args(cv.gapi.networks(pp))) - - # Check - self.assertEqual(0.0, cv.norm(dnn_gender, gapi_gender, cv.NORM_INF)) - self.assertEqual(0.0, cv.norm(dnn_age, gapi_age, cv.NORM_INF)) + def make_roi(self, img, roi): + return img[roi[1]:roi[1] + roi[3], roi[0]:roi[0] + roi[2], ...] - def test_age_gender_infer_roi(self): - # NB: Check IE - if not cv.dnn.DNN_TARGET_CPU in cv.dnn.getAvailableTargets(cv.dnn.DNN_BACKEND_INFERENCE_ENGINE): - return + def test_age_gender_infer(self): + # NB: Check IE + if not cv.dnn.DNN_TARGET_CPU in cv.dnn.getAvailableTargets(cv.dnn.DNN_BACKEND_INFERENCE_ENGINE): + return - root_path = '/omz_intel_models/intel/age-gender-recognition-retail-0013/FP32/age-gender-recognition-retail-0013' - model_path = self.find_file(root_path + '.xml', [os.environ.get('OPENCV_DNN_TEST_DATA_PATH')]) - weights_path = self.find_file(root_path + '.bin', [os.environ.get('OPENCV_DNN_TEST_DATA_PATH')]) - device_id = 'CPU' + root_path = '/omz_intel_models/intel/age-gender-recognition-retail-0013/FP32/age-gender-recognition-retail-0013' + model_path = self.find_file(root_path + '.xml', [os.environ.get('OPENCV_DNN_TEST_DATA_PATH')]) + weights_path = self.find_file(root_path + '.bin', [os.environ.get('OPENCV_DNN_TEST_DATA_PATH')]) + device_id = 'CPU' - img_path = self.find_file('cv/face/david2.jpg', [os.environ.get('OPENCV_TEST_DATA_PATH')]) - img = cv.imread(img_path) - roi = (10, 10, 62, 62) + img_path = self.find_file('cv/face/david2.jpg', [os.environ.get('OPENCV_TEST_DATA_PATH')]) + img = cv.resize(cv.imread(img_path), (62,62)) - # OpenCV DNN - dnn_age, dnn_gender = self.infer_reference_network(model_path, + # OpenCV DNN + dnn_age, dnn_gender = self.infer_reference_network(model_path, weights_path, img) + + # OpenCV G-API + g_in = cv.GMat() + inputs = cv.GInferInputs() + inputs.setInput('data', g_in) + + outputs = cv.gapi.infer("net", inputs) + age_g = outputs.at("age_conv3") + gender_g = outputs.at("prob") + + comp = cv.GComputation(cv.GIn(g_in), cv.GOut(age_g, gender_g)) + pp = cv.gapi.ie.params("net", model_path, weights_path, device_id) + + gapi_age, gapi_gender = comp.apply(cv.gin(img), args=cv.gapi.compile_args(cv.gapi.networks(pp))) + + # Check + self.assertEqual(0.0, cv.norm(dnn_gender, gapi_gender, cv.NORM_INF)) + self.assertEqual(0.0, cv.norm(dnn_age, gapi_age, cv.NORM_INF)) + + + def test_age_gender_infer_roi(self): + # NB: Check IE + if not cv.dnn.DNN_TARGET_CPU in cv.dnn.getAvailableTargets(cv.dnn.DNN_BACKEND_INFERENCE_ENGINE): + return + + root_path = '/omz_intel_models/intel/age-gender-recognition-retail-0013/FP32/age-gender-recognition-retail-0013' + model_path = self.find_file(root_path + '.xml', [os.environ.get('OPENCV_DNN_TEST_DATA_PATH')]) + weights_path = self.find_file(root_path + '.bin', [os.environ.get('OPENCV_DNN_TEST_DATA_PATH')]) + device_id = 'CPU' + + img_path = self.find_file('cv/face/david2.jpg', [os.environ.get('OPENCV_TEST_DATA_PATH')]) + img = cv.imread(img_path) + roi = (10, 10, 62, 62) + + # OpenCV DNN + dnn_age, dnn_gender = self.infer_reference_network(model_path, + weights_path, + self.make_roi(img, roi)) + + # OpenCV G-API + g_in = cv.GMat() + g_roi = cv.GOpaqueT(cv.gapi.CV_RECT) + inputs = cv.GInferInputs() + inputs.setInput('data', g_in) + + outputs = cv.gapi.infer("net", g_roi, inputs) + age_g = outputs.at("age_conv3") + gender_g = outputs.at("prob") + + comp = cv.GComputation(cv.GIn(g_in, g_roi), cv.GOut(age_g, gender_g)) + pp = cv.gapi.ie.params("net", model_path, weights_path, device_id) + + gapi_age, gapi_gender = comp.apply(cv.gin(img, roi), args=cv.gapi.compile_args(cv.gapi.networks(pp))) + + # Check + self.assertEqual(0.0, cv.norm(dnn_gender, gapi_gender, cv.NORM_INF)) + self.assertEqual(0.0, cv.norm(dnn_age, gapi_age, cv.NORM_INF)) + + + def test_age_gender_infer_roi_list(self): + # NB: Check IE + if not cv.dnn.DNN_TARGET_CPU in cv.dnn.getAvailableTargets(cv.dnn.DNN_BACKEND_INFERENCE_ENGINE): + return + + root_path = '/omz_intel_models/intel/age-gender-recognition-retail-0013/FP32/age-gender-recognition-retail-0013' + model_path = self.find_file(root_path + '.xml', [os.environ.get('OPENCV_DNN_TEST_DATA_PATH')]) + weights_path = self.find_file(root_path + '.bin', [os.environ.get('OPENCV_DNN_TEST_DATA_PATH')]) + device_id = 'CPU' + + rois = [(10, 15, 62, 62), (23, 50, 62, 62), (14, 100, 62, 62), (80, 50, 62, 62)] + img_path = self.find_file('cv/face/david2.jpg', [os.environ.get('OPENCV_TEST_DATA_PATH')]) + img = cv.imread(img_path) + + # OpenCV DNN + dnn_age_list = [] + dnn_gender_list = [] + for roi in rois: + age, gender = self.infer_reference_network(model_path, weights_path, self.make_roi(img, roi)) + dnn_age_list.append(age) + dnn_gender_list.append(gender) - # OpenCV G-API - g_in = cv.GMat() - g_roi = cv.GOpaqueT(cv.gapi.CV_RECT) - inputs = cv.GInferInputs() - inputs.setInput('data', g_in) + # OpenCV G-API + g_in = cv.GMat() + g_rois = cv.GArrayT(cv.gapi.CV_RECT) + inputs = cv.GInferInputs() + inputs.setInput('data', g_in) - outputs = cv.gapi.infer("net", g_roi, inputs) - age_g = outputs.at("age_conv3") - gender_g = outputs.at("prob") + outputs = cv.gapi.infer("net", g_rois, inputs) + age_g = outputs.at("age_conv3") + gender_g = outputs.at("prob") - comp = cv.GComputation(cv.GIn(g_in, g_roi), cv.GOut(age_g, gender_g)) - pp = cv.gapi.ie.params("net", model_path, weights_path, device_id) + comp = cv.GComputation(cv.GIn(g_in, g_rois), cv.GOut(age_g, gender_g)) + pp = cv.gapi.ie.params("net", model_path, weights_path, device_id) - gapi_age, gapi_gender = comp.apply(cv.gin(img, roi), args=cv.compile_args(cv.gapi.networks(pp))) + gapi_age_list, gapi_gender_list = comp.apply(cv.gin(img, rois), + args=cv.gapi.compile_args(cv.gapi.networks(pp))) - # Check - self.assertEqual(0.0, cv.norm(dnn_gender, gapi_gender, cv.NORM_INF)) - self.assertEqual(0.0, cv.norm(dnn_age, gapi_age, cv.NORM_INF)) + # Check + for gapi_age, gapi_gender, dnn_age, dnn_gender in zip(gapi_age_list, + gapi_gender_list, + dnn_age_list, + dnn_gender_list): + self.assertEqual(0.0, cv.norm(dnn_gender, gapi_gender, cv.NORM_INF)) + self.assertEqual(0.0, cv.norm(dnn_age, gapi_age, cv.NORM_INF)) - def test_age_gender_infer_roi_list(self): - # NB: Check IE - if not cv.dnn.DNN_TARGET_CPU in cv.dnn.getAvailableTargets(cv.dnn.DNN_BACKEND_INFERENCE_ENGINE): - return + def test_age_gender_infer2_roi(self): + # NB: Check IE + if not cv.dnn.DNN_TARGET_CPU in cv.dnn.getAvailableTargets(cv.dnn.DNN_BACKEND_INFERENCE_ENGINE): + return - root_path = '/omz_intel_models/intel/age-gender-recognition-retail-0013/FP32/age-gender-recognition-retail-0013' - model_path = self.find_file(root_path + '.xml', [os.environ.get('OPENCV_DNN_TEST_DATA_PATH')]) - weights_path = self.find_file(root_path + '.bin', [os.environ.get('OPENCV_DNN_TEST_DATA_PATH')]) - device_id = 'CPU' + root_path = '/omz_intel_models/intel/age-gender-recognition-retail-0013/FP32/age-gender-recognition-retail-0013' + model_path = self.find_file(root_path + '.xml', [os.environ.get('OPENCV_DNN_TEST_DATA_PATH')]) + weights_path = self.find_file(root_path + '.bin', [os.environ.get('OPENCV_DNN_TEST_DATA_PATH')]) + device_id = 'CPU' - rois = [(10, 15, 62, 62), (23, 50, 62, 62), (14, 100, 62, 62), (80, 50, 62, 62)] - img_path = self.find_file('cv/face/david2.jpg', [os.environ.get('OPENCV_TEST_DATA_PATH')]) - img = cv.imread(img_path) + rois = [(10, 15, 62, 62), (23, 50, 62, 62), (14, 100, 62, 62), (80, 50, 62, 62)] + img_path = self.find_file('cv/face/david2.jpg', [os.environ.get('OPENCV_TEST_DATA_PATH')]) + img = cv.imread(img_path) - # OpenCV DNN - dnn_age_list = [] - dnn_gender_list = [] - for roi in rois: - age, gender = self.infer_reference_network(model_path, - weights_path, - self.make_roi(img, roi)) - dnn_age_list.append(age) - dnn_gender_list.append(gender) + # OpenCV DNN + dnn_age_list = [] + dnn_gender_list = [] + for roi in rois: + age, gender = self.infer_reference_network(model_path, + weights_path, + self.make_roi(img, roi)) + dnn_age_list.append(age) + dnn_gender_list.append(gender) - # OpenCV G-API - g_in = cv.GMat() - g_rois = cv.GArrayT(cv.gapi.CV_RECT) - inputs = cv.GInferInputs() - inputs.setInput('data', g_in) + # OpenCV G-API + g_in = cv.GMat() + g_rois = cv.GArrayT(cv.gapi.CV_RECT) + inputs = cv.GInferListInputs() + inputs.setInput('data', g_rois) - outputs = cv.gapi.infer("net", g_rois, inputs) - age_g = outputs.at("age_conv3") - gender_g = outputs.at("prob") + outputs = cv.gapi.infer2("net", g_in, inputs) + age_g = outputs.at("age_conv3") + gender_g = outputs.at("prob") - comp = cv.GComputation(cv.GIn(g_in, g_rois), cv.GOut(age_g, gender_g)) - pp = cv.gapi.ie.params("net", model_path, weights_path, device_id) + comp = cv.GComputation(cv.GIn(g_in, g_rois), cv.GOut(age_g, gender_g)) + pp = cv.gapi.ie.params("net", model_path, weights_path, device_id) - gapi_age_list, gapi_gender_list = comp.apply(cv.gin(img, rois), - args=cv.compile_args(cv.gapi.networks(pp))) + gapi_age_list, gapi_gender_list = comp.apply(cv.gin(img, rois), + args=cv.gapi.compile_args(cv.gapi.networks(pp))) - # Check - for gapi_age, gapi_gender, dnn_age, dnn_gender in zip(gapi_age_list, - gapi_gender_list, - dnn_age_list, - dnn_gender_list): - self.assertEqual(0.0, cv.norm(dnn_gender, gapi_gender, cv.NORM_INF)) - self.assertEqual(0.0, cv.norm(dnn_age, gapi_age, cv.NORM_INF)) - - - def test_age_gender_infer2_roi(self): - # NB: Check IE - if not cv.dnn.DNN_TARGET_CPU in cv.dnn.getAvailableTargets(cv.dnn.DNN_BACKEND_INFERENCE_ENGINE): - return - - root_path = '/omz_intel_models/intel/age-gender-recognition-retail-0013/FP32/age-gender-recognition-retail-0013' - model_path = self.find_file(root_path + '.xml', [os.environ.get('OPENCV_DNN_TEST_DATA_PATH')]) - weights_path = self.find_file(root_path + '.bin', [os.environ.get('OPENCV_DNN_TEST_DATA_PATH')]) - device_id = 'CPU' - - rois = [(10, 15, 62, 62), (23, 50, 62, 62), (14, 100, 62, 62), (80, 50, 62, 62)] - img_path = self.find_file('cv/face/david2.jpg', [os.environ.get('OPENCV_TEST_DATA_PATH')]) - img = cv.imread(img_path) - - # OpenCV DNN - dnn_age_list = [] - dnn_gender_list = [] - for roi in rois: - age, gender = self.infer_reference_network(model_path, - weights_path, - self.make_roi(img, roi)) - dnn_age_list.append(age) - dnn_gender_list.append(gender) - - # OpenCV G-API - g_in = cv.GMat() - g_rois = cv.GArrayT(cv.gapi.CV_RECT) - inputs = cv.GInferListInputs() - inputs.setInput('data', g_rois) - - outputs = cv.gapi.infer2("net", g_in, inputs) - age_g = outputs.at("age_conv3") - gender_g = outputs.at("prob") - - comp = cv.GComputation(cv.GIn(g_in, g_rois), cv.GOut(age_g, gender_g)) - pp = cv.gapi.ie.params("net", model_path, weights_path, device_id) - - gapi_age_list, gapi_gender_list = comp.apply(cv.gin(img, rois), - args=cv.compile_args(cv.gapi.networks(pp))) - - # Check - for gapi_age, gapi_gender, dnn_age, dnn_gender in zip(gapi_age_list, - gapi_gender_list, - dnn_age_list, - dnn_gender_list): - self.assertEqual(0.0, cv.norm(dnn_gender, gapi_gender, cv.NORM_INF)) - self.assertEqual(0.0, cv.norm(dnn_age, gapi_age, cv.NORM_INF)) + # Check + for gapi_age, gapi_gender, dnn_age, dnn_gender in zip(gapi_age_list, + gapi_gender_list, + dnn_age_list, + dnn_gender_list): + self.assertEqual(0.0, cv.norm(dnn_gender, gapi_gender, cv.NORM_INF)) + self.assertEqual(0.0, cv.norm(dnn_age, gapi_age, cv.NORM_INF)) - def test_person_detection_retail_0013(self): - # NB: Check IE - if not cv.dnn.DNN_TARGET_CPU in cv.dnn.getAvailableTargets(cv.dnn.DNN_BACKEND_INFERENCE_ENGINE): - return + def test_person_detection_retail_0013(self): + # NB: Check IE + if not cv.dnn.DNN_TARGET_CPU in cv.dnn.getAvailableTargets(cv.dnn.DNN_BACKEND_INFERENCE_ENGINE): + return - root_path = '/omz_intel_models/intel/person-detection-retail-0013/FP32/person-detection-retail-0013' - model_path = self.find_file(root_path + '.xml', [os.environ.get('OPENCV_DNN_TEST_DATA_PATH')]) - weights_path = self.find_file(root_path + '.bin', [os.environ.get('OPENCV_DNN_TEST_DATA_PATH')]) - img_path = self.find_file('gpu/lbpcascade/er.png', [os.environ.get('OPENCV_TEST_DATA_PATH')]) - device_id = 'CPU' - img = cv.resize(cv.imread(img_path), (544, 320)) + root_path = '/omz_intel_models/intel/person-detection-retail-0013/FP32/person-detection-retail-0013' + model_path = self.find_file(root_path + '.xml', [os.environ.get('OPENCV_DNN_TEST_DATA_PATH')]) + weights_path = self.find_file(root_path + '.bin', [os.environ.get('OPENCV_DNN_TEST_DATA_PATH')]) + img_path = self.find_file('gpu/lbpcascade/er.png', [os.environ.get('OPENCV_TEST_DATA_PATH')]) + device_id = 'CPU' + img = cv.resize(cv.imread(img_path), (544, 320)) - # OpenCV DNN - net = cv.dnn.readNetFromModelOptimizer(model_path, weights_path) - net.setPreferableBackend(cv.dnn.DNN_BACKEND_INFERENCE_ENGINE) - net.setPreferableTarget(cv.dnn.DNN_TARGET_CPU) + # OpenCV DNN + net = cv.dnn.readNetFromModelOptimizer(model_path, weights_path) + net.setPreferableBackend(cv.dnn.DNN_BACKEND_INFERENCE_ENGINE) + net.setPreferableTarget(cv.dnn.DNN_TARGET_CPU) - blob = cv.dnn.blobFromImage(img) + blob = cv.dnn.blobFromImage(img) - def parseSSD(detections, size): - h, w = size - bboxes = [] - detections = detections.reshape(-1, 7) - for sample_id, class_id, confidence, xmin, ymin, xmax, ymax in detections: - if confidence >= 0.5: - x = int(xmin * w) - y = int(ymin * h) - width = int(xmax * w - x) - height = int(ymax * h - y) - bboxes.append((x, y, width, height)) + def parseSSD(detections, size): + h, w = size + bboxes = [] + detections = detections.reshape(-1, 7) + for sample_id, class_id, confidence, xmin, ymin, xmax, ymax in detections: + if confidence >= 0.5: + x = int(xmin * w) + y = int(ymin * h) + width = int(xmax * w - x) + height = int(ymax * h - y) + bboxes.append((x, y, width, height)) - return bboxes + return bboxes - net.setInput(blob) - dnn_detections = net.forward() - dnn_boxes = parseSSD(np.array(dnn_detections), img.shape[:2]) + net.setInput(blob) + dnn_detections = net.forward() + dnn_boxes = parseSSD(np.array(dnn_detections), img.shape[:2]) - # OpenCV G-API - g_in = cv.GMat() - inputs = cv.GInferInputs() - inputs.setInput('data', g_in) + # OpenCV G-API + g_in = cv.GMat() + inputs = cv.GInferInputs() + inputs.setInput('data', g_in) - g_sz = cv.gapi.streaming.size(g_in) - outputs = cv.gapi.infer("net", inputs) - detections = outputs.at("detection_out") - bboxes = cv.gapi.parseSSD(detections, g_sz, 0.5, False, False) + g_sz = cv.gapi.streaming.size(g_in) + outputs = cv.gapi.infer("net", inputs) + detections = outputs.at("detection_out") + bboxes = cv.gapi.parseSSD(detections, g_sz, 0.5, False, False) - comp = cv.GComputation(cv.GIn(g_in), cv.GOut(bboxes)) - pp = cv.gapi.ie.params("net", model_path, weights_path, device_id) + comp = cv.GComputation(cv.GIn(g_in), cv.GOut(bboxes)) + pp = cv.gapi.ie.params("net", model_path, weights_path, device_id) - gapi_age, gapi_gender = comp.apply(cv.gin(img), args=cv.compile_args(cv.gapi.networks(pp))) + gapi_boxes = comp.apply(cv.gin(img.astype(np.float32)), + args=cv.gapi.compile_args(cv.gapi.networks(pp))) - gapi_boxes = comp.apply(cv.gin(img.astype(np.float32)), - args=cv.compile_args(cv.gapi.networks(pp))) - - # Comparison - self.assertEqual(0.0, cv.norm(np.array(dnn_boxes).flatten(), - np.array(gapi_boxes).flatten(), - cv.NORM_INF)) + # Comparison + self.assertEqual(0.0, cv.norm(np.array(dnn_boxes).flatten(), + np.array(gapi_boxes).flatten(), + cv.NORM_INF)) - def test_person_detection_retail_0013(self): - # NB: Check IE - if not cv.dnn.DNN_TARGET_CPU in cv.dnn.getAvailableTargets(cv.dnn.DNN_BACKEND_INFERENCE_ENGINE): - return + def test_person_detection_retail_0013(self): + # NB: Check IE + if not cv.dnn.DNN_TARGET_CPU in cv.dnn.getAvailableTargets(cv.dnn.DNN_BACKEND_INFERENCE_ENGINE): + return - root_path = '/omz_intel_models/intel/person-detection-retail-0013/FP32/person-detection-retail-0013' - model_path = self.find_file(root_path + '.xml', [os.environ.get('OPENCV_DNN_TEST_DATA_PATH')]) - weights_path = self.find_file(root_path + '.bin', [os.environ.get('OPENCV_DNN_TEST_DATA_PATH')]) - img_path = self.find_file('gpu/lbpcascade/er.png', [os.environ.get('OPENCV_TEST_DATA_PATH')]) - device_id = 'CPU' - img = cv.resize(cv.imread(img_path), (544, 320)) + root_path = '/omz_intel_models/intel/person-detection-retail-0013/FP32/person-detection-retail-0013' + model_path = self.find_file(root_path + '.xml', [os.environ.get('OPENCV_DNN_TEST_DATA_PATH')]) + weights_path = self.find_file(root_path + '.bin', [os.environ.get('OPENCV_DNN_TEST_DATA_PATH')]) + img_path = self.find_file('gpu/lbpcascade/er.png', [os.environ.get('OPENCV_TEST_DATA_PATH')]) + device_id = 'CPU' + img = cv.resize(cv.imread(img_path), (544, 320)) - # OpenCV DNN - net = cv.dnn.readNetFromModelOptimizer(model_path, weights_path) - net.setPreferableBackend(cv.dnn.DNN_BACKEND_INFERENCE_ENGINE) - net.setPreferableTarget(cv.dnn.DNN_TARGET_CPU) + # OpenCV DNN + net = cv.dnn.readNetFromModelOptimizer(model_path, weights_path) + net.setPreferableBackend(cv.dnn.DNN_BACKEND_INFERENCE_ENGINE) + net.setPreferableTarget(cv.dnn.DNN_TARGET_CPU) - blob = cv.dnn.blobFromImage(img) + blob = cv.dnn.blobFromImage(img) - def parseSSD(detections, size): - h, w = size - bboxes = [] - detections = detections.reshape(-1, 7) - for sample_id, class_id, confidence, xmin, ymin, xmax, ymax in detections: - if confidence >= 0.5: - x = int(xmin * w) - y = int(ymin * h) - width = int(xmax * w - x) - height = int(ymax * h - y) - bboxes.append((x, y, width, height)) + def parseSSD(detections, size): + h, w = size + bboxes = [] + detections = detections.reshape(-1, 7) + for sample_id, class_id, confidence, xmin, ymin, xmax, ymax in detections: + if confidence >= 0.5: + x = int(xmin * w) + y = int(ymin * h) + width = int(xmax * w - x) + height = int(ymax * h - y) + bboxes.append((x, y, width, height)) - return bboxes + return bboxes - net.setInput(blob) - dnn_detections = net.forward() - dnn_boxes = parseSSD(np.array(dnn_detections), img.shape[:2]) + net.setInput(blob) + dnn_detections = net.forward() + dnn_boxes = parseSSD(np.array(dnn_detections), img.shape[:2]) - # OpenCV G-API - g_in = cv.GMat() - inputs = cv.GInferInputs() - inputs.setInput('data', g_in) + # OpenCV G-API + g_in = cv.GMat() + inputs = cv.GInferInputs() + inputs.setInput('data', g_in) - g_sz = cv.gapi.streaming.size(g_in) - outputs = cv.gapi.infer("net", inputs) - detections = outputs.at("detection_out") - bboxes = cv.gapi.parseSSD(detections, g_sz, 0.5, False, False) + g_sz = cv.gapi.streaming.size(g_in) + outputs = cv.gapi.infer("net", inputs) + detections = outputs.at("detection_out") + bboxes = cv.gapi.parseSSD(detections, g_sz, 0.5, False, False) - comp = cv.GComputation(cv.GIn(g_in), cv.GOut(bboxes)) - pp = cv.gapi.ie.params("net", model_path, weights_path, device_id) + comp = cv.GComputation(cv.GIn(g_in), cv.GOut(bboxes)) + pp = cv.gapi.ie.params("net", model_path, weights_path, device_id) - gapi_boxes = comp.apply(cv.gin(img.astype(np.float32)), - args=cv.compile_args(cv.gapi.networks(pp))) + gapi_boxes = comp.apply(cv.gin(img.astype(np.float32)), + args=cv.gapi.compile_args(cv.gapi.networks(pp))) - # Comparison - self.assertEqual(0.0, cv.norm(np.array(dnn_boxes).flatten(), - np.array(gapi_boxes).flatten(), - cv.NORM_INF)) + # Comparison + self.assertEqual(0.0, cv.norm(np.array(dnn_boxes).flatten(), + np.array(gapi_boxes).flatten(), + cv.NORM_INF)) + + +except unittest.SkipTest as e: + + message = str(e) + + class TestSkip(unittest.TestCase): + def setUp(self): + self.skipTest('Skip tests: ' + message) + + def test_skip(): + pass + + pass if __name__ == '__main__': diff --git a/modules/gapi/misc/python/test/test_gapi_sample_pipelines.py b/modules/gapi/misc/python/test/test_gapi_sample_pipelines.py index 2f921901db..a10d63f09e 100644 --- a/modules/gapi/misc/python/test/test_gapi_sample_pipelines.py +++ b/modules/gapi/misc/python/test/test_gapi_sample_pipelines.py @@ -225,7 +225,7 @@ try: comp = cv.GComputation(cv.GIn(g_in1, g_in2), cv.GOut(g_out)) pkg = cv.gapi.kernels(GAddImpl) - actual = comp.apply(cv.gin(in_mat1, in_mat2), args=cv.compile_args(pkg)) + actual = comp.apply(cv.gin(in_mat1, in_mat2), args=cv.gapi.compile_args(pkg)) self.assertEqual(0.0, cv.norm(expected, actual, cv.NORM_INF)) @@ -245,7 +245,7 @@ try: comp = cv.GComputation(cv.GIn(g_in), cv.GOut(g_ch1, g_ch2, g_ch3)) pkg = cv.gapi.kernels(GSplit3Impl) - ch1, ch2, ch3 = comp.apply(cv.gin(in_mat), args=cv.compile_args(pkg)) + ch1, ch2, ch3 = comp.apply(cv.gin(in_mat), args=cv.gapi.compile_args(pkg)) self.assertEqual(0.0, cv.norm(in_ch1, ch1, cv.NORM_INF)) self.assertEqual(0.0, cv.norm(in_ch2, ch2, cv.NORM_INF)) @@ -266,7 +266,7 @@ try: comp = cv.GComputation(g_in, g_out) pkg = cv.gapi.kernels(GMeanImpl) - actual = comp.apply(cv.gin(in_mat), args=cv.compile_args(pkg)) + actual = comp.apply(cv.gin(in_mat), args=cv.gapi.compile_args(pkg)) # Comparison self.assertEqual(expected, actual) @@ -287,7 +287,7 @@ try: comp = cv.GComputation(cv.GIn(g_in, g_sc), cv.GOut(g_out)) pkg = cv.gapi.kernels(GAddCImpl) - actual = comp.apply(cv.gin(in_mat, sc), args=cv.compile_args(pkg)) + actual = comp.apply(cv.gin(in_mat, sc), args=cv.gapi.compile_args(pkg)) self.assertEqual(0.0, cv.norm(expected, actual, cv.NORM_INF)) @@ -305,7 +305,7 @@ try: comp = cv.GComputation(cv.GIn(g_in), cv.GOut(g_sz)) pkg = cv.gapi.kernels(GSizeImpl) - actual = comp.apply(cv.gin(in_mat), args=cv.compile_args(pkg)) + actual = comp.apply(cv.gin(in_mat), args=cv.gapi.compile_args(pkg)) self.assertEqual(0.0, cv.norm(expected, actual, cv.NORM_INF)) @@ -322,7 +322,7 @@ try: comp = cv.GComputation(cv.GIn(g_r), cv.GOut(g_sz)) pkg = cv.gapi.kernels(GSizeRImpl) - actual = comp.apply(cv.gin(roi), args=cv.compile_args(pkg)) + actual = comp.apply(cv.gin(roi), args=cv.gapi.compile_args(pkg)) # cv.norm works with tuples ? self.assertEqual(0.0, cv.norm(expected, actual, cv.NORM_INF)) @@ -340,7 +340,7 @@ try: comp = cv.GComputation(cv.GIn(g_pts), cv.GOut(g_br)) pkg = cv.gapi.kernels(GBoundingRectImpl) - actual = comp.apply(cv.gin(points), args=cv.compile_args(pkg)) + actual = comp.apply(cv.gin(points), args=cv.gapi.compile_args(pkg)) # cv.norm works with tuples ? self.assertEqual(0.0, cv.norm(expected, actual, cv.NORM_INF)) @@ -371,7 +371,7 @@ try: comp = cv.GComputation(cv.GIn(g_in), cv.GOut(g_out)) pkg = cv.gapi.kernels(GGoodFeaturesImpl) - actual = comp.apply(cv.gin(in_mat), args=cv.compile_args(pkg)) + actual = comp.apply(cv.gin(in_mat), args=cv.gapi.compile_args(pkg)) # NB: OpenCV & G-API have different output types. # OpenCV - numpy array with shape (num_points, 1, 2) @@ -453,10 +453,10 @@ try: g_in = cv.GArray.Int() comp = cv.GComputation(cv.GIn(g_in), cv.GOut(GSum.on(g_in))) - s = comp.apply(cv.gin([1, 2, 3, 4]), args=cv.compile_args(cv.gapi.kernels(GSumImpl))) + s = comp.apply(cv.gin([1, 2, 3, 4]), args=cv.gapi.compile_args(cv.gapi.kernels(GSumImpl))) self.assertEqual(10, s) - s = comp.apply(cv.gin([1, 2, 8, 7]), args=cv.compile_args(cv.gapi.kernels(GSumImpl))) + s = comp.apply(cv.gin([1, 2, 8, 7]), args=cv.gapi.compile_args(cv.gapi.kernels(GSumImpl))) self.assertEqual(18, s) self.assertEqual(18, GSumImpl.last_result) @@ -488,13 +488,13 @@ try: 'tuple': (42, 42) } - out = comp.apply(cv.gin(table, 'int'), args=cv.compile_args(cv.gapi.kernels(GLookUpImpl))) + out = comp.apply(cv.gin(table, 'int'), args=cv.gapi.compile_args(cv.gapi.kernels(GLookUpImpl))) self.assertEqual(42, out) - out = comp.apply(cv.gin(table, 'str'), args=cv.compile_args(cv.gapi.kernels(GLookUpImpl))) + out = comp.apply(cv.gin(table, 'str'), args=cv.gapi.compile_args(cv.gapi.kernels(GLookUpImpl))) self.assertEqual('hello, world!', out) - out = comp.apply(cv.gin(table, 'tuple'), args=cv.compile_args(cv.gapi.kernels(GLookUpImpl))) + out = comp.apply(cv.gin(table, 'tuple'), args=cv.gapi.compile_args(cv.gapi.kernels(GLookUpImpl))) self.assertEqual((42, 42), out) @@ -521,7 +521,7 @@ try: arr1 = [3, 'str'] out = comp.apply(cv.gin(arr0, arr1), - args=cv.compile_args(cv.gapi.kernels(GConcatImpl))) + args=cv.gapi.compile_args(cv.gapi.kernels(GConcatImpl))) self.assertEqual(arr0 + arr1, out) @@ -550,7 +550,7 @@ try: img1 = np.array([1, 2, 3]) with self.assertRaises(Exception): comp.apply(cv.gin(img0, img1), - args=cv.compile_args( + args=cv.gapi.compile_args( cv.gapi.kernels(GAddImpl))) @@ -577,7 +577,7 @@ try: img1 = np.array([1, 2, 3]) with self.assertRaises(Exception): comp.apply(cv.gin(img0, img1), - args=cv.compile_args( + args=cv.gapi.compile_args( cv.gapi.kernels(GAddImpl))) @@ -607,7 +607,7 @@ try: # FIXME: Cause Bad variant access. # Need to provide more descriptive error messsage. with self.assertRaises(Exception): comp.apply(cv.gin(img0, img1), - args=cv.compile_args( + args=cv.gapi.compile_args( cv.gapi.kernels(GAddImpl))) def test_pipeline_with_custom_kernels(self): @@ -657,7 +657,7 @@ try: g_mean = cv.gapi.mean(g_transposed) comp = cv.GComputation(cv.GIn(g_bgr), cv.GOut(g_mean)) - actual = comp.apply(cv.gin(img), args=cv.compile_args( + actual = comp.apply(cv.gin(img), args=cv.gapi.compile_args( cv.gapi.kernels(GResizeImpl, GTransposeImpl))) self.assertEqual(0.0, cv.norm(expected, actual, cv.NORM_INF)) diff --git a/modules/gapi/misc/python/test/test_gapi_streaming.py b/modules/gapi/misc/python/test/test_gapi_streaming.py index 5356abc76a..f1cce4fb72 100644 --- a/modules/gapi/misc/python/test/test_gapi_streaming.py +++ b/modules/gapi/misc/python/test/test_gapi_streaming.py @@ -3,201 +3,225 @@ import numpy as np import cv2 as cv import os +import sys +import unittest from tests_common import NewOpenCVTests -class test_gapi_streaming(NewOpenCVTests): - def test_image_input(self): - sz = (1280, 720) - in_mat = np.random.randint(0, 100, sz).astype(np.uint8) +try: - # OpenCV - expected = cv.medianBlur(in_mat, 3) - - # G-API - g_in = cv.GMat() - g_out = cv.gapi.medianBlur(g_in, 3) - c = cv.GComputation(g_in, g_out) - ccomp = c.compileStreaming(cv.descr_of(in_mat)) - ccomp.setSource(cv.gin(in_mat)) - ccomp.start() - - _, actual = ccomp.pull() - - # Assert - self.assertEqual(0.0, cv.norm(expected, actual, cv.NORM_INF)) + if sys.version_info[:2] < (3, 0): + raise unittest.SkipTest('Python 2.x is not supported') - def test_video_input(self): - ksize = 3 - path = self.find_file('cv/video/768x576.avi', [os.environ['OPENCV_TEST_DATA_PATH']]) + class test_gapi_streaming(NewOpenCVTests): - # OpenCV - cap = cv.VideoCapture(path) - - # G-API - g_in = cv.GMat() - g_out = cv.gapi.medianBlur(g_in, ksize) - c = cv.GComputation(g_in, g_out) - - ccomp = c.compileStreaming() - source = cv.gapi.wip.make_capture_src(path) - ccomp.setSource(source) - ccomp.start() - - # Assert - max_num_frames = 10 - proc_num_frames = 0 - while cap.isOpened(): - has_expected, expected = cap.read() - has_actual, actual = ccomp.pull() - - self.assertEqual(has_expected, has_actual) - - if not has_actual: - break - - self.assertEqual(0.0, cv.norm(cv.medianBlur(expected, ksize), actual, cv.NORM_INF)) - - proc_num_frames += 1 - if proc_num_frames == max_num_frames: - break; - - - def test_video_split3(self): - path = self.find_file('cv/video/768x576.avi', [os.environ['OPENCV_TEST_DATA_PATH']]) - - # OpenCV - cap = cv.VideoCapture(path) - - # G-API - g_in = cv.GMat() - b, g, r = cv.gapi.split3(g_in) - c = cv.GComputation(cv.GIn(g_in), cv.GOut(b, g, r)) - - ccomp = c.compileStreaming() - source = cv.gapi.wip.make_capture_src(path) - ccomp.setSource(source) - ccomp.start() - - # Assert - max_num_frames = 10 - proc_num_frames = 0 - while cap.isOpened(): - has_expected, frame = cap.read() - has_actual, actual = ccomp.pull() - - self.assertEqual(has_expected, has_actual) - - if not has_actual: - break - - expected = cv.split(frame) - for e, a in zip(expected, actual): - self.assertEqual(0.0, cv.norm(e, a, cv.NORM_INF)) - - proc_num_frames += 1 - if proc_num_frames == max_num_frames: - break; - - - def test_video_add(self): - sz = (576, 768, 3) - in_mat = np.random.randint(0, 100, sz).astype(np.uint8) - - path = self.find_file('cv/video/768x576.avi', [os.environ['OPENCV_TEST_DATA_PATH']]) - - # OpenCV - cap = cv.VideoCapture(path) - - # G-API - g_in1 = cv.GMat() - g_in2 = cv.GMat() - out = cv.gapi.add(g_in1, g_in2) - c = cv.GComputation(cv.GIn(g_in1, g_in2), cv.GOut(out)) - - ccomp = c.compileStreaming() - source = cv.gapi.wip.make_capture_src(path) - ccomp.setSource(cv.gin(source, in_mat)) - ccomp.start() - - # Assert - max_num_frames = 10 - proc_num_frames = 0 - while cap.isOpened(): - has_expected, frame = cap.read() - has_actual, actual = ccomp.pull() - - self.assertEqual(has_expected, has_actual) - - if not has_actual: - break - - expected = cv.add(frame, in_mat) - self.assertEqual(0.0, cv.norm(expected, actual, cv.NORM_INF)) - - proc_num_frames += 1 - if proc_num_frames == max_num_frames: - break; - - - def test_video_good_features_to_track(self): - path = self.find_file('cv/video/768x576.avi', [os.environ['OPENCV_TEST_DATA_PATH']]) - - # NB: goodFeaturesToTrack configuration - max_corners = 50 - quality_lvl = 0.01 - min_distance = 10 - block_sz = 3 - use_harris_detector = True - k = 0.04 - mask = None - - # OpenCV - cap = cv.VideoCapture(path) - - # G-API - g_in = cv.GMat() - g_gray = cv.gapi.RGB2Gray(g_in) - g_out = cv.gapi.goodFeaturesToTrack(g_gray, max_corners, quality_lvl, - min_distance, mask, block_sz, use_harris_detector, k) - - c = cv.GComputation(cv.GIn(g_in), cv.GOut(g_out)) - - ccomp = c.compileStreaming() - source = cv.gapi.wip.make_capture_src(path) - ccomp.setSource(source) - ccomp.start() - - # Assert - max_num_frames = 10 - proc_num_frames = 0 - while cap.isOpened(): - has_expected, frame = cap.read() - has_actual, actual = ccomp.pull() - - self.assertEqual(has_expected, has_actual) - - if not has_actual: - break + def test_image_input(self): + sz = (1280, 720) + in_mat = np.random.randint(0, 100, sz).astype(np.uint8) # OpenCV - frame = cv.cvtColor(frame, cv.COLOR_RGB2GRAY) - expected = cv.goodFeaturesToTrack(frame, max_corners, quality_lvl, - min_distance, mask=mask, - blockSize=block_sz, useHarrisDetector=use_harris_detector, k=k) - for e, a in zip(expected, actual): - # NB: OpenCV & G-API have different output shapes: - # OpenCV - (num_points, 1, 2) - # G-API - (num_points, 2) - self.assertEqual(0.0, cv.norm(e.flatten(), - np.array(a, np.float32).flatten(), - cv.NORM_INF)) + expected = cv.medianBlur(in_mat, 3) + + # G-API + g_in = cv.GMat() + g_out = cv.gapi.medianBlur(g_in, 3) + c = cv.GComputation(g_in, g_out) + ccomp = c.compileStreaming(cv.descr_of(in_mat)) + ccomp.setSource(cv.gin(in_mat)) + ccomp.start() + + _, actual = ccomp.pull() + + # Assert + self.assertEqual(0.0, cv.norm(expected, actual, cv.NORM_INF)) + + + def test_video_input(self): + ksize = 3 + path = self.find_file('cv/video/768x576.avi', [os.environ['OPENCV_TEST_DATA_PATH']]) + + # OpenCV + cap = cv.VideoCapture(path) + + # G-API + g_in = cv.GMat() + g_out = cv.gapi.medianBlur(g_in, ksize) + c = cv.GComputation(g_in, g_out) + + ccomp = c.compileStreaming() + source = cv.gapi.wip.make_capture_src(path) + ccomp.setSource(source) + ccomp.start() + + # Assert + max_num_frames = 10 + proc_num_frames = 0 + while cap.isOpened(): + has_expected, expected = cap.read() + has_actual, actual = ccomp.pull() + + self.assertEqual(has_expected, has_actual) + + if not has_actual: + break + + self.assertEqual(0.0, cv.norm(cv.medianBlur(expected, ksize), actual, cv.NORM_INF)) + + proc_num_frames += 1 + if proc_num_frames == max_num_frames: + break + + + def test_video_split3(self): + path = self.find_file('cv/video/768x576.avi', [os.environ['OPENCV_TEST_DATA_PATH']]) + + # OpenCV + cap = cv.VideoCapture(path) + + # G-API + g_in = cv.GMat() + b, g, r = cv.gapi.split3(g_in) + c = cv.GComputation(cv.GIn(g_in), cv.GOut(b, g, r)) + + ccomp = c.compileStreaming() + source = cv.gapi.wip.make_capture_src(path) + ccomp.setSource(source) + ccomp.start() + + # Assert + max_num_frames = 10 + proc_num_frames = 0 + while cap.isOpened(): + has_expected, frame = cap.read() + has_actual, actual = ccomp.pull() + + self.assertEqual(has_expected, has_actual) + + if not has_actual: + break + + expected = cv.split(frame) + for e, a in zip(expected, actual): + self.assertEqual(0.0, cv.norm(e, a, cv.NORM_INF)) + + proc_num_frames += 1 + if proc_num_frames == max_num_frames: + break + + + def test_video_add(self): + sz = (576, 768, 3) + in_mat = np.random.randint(0, 100, sz).astype(np.uint8) + + path = self.find_file('cv/video/768x576.avi', [os.environ['OPENCV_TEST_DATA_PATH']]) + + # OpenCV + cap = cv.VideoCapture(path) + + # G-API + g_in1 = cv.GMat() + g_in2 = cv.GMat() + out = cv.gapi.add(g_in1, g_in2) + c = cv.GComputation(cv.GIn(g_in1, g_in2), cv.GOut(out)) + + ccomp = c.compileStreaming() + source = cv.gapi.wip.make_capture_src(path) + ccomp.setSource(cv.gin(source, in_mat)) + ccomp.start() + + # Assert + max_num_frames = 10 + proc_num_frames = 0 + while cap.isOpened(): + has_expected, frame = cap.read() + has_actual, actual = ccomp.pull() + + self.assertEqual(has_expected, has_actual) + + if not has_actual: + break + + expected = cv.add(frame, in_mat) + self.assertEqual(0.0, cv.norm(expected, actual, cv.NORM_INF)) + + proc_num_frames += 1 + if proc_num_frames == max_num_frames: + break; + + + def test_video_good_features_to_track(self): + path = self.find_file('cv/video/768x576.avi', [os.environ['OPENCV_TEST_DATA_PATH']]) + + # NB: goodFeaturesToTrack configuration + max_corners = 50 + quality_lvl = 0.01 + min_distance = 10 + block_sz = 3 + use_harris_detector = True + k = 0.04 + mask = None + + # OpenCV + cap = cv.VideoCapture(path) + + # G-API + g_in = cv.GMat() + g_gray = cv.gapi.RGB2Gray(g_in) + g_out = cv.gapi.goodFeaturesToTrack(g_gray, max_corners, quality_lvl, + min_distance, mask, block_sz, use_harris_detector, k) + + c = cv.GComputation(cv.GIn(g_in), cv.GOut(g_out)) + + ccomp = c.compileStreaming() + source = cv.gapi.wip.make_capture_src(path) + ccomp.setSource(source) + ccomp.start() + + # Assert + max_num_frames = 10 + proc_num_frames = 0 + while cap.isOpened(): + has_expected, frame = cap.read() + has_actual, actual = ccomp.pull() + + self.assertEqual(has_expected, has_actual) + + if not has_actual: + break + + # OpenCV + frame = cv.cvtColor(frame, cv.COLOR_RGB2GRAY) + expected = cv.goodFeaturesToTrack(frame, max_corners, quality_lvl, + min_distance, mask=mask, + blockSize=block_sz, useHarrisDetector=use_harris_detector, k=k) + for e, a in zip(expected, actual): + # NB: OpenCV & G-API have different output shapes: + # OpenCV - (num_points, 1, 2) + # G-API - (num_points, 2) + self.assertEqual(0.0, cv.norm(e.flatten(), + np.array(a, np.float32).flatten(), + cv.NORM_INF)) + + proc_num_frames += 1 + if proc_num_frames == max_num_frames: + break + + +except unittest.SkipTest as e: + + message = str(e) + + class TestSkip(unittest.TestCase): + def setUp(self): + self.skipTest('Skip tests: ' + message) + + def test_skip(): + pass + + pass - proc_num_frames += 1 - if proc_num_frames == max_num_frames: - break; if __name__ == '__main__': NewOpenCVTests.bootstrap() diff --git a/modules/gapi/misc/python/test/test_gapi_types.py b/modules/gapi/misc/python/test/test_gapi_types.py index 0f3b194a2f..dde554f5e1 100644 --- a/modules/gapi/misc/python/test/test_gapi_types.py +++ b/modules/gapi/misc/python/test/test_gapi_types.py @@ -3,29 +3,51 @@ import numpy as np import cv2 as cv import os +import sys +import unittest from tests_common import NewOpenCVTests -class gapi_types_test(NewOpenCVTests): - def test_garray_type(self): - types = [cv.gapi.CV_BOOL , cv.gapi.CV_INT , cv.gapi.CV_DOUBLE , cv.gapi.CV_FLOAT, - cv.gapi.CV_STRING, cv.gapi.CV_POINT , cv.gapi.CV_POINT2F, cv.gapi.CV_SIZE , - cv.gapi.CV_RECT , cv.gapi.CV_SCALAR, cv.gapi.CV_MAT , cv.gapi.CV_GMAT] +try: - for t in types: - g_array = cv.GArrayT(t) - self.assertEqual(t, g_array.type()) + if sys.version_info[:2] < (3, 0): + raise unittest.SkipTest('Python 2.x is not supported') + + class gapi_types_test(NewOpenCVTests): + + def test_garray_type(self): + types = [cv.gapi.CV_BOOL , cv.gapi.CV_INT , cv.gapi.CV_DOUBLE , cv.gapi.CV_FLOAT, + cv.gapi.CV_STRING, cv.gapi.CV_POINT , cv.gapi.CV_POINT2F, cv.gapi.CV_SIZE , + cv.gapi.CV_RECT , cv.gapi.CV_SCALAR, cv.gapi.CV_MAT , cv.gapi.CV_GMAT] + + for t in types: + g_array = cv.GArrayT(t) + self.assertEqual(t, g_array.type()) - def test_gopaque_type(self): - types = [cv.gapi.CV_BOOL , cv.gapi.CV_INT , cv.gapi.CV_DOUBLE , cv.gapi.CV_FLOAT, - cv.gapi.CV_STRING, cv.gapi.CV_POINT , cv.gapi.CV_POINT2F, cv.gapi.CV_SIZE , - cv.gapi.CV_RECT] + def test_gopaque_type(self): + types = [cv.gapi.CV_BOOL , cv.gapi.CV_INT , cv.gapi.CV_DOUBLE , cv.gapi.CV_FLOAT, + cv.gapi.CV_STRING, cv.gapi.CV_POINT , cv.gapi.CV_POINT2F, cv.gapi.CV_SIZE , + cv.gapi.CV_RECT] - for t in types: - g_opaque = cv.GOpaqueT(t) - self.assertEqual(t, g_opaque.type()) + for t in types: + g_opaque = cv.GOpaqueT(t) + self.assertEqual(t, g_opaque.type()) + + +except unittest.SkipTest as e: + + message = str(e) + + class TestSkip(unittest.TestCase): + def setUp(self): + self.skipTest('Skip tests: ' + message) + + def test_skip(): + pass + + pass if __name__ == '__main__': From ef2b400c61ee61211ffaa4233dd44b65974ae2aa Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Sat, 19 Jun 2021 09:16:23 +0000 Subject: [PATCH 013/262] highgui: win32ui plugin --- cmake/OpenCVFindLibsGUI.cmake | 9 - cmake/templates/cvconfig.h.in | 3 - modules/highgui/CMakeLists.txt | 16 +- modules/highgui/cmake/detect_win32ui.cmake | 17 + modules/highgui/cmake/init.cmake | 3 +- modules/highgui/src/backend.hpp | 4 + modules/highgui/src/precomp.hpp | 8 +- modules/highgui/src/registry.impl.hpp | 8 + modules/highgui/src/window.cpp | 65 +- modules/highgui/src/window_QT.cpp | 3 +- modules/highgui/src/window_cocoa.mm | 6 +- modules/highgui/src/window_gtk.cpp | 4 +- modules/highgui/src/window_w32.cpp | 2351 ++++++++++++-------- 13 files changed, 1510 insertions(+), 987 deletions(-) create mode 100644 modules/highgui/cmake/detect_win32ui.cmake diff --git a/cmake/OpenCVFindLibsGUI.cmake b/cmake/OpenCVFindLibsGUI.cmake index 8030e8b0c0..c8ec55b588 100644 --- a/cmake/OpenCVFindLibsGUI.cmake +++ b/cmake/OpenCVFindLibsGUI.cmake @@ -2,15 +2,6 @@ # Detect 3rd-party GUI libraries # ---------------------------------------------------------------------------- -#--- Win32 UI --- -ocv_clear_vars(HAVE_WIN32UI) -if(WITH_WIN32UI) - try_compile(HAVE_WIN32UI - "${OpenCV_BINARY_DIR}" - "${OpenCV_SOURCE_DIR}/cmake/checks/win32uitest.cpp" - CMAKE_FLAGS "-DLINK_LIBRARIES:STRING=user32;gdi32") -endif() - # --- QT4/5 --- ocv_clear_vars(HAVE_QT HAVE_QT5) if(WITH_QT) diff --git a/cmake/templates/cvconfig.h.in b/cmake/templates/cvconfig.h.in index e79e1ec0a1..6439d8b43f 100644 --- a/cmake/templates/cvconfig.h.in +++ b/cmake/templates/cvconfig.h.in @@ -121,9 +121,6 @@ /* TIFF codec */ #cmakedefine HAVE_TIFF -/* Win32 UI */ -#cmakedefine HAVE_WIN32UI - /* Define if your processor stores words with the most significant byte first (like Motorola and SPARC, unlike Intel and VAX). */ #cmakedefine WORDS_BIGENDIAN diff --git a/modules/highgui/CMakeLists.txt b/modules/highgui/CMakeLists.txt index b4d4b9f503..5eb9f5ab5e 100644 --- a/modules/highgui/CMakeLists.txt +++ b/modules/highgui/CMakeLists.txt @@ -131,12 +131,6 @@ elseif(WINRT) message(STATUS " ${name}: Removing 'comctl32.lib, gdi32.lib, ole32.lib, setupapi.lib'") message(STATUS " ${name}: Leaving '${HIGHGUI_LIBRARIES}'") endif() -elseif(HAVE_WIN32UI) - set(OPENCV_HIGHGUI_BUILTIN_BACKEND "WIN32UI") - list(APPEND highgui_srcs ${CMAKE_CURRENT_LIST_DIR}/src/window_w32.cpp) - if(OpenCV_ARCH STREQUAL "ARM64") - list(APPEND HIGHGUI_LIBRARIES "comdlg32" "advapi32") - endif() elseif(HAVE_COCOA) set(OPENCV_HIGHGUI_BUILTIN_BACKEND "COCOA") add_definitions(-DHAVE_COCOA) @@ -144,6 +138,16 @@ elseif(HAVE_COCOA) list(APPEND HIGHGUI_LIBRARIES "-framework Cocoa") endif() +if(TARGET ocv.3rdparty.win32ui) + if("win32ui" IN_LIST HIGHGUI_PLUGIN_LIST OR HIGHGUI_PLUGIN_LIST STREQUAL "all") + ocv_create_builtin_highgui_plugin(opencv_highgui_win32 ocv.3rdparty.win32ui "window_w32.cpp") + else() + set(OPENCV_HIGHGUI_BUILTIN_BACKEND "WIN32UI") + list(APPEND highgui_srcs ${CMAKE_CURRENT_LIST_DIR}/src/window_w32.cpp) + list(APPEND tgts ocv.3rdparty.win32ui) + endif() +endif() + if(TARGET ocv.3rdparty.gtk3 OR TARGET ocv.3rdparty.gtk2) if(TARGET ocv.3rdparty.gtk3 AND NOT WITH_GTK_2_X) set(__gtk_dependency "ocv.3rdparty.gtk3") diff --git a/modules/highgui/cmake/detect_win32ui.cmake b/modules/highgui/cmake/detect_win32ui.cmake new file mode 100644 index 0000000000..1d2fdc5d46 --- /dev/null +++ b/modules/highgui/cmake/detect_win32ui.cmake @@ -0,0 +1,17 @@ +#--- Win32 UI --- +ocv_clear_vars(HAVE_WIN32UI) +if(WITH_WIN32UI) + try_compile(HAVE_WIN32UI + "${CMAKE_CURRENT_BINARY_DIR}" + "${OpenCV_SOURCE_DIR}/cmake/checks/win32uitest.cpp" + CMAKE_FLAGS "-DLINK_LIBRARIES:STRING=user32;gdi32") + if(HAVE_WIN32UI) + set(__libs "user32" "gdi32") + if(OpenCV_ARCH STREQUAL "ARM64") + list(APPEND __libs "comdlg32" "advapi32") + endif() + ocv_add_external_target(win32ui "" "${__libs}" "HAVE_WIN32UI") + endif() +endif() + +set(HAVE_WIN32UI "${HAVE_WIN32UI}" PARENT_SCOPE) # informational diff --git a/modules/highgui/cmake/init.cmake b/modules/highgui/cmake/init.cmake index 3b766b3758..1626d254da 100644 --- a/modules/highgui/cmake/init.cmake +++ b/modules/highgui/cmake/init.cmake @@ -43,8 +43,7 @@ else() endif() add_backend("gtk" WITH_GTK) - -# TODO win32 +add_backend("win32ui" WITH_WIN32UI) # TODO cocoa # TODO qt # TODO opengl diff --git a/modules/highgui/src/backend.hpp b/modules/highgui/src/backend.hpp index 14c88b2387..7c32846ce4 100644 --- a/modules/highgui/src/backend.hpp +++ b/modules/highgui/src/backend.hpp @@ -114,6 +114,10 @@ bool setUIBackend(const std::string& backendName); #ifndef BUILD_PLUGIN +#ifdef HAVE_WIN32UI +std::shared_ptr createUIBackendWin32UI(); +#endif + #ifdef HAVE_GTK std::shared_ptr createUIBackendGTK(); #endif diff --git a/modules/highgui/src/precomp.hpp b/modules/highgui/src/precomp.hpp index 6ad5bce8b4..0d26b957ad 100644 --- a/modules/highgui/src/precomp.hpp +++ b/modules/highgui/src/precomp.hpp @@ -67,7 +67,6 @@ #include #include #include -#include #if defined _WIN32 || defined WINCE #include @@ -127,6 +126,13 @@ void cvSetPropTopmost_COCOA(const char* name, const bool topmost); double cvGetPropVsync_W32(const char* name); void cvSetPropVsync_W32(const char* name, const bool enabled); +void setWindowTitle_W32(const cv::String& name, const cv::String& title); +void setWindowTitle_GTK(const cv::String& name, const cv::String& title); +void setWindowTitle_QT(const cv::String& name, const cv::String& title); +void setWindowTitle_COCOA(const cv::String& name, const cv::String& title); + +int pollKey_W32(); + //for QT #if defined (HAVE_QT) CvRect cvGetWindowRect_QT(const char* name); diff --git a/modules/highgui/src/registry.impl.hpp b/modules/highgui/src/registry.impl.hpp index ccf81f9280..66693f1b07 100644 --- a/modules/highgui/src/registry.impl.hpp +++ b/modules/highgui/src/registry.impl.hpp @@ -50,6 +50,14 @@ std::vector& getBuiltinBackendsInfo() #elif defined(ENABLE_PLUGINS) DECLARE_DYNAMIC_BACKEND("QT") #endif +#endif + +#ifdef _WIN32 +#ifdef HAVE_WIN32UI + DECLARE_STATIC_BACKEND("WIN32", createUIBackendWin32UI) +#elif defined(ENABLE_PLUGINS) + DECLARE_DYNAMIC_BACKEND("WIN32") +#endif #endif }; return g_backends; diff --git a/modules/highgui/src/window.cpp b/modules/highgui/src/window.cpp index 56c1456a5d..d1ccd1dbc3 100644 --- a/modules/highgui/src/window.cpp +++ b/modules/highgui/src/window.cpp @@ -586,6 +586,46 @@ void cv::moveWindow( const String& winname, int x, int y ) #endif } +void cv::setWindowTitle(const String& winname, const String& title) +{ + CV_TRACE_FUNCTION(); + + { + cv::AutoLock lock(cv::getWindowMutex()); + auto window = findWindow_(winname); + if (window) + { + return window->setTitle(title); + } + } + +#if defined(OPENCV_HIGHGUI_WITHOUT_BUILTIN_BACKEND) && defined(ENABLE_PLUGINS) + auto backend = getCurrentUIBackend(); + if (backend) + { + CV_LOG_WARNING(NULL, "Can't find window with name: '" << winname << "'. Do nothing"); + CV_NOT_FOUND_DEPRECATION; + } + else + { + CV_LOG_WARNING(NULL, "No UI backends available. Use OPENCV_LOG_LEVEL=DEBUG for investigation"); + } + return; +#elif defined(HAVE_WIN32UI) + return setWindowTitle_W32(winname, title); +#elif defined (HAVE_GTK) + return setWindowTitle_GTK(winname, title); +#elif defined (HAVE_QT) + return setWindowTitle_QT(winname, title); +#elif defined (HAVE_COCOA) + return setWindowTitle_COCOA(winname, title); +#else + CV_Error(Error::StsNotImplemented, "The function is not implemented. " + "Rebuild the library with Windows, GTK+ 2.x or Cocoa support. " + "If you are on Ubuntu or Debian, install libgtk2.0-dev and pkg-config, then re-run cmake or configure script"); +#endif +} + void cv::setWindowProperty(const String& winname, int prop_id, double prop_value) { CV_TRACE_FUNCTION(); @@ -630,9 +670,9 @@ int cv::waitKey(int delay) return (code != -1) ? (code & 0xff) : -1; } -#if defined(HAVE_QT) || (defined (WINRT) && !defined (WINRT_8_0)) || \ - !defined(HAVE_WIN32UI) && (defined(HAVE_GTK) || defined(HAVE_COCOA)) -// pollKey() fallback implementation +/* + * process until queue is empty but don't wait. + */ int cv::pollKey() { CV_TRACE_FUNCTION(); @@ -646,12 +686,13 @@ int cv::pollKey() } } +#if defined(HAVE_WIN32UI) + return pollKey_W32(); +#else // fallback. please implement a proper polling function return cvWaitKey(1); -} -#elif defined(HAVE_WIN32UI) -// pollKey() implemented in window_w32.cpp #endif +} int cv::createTrackbar(const String& trackbarName, const String& winName, int* value, int count, TrackbarCallback callback, @@ -1203,13 +1244,6 @@ int cv::createButton(const String&, ButtonCallback, void*, int , bool ) // version with a more capable one without a need to recompile dependent // applications or libraries. -void cv::setWindowTitle(const String&, const String&) -{ - CV_Error(Error::StsNotImplemented, "The function is not implemented. " - "Rebuild the library with Windows, GTK+ 2.x or Cocoa support. " - "If you are on Ubuntu or Debian, install libgtk2.0-dev and pkg-config, then re-run cmake or configure script"); -} - #define CV_NO_GUI_ERROR(funcname) \ cv::error(cv::Error::StsError, \ "The function is not implemented. " \ @@ -1360,11 +1394,6 @@ CV_IMPL int cvCreateButton(const char*, void (*)(int, void*), void*, int, int) CV_NO_GUI_ERROR("cvCreateButton"); } -int cv::pollKey() -{ - CV_NO_GUI_ERROR("cv::pollKey()"); -} - #endif /* End of file. */ diff --git a/modules/highgui/src/window_QT.cpp b/modules/highgui/src/window_QT.cpp index 60d7d69a59..9899dfdcf0 100644 --- a/modules/highgui/src/window_QT.cpp +++ b/modules/highgui/src/window_QT.cpp @@ -63,6 +63,7 @@ #endif #endif +using namespace cv; //Static and global first static GuiReceiver *guiMainThread = NULL; @@ -197,7 +198,7 @@ void cvSetPropWindow_QT(const char* name,double prop_value) Q_ARG(double, prop_value)); } -void cv::setWindowTitle(const String& winname, const String& title) +void setWindowTitle_QT(const String& winname, const String& title) { if (!guiMainThread) CV_Error(Error::StsNullPtr, "NULL guiReceiver (please create a window)"); diff --git a/modules/highgui/src/window_cocoa.mm b/modules/highgui/src/window_cocoa.mm index 29a0278c98..e8e9034406 100644 --- a/modules/highgui/src/window_cocoa.mm +++ b/modules/highgui/src/window_cocoa.mm @@ -795,18 +795,18 @@ void cvSetPropTopmost_COCOA( const char* name, const bool topmost ) __END__; } -void cv::setWindowTitle(const String& winname, const String& title) +void setWindowTitle_COCOA(const cv::String& winname, const cv::String& title) { CVWindow *window = cvGetWindow(winname.c_str()); if (window == NULL) { - namedWindow(winname); + cv::namedWindow(winname); window = cvGetWindow(winname.c_str()); } if (window == NULL) - CV_Error(Error::StsNullPtr, "NULL window"); + CV_Error(cv::Error::StsNullPtr, "NULL window"); NSAutoreleasePool* localpool = [[NSAutoreleasePool alloc] init]; diff --git a/modules/highgui/src/window_gtk.cpp b/modules/highgui/src/window_gtk.cpp index efa3fbd96f..8eaf98fb36 100644 --- a/modules/highgui/src/window_gtk.cpp +++ b/modules/highgui/src/window_gtk.cpp @@ -364,7 +364,7 @@ static void cvImageWidget_set_size(GtkWidget * widget, int max_width, int max_he } - assert( image_widget->scaled_image ); + CV_Assert(image_widget->scaled_image); } static void @@ -849,7 +849,7 @@ static bool setModeWindow_(const std::shared_ptr& window, int mode) return false; } -void cv::setWindowTitle(const String& winname, const String& title) +void setWindowTitle_GTK(const String& winname, const String& title) { CV_LOCK_MUTEX(); diff --git a/modules/highgui/src/window_w32.cpp b/modules/highgui/src/window_w32.cpp index c4f2ddd2a6..d9a9d73222 100644 --- a/modules/highgui/src/window_w32.cpp +++ b/modules/highgui/src/window_w32.cpp @@ -41,12 +41,17 @@ #include "precomp.hpp" +#ifdef HAVE_WIN32UI + +#include +#include + +#include "backend.hpp" + using namespace cv; #include // required for GET_X_LPARAM() and GET_Y_LPARAM() macros -#if defined _WIN32 - #ifdef __GNUC__ # pragma GCC diagnostic ignored "-Wmissing-declarations" #endif @@ -60,14 +65,12 @@ using namespace cv; #include #include #include -#include #ifdef HAVE_OPENGL #include #include #include #include -#include "opencv2/highgui.hpp" #include #include "opencv2/core/opengl.hpp" #endif @@ -78,7 +81,7 @@ static const char* trackbar_text = #if defined _M_X64 || defined __x86_64 || defined _M_ARM64 #define icvGetWindowLongPtr GetWindowLongPtr -#define icvSetWindowLongPtr( hwnd, id, ptr ) SetWindowLongPtr( hwnd, id, (LONG_PTR)(ptr) ) +#define icvSetWindowLongPtr(hwnd, id, ptr) SetWindowLongPtr(hwnd, id, (LONG_PTR)(ptr)) #define icvGetClassLongPtr GetClassLongPtr #define CV_USERDATA GWLP_USERDATA @@ -89,7 +92,7 @@ static const char* trackbar_text = #else #define icvGetWindowLongPtr GetWindowLong -#define icvSetWindowLongPtr( hwnd, id, ptr ) SetWindowLong( hwnd, id, (size_t)ptr ) +#define icvSetWindowLongPtr(hwnd, id, ptr) SetWindowLong(hwnd, id, (size_t)ptr) #define icvGetClassLongPtr GetClassLong #define CV_USERDATA GWL_USERDATA @@ -116,13 +119,13 @@ static inline void mingw_strcat_s(char *dest, size_t destsz, const char *src){ #define strcat_s mingw_strcat_s #endif -static void FillBitmapInfo( BITMAPINFO* bmi, int width, int height, int bpp, int origin ) +static void FillBitmapInfo(BITMAPINFO* bmi, int width, int height, int bpp, int origin) { - assert( bmi && width >= 0 && height >= 0 && (bpp == 8 || bpp == 24 || bpp == 32)); + CV_Assert(bmi && width >= 0 && height >= 0 && (bpp == 8 || bpp == 24 || bpp == 32)); BITMAPINFOHEADER* bmih = &(bmi->bmiHeader); - memset( bmih, 0, sizeof(*bmih)); + memset(bmih, 0, sizeof(*bmih)); bmih->biSize = sizeof(BITMAPINFOHEADER); bmih->biWidth = width; bmih->biHeight = origin ? abs(height) : -abs(height); @@ -130,11 +133,11 @@ static void FillBitmapInfo( BITMAPINFO* bmi, int width, int height, int bpp, int bmih->biBitCount = (unsigned short)bpp; bmih->biCompression = BI_RGB; - if( bpp == 8 ) + if (bpp == 8) { RGBQUAD* palette = bmi->bmiColors; int i; - for( i = 0; i < 256; i++ ) + for (i = 0; i < 256; i++) { palette[i].rgbBlue = palette[i].rgbGreen = palette[i].rgbRed = (BYTE)i; palette[i].rgbReserved = 0; @@ -144,68 +147,91 @@ static void FillBitmapInfo( BITMAPINFO* bmi, int width, int height, int bpp, int struct CvWindow; -typedef struct CvTrackbar +struct CvTrackbar : public std::enable_shared_from_this { + CvTrackbar(CvWindow& window, const std::string& name_) + : signature(CV_TRACKBAR_MAGIC_VAL) + , name(name_) + , parent(&window) + { + // nothing + } + ~CvTrackbar() + { + signature = -1; + } + int signature; - HWND hwnd; - char* name; - CvTrackbar* next; - CvWindow* parent; - HWND buddy; - int* data; - int pos; - int maxval; - int minval; - void (*notify)(int); - void (*notify2)(int, void*); - void* userdata; - int id; -} -CvTrackbar; + HWND hwnd = 0; + std::string name; + CvWindow* parent; // TODO weak_ptr + HWND buddy = 0; + int* data = nullptr; + int pos = 0; + int maxval = 0; + int minval = 0; + void (*notify)(int) = nullptr; // deprecated + void (*notify2)(int, void*) = nullptr; // deprecated + TrackbarCallback onChangeCallback = nullptr; + void* userdata = nullptr; + int id = -1; +}; -typedef struct CvWindow +struct CvWindow : public std::enable_shared_from_this { + CvWindow(const std::string& name_) + : signature(CV_WINDOW_MAGIC_VAL) + , name(name_) + { + // nothing + } + + ~CvWindow() + { + signature = -1; + } + + void destroy(); + int signature; - HWND hwnd; - char* name; - CvWindow* prev; - CvWindow* next; - HWND frame; + cv::Mutex mutex; + HWND hwnd = 0; + std::string name; + HWND frame = 0; - HDC dc; - HGDIOBJ image; - int last_key; - int flags; - int status;//0 normal, 1 fullscreen (YV) + HDC dc = 0; + HGDIOBJ image = 0; + int last_key = 0; + int flags = 0; + int status = 0;//0 normal, 1 fullscreen (YV) - CvMouseCallback on_mouse; - void* on_mouse_param; + CvMouseCallback on_mouse = nullptr; + void* on_mouse_param = nullptr; struct { - HWND toolbar; - int pos; - int rows; - WNDPROC toolBarProc; - CvTrackbar* first; + HWND toolbar = 0; + int pos = 0; + int rows = 0; + WNDPROC toolBarProc = nullptr; + std::vector< std::shared_ptr > trackbars; } toolbar; - int width; - int height; + int width = -1; + int height = -1; // OpenGL support #ifdef HAVE_OPENGL - bool useGl; - HGLRC hGLRC; + bool useGl = false; + HGLRC hGLRC = 0; - CvOpenGlDrawCallback glDrawCallback; - void* glDrawData; + CvOpenGlDrawCallback glDrawCallback = nullptr; + void* glDrawData = nullptr; #endif -} -CvWindow; +}; #define HG_BUDDY_WIDTH 130 @@ -221,19 +247,50 @@ CvWindow; #define TBM_GETTOOLTIPS (WM_USER + 30) #endif -static LRESULT CALLBACK HighGUIProc( HWND hwnd, UINT uMsg, WPARAM wParam, LPARAM lParam); -static LRESULT CALLBACK WindowProc( HWND hwnd, UINT uMsg, WPARAM wParam, LPARAM lParam); -static LRESULT CALLBACK MainWindowProc( HWND hwnd, UINT uMsg, WPARAM wParam, LPARAM lParam); -static void icvUpdateWindowPos( CvWindow* window ); +static +std::vector< std::shared_ptr >& getWindowsList() +{ + static std::vector< std::shared_ptr > g_windows; + return g_windows; +} -static CvWindow* hg_windows = 0; + +// Mutex must be locked +static +std::shared_ptr icvFindWindowByName(const std::string& name) +{ + auto& g_windows = getWindowsList(); + for (auto it = g_windows.begin(); it != g_windows.end(); ++it) + { + auto window = *it; + if (!window) + continue; + if (window->name == name) + return window; + } + return std::shared_ptr(); +} + +static inline +std::shared_ptr icvFindWindowByName(const char* name) +{ + CV_Assert(name); + return icvFindWindowByName(std::string(name)); +} + + + +static LRESULT CALLBACK HighGUIProc(HWND hwnd, UINT uMsg, WPARAM wParam, LPARAM lParam); +static LRESULT CALLBACK WindowProc(HWND hwnd, UINT uMsg, WPARAM wParam, LPARAM lParam); +static LRESULT CALLBACK MainWindowProc(HWND hwnd, UINT uMsg, WPARAM wParam, LPARAM lParam); +static void icvUpdateWindowPos(CvWindow& window); typedef int (CV_CDECL * CvWin32WindowCallback)(HWND, UINT, WPARAM, LPARAM, int*); static CvWin32WindowCallback hg_on_preprocess = 0, hg_on_postprocess = 0; static HINSTANCE hg_hinstance = 0; -static const char* highGUIclassName = "HighGUI class"; -static const char* mainHighGUIclassName = "Main HighGUI class"; +static const char* const highGUIclassName = "HighGUI class"; +static const char* const mainHighGUIclassName = "Main HighGUI class"; static void icvCleanupHighgui() { @@ -242,15 +299,15 @@ static void icvCleanupHighgui() UnregisterClass(mainHighGUIclassName, hg_hinstance); } -CV_IMPL int cvInitSystem( int, char** ) +CV_IMPL int cvInitSystem(int, char**) { static int wasInitialized = 0; // check initialization status - if( !wasInitialized ) + if (!wasInitialized) { - // Initialize the storage - hg_windows = 0; + (void)getWindowMutex(); // force mutex initialization + (void)getWindowsList(); // Initialize the storage // Register the class WNDCLASS wndc; @@ -262,7 +319,7 @@ CV_IMPL int cvInitSystem( int, char** ) wndc.lpszClassName = highGUIclassName; wndc.lpszMenuName = highGUIclassName; wndc.hIcon = LoadIcon(0, IDI_APPLICATION); - wndc.hCursor = (HCURSOR)LoadCursor(0, (LPSTR)(size_t)IDC_CROSS ); + wndc.hCursor = (HCURSOR)LoadCursor(0, (LPSTR)(size_t)IDC_CROSS); wndc.hbrBackground = (HBRUSH)GetStockObject(DKGRAY_BRUSH); RegisterClass(&wndc); @@ -273,12 +330,12 @@ CV_IMPL int cvInitSystem( int, char** ) wndc.lpfnWndProc = MainWindowProc; RegisterClass(&wndc); - atexit( icvCleanupHighgui ); + atexit(icvCleanupHighgui); wasInitialized = 1; } - setlocale(LC_NUMERIC,"C"); + setlocale(LC_NUMERIC,"C"); // FIXIT must be removed return 0; } @@ -287,50 +344,58 @@ CV_IMPL int cvStartWindowThread(){ return 0; } -static CvWindow* icvFindWindowByName( const char* name ) + +static std::shared_ptr icvWindowByHWND(HWND hwnd) { - CvWindow* window = hg_windows; - - for( ; window != 0 && strcmp( name, window->name) != 0; window = window->next ) - ; - - return window; -} - - -static CvWindow* icvWindowByHWND( HWND hwnd ) -{ - CvWindow* window = (CvWindow*)icvGetWindowLongPtr( hwnd, CV_USERDATA ); - return window != 0 && hg_windows != 0 && + AutoLock lock(getWindowMutex()); + CvWindow* window = (CvWindow*)icvGetWindowLongPtr(hwnd, CV_USERDATA); + window = window != 0 && window->signature == CV_WINDOW_MAGIC_VAL ? window : 0; + if (window) + { + return window->shared_from_this(); + } + else + { + return std::shared_ptr(); + } } -static CvTrackbar* icvTrackbarByHWND( HWND hwnd ) +static std::shared_ptr icvTrackbarByHWND(HWND hwnd) { - CvTrackbar* trackbar = (CvTrackbar*)icvGetWindowLongPtr( hwnd, CV_USERDATA ); - return trackbar != 0 && trackbar->signature == CV_TRACKBAR_MAGIC_VAL && + AutoLock lock(getWindowMutex()); + CvTrackbar* trackbar = (CvTrackbar*)icvGetWindowLongPtr(hwnd, CV_USERDATA); + trackbar = trackbar != 0 && trackbar->signature == CV_TRACKBAR_MAGIC_VAL && trackbar->hwnd == hwnd ? trackbar : 0; + if (trackbar) + { + return trackbar->shared_from_this(); + } + else + { + return std::shared_ptr(); + } } -static const char* icvWindowPosRootKey = "Software\\OpenCV\\HighGUI\\Windows\\"; +static const char* const icvWindowPosRootKey = "Software\\OpenCV\\HighGUI\\Windows\\"; // Window positions saving/loading added by Philip Gruebele. //pgruebele@cox.net // Restores the window position from the registry saved position. static void -icvLoadWindowPos( const char* name, CvRect& rect ) +icvLoadWindowPos(const char* name, CvRect& rect) { HKEY hkey; char szKey[1024]; - strcpy_s( szKey, 1024, icvWindowPosRootKey ); - strcat_s( szKey, 1024, name ); + strcpy_s(szKey, 1024, icvWindowPosRootKey); + strcat_s(szKey, 1024, name); rect.x = rect.y = CW_USEDEFAULT; rect.width = rect.height = 320; - if( RegOpenKeyEx(HKEY_CURRENT_USER,szKey,0,KEY_QUERY_VALUE,&hkey) == ERROR_SUCCESS ) + if (RegOpenKeyEx(HKEY_CURRENT_USER,szKey,0,KEY_QUERY_VALUE,&hkey) == ERROR_SUCCESS) { // Yes we are installed. DWORD dwType = 0; @@ -379,16 +444,16 @@ icvLoadWindowPos( const char* name, CvRect& rect ) //pgruebele@cox.net // philipg. Saves the window position in the registry static void -icvSaveWindowPos( const char* name, CvRect rect ) +icvSaveWindowPos(const char* name, CvRect rect) { static const DWORD MAX_RECORD_COUNT = 100; HKEY hkey; char szKey[1024]; char rootKey[1024]; - strcpy_s( szKey, 1024, icvWindowPosRootKey ); - strcat_s( szKey, 1024, name ); + strcpy_s(szKey, 1024, icvWindowPosRootKey); + strcat_s(szKey, 1024, name); - if( RegOpenKeyEx( HKEY_CURRENT_USER,szKey,0,KEY_READ,&hkey) != ERROR_SUCCESS ) + if (RegOpenKeyEx(HKEY_CURRENT_USER,szKey,0,KEY_READ,&hkey) != ERROR_SUCCESS) { HKEY hroot; DWORD count = 0; @@ -396,40 +461,40 @@ icvSaveWindowPos( const char* name, CvRect rect ) char oldestKey[1024]; char currentKey[1024]; - strcpy_s( rootKey, 1024, icvWindowPosRootKey ); + strcpy_s(rootKey, 1024, icvWindowPosRootKey); rootKey[strlen(rootKey)-1] = '\0'; - if( RegCreateKeyEx(HKEY_CURRENT_USER, rootKey, 0, NULL, REG_OPTION_NON_VOLATILE, KEY_READ+KEY_WRITE, 0, &hroot, NULL) != ERROR_SUCCESS ) - //RegOpenKeyEx( HKEY_CURRENT_USER,rootKey,0,KEY_READ,&hroot) != ERROR_SUCCESS ) + if (RegCreateKeyEx(HKEY_CURRENT_USER, rootKey, 0, NULL, REG_OPTION_NON_VOLATILE, KEY_READ+KEY_WRITE, 0, &hroot, NULL) != ERROR_SUCCESS) + //RegOpenKeyEx(HKEY_CURRENT_USER,rootKey,0,KEY_READ,&hroot) != ERROR_SUCCESS) return; for(;;) { DWORD csize = sizeof(currentKey); FILETIME accesstime = { 0, 0 }; - LONG code = RegEnumKeyEx( hroot, count, currentKey, &csize, NULL, NULL, NULL, &accesstime ); - if( code != ERROR_SUCCESS && code != ERROR_MORE_DATA ) + LONG code = RegEnumKeyEx(hroot, count, currentKey, &csize, NULL, NULL, NULL, &accesstime); + if (code != ERROR_SUCCESS && code != ERROR_MORE_DATA) break; count++; - if( oldestTime.dwHighDateTime > accesstime.dwHighDateTime || + if (oldestTime.dwHighDateTime > accesstime.dwHighDateTime || (oldestTime.dwHighDateTime == accesstime.dwHighDateTime && - oldestTime.dwLowDateTime > accesstime.dwLowDateTime) ) + oldestTime.dwLowDateTime > accesstime.dwLowDateTime)) { oldestTime = accesstime; - strcpy_s( oldestKey, 1024, currentKey ); + strcpy_s(oldestKey, 1024, currentKey); } } - if( count >= MAX_RECORD_COUNT ) - RegDeleteKey( hroot, oldestKey ); - RegCloseKey( hroot ); + if (count >= MAX_RECORD_COUNT) + RegDeleteKey(hroot, oldestKey); + RegCloseKey(hroot); - if( RegCreateKeyEx(HKEY_CURRENT_USER,szKey,0,NULL,REG_OPTION_NON_VOLATILE, KEY_WRITE, 0, &hkey, NULL) != ERROR_SUCCESS ) + if (RegCreateKeyEx(HKEY_CURRENT_USER,szKey,0,NULL,REG_OPTION_NON_VOLATILE, KEY_WRITE, 0, &hkey, NULL) != ERROR_SUCCESS) return; } else { - RegCloseKey( hkey ); - if( RegOpenKeyEx( HKEY_CURRENT_USER,szKey,0,KEY_WRITE,&hkey) != ERROR_SUCCESS ) + RegCloseKey(hkey); + if (RegOpenKeyEx(HKEY_CURRENT_USER,szKey,0,KEY_WRITE,&hkey) != ERROR_SUCCESS) return; } @@ -440,96 +505,101 @@ icvSaveWindowPos( const char* name, CvRect rect ) RegCloseKey(hkey); } +static Rect getImageRect_(CvWindow& window); + CvRect cvGetWindowRect_W32(const char* name) { - RECT rect = { 0 }; - CvRect result = cvRect(-1, -1, -1, -1); + CV_FUNCNAME("cvGetWindowRect_W32"); - CV_FUNCNAME( "cvGetWindowRect_W32" ); - - __BEGIN__; - - CvWindow* window; + AutoLock lock(getWindowMutex()); if (!name) - CV_ERROR( CV_StsNullPtr, "NULL name string" ); - window = icvFindWindowByName( name ); - if (!window) - EXIT; // keep silence here + CV_Error(Error::StsNullPtr, "NULL name string"); - GetClientRect(window->hwnd, &rect); - { + auto window = icvFindWindowByName(name); + if (!window) + CV_Error_(Error::StsNullPtr, ("NULL window: '%s'", name)); + + Rect r = getImageRect_(*window); + + CvRect result = cvRect(r.x, r.y, r.width, r.height); + return result; +} + +static Rect getImageRect_(CvWindow& window) +{ + RECT rect = { 0 }; + GetClientRect(window.hwnd, &rect); POINT pt = {rect.left, rect.top}; - ClientToScreen(window->hwnd, &pt); - result = cvRect(pt.x, pt.y, rect.right - rect.left, rect.bottom - rect.top); - } - __END__; + ClientToScreen(window.hwnd, &pt); + Rect result(pt.x, pt.y, rect.right - rect.left, rect.bottom - rect.top); return result; } double cvGetModeWindow_W32(const char* name)//YV { - double result = -1; + CV_FUNCNAME("cvGetModeWindow_W32"); - CV_FUNCNAME( "cvGetModeWindow_W32" ); - - __BEGIN__; - - CvWindow* window; + AutoLock lock(getWindowMutex()); if (!name) - CV_ERROR( CV_StsNullPtr, "NULL name string" ); + CV_Error(Error::StsNullPtr, "NULL name string"); - window = icvFindWindowByName( name ); + auto window = icvFindWindowByName(name); if (!window) - EXIT; // keep silence here + CV_Error_(Error::StsNullPtr, ("NULL window: '%s'", name)); - result = window->status; - - __END__; - return result; + return window->status; } -void cvSetModeWindow_W32( const char* name, double prop_value)//Yannick Verdie +static bool setModeWindow_(CvWindow& window, int mode); + +void cvSetModeWindow_W32(const char* name, double prop_value)//Yannick Verdie { - CV_FUNCNAME( "cvSetModeWindow_W32" ); + CV_FUNCNAME("cvSetModeWindow_W32"); - __BEGIN__; + AutoLock lock(getWindowMutex()); - CvWindow* window; + if (!name) + CV_Error(Error::StsNullPtr, "NULL name string"); - if(!name) - CV_ERROR( CV_StsNullPtr, "NULL name string" ); + auto window = icvFindWindowByName(name); + if (!window) + CV_Error_(Error::StsNullPtr, ("NULL window: '%s'", name)); - window = icvFindWindowByName( name ); - if( !window ) - CV_ERROR( CV_StsNullPtr, "NULL window" ); + (void)setModeWindow_(*window, (int)prop_value); +} - if(window->flags & CV_WINDOW_AUTOSIZE)//if the flag CV_WINDOW_AUTOSIZE is set - EXIT; +static bool setModeWindow_(CvWindow& window, int mode) +{ + if (window.flags & CV_WINDOW_AUTOSIZE)//if the flag CV_WINDOW_AUTOSIZE is set + return false; + + if (window.status == mode) + return true; { - DWORD dwStyle = (DWORD)GetWindowLongPtr(window->frame, GWL_STYLE); + DWORD dwStyle = (DWORD)GetWindowLongPtr(window.frame, GWL_STYLE); CvRect position; - if (window->status==CV_WINDOW_FULLSCREEN && prop_value==CV_WINDOW_NORMAL) + if (window.status == CV_WINDOW_FULLSCREEN && mode == CV_WINDOW_NORMAL) { - icvLoadWindowPos(window->name,position ); - SetWindowLongPtr(window->frame, GWL_STYLE, dwStyle | WS_CAPTION | WS_THICKFRAME); + icvLoadWindowPos(window.name.c_str(), position); + SetWindowLongPtr(window.frame, GWL_STYLE, dwStyle | WS_CAPTION | WS_THICKFRAME); - SetWindowPos(window->frame, HWND_TOP, position.x, position.y , position.width,position.height, SWP_NOZORDER | SWP_FRAMECHANGED); - window->status=CV_WINDOW_NORMAL; + SetWindowPos(window.frame, HWND_TOP, position.x, position.y , position.width,position.height, SWP_NOZORDER | SWP_FRAMECHANGED); + window.status=CV_WINDOW_NORMAL; - EXIT; + return true; } - if (window->status==CV_WINDOW_NORMAL && prop_value==CV_WINDOW_FULLSCREEN) + if (window.status == CV_WINDOW_NORMAL && mode == CV_WINDOW_FULLSCREEN) { //save dimension RECT rect = { 0 }; - GetWindowRect(window->frame, &rect); - CvRect RectCV = cvRect(rect.left, rect.top,rect.right - rect.left, rect.bottom - rect.top); - icvSaveWindowPos(window->name,RectCV ); + GetWindowRect(window.frame, &rect); + CvRect rectCV = cvRect(rect.left, rect.top,rect.right - rect.left, rect.bottom - rect.top); + icvSaveWindowPos(window.name.c_str(), rectCV); //Look at coordinate for fullscreen HMONITOR hMonitor; @@ -542,60 +612,75 @@ void cvSetModeWindow_W32( const char* name, double prop_value)//Yannick Verdie //fullscreen position.x=mi.rcMonitor.left;position.y=mi.rcMonitor.top; position.width=mi.rcMonitor.right - mi.rcMonitor.left;position.height=mi.rcMonitor.bottom - mi.rcMonitor.top; - SetWindowLongPtr(window->frame, GWL_STYLE, dwStyle & ~WS_CAPTION & ~WS_THICKFRAME); + SetWindowLongPtr(window.frame, GWL_STYLE, dwStyle & ~WS_CAPTION & ~WS_THICKFRAME); - SetWindowPos(window->frame, HWND_TOP, position.x, position.y , position.width,position.height, SWP_NOZORDER | SWP_FRAMECHANGED); - window->status=CV_WINDOW_FULLSCREEN; + SetWindowPos(window.frame, HWND_TOP, position.x, position.y , position.width,position.height, SWP_NOZORDER | SWP_FRAMECHANGED); + window.status=CV_WINDOW_FULLSCREEN; - EXIT; + return true; } } - __END__; + return false; } +static double getPropTopmost_(CvWindow& window); + double cvGetPropTopmost_W32(const char* name) { - double result = -1; - CV_Assert(name); - CvWindow* window = icvFindWindowByName(name); + auto window = icvFindWindowByName(name); if (!window) CV_Error(Error::StsNullPtr, "NULL window"); - LONG style = GetWindowLongA(window->frame, GWL_EXSTYLE); // -20 + return getPropTopmost_(*window); +} + +static double getPropTopmost_(CvWindow& window) +{ + LONG style = GetWindowLongA(window.frame, GWL_EXSTYLE); // -20 if (!style) { std::ostringstream errorMsg; - errorMsg << "window(" << name << "): failed to retrieve extended window style using GetWindowLongA(); error code: " << GetLastError(); - CV_Error(Error::StsError, errorMsg.str().c_str()); + errorMsg << "window(" << window.name << "): failed to retrieve extended window style using GetWindowLongA(); error code: " << GetLastError(); + CV_Error(Error::StsError, errorMsg.str()); } - result = (style & WS_EX_TOPMOST) == WS_EX_TOPMOST; - - return result; + bool result = (style & WS_EX_TOPMOST) == WS_EX_TOPMOST; + return result ? 1.0 : 0.0; } +static bool setPropTopmost_(CvWindow& window, bool topmost); + void cvSetPropTopmost_W32(const char* name, const bool topmost) { CV_Assert(name); - CvWindow* window = icvFindWindowByName(name); + auto window = icvFindWindowByName(name); if (!window) CV_Error(Error::StsNullPtr, "NULL window"); + (void)setPropTopmost_(*window, topmost); +} + +static bool setPropTopmost_(CvWindow& window, bool topmost) +{ HWND flag = topmost ? HWND_TOPMOST : HWND_TOP; - BOOL success = SetWindowPos(window->frame, flag, 0, 0, 0, 0, SWP_NOMOVE | SWP_NOSIZE); + BOOL success = SetWindowPos(window.frame, flag, 0, 0, 0, 0, SWP_NOMOVE | SWP_NOSIZE); if (!success) { std::ostringstream errorMsg; - errorMsg << "window(" << name << "): error reported by SetWindowPos(" << (topmost ? "HWND_TOPMOST" : "HWND_TOP") << "), error code: " << GetLastError(); - CV_Error(Error::StsError, errorMsg.str().c_str()); + errorMsg << "window(" << window.name << "): error reported by SetWindowPos(" << (topmost ? "HWND_TOPMOST" : "HWND_TOP") << "), error code: " << GetLastError(); + CV_Error(Error::StsError, errorMsg.str()); + return false; } + return true; } +static double getPropVsync_(CvWindow& window); + double cvGetPropVsync_W32(const char* name) { #ifndef HAVE_OPENGL @@ -605,40 +690,53 @@ double cvGetPropVsync_W32(const char* name) if (!name) CV_Error(Error::StsNullPtr, "'name' argument must not be NULL"); - CvWindow* window = icvFindWindowByName(name); + auto window = icvFindWindowByName(name); if (!window) CV_Error_(Error::StsBadArg, ("there is no window named '%s'", name)); + double result = getPropVsync_(*window); + return cvIsNaN(result) ? -1.0 : result; +#endif +} + +static double getPropVsync_(CvWindow& window) +{ +#ifndef HAVE_OPENGL + CV_UNUSED(window); + CV_Error(Error::OpenGlNotSupported, "Library was built without OpenGL support"); +#else // https://www.khronos.org/opengl/wiki/Swap_Interval // https://www.khronos.org/registry/OpenGL/extensions/EXT/WGL_EXT_extensions_string.txt // https://www.khronos.org/registry/OpenGL/extensions/EXT/WGL_EXT_swap_control.txt - if (!wglMakeCurrent(window->dc, window->hGLRC)) + if (!wglMakeCurrent(window.dc, window.hGLRC)) CV_Error(Error::OpenGlApiCallError, "Can't Activate The GL Rendering Context"); typedef const char* (APIENTRY* PFNWGLGETEXTENSIONSSTRINGEXTPROC)(void); PFNWGLGETEXTENSIONSSTRINGEXTPROC wglGetExtensionsString = NULL; wglGetExtensionsString = (PFNWGLGETEXTENSIONSSTRINGEXTPROC)wglGetProcAddress("wglGetExtensionsStringEXT"); if (wglGetExtensionsString == NULL) - return -1; // wglGetProcAddress failed to get wglGetExtensionsStringEXT + return std::numeric_limits::quiet_NaN(); // wglGetProcAddress failed to get wglGetExtensionsStringEXT const char* wgl_extensions = wglGetExtensionsString(); if (wgl_extensions == NULL) - return -1; // Can't get WGL extensions string + return std::numeric_limits::quiet_NaN(); // Can't get WGL extensions string if (strstr(wgl_extensions, "WGL_EXT_swap_control") == NULL) - return -1; // WGL extensions don't contain WGL_EXT_swap_control + return std::numeric_limits::quiet_NaN(); // WGL extensions don't contain WGL_EXT_swap_control typedef int (APIENTRY* PFNWGLGETSWAPINTERVALPROC)(void); PFNWGLGETSWAPINTERVALPROC wglGetSwapInterval = 0; wglGetSwapInterval = (PFNWGLGETSWAPINTERVALPROC)wglGetProcAddress("wglGetSwapIntervalEXT"); if (wglGetSwapInterval == NULL) - return -1; // wglGetProcAddress failed to get wglGetSwapIntervalEXT + return std::numeric_limits::quiet_NaN(); // wglGetProcAddress failed to get wglGetSwapIntervalEXT return wglGetSwapInterval(); #endif } +static bool setPropVsync_(CvWindow& window, bool enable_vsync); + void cvSetPropVsync_W32(const char* name, const bool enable_vsync) { #ifndef HAVE_OPENGL @@ -649,11 +747,22 @@ void cvSetPropVsync_W32(const char* name, const bool enable_vsync) if (!name) CV_Error(Error::StsNullPtr, "'name' argument must not be NULL"); - CvWindow* window = icvFindWindowByName(name); + auto window = icvFindWindowByName(name); if (!window) CV_Error_(Error::StsBadArg, ("there is no window named '%s'", name)); - if (!wglMakeCurrent(window->dc, window->hGLRC)) + (void)setPropVsync_(*window, enable_vsync); +#endif +} + +static bool setPropVsync_(CvWindow& window, bool enable_vsync) +{ +#ifndef HAVE_OPENGL + CV_UNUSED(window); + CV_UNUSED(enable_vsync); + CV_Error(Error::OpenGlNotSupported, "Library was built without OpenGL support"); +#else + if (!wglMakeCurrent(window.dc, window.hGLRC)) CV_Error(Error::OpenGlApiCallError, "Can't Activate The GL Rendering Context"); typedef const char* (APIENTRY* PFNWGLGETEXTENSIONSSTRINGEXTPROC)(void); @@ -676,47 +785,44 @@ void cvSetPropVsync_W32(const char* name, const bool enable_vsync) CV_Error(Error::OpenGlApiCallError, "wglGetProcAddress failed to get wglSwapIntervalEXT"); wglSwapInterval(enable_vsync); + return true; #endif } -void cv::setWindowTitle(const String& winname, const String& title) +void setWindowTitle_W32(const std::string& name, const std::string& title) { - CvWindow* window = icvFindWindowByName(winname.c_str()); + auto window = icvFindWindowByName(name); if (!window) { - namedWindow(winname); - window = icvFindWindowByName(winname.c_str()); + namedWindow(name); + window = icvFindWindowByName(name); } if (!window) CV_Error(Error::StsNullPtr, "NULL window"); if (!SetWindowText(window->frame, title.c_str())) - CV_Error_(Error::StsError, ("Failed to set \"%s\" window title to \"%s\"", winname.c_str(), title.c_str())); + CV_Error_(Error::StsError, ("Failed to set \"%s\" window title to \"%s\"", name.c_str(), title.c_str())); } double cvGetPropWindowAutoSize_W32(const char* name) { double result = -1; - CV_FUNCNAME( "cvSetCloseCallback" ); + CV_FUNCNAME("cvSetCloseCallback"); - __BEGIN__; - - CvWindow* window; + AutoLock lock(getWindowMutex()); if (!name) - CV_ERROR( CV_StsNullPtr, "NULL name string" ); + CV_Error(Error::StsNullPtr, "NULL name string"); - window = icvFindWindowByName( name ); + auto window = icvFindWindowByName(name); if (!window) - EXIT; // keep silence here + CV_Error_(Error::StsNullPtr, ("NULL window: '%s'", name)); result = window->flags & CV_WINDOW_AUTOSIZE; - __END__; - return result; } @@ -724,23 +830,19 @@ double cvGetRatioWindow_W32(const char* name) { double result = -1; - CV_FUNCNAME( "cvGetRatioWindow_W32" ); + CV_FUNCNAME("cvGetRatioWindow_W32"); - __BEGIN__; - - CvWindow* window; + AutoLock lock(getWindowMutex()); if (!name) - CV_ERROR( CV_StsNullPtr, "NULL name string" ); + CV_Error(Error::StsNullPtr, "NULL name string"); - window = icvFindWindowByName( name ); + auto window = icvFindWindowByName(name); if (!window) - EXIT; // keep silence here + CV_Error_(Error::StsNullPtr, ("NULL window: '%s'", name)); result = static_cast(window->width) / window->height; - __END__; - return result; } @@ -749,23 +851,20 @@ double cvGetOpenGlProp_W32(const char* name) double result = -1; #ifdef HAVE_OPENGL - CV_FUNCNAME( "cvGetOpenGlProp_W32" ); + CV_FUNCNAME("cvGetOpenGlProp_W32"); - __BEGIN__; - - CvWindow* window; + AutoLock lock(getWindowMutex()); if (!name) - CV_ERROR( CV_StsNullPtr, "NULL name string" ); + CV_Error(Error::StsNullPtr, "NULL name string"); - window = icvFindWindowByName( name ); + auto window = icvFindWindowByName(name); if (!window) - EXIT; // keep silence here + return -1; result = window->useGl; - - __END__; #endif + CV_UNUSED(name); return result; @@ -775,16 +874,15 @@ double cvGetPropVisible_W32(const char* name) { double result = -1; - CV_FUNCNAME( "cvGetPropVisible_W32" ); + CV_FUNCNAME("cvGetPropVisible_W32"); - __BEGIN__; + AutoLock lock(getWindowMutex()); if (!name) - CV_ERROR( CV_StsNullPtr, "NULL name string" ); + CV_Error(Error::StsNullPtr, "NULL name string"); - result = (icvFindWindowByName( name ) != NULL); - - __END__; + auto window = icvFindWindowByName(name); + result = (bool)window ? 1.0 : 0.0; return result; } @@ -798,9 +896,9 @@ namespace { void createGlContext(HWND hWnd, HDC& hGLDC, HGLRC& hGLRC, bool& useGl) { - CV_FUNCNAME( "createGlContext" ); + CV_FUNCNAME("createGlContext"); - __BEGIN__; + AutoLock lock(getWindowMutex()); useGl = false; @@ -830,120 +928,119 @@ namespace hGLDC = GetDC(hWnd); if (!hGLDC) - CV_ERROR( CV_OpenGlApiCallError, "Can't Create A GL Device Context" ); + CV_Error(Error::OpenGlApiCallError, "Can't Create A GL Device Context"); PixelFormat = ChoosePixelFormat(hGLDC, &pfd); if (!PixelFormat) - CV_ERROR( CV_OpenGlApiCallError, "Can't Find A Suitable PixelFormat" ); + CV_Error(Error::OpenGlApiCallError, "Can't Find A Suitable PixelFormat"); if (!SetPixelFormat(hGLDC, PixelFormat, &pfd)) - CV_ERROR( CV_OpenGlApiCallError, "Can't Set The PixelFormat" ); + CV_Error(Error::OpenGlApiCallError, "Can't Set The PixelFormat"); hGLRC = wglCreateContext(hGLDC); if (!hGLRC) - CV_ERROR( CV_OpenGlApiCallError, "Can't Create A GL Rendering Context" ); + CV_Error(Error::OpenGlApiCallError, "Can't Create A GL Rendering Context"); if (!wglMakeCurrent(hGLDC, hGLRC)) - CV_ERROR( CV_OpenGlApiCallError, "Can't Activate The GL Rendering Context" ); + CV_Error(Error::OpenGlApiCallError, "Can't Activate The GL Rendering Context"); useGl = true; - - __END__; } - void releaseGlContext(CvWindow* window) + void releaseGlContext(CvWindow& window) { - //CV_FUNCNAME( "releaseGlContext" ); + //CV_FUNCNAME("releaseGlContext"); - __BEGIN__; + AutoLock lock(getWindowMutex()); - if (window->hGLRC) + if (window.hGLRC) { - wglDeleteContext(window->hGLRC); - window->hGLRC = NULL; + wglDeleteContext(window.hGLRC); + window.hGLRC = NULL; } - if (window->dc) + if (window.dc) { - ReleaseDC(window->hwnd, window->dc); - window->dc = NULL; + ReleaseDC(window.hwnd, window.dc); + window.dc = NULL; } - window->useGl = false; - - __END__; + window.useGl = false; } - void drawGl(CvWindow* window) + void drawGl(CvWindow& window) { - CV_FUNCNAME( "drawGl" ); + CV_FUNCNAME("drawGl"); - __BEGIN__; + AutoLock lock(getWindowMutex()); - if (!wglMakeCurrent(window->dc, window->hGLRC)) - CV_ERROR( CV_OpenGlApiCallError, "Can't Activate The GL Rendering Context" ); + if (!wglMakeCurrent(window.dc, window.hGLRC)) + CV_Error(Error::OpenGlApiCallError, "Can't Activate The GL Rendering Context"); glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT); - if (window->glDrawCallback) - window->glDrawCallback(window->glDrawData); + if (window.glDrawCallback) + window.glDrawCallback(window.glDrawData); - if (!SwapBuffers(window->dc)) - CV_ERROR( CV_OpenGlApiCallError, "Can't swap OpenGL buffers" ); - - __END__; + if (!SwapBuffers(window.dc)) + CV_Error(Error::OpenGlApiCallError, "Can't swap OpenGL buffers"); } - void resizeGl(CvWindow* window) + void resizeGl(CvWindow& window) { - CV_FUNCNAME( "resizeGl" ); + CV_FUNCNAME("resizeGl"); - __BEGIN__; + AutoLock lock(getWindowMutex()); - if (!wglMakeCurrent(window->dc, window->hGLRC)) - CV_ERROR( CV_OpenGlApiCallError, "Can't Activate The GL Rendering Context" ); + if (!wglMakeCurrent(window.dc, window.hGLRC)) + CV_Error(Error::OpenGlApiCallError, "Can't Activate The GL Rendering Context"); - glViewport(0, 0, window->width, window->height); - - __END__; + glViewport(0, 0, window.width, window.height); } } #endif // HAVE_OPENGL +static std::shared_ptr namedWindow_(const std::string& name, int flags); -CV_IMPL int cvNamedWindow( const char* name, int flags ) +CV_IMPL int cvNamedWindow(const char* name, int flags) { - int result = 0; - CV_FUNCNAME( "cvNamedWindow" ); + CV_FUNCNAME("cvNamedWindow"); - __BEGIN__; + AutoLock lock(getWindowMutex()); + + if (!name) + CV_Error(Error::StsNullPtr, "NULL name string"); + + // Check the name in the storage + auto window = icvFindWindowByName(name); + if (window) + { + return 1; + } + + window = namedWindow_(name, flags); + return (bool)window; +} + +static std::shared_ptr namedWindow_(const std::string& name, int flags) +{ + AutoLock lock(getWindowMutex()); + + cvInitSystem(0,0); HWND hWnd, mainhWnd; - CvWindow* window; DWORD defStyle = WS_VISIBLE | WS_MINIMIZEBOX | WS_MAXIMIZEBOX | WS_SYSMENU; - int len; - CvRect rect; #ifdef HAVE_OPENGL bool useGl; HDC hGLDC; HGLRC hGLRC; #endif - cvInitSystem(0,0); + CvRect rect; + icvLoadWindowPos(name.c_str(), rect); - if( !name ) - CV_ERROR( CV_StsNullPtr, "NULL name string" ); - - // Check the name in the storage - window = icvFindWindowByName( name ); - if (window != 0) - { - result = 1; - EXIT; - } - - if( !(flags & CV_WINDOW_AUTOSIZE))//YV add border in order to resize the window + if (!(flags & CV_WINDOW_AUTOSIZE))//YV add border in order to resize the window defStyle |= WS_SIZEBOX; #ifdef HAVE_OPENGL @@ -951,23 +1048,21 @@ CV_IMPL int cvNamedWindow( const char* name, int flags ) defStyle |= WS_CLIPCHILDREN | WS_CLIPSIBLINGS; #endif - icvLoadWindowPos( name, rect ); - - mainhWnd = CreateWindow( "Main HighGUI class", name, defStyle | WS_OVERLAPPED, - rect.x, rect.y, rect.width, rect.height, 0, 0, hg_hinstance, 0 ); - if( !mainhWnd ) - CV_ERROR( CV_StsError, "Frame window can not be created" ); + mainhWnd = CreateWindow(mainHighGUIclassName, name.c_str(), defStyle | WS_OVERLAPPED, + rect.x, rect.y, rect.width, rect.height, 0, 0, hg_hinstance, 0); + if (!mainhWnd) + CV_Error_(Error::StsError, ("Frame window can not be created: '%s'", name.c_str())); ShowWindow(mainhWnd, SW_SHOW); //YV- remove one border by changing the style - hWnd = CreateWindow("HighGUI class", "", (defStyle & ~WS_SIZEBOX) | WS_CHILD, CW_USEDEFAULT, 0, rect.width, rect.height, mainhWnd, 0, hg_hinstance, 0); - if( !hWnd ) - CV_ERROR( CV_StsError, "Frame window can not be created" ); + hWnd = CreateWindow(highGUIclassName, "", (defStyle & ~WS_SIZEBOX) | WS_CHILD, CW_USEDEFAULT, 0, rect.width, rect.height, mainhWnd, 0, hg_hinstance, 0); + if (!hWnd) + CV_Error(Error::StsError, "Frame window can not be created"); #ifndef HAVE_OPENGL if (flags & CV_WINDOW_OPENGL) - CV_ERROR( CV_OpenGlNotSupported, "Library was built without OpenGL support" ); + CV_Error(Error::OpenGlNotSupported, "Library was built without OpenGL support"); #else useGl = false; hGLDC = 0; @@ -979,14 +1074,10 @@ CV_IMPL int cvNamedWindow( const char* name, int flags ) ShowWindow(hWnd, SW_SHOW); - len = (int)strlen(name); - CV_CALL( window = (CvWindow*)cvAlloc(sizeof(CvWindow) + len + 1)); + auto window = std::make_shared(name); - window->signature = CV_WINDOW_MAGIC_VAL; window->hwnd = hWnd; window->frame = mainhWnd; - window->name = (char*)(window + 1); - memcpy( window->name, name, len + 1 ); window->flags = flags; window->image = 0; @@ -1016,200 +1107,175 @@ CV_IMPL int cvNamedWindow( const char* name, int flags ) window->on_mouse = 0; window->on_mouse_param = 0; - memset( &window->toolbar, 0, sizeof(window->toolbar)); + icvSetWindowLongPtr(hWnd, CV_USERDATA, window.get()); + icvSetWindowLongPtr(mainhWnd, CV_USERDATA, window.get()); - window->next = hg_windows; - window->prev = 0; - if( hg_windows ) - hg_windows->prev = window; - hg_windows = window; - icvSetWindowLongPtr( hWnd, CV_USERDATA, window ); - icvSetWindowLongPtr( mainhWnd, CV_USERDATA, window ); + auto& g_windows = getWindowsList(); + g_windows.push_back(window); // Recalculate window pos - icvUpdateWindowPos( window ); + icvUpdateWindowPos(*window); - result = 1; - __END__; - - return result; + return window; } #ifdef HAVE_OPENGL CV_IMPL void cvSetOpenGlContext(const char* name) { - CV_FUNCNAME( "cvSetOpenGlContext" ); + CV_FUNCNAME("cvSetOpenGlContext"); - __BEGIN__; + AutoLock lock(getWindowMutex()); - CvWindow* window; + if (!name) + CV_Error(Error::StsNullPtr, "NULL name string"); - if(!name) - CV_ERROR( CV_StsNullPtr, "NULL name string" ); - - window = icvFindWindowByName( name ); + auto window = icvFindWindowByName(name); if (!window) - CV_ERROR( CV_StsNullPtr, "NULL window" ); + CV_Error_(Error::StsNullPtr, ("NULL window: '%s'", name)); if (!window->useGl) - CV_ERROR( CV_OpenGlNotSupported, "Window doesn't support OpenGL" ); + CV_Error(Error::OpenGlNotSupported, "Window doesn't support OpenGL"); if (!wglMakeCurrent(window->dc, window->hGLRC)) - CV_ERROR( CV_OpenGlApiCallError, "Can't Activate The GL Rendering Context" ); - - __END__; + CV_Error(Error::OpenGlApiCallError, "Can't Activate The GL Rendering Context"); } CV_IMPL void cvUpdateWindow(const char* name) { - CV_FUNCNAME( "cvUpdateWindow" ); + CV_FUNCNAME("cvUpdateWindow"); - __BEGIN__; - - CvWindow* window; + AutoLock lock(getWindowMutex()); if (!name) - CV_ERROR( CV_StsNullPtr, "NULL name string" ); + CV_Error(Error::StsNullPtr, "NULL name string"); - window = icvFindWindowByName( name ); + auto window = icvFindWindowByName(name); if (!window) - EXIT; + CV_Error_(Error::StsNullPtr, ("NULL window: '%s'", name)); InvalidateRect(window->hwnd, 0, 0); - - __END__; } CV_IMPL void cvSetOpenGlDrawCallback(const char* name, CvOpenGlDrawCallback callback, void* userdata) { - CV_FUNCNAME( "cvCreateOpenGLCallback" ); + CV_FUNCNAME("cvCreateOpenGLCallback"); - __BEGIN__; + AutoLock lock(getWindowMutex()); - CvWindow* window; + if (!name) + CV_Error(Error::StsNullPtr, "NULL name string"); - if(!name) - CV_ERROR( CV_StsNullPtr, "NULL name string" ); - - window = icvFindWindowByName( name ); - if( !window ) - EXIT; + auto window = icvFindWindowByName(name); + if (!window) + CV_Error_(Error::StsNullPtr, ("NULL window: '%s'", name)); if (!window->useGl) - CV_ERROR( CV_OpenGlNotSupported, "Window was created without OpenGL context" ); + CV_Error(Error::OpenGlNotSupported, "Window was created without OpenGL context"); window->glDrawCallback = callback; window->glDrawData = userdata; - - __END__; } #endif // HAVE_OPENGL -static void icvRemoveWindow( CvWindow* window ) +static void icvRemoveWindow(const std::shared_ptr& window_) { - CvTrackbar* trackbar = NULL; + CV_Assert(window_); + AutoLock lock(getWindowMutex()); + CvWindow& window = *window_; + RECT wrect={0,0,0,0}; + auto& g_windows = getWindowsList(); + for (auto it = g_windows.begin(); it != g_windows.end(); ++it) + { + const std::shared_ptr& w = *it; + if (w.get() == &window) + { + g_windows.erase(it); + break; + } + } + #ifdef HAVE_OPENGL - if (window->useGl) + if (window.useGl) releaseGlContext(window); #endif - if( window->frame ) - GetWindowRect( window->frame, &wrect ); - if( window->name ) - icvSaveWindowPos( window->name, cvRect(wrect.left, wrect.top, - wrect.right-wrect.left, wrect.bottom-wrect.top) ); + if (window.frame) + GetWindowRect(window.frame, &wrect); + icvSaveWindowPos(window.name.c_str(), cvRect(wrect.left, wrect.top, wrect.right-wrect.left, wrect.bottom-wrect.top)); - if( window->hwnd ) - icvSetWindowLongPtr( window->hwnd, CV_USERDATA, 0 ); - if( window->frame ) - icvSetWindowLongPtr( window->frame, CV_USERDATA, 0 ); + if (window.hwnd) + icvSetWindowLongPtr(window.hwnd, CV_USERDATA, 0); + if (window.frame) + icvSetWindowLongPtr(window.frame, CV_USERDATA, 0); - if( window->toolbar.toolbar ) - icvSetWindowLongPtr(window->toolbar.toolbar, CV_USERDATA, 0); + if (window.toolbar.toolbar) + icvSetWindowLongPtr(window.toolbar.toolbar, CV_USERDATA, 0); - if( window->prev ) - window->prev->next = window->next; - else - hg_windows = window->next; + if (window.dc && window.image) + DeleteObject(SelectObject(window.dc, window.image)); - if( window->next ) - window->next->prev = window->prev; + if (window.dc) + DeleteDC(window.dc); - window->prev = window->next = 0; - - if( window->dc && window->image ) - DeleteObject(SelectObject(window->dc,window->image)); - - if( window->dc ) - DeleteDC(window->dc); - - for( trackbar = window->toolbar.first; trackbar != 0; ) + for (auto it = window.toolbar.trackbars.begin(); it != window.toolbar.trackbars.end(); ++it) { - CvTrackbar* next = trackbar->next; - if( trackbar->hwnd ) + auto trackbar = (*it).get(); + if (trackbar && trackbar->hwnd) { - icvSetWindowLongPtr( trackbar->hwnd, CV_USERDATA, 0 ); - cvFree( &trackbar ); + icvSetWindowLongPtr(trackbar->hwnd, CV_USERDATA, 0); } - trackbar = next; } - - cvFree( &window ); } -CV_IMPL void cvDestroyWindow( const char* name ) +CV_IMPL void cvDestroyWindow(const char* name) { - CV_FUNCNAME( "cvDestroyWindow" ); + CV_FUNCNAME("cvDestroyWindow"); - __BEGIN__; + AutoLock lock(getWindowMutex()); - CvWindow* window; - HWND mainhWnd; + if (!name) + CV_Error(Error::StsNullPtr, "NULL name string"); - if(!name) - CV_ERROR( CV_StsNullPtr, "NULL name string" ); + auto window = icvFindWindowByName(name); + if (!window) + CV_Error_(Error::StsNullPtr, ("NULL window: '%s'", name)); - window = icvFindWindowByName( name ); - if( !window ) - EXIT; - - mainhWnd = window->frame; - - SendMessage(window->hwnd, WM_CLOSE, 0, 0); - SendMessage( mainhWnd, WM_CLOSE, 0, 0); - // Do NOT call _remove_window -- CvWindow list will be updated automatically ... - - __END__; + window->destroy(); } -static void icvScreenToClient( HWND hwnd, RECT* rect ) +void CvWindow::destroy() +{ + SendMessage(hwnd, WM_CLOSE, 0, 0); + SendMessage(frame, WM_CLOSE, 0, 0); + // Do NOT call _remove_window -- CvWindow list will be updated automatically ... +} + +static void icvScreenToClient(HWND hwnd, RECT* rect) { POINT p; p.x = rect->left; p.y = rect->top; ScreenToClient(hwnd, &p); - OffsetRect( rect, p.x - rect->left, p.y - rect->top ); + OffsetRect(rect, p.x - rect->left, p.y - rect->top); } /* Calculatess the window coordinates relative to the upper left corner of the mainhWnd window */ -static RECT icvCalcWindowRect( CvWindow* window ) +static RECT icvCalcWindowRect(CvWindow& window) { RECT crect = { 0 }, trect = { 0 }, rect = { 0 }; - assert(window); - - GetClientRect(window->frame, &crect); - if (window->toolbar.toolbar) + GetClientRect(window.frame, &crect); + if (window.toolbar.toolbar) { - GetWindowRect(window->toolbar.toolbar, &trect); - icvScreenToClient(window->frame, &trect); + GetWindowRect(window.toolbar.toolbar, &trect); + icvScreenToClient(window.frame, &trect); SubtractRect(&rect, &crect, &trect); } else @@ -1217,138 +1283,153 @@ static RECT icvCalcWindowRect( CvWindow* window ) return rect; } +static inline RECT icvCalcWindowRect(CvWindow* window) { CV_Assert(window); return icvCalcWindowRect(*window); } -// returns TRUE if there is a problem such as ERROR_IO_PENDING. -static bool icvGetBitmapData( CvWindow* window, SIZE* size, int* channels, void** data ) + +// returns FALSE if there is a problem such as ERROR_IO_PENDING. +static bool icvGetBitmapData(CvWindow& window, SIZE& size, int& channels, void*& data) { - BITMAP bmp; GdiFlush(); - HGDIOBJ h = GetCurrentObject( window->dc, OBJ_BITMAP ); - if( size ) - size->cx = size->cy = 0; - if( data ) - *data = 0; + + HGDIOBJ h = GetCurrentObject(window.dc, OBJ_BITMAP); + size.cx = size.cy = 0; + data = 0; if (h == NULL) - return true; + return false; + + BITMAP bmp = {}; if (GetObject(h, sizeof(bmp), &bmp) == 0) - return true; + return false; - if( size ) - { - size->cx = abs(bmp.bmWidth); - size->cy = abs(bmp.bmHeight); - } + size.cx = abs(bmp.bmWidth); + size.cy = abs(bmp.bmHeight); - if( channels ) - *channels = bmp.bmBitsPixel/8; + channels = bmp.bmBitsPixel/8; - if( data ) - *data = bmp.bmBits; + data = bmp.bmBits; - return false; + return true; +} +static bool icvGetBitmapData(CvWindow& window, SIZE& size) +{ + int channels = 0; + void* data = nullptr; + return icvGetBitmapData(window, size, channels, data); } -static void icvUpdateWindowPos( CvWindow* window ) +static void icvUpdateWindowPos(CvWindow& window) { RECT rect = { 0 }; - assert(window); - if( (window->flags & CV_WINDOW_AUTOSIZE) && window->image ) + if ((window.flags & CV_WINDOW_AUTOSIZE) && window.image) { int i; SIZE size = {0,0}; - icvGetBitmapData( window, &size, 0, 0 ); + icvGetBitmapData(window, size); // TODO check return value? // Repeat two times because after the first resizing of the mainhWnd window // toolbar may resize too - for(i = 0; i < (window->toolbar.toolbar ? 2 : 1); i++) + for(i = 0; i < (window.toolbar.toolbar ? 2 : 1); i++) { - RECT rmw = { 0 }, rw = icvCalcWindowRect(window ); - MoveWindow(window->hwnd, rw.left, rw.top, + RECT rmw = { 0 }, rw = icvCalcWindowRect(&window); + MoveWindow(window.hwnd, rw.left, rw.top, rw.right - rw.left, rw.bottom - rw.top, FALSE); - GetClientRect(window->hwnd, &rw); - GetWindowRect(window->frame, &rmw); + GetClientRect(window.hwnd, &rw); + GetWindowRect(window.frame, &rmw); // Resize the mainhWnd window in order to make the bitmap fit into the child window - MoveWindow(window->frame, rmw.left, rmw.top, + MoveWindow(window.frame, rmw.left, rmw.top, size.cx + (rmw.right - rmw.left) - (rw.right - rw.left), - size.cy + (rmw.bottom - rmw.top) - (rw.bottom - rw.top), TRUE ); + size.cy + (rmw.bottom - rmw.top) - (rw.bottom - rw.top), TRUE); } } rect = icvCalcWindowRect(window); - MoveWindow(window->hwnd, rect.left, rect.top, + MoveWindow(window.hwnd, rect.left, rect.top, rect.right - rect.left, - rect.bottom - rect.top, TRUE ); + rect.bottom - rect.top, TRUE); } +static void showImage_(CvWindow& window, const Mat& image); + CV_IMPL void -cvShowImage( const char* name, const CvArr* arr ) +cvShowImage(const char* name, const CvArr* arr) { - CV_FUNCNAME( "cvShowImage" ); + CV_FUNCNAME("cvShowImage"); - __BEGIN__; + if (!name) + CV_Error(Error::StsNullPtr, "NULL name"); + + std::shared_ptr window; + { + AutoLock lock(getWindowMutex()); + + window = icvFindWindowByName(name); + if (!window) + { + cvNamedWindow(name, CV_WINDOW_AUTOSIZE); + window = icvFindWindowByName(name); + } + } + + if (!window || !arr) + return; // keep silence here. + + CvMat stub = {}; + CvMat* image_c = cvGetMat(arr, &stub); + Mat image = cv::cvarrToMat(image_c); +#ifdef HAVE_OPENGL + if (window->useGl) + { + cv::imshow(name, image); + return; + } +#endif + return showImage_(*window, image); +} + +static void showImage_(CvWindow& window, const Mat& image) +{ + AutoLock lock(window.mutex); - CvWindow* window; SIZE size = { 0, 0 }; int channels = 0; void* dst_ptr = 0; const int channels0 = 3; - CvMat stub, *image; bool changed_size = false; // philipg - if( !name ) - CV_ERROR( CV_StsNullPtr, "NULL name" ); - - window = icvFindWindowByName(name); - if(!window) + if (window.image) { - cvNamedWindow(name, CV_WINDOW_AUTOSIZE); - window = icvFindWindowByName(name); - } - - if( !window || !arr ) - EXIT; // keep silence here. - - CV_CALL( image = cvGetMat( arr, &stub )); - -#ifdef HAVE_OPENGL - if (window->useGl) - { - cv::imshow(name, cv::cvarrToMat(image)); - return; - } -#endif - - if (window->image) // if there is something wrong with these system calls, we cannot display image... - if (icvGetBitmapData( window, &size, &channels, &dst_ptr )) + if (!icvGetBitmapData(window, size, channels, dst_ptr)) return; + } - if( size.cx != image->width || size.cy != image->height || channels != channels0 ) + if (size.cx != image.cols || size.cy != image.rows || channels != channels0) { changed_size = true; uchar buffer[sizeof(BITMAPINFO) + 255*sizeof(RGBQUAD)]; BITMAPINFO* binfo = (BITMAPINFO*)buffer; - DeleteObject( SelectObject( window->dc, window->image )); - window->image = 0; + DeleteObject(SelectObject(window.dc, window.image)); + window.image = 0; - size.cx = image->width; - size.cy = image->height; + size.cx = image.cols; + size.cy = image.rows; channels = channels0; - FillBitmapInfo( binfo, size.cx, size.cy, channels*8, 1 ); + FillBitmapInfo(binfo, size.cx, size.cy, channels*8, 1); - window->image = SelectObject( window->dc, CreateDIBSection(window->dc, binfo, - DIB_RGB_COLORS, &dst_ptr, 0, 0)); + window.image = SelectObject(window.dc, + CreateDIBSection(window.dc, binfo, DIB_RGB_COLORS, &dst_ptr, 0, 0) + ); } { cv::Mat dst(size.cy, size.cx, CV_8UC3, dst_ptr, (size.cx * channels + 3) & -4); - convertToShow(cv::cvarrToMat(image), dst, false); + convertToShow(image, dst, false); CV_Assert(dst.data == (uchar*)dst_ptr); cv::flip(dst, dst, 0); } @@ -1356,98 +1437,103 @@ cvShowImage( const char* name, const CvArr* arr ) // only resize window if needed if (changed_size) icvUpdateWindowPos(window); - InvalidateRect(window->hwnd, 0, 0); + InvalidateRect(window.hwnd, 0, 0); // philipg: this is not needed and just slows things down // UpdateWindow(window->hwnd); - - __END__; } -CV_IMPL void cvResizeWindow(const char* name, int width, int height ) +static void resizeWindow_(CvWindow& window, const Size& size); + +CV_IMPL void cvResizeWindow(const char* name, int width, int height) { - CV_FUNCNAME( "cvResizeWindow" ); + CV_FUNCNAME("cvResizeWindow"); - __BEGIN__; + AutoLock lock(getWindowMutex()); - int i; - CvWindow* window; + if (!name) + CV_Error(Error::StsNullPtr, "NULL name"); + + auto window = icvFindWindowByName(name); + if (!window) + CV_Error_(Error::StsNullPtr, ("NULL window: '%s'", name)); + + return resizeWindow_(*window, Size(width, height)); +} + +static void resizeWindow_(CvWindow& window, const Size& size) +{ RECT rmw = { 0 }, rw = { 0 }, rect = { 0 }; - if( !name ) - CV_ERROR( CV_StsNullPtr, "NULL name" ); - - window = icvFindWindowByName(name); - if(!window) - EXIT; - // Repeat two times because after the first resizing of the mainhWnd window // toolbar may resize too - for(i = 0; i < (window->toolbar.toolbar ? 2 : 1); i++) + for (int i = 0; i < (window.toolbar.toolbar ? 2 : 1); i++) { rw = icvCalcWindowRect(window); - MoveWindow(window->hwnd, rw.left, rw.top, + MoveWindow(window.hwnd, rw.left, rw.top, rw.right - rw.left, rw.bottom - rw.top, FALSE); - GetClientRect(window->hwnd, &rw); - GetWindowRect(window->frame, &rmw); + GetClientRect(window.hwnd, &rw); + GetWindowRect(window.frame, &rmw); // Resize the mainhWnd window in order to make the bitmap fit into the child window - MoveWindow(window->frame, rmw.left, rmw.top, - width + (rmw.right - rmw.left) - (rw.right - rw.left), - height + (rmw.bottom - rmw.top) - (rw.bottom - rw.top), TRUE); + MoveWindow(window.frame, rmw.left, rmw.top, + size.width + (rmw.right - rmw.left) - (rw.right - rw.left), + size.height + (rmw.bottom - rmw.top) - (rw.bottom - rw.top), TRUE); } rect = icvCalcWindowRect(window); - MoveWindow(window->hwnd, rect.left, rect.top, + MoveWindow(window.hwnd, rect.left, rect.top, rect.right - rect.left, rect.bottom - rect.top, TRUE); - - __END__; } +static void moveWindow_(CvWindow& window, const Point& pt); -CV_IMPL void cvMoveWindow( const char* name, int x, int y ) +CV_IMPL void cvMoveWindow(const char* name, int x, int y) { - CV_FUNCNAME( "cvMoveWindow" ); + CV_FUNCNAME("cvMoveWindow"); - __BEGIN__; + AutoLock lock(getWindowMutex()); - CvWindow* window; + if (!name) + CV_Error(Error::StsNullPtr, "NULL name"); + + auto window = icvFindWindowByName(name); + if (!window) + CV_Error_(Error::StsNullPtr, ("NULL window: '%s'", name)); + + (void)moveWindow_(*window, Point(x, y)); +} + +static void moveWindow_(CvWindow& window, const Point& pt) +{ RECT rect = { 0 }; - - if( !name ) - CV_ERROR( CV_StsNullPtr, "NULL name" ); - - window = icvFindWindowByName(name); - if(!window) - EXIT; - - GetWindowRect( window->frame, &rect ); - MoveWindow( window->frame, x, y, rect.right - rect.left, rect.bottom - rect.top, TRUE); - - __END__; + GetWindowRect(window.frame, &rect); // TODO check return value + MoveWindow(window.frame, pt.x, pt.y, rect.right - rect.left, rect.bottom - rect.top, TRUE); } static LRESULT CALLBACK -MainWindowProc( HWND hwnd, UINT uMsg, WPARAM wParam, LPARAM lParam ) +MainWindowProc(HWND hwnd, UINT uMsg, WPARAM wParam, LPARAM lParam) { - CvWindow* window = icvWindowByHWND( hwnd ); - if( !window ) + auto window_ = icvWindowByHWND(hwnd); + if (!window_) return DefWindowProc(hwnd, uMsg, wParam, lParam); + CvWindow& window = *window_; + switch(uMsg) { case WM_COPY: - ::SendMessage(window->hwnd, uMsg, wParam, lParam); + ::SendMessage(window.hwnd, uMsg, wParam, lParam); break; case WM_DESTROY: - icvRemoveWindow(window); + icvRemoveWindow(window_); // Do nothing!!! //PostQuitMessage(0); break; case WM_GETMINMAXINFO: - if( !(window->flags & CV_WINDOW_AUTOSIZE) ) + if (!(window.flags & CV_WINDOW_AUTOSIZE)) { MINMAXINFO* minmax = (MINMAXINFO*)lParam; RECT rect = { 0 }; @@ -1456,10 +1542,10 @@ MainWindowProc( HWND hwnd, UINT uMsg, WPARAM wParam, LPARAM lParam ) minmax->ptMinTrackSize.y = 100; minmax->ptMinTrackSize.x = 100; - if( window->toolbar.first ) + if (!window.toolbar.trackbars.empty()) { - GetWindowRect( window->toolbar.first->hwnd, &rect ); - minmax->ptMinTrackSize.y += window->toolbar.rows*(rect.bottom - rect.top); + GetWindowRect(window.toolbar.trackbars[0]->hwnd, &rect); + minmax->ptMinTrackSize.y += window.toolbar.rows*(rect.bottom - rect.top); minmax->ptMinTrackSize.x = MAX(rect.right - rect.left + HG_BUDDY_WIDTH, HG_BUDDY_WIDTH*2); } return retval; @@ -1471,14 +1557,14 @@ MainWindowProc( HWND hwnd, UINT uMsg, WPARAM wParam, LPARAM lParam ) WINDOWPOS* pos = (WINDOWPOS*)lParam; // Update the toolbar pos/size - if(window->toolbar.toolbar) + if (window.toolbar.toolbar) { RECT rect = { 0 }; - GetWindowRect(window->toolbar.toolbar, &rect); - MoveWindow(window->toolbar.toolbar, 0, 0, pos->cx, rect.bottom - rect.top, TRUE); + GetWindowRect(window.toolbar.toolbar, &rect); + MoveWindow(window.toolbar.toolbar, 0, 0, pos->cx, rect.bottom - rect.top, TRUE); } - if(!(window->flags & CV_WINDOW_AUTOSIZE)) + if (!(window.flags & CV_WINDOW_AUTOSIZE)) icvUpdateWindowPos(window); break; @@ -1490,7 +1576,7 @@ MainWindowProc( HWND hwnd, UINT uMsg, WPARAM wParam, LPARAM lParam ) LPWINDOWPOS pos = (LPWINDOWPOS)lParam; RECT rect = { 0 }; - GetWindowRect(window->frame, &rect); + GetWindowRect(window.frame, &rect); HMONITOR hMonitor; hMonitor = MonitorFromRect(&rect, MONITOR_DEFAULTTONEAREST); @@ -1515,13 +1601,13 @@ MainWindowProc( HWND hwnd, UINT uMsg, WPARAM wParam, LPARAM lParam ) } case WM_ACTIVATE: - if(LOWORD(wParam) == WA_ACTIVE || LOWORD(wParam) == WA_CLICKACTIVE) - SetFocus(window->hwnd); + if (LOWORD(wParam) == WA_ACTIVE || LOWORD(wParam) == WA_CLICKACTIVE) + SetFocus(window.hwnd); break; case WM_MOUSEWHEEL: case WM_MOUSEHWHEEL: - if( window->on_mouse ) + if (window.on_mouse) { int flags = (wParam & MK_LBUTTON ? CV_EVENT_FLAG_LBUTTON : 0)| (wParam & MK_RBUTTON ? CV_EVENT_FLAG_RBUTTON : 0)| @@ -1536,32 +1622,32 @@ MainWindowProc( HWND hwnd, UINT uMsg, WPARAM wParam, LPARAM lParam ) flags |= (delta << 16); POINT pt; - pt.x = GET_X_LPARAM( lParam ); - pt.y = GET_Y_LPARAM( lParam ); + pt.x = GET_X_LPARAM(lParam); + pt.y = GET_Y_LPARAM(lParam); ::ScreenToClient(hwnd, &pt); // Convert screen coordinates to client coordinates. RECT rect = { 0 }; - GetClientRect( window->hwnd, &rect ); + GetClientRect(window.hwnd, &rect); SIZE size = {0,0}; #ifdef HAVE_OPENGL - if (window->useGl) + if (window.useGl) { - cv::ogl::Texture2D* texObj = static_cast(window->glDrawData); + cv::ogl::Texture2D* texObj = static_cast(window.glDrawData); size.cx = texObj->cols(); size.cy = texObj->rows(); } else { - icvGetBitmapData(window, &size, 0, 0); + icvGetBitmapData(window, size); } #else - icvGetBitmapData(window, &size, 0, 0); + icvGetBitmapData(window, size); #endif - window->on_mouse( event, pt.x*size.cx/MAX(rect.right - rect.left,1), - pt.y*size.cy/MAX(rect.bottom - rect.top,1), flags, - window->on_mouse_param ); + int x = cvRound((float)pt.x*size.cx/MAX(rect.right - rect.left,1)); + int y = cvRound((float)pt.y*size.cy/MAX(rect.bottom - rect.top,1)); + window.on_mouse(event, x, y, flags, window.on_mouse_param); } break; @@ -1571,17 +1657,17 @@ MainWindowProc( HWND hwnd, UINT uMsg, WPARAM wParam, LPARAM lParam ) HRGN rgn, rgn1, rgn2; int ret; HDC hdc = (HDC)wParam; - GetWindowRect(window->hwnd, &cr); - icvScreenToClient(window->frame, &cr); - if(window->toolbar.toolbar) + GetWindowRect(window.hwnd, &cr); + icvScreenToClient(window.frame, &cr); + if (window.toolbar.toolbar) { - GetWindowRect(window->toolbar.toolbar, &tr); - icvScreenToClient(window->frame, &tr); + GetWindowRect(window.toolbar.toolbar, &tr); + icvScreenToClient(window.frame, &tr); } else tr.left = tr.top = tr.right = tr.bottom = 0; - GetClientRect(window->frame, &wrc); + GetClientRect(window.frame, &wrc); rgn = CreateRectRgn(0, 0, wrc.right, wrc.bottom); rgn1 = CreateRectRgn(cr.left, cr.top, cr.right, cr.bottom); @@ -1591,7 +1677,7 @@ MainWindowProc( HWND hwnd, UINT uMsg, WPARAM wParam, LPARAM lParam ) ret = CombineRgn(rgn, rgn, rgn1, RGN_DIFF); ret = CombineRgn(rgn, rgn, rgn2, RGN_DIFF); - if(ret != NULLREGION && ret != ERROR) + if (ret != NULLREGION && ret != ERROR) FillRgn(hdc, rgn, (HBRUSH)icvGetClassLongPtr(hwnd, CV_HBRBACKGROUND)); DeleteObject(rgn); @@ -1605,20 +1691,24 @@ MainWindowProc( HWND hwnd, UINT uMsg, WPARAM wParam, LPARAM lParam ) } -static LRESULT CALLBACK HighGUIProc( HWND hwnd, UINT uMsg, WPARAM wParam, LPARAM lParam ) +static LRESULT CALLBACK HighGUIProc(HWND hwnd, UINT uMsg, WPARAM wParam, LPARAM lParam) { - CvWindow* window = icvWindowByHWND(hwnd); - if( !window ) + auto window_ = icvWindowByHWND(hwnd); + if (!window_) + { // This window is not mentioned in HighGUI storage // Actually, this should be error except for the case of calls to CreateWindow return DefWindowProc(hwnd, uMsg, wParam, lParam); + } + + CvWindow& window = *window_; // Process the message switch(uMsg) { case WM_COPY: { - if (!::OpenClipboard(hwnd) ) + if (!::OpenClipboard(hwnd)) break; HDC hDC = 0; @@ -1632,7 +1722,7 @@ static LRESULT CALLBACK HighGUIProc( HWND hwnd, UINT uMsg, WPARAM wParam, LPARAM if (!::EmptyClipboard()) break; - if(!window->image) + if (!window.image) break; // Get window device context @@ -1640,19 +1730,20 @@ static LRESULT CALLBACK HighGUIProc( HWND hwnd, UINT uMsg, WPARAM wParam, LPARAM break; // Create another DC compatible with hDC - if (0 == (memDC = ::CreateCompatibleDC( hDC ))) + if (0 == (memDC = ::CreateCompatibleDC(hDC))) break; // Determine the bitmap's dimensions - int nchannels = 3; SIZE size = {0,0}; - icvGetBitmapData( window, &size, &nchannels, 0 ); + int nchannels = 3; + void* data = NULL; // unused + icvGetBitmapData(window, size, nchannels, data); // Create bitmap to draw on and it in the new DC - if (0 == (memBM = ::CreateCompatibleBitmap ( hDC, size.cx, size.cy))) + if (0 == (memBM = ::CreateCompatibleBitmap(hDC, size.cx, size.cy))) break; - if (!::SelectObject( memDC, memBM )) + if (!::SelectObject(memDC, memBM)) break; // Begin drawing to DC @@ -1660,7 +1751,7 @@ static LRESULT CALLBACK HighGUIProc( HWND hwnd, UINT uMsg, WPARAM wParam, LPARAM break; RGBQUAD table[256]; - if( 1 == nchannels ) + if (1 == nchannels) { for(int i = 0; i < 256; ++i) { @@ -1668,14 +1759,14 @@ static LRESULT CALLBACK HighGUIProc( HWND hwnd, UINT uMsg, WPARAM wParam, LPARAM table[i].rgbGreen = (unsigned char)i; table[i].rgbRed = (unsigned char)i; } - if (!::SetDIBColorTable(window->dc, 0, 255, table)) + if (!::SetDIBColorTable(window.dc, 0, 255, table)) break; } // The image copied to the clipboard will be in its original size, regardless if the window itself was resized. // Render the image to the dc/bitmap (at original size). - if (!::BitBlt( memDC, 0, 0, size.cx, size.cy, window->dc, 0, 0, SRCCOPY )) + if (!::BitBlt(memDC, 0, 0, size.cx, size.cy, window.dc, 0, 0, SRCCOPY)) break; // Finally, set bitmap to clipboard @@ -1712,7 +1803,7 @@ static LRESULT CALLBACK HighGUIProc( HWND hwnd, UINT uMsg, WPARAM wParam, LPARAM case WM_RBUTTONUP: case WM_MBUTTONUP: case WM_MOUSEMOVE: - if( window->on_mouse ) + if (window.on_mouse) { POINT pt; @@ -1732,50 +1823,50 @@ static LRESULT CALLBACK HighGUIProc( HWND hwnd, UINT uMsg, WPARAM wParam, LPARAM uMsg == WM_RBUTTONDBLCLK ? CV_EVENT_RBUTTONDBLCLK : uMsg == WM_MBUTTONDBLCLK ? CV_EVENT_MBUTTONDBLCLK : CV_EVENT_MOUSEMOVE; - if( uMsg == WM_LBUTTONDOWN || uMsg == WM_RBUTTONDOWN || uMsg == WM_MBUTTONDOWN ) - SetCapture( hwnd ); - if( uMsg == WM_LBUTTONUP || uMsg == WM_RBUTTONUP || uMsg == WM_MBUTTONUP ) + if (uMsg == WM_LBUTTONDOWN || uMsg == WM_RBUTTONDOWN || uMsg == WM_MBUTTONDOWN) + SetCapture(hwnd); + if (uMsg == WM_LBUTTONUP || uMsg == WM_RBUTTONUP || uMsg == WM_MBUTTONUP) ReleaseCapture(); - pt.x = GET_X_LPARAM( lParam ); - pt.y = GET_Y_LPARAM( lParam ); + pt.x = GET_X_LPARAM(lParam); + pt.y = GET_Y_LPARAM(lParam); - if (window->flags & CV_WINDOW_AUTOSIZE) + if (window.flags & CV_WINDOW_AUTOSIZE) { // As user can't change window size, do not scale window coordinates. Underlying windowing system // may prevent full window from being displayed and in this case coordinates should not be scaled. - window->on_mouse( event, pt.x, pt.y, flags, window->on_mouse_param ); + window.on_mouse(event, pt.x, pt.y, flags, window.on_mouse_param); } else { // Full window is displayed using different size. Scale coordinates to match underlying positions. RECT rect = { 0 }; SIZE size = {0, 0}; - GetClientRect( window->hwnd, &rect ); + GetClientRect(window.hwnd, &rect); #ifdef HAVE_OPENGL - if (window->useGl) + if (window.useGl) { - cv::ogl::Texture2D* texObj = static_cast(window->glDrawData); + cv::ogl::Texture2D* texObj = static_cast(window.glDrawData); size.cx = texObj->cols(); size.cy = texObj->rows(); } else { - icvGetBitmapData(window, &size, 0, 0); + icvGetBitmapData(window, size); } #else - icvGetBitmapData( window, &size, 0, 0 ); + icvGetBitmapData(window, size); #endif - window->on_mouse( event, pt.x*size.cx/MAX(rect.right - rect.left,1), - pt.y*size.cy/MAX(rect.bottom - rect.top,1), flags, - window->on_mouse_param ); + int x = cvRound((float)pt.x*size.cx/MAX(rect.right - rect.left,1)); + int y = cvRound((float)pt.y*size.cy/MAX(rect.bottom - rect.top,1)); + window.on_mouse(event, x, y, flags, window.on_mouse_param); } } break; case WM_PAINT: - if(window->image != 0) + if (window.image != 0) { int nchannels = 3; SIZE size = {0,0}; @@ -1784,12 +1875,13 @@ static LRESULT CALLBACK HighGUIProc( HWND hwnd, UINT uMsg, WPARAM wParam, LPARAM RGBQUAD table[256]; // Determine the bitmap's dimensions - icvGetBitmapData( window, &size, &nchannels, 0 ); + void* data = 0; // unused + icvGetBitmapData(window, size, nchannels, data); hdc = BeginPaint(hwnd, &paint); SetStretchBltMode(hdc, COLORONCOLOR); - if( nchannels == 1 ) + if (nchannels == 1) { int i; for(i = 0; i < 256; i++) @@ -1798,25 +1890,25 @@ static LRESULT CALLBACK HighGUIProc( HWND hwnd, UINT uMsg, WPARAM wParam, LPARAM table[i].rgbGreen = (unsigned char)i; table[i].rgbRed = (unsigned char)i; } - SetDIBColorTable(window->dc, 0, 255, table); + SetDIBColorTable(window.dc, 0, 255, table); } - if(window->flags & CV_WINDOW_AUTOSIZE) + if (window.flags & CV_WINDOW_AUTOSIZE) { - BitBlt( hdc, 0, 0, size.cx, size.cy, window->dc, 0, 0, SRCCOPY ); + BitBlt(hdc, 0, 0, size.cx, size.cy, window.dc, 0, 0, SRCCOPY); } else { RECT rect = { 0 }; - GetClientRect(window->hwnd, &rect); - StretchBlt( hdc, 0, 0, rect.right - rect.left, rect.bottom - rect.top, - window->dc, 0, 0, size.cx, size.cy, SRCCOPY ); + GetClientRect(window.hwnd, &rect); + StretchBlt(hdc, 0, 0, rect.right - rect.left, rect.bottom - rect.top, + window.dc, 0, 0, size.cx, size.cy, SRCCOPY); } //DeleteDC(hdc); EndPaint(hwnd, &paint); } #ifdef HAVE_OPENGL - else if(window->useGl) + else if (window.useGl) { drawGl(window); return DefWindowProc(hwnd, uMsg, wParam, lParam); @@ -1829,13 +1921,13 @@ static LRESULT CALLBACK HighGUIProc( HWND hwnd, UINT uMsg, WPARAM wParam, LPARAM return 0; case WM_ERASEBKGND: - if(window->image) + if (window.image) return 0; break; case WM_DESTROY: - icvRemoveWindow(window); + icvRemoveWindow(window_); // Do nothing!!! //PostQuitMessage(0); break; @@ -1845,15 +1937,15 @@ static LRESULT CALLBACK HighGUIProc( HWND hwnd, UINT uMsg, WPARAM wParam, LPARAM return 0; case WM_KEYDOWN: - window->last_key = (int)wParam; + window.last_key = (int)wParam; return 0; case WM_SIZE: - window->width = LOWORD(lParam); - window->height = HIWORD(lParam); + window.width = LOWORD(lParam); + window.height = HIWORD(lParam); #ifdef HAVE_OPENGL - if (window->useGl) + if (window.useGl) resizeGl(window); #endif } @@ -1862,24 +1954,24 @@ static LRESULT CALLBACK HighGUIProc( HWND hwnd, UINT uMsg, WPARAM wParam, LPARAM } -static LRESULT CALLBACK WindowProc( HWND hwnd, UINT uMsg, WPARAM wParam, LPARAM lParam ) +static LRESULT CALLBACK WindowProc(HWND hwnd, UINT uMsg, WPARAM wParam, LPARAM lParam) { LRESULT ret; - if( hg_on_preprocess ) + if (hg_on_preprocess) { int was_processed = 0; int rethg = hg_on_preprocess(hwnd, uMsg, wParam, lParam, &was_processed); - if( was_processed ) + if (was_processed) return rethg; } ret = HighGUIProc(hwnd, uMsg, wParam, lParam); - if(hg_on_postprocess) + if (hg_on_postprocess) { int was_processed = 0; int rethg = hg_on_postprocess(hwnd, uMsg, wParam, lParam, &was_processed); - if( was_processed ) + if (was_processed) return rethg; } @@ -1887,51 +1979,56 @@ static LRESULT CALLBACK WindowProc( HWND hwnd, UINT uMsg, WPARAM wParam, LPARAM } -static void icvUpdateTrackbar( CvTrackbar* trackbar, int pos ) +static void icvUpdateTrackbar(CvTrackbar& trackbar, int pos) { const int max_name_len = 10; const char* suffix = ""; char pos_text[32]; int name_len; - if( trackbar->data ) - *trackbar->data = pos; + if (trackbar.data) + *trackbar.data = pos; - if( trackbar->pos != pos ) + if (trackbar.pos != pos) { - trackbar->pos = pos; - if( trackbar->notify2 ) - trackbar->notify2(pos, trackbar->userdata); - if( trackbar->notify ) - trackbar->notify(pos); + trackbar.pos = pos; + if (trackbar.onChangeCallback) + trackbar.onChangeCallback(pos, trackbar.userdata); + if (trackbar.notify2) + trackbar.notify2(pos, trackbar.userdata); + if (trackbar.notify) + trackbar.notify(pos); - name_len = (int)strlen(trackbar->name); + name_len = (int)trackbar.name.size(); - if( name_len > max_name_len ) + // TODO replace C strings manipulation + if (name_len > max_name_len) { int start_len = max_name_len*2/3; int end_len = max_name_len - start_len - 2; - memcpy( pos_text, trackbar->name, start_len ); - memcpy( pos_text + start_len, "...", 3 ); - memcpy( pos_text + start_len + 3, trackbar->name + name_len - end_len, end_len + 1 ); + memcpy(pos_text, trackbar.name.c_str(), start_len); + memcpy(pos_text + start_len, "...", 3); + memcpy(pos_text + start_len + 3, trackbar.name.c_str() + name_len - end_len, end_len + 1); } else { - memcpy( pos_text, trackbar->name, name_len + 1); + memcpy(pos_text, trackbar.name.c_str(), name_len + 1); } - sprintf( pos_text + strlen(pos_text), "%s: %d\n", suffix, pos ); - SetWindowText( trackbar->buddy, pos_text ); + sprintf(pos_text + strlen(pos_text), "%s: %d\n", suffix, pos); + SetWindowText(trackbar.buddy, pos_text); } } -static LRESULT CALLBACK HGToolbarProc( HWND hwnd, UINT uMsg, WPARAM wParam, LPARAM lParam ) +static LRESULT CALLBACK HGToolbarProc(HWND hwnd, UINT uMsg, WPARAM wParam, LPARAM lParam) { - CvWindow* window = icvWindowByHWND( hwnd ); - if(!window) + auto window_ = icvWindowByHWND(hwnd); + if (!window_) return DefWindowProc(hwnd, uMsg, wParam, lParam); + CvWindow& window = *window_; + // Control messages processing switch(uMsg) { @@ -1940,32 +2037,34 @@ static LRESULT CALLBACK HGToolbarProc( HWND hwnd, UINT uMsg, WPARAM wParam, LPAR { HWND slider = (HWND)lParam; int pos = (int)SendMessage(slider, TBM_GETPOS, 0, 0); - CvTrackbar* trackbar = icvTrackbarByHWND( slider ); + auto trackbar = icvTrackbarByHWND(slider); - if( trackbar ) + if (trackbar) { - if( trackbar->pos != pos ) - icvUpdateTrackbar( trackbar, pos ); + if (trackbar->pos != pos) + icvUpdateTrackbar(*trackbar, pos); } - SetFocus( window->hwnd ); + SetFocus(window.hwnd); return 0; } case WM_NCCALCSIZE: { - LRESULT ret = CallWindowProc(window->toolbar.toolBarProc, hwnd, uMsg, wParam, lParam); + LRESULT ret = CallWindowProc(window.toolbar.toolBarProc, hwnd, uMsg, wParam, lParam); int rows = (int)SendMessage(hwnd, TB_GETROWS, 0, 0); - if(window->toolbar.rows != rows) + if (window.toolbar.rows != rows) { - SendMessage(window->toolbar.toolbar, TB_BUTTONCOUNT, 0, 0); - CvTrackbar* trackbar = window->toolbar.first; + SendMessage(window.toolbar.toolbar, TB_BUTTONCOUNT, 0, 0); + auto& trakbars = window.toolbar.trackbars; - for( ; trackbar != 0; trackbar = trackbar->next ) + for (auto it = trakbars.begin(); it != trakbars.end(); ++it) { + auto trackbar = *it; + CV_Assert(trackbar); RECT rect = { 0 }; - SendMessage(window->toolbar.toolbar, TB_GETITEMRECT, + SendMessage(window.toolbar.toolbar, TB_GETITEMRECT, (WPARAM)trackbar->id, (LPARAM)&rect); MoveWindow(trackbar->hwnd, rect.left + HG_BUDDY_WIDTH, rect.top, rect.right - rect.left - HG_BUDDY_WIDTH, @@ -1973,46 +2072,63 @@ static LRESULT CALLBACK HGToolbarProc( HWND hwnd, UINT uMsg, WPARAM wParam, LPAR MoveWindow(trackbar->buddy, rect.left, rect.top, HG_BUDDY_WIDTH, rect.bottom - rect.top, FALSE); } - window->toolbar.rows = rows; + window.toolbar.rows = rows; } return ret; } } - return CallWindowProc(window->toolbar.toolBarProc, hwnd, uMsg, wParam, lParam); + return CallWindowProc(window.toolbar.toolBarProc, hwnd, uMsg, wParam, lParam); } CV_IMPL void cvDestroyAllWindows(void) { - CvWindow* window = hg_windows; - - while( window ) + std::vector< std::shared_ptr > g_windows; { - HWND mainhWnd = window->frame; - HWND hwnd = window->hwnd; - window = window->next; + AutoLock lock(getWindowMutex()); + g_windows = getWindowsList(); // copy + } + for (auto it = g_windows.begin(); it != g_windows.end(); ++it) + { + auto window_ = *it; + if (!window_) + continue; - SendMessage( hwnd, WM_CLOSE, 0, 0 ); - SendMessage( mainhWnd, WM_CLOSE, 0, 0 ); + { + CvWindow& window = *window_; + + HWND mainhWnd = window.frame; + HWND hwnd = window.hwnd; + + SendMessage(hwnd, WM_CLOSE, 0, 0); + SendMessage(mainhWnd, WM_CLOSE, 0, 0); + } + + window_.reset(); + } + // TODO needed? + { + AutoLock lock(getWindowMutex()); + getWindowsList().clear(); } } -static void showSaveDialog(CvWindow* window) +static void showSaveDialog(CvWindow& window) { - if (!window || !window->image) + if (!window.image) return; SIZE sz; int channels; void* data; - if (icvGetBitmapData(window, &sz, &channels, &data)) + if (icvGetBitmapData(window, sz, channels, data)) return; // nothing to save char szFileName[MAX_PATH] = ""; // try to use window title as file name - GetWindowText(window->frame, szFileName, MAX_PATH); + GetWindowText(window.frame, szFileName, MAX_PATH); OPENFILENAME ofn; ZeroMemory(&ofn, sizeof(ofn)); @@ -2022,7 +2138,7 @@ static void showSaveDialog(CvWindow* window) #else ofn.lStructSize = sizeof(ofn); #endif - ofn.hwndOwner = window->hwnd; + ofn.hwndOwner = window.hwnd; ofn.lpstrFilter = #ifdef HAVE_PNG "Portable Network Graphics files (*.png)\0*.png\0" @@ -2075,9 +2191,15 @@ static bool handleMessage(MSG& message, int& keyCode) // otherwise the message was handled specifically bool is_processed = false; - for (CvWindow* window = hg_windows; window != 0 && is_processed == 0; window = window->next) + AutoLock lock(getWindowMutex()); + auto& g_windows = getWindowsList(); + for (auto it = g_windows.begin(); it != g_windows.end() && !is_processed; ++it) { - if (!(window->hwnd == message.hwnd || window->frame == message.hwnd)) + auto window_ = *it; + if (!window_) + continue; + CvWindow& window = *window_; + if (!(window.hwnd == message.hwnd || window.frame == message.hwnd)) continue; is_processed = true; @@ -2140,7 +2262,7 @@ static bool handleMessage(MSG& message, int& keyCode) /* * process until queue is empty but don't wait. */ -int cv::pollKey() +int pollKey_W32() { CV_TRACE_FUNCTION(); for(;;) @@ -2156,7 +2278,7 @@ int cv::pollKey() } CV_IMPL int -cvWaitKey( int delay ) +cvWaitKey(int delay) { int64 time0 = cv::getTickCount(); int64 timeEnd = time0 + (int64)(delay * 0.001f * cv::getTickFrequency()); @@ -2165,9 +2287,9 @@ cvWaitKey( int delay ) { MSG message; - if( (delay <= 0) && hg_windows) + if ((delay <= 0) && !getWindowsList().empty()) GetMessage(&message, 0, 0, 0); - else if( PeekMessage(&message, 0, 0, 0, PM_REMOVE) == FALSE ) + else if (PeekMessage(&message, 0, 0, 0, PM_REMOVE) == FALSE) { int64 t = cv::getTickCount(); if (t - timeEnd >= 0) @@ -2183,110 +2305,135 @@ cvWaitKey( int delay ) } -static CvTrackbar* -icvFindTrackbarByName( const CvWindow* window, const char* name ) +static +std::shared_ptr icvFindTrackbarByName(CvWindow& window, const std::string& name) { - CvTrackbar* trackbar = window->toolbar.first; - - for( ; trackbar != 0 && strcmp( trackbar->name, name ) != 0; trackbar = trackbar->next ) - ; - - return trackbar; + auto trackbars = window.toolbar.trackbars; + for (auto it = trackbars.begin(); it != trackbars.end(); ++it) + { + auto& trackbar = *it; + CV_Assert(trackbar); + if (trackbar->name == name) + return trackbar; + } + return std::shared_ptr(); +} +static inline +std::shared_ptr icvFindTrackbarByName(const std::shared_ptr& window, const std::string& name) +{ + CV_Assert(window); + return icvFindTrackbarByName(window, name); } +static +std::shared_ptr createTrackbar_(CvWindow& window, const std::string& trackbar_name, + int count, + TrackbarCallback onChange, void* userdata); static int -icvCreateTrackbar( const char* trackbar_name, const char* window_name, - int* val, int count, CvTrackbarCallback on_notify, - CvTrackbarCallback2 on_notify2, void* userdata ) +icvCreateTrackbar(const char* trackbar_name, const char* window_name, + int* val, int count, CvTrackbarCallback on_notify, + CvTrackbarCallback2 on_notify2, void* userdata) { - int result = 0; + CV_FUNCNAME("icvCreateTrackbar"); - CV_FUNCNAME( "icvCreateTrackbar" ); + AutoLock lock(getWindowMutex()); - __BEGIN__; + if (!window_name || !trackbar_name) + CV_Error(Error::StsNullPtr, "NULL window or trackbar name"); - char slider_name[32]; - CvWindow* window = 0; - CvTrackbar* trackbar = 0; - int pos = 0; + if (count < 0) + CV_Error(Error::StsOutOfRange, "Bad trackbar maximal value"); - if( !window_name || !trackbar_name ) - CV_ERROR( CV_StsNullPtr, "NULL window or trackbar name" ); + auto window = icvFindWindowByName(window_name); + if (!window) + CV_Error_(Error::StsNullPtr, ("NULL window: '%s'", window_name)); - if( count < 0 ) - CV_ERROR( CV_StsOutOfRange, "Bad trackbar maximal value" ); + auto trackbar = icvFindTrackbarByName(*window, trackbar_name); + if (!trackbar) + trackbar = createTrackbar_(*window, trackbar_name, count, nullptr, userdata); + CV_Assert(trackbar); - window = icvFindWindowByName(window_name); - if( !window ) - EXIT; + trackbar->notify = on_notify; + trackbar->notify2 = on_notify2; + trackbar->userdata = userdata; + trackbar->data = val; - trackbar = icvFindTrackbarByName(window,trackbar_name); - if( !trackbar ) + return 1; +} + +static void createToolbar_(CvWindow& window) +{ + CV_Assert(!window.toolbar.toolbar); + + const int default_height = 30; + + // CreateToolbarEx is deprecated and forces linking against Comctl32.lib. + window.toolbar.toolbar = CreateWindowEx(0, TOOLBARCLASSNAME, NULL, + WS_CHILD | CCS_TOP | TBSTYLE_WRAPABLE | BTNS_AUTOSIZE | BTNS_BUTTON, + 0, 0, 0, 0, + window.frame, NULL, GetModuleHandle(NULL), NULL); + // CreateToolbarEx automatically sends this but CreateWindowEx doesn't. + SendMessage(window.toolbar.toolbar, TB_BUTTONSTRUCTSIZE, (WPARAM)sizeof(TBBUTTON), 0); + + RECT rect; + GetClientRect(window.frame, &rect); + MoveWindow(window.toolbar.toolbar, 0, 0, + rect.right - rect.left, default_height, TRUE); + SendMessage(window.toolbar.toolbar, TB_AUTOSIZE, 0, 0); + ShowWindow(window.toolbar.toolbar, SW_SHOW); + + window.toolbar.pos = 0; + window.toolbar.rows = 0; + window.toolbar.toolBarProc = + (WNDPROC)icvGetWindowLongPtr(window.toolbar.toolbar, CV_WNDPROC); + + icvUpdateWindowPos(window); + + // Subclassing from toolbar + icvSetWindowLongPtr(window.toolbar.toolbar, CV_WNDPROC, HGToolbarProc); + icvSetWindowLongPtr(window.toolbar.toolbar, CV_USERDATA, (void*)&window); + +} + +static +std::shared_ptr createTrackbar_(CvWindow& window, const std::string& trackbar_name, + int count, + TrackbarCallback onChange, void* userdata) +{ + // create toolbar if it is not created yet + if (!window.toolbar.toolbar) { - TBBUTTON tbs = {}; - TBBUTTONINFO tbis = {}; - RECT rect = { 0 }; - int bcount; - int len = (int)strlen( trackbar_name ); + createToolbar_(window); + } - // create toolbar if it is not created yet - if( !window->toolbar.toolbar ) - { - const int default_height = 30; + TBBUTTON tbs = {}; - // CreateToolbarEx is deprecated and forces linking against Comctl32.lib. - window->toolbar.toolbar = CreateWindowEx(0, TOOLBARCLASSNAME, NULL, - WS_CHILD | CCS_TOP | TBSTYLE_WRAPABLE | BTNS_AUTOSIZE | BTNS_BUTTON, - 0, 0, 0, 0, - window->frame, NULL, GetModuleHandle(NULL), NULL); - // CreateToolbarEx automatically sends this but CreateWindowEx doesn't. - SendMessage(window->toolbar.toolbar, TB_BUTTONSTRUCTSIZE, (WPARAM)sizeof(TBBUTTON), 0); + /* Retrieve current buttons count */ + int bcount = (int)SendMessage(window.toolbar.toolbar, TB_BUTTONCOUNT, 0, 0); - GetClientRect(window->frame, &rect); - MoveWindow( window->toolbar.toolbar, 0, 0, - rect.right - rect.left, default_height, TRUE); - SendMessage(window->toolbar.toolbar, TB_AUTOSIZE, 0, 0); - ShowWindow(window->toolbar.toolbar, SW_SHOW); - - window->toolbar.first = 0; - window->toolbar.pos = 0; - window->toolbar.rows = 0; - window->toolbar.toolBarProc = - (WNDPROC)icvGetWindowLongPtr(window->toolbar.toolbar, CV_WNDPROC); - - icvUpdateWindowPos(window); - - // Subclassing from toolbar - icvSetWindowLongPtr(window->toolbar.toolbar, CV_WNDPROC, HGToolbarProc); - icvSetWindowLongPtr(window->toolbar.toolbar, CV_USERDATA, window); - } - - /* Retrieve current buttons count */ - bcount = (int)SendMessage(window->toolbar.toolbar, TB_BUTTONCOUNT, 0, 0); - - if(bcount > 1) - { - /* If this is not the first button then we need to - separate it from the previous one */ - tbs.iBitmap = 0; - tbs.idCommand = bcount; // Set button id to it's number - tbs.iString = 0; - tbs.fsStyle = TBSTYLE_SEP; - tbs.fsState = TBSTATE_ENABLED; - SendMessage(window->toolbar.toolbar, TB_ADDBUTTONS, 1, (LPARAM)&tbs); - - // Retrieve current buttons count - bcount = (int)SendMessage(window->toolbar.toolbar, TB_BUTTONCOUNT, 0, 0); - } - - /* Add a button which we're going to cover with the slider */ + if (bcount > 1) + { + /* If this is not the first button then we need to + separate it from the previous one */ tbs.iBitmap = 0; tbs.idCommand = bcount; // Set button id to it's number - tbs.fsState = TBSTATE_ENABLED; -#if 0/*!defined WIN64 && !defined EM64T*/ - tbs.fsStyle = 0; tbs.iString = 0; + tbs.fsStyle = TBSTYLE_SEP; + tbs.fsState = TBSTATE_ENABLED; + SendMessage(window.toolbar.toolbar, TB_ADDBUTTONS, 1, (LPARAM)&tbs); + + // Retrieve current buttons count + bcount = (int)SendMessage(window.toolbar.toolbar, TB_BUTTONCOUNT, 0, 0); + } + + /* Add a button which we're going to cover with the slider */ + tbs.iBitmap = 0; + tbs.idCommand = bcount; // Set button id to it's number + tbs.fsState = TBSTATE_ENABLED; +#if 0/*!defined WIN64 && !defined EM64T*/ + tbs.fsStyle = 0; + tbs.iString = 0; #else #ifndef TBSTYLE_AUTOSIZE @@ -2296,307 +2443,257 @@ icvCreateTrackbar( const char* trackbar_name, const char* window_name, #ifndef TBSTYLE_GROUP #define TBSTYLE_GROUP 0x0004 #endif - //tbs.fsStyle = TBSTYLE_AUTOSIZE; - tbs.fsStyle = TBSTYLE_GROUP; - tbs.iString = (INT_PTR)trackbar_text; + //tbs.fsStyle = TBSTYLE_AUTOSIZE; + tbs.fsStyle = TBSTYLE_GROUP; + tbs.iString = (INT_PTR)trackbar_text; #endif - SendMessage(window->toolbar.toolbar, TB_ADDBUTTONS, 1, (LPARAM)&tbs); + SendMessage(window.toolbar.toolbar, TB_ADDBUTTONS, 1, (LPARAM)&tbs); - /* Adjust button size to the slider */ - tbis.cbSize = sizeof(tbis); - tbis.dwMask = TBIF_SIZE; + TBBUTTONINFO tbis = {}; - GetClientRect(window->hwnd, &rect); - tbis.cx = (unsigned short)(rect.right - rect.left); + /* Adjust button size to the slider */ + tbis.cbSize = sizeof(tbis); + tbis.dwMask = TBIF_SIZE; - SendMessage(window->toolbar.toolbar, TB_SETBUTTONINFO, - (WPARAM)tbs.idCommand, (LPARAM)&tbis); + RECT rect = { 0 }; + GetClientRect(window.hwnd, &rect); + tbis.cx = (unsigned short)(rect.right - rect.left); - /* Get button pos */ - SendMessage(window->toolbar.toolbar, TB_GETITEMRECT, - (WPARAM)tbs.idCommand, (LPARAM)&rect); + SendMessage(window.toolbar.toolbar, TB_SETBUTTONINFO, + (WPARAM)tbs.idCommand, (LPARAM)&tbis); - /* Create a slider */ - trackbar = (CvTrackbar*)cvAlloc( sizeof(CvTrackbar) + len + 1 ); - trackbar->signature = CV_TRACKBAR_MAGIC_VAL; - trackbar->notify = 0; - trackbar->notify2 = 0; - trackbar->parent = window; - trackbar->pos = 0; - trackbar->data = 0; - trackbar->id = bcount; - trackbar->next = window->toolbar.first; - trackbar->name = (char*)(trackbar + 1); - memcpy( trackbar->name, trackbar_name, len + 1 ); - window->toolbar.first = trackbar; + /* Get button pos */ + SendMessage(window.toolbar.toolbar, TB_GETITEMRECT, + (WPARAM)tbs.idCommand, (LPARAM)&rect); - sprintf(slider_name, "Trackbar%p", val); - trackbar->hwnd = CreateWindowEx(0, TRACKBAR_CLASS, slider_name, - WS_CHILD | WS_VISIBLE | TBS_AUTOTICKS | - TBS_FIXEDLENGTH | TBS_HORZ | TBS_BOTTOM, - rect.left + HG_BUDDY_WIDTH, rect.top, - rect.right - rect.left - HG_BUDDY_WIDTH, - rect.bottom - rect.top, window->toolbar.toolbar, - (HMENU)(size_t)bcount, hg_hinstance, 0); + /* Create a slider */ + auto trackbar = std::make_shared(window, trackbar_name); + trackbar->id = bcount; + window.toolbar.trackbars.push_back(trackbar); - sprintf(slider_name,"Buddy%p", val); - trackbar->buddy = CreateWindowEx(0, "STATIC", slider_name, - WS_CHILD | SS_RIGHT, - rect.left, rect.top, - HG_BUDDY_WIDTH, rect.bottom - rect.top, - window->toolbar.toolbar, 0, hg_hinstance, 0); + auto slider_name = cv::format("Trackbar%p", trackbar.get()); + trackbar->hwnd = CreateWindowEx(0, TRACKBAR_CLASS, slider_name.c_str(), + WS_CHILD | WS_VISIBLE | TBS_AUTOTICKS | + TBS_FIXEDLENGTH | TBS_HORZ | TBS_BOTTOM, + rect.left + HG_BUDDY_WIDTH, rect.top, + rect.right - rect.left - HG_BUDDY_WIDTH, + rect.bottom - rect.top, window.toolbar.toolbar, + (HMENU)(size_t)bcount, hg_hinstance, 0); - icvSetWindowLongPtr( trackbar->hwnd, CV_USERDATA, trackbar ); + slider_name = cv::format("Buddy%p", trackbar.get()); + trackbar->buddy = CreateWindowEx(0, "STATIC", slider_name.c_str(), + WS_CHILD | SS_RIGHT, + rect.left, rect.top, + HG_BUDDY_WIDTH, rect.bottom - rect.top, + window.toolbar.toolbar, 0, hg_hinstance, 0); - /* Minimize the number of rows */ - SendMessage( window->toolbar.toolbar, TB_SETROWS, - MAKEWPARAM(1, FALSE), (LPARAM)&rect ); - } - else - { - trackbar->data = 0; - trackbar->notify = 0; - trackbar->notify2 = 0; - } + icvSetWindowLongPtr(trackbar->hwnd, CV_USERDATA, (void*)trackbar.get()); + + /* Minimize the number of rows */ + SendMessage(window.toolbar.toolbar, TB_SETROWS, + MAKEWPARAM(1, FALSE), (LPARAM)&rect); trackbar->maxval = count; /* Adjust slider parameters */ SendMessage(trackbar->hwnd, TBM_SETRANGEMIN, (WPARAM)TRUE, (LPARAM)0); SendMessage(trackbar->hwnd, TBM_SETRANGEMAX, (WPARAM)TRUE, (LPARAM)count); - SendMessage(trackbar->hwnd, TBM_SETTICFREQ, (WPARAM)1, (LPARAM)0 ); - if( val ) - pos = *val; + SendMessage(trackbar->hwnd, TBM_SETTICFREQ, (WPARAM)1, (LPARAM)0); - SendMessage(trackbar->hwnd, TBM_SETPOS, (WPARAM)TRUE, (LPARAM)pos ); - SendMessage(window->toolbar.toolbar, TB_AUTOSIZE, 0, 0); + int pos = 0; + SendMessage(trackbar->hwnd, TBM_SETPOS, (WPARAM)TRUE, (LPARAM)pos); + SendMessage(window.toolbar.toolbar, TB_AUTOSIZE, 0, 0); trackbar->pos = -1; - icvUpdateTrackbar( trackbar, pos ); - ShowWindow( trackbar->buddy, SW_SHOW ); - ShowWindow( trackbar->hwnd, SW_SHOW ); - - trackbar->notify = on_notify; - trackbar->notify2 = on_notify2; - trackbar->userdata = userdata; - trackbar->data = val; + icvUpdateTrackbar(*trackbar, pos); + ShowWindow(trackbar->buddy, SW_SHOW); + ShowWindow(trackbar->hwnd, SW_SHOW); /* Resize the window to reflect the toolbar resizing*/ icvUpdateWindowPos(window); - result = 1; + trackbar->onChangeCallback = onChange; + trackbar->userdata = userdata; - __END__; - - return result; + return trackbar; } CV_IMPL int -cvCreateTrackbar( const char* trackbar_name, const char* window_name, - int* val, int count, CvTrackbarCallback on_notify ) +cvCreateTrackbar(const char* trackbar_name, const char* window_name, + int* val, int count, CvTrackbarCallback on_notify) { - return icvCreateTrackbar( trackbar_name, window_name, val, count, - on_notify, 0, 0 ); + return icvCreateTrackbar(trackbar_name, window_name, val, count, + on_notify, 0, 0); } CV_IMPL int -cvCreateTrackbar2( const char* trackbar_name, const char* window_name, - int* val, int count, CvTrackbarCallback2 on_notify2, - void* userdata ) +cvCreateTrackbar2(const char* trackbar_name, const char* window_name, + int* val, int count, CvTrackbarCallback2 on_notify2, + void* userdata) { - return icvCreateTrackbar( trackbar_name, window_name, val, count, - 0, on_notify2, userdata ); + return icvCreateTrackbar(trackbar_name, window_name, val, count, + 0, on_notify2, userdata); } CV_IMPL void -cvSetMouseCallback( const char* window_name, CvMouseCallback on_mouse, void* param ) +cvSetMouseCallback(const char* name, CvMouseCallback on_mouse, void* param) { - CV_FUNCNAME( "cvSetMouseCallback" ); + CV_FUNCNAME("cvSetMouseCallback"); - __BEGIN__; + if (!name) + CV_Error(Error::StsNullPtr, "NULL window name"); - CvWindow* window = 0; + AutoLock lock(getWindowMutex()); - if( !window_name ) - CV_ERROR( CV_StsNullPtr, "NULL window name" ); - - window = icvFindWindowByName(window_name); - if( !window ) - EXIT; + auto window = icvFindWindowByName(name); + if (!window) + CV_Error_(Error::StsNullPtr, ("NULL window: '%s'", name)); window->on_mouse = on_mouse; window->on_mouse_param = param; - - __END__; } -CV_IMPL int cvGetTrackbarPos( const char* trackbar_name, const char* window_name ) +CV_IMPL int cvGetTrackbarPos(const char* trackbar_name, const char* window_name) { - int pos = -1; + CV_FUNCNAME("cvGetTrackbarPos"); - CV_FUNCNAME( "cvGetTrackbarPos" ); + AutoLock lock(getWindowMutex()); - __BEGIN__; + if (trackbar_name == 0 || window_name == 0) + CV_Error(Error::StsNullPtr, "NULL trackbar or window name"); - CvWindow* window; - CvTrackbar* trackbar = 0; + auto window = icvFindWindowByName(window_name); + if (!window) + CV_Error_(Error::StsNullPtr, ("NULL window: '%s'", window_name)); - if( trackbar_name == 0 || window_name == 0 ) - CV_ERROR( CV_StsNullPtr, "NULL trackbar or window name" ); + auto trackbar = icvFindTrackbarByName(window, trackbar_name); + if (!trackbar) + CV_Error_(Error::StsNullPtr, ("NULL trackbar: '%s'", trackbar_name)); - window = icvFindWindowByName( window_name ); - if( window ) - trackbar = icvFindTrackbarByName( window, trackbar_name ); - - if( trackbar ) - pos = trackbar->pos; - - __END__; - - return pos; + return trackbar->pos; } -CV_IMPL void cvSetTrackbarPos( const char* trackbar_name, const char* window_name, int pos ) +CV_IMPL void cvSetTrackbarPos(const char* trackbar_name, const char* window_name, int pos) { - CV_FUNCNAME( "cvSetTrackbarPos" ); + CV_FUNCNAME("cvSetTrackbarPos"); - __BEGIN__; + AutoLock lock(getWindowMutex()); - CvWindow* window; - CvTrackbar* trackbar = 0; + if (trackbar_name == 0 || window_name == 0) + CV_Error(Error::StsNullPtr, "NULL trackbar or window name"); - if( trackbar_name == 0 || window_name == 0 ) - CV_ERROR( CV_StsNullPtr, "NULL trackbar or window name" ); + auto window = icvFindWindowByName(window_name); + if (!window) + CV_Error_(Error::StsNullPtr, ("NULL window: '%s'", window_name)); - window = icvFindWindowByName( window_name ); - if( window ) - trackbar = icvFindTrackbarByName( window, trackbar_name ); + auto trackbar = icvFindTrackbarByName(window, trackbar_name); + if (!trackbar) + CV_Error_(Error::StsNullPtr, ("NULL trackbar: '%s'", trackbar_name)); - if( trackbar ) { - if( pos < 0 ) + if (pos < 0) pos = 0; - if( pos > trackbar->maxval ) + if (pos > trackbar->maxval) pos = trackbar->maxval; - SendMessage( trackbar->hwnd, TBM_SETPOS, (WPARAM)TRUE, (LPARAM)pos ); - icvUpdateTrackbar( trackbar, pos ); + SendMessage(trackbar->hwnd, TBM_SETPOS, (WPARAM)TRUE, (LPARAM)pos); + icvUpdateTrackbar(*trackbar, pos); } - - __END__; } CV_IMPL void cvSetTrackbarMax(const char* trackbar_name, const char* window_name, int maxval) { - CV_FUNCNAME( "cvSetTrackbarMax" ); + CV_FUNCNAME("cvSetTrackbarMax"); - __BEGIN__; - - if (maxval >= 0) + if (trackbar_name == 0 || window_name == 0) { - CvWindow* window = 0; - CvTrackbar* trackbar = 0; - if (trackbar_name == 0 || window_name == 0) - { - CV_ERROR(CV_StsNullPtr, "NULL trackbar or window name"); - } - - window = icvFindWindowByName(window_name); - if (window) - { - trackbar = icvFindTrackbarByName(window, trackbar_name); - if (trackbar) - { - // The position will be min(pos, maxval). - trackbar->maxval = (trackbar->minval>maxval)?trackbar->minval:maxval; - SendMessage(trackbar->hwnd, TBM_SETRANGEMAX, (WPARAM)TRUE, (LPARAM)maxval); - } - } + CV_Error(Error::StsNullPtr, "NULL trackbar or window name"); } - __END__; + AutoLock lock(getWindowMutex()); + + auto window = icvFindWindowByName(window_name); + if (!window) + CV_Error_(Error::StsNullPtr, ("NULL window: '%s'", window_name)); + + auto trackbar = icvFindTrackbarByName(window, trackbar_name); + if (!trackbar) + CV_Error_(Error::StsNullPtr, ("NULL trackbar: '%s'", trackbar_name)); + + // FIXIT + if (maxval >= 0) + { + // The position will be min(pos, maxval). + trackbar->maxval = (trackbar->minval>maxval)?trackbar->minval:maxval; + SendMessage(trackbar->hwnd, TBM_SETRANGEMAX, (WPARAM)TRUE, (LPARAM)maxval); + } } CV_IMPL void cvSetTrackbarMin(const char* trackbar_name, const char* window_name, int minval) { - CV_FUNCNAME( "cvSetTrackbarMin" ); + CV_FUNCNAME("cvSetTrackbarMin"); - __BEGIN__; - - if (minval >= 0) + if (trackbar_name == 0 || window_name == 0) { - CvWindow* window = 0; - CvTrackbar* trackbar = 0; - if (trackbar_name == 0 || window_name == 0) - { - CV_ERROR(CV_StsNullPtr, "NULL trackbar or window name"); - } - - window = icvFindWindowByName(window_name); - if (window) - { - trackbar = icvFindTrackbarByName(window, trackbar_name); - if (trackbar) - { - // The position will be min(pos, maxval). - trackbar->minval = (minvalmaxval)?minval:trackbar->maxval; - SendMessage(trackbar->hwnd, TBM_SETRANGEMIN, (WPARAM)TRUE, (LPARAM)minval); - } - } + CV_Error(Error::StsNullPtr, "NULL trackbar or window name"); } - __END__; + AutoLock lock(getWindowMutex()); + + auto window = icvFindWindowByName(window_name); + if (!window) + CV_Error_(Error::StsNullPtr, ("NULL window: '%s'", window_name)); + + auto trackbar = icvFindTrackbarByName(window, trackbar_name); + if (!trackbar) + CV_Error_(Error::StsNullPtr, ("NULL trackbar: '%s'", trackbar_name)); + + // FIXIT + if (minval >= 0) + { + // The position will be min(pos, maxval). + trackbar->minval = (minvalmaxval)?minval:trackbar->maxval; + SendMessage(trackbar->hwnd, TBM_SETRANGEMIN, (WPARAM)TRUE, (LPARAM)minval); + } } -CV_IMPL void* cvGetWindowHandle( const char* window_name ) +CV_IMPL void* cvGetWindowHandle(const char* window_name) { - void* hwnd = 0; + CV_FUNCNAME("cvGetWindowHandle"); - CV_FUNCNAME( "cvGetWindowHandle" ); + AutoLock lock(getWindowMutex()); - __BEGIN__; + if (window_name == 0) + CV_Error(Error::StsNullPtr, "NULL window name"); - CvWindow* window; + auto window = icvFindWindowByName(window_name); + if (!window) + CV_Error_(Error::StsNullPtr, ("NULL window: '%s'", window_name)); - if( window_name == 0 ) - CV_ERROR( CV_StsNullPtr, "NULL window name" ); - - window = icvFindWindowByName( window_name ); - if( window ) - hwnd = (void*)window->hwnd; - - __END__; - - return hwnd; + return (void*)window->hwnd; } - -CV_IMPL const char* cvGetWindowName( void* window_handle ) +// FIXIT: result is not safe to use +CV_IMPL const char* cvGetWindowName(void* window_handle) { - const char* window_name = ""; + CV_FUNCNAME("cvGetWindowName"); - CV_FUNCNAME( "cvGetWindowName" ); + AutoLock lock(getWindowMutex()); - __BEGIN__; + if (window_handle == 0) + CV_Error(Error::StsNullPtr, "NULL window handle"); - CvWindow* window; + auto window = icvWindowByHWND((HWND)window_handle); + if (!window) + CV_Error_(Error::StsNullPtr, ("NULL window: '%p'", window_handle)); - if( window_handle == 0 ) - CV_ERROR( CV_StsNullPtr, "NULL window" ); - - window = icvWindowByHWND( (HWND)window_handle ); - if( window ) - window_name = window->name; - - __END__; - - return window_name; + return window->name.c_str(); } @@ -2612,4 +2709,374 @@ cvSetPostprocessFuncWin32_(const void* callback) hg_on_postprocess = (CvWin32WindowCallback)callback; } -#endif //_WIN32 + + +namespace cv { namespace impl { + +using namespace cv::highgui_backend; + +class Win32UITrackbar; + +class Win32UIWindow + : public UIWindow + , public std::enable_shared_from_this +{ +protected: + const std::string name_; + std::weak_ptr window_; + std::map > trackbars_; +public: + Win32UIWindow(const std::string& name, const std::shared_ptr& window) + : name_(name) + , window_(window) + { + // nothing + } + + ~Win32UIWindow() CV_OVERRIDE + { + if (!window_.expired()) + destroy(); + CV_LOG_DEBUG(NULL, "OpenCV/UI/Win32UI: Win32UIWindow(" << name_ << ") is disposed"); + } + + const std::string& getID() const CV_OVERRIDE { return name_; } + + bool isActive() const CV_OVERRIDE { return !window_.expired(); } + + void destroy() CV_OVERRIDE + { + cv::AutoLock lock(getWindowMutex()); + if (!window_.expired()) + { + auto window = window_.lock(); + if (window) + window->destroy(); + window_.reset(); + } + } + + void imshow(InputArray image) CV_OVERRIDE + { + auto window_ptr = window_.lock(); + CV_Assert(window_ptr); + CvWindow& window = *window_ptr; + Mat image_mat = image.getMat(); + showImage_(window, image_mat); + } + + double getProperty(int prop) const CV_OVERRIDE + { + auto window_ptr = window_.lock(); + CV_Assert(window_ptr); + CvWindow& window = *window_ptr; + // see cvGetWindowProperty + switch ((WindowPropertyFlags)prop) + { + case WND_PROP_FULLSCREEN: + return (double)window.status; + + case WND_PROP_AUTOSIZE: + return (window.flags & WINDOW_AUTOSIZE) ? 1.0 : 0.0; + + case WND_PROP_ASPECT_RATIO: + return static_cast(window.width) / window.height; + +#ifdef HAVE_OPENGL + case WND_PROP_OPENGL: + return window.useGl ? 1.0 : 0.0; +#endif + + case WND_PROP_VISIBLE: + return 1.0; + + case WND_PROP_TOPMOST: + return getPropTopmost_(window); + + case WND_PROP_VSYNC: + return getPropVsync_(window); + + // don't use default, add unsupported cases below: + // case WND_PROP_UNSUPPORTED: // fallthru + // break; + } + return std::numeric_limits::quiet_NaN(); + } + + bool setProperty(int prop, double value) CV_OVERRIDE + { + auto window_ptr = window_.lock(); + CV_Assert(window_ptr); + CvWindow& window = *window_ptr; + // see cvSetWindowProperty + switch ((WindowPropertyFlags)prop) + { + case WND_PROP_FULLSCREEN: + if (value != WINDOW_NORMAL && value != WINDOW_FULLSCREEN) // bad arg + break; + setModeWindow_(window, (int)value); + return true; + + case WND_PROP_TOPMOST: + return setPropTopmost_(window, value != 0.0); + + case WND_PROP_VSYNC: + return setPropVsync_(window, value != 0.0); + + // don't use default, add unsupported cases below: + // case WND_PROP_UNSUPPORTED: // fallthru + case WND_PROP_AUTOSIZE: // fallthru + case WND_PROP_ASPECT_RATIO: // fallthru + case WND_PROP_OPENGL: // fallthru + case WND_PROP_VISIBLE: // fallthru + break; + } + return false; + } + + void resize(int width, int height) CV_OVERRIDE + { + auto window_ptr = window_.lock(); + CV_Assert(window_ptr); + CvWindow& window = *window_ptr; + resizeWindow_(window, Size(width, height)); + } + + void move(int x, int y) CV_OVERRIDE + { + auto window_ptr = window_.lock(); + CV_Assert(window_ptr); + CvWindow& window = *window_ptr; + moveWindow_(window, Point(x, y)); + } + + Rect getImageRect() const CV_OVERRIDE + { + auto window_ptr = window_.lock(); + CV_Assert(window_ptr); + CvWindow& window = *window_ptr; + return getImageRect_(window); + } + + void setTitle(const std::string& title) CV_OVERRIDE + { + auto window_ptr = window_.lock(); + CV_Assert(window_ptr); + CvWindow& window = *window_ptr; + if (!SetWindowText(window.frame, title.c_str())) + CV_Error_(Error::StsError, ("Failed to set \"%s\" window title to \"%s\"", window.name.c_str(), title.c_str())); + } + + void setMouseCallback(MouseCallback onMouse, void* userdata /*= 0*/) CV_OVERRIDE + { + auto window_ptr = window_.lock(); + CV_Assert(window_ptr); + CvWindow& window = *window_ptr; + window.on_mouse = onMouse; + window.on_mouse_param = userdata; + } + + std::shared_ptr createTrackbar( + const std::string& name, + int count, + TrackbarCallback onChange /*= 0*/, + void* userdata /*= 0*/ + ) CV_OVERRIDE + { + auto window_ptr = window_.lock(); + CV_Assert(window_ptr); + CvWindow& window = *window_ptr; + CV_LOG_INFO(NULL, "OpenCV/UI: Creating Win32UI trackbar at '" << name_ << "': '" << name << "'"); + auto trackbar = createTrackbar_(window, name, count, onChange, userdata); + auto ui_trackbar = std::make_shared(name, trackbar, shared_from_this()); + { + cv::AutoLock lock(getWindowMutex()); + trackbars_.emplace(name, ui_trackbar); + } + return std::static_pointer_cast(ui_trackbar); + } + + std::shared_ptr findTrackbar(const std::string& name) CV_OVERRIDE + { + cv::AutoLock lock(getWindowMutex()); + auto i = trackbars_.find(name); + if (i != trackbars_.end()) + { + return std::static_pointer_cast(i->second); + } + return std::shared_ptr(); + } +}; // Win32UIWindow + + +class Win32UITrackbar : public UITrackbar +{ +protected: + /*const*/ std::string name_; + std::weak_ptr trackbar_; + std::weak_ptr parent_; + std::map > trackbars_; +public: + Win32UITrackbar(const std::string& name, const std::shared_ptr& trackbar, const std::shared_ptr& parent) + : trackbar_(trackbar) + , parent_(parent) + { + name_ = std::string("<") + name + ">@" + parent->getID(); + } + + ~Win32UITrackbar() CV_OVERRIDE + { + if (!trackbar_.expired()) + destroy(); + CV_LOG_DEBUG(NULL, "OpenCV/UI/Win32UI: Win32UITrackbar(" << name_ << ") is disposed"); + } + + const std::string& getID() const CV_OVERRIDE { return name_; } + + bool isActive() const CV_OVERRIDE { return !trackbar_.expired(); } + + void destroy() CV_OVERRIDE + { + // nothing (destroyed with parent window, dedicated trackbar removal is not supported) + } + + int getPos() const CV_OVERRIDE + { + auto trackbar_ptr = trackbar_.lock(); + CV_Assert(trackbar_ptr); + CvTrackbar& trackbar = *trackbar_ptr; + return trackbar.pos; + } + void setPos(int pos) CV_OVERRIDE + { + auto trackbar_ptr = trackbar_.lock(); + CV_Assert(trackbar_ptr); + CvTrackbar& trackbar = *trackbar_ptr; + SendMessage(trackbar.hwnd, TBM_SETPOS, (WPARAM)TRUE, (LPARAM)pos); + icvUpdateTrackbar(trackbar, pos); + } + + cv::Range getRange() const CV_OVERRIDE + { + auto trackbar_ptr = trackbar_.lock(); + CV_Assert(trackbar_ptr); + CvTrackbar& trackbar = *trackbar_ptr; + return cv::Range(trackbar.minval, trackbar.maxval); + } + + void setRange(const cv::Range& range) CV_OVERRIDE + { + auto trackbar_ptr = trackbar_.lock(); + CV_Assert(trackbar_ptr); + CvTrackbar& trackbar = *trackbar_ptr; + CV_CheckLE(range.start, range.end, "Invalid trackbar range"); + trackbar.minval = range.start; + trackbar.maxval = range.start; + SendMessage(trackbar.hwnd, TBM_SETRANGEMIN, (WPARAM)TRUE, (LPARAM)trackbar.minval); + SendMessage(trackbar.hwnd, TBM_SETRANGEMAX, (WPARAM)TRUE, (LPARAM)trackbar.maxval); + } +}; // Win32UITrackbar + + +class Win32BackendUI : public UIBackend +{ +public: + ~Win32BackendUI() CV_OVERRIDE + { + destroyAllWindows(); + } + + void destroyAllWindows() CV_OVERRIDE + { + cvDestroyAllWindows(); + } + + // namedWindow + virtual std::shared_ptr createWindow( + const std::string& winname, + int flags + ) CV_OVERRIDE + { + CV_LOG_INFO(NULL, "OpenCV/UI: Creating Win32UI window: " << winname << " (" << flags << ")"); + auto window = namedWindow_(winname, flags); + auto ui_window = std::make_shared(winname, window); + return ui_window; + } + + int waitKeyEx(int delay) CV_OVERRIDE + { + return cvWaitKey(delay); + } + int pollKey() CV_OVERRIDE + { + return pollKey_W32(); + } +}; // Win32BackendUI + +static +std::shared_ptr& getInstance() +{ + static std::shared_ptr g_instance = std::make_shared(); + return g_instance; +} + +} // namespace impl + +#ifndef BUILD_PLUGIN +namespace highgui_backend { + +std::shared_ptr createUIBackendWin32UI() +{ + return impl::getInstance(); +} + +} // namespace highgui_backend +#endif + +} // namespace + + +#ifdef BUILD_PLUGIN + +#define ABI_VERSION 0 +#define API_VERSION 0 +#include "plugin_api.hpp" + +static +CvResult cv_getInstance(CV_OUT CvPluginUIBackend* handle) CV_NOEXCEPT +{ + try + { + if (!handle) + return CV_ERROR_FAIL; + *handle = cv::impl::getInstance().get(); + return CV_ERROR_OK; + } + catch (...) + { + return CV_ERROR_FAIL; + } +} + +static const OpenCV_UI_Plugin_API plugin_api = +{ + { + sizeof(OpenCV_UI_Plugin_API), ABI_VERSION, API_VERSION, + CV_VERSION_MAJOR, CV_VERSION_MINOR, CV_VERSION_REVISION, CV_VERSION_STATUS, + "Win32 OpenCV UI plugin" + }, + { + /* 1*/cv_getInstance + } +}; + +const OpenCV_UI_Plugin_API* CV_API_CALL opencv_ui_plugin_init_v0(int requested_abi_version, int requested_api_version, void* /*reserved=NULL*/) CV_NOEXCEPT +{ + if (requested_abi_version == ABI_VERSION && requested_api_version <= API_VERSION) + return &plugin_api; + return NULL; +} + +#endif // BUILD_PLUGIN + +#endif // HAVE_WIN32UI From 55e1dfb778b806272262871c3060ae32a4ff20d0 Mon Sep 17 00:00:00 2001 From: SamFC10 Date: Sun, 20 Jun 2021 13:19:29 +0530 Subject: [PATCH 014/262] Fix BatchNorm reinitialization --- modules/dnn/src/layers/batch_norm_layer.cpp | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/modules/dnn/src/layers/batch_norm_layer.cpp b/modules/dnn/src/layers/batch_norm_layer.cpp index 27c3db6c44..42676c7938 100644 --- a/modules/dnn/src/layers/batch_norm_layer.cpp +++ b/modules/dnn/src/layers/batch_norm_layer.cpp @@ -29,6 +29,7 @@ namespace dnn class BatchNormLayerImpl CV_FINAL : public BatchNormLayer { public: + Mat origin_weights, origin_bias; Mat weights_, bias_; UMat umat_weight, umat_bias; mutable int dims; @@ -82,11 +83,11 @@ public: const float* weightsData = hasWeights ? blobs[weightsBlobIndex].ptr() : 0; const float* biasData = hasBias ? blobs[biasBlobIndex].ptr() : 0; - weights_.create(1, (int)n, CV_32F); - bias_.create(1, (int)n, CV_32F); + origin_weights.create(1, (int)n, CV_32F); + origin_bias.create(1, (int)n, CV_32F); - float* dstWeightsData = weights_.ptr(); - float* dstBiasData = bias_.ptr(); + float* dstWeightsData = origin_weights.ptr(); + float* dstBiasData = origin_bias.ptr(); for (size_t i = 0; i < n; ++i) { @@ -94,15 +95,12 @@ public: dstWeightsData[i] = w; dstBiasData[i] = (hasBias ? biasData[i] : 0.0f) - w * meanData[i] * varMeanScale; } - // We will use blobs to store origin weights and bias to restore them in case of reinitialization. - weights_.copyTo(blobs[0].reshape(1, 1)); - bias_.copyTo(blobs[1].reshape(1, 1)); } virtual void finalize(InputArrayOfArrays, OutputArrayOfArrays) CV_OVERRIDE { - blobs[0].reshape(1, 1).copyTo(weights_); - blobs[1].reshape(1, 1).copyTo(bias_); + origin_weights.reshape(1, 1).copyTo(weights_); + origin_bias.reshape(1, 1).copyTo(bias_); } void getScaleShift(Mat& scale, Mat& shift) const CV_OVERRIDE From bb60cb0bf97338848859e5402518ff9b53c2a6d1 Mon Sep 17 00:00:00 2001 From: kikaxa Date: Thu, 20 May 2021 16:01:36 +0300 Subject: [PATCH 015/262] Reenable filesystem for ios builds --- .../core/include/opencv2/core/utils/filesystem.private.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/core/include/opencv2/core/utils/filesystem.private.hpp b/modules/core/include/opencv2/core/utils/filesystem.private.hpp index ea2591c9de..72b2bb9479 100644 --- a/modules/core/include/opencv2/core/utils/filesystem.private.hpp +++ b/modules/core/include/opencv2/core/utils/filesystem.private.hpp @@ -16,8 +16,8 @@ # define OPENCV_HAVE_FILESYSTEM_SUPPORT 1 # elif defined(__APPLE__) # include -# if (defined(TARGET_OS_OSX) && TARGET_OS_OSX) || (!defined(TARGET_OS_OSX) && !TARGET_OS_IPHONE) -# define OPENCV_HAVE_FILESYSTEM_SUPPORT 1 // OSX only +# if (defined(TARGET_OS_OSX) && TARGET_OS_OSX) || (defined(TARGET_OS_IOS) && TARGET_OS_IOS) +# define OPENCV_HAVE_FILESYSTEM_SUPPORT 1 // OSX, iOS only # endif # else /* unknown */ From 8be86cbdfd3c941fc12500bcb61a034a9fa9e148 Mon Sep 17 00:00:00 2001 From: Dale Phurrough Date: Tue, 22 Jun 2021 05:32:54 +0200 Subject: [PATCH 016/262] add usageFlags to UMat static factories - add abi compatible overloads - add test case --- modules/core/include/opencv2/core/mat.hpp | 27 ++++++++++++++------- modules/core/src/matrix_operations.cpp | 8 +++---- modules/core/src/umatrix.cpp | 28 +++++++++++----------- modules/core/test/ocl/test_matrix_expr.cpp | 18 ++++++++++++++ 4 files changed, 54 insertions(+), 27 deletions(-) diff --git a/modules/core/include/opencv2/core/mat.hpp b/modules/core/include/opencv2/core/mat.hpp index 5e667b2132..6768be7683 100644 --- a/modules/core/include/opencv2/core/mat.hpp +++ b/modules/core/include/opencv2/core/mat.hpp @@ -2451,7 +2451,8 @@ public: //! <0 - a diagonal from the lower half) UMat diag(int d=0) const; //! constructs a square diagonal matrix which main diagonal is vector "d" - static UMat diag(const UMat& d); + static UMat diag(const UMat& d, UMatUsageFlags usageFlags /*= USAGE_DEFAULT*/); + static UMat diag(const UMat& d) { return diag(d, USAGE_DEFAULT); } // OpenCV 5.0: remove abi compatibility overload //! returns deep copy of the matrix, i.e. the data is copied UMat clone() const CV_NODISCARD; @@ -2485,14 +2486,22 @@ public: double dot(InputArray m) const; //! Matlab-style matrix initialization - static UMat zeros(int rows, int cols, int type); - static UMat zeros(Size size, int type); - static UMat zeros(int ndims, const int* sz, int type); - static UMat ones(int rows, int cols, int type); - static UMat ones(Size size, int type); - static UMat ones(int ndims, const int* sz, int type); - static UMat eye(int rows, int cols, int type); - static UMat eye(Size size, int type); + static UMat zeros(int rows, int cols, int type, UMatUsageFlags usageFlags /*= USAGE_DEFAULT*/); + static UMat zeros(Size size, int type, UMatUsageFlags usageFlags /*= USAGE_DEFAULT*/); + static UMat zeros(int ndims, const int* sz, int type, UMatUsageFlags usageFlags /*= USAGE_DEFAULT*/); + static UMat zeros(int rows, int cols, int type) { return zeros(rows, cols, type, USAGE_DEFAULT); } // OpenCV 5.0: remove abi compatibility overload + static UMat zeros(Size size, int type) { return zeros(size, type, USAGE_DEFAULT); } // OpenCV 5.0: remove abi compatibility overload + static UMat zeros(int ndims, const int* sz, int type) { return zeros(ndims, sz, type, USAGE_DEFAULT); } // OpenCV 5.0: remove abi compatibility overload + static UMat ones(int rows, int cols, int type, UMatUsageFlags usageFlags /*= USAGE_DEFAULT*/); + static UMat ones(Size size, int type, UMatUsageFlags usageFlags /*= USAGE_DEFAULT*/); + static UMat ones(int ndims, const int* sz, int type, UMatUsageFlags usageFlags /*= USAGE_DEFAULT*/); + static UMat ones(int rows, int cols, int type) { return ones(rows, cols, type, USAGE_DEFAULT); } // OpenCV 5.0: remove abi compatibility overload + static UMat ones(Size size, int type) { return ones(size, type, USAGE_DEFAULT); } // OpenCV 5.0: remove abi compatibility overload + static UMat ones(int ndims, const int* sz, int type) { return ones(ndims, sz, type, USAGE_DEFAULT); } // OpenCV 5.0: remove abi compatibility overload + static UMat eye(int rows, int cols, int type, UMatUsageFlags usageFlags /*= USAGE_DEFAULT*/); + static UMat eye(Size size, int type, UMatUsageFlags usageFlags /*= USAGE_DEFAULT*/); + static UMat eye(int rows, int cols, int type) { return eye(rows, cols, type, USAGE_DEFAULT); } // OpenCV 5.0: remove abi compatibility overload + static UMat eye(Size size, int type) { return eye(size, type, USAGE_DEFAULT); } // OpenCV 5.0: remove abi compatibility overload //! allocates new matrix data unless the matrix already has specified size and type. // previous data is unreferenced if needed. diff --git a/modules/core/src/matrix_operations.cpp b/modules/core/src/matrix_operations.cpp index 83c8aaeb57..227c7aaef7 100644 --- a/modules/core/src/matrix_operations.cpp +++ b/modules/core/src/matrix_operations.cpp @@ -229,14 +229,14 @@ void cv::setIdentity( InputOutputArray _m, const Scalar& s ) namespace cv { -UMat UMat::eye(int rows, int cols, int type) +UMat UMat::eye(int rows, int cols, int type, UMatUsageFlags usageFlags) { - return UMat::eye(Size(cols, rows), type); + return UMat::eye(Size(cols, rows), type, usageFlags); } -UMat UMat::eye(Size size, int type) +UMat UMat::eye(Size size, int type, UMatUsageFlags usageFlags) { - UMat m(size, type); + UMat m(size, type, usageFlags); setIdentity(m); return m; } diff --git a/modules/core/src/umatrix.cpp b/modules/core/src/umatrix.cpp index c80d240ecc..bf5dfb68a3 100644 --- a/modules/core/src/umatrix.cpp +++ b/modules/core/src/umatrix.cpp @@ -951,11 +951,11 @@ UMat UMat::reshape(int new_cn, int new_rows) const return hdr; } -UMat UMat::diag(const UMat& d) +UMat UMat::diag(const UMat& d, UMatUsageFlags usageFlags) { CV_Assert( d.cols == 1 || d.rows == 1 ); int len = d.rows + d.cols - 1; - UMat m(len, len, d.type(), Scalar(0)); + UMat m(len, len, d.type(), Scalar(0), usageFlags); UMat md = m.diag(); if( d.cols == 1 ) d.copyTo(md); @@ -1323,34 +1323,34 @@ UMat UMat::t() const return m; } -UMat UMat::zeros(int rows, int cols, int type) +UMat UMat::zeros(int rows, int cols, int type, UMatUsageFlags usageFlags) { - return UMat(rows, cols, type, Scalar::all(0)); + return UMat(rows, cols, type, Scalar::all(0), usageFlags); } -UMat UMat::zeros(Size size, int type) +UMat UMat::zeros(Size size, int type, UMatUsageFlags usageFlags) { - return UMat(size, type, Scalar::all(0)); + return UMat(size, type, Scalar::all(0), usageFlags); } -UMat UMat::zeros(int ndims, const int* sz, int type) +UMat UMat::zeros(int ndims, const int* sz, int type, UMatUsageFlags usageFlags) { - return UMat(ndims, sz, type, Scalar::all(0)); + return UMat(ndims, sz, type, Scalar::all(0), usageFlags); } -UMat UMat::ones(int rows, int cols, int type) +UMat UMat::ones(int rows, int cols, int type, UMatUsageFlags usageFlags) { - return UMat::ones(Size(cols, rows), type); + return UMat(rows, cols, type, Scalar(1), usageFlags); } -UMat UMat::ones(Size size, int type) +UMat UMat::ones(Size size, int type, UMatUsageFlags usageFlags) { - return UMat(size, type, Scalar(1)); + return UMat(size, type, Scalar(1), usageFlags); } -UMat UMat::ones(int ndims, const int* sz, int type) +UMat UMat::ones(int ndims, const int* sz, int type, UMatUsageFlags usageFlags) { - return UMat(ndims, sz, type, Scalar(1)); + return UMat(ndims, sz, type, Scalar(1), usageFlags); } } diff --git a/modules/core/test/ocl/test_matrix_expr.cpp b/modules/core/test/ocl/test_matrix_expr.cpp index 7a5ff72cb2..f11c0a6ebb 100644 --- a/modules/core/test/ocl/test_matrix_expr.cpp +++ b/modules/core/test/ocl/test_matrix_expr.cpp @@ -76,6 +76,24 @@ OCL_TEST_P(UMatExpr, Ones) } } +//////////////////////////////// with usageFlags ///////////////////////////////////////////////// + +OCL_TEST_P(UMatExpr, WithUsageFlags) +{ + for (int j = 0; j < test_loop_times; j++) + { + generateTestData(); + + UMat u0 = UMat::zeros(size, type, cv::USAGE_ALLOCATE_HOST_MEMORY); + UMat u1 = UMat::ones(size, type, cv::USAGE_ALLOCATE_HOST_MEMORY); + UMat u8 = UMat::eye(size, type, cv::USAGE_ALLOCATE_HOST_MEMORY); + + EXPECT_EQ(cv::USAGE_ALLOCATE_HOST_MEMORY, u0.usageFlags); + EXPECT_EQ(cv::USAGE_ALLOCATE_HOST_MEMORY, u1.usageFlags); + EXPECT_EQ(cv::USAGE_ALLOCATE_HOST_MEMORY, u8.usageFlags); + } +} + //////////////////////////////// Instantiation ///////////////////////////////////////////////// OCL_INSTANTIATE_TEST_CASE_P(MatrixOperation, UMatExpr, Combine(OCL_ALL_DEPTHS_16F, OCL_ALL_CHANNELS)); From b68057d92701e24765d9ff199011f5171d320143 Mon Sep 17 00:00:00 2001 From: Vincent Rabaud Date: Wed, 23 Jun 2021 21:27:54 +0200 Subject: [PATCH 017/262] Do not use = 0 for a cv::Mat. There are several operator= overloads and some compilers can be confused. --- modules/calib3d/src/chessboard.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/calib3d/src/chessboard.cpp b/modules/calib3d/src/chessboard.cpp index dbc47722cb..18e2605f53 100644 --- a/modules/calib3d/src/chessboard.cpp +++ b/modules/calib3d/src/chessboard.cpp @@ -3924,7 +3924,7 @@ bool findChessboardCornersSB(cv::InputArray image_, cv::Size pattern_size, { meta_.create(int(board.rowCount()),int(board.colCount()),CV_8UC1); cv::Mat meta = meta_.getMat(); - meta = 0; + meta.setTo(cv::Scalar::all(0)); for(int row =0;row < meta.rows-1;++row) { for(int col=0;col< meta.cols-1;++col) From dc5199feeae9cef33bd55cc8c161917bc3ef367b Mon Sep 17 00:00:00 2001 From: Smirnov Egor Date: Mon, 21 Jun 2021 11:11:14 +0300 Subject: [PATCH 018/262] skipping missing layers and layer failures --- apps/model-diagnostics/model_diagnostics.cpp | 6 +-- modules/dnn/src/tensorflow/tf_importer.cpp | 43 +++++++++++++++++--- 2 files changed, 41 insertions(+), 8 deletions(-) diff --git a/apps/model-diagnostics/model_diagnostics.cpp b/apps/model-diagnostics/model_diagnostics.cpp index 2ffeaa1ea5..d3934577ae 100644 --- a/apps/model-diagnostics/model_diagnostics.cpp +++ b/apps/model-diagnostics/model_diagnostics.cpp @@ -1,6 +1,6 @@ /************************************************* USAGE: -./model_diagnostics -m +./model_diagnostics -m **************************************************/ #include #include @@ -32,7 +32,7 @@ static std::string checkFileExists(const std::string& fileName) } std::string diagnosticKeys = - "{ model m | | Path to the model .onnx file. }" + "{ model m | | Path to the model file. }" "{ config c | | Path to the model configuration file. }" "{ framework f | | [Optional] Name of the model framework. }"; @@ -41,7 +41,7 @@ std::string diagnosticKeys = int main( int argc, const char** argv ) { CommandLineParser argParser(argc, argv, diagnosticKeys); - argParser.about("Use this tool to run the diagnostics of provided ONNX model" + argParser.about("Use this tool to run the diagnostics of provided ONNX/TF model" "to obtain the information about its support (supported layers)."); if (argc == 1) diff --git a/modules/dnn/src/tensorflow/tf_importer.cpp b/modules/dnn/src/tensorflow/tf_importer.cpp index ed6a407920..15f88007b4 100644 --- a/modules/dnn/src/tensorflow/tf_importer.cpp +++ b/modules/dnn/src/tensorflow/tf_importer.cpp @@ -32,6 +32,8 @@ namespace cv { namespace dnn { CV__DNN_INLINE_NS_BEGIN +extern bool DNN_DIAGNOSTICS_RUN; + #if HAVE_PROTOBUF using ::google::protobuf::RepeatedField; @@ -471,6 +473,7 @@ public: TFImporter(Net& net, const char *dataModel, size_t lenModel, const char *dataConfig = NULL, size_t lenConfig = 0); protected: + std::unique_ptr utilNet; Net& dstNet; void populateNet(); @@ -2337,7 +2340,8 @@ void TFImporter::parseCustomLayer(tensorflow::GraphDef& net, const tensorflow::N } TFImporter::TFImporter(Net& net, const char *model, const char *config) - : dstNet(net), dispatch(buildDispatchMap()) + : utilNet(DNN_DIAGNOSTICS_RUN ? new Net : nullptr), + dstNet(DNN_DIAGNOSTICS_RUN ? *utilNet : net), dispatch(buildDispatchMap()) { if (model && model[0]) { @@ -2358,7 +2362,8 @@ TFImporter::TFImporter( const char *dataModel, size_t lenModel, const char *dataConfig, size_t lenConfig ) - : dstNet(net), dispatch(buildDispatchMap()) + : utilNet(DNN_DIAGNOSTICS_RUN ? new Net : nullptr), + dstNet(DNN_DIAGNOSTICS_RUN ? *utilNet : net), dispatch(buildDispatchMap()) { if (dataModel != NULL && lenModel > 0) { @@ -2615,6 +2620,11 @@ DataLayout TFImporter::predictOutputDataLayout(const tensorflow::NodeDef& layer) return it->second; } +Ptr dummy_constructor(LayerParams & params) +{ + return new Layer(params); +} + void TFImporter::populateNet() { CV_Assert(netBin.ByteSize() || netTxt.ByteSize()); @@ -2757,9 +2767,9 @@ void TFImporter::parseNode(const tensorflow::NodeDef& layer) const std::string& name = layer.name(); const std::string& type = layer.op(); + LayerParams layerParams; try { - LayerParams layerParams; if (layers_to_ignore.find(name) != layers_to_ignore.end()) { @@ -2777,13 +2787,36 @@ void TFImporter::parseNode(const tensorflow::NodeDef& layer) } else { + if (DNN_DIAGNOSTICS_RUN && !LayerFactory::createLayerInstance(type, layerParams)) + { + CV_LOG_ERROR(NULL, "DNN/TF: Node='" << name << "' of type='"<< type + << "' is not supported. This error won't be displayed again."); + LayerFactory::registerLayer(type, dummy_constructor); + } + parseCustomLayer(net, layer, layerParams); } } catch (const std::exception& e) { - CV_LOG_ERROR(NULL, "DNN/TF: Can't parse layer for node='" << name << "'. Exception: " << e.what()); - throw; + if (!DNN_DIAGNOSTICS_RUN) + { + CV_LOG_ERROR(NULL, "DNN/TF: Can't parse layer for node='" << name << "' of type='" << type + << "'. Exception: " << e.what()); + throw; + } + else + { + CV_LOG_ERROR(NULL, "DNN/TF: Can't parse layer for node='" << name << "' of type='" << type + << "'. Exception: " << e.what()); + + // internal layer failure (didnt call addLayer) + if (dstNet.getLayerId(name) == -1) + { + int id = dstNet.addLayer(name, type, layerParams); + layer_id[name] = id; + } + } } } From c95a56450dd8c612b3f8e83d680a0a6c6533acf0 Mon Sep 17 00:00:00 2001 From: Alexey Smirnov Date: Sat, 26 Jun 2021 00:09:33 +0300 Subject: [PATCH 019/262] Merge pull request #20156 from smirnov-alexey:as/gapi_remote_infer G-API: Support remote inference * Extend MediaFrame to be able to extract additional info besides access * Add API for remote inference * Add default implementation for blobParams() * Add default implementation for blobParams() * Address review comments * Fix any_cast usage * Add comment on the default blobParams() * Address review comments * Add missing rctx * Minor fix * Fix indentation and comment * Address review comments * Add documentation --- .../gapi/include/opencv2/gapi/infer/ie.hpp | 44 +++++++++++++++++-- modules/gapi/src/backends/ie/giebackend.cpp | 40 ++++++++++++++++- .../src/backends/ie/giebackend/giewrapper.hpp | 26 +++++++++-- 3 files changed, 100 insertions(+), 10 deletions(-) diff --git a/modules/gapi/include/opencv2/gapi/infer/ie.hpp b/modules/gapi/include/opencv2/gapi/infer/ie.hpp index 70712ba740..2be739e518 100644 --- a/modules/gapi/include/opencv2/gapi/infer/ie.hpp +++ b/modules/gapi/include/opencv2/gapi/infer/ie.hpp @@ -74,7 +74,11 @@ struct ParamDesc { std::map> reshape_table; std::unordered_set layer_names_to_reshape; + // NB: Number of asyncrhonious infer requests size_t nireq; + + // NB: An optional config to setup RemoteContext for IE + cv::util::any context_config; }; } // namespace detail @@ -115,7 +119,8 @@ public: , {} , {} , {} - , 1u} { + , 1u + , {}} { }; /** @overload @@ -135,7 +140,8 @@ public: , {} , {} , {} - , 1u} { + , 1u + , {}} { }; /** @brief Specifies sequence of network input layers names for inference. @@ -217,6 +223,30 @@ public: return *this; } + /** @brief Specifies configuration for RemoteContext in InferenceEngine. + + When RemoteContext is configured the backend imports the networks using the context. + It also expects cv::MediaFrames to be actually remote, to operate with blobs via the context. + + @param ctx_cfg cv::util::any value which holds InferenceEngine::ParamMap. + @return reference to this parameter structure. + */ + Params& cfgContextParams(const cv::util::any& ctx_cfg) { + desc.context_config = ctx_cfg; + return *this; + } + + /** @overload + Function with an rvalue parameter. + + @param ctx_cfg cv::util::any value which holds InferenceEngine::ParamMap. + @return reference to this parameter structure. + */ + Params& cfgContextParams(cv::util::any&& ctx_cfg) { + desc.context_config = std::move(ctx_cfg); + return *this; + } + /** @brief Specifies number of asynchronous inference requests. @param nireq Number of inference asynchronous requests. @@ -318,7 +348,10 @@ public: const std::string &model, const std::string &weights, const std::string &device) - : desc{ model, weights, device, {}, {}, {}, 0u, 0u, detail::ParamDesc::Kind::Load, true, {}, {}, {}, 1u}, m_tag(tag) { + : desc{ model, weights, device, {}, {}, {}, 0u, 0u, + detail::ParamDesc::Kind::Load, true, {}, {}, {}, 1u, + {}}, + m_tag(tag) { }; /** @overload @@ -333,7 +366,10 @@ public: Params(const std::string &tag, const std::string &model, const std::string &device) - : desc{ model, {}, device, {}, {}, {}, 0u, 0u, detail::ParamDesc::Kind::Import, true, {}, {}, {}, 1u}, m_tag(tag) { + : desc{ model, {}, device, {}, {}, {}, 0u, 0u, + detail::ParamDesc::Kind::Import, true, {}, {}, {}, 1u, + {}}, + m_tag(tag) { }; /** @see ie::Params::pluginConfig. */ diff --git a/modules/gapi/src/backends/ie/giebackend.cpp b/modules/gapi/src/backends/ie/giebackend.cpp index 46b6bdbb97..77a6515f85 100644 --- a/modules/gapi/src/backends/ie/giebackend.cpp +++ b/modules/gapi/src/backends/ie/giebackend.cpp @@ -222,8 +222,17 @@ struct IEUnit { IE::ExecutableNetwork this_network; cv::gimpl::ie::wrap::Plugin this_plugin; + InferenceEngine::RemoteContext::Ptr rctx = nullptr; + explicit IEUnit(const cv::gapi::ie::detail::ParamDesc &pp) : params(pp) { + InferenceEngine::ParamMap* ctx_params = + cv::util::any_cast(¶ms.context_config); + if (ctx_params != nullptr) { + auto ie_core = cv::gimpl::ie::wrap::getCore(); + rctx = ie_core.CreateContext(params.device_id, *ctx_params); + } + if (params.kind == cv::gapi::ie::detail::ParamDesc::Kind::Load) { net = cv::gimpl::ie::wrap::readNetwork(params); inputs = net.getInputsInfo(); @@ -231,7 +240,7 @@ struct IEUnit { } else if (params.kind == cv::gapi::ie::detail::ParamDesc::Kind::Import) { this_plugin = cv::gimpl::ie::wrap::getPlugin(params); this_plugin.SetConfig(params.config); - this_network = cv::gimpl::ie::wrap::importNetwork(this_plugin, params); + this_network = cv::gimpl::ie::wrap::importNetwork(this_plugin, params, rctx); // FIXME: ICNNetwork returns InputsDataMap/OutputsDataMap, // but ExecutableNetwork returns ConstInputsDataMap/ConstOutputsDataMap inputs = cv::gimpl::ie::wrap::toInputsDataMap(this_network.GetInputsInfo()); @@ -279,7 +288,8 @@ struct IEUnit { // for loadNetwork they can be obtained by using readNetwork non_const_this->this_plugin = cv::gimpl::ie::wrap::getPlugin(params); non_const_this->this_plugin.SetConfig(params.config); - non_const_this->this_network = cv::gimpl::ie::wrap::loadNetwork(non_const_this->this_plugin, net, params); + non_const_this->this_network = cv::gimpl::ie::wrap::loadNetwork(non_const_this->this_plugin, + net, params, rctx); } return {params, this_plugin, this_network}; @@ -481,7 +491,32 @@ using GConstGIEModel = ade::ConstTypedGraph , IECallable >; +inline IE::Blob::Ptr extractRemoteBlob(IECallContext& ctx, std::size_t i) { + GAPI_Assert(ctx.inShape(i) == cv::GShape::GFRAME && + "Remote blob is supported for MediaFrame only"); + + cv::util::any any_blob_params = ctx.inFrame(i).blobParams(); + auto ie_core = cv::gimpl::ie::wrap::getCore(); + + using ParamType = std::pair; + + ParamType* blob_params = cv::util::any_cast(&any_blob_params); + if (blob_params == nullptr) { + GAPI_Assert(false && "Incorrect type of blobParams: " + "expected std::pair"); + } + + return ctx.uu.rctx->CreateBlob(blob_params->first, + blob_params->second); +} + inline IE::Blob::Ptr extractBlob(IECallContext& ctx, std::size_t i) { + if (ctx.uu.rctx != nullptr) { + return extractRemoteBlob(ctx, i); + } + switch (ctx.inShape(i)) { case cv::GShape::GFRAME: { const auto& frame = ctx.inFrame(i); @@ -1060,6 +1095,7 @@ struct InferList: public cv::detail::KernelTag { } IE::Blob::Ptr this_blob = extractBlob(*ctx, 1); + std::vector> cached_dims(ctx->uu.params.num_out); for (auto i : ade::util::iota(ctx->uu.params.num_out)) { const IE::DataPtr& ie_out = ctx->uu.outputs.at(ctx->uu.params.output_names[i]); diff --git a/modules/gapi/src/backends/ie/giebackend/giewrapper.hpp b/modules/gapi/src/backends/ie/giebackend/giewrapper.hpp index 3927c802b7..7e67cb8989 100644 --- a/modules/gapi/src/backends/ie/giebackend/giewrapper.hpp +++ b/modules/gapi/src/backends/ie/giebackend/giewrapper.hpp @@ -13,6 +13,7 @@ #include #include +#include #include "opencv2/gapi/infer/ie.hpp" @@ -50,12 +51,29 @@ GAPI_EXPORTS IE::Core getCore(); GAPI_EXPORTS IE::Core getPlugin(const GIEParam& params); GAPI_EXPORTS inline IE::ExecutableNetwork loadNetwork( IE::Core& core, const IE::CNNNetwork& net, - const GIEParam& params) { - return core.LoadNetwork(net, params.device_id); + const GIEParam& params, + IE::RemoteContext::Ptr rctx = nullptr) { + if (rctx != nullptr) { + return core.LoadNetwork(net, rctx); + } else { + return core.LoadNetwork(net, params.device_id); + } } GAPI_EXPORTS inline IE::ExecutableNetwork importNetwork( IE::Core& core, - const GIEParam& param) { - return core.ImportNetwork(param.model_path, param.device_id, {}); + const GIEParam& params, + IE::RemoteContext::Ptr rctx = nullptr) { + if (rctx != nullptr) { + std::filebuf blobFile; + if (!blobFile.open(params.model_path, std::ios::in | std::ios::binary)) + { + blobFile.close(); + throw std::runtime_error("Could not open file"); + } + std::istream graphBlob(&blobFile); + return core.ImportNetwork(graphBlob, rctx); + } else { + return core.ImportNetwork(params.model_path, params.device_id, {}); + } } #endif // INF_ENGINE_RELEASE < 2019020000 }}}} From 42d644ef9134bcb620dd00f7ab7a6d7d039bfdf2 Mon Sep 17 00:00:00 2001 From: xzvno Date: Sun, 27 Jun 2021 05:01:31 +0800 Subject: [PATCH 020/262] Merge pull request #20293 from endjkv:fix-mem-leak-when-throw * fix memory leak when exception is thrown --- modules/core/src/system.cpp | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/modules/core/src/system.cpp b/modules/core/src/system.cpp index af4a621816..441457d50f 100644 --- a/modules/core/src/system.cpp +++ b/modules/core/src/system.cpp @@ -1835,7 +1835,15 @@ void* TLSDataContainer::getData() const { // Create new data instance and save it to TLS storage pData = createDataInstance(); - getTlsStorage().setData(key_, pData); + try + { + getTlsStorage().setData(key_, pData); + } + catch (...) + { + deleteDataInstance(pData); + throw; + } } return pData; } From 61a5378aeb3a6be13cbca1a1e6a6874358996fb4 Mon Sep 17 00:00:00 2001 From: Giles Payne Date: Sun, 27 Jun 2021 21:08:25 +0900 Subject: [PATCH 021/262] Improvements/fixes for unsigned type handling in Swift/Kotlin --- modules/core/misc/java/src/java/core+MatAt.kt | 93 ++++++++-- modules/core/misc/objc/common/Mat.mm | 2 +- modules/core/misc/objc/common/MatExt.swift | 146 +++++++++++++-- modules/core/misc/objc/test/MatTest.swift | 172 ++++++++++++++++-- 4 files changed, 366 insertions(+), 47 deletions(-) diff --git a/modules/core/misc/java/src/java/core+MatAt.kt b/modules/core/misc/java/src/java/core+MatAt.kt index f48a3deaed..c81e21057f 100644 --- a/modules/core/misc/java/src/java/core+MatAt.kt +++ b/modules/core/misc/java/src/java/core+MatAt.kt @@ -3,6 +3,16 @@ package org.opencv.core import org.opencv.core.Mat.* import java.lang.RuntimeException +fun Mat.get(row: Int, col: Int, data: UByteArray) = this.get(row, col, data.asByteArray()) +fun Mat.get(indices: IntArray, data: UByteArray) = this.get(indices, data.asByteArray()) +fun Mat.put(row: Int, col: Int, data: UByteArray) = this.put(row, col, data.asByteArray()) +fun Mat.put(indices: IntArray, data: UByteArray) = this.put(indices, data.asByteArray()) + +fun Mat.get(row: Int, col: Int, data: UShortArray) = this.get(row, col, data.asShortArray()) +fun Mat.get(indices: IntArray, data: UShortArray) = this.get(indices, data.asShortArray()) +fun Mat.put(row: Int, col: Int, data: UShortArray) = this.put(row, col, data.asShortArray()) +fun Mat.put(indices: IntArray, data: UShortArray) = this.put(indices, data.asShortArray()) + /*** * Example use: * @@ -19,6 +29,7 @@ inline fun Mat.at(row: Int, col: Int) : Atable = col ) UByte::class -> AtableUByte(this, row, col) as Atable + UShort::class -> AtableUShort(this, row, col) as Atable else -> throw RuntimeException("Unsupported class type") } @@ -30,6 +41,7 @@ inline fun Mat.at(idx: IntArray) : Atable = idx ) UByte::class -> AtableUByte(this, idx) as Atable + UShort::class -> AtableUShort(this, idx) as Atable else -> throw RuntimeException("Unsupported class type") } @@ -38,46 +50,95 @@ class AtableUByte(val mat: Mat, val indices: IntArray): Atable { constructor(mat: Mat, row: Int, col: Int) : this(mat, intArrayOf(row, col)) override fun getV(): UByte { - val data = ByteArray(1) - mat[indices, data] - return data[0].toUByte() + val data = UByteArray(1) + mat.get(indices, data) + return data[0] } override fun setV(v: UByte) { - val data = byteArrayOf(v.toByte()) + val data = ubyteArrayOf(v) mat.put(indices, data) } override fun getV2c(): Tuple2 { - val data = ByteArray(2) - mat[indices, data] - return Tuple2(data[0].toUByte(), data[1].toUByte()) + val data = UByteArray(2) + mat.get(indices, data) + return Tuple2(data[0], data[1]) } override fun setV2c(v: Tuple2) { - val data = byteArrayOf(v._0.toByte(), v._1.toByte()) + val data = ubyteArrayOf(v._0, v._1) mat.put(indices, data) } override fun getV3c(): Tuple3 { - val data = ByteArray(3) - mat[indices, data] - return Tuple3(data[0].toUByte(), data[1].toUByte(), data[2].toUByte()) + val data = UByteArray(3) + mat.get(indices, data) + return Tuple3(data[0], data[1], data[2]) } override fun setV3c(v: Tuple3) { - val data = byteArrayOf(v._0.toByte(), v._1.toByte(), v._2.toByte()) + val data = ubyteArrayOf(v._0, v._1, v._2) mat.put(indices, data) } override fun getV4c(): Tuple4 { - val data = ByteArray(4) - mat[indices, data] - return Tuple4(data[0].toUByte(), data[1].toUByte(), data[2].toUByte(), data[3].toUByte()) + val data = UByteArray(4) + mat.get(indices, data) + return Tuple4(data[0], data[1], data[2], data[3]) } override fun setV4c(v: Tuple4) { - val data = byteArrayOf(v._0.toByte(), v._1.toByte(), v._2.toByte(), v._3.toByte()) + val data = ubyteArrayOf(v._0, v._1, v._2, v._3) + mat.put(indices, data) + } +} + +class AtableUShort(val mat: Mat, val indices: IntArray): Atable { + + constructor(mat: Mat, row: Int, col: Int) : this(mat, intArrayOf(row, col)) + + override fun getV(): UShort { + val data = UShortArray(1) + mat.get(indices, data) + return data[0] + } + + override fun setV(v: UShort) { + val data = ushortArrayOf(v) + mat.put(indices, data) + } + + override fun getV2c(): Tuple2 { + val data = UShortArray(2) + mat.get(indices, data) + return Tuple2(data[0], data[1]) + } + + override fun setV2c(v: Tuple2) { + val data = ushortArrayOf(v._0, v._1) + mat.put(indices, data) + } + + override fun getV3c(): Tuple3 { + val data = UShortArray(3) + mat.get(indices, data) + return Tuple3(data[0], data[1], data[2]) + } + + override fun setV3c(v: Tuple3) { + val data = ushortArrayOf(v._0, v._1, v._2) + mat.put(indices, data) + } + + override fun getV4c(): Tuple4 { + val data = UShortArray(4) + mat.get(indices, data) + return Tuple4(data[0], data[1], data[2], data[3]) + } + + override fun setV4c(v: Tuple4) { + val data = ushortArrayOf(v._0, v._1, v._2, v._3) mat.put(indices, data) } } diff --git a/modules/core/misc/objc/common/Mat.mm b/modules/core/misc/objc/common/Mat.mm index 5d41a3622e..045bd8393e 100644 --- a/modules/core/misc/objc/common/Mat.mm +++ b/modules/core/misc/objc/common/Mat.mm @@ -548,7 +548,7 @@ template void putData(uchar* dataDest, int count, T (^readData)(int) if (depth == CV_8U) { putData(dest, count, ^uchar (int index) { return cv::saturate_cast(data[offset + index].doubleValue);} ); } else if (depth == CV_8S) { - putData(dest, count, ^char (int index) { return cv::saturate_cast(data[offset + index].doubleValue);} ); + putData(dest, count, ^schar (int index) { return cv::saturate_cast(data[offset + index].doubleValue);} ); } else if (depth == CV_16U) { putData(dest, count, ^ushort (int index) { return cv::saturate_cast(data[offset + index].doubleValue);} ); } else if (depth == CV_16S) { diff --git a/modules/core/misc/objc/common/MatExt.swift b/modules/core/misc/objc/common/MatExt.swift index 5ce3a5e6fb..a6ba548599 100644 --- a/modules/core/misc/objc/common/MatExt.swift +++ b/modules/core/misc/objc/common/MatExt.swift @@ -62,6 +62,21 @@ public extension Mat { } } + @discardableResult func get(indices:[Int32], data:inout [UInt8]) throws -> Int32 { + let channels = CvType.channels(Int32(type())) + if Int32(data.count) % channels != 0 { + try throwIncompatibleBufferSize(count: data.count, channels: channels) + } else if depth() != CvType.CV_8U { + try throwIncompatibleDataType(typeName: CvType.type(toString: type())) + } + let count = Int32(data.count) + return data.withUnsafeMutableBufferPointer { body in + body.withMemoryRebound(to: Int8.self) { reboundBody in + return __get(indices as [NSNumber], count: count, byteBuffer: reboundBody.baseAddress!) + } + } + } + @discardableResult func get(indices:[Int32], data:inout [Double]) throws -> Int32 { let channels = CvType.channels(Int32(type())) if Int32(data.count) % channels != 0 { @@ -114,10 +129,29 @@ public extension Mat { } } + @discardableResult func get(indices:[Int32], data:inout [UInt16]) throws -> Int32 { + let channels = CvType.channels(Int32(type())) + if Int32(data.count) % channels != 0 { + try throwIncompatibleBufferSize(count: data.count, channels: channels) + } else if depth() != CvType.CV_16U { + try throwIncompatibleDataType(typeName: CvType.type(toString: type())) + } + let count = Int32(data.count) + return data.withUnsafeMutableBufferPointer { body in + body.withMemoryRebound(to: Int16.self) { reboundBody in + return __get(indices as [NSNumber], count: count, shortBuffer: reboundBody.baseAddress!) + } + } + } + @discardableResult func get(row: Int32, col: Int32, data:inout [Int8]) throws -> Int32 { return try get(indices: [row, col], data: &data) } + @discardableResult func get(row: Int32, col: Int32, data:inout [UInt8]) throws -> Int32 { + return try get(indices: [row, col], data: &data) + } + @discardableResult func get(row: Int32, col: Int32, data:inout [Double]) throws -> Int32 { return try get(indices: [row, col], data: &data) } @@ -134,6 +168,10 @@ public extension Mat { return try get(indices: [row, col], data: &data) } + @discardableResult func get(row: Int32, col: Int32, data:inout [UInt16]) throws -> Int32 { + return try get(indices: [row, col], data: &data) + } + @discardableResult func put(indices:[Int32], data:[Int8]) throws -> Int32 { let channels = CvType.channels(Int32(type())) if Int32(data.count) % channels != 0 { @@ -147,6 +185,21 @@ public extension Mat { } } + @discardableResult func put(indices:[Int32], data:[UInt8]) throws -> Int32 { + let channels = CvType.channels(Int32(type())) + if Int32(data.count) % channels != 0 { + try throwIncompatibleBufferSize(count: data.count, channels: channels) + } else if depth() != CvType.CV_8U { + try throwIncompatibleDataType(typeName: CvType.type(toString: type())) + } + let count = Int32(data.count) + return data.withUnsafeBufferPointer { body in + body.withMemoryRebound(to: Int8.self) { reboundBody in + return __put(indices as [NSNumber], count: count, byteBuffer: reboundBody.baseAddress!) + } + } + } + @discardableResult func put(indices:[Int32], data:[Int8], offset: Int, length: Int32) throws -> Int32 { let channels = CvType.channels(Int32(type())) if Int32(data.count) % channels != 0 { @@ -214,10 +267,29 @@ public extension Mat { } } + @discardableResult func put(indices:[Int32], data:[UInt16]) throws -> Int32 { + let channels = CvType.channels(Int32(type())) + if Int32(data.count) % channels != 0 { + try throwIncompatibleBufferSize(count: data.count, channels: channels) + } else if depth() != CvType.CV_16U { + try throwIncompatibleDataType(typeName: CvType.type(toString: type())) + } + let count = Int32(data.count) + return data.withUnsafeBufferPointer { body in + body.withMemoryRebound(to: Int16.self) { reboundBody in + return __put(indices as [NSNumber], count: count, shortBuffer: reboundBody.baseAddress!) + } + } + } + @discardableResult func put(row: Int32, col: Int32, data:[Int8]) throws -> Int32 { return try put(indices: [row, col], data: data) } + @discardableResult func put(row: Int32, col: Int32, data:[UInt8]) throws -> Int32 { + return try put(indices: [row, col], data: data) + } + @discardableResult func put(row: Int32, col: Int32, data: [Int8], offset: Int, length: Int32) throws -> Int32 { return try put(indices: [row, col], data: data, offset: offset, length: length) } @@ -238,6 +310,10 @@ public extension Mat { return try put(indices: [row, col], data: data) } + @discardableResult func put(row: Int32, col: Int32, data: [UInt16]) throws -> Int32 { + return try put(indices: [row, col], data: data) + } + @discardableResult func get(row: Int32, col: Int32) -> [Double] { return get(indices: [row, col]) } @@ -303,46 +379,46 @@ public class MatAt { extension UInt8: Atable { public static func getAt(m: Mat, indices:[Int32]) -> UInt8 { - var tmp = [Int8](repeating: 0, count: 1) + var tmp = [UInt8](repeating: 0, count: 1) try! m.get(indices: indices, data: &tmp) - return UInt8(bitPattern: tmp[0]) + return tmp[0] } public static func putAt(m: Mat, indices: [Int32], v: UInt8) { - let tmp = [Int8(bitPattern: v)] + let tmp = [v] try! m.put(indices: indices, data: tmp) } public static func getAt2c(m: Mat, indices:[Int32]) -> (UInt8, UInt8) { - var tmp = [Int8](repeating: 0, count: 2) + var tmp = [UInt8](repeating: 0, count: 2) try! m.get(indices: indices, data: &tmp) - return (UInt8(bitPattern: tmp[0]), UInt8(bitPattern: tmp[1])) + return (tmp[0], tmp[1]) } public static func putAt2c(m: Mat, indices: [Int32], v: (UInt8, UInt8)) { - let tmp = [Int8(bitPattern: v.0), Int8(bitPattern: v.1)] + let tmp = [v.0, v.1] try! m.put(indices: indices, data: tmp) } public static func getAt3c(m: Mat, indices:[Int32]) -> (UInt8, UInt8, UInt8) { - var tmp = [Int8](repeating: 0, count: 3) + var tmp = [UInt8](repeating: 0, count: 3) try! m.get(indices: indices, data: &tmp) - return (UInt8(bitPattern: tmp[0]), UInt8(bitPattern: tmp[1]), UInt8(bitPattern: tmp[2])) + return (tmp[0], tmp[1], tmp[2]) } public static func putAt3c(m: Mat, indices: [Int32], v: (UInt8, UInt8, UInt8)) { - let tmp = [Int8(bitPattern: v.0), Int8(bitPattern: v.1), Int8(bitPattern: v.2)] + let tmp = [v.0, v.1, v.2] try! m.put(indices: indices, data: tmp) } public static func getAt4c(m: Mat, indices:[Int32]) -> (UInt8, UInt8, UInt8, UInt8) { - var tmp = [Int8](repeating: 0, count: 4) + var tmp = [UInt8](repeating: 0, count: 4) try! m.get(indices: indices, data: &tmp) - return (UInt8(bitPattern: tmp[0]), UInt8(bitPattern: tmp[1]), UInt8(bitPattern: tmp[2]), UInt8(bitPattern: tmp[3])) + return (tmp[0], tmp[1], tmp[2], tmp[3]) } public static func putAt4c(m: Mat, indices: [Int32], v: (UInt8, UInt8, UInt8, UInt8)) { - let tmp = [Int8(bitPattern: v.0), Int8(bitPattern: v.1), Int8(bitPattern: v.2), Int8(bitPattern: v.3)] + let tmp = [v.0, v.1, v.2, v.3] try! m.put(indices: indices, data: tmp) } } @@ -531,6 +607,52 @@ extension Int32: Atable { } } +extension UInt16: Atable { + public static func getAt(m: Mat, indices:[Int32]) -> UInt16 { + var tmp = [UInt16](repeating: 0, count: 1) + try! m.get(indices: indices, data: &tmp) + return tmp[0] + } + + public static func putAt(m: Mat, indices: [Int32], v: UInt16) { + let tmp = [v] + try! m.put(indices: indices, data: tmp) + } + + public static func getAt2c(m: Mat, indices:[Int32]) -> (UInt16, UInt16) { + var tmp = [UInt16](repeating: 0, count: 2) + try! m.get(indices: indices, data: &tmp) + return (tmp[0], tmp[1]) + } + + public static func putAt2c(m: Mat, indices: [Int32], v: (UInt16, UInt16)) { + let tmp = [v.0, v.1] + try! m.put(indices: indices, data: tmp) + } + + public static func getAt3c(m: Mat, indices:[Int32]) -> (UInt16, UInt16, UInt16) { + var tmp = [UInt16](repeating: 0, count: 3) + try! m.get(indices: indices, data: &tmp) + return (tmp[0], tmp[1], tmp[2]) + } + + public static func putAt3c(m: Mat, indices: [Int32], v: (UInt16, UInt16, UInt16)) { + let tmp = [v.0, v.1, v.2] + try! m.put(indices: indices, data: tmp) + } + + public static func getAt4c(m: Mat, indices:[Int32]) -> (UInt16, UInt16, UInt16, UInt16) { + var tmp = [UInt16](repeating: 0, count: 4) + try! m.get(indices: indices, data: &tmp) + return (tmp[0], tmp[1], tmp[2], tmp[3]) + } + + public static func putAt4c(m: Mat, indices: [Int32], v: (UInt16, UInt16, UInt16, UInt16)) { + let tmp = [v.0, v.1, v.2, v.3] + try! m.put(indices: indices, data: tmp) + } +} + extension Int16: Atable { public static func getAt(m: Mat, indices:[Int32]) -> Int16 { var tmp = [Int16](repeating: 0, count: 1) diff --git a/modules/core/misc/objc/test/MatTest.swift b/modules/core/misc/objc/test/MatTest.swift index 14c440b5eb..8a513505cc 100644 --- a/modules/core/misc/objc/test/MatTest.swift +++ b/modules/core/misc/objc/test/MatTest.swift @@ -308,15 +308,15 @@ class MatTests: OpenCVTestCase { XCTAssert([340] == sm.get(row: 1, col: 1)) } - func testGetIntIntByteArray() throws { - let m = try getTestMat(size: 5, type: CvType.CV_8UC3) + func testGetIntIntInt8Array() throws { + let m = try getTestMat(size: 5, type: CvType.CV_8SC3) var goodData = [Int8](repeating: 0, count: 9) // whole Mat var bytesNum = try m.get(row: 1, col: 1, data: &goodData) XCTAssertEqual(9, bytesNum) - XCTAssert([110, 111, 112, 120, 121, 122, -126, -125, -124] == goodData) + XCTAssert([110, 111, 112, 120, 121, 122, 127, 127, 127] == goodData) var badData = [Int8](repeating: 0, count: 7) XCTAssertThrowsError(bytesNum = try m.get(row: 0, col: 0, data: &badData)) @@ -326,11 +326,36 @@ class MatTests: OpenCVTestCase { var buff00 = [Int8](repeating: 0, count: 3) bytesNum = try sm.get(row: 0, col: 0, data: &buff00) XCTAssertEqual(3, bytesNum) - XCTAssert(buff00 == [-26, -25, -24]) + XCTAssert(buff00 == [127, 127, 127]) var buff11 = [Int8](repeating: 0, count: 3) bytesNum = try sm.get(row: 1, col: 1, data: &buff11) XCTAssertEqual(3, bytesNum) - XCTAssert(buff11 == [-1, -1, -1]) + XCTAssert(buff11 == [127, 127, 127]) + } + + func testGetIntIntUInt8Array() throws { + let m = try getTestMat(size: 5, type: CvType.CV_8UC3) + var goodData = [UInt8](repeating: 0, count: 9) + + // whole Mat + var bytesNum = try m.get(row: 1, col: 1, data: &goodData) + + XCTAssertEqual(9, bytesNum) + XCTAssert([110, 111, 112, 120, 121, 122, 130, 131, 132] == goodData) + + var badData = [UInt8](repeating: 0, count: 7) + XCTAssertThrowsError(bytesNum = try m.get(row: 0, col: 0, data: &badData)) + + // sub-Mat + let sm = m.submat(rowStart: 2, rowEnd: 4, colStart: 3, colEnd: 5) + var buff00 = [UInt8](repeating: 0, count: 3) + bytesNum = try sm.get(row: 0, col: 0, data: &buff00) + XCTAssertEqual(3, bytesNum) + XCTAssert(buff00 == [230, 231, 232]) + var buff11 = [UInt8](repeating: 0, count: 3) + bytesNum = try sm.get(row: 1, col: 1, data: &buff11) + XCTAssertEqual(3, bytesNum) + XCTAssert(buff11 == [255, 255, 255]) } func testGetIntIntDoubleArray() throws { @@ -399,7 +424,7 @@ class MatTests: OpenCVTestCase { XCTAssert(buff11 == [340, 341, 0, 0]) } - func testGetIntIntShortArray() throws { + func testGetIntIntInt16Array() throws { let m = try getTestMat(size: 5, type: CvType.CV_16SC2) var buff = [Int16](repeating: 0, count: 6) @@ -421,6 +446,28 @@ class MatTests: OpenCVTestCase { XCTAssert(buff11 == [340, 341, 0, 0]) } + func testGetIntIntUInt16Array() throws { + let m = try getTestMat(size: 5, type: CvType.CV_16UC2) + var buff = [UInt16](repeating: 0, count: 6) + + // whole Mat + var bytesNum = try m.get(row: 1, col: 1, data: &buff) + + XCTAssertEqual(12, bytesNum); + XCTAssert(buff == [110, 111, 120, 121, 130, 131]) + + // sub-Mat + let sm = m.submat(rowStart: 2, rowEnd: 4, colStart: 3, colEnd: 5) + var buff00 = [UInt16](repeating: 0, count: 4) + bytesNum = try sm.get(row: 0, col: 0, data: &buff00) + XCTAssertEqual(8, bytesNum) + XCTAssert(buff00 == [230, 231, 240, 241]) + var buff11 = [UInt16](repeating: 0, count: 4) + bytesNum = try sm.get(row: 1, col: 1, data: &buff11) + XCTAssertEqual(4, bytesNum); + XCTAssert(buff11 == [340, 341, 0, 0]) + } + func testHeight() { XCTAssertEqual(gray0.rows(), gray0.height()) XCTAssertEqual(rgbLena.rows(), rgbLena.height()) @@ -653,7 +700,7 @@ class MatTests: OpenCVTestCase { try assertMatEqual(truth!, m1, OpenCVTestCase.EPS) } - func testPutIntIntByteArray() throws { + func testPutIntIntInt8Array() throws { let m = Mat(rows: 5, cols: 5, type: CvType.CV_8SC3, scalar: Scalar(1, 2, 3)) let sm = m.submat(rowStart: 2, rowEnd: 4, colStart: 3, colEnd: 5) var buff = [Int8](repeating: 0, count: 6) @@ -683,7 +730,37 @@ class MatTests: OpenCVTestCase { XCTAssert(buff == buff0) } - func testPutIntArrayByteArray() throws { + func testPutIntIntUInt8Array() throws { + let m = Mat(rows: 5, cols: 5, type: CvType.CV_8UC3, scalar: Scalar(1, 2, 3)) + let sm = m.submat(rowStart: 2, rowEnd: 4, colStart: 3, colEnd: 5) + var buff = [UInt8](repeating: 0, count: 6) + let buff0:[UInt8] = [10, 20, 30, 40, 50, 60] + let buff1:[UInt8] = [255, 254, 253, 252, 251, 250] + + var bytesNum = try m.put(row:1, col:2, data:buff0) + + XCTAssertEqual(6, bytesNum) + bytesNum = try m.get(row: 1, col: 2, data: &buff) + XCTAssertEqual(6, bytesNum) + XCTAssert(buff == buff0) + + bytesNum = try sm.put(row:0, col:0, data:buff1) + + XCTAssertEqual(6, bytesNum) + bytesNum = try sm.get(row: 0, col: 0, data: &buff) + XCTAssertEqual(6, bytesNum) + XCTAssert(buff == buff1) + bytesNum = try m.get(row: 2, col: 3, data: &buff) + XCTAssertEqual(6, bytesNum); + XCTAssert(buff == buff1) + + let m1 = m.row(1) + bytesNum = try m1.get(row: 0, col: 2, data: &buff) + XCTAssertEqual(6, bytesNum) + XCTAssert(buff == buff0) + } + + func testPutIntArrayInt8Array() throws { let m = Mat(sizes: [5, 5, 5], type: CvType.CV_8SC3, scalar: Scalar(1, 2, 3)) let sm = m.submat(ranges: [Range(start: 0, end: 2), Range(start: 1, end: 3), Range(start: 2, end: 4)]) var buff = [Int8](repeating: 0, count: 6) @@ -714,10 +791,41 @@ class MatTests: OpenCVTestCase { XCTAssert(buff == buff0) } + func testPutIntArrayUInt8Array() throws { + let m = Mat(sizes: [5, 5, 5], type: CvType.CV_8UC3, scalar: Scalar(1, 2, 3)) + let sm = m.submat(ranges: [Range(start: 0, end: 2), Range(start: 1, end: 3), Range(start: 2, end: 4)]) + var buff = [UInt8](repeating: 0, count: 6) + let buff0:[UInt8] = [10, 20, 30, 40, 50, 60] + let buff1:[UInt8] = [255, 254, 253, 252, 251, 250] + + var bytesNum = try m.put(indices:[1, 2, 0], data:buff0) + + XCTAssertEqual(6, bytesNum) + bytesNum = try m.get(indices: [1, 2, 0], data: &buff) + XCTAssertEqual(6, bytesNum) + XCTAssert(buff == buff0) + + bytesNum = try sm.put(indices: [0, 0, 0], data: buff1) + + XCTAssertEqual(6, bytesNum) + bytesNum = try sm.get(indices: [0, 0, 0], data: &buff) + XCTAssertEqual(6, bytesNum) + XCTAssert(buff == buff1) + + bytesNum = try m.get(indices: [0, 1, 2], data: &buff) + XCTAssertEqual(6, bytesNum) + XCTAssert(buff == buff1) + + let m1 = m.submat(ranges: [Range(start: 1,end: 2), Range.all(), Range.all()]) + bytesNum = try m1.get(indices: [0, 2, 0], data: &buff) + XCTAssertEqual(6, bytesNum) + XCTAssert(buff == buff0) + } + func testPutIntIntDoubleArray() throws { - let m = Mat(rows: 5, cols: 5, type: CvType.CV_8SC3, scalar: Scalar(1, 2, 3)) + let m = Mat(rows: 5, cols: 5, type: CvType.CV_8UC3, scalar: Scalar(1, 2, 3)) let sm = m.submat(rowStart: 2, rowEnd: 4, colStart: 3, colEnd: 5) - var buff = [Int8](repeating: 0, count: 6) + var buff = [UInt8](repeating: 0, count: 6) var bytesNum = try m.put(row: 1, col: 2, data: [10, 20, 30, 40, 50, 60] as [Double]) @@ -731,16 +839,16 @@ class MatTests: OpenCVTestCase { XCTAssertEqual(6, bytesNum) bytesNum = try sm.get(row: 0, col: 0, data: &buff) XCTAssertEqual(6, bytesNum); - XCTAssert(buff == [-1, -2, -3, -4, -5, -6]) + XCTAssert(buff == [255, 254, 253, 252, 251, 250]) bytesNum = try m.get(row: 2, col: 3, data: &buff) XCTAssertEqual(6, bytesNum); - XCTAssert(buff == [-1, -2, -3, -4, -5, -6]) + XCTAssert(buff == [255, 254, 253, 252, 251, 250]) } func testPutIntArrayDoubleArray() throws { - let m = Mat(sizes: [5, 5, 5], type: CvType.CV_8SC3, scalar: Scalar(1, 2, 3)) + let m = Mat(sizes: [5, 5, 5], type: CvType.CV_8UC3, scalar: Scalar(1, 2, 3)) let sm = m.submat(ranges: [Range(start: 0, end: 2), Range(start: 1, end: 3), Range(start: 2, end: 4)]) - var buff = [Int8](repeating: 0, count: 6) + var buff = [UInt8](repeating: 0, count: 6) var bytesNum = try m.put(indices: [1, 2, 0], data: [10, 20, 30, 40, 50, 60] as [Double]) @@ -754,10 +862,10 @@ class MatTests: OpenCVTestCase { XCTAssertEqual(6, bytesNum); bytesNum = try sm.get(indices: [0, 0, 0], data: &buff) XCTAssertEqual(6, bytesNum); - XCTAssert(buff == [-1, -2, -3, -4, -5, -6]) + XCTAssert(buff == [255, 254, 253, 252, 251, 250]) bytesNum = try m.get(indices: [0, 1, 2], data: &buff) XCTAssertEqual(6, bytesNum) - XCTAssert(buff == [-1, -2, -3, -4, -5, -6]) + XCTAssert(buff == [255, 254, 253, 252, 251, 250]) } func testPutIntIntFloatArray() throws { @@ -820,7 +928,7 @@ class MatTests: OpenCVTestCase { XCTAssert([40, 50, 60] == m.get(indices: [0, 1, 0])) } - func testPutIntIntShortArray() throws { + func testPutIntIntInt16Array() throws { let m = Mat(rows: 5, cols: 5, type: CvType.CV_16SC3, scalar: Scalar(-1, -2, -3)) let elements: [Int16] = [ 10, 20, 30, 40, 50, 60] @@ -834,7 +942,21 @@ class MatTests: OpenCVTestCase { XCTAssert([40, 50, 60] == m.get(row: 2, col: 4)) } - func testPutIntArrayShortArray() throws { + func testPutIntIntUInt16Array() throws { + let m = Mat(rows: 5, cols: 5, type: CvType.CV_16UC3, scalar: Scalar(-1, -2, -3)) + let elements: [UInt16] = [ 10, 20, 30, 40, 50, 60] + + var bytesNum = try m.put(row: 2, col: 3, data: elements) + + XCTAssertEqual(Int32(elements.count * 2), bytesNum) + let m1 = m.col(3) + var buff = [UInt16](repeating: 0, count: 3) + bytesNum = try m1.get(row: 2, col: 0, data: &buff) + XCTAssert(buff == [10, 20, 30]) + XCTAssert([40, 50, 60] == m.get(row: 2, col: 4)) + } + + func testPutIntArrayInt16Array() throws { let m = Mat(sizes: [5, 5, 5], type: CvType.CV_16SC3, scalar: Scalar(-1, -2, -3)) let elements: [Int16] = [ 10, 20, 30, 40, 50, 60] @@ -848,6 +970,20 @@ class MatTests: OpenCVTestCase { XCTAssert([40, 50, 60] == m.get(indices: [0, 2, 4])) } + func testPutIntArrayUInt16Array() throws { + let m = Mat(sizes: [5, 5, 5], type: CvType.CV_16UC3, scalar: Scalar(-1, -2, -3)) + let elements: [UInt16] = [ 10, 20, 30, 40, 50, 60] + + var bytesNum = try m.put(indices: [0, 2, 3], data: elements) + + XCTAssertEqual(Int32(elements.count * 2), bytesNum) + let m1 = m.submat(ranges: [Range.all(), Range.all(), Range(start: 3, end: 4)]) + var buff = [UInt16](repeating: 0, count: 3) + bytesNum = try m1.get(indices: [0, 2, 0], data: &buff) + XCTAssert(buff == [10, 20, 30]) + XCTAssert([40, 50, 60] == m.get(indices: [0, 2, 4])) + } + func testReshapeInt() throws { let src = Mat(rows: 4, cols: 4, type: CvType.CV_8U, scalar: Scalar(0)) dst = src.reshape(channels: 4) From 4eac198270783d8924ed26ecfb82f8aa54d9e67d Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Tue, 29 Jun 2021 09:00:10 +0000 Subject: [PATCH 022/262] core(persistence): fix types format handling, fix 16F support --- modules/core/src/persistence.cpp | 19 +++++++--- modules/core/test/test_io.cpp | 65 ++++++++++++++++++++++++++++++++ 2 files changed, 78 insertions(+), 6 deletions(-) diff --git a/modules/core/src/persistence.cpp b/modules/core/src/persistence.cpp index 4bf52a3134..32328361e8 100644 --- a/modules/core/src/persistence.cpp +++ b/modules/core/src/persistence.cpp @@ -143,17 +143,17 @@ static const char symbols[9] = "ucwsifdh"; static char typeSymbol(int depth) { CV_StaticAssert(CV_64F == 6, ""); - CV_Assert(depth >=0 && depth <= CV_64F); + CV_CheckDepth(depth, depth >=0 && depth <= CV_16F, ""); return symbols[depth]; } static int symbolToType(char c) { + if (c == 'r') + return CV_SEQ_ELTYPE_PTR; const char* pos = strchr( symbols, c ); if( !pos ) CV_Error( CV_StsBadArg, "Invalid data type specification" ); - if (c == 'r') - return CV_SEQ_ELTYPE_PTR; return static_cast(pos - symbols); } @@ -245,8 +245,12 @@ int calcStructSize( const char* dt, int initial_size ) { int size = calcElemSize( dt, initial_size ); size_t elem_max_size = 0; - for ( const char * type = dt; *type != '\0'; type++ ) { - switch ( *type ) + for ( const char * type = dt; *type != '\0'; type++ ) + { + char v = *type; + if (v >= '0' && v <= '9') + continue; // skip vector size + switch (v) { case 'u': { elem_max_size = std::max( elem_max_size, sizeof(uchar ) ); break; } case 'c': { elem_max_size = std::max( elem_max_size, sizeof(schar ) ); break; } @@ -255,7 +259,9 @@ int calcStructSize( const char* dt, int initial_size ) case 'i': { elem_max_size = std::max( elem_max_size, sizeof(int ) ); break; } case 'f': { elem_max_size = std::max( elem_max_size, sizeof(float ) ); break; } case 'd': { elem_max_size = std::max( elem_max_size, sizeof(double) ); break; } - default: break; + case 'h': { elem_max_size = std::max(elem_max_size, sizeof(float16_t)); break; } + default: + CV_Error_(Error::StsNotImplemented, ("Unknown type identifier: '%c' in '%s'", (char)(*type), dt)); } } size = cvAlign( size, static_cast(elem_max_size) ); @@ -1054,6 +1060,7 @@ public: CV_Assert(write_mode); size_t elemSize = fs::calcStructSize(dt.c_str(), 0); + CV_Assert(elemSize); CV_Assert( len % elemSize == 0 ); len /= elemSize; diff --git a/modules/core/test/test_io.cpp b/modules/core/test/test_io.cpp index d30c485368..82bd05372d 100644 --- a/modules/core/test/test_io.cpp +++ b/modules/core/test/test_io.cpp @@ -1837,4 +1837,69 @@ TEST(Core_InputOutput, FileStorage_copy_constructor_17412_heap) EXPECT_EQ(0, remove(fname.c_str())); } + +static void test_20279(FileStorage& fs) +{ + Mat m32fc1(5, 10, CV_32FC1, Scalar::all(0)); + for (size_t i = 0; i < m32fc1.total(); i++) + { + float v = (float)i; + m32fc1.at((int)i) = v * 0.5f; + } + Mat m16fc1; + // produces CV_16S output: convertFp16(m32fc1, m16fc1); + m32fc1.convertTo(m16fc1, CV_16FC1); + EXPECT_EQ(CV_16FC1, m16fc1.type()) << typeToString(m16fc1.type()); + //std::cout << m16fc1 << std::endl; + + Mat m32fc3(4, 3, CV_32FC3, Scalar::all(0)); + for (size_t i = 0; i < m32fc3.total(); i++) + { + float v = (float)i; + m32fc3.at((int)i) = Vec3f(v, v * 0.2f, -v); + } + Mat m16fc3; + m32fc3.convertTo(m16fc3, CV_16FC3); + EXPECT_EQ(CV_16FC3, m16fc3.type()) << typeToString(m16fc3.type()); + //std::cout << m16fc3 << std::endl; + + fs << "m16fc1" << m16fc1; + fs << "m16fc3" << m16fc3; + + string content = fs.releaseAndGetString(); + if (cvtest::debugLevel > 0) std::cout << content << std::endl; + + FileStorage fs_read(content, FileStorage::READ + FileStorage::MEMORY); + Mat m16fc1_result; + Mat m16fc3_result; + fs_read["m16fc1"] >> m16fc1_result; + ASSERT_FALSE(m16fc1_result.empty()); + EXPECT_EQ(CV_16FC1, m16fc1_result.type()) << typeToString(m16fc1_result.type()); + EXPECT_LE(cvtest::norm(m16fc1_result, m16fc1, NORM_INF), 1e-2); + + fs_read["m16fc3"] >> m16fc3_result; + ASSERT_FALSE(m16fc3_result.empty()); + EXPECT_EQ(CV_16FC3, m16fc3_result.type()) << typeToString(m16fc3_result.type()); + EXPECT_LE(cvtest::norm(m16fc3_result, m16fc3, NORM_INF), 1e-2); +} + +TEST(Core_InputOutput, FileStorage_16F_xml) +{ + FileStorage fs("test.xml", cv::FileStorage::WRITE | cv::FileStorage::MEMORY); + test_20279(fs); +} + +TEST(Core_InputOutput, FileStorage_16F_yml) +{ + FileStorage fs("test.yml", cv::FileStorage::WRITE | cv::FileStorage::MEMORY); + test_20279(fs); +} + +TEST(Core_InputOutput, FileStorage_16F_json) +{ + FileStorage fs("test.json", cv::FileStorage::WRITE | cv::FileStorage::MEMORY); + test_20279(fs); +} + + }} // namespace From 7d842f5bcffc54e25b365935a79b99b96c49e01d Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Tue, 29 Jun 2021 18:48:21 +0000 Subject: [PATCH 023/262] dnn: use OpenVINO 2021.4 defines --- cmake/OpenCVDetectInferenceEngine.cmake | 4 ++-- modules/dnn/src/op_inf_engine.hpp | 5 +++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/cmake/OpenCVDetectInferenceEngine.cmake b/cmake/OpenCVDetectInferenceEngine.cmake index aa4bb4a864..829ddbfe7e 100644 --- a/cmake/OpenCVDetectInferenceEngine.cmake +++ b/cmake/OpenCVDetectInferenceEngine.cmake @@ -141,9 +141,9 @@ endif() if(INF_ENGINE_TARGET) if(NOT INF_ENGINE_RELEASE) - message(WARNING "InferenceEngine version has not been set, 2021.3 will be used by default. Set INF_ENGINE_RELEASE variable if you experience build errors.") + message(WARNING "InferenceEngine version has not been set, 2021.4 will be used by default. Set INF_ENGINE_RELEASE variable if you experience build errors.") endif() - set(INF_ENGINE_RELEASE "2021030000" CACHE STRING "Force IE version, should be in form YYYYAABBCC (e.g. 2020.1.0.2 -> 2020010002)") + set(INF_ENGINE_RELEASE "2021040000" CACHE STRING "Force IE version, should be in form YYYYAABBCC (e.g. 2020.1.0.2 -> 2020010002)") set_target_properties(${INF_ENGINE_TARGET} PROPERTIES INTERFACE_COMPILE_DEFINITIONS "HAVE_INF_ENGINE=1;INF_ENGINE_RELEASE=${INF_ENGINE_RELEASE}" ) diff --git a/modules/dnn/src/op_inf_engine.hpp b/modules/dnn/src/op_inf_engine.hpp index 42008b0f10..a825431627 100644 --- a/modules/dnn/src/op_inf_engine.hpp +++ b/modules/dnn/src/op_inf_engine.hpp @@ -30,10 +30,11 @@ #define INF_ENGINE_RELEASE_2021_1 2021010000 #define INF_ENGINE_RELEASE_2021_2 2021020000 #define INF_ENGINE_RELEASE_2021_3 2021030000 +#define INF_ENGINE_RELEASE_2021_4 2021040000 #ifndef INF_ENGINE_RELEASE -#warning("IE version have not been provided via command-line. Using 2021.3 by default") -#define INF_ENGINE_RELEASE INF_ENGINE_RELEASE_2021_3 +#warning("IE version have not been provided via command-line. Using 2021.4 by default") +#define INF_ENGINE_RELEASE INF_ENGINE_RELEASE_2021_4 #endif #define INF_ENGINE_VER_MAJOR_GT(ver) (((INF_ENGINE_RELEASE) / 10000) > ((ver) / 10000)) From db4b1e613ccdb449fad83c0711a3d44cb21662bc Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Tue, 29 Jun 2021 09:00:10 +0000 Subject: [PATCH 024/262] core(persistence): fix types format handling partial backport of 4eac198270783d8924ed26ecfb82f8aa54d9e67d --- modules/core/src/persistence.cpp | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/modules/core/src/persistence.cpp b/modules/core/src/persistence.cpp index 40d1cdfa07..7e9d107c35 100644 --- a/modules/core/src/persistence.cpp +++ b/modules/core/src/persistence.cpp @@ -6,6 +6,8 @@ #include "precomp.hpp" #include "persistence.hpp" +using namespace cv; + char* icv_itoa( int _val, char* buffer, int /*radix*/ ) { const int radix = 10; @@ -519,12 +521,16 @@ static const char symbols[9] = "ucwsifdr"; char icvTypeSymbol(int depth) { - CV_Assert(depth >=0 && depth < 9); + CV_StaticAssert(CV_64F == 6, ""); + CV_Assert(depth >=0 && depth <= CV_64F); + CV_CheckDepth(depth, depth >=0 && depth <= CV_64F, ""); return symbols[depth]; } static int icvSymbolToType(char c) { + if (c == 'r') + return CV_SEQ_ELTYPE_PTR; const char* pos = strchr( symbols, c ); if( !pos ) CV_Error( CV_StsBadArg, "Invalid data type specification" ); @@ -618,8 +624,12 @@ int icvCalcStructSize( const char* dt, int initial_size ) { int size = icvCalcElemSize( dt, initial_size ); size_t elem_max_size = 0; - for ( const char * type = dt; *type != '\0'; type++ ) { - switch ( *type ) + for ( const char * type = dt; *type != '\0'; type++ ) + { + char v = *type; + if (v >= '0' && v <= '9') + continue; // skip vector size + switch (v) { case 'u': { elem_max_size = std::max( elem_max_size, sizeof(uchar ) ); break; } case 'c': { elem_max_size = std::max( elem_max_size, sizeof(schar ) ); break; } @@ -628,7 +638,8 @@ int icvCalcStructSize( const char* dt, int initial_size ) case 'i': { elem_max_size = std::max( elem_max_size, sizeof(int ) ); break; } case 'f': { elem_max_size = std::max( elem_max_size, sizeof(float ) ); break; } case 'd': { elem_max_size = std::max( elem_max_size, sizeof(double) ); break; } - default: break; + default: + CV_Error_(Error::StsNotImplemented, ("Unknown type identifier: '%c' in '%s'", (char)(*type), dt)); } } size = cvAlign( size, static_cast(elem_max_size) ); From fb7ef76e742f408cc4e23ba7b67c99db06d0140d Mon Sep 17 00:00:00 2001 From: Anatoliy Talamanov Date: Wed, 30 Jun 2021 12:04:09 +0300 Subject: [PATCH 025/262] Merge pull request #20271 from TolyaTalamanov:at/extend-python-bindings G-API: Extend python bindings * Extend G-API bindings * Wrap timestamp, seqNo, seq_id * Wrap copy * Wrap parseSSD, parseYolo * Rewrap cv.gapi.networks * Add test for metabackend in pytnon * Remove int64 pyopencv_to --- modules/gapi/include/opencv2/gapi/gcommon.hpp | 2 + .../gapi/include/opencv2/gapi/gstreaming.hpp | 2 +- modules/gapi/include/opencv2/gapi/infer.hpp | 6 +- .../include/opencv2/gapi/infer/parsers.hpp | 20 +- .../include/opencv2/gapi/streaming/format.hpp | 2 +- .../gapi/misc/python/package/gapi/__init__.py | 25 ++ modules/gapi/misc/python/pyopencv_gapi.hpp | 230 ++++++++++-------- modules/gapi/misc/python/python_bridge.hpp | 3 + modules/gapi/misc/python/shadow_gapi.hpp | 29 +-- .../misc/python/test/test_gapi_streaming.py | 42 +++- modules/gapi/src/api/ginfer.cpp | 4 + .../gapi/src/backends/common/gmetabackend.cpp | 13 + modules/python/src2/cv2.cpp | 6 - 13 files changed, 240 insertions(+), 144 deletions(-) diff --git a/modules/gapi/include/opencv2/gapi/gcommon.hpp b/modules/gapi/include/opencv2/gapi/gcommon.hpp index a9cb015901..d3c280816f 100644 --- a/modules/gapi/include/opencv2/gapi/gcommon.hpp +++ b/modules/gapi/include/opencv2/gapi/gcommon.hpp @@ -44,6 +44,7 @@ namespace detail CV_UNKNOWN, // Unknown, generic, opaque-to-GAPI data type unsupported in graph seriallization CV_BOOL, // bool user G-API data CV_INT, // int user G-API data + CV_INT64, // int64_t user G-API data CV_DOUBLE, // double user G-API data CV_FLOAT, // float user G-API data CV_UINT64, // uint64_t user G-API data @@ -61,6 +62,7 @@ namespace detail template struct GOpaqueTraits; template struct GOpaqueTraits { static constexpr const OpaqueKind kind = OpaqueKind::CV_UNKNOWN; }; template<> struct GOpaqueTraits { static constexpr const OpaqueKind kind = OpaqueKind::CV_INT; }; + template<> struct GOpaqueTraits { static constexpr const OpaqueKind kind = OpaqueKind::CV_INT64; }; template<> struct GOpaqueTraits { static constexpr const OpaqueKind kind = OpaqueKind::CV_DOUBLE; }; template<> struct GOpaqueTraits { static constexpr const OpaqueKind kind = OpaqueKind::CV_FLOAT; }; template<> struct GOpaqueTraits { static constexpr const OpaqueKind kind = OpaqueKind::CV_UINT64; }; diff --git a/modules/gapi/include/opencv2/gapi/gstreaming.hpp b/modules/gapi/include/opencv2/gapi/gstreaming.hpp index 5bbed5e12d..47e103fd0e 100644 --- a/modules/gapi/include/opencv2/gapi/gstreaming.hpp +++ b/modules/gapi/include/opencv2/gapi/gstreaming.hpp @@ -196,7 +196,7 @@ public: * @param s a shared pointer to IStreamSource representing the * input video stream. */ - GAPI_WRAP void setSource(const gapi::wip::IStreamSource::Ptr& s); + void setSource(const gapi::wip::IStreamSource::Ptr& s); /** * @brief Constructs and specifies an input video stream for a diff --git a/modules/gapi/include/opencv2/gapi/infer.hpp b/modules/gapi/include/opencv2/gapi/infer.hpp index 93701856bb..807c82d31f 100644 --- a/modules/gapi/include/opencv2/gapi/infer.hpp +++ b/modules/gapi/include/opencv2/gapi/infer.hpp @@ -136,11 +136,12 @@ public: } template - void setInput(const std::string& name, U in) + GInferInputsTyped& setInput(const std::string& name, U in) { m_priv->blobs.emplace(std::piecewise_construct, std::forward_as_tuple(name), std::forward_as_tuple(in)); + return *this; } using StorageT = cv::util::variant; @@ -654,7 +655,7 @@ namespace gapi { // A type-erased form of network parameters. // Similar to how a type-erased GKernel is represented and used. /// @private -struct GAPI_EXPORTS GNetParam { +struct GAPI_EXPORTS_W_SIMPLE GNetParam { std::string tag; // FIXME: const? GBackend backend; // Specifies the execution model util::any params; // Backend-interpreted parameter structure @@ -671,6 +672,7 @@ struct GAPI_EXPORTS GNetParam { */ struct GAPI_EXPORTS_W_SIMPLE GNetPackage { GAPI_WRAP GNetPackage() = default; + GAPI_WRAP explicit GNetPackage(std::vector nets); explicit GNetPackage(std::initializer_list ii); std::vector backends() const; std::vector networks; diff --git a/modules/gapi/include/opencv2/gapi/infer/parsers.hpp b/modules/gapi/include/opencv2/gapi/infer/parsers.hpp index 22c8701a6c..c7308dd39f 100644 --- a/modules/gapi/include/opencv2/gapi/infer/parsers.hpp +++ b/modules/gapi/include/opencv2/gapi/infer/parsers.hpp @@ -64,10 +64,10 @@ detection is smaller than confidence threshold, detection is rejected. given label will get to the output. @return a tuple with a vector of detected boxes and a vector of appropriate labels. */ -GAPI_EXPORTS std::tuple, GArray> parseSSD(const GMat& in, - const GOpaque& inSz, - const float confidenceThreshold = 0.5f, - const int filterLabel = -1); +GAPI_EXPORTS_W std::tuple, GArray> parseSSD(const GMat& in, + const GOpaque& inSz, + const float confidenceThreshold = 0.5f, + const int filterLabel = -1); /** @brief Parses output of SSD network. @@ -113,12 +113,12 @@ If 1.f, nms is not performed and no boxes are rejected. documentation. @return a tuple with a vector of detected boxes and a vector of appropriate labels. */ -GAPI_EXPORTS std::tuple, GArray> parseYolo(const GMat& in, - const GOpaque& inSz, - const float confidenceThreshold = 0.5f, - const float nmsThreshold = 0.5f, - const std::vector& anchors - = nn::parsers::GParseYolo::defaultAnchors()); +GAPI_EXPORTS_W std::tuple, GArray> parseYolo(const GMat& in, + const GOpaque& inSz, + const float confidenceThreshold = 0.5f, + const float nmsThreshold = 0.5f, + const std::vector& anchors + = nn::parsers::GParseYolo::defaultAnchors()); } // namespace gapi } // namespace cv diff --git a/modules/gapi/include/opencv2/gapi/streaming/format.hpp b/modules/gapi/include/opencv2/gapi/streaming/format.hpp index c9d2fa3e0a..f7c3bd457d 100644 --- a/modules/gapi/include/opencv2/gapi/streaming/format.hpp +++ b/modules/gapi/include/opencv2/gapi/streaming/format.hpp @@ -74,7 +74,7 @@ e.g when graph's input needs to be passed directly to output, like in Streaming @param in Input image @return Copy of the input */ -GAPI_EXPORTS GMat copy(const GMat& in); +GAPI_EXPORTS_W GMat copy(const GMat& in); /** @brief Makes a copy of the input frame. Note that this copy may be not real (no actual data copied). Use this function to maintain graph contracts, diff --git a/modules/gapi/misc/python/package/gapi/__init__.py b/modules/gapi/misc/python/package/gapi/__init__.py index 23f5f41846..587f641fd3 100644 --- a/modules/gapi/misc/python/package/gapi/__init__.py +++ b/modules/gapi/misc/python/package/gapi/__init__.py @@ -11,11 +11,36 @@ def register(mname): return parameterized +@register('cv2.gapi') +def networks(*args): + return cv.gapi_GNetPackage(list(map(cv.detail.strip, args))) + + @register('cv2.gapi') def compile_args(*args): return list(map(cv.GCompileArg, args)) +@register('cv2') +def GIn(*args): + return [*args] + + +@register('cv2') +def GOut(*args): + return [*args] + + +@register('cv2') +def gin(*args): + return [*args] + + +@register('cv2.gapi') +def descr_of(*args): + return [*args] + + @register('cv2') class GOpaque(): # NB: Inheritance from c++ class cause segfault. diff --git a/modules/gapi/misc/python/pyopencv_gapi.hpp b/modules/gapi/misc/python/pyopencv_gapi.hpp index 6b782cfc8d..6cd79e4a73 100644 --- a/modules/gapi/misc/python/pyopencv_gapi.hpp +++ b/modules/gapi/misc/python/pyopencv_gapi.hpp @@ -17,6 +17,7 @@ using gapi_ie_PyParams = cv::gapi::ie::PyParams; using gapi_wip_IStreamSource_Ptr = cv::Ptr; using detail_ExtractArgsCallback = cv::detail::ExtractArgsCallback; using detail_ExtractMetaCallback = cv::detail::ExtractMetaCallback; +using vector_GNetParam = std::vector; // NB: Python wrapper generate T_U for T // This behavior is only observed for inputs @@ -138,6 +139,7 @@ PyObject* pyopencv_from(const cv::GArg& value) { HANDLE_CASE(BOOL, bool); HANDLE_CASE(INT, int); + HANDLE_CASE(INT64, int64_t); HANDLE_CASE(DOUBLE, double); HANDLE_CASE(FLOAT, float); HANDLE_CASE(STRING, std::string); @@ -163,6 +165,18 @@ bool pyopencv_to(PyObject* obj, cv::GArg& value, const ArgInfo& info) return true; } +template <> +bool pyopencv_to(PyObject* obj, std::vector& value, const ArgInfo& info) +{ + return pyopencv_to_generic_vec(obj, value, info); +} + +template <> +PyObject* pyopencv_from(const std::vector& value) +{ + return pyopencv_from_generic_vec(value); +} + template <> bool pyopencv_to(PyObject* obj, std::vector& value, const ArgInfo& info) { @@ -175,12 +189,6 @@ PyObject* pyopencv_from(const std::vector& value) return pyopencv_from_generic_vec(value); } -template <> -bool pyopencv_to(PyObject* obj, GRunArgs& value, const ArgInfo& info) -{ - return pyopencv_to_generic_vec(obj, value, info); -} - template<> PyObject* pyopencv_from(const cv::detail::OpaqueRef& o) { @@ -188,6 +196,7 @@ PyObject* pyopencv_from(const cv::detail::OpaqueRef& o) { case cv::detail::OpaqueKind::CV_BOOL : return pyopencv_from(o.rref()); case cv::detail::OpaqueKind::CV_INT : return pyopencv_from(o.rref()); + case cv::detail::OpaqueKind::CV_INT64 : return pyopencv_from(o.rref()); case cv::detail::OpaqueKind::CV_DOUBLE : return pyopencv_from(o.rref()); case cv::detail::OpaqueKind::CV_FLOAT : return pyopencv_from(o.rref()); case cv::detail::OpaqueKind::CV_STRING : return pyopencv_from(o.rref()); @@ -213,6 +222,7 @@ PyObject* pyopencv_from(const cv::detail::VectorRef& v) { case cv::detail::OpaqueKind::CV_BOOL : return pyopencv_from_generic_vec(v.rref()); case cv::detail::OpaqueKind::CV_INT : return pyopencv_from_generic_vec(v.rref()); + case cv::detail::OpaqueKind::CV_INT64 : return pyopencv_from_generic_vec(v.rref()); case cv::detail::OpaqueKind::CV_DOUBLE : return pyopencv_from_generic_vec(v.rref()); case cv::detail::OpaqueKind::CV_FLOAT : return pyopencv_from_generic_vec(v.rref()); case cv::detail::OpaqueKind::CV_STRING : return pyopencv_from_generic_vec(v.rref()); @@ -285,18 +295,6 @@ PyObject* pyopencv_from(const GRunArgs& value) return list; } -template<> -bool pyopencv_to(PyObject* obj, GMetaArgs& value, const ArgInfo& info) -{ - return pyopencv_to_generic_vec(obj, value, info); -} - -template<> -PyObject* pyopencv_from(const GMetaArgs& value) -{ - return pyopencv_from_generic_vec(value); -} - template void pyopencv_to_with_check(PyObject* from, T& to, const std::string& msg = "") { @@ -318,16 +316,16 @@ void pyopencv_to_generic_vec_with_check(PyObject* from, } template -static PyObject* extract_proto_args(PyObject* py_args, PyObject* kw) +static T extract_proto_args(PyObject* py_args) { using namespace cv; GProtoArgs args; - Py_ssize_t size = PyTuple_Size(py_args); + Py_ssize_t size = PyList_Size(py_args); args.reserve(size); for (int i = 0; i < size; ++i) { - PyObject* item = PyTuple_GetItem(py_args, i); + PyObject* item = PyList_GetItem(py_args, i); if (PyObject_TypeCheck(item, reinterpret_cast(pyopencv_GScalar_TypePtr))) { args.emplace_back(reinterpret_cast(item)->v); @@ -346,22 +344,11 @@ static PyObject* extract_proto_args(PyObject* py_args, PyObject* kw) } else { - PyErr_SetString(PyExc_TypeError, "Unsupported type for cv.GIn()/cv.GOut()"); - return NULL; + util::throw_error(std::logic_error("Unsupported type for GProtoArgs")); } } - return pyopencv_from(T{std::move(args)}); -} - -static PyObject* pyopencv_cv_GIn(PyObject* , PyObject* py_args, PyObject* kw) -{ - return extract_proto_args(py_args, kw); -} - -static PyObject* pyopencv_cv_GOut(PyObject* , PyObject* py_args, PyObject* kw) -{ - return extract_proto_args(py_args, kw); + return T(std::move(args)); } static cv::detail::OpaqueRef extract_opaque_ref(PyObject* from, cv::detail::OpaqueKind kind) @@ -386,6 +373,7 @@ static cv::detail::OpaqueRef extract_opaque_ref(PyObject* from, cv::detail::Opaq HANDLE_CASE(RECT, cv::Rect); HANDLE_CASE(UNKNOWN, cv::GArg); UNSUPPORTED(UINT64); + UNSUPPORTED(INT64); UNSUPPORTED(SCALAR); UNSUPPORTED(MAT); UNSUPPORTED(DRAW_PRIM); @@ -419,6 +407,7 @@ static cv::detail::VectorRef extract_vector_ref(PyObject* from, cv::detail::Opaq HANDLE_CASE(MAT, cv::Mat); HANDLE_CASE(UNKNOWN, cv::GArg); UNSUPPORTED(UINT64); + UNSUPPORTED(INT64); UNSUPPORTED(DRAW_PRIM); #undef HANDLE_CASE #undef UNSUPPORTED @@ -470,13 +459,15 @@ static cv::GRunArg extract_run_arg(const cv::GTypeInfo& info, PyObject* item) static cv::GRunArgs extract_run_args(const cv::GTypesInfo& info, PyObject* py_args) { - cv::GRunArgs args; - Py_ssize_t tuple_size = PyTuple_Size(py_args); - args.reserve(tuple_size); + GAPI_Assert(PyList_Check(py_args)); - for (int i = 0; i < tuple_size; ++i) + cv::GRunArgs args; + Py_ssize_t list_size = PyList_Size(py_args); + args.reserve(list_size); + + for (int i = 0; i < list_size; ++i) { - args.push_back(extract_run_arg(info[i], PyTuple_GetItem(py_args, i))); + args.push_back(extract_run_arg(info[i], PyList_GetItem(py_args, i))); } return args; @@ -517,13 +508,15 @@ static cv::GMetaArg extract_meta_arg(const cv::GTypeInfo& info, PyObject* item) static cv::GMetaArgs extract_meta_args(const cv::GTypesInfo& info, PyObject* py_args) { - cv::GMetaArgs metas; - Py_ssize_t tuple_size = PyTuple_Size(py_args); - metas.reserve(tuple_size); + GAPI_Assert(PyList_Check(py_args)); - for (int i = 0; i < tuple_size; ++i) + cv::GMetaArgs metas; + Py_ssize_t list_size = PyList_Size(py_args); + metas.reserve(list_size); + + for (int i = 0; i < list_size; ++i) { - metas.push_back(extract_meta_arg(info[i], PyTuple_GetItem(py_args, i))); + metas.push_back(extract_meta_arg(info[i], PyList_GetItem(py_args, i))); } return metas; @@ -589,8 +582,27 @@ static cv::GRunArgs run_py_kernel(cv::detail::PyObjectHolder kernel, // NB: In fact it's impossible situation, becase errors were handled above. GAPI_Assert(result.get() && "Python kernel returned NULL!"); - outs = out_info.size() == 1 ? cv::GRunArgs{extract_run_arg(out_info[0], result.get())} - : extract_run_args(out_info, result.get()); + if (out_info.size() == 1) + { + outs = cv::GRunArgs{extract_run_arg(out_info[0], result.get())}; + } + else if (out_info.size() > 1) + { + GAPI_Assert(PyTuple_Check(result.get())); + + Py_ssize_t tuple_size = PyTuple_Size(result.get()); + outs.reserve(tuple_size); + + for (int i = 0; i < tuple_size; ++i) + { + outs.push_back(extract_run_arg(out_info[i], PyTuple_GetItem(result.get(), i))); + } + } + else + { + // Seems to be impossible case. + GAPI_Assert(false); + } } catch (...) { @@ -756,23 +768,6 @@ static PyObject* pyopencv_cv_gapi_kernels(PyObject* , PyObject* py_args, PyObjec return pyopencv_from(pkg); } -static PyObject* pyopencv_cv_gapi_networks(PyObject*, PyObject* py_args, PyObject*) -{ - using namespace cv; - gapi::GNetPackage pkg; - Py_ssize_t size = PyTuple_Size(py_args); - for (int i = 0; i < size; ++i) - { - gapi_ie_PyParams params; - PyObject* item = PyTuple_GetItem(py_args, i); - if (pyopencv_to(item, params, ArgInfo("PyParams", false))) - { - pkg += gapi::networks(params); - } - } - return pyopencv_from(pkg); -} - static PyObject* pyopencv_cv_gapi_op(PyObject* , PyObject* py_args, PyObject*) { using namespace cv; @@ -834,53 +829,54 @@ static PyObject* pyopencv_cv_gapi_op(PyObject* , PyObject* py_args, PyObject*) return pyopencv_from(cv::gapi::wip::op(id, outMetaWrapper, std::move(args))); } -static PyObject* pyopencv_cv_gin(PyObject*, PyObject* py_args, PyObject*) +template<> +bool pyopencv_to(PyObject* obj, cv::detail::ExtractArgsCallback& value, const ArgInfo&) { - cv::detail::PyObjectHolder holder{py_args}; - auto callback = cv::detail::ExtractArgsCallback{[=](const cv::GTypesInfo& info) + cv::detail::PyObjectHolder holder{obj}; + value = cv::detail::ExtractArgsCallback{[=](const cv::GTypesInfo& info) + { + PyGILState_STATE gstate; + gstate = PyGILState_Ensure(); + + cv::GRunArgs args; + try + { + args = extract_run_args(info, holder.get()); + } + catch (...) { - PyGILState_STATE gstate; - gstate = PyGILState_Ensure(); - - cv::GRunArgs args; - try - { - args = extract_run_args(info, holder.get()); - } - catch (...) - { - PyGILState_Release(gstate); - throw; - } PyGILState_Release(gstate); - return args; - }}; - - return pyopencv_from(callback); + throw; + } + PyGILState_Release(gstate); + return args; + }}; + return true; } -static PyObject* pyopencv_cv_descr_of(PyObject*, PyObject* py_args, PyObject*) +template<> +bool pyopencv_to(PyObject* obj, cv::detail::ExtractMetaCallback& value, const ArgInfo&) { - Py_INCREF(py_args); - auto callback = cv::detail::ExtractMetaCallback{[=](const cv::GTypesInfo& info) - { - PyGILState_STATE gstate; - gstate = PyGILState_Ensure(); + cv::detail::PyObjectHolder holder{obj}; + value = cv::detail::ExtractMetaCallback{[=](const cv::GTypesInfo& info) + { + PyGILState_STATE gstate; + gstate = PyGILState_Ensure(); - cv::GMetaArgs args; - try - { - args = extract_meta_args(info, py_args); - } - catch (...) - { - PyGILState_Release(gstate); - throw; - } + cv::GMetaArgs args; + try + { + args = extract_meta_args(info, holder.get()); + } + catch (...) + { PyGILState_Release(gstate); - return args; - }}; - return pyopencv_from(callback); + throw; + } + PyGILState_Release(gstate); + return args; + }}; + return true; } template @@ -929,11 +925,39 @@ struct PyOpenCV_Converter> } }; +template<> +bool pyopencv_to(PyObject* obj, cv::GProtoInputArgs& value, const ArgInfo& info) +{ + try + { + value = extract_proto_args(obj); + return true; + } + catch (...) + { + failmsg("Can't parse cv::GProtoInputArgs"); + return false; + } +} + +template<> +bool pyopencv_to(PyObject* obj, cv::GProtoOutputArgs& value, const ArgInfo& info) +{ + try + { + value = extract_proto_args(obj); + return true; + } + catch (...) + { + failmsg("Can't parse cv::GProtoOutputArgs"); + return false; + } +} // extend cv.gapi methods #define PYOPENCV_EXTRA_METHODS_GAPI \ {"kernels", CV_PY_FN_WITH_KW(pyopencv_cv_gapi_kernels), "kernels(...) -> GKernelPackage"}, \ - {"networks", CV_PY_FN_WITH_KW(pyopencv_cv_gapi_networks), "networks(...) -> GNetPackage"}, \ {"__op", CV_PY_FN_WITH_KW(pyopencv_cv_gapi_op), "__op(...) -> retval\n"}, diff --git a/modules/gapi/misc/python/python_bridge.hpp b/modules/gapi/misc/python/python_bridge.hpp index 0d1c6d51c5..b212babe45 100644 --- a/modules/gapi/misc/python/python_bridge.hpp +++ b/modules/gapi/misc/python/python_bridge.hpp @@ -27,6 +27,7 @@ #define GARRAY_TYPE_LIST_G(G, G2) \ WRAP_ARGS(bool , cv::gapi::ArgType::CV_BOOL, G) \ WRAP_ARGS(int , cv::gapi::ArgType::CV_INT, G) \ +WRAP_ARGS(int64_t , cv::gapi::ArgType::CV_INT64, G) \ WRAP_ARGS(double , cv::gapi::ArgType::CV_DOUBLE, G) \ WRAP_ARGS(float , cv::gapi::ArgType::CV_FLOAT, G) \ WRAP_ARGS(std::string , cv::gapi::ArgType::CV_STRING, G) \ @@ -42,6 +43,7 @@ WRAP_ARGS(cv::GMat , cv::gapi::ArgType::CV_GMAT, G2) \ #define GOPAQUE_TYPE_LIST_G(G, G2) \ WRAP_ARGS(bool , cv::gapi::ArgType::CV_BOOL, G) \ WRAP_ARGS(int , cv::gapi::ArgType::CV_INT, G) \ +WRAP_ARGS(int64_t , cv::gapi::ArgType::CV_INT64, G) \ WRAP_ARGS(double , cv::gapi::ArgType::CV_DOUBLE, G) \ WRAP_ARGS(float , cv::gapi::ArgType::CV_FLOAT, G) \ WRAP_ARGS(std::string , cv::gapi::ArgType::CV_STRING, G) \ @@ -58,6 +60,7 @@ namespace gapi { enum ArgType { CV_BOOL, CV_INT, + CV_INT64, CV_DOUBLE, CV_FLOAT, CV_STRING, diff --git a/modules/gapi/misc/python/shadow_gapi.hpp b/modules/gapi/misc/python/shadow_gapi.hpp index 941250c2fb..e777aa5d93 100644 --- a/modules/gapi/misc/python/shadow_gapi.hpp +++ b/modules/gapi/misc/python/shadow_gapi.hpp @@ -8,31 +8,20 @@ namespace cv GAPI_WRAP GCompileArg(gapi::GNetPackage pkg); }; - // NB: This classes doesn't exist in *.so - // HACK: Mark them as a class to force python wrapper generate code for this entities - class GAPI_EXPORTS_W_SIMPLE GProtoArg { }; - class GAPI_EXPORTS_W_SIMPLE GProtoInputArgs { }; - class GAPI_EXPORTS_W_SIMPLE GProtoOutputArgs { }; - class GAPI_EXPORTS_W_SIMPLE GRunArg { }; - class GAPI_EXPORTS_W_SIMPLE GMetaArg { GAPI_WRAP GMetaArg(); }; - - using GProtoInputArgs = GIOProtoArgs; - using GProtoOutputArgs = GIOProtoArgs; - class GAPI_EXPORTS_W_SIMPLE GInferInputs { public: GAPI_WRAP GInferInputs(); - GAPI_WRAP void setInput(const std::string& name, const cv::GMat& value); - GAPI_WRAP void setInput(const std::string& name, const cv::GFrame& value); + GAPI_WRAP GInferInputs& setInput(const std::string& name, const cv::GMat& value); + GAPI_WRAP GInferInputs& setInput(const std::string& name, const cv::GFrame& value); }; class GAPI_EXPORTS_W_SIMPLE GInferListInputs { public: GAPI_WRAP GInferListInputs(); - GAPI_WRAP void setInput(const std::string& name, const cv::GArray& value); - GAPI_WRAP void setInput(const std::string& name, const cv::GArray& value); + GAPI_WRAP GInferListInputs setInput(const std::string& name, const cv::GArray& value); + GAPI_WRAP GInferListInputs setInput(const std::string& name, const cv::GArray& value); }; class GAPI_EXPORTS_W_SIMPLE GInferOutputs @@ -51,12 +40,18 @@ namespace cv namespace detail { - struct GAPI_EXPORTS_W_SIMPLE ExtractArgsCallback { }; - struct GAPI_EXPORTS_W_SIMPLE ExtractMetaCallback { }; + gapi::GNetParam GAPI_EXPORTS_W strip(gapi::ie::PyParams params); } // namespace detail namespace gapi { + namespace streaming + { + // FIXME: Extend to work with an arbitrary G-type. + cv::GOpaque GAPI_EXPORTS_W timestamp(cv::GMat); + cv::GOpaque GAPI_EXPORTS_W seqNo(cv::GMat); + cv::GOpaque GAPI_EXPORTS_W seq_id(cv::GMat); + } // namespace streaming namespace wip { class GAPI_EXPORTS_W IStreamSource { }; diff --git a/modules/gapi/misc/python/test/test_gapi_streaming.py b/modules/gapi/misc/python/test/test_gapi_streaming.py index f1cce4fb72..4ea88878ee 100644 --- a/modules/gapi/misc/python/test/test_gapi_streaming.py +++ b/modules/gapi/misc/python/test/test_gapi_streaming.py @@ -28,7 +28,7 @@ try: g_in = cv.GMat() g_out = cv.gapi.medianBlur(g_in, 3) c = cv.GComputation(g_in, g_out) - ccomp = c.compileStreaming(cv.descr_of(in_mat)) + ccomp = c.compileStreaming(cv.gapi.descr_of(in_mat)) ccomp.setSource(cv.gin(in_mat)) ccomp.start() @@ -52,7 +52,7 @@ try: ccomp = c.compileStreaming() source = cv.gapi.wip.make_capture_src(path) - ccomp.setSource(source) + ccomp.setSource(cv.gin(source)) ccomp.start() # Assert @@ -87,7 +87,7 @@ try: ccomp = c.compileStreaming() source = cv.gapi.wip.make_capture_src(path) - ccomp.setSource(source) + ccomp.setSource(cv.gin(source)) ccomp.start() # Assert @@ -176,7 +176,7 @@ try: ccomp = c.compileStreaming() source = cv.gapi.wip.make_capture_src(path) - ccomp.setSource(source) + ccomp.setSource(cv.gin(source)) ccomp.start() # Assert @@ -209,6 +209,40 @@ try: break + def test_gapi_streaming_meta(self): + ksize = 3 + path = self.find_file('cv/video/768x576.avi', [os.environ['OPENCV_TEST_DATA_PATH']]) + + # G-API + g_in = cv.GMat() + g_ts = cv.gapi.streaming.timestamp(g_in) + g_seqno = cv.gapi.streaming.seqNo(g_in) + g_seqid = cv.gapi.streaming.seq_id(g_in) + + c = cv.GComputation(cv.GIn(g_in), cv.GOut(g_ts, g_seqno, g_seqid)) + + ccomp = c.compileStreaming() + source = cv.gapi.wip.make_capture_src(path) + ccomp.setSource(cv.gin(source)) + ccomp.start() + + # Assert + max_num_frames = 10 + curr_frame_number = 0 + while True: + has_frame, (ts, seqno, seqid) = ccomp.pull() + + if not has_frame: + break + + self.assertEqual(curr_frame_number, seqno) + self.assertEqual(curr_frame_number, seqid) + + curr_frame_number += 1 + if curr_frame_number == max_num_frames: + break + + except unittest.SkipTest as e: message = str(e) diff --git a/modules/gapi/src/api/ginfer.cpp b/modules/gapi/src/api/ginfer.cpp index e3cc94041c..9db05a43c3 100644 --- a/modules/gapi/src/api/ginfer.cpp +++ b/modules/gapi/src/api/ginfer.cpp @@ -15,6 +15,10 @@ cv::gapi::GNetPackage::GNetPackage(std::initializer_list ii) : networks(ii) { } +cv::gapi::GNetPackage::GNetPackage(std::vector nets) + : networks(nets) { +} + std::vector cv::gapi::GNetPackage::backends() const { std::unordered_set unique_set; for (const auto &nn : networks) unique_set.insert(nn.backend); diff --git a/modules/gapi/src/backends/common/gmetabackend.cpp b/modules/gapi/src/backends/common/gmetabackend.cpp index c535569b0c..40e87c3ea0 100644 --- a/modules/gapi/src/backends/common/gmetabackend.cpp +++ b/modules/gapi/src/backends/common/gmetabackend.cpp @@ -85,6 +85,19 @@ class GGraphMetaBackendImpl final: public cv::gapi::GBackend::Priv { const std::vector&) const override { return EPtr{new GraphMetaExecutable(graph, nodes)}; } + + virtual bool controlsMerge() const override + { + return true; + } + + virtual bool allowsMerge(const cv::gimpl::GIslandModel::Graph &, + const ade::NodeHandle &, + const ade::NodeHandle &, + const ade::NodeHandle &) const override + { + return false; + } }; cv::gapi::GBackend graph_meta_backend() { diff --git a/modules/python/src2/cv2.cpp b/modules/python/src2/cv2.cpp index 9e8a6ee13b..795afb13f2 100644 --- a/modules/python/src2/cv2.cpp +++ b/modules/python/src2/cv2.cpp @@ -2219,12 +2219,6 @@ static PyMethodDef special_methods[] = { #ifdef HAVE_OPENCV_DNN {"dnn_registerLayer", CV_PY_FN_WITH_KW(pyopencv_cv_dnn_registerLayer), "registerLayer(type, class) -> None"}, {"dnn_unregisterLayer", CV_PY_FN_WITH_KW(pyopencv_cv_dnn_unregisterLayer), "unregisterLayer(type) -> None"}, -#endif -#ifdef HAVE_OPENCV_GAPI - {"GIn", CV_PY_FN_WITH_KW(pyopencv_cv_GIn), "GIn(...) -> GInputProtoArgs"}, - {"GOut", CV_PY_FN_WITH_KW(pyopencv_cv_GOut), "GOut(...) -> GOutputProtoArgs"}, - {"gin", CV_PY_FN_WITH_KW(pyopencv_cv_gin), "gin(...) -> ExtractArgsCallback"}, - {"descr_of", CV_PY_FN_WITH_KW(pyopencv_cv_descr_of), "descr_of(...) -> ExtractMetaCallback"}, #endif {NULL, NULL}, }; From 5e80bd3cc922bad5eb1fa02b56bc3db44abb96e9 Mon Sep 17 00:00:00 2001 From: APrigarina Date: Wed, 30 Jun 2021 12:50:21 +0300 Subject: [PATCH 026/262] fix samples 3.4 --- samples/python/camera_calibration_show_extrinsics.py | 2 +- samples/python/gaussian_mix.py | 2 +- samples/python/hist.py | 2 +- samples/python/lk_homography.py | 6 +++--- samples/python/lk_track.py | 2 +- samples/python/video_v4l2.py | 4 ++-- 6 files changed, 9 insertions(+), 9 deletions(-) diff --git a/samples/python/camera_calibration_show_extrinsics.py b/samples/python/camera_calibration_show_extrinsics.py index 0118b5b913..d676691f15 100755 --- a/samples/python/camera_calibration_show_extrinsics.py +++ b/samples/python/camera_calibration_show_extrinsics.py @@ -188,7 +188,7 @@ def main(): fig = plt.figure() ax = fig.gca(projection='3d') - ax.set_aspect("equal") + ax.set_aspect("auto") cam_width = args.cam_width cam_height = args.cam_height diff --git a/samples/python/gaussian_mix.py b/samples/python/gaussian_mix.py index 5f2dfcc440..6a656647dd 100755 --- a/samples/python/gaussian_mix.py +++ b/samples/python/gaussian_mix.py @@ -32,7 +32,7 @@ def draw_gaussain(img, mean, cov, color): w, u, _vt = cv.SVDecomp(cov) ang = np.arctan2(u[1, 0], u[0, 0])*(180/np.pi) s1, s2 = np.sqrt(w)*3.0 - cv.ellipse(img, (x, y), (s1, s2), ang, 0, 360, color, 1, cv.LINE_AA) + cv.ellipse(img, (int(x), int(y)), (int(s1), int(s2)), ang, 0, 360, color, 1, cv.LINE_AA) def main(): diff --git a/samples/python/hist.py b/samples/python/hist.py index 4c2c1ad395..157d5ff0ba 100755 --- a/samples/python/hist.py +++ b/samples/python/hist.py @@ -48,7 +48,7 @@ def hist_lines(im): cv.normalize(hist_item,hist_item,0,255,cv.NORM_MINMAX) hist=np.int32(np.around(hist_item)) for x,y in enumerate(hist): - cv.line(h,(x,0),(x,y),(255,255,255)) + cv.line(h,(x,0),(x,y[0]),(255,255,255)) y = np.flipud(h) return y diff --git a/samples/python/lk_homography.py b/samples/python/lk_homography.py index 808f30965f..38a05f63b6 100755 --- a/samples/python/lk_homography.py +++ b/samples/python/lk_homography.py @@ -77,8 +77,8 @@ class App: for (x0, y0), (x1, y1), good in zip(self.p0[:,0], self.p1[:,0], status[:,0]): if good: - cv.line(vis, (x0, y0), (x1, y1), (0, 128, 0)) - cv.circle(vis, (x1, y1), 2, (red, green)[good], -1) + cv.line(vis, (int(x0), int(y0)), (int(x1), int(y1)), (0, 128, 0)) + cv.circle(vis, (int(x1), int(y1)), 2, (red, green)[good], -1) draw_str(vis, (20, 20), 'track count: %d' % len(self.p1)) if self.use_ransac: draw_str(vis, (20, 40), 'RANSAC') @@ -86,7 +86,7 @@ class App: p = cv.goodFeaturesToTrack(frame_gray, **feature_params) if p is not None: for x, y in p[:,0]: - cv.circle(vis, (x, y), 2, green, -1) + cv.circle(vis, (int(x), int(y)), 2, green, -1) draw_str(vis, (20, 20), 'feature count: %d' % len(p)) cv.imshow('lk_homography', vis) diff --git a/samples/python/lk_track.py b/samples/python/lk_track.py index 7b77f1b335..97a8c40241 100755 --- a/samples/python/lk_track.py +++ b/samples/python/lk_track.py @@ -65,7 +65,7 @@ class App: if len(tr) > self.track_len: del tr[0] new_tracks.append(tr) - cv.circle(vis, (x, y), 2, (0, 255, 0), -1) + cv.circle(vis, (int(x), int(y)), 2, (0, 255, 0), -1) self.tracks = new_tracks cv.polylines(vis, [np.int32(tr) for tr in self.tracks], False, (0, 255, 0)) draw_str(vis, (20, 20), 'track count: %d' % len(self.tracks)) diff --git a/samples/python/video_v4l2.py b/samples/python/video_v4l2.py index 61b1e35804..abebb2a2ca 100644 --- a/samples/python/video_v4l2.py +++ b/samples/python/video_v4l2.py @@ -30,7 +30,7 @@ def main(): color = (0, 255, 0) cap = cv.VideoCapture(0) - cap.set(cv.CAP_PROP_AUTOFOCUS, False) # Known bug: https://github.com/opencv/opencv/pull/5474 + cap.set(cv.CAP_PROP_AUTOFOCUS, 0) # Known bug: https://github.com/opencv/opencv/pull/5474 cv.namedWindow("Video") @@ -67,7 +67,7 @@ def main(): break elif k == ord('g'): convert_rgb = not convert_rgb - cap.set(cv.CAP_PROP_CONVERT_RGB, convert_rgb) + cap.set(cv.CAP_PROP_CONVERT_RGB, 1 if convert_rgb else 0) print('Done') From 90be83ae99cb719619f26fedf6d199f6422b551e Mon Sep 17 00:00:00 2001 From: Vladimir <10669582+Wovchena@users.noreply.github.com> Date: Wed, 30 Jun 2021 10:15:58 +0300 Subject: [PATCH 027/262] Fix an arg for calcHist() in demos `float* histRange = { range };` doesn't make much sense. `histRange` is an array of array(s), so it should have a type of ptr to ptr. Strangely some domos are correct as well as the example for the function https://docs.opencv.org/master/d6/dc7/group__imgproc__hist.html#ga4b2b5fd75503ff9e6844cc4dcdaed35d --- .../Histograms_Matching/MatchTemplate_Demo.cpp | 2 +- .../Histograms_Matching/calcBackProject_Demo1.cpp | 6 +++--- .../tutorial_code/Histograms_Matching/calcHist_Demo.cpp | 8 ++++---- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/samples/cpp/tutorial_code/Histograms_Matching/MatchTemplate_Demo.cpp b/samples/cpp/tutorial_code/Histograms_Matching/MatchTemplate_Demo.cpp index 5bcc878965..f9abbae945 100644 --- a/samples/cpp/tutorial_code/Histograms_Matching/MatchTemplate_Demo.cpp +++ b/samples/cpp/tutorial_code/Histograms_Matching/MatchTemplate_Demo.cpp @@ -89,7 +89,7 @@ void MatchingMethod( int, void* ) //! [create_result_matrix] /// Create the result matrix - int result_cols = img.cols - templ.cols + 1; + int result_cols = img.cols - templ.cols + 1; int result_rows = img.rows - templ.rows + 1; result.create( result_rows, result_cols, CV_32FC1 ); diff --git a/samples/cpp/tutorial_code/Histograms_Matching/calcBackProject_Demo1.cpp b/samples/cpp/tutorial_code/Histograms_Matching/calcBackProject_Demo1.cpp index 61b6d607ce..bcb547a2fb 100644 --- a/samples/cpp/tutorial_code/Histograms_Matching/calcBackProject_Demo1.cpp +++ b/samples/cpp/tutorial_code/Histograms_Matching/calcBackProject_Demo1.cpp @@ -72,18 +72,18 @@ void Hist_and_Backproj(int, void* ) //! [initialize] int histSize = MAX( bins, 2 ); float hue_range[] = { 0, 180 }; - const float* ranges = { hue_range }; + const float* ranges[] = { hue_range }; //! [initialize] //! [Get the Histogram and normalize it] Mat hist; - calcHist( &hue, 1, 0, Mat(), hist, 1, &histSize, &ranges, true, false ); + calcHist( &hue, 1, 0, Mat(), hist, 1, &histSize, ranges, true, false ); normalize( hist, hist, 0, 255, NORM_MINMAX, -1, Mat() ); //! [Get the Histogram and normalize it] //! [Get Backprojection] Mat backproj; - calcBackProject( &hue, 1, 0, hist, backproj, &ranges, 1, true ); + calcBackProject( &hue, 1, 0, hist, backproj, ranges, 1, true ); //! [Get Backprojection] //! [Draw the backproj] diff --git a/samples/cpp/tutorial_code/Histograms_Matching/calcHist_Demo.cpp b/samples/cpp/tutorial_code/Histograms_Matching/calcHist_Demo.cpp index 86167e519a..a7582e4282 100644 --- a/samples/cpp/tutorial_code/Histograms_Matching/calcHist_Demo.cpp +++ b/samples/cpp/tutorial_code/Histograms_Matching/calcHist_Demo.cpp @@ -37,7 +37,7 @@ int main(int argc, char** argv) //! [Set the ranges ( for B,G,R) )] float range[] = { 0, 256 }; //the upper boundary is exclusive - const float* histRange = { range }; + const float* histRange[] = { range }; //! [Set the ranges ( for B,G,R) )] //! [Set histogram param] @@ -46,9 +46,9 @@ int main(int argc, char** argv) //! [Compute the histograms] Mat b_hist, g_hist, r_hist; - calcHist( &bgr_planes[0], 1, 0, Mat(), b_hist, 1, &histSize, &histRange, uniform, accumulate ); - calcHist( &bgr_planes[1], 1, 0, Mat(), g_hist, 1, &histSize, &histRange, uniform, accumulate ); - calcHist( &bgr_planes[2], 1, 0, Mat(), r_hist, 1, &histSize, &histRange, uniform, accumulate ); + calcHist( &bgr_planes[0], 1, 0, Mat(), b_hist, 1, &histSize, histRange, uniform, accumulate ); + calcHist( &bgr_planes[1], 1, 0, Mat(), g_hist, 1, &histSize, histRange, uniform, accumulate ); + calcHist( &bgr_planes[2], 1, 0, Mat(), r_hist, 1, &histSize, histRange, uniform, accumulate ); //! [Compute the histograms] //! [Draw the histograms for B, G and R] From 6a3d925a47d54945eab7b50a769531703cde99a2 Mon Sep 17 00:00:00 2001 From: Joe Howse Date: Mon, 21 Jun 2021 00:46:32 -0300 Subject: [PATCH 028/262] OpenCL: core support for FP16, more channel orders * Support cl_image conversion for CL_HALF_FLOAT (float16) * Support cl_image conversion for additional channel orders: CL_A, CL_INTENSITY, CL_LUMINANCE, CL_RG, CL_RA * Comment on why cl_image conversion is unsupported for CL_RGB * Predict optimal vector width for float16 * ocl::kernelToStr: support float16 * ocl::Device::halfFPConfig: drop artificial requirement for OpenCL version >= 1.2. Even OpenCL 1.0 supports the underlying config property, CL_DEVICE_HALF_FP_CONFIG. * dumpOpenCLInformation: provide info on OpenCL half-float support and preferred half-float vector width * randu: support default range [-1.0, 1.0] for float16 * TestBase::warmup: support float16 --- .../opencv2/core/opencl/opencl_info.hpp | 7 +++ modules/core/src/ocl.cpp | 45 ++++++++++++++----- modules/ts/src/ocl_perf.cpp | 2 +- modules/ts/src/ts_perf.cpp | 2 +- 4 files changed, 44 insertions(+), 12 deletions(-) diff --git a/modules/core/include/opencv2/core/opencl/opencl_info.hpp b/modules/core/include/opencv2/core/opencl/opencl_info.hpp index 5e5c846ad0..3ead76e5c4 100644 --- a/modules/core/include/opencv2/core/opencl/opencl_info.hpp +++ b/modules/core/include/opencv2/core/opencl/opencl_info.hpp @@ -144,6 +144,10 @@ static void dumpOpenCLInformation() DUMP_MESSAGE_STDOUT(" Double support = " << doubleSupportStr); DUMP_CONFIG_PROPERTY("cv_ocl_current_haveDoubleSupport", device.doubleFPConfig() > 0); + const char* halfSupportStr = device.halfFPConfig() > 0 ? "Yes" : "No"; + DUMP_MESSAGE_STDOUT(" Half support = " << halfSupportStr); + DUMP_CONFIG_PROPERTY("cv_ocl_current_haveHalfSupport", device.halfFPConfig() > 0); + const char* isUnifiedMemoryStr = device.hostUnifiedMemory() ? "Yes" : "No"; DUMP_MESSAGE_STDOUT(" Host unified memory = " << isUnifiedMemoryStr); DUMP_CONFIG_PROPERTY("cv_ocl_current_hostUnifiedMemory", device.hostUnifiedMemory()); @@ -191,6 +195,9 @@ static void dumpOpenCLInformation() DUMP_MESSAGE_STDOUT(" Preferred vector width double = " << device.preferredVectorWidthDouble()); DUMP_CONFIG_PROPERTY("cv_ocl_current_preferredVectorWidthDouble", device.preferredVectorWidthDouble()); + + DUMP_MESSAGE_STDOUT(" Preferred vector width half = " << device.preferredVectorWidthHalf()); + DUMP_CONFIG_PROPERTY("cv_ocl_current_preferredVectorWidthHalf", device.preferredVectorWidthHalf()); } catch (...) { diff --git a/modules/core/src/ocl.cpp b/modules/core/src/ocl.cpp index 0e97cf52fe..46185446f7 100644 --- a/modules/core/src/ocl.cpp +++ b/modules/core/src/ocl.cpp @@ -1566,6 +1566,7 @@ struct Device::Impl version_ = getStrProp(CL_DEVICE_VERSION); extensions_ = getStrProp(CL_DEVICE_EXTENSIONS); doubleFPConfig_ = getProp(CL_DEVICE_DOUBLE_FP_CONFIG); + halfFPConfig_ = getProp(CL_DEVICE_HALF_FP_CONFIG); hostUnifiedMemory_ = getBoolProp(CL_DEVICE_HOST_UNIFIED_MEMORY); maxComputeUnits_ = getProp(CL_DEVICE_MAX_COMPUTE_UNITS); maxWorkGroupSize_ = getProp(CL_DEVICE_MAX_WORK_GROUP_SIZE); @@ -1678,6 +1679,7 @@ struct Device::Impl String version_; std::string extensions_; int doubleFPConfig_; + int halfFPConfig_; bool hostUnifiedMemory_; int maxComputeUnits_; size_t maxWorkGroupSize_; @@ -1827,11 +1829,7 @@ int Device::singleFPConfig() const { return p ? p->getProp(CL_DEVICE_SINGLE_FP_CONFIG) : 0; } int Device::halfFPConfig() const -#ifdef CL_VERSION_1_2 -{ return p ? p->getProp(CL_DEVICE_HALF_FP_CONFIG) : 0; } -#else -{ CV_REQUIRE_OPENCL_1_2_ERROR; } -#endif +{ return p ? p->halfFPConfig_ : 0; } bool Device::endianLittle() const { return p ? p->getBoolProp(CL_DEVICE_ENDIAN_LITTLE) : false; } @@ -6668,6 +6666,10 @@ void convertFromImage(void* cl_mem_image, UMat& dst) depth = CV_32F; break; + case CL_HALF_FLOAT: + depth = CV_16F; + break; + default: CV_Error(cv::Error::OpenCLApiCallError, "Not supported image_channel_data_type"); } @@ -6676,9 +6678,23 @@ void convertFromImage(void* cl_mem_image, UMat& dst) switch (fmt.image_channel_order) { case CL_R: + case CL_A: + case CL_INTENSITY: + case CL_LUMINANCE: type = CV_MAKE_TYPE(depth, 1); break; + case CL_RG: + case CL_RA: + type = CV_MAKE_TYPE(depth, 2); + break; + + // CL_RGB has no mappings to OpenCV types because CL_RGB can only be used with + // CL_UNORM_SHORT_565, CL_UNORM_SHORT_555, or CL_UNORM_INT_101010. + /*case CL_RGB: + type = CV_MAKE_TYPE(depth, 3); + break;*/ + case CL_RGBA: case CL_BGRA: case CL_ARGB: @@ -7068,6 +7084,13 @@ static std::string kerToStr(const Mat & k) stream << "DIG(" << data[i] << "f)"; stream << "DIG(" << data[width] << "f)"; } + else if (depth == CV_16F) + { + stream.setf(std::ios_base::showpoint); + for (int i = 0; i < width; ++i) + stream << "DIG(" << (float)data[i] << "h)"; + stream << "DIG(" << (float)data[width] << "h)"; + } else { for (int i = 0; i < width; ++i) @@ -7091,7 +7114,7 @@ String kernelToStr(InputArray _kernel, int ddepth, const char * name) typedef std::string (* func_t)(const Mat &); static const func_t funcs[] = { kerToStr, kerToStr, kerToStr, kerToStr, - kerToStr, kerToStr, kerToStr, 0 }; + kerToStr, kerToStr, kerToStr, kerToStr }; const func_t func = funcs[ddepth]; CV_Assert(func != 0); @@ -7130,14 +7153,14 @@ int predictOptimalVectorWidth(InputArray src1, InputArray src2, InputArray src3, int vectorWidths[] = { d.preferredVectorWidthChar(), d.preferredVectorWidthChar(), d.preferredVectorWidthShort(), d.preferredVectorWidthShort(), d.preferredVectorWidthInt(), d.preferredVectorWidthFloat(), - d.preferredVectorWidthDouble(), -1 }; + d.preferredVectorWidthDouble(), d.preferredVectorWidthHalf() }; // if the device says don't use vectors if (vectorWidths[0] == 1) { // it's heuristic vectorWidths[CV_8U] = vectorWidths[CV_8S] = 4; - vectorWidths[CV_16U] = vectorWidths[CV_16S] = 2; + vectorWidths[CV_16U] = vectorWidths[CV_16S] = vectorWidths[CV_16F] = 2; vectorWidths[CV_32S] = vectorWidths[CV_32F] = vectorWidths[CV_64F] = 1; } @@ -7225,10 +7248,12 @@ struct Image2D::Impl { cl_image_format format; static const int channelTypes[] = { CL_UNSIGNED_INT8, CL_SIGNED_INT8, CL_UNSIGNED_INT16, - CL_SIGNED_INT16, CL_SIGNED_INT32, CL_FLOAT, -1, -1 }; + CL_SIGNED_INT16, CL_SIGNED_INT32, CL_FLOAT, -1, CL_HALF_FLOAT }; static const int channelTypesNorm[] = { CL_UNORM_INT8, CL_SNORM_INT8, CL_UNORM_INT16, CL_SNORM_INT16, -1, -1, -1, -1 }; - static const int channelOrders[] = { -1, CL_R, CL_RG, -1, CL_RGBA }; + // CL_RGB has no mappings to OpenCV types because CL_RGB can only be used with + // CL_UNORM_SHORT_565, CL_UNORM_SHORT_555, or CL_UNORM_INT_101010. + static const int channelOrders[] = { -1, CL_R, CL_RG, /*CL_RGB*/ -1, CL_RGBA }; int channelType = norm ? channelTypesNorm[depth] : channelTypes[depth]; int channelOrder = channelOrders[cn]; diff --git a/modules/ts/src/ocl_perf.cpp b/modules/ts/src/ocl_perf.cpp index 8dacf219f6..fe521f2c00 100644 --- a/modules/ts/src/ocl_perf.cpp +++ b/modules/ts/src/ocl_perf.cpp @@ -70,7 +70,7 @@ void randu(InputOutputArray dst) cv::randu(dst, -128, 128); else if (dst.depth() == CV_16U) cv::randu(dst, 0, 1024); - else if (dst.depth() == CV_32F || dst.depth() == CV_64F) + else if (dst.depth() == CV_32F || dst.depth() == CV_64F || dst.depth() == CV_16F) cv::randu(dst, -1.0, 1.0); else if (dst.depth() == CV_16S || dst.depth() == CV_32S) cv::randu(dst, -4096, 4096); diff --git a/modules/ts/src/ts_perf.cpp b/modules/ts/src/ts_perf.cpp index 2a9169fd13..5a42ca01cd 100644 --- a/modules/ts/src/ts_perf.cpp +++ b/modules/ts/src/ts_perf.cpp @@ -1297,7 +1297,7 @@ void TestBase::warmup(cv::InputOutputArray a, WarmUpType wtype) cv::randu(a, -128, 128); else if (depth == CV_16U) cv::randu(a, 0, 1024); - else if (depth == CV_32F || depth == CV_64F) + else if (depth == CV_32F || depth == CV_64F || depth == CV_16F) cv::randu(a, -1.0, 1.0); else if (depth == CV_16S || depth == CV_32S) cv::randu(a, -4096, 4096); From bf489feef11f365f8bd911bf4491d207d04ca46f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafael=20H=20Tib=C3=A3es?= Date: Wed, 30 Jun 2021 19:08:24 -0300 Subject: [PATCH 029/262] Merge pull request #20327 from tibaes:MSMF-Slow-Webcam-Startup * fixes MSMF slow webcam startup * add variable to change MF_READWRITE_ENABLE_HARDWARE_TRANSFORMS at runtime --- modules/videoio/src/cap_msmf.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/modules/videoio/src/cap_msmf.cpp b/modules/videoio/src/cap_msmf.cpp index 73288c3d03..9e45fd1bac 100644 --- a/modules/videoio/src/cap_msmf.cpp +++ b/modules/videoio/src/cap_msmf.cpp @@ -708,9 +708,10 @@ bool CvCapture_MSMF::initStream(DWORD streamID, const MediaType& mt) _ComPtr CvCapture_MSMF::getDefaultSourceConfig(UINT32 num) { CV_Assert(num > 0); + const bool OPENCV_VIDEOIO_MSMF_ENABLE_HW_TRANSFORMS = utils::getConfigurationParameterBool("OPENCV_VIDEOIO_MSMF_ENABLE_HW_TRANSFORMS", true); _ComPtr res; if (FAILED(MFCreateAttributes(&res, num)) || - FAILED(res->SetUINT32(MF_READWRITE_ENABLE_HARDWARE_TRANSFORMS, true)) || + FAILED(res->SetUINT32(MF_READWRITE_ENABLE_HARDWARE_TRANSFORMS, OPENCV_VIDEOIO_MSMF_ENABLE_HW_TRANSFORMS)) || FAILED(res->SetUINT32(MF_SOURCE_READER_DISABLE_DXVA, false)) || FAILED(res->SetUINT32(MF_SOURCE_READER_ENABLE_VIDEO_PROCESSING, false)) || FAILED(res->SetUINT32(MF_SOURCE_READER_ENABLE_ADVANCED_VIDEO_PROCESSING, true)) From 6797fd65a5de608d30273ace911c5de14ac7693e Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Tue, 29 Jun 2021 20:25:22 +0000 Subject: [PATCH 030/262] dnn(test): update tests for OpenVINO 2021.4 --- modules/dnn/test/test_backends.cpp | 10 +++++----- modules/dnn/test/test_ie_models.cpp | 9 +++++++++ modules/dnn/test/test_torch_importer.cpp | 9 +++++++-- 3 files changed, 21 insertions(+), 7 deletions(-) diff --git a/modules/dnn/test/test_backends.cpp b/modules/dnn/test/test_backends.cpp index b9958c107e..5426a11a3f 100644 --- a/modules/dnn/test/test_backends.cpp +++ b/modules/dnn/test/test_backends.cpp @@ -196,7 +196,7 @@ TEST_P(DNNTestNetwork, MobileNet_SSD_Caffe) Mat inp = blobFromImage(sample, 1.0f / 127.5, Size(300, 300), Scalar(127.5, 127.5, 127.5), false); float diffScores = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 1.5e-2 : 0.0; float diffSquares = (target == DNN_TARGET_MYRIAD) ? 0.063 : 0.0; - float detectionConfThresh = (target == DNN_TARGET_MYRIAD) ? 0.252 : FLT_MIN; + float detectionConfThresh = (target == DNN_TARGET_MYRIAD) ? 0.262 : FLT_MIN; processNet("dnn/MobileNetSSD_deploy.caffemodel", "dnn/MobileNetSSD_deploy.prototxt", inp, "detection_out", "", diffScores, diffSquares, detectionConfThresh); expectNoFallbacksFromIE(net); @@ -301,8 +301,8 @@ TEST_P(DNNTestNetwork, OpenPose_pose_coco) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD_X, CV_TEST_TAG_DNN_SKIP_IE_VERSION); #endif - const float l1 = (target == DNN_TARGET_MYRIAD) ? 0.0056 : 0.0; - const float lInf = (target == DNN_TARGET_MYRIAD) ? 0.072 : 0.0; + const float l1 = (target == DNN_TARGET_MYRIAD) ? 0.009 : 0.0; + const float lInf = (target == DNN_TARGET_MYRIAD) ? 0.09 : 0.0; processNet("dnn/openpose_pose_coco.caffemodel", "dnn/openpose_pose_coco.prototxt", Size(46, 46), "", "", l1, lInf); expectNoFallbacksFromIE(net); @@ -321,8 +321,8 @@ TEST_P(DNNTestNetwork, OpenPose_pose_mpi) #endif // output range: [-0.001, 0.97] - const float l1 = (target == DNN_TARGET_MYRIAD) ? 0.012 : 0.0; - const float lInf = (target == DNN_TARGET_MYRIAD || target == DNN_TARGET_OPENCL_FP16) ? 0.16 : 0.0; + const float l1 = (target == DNN_TARGET_MYRIAD) ? 0.02 : 0.0; + const float lInf = (target == DNN_TARGET_MYRIAD || target == DNN_TARGET_OPENCL_FP16) ? 0.2 : 0.0; processNet("dnn/openpose_pose_mpi.caffemodel", "dnn/openpose_pose_mpi.prototxt", Size(46, 46), "", "", l1, lInf); expectNoFallbacksFromIE(net); diff --git a/modules/dnn/test/test_ie_models.cpp b/modules/dnn/test/test_ie_models.cpp index 2ba7d80f58..da6cbd6fbc 100644 --- a/modules/dnn/test/test_ie_models.cpp +++ b/modules/dnn/test/test_ie_models.cpp @@ -288,6 +288,15 @@ TEST_P(DNNTestOpenVINO, models) ASSERT_FALSE(backendId != DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 && backendId != DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) << "Inference Engine backend is required"; +#if INF_ENGINE_VER_MAJOR_EQ(2021040000) + if (targetId == DNN_TARGET_MYRIAD && ( + modelName == "person-detection-retail-0013" || // ncDeviceOpen:1013 Failed to find booted device after boot + modelName == "age-gender-recognition-retail-0013" // ncDeviceOpen:1013 Failed to find booted device after boot + ) + ) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD, CV_DNN_BACKEND_INFERENCE_ENGINE_NGRAPH, CV_TEST_TAG_DNN_SKIP_IE_VERSION); +#endif + #if INF_ENGINE_VER_MAJOR_GE(2020020000) if (targetId == DNN_TARGET_MYRIAD && backendId == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019) { diff --git a/modules/dnn/test/test_torch_importer.cpp b/modules/dnn/test/test_torch_importer.cpp index 8738e5e25c..7316a06856 100644 --- a/modules/dnn/test/test_torch_importer.cpp +++ b/modules/dnn/test/test_torch_importer.cpp @@ -254,9 +254,14 @@ TEST_P(Test_Torch_layers, net_padding) TEST_P(Test_Torch_layers, net_non_spatial) { -#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2021030000) +#if defined(INF_ENGINE_RELEASE) && ( \ + INF_ENGINE_VER_MAJOR_EQ(2021030000) || \ + INF_ENGINE_VER_MAJOR_EQ(2021040000) \ +) if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target == DNN_TARGET_MYRIAD) - applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD, CV_TEST_TAG_DNN_SKIP_IE_NGRAPH); // crash + // 2021.3: crash + // 2021.4: [ GENERAL_ERROR ] AssertionFailed: !out.networkInputs.empty() + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD, CV_TEST_TAG_DNN_SKIP_IE_NGRAPH); if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target == DNN_TARGET_OPENCL) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL, CV_TEST_TAG_DNN_SKIP_IE_NGRAPH); // exception if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target == DNN_TARGET_OPENCL_FP16) From f2057ce1ab7d92f2deb4de29e7a81f401965a127 Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Tue, 29 Jun 2021 21:55:18 +0000 Subject: [PATCH 031/262] dnn(ie): replace deprecated calls --- modules/dnn/src/ie_ngraph.cpp | 51 +++++++++++++++++++++++++---------- 1 file changed, 37 insertions(+), 14 deletions(-) diff --git a/modules/dnn/src/ie_ngraph.cpp b/modules/dnn/src/ie_ngraph.cpp index 3b87243717..e6c219f13e 100644 --- a/modules/dnn/src/ie_ngraph.cpp +++ b/modules/dnn/src/ie_ngraph.cpp @@ -654,7 +654,11 @@ void InfEngineNgraphNet::initPlugin(InferenceEngine::CNNNetwork& net) try { InferenceEngine::IExtensionPtr extension = +#if INF_ENGINE_VER_MAJOR_GE(INF_ENGINE_RELEASE_2021_4) + std::make_shared(libName); +#else InferenceEngine::make_so_pointer(libName); +#endif ie.AddExtension(extension, "CPU"); CV_LOG_INFO(NULL, "DNN-IE: Loaded extension plugin: " << libName); @@ -1002,35 +1006,54 @@ void InfEngineNgraphNet::forward(const std::vector >& outBlo reqWrapper->req.SetInput(inpBlobs); reqWrapper->req.SetOutput(outBlobs); +#if INF_ENGINE_VER_MAJOR_GE(INF_ENGINE_RELEASE_2021_4) + InferenceEngine::InferRequest infRequest = reqWrapper->req; + NgraphReqWrapper* wrapperPtr = reqWrapper.get(); + CV_Assert(wrapperPtr && "Internal error"); +#else InferenceEngine::IInferRequest::Ptr infRequestPtr = reqWrapper->req; - infRequestPtr->SetUserData(reqWrapper.get(), 0); + CV_Assert(infRequestPtr); + InferenceEngine::IInferRequest& infRequest = *infRequestPtr.get(); + infRequest.SetUserData(reqWrapper.get(), 0); +#endif - infRequestPtr->SetCompletionCallback( - [](InferenceEngine::IInferRequest::Ptr request, InferenceEngine::StatusCode status) +#if INF_ENGINE_VER_MAJOR_GE(INF_ENGINE_RELEASE_2021_4) + // do NOT capture 'reqWrapper' (smart ptr) in the lambda callback + infRequest.SetCompletionCallback>( + [wrapperPtr](InferenceEngine::InferRequest /*request*/, InferenceEngine::StatusCode status) +#else + infRequest.SetCompletionCallback( + [](InferenceEngine::IInferRequest::Ptr requestPtr, InferenceEngine::StatusCode status) +#endif { CV_LOG_DEBUG(NULL, "DNN(nGraph): completionCallback(" << (int)status << ")"); +#if !INF_ENGINE_VER_MAJOR_GE(INF_ENGINE_RELEASE_2021_4) + CV_Assert(requestPtr); + InferenceEngine::IInferRequest& request = *requestPtr.get(); - NgraphReqWrapper* wrapper; - request->GetUserData((void**)&wrapper, 0); - CV_Assert(wrapper && "Internal error"); + NgraphReqWrapper* wrapperPtr; + request.GetUserData((void**)&wrapperPtr, 0); + CV_Assert(wrapperPtr && "Internal error"); +#endif + NgraphReqWrapper& wrapper = *wrapperPtr; size_t processedOutputs = 0; try { - for (; processedOutputs < wrapper->outProms.size(); ++processedOutputs) + for (; processedOutputs < wrapper.outProms.size(); ++processedOutputs) { - const std::string& name = wrapper->outsNames[processedOutputs]; - Mat m = ngraphBlobToMat(wrapper->req.GetBlob(name)); + const std::string& name = wrapper.outsNames[processedOutputs]; + Mat m = ngraphBlobToMat(wrapper.req.GetBlob(name)); try { CV_Assert(status == InferenceEngine::StatusCode::OK); - wrapper->outProms[processedOutputs].setValue(m.clone()); + wrapper.outProms[processedOutputs].setValue(m.clone()); } catch (...) { try { - wrapper->outProms[processedOutputs].setException(std::current_exception()); + wrapper.outProms[processedOutputs].setException(std::current_exception()); } catch(...) { CV_LOG_ERROR(NULL, "DNN: Exception occurred during async inference exception propagation"); } @@ -1040,16 +1063,16 @@ void InfEngineNgraphNet::forward(const std::vector >& outBlo catch (...) { std::exception_ptr e = std::current_exception(); - for (; processedOutputs < wrapper->outProms.size(); ++processedOutputs) + for (; processedOutputs < wrapper.outProms.size(); ++processedOutputs) { try { - wrapper->outProms[processedOutputs].setException(e); + wrapper.outProms[processedOutputs].setException(e); } catch(...) { CV_LOG_ERROR(NULL, "DNN: Exception occurred during async inference exception propagation"); } } } - wrapper->isReady = true; + wrapper.isReady = true; } ); } From 5b8c10f2f85eb54fe3c945d1b5c8d0e4344ddbbd Mon Sep 17 00:00:00 2001 From: SamFC10 Date: Wed, 30 Jun 2021 21:55:42 +0530 Subject: [PATCH 032/262] modified onnx importer to concat const input blobs --- modules/dnn/src/onnx/onnx_importer.cpp | 17 +++++++++++++++++ modules/dnn/test/test_onnx_importer.cpp | 1 + 2 files changed, 18 insertions(+) diff --git a/modules/dnn/src/onnx/onnx_importer.cpp b/modules/dnn/src/onnx/onnx_importer.cpp index 651a2ab333..3668c9b51e 100644 --- a/modules/dnn/src/onnx/onnx_importer.cpp +++ b/modules/dnn/src/onnx/onnx_importer.cpp @@ -1792,6 +1792,23 @@ void ONNXImporter::handleNode(const opencv_onnx::NodeProto& node_proto_) addConstant(layerParams.name, concatenated[0]); return; } + else + { + for (int i = 0; i < node_proto.input_size(); ++i) + { + if (constBlobs.find(node_proto.input(i)) != constBlobs.end()) + { + LayerParams constParams; + constParams.name = node_proto.input(i); + constParams.type = "Const"; + constParams.blobs.push_back(getBlob(node_proto, i)); + + opencv_onnx::NodeProto proto; + proto.add_output(constParams.name); + addLayer(constParams, proto); + } + } + } } else if (layer_type == "Resize") { diff --git a/modules/dnn/test/test_onnx_importer.cpp b/modules/dnn/test/test_onnx_importer.cpp index c4cb877172..600f727d7d 100644 --- a/modules/dnn/test/test_onnx_importer.cpp +++ b/modules/dnn/test/test_onnx_importer.cpp @@ -327,6 +327,7 @@ TEST_P(Test_ONNX_layers, Concatenation) if (target == DNN_TARGET_MYRIAD) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD, CV_TEST_TAG_DNN_SKIP_IE_NN_BUILDER); } testONNXModels("concatenation"); + testONNXModels("concat_const_blobs"); } TEST_P(Test_ONNX_layers, Eltwise3D) From 9fe49497bb8b3e9c5b2043b4c74d3af495a66927 Mon Sep 17 00:00:00 2001 From: Anatoliy Talamanov Date: Thu, 1 Jul 2021 12:36:19 +0300 Subject: [PATCH 033/262] Merge pull request #20284 from TolyaTalamanov:at/wrap-render G-API: Wrap render functionality to python * Wrap render Rect prim * Add all primitives and tests * Cover mosaic and image * Handle error in pyopencv_to(Prim) * Move Mosaic and Rect ctors wrappers to shadow file * Use GAPI_PROP_RW * Fix indent --- .../gapi/include/opencv2/gapi/own/exports.hpp | 2 + .../include/opencv2/gapi/render/render.hpp | 24 +- .../opencv2/gapi/render/render_types.hpp | 100 ++++---- .../gapi/misc/python/package/gapi/__init__.py | 45 +++- modules/gapi/misc/python/pyopencv_gapi.hpp | 116 ++++++--- modules/gapi/misc/python/python_bridge.hpp | 33 +-- modules/gapi/misc/python/shadow_gapi.hpp | 115 +++++---- .../gapi/misc/python/test/test_gapi_render.py | 227 ++++++++++++++++++ modules/gapi/src/api/render_ocv.cpp | 6 +- .../test/render/gapi_render_tests_ocv.cpp | 8 +- modules/python/src2/hdr_parser.py | 1 + 11 files changed, 512 insertions(+), 165 deletions(-) create mode 100644 modules/gapi/misc/python/test/test_gapi_render.py diff --git a/modules/gapi/include/opencv2/gapi/own/exports.hpp b/modules/gapi/include/opencv2/gapi/own/exports.hpp index 1978991b75..c36f4003d0 100644 --- a/modules/gapi/include/opencv2/gapi/own/exports.hpp +++ b/modules/gapi/include/opencv2/gapi/own/exports.hpp @@ -13,11 +13,13 @@ # define GAPI_EXPORTS CV_EXPORTS /* special informative macros for wrapper generators */ # define GAPI_PROP CV_PROP +# define GAPI_PROP_RW CV_PROP_RW # define GAPI_WRAP CV_WRAP # define GAPI_EXPORTS_W_SIMPLE CV_EXPORTS_W_SIMPLE # define GAPI_EXPORTS_W CV_EXPORTS_W # else # define GAPI_PROP +# define GAPI_PROP_RW # define GAPI_WRAP # define GAPI_EXPORTS # define GAPI_EXPORTS_W_SIMPLE diff --git a/modules/gapi/include/opencv2/gapi/render/render.hpp b/modules/gapi/include/opencv2/gapi/render/render.hpp index 6bfe92388a..5375412224 100644 --- a/modules/gapi/include/opencv2/gapi/render/render.hpp +++ b/modules/gapi/include/opencv2/gapi/render/render.hpp @@ -81,9 +81,9 @@ using GMatDesc2 = std::tuple; @param prims vector of drawing primitivies @param args graph compile time parameters */ -void GAPI_EXPORTS render(cv::Mat& bgr, - const Prims& prims, - cv::GCompileArgs&& args = {}); +void GAPI_EXPORTS_W render(cv::Mat& bgr, + const Prims& prims, + cv::GCompileArgs&& args = {}); /** @brief The function renders on two NV12 planes passed drawing primitivies @@ -92,10 +92,10 @@ void GAPI_EXPORTS render(cv::Mat& bgr, @param prims vector of drawing primitivies @param args graph compile time parameters */ -void GAPI_EXPORTS render(cv::Mat& y_plane, - cv::Mat& uv_plane, - const Prims& prims, - cv::GCompileArgs&& args = {}); +void GAPI_EXPORTS_W render(cv::Mat& y_plane, + cv::Mat& uv_plane, + const Prims& prims, + cv::GCompileArgs&& args = {}); /** @brief The function renders on the input media frame passed drawing primitivies @@ -139,7 +139,7 @@ Output image must be 8-bit unsigned planar 3-channel image @param src input image: 8-bit unsigned 3-channel image @ref CV_8UC3 @param prims draw primitives */ -GAPI_EXPORTS GMat render3ch(const GMat& src, const GArray& prims); +GAPI_EXPORTS_W GMat render3ch(const GMat& src, const GArray& prims); /** @brief Renders on two planes @@ -150,9 +150,9 @@ uv image must be 8-bit unsigned planar 2-channel image @ref CV_8UC2 @param uv input image: 8-bit unsigned 2-channel image @ref CV_8UC2 @param prims draw primitives */ -GAPI_EXPORTS GMat2 renderNV12(const GMat& y, - const GMat& uv, - const GArray& prims); +GAPI_EXPORTS_W GMat2 renderNV12(const GMat& y, + const GMat& uv, + const GArray& prims); /** @brief Renders Media Frame @@ -177,7 +177,7 @@ namespace render { namespace ocv { - GAPI_EXPORTS cv::gapi::GKernelPackage kernels(); + GAPI_EXPORTS_W cv::gapi::GKernelPackage kernels(); } // namespace ocv } // namespace render diff --git a/modules/gapi/include/opencv2/gapi/render/render_types.hpp b/modules/gapi/include/opencv2/gapi/render/render_types.hpp index ca403be361..6d70e3a877 100644 --- a/modules/gapi/include/opencv2/gapi/render/render_types.hpp +++ b/modules/gapi/include/opencv2/gapi/render/render_types.hpp @@ -41,7 +41,7 @@ struct freetype_font * * Parameters match cv::putText(). */ -struct Text +struct GAPI_EXPORTS_W_SIMPLE Text { /** * @brief Text constructor @@ -55,6 +55,7 @@ struct Text * @param lt_ The line type. See #LineTypes * @param bottom_left_origin_ When true, the image data origin is at the bottom-left corner. Otherwise, it is at the top-left corner */ + GAPI_WRAP Text(const std::string& text_, const cv::Point& org_, int ff_, @@ -68,17 +69,18 @@ struct Text { } + GAPI_WRAP Text() = default; /*@{*/ - std::string text; //!< The text string to be drawn - cv::Point org; //!< The bottom-left corner of the text string in the image - int ff; //!< The font type, see #HersheyFonts - double fs; //!< The font scale factor that is multiplied by the font-specific base size - cv::Scalar color; //!< The text color - int thick; //!< The thickness of the lines used to draw a text - int lt; //!< The line type. See #LineTypes - bool bottom_left_origin; //!< When true, the image data origin is at the bottom-left corner. Otherwise, it is at the top-left corner + GAPI_PROP_RW std::string text; //!< The text string to be drawn + GAPI_PROP_RW cv::Point org; //!< The bottom-left corner of the text string in the image + GAPI_PROP_RW int ff; //!< The font type, see #HersheyFonts + GAPI_PROP_RW double fs; //!< The font scale factor that is multiplied by the font-specific base size + GAPI_PROP_RW cv::Scalar color; //!< The text color + GAPI_PROP_RW int thick; //!< The thickness of the lines used to draw a text + GAPI_PROP_RW int lt; //!< The line type. See #LineTypes + GAPI_PROP_RW bool bottom_left_origin; //!< When true, the image data origin is at the bottom-left corner. Otherwise, it is at the top-left corner /*@{*/ }; @@ -122,7 +124,7 @@ struct FText * * Parameters match cv::rectangle(). */ -struct Rect +struct GAPI_EXPORTS_W_SIMPLE Rect { /** * @brief Rect constructor @@ -142,14 +144,15 @@ struct Rect { } + GAPI_WRAP Rect() = default; /*@{*/ - cv::Rect rect; //!< Coordinates of the rectangle - cv::Scalar color; //!< The rectangle color or brightness (grayscale image) - int thick; //!< The thickness of lines that make up the rectangle. Negative values, like #FILLED, mean that the function has to draw a filled rectangle - int lt; //!< The type of the line. See #LineTypes - int shift; //!< The number of fractional bits in the point coordinates + GAPI_PROP_RW cv::Rect rect; //!< Coordinates of the rectangle + GAPI_PROP_RW cv::Scalar color; //!< The rectangle color or brightness (grayscale image) + GAPI_PROP_RW int thick; //!< The thickness of lines that make up the rectangle. Negative values, like #FILLED, mean that the function has to draw a filled rectangle + GAPI_PROP_RW int lt; //!< The type of the line. See #LineTypes + GAPI_PROP_RW int shift; //!< The number of fractional bits in the point coordinates /*@{*/ }; @@ -158,7 +161,7 @@ struct Rect * * Parameters match cv::circle(). */ -struct Circle +struct GAPI_EXPORTS_W_SIMPLE Circle { /** * @brief Circle constructor @@ -170,6 +173,7 @@ struct Circle * @param lt_ The Type of the circle boundary. See #LineTypes * @param shift_ The Number of fractional bits in the coordinates of the center and in the radius value */ + GAPI_WRAP Circle(const cv::Point& center_, int radius_, const cv::Scalar& color_, @@ -180,15 +184,16 @@ struct Circle { } + GAPI_WRAP Circle() = default; /*@{*/ - cv::Point center; //!< The center of the circle - int radius; //!< The radius of the circle - cv::Scalar color; //!< The color of the circle - int thick; //!< The thickness of the circle outline, if positive. Negative values, like #FILLED, mean that a filled circle is to be drawn - int lt; //!< The Type of the circle boundary. See #LineTypes - int shift; //!< The Number of fractional bits in the coordinates of the center and in the radius value + GAPI_PROP_RW cv::Point center; //!< The center of the circle + GAPI_PROP_RW int radius; //!< The radius of the circle + GAPI_PROP_RW cv::Scalar color; //!< The color of the circle + GAPI_PROP_RW int thick; //!< The thickness of the circle outline, if positive. Negative values, like #FILLED, mean that a filled circle is to be drawn + GAPI_PROP_RW int lt; //!< The Type of the circle boundary. See #LineTypes + GAPI_PROP_RW int shift; //!< The Number of fractional bits in the coordinates of the center and in the radius value /*@{*/ }; @@ -197,7 +202,7 @@ struct Circle * * Parameters match cv::line(). */ -struct Line +struct GAPI_EXPORTS_W_SIMPLE Line { /** * @brief Line constructor @@ -209,6 +214,7 @@ struct Line * @param lt_ The Type of the line. See #LineTypes * @param shift_ The number of fractional bits in the point coordinates */ + GAPI_WRAP Line(const cv::Point& pt1_, const cv::Point& pt2_, const cv::Scalar& color_, @@ -219,15 +225,16 @@ struct Line { } + GAPI_WRAP Line() = default; /*@{*/ - cv::Point pt1; //!< The first point of the line segment - cv::Point pt2; //!< The second point of the line segment - cv::Scalar color; //!< The line color - int thick; //!< The thickness of line - int lt; //!< The Type of the line. See #LineTypes - int shift; //!< The number of fractional bits in the point coordinates + GAPI_PROP_RW cv::Point pt1; //!< The first point of the line segment + GAPI_PROP_RW cv::Point pt2; //!< The second point of the line segment + GAPI_PROP_RW cv::Scalar color; //!< The line color + GAPI_PROP_RW int thick; //!< The thickness of line + GAPI_PROP_RW int lt; //!< The Type of the line. See #LineTypes + GAPI_PROP_RW int shift; //!< The number of fractional bits in the point coordinates /*@{*/ }; @@ -236,7 +243,7 @@ struct Line * * Mosaicing is a very basic method to obfuscate regions in the image. */ -struct Mosaic +struct GAPI_EXPORTS_W_SIMPLE Mosaic { /** * @brief Mosaic constructor @@ -252,12 +259,13 @@ struct Mosaic { } + GAPI_WRAP Mosaic() : cellSz(0), decim(0) {} /*@{*/ - cv::Rect mos; //!< Coordinates of the mosaic - int cellSz; //!< Cell size (same for X, Y) - int decim; //!< Decimation (0 stands for no decimation) + GAPI_PROP_RW cv::Rect mos; //!< Coordinates of the mosaic + GAPI_PROP_RW int cellSz; //!< Cell size (same for X, Y) + GAPI_PROP_RW int decim; //!< Decimation (0 stands for no decimation) /*@{*/ }; @@ -266,7 +274,7 @@ struct Mosaic * * Image is blended on a frame using the specified mask. */ -struct Image +struct GAPI_EXPORTS_W_SIMPLE Image { /** * @brief Mosaic constructor @@ -275,6 +283,7 @@ struct Image * @param img_ Image to draw * @param alpha_ Alpha channel for image to draw (same size and number of channels) */ + GAPI_WRAP Image(const cv::Point& org_, const cv::Mat& img_, const cv::Mat& alpha_) : @@ -282,19 +291,20 @@ struct Image { } + GAPI_WRAP Image() = default; /*@{*/ - cv::Point org; //!< The bottom-left corner of the image - cv::Mat img; //!< Image to draw - cv::Mat alpha; //!< Alpha channel for image to draw (same size and number of channels) + GAPI_PROP_RW cv::Point org; //!< The bottom-left corner of the image + GAPI_PROP_RW cv::Mat img; //!< Image to draw + GAPI_PROP_RW cv::Mat alpha; //!< Alpha channel for image to draw (same size and number of channels) /*@{*/ }; /** * @brief This structure represents a polygon to draw. */ -struct Poly +struct GAPI_EXPORTS_W_SIMPLE Poly { /** * @brief Mosaic constructor @@ -305,6 +315,7 @@ struct Poly * @param lt_ The Type of the line. See #LineTypes * @param shift_ The number of fractional bits in the point coordinate */ + GAPI_WRAP Poly(const std::vector& points_, const cv::Scalar& color_, int thick_ = 1, @@ -314,14 +325,15 @@ struct Poly { } + GAPI_WRAP Poly() = default; /*@{*/ - std::vector points; //!< Points to connect - cv::Scalar color; //!< The line color - int thick; //!< The thickness of line - int lt; //!< The Type of the line. See #LineTypes - int shift; //!< The number of fractional bits in the point coordinate + GAPI_PROP_RW std::vector points; //!< Points to connect + GAPI_PROP_RW cv::Scalar color; //!< The line color + GAPI_PROP_RW int thick; //!< The thickness of line + GAPI_PROP_RW int lt; //!< The Type of the line. See #LineTypes + GAPI_PROP_RW int shift; //!< The number of fractional bits in the point coordinate /*@{*/ }; @@ -336,7 +348,7 @@ using Prim = util::variant , Poly >; -using Prims = std::vector; +using Prims = std::vector; //! @} gapi_draw_prims } // namespace draw diff --git a/modules/gapi/misc/python/package/gapi/__init__.py b/modules/gapi/misc/python/package/gapi/__init__.py index 587f641fd3..dc874f0b0c 100644 --- a/modules/gapi/misc/python/package/gapi/__init__.py +++ b/modules/gapi/misc/python/package/gapi/__init__.py @@ -84,6 +84,10 @@ class GOpaque(): def __new__(self): return cv.GOpaqueT(cv.gapi.CV_RECT) + class Prim(): + def __new__(self): + return cv.GOpaqueT(cv.gapi.CV_DRAW_PRIM) + class Any(): def __new__(self): return cv.GOpaqueT(cv.gapi.CV_ANY) @@ -143,6 +147,10 @@ class GArray(): def __new__(self): return cv.GArrayT(cv.gapi.CV_GMAT) + class Prim(): + def __new__(self): + return cv.GArray(cv.gapi.CV_DRAW_PRIM) + class Any(): def __new__(self): return cv.GArray(cv.gapi.CV_ANY) @@ -164,6 +172,7 @@ def op(op_id, in_types, out_types): cv.GArray.Scalar: cv.gapi.CV_SCALAR, cv.GArray.Mat: cv.gapi.CV_MAT, cv.GArray.GMat: cv.gapi.CV_GMAT, + cv.GArray.Prim: cv.gapi.CV_DRAW_PRIM, cv.GArray.Any: cv.gapi.CV_ANY } @@ -179,22 +188,24 @@ def op(op_id, in_types, out_types): cv.GOpaque.Point2f: cv.gapi.CV_POINT2F, cv.GOpaque.Size: cv.gapi.CV_SIZE, cv.GOpaque.Rect: cv.gapi.CV_RECT, + cv.GOpaque.Prim: cv.gapi.CV_DRAW_PRIM, cv.GOpaque.Any: cv.gapi.CV_ANY } type2str = { - cv.gapi.CV_BOOL: 'cv.gapi.CV_BOOL' , - cv.gapi.CV_INT: 'cv.gapi.CV_INT' , - cv.gapi.CV_DOUBLE: 'cv.gapi.CV_DOUBLE' , - cv.gapi.CV_FLOAT: 'cv.gapi.CV_FLOAT' , - cv.gapi.CV_STRING: 'cv.gapi.CV_STRING' , - cv.gapi.CV_POINT: 'cv.gapi.CV_POINT' , - cv.gapi.CV_POINT2F: 'cv.gapi.CV_POINT2F' , - cv.gapi.CV_SIZE: 'cv.gapi.CV_SIZE', - cv.gapi.CV_RECT: 'cv.gapi.CV_RECT', - cv.gapi.CV_SCALAR: 'cv.gapi.CV_SCALAR', - cv.gapi.CV_MAT: 'cv.gapi.CV_MAT', - cv.gapi.CV_GMAT: 'cv.gapi.CV_GMAT' + cv.gapi.CV_BOOL: 'cv.gapi.CV_BOOL' , + cv.gapi.CV_INT: 'cv.gapi.CV_INT' , + cv.gapi.CV_DOUBLE: 'cv.gapi.CV_DOUBLE' , + cv.gapi.CV_FLOAT: 'cv.gapi.CV_FLOAT' , + cv.gapi.CV_STRING: 'cv.gapi.CV_STRING' , + cv.gapi.CV_POINT: 'cv.gapi.CV_POINT' , + cv.gapi.CV_POINT2F: 'cv.gapi.CV_POINT2F' , + cv.gapi.CV_SIZE: 'cv.gapi.CV_SIZE', + cv.gapi.CV_RECT: 'cv.gapi.CV_RECT', + cv.gapi.CV_SCALAR: 'cv.gapi.CV_SCALAR', + cv.gapi.CV_MAT: 'cv.gapi.CV_MAT', + cv.gapi.CV_GMAT: 'cv.gapi.CV_GMAT', + cv.gapi.CV_DRAW_PRIM: 'cv.gapi.CV_DRAW_PRIM' } # NB: Second lvl decorator takes class to decorate @@ -274,3 +285,13 @@ def kernel(op_cls): return cls return kernel_with_params + + +# FIXME: On the c++ side every class is placed in cv2 module. +cv.gapi.wip.draw.Rect = cv.gapi_wip_draw_Rect +cv.gapi.wip.draw.Text = cv.gapi_wip_draw_Text +cv.gapi.wip.draw.Circle = cv.gapi_wip_draw_Circle +cv.gapi.wip.draw.Line = cv.gapi_wip_draw_Line +cv.gapi.wip.draw.Mosaic = cv.gapi_wip_draw_Mosaic +cv.gapi.wip.draw.Image = cv.gapi_wip_draw_Image +cv.gapi.wip.draw.Poly = cv.gapi_wip_draw_Poly diff --git a/modules/gapi/misc/python/pyopencv_gapi.hpp b/modules/gapi/misc/python/pyopencv_gapi.hpp index 6cd79e4a73..3c428dde6d 100644 --- a/modules/gapi/misc/python/pyopencv_gapi.hpp +++ b/modules/gapi/misc/python/pyopencv_gapi.hpp @@ -43,6 +43,7 @@ using GArray_Rect = cv::GArray; using GArray_Scalar = cv::GArray; using GArray_Mat = cv::GArray; using GArray_GMat = cv::GArray; +using GArray_Prim = cv::GArray; // FIXME: Python wrapper generate code without namespace std, // so it cause error: "string wasn't declared" @@ -125,6 +126,65 @@ PyObject* pyopencv_from(const cv::detail::PyObjectHolder& v) return o; } +// #FIXME: Is it possible to implement pyopencv_from/pyopencv_to for generic +// cv::variant ? +template <> +PyObject* pyopencv_from(const cv::gapi::wip::draw::Prim& prim) +{ + switch (prim.index()) { + case cv::gapi::wip::draw::Prim::index_of(): + return pyopencv_from(cv::util::get(prim)); + case cv::gapi::wip::draw::Prim::index_of(): + return pyopencv_from(cv::util::get(prim)); + case cv::gapi::wip::draw::Prim::index_of(): + return pyopencv_from(cv::util::get(prim)); + case cv::gapi::wip::draw::Prim::index_of(): + return pyopencv_from(cv::util::get(prim)); + case cv::gapi::wip::draw::Prim::index_of(): + return pyopencv_from(cv::util::get(prim)); + case cv::gapi::wip::draw::Prim::index_of(): + return pyopencv_from(cv::util::get(prim)); + case cv::gapi::wip::draw::Prim::index_of(): + return pyopencv_from(cv::util::get(prim)); + } + + util::throw_error(std::logic_error("Unsupported draw primitive type")); +} + +template <> +PyObject* pyopencv_from(const cv::gapi::wip::draw::Prims& value) +{ + return pyopencv_from_generic_vec(value); +} + +template<> +bool pyopencv_to(PyObject* obj, cv::gapi::wip::draw::Prim& value, const ArgInfo& info) +{ +#define TRY_EXTRACT(Prim) \ + if (PyObject_TypeCheck(obj, reinterpret_cast(pyopencv_gapi_wip_draw_##Prim##_TypePtr))) \ + { \ + value = reinterpret_cast(obj)->v; \ + return true; \ + } \ + + TRY_EXTRACT(Rect) + TRY_EXTRACT(Text) + TRY_EXTRACT(Circle) + TRY_EXTRACT(Line) + TRY_EXTRACT(Mosaic) + TRY_EXTRACT(Image) + TRY_EXTRACT(Poly) + + failmsg("Unsupported primitive type"); + return false; +} + +template <> +bool pyopencv_to(PyObject* obj, cv::gapi::wip::draw::Prims& value, const ArgInfo& info) +{ + return pyopencv_to_generic_vec(obj, value, info); +} + template<> PyObject* pyopencv_from(const cv::GArg& value) { @@ -137,21 +197,21 @@ PyObject* pyopencv_from(const cv::GArg& value) #define UNSUPPORTED(T) case cv::detail::OpaqueKind::CV_##T: break switch (value.opaque_kind) { - HANDLE_CASE(BOOL, bool); - HANDLE_CASE(INT, int); + HANDLE_CASE(BOOL, bool); + HANDLE_CASE(INT, int); HANDLE_CASE(INT64, int64_t); - HANDLE_CASE(DOUBLE, double); - HANDLE_CASE(FLOAT, float); - HANDLE_CASE(STRING, std::string); - HANDLE_CASE(POINT, cv::Point); - HANDLE_CASE(POINT2F, cv::Point2f); - HANDLE_CASE(SIZE, cv::Size); - HANDLE_CASE(RECT, cv::Rect); - HANDLE_CASE(SCALAR, cv::Scalar); - HANDLE_CASE(MAT, cv::Mat); - HANDLE_CASE(UNKNOWN, cv::detail::PyObjectHolder); + HANDLE_CASE(DOUBLE, double); + HANDLE_CASE(FLOAT, float); + HANDLE_CASE(STRING, std::string); + HANDLE_CASE(POINT, cv::Point); + HANDLE_CASE(POINT2F, cv::Point2f); + HANDLE_CASE(SIZE, cv::Size); + HANDLE_CASE(RECT, cv::Rect); + HANDLE_CASE(SCALAR, cv::Scalar); + HANDLE_CASE(MAT, cv::Mat); + HANDLE_CASE(UNKNOWN, cv::detail::PyObjectHolder); + HANDLE_CASE(DRAW_PRIM, cv::gapi::wip::draw::Prim); UNSUPPORTED(UINT64); - UNSUPPORTED(DRAW_PRIM); #undef HANDLE_CASE #undef UNSUPPORTED } @@ -205,10 +265,10 @@ PyObject* pyopencv_from(const cv::detail::OpaqueRef& o) case cv::detail::OpaqueKind::CV_SIZE : return pyopencv_from(o.rref()); case cv::detail::OpaqueKind::CV_RECT : return pyopencv_from(o.rref()); case cv::detail::OpaqueKind::CV_UNKNOWN : return pyopencv_from(o.rref()); + case cv::detail::OpaqueKind::CV_DRAW_PRIM : return pyopencv_from(o.rref()); case cv::detail::OpaqueKind::CV_UINT64 : break; case cv::detail::OpaqueKind::CV_SCALAR : break; case cv::detail::OpaqueKind::CV_MAT : break; - case cv::detail::OpaqueKind::CV_DRAW_PRIM : break; } PyErr_SetString(PyExc_TypeError, "Unsupported GOpaque type"); @@ -233,8 +293,8 @@ PyObject* pyopencv_from(const cv::detail::VectorRef& v) case cv::detail::OpaqueKind::CV_SCALAR : return pyopencv_from_generic_vec(v.rref()); case cv::detail::OpaqueKind::CV_MAT : return pyopencv_from_generic_vec(v.rref()); case cv::detail::OpaqueKind::CV_UNKNOWN : return pyopencv_from_generic_vec(v.rref()); + case cv::detail::OpaqueKind::CV_DRAW_PRIM : return pyopencv_from_generic_vec(v.rref()); case cv::detail::OpaqueKind::CV_UINT64 : break; - case cv::detail::OpaqueKind::CV_DRAW_PRIM : break; } PyErr_SetString(PyExc_TypeError, "Unsupported GArray type"); @@ -394,21 +454,21 @@ static cv::detail::VectorRef extract_vector_ref(PyObject* from, cv::detail::Opaq #define UNSUPPORTED(T) case cv::detail::OpaqueKind::CV_##T: break switch (kind) { - HANDLE_CASE(BOOL, bool); - HANDLE_CASE(INT, int); - HANDLE_CASE(DOUBLE, double); - HANDLE_CASE(FLOAT, float); - HANDLE_CASE(STRING, std::string); - HANDLE_CASE(POINT, cv::Point); - HANDLE_CASE(POINT2F, cv::Point2f); - HANDLE_CASE(SIZE, cv::Size); - HANDLE_CASE(RECT, cv::Rect); - HANDLE_CASE(SCALAR, cv::Scalar); - HANDLE_CASE(MAT, cv::Mat); - HANDLE_CASE(UNKNOWN, cv::GArg); + HANDLE_CASE(BOOL, bool); + HANDLE_CASE(INT, int); + HANDLE_CASE(DOUBLE, double); + HANDLE_CASE(FLOAT, float); + HANDLE_CASE(STRING, std::string); + HANDLE_CASE(POINT, cv::Point); + HANDLE_CASE(POINT2F, cv::Point2f); + HANDLE_CASE(SIZE, cv::Size); + HANDLE_CASE(RECT, cv::Rect); + HANDLE_CASE(SCALAR, cv::Scalar); + HANDLE_CASE(MAT, cv::Mat); + HANDLE_CASE(UNKNOWN, cv::GArg); + HANDLE_CASE(DRAW_PRIM, cv::gapi::wip::draw::Prim); UNSUPPORTED(UINT64); UNSUPPORTED(INT64); - UNSUPPORTED(DRAW_PRIM); #undef HANDLE_CASE #undef UNSUPPORTED } diff --git a/modules/gapi/misc/python/python_bridge.hpp b/modules/gapi/misc/python/python_bridge.hpp index b212babe45..11d1728730 100644 --- a/modules/gapi/misc/python/python_bridge.hpp +++ b/modules/gapi/misc/python/python_bridge.hpp @@ -10,6 +10,7 @@ #include #include #include +#include // Prim #define ID(T, E) T #define ID_(T, E) ID(T, E), @@ -24,21 +25,24 @@ GAPI_Assert(false && "Unsupported type"); \ } +using cv::gapi::wip::draw::Prim; + #define GARRAY_TYPE_LIST_G(G, G2) \ -WRAP_ARGS(bool , cv::gapi::ArgType::CV_BOOL, G) \ -WRAP_ARGS(int , cv::gapi::ArgType::CV_INT, G) \ -WRAP_ARGS(int64_t , cv::gapi::ArgType::CV_INT64, G) \ -WRAP_ARGS(double , cv::gapi::ArgType::CV_DOUBLE, G) \ -WRAP_ARGS(float , cv::gapi::ArgType::CV_FLOAT, G) \ -WRAP_ARGS(std::string , cv::gapi::ArgType::CV_STRING, G) \ -WRAP_ARGS(cv::Point , cv::gapi::ArgType::CV_POINT, G) \ -WRAP_ARGS(cv::Point2f , cv::gapi::ArgType::CV_POINT2F, G) \ -WRAP_ARGS(cv::Size , cv::gapi::ArgType::CV_SIZE, G) \ -WRAP_ARGS(cv::Rect , cv::gapi::ArgType::CV_RECT, G) \ -WRAP_ARGS(cv::Scalar , cv::gapi::ArgType::CV_SCALAR, G) \ -WRAP_ARGS(cv::Mat , cv::gapi::ArgType::CV_MAT, G) \ -WRAP_ARGS(cv::GArg , cv::gapi::ArgType::CV_ANY, G) \ -WRAP_ARGS(cv::GMat , cv::gapi::ArgType::CV_GMAT, G2) \ +WRAP_ARGS(bool , cv::gapi::ArgType::CV_BOOL, G) \ +WRAP_ARGS(int , cv::gapi::ArgType::CV_INT, G) \ +WRAP_ARGS(int64_t , cv::gapi::ArgType::CV_INT64, G) \ +WRAP_ARGS(double , cv::gapi::ArgType::CV_DOUBLE, G) \ +WRAP_ARGS(float , cv::gapi::ArgType::CV_FLOAT, G) \ +WRAP_ARGS(std::string , cv::gapi::ArgType::CV_STRING, G) \ +WRAP_ARGS(cv::Point , cv::gapi::ArgType::CV_POINT, G) \ +WRAP_ARGS(cv::Point2f , cv::gapi::ArgType::CV_POINT2F, G) \ +WRAP_ARGS(cv::Size , cv::gapi::ArgType::CV_SIZE, G) \ +WRAP_ARGS(cv::Rect , cv::gapi::ArgType::CV_RECT, G) \ +WRAP_ARGS(cv::Scalar , cv::gapi::ArgType::CV_SCALAR, G) \ +WRAP_ARGS(cv::Mat , cv::gapi::ArgType::CV_MAT, G) \ +WRAP_ARGS(Prim , cv::gapi::ArgType::CV_DRAW_PRIM, G) \ +WRAP_ARGS(cv::GArg , cv::gapi::ArgType::CV_ANY, G) \ +WRAP_ARGS(cv::GMat , cv::gapi::ArgType::CV_GMAT, G2) \ #define GOPAQUE_TYPE_LIST_G(G, G2) \ WRAP_ARGS(bool , cv::gapi::ArgType::CV_BOOL, G) \ @@ -71,6 +75,7 @@ enum ArgType { CV_SCALAR, CV_MAT, CV_GMAT, + CV_DRAW_PRIM, CV_ANY, }; diff --git a/modules/gapi/misc/python/shadow_gapi.hpp b/modules/gapi/misc/python/shadow_gapi.hpp index e777aa5d93..41d0f19732 100644 --- a/modules/gapi/misc/python/shadow_gapi.hpp +++ b/modules/gapi/misc/python/shadow_gapi.hpp @@ -3,58 +3,77 @@ namespace cv { - struct GAPI_EXPORTS_W_SIMPLE GCompileArg { - GAPI_WRAP GCompileArg(gapi::GKernelPackage pkg); - GAPI_WRAP GCompileArg(gapi::GNetPackage pkg); - }; +struct GAPI_EXPORTS_W_SIMPLE GCompileArg { + GAPI_WRAP GCompileArg(gapi::GKernelPackage pkg); + GAPI_WRAP GCompileArg(gapi::GNetPackage pkg); +}; - class GAPI_EXPORTS_W_SIMPLE GInferInputs - { - public: - GAPI_WRAP GInferInputs(); - GAPI_WRAP GInferInputs& setInput(const std::string& name, const cv::GMat& value); - GAPI_WRAP GInferInputs& setInput(const std::string& name, const cv::GFrame& value); - }; +class GAPI_EXPORTS_W_SIMPLE GInferInputs +{ +public: + GAPI_WRAP GInferInputs(); + GAPI_WRAP GInferInputs& setInput(const std::string& name, const cv::GMat& value); + GAPI_WRAP GInferInputs& setInput(const std::string& name, const cv::GFrame& value); +}; - class GAPI_EXPORTS_W_SIMPLE GInferListInputs - { - public: - GAPI_WRAP GInferListInputs(); - GAPI_WRAP GInferListInputs setInput(const std::string& name, const cv::GArray& value); - GAPI_WRAP GInferListInputs setInput(const std::string& name, const cv::GArray& value); - }; +class GAPI_EXPORTS_W_SIMPLE GInferListInputs +{ +public: + GAPI_WRAP GInferListInputs(); + GAPI_WRAP GInferListInputs setInput(const std::string& name, const cv::GArray& value); + GAPI_WRAP GInferListInputs setInput(const std::string& name, const cv::GArray& value); +}; - class GAPI_EXPORTS_W_SIMPLE GInferOutputs - { - public: - GAPI_WRAP GInferOutputs(); - GAPI_WRAP cv::GMat at(const std::string& name); - }; +class GAPI_EXPORTS_W_SIMPLE GInferOutputs +{ +public: + GAPI_WRAP GInferOutputs(); + GAPI_WRAP cv::GMat at(const std::string& name); +}; - class GAPI_EXPORTS_W_SIMPLE GInferListOutputs - { - public: - GAPI_WRAP GInferListOutputs(); - GAPI_WRAP cv::GArray at(const std::string& name); - }; +class GAPI_EXPORTS_W_SIMPLE GInferListOutputs +{ +public: + GAPI_WRAP GInferListOutputs(); + GAPI_WRAP cv::GArray at(const std::string& name); +}; - namespace detail - { - gapi::GNetParam GAPI_EXPORTS_W strip(gapi::ie::PyParams params); - } // namespace detail +namespace gapi +{ +namespace wip +{ +class GAPI_EXPORTS_W IStreamSource { }; +namespace draw +{ + // NB: These render primitives are partially wrapped in shadow file + // because cv::Rect conflicts with cv::gapi::wip::draw::Rect in python generator + // and cv::Rect2i breaks standalone mode. + struct Rect + { + GAPI_WRAP Rect(const cv::Rect2i& rect_, + const cv::Scalar& color_, + int thick_ = 1, + int lt_ = 8, + int shift_ = 0); + }; - namespace gapi - { - namespace streaming - { - // FIXME: Extend to work with an arbitrary G-type. - cv::GOpaque GAPI_EXPORTS_W timestamp(cv::GMat); - cv::GOpaque GAPI_EXPORTS_W seqNo(cv::GMat); - cv::GOpaque GAPI_EXPORTS_W seq_id(cv::GMat); - } // namespace streaming - namespace wip - { - class GAPI_EXPORTS_W IStreamSource { }; - } // namespace wip - } // namespace gapi + struct Mosaic + { + GAPI_WRAP Mosaic(const cv::Rect2i& mos_, int cellSz_, int decim_); + }; +} // namespace draw +} // namespace wip +namespace streaming +{ + // FIXME: Extend to work with an arbitrary G-type. + cv::GOpaque GAPI_EXPORTS_W timestamp(cv::GMat); + cv::GOpaque GAPI_EXPORTS_W seqNo(cv::GMat); + cv::GOpaque GAPI_EXPORTS_W seq_id(cv::GMat); +} // namespace streaming +} // namespace gapi + +namespace detail +{ + gapi::GNetParam GAPI_EXPORTS_W strip(gapi::ie::PyParams params); +} // namespace detail } // namespace cv diff --git a/modules/gapi/misc/python/test/test_gapi_render.py b/modules/gapi/misc/python/test/test_gapi_render.py new file mode 100644 index 0000000000..70601a72e5 --- /dev/null +++ b/modules/gapi/misc/python/test/test_gapi_render.py @@ -0,0 +1,227 @@ +#!/usr/bin/env python + +import numpy as np +import cv2 as cv +import os +import sys +import unittest + +from tests_common import NewOpenCVTests + +try: + + if sys.version_info[:2] < (3, 0): + raise unittest.SkipTest('Python 2.x is not supported') + + # FIXME: FText isn't supported yet. + class gapi_render_test(NewOpenCVTests): + def __init__(self, *args): + super().__init__(*args) + + self.size = (300, 300, 3) + + # Rect + self.rect = (30, 30, 50, 50) + self.rcolor = (0, 255, 0) + self.rlt = cv.LINE_4 + self.rthick = 2 + self.rshift = 3 + + # Text + self.text = 'Hello, world!' + self.org = (100, 100) + self.ff = cv.FONT_HERSHEY_SIMPLEX + self.fs = 1.0 + self.tthick = 2 + self.tlt = cv.LINE_8 + self.tcolor = (255, 255, 255) + self.blo = False + + # Circle + self.center = (200, 200) + self.radius = 200 + self.ccolor = (255, 255, 0) + self.cthick = 2 + self.clt = cv.LINE_4 + self.cshift = 1 + + # Line + self.pt1 = (50, 50) + self.pt2 = (200, 200) + self.lcolor = (0, 255, 128) + self.lthick = 5 + self.llt = cv.LINE_8 + self.lshift = 2 + + # Poly + self.pts = [(50, 100), (100, 200), (25, 250)] + self.pcolor = (0, 0, 255) + self.pthick = 3 + self.plt = cv.LINE_4 + self.pshift = 1 + + # Image + self.iorg = (150, 150) + img_path = self.find_file('cv/face/david2.jpg', [os.environ.get('OPENCV_TEST_DATA_PATH')]) + self.img = cv.resize(cv.imread(img_path), (50, 50)) + self.alpha = np.full(self.img.shape[:2], 0.8, dtype=np.float32) + + # Mosaic + self.mos = (100, 100, 100, 100) + self.cell_sz = 25 + self.decim = 0 + + # Render primitives + self.prims = [cv.gapi.wip.draw.Rect(self.rect, self.rcolor, self.rthick, self.rlt, self.rshift), + cv.gapi.wip.draw.Text(self.text, self.org, self.ff, self.fs, self.tcolor, self.tthick, self.tlt, self.blo), + cv.gapi.wip.draw.Circle(self.center, self.radius, self.ccolor, self.cthick, self.clt, self.cshift), + cv.gapi.wip.draw.Line(self.pt1, self.pt2, self.lcolor, self.lthick, self.llt, self.lshift), + cv.gapi.wip.draw.Mosaic(self.mos, self.cell_sz, self.decim), + cv.gapi.wip.draw.Image(self.iorg, self.img, self.alpha), + cv.gapi.wip.draw.Poly(self.pts, self.pcolor, self.pthick, self.plt, self.pshift)] + + def cvt_nv12_to_yuv(self, y, uv): + h,w,_ = uv.shape + upsample_uv = cv.resize(uv, (h * 2, w * 2)) + return cv.merge([y, upsample_uv]) + + def cvt_yuv_to_nv12(self, yuv, y_out, uv_out): + chs = cv.split(yuv, [y_out, None, None]) + uv = cv.merge([chs[1], chs[2]]) + uv_out = cv.resize(uv, (uv.shape[0] // 2, uv.shape[1] // 2), dst=uv_out) + return y_out, uv_out + + def cvt_bgr_to_yuv_color(self, bgr): + y = bgr[2] * 0.299000 + bgr[1] * 0.587000 + bgr[0] * 0.114000; + u = bgr[2] * -0.168736 + bgr[1] * -0.331264 + bgr[0] * 0.500000 + 128; + v = bgr[2] * 0.500000 + bgr[1] * -0.418688 + bgr[0] * -0.081312 + 128; + return (y, u, v) + + def blend_img(self, background, org, img, alpha): + x, y = org + h, w, _ = img.shape + roi_img = background[x:x+w, y:y+h, :] + img32f_w = cv.merge([alpha] * 3).astype(np.float32) + roi32f_w = np.full(roi_img.shape, 1.0, dtype=np.float32) + roi32f_w -= img32f_w + img32f = (img / 255).astype(np.float32) + roi32f = (roi_img / 255).astype(np.float32) + cv.multiply(img32f, img32f_w, dst=img32f) + cv.multiply(roi32f, roi32f_w, dst=roi32f) + roi32f += img32f + roi_img[...] = np.round(roi32f * 255) + + # This is quite naive implementations used as a simple reference + # doesn't consider corner cases. + def draw_mosaic(self, img, mos, cell_sz, decim): + x,y,w,h = mos + mosaic_area = img[x:x+w, y:y+h, :] + for i in range(0, mosaic_area.shape[0], cell_sz): + for j in range(0, mosaic_area.shape[1], cell_sz): + cell_roi = mosaic_area[j:j+cell_sz, i:i+cell_sz, :] + s0, s1, s2 = cv.mean(cell_roi)[:3] + mosaic_area[j:j+cell_sz, i:i+cell_sz] = (round(s0), round(s1), round(s2)) + + def render_primitives_bgr_ref(self, img): + cv.rectangle(img, self.rect, self.rcolor, self.rthick, self.rlt, self.rshift) + cv.putText(img, self.text, self.org, self.ff, self.fs, self.tcolor, self.tthick, self.tlt, self.blo) + cv.circle(img, self.center, self.radius, self.ccolor, self.cthick, self.clt, self.cshift) + cv.line(img, self.pt1, self.pt2, self.lcolor, self.lthick, self.llt, self.lshift) + cv.fillPoly(img, np.expand_dims(np.array([self.pts]), axis=0), self.pcolor, self.plt, self.pshift) + self.draw_mosaic(img, self.mos, self.cell_sz, self.decim) + self.blend_img(img, self.iorg, self.img, self.alpha) + + def render_primitives_nv12_ref(self, y_plane, uv_plane): + yuv = self.cvt_nv12_to_yuv(y_plane, uv_plane) + cv.rectangle(yuv, self.rect, self.cvt_bgr_to_yuv_color(self.rcolor), self.rthick, self.rlt, self.rshift) + cv.putText(yuv, self.text, self.org, self.ff, self.fs, self.cvt_bgr_to_yuv_color(self.tcolor), self.tthick, self.tlt, self.blo) + cv.circle(yuv, self.center, self.radius, self.cvt_bgr_to_yuv_color(self.ccolor), self.cthick, self.clt, self.cshift) + cv.line(yuv, self.pt1, self.pt2, self.cvt_bgr_to_yuv_color(self.lcolor), self.lthick, self.llt, self.lshift) + cv.fillPoly(yuv, np.expand_dims(np.array([self.pts]), axis=0), self.cvt_bgr_to_yuv_color(self.pcolor), self.plt, self.pshift) + self.draw_mosaic(yuv, self.mos, self.cell_sz, self.decim) + self.blend_img(yuv, self.iorg, cv.cvtColor(self.img, cv.COLOR_BGR2YUV), self.alpha) + self.cvt_yuv_to_nv12(yuv, y_plane, uv_plane) + + def test_render_primitives_on_bgr_graph(self): + expected = np.zeros(self.size, dtype=np.uint8) + actual = np.array(expected, copy=True) + + # OpenCV + self.render_primitives_bgr_ref(expected) + + # G-API + g_in = cv.GMat() + g_prims = cv.GArray.Prim() + g_out = cv.gapi.wip.draw.render3ch(g_in, g_prims) + + + comp = cv.GComputation(cv.GIn(g_in, g_prims), cv.GOut(g_out)) + actual = comp.apply(cv.gin(actual, self.prims)) + + self.assertEqual(0.0, cv.norm(expected, actual, cv.NORM_INF)) + + def test_render_primitives_on_bgr_function(self): + expected = np.zeros(self.size, dtype=np.uint8) + actual = np.array(expected, copy=True) + + # OpenCV + self.render_primitives_bgr_ref(expected) + + # G-API + cv.gapi.wip.draw.render(actual, self.prims) + self.assertEqual(0.0, cv.norm(expected, actual, cv.NORM_INF)) + + def test_render_primitives_on_nv12_graph(self): + y_expected = np.zeros((self.size[0], self.size[1], 1), dtype=np.uint8) + uv_expected = np.zeros((self.size[0] // 2, self.size[1] // 2, 2), dtype=np.uint8) + + y_actual = np.array(y_expected, copy=True) + uv_actual = np.array(uv_expected, copy=True) + + # OpenCV + self.render_primitives_nv12_ref(y_expected, uv_expected) + + # G-API + g_y = cv.GMat() + g_uv = cv.GMat() + g_prims = cv.GArray.Prim() + g_out_y, g_out_uv = cv.gapi.wip.draw.renderNV12(g_y, g_uv, g_prims) + + comp = cv.GComputation(cv.GIn(g_y, g_uv, g_prims), cv.GOut(g_out_y, g_out_uv)) + y_actual, uv_actual = comp.apply(cv.gin(y_actual, uv_actual, self.prims)) + + self.assertEqual(0.0, cv.norm(y_expected, y_actual, cv.NORM_INF)) + self.assertEqual(0.0, cv.norm(uv_expected, uv_actual, cv.NORM_INF)) + + def test_render_primitives_on_nv12_function(self): + y_expected = np.zeros((self.size[0], self.size[1], 1), dtype=np.uint8) + uv_expected = np.zeros((self.size[0] // 2, self.size[1] // 2, 2), dtype=np.uint8) + + y_actual = np.array(y_expected, copy=True) + uv_actual = np.array(uv_expected, copy=True) + + # OpenCV + self.render_primitives_nv12_ref(y_expected, uv_expected) + + # G-API + cv.gapi.wip.draw.render(y_actual, uv_actual, self.prims) + + self.assertEqual(0.0, cv.norm(y_expected, y_actual, cv.NORM_INF)) + self.assertEqual(0.0, cv.norm(uv_expected, uv_actual, cv.NORM_INF)) + + +except unittest.SkipTest as e: + + message = str(e) + + class TestSkip(unittest.TestCase): + def setUp(self): + self.skipTest('Skip tests: ' + message) + + def test_skip(): + pass + + pass + +if __name__ == '__main__': + NewOpenCVTests.bootstrap() diff --git a/modules/gapi/src/api/render_ocv.cpp b/modules/gapi/src/api/render_ocv.cpp index 5ab2e1dd07..f1e9be4b48 100644 --- a/modules/gapi/src/api/render_ocv.cpp +++ b/modules/gapi/src/api/render_ocv.cpp @@ -159,7 +159,7 @@ void drawPrimitivesOCV(cv::Mat& in, { const auto& rp = cv::util::get(p); const auto color = converter.cvtColor(rp.color); - cv::rectangle(in, rp.rect, color , rp.thick); + cv::rectangle(in, rp.rect, color, rp.thick, rp.lt, rp.shift); break; } @@ -198,7 +198,7 @@ void drawPrimitivesOCV(cv::Mat& in, { const auto& cp = cv::util::get(p); const auto color = converter.cvtColor(cp.color); - cv::circle(in, cp.center, cp.radius, color, cp.thick); + cv::circle(in, cp.center, cp.radius, color, cp.thick, cp.lt, cp.shift); break; } @@ -206,7 +206,7 @@ void drawPrimitivesOCV(cv::Mat& in, { const auto& lp = cv::util::get(p); const auto color = converter.cvtColor(lp.color); - cv::line(in, lp.pt1, lp.pt2, color, lp.thick); + cv::line(in, lp.pt1, lp.pt2, color, lp.thick, lp.lt, lp.shift); break; } diff --git a/modules/gapi/test/render/gapi_render_tests_ocv.cpp b/modules/gapi/test/render/gapi_render_tests_ocv.cpp index 010df5dff7..95f6954156 100644 --- a/modules/gapi/test/render/gapi_render_tests_ocv.cpp +++ b/modules/gapi/test/render/gapi_render_tests_ocv.cpp @@ -639,8 +639,8 @@ INSTANTIATE_TEST_CASE_P(RenderBGROCVTestRectsImpl, RenderBGROCVTestRects, Values(cv::Rect(100, 100, 200, 200)), Values(cv::Scalar(100, 50, 150)), Values(2), - Values(LINE_8), - Values(0))); + Values(LINE_8, LINE_4), + Values(0, 1))); INSTANTIATE_TEST_CASE_P(RenderNV12OCVTestRectsImpl, RenderNV12OCVTestRects, Combine(Values(cv::Size(1280, 720)), @@ -673,8 +673,8 @@ INSTANTIATE_TEST_CASE_P(RenderNV12OCVTestCirclesImpl, RenderNV12OCVTestCircles, Values(10), Values(cv::Scalar(100, 50, 150)), Values(2), - Values(LINE_8), - Values(0))); + Values(LINE_8, LINE_4), + Values(0, 1))); INSTANTIATE_TEST_CASE_P(RenderMFrameOCVTestCirclesImpl, RenderMFrameOCVTestCircles, Combine(Values(cv::Size(1280, 720)), diff --git a/modules/python/src2/hdr_parser.py b/modules/python/src2/hdr_parser.py index 412d41a4df..1e0f9b3a95 100755 --- a/modules/python/src2/hdr_parser.py +++ b/modules/python/src2/hdr_parser.py @@ -832,6 +832,7 @@ class CppHeaderParser(object): ("GAPI_EXPORTS_W_SIMPLE","CV_EXPORTS_W_SIMPLE"), ("GAPI_WRAP", "CV_WRAP"), ("GAPI_PROP", "CV_PROP"), + ("GAPI_PROP_RW", "CV_PROP_RW"), ('defined(GAPI_STANDALONE)', '0'), ]) From d70053aba528ec5ddf8d6906109c7cc9310654a5 Mon Sep 17 00:00:00 2001 From: Maxim Pashchenkov Date: Thu, 1 Jul 2021 13:27:28 +0300 Subject: [PATCH 034/262] Merge pull request #20144 from mpashchenkov:mp/python-ge G-API: Python. Gaze Estimation sample. * GE pep8 * Added function description, wrapped copy * Applying review comments * One more change * Added gin * Rstrt bb --- .../misc/python/samples/gaze_estimation.py | 467 ++++++++++++++++++ 1 file changed, 467 insertions(+) create mode 100644 modules/gapi/misc/python/samples/gaze_estimation.py diff --git a/modules/gapi/misc/python/samples/gaze_estimation.py b/modules/gapi/misc/python/samples/gaze_estimation.py new file mode 100644 index 0000000000..db190f67bb --- /dev/null +++ b/modules/gapi/misc/python/samples/gaze_estimation.py @@ -0,0 +1,467 @@ +import argparse +import time +import numpy as np +import cv2 as cv + +# ------------------------Service operations------------------------ +def weight_path(model_path): + """ Get path of weights based on path to IR + + Params: + model_path: the string contains path to IR file + + Return: + Path to weights file + """ + assert model_path.endswith('.xml'), "Wrong topology path was provided" + return model_path[:-3] + 'bin' + + +def build_argparser(): + """ Parse arguments from command line + + Return: + Pack of arguments from command line + """ + parser = argparse.ArgumentParser(description='This is an OpenCV-based version of Gaze Estimation example') + + parser.add_argument('--input', + help='Path to the input video file') + parser.add_argument('--out', + help='Path to the output video file') + parser.add_argument('--facem', + default='face-detection-retail-0005.xml', + help='Path to OpenVINO face detection model (.xml)') + parser.add_argument('--faced', + default='CPU', + help='Target device for the face detection' + + '(e.g. CPU, GPU, VPU, ...)') + parser.add_argument('--headm', + default='head-pose-estimation-adas-0001.xml', + help='Path to OpenVINO head pose estimation model (.xml)') + parser.add_argument('--headd', + default='CPU', + help='Target device for the head pose estimation inference ' + + '(e.g. CPU, GPU, VPU, ...)') + parser.add_argument('--landm', + default='facial-landmarks-35-adas-0002.xml', + help='Path to OpenVINO landmarks detector model (.xml)') + parser.add_argument('--landd', + default='CPU', + help='Target device for the landmarks detector (e.g. CPU, GPU, VPU, ...)') + parser.add_argument('--gazem', + default='gaze-estimation-adas-0002.xml', + help='Path to OpenVINO gaze vector estimaiton model (.xml)') + parser.add_argument('--gazed', + default='CPU', + help='Target device for the gaze vector estimation inference ' + + '(e.g. CPU, GPU, VPU, ...)') + parser.add_argument('--eyem', + default='open-closed-eye-0001.xml', + help='Path to OpenVINO open closed eye model (.xml)') + parser.add_argument('--eyed', + default='CPU', + help='Target device for the eyes state inference (e.g. CPU, GPU, VPU, ...)') + return parser + + +# ------------------------Support functions for custom kernels------------------------ +def intersection(surface, rect): + """ Remove zone of out of bound from ROI + + Params: + surface: image bounds is rect representation (top left coordinates and width and height) + rect: region of interest is also has rect representation + + Return: + Modified ROI with correct bounds + """ + l_x = max(surface[0], rect[0]) + l_y = max(surface[1], rect[1]) + width = min(surface[0] + surface[2], rect[0] + rect[2]) - l_x + height = min(surface[1] + surface[3], rect[1] + rect[3]) - l_y + if width < 0 or height < 0: + return (0, 0, 0, 0) + return (l_x, l_y, width, height) + + +def process_landmarks(r_x, r_y, r_w, r_h, landmarks): + """ Create points from result of inference of facial-landmarks network and size of input image + + Params: + r_x: x coordinate of top left corner of input image + r_y: y coordinate of top left corner of input image + r_w: width of input image + r_h: height of input image + landmarks: result of inference of facial-landmarks network + + Return: + Array of landmarks points for one face + """ + lmrks = landmarks[0] + raw_x = lmrks[::2] * r_w + r_x + raw_y = lmrks[1::2] * r_h + r_y + return np.array([[int(x), int(y)] for x, y in zip(raw_x, raw_y)]) + + +def eye_box(p_1, p_2, scale=1.8): + """ Get bounding box of eye + + Params: + p_1: point of left edge of eye + p_2: point of right edge of eye + scale: change size of box with this value + + Return: + Bounding box of eye and its midpoint + """ + + size = np.linalg.norm(p_1 - p_2) + midpoint = (p_1 + p_2) / 2 + width = scale * size + height = width + p_x = midpoint[0] - (width / 2) + p_y = midpoint[1] - (height / 2) + return (int(p_x), int(p_y), int(width), int(height)), list(map(int, midpoint)) + + +# ------------------------Custom graph operations------------------------ +@cv.gapi.op('custom.GProcessPoses', + in_types=[cv.GArray.GMat, cv.GArray.GMat, cv.GArray.GMat], + out_types=[cv.GArray.GMat]) +class GProcessPoses: + @staticmethod + def outMeta(arr_desc0, arr_desc1, arr_desc2): + return cv.empty_array_desc() + + +@cv.gapi.op('custom.GParseEyes', + in_types=[cv.GArray.GMat, cv.GArray.Rect, cv.GOpaque.Size], + out_types=[cv.GArray.Rect, cv.GArray.Rect, cv.GArray.Point, cv.GArray.Point]) +class GParseEyes: + @staticmethod + def outMeta(arr_desc0, arr_desc1, arr_desc2): + return cv.empty_array_desc(), cv.empty_array_desc(), \ + cv.empty_array_desc(), cv.empty_array_desc() + + +@cv.gapi.op('custom.GGetStates', + in_types=[cv.GArray.GMat, cv.GArray.GMat], + out_types=[cv.GArray.Int, cv.GArray.Int]) +class GGetStates: + @staticmethod + def outMeta(arr_desc0, arr_desc1): + return cv.empty_array_desc(), cv.empty_array_desc() + + +# ------------------------Custom kernels------------------------ +@cv.gapi.kernel(GProcessPoses) +class GProcessPosesImpl: + """ Custom kernel. Processed poses of heads + """ + @staticmethod + def run(in_ys, in_ps, in_rs): + """ Сustom kernel executable code + + Params: + in_ys: yaw angle of head + in_ps: pitch angle of head + in_rs: roll angle of head + + Return: + Arrays with heads poses + """ + out_poses = [] + size = len(in_ys) + for i in range(size): + out_poses.append(np.array([in_ys[i][0], in_ps[i][0], in_rs[i][0]]).T) + return out_poses + + +@cv.gapi.kernel(GParseEyes) +class GParseEyesImpl: + """ Custom kernel. Get information about eyes + """ + @staticmethod + def run(in_landm_per_face, in_face_rcs, frame_size): + """ Сustom kernel executable code + + Params: + in_landm_per_face: landmarks from inference of facial-landmarks network for each face + in_face_rcs: bounding boxes for each face + frame_size: size of input image + + Return: + Arrays of ROI for left and right eyes, array of midpoints and + array of landmarks points + """ + left_eyes = [] + right_eyes = [] + midpoints = [] + lmarks = [] + num_faces = len(in_landm_per_face) + surface = (0, 0, *frame_size) + for i in range(num_faces): + rect = in_face_rcs[i] + points = process_landmarks(*rect, in_landm_per_face[i]) + for p in points: + lmarks.append(p) + size = int(len(in_landm_per_face[i][0]) / 2) + + rect, midpoint_l = eye_box(lmarks[0 + i * size], lmarks[1 + i * size]) + left_eyes.append(intersection(surface, rect)) + rect, midpoint_r = eye_box(lmarks[2 + i * size], lmarks[3 + i * size]) + right_eyes.append(intersection(surface, rect)) + midpoints += [midpoint_l, midpoint_r] + return left_eyes, right_eyes, midpoints, lmarks + + +@cv.gapi.kernel(GGetStates) +class GGetStatesImpl: + """ Custom kernel. Get state of eye - open or closed + """ + @staticmethod + def run(eyesl, eyesr): + """ Сustom kernel executable code + + Params: + eyesl: result of inference of open-closed-eye network for left eye + eyesr: result of inference of open-closed-eye network for right eye + + Return: + States of left eyes and states of right eyes + """ + size = len(eyesl) + out_l_st = [] + out_r_st = [] + for i in range(size): + for st in eyesl[i]: + out_l_st += [1 if st[0] < st[1] else 0] + for st in eyesr[i]: + out_r_st += [1 if st[0] < st[1] else 0] + return out_l_st, out_r_st + + +if __name__ == '__main__': + ARGUMENTS = build_argparser().parse_args() + + # ------------------------Demo's graph------------------------ + g_in = cv.GMat() + + # Detect faces + face_inputs = cv.GInferInputs() + face_inputs.setInput('data', g_in) + face_outputs = cv.gapi.infer('face-detection', face_inputs) + faces = face_outputs.at('detection_out') + + # Parse faces + sz = cv.gapi.streaming.size(g_in) + faces_rc = cv.gapi.parseSSD(faces, sz, 0.5, False, False) + + # Detect poses + head_inputs = cv.GInferInputs() + head_inputs.setInput('data', g_in) + face_outputs = cv.gapi.infer('head-pose', faces_rc, head_inputs) + angles_y = face_outputs.at('angle_y_fc') + angles_p = face_outputs.at('angle_p_fc') + angles_r = face_outputs.at('angle_r_fc') + + # Parse poses + heads_pos = GProcessPoses.on(angles_y, angles_p, angles_r) + + # Detect landmarks + landmark_inputs = cv.GInferInputs() + landmark_inputs.setInput('data', g_in) + landmark_outputs = cv.gapi.infer('facial-landmarks', faces_rc, + landmark_inputs) + landmark = landmark_outputs.at('align_fc3') + + # Parse landmarks + left_eyes, right_eyes, mids, lmarks = GParseEyes.on(landmark, faces_rc, sz) + + # Detect eyes + eyes_inputs = cv.GInferInputs() + eyes_inputs.setInput('input.1', g_in) + eyesl_outputs = cv.gapi.infer('open-closed-eye', left_eyes, eyes_inputs) + eyesr_outputs = cv.gapi.infer('open-closed-eye', right_eyes, eyes_inputs) + eyesl = eyesl_outputs.at('19') + eyesr = eyesr_outputs.at('19') + + # Process eyes states + l_eye_st, r_eye_st = GGetStates.on(eyesl, eyesr) + + # Gaze estimation + gaze_inputs = cv.GInferListInputs() + gaze_inputs.setInput('left_eye_image', left_eyes) + gaze_inputs.setInput('right_eye_image', right_eyes) + gaze_inputs.setInput('head_pose_angles', heads_pos) + gaze_outputs = cv.gapi.infer2('gaze-estimation', g_in, gaze_inputs) + gaze_vectors = gaze_outputs.at('gaze_vector') + + out = cv.gapi.copy(g_in) + # ------------------------End of graph------------------------ + + comp = cv.GComputation(cv.GIn(g_in), cv.GOut(out, + faces_rc, + left_eyes, + right_eyes, + gaze_vectors, + angles_y, + angles_p, + angles_r, + l_eye_st, + r_eye_st, + mids, + lmarks)) + + # Networks + face_net = cv.gapi.ie.params('face-detection', ARGUMENTS.facem, + weight_path(ARGUMENTS.facem), ARGUMENTS.faced) + head_pose_net = cv.gapi.ie.params('head-pose', ARGUMENTS.headm, + weight_path(ARGUMENTS.headm), ARGUMENTS.headd) + landmarks_net = cv.gapi.ie.params('facial-landmarks', ARGUMENTS.landm, + weight_path(ARGUMENTS.landm), ARGUMENTS.landd) + gaze_net = cv.gapi.ie.params('gaze-estimation', ARGUMENTS.gazem, + weight_path(ARGUMENTS.gazem), ARGUMENTS.gazed) + eye_net = cv.gapi.ie.params('open-closed-eye', ARGUMENTS.eyem, + weight_path(ARGUMENTS.eyem), ARGUMENTS.eyed) + + nets = cv.gapi.networks(face_net, head_pose_net, landmarks_net, gaze_net, eye_net) + + # Kernels pack + kernels = cv.gapi.kernels(GParseEyesImpl, GProcessPosesImpl, GGetStatesImpl) + + # ------------------------Execution part------------------------ + ccomp = comp.compileStreaming(args=cv.gapi.compile_args(kernels, nets)) + source = cv.gapi.wip.make_capture_src(ARGUMENTS.input) + ccomp.setSource(cv.gin(source)) + ccomp.start() + + frames = 0 + fps = 0 + print('Processing') + START_TIME = time.time() + + while True: + start_time_cycle = time.time() + has_frame, (oimg, + outr, + l_eyes, + r_eyes, + outg, + out_y, + out_p, + out_r, + out_st_l, + out_st_r, + out_mids, + outl) = ccomp.pull() + + if not has_frame: + break + + # Draw + GREEN = (0, 255, 0) + RED = (0, 0, 255) + WHITE = (255, 255, 255) + BLUE = (255, 0, 0) + PINK = (255, 0, 255) + YELLOW = (0, 255, 255) + + M_PI_180 = np.pi / 180 + M_PI_2 = np.pi / 2 + M_PI = np.pi + + FACES_SIZE = len(outr) + + for i, out_rect in enumerate(outr): + # Face box + cv.rectangle(oimg, out_rect, WHITE, 1) + rx, ry, rwidth, rheight = out_rect + + # Landmarks + lm_radius = int(0.01 * rwidth + 1) + lmsize = int(len(outl) / FACES_SIZE) + for j in range(lmsize): + cv.circle(oimg, outl[j + i * lmsize], lm_radius, YELLOW, -1) + + # Headposes + yaw = out_y[i] + pitch = out_p[i] + roll = out_r[i] + sin_y = np.sin(yaw[:] * M_PI_180) + sin_p = np.sin(pitch[:] * M_PI_180) + sin_r = np.sin(roll[:] * M_PI_180) + + cos_y = np.cos(yaw[:] * M_PI_180) + cos_p = np.cos(pitch[:] * M_PI_180) + cos_r = np.cos(roll[:] * M_PI_180) + + axis_length = 0.4 * rwidth + x_center = int(rx + rwidth / 2) + y_center = int(ry + rheight / 2) + + # center to right + cv.line(oimg, [x_center, y_center], + [int(x_center + axis_length * (cos_r * cos_y + sin_y * sin_p * sin_r)), + int(y_center + axis_length * cos_p * sin_r)], + RED, 2) + + # center to top + cv.line(oimg, [x_center, y_center], + [int(x_center + axis_length * (cos_r * sin_y * sin_p + cos_y * sin_r)), + int(y_center - axis_length * cos_p * cos_r)], + GREEN, 2) + + # center to forward + cv.line(oimg, [x_center, y_center], + [int(x_center + axis_length * sin_y * cos_p), + int(y_center + axis_length * sin_p)], + PINK, 2) + + scale_box = 0.002 * rwidth + cv.putText(oimg, "head pose: (y=%0.0f, p=%0.0f, r=%0.0f)" % + (np.round(yaw), np.round(pitch), np.round(roll)), + [int(rx), int(ry + rheight + 5 * rwidth / 100)], + cv.FONT_HERSHEY_PLAIN, scale_box * 2, WHITE, 1) + + # Eyes boxes + color_l = GREEN if out_st_l[i] else RED + cv.rectangle(oimg, l_eyes[i], color_l, 1) + color_r = GREEN if out_st_r[i] else RED + cv.rectangle(oimg, r_eyes[i], color_r, 1) + + # Gaze vectors + norm_gazes = np.linalg.norm(outg[i][0]) + gaze_vector = outg[i][0] / norm_gazes + + arrow_length = 0.4 * rwidth + gaze_arrow = [arrow_length * gaze_vector[0], -arrow_length * gaze_vector[1]] + left_arrow = [int(a+b) for a, b in zip(out_mids[0 + i * 2], gaze_arrow)] + right_arrow = [int(a+b) for a, b in zip(out_mids[1 + i * 2], gaze_arrow)] + if out_st_l[i]: + cv.arrowedLine(oimg, out_mids[0 + i * 2], left_arrow, BLUE, 2) + if out_st_r[i]: + cv.arrowedLine(oimg, out_mids[1 + i * 2], right_arrow, BLUE, 2) + + v0, v1, v2 = outg[i][0] + + gaze_angles = [180 / M_PI * (M_PI_2 + np.arctan2(v2, v0)), + 180 / M_PI * (M_PI_2 - np.arccos(v1 / norm_gazes))] + cv.putText(oimg, "gaze angles: (h=%0.0f, v=%0.0f)" % + (np.round(gaze_angles[0]), np.round(gaze_angles[1])), + [int(rx), int(ry + rheight + 12 * rwidth / 100)], + cv.FONT_HERSHEY_PLAIN, scale_box * 2, WHITE, 1) + + # Add FPS value to frame + cv.putText(oimg, "FPS: %0i" % (fps), [int(20), int(40)], + cv.FONT_HERSHEY_PLAIN, 2, RED, 2) + + # Show result + cv.imshow('Gaze Estimation', oimg) + + fps = int(1. / (time.time() - start_time_cycle)) + frames += 1 + EXECUTION_TIME = time.time() - START_TIME + print('Execution successful') + print('Mean FPS is ', int(frames / EXECUTION_TIME)) From fc799191f4f19c523343c41a99ec9e6e4269da9e Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Thu, 1 Jul 2021 13:48:11 +0000 Subject: [PATCH 035/262] gapi(ie): replace deprecated calls --- modules/gapi/src/backends/ie/giebackend/giewrapper.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/modules/gapi/src/backends/ie/giebackend/giewrapper.cpp b/modules/gapi/src/backends/ie/giebackend/giewrapper.cpp index ba0632d4f0..d4ec806e48 100644 --- a/modules/gapi/src/backends/ie/giebackend/giewrapper.cpp +++ b/modules/gapi/src/backends/ie/giebackend/giewrapper.cpp @@ -124,7 +124,11 @@ IE::Core giewrap::getPlugin(const GIEParam& params) { { try { +#if INF_ENGINE_RELEASE >= 2021040000 + plugin.AddExtension(std::make_shared(extlib), params.device_id); +#else plugin.AddExtension(IE::make_so_pointer(extlib), params.device_id); +#endif CV_LOG_INFO(NULL, "DNN-IE: Loaded extension plugin: " << extlib); break; } From 0f24d4d2a14d8f79eaa4ca0710d7770aeeb833f2 Mon Sep 17 00:00:00 2001 From: APrigarina Date: Tue, 29 Jun 2021 22:15:37 +0300 Subject: [PATCH 036/262] fix samples --- samples/python/gaussian_mix.py | 2 +- samples/python/hist.py | 2 +- samples/python/morphology.py | 7 +++++-- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/samples/python/gaussian_mix.py b/samples/python/gaussian_mix.py index 6a656647dd..4c1f86794c 100755 --- a/samples/python/gaussian_mix.py +++ b/samples/python/gaussian_mix.py @@ -28,7 +28,7 @@ def make_gaussians(cluster_n, img_size): return points, ref_distrs def draw_gaussain(img, mean, cov, color): - x, y = np.int32(mean) + x, y = mean w, u, _vt = cv.SVDecomp(cov) ang = np.arctan2(u[1, 0], u[0, 0])*(180/np.pi) s1, s2 = np.sqrt(w)*3.0 diff --git a/samples/python/hist.py b/samples/python/hist.py index 157d5ff0ba..8c1f4546a8 100755 --- a/samples/python/hist.py +++ b/samples/python/hist.py @@ -46,7 +46,7 @@ def hist_lines(im): im = cv.cvtColor(im,cv.COLOR_BGR2GRAY) hist_item = cv.calcHist([im],[0],None,[256],[0,256]) cv.normalize(hist_item,hist_item,0,255,cv.NORM_MINMAX) - hist=np.int32(np.around(hist_item)) + hist = np.int32(np.around(hist_item)) for x,y in enumerate(hist): cv.line(h,(x,0),(x,y[0]),(255,255,255)) y = np.flipud(h) diff --git a/samples/python/morphology.py b/samples/python/morphology.py index 9ecf5b0682..183f5e8288 100755 --- a/samples/python/morphology.py +++ b/samples/python/morphology.py @@ -50,8 +50,11 @@ def main(): cur_str_mode = str_modes.next() def update(dummy=None): - sz = cv.getTrackbarPos('op/size', 'morphology') - iters = cv.getTrackbarPos('iters', 'morphology') + try: # do not get trackbar position while trackbar is not created + sz = cv.getTrackbarPos('op/size', 'morphology') + iters = cv.getTrackbarPos('iters', 'morphology') + except: + return opers = cur_mode.split('/') if len(opers) > 1: sz = sz - 10 From 05f1939b0284c55f987b6caa92f34affd50dd454 Mon Sep 17 00:00:00 2001 From: Maxim Pashchenkov Date: Thu, 1 Jul 2021 22:06:35 +0300 Subject: [PATCH 037/262] Merge pull request #20298 from mpashchenkov:mp/python-desync G-API: Python. Desync. * Desync. GMat. * Alignment --- .../gapi/include/opencv2/gapi/gstreaming.hpp | 11 +- modules/gapi/misc/python/pyopencv_gapi.hpp | 114 ++++++++++++------ modules/gapi/misc/python/shadow_gapi.hpp | 31 ++--- .../misc/python/test/test_gapi_streaming.py | 68 ++++++++++- modules/gapi/src/compiler/gstreaming.cpp | 18 ++- modules/gapi/src/compiler/gstreaming_priv.hpp | 1 + .../gapi/src/executor/gstreamingexecutor.cpp | 78 ++++++++++++ .../gapi/src/executor/gstreamingexecutor.hpp | 3 + .../test/streaming/gapi_streaming_tests.cpp | 81 ++++++++++++- 9 files changed, 335 insertions(+), 70 deletions(-) diff --git a/modules/gapi/include/opencv2/gapi/gstreaming.hpp b/modules/gapi/include/opencv2/gapi/gstreaming.hpp index 47e103fd0e..50abe69f87 100644 --- a/modules/gapi/include/opencv2/gapi/gstreaming.hpp +++ b/modules/gapi/include/opencv2/gapi/gstreaming.hpp @@ -71,6 +71,15 @@ using GOptRunArgP = util::variant< >; using GOptRunArgsP = std::vector; +using GOptRunArg = util::variant< + optional, + optional, + optional, + optional, + optional +>; +using GOptRunArgs = std::vector; + namespace detail { template inline GOptRunArgP wrap_opt_arg(optional& arg) { @@ -255,7 +264,7 @@ public: // NB: Used from python /// @private -- Exclude this function from OpenCV documentation - GAPI_WRAP std::tuple pull(); + GAPI_WRAP std::tuple> pull(); /** * @brief Get some next available data from the pipeline. diff --git a/modules/gapi/misc/python/pyopencv_gapi.hpp b/modules/gapi/misc/python/pyopencv_gapi.hpp index 3c428dde6d..d378a91b5f 100644 --- a/modules/gapi/misc/python/pyopencv_gapi.hpp +++ b/modules/gapi/misc/python/pyopencv_gapi.hpp @@ -131,7 +131,8 @@ PyObject* pyopencv_from(const cv::detail::PyObjectHolder& v) template <> PyObject* pyopencv_from(const cv::gapi::wip::draw::Prim& prim) { - switch (prim.index()) { + switch (prim.index()) + { case cv::gapi::wip::draw::Prim::index_of(): return pyopencv_from(cv::util::get(prim)); case cv::gapi::wip::draw::Prim::index_of(): @@ -319,40 +320,69 @@ PyObject* pyopencv_from(const GRunArg& v) return pyopencv_from(util::get(v)); } - PyErr_SetString(PyExc_TypeError, "Failed to unpack GRunArgs"); + PyErr_SetString(PyExc_TypeError, "Failed to unpack GRunArgs. Index of variant is unknown"); + return NULL; +} + +template +PyObject* pyopencv_from(const cv::optional& opt) +{ + if (!opt.has_value()) + { + Py_RETURN_NONE; + } + return pyopencv_from(*opt); +} + +template <> +PyObject* pyopencv_from(const GOptRunArg& v) +{ + switch (v.index()) + { + case GOptRunArg::index_of>(): + return pyopencv_from(util::get>(v)); + + case GOptRunArg::index_of>(): + return pyopencv_from(util::get>(v)); + + case GOptRunArg::index_of>(): + return pyopencv_from(util::get>(v)); + + case GOptRunArg::index_of>(): + return pyopencv_from(util::get>(v)); + } + + PyErr_SetString(PyExc_TypeError, "Failed to unpack GOptRunArg. Index of variant is unknown"); return NULL; } template<> PyObject* pyopencv_from(const GRunArgs& value) { - size_t i, n = value.size(); + return value.size() == 1 ? pyopencv_from(value[0]) : pyopencv_from_generic_vec(value); +} - // NB: It doesn't make sense to return list with a single element - if (n == 1) +template<> +PyObject* pyopencv_from(const GOptRunArgs& value) +{ + return value.size() == 1 ? pyopencv_from(value[0]) : pyopencv_from_generic_vec(value); +} + +// FIXME: cv::variant should be wrapped once for all types. +template <> +PyObject* pyopencv_from(const cv::util::variant& v) +{ + using RunArgs = cv::util::variant; + switch (v.index()) { - PyObject* item = pyopencv_from(value[0]); - if(!item) - { - return NULL; - } - return item; + case RunArgs::index_of(): + return pyopencv_from(util::get(v)); + case RunArgs::index_of(): + return pyopencv_from(util::get(v)); } - PyObject* list = PyList_New(n); - for(i = 0; i < n; ++i) - { - PyObject* item = pyopencv_from(value[i]); - if(!item) - { - Py_DECREF(list); - PyErr_SetString(PyExc_TypeError, "Failed to unpack GRunArgs"); - return NULL; - } - PyList_SetItem(list, i, item); - } - - return list; + PyErr_SetString(PyExc_TypeError, "Failed to recognize kind of RunArgs. Index of variant is unknown"); + return NULL; } template @@ -634,7 +664,8 @@ static cv::GRunArgs run_py_kernel(cv::detail::PyObjectHolder kernel, cv::detail::PyObjectHolder result( PyObject_CallObject(kernel.get(), args.get()), false); - if (PyErr_Occurred()) { + if (PyErr_Occurred()) + { PyErr_PrintEx(0); PyErr_Clear(); throw std::logic_error("Python kernel failed with error!"); @@ -717,8 +748,9 @@ static cv::GMetaArgs get_meta_args(PyObject* tuple) } static GMetaArgs run_py_meta(cv::detail::PyObjectHolder out_meta, - const cv::GMetaArgs &meta, - const cv::GArgs &gargs) { + const cv::GMetaArgs &meta, + const cv::GArgs &gargs) +{ PyGILState_STATE gstate; gstate = PyGILState_Ensure(); @@ -760,7 +792,8 @@ static GMetaArgs run_py_meta(cv::detail::PyObjectHolder out_meta, cv::detail::PyObjectHolder result( PyObject_CallObject(out_meta.get(), args.get()), false); - if (PyErr_Occurred()) { + if (PyErr_Occurred()) + { PyErr_PrintEx(0); PyErr_Clear(); throw std::logic_error("Python outMeta failed with error!"); @@ -792,21 +825,24 @@ static PyObject* pyopencv_cv_gapi_kernels(PyObject* , PyObject* py_args, PyObjec PyObject* user_kernel = PyTuple_GetItem(py_args, i); PyObject* id_obj = PyObject_GetAttrString(user_kernel, "id"); - if (!id_obj) { + if (!id_obj) + { PyErr_SetString(PyExc_TypeError, "Python kernel should contain id, please use cv.gapi.kernel to define kernel"); return NULL; } PyObject* out_meta = PyObject_GetAttrString(user_kernel, "outMeta"); - if (!out_meta) { + if (!out_meta) + { PyErr_SetString(PyExc_TypeError, "Python kernel should contain outMeta, please use cv.gapi.kernel to define kernel"); return NULL; } PyObject* run = PyObject_GetAttrString(user_kernel, "run"); - if (!run) { + if (!run) + { PyErr_SetString(PyExc_TypeError, "Python kernel should contain run, please use cv.gapi.kernel to define kernel"); return NULL; @@ -951,9 +987,12 @@ struct PyOpenCV_Converter> if (PyObject_TypeCheck(obj, reinterpret_cast(pyopencv_GArrayT_TypePtr))) { auto& array = reinterpret_cast(obj)->v; - try { + try + { value = cv::util::get>(array.arg()); - } catch (...) { + } + catch (...) + { return false; } return true; @@ -974,9 +1013,12 @@ struct PyOpenCV_Converter> if (PyObject_TypeCheck(obj, reinterpret_cast(pyopencv_GOpaqueT_TypePtr))) { auto& opaque = reinterpret_cast(obj)->v; - try { + try + { value = cv::util::get>(opaque.arg()); - } catch (...) { + } + catch (...) + { return false; } return true; diff --git a/modules/gapi/misc/python/shadow_gapi.hpp b/modules/gapi/misc/python/shadow_gapi.hpp index 41d0f19732..0b489dde0f 100644 --- a/modules/gapi/misc/python/shadow_gapi.hpp +++ b/modules/gapi/misc/python/shadow_gapi.hpp @@ -3,39 +3,40 @@ namespace cv { -struct GAPI_EXPORTS_W_SIMPLE GCompileArg { - GAPI_WRAP GCompileArg(gapi::GKernelPackage pkg); - GAPI_WRAP GCompileArg(gapi::GNetPackage pkg); +struct GAPI_EXPORTS_W_SIMPLE GCompileArg +{ + GAPI_WRAP GCompileArg(gapi::GKernelPackage pkg); + GAPI_WRAP GCompileArg(gapi::GNetPackage pkg); }; class GAPI_EXPORTS_W_SIMPLE GInferInputs { public: - GAPI_WRAP GInferInputs(); - GAPI_WRAP GInferInputs& setInput(const std::string& name, const cv::GMat& value); - GAPI_WRAP GInferInputs& setInput(const std::string& name, const cv::GFrame& value); + GAPI_WRAP GInferInputs(); + GAPI_WRAP GInferInputs& setInput(const std::string& name, const cv::GMat& value); + GAPI_WRAP GInferInputs& setInput(const std::string& name, const cv::GFrame& value); }; class GAPI_EXPORTS_W_SIMPLE GInferListInputs { public: - GAPI_WRAP GInferListInputs(); - GAPI_WRAP GInferListInputs setInput(const std::string& name, const cv::GArray& value); - GAPI_WRAP GInferListInputs setInput(const std::string& name, const cv::GArray& value); + GAPI_WRAP GInferListInputs(); + GAPI_WRAP GInferListInputs setInput(const std::string& name, const cv::GArray& value); + GAPI_WRAP GInferListInputs setInput(const std::string& name, const cv::GArray& value); }; class GAPI_EXPORTS_W_SIMPLE GInferOutputs { public: - GAPI_WRAP GInferOutputs(); - GAPI_WRAP cv::GMat at(const std::string& name); + GAPI_WRAP GInferOutputs(); + GAPI_WRAP cv::GMat at(const std::string& name); }; class GAPI_EXPORTS_W_SIMPLE GInferListOutputs { public: - GAPI_WRAP GInferListOutputs(); - GAPI_WRAP cv::GArray at(const std::string& name); + GAPI_WRAP GInferListOutputs(); + GAPI_WRAP cv::GArray at(const std::string& name); }; namespace gapi @@ -69,11 +70,13 @@ namespace streaming cv::GOpaque GAPI_EXPORTS_W timestamp(cv::GMat); cv::GOpaque GAPI_EXPORTS_W seqNo(cv::GMat); cv::GOpaque GAPI_EXPORTS_W seq_id(cv::GMat); + + GAPI_EXPORTS_W cv::GMat desync(const cv::GMat &g); } // namespace streaming } // namespace gapi namespace detail { - gapi::GNetParam GAPI_EXPORTS_W strip(gapi::ie::PyParams params); + gapi::GNetParam GAPI_EXPORTS_W strip(gapi::ie::PyParams params); } // namespace detail } // namespace cv diff --git a/modules/gapi/misc/python/test/test_gapi_streaming.py b/modules/gapi/misc/python/test/test_gapi_streaming.py index 4ea88878ee..7ede1b5cf3 100644 --- a/modules/gapi/misc/python/test/test_gapi_streaming.py +++ b/modules/gapi/misc/python/test/test_gapi_streaming.py @@ -5,16 +5,35 @@ import cv2 as cv import os import sys import unittest +import time from tests_common import NewOpenCVTests try: - if sys.version_info[:2] < (3, 0): raise unittest.SkipTest('Python 2.x is not supported') + @cv.gapi.op('custom.delay', in_types=[cv.GMat], out_types=[cv.GMat]) + class GDelay: + """Delay for 10 ms.""" + + @staticmethod + def outMeta(desc): + return desc + + + @cv.gapi.kernel(GDelay) + class GDelayImpl: + """Implementation for GDelay operation.""" + + @staticmethod + def run(img): + time.sleep(0.01) + return img + + class test_gapi_streaming(NewOpenCVTests): def test_image_input(self): @@ -148,7 +167,7 @@ try: proc_num_frames += 1 if proc_num_frames == max_num_frames: - break; + break def test_video_good_features_to_track(self): @@ -242,6 +261,51 @@ try: if curr_frame_number == max_num_frames: break + def test_desync(self): + path = self.find_file('cv/video/768x576.avi', [os.environ['OPENCV_TEST_DATA_PATH']]) + + # G-API + g_in = cv.GMat() + g_out1 = cv.gapi.copy(g_in) + des = cv.gapi.streaming.desync(g_in) + g_out2 = GDelay.on(des) + + c = cv.GComputation(cv.GIn(g_in), cv.GOut(g_out1, g_out2)) + + kernels = cv.gapi.kernels(GDelayImpl) + ccomp = c.compileStreaming(args=cv.gapi.compile_args(kernels)) + source = cv.gapi.wip.make_capture_src(path) + ccomp.setSource(cv.gin(source)) + ccomp.start() + + # Assert + max_num_frames = 10 + proc_num_frames = 0 + + out_counter = 0 + desync_out_counter = 0 + none_counter = 0 + while True: + has_frame, (out1, out2) = ccomp.pull() + if not has_frame: + break + + if not out1 is None: + out_counter += 1 + if not out2 is None: + desync_out_counter += 1 + else: + none_counter += 1 + + proc_num_frames += 1 + if proc_num_frames == max_num_frames: + ccomp.stop() + break + + self.assertLess(0, proc_num_frames) + self.assertLess(desync_out_counter, out_counter) + self.assertLess(0, none_counter) + except unittest.SkipTest as e: diff --git a/modules/gapi/src/compiler/gstreaming.cpp b/modules/gapi/src/compiler/gstreaming.cpp index 3bdc0323b5..e45e770427 100644 --- a/modules/gapi/src/compiler/gstreaming.cpp +++ b/modules/gapi/src/compiler/gstreaming.cpp @@ -75,6 +75,11 @@ bool cv::GStreamingCompiled::Priv::pull(cv::GOptRunArgsP &&outs) return m_exec->pull(std::move(outs)); } +std::tuple> cv::GStreamingCompiled::Priv::pull() +{ + return m_exec->pull(); +} + bool cv::GStreamingCompiled::Priv::try_pull(cv::GRunArgsP &&outs) { return m_exec->try_pull(std::move(outs)); @@ -123,18 +128,9 @@ bool cv::GStreamingCompiled::pull(cv::GRunArgsP &&outs) return m_priv->pull(std::move(outs)); } -std::tuple cv::GStreamingCompiled::pull() +std::tuple> cv::GStreamingCompiled::pull() { - GRunArgs run_args; - GRunArgsP outs; - const auto& out_info = m_priv->outInfo(); - run_args.reserve(out_info.size()); - outs.reserve(out_info.size()); - - cv::detail::constructGraphOutputs(m_priv->outInfo(), run_args, outs); - - bool is_over = m_priv->pull(std::move(outs)); - return std::make_tuple(is_over, run_args); + return m_priv->pull(); } bool cv::GStreamingCompiled::pull(cv::GOptRunArgsP &&outs) diff --git a/modules/gapi/src/compiler/gstreaming_priv.hpp b/modules/gapi/src/compiler/gstreaming_priv.hpp index 59b19d4252..1b559ba310 100644 --- a/modules/gapi/src/compiler/gstreaming_priv.hpp +++ b/modules/gapi/src/compiler/gstreaming_priv.hpp @@ -46,6 +46,7 @@ public: void start(); bool pull(cv::GRunArgsP &&outs); bool pull(cv::GOptRunArgsP &&outs); + std::tuple> pull(); bool try_pull(cv::GRunArgsP &&outs); void stop(); diff --git a/modules/gapi/src/executor/gstreamingexecutor.cpp b/modules/gapi/src/executor/gstreamingexecutor.cpp index 74c96bdf3e..27049aef63 100644 --- a/modules/gapi/src/executor/gstreamingexecutor.cpp +++ b/modules/gapi/src/executor/gstreamingexecutor.cpp @@ -1017,6 +1017,49 @@ void check_DesyncObjectConsumedByMultipleIslands(const cv::gimpl::GIslandModel:: } // for(nodes) } +// NB: Construct GRunArgsP based on passed info and store the memory in passed cv::GRunArgs. +// Needed for python bridge, because in case python user doesn't pass output arguments to apply. +void constructOptGraphOutputs(const cv::GTypesInfo &out_info, + cv::GOptRunArgs &args, + cv::GOptRunArgsP &outs) +{ + for (auto&& info : out_info) + { + switch (info.shape) + { + case cv::GShape::GMAT: + { + args.emplace_back(cv::optional{}); + outs.emplace_back(&cv::util::get>(args.back())); + break; + } + case cv::GShape::GSCALAR: + { + args.emplace_back(cv::optional{}); + outs.emplace_back(&cv::util::get>(args.back())); + break; + } + case cv::GShape::GARRAY: + { + cv::detail::VectorRef ref; + cv::util::get(info.ctor)(ref); + args.emplace_back(cv::util::make_optional(std::move(ref))); + outs.emplace_back(wrap_opt_arg(cv::util::get>(args.back()))); + break; + } + case cv::GShape::GOPAQUE: + { + cv::detail::OpaqueRef ref; + cv::util::get(info.ctor)(ref); + args.emplace_back(cv::util::make_optional(std::move(ref))); + outs.emplace_back(wrap_opt_arg(cv::util::get>(args.back()))); + break; + } + default: + cv::util::throw_error(std::logic_error("Unsupported optional output shape for Python")); + } + } +} } // anonymous namespace class cv::gimpl::GStreamingExecutor::Synchronizer final { @@ -1320,6 +1363,16 @@ cv::gimpl::GStreamingExecutor::GStreamingExecutor(std::unique_ptr && // per the same input frame, so the output traffic multiplies) GAPI_Assert(m_collector_map.size() > 0u); m_out_queue.set_capacity(queue_capacity * m_collector_map.size()); + + // FIXME: The code duplicates logic of collectGraphInfo() + cv::gimpl::GModel::ConstGraph cgr(*m_orig_graph); + auto meta = cgr.metadata().get().out_nhs; + out_info.reserve(meta.size()); + + ade::util::transform(meta, std::back_inserter(out_info), [&cgr](const ade::NodeHandle& nh) { + const auto& data = cgr.metadata(nh).get(); + return cv::GTypeInfo{data.shape, data.kind, data.ctor}; + }); } cv::gimpl::GStreamingExecutor::~GStreamingExecutor() @@ -1653,6 +1706,31 @@ bool cv::gimpl::GStreamingExecutor::pull(cv::GOptRunArgsP &&outs) return true; } +std::tuple> cv::gimpl::GStreamingExecutor::pull() +{ + using RunArgs = cv::util::variant; + bool is_over = false; + + if (m_desync) { + GOptRunArgs opt_run_args; + GOptRunArgsP opt_outs; + opt_outs.reserve(out_info.size()); + opt_run_args.reserve(out_info.size()); + + constructOptGraphOutputs(out_info, opt_run_args, opt_outs); + is_over = pull(std::move(opt_outs)); + return std::make_tuple(is_over, RunArgs(opt_run_args)); + } + + GRunArgs run_args; + GRunArgsP outs; + run_args.reserve(out_info.size()); + outs.reserve(out_info.size()); + + constructGraphOutputs(out_info, run_args, outs); + is_over = pull(std::move(outs)); + return std::make_tuple(is_over, RunArgs(run_args)); +} bool cv::gimpl::GStreamingExecutor::try_pull(cv::GRunArgsP &&outs) { diff --git a/modules/gapi/src/executor/gstreamingexecutor.hpp b/modules/gapi/src/executor/gstreamingexecutor.hpp index 40b7872682..b4aadcbbaf 100644 --- a/modules/gapi/src/executor/gstreamingexecutor.hpp +++ b/modules/gapi/src/executor/gstreamingexecutor.hpp @@ -195,6 +195,8 @@ protected: void wait_shutdown(); + cv::GTypesInfo out_info; + public: explicit GStreamingExecutor(std::unique_ptr &&g_model, const cv::GCompileArgs &comp_args); @@ -203,6 +205,7 @@ public: void start(); bool pull(cv::GRunArgsP &&outs); bool pull(cv::GOptRunArgsP &&outs); + std::tuple> pull(); bool try_pull(cv::GRunArgsP &&outs); void stop(); bool running() const; diff --git a/modules/gapi/test/streaming/gapi_streaming_tests.cpp b/modules/gapi/test/streaming/gapi_streaming_tests.cpp index f3179a7081..5386d1736f 100644 --- a/modules/gapi/test/streaming/gapi_streaming_tests.cpp +++ b/modules/gapi/test/streaming/gapi_streaming_tests.cpp @@ -244,6 +244,35 @@ public: } }; +void checkPullOverload(const cv::Mat& ref, + const bool has_output, + cv::util::variant& args) { + EXPECT_TRUE(has_output); + using runArgs = cv::util::variant; + cv::Mat out_mat; + switch (args.index()) { + case runArgs::index_of(): + { + auto outputs = util::get(args); + EXPECT_EQ(1u, outputs.size()); + out_mat = cv::util::get(outputs[0]); + break; + } + case runArgs::index_of(): + { + auto outputs = util::get(args); + EXPECT_EQ(1u, outputs.size()); + auto opt_mat = cv::util::get>(outputs[0]); + ASSERT_TRUE(opt_mat.has_value()); + out_mat = *opt_mat; + break; + } + default: GAPI_Assert(false && "Incorrect type of Args"); + } + + EXPECT_EQ(0., cv::norm(ref, out_mat, cv::NORM_INF)); +} + } // anonymous namespace TEST_P(GAPI_Streaming, SmokeTest_ConstInput_GMat) @@ -1336,13 +1365,45 @@ TEST(Streaming, Python_Pull_Overload) bool has_output; cv::GRunArgs outputs; - std::tie(has_output, outputs) = ccomp.pull(); + using RunArgs = cv::util::variant; + RunArgs args; - EXPECT_TRUE(has_output); - EXPECT_EQ(1u, outputs.size()); + std::tie(has_output, args) = ccomp.pull(); - auto out_mat = cv::util::get(outputs[0]); - EXPECT_EQ(0., cv::norm(in_mat, out_mat, cv::NORM_INF)); + checkPullOverload(in_mat, has_output, args); + + ccomp.stop(); + EXPECT_FALSE(ccomp.running()); +} + +TEST(GAPI_Streaming_Desync, Python_Pull_Overload) +{ + cv::GMat in; + cv::GMat out = cv::gapi::streaming::desync(in); + cv::GComputation c(in, out); + + cv::Size sz(3,3); + cv::Mat in_mat(sz, CV_8UC3); + cv::randu(in_mat, cv::Scalar::all(0), cv::Scalar(255)); + + auto ccomp = c.compileStreaming(); + + EXPECT_TRUE(ccomp); + EXPECT_FALSE(ccomp.running()); + + ccomp.setSource(cv::gin(in_mat)); + + ccomp.start(); + EXPECT_TRUE(ccomp.running()); + + bool has_output; + cv::GRunArgs outputs; + using RunArgs = cv::util::variant; + RunArgs args; + + std::tie(has_output, args) = ccomp.pull(); + + checkPullOverload(in_mat, has_output, args); ccomp.stop(); EXPECT_FALSE(ccomp.running()); @@ -2132,9 +2193,17 @@ TEST(GAPI_Streaming, TestPythonAPI) bool is_over = false; cv::GRunArgs out_args; + using RunArgs = cv::util::variant; + RunArgs args; // NB: Used by python bridge - std::tie(is_over, out_args) = cc.pull(); + std::tie(is_over, args) = cc.pull(); + + switch (args.index()) { + case RunArgs::index_of(): + out_args = util::get(args); break; + default: GAPI_Assert(false && "Incorrect type of return value"); + } ASSERT_EQ(1u, out_args.size()); ASSERT_TRUE(cv::util::holds_alternative(out_args[0])); From 90df3af6cffec4a3255ae0f217471d013fd68492 Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Wed, 30 Jun 2021 21:41:25 +0000 Subject: [PATCH 038/262] build: winpack_dldt with dldt 2021.4.0 --- ...-dldt-disable-multidevice-autoplugin.patch | 16 ++ ...20210630-dldt-disable-unused-targets.patch | 219 ++++++++++++++++++ .../2021.4/20210630-dldt-pdb.patch | 15 ++ .../2021.4/20210630-dldt-vs-version.patch | 16 ++ platforms/winpack_dldt/2021.4/build.config.py | 1 + platforms/winpack_dldt/2021.4/patch.config.py | 4 + .../winpack_dldt/2021.4/sysroot.config.py | 56 +++++ platforms/winpack_dldt/build_package.py | 14 +- 8 files changed, 336 insertions(+), 5 deletions(-) create mode 100644 platforms/winpack_dldt/2021.4/20210630-dldt-disable-multidevice-autoplugin.patch create mode 100644 platforms/winpack_dldt/2021.4/20210630-dldt-disable-unused-targets.patch create mode 100644 platforms/winpack_dldt/2021.4/20210630-dldt-pdb.patch create mode 100644 platforms/winpack_dldt/2021.4/20210630-dldt-vs-version.patch create mode 100644 platforms/winpack_dldt/2021.4/build.config.py create mode 100644 platforms/winpack_dldt/2021.4/patch.config.py create mode 100644 platforms/winpack_dldt/2021.4/sysroot.config.py diff --git a/platforms/winpack_dldt/2021.4/20210630-dldt-disable-multidevice-autoplugin.patch b/platforms/winpack_dldt/2021.4/20210630-dldt-disable-multidevice-autoplugin.patch new file mode 100644 index 0000000000..f1e7487442 --- /dev/null +++ b/platforms/winpack_dldt/2021.4/20210630-dldt-disable-multidevice-autoplugin.patch @@ -0,0 +1,16 @@ +diff --git a/inference-engine/src/CMakeLists.txt b/inference-engine/src/CMakeLists.txt +index 0ba0dd78..7d34e7cb 100644 +--- a/inference-engine/src/CMakeLists.txt ++++ b/inference-engine/src/CMakeLists.txt +@@ -26,9 +26,9 @@ endif() + + add_subdirectory(hetero_plugin) + +-add_subdirectory(auto_plugin) ++#add_subdirectory(auto_plugin) + +-add_subdirectory(multi_device) ++#add_subdirectory(multi_device) + + add_subdirectory(transformations) + diff --git a/platforms/winpack_dldt/2021.4/20210630-dldt-disable-unused-targets.patch b/platforms/winpack_dldt/2021.4/20210630-dldt-disable-unused-targets.patch new file mode 100644 index 0000000000..9d44cdadc6 --- /dev/null +++ b/platforms/winpack_dldt/2021.4/20210630-dldt-disable-unused-targets.patch @@ -0,0 +1,219 @@ +diff --git a/cmake/developer_package/add_ie_target.cmake b/cmake/developer_package/add_ie_target.cmake +index d49f16a4d..2726ca787 100644 +--- a/cmake/developer_package/add_ie_target.cmake ++++ b/cmake/developer_package/add_ie_target.cmake +@@ -92,7 +92,7 @@ function(addIeTarget) + if (ARG_TYPE STREQUAL EXECUTABLE) + add_executable(${ARG_NAME} ${all_sources}) + elseif(ARG_TYPE STREQUAL STATIC OR ARG_TYPE STREQUAL SHARED) +- add_library(${ARG_NAME} ${ARG_TYPE} ${all_sources}) ++ add_library(${ARG_NAME} ${ARG_TYPE} EXCLUDE_FROM_ALL ${all_sources}) + else() + message(SEND_ERROR "Invalid target type ${ARG_TYPE} specified for target name ${ARG_NAME}") + endif() +diff --git a/inference-engine/CMakeLists.txt b/inference-engine/CMakeLists.txt +index 1ac7fd8bf..df7091e51 100644 +--- a/inference-engine/CMakeLists.txt ++++ b/inference-engine/CMakeLists.txt +@@ -39,7 +39,7 @@ if(ENABLE_TESTS) + add_subdirectory(tests) + endif() + +-add_subdirectory(tools) ++#add_subdirectory(tools) + + function(ie_build_samples) + # samples should be build with the same flags as from OpenVINO package, +@@ -58,7 +58,7 @@ endfunction() + + # gflags and format_reader targets are kept inside of samples directory and + # they must be built even if samples build is disabled (required for tests and tools). +-ie_build_samples() ++#ie_build_samples() + + if(ENABLE_PYTHON) + add_subdirectory(ie_bridges/python) +@@ -142,7 +142,7 @@ endif() + # Developer package + # + +-openvino_developer_export_targets(COMPONENT openvino_common TARGETS format_reader gflags ie_samples_utils) ++#openvino_developer_export_targets(COMPONENT openvino_common TARGETS format_reader gflags ie_samples_utils) + + # for Template plugin + if(NGRAPH_INTERPRETER_ENABLE) +@@ -166,7 +166,7 @@ function(ie_generate_dev_package_config) + @ONLY) + endfunction() + +-ie_generate_dev_package_config() ++#ie_generate_dev_package_config() + + # + # Coverage +diff --git a/inference-engine/src/inference_engine/CMakeLists.txt b/inference-engine/src/inference_engine/CMakeLists.txt +index e8ed1a5c4..1fc9fc3ff 100644 +--- a/inference-engine/src/inference_engine/CMakeLists.txt ++++ b/inference-engine/src/inference_engine/CMakeLists.txt +@@ -110,7 +110,7 @@ add_cpplint_target(${TARGET_NAME}_plugin_api_cpplint FOR_SOURCES ${plugin_api_sr + + # Create object library + +-add_library(${TARGET_NAME}_obj OBJECT ++add_library(${TARGET_NAME}_obj OBJECT EXCLUDE_FROM_ALL + ${LIBRARY_SRC} + ${LIBRARY_HEADERS} + ${PUBLIC_HEADERS}) +@@ -181,7 +181,7 @@ ie_add_api_validator_post_build_step(TARGET ${TARGET_NAME}) + + # Static library used for unit tests which are always built + +-add_library(${TARGET_NAME}_s STATIC ++add_library(${TARGET_NAME}_s STATIC EXCLUDE_FROM_ALL + $ + $ + ${IE_STATIC_DEPENDENT_FILES}) +diff --git a/inference-engine/src/legacy_api/CMakeLists.txt b/inference-engine/src/legacy_api/CMakeLists.txt +index 8eae82bd2..e0e6745b1 100644 +--- a/inference-engine/src/legacy_api/CMakeLists.txt ++++ b/inference-engine/src/legacy_api/CMakeLists.txt +@@ -26,7 +26,7 @@ endif() + + file(TOUCH ${CMAKE_CURRENT_BINARY_DIR}/dummy.cpp) + +-add_library(${TARGET_NAME}_obj OBJECT ++add_library(${TARGET_NAME}_obj OBJECT EXCLUDE_FROM_ALL + ${LIBRARY_SRC} + ${PUBLIC_HEADERS}) + +diff --git a/inference-engine/src/mkldnn_plugin/CMakeLists.txt b/inference-engine/src/mkldnn_plugin/CMakeLists.txt +index fe57b29dd..07831e2fb 100644 +--- a/inference-engine/src/mkldnn_plugin/CMakeLists.txt ++++ b/inference-engine/src/mkldnn_plugin/CMakeLists.txt +@@ -67,7 +67,7 @@ ie_add_api_validator_post_build_step(TARGET ${TARGET_NAME}) + + # add test object library + +-add_library(${TARGET_NAME}_obj OBJECT ${SOURCES} ${HEADERS}) ++add_library(${TARGET_NAME}_obj OBJECT EXCLUDE_FROM_ALL ${SOURCES} ${HEADERS}) + target_link_libraries(${TARGET_NAME}_obj PUBLIC mkldnn) + + target_include_directories(${TARGET_NAME}_obj PRIVATE $ +diff --git a/inference-engine/src/preprocessing/CMakeLists.txt b/inference-engine/src/preprocessing/CMakeLists.txt +index f9548339d..ef962145a 100644 +--- a/inference-engine/src/preprocessing/CMakeLists.txt ++++ b/inference-engine/src/preprocessing/CMakeLists.txt +@@ -101,7 +101,7 @@ endif() + + # Create object library + +-add_library(${TARGET_NAME}_obj OBJECT ++add_library(${TARGET_NAME}_obj OBJECT EXCLUDE_FROM_ALL + ${LIBRARY_SRC} + ${LIBRARY_HEADERS}) + +@@ -153,7 +153,7 @@ ie_add_api_validator_post_build_step(TARGET ${TARGET_NAME}) + + # Static library used for unit tests which are always built + +-add_library(${TARGET_NAME}_s STATIC ++add_library(${TARGET_NAME}_s STATIC EXCLUDE_FROM_ALL + $) + + set_ie_threading_interface_for(${TARGET_NAME}_s) +diff --git a/inference-engine/src/vpu/common/CMakeLists.txt b/inference-engine/src/vpu/common/CMakeLists.txt +index 249e47c28..4ddf63049 100644 +--- a/inference-engine/src/vpu/common/CMakeLists.txt ++++ b/inference-engine/src/vpu/common/CMakeLists.txt +@@ -5,7 +5,7 @@ + file(GLOB_RECURSE SOURCES *.cpp *.hpp *.h) + + function(add_common_target TARGET_NAME STATIC_IE) +- add_library(${TARGET_NAME} STATIC ${SOURCES}) ++ add_library(${TARGET_NAME} STATIC EXCLUDE_FROM_ALL ${SOURCES}) + + ie_faster_build(${TARGET_NAME} + UNITY +@@ -60,7 +60,7 @@ add_common_target("vpu_common_lib" FALSE) + + # Unit tests support for graph transformer + if(WIN32) +- add_common_target("vpu_common_lib_test_static" TRUE) ++ #add_common_target("vpu_common_lib_test_static" TRUE) + else() + add_library("vpu_common_lib_test_static" ALIAS "vpu_common_lib") + endif() +diff --git a/inference-engine/src/vpu/graph_transformer/CMakeLists.txt b/inference-engine/src/vpu/graph_transformer/CMakeLists.txt +index bc73ab5b1..b4c1547fc 100644 +--- a/inference-engine/src/vpu/graph_transformer/CMakeLists.txt ++++ b/inference-engine/src/vpu/graph_transformer/CMakeLists.txt +@@ -5,7 +5,7 @@ + file(GLOB_RECURSE SOURCES *.cpp *.hpp *.h *.inc) + + function(add_graph_transformer_target TARGET_NAME STATIC_IE) +- add_library(${TARGET_NAME} STATIC ${SOURCES}) ++ add_library(${TARGET_NAME} STATIC EXCLUDE_FROM_ALL ${SOURCES}) + + set_ie_threading_interface_for(${TARGET_NAME}) + +@@ -70,7 +70,7 @@ add_graph_transformer_target("vpu_graph_transformer" FALSE) + + # Unit tests support for graph transformer + if(WIN32) +- add_graph_transformer_target("vpu_graph_transformer_test_static" TRUE) ++ #add_graph_transformer_target("vpu_graph_transformer_test_static" TRUE) + else() + add_library("vpu_graph_transformer_test_static" ALIAS "vpu_graph_transformer") + endif() +diff --git a/inference-engine/thirdparty/pugixml/CMakeLists.txt b/inference-engine/thirdparty/pugixml/CMakeLists.txt +index 8bcb2801a..f7e031c01 100644 +--- a/inference-engine/thirdparty/pugixml/CMakeLists.txt ++++ b/inference-engine/thirdparty/pugixml/CMakeLists.txt +@@ -41,7 +41,7 @@ if(BUILD_SHARED_LIBS) + else() + add_library(pugixml STATIC ${SOURCES}) + if (MSVC) +- add_library(pugixml_mt STATIC ${SOURCES}) ++ #add_library(pugixml_mt STATIC ${SOURCES}) + #if (WIN32) + # set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} /MT") + # set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} /MTd") +diff --git a/ngraph/core/builder/CMakeLists.txt b/ngraph/core/builder/CMakeLists.txt +index ff5c381e7..2797ec9ab 100644 +--- a/ngraph/core/builder/CMakeLists.txt ++++ b/ngraph/core/builder/CMakeLists.txt +@@ -16,7 +16,7 @@ source_group("src" FILES ${LIBRARY_SRC}) + source_group("include" FILES ${PUBLIC_HEADERS}) + + # Create shared library +-add_library(${TARGET_NAME} STATIC ${LIBRARY_SRC} ${PUBLIC_HEADERS}) ++add_library(${TARGET_NAME} STATIC EXCLUDE_FROM_ALL ${LIBRARY_SRC} ${PUBLIC_HEADERS}) + + if(COMMAND ie_faster_build) + ie_faster_build(${TARGET_NAME} +diff --git a/ngraph/core/reference/CMakeLists.txt b/ngraph/core/reference/CMakeLists.txt +index ef4a764ab..f6d3172e2 100644 +--- a/ngraph/core/reference/CMakeLists.txt ++++ b/ngraph/core/reference/CMakeLists.txt +@@ -16,7 +16,7 @@ source_group("src" FILES ${LIBRARY_SRC}) + source_group("include" FILES ${PUBLIC_HEADERS}) + + # Create shared library +-add_library(${TARGET_NAME} STATIC ${LIBRARY_SRC} ${PUBLIC_HEADERS}) ++add_library(${TARGET_NAME} STATIC EXCLUDE_FROM_ALL ${LIBRARY_SRC} ${PUBLIC_HEADERS}) + + if(COMMAND ie_faster_build) + ie_faster_build(${TARGET_NAME} +diff --git a/openvino/itt/CMakeLists.txt b/openvino/itt/CMakeLists.txt +index e9f880b8c..c63f4df63 100644 +--- a/openvino/itt/CMakeLists.txt ++++ b/openvino/itt/CMakeLists.txt +@@ -6,7 +6,7 @@ set(TARGET_NAME itt) + + file(GLOB_RECURSE SOURCES "src/*.cpp" "src/*.hpp") + +-add_library(${TARGET_NAME} STATIC ${SOURCES}) ++add_library(${TARGET_NAME} STATIC EXCLUDE_FROM_ALL ${SOURCES}) + + add_library(openvino::itt ALIAS ${TARGET_NAME}) + diff --git a/platforms/winpack_dldt/2021.4/20210630-dldt-pdb.patch b/platforms/winpack_dldt/2021.4/20210630-dldt-pdb.patch new file mode 100644 index 0000000000..65e6f84dc8 --- /dev/null +++ b/platforms/winpack_dldt/2021.4/20210630-dldt-pdb.patch @@ -0,0 +1,15 @@ +iff --git a/CMakeLists.txt b/CMakeLists.txt +index e0706a72e..9a053b1e4 100644 +--- a/CMakeLists.txt ++++ b/CMakeLists.txt +@@ -6,6 +6,10 @@ cmake_minimum_required(VERSION 3.13) + + project(OpenVINO) + ++set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} /Zi /FS") ++set(CMAKE_SHARED_LINKER_FLAGS_RELEASE "${CMAKE_SHARED_LINKER_FLAGS_RELEASE} /DEBUG /OPT:REF /OPT:ICF") ++set(CMAKE_MODULE_LINKER_FLAGS_RELEASE "${CMAKE_SHARED_LINKER_FLAGS_RELEASE} /DEBUG /OPT:REF /OPT:ICF") ++ + set(OpenVINO_MAIN_SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}) + set(IE_MAIN_SOURCE_DIR ${OpenVINO_MAIN_SOURCE_DIR}/inference-engine) + diff --git a/platforms/winpack_dldt/2021.4/20210630-dldt-vs-version.patch b/platforms/winpack_dldt/2021.4/20210630-dldt-vs-version.patch new file mode 100644 index 0000000000..36b0068775 --- /dev/null +++ b/platforms/winpack_dldt/2021.4/20210630-dldt-vs-version.patch @@ -0,0 +1,16 @@ +diff --git a/cmake/developer_package/vs_version/vs_version.cmake b/cmake/developer_package/vs_version/vs_version.cmake +index 14d4c0e1e..6a44f73b9 100644 +--- a/cmake/developer_package/vs_version/vs_version.cmake ++++ b/cmake/developer_package/vs_version/vs_version.cmake +@@ -8,9 +8,9 @@ set(IE_VS_VER_FILEVERSION_STR "${IE_VERSION_MAJOR}.${IE_VERSION_MINOR}.${IE_VERS + + set(IE_VS_VER_COMPANY_NAME_STR "Intel Corporation") + set(IE_VS_VER_PRODUCTVERSION_STR "${CI_BUILD_NUMBER}") +-set(IE_VS_VER_PRODUCTNAME_STR "OpenVINO toolkit") ++set(IE_VS_VER_PRODUCTNAME_STR "OpenVINO toolkit (for OpenCV Windows package)") + set(IE_VS_VER_COPYRIGHT_STR "Copyright (C) 2018-2021, Intel Corporation") +-set(IE_VS_VER_COMMENTS_STR "https://docs.openvinotoolkit.org/") ++set(IE_VS_VER_COMMENTS_STR "https://github.com/opencv/opencv/wiki/Intel%27s-Deep-Learning-Inference-Engine-backend") + + # + # ie_add_vs_version_file(NAME diff --git a/platforms/winpack_dldt/2021.4/build.config.py b/platforms/winpack_dldt/2021.4/build.config.py new file mode 100644 index 0000000000..33ef1050ca --- /dev/null +++ b/platforms/winpack_dldt/2021.4/build.config.py @@ -0,0 +1 @@ +os.environ['CI_BUILD_NUMBER'] = '2021.4.0-opencv_winpack_dldt' diff --git a/platforms/winpack_dldt/2021.4/patch.config.py b/platforms/winpack_dldt/2021.4/patch.config.py new file mode 100644 index 0000000000..7f8715aae2 --- /dev/null +++ b/platforms/winpack_dldt/2021.4/patch.config.py @@ -0,0 +1,4 @@ +applyPatch('20210630-dldt-disable-unused-targets.patch') +applyPatch('20210630-dldt-pdb.patch') +applyPatch('20210630-dldt-disable-multidevice-autoplugin.patch') +applyPatch('20210630-dldt-vs-version.patch') diff --git a/platforms/winpack_dldt/2021.4/sysroot.config.py b/platforms/winpack_dldt/2021.4/sysroot.config.py new file mode 100644 index 0000000000..fa4281107d --- /dev/null +++ b/platforms/winpack_dldt/2021.4/sysroot.config.py @@ -0,0 +1,56 @@ +sysroot_bin_dir = prepare_dir(self.sysrootdir / 'bin') +copytree(self.build_dir / 'install', self.sysrootdir / 'ngraph') +#rm_one(self.sysrootdir / 'ngraph' / 'lib' / 'ngraph.dll') + +build_config = 'Release' if not self.config.build_debug else 'Debug' +build_bin_dir = self.build_dir / 'bin' / 'intel64' / build_config + +def copy_bin(name): + global build_bin_dir, sysroot_bin_dir + copytree(build_bin_dir / name, sysroot_bin_dir / name) + +dll_suffix = 'd' if self.config.build_debug else '' +def copy_dll(name): + global copy_bin, dll_suffix + copy_bin(name + dll_suffix + '.dll') + copy_bin(name + dll_suffix + '.pdb') + +copy_bin('cache.json') +copy_dll('clDNNPlugin') +copy_dll('HeteroPlugin') +copy_dll('inference_engine') +copy_dll('inference_engine_ir_reader') +#copy_dll('inference_engine_ir_v7_reader') +copy_dll('inference_engine_legacy') +copy_dll('inference_engine_transformations') # runtime +copy_dll('inference_engine_lp_transformations') # runtime +#copy_dll('inference_engine_preproc') # runtime +copy_dll('MKLDNNPlugin') # runtime +copy_dll('myriadPlugin') # runtime +#copy_dll('MultiDevicePlugin') # runtime, not used +copy_dll('ngraph') +copy_bin('plugins.xml') +copy_bin('pcie-ma2x8x.elf') +copy_bin('usb-ma2x8x.mvcmd') + +copytree(self.srcdir / 'inference-engine' / 'temp' / 'tbb' / 'bin', sysroot_bin_dir) +copytree(self.srcdir / 'inference-engine' / 'temp' / 'tbb', self.sysrootdir / 'tbb') + +sysroot_ie_dir = prepare_dir(self.sysrootdir / 'deployment_tools' / 'inference_engine') +sysroot_ie_lib_dir = prepare_dir(sysroot_ie_dir / 'lib' / 'intel64') + +copytree(self.srcdir / 'inference-engine' / 'include', sysroot_ie_dir / 'include') +if not self.config.build_debug: + copytree(build_bin_dir / 'ngraph.lib', sysroot_ie_lib_dir / 'ngraph.lib') + copytree(build_bin_dir / 'inference_engine.lib', sysroot_ie_lib_dir / 'inference_engine.lib') + copytree(build_bin_dir / 'inference_engine_ir_reader.lib', sysroot_ie_lib_dir / 'inference_engine_ir_reader.lib') + copytree(build_bin_dir / 'inference_engine_legacy.lib', sysroot_ie_lib_dir / 'inference_engine_legacy.lib') +else: + copytree(build_bin_dir / 'ngraphd.lib', sysroot_ie_lib_dir / 'ngraphd.lib') + copytree(build_bin_dir / 'inference_engined.lib', sysroot_ie_lib_dir / 'inference_engined.lib') + copytree(build_bin_dir / 'inference_engine_ir_readerd.lib', sysroot_ie_lib_dir / 'inference_engine_ir_readerd.lib') + copytree(build_bin_dir / 'inference_engine_legacyd.lib', sysroot_ie_lib_dir / 'inference_engine_legacyd.lib') + +sysroot_license_dir = prepare_dir(self.sysrootdir / 'etc' / 'licenses') +copytree(self.srcdir / 'LICENSE', sysroot_license_dir / 'dldt-LICENSE') +copytree(self.sysrootdir / 'tbb/LICENSE', sysroot_license_dir / 'tbb-LICENSE') diff --git a/platforms/winpack_dldt/build_package.py b/platforms/winpack_dldt/build_package.py index b993f076d9..6fde62241a 100644 --- a/platforms/winpack_dldt/build_package.py +++ b/platforms/winpack_dldt/build_package.py @@ -214,7 +214,7 @@ class BuilderDLDT: patch_hashsum = hashlib.md5(self.patch_file_contents.encode('utf-8')).hexdigest() except: log.warn("Can't compute hashsum of patches: %s", self.patch_file) - self.patch_hashsum = patch_hashsum + self.patch_hashsum = self.config.override_patch_hashsum if self.config.override_patch_hashsum else patch_hashsum def prepare_sources(self): @@ -355,7 +355,6 @@ class Builder: BUILD_PERF_TESTS='OFF', ENABLE_CXX11='ON', WITH_INF_ENGINE='ON', - INF_ENGINE_RELEASE=str(self.config.dldt_release), WITH_TBB='ON', CPU_BASELINE='AVX2', CMAKE_INSTALL_PREFIX=str(self.install_dir), @@ -381,6 +380,9 @@ class Builder: OPENCV_PYTHON_INSTALL_PATH='python', ) + if self.config.dldt_release: + cmake_vars['INF_ENGINE_RELEASE'] = str(self.config.dldt_release) + cmake_vars['INF_ENGINE_LIB_DIRS:PATH'] = str(builderDLDT.sysrootdir / 'deployment_tools/inference_engine/lib/intel64') assert os.path.exists(cmake_vars['INF_ENGINE_LIB_DIRS:PATH']), cmake_vars['INF_ENGINE_LIB_DIRS:PATH'] cmake_vars['INF_ENGINE_INCLUDE_DIRS:PATH'] = str(builderDLDT.sysrootdir / 'deployment_tools/inference_engine/include') @@ -464,8 +466,8 @@ class Builder: def main(): dldt_src_url = 'https://github.com/openvinotoolkit/openvino' - dldt_src_commit = '2021.3' - dldt_release = '2021030000' + dldt_src_commit = '2021.4' + dldt_release = None build_cache_dir_default = os.environ.get('BUILD_CACHE_DIR', '.build_cache') build_subst_drive = os.environ.get('BUILD_SUBST_DRIVE', None) @@ -492,13 +494,15 @@ def main(): parser.add_argument('--dldt_src_branch', help='DLDT checkout branch') parser.add_argument('--dldt_src_commit', default=dldt_src_commit, help='DLDT source commit / tag (default: %s)' % dldt_src_commit) parser.add_argument('--dldt_src_git_clone_extra', action='append', help='DLDT git clone extra args') - parser.add_argument('--dldt_release', default=dldt_release, help='DLDT release code for INF_ENGINE_RELEASE (default: %s)' % dldt_release) + parser.add_argument('--dldt_release', default=dldt_release, help='DLDT release code for INF_ENGINE_RELEASE, e.g 2021030000 (default: %s)' % dldt_release) parser.add_argument('--dldt_reference_dir', help='DLDT reference git repository (optional)') parser.add_argument('--dldt_src_dir', help='DLDT custom source repository (skip git checkout and patching, use for TESTING only)') parser.add_argument('--dldt_config', help='Specify DLDT build configuration (defaults to evaluate from DLDT commit/branch)') + parser.add_argument('--override_patch_hashsum', default='', help='(script debug mode)') + args = parser.parse_args() log.basicConfig( From e5841d3126527ff1151ab480ba8e291e29942e07 Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Fri, 2 Jul 2021 10:41:41 +0000 Subject: [PATCH 039/262] java: force using of 'Ptr<>' for OpenCV classes --- modules/dnn/misc/java/gen_dict.json | 2 +- modules/java/generator/gen_java.py | 34 +++++++++++-------- .../misc/java/test/TrackerCreateTest.java | 7 ++++ 3 files changed, 27 insertions(+), 16 deletions(-) diff --git a/modules/dnn/misc/java/gen_dict.json b/modules/dnn/misc/java/gen_dict.json index 5a397eac51..65ecfdc25e 100644 --- a/modules/dnn/misc/java/gen_dict.json +++ b/modules/dnn/misc/java/gen_dict.json @@ -54,7 +54,7 @@ ] ], - "jni_name": "(*(cv::dnn::DictValue*)%(n)s_nativeObj)", + "jni_name": "(*(*(Ptr*)%(n)s_nativeObj))", "jni_type": "jlong", "suffix": "J", "j_import": "org.opencv.dnn.DictValue" diff --git a/modules/java/generator/gen_java.py b/modules/java/generator/gen_java.py index 6019ca340d..c5b4f34a8f 100755 --- a/modules/java/generator/gen_java.py +++ b/modules/java/generator/gen_java.py @@ -258,6 +258,8 @@ class ClassInfo(GeneralInfo): for m in decl[2]: if m.startswith("="): self.jname = m[1:] + if m == '/Simple': + self.smart = False if self.classpath: prefix = self.classpath.replace('.', '_') @@ -445,7 +447,7 @@ class JavaWrapperGenerator(object): def clear(self): self.namespaces = ["cv"] - classinfo_Mat = ClassInfo([ 'class cv.Mat', '', [], [] ], self.namespaces) + classinfo_Mat = ClassInfo([ 'class cv.Mat', '', ['/Simple'], [] ], self.namespaces) self.classes = { "Mat" : classinfo_Mat } self.module = "" self.Module = "" @@ -466,10 +468,15 @@ class JavaWrapperGenerator(object): if name in type_dict and not classinfo.base: logging.warning('duplicated: %s', classinfo) return + if self.isSmartClass(classinfo): + jni_name = "*((*(Ptr<"+classinfo.fullNameCPP()+">*)%(n)s_nativeObj).get())" + else: + jni_name = "(*("+classinfo.fullNameCPP()+"*)%(n)s_nativeObj)" type_dict.setdefault(name, {}).update( { "j_type" : classinfo.jname, "jn_type" : "long", "jn_args" : (("__int64", ".nativeObj"),), - "jni_name" : "(*("+classinfo.fullNameCPP()+"*)%(n)s_nativeObj)", "jni_type" : "jlong", + "jni_name" : jni_name, + "jni_type" : "jlong", "suffix" : "J", "j_import" : "org.opencv.%s.%s" % (self.module, classinfo.jname) } @@ -477,7 +484,8 @@ class JavaWrapperGenerator(object): type_dict.setdefault(name+'*', {}).update( { "j_type" : classinfo.jname, "jn_type" : "long", "jn_args" : (("__int64", ".nativeObj"),), - "jni_name" : "("+classinfo.fullNameCPP()+"*)%(n)s_nativeObj", "jni_type" : "jlong", + "jni_name" : "&("+jni_name+")", + "jni_type" : "jlong", "suffix" : "J", "j_import" : "org.opencv.%s.%s" % (self.module, classinfo.jname) } @@ -966,7 +974,13 @@ class JavaWrapperGenerator(object): ret = "return env->NewStringUTF(_retval_.c_str());" default = 'return env->NewStringUTF("");' elif self.isWrapped(fi.ctype): # wrapped class: - ret = "return (jlong) new %s(_retval_);" % self.fullTypeNameCPP(fi.ctype) + ret = None + if fi.ctype in self.classes: + ret_ci = self.classes[fi.ctype] + if self.isSmartClass(ret_ci): + ret = "return (jlong)(new Ptr<%(ctype)s>(new %(ctype)s(_retval_)));" % { 'ctype': ret_ci.fullNameCPP() } + if ret is None: + ret = "return (jlong) new %s(_retval_);" % self.fullTypeNameCPP(fi.ctype) elif fi.ctype.startswith('Ptr_'): c_prologue.append("typedef Ptr<%s> %s;" % (self.fullTypeNameCPP(fi.ctype[4:]), fi.ctype)) ret = "return (jlong)(new %(ctype)s(_retval_));" % { 'ctype':fi.ctype } @@ -1207,17 +1221,7 @@ JNIEXPORT void JNICALL Java_org_opencv_%(module)s_%(j_cls)s_delete if ci.smart != None: return ci.smart - # if parents are smart (we hope) then children are! - # if not we believe the class is smart if it has "create" method - ci.smart = False - if ci.base or ci.name == 'Algorithm': - ci.smart = True - else: - for fi in ci.methods: - if fi.name == "create": - ci.smart = True - break - + ci.smart = True # smart class is not properly handled in case of base/derived classes return ci.smart def smartWrap(self, ci, fullname): diff --git a/modules/video/misc/java/test/TrackerCreateTest.java b/modules/video/misc/java/test/TrackerCreateTest.java index dad696bebf..83bbd0b5d5 100644 --- a/modules/video/misc/java/test/TrackerCreateTest.java +++ b/modules/video/misc/java/test/TrackerCreateTest.java @@ -1,7 +1,10 @@ package org.opencv.test.video; import org.opencv.core.Core; +import org.opencv.core.CvType; import org.opencv.core.CvException; +import org.opencv.core.Mat; +import org.opencv.core.Rect; import org.opencv.test.OpenCVTestCase; import org.opencv.video.Tracker; @@ -27,6 +30,10 @@ public class TrackerCreateTest extends OpenCVTestCase { public void testCreateTrackerMIL() { Tracker tracker = TrackerMIL.create(); + assert(tracker != null); + Mat mat = new Mat(100, 100, CvType.CV_8UC1); + Rect rect = new Rect(10, 10, 30, 30); + tracker.init(mat, rect); // should not crash (https://github.com/opencv/opencv/issues/19915) } } From 8d1f254dcc2b4f413b1a610ec77b92896702cfca Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Fri, 2 Jul 2021 10:41:41 +0000 Subject: [PATCH 040/262] java: force using of 'Ptr<>' for OpenCV classes backport of commit: e5841d3126527ff1151ab480ba8e291e29942e07 --- modules/dnn/misc/java/gen_dict.json | 2 +- modules/java/generator/gen_java.py | 35 ++++++++++++++++------------- 2 files changed, 21 insertions(+), 16 deletions(-) diff --git a/modules/dnn/misc/java/gen_dict.json b/modules/dnn/misc/java/gen_dict.json index 5a397eac51..65ecfdc25e 100644 --- a/modules/dnn/misc/java/gen_dict.json +++ b/modules/dnn/misc/java/gen_dict.json @@ -54,7 +54,7 @@ ] ], - "jni_name": "(*(cv::dnn::DictValue*)%(n)s_nativeObj)", + "jni_name": "(*(*(Ptr*)%(n)s_nativeObj))", "jni_type": "jlong", "suffix": "J", "j_import": "org.opencv.dnn.DictValue" diff --git a/modules/java/generator/gen_java.py b/modules/java/generator/gen_java.py index 8e5c69e788..f3b5e132d1 100755 --- a/modules/java/generator/gen_java.py +++ b/modules/java/generator/gen_java.py @@ -224,6 +224,9 @@ class ClassInfo(GeneralInfo): for m in decl[2]: if m.startswith("="): self.jname = m[1:] + if m == '/Simple': + self.smart = False + self.base = '' if decl[1]: #self.base = re.sub(r"\b"+self.jname+r"\b", "", decl[1].replace(":", "")).strip() @@ -370,7 +373,7 @@ class JavaWrapperGenerator(object): def clear(self): self.namespaces = ["cv"] - self.classes = { "Mat" : ClassInfo([ 'class Mat', '', [], [] ], self.namespaces) } + self.classes = { "Mat" : ClassInfo([ 'class Mat', '', ['/Simple'], [] ], self.namespaces) } self.module = "" self.Module = "" self.ported_func_list = [] @@ -390,10 +393,15 @@ class JavaWrapperGenerator(object): if name in type_dict and not classinfo.base: logging.warning('duplicated: %s', classinfo) return + if self.isSmartClass(classinfo): + jni_name = "*((*(Ptr<"+classinfo.fullName(isCPP=True)+">*)%(n)s_nativeObj).get())" + else: + jni_name = "(*("+classinfo.fullName(isCPP=True)+"*)%(n)s_nativeObj)" type_dict.setdefault(name, {}).update( { "j_type" : classinfo.jname, "jn_type" : "long", "jn_args" : (("__int64", ".nativeObj"),), - "jni_name" : "(*("+classinfo.fullName(isCPP=True)+"*)%(n)s_nativeObj)", "jni_type" : "jlong", + "jni_name" : jni_name, + "jni_type" : "jlong", "suffix" : "J", "j_import" : "org.opencv.%s.%s" % (self.module, classinfo.jname) } @@ -401,7 +409,8 @@ class JavaWrapperGenerator(object): type_dict.setdefault(name+'*', {}).update( { "j_type" : classinfo.jname, "jn_type" : "long", "jn_args" : (("__int64", ".nativeObj"),), - "jni_name" : "("+classinfo.fullName(isCPP=True)+"*)%(n)s_nativeObj", "jni_type" : "jlong", + "jni_name" : "&("+jni_name+")", + "jni_type" : "jlong", "suffix" : "J", "j_import" : "org.opencv.%s.%s" % (self.module, classinfo.jname) } @@ -889,7 +898,13 @@ class JavaWrapperGenerator(object): ret = "return env->NewStringUTF(_retval_.c_str());" default = 'return env->NewStringUTF("");' elif self.isWrapped(fi.ctype): # wrapped class: - ret = "return (jlong) new %s(_retval_);" % self.fullTypeName(fi.ctype) + ret = None + if fi.ctype in self.classes: + ret_ci = self.classes[fi.ctype] + if self.isSmartClass(ret_ci): + ret = "return (jlong)(new Ptr<%(ctype)s>(new %(ctype)s(_retval_)));" % { 'ctype': self.fullTypeName(fi.ctype) } + if ret is None: + ret = "return (jlong) new %s(_retval_);" % self.fullTypeName(fi.ctype) elif fi.ctype.startswith('Ptr_'): c_prologue.append("typedef Ptr<%s> %s;" % (self.fullTypeName(fi.ctype[4:]), fi.ctype)) ret = "return (jlong)(new %(ctype)s(_retval_));" % { 'ctype':fi.ctype } @@ -1128,17 +1143,7 @@ JNIEXPORT void JNICALL Java_org_opencv_%(module)s_%(j_cls)s_delete if ci.smart != None: return ci.smart - # if parents are smart (we hope) then children are! - # if not we believe the class is smart if it has "create" method - ci.smart = False - if ci.base or ci.name == 'Algorithm': - ci.smart = True - else: - for fi in ci.methods: - if fi.name == "create": - ci.smart = True - break - + ci.smart = True # smart class is not properly handled in case of base/derived classes return ci.smart def smartWrap(self, ci, fullname): From 18dbac203f3b55ad445df26a5728798e26f78633 Mon Sep 17 00:00:00 2001 From: mitruska Date: Fri, 2 Jul 2021 15:29:34 +0200 Subject: [PATCH 041/262] Use explicit version of ngraph NormalizeL2 --- modules/dnn/src/layers/normalize_bbox_layer.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/dnn/src/layers/normalize_bbox_layer.cpp b/modules/dnn/src/layers/normalize_bbox_layer.cpp index cdaa87bde5..001ea3bd16 100644 --- a/modules/dnn/src/layers/normalize_bbox_layer.cpp +++ b/modules/dnn/src/layers/normalize_bbox_layer.cpp @@ -328,7 +328,7 @@ public: std::iota(axes_data.begin(), axes_data.end(), 1); } auto axes = std::make_shared(ngraph::element::i64, ngraph::Shape{axes_data.size()}, axes_data); - auto norm = std::make_shared(ieInpNode, axes, epsilon, ngraph::op::EpsMode::ADD); + auto norm = std::make_shared(ieInpNode, axes, epsilon, ngraph::op::EpsMode::ADD); CV_Assert(blobs.empty() || numChannels == blobs[0].total()); std::vector shape(ieInpNode->get_shape().size(), 1); From 9b0d6862c474b4edb012d76492834d58451d5fec Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Fri, 2 Jul 2021 21:37:37 +0000 Subject: [PATCH 042/262] cmake(IE): extract INF_ENGINE_RELEASE from InferenceEngine package --- cmake/OpenCVDetectInferenceEngine.cmake | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/cmake/OpenCVDetectInferenceEngine.cmake b/cmake/OpenCVDetectInferenceEngine.cmake index 829ddbfe7e..6308d1b424 100644 --- a/cmake/OpenCVDetectInferenceEngine.cmake +++ b/cmake/OpenCVDetectInferenceEngine.cmake @@ -140,12 +140,21 @@ endif() # Add more features to the target if(INF_ENGINE_TARGET) - if(NOT INF_ENGINE_RELEASE) - message(WARNING "InferenceEngine version has not been set, 2021.4 will be used by default. Set INF_ENGINE_RELEASE variable if you experience build errors.") + if(DEFINED InferenceEngine_VERSION) + message(STATUS "InferenceEngine: ${InferenceEngine_VERSION}") + if(NOT INF_ENGINE_RELEASE AND NOT (InferenceEngine_VERSION VERSION_LESS "2021.4")) + math(EXPR INF_ENGINE_RELEASE_INIT "${InferenceEngine_VERSION_MAJOR} * 1000000 + ${InferenceEngine_VERSION_MINOR} * 10000 + ${InferenceEngine_VERSION_PATCH} * 100") + endif() endif() - set(INF_ENGINE_RELEASE "2021040000" CACHE STRING "Force IE version, should be in form YYYYAABBCC (e.g. 2020.1.0.2 -> 2020010002)") + if(NOT INF_ENGINE_RELEASE AND NOT INF_ENGINE_RELEASE_INIT) + message(WARNING "InferenceEngine version has not been set, 2021.4 will be used by default. Set INF_ENGINE_RELEASE variable if you experience build errors.") + set(INF_ENGINE_RELEASE_INIT "2021040000") + elseif(DEFINED INF_ENGINE_RELEASE) + set(INF_ENGINE_RELEASE_INIT "${INF_ENGINE_RELEASE}") + endif() + set(INF_ENGINE_RELEASE "${INF_ENGINE_RELEASE_INIT}" CACHE STRING "Force IE version, should be in form YYYYAABBCC (e.g. 2020.1.0.2 -> 2020010002)") set_target_properties(${INF_ENGINE_TARGET} PROPERTIES - INTERFACE_COMPILE_DEFINITIONS "HAVE_INF_ENGINE=1;INF_ENGINE_RELEASE=${INF_ENGINE_RELEASE}" + INTERFACE_COMPILE_DEFINITIONS "HAVE_INF_ENGINE=1;INF_ENGINE_RELEASE=${INF_ENGINE_RELEASE}" ) endif() From 0e523618a17a99647fc13a2be1577b9c887d6d64 Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Sat, 3 Jul 2021 10:57:18 +0000 Subject: [PATCH 043/262] cmake: exclude -pthread from Emscripten default build --- cmake/OpenCVCompilerOptions.cmake | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/cmake/OpenCVCompilerOptions.cmake b/cmake/OpenCVCompilerOptions.cmake index 0a10bfffcf..303d4f451e 100644 --- a/cmake/OpenCVCompilerOptions.cmake +++ b/cmake/OpenCVCompilerOptions.cmake @@ -177,7 +177,13 @@ if(CV_GCC OR CV_CLANG) endif() # We need pthread's - if(UNIX AND NOT ANDROID AND NOT (APPLE AND CV_CLANG)) # TODO + if((UNIX + AND NOT ANDROID + AND NOT (APPLE AND CV_CLANG) + AND NOT EMSCRIPTEN + ) + OR (EMSCRIPTEN AND WITH_PTHREADS_PF) # https://github.com/opencv/opencv/issues/20285 + ) add_extra_compiler_option(-pthread) endif() From 5d0cfa252797e115010eec269043f958c3305a51 Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Sat, 3 Jul 2021 11:36:29 +0000 Subject: [PATCH 044/262] cmake(highgui): don't allow multiple builtin backends --- modules/highgui/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/highgui/CMakeLists.txt b/modules/highgui/CMakeLists.txt index 5eb9f5ab5e..bc31b84c74 100644 --- a/modules/highgui/CMakeLists.txt +++ b/modules/highgui/CMakeLists.txt @@ -141,7 +141,7 @@ endif() if(TARGET ocv.3rdparty.win32ui) if("win32ui" IN_LIST HIGHGUI_PLUGIN_LIST OR HIGHGUI_PLUGIN_LIST STREQUAL "all") ocv_create_builtin_highgui_plugin(opencv_highgui_win32 ocv.3rdparty.win32ui "window_w32.cpp") - else() + elseif(NOT OPENCV_HIGHGUI_BUILTIN_BACKEND) set(OPENCV_HIGHGUI_BUILTIN_BACKEND "WIN32UI") list(APPEND highgui_srcs ${CMAKE_CURRENT_LIST_DIR}/src/window_w32.cpp) list(APPEND tgts ocv.3rdparty.win32ui) From 4c3f9b2ef4ca1a74c0fd15f1747bd131e249c57f Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Sun, 4 Jul 2021 13:07:34 +0300 Subject: [PATCH 045/262] cmake: update Halide detection --- cmake/OpenCVDetectHalide.cmake | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/cmake/OpenCVDetectHalide.cmake b/cmake/OpenCVDetectHalide.cmake index 790f692056..4828c299ae 100644 --- a/cmake/OpenCVDetectHalide.cmake +++ b/cmake/OpenCVDetectHalide.cmake @@ -9,9 +9,14 @@ set(HALIDE_ROOT_DIR "${HALIDE_ROOT_DIR}" CACHE PATH "Halide root directory") if(NOT HAVE_HALIDE) find_package(Halide QUIET) # Try CMake-based config files if(Halide_FOUND) - set(HALIDE_INCLUDE_DIRS "${Halide_INCLUDE_DIRS}" CACHE PATH "Halide include directories" FORCE) - set(HALIDE_LIBRARIES "${Halide_LIBRARIES}" CACHE PATH "Halide libraries" FORCE) - set(HAVE_HALIDE TRUE) + if(TARGET Halide::Halide) # modern Halide scripts defines imported target + set(HALIDE_INCLUDE_DIRS "") + set(HALIDE_LIBRARIES "Halide::Halide") + set(HAVE_HALIDE TRUE) + else() + # using HALIDE_INCLUDE_DIRS / Halide_LIBRARIES + set(HAVE_HALIDE TRUE) + endif() endif() endif() @@ -28,18 +33,15 @@ if(NOT HAVE_HALIDE AND HALIDE_ROOT_DIR) ) if(HALIDE_LIBRARY AND HALIDE_INCLUDE_DIR) # TODO try_compile - set(HALIDE_INCLUDE_DIRS "${HALIDE_INCLUDE_DIR}" CACHE PATH "Halide include directories" FORCE) - set(HALIDE_LIBRARIES "${HALIDE_LIBRARY}" CACHE PATH "Halide libraries" FORCE) + set(HALIDE_INCLUDE_DIRS "${HALIDE_INCLUDE_DIR}") + set(HALIDE_LIBRARIES "${HALIDE_LIBRARY}") set(HAVE_HALIDE TRUE) endif() - if(NOT HAVE_HALIDE) - ocv_clear_vars(HALIDE_LIBRARIES HALIDE_INCLUDE_DIRS CACHE) - endif() endif() if(HAVE_HALIDE) - include_directories(${HALIDE_INCLUDE_DIRS}) + if(HALIDE_INCLUDE_DIRS) + include_directories(${HALIDE_INCLUDE_DIRS}) + endif() list(APPEND OPENCV_LINKER_LIBS ${HALIDE_LIBRARIES}) -else() - ocv_clear_vars(HALIDE_INCLUDE_DIRS HALIDE_LIBRARIES) endif() From cbff19ff1a8be873c3c2aa0cafa2ff77738cc437 Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Sun, 4 Jul 2021 17:33:18 +0300 Subject: [PATCH 046/262] highgui: fix win32 backend behavior --- modules/highgui/src/window.cpp | 2 +- modules/highgui/src/window_w32.cpp | 25 +++++++++++++++++-------- 2 files changed, 18 insertions(+), 9 deletions(-) diff --git a/modules/highgui/src/window.cpp b/modules/highgui/src/window.cpp index d1ccd1dbc3..d9481de6da 100644 --- a/modules/highgui/src/window.cpp +++ b/modules/highgui/src/window.cpp @@ -990,7 +990,7 @@ void cv::imshow( const String& winname, InputArray _img ) auto backend = getCurrentUIBackend(); if (backend) { - auto window = backend->createWindow(winname, WINDOW_NORMAL); + auto window = backend->createWindow(winname, WINDOW_AUTOSIZE); if (!window) { CV_LOG_ERROR(NULL, "OpenCV/UI: Can't create window: '" << winname << "'"); diff --git a/modules/highgui/src/window_w32.cpp b/modules/highgui/src/window_w32.cpp index d9a9d73222..716af1094c 100644 --- a/modules/highgui/src/window_w32.cpp +++ b/modules/highgui/src/window_w32.cpp @@ -2123,7 +2123,7 @@ static void showSaveDialog(CvWindow& window) SIZE sz; int channels; void* data; - if (icvGetBitmapData(window, sz, channels, data)) + if (!icvGetBitmapData(window, sz, channels, data)) return; // nothing to save char szFileName[MAX_PATH] = ""; @@ -2206,6 +2206,7 @@ static bool handleMessage(MSG& message, int& keyCode) switch (message.message) { case WM_DESTROY: + // fallthru case WM_CHAR: DispatchMessage(&message); keyCode = (int)message.wParam; @@ -2221,6 +2222,20 @@ static bool handleMessage(MSG& message, int& keyCode) break; case WM_KEYDOWN: + // Intercept Ctrl+C for copy to clipboard + if ('C' == message.wParam && (::GetKeyState(VK_CONTROL) >> 15)) + { + ::SendMessage(message.hwnd, WM_COPY, 0, 0); + return false; + } + + // Intercept Ctrl+S for "save as" dialog + if ('S' == message.wParam && (::GetKeyState(VK_CONTROL) >> 15)) + { + showSaveDialog(window); + return false; + } + TranslateMessage(&message); if ((message.wParam >= VK_F1 && message.wParam <= VK_F24) || message.wParam == VK_HOME || message.wParam == VK_END || @@ -2235,13 +2250,7 @@ static bool handleMessage(MSG& message, int& keyCode) return true; } - // Intercept Ctrl+C for copy to clipboard - if ('C' == message.wParam && (::GetKeyState(VK_CONTROL) >> 15)) - ::SendMessage(message.hwnd, WM_COPY, 0, 0); - - // Intercept Ctrl+S for "save as" dialog - if ('S' == message.wParam && (::GetKeyState(VK_CONTROL) >> 15)) - showSaveDialog(window); + // fallthru default: DispatchMessage(&message); From 591708903b4393b5d33c3a2f4af5be4daeb94c4d Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Sun, 4 Jul 2021 21:10:13 +0000 Subject: [PATCH 047/262] release: OpenCV 3.4.15 --- modules/core/include/opencv2/core/version.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/core/include/opencv2/core/version.hpp b/modules/core/include/opencv2/core/version.hpp index a71c13ebd1..ecd8e733c7 100644 --- a/modules/core/include/opencv2/core/version.hpp +++ b/modules/core/include/opencv2/core/version.hpp @@ -8,7 +8,7 @@ #define CV_VERSION_MAJOR 3 #define CV_VERSION_MINOR 4 #define CV_VERSION_REVISION 15 -#define CV_VERSION_STATUS "-pre" +#define CV_VERSION_STATUS "" #define CVAUX_STR_EXP(__A) #__A #define CVAUX_STR(__A) CVAUX_STR_EXP(__A) From ad6e82942b37be8ee2c71c1d9bc7fe79cd16f7ab Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Mon, 5 Jul 2021 12:03:22 +0000 Subject: [PATCH 048/262] release: OpenCV 4.5.3 --- modules/core/include/opencv2/core/version.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/core/include/opencv2/core/version.hpp b/modules/core/include/opencv2/core/version.hpp index 4757e30f9b..c9f4b7ed11 100644 --- a/modules/core/include/opencv2/core/version.hpp +++ b/modules/core/include/opencv2/core/version.hpp @@ -8,7 +8,7 @@ #define CV_VERSION_MAJOR 4 #define CV_VERSION_MINOR 5 #define CV_VERSION_REVISION 3 -#define CV_VERSION_STATUS "-pre" +#define CV_VERSION_STATUS "" #define CVAUX_STR_EXP(__A) #__A #define CVAUX_STR(__A) CVAUX_STR_EXP(__A) From ed2a69839293625ecae9f2b9e53eeefc23ceedd6 Mon Sep 17 00:00:00 2001 From: Maxim Pashchenkov Date: Tue, 6 Jul 2021 21:35:41 +0300 Subject: [PATCH 049/262] Merge pull request #20359 from mpashchenkov:mp/onnx-tests G-API: ONNX. Skip tests. * imread for every test * Changed name for Yolo function --- .../gapi/test/infer/gapi_infer_onnx_test.cpp | 107 ++++++++++-------- 1 file changed, 62 insertions(+), 45 deletions(-) diff --git a/modules/gapi/test/infer/gapi_infer_onnx_test.cpp b/modules/gapi/test/infer/gapi_infer_onnx_test.cpp index ef192b9d6a..b1bf9c9356 100644 --- a/modules/gapi/test/infer/gapi_infer_onnx_test.cpp +++ b/modules/gapi/test/infer/gapi_infer_onnx_test.cpp @@ -67,17 +67,17 @@ struct ONNXInitPath { static ONNXInitPath g_init_path; cv::Mat initMatrixRandU(const int type, const cv::Size& sz_in) { - const cv::Mat in_mat1 = cv::Mat(sz_in, type); + const cv::Mat in_mat = cv::Mat(sz_in, type); if (CV_MAT_DEPTH(type) < CV_32F) { - cv::randu(in_mat1, cv::Scalar::all(0), cv::Scalar::all(255)); + cv::randu(in_mat, cv::Scalar::all(0), cv::Scalar::all(255)); } else { const int fscale = 256; // avoid bits near ULP, generate stable test input - cv::Mat in_mat32s(in_mat1.size(), CV_MAKE_TYPE(CV_32S, CV_MAT_CN(type))); + cv::Mat in_mat32s(in_mat.size(), CV_MAKE_TYPE(CV_32S, CV_MAT_CN(type))); cv::randu(in_mat32s, cv::Scalar::all(0), cv::Scalar::all(255 * fscale)); - in_mat32s.convertTo(in_mat1, type, 1.0f / fscale, 0); + in_mat32s.convertTo(in_mat, type, 1.0f / fscale, 0); } - return in_mat1; + return in_mat; } } // anonymous namespace namespace opencv_test @@ -319,15 +319,13 @@ public: size_t num_in, num_out; std::vector out_gapi; std::vector out_onnx; - cv::Mat in_mat1; + cv::Mat in_mat; ONNXtest() { initTestDataPath(); env = Ort::Env(ORT_LOGGING_LEVEL_WARNING, "test"); memory_info = Ort::MemoryInfo::CreateCpu(OrtArenaAllocator, OrtMemTypeDefault); out_gapi.resize(1); - // FIXME: It should be an image from own (gapi) directory in opencv extra - in_mat1 = cv::imread(findDataFile("cv/dpm/cat.png")); } template @@ -463,13 +461,9 @@ public: cv::Rect(cv::Point{70, 10}, cv::Size{20, 260}), cv::Rect(cv::Point{5, 15}, cv::Size{200, 160}), }; - cv::Mat m_in_y; - cv::Mat m_in_uv; - virtual void SetUp() { - cv::Size sz{640, 480}; - m_in_y = initMatrixRandU(CV_8UC1, sz); - m_in_uv = initMatrixRandU(CV_8UC2, sz / 2); - } + const cv::Size sz{640, 480}; + const cv::Mat m_in_y = initMatrixRandU(CV_8UC1, sz); + const cv::Mat m_in_uv = initMatrixRandU(CV_8UC2, sz / 2); }; class ONNXGRayScale : public ONNXtest { @@ -545,20 +539,20 @@ class ONNXYoloV3 : public ONNXWithRemap { public: std::vector ins; -private: - virtual void SetUp() { + void constructYoloInputs(const cv::Mat& src) { const int yolo_in_h = 416; const int yolo_in_w = 416; cv::Mat yolov3_input, shape, prep_mat; - cv::resize(in_mat1, yolov3_input, cv::Size(yolo_in_w, yolo_in_h)); + cv::resize(src, yolov3_input, cv::Size(yolo_in_w, yolo_in_h)); shape.create(cv::Size(2, 1), CV_32F); float* ptr = shape.ptr(); - ptr[0] = in_mat1.cols; - ptr[1] = in_mat1.rows; + ptr[0] = src.cols; + ptr[1] = src.rows; preprocess(yolov3_input, prep_mat); ins = {prep_mat, shape}; } +private: void preprocess(const cv::Mat& src, cv::Mat& dst) { cv::Mat cvt; src.convertTo(cvt, CV_32F, 1.f / 255.f); @@ -571,9 +565,10 @@ private: TEST_F(ONNXClassification, Infer) { useModel("classification/squeezenet/model/squeezenet1.0-9"); + in_mat = cv::imread(findDataFile("cv/dpm/cat.png", false)); // ONNX_API code cv::Mat processed_mat; - preprocess(in_mat1, processed_mat); + preprocess(in_mat, processed_mat); infer(processed_mat, out_onnx); // G_API code G_API_NET(SqueezNet, , "squeeznet"); @@ -583,7 +578,7 @@ TEST_F(ONNXClassification, Infer) // NOTE: We have to normalize U8 tensor // so cfgMeanStd() is here auto net = cv::gapi::onnx::Params { model_path }.cfgMeanStd({ mean }, { std }); - comp.apply(cv::gin(in_mat1), + comp.apply(cv::gin(in_mat), cv::gout(out_gapi.front()), cv::compile_args(cv::gapi::networks(net))); // Validate @@ -593,9 +588,10 @@ TEST_F(ONNXClassification, Infer) TEST_F(ONNXClassification, InferTensor) { useModel("classification/squeezenet/model/squeezenet1.0-9"); + in_mat = cv::imread(findDataFile("cv/dpm/cat.png", false)); // Create tensor cv::Mat tensor; - preprocess(in_mat1, tensor); + preprocess(in_mat, tensor); // ONNX_API code infer(tensor, out_onnx); // G_API code @@ -614,10 +610,11 @@ TEST_F(ONNXClassification, InferTensor) TEST_F(ONNXClassification, InferROI) { useModel("classification/squeezenet/model/squeezenet1.0-9"); + in_mat = cv::imread(findDataFile("cv/dpm/cat.png", false)); const auto ROI = rois.at(0); // ONNX_API code cv::Mat roi_mat; - preprocess(in_mat1(ROI), roi_mat); + preprocess(in_mat(ROI), roi_mat); infer(roi_mat, out_onnx); // G_API code G_API_NET(SqueezNet, , "squeeznet"); @@ -628,7 +625,7 @@ TEST_F(ONNXClassification, InferROI) // NOTE: We have to normalize U8 tensor // so cfgMeanStd() is here auto net = cv::gapi::onnx::Params { model_path }.cfgMeanStd({ mean }, { std }); - comp.apply(cv::gin(in_mat1, ROI), + comp.apply(cv::gin(in_mat, ROI), cv::gout(out_gapi.front()), cv::compile_args(cv::gapi::networks(net))); // Validate @@ -638,10 +635,11 @@ TEST_F(ONNXClassification, InferROI) TEST_F(ONNXClassification, InferROIList) { useModel("classification/squeezenet/model/squeezenet1.0-9"); + in_mat = cv::imread(findDataFile("cv/dpm/cat.png", false)); // ONNX_API code for (size_t i = 0; i < rois.size(); ++i) { cv::Mat roi_mat; - preprocess(in_mat1(rois[i]), roi_mat); + preprocess(in_mat(rois[i]), roi_mat); infer(roi_mat, out_onnx); } // G_API code @@ -653,7 +651,7 @@ TEST_F(ONNXClassification, InferROIList) // NOTE: We have to normalize U8 tensor // so cfgMeanStd() is here auto net = cv::gapi::onnx::Params { model_path }.cfgMeanStd({ mean }, { std }); - comp.apply(cv::gin(in_mat1, rois), + comp.apply(cv::gin(in_mat, rois), cv::gout(out_gapi), cv::compile_args(cv::gapi::networks(net))); // Validate @@ -663,10 +661,11 @@ TEST_F(ONNXClassification, InferROIList) TEST_F(ONNXClassification, Infer2ROIList) { useModel("classification/squeezenet/model/squeezenet1.0-9"); + in_mat = cv::imread(findDataFile("cv/dpm/cat.png", false)); // ONNX_API code for (size_t i = 0; i < rois.size(); ++i) { cv::Mat roi_mat; - preprocess(in_mat1(rois[i]), roi_mat); + preprocess(in_mat(rois[i]), roi_mat); infer(roi_mat, out_onnx); } // G_API code @@ -678,7 +677,7 @@ TEST_F(ONNXClassification, Infer2ROIList) // NOTE: We have to normalize U8 tensor // so cfgMeanStd() is here auto net = cv::gapi::onnx::Params { model_path }.cfgMeanStd({ mean }, { std }); - comp.apply(cv::gin(in_mat1, rois), + comp.apply(cv::gin(in_mat, rois), cv::gout(out_gapi), cv::compile_args(cv::gapi::networks(net))); // Validate @@ -688,9 +687,10 @@ TEST_F(ONNXClassification, Infer2ROIList) TEST_F(ONNXWithRemap, InferDynamicInputTensor) { useModel("object_detection_segmentation/tiny-yolov2/model/tinyyolov2-8"); + in_mat = cv::imread(findDataFile("cv/dpm/cat.png", false)); // Create tensor cv::Mat cvt, rsz, tensor; - cv::resize(in_mat1, rsz, cv::Size{416, 416}); + cv::resize(in_mat, rsz, cv::Size{416, 416}); rsz.convertTo(cvt, CV_32F, 1.f / 255.f); toCHW(cvt, tensor); tensor = tensor.reshape(1, {1, 3, 416, 416}); @@ -714,9 +714,10 @@ TEST_F(ONNXWithRemap, InferDynamicInputTensor) TEST_F(ONNXGRayScale, InferImage) { useModel("body_analysis/emotion_ferplus/model/emotion-ferplus-8"); + in_mat = cv::imread(findDataFile("cv/dpm/cat.png", false)); // ONNX_API code cv::Mat prep_mat; - preprocess(in_mat1, prep_mat); + preprocess(in_mat, prep_mat); infer(prep_mat, out_onnx); // G_API code G_API_NET(EmotionNet, , "emotion-ferplus"); @@ -725,7 +726,7 @@ TEST_F(ONNXGRayScale, InferImage) cv::GComputation comp(cv::GIn(in), cv::GOut(out)); auto net = cv::gapi::onnx::Params { model_path } .cfgNormalize({ false }); // model accepts 0..255 range in FP32; - comp.apply(cv::gin(in_mat1), + comp.apply(cv::gin(in_mat), cv::gout(out_gapi.front()), cv::compile_args(cv::gapi::networks(net))); // Validate @@ -735,8 +736,9 @@ TEST_F(ONNXGRayScale, InferImage) TEST_F(ONNXWithRemap, InferMultiOutput) { useModel("object_detection_segmentation/ssd-mobilenetv1/model/ssd_mobilenet_v1_10"); + in_mat = cv::imread(findDataFile("cv/dpm/cat.png", false)); // ONNX_API code - const auto prep_mat = in_mat1.reshape(1, {1, in_mat1.rows, in_mat1.cols, in_mat1.channels()}); + const auto prep_mat = in_mat.reshape(1, {1, in_mat.rows, in_mat.cols, in_mat.channels()}); infer(prep_mat, out_onnx); cv::Mat onnx_conv_out({1, 1, 200, 7}, CV_32F); remapToIESSDOut({out_onnx[3], out_onnx[0], out_onnx[2], out_onnx[1]}, onnx_conv_out); @@ -750,7 +752,7 @@ TEST_F(ONNXWithRemap, InferMultiOutput) auto net = cv::gapi::onnx::Params{ model_path } .cfgOutputLayers({"detection_output"}) .cfgPostProc({cv::GMatDesc{CV_32F, {1, 1, 200, 7}}}, remapSSDPorts); - comp.apply(cv::gin(in_mat1), + comp.apply(cv::gin(in_mat), cv::gout(out_gapi.front()), cv::compile_args(cv::gapi::networks(net))); // Validate @@ -760,12 +762,13 @@ TEST_F(ONNXWithRemap, InferMultiOutput) TEST_F(ONNXMediaFrame, InferBGR) { useModel("classification/squeezenet/model/squeezenet1.0-9"); + in_mat = cv::imread(findDataFile("cv/dpm/cat.png", false)); // ONNX_API code cv::Mat processed_mat; - preprocess(in_mat1, processed_mat); + preprocess(in_mat, processed_mat); infer(processed_mat, out_onnx); // G_API code - auto frame = MediaFrame::Create(in_mat1); + auto frame = MediaFrame::Create(in_mat); G_API_NET(SqueezNet, , "squeeznet"); cv::GFrame in; cv::GMat out = cv::gapi::infer(in); @@ -783,6 +786,7 @@ TEST_F(ONNXMediaFrame, InferBGR) TEST_F(ONNXMediaFrame, InferYUV) { useModel("classification/squeezenet/model/squeezenet1.0-9"); + in_mat = cv::imread(findDataFile("cv/dpm/cat.png", false)); const auto frame = MediaFrame::Create(m_in_y, m_in_uv); // ONNX_API code cv::Mat pp; @@ -808,10 +812,11 @@ TEST_F(ONNXMediaFrame, InferYUV) TEST_F(ONNXMediaFrame, InferROIBGR) { useModel("classification/squeezenet/model/squeezenet1.0-9"); - auto frame = MediaFrame::Create(in_mat1); + in_mat = cv::imread(findDataFile("cv/dpm/cat.png", false)); + auto frame = MediaFrame::Create(in_mat); // ONNX_API code cv::Mat roi_mat; - preprocess(in_mat1(rois.front()), roi_mat); + preprocess(in_mat(rois.front()), roi_mat); infer(roi_mat, out_onnx); // G_API code G_API_NET(SqueezNet, , "squeeznet"); @@ -832,6 +837,7 @@ TEST_F(ONNXMediaFrame, InferROIBGR) TEST_F(ONNXMediaFrame, InferROIYUV) { useModel("classification/squeezenet/model/squeezenet1.0-9"); + in_mat = cv::imread(findDataFile("cv/dpm/cat.png", false)); const auto frame = MediaFrame::Create(m_in_y, m_in_uv); // ONNX_API code cv::Mat pp; @@ -858,11 +864,12 @@ TEST_F(ONNXMediaFrame, InferROIYUV) TEST_F(ONNXMediaFrame, InferListBGR) { useModel("classification/squeezenet/model/squeezenet1.0-9"); - const auto frame = MediaFrame::Create(in_mat1); + in_mat = cv::imread(findDataFile("cv/dpm/cat.png", false)); + const auto frame = MediaFrame::Create(in_mat); // ONNX_API code for (size_t i = 0; i < rois.size(); ++i) { cv::Mat roi_mat; - preprocess(in_mat1(rois[i]), roi_mat); + preprocess(in_mat(rois[i]), roi_mat); infer(roi_mat, out_onnx); } // G_API code @@ -884,6 +891,7 @@ TEST_F(ONNXMediaFrame, InferListBGR) TEST_F(ONNXMediaFrame, InferListYUV) { useModel("classification/squeezenet/model/squeezenet1.0-9"); + in_mat = cv::imread(findDataFile("cv/dpm/cat.png", false)); const auto frame = MediaFrame::Create(m_in_y, m_in_uv); // ONNX_API code cv::Mat pp; @@ -911,8 +919,9 @@ TEST_F(ONNXMediaFrame, InferListYUV) TEST_F(ONNXRCNN, InferWithDisabledOut) { useModel("object_detection_segmentation/faster-rcnn/model/FasterRCNN-10"); + in_mat = cv::imread(findDataFile("cv/dpm/cat.png", false)); cv::Mat pp; - preprocess(in_mat1, pp); + preprocess(in_mat, pp); // ONNX_API code infer(pp, out_onnx, {"6379", "6383"}); // G_API code @@ -937,11 +946,12 @@ TEST_F(ONNXRCNN, InferWithDisabledOut) TEST_F(ONNXMediaFrame, InferList2BGR) { useModel("classification/squeezenet/model/squeezenet1.0-9"); - const auto frame = MediaFrame::Create(in_mat1); + in_mat = cv::imread(findDataFile("cv/dpm/cat.png", false)); + const auto frame = MediaFrame::Create(in_mat); // ONNX_API code for (size_t i = 0; i < rois.size(); ++i) { cv::Mat roi_mat; - preprocess(in_mat1(rois[i]), roi_mat); + preprocess(in_mat(rois[i]), roi_mat); infer(roi_mat, out_onnx); } // G_API code @@ -963,6 +973,7 @@ TEST_F(ONNXMediaFrame, InferList2BGR) TEST_F(ONNXMediaFrame, InferList2YUV) { useModel("classification/squeezenet/model/squeezenet1.0-9"); + in_mat = cv::imread(findDataFile("cv/dpm/cat.png", false)); const auto frame = MediaFrame::Create(m_in_y, m_in_uv); // ONNX_API code cv::Mat pp; @@ -991,6 +1002,8 @@ TEST_F(ONNXMediaFrame, InferList2YUV) TEST_F(ONNXYoloV3, InferConstInput) { useModel("object_detection_segmentation/yolov3/model/yolov3-10"); + in_mat = cv::imread(findDataFile("cv/dpm/cat.png", false)); + constructYoloInputs(in_mat); // ONNX_API code infer(ins, out_onnx); // G_API code @@ -1022,6 +1035,8 @@ TEST_F(ONNXYoloV3, InferBSConstInput) // and all input layer names are specified. // Const input has the advantage. It is expected behavior. useModel("object_detection_segmentation/yolov3/model/yolov3-10"); + in_mat = cv::imread(findDataFile("cv/dpm/cat.png", false)); + constructYoloInputs(in_mat); // Tensor with incorrect image size // is used for check case when InputLayers and constInput have same names cv::Mat bad_shape; @@ -1059,8 +1074,9 @@ TEST_F(ONNXYoloV3, InferBSConstInput) TEST_F(ONNXRCNN, ConversionInt64to32) { useModel("object_detection_segmentation/faster-rcnn/model/FasterRCNN-10"); + in_mat = cv::imread(findDataFile("cv/dpm/cat.png", false)); cv::Mat dst; - preprocess(in_mat1, dst); + preprocess(in_mat, dst); // ONNX_API code infer(dst, out_onnx); // G_API code @@ -1087,6 +1103,7 @@ TEST_F(ONNXRCNN, ConversionInt64to32) TEST_F(ONNXWithRemap, InferOutReallocation) { useModel("object_detection_segmentation/ssd-mobilenetv1/model/ssd_mobilenet_v1_10"); + in_mat = cv::imread(findDataFile("cv/dpm/cat.png", false)); // G_API code G_API_NET(MobileNet, , "ssd_mobilenet"); auto net = cv::gapi::onnx::Params{model_path} @@ -1096,7 +1113,7 @@ TEST_F(ONNXWithRemap, InferOutReallocation) cv::GMat out1; out1 = cv::gapi::infer(in); cv::GComputation comp(cv::GIn(in), cv::GOut(out1)); - EXPECT_THROW(comp.apply(cv::gin(in_mat1), + EXPECT_THROW(comp.apply(cv::gin(in_mat), cv::gout(out_gapi[0]), cv::compile_args(cv::gapi::networks(net))), std::exception); } From 5627a0cbdf94a053e55aa4749a5c83d8b18ad34c Mon Sep 17 00:00:00 2001 From: Xinguang Bian Date: Wed, 7 Jul 2021 12:35:11 +0800 Subject: [PATCH 050/262] fix scale problem in DefaultViewPort::controlImagePosition() --- modules/highgui/src/window_QT.cpp | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/modules/highgui/src/window_QT.cpp b/modules/highgui/src/window_QT.cpp index 68289eb876..8dff03117e 100644 --- a/modules/highgui/src/window_QT.cpp +++ b/modules/highgui/src/window_QT.cpp @@ -2883,18 +2883,19 @@ inline bool DefaultViewPort::isSameSize(IplImage* img1, IplImage* img2) void DefaultViewPort::controlImagePosition() { qreal left, top, right, bottom; + qreal factor = 1.0 / param_matrixWorld.m11(); //after check top-left, bottom right corner to avoid getting "out" during zoom/panning param_matrixWorld.map(0,0,&left,&top); if (left > 0) { - param_matrixWorld.translate(-left,0); + param_matrixWorld.translate(-left * factor, 0); left = 0; } if (top > 0) { - param_matrixWorld.translate(0,-top); + param_matrixWorld.translate(0, -top * factor); top = 0; } //------- @@ -2903,12 +2904,12 @@ void DefaultViewPort::controlImagePosition() param_matrixWorld.map(sizeImage.width(),sizeImage.height(),&right,&bottom); if (right < sizeImage.width()) { - param_matrixWorld.translate(sizeImage.width()-right,0); + param_matrixWorld.translate((sizeImage.width() - right) * factor, 0); right = sizeImage.width(); } if (bottom < sizeImage.height()) { - param_matrixWorld.translate(0,sizeImage.height()-bottom); + param_matrixWorld.translate(0, (sizeImage.height() - bottom) * factor); bottom = sizeImage.height(); } From c0f63eb21f044415bf7a21cb98fa6f4ae2586e1b Mon Sep 17 00:00:00 2001 From: Sergey Ivanov Date: Wed, 7 Jul 2021 15:33:40 +0300 Subject: [PATCH 051/262] Merge pull request #20039 from sivanov-work:gapi_empty_input G-API: Implement variant visit() * Add variant visitor, use visitor for check compile args * Fix GAPI UT: variant *compiler * Aling apply_visior with std, fix indentations * Fix compilation (included compiler_hints.hpp) * Fix compilation (due gapi standalone) * Fix compilation2 (Docs) * Add Lambdas overload, Refactor visit() * Add ReturnType auto deduction * Fix comilation * Fix compilation * Fix warnings * Try to fix MSVC14 * Fix docs * Try fix Win compile * Fix Docs again * Revert GAPI empty input fix * Apply comment for `tuple_element` * Add std::decay for std::base_of to work arounf armv7 problem * Apply review comments * Apply review comments: added comment & removed unused args * Fix docs compilation --- .../include/opencv2/gapi/gtype_traits.hpp | 13 - modules/gapi/include/opencv2/gapi/gtyped.hpp | 1 - .../gapi/include/opencv2/gapi/util/util.hpp | 37 +++ .../include/opencv2/gapi/util/variant.hpp | 226 +++++++++++++++- modules/gapi/src/api/gproto.cpp | 5 - modules/gapi/test/util/variant_tests.cpp | 250 ++++++++++++++++++ 6 files changed, 512 insertions(+), 20 deletions(-) diff --git a/modules/gapi/include/opencv2/gapi/gtype_traits.hpp b/modules/gapi/include/opencv2/gapi/gtype_traits.hpp index 0b11b18485..2e8dcb1aec 100644 --- a/modules/gapi/include/opencv2/gapi/gtype_traits.hpp +++ b/modules/gapi/include/opencv2/gapi/gtype_traits.hpp @@ -43,19 +43,6 @@ namespace detail GOPAQUE, // a cv::GOpaqueU (note - exactly GOpaqueU, not GOpaque!) }; - template - constexpr const char* meta_to_string() noexcept; - template<> - constexpr const char* meta_to_string() noexcept { return "GMatDesc"; } - template<> - constexpr const char* meta_to_string() noexcept { return "GScalarDesc"; } - template<> - constexpr const char* meta_to_string() noexcept { return "GArrayDesc"; } - template<> - constexpr const char* meta_to_string() noexcept { return "GOpaqueDesc"; } - template<> - constexpr const char* meta_to_string() noexcept { return "GFrameDesc";} - // Describe G-API types (G-types) with traits. Mostly used by // cv::GArg to store meta information about types passed into // operation arguments. Please note that cv::GComputation is diff --git a/modules/gapi/include/opencv2/gapi/gtyped.hpp b/modules/gapi/include/opencv2/gapi/gtyped.hpp index 27d9777944..6fe52a62e1 100644 --- a/modules/gapi/include/opencv2/gapi/gtyped.hpp +++ b/modules/gapi/include/opencv2/gapi/gtyped.hpp @@ -35,7 +35,6 @@ namespace detail template<> struct ProtoToMeta { using type = cv::GScalarDesc; }; template struct ProtoToMeta > { using type = cv::GArrayDesc; }; template struct ProtoToMeta > { using type = cv::GOpaqueDesc; }; - template<> struct ProtoToMeta { using type = cv::GFrameDesc; }; template using ProtoToMetaT = typename ProtoToMeta::type; //workaround for MSVC 19.0 bug diff --git a/modules/gapi/include/opencv2/gapi/util/util.hpp b/modules/gapi/include/opencv2/gapi/util/util.hpp index afcf5596fd..c6ad0632e2 100644 --- a/modules/gapi/include/opencv2/gapi/util/util.hpp +++ b/modules/gapi/include/opencv2/gapi/util/util.hpp @@ -117,6 +117,43 @@ namespace detail static type get(std::tuple&& objs) { return std::forward>(objs); } }; } // namespace detail + +namespace util +{ +template +struct overload_lamba_set; + +template +struct overload_lamba_set : public L1 +{ + overload_lamba_set(L1&& lambda) : L1(std::move(lambda)) {} + overload_lamba_set(const L1& lambda) : L1(lambda) {} + + using L1::operator(); +}; + +template +struct overload_lamba_set : public L1, public overload_lamba_set +{ + using base_type = overload_lamba_set; + overload_lamba_set(L1 &&lambda1, L&& ...lambdas): + L1(std::move(lambda1)), + base_type(std::forward(lambdas)...) {} + + overload_lamba_set(const L1 &lambda1, L&& ...lambdas): + L1(lambda1), + base_type(std::forward(lambdas)...) {} + + using L1::operator(); + using base_type::operator(); +}; + +template +overload_lamba_set overload_lambdas(L&& ...lambdas) +{ + return overload_lamba_set(std::forward(lambdas)...); +} +} } // namespace cv // \endcond diff --git a/modules/gapi/include/opencv2/gapi/util/variant.hpp b/modules/gapi/include/opencv2/gapi/util/variant.hpp index 71a06d2dcf..f412110deb 100644 --- a/modules/gapi/include/opencv2/gapi/util/variant.hpp +++ b/modules/gapi/include/opencv2/gapi/util/variant.hpp @@ -11,6 +11,7 @@ #include #include +#include #include #include // max_of_t #include @@ -44,6 +45,12 @@ namespace util static const constexpr std::size_t value = detail::type_list_index_helper<0, Target, Types...>::value; }; + template + struct type_list_element + { + using type = typename std::tuple_element >::type; + }; + class bad_variant_access: public std::exception { public: @@ -233,9 +240,87 @@ namespace util template const T& get(const util::variant &v); + template + typename util::type_list_element::type& get(util::variant &v); + + template + const typename util::type_list_element::type& get(const util::variant &v); + template bool holds_alternative(const util::variant &v) noexcept; + + // Visitor + namespace detail + { + struct visitor_interface {}; + + // Class `visitor_return_type_deduction_helper` + // introduces solution for deduction `return_type` in `visit` function in common way + // for both Lambda and class Visitor and keep one interface invocation point: `visit` only + // his helper class is required to unify return_type deduction mechanism because + // for Lambda it is possible to take type of `decltype(visitor(get<0>(var)))` + // but for class Visitor there is no operator() in base case, + // because it provides `operator() (std::size_t index, ...)` + // So `visitor_return_type_deduction_helper` expose `operator()` + // uses only for class Visitor only for deduction `return type` in visit() + template + struct visitor_return_type_deduction_helper + { + using return_type = R; + + // to be used in Lambda return type deduction context only + template + return_type operator() (T&&); + }; + } + + // Special purpose `static_visitor` can receive additional arguments + template + struct static_visitor : public detail::visitor_interface, + public detail::visitor_return_type_deduction_helper { + + // assign responsibility for return type deduction to helper class + using return_type = typename detail::visitor_return_type_deduction_helper::return_type; + using detail::visitor_return_type_deduction_helper::operator(); + friend Impl; + + template + return_type operator() (std::size_t index, VariantValue&& value, Args&& ...args) + { + suppress_unused_warning(index); + return static_cast(this)-> visit( + std::forward(value), + std::forward(args)...); + } + }; + + // Special purpose `static_indexed_visitor` can receive additional arguments + // And make forwarding current variant index as runtime function argument to its `Impl` + template + struct static_indexed_visitor : public detail::visitor_interface, + public detail::visitor_return_type_deduction_helper { + + // assign responsibility for return type deduction to helper class + using return_type = typename detail::visitor_return_type_deduction_helper::return_type; + using detail::visitor_return_type_deduction_helper::operator(); + friend Impl; + + template + return_type operator() (std::size_t Index, VariantValue&& value, Args&& ...args) + { + return static_cast(this)-> visit(Index, + std::forward(value), + std::forward(args)...); + } + }; + + template + struct variant_size; + + template + struct variant_size> + : std::integral_constant { }; // FIXME: T&&, const TT&& versions. // Implementation ////////////////////////////////////////////////////////// @@ -402,6 +487,22 @@ namespace util throw_error(bad_variant_access()); } + template + typename util::type_list_element::type& get(util::variant &v) + { + using ReturnType = typename util::type_list_element::type; + return const_cast(get(static_cast &>(v))); + } + + template + const typename util::type_list_element::type& get(const util::variant &v) + { + static_assert(Index < sizeof...(Types), + "`Index` it out of bound of `util::variant` type list"); + using ReturnType = typename util::type_list_element::type; + return get(v); + } + template bool holds_alternative(const util::variant &v) noexcept { @@ -428,7 +529,130 @@ namespace util { return !(lhs == rhs); } -} // namespace cv + +namespace detail +{ + // terminate recursion implementation for `non-void` ReturnType + template + ReturnType apply_visitor_impl(Visitor&&, Variant&, + std::true_type, std::false_type, + VisitorArgs&& ...) + { + return {}; + } + + // terminate recursion implementation for `void` ReturnType + template + void apply_visitor_impl(Visitor&&, Variant&, + std::true_type, std::true_type, + VisitorArgs&& ...) + { + } + + // Intermediate resursion processor for Lambda Visitors + template + typename std::enable_if::type>::value, ReturnType>::type + apply_visitor_impl(Visitor&& visitor, Variant&& v, std::false_type not_processed, + std::integral_constant should_no_return, + VisitorArgs&& ...args) + { + static_assert(std::is_same(v)))>::value, + "Different `ReturnType`s detected! All `Visitor::visit` or `overload_lamba_set`" + " must return the same type"); + suppress_unused_warning(not_processed); + if (v.index() == CurIndex) + { + return visitor.operator()(get(v), std::forward(args)... ); + } + + using is_variant_processed_t = std::integral_constant= ElemCount>; + return apply_visitor_impl( + std::forward(visitor), + std::forward(v), + is_variant_processed_t{}, + should_no_return, + std::forward(args)...); + } + + //Visual Studio 2014 compilation fix: cast visitor to base class before invoke operator() + template + typename std::enable_if::type>, + typename std::decay::type>::value, ReturnType>::type + invoke_class_visitor(Visitor& visitor, Value&& v, VisitorArgs&&...args) + { + return static_cast::type>&>(visitor).operator() (CurIndex, std::forward(v), std::forward(args)... ); + } + + //Visual Studio 2014 compilation fix: cast visitor to base class before invoke operator() + template + typename std::enable_if::type>, + typename std::decay::type>::value, ReturnType>::type + invoke_class_visitor(Visitor& visitor, Value&& v, VisitorArgs&&...args) + { + return static_cast::type>&>(visitor).operator() (CurIndex, std::forward(v), std::forward(args)... ); + } + + // Intermediate recursion processor for special case `visitor_interface` derived Visitors + template + typename std::enable_if::type>::value, ReturnType>::type + apply_visitor_impl(Visitor&& visitor, Variant&& v, std::false_type not_processed, + std::integral_constant should_no_return, + VisitorArgs&& ...args) + { + static_assert(std::is_same(v)))>::value, + "Different `ReturnType`s detected! All `Visitor::visit` or `overload_lamba_set`" + " must return the same type"); + suppress_unused_warning(not_processed); + if (v.index() == CurIndex) + { + return invoke_class_visitor(visitor, get(v), std::forward(args)... ); + } + + using is_variant_processed_t = std::integral_constant= ElemCount>; + return apply_visitor_impl( + std::forward(visitor), + std::forward(v), + is_variant_processed_t{}, + should_no_return, + std::forward(args)...); + } +} // namespace detail + + template + auto visit(Visitor &visitor, const Variant& var, VisitorArg &&...args) -> decltype(visitor(get<0>(var))) + { + constexpr std::size_t varsize = util::variant_size::value; + static_assert(varsize != 0, "utils::variant must contains one type at least "); + using is_variant_processed_t = std::false_type; + + using ReturnType = decltype(visitor(get<0>(var))); + using return_t = std::is_same; + return detail::apply_visitor_impl( + std::forward(visitor), + var, is_variant_processed_t{}, + return_t{}, + std::forward(args)...); + } + + template + auto visit(Visitor&& visitor, const Variant& var) -> decltype(visitor(get<0>(var))) + { + constexpr std::size_t varsize = util::variant_size::value; + static_assert(varsize != 0, "utils::variant must contains one type at least "); + using is_variant_processed_t = std::false_type; + + using ReturnType = decltype(visitor(get<0>(var))); + using return_t = std::is_same; + return detail::apply_visitor_impl( + std::forward(visitor), + var, is_variant_processed_t{}, + return_t{}); + } } // namespace util +} // namespace cv #endif // OPENCV_GAPI_UTIL_VARIANT_HPP diff --git a/modules/gapi/src/api/gproto.cpp b/modules/gapi/src/api/gproto.cpp index 94234c9b4d..9b012770ca 100644 --- a/modules/gapi/src/api/gproto.cpp +++ b/modules/gapi/src/api/gproto.cpp @@ -14,7 +14,6 @@ #include "api/gorigin.hpp" #include "api/gproto_priv.hpp" -#include "logger.hpp" // FIXME: it should be a visitor! // FIXME: Reimplement with traits? @@ -277,13 +276,9 @@ void cv::validate_input_arg(const GRunArg& arg) void cv::validate_input_args(const GRunArgs& args) { - GAPI_LOG_DEBUG(nullptr, "Total count: " << args.size()); - size_t index = 0; for (const auto& arg : args) { - GAPI_LOG_DEBUG(nullptr, "Process index: " << index); validate_input_arg(arg); - index ++; } } diff --git a/modules/gapi/test/util/variant_tests.cpp b/modules/gapi/test/util/variant_tests.cpp index 65d5e579f8..7725f9a702 100644 --- a/modules/gapi/test/util/variant_tests.cpp +++ b/modules/gapi/test/util/variant_tests.cpp @@ -354,6 +354,20 @@ TEST(Variant, Get) EXPECT_THROW(util::get(cv2), util::bad_variant_access); } +TEST(Variant, GetIndexed) +{ + const TestVar cv(42); + + // Test const& get() + EXPECT_EQ(42, util::get<0>(cv)); + EXPECT_THROW(util::get<1>(cv), util::bad_variant_access); + + // Test &get + TestVar cv2(std::string("42")); + EXPECT_EQ("42", util::get<1>(cv2)); + EXPECT_THROW(util::get<0>(cv2), util::bad_variant_access); +} + TEST(Variant, GetWrite) { util::variant v(42); @@ -486,4 +500,240 @@ TEST(Variant, EXT_IndexOf) static_assert(6u == V::index_of(), "Index is incorrect"); } +namespace test_validation +{ +struct MyType +{ + friend std::ostream& operator<<(std::ostream& out, const MyType& src) + { + return out << "MyType"; (void) src; + } +}; +class MyClass +{ + friend std::ostream& operator<<(std::ostream& out, const MyClass& src) + { + return out << "MyClass"; (void) src; + } +}; + +struct MyBoolParamIndexedVisitor : cv::util::static_indexed_visitor +{ + MyBoolParamIndexedVisitor(std::ostream &output) : out(output) {} + + template + bool visit(std::size_t index, Type val, int check) + { + bool result = false; + out << index << ":" << val <<","; + if(std::is_same::value) + { + result = !memcmp(&val, &check, sizeof(int)); + } + return result; + } + + std::ostream &out; +}; + +struct MyBoolNoParamNonIndexedVisitor : cv::util::static_indexed_visitor +{ + MyBoolNoParamNonIndexedVisitor(std::ostream &output) : out(output) {} + + template + bool visit(std::size_t index, Type val) + { + out << index << ":" << val <<","; + return true; + } + std::ostream &out; +}; + + +struct MyVoidNoParamNonIndexedVisitor : cv::util::static_visitor +{ + MyVoidNoParamNonIndexedVisitor(std::ostream &output) : out(output) {} + + template + void visit(Type val) + { + out << val << ","; + } + + std::ostream &out; +}; + + +struct MyVoidNoParamIndexedVisitor : cv::util::static_indexed_visitor +{ + MyVoidNoParamIndexedVisitor(std::ostream &output) : out(output) {} + + template + void visit(std::size_t Index, Type val) + { + out << Index << ":" << val <<","; + } + + std::ostream &out; +}; +} + +TEST(Variant, DynamicVisitor) +{ + using V = cv::util::variant; + V var{42}; + { + std::stringstream ss; + test_validation::MyBoolParamIndexedVisitor visitor(ss); + + EXPECT_TRUE(cv::util::visit(visitor, var, int{42})); + EXPECT_EQ(ss.str(), std::string("0:42,")); + } + + std::stringstream ss; + test_validation::MyBoolNoParamNonIndexedVisitor visitor(ss); + + cv::util::visit(visitor, var); + EXPECT_EQ(ss.str(), std::string("0:42,")); + + var = double{1.0}; + EXPECT_TRUE(cv::util::visit(visitor, var)); + EXPECT_EQ(ss.str(), std::string("0:42,1:1,")); + + var = char{'a'}; + EXPECT_TRUE(cv::util::visit(visitor, var)); + EXPECT_EQ(ss.str(), std::string("0:42,1:1,2:a,")); + + var = float{6.0}; + EXPECT_TRUE(cv::util::visit(visitor, var)); + EXPECT_EQ(ss.str(), std::string("0:42,1:1,2:a,3:6,")); + + var = test_validation::MyType{}; + EXPECT_TRUE(cv::util::visit(visitor, var)); + EXPECT_EQ(ss.str(), std::string("0:42,1:1,2:a,3:6,4:MyType,")); + + var = test_validation::MyClass{}; + EXPECT_TRUE(cv::util::visit(visitor, var)); + EXPECT_EQ(ss.str(), std::string("0:42,1:1,2:a,3:6,4:MyType,5:MyClass,")); +} + +TEST(Variant, StaticVisitor) +{ + using V = cv::util::variant; + V var{42}; + std::stringstream ss; + test_validation::MyVoidNoParamNonIndexedVisitor visitor(ss); + + cv::util::visit(visitor, var); + EXPECT_EQ(ss.str(), std::string("42,")); + + var = double{1.0}; + cv::util::visit(visitor, var); + EXPECT_EQ(ss.str(), std::string("42,1,")); + + var = char{'a'}; + cv::util::visit(visitor, var); + EXPECT_EQ(ss.str(), std::string("42,1,a,")); + + var = float{6.0}; + cv::util::visit(visitor, var); + EXPECT_EQ(ss.str(), std::string("42,1,a,6,")); + + var = test_validation::MyType{}; + cv::util::visit(visitor, var); + EXPECT_EQ(ss.str(), std::string("42,1,a,6,MyType,")); + + var = test_validation::MyClass{}; + cv::util::visit(visitor, var); + EXPECT_EQ(ss.str(), std::string("42,1,a,6,MyType,MyClass,")); +} + +TEST(Variant, StaticIndexedVisitor) +{ + using V = cv::util::variant; + V var{42}; + + std::stringstream ss; + cv::util::visit(test_validation::MyVoidNoParamIndexedVisitor {ss}, var); + EXPECT_EQ(ss.str(), std::string("0:42,")); + + var = double{1.0}; + cv::util::visit(test_validation::MyVoidNoParamIndexedVisitor (ss), var); + EXPECT_EQ(ss.str(), std::string("0:42,1:1,")); + + var = char{'a'}; + cv::util::visit(test_validation::MyVoidNoParamIndexedVisitor (ss), var); + EXPECT_EQ(ss.str(), std::string("0:42,1:1,2:a,")); + + var = float{6.0}; + cv::util::visit(test_validation::MyVoidNoParamIndexedVisitor (ss), var); + EXPECT_EQ(ss.str(), std::string("0:42,1:1,2:a,3:6,")); + + var = test_validation::MyType{}; + cv::util::visit(test_validation::MyVoidNoParamIndexedVisitor (ss), var); + EXPECT_EQ(ss.str(), std::string("0:42,1:1,2:a,3:6,4:MyType,")); + + var = test_validation::MyClass{}; + cv::util::visit(test_validation::MyVoidNoParamIndexedVisitor (ss), var); + EXPECT_EQ(ss.str(), std::string("0:42,1:1,2:a,3:6,4:MyType,5:MyClass,")); +} + + +TEST(Variant, LambdaVisitor) +{ + using V = cv::util::variant; + V var{42}; + { + cv::util::visit(cv::util::overload_lambdas( + [](int value) { + EXPECT_EQ(value, 42); + }, + [](double) { + ADD_FAILURE() << "can't be called for `double`"; + }, + [](char) { + ADD_FAILURE() << "can't be called for `char`"; + }, + [](float) { + ADD_FAILURE() << "can't be called for `float`"; + }, + [](test_validation::MyType) { + ADD_FAILURE() << "can't be called for `MyType`"; + }, + [](test_validation::MyClass) { + ADD_FAILURE() << "can't be called for `MyClass`"; + }, + [](std::string) { + ADD_FAILURE() << "can't be called for `std::string`, invalid type"; + } + ), var); + } + + var = 'c'; + { + cv::util::visit(cv::util::overload_lambdas( + [](int) { + ADD_FAILURE() << "can't be called for `int`"; + }, + [](double) { + ADD_FAILURE() << "can't be called for `double`"; + }, + [](char value) { + EXPECT_EQ(value, 'c'); + }, + [](float) { + ADD_FAILURE() << "can't be called for `float`"; + }, + [](test_validation::MyType) { + ADD_FAILURE() << "can't be called for `MyType`"; + }, + [](test_validation::MyClass) { + ADD_FAILURE() << "can't be called for `MyClass`"; + }, + [](std::string) { + ADD_FAILURE() << "can't be called for `std::string`, invalid type"; + } + ), var); + } +} } // namespace opencv_test From 926535469d82649204a9b2819f3f7341c140b7cb Mon Sep 17 00:00:00 2001 From: kikaxa Date: Wed, 7 Jul 2021 18:31:53 +0300 Subject: [PATCH 052/262] fix videoio/src/container_avi.cpp VideoInputStream alignment --- modules/videoio/src/container_avi.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/videoio/src/container_avi.cpp b/modules/videoio/src/container_avi.cpp index 8664d198df..c0c3234f6f 100644 --- a/modules/videoio/src/container_avi.cpp +++ b/modules/videoio/src/container_avi.cpp @@ -124,6 +124,7 @@ struct RiffList uint32_t m_size; uint32_t m_list_type_cc; }; +#pragma pack(pop) class VideoInputStream { @@ -149,7 +150,6 @@ private: String m_fname; }; -#pragma pack(pop) inline VideoInputStream& operator >> (VideoInputStream& is, AviMainHeader& avih) { From 59ae0e0013d85bf4a6d064e7c258c00e56858b35 Mon Sep 17 00:00:00 2001 From: Alexey Smirnov Date: Wed, 7 Jul 2021 22:07:59 +0300 Subject: [PATCH 053/262] Merge pull request #20163 from smirnov-alexey:as/gapi_serialization_docs G-API: add documentation on serialization functionality * Add documentation on serialization/deserialization * Add docs on bind() methods * Fix typo * Docs refactoring * Fix s11n docs * Fix deserialize() docs * Change deserialize docs * Fix warning * Address review comments * Fix sample * Fix warnings and errors * Fix docs warnings * Fix warnings * Address review comments * Add prefixes to snippets and fix indentation * Address review comments and move snippets to a single file --- modules/gapi/include/opencv2/gapi.hpp | 3 +- modules/gapi/include/opencv2/gapi/garg.hpp | 41 +++- modules/gapi/include/opencv2/gapi/gproto.hpp | 2 +- modules/gapi/include/opencv2/gapi/s11n.hpp | 183 ++++++++++++++---- .../gapi/include/opencv2/gapi/s11n/base.hpp | 35 +++- modules/gapi/samples/api_ref_snippets.cpp | 121 ++++++++++++ modules/gapi/src/api/s11n.cpp | 6 +- modules/gapi/test/s11n/gapi_s11n_tests.cpp | 2 - 8 files changed, 338 insertions(+), 55 deletions(-) diff --git a/modules/gapi/include/opencv2/gapi.hpp b/modules/gapi/include/opencv2/gapi.hpp index e4b2021479..f10dfd471d 100644 --- a/modules/gapi/include/opencv2/gapi.hpp +++ b/modules/gapi/include/opencv2/gapi.hpp @@ -2,7 +2,7 @@ // It is subject to the license terms in the LICENSE file found in the top-level directory // of this distribution and at http://opencv.org/license.html. // -// Copyright (C) 2018 Intel Corporation +// Copyright (C) 2018-2021 Intel Corporation #ifndef OPENCV_GAPI_HPP @@ -19,6 +19,7 @@ @} @defgroup gapi_std_backends G-API Standard Backends @defgroup gapi_compile_args G-API Graph Compilation Arguments + @defgroup gapi_serialization G-API Serialization functionality @} */ diff --git a/modules/gapi/include/opencv2/gapi/garg.hpp b/modules/gapi/include/opencv2/gapi/garg.hpp index 20f2233bf9..ee6ee81e1c 100644 --- a/modules/gapi/include/opencv2/gapi/garg.hpp +++ b/modules/gapi/include/opencv2/gapi/garg.hpp @@ -2,7 +2,7 @@ // It is subject to the license terms in the LICENSE file found in the top-level directory // of this distribution and at http://opencv.org/license.html. // -// Copyright (C) 2018-2020 Intel Corporation +// Copyright (C) 2018-2021 Intel Corporation #ifndef OPENCV_GAPI_GARG_HPP @@ -171,7 +171,7 @@ using GRunArgs = std::vector; * It's an ordinary overload of addition assignment operator. * * Example of usage: - * @snippet dynamic_graph.cpp GRunArgs usage + * @snippet modules/gapi/samples/dynamic_graph.cpp GRunArgs usage * */ inline GRunArgs& operator += (GRunArgs &lhs, const GRunArgs &rhs) @@ -223,7 +223,7 @@ using GRunArgsP = std::vector; * It's an ordinary overload of addition assignment operator. * * Example of usage: - * @snippet dynamic_graph.cpp GRunArgsP usage + * @snippet modules/gapi/samples/dynamic_graph.cpp GRunArgsP usage * */ inline GRunArgsP& operator += (GRunArgsP &lhs, const GRunArgsP &rhs) @@ -235,8 +235,39 @@ inline GRunArgsP& operator += (GRunArgsP &lhs, const GRunArgsP &rhs) namespace gapi { - GAPI_EXPORTS cv::GRunArgsP bind(cv::GRunArgs &results); - GAPI_EXPORTS cv::GRunArg bind(cv::GRunArgP &out); // FIXME: think more about it +/** + * \addtogroup gapi_serialization + * @{ + * + * @brief G-API functions and classes for serialization and deserialization. + */ +/** @brief Wraps deserialized output GRunArgs to GRunArgsP which can be used by GCompiled. + * + * Since it's impossible to get modifiable output arguments from deserialization + * it needs to be wrapped by this function. + * + * Example of usage: + * @snippet modules/gapi/samples/api_ref_snippets.cpp bind after deserialization + * + * @param out_args deserialized GRunArgs. + * @return the same GRunArgs wrapped in GRunArgsP. + * @see deserialize + */ +GAPI_EXPORTS cv::GRunArgsP bind(cv::GRunArgs &out_args); +/** @brief Wraps output GRunArgsP available during graph execution to GRunArgs which can be serialized. + * + * GRunArgsP is pointer-to-value, so to be serialized they need to be binded to real values + * which this function does. + * + * Example of usage: + * @snippet modules/gapi/samples/api_ref_snippets.cpp bind before serialization + * + * @param out output GRunArgsP available during graph execution. + * @return the same GRunArgsP wrapped in serializable GRunArgs. + * @see serialize + */ +GAPI_EXPORTS cv::GRunArg bind(cv::GRunArgP &out); // FIXME: think more about it +/** @} */ } template inline GRunArgs gin(const Ts&... args) diff --git a/modules/gapi/include/opencv2/gapi/gproto.hpp b/modules/gapi/include/opencv2/gapi/gproto.hpp index fbcccb38ea..6271e470b0 100644 --- a/modules/gapi/include/opencv2/gapi/gproto.hpp +++ b/modules/gapi/include/opencv2/gapi/gproto.hpp @@ -71,7 +71,7 @@ public: * It's an ordinary overload of addition assignment operator. * * Example of usage: - * @snippet dynamic_graph.cpp GIOProtoArgs usage + * @snippet modules/gapi/samples/dynamic_graph.cpp GIOProtoArgs usage * */ template diff --git a/modules/gapi/include/opencv2/gapi/s11n.hpp b/modules/gapi/include/opencv2/gapi/s11n.hpp index 5a64410e5a..ca8e32c98b 100644 --- a/modules/gapi/include/opencv2/gapi/s11n.hpp +++ b/modules/gapi/include/opencv2/gapi/s11n.hpp @@ -17,65 +17,135 @@ namespace cv { namespace gapi { +/** +* \addtogroup gapi_serialization +* @{ +*/ + namespace detail { - GAPI_EXPORTS cv::GComputation getGraph(const std::vector &p); + GAPI_EXPORTS cv::GComputation getGraph(const std::vector &bytes); - GAPI_EXPORTS cv::GMetaArgs getMetaArgs(const std::vector &p); + GAPI_EXPORTS cv::GMetaArgs getMetaArgs(const std::vector &bytes); - GAPI_EXPORTS cv::GRunArgs getRunArgs(const std::vector &p); + GAPI_EXPORTS cv::GRunArgs getRunArgs(const std::vector &bytes); - GAPI_EXPORTS std::vector getVectorOfStrings(const std::vector &p); + GAPI_EXPORTS std::vector getVectorOfStrings(const std::vector &bytes); template - cv::GCompileArgs getCompileArgs(const std::vector &p); + cv::GCompileArgs getCompileArgs(const std::vector &bytes); template - cv::GRunArgs getRunArgsWithRMats(const std::vector &p); + cv::GRunArgs getRunArgsWithRMats(const std::vector &bytes); } // namespace detail +/** @brief Serialize a graph represented by GComputation into an array of bytes. + * + * Check different overloads for more examples. + * @param c GComputation to serialize. + * @return serialized vector of bytes. + */ GAPI_EXPORTS std::vector serialize(const cv::GComputation &c); -//namespace{ +/** @overload + * @param ca GCompileArgs to serialize. + */ +GAPI_EXPORTS std::vector serialize(const cv::GCompileArgs& ca); + +/** @overload + * @param ma GMetaArgs to serialize. + */ +GAPI_EXPORTS std::vector serialize(const cv::GMetaArgs& ma); + +/** @overload + * @param ra GRunArgs to serialize. + */ +GAPI_EXPORTS std::vector serialize(const cv::GRunArgs& ra); + +/** @overload + * @param vs std::vector to serialize. + */ +GAPI_EXPORTS std::vector serialize(const std::vector& vs); + +/** + * @private + */ template static inline -T deserialize(const std::vector &p); - -//} //ananymous namespace - -GAPI_EXPORTS std::vector serialize(const cv::GCompileArgs&); -GAPI_EXPORTS std::vector serialize(const cv::GMetaArgs&); -GAPI_EXPORTS std::vector serialize(const cv::GRunArgs&); -GAPI_EXPORTS std::vector serialize(const std::vector&); +T deserialize(const std::vector &bytes); +/** @brief Deserialize GComputation from a byte array. + * + * Check different overloads for more examples. + * @param bytes serialized vector of bytes. + * @return deserialized GComputation object. + */ template<> inline -cv::GComputation deserialize(const std::vector &p) { - return detail::getGraph(p); +cv::GComputation deserialize(const std::vector &bytes) { + return detail::getGraph(bytes); } +/** @brief Deserialize GMetaArgs from a byte array. + * + * Check different overloads for more examples. + * @param bytes serialized vector of bytes. + * @return deserialized GMetaArgs object. + */ template<> inline -cv::GMetaArgs deserialize(const std::vector &p) { - return detail::getMetaArgs(p); +cv::GMetaArgs deserialize(const std::vector &bytes) { + return detail::getMetaArgs(bytes); } +/** @brief Deserialize GRunArgs from a byte array. + * + * Check different overloads for more examples. + * @param bytes serialized vector of bytes. + * @return deserialized GRunArgs object. + */ template<> inline -cv::GRunArgs deserialize(const std::vector &p) { - return detail::getRunArgs(p); +cv::GRunArgs deserialize(const std::vector &bytes) { + return detail::getRunArgs(bytes); } +/** @brief Deserialize std::vector from a byte array. + * + * Check different overloads for more examples. + * @param bytes serialized vector of bytes. + * @return deserialized std::vector object. + */ template<> inline -std::vector deserialize(const std::vector &p) { - return detail::getVectorOfStrings(p); +std::vector deserialize(const std::vector &bytes) { + return detail::getVectorOfStrings(bytes); } +/** + * @brief Deserialize GCompileArgs which types were specified in the template from a byte array. + * + * @note cv::gapi::s11n::detail::S11N template specialization must be provided to make a custom type + * in GCompileArgs deserializable. + * + * @param bytes vector of bytes to deserialize GCompileArgs object from. + * @return GCompileArgs object. + * @see GCompileArgs cv::gapi::s11n::detail::S11N + */ template inline typename std::enable_if::value, GCompileArgs>:: -type deserialize(const std::vector &p) { - return detail::getCompileArgs(p); +type deserialize(const std::vector &bytes) { + return detail::getCompileArgs(bytes); } +/** + * @brief Deserialize GRunArgs including RMat objects if any from a byte array. + * + * RMat adapter type is specified in the template. + * @note To be used properly specified adapter type must overload its serialize() and + * deserialize() methods. + * @param bytes vector of bytes to deserialize GRunArgs object from. + * @return GRunArgs including RMat objects if any. + * @see RMat + */ template inline typename std::enable_if::value, GRunArgs>:: -type deserialize(const std::vector &p) { - return detail::getRunArgsWithRMats(p); +type deserialize(const std::vector &bytes) { + return detail::getRunArgsWithRMats(bytes); } } // namespace gapi } // namespace cv @@ -83,6 +153,17 @@ type deserialize(const std::vector &p) { namespace cv { namespace gapi { namespace s11n { + +/** @brief This structure is an interface for serialization routines. + * + * It's main purpose is to provide multiple overloads for operator<<() + * with basic C++ in addition to OpenCV/G-API types. + * + * This sctructure can be inherited and further extended with additional types. + * + * For example, it is utilized in cv::gapi::s11n::detail::S11N as input parameter + * in serialize() method. + */ struct GAPI_EXPORTS IOStream { virtual ~IOStream() = default; // Define the native support for basic C++ types at the API level: @@ -99,6 +180,16 @@ struct GAPI_EXPORTS IOStream { virtual IOStream& operator<< (const std::string&) = 0; }; +/** @brief This structure is an interface for deserialization routines. + * + * It's main purpose is to provide multiple overloads for operator>>() + * with basic C++ in addition to OpenCV/G-API types. + * + * This structure can be inherited and further extended with additional types. + * + * For example, it is utilized in cv::gapi::s11n::detail::S11N as input parameter + * in deserialize() method. + */ struct GAPI_EXPORTS IIStream { virtual ~IIStream() = default; virtual IIStream& operator>> (bool &) = 0; @@ -116,7 +207,7 @@ struct GAPI_EXPORTS IIStream { }; namespace detail { -GAPI_EXPORTS std::unique_ptr getInStream(const std::vector &p); +GAPI_EXPORTS std::unique_ptr getInStream(const std::vector &bytes); } // namespace detail //////////////////////////////////////////////////////////////////////////////// @@ -146,24 +237,26 @@ GAPI_EXPORTS IIStream& operator>> (IIStream& is, cv::Mat &m); // FIXME: for GRunArgs serailization #if !defined(GAPI_STANDALONE) -GAPI_EXPORTS IOStream& operator<< (IOStream& os, const cv::UMat &); -GAPI_EXPORTS IIStream& operator>> (IIStream& is, cv::UMat &); +GAPI_EXPORTS IOStream& operator<< (IOStream& os, const cv::UMat & um); +GAPI_EXPORTS IIStream& operator>> (IIStream& is, cv::UMat & um); #endif // !defined(GAPI_STANDALONE) GAPI_EXPORTS IOStream& operator<< (IOStream& os, const cv::RMat &r); GAPI_EXPORTS IIStream& operator>> (IIStream& is, cv::RMat &r); -GAPI_EXPORTS IOStream& operator<< (IOStream& os, const cv::gapi::wip::IStreamSource::Ptr &); -GAPI_EXPORTS IIStream& operator>> (IIStream& is, cv::gapi::wip::IStreamSource::Ptr &); +GAPI_EXPORTS IOStream& operator<< (IOStream& os, const cv::gapi::wip::IStreamSource::Ptr &issptr); +GAPI_EXPORTS IIStream& operator>> (IIStream& is, cv::gapi::wip::IStreamSource::Ptr &issptr); -GAPI_EXPORTS IOStream& operator<< (IOStream& os, const cv::detail::VectorRef &); -GAPI_EXPORTS IIStream& operator>> (IIStream& is, cv::detail::VectorRef &); +GAPI_EXPORTS IOStream& operator<< (IOStream& os, const cv::detail::VectorRef &vr); +GAPI_EXPORTS IIStream& operator>> (IIStream& is, cv::detail::VectorRef &vr); -GAPI_EXPORTS IOStream& operator<< (IOStream& os, const cv::detail::OpaqueRef &); -GAPI_EXPORTS IIStream& operator>> (IIStream& is, cv::detail::OpaqueRef &); +GAPI_EXPORTS IOStream& operator<< (IOStream& os, const cv::detail::OpaqueRef &opr); +GAPI_EXPORTS IIStream& operator>> (IIStream& is, cv::detail::OpaqueRef &opr); -GAPI_EXPORTS IOStream& operator<< (IOStream& os, const cv::MediaFrame &); -GAPI_EXPORTS IIStream& operator>> (IIStream& is, cv::MediaFrame &); +/// @private -- Exclude this function from OpenCV documentation +GAPI_EXPORTS IOStream& operator<< (IOStream& os, const cv::MediaFrame &mf); +/// @private -- Exclude this function from OpenCV documentation +GAPI_EXPORTS IIStream& operator>> (IIStream& is, cv::MediaFrame &mf); // Generic STL types //////////////////////////////////////////////////////////////// template @@ -186,6 +279,7 @@ IIStream& operator>> (IIStream& is, std::map &m) { } return is; } + template IOStream& operator<< (IOStream& os, const std::unordered_map &m) { const uint32_t sz = static_cast(m.size()); @@ -206,6 +300,7 @@ IIStream& operator>> (IIStream& is, std::unordered_map &m) { } return is; } + template IOStream& operator<< (IOStream& os, const std::vector &ts) { const uint32_t sz = static_cast(ts.size()); @@ -233,16 +328,19 @@ template IOStream& put_v(IOStream&, const V&, std::size_t) { GAPI_Assert(false && "variant>>: requested index is invalid"); }; + template IOStream& put_v(IOStream& os, const V& v, std::size_t x) { return (x == 0u) ? os << cv::util::get(v) : put_v(os, v, x-1); } + template IIStream& get_v(IIStream&, V&, std::size_t, std::size_t) { GAPI_Assert(false && "variant<<: requested index is invalid"); } + template IIStream& get_v(IIStream& is, V& v, std::size_t i, std::size_t gi) { if (i == gi) { @@ -254,11 +352,13 @@ IIStream& get_v(IIStream& is, V& v, std::size_t i, std::size_t gi) { } } // namespace detail +//! @overload template IOStream& operator<< (IOStream& os, const cv::util::variant &v) { os << static_cast(v.index()); return detail::put_v, Ts...>(os, v, v.index()); } +//! @overload template IIStream& operator>> (IIStream& is, cv::util::variant &v) { int idx = -1; @@ -268,6 +368,7 @@ IIStream& operator>> (IIStream& is, cv::util::variant &v) { } // FIXME: consider a better solution +/// @private -- Exclude this function from OpenCV documentation template void getRunArgByIdx (IIStream& is, cv::util::variant &v, uint32_t idx) { is = detail::get_v, Ts...>(is, v, 0u, idx); @@ -351,8 +452,8 @@ cv::GCompileArgs getCompileArgs(const std::vector &sArgs) { } template -cv::GRunArgs getRunArgsWithRMats(const std::vector &p) { - std::unique_ptr pIs = cv::gapi::s11n::detail::getInStream(p); +cv::GRunArgs getRunArgsWithRMats(const std::vector &bytes) { + std::unique_ptr pIs = cv::gapi::s11n::detail::getInStream(bytes); cv::gapi::s11n::IIStream& is = *pIs; cv::GRunArgs args; @@ -367,6 +468,8 @@ cv::GRunArgs getRunArgsWithRMats(const std::vector &p) { return args; } } // namespace detail +/** @} */ + } // namespace gapi } // namespace cv diff --git a/modules/gapi/include/opencv2/gapi/s11n/base.hpp b/modules/gapi/include/opencv2/gapi/s11n/base.hpp index b8ec8cfaff..11440b27e5 100644 --- a/modules/gapi/include/opencv2/gapi/s11n/base.hpp +++ b/modules/gapi/include/opencv2/gapi/s11n/base.hpp @@ -2,7 +2,7 @@ // It is subject to the license terms in the LICENSE file found in the top-level directory // of this distribution and at http://opencv.org/license.html. // -// Copyright (C) 2020 Intel Corporation +// Copyright (C) 2020-2021 Intel Corporation #ifndef OPENCV_GAPI_S11N_BASE_HPP #define OPENCV_GAPI_S11N_BASE_HPP @@ -23,25 +23,54 @@ struct IIStream; namespace detail { +//! @addtogroup gapi_serialization +//! @{ + struct NotImplemented { }; -// The default S11N for custom types is NotImplemented -// Don't! sublass from NotImplemented if you actually implement S11N. +/** @brief This structure allows to implement serialization routines for custom types. + * + * The default S11N for custom types is not implemented. + * + * @note When providing an overloaded implementation for S11N with your type + * don't inherit it from NotImplemented structure. + * + * @note There are lots of overloaded >> and << operators for basic and OpenCV/G-API types + * which can be utilized when serializing a custom type. + * + * Example of usage: + * @snippet modules/gapi/samples/api_ref_snippets.cpp S11N usage + * + */ template struct S11N: public NotImplemented { + /** + * @brief This function allows user to serialize their custom type. + * + * @note The default overload throws an exception if called. User need to + * properly overload the function to use it. + */ static void serialize(IOStream &, const T &) { GAPI_Assert(false && "No serialization routine is provided!"); } + /** + * @brief This function allows user to deserialize their custom type. + * + * @note The default overload throws an exception if called. User need to + * properly overload the function to use it. + */ static T deserialize(IIStream &) { GAPI_Assert(false && "No deserialization routine is provided!"); } }; +/// @private -- Exclude this struct from OpenCV documentation template struct has_S11N_spec { static constexpr bool value = !std::is_base_of::type>>::value; }; +//! @} gapi_serialization } // namespace detail } // namespace s11n diff --git a/modules/gapi/samples/api_ref_snippets.cpp b/modules/gapi/samples/api_ref_snippets.cpp index 6c660fb8fa..0abcab89b3 100644 --- a/modules/gapi/samples/api_ref_snippets.cpp +++ b/modules/gapi/samples/api_ref_snippets.cpp @@ -4,6 +4,10 @@ #include #include +#include +#include +#include + #include #include @@ -55,6 +59,120 @@ static void typed_example() //! [Typed_Example] } +static void bind_serialization_example() +{ + // ! [bind after deserialization] + cv::GCompiled compd; + std::vector bytes; + auto graph = cv::gapi::deserialize(bytes); + auto meta = cv::gapi::deserialize(bytes); + + compd = graph.compile(std::move(meta), cv::compile_args()); + auto in_args = cv::gapi::deserialize(bytes); + auto out_args = cv::gapi::deserialize(bytes); + compd(std::move(in_args), cv::gapi::bind(out_args)); + // ! [bind after deserialization] +} + +static void bind_deserialization_example() +{ + // ! [bind before serialization] + std::vector graph_outs; + cv::GRunArgs out_args; + + for (auto &&out : graph_outs) { + out_args.emplace_back(cv::gapi::bind(out)); + } + const auto sargsout = cv::gapi::serialize(out_args); + // ! [bind before serialization] +} + +struct SimpleCustomType { + bool val; + bool operator==(const SimpleCustomType& other) const { + return val == other.val; + } +}; + +struct SimpleCustomType2 { + int val; + std::string name; + std::vector vec; + std::map mmap; + bool operator==(const SimpleCustomType2& other) const { + return val == other.val && name == other.name && + vec == other.vec && mmap == other.mmap; + } +}; + +// ! [S11N usage] +namespace cv { +namespace gapi { +namespace s11n { +namespace detail { +template<> struct S11N { + static void serialize(IOStream &os, const SimpleCustomType &p) { + os << p.val; + } + static SimpleCustomType deserialize(IIStream &is) { + SimpleCustomType p; + is >> p.val; + return p; + } +}; + +template<> struct S11N { + static void serialize(IOStream &os, const SimpleCustomType2 &p) { + os << p.val << p.name << p.vec << p.mmap; + } + static SimpleCustomType2 deserialize(IIStream &is) { + SimpleCustomType2 p; + is >> p.val >> p.name >> p.vec >> p.mmap; + return p; + } +}; +} // namespace detail +} // namespace s11n +} // namespace gapi +} // namespace cv +// ! [S11N usage] + +namespace cv { +namespace detail { +template<> struct CompileArgTag { + static const char* tag() { + return "org.opencv.test.simple_custom_type"; + } +}; + +template<> struct CompileArgTag { + static const char* tag() { + return "org.opencv.test.simple_custom_type_2"; + } +}; +} // namespace detail +} // namespace cv + +static void s11n_example() +{ + SimpleCustomType customVar1 { false }; + SimpleCustomType2 customVar2 { 1248, "World", {1280, 720, 640, 480}, + { {5, 32434142342}, {7, 34242432} } }; + + std::vector sArgs = cv::gapi::serialize( + cv::compile_args(customVar1, customVar2)); + + cv::GCompileArgs dArgs = cv::gapi::deserialize(sArgs); + + SimpleCustomType dCustomVar1 = cv::gapi::getCompileArg(dArgs).value(); + SimpleCustomType2 dCustomVar2 = cv::gapi::getCompileArg(dArgs).value(); + + (void) dCustomVar1; + (void) dCustomVar2; +} + G_TYPED_KERNEL(IAdd, , "test.custom.add") { static cv::GMatDesc outMeta(const cv::GMatDesc &in) { return in; } }; @@ -128,5 +246,8 @@ int main(int argc, char *argv[]) // unused functions typed_example(); gscalar_example(); + bind_serialization_example(); + bind_deserialization_example(); + s11n_example(); return 0; } diff --git a/modules/gapi/src/api/s11n.cpp b/modules/gapi/src/api/s11n.cpp index d08f47fd26..97f5a95c42 100644 --- a/modules/gapi/src/api/s11n.cpp +++ b/modules/gapi/src/api/s11n.cpp @@ -65,11 +65,11 @@ std::vector cv::gapi::serialize(const std::vector& vs) // FIXME: This function should move from S11N to GRunArg-related entities. // it has nothing to do with the S11N as it is -cv::GRunArgsP cv::gapi::bind(cv::GRunArgs &results) +cv::GRunArgsP cv::gapi::bind(cv::GRunArgs &out_args) { cv::GRunArgsP outputs; - outputs.reserve(results.size()); - for (cv::GRunArg &res_obj : results) + outputs.reserve(out_args.size()); + for (cv::GRunArg &res_obj : out_args) { using T = cv::GRunArg; switch (res_obj.index()) diff --git a/modules/gapi/test/s11n/gapi_s11n_tests.cpp b/modules/gapi/test/s11n/gapi_s11n_tests.cpp index f4a30b3946..c2b17521d9 100644 --- a/modules/gapi/test/s11n/gapi_s11n_tests.cpp +++ b/modules/gapi/test/s11n/gapi_s11n_tests.cpp @@ -754,8 +754,6 @@ TEST_F(S11N_Basic, Test_Deserialize_CompileArgs_RandomOrder) { std::vector sArgs = cv::gapi::serialize( cv::compile_args(simpleCustomVar, simpleCustomVar2)); GCompileArgs dArgs = cv::gapi::deserialize(sArgs); From b42623ff9d6b1baf9fda11b16a8d2f512720d424 Mon Sep 17 00:00:00 2001 From: Smirnov Egor Date: Thu, 8 Jul 2021 10:42:44 +0300 Subject: [PATCH 054/262] port base64 encoding from 3.4 --- modules/core/src/persistence.cpp | 2659 +++++++++-------- modules/core/src/persistence.hpp | 18 + .../core/src/persistence_base64_encoding.cpp | 370 +++ .../core/src/persistence_base64_encoding.hpp | 127 + modules/core/src/persistence_impl.hpp | 231 ++ modules/core/src/persistence_json.cpp | 69 +- modules/core/src/persistence_xml.cpp | 30 +- modules/core/src/persistence_yml.cpp | 28 +- modules/core/test/test_io.cpp | 29 +- modules/python/test/test_filestorage_io.py | 93 + 10 files changed, 2289 insertions(+), 1365 deletions(-) create mode 100644 modules/core/src/persistence_base64_encoding.cpp create mode 100644 modules/core/src/persistence_base64_encoding.hpp create mode 100644 modules/core/src/persistence_impl.hpp diff --git a/modules/core/src/persistence.cpp b/modules/core/src/persistence.cpp index 32328361e8..291931b5ae 100644 --- a/modules/core/src/persistence.cpp +++ b/modules/core/src/persistence.cpp @@ -4,6 +4,8 @@ #include "precomp.hpp" #include "persistence.hpp" +#include "persistence_impl.hpp" +#include "persistence_base64_encoding.hpp" #include #include @@ -153,7 +155,7 @@ static int symbolToType(char c) return CV_SEQ_ELTYPE_PTR; const char* pos = strchr( symbols, c ); if( !pos ) - CV_Error( CV_StsBadArg, "Invalid data type specification" ); + CV_Error( cv::Error::StsBadArg, "Invalid data type specification" ); return static_cast(pos - symbols); } @@ -192,7 +194,7 @@ int decodeFormat( const char* dt, int* fmt_pairs, int max_len ) } if( count <= 0 ) - CV_Error( CV_StsBadArg, "Invalid data type specification" ); + CV_Error( cv::Error::StsBadArg, "Invalid data type specification" ); fmt_pairs[i] = count; } @@ -208,7 +210,7 @@ int decodeFormat( const char* dt, int* fmt_pairs, int max_len ) { i += 2; if( i >= max_len ) - CV_Error( CV_StsBadArg, "Too long data type specification" ); + CV_Error( cv::Error::StsBadArg, "Too long data type specification" ); } fmt_pairs[i] = 0; } @@ -275,7 +277,7 @@ int decodeSimpleFormat( const char* dt ) fmt_pair_count = decodeFormat( dt, fmt_pairs, CV_FS_MAX_FMT_PAIRS ); if( fmt_pair_count != 1 || fmt_pairs[0] >= CV_CN_MAX) - CV_Error( CV_StsError, "Too complex format for the matrix" ); + CV_Error( cv::Error::StsError, "Too complex format for the matrix" ); elem_type = CV_MAKETYPE( fmt_pairs[1], fmt_pairs[0] ); @@ -345,1450 +347,1483 @@ static inline void writeReal(uchar* p, double fval) #endif } -class FileStorage::Impl : public FileStorage_API -{ -public: - void init() - { - flags = 0; - buffer.clear(); - bufofs = 0; - state = UNDEFINED; - is_opened = false; - dummy_eof = false; - write_mode = false; - mem_mode = false; - space = 0; - wrap_margin = 71; - fmt = 0; - file = 0; - gzfile = 0; - empty_stream = true; - strbufv.clear(); - strbuf = 0; - strbufsize = strbufpos = 0; - roots.clear(); - fs_data.clear(); - fs_data_ptrs.clear(); - fs_data_blksz.clear(); - freeSpaceOfs = 0; - - str_hash.clear(); - str_hash_data.clear(); - str_hash_data.resize(1); - str_hash_data[0] = '\0'; - - filename.clear(); - lineno = 0; - } - - Impl(FileStorage* _fs) - { - fs_ext = _fs; - init(); - } - - virtual ~Impl() - { - release(); - } - - void release(String* out=0) - { - if( is_opened ) - { - if(out) - out->clear(); - if( write_mode ) - { - while( write_stack.size() > 1 ) - { - endWriteStruct(); - } - flush(); - if( fmt == FileStorage::FORMAT_XML ) - puts( "\n" ); - else if ( fmt == FileStorage::FORMAT_JSON ) - puts( "}\n" ); - } - if( mem_mode && out ) - { - *out = cv::String(outbuf.begin(), outbuf.end()); - } - } - closeFile(); - init(); - } - - void analyze_file_name( const std::string& file_name, std::vector& params ) - { - params.clear(); - static const char not_file_name = '\n'; - static const char parameter_begin = '?'; - static const char parameter_separator = '&'; - - if( file_name.find(not_file_name, (size_t)0) != std::string::npos ) - return; - - size_t beg = file_name.find_last_of(parameter_begin); - params.push_back(file_name.substr((size_t)0, beg)); - - if( beg != std::string::npos ) - { - size_t end = file_name.size(); - beg++; - for( size_t param_beg = beg, param_end = beg; - param_end < end; - param_beg = param_end + 1 ) - { - param_end = file_name.find_first_of( parameter_separator, param_beg ); - if( (param_end == std::string::npos || param_end != param_beg) && param_beg + 1 < end ) - { - params.push_back( file_name.substr( param_beg, param_end - param_beg ) ); - } - } - } - } - - bool open( const char* filename_or_buf, int _flags, const char* encoding ) - { - _flags &= ~FileStorage::BASE64; - - bool ok = true; - release(); - - bool append = (_flags & 3) == FileStorage::APPEND; - mem_mode = (_flags & FileStorage::MEMORY) != 0; - - write_mode = (_flags & 3) != 0; - - bool isGZ = false; - size_t fnamelen = 0; - - std::vector params; - //if ( !mem_mode ) - { - analyze_file_name( filename_or_buf, params ); - if( !params.empty() ) - filename = params[0]; - - /*if( !write_base64 && params.size() >= 2 && - std::find(params.begin()+1, params.end(), std::string("base64")) != params.end()) - write_base64 = (write_mode || append);*/ - } - - if( filename.size() == 0 && !mem_mode && !write_mode ) - CV_Error( CV_StsNullPtr, "NULL or empty filename" ); - - if( mem_mode && append ) - CV_Error( CV_StsBadFlag, "FileStorage::APPEND and FileStorage::MEMORY are not currently compatible" ); - - flags = _flags; - - if( !mem_mode ) - { - char* dot_pos = strrchr((char*)filename.c_str(), '.'); - char compression = '\0'; - - if( dot_pos && dot_pos[1] == 'g' && dot_pos[2] == 'z' && - (dot_pos[3] == '\0' || (cv_isdigit(dot_pos[3]) && dot_pos[4] == '\0')) ) - { - if( append ) - { - CV_Error(CV_StsNotImplemented, "Appending data to compressed file is not implemented" ); - } - isGZ = true; - compression = dot_pos[3]; - if( compression ) - dot_pos[3] = '\0', fnamelen--; - } - - if( !isGZ ) - { - file = fopen(filename.c_str(), !write_mode ? "rt" : !append ? "wt" : "a+t" ); - if( !file ) - return false; - } - else - { -#if USE_ZLIB - char mode[] = { write_mode ? 'w' : 'r', 'b', compression ? compression : '3', '\0' }; - gzfile = gzopen(filename.c_str(), mode); - if( !gzfile ) - return false; -#else - CV_Error(CV_StsNotImplemented, "There is no compressed file storage support in this configuration"); -#endif - } - } - - roots.clear(); - fs_data.clear(); - wrap_margin = 71; - fmt = FileStorage::FORMAT_AUTO; - - if( write_mode ) - { - fmt = flags & FileStorage::FORMAT_MASK; - - if( mem_mode ) - outbuf.clear(); - - if( fmt == FileStorage::FORMAT_AUTO && !filename.empty() ) - { - const char* dot_pos = NULL; - const char* dot_pos2 = NULL; - // like strrchr() implementation, but save two last positions simultaneously - for (const char* pos = &filename[0]; pos[0] != 0; pos++) - { - if( pos[0] == '.' ) - { - dot_pos2 = dot_pos; - dot_pos = pos; - } - } - if (fs::strcasecmp(dot_pos, ".gz") == 0 && dot_pos2 != NULL) - { - dot_pos = dot_pos2; - } - fmt = (fs::strcasecmp(dot_pos, ".xml") == 0 || fs::strcasecmp(dot_pos, ".xml.gz") == 0 ) - ? FileStorage::FORMAT_XML - : (fs::strcasecmp(dot_pos, ".json") == 0 || fs::strcasecmp(dot_pos, ".json.gz") == 0) - ? FileStorage::FORMAT_JSON - : FileStorage::FORMAT_YAML; - } - else if( fmt == FileStorage::FORMAT_AUTO ) - { - fmt = FileStorage::FORMAT_XML; - } - - // we use factor=6 for XML (the longest characters (' and ") are encoded with 6 bytes (' and ") - // and factor=4 for YAML ( as we use 4 bytes for non ASCII characters (e.g. \xAB)) - int buf_size = CV_FS_MAX_LEN*(fmt == FileStorage::FORMAT_XML ? 6 : 4) + 1024; - - if (append) - { - fseek( file, 0, SEEK_END ); - if (ftell(file) == 0) - append = false; - } - - write_stack.clear(); - empty_stream = true; - write_stack.push_back(FStructData("", FileNode::MAP | FileNode::EMPTY, 0)); - buffer.reserve(buf_size + 1024); - buffer.resize(buf_size); - bufofs = 0; - - if( fmt == FileStorage::FORMAT_XML ) - { - size_t file_size = file ? (size_t)ftell(file) : (size_t)0; - if( !append || file_size == 0 ) - { - if( encoding && *encoding != '\0' ) - { - if( fs::strcasecmp(encoding, "UTF-16" ) == 0 ) - { - release(); - CV_Error( CV_StsBadArg, "UTF-16 XML encoding is not supported! Use 8-bit encoding\n"); - } - - CV_Assert( strlen(encoding) < 1000 ); - char buf[1100]; - sprintf(buf, "\n", encoding); - puts( buf ); - } - else - puts( "\n" ); - puts( "\n" ); - } - else - { - int xml_buf_size = 1 << 10; - char substr[] = ""; - int last_occurrence = -1; - xml_buf_size = MIN(xml_buf_size, int(file_size)); - fseek( file, -xml_buf_size, SEEK_END ); - // find the last occurrence of - for(;;) - { - int line_offset = (int)ftell( file ); - const char* ptr0 = this->gets(xml_buf_size); - const char* ptr = NULL; - if( !ptr0 ) - break; - ptr = ptr0; - for(;;) - { - ptr = strstr( ptr, substr ); - if( !ptr ) - break; - last_occurrence = line_offset + (int)(ptr - ptr0); - ptr += strlen(substr); - } - } - if( last_occurrence < 0 ) - { - release(); - CV_Error( CV_StsError, "Could not find in the end of file.\n" ); - } - closeFile(); - file = fopen( filename.c_str(), "r+t" ); - CV_Assert(file != 0); - fseek( file, last_occurrence, SEEK_SET ); - // replace the last "" with " ", which has the same length - puts( " " ); - fseek( file, 0, SEEK_END ); - puts( "\n" ); - } - - emitter = createXMLEmitter(this); - } - else if( fmt == FileStorage::FORMAT_YAML ) - { - if( !append) - puts( "%YAML:1.0\n---\n" ); - else - puts( "...\n---\n" ); - - emitter = createYAMLEmitter(this); - } - else - { - CV_Assert( fmt == FileStorage::FORMAT_JSON ); - if( !append ) - puts( "{\n" ); - else - { - bool valid = false; - long roffset = 0; - for ( ; - fseek( file, roffset, SEEK_END ) == 0; - roffset -= 1 ) - { - const char end_mark = '}'; - if ( fgetc( file ) == end_mark ) - { - fseek( file, roffset, SEEK_END ); - valid = true; - break; - } - } - - if ( valid ) - { - closeFile(); - file = fopen( filename.c_str(), "r+t" ); - CV_Assert(file != 0); - fseek( file, roffset, SEEK_END ); - fputs( ",", file ); - } - else - { - CV_Error( CV_StsError, "Could not find '}' in the end of file.\n" ); - } - } - write_stack.back().indent = 4; - emitter = createJSONEmitter(this); - } - is_opened = true; - } - else - { - const size_t buf_size0 = 40; - buffer.resize(buf_size0); - if( mem_mode ) - { - strbuf = (char*)filename_or_buf; - strbufsize = strlen(strbuf); - } - - const char* yaml_signature = "%YAML"; - const char* json_signature = "{"; - const char* xml_signature = "gets(16); - CV_Assert(buf); - char* bufPtr = cv_skip_BOM(buf); - size_t bufOffset = bufPtr - buf; - - if(strncmp( bufPtr, yaml_signature, strlen(yaml_signature) ) == 0) - fmt = FileStorage::FORMAT_YAML; - else if(strncmp( bufPtr, json_signature, strlen(json_signature) ) == 0) - fmt = FileStorage::FORMAT_JSON; - else if(strncmp( bufPtr, xml_signature, strlen(xml_signature) ) == 0) - fmt = FileStorage::FORMAT_XML; - else if(strbufsize == bufOffset) - CV_Error(CV_BADARG_ERR, "Input file is invalid"); - else - CV_Error(CV_BADARG_ERR, "Unsupported file storage format"); - - rewind(); - strbufpos = bufOffset; - bufofs = 0; - - try - { - char* ptr = bufferStart(); - ptr[0] = ptr[1] = ptr[2] = '\0'; - FileNode root_nodes(fs_ext, 0, 0); - - uchar* rptr = reserveNodeSpace(root_nodes, 9); - *rptr = FileNode::SEQ; - writeInt(rptr + 1, 4); - writeInt(rptr + 5, 0); - - roots.clear(); - - switch (fmt) - { - case FileStorage::FORMAT_XML: parser = createXMLParser(this); break; - case FileStorage::FORMAT_YAML: parser = createYAMLParser(this); break; - case FileStorage::FORMAT_JSON: parser = createJSONParser(this); break; - default: parser = Ptr(); - } - - if( !parser.empty() ) - { - ok = parser->parse(ptr); - if( ok ) - { - finalizeCollection(root_nodes); - - CV_Assert( !fs_data_ptrs.empty() ); - FileNode roots_node(fs_ext, 0, 0); - size_t i, nroots = roots_node.size(); - FileNodeIterator it = roots_node.begin(); - - for( i = 0; i < nroots; i++, ++it ) - roots.push_back(*it); - } - } - } - catch(...) - { - is_opened = true; - release(); - throw; - } - - // release resources that we do not need anymore - closeFile(); - is_opened = true; - std::vector tmpbuf; - std::swap(buffer, tmpbuf); - bufofs = 0; - } - return ok; - } - - void puts( const char* str ) - { - CV_Assert( write_mode ); - if( mem_mode ) - std::copy(str, str + strlen(str), std::back_inserter(outbuf)); - else if( file ) - fputs( str, file ); -#if USE_ZLIB - else if( gzfile ) - gzputs( gzfile, str ); -#endif - else - CV_Error( CV_StsError, "The storage is not opened" ); - } - - char* getsFromFile( char* buf, int count ) - { - if( file ) - return fgets( buf, count, file ); - #if USE_ZLIB - if( gzfile ) - return gzgets( gzfile, buf, count ); - #endif - CV_Error(CV_StsError, "The storage is not opened"); - } - - char* gets( size_t maxCount ) - { - if( strbuf ) - { - size_t i = strbufpos, len = strbufsize; - const char* instr = strbuf; - for( ; i < len; i++ ) - { - char c = instr[i]; - if( c == '\0' || c == '\n' ) - { - if( c == '\n' ) - i++; - break; - } - } - size_t count = i - strbufpos; - if( maxCount == 0 || maxCount > count ) - maxCount = count; - buffer.resize(std::max(buffer.size(), maxCount + 8)); - memcpy(&buffer[0], instr + strbufpos, maxCount); - buffer[maxCount] = '\0'; - strbufpos = i; - return maxCount > 0 ? &buffer[0] : 0; - } - - const size_t MAX_BLOCK_SIZE = INT_MAX/2; // hopefully, that will be enough - if( maxCount == 0 ) - maxCount = MAX_BLOCK_SIZE; - else - CV_Assert(maxCount < MAX_BLOCK_SIZE); - size_t ofs = 0; - - for(;;) - { - int count = (int)std::min(buffer.size() - ofs - 16, maxCount); - char* ptr = getsFromFile( &buffer[ofs], count+1 ); - if( !ptr ) - break; - int delta = (int)strlen(ptr); - ofs += delta; - maxCount -= delta; - if( ptr[delta-1] == '\n' || maxCount == 0 ) - break; - if( delta == count ) - buffer.resize((size_t)(buffer.size()*1.5)); - } - return ofs > 0 ? &buffer[0] : 0; - } - - char* gets() - { - char* ptr = this->gets(0); - if( !ptr ) - { - ptr = bufferStart(); // FIXIT Why do we need this hack? What is about other parsers JSON/YAML? - *ptr = '\0'; - setEof(); - return 0; - } - else - { - size_t l = strlen(ptr); - if( l > 0 && ptr[l-1] != '\n' && ptr[l-1] != '\r' && !eof() ) - { - ptr[l] = '\n'; - ptr[l+1] = '\0'; - } - } - lineno++; - return ptr; - } - - bool eof() - { - if( dummy_eof ) - return true; - if( strbuf ) - return strbufpos >= strbufsize; - if( file ) - return feof(file) != 0; -#if USE_ZLIB - if( gzfile ) - return gzeof(gzfile) != 0; -#endif - return false; - } - - void setEof() - { - dummy_eof = true; - } - - void closeFile() - { - if( file ) - fclose( file ); -#if USE_ZLIB - else if( gzfile ) - gzclose( gzfile ); -#endif - file = 0; - gzfile = 0; - strbuf = 0; - strbufpos = 0; - is_opened = false; - } - - void rewind() - { - if( file ) - ::rewind(file); -#if USE_ZLIB - else if( gzfile ) - gzrewind(gzfile); -#endif - strbufpos = 0; - } - - char* resizeWriteBuffer( char* ptr, int len ) - { - const char* buffer_end = &buffer[0] + buffer.size(); - if( ptr + len < buffer_end ) - return ptr; - - const char* buffer_start = &buffer[0]; - int written_len = (int)(ptr - buffer_start); - - CV_Assert(written_len <= (int)buffer.size()); - int new_size = (int)((buffer_end - buffer_start)*3/2); - new_size = MAX( written_len + len, new_size ); - buffer.reserve( new_size + 256 ); - buffer.resize( new_size ); - bufofs = written_len; - return &buffer[0] + bufofs; - } - - char* flush() - { - char* buffer_start = &buffer[0]; - char* ptr = buffer_start + bufofs; - - if( ptr > buffer_start + space ) - { - ptr[0] = '\n'; - ptr[1] = '\0'; - puts( buffer_start ); - bufofs = 0; - } - - int indent = write_stack.back().indent; - - if( space != indent ) - { - memset( buffer_start, ' ', indent ); - space = indent; - } - bufofs = space; - ptr = buffer_start + bufofs; - - return ptr; - } - - void endWriteStruct() - { - CV_Assert( write_mode ); - CV_Assert( !write_stack.empty() ); - - FStructData& current_struct = write_stack.back(); - if( fmt == FileStorage::FORMAT_JSON && !FileNode::isFlow(current_struct.flags) && write_stack.size() > 1 ) - current_struct.indent = write_stack[write_stack.size() - 2].indent; - - emitter->endWriteStruct(current_struct); - - write_stack.pop_back(); - if( !write_stack.empty() ) - write_stack.back().flags &= ~FileNode::EMPTY; - } - - void startWriteStruct( const char* key, int struct_flags, - const char* type_name ) - { - CV_Assert( write_mode ); - - struct_flags = (struct_flags & (FileNode::TYPE_MASK|FileNode::FLOW)) | FileNode::EMPTY; - if( !FileNode::isCollection(struct_flags)) - CV_Error( CV_StsBadArg, - "Some collection type: FileNode::SEQ or FileNode::MAP must be specified" ); - - if( type_name && type_name[0] == '\0' ) - type_name = 0; - - FStructData s = emitter->startWriteStruct( write_stack.back(), key, struct_flags, type_name ); - write_stack.push_back(s); - size_t write_stack_size = write_stack.size(); - if( write_stack_size > 1 ) - write_stack[write_stack_size-2].flags &= ~FileNode::EMPTY; - - if( !FileNode::isFlow(s.flags) ) - flush(); - - if( fmt == FileStorage::FORMAT_JSON && type_name && type_name[0] && FileNode::isMap(struct_flags)) - { - emitter->write("type_id", type_name, false); - } - } - - void writeComment( const char* comment, bool eol_comment ) - { - CV_Assert(write_mode); - emitter->writeComment( comment, eol_comment ); - } - - void startNextStream() - { - CV_Assert(write_mode); - if( !empty_stream ) - { - while( !write_stack.empty() ) +void FileStorage::Impl::init() { + flags = 0; + buffer.clear(); + bufofs = 0; + state = UNDEFINED; + is_using_base64 = false; + state_of_writing_base64 = FileStorage_API::Base64State::Uncertain; + is_write_struct_delayed = false; + delayed_struct_key = nullptr; + delayed_struct_flags = 0; + delayed_type_name = nullptr; + base64_writer = nullptr; + is_opened = false; + dummy_eof = false; + write_mode = false; + mem_mode = false; + space = 0; + wrap_margin = 71; + fmt = 0; + file = 0; + gzfile = 0; + empty_stream = true; + + strbufv.clear(); + strbuf = 0; + strbufsize = strbufpos = 0; + roots.clear(); + + fs_data.clear(); + fs_data_ptrs.clear(); + fs_data_blksz.clear(); + freeSpaceOfs = 0; + + str_hash.clear(); + str_hash_data.clear(); + str_hash_data.resize(1); + str_hash_data[0] = '\0'; + + filename.clear(); + lineno = 0; +} + +FileStorage::Impl::Impl(FileStorage *_fs) { + fs_ext = _fs; + init(); +} + +FileStorage::Impl::~Impl() { + release(); +} + +void FileStorage::Impl::release(String *out) { + if (is_opened) { + if (out) + out->clear(); + if (write_mode) { + while (write_stack.size() > 1) { endWriteStruct(); + } flush(); - emitter->startNextStream(); - empty_stream = true; - write_stack.push_back(FStructData("", FileNode::EMPTY, 0)); - bufofs = 0; + if (fmt == FileStorage::FORMAT_XML) + puts("\n"); + else if (fmt == FileStorage::FORMAT_JSON) + puts("}\n"); + } + if (mem_mode && out) { + *out = cv::String(outbuf.begin(), outbuf.end()); + } + } + closeFile(); + init(); +} + +void FileStorage::Impl::analyze_file_name(const std::string &file_name, std::vector ¶ms) { + params.clear(); + static const char not_file_name = '\n'; + static const char parameter_begin = '?'; + static const char parameter_separator = '&'; + + if (file_name.find(not_file_name, (size_t) 0) != std::string::npos) + return; + + size_t beg = file_name.find_last_of(parameter_begin); + params.push_back(file_name.substr((size_t) 0, beg)); + + if (beg != std::string::npos) { + size_t end = file_name.size(); + beg++; + for (size_t param_beg = beg, param_end = beg; + param_end < end; + param_beg = param_end + 1) { + param_end = file_name.find_first_of(parameter_separator, param_beg); + if ((param_end == std::string::npos || param_end != param_beg) && param_beg + 1 < end) { + params.push_back(file_name.substr(param_beg, param_end - param_beg)); + } + } + } +} + +bool FileStorage::Impl::open(const char *filename_or_buf, int _flags, const char *encoding) { + bool ok = true; + release(); + + bool append = (_flags & 3) == FileStorage::APPEND; + mem_mode = (_flags & FileStorage::MEMORY) != 0; + + write_mode = (_flags & 3) != 0; + bool write_base64 = (write_mode || append) && (_flags & FileStorage::BASE64) != 0; + + bool isGZ = false; + size_t fnamelen = 0; + + std::vector params; + //if ( !mem_mode ) + { + analyze_file_name(filename_or_buf, params); + if (!params.empty()) + filename = params[0]; + + if (!write_base64 && params.size() >= 2 && + std::find(params.begin() + 1, params.end(), std::string("base64")) != params.end()) + write_base64 = (write_mode || append); + } + + if (filename.size() == 0 && !mem_mode && !write_mode) + CV_Error(cv::Error::StsNullPtr, "NULL or empty filename"); + + if (mem_mode && append) + CV_Error(cv::Error::StsBadFlag, "FileStorage::APPEND and FileStorage::MEMORY are not currently compatible"); + + flags = _flags; + + if (!mem_mode) { + char *dot_pos = strrchr((char *) filename.c_str(), '.'); + char compression = '\0'; + + if (dot_pos && dot_pos[1] == 'g' && dot_pos[2] == 'z' && + (dot_pos[3] == '\0' || (cv_isdigit(dot_pos[3]) && dot_pos[4] == '\0'))) { + if (append) { + CV_Error(cv::Error::StsNotImplemented, "Appending data to compressed file is not implemented"); + } + isGZ = true; + compression = dot_pos[3]; + if (compression) + dot_pos[3] = '\0', fnamelen--; + } + + if (!isGZ) { + file = fopen(filename.c_str(), !write_mode ? "rt" : !append ? "wt" : "a+t"); + if (!file) + return false; + } else { +#if USE_ZLIB + char mode[] = {write_mode ? 'w' : 'r', 'b', compression ? compression : '3', '\0'}; + gzfile = gzopen(filename.c_str(), mode); + if (!gzfile) + return false; +#else + CV_Error(cv::Error::StsNotImplemented, "There is no compressed file storage support in this configuration"); +#endif } } - void write( const String& key, int value ) - { - CV_Assert(write_mode); - emitter->write(key.c_str(), value); - } + roots.clear(); + fs_data.clear(); + wrap_margin = 71; + fmt = FileStorage::FORMAT_AUTO; - void write( const String& key, double value ) - { - CV_Assert(write_mode); - emitter->write(key.c_str(), value); - } + if (write_mode) { + fmt = flags & FileStorage::FORMAT_MASK; - void write( const String& key, const String& value ) - { - CV_Assert(write_mode); - emitter->write(key.c_str(), value.c_str(), false); - } + if (mem_mode) + outbuf.clear(); - void writeRawData( const std::string& dt, const void* _data, size_t len ) - { - CV_Assert(write_mode); - - size_t elemSize = fs::calcStructSize(dt.c_str(), 0); - CV_Assert(elemSize); - CV_Assert( len % elemSize == 0 ); - len /= elemSize; - - bool explicitZero = fmt == FileStorage::FORMAT_JSON; - const uchar* data0 = (const uchar*)_data; - int fmt_pairs[CV_FS_MAX_FMT_PAIRS*2], k, fmt_pair_count; - char buf[256] = ""; - - fmt_pair_count = fs::decodeFormat( dt.c_str(), fmt_pairs, CV_FS_MAX_FMT_PAIRS ); - - if( !len ) - return; - - if( !data0 ) - CV_Error( CV_StsNullPtr, "Null data pointer" ); - - if( fmt_pair_count == 1 ) - { - fmt_pairs[0] *= (int)len; - len = 1; + if (fmt == FileStorage::FORMAT_AUTO && !filename.empty()) { + const char *dot_pos = NULL; + const char *dot_pos2 = NULL; + // like strrchr() implementation, but save two last positions simultaneously + for (const char *pos = &filename[0]; pos[0] != 0; pos++) { + if (pos[0] == '.') { + dot_pos2 = dot_pos; + dot_pos = pos; + } + } + if (fs::strcasecmp(dot_pos, ".gz") == 0 && dot_pos2 != NULL) { + dot_pos = dot_pos2; + } + fmt = (fs::strcasecmp(dot_pos, ".xml") == 0 || fs::strcasecmp(dot_pos, ".xml.gz") == 0) + ? FileStorage::FORMAT_XML + : (fs::strcasecmp(dot_pos, ".json") == 0 || fs::strcasecmp(dot_pos, ".json.gz") == 0) + ? FileStorage::FORMAT_JSON + : FileStorage::FORMAT_YAML; + } else if (fmt == FileStorage::FORMAT_AUTO) { + fmt = FileStorage::FORMAT_XML; } - for(;len--; data0 += elemSize) - { - int offset = 0; - for( k = 0; k < fmt_pair_count; k++ ) - { - int i, count = fmt_pairs[k*2]; - int elem_type = fmt_pairs[k*2+1]; - int elem_size = CV_ELEM_SIZE(elem_type); - const char *ptr; + // we use factor=6 for XML (the longest characters (' and ") are encoded with 6 bytes (' and ") + // and factor=4 for YAML ( as we use 4 bytes for non ASCII characters (e.g. \xAB)) + int buf_size = CV_FS_MAX_LEN * (fmt == FileStorage::FORMAT_XML ? 6 : 4) + 1024; - offset = cvAlign( offset, elem_size ); - const uchar* data = data0 + offset; + if (append) { + fseek(file, 0, SEEK_END); + if (ftell(file) == 0) + append = false; + } - for( i = 0; i < count; i++ ) - { - switch( elem_type ) - { + write_stack.clear(); + empty_stream = true; + write_stack.push_back(FStructData("", FileNode::MAP | FileNode::EMPTY, 0)); + buffer.reserve(buf_size + 1024); + buffer.resize(buf_size); + bufofs = 0; + is_using_base64 = write_base64; + state_of_writing_base64 = FileStorage_API::Base64State::Uncertain; + + if (fmt == FileStorage::FORMAT_XML) { + size_t file_size = file ? (size_t) ftell(file) : (size_t) 0; + if (!append || file_size == 0) { + if (encoding && *encoding != '\0') { + if (fs::strcasecmp(encoding, "UTF-16") == 0) { + release(); + CV_Error(cv::Error::StsBadArg, "UTF-16 XML encoding is not supported! Use 8-bit encoding\n"); + } + + CV_Assert(strlen(encoding) < 1000); + char buf[1100]; + sprintf(buf, "\n", encoding); + puts(buf); + } else + puts("\n"); + puts("\n"); + } else { + int xml_buf_size = 1 << 10; + char substr[] = ""; + int last_occurrence = -1; + xml_buf_size = MIN(xml_buf_size, int(file_size)); + fseek(file, -xml_buf_size, SEEK_END); + // find the last occurrence of + for (;;) { + int line_offset = (int) ftell(file); + const char *ptr0 = this->gets(xml_buf_size); + const char *ptr = NULL; + if (!ptr0) + break; + ptr = ptr0; + for (;;) { + ptr = strstr(ptr, substr); + if (!ptr) + break; + last_occurrence = line_offset + (int) (ptr - ptr0); + ptr += strlen(substr); + } + } + if (last_occurrence < 0) { + release(); + CV_Error(cv::Error::StsError, "Could not find in the end of file.\n"); + } + closeFile(); + file = fopen(filename.c_str(), "r+t"); + CV_Assert(file != 0); + fseek(file, last_occurrence, SEEK_SET); + // replace the last "" with " ", which has the same length + puts(" "); + fseek(file, 0, SEEK_END); + puts("\n"); + } + + emitter = createXMLEmitter(this); + } else if (fmt == FileStorage::FORMAT_YAML) { + if (!append) + puts("%YAML:1.0\n---\n"); + else + puts("...\n---\n"); + + emitter = createYAMLEmitter(this); + } else { + CV_Assert(fmt == FileStorage::FORMAT_JSON); + if (!append) + puts("{\n"); + else { + bool valid = false; + long roffset = 0; + for (; + fseek(file, roffset, SEEK_END) == 0; + roffset -= 1) { + const char end_mark = '}'; + if (fgetc(file) == end_mark) { + fseek(file, roffset, SEEK_END); + valid = true; + break; + } + } + + if (valid) { + closeFile(); + file = fopen(filename.c_str(), "r+t"); + CV_Assert(file != 0); + fseek(file, roffset, SEEK_END); + fputs(",", file); + } else { + CV_Error(cv::Error::StsError, "Could not find '}' in the end of file.\n"); + } + } + write_stack.back().indent = 4; + emitter = createJSONEmitter(this); + } + is_opened = true; + } else { + const size_t buf_size0 = 40; + buffer.resize(buf_size0); + if (mem_mode) { + strbuf = (char *) filename_or_buf; + strbufsize = strlen(strbuf); + } + + const char *yaml_signature = "%YAML"; + const char *json_signature = "{"; + const char *xml_signature = "gets(16); + CV_Assert(buf); + char *bufPtr = cv_skip_BOM(buf); + size_t bufOffset = bufPtr - buf; + + if (strncmp(bufPtr, yaml_signature, strlen(yaml_signature)) == 0) + fmt = FileStorage::FORMAT_YAML; + else if (strncmp(bufPtr, json_signature, strlen(json_signature)) == 0) + fmt = FileStorage::FORMAT_JSON; + else if (strncmp(bufPtr, xml_signature, strlen(xml_signature)) == 0) + fmt = FileStorage::FORMAT_XML; + else if (strbufsize == bufOffset) + CV_Error(cv::Error::StsBadArg, "Input file is invalid"); + else + CV_Error(cv::Error::StsBadArg, "Unsupported file storage format"); + + rewind(); + strbufpos = bufOffset; + bufofs = 0; + + try { + char *ptr = bufferStart(); + ptr[0] = ptr[1] = ptr[2] = '\0'; + FileNode root_nodes(fs_ext, 0, 0); + + uchar *rptr = reserveNodeSpace(root_nodes, 9); + *rptr = FileNode::SEQ; + writeInt(rptr + 1, 4); + writeInt(rptr + 5, 0); + + roots.clear(); + + switch (fmt) { + case FileStorage::FORMAT_XML: + parser = createXMLParser(this); + break; + case FileStorage::FORMAT_YAML: + parser = createYAMLParser(this); + break; + case FileStorage::FORMAT_JSON: + parser = createJSONParser(this); + break; + default: + parser = Ptr(); + } + + if (!parser.empty()) { + ok = parser->parse(ptr); + if (ok) { + finalizeCollection(root_nodes); + + CV_Assert(!fs_data_ptrs.empty()); + FileNode roots_node(fs_ext, 0, 0); + size_t i, nroots = roots_node.size(); + FileNodeIterator it = roots_node.begin(); + + for (i = 0; i < nroots; i++, ++it) + roots.push_back(*it); + } + } + } + catch (...) { + is_opened = true; + release(); + throw; + } + + // release resources that we do not need anymore + closeFile(); + is_opened = true; + std::vector tmpbuf; + std::swap(buffer, tmpbuf); + bufofs = 0; + } + return ok; +} + +void FileStorage::Impl::puts(const char *str) { + CV_Assert(write_mode); + if (mem_mode) + std::copy(str, str + strlen(str), std::back_inserter(outbuf)); + else if (file) + fputs(str, file); +#if USE_ZLIB + else if (gzfile) + gzputs(gzfile, str); +#endif + else + CV_Error(cv::Error::StsError, "The storage is not opened"); +} + +char *FileStorage::Impl::getsFromFile(char *buf, int count) { + if (file) + return fgets(buf, count, file); +#if USE_ZLIB + if (gzfile) + return gzgets(gzfile, buf, count); +#endif + CV_Error(cv::Error::StsError, "The storage is not opened"); +} + +char *FileStorage::Impl::gets(size_t maxCount) { + if (strbuf) { + size_t i = strbufpos, len = strbufsize; + const char *instr = strbuf; + for (; i < len; i++) { + char c = instr[i]; + if (c == '\0' || c == '\n') { + if (c == '\n') + i++; + break; + } + } + size_t count = i - strbufpos; + if (maxCount == 0 || maxCount > count) + maxCount = count; + buffer.resize(std::max(buffer.size(), maxCount + 8)); + memcpy(&buffer[0], instr + strbufpos, maxCount); + buffer[maxCount] = '\0'; + strbufpos = i; + return maxCount > 0 ? &buffer[0] : 0; + } + + const size_t MAX_BLOCK_SIZE = INT_MAX / 2; // hopefully, that will be enough + if (maxCount == 0) + maxCount = MAX_BLOCK_SIZE; + else + CV_Assert(maxCount < MAX_BLOCK_SIZE); + size_t ofs = 0; + + for (;;) { + int count = (int) std::min(buffer.size() - ofs - 16, maxCount); + char *ptr = getsFromFile(&buffer[ofs], count + 1); + if (!ptr) + break; + int delta = (int) strlen(ptr); + ofs += delta; + maxCount -= delta; + if (ptr[delta - 1] == '\n' || maxCount == 0) + break; + if (delta == count) + buffer.resize((size_t) (buffer.size() * 1.5)); + } + return ofs > 0 ? &buffer[0] : 0; +} + +char *FileStorage::Impl::gets() { + char *ptr = this->gets(0); + if (!ptr) { + ptr = bufferStart(); // FIXIT Why do we need this hack? What is about other parsers JSON/YAML? + *ptr = '\0'; + setEof(); + return 0; + } else { + size_t l = strlen(ptr); + if (l > 0 && ptr[l - 1] != '\n' && ptr[l - 1] != '\r' && !eof()) { + ptr[l] = '\n'; + ptr[l + 1] = '\0'; + } + } + lineno++; + return ptr; +} + +bool FileStorage::Impl::eof() { + if (dummy_eof) + return true; + if (strbuf) + return strbufpos >= strbufsize; + if (file) + return feof(file) != 0; +#if USE_ZLIB + if (gzfile) + return gzeof(gzfile) != 0; +#endif + return false; +} + +void FileStorage::Impl::setEof() { + dummy_eof = true; +} + +void FileStorage::Impl::closeFile() { + if (file) + fclose(file); +#if USE_ZLIB + else if (gzfile) + gzclose(gzfile); +#endif + file = 0; + gzfile = 0; + strbuf = 0; + strbufpos = 0; + is_opened = false; +} + +void FileStorage::Impl::rewind() { + if (file) + ::rewind(file); +#if USE_ZLIB + else if (gzfile) + gzrewind(gzfile); +#endif + strbufpos = 0; +} + +char *FileStorage::Impl::resizeWriteBuffer(char *ptr, int len) { + const char *buffer_end = &buffer[0] + buffer.size(); + if (ptr + len < buffer_end) + return ptr; + + const char *buffer_start = &buffer[0]; + int written_len = (int) (ptr - buffer_start); + + CV_Assert(written_len <= (int) buffer.size()); + int new_size = (int) ((buffer_end - buffer_start) * 3 / 2); + new_size = MAX(written_len + len, new_size); + buffer.reserve(new_size + 256); + buffer.resize(new_size); + bufofs = written_len; + return &buffer[0] + bufofs; +} + +char *FileStorage::Impl::flush() { + char *buffer_start = &buffer[0]; + char *ptr = buffer_start + bufofs; + + if (ptr > buffer_start + space) { + ptr[0] = '\n'; + ptr[1] = '\0'; + puts(buffer_start); + bufofs = 0; + } + + int indent = write_stack.back().indent; + + if (space != indent) { + memset(buffer_start, ' ', indent); + space = indent; + } + bufofs = space; + ptr = buffer_start + bufofs; + + return ptr; +} + +void FileStorage::Impl::endWriteStruct() { + CV_Assert(write_mode); + + check_if_write_struct_is_delayed(false); + if (state_of_writing_base64 != FileStorage_API::Uncertain) + switch_to_Base64_state(FileStorage_API::Uncertain); + + CV_Assert(!write_stack.empty()); + + FStructData ¤t_struct = write_stack.back(); + if (fmt == FileStorage::FORMAT_JSON && !FileNode::isFlow(current_struct.flags) && write_stack.size() > 1) + current_struct.indent = write_stack[write_stack.size() - 2].indent; + + emitter->endWriteStruct(current_struct); + + write_stack.pop_back(); + if (!write_stack.empty()) + write_stack.back().flags &= ~FileNode::EMPTY; +} + +void FileStorage::Impl::startWriteStruct_helper(const char *key, int struct_flags, + const char *type_name) { + CV_Assert(write_mode); + + struct_flags = (struct_flags & (FileNode::TYPE_MASK | FileNode::FLOW)) | FileNode::EMPTY; + if (!FileNode::isCollection(struct_flags)) + CV_Error(cv::Error::StsBadArg, + "Some collection type: FileNode::SEQ or FileNode::MAP must be specified"); + + if (type_name && type_name[0] == '\0') + type_name = 0; + + FStructData s = emitter->startWriteStruct(write_stack.back(), key, struct_flags, type_name); + + write_stack.push_back(s); + size_t write_stack_size = write_stack.size(); + if (write_stack_size > 1) + write_stack[write_stack_size - 2].flags &= ~FileNode::EMPTY; + + if (fmt != FileStorage::FORMAT_JSON && !FileNode::isFlow(s.flags)) + flush(); + + if (fmt == FileStorage::FORMAT_JSON && type_name && type_name[0] && FileNode::isMap(struct_flags)) { + emitter->write("type_id", type_name, false); + } +} + +void FileStorage::Impl::startWriteStruct(const char *key, int struct_flags, + const char *type_name) { + check_if_write_struct_is_delayed(false); + if (state_of_writing_base64 == FileStorage_API::NotUse) + switch_to_Base64_state(FileStorage_API::Uncertain); + + if (state_of_writing_base64 == FileStorage_API::Uncertain && FileNode::isSeq(struct_flags) + && is_using_base64 && type_name == 0) { + /* Uncertain whether output Base64 data */ + make_write_struct_delayed(key, struct_flags, type_name); + } else if (type_name && memcmp(type_name, "binary", 6) == 0) { + /* Must output Base64 data */ + if ((FileNode::TYPE_MASK & struct_flags) != FileNode::SEQ) + CV_Error(cv::Error::StsBadArg, "must set 'struct_flags |= CV_NODE_SEQ' if using Base64."); + else if (state_of_writing_base64 != FileStorage_API::Uncertain) + CV_Error(cv::Error::StsError, "function \'cvStartWriteStruct\' calls cannot be nested if using Base64."); + + startWriteStruct_helper(key, struct_flags, "binary"); + + if (state_of_writing_base64 != FileStorage_API::Uncertain) + switch_to_Base64_state(FileStorage_API::Uncertain); + switch_to_Base64_state(FileStorage_API::InUse); + } else { + /* Won't output Base64 data */ + if (state_of_writing_base64 == FileStorage_API::InUse) + CV_Error(cv::Error::StsError, "At the end of the output Base64, `cvEndWriteStruct` is needed."); + + startWriteStruct_helper(key, struct_flags, type_name); + + if (state_of_writing_base64 != FileStorage_API::Uncertain) + switch_to_Base64_state(FileStorage_API::Uncertain); + switch_to_Base64_state(FileStorage_API::NotUse); + } +} + +void FileStorage::Impl::writeComment(const char *comment, bool eol_comment) { + CV_Assert(write_mode); + emitter->writeComment(comment, eol_comment); +} + +void FileStorage::Impl::startNextStream() { + CV_Assert(write_mode); + if (!empty_stream) { + while (!write_stack.empty()) + endWriteStruct(); + flush(); + emitter->startNextStream(); + empty_stream = true; + write_stack.push_back(FStructData("", FileNode::EMPTY, 0)); + bufofs = 0; + } +} + +void FileStorage::Impl::write(const String &key, int value) { + CV_Assert(write_mode); + emitter->write(key.c_str(), value); +} + +void FileStorage::Impl::write(const String &key, double value) { + CV_Assert(write_mode); + emitter->write(key.c_str(), value); +} + +void FileStorage::Impl::write(const String &key, const String &value) { + CV_Assert(write_mode); + emitter->write(key.c_str(), value.c_str(), false); +} + +void FileStorage::Impl::writeRawData(const std::string &dt, const void *_data, size_t len) { + CV_Assert(write_mode); + + if (is_using_base64 || state_of_writing_base64 == FileStorage_API::Base64State::InUse) { + writeRawDataBase64(_data, len, dt.c_str()); + return; + } else if (state_of_writing_base64 == FileStorage_API::Base64State::Uncertain) { + switch_to_Base64_state(FileStorage_API::Base64State::NotUse); + } + + size_t elemSize = fs::calcStructSize(dt.c_str(), 0); + CV_Assert(elemSize); + CV_Assert(len % elemSize == 0); + len /= elemSize; + + bool explicitZero = fmt == FileStorage::FORMAT_JSON; + const uchar *data0 = (const uchar *) _data; + int fmt_pairs[CV_FS_MAX_FMT_PAIRS * 2], k, fmt_pair_count; + char buf[256] = ""; + + fmt_pair_count = fs::decodeFormat(dt.c_str(), fmt_pairs, CV_FS_MAX_FMT_PAIRS); + + if (!len) + return; + + if (!data0) + CV_Error(cv::Error::StsNullPtr, "Null data pointer"); + + if (fmt_pair_count == 1) { + fmt_pairs[0] *= (int) len; + len = 1; + } + + for (; len--; data0 += elemSize) { + int offset = 0; + for (k = 0; k < fmt_pair_count; k++) { + int i, count = fmt_pairs[k * 2]; + int elem_type = fmt_pairs[k * 2 + 1]; + int elem_size = CV_ELEM_SIZE(elem_type); + const char *ptr; + + offset = cvAlign(offset, elem_size); + const uchar *data = data0 + offset; + + for (i = 0; i < count; i++) { + switch (elem_type) { case CV_8U: - ptr = fs::itoa( *(uchar*)data, buf, 10 ); + ptr = fs::itoa(*(uchar *) data, buf, 10); data++; break; case CV_8S: - ptr = fs::itoa( *(char*)data, buf, 10 ); + ptr = fs::itoa(*(char *) data, buf, 10); data++; break; case CV_16U: - ptr = fs::itoa( *(ushort*)data, buf, 10 ); + ptr = fs::itoa(*(ushort *) data, buf, 10); data += sizeof(ushort); break; case CV_16S: - ptr = fs::itoa( *(short*)data, buf, 10 ); + ptr = fs::itoa(*(short *) data, buf, 10); data += sizeof(short); break; case CV_32S: - ptr = fs::itoa( *(int*)data, buf, 10 ); + ptr = fs::itoa(*(int *) data, buf, 10); data += sizeof(int); break; case CV_32F: - ptr = fs::floatToString( buf, *(float*)data, false, explicitZero ); + ptr = fs::floatToString(buf, *(float *) data, false, explicitZero); data += sizeof(float); break; case CV_64F: - ptr = fs::doubleToString( buf, *(double*)data, explicitZero ); + ptr = fs::doubleToString(buf, *(double *) data, explicitZero); data += sizeof(double); break; case CV_16F: /* reference */ - ptr = fs::floatToString( buf, (float)*(float16_t*)data, true, explicitZero ); + ptr = fs::floatToString(buf, (float) *(float16_t *) data, true, explicitZero); data += sizeof(float16_t); break; default: - CV_Error( CV_StsUnsupportedFormat, "Unsupported type" ); + CV_Error(cv::Error::StsUnsupportedFormat, "Unsupported type"); return; - } - - emitter->writeScalar(0, ptr); } - offset = (int)(data - data0); + emitter->writeScalar(0, ptr); } + + offset = (int) (data - data0); } } +} - void writeRawDataBase64(const void* /*data*/, int /*len*/, const char* /*dt*/ ) - { +void FileStorage::Impl::workaround() { + check_if_write_struct_is_delayed(false); + if (state_of_writing_base64 != FileStorage_API::Base64State::Uncertain) + switch_to_Base64_state(FileStorage_API::Base64State::Uncertain); +} + +void FileStorage::Impl::switch_to_Base64_state(FileStorage_API::Base64State new_state) { + const char *err_unkonwn_state = "Unexpected error, unable to determine the Base64 state."; + const char *err_unable_to_switch = "Unexpected error, unable to switch to this state."; + + /* like a finite state machine */ + switch (state_of_writing_base64) { + case FileStorage_API::Base64State::Uncertain: + switch (new_state) { + case FileStorage_API::Base64State::InUse: + { + CV_DbgAssert(base64_writer == 0); + bool can_indent = (fmt != cv::FileStorage::Mode::FORMAT_JSON); + base64_writer = new base64::Base64Writer(*this, can_indent); + if (!can_indent) { + char *ptr = bufferPtr(); + *ptr++ = '\0'; + puts(bufferStart()); + setBufferPtr(bufferStart()); + memset(bufferStart(), 0, static_cast(space)); + puts("\"$base64$"); + } + break; + } + case FileStorage_API::Base64State::Uncertain: + break; + case FileStorage_API::Base64State::NotUse: + break; + default: + CV_Error(cv::Error::StsError, err_unkonwn_state); + break; + } + break; + case FileStorage_API::Base64State::InUse: + switch (new_state) { + case FileStorage_API::Base64State::InUse: + case FileStorage_API::Base64State::NotUse: + CV_Error(cv::Error::StsError, err_unable_to_switch); + break; + case FileStorage_API::Base64State::Uncertain: + delete base64_writer; + base64_writer = 0; + if ( fmt == cv::FileStorage::FORMAT_JSON ) + { + puts("\""); + setBufferPtr(bufferStart()); + flush(); + memset(bufferStart(), 0, static_cast(space) ); + setBufferPtr(bufferStart()); + } + break; + default: + CV_Error(cv::Error::StsError, err_unkonwn_state); + break; + } + break; + case FileStorage_API::Base64State::NotUse: + switch (new_state) { + case FileStorage_API::Base64State::InUse: + case FileStorage_API::Base64State::NotUse: + CV_Error(cv::Error::StsError, err_unable_to_switch); + break; + case FileStorage_API::Base64State::Uncertain: + break; + default: + CV_Error(cv::Error::StsError, err_unkonwn_state); + break; + } + break; + default: + CV_Error(cv::Error::StsError, err_unkonwn_state); + break; } - String releaseAndGetString(); + state_of_writing_base64 = new_state; +} - FileNode getFirstTopLevelNode() const - { - return roots.empty() ? FileNode() : roots[0]; +void FileStorage::Impl::make_write_struct_delayed(const char *key, int struct_flags, const char *type_name) { + CV_Assert(is_write_struct_delayed == false); + CV_DbgAssert(delayed_struct_key == nullptr); + CV_DbgAssert(delayed_struct_flags == 0); + CV_DbgAssert(delayed_type_name == nullptr); + + delayed_struct_flags = struct_flags; + + if (key != nullptr) { + delayed_struct_key = new char[strlen(key) + 1U]; + strcpy(delayed_struct_key, key); } - FileNode root(int streamIdx=0) const - { - return streamIdx >= 0 && streamIdx < (int)roots.size() ? roots[streamIdx] : FileNode(); + if (type_name != nullptr) { + delayed_type_name = new char[strlen(type_name) + 1U]; + strcpy(delayed_type_name, type_name); } - FileNode operator[](const String& nodename) const - { - return this->operator[](nodename.c_str()); - } + is_write_struct_delayed = true; +} - FileNode operator[](const char* /*nodename*/) const - { - return FileNode(); - } +void FileStorage::Impl::check_if_write_struct_is_delayed(bool change_type_to_base64) { + if (is_write_struct_delayed) { + /* save data to prevent recursive call errors */ + std::string struct_key; + std::string type_name; + int struct_flags = delayed_struct_flags; - int getFormat() const { return fmt; } - - char* bufferPtr() const { return (char*)(&buffer[0] + bufofs); } - char* bufferStart() const { return (char*)&buffer[0]; } - char* bufferEnd() const { return (char*)(&buffer[0] + buffer.size()); } - void setBufferPtr(char* ptr) - { - char* bufferstart = bufferStart(); - CV_Assert( ptr >= bufferstart && ptr <= bufferEnd() ); - bufofs = ptr - bufferstart; - } - int wrapMargin() const { return wrap_margin; } - - FStructData& getCurrentStruct() - { - CV_Assert(!write_stack.empty()); - return write_stack.back(); - } - - void setNonEmpty() - { - empty_stream = false; - } - - void processSpecialDouble( char* buf, double* value, char** endptr ) - { - FileStorage_API* fs = this; - char c = buf[0]; - int inf_hi = 0x7ff00000; - - if( c == '-' || c == '+' ) - { - inf_hi = c == '-' ? 0xfff00000 : 0x7ff00000; - c = *++buf; + if (delayed_struct_key != nullptr && *delayed_struct_key != '\0') { + struct_key.assign(delayed_struct_key); + } + if (delayed_type_name != nullptr && *delayed_type_name != '\0') { + type_name.assign(delayed_type_name); } - if( c != '.' ) - CV_PARSE_ERROR_CPP( "Bad format of floating-point constant" ); + /* reset */ + delete[] delayed_struct_key; + delete[] delayed_type_name; + delayed_struct_key = nullptr; + delayed_struct_flags = 0; + delayed_type_name = nullptr; - Cv64suf v; - v.f = 0.; - if( toupper(buf[1]) == 'I' && toupper(buf[2]) == 'N' && toupper(buf[3]) == 'F' ) - v.u = (uint64)inf_hi << 32; - else if( toupper(buf[1]) == 'N' && toupper(buf[2]) == 'A' && toupper(buf[3]) == 'N' ) - v.u = (uint64)-1; + is_write_struct_delayed = false; + + /* call */ + if (change_type_to_base64) { + startWriteStruct_helper(struct_key.c_str(), struct_flags, "binary"); + if (state_of_writing_base64 != FileStorage_API::Uncertain) + switch_to_Base64_state(FileStorage_API::Uncertain); + switch_to_Base64_state(FileStorage_API::InUse); + } else { + startWriteStruct_helper(struct_key.c_str(), struct_flags, type_name.c_str()); + if (state_of_writing_base64 != FileStorage_API::Uncertain) + switch_to_Base64_state(FileStorage_API::Uncertain); + switch_to_Base64_state(FileStorage_API::NotUse); + } + } +} + +void FileStorage::Impl::writeRawDataBase64(const void *_data, size_t len, const char *dt) { + CV_Assert(write_mode); + + check_if_write_struct_is_delayed(true); + + if (state_of_writing_base64 == FileStorage_API::Base64State::Uncertain) { + switch_to_Base64_state(FileStorage_API::Base64State::InUse); + } else if (state_of_writing_base64 != FileStorage_API::Base64State::InUse) { + CV_Error(cv::Error::StsError, "Base64 should not be used at present."); + } + + base64_writer->write(_data, len, dt); +} + +FileNode FileStorage::Impl::getFirstTopLevelNode() const { + return roots.empty() ? FileNode() : roots[0]; +} + +FileNode FileStorage::Impl::root(int streamIdx) const { + return streamIdx >= 0 && streamIdx < (int) roots.size() ? roots[streamIdx] : FileNode(); +} + +FileNode FileStorage::Impl::operator[](const String &nodename) const { + return this->operator[](nodename.c_str()); +} + +FileNode FileStorage::Impl::operator[](const char * /*nodename*/) const { + return FileNode(); +} + +int FileStorage::Impl::getFormat() const { return fmt; } + +char *FileStorage::Impl::bufferPtr() const { return (char *) (&buffer[0] + bufofs); } + +char *FileStorage::Impl::bufferStart() const { return (char *) &buffer[0]; } + +char *FileStorage::Impl::bufferEnd() const { return (char *) (&buffer[0] + buffer.size()); } + +void FileStorage::Impl::setBufferPtr(char *ptr) { + char *bufferstart = bufferStart(); + CV_Assert(ptr >= bufferstart && ptr <= bufferEnd()); + bufofs = ptr - bufferstart; +} + +int FileStorage::Impl::wrapMargin() const { return wrap_margin; } + +FStructData &FileStorage::Impl::getCurrentStruct() { + CV_Assert(!write_stack.empty()); + return write_stack.back(); +} + +void FileStorage::Impl::setNonEmpty() { + empty_stream = false; +} + +void FileStorage::Impl::processSpecialDouble(char *buf, double *value, char **endptr) { + FileStorage_API *fs = this; + char c = buf[0]; + int inf_hi = 0x7ff00000; + + if (c == '-' || c == '+') { + inf_hi = c == '-' ? 0xfff00000 : 0x7ff00000; + c = *++buf; + } + + if (c != '.') + CV_PARSE_ERROR_CPP("Bad format of floating-point constant"); + + Cv64suf v; + v.f = 0.; + if (toupper(buf[1]) == 'I' && toupper(buf[2]) == 'N' && toupper(buf[3]) == 'F') + v.u = (uint64) inf_hi << 32; + else if (toupper(buf[1]) == 'N' && toupper(buf[2]) == 'A' && toupper(buf[3]) == 'N') + v.u = (uint64) -1; + else + CV_PARSE_ERROR_CPP("Bad format of floating-point constant"); + *value = v.f; + *endptr = buf + 4; +} + +double FileStorage::Impl::strtod(char *ptr, char **endptr) { + double fval = ::strtod(ptr, endptr); + if (**endptr == '.') { + char *dot_pos = *endptr; + *dot_pos = ','; + double fval2 = ::strtod(ptr, endptr); + *dot_pos = '.'; + if (*endptr > dot_pos) + fval = fval2; else - CV_PARSE_ERROR_CPP( "Bad format of floating-point constant" ); - *value = v.f; - *endptr = buf + 4; + *endptr = dot_pos; } - double strtod( char* ptr, char** endptr ) - { - double fval = ::strtod( ptr, endptr ); - if( **endptr == '.' ) - { - char* dot_pos = *endptr; - *dot_pos = ','; - double fval2 = ::strtod( ptr, endptr ); - *dot_pos = '.'; - if( *endptr > dot_pos ) - fval = fval2; - else - *endptr = dot_pos; - } + if (*endptr == ptr || cv_isalpha(**endptr)) + processSpecialDouble(ptr, &fval, endptr); - if( *endptr == ptr || cv_isalpha(**endptr) ) - processSpecialDouble( ptr, &fval, endptr ); + return fval; +} - return fval; +void FileStorage::Impl::convertToCollection(int type, FileNode &node) { + CV_Assert(type == FileNode::SEQ || type == FileNode::MAP); + + int node_type = node.type(); + if (node_type == type) + return; + + bool named = node.isNamed(); + uchar *ptr = node.ptr() + 1 + (named ? 4 : 0); + + int ival = 0; + double fval = 0; + std::string sval; + bool add_first_scalar = false; + + if (node_type != FileNode::NONE) { + // scalar nodes can only be converted to sequences, e.g. in XML: + // 5[parser_position]... => create 5 with name "a" + // 5 6[parser_position]... => 5 is converted to [5] and then 6 is added to it + // + // otherwise we don't know where to get the element names from + CV_Assert(type == FileNode::SEQ); + if (node_type == FileNode::INT) { + ival = readInt(ptr); + add_first_scalar = true; + } else if (node_type == FileNode::REAL) { + fval = readReal(ptr); + add_first_scalar = true; + } else if (node_type == FileNode::STRING) { + sval = std::string(node); + add_first_scalar = true; + } else + CV_Error_(Error::StsError, ("The node of type %d cannot be converted to collection", node_type)); } - void convertToCollection(int type, FileNode& node) - { - CV_Assert( type == FileNode::SEQ || type == FileNode::MAP ); + ptr = reserveNodeSpace(node, 1 + (named ? 4 : 0) + 4 + 4); + *ptr++ = (uchar) (type | (named ? FileNode::NAMED : 0)); + // name has been copied automatically + if (named) + ptr += 4; + // set raw_size(collection)==4, nelems(collection)==1 + writeInt(ptr, 4); + writeInt(ptr + 4, 0); - int node_type = node.type(); - if( node_type == type ) - return; + if (add_first_scalar) + addNode(node, std::string(), node_type, + node_type == FileNode::INT ? (const void *) &ival : + node_type == FileNode::REAL ? (const void *) &fval : + node_type == FileNode::STRING ? (const void *) sval.c_str() : 0, + -1); +} - bool named = node.isNamed(); - uchar* ptr = node.ptr() + 1 + (named ? 4 : 0); +// a) allocates new FileNode (for that just set blockIdx to the last block and ofs to freeSpaceOfs) or +// b) reallocates just created new node (blockIdx and ofs must be taken from FileNode). +// If there is no enough space in the current block (it should be the last block added so far), +// the last block is shrunk so that it ends immediately before the reallocated node. Then, +// a new block of sufficient size is allocated and the FileNode is placed in the beginning of it. +// The case (a) can be used to allocate the very first node by setting blockIdx == ofs == 0. +// In the case (b) the existing tag and the name are copied automatically. +uchar *FileStorage::Impl::reserveNodeSpace(FileNode &node, size_t sz) { + bool shrinkBlock = false; + size_t shrinkBlockIdx = 0, shrinkSize = 0; - int ival = 0; - double fval = 0; - std::string sval; - bool add_first_scalar = false; + uchar *ptr = 0, *blockEnd = 0; - if( node_type != FileNode::NONE ) - { - // scalar nodes can only be converted to sequences, e.g. in XML: - // 5[parser_position]... => create 5 with name "a" - // 5 6[parser_position]... => 5 is converted to [5] and then 6 is added to it - // - // otherwise we don't know where to get the element names from - CV_Assert( type == FileNode::SEQ ); - if( node_type == FileNode::INT ) - { - ival = readInt(ptr); - add_first_scalar = true; - } - else if( node_type == FileNode::REAL ) - { - fval = readReal(ptr); - add_first_scalar = true; - } - else if( node_type == FileNode::STRING ) - { - sval = std::string(node); - add_first_scalar = true; - } - else - CV_Error_(Error::StsError, ("The node of type %d cannot be converted to collection", node_type)); + if (!fs_data_ptrs.empty()) { + size_t blockIdx = node.blockIdx; + size_t ofs = node.ofs; + CV_Assert(blockIdx == fs_data_ptrs.size() - 1); + CV_Assert(ofs <= fs_data_blksz[blockIdx]); + CV_Assert(freeSpaceOfs <= fs_data_blksz[blockIdx]); + //CV_Assert( freeSpaceOfs <= ofs + sz ); + + ptr = fs_data_ptrs[blockIdx] + ofs; + blockEnd = fs_data_ptrs[blockIdx] + fs_data_blksz[blockIdx]; + + CV_Assert(ptr >= fs_data_ptrs[blockIdx] && ptr <= blockEnd); + if (ptr + sz <= blockEnd) { + freeSpaceOfs = ofs + sz; + return ptr; } - ptr = reserveNodeSpace(node, 1 + (named ? 4 : 0) + 4 + 4); - *ptr++ = (uchar)(type | (named ? FileNode::NAMED : 0)); - // name has been copied automatically - if( named ) - ptr += 4; - // set raw_size(collection)==4, nelems(collection)==1 + if (ofs == + 0) // FileNode is a first component of this block. Resize current block instead of allocation of new one. + { + fs_data[blockIdx]->resize(sz); + ptr = &fs_data[blockIdx]->at(0); + fs_data_ptrs[blockIdx] = ptr; + fs_data_blksz[blockIdx] = sz; + freeSpaceOfs = sz; + return ptr; + } + + shrinkBlock = true; + shrinkBlockIdx = blockIdx; + shrinkSize = ofs; + } + + size_t blockSize = std::max((size_t) CV_FS_MAX_LEN * 4 - 256, sz) + 256; + Ptr > pv = makePtr >(blockSize); + fs_data.push_back(pv); + uchar *new_ptr = &pv->at(0); + fs_data_ptrs.push_back(new_ptr); + fs_data_blksz.push_back(blockSize); + node.blockIdx = fs_data_ptrs.size() - 1; + node.ofs = 0; + freeSpaceOfs = sz; + + if (ptr && ptr + 5 <= blockEnd) { + new_ptr[0] = ptr[0]; + if (ptr[0] & FileNode::NAMED) { + new_ptr[1] = ptr[1]; + new_ptr[2] = ptr[2]; + new_ptr[3] = ptr[3]; + new_ptr[4] = ptr[4]; + } + } + + if (shrinkBlock) { + fs_data[shrinkBlockIdx]->resize(shrinkSize); + fs_data_blksz[shrinkBlockIdx] = shrinkSize; + } + + return new_ptr; +} + +unsigned FileStorage::Impl::getStringOfs(const std::string &key) const { + str_hash_t::const_iterator it = str_hash.find(key); + return it != str_hash.end() ? it->second : 0; +} + +FileNode FileStorage::Impl::addNode(FileNode &collection, const std::string &key, + int elem_type, const void *value, int len) { + FileStorage_API *fs = this; + bool noname = key.empty() || (fmt == FileStorage::FORMAT_XML && strcmp(key.c_str(), "_") == 0); + convertToCollection(noname ? FileNode::SEQ : FileNode::MAP, collection); + + bool isseq = collection.empty() ? false : collection.isSeq(); + if (noname != isseq) + CV_PARSE_ERROR_CPP(noname ? "Map element should have a name" : + "Sequence element should not have name (use <_>)"); + unsigned strofs = 0; + if (!noname) { + strofs = getStringOfs(key); + if (!strofs) { + strofs = (unsigned) str_hash_data.size(); + size_t keysize = key.size() + 1; + str_hash_data.resize(strofs + keysize); + memcpy(&str_hash_data[0] + strofs, &key[0], keysize); + str_hash.insert(std::make_pair(key, strofs)); + } + } + + uchar *cp = collection.ptr(); + + size_t blockIdx = fs_data_ptrs.size() - 1; + size_t ofs = freeSpaceOfs; + FileNode node(fs_ext, blockIdx, ofs); + + size_t sz0 = 1 + (noname ? 0 : 4) + 8; + uchar *ptr = reserveNodeSpace(node, sz0); + + *ptr++ = (uchar) (elem_type | (noname ? 0 : FileNode::NAMED)); + if (elem_type == FileNode::NONE) + freeSpaceOfs -= 8; + + if (!noname) { + writeInt(ptr, (int) strofs); + ptr += 4; + } + + if (elem_type == FileNode::SEQ || elem_type == FileNode::MAP) { writeInt(ptr, 4); - writeInt(ptr + 4, 0); - - if( add_first_scalar ) - addNode(node, std::string(), node_type, - node_type == FileNode::INT ? (const void*)&ival : - node_type == FileNode::REAL ? (const void*)&fval : - node_type == FileNode::STRING ? (const void*)sval.c_str() : 0, - -1); + writeInt(ptr, 0); } - // a) allocates new FileNode (for that just set blockIdx to the last block and ofs to freeSpaceOfs) or - // b) reallocates just created new node (blockIdx and ofs must be taken from FileNode). - // If there is no enough space in the current block (it should be the last block added so far), - // the last block is shrunk so that it ends immediately before the reallocated node. Then, - // a new block of sufficient size is allocated and the FileNode is placed in the beginning of it. - // The case (a) can be used to allocate the very first node by setting blockIdx == ofs == 0. - // In the case (b) the existing tag and the name are copied automatically. - uchar* reserveNodeSpace(FileNode& node, size_t sz) - { - bool shrinkBlock = false; - size_t shrinkBlockIdx = 0, shrinkSize = 0; + if (value) + node.setValue(elem_type, value, len); - uchar *ptr = 0, *blockEnd = 0; + if (collection.isNamed()) + cp += 4; + int nelems = readInt(cp + 5); + writeInt(cp + 5, nelems + 1); - if( !fs_data_ptrs.empty() ) - { - size_t blockIdx = node.blockIdx; - size_t ofs = node.ofs; - CV_Assert( blockIdx == fs_data_ptrs.size()-1 ); - CV_Assert( ofs <= fs_data_blksz[blockIdx] ); - CV_Assert( freeSpaceOfs <= fs_data_blksz[blockIdx] ); - //CV_Assert( freeSpaceOfs <= ofs + sz ); + return node; +} - ptr = fs_data_ptrs[blockIdx] + ofs; - blockEnd = fs_data_ptrs[blockIdx] + fs_data_blksz[blockIdx]; +void FileStorage::Impl::finalizeCollection(FileNode &collection) { + if (!collection.isSeq() && !collection.isMap()) + return; + uchar *ptr0 = collection.ptr(), *ptr = ptr0 + 1; + if (*ptr0 & FileNode::NAMED) + ptr += 4; + size_t blockIdx = collection.blockIdx; + size_t ofs = collection.ofs + (size_t) (ptr + 8 - ptr0); + size_t rawSize = 4; + unsigned sz = (unsigned) readInt(ptr + 4); + if (sz > 0) { + size_t lastBlockIdx = fs_data_ptrs.size() - 1; - CV_Assert(ptr >= fs_data_ptrs[blockIdx] && ptr <= blockEnd); - if( ptr + sz <= blockEnd ) - { - freeSpaceOfs = ofs + sz; - return ptr; - } - - if (ofs == 0) // FileNode is a first component of this block. Resize current block instead of allocation of new one. - { - fs_data[blockIdx]->resize(sz); - ptr = &fs_data[blockIdx]->at(0); - fs_data_ptrs[blockIdx] = ptr; - fs_data_blksz[blockIdx] = sz; - freeSpaceOfs = sz; - return ptr; - } - - shrinkBlock = true; - shrinkBlockIdx = blockIdx; - shrinkSize = ofs; - } - - size_t blockSize = std::max((size_t)CV_FS_MAX_LEN*4 - 256, sz) + 256; - Ptr > pv = makePtr >(blockSize); - fs_data.push_back(pv); - uchar* new_ptr = &pv->at(0); - fs_data_ptrs.push_back(new_ptr); - fs_data_blksz.push_back(blockSize); - node.blockIdx = fs_data_ptrs.size()-1; - node.ofs = 0; - freeSpaceOfs = sz; - - if( ptr && ptr + 5 <= blockEnd ) - { - new_ptr[0] = ptr[0]; - if( ptr[0] & FileNode::NAMED ) - { - new_ptr[1] = ptr[1]; - new_ptr[2] = ptr[2]; - new_ptr[3] = ptr[3]; - new_ptr[4] = ptr[4]; - } - } - - if (shrinkBlock) - { - fs_data[shrinkBlockIdx]->resize(shrinkSize); - fs_data_blksz[shrinkBlockIdx] = shrinkSize; - } - - return new_ptr; - } - - unsigned getStringOfs( const std::string& key ) const - { - str_hash_t::const_iterator it = str_hash.find(key); - return it != str_hash.end() ? it->second : 0; - } - - FileNode addNode( FileNode& collection, const std::string& key, - int elem_type, const void* value, int len ) - { - FileStorage_API* fs = this; - bool noname = key.empty() || (fmt == FileStorage::FORMAT_XML && strcmp(key.c_str(), "_") == 0); - convertToCollection( noname ? FileNode::SEQ : FileNode::MAP, collection ); - - bool isseq = collection.empty() ? false : collection.isSeq(); - if( noname != isseq ) - CV_PARSE_ERROR_CPP( noname ? "Map element should have a name" : - "Sequence element should not have name (use <_>)" ); - unsigned strofs = 0; - if( !noname ) - { - strofs = getStringOfs(key); - if( !strofs ) - { - strofs = (unsigned)str_hash_data.size(); - size_t keysize = key.size() + 1; - str_hash_data.resize(strofs + keysize); - memcpy(&str_hash_data[0] + strofs, &key[0], keysize); - str_hash.insert(std::make_pair(key, strofs)); - } - } - - uchar* cp = collection.ptr(); - - size_t blockIdx = fs_data_ptrs.size() - 1; - size_t ofs = freeSpaceOfs; - FileNode node(fs_ext, blockIdx, ofs); - - size_t sz0 = 1 + (noname ? 0 : 4) + 8; - uchar* ptr = reserveNodeSpace(node, sz0); - - *ptr++ = (uchar)(elem_type | (noname ? 0 : FileNode::NAMED)); - if( elem_type == FileNode::NONE ) - freeSpaceOfs -= 8; - - if( !noname ) - { - writeInt(ptr, (int)strofs); - ptr += 4; - } - - if( elem_type == FileNode::SEQ || elem_type == FileNode::MAP ) - { - writeInt(ptr, 4); - writeInt(ptr, 0); - } - - if( value ) - node.setValue(elem_type, value, len); - - if( collection.isNamed() ) - cp += 4; - int nelems = readInt(cp + 5); - writeInt(cp + 5, nelems + 1); - - return node; - } - - void finalizeCollection( FileNode& collection ) - { - if( !collection.isSeq() && !collection.isMap() ) - return; - uchar* ptr0 = collection.ptr(), *ptr = ptr0 + 1; - if( *ptr0 & FileNode::NAMED ) - ptr += 4; - size_t blockIdx = collection.blockIdx; - size_t ofs = collection.ofs + (size_t)(ptr + 8 - ptr0); - size_t rawSize = 4; - unsigned sz = (unsigned)readInt(ptr + 4); - if( sz > 0 ) - { - size_t lastBlockIdx = fs_data_ptrs.size() - 1; - - for( ; blockIdx < lastBlockIdx; blockIdx++ ) - { - rawSize += fs_data_blksz[blockIdx] - ofs; - ofs = 0; - } - } - rawSize += freeSpaceOfs - ofs; - writeInt(ptr, (int)rawSize); - } - - void normalizeNodeOfs(size_t& blockIdx, size_t& ofs) const - { - while( ofs >= fs_data_blksz[blockIdx] ) - { - if( blockIdx == fs_data_blksz.size() - 1 ) - { - CV_Assert( ofs == fs_data_blksz[blockIdx] ); - break; - } - ofs -= fs_data_blksz[blockIdx]; - blockIdx++; - } - } - - class Base64Decoder - { - public: - Base64Decoder() { ofs = 0; ptr = 0; indent = 0; totalchars = 0; eos = true; } - void init(Ptr& _parser, char* _ptr, int _indent) - { - parser = _parser; - ptr = _ptr; - indent = _indent; - encoded.clear(); - decoded.clear(); + for (; blockIdx < lastBlockIdx; blockIdx++) { + rawSize += fs_data_blksz[blockIdx] - ofs; ofs = 0; - totalchars = 0; - eos = false; } + } + rawSize += freeSpaceOfs - ofs; + writeInt(ptr, (int) rawSize); +} - bool readMore(int needed) - { - static const uchar base64tab[] = +void FileStorage::Impl::normalizeNodeOfs(size_t &blockIdx, size_t &ofs) const { + while (ofs >= fs_data_blksz[blockIdx]) { + if (blockIdx == fs_data_blksz.size() - 1) { + CV_Assert(ofs == fs_data_blksz[blockIdx]); + break; + } + ofs -= fs_data_blksz[blockIdx]; + blockIdx++; + } +} + +FileStorage::Impl::Base64State FileStorage::Impl::get_state_of_writing_base64() { + return state_of_writing_base64; +} + +int FileStorage::Impl::get_space() { + return space; +} + + +FileStorage::Impl::Base64Decoder::Base64Decoder() { + ofs = 0; + ptr = 0; + indent = 0; + totalchars = 0; + eos = true; +} + +void FileStorage::Impl::Base64Decoder::init(Ptr &_parser, char *_ptr, int _indent) { + parser = _parser; + ptr = _ptr; + indent = _indent; + encoded.clear(); + decoded.clear(); + ofs = 0; + totalchars = 0; + eos = false; +} + +bool FileStorage::Impl::Base64Decoder::readMore(int needed) { + static const uchar base64tab[] = { - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 62, 0, 0, 0, 63, - 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 0, 0, 0, 0, 0, 0, - 0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, - 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 0, 0, 0, 0, 0, - 0, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, - 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 62, 0, 0, 0, 63, + 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 0, 0, 0, 0, 0, 0, + 0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, + 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 0, 0, 0, 0, 0, + 0, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, + 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }; - if( eos ) - return false; + if (eos) + return false; - size_t sz = decoded.size(); - CV_Assert( ofs <= sz ); - sz -= ofs; - for( size_t i = 0; i < sz; i++ ) - decoded[i] = decoded[ofs + i]; + size_t sz = decoded.size(); + CV_Assert(ofs <= sz); + sz -= ofs; + for (size_t i = 0; i < sz; i++) + decoded[i] = decoded[ofs + i]; - decoded.resize(sz); - ofs = 0; + decoded.resize(sz); + ofs = 0; - CV_Assert( !parser.empty() && ptr ); - char *beg = 0, *end = 0; - bool ok = parser->getBase64Row(ptr, indent, beg, end); - ptr = end; - std::copy(beg, end, std::back_inserter(encoded)); - totalchars += end - beg; + CV_Assert(!parser.empty() && ptr); + char *beg = 0, *end = 0; + bool ok = parser->getBase64Row(ptr, indent, beg, end); + ptr = end; + std::copy(beg, end, std::back_inserter(encoded)); + totalchars += end - beg; - if( !ok || beg == end ) - { - // in the end of base64 sequence pad it with '=' characters so that - // its total length is multiple of - eos = true; - size_t tc = totalchars; - for( ; tc % 4 != 0; tc++ ) - encoded.push_back('='); - } + if (!ok || beg == end) { + // in the end of base64 sequence pad it with '=' characters so that + // its total length is multiple of + eos = true; + size_t tc = totalchars; + for (; tc % 4 != 0; tc++) + encoded.push_back('='); + } - int i = 0, j, n = (int)encoded.size(); - if( n > 0 ) - { - const uchar* tab = base64tab; - char* src = &encoded[0]; + int i = 0, j, n = (int) encoded.size(); + if (n > 0) { + const uchar *tab = base64tab; + char *src = &encoded[0]; - for( ; i <= n - 4; i += 4 ) - { - // dddddd cccccc bbbbbb aaaaaa => ddddddcc ccccbbbb bbaaaaaa - uchar d = tab[(int)(uchar)src[i]], c = tab[(int)(uchar)src[i+1]]; - uchar b = tab[(int)(uchar)src[i+2]], a = tab[(int)(uchar)src[i+3]]; + for (; i <= n - 4; i += 4) { + // dddddd cccccc bbbbbb aaaaaa => ddddddcc ccccbbbb bbaaaaaa + uchar d = tab[(int) (uchar) src[i]], c = tab[(int) (uchar) src[i + 1]]; + uchar b = tab[(int) (uchar) src[i + 2]], a = tab[(int) (uchar) src[i + 3]]; - decoded.push_back((uchar)((d << 2) | (c >> 4))); - decoded.push_back((uchar)((c << 4) | (b >> 2))); - decoded.push_back((uchar)((b << 6) | a)); - } - } - - if( i > 0 && encoded[i-1] == '=' ) - { - if( i > 1 && encoded[i-2] == '=' && !decoded.empty() ) - decoded.pop_back(); - if( !decoded.empty() ) - decoded.pop_back(); - } - - n -= i; - for( j = 0; j < n; j++ ) - encoded[j] = encoded[i + j]; - encoded.resize(n); - - return (int)decoded.size() >= needed; + decoded.push_back((uchar) ((d << 2) | (c >> 4))); + decoded.push_back((uchar) ((c << 4) | (b >> 2))); + decoded.push_back((uchar) ((b << 6) | a)); } + } - uchar getUInt8() - { - size_t sz = decoded.size(); - if( ofs >= sz && !readMore(1) ) - return (uchar)0; - return decoded[ofs++]; - } + if (i > 0 && encoded[i - 1] == '=') { + if (i > 1 && encoded[i - 2] == '=' && !decoded.empty()) + decoded.pop_back(); + if (!decoded.empty()) + decoded.pop_back(); + } - ushort getUInt16() - { - size_t sz = decoded.size(); - if( ofs + 2 > sz && !readMore(2) ) - return (ushort)0; - ushort val = (decoded[ofs] + (decoded[ofs + 1] << 8)); - ofs += 2; - return val; - } + n -= i; + for (j = 0; j < n; j++) + encoded[j] = encoded[i + j]; + encoded.resize(n); - int getInt32() - { - size_t sz = decoded.size(); - if( ofs + 4 > sz && !readMore(4) ) - return 0; - int ival = readInt(&decoded[ofs]); - ofs += 4; - return ival; - } + return (int) decoded.size() >= needed; +} - double getFloat64() - { - size_t sz = decoded.size(); - if( ofs + 8 > sz && !readMore(8) ) - return 0; - double fval = readReal(&decoded[ofs]); - ofs += 8; - return fval; - } +uchar FileStorage::Impl::Base64Decoder::getUInt8() { + size_t sz = decoded.size(); + if (ofs >= sz && !readMore(1)) + return (uchar) 0; + return decoded[ofs++]; +} - bool endOfStream() const { return eos; } - char* getPtr() const { return ptr; } - protected: +ushort FileStorage::Impl::Base64Decoder::getUInt16() { + size_t sz = decoded.size(); + if (ofs + 2 > sz && !readMore(2)) + return (ushort) 0; + ushort val = (decoded[ofs] + (decoded[ofs + 1] << 8)); + ofs += 2; + return val; +} - Ptr parser; - char* ptr; - int indent; - std::vector encoded; - std::vector decoded; - size_t ofs; - size_t totalchars; - bool eos; - }; +int FileStorage::Impl::Base64Decoder::getInt32() { + size_t sz = decoded.size(); + if (ofs + 4 > sz && !readMore(4)) + return 0; + int ival = readInt(&decoded[ofs]); + ofs += 4; + return ival; +} - char* parseBase64(char* ptr, int indent, FileNode& collection) - { - const int BASE64_HDR_SIZE = 24; - char dt[BASE64_HDR_SIZE+1] = {0}; - base64decoder.init(parser, ptr, indent); +double FileStorage::Impl::Base64Decoder::getFloat64() { + size_t sz = decoded.size(); + if (ofs + 8 > sz && !readMore(8)) + return 0; + double fval = readReal(&decoded[ofs]); + ofs += 8; + return fval; +} - int i, k; +bool FileStorage::Impl::Base64Decoder::endOfStream() const { return eos; } - for( i = 0; i < BASE64_HDR_SIZE; i++ ) - dt[i] = (char)base64decoder.getUInt8(); - for( i = 0; i < BASE64_HDR_SIZE; i++ ) - if( isspace(dt[i])) - break; - dt[i] = '\0'; +char *FileStorage::Impl::Base64Decoder::getPtr() const { return ptr; } - CV_Assert( !base64decoder.endOfStream() ); - int fmt_pairs[CV_FS_MAX_FMT_PAIRS*2]; - int fmt_pair_count = fs::decodeFormat( dt, fmt_pairs, CV_FS_MAX_FMT_PAIRS ); - int ival = 0; - double fval = 0; +char *FileStorage::Impl::parseBase64(char *ptr, int indent, FileNode &collection) { + const int BASE64_HDR_SIZE = 24; + char dt[BASE64_HDR_SIZE + 1] = {0}; + base64decoder.init(parser, ptr, indent); - for(;;) - { - for( k = 0; k < fmt_pair_count; k++ ) - { - int elem_type = fmt_pairs[k*2+1]; - int count = fmt_pairs[k*2]; + int i, k; - for( i = 0; i < count; i++ ) - { - int node_type = FileNode::INT; - switch( elem_type ) - { + for (i = 0; i < BASE64_HDR_SIZE; i++) + dt[i] = (char) base64decoder.getUInt8(); + for (i = 0; i < BASE64_HDR_SIZE; i++) + if (isspace(dt[i])) + break; + dt[i] = '\0'; + + CV_Assert(!base64decoder.endOfStream()); + + int fmt_pairs[CV_FS_MAX_FMT_PAIRS * 2]; + int fmt_pair_count = fs::decodeFormat(dt, fmt_pairs, CV_FS_MAX_FMT_PAIRS); + int ival = 0; + double fval = 0; + + for (;;) { + for (k = 0; k < fmt_pair_count; k++) { + int elem_type = fmt_pairs[k * 2 + 1]; + int count = fmt_pairs[k * 2]; + + for (i = 0; i < count; i++) { + int node_type = FileNode::INT; + switch (elem_type) { case CV_8U: ival = base64decoder.getUInt8(); break; case CV_8S: - ival = (char)base64decoder.getUInt8(); + ival = (char) base64decoder.getUInt8(); break; case CV_16U: ival = base64decoder.getUInt16(); break; case CV_16S: - ival = (short)base64decoder.getUInt16(); + ival = (short) base64decoder.getUInt16(); break; case CV_32S: ival = base64decoder.getInt32(); break; - case CV_32F: - { + case CV_32F: { Cv32suf v; v.i = base64decoder.getInt32(); fval = v.f; node_type = FileNode::REAL; - } + } break; case CV_64F: fval = base64decoder.getFloat64(); node_type = FileNode::REAL; break; case CV_16F: - fval = (float)float16_t::fromBits(base64decoder.getUInt16()); + fval = (float) float16_t::fromBits(base64decoder.getUInt16()); node_type = FileNode::REAL; break; default: - CV_Error( Error::StsUnsupportedFormat, "Unsupported type" ); - } - - if( base64decoder.endOfStream() ) - break; - addNode(collection, std::string(), node_type, - node_type == FileNode::INT ? (void*)&ival : (void*)&fval, -1); + CV_Error(Error::StsUnsupportedFormat, "Unsupported type"); } + + if (base64decoder.endOfStream()) + break; + addNode(collection, std::string(), node_type, + node_type == FileNode::INT ? (void *) &ival : (void *) &fval, -1); } - if( base64decoder.endOfStream() ) - break; } - - finalizeCollection(collection); - return base64decoder.getPtr(); + if (base64decoder.endOfStream()) + break; } - void parseError( const char* func_name, const std::string& err_msg, const char* source_file, int source_line ) - { - std::string msg = format("%s(%d): %s", filename.c_str(), lineno, err_msg.c_str()); - error(Error::StsParseError, func_name, msg.c_str(), source_file, source_line ); - } + finalizeCollection(collection); + return base64decoder.getPtr(); +} - const uchar* getNodePtr(size_t blockIdx, size_t ofs) const - { - CV_Assert( blockIdx < fs_data_ptrs.size()); - CV_Assert( ofs < fs_data_blksz[blockIdx]); +void FileStorage::Impl::parseError(const char *func_name, const std::string &err_msg, const char *source_file, + int source_line) { + std::string msg = format("%s(%d): %s", filename.c_str(), lineno, err_msg.c_str()); + error(Error::StsParseError, func_name, msg.c_str(), source_file, source_line); +} - return fs_data_ptrs[blockIdx] + ofs; - } +const uchar *FileStorage::Impl::getNodePtr(size_t blockIdx, size_t ofs) const { + CV_Assert(blockIdx < fs_data_ptrs.size()); + CV_Assert(ofs < fs_data_blksz[blockIdx]); - std::string getName( size_t nameofs ) const - { - CV_Assert( nameofs < str_hash_data.size() ); - return std::string(&str_hash_data[nameofs]); - } + return fs_data_ptrs[blockIdx] + ofs; +} - FileStorage* getFS() { return fs_ext; } +std::string FileStorage::Impl::getName(size_t nameofs) const { + CV_Assert(nameofs < str_hash_data.size()); + return std::string(&str_hash_data[nameofs]); +} - FileStorage* fs_ext; +FileStorage *FileStorage::Impl::getFS() { return fs_ext; } - std::string filename; - int flags; - bool empty_stream; - - FILE* file; - gzFile gzfile; - - bool is_opened; - bool dummy_eof; - bool write_mode; - bool mem_mode; - int fmt; - - State state; //!< current state of the FileStorage (used only for writing) - int space, wrap_margin; - std::deque write_stack; - std::vector buffer; - size_t bufofs; - - std::deque outbuf; - - Ptr emitter; - Ptr parser; - Base64Decoder base64decoder; - - std::vector roots; - std::vector > > fs_data; - std::vector fs_data_ptrs; - std::vector fs_data_blksz; - size_t freeSpaceOfs; - typedef std::unordered_map str_hash_t; - str_hash_t str_hash; - std::vector str_hash_data; - - std::vector strbufv; - char* strbuf; - size_t strbufsize; - size_t strbufpos; - int lineno; -}; FileStorage::FileStorage() : state(0) @@ -1807,7 +1842,7 @@ FileStorage::FileStorage(const String& filename, int flags, const String& encodi void FileStorage::startWriteStruct(const String& name, int struct_flags, const String& typeName) { - p->startWriteStruct(name.c_str(), struct_flags, typeName.c_str()); + p->startWriteStruct(name.size() ? name.c_str() : 0, struct_flags, typeName.size() ? typeName.c_str() : 0); elname = String(); if ((struct_flags & FileNode::TYPE_MASK) == FileNode::SEQ) state = FileStorage::VALUE_EXPECTED; @@ -1882,7 +1917,7 @@ std::string FileStorage::getDefaultObjectName(const std::string& _filename) } ptr++; if( ptr == ptr2 ) - CV_Error( CV_StsBadArg, "Invalid filename" ); + CV_Error( cv::Error::StsBadArg, "Invalid filename" ); char* name = name_buf.data(); @@ -2005,12 +2040,14 @@ FileStorage& operator << (FileStorage& fs, const String& str) if( c == '}' || c == ']' ) { if( fs_impl->write_stack.empty() ) - CV_Error_( CV_StsError, ("Extra closing '%c'", *_str) ); + CV_Error_( cv::Error::StsError, ("Extra closing '%c'", *_str) ); + + fs_impl->workaround(); int struct_flags = fs_impl->write_stack.back().flags; char expected_bracket = FileNode::isMap(struct_flags) ? '}' : ']'; if( c != expected_bracket ) - CV_Error_( CV_StsError, ("The closing '%c' does not match the opening '%c'", c, expected_bracket)); + CV_Error_( cv::Error::StsError, ("The closing '%c' does not match the opening '%c'", c, expected_bracket)); fs_impl->endWriteStruct(); CV_Assert(!fs_impl->write_stack.empty()); struct_flags = fs_impl->write_stack.back().flags; @@ -2020,7 +2057,7 @@ FileStorage& operator << (FileStorage& fs, const String& str) else if( fs.state == NAME_EXPECTED + INSIDE_MAP ) { if (!cv_isalpha(c) && c != '_') - CV_Error_( CV_StsError, ("Incorrect element name %s; should start with a letter or '_'", _str) ); + CV_Error_( cv::Error::StsError, ("Incorrect element name %s; should start with a letter or '_'", _str) ); fs.elname = str; fs.state = VALUE_EXPECTED + INSIDE_MAP; } @@ -2049,7 +2086,7 @@ FileStorage& operator << (FileStorage& fs, const String& str) } } else - CV_Error( CV_StsError, "Invalid fs.state" ); + CV_Error( cv::Error::StsError, "Invalid fs.state" ); return fs; } diff --git a/modules/core/src/persistence.hpp b/modules/core/src/persistence.hpp index 05c7adc17c..1a9dbecf7c 100644 --- a/modules/core/src/persistence.hpp +++ b/modules/core/src/persistence.hpp @@ -163,6 +163,24 @@ public: CV_NORETURN virtual void parseError(const char* funcname, const std::string& msg, const char* filename, int lineno) = 0; + +private: + enum Base64State{ + Uncertain, + NotUse, + InUse, + }; + + friend class cv::FileStorage::Impl; + friend class cv::FileStorage; + friend class JSONEmitter; + friend class XMLEmitter; + friend class YAMLEmitter; + + virtual void check_if_write_struct_is_delayed(bool change_type_to_base64 = false) = 0; + virtual void switch_to_Base64_state(Base64State state) = 0; + virtual Base64State get_state_of_writing_base64() = 0; + virtual int get_space() = 0; }; class FileStorageEmitter diff --git a/modules/core/src/persistence_base64_encoding.cpp b/modules/core/src/persistence_base64_encoding.cpp new file mode 100644 index 0000000000..7d90fd422b --- /dev/null +++ b/modules/core/src/persistence_base64_encoding.cpp @@ -0,0 +1,370 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html + +#include "precomp.hpp" +#include "persistence_impl.hpp" +#include "persistence_base64_encoding.hpp" + +namespace cv +{ + +class base64::Base64ContextEmitter +{ +public: + explicit Base64ContextEmitter(cv::FileStorage::Impl& fs, bool needs_indent_) + : file_storage(fs) + , needs_indent(needs_indent_) + , binary_buffer(BUFFER_LEN) + , base64_buffer(base64_encode_buffer_size(BUFFER_LEN)) + , src_beg(0) + , src_cur(0) + , src_end(0) + { + src_beg = binary_buffer.data(); + src_end = src_beg + BUFFER_LEN; + src_cur = src_beg; + + CV_Assert(fs.write_mode); + + if (needs_indent) + { + file_storage.flush(); + } + } + + ~Base64ContextEmitter() + { + /* cleaning */ + if (src_cur != src_beg) + flush(); /* encode the rest binary data to base64 buffer */ + } + + Base64ContextEmitter & write(const uchar * beg, const uchar * end) + { + if (beg >= end) + return *this; + + while (beg < end) { + /* collect binary data and copy to binary buffer */ + size_t len = std::min(end - beg, src_end - src_cur); + std::memcpy(src_cur, beg, len); + beg += len; + src_cur += len; + + if (src_cur >= src_end) { + /* binary buffer is full. */ + /* encode it to base64 and send result to fs */ + flush(); + } + } + + return *this; + } + + /* + * a convertor must provide : + * - `operator >> (uchar * & dst)` for writing current binary data to `dst` and moving to next data. + * - `operator bool` for checking if current loaction is valid and not the end. + */ + template inline + Base64ContextEmitter & write(_to_binary_convertor_t & convertor) + { + static const size_t BUFFER_MAX_LEN = 1024U; + + std::vector buffer(BUFFER_MAX_LEN); + uchar * beg = buffer.data(); + uchar * end = beg; + + while (convertor) { + convertor >> end; + write(beg, end); + end = beg; + } + + return *this; + } + + bool flush() + { + /* control line width, so on. */ + size_t len = base64_encode(src_beg, base64_buffer.data(), 0U, src_cur - src_beg); + if (len == 0U) + return false; + + src_cur = src_beg; + + if ( !needs_indent) + { + file_storage.puts((const char*)base64_buffer.data()); + } + else + { + const char newline[] = "\n"; + char space[80]; + int ident = file_storage.write_stack.back().indent; + memset(space, ' ', static_cast(ident)); + space[ident] = '\0'; + + file_storage.puts(space); + file_storage.puts((const char*)base64_buffer.data()); + file_storage.puts(newline); + file_storage.flush(); + } + + return true; + } + +private: + /* because of Base64, we must keep its length a multiple of 3 */ + static const size_t BUFFER_LEN = 48U; + // static_assert(BUFFER_LEN % 3 == 0, "BUFFER_LEN is invalid"); + +private: + cv::FileStorage::Impl& file_storage; + bool needs_indent; + + std::vector binary_buffer; + std::vector base64_buffer; + uchar * src_beg; + uchar * src_cur; + uchar * src_end; +}; + +std::string base64::make_base64_header(const char *dt) { + std::ostringstream oss; + oss << dt << ' '; + std::string buffer(oss.str()); + CV_Assert(buffer.size() < ::base64::HEADER_SIZE); + + buffer.reserve(::base64::HEADER_SIZE); + while (buffer.size() < ::base64::HEADER_SIZE) + buffer += ' '; + + return buffer; +} + +size_t base64::base64_encode(const uint8_t *src, uint8_t *dst, size_t off, size_t cnt) { + if (!src || !dst || !cnt) + return 0; + + /* initialize beginning and end */ + uint8_t * dst_beg = dst; + uint8_t * dst_cur = dst_beg; + + uint8_t const * src_beg = src + off; + uint8_t const * src_cur = src_beg; + uint8_t const * src_end = src_cur + cnt / 3U * 3U; + + /* integer multiples part */ + while (src_cur < src_end) { + uint8_t _2 = *src_cur++; + uint8_t _1 = *src_cur++; + uint8_t _0 = *src_cur++; + *dst_cur++ = base64_mapping[ _2 >> 2U]; + *dst_cur++ = base64_mapping[(_1 & 0xF0U) >> 4U | (_2 & 0x03U) << 4U]; + *dst_cur++ = base64_mapping[(_0 & 0xC0U) >> 6U | (_1 & 0x0FU) << 2U]; + *dst_cur++ = base64_mapping[ _0 & 0x3FU]; + } + + /* remainder part */ + size_t rst = src_beg + cnt - src_cur; + if (rst == 1U) { + uint8_t _2 = *src_cur++; + *dst_cur++ = base64_mapping[ _2 >> 2U]; + *dst_cur++ = base64_mapping[(_2 & 0x03U) << 4U]; + } else if (rst == 2U) { + uint8_t _2 = *src_cur++; + uint8_t _1 = *src_cur++; + *dst_cur++ = base64_mapping[ _2 >> 2U]; + *dst_cur++ = base64_mapping[(_2 & 0x03U) << 4U | (_1 & 0xF0U) >> 4U]; + *dst_cur++ = base64_mapping[(_1 & 0x0FU) << 2U]; + } + + /* padding */ + switch (rst) + { + case 1U: *dst_cur++ = base64_padding; + /* fallthrough */ + case 2U: *dst_cur++ = base64_padding; + /* fallthrough */ + default: *dst_cur = 0; + break; + } + + return static_cast(dst_cur - dst_beg); +} + +int base64::icvCalcStructSize(const char *dt, int initial_size) { + int size = cv::fs::calcElemSize( dt, initial_size ); + size_t elem_max_size = 0; + for ( const char * type = dt; *type != '\0'; type++ ) { + switch ( *type ) + { + case 'u': { elem_max_size = std::max( elem_max_size, sizeof(uchar ) ); break; } + case 'c': { elem_max_size = std::max( elem_max_size, sizeof(schar ) ); break; } + case 'w': { elem_max_size = std::max( elem_max_size, sizeof(ushort) ); break; } + case 's': { elem_max_size = std::max( elem_max_size, sizeof(short ) ); break; } + case 'i': { elem_max_size = std::max( elem_max_size, sizeof(int ) ); break; } + case 'f': { elem_max_size = std::max( elem_max_size, sizeof(float ) ); break; } + case 'd': { elem_max_size = std::max( elem_max_size, sizeof(double) ); break; } + default: break; + } + } + size = cvAlign( size, static_cast(elem_max_size) ); + return size; +} + +size_t base64::base64_encode_buffer_size(size_t cnt, bool is_end_with_zero) { + size_t additional = static_cast(is_end_with_zero == true); + return (cnt + 2U) / 3U * 4U + additional; +} + +base64::Base64Writer::Base64Writer(cv::FileStorage::Impl& fs, bool can_indent) + : emitter(new Base64ContextEmitter(fs, can_indent)) + , data_type_string() +{ + CV_Assert(fs.write_mode); +} + +void base64::Base64Writer::write(const void* _data, size_t len, const char* dt) +{ + check_dt(dt); + RawDataToBinaryConvertor convertor(_data, static_cast(len), data_type_string); + emitter->write(convertor); +} + +template inline +void base64::Base64Writer::write(_to_binary_convertor_t & convertor, const char* dt) +{ + check_dt(dt); + emitter->write(convertor); +} + +base64::Base64Writer::~Base64Writer() +{ + delete emitter; +} + +void base64::Base64Writer::check_dt(const char* dt) +{ + if ( dt == 0 ) + CV_Error( cv::Error::StsBadArg, "Invalid \'dt\'." ); + else if (data_type_string.empty()) { + data_type_string = dt; + + /* output header */ + std::string buffer = make_base64_header(dt); + const uchar * beg = reinterpret_cast(buffer.data()); + const uchar * end = beg + buffer.size(); + + emitter->write(beg, end); + } else if ( data_type_string != dt ) + CV_Error( cv::Error::StsBadArg, "\'dt\' does not match." ); +} + +base64::RawDataToBinaryConvertor::RawDataToBinaryConvertor(const void* src, int len, const std::string & dt) + : beg(reinterpret_cast(src)) + , cur(0) + , end(0) +{ + CV_Assert(src); + CV_Assert(!dt.empty()); + CV_Assert(len > 0); + + /* calc step and to_binary_funcs */ + step_packed = make_to_binary_funcs(dt); + + end = beg; + cur = beg; + + step = icvCalcStructSize(dt.c_str(), 0); + end = beg + static_cast(len); +} + +inline base64::RawDataToBinaryConvertor& base64::RawDataToBinaryConvertor::operator >>(uchar * & dst) +{ + CV_DbgAssert(*this); + + for (size_t i = 0U, n = to_binary_funcs.size(); i < n; i++) { + elem_to_binary_t & pack = to_binary_funcs[i]; + pack.func(cur + pack.offset, dst + pack.offset_packed); + } + cur += step; + dst += step_packed; + + return *this; +} + +inline base64::RawDataToBinaryConvertor::operator bool() const +{ + return cur < end; +} + +size_t base64::RawDataToBinaryConvertor::make_to_binary_funcs(const std::string &dt) +{ + size_t cnt = 0; + size_t offset = 0; + size_t offset_packed = 0; + char type = '\0'; + + std::istringstream iss(dt); + while (!iss.eof()) { + if (!(iss >> cnt)) { + iss.clear(); + cnt = 1; + } + CV_Assert(cnt > 0U); + if (!(iss >> type)) + break; + + while (cnt-- > 0) + { + elem_to_binary_t pack; + + size_t size = 0; + switch (type) + { + case 'u': + case 'c': + size = sizeof(uchar); + pack.func = to_binary; + break; + case 'w': + case 's': + size = sizeof(ushort); + pack.func = to_binary; + break; + case 'i': + size = sizeof(uint); + pack.func = to_binary; + break; + case 'f': + size = sizeof(float); + pack.func = to_binary; + break; + case 'd': + size = sizeof(double); + pack.func = to_binary; + break; + case 'r': + default: + CV_Error(cv::Error::StsError, "type is not supported"); + }; + + offset = static_cast(cvAlign(static_cast(offset), static_cast(size))); + pack.offset = offset; + offset += size; + + pack.offset_packed = offset_packed; + offset_packed += size; + + to_binary_funcs.push_back(pack); + } + } + + CV_Assert(iss.eof()); + return offset_packed; +} + +} \ No newline at end of file diff --git a/modules/core/src/persistence_base64_encoding.hpp b/modules/core/src/persistence_base64_encoding.hpp new file mode 100644 index 0000000000..1ee5201e14 --- /dev/null +++ b/modules/core/src/persistence_base64_encoding.hpp @@ -0,0 +1,127 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html + +#ifndef OPENCV_CORE_BASE64_ENCODING_HPP +#define OPENCV_CORE_BASE64_ENCODING_HPP + +namespace cv +{ + +namespace base64 +{ +/* A decorator for CvFileStorage +* - no copyable +* - not safe for now +* - move constructor may be needed if C++11 +*/ +uint8_t const base64_mapping[] = + "ABCDEFGHIJKLMNOPQRSTUVWXYZ" + "abcdefghijklmnopqrstuvwxyz" + "0123456789+/"; + +uint8_t const base64_padding = '='; + +std::string make_base64_header(const char * dt); + +size_t base64_encode(uint8_t const * src, uint8_t * dst, size_t off, size_t cnt); + + +int icvCalcStructSize( const char* dt, int initial_size ); + +class Base64ContextEmitter; +class Impl; + +class Base64Writer +{ +public: + Base64Writer(cv::FileStorage::Impl& fs, bool can_indent); + ~Base64Writer(); + void write(const void* _data, size_t len, const char* dt); + template void write(_to_binary_convertor_t & convertor, const char* dt); + +private: + void check_dt(const char* dt); + +private: + // disable copy and assignment + Base64Writer(const Base64Writer &); + Base64Writer & operator=(const Base64Writer &); + +private: + + Base64ContextEmitter * emitter; + std::string data_type_string; +}; + +size_t base64_encode_buffer_size(size_t cnt, bool is_end_with_zero = true); + +template inline size_t +to_binary(_uint_t val, uchar * cur) +{ + size_t delta = CHAR_BIT; + size_t cnt = sizeof(_uint_t); + while (cnt --> static_cast(0U)) { + *cur++ = static_cast(val); + val >>= delta; + } + return sizeof(_uint_t); +} + +template<> inline size_t to_binary(double val, uchar * cur) +{ + Cv64suf bit64; + bit64.f = val; + return to_binary(bit64.u, cur); +} + +template<> inline size_t to_binary(float val, uchar * cur) +{ + Cv32suf bit32; + bit32.f = val; + return to_binary(bit32.u, cur); +} + +template inline size_t +to_binary(uchar const * val, uchar * cur) +{ + return to_binary<_primitive_t>(*reinterpret_cast<_primitive_t const *>(val), cur); +} + + + +class RawDataToBinaryConvertor +{ +public: + // NOTE: len is already multiplied by element size here + RawDataToBinaryConvertor(const void* src, int len, const std::string & dt); + + inline RawDataToBinaryConvertor & operator >>(uchar * & dst); + inline operator bool() const; + +private: + typedef size_t(*to_binary_t)(const uchar *, uchar *); + struct elem_to_binary_t + { + size_t offset; + size_t offset_packed; + to_binary_t func; + }; + +private: + size_t make_to_binary_funcs(const std::string &dt); + +private: + const uchar * beg; + const uchar * cur; + const uchar * end; + + size_t step; + size_t step_packed; + std::vector to_binary_funcs; +}; + +} + +} +#endif \ No newline at end of file diff --git a/modules/core/src/persistence_impl.hpp b/modules/core/src/persistence_impl.hpp new file mode 100644 index 0000000000..4ea2dc3502 --- /dev/null +++ b/modules/core/src/persistence_impl.hpp @@ -0,0 +1,231 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html + +#ifndef OPENCV_CORE_PERSISTENCE_IMPL_HPP +#define OPENCV_CORE_PERSISTENCE_IMPL_HPP + +#include "persistence.hpp" +#include "persistence_base64_encoding.hpp" +#include +#include + + +namespace cv +{ + +enum Base64State{ + Uncertain, + NotUse, + InUse, +}; + +class cv::FileStorage::Impl : public FileStorage_API +{ +public: + void init(); + + Impl(FileStorage* _fs); + + virtual ~Impl(); + + void release(String* out=0); + + void analyze_file_name( const std::string& file_name, std::vector& params ); + + bool open( const char* filename_or_buf, int _flags, const char* encoding ); + + void puts( const char* str ); + + char* getsFromFile( char* buf, int count ); + + char* gets( size_t maxCount ); + + char* gets(); + + bool eof(); + + void setEof(); + + void closeFile(); + + void rewind(); + + char* resizeWriteBuffer( char* ptr, int len ); + + char* flush(); + + void endWriteStruct(); + + void startWriteStruct_helper( const char* key, int struct_flags, + const char* type_name ); + + void startWriteStruct( const char* key, int struct_flags, + const char* type_name ); + + void writeComment( const char* comment, bool eol_comment ); + + void startNextStream(); + + void write( const String& key, int value ); + + void write( const String& key, double value ); + + void write( const String& key, const String& value ); + + void writeRawData( const std::string& dt, const void* _data, size_t len ); + + void workaround(); + + void switch_to_Base64_state( FileStorage_API::Base64State new_state); + + void make_write_struct_delayed( const char* key, int struct_flags, const char* type_name ); + + void check_if_write_struct_is_delayed( bool change_type_to_base64 ); + + void writeRawDataBase64(const void* _data, size_t len, const char* dt ); + + String releaseAndGetString(); + + FileNode getFirstTopLevelNode() const; + + FileNode root(int streamIdx=0) const; + + FileNode operator[](const String& nodename) const; + + FileNode operator[](const char* /*nodename*/) const; + + int getFormat() const; + + char* bufferPtr() const; + char* bufferStart() const; + char* bufferEnd() const; + void setBufferPtr(char* ptr); + int wrapMargin() const; + + FStructData& getCurrentStruct(); + + void setNonEmpty(); + + void processSpecialDouble( char* buf, double* value, char** endptr ); + + double strtod( char* ptr, char** endptr ); + + void convertToCollection(int type, FileNode& node); + + // a) allocates new FileNode (for that just set blockIdx to the last block and ofs to freeSpaceOfs) or + // b) reallocates just created new node (blockIdx and ofs must be taken from FileNode). + // If there is no enough space in the current block (it should be the last block added so far), + // the last block is shrunk so that it ends immediately before the reallocated node. Then, + // a new block of sufficient size is allocated and the FileNode is placed in the beginning of it. + // The case (a) can be used to allocate the very first node by setting blockIdx == ofs == 0. + // In the case (b) the existing tag and the name are copied automatically. + uchar* reserveNodeSpace(FileNode& node, size_t sz); + + unsigned getStringOfs( const std::string& key ) const; + + FileNode addNode( FileNode& collection, const std::string& key, + int elem_type, const void* value, int len ); + + void finalizeCollection( FileNode& collection ); + + void normalizeNodeOfs(size_t& blockIdx, size_t& ofs) const; + + Base64State get_state_of_writing_base64(); + + int get_space(); + + class Base64Decoder + { + public: + Base64Decoder(); + void init(Ptr& _parser, char* _ptr, int _indent); + + bool readMore(int needed); + + uchar getUInt8(); + + ushort getUInt16(); + + int getInt32(); + + double getFloat64(); + + bool endOfStream() const; + char* getPtr() const; + protected: + + Ptr parser; + char* ptr; + int indent; + std::vector encoded; + std::vector decoded; + size_t ofs; + size_t totalchars; + bool eos; + }; + + char* parseBase64(char* ptr, int indent, FileNode& collection); + + void parseError( const char* func_name, const std::string& err_msg, const char* source_file, int source_line ); + + const uchar* getNodePtr(size_t blockIdx, size_t ofs) const; + + std::string getName( size_t nameofs ) const; + + FileStorage* getFS(); + + FileStorage* fs_ext; + + std::string filename; + int flags; + bool empty_stream; + + FILE* file; + gzFile gzfile; + + bool is_opened; + bool dummy_eof; + bool write_mode; + bool mem_mode; + int fmt; + + State state; //!< current state of the FileStorage (used only for writing) + bool is_using_base64; + bool is_write_struct_delayed; + char* delayed_struct_key; + int delayed_struct_flags; + char* delayed_type_name; + FileStorage_API::Base64State state_of_writing_base64; + + int space, wrap_margin; + std::deque write_stack; + std::vector buffer; + size_t bufofs; + + std::deque outbuf; + + Ptr emitter; + Ptr parser; + Base64Decoder base64decoder; + base64::Base64Writer* base64_writer; + + std::vector roots; + std::vector > > fs_data; + std::vector fs_data_ptrs; + std::vector fs_data_blksz; + size_t freeSpaceOfs; + typedef std::unordered_map str_hash_t; + str_hash_t str_hash; + std::vector str_hash_data; + + std::vector strbufv; + char* strbuf; + size_t strbufsize; + size_t strbufpos; + int lineno; +}; + +} + +#endif \ No newline at end of file diff --git a/modules/core/src/persistence_json.cpp b/modules/core/src/persistence_json.cpp index 667895fbc5..12a58e80bf 100644 --- a/modules/core/src/persistence_json.cpp +++ b/modules/core/src/persistence_json.cpp @@ -23,7 +23,7 @@ public: struct_flags = (struct_flags & (FileNode::TYPE_MASK|FileNode::FLOW)) | FileNode::EMPTY; if( !FileNode::isCollection(struct_flags)) - CV_Error( CV_StsBadArg, + CV_Error( cv::Error::StsBadArg, "Some collection type - FileNode::SEQ or FileNode::MAP, must be specified" ); if( type_name && *type_name == '\0' ) @@ -53,29 +53,26 @@ public: void endWriteStruct(const FStructData& current_struct) { int struct_flags = current_struct.flags; - CV_Assert( FileNode::isCollection(struct_flags) ); - if( !FileNode::isFlow(struct_flags) ) - { -#if 0 - if ( fs->bufferPtr() <= fs->bufferStart() + fs->space ) - { - /* some bad code for base64_writer... */ - ptr = fs->bufferPtr(); - *ptr++ = '\n'; - *ptr++ = '\0'; - fs->puts( fs->bufferStart() ); - fs->setBufferPtr(fs->bufferStart()); + if (FileNode::isCollection(struct_flags)) { + if (!FileNode::isFlow(struct_flags)) { + if (fs->bufferPtr() <= fs->bufferStart() + fs->get_space()) { + /* some bad code for base64_writer... */ + char *ptr = fs->bufferPtr(); + *ptr++ = '\n'; + *ptr++ = '\0'; + fs->puts(fs->bufferStart()); + fs->setBufferPtr(fs->bufferStart()); + } + fs->flush(); } -#endif - fs->flush(); - } - char* ptr = fs->bufferPtr(); - if( ptr > fs->bufferStart() + current_struct.indent && !FileNode::isEmptyCollection(struct_flags) ) - *ptr++ = ' '; - *ptr++ = FileNode::isMap(struct_flags) ? '}' : ']'; - fs->setBufferPtr(ptr); + char *ptr = fs->bufferPtr(); + if (ptr > fs->bufferStart() + current_struct.indent && !FileNode::isEmptyCollection(struct_flags)) + *ptr++ = ' '; + *ptr++ = FileNode::isMap(struct_flags) ? '}' : ']'; + fs->setBufferPtr(ptr); + } } void write(const char* key, int value) @@ -97,11 +94,11 @@ public: int i, len; if( !str ) - CV_Error( CV_StsNullPtr, "Null string pointer" ); + CV_Error( cv::Error::StsNullPtr, "Null string pointer" ); len = (int)strlen(str); if( len > CV_FS_MAX_LEN ) - CV_Error( CV_StsBadArg, "The written string is too long" ); + CV_Error( cv::Error::StsBadArg, "The written string is too long" ); if( quote || len == 0 || str[0] != str[len-1] || (str[0] != '\"' && str[0] != '\'') ) { @@ -136,6 +133,20 @@ public: void writeScalar(const char* key, const char* data) { + /* check write_struct */ + + fs->check_if_write_struct_is_delayed(false); + if ( fs->get_state_of_writing_base64() == FileStorage_API::Uncertain ) + { + fs->switch_to_Base64_state( FileStorage_API::NotUse ); + } + else if ( fs->get_state_of_writing_base64() == FileStorage_API::InUse ) + { + CV_Error( cv::Error::StsError, "At present, output Base64 data only." ); + } + + /* check parameters */ + size_t key_len = 0u; if( key && *key == '\0' ) key = 0; @@ -143,9 +154,9 @@ public: { key_len = strlen(key); if ( key_len == 0u ) - CV_Error( CV_StsBadArg, "The key is an empty" ); + CV_Error( cv::Error::StsBadArg, "The key is an empty" ); else if ( static_cast(key_len) > CV_FS_MAX_LEN ) - CV_Error( CV_StsBadArg, "The key is too long" ); + CV_Error( cv::Error::StsBadArg, "The key is too long" ); } size_t data_len = 0u; @@ -157,7 +168,7 @@ public: if( FileNode::isCollection(struct_flags) ) { if ( (FileNode::isMap(struct_flags) ^ (key != 0)) ) - CV_Error( CV_StsBadArg, "An attempt to add element without a key to a map, " + CV_Error( cv::Error::StsBadArg, "An attempt to add element without a key to a map, " "or add element with key to sequence" ); } else { fs->setNonEmpty(); @@ -199,7 +210,7 @@ public: if( key ) { if( !cv_isalpha(key[0]) && key[0] != '_' ) - CV_Error( CV_StsBadArg, "Key must start with a letter or _" ); + CV_Error( cv::Error::StsBadArg, "Key must start with a letter or _" ); ptr = fs->resizeWriteBuffer( ptr, static_cast(key_len) ); *ptr++ = '\"'; @@ -210,7 +221,7 @@ public: ptr[i] = c; if( !cv_isalnum(c) && c != '-' && c != '_' && c != ' ' ) - CV_Error( CV_StsBadArg, "Key names may only contain alphanumeric characters [a-zA-Z0-9], '-', '_' and ' '" ); + CV_Error( cv::Error::StsBadArg, "Key names may only contain alphanumeric characters [a-zA-Z0-9], '-', '_' and ' '" ); } ptr += key_len; @@ -233,7 +244,7 @@ public: void writeComment(const char* comment, bool eol_comment) { if( !comment ) - CV_Error( CV_StsNullPtr, "Null comment" ); + CV_Error( cv::Error::StsNullPtr, "Null comment" ); int len = static_cast(strlen(comment)); char* ptr = fs->bufferPtr(); diff --git a/modules/core/src/persistence_xml.cpp b/modules/core/src/persistence_xml.cpp index 52b5374425..62b7b1eb59 100644 --- a/modules/core/src/persistence_xml.cpp +++ b/modules/core/src/persistence_xml.cpp @@ -45,7 +45,7 @@ public: if( FileNode::isCollection(struct_flags) ) { if( FileNode::isMap(struct_flags) ^ (key != 0) ) - CV_Error( CV_StsBadArg, "An attempt to add element without a key to a map, " + CV_Error( cv::Error::StsBadArg, "An attempt to add element without a key to a map, " "or add element with key to sequence" ); } else @@ -61,26 +61,26 @@ public: if( !key ) key = "_"; else if( key[0] == '_' && key[1] == '\0' ) - CV_Error( CV_StsBadArg, "A single _ is a reserved tag name" ); + CV_Error( cv::Error::StsBadArg, "A single _ is a reserved tag name" ); len = (int)strlen( key ); *ptr++ = '<'; if( tag_type == CV_XML_CLOSING_TAG ) { if( !attrlist.empty() ) - CV_Error( CV_StsBadArg, "Closing tag should not include any attributes" ); + CV_Error( cv::Error::StsBadArg, "Closing tag should not include any attributes" ); *ptr++ = '/'; } if( !cv_isalpha(key[0]) && key[0] != '_' ) - CV_Error( CV_StsBadArg, "Key should start with a letter or _" ); + CV_Error( cv::Error::StsBadArg, "Key should start with a letter or _" ); ptr = fs->resizeWriteBuffer( ptr, len ); for( i = 0; i < len; i++ ) { char c = key[i]; if( !cv_isalnum(c) && c != '_' && c != '-' ) - CV_Error( CV_StsBadArg, "Key name may only contain alphanumeric characters [a-zA-Z0-9], '-' and '_'" ); + CV_Error( cv::Error::StsBadArg, "Key name may only contain alphanumeric characters [a-zA-Z0-9], '-' and '_'" ); ptr[i] = c; } ptr += len; @@ -158,11 +158,11 @@ public: int i, len; if( !str ) - CV_Error( CV_StsNullPtr, "Null string pointer" ); + CV_Error( cv::Error::StsNullPtr, "Null string pointer" ); len = (int)strlen(str); if( len > CV_FS_MAX_LEN ) - CV_Error( CV_StsBadArg, "The written string is too long" ); + CV_Error( cv::Error::StsBadArg, "The written string is too long" ); if( quote || len == 0 || str[0] != '\"' || str[0] != str[len-1] ) { @@ -233,6 +233,16 @@ public: void writeScalar(const char* key, const char* data) { + fs->check_if_write_struct_is_delayed(false); + if ( fs->get_state_of_writing_base64() == FileStorage_API::Uncertain ) + { + fs->switch_to_Base64_state( FileStorage_API::NotUse ); + } + else if ( fs->get_state_of_writing_base64() == FileStorage_API::InUse ) + { + CV_Error( cv::Error::StsError, "At present, output Base64 data only." ); + } + int len = (int)strlen(data); if( key && *key == '\0' ) key = 0; @@ -255,7 +265,7 @@ public: int new_offset = (int)(ptr - fs->bufferStart()) + len; if( key ) - CV_Error( CV_StsBadArg, "elements with keys can not be written to sequence" ); + CV_Error( cv::Error::StsBadArg, "elements with keys can not be written to sequence" ); current_struct.flags = FileNode::SEQ; @@ -281,10 +291,10 @@ public: char* ptr; if( !comment ) - CV_Error( CV_StsNullPtr, "Null comment" ); + CV_Error( cv::Error::StsNullPtr, "Null comment" ); if( strstr(comment, "--") != 0 ) - CV_Error( CV_StsBadArg, "Double hyphen \'--\' is not allowed in the comments" ); + CV_Error( cv::Error::StsBadArg, "Double hyphen \'--\' is not allowed in the comments" ); len = (int)strlen(comment); eol = strchr(comment, '\n'); diff --git a/modules/core/src/persistence_yml.cpp b/modules/core/src/persistence_yml.cpp index 3f3742b8d1..95db1450c6 100644 --- a/modules/core/src/persistence_yml.cpp +++ b/modules/core/src/persistence_yml.cpp @@ -33,7 +33,7 @@ public: struct_flags = (struct_flags & (FileNode::TYPE_MASK|FileNode::FLOW)) | FileNode::EMPTY; if( !FileNode::isCollection(struct_flags)) - CV_Error( CV_StsBadArg, + CV_Error( cv::Error::StsBadArg, "Some collection type - FileNode::SEQ or FileNode::MAP, must be specified" ); if (type_name && memcmp(type_name, "binary", 6) == 0) @@ -120,11 +120,11 @@ public: int i, len; if( !str ) - CV_Error( CV_StsNullPtr, "Null string pointer" ); + CV_Error( cv::Error::StsNullPtr, "Null string pointer" ); len = (int)strlen(str); if( len > CV_FS_MAX_LEN ) - CV_Error( CV_StsBadArg, "The written string is too long" ); + CV_Error( cv::Error::StsBadArg, "The written string is too long" ); if( quote || len == 0 || str[0] != str[len-1] || (str[0] != '\"' && str[0] != '\'') ) { @@ -174,6 +174,16 @@ public: void writeScalar(const char* key, const char* data) { + fs->check_if_write_struct_is_delayed(false); + if ( fs->get_state_of_writing_base64() == FileStorage_API::Uncertain ) + { + fs->switch_to_Base64_state( FileStorage_API::NotUse ); + } + else if ( fs->get_state_of_writing_base64() == FileStorage_API::InUse ) + { + CV_Error( cv::Error::StsError, "At present, output Base64 data only." ); + } + int i, keylen = 0; int datalen = 0; char* ptr; @@ -188,7 +198,7 @@ public: if( FileNode::isCollection(struct_flags) ) { if( (FileNode::isMap(struct_flags) ^ (key != 0)) ) - CV_Error( CV_StsBadArg, "An attempt to add element without a key to a map, " + CV_Error( cv::Error::StsBadArg, "An attempt to add element without a key to a map, " "or add element with key to sequence" ); } else @@ -201,10 +211,10 @@ public: { keylen = (int)strlen(key); if( keylen == 0 ) - CV_Error( CV_StsBadArg, "The key is an empty" ); + CV_Error( cv::Error::StsBadArg, "The key is an empty" ); if( keylen > CV_FS_MAX_LEN ) - CV_Error( CV_StsBadArg, "The key is too long" ); + CV_Error( cv::Error::StsBadArg, "The key is too long" ); } if( data ) @@ -238,7 +248,7 @@ public: if( key ) { if( !cv_isalpha(key[0]) && key[0] != '_' ) - CV_Error( CV_StsBadArg, "Key must start with a letter or _" ); + CV_Error( cv::Error::StsBadArg, "Key must start with a letter or _" ); ptr = fs->resizeWriteBuffer( ptr, keylen ); @@ -248,7 +258,7 @@ public: ptr[i] = c; if( !cv_isalnum(c) && c != '-' && c != '_' && c != ' ' ) - CV_Error( CV_StsBadArg, "Key names may only contain alphanumeric characters [a-zA-Z0-9], '-', '_' and ' '" ); + CV_Error( cv::Error::StsBadArg, "Key names may only contain alphanumeric characters [a-zA-Z0-9], '-', '_' and ' '" ); } ptr += keylen; @@ -271,7 +281,7 @@ public: void writeComment(const char* comment, bool eol_comment) { if( !comment ) - CV_Error( CV_StsNullPtr, "Null comment" ); + CV_Error( cv::Error::StsNullPtr, "Null comment" ); int len = (int)strlen(comment); const char* eol = strchr(comment, '\n'); diff --git a/modules/core/test/test_io.cpp b/modules/core/test/test_io.cpp index 82bd05372d..3712be9f2e 100644 --- a/modules/core/test/test_io.cpp +++ b/modules/core/test/test_io.cpp @@ -586,6 +586,7 @@ static void test_filestorage_basic(int write_flags, const char* suffix_name, boo const ::testing::TestInfo* const test_info = ::testing::UnitTest::GetInstance()->current_test_info(); CV_Assert(test_info); std::string name = (std::string(test_info->test_case_name()) + "--" + test_info->name() + suffix_name); + std::string name_34 = string(cvtest::TS::ptr()->get_data_path()) + "io/3_4/" + name; if (!testReadWrite) name = string(cvtest::TS::ptr()->get_data_path()) + "io/" + name; @@ -661,7 +662,23 @@ static void test_filestorage_basic(int write_flags, const char* suffix_name, boo std::ifstream f(name.c_str(), std::ios::in|std::ios::binary); f.seekg(0, std::fstream::end); sz = (size_t)f.tellg(); + + f.seekg(0, std::ios::beg); + std::vector test_data(sz); + f.read(&test_data[0], sz); f.close(); + + std::ifstream reference(name_34.c_str(), std::ios::in|std::ios::binary); + ASSERT_TRUE(reference.is_open()); + reference.seekg(0, std::fstream::end); + size_t ref_sz = (size_t)reference.tellg(); + + reference.seekg(0, std::ios::beg); + std::vector reference_data(ref_sz); + reference.read(&reference_data[0], ref_sz); + reference.close(); + + EXPECT_EQ(reference_data, test_data); } std::cout << "Storage size: " << sz << std::endl; EXPECT_LE(sz, (size_t)6000); @@ -757,27 +774,27 @@ TEST(Core_InputOutput, filestorage_base64_basic_read_JSON) { test_filestorage_basic(cv::FileStorage::WRITE_BASE64, ".json", false); } -TEST(Core_InputOutput, DISABLED_filestorage_base64_basic_rw_XML) +TEST(Core_InputOutput, filestorage_base64_basic_rw_XML) { test_filestorage_basic(cv::FileStorage::WRITE_BASE64, ".xml", true); } -TEST(Core_InputOutput, DISABLED_filestorage_base64_basic_rw_YAML) +TEST(Core_InputOutput, filestorage_base64_basic_rw_YAML) { test_filestorage_basic(cv::FileStorage::WRITE_BASE64, ".yml", true); } -TEST(Core_InputOutput, DISABLED_filestorage_base64_basic_rw_JSON) +TEST(Core_InputOutput, filestorage_base64_basic_rw_JSON) { test_filestorage_basic(cv::FileStorage::WRITE_BASE64, ".json", true); } -TEST(Core_InputOutput, DISABLED_filestorage_base64_basic_memory_XML) +TEST(Core_InputOutput, filestorage_base64_basic_memory_XML) { test_filestorage_basic(cv::FileStorage::WRITE_BASE64, ".xml", true, true); } -TEST(Core_InputOutput, DISABLED_filestorage_base64_basic_memory_YAML) +TEST(Core_InputOutput, filestorage_base64_basic_memory_YAML) { test_filestorage_basic(cv::FileStorage::WRITE_BASE64, ".yml", true, true); } -TEST(Core_InputOutput, DISABLED_filestorage_base64_basic_memory_JSON) +TEST(Core_InputOutput, filestorage_base64_basic_memory_JSON) { test_filestorage_basic(cv::FileStorage::WRITE_BASE64, ".json", true, true); } diff --git a/modules/python/test/test_filestorage_io.py b/modules/python/test/test_filestorage_io.py index 62b540d79c..01e0a72300 100755 --- a/modules/python/test/test_filestorage_io.py +++ b/modules/python/test/test_filestorage_io.py @@ -1,6 +1,8 @@ #!/usr/bin/env python """Algorithm serialization test.""" from __future__ import print_function +import base64 +import json import tempfile import os import cv2 as cv @@ -109,5 +111,96 @@ class filestorage_io_test(NewOpenCVTests): def test_json(self): self.run_fs_test(".json") + def test_base64(self): + fd, fname = tempfile.mkstemp(prefix="opencv_python_sample_filestorage_base64", suffix=".json") + os.close(fd) + np.random.seed(42) + self.write_base64_json(fname) + os.remove(fname) + + @staticmethod + def get_normal_2d_mat(): + rows = 10 + cols = 20 + cn = 3 + + image = np.zeros((rows, cols, cn), np.uint8) + image[:] = (1, 2, 127) + + for i in range(rows): + for j in range(cols): + image[i, j, 1] = (i + j) % 256 + + return image + + @staticmethod + def get_normal_nd_mat(): + shape = (2, 2, 1, 2) + cn = 4 + + image = np.zeros(shape + (cn,), np.float64) + image[:] = (0.888, 0.111, 0.666, 0.444) + + return image + + @staticmethod + def get_empty_2d_mat(): + shape = (0, 0) + cn = 1 + + image = np.zeros(shape + (cn,), np.uint8) + + return image + + @staticmethod + def get_random_mat(): + rows = 8 + cols = 16 + cn = 1 + + image = np.random.rand(rows, cols, cn) + + return image + + @staticmethod + def decode(data): + # strip $base64$ + encoded = data[8:] + + if len(encoded) == 0: + return b'' + + # strip info about datatype and padding + return base64.b64decode(encoded)[24:] + + def write_base64_json(self, fname): + fs = cv.FileStorage(fname, cv.FileStorage_WRITE_BASE64) + + mats = {'normal_2d_mat': self.get_normal_2d_mat(), + 'normal_nd_mat': self.get_normal_nd_mat(), + 'empty_2d_mat': self.get_empty_2d_mat(), + 'random_mat': self.get_random_mat()} + + for name, mat in mats.items(): + fs.write(name, mat) + + fs.release() + + data = {} + with open(fname) as file: + data = json.load(file) + + for name, mat in mats.items(): + buffer = b'' + + if mat.size != 0: + if hasattr(mat, 'tobytes'): + buffer = mat.tobytes() + else: + buffer = mat.tostring() + + self.assertEqual(buffer, self.decode(data[name]['data'])) + + if __name__ == '__main__': NewOpenCVTests.bootstrap() From b928ebdd53b3db10864ff675461c44f04eef3e8c Mon Sep 17 00:00:00 2001 From: Francesco Petrogalli <25690309+fpetrogalli@users.noreply.github.com> Date: Thu, 8 Jul 2021 21:21:21 +0100 Subject: [PATCH 055/262] Merge pull request #19985 from fpetrogalli:disable_threads * [build][option] Introduce `OPENCV_DISABLE_THREAD_SUPPORT` option. The option forces the library to build without thread support. * update handling of OPENCV_DISABLE_THREAD_SUPPORT - reduce amount of #if conditions * [to squash] cmake: apply mode vars in toolchains too Co-authored-by: Alexander Alekhin --- 3rdparty/libwebp/CMakeLists.txt | 4 +- CMakeLists.txt | 15 ++ cmake/OpenCVCompilerOptions.cmake | 11 +- cmake/OpenCVUtils.cmake | 6 + cmake/vars/EnableModeVars.cmake | 18 ++ .../vars/OPENCV_DISABLE_THREAD_SUPPORT.cmake | 28 +++ modules/core/CMakeLists.txt | 4 + modules/core/include/opencv2/core/utility.hpp | 20 ++- modules/core/src/async.cpp | 166 ++++++++++++++++++ modules/core/src/parallel.cpp | 10 +- modules/core/src/system.cpp | 112 ++++++++++++ modules/core/src/umatrix.cpp | 37 +++- modules/core/src/utils/logtagmanager.hpp | 4 +- modules/core/test/test_async.cpp | 5 +- modules/core/test/test_utils.cpp | 5 +- modules/ts/CMakeLists.txt | 6 + 16 files changed, 435 insertions(+), 16 deletions(-) create mode 100644 cmake/vars/EnableModeVars.cmake create mode 100644 cmake/vars/OPENCV_DISABLE_THREAD_SUPPORT.cmake diff --git a/3rdparty/libwebp/CMakeLists.txt b/3rdparty/libwebp/CMakeLists.txt index 80ab0b86ab..9160e2024c 100644 --- a/3rdparty/libwebp/CMakeLists.txt +++ b/3rdparty/libwebp/CMakeLists.txt @@ -32,7 +32,9 @@ endif() # Define the library target: # ---------------------------------------------------------------------------------- -add_definitions(-DWEBP_USE_THREAD) +if(NOT OPENCV_DISABLE_THREAD_SUPPORT) + add_definitions(-DWEBP_USE_THREAD) +endif() add_library(${WEBP_LIBRARY} STATIC ${OPENCV_3RDPARTY_EXCLUDE_FROM_ALL} ${lib_srcs} ${lib_hdrs}) if(ANDROID) diff --git a/CMakeLists.txt b/CMakeLists.txt index 49abe017a5..dd862bb154 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -512,6 +512,7 @@ OCV_OPTION(OPENCV_GENERATE_SETUPVARS "Generate setup_vars* scripts" ON IF (NOT OCV_OPTION(ENABLE_CONFIG_VERIFICATION "Fail build if actual configuration doesn't match requested (WITH_XXX != HAVE_XXX)" OFF) OCV_OPTION(OPENCV_ENABLE_MEMALIGN "Enable posix_memalign or memalign usage" ON) OCV_OPTION(OPENCV_DISABLE_FILESYSTEM_SUPPORT "Disable filesystem support" OFF) +OCV_OPTION(OPENCV_DISABLE_THREAD_SUPPORT "Build the library without multi-threaded code." OFF) OCV_OPTION(ENABLE_PYLINT "Add target with Pylint checks" (BUILD_DOCS OR BUILD_EXAMPLES) IF (NOT CMAKE_CROSSCOMPILING AND NOT APPLE_FRAMEWORK) ) OCV_OPTION(ENABLE_FLAKE8 "Add target with Python flake8 checker" (BUILD_DOCS OR BUILD_EXAMPLES) IF (NOT CMAKE_CROSSCOMPILING AND NOT APPLE_FRAMEWORK) ) @@ -666,6 +667,11 @@ if(UNIX) set(HAVE_PTHREAD 1) endif() + # Ensure that libpthread is not listed as one of the libraries to pass to the linker. + if (OPENCV_DISABLE_THREAD_SUPPORT) + list(REMOVE_ITEM OPENCV_LINKER_LIBS pthread) + endif() + if(OPENCV_ENABLE_MEMALIGN) CHECK_SYMBOL_EXISTS(posix_memalign stdlib.h HAVE_POSIX_MEMALIGN) CHECK_INCLUDE_FILE(malloc.h HAVE_MALLOC_H) @@ -1459,6 +1465,15 @@ ocv_build_features_string(parallel_status EXCLUSIVE ELSE "none") status("") status(" Parallel framework:" "${parallel_status}") +if (OPENCV_DISABLE_THREAD_SUPPORT) + status("" "Multi thread code explicitly disabled with OPENCV_DISABLE_THREAD_SUPPORT.") + if(HAVE_PTHREADS_PF OR HAVE_HPX OR HAVE_OPENMP OR HAVE_GCD OR HAVE_CONCURRENCY) + message(FATAL_ERROR "Not all parallel frameworks have been disabled (using ${parallel_status}).") + endif() + if(HAVE_PTHREAD) + message(FATAL_ERROR "Thread execution might be in use in some component.") + endif() +endif() if(CV_TRACE OR OPENCV_TRACE) ocv_build_features_string(trace_status EXCLUSIVE diff --git a/cmake/OpenCVCompilerOptions.cmake b/cmake/OpenCVCompilerOptions.cmake index 6e56a2e34a..2917dd33d5 100644 --- a/cmake/OpenCVCompilerOptions.cmake +++ b/cmake/OpenCVCompilerOptions.cmake @@ -178,14 +178,17 @@ if(CV_GCC OR CV_CLANG) add_extra_compiler_option(-Wno-long-long) endif() - # We need pthread's - if((UNIX + # We need pthread's, unless we have explicitly disabled multi-thread execution. + if(NOT OPENCV_DISABLE_THREAD_SUPPORT + AND ( + (UNIX AND NOT ANDROID AND NOT (APPLE AND CV_CLANG) AND NOT EMSCRIPTEN + ) + OR (EMSCRIPTEN AND WITH_PTHREADS_PF) # https://github.com/opencv/opencv/issues/20285 ) - OR (EMSCRIPTEN AND WITH_PTHREADS_PF) # https://github.com/opencv/opencv/issues/20285 - ) + ) # TODO add_extra_compiler_option(-pthread) endif() diff --git a/cmake/OpenCVUtils.cmake b/cmake/OpenCVUtils.cmake index 252078bdf7..39445150a9 100644 --- a/cmake/OpenCVUtils.cmake +++ b/cmake/OpenCVUtils.cmake @@ -1973,3 +1973,9 @@ if(NOT BUILD_SHARED_LIBS AND (CMAKE_VERSION VERSION_LESS "3.14.0")) else() ocv_update(OPENCV_3RDPARTY_EXCLUDE_FROM_ALL "EXCLUDE_FROM_ALL") endif() + + +# +# Include configuration override settings +# +include(cmake/vars/EnableModeVars.cmake) diff --git a/cmake/vars/EnableModeVars.cmake b/cmake/vars/EnableModeVars.cmake new file mode 100644 index 0000000000..b3c4e79c46 --- /dev/null +++ b/cmake/vars/EnableModeVars.cmake @@ -0,0 +1,18 @@ +set(__OCV_MODE_VARS_DIR "${CMAKE_CURRENT_LIST_DIR}") + +macro(ocv_change_mode_var) + set(__var "${ARGV0}") + set(__mode "${ARGV1}") + set(__value "${ARGV2}") + if(__mode STREQUAL "MODIFIED_ACCESS" AND __value) + if(NOT __applied_mode_${__var}) + include("${__OCV_MODE_VARS_DIR}/${__var}.cmake") + set(__applied_mode_${__var} 1) + else() + #message("Mode is already applied: ${__var}") + endif() + endif() +endmacro() + +variable_watch(OPENCV_DISABLE_THREAD_SUPPORT ocv_change_mode_var) +set(OPENCV_DISABLE_THREAD_SUPPORT "${OPENCV_DISABLE_THREAD_SUPPORT}") diff --git a/cmake/vars/OPENCV_DISABLE_THREAD_SUPPORT.cmake b/cmake/vars/OPENCV_DISABLE_THREAD_SUPPORT.cmake new file mode 100644 index 0000000000..5f5fc0204d --- /dev/null +++ b/cmake/vars/OPENCV_DISABLE_THREAD_SUPPORT.cmake @@ -0,0 +1,28 @@ +# Force removal of code conditionally compiled with `#if +# HAVE_PTHREAD`. +ocv_update(HAVE_PTHREAD 0) + +# There components are disabled because they require +# multi-threaded execution. +ocv_update(WITH_PROTOBUF OFF) +ocv_update(WITH_GSTREAMER OFF) +ocv_update(WITH_IPP OFF) +ocv_update(WITH_ITT OFF) +ocv_update(WITH_OPENCL OFF) +ocv_update(WITH_VA OFF) +ocv_update(WITH_VA_INTEL OFF) + +# Disable bindings +ocv_update(BUILD_opencv_python2 OFF) +ocv_update(BUILD_opencv_python3 OFF) +ocv_update(BUILD_JAVA OFF) +ocv_update(BUILD_opencv_java OFF) + +# These modules require `#include +# <[thread|mutex|condition_variable|future]>` and linkage into +# `libpthread` to work. +ocv_update(BUILD_opencv_objdetect OFF) +ocv_update(BUILD_opencv_gapi OFF) +ocv_update(BUILD_opencv_dnn OFF) + +set(OPJ_USE_THREAD "OFF" CACHE INTERNAL "") diff --git a/modules/core/CMakeLists.txt b/modules/core/CMakeLists.txt index b2797ab31f..6a969e5fc3 100644 --- a/modules/core/CMakeLists.txt +++ b/modules/core/CMakeLists.txt @@ -153,6 +153,10 @@ if(OPENCV_CORE_EXCLUDE_C_API) ocv_target_compile_definitions(${the_module} PRIVATE "OPENCV_EXCLUDE_C_API=1") endif() +if(OPENCV_DISABLE_THREAD_SUPPORT) + ocv_target_compile_definitions(${the_module} PUBLIC "OPENCV_DISABLE_THREAD_SUPPORT=1") +endif() + if(HAVE_HPX) ocv_target_link_libraries(${the_module} LINK_PRIVATE "${HPX_LIBRARIES}") endif() diff --git a/modules/core/include/opencv2/core/utility.hpp b/modules/core/include/opencv2/core/utility.hpp index f0368027aa..108c0d93e7 100644 --- a/modules/core/include/opencv2/core/utility.hpp +++ b/modules/core/include/opencv2/core/utility.hpp @@ -714,9 +714,27 @@ void Mat::forEach_impl(const Functor& operation) { /////////////////////////// Synchronization Primitives /////////////////////////////// #if !defined(_M_CEE) +#ifndef OPENCV_DISABLE_THREAD_SUPPORT typedef std::recursive_mutex Mutex; typedef std::lock_guard AutoLock; -#endif +#else // OPENCV_DISABLE_THREAD_SUPPORT +// Custom (failing) implementation of `std::recursive_mutex`. +struct Mutex { + void lock(){ + CV_Error(cv::Error::StsNotImplemented, + "cv::Mutex is disabled by OPENCV_DISABLE_THREAD_SUPPORT=ON"); + } + void unlock(){ + CV_Error(cv::Error::StsNotImplemented, + "cv::Mutex is disabled by OPENCV_DISABLE_THREAD_SUPPORT=ON"); + } +}; +// Stub for cv::AutoLock when threads are disabled. +struct AutoLock { + AutoLock(Mutex &) { } +}; +#endif // OPENCV_DISABLE_THREAD_SUPPORT +#endif // !defined(_M_CEE) /** @brief Designed for command line parsing diff --git a/modules/core/src/async.cpp b/modules/core/src/async.cpp index a2f4612365..78c0a1ee81 100644 --- a/modules/core/src/async.cpp +++ b/modules/core/src/async.cpp @@ -14,6 +14,7 @@ #define CV_LOG_STRIP_LEVEL CV_LOG_LEVEL_DEBUG + 1 #include +#ifndef OPENCV_DISABLE_THREAD_SUPPORT #ifdef CV_CXX11 #include @@ -236,6 +237,171 @@ struct AsyncArray::Impl } }; +} // namespace + +#else // OPENCV_DISABLE_THREAD_SUPPORT + +namespace cv { + +// no threading +struct AsyncArray::Impl +{ + int refcount; + void addrefFuture() CV_NOEXCEPT { refcount_future++; refcount++; } + void releaseFuture() CV_NOEXCEPT { refcount_future--; if (0 == --refcount) delete this; } + int refcount_future; + void addrefPromise() CV_NOEXCEPT { refcount_promise++; refcount++; } \ + void releasePromise() CV_NOEXCEPT { refcount_promise--; if (0 == --refcount) delete this; } + int refcount_promise; + + mutable bool has_result; // Mat, UMat or exception + + mutable cv::Ptr result_mat; + mutable cv::Ptr result_umat; + + + bool has_exception; +#if CV__EXCEPTION_PTR + std::exception_ptr exception; +#endif + cv::Exception cv_exception; + + mutable bool result_is_fetched; + + bool future_is_returned; + + Impl() + : refcount(1), refcount_future(0), refcount_promise(1) + , has_result(false) + , has_exception(false) + , result_is_fetched(false) + , future_is_returned(false) + { + // nothing + } + + ~Impl() + { + if (has_result && !result_is_fetched) + { + CV_LOG_INFO(NULL, "Asynchronous result has not been fetched"); + } + } + + bool get(OutputArray dst, int64 timeoutNs) const + { + CV_Assert(!result_is_fetched); + if (!has_result) + { + CV_UNUSED(timeoutNs); + CV_Error(Error::StsError, "Result is not produced (unable to wait for result in OPENCV_DISABLE_THREAD_SUPPORT mode)"); + } + if (!result_mat.empty()) + { + dst.move(*result_mat.get()); + result_mat.release(); + result_is_fetched = true; + return true; + } + if (!result_umat.empty()) + { + dst.move(*result_umat.get()); + result_umat.release(); + result_is_fetched = true; + return true; + } +#if CV__EXCEPTION_PTR + if (has_exception && exception) + { + result_is_fetched = true; + std::rethrow_exception(exception); + } +#endif + if (has_exception) + { + result_is_fetched = true; + throw cv_exception; + } + CV_Error(Error::StsInternal, "AsyncArray: invalid state of 'has_result = true'"); + return false; + } + + bool valid() const CV_NOEXCEPT + { + if (result_is_fetched) + return false; + if (refcount_promise == 0 && !has_result) + return false; + return true; + } + + bool wait_for(int64 timeoutNs) const + { + CV_Assert(valid()); + if (has_result) + return has_result; + if (timeoutNs == 0) + return has_result; + CV_Error(Error::StsError, "Unable to wait in OPENCV_DISABLE_THREAD_SUPPORT mode"); + } + + AsyncArray getArrayResult() + { + CV_Assert(refcount_future == 0); + AsyncArray result; + addrefFuture(); + result.p = this; + future_is_returned = true; + return result; + } + + void setValue(InputArray value) + { + if (future_is_returned && refcount_future == 0) + CV_Error(Error::StsError, "Associated AsyncArray has been destroyed"); + CV_Assert(!has_result); + int k = value.kind(); + if (k == _InputArray::UMAT) + { + result_umat = makePtr(); + value.copyTo(*result_umat.get()); + } + else + { + result_mat = makePtr(); + value.copyTo(*result_mat.get()); + } + has_result = true; + } + +#if CV__EXCEPTION_PTR + void setException(std::exception_ptr e) + { + if (future_is_returned && refcount_future == 0) + CV_Error(Error::StsError, "Associated AsyncArray has been destroyed"); + CV_Assert(!has_result); + has_exception = true; + exception = e; + has_result = true; + } +#endif + + void setException(const cv::Exception e) + { + if (future_is_returned && refcount_future == 0) + CV_Error(Error::StsError, "Associated AsyncArray has been destroyed"); + CV_Assert(!has_result); + has_exception = true; + cv_exception = e; + has_result = true; + } +}; + +} + +#endif // OPENCV_DISABLE_THREAD_SUPPORT + +namespace cv { AsyncArray::AsyncArray() CV_NOEXCEPT : p(NULL) diff --git a/modules/core/src/parallel.cpp b/modules/core/src/parallel.cpp index 7bb7e4633d..81ddd0c5dd 100644 --- a/modules/core/src/parallel.cpp +++ b/modules/core/src/parallel.cpp @@ -72,7 +72,7 @@ #endif #endif -#if defined CV_CXX11 +#ifndef OPENCV_DISABLE_THREAD_SUPPORT #include #endif @@ -884,6 +884,7 @@ T minNonZero(const T& val_1, const T& val_2) return (val_1 != 0) ? val_1 : val_2; } +#ifndef OPENCV_DISABLE_THREAD_SUPPORT static int getNumberOfCPUs_() { @@ -986,6 +987,13 @@ int getNumberOfCPUs() return nCPUs; // cached value } +#else // OPENCV_DISABLE_THREAD_SUPPORT +int getNumberOfCPUs() +{ + return 1; +} +#endif // OPENCV_DISABLE_THREAD_SUPPORT + const char* currentParallelFramework() { std::shared_ptr& api = getCurrentParallelForAPI(); diff --git a/modules/core/src/system.cpp b/modules/core/src/system.cpp index 441457d50f..777efceca0 100644 --- a/modules/core/src/system.cpp +++ b/modules/core/src/system.cpp @@ -216,7 +216,9 @@ std::wstring GetTempFileNameWinRT(std::wstring prefix) #endif #else +#ifndef OPENCV_DISABLE_THREAD_SUPPORT #include +#endif #include #include @@ -1366,6 +1368,8 @@ bool __termination = false; namespace details { +#ifndef OPENCV_DISABLE_THREAD_SUPPORT + #ifdef _WIN32 #ifdef _MSC_VER #pragma warning(disable:4505) // unreferenced local function has been removed @@ -1778,14 +1782,122 @@ static void WINAPI opencv_fls_destructor(void* pData) #endif // CV_USE_FLS #endif // _WIN32 +#else // OPENCV_DISABLE_THREAD_SUPPORT + +// no threading (OPENCV_DISABLE_THREAD_SUPPORT=ON) +class TlsStorage +{ +public: + TlsStorage() + { + slots.reserve(32); + } + ~TlsStorage() + { + for (size_t slotIdx = 0; slotIdx < slots.size(); slotIdx++) + { + SlotInfo& s = slots[slotIdx]; + TLSDataContainer* container = s.container; + if (container && s.data) + { + container->deleteDataInstance(s.data); // Can't use from SlotInfo destructor + s.data = nullptr; + } + } + } + + // Reserve TLS storage index + size_t reserveSlot(TLSDataContainer* container) + { + size_t slotsSize = slots.size(); + for (size_t slot = 0; slot < slotsSize; slot++) + { + SlotInfo& s = slots[slot]; + if (s.container == NULL) + { + CV_Assert(!s.data); + s.container = container; + return slot; + } + } + + // create new slot + slots.push_back(SlotInfo(container)); + return slotsSize; + } + + // Release TLS storage index and pass associated data to caller + void releaseSlot(size_t slotIdx, std::vector &dataVec, bool keepSlot = false) + { + CV_Assert(slotIdx < slots.size()); + SlotInfo& s = slots[slotIdx]; + void* data = s.data; + if (data) + { + dataVec.push_back(data); + s.data = nullptr; + } + if (!keepSlot) + { + s.container = NULL; // mark slot as free (see reserveSlot() implementation) + } + } + + // Get data by TLS storage index + void* getData(size_t slotIdx) const + { + CV_Assert(slotIdx < slots.size()); + const SlotInfo& s = slots[slotIdx]; + return s.data; + } + + // Gather data from threads by TLS storage index + void gather(size_t slotIdx, std::vector &dataVec) + { + CV_Assert(slotIdx < slots.size()); + SlotInfo& s = slots[slotIdx]; + void* data = s.data; + if (data) + dataVec.push_back(data); + return; + } + + // Set data to storage index + void setData(size_t slotIdx, void* pData) + { + CV_Assert(slotIdx < slots.size()); + SlotInfo& s = slots[slotIdx]; + s.data = pData; + } + +private: + struct SlotInfo + { + SlotInfo(TLSDataContainer* _container) : container(_container), data(nullptr) {} + TLSDataContainer* container; // attached container (to dispose data) + void* data; + }; + std::vector slots; +}; + +static TlsStorage& getTlsStorage() +{ + static TlsStorage g_storage; // no threading + return g_storage; +} + +#endif // OPENCV_DISABLE_THREAD_SUPPORT + } // namespace details using namespace details; void releaseTlsStorageThread() { +#ifndef OPENCV_DISABLE_THREAD_SUPPORT if (!g_isTlsStorageInitialized) return; // nothing to release, so prefer to avoid creation of new global structures getTlsStorage().releaseThread(); +#endif } TLSDataContainer::TLSDataContainer() diff --git a/modules/core/src/umatrix.cpp b/modules/core/src/umatrix.cpp index bf5dfb68a3..bbb34a7256 100644 --- a/modules/core/src/umatrix.cpp +++ b/modules/core/src/umatrix.cpp @@ -56,10 +56,6 @@ void setSize(UMat& m, int _dims, const int* _sz, const size_t* _steps, void updateContinuityFlag(UMat& m); void finalizeHdr(UMat& m); -// it should be a prime number for the best hash function -enum { UMAT_NLOCKS = 31 }; -static Mutex umatLocks[UMAT_NLOCKS]; - UMatData::UMatData(const MatAllocator* allocator) { prevAllocator = currAllocator = allocator; @@ -131,6 +127,12 @@ UMatData::~UMatData() } } +#ifndef OPENCV_DISABLE_THREAD_SUPPORT + +// it should be a prime number for the best hash function +enum { UMAT_NLOCKS = 31 }; +static Mutex umatLocks[UMAT_NLOCKS]; + static size_t getUMatDataLockIndex(const UMatData* u) { size_t idx = ((size_t)(void*)u) % UMAT_NLOCKS; @@ -228,6 +230,33 @@ UMatDataAutoLock::~UMatDataAutoLock() getUMatDataAutoLocker().release(u1, u2); } +#else + +void UMatData::lock() +{ + // nothing in OPENCV_DISABLE_THREAD_SUPPORT mode +} + +void UMatData::unlock() +{ + // nothing in OPENCV_DISABLE_THREAD_SUPPORT mode +} + +UMatDataAutoLock::UMatDataAutoLock(UMatData* u) : u1(u), u2(NULL) +{ + // nothing in OPENCV_DISABLE_THREAD_SUPPORT mode +} +UMatDataAutoLock::UMatDataAutoLock(UMatData* u1_, UMatData* u2_) : u1(u1_), u2(u2_) +{ + // nothing in OPENCV_DISABLE_THREAD_SUPPORT mode +} +UMatDataAutoLock::~UMatDataAutoLock() +{ + // nothing in OPENCV_DISABLE_THREAD_SUPPORT mode +} + +#endif // OPENCV_DISABLE_THREAD_SUPPORT + //////////////////////////////// UMat //////////////////////////////// UMat::UMat(UMatUsageFlags _usageFlags) CV_NOEXCEPT diff --git a/modules/core/src/utils/logtagmanager.hpp b/modules/core/src/utils/logtagmanager.hpp index 29a1776ada..ab4bb9b7d3 100644 --- a/modules/core/src/utils/logtagmanager.hpp +++ b/modules/core/src/utils/logtagmanager.hpp @@ -37,8 +37,8 @@ private: // also, extensible functions (accepting user-provided callback) are not allowed // to call LogTagManger (to prevent iterator invalidation), which needs enforced // with a non-recursive mutex. - using MutexType = std::mutex; - using LockType = std::lock_guard; + using MutexType = cv::Mutex; + using LockType = cv::AutoLock; enum class MatchingScope { diff --git a/modules/core/test/test_async.cpp b/modules/core/test/test_async.cpp index f898a22878..58bcfddcd7 100644 --- a/modules/core/test/test_async.cpp +++ b/modules/core/test/test_async.cpp @@ -7,7 +7,7 @@ #include -#ifdef CV_CXX11 +#if defined(CV_CXX11) && !defined(OPENCV_DISABLE_THREAD_SUPPORT) #include #include #endif @@ -85,7 +85,8 @@ TEST(Core_Async, LikePythonTest) } -#ifdef CV_CXX11 +#if defined(CV_CXX11) && !defined(OPENCV_DISABLE_THREAD_SUPPORT) + TEST(Core_Async, AsyncThread_Simple) { Mat m(3, 3, CV_32FC1, Scalar::all(5.0f)); diff --git a/modules/core/test/test_utils.cpp b/modules/core/test/test_utils.cpp index ed5f34603d..c31ca75667 100644 --- a/modules/core/test/test_utils.cpp +++ b/modules/core/test/test_utils.cpp @@ -8,9 +8,12 @@ #include "opencv2/core/utils/logger.hpp" #include "opencv2/core/utils/buffer_area.private.hpp" -#include "test_utils_tls.impl.hpp" #include "opencv2/core/utils/filesystem.private.hpp" +#ifndef OPENCV_DISABLE_THREAD_SUPPORT +#include "test_utils_tls.impl.hpp" +#endif + namespace opencv_test { namespace { static const char * const keys = diff --git a/modules/ts/CMakeLists.txt b/modules/ts/CMakeLists.txt index f95bed0793..c1d249ea14 100644 --- a/modules/ts/CMakeLists.txt +++ b/modules/ts/CMakeLists.txt @@ -41,3 +41,9 @@ endif() if(NOT OPENCV_TESTS_CONFIG_STR STREQUAL "${__content}") file(WRITE "${OPENCV_TESTS_CONFIG_FILE}" "${OPENCV_TESTS_CONFIG_STR}") endif() + +if(OPENCV_DISABLE_THREAD_SUPPORT) + # This is required to disable threads in the ts module, as + # described in `ts_gtest.h`. + ocv_target_compile_definitions(${the_module} PUBLIC GTEST_HAS_PTHREAD=0) +endif() From fd16222613f06201fa1c9b503aee3cdb1b13fa8b Mon Sep 17 00:00:00 2001 From: berak Date: Fri, 9 Jul 2021 13:21:44 +0200 Subject: [PATCH 056/262] dnn: update links for the colorization samples --- samples/dnn/colorization.cpp | 2 +- samples/dnn/colorization.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/samples/dnn/colorization.cpp b/samples/dnn/colorization.cpp index b68e0ec4d8..6d751590d1 100644 --- a/samples/dnn/colorization.cpp +++ b/samples/dnn/colorization.cpp @@ -50,7 +50,7 @@ int main(int argc, char **argv) " https://github.com/richzhang/colorization\n" "Download caffemodel and prototxt files:\n" " http://eecs.berkeley.edu/~rich.zhang/projects/2016_colorization/files/demo_v2/colorization_release_v2.caffemodel\n" - " https://raw.githubusercontent.com/richzhang/colorization/master/colorization/models/colorization_deploy_v2.prototxt\n"; + " https://raw.githubusercontent.com/richzhang/colorization/caffe/models/colorization_deploy_v2.prototxt\n"; const string keys = "{ h help | | print this help message }" "{ proto | colorization_deploy_v2.prototxt | model configuration }" diff --git a/samples/dnn/colorization.py b/samples/dnn/colorization.py index c9eb2af3b6..5bdef9793e 100644 --- a/samples/dnn/colorization.py +++ b/samples/dnn/colorization.py @@ -1,6 +1,6 @@ # Script is based on https://github.com/richzhang/colorization/blob/master/colorization/colorize.py -# To download the caffemodel and the prototxt, see: https://github.com/richzhang/colorization/tree/master/colorization/models -# To download pts_in_hull.npy, see: https://github.com/richzhang/colorization/blob/master/colorization/resources/pts_in_hull.npy +# To download the caffemodel and the prototxt, see: https://github.com/richzhang/colorization/tree/caffe/colorization/models +# To download pts_in_hull.npy, see: https://github.com/richzhang/colorization/tree/caffe/colorization/resources/pts_in_hull.npy import numpy as np import argparse import cv2 as cv From 34b65be44a265cac7921a63bb20b09786802de99 Mon Sep 17 00:00:00 2001 From: Smirnov Egor Date: Fri, 9 Jul 2021 19:15:45 +0300 Subject: [PATCH 057/262] fix find_package cache pollution --- cmake/OpenCVDetectVTK.cmake | 60 ++++++++++++++++++------------------- 1 file changed, 30 insertions(+), 30 deletions(-) diff --git a/cmake/OpenCVDetectVTK.cmake b/cmake/OpenCVDetectVTK.cmake index b8cf36007c..57c154475c 100644 --- a/cmake/OpenCVDetectVTK.cmake +++ b/cmake/OpenCVDetectVTK.cmake @@ -1,34 +1,34 @@ -# VTK 9.0 if(NOT VTK_FOUND) - find_package(VTK 9 QUIET NAMES vtk COMPONENTS - FiltersExtraction - FiltersSources - FiltersTexture - IOExport - IOGeometry - IOPLY - InteractionStyle - RenderingCore - RenderingLOD - RenderingOpenGL2 - NO_MODULE) -endif() - -# VTK 6.x components -if(NOT VTK_FOUND) - find_package(VTK QUIET COMPONENTS vtkInteractionStyle vtkRenderingLOD vtkIOPLY vtkFiltersTexture vtkRenderingFreeType vtkIOExport NO_MODULE) - IF(VTK_FOUND) - IF(VTK_RENDERING_BACKEND) #in vtk 7, the rendering backend is exported as a var. - find_package(VTK QUIET COMPONENTS vtkRendering${VTK_RENDERING_BACKEND} vtkInteractionStyle vtkRenderingLOD vtkIOPLY vtkFiltersTexture vtkRenderingFreeType vtkIOExport vtkIOGeometry NO_MODULE) - ELSE(VTK_RENDERING_BACKEND) - find_package(VTK QUIET COMPONENTS vtkRenderingOpenGL vtkInteractionStyle vtkRenderingLOD vtkIOPLY vtkFiltersTexture vtkRenderingFreeType vtkIOExport NO_MODULE) - ENDIF(VTK_RENDERING_BACKEND) - ENDIF(VTK_FOUND) -endif() - -# VTK 5.x components -if(NOT VTK_FOUND) - find_package(VTK QUIET COMPONENTS vtkCommon NO_MODULE) + find_package(VTK QUIET NAMES vtk VTK) + if(VTK_FOUND) + if(VTK_VERSION VERSION_EQUAL "9") # VTK 9.0 + find_package(VTK 9 QUIET NAMES vtk COMPONENTS + FiltersExtraction + FiltersSources + FiltersTexture + IOExport + IOGeometry + IOPLY + InteractionStyle + RenderingCore + RenderingLOD + RenderingOpenGL2 + NO_MODULE) + elseif(VTK_VERSION VERSION_GREATER "5") # VTK 6.x components + find_package(VTK QUIET COMPONENTS vtkInteractionStyle vtkRenderingLOD vtkIOPLY vtkFiltersTexture vtkRenderingFreeType vtkIOExport NO_MODULE) + IF(VTK_FOUND) + IF(VTK_RENDERING_BACKEND) #in vtk 7, the rendering backend is exported as a var. + find_package(VTK QUIET COMPONENTS vtkRendering${VTK_RENDERING_BACKEND} vtkInteractionStyle vtkRenderingLOD vtkIOPLY vtkFiltersTexture vtkRenderingFreeType vtkIOExport vtkIOGeometry NO_MODULE) + ELSE(VTK_RENDERING_BACKEND) + find_package(VTK QUIET COMPONENTS vtkRenderingOpenGL vtkInteractionStyle vtkRenderingLOD vtkIOPLY vtkFiltersTexture vtkRenderingFreeType vtkIOExport NO_MODULE) + ENDIF(VTK_RENDERING_BACKEND) + ENDIF(VTK_FOUND) + elseif(VTK_VERSION VERSION_EQUAL "5") # VTK 5.x components + find_package(VTK QUIET COMPONENTS vtkCommon NO_MODULE) + else() + set(VTK_FOUND FALSE) + endif() + endif() endif() if(NOT VTK_FOUND) From 167a12028daa7b8b0f52fc828b9693c161cc01f4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9sar=20Gouveia?= <33461054+cesarpgouveia@users.noreply.github.com> Date: Fri, 9 Jul 2021 19:21:56 +0100 Subject: [PATCH 058/262] Merge pull request #20374 from cesarpgouveia:bugfix/fix_load_onnxModel_debug * Fix bug while loading onnx model in debug * dnn: fix other .at using Co-authored-by: Alexander Alekhin --- modules/dnn/src/onnx/onnx_importer.cpp | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/modules/dnn/src/onnx/onnx_importer.cpp b/modules/dnn/src/onnx/onnx_importer.cpp index 3668c9b51e..db16cfd56d 100644 --- a/modules/dnn/src/onnx/onnx_importer.cpp +++ b/modules/dnn/src/onnx/onnx_importer.cpp @@ -788,7 +788,7 @@ void ONNXImporter::handleNode(const opencv_onnx::NodeProto& node_proto_) int blob_total = blob.total(); if (blob_total == 1) { layerParams.type = "Power"; - layerParams.set("shift", (isSub ? -1 : 1) * blob.at(0)); + layerParams.set("shift", (isSub ? -1 : 1) * blob.ptr()[0]); } else { MatShape inpShape = outShapes[node_proto.input(1 - const_blob_id)]; @@ -871,7 +871,7 @@ void ONNXImporter::handleNode(const opencv_onnx::NodeProto& node_proto_) blob.convertTo(blob, CV_32F); layerParams.type = "Power"; - layerParams.set("power", blob.at(0)); + layerParams.set("power", blob.ptr()[0]); } else if (layer_type == "Max") { @@ -1150,7 +1150,8 @@ void ONNXImporter::handleNode(const opencv_onnx::NodeProto& node_proto_) Mat blob = getBlob(node_proto, constId); blob = blob.reshape(1, 1); if (blob.total() == 1) { - float coeff = isDiv ? 1.0 / blob.at(0) : blob.at(0); + float blob_value = blob.ptr()[0]; + float coeff = isDiv ? 1.0 / blob_value : blob_value; layerParams.set("scale", coeff); layerParams.type = "Power"; } @@ -1188,12 +1189,14 @@ void ONNXImporter::handleNode(const opencv_onnx::NodeProto& node_proto_) { if (inp0.total() == 1) { - float coeff = isDiv ? 1.0 / inp0.at(0) : inp0.at(0); + float inp0_value = inp0.ptr()[0]; + float coeff = isDiv ? 1.0 / inp0_value : inp0_value; multiply(inp1, coeff, out); } else { - float coeff = isDiv ? 1.0 / inp1.at(0) : inp1.at(0); + float inp1_value = inp1.ptr()[0]; + float coeff = isDiv ? 1.0 / inp1_value : inp1_value; multiply(inp0, coeff, out); } @@ -1605,7 +1608,7 @@ void ONNXImporter::handleNode(const opencv_onnx::NodeProto& node_proto_) if (node_proto.input_size() == 3) { Mat value = getBlob(node_proto, 2); - layerParams.set("value", value.at(0)); + layerParams.set("value", value.ptr()[0]); } } } From 3f3c5de851dc8e28240a5fa292708f3bdcfe8851 Mon Sep 17 00:00:00 2001 From: Sergey Ivanov Date: Fri, 9 Jul 2021 21:46:38 +0300 Subject: [PATCH 059/262] Merge pull request #20372 from sivanov-work:serialize GAPI: Implement ConstValue serialize/deserialize * Implement ConstValue ser/deser * Fix MacOs compile issue * Fix Docs compile * Change uint32 -> uint64 for serialize tag --- .../src/backends/common/serialization.cpp | 48 +++++++++++++++++-- .../src/backends/common/serialization.hpp | 6 +++ .../test/s11n/gapi_sample_pipelines_s11n.cpp | 29 +++++++++++ 3 files changed, 79 insertions(+), 4 deletions(-) diff --git a/modules/gapi/src/backends/common/serialization.cpp b/modules/gapi/src/backends/common/serialization.cpp index 7389bacb02..f2c956874c 100644 --- a/modules/gapi/src/backends/common/serialization.cpp +++ b/modules/gapi/src/backends/common/serialization.cpp @@ -32,6 +32,14 @@ void putData(GSerialized& s, const cv::gimpl::GModel::ConstGraph& cg, const ade: }); if (s.m_datas.end() == it) { s.m_datas.push_back(gdata); + + if (cg.metadata(nh).contains()) { + size_t datas_num = s.m_datas.size() - 1; + GAPI_DbgAssert(datas_num <= static_cast(std::numeric_limits::max())); + GSerialized::data_tag_t tag = static_cast(datas_num); + s.m_const_datas.emplace(tag, + cg.metadata(nh).get()); + } } } @@ -42,11 +50,20 @@ void putOp(GSerialized& s, const cv::gimpl::GModel::ConstGraph& cg, const ade::N s.m_ops.push_back(op); } -void mkDataNode(ade::Graph& g, const cv::gimpl::Data& data) { +ade::NodeHandle mkDataNode(ade::Graph& g, const cv::gimpl::Data& data) { cv::gimpl::GModel::Graph gm(g); auto nh = gm.createNode(); gm.metadata(nh).set(cv::gimpl::NodeType{cv::gimpl::NodeType::DATA}); gm.metadata(nh).set(data); + return nh; +} + +ade::NodeHandle mkConstDataNode(ade::Graph& g, const cv::gimpl::Data& data, const cv::gimpl::ConstValue& const_data) { + auto nh = mkDataNode(g, data); + + cv::gimpl::GModel::Graph gm(g); + gm.metadata(nh).set(const_data); + return nh; } void mkOpNode(ade::Graph& g, const cv::gimpl::Op& op) { @@ -624,6 +641,10 @@ IOStream& operator<< (IOStream& os, const cv::gimpl::Data &d) { return os << d.shape << d.rc << d.meta << d.storage << d.kind; } +IOStream& operator<< (IOStream& os, const cv::gimpl::ConstValue &cd) { + return os << cd.arg; +} + namespace { template @@ -667,6 +688,9 @@ IIStream& operator>> (IIStream& is, cv::gimpl::Data &d) { return is; } +IIStream& operator>> (IIStream& is, cv::gimpl::ConstValue &cd) { + return is >> cd.arg; +} IOStream& operator<< (IOStream& os, const cv::gimpl::DataObjectCounter &c) { return os << c.m_next_data_id; @@ -709,18 +733,34 @@ void serialize( IOStream& os } s.m_counter = cg.metadata().get(); s.m_proto = p; - os << s.m_ops << s.m_datas << s.m_counter << s.m_proto; + os << s.m_ops << s.m_datas << s.m_counter << s.m_proto << s.m_const_datas; } GSerialized deserialize(IIStream &is) { GSerialized s; - is >> s.m_ops >> s.m_datas >> s.m_counter >> s.m_proto; + is >> s.m_ops >> s.m_datas >> s.m_counter >> s.m_proto >> s.m_const_datas; return s; } void reconstruct(const GSerialized &s, ade::Graph &g) { GAPI_Assert(g.nodes().empty()); - for (const auto& d : s.m_datas) cv::gapi::s11n::mkDataNode(g, d); + + GSerialized::data_tag_t tag = 0; + for (const auto& d : s.m_datas) { + if (d.storage == gimpl::Data::Storage::CONST_VAL) { + auto cit = s.m_const_datas.find(tag); + if (cit == s.m_const_datas.end()) { + util::throw_error(std::logic_error("Cannot reconstruct graph: Data::Storage::CONST_VAL by tag: " + + std::to_string(tag) + " requires ConstValue")); + } + + mkConstDataNode(g, d, cit->second); + } else { + cv::gapi::s11n::mkDataNode(g, d); + } + + tag ++; + } for (const auto& op : s.m_ops) cv::gapi::s11n::mkOpNode(g, op); cv::gapi::s11n::linkNodes(g); diff --git a/modules/gapi/src/backends/common/serialization.hpp b/modules/gapi/src/backends/common/serialization.hpp index b4204ca64e..529fdc635d 100644 --- a/modules/gapi/src/backends/common/serialization.hpp +++ b/modules/gapi/src/backends/common/serialization.hpp @@ -31,6 +31,9 @@ struct GSerialized { std::vector m_datas; cv::gimpl::DataObjectCounter m_counter; cv::gimpl::Protocol m_proto; + + using data_tag_t = uint64_t; + std::map m_const_datas; }; //////////////////////////////////////////////////////////////////////////////// @@ -97,6 +100,9 @@ GAPI_EXPORTS IIStream& operator>> (IIStream& is, cv::gimpl::Op &op); GAPI_EXPORTS IOStream& operator<< (IOStream& os, const cv::gimpl::Data &op); GAPI_EXPORTS IIStream& operator>> (IIStream& is, cv::gimpl::Data &op); +GAPI_EXPORTS IOStream& operator<< (IOStream& os, const cv::gimpl::ConstValue &cd); +GAPI_EXPORTS IIStream& operator>> (IIStream& os, cv::gimpl::ConstValue &cd); + // Render types //////////////////////////////////////////////////////////////// GAPI_EXPORTS IOStream& operator<< (IOStream& os, const cv::gapi::wip::draw::Text &t); diff --git a/modules/gapi/test/s11n/gapi_sample_pipelines_s11n.cpp b/modules/gapi/test/s11n/gapi_sample_pipelines_s11n.cpp index 885457cd90..c3d21a3f6f 100644 --- a/modules/gapi/test/s11n/gapi_sample_pipelines_s11n.cpp +++ b/modules/gapi/test/s11n/gapi_sample_pipelines_s11n.cpp @@ -806,4 +806,33 @@ TEST(S11N, Pipeline_Render_RGB) EXPECT_EQ(cv::norm(input, ref_mat), 0); } + +TEST(S11N, Pipeline_Const_GScalar) +{ + static constexpr auto in_scalar = 10; + + cv::GMat a; + cv::GScalar s; + + cv::GComputation computation(GIn(a), GOut(cv::gapi::addC(a, in_scalar))); + auto p = cv::gapi::serialize(computation); + auto deserialized_computation = cv::gapi::deserialize(p); + + cv::Mat in_mat = cv::Mat::eye(32, 32, CV_8UC1); + cv::Mat ref_mat; + cv::add(in_mat, in_scalar, ref_mat); + + cv::Mat out_mat; + computation.apply(cv::gin(in_mat/*, in_scalar*/), cv::gout(out_mat)); + EXPECT_EQ(0, cvtest::norm(out_mat, ref_mat, NORM_INF)); + + out_mat = cv::Mat(); + deserialized_computation.apply(cv::gin(in_mat/*, in_scalar*/), cv::gout(out_mat)); + EXPECT_EQ(0, cvtest::norm(out_mat, ref_mat, NORM_INF)); + + out_mat = cv::Mat(); + auto cc = deserialized_computation.compile(cv::descr_of(in_mat)); + cc(cv::gin(in_mat/*, in_scalar*/), cv::gout(out_mat)); + EXPECT_EQ(0, cvtest::norm(out_mat, ref_mat, NORM_INF)); +} } // namespace opencv_test From fd22e9829815860d25142f9e42aa68fb8f192b3f Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Mon, 12 Jul 2021 19:32:11 +0000 Subject: [PATCH 060/262] build(winpack_dldt): avoid stale sysroot contents --- platforms/winpack_dldt/build_package.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/platforms/winpack_dldt/build_package.py b/platforms/winpack_dldt/build_package.py index 6fde62241a..bd4355e1cd 100644 --- a/platforms/winpack_dldt/build_package.py +++ b/platforms/winpack_dldt/build_package.py @@ -189,7 +189,10 @@ class BuilderDLDT: if self.srcdir is None: self.srcdir = prepare_dir(self.outdir / 'sources', clean=clean_src_dir) self.build_dir = prepare_dir(self.outdir / 'build', clean=self.config.clean_dldt) - self.sysrootdir = prepare_dir(self.outdir / 'sysroot', clean=self.config.clean_dldt) + self.sysrootdir = prepare_dir(self.outdir / 'sysroot', clean=self.config.clean_dldt or self.config.clean_dldt_sysroot) + if not (self.config.clean_dldt or self.config.clean_dldt_sysroot): + _ = prepare_dir(self.sysrootdir / 'bin', clean=True) # always clean sysroot/bin (package files) + _ = prepare_dir(self.sysrootdir / 'etc', clean=True) # always clean sysroot/etc (package files) if self.config.build_subst_drive: if os.path.exists(self.config.build_subst_drive + ':\\'): @@ -483,8 +486,9 @@ def main(): parser.add_argument('--cmake_option', action='append', help='Append OpenCV CMake option') parser.add_argument('--cmake_option_dldt', action='append', help='Append CMake option for DLDT project') - parser.add_argument('--clean_dldt', action='store_true', help='Clear DLDT build and sysroot directories') - parser.add_argument('--clean_opencv', action='store_true', help='Clear OpenCV build directory') + parser.add_argument('--clean_dldt', action='store_true', help='Clean DLDT build and sysroot directories') + parser.add_argument('--clean_dldt_sysroot', action='store_true', help='Clean DLDT sysroot directories') + parser.add_argument('--clean_opencv', action='store_true', help='Clean OpenCV build directory') parser.add_argument('--build_debug', action='store_true', help='Build debug binaries') parser.add_argument('--build_tests', action='store_true', help='Build OpenCV tests') From 4af1f31a3fb3530499826e516fdc5b5121cdf600 Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Tue, 13 Jul 2021 09:15:03 +0000 Subject: [PATCH 061/262] cmake: use relative path for mode vars --- cmake/OpenCVUtils.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/OpenCVUtils.cmake b/cmake/OpenCVUtils.cmake index 39445150a9..da0ee3b36b 100644 --- a/cmake/OpenCVUtils.cmake +++ b/cmake/OpenCVUtils.cmake @@ -1978,4 +1978,4 @@ endif() # # Include configuration override settings # -include(cmake/vars/EnableModeVars.cmake) +include("${CMAKE_CURRENT_LIST_DIR}/vars/EnableModeVars.cmake") From 5179e37bd1670e025735ee9ed9aa7e4cbe38ccd9 Mon Sep 17 00:00:00 2001 From: Alexey Smirnov Date: Tue, 13 Jul 2021 22:31:46 +0300 Subject: [PATCH 062/262] Merge pull request #20329 from smirnov-alexey:as/mediaframe_serialization [G-API]: Add serialization mechanism for cv::MediaFrame * Stub initial interface * Fix templates for deserialization * Fix tests * Disable a warning on windows * Address review comments * Change enable_ifs to other template helpers * Resolve ambiguous template * Fix warnings in docs --- modules/gapi/include/opencv2/gapi/media.hpp | 28 +++++ modules/gapi/include/opencv2/gapi/rmat.hpp | 10 +- modules/gapi/include/opencv2/gapi/s11n.hpp | 67 +++++++--- .../gapi/include/opencv2/gapi/util/util.hpp | 24 +++- modules/gapi/src/api/media.cpp | 4 + modules/gapi/src/api/s11n.cpp | 14 ++- .../src/backends/common/serialization.cpp | 14 ++- modules/gapi/test/s11n/gapi_s11n_tests.cpp | 116 ++++++++++++++++++ 8 files changed, 243 insertions(+), 34 deletions(-) diff --git a/modules/gapi/include/opencv2/gapi/media.hpp b/modules/gapi/include/opencv2/gapi/media.hpp index aa7d6d6a1f..19aaef3fd1 100644 --- a/modules/gapi/include/opencv2/gapi/media.hpp +++ b/modules/gapi/include/opencv2/gapi/media.hpp @@ -15,6 +15,16 @@ #include #include +// Forward declaration +namespace cv { +namespace gapi { +namespace s11n { +struct IOStream; +struct IIStream; +} // namespace s11n +} // namespace gapi +} // namespace cv + namespace cv { /** \addtogroup gapi_data_structures @@ -125,6 +135,16 @@ public: return dynamic_cast(adapter); } + /** + * @brief Serialize MediaFrame's data to a byte array. + * + * @note The actual logic is implemented by frame's adapter class. + * Does nothing by default. + * + * @param os Bytestream to store serialized MediaFrame data in. + */ + void serialize(cv::gapi::s11n::IOStream& os) const; + private: struct Priv; std::shared_ptr m; @@ -221,6 +241,14 @@ public: // FIXME: design a better solution // The default implementation does nothing virtual cv::util::any blobParams() const; + virtual void serialize(cv::gapi::s11n::IOStream&) { + GAPI_Assert(false && "Generic serialize method of MediaFrame::IAdapter does nothing by default. " + "Please, implement it in derived class to properly serialize the object."); + } + virtual void deserialize(cv::gapi::s11n::IIStream&) { + GAPI_Assert(false && "Generic deserialize method of MediaFrame::IAdapter does nothing by default. " + "Please, implement it in derived class to properly deserialize the object."); + } }; /** @} */ diff --git a/modules/gapi/include/opencv2/gapi/rmat.hpp b/modules/gapi/include/opencv2/gapi/rmat.hpp index cc27f48664..6b289001e7 100644 --- a/modules/gapi/include/opencv2/gapi/rmat.hpp +++ b/modules/gapi/include/opencv2/gapi/rmat.hpp @@ -14,8 +14,8 @@ namespace cv { namespace gapi { namespace s11n { - struct IOStream; - struct IIStream; +struct IOStream; +struct IIStream; } // namespace s11n } // namespace gapi } // namespace cv @@ -111,10 +111,12 @@ public: // is transferred to the device when the view is destroyed virtual View access(Access) = 0; virtual void serialize(cv::gapi::s11n::IOStream&) { - GAPI_Assert(false && "Generic serialize method should never be called for RMat adapter"); + GAPI_Assert(false && "Generic serialize method of RMat::Adapter does nothing by default. " + "Please, implement it in derived class to properly serialize the object."); } virtual void deserialize(cv::gapi::s11n::IIStream&) { - GAPI_Assert(false && "Generic deserialize method should never be called for RMat adapter"); + GAPI_Assert(false && "Generic deserialize method of RMat::Adapter does nothing by default. " + "Please, implement it in derived class to properly deserialize the object."); } }; using AdapterP = std::shared_ptr; diff --git a/modules/gapi/include/opencv2/gapi/s11n.hpp b/modules/gapi/include/opencv2/gapi/s11n.hpp index ca8e32c98b..53800970d1 100644 --- a/modules/gapi/include/opencv2/gapi/s11n.hpp +++ b/modules/gapi/include/opencv2/gapi/s11n.hpp @@ -13,6 +13,13 @@ #include #include #include +#include +#include + +// FIXME: caused by deserialize_runarg +#if (defined _WIN32 || defined _WIN64) && defined _MSC_VER +#pragma warning(disable: 4702) +#endif namespace cv { namespace gapi { @@ -34,8 +41,8 @@ namespace detail { template cv::GCompileArgs getCompileArgs(const std::vector &bytes); - template - cv::GRunArgs getRunArgsWithRMats(const std::vector &bytes); + template + cv::GRunArgs getRunArgsWithAdapters(const std::vector &bytes); } // namespace detail /** @brief Serialize a graph represented by GComputation into an array of bytes. @@ -133,19 +140,18 @@ type deserialize(const std::vector &bytes) { } /** - * @brief Deserialize GRunArgs including RMat objects if any from a byte array. + * @brief Deserialize GRunArgs including RMat and MediaFrame objects if any from a byte array. * - * RMat adapter type is specified in the template. - * @note To be used properly specified adapter type must overload its serialize() and - * deserialize() methods. + * Adapter types are specified in the template. + * @note To be used properly specified adapter types must overload their deserialize() method. * @param bytes vector of bytes to deserialize GRunArgs object from. - * @return GRunArgs including RMat objects if any. - * @see RMat + * @return GRunArgs including RMat and MediaFrame objects if any. + * @see RMat MediaFrame */ -template inline +template inline typename std::enable_if::value, GRunArgs>:: type deserialize(const std::vector &bytes) { - return detail::getRunArgsWithRMats(bytes); + return detail::getRunArgsWithAdapters(bytes); } } // namespace gapi } // namespace cv @@ -399,16 +405,39 @@ static cv::util::optional exec(const std::string& tag, cv::gapi::s1 } }; -template struct deserialize_runarg; +template +struct deserialize_arg_with_adapter; -template +template +struct deserialize_arg_with_adapter { +static GRunArg exec(cv::gapi::s11n::IIStream& is) { + std::unique_ptr ptr(new TA); + ptr->deserialize(is); + return GRunArg { RA(std::move(ptr)) }; +} +}; + +template +struct deserialize_arg_with_adapter { +static GRunArg exec(cv::gapi::s11n::IIStream&) { + GAPI_Assert(false && "No suitable adapter class found during RMat/MediaFrame deserialization. " + "Please, make sure you've passed them in cv::gapi::deserialize() template"); + return GRunArg{}; +} +}; + +template struct deserialize_runarg { static GRunArg exec(cv::gapi::s11n::IIStream& is, uint32_t idx) { if (idx == GRunArg::index_of()) { - auto ptr = std::make_shared(); - ptr->deserialize(is); - return GRunArg { RMat(std::move(ptr)) }; - } else { // non-RMat arg - use default deserialization + // Type or void (if not found) + using TA = typename cv::util::find_adapter_impl::type; + return deserialize_arg_with_adapter::exec(is); + } else if (idx == GRunArg::index_of()) { + // Type or void (if not found) + using TA = typename cv::util::find_adapter_impl::type; + return deserialize_arg_with_adapter::exec(is); + } else { // not an adapter holding type runarg - use default deserialization GRunArg arg; getRunArgByIdx(is, arg, idx); return arg; @@ -451,8 +480,8 @@ cv::GCompileArgs getCompileArgs(const std::vector &sArgs) { return args; } -template -cv::GRunArgs getRunArgsWithRMats(const std::vector &bytes) { +template +cv::GRunArgs getRunArgsWithAdapters(const std::vector &bytes) { std::unique_ptr pIs = cv::gapi::s11n::detail::getInStream(bytes); cv::gapi::s11n::IIStream& is = *pIs; cv::GRunArgs args; @@ -462,7 +491,7 @@ cv::GRunArgs getRunArgsWithRMats(const std::vector &bytes) { for (uint32_t i = 0; i < sz; ++i) { uint32_t idx = 0; is >> idx; - args.push_back(cv::gapi::detail::deserialize_runarg::exec(is, idx)); + args.push_back(cv::gapi::detail::deserialize_runarg::exec(is, idx)); } return args; diff --git a/modules/gapi/include/opencv2/gapi/util/util.hpp b/modules/gapi/include/opencv2/gapi/util/util.hpp index c6ad0632e2..eb435a3eef 100644 --- a/modules/gapi/include/opencv2/gapi/util/util.hpp +++ b/modules/gapi/include/opencv2/gapi/util/util.hpp @@ -153,7 +153,29 @@ overload_lamba_set overload_lambdas(L&& ...lambdas) { return overload_lamba_set(std::forward(lambdas)...); } -} + +template +struct find_adapter_impl; + +template +struct find_adapter_impl +{ + using type = typename std::conditional::value, + T, + void>::type; + static constexpr bool found = std::is_base_of::value; +}; + +template +struct find_adapter_impl +{ + using type = typename std::conditional::value, + T, + typename find_adapter_impl::type>::type; + static constexpr bool found = std::is_base_of::value || + find_adapter_impl::found; +}; +} // namespace util } // namespace cv // \endcond diff --git a/modules/gapi/src/api/media.cpp b/modules/gapi/src/api/media.cpp index 884fc9e83d..b1c455d40a 100644 --- a/modules/gapi/src/api/media.cpp +++ b/modules/gapi/src/api/media.cpp @@ -35,6 +35,10 @@ cv::MediaFrame::IAdapter* cv::MediaFrame::getAdapter() const { return m->adapter.get(); } +void cv::MediaFrame::serialize(cv::gapi::s11n::IOStream& os) const { + return m->adapter->serialize(os); +} + cv::MediaFrame::View::View(Ptrs&& ptrs, Strides&& strs, Callback &&cb) : ptr (std::move(ptrs)) , stride(std::move(strs)) diff --git a/modules/gapi/src/api/s11n.cpp b/modules/gapi/src/api/s11n.cpp index 97f5a95c42..bd7f46c88a 100644 --- a/modules/gapi/src/api/s11n.cpp +++ b/modules/gapi/src/api/s11n.cpp @@ -76,14 +76,14 @@ cv::GRunArgsP cv::gapi::bind(cv::GRunArgs &out_args) { #if !defined(GAPI_STANDALONE) case T::index_of() : - outputs.emplace_back((cv::UMat*)(&(cv::util::get(res_obj)))); + outputs.emplace_back(&(cv::util::get(res_obj))); break; #endif case cv::GRunArg::index_of() : - outputs.emplace_back((cv::Mat*)(&(cv::util::get(res_obj)))); + outputs.emplace_back(&(cv::util::get(res_obj))); break; case cv::GRunArg::index_of() : - outputs.emplace_back((cv::Scalar*)(&(cv::util::get(res_obj)))); + outputs.emplace_back(&(cv::util::get(res_obj))); break; case T::index_of() : outputs.emplace_back(cv::util::get(res_obj)); @@ -92,7 +92,10 @@ cv::GRunArgsP cv::gapi::bind(cv::GRunArgs &out_args) outputs.emplace_back(cv::util::get(res_obj)); break; case cv::GRunArg::index_of() : - outputs.emplace_back((cv::RMat*)(&(cv::util::get(res_obj)))); + outputs.emplace_back(&(cv::util::get(res_obj))); + break; + case cv::GRunArg::index_of() : + outputs.emplace_back(&(cv::util::get(res_obj))); break; default: GAPI_Assert(false && "This value type is not supported!"); // ...maybe because of STANDALONE mode. @@ -130,6 +133,9 @@ cv::GRunArg cv::gapi::bind(cv::GRunArgP &out) case T::index_of() : return cv::GRunArg(*cv::util::get(out)); + case T::index_of() : + return cv::GRunArg(*cv::util::get(out)); + default: // ...maybe our types were extended GAPI_Assert(false && "This value type is UNKNOWN!"); diff --git a/modules/gapi/src/backends/common/serialization.cpp b/modules/gapi/src/backends/common/serialization.cpp index f2c956874c..619b2feb74 100644 --- a/modules/gapi/src/backends/common/serialization.cpp +++ b/modules/gapi/src/backends/common/serialization.cpp @@ -201,18 +201,20 @@ IOStream& operator<< (IOStream& os, const cv::RMat& mat) { return os; } IIStream& operator>> (IIStream& is, cv::RMat&) { - util::throw_error(std::logic_error("operator>> for RMat should never be called")); + util::throw_error(std::logic_error("operator>> for RMat should never be called. " + "Instead, cv::gapi::deserialize() " + "should be used")); return is; } -IOStream& operator<< (IOStream& os, const cv::MediaFrame &) { - // Stub - GAPI_Assert(false && "cv::MediaFrame serialization is not supported!"); +IOStream& operator<< (IOStream& os, const cv::MediaFrame &frame) { + frame.serialize(os); return os; } IIStream& operator>> (IIStream& is, cv::MediaFrame &) { - // Stub - GAPI_Assert(false && "cv::MediaFrame serialization is not supported!"); + util::throw_error(std::logic_error("operator>> for MediaFrame should never be called. " + "Instead, cv::gapi::deserialize() " + "should be used")); return is; } diff --git a/modules/gapi/test/s11n/gapi_s11n_tests.cpp b/modules/gapi/test/s11n/gapi_s11n_tests.cpp index c2b17521d9..4c6e63b552 100644 --- a/modules/gapi/test/s11n/gapi_s11n_tests.cpp +++ b/modules/gapi/test/s11n/gapi_s11n_tests.cpp @@ -2,6 +2,7 @@ #include "backends/common/serialization.hpp" #include +#include #include <../src/backends/common/gbackend.hpp> // asView namespace { @@ -148,6 +149,29 @@ public: int getVal() { return m_value; } std::string getStr() { return m_str; } }; + +class MyMediaFrameAdapter : public cv::MediaFrame::IAdapter { + cv::Mat m_mat; + int m_value; + std::string m_str; +public: + MyMediaFrameAdapter() = default; + MyMediaFrameAdapter(cv::Mat m, int value, const std::string& str) + : m_mat(m), m_value(value), m_str(str) + {} + virtual cv::MediaFrame::View access(cv::MediaFrame::Access) override { + return cv::MediaFrame::View({m_mat.data}, {m_mat.step}); + } + virtual cv::GFrameDesc meta() const override { return {cv::MediaFormat::BGR, m_mat.size()}; } + virtual void serialize(cv::gapi::s11n::IOStream& os) override { + os << m_value << m_str; + } + virtual void deserialize(cv::gapi::s11n::IIStream& is) override { + is >> m_value >> m_str; + } + int getVal() { return m_value; } + std::string getStr() { return m_str; } +}; } namespace opencv_test { @@ -581,6 +605,17 @@ TEST_F(S11N_Basic, Test_Vector_Of_Strings) { EXPECT_EQ("42", des[2]); } +TEST_F(S11N_Basic, Test_RunArg) { + cv::Mat mat = cv::Mat::eye(cv::Size(128, 64), CV_8UC3); + auto v = cv::GRunArgs{ cv::GRunArg{ mat } }; + + const std::vector sargsin = cv::gapi::serialize(v); + cv::GRunArgs out = cv::gapi::deserialize(sargsin); + cv::Mat out_mat = cv::util::get(out[0]); + + EXPECT_EQ(0, cv::norm(mat, out_mat)); +} + TEST_F(S11N_Basic, Test_RunArg_RMat) { cv::Mat mat = cv::Mat::eye(cv::Size(128, 64), CV_8UC3); cv::RMat rmat = cv::make_rmat(mat, 42, "It actually works"); @@ -614,6 +649,87 @@ TEST_F(S11N_Basic, Test_RunArg_RMat_Scalar_Mat) { EXPECT_EQ(0, cv::norm(mat, out_mat)); } +TEST_F(S11N_Basic, Test_RunArg_MediaFrame) { + cv::Mat mat = cv::Mat::eye(cv::Size(128, 64), CV_8UC3); + auto frame = cv::MediaFrame::Create(mat, 42, "It actually works"); + auto v = cv::GRunArgs{ cv::GRunArg{ frame } }; + + const std::vector sargsin = cv::gapi::serialize(v); + cv::GRunArgs out = cv::gapi::deserialize(sargsin); + cv::MediaFrame out_mat = cv::util::get(out[0]); + auto adapter = out_mat.get(); + EXPECT_EQ(42, adapter->getVal()); + EXPECT_EQ("It actually works", adapter->getStr()); +} + +TEST_F(S11N_Basic, Test_RunArg_MediaFrame_Scalar_Mat) { + cv::Mat mat = cv::Mat::eye(cv::Size(128, 64), CV_8UC3); + auto frame = cv::MediaFrame::Create(mat, 42, "It actually works"); + cv::Scalar sc(111); + auto v = cv::GRunArgs{ cv::GRunArg{ frame }, cv::GRunArg{ sc }, cv::GRunArg{ mat } }; + + const std::vector sargsin = cv::gapi::serialize(v); + cv::GRunArgs out = cv::gapi::deserialize(sargsin); + cv::MediaFrame out_frame = cv::util::get(out[0]); + auto adapter = out_frame.get(); + EXPECT_EQ(42, adapter->getVal()); + EXPECT_EQ("It actually works", adapter->getStr()); + + cv::Scalar out_sc = cv::util::get(out[1]); + EXPECT_EQ(sc, out_sc); + + cv::Mat out_mat = cv::util::get(out[2]); + EXPECT_EQ(0, cv::norm(mat, out_mat)); +} + +TEST_F(S11N_Basic, Test_RunArg_MediaFrame_RMat) { + cv::Mat mat = cv::Mat::eye(cv::Size(128, 64), CV_8UC3); + cv::Mat mat2 = cv::Mat::eye(cv::Size(128, 64), CV_8UC3); + + auto frame = cv::MediaFrame::Create(mat, 42, "It actually works"); + auto rmat = cv::make_rmat(mat2, 24, "Hello there"); + + auto v = cv::GRunArgs{ cv::GRunArg{ frame }, cv::GRunArg{ rmat } }; + + const std::vector sargsin = cv::gapi::serialize(v); + cv::GRunArgs out = cv::gapi::deserialize(sargsin); + + cv::MediaFrame out_frame = cv::util::get(out[0]); + cv::RMat out_rmat = cv::util::get(out[1]); + + auto adapter = out_frame.get(); + EXPECT_EQ(42, adapter->getVal()); + EXPECT_EQ("It actually works", adapter->getStr()); + + auto adapter2 = out_rmat.get(); + EXPECT_EQ(24, adapter2->getVal()); + EXPECT_EQ("Hello there", adapter2->getStr()); +} + +TEST_F(S11N_Basic, Test_RunArg_RMat_MediaFrame) { + cv::Mat mat = cv::Mat::eye(cv::Size(128, 64), CV_8UC3); + cv::Mat mat2 = cv::Mat::eye(cv::Size(128, 64), CV_8UC3); + + auto frame = cv::MediaFrame::Create(mat, 42, "It actually works"); + auto rmat = cv::make_rmat(mat2, 24, "Hello there"); + + auto v = cv::GRunArgs{ cv::GRunArg{ rmat }, cv::GRunArg{ frame } }; + + const std::vector sargsin = cv::gapi::serialize(v); + cv::GRunArgs out = cv::gapi::deserialize(sargsin); + + cv::RMat out_rmat = cv::util::get(out[0]); + cv::MediaFrame out_frame = cv::util::get(out[1]); + + auto adapter = out_frame.get(); + EXPECT_EQ(42, adapter->getVal()); + EXPECT_EQ("It actually works", adapter->getStr()); + + auto adapter2 = out_rmat.get(); + EXPECT_EQ(24, adapter2->getVal()); + EXPECT_EQ("Hello there", adapter2->getStr()); +} + namespace { template bool verifyOpaqueKind(T&& in) { From a7742d7d631b00346aed8a681e1bbc338d58651d Mon Sep 17 00:00:00 2001 From: Dmitry Budnikov Date: Tue, 13 Jul 2021 22:33:13 +0300 Subject: [PATCH 063/262] Merge pull request #20383 from dbudniko:dbudniko/mtcnn_1st_pnet_simplification MTCNN 1st pnet simplification to ensure single graph input * 1st pnet simplification to ensure single graph input * address comment from Dmitry M regarding unused variable --- modules/gapi/samples/face_detection_mtcnn.cpp | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/modules/gapi/samples/face_detection_mtcnn.cpp b/modules/gapi/samples/face_detection_mtcnn.cpp index c437bdbba4..d679ba0529 100644 --- a/modules/gapi/samples/face_detection_mtcnn.cpp +++ b/modules/gapi/samples/face_detection_mtcnn.cpp @@ -596,7 +596,6 @@ int main(int argc, char* argv[]) { cv::GMat scores[MAX_PYRAMID_LEVELS]; cv::GArray nms_p_faces[MAX_PYRAMID_LEVELS]; cv::GArray total_faces[MAX_PYRAMID_LEVELS]; - cv::GArray faces_init(std::vector{}); //The very first PNet pyramid layer to init total_faces[0] in_resized[0] = cv::gapi::resize(in_originalRGB, level_size[0]); @@ -605,8 +604,7 @@ int main(int argc, char* argv[]) { cv::GArray faces0 = custom::BuildFaces::on(scores[0], regressions[0], static_cast(scales[0]), conf_thresh_p); cv::GArray final_p_faces_for_bb2squares = custom::ApplyRegression::on(faces0, true); cv::GArray final_faces_pnet0 = custom::BBoxesToSquares::on(final_p_faces_for_bb2squares); - nms_p_faces[0] = custom::RunNMS::on(final_faces_pnet0, 0.5f, false); - total_faces[0] = custom::AccumulatePyramidOutputs::on(faces_init, nms_p_faces[0]); + total_faces[0] = custom::RunNMS::on(final_faces_pnet0, 0.5f, false); //The rest PNet pyramid layers to accumlate all layers result in total_faces[PYRAMID_LEVELS - 1]] for (int i = 1; i < pyramid_levels; ++i) { From 6f417b57c1bbb3b113104b3023ae9f2361b4618d Mon Sep 17 00:00:00 2001 From: Pablo Romero Date: Tue, 13 Jul 2021 21:40:15 +0200 Subject: [PATCH 064/262] Merge pull request #20399 from pablorcum:3.4 Improves support for Unix non-Linux systems, including QNX * Fixes #20395. Improves support for Unix non-Linux systems. Focus on QNX Neutrino. Signed-off-by: promero * Update system.cpp --- CMakeLists.txt | 2 ++ modules/core/src/parallel.cpp | 2 +- modules/core/src/system.cpp | 22 +++++++++++++--------- 3 files changed, 16 insertions(+), 10 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index f6a2da5310..94ef43fcb4 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -648,6 +648,8 @@ if(UNIX) set(OPENCV_LINKER_LIBS ${OPENCV_LINKER_LIBS} m pthread) elseif(EMSCRIPTEN) # no need to link to system libs with emscripten + elseif(QNXNTO) + set(OPENCV_LINKER_LIBS ${OPENCV_LINKER_LIBS} m) else() set(OPENCV_LINKER_LIBS ${OPENCV_LINKER_LIBS} dl m pthread rt) endif() diff --git a/modules/core/src/parallel.cpp b/modules/core/src/parallel.cpp index 9ac7d3e4c0..cfff4cea4b 100644 --- a/modules/core/src/parallel.cpp +++ b/modules/core/src/parallel.cpp @@ -53,7 +53,7 @@ #undef abs #endif -#if defined __linux__ || defined __APPLE__ || defined __GLIBC__ \ +#if defined __unix__ || defined __APPLE__ || defined __GLIBC__ \ || defined __HAIKU__ || defined __EMSCRIPTEN__ || defined __FreeBSD__ \ || defined __OpenBSD__ #include diff --git a/modules/core/src/system.cpp b/modules/core/src/system.cpp index b6810fa9f5..d8b8f67559 100644 --- a/modules/core/src/system.cpp +++ b/modules/core/src/system.cpp @@ -114,10 +114,14 @@ void* allocSingletonNewBuffer(size_t size) { return malloc(size); } #include // std::abort #endif -#if defined __ANDROID__ || defined __linux__ || defined __FreeBSD__ || defined __OpenBSD__ || defined __HAIKU__ +#if defined __ANDROID__ || defined __unix__ || defined __FreeBSD__ || defined __OpenBSD__ || defined __HAIKU__ # include # include +#if defined __QNXNTO__ +# include +#else # include +#endif #if defined __ANDROID__ || defined __linux__ # include #endif @@ -128,7 +132,7 @@ void* allocSingletonNewBuffer(size_t size) { return malloc(size); } #endif -#if (defined __ppc64__ || defined __PPC64__) && defined __linux__ +#if (defined __ppc64__ || defined __PPC64__) && defined __unix__ # include "sys/auxv.h" # ifndef AT_HWCAP2 # define AT_HWCAP2 26 @@ -229,7 +233,7 @@ std::wstring GetTempFileNameWinRT(std::wstring prefix) #include "omp.h" #endif -#if defined __linux__ || defined __APPLE__ || defined __EMSCRIPTEN__ || defined __FreeBSD__ || defined __GLIBC__ || defined __HAIKU__ +#if defined __unix__ || defined __APPLE__ || defined __EMSCRIPTEN__ || defined __FreeBSD__ || defined __GLIBC__ || defined __HAIKU__ #include #include #include @@ -591,7 +595,7 @@ struct HWFeatures have[CV_CPU_MSA] = true; #endif - #if (defined __ppc64__ || defined __PPC64__) && defined __linux__ + #if (defined __ppc64__ || defined __PPC64__) && defined __unix__ unsigned int hwcap = getauxval(AT_HWCAP); if (hwcap & PPC_FEATURE_HAS_VSX) { hwcap = getauxval(AT_HWCAP2); @@ -804,12 +808,12 @@ int64 getTickCount(void) LARGE_INTEGER counter; QueryPerformanceCounter( &counter ); return (int64)counter.QuadPart; -#elif defined __linux || defined __linux__ +#elif defined __MACH__ && defined __APPLE__ + return (int64)mach_absolute_time(); +#elif defined __unix__ struct timespec tp; clock_gettime(CLOCK_MONOTONIC, &tp); return (int64)tp.tv_sec*1000000000 + tp.tv_nsec; -#elif defined __MACH__ && defined __APPLE__ - return (int64)mach_absolute_time(); #else struct timeval tv; gettimeofday(&tv, NULL); @@ -823,8 +827,6 @@ double getTickFrequency(void) LARGE_INTEGER freq; QueryPerformanceFrequency(&freq); return (double)freq.QuadPart; -#elif defined __linux || defined __linux__ - return 1e9; #elif defined __MACH__ && defined __APPLE__ static double freq = 0; if( freq == 0 ) @@ -834,6 +836,8 @@ double getTickFrequency(void) freq = sTimebaseInfo.denom*1e9/sTimebaseInfo.numer; } return freq; +#elif defined __unix__ + return 1e9; #else return 1e6; #endif From 2113af9c52b4756809004dc932489e518dea4837 Mon Sep 17 00:00:00 2001 From: Roland Meertens Date: Thu, 3 Jun 2021 20:59:22 +0200 Subject: [PATCH 065/262] Updated grabcut example to show the background in a transparant way --- samples/cpp/grabcut.cpp | 37 +++++++++++++++++++++++++++---------- 1 file changed, 27 insertions(+), 10 deletions(-) diff --git a/samples/cpp/grabcut.cpp b/samples/cpp/grabcut.cpp index d3e3db49f9..25492166a7 100644 --- a/samples/cpp/grabcut.cpp +++ b/samples/cpp/grabcut.cpp @@ -107,12 +107,14 @@ void GCApplication::showImage() const Mat res; Mat binMask; - if( !isInitialized ) - image->copyTo( res ); - else - { - getBinMask( mask, binMask ); - image->copyTo( res, binMask ); + image->copyTo( res ); + if( isInitialized ){ + getBinMask( mask, binMask); + + Mat black (binMask.rows, binMask.cols, CV_8UC3, cv::Scalar(0,0,0)); + black.setTo(Scalar::all(255), binMask); + + addWeighted(black, 0.5, res, 0.5, 0.0, res); } vector::const_iterator it; @@ -201,24 +203,39 @@ void GCApplication::mouseClick( int event, int x, int y, int flags, void* ) case EVENT_LBUTTONUP: if( rectState == IN_PROCESS ) { - rect = Rect( Point(rect.x, rect.y), Point(x,y) ); - rectState = SET; - setRectInMask(); - CV_Assert( bgdPxls.empty() && fgdPxls.empty() && prBgdPxls.empty() && prFgdPxls.empty() ); + if(rect.x == x || rect.y == y){ + rectState = NOT_SET; + } + else{ + rect = Rect( Point(rect.x, rect.y), Point(x,y) ); + rectState = SET; + setRectInMask(); + CV_Assert( bgdPxls.empty() && fgdPxls.empty() && prBgdPxls.empty() && prFgdPxls.empty() ); + } showImage(); } if( lblsState == IN_PROCESS ) { setLblsInMask(flags, Point(x,y), false); lblsState = SET; + nextIter(); showImage(); } + else{ + if(rectState == SET){ + nextIter(); + showImage(); + } + } break; case EVENT_RBUTTONUP: if( prLblsState == IN_PROCESS ) { setLblsInMask(flags, Point(x,y), true); prLblsState = SET; + } + if(rectState == SET){ + nextIter(); showImage(); } break; From bc210b292b7df85d0582211e85c8017aca165444 Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Fri, 9 Jul 2021 16:22:13 +0000 Subject: [PATCH 066/262] dnn(test): backport test_ie_models.cpp from 4.5.3 --- modules/dnn/test/test_ie_models.cpp | 28 ++++++++++++++++++++++------ 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/modules/dnn/test/test_ie_models.cpp b/modules/dnn/test/test_ie_models.cpp index da6cbd6fbc..06d2e1776d 100644 --- a/modules/dnn/test/test_ie_models.cpp +++ b/modules/dnn/test/test_ie_models.cpp @@ -103,11 +103,15 @@ static const std::map& getOpenVINOTestMo #if INF_ENGINE_RELEASE >= 2020010000 // Downloaded using these parameters for Open Model Zoo downloader (2020.1): // ./downloader.py -o ${OPENCV_DNN_TEST_DATA_PATH}/omz_intel_models --cache_dir ${OPENCV_DNN_TEST_DATA_PATH}/.omz_cache/ \ - // --name person-detection-retail-0013 + // --name person-detection-retail-0013,age-gender-recognition-retail-0013 { "person-detection-retail-0013", { // IRv10 "intel/person-detection-retail-0013/FP32/person-detection-retail-0013", "intel/person-detection-retail-0013/FP16/person-detection-retail-0013" }}, + { "age-gender-recognition-retail-0013", { + "intel/age-gender-recognition-retail-0013/FP16/age-gender-recognition-retail-0013", + "intel/age-gender-recognition-retail-0013/FP32/age-gender-recognition-retail-0013" + }}, #endif }; @@ -123,6 +127,21 @@ static const std::vector getOpenVINOTestModelsList() return result; } +inline static std::string getOpenVINOModel(const std::string &modelName, bool isFP16) +{ + const std::map& models = getOpenVINOTestModels(); + const auto it = models.find(modelName); + if (it != models.end()) + { + OpenVINOModelTestCaseInfo modelInfo = it->second; + if (isFP16 && modelInfo.modelPathFP16) + return std::string(modelInfo.modelPathFP16); + else if (!isFP16 && modelInfo.modelPathFP32) + return std::string(modelInfo.modelPathFP32); + } + return std::string(); +} + static inline void genData(const InferenceEngine::TensorDesc& desc, Mat& m, Blob::Ptr& dataPtr) { const std::vector& dims = desc.getDims(); @@ -319,11 +338,8 @@ TEST_P(DNNTestOpenVINO, models) bool isFP16 = (targetId == DNN_TARGET_OPENCL_FP16 || targetId == DNN_TARGET_MYRIAD); - const std::map& models = getOpenVINOTestModels(); - const auto it = models.find(modelName); - ASSERT_TRUE(it != models.end()) << modelName; - OpenVINOModelTestCaseInfo modelInfo = it->second; - std::string modelPath = isFP16 ? modelInfo.modelPathFP16 : modelInfo.modelPathFP32; + const std::string modelPath = getOpenVINOModel(modelName, isFP16); + ASSERT_FALSE(modelPath.empty()) << modelName; std::string xmlPath = findDataFile(modelPath + ".xml", false); std::string binPath = findDataFile(modelPath + ".bin", false); From 9f2dcc3f13d8484d8f97df866f8bd51be4875c36 Mon Sep 17 00:00:00 2001 From: berak Date: Thu, 15 Jul 2021 17:02:23 +0200 Subject: [PATCH 067/262] python: fix trackbar warning --- modules/python/src2/cv2.cpp | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/modules/python/src2/cv2.cpp b/modules/python/src2/cv2.cpp index 4bdb0fcc14..6c5e6463d2 100644 --- a/modules/python/src2/cv2.cpp +++ b/modules/python/src2/cv2.cpp @@ -1971,15 +1971,23 @@ static void OnChange(int pos, void *param) } #ifdef HAVE_OPENCV_HIGHGUI +// workaround for #20408, use nullptr, set value later +static int _createTrackbar(const String &trackbar_name, const String &window_name, int value, int count, + TrackbarCallback onChange, PyObject* py_callback_info) +{ + int n = createTrackbar(trackbar_name, window_name, NULL, count, onChange, py_callback_info); + setTrackbarPos(trackbar_name, window_name, value); + return n; +} static PyObject *pycvCreateTrackbar(PyObject*, PyObject *args) { PyObject *on_change; char* trackbar_name; char* window_name; - int *value = new int; + int value; int count; - if (!PyArg_ParseTuple(args, "ssiiO", &trackbar_name, &window_name, value, &count, &on_change)) + if (!PyArg_ParseTuple(args, "ssiiO", &trackbar_name, &window_name, &value, &count, &on_change)) return NULL; if (!PyCallable_Check(on_change)) { PyErr_SetString(PyExc_TypeError, "on_change must be callable"); @@ -1998,7 +2006,7 @@ static PyObject *pycvCreateTrackbar(PyObject*, PyObject *args) { registered_callbacks.insert(std::pair(name, py_callback_info)); } - ERRWRAP2(createTrackbar(trackbar_name, window_name, value, count, OnChange, py_callback_info)); + ERRWRAP2(_createTrackbar(trackbar_name, window_name, value, count, OnChange, py_callback_info)); Py_RETURN_NONE; } From 602e7c83e2ea3cf7d094e883c69f0911a37e6d1a Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Sat, 10 Jul 2021 13:06:33 +0000 Subject: [PATCH 068/262] dnn(test): add extra IR models, more checks in IE testing code --- modules/dnn/test/test_ie_models.cpp | 98 ++++++++++++++++++++++++++--- 1 file changed, 88 insertions(+), 10 deletions(-) diff --git a/modules/dnn/test/test_ie_models.cpp b/modules/dnn/test/test_ie_models.cpp index 06d2e1776d..3407e95e9b 100644 --- a/modules/dnn/test/test_ie_models.cpp +++ b/modules/dnn/test/test_ie_models.cpp @@ -112,6 +112,25 @@ static const std::map& getOpenVINOTestMo "intel/age-gender-recognition-retail-0013/FP16/age-gender-recognition-retail-0013", "intel/age-gender-recognition-retail-0013/FP32/age-gender-recognition-retail-0013" }}, +#endif +#if INF_ENGINE_RELEASE >= 2021020000 + // OMZ: 2020.2 + { "face-detection-0105", { + "intel/face-detection-0105/FP32/face-detection-0105", + "intel/face-detection-0105/FP16/face-detection-0105" + }}, + { "face-detection-0106", { + "intel/face-detection-0106/FP32/face-detection-0106", + "intel/face-detection-0106/FP16/face-detection-0106" + }}, +#endif +#if INF_ENGINE_RELEASE >= 2021040000 + // OMZ: 2021.4 + { "person-vehicle-bike-detection-2004", { + "intel/person-vehicle-bike-detection-2004/FP32/person-vehicle-bike-detection-2004", + "intel/person-vehicle-bike-detection-2004/FP16/person-vehicle-bike-detection-2004" + //"intel/person-vehicle-bike-detection-2004/FP16-INT8/person-vehicle-bike-detection-2004" + }}, #endif }; @@ -145,10 +164,22 @@ inline static std::string getOpenVINOModel(const std::string &modelName, bool is static inline void genData(const InferenceEngine::TensorDesc& desc, Mat& m, Blob::Ptr& dataPtr) { const std::vector& dims = desc.getDims(); - m.create(std::vector(dims.begin(), dims.end()), CV_32F); - randu(m, -1, 1); - - dataPtr = make_shared_blob(desc, (float*)m.data); + if (desc.getPrecision() == InferenceEngine::Precision::FP32) + { + m.create(std::vector(dims.begin(), dims.end()), CV_32F); + randu(m, -1, 1); + dataPtr = make_shared_blob(desc, (float*)m.data); + } + else if (desc.getPrecision() == InferenceEngine::Precision::I32) + { + m.create(std::vector(dims.begin(), dims.end()), CV_32S); + randu(m, -100, 100); + dataPtr = make_shared_blob(desc, (int*)m.data); + } + else + { + FAIL() << "Unsupported precision: " << desc.getPrecision(); + } } void runIE(Target target, const std::string& xmlPath, const std::string& binPath, @@ -254,7 +285,16 @@ void runIE(Target target, const std::string& xmlPath, const std::string& binPath BlobMap inputBlobs; for (auto& it : net.getInputsInfo()) { - genData(it.second->getTensorDesc(), inputsMap[it.first], inputBlobs[it.first]); + const InferenceEngine::TensorDesc& desc = it.second->getTensorDesc(); + genData(desc, inputsMap[it.first], inputBlobs[it.first]); + if (cvtest::debugLevel > 0) + { + const std::vector& dims = desc.getDims(); + std::cout << "Input: '" << it.first << "' precison=" << desc.getPrecision() << " dims=" << dims.size() << " ["; + for (auto d : dims) + std::cout << " " << d; + std::cout << "] ocv_mat=" << inputsMap[it.first].size << " of " << typeToString(inputsMap[it.first].type()) << std::endl; + } } infRequest.SetInput(inputBlobs); @@ -263,7 +303,16 @@ void runIE(Target target, const std::string& xmlPath, const std::string& binPath BlobMap outputBlobs; for (auto& it : net.getOutputsInfo()) { - genData(it.second->getTensorDesc(), outputsMap[it.first], outputBlobs[it.first]); + const InferenceEngine::TensorDesc& desc = it.second->getTensorDesc(); + genData(desc, outputsMap[it.first], outputBlobs[it.first]); + if (cvtest::debugLevel > 0) + { + const std::vector& dims = desc.getDims(); + std::cout << "Output: '" << it.first << "' precison=" << desc.getPrecision() << " dims=" << dims.size() << " ["; + for (auto d : dims) + std::cout << " " << d; + std::cout << "] ocv_mat=" << outputsMap[it.first].size << " of " << typeToString(outputsMap[it.first].type()) << std::endl; + } } infRequest.SetOutput(outputBlobs); @@ -284,6 +333,12 @@ void runCV(Backend backendId, Target targetId, const std::string& xmlPath, const net.setPreferableTarget(targetId); std::vector outNames = net.getUnconnectedOutLayersNames(); + if (cvtest::debugLevel > 0) + { + std::cout << "OpenCV output names: " << outNames.size() << std::endl; + for (auto name : outNames) + std::cout << "- " << name << std::endl; + } std::vector outs; net.forward(outs, outNames); @@ -307,13 +362,26 @@ TEST_P(DNNTestOpenVINO, models) ASSERT_FALSE(backendId != DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 && backendId != DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) << "Inference Engine backend is required"; -#if INF_ENGINE_VER_MAJOR_EQ(2021040000) - if (targetId == DNN_TARGET_MYRIAD && ( - modelName == "person-detection-retail-0013" || // ncDeviceOpen:1013 Failed to find booted device after boot - modelName == "age-gender-recognition-retail-0013" // ncDeviceOpen:1013 Failed to find booted device after boot +#if INF_ENGINE_VER_MAJOR_GE(2021030000) + if (targetId == DNN_TARGET_MYRIAD && (false + || modelName == "person-detection-retail-0013" // ncDeviceOpen:1013 Failed to find booted device after boot + || modelName == "age-gender-recognition-retail-0013" // ncDeviceOpen:1013 Failed to find booted device after boot + || modelName == "face-detection-0105" // get_element_type() must be called on a node with exactly one output + || modelName == "face-detection-0106" // get_element_type() must be called on a node with exactly one output + || modelName == "person-vehicle-bike-detection-2004" // 2021.4+: ncDeviceOpen:1013 Failed to find booted device after boot ) ) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD, CV_DNN_BACKEND_INFERENCE_ENGINE_NGRAPH, CV_TEST_TAG_DNN_SKIP_IE_VERSION); + if (targetId == DNN_TARGET_OPENCL && (false + || modelName == "face-detection-0106" // Operation: 2278 of type ExperimentalDetectronPriorGridGenerator(op::v6) is not supported + ) + ) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL, CV_DNN_BACKEND_INFERENCE_ENGINE_NGRAPH, CV_TEST_TAG_DNN_SKIP_IE_VERSION); + if (targetId == DNN_TARGET_OPENCL_FP16 && (false + || modelName == "face-detection-0106" // Operation: 2278 of type ExperimentalDetectronPriorGridGenerator(op::v6) is not supported + ) + ) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16, CV_DNN_BACKEND_INFERENCE_ENGINE_NGRAPH, CV_TEST_TAG_DNN_SKIP_IE_VERSION); #endif #if INF_ENGINE_VER_MAJOR_GE(2020020000) @@ -350,6 +418,8 @@ TEST_P(DNNTestOpenVINO, models) if (targetId == DNN_TARGET_MYRIAD) resetMyriadDevice(); EXPECT_NO_THROW(runIE(targetId, xmlPath, binPath, inputsMap, ieOutputsMap)) << "runIE"; + if (targetId == DNN_TARGET_MYRIAD) + resetMyriadDevice(); EXPECT_NO_THROW(runCV(backendId, targetId, xmlPath, binPath, inputsMap, cvOutputsMap)) << "runCV"; double eps = 0; @@ -357,6 +427,14 @@ TEST_P(DNNTestOpenVINO, models) if (targetId == DNN_TARGET_CPU && checkHardwareSupport(CV_CPU_AVX_512F)) eps = 1e-5; #endif +#if INF_ENGINE_VER_MAJOR_GE(2021030000) + if (targetId == DNN_TARGET_CPU && modelName == "face-detection-0105") + eps = 2e-4; +#endif +#if INF_ENGINE_VER_MAJOR_GE(2021040000) + if (targetId == DNN_TARGET_CPU && modelName == "person-vehicle-bike-detection-2004") + eps = 1e-6; +#endif EXPECT_EQ(ieOutputsMap.size(), cvOutputsMap.size()); for (auto& srcIt : ieOutputsMap) From fbde0c6c961e631c4c91ce4c94d6c1a891e282dd Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Wed, 14 Jul 2021 23:31:41 +0000 Subject: [PATCH 069/262] dnn(ie): fix handling of 1D and non-32F outputs of InferenceEngine --- modules/dnn/src/dnn.cpp | 29 ++++++++--- modules/dnn/src/ie_ngraph.cpp | 97 ++++++++++++++++++++++++++++++++--- modules/dnn/src/ie_ngraph.hpp | 10 +++- 3 files changed, 120 insertions(+), 16 deletions(-) diff --git a/modules/dnn/src/dnn.cpp b/modules/dnn/src/dnn.cpp index 45be6eb97c..8182394387 100644 --- a/modules/dnn/src/dnn.cpp +++ b/modules/dnn/src/dnn.cpp @@ -1944,7 +1944,10 @@ struct Net::Impl : public detail::NetImplBase Ptr ieNode = node.dynamicCast(); CV_Assert(!ieNode.empty()); - ieNode->net->reset(); + + CV_Assert(ieNode->net); + InfEngineNgraphNet& ienet = *ieNode->net; + ienet.reset(); for (it = layers.begin(); it != layers.end(); ++it) { @@ -1961,16 +1964,26 @@ struct Net::Impl : public detail::NetImplBase { for (int i = 0; i < ld.outputBlobsWrappers.size(); ++i) { - InferenceEngine::DataPtr dataPtr = ngraphDataNode(ld.outputBlobsWrappers[i]); - dataPtr->setName(ld.name); + auto it = ienet.outputsDesc.find(ld.name); + if (it != ienet.outputsDesc.end()) + { + const InferenceEngine::TensorDesc& descriptor = it->second; + InferenceEngine::DataPtr dataPtr = ngraphDataOutputNode(ld.outputBlobsWrappers[i], descriptor, ld.name); + dataPtr->setName(ld.name); + } + else + { + InferenceEngine::DataPtr dataPtr = ngraphDataNode(ld.outputBlobsWrappers[i]); + dataPtr->setName(ld.name); + } } } - ieNode->net->addBlobs(ld.inputBlobsWrappers); - ieNode->net->addBlobs(ld.outputBlobsWrappers); + ienet.addBlobs(ld.inputBlobsWrappers); + ienet.addBlobs(ld.outputBlobsWrappers); ld.skip = true; } layers[lastLayerId].skip = false; - ieNode->net->init((Target)preferableTarget); + ienet.init((Target)preferableTarget); return; } @@ -3719,8 +3732,8 @@ void Net::forward(OutputArrayOfArrays outputBlobs, matvec.push_back(impl->getBlob(pins[i])); } - std::vector & outputvec = *(std::vector *)outputBlobs.getObj(); - outputvec = matvec; + outputBlobs.create((int)matvec.size(), 1, CV_32F/*FIXIT*/, -1); // allocate vector + outputBlobs.assign(matvec); } void Net::forward(std::vector >& outputBlobs, diff --git a/modules/dnn/src/ie_ngraph.cpp b/modules/dnn/src/ie_ngraph.cpp index e6c219f13e..6736590161 100644 --- a/modules/dnn/src/ie_ngraph.cpp +++ b/modules/dnn/src/ie_ngraph.cpp @@ -789,21 +789,32 @@ void NgraphBackendLayer::forward(InputArrayOfArrays inputs, OutputArrayOfArrays } -static InferenceEngine::Layout estimateLayout(const Mat& m) +static InferenceEngine::Layout estimateLayout(int dims) { - if (m.dims == 4) + if (dims == 4) return InferenceEngine::Layout::NCHW; - else if (m.dims == 3) + else if (dims == 3) return InferenceEngine::Layout::CHW; - else if (m.dims == 2) + else if (dims == 2) return InferenceEngine::Layout::NC; - else if (m.dims == 1) + else if (dims == 1) return InferenceEngine::Layout::C; - else if (m.dims == 5) + else if (dims == 5) return InferenceEngine::Layout::NCDHW; else return InferenceEngine::Layout::ANY; } +static inline +InferenceEngine::Layout estimateLayout(size_t dims) +{ + return estimateLayout((int)dims); +} + +static inline +InferenceEngine::Layout estimateLayout(const Mat& m) +{ + return estimateLayout(m.dims); +} static InferenceEngine::DataPtr wrapToInfEngineDataNode(const Mat& m, const std::string& name = "") { @@ -839,6 +850,7 @@ InferenceEngine::Blob::Ptr wrapToNgraphBlob(const Mat& m, InferenceEngine::Layou NgraphBackendWrapper::NgraphBackendWrapper(int targetId, const cv::Mat& m) : BackendWrapper(DNN_BACKEND_INFERENCE_ENGINE_NGRAPH, targetId) + , host((Mat*)&m) { dataPtr = wrapToInfEngineDataNode(m); blob = wrapToNgraphBlob(m, estimateLayout(m)); @@ -890,7 +902,11 @@ InferenceEngine::Blob::Ptr copyBlob(const InferenceEngine::Blob::Ptr& blob) copy = InferenceEngine::make_shared_blob(description); } else - CV_Error(Error::StsNotImplemented, "Unsupported blob precision"); + { + std::ostringstream msg; + msg << precision; + CV_Error_(Error::StsNotImplemented, ("Unsupported blob precision: %s", msg.str().c_str())); + } copy->allocate(); return copy; } @@ -903,6 +919,66 @@ InferenceEngine::DataPtr ngraphDataNode(const Ptr& ptr) return p->dataPtr; } +static +InferenceEngine::Blob::Ptr reallocateBlob(Mat &m, const InferenceEngine::TensorDesc& description) +{ + auto dims = description.getDims(); + auto layout = estimateLayout(dims.size()); + MatShape matShape(dims.begin(), dims.end()); + if (description.getPrecision() == InferenceEngine::Precision::FP32) + { + m.create(matShape, CV_32FC1); + return InferenceEngine::make_shared_blob( + {description.getPrecision(), dims, layout}, (float*)m.data); + } + else if (description.getPrecision() == InferenceEngine::Precision::I32) + { + m.create(matShape, CV_32SC1); + return InferenceEngine::make_shared_blob( + {description.getPrecision(), dims, layout}, (int*)m.data); + } + else if (description.getPrecision() == InferenceEngine::Precision::U8) + { + m.create(matShape, CV_8UC1); + return InferenceEngine::make_shared_blob( + {description.getPrecision(), dims, layout}, (uchar*)m.data); + } + std::ostringstream msg; + msg << "Unsupported IE precision: " << description.getPrecision(); + CV_Error(Error::StsNotImplemented, msg.str()); +} + +InferenceEngine::DataPtr ngraphDataOutputNode( + const Ptr& ptr, + const InferenceEngine::TensorDesc& description, + const std::string name) +{ + CV_Assert(!ptr.empty()); + Ptr p = ptr.dynamicCast(); + CV_Assert(!p.empty()); + NgraphBackendWrapper& w = *p; + const InferenceEngine::TensorDesc& blobDesc = w.blob.get()->getTensorDesc(); + auto dims = description.getDims(); + bool reallocate = false; + if (blobDesc.getPrecision() != description.getPrecision()) + { + reallocate = true; + CV_LOG_WARNING(NULL, "Reallocate output '" << name << "' blob due to wrong precision: " << blobDesc.getPrecision() << " => " << description.getPrecision() << " ndims=" << dims.size()); + } + if (dims.size() != blobDesc.getDims().size()) + { + reallocate = true; + CV_LOG_WARNING(NULL, "Reallocate output '" << name << "' blob due to wrong dims: " << blobDesc.getDims().size() << " => " << dims.size()); + } + if (reallocate) + { + auto layout = estimateLayout(dims.size()); + w.dataPtr = InferenceEngine::DataPtr(new InferenceEngine::Data(name, + {description.getPrecision(), dims, layout})); + w.blob = reallocateBlob(*w.host, description); + } + return w.dataPtr; +} void forwardNgraph(const std::vector >& outBlobsWrappers, Ptr& node, bool isAsync) @@ -918,6 +994,13 @@ void InfEngineNgraphNet::reset() allBlobs.clear(); infRequests.clear(); isInit = false; + + outputsDesc.clear(); + for (const auto& it : cnn.getOutputsInfo()) + { + const std::string& name = it.first; + outputsDesc.insert({name, it.second->getTensorDesc()}); + } } void InfEngineNgraphNet::addBlobs(const std::vector >& ptrs) diff --git a/modules/dnn/src/ie_ngraph.hpp b/modules/dnn/src/ie_ngraph.hpp index 7a8c4bef8d..617f1d4542 100644 --- a/modules/dnn/src/ie_ngraph.hpp +++ b/modules/dnn/src/ie_ngraph.hpp @@ -54,7 +54,8 @@ public: void setNodePtr(std::shared_ptr* ptr); void reset(); -private: + +//private: detail::NetImplBase& netImpl_; void release(); @@ -89,6 +90,8 @@ private: bool hasNetOwner; std::vector requestedOutputs; std::unordered_set> unconnectedNodes; + + std::map outputsDesc; }; class InfEngineNgraphNode : public BackendNode @@ -121,12 +124,17 @@ public: virtual void copyToHost() CV_OVERRIDE; virtual void setHostDirty() CV_OVERRIDE; + Mat* host; InferenceEngine::DataPtr dataPtr; InferenceEngine::Blob::Ptr blob; AsyncArray futureMat; }; InferenceEngine::DataPtr ngraphDataNode(const Ptr& ptr); +InferenceEngine::DataPtr ngraphDataOutputNode( + const Ptr& ptr, + const InferenceEngine::TensorDesc& description, + const std::string name); // This is a fake class to run networks from Model Optimizer. Objects of that // class simulate responses of layers are imported by OpenCV and supported by From 96d35f7c54e6482e2c041c20dbc78bb3ef568a88 Mon Sep 17 00:00:00 2001 From: SamFC10 Date: Fri, 16 Jul 2021 09:39:41 +0530 Subject: [PATCH 070/262] Fix convolution asymmetric padding bug in onnx importer --- modules/dnn/src/onnx/onnx_importer.cpp | 39 +++++++++++++++++++++++++ modules/dnn/test/test_onnx_importer.cpp | 1 + 2 files changed, 40 insertions(+) diff --git a/modules/dnn/src/onnx/onnx_importer.cpp b/modules/dnn/src/onnx/onnx_importer.cpp index db16cfd56d..ec61a9707e 100644 --- a/modules/dnn/src/onnx/onnx_importer.cpp +++ b/modules/dnn/src/onnx/onnx_importer.cpp @@ -1263,6 +1263,45 @@ void ONNXImporter::handleNode(const opencv_onnx::NodeProto& node_proto_) } int outCn = layerParams.blobs.empty() ? outShapes[node_proto.input(1)][0] : layerParams.blobs[0].size[0]; layerParams.set("num_output", outCn); + + // Check for asymmetric padding in Conv2D + if (layerParams.has("pad")) + { + bool asymmetricPadding = false; + DictValue pads = layerParams.get("pad"); + const int dims = pads.size() / 2; + for (int i = 0; i < dims; ++i) + { + if (pads.get(i) != pads.get(i + dims)) + { + asymmetricPadding = true; + break; + } + } + if (asymmetricPadding && pads.size() == 4) // [pad_t, pad_l, pad_b, pad_r] + { + layerParams.erase("pad"); + // No paddings required for N, C axis + std::vector paddings(4, 0); + // Add paddings for H, W axis + for (int i = 0; i < dims; ++i) + { + paddings.push_back(pads.get(i)); + paddings.push_back(pads.get(dims + i)); + } + LayerParams padLp; + padLp.name = layerParams.name + "/pad"; + padLp.type = "Padding"; + padLp.set("paddings", DictValue::arrayInt(&paddings[0], paddings.size())); + + opencv_onnx::NodeProto proto; + proto.add_input(node_proto.input(0)); + proto.add_output(padLp.name); + + addLayer(padLp, proto); + node_proto.set_input(0, padLp.name); + } + } } else if (layer_type == "ConvTranspose") { diff --git a/modules/dnn/test/test_onnx_importer.cpp b/modules/dnn/test/test_onnx_importer.cpp index 600f727d7d..3923068dbf 100644 --- a/modules/dnn/test/test_onnx_importer.cpp +++ b/modules/dnn/test/test_onnx_importer.cpp @@ -109,6 +109,7 @@ TEST_P(Test_ONNX_layers, MaxPooling_2) TEST_P(Test_ONNX_layers, Convolution) { testONNXModels("convolution"); + testONNXModels("conv_asymmetric_pads"); } TEST_P(Test_ONNX_layers, Convolution_variable_weight) From c30078c5a3e852e6df7ec825280ecd8bc03e3107 Mon Sep 17 00:00:00 2001 From: Smirnov Egor Date: Tue, 13 Jul 2021 12:20:35 +0300 Subject: [PATCH 071/262] add NotImplemented layer --- modules/dnn/src/dnn.cpp | 28 ++- modules/dnn/src/dnn_common.hpp | 9 + .../dnn/src/layers/not_implemented_layer.cpp | 194 ++++++++++++++++++ modules/dnn/src/tensorflow/tf_importer.cpp | 95 ++++++--- modules/dnn/test/test_tf_importer.cpp | 33 +++ 5 files changed, 326 insertions(+), 33 deletions(-) create mode 100644 modules/dnn/src/layers/not_implemented_layer.cpp diff --git a/modules/dnn/src/dnn.cpp b/modules/dnn/src/dnn.cpp index 668cce8fa6..2d1a093ef4 100644 --- a/modules/dnn/src/dnn.cpp +++ b/modules/dnn/src/dnn.cpp @@ -99,6 +99,15 @@ bool DNN_DIAGNOSTICS_RUN = false; void enableModelDiagnostics(bool isDiagnosticsMode) { DNN_DIAGNOSTICS_RUN = isDiagnosticsMode; + + if (DNN_DIAGNOSTICS_RUN) + { + detail::NotImplemented::Register(); + } + else + { + detail::NotImplemented::unRegister(); + } } using std::vector; @@ -4001,13 +4010,24 @@ int Net::addLayer(const String &name, const String &type, LayerParams ¶ms) { CV_TRACE_FUNCTION(); - if (impl->getLayerId(name) >= 0) + int id = impl->getLayerId(name); + if (id >= 0) { - CV_Error(Error::StsBadArg, "Layer \"" + name + "\" already into net"); - return -1; + if (!DNN_DIAGNOSTICS_RUN || type != "NotImplemented") + { + CV_Error(Error::StsBadArg, "Layer \"" + name + "\" already into net"); + return -1; + } + else + { + LayerData& ld = impl->layers.find(id)->second; + ld.type = type; + ld.params = params; + return -1; + } } - int id = ++impl->lastLayerId; + id = ++impl->lastLayerId; impl->layerNameToId.insert(std::make_pair(name, id)); impl->layers.insert(std::make_pair(id, LayerData(id, name, type, params))); if (params.get("has_dynamic_shapes", false)) diff --git a/modules/dnn/src/dnn_common.hpp b/modules/dnn/src/dnn_common.hpp index ff8f5e8467..46fae41cc2 100644 --- a/modules/dnn/src/dnn_common.hpp +++ b/modules/dnn/src/dnn_common.hpp @@ -15,6 +15,15 @@ void initializeLayerFactory(); namespace detail { +class NotImplemented : public Layer +{ +public: + static Ptr create(const LayerParams ¶ms); + + static void Register(); + static void unRegister(); +}; + struct NetImplBase { const int networkId; // network global identifier diff --git a/modules/dnn/src/layers/not_implemented_layer.cpp b/modules/dnn/src/layers/not_implemented_layer.cpp new file mode 100644 index 0000000000..c4b1343902 --- /dev/null +++ b/modules/dnn/src/layers/not_implemented_layer.cpp @@ -0,0 +1,194 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. + +#include "../precomp.hpp" +#include "../dnn_common.hpp" + +namespace cv { namespace dnn { +CV__DNN_INLINE_NS_BEGIN + +namespace detail { + +class NotImplementedImpl CV_FINAL : public NotImplemented +{ +public: + NotImplementedImpl(const LayerParams& params) + { + setParamsFrom(params); + CV_Assert(params.has("type")); + std::stringstream ss; + ss << "Node for layer '" << params.name << "' of type '" << params.get("type") << "' wasn't initialized."; + msg = ss.str(); + } + + CV_DEPRECATED_EXTERNAL + virtual void finalize(const std::vector &input, std::vector &output) CV_OVERRIDE + { + CV_Error(Error::StsNotImplemented, msg); + } + + virtual void finalize(InputArrayOfArrays inputs, OutputArrayOfArrays outputs) CV_OVERRIDE + { + CV_Error(Error::StsNotImplemented, msg); + } + + CV_DEPRECATED_EXTERNAL + virtual void forward(std::vector &input, std::vector &output, std::vector &internals) CV_OVERRIDE + { + CV_Error(Error::StsNotImplemented, msg); + } + + virtual void forward(InputArrayOfArrays inputs_arr, OutputArrayOfArrays outputs_arr, OutputArrayOfArrays internals_arr) CV_OVERRIDE + { + CV_Error(Error::StsNotImplemented, msg); + } + + void forward_fallback(InputArrayOfArrays inputs, OutputArrayOfArrays outputs, OutputArrayOfArrays internals) + { + CV_Error(Error::StsNotImplemented, msg); + } + + CV_DEPRECATED_EXTERNAL + void finalize(const std::vector &inputs, CV_OUT std::vector &outputs) + { + CV_Error(Error::StsNotImplemented, msg); + } + + CV_DEPRECATED std::vector finalize(const std::vector &inputs) + { + CV_Error(Error::StsNotImplemented, msg); + } + + CV_DEPRECATED void run(const std::vector &inputs, + CV_OUT std::vector &outputs, + CV_IN_OUT std::vector &internals) + { + CV_Error(Error::StsNotImplemented, msg); + } + + virtual int inputNameToIndex(String inputName) CV_OVERRIDE + { + CV_Error(Error::StsNotImplemented, msg); + } + + virtual int outputNameToIndex(const String& outputName) CV_OVERRIDE + { + CV_Error(Error::StsNotImplemented, msg); + } + + virtual bool supportBackend(int backendId) CV_OVERRIDE + { + CV_Error(Error::StsNotImplemented, msg); + } + + virtual Ptr initHalide(const std::vector > &inputs) CV_OVERRIDE + { + CV_Error(Error::StsNotImplemented, msg); + } + + virtual Ptr initInfEngine(const std::vector > &inputs) CV_OVERRIDE + { + CV_Error(Error::StsNotImplemented, msg); + } + + virtual Ptr initNgraph(const std::vector > &inputs, + const std::vector >& nodes) CV_OVERRIDE + { + CV_Error(Error::StsNotImplemented, msg); + } + + virtual Ptr initVkCom(const std::vector > &inputs) CV_OVERRIDE + { + CV_Error(Error::StsNotImplemented, msg); + } + + virtual Ptr initCUDA( + void *context, + const std::vector>& inputs, + const std::vector>& outputs + ) CV_OVERRIDE + { + CV_Error(Error::StsNotImplemented, msg); + } + + virtual void applyHalideScheduler(Ptr& node, + const std::vector &inputs, + const std::vector &outputs, + int targetId) const CV_OVERRIDE + { + CV_Error(Error::StsNotImplemented, msg); + } + + virtual Ptr tryAttach(const Ptr& node) CV_OVERRIDE + { + CV_Error(Error::StsNotImplemented, msg); + } + + virtual bool setActivation(const Ptr& layer) CV_OVERRIDE + { + CV_Error(Error::StsNotImplemented, msg); + } + + virtual bool tryFuse(Ptr& top) CV_OVERRIDE + { + CV_Error(Error::StsNotImplemented, msg); + } + + virtual void getScaleShift(Mat& scale, Mat& shift) const CV_OVERRIDE + { + CV_Error(Error::StsNotImplemented, msg); + } + + virtual void unsetAttached() CV_OVERRIDE + { + CV_Error(Error::StsNotImplemented, msg); + } + + virtual bool getMemoryShapes(const std::vector &inputs, + const int requiredOutputs, + std::vector &outputs, + std::vector &internals) const CV_OVERRIDE + { + CV_Error(Error::StsNotImplemented, msg); + } + + virtual int64 getFLOPS(const std::vector &inputs, + const std::vector &outputs) const CV_OVERRIDE + { + CV_Error(Error::StsNotImplemented, msg); + } + + virtual bool updateMemoryShapes(const std::vector &inputs) CV_OVERRIDE + { + CV_Error(Error::StsNotImplemented, msg); + } + +private: + std::string msg; +}; + +Ptr NotImplemented::create(const LayerParams& params) +{ + return makePtr(params); +} + +Ptr notImplementedRegisterer(LayerParams ¶ms) +{ + return detail::NotImplemented::create(params); +} + +void NotImplemented::Register() +{ + LayerFactory::registerLayer("NotImplemented", detail::notImplementedRegisterer); +} + +void NotImplemented::unRegister() +{ + LayerFactory::unregisterLayer("NotImplemented"); +} + +} // namespace detail + +CV__DNN_INLINE_NS_END +}} // namespace cv::dnn diff --git a/modules/dnn/src/tensorflow/tf_importer.cpp b/modules/dnn/src/tensorflow/tf_importer.cpp index 15f88007b4..10670bfef9 100644 --- a/modules/dnn/src/tensorflow/tf_importer.cpp +++ b/modules/dnn/src/tensorflow/tf_importer.cpp @@ -466,6 +466,8 @@ void ExcludeLayer(tensorflow::GraphDef& net, const int layer_index, const int in net.mutable_node()->DeleteSubrange(layer_index, 1); } +class LayerHandler; + class TFImporter { public: @@ -473,6 +475,7 @@ public: TFImporter(Net& net, const char *dataModel, size_t lenModel, const char *dataConfig = NULL, size_t lenConfig = 0); protected: + std::unique_ptr layerHandler; std::unique_ptr utilNet; Net& dstNet; void populateNet(); @@ -514,6 +517,7 @@ protected: private: void addPermuteLayer(const int* order, const std::string& permName, Pin& inpId); + friend class LayerHandler; typedef void (TFImporter::*TFImporterNodeParser)(tensorflow::GraphDef&, const tensorflow::NodeDef&, LayerParams&); typedef std::map DispatchMap; @@ -554,6 +558,20 @@ private: void parseCustomLayer (tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams); }; +class LayerHandler +{ +public: + LayerHandler(TFImporter* importer_); + ~LayerHandler() = default; + + bool handleMissing(const opencv_tensorflow::NodeDef& layer); + void handleFailed(const opencv_tensorflow::NodeDef& layer); + +private: + TFImporter* importer; + std::set layers; +}; + const TFImporter::DispatchMap TFImporter::buildDispatchMap() { static DispatchMap dispatch; @@ -2340,7 +2358,8 @@ void TFImporter::parseCustomLayer(tensorflow::GraphDef& net, const tensorflow::N } TFImporter::TFImporter(Net& net, const char *model, const char *config) - : utilNet(DNN_DIAGNOSTICS_RUN ? new Net : nullptr), + : layerHandler(DNN_DIAGNOSTICS_RUN ? new LayerHandler(this) : nullptr), + utilNet(DNN_DIAGNOSTICS_RUN ? new Net : nullptr), dstNet(DNN_DIAGNOSTICS_RUN ? *utilNet : net), dispatch(buildDispatchMap()) { if (model && model[0]) @@ -2362,7 +2381,8 @@ TFImporter::TFImporter( const char *dataModel, size_t lenModel, const char *dataConfig, size_t lenConfig ) - : utilNet(DNN_DIAGNOSTICS_RUN ? new Net : nullptr), + : layerHandler(DNN_DIAGNOSTICS_RUN ? new LayerHandler(this) : nullptr), + utilNet(DNN_DIAGNOSTICS_RUN ? new Net : nullptr), dstNet(DNN_DIAGNOSTICS_RUN ? *utilNet : net), dispatch(buildDispatchMap()) { if (dataModel != NULL && lenModel > 0) @@ -2620,11 +2640,6 @@ DataLayout TFImporter::predictOutputDataLayout(const tensorflow::NodeDef& layer) return it->second; } -Ptr dummy_constructor(LayerParams & params) -{ - return new Layer(params); -} - void TFImporter::populateNet() { CV_Assert(netBin.ByteSize() || netTxt.ByteSize()); @@ -2727,7 +2742,6 @@ void TFImporter::populateNet() addConstNodes(netBin, value_id, layers_to_ignore); addConstNodes(netTxt, value_id, layers_to_ignore); - for (int li = 0; li < layersSize; li++) { const tensorflow::NodeDef& layer = net.node(li); @@ -2785,41 +2799,64 @@ void TFImporter::parseNode(const tensorflow::NodeDef& layer) { ((*this).*(iter->second))(net, layer, layerParams); } - else + else if (!DNN_DIAGNOSTICS_RUN || !layerHandler->handleMissing(layer)) { - if (DNN_DIAGNOSTICS_RUN && !LayerFactory::createLayerInstance(type, layerParams)) - { - CV_LOG_ERROR(NULL, "DNN/TF: Node='" << name << "' of type='"<< type - << "' is not supported. This error won't be displayed again."); - LayerFactory::registerLayer(type, dummy_constructor); - } - parseCustomLayer(net, layer, layerParams); } } catch (const std::exception& e) { - if (!DNN_DIAGNOSTICS_RUN) + CV_LOG_ERROR(NULL, "DNN/TF: Can't parse layer for node='" << name << "' of type='" << type + << "'. Exception: " << e.what()); + + if (DNN_DIAGNOSTICS_RUN) { - CV_LOG_ERROR(NULL, "DNN/TF: Can't parse layer for node='" << name << "' of type='" << type - << "'. Exception: " << e.what()); - throw; + layerHandler->handleFailed(layer); } else { - CV_LOG_ERROR(NULL, "DNN/TF: Can't parse layer for node='" << name << "' of type='" << type - << "'. Exception: " << e.what()); - - // internal layer failure (didnt call addLayer) - if (dstNet.getLayerId(name) == -1) - { - int id = dstNet.addLayer(name, type, layerParams); - layer_id[name] = id; - } + throw; } } } +LayerHandler::LayerHandler(TFImporter* importer_) : importer(importer_) {} + +void LayerHandler::handleFailed(const opencv_tensorflow::NodeDef& layer) +{ + LayerParams lp; + lp.name = layer.name(); + lp.type = "NotImplemented"; + lp.set("type", layer.op()); + + // the layer will be created or its params and type will be replaced + int id = importer->dstNet.addLayer(lp.name, "NotImplemented", lp); + if (id != -1) // internal layer failure before the call to addLayer() + { + importer->layer_id[lp.name] = id; + } +} + +bool LayerHandler::handleMissing(const opencv_tensorflow::NodeDef& layer) +{ + LayerParams lp; + // If we didn't add it, but can create it, it's custom and not missing. + if (layers.find(layer.op()) == layers.end() && LayerFactory::createLayerInstance(layer.op(), lp)) + { + return false; + } + + if (layers.insert(layer.op()).second) + { + CV_LOG_ERROR(NULL, "DNN/TF: Node='" << layer.name() << "' of type='"<< layer.op() + << "' is not supported. This error won't be displayed again."); + } + + handleFailed(layer); + + return true; +} + } // namespace #endif //HAVE_PROTOBUF diff --git a/modules/dnn/test/test_tf_importer.cpp b/modules/dnn/test/test_tf_importer.cpp index 2c36134724..35751b4824 100644 --- a/modules/dnn/test/test_tf_importer.cpp +++ b/modules/dnn/test/test_tf_importer.cpp @@ -568,6 +568,39 @@ TEST_P(Test_TensorFlow_layers, l2_normalize_3d) runTensorFlowNet("l2_normalize_3d"); } +class Test_TensorFlow_diagnostics : public DNNTestLayer { +public: + Test_TensorFlow_diagnostics() + { + enableModelDiagnostics(true); + } + + ~Test_TensorFlow_diagnostics() + { + enableModelDiagnostics(false); + } + + void runFailingTensorFlowNet(const std::string& prefix, bool hasText = false) + { + std::string netPath = path(prefix + "_net.pb"); + std::string netConfig = (hasText ? path(prefix + "_net.pbtxt") : ""); + + Net net = readNetFromTensorflow(netPath, netConfig); + } +}; + +TEST_P(Test_TensorFlow_diagnostics, not_implemented_layer) +{ + runFailingTensorFlowNet("not_implemented_layer"); +} + +TEST_P(Test_TensorFlow_diagnostics, broken_parameters) +{ + runFailingTensorFlowNet("broken_layer"); +} + +INSTANTIATE_TEST_CASE_P(/**/, Test_TensorFlow_diagnostics, dnnBackendsAndTargets()); + class Test_TensorFlow_nets : public DNNTestLayer {}; TEST_P(Test_TensorFlow_nets, MobileNet_SSD) From 2062a7ca8fd313af7ebcd510d8dca10b77858ec3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sa=C5=A1a=20Bajtl?= Date: Sun, 18 Jul 2021 10:12:39 +0200 Subject: [PATCH 072/262] Bugfix on import script with web worker. --- modules/js/src/make_umd.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/js/src/make_umd.py b/modules/js/src/make_umd.py index bed6ee9bcc..1096a8eb31 100644 --- a/modules/js/src/make_umd.py +++ b/modules/js/src/make_umd.py @@ -95,7 +95,7 @@ def make_umd(opencvjs, cvjs): root.cv = factory(); } else if (typeof importScripts === 'function') { // Web worker - root.cv = factory; + root.cv = factory(); } else { // Other shells, e.g. d8 root.cv = factory(); From 863ab0e72ee9aa530cbf936489be639837739b86 Mon Sep 17 00:00:00 2001 From: Lukas-Alexander Weber <32765578+lukasalexanderweber@users.noreply.github.com> Date: Tue, 20 Jul 2021 10:59:15 +0200 Subject: [PATCH 073/262] fix TypeError when specifying compose_megapix without rounding the composed image sizes (variable "sz") they will be odly fractions of a pixel (e.g. (5300.965, 3772.897)) and therefore cause a "TypeError: integer argument expected, got float" in line 456 roi = warper.warpRoi(sz, K, cameras[i].R) --- samples/python/stitching_detailed.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/samples/python/stitching_detailed.py b/samples/python/stitching_detailed.py index a7e316105e..4ee29048d1 100644 --- a/samples/python/stitching_detailed.py +++ b/samples/python/stitching_detailed.py @@ -450,7 +450,8 @@ def main(): cameras[i].focal *= compose_work_aspect cameras[i].ppx *= compose_work_aspect cameras[i].ppy *= compose_work_aspect - sz = (full_img_sizes[i][0] * compose_scale, full_img_sizes[i][1] * compose_scale) + sz = (int(round(full_img_sizes[i][0] * compose_scale)), + int(round(full_img_sizes[i][1] * compose_scale))) K = cameras[i].K().astype(np.float32) roi = warper.warpRoi(sz, K, cameras[i].R) corners.append(roi[0:2]) From 3817f3a89bec25347568dab775fdfa1eea4704cd Mon Sep 17 00:00:00 2001 From: Xiaoxiao Tian Date: Wed, 21 Jul 2021 13:32:50 +0800 Subject: [PATCH 074/262] fix: ocv_target_link_libraries could not handle the keyword rightly #20430 --- cmake/OpenCVUtils.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/OpenCVUtils.cmake b/cmake/OpenCVUtils.cmake index d03dc9c551..1e0ea947ef 100644 --- a/cmake/OpenCVUtils.cmake +++ b/cmake/OpenCVUtils.cmake @@ -1488,8 +1488,8 @@ function(ocv_target_link_libraries target) if(NOT LINK_PENDING STREQUAL "") __ocv_push_target_link_libraries(${LINK_MODE} ${LINK_PENDING}) set(LINK_PENDING "") - set(LINK_MODE "${dep}") endif() + set(LINK_MODE "${dep}") else() if(BUILD_opencv_world) if(OPENCV_MODULE_${dep}_IS_PART_OF_WORLD) From d29c7e787159fb01ea31b4673a07a52d752aa66d Mon Sep 17 00:00:00 2001 From: Francesco Petrogalli <25690309+fpetrogalli@users.noreply.github.com> Date: Wed, 21 Jul 2021 16:46:05 +0100 Subject: [PATCH 075/262] Merge pull request #20392 from fpetrogalli:aarch64-semihosting AArch64 semihosting * [ts] Disable filesystem support in the TS module. Because of this change, all the tests loading data will file, but tat least the core module can be tested with the following line: opencv_test_core --gtest_filter=-"*Core_InputOutput*:*Core_globbing.accuracy*" * [aarch64] Build OpenCV for AArch64 semihosting. This patch provide a toolchain file that allows to build the library for semihosting applications [1]. Minimal changes have been applied to the code to be able to compile with a baremetal toolchain. [1] https://developer.arm.com/documentation/100863/latest The option `CV_SEMIHOSTING` is used to guard the bits in the code that are specific to the target. To build the code: cmake ../opencv/ \ -DCMAKE_TOOLCHAIN_FILE=../opencv/platforms/semihosting/aarch64-semihosting.toolchain.cmake \ -DSEMIHOSTING_TOOLCHAIN_PATH=/path/to/baremetal-toolchain/bin/ \ -DBUILD_EXAMPLES=ON -GNinja A barematel toolchain for targeting aarch64 semihosting can be found at [2], under `aarch64-none-elf`. [2] https://developer.arm.com/tools-and-software/open-source-software/developer-tools/gnu-toolchain/gnu-a/downloads The folder `samples/semihosting` provides two example semihosting applications. The two binaries can be executed on the host platform with: qemu-aarch64 ./bin/example_semihosting_histogram qemu-aarch64 ./bin/example_semihosting_norm Similarly, the test and perf executables of the modules can be run with: qemu-aarch64 ./bin/opecv_[test|perf]_ Notice that filesystem support is disabled by the toolchain file, hence some of the test that depend on filesystem support will fail. * [semihosting] Remove blank like at the end of file. [NFC] The spurious blankline was reported by https://pullrequest.opencv.org/buildbot/builders/precommit_docs/builds/31158. * [semihosting] Make the raw pixel file generation OS independent. Use the facilities provided by Cmake to generate the header file instead of a shell script, so that the build doesn't fail on systems that do not have a unix shell. * [semihosting] Rename variable for semihosting compilation. * [semihosting] Move the cmake configuration to a variable file. * [semihosting] Make the guard macro private for the core module. * [semihosting] Remove space. [NFC] * [semihosting] Improve comment with information about semihosting. [NFC] * [semihosting] Update license statement on top of sourvce file. [NFC] * [semihosting] Replace BM_SUFFIX with SEMIHOSTING_SUFFIX. [NFC] * [semihosting] Remove double space. [NFC] * [semihosting] Add some text output to the sample applications. * [semihosting] Remove duplicate entry in cmake configuration. [NFCI] * [semihosting] Replace `long` with `int` in sample apps. [NFCI] * [semihosting] Use `configure_file` to create the random pixels. [NFCI] * [semihosting][bugfix] Fix name of cmakedefine variable. * [semihosting][samples] Use CV_8UC1 for grayscale images. [NFCI] * [semihosting] Add readme file. * [semihosting] Remove blank like at the end of README. [NFC] This fixes the failure at https://pullrequest.opencv.org/buildbot/builders/precommit_docs/builds/31272. --- CMakeLists.txt | 1 + cmake/vars/EnableModeVars.cmake | 3 ++ cmake/vars/OPENCV_SEMIHOSTING.cmake | 10 +++++ modules/calib3d/src/ap3p.cpp | 18 ++++---- modules/core/CMakeLists.txt | 4 ++ modules/core/src/parallel.cpp | 4 ++ modules/ts/src/ts.cpp | 2 + modules/ts/src/ts_gtest.cpp | 10 +++++ .../aarch64-semihosting.toolchain.cmake | 40 +++++++++++++++++ .../include/aarch64_semihosting_port.hpp | 42 ++++++++++++++++++ samples/CMakeLists.txt | 7 ++- samples/semihosting/CMakeLists.txt | 10 +++++ samples/semihosting/README.md | 27 ++++++++++++ samples/semihosting/histogram/CMakeLists.txt | 26 +++++++++++ samples/semihosting/histogram/histogram.cpp | 43 +++++++++++++++++++ samples/semihosting/include/CMakeLists.txt | 16 +++++++ samples/semihosting/include/raw_pixels.hpp.in | 11 +++++ samples/semihosting/norm/CMakeLists.txt | 25 +++++++++++ samples/semihosting/norm/norm.cpp | 33 ++++++++++++++ 19 files changed, 321 insertions(+), 11 deletions(-) create mode 100644 cmake/vars/OPENCV_SEMIHOSTING.cmake create mode 100644 platforms/semihosting/aarch64-semihosting.toolchain.cmake create mode 100644 platforms/semihosting/include/aarch64_semihosting_port.hpp create mode 100644 samples/semihosting/CMakeLists.txt create mode 100644 samples/semihosting/README.md create mode 100644 samples/semihosting/histogram/CMakeLists.txt create mode 100644 samples/semihosting/histogram/histogram.cpp create mode 100644 samples/semihosting/include/CMakeLists.txt create mode 100644 samples/semihosting/include/raw_pixels.hpp.in create mode 100644 samples/semihosting/norm/CMakeLists.txt create mode 100644 samples/semihosting/norm/norm.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index f4fe0385d1..b7e5b58837 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -513,6 +513,7 @@ OCV_OPTION(ENABLE_CONFIG_VERIFICATION "Fail build if actual configuration doesn' OCV_OPTION(OPENCV_ENABLE_MEMALIGN "Enable posix_memalign or memalign usage" ON) OCV_OPTION(OPENCV_DISABLE_FILESYSTEM_SUPPORT "Disable filesystem support" OFF) OCV_OPTION(OPENCV_DISABLE_THREAD_SUPPORT "Build the library without multi-threaded code." OFF) +OCV_OPTION(OPENCV_SEMIHOSTING "Build the library for semihosting target (Arm). See https://developer.arm.com/documentation/100863/latest." OFF) OCV_OPTION(ENABLE_PYLINT "Add target with Pylint checks" (BUILD_DOCS OR BUILD_EXAMPLES) IF (NOT CMAKE_CROSSCOMPILING AND NOT APPLE_FRAMEWORK) ) OCV_OPTION(ENABLE_FLAKE8 "Add target with Python flake8 checker" (BUILD_DOCS OR BUILD_EXAMPLES) IF (NOT CMAKE_CROSSCOMPILING AND NOT APPLE_FRAMEWORK) ) diff --git a/cmake/vars/EnableModeVars.cmake b/cmake/vars/EnableModeVars.cmake index b3c4e79c46..3f017af496 100644 --- a/cmake/vars/EnableModeVars.cmake +++ b/cmake/vars/EnableModeVars.cmake @@ -16,3 +16,6 @@ endmacro() variable_watch(OPENCV_DISABLE_THREAD_SUPPORT ocv_change_mode_var) set(OPENCV_DISABLE_THREAD_SUPPORT "${OPENCV_DISABLE_THREAD_SUPPORT}") + +variable_watch(OPENCV_SEMIHOSTING ocv_change_mode_var) +set(OPENCV_SEMIHOSTING "${OPENCV_SEMIHOSTING}") diff --git a/cmake/vars/OPENCV_SEMIHOSTING.cmake b/cmake/vars/OPENCV_SEMIHOSTING.cmake new file mode 100644 index 0000000000..66f21c7ebd --- /dev/null +++ b/cmake/vars/OPENCV_SEMIHOSTING.cmake @@ -0,0 +1,10 @@ +set(CV_TRACE OFF) + +# These third parties libraries are incompatible with the semihosting +# toolchain. +set(WITH_JPEG OFF) +set(WITH_OPENEXR OFF) +set(WITH_TIFF OFF) + +# Turn off `libpng` for some linking issues. +set(WITH_PNG OFF) diff --git a/modules/calib3d/src/ap3p.cpp b/modules/calib3d/src/ap3p.cpp index 386a4499ef..582b201b36 100644 --- a/modules/calib3d/src/ap3p.cpp +++ b/modules/calib3d/src/ap3p.cpp @@ -7,8 +7,6 @@ static inline double cbrt(double x) { return (double)cv::cubeRoot((float)x); }; #endif -using namespace std; - namespace { void solveQuartic(const double *factors, double *realRoots) { const double &a4 = factors[0]; @@ -30,29 +28,29 @@ void solveQuartic(const double *factors, double *realRoots) { double q3 = (72 * r4 * p4 - 2 * p4 * p4 * p4 - 27 * q4 * q4) / 432; // /=2 double t; // *=2 - complex w; + std::complex w; if (q3 >= 0) - w = -sqrt(static_cast >(q3 * q3 - p3 * p3 * p3)) - q3; + w = -std::sqrt(static_cast >(q3 * q3 - p3 * p3 * p3)) - q3; else - w = sqrt(static_cast >(q3 * q3 - p3 * p3 * p3)) - q3; + w = std::sqrt(static_cast >(q3 * q3 - p3 * p3 * p3)) - q3; if (w.imag() == 0.0) { - w.real(cbrt(w.real())); + w.real(std::cbrt(w.real())); t = 2.0 * (w.real() + p3 / w.real()); } else { w = pow(w, 1.0 / 3); t = 4.0 * w.real(); } - complex sqrt_2m = sqrt(static_cast >(-2 * p4 / 3 + t)); + std::complex sqrt_2m = sqrt(static_cast >(-2 * p4 / 3 + t)); double B_4A = -a3 / (4 * a4); double complex1 = 4 * p4 / 3 + t; #if defined(__clang__) && defined(__arm__) && (__clang_major__ == 3 || __clang_major__ == 4) && !defined(__ANDROID__) // details: https://github.com/opencv/opencv/issues/11135 // details: https://github.com/opencv/opencv/issues/11056 - complex complex2 = 2 * q4; - complex2 = complex(complex2.real() / sqrt_2m.real(), 0); + std::complex complex2 = 2 * q4; + complex2 = std::complex(complex2.real() / sqrt_2m.real(), 0); #else - complex complex2 = 2 * q4 / sqrt_2m; + std::complex complex2 = 2 * q4 / sqrt_2m; #endif double sqrt_2m_rh = sqrt_2m.real() / 2; double sqrt1 = sqrt(-(complex1 + complex2)).real() / 2; diff --git a/modules/core/CMakeLists.txt b/modules/core/CMakeLists.txt index 6a969e5fc3..13d0af4db8 100644 --- a/modules/core/CMakeLists.txt +++ b/modules/core/CMakeLists.txt @@ -157,6 +157,10 @@ if(OPENCV_DISABLE_THREAD_SUPPORT) ocv_target_compile_definitions(${the_module} PUBLIC "OPENCV_DISABLE_THREAD_SUPPORT=1") endif() +if(OPENCV_SEMIHOSTING) + ocv_target_compile_definitions(${the_module} PRIVATE "-DOPENCV_SEMIHOSTING") +endif(OPENCV_SEMIHOSTING) + if(HAVE_HPX) ocv_target_link_libraries(${the_module} LINK_PRIVATE "${HPX_LIBRARIES}") endif() diff --git a/modules/core/src/parallel.cpp b/modules/core/src/parallel.cpp index 1d4179b7b4..8fccd19798 100644 --- a/modules/core/src/parallel.cpp +++ b/modules/core/src/parallel.cpp @@ -888,6 +888,7 @@ T minNonZero(const T& val_1, const T& val_2) static int getNumberOfCPUs_() { +#ifndef OPENCV_SEMIHOSTING /* * Logic here is to try different methods of getting CPU counts and return * the minimum most value as it has high probablity of being right and safe. @@ -979,6 +980,9 @@ int getNumberOfCPUs_() #endif return ncpus != 0 ? ncpus : 1; +#else // OPENCV_SEMIHOSTING + return 1; +#endif //OPENCV_SEMIHOSTING } int getNumberOfCPUs() diff --git a/modules/ts/src/ts.cpp b/modules/ts/src/ts.cpp index 3aa403ad87..3af3a7b8d5 100644 --- a/modules/ts/src/ts.cpp +++ b/modules/ts/src/ts.cpp @@ -72,7 +72,9 @@ #if defined _WIN32 || defined WINCE # include #else +#if OPENCV_HAVE_FILESYSTEM_SUPPORT # include +#endif # include #endif diff --git a/modules/ts/src/ts_gtest.cpp b/modules/ts/src/ts_gtest.cpp index a65ef721a2..b3debd54d2 100644 --- a/modules/ts/src/ts_gtest.cpp +++ b/modules/ts/src/ts_gtest.cpp @@ -1067,6 +1067,7 @@ class GTEST_API_ UnitTestImpl { void AddTestInfo(Test::SetUpTestCaseFunc set_up_tc, Test::TearDownTestCaseFunc tear_down_tc, TestInfo* test_info) { +#if OPENCV_HAVE_FILESYSTEM_SUPPORT // In order to support thread-safe death tests, we need to // remember the original working directory when the test program // was first invoked. We cannot do this in RUN_ALL_TESTS(), as @@ -1079,6 +1080,7 @@ class GTEST_API_ UnitTestImpl { GTEST_CHECK_(!original_working_dir_.IsEmpty()) << "Failed to get the current working directory."; } +#endif GetTestCase(test_info->test_case_name(), test_info->type_param(), @@ -9165,6 +9167,7 @@ static bool IsPathSeparator(char c) { // Returns the current working directory, or "" if unsuccessful. FilePath FilePath::GetCurrentDir() { +#if OPENCV_HAVE_FILESYSTEM_SUPPORT #if GTEST_OS_WINDOWS_MOBILE || GTEST_OS_WINDOWS_PHONE || GTEST_OS_WINDOWS_RT // Windows CE doesn't have a current directory, so we just return // something reasonable. @@ -9183,6 +9186,9 @@ FilePath FilePath::GetCurrentDir() { # endif // GTEST_OS_NACL return FilePath(result == NULL ? "" : cwd); #endif // GTEST_OS_WINDOWS_MOBILE +#else // OPENCV_HAVE_FILESYSTEM_SUPPORT + return FilePath(""); +#endif // OPENCV_HAVE_FILESYSTEM_SUPPORT } // Returns a copy of the FilePath with the case-insensitive extension removed. @@ -9391,6 +9397,7 @@ bool FilePath::CreateDirectoriesRecursively() const { // directory for any reason, including if the parent directory does not // exist. Not named "CreateDirectory" because that's a macro on Windows. bool FilePath::CreateFolder() const { +#if OPENCV_HAVE_FILESYSTEM_SUPPORT #if GTEST_OS_WINDOWS_MOBILE FilePath removed_sep(this->RemoveTrailingPathSeparator()); LPCWSTR unicode = String::AnsiToUtf16(removed_sep.c_str()); @@ -9406,6 +9413,9 @@ bool FilePath::CreateFolder() const { return this->DirectoryExists(); // An error is OK if the directory exists. } return true; // No error. +#else // OPENCV_HAVE_FILESYSTEM_SUPPORT + return false; +#endif // OPENCV_HAVE_FILESYSTEM_SUPPORT } // If input name has a trailing separator character, remove it and return the diff --git a/platforms/semihosting/aarch64-semihosting.toolchain.cmake b/platforms/semihosting/aarch64-semihosting.toolchain.cmake new file mode 100644 index 0000000000..95bbda3bed --- /dev/null +++ b/platforms/semihosting/aarch64-semihosting.toolchain.cmake @@ -0,0 +1,40 @@ +# This file is part of OpenCV project. +# It is subject to the license terms in the LICENSE file found in the top-level directory +# of this distribution and at http://opencv.org/license.html + +set(CMAKE_SYSTEM_NAME Generic) +set(CMAKE_SYSTEM_PROCESSOR AArch64) + +set(CMAKE_TRY_COMPILE_TARGET_TYPE STATIC_LIBRARY) + +set(PORT_FILE ${CMAKE_SOURCE_DIR}/platforms/semihosting/include/aarch64_semihosting_port.hpp) + +set(COMMON_FLAGS "--specs=rdimon.specs -DOPENCV_INCLUDE_PORT_FILE=\\\"${PORT_FILE}\\\"") + +set(CMAKE_AR ${SEMIHOSTING_TOOLCHAIN_PATH}aarch64-none-elf-ar${CMAKE_EXECUTABLE_SUFFIX}) +set(CMAKE_ASM_COMPILER ${SEMIHOSTING_TOOLCHAIN_PATH}aarch64-none-elf-gcc${CMAKE_EXECUTABLE_SUFFIX}) +set(CMAKE_C_COMPILER ${SEMIHOSTING_TOOLCHAIN_PATH}aarch64-none-elf-gcc${CMAKE_EXECUTABLE_SUFFIX}) +set(CMAKE_CXX_COMPILER ${SEMIHOSTING_TOOLCHAIN_PATH}aarch64-none-elf-g++${CMAKE_EXECUTABLE_SUFFIX}) +set(CMAKE_LINKER ${SEMIHOSTING_TOOLCHAIN_PATH}aarch64-none-elf-ld${CMAKE_EXECUTABLE_SUFFIX}) +set(CMAKE_OBJCOPY ${SEMIHOSTING_TOOLCHAIN_PATH}aarch64-none-elf-objcopy${CMAKE_EXECUTABLE_SUFFIX} CACHE INTERNAL "") +set(CMAKE_RANLIB ${SEMIHOSTING_TOOLCHAIN_PATH}aarch64-none-elf-ranlib${CMAKE_EXECUTABLE_SUFFIX} CACHE INTERNAL "") +set(CMAKE_SIZE ${SEMIHOSTING_TOOLCHAIN_PATH}aarch64-none-elf-size${CMAKE_EXECUTABLE_SUFFIX} CACHE INTERNAL "") +set(CMAKE_STRIP ${SEMIHOSTING_TOOLCHAIN_PATH}aarch64-none-elf-strip${CMAKE_EXECUTABLE_SUFFIX} CACHE INTERNAL "") +set(CMAKE_C_FLAGS ${COMMON_FLAGS} CACHE INTERNAL "") +set(CMAKE_CXX_FLAGS ${COMMON_FLAGS} CACHE INTERNAL "") + +set(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER) +set(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY) +set(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY) + +set(OPENCV_SEMIHOSTING ON) +set(OPENCV_DISABLE_THREAD_SUPPORT ON) +set(OPENCV_DISABLE_FILESYSTEM_SUPPORT ON) +set(BUILD_SHARED_LIBS OFF) +set(OPENCV_FORCE_3RDPARTY_BUILD OFF) + + +# Enable newlib. +add_definitions(-D_GNU_SOURCE) + +add_definitions(-D_POSIX_PATH_MAX=0) diff --git a/platforms/semihosting/include/aarch64_semihosting_port.hpp b/platforms/semihosting/include/aarch64_semihosting_port.hpp new file mode 100644 index 0000000000..d3151c240a --- /dev/null +++ b/platforms/semihosting/include/aarch64_semihosting_port.hpp @@ -0,0 +1,42 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. + +#ifndef AARCH64_BAREMETAL_PORT_HPP +#define AARCH64_BAREMETAL_PORT_HPP + +#include // Needed for `memalign`. +#include // Needed for `ENOMEM`. + +// -std=c++11 is missing the following definitions when targeting +// semihosting on aarch64. +#if __cplusplus == 201103L +#include +#define M_PI 3.14159265358979323846 +#define M_SQRT2 1.41421356237309504880 + +namespace std { +inline double cbrt(double x) { + return ::cbrt(x); +} +inline double copysign(double mag, double sgn) { + return ::copysign(mag, sgn); +} +} //namespace std +#endif // __cplusplus == 201103L + +extern "C" { +// Redirect the implementation of `posix_memalign` to `memalign` +// as the former is +// missing at link time. https://pubs.opengroup.org/onlinepubs/9699919799/functions/posix_memalign.html +__attribute__((weak)) int posix_memalign(void **memptr, size_t alignment, size_t size) { + void * ptr = memalign(alignment, size); + if (ptr != NULL) { + *memptr = ptr; + return 0; + } + return ENOMEM; +} +} // extern "C" + +#endif diff --git a/samples/CMakeLists.txt b/samples/CMakeLists.txt index 0c70698ccb..9bfc2bf8ad 100644 --- a/samples/CMakeLists.txt +++ b/samples/CMakeLists.txt @@ -45,7 +45,12 @@ endif() if(INSTALL_PYTHON_EXAMPLES) add_subdirectory(python) endif() - +# The examples in this folder will work with a semihosting version of +# OpenCV. For more information about semihosting, see +# https://developer.arm.com/documentation/100863/latest +if(OPENCV_SEMIHOSTING) + add_subdirectory(semihosting) +endif() ocv_install_example_src("." CMakeLists.txt samples_utils.cmake) if(INSTALL_C_EXAMPLES) install(DIRECTORY data DESTINATION "${OPENCV_SAMPLES_SRC_INSTALL_PATH}" COMPONENT samples_data) diff --git a/samples/semihosting/CMakeLists.txt b/samples/semihosting/CMakeLists.txt new file mode 100644 index 0000000000..9fddb0587b --- /dev/null +++ b/samples/semihosting/CMakeLists.txt @@ -0,0 +1,10 @@ +# This file is part of OpenCV project. +# It is subject to the license terms in the LICENSE file found in the top-level directory +# of this distribution and at http://opencv.org/license.html + +set(SEMIHOSTING_SUFFIX semihosting) + +add_subdirectory(include) +set(RAW_PIXEL_INCLUDE ${CMAKE_CURRENT_BINARY_DIR}/include) +add_subdirectory(histogram) +add_subdirectory(norm) diff --git a/samples/semihosting/README.md b/samples/semihosting/README.md new file mode 100644 index 0000000000..881b09b735 --- /dev/null +++ b/samples/semihosting/README.md @@ -0,0 +1,27 @@ +# Arm semihosting + +This folder contain a toolchain file and a couple of examples for +building OpenCV based applications that can run in an [Arm +semihosting](https://developer.arm.com/documentation/100863/latest) +setup. + +OpenCV can be compiled to target a semihosting platform as follows: + +``` +cmake ../opencv/ \ + -DCMAKE_TOOLCHAIN_FILE=../opencv/platforms/semihosting/aarch64-semihosting.toolchain.cmake \ + -DSEMIHOSTING_TOOLCHAIN_PATH=/path/to/baremetal-toolchain/bin/ \ + -DBUILD_EXAMPLES=ON -GNinja +``` + +A barematel toolchain for targeting aarch64 semihosting can be found +[here](https://developer.arm.com/tools-and-software/open-source-software/developer-tools/gnu-toolchain/gnu-a/downloads), +under `aarch64-none-elf`. + +The code of the examples in the `norm` and `histogram` folders can be +executed with qemu in Linux userspace: + +``` + qemu-aarch64 ./bin/example_semihosting_histogram + qemu-aarch64 ./bin/example_semihosting_norm +``` diff --git a/samples/semihosting/histogram/CMakeLists.txt b/samples/semihosting/histogram/CMakeLists.txt new file mode 100644 index 0000000000..d2f065d1b9 --- /dev/null +++ b/samples/semihosting/histogram/CMakeLists.txt @@ -0,0 +1,26 @@ +# This file is part of OpenCV project. +# It is subject to the license terms in the LICENSE file found in the top-level directory +# of this distribution and at http://opencv.org/license.html + +set(PROJECT_NAME histogram) +project(${PROJECT_NAME}) + +ocv_install_example_src(histogram *.cpp *.hpp CMakeLists.txt) + +set(LOCAL_DEPS + opencv_core + opencv_imgproc + ${OPENCV_MODULES_PUBLIC} + ${OpenCV_LIB_COMPONENTS}) +ocv_check_dependencies(${LOCAL_DEPS}) + +if(NOT OCV_DEPENDENCIES_FOUND) + return() +endif() + +ocv_define_sample(histogram histogram.cpp ${SEMIHOSTING_SUFFIX}) +ocv_include_modules_recurse(${LOCAL_DEPS}) +target_include_directories(${histogram} PRIVATE ${CMAKE_CURRENT_BINARY_DIR}) +target_include_directories(${histogram} PRIVATE ${RAW_PIXEL_INCLUDE}) +ocv_target_link_libraries(${histogram} PRIVATE ${OPENCV_LINKER_LIBS} + ${LOCAL_DEPS}) diff --git a/samples/semihosting/histogram/histogram.cpp b/samples/semihosting/histogram/histogram.cpp new file mode 100644 index 0000000000..daa568d0bb --- /dev/null +++ b/samples/semihosting/histogram/histogram.cpp @@ -0,0 +1,43 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html + +#include +#include + +#include +#include +#include +#include "raw_pixels.hpp" + +#define IMG_ROWS 100 +#define IMG_COLS 100 + +static_assert(IMG_ROWS * IMG_COLS <= RAW_PIXELS_SIZE, "Incompatible size"); + +int main(void) +{ + // Number of experiment runs + int no_runs = 2; + + // https://docs.opencv.org/master/d3/d63/classcv_1_1Mat.html + cv::Mat src_new(IMG_ROWS, IMG_COLS, CV_8UC1, (void *)raw_pixels); + + // Set parameters + int imgCount = 1; + const int channels[] = {0}; + cv::Mat mask = cv::Mat(); + cv::Mat hist; + int dims = 1; + const int hist_sizes[] = {256}; + float Range[] = {0,256}; + const float *ranges[] = {Range}; + + // Run calc Hist + for(int i=0; i < no_runs; i++){ + std::cout << "Running iteration # "<< i << std::endl; + cv::calcHist(&src_new, imgCount, channels, mask, hist, dims, hist_sizes, ranges); + } + + return 0; +} diff --git a/samples/semihosting/include/CMakeLists.txt b/samples/semihosting/include/CMakeLists.txt new file mode 100644 index 0000000000..3c429b8adf --- /dev/null +++ b/samples/semihosting/include/CMakeLists.txt @@ -0,0 +1,16 @@ +# Populate a C array with random data. +set(RAW_PIXELS_SIZE 102400) +set(RAW_PIXELS_HEADER ${CMAKE_CURRENT_BINARY_DIR}/raw_pixels.hpp) +set(RAW_PIXELS_HEADER_IN ${CMAKE_CURRENT_SOURCE_DIR}/raw_pixels.hpp.in) + +set(RAW_PIXEL_VALUES "") +# Seed the random number generator. +string(RANDOM LENGTH 8 ALPHABET 0123456789abcdf RANDOM_SEED 314 number) +math(EXPR LOOP_RANGE "${RAW_PIXELS_SIZE} - 1") + +foreach(i RANGE ${LOOP_RANGE}) + string(RANDOM LENGTH 8 ALPHABET 0123456789abcdf number) + string(CONCAT RAW_PIXEL_VALUES ${RAW_PIXEL_VALUES} "0x${number}, \\\n") +endforeach() + +configure_file(${RAW_PIXELS_HEADER_IN} ${RAW_PIXELS_HEADER}) diff --git a/samples/semihosting/include/raw_pixels.hpp.in b/samples/semihosting/include/raw_pixels.hpp.in new file mode 100644 index 0000000000..6ee98222cc --- /dev/null +++ b/samples/semihosting/include/raw_pixels.hpp.in @@ -0,0 +1,11 @@ +#ifndef RAW_PIXELS_HPP +#define RAW_PIXELS_HP +#include + +#cmakedefine RAW_PIXEL_VALUES @RAW_PIXEL_VALUES@ +#cmakedefine RAW_PIXELS_SIZE @RAW_PIXELS_SIZE@ + +static std::uint32_t raw_pixels[RAW_PIXELS_SIZE] = { + RAW_PIXEL_VALUES +}; +#endif //RAW_PIXELS_HPP diff --git a/samples/semihosting/norm/CMakeLists.txt b/samples/semihosting/norm/CMakeLists.txt new file mode 100644 index 0000000000..6f23d74627 --- /dev/null +++ b/samples/semihosting/norm/CMakeLists.txt @@ -0,0 +1,25 @@ +# This file is part of OpenCV project. +# It is subject to the license terms in the LICENSE file found in the top-level directory +# of this distribution and at http://opencv.org/license.html + +set(PROJECT_NAME norm) +project(${PROJECT_NAME}) + +ocv_install_example_src(norm *.cpp *.hpp CMakeLists.txt) + +set(LOCAL_DEPS + opencv_core + ${OPENCV_MODULES_PUBLIC} + ${OpenCV_LIB_COMPONENTS}) +ocv_check_dependencies(${LOCAL_DEPS}) + +if(NOT OCV_DEPENDENCIES_FOUND) + return() +endif() + +ocv_define_sample(norm norm.cpp ${SEMIHOSTING_SUFFIX}) +ocv_include_modules_recurse(${LOCAL_DEPS}) +target_include_directories(${norm} PRIVATE ${CMAKE_CURRENT_BINARY_DIR}) +target_include_directories(${norm} PRIVATE ${RAW_PIXEL_INCLUDE}) +ocv_target_link_libraries(${norm} PRIVATE ${OPENCV_LINKER_LIBS} + ${LOCAL_DEPS}) diff --git a/samples/semihosting/norm/norm.cpp b/samples/semihosting/norm/norm.cpp new file mode 100644 index 0000000000..f911754be1 --- /dev/null +++ b/samples/semihosting/norm/norm.cpp @@ -0,0 +1,33 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html + +#include +#include + +#include +#include +#include +#include "raw_pixels.hpp" + +#define IMG_ROWS 100 +#define IMG_COLS 100 + +static_assert(IMG_ROWS * IMG_COLS <= RAW_PIXELS_SIZE, "Incompatible size"); + +int main(void) +{ + // Number of experiment runs + int no_runs = 2; + + // https://docs.opencv.org/master/d3/d63/classcv_1_1Mat.html + cv::Mat src(IMG_ROWS, IMG_COLS, CV_8UC1, (void *)raw_pixels); + + // Run calc Hist + for(int i=0; i < no_runs; i++){ + std::cout << "Running iteration # "<< i << std::endl; + cv::norm(src); + } + + return 0; +} From aae48e6fd77a30e45b56c60ceef20de21886d5a9 Mon Sep 17 00:00:00 2001 From: Dmitry Budnikov Date: Thu, 22 Jul 2021 16:52:11 +0300 Subject: [PATCH 076/262] single transpose MTCNN version --- modules/gapi/samples/face_detection_mtcnn.cpp | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/modules/gapi/samples/face_detection_mtcnn.cpp b/modules/gapi/samples/face_detection_mtcnn.cpp index d679ba0529..ad7c20c117 100644 --- a/modules/gapi/samples/face_detection_mtcnn.cpp +++ b/modules/gapi/samples/face_detection_mtcnn.cpp @@ -589,18 +589,18 @@ int main(int argc, char* argv[]) { //Preprocessing BGR2RGB + transpose (NCWH is expected instead of NCHW) cv::GMat in_original; cv::GMat in_originalRGB = cv::gapi::BGR2RGB(in_original); + cv::GMat in_transposedRGB = cv::gapi::transpose(in_originalRGB); cv::GOpaque in_sz = cv::gapi::streaming::size(in_original); cv::GMat in_resized[MAX_PYRAMID_LEVELS]; - cv::GMat in_transposed[MAX_PYRAMID_LEVELS]; cv::GMat regressions[MAX_PYRAMID_LEVELS]; cv::GMat scores[MAX_PYRAMID_LEVELS]; cv::GArray nms_p_faces[MAX_PYRAMID_LEVELS]; cv::GArray total_faces[MAX_PYRAMID_LEVELS]; //The very first PNet pyramid layer to init total_faces[0] - in_resized[0] = cv::gapi::resize(in_originalRGB, level_size[0]); - in_transposed[0] = cv::gapi::transpose(in_resized[0]); - std::tie(regressions[0], scores[0]) = run_mtcnn_p(in_transposed[0], get_pnet_level_name(level_size[0])); + cv::Size currentSize = cv::Size(level_size[0].height, level_size[0].width); + in_resized[0] = cv::gapi::resize(in_transposedRGB, currentSize); + std::tie(regressions[0], scores[0]) = run_mtcnn_p(in_resized[0], get_pnet_level_name(level_size[0])); cv::GArray faces0 = custom::BuildFaces::on(scores[0], regressions[0], static_cast(scales[0]), conf_thresh_p); cv::GArray final_p_faces_for_bb2squares = custom::ApplyRegression::on(faces0, true); cv::GArray final_faces_pnet0 = custom::BBoxesToSquares::on(final_p_faces_for_bb2squares); @@ -608,9 +608,9 @@ int main(int argc, char* argv[]) { //The rest PNet pyramid layers to accumlate all layers result in total_faces[PYRAMID_LEVELS - 1]] for (int i = 1; i < pyramid_levels; ++i) { - in_resized[i] = cv::gapi::resize(in_originalRGB, level_size[i]); - in_transposed[i] = cv::gapi::transpose(in_resized[i]); - std::tie(regressions[i], scores[i]) = run_mtcnn_p(in_transposed[i], get_pnet_level_name(level_size[i])); + currentSize = cv::Size(level_size[i].height, level_size[i].width); + in_resized[i] = cv::gapi::resize(in_transposedRGB, currentSize); + std::tie(regressions[i], scores[i]) = run_mtcnn_p(in_resized[i], get_pnet_level_name(level_size[i])); cv::GArray faces = custom::BuildFaces::on(scores[i], regressions[i], static_cast(scales[i]), conf_thresh_p); cv::GArray final_p_faces_for_bb2squares_i = custom::ApplyRegression::on(faces, true); cv::GArray final_faces_pnet_i = custom::BBoxesToSquares::on(final_p_faces_for_bb2squares_i); @@ -624,8 +624,7 @@ int main(int argc, char* argv[]) { //Refinement part of MTCNN graph cv::GArray faces_roi_pnet = custom::R_O_NetPreProcGetROIs::on(final_faces_pnet, in_sz); cv::GArray regressionsRNet, scoresRNet; - cv::GMat in_originalRGB_transposed = cv::gapi::transpose(in_originalRGB); - std::tie(regressionsRNet, scoresRNet) = cv::gapi::infer(faces_roi_pnet, in_originalRGB_transposed); + std::tie(regressionsRNet, scoresRNet) = cv::gapi::infer(faces_roi_pnet, in_transposedRGB); //Refinement post-processing cv::GArray rnet_post_proc_faces = custom::RNetPostProc::on(final_faces_pnet, scoresRNet, regressionsRNet, conf_thresh_r); @@ -636,7 +635,7 @@ int main(int argc, char* argv[]) { //Output part of MTCNN graph cv::GArray faces_roi_rnet = custom::R_O_NetPreProcGetROIs::on(final_faces_rnet, in_sz); cv::GArray regressionsONet, scoresONet, landmarksONet; - std::tie(regressionsONet, landmarksONet, scoresONet) = cv::gapi::infer(faces_roi_rnet, in_originalRGB_transposed); + std::tie(regressionsONet, landmarksONet, scoresONet) = cv::gapi::infer(faces_roi_rnet, in_transposedRGB); //Output post-processing cv::GArray onet_post_proc_faces = custom::ONetPostProc::on(final_faces_rnet, scoresONet, regressionsONet, landmarksONet, conf_thresh_o); From 024b43ca06dc7ab5e612aeee55fb44cdb534c49d Mon Sep 17 00:00:00 2001 From: Smirnov Egor Date: Mon, 19 Jul 2021 18:24:15 +0300 Subject: [PATCH 077/262] implement asymmetric padding for conv2d, max_pool and conv2d_backprop_input --- modules/dnn/src/tensorflow/tf_importer.cpp | 112 +++++++++++++++++++-- modules/dnn/test/test_tf_importer.cpp | 10 ++ 2 files changed, 112 insertions(+), 10 deletions(-) diff --git a/modules/dnn/src/tensorflow/tf_importer.cpp b/modules/dnn/src/tensorflow/tf_importer.cpp index 39c2309394..426710989e 100644 --- a/modules/dnn/src/tensorflow/tf_importer.cpp +++ b/modules/dnn/src/tensorflow/tf_importer.cpp @@ -404,12 +404,53 @@ void setKSize(LayerParams &layerParams, const tensorflow::NodeDef &layer) } } -void setPadding(LayerParams &layerParams, const tensorflow::NodeDef &layer) +void setPadMode(LayerParams &layerParams, const tensorflow::NodeDef &layer) { if (hasLayerAttr(layer, "padding")) layerParams.set("pad_mode", getLayerAttr(layer, "padding").s()); } +bool getExplicitPadding(LayerParams &layerParams, const tensorflow::NodeDef &layer, int64_t (&pads)[8]) +{ + if (!layerParams.has("pad_mode") || + layerParams.get("pad_mode").getStringValue() != "EXPLICIT") + { + return false; + } + + CV_Assert(hasLayerAttr(layer, "explicit_paddings")); + + const tensorflow::AttrValue& protoPads = getLayerAttr(layer, "explicit_paddings"); + if (protoPads.list().i_size() != 8) + { + CV_Error(Error::StsNotImplemented, "Unsupported asymmetric padding configuration."); + } + + int n = sizeof(pads) / sizeof(pads[0]); + for (int i = 0; i < n; ++i) + { + pads[i] = protoPads.list().i(i); + } + + if (getDataLayout(layer) != DATA_LAYOUT_NCHW) + { + CV_LOG_DEBUG(NULL, "DNN/TF: Data format " << getLayerAttr(layer, "data_format").s() << ", assuming NHWC."); + // Perhaps, we have NHWC padding dimensions order. + // N H W C + // 0 1 2 3 4 5 6 7 + std::swap(pads[2], pads[6]); + std::swap(pads[3], pads[7]); + // N C W H + // 0 1 2 3 4 5 6 7 + std::swap(pads[4], pads[6]); + std::swap(pads[5], pads[7]); + // N C H W + // 0 1 2 3 4 5 6 7 + } + + return true; +} + Pin parsePin(const std::string &name) { Pin pin(name); @@ -510,6 +551,7 @@ protected: private: void addPermuteLayer(const int* order, const std::string& permName, Pin& inpId); + void setPadding(LayerParams &layerParams, const tensorflow::NodeDef &layer, std::string& inputName, float value = 0.); typedef void (TFImporter::*TFImporterNodeParser)(tensorflow::GraphDef&, const tensorflow::NodeDef&, LayerParams&); typedef std::map DispatchMap; @@ -551,6 +593,31 @@ private: void parseCustomLayer (tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams); }; +void TFImporter::setPadding(LayerParams &layerParams, const tensorflow::NodeDef &layer, std::string& inputName, float value) +{ + setPadMode(layerParams, layer); + int64_t pads[8]; + + if (!getExplicitPadding(layerParams, layer, pads)) + { + return; + } + + LayerParams padLp; + padLp.name = layer.name() + "/pad"; + padLp.type = "Padding"; + padLp.set("paddings", DictValue::arrayInt(pads, sizeof(pads) / sizeof(pads[0]))); + padLp.set("value", value); + + int id = dstNet.addLayer(padLp.name, padLp.type, padLp); + layer_id[padLp.name] = id; + + connect(layer_id, dstNet, parsePin(inputName), id, 0); + inputName = padLp.name; + + layerParams.set("pad_mode", "VALID"); +} + const TFImporter::DispatchMap TFImporter::buildDispatchMap() { static DispatchMap dispatch; @@ -787,7 +854,7 @@ void TFImporter::parseConvolution(tensorflow::GraphDef& net, const tensorflow::N setStrides(layerParams, layer); if (!layerParams.has("pad_w") && !layerParams.has("pad_h")) - setPadding(layerParams, layer); + setPadding(layerParams, layer, input); // The final node of dilated convolution subgraph. next_layers = getNextLayers(net, name, "BatchToSpaceND"); @@ -1232,20 +1299,21 @@ void TFImporter::parseMaxPool(tensorflow::GraphDef& net, const tensorflow::NodeD { const std::string& name = layer.name(); const int num_inputs = layer.input_size(); + std::string inputName = layer.input(0); CV_CheckGT(num_inputs, 0, ""); layerParams.set("pool", "max"); setKSize(layerParams, layer); setStrides(layerParams, layer); - setPadding(layerParams, layer); + setPadding(layerParams, layer, inputName, -std::numeric_limits::infinity()); // Test_TensorFlow_nets.EAST_text_detection/1, NGRAPH/CPU layerParams.set("ceil_mode", false); int id = dstNet.addLayer(name, "Pooling", layerParams); layer_id[name] = id; - connectToAllBlobs(layer_id, dstNet, parsePin(layer.input(0)), id, num_inputs); + connectToAllBlobs(layer_id, dstNet, parsePin(inputName), id, num_inputs); } void TFImporter::parseAvgPool(tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams) @@ -1258,7 +1326,7 @@ void TFImporter::parseAvgPool(tensorflow::GraphDef& net, const tensorflow::NodeD layerParams.set("ave_pool_padded_area", false); setKSize(layerParams, layer); setStrides(layerParams, layer); - setPadding(layerParams, layer); + setPadMode(layerParams, layer); int id = dstNet.addLayer(name, "Pooling", layerParams); layer_id[name] = id; @@ -1673,7 +1741,7 @@ void TFImporter::parseConv2DBackpropInput(tensorflow::GraphDef& net, const tenso // input: "weights" // input: "input" - const std::string& name = layer.name(); + std::string name = layer.name(); const int num_inputs = layer.input_size(); CV_CheckEQ(num_inputs, 3, "Expected output shape, weights and input nodes"); @@ -1704,7 +1772,21 @@ void TFImporter::parseConv2DBackpropInput(tensorflow::GraphDef& net, const tenso layerParams.set("num_output", kshape[1]); setStrides(layerParams, layer); - setPadding(layerParams, layer); + setPadMode(layerParams, layer); + int64_t pads[8]; + bool explicit_pads = getExplicitPadding(layerParams, layer, pads); + int64_t begs[4] = {}; + int64_t ends[4] = {-1, -1, -1, -1}; + if (explicit_pads) + { + name += "/deconv"; + layerParams.set("pad_mode", "VALID"); + for (int i = 2; i < 4; ++i) // begins=[0, 0, a, b], ends=[-1, -1, c, d] + { + begs[i] = pads[2*i]; + ends[i] = -1 - pads[2*i + 1]; + } + } // For convolution layer, output shape computes as // o = 1 + (i - k + 2*p) / s @@ -1721,8 +1803,9 @@ void TFImporter::parseConv2DBackpropInput(tensorflow::GraphDef& net, const tenso const int strideY = layerParams.get("stride_h"); const int strideX = layerParams.get("stride_w"); Mat outShape = getTensorContent(getConstBlob(layer, value_id, 0)); - const int outH = outShape.at(1); - const int outW = outShape.at(2); + int shift = (getDataLayout(layer) == DATA_LAYOUT_NCHW); + const int outH = outShape.at(1 + shift) + begs[2] - 1 - ends[2]; + const int outW = outShape.at(2 + shift) + begs[3] - 1 - ends[3]; if (layerParams.get("pad_mode") == "SAME") { layerParams.set("adj_w", (outW - 1) % strideX); @@ -1738,6 +1821,16 @@ void TFImporter::parseConv2DBackpropInput(tensorflow::GraphDef& net, const tenso // one input only connect(layer_id, dstNet, parsePin(layer.input(2)), id, 0); + if (explicit_pads) // If we have explicit paddings, remove extra data + { + layerParams.set("begin", DictValue::arrayInt(begs, sizeof(begs) / sizeof(begs[0]))); + layerParams.set("end", DictValue::arrayInt(ends, sizeof(ends) / sizeof(ends[0]))); + + int id = dstNet.addLayer(layer.name(), "Slice", layerParams); + layer_id[layer.name()] = id; + + connect(layer_id, dstNet, parsePin(name), id, 0); + } } void TFImporter::parseBlockLSTM(tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams) @@ -2717,7 +2810,6 @@ void TFImporter::populateNet() addConstNodes(netBin, value_id, layers_to_ignore); addConstNodes(netTxt, value_id, layers_to_ignore); - for (int li = 0; li < layersSize; li++) { const tensorflow::NodeDef& layer = net.node(li); diff --git a/modules/dnn/test/test_tf_importer.cpp b/modules/dnn/test/test_tf_importer.cpp index 4ba4f29322..4f7840f9e4 100644 --- a/modules/dnn/test/test_tf_importer.cpp +++ b/modules/dnn/test/test_tf_importer.cpp @@ -203,6 +203,16 @@ TEST_P(Test_TensorFlow_layers, padding) runTensorFlowNet("keras_pad_concat"); } +TEST_P(Test_TensorFlow_layers, padding_asymmetric) +{ + runTensorFlowNet("conv2d_asymmetric_pads_nchw"); + runTensorFlowNet("conv2d_asymmetric_pads_nhwc"); + runTensorFlowNet("max_pool2d_asymmetric_pads_nchw"); + runTensorFlowNet("max_pool2d_asymmetric_pads_nhwc"); + runTensorFlowNet("conv2d_backprop_input_asymmetric_pads_nchw"); + runTensorFlowNet("conv2d_backprop_input_asymmetric_pads_nhwc"); +} + TEST_P(Test_TensorFlow_layers, padding_same) { // Reference output values are in range [0.0006, 2.798] From 4015a5486c5cd831923c6d8019233228c06b99d1 Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Wed, 21 Jul 2021 04:06:31 +0000 Subject: [PATCH 078/262] cmake: process modules in the same CMake scope --- CMakeLists.txt | 4 +- cmake/OpenCVModule.cmake | 39 +++++++++++++++++-- modules/CMakeLists.txt | 28 ------------- modules/highgui/CMakeLists.txt | 3 +- modules/highgui/cmake/detect_gtk.cmake | 11 ------ modules/highgui/cmake/detect_win32ui.cmake | 2 - modules/highgui/cmake/init.cmake | 19 +++------ .../videoio/cmake/detect_android_camera.cmake | 2 - .../cmake/detect_android_mediandk.cmake | 2 - modules/videoio/cmake/detect_aravis.cmake | 4 +- .../videoio/cmake/detect_avfoundation.cmake | 2 - modules/videoio/cmake/detect_dc1394.cmake | 5 +-- modules/videoio/cmake/detect_dshow.cmake | 2 - modules/videoio/cmake/detect_ffmpeg.cmake | 7 ---- modules/videoio/cmake/detect_gphoto.cmake | 2 - modules/videoio/cmake/detect_gstreamer.cmake | 6 +-- modules/videoio/cmake/detect_ios.cmake | 2 - modules/videoio/cmake/detect_msdk.cmake | 2 - modules/videoio/cmake/detect_msmf.cmake | 3 -- modules/videoio/cmake/detect_openni2.cmake | 4 +- modules/videoio/cmake/detect_pvapi.cmake | 2 - modules/videoio/cmake/detect_realsense.cmake | 4 +- modules/videoio/cmake/detect_ueye.cmake | 2 - modules/videoio/cmake/detect_v4l.cmake | 2 - modules/videoio/cmake/detect_ximea.cmake | 2 - modules/videoio/cmake/detect_xine.cmake | 2 - modules/videoio/cmake/init.cmake | 19 +++------ 27 files changed, 56 insertions(+), 126 deletions(-) delete mode 100644 modules/CMakeLists.txt diff --git a/CMakeLists.txt b/CMakeLists.txt index f4fe0385d1..48bce581f4 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -913,7 +913,7 @@ add_subdirectory(include) ocv_add_modules_compiler_options() # OpenCV modules -add_subdirectory(modules) +ocv_register_modules() # Generate targets for documentation add_subdirectory(doc) @@ -1243,7 +1243,7 @@ endif(WIN32) # ========================== GUI ========================== status("") -status(" GUI: ") +status(" GUI: " "${OPENCV_HIGHGUI_BUILTIN_BACKEND}") if(WITH_QT OR HAVE_QT) if(HAVE_QT5) diff --git a/cmake/OpenCVModule.cmake b/cmake/OpenCVModule.cmake index 7c48aad9c2..9981620f25 100644 --- a/cmake/OpenCVModule.cmake +++ b/cmake/OpenCVModule.cmake @@ -254,7 +254,7 @@ function(_glob_locations out_paths out_names) list(LENGTH paths before) get_filename_component(path "${path}" ABSOLUTE) # Either module itself - if(NOT path STREQUAL CMAKE_CURRENT_SOURCE_DIR AND EXISTS "${path}/CMakeLists.txt") + if(NOT path STREQUAL "${OpenCV_SOURCE_DIR}/modules" AND EXISTS "${path}/CMakeLists.txt") get_filename_component(name "${path}" NAME) list(APPEND paths "${path}") list(APPEND names "${name}") @@ -296,7 +296,7 @@ macro(_add_modules_1 paths names) list(GET ${names} ${i} __name) #message(STATUS "First pass: ${__name} => ${__path}") include("${__path}/cmake/init.cmake" OPTIONAL) - add_subdirectory("${__path}" "${CMAKE_CURRENT_BINARY_DIR}/.firstpass/${__name}") + add_subdirectory("${__path}" "${OpenCV_BINARY_DIR}/modules/.firstpass/${__name}") endforeach() endif() endmacro() @@ -316,7 +316,7 @@ macro(_add_modules_2) endif() string(REGEX REPLACE "^opencv_" "" name "${m}") #message(STATUS "Second pass: ${name} => ${OPENCV_MODULE_${m}_LOCATION}") - add_subdirectory("${OPENCV_MODULE_${m}_LOCATION}" "${CMAKE_CURRENT_BINARY_DIR}/${name}") + add_subdirectory("${OPENCV_MODULE_${m}_LOCATION}" "${OpenCV_BINARY_DIR}/modules/${name}") endif() ocv_cmake_hook(POST_MODULES_CREATE_${the_module}) endforeach() @@ -369,7 +369,6 @@ macro(ocv_glob_modules main_root) __ocv_resolve_dependencies() # create modules - set(OPENCV_INITIAL_PASS OFF PARENT_SCOPE) set(OPENCV_INITIAL_PASS OFF) ocv_cmake_hook(PRE_MODULES_CREATE) _add_modules_2(${OPENCV_MODULES_BUILD}) @@ -377,6 +376,37 @@ macro(ocv_glob_modules main_root) endmacro() +# called by root CMakeLists.txt +macro(ocv_register_modules) + if(NOT OPENCV_MODULES_PATH) + set(OPENCV_MODULES_PATH "${OpenCV_SOURCE_DIR}/modules") + endif() + + ocv_glob_modules(${OPENCV_MODULES_PATH} ${OPENCV_EXTRA_MODULES_PATH}) + + # build lists of modules to be documented + set(OPENCV_MODULES_MAIN "") + set(OPENCV_MODULES_EXTRA "") + + foreach(mod ${OPENCV_MODULES_BUILD} ${OPENCV_MODULES_DISABLED_USER} ${OPENCV_MODULES_DISABLED_AUTO} ${OPENCV_MODULES_DISABLED_FORCE}) + string(REGEX REPLACE "^opencv_" "" mod "${mod}") + if("${OPENCV_MODULE_opencv_${mod}_LOCATION}" STREQUAL "${OpenCV_SOURCE_DIR}/modules/${mod}") + list(APPEND OPENCV_MODULES_MAIN ${mod}) + else() + list(APPEND OPENCV_MODULES_EXTRA ${mod}) + endif() + endforeach() + ocv_list_sort(OPENCV_MODULES_MAIN) + ocv_list_sort(OPENCV_MODULES_EXTRA) + set(FIXED_ORDER_MODULES core imgproc imgcodecs videoio highgui video calib3d features2d objdetect dnn ml flann photo stitching) + list(REMOVE_ITEM OPENCV_MODULES_MAIN ${FIXED_ORDER_MODULES}) + set(OPENCV_MODULES_MAIN ${FIXED_ORDER_MODULES} ${OPENCV_MODULES_MAIN}) + + set(OPENCV_MODULES_MAIN ${OPENCV_MODULES_MAIN} CACHE INTERNAL "List of main modules" FORCE) + set(OPENCV_MODULES_EXTRA ${OPENCV_MODULES_EXTRA} CACHE INTERNAL "List of extra modules" FORCE) +endmacro() + + # disables OpenCV module with missing dependencies function(__ocv_module_turn_off the_module) list(REMOVE_ITEM OPENCV_MODULES_DISABLED_AUTO "${the_module}") @@ -877,6 +907,7 @@ macro(ocv_create_module) endmacro() macro(_ocv_create_module) + add_definitions(-D__OPENCV_BUILD=1) ocv_compiler_optimization_process_sources(OPENCV_MODULE_${the_module}_SOURCES OPENCV_MODULE_${the_module}_DEPS_EXT ${the_module}) set(__module_headers ${OPENCV_MODULE_${the_module}_HEADERS}) diff --git a/modules/CMakeLists.txt b/modules/CMakeLists.txt deleted file mode 100644 index 6a8004036b..0000000000 --- a/modules/CMakeLists.txt +++ /dev/null @@ -1,28 +0,0 @@ -add_definitions(-D__OPENCV_BUILD=1) - -if(NOT OPENCV_MODULES_PATH) - set(OPENCV_MODULES_PATH "${CMAKE_CURRENT_SOURCE_DIR}") -endif() - -ocv_glob_modules(${OPENCV_MODULES_PATH} ${OPENCV_EXTRA_MODULES_PATH}) - -# build lists of modules to be documented -set(OPENCV_MODULES_MAIN "") -set(OPENCV_MODULES_EXTRA "") - -foreach(mod ${OPENCV_MODULES_BUILD} ${OPENCV_MODULES_DISABLED_USER} ${OPENCV_MODULES_DISABLED_AUTO} ${OPENCV_MODULES_DISABLED_FORCE}) - string(REGEX REPLACE "^opencv_" "" mod "${mod}") - if("${OPENCV_MODULE_opencv_${mod}_LOCATION}" STREQUAL "${OpenCV_SOURCE_DIR}/modules/${mod}") - list(APPEND OPENCV_MODULES_MAIN ${mod}) - else() - list(APPEND OPENCV_MODULES_EXTRA ${mod}) - endif() -endforeach() -ocv_list_sort(OPENCV_MODULES_MAIN) -ocv_list_sort(OPENCV_MODULES_EXTRA) -set(FIXED_ORDER_MODULES core imgproc imgcodecs videoio highgui video calib3d features2d objdetect dnn ml flann photo stitching) -list(REMOVE_ITEM OPENCV_MODULES_MAIN ${FIXED_ORDER_MODULES}) -set(OPENCV_MODULES_MAIN ${FIXED_ORDER_MODULES} ${OPENCV_MODULES_MAIN}) - -set(OPENCV_MODULES_MAIN ${OPENCV_MODULES_MAIN} CACHE INTERNAL "List of main modules" FORCE) -set(OPENCV_MODULES_EXTRA ${OPENCV_MODULES_EXTRA} CACHE INTERNAL "List of extra modules" FORCE) diff --git a/modules/highgui/CMakeLists.txt b/modules/highgui/CMakeLists.txt index bc31b84c74..2b630bfed8 100644 --- a/modules/highgui/CMakeLists.txt +++ b/modules/highgui/CMakeLists.txt @@ -218,7 +218,8 @@ endif() if(NOT OPENCV_HIGHGUI_BUILTIN_BACKEND) set(OPENCV_HIGHGUI_BUILTIN_BACKEND "NONE") endif() -message(STATUS "highgui: using builtin backend: ${OPENCV_HIGHGUI_BUILTIN_BACKEND}") # FIXIT: propagate to root CMake +message(STATUS "highgui: using builtin backend: ${OPENCV_HIGHGUI_BUILTIN_BACKEND}") +set(OPENCV_HIGHGUI_BUILTIN_BACKEND "${OPENCV_HIGHGUI_BUILTIN_BACKEND}" PARENT_SCOPE) # informational if(TRUE) # these variables are set by 'ocv_append_build_options(HIGHGUI ...)' diff --git a/modules/highgui/cmake/detect_gtk.cmake b/modules/highgui/cmake/detect_gtk.cmake index cdc054fad0..c58246ac54 100644 --- a/modules/highgui/cmake/detect_gtk.cmake +++ b/modules/highgui/cmake/detect_gtk.cmake @@ -6,8 +6,6 @@ if(WITH_GTK) if(HAVE_GTK3) ocv_add_external_target(gtk3 "${GTK3_INCLUDE_DIRS}" "${GTK3_LIBRARIES}" "HAVE_GTK3;HAVE_GTK") set(HAVE_GTK TRUE) - set(HAVE_GTK3 ${HAVE_GTK3} PARENT_SCOPE) - set(GTK3_VERSION "${GTK3_VERSION}" PARENT_SCOPE) # informational endif() endif() if((PROJECT_NAME STREQUAL "OpenCV" AND HIGHGUI_ENABLE_PLUGINS) OR NOT HAVE_GTK3) @@ -19,8 +17,6 @@ if(WITH_GTK) else() ocv_add_external_target(gtk2 "${GTK2_INCLUDE_DIRS}" "${GTK2_LIBRARIES}" "HAVE_GTK2;HAVE_GTK") set(HAVE_GTK TRUE) - set(HAVE_GTK2 ${HAVE_GTK2} PARENT_SCOPE) - set(GTK2_VERSION "${GTK2_VERSION}" PARENT_SCOPE) # informational endif() endif() endif() @@ -29,15 +25,11 @@ if(WITH_GTK) message(FATAL_ERROR "gthread not found. This library is required when building with GTK support") else() ocv_add_external_target(gthread "${GTHREAD_INCLUDE_DIRS}" "${GTHREAD_LIBRARIES}" "HAVE_GTHREAD") - set(HAVE_GTHREAD "${HAVE_GTHREAD}" PARENT_SCOPE) # informational - set(GTHREAD_VERSION "${GTHREAD_VERSION}" PARENT_SCOPE) # informational endif() if((WITH_OPENGL OR HAVE_OPENGL) AND HAVE_GTK2) ocv_check_modules(GTKGLEXT gtkglext-1.0) if(HAVE_GTKGLEXT) ocv_add_external_target(gtkglext "${GTKGLEXT_INCLUDE_DIRS}" "${GTKGLEXT_LIBRARIES}" "HAVE_GTKGLEXT") - set(HAVE_GTKGLEXT "${HAVE_GTKGLEXT}" PARENT_SCOPE) # informational - set(GTKGLEXT_VERSION "${GTKGLEXT_VERSION}" PARENT_SCOPE) # informational endif() endif() elseif(HAVE_GTK) @@ -48,9 +40,6 @@ if(WITH_OPENGL AND HAVE_GTKGLEXT) find_package(OpenGL QUIET) if(OPENGL_FOUND) set(HAVE_OPENGL TRUE) - #set(HAVE_OPENGL ${HAVE_OPENGL} PARENT_SCOPE) ocv_add_external_target(gtk_opengl "${OPENGL_INCLUDE_DIRS}" "${OPENGL_LIBRARIES}" "HAVE_OPENGL") endif() endif() - -set(HAVE_GTK ${HAVE_GTK} PARENT_SCOPE) diff --git a/modules/highgui/cmake/detect_win32ui.cmake b/modules/highgui/cmake/detect_win32ui.cmake index 1d2fdc5d46..c5e358ffa7 100644 --- a/modules/highgui/cmake/detect_win32ui.cmake +++ b/modules/highgui/cmake/detect_win32ui.cmake @@ -13,5 +13,3 @@ if(WITH_WIN32UI) ocv_add_external_target(win32ui "" "${__libs}" "HAVE_WIN32UI") endif() endif() - -set(HAVE_WIN32UI "${HAVE_WIN32UI}" PARENT_SCOPE) # informational diff --git a/modules/highgui/cmake/init.cmake b/modules/highgui/cmake/init.cmake index 1626d254da..a302c4d534 100644 --- a/modules/highgui/cmake/init.cmake +++ b/modules/highgui/cmake/init.cmake @@ -27,20 +27,11 @@ endif() include(FindPkgConfig) -# FIXIT: stop using PARENT_SCOPE in dependencies -if(PROJECT_NAME STREQUAL "OpenCV") - macro(add_backend backend_id cond_var) - if(${cond_var}) - include("${CMAKE_CURRENT_LIST_DIR}/detect_${backend_id}.cmake") - endif() - endmacro() -else() - function(add_backend backend_id cond_var) - if(${cond_var}) - include("${CMAKE_CURRENT_LIST_DIR}/detect_${backend_id}.cmake") - endif() - endfunction() -endif() +macro(add_backend backend_id cond_var) + if(${cond_var}) + include("${CMAKE_CURRENT_LIST_DIR}/detect_${backend_id}.cmake") + endif() +endmacro() add_backend("gtk" WITH_GTK) add_backend("win32ui" WITH_WIN32UI) diff --git a/modules/videoio/cmake/detect_android_camera.cmake b/modules/videoio/cmake/detect_android_camera.cmake index ded4c91ccf..a465751334 100644 --- a/modules/videoio/cmake/detect_android_camera.cmake +++ b/modules/videoio/cmake/detect_android_camera.cmake @@ -4,5 +4,3 @@ if(ANDROID AND ANDROID_NATIVE_API_LEVEL GREATER 23) set(libs "-landroid -llog -lcamera2ndk") ocv_add_external_target(android_native_camera "" "${libs}" "HAVE_ANDROID_NATIVE_CAMERA") endif() - -set(HAVE_ANDROID_NATIVE_CAMERA ${HAVE_ANDROID_NATIVE_CAMERA} PARENT_SCOPE) diff --git a/modules/videoio/cmake/detect_android_mediandk.cmake b/modules/videoio/cmake/detect_android_mediandk.cmake index edfb4bbbc5..cee64ab549 100644 --- a/modules/videoio/cmake/detect_android_mediandk.cmake +++ b/modules/videoio/cmake/detect_android_mediandk.cmake @@ -4,5 +4,3 @@ if(ANDROID AND ANDROID_NATIVE_API_LEVEL GREATER 20) set(libs "-landroid -llog -lmediandk") ocv_add_external_target(android_mediandk "" "${libs}" "HAVE_ANDROID_MEDIANDK") endif() - -set(HAVE_ANDROID_MEDIANDK ${HAVE_ANDROID_MEDIANDK} PARENT_SCOPE) diff --git a/modules/videoio/cmake/detect_aravis.cmake b/modules/videoio/cmake/detect_aravis.cmake index 79d6a217db..e7b3828993 100644 --- a/modules/videoio/cmake/detect_aravis.cmake +++ b/modules/videoio/cmake/detect_aravis.cmake @@ -21,7 +21,7 @@ if(NOT HAVE_ARAVIS_API) string(REGEX REPLACE ".*ARAVIS_MAJOR_VERSION[^0-9]+([0-9]+).*" "\\1" ver_major "${ver_strings}") string(REGEX REPLACE ".*ARAVIS_MINOR_VERSION[^0-9]+([0-9]+).*" "\\1" ver_minor "${ver_strings}") string(REGEX REPLACE ".*ARAVIS_MICRO_VERSION[^0-9]+([0-9]+).*" "\\1" ver_micro "${ver_strings}") - set(ARAVIS_VERSION "${ver_major}.${ver_minor}.${ver_micro}" PARENT_SCOPE) # informational + set(ARAVIS_VERSION "${ver_major}.${ver_minor}.${ver_micro}") # informational set(ARAVIS_INCLUDE_DIRS "${ARAVIS_INCLUDE}") set(ARAVIS_LIBRARIES "${ARAVIS_LIBRARY}") endif() @@ -30,5 +30,3 @@ endif() if(HAVE_ARAVIS_API) ocv_add_external_target(aravis "${ARAVIS_INCLUDE_DIRS}" "${ARAVIS_LIBRARIES}" "HAVE_ARAVIS_API") endif() - -set(HAVE_ARAVIS_API ${HAVE_ARAVIS_API} PARENT_SCOPE) diff --git a/modules/videoio/cmake/detect_avfoundation.cmake b/modules/videoio/cmake/detect_avfoundation.cmake index a341f587a1..2da4fabfab 100644 --- a/modules/videoio/cmake/detect_avfoundation.cmake +++ b/modules/videoio/cmake/detect_avfoundation.cmake @@ -14,5 +14,3 @@ if(APPLE) endif() ocv_add_external_target(avfoundation "" "${libs}" "HAVE_AVFOUNDATION") endif() - -set(HAVE_AVFOUNDATION ${HAVE_AVFOUNDATION} PARENT_SCOPE) diff --git a/modules/videoio/cmake/detect_dc1394.cmake b/modules/videoio/cmake/detect_dc1394.cmake index 51ab2dd80e..8bcee4bf70 100644 --- a/modules/videoio/cmake/detect_dc1394.cmake +++ b/modules/videoio/cmake/detect_dc1394.cmake @@ -2,7 +2,6 @@ if(NOT HAVE_DC1394_2 AND PKG_CONFIG_FOUND) ocv_check_modules(DC1394_2 libdc1394-2) if(DC1394_2_FOUND) - set(DC1394_2_VERSION "${DC1394_2_VERSION}" PARENT_SCOPE) # informational set(HAVE_DC1394_2 TRUE) endif() endif() @@ -20,12 +19,10 @@ if(NOT HAVE_DC1394_2) set(HAVE_DC1394_2 TRUE) set(DC1394_2_INCLUDE_DIRS "${DC1394_INCLUDE}") set(DC1394_2_LIBRARIES "${DC1394_LIBRARY}") - set(DC1394_2_VERSION "unknown" PARENT_SCOPE) # informational + set(DC1394_2_VERSION "unknown") # informational endif() endif() if(HAVE_DC1394_2) ocv_add_external_target(dc1394_2 "${DC1394_2_INCLUDE_DIRS}" "${DC1394_2_LIBRARIES}" "HAVE_DC1394_2") endif() - -set(HAVE_DC1394_2 ${HAVE_DC1394_2} PARENT_SCOPE) diff --git a/modules/videoio/cmake/detect_dshow.cmake b/modules/videoio/cmake/detect_dshow.cmake index 3f41b3fd34..928134c08c 100644 --- a/modules/videoio/cmake/detect_dshow.cmake +++ b/modules/videoio/cmake/detect_dshow.cmake @@ -10,5 +10,3 @@ endif() if(HAVE_DSHOW) ocv_add_external_target(dshow "" "" "HAVE_DSHOW") endif() - -set(HAVE_DSHOW ${HAVE_DSHOW} PARENT_SCOPE) diff --git a/modules/videoio/cmake/detect_ffmpeg.cmake b/modules/videoio/cmake/detect_ffmpeg.cmake index 58de4b9515..c33eaf221b 100644 --- a/modules/videoio/cmake/detect_ffmpeg.cmake +++ b/modules/videoio/cmake/detect_ffmpeg.cmake @@ -14,11 +14,6 @@ if(NOT HAVE_FFMPEG AND WIN32 AND NOT ARM AND NOT OPENCV_FFMPEG_SKIP_DOWNLOAD) download_win_ffmpeg(FFMPEG_CMAKE_SCRIPT) if(FFMPEG_CMAKE_SCRIPT) include("${FFMPEG_CMAKE_SCRIPT}") - set(FFMPEG_libavcodec_VERSION ${FFMPEG_libavcodec_VERSION} PARENT_SCOPE) # info - set(FFMPEG_libavformat_VERSION ${FFMPEG_libavformat_VERSION} PARENT_SCOPE) # info - set(FFMPEG_libavutil_VERSION ${FFMPEG_libavutil_VERSION} PARENT_SCOPE) # info - set(FFMPEG_libswscale_VERSION ${FFMPEG_libswscale_VERSION} PARENT_SCOPE) # info - set(FFMPEG_libavresample_VERSION ${FFMPEG_libavresample_VERSION} PARENT_SCOPE) # info set(HAVE_FFMPEG TRUE) set(HAVE_FFMPEG_WRAPPER TRUE) endif() @@ -132,5 +127,3 @@ elseif(HAVE_FFMPEG) ocv_add_external_target(ffmpeg.plugin_deps "${__plugin_include_dirs}" "${__plugin_include_libs}" "${__plugin_defines}") endif() endif() - -set(HAVE_FFMPEG ${HAVE_FFMPEG} PARENT_SCOPE) diff --git a/modules/videoio/cmake/detect_gphoto.cmake b/modules/videoio/cmake/detect_gphoto.cmake index 0d6f1212eb..2cb23c0033 100644 --- a/modules/videoio/cmake/detect_gphoto.cmake +++ b/modules/videoio/cmake/detect_gphoto.cmake @@ -9,5 +9,3 @@ endif() if(HAVE_GPHOTO2) ocv_add_external_target(gphoto2 "${GPHOTO2_INCLUDE_DIRS}" "${GPHOTO2_LIBRARIES}" "HAVE_GPHOTO2") endif() - -set(HAVE_GPHOTO2 ${HAVE_GPHOTO2} PARENT_SCOPE) diff --git a/modules/videoio/cmake/detect_gstreamer.cmake b/modules/videoio/cmake/detect_gstreamer.cmake index 2198786161..47ea7a0b30 100644 --- a/modules/videoio/cmake/detect_gstreamer.cmake +++ b/modules/videoio/cmake/detect_gstreamer.cmake @@ -69,7 +69,7 @@ if(NOT HAVE_GSTREAMER AND WIN32) string(REGEX REPLACE ".*GST_VERSION_MAJOR[^0-9]+([0-9]+).*" "\\1" ver_major "${ver_strings}") string(REGEX REPLACE ".*GST_VERSION_MINOR[^0-9]+([0-9]+).*" "\\1" ver_minor "${ver_strings}") string(REGEX REPLACE ".*GST_VERSION_MICRO[^0-9]+([0-9]+).*" "\\1" ver_micro "${ver_strings}") - set(GSTREAMER_VERSION "${ver_major}.${ver_minor}.${ver_micro}" PARENT_SCOPE) # informational + set(GSTREAMER_VERSION "${ver_major}.${ver_minor}.${ver_micro}") # informational set(HAVE_GSTREAMER TRUE) set(GSTREAMER_LIBRARIES ${GSTREAMER_gstreamer_LIBRARY} @@ -95,7 +95,7 @@ if(NOT HAVE_GSTREAMER AND PKG_CONFIG_FOUND) ocv_check_modules(GSTREAMER_video gstreamer-video-1.0) if(GSTREAMER_base_FOUND AND GSTREAMER_app_FOUND AND GSTREAMER_riff_FOUND AND GSTREAMER_pbutils_FOUND AND GSTREAMER_video_FOUND) set(HAVE_GSTREAMER TRUE) - set(GSTREAMER_VERSION ${GSTREAMER_base_VERSION} PARENT_SCOPE) # informational + set(GSTREAMER_VERSION ${GSTREAMER_base_VERSION}) # informational set(GSTREAMER_LIBRARIES ${GSTREAMER_base_LIBRARIES} ${GSTREAMER_app_LIBRARIES} ${GSTREAMER_riff_LIBRARIES} ${GSTREAMER_pbutils_LIBRARIES} ${GSTREAMER_video_LIBRARIES}) set(GSTREAMER_INCLUDE_DIRS ${GSTREAMER_base_INCLUDE_DIRS} ${GSTREAMER_app_INCLUDE_DIRS} ${GSTREAMER_riff_INCLUDE_DIRS} ${GSTREAMER_pbutils_INCLUDE_DIRS} ${GSTREAMER_video_INCLUDE_DIRS}) endif() @@ -104,5 +104,3 @@ endif() if(HAVE_GSTREAMER) ocv_add_external_target(gstreamer "${GSTREAMER_INCLUDE_DIRS}" "${GSTREAMER_LIBRARIES}" "HAVE_GSTREAMER") endif() - -set(HAVE_GSTREAMER ${HAVE_GSTREAMER} PARENT_SCOPE) diff --git a/modules/videoio/cmake/detect_ios.cmake b/modules/videoio/cmake/detect_ios.cmake index c75426060b..8d48dd6f3b 100644 --- a/modules/videoio/cmake/detect_ios.cmake +++ b/modules/videoio/cmake/detect_ios.cmake @@ -11,5 +11,3 @@ if(APPLE AND IOS) "-framework UIKit") ocv_add_external_target(cap_ios "" "${libs}" "HAVE_CAP_IOS") endif() - -set(HAVE_CAP_IOS ${HAVE_CAP_IOS} PARENT_SCOPE) diff --git a/modules/videoio/cmake/detect_msdk.cmake b/modules/videoio/cmake/detect_msdk.cmake index d035c3f5cc..83701425e1 100644 --- a/modules/videoio/cmake/detect_msdk.cmake +++ b/modules/videoio/cmake/detect_msdk.cmake @@ -70,5 +70,3 @@ if(HAVE_MFX) list(APPEND MFX_DEFS "HAVE_MFX") ocv_add_external_target(mediasdk "${MFX_INCLUDE_DIRS}" "${MFX_LIBRARIES}" "${MFX_DEFS}") endif() - -set(HAVE_MFX ${HAVE_MFX} PARENT_SCOPE) diff --git a/modules/videoio/cmake/detect_msmf.cmake b/modules/videoio/cmake/detect_msmf.cmake index a1c91dab67..aebc226bcf 100644 --- a/modules/videoio/cmake/detect_msmf.cmake +++ b/modules/videoio/cmake/detect_msmf.cmake @@ -20,6 +20,3 @@ if(HAVE_MSMF) endif() ocv_add_external_target(msmf "" "" "${defs}") endif() - -set(HAVE_MSMF ${HAVE_MSMF} PARENT_SCOPE) -set(HAVE_MSMF_DXVA ${HAVE_MSMF_DXVA} PARENT_SCOPE) diff --git a/modules/videoio/cmake/detect_openni2.cmake b/modules/videoio/cmake/detect_openni2.cmake index 76c31454da..54a5c62bed 100644 --- a/modules/videoio/cmake/detect_openni2.cmake +++ b/modules/videoio/cmake/detect_openni2.cmake @@ -42,8 +42,6 @@ if(HAVE_OPENNI2) string(REGEX REPLACE ".*ONI_VERSION_MAJOR[^0-9]+([0-9]+).*" "\\1" ver_major "${ver_strings}") string(REGEX REPLACE ".*ONI_VERSION_MINOR[^0-9]+([0-9]+).*" "\\1" ver_minor "${ver_strings}") string(REGEX REPLACE ".*ONI_VERSION_MAINTENANCE[^0-9]+([0-9]+).*" "\\1" ver_maint "${ver_strings}") - set(OPENNI2_VERSION "${ver_major}.${ver_minor}.${ver_maint}" PARENT_SCOPE) # informational + set(OPENNI2_VERSION "${ver_major}.${ver_minor}.${ver_maint}") # informational ocv_add_external_target(openni2 "${OPENNI2_INCLUDE_DIRS}" "${OPENNI2_LIBRARIES}" "HAVE_OPENNI2") endif() - -set(HAVE_OPENNI2 ${HAVE_OPENNI2} PARENT_SCOPE) diff --git a/modules/videoio/cmake/detect_pvapi.cmake b/modules/videoio/cmake/detect_pvapi.cmake index a0f4673fdc..f2c6d4bcea 100644 --- a/modules/videoio/cmake/detect_pvapi.cmake +++ b/modules/videoio/cmake/detect_pvapi.cmake @@ -19,5 +19,3 @@ endif() if(HAVE_PVAPI) ocv_add_external_target(pvapi "${PVAPI_INCLUDE}" "${PVAPI_LIBRARY}" "HAVE_PVAPI") endif() - -set(HAVE_PVAPI ${HAVE_PVAPI} PARENT_SCOPE) diff --git a/modules/videoio/cmake/detect_realsense.cmake b/modules/videoio/cmake/detect_realsense.cmake index 32e5e02c9e..065f548830 100644 --- a/modules/videoio/cmake/detect_realsense.cmake +++ b/modules/videoio/cmake/detect_realsense.cmake @@ -4,7 +4,7 @@ if(NOT HAVE_LIBREALSENSE) find_package(realsense2 QUIET) if(realsense2_FOUND) set(HAVE_LIBREALSENSE TRUE) - set(LIBREALSENSE_VERSION "${realsense2_VERSION}" PARENT_SCOPE) # informational + set(LIBREALSENSE_VERSION "${realsense2_VERSION}") # informational ocv_add_external_target(librealsense "" "${realsense2_LIBRARY}" "HAVE_LIBREALSENSE") endif() endif() @@ -20,7 +20,7 @@ if(NOT HAVE_LIBREALSENSE) string(REGEX REPLACE ".*RS2_API_MAJOR_VERSION[^0-9]+([0-9]+).*" "\\1" ver_major "${ver_strings}") string(REGEX REPLACE ".*RS2_API_MINOR_VERSION[^0-9]+([0-9]+).*" "\\1" ver_minor "${ver_strings}") string(REGEX REPLACE ".*RS2_API_PATCH_VERSION[^0-9]+([0-9]+).*" "\\1" ver_patch "${ver_strings}") - set(LIBREALSENSE_VERSION "${ver_major}.${ver_minor}.${ver_patch}" PARENT_SCOPE) # informational + set(LIBREALSENSE_VERSION "${ver_major}.${ver_minor}.${ver_patch}") # informational ocv_add_external_target(librealsense "${LIBREALSENSE_INCLUDE_DIR}" "${LIBREALSENSE_LIBRARIES}" "HAVE_LIBREALSENSE") endif() endif() diff --git a/modules/videoio/cmake/detect_ueye.cmake b/modules/videoio/cmake/detect_ueye.cmake index 495e9c2450..9428f9e596 100644 --- a/modules/videoio/cmake/detect_ueye.cmake +++ b/modules/videoio/cmake/detect_ueye.cmake @@ -21,5 +21,3 @@ unset(_WIN_LIB_SUFFIX) if(HAVE_UEYE) ocv_add_external_target(ueye "${UEYE_INCLUDE}" "${UEYE_LIBRARY}" "HAVE_UEYE") endif() - -set(HAVE_UEYE ${HAVE_UEYE} PARENT_SCOPE) diff --git a/modules/videoio/cmake/detect_v4l.cmake b/modules/videoio/cmake/detect_v4l.cmake index 05b73b003c..e413dae9ca 100644 --- a/modules/videoio/cmake/detect_v4l.cmake +++ b/modules/videoio/cmake/detect_v4l.cmake @@ -15,5 +15,3 @@ if(NOT HAVE_V4L) ocv_add_external_target(v4l "" "" "${defs}") endif() endif() - -set(HAVE_V4L ${HAVE_V4L} PARENT_SCOPE) diff --git a/modules/videoio/cmake/detect_ximea.cmake b/modules/videoio/cmake/detect_ximea.cmake index 9cf295e352..7521e619b0 100644 --- a/modules/videoio/cmake/detect_ximea.cmake +++ b/modules/videoio/cmake/detect_ximea.cmake @@ -28,5 +28,3 @@ endif() if(HAVE_XIMEA) ocv_add_external_target(ximea "${XIMEA_INCLUDE}" "${XIMEA_LIBRARY}" "HAVE_XIMEA") endif() - -set(HAVE_XIMEA ${HAVE_XIMEA} PARENT_SCOPE) diff --git a/modules/videoio/cmake/detect_xine.cmake b/modules/videoio/cmake/detect_xine.cmake index 3e1f3010a4..0a6f642353 100644 --- a/modules/videoio/cmake/detect_xine.cmake +++ b/modules/videoio/cmake/detect_xine.cmake @@ -5,5 +5,3 @@ endif() if(HAVE_XINE) ocv_add_external_target(xine "${XINE_INCLUDE_DIRS}" "${XINE_LIBRARIES}" "HAVE_XINE") endif() - -set(HAVE_XINE ${HAVE_XINE} PARENT_SCOPE) diff --git a/modules/videoio/cmake/init.cmake b/modules/videoio/cmake/init.cmake index 310df2d249..68838790b8 100644 --- a/modules/videoio/cmake/init.cmake +++ b/modules/videoio/cmake/init.cmake @@ -1,19 +1,10 @@ include(FindPkgConfig) -# FIXIT: stop using PARENT_SCOPE in dependencies -if(PROJECT_NAME STREQUAL "OpenCV") - macro(add_backend backend_id cond_var) - if(${cond_var}) - include("${CMAKE_CURRENT_LIST_DIR}/detect_${backend_id}.cmake") - endif() - endmacro() -else() - function(add_backend backend_id cond_var) - if(${cond_var}) - include("${CMAKE_CURRENT_LIST_DIR}/detect_${backend_id}.cmake") - endif() - endfunction() -endif() +macro(add_backend backend_id cond_var) + if(${cond_var}) + include("${CMAKE_CURRENT_LIST_DIR}/detect_${backend_id}.cmake") + endif() +endmacro() add_backend("ffmpeg" WITH_FFMPEG) add_backend("gstreamer" WITH_GSTREAMER) From 803ff8ebb9ed7960c28e28c6e11e77a783f77d0a Mon Sep 17 00:00:00 2001 From: Shreyas Taware Date: Wed, 7 Jul 2021 14:40:08 +0530 Subject: [PATCH 079/262] Update py_canny.markdown Fixed a word that was previously written as third argument but it is instead the fourth argument of cv.Canny() function --- doc/py_tutorials/py_imgproc/py_canny/py_canny.markdown | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/py_tutorials/py_imgproc/py_canny/py_canny.markdown b/doc/py_tutorials/py_imgproc/py_canny/py_canny.markdown index cbc2a72eec..d36e5784eb 100644 --- a/doc/py_tutorials/py_imgproc/py_canny/py_canny.markdown +++ b/doc/py_tutorials/py_imgproc/py_canny/py_canny.markdown @@ -74,7 +74,7 @@ Canny Edge Detection in OpenCV OpenCV puts all the above in single function, **cv.Canny()**. We will see how to use it. First argument is our input image. Second and third arguments are our minVal and maxVal respectively. -Third argument is aperture_size. It is the size of Sobel kernel used for find image gradients. By +Fourth argument is aperture_size. It is the size of Sobel kernel used for find image gradients. By default it is 3. Last argument is L2gradient which specifies the equation for finding gradient magnitude. If it is True, it uses the equation mentioned above which is more accurate, otherwise it uses this function: \f$Edge\_Gradient \; (G) = |G_x| + |G_y|\f$. By default, it is False. From a76274b549632464987805a29472a9520fb45389 Mon Sep 17 00:00:00 2001 From: Scott Noyes Date: Thu, 22 Jul 2021 14:58:20 -0500 Subject: [PATCH 080/262] minor grammar edits --- .../mat_the_basic_image_container.markdown | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/doc/tutorials/core/mat_the_basic_image_container/mat_the_basic_image_container.markdown b/doc/tutorials/core/mat_the_basic_image_container/mat_the_basic_image_container.markdown index f6a1a0a4fb..aafa9687d9 100644 --- a/doc/tutorials/core/mat_the_basic_image_container/mat_the_basic_image_container.markdown +++ b/doc/tutorials/core/mat_the_basic_image_container/mat_the_basic_image_container.markdown @@ -84,8 +84,8 @@ a new header with the new boundaries: Mat D (A, Rect(10, 10, 100, 100) ); // using a rectangle Mat E = A(Range::all(), Range(1,3)); // using row and column boundaries @endcode -Now you may ask -- if the matrix itself may belong to multiple *Mat* objects who takes responsibility -for cleaning it up when it's no longer needed. The short answer is: the last object that used it. +Now you may ask -- if the matrix itself may belong to multiple *Mat* objects, who takes responsibility +for cleaning it up when it's no longer needed? The short answer is: the last object that used it. This is handled by using a reference counting mechanism. Whenever somebody copies a header of a *Mat* object, a counter is increased for the matrix. Whenever a header is cleaned, this counter is decreased. When the counter reaches zero the matrix is freed. Sometimes you will want to copy @@ -95,12 +95,12 @@ Mat F = A.clone(); Mat G; A.copyTo(G); @endcode -Now modifying *F* or *G* will not affect the matrix pointed by the *A*'s header. What you need to +Now modifying *F* or *G* will not affect the matrix pointed to by the *A*'s header. What you need to remember from all this is that: - Output image allocation for OpenCV functions is automatic (unless specified otherwise). - You do not need to think about memory management with OpenCV's C++ interface. -- The assignment operator and the copy constructor only copies the header. +- The assignment operator and the copy constructor only copy the header. - The underlying matrix of an image may be copied using the @ref cv::Mat::clone() and @ref cv::Mat::copyTo() functions. @@ -115,10 +115,10 @@ of these allows us to create many shades of gray. For *colorful* ways we have a lot more methods to choose from. Each of them breaks it down to three or four basic components and we can use the combination of these to create the others. The most popular one is RGB, mainly because this is also how our eye builds up colors. Its base colors are -red, green and blue. To code the transparency of a color sometimes a fourth element: alpha (A) is +red, green and blue. To code the transparency of a color sometimes a fourth element, alpha (A), is added. -There are, however, many other color systems each with their own advantages: +There are, however, many other color systems, each with their own advantages: - RGB is the most common as our eyes use something similar, however keep in mind that OpenCV standard display system composes colors using the BGR color space (red and blue channels are swapped places). @@ -132,11 +132,11 @@ There are, however, many other color systems each with their own advantages: Each of the building components has its own valid domains. This leads to the data type used. How we store a component defines the control we have over its domain. The smallest data type possible is *char*, which means one byte or 8 bits. This may be unsigned (so can store values from 0 to 255) or -signed (values from -127 to +127). Although in case of three components this already gives 16 -million possible colors to represent (like in case of RGB) we may acquire an even finer control by +signed (values from -127 to +127). Although this width, in the case of three components (like RGB), already gives 16 +million possible colors to represent, we may acquire an even finer control by using the float (4 byte = 32 bit) or double (8 byte = 64 bit) data types for each component. Nevertheless, remember that increasing the size of a component also increases the size of the whole -picture in the memory. +picture in memory. Creating a Mat object explicitly ---------------------------------- From acc576658ad628d46fd4e79c68c1419d438ce716 Mon Sep 17 00:00:00 2001 From: ZhangYin Date: Fri, 23 Jul 2021 22:08:43 +0800 Subject: [PATCH 081/262] Merge pull request #20412 from joy2myself:rvv-0.10 bug fixes for universal intrinsics of RISC-V back-end * Align universal intrinsic comparator behaviour with other platforms Set all bits to one for return value of int and fp comparators. * fix v_pack_triplets, v_pack_store and v_pack_u_store * Remove redundant CV_DECL_ALIGNED statements Co-authored-by: Alexander Smorkalov --- .../include/opencv2/core/hal/intrin_rvv.hpp | 299 +++++++++--------- 1 file changed, 151 insertions(+), 148 deletions(-) diff --git a/modules/core/include/opencv2/core/hal/intrin_rvv.hpp b/modules/core/include/opencv2/core/hal/intrin_rvv.hpp index 4a3455b073..51433cdbae 100644 --- a/modules/core/include/opencv2/core/hal/intrin_rvv.hpp +++ b/modules/core/include/opencv2/core/hal/intrin_rvv.hpp @@ -737,7 +737,7 @@ OPENCV_HAL_IMPL_RVV_LOADSTORE_OP(v_float64x2, vfloat64m1_t, double, 1, 2, 64, f6 inline v_int8x16 v_load_halves(const schar* ptr0, const schar* ptr1) { - schar CV_DECL_ALIGNED(32) elems[16] = + schar elems[16] = { ptr0[0], ptr0[1], ptr0[2], ptr0[3], ptr0[4], ptr0[5], ptr0[6], ptr0[7], ptr1[0], ptr1[1], ptr1[2], ptr1[3], ptr1[4], ptr1[5], ptr1[6], ptr1[7] @@ -748,7 +748,7 @@ inline v_uint8x16 v_load_halves(const uchar* ptr0, const uchar* ptr1) { return v inline v_int16x8 v_load_halves(const short* ptr0, const short* ptr1) { - short CV_DECL_ALIGNED(32) elems[8] = + short elems[8] = { ptr0[0], ptr0[1], ptr0[2], ptr0[3], ptr1[0], ptr1[1], ptr1[2], ptr1[3] }; @@ -758,7 +758,7 @@ inline v_uint16x8 v_load_halves(const ushort* ptr0, const ushort* ptr1) { return inline v_int32x4 v_load_halves(const int* ptr0, const int* ptr1) { - int CV_DECL_ALIGNED(32) elems[4] = + int elems[4] = { ptr0[0], ptr0[1], ptr1[0], ptr1[1] }; @@ -766,7 +766,7 @@ inline v_int32x4 v_load_halves(const int* ptr0, const int* ptr1) } inline v_float32x4 v_load_halves(const float* ptr0, const float* ptr1) { - float CV_DECL_ALIGNED(32) elems[4] = + float elems[4] = { ptr0[0], ptr0[1], ptr1[0], ptr1[1] }; @@ -776,7 +776,7 @@ inline v_uint32x4 v_load_halves(const unsigned* ptr0, const unsigned* ptr1) { re inline v_int64x2 v_load_halves(const int64* ptr0, const int64* ptr1) { - int64 CV_DECL_ALIGNED(32) elems[2] = + int64 elems[2] = { ptr0[0], ptr1[0] }; @@ -787,7 +787,7 @@ inline v_uint64x2 v_load_halves(const uint64* ptr0, const uint64* ptr1) { return #if CV_SIMD128_64F inline v_float64x2 v_load_halves(const double* ptr0, const double* ptr1) { - double CV_DECL_ALIGNED(32) elems[2] = + double elems[2] = { ptr0[0], ptr1[0] }; @@ -800,7 +800,7 @@ inline v_float64x2 v_load_halves(const double* ptr0, const double* ptr1) inline v_int8x16 v_lut(const schar* tab, const int* idx) { - schar CV_DECL_ALIGNED(32) elems[16] = + schar elems[16] = { tab[idx[ 0]], tab[idx[ 1]], @@ -823,7 +823,7 @@ inline v_int8x16 v_lut(const schar* tab, const int* idx) } inline v_int8x16 v_lut_pairs(const schar* tab, const int* idx) { - schar CV_DECL_ALIGNED(32) elems[16] = + schar elems[16] = { tab[idx[0]], tab[idx[0] + 1], @@ -846,7 +846,7 @@ inline v_int8x16 v_lut_pairs(const schar* tab, const int* idx) } inline v_int8x16 v_lut_quads(const schar* tab, const int* idx) { - schar CV_DECL_ALIGNED(32) elems[16] = + schar elems[16] = { tab[idx[0]], tab[idx[0] + 1], @@ -873,7 +873,7 @@ inline v_uint8x16 v_lut_quads(const uchar* tab, const int* idx) { return v_reint inline v_int16x8 v_lut(const short* tab, const int* idx) { - short CV_DECL_ALIGNED(32) elems[8] = + short elems[8] = { tab[idx[0]], tab[idx[1]], @@ -888,7 +888,7 @@ inline v_int16x8 v_lut(const short* tab, const int* idx) } inline v_int16x8 v_lut_pairs(const short* tab, const int* idx) { - short CV_DECL_ALIGNED(32) elems[8] = + short elems[8] = { tab[idx[0]], tab[idx[0] + 1], @@ -903,7 +903,7 @@ inline v_int16x8 v_lut_pairs(const short* tab, const int* idx) } inline v_int16x8 v_lut_quads(const short* tab, const int* idx) { - short CV_DECL_ALIGNED(32) elems[8] = + short elems[8] = { tab[idx[0]], tab[idx[0] + 1], @@ -922,7 +922,7 @@ inline v_uint16x8 v_lut_quads(const ushort* tab, const int* idx) { return v_rein inline v_int32x4 v_lut(const int* tab, const int* idx) { - int CV_DECL_ALIGNED(32) elems[4] = + int elems[4] = { tab[idx[0]], tab[idx[1]], @@ -933,7 +933,7 @@ inline v_int32x4 v_lut(const int* tab, const int* idx) } inline v_int32x4 v_lut_pairs(const int* tab, const int* idx) { - int CV_DECL_ALIGNED(32) elems[4] = + int elems[4] = { tab[idx[0]], tab[idx[0] + 1], @@ -953,7 +953,7 @@ inline v_uint32x4 v_lut_quads(const unsigned* tab, const int* idx) { return v_re inline v_int64x2 v_lut(const int64_t* tab, const int* idx) { - int64_t CV_DECL_ALIGNED(32) elems[2] = + int64_t elems[2] = { tab[idx[0]], tab[idx[1]] @@ -969,7 +969,7 @@ inline v_uint64x2 v_lut_pairs(const uint64* tab, const int* idx) { return v_rein inline v_float32x4 v_lut(const float* tab, const int* idx) { - float CV_DECL_ALIGNED(32) elems[4] = + float elems[4] = { tab[idx[0]], tab[idx[1]], @@ -980,7 +980,7 @@ inline v_float32x4 v_lut(const float* tab, const int* idx) } inline v_float32x4 v_lut_pairs(const float* tab, const int* idx) { - float CV_DECL_ALIGNED(32) elems[4] = + float elems[4] = { tab[idx[0]], tab[idx[0] + 1], @@ -996,7 +996,7 @@ inline v_float32x4 v_lut_quads(const float* tab, const int* idx) inline v_int32x4 v_lut(const int* tab, const v_int32x4& idxvec) { - int CV_DECL_ALIGNED(32) elems[4] = + int elems[4] = { tab[v_extract_n<0>(idxvec)], tab[v_extract_n<1>(idxvec)], @@ -1008,7 +1008,7 @@ inline v_int32x4 v_lut(const int* tab, const v_int32x4& idxvec) inline v_uint32x4 v_lut(const unsigned* tab, const v_int32x4& idxvec) { - unsigned CV_DECL_ALIGNED(32) elems[4] = + unsigned elems[4] = { tab[v_extract_n<0>(idxvec)], tab[v_extract_n<1>(idxvec)], @@ -1020,7 +1020,7 @@ inline v_uint32x4 v_lut(const unsigned* tab, const v_int32x4& idxvec) inline v_float32x4 v_lut(const float* tab, const v_int32x4& idxvec) { - float CV_DECL_ALIGNED(32) elems[4] = + float elems[4] = { tab[v_extract_n<0>(idxvec)], tab[v_extract_n<1>(idxvec)], @@ -1032,7 +1032,7 @@ inline v_float32x4 v_lut(const float* tab, const v_int32x4& idxvec) inline void v_lut_deinterleave(const float* tab, const v_int32x4& idxvec, v_float32x4& x, v_float32x4& y) { - int CV_DECL_ALIGNED(32) idx[4]; + int idx[4]; v_store_aligned(idx, idxvec); x = v_float32x4(tab[idx[0]], tab[idx[1]], tab[idx[2]], tab[idx[3]]); @@ -1042,7 +1042,7 @@ inline void v_lut_deinterleave(const float* tab, const v_int32x4& idxvec, v_floa #if CV_SIMD128_64F inline v_float64x2 v_lut(const double* tab, const int* idx) { - double CV_DECL_ALIGNED(32) elems[2] = + double elems[2] = { tab[idx[0]], tab[idx[1]] @@ -1057,7 +1057,7 @@ inline v_float64x2 v_lut_pairs(const double* tab, const int* idx) inline v_float64x2 v_lut(const double* tab, const v_int32x4& idxvec) { - double CV_DECL_ALIGNED(32) elems[2] = + double elems[2] = { tab[v_extract_n<0>(idxvec)], tab[v_extract_n<1>(idxvec)] @@ -1067,7 +1067,7 @@ inline v_float64x2 v_lut(const double* tab, const v_int32x4& idxvec) inline void v_lut_deinterleave(const double* tab, const v_int32x4& idxvec, v_float64x2& x, v_float64x2& y) { - int CV_DECL_ALIGNED(32) idx[4] = {0}; + int idx[4] = {0}; v_store_aligned(idx, idxvec); x = v_float64x2(tab[idx[0]], tab[idx[1]]); @@ -1079,7 +1079,7 @@ inline void v_lut_deinterleave(const double* tab, const v_int32x4& idxvec, v_flo inline v_uint8x16 v_pack_b(const v_uint16x8& a, const v_uint16x8& b) { - ushort CV_DECL_ALIGNED(32) ptr[16] = {0}; + ushort ptr[16] = {0}; v_store(ptr, a); v_store(ptr + 8, b); return v_uint8x16(vnsrl_wx_u8m1(vle16_v_u16m2(ptr, 16), 0, 16)); @@ -1088,7 +1088,7 @@ inline v_uint8x16 v_pack_b(const v_uint16x8& a, const v_uint16x8& b) inline v_uint8x16 v_pack_b(const v_uint32x4& a, const v_uint32x4& b, const v_uint32x4& c, const v_uint32x4& d) { - unsigned CV_DECL_ALIGNED(32) ptr[16] = {0}; + unsigned ptr[16] = {0}; v_store(ptr, a); v_store(ptr + 4, b); v_store(ptr + 8, c); @@ -1100,7 +1100,7 @@ inline v_uint8x16 v_pack_b(const v_uint64x2& a, const v_uint64x2& b, const v_uin const v_uint64x2& d, const v_uint64x2& e, const v_uint64x2& f, const v_uint64x2& g, const v_uint64x2& h) { - uint64 CV_DECL_ALIGNED(32) ptr[16] = {0}; + uint64 ptr[16] = {0}; v_store(ptr, a); v_store(ptr + 2, b); v_store(ptr + 4, c); @@ -1279,13 +1279,15 @@ OPENCV_HAL_IMPL_RVV_SIGNED_SHIFT_OP(v_int64x2, i64, 2) #define OPENCV_HAL_IMPL_RVV_INT_CMP_OP(_Tpvec, op, intrin, suffix, vl) \ inline _Tpvec operator op (const _Tpvec& a, const _Tpvec& b) \ { \ - return _Tpvec(vmerge_vxm_##suffix##m1(intrin(a, b, vl), vmv_v_x_##suffix##m1(0, vl), 1, vl)); \ + uint64_t ones = -1; \ + return _Tpvec(vmerge_vxm_##suffix##m1(intrin(a, b, vl), vmv_v_x_##suffix##m1(0, vl), ones, vl)); \ } #define OPENCV_HAL_IMPL_RVV_FLOAT_CMP_OP(_Tpvec, op, intrin, suffix, vl) \ inline _Tpvec operator op (const _Tpvec& a, const _Tpvec& b) \ { \ - return _Tpvec(vfmerge_vfm_##suffix##m1(intrin(a, b, vl), vfmv_v_f_##suffix##m1(0, vl), 1, vl)); \ + union { uint64 u; double d; } ones; ones.u = -1; \ + return _Tpvec(vfmerge_vfm_##suffix##m1(intrin(a, b, vl), vfmv_v_f_##suffix##m1(0, vl), ones.d, vl)); \ } #define OPENCV_HAL_IMPL_RVV_UNSIGNED_CMP(_Tpvec, suffix, width, vl) \ @@ -1441,7 +1443,7 @@ OPENCV_HAL_IMPL_RVV_REDUCE(v_float32x4, max, float, f32, 4, fredmax) inline v_float32x4 v_reduce_sum4(const v_float32x4& a, const v_float32x4& b, const v_float32x4& c, const v_float32x4& d) { - float CV_DECL_ALIGNED(32) elems[4] = + float elems[4] = { v_reduce_sum(a), v_reduce_sum(b), @@ -1746,9 +1748,9 @@ inline v_float32x4 v_cvt_f32(const v_float64x2& a, const v_float64x2& b) inline v_float64x2 v_cvt_f64(const v_int32x4& a) { - double CV_DECL_ALIGNED(32) ptr[4] = {0}; + double ptr[4] = {0}; vse64_v_f64m2(ptr, vfwcvt_f_x_v_f64m2(a, 4), 4); - double CV_DECL_ALIGNED(32) elems[2] = + double elems[2] = { ptr[0], ptr[1] }; @@ -1757,9 +1759,9 @@ inline v_float64x2 v_cvt_f64(const v_int32x4& a) inline v_float64x2 v_cvt_f64_high(const v_int32x4& a) { - double CV_DECL_ALIGNED(32) ptr[4] = {0}; + double ptr[4] = {0}; vse64_v_f64m2(ptr, vfwcvt_f_x_v_f64m2(a, 4), 4); - double CV_DECL_ALIGNED(32) elems[2] = + double elems[2] = { ptr[2], ptr[3] }; @@ -1768,9 +1770,9 @@ inline v_float64x2 v_cvt_f64_high(const v_int32x4& a) inline v_float64x2 v_cvt_f64(const v_float32x4& a) { - double CV_DECL_ALIGNED(32) ptr[4] = {0}; + double ptr[4] = {0}; vse64_v_f64m2(ptr, vfwcvt_f_f_v_f64m2(a, 4), 4); - double CV_DECL_ALIGNED(32) elems[2] = + double elems[2] = { ptr[0], ptr[1] }; @@ -1779,9 +1781,9 @@ inline v_float64x2 v_cvt_f64(const v_float32x4& a) inline v_float64x2 v_cvt_f64_high(const v_float32x4& a) { - double CV_DECL_ALIGNED(32) ptr[4] = {0}; + double ptr[4] = {0}; vse64_v_f64m2(ptr, vfwcvt_f_f_v_f64m2(a, 4), 4); - double CV_DECL_ALIGNED(32) elems[2] = + double elems[2] = { ptr[2], ptr[3] }; @@ -1823,7 +1825,7 @@ inline void v_transpose4x4(const v_##_Tpvec& a0, const v_##_Tpvec& a1, \ v_##_Tpvec& b0, v_##_Tpvec& b1, \ v_##_Tpvec& b2, v_##_Tpvec& b3) \ { \ - _Tp CV_DECL_ALIGNED(32) elems0[4] = \ + _Tp elems0[4] = \ { \ v_extract_n<0>(a0), \ v_extract_n<0>(a1), \ @@ -1831,7 +1833,7 @@ inline void v_transpose4x4(const v_##_Tpvec& a0, const v_##_Tpvec& a1, \ v_extract_n<0>(a3) \ }; \ b0 = v_load(elems0); \ - _Tp CV_DECL_ALIGNED(32) elems1[4] = \ + _Tp elems1[4] = \ { \ v_extract_n<1>(a0), \ v_extract_n<1>(a1), \ @@ -1839,7 +1841,7 @@ inline void v_transpose4x4(const v_##_Tpvec& a0, const v_##_Tpvec& a1, \ v_extract_n<1>(a3) \ }; \ b1 = v_load(elems1); \ - _Tp CV_DECL_ALIGNED(32) elems2[4] = \ + _Tp elems2[4] = \ { \ v_extract_n<2>(a0), \ v_extract_n<2>(a1), \ @@ -1847,7 +1849,7 @@ inline void v_transpose4x4(const v_##_Tpvec& a0, const v_##_Tpvec& a1, \ v_extract_n<2>(a3) \ }; \ b2 = v_load(elems2); \ - _Tp CV_DECL_ALIGNED(32) elems3[4] = \ + _Tp elems3[4] = \ { \ v_extract_n<3>(a0), \ v_extract_n<3>(a1), \ @@ -1866,8 +1868,8 @@ OPENCV_HAL_IMPL_RVV_TRANSPOSE4x4(float32x4, float, f32) #define OPENCV_HAL_IMPL_RVV_REVERSE(_Tpvec, _Tp, suffix) \ inline _Tpvec v_reverse(const _Tpvec& a) \ { \ - _Tp CV_DECL_ALIGNED(32) ptr[_Tpvec::nlanes] = {0}; \ - _Tp CV_DECL_ALIGNED(32) ptra[_Tpvec::nlanes] = {0}; \ + _Tp ptr[_Tpvec::nlanes] = {0}; \ + _Tp ptra[_Tpvec::nlanes] = {0}; \ v_store(ptra, a); \ for (int i = 0; i < _Tpvec::nlanes; i++) \ { \ @@ -1894,8 +1896,8 @@ OPENCV_HAL_IMPL_RVV_REVERSE(v_float64x2, double, f64) #define OPENCV_HAL_IMPL_RVV_EXPAND(_Tpwvec, _Tp, _Tpvec, width, suffix, wcvt, vl) \ inline void v_expand(const _Tpvec& a, _Tpwvec& b0, _Tpwvec& b1) \ { \ - _Tp CV_DECL_ALIGNED(32) lptr[_Tpvec::nlanes/2] = {0}; \ - _Tp CV_DECL_ALIGNED(32) hptr[_Tpvec::nlanes/2] = {0}; \ + _Tp lptr[_Tpvec::nlanes/2] = {0}; \ + _Tp hptr[_Tpvec::nlanes/2] = {0}; \ v_store_low(lptr, a); \ v_store_high(hptr, a); \ b0 = _Tpwvec(wcvt(vle##width##_v_##suffix##mf2(lptr, vl), vl)); \ @@ -1903,13 +1905,13 @@ inline void v_expand(const _Tpvec& a, _Tpwvec& b0, _Tpwvec& b1) \ } \ inline _Tpwvec v_expand_low(const _Tpvec& a) \ { \ - _Tp CV_DECL_ALIGNED(32) lptr[_Tpvec::nlanes/2] = {0}; \ + _Tp lptr[_Tpvec::nlanes/2] = {0}; \ v_store_low(lptr, a); \ return _Tpwvec(wcvt(vle##width##_v_##suffix##mf2(lptr, vl), vl)); \ } \ inline _Tpwvec v_expand_high(const _Tpvec& a) \ { \ - _Tp CV_DECL_ALIGNED(32) hptr[_Tpvec::nlanes/2] = {0}; \ + _Tp hptr[_Tpvec::nlanes/2] = {0}; \ v_store_high(hptr, a); \ return _Tpwvec(wcvt(vle##width##_v_##suffix##mf2(hptr, vl), vl)); \ } \ @@ -1936,25 +1938,25 @@ inline v_int32x4 v_load_expand_q(const schar* ptr) } -#define OPENCV_HAL_IMPL_RVV_PACK(_Tpvec, _Tp, _wTpvec, _wTp, width, suffix, rshr, shr, hvl, vl) \ +#define OPENCV_HAL_IMPL_RVV_PACK(_Tpvec, _Tp, _wTpvec, _wTp, hwidth, width, hsuffix, suffix, rshr, shr, hvl, vl) \ inline _Tpvec v_pack(const _wTpvec& a, const _wTpvec& b) \ { \ - _wTp CV_DECL_ALIGNED(32) arr[_Tpvec::nlanes] = {0}; \ + _wTp arr[_Tpvec::nlanes] = {0}; \ v_store(arr, a); \ v_store(arr + _wTpvec::nlanes, b); \ return _Tpvec(shr(vle##width##_v_##suffix##m2(arr, vl), 0, vl)); \ } \ inline void v_pack_store(_Tp* ptr, const _wTpvec& a) \ { \ - _wTp CV_DECL_ALIGNED(32) arr[_Tpvec::nlanes] = {0}; \ + _wTp arr[_Tpvec::nlanes] = {0}; \ v_store(arr, a); \ v_store(arr + _wTpvec::nlanes, _wTpvec(vmv_v_x_##suffix##m1(0, hvl))); \ - v_store(ptr, _Tpvec(shr(vle##width##_v_##suffix##m2(arr, vl), 0, vl))); \ + vse##hwidth##_v_##hsuffix##m1(ptr, shr(vle##width##_v_##suffix##m2(arr, vl), 0, vl), hvl); \ } \ template inline \ _Tpvec v_rshr_pack(const _wTpvec& a, const _wTpvec& b) \ { \ - _wTp CV_DECL_ALIGNED(32) arr[_Tpvec::nlanes] = {0}; \ + _wTp arr[_Tpvec::nlanes] = {0}; \ v_store(arr, a); \ v_store(arr + _wTpvec::nlanes, b); \ return _Tpvec(rshr(vle##width##_v_##suffix##m2(arr, vl), n, vl)); \ @@ -1962,39 +1964,39 @@ _Tpvec v_rshr_pack(const _wTpvec& a, const _wTpvec& b) \ template inline \ void v_rshr_pack_store(_Tp* ptr, const _wTpvec& a) \ { \ - _wTp CV_DECL_ALIGNED(32) arr[_Tpvec::nlanes] = {0}; \ + _wTp arr[_Tpvec::nlanes] = {0}; \ v_store(arr, a); \ v_store(arr + _wTpvec::nlanes, _wTpvec(vmv_v_x_##suffix##m1(0, hvl))); \ v_store(ptr, _Tpvec(rshr(vle##width##_v_##suffix##m2(arr, vl), n, vl))); \ } -OPENCV_HAL_IMPL_RVV_PACK(v_uint8x16, uchar, v_uint16x8, ushort, 16, u16, vnclipu_wx_u8m1, vnclipu_wx_u8m1, 8, 16) -OPENCV_HAL_IMPL_RVV_PACK(v_int8x16, schar, v_int16x8, short, 16, i16, vnclip_wx_i8m1, vnclip_wx_i8m1, 8, 16) -OPENCV_HAL_IMPL_RVV_PACK(v_uint16x8, ushort, v_uint32x4, unsigned, 32, u32, vnclipu_wx_u16m1, vnclipu_wx_u16m1, 4, 8) -OPENCV_HAL_IMPL_RVV_PACK(v_int16x8, short, v_int32x4, int, 32, i32, vnclip_wx_i16m1, vnclip_wx_i16m1, 4, 8) -OPENCV_HAL_IMPL_RVV_PACK(v_uint32x4, unsigned, v_uint64x2, uint64, 64, u64, vnclipu_wx_u32m1, vnsrl_wx_u32m1, 2, 4) -OPENCV_HAL_IMPL_RVV_PACK(v_int32x4, int, v_int64x2, int64, 64, i64, vnclip_wx_i32m1, vnsra_wx_i32m1, 2, 4) +OPENCV_HAL_IMPL_RVV_PACK(v_uint8x16, uchar, v_uint16x8, ushort, 8, 16, u8, u16, vnclipu_wx_u8m1, vnclipu_wx_u8m1, 8, 16) +OPENCV_HAL_IMPL_RVV_PACK(v_int8x16, schar, v_int16x8, short, 8, 16, i8, i16, vnclip_wx_i8m1, vnclip_wx_i8m1, 8, 16) +OPENCV_HAL_IMPL_RVV_PACK(v_uint16x8, ushort, v_uint32x4, unsigned, 16, 32, u16, u32, vnclipu_wx_u16m1, vnclipu_wx_u16m1, 4, 8) +OPENCV_HAL_IMPL_RVV_PACK(v_int16x8, short, v_int32x4, int, 16, 32, i16, i32, vnclip_wx_i16m1, vnclip_wx_i16m1, 4, 8) +OPENCV_HAL_IMPL_RVV_PACK(v_uint32x4, unsigned, v_uint64x2, uint64, 32, 64, u32, u64, vnclipu_wx_u32m1, vnsrl_wx_u32m1, 2, 4) +OPENCV_HAL_IMPL_RVV_PACK(v_int32x4, int, v_int64x2, int64, 32, 64, i32, i64, vnclip_wx_i32m1, vnsra_wx_i32m1, 2, 4) -#define OPENCV_HAL_IMPL_RVV_PACK_U(_Tpvec, _Tp, _wTpvec, _wTp, width, suffix, rshr, cast, vl) \ +#define OPENCV_HAL_IMPL_RVV_PACK_U(_Tpvec, _Tp, _wTpvec, _wTp, hwidth, width, hsuffix, suffix, rshr, cast, hvl, vl) \ inline _Tpvec v_pack_u(const _wTpvec& a, const _wTpvec& b) \ { \ - _wTp CV_DECL_ALIGNED(32) arr[_Tpvec::nlanes] = {0}; \ + _wTp arr[_Tpvec::nlanes] = {0}; \ v_store(arr, a); \ v_store(arr + _wTpvec::nlanes, b); \ return _Tpvec(rshr(cast(vmax_vx_##suffix##m2(vle##width##_v_##suffix##m2(arr, vl), 0, vl)), 0, vl)); \ } \ inline void v_pack_u_store(_Tp* ptr, const _wTpvec& a) \ { \ - _wTp CV_DECL_ALIGNED(32) arr[_Tpvec::nlanes] = {0}; \ + _wTp arr[_Tpvec::nlanes] = {0}; \ v_store(arr, a); \ - v_store(arr + _wTpvec::nlanes, _wTpvec(vmv_v_x_##suffix##m1(0, vl))); \ - v_store(ptr, _Tpvec(rshr(cast(vmax_vx_##suffix##m2(vle##width##_v_##suffix##m2(arr, vl), 0, vl)), 0, vl))); \ + v_store(arr + _wTpvec::nlanes, _wTpvec(vmv_v_x_##suffix##m1(0, hvl))); \ + vse##hwidth##_v_##hsuffix##m1(ptr, rshr(cast(vmax_vx_##suffix##m2(vle##width##_v_##suffix##m2(arr, vl), 0, vl)), 0, vl), hvl); \ } \ template inline \ _Tpvec v_rshr_pack_u(const _wTpvec& a, const _wTpvec& b) \ { \ - _wTp CV_DECL_ALIGNED(32) arr[_Tpvec::nlanes] = {0}; \ + _wTp arr[_Tpvec::nlanes] = {0}; \ v_store(arr, a); \ v_store(arr + _wTpvec::nlanes, b); \ return _Tpvec(rshr(cast(vmax_vx_##suffix##m2(vle##width##_v_##suffix##m2(arr, vl), 0, vl)), n, vl)); \ @@ -2002,23 +2004,23 @@ _Tpvec v_rshr_pack_u(const _wTpvec& a, const _wTpvec& b) \ template inline \ void v_rshr_pack_u_store(_Tp* ptr, const _wTpvec& a) \ { \ - _wTp CV_DECL_ALIGNED(32) arr[_Tpvec::nlanes] = {0}; \ + _wTp arr[_Tpvec::nlanes] = {0}; \ v_store(arr, a); \ - v_store(arr + _wTpvec::nlanes, _wTpvec(vmv_v_x_##suffix##m1(0, vl))); \ + v_store(arr + _wTpvec::nlanes, _wTpvec(vmv_v_x_##suffix##m1(0, hvl))); \ v_store(ptr, _Tpvec(rshr(cast(vmax_vx_##suffix##m2(vle##width##_v_##suffix##m2(arr, vl), 0, vl)), n, vl))); \ } -OPENCV_HAL_IMPL_RVV_PACK_U(v_uint8x16, uchar, v_int16x8, short, 16, i16, vnclipu_wx_u8m1, vreinterpret_v_i16m2_u16m2, 16) -OPENCV_HAL_IMPL_RVV_PACK_U(v_uint16x8, ushort, v_int32x4, int, 32, i32, vnclipu_wx_u16m1, vreinterpret_v_i32m2_u32m2, 8) +OPENCV_HAL_IMPL_RVV_PACK_U(v_uint8x16, uchar, v_int16x8, short, 8, 16, u8, i16, vnclipu_wx_u8m1, vreinterpret_v_i16m2_u16m2, 8, 16) +OPENCV_HAL_IMPL_RVV_PACK_U(v_uint16x8, ushort, v_int32x4, int, 16, 32, u16, i32, vnclipu_wx_u16m1, vreinterpret_v_i32m2_u32m2, 4, 8) #define OPENCV_HAL_IMPL_RVV_UNPACKS(_Tpvec, _Tp, suffix) \ inline void v_zip(const v_##_Tpvec& a0, const v_##_Tpvec& a1, v_##_Tpvec& b0, v_##_Tpvec& b1) \ { \ - _Tp CV_DECL_ALIGNED(32) ptra0[v_##_Tpvec::nlanes] = {0}; \ - _Tp CV_DECL_ALIGNED(32) ptra1[v_##_Tpvec::nlanes] = {0}; \ - _Tp CV_DECL_ALIGNED(32) ptrb0[v_##_Tpvec::nlanes] = {0}; \ - _Tp CV_DECL_ALIGNED(32) ptrb1[v_##_Tpvec::nlanes] = {0}; \ + _Tp ptra0[v_##_Tpvec::nlanes] = {0}; \ + _Tp ptra1[v_##_Tpvec::nlanes] = {0}; \ + _Tp ptrb0[v_##_Tpvec::nlanes] = {0}; \ + _Tp ptrb1[v_##_Tpvec::nlanes] = {0}; \ v_store(ptra0, a0); \ v_store(ptra1, a1); \ int i; \ @@ -2037,16 +2039,16 @@ inline void v_zip(const v_##_Tpvec& a0, const v_##_Tpvec& a1, v_##_Tpvec& b0, v_ } \ inline v_##_Tpvec v_combine_low(const v_##_Tpvec& a, const v_##_Tpvec& b) \ { \ - _Tp CV_DECL_ALIGNED(32) ptra[v_##_Tpvec::nlanes/2] = {0}; \ - _Tp CV_DECL_ALIGNED(32) ptrb[v_##_Tpvec::nlanes/2] = {0}; \ + _Tp ptra[v_##_Tpvec::nlanes/2] = {0}; \ + _Tp ptrb[v_##_Tpvec::nlanes/2] = {0}; \ v_store_low(ptra, a); \ v_store_low(ptrb, b); \ return v_load_halves(ptra, ptrb); \ } \ inline v_##_Tpvec v_combine_high(const v_##_Tpvec& a, const v_##_Tpvec& b) \ { \ - _Tp CV_DECL_ALIGNED(32) ptra[v_##_Tpvec::nlanes/2] = {0}; \ - _Tp CV_DECL_ALIGNED(32) ptrb[v_##_Tpvec::nlanes/2] = {0}; \ + _Tp ptra[v_##_Tpvec::nlanes/2] = {0}; \ + _Tp ptrb[v_##_Tpvec::nlanes/2] = {0}; \ v_store_high(ptra, a); \ v_store_high(ptrb, b); \ return v_load_halves(ptra, ptrb); \ @@ -2072,8 +2074,8 @@ OPENCV_HAL_IMPL_RVV_UNPACKS(float64x2, double, f64) #define OPENCV_HAL_IMPL_RVV_INTERLEAVED(_Tpvec, _Tp) \ inline void v_load_deinterleave(const _Tp* ptr, v_##_Tpvec& a, v_##_Tpvec& b) \ { \ - _Tp CV_DECL_ALIGNED(32) ptra[v_##_Tpvec::nlanes] = {0}; \ - _Tp CV_DECL_ALIGNED(32) ptrb[v_##_Tpvec::nlanes] = {0}; \ + _Tp ptra[v_##_Tpvec::nlanes] = {0}; \ + _Tp ptrb[v_##_Tpvec::nlanes] = {0}; \ int i, i2; \ for( i = i2 = 0; i < v_##_Tpvec::nlanes; i++, i2 += 2 ) \ { \ @@ -2085,9 +2087,9 @@ inline void v_load_deinterleave(const _Tp* ptr, v_##_Tpvec& a, v_##_Tpvec& b) \ } \ inline void v_load_deinterleave(const _Tp* ptr, v_##_Tpvec& a, v_##_Tpvec& b, v_##_Tpvec& c) \ { \ - _Tp CV_DECL_ALIGNED(32) ptra[v_##_Tpvec::nlanes] = {0}; \ - _Tp CV_DECL_ALIGNED(32) ptrb[v_##_Tpvec::nlanes] = {0}; \ - _Tp CV_DECL_ALIGNED(32) ptrc[v_##_Tpvec::nlanes] = {0}; \ + _Tp ptra[v_##_Tpvec::nlanes] = {0}; \ + _Tp ptrb[v_##_Tpvec::nlanes] = {0}; \ + _Tp ptrc[v_##_Tpvec::nlanes] = {0}; \ int i, i3; \ for( i = i3 = 0; i < v_##_Tpvec::nlanes; i++, i3 += 3 ) \ { \ @@ -2102,10 +2104,10 @@ inline void v_load_deinterleave(const _Tp* ptr, v_##_Tpvec& a, v_##_Tpvec& b, v_ inline void v_load_deinterleave(const _Tp* ptr, v_##_Tpvec& a, v_##_Tpvec& b, \ v_##_Tpvec& c, v_##_Tpvec& d) \ { \ - _Tp CV_DECL_ALIGNED(32) ptra[v_##_Tpvec::nlanes] = {0}; \ - _Tp CV_DECL_ALIGNED(32) ptrb[v_##_Tpvec::nlanes] = {0}; \ - _Tp CV_DECL_ALIGNED(32) ptrc[v_##_Tpvec::nlanes] = {0}; \ - _Tp CV_DECL_ALIGNED(32) ptrd[v_##_Tpvec::nlanes] = {0}; \ + _Tp ptra[v_##_Tpvec::nlanes] = {0}; \ + _Tp ptrb[v_##_Tpvec::nlanes] = {0}; \ + _Tp ptrc[v_##_Tpvec::nlanes] = {0}; \ + _Tp ptrd[v_##_Tpvec::nlanes] = {0}; \ int i, i4; \ for( i = i4 = 0; i < v_##_Tpvec::nlanes; i++, i4 += 4 ) \ { \ @@ -2123,8 +2125,8 @@ inline void v_store_interleave( _Tp* ptr, const v_##_Tpvec& a, const v_##_Tpvec& hal::StoreMode /*mode*/=hal::STORE_UNALIGNED) \ { \ int i, i2; \ - _Tp CV_DECL_ALIGNED(32) ptra[v_##_Tpvec::nlanes] = {0}; \ - _Tp CV_DECL_ALIGNED(32) ptrb[v_##_Tpvec::nlanes] = {0}; \ + _Tp ptra[v_##_Tpvec::nlanes] = {0}; \ + _Tp ptrb[v_##_Tpvec::nlanes] = {0}; \ v_store(ptra, a); \ v_store(ptrb, b); \ for( i = i2 = 0; i < v_##_Tpvec::nlanes; i++, i2 += 2 ) \ @@ -2137,9 +2139,9 @@ inline void v_store_interleave( _Tp* ptr, const v_##_Tpvec& a, const v_##_Tpvec& const v_##_Tpvec& c, hal::StoreMode /*mode*/=hal::STORE_UNALIGNED) \ { \ int i, i3; \ - _Tp CV_DECL_ALIGNED(32) ptra[v_##_Tpvec::nlanes] = {0}; \ - _Tp CV_DECL_ALIGNED(32) ptrb[v_##_Tpvec::nlanes] = {0}; \ - _Tp CV_DECL_ALIGNED(32) ptrc[v_##_Tpvec::nlanes] = {0}; \ + _Tp ptra[v_##_Tpvec::nlanes] = {0}; \ + _Tp ptrb[v_##_Tpvec::nlanes] = {0}; \ + _Tp ptrc[v_##_Tpvec::nlanes] = {0}; \ v_store(ptra, a); \ v_store(ptrb, b); \ v_store(ptrc, c); \ @@ -2155,10 +2157,10 @@ inline void v_store_interleave( _Tp* ptr, const v_##_Tpvec& a, const v_##_Tpvec& hal::StoreMode /*mode*/=hal::STORE_UNALIGNED ) \ { \ int i, i4; \ - _Tp CV_DECL_ALIGNED(32) ptra[v_##_Tpvec::nlanes] = {0}; \ - _Tp CV_DECL_ALIGNED(32) ptrb[v_##_Tpvec::nlanes] = {0}; \ - _Tp CV_DECL_ALIGNED(32) ptrc[v_##_Tpvec::nlanes] = {0}; \ - _Tp CV_DECL_ALIGNED(32) ptrd[v_##_Tpvec::nlanes] = {0}; \ + _Tp ptra[v_##_Tpvec::nlanes] = {0}; \ + _Tp ptrb[v_##_Tpvec::nlanes] = {0}; \ + _Tp ptrc[v_##_Tpvec::nlanes] = {0}; \ + _Tp ptrd[v_##_Tpvec::nlanes] = {0}; \ v_store(ptra, a); \ v_store(ptrb, b); \ v_store(ptrc, c); \ @@ -2173,8 +2175,8 @@ inline void v_store_interleave( _Tp* ptr, const v_##_Tpvec& a, const v_##_Tpvec& } \ inline v_##_Tpvec v_interleave_pairs(const v_##_Tpvec& vec) \ { \ - _Tp CV_DECL_ALIGNED(32) ptr[v_##_Tpvec::nlanes] = {0}; \ - _Tp CV_DECL_ALIGNED(32) ptrvec[v_##_Tpvec::nlanes] = {0}; \ + _Tp ptr[v_##_Tpvec::nlanes] = {0}; \ + _Tp ptrvec[v_##_Tpvec::nlanes] = {0}; \ v_store(ptrvec, vec); \ for (int i = 0; i < v_##_Tpvec::nlanes/4; i++) \ { \ @@ -2187,8 +2189,8 @@ inline v_##_Tpvec v_interleave_pairs(const v_##_Tpvec& vec) \ } \ inline v_##_Tpvec v_interleave_quads(const v_##_Tpvec& vec) \ { \ - _Tp CV_DECL_ALIGNED(32) ptr[v_##_Tpvec::nlanes] = {0}; \ - _Tp CV_DECL_ALIGNED(32) ptrvec[v_##_Tpvec::nlanes] = {0}; \ + _Tp ptr[v_##_Tpvec::nlanes] = {0}; \ + _Tp ptrvec[v_##_Tpvec::nlanes] = {0}; \ v_store(ptrvec, vec); \ for (int i = 0; i < v_##_Tpvec::nlanes/8; i++) \ { \ @@ -2242,9 +2244,9 @@ static const unsigned char popCountTable[] = #define OPENCV_HAL_IMPL_RVV_POPCOUNT_OP(_rTpvec, _Tpvec, _rTp, _Tp, suffix) \ inline _rTpvec v_popcount(const _Tpvec& a) \ { \ - uchar CV_DECL_ALIGNED(32) ptra[16] = {0}; \ + uchar ptra[16] = {0}; \ v_store(ptra, v_reinterpret_as_u8(a)); \ - _rTp CV_DECL_ALIGNED(32) ptr[_Tpvec::nlanes] = {0}; \ + _rTp ptr[_Tpvec::nlanes] = {0}; \ v_store(ptr, v_setzero_##suffix()); \ for (int i = 0; i < _Tpvec::nlanes*(int)sizeof(_Tp); i++) \ ptr[i/sizeof(_Tp)] += popCountTable[ptra[i]]; \ @@ -2298,7 +2300,7 @@ inline int v_signmask(const v_float64x2& a) #define OPENCV_HAL_IMPL_RVV_SCAN_FORWOARD_OP(_Tpvec, _Tp, suffix) \ inline int v_scan_forward(const _Tpvec& a) \ { \ - _Tp CV_DECL_ALIGNED(32) ptr[_Tpvec::nlanes] = {0}; \ + _Tp ptr[_Tpvec::nlanes] = {0}; \ v_store(ptr, v_reinterpret_as_##suffix(a)); \ for (int i = 0; i < _Tpvec::nlanes; i++) \ if(int(ptr[i]) < 0) \ @@ -2321,28 +2323,29 @@ OPENCV_HAL_IMPL_RVV_SCAN_FORWOARD_OP(v_float64x2, double, f64) //////////// Pack triplets //////////// -#define OPENCV_HAL_IMPL_RVV_PACK_TRIPLETS(_Tpvec, _Tp) \ -inline _Tpvec v_pack_triplets(const _Tpvec& vec) \ -{ \ - _Tp CV_DECL_ALIGNED(32) ptr[_Tpvec::nlanes] = {0}; \ - _Tp CV_DECL_ALIGNED(32) ptrvec[_Tpvec::nlanes] = {0}; \ - v_store(ptrvec, vec); \ - for (int i = 0; i < _Tpvec::nlanes/4; i++) \ - { \ - ptr[3*i ] = ptrvec[4*i ]; \ - ptr[3*i+1] = ptrvec[4*i+2]; \ - ptr[3*i+2] = ptrvec[4*i+2]; \ - } \ - return v_load(ptr); \ +inline v_int8x16 v_pack_triplets(const v_int8x16& vec) +{ + uint64 ptr[2] = {0x0908060504020100, 0xFFFFFFFF0E0D0C0A}; + return v_int8x16((vint8m1_t)vrgather_vv_u8m1((vuint8m1_t)vint8m1_t(vec), (vuint8m1_t)vle64_v_u64m1(ptr, 2), 16)); +} +inline v_uint8x16 v_pack_triplets(const v_uint8x16& vec) +{ + return v_reinterpret_as_u8(v_pack_triplets(v_reinterpret_as_s8(vec))); } -OPENCV_HAL_IMPL_RVV_PACK_TRIPLETS(v_uint8x16, uchar) -OPENCV_HAL_IMPL_RVV_PACK_TRIPLETS(v_int8x16, schar) -OPENCV_HAL_IMPL_RVV_PACK_TRIPLETS(v_uint16x8, ushort) -OPENCV_HAL_IMPL_RVV_PACK_TRIPLETS(v_int16x8, short) -OPENCV_HAL_IMPL_RVV_PACK_TRIPLETS(v_uint32x4, unsigned) -OPENCV_HAL_IMPL_RVV_PACK_TRIPLETS(v_int32x4, int) -OPENCV_HAL_IMPL_RVV_PACK_TRIPLETS(v_float32x4, float) +inline v_int16x8 v_pack_triplets(const v_int16x8& vec) +{ + uint64 ptr[2] = {0x0908060504020100, 0xFFFFFFFF0E0D0C0A}; + return v_int16x8((vint16m1_t)vrgather_vv_u8m1((vuint8m1_t)vint16m1_t(vec), (vuint8m1_t)vle64_v_u64m1(ptr, 2), 16)); +} +inline v_uint16x8 v_pack_triplets(const v_uint16x8& vec) +{ + return v_reinterpret_as_u16(v_pack_triplets(v_reinterpret_as_s16(vec))); +} + +inline v_int32x4 v_pack_triplets(const v_int32x4& vec) { return vec; } +inline v_uint32x4 v_pack_triplets(const v_uint32x4& vec) { return vec; } +inline v_float32x4 v_pack_triplets(const v_float32x4& vec) { return vec; } ////// FP16 support /////// @@ -2443,7 +2446,7 @@ inline v_int32x4 v_trunc(const v_float64x2& a) // 16 >> 32 inline v_int32x4 v_dotprod(const v_int16x8& a, const v_int16x8& b) { - int CV_DECL_ALIGNED(32) ptr[8] = {0}; + int ptr[8] = {0}; v_int32x4 t1, t2; vse32_v_i32m2(ptr, vwmul_vv_i32m2(a, b, 8), 8); v_load_deinterleave(ptr, t1, t2); @@ -2451,7 +2454,7 @@ inline v_int32x4 v_dotprod(const v_int16x8& a, const v_int16x8& b) } inline v_int32x4 v_dotprod(const v_int16x8& a, const v_int16x8& b, const v_int32x4& c) { - int CV_DECL_ALIGNED(32) ptr[8] = {0}; + int ptr[8] = {0}; v_int32x4 t1, t2; vse32_v_i32m2(ptr, vwmul_vv_i32m2(a, b, 8), 8); v_load_deinterleave(ptr, t1, t2); @@ -2461,7 +2464,7 @@ inline v_int32x4 v_dotprod(const v_int16x8& a, const v_int16x8& b, const v_int32 // 32 >> 64 inline v_int64x2 v_dotprod(const v_int32x4& a, const v_int32x4& b) { - int64 CV_DECL_ALIGNED(32) ptr[4] = {0}; + int64 ptr[4] = {0}; v_int64x2 t1, t2; vse64_v_i64m2(ptr, vwmul_vv_i64m2(a, b, 4), 4); v_load_deinterleave(ptr, t1, t2); @@ -2469,7 +2472,7 @@ inline v_int64x2 v_dotprod(const v_int32x4& a, const v_int32x4& b) } inline v_int64x2 v_dotprod(const v_int32x4& a, const v_int32x4& b, const v_int64x2& c) { - int64 CV_DECL_ALIGNED(32) ptr[4] = {0}; + int64 ptr[4] = {0}; v_int64x2 t1, t2; vse64_v_i64m2(ptr, vwmul_vv_i64m2(a, b, 4), 4); v_load_deinterleave(ptr, t1, t2); @@ -2479,7 +2482,7 @@ inline v_int64x2 v_dotprod(const v_int32x4& a, const v_int32x4& b, const v_int64 // 8 >> 32 inline v_uint32x4 v_dotprod_expand(const v_uint8x16& a, const v_uint8x16& b) { - unsigned CV_DECL_ALIGNED(32) ptr[16] = {0}; + unsigned ptr[16] = {0}; v_uint32x4 t1, t2, t3, t4; vse32_v_u32m4(ptr, vwcvtu_x_x_v_u32m4(vwmulu_vv_u16m2(a, b, 16), 16), 16); v_load_deinterleave(ptr, t1, t2, t3, t4); @@ -2488,7 +2491,7 @@ inline v_uint32x4 v_dotprod_expand(const v_uint8x16& a, const v_uint8x16& b) inline v_uint32x4 v_dotprod_expand(const v_uint8x16& a, const v_uint8x16& b, const v_uint32x4& c) { - unsigned CV_DECL_ALIGNED(32) ptr[16] = {0}; + unsigned ptr[16] = {0}; v_uint32x4 t1, t2, t3, t4; vse32_v_u32m4(ptr, vwcvtu_x_x_v_u32m4(vwmulu_vv_u16m2(a, b, 16), 16), 16); v_load_deinterleave(ptr, t1, t2, t3, t4); @@ -2497,7 +2500,7 @@ inline v_uint32x4 v_dotprod_expand(const v_uint8x16& a, const v_uint8x16& b, inline v_int32x4 v_dotprod_expand(const v_int8x16& a, const v_int8x16& b) { - int CV_DECL_ALIGNED(32) ptr[16] = {0}; + int ptr[16] = {0}; v_int32x4 t1, t2, t3, t4; vse32_v_i32m4(ptr, vwcvt_x_x_v_i32m4(vwmul_vv_i16m2(a, b, 16), 16), 16); v_load_deinterleave(ptr, t1, t2, t3, t4); @@ -2506,7 +2509,7 @@ inline v_int32x4 v_dotprod_expand(const v_int8x16& a, const v_int8x16& b) inline v_int32x4 v_dotprod_expand(const v_int8x16& a, const v_int8x16& b, const v_int32x4& c) { - int CV_DECL_ALIGNED(32) ptr[16] = {0}; + int ptr[16] = {0}; v_int32x4 t1, t2, t3, t4; vse32_v_i32m4(ptr, vwcvt_x_x_v_i32m4(vwmul_vv_i16m2(a, b, 16), 16), 16); v_load_deinterleave(ptr, t1, t2, t3, t4); @@ -2516,7 +2519,7 @@ inline v_int32x4 v_dotprod_expand(const v_int8x16& a, const v_int8x16& b, // 16 >> 64 inline v_uint64x2 v_dotprod_expand(const v_uint16x8& a, const v_uint16x8& b) { - uint64 CV_DECL_ALIGNED(32) ptr[8] = {0}; + uint64 ptr[8] = {0}; v_uint64x2 t1, t2, t3, t4; vse64_v_u64m4(ptr, vwcvtu_x_x_v_u64m4(vwmulu_vv_u32m2(a, b, 8), 8), 8); v_load_deinterleave(ptr, t1, t2, t3, t4); @@ -2524,7 +2527,7 @@ inline v_uint64x2 v_dotprod_expand(const v_uint16x8& a, const v_uint16x8& b) } inline v_uint64x2 v_dotprod_expand(const v_uint16x8& a, const v_uint16x8& b, const v_uint64x2& c) { - uint64 CV_DECL_ALIGNED(32) ptr[8] = {0}; + uint64 ptr[8] = {0}; v_uint64x2 t1, t2, t3, t4; vse64_v_u64m4(ptr, vwcvtu_x_x_v_u64m4(vwmulu_vv_u32m2(a, b, 8), 8), 8); v_load_deinterleave(ptr, t1, t2, t3, t4); @@ -2533,7 +2536,7 @@ inline v_uint64x2 v_dotprod_expand(const v_uint16x8& a, const v_uint16x8& b, con inline v_int64x2 v_dotprod_expand(const v_int16x8& a, const v_int16x8& b) { - int64 CV_DECL_ALIGNED(32) ptr[8] = {0}; + int64 ptr[8] = {0}; v_int64x2 t1, t2, t3, t4; vse64_v_i64m4(ptr, vwcvt_x_x_v_i64m4(vwmul_vv_i32m2(a, b, 8), 8), 8); v_load_deinterleave(ptr, t1, t2, t3, t4); @@ -2542,7 +2545,7 @@ inline v_int64x2 v_dotprod_expand(const v_int16x8& a, const v_int16x8& b) inline v_int64x2 v_dotprod_expand(const v_int16x8& a, const v_int16x8& b, const v_int64x2& c) { - int64 CV_DECL_ALIGNED(32) ptr[8] = {0}; + int64 ptr[8] = {0}; v_int64x2 t1, t2, t3, t4; vse64_v_i64m4(ptr, vwcvt_x_x_v_i64m4(vwmul_vv_i32m2(a, b, 8), 8), 8); v_load_deinterleave(ptr, t1, t2, t3, t4); @@ -2563,7 +2566,7 @@ inline v_float64x2 v_dotprod_expand(const v_int32x4& a, const v_int32x4& b, // 16 >> 32 inline v_int32x4 v_dotprod_fast(const v_int16x8& a, const v_int16x8& b) { - int CV_DECL_ALIGNED(32) ptr[8] = {0}; + int ptr[8] = {0}; vse32_v_i32m2(ptr, vwmul_vv_i32m2(a, b, 8), 8); v_int32x4 t1 = v_load(ptr); v_int32x4 t2 = v_load(ptr+4); @@ -2571,7 +2574,7 @@ inline v_int32x4 v_dotprod_fast(const v_int16x8& a, const v_int16x8& b) } inline v_int32x4 v_dotprod_fast(const v_int16x8& a, const v_int16x8& b, const v_int32x4& c) { - int CV_DECL_ALIGNED(32) ptr[8] = {0}; + int ptr[8] = {0}; vse32_v_i32m2(ptr, vwmul_vv_i32m2(a, b, 8), 8); v_int32x4 t1 = v_load(ptr); v_int32x4 t2 = v_load(ptr+4); @@ -2581,7 +2584,7 @@ inline v_int32x4 v_dotprod_fast(const v_int16x8& a, const v_int16x8& b, const v_ // 32 >> 64 inline v_int64x2 v_dotprod_fast(const v_int32x4& a, const v_int32x4& b) { - int64 CV_DECL_ALIGNED(32) ptr[4] = {0}; + int64 ptr[4] = {0}; vse64_v_i64m2(ptr, vwmul_vv_i64m2(a, b, 4), 4); v_int64x2 t1 = v_load(ptr); v_int64x2 t2 = v_load(ptr+2); @@ -2589,7 +2592,7 @@ inline v_int64x2 v_dotprod_fast(const v_int32x4& a, const v_int32x4& b) } inline v_int64x2 v_dotprod_fast(const v_int32x4& a, const v_int32x4& b, const v_int64x2& c) { - int64 CV_DECL_ALIGNED(32) ptr[4] = {0}; + int64 ptr[4] = {0}; vse64_v_i64m2(ptr, vwmul_vv_i64m2(a, b, 4), 4); v_int64x2 t1 = v_load(ptr); v_int64x2 t2 = v_load(ptr+2); @@ -2600,7 +2603,7 @@ inline v_int64x2 v_dotprod_fast(const v_int32x4& a, const v_int32x4& b, const v_ // 8 >> 32 inline v_uint32x4 v_dotprod_expand_fast(const v_uint8x16& a, const v_uint8x16& b) { - unsigned CV_DECL_ALIGNED(32) ptr[16] = {0}; + unsigned ptr[16] = {0}; vse32_v_u32m4(ptr, vwcvtu_x_x_v_u32m4(vwmulu_vv_u16m2(a, b, 16), 16), 16); v_uint32x4 t1 = v_load(ptr); v_uint32x4 t2 = v_load(ptr+4); @@ -2610,7 +2613,7 @@ inline v_uint32x4 v_dotprod_expand_fast(const v_uint8x16& a, const v_uint8x16& b } inline v_uint32x4 v_dotprod_expand_fast(const v_uint8x16& a, const v_uint8x16& b, const v_uint32x4& c) { - unsigned CV_DECL_ALIGNED(32) ptr[16] = {0}; + unsigned ptr[16] = {0}; vse32_v_u32m4(ptr, vwcvtu_x_x_v_u32m4(vwmulu_vv_u16m2(a, b, 16), 16), 16); v_uint32x4 t1 = v_load(ptr); v_uint32x4 t2 = v_load(ptr+4); @@ -2620,7 +2623,7 @@ inline v_uint32x4 v_dotprod_expand_fast(const v_uint8x16& a, const v_uint8x16& b } inline v_int32x4 v_dotprod_expand_fast(const v_int8x16& a, const v_int8x16& b) { - int CV_DECL_ALIGNED(32) ptr[16] = {0}; + int ptr[16] = {0}; vse32_v_i32m4(ptr, vwcvt_x_x_v_i32m4(vwmul_vv_i16m2(a, b, 16), 16), 16); v_int32x4 t1 = v_load(ptr); v_int32x4 t2 = v_load(ptr+4); @@ -2630,7 +2633,7 @@ inline v_int32x4 v_dotprod_expand_fast(const v_int8x16& a, const v_int8x16& b) } inline v_int32x4 v_dotprod_expand_fast(const v_int8x16& a, const v_int8x16& b, const v_int32x4& c) { - int CV_DECL_ALIGNED(32) ptr[16] = {0}; + int ptr[16] = {0}; vse32_v_i32m4(ptr, vwcvt_x_x_v_i32m4(vwmul_vv_i16m2(a, b, 16), 16), 16); v_int32x4 t1 = v_load(ptr); v_int32x4 t2 = v_load(ptr+4); @@ -2642,7 +2645,7 @@ inline v_int32x4 v_dotprod_expand_fast(const v_int8x16& a, const v_int8x16& b, c // 16 >> 64 inline v_uint64x2 v_dotprod_expand_fast(const v_uint16x8& a, const v_uint16x8& b) { - uint64 CV_DECL_ALIGNED(32) ptr[8] = {0}; + uint64 ptr[8] = {0}; vse64_v_u64m4(ptr, vwcvtu_x_x_v_u64m4(vwmulu_vv_u32m2(a, b, 8), 8), 8); v_uint64x2 t1 = v_load(ptr); v_uint64x2 t2 = v_load(ptr+2); @@ -2652,7 +2655,7 @@ inline v_uint64x2 v_dotprod_expand_fast(const v_uint16x8& a, const v_uint16x8& b } inline v_uint64x2 v_dotprod_expand_fast(const v_uint16x8& a, const v_uint16x8& b, const v_uint64x2& c) { - uint64 CV_DECL_ALIGNED(32) ptr[8] = {0}; + uint64 ptr[8] = {0}; vse64_v_u64m4(ptr, vwcvtu_x_x_v_u64m4(vwmulu_vv_u32m2(a, b, 8), 8), 8); v_uint64x2 t1 = v_load(ptr); v_uint64x2 t2 = v_load(ptr+2); @@ -2662,7 +2665,7 @@ inline v_uint64x2 v_dotprod_expand_fast(const v_uint16x8& a, const v_uint16x8& b } inline v_int64x2 v_dotprod_expand_fast(const v_int16x8& a, const v_int16x8& b) { - int64 CV_DECL_ALIGNED(32) ptr[8] = {0}; + int64 ptr[8] = {0}; vse64_v_i64m4(ptr, vwcvt_x_x_v_i64m4(vwmul_vv_i32m2(a, b, 8), 8), 8); v_int64x2 t1 = v_load(ptr); v_int64x2 t2 = v_load(ptr+2); @@ -2672,7 +2675,7 @@ inline v_int64x2 v_dotprod_expand_fast(const v_int16x8& a, const v_int16x8& b) } inline v_int64x2 v_dotprod_expand_fast(const v_int16x8& a, const v_int16x8& b, const v_int64x2& c) { - int64 CV_DECL_ALIGNED(32) ptr[8] = {0}; + int64 ptr[8] = {0}; vse64_v_i64m4(ptr, vwcvt_x_x_v_i64m4(vwmul_vv_i32m2(a, b, 8), 8), 8); v_int64x2 t1 = v_load(ptr); v_int64x2 t2 = v_load(ptr+2); @@ -2714,7 +2717,7 @@ inline v_float32x4 v_matmuladd(const v_float32x4& v, const v_float32x4& m0, #define OPENCV_HAL_IMPL_RVV_MUL_EXPAND(_Tpvec, _Tpwvec, _Tpw, suffix, wmul, width, vl, hvl) \ inline void v_mul_expand(const _Tpvec& a, const _Tpvec& b, _Tpwvec& c, _Tpwvec& d) \ { \ - _Tpw CV_DECL_ALIGNED(32) ptr[_Tpwvec::nlanes*2] = {0}; \ + _Tpw ptr[_Tpwvec::nlanes*2] = {0}; \ vse##width##_v_##suffix##m2(ptr, wmul(a, b, vl), vl); \ c = _Tpwvec(vle##width##_v_##suffix##m1(ptr, hvl)); \ d = _Tpwvec(vle##width##_v_##suffix##m1(ptr+_Tpwvec::nlanes, hvl)); \ From 4e5699fa716a3e5b0faddf1d6a00213aeb5c60cc Mon Sep 17 00:00:00 2001 From: Julia Bareeva <34717687+JulieBar@users.noreply.github.com> Date: Fri, 23 Jul 2021 17:11:50 +0300 Subject: [PATCH 082/262] Merge pull request #20450 from JulieBar:lstm_inside Support non-zero hidden state for LSTM * fully support non-zero hidden state for LSTM * check dims of hidden state for LSTM * fix failed test Test_Model.TextRecognition * add new tests for LSTM w/ non-zero hidden params Co-authored-by: Julie Bareeva --- modules/dnn/src/layers/recurrent_layers.cpp | 28 +++++--- modules/dnn/src/onnx/onnx_importer.cpp | 11 ++- modules/dnn/src/tensorflow/tf_importer.cpp | 16 +++-- modules/dnn/test/test_layers.cpp | 80 ++++++++++++++++++++- modules/dnn/test/test_onnx_importer.cpp | 10 +++ 5 files changed, 122 insertions(+), 23 deletions(-) diff --git a/modules/dnn/src/layers/recurrent_layers.cpp b/modules/dnn/src/layers/recurrent_layers.cpp index 69606a6b4e..a6715aefca 100644 --- a/modules/dnn/src/layers/recurrent_layers.cpp +++ b/modules/dnn/src/layers/recurrent_layers.cpp @@ -112,19 +112,24 @@ public: const Mat& Wh = blobs[0]; const Mat& Wx = blobs[1]; const Mat& bias = blobs[2]; + const Mat& hInternal = blobs[3]; + const Mat& cInternal = blobs[4]; CV_CheckEQ(Wh.dims, 2, ""); CV_CheckEQ(Wx.dims, 2, ""); CV_CheckEQ(Wh.rows, Wx.rows, ""); CV_CheckEQ(Wh.rows, (1 + static_cast(bidirectional))*4*Wh.cols, ""); CV_CheckEQ(Wh.rows, (int)bias.total(), ""); + CV_CheckEQ(hInternal.cols, Wh.cols, ""); + CV_CheckEQ(hInternal.cols, cInternal.cols, ""); + CV_CheckEQ(hInternal.rows, cInternal.rows, ""); CV_Assert(Wh.type() == Wx.type() && Wx.type() == bias.type()); // Peephole weights. - if (blobs.size() > 3) + if (blobs.size() > 5) { - CV_Assert(blobs.size() == 6); + CV_Assert(blobs.size() == 8); const int N = Wh.cols; - for (int i = 3; i < 6; ++i) + for (int i = 5; i < 8; ++i) { CV_Assert(blobs[i].rows == N && blobs[i].cols == N); CV_Assert(blobs[i].type() == bias.type()); @@ -181,7 +186,7 @@ public: std::vector &outputs, std::vector &internals) const CV_OVERRIDE { - CV_Assert((!usePeephole && blobs.size() == 3) || (usePeephole && blobs.size() == 6)); + CV_Assert((!usePeephole && blobs.size() == 5) || (usePeephole && blobs.size() == 8)); CV_Assert(inputs.size() == 1); const MatShape& inp0 = inputs[0]; @@ -228,7 +233,7 @@ public: std::vector input; inputs_arr.getMatVector(input); - CV_Assert((!usePeephole && blobs.size() == 3) || (usePeephole && blobs.size() == 6)); + CV_Assert((!usePeephole && blobs.size() == 5) || (usePeephole && blobs.size() == 8)); CV_Assert(input.size() == 1); const Mat& inp0 = input[0]; @@ -284,13 +289,14 @@ public: const Mat &Wh = blobs[0].rowRange(i * blobs[0].rows / numDirs, (i + 1) * blobs[0].rows / numDirs); const Mat &Wx = blobs[1].rowRange(i * blobs[1].rows / numDirs, (i + 1) * blobs[1].rows / numDirs); const Mat &bias = blobs[2].colRange(i * blobs[2].cols / numDirs, (i + 1) * blobs[2].cols / numDirs); + const Mat &h_0 = blobs[3].rowRange(i * blobs[3].rows / numDirs, (i + 1) * blobs[3].rows / numDirs); + const Mat &c_0 = blobs[4].rowRange(i * blobs[4].rows / numDirs, (i + 1) * blobs[4].rows / numDirs); int numOut = Wh.size[1]; - Mat hInternal = internals[0], cInternal = internals[1], dummyOnes = internals[2], gates = internals[3]; - hInternal.setTo(0.); - cInternal.setTo(0.); + h_0.copyTo(hInternal); + c_0.copyTo(cInternal); dummyOnes.setTo(1.); int numSamplesTotal = numTimeStamps*numSamples; @@ -331,8 +337,8 @@ public: if (usePeephole) { Mat gatesIF = gates.colRange(0, 2*numOut); - gemm(cInternal, blobs[3], 1, gateI, 1, gateI); - gemm(cInternal, blobs[4], 1, gateF, 1, gateF); + gemm(cInternal, blobs[5], 1, gateI, 1, gateI); + gemm(cInternal, blobs[6], 1, gateF, 1, gateF); sigmoid(gatesIF, gatesIF); } else @@ -355,7 +361,7 @@ public: } if (usePeephole) { - gemm(cInternal, blobs[5], 1, gateO, 1, gateO); + gemm(cInternal, blobs[7], 1, gateO, 1, gateO); sigmoid(gateO, gateO); } diff --git a/modules/dnn/src/onnx/onnx_importer.cpp b/modules/dnn/src/onnx/onnx_importer.cpp index ec61a9707e..4ad0fd496e 100644 --- a/modules/dnn/src/onnx/onnx_importer.cpp +++ b/modules/dnn/src/onnx/onnx_importer.cpp @@ -900,8 +900,9 @@ void ONNXImporter::handleNode(const opencv_onnx::NodeProto& node_proto_) Mat Wx = getBlob(node_proto, 1); Mat Wh = getBlob(node_proto, 2); Mat b = getBlob(node_proto, 3); - CV_CheckEQ(countNonZero(getBlob(node_proto, 5)), 0, "Unsupported non zero initial_h"); - CV_CheckEQ(countNonZero(getBlob(node_proto, 6)), 0, "Unsupported non zero initial_c"); + Mat h0 = getBlob(node_proto, 5); + Mat c0 = getBlob(node_proto, 6); + b = b.reshape(1, b.size[0]); const int numHidden = lstmParams.get("hidden_size"); @@ -934,11 +935,15 @@ void ONNXImporter::handleNode(const opencv_onnx::NodeProto& node_proto_) } Wx = Wx.reshape(1, Wx.size[0] * Wx.size[1]); Wh = Wh.reshape(1, Wh.size[0] * Wh.size[1]); + h0 = h0.reshape(1, h0.size[0] * h0.size[1]); + c0 = c0.reshape(1, c0.size[0] * c0.size[1]); - lstmParams.blobs.resize(3); + lstmParams.blobs.resize(5); lstmParams.blobs[0] = Wh; lstmParams.blobs[1] = Wx; lstmParams.blobs[2] = b; + lstmParams.blobs[3] = h0; + lstmParams.blobs[4] = c0; lstmParams.set("bidirectional", lstmParams.get("direction", "") == "bidirectional"); node_proto.set_output(0, lstmParams.name); // set different name so output shapes will be registered on that name diff --git a/modules/dnn/src/tensorflow/tf_importer.cpp b/modules/dnn/src/tensorflow/tf_importer.cpp index 426710989e..01fa0df985 100644 --- a/modules/dnn/src/tensorflow/tf_importer.cpp +++ b/modules/dnn/src/tensorflow/tf_importer.cpp @@ -1838,8 +1838,8 @@ void TFImporter::parseBlockLSTM(tensorflow::GraphDef& net, const tensorflow::Nod // op: "BlockLSTM" // input: "lstm_block_wrapper/ToInt64/x" (ignore, number of time stamps) // input: "input" - // input: "lstm_block_wrapper/zeros" (ignore) - // input: "lstm_block_wrapper/zeros" (ignore) + // input: "lstm_block_wrapper/zeros" + // input: "lstm_block_wrapper/zeros" // input: "lstm_block_wrapper/kernel" // input: "lstm_block_wrapper/w_i_diag" // input: "lstm_block_wrapper/w_f_diag" @@ -1865,9 +1865,11 @@ void TFImporter::parseBlockLSTM(tensorflow::GraphDef& net, const tensorflow::Nod } } - Mat W, Wh, Wx, b; + Mat W, Wh, Wx, b, cs_prev, h_prev; blobFromTensor(getConstBlob(layer, value_id, 4), W); blobFromTensor(getConstBlob(layer, value_id, 8), b); + blobFromTensor(getConstBlob(layer, value_id, 2), cs_prev); + blobFromTensor(getConstBlob(layer, value_id, 3), h_prev); const int outSize = W.cols / 4; // IGFO->IFOG @@ -1883,10 +1885,12 @@ void TFImporter::parseBlockLSTM(tensorflow::GraphDef& net, const tensorflow::Nod Wx = W.rowRange(0, W.rows - outSize).t(); Wh = W.rowRange(W.rows - outSize, W.rows).t(); - layerParams.blobs.resize(3); + layerParams.blobs.resize(5); layerParams.blobs[0] = Wh; layerParams.blobs[1] = Wx; layerParams.blobs[2] = b; + layerParams.blobs[3] = h_prev; + layerParams.blobs[4] = cs_prev; if (hasLayerAttr(layer, "use_peephole")) { @@ -1894,14 +1898,14 @@ void TFImporter::parseBlockLSTM(tensorflow::GraphDef& net, const tensorflow::Nod if (usePeephole) { layerParams.set("use_peephole", true); - layerParams.blobs.resize(6); + layerParams.blobs.resize(8); for (int i = 0; i < 3; ++i) { Mat w; blobFromTensor(getConstBlob(layer, value_id, 5 + i), w); w = w.reshape(1, w.total()); // Single column. w = Mat::diag(w); // Make a diagonal matrix. - layerParams.blobs[3 + i] = w; + layerParams.blobs[5 + i] = w; } } } diff --git a/modules/dnn/test/test_layers.cpp b/modules/dnn/test/test_layers.cpp index 897603d274..fbe9605e7f 100644 --- a/modules/dnn/test/test_layers.cpp +++ b/modules/dnn/test/test_layers.cpp @@ -434,7 +434,7 @@ class Layer_LSTM_Test : public ::testing::Test { public: int numInp, numOut; - Mat Wh, Wx, b; + Mat Wh, Wx, b, h, c; Ptr layer; std::vector inputs, outputs; @@ -449,12 +449,17 @@ public: Wh = Mat::ones(4 * numOut, numOut, CV_32F); Wx = Mat::ones(4 * numOut, numInp, CV_32F); b = Mat::ones(4 * numOut, 1, CV_32F); + h = Mat::ones(4, numOut, CV_32F); + c = Mat::ones(4, numOut, CV_32F); LayerParams lp; - lp.blobs.resize(3); + lp.blobs.resize(5); lp.blobs[0] = Wh; lp.blobs[1] = Wx; lp.blobs[2] = b; + lp.blobs[3] = h; + lp.blobs[4] = c; + lp.set("produce_cell_output", produceCellOutput); lp.set("use_timestamp_dim", useTimestampDim); @@ -502,10 +507,12 @@ TEST_F(Layer_LSTM_Test, get_set_test) TEST(Layer_LSTM_Test_Accuracy_with_, CaffeRecurrent) { LayerParams lp; - lp.blobs.resize(3); + lp.blobs.resize(5); lp.blobs[0] = blobFromNPY(_tf("lstm.prototxt.w_2.npy")); // Wh lp.blobs[1] = blobFromNPY(_tf("lstm.prototxt.w_0.npy")); // Wx lp.blobs[2] = blobFromNPY(_tf("lstm.prototxt.w_1.npy")); // bias + lp.blobs[3] = Mat::zeros(2, 17, CV_32F); // h_0 + lp.blobs[4] = Mat::zeros(2, 17, CV_32F); // c_0 Ptr layer = LSTMLayer::create(lp); Mat inp = blobFromNPY(_tf("recurrent.input.npy")); @@ -516,6 +523,68 @@ TEST(Layer_LSTM_Test_Accuracy_with_, CaffeRecurrent) normAssert(h_t_reference, outputs[0]); } +TEST(Layer_LSTM_Test_Accuracy_with_, HiddenParams) +{ + Mat Wx = blobFromNPY(_tf("lstm.hidden.W.npy")); + Mat Wh = blobFromNPY(_tf("lstm.hidden.R.npy")); + Mat b = blobFromNPY(_tf("lstm.hidden.B.npy")); + Mat h0 = blobFromNPY(_tf("lstm.hidden.h0.npy")); + Mat c0 = blobFromNPY(_tf("lstm.hidden.c0.npy")); + + const int numHidden = 3; + const int numDirs = Wx.size[0]; + const int numFeatures = Wx.size[2]; + + b = b.reshape(1, b.size[0]); + Mat bx = b.colRange(0, b.cols / 2); + Mat bh = b.colRange(b.cols / 2, b.cols); + b = bx + bh; + + // IFGO->IGFO + for (int k = 0; k < numDirs; ++k) + { + float* WxData = Wx.ptr(k); + float* WhData = Wh.ptr(k); + float* biasData = b.ptr(k); + for (int j = 0; j < numHidden; ++j) + { + for (int i = 0; i < numFeatures; ++i) + { + std::swap(WxData[(numHidden + j) * numFeatures + i], + WxData[(numHidden * 2 + j) * numFeatures + i]); + } + for (int i = 0; i < numHidden; ++i) + { + std::swap(WhData[(numHidden + j) * numHidden + i], + WhData[(numHidden * 2 + j) * numHidden + i]); + } + std::swap(biasData[numHidden + j], biasData[numHidden * 2 + j]); + } + } + + Wx = Wx.reshape(1, Wx.size[0] * Wx.size[1]); + Wh = Wh.reshape(1, Wh.size[0] * Wh.size[1]); + h0 = h0.reshape(1, h0.size[0] * h0.size[1]); + c0 = c0.reshape(1, c0.size[0] * c0.size[1]); + + LayerParams lstmParams; + lstmParams.blobs.resize(5); + lstmParams.blobs[0] = Wh; + lstmParams.blobs[1] = Wx; + lstmParams.blobs[2] = b; + lstmParams.blobs[3] = h0; + lstmParams.blobs[4] = c0; + lstmParams.set("bidirectional", false); + Ptr layer = LSTMLayer::create(lstmParams); + + Mat inp = blobFromNPY(_tf("lstm.hidden.input.npy")); + std::vector inputs(1, inp), outputs; + runLayer(layer, inputs, outputs); + + Mat h_t_reference = blobFromNPY(_tf("lstm.hidden.output.npy")); + normAssert(h_t_reference, outputs[0]); +} + TEST(Layer_RNN_Test_Accuracy_with_, CaffeRecurrent) { Ptr layer = RNNLayer::create(LayerParams()); @@ -560,6 +629,9 @@ TEST(Layer_LSTM_Test_Accuracy_, Reverse) bias.at(2, 0) = 1e10f; // Output gate - always output everything bias.at(3, 0) = 0.f; // Update signal + cv::Mat hInternal = cv::Mat::zeros(1, 1, CV_32FC1); + cv::Mat cInternal = cv::Mat::zeros(1, 1, CV_32FC1); + LayerParams lp; lp.set("reverse", true); lp.set("use_timestamp_dim", true); @@ -567,6 +639,8 @@ TEST(Layer_LSTM_Test_Accuracy_, Reverse) lp.blobs.push_back(Wh); lp.blobs.push_back(Wx); lp.blobs.push_back(bias); + lp.blobs.push_back(hInternal); + lp.blobs.push_back(cInternal); cv::Ptr layer = LSTMLayer::create(lp); std::vector outputs; diff --git a/modules/dnn/test/test_onnx_importer.cpp b/modules/dnn/test/test_onnx_importer.cpp index 3923068dbf..05f77730af 100644 --- a/modules/dnn/test/test_onnx_importer.cpp +++ b/modules/dnn/test/test_onnx_importer.cpp @@ -675,6 +675,16 @@ TEST_P(Test_ONNX_layers, LSTM_bidirectional) testONNXModels("lstm_bidirectional", npy, 0, 0, false, false); } +TEST_P(Test_ONNX_layers, LSTM_hidden) +{ + testONNXModels("hidden_lstm", npy, 0, 0, false, false); +} + +TEST_P(Test_ONNX_layers, LSTM_hidden_bidirectional) +{ + testONNXModels("hidden_lstm_bi", npy, 0, 0, false, false); +} + TEST_P(Test_ONNX_layers, Pad2d_Unfused) { testONNXModels("ReflectionPad2d"); From 7aa922ceac7d61c0a2b6c8c76debb70e256f6e2f Mon Sep 17 00:00:00 2001 From: Parsa Date: Sat, 24 Jul 2021 19:59:24 +0430 Subject: [PATCH 083/262] Merge pull request #20440 from parsa-ra:patch-1 * Update config_reference.markdown Added description for `WITH_CLP` build option. * Added extra description Can't cross-reference with anchors to other sections of the markdown file due to the presence of markdown link extension in the form of `## Header {#id-of-header}` * Fixed trailing space issue --- .../config_reference/config_reference.markdown | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/doc/tutorials/introduction/config_reference/config_reference.markdown b/doc/tutorials/introduction/config_reference/config_reference.markdown index 58b4ed55ca..0ba5627249 100644 --- a/doc/tutorials/introduction/config_reference/config_reference.markdown +++ b/doc/tutorials/introduction/config_reference/config_reference.markdown @@ -589,6 +589,14 @@ Some features have been added specifically for automated build environments, lik | `OPENCV_CMAKE_HOOKS_DIR` | _empty_ | OpenCV allows to customize configuration process by adding custom hook scripts at each stage and substage. cmake scripts with predefined names located in the directory set by this variable will be included before and after various configuration stages. Examples of file names: _CMAKE_INIT.cmake_, _PRE_CMAKE_BOOTSTRAP.cmake_, _POST_CMAKE_BOOTSTRAP.cmake_, etc.. Other names are not documented and can be found in the project cmake files by searching for the _ocv_cmake_hook_ macro calls. | | `OPENCV_DUMP_HOOKS_FLOW` | _OFF_ | Enables a debug message print on each cmake hook script call. | +## Contrib Modules + +Following build options are utilized in `opencv_contrib` modules, as stated [previously](#tutorial_config_reference_general_contrib), these extra modules can be added to your final build by setting `DOPENCV_EXTRA_MODULES_PATH` option. + +| Option | Default | Description | +| ------ | ------- | ----------- | +| `WITH_CLP` | _OFF_ | Will add [coinor](https://projects.coin-or.org/Clp) linear programming library build support which is required in `videostab` module. Make sure to install the development libraries of coinor-clp. | + # Other non-documented options @@ -605,7 +613,6 @@ Some features have been added specifically for automated build environments, lik `WITH_CPUFEATURES` `WITH_EIGEN` `WITH_OPENVX` -`WITH_CLP` `WITH_DIRECTX` `WITH_VA` `WITH_LAPACK` From 2f180cea7f6e2519aed9665c37dafdf4ab0e80ba Mon Sep 17 00:00:00 2001 From: Giles Payne Date: Sun, 25 Jul 2021 14:23:52 +0900 Subject: [PATCH 084/262] Add Quicklook for Mat on iOS and macOS --- .../imgcodecs/misc/objc/ios/Mat+Converters.h | 2 +- .../imgcodecs/misc/objc/ios/Mat+Converters.mm | 2 +- .../imgcodecs/misc/objc/ios/Mat+QuickLook.h | 27 +++ .../imgcodecs/misc/objc/ios/Mat+QuickLook.mm | 155 ++++++++++++++++++ .../misc/objc/macosx/Mat+QuickLook.h | 27 +++ .../misc/objc/macosx/Mat+QuickLook.mm | 154 +++++++++++++++++ 6 files changed, 365 insertions(+), 2 deletions(-) create mode 100644 modules/imgcodecs/misc/objc/ios/Mat+QuickLook.h create mode 100644 modules/imgcodecs/misc/objc/ios/Mat+QuickLook.mm create mode 100644 modules/imgcodecs/misc/objc/macosx/Mat+QuickLook.h create mode 100644 modules/imgcodecs/misc/objc/macosx/Mat+QuickLook.mm diff --git a/modules/imgcodecs/misc/objc/ios/Mat+Converters.h b/modules/imgcodecs/misc/objc/ios/Mat+Converters.h index a3ee005c18..0f74bb2f5d 100644 --- a/modules/imgcodecs/misc/objc/ios/Mat+Converters.h +++ b/modules/imgcodecs/misc/objc/ios/Mat+Converters.h @@ -1,5 +1,5 @@ // -// Mat+UIImage.h +// Mat+Converters.h // // Created by Giles Payne on 2020/03/03. // diff --git a/modules/imgcodecs/misc/objc/ios/Mat+Converters.mm b/modules/imgcodecs/misc/objc/ios/Mat+Converters.mm index 69250eb994..79358cb6de 100644 --- a/modules/imgcodecs/misc/objc/ios/Mat+Converters.mm +++ b/modules/imgcodecs/misc/objc/ios/Mat+Converters.mm @@ -1,5 +1,5 @@ // -// Mat+UIImage.mm +// Mat+Converters.mm // // Created by Giles Payne on 2020/03/03. // diff --git a/modules/imgcodecs/misc/objc/ios/Mat+QuickLook.h b/modules/imgcodecs/misc/objc/ios/Mat+QuickLook.h new file mode 100644 index 0000000000..341172798e --- /dev/null +++ b/modules/imgcodecs/misc/objc/ios/Mat+QuickLook.h @@ -0,0 +1,27 @@ +// +// Mat+QuickLook.h +// +// Created by Giles Payne on 2021/07/18. +// + +#pragma once + +#ifdef __cplusplus +#import "opencv2/core.hpp" +#else +#define CV_EXPORTS +#endif + +#import "Mat.h" +#import +#import + +NS_ASSUME_NONNULL_BEGIN + +CV_EXPORTS @interface Mat (QuickLook) + +- (id)debugQuickLookObject; + +@end + +NS_ASSUME_NONNULL_END diff --git a/modules/imgcodecs/misc/objc/ios/Mat+QuickLook.mm b/modules/imgcodecs/misc/objc/ios/Mat+QuickLook.mm new file mode 100644 index 0000000000..7bfee07eb1 --- /dev/null +++ b/modules/imgcodecs/misc/objc/ios/Mat+QuickLook.mm @@ -0,0 +1,155 @@ +// +// Mat+QuickLook.mm +// +// Created by Giles Payne on 2021/07/18. +// + +#import "Mat+QuickLook.h" +#import "Mat+Converters.h" +#import "Rect2i.h" +#import "Core.h" +#import "Imgproc.h" +#import + +#define SIZE 20 + +static UIFont* getCMU() { + return [UIFont fontWithName:@"CMU Serif" size:SIZE]; +} + +static UIFont* getBodoni72() { + return [UIFont fontWithName:@"Bodoni 72" size:SIZE]; +} + +static UIFont* getAnySerif() { + if (@available(iOS 13.0, *)) { + return [UIFont fontWithDescriptor:[[UIFontDescriptor preferredFontDescriptorWithTextStyle:UIFontTextStyleBody] fontDescriptorWithDesign:UIFontDescriptorSystemDesignSerif] size:SIZE]; + } else { + return nil; + } +} + +static UIFont* getSystemFont() { + return [UIFont systemFontOfSize:SIZE]; +} + +typedef UIFont* (*FontGetter)(); + +@implementation Mat (QuickLook) + +- (NSString*)makeLabel:(BOOL)isIntType val:(NSNumber*)num { + if (isIntType) { + return [NSString stringWithFormat:@"%d", num.intValue]; + } else { + int exponent = 1 + (int)log10(abs(num.doubleValue)); + if (num.doubleValue == (double)num.intValue && num.doubleValue < 10000 && num.doubleValue > -10000) { + return [NSString stringWithFormat:@"%d", num.intValue];; + } else if (exponent <= 5 && exponent >= -1) { + return [NSString stringWithFormat:[NSString stringWithFormat:@"%%%d.%df", 6, MIN(5 - exponent, 4)], num.doubleValue]; + } else { + return [[[NSString stringWithFormat:@"%.2e", num.doubleValue] stringByReplacingOccurrencesOfString:@"e+0" withString:@"e"] stringByReplacingOccurrencesOfString:@"e-0" withString:@"e-"]; + } + } +} + +- (void)relativeLine:(UIBezierPath*)path relX:(CGFloat)x relY:(CGFloat)y { + CGPoint curr = path.currentPoint; + [path addLineToPoint:CGPointMake(curr.x + x, curr.y + y)]; +} + +- (id)debugQuickLookObject { + if ([self dims] == 2 && [self rows] <= 10 && [self cols] <= 10) { + FontGetter fontGetters[] = { getCMU, getBodoni72, getAnySerif, getSystemFont }; + UIFont* font = nil; + for (int fontGetterIndex = 0; font==nil && fontGetterIndex < (sizeof(fontGetters)) / (sizeof(fontGetters[0])); fontGetterIndex++) { + font = fontGetters[fontGetterIndex](); + } + int elements = [self rows] * [self cols]; + NSDictionary* textFontAttributes = @{ NSFontAttributeName: font, NSForegroundColorAttributeName: UIColor.blackColor }; + NSMutableArray* rawData = [NSMutableArray new]; + for (int dataIndex = 0; dataIndex < elements; dataIndex++) { + [rawData addObject:[NSNumber numberWithDouble:0]]; + } + [self get:0 col: 0 data: rawData]; + BOOL isIntType = [self depth] <= CV_32S; + NSMutableArray* labels = [NSMutableArray new]; + NSMutableDictionary* boundingRects = [NSMutableDictionary dictionaryWithCapacity:elements]; + int maxWidth = 0, maxHeight = 0; + for (NSNumber* number in rawData) { + NSString* label = [self makeLabel:isIntType val:number]; + [labels addObject:label]; + CGRect boundingRect = [label boundingRectWithSize:CGSizeMake(CGFLOAT_MAX, CGFLOAT_MAX) options:NSStringDrawingUsesLineFragmentOrigin attributes:textFontAttributes context:nil]; + if (boundingRect.size.width > maxWidth) { + maxWidth = boundingRect.size.width; + } + if (boundingRect.size.height > maxHeight) { + maxHeight = boundingRect.size.height; + } + boundingRects[label] = [NSValue valueWithCGRect:boundingRect]; + } + + int rowGap = 6; + int colGap = 6; + int borderGap = 8; + int lineThickness = 3; + int lipWidth = 6; + int imageWidth = 2 * (borderGap + lipWidth) + maxWidth * [self cols] + colGap * ([self cols] - 1); + int imageHeight = 2 * (borderGap + lipWidth) + maxHeight * [self rows] + rowGap * ([self rows] - 1); + + UIBezierPath* leftBracket = [UIBezierPath new]; + [leftBracket moveToPoint:CGPointMake(borderGap, borderGap)]; + [self relativeLine:leftBracket relX:0 relY:imageHeight - 2 * borderGap]; + [self relativeLine:leftBracket relX:lineThickness + lipWidth relY:0]; + [self relativeLine:leftBracket relX:0 relY:-lineThickness]; + [self relativeLine:leftBracket relX:-lipWidth relY:0]; + [self relativeLine:leftBracket relX:0 relY:-(imageHeight - 2 * (borderGap + lineThickness))]; + [self relativeLine:leftBracket relX:lipWidth relY:0]; + [self relativeLine:leftBracket relX:0 relY:-lineThickness]; + [leftBracket closePath]; + CGAffineTransform reflect = CGAffineTransformConcat(CGAffineTransformMakeTranslation(-imageWidth, 0), CGAffineTransformMakeScale(-1, 1)); + UIBezierPath* rightBracket = [leftBracket copy]; + [rightBracket applyTransform:reflect]; + + CGRect rect = CGRectMake(0, 0, imageWidth, imageHeight); + UIGraphicsBeginImageContextWithOptions(rect.size, false, 0.0); + [UIColor.whiteColor setFill]; + UIRectFill(rect); + [UIColor.blackColor setFill]; + [leftBracket fill]; + [rightBracket fill]; + [labels enumerateObjectsUsingBlock:^(id label, NSUInteger index, BOOL *stop) + { + CGRect boundingRect = boundingRects[label].CGRectValue; + int row = (int)index / [self cols]; + int col = (int)index % [self cols]; + int x = borderGap + lipWidth + col * (maxWidth + colGap) + (maxWidth - boundingRect.size.width) / 2; + int y = borderGap + lipWidth + row * (maxHeight + rowGap) + (maxHeight - boundingRect.size.height) / 2; + CGRect textRect = CGRectMake(x, y, boundingRect.size.width, boundingRect.size.height); + [label drawInRect:textRect withAttributes:textFontAttributes]; + }]; + UIImage* image = UIGraphicsGetImageFromCurrentImageContext(); + UIGraphicsEndImageContext(); + return image; + } else if (([self dims] == 2) && ([self type] == CV_8U || [self type] == CV_8UC3 || [self type] == CV_8UC4)) { + return [self toUIImage]; + } else if ([self dims] == 2 && [self channels] == 1) { + Mat* normalized = [Mat new]; + [Core normalize:self dst:normalized alpha:0 beta:255 norm_type:NORM_MINMAX dtype:CV_8U]; + Mat* normalizedKey = [[Mat alloc] initWithRows:[self rows] + 10 cols:[self cols] type:CV_8U]; + std::vector key; + for (int index = 0; index < [self cols]; index++) { + key.push_back((char)(index * 256 / [self cols])); + } + for (int index = 0; index < 10; index++) { + [normalizedKey put:@[[NSNumber numberWithInt:index], [NSNumber numberWithInt:0]] count:[self cols] byteBuffer:key.data()]; + } + [normalized copyTo:[normalizedKey submatRoi:[[Rect2i alloc] initWithX:0 y:10 width:[self cols] height:[self rows]]]]; + Mat* colorMap = [Mat new]; + [Imgproc applyColorMap:normalizedKey dst:colorMap colormap:COLORMAP_JET]; + [Imgproc cvtColor:colorMap dst:colorMap code:COLOR_BGR2RGB]; + return [colorMap toUIImage]; + } + return [self description]; +} + +@end diff --git a/modules/imgcodecs/misc/objc/macosx/Mat+QuickLook.h b/modules/imgcodecs/misc/objc/macosx/Mat+QuickLook.h new file mode 100644 index 0000000000..9fa31aba39 --- /dev/null +++ b/modules/imgcodecs/misc/objc/macosx/Mat+QuickLook.h @@ -0,0 +1,27 @@ +// +// Mat+QuickLook.h +// +// Created by Giles Payne on 2021/07/18. +// + +#pragma once + +#ifdef __cplusplus +#import "opencv2/core.hpp" +#else +#define CV_EXPORTS +#endif + +#import "Mat.h" +#import +#import + +NS_ASSUME_NONNULL_BEGIN + +CV_EXPORTS @interface Mat (QuickLook) + +- (id)debugQuickLookObject; + +@end + +NS_ASSUME_NONNULL_END diff --git a/modules/imgcodecs/misc/objc/macosx/Mat+QuickLook.mm b/modules/imgcodecs/misc/objc/macosx/Mat+QuickLook.mm new file mode 100644 index 0000000000..6775f81780 --- /dev/null +++ b/modules/imgcodecs/misc/objc/macosx/Mat+QuickLook.mm @@ -0,0 +1,154 @@ +// +// Mat+QuickLook.mm +// +// Created by Giles Payne on 2021/07/18. +// + +#import "Mat+QuickLook.h" +#import "Mat+Converters.h" +#import "Rect2i.h" +#import "Core.h" +#import "Imgproc.h" +#import + +#define SIZE 20 + +static NSFont* getCMU() { + return [NSFont fontWithName:@"CMU Serif" size:SIZE]; +} + +static NSFont* getBodoni72() { + return [NSFont fontWithName:@"Bodoni 72" size:SIZE]; +} + +static NSFont* getAnySerif() { + if (@available(macOS 11.0, *)) { + return [NSFont fontWithDescriptor:[[NSFontDescriptor preferredFontDescriptorForTextStyle:NSFontTextStyleBody options:@{}] fontDescriptorWithDesign:NSFontDescriptorSystemDesignSerif] size:SIZE]; + } else { + return nil; + } +} + +static NSFont* getSystemFont() { + return [NSFont systemFontOfSize:SIZE]; +} + +typedef NSFont* (*FontGetter)(); + +@implementation Mat (QuickLook) + +- (NSString*)makeLabel:(BOOL)isIntType val:(NSNumber*)num { + if (isIntType) { + return [NSString stringWithFormat:@"%d", num.intValue]; + } else { + int exponent = 1 + (int)log10(abs(num.doubleValue)); + if (num.doubleValue == (double)num.intValue && num.doubleValue < 10000 && num.doubleValue > -10000) { + return [NSString stringWithFormat:@"%d", num.intValue];; + } else if (exponent <= 5 && exponent >= -1) { + return [NSString stringWithFormat:[NSString stringWithFormat:@"%%%d.%df", 6, MIN(5 - exponent, 4)], num.doubleValue]; + } else { + return [[[NSString stringWithFormat:@"%.2e", num.doubleValue] stringByReplacingOccurrencesOfString:@"e+0" withString:@"e"] stringByReplacingOccurrencesOfString:@"e-0" withString:@"e-"]; + } + } +} + +- (id)debugQuickLookObject { + // for smallish Mat objects display as a matrix + if ([self dims] == 2 && [self rows] <= 10 && [self cols] <= 10) { + FontGetter fontGetters[] = { getCMU, getBodoni72, getAnySerif, getSystemFont }; + NSFont* font = nil; + for (int fontGetterIndex = 0; font==nil && fontGetterIndex < (sizeof(fontGetters)) / (sizeof(fontGetters[0])); fontGetterIndex++) { + font = fontGetters[fontGetterIndex](); + } + int elements = [self rows] * [self cols]; + NSDictionary* textFontAttributes = @{ NSFontAttributeName: font, NSForegroundColorAttributeName: NSColor.blackColor }; + NSMutableArray* rawData = [NSMutableArray new]; + for (int dataIndex = 0; dataIndex < elements; dataIndex++) { + [rawData addObject:[NSNumber numberWithDouble:0]]; + } + [self get:0 col: 0 data: rawData]; + BOOL isIntType = [self depth] <= CV_32S; + NSMutableArray* labels = [NSMutableArray new]; + NSMutableDictionary* boundingRects = [NSMutableDictionary dictionaryWithCapacity:elements]; + int maxWidth = 0, maxHeight = 0; + for (NSNumber* number in rawData) { + NSString* label = [self makeLabel:isIntType val:number]; + [labels addObject:label]; + NSRect boundingRect = [label boundingRectWithSize:NSMakeSize(CGFLOAT_MAX, CGFLOAT_MAX) options:NSStringDrawingUsesLineFragmentOrigin attributes:textFontAttributes]; + if (boundingRect.size.width > maxWidth) { + maxWidth = boundingRect.size.width; + } + if (boundingRect.size.height > maxHeight) { + maxHeight = boundingRect.size.height; + } + boundingRects[label] = [NSValue valueWithRect:boundingRect]; + } + + int rowGap = 8; + int colGap = 8; + int borderGap = 9; + int lineThickness = 4; + int lipWidth = 8; + int imageWidth = 2 * (borderGap + lipWidth) + maxWidth * [self cols] + colGap * ([self cols] - 1); + int imageHeight = 2 * (borderGap + lipWidth) + maxHeight * [self rows] + rowGap * ([self rows] - 1); + NSImage* image = [[NSImage alloc] initWithSize:NSMakeSize(imageWidth, imageHeight)]; + NSBezierPath* leftBracket = [NSBezierPath new]; + [leftBracket moveToPoint:NSMakePoint(borderGap, borderGap)]; + [leftBracket relativeLineToPoint:NSMakePoint(0, imageHeight - 2 * borderGap)]; + [leftBracket relativeLineToPoint:NSMakePoint(lineThickness + lipWidth, 0)]; + [leftBracket relativeLineToPoint:NSMakePoint(0, -lineThickness)]; + [leftBracket relativeLineToPoint:NSMakePoint(-lipWidth, 0)]; + [leftBracket relativeLineToPoint:NSMakePoint(0, -(imageHeight - 2 * (borderGap + lineThickness)))]; + [leftBracket relativeLineToPoint:NSMakePoint(lipWidth, 0)]; + [leftBracket relativeLineToPoint:NSMakePoint(0, -lineThickness)]; + [leftBracket relativeLineToPoint:NSMakePoint(-(lineThickness + lipWidth), 0)]; + NSAffineTransform* reflect = [NSAffineTransform new]; + [reflect scaleXBy:-1 yBy:1]; + [reflect translateXBy:-imageWidth yBy:0]; + NSBezierPath* rightBracket = [leftBracket copy]; + [rightBracket transformUsingAffineTransform:reflect]; + + [image lockFocus]; + [NSColor.whiteColor drawSwatchInRect:NSMakeRect(0, 0, imageWidth, imageHeight)]; + [NSColor.blackColor set]; + [leftBracket fill]; + [rightBracket fill]; + + [labels enumerateObjectsUsingBlock:^(id label, NSUInteger index, BOOL *stop) + { + NSRect boundingRect = boundingRects[label].rectValue; + int row = [self rows] - 1 - ((int)index / [self cols]); + int col = (int)index % [self cols]; + int x = borderGap + lipWidth + col * (maxWidth + colGap) + (maxWidth - boundingRect.size.width) / 2; + int y = borderGap + lipWidth + row * (maxHeight + rowGap) + (maxHeight - boundingRect.size.height) / 2; + NSRect textRect = NSMakeRect(x, y, boundingRect.size.width, boundingRect.size.height); + [label drawInRect:textRect withAttributes:textFontAttributes]; + }]; + [image unlockFocus]; + return image; + } else if (([self dims] == 2) && ([self type] == CV_8U || [self type] == CV_8UC3 || [self type] == CV_8UC4)) { + // convert to NSImage if the Mats has 2 dimensions and a type and number of channels consistent with it being a image + return [self toNSImage]; + } else if ([self dims] == 2 && [self channels] == 1) { + // for other Mats with 2 dimensions and one channel - generate heat map + Mat* normalized = [Mat new]; + [Core normalize:self dst:normalized alpha:0 beta:255 norm_type:NORM_MINMAX dtype:CV_8U]; + Mat* normalizedKey = [[Mat alloc] initWithRows:[self rows] + 10 cols:[self cols] type:CV_8U]; + std::vector key; + for (int index = 0; index < [self cols]; index++) { + key.push_back((char)(index * 256 / [self cols])); + } + for (int index = 0; index < 10; index++) { + [normalizedKey put:@[[NSNumber numberWithInt:index], [NSNumber numberWithInt:0]] count:[self cols] byteBuffer:key.data()]; + } + [normalized copyTo:[normalizedKey submatRoi:[[Rect2i alloc] initWithX:0 y:10 width:[self cols] height:[self rows]]]]; + Mat* colorMap = [Mat new]; + [Imgproc applyColorMap:normalizedKey dst:colorMap colormap:COLORMAP_JET]; + [Imgproc cvtColor:colorMap dst:colorMap code:COLOR_BGR2RGB]; + return [colorMap toNSImage]; + } + //everything just return the Mat description + return [self description]; +} + +@end From 1e1984a586028ab233d26f0a4a5668653fc410ba Mon Sep 17 00:00:00 2001 From: Xerxes Battiwalla Date: Mon, 26 Jul 2021 14:54:27 +1000 Subject: [PATCH 085/262] Fixed typo in error message in OpenCVDetectCUDA.cmake There was a minor typo in the FATAL error message when the specified CUDA generation does not match any known generation --- cmake/OpenCVDetectCUDA.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/OpenCVDetectCUDA.cmake b/cmake/OpenCVDetectCUDA.cmake index c7cfebe50f..ac29e600d3 100644 --- a/cmake/OpenCVDetectCUDA.cmake +++ b/cmake/OpenCVDetectCUDA.cmake @@ -102,7 +102,7 @@ if(CUDA_FOUND) if(CUDA_GENERATION) if(NOT ";${_generations};" MATCHES ";${CUDA_GENERATION};") string(REPLACE ";" ", " _generations "${_generations}") - message(FATAL_ERROR "ERROR: ${_generations} Generations are suppered.") + message(FATAL_ERROR "ERROR: ${_generations} Generations are supported.") endif() unset(CUDA_ARCH_BIN CACHE) unset(CUDA_ARCH_PTX CACHE) From cff0168f3a136b86ac4f415c8332abcd39212f67 Mon Sep 17 00:00:00 2001 From: rogday Date: Wed, 28 Jul 2021 18:06:24 +0300 Subject: [PATCH 086/262] Merge pull request #20453 from rogday:onnx_importer_fix Split layer dispatch into functions in ONNXImporter * split layer dispatch into functions * fixes * identation and comment fixes * fix constness --- modules/dnn/src/dnn_common.hpp | 1 + modules/dnn/src/onnx/onnx_importer.cpp | 3238 +++++++++++--------- modules/dnn/src/tensorflow/tf_importer.cpp | 2 +- 3 files changed, 1724 insertions(+), 1517 deletions(-) diff --git a/modules/dnn/src/dnn_common.hpp b/modules/dnn/src/dnn_common.hpp index cd6cea0c6b..0f3feda91b 100644 --- a/modules/dnn/src/dnn_common.hpp +++ b/modules/dnn/src/dnn_common.hpp @@ -14,6 +14,7 @@ Mutex& getInitializationMutex(); void initializeLayerFactory(); namespace detail { +#define CALL_MEMBER_FN(object, ptrToMemFn) ((object).*(ptrToMemFn)) struct NetImplBase { diff --git a/modules/dnn/src/onnx/onnx_importer.cpp b/modules/dnn/src/onnx/onnx_importer.cpp index 4ad0fd496e..b833b2ea44 100644 --- a/modules/dnn/src/onnx/onnx_importer.cpp +++ b/modules/dnn/src/onnx/onnx_importer.cpp @@ -62,7 +62,7 @@ class ONNXImporter public: ONNXImporter(Net& net, const char *onnxFile) - : dstNet(net) + : dstNet(net), dispatch(buildDispatchMap()) { hasDynamicShapes = false; CV_Assert(onnxFile); @@ -83,7 +83,7 @@ public: } ONNXImporter(Net& net, const char* buffer, size_t sizeBuffer) - : dstNet(net) + : dstNet(net), dispatch(buildDispatchMap()) { hasDynamicShapes = false; CV_LOG_DEBUG(NULL, "DNN/ONNX: processing in-memory ONNX model (" << sizeBuffer << " bytes)"); @@ -124,6 +124,57 @@ protected: typedef std::map::iterator IterLayerId_t; void handleNode(const opencv_onnx::NodeProto& node_proto); + +private: + typedef void (ONNXImporter::*ONNXImporterNodeParser)(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + typedef std::map DispatchMap; + + void parseMaxPool (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseAveragePool (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseReduce (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseSlice (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseSplit (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseBias (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parsePow (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseMax (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseNeg (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseConstant (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseLSTM (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseImageScaler (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseClip (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseLeakyRelu (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseRelu (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseElu (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseTanh (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parsePRelu (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseLRN (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseInstanceNormalization(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseBatchNormalization (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseGemm (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseMatMul (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseMul (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseConv (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseConvTranspose (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseTranspose (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseSqueeze (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseFlatten (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseUnsqueeze (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseExpand (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseReshape (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parsePad (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseShape (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseCast (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseConstantFill (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseGather (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseConcat (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseResize (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseUpsample (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseSoftMax (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseDetectionOutput (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseCustom (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + + const DispatchMap dispatch; + static const DispatchMap buildDispatchMap(); }; inline void replaceLayerParam(LayerParams& layerParams, const String& oldKey, const String& newKey) @@ -448,13 +499,11 @@ void ONNXImporter::populateNet() CV_LOG_DEBUG(NULL, "DNN/ONNX: import completed!"); } -void ONNXImporter::handleNode(const opencv_onnx::NodeProto& node_proto_) +void ONNXImporter::handleNode(const opencv_onnx::NodeProto& node_proto) { - opencv_onnx::NodeProto node_proto = node_proto_; // TODO FIXIT - CV_Assert(node_proto.output_size() >= 1); std::string name = node_proto.output(0); - std::string layer_type = node_proto.op_type(); + const std::string& layer_type = node_proto.op_type(); CV_LOG_DEBUG(NULL, "DNN/ONNX: processing node with " << node_proto.input_size() << " inputs and " << node_proto.output_size() << " outputs: " << cv::format("[%s]:(%s)", layer_type.c_str(), name.c_str()) ); @@ -468,1521 +517,15 @@ void ONNXImporter::handleNode(const opencv_onnx::NodeProto& node_proto_) layerParams.type = layer_type; layerParams.set("has_dynamic_shapes", hasDynamicShapes); - if (layer_type == "MaxPool") + DispatchMap::const_iterator iter = dispatch.find(layer_type); + if (iter != dispatch.end()) { - layerParams.type = "Pooling"; - layerParams.set("pool", "MAX"); - layerParams.set("ceil_mode", layerParams.has("pad_mode")); - } - else if (layer_type == "AveragePool") - { - layerParams.type = "Pooling"; - layerParams.set("pool", "AVE"); - layerParams.set("ceil_mode", layerParams.has("pad_mode")); - layerParams.set("ave_pool_padded_area", framework_name == "pytorch"); - } - else if (layer_type == "GlobalAveragePool" || layer_type == "GlobalMaxPool" || - layer_type == "ReduceMean" || layer_type == "ReduceSum" || layer_type == "ReduceMax") - { - CV_Assert(node_proto.input_size() == 1); - layerParams.type = "Pooling"; - String pool; - if (layer_type == "GlobalMaxPool" || layer_type == "ReduceMax") - pool = "MAX"; - else if (layer_type == "ReduceSum") - pool = "SUM"; - else - pool = "AVE"; - layerParams.set("pool", pool); - layerParams.set("global_pooling", !layerParams.has("axes")); - if (layerParams.has("axes") && (layer_type == "ReduceMean" || layer_type == "ReduceSum" || layer_type == "ReduceMax")) - { - MatShape inpShape = outShapes[node_proto.input(0)]; - DictValue axes = layerParams.get("axes"); - bool keepdims = layerParams.get("keepdims"); - MatShape targetShape; - std::vector shouldDelete(inpShape.size(), false); - for (int i = 0; i < axes.size(); i++) { - int axis = normalize_axis(axes.get(i), inpShape.size()); - shouldDelete[axis] = true; - } - for (int axis = 0; axis < inpShape.size(); ++axis){ - if (!shouldDelete[axis]) - targetShape.push_back(inpShape[axis]); - else if (keepdims) - targetShape.push_back(1); - } - - if (inpShape.size() == 3 && axes.size() <= 2) - { - int axis = normalize_axis(axes.get(0), inpShape.size()); - CV_CheckNE(axis, 0, ""); - - LayerParams reshapeLp; - reshapeLp.name = layerParams.name + "/reshape"; - reshapeLp.type = "Reshape"; - CV_Assert(layer_id.find(reshapeLp.name) == layer_id.end()); - reshapeLp.set("axis", 0); - reshapeLp.set("num_axes", 1); - int newShape[] = {1, -1}; - reshapeLp.set("dim", DictValue::arrayInt(&newShape[0], 2)); - - opencv_onnx::NodeProto proto; - proto.add_input(node_proto.input(0)); - proto.add_output(reshapeLp.name); - addLayer(reshapeLp, proto); - - LayerParams avgLp; - avgLp.name = layerParams.name + "/avg"; - avgLp.type = "Pooling"; - CV_Assert(layer_id.find(avgLp.name) == layer_id.end()); - avgLp.set("pool", pool); - if (axes.size() == 2) - { - CV_CheckEQ(normalize_axis(axes.get(0), inpShape.size()), 1, "Unsupported mode"); - CV_CheckEQ(normalize_axis(axes.get(1), inpShape.size()), 2, "Unsupported mode"); - avgLp.set("global_pooling", true); - } - else - { - avgLp.set(axis == 2 ? "global_pooling_w" : "global_pooling_h", true); - avgLp.set(axis == 2 ? "kernel_h" : "kernel_w", 1); - } - - node_proto.set_input(0, reshapeLp.name); - node_proto.set_output(0, avgLp.name); - addLayer(avgLp, node_proto); - } - else - { - if (inpShape.size() != 4 && inpShape.size() != 5) - CV_Error(Error::StsNotImplemented, "Unsupported input shape of " + layer_type + " operation."); - - CV_Assert(axes.size() <= inpShape.size() - 2); - std::vector kernel_size(inpShape.size() - 2, 1); - if (axes.size() == 1 && (normalize_axis(axes.get(0), inpShape.size()) <= 1)) - { - int axis = normalize_axis(axes.get(0), inpShape.size()); - MatShape newShape = inpShape; - newShape[axis + 1] = total(newShape, axis + 1); - newShape.resize(axis + 2); - newShape.insert(newShape.begin(), 2 - axis, 1); - - LayerParams reshapeLp; - reshapeLp.type = "Reshape"; - reshapeLp.name = layerParams.name + "/reshape"; - CV_Assert(layer_id.find(reshapeLp.name) == layer_id.end()); - reshapeLp.set("dim", DictValue::arrayInt(&newShape[0], newShape.size())); - - node_proto.set_output(0, reshapeLp.name); - addLayer(reshapeLp, node_proto); - - kernel_size.resize(2); - kernel_size[0] = inpShape[axis]; - node_proto.set_input(0, node_proto.output(0)); - } - else - { - for (int i = 0; i < axes.size(); i++) { - int axis = normalize_axis(axes.get(i), inpShape.size()); - CV_Assert_N(axis >= 2 + i, axis < inpShape.size()); - kernel_size[axis - 2] = inpShape[axis]; - } - } - - LayerParams poolLp = layerParams; - poolLp.name = layerParams.name + "/avg"; - CV_Assert(layer_id.find(poolLp.name) == layer_id.end()); - poolLp.set("kernel_size", DictValue::arrayInt(&kernel_size[0], kernel_size.size())); - - node_proto.set_output(0, poolLp.name); - addLayer(poolLp, node_proto); - } - - layerParams.type = "Reshape"; - layerParams.set("dim", DictValue::arrayInt(&targetShape[0], targetShape.size())); - - node_proto.set_input(0, node_proto.output(0)); - node_proto.set_output(0, layerParams.name); - } - else if (!layerParams.has("axes") && (layer_type == "ReduceMean" || layer_type == "ReduceSum" || layer_type == "ReduceMax")) - { - CV_CheckEQ(layerParams.get("keepdims"), 0, "layer only supports keepdims = false"); - LayerParams reshapeLp; - reshapeLp.name = layerParams.name + "/reshape"; - reshapeLp.type = "Reshape"; - CV_Assert(layer_id.find(reshapeLp.name) == layer_id.end()); - int newShape[] = {1, 1, 1, -1}; - reshapeLp.set("dim", DictValue::arrayInt(&newShape[0], 4)); - - opencv_onnx::NodeProto proto; - proto.add_input(node_proto.input(0)); - proto.add_output(reshapeLp.name); - addLayer(reshapeLp, proto); - - LayerParams poolLp = layerParams; - poolLp.name = layerParams.name + "/pool"; - CV_Assert(layer_id.find(poolLp.name) == layer_id.end()); - - node_proto.set_input(0, reshapeLp.name); - node_proto.set_output(0, poolLp.name); - addLayer(poolLp, node_proto); - - layerParams.type = "Reshape"; - int targetShape[] = {1}; - layerParams.set("dim", DictValue::arrayInt(&targetShape[0], 1)); - - node_proto.set_input(0, node_proto.output(0)); - node_proto.set_output(0, layerParams.name); - } - } - else if (layer_type == "Slice") - { - int axis = 0; - std::vector begin; - std::vector end; - std::vector steps; - int inp_size = node_proto.input_size(); - - if (inp_size == 1) - { - if (layerParams.has("axes")) { - DictValue axes = layerParams.get("axes"); - for (int i = 1; i < axes.size(); ++i) { - CV_Assert(axes.get(i - 1) == axes.get(i) - 1); - } - axis = axes.get(0); - } - - DictValue starts = layerParams.get("starts"); - DictValue ends = layerParams.get("ends"); - CV_Assert(starts.size() == ends.size()); - - if (axis > 0) { - begin.resize(axis, 0); - end.resize(axis, -1); - } - for (int i = 0; i < starts.size(); ++i) - { - begin.push_back(starts.get(i)); - int finish = ends.get(i); - end.push_back((finish < 0) ? --finish : finish); // numpy doesn't include last dim - } - } else { // inp_size > 1 - CV_Assert(inp_size >= 3); - for (int i = 1; i < inp_size; i++) { - CV_Assert(constBlobs.find(node_proto.input(i)) != constBlobs.end()); - } - Mat start_blob = getBlob(node_proto, 1); - Mat end_blob = getBlob(node_proto, 2); - CV_Assert(start_blob.total() == end_blob.total()); - - if (inp_size > 3) { - Mat axes_blob = getBlob(node_proto, 3); - const int* axes = (int*)axes_blob.data; - for (int i = 1; i < axes_blob.total(); ++i) { - CV_Assert(axes[i - 1] == axes[i] - 1); - } - axis = axes[0]; - } - - const int* starts = start_blob.ptr(); - const int* ends = end_blob.ptr(); - if (axis > 0) { - begin.resize(axis, 0); - end.resize(axis, -1); - } - std::copy(starts, starts + start_blob.total(), std::back_inserter(begin)); - for (int i = 0; i < end_blob.total(); ++i) - { - int finish = ends[i]; - end.push_back((finish < 0) ? --finish : finish); // numpy doesn't include last dim - } - - if (inp_size == 5) { - CV_Assert(constBlobs.find(node_proto.input(4)) != constBlobs.end()); - Mat step_blob = getBlob(node_proto, 4); - const int* steps_ptr = step_blob.ptr(); - - if (axis > 0) - steps.resize(axis, 1); - - std::copy(steps_ptr, steps_ptr + step_blob.total(), std::back_inserter(steps)); - - // Very strange application for Slice op with tensor reversing. - // We just workaround it for 2d constants. - if (constBlobs.find(node_proto.input(0)) != constBlobs.end() && - axis == 0 && - start_blob.at(0) == -1 && step_blob.at(0) == -1 && - end_blob.at(0) == std::numeric_limits::min()) - { - Mat inp = getBlob(node_proto, 0); - if (inp.dims == 2) - { - Mat flipped; - flip(inp, flipped, 0); - addConstant(layerParams.name, flipped); - return; - } - } - } - } - layerParams.set("begin", DictValue::arrayInt(&begin[0], begin.size())); - layerParams.set("end", DictValue::arrayInt(&end[0], end.size())); - layerParams.set("axis", axis); - - if (!steps.empty()) - layerParams.set("steps", DictValue::arrayInt(&steps[0], steps.size())); - - if (constBlobs.find(node_proto.input(0)) != constBlobs.end()) - { - Mat inp = getBlob(node_proto, 0); - std::vector inputs, sliced; - inputs.push_back(inp); - runLayer(layerParams, inputs, sliced); - CV_Assert(sliced.size() == 1); - addConstant(layerParams.name, sliced[0]); - return; - } - } - else if (layer_type == "Split") - { - if (layerParams.has("split")) - { - DictValue splits = layerParams.get("split"); - const int numSplits = splits.size(); - CV_Assert(numSplits > 1); - - std::vector slicePoints(numSplits - 1, splits.get(0)); - for (int i = 1; i < splits.size() - 1; ++i) - { - slicePoints[i] = slicePoints[i - 1] + splits.get(i - 1); - } - layerParams.set("slice_point", DictValue::arrayInt(&slicePoints[0], slicePoints.size())); - } - else - { - layerParams.set("num_split", node_proto.output_size()); - } - layerParams.type = "Slice"; - } - else if (layer_type == "Add" || layer_type == "Sum" || layer_type == "Sub") - { - bool isSub = layer_type == "Sub"; - CV_CheckEQ(node_proto.input_size(), 2, ""); - bool is_const_0 = layer_id.find(node_proto.input(0)) == layer_id.end(); - bool is_const_1 = layer_id.find(node_proto.input(1)) == layer_id.end(); - if (is_const_0 && is_const_1) - { - Mat blob_0 = getBlob(node_proto, 0); - Mat blob_1 = getBlob(node_proto, 1); - CV_Assert(blob_0.size == blob_1.size); - Mat output = isSub ? (blob_0 - blob_1) : (blob_0 + blob_1); - addConstant(layerParams.name, output); - return; - } - else if (is_const_0 || is_const_1) - { - int const_blob_id = is_const_0 ? 0 : 1; - Mat blob = getBlob(node_proto, const_blob_id); - int blob_total = blob.total(); - if (blob_total == 1) { - layerParams.type = "Power"; - layerParams.set("shift", (isSub ? -1 : 1) * blob.ptr()[0]); - } - else { - MatShape inpShape = outShapes[node_proto.input(1 - const_blob_id)]; - if (shape(blob) == inpShape) - { - LayerParams constParams; - constParams.name = layerParams.name + "/const"; - constParams.type = "Const"; - constParams.blobs.push_back((isSub ? -1 : 1) * blob); - int id = dstNet.addLayer(constParams.name, constParams.type, constParams); - layer_id.insert(std::make_pair(constParams.name, LayerInfo(id, 0))); - outShapes[constParams.name] = shape(blob); - - layerParams.type = "Eltwise"; - node_proto.set_input(const_blob_id, constParams.name); - } - else - { - layerParams.type = "Scale"; - layerParams.set("bias_term", true); - int axis = 1; - for (int i = 0; i < graph_proto.initializer_size(); i++) - { - opencv_onnx::TensorProto tensor_proto = graph_proto.initializer(i); - if (tensor_proto.name() == node_proto.input(const_blob_id)) - { - axis = inpShape.size() - tensor_proto.dims_size(); - break; - } - } - layerParams.set("axis", axis); - blob = blob.reshape(1, 1); - layerParams.blobs.push_back((isSub ? -1 : 1) * blob); - } - } - } - else if (outShapes[node_proto.input(0)] == outShapes[node_proto.input(1)]) - { - layerParams.type = "Eltwise"; - if (isSub) - { - static float subCoeffs[] = {1.f, -1.f}; - layerParams.set("coeff", DictValue::arrayReal(subCoeffs, 2)); - } - } - else - { - if (isSub) - { - LayerParams powerParams; - powerParams.name = layerParams.name + "/neg"; - powerParams.type = "Power"; - powerParams.set("scale", -1); - - //Create Power layer - int id = dstNet.addLayer(powerParams.name, powerParams.type, powerParams); - //Connect to input - IterLayerId_t layerId = layer_id.find(node_proto.input(1)); - CV_Assert(layerId != layer_id.end()); - dstNet.connect(layerId->second.layerId, layerId->second.outputId, id, 0); - //Add shape - layer_id.insert(std::make_pair(powerParams.name, LayerInfo(id, 0))); - outShapes[powerParams.name] = outShapes[node_proto.input(1)]; - - //Replace input to Power - node_proto.set_input(1, powerParams.name); - } - layerParams.type = "Scale"; - layerParams.set("bias_term", true); - } - } - else if (layer_type == "Pow") - { - if (layer_id.find(node_proto.input(1)) != layer_id.end()) - CV_Error(Error::StsNotImplemented, "Unsupported Pow op with variable power"); - - Mat blob = getBlob(node_proto, 1); - if (blob.total() != 1) - CV_Error(Error::StsNotImplemented, "Pow op supports only scalar power"); - - blob.convertTo(blob, CV_32F); - layerParams.type = "Power"; - layerParams.set("power", blob.ptr()[0]); - } - else if (layer_type == "Max") - { - layerParams.type = "Eltwise"; - layerParams.set("operation", "max"); - } - else if (layer_type == "Neg") - { - layerParams.type = "Power"; - layerParams.set("scale", -1); - } - else if (layer_type == "Constant") - { - CV_Assert(node_proto.input_size() == 0); - CV_Assert(layerParams.blobs.size() == 1); - addConstant(layerParams.name, layerParams.blobs[0]); - return; - } - else if (layer_type == "LSTM") - { - LayerParams lstmParams = layerParams; - lstmParams.name += "/lstm"; - - // https://pytorch.org/docs/stable/nn.html#lstm - CV_Assert(node_proto.input_size() == 7); - Mat Wx = getBlob(node_proto, 1); - Mat Wh = getBlob(node_proto, 2); - Mat b = getBlob(node_proto, 3); - Mat h0 = getBlob(node_proto, 5); - Mat c0 = getBlob(node_proto, 6); - - b = b.reshape(1, b.size[0]); - - const int numHidden = lstmParams.get("hidden_size"); - const int numDirs = Wx.size[0]; // Is 1 for forward only and 2 for bidirectional LSTM. - const int numFeatures = Wx.size[2]; - Mat bx = b.colRange(0, b.cols / 2); - Mat bh = b.colRange(b.cols / 2, b.cols); - b = bx + bh; - - // IFGO->IGFO - for (int k = 0; k < numDirs; ++k) - { - float* WxData = Wx.ptr(k); - float* WhData = Wh.ptr(k); - float* biasData = b.ptr(k); - for (int j = 0; j < numHidden; ++j) - { - for (int i = 0; i < numFeatures; ++i) - { - std::swap(WxData[(numHidden + j) * numFeatures + i], - WxData[(numHidden * 2 + j) * numFeatures + i]); - } - for (int i = 0; i < numHidden; ++i) - { - std::swap(WhData[(numHidden + j) * numHidden + i], - WhData[(numHidden * 2 + j) * numHidden + i]); - } - std::swap(biasData[numHidden + j], biasData[numHidden * 2 + j]); - } - } - Wx = Wx.reshape(1, Wx.size[0] * Wx.size[1]); - Wh = Wh.reshape(1, Wh.size[0] * Wh.size[1]); - h0 = h0.reshape(1, h0.size[0] * h0.size[1]); - c0 = c0.reshape(1, c0.size[0] * c0.size[1]); - - lstmParams.blobs.resize(5); - lstmParams.blobs[0] = Wh; - lstmParams.blobs[1] = Wx; - lstmParams.blobs[2] = b; - lstmParams.blobs[3] = h0; - lstmParams.blobs[4] = c0; - lstmParams.set("bidirectional", lstmParams.get("direction", "") == "bidirectional"); - - node_proto.set_output(0, lstmParams.name); // set different name so output shapes will be registered on that name - addLayer(lstmParams, node_proto); - - MatShape lstmShape = outShapes[node_proto.output(0)]; - - // Add fake 1 as it is done in ONNX - lstmShape.insert(lstmShape.begin() + 1, 1); - - layerParams.type = "Reshape"; - layerParams.set("dim", DictValue::arrayInt(&lstmShape[0], lstmShape.size())); - node_proto.set_input(0, lstmParams.name); // redirect input to LSTM - node_proto.set_output(0, layerParams.name); // keep origin LSTM's name - } - else if (layer_type == "ImageScaler") - { - const float scale = layerParams.has("scale") ? layerParams.get("scale") : 1.0f; - layerParams.erase("scale"); - - if (layerParams.has("bias")) - { - layerParams.type = "Scale"; - layerParams.blobs.push_back( - Mat(Size(1, layerParams.get("bias").size()), CV_32FC1, scale)); - - layerParams.set("bias_term", true); - Mat bias(1, layerParams.get("bias").size(), CV_32FC1); - for (int j = 0; j < bias.total(); j++) { - bias.at(0, j) = layerParams.get("bias").getRealValue(j); - } - layerParams.blobs.push_back(bias); - layerParams.erase("bias"); - } - else { - layerParams.set("scale", scale); - layerParams.type = "Power"; - } - } - else if (layer_type == "Clip") - { - layerParams.type = "ReLU6"; - replaceLayerParam(layerParams, "min", "min_value"); - replaceLayerParam(layerParams, "max", "max_value"); - - } - else if (layer_type == "LeakyRelu") - { - layerParams.type = "ReLU"; - replaceLayerParam(layerParams, "alpha", "negative_slope"); - } - else if (layer_type == "Relu") - { - layerParams.type = "ReLU"; - } - else if (layer_type == "Elu") - { - layerParams.type = "ELU"; - } - else if (layer_type == "Tanh") - { - layerParams.type = "TanH"; - } - else if (layer_type == "PRelu") - { - layerParams.type = "PReLU"; - layerParams.blobs.push_back(getBlob(node_proto, 1)); - } - else if (layer_type == "LRN") - { - replaceLayerParam(layerParams, "size", "local_size"); - } - else if (layer_type == "InstanceNormalization") - { - if (node_proto.input_size() != 3) - CV_Error(Error::StsNotImplemented, - "Expected input, scale, bias"); - - layerParams.blobs.resize(4); - layerParams.blobs[2] = getBlob(node_proto, 1); // weightData - layerParams.blobs[3] = getBlob(node_proto, 2); // biasData - layerParams.set("has_bias", true); - layerParams.set("has_weight", true); - - // Get number of channels in input - int size = layerParams.blobs[2].total(); - layerParams.blobs[0] = Mat::zeros(size, 1, CV_32F); // mean - layerParams.blobs[1] = Mat::ones(size, 1, CV_32F); // std - - LayerParams mvnParams; - mvnParams.name = layerParams.name + "/MVN"; - mvnParams.type = "MVN"; - mvnParams.set("eps", layerParams.get("epsilon")); - layerParams.erase("epsilon"); - - //Create MVN layer - int id = dstNet.addLayer(mvnParams.name, mvnParams.type, mvnParams); - //Connect to input - IterLayerId_t layerId = layer_id.find(node_proto.input(0)); - CV_Assert(layerId != layer_id.end()); - dstNet.connect(layerId->second.layerId, layerId->second.outputId, id, 0); - //Add shape - layer_id.insert(std::make_pair(mvnParams.name, LayerInfo(id, 0))); - outShapes[mvnParams.name] = outShapes[node_proto.input(0)]; - - //Replace Batch Norm's input to MVN - node_proto.set_input(0, mvnParams.name); - layerParams.type = "BatchNorm"; - } - else if (layer_type == "BatchNormalization") - { - if (node_proto.input_size() != 5) - CV_Error(Error::StsNotImplemented, - "Expected input, scale, bias, mean and var"); - - layerParams.type = "BatchNorm"; - replaceLayerParam(layerParams, "epsilon", "eps"); - replaceLayerParam(layerParams, "spatial", "use_global_stats"); - - Mat meanData = getBlob(node_proto, 3); - Mat stdData = getBlob(node_proto, 4); - - layerParams.blobs.push_back(meanData); - layerParams.blobs.push_back(stdData); - - if (!node_proto.input(1).empty()) { - layerParams.set("has_weight", true); - layerParams.blobs.push_back(getBlob(node_proto, 1)); // weightData - } else { - layerParams.set("has_weight", false); - } - - if (!node_proto.input(2).empty()) { - layerParams.set("has_bias", true); - layerParams.blobs.push_back(getBlob(node_proto, 2)); // biasData - } else { - layerParams.set("has_bias", false); - } - } - else if (layer_type == "Gemm") - { - CV_Assert(node_proto.input_size() >= 2); - layerParams.type = "InnerProduct"; - Mat weights = getBlob(node_proto, 1); - int ind_num_out = 0; - if (layerParams.has("transB") && !layerParams.get("transB")) { - transpose(weights, weights); - ind_num_out = 1; - } - layerParams.blobs.push_back(weights); - - if (node_proto.input_size() == 3) { - Mat bias = getBlob(node_proto, 2); - layerParams.blobs.push_back(bias); - } - if (constBlobs.find(node_proto.input(0)) != constBlobs.end()) - { - Mat inputBuf = getBlob(node_proto, 0); - - LayerParams constParams; - constParams.name = node_proto.input(0); - constParams.type = "Const"; - constParams.blobs.push_back(inputBuf); - - opencv_onnx::NodeProto proto; - proto.add_output(constParams.name); - addLayer(constParams, proto); - } - - layerParams.set("num_output", layerParams.blobs[0].size[ind_num_out]); - layerParams.set("bias_term", node_proto.input_size() == 3); - } - else if (layer_type == "MatMul") - { - CV_Assert(node_proto.input_size() == 2); - layerParams.type = "InnerProduct"; - layerParams.set("bias_term", false); - CV_Assert(constBlobs.find(node_proto.input(0)) == constBlobs.end()); - int firstInpDims = outShapes[node_proto.input(0)].size(); - int secondInpDims; - - if (constBlobs.find(node_proto.input(1)) != constBlobs.end()) - { - Mat blob = getBlob(node_proto, 1); - secondInpDims = blob.dims; - layerParams.blobs.push_back(blob.t()); - layerParams.set("num_output", layerParams.blobs[0].size[0]); - } else { - secondInpDims = outShapes[node_proto.input(1)].size(); - } - layerParams.set("axis", firstInpDims - secondInpDims + 1); - } - else if (layer_type == "Mul" || layer_type == "Div") - { - CV_Assert(node_proto.input_size() == 2); - - bool isDiv = layer_type == "Div"; - int constId = -1; - bool haveVariables = false; - for (int i = 0; i < 2; ++i) - { - if (constBlobs.find(node_proto.input(i)) != constBlobs.end()) - constId = i; - else - haveVariables = true; - } - if (constId != -1 && haveVariables) - { - Mat blob = getBlob(node_proto, constId); - blob = blob.reshape(1, 1); - if (blob.total() == 1) { - float blob_value = blob.ptr()[0]; - float coeff = isDiv ? 1.0 / blob_value : blob_value; - layerParams.set("scale", coeff); - layerParams.type = "Power"; - } - else { - if (isDiv) - divide(1.0, blob, blob); - layerParams.blobs.push_back(blob); - layerParams.type = "Scale"; - } - } - else if (!haveVariables) - { - Mat inp0 = getBlob(node_proto, 0); - Mat inp1 = getBlob(node_proto, 1); - - if (inp0.size != inp1.size && (inp0.total() != 1 || inp1.total() != 1)) - CV_Error_(Error::StsNotImplemented, ("Different shapes case is not supported with constant inputs: %s", layer_type.c_str())); - - if (inp0.total() == 1 && inp1.total() == 1 && inp0.dims != inp1.dims) - { - if (inp0.dims < inp1.dims) - { - inp0 = inp0.reshape(1, inp1.dims, inp1.size); - inp0.dims = inp1.dims; - } - else - { - inp1 = inp1.reshape(1, inp0.dims, inp0.size); - inp1.dims = inp0.dims; - } - } - - Mat out; - if (inp0.total() != inp1.total()) - { - if (inp0.total() == 1) - { - float inp0_value = inp0.ptr()[0]; - float coeff = isDiv ? 1.0 / inp0_value : inp0_value; - multiply(inp1, coeff, out); - } - else - { - float inp1_value = inp1.ptr()[0]; - float coeff = isDiv ? 1.0 / inp1_value : inp1_value; - multiply(inp0, coeff, out); - } - - } - else - { - out = isDiv ? inp0 / inp1 : inp0.mul(inp1); - } - - if (inp0.dims == 1 && inp1.dims == 1) - out.dims = 1; // to workaround dims == 1 - addConstant(layerParams.name, out); - return; - } - else if (outShapes[node_proto.input(0)] == outShapes[node_proto.input(1)]) - { - layerParams.type = "Eltwise"; - layerParams.set("operation", isDiv ? "div" : "prod"); - } - else - { - // Scale layer allocate output with the first input shape - if (total(outShapes[node_proto.input(0)]) < total(outShapes[node_proto.input(1)])) - { - opencv_onnx::NodeProto proto; - proto.add_input(node_proto.input(1)); - proto.add_input(node_proto.input(0)); - proto.add_output(layerParams.name); - node_proto = proto; - } - - if (isDiv) - { - LayerParams powerParams; - powerParams.name = layerParams.name + "/inv"; - powerParams.type = "Power"; - powerParams.set("power", -1); - - //Create Power layer - int id = dstNet.addLayer(powerParams.name, powerParams.type, powerParams); - //Connect to input - IterLayerId_t layerId = layer_id.find(node_proto.input(1)); - CV_Assert(layerId != layer_id.end()); - dstNet.connect(layerId->second.layerId, layerId->second.outputId, id, 0); - //Add shape - layer_id.insert(std::make_pair(powerParams.name, LayerInfo(id, 0))); - outShapes[powerParams.name] = outShapes[node_proto.input(1)]; - - //Replace input to Power - node_proto.set_input(1, powerParams.name); - } - layerParams.type = "Scale"; - } - } - else if (layer_type == "Conv") - { - CV_Assert(node_proto.input_size() >= 2); - layerParams.type = "Convolution"; - for (int j = 1; j < node_proto.input_size(); j++) { - if (constBlobs.find(node_proto.input(j)) != constBlobs.end()) - { - layerParams.blobs.push_back(getBlob(node_proto, j)); - } - } - int outCn = layerParams.blobs.empty() ? outShapes[node_proto.input(1)][0] : layerParams.blobs[0].size[0]; - layerParams.set("num_output", outCn); - - // Check for asymmetric padding in Conv2D - if (layerParams.has("pad")) - { - bool asymmetricPadding = false; - DictValue pads = layerParams.get("pad"); - const int dims = pads.size() / 2; - for (int i = 0; i < dims; ++i) - { - if (pads.get(i) != pads.get(i + dims)) - { - asymmetricPadding = true; - break; - } - } - if (asymmetricPadding && pads.size() == 4) // [pad_t, pad_l, pad_b, pad_r] - { - layerParams.erase("pad"); - // No paddings required for N, C axis - std::vector paddings(4, 0); - // Add paddings for H, W axis - for (int i = 0; i < dims; ++i) - { - paddings.push_back(pads.get(i)); - paddings.push_back(pads.get(dims + i)); - } - LayerParams padLp; - padLp.name = layerParams.name + "/pad"; - padLp.type = "Padding"; - padLp.set("paddings", DictValue::arrayInt(&paddings[0], paddings.size())); - - opencv_onnx::NodeProto proto; - proto.add_input(node_proto.input(0)); - proto.add_output(padLp.name); - - addLayer(padLp, proto); - node_proto.set_input(0, padLp.name); - } - } - } - else if (layer_type == "ConvTranspose") - { - CV_Assert(node_proto.input_size() >= 2); - layerParams.type = "Deconvolution"; - for (int j = 1; j < node_proto.input_size(); j++) { - layerParams.blobs.push_back(getBlob(node_proto, j)); - } - layerParams.set("num_output", layerParams.blobs[0].size[1] * layerParams.get("group", 1)); - layerParams.set("bias_term", node_proto.input_size() == 3); - - if (!layerParams.has("kernel_size")) - CV_Error(Error::StsNotImplemented, - "Required attribute 'kernel_size' is not present."); - - if (layerParams.has("output_shape")) - { - const DictValue& outShape = layerParams.get("output_shape"); - DictValue strides = layerParams.get("stride"); - DictValue kernel = layerParams.get("kernel_size"); - - String padMode; - std::vector adjust_pads; - if (layerParams.has("pad_mode")) - { - padMode = toUpperCase(layerParams.get("pad_mode")); - if (padMode != "SAME" && padMode != "VALID") - CV_Error(Error::StsError, "Unsupported padding mode " + padMode); - - for (int i = 0; i < strides.size(); i++) - { - int sz = outShape.get(2 + i); - int stride = strides.get(i); - adjust_pads.push_back(padMode == "SAME"? (sz - 1) % stride : - (sz - kernel.get(i)) % stride); - } - layerParams.set("adj", DictValue::arrayInt(&adjust_pads[0], adjust_pads.size())); - } - } - else if (layerParams.has("output_padding")) - { - replaceLayerParam(layerParams, "output_padding", "adj"); - } - } - else if (layer_type == "Transpose") - { - layerParams.type = "Permute"; - replaceLayerParam(layerParams, "perm", "order"); - - CV_Assert(node_proto.input_size() == 1); - if (constBlobs.find(node_proto.input(0)) != constBlobs.end()) - { - std::vector inputs(1, getBlob(node_proto, 0)), transposed; - runLayer(layerParams, inputs, transposed); - CV_Assert(transposed.size() == 1); - addConstant(layerParams.name, transposed[0]); - return; - } - } - else if (layer_type == "Squeeze") - { - CV_Assert_N(node_proto.input_size() == 1, layerParams.has("axes")); - DictValue axes_dict = layerParams.get("axes"); - MatShape inpShape = outShapes[node_proto.input(0)]; - - std::vector maskedAxes(inpShape.size(), false); - for (int i = 0; i < axes_dict.size(); ++i) - { - int axis = axes_dict.getIntValue(i); - CV_CheckLE(axis, static_cast(inpShape.size()), "Squeeze axis"); - maskedAxes[axis] = inpShape[axis] == 1; - } - MatShape outShape; - for (int i = 0; i < inpShape.size(); ++i) - { - if (!maskedAxes[i]) - outShape.push_back(inpShape[i]); - } - if (outShape.size() != inpShape.size()) - { - layerParams.type = "Reshape"; - layerParams.set("dim", DictValue::arrayInt(&outShape[0], outShape.size())); - if (hasDynamicShapes) - { - std::vector dynamicAxes; - std::vector inputIndices; - for (int index = 0; index < inpShape.size(); ++index) - { - if (!maskedAxes[index]) - inputIndices.push_back(index); - } - for (int index = 0; index < outShape.size(); ++index) - dynamicAxes.push_back(index); - layerParams.set("dynamic_axes", DictValue::arrayInt(dynamicAxes.data(), dynamicAxes.size())); - layerParams.set("input_indices", DictValue::arrayInt(inputIndices.data(), inputIndices.size())); - } - } - else - layerParams.type = "Identity"; - - if (constBlobs.find(node_proto.input(0)) != constBlobs.end()) - { - Mat inp = getBlob(node_proto, 0); - Mat out = inp.reshape(1, outShape); - out.dims = outShape.size(); // to workaround dims == 1 - addConstant(layerParams.name, out); - return; - } - } - else if (layer_type == "Flatten") - { - CV_CheckEQ(node_proto.input_size(), 1, ""); - if (constBlobs.find(node_proto.input(0)) != constBlobs.end()) - { - Mat input = getBlob(node_proto, 0); - int axis = normalize_axis(layerParams.get("axis", 1), input.dims); - - std::vector out_size(&input.size[0], &input.size[0] + axis); - out_size.push_back(input.total(axis)); - Mat output = input.reshape(1, out_size); - addConstant(layerParams.name, output); - return; - } - } - else if (layer_type == "Unsqueeze") - { - CV_Assert(node_proto.input_size() == 1); - DictValue axes = layerParams.get("axes"); - if (constBlobs.find(node_proto.input(0)) != constBlobs.end()) - { - // Constant input. - Mat input = getBlob(node_proto, 0); - - std::vector dims; - for (int j = 0; j < input.dims; j++) { - dims.push_back(input.size[j]); - } - CV_Assert(axes.getIntValue(axes.size()-1) <= dims.size()); - for (int j = 0; j < axes.size(); j++) { - dims.insert(dims.begin() + axes.getIntValue(j), 1); - } - - Mat out = input.reshape(0, dims); - addConstant(layerParams.name, out); - return; - } - - // Variable input. - if (axes.size() != 1) - CV_Error(Error::StsNotImplemented, "Multidimensional unsqueeze"); - - MatShape inpShape = outShapes[node_proto.input(0)]; - int axis = axes.getIntValue(0); - CV_Assert(0 <= axis && axis <= inpShape.size()); - std::vector outShape = inpShape; - outShape.insert(outShape.begin() + axis, 1); - layerParams.type = "Reshape"; - layerParams.set("dim", DictValue::arrayInt(&outShape[0], outShape.size())); - if (hasDynamicShapes) - { - std::vector dynamicAxes; - std::vector inputIndices; - for (int index = 0; index < outShape.size(); ++index) { - if (index != axis) - dynamicAxes.push_back(index); - } - for (int index = 0; index < inpShape.size(); ++index) - inputIndices.push_back(index); - layerParams.set("dynamic_axes", DictValue::arrayInt(dynamicAxes.data(), dynamicAxes.size())); - layerParams.set("input_indices", DictValue::arrayInt(inputIndices.data(), inputIndices.size())); - } - } - else if (layer_type == "Expand") - { - CV_CheckEQ(node_proto.input_size(), 2, ""); - const std::string& input0 = node_proto.input(0); - const std::string& input1 = node_proto.input(1); - Mat newShapeMat = getBlob(input1); - MatShape targetShape(newShapeMat.ptr(), newShapeMat.ptr() + newShapeMat.total()); - - MatShape inpShape; - bool haveVariables = constBlobs.find(input0) == constBlobs.end(); - if (haveVariables) - { - IterShape_t shapeIt = outShapes.find(input0); - CV_Assert(shapeIt != outShapes.end()); - inpShape = shapeIt->second; - } - else - { - inpShape = shape(getBlob(input0)); - } - - String srcName = input0; - // Unsqueeze and repeat along new axis - if (targetShape.size() == inpShape.size() + 1) - { - for (int i = 0; i < targetShape.size(); i++) - { - if (targetShape[i] == -1 && i < inpShape.size()) - targetShape[i] = inpShape[i]; - else if (i < inpShape.size() && targetShape[i] != inpShape[i]) - inpShape.insert(inpShape.begin() + i, 1); - } - if (haveVariables) - { - LayerParams reshapeLp; - reshapeLp.name = layerParams.name + "/reshape"; - reshapeLp.type = "Reshape"; - CV_Assert(layer_id.find(reshapeLp.name) == layer_id.end()); - reshapeLp.set("dim", DictValue::arrayInt(&inpShape[0], inpShape.size())); - - opencv_onnx::NodeProto proto; - proto.add_input(node_proto.input(0)); - proto.add_output(reshapeLp.name); - addLayer(reshapeLp, proto); - srcName = reshapeLp.name; - } - } - CV_CheckEQ(inpShape.size(), targetShape.size(), "Unsupported Expand op with different dims"); - - std::vector broadcast_axes; - for (int i = 0; i < targetShape.size(); i++) - { - if (targetShape[i] != inpShape[i]) - { - if (inpShape[i] == 1) - broadcast_axes.push_back(i); - else - CV_Error(Error::StsError, format("Could not be broadcast by axis: %d", i)); - } - } - - if (!haveVariables) - { - if (broadcast_axes.size() != 1) - CV_Error(Error::StsNotImplemented, "Expand op doesn't support multiple axes for constant input"); - - Mat input = getBlob(node_proto, 0); - input = input.reshape(0, total(inpShape, 0, broadcast_axes[0])); - Mat output = cv::repeat(input, 1, targetShape[broadcast_axes[0]]); - output = output.reshape(0, targetShape); - addConstant(layerParams.name, output); - return; - } - - if (broadcast_axes.size() == 2 && - broadcast_axes[0] == broadcast_axes[1] - 1 && broadcast_axes[1] == inpShape.size() - 1) - { - LayerParams constParams; - constParams.name = layerParams.name + "/const"; - CV_Assert(layer_id.find(constParams.name) == layer_id.end()); - constParams.type = "Const"; - - Mat inp = Mat::ones(newShapeMat.total(), newShapeMat.ptr(), CV_32F); - constParams.blobs.push_back(inp); - - opencv_onnx::NodeProto proto; - proto.add_output(constParams.name); - addLayer(constParams, proto); - - layerParams.type = "Scale"; - layerParams.set("bias_term", false); - node_proto.set_input(0, constParams.name); - node_proto.set_input(1, srcName); - } - else if (broadcast_axes.size() == 1 && broadcast_axes[0] <= 1) - { - String base_name = layerParams.name + "/copy_"; - std::vector input_names; - for (int j = 0; j < targetShape[broadcast_axes[0]]; j++) - { - std::ostringstream ss; - ss << j; - LayerParams copyLP; - copyLP.name = base_name + ss.str(); - copyLP.type = "Identity"; - CV_Assert(layer_id.find(copyLP.name) == layer_id.end()); - input_names.push_back(copyLP.name); - - node_proto.set_input(0, srcName); - node_proto.set_output(0, copyLP.name); - addLayer(copyLP, node_proto); - } - node_proto.clear_input(); - for (int i = 0; i < input_names.size(); i++) - { - node_proto.add_input(input_names[i]); - } - layerParams.set("axis", broadcast_axes[0]); - layerParams.type = "Concat"; - node_proto.set_output(0, layerParams.name); - } - else - CV_Error(Error::StsNotImplemented, "Unsupported Expand op"); - } - else if (layer_type == "Reshape") - { - CV_Assert(node_proto.input_size() == 2 || layerParams.has("shape")); - - if (node_proto.input_size() == 2) { - Mat blob = getBlob(node_proto, 1); - CV_Assert(blob.type() == CV_32SC1); - - layerParams.set("dim", DictValue::arrayInt( - blob.ptr(), blob.total() )); - - if (layer_id.find(node_proto.input(0)) == layer_id.end()) { - std::vector inputs(1, getBlob(node_proto, 0)), outputs; - runLayer(layerParams, inputs, outputs); - addConstant(layerParams.name, outputs[0]); - return; - } - } - else { - DictValue shape = layerParams.get("shape"); - std::vector dim; - for (int j = 0; j < shape.size(); j++) { - dim.push_back(shape.getIntValue(j)); - } - - if (layer_id.find(node_proto.input(0)) == layer_id.end()) { - Mat input = getBlob(node_proto, 0); - Mat out = input.reshape(0, dim); - addConstant(layerParams.name, out); - return; - } - replaceLayerParam(layerParams, "shape", "dim"); - } - } - else if (layer_type == "Pad") - { - layerParams.type = "Padding"; - replaceLayerParam(layerParams, "mode", "type"); - if (node_proto.input_size() == 3 || node_proto.input_size() == 2) - { - // Paddings are in order begin0, begin1, .. beginN, end0, end1, ..., endN. - // We need to shuffle it to begin0, end0, begin1, end1, ... - Mat paddings = getBlob(node_proto, 1).reshape(1, 2); - paddings = paddings.t(); - layerParams.set("paddings", DictValue::arrayInt(paddings.ptr(), paddings.total())); - - if (node_proto.input_size() == 3) - { - Mat value = getBlob(node_proto, 2); - layerParams.set("value", value.ptr()[0]); - } - } - } - else if (layer_type == "Shape") - { - CV_Assert(node_proto.input_size() == 1); - IterShape_t shapeIt = outShapes.find(node_proto.input(0)); - CV_Assert(shapeIt != outShapes.end()); - const MatShape& inpShape = shapeIt->second; - - Mat shapeMat(inpShape.size(), 1, CV_32S); - for (int j = 0; j < inpShape.size(); ++j) - shapeMat.at(j) = inpShape[j]; - shapeMat.dims = 1; - - addConstant(layerParams.name, shapeMat); - return; - } - else if (layer_type == "Cast") - { - if (constBlobs.find(node_proto.input(0)) != constBlobs.end()) - { - Mat blob = getBlob(node_proto, 0); - int type; - switch (layerParams.get("to")) - { - case opencv_onnx::TensorProto_DataType_FLOAT: type = CV_32F; break; - case opencv_onnx::TensorProto_DataType_UINT8: type = CV_8U; break; - case opencv_onnx::TensorProto_DataType_UINT16: type = CV_16U; break; - case opencv_onnx::TensorProto_DataType_FLOAT16: type = CV_16S; break; - case opencv_onnx::TensorProto_DataType_INT8: - case opencv_onnx::TensorProto_DataType_INT16: - case opencv_onnx::TensorProto_DataType_INT32: - case opencv_onnx::TensorProto_DataType_INT64: type = CV_32S; break; - default: type = blob.type(); - } - Mat dst; - blob.convertTo(dst, type); - dst.dims = blob.dims; - addConstant(layerParams.name, dst); - return; - } - else - layerParams.type = "Identity"; - } - else if (layer_type == "ConstantOfShape" || layer_type == "ConstantFill") - { - int depth = CV_32F; - float fill_value; - if (!layerParams.blobs.empty()) - { - CV_Assert(!layerParams.has("value")); - depth = layerParams.blobs[0].depth(); - Mat floats; - layerParams.blobs[0].convertTo(floats, CV_32F); - fill_value = floats.at(0, 0); - } - else - fill_value = layerParams.get("value", 0); - - MatShape inpShape = getBlob(node_proto, 0); - for (int i = 0; i < inpShape.size(); i++) - CV_CheckGT(inpShape[i], 0, ""); - Mat tensor(inpShape.size(), &inpShape[0], depth, Scalar(fill_value)); - addConstant(layerParams.name, tensor); - return; - } - else if (layer_type == "Gather") - { - CV_Assert(node_proto.input_size() == 2); - Mat indexMat = getBlob(node_proto, 1); - CV_Assert_N(indexMat.type() == CV_32S, indexMat.total() == 1); - int index = indexMat.at(0); - int axis = layerParams.get("axis", 0); - - if ((constBlobs.find(node_proto.input(0)) != constBlobs.end())) - { - Mat input = getBlob(node_proto, 0); - Mat out; - std::vector ranges(input.dims, Range::all()); - ranges[axis] = Range(index, index + 1); - - out = input(ranges); - MatShape outShape = shape(out); - if (outShape.size() > 1) - { - outShape.erase(outShape.begin() + axis); - out.reshape(0, outShape); - } else { - out.dims = 1; - } - addConstant(layerParams.name, out); - return; - } - else - { - IterShape_t shapeIt = outShapes.find(node_proto.input(0)); - CV_Assert(shapeIt != outShapes.end()); - MatShape inpShape = shapeIt->second; - - LayerParams sliceLp; - sliceLp.type = "Slice"; - sliceLp.name = inpShape.size() > 1 ? layerParams.name + "/slice" : layerParams.name; - std::vector begin(inpShape.size(), 0); - std::vector end(inpShape.size(), -1); - begin[axis] = index; - end[axis] = index + 1; - - cv::dnn::DictValue paramBegin = cv::dnn::DictValue::arrayInt(begin.data(), begin.size()); - cv::dnn::DictValue paramEnd = cv::dnn::DictValue::arrayInt(end.data(), end.size()); - sliceLp.set("begin", paramBegin); - sliceLp.set("end", paramEnd); - sliceLp.set("has_dynamic_shapes", hasDynamicShapes); - - if (inpShape.size() > 1) - { - opencv_onnx::NodeProto proto; - proto.add_input(node_proto.input(0)); - proto.add_output(sliceLp.name); - addLayer(sliceLp, proto); - - inpShape.erase(inpShape.begin() + axis); - layerParams.type = "Reshape"; - layerParams.set("axis", 0); - layerParams.set("dim", DictValue::arrayInt(&inpShape[0], inpShape.size())); - if (hasDynamicShapes) - { - std::vector dynamicAxes; - std::vector inputIndices; - for (int index = 0; index < inpShape.size(); ++index) - dynamicAxes.push_back(index); - for (int index = 0; index < inpShape.size(); ++index) - inputIndices.push_back(index); - layerParams.set("dynamic_axes", DictValue::arrayInt(dynamicAxes.data(), dynamicAxes.size())); - layerParams.set("input_indices", DictValue::arrayInt(inputIndices.data(), inputIndices.size())); - } - node_proto.set_input(0, sliceLp.name); - } - else - { - layerParams = sliceLp; - } - } - } - else if (layer_type == "Concat") - { - bool hasVariableInps = false; - for (int i = 0; i < node_proto.input_size(); ++i) - { - if (layer_id.find(node_proto.input(i)) != layer_id.end()) - { - hasVariableInps = true; - break; - } - } - - if (!hasVariableInps) - { - std::vector inputs(node_proto.input_size()), concatenated; - // Due constant folding we can get inputs with different number of dimensions - // Insert the missing dimension to inputs - MatShape inputShape; - for (size_t i = 0; i < inputs.size(); ++i) - { - inputs[i] = getBlob(node_proto, i); - if (inputs[i].size.dims() > inputShape.size()) - { - inputShape = shape(inputs[i]); - } - } - - // Concat-1 has default value for axis is 1: https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Concat-1 - int axis = layerParams.get("axis", 1); - for (size_t i = 0; i < inputs.size(); ++i) - { - MatShape targetShape = inputShape; - targetShape[axis] = shape(inputs[i])[axis]; - CV_CheckEQ(total(targetShape), total(shape(inputs[i])), ""); - inputs[i] = inputs[i].reshape(0, targetShape); - } - runLayer(layerParams, inputs, concatenated); - - CV_Assert(concatenated.size() == 1); - addConstant(layerParams.name, concatenated[0]); - return; - } - else - { - for (int i = 0; i < node_proto.input_size(); ++i) - { - if (constBlobs.find(node_proto.input(i)) != constBlobs.end()) - { - LayerParams constParams; - constParams.name = node_proto.input(i); - constParams.type = "Const"; - constParams.blobs.push_back(getBlob(node_proto, i)); - - opencv_onnx::NodeProto proto; - proto.add_output(constParams.name); - addLayer(constParams, proto); - } - } - } - } - else if (layer_type == "Resize") - { - for (int i = 1; i < node_proto.input_size(); i++) - CV_Assert(layer_id.find(node_proto.input(i)) == layer_id.end()); - - if (layerParams.has("coordinate_transformation_mode")) - { - String interp_mode = layerParams.get("coordinate_transformation_mode"); - CV_Assert_N(interp_mode != "tf_crop_and_resize", interp_mode != "tf_half_pixel_for_nn"); - - layerParams.set("align_corners", interp_mode == "align_corners"); - if (layerParams.get("mode") == "linear") - { - layerParams.set("mode", interp_mode == "pytorch_half_pixel" ? - "opencv_linear" : "bilinear"); - } - } - if (layerParams.get("mode") == "linear" && framework_name == "pytorch") - layerParams.set("mode", "opencv_linear"); - - // input = [X, scales], [X, roi, scales] or [x, roi, scales, sizes] - int foundScaleId = hasDynamicShapes ? node_proto.input_size() - 1 - : node_proto.input_size() > 2 ? 2 : 1; - - Mat scales = getBlob(node_proto, foundScaleId); - if (scales.total() == 4) - { - layerParams.set("zoom_factor_y", scales.at(2)); - layerParams.set("zoom_factor_x", scales.at(3)); - } - else - { - const std::string& inputLast = node_proto.input(node_proto.input_size() - 1); - if (constBlobs.find(inputLast) != constBlobs.end()) - { - Mat shapes = getBlob(inputLast); - CV_CheckEQ(shapes.size[0], 4, ""); - CV_CheckEQ(shapes.size[1], 1, ""); - CV_CheckDepth(shapes.depth(), shapes.depth() == CV_32S || shapes.depth() == CV_32F, ""); - if (shapes.depth() == CV_32F) - shapes.convertTo(shapes, CV_32S); - layerParams.set("width", shapes.at(3)); - layerParams.set("height", shapes.at(2)); - } - } - replaceLayerParam(layerParams, "mode", "interpolation"); - } - else if (layer_type == "Upsample") - { - //fused from Resize Subgraph - if (layerParams.has("coordinate_transformation_mode")) - { - String interp_mode = layerParams.get("coordinate_transformation_mode"); - CV_Assert_N(interp_mode != "tf_crop_and_resize", interp_mode != "tf_half_pixel_for_nn"); - - layerParams.set("align_corners", interp_mode == "align_corners"); - if (layerParams.get("mode") == "linear") - { - layerParams.set("mode", interp_mode == "pytorch_half_pixel" ? - "opencv_linear" : "bilinear"); - } - } - if (layerParams.get("mode") == "linear" && framework_name == "pytorch") - layerParams.set("mode", "opencv_linear"); - - layerParams.type = "Resize"; - if (layerParams.has("scales")) - { - // Pytorch layer - DictValue scales = layerParams.get("scales"); - CV_Assert(scales.size() == 4); - layerParams.set("zoom_factor_y", scales.getIntValue(2)); - layerParams.set("zoom_factor_x", scales.getIntValue(3)); - } - else if (layerParams.has("height_scale") && layerParams.has("width_scale")) - { - // Caffe2 layer - replaceLayerParam(layerParams, "height_scale", "zoom_factor_y"); - replaceLayerParam(layerParams, "width_scale", "zoom_factor_x"); - } - else - { - // scales as input - const std::string& input1 = node_proto.input(1); - if (constBlobs.find(input1) != constBlobs.end()) - { - Mat scales = getBlob(input1); - CV_Assert(scales.total() == 4); - layerParams.set("zoom_factor_y", scales.at(2)); - layerParams.set("zoom_factor_x", scales.at(3)); - } - } - replaceLayerParam(layerParams, "mode", "interpolation"); - } - else if (layer_type == "SoftMax" || layer_type == "LogSoftmax") - { - layerParams.type = "Softmax"; - layerParams.set("log_softmax", layer_type == "LogSoftmax"); - } - else if (layer_type == "DetectionOutput") - { - CV_CheckEQ(node_proto.input_size(), 3, ""); - if (constBlobs.find(node_proto.input(2)) != constBlobs.end()) - { - Mat priors = getBlob(node_proto, 2); - - LayerParams constParams; - constParams.name = layerParams.name + "/priors"; - constParams.type = "Const"; - constParams.blobs.push_back(priors); - - opencv_onnx::NodeProto priorsProto; - priorsProto.add_output(constParams.name); - addLayer(constParams, priorsProto); - - node_proto.set_input(2, constParams.name); - } + CALL_MEMBER_FN(*this, iter->second)(layerParams, node_proto); } else { - for (int j = 0; j < node_proto.input_size(); j++) { - if (layer_id.find(node_proto.input(j)) == layer_id.end()) - layerParams.blobs.push_back(getBlob(node_proto, j)); - } + parseCustom(layerParams, node_proto); } - addLayer(layerParams, node_proto); } catch (const cv::Exception& e) { @@ -2001,6 +544,1669 @@ void ONNXImporter::handleNode(const opencv_onnx::NodeProto& node_proto_) } } +void ONNXImporter::parseMaxPool(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +{ + layerParams.type = "Pooling"; + layerParams.set("pool", "MAX"); + layerParams.set("ceil_mode", layerParams.has("pad_mode")); + addLayer(layerParams, node_proto); +} + +void ONNXImporter::parseAveragePool(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +{ + layerParams.type = "Pooling"; + layerParams.set("pool", "AVE"); + layerParams.set("ceil_mode", layerParams.has("pad_mode")); + layerParams.set("ave_pool_padded_area", framework_name == "pytorch"); + addLayer(layerParams, node_proto); +} + +// "GlobalAveragePool" "GlobalMaxPool" "ReduceMean" "ReduceSum" "ReduceMax" +void ONNXImporter::parseReduce(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto_) +{ + opencv_onnx::NodeProto node_proto = node_proto_; + const std::string& layer_type = node_proto.op_type(); + + CV_Assert(node_proto.input_size() == 1); + layerParams.type = "Pooling"; + String pool; + if (layer_type == "GlobalMaxPool" || layer_type == "ReduceMax") + pool = "MAX"; + else if (layer_type == "ReduceSum") + pool = "SUM"; + else + pool = "AVE"; + layerParams.set("pool", pool); + layerParams.set("global_pooling", !layerParams.has("axes")); + if (layerParams.has("axes") && (layer_type == "ReduceMean" || layer_type == "ReduceSum" || layer_type == "ReduceMax")) + { + MatShape inpShape = outShapes[node_proto.input(0)]; + DictValue axes = layerParams.get("axes"); + bool keepdims = layerParams.get("keepdims"); + MatShape targetShape; + std::vector shouldDelete(inpShape.size(), false); + for (int i = 0; i < axes.size(); i++) { + int axis = normalize_axis(axes.get(i), inpShape.size()); + shouldDelete[axis] = true; + } + for (int axis = 0; axis < inpShape.size(); ++axis){ + if (!shouldDelete[axis]) + targetShape.push_back(inpShape[axis]); + else if (keepdims) + targetShape.push_back(1); + } + + if (inpShape.size() == 3 && axes.size() <= 2) + { + int axis = normalize_axis(axes.get(0), inpShape.size()); + CV_CheckNE(axis, 0, ""); + + LayerParams reshapeLp; + reshapeLp.name = layerParams.name + "/reshape"; + reshapeLp.type = "Reshape"; + CV_Assert(layer_id.find(reshapeLp.name) == layer_id.end()); + reshapeLp.set("axis", 0); + reshapeLp.set("num_axes", 1); + int newShape[] = {1, -1}; + reshapeLp.set("dim", DictValue::arrayInt(&newShape[0], 2)); + + opencv_onnx::NodeProto proto; + proto.add_input(node_proto.input(0)); + proto.add_output(reshapeLp.name); + addLayer(reshapeLp, proto); + + LayerParams avgLp; + avgLp.name = layerParams.name + "/avg"; + avgLp.type = "Pooling"; + CV_Assert(layer_id.find(avgLp.name) == layer_id.end()); + avgLp.set("pool", pool); + if (axes.size() == 2) + { + CV_CheckEQ(normalize_axis(axes.get(0), inpShape.size()), 1, "Unsupported mode"); + CV_CheckEQ(normalize_axis(axes.get(1), inpShape.size()), 2, "Unsupported mode"); + avgLp.set("global_pooling", true); + } + else + { + avgLp.set(axis == 2 ? "global_pooling_w" : "global_pooling_h", true); + avgLp.set(axis == 2 ? "kernel_h" : "kernel_w", 1); + } + + node_proto.set_input(0, reshapeLp.name); + node_proto.set_output(0, avgLp.name); + addLayer(avgLp, node_proto); + } + else + { + if (inpShape.size() != 4 && inpShape.size() != 5) + CV_Error(Error::StsNotImplemented, "Unsupported input shape of " + layer_type + " operation."); + + CV_Assert(axes.size() <= inpShape.size() - 2); + std::vector kernel_size(inpShape.size() - 2, 1); + if (axes.size() == 1 && (normalize_axis(axes.get(0), inpShape.size()) <= 1)) + { + int axis = normalize_axis(axes.get(0), inpShape.size()); + MatShape newShape = inpShape; + newShape[axis + 1] = total(newShape, axis + 1); + newShape.resize(axis + 2); + newShape.insert(newShape.begin(), 2 - axis, 1); + + LayerParams reshapeLp; + reshapeLp.type = "Reshape"; + reshapeLp.name = layerParams.name + "/reshape"; + CV_Assert(layer_id.find(reshapeLp.name) == layer_id.end()); + reshapeLp.set("dim", DictValue::arrayInt(&newShape[0], newShape.size())); + + node_proto.set_output(0, reshapeLp.name); + addLayer(reshapeLp, node_proto); + + kernel_size.resize(2); + kernel_size[0] = inpShape[axis]; + node_proto.set_input(0, node_proto.output(0)); + } + else + { + for (int i = 0; i < axes.size(); i++) { + int axis = normalize_axis(axes.get(i), inpShape.size()); + CV_Assert_N(axis >= 2 + i, axis < inpShape.size()); + kernel_size[axis - 2] = inpShape[axis]; + } + } + + LayerParams poolLp = layerParams; + poolLp.name = layerParams.name + "/avg"; + CV_Assert(layer_id.find(poolLp.name) == layer_id.end()); + poolLp.set("kernel_size", DictValue::arrayInt(&kernel_size[0], kernel_size.size())); + + node_proto.set_output(0, poolLp.name); + addLayer(poolLp, node_proto); + } + + layerParams.type = "Reshape"; + layerParams.set("dim", DictValue::arrayInt(&targetShape[0], targetShape.size())); + + node_proto.set_input(0, node_proto.output(0)); + node_proto.set_output(0, layerParams.name); + } + else if (!layerParams.has("axes") && (layer_type == "ReduceMean" || layer_type == "ReduceSum" || layer_type == "ReduceMax")) + { + CV_CheckEQ(layerParams.get("keepdims"), 0, "layer only supports keepdims = false"); + LayerParams reshapeLp; + reshapeLp.name = layerParams.name + "/reshape"; + reshapeLp.type = "Reshape"; + CV_Assert(layer_id.find(reshapeLp.name) == layer_id.end()); + int newShape[] = {1, 1, 1, -1}; + reshapeLp.set("dim", DictValue::arrayInt(&newShape[0], 4)); + + opencv_onnx::NodeProto proto; + proto.add_input(node_proto.input(0)); + proto.add_output(reshapeLp.name); + addLayer(reshapeLp, proto); + + LayerParams poolLp = layerParams; + poolLp.name = layerParams.name + "/pool"; + CV_Assert(layer_id.find(poolLp.name) == layer_id.end()); + + node_proto.set_input(0, reshapeLp.name); + node_proto.set_output(0, poolLp.name); + addLayer(poolLp, node_proto); + + layerParams.type = "Reshape"; + int targetShape[] = {1}; + layerParams.set("dim", DictValue::arrayInt(&targetShape[0], 1)); + + node_proto.set_input(0, node_proto.output(0)); + node_proto.set_output(0, layerParams.name); + } + addLayer(layerParams, node_proto); +} + +void ONNXImporter::parseSlice(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +{ + int axis = 0; + std::vector begin; + std::vector end; + std::vector steps; + int inp_size = node_proto.input_size(); + + if (inp_size == 1) + { + if (layerParams.has("axes")) { + DictValue axes = layerParams.get("axes"); + for (int i = 1; i < axes.size(); ++i) { + CV_Assert(axes.get(i - 1) == axes.get(i) - 1); + } + axis = axes.get(0); + } + + DictValue starts = layerParams.get("starts"); + DictValue ends = layerParams.get("ends"); + CV_Assert(starts.size() == ends.size()); + + if (axis > 0) { + begin.resize(axis, 0); + end.resize(axis, -1); + } + for (int i = 0; i < starts.size(); ++i) + { + begin.push_back(starts.get(i)); + int finish = ends.get(i); + end.push_back((finish < 0) ? --finish : finish); // numpy doesn't include last dim + } + } else { // inp_size > 1 + CV_Assert(inp_size >= 3); + for (int i = 1; i < inp_size; i++) { + CV_Assert(constBlobs.find(node_proto.input(i)) != constBlobs.end()); + } + Mat start_blob = getBlob(node_proto, 1); + Mat end_blob = getBlob(node_proto, 2); + CV_Assert(start_blob.total() == end_blob.total()); + + if (inp_size > 3) { + Mat axes_blob = getBlob(node_proto, 3); + const int* axes = (int*)axes_blob.data; + for (int i = 1; i < axes_blob.total(); ++i) { + CV_Assert(axes[i - 1] == axes[i] - 1); + } + axis = axes[0]; + } + + const int* starts = start_blob.ptr(); + const int* ends = end_blob.ptr(); + if (axis > 0) { + begin.resize(axis, 0); + end.resize(axis, -1); + } + std::copy(starts, starts + start_blob.total(), std::back_inserter(begin)); + for (int i = 0; i < end_blob.total(); ++i) + { + int finish = ends[i]; + end.push_back((finish < 0) ? --finish : finish); // numpy doesn't include last dim + } + + if (inp_size == 5) { + CV_Assert(constBlobs.find(node_proto.input(4)) != constBlobs.end()); + Mat step_blob = getBlob(node_proto, 4); + const int* steps_ptr = step_blob.ptr(); + + if (axis > 0) + steps.resize(axis, 1); + + std::copy(steps_ptr, steps_ptr + step_blob.total(), std::back_inserter(steps)); + + // Very strange application for Slice op with tensor reversing. + // We just workaround it for 2d constants. + if (constBlobs.find(node_proto.input(0)) != constBlobs.end() && + axis == 0 && + start_blob.at(0) == -1 && step_blob.at(0) == -1 && + end_blob.at(0) == std::numeric_limits::min()) + { + Mat inp = getBlob(node_proto, 0); + if (inp.dims == 2) + { + Mat flipped; + flip(inp, flipped, 0); + addConstant(layerParams.name, flipped); + return; + } + } + } + } + layerParams.set("begin", DictValue::arrayInt(&begin[0], begin.size())); + layerParams.set("end", DictValue::arrayInt(&end[0], end.size())); + layerParams.set("axis", axis); + + if (!steps.empty()) + layerParams.set("steps", DictValue::arrayInt(&steps[0], steps.size())); + + if (constBlobs.find(node_proto.input(0)) != constBlobs.end()) + { + Mat inp = getBlob(node_proto, 0); + std::vector inputs, sliced; + inputs.push_back(inp); + runLayer(layerParams, inputs, sliced); + CV_Assert(sliced.size() == 1); + addConstant(layerParams.name, sliced[0]); + return; + } + addLayer(layerParams, node_proto); +} + +void ONNXImporter::parseSplit(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +{ + if (layerParams.has("split")) + { + DictValue splits = layerParams.get("split"); + const int numSplits = splits.size(); + CV_Assert(numSplits > 1); + + std::vector slicePoints(numSplits - 1, splits.get(0)); + for (int i = 1; i < splits.size() - 1; ++i) + { + slicePoints[i] = slicePoints[i - 1] + splits.get(i - 1); + } + layerParams.set("slice_point", DictValue::arrayInt(&slicePoints[0], slicePoints.size())); + } + else + { + layerParams.set("num_split", node_proto.output_size()); + } + layerParams.type = "Slice"; + addLayer(layerParams, node_proto); +} + +// "Add" "Sum" "Sub" +void ONNXImporter::parseBias(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto_) +{ + opencv_onnx::NodeProto node_proto = node_proto_; + const std::string& layer_type = node_proto.op_type(); + bool isSub = layer_type == "Sub"; + CV_CheckEQ(node_proto.input_size(), 2, ""); + bool is_const_0 = layer_id.find(node_proto.input(0)) == layer_id.end(); + bool is_const_1 = layer_id.find(node_proto.input(1)) == layer_id.end(); + if (is_const_0 && is_const_1) + { + Mat blob_0 = getBlob(node_proto, 0); + Mat blob_1 = getBlob(node_proto, 1); + CV_Assert(blob_0.size == blob_1.size); + Mat output = isSub ? (blob_0 - blob_1) : (blob_0 + blob_1); + addConstant(layerParams.name, output); + return; + } + else if (is_const_0 || is_const_1) + { + int const_blob_id = is_const_0 ? 0 : 1; + Mat blob = getBlob(node_proto, const_blob_id); + int blob_total = blob.total(); + if (blob_total == 1) { + layerParams.type = "Power"; + layerParams.set("shift", (isSub ? -1 : 1) * blob.ptr()[0]); + } + else { + MatShape inpShape = outShapes[node_proto.input(1 - const_blob_id)]; + if (shape(blob) == inpShape) + { + LayerParams constParams; + constParams.name = layerParams.name + "/const"; + constParams.type = "Const"; + constParams.blobs.push_back((isSub ? -1 : 1) * blob); + int id = dstNet.addLayer(constParams.name, constParams.type, constParams); + layer_id.insert(std::make_pair(constParams.name, LayerInfo(id, 0))); + outShapes[constParams.name] = shape(blob); + + layerParams.type = "Eltwise"; + node_proto.set_input(const_blob_id, constParams.name); + } + else + { + layerParams.type = "Scale"; + layerParams.set("bias_term", true); + int axis = 1; + for (int i = 0; i < graph_proto.initializer_size(); i++) + { + opencv_onnx::TensorProto tensor_proto = graph_proto.initializer(i); + if (tensor_proto.name() == node_proto.input(const_blob_id)) + { + axis = inpShape.size() - tensor_proto.dims_size(); + break; + } + } + layerParams.set("axis", axis); + blob = blob.reshape(1, 1); + layerParams.blobs.push_back((isSub ? -1 : 1) * blob); + } + } + } + else if (outShapes[node_proto.input(0)] == outShapes[node_proto.input(1)]) + { + layerParams.type = "Eltwise"; + if (isSub) + { + static float subCoeffs[] = {1.f, -1.f}; + layerParams.set("coeff", DictValue::arrayReal(subCoeffs, 2)); + } + } + else + { + if (isSub) + { + LayerParams powerParams; + powerParams.name = layerParams.name + "/neg"; + powerParams.type = "Power"; + powerParams.set("scale", -1); + + //Create Power layer + int id = dstNet.addLayer(powerParams.name, powerParams.type, powerParams); + //Connect to input + IterLayerId_t layerId = layer_id.find(node_proto.input(1)); + CV_Assert(layerId != layer_id.end()); + dstNet.connect(layerId->second.layerId, layerId->second.outputId, id, 0); + //Add shape + layer_id.insert(std::make_pair(powerParams.name, LayerInfo(id, 0))); + outShapes[powerParams.name] = outShapes[node_proto.input(1)]; + + //Replace input to Power + node_proto.set_input(1, powerParams.name); + } + layerParams.type = "Scale"; + layerParams.set("bias_term", true); + } + addLayer(layerParams, node_proto); +} + +void ONNXImporter::parsePow(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +{ + if (layer_id.find(node_proto.input(1)) != layer_id.end()) + CV_Error(Error::StsNotImplemented, "Unsupported Pow op with variable power"); + + Mat blob = getBlob(node_proto, 1); + if (blob.total() != 1) + CV_Error(Error::StsNotImplemented, "Pow op supports only scalar power"); + + blob.convertTo(blob, CV_32F); + layerParams.type = "Power"; + layerParams.set("power", blob.ptr()[0]); + addLayer(layerParams, node_proto); +} + +void ONNXImporter::parseMax(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +{ + layerParams.type = "Eltwise"; + layerParams.set("operation", "max"); + addLayer(layerParams, node_proto); +} + +void ONNXImporter::parseNeg(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +{ + layerParams.type = "Power"; + layerParams.set("scale", -1); + addLayer(layerParams, node_proto); +} + +void ONNXImporter::parseConstant(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +{ + CV_Assert(node_proto.input_size() == 0); + CV_Assert(layerParams.blobs.size() == 1); + addConstant(layerParams.name, layerParams.blobs[0]); +} + +void ONNXImporter::parseLSTM(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto_) +{ + opencv_onnx::NodeProto node_proto = node_proto_; + LayerParams lstmParams = layerParams; + lstmParams.name += "/lstm"; + + // https://pytorch.org/docs/stable/nn.html#lstm + CV_Assert(node_proto.input_size() == 7); + Mat Wx = getBlob(node_proto, 1); + Mat Wh = getBlob(node_proto, 2); + Mat b = getBlob(node_proto, 3); + Mat h0 = getBlob(node_proto, 5); + Mat c0 = getBlob(node_proto, 6); + + b = b.reshape(1, b.size[0]); + + const int numHidden = lstmParams.get("hidden_size"); + const int numDirs = Wx.size[0]; // Is 1 for forward only and 2 for bidirectional LSTM. + const int numFeatures = Wx.size[2]; + Mat bx = b.colRange(0, b.cols / 2); + Mat bh = b.colRange(b.cols / 2, b.cols); + b = bx + bh; + + // IFGO->IGFO + for (int k = 0; k < numDirs; ++k) + { + float* WxData = Wx.ptr(k); + float* WhData = Wh.ptr(k); + float* biasData = b.ptr(k); + for (int j = 0; j < numHidden; ++j) + { + for (int i = 0; i < numFeatures; ++i) + { + std::swap(WxData[(numHidden + j) * numFeatures + i], + WxData[(numHidden * 2 + j) * numFeatures + i]); + } + for (int i = 0; i < numHidden; ++i) + { + std::swap(WhData[(numHidden + j) * numHidden + i], + WhData[(numHidden * 2 + j) * numHidden + i]); + } + std::swap(biasData[numHidden + j], biasData[numHidden * 2 + j]); + } + } + Wx = Wx.reshape(1, Wx.size[0] * Wx.size[1]); + Wh = Wh.reshape(1, Wh.size[0] * Wh.size[1]); + h0 = h0.reshape(1, h0.size[0] * h0.size[1]); + c0 = c0.reshape(1, c0.size[0] * c0.size[1]); + + lstmParams.blobs.resize(5); + lstmParams.blobs[0] = Wh; + lstmParams.blobs[1] = Wx; + lstmParams.blobs[2] = b; + lstmParams.blobs[3] = h0; + lstmParams.blobs[4] = c0; + lstmParams.set("bidirectional", lstmParams.get("direction", "") == "bidirectional"); + + node_proto.set_output(0, lstmParams.name); // set different name so output shapes will be registered on that name + addLayer(lstmParams, node_proto); + + MatShape lstmShape = outShapes[node_proto.output(0)]; + + // Add fake 1 as it is done in ONNX + lstmShape.insert(lstmShape.begin() + 1, 1); + + layerParams.type = "Reshape"; + layerParams.set("dim", DictValue::arrayInt(&lstmShape[0], lstmShape.size())); + node_proto.set_input(0, lstmParams.name); // redirect input to LSTM + node_proto.set_output(0, layerParams.name); // keep origin LSTM's name + addLayer(layerParams, node_proto); +} + +void ONNXImporter::parseImageScaler(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +{ + const float scale = layerParams.has("scale") ? layerParams.get("scale") : 1.0f; + layerParams.erase("scale"); + + if (layerParams.has("bias")) + { + layerParams.type = "Scale"; + layerParams.blobs.push_back( + Mat(Size(1, layerParams.get("bias").size()), CV_32FC1, scale)); + + layerParams.set("bias_term", true); + Mat bias(1, layerParams.get("bias").size(), CV_32FC1); + for (int j = 0; j < bias.total(); j++) { + bias.at(0, j) = layerParams.get("bias").getRealValue(j); + } + layerParams.blobs.push_back(bias); + layerParams.erase("bias"); + } + else { + layerParams.set("scale", scale); + layerParams.type = "Power"; + } + addLayer(layerParams, node_proto); +} + +void ONNXImporter::parseClip(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +{ + layerParams.type = "ReLU6"; + replaceLayerParam(layerParams, "min", "min_value"); + replaceLayerParam(layerParams, "max", "max_value"); + addLayer(layerParams, node_proto); +} + +void ONNXImporter::parseLeakyRelu(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +{ + layerParams.type = "ReLU"; + replaceLayerParam(layerParams, "alpha", "negative_slope"); + addLayer(layerParams, node_proto); +} + +void ONNXImporter::parseRelu(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +{ + layerParams.type = "ReLU"; + addLayer(layerParams, node_proto); +} + +void ONNXImporter::parseElu(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +{ + layerParams.type = "ELU"; + addLayer(layerParams, node_proto); +} + +void ONNXImporter::parseTanh(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +{ + layerParams.type = "TanH"; + addLayer(layerParams, node_proto); +} + +void ONNXImporter::parsePRelu(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +{ + layerParams.type = "PReLU"; + layerParams.blobs.push_back(getBlob(node_proto, 1)); + addLayer(layerParams, node_proto); +} + +void ONNXImporter::parseLRN(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +{ + replaceLayerParam(layerParams, "size", "local_size"); + addLayer(layerParams, node_proto); +} + +void ONNXImporter::parseInstanceNormalization(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto_) +{ + opencv_onnx::NodeProto node_proto = node_proto_; + if (node_proto.input_size() != 3) + CV_Error(Error::StsNotImplemented, + "Expected input, scale, bias"); + + layerParams.blobs.resize(4); + layerParams.blobs[2] = getBlob(node_proto, 1); // weightData + layerParams.blobs[3] = getBlob(node_proto, 2); // biasData + layerParams.set("has_bias", true); + layerParams.set("has_weight", true); + + // Get number of channels in input + int size = layerParams.blobs[2].total(); + layerParams.blobs[0] = Mat::zeros(size, 1, CV_32F); // mean + layerParams.blobs[1] = Mat::ones(size, 1, CV_32F); // std + + LayerParams mvnParams; + mvnParams.name = layerParams.name + "/MVN"; + mvnParams.type = "MVN"; + mvnParams.set("eps", layerParams.get("epsilon")); + layerParams.erase("epsilon"); + + //Create MVN layer + int id = dstNet.addLayer(mvnParams.name, mvnParams.type, mvnParams); + //Connect to input + IterLayerId_t layerId = layer_id.find(node_proto.input(0)); + CV_Assert(layerId != layer_id.end()); + dstNet.connect(layerId->second.layerId, layerId->second.outputId, id, 0); + //Add shape + layer_id.insert(std::make_pair(mvnParams.name, LayerInfo(id, 0))); + outShapes[mvnParams.name] = outShapes[node_proto.input(0)]; + + //Replace Batch Norm's input to MVN + node_proto.set_input(0, mvnParams.name); + layerParams.type = "BatchNorm"; + addLayer(layerParams, node_proto); +} + +void ONNXImporter::parseBatchNormalization(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +{ + if (node_proto.input_size() != 5) + CV_Error(Error::StsNotImplemented, + "Expected input, scale, bias, mean and var"); + + layerParams.type = "BatchNorm"; + replaceLayerParam(layerParams, "epsilon", "eps"); + replaceLayerParam(layerParams, "spatial", "use_global_stats"); + + Mat meanData = getBlob(node_proto, 3); + Mat stdData = getBlob(node_proto, 4); + + layerParams.blobs.push_back(meanData); + layerParams.blobs.push_back(stdData); + + if (!node_proto.input(1).empty()) { + layerParams.set("has_weight", true); + layerParams.blobs.push_back(getBlob(node_proto, 1)); // weightData + } else { + layerParams.set("has_weight", false); + } + + if (!node_proto.input(2).empty()) { + layerParams.set("has_bias", true); + layerParams.blobs.push_back(getBlob(node_proto, 2)); // biasData + } else { + layerParams.set("has_bias", false); + } + addLayer(layerParams, node_proto); +} + +void ONNXImporter::parseGemm(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +{ + CV_Assert(node_proto.input_size() >= 2); + layerParams.type = "InnerProduct"; + Mat weights = getBlob(node_proto, 1); + int ind_num_out = 0; + if (layerParams.has("transB") && !layerParams.get("transB")) { + transpose(weights, weights); + ind_num_out = 1; + } + layerParams.blobs.push_back(weights); + + if (node_proto.input_size() == 3) { + Mat bias = getBlob(node_proto, 2); + layerParams.blobs.push_back(bias); + } + if (constBlobs.find(node_proto.input(0)) != constBlobs.end()) + { + Mat inputBuf = getBlob(node_proto, 0); + + LayerParams constParams; + constParams.name = node_proto.input(0); + constParams.type = "Const"; + constParams.blobs.push_back(inputBuf); + + opencv_onnx::NodeProto proto; + proto.add_output(constParams.name); + addLayer(constParams, proto); + } + + layerParams.set("num_output", layerParams.blobs[0].size[ind_num_out]); + layerParams.set("bias_term", node_proto.input_size() == 3); + addLayer(layerParams, node_proto); +} + +void ONNXImporter::parseMatMul(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +{ + CV_Assert(node_proto.input_size() == 2); + layerParams.type = "InnerProduct"; + layerParams.set("bias_term", false); + CV_Assert(constBlobs.find(node_proto.input(0)) == constBlobs.end()); + int firstInpDims = outShapes[node_proto.input(0)].size(); + int secondInpDims; + + if (constBlobs.find(node_proto.input(1)) != constBlobs.end()) + { + Mat blob = getBlob(node_proto, 1); + secondInpDims = blob.dims; + layerParams.blobs.push_back(blob.t()); + layerParams.set("num_output", layerParams.blobs[0].size[0]); + } else { + secondInpDims = outShapes[node_proto.input(1)].size(); + } + layerParams.set("axis", firstInpDims - secondInpDims + 1); + addLayer(layerParams, node_proto); +} + +// "Mul" "Div" +void ONNXImporter::parseMul(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto_) +{ + opencv_onnx::NodeProto node_proto = node_proto_; + const std::string& layer_type = node_proto.op_type(); + CV_Assert(node_proto.input_size() == 2); + + bool isDiv = layer_type == "Div"; + int constId = -1; + bool haveVariables = false; + for (int i = 0; i < 2; ++i) + { + if (constBlobs.find(node_proto.input(i)) != constBlobs.end()) + constId = i; + else + haveVariables = true; + } + if (constId != -1 && haveVariables) + { + Mat blob = getBlob(node_proto, constId); + blob = blob.reshape(1, 1); + if (blob.total() == 1) { + float blob_value = blob.ptr()[0]; + float coeff = isDiv ? 1.0 / blob_value : blob_value; + layerParams.set("scale", coeff); + layerParams.type = "Power"; + } + else { + if (isDiv) + divide(1.0, blob, blob); + layerParams.blobs.push_back(blob); + layerParams.type = "Scale"; + } + } + else if (!haveVariables) + { + Mat inp0 = getBlob(node_proto, 0); + Mat inp1 = getBlob(node_proto, 1); + + if (inp0.size != inp1.size && (inp0.total() != 1 || inp1.total() != 1)) + CV_Error_(Error::StsNotImplemented, ("Different shapes case is not supported with constant inputs: %s", layer_type.c_str())); + + if (inp0.total() == 1 && inp1.total() == 1 && inp0.dims != inp1.dims) + { + if (inp0.dims < inp1.dims) + { + inp0 = inp0.reshape(1, inp1.dims, inp1.size); + inp0.dims = inp1.dims; + } + else + { + inp1 = inp1.reshape(1, inp0.dims, inp0.size); + inp1.dims = inp0.dims; + } + } + + Mat out; + if (inp0.total() != inp1.total()) + { + if (inp0.total() == 1) + { + float inp0_value = inp0.ptr()[0]; + float coeff = isDiv ? 1.0 / inp0_value : inp0_value; + multiply(inp1, coeff, out); + } + else + { + float inp1_value = inp1.ptr()[0]; + float coeff = isDiv ? 1.0 / inp1_value : inp1_value; + multiply(inp0, coeff, out); + } + + } + else + { + out = isDiv ? inp0 / inp1 : inp0.mul(inp1); + } + + if (inp0.dims == 1 && inp1.dims == 1) + out.dims = 1; // to workaround dims == 1 + addConstant(layerParams.name, out); + return; + } + else if (outShapes[node_proto.input(0)] == outShapes[node_proto.input(1)]) + { + layerParams.type = "Eltwise"; + layerParams.set("operation", isDiv ? "div" : "prod"); + } + else + { + // Scale layer allocate output with the first input shape + if (total(outShapes[node_proto.input(0)]) < total(outShapes[node_proto.input(1)])) + { + opencv_onnx::NodeProto proto; + proto.add_input(node_proto.input(1)); + proto.add_input(node_proto.input(0)); + proto.add_output(layerParams.name); + node_proto = proto; + } + + if (isDiv) + { + LayerParams powerParams; + powerParams.name = layerParams.name + "/inv"; + powerParams.type = "Power"; + powerParams.set("power", -1); + + //Create Power layer + int id = dstNet.addLayer(powerParams.name, powerParams.type, powerParams); + //Connect to input + IterLayerId_t layerId = layer_id.find(node_proto.input(1)); + CV_Assert(layerId != layer_id.end()); + dstNet.connect(layerId->second.layerId, layerId->second.outputId, id, 0); + //Add shape + layer_id.insert(std::make_pair(powerParams.name, LayerInfo(id, 0))); + outShapes[powerParams.name] = outShapes[node_proto.input(1)]; + + //Replace input to Power + node_proto.set_input(1, powerParams.name); + } + layerParams.type = "Scale"; + } + addLayer(layerParams, node_proto); +} + +void ONNXImporter::parseConv(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto_) +{ + opencv_onnx::NodeProto node_proto = node_proto_; + CV_Assert(node_proto.input_size() >= 2); + layerParams.type = "Convolution"; + for (int j = 1; j < node_proto.input_size(); j++) { + if (constBlobs.find(node_proto.input(j)) != constBlobs.end()) + { + layerParams.blobs.push_back(getBlob(node_proto, j)); + } + } + int outCn = layerParams.blobs.empty() ? outShapes[node_proto.input(1)][0] : layerParams.blobs[0].size[0]; + layerParams.set("num_output", outCn); + + // Check for asymmetric padding in Conv2D + if (layerParams.has("pad")) + { + bool asymmetricPadding = false; + DictValue pads = layerParams.get("pad"); + const int dims = pads.size() / 2; + for (int i = 0; i < dims; ++i) + { + if (pads.get(i) != pads.get(i + dims)) + { + asymmetricPadding = true; + break; + } + } + if (asymmetricPadding && pads.size() == 4) // [pad_t, pad_l, pad_b, pad_r] + { + layerParams.erase("pad"); + // No paddings required for N, C axis + std::vector paddings(4, 0); + // Add paddings for H, W axis + for (int i = 0; i < dims; ++i) + { + paddings.push_back(pads.get(i)); + paddings.push_back(pads.get(dims + i)); + } + LayerParams padLp; + padLp.name = layerParams.name + "/pad"; + padLp.type = "Padding"; + padLp.set("paddings", DictValue::arrayInt(&paddings[0], paddings.size())); + + opencv_onnx::NodeProto proto; + proto.add_input(node_proto.input(0)); + proto.add_output(padLp.name); + + addLayer(padLp, proto); + node_proto.set_input(0, padLp.name); + } + } + addLayer(layerParams, node_proto); +} + +void ONNXImporter::parseConvTranspose(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +{ + CV_Assert(node_proto.input_size() >= 2); + layerParams.type = "Deconvolution"; + for (int j = 1; j < node_proto.input_size(); j++) { + layerParams.blobs.push_back(getBlob(node_proto, j)); + } + layerParams.set("num_output", layerParams.blobs[0].size[1] * layerParams.get("group", 1)); + layerParams.set("bias_term", node_proto.input_size() == 3); + + if (!layerParams.has("kernel_size")) + CV_Error(Error::StsNotImplemented, + "Required attribute 'kernel_size' is not present."); + + if (layerParams.has("output_shape")) + { + const DictValue& outShape = layerParams.get("output_shape"); + DictValue strides = layerParams.get("stride"); + DictValue kernel = layerParams.get("kernel_size"); + + String padMode; + std::vector adjust_pads; + if (layerParams.has("pad_mode")) + { + padMode = toUpperCase(layerParams.get("pad_mode")); + if (padMode != "SAME" && padMode != "VALID") + CV_Error(Error::StsError, "Unsupported padding mode " + padMode); + + for (int i = 0; i < strides.size(); i++) + { + int sz = outShape.get(2 + i); + int stride = strides.get(i); + adjust_pads.push_back(padMode == "SAME"? (sz - 1) % stride : + (sz - kernel.get(i)) % stride); + } + layerParams.set("adj", DictValue::arrayInt(&adjust_pads[0], adjust_pads.size())); + } + } + else if (layerParams.has("output_padding")) + { + replaceLayerParam(layerParams, "output_padding", "adj"); + } + addLayer(layerParams, node_proto); +} + +void ONNXImporter::parseTranspose(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +{ + layerParams.type = "Permute"; + replaceLayerParam(layerParams, "perm", "order"); + + CV_Assert(node_proto.input_size() == 1); + if (constBlobs.find(node_proto.input(0)) != constBlobs.end()) + { + std::vector inputs(1, getBlob(node_proto, 0)), transposed; + runLayer(layerParams, inputs, transposed); + CV_Assert(transposed.size() == 1); + addConstant(layerParams.name, transposed[0]); + return; + } + addLayer(layerParams, node_proto); +} + +void ONNXImporter::parseSqueeze(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +{ + CV_Assert_N(node_proto.input_size() == 1, layerParams.has("axes")); + DictValue axes_dict = layerParams.get("axes"); + MatShape inpShape = outShapes[node_proto.input(0)]; + + std::vector maskedAxes(inpShape.size(), false); + for (int i = 0; i < axes_dict.size(); ++i) + { + int axis = axes_dict.getIntValue(i); + CV_CheckLE(axis, static_cast(inpShape.size()), "Squeeze axis"); + maskedAxes[axis] = inpShape[axis] == 1; + } + MatShape outShape; + for (int i = 0; i < inpShape.size(); ++i) + { + if (!maskedAxes[i]) + outShape.push_back(inpShape[i]); + } + if (outShape.size() != inpShape.size()) + { + layerParams.type = "Reshape"; + layerParams.set("dim", DictValue::arrayInt(&outShape[0], outShape.size())); + if (hasDynamicShapes) + { + std::vector dynamicAxes; + std::vector inputIndices; + for (int index = 0; index < inpShape.size(); ++index) + { + if (!maskedAxes[index]) + inputIndices.push_back(index); + } + for (int index = 0; index < outShape.size(); ++index) + dynamicAxes.push_back(index); + layerParams.set("dynamic_axes", DictValue::arrayInt(dynamicAxes.data(), dynamicAxes.size())); + layerParams.set("input_indices", DictValue::arrayInt(inputIndices.data(), inputIndices.size())); + } + } + else + layerParams.type = "Identity"; + + if (constBlobs.find(node_proto.input(0)) != constBlobs.end()) + { + Mat inp = getBlob(node_proto, 0); + Mat out = inp.reshape(1, outShape); + out.dims = outShape.size(); // to workaround dims == 1 + addConstant(layerParams.name, out); + return; + } + addLayer(layerParams, node_proto); +} + +void ONNXImporter::parseFlatten(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +{ + CV_CheckEQ(node_proto.input_size(), 1, ""); + if (constBlobs.find(node_proto.input(0)) != constBlobs.end()) + { + Mat input = getBlob(node_proto, 0); + int axis = normalize_axis(layerParams.get("axis", 1), input.dims); + + std::vector out_size(&input.size[0], &input.size[0] + axis); + out_size.push_back(input.total(axis)); + Mat output = input.reshape(1, out_size); + addConstant(layerParams.name, output); + return; + } + addLayer(layerParams, node_proto); +} + +void ONNXImporter::parseUnsqueeze(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +{ + CV_Assert(node_proto.input_size() == 1); + DictValue axes = layerParams.get("axes"); + if (constBlobs.find(node_proto.input(0)) != constBlobs.end()) + { + // Constant input. + Mat input = getBlob(node_proto, 0); + + std::vector dims; + for (int j = 0; j < input.dims; j++) { + dims.push_back(input.size[j]); + } + CV_Assert(axes.getIntValue(axes.size()-1) <= dims.size()); + for (int j = 0; j < axes.size(); j++) { + dims.insert(dims.begin() + axes.getIntValue(j), 1); + } + + Mat out = input.reshape(0, dims); + addConstant(layerParams.name, out); + return; + } + + // Variable input. + if (axes.size() != 1) + CV_Error(Error::StsNotImplemented, "Multidimensional unsqueeze"); + + MatShape inpShape = outShapes[node_proto.input(0)]; + int axis = axes.getIntValue(0); + CV_Assert(0 <= axis && axis <= inpShape.size()); + std::vector outShape = inpShape; + outShape.insert(outShape.begin() + axis, 1); + layerParams.type = "Reshape"; + layerParams.set("dim", DictValue::arrayInt(&outShape[0], outShape.size())); + if (hasDynamicShapes) + { + std::vector dynamicAxes; + std::vector inputIndices; + for (int index = 0; index < outShape.size(); ++index) { + if (index != axis) + dynamicAxes.push_back(index); + } + for (int index = 0; index < inpShape.size(); ++index) + inputIndices.push_back(index); + layerParams.set("dynamic_axes", DictValue::arrayInt(dynamicAxes.data(), dynamicAxes.size())); + layerParams.set("input_indices", DictValue::arrayInt(inputIndices.data(), inputIndices.size())); + } + addLayer(layerParams, node_proto); +} + +void ONNXImporter::parseExpand(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto_) +{ + opencv_onnx::NodeProto node_proto = node_proto_; + CV_CheckEQ(node_proto.input_size(), 2, ""); + const std::string& input0 = node_proto.input(0); + const std::string& input1 = node_proto.input(1); + Mat newShapeMat = getBlob(input1); + MatShape targetShape(newShapeMat.ptr(), newShapeMat.ptr() + newShapeMat.total()); + + MatShape inpShape; + bool haveVariables = constBlobs.find(input0) == constBlobs.end(); + if (haveVariables) + { + IterShape_t shapeIt = outShapes.find(input0); + CV_Assert(shapeIt != outShapes.end()); + inpShape = shapeIt->second; + } + else + { + inpShape = shape(getBlob(input0)); + } + + String srcName = input0; + // Unsqueeze and repeat along new axis + if (targetShape.size() == inpShape.size() + 1) + { + for (int i = 0; i < targetShape.size(); i++) + { + if (targetShape[i] == -1 && i < inpShape.size()) + targetShape[i] = inpShape[i]; + else if (i < inpShape.size() && targetShape[i] != inpShape[i]) + inpShape.insert(inpShape.begin() + i, 1); + } + if (haveVariables) + { + LayerParams reshapeLp; + reshapeLp.name = layerParams.name + "/reshape"; + reshapeLp.type = "Reshape"; + CV_Assert(layer_id.find(reshapeLp.name) == layer_id.end()); + reshapeLp.set("dim", DictValue::arrayInt(&inpShape[0], inpShape.size())); + + opencv_onnx::NodeProto proto; + proto.add_input(node_proto.input(0)); + proto.add_output(reshapeLp.name); + addLayer(reshapeLp, proto); + srcName = reshapeLp.name; + } + } + CV_CheckEQ(inpShape.size(), targetShape.size(), "Unsupported Expand op with different dims"); + + std::vector broadcast_axes; + for (int i = 0; i < targetShape.size(); i++) + { + if (targetShape[i] != inpShape[i]) + { + if (inpShape[i] == 1) + broadcast_axes.push_back(i); + else + CV_Error(Error::StsError, format("Could not be broadcast by axis: %d", i)); + } + } + + if (!haveVariables) + { + if (broadcast_axes.size() != 1) + CV_Error(Error::StsNotImplemented, "Expand op doesn't support multiple axes for constant input"); + + Mat input = getBlob(node_proto, 0); + input = input.reshape(0, total(inpShape, 0, broadcast_axes[0])); + Mat output = cv::repeat(input, 1, targetShape[broadcast_axes[0]]); + output = output.reshape(0, targetShape); + addConstant(layerParams.name, output); + return; + } + + if (broadcast_axes.size() == 2 && + broadcast_axes[0] == broadcast_axes[1] - 1 && broadcast_axes[1] == inpShape.size() - 1) + { + LayerParams constParams; + constParams.name = layerParams.name + "/const"; + CV_Assert(layer_id.find(constParams.name) == layer_id.end()); + constParams.type = "Const"; + + Mat inp = Mat::ones(newShapeMat.total(), newShapeMat.ptr(), CV_32F); + constParams.blobs.push_back(inp); + + opencv_onnx::NodeProto proto; + proto.add_output(constParams.name); + addLayer(constParams, proto); + + layerParams.type = "Scale"; + layerParams.set("bias_term", false); + node_proto.set_input(0, constParams.name); + node_proto.set_input(1, srcName); + } + else if (broadcast_axes.size() == 1 && broadcast_axes[0] <= 1) + { + String base_name = layerParams.name + "/copy_"; + std::vector input_names; + for (int j = 0; j < targetShape[broadcast_axes[0]]; j++) + { + std::ostringstream ss; + ss << j; + LayerParams copyLP; + copyLP.name = base_name + ss.str(); + copyLP.type = "Identity"; + CV_Assert(layer_id.find(copyLP.name) == layer_id.end()); + input_names.push_back(copyLP.name); + + node_proto.set_input(0, srcName); + node_proto.set_output(0, copyLP.name); + addLayer(copyLP, node_proto); + } + node_proto.clear_input(); + for (int i = 0; i < input_names.size(); i++) + { + node_proto.add_input(input_names[i]); + } + layerParams.set("axis", broadcast_axes[0]); + layerParams.type = "Concat"; + node_proto.set_output(0, layerParams.name); + } + else + CV_Error(Error::StsNotImplemented, "Unsupported Expand op"); + addLayer(layerParams, node_proto); +} + +void ONNXImporter::parseReshape(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +{ + CV_Assert(node_proto.input_size() == 2 || layerParams.has("shape")); + + if (node_proto.input_size() == 2) { + Mat blob = getBlob(node_proto, 1); + CV_Assert(blob.type() == CV_32SC1); + + layerParams.set("dim", DictValue::arrayInt( + blob.ptr(), blob.total() )); + + if (layer_id.find(node_proto.input(0)) == layer_id.end()) { + std::vector inputs(1, getBlob(node_proto, 0)), outputs; + runLayer(layerParams, inputs, outputs); + addConstant(layerParams.name, outputs[0]); + return; + } + } + else { + DictValue shape = layerParams.get("shape"); + std::vector dim; + for (int j = 0; j < shape.size(); j++) { + dim.push_back(shape.getIntValue(j)); + } + + if (layer_id.find(node_proto.input(0)) == layer_id.end()) { + Mat input = getBlob(node_proto, 0); + Mat out = input.reshape(0, dim); + addConstant(layerParams.name, out); + return; + } + replaceLayerParam(layerParams, "shape", "dim"); + } + addLayer(layerParams, node_proto); +} + +void ONNXImporter::parsePad(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +{ + layerParams.type = "Padding"; + replaceLayerParam(layerParams, "mode", "type"); + if (node_proto.input_size() == 3 || node_proto.input_size() == 2) + { + // Paddings are in order begin0, begin1, .. beginN, end0, end1, ..., endN. + // We need to shuffle it to begin0, end0, begin1, end1, ... + Mat paddings = getBlob(node_proto, 1).reshape(1, 2); + paddings = paddings.t(); + layerParams.set("paddings", DictValue::arrayInt(paddings.ptr(), paddings.total())); + + if (node_proto.input_size() == 3) + { + Mat value = getBlob(node_proto, 2); + layerParams.set("value", value.ptr()[0]); + } + } + addLayer(layerParams, node_proto); +} + +void ONNXImporter::parseShape(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +{ + CV_Assert(node_proto.input_size() == 1); + IterShape_t shapeIt = outShapes.find(node_proto.input(0)); + CV_Assert(shapeIt != outShapes.end()); + const MatShape& inpShape = shapeIt->second; + + Mat shapeMat(inpShape.size(), 1, CV_32S); + for (int j = 0; j < inpShape.size(); ++j) + shapeMat.at(j) = inpShape[j]; + shapeMat.dims = 1; + + addConstant(layerParams.name, shapeMat); +} + +void ONNXImporter::parseCast(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +{ + if (constBlobs.find(node_proto.input(0)) != constBlobs.end()) + { + Mat blob = getBlob(node_proto, 0); + int type; + switch (layerParams.get("to")) + { + case opencv_onnx::TensorProto_DataType_FLOAT: type = CV_32F; break; + case opencv_onnx::TensorProto_DataType_UINT8: type = CV_8U; break; + case opencv_onnx::TensorProto_DataType_UINT16: type = CV_16U; break; + case opencv_onnx::TensorProto_DataType_FLOAT16: type = CV_16S; break; + case opencv_onnx::TensorProto_DataType_INT8: + case opencv_onnx::TensorProto_DataType_INT16: + case opencv_onnx::TensorProto_DataType_INT32: + case opencv_onnx::TensorProto_DataType_INT64: type = CV_32S; break; + default: type = blob.type(); + } + Mat dst; + blob.convertTo(dst, type); + dst.dims = blob.dims; + addConstant(layerParams.name, dst); + return; + } + else + layerParams.type = "Identity"; + addLayer(layerParams, node_proto); +} + +// "ConstantOfShape" "ConstantFill" +void ONNXImporter::parseConstantFill(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +{ + int depth = CV_32F; + float fill_value; + if (!layerParams.blobs.empty()) + { + CV_Assert(!layerParams.has("value")); + depth = layerParams.blobs[0].depth(); + Mat floats; + layerParams.blobs[0].convertTo(floats, CV_32F); + fill_value = floats.at(0, 0); + } + else + fill_value = layerParams.get("value", 0); + + MatShape inpShape = getBlob(node_proto, 0); + for (int i = 0; i < inpShape.size(); i++) + CV_CheckGT(inpShape[i], 0, ""); + Mat tensor(inpShape.size(), &inpShape[0], depth, Scalar(fill_value)); + addConstant(layerParams.name, tensor); +} + +void ONNXImporter::parseGather(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto_) +{ + opencv_onnx::NodeProto node_proto = node_proto_; + CV_Assert(node_proto.input_size() == 2); + Mat indexMat = getBlob(node_proto, 1); + CV_Assert_N(indexMat.type() == CV_32S, indexMat.total() == 1); + int index = indexMat.at(0); + int axis = layerParams.get("axis", 0); + + if ((constBlobs.find(node_proto.input(0)) != constBlobs.end())) + { + Mat input = getBlob(node_proto, 0); + Mat out; + std::vector ranges(input.dims, Range::all()); + ranges[axis] = Range(index, index + 1); + + out = input(ranges); + MatShape outShape = shape(out); + if (outShape.size() > 1) + { + outShape.erase(outShape.begin() + axis); + out.reshape(0, outShape); + } else { + out.dims = 1; + } + addConstant(layerParams.name, out); + return; + } + else + { + IterShape_t shapeIt = outShapes.find(node_proto.input(0)); + CV_Assert(shapeIt != outShapes.end()); + MatShape inpShape = shapeIt->second; + + LayerParams sliceLp; + sliceLp.type = "Slice"; + sliceLp.name = inpShape.size() > 1 ? layerParams.name + "/slice" : layerParams.name; + std::vector begin(inpShape.size(), 0); + std::vector end(inpShape.size(), -1); + begin[axis] = index; + end[axis] = index + 1; + + cv::dnn::DictValue paramBegin = cv::dnn::DictValue::arrayInt(begin.data(), begin.size()); + cv::dnn::DictValue paramEnd = cv::dnn::DictValue::arrayInt(end.data(), end.size()); + sliceLp.set("begin", paramBegin); + sliceLp.set("end", paramEnd); + sliceLp.set("has_dynamic_shapes", hasDynamicShapes); + + if (inpShape.size() > 1) + { + opencv_onnx::NodeProto proto; + proto.add_input(node_proto.input(0)); + proto.add_output(sliceLp.name); + addLayer(sliceLp, proto); + + inpShape.erase(inpShape.begin() + axis); + layerParams.type = "Reshape"; + layerParams.set("axis", 0); + layerParams.set("dim", DictValue::arrayInt(&inpShape[0], inpShape.size())); + if (hasDynamicShapes) + { + std::vector dynamicAxes; + std::vector inputIndices; + for (int index = 0; index < inpShape.size(); ++index) + dynamicAxes.push_back(index); + for (int index = 0; index < inpShape.size(); ++index) + inputIndices.push_back(index); + layerParams.set("dynamic_axes", DictValue::arrayInt(dynamicAxes.data(), dynamicAxes.size())); + layerParams.set("input_indices", DictValue::arrayInt(inputIndices.data(), inputIndices.size())); + } + node_proto.set_input(0, sliceLp.name); + } + else + { + layerParams = sliceLp; + } + } + addLayer(layerParams, node_proto); +} + +void ONNXImporter::parseConcat(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +{ + bool hasVariableInps = false; + for (int i = 0; i < node_proto.input_size(); ++i) + { + if (layer_id.find(node_proto.input(i)) != layer_id.end()) + { + hasVariableInps = true; + break; + } + } + + if (!hasVariableInps) + { + std::vector inputs(node_proto.input_size()), concatenated; + // Due constant folding we can get inputs with different number of dimensions + // Insert the missing dimension to inputs + MatShape inputShape; + for (size_t i = 0; i < inputs.size(); ++i) + { + inputs[i] = getBlob(node_proto, i); + if (inputs[i].size.dims() > inputShape.size()) + { + inputShape = shape(inputs[i]); + } + } + + // Concat-1 has default value for axis is 1: https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Concat-1 + int axis = layerParams.get("axis", 1); + for (size_t i = 0; i < inputs.size(); ++i) + { + MatShape targetShape = inputShape; + targetShape[axis] = shape(inputs[i])[axis]; + CV_CheckEQ(total(targetShape), total(shape(inputs[i])), ""); + inputs[i] = inputs[i].reshape(0, targetShape); + } + runLayer(layerParams, inputs, concatenated); + + CV_Assert(concatenated.size() == 1); + addConstant(layerParams.name, concatenated[0]); + return; + } + else + { + for (int i = 0; i < node_proto.input_size(); ++i) + { + if (constBlobs.find(node_proto.input(i)) != constBlobs.end()) + { + LayerParams constParams; + constParams.name = node_proto.input(i); + constParams.type = "Const"; + constParams.blobs.push_back(getBlob(node_proto, i)); + + opencv_onnx::NodeProto proto; + proto.add_output(constParams.name); + addLayer(constParams, proto); + } + } + } + addLayer(layerParams, node_proto); +} + +void ONNXImporter::parseResize(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +{ + for (int i = 1; i < node_proto.input_size(); i++) + CV_Assert(layer_id.find(node_proto.input(i)) == layer_id.end()); + + if (layerParams.has("coordinate_transformation_mode")) + { + String interp_mode = layerParams.get("coordinate_transformation_mode"); + CV_Assert_N(interp_mode != "tf_crop_and_resize", interp_mode != "tf_half_pixel_for_nn"); + + layerParams.set("align_corners", interp_mode == "align_corners"); + if (layerParams.get("mode") == "linear") + { + layerParams.set("mode", interp_mode == "pytorch_half_pixel" ? + "opencv_linear" : "bilinear"); + } + } + if (layerParams.get("mode") == "linear" && framework_name == "pytorch") + layerParams.set("mode", "opencv_linear"); + + // input = [X, scales], [X, roi, scales] or [x, roi, scales, sizes] + int foundScaleId = hasDynamicShapes ? node_proto.input_size() - 1 + : node_proto.input_size() > 2 ? 2 : 1; + + Mat scales = getBlob(node_proto, foundScaleId); + if (scales.total() == 4) + { + layerParams.set("zoom_factor_y", scales.at(2)); + layerParams.set("zoom_factor_x", scales.at(3)); + } + else + { + const std::string& inputLast = node_proto.input(node_proto.input_size() - 1); + if (constBlobs.find(inputLast) != constBlobs.end()) + { + Mat shapes = getBlob(inputLast); + CV_CheckEQ(shapes.size[0], 4, ""); + CV_CheckEQ(shapes.size[1], 1, ""); + CV_CheckDepth(shapes.depth(), shapes.depth() == CV_32S || shapes.depth() == CV_32F, ""); + if (shapes.depth() == CV_32F) + shapes.convertTo(shapes, CV_32S); + layerParams.set("width", shapes.at(3)); + layerParams.set("height", shapes.at(2)); + } + } + replaceLayerParam(layerParams, "mode", "interpolation"); + addLayer(layerParams, node_proto); +} + +void ONNXImporter::parseUpsample(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +{ + //fused from Resize Subgraph + if (layerParams.has("coordinate_transformation_mode")) + { + String interp_mode = layerParams.get("coordinate_transformation_mode"); + CV_Assert_N(interp_mode != "tf_crop_and_resize", interp_mode != "tf_half_pixel_for_nn"); + + layerParams.set("align_corners", interp_mode == "align_corners"); + if (layerParams.get("mode") == "linear") + { + layerParams.set("mode", interp_mode == "pytorch_half_pixel" ? + "opencv_linear" : "bilinear"); + } + } + if (layerParams.get("mode") == "linear" && framework_name == "pytorch") + layerParams.set("mode", "opencv_linear"); + + layerParams.type = "Resize"; + if (layerParams.has("scales")) + { + // Pytorch layer + DictValue scales = layerParams.get("scales"); + CV_Assert(scales.size() == 4); + layerParams.set("zoom_factor_y", scales.getIntValue(2)); + layerParams.set("zoom_factor_x", scales.getIntValue(3)); + } + else if (layerParams.has("height_scale") && layerParams.has("width_scale")) + { + // Caffe2 layer + replaceLayerParam(layerParams, "height_scale", "zoom_factor_y"); + replaceLayerParam(layerParams, "width_scale", "zoom_factor_x"); + } + else + { + // scales as input + const std::string& input1 = node_proto.input(1); + if (constBlobs.find(input1) != constBlobs.end()) + { + Mat scales = getBlob(input1); + CV_Assert(scales.total() == 4); + layerParams.set("zoom_factor_y", scales.at(2)); + layerParams.set("zoom_factor_x", scales.at(3)); + } + } + replaceLayerParam(layerParams, "mode", "interpolation"); + addLayer(layerParams, node_proto); +} + +// "SoftMax" "LogSoftmax" +void ONNXImporter::parseSoftMax(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +{ + const std::string& layer_type = node_proto.op_type(); + layerParams.type = "Softmax"; + layerParams.set("log_softmax", layer_type == "LogSoftmax"); + addLayer(layerParams, node_proto); +} + +void ONNXImporter::parseDetectionOutput(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto_) +{ + opencv_onnx::NodeProto node_proto = node_proto_; + CV_CheckEQ(node_proto.input_size(), 3, ""); + if (constBlobs.find(node_proto.input(2)) != constBlobs.end()) + { + Mat priors = getBlob(node_proto, 2); + + LayerParams constParams; + constParams.name = layerParams.name + "/priors"; + constParams.type = "Const"; + constParams.blobs.push_back(priors); + + opencv_onnx::NodeProto priorsProto; + priorsProto.add_output(constParams.name); + addLayer(constParams, priorsProto); + + node_proto.set_input(2, constParams.name); + } + addLayer(layerParams, node_proto); +} + +void ONNXImporter::parseCustom(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +{ + for (int j = 0; j < node_proto.input_size(); j++) { + if (layer_id.find(node_proto.input(j)) == layer_id.end()) + layerParams.blobs.push_back(getBlob(node_proto, j)); + } + addLayer(layerParams, node_proto); +} + +const ONNXImporter::DispatchMap ONNXImporter::buildDispatchMap() +{ + DispatchMap dispatch; + + dispatch["MaxPool"] = &ONNXImporter::parseMaxPool; + dispatch["AveragePool"] = &ONNXImporter::parseAveragePool; + dispatch["GlobalAveragePool"] = dispatch["GlobalMaxPool"] = dispatch["ReduceMean"] = dispatch["ReduceSum"] = + dispatch["ReduceMax"] = &ONNXImporter::parseReduce; + dispatch["Slice"] = &ONNXImporter::parseSlice; + dispatch["Split"] = &ONNXImporter::parseSplit; + dispatch["Add"] = dispatch["Sum"] = dispatch["Sub"] = &ONNXImporter::parseBias; + dispatch["Pow"] = &ONNXImporter::parsePow; + dispatch["Max"] = &ONNXImporter::parseMax; + dispatch["Neg"] = &ONNXImporter::parseNeg; + dispatch["Constant"] = &ONNXImporter::parseConstant; + dispatch["LSTM"] = &ONNXImporter::parseLSTM; + dispatch["ImageScaler"] = &ONNXImporter::parseImageScaler; + dispatch["Clip"] = &ONNXImporter::parseClip; + dispatch["LeakyRelu"] = &ONNXImporter::parseLeakyRelu; + dispatch["Relu"] = &ONNXImporter::parseRelu; + dispatch["Elu"] = &ONNXImporter::parseElu; + dispatch["Tanh"] = &ONNXImporter::parseTanh; + dispatch["PRelu"] = &ONNXImporter::parsePRelu; + dispatch["LRN"] = &ONNXImporter::parseLRN; + dispatch["InstanceNormalization"] = &ONNXImporter::parseInstanceNormalization; + dispatch["BatchNormalization"] = &ONNXImporter::parseBatchNormalization; + dispatch["Gemm"] = &ONNXImporter::parseGemm; + dispatch["MatMul"] = &ONNXImporter::parseMatMul; + dispatch["Mul"] = dispatch["Div"] = &ONNXImporter::parseMul; + dispatch["Conv"] = &ONNXImporter::parseConv; + dispatch["ConvTranspose"] = &ONNXImporter::parseConvTranspose; + dispatch["Transpose"] = &ONNXImporter::parseTranspose; + dispatch["Squeeze"] = &ONNXImporter::parseSqueeze; + dispatch["Flatten"] = &ONNXImporter::parseFlatten; + dispatch["Unsqueeze"] = &ONNXImporter::parseUnsqueeze; + dispatch["Expand"] = &ONNXImporter::parseExpand; + dispatch["Reshape"] = &ONNXImporter::parseReshape; + dispatch["Pad"] = &ONNXImporter::parsePad; + dispatch["Shape"] = &ONNXImporter::parseShape; + dispatch["Cast"] = &ONNXImporter::parseCast; + dispatch["ConstantFill"] = dispatch["ConstantOfShape"] = &ONNXImporter::parseConstantFill; + dispatch["Gather"] = &ONNXImporter::parseGather; + dispatch["Concat"] = &ONNXImporter::parseConcat; + dispatch["Resize"] = &ONNXImporter::parseResize; + dispatch["Upsample"] = &ONNXImporter::parseUpsample; + dispatch["SoftMax"] = dispatch["LogSoftmax"] = &ONNXImporter::parseSoftMax; + dispatch["DetectionOutput"] = &ONNXImporter::parseDetectionOutput; + dispatch["Custom"] = &ONNXImporter::parseCustom; + + return dispatch; +} + Net readNetFromONNX(const String& onnxFile) { Net net; diff --git a/modules/dnn/src/tensorflow/tf_importer.cpp b/modules/dnn/src/tensorflow/tf_importer.cpp index 01fa0df985..fa33211a50 100644 --- a/modules/dnn/src/tensorflow/tf_importer.cpp +++ b/modules/dnn/src/tensorflow/tf_importer.cpp @@ -2869,7 +2869,7 @@ void TFImporter::parseNode(const tensorflow::NodeDef& layer) DispatchMap::const_iterator iter = dispatch.find(type); if (iter != dispatch.end()) { - ((*this).*(iter->second))(net, layer, layerParams); + CALL_MEMBER_FN(*this, iter->second)(net, layer, layerParams); } else { From bdd3930855c06cf9e48b7b836837414c36660c9a Mon Sep 17 00:00:00 2001 From: Zhuo Zhang Date: Thu, 29 Jul 2021 09:34:09 +0800 Subject: [PATCH 087/262] Fix typo in comment, OpenMP => TBB --- .../include/opencv2/core/parallel/backend/parallel_for.tbb.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/core/include/opencv2/core/parallel/backend/parallel_for.tbb.hpp b/modules/core/include/opencv2/core/parallel/backend/parallel_for.tbb.hpp index 264def5f50..04b0c4c6cb 100644 --- a/modules/core/include/opencv2/core/parallel/backend/parallel_for.tbb.hpp +++ b/modules/core/include/opencv2/core/parallel/backend/parallel_for.tbb.hpp @@ -38,7 +38,7 @@ static tbb::task_scheduler_init& getScheduler() } #endif -/** OpenMP parallel_for API implementation +/** TBB parallel_for API implementation * * @sa setParallelForBackend * @ingroup core_parallel_backend From 27392f832d99714e5134f877b7b8f72716c916f0 Mon Sep 17 00:00:00 2001 From: Smirnov Egor Date: Fri, 30 Jul 2021 13:00:13 +0300 Subject: [PATCH 088/262] reimplement onnx refactor for master --- modules/dnn/src/dnn_common.hpp | 1 + modules/dnn/src/onnx/onnx_importer.cpp | 3283 +++++++++++--------- modules/dnn/src/tensorflow/tf_importer.cpp | 2 +- 3 files changed, 1744 insertions(+), 1542 deletions(-) diff --git a/modules/dnn/src/dnn_common.hpp b/modules/dnn/src/dnn_common.hpp index 46fae41cc2..591be88079 100644 --- a/modules/dnn/src/dnn_common.hpp +++ b/modules/dnn/src/dnn_common.hpp @@ -14,6 +14,7 @@ Mutex& getInitializationMutex(); void initializeLayerFactory(); namespace detail { +#define CALL_MEMBER_FN(object, ptrToMemFn) ((object).*(ptrToMemFn)) class NotImplemented : public Layer { diff --git a/modules/dnn/src/onnx/onnx_importer.cpp b/modules/dnn/src/onnx/onnx_importer.cpp index d33fb68ac1..33dc648b2c 100644 --- a/modules/dnn/src/onnx/onnx_importer.cpp +++ b/modules/dnn/src/onnx/onnx_importer.cpp @@ -66,7 +66,7 @@ class ONNXImporter public: ONNXImporter(Net& net, const char *onnxFile) - : dstNet(net), utilNet() + : dstNet(net), utilNet(), dispatch(buildDispatchMap()) { hasDynamicShapes = false; CV_Assert(onnxFile); @@ -87,7 +87,7 @@ public: } ONNXImporter(Net& net, const char* buffer, size_t sizeBuffer) - : dstNet(net), utilNet() + : dstNet(net), utilNet(), dispatch(buildDispatchMap()) { hasDynamicShapes = false; CV_LOG_DEBUG(NULL, "DNN/ONNX: processing in-memory ONNX model (" << sizeBuffer << " bytes)"); @@ -129,6 +129,57 @@ protected: typedef std::map::iterator IterLayerId_t; void handleNode(const opencv_onnx::NodeProto& node_proto); + +private: + typedef void (ONNXImporter::*ONNXImporterNodeParser)(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + typedef std::map DispatchMap; + + void parseMaxPool (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseAveragePool (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseReduce (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseSlice (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseSplit (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseBias (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parsePow (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseMax (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseNeg (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseConstant (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseLSTM (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseImageScaler (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseClip (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseLeakyRelu (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseRelu (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseElu (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseTanh (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parsePRelu (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseLRN (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseInstanceNormalization(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseBatchNormalization (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseGemm (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseMatMul (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseMul (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseConv (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseConvTranspose (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseTranspose (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseSqueeze (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseFlatten (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseUnsqueeze (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseExpand (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseReshape (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parsePad (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseShape (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseCast (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseConstantFill (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseGather (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseConcat (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseResize (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseUpsample (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseSoftMax (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseDetectionOutput (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseCustom (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + + const DispatchMap dispatch; + static const DispatchMap buildDispatchMap(); }; inline void replaceLayerParam(LayerParams& layerParams, const String& oldKey, const String& newKey) @@ -585,13 +636,11 @@ const std::set& ONNXImporter::getSupportedTypes() return layerTypes; } -void ONNXImporter::handleNode(const opencv_onnx::NodeProto& node_proto_) +void ONNXImporter::handleNode(const opencv_onnx::NodeProto& node_proto) { - opencv_onnx::NodeProto node_proto = node_proto_; // TODO FIXIT - CV_Assert(node_proto.output_size() >= 1); std::string name = node_proto.output(0); - std::string layer_type = node_proto.op_type(); + const std::string& layer_type = node_proto.op_type(); CV_LOG_DEBUG(NULL, "DNN/ONNX: processing node with " << node_proto.input_size() << " inputs and " << node_proto.output_size() << " outputs: " << cv::format("[%s]:(%s)", layer_type.c_str(), name.c_str()) ); @@ -605,1546 +654,15 @@ void ONNXImporter::handleNode(const opencv_onnx::NodeProto& node_proto_) layerParams.type = layer_type; layerParams.set("has_dynamic_shapes", hasDynamicShapes); - if (layer_type == "MaxPool") + DispatchMap::const_iterator iter = dispatch.find(layer_type); + if (iter != dispatch.end()) { - layerParams.type = "Pooling"; - layerParams.set("pool", "MAX"); - layerParams.set("ceil_mode", layerParams.has("pad_mode")); - } - else if (layer_type == "AveragePool") - { - layerParams.type = "Pooling"; - layerParams.set("pool", "AVE"); - layerParams.set("ceil_mode", layerParams.has("pad_mode")); - layerParams.set("ave_pool_padded_area", framework_name == "pytorch"); - } - else if (layer_type == "GlobalAveragePool" || layer_type == "GlobalMaxPool" || - layer_type == "ReduceMean" || layer_type == "ReduceSum" || layer_type == "ReduceMax") - { - CV_Assert(node_proto.input_size() == 1); - layerParams.type = "Pooling"; - String pool; - if (layer_type == "GlobalMaxPool" || layer_type == "ReduceMax") - pool = "MAX"; - else if (layer_type == "ReduceSum") - pool = "SUM"; - else - pool = "AVE"; - layerParams.set("pool", pool); - layerParams.set("global_pooling", !layerParams.has("axes")); - if (layerParams.has("axes") && (layer_type == "ReduceMean" || layer_type == "ReduceSum" || layer_type == "ReduceMax")) - { - MatShape inpShape = outShapes[node_proto.input(0)]; - DictValue axes = layerParams.get("axes"); - bool keepdims = layerParams.get("keepdims"); - MatShape targetShape; - std::vector shouldDelete(inpShape.size(), false); - for (int i = 0; i < axes.size(); i++) { - int axis = normalize_axis(axes.get(i), inpShape.size()); - shouldDelete[axis] = true; - } - for (int axis = 0; axis < inpShape.size(); ++axis){ - if (!shouldDelete[axis]) - targetShape.push_back(inpShape[axis]); - else if (keepdims) - targetShape.push_back(1); - } - - if (inpShape.size() == 3 && axes.size() <= 2) - { - int axis = normalize_axis(axes.get(0), inpShape.size()); - CV_CheckNE(axis, 0, ""); - - LayerParams reshapeLp; - reshapeLp.name = layerParams.name + "/reshape"; - reshapeLp.type = "Reshape"; - CV_Assert(layer_id.find(reshapeLp.name) == layer_id.end()); - reshapeLp.set("axis", 0); - reshapeLp.set("num_axes", 1); - int newShape[] = {1, -1}; - reshapeLp.set("dim", DictValue::arrayInt(&newShape[0], 2)); - - opencv_onnx::NodeProto proto; - proto.add_input(node_proto.input(0)); - proto.add_output(reshapeLp.name); - addLayer(reshapeLp, proto); - - LayerParams avgLp; - avgLp.name = layerParams.name + "/avg"; - avgLp.type = "Pooling"; - CV_Assert(layer_id.find(avgLp.name) == layer_id.end()); - avgLp.set("pool", pool); - if (axes.size() == 2) - { - CV_CheckEQ(normalize_axis(axes.get(0), inpShape.size()), 1, "Unsupported mode"); - CV_CheckEQ(normalize_axis(axes.get(1), inpShape.size()), 2, "Unsupported mode"); - avgLp.set("global_pooling", true); - } - else - { - avgLp.set(axis == 2 ? "global_pooling_w" : "global_pooling_h", true); - avgLp.set(axis == 2 ? "kernel_h" : "kernel_w", 1); - } - - node_proto.set_input(0, reshapeLp.name); - node_proto.set_output(0, avgLp.name); - addLayer(avgLp, node_proto); - } - else - { - if (inpShape.size() != 4 && inpShape.size() != 5) - CV_Error(Error::StsNotImplemented, "Unsupported input shape of " + layer_type + " operation."); - - CV_Assert(axes.size() <= inpShape.size() - 2); - std::vector kernel_size(inpShape.size() - 2, 1); - if (axes.size() == 1 && (normalize_axis(axes.get(0), inpShape.size()) <= 1)) - { - int axis = normalize_axis(axes.get(0), inpShape.size()); - MatShape newShape = inpShape; - newShape[axis + 1] = total(newShape, axis + 1); - newShape.resize(axis + 2); - newShape.insert(newShape.begin(), 2 - axis, 1); - - LayerParams reshapeLp; - reshapeLp.type = "Reshape"; - reshapeLp.name = layerParams.name + "/reshape"; - CV_Assert(layer_id.find(reshapeLp.name) == layer_id.end()); - reshapeLp.set("dim", DictValue::arrayInt(&newShape[0], newShape.size())); - - node_proto.set_output(0, reshapeLp.name); - addLayer(reshapeLp, node_proto); - - kernel_size.resize(2); - kernel_size[0] = inpShape[axis]; - node_proto.set_input(0, node_proto.output(0)); - } - else - { - for (int i = 0; i < axes.size(); i++) { - int axis = normalize_axis(axes.get(i), inpShape.size()); - CV_Assert_N(axis >= 2 + i, axis < inpShape.size()); - kernel_size[axis - 2] = inpShape[axis]; - } - } - - LayerParams poolLp = layerParams; - poolLp.name = layerParams.name + "/avg"; - CV_Assert(layer_id.find(poolLp.name) == layer_id.end()); - poolLp.set("kernel_size", DictValue::arrayInt(&kernel_size[0], kernel_size.size())); - - node_proto.set_output(0, poolLp.name); - addLayer(poolLp, node_proto); - } - - layerParams.type = "Reshape"; - layerParams.set("dim", DictValue::arrayInt(&targetShape[0], targetShape.size())); - - node_proto.set_input(0, node_proto.output(0)); - node_proto.set_output(0, layerParams.name); - } - else if (!layerParams.has("axes") && (layer_type == "ReduceMean" || layer_type == "ReduceSum" || layer_type == "ReduceMax")) - { - CV_CheckEQ(layerParams.get("keepdims"), 0, "layer only supports keepdims = false"); - LayerParams reshapeLp; - reshapeLp.name = layerParams.name + "/reshape"; - reshapeLp.type = "Reshape"; - CV_Assert(layer_id.find(reshapeLp.name) == layer_id.end()); - int newShape[] = {1, 1, 1, -1}; - reshapeLp.set("dim", DictValue::arrayInt(&newShape[0], 4)); - - opencv_onnx::NodeProto proto; - proto.add_input(node_proto.input(0)); - proto.add_output(reshapeLp.name); - addLayer(reshapeLp, proto); - - LayerParams poolLp = layerParams; - poolLp.name = layerParams.name + "/pool"; - CV_Assert(layer_id.find(poolLp.name) == layer_id.end()); - - node_proto.set_input(0, reshapeLp.name); - node_proto.set_output(0, poolLp.name); - addLayer(poolLp, node_proto); - - layerParams.type = "Reshape"; - int targetShape[] = {1}; - layerParams.set("dim", DictValue::arrayInt(&targetShape[0], 1)); - - node_proto.set_input(0, node_proto.output(0)); - node_proto.set_output(0, layerParams.name); - } - } - else if (layer_type == "Slice") - { - int axis = 0; - std::vector begin; - std::vector end; - std::vector steps; - int inp_size = node_proto.input_size(); - - if (inp_size == 1) - { - if (layerParams.has("axes")) { - DictValue axes = layerParams.get("axes"); - for (int i = 1; i < axes.size(); ++i) { - CV_Assert(axes.get(i - 1) == axes.get(i) - 1); - } - axis = axes.get(0); - } - - DictValue starts = layerParams.get("starts"); - DictValue ends = layerParams.get("ends"); - CV_Assert(starts.size() == ends.size()); - - if (axis > 0) { - begin.resize(axis, 0); - end.resize(axis, -1); - } - for (int i = 0; i < starts.size(); ++i) - { - begin.push_back(starts.get(i)); - int finish = ends.get(i); - end.push_back((finish < 0) ? --finish : finish); // numpy doesn't include last dim - } - } else { // inp_size > 1 - CV_Assert(inp_size >= 3); - for (int i = 1; i < inp_size; i++) { - CV_Assert(constBlobs.find(node_proto.input(i)) != constBlobs.end()); - } - Mat start_blob = getBlob(node_proto, 1); - Mat end_blob = getBlob(node_proto, 2); - CV_Assert(start_blob.total() == end_blob.total()); - - if (inp_size > 3) { - Mat axes_blob = getBlob(node_proto, 3); - const int* axes = (int*)axes_blob.data; - for (int i = 1; i < axes_blob.total(); ++i) { - CV_Assert(axes[i - 1] == axes[i] - 1); - } - axis = axes[0]; - } - - const int* starts = start_blob.ptr(); - const int* ends = end_blob.ptr(); - if (axis > 0) { - begin.resize(axis, 0); - end.resize(axis, -1); - } - std::copy(starts, starts + start_blob.total(), std::back_inserter(begin)); - for (int i = 0; i < end_blob.total(); ++i) - { - int finish = ends[i]; - end.push_back((finish < 0) ? --finish : finish); // numpy doesn't include last dim - } - - if (inp_size == 5) { - CV_Assert(constBlobs.find(node_proto.input(4)) != constBlobs.end()); - Mat step_blob = getBlob(node_proto, 4); - const int* steps_ptr = step_blob.ptr(); - - if (axis > 0) - steps.resize(axis, 1); - - std::copy(steps_ptr, steps_ptr + step_blob.total(), std::back_inserter(steps)); - - // Very strange application for Slice op with tensor reversing. - // We just workaround it for 2d constants. - if (constBlobs.find(node_proto.input(0)) != constBlobs.end() && - axis == 0 && - start_blob.at(0) == -1 && step_blob.at(0) == -1 && - end_blob.at(0) == std::numeric_limits::min()) - { - Mat inp = getBlob(node_proto, 0); - if (inp.dims == 2) - { - Mat flipped; - flip(inp, flipped, 0); - addConstant(layerParams.name, flipped); - return; - } - } - } - } - layerParams.set("begin", DictValue::arrayInt(&begin[0], begin.size())); - layerParams.set("end", DictValue::arrayInt(&end[0], end.size())); - layerParams.set("axis", axis); - - if (!steps.empty()) - layerParams.set("steps", DictValue::arrayInt(&steps[0], steps.size())); - - if (constBlobs.find(node_proto.input(0)) != constBlobs.end()) - { - Mat inp = getBlob(node_proto, 0); - std::vector inputs, sliced; - inputs.push_back(inp); - runLayer(layerParams, inputs, sliced); - CV_Assert(sliced.size() == 1); - addConstant(layerParams.name, sliced[0]); - return; - } - } - else if (layer_type == "Split") - { - if (layerParams.has("split")) - { - DictValue splits = layerParams.get("split"); - const int numSplits = splits.size(); - CV_Assert(numSplits > 1); - - std::vector slicePoints(numSplits - 1, splits.get(0)); - for (int i = 1; i < splits.size() - 1; ++i) - { - slicePoints[i] = slicePoints[i - 1] + splits.get(i - 1); - } - layerParams.set("slice_point", DictValue::arrayInt(&slicePoints[0], slicePoints.size())); - } - else - { - layerParams.set("num_split", node_proto.output_size()); - } - layerParams.type = "Slice"; - } - else if (layer_type == "Add" || layer_type == "Sum" || layer_type == "Sub") - { - bool isSub = layer_type == "Sub"; - CV_CheckEQ(node_proto.input_size(), 2, ""); - bool is_const_0 = layer_id.find(node_proto.input(0)) == layer_id.end(); - bool is_const_1 = layer_id.find(node_proto.input(1)) == layer_id.end(); - if (is_const_0 && is_const_1) - { - Mat blob_0 = getBlob(node_proto, 0); - Mat blob_1 = getBlob(node_proto, 1); - CV_Assert(blob_0.size == blob_1.size); - Mat output = isSub ? (blob_0 - blob_1) : (blob_0 + blob_1); - addConstant(layerParams.name, output); - return; - } - else if (is_const_0 || is_const_1) - { - int const_blob_id = is_const_0 ? 0 : 1; - Mat blob = getBlob(node_proto, const_blob_id); - int blob_total = blob.total(); - if (blob_total == 1) { - layerParams.type = "Power"; - layerParams.set("shift", (isSub ? -1 : 1) * blob.ptr()[0]); - } - else { - MatShape inpShape = outShapes[node_proto.input(1 - const_blob_id)]; - if (shape(blob) == inpShape) - { - LayerParams constParams; - constParams.name = layerParams.name + "/const"; - constParams.type = "Const"; - constParams.blobs.push_back((isSub ? -1 : 1) * blob); - int id; - if (DNN_DIAGNOSTICS_RUN) - id = utilNet.addLayer(constParams.name, constParams.type, constParams); - else - id = dstNet.addLayer(constParams.name, constParams.type, constParams); - layer_id.insert(std::make_pair(constParams.name, LayerInfo(id, 0))); - outShapes[constParams.name] = shape(blob); - - layerParams.type = "Eltwise"; - node_proto.set_input(const_blob_id, constParams.name); - } - else - { - layerParams.type = "Scale"; - layerParams.set("bias_term", true); - int axis = 1; - for (int i = 0; i < graph_proto.initializer_size(); i++) - { - opencv_onnx::TensorProto tensor_proto = graph_proto.initializer(i); - if (tensor_proto.name() == node_proto.input(const_blob_id)) - { - axis = inpShape.size() - tensor_proto.dims_size(); - break; - } - } - layerParams.set("axis", axis); - blob = blob.reshape(1, 1); - layerParams.blobs.push_back((isSub ? -1 : 1) * blob); - } - } - } - else if (outShapes[node_proto.input(0)] == outShapes[node_proto.input(1)]) - { - layerParams.type = "Eltwise"; - if (isSub) - { - static float subCoeffs[] = {1.f, -1.f}; - layerParams.set("coeff", DictValue::arrayReal(subCoeffs, 2)); - } - } - else - { - if (isSub) - { - LayerParams powerParams; - powerParams.name = layerParams.name + "/neg"; - powerParams.type = "Power"; - powerParams.set("scale", -1); - - int id; - //Create Power layer - if (DNN_DIAGNOSTICS_RUN) - id = utilNet.addLayer(powerParams.name, powerParams.type, powerParams); - else - id = dstNet.addLayer(powerParams.name, powerParams.type, powerParams); - //Connect to input - IterLayerId_t layerId = layer_id.find(node_proto.input(1)); - CV_Assert(layerId != layer_id.end()); - if (DNN_DIAGNOSTICS_RUN) - utilNet.connect(layerId->second.layerId, layerId->second.outputId, id, 0); - else - dstNet.connect(layerId->second.layerId, layerId->second.outputId, id, 0); - //Add shape - layer_id.insert(std::make_pair(powerParams.name, LayerInfo(id, 0))); - outShapes[powerParams.name] = outShapes[node_proto.input(1)]; - - //Replace input to Power - node_proto.set_input(1, powerParams.name); - } - layerParams.type = "Scale"; - layerParams.set("bias_term", true); - } - } - else if (layer_type == "Pow") - { - if (layer_id.find(node_proto.input(1)) != layer_id.end()) - CV_Error(Error::StsNotImplemented, "Unsupported Pow op with variable power"); - - Mat blob = getBlob(node_proto, 1); - if (blob.total() != 1) - CV_Error(Error::StsNotImplemented, "Pow op supports only scalar power"); - - blob.convertTo(blob, CV_32F); - layerParams.type = "Power"; - layerParams.set("power", blob.ptr()[0]); - } - else if (layer_type == "Max") - { - layerParams.type = "Eltwise"; - layerParams.set("operation", "max"); - } - else if (layer_type == "Neg") - { - layerParams.type = "Power"; - layerParams.set("scale", -1); - } - else if (layer_type == "Constant") - { - CV_Assert(node_proto.input_size() == 0); - CV_Assert(layerParams.blobs.size() == 1); - addConstant(layerParams.name, layerParams.blobs[0]); - return; - } - else if (layer_type == "LSTM") - { - LayerParams lstmParams = layerParams; - lstmParams.name += "/lstm"; - - // https://pytorch.org/docs/stable/nn.html#lstm - CV_Assert(node_proto.input_size() == 7); - Mat Wx = getBlob(node_proto, 1); - Mat Wh = getBlob(node_proto, 2); - Mat b = getBlob(node_proto, 3); - Mat h0 = getBlob(node_proto, 5); - Mat c0 = getBlob(node_proto, 6); - - b = b.reshape(1, b.size[0]); - - const int numHidden = lstmParams.get("hidden_size"); - const int numDirs = Wx.size[0]; // Is 1 for forward only and 2 for bidirectional LSTM. - const int numFeatures = Wx.size[2]; - Mat bx = b.colRange(0, b.cols / 2); - Mat bh = b.colRange(b.cols / 2, b.cols); - b = bx + bh; - - // IFGO->IGFO - for (int k = 0; k < numDirs; ++k) - { - float* WxData = Wx.ptr(k); - float* WhData = Wh.ptr(k); - float* biasData = b.ptr(k); - for (int j = 0; j < numHidden; ++j) - { - for (int i = 0; i < numFeatures; ++i) - { - std::swap(WxData[(numHidden + j) * numFeatures + i], - WxData[(numHidden * 2 + j) * numFeatures + i]); - } - for (int i = 0; i < numHidden; ++i) - { - std::swap(WhData[(numHidden + j) * numHidden + i], - WhData[(numHidden * 2 + j) * numHidden + i]); - } - std::swap(biasData[numHidden + j], biasData[numHidden * 2 + j]); - } - } - Wx = Wx.reshape(1, Wx.size[0] * Wx.size[1]); - Wh = Wh.reshape(1, Wh.size[0] * Wh.size[1]); - h0 = h0.reshape(1, h0.size[0] * h0.size[1]); - c0 = c0.reshape(1, c0.size[0] * c0.size[1]); - - lstmParams.blobs.resize(5); - lstmParams.blobs[0] = Wh; - lstmParams.blobs[1] = Wx; - lstmParams.blobs[2] = b; - lstmParams.blobs[3] = h0; - lstmParams.blobs[4] = c0; - lstmParams.set("bidirectional", lstmParams.get("direction", "") == "bidirectional"); - - node_proto.set_output(0, lstmParams.name); // set different name so output shapes will be registered on that name - addLayer(lstmParams, node_proto); - - MatShape lstmShape = outShapes[node_proto.output(0)]; - - // Add fake 1 as it is done in ONNX - lstmShape.insert(lstmShape.begin() + 1, 1); - - layerParams.type = "Reshape"; - layerParams.set("dim", DictValue::arrayInt(&lstmShape[0], lstmShape.size())); - node_proto.set_input(0, lstmParams.name); // redirect input to LSTM - node_proto.set_output(0, layerParams.name); // keep origin LSTM's name - } - else if (layer_type == "ImageScaler") - { - const float scale = layerParams.has("scale") ? layerParams.get("scale") : 1.0f; - layerParams.erase("scale"); - - if (layerParams.has("bias")) - { - layerParams.type = "Scale"; - layerParams.blobs.push_back( - Mat(Size(1, layerParams.get("bias").size()), CV_32FC1, scale)); - - layerParams.set("bias_term", true); - Mat bias(1, layerParams.get("bias").size(), CV_32FC1); - for (int j = 0; j < bias.total(); j++) { - bias.at(0, j) = layerParams.get("bias").getRealValue(j); - } - layerParams.blobs.push_back(bias); - layerParams.erase("bias"); - } - else { - layerParams.set("scale", scale); - layerParams.type = "Power"; - } - } - else if (layer_type == "Clip") - { - layerParams.type = "ReLU6"; - replaceLayerParam(layerParams, "min", "min_value"); - replaceLayerParam(layerParams, "max", "max_value"); - - } - else if (layer_type == "LeakyRelu") - { - layerParams.type = "ReLU"; - replaceLayerParam(layerParams, "alpha", "negative_slope"); - } - else if (layer_type == "Relu") - { - layerParams.type = "ReLU"; - } - else if (layer_type == "Elu") - { - layerParams.type = "ELU"; - } - else if (layer_type == "Tanh") - { - layerParams.type = "TanH"; - } - else if (layer_type == "PRelu") - { - layerParams.type = "PReLU"; - layerParams.blobs.push_back(getBlob(node_proto, 1)); - } - else if (layer_type == "LRN") - { - replaceLayerParam(layerParams, "size", "local_size"); - } - else if (layer_type == "InstanceNormalization") - { - if (node_proto.input_size() != 3) - CV_Error(Error::StsNotImplemented, - "Expected input, scale, bias"); - - layerParams.blobs.resize(4); - layerParams.blobs[2] = getBlob(node_proto, 1); // weightData - layerParams.blobs[3] = getBlob(node_proto, 2); // biasData - layerParams.set("has_bias", true); - layerParams.set("has_weight", true); - - // Get number of channels in input - int size = layerParams.blobs[2].total(); - layerParams.blobs[0] = Mat::zeros(size, 1, CV_32F); // mean - layerParams.blobs[1] = Mat::ones(size, 1, CV_32F); // std - - LayerParams mvnParams; - mvnParams.name = layerParams.name + "/MVN"; - mvnParams.type = "MVN"; - mvnParams.set("eps", layerParams.get("epsilon")); - layerParams.erase("epsilon"); - - //Create MVN layer - int id; - if (DNN_DIAGNOSTICS_RUN) - id = utilNet.addLayer(mvnParams.name, mvnParams.type, mvnParams); - else - id = dstNet.addLayer(mvnParams.name, mvnParams.type, mvnParams); - //Connect to input - IterLayerId_t layerId = layer_id.find(node_proto.input(0)); - CV_Assert(layerId != layer_id.end()); - if (DNN_DIAGNOSTICS_RUN) - utilNet.connect(layerId->second.layerId, layerId->second.outputId, id, 0); - else - dstNet.connect(layerId->second.layerId, layerId->second.outputId, id, 0); - //Add shape - layer_id.insert(std::make_pair(mvnParams.name, LayerInfo(id, 0))); - outShapes[mvnParams.name] = outShapes[node_proto.input(0)]; - - //Replace Batch Norm's input to MVN - node_proto.set_input(0, mvnParams.name); - layerParams.type = "BatchNorm"; - } - else if (layer_type == "BatchNormalization") - { - if (node_proto.input_size() != 5) - CV_Error(Error::StsNotImplemented, - "Expected input, scale, bias, mean and var"); - - layerParams.type = "BatchNorm"; - replaceLayerParam(layerParams, "epsilon", "eps"); - replaceLayerParam(layerParams, "spatial", "use_global_stats"); - - Mat meanData = getBlob(node_proto, 3); - Mat stdData = getBlob(node_proto, 4); - - layerParams.blobs.push_back(meanData); - layerParams.blobs.push_back(stdData); - - if (!node_proto.input(1).empty()) { - layerParams.set("has_weight", true); - layerParams.blobs.push_back(getBlob(node_proto, 1)); // weightData - } else { - layerParams.set("has_weight", false); - } - - if (!node_proto.input(2).empty()) { - layerParams.set("has_bias", true); - layerParams.blobs.push_back(getBlob(node_proto, 2)); // biasData - } else { - layerParams.set("has_bias", false); - } - } - else if (layer_type == "Gemm") - { - CV_Assert(node_proto.input_size() >= 2); - layerParams.type = "InnerProduct"; - Mat weights = getBlob(node_proto, 1); - int ind_num_out = 0; - if (layerParams.has("transB") && !layerParams.get("transB")) { - transpose(weights, weights); - ind_num_out = 1; - } - layerParams.blobs.push_back(weights); - - if (node_proto.input_size() == 3) { - Mat bias = getBlob(node_proto, 2); - layerParams.blobs.push_back(bias); - } - if (constBlobs.find(node_proto.input(0)) != constBlobs.end()) - { - Mat inputBuf = getBlob(node_proto, 0); - - LayerParams constParams; - constParams.name = node_proto.input(0); - constParams.type = "Const"; - constParams.blobs.push_back(inputBuf); - - opencv_onnx::NodeProto proto; - proto.add_output(constParams.name); - addLayer(constParams, proto); - } - - layerParams.set("num_output", layerParams.blobs[0].size[ind_num_out]); - layerParams.set("bias_term", node_proto.input_size() == 3); - } - else if (layer_type == "MatMul") - { - CV_Assert(node_proto.input_size() == 2); - layerParams.type = "InnerProduct"; - layerParams.set("bias_term", false); - CV_Assert(constBlobs.find(node_proto.input(0)) == constBlobs.end()); - int firstInpDims = outShapes[node_proto.input(0)].size(); - int secondInpDims; - - if (constBlobs.find(node_proto.input(1)) != constBlobs.end()) - { - Mat blob = getBlob(node_proto, 1); - secondInpDims = blob.dims; - layerParams.blobs.push_back(blob.t()); - layerParams.set("num_output", layerParams.blobs[0].size[0]); - } else { - secondInpDims = outShapes[node_proto.input(1)].size(); - } - layerParams.set("axis", firstInpDims - secondInpDims + 1); - } - else if (layer_type == "Mul" || layer_type == "Div") - { - CV_Assert(node_proto.input_size() == 2); - - bool isDiv = layer_type == "Div"; - int constId = -1; - bool haveVariables = false; - for (int i = 0; i < 2; ++i) - { - if (constBlobs.find(node_proto.input(i)) != constBlobs.end()) - constId = i; - else - haveVariables = true; - } - if (constId != -1 && haveVariables) - { - Mat blob = getBlob(node_proto, constId); - blob = blob.reshape(1, 1); - if (blob.total() == 1) { - float blob_value = blob.ptr()[0]; - float coeff = isDiv ? 1.0 / blob_value : blob_value; - layerParams.set("scale", coeff); - layerParams.type = "Power"; - } - else { - if (isDiv) - divide(1.0, blob, blob); - layerParams.blobs.push_back(blob); - layerParams.type = "Scale"; - } - } - else if (!haveVariables) - { - Mat inp0 = getBlob(node_proto, 0); - Mat inp1 = getBlob(node_proto, 1); - - if (inp0.size != inp1.size && (inp0.total() != 1 || inp1.total() != 1)) - CV_Error_(Error::StsNotImplemented, ("Different shapes case is not supported with constant inputs: %s", layer_type.c_str())); - - if (inp0.total() == 1 && inp1.total() == 1 && inp0.dims != inp1.dims) - { - if (inp0.dims < inp1.dims) - { - inp0 = inp0.reshape(1, inp1.dims, inp1.size); - inp0.dims = inp1.dims; - } - else - { - inp1 = inp1.reshape(1, inp0.dims, inp0.size); - inp1.dims = inp0.dims; - } - } - - Mat out; - if (inp0.total() != inp1.total()) - { - if (inp0.total() == 1) - { - float inp0_value = inp0.ptr()[0]; - float coeff = isDiv ? 1.0 / inp0_value : inp0_value; - multiply(inp1, coeff, out); - } - else - { - float inp1_value = inp1.ptr()[0]; - float coeff = isDiv ? 1.0 / inp1_value : inp1_value; - multiply(inp0, coeff, out); - } - - } - else - { - out = isDiv ? inp0 / inp1 : inp0.mul(inp1); - } - - if (inp0.dims == 1 && inp1.dims == 1) - out.dims = 1; // to workaround dims == 1 - addConstant(layerParams.name, out); - return; - } - else if (outShapes[node_proto.input(0)] == outShapes[node_proto.input(1)]) - { - layerParams.type = "Eltwise"; - layerParams.set("operation", isDiv ? "div" : "prod"); - } - else - { - // Scale layer allocate output with the first input shape - if (total(outShapes[node_proto.input(0)]) < total(outShapes[node_proto.input(1)])) - { - opencv_onnx::NodeProto proto; - proto.add_input(node_proto.input(1)); - proto.add_input(node_proto.input(0)); - proto.add_output(layerParams.name); - node_proto = proto; - } - - if (isDiv) - { - LayerParams powerParams; - powerParams.name = layerParams.name + "/inv"; - powerParams.type = "Power"; - powerParams.set("power", -1); - - int id; - //Create Power layer - if (DNN_DIAGNOSTICS_RUN) - id = utilNet.addLayer(powerParams.name, powerParams.type, powerParams); - else - id = dstNet.addLayer(powerParams.name, powerParams.type, powerParams); - //Connect to input - IterLayerId_t layerId = layer_id.find(node_proto.input(1)); - CV_Assert(layerId != layer_id.end()); - if (DNN_DIAGNOSTICS_RUN) - utilNet.connect(layerId->second.layerId, layerId->second.outputId, id, 0); - else - dstNet.connect(layerId->second.layerId, layerId->second.outputId, id, 0); - //Add shape - layer_id.insert(std::make_pair(powerParams.name, LayerInfo(id, 0))); - outShapes[powerParams.name] = outShapes[node_proto.input(1)]; - - //Replace input to Power - node_proto.set_input(1, powerParams.name); - } - layerParams.type = "Scale"; - } - } - else if (layer_type == "Conv") - { - CV_Assert(node_proto.input_size() >= 2); - layerParams.type = "Convolution"; - for (int j = 1; j < node_proto.input_size(); j++) { - if (constBlobs.find(node_proto.input(j)) != constBlobs.end()) - { - layerParams.blobs.push_back(getBlob(node_proto, j)); - } - } - int outCn = layerParams.blobs.empty() ? outShapes[node_proto.input(1)][0] : layerParams.blobs[0].size[0]; - layerParams.set("num_output", outCn); - - // Check for asymmetric padding in Conv2D - if (layerParams.has("pad")) - { - bool asymmetricPadding = false; - DictValue pads = layerParams.get("pad"); - const int dims = pads.size() / 2; - for (int i = 0; i < dims; ++i) - { - if (pads.get(i) != pads.get(i + dims)) - { - asymmetricPadding = true; - break; - } - } - if (asymmetricPadding && pads.size() == 4) // [pad_t, pad_l, pad_b, pad_r] - { - layerParams.erase("pad"); - // No paddings required for N, C axis - std::vector paddings(4, 0); - // Add paddings for H, W axis - for (int i = 0; i < dims; ++i) - { - paddings.push_back(pads.get(i)); - paddings.push_back(pads.get(dims + i)); - } - LayerParams padLp; - padLp.name = layerParams.name + "/pad"; - padLp.type = "Padding"; - padLp.set("paddings", DictValue::arrayInt(&paddings[0], paddings.size())); - - opencv_onnx::NodeProto proto; - proto.add_input(node_proto.input(0)); - proto.add_output(padLp.name); - - addLayer(padLp, proto); - node_proto.set_input(0, padLp.name); - } - } - } - else if (layer_type == "ConvTranspose") - { - CV_Assert(node_proto.input_size() >= 2); - layerParams.type = "Deconvolution"; - for (int j = 1; j < node_proto.input_size(); j++) { - layerParams.blobs.push_back(getBlob(node_proto, j)); - } - layerParams.set("num_output", layerParams.blobs[0].size[1] * layerParams.get("group", 1)); - layerParams.set("bias_term", node_proto.input_size() == 3); - - if (!layerParams.has("kernel_size")) - CV_Error(Error::StsNotImplemented, - "Required attribute 'kernel_size' is not present."); - - if (layerParams.has("output_shape")) - { - const DictValue& outShape = layerParams.get("output_shape"); - DictValue strides = layerParams.get("stride"); - DictValue kernel = layerParams.get("kernel_size"); - - String padMode; - std::vector adjust_pads; - if (layerParams.has("pad_mode")) - { - padMode = toUpperCase(layerParams.get("pad_mode")); - if (padMode != "SAME" && padMode != "VALID") - CV_Error(Error::StsError, "Unsupported padding mode " + padMode); - - for (int i = 0; i < strides.size(); i++) - { - int sz = outShape.get(2 + i); - int stride = strides.get(i); - adjust_pads.push_back(padMode == "SAME"? (sz - 1) % stride : - (sz - kernel.get(i)) % stride); - } - layerParams.set("adj", DictValue::arrayInt(&adjust_pads[0], adjust_pads.size())); - } - } - else if (layerParams.has("output_padding")) - { - replaceLayerParam(layerParams, "output_padding", "adj"); - } - } - else if (layer_type == "Transpose") - { - layerParams.type = "Permute"; - replaceLayerParam(layerParams, "perm", "order"); - - CV_Assert(node_proto.input_size() == 1); - if (constBlobs.find(node_proto.input(0)) != constBlobs.end()) - { - std::vector inputs(1, getBlob(node_proto, 0)), transposed; - runLayer(layerParams, inputs, transposed); - CV_Assert(transposed.size() == 1); - addConstant(layerParams.name, transposed[0]); - return; - } - } - else if (layer_type == "Squeeze") - { - CV_Assert_N(node_proto.input_size() == 1, layerParams.has("axes")); - DictValue axes_dict = layerParams.get("axes"); - MatShape inpShape = outShapes[node_proto.input(0)]; - - std::vector maskedAxes(inpShape.size(), false); - for (int i = 0; i < axes_dict.size(); ++i) - { - int axis = axes_dict.getIntValue(i); - CV_CheckLE(axis, static_cast(inpShape.size()), "Squeeze axis"); - maskedAxes[axis] = inpShape[axis] == 1; - } - MatShape outShape; - for (int i = 0; i < inpShape.size(); ++i) - { - if (!maskedAxes[i]) - outShape.push_back(inpShape[i]); - } - if (outShape.size() != inpShape.size()) - { - layerParams.type = "Reshape"; - layerParams.set("dim", DictValue::arrayInt(&outShape[0], outShape.size())); - if (hasDynamicShapes) - { - std::vector dynamicAxes; - std::vector inputIndices; - for (int index = 0; index < inpShape.size(); ++index) - { - if (!maskedAxes[index]) - inputIndices.push_back(index); - } - for (int index = 0; index < outShape.size(); ++index) - dynamicAxes.push_back(index); - layerParams.set("dynamic_axes", DictValue::arrayInt(dynamicAxes.data(), dynamicAxes.size())); - layerParams.set("input_indices", DictValue::arrayInt(inputIndices.data(), inputIndices.size())); - } - } - else - layerParams.type = "Identity"; - - if (constBlobs.find(node_proto.input(0)) != constBlobs.end()) - { - Mat inp = getBlob(node_proto, 0); - Mat out = inp.reshape(1, outShape); - out.dims = outShape.size(); // to workaround dims == 1 - addConstant(layerParams.name, out); - return; - } - } - else if (layer_type == "Flatten") - { - CV_CheckEQ(node_proto.input_size(), 1, ""); - if (constBlobs.find(node_proto.input(0)) != constBlobs.end()) - { - Mat input = getBlob(node_proto, 0); - int axis = normalize_axis(layerParams.get("axis", 1), input.dims); - - std::vector out_size(&input.size[0], &input.size[0] + axis); - out_size.push_back(input.total(axis)); - Mat output = input.reshape(1, out_size); - addConstant(layerParams.name, output); - return; - } - } - else if (layer_type == "Unsqueeze") - { - CV_Assert(node_proto.input_size() == 1); - DictValue axes = layerParams.get("axes"); - if (constBlobs.find(node_proto.input(0)) != constBlobs.end()) - { - // Constant input. - Mat input = getBlob(node_proto, 0); - - std::vector dims; - for (int j = 0; j < input.dims; j++) { - dims.push_back(input.size[j]); - } - CV_Assert(axes.getIntValue(axes.size()-1) <= dims.size()); - for (int j = 0; j < axes.size(); j++) { - dims.insert(dims.begin() + axes.getIntValue(j), 1); - } - - Mat out = input.reshape(0, dims); - addConstant(layerParams.name, out); - return; - } - - // Variable input. - if (axes.size() != 1) - CV_Error(Error::StsNotImplemented, "Multidimensional unsqueeze"); - - MatShape inpShape = outShapes[node_proto.input(0)]; - int axis = axes.getIntValue(0); - CV_Assert(0 <= axis && axis <= inpShape.size()); - std::vector outShape = inpShape; - outShape.insert(outShape.begin() + axis, 1); - layerParams.type = "Reshape"; - layerParams.set("dim", DictValue::arrayInt(&outShape[0], outShape.size())); - if (hasDynamicShapes) - { - std::vector dynamicAxes; - std::vector inputIndices; - for (int index = 0; index < outShape.size(); ++index) { - if (index != axis) - dynamicAxes.push_back(index); - } - for (int index = 0; index < inpShape.size(); ++index) - inputIndices.push_back(index); - layerParams.set("dynamic_axes", DictValue::arrayInt(dynamicAxes.data(), dynamicAxes.size())); - layerParams.set("input_indices", DictValue::arrayInt(inputIndices.data(), inputIndices.size())); - } - } - else if (layer_type == "Expand") - { - CV_CheckEQ(node_proto.input_size(), 2, ""); - const std::string& input0 = node_proto.input(0); - const std::string& input1 = node_proto.input(1); - Mat newShapeMat = getBlob(input1); - MatShape targetShape(newShapeMat.ptr(), newShapeMat.ptr() + newShapeMat.total()); - - MatShape inpShape; - bool haveVariables = constBlobs.find(input0) == constBlobs.end(); - if (haveVariables) - { - IterShape_t shapeIt = outShapes.find(input0); - CV_Assert(shapeIt != outShapes.end()); - inpShape = shapeIt->second; - } - else - { - inpShape = shape(getBlob(input0)); - } - - String srcName = input0; - // Unsqueeze and repeat along new axis - if (targetShape.size() == inpShape.size() + 1) - { - for (int i = 0; i < targetShape.size(); i++) - { - if (targetShape[i] == -1 && i < inpShape.size()) - targetShape[i] = inpShape[i]; - else if (i < inpShape.size() && targetShape[i] != inpShape[i]) - inpShape.insert(inpShape.begin() + i, 1); - } - if (haveVariables) - { - LayerParams reshapeLp; - reshapeLp.name = layerParams.name + "/reshape"; - reshapeLp.type = "Reshape"; - CV_Assert(layer_id.find(reshapeLp.name) == layer_id.end()); - reshapeLp.set("dim", DictValue::arrayInt(&inpShape[0], inpShape.size())); - - opencv_onnx::NodeProto proto; - proto.add_input(node_proto.input(0)); - proto.add_output(reshapeLp.name); - addLayer(reshapeLp, proto); - srcName = reshapeLp.name; - } - } - CV_CheckEQ(inpShape.size(), targetShape.size(), "Unsupported Expand op with different dims"); - - std::vector broadcast_axes; - for (int i = 0; i < targetShape.size(); i++) - { - if (targetShape[i] != inpShape[i]) - { - if (inpShape[i] == 1) - broadcast_axes.push_back(i); - else - CV_Error(Error::StsError, format("Could not be broadcast by axis: %d", i)); - } - } - - if (!haveVariables) - { - if (broadcast_axes.size() != 1) - CV_Error(Error::StsNotImplemented, "Expand op doesn't support multiple axes for constant input"); - - Mat input = getBlob(node_proto, 0); - input = input.reshape(0, total(inpShape, 0, broadcast_axes[0])); - Mat output = cv::repeat(input, 1, targetShape[broadcast_axes[0]]); - output = output.reshape(0, targetShape); - addConstant(layerParams.name, output); - return; - } - - if (broadcast_axes.size() == 2 && - broadcast_axes[0] == broadcast_axes[1] - 1 && broadcast_axes[1] == inpShape.size() - 1) - { - LayerParams constParams; - constParams.name = layerParams.name + "/const"; - CV_Assert(layer_id.find(constParams.name) == layer_id.end()); - constParams.type = "Const"; - - Mat inp = Mat::ones(newShapeMat.total(), newShapeMat.ptr(), CV_32F); - constParams.blobs.push_back(inp); - - opencv_onnx::NodeProto proto; - proto.add_output(constParams.name); - addLayer(constParams, proto); - - layerParams.type = "Scale"; - layerParams.set("bias_term", false); - node_proto.set_input(0, constParams.name); - node_proto.set_input(1, srcName); - } - else if (broadcast_axes.size() == 1 && broadcast_axes[0] <= 1) - { - String base_name = layerParams.name + "/copy_"; - std::vector input_names; - for (int j = 0; j < targetShape[broadcast_axes[0]]; j++) - { - std::ostringstream ss; - ss << j; - LayerParams copyLP; - copyLP.name = base_name + ss.str(); - copyLP.type = "Identity"; - CV_Assert(layer_id.find(copyLP.name) == layer_id.end()); - input_names.push_back(copyLP.name); - - node_proto.set_input(0, srcName); - node_proto.set_output(0, copyLP.name); - addLayer(copyLP, node_proto); - } - node_proto.clear_input(); - for (int i = 0; i < input_names.size(); i++) - { - node_proto.add_input(input_names[i]); - } - layerParams.set("axis", broadcast_axes[0]); - layerParams.type = "Concat"; - node_proto.set_output(0, layerParams.name); - } - else - CV_Error(Error::StsNotImplemented, "Unsupported Expand op"); - } - else if (layer_type == "Reshape") - { - CV_Assert(node_proto.input_size() == 2 || layerParams.has("shape")); - - if (node_proto.input_size() == 2) { - Mat blob = getBlob(node_proto, 1); - CV_Assert(blob.type() == CV_32SC1); - - layerParams.set("dim", DictValue::arrayInt( - blob.ptr(), blob.total() )); - - if (layer_id.find(node_proto.input(0)) == layer_id.end()) { - std::vector inputs(1, getBlob(node_proto, 0)), outputs; - runLayer(layerParams, inputs, outputs); - addConstant(layerParams.name, outputs[0]); - return; - } - } - else { - DictValue shape = layerParams.get("shape"); - std::vector dim; - for (int j = 0; j < shape.size(); j++) { - dim.push_back(shape.getIntValue(j)); - } - - if (layer_id.find(node_proto.input(0)) == layer_id.end()) { - Mat input = getBlob(node_proto, 0); - Mat out = input.reshape(0, dim); - addConstant(layerParams.name, out); - return; - } - replaceLayerParam(layerParams, "shape", "dim"); - } - } - else if (layer_type == "Pad") - { - layerParams.type = "Padding"; - replaceLayerParam(layerParams, "mode", "type"); - if (node_proto.input_size() == 3 || node_proto.input_size() == 2) - { - // Paddings are in order begin0, begin1, .. beginN, end0, end1, ..., endN. - // We need to shuffle it to begin0, end0, begin1, end1, ... - Mat paddings = getBlob(node_proto, 1).reshape(1, 2); - paddings = paddings.t(); - layerParams.set("paddings", DictValue::arrayInt(paddings.ptr(), paddings.total())); - - if (node_proto.input_size() == 3) - { - Mat value = getBlob(node_proto, 2); - layerParams.set("value", value.ptr()[0]); - } - } - } - else if (layer_type == "Shape") - { - CV_Assert(node_proto.input_size() == 1); - IterShape_t shapeIt = outShapes.find(node_proto.input(0)); - CV_Assert(shapeIt != outShapes.end()); - const MatShape& inpShape = shapeIt->second; - - Mat shapeMat(inpShape.size(), 1, CV_32S); - for (int j = 0; j < inpShape.size(); ++j) - shapeMat.at(j) = inpShape[j]; - shapeMat.dims = 1; - - addConstant(layerParams.name, shapeMat); - return; - } - else if (layer_type == "Cast") - { - if (constBlobs.find(node_proto.input(0)) != constBlobs.end()) - { - Mat blob = getBlob(node_proto, 0); - int type; - switch (layerParams.get("to")) - { - case opencv_onnx::TensorProto_DataType_FLOAT: type = CV_32F; break; - case opencv_onnx::TensorProto_DataType_UINT8: type = CV_8U; break; - case opencv_onnx::TensorProto_DataType_UINT16: type = CV_16U; break; - case opencv_onnx::TensorProto_DataType_FLOAT16: type = CV_16S; break; - case opencv_onnx::TensorProto_DataType_INT8: - case opencv_onnx::TensorProto_DataType_INT16: - case opencv_onnx::TensorProto_DataType_INT32: - case opencv_onnx::TensorProto_DataType_INT64: type = CV_32S; break; - default: type = blob.type(); - } - Mat dst; - blob.convertTo(dst, type); - dst.dims = blob.dims; - addConstant(layerParams.name, dst); - return; - } - else - layerParams.type = "Identity"; - } - else if (layer_type == "ConstantOfShape" || layer_type == "ConstantFill") - { - int depth = CV_32F; - float fill_value; - if (!layerParams.blobs.empty()) - { - CV_Assert(!layerParams.has("value")); - depth = layerParams.blobs[0].depth(); - Mat floats; - layerParams.blobs[0].convertTo(floats, CV_32F); - fill_value = floats.at(0, 0); - } - else - fill_value = layerParams.get("value", 0); - - MatShape inpShape = getBlob(node_proto, 0); - for (int i = 0; i < inpShape.size(); i++) - CV_CheckGT(inpShape[i], 0, ""); - Mat tensor(inpShape.size(), &inpShape[0], depth, Scalar(fill_value)); - addConstant(layerParams.name, tensor); - return; - } - else if (layer_type == "Gather") - { - CV_Assert(node_proto.input_size() == 2); - Mat indexMat = getBlob(node_proto, 1); - CV_Assert_N(indexMat.type() == CV_32S, indexMat.total() == 1); - int index = indexMat.at(0); - int axis = layerParams.get("axis", 0); - - if ((constBlobs.find(node_proto.input(0)) != constBlobs.end())) - { - Mat input = getBlob(node_proto, 0); - Mat out; - std::vector ranges(input.dims, Range::all()); - ranges[axis] = Range(index, index + 1); - - out = input(ranges); - MatShape outShape = shape(out); - if (outShape.size() > 1) - { - outShape.erase(outShape.begin() + axis); - out.reshape(0, outShape); - } else { - out.dims = 1; - } - addConstant(layerParams.name, out); - return; - } - else - { - IterShape_t shapeIt = outShapes.find(node_proto.input(0)); - CV_Assert(shapeIt != outShapes.end()); - MatShape inpShape = shapeIt->second; - - LayerParams sliceLp; - sliceLp.type = "Slice"; - sliceLp.name = inpShape.size() > 1 ? layerParams.name + "/slice" : layerParams.name; - std::vector begin(inpShape.size(), 0); - std::vector end(inpShape.size(), -1); - begin[axis] = index; - end[axis] = index + 1; - - cv::dnn::DictValue paramBegin = cv::dnn::DictValue::arrayInt(begin.data(), begin.size()); - cv::dnn::DictValue paramEnd = cv::dnn::DictValue::arrayInt(end.data(), end.size()); - sliceLp.set("begin", paramBegin); - sliceLp.set("end", paramEnd); - sliceLp.set("has_dynamic_shapes", hasDynamicShapes); - - if (inpShape.size() > 1) - { - opencv_onnx::NodeProto proto; - proto.add_input(node_proto.input(0)); - proto.add_output(sliceLp.name); - addLayer(sliceLp, proto); - - inpShape.erase(inpShape.begin() + axis); - layerParams.type = "Reshape"; - layerParams.set("axis", 0); - layerParams.set("dim", DictValue::arrayInt(&inpShape[0], inpShape.size())); - if (hasDynamicShapes) - { - std::vector dynamicAxes; - std::vector inputIndices; - for (int index = 0; index < inpShape.size(); ++index) - dynamicAxes.push_back(index); - for (int index = 0; index < inpShape.size(); ++index) - inputIndices.push_back(index); - layerParams.set("dynamic_axes", DictValue::arrayInt(dynamicAxes.data(), dynamicAxes.size())); - layerParams.set("input_indices", DictValue::arrayInt(inputIndices.data(), inputIndices.size())); - } - node_proto.set_input(0, sliceLp.name); - } - else - { - layerParams = sliceLp; - } - } - } - else if (layer_type == "Concat") - { - bool hasVariableInps = false; - for (int i = 0; i < node_proto.input_size(); ++i) - { - if (layer_id.find(node_proto.input(i)) != layer_id.end()) - { - hasVariableInps = true; - break; - } - } - - if (!hasVariableInps) - { - std::vector inputs(node_proto.input_size()), concatenated; - // Due constant folding we can get inputs with different number of dimensions - // Insert the missing dimension to inputs - MatShape inputShape; - for (size_t i = 0; i < inputs.size(); ++i) - { - inputs[i] = getBlob(node_proto, i); - if (inputs[i].size.dims() > inputShape.size()) - { - inputShape = shape(inputs[i]); - } - } - - // Concat-1 has default value for axis is 1: https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Concat-1 - int axis = layerParams.get("axis", 1); - for (size_t i = 0; i < inputs.size(); ++i) - { - MatShape targetShape = inputShape; - targetShape[axis] = shape(inputs[i])[axis]; - CV_CheckEQ(total(targetShape), total(shape(inputs[i])), ""); - inputs[i] = inputs[i].reshape(0, targetShape); - } - runLayer(layerParams, inputs, concatenated); - - CV_Assert(concatenated.size() == 1); - addConstant(layerParams.name, concatenated[0]); - return; - } - else - { - for (int i = 0; i < node_proto.input_size(); ++i) - { - if (constBlobs.find(node_proto.input(i)) != constBlobs.end()) - { - LayerParams constParams; - constParams.name = node_proto.input(i); - constParams.type = "Const"; - constParams.blobs.push_back(getBlob(node_proto, i)); - - opencv_onnx::NodeProto proto; - proto.add_output(constParams.name); - addLayer(constParams, proto); - } - } - } - } - else if (layer_type == "Resize") - { - for (int i = 1; i < node_proto.input_size(); i++) - CV_Assert(layer_id.find(node_proto.input(i)) == layer_id.end()); - - if (layerParams.has("coordinate_transformation_mode")) - { - String interp_mode = layerParams.get("coordinate_transformation_mode"); - CV_Assert_N(interp_mode != "tf_crop_and_resize", interp_mode != "tf_half_pixel_for_nn"); - - layerParams.set("align_corners", interp_mode == "align_corners"); - if (layerParams.get("mode") == "linear") - { - layerParams.set("mode", interp_mode == "pytorch_half_pixel" ? - "opencv_linear" : "bilinear"); - } - } - if (layerParams.get("mode") == "linear" && framework_name == "pytorch") - layerParams.set("mode", "opencv_linear"); - - // input = [X, scales], [X, roi, scales] or [x, roi, scales, sizes] - int foundScaleId = hasDynamicShapes ? node_proto.input_size() - 1 - : node_proto.input_size() > 2 ? 2 : 1; - - Mat scales = getBlob(node_proto, foundScaleId); - if (scales.total() == 4) - { - layerParams.set("zoom_factor_y", scales.at(2)); - layerParams.set("zoom_factor_x", scales.at(3)); - } - else - { - const std::string& inputLast = node_proto.input(node_proto.input_size() - 1); - if (constBlobs.find(inputLast) != constBlobs.end()) - { - Mat shapes = getBlob(inputLast); - CV_CheckEQ(shapes.size[0], 4, ""); - CV_CheckEQ(shapes.size[1], 1, ""); - CV_CheckDepth(shapes.depth(), shapes.depth() == CV_32S || shapes.depth() == CV_32F, ""); - if (shapes.depth() == CV_32F) - shapes.convertTo(shapes, CV_32S); - layerParams.set("width", shapes.at(3)); - layerParams.set("height", shapes.at(2)); - } - } - replaceLayerParam(layerParams, "mode", "interpolation"); - } - else if (layer_type == "Upsample") - { - //fused from Resize Subgraph - if (layerParams.has("coordinate_transformation_mode")) - { - String interp_mode = layerParams.get("coordinate_transformation_mode"); - CV_Assert_N(interp_mode != "tf_crop_and_resize", interp_mode != "tf_half_pixel_for_nn"); - - layerParams.set("align_corners", interp_mode == "align_corners"); - if (layerParams.get("mode") == "linear") - { - layerParams.set("mode", interp_mode == "pytorch_half_pixel" ? - "opencv_linear" : "bilinear"); - } - } - if (layerParams.get("mode") == "linear" && framework_name == "pytorch") - layerParams.set("mode", "opencv_linear"); - - layerParams.type = "Resize"; - if (layerParams.has("scales")) - { - // Pytorch layer - DictValue scales = layerParams.get("scales"); - CV_Assert(scales.size() == 4); - layerParams.set("zoom_factor_y", scales.getIntValue(2)); - layerParams.set("zoom_factor_x", scales.getIntValue(3)); - } - else if (layerParams.has("height_scale") && layerParams.has("width_scale")) - { - // Caffe2 layer - replaceLayerParam(layerParams, "height_scale", "zoom_factor_y"); - replaceLayerParam(layerParams, "width_scale", "zoom_factor_x"); - } - else - { - // scales as input - const std::string& input1 = node_proto.input(1); - if (constBlobs.find(input1) != constBlobs.end()) - { - Mat scales = getBlob(input1); - CV_Assert(scales.total() == 4); - layerParams.set("zoom_factor_y", scales.at(2)); - layerParams.set("zoom_factor_x", scales.at(3)); - } - } - replaceLayerParam(layerParams, "mode", "interpolation"); - } - else if (layer_type == "SoftMax" || layer_type == "LogSoftmax") - { - layerParams.type = "Softmax"; - layerParams.set("log_softmax", layer_type == "LogSoftmax"); - } - else if (layer_type == "DetectionOutput") - { - CV_CheckEQ(node_proto.input_size(), 3, ""); - if (constBlobs.find(node_proto.input(2)) != constBlobs.end()) - { - Mat priors = getBlob(node_proto, 2); - - LayerParams constParams; - constParams.name = layerParams.name + "/priors"; - constParams.type = "Const"; - constParams.blobs.push_back(priors); - - opencv_onnx::NodeProto priorsProto; - priorsProto.add_output(constParams.name); - addLayer(constParams, priorsProto); - - node_proto.set_input(2, constParams.name); - } + CALL_MEMBER_FN(*this, iter->second)(layerParams, node_proto); } else { - for (int j = 0; j < node_proto.input_size(); j++) { - if (layer_id.find(node_proto.input(j)) == layer_id.end()) - layerParams.blobs.push_back(getBlob(node_proto, j)); - } + parseCustom(layerParams, node_proto); } - addLayer(layerParams, node_proto); } catch (const cv::Exception& e) { @@ -2194,6 +712,1689 @@ void ONNXImporter::handleNode(const opencv_onnx::NodeProto& node_proto_) } } +void ONNXImporter::parseMaxPool(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +{ + layerParams.type = "Pooling"; + layerParams.set("pool", "MAX"); + layerParams.set("ceil_mode", layerParams.has("pad_mode")); + addLayer(layerParams, node_proto); +} + +void ONNXImporter::parseAveragePool(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +{ + layerParams.type = "Pooling"; + layerParams.set("pool", "AVE"); + layerParams.set("ceil_mode", layerParams.has("pad_mode")); + layerParams.set("ave_pool_padded_area", framework_name == "pytorch"); + addLayer(layerParams, node_proto); +} + +void ONNXImporter::parseReduce(LayerParams &layerParams, const opencv_onnx::NodeProto &node_proto_) +{ + opencv_onnx::NodeProto node_proto = node_proto_; + const std::string& layer_type = node_proto.op_type(); + + CV_Assert(node_proto.input_size() == 1); + layerParams.type = "Pooling"; + String pool; + if (layer_type == "GlobalMaxPool" || layer_type == "ReduceMax") + pool = "MAX"; + else if (layer_type == "ReduceSum") + pool = "SUM"; + else + pool = "AVE"; + layerParams.set("pool", pool); + layerParams.set("global_pooling", !layerParams.has("axes")); + if (layerParams.has("axes") && (layer_type == "ReduceMean" || layer_type == "ReduceSum" || layer_type == "ReduceMax")) + { + MatShape inpShape = outShapes[node_proto.input(0)]; + DictValue axes = layerParams.get("axes"); + bool keepdims = layerParams.get("keepdims"); + MatShape targetShape; + std::vector shouldDelete(inpShape.size(), false); + for (int i = 0; i < axes.size(); i++) { + int axis = normalize_axis(axes.get(i), inpShape.size()); + shouldDelete[axis] = true; + } + for (int axis = 0; axis < inpShape.size(); ++axis){ + if (!shouldDelete[axis]) + targetShape.push_back(inpShape[axis]); + else if (keepdims) + targetShape.push_back(1); + } + + if (inpShape.size() == 3 && axes.size() <= 2) + { + int axis = normalize_axis(axes.get(0), inpShape.size()); + CV_CheckNE(axis, 0, ""); + + LayerParams reshapeLp; + reshapeLp.name = layerParams.name + "/reshape"; + reshapeLp.type = "Reshape"; + CV_Assert(layer_id.find(reshapeLp.name) == layer_id.end()); + reshapeLp.set("axis", 0); + reshapeLp.set("num_axes", 1); + int newShape[] = {1, -1}; + reshapeLp.set("dim", DictValue::arrayInt(&newShape[0], 2)); + + opencv_onnx::NodeProto proto; + proto.add_input(node_proto.input(0)); + proto.add_output(reshapeLp.name); + addLayer(reshapeLp, proto); + + LayerParams avgLp; + avgLp.name = layerParams.name + "/avg"; + avgLp.type = "Pooling"; + CV_Assert(layer_id.find(avgLp.name) == layer_id.end()); + avgLp.set("pool", pool); + if (axes.size() == 2) + { + CV_CheckEQ(normalize_axis(axes.get(0), inpShape.size()), 1, "Unsupported mode"); + CV_CheckEQ(normalize_axis(axes.get(1), inpShape.size()), 2, "Unsupported mode"); + avgLp.set("global_pooling", true); + } + else + { + avgLp.set(axis == 2 ? "global_pooling_w" : "global_pooling_h", true); + avgLp.set(axis == 2 ? "kernel_h" : "kernel_w", 1); + } + + node_proto.set_input(0, reshapeLp.name); + node_proto.set_output(0, avgLp.name); + addLayer(avgLp, node_proto); + } + else + { + if (inpShape.size() != 4 && inpShape.size() != 5) + CV_Error(Error::StsNotImplemented, "Unsupported input shape of " + layer_type + " operation."); + + CV_Assert(axes.size() <= inpShape.size() - 2); + std::vector kernel_size(inpShape.size() - 2, 1); + if (axes.size() == 1 && (normalize_axis(axes.get(0), inpShape.size()) <= 1)) + { + int axis = normalize_axis(axes.get(0), inpShape.size()); + MatShape newShape = inpShape; + newShape[axis + 1] = total(newShape, axis + 1); + newShape.resize(axis + 2); + newShape.insert(newShape.begin(), 2 - axis, 1); + + LayerParams reshapeLp; + reshapeLp.type = "Reshape"; + reshapeLp.name = layerParams.name + "/reshape"; + CV_Assert(layer_id.find(reshapeLp.name) == layer_id.end()); + reshapeLp.set("dim", DictValue::arrayInt(&newShape[0], newShape.size())); + + node_proto.set_output(0, reshapeLp.name); + addLayer(reshapeLp, node_proto); + + kernel_size.resize(2); + kernel_size[0] = inpShape[axis]; + node_proto.set_input(0, node_proto.output(0)); + } + else + { + for (int i = 0; i < axes.size(); i++) { + int axis = normalize_axis(axes.get(i), inpShape.size()); + CV_Assert_N(axis >= 2 + i, axis < inpShape.size()); + kernel_size[axis - 2] = inpShape[axis]; + } + } + + LayerParams poolLp = layerParams; + poolLp.name = layerParams.name + "/avg"; + CV_Assert(layer_id.find(poolLp.name) == layer_id.end()); + poolLp.set("kernel_size", DictValue::arrayInt(&kernel_size[0], kernel_size.size())); + + node_proto.set_output(0, poolLp.name); + addLayer(poolLp, node_proto); + } + + layerParams.type = "Reshape"; + layerParams.set("dim", DictValue::arrayInt(&targetShape[0], targetShape.size())); + + node_proto.set_input(0, node_proto.output(0)); + node_proto.set_output(0, layerParams.name); + } + else if (!layerParams.has("axes") && (layer_type == "ReduceMean" || layer_type == "ReduceSum" || layer_type == "ReduceMax")) + { + CV_CheckEQ(layerParams.get("keepdims"), 0, "layer only supports keepdims = false"); + LayerParams reshapeLp; + reshapeLp.name = layerParams.name + "/reshape"; + reshapeLp.type = "Reshape"; + CV_Assert(layer_id.find(reshapeLp.name) == layer_id.end()); + int newShape[] = {1, 1, 1, -1}; + reshapeLp.set("dim", DictValue::arrayInt(&newShape[0], 4)); + + opencv_onnx::NodeProto proto; + proto.add_input(node_proto.input(0)); + proto.add_output(reshapeLp.name); + addLayer(reshapeLp, proto); + + LayerParams poolLp = layerParams; + poolLp.name = layerParams.name + "/pool"; + CV_Assert(layer_id.find(poolLp.name) == layer_id.end()); + + node_proto.set_input(0, reshapeLp.name); + node_proto.set_output(0, poolLp.name); + addLayer(poolLp, node_proto); + + layerParams.type = "Reshape"; + int targetShape[] = {1}; + layerParams.set("dim", DictValue::arrayInt(&targetShape[0], 1)); + + node_proto.set_input(0, node_proto.output(0)); + node_proto.set_output(0, layerParams.name); + } + addLayer(layerParams, node_proto); +} + +void ONNXImporter::parseSlice(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +{ + int axis = 0; + std::vector begin; + std::vector end; + std::vector steps; + int inp_size = node_proto.input_size(); + + if (inp_size == 1) + { + if (layerParams.has("axes")) { + DictValue axes = layerParams.get("axes"); + for (int i = 1; i < axes.size(); ++i) { + CV_Assert(axes.get(i - 1) == axes.get(i) - 1); + } + axis = axes.get(0); + } + + DictValue starts = layerParams.get("starts"); + DictValue ends = layerParams.get("ends"); + CV_Assert(starts.size() == ends.size()); + + if (axis > 0) { + begin.resize(axis, 0); + end.resize(axis, -1); + } + for (int i = 0; i < starts.size(); ++i) + { + begin.push_back(starts.get(i)); + int finish = ends.get(i); + end.push_back((finish < 0) ? --finish : finish); // numpy doesn't include last dim + } + } else { // inp_size > 1 + CV_Assert(inp_size >= 3); + for (int i = 1; i < inp_size; i++) { + CV_Assert(constBlobs.find(node_proto.input(i)) != constBlobs.end()); + } + Mat start_blob = getBlob(node_proto, 1); + Mat end_blob = getBlob(node_proto, 2); + CV_Assert(start_blob.total() == end_blob.total()); + + if (inp_size > 3) { + Mat axes_blob = getBlob(node_proto, 3); + const int* axes = (int*)axes_blob.data; + for (int i = 1; i < axes_blob.total(); ++i) { + CV_Assert(axes[i - 1] == axes[i] - 1); + } + axis = axes[0]; + } + + const int* starts = start_blob.ptr(); + const int* ends = end_blob.ptr(); + if (axis > 0) { + begin.resize(axis, 0); + end.resize(axis, -1); + } + std::copy(starts, starts + start_blob.total(), std::back_inserter(begin)); + for (int i = 0; i < end_blob.total(); ++i) + { + int finish = ends[i]; + end.push_back((finish < 0) ? --finish : finish); // numpy doesn't include last dim + } + + if (inp_size == 5) { + CV_Assert(constBlobs.find(node_proto.input(4)) != constBlobs.end()); + Mat step_blob = getBlob(node_proto, 4); + const int* steps_ptr = step_blob.ptr(); + + if (axis > 0) + steps.resize(axis, 1); + + std::copy(steps_ptr, steps_ptr + step_blob.total(), std::back_inserter(steps)); + + // Very strange application for Slice op with tensor reversing. + // We just workaround it for 2d constants. + if (constBlobs.find(node_proto.input(0)) != constBlobs.end() && + axis == 0 && + start_blob.at(0) == -1 && step_blob.at(0) == -1 && + end_blob.at(0) == std::numeric_limits::min()) + { + Mat inp = getBlob(node_proto, 0); + if (inp.dims == 2) + { + Mat flipped; + flip(inp, flipped, 0); + addConstant(layerParams.name, flipped); + return; + } + } + } + } + layerParams.set("begin", DictValue::arrayInt(&begin[0], begin.size())); + layerParams.set("end", DictValue::arrayInt(&end[0], end.size())); + layerParams.set("axis", axis); + + if (!steps.empty()) + layerParams.set("steps", DictValue::arrayInt(&steps[0], steps.size())); + + if (constBlobs.find(node_proto.input(0)) != constBlobs.end()) + { + Mat inp = getBlob(node_proto, 0); + std::vector inputs, sliced; + inputs.push_back(inp); + runLayer(layerParams, inputs, sliced); + CV_Assert(sliced.size() == 1); + addConstant(layerParams.name, sliced[0]); + return; + } + addLayer(layerParams, node_proto); +} + +void ONNXImporter::parseSplit(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +{ + if (layerParams.has("split")) + { + DictValue splits = layerParams.get("split"); + const int numSplits = splits.size(); + CV_Assert(numSplits > 1); + + std::vector slicePoints(numSplits - 1, splits.get(0)); + for (int i = 1; i < splits.size() - 1; ++i) + { + slicePoints[i] = slicePoints[i - 1] + splits.get(i - 1); + } + layerParams.set("slice_point", DictValue::arrayInt(&slicePoints[0], slicePoints.size())); + } + else + { + layerParams.set("num_split", node_proto.output_size()); + } + layerParams.type = "Slice"; + addLayer(layerParams, node_proto); +} + +void ONNXImporter::parseBias(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto_) +{ + opencv_onnx::NodeProto node_proto = node_proto_; + const std::string& layer_type = node_proto.op_type(); + bool isSub = layer_type == "Sub"; + CV_CheckEQ(node_proto.input_size(), 2, ""); + bool is_const_0 = layer_id.find(node_proto.input(0)) == layer_id.end(); + bool is_const_1 = layer_id.find(node_proto.input(1)) == layer_id.end(); + if (is_const_0 && is_const_1) + { + Mat blob_0 = getBlob(node_proto, 0); + Mat blob_1 = getBlob(node_proto, 1); + CV_Assert(blob_0.size == blob_1.size); + Mat output = isSub ? (blob_0 - blob_1) : (blob_0 + blob_1); + addConstant(layerParams.name, output); + return; + } + else if (is_const_0 || is_const_1) + { + int const_blob_id = is_const_0 ? 0 : 1; + Mat blob = getBlob(node_proto, const_blob_id); + int blob_total = blob.total(); + if (blob_total == 1) { + layerParams.type = "Power"; + layerParams.set("shift", (isSub ? -1 : 1) * blob.ptr()[0]); + } + else { + MatShape inpShape = outShapes[node_proto.input(1 - const_blob_id)]; + if (shape(blob) == inpShape) + { + LayerParams constParams; + constParams.name = layerParams.name + "/const"; + constParams.type = "Const"; + constParams.blobs.push_back((isSub ? -1 : 1) * blob); + int id; + if (DNN_DIAGNOSTICS_RUN) + id = utilNet.addLayer(constParams.name, constParams.type, constParams); + else + id = dstNet.addLayer(constParams.name, constParams.type, constParams); + layer_id.insert(std::make_pair(constParams.name, LayerInfo(id, 0))); + outShapes[constParams.name] = shape(blob); + + layerParams.type = "Eltwise"; + node_proto.set_input(const_blob_id, constParams.name); + } + else + { + layerParams.type = "Scale"; + layerParams.set("bias_term", true); + int axis = 1; + for (int i = 0; i < graph_proto.initializer_size(); i++) + { + opencv_onnx::TensorProto tensor_proto = graph_proto.initializer(i); + if (tensor_proto.name() == node_proto.input(const_blob_id)) + { + axis = inpShape.size() - tensor_proto.dims_size(); + break; + } + } + layerParams.set("axis", axis); + blob = blob.reshape(1, 1); + layerParams.blobs.push_back((isSub ? -1 : 1) * blob); + } + } + } + else if (outShapes[node_proto.input(0)] == outShapes[node_proto.input(1)]) + { + layerParams.type = "Eltwise"; + if (isSub) + { + static float subCoeffs[] = {1.f, -1.f}; + layerParams.set("coeff", DictValue::arrayReal(subCoeffs, 2)); + } + } + else + { + if (isSub) + { + LayerParams powerParams; + powerParams.name = layerParams.name + "/neg"; + powerParams.type = "Power"; + powerParams.set("scale", -1); + + int id; + //Create Power layer + if (DNN_DIAGNOSTICS_RUN) + id = utilNet.addLayer(powerParams.name, powerParams.type, powerParams); + else + id = dstNet.addLayer(powerParams.name, powerParams.type, powerParams); + //Connect to input + IterLayerId_t layerId = layer_id.find(node_proto.input(1)); + CV_Assert(layerId != layer_id.end()); + if (DNN_DIAGNOSTICS_RUN) + utilNet.connect(layerId->second.layerId, layerId->second.outputId, id, 0); + else + dstNet.connect(layerId->second.layerId, layerId->second.outputId, id, 0); + //Add shape + layer_id.insert(std::make_pair(powerParams.name, LayerInfo(id, 0))); + outShapes[powerParams.name] = outShapes[node_proto.input(1)]; + + //Replace input to Power + node_proto.set_input(1, powerParams.name); + } + layerParams.type = "Scale"; + layerParams.set("bias_term", true); + } + addLayer(layerParams, node_proto); +} + +void ONNXImporter::parsePow(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +{ + if (layer_id.find(node_proto.input(1)) != layer_id.end()) + CV_Error(Error::StsNotImplemented, "Unsupported Pow op with variable power"); + + Mat blob = getBlob(node_proto, 1); + if (blob.total() != 1) + CV_Error(Error::StsNotImplemented, "Pow op supports only scalar power"); + + blob.convertTo(blob, CV_32F); + layerParams.type = "Power"; + layerParams.set("power", blob.ptr()[0]); + addLayer(layerParams, node_proto); +} + +void ONNXImporter::parseMax(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +{ + layerParams.type = "Eltwise"; + layerParams.set("operation", "max"); + addLayer(layerParams, node_proto); +} + +void ONNXImporter::parseNeg(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +{ + layerParams.type = "Power"; + layerParams.set("scale", -1); + addLayer(layerParams, node_proto); +} + +void ONNXImporter::parseConstant(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +{ + CV_Assert(node_proto.input_size() == 0); + CV_Assert(layerParams.blobs.size() == 1); + addConstant(layerParams.name, layerParams.blobs[0]); +} + +void ONNXImporter::parseLSTM(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto_) +{ + opencv_onnx::NodeProto node_proto = node_proto_; + LayerParams lstmParams = layerParams; + lstmParams.name += "/lstm"; + + // https://pytorch.org/docs/stable/nn.html#lstm + CV_Assert(node_proto.input_size() == 7); + Mat Wx = getBlob(node_proto, 1); + Mat Wh = getBlob(node_proto, 2); + Mat b = getBlob(node_proto, 3); + Mat h0 = getBlob(node_proto, 5); + Mat c0 = getBlob(node_proto, 6); + + b = b.reshape(1, b.size[0]); + + const int numHidden = lstmParams.get("hidden_size"); + const int numDirs = Wx.size[0]; // Is 1 for forward only and 2 for bidirectional LSTM. + const int numFeatures = Wx.size[2]; + Mat bx = b.colRange(0, b.cols / 2); + Mat bh = b.colRange(b.cols / 2, b.cols); + b = bx + bh; + + // IFGO->IGFO + for (int k = 0; k < numDirs; ++k) + { + float* WxData = Wx.ptr(k); + float* WhData = Wh.ptr(k); + float* biasData = b.ptr(k); + for (int j = 0; j < numHidden; ++j) + { + for (int i = 0; i < numFeatures; ++i) + { + std::swap(WxData[(numHidden + j) * numFeatures + i], + WxData[(numHidden * 2 + j) * numFeatures + i]); + } + for (int i = 0; i < numHidden; ++i) + { + std::swap(WhData[(numHidden + j) * numHidden + i], + WhData[(numHidden * 2 + j) * numHidden + i]); + } + std::swap(biasData[numHidden + j], biasData[numHidden * 2 + j]); + } + } + Wx = Wx.reshape(1, Wx.size[0] * Wx.size[1]); + Wh = Wh.reshape(1, Wh.size[0] * Wh.size[1]); + h0 = h0.reshape(1, h0.size[0] * h0.size[1]); + c0 = c0.reshape(1, c0.size[0] * c0.size[1]); + + lstmParams.blobs.resize(5); + lstmParams.blobs[0] = Wh; + lstmParams.blobs[1] = Wx; + lstmParams.blobs[2] = b; + lstmParams.blobs[3] = h0; + lstmParams.blobs[4] = c0; + lstmParams.set("bidirectional", lstmParams.get("direction", "") == "bidirectional"); + + node_proto.set_output(0, lstmParams.name); // set different name so output shapes will be registered on that name + addLayer(lstmParams, node_proto); + + MatShape lstmShape = outShapes[node_proto.output(0)]; + + // Add fake 1 as it is done in ONNX + lstmShape.insert(lstmShape.begin() + 1, 1); + + layerParams.type = "Reshape"; + layerParams.set("dim", DictValue::arrayInt(&lstmShape[0], lstmShape.size())); + node_proto.set_input(0, lstmParams.name); // redirect input to LSTM + node_proto.set_output(0, layerParams.name); // keep origin LSTM's name + addLayer(layerParams, node_proto); +} + +void ONNXImporter::parseImageScaler(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +{ + const float scale = layerParams.has("scale") ? layerParams.get("scale") : 1.0f; + layerParams.erase("scale"); + + if (layerParams.has("bias")) + { + layerParams.type = "Scale"; + layerParams.blobs.push_back( + Mat(Size(1, layerParams.get("bias").size()), CV_32FC1, scale)); + + layerParams.set("bias_term", true); + Mat bias(1, layerParams.get("bias").size(), CV_32FC1); + for (int j = 0; j < bias.total(); j++) { + bias.at(0, j) = layerParams.get("bias").getRealValue(j); + } + layerParams.blobs.push_back(bias); + layerParams.erase("bias"); + } + else { + layerParams.set("scale", scale); + layerParams.type = "Power"; + } + addLayer(layerParams, node_proto); +} + +void ONNXImporter::parseClip(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +{ + layerParams.type = "ReLU6"; + replaceLayerParam(layerParams, "min", "min_value"); + replaceLayerParam(layerParams, "max", "max_value"); + addLayer(layerParams, node_proto); +} + +void ONNXImporter::parseLeakyRelu(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +{ + layerParams.type = "ReLU"; + replaceLayerParam(layerParams, "alpha", "negative_slope"); + addLayer(layerParams, node_proto); +} + +void ONNXImporter::parseRelu(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +{ + layerParams.type = "ReLU"; + addLayer(layerParams, node_proto); +} + +void ONNXImporter::parseElu(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +{ + layerParams.type = "ELU"; + addLayer(layerParams, node_proto); +} + +void ONNXImporter::parseTanh(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +{ + layerParams.type = "TanH"; + addLayer(layerParams, node_proto); +} + +void ONNXImporter::parsePRelu(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +{ + layerParams.type = "PReLU"; + layerParams.blobs.push_back(getBlob(node_proto, 1)); + addLayer(layerParams, node_proto); +} + +void ONNXImporter::parseLRN(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +{ + replaceLayerParam(layerParams, "size", "local_size"); + addLayer(layerParams, node_proto); +} + +void ONNXImporter::parseInstanceNormalization(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto_) +{ + opencv_onnx::NodeProto node_proto = node_proto_; + if (node_proto.input_size() != 3) + CV_Error(Error::StsNotImplemented, + "Expected input, scale, bias"); + + layerParams.blobs.resize(4); + layerParams.blobs[2] = getBlob(node_proto, 1); // weightData + layerParams.blobs[3] = getBlob(node_proto, 2); // biasData + layerParams.set("has_bias", true); + layerParams.set("has_weight", true); + + // Get number of channels in input + int size = layerParams.blobs[2].total(); + layerParams.blobs[0] = Mat::zeros(size, 1, CV_32F); // mean + layerParams.blobs[1] = Mat::ones(size, 1, CV_32F); // std + + LayerParams mvnParams; + mvnParams.name = layerParams.name + "/MVN"; + mvnParams.type = "MVN"; + mvnParams.set("eps", layerParams.get("epsilon")); + layerParams.erase("epsilon"); + + //Create MVN layer + int id; + if (DNN_DIAGNOSTICS_RUN) + id = utilNet.addLayer(mvnParams.name, mvnParams.type, mvnParams); + else + id = dstNet.addLayer(mvnParams.name, mvnParams.type, mvnParams); + //Connect to input + IterLayerId_t layerId = layer_id.find(node_proto.input(0)); + CV_Assert(layerId != layer_id.end()); + if (DNN_DIAGNOSTICS_RUN) + utilNet.connect(layerId->second.layerId, layerId->second.outputId, id, 0); + else + dstNet.connect(layerId->second.layerId, layerId->second.outputId, id, 0); + //Add shape + layer_id.insert(std::make_pair(mvnParams.name, LayerInfo(id, 0))); + outShapes[mvnParams.name] = outShapes[node_proto.input(0)]; + + //Replace Batch Norm's input to MVN + node_proto.set_input(0, mvnParams.name); + layerParams.type = "BatchNorm"; + addLayer(layerParams, node_proto); +} + +void ONNXImporter::parseBatchNormalization(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +{ + if (node_proto.input_size() != 5) + CV_Error(Error::StsNotImplemented, + "Expected input, scale, bias, mean and var"); + + layerParams.type = "BatchNorm"; + replaceLayerParam(layerParams, "epsilon", "eps"); + replaceLayerParam(layerParams, "spatial", "use_global_stats"); + + Mat meanData = getBlob(node_proto, 3); + Mat stdData = getBlob(node_proto, 4); + + layerParams.blobs.push_back(meanData); + layerParams.blobs.push_back(stdData); + + if (!node_proto.input(1).empty()) { + layerParams.set("has_weight", true); + layerParams.blobs.push_back(getBlob(node_proto, 1)); // weightData + } else { + layerParams.set("has_weight", false); + } + + if (!node_proto.input(2).empty()) { + layerParams.set("has_bias", true); + layerParams.blobs.push_back(getBlob(node_proto, 2)); // biasData + } else { + layerParams.set("has_bias", false); + } + addLayer(layerParams, node_proto); +} + +void ONNXImporter::parseGemm(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +{ + CV_Assert(node_proto.input_size() >= 2); + layerParams.type = "InnerProduct"; + Mat weights = getBlob(node_proto, 1); + int ind_num_out = 0; + if (layerParams.has("transB") && !layerParams.get("transB")) { + transpose(weights, weights); + ind_num_out = 1; + } + layerParams.blobs.push_back(weights); + + if (node_proto.input_size() == 3) { + Mat bias = getBlob(node_proto, 2); + layerParams.blobs.push_back(bias); + } + if (constBlobs.find(node_proto.input(0)) != constBlobs.end()) + { + Mat inputBuf = getBlob(node_proto, 0); + + LayerParams constParams; + constParams.name = node_proto.input(0); + constParams.type = "Const"; + constParams.blobs.push_back(inputBuf); + + opencv_onnx::NodeProto proto; + proto.add_output(constParams.name); + addLayer(constParams, proto); + } + + layerParams.set("num_output", layerParams.blobs[0].size[ind_num_out]); + layerParams.set("bias_term", node_proto.input_size() == 3); + addLayer(layerParams, node_proto); +} + +void ONNXImporter::parseMatMul(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +{ + CV_Assert(node_proto.input_size() == 2); + layerParams.type = "InnerProduct"; + layerParams.set("bias_term", false); + CV_Assert(constBlobs.find(node_proto.input(0)) == constBlobs.end()); + int firstInpDims = outShapes[node_proto.input(0)].size(); + int secondInpDims; + + if (constBlobs.find(node_proto.input(1)) != constBlobs.end()) + { + Mat blob = getBlob(node_proto, 1); + secondInpDims = blob.dims; + layerParams.blobs.push_back(blob.t()); + layerParams.set("num_output", layerParams.blobs[0].size[0]); + } else { + secondInpDims = outShapes[node_proto.input(1)].size(); + } + layerParams.set("axis", firstInpDims - secondInpDims + 1); + addLayer(layerParams, node_proto); +} + +void ONNXImporter::parseMul(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto_) +{ + opencv_onnx::NodeProto node_proto = node_proto_; + const std::string& layer_type = node_proto.op_type(); + CV_Assert(node_proto.input_size() == 2); + + bool isDiv = layer_type == "Div"; + int constId = -1; + bool haveVariables = false; + for (int i = 0; i < 2; ++i) + { + if (constBlobs.find(node_proto.input(i)) != constBlobs.end()) + constId = i; + else + haveVariables = true; + } + if (constId != -1 && haveVariables) + { + Mat blob = getBlob(node_proto, constId); + blob = blob.reshape(1, 1); + if (blob.total() == 1) { + float blob_value = blob.ptr()[0]; + float coeff = isDiv ? 1.0 / blob_value : blob_value; + layerParams.set("scale", coeff); + layerParams.type = "Power"; + } + else { + if (isDiv) + divide(1.0, blob, blob); + layerParams.blobs.push_back(blob); + layerParams.type = "Scale"; + } + } + else if (!haveVariables) + { + Mat inp0 = getBlob(node_proto, 0); + Mat inp1 = getBlob(node_proto, 1); + + if (inp0.size != inp1.size && (inp0.total() != 1 || inp1.total() != 1)) + CV_Error_(Error::StsNotImplemented, ("Different shapes case is not supported with constant inputs: %s", layer_type.c_str())); + + if (inp0.total() == 1 && inp1.total() == 1 && inp0.dims != inp1.dims) + { + if (inp0.dims < inp1.dims) + { + inp0 = inp0.reshape(1, inp1.dims, inp1.size); + inp0.dims = inp1.dims; + } + else + { + inp1 = inp1.reshape(1, inp0.dims, inp0.size); + inp1.dims = inp0.dims; + } + } + + Mat out; + if (inp0.total() != inp1.total()) + { + if (inp0.total() == 1) + { + float inp0_value = inp0.ptr()[0]; + float coeff = isDiv ? 1.0 / inp0_value : inp0_value; + multiply(inp1, coeff, out); + } + else + { + float inp1_value = inp1.ptr()[0]; + float coeff = isDiv ? 1.0 / inp1_value : inp1_value; + multiply(inp0, coeff, out); + } + + } + else + { + out = isDiv ? inp0 / inp1 : inp0.mul(inp1); + } + + if (inp0.dims == 1 && inp1.dims == 1) + out.dims = 1; // to workaround dims == 1 + addConstant(layerParams.name, out); + return; + } + else if (outShapes[node_proto.input(0)] == outShapes[node_proto.input(1)]) + { + layerParams.type = "Eltwise"; + layerParams.set("operation", isDiv ? "div" : "prod"); + } + else + { + // Scale layer allocate output with the first input shape + if (total(outShapes[node_proto.input(0)]) < total(outShapes[node_proto.input(1)])) + { + opencv_onnx::NodeProto proto; + proto.add_input(node_proto.input(1)); + proto.add_input(node_proto.input(0)); + proto.add_output(layerParams.name); + node_proto = proto; + } + + if (isDiv) + { + LayerParams powerParams; + powerParams.name = layerParams.name + "/inv"; + powerParams.type = "Power"; + powerParams.set("power", -1); + + int id; + //Create Power layer + if (DNN_DIAGNOSTICS_RUN) + id = utilNet.addLayer(powerParams.name, powerParams.type, powerParams); + else + id = dstNet.addLayer(powerParams.name, powerParams.type, powerParams); + //Connect to input + IterLayerId_t layerId = layer_id.find(node_proto.input(1)); + CV_Assert(layerId != layer_id.end()); + if (DNN_DIAGNOSTICS_RUN) + utilNet.connect(layerId->second.layerId, layerId->second.outputId, id, 0); + else + dstNet.connect(layerId->second.layerId, layerId->second.outputId, id, 0); + //Add shape + layer_id.insert(std::make_pair(powerParams.name, LayerInfo(id, 0))); + outShapes[powerParams.name] = outShapes[node_proto.input(1)]; + + //Replace input to Power + node_proto.set_input(1, powerParams.name); + } + layerParams.type = "Scale"; + } + addLayer(layerParams, node_proto); +} + +void ONNXImporter::parseConv(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto_) +{ + opencv_onnx::NodeProto node_proto = node_proto_; + CV_Assert(node_proto.input_size() >= 2); + layerParams.type = "Convolution"; + for (int j = 1; j < node_proto.input_size(); j++) { + if (constBlobs.find(node_proto.input(j)) != constBlobs.end()) + { + layerParams.blobs.push_back(getBlob(node_proto, j)); + } + } + int outCn = layerParams.blobs.empty() ? outShapes[node_proto.input(1)][0] : layerParams.blobs[0].size[0]; + layerParams.set("num_output", outCn); + + // Check for asymmetric padding in Conv2D + if (layerParams.has("pad")) + { + bool asymmetricPadding = false; + DictValue pads = layerParams.get("pad"); + const int dims = pads.size() / 2; + for (int i = 0; i < dims; ++i) + { + if (pads.get(i) != pads.get(i + dims)) + { + asymmetricPadding = true; + break; + } + } + if (asymmetricPadding && pads.size() == 4) // [pad_t, pad_l, pad_b, pad_r] + { + layerParams.erase("pad"); + // No paddings required for N, C axis + std::vector paddings(4, 0); + // Add paddings for H, W axis + for (int i = 0; i < dims; ++i) + { + paddings.push_back(pads.get(i)); + paddings.push_back(pads.get(dims + i)); + } + LayerParams padLp; + padLp.name = layerParams.name + "/pad"; + padLp.type = "Padding"; + padLp.set("paddings", DictValue::arrayInt(&paddings[0], paddings.size())); + + opencv_onnx::NodeProto proto; + proto.add_input(node_proto.input(0)); + proto.add_output(padLp.name); + + addLayer(padLp, proto); + node_proto.set_input(0, padLp.name); + } + } + addLayer(layerParams, node_proto); +} + +void ONNXImporter::parseConvTranspose(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +{ + CV_Assert(node_proto.input_size() >= 2); + layerParams.type = "Deconvolution"; + for (int j = 1; j < node_proto.input_size(); j++) { + layerParams.blobs.push_back(getBlob(node_proto, j)); + } + layerParams.set("num_output", layerParams.blobs[0].size[1] * layerParams.get("group", 1)); + layerParams.set("bias_term", node_proto.input_size() == 3); + + if (!layerParams.has("kernel_size")) + CV_Error(Error::StsNotImplemented, + "Required attribute 'kernel_size' is not present."); + + if (layerParams.has("output_shape")) + { + const DictValue& outShape = layerParams.get("output_shape"); + DictValue strides = layerParams.get("stride"); + DictValue kernel = layerParams.get("kernel_size"); + + String padMode; + std::vector adjust_pads; + if (layerParams.has("pad_mode")) + { + padMode = toUpperCase(layerParams.get("pad_mode")); + if (padMode != "SAME" && padMode != "VALID") + CV_Error(Error::StsError, "Unsupported padding mode " + padMode); + + for (int i = 0; i < strides.size(); i++) + { + int sz = outShape.get(2 + i); + int stride = strides.get(i); + adjust_pads.push_back(padMode == "SAME"? (sz - 1) % stride : + (sz - kernel.get(i)) % stride); + } + layerParams.set("adj", DictValue::arrayInt(&adjust_pads[0], adjust_pads.size())); + } + } + else if (layerParams.has("output_padding")) + { + replaceLayerParam(layerParams, "output_padding", "adj"); + } + addLayer(layerParams, node_proto); +} + +void ONNXImporter::parseTranspose(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +{ + layerParams.type = "Permute"; + replaceLayerParam(layerParams, "perm", "order"); + + CV_Assert(node_proto.input_size() == 1); + if (constBlobs.find(node_proto.input(0)) != constBlobs.end()) + { + std::vector inputs(1, getBlob(node_proto, 0)), transposed; + runLayer(layerParams, inputs, transposed); + CV_Assert(transposed.size() == 1); + addConstant(layerParams.name, transposed[0]); + return; + } + addLayer(layerParams, node_proto); +} + +void ONNXImporter::parseSqueeze(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +{ + CV_Assert_N(node_proto.input_size() == 1, layerParams.has("axes")); + DictValue axes_dict = layerParams.get("axes"); + MatShape inpShape = outShapes[node_proto.input(0)]; + + std::vector maskedAxes(inpShape.size(), false); + for (int i = 0; i < axes_dict.size(); ++i) + { + int axis = axes_dict.getIntValue(i); + CV_CheckLE(axis, static_cast(inpShape.size()), "Squeeze axis"); + maskedAxes[axis] = inpShape[axis] == 1; + } + MatShape outShape; + for (int i = 0; i < inpShape.size(); ++i) + { + if (!maskedAxes[i]) + outShape.push_back(inpShape[i]); + } + if (outShape.size() != inpShape.size()) + { + layerParams.type = "Reshape"; + layerParams.set("dim", DictValue::arrayInt(&outShape[0], outShape.size())); + if (hasDynamicShapes) + { + std::vector dynamicAxes; + std::vector inputIndices; + for (int index = 0; index < inpShape.size(); ++index) + { + if (!maskedAxes[index]) + inputIndices.push_back(index); + } + for (int index = 0; index < outShape.size(); ++index) + dynamicAxes.push_back(index); + layerParams.set("dynamic_axes", DictValue::arrayInt(dynamicAxes.data(), dynamicAxes.size())); + layerParams.set("input_indices", DictValue::arrayInt(inputIndices.data(), inputIndices.size())); + } + } + else + layerParams.type = "Identity"; + + if (constBlobs.find(node_proto.input(0)) != constBlobs.end()) + { + Mat inp = getBlob(node_proto, 0); + Mat out = inp.reshape(1, outShape); + out.dims = outShape.size(); // to workaround dims == 1 + addConstant(layerParams.name, out); + return; + } + addLayer(layerParams, node_proto); +} + +void ONNXImporter::parseFlatten(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +{ + CV_CheckEQ(node_proto.input_size(), 1, ""); + if (constBlobs.find(node_proto.input(0)) != constBlobs.end()) + { + Mat input = getBlob(node_proto, 0); + int axis = normalize_axis(layerParams.get("axis", 1), input.dims); + + std::vector out_size(&input.size[0], &input.size[0] + axis); + out_size.push_back(input.total(axis)); + Mat output = input.reshape(1, out_size); + addConstant(layerParams.name, output); + return; + } + addLayer(layerParams, node_proto); +} + +void ONNXImporter::parseUnsqueeze(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +{ + CV_Assert(node_proto.input_size() == 1); + DictValue axes = layerParams.get("axes"); + if (constBlobs.find(node_proto.input(0)) != constBlobs.end()) + { + // Constant input. + Mat input = getBlob(node_proto, 0); + + std::vector dims; + for (int j = 0; j < input.dims; j++) { + dims.push_back(input.size[j]); + } + CV_Assert(axes.getIntValue(axes.size()-1) <= dims.size()); + for (int j = 0; j < axes.size(); j++) { + dims.insert(dims.begin() + axes.getIntValue(j), 1); + } + + Mat out = input.reshape(0, dims); + addConstant(layerParams.name, out); + return; + } + + // Variable input. + if (axes.size() != 1) + CV_Error(Error::StsNotImplemented, "Multidimensional unsqueeze"); + + MatShape inpShape = outShapes[node_proto.input(0)]; + int axis = axes.getIntValue(0); + CV_Assert(0 <= axis && axis <= inpShape.size()); + std::vector outShape = inpShape; + outShape.insert(outShape.begin() + axis, 1); + layerParams.type = "Reshape"; + layerParams.set("dim", DictValue::arrayInt(&outShape[0], outShape.size())); + if (hasDynamicShapes) + { + std::vector dynamicAxes; + std::vector inputIndices; + for (int index = 0; index < outShape.size(); ++index) { + if (index != axis) + dynamicAxes.push_back(index); + } + for (int index = 0; index < inpShape.size(); ++index) + inputIndices.push_back(index); + layerParams.set("dynamic_axes", DictValue::arrayInt(dynamicAxes.data(), dynamicAxes.size())); + layerParams.set("input_indices", DictValue::arrayInt(inputIndices.data(), inputIndices.size())); + } + addLayer(layerParams, node_proto); +} + +void ONNXImporter::parseExpand(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto_) +{ + opencv_onnx::NodeProto node_proto = node_proto_; + CV_CheckEQ(node_proto.input_size(), 2, ""); + const std::string& input0 = node_proto.input(0); + const std::string& input1 = node_proto.input(1); + Mat newShapeMat = getBlob(input1); + MatShape targetShape(newShapeMat.ptr(), newShapeMat.ptr() + newShapeMat.total()); + + MatShape inpShape; + bool haveVariables = constBlobs.find(input0) == constBlobs.end(); + if (haveVariables) + { + IterShape_t shapeIt = outShapes.find(input0); + CV_Assert(shapeIt != outShapes.end()); + inpShape = shapeIt->second; + } + else + { + inpShape = shape(getBlob(input0)); + } + + String srcName = input0; + // Unsqueeze and repeat along new axis + if (targetShape.size() == inpShape.size() + 1) + { + for (int i = 0; i < targetShape.size(); i++) + { + if (targetShape[i] == -1 && i < inpShape.size()) + targetShape[i] = inpShape[i]; + else if (i < inpShape.size() && targetShape[i] != inpShape[i]) + inpShape.insert(inpShape.begin() + i, 1); + } + if (haveVariables) + { + LayerParams reshapeLp; + reshapeLp.name = layerParams.name + "/reshape"; + reshapeLp.type = "Reshape"; + CV_Assert(layer_id.find(reshapeLp.name) == layer_id.end()); + reshapeLp.set("dim", DictValue::arrayInt(&inpShape[0], inpShape.size())); + + opencv_onnx::NodeProto proto; + proto.add_input(node_proto.input(0)); + proto.add_output(reshapeLp.name); + addLayer(reshapeLp, proto); + srcName = reshapeLp.name; + } + } + CV_CheckEQ(inpShape.size(), targetShape.size(), "Unsupported Expand op with different dims"); + + std::vector broadcast_axes; + for (int i = 0; i < targetShape.size(); i++) + { + if (targetShape[i] != inpShape[i]) + { + if (inpShape[i] == 1) + broadcast_axes.push_back(i); + else + CV_Error(Error::StsError, format("Could not be broadcast by axis: %d", i)); + } + } + + if (!haveVariables) + { + if (broadcast_axes.size() != 1) + CV_Error(Error::StsNotImplemented, "Expand op doesn't support multiple axes for constant input"); + + Mat input = getBlob(node_proto, 0); + input = input.reshape(0, total(inpShape, 0, broadcast_axes[0])); + Mat output = cv::repeat(input, 1, targetShape[broadcast_axes[0]]); + output = output.reshape(0, targetShape); + addConstant(layerParams.name, output); + return; + } + + if (broadcast_axes.size() == 2 && + broadcast_axes[0] == broadcast_axes[1] - 1 && broadcast_axes[1] == inpShape.size() - 1) + { + LayerParams constParams; + constParams.name = layerParams.name + "/const"; + CV_Assert(layer_id.find(constParams.name) == layer_id.end()); + constParams.type = "Const"; + + Mat inp = Mat::ones(newShapeMat.total(), newShapeMat.ptr(), CV_32F); + constParams.blobs.push_back(inp); + + opencv_onnx::NodeProto proto; + proto.add_output(constParams.name); + addLayer(constParams, proto); + + layerParams.type = "Scale"; + layerParams.set("bias_term", false); + node_proto.set_input(0, constParams.name); + node_proto.set_input(1, srcName); + } + else if (broadcast_axes.size() == 1 && broadcast_axes[0] <= 1) + { + String base_name = layerParams.name + "/copy_"; + std::vector input_names; + for (int j = 0; j < targetShape[broadcast_axes[0]]; j++) + { + std::ostringstream ss; + ss << j; + LayerParams copyLP; + copyLP.name = base_name + ss.str(); + copyLP.type = "Identity"; + CV_Assert(layer_id.find(copyLP.name) == layer_id.end()); + input_names.push_back(copyLP.name); + + node_proto.set_input(0, srcName); + node_proto.set_output(0, copyLP.name); + addLayer(copyLP, node_proto); + } + node_proto.clear_input(); + for (int i = 0; i < input_names.size(); i++) + { + node_proto.add_input(input_names[i]); + } + layerParams.set("axis", broadcast_axes[0]); + layerParams.type = "Concat"; + node_proto.set_output(0, layerParams.name); + } + else + CV_Error(Error::StsNotImplemented, "Unsupported Expand op"); + addLayer(layerParams, node_proto); +} + +void ONNXImporter::parseReshape(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +{ + CV_Assert(node_proto.input_size() == 2 || layerParams.has("shape")); + + if (node_proto.input_size() == 2) { + Mat blob = getBlob(node_proto, 1); + CV_Assert(blob.type() == CV_32SC1); + + layerParams.set("dim", DictValue::arrayInt( + blob.ptr(), blob.total() )); + + if (layer_id.find(node_proto.input(0)) == layer_id.end()) { + std::vector inputs(1, getBlob(node_proto, 0)), outputs; + runLayer(layerParams, inputs, outputs); + addConstant(layerParams.name, outputs[0]); + return; + } + } + else { + DictValue shape = layerParams.get("shape"); + std::vector dim; + for (int j = 0; j < shape.size(); j++) { + dim.push_back(shape.getIntValue(j)); + } + + if (layer_id.find(node_proto.input(0)) == layer_id.end()) { + Mat input = getBlob(node_proto, 0); + Mat out = input.reshape(0, dim); + addConstant(layerParams.name, out); + return; + } + replaceLayerParam(layerParams, "shape", "dim"); + } + addLayer(layerParams, node_proto); +} + +void ONNXImporter::parsePad(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +{ + layerParams.type = "Padding"; + replaceLayerParam(layerParams, "mode", "type"); + if (node_proto.input_size() == 3 || node_proto.input_size() == 2) + { + // Paddings are in order begin0, begin1, .. beginN, end0, end1, ..., endN. + // We need to shuffle it to begin0, end0, begin1, end1, ... + Mat paddings = getBlob(node_proto, 1).reshape(1, 2); + paddings = paddings.t(); + layerParams.set("paddings", DictValue::arrayInt(paddings.ptr(), paddings.total())); + + if (node_proto.input_size() == 3) + { + Mat value = getBlob(node_proto, 2); + layerParams.set("value", value.ptr()[0]); + } + } + addLayer(layerParams, node_proto); +} + +void ONNXImporter::parseShape(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +{ + CV_Assert(node_proto.input_size() == 1); + IterShape_t shapeIt = outShapes.find(node_proto.input(0)); + CV_Assert(shapeIt != outShapes.end()); + const MatShape& inpShape = shapeIt->second; + + Mat shapeMat(inpShape.size(), 1, CV_32S); + for (int j = 0; j < inpShape.size(); ++j) + shapeMat.at(j) = inpShape[j]; + shapeMat.dims = 1; + + addConstant(layerParams.name, shapeMat); +} + +void ONNXImporter::parseCast(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +{ + if (constBlobs.find(node_proto.input(0)) != constBlobs.end()) + { + Mat blob = getBlob(node_proto, 0); + int type; + switch (layerParams.get("to")) + { + case opencv_onnx::TensorProto_DataType_FLOAT: type = CV_32F; break; + case opencv_onnx::TensorProto_DataType_UINT8: type = CV_8U; break; + case opencv_onnx::TensorProto_DataType_UINT16: type = CV_16U; break; + case opencv_onnx::TensorProto_DataType_FLOAT16: type = CV_16S; break; + case opencv_onnx::TensorProto_DataType_INT8: + case opencv_onnx::TensorProto_DataType_INT16: + case opencv_onnx::TensorProto_DataType_INT32: + case opencv_onnx::TensorProto_DataType_INT64: type = CV_32S; break; + default: type = blob.type(); + } + Mat dst; + blob.convertTo(dst, type); + dst.dims = blob.dims; + addConstant(layerParams.name, dst); + return; + } + else + layerParams.type = "Identity"; + addLayer(layerParams, node_proto); +} + +void ONNXImporter::parseConstantFill(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +{ + int depth = CV_32F; + float fill_value; + if (!layerParams.blobs.empty()) + { + CV_Assert(!layerParams.has("value")); + depth = layerParams.blobs[0].depth(); + Mat floats; + layerParams.blobs[0].convertTo(floats, CV_32F); + fill_value = floats.at(0, 0); + } + else + fill_value = layerParams.get("value", 0); + + MatShape inpShape = getBlob(node_proto, 0); + for (int i = 0; i < inpShape.size(); i++) + CV_CheckGT(inpShape[i], 0, ""); + Mat tensor(inpShape.size(), &inpShape[0], depth, Scalar(fill_value)); + addConstant(layerParams.name, tensor); +} + +void ONNXImporter::parseGather(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto_) +{ + opencv_onnx::NodeProto node_proto = node_proto_; + CV_Assert(node_proto.input_size() == 2); + Mat indexMat = getBlob(node_proto, 1); + CV_Assert_N(indexMat.type() == CV_32S, indexMat.total() == 1); + int index = indexMat.at(0); + int axis = layerParams.get("axis", 0); + + if ((constBlobs.find(node_proto.input(0)) != constBlobs.end())) + { + Mat input = getBlob(node_proto, 0); + Mat out; + std::vector ranges(input.dims, Range::all()); + ranges[axis] = Range(index, index + 1); + + out = input(ranges); + MatShape outShape = shape(out); + if (outShape.size() > 1) + { + outShape.erase(outShape.begin() + axis); + out.reshape(0, outShape); + } else { + out.dims = 1; + } + addConstant(layerParams.name, out); + return; + } + else + { + IterShape_t shapeIt = outShapes.find(node_proto.input(0)); + CV_Assert(shapeIt != outShapes.end()); + MatShape inpShape = shapeIt->second; + + LayerParams sliceLp; + sliceLp.type = "Slice"; + sliceLp.name = inpShape.size() > 1 ? layerParams.name + "/slice" : layerParams.name; + std::vector begin(inpShape.size(), 0); + std::vector end(inpShape.size(), -1); + begin[axis] = index; + end[axis] = index + 1; + + cv::dnn::DictValue paramBegin = cv::dnn::DictValue::arrayInt(begin.data(), begin.size()); + cv::dnn::DictValue paramEnd = cv::dnn::DictValue::arrayInt(end.data(), end.size()); + sliceLp.set("begin", paramBegin); + sliceLp.set("end", paramEnd); + sliceLp.set("has_dynamic_shapes", hasDynamicShapes); + + if (inpShape.size() > 1) + { + opencv_onnx::NodeProto proto; + proto.add_input(node_proto.input(0)); + proto.add_output(sliceLp.name); + addLayer(sliceLp, proto); + + inpShape.erase(inpShape.begin() + axis); + layerParams.type = "Reshape"; + layerParams.set("axis", 0); + layerParams.set("dim", DictValue::arrayInt(&inpShape[0], inpShape.size())); + if (hasDynamicShapes) + { + std::vector dynamicAxes; + std::vector inputIndices; + for (int index = 0; index < inpShape.size(); ++index) + dynamicAxes.push_back(index); + for (int index = 0; index < inpShape.size(); ++index) + inputIndices.push_back(index); + layerParams.set("dynamic_axes", DictValue::arrayInt(dynamicAxes.data(), dynamicAxes.size())); + layerParams.set("input_indices", DictValue::arrayInt(inputIndices.data(), inputIndices.size())); + } + node_proto.set_input(0, sliceLp.name); + } + else + { + layerParams = sliceLp; + } + } + addLayer(layerParams, node_proto); +} + +void ONNXImporter::parseConcat(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +{ + bool hasVariableInps = false; + for (int i = 0; i < node_proto.input_size(); ++i) + { + if (layer_id.find(node_proto.input(i)) != layer_id.end()) + { + hasVariableInps = true; + break; + } + } + + if (!hasVariableInps) + { + std::vector inputs(node_proto.input_size()), concatenated; + // Due constant folding we can get inputs with different number of dimensions + // Insert the missing dimension to inputs + MatShape inputShape; + for (size_t i = 0; i < inputs.size(); ++i) + { + inputs[i] = getBlob(node_proto, i); + if (inputs[i].size.dims() > inputShape.size()) + { + inputShape = shape(inputs[i]); + } + } + + // Concat-1 has default value for axis is 1: https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Concat-1 + int axis = layerParams.get("axis", 1); + for (size_t i = 0; i < inputs.size(); ++i) + { + MatShape targetShape = inputShape; + targetShape[axis] = shape(inputs[i])[axis]; + CV_CheckEQ(total(targetShape), total(shape(inputs[i])), ""); + inputs[i] = inputs[i].reshape(0, targetShape); + } + runLayer(layerParams, inputs, concatenated); + + CV_Assert(concatenated.size() == 1); + addConstant(layerParams.name, concatenated[0]); + return; + } + else + { + for (int i = 0; i < node_proto.input_size(); ++i) + { + if (constBlobs.find(node_proto.input(i)) != constBlobs.end()) + { + LayerParams constParams; + constParams.name = node_proto.input(i); + constParams.type = "Const"; + constParams.blobs.push_back(getBlob(node_proto, i)); + + opencv_onnx::NodeProto proto; + proto.add_output(constParams.name); + addLayer(constParams, proto); + } + } + } + addLayer(layerParams, node_proto); +} + +void ONNXImporter::parseResize(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +{ + for (int i = 1; i < node_proto.input_size(); i++) + CV_Assert(layer_id.find(node_proto.input(i)) == layer_id.end()); + + if (layerParams.has("coordinate_transformation_mode")) + { + String interp_mode = layerParams.get("coordinate_transformation_mode"); + CV_Assert_N(interp_mode != "tf_crop_and_resize", interp_mode != "tf_half_pixel_for_nn"); + + layerParams.set("align_corners", interp_mode == "align_corners"); + if (layerParams.get("mode") == "linear") + { + layerParams.set("mode", interp_mode == "pytorch_half_pixel" ? + "opencv_linear" : "bilinear"); + } + } + if (layerParams.get("mode") == "linear" && framework_name == "pytorch") + layerParams.set("mode", "opencv_linear"); + + // input = [X, scales], [X, roi, scales] or [x, roi, scales, sizes] + int foundScaleId = hasDynamicShapes ? node_proto.input_size() - 1 + : node_proto.input_size() > 2 ? 2 : 1; + + Mat scales = getBlob(node_proto, foundScaleId); + if (scales.total() == 4) + { + layerParams.set("zoom_factor_y", scales.at(2)); + layerParams.set("zoom_factor_x", scales.at(3)); + } + else + { + const std::string& inputLast = node_proto.input(node_proto.input_size() - 1); + if (constBlobs.find(inputLast) != constBlobs.end()) + { + Mat shapes = getBlob(inputLast); + CV_CheckEQ(shapes.size[0], 4, ""); + CV_CheckEQ(shapes.size[1], 1, ""); + CV_CheckDepth(shapes.depth(), shapes.depth() == CV_32S || shapes.depth() == CV_32F, ""); + if (shapes.depth() == CV_32F) + shapes.convertTo(shapes, CV_32S); + layerParams.set("width", shapes.at(3)); + layerParams.set("height", shapes.at(2)); + } + } + replaceLayerParam(layerParams, "mode", "interpolation"); + addLayer(layerParams, node_proto); +} + +void ONNXImporter::parseUpsample(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +{ + //fused from Resize Subgraph + if (layerParams.has("coordinate_transformation_mode")) + { + String interp_mode = layerParams.get("coordinate_transformation_mode"); + CV_Assert_N(interp_mode != "tf_crop_and_resize", interp_mode != "tf_half_pixel_for_nn"); + + layerParams.set("align_corners", interp_mode == "align_corners"); + if (layerParams.get("mode") == "linear") + { + layerParams.set("mode", interp_mode == "pytorch_half_pixel" ? + "opencv_linear" : "bilinear"); + } + } + if (layerParams.get("mode") == "linear" && framework_name == "pytorch") + layerParams.set("mode", "opencv_linear"); + + layerParams.type = "Resize"; + if (layerParams.has("scales")) + { + // Pytorch layer + DictValue scales = layerParams.get("scales"); + CV_Assert(scales.size() == 4); + layerParams.set("zoom_factor_y", scales.getIntValue(2)); + layerParams.set("zoom_factor_x", scales.getIntValue(3)); + } + else if (layerParams.has("height_scale") && layerParams.has("width_scale")) + { + // Caffe2 layer + replaceLayerParam(layerParams, "height_scale", "zoom_factor_y"); + replaceLayerParam(layerParams, "width_scale", "zoom_factor_x"); + } + else + { + // scales as input + const std::string& input1 = node_proto.input(1); + if (constBlobs.find(input1) != constBlobs.end()) + { + Mat scales = getBlob(input1); + CV_Assert(scales.total() == 4); + layerParams.set("zoom_factor_y", scales.at(2)); + layerParams.set("zoom_factor_x", scales.at(3)); + } + } + replaceLayerParam(layerParams, "mode", "interpolation"); + addLayer(layerParams, node_proto); +} + +void ONNXImporter::parseSoftMax(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +{ + const std::string& layer_type = node_proto.op_type(); + layerParams.type = "Softmax"; + layerParams.set("log_softmax", layer_type == "LogSoftmax"); + addLayer(layerParams, node_proto); +} + +void ONNXImporter::parseDetectionOutput(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto_) +{ + opencv_onnx::NodeProto node_proto = node_proto_; + CV_CheckEQ(node_proto.input_size(), 3, ""); + if (constBlobs.find(node_proto.input(2)) != constBlobs.end()) + { + Mat priors = getBlob(node_proto, 2); + + LayerParams constParams; + constParams.name = layerParams.name + "/priors"; + constParams.type = "Const"; + constParams.blobs.push_back(priors); + + opencv_onnx::NodeProto priorsProto; + priorsProto.add_output(constParams.name); + addLayer(constParams, priorsProto); + + node_proto.set_input(2, constParams.name); + } + addLayer(layerParams, node_proto); +} + +void ONNXImporter::parseCustom(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +{ + for (int j = 0; j < node_proto.input_size(); j++) { + if (layer_id.find(node_proto.input(j)) == layer_id.end()) + layerParams.blobs.push_back(getBlob(node_proto, j)); + } + addLayer(layerParams, node_proto); +} + +const ONNXImporter::DispatchMap ONNXImporter::buildDispatchMap() +{ + DispatchMap dispatch; + + dispatch["MaxPool"] = &ONNXImporter::parseMaxPool; + dispatch["AveragePool"] = &ONNXImporter::parseAveragePool; + dispatch["GlobalAveragePool"] = dispatch["GlobalMaxPool"] = dispatch["ReduceMean"] = dispatch["ReduceSum"] = + dispatch["ReduceMax"] = &ONNXImporter::parseReduce; + dispatch["Slice"] = &ONNXImporter::parseSlice; + dispatch["Split"] = &ONNXImporter::parseSplit; + dispatch["Add"] = dispatch["Sum"] = dispatch["Sub"] = &ONNXImporter::parseBias; + dispatch["Pow"] = &ONNXImporter::parsePow; + dispatch["Max"] = &ONNXImporter::parseMax; + dispatch["Neg"] = &ONNXImporter::parseNeg; + dispatch["Constant"] = &ONNXImporter::parseConstant; + dispatch["LSTM"] = &ONNXImporter::parseLSTM; + dispatch["ImageScaler"] = &ONNXImporter::parseImageScaler; + dispatch["Clip"] = &ONNXImporter::parseClip; + dispatch["LeakyRelu"] = &ONNXImporter::parseLeakyRelu; + dispatch["Relu"] = &ONNXImporter::parseRelu; + dispatch["Elu"] = &ONNXImporter::parseElu; + dispatch["Tanh"] = &ONNXImporter::parseTanh; + dispatch["PRelu"] = &ONNXImporter::parsePRelu; + dispatch["LRN"] = &ONNXImporter::parseLRN; + dispatch["InstanceNormalization"] = &ONNXImporter::parseInstanceNormalization; + dispatch["BatchNormalization"] = &ONNXImporter::parseBatchNormalization; + dispatch["Gemm"] = &ONNXImporter::parseGemm; + dispatch["MatMul"] = &ONNXImporter::parseMatMul; + dispatch["Mul"] = dispatch["Div"] = &ONNXImporter::parseMul; + dispatch["Conv"] = &ONNXImporter::parseConv; + dispatch["ConvTranspose"] = &ONNXImporter::parseConvTranspose; + dispatch["Transpose"] = &ONNXImporter::parseTranspose; + dispatch["Squeeze"] = &ONNXImporter::parseSqueeze; + dispatch["Flatten"] = &ONNXImporter::parseFlatten; + dispatch["Unsqueeze"] = &ONNXImporter::parseUnsqueeze; + dispatch["Expand"] = &ONNXImporter::parseExpand; + dispatch["Reshape"] = &ONNXImporter::parseReshape; + dispatch["Pad"] = &ONNXImporter::parsePad; + dispatch["Shape"] = &ONNXImporter::parseShape; + dispatch["Cast"] = &ONNXImporter::parseCast; + dispatch["ConstantFill"] = dispatch["ConstantOfShape"] = &ONNXImporter::parseConstantFill; + dispatch["Gather"] = &ONNXImporter::parseGather; + dispatch["Concat"] = &ONNXImporter::parseConcat; + dispatch["Resize"] = &ONNXImporter::parseResize; + dispatch["Upsample"] = &ONNXImporter::parseUpsample; + dispatch["SoftMax"] = dispatch["LogSoftmax"] = &ONNXImporter::parseSoftMax; + dispatch["DetectionOutput"] = &ONNXImporter::parseDetectionOutput; + dispatch["Custom"] = &ONNXImporter::parseCustom; + + return dispatch; +} + Net readNetFromONNX(const String& onnxFile) { Net net; diff --git a/modules/dnn/src/tensorflow/tf_importer.cpp b/modules/dnn/src/tensorflow/tf_importer.cpp index 4bd09adda0..38a55d12b9 100644 --- a/modules/dnn/src/tensorflow/tf_importer.cpp +++ b/modules/dnn/src/tensorflow/tf_importer.cpp @@ -2894,7 +2894,7 @@ void TFImporter::parseNode(const tensorflow::NodeDef& layer) DispatchMap::const_iterator iter = dispatch.find(type); if (iter != dispatch.end()) { - ((*this).*(iter->second))(net, layer, layerParams); + CALL_MEMBER_FN(*this, iter->second)(net, layer, layerParams); } else if (!DNN_DIAGNOSTICS_RUN || !layerHandler->handleMissing(layer)) { From b468468e7e24d78387fe1cf75ec7b46ae2319457 Mon Sep 17 00:00:00 2001 From: Mahendra Kumar <66687425+kumar-mahendra@users.noreply.github.com> Date: Fri, 23 Jul 2021 16:44:48 +0530 Subject: [PATCH 089/262] Closing brackets missing In line 94, closing brackets are added which were missing . --- .../py_contour_features/py_contour_features.markdown | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/py_tutorials/py_imgproc/py_contours/py_contour_features/py_contour_features.markdown b/doc/py_tutorials/py_imgproc/py_contours/py_contour_features/py_contour_features.markdown index ecd0e97af2..d0e6c4b2ac 100644 --- a/doc/py_tutorials/py_imgproc/py_contours/py_contour_features/py_contour_features.markdown +++ b/doc/py_tutorials/py_imgproc/py_contours/py_contour_features/py_contour_features.markdown @@ -91,7 +91,7 @@ convexity defects, which are the local maximum deviations of hull from contours. There is a little bit things to discuss about it its syntax: @code{.py} -hull = cv.convexHull(points[, hull[, clockwise[, returnPoints]] +hull = cv.convexHull(points[, hull[, clockwise[, returnPoints]]]) @endcode Arguments details: From 531ea5b3a21a5632789e592cfd71e9515849523b Mon Sep 17 00:00:00 2001 From: Vadim Levin Date: Sun, 1 Aug 2021 11:59:16 +0300 Subject: [PATCH 090/262] fix: convert arguments names that are keywords reserved by Python --- .../include/opencv2/core/bindings_utils.hpp | 6 ++++++ modules/python/src2/gen2.py | 12 ++++++++++++ modules/python/test/test_misc.py | 17 +++++++++++++++++ 3 files changed, 35 insertions(+) diff --git a/modules/core/include/opencv2/core/bindings_utils.hpp b/modules/core/include/opencv2/core/bindings_utils.hpp index 98a4a2b785..a3f83d9c2c 100644 --- a/modules/core/include/opencv2/core/bindings_utils.hpp +++ b/modules/core/include/opencv2/core/bindings_utils.hpp @@ -116,6 +116,12 @@ String dumpRange(const Range& argument) } } +CV_WRAP static inline +String testReservedKeywordConversion(int positional_argument, int lambda = 2, int from = 3) +{ + return format("arg=%d, lambda=%d, from=%d", positional_argument, lambda, from); +} + CV_WRAP static inline void testRaiseGeneralException() { diff --git a/modules/python/src2/gen2.py b/modules/python/src2/gen2.py index bccf0d2767..51566fc248 100755 --- a/modules/python/src2/gen2.py +++ b/modules/python/src2/gen2.py @@ -212,6 +212,16 @@ simple_argtype_mapping = { "c_string": ArgTypeInfo("char*", FormatStrings.string, '(char*)""') } +# Set of reserved keywords for Python. Can be acquired via the following call +# $ python -c "help('keywords')" +# Keywords that are reserved in C/C++ are excluded because they can not be +# used as variables identifiers +python_reserved_keywords = { + "True", "None", "False", "as", "assert", "def", "del", "elif", "except", "exec", + "finally", "from", "global", "import", "in", "is", "lambda", "nonlocal", + "pass", "print", "raise", "with", "yield" +} + def normalize_class_name(name): return re.sub(r"^cv\.", "", name).replace(".", "_") @@ -369,6 +379,8 @@ class ArgInfo(object): def __init__(self, arg_tuple): self.tp = handle_ptr(arg_tuple[0]) self.name = arg_tuple[1] + if self.name in python_reserved_keywords: + self.name += "_" self.defval = arg_tuple[2] self.isarray = False self.arraylen = 0 diff --git a/modules/python/test/test_misc.py b/modules/python/test/test_misc.py index 121e86a64c..4d435a46b6 100644 --- a/modules/python/test/test_misc.py +++ b/modules/python/test/test_misc.py @@ -463,6 +463,23 @@ class Arguments(NewOpenCVTests): with self.assertRaises((TypeError), msg=get_no_exception_msg(not_convertible)): _ = cv.utils.dumpRange(not_convertible) + def test_reserved_keywords_are_transformed(self): + default_lambda_value = 2 + default_from_value = 3 + format_str = "arg={}, lambda={}, from={}" + self.assertEqual( + cv.utils.testReservedKeywordConversion(20), format_str.format(20, default_lambda_value, default_from_value) + ) + self.assertEqual( + cv.utils.testReservedKeywordConversion(10, lambda_=10), format_str.format(10, 10, default_from_value) + ) + self.assertEqual( + cv.utils.testReservedKeywordConversion(10, from_=10), format_str.format(10, default_lambda_value, 10) + ) + self.assertEqual( + cv.utils.testReservedKeywordConversion(20, lambda_=-4, from_=12), format_str.format(20, -4, 12) + ) + class SamplesFindFile(NewOpenCVTests): From ba0cea6826a90689d4d58efcd28d1aa53f3478cc Mon Sep 17 00:00:00 2001 From: Dmitry Budnikov Date: Mon, 2 Aug 2021 13:58:18 +0300 Subject: [PATCH 091/262] Merge pull request #20474 from dbudniko:dbudniko/mtcnn_graph_without_resizes Remove explicit PNet resizes from MTCNN graph * remove PNet resizes * address comment from Ruslan --- modules/gapi/samples/face_detection_mtcnn.cpp | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/modules/gapi/samples/face_detection_mtcnn.cpp b/modules/gapi/samples/face_detection_mtcnn.cpp index ad7c20c117..50cb666a90 100644 --- a/modules/gapi/samples/face_detection_mtcnn.cpp +++ b/modules/gapi/samples/face_detection_mtcnn.cpp @@ -591,16 +591,13 @@ int main(int argc, char* argv[]) { cv::GMat in_originalRGB = cv::gapi::BGR2RGB(in_original); cv::GMat in_transposedRGB = cv::gapi::transpose(in_originalRGB); cv::GOpaque in_sz = cv::gapi::streaming::size(in_original); - cv::GMat in_resized[MAX_PYRAMID_LEVELS]; cv::GMat regressions[MAX_PYRAMID_LEVELS]; cv::GMat scores[MAX_PYRAMID_LEVELS]; cv::GArray nms_p_faces[MAX_PYRAMID_LEVELS]; cv::GArray total_faces[MAX_PYRAMID_LEVELS]; //The very first PNet pyramid layer to init total_faces[0] - cv::Size currentSize = cv::Size(level_size[0].height, level_size[0].width); - in_resized[0] = cv::gapi::resize(in_transposedRGB, currentSize); - std::tie(regressions[0], scores[0]) = run_mtcnn_p(in_resized[0], get_pnet_level_name(level_size[0])); + std::tie(regressions[0], scores[0]) = run_mtcnn_p(in_transposedRGB, get_pnet_level_name(level_size[0])); cv::GArray faces0 = custom::BuildFaces::on(scores[0], regressions[0], static_cast(scales[0]), conf_thresh_p); cv::GArray final_p_faces_for_bb2squares = custom::ApplyRegression::on(faces0, true); cv::GArray final_faces_pnet0 = custom::BBoxesToSquares::on(final_p_faces_for_bb2squares); @@ -608,9 +605,7 @@ int main(int argc, char* argv[]) { //The rest PNet pyramid layers to accumlate all layers result in total_faces[PYRAMID_LEVELS - 1]] for (int i = 1; i < pyramid_levels; ++i) { - currentSize = cv::Size(level_size[i].height, level_size[i].width); - in_resized[i] = cv::gapi::resize(in_transposedRGB, currentSize); - std::tie(regressions[i], scores[i]) = run_mtcnn_p(in_resized[i], get_pnet_level_name(level_size[i])); + std::tie(regressions[i], scores[i]) = run_mtcnn_p(in_transposedRGB, get_pnet_level_name(level_size[i])); cv::GArray faces = custom::BuildFaces::on(scores[i], regressions[i], static_cast(scales[i]), conf_thresh_p); cv::GArray final_p_faces_for_bb2squares_i = custom::ApplyRegression::on(faces, true); cv::GArray final_faces_pnet_i = custom::BBoxesToSquares::on(final_p_faces_for_bb2squares_i); From 2d8ce500fa4961a9122346e51c2d6d421f0b8cef Mon Sep 17 00:00:00 2001 From: AleksandrPanov Date: Mon, 2 Aug 2021 18:41:53 +0300 Subject: [PATCH 092/262] add note about using version 3.4 to traincascade documentation --- doc/tutorials/objdetect/traincascade.markdown | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/tutorials/objdetect/traincascade.markdown b/doc/tutorials/objdetect/traincascade.markdown index d7acdca553..d78de2ec9a 100644 --- a/doc/tutorials/objdetect/traincascade.markdown +++ b/doc/tutorials/objdetect/traincascade.markdown @@ -10,6 +10,8 @@ Working with a boosted cascade of weak classifiers includes two major stages: th To support this tutorial, several official OpenCV applications will be used: [opencv_createsamples](https://github.com/opencv/opencv/tree/3.4/apps/createsamples), [opencv_annotation](https://github.com/opencv/opencv/tree/3.4/apps/annotation), [opencv_traincascade](https://github.com/opencv/opencv/tree/3.4/apps/traincascade) and [opencv_visualisation](https://github.com/opencv/opencv/tree/3.4/apps/visualisation). +@note Createsamples and traincascade are disabled since OpenCV 4.0. Consider using these apps for training from 3.4 branch for Cascade Classifier. Model format is the same between 3.4 and 4.x. + ### Important notes - If you come across any tutorial mentioning the old opencv_haartraining tool (which is deprecated and still using the OpenCV1.x interface), then please ignore that tutorial and stick to the opencv_traincascade tool. This tool is a newer version, written in C++ in accordance to the OpenCV 2.x and OpenCV 3.x API. The opencv_traincascade supports both HAAR like wavelet features @cite Viola01 and LBP (Local Binary Patterns) @cite Liao2007 features. LBP features yield integer precision in contrast to HAAR features, yielding floating point precision, so both training and detection with LBP are several times faster then with HAAR features. Regarding the LBP and HAAR detection quality, it mainly depends on the training data used and the training parameters selected. It's possible to train a LBP-based classifier that will provide almost the same quality as HAAR-based one, within a percentage of the training time. From d773691848f6850eb1b21b2b6031a5f64c321efa Mon Sep 17 00:00:00 2001 From: AleksandrPanov Date: Wed, 4 Aug 2021 15:37:20 +0300 Subject: [PATCH 093/262] add note about hierarchy in python version --- modules/imgproc/include/opencv2/imgproc.hpp | 1 + 1 file changed, 1 insertion(+) diff --git a/modules/imgproc/include/opencv2/imgproc.hpp b/modules/imgproc/include/opencv2/imgproc.hpp index a1cfff991d..5e66b14e3b 100644 --- a/modules/imgproc/include/opencv2/imgproc.hpp +++ b/modules/imgproc/include/opencv2/imgproc.hpp @@ -3951,6 +3951,7 @@ hierarchy[i][0] , hierarchy[i][1] , hierarchy[i][2] , and hierarchy[i][3] are se in contours of the next and previous contours at the same hierarchical level, the first child contour and the parent contour, respectively. If for the contour i there are no next, previous, parent, or nested contours, the corresponding elements of hierarchy[i] will be negative. +@note In Python, hierarchy is nested inside a top level array. Use hierarchy[0][i] to access hierarchical elements of i-th contour. @param mode Contour retrieval mode, see #RetrievalModes @param method Contour approximation method, see #ContourApproximationModes @param offset Optional offset by which every contour point is shifted. This is useful if the From cefa60260105aaa33d54daddacd3c60a1ce93e57 Mon Sep 17 00:00:00 2001 From: Duong Dac Date: Wed, 4 Aug 2021 15:17:25 +0200 Subject: [PATCH 094/262] Avoid adding false UMat/GpuMat declaration --- modules/python/src2/hdr_parser.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/modules/python/src2/hdr_parser.py b/modules/python/src2/hdr_parser.py index ac3f383dc8..749a9033ee 100755 --- a/modules/python/src2/hdr_parser.py +++ b/modules/python/src2/hdr_parser.py @@ -954,7 +954,8 @@ class CppHeaderParser(object): has_mat = len(list(filter(lambda x: x[0] in {"Mat", "vector_Mat"}, args))) > 0 if has_mat: _, _, _, gpumat_decl = self.parse_stmt(stmt, token, mat="cuda::GpuMat", docstring=docstring) - decls.append(gpumat_decl) + if gpumat_decl != decl: + decls.append(gpumat_decl) if self._generate_umat_decls: # If function takes as one of arguments Mat or vector - we want to create the @@ -963,7 +964,8 @@ class CppHeaderParser(object): has_mat = len(list(filter(lambda x: x[0] in {"Mat", "vector_Mat"}, args))) > 0 if has_mat: _, _, _, umat_decl = self.parse_stmt(stmt, token, mat="UMat", docstring=docstring) - decls.append(umat_decl) + if umat_decl != decl: + decls.append(umat_decl) docstring = "" if stmt_type == "namespace": From 6a2e559222e3950d7323e5703b3f5857f59dafce Mon Sep 17 00:00:00 2001 From: Alexander Smorkalov Date: Wed, 4 Aug 2021 20:04:03 +0300 Subject: [PATCH 095/262] Fixed memory access issue in v_rshr_pack_store intrinsic on RISC-V RVV. --- modules/core/include/opencv2/core/hal/intrin_rvv.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/core/include/opencv2/core/hal/intrin_rvv.hpp b/modules/core/include/opencv2/core/hal/intrin_rvv.hpp index 51433cdbae..5b33193781 100644 --- a/modules/core/include/opencv2/core/hal/intrin_rvv.hpp +++ b/modules/core/include/opencv2/core/hal/intrin_rvv.hpp @@ -1967,7 +1967,7 @@ void v_rshr_pack_store(_Tp* ptr, const _wTpvec& a) \ _wTp arr[_Tpvec::nlanes] = {0}; \ v_store(arr, a); \ v_store(arr + _wTpvec::nlanes, _wTpvec(vmv_v_x_##suffix##m1(0, hvl))); \ - v_store(ptr, _Tpvec(rshr(vle##width##_v_##suffix##m2(arr, vl), n, vl))); \ + vse##hwidth##_v_##hsuffix##m1(ptr, _Tpvec(rshr(vle##width##_v_##suffix##m2(arr, vl), n, vl)), hvl); \ } OPENCV_HAL_IMPL_RVV_PACK(v_uint8x16, uchar, v_uint16x8, ushort, 8, 16, u8, u16, vnclipu_wx_u8m1, vnclipu_wx_u8m1, 8, 16) From d5f34cf34c7088ed69e3e3a922435743e3d95d81 Mon Sep 17 00:00:00 2001 From: Abhinav Sharma <63901956+abhi-bhatra@users.noreply.github.com> Date: Wed, 4 Aug 2021 22:58:59 +0530 Subject: [PATCH 096/262] Merge pull request #20493 from abhi-bhatra:patch-1 Fix Broken Link * Update README.md Fix broken link * Update Readme.txt * Update readme.txt Add missing link * Update readme.txt Fix names --- 3rdparty/readme.txt | 6 ++++-- platforms/winrt/readme.txt | 4 ++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/3rdparty/readme.txt b/3rdparty/readme.txt index 3b961782b0..4e4a6ba0a6 100644 --- a/3rdparty/readme.txt +++ b/3rdparty/readme.txt @@ -31,7 +31,7 @@ libpng Portable Network Graphics library. libtiff Tag Image File Format (TIFF) Software Copyright (c) 1988-1997 Sam Leffler Copyright (c) 1991-1997 Silicon Graphics, Inc. - See libtiff home page http://www.remotesensing.org/libtiff/ + See libtiff home page http://www.libtiff.org/ for details and links to the source code WITH_TIFF CMake option must be ON to add libtiff & zlib support to imgcodecs. @@ -51,7 +51,9 @@ jasper JasPer is a collection of software Copyright (c) 1999-2000 The University of British Columbia Copyright (c) 2001-2003 Michael David Adams - The JasPer license can be found in libjasper. + See JasPer official GitHub repository + https://github.com/jasper-software/jasper.git + for details and links to source code ------------------------------------------------------------------------------------ openexr OpenEXR is a high dynamic-range (HDR) image file format developed by Industrial Light & Magic for use in computer imaging applications. diff --git a/platforms/winrt/readme.txt b/platforms/winrt/readme.txt index 2fb4ce1f54..2d1b4e6c30 100644 --- a/platforms/winrt/readme.txt +++ b/platforms/winrt/readme.txt @@ -13,7 +13,7 @@ Install Visual Studio 2013 Community Edition http://go.microsoft.com/?linkid=9863608 Install Visual Studio Express 2012 for Windows Desktop - http://www.microsoft.com/en-us/download/details.aspx?id=34673 + https://devblogs.microsoft.com/visualstudio/visual-studio-express-2012-for-windows-desktop-is-here/ @@ -156,4 +156,4 @@ Manual build cmake -G "Visual Studio 12 2013 Win64" -DCMAKE_SYSTEM_NAME:String=WindowsStore -DCMAKE_SYSTEM_VERSION:String=8.1 -DCMAKE_VS_EFFECTIVE_PLATFORMS:String=x64 -DCMAKE_INSTALL_PREFIX:PATH=.\install\WS\8.1\x64\ .. -Return to "Running tests for Windows Store", list item 4. \ No newline at end of file +Return to "Running tests for Windows Store", list item 4. From 5af09e73f250946e3e5447099a23c1e07131b1fe Mon Sep 17 00:00:00 2001 From: Anatoliy Talamanov Date: Thu, 5 Aug 2021 11:59:24 +0300 Subject: [PATCH 097/262] Merge pull request #20490 from TolyaTalamanov:at/support-fp16-input-precision [G-API] Support FP16 input precision for IE backend * Support fp16 input precision IE backend * Add support to wrapIE --- modules/gapi/src/backends/ie/giebackend.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/modules/gapi/src/backends/ie/giebackend.cpp b/modules/gapi/src/backends/ie/giebackend.cpp index 77a6515f85..fc9fc502ef 100644 --- a/modules/gapi/src/backends/ie/giebackend.cpp +++ b/modules/gapi/src/backends/ie/giebackend.cpp @@ -108,6 +108,7 @@ inline IE::Precision toIE(int depth) { case CV_8U: return IE::Precision::U8; case CV_32S: return IE::Precision::I32; case CV_32F: return IE::Precision::FP32; + case CV_16F: return IE::Precision::FP16; default: GAPI_Assert(false && "IE. Unsupported data type"); } return IE::Precision::UNSPECIFIED; @@ -161,6 +162,7 @@ inline IE::Blob::Ptr wrapIE(const cv::Mat &mat, cv::gapi::ie::TraitAs hint) { HANDLE(8U, uint8_t); HANDLE(32F, float); HANDLE(32S, int); + HANDLE(16F, int16_t); #undef HANDLE default: GAPI_Assert(false && "IE. Unsupported data type"); } From 65134c793bc7c80cf63f7a4d32158fa4e17ffed3 Mon Sep 17 00:00:00 2001 From: AleksandrPanov Date: Thu, 5 Aug 2021 14:21:34 +0300 Subject: [PATCH 098/262] add links in python and js contours tutorial --- .../js_contour_features/js_contour_features.markdown | 3 +++ .../js_contour_properties/js_contour_properties.markdown | 3 +++ .../js_contours/js_contours_begin/js_contours_begin.markdown | 2 ++ .../js_contours_hierarchy/js_contours_hierarchy.markdown | 2 ++ .../js_contours_more_functions.markdown | 3 +++ .../py_contour_features/py_contour_features.markdown | 3 +++ .../py_contour_properties/py_contour_properties.markdown | 3 +++ .../py_contours/py_contours_begin/py_contours_begin.markdown | 2 ++ .../py_contours_hierarchy/py_contours_hierarchy.markdown | 2 ++ .../py_contours_more_functions.markdown | 4 ++++ 10 files changed, 27 insertions(+) diff --git a/doc/js_tutorials/js_imgproc/js_contours/js_contour_features/js_contour_features.markdown b/doc/js_tutorials/js_imgproc/js_contours/js_contour_features/js_contour_features.markdown index 22544b280c..8421269587 100644 --- a/doc/js_tutorials/js_imgproc/js_contours/js_contour_features/js_contour_features.markdown +++ b/doc/js_tutorials/js_imgproc/js_contours/js_contour_features/js_contour_features.markdown @@ -1,6 +1,9 @@ Contour Features {#tutorial_js_contour_features} ================ +@prev_tutorial{tutorial_js_contours_begin} +@next_tutorial{tutorial_js_contour_properties} + Goal ---- diff --git a/doc/js_tutorials/js_imgproc/js_contours/js_contour_properties/js_contour_properties.markdown b/doc/js_tutorials/js_imgproc/js_contours/js_contour_properties/js_contour_properties.markdown index 1dbb15c4cf..3392283305 100644 --- a/doc/js_tutorials/js_imgproc/js_contours/js_contour_properties/js_contour_properties.markdown +++ b/doc/js_tutorials/js_imgproc/js_contours/js_contour_properties/js_contour_properties.markdown @@ -1,6 +1,9 @@ Contour Properties {#tutorial_js_contour_properties} ================== +@prev_tutorial{tutorial_js_contour_features} +@next_tutorial{tutorial_js_contours_more_functions} + Goal ---- diff --git a/doc/js_tutorials/js_imgproc/js_contours/js_contours_begin/js_contours_begin.markdown b/doc/js_tutorials/js_imgproc/js_contours/js_contours_begin/js_contours_begin.markdown index 9678a7c9f0..3caf17f873 100644 --- a/doc/js_tutorials/js_imgproc/js_contours/js_contours_begin/js_contours_begin.markdown +++ b/doc/js_tutorials/js_imgproc/js_contours/js_contours_begin/js_contours_begin.markdown @@ -1,6 +1,8 @@ Contours : Getting Started {#tutorial_js_contours_begin} ========================== +@next_tutorial{tutorial_js_contour_features} + Goal ---- diff --git a/doc/js_tutorials/js_imgproc/js_contours/js_contours_hierarchy/js_contours_hierarchy.markdown b/doc/js_tutorials/js_imgproc/js_contours/js_contours_hierarchy/js_contours_hierarchy.markdown index c98628e486..c2e408a962 100644 --- a/doc/js_tutorials/js_imgproc/js_contours/js_contours_hierarchy/js_contours_hierarchy.markdown +++ b/doc/js_tutorials/js_imgproc/js_contours/js_contours_hierarchy/js_contours_hierarchy.markdown @@ -1,6 +1,8 @@ Contours Hierarchy {#tutorial_js_contours_hierarchy} ================== +@prev_tutorial{tutorial_js_contours_more_functions} + Goal ---- diff --git a/doc/js_tutorials/js_imgproc/js_contours/js_contours_more_functions/js_contours_more_functions.markdown b/doc/js_tutorials/js_imgproc/js_contours/js_contours_more_functions/js_contours_more_functions.markdown index b753116662..941f0c486a 100644 --- a/doc/js_tutorials/js_imgproc/js_contours/js_contours_more_functions/js_contours_more_functions.markdown +++ b/doc/js_tutorials/js_imgproc/js_contours/js_contours_more_functions/js_contours_more_functions.markdown @@ -1,6 +1,9 @@ Contours : More Functions {#tutorial_js_contours_more_functions} ========================= +@prev_tutorial{tutorial_js_contour_properties} +@next_tutorial{tutorial_js_contours_hierarchy} + Goal ---- diff --git a/doc/py_tutorials/py_imgproc/py_contours/py_contour_features/py_contour_features.markdown b/doc/py_tutorials/py_imgproc/py_contours/py_contour_features/py_contour_features.markdown index d0e6c4b2ac..e8cfbd6597 100644 --- a/doc/py_tutorials/py_imgproc/py_contours/py_contour_features/py_contour_features.markdown +++ b/doc/py_tutorials/py_imgproc/py_contours/py_contour_features/py_contour_features.markdown @@ -1,6 +1,9 @@ Contour Features {#tutorial_py_contour_features} ================ +@prev_tutorial{tutorial_py_contours_begin} +@next_tutorial{tutorial_py_contour_properties} + Goal ---- diff --git a/doc/py_tutorials/py_imgproc/py_contours/py_contour_properties/py_contour_properties.markdown b/doc/py_tutorials/py_imgproc/py_contours/py_contour_properties/py_contour_properties.markdown index 461c870343..282f62ddf9 100644 --- a/doc/py_tutorials/py_imgproc/py_contours/py_contour_properties/py_contour_properties.markdown +++ b/doc/py_tutorials/py_imgproc/py_contours/py_contour_properties/py_contour_properties.markdown @@ -1,6 +1,9 @@ Contour Properties {#tutorial_py_contour_properties} ================== +@prev_tutorial{tutorial_py_contour_features} +@next_tutorial{tutorial_py_contours_more_functions} + Here we will learn to extract some frequently used properties of objects like Solidity, Equivalent Diameter, Mask image, Mean Intensity etc. More features can be found at [Matlab regionprops documentation](http://www.mathworks.in/help/images/ref/regionprops.html). diff --git a/doc/py_tutorials/py_imgproc/py_contours/py_contours_begin/py_contours_begin.markdown b/doc/py_tutorials/py_imgproc/py_contours/py_contours_begin/py_contours_begin.markdown index c2055f75af..74d7b252a5 100644 --- a/doc/py_tutorials/py_imgproc/py_contours/py_contours_begin/py_contours_begin.markdown +++ b/doc/py_tutorials/py_imgproc/py_contours/py_contours_begin/py_contours_begin.markdown @@ -1,6 +1,8 @@ Contours : Getting Started {#tutorial_py_contours_begin} ========================== +@next_tutorial{tutorial_py_contour_features} + Goal ---- diff --git a/doc/py_tutorials/py_imgproc/py_contours/py_contours_hierarchy/py_contours_hierarchy.markdown b/doc/py_tutorials/py_imgproc/py_contours/py_contours_hierarchy/py_contours_hierarchy.markdown index 2619ea2a70..075e6ec81f 100644 --- a/doc/py_tutorials/py_imgproc/py_contours/py_contours_hierarchy/py_contours_hierarchy.markdown +++ b/doc/py_tutorials/py_imgproc/py_contours/py_contours_hierarchy/py_contours_hierarchy.markdown @@ -1,6 +1,8 @@ Contours Hierarchy {#tutorial_py_contours_hierarchy} ================== +@prev_tutorial{tutorial_py_contours_more_functions} + Goal ---- diff --git a/doc/py_tutorials/py_imgproc/py_contours/py_contours_more_functions/py_contours_more_functions.markdown b/doc/py_tutorials/py_imgproc/py_contours/py_contours_more_functions/py_contours_more_functions.markdown index fb5f59bef6..397a2a63a0 100644 --- a/doc/py_tutorials/py_imgproc/py_contours/py_contours_more_functions/py_contours_more_functions.markdown +++ b/doc/py_tutorials/py_imgproc/py_contours/py_contours_more_functions/py_contours_more_functions.markdown @@ -1,6 +1,10 @@ Contours : More Functions {#tutorial_py_contours_more_functions} ========================= +@prev_tutorial{tutorial_py_contour_properties} +@next_tutorial{tutorial_py_contours_hierarchy} + + Goal ---- From 4ff76cad2a0b204a43ec4d57aebe65aa2a616543 Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Thu, 5 Aug 2021 11:42:58 +0000 Subject: [PATCH 099/262] cmake: fix cross-compilation problems - unexpected pkg-config module (we should not use host binary) - bump cmake_minimum_required to 3.5 in toolchain files --- modules/highgui/cmake/init.cmake | 4 +++- modules/videoio/cmake/init.cmake | 4 +++- platforms/linux/gnu.toolchain.cmake | 2 +- platforms/linux/riscv.toolchain.cmake | 2 +- 4 files changed, 8 insertions(+), 4 deletions(-) diff --git a/modules/highgui/cmake/init.cmake b/modules/highgui/cmake/init.cmake index a302c4d534..2002ff0e9d 100644 --- a/modules/highgui/cmake/init.cmake +++ b/modules/highgui/cmake/init.cmake @@ -25,7 +25,9 @@ endif() # Detect available dependencies # -include(FindPkgConfig) +if(NOT PROJECT_NAME STREQUAL "OpenCV") + include(FindPkgConfig) +endif() macro(add_backend backend_id cond_var) if(${cond_var}) diff --git a/modules/videoio/cmake/init.cmake b/modules/videoio/cmake/init.cmake index 68838790b8..af664f94df 100644 --- a/modules/videoio/cmake/init.cmake +++ b/modules/videoio/cmake/init.cmake @@ -1,4 +1,6 @@ -include(FindPkgConfig) +if(NOT PROJECT_NAME STREQUAL "OpenCV") + include(FindPkgConfig) +endif() macro(add_backend backend_id cond_var) if(${cond_var}) diff --git a/platforms/linux/gnu.toolchain.cmake b/platforms/linux/gnu.toolchain.cmake index cba08e7fbb..64258e65b3 100644 --- a/platforms/linux/gnu.toolchain.cmake +++ b/platforms/linux/gnu.toolchain.cmake @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 2.8) +cmake_minimum_required(VERSION 3.5) # load settings in case of "try compile" set(TOOLCHAIN_CONFIG_FILE "${CMAKE_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/toolchain.config.cmake") diff --git a/platforms/linux/riscv.toolchain.cmake b/platforms/linux/riscv.toolchain.cmake index 2a69d7e004..cea80bd9ba 100644 --- a/platforms/linux/riscv.toolchain.cmake +++ b/platforms/linux/riscv.toolchain.cmake @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 2.8) +cmake_minimum_required(VERSION 3.5) if(COMMAND toolchain_save_config) return() # prevent recursive call From 633fedaa96d1a6e48db5f0897d82fa5d2c5f4560 Mon Sep 17 00:00:00 2001 From: Julia Bareeva <34717687+JulieBar@users.noreply.github.com> Date: Thu, 5 Aug 2021 18:13:17 +0300 Subject: [PATCH 100/262] Merge pull request #20480 from JulieBar:lstm_pytest Add Python's test for LSTM layer * Add Python's test for LSTM layer * Set different test threshold for FP16 target * rename test to test_input_3d Co-authored-by: Julie Bareeva --- modules/dnn/misc/python/test/test_dnn.py | 36 ++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/modules/dnn/misc/python/test/test_dnn.py b/modules/dnn/misc/python/test/test_dnn.py index 932984f1c2..f7bfc01119 100644 --- a/modules/dnn/misc/python/test/test_dnn.py +++ b/modules/dnn/misc/python/test/test_dnn.py @@ -62,6 +62,12 @@ def printParams(backend, target): } print('%s/%s' % (backendNames[backend], targetNames[target])) +def getDefaultThreshold(target): + if target == cv.dnn.DNN_TARGET_OPENCL_FP16 or target == cv.dnn.DNN_TARGET_MYRIAD: + return 4e-3 + else: + return 1e-5 + testdata_required = bool(os.environ.get('OPENCV_DNN_TEST_REQUIRE_TESTDATA', False)) g_dnnBackendsAndTargets = None @@ -305,5 +311,35 @@ class dnn_test(NewOpenCVTests): cv.dnn_unregisterLayer('CropCaffe') + # check that dnn module can work with 3D tensor as input for network + def test_input_3d(self): + model = self.find_dnn_file('dnn/onnx/models/hidden_lstm.onnx') + input_file = self.find_dnn_file('dnn/onnx/data/input_hidden_lstm.npy') + output_file = self.find_dnn_file('dnn/onnx/data/output_hidden_lstm.npy') + if model is None: + raise unittest.SkipTest("Missing DNN test files (dnn/onnx/models/hidden_lstm.onnx). " + "Verify OPENCV_DNN_TEST_DATA_PATH configuration parameter.") + if input_file is None or output_file is None: + raise unittest.SkipTest("Missing DNN test files (dnn/onnx/data/{input/output}_hidden_lstm.npy). " + "Verify OPENCV_DNN_TEST_DATA_PATH configuration parameter.") + + net = cv.dnn.readNet(model) + input = np.load(input_file) + # we have to expand the shape of input tensor because Python bindings cut 3D tensors to 2D + # it should be fixed in future. see : https://github.com/opencv/opencv/issues/19091 + # please remove `expand_dims` after that + input = np.expand_dims(input, axis=3) + gold_output = np.load(output_file) + net.setInput(input) + + for backend, target in self.dnnBackendsAndTargets: + printParams(backend, target) + + net.setPreferableBackend(backend) + net.setPreferableTarget(target) + real_output = net.forward() + + normAssert(self, real_output, gold_output, "", getDefaultThreshold(target)) + if __name__ == '__main__': NewOpenCVTests.bootstrap() From 6306bc3ddcbe3480a6d113081bebedb5745dff24 Mon Sep 17 00:00:00 2001 From: Maxim Pashchenkov Date: Fri, 6 Aug 2021 09:52:46 +0300 Subject: [PATCH 101/262] Applying of comments --- .../misc/python/samples/gaze_estimation.py | 37 +++++++------------ 1 file changed, 14 insertions(+), 23 deletions(-) diff --git a/modules/gapi/misc/python/samples/gaze_estimation.py b/modules/gapi/misc/python/samples/gaze_estimation.py index db190f67bb..5536787e60 100644 --- a/modules/gapi/misc/python/samples/gaze_estimation.py +++ b/modules/gapi/misc/python/samples/gaze_estimation.py @@ -3,6 +3,7 @@ import time import numpy as np import cv2 as cv + # ------------------------Service operations------------------------ def weight_path(model_path): """ Get path of weights based on path to IR @@ -171,11 +172,7 @@ class GProcessPosesImpl: Return: Arrays with heads poses """ - out_poses = [] - size = len(in_ys) - for i in range(size): - out_poses.append(np.array([in_ys[i][0], in_ps[i][0], in_rs[i][0]]).T) - return out_poses + return [np.array([ys[0], ps[0], rs[0]]).T for ys, ps, rs in zip(in_ys, in_ps, in_rs)] @cv.gapi.kernel(GParseEyes) @@ -199,20 +196,19 @@ class GParseEyesImpl: right_eyes = [] midpoints = [] lmarks = [] - num_faces = len(in_landm_per_face) surface = (0, 0, *frame_size) - for i in range(num_faces): - rect = in_face_rcs[i] - points = process_landmarks(*rect, in_landm_per_face[i]) - for p in points: - lmarks.append(p) - size = int(len(in_landm_per_face[i][0]) / 2) + for landm_face, rect in zip(in_landm_per_face, in_face_rcs): + points = process_landmarks(*rect, landm_face) + lmarks.extend(points) - rect, midpoint_l = eye_box(lmarks[0 + i * size], lmarks[1 + i * size]) + rect, midpoint_l = eye_box(points[0], points[1]) left_eyes.append(intersection(surface, rect)) - rect, midpoint_r = eye_box(lmarks[2 + i * size], lmarks[3 + i * size]) + + rect, midpoint_r = eye_box(points[2], points[3]) right_eyes.append(intersection(surface, rect)) - midpoints += [midpoint_l, midpoint_r] + + midpoints.append(midpoint_l) + midpoints.append(midpoint_r) return left_eyes, right_eyes, midpoints, lmarks @@ -231,14 +227,8 @@ class GGetStatesImpl: Return: States of left eyes and states of right eyes """ - size = len(eyesl) - out_l_st = [] - out_r_st = [] - for i in range(size): - for st in eyesl[i]: - out_l_st += [1 if st[0] < st[1] else 0] - for st in eyesr[i]: - out_r_st += [1 if st[0] < st[1] else 0] + out_l_st = [int(st) for eye_l in eyesl for st in (eye_l[:, 0] < eye_l[:, 1]).ravel()] + out_r_st = [int(st) for eye_r in eyesr for st in (eye_r[:, 0] < eye_r[:, 1]).ravel()] return out_l_st, out_r_st @@ -459,6 +449,7 @@ if __name__ == '__main__': # Show result cv.imshow('Gaze Estimation', oimg) + cv.waitKey(1) fps = int(1. / (time.time() - start_time_cycle)) frames += 1 From 24de676a6480aae151b1a2c80a483ec3ef98ab2a Mon Sep 17 00:00:00 2001 From: Anatoliy Talamanov Date: Fri, 6 Aug 2021 13:26:49 +0300 Subject: [PATCH 102/262] Merge pull request #20476 from TolyaTalamanov:at/support-unet-camvid-0001-segm-sample [G-API] Support postprocessing for not argmaxed outputs * Support postprocessing for not argmaxed outputs * Fix typo * Add assert * Remove static cast * CamelCast to snake_case * Fix windows warning * Add static_cast to uint8_t * Add const to variables --- .../gapi/samples/semantic_segmentation.cpp | 83 ++++++++++++++----- 1 file changed, 63 insertions(+), 20 deletions(-) diff --git a/modules/gapi/samples/semantic_segmentation.cpp b/modules/gapi/samples/semantic_segmentation.cpp index 0a6e7231c4..4cdb14cc5c 100644 --- a/modules/gapi/samples/semantic_segmentation.cpp +++ b/modules/gapi/samples/semantic_segmentation.cpp @@ -47,6 +47,53 @@ std::string get_weights_path(const std::string &model_path) { CV_Assert(ext == ".xml"); return model_path.substr(0u, sz - EXT_LEN) + ".bin"; } + +void classesToColors(const cv::Mat &out_blob, + cv::Mat &mask_img) { + const int H = out_blob.size[0]; + const int W = out_blob.size[1]; + + mask_img.create(H, W, CV_8UC3); + GAPI_Assert(out_blob.type() == CV_8UC1); + const uint8_t* const classes = out_blob.ptr(); + + for (int rowId = 0; rowId < H; ++rowId) { + for (int colId = 0; colId < W; ++colId) { + uint8_t class_id = classes[rowId * W + colId]; + mask_img.at(rowId, colId) = + class_id < colors.size() + ? colors[class_id] + : cv::Vec3b{0, 0, 0}; // NB: sample supports 20 classes + } + } +} + +void probsToClasses(const cv::Mat& probs, cv::Mat& classes) { + const int C = probs.size[1]; + const int H = probs.size[2]; + const int W = probs.size[3]; + + classes.create(H, W, CV_8UC1); + GAPI_Assert(probs.depth() == CV_32F); + float* out_p = reinterpret_cast(probs.data); + uint8_t* classes_p = reinterpret_cast(classes.data); + + for (int h = 0; h < H; ++h) { + for (int w = 0; w < W; ++w) { + double max = 0; + int class_id = 0; + for (int c = 0; c < C; ++c) { + int idx = c * H * W + h * W + w; + if (out_p[idx] > max) { + max = out_p[idx]; + class_id = c; + } + } + classes_p[h * W + w] = static_cast(class_id); + } + } +} + } // anonymous namespace namespace custom { @@ -57,25 +104,21 @@ G_API_OP(PostProcessing, , "sample.custom.post_pro }; GAPI_OCV_KERNEL(OCVPostProcessing, PostProcessing) { - static void run(const cv::Mat &in, const cv::Mat &detected_classes, cv::Mat &out) { - // This kernel constructs output image by class table and colors vector - - // The semantic-segmentation-adas-0001 output a blob with the shape - // [B, C=1, H=1024, W=2048] - const int outHeight = 1024; - const int outWidth = 2048; - cv::Mat maskImg(outHeight, outWidth, CV_8UC3); - const int* const classes = detected_classes.ptr(); - for (int rowId = 0; rowId < outHeight; ++rowId) { - for (int colId = 0; colId < outWidth; ++colId) { - size_t classId = static_cast(classes[rowId * outWidth + colId]); - maskImg.at(rowId, colId) = - classId < colors.size() - ? colors[classId] - : cv::Vec3b{0, 0, 0}; // sample detects 20 classes - } + static void run(const cv::Mat &in, const cv::Mat &out_blob, cv::Mat &out) { + cv::Mat classes; + // NB: If output has more than single plane, it contains probabilities + // otherwise class id. + if (out_blob.size[1] > 1) { + probsToClasses(out_blob, classes); + } else { + out_blob.convertTo(classes, CV_8UC1); + classes = classes.reshape(1, out_blob.size[2]); } - cv::resize(maskImg, out, in.size()); + + cv::Mat mask_img; + classesToColors(classes, mask_img); + + cv::resize(mask_img, out, in.size()); const float blending = 0.3f; out = in * blending + out * (1 - blending); } @@ -104,8 +147,8 @@ int main(int argc, char *argv[]) { // Now build the graph cv::GMat in; - cv::GMat detected_classes = cv::gapi::infer(in); - cv::GMat out = custom::PostProcessing::on(in, detected_classes); + cv::GMat out_blob = cv::gapi::infer(in); + cv::GMat out = custom::PostProcessing::on(in, out_blob); cv::GStreamingCompiled pipeline = cv::GComputation(cv::GIn(in), cv::GOut(out)) .compileStreaming(cv::compile_args(kernels, networks)); From ba539eb9aad7571361657ea7e00a7c3efcc2f9ba Mon Sep 17 00:00:00 2001 From: Anatoliy Talamanov Date: Fri, 6 Aug 2021 14:41:56 +0300 Subject: [PATCH 103/262] Merge pull request #20508 from TolyaTalamanov:at/expand-python-pyparams [G-API] Expand PyParams to support constInput * Wrap constInputs to python * Wrap cfgNumRequests * Fix alignment * Move macro to the line above --- .../gapi/include/opencv2/gapi/infer/bindings_ie.hpp | 11 +++++++++++ modules/gapi/src/backends/ie/bindings_ie.cpp | 12 ++++++++++++ 2 files changed, 23 insertions(+) diff --git a/modules/gapi/include/opencv2/gapi/infer/bindings_ie.hpp b/modules/gapi/include/opencv2/gapi/infer/bindings_ie.hpp index fdd4128b1a..92ef2101a1 100644 --- a/modules/gapi/include/opencv2/gapi/infer/bindings_ie.hpp +++ b/modules/gapi/include/opencv2/gapi/infer/bindings_ie.hpp @@ -22,17 +22,28 @@ namespace ie { // This class can be marked as SIMPLE, because it's implemented as pimpl class GAPI_EXPORTS_W_SIMPLE PyParams { public: + GAPI_WRAP PyParams() = default; + GAPI_WRAP PyParams(const std::string &tag, const std::string &model, const std::string &weights, const std::string &device); + GAPI_WRAP PyParams(const std::string &tag, const std::string &model, const std::string &device); + GAPI_WRAP + PyParams& constInput(const std::string &layer_name, + const cv::Mat &data, + TraitAs hint = TraitAs::TENSOR); + + GAPI_WRAP + PyParams& cfgNumRequests(size_t nireq); + GBackend backend() const; std::string tag() const; cv::util::any params() const; diff --git a/modules/gapi/src/backends/ie/bindings_ie.cpp b/modules/gapi/src/backends/ie/bindings_ie.cpp index 35191d7bcb..5874fe1378 100644 --- a/modules/gapi/src/backends/ie/bindings_ie.cpp +++ b/modules/gapi/src/backends/ie/bindings_ie.cpp @@ -37,3 +37,15 @@ cv::gapi::ie::PyParams cv::gapi::ie::params(const std::string &tag, const std::string &device) { return {tag, model, device}; } + +cv::gapi::ie::PyParams& cv::gapi::ie::PyParams::constInput(const std::string &layer_name, + const cv::Mat &data, + TraitAs hint) { + m_priv->constInput(layer_name, data, hint); + return *this; +} + +cv::gapi::ie::PyParams& cv::gapi::ie::PyParams::cfgNumRequests(size_t nireq) { + m_priv->cfgNumRequests(nireq); + return *this; +} From e1cafa383431a65e4ca0493a21668444d10a14b3 Mon Sep 17 00:00:00 2001 From: Julia Bareeva <34717687+JulieBar@users.noreply.github.com> Date: Sat, 7 Aug 2021 10:07:37 +0300 Subject: [PATCH 104/262] Merge pull request #20442 from JulieBar:gru_layer * Add initialization and inference for GRU layer * fix issues found on review --- .../dnn/include/opencv2/dnn/all_layers.hpp | 34 +++ modules/dnn/src/init.cpp | 1 + modules/dnn/src/layers/recurrent_layers.cpp | 209 ++++++++++++++++++ modules/dnn/src/onnx/onnx_importer.cpp | 43 ++++ modules/dnn/test/test_layers.cpp | 29 +++ modules/dnn/test/test_onnx_importer.cpp | 10 + 6 files changed, 326 insertions(+) diff --git a/modules/dnn/include/opencv2/dnn/all_layers.hpp b/modules/dnn/include/opencv2/dnn/all_layers.hpp index 24d35646df..9c96c5a5f1 100644 --- a/modules/dnn/include/opencv2/dnn/all_layers.hpp +++ b/modules/dnn/include/opencv2/dnn/all_layers.hpp @@ -165,6 +165,40 @@ CV__DNN_INLINE_NS_BEGIN int outputNameToIndex(const String& outputName) CV_OVERRIDE; }; + /** @brief GRU recurrent one-layer + * + * Accepts input sequence and computes the final hidden state for each element in the batch. + * + * - input[0] containing the features of the input sequence. + * input[0] should have shape [`T`, `N`, `data_dims`] where `T` is sequence length, `N` is batch size, `data_dims` is input size + * - output would have shape [`T`, `N`, `D` * `hidden_size`] where `D = 2` if layer is bidirectional otherwise `D = 1` + * + * Depends on the following attributes: + * - hidden_size - Number of neurons in the hidden layer + * - direction - RNN could be bidirectional or forward + * + * The final hidden state @f$ h_t @f$ computes by the following formulas: + * + @f{eqnarray*}{ + r_t = \sigma(W_{ir} x_t + b_{ir} + W_{hr} h_{(t-1)} + b_{hr}) \\ + z_t = \sigma(W_{iz} x_t + b_{iz} + W_{hz} h_{(t-1)} + b_{hz}) \\ + n_t = \tanh(W_{in} x_t + b_{in} + r_t \odot (W_{hn} h_{(t-1)}+ b_{hn})) \\ + h_t = (1 - z_t) \odot n_t + z_t \odot h_{(t-1)} \\ + @f} + * Where @f$x_t@f$ is current input, @f$h_{(t-1)}@f$ is previous or initial hidden state. + * + * @f$W_{x?}@f$, @f$W_{h?}@f$ and @f$b_{?}@f$ are learned weights represented as matrices: + * @f$W_{x?} \in R^{N_h \times N_x}@f$, @f$W_{h?} \in R^{N_h \times N_h}@f$, @f$b_? \in R^{N_h}@f$. + * + * @f$\odot@f$ is per-element multiply operation. + */ + class CV_EXPORTS GRULayer : public Layer + { + public: + /** Creates instance of GRU layer */ + static Ptr create(const LayerParams& params); + }; + /** @brief Classical recurrent layer Accepts two inputs @f$x_t@f$ and @f$h_{t-1}@f$ and compute two outputs @f$o_t@f$ and @f$h_t@f$. diff --git a/modules/dnn/src/init.cpp b/modules/dnn/src/init.cpp index 698168817f..ebd887999b 100644 --- a/modules/dnn/src/init.cpp +++ b/modules/dnn/src/init.cpp @@ -139,6 +139,7 @@ void initializeLayerFactory() CV_DNN_REGISTER_LAYER_CLASS(FlowWarp, FlowWarpLayer); CV_DNN_REGISTER_LAYER_CLASS(LSTM, LSTMLayer); + CV_DNN_REGISTER_LAYER_CLASS(GRU, GRULayer); } CV__DNN_INLINE_NS_END diff --git a/modules/dnn/src/layers/recurrent_layers.cpp b/modules/dnn/src/layers/recurrent_layers.cpp index feae35dac0..cb2ffb8cc9 100644 --- a/modules/dnn/src/layers/recurrent_layers.cpp +++ b/modules/dnn/src/layers/recurrent_layers.cpp @@ -563,5 +563,214 @@ CV_EXPORTS_W Ptr RNNLayer::create(const LayerParams& params) return Ptr(new RNNLayerImpl(params)); } +class GRULayerImpl CV_FINAL : public GRULayer +{ + int numTimeStamps, numSamples; + bool allocated; + + MatShape outTailShape; //shape of single output sample + MatShape outTsShape; //shape of N output samples + bool bidirectional; // If true, produces both forward and reversed directions along time axis + +public: + + GRULayerImpl(const LayerParams& params) : numTimeStamps(0), numSamples(0) + { + setParamsFrom(params); + + bidirectional = params.get("bidirectional", false); + if (!blobs.empty()) + { + CV_Assert(blobs.size() >= 3); + + blobs[2] = blobs[2].reshape(1, 1); + + const Mat& Wh = blobs[0]; + const Mat& Wx = blobs[1]; + const Mat& bias = blobs[2]; + const Mat& hInternal = blobs[3]; + CV_CheckEQ(Wh.dims, 2, ""); + CV_CheckEQ(Wx.dims, 2, ""); + CV_CheckEQ(Wh.rows, Wx.rows, ""); + CV_CheckEQ(Wh.rows, (1 + static_cast(bidirectional)) * 3 * Wh.cols, ""); + CV_CheckEQ(Wh.rows * 2, (int)bias.total(), ""); + CV_CheckEQ(hInternal.cols, Wh.cols, ""); + CV_CheckTypeEQ(Wh.type(), Wx.type(), ""); + CV_CheckTypeEQ(Wx.type(), bias.type(), ""); + } + + allocated = false; + outTailShape.clear(); + } + + bool getMemoryShapes(const std::vector &inputs, + const int requiredOutputs, + std::vector &outputs, + std::vector &internals) const CV_OVERRIDE + { + CV_Assert(inputs.size() == 1); + const MatShape& inp0 = inputs[0]; + + const Mat &Wh = blobs[0], &Wx = blobs[1]; + int _numOut = Wh.size[1]; + int _numInp = Wx.size[1]; + MatShape outTailShape_(outTailShape), outResShape; + + if (!outTailShape_.empty()) + CV_Assert(total(outTailShape_) == _numOut); + else + outTailShape_.assign(1, _numOut); + + int _numSamples; + CV_Assert(inp0.size() >= 2 && total(inp0, 2) == _numInp); + _numSamples = inp0[1]; + outResShape.push_back(inp0[0]); + + outResShape.push_back(_numSamples); + outResShape.insert(outResShape.end(), outTailShape_.begin(), outTailShape_.end()); + outResShape.back() *= (1 + static_cast(bidirectional)); + + outputs.assign(1, outResShape); + + internals.assign(1, shape(_numSamples, _numOut)); // hInternal + internals.push_back(shape(_numSamples, 1)); // dummyOnes + internals.push_back(shape(_numSamples, 2 * _numOut)); // gates + internals.push_back(shape(_numSamples, 2 * _numOut)); // gates_b + internals.push_back(shape(_numSamples, 1 * _numOut)); // h_linear + internals.push_back(shape(_numSamples, _numOut)); // ones + + return false; + } + + void finalize(InputArrayOfArrays inputs_arr, OutputArrayOfArrays) CV_OVERRIDE + { + std::vector input; + inputs_arr.getMatVector(input); + + CV_Assert(input.size() == 1); + const Mat& inp0 = input[0]; + + Mat &Wh = blobs[0], &Wx = blobs[1]; + int numOut = Wh.size[1]; + int numInp = Wx.size[1]; + + if (!outTailShape.empty()) + CV_Assert(total(outTailShape) == numOut); + else + outTailShape.assign(1, numOut); + + CV_Assert(inp0.dims >= 2 && (int)inp0.total(2) == numInp); + numTimeStamps = inp0.size[0]; + numSamples = inp0.size[1]; + + outTsShape.clear(); + outTsShape.push_back(numSamples); + outTsShape.insert(outTsShape.end(), outTailShape.begin(), outTailShape.end()); + outTsShape.back() *= (1 + static_cast(bidirectional)); + + allocated = true; + } + + void forward(InputArrayOfArrays inputs_arr, OutputArrayOfArrays outputs_arr, OutputArrayOfArrays internals_arr) CV_OVERRIDE + { + CV_TRACE_FUNCTION(); + CV_TRACE_ARG_VALUE(name, "name", name.c_str()); + + if (inputs_arr.depth() == CV_16S) + { + forward_fallback(inputs_arr, outputs_arr, internals_arr); + return; + } + + std::vector input, output, internals; + inputs_arr.getMatVector(input); + outputs_arr.getMatVector(output); + internals_arr.getMatVector(internals); + + const int numDirs = 1 + static_cast(bidirectional); + for (int i = 0; i < numDirs; ++i) + { + const Mat &Wh = blobs[0].rowRange(i * blobs[0].rows / numDirs, (i + 1) * blobs[0].rows / numDirs); + const Mat &Wx = blobs[1].rowRange(i * blobs[1].rows / numDirs, (i + 1) * blobs[1].rows / numDirs); + const Mat &bias = blobs[2].colRange(i * blobs[2].cols / numDirs, (i + 1) * blobs[2].cols / numDirs); + const Mat &h_0 = blobs[3].rowRange(i * blobs[3].rows / numDirs, (i + 1) * blobs[3].rows / numDirs); + + const Mat &bx = bias.colRange(0, bias.cols / 2); + const Mat &bh = bias.colRange(bias.cols / 2, bias.cols); + + Mat hInternal = internals[0], dummyOnes = internals[1], gates = internals[2], + b_rz = internals[3], n_t = internals[4], ones = internals[5]; + h_0.copyTo(hInternal); + dummyOnes.setTo(1.); + ones.setTo(1.); + + int numOut = Wh.size[1]; + const Mat& wx_rz = Wx.rowRange(0, 2 * numOut); + const Mat& wh_rz = Wh.rowRange(0, 2 * numOut); + b_rz = bx.colRange(0, 2 * numOut) + bh.colRange(0, 2 * numOut); + const Mat& wx_n = Wx.rowRange(2 * numOut, 3 * numOut); + const Mat& wh_n = Wh.rowRange(2 * numOut, 3 * numOut); + const Mat& b_in = bx.colRange(2 * numOut, 3 * numOut); + const Mat& b_hn = bh.colRange(2 * numOut, 3 * numOut); + + int numSamplesTotal = numTimeStamps * numSamples; + Mat xTs = input[0].reshape(1, numSamplesTotal); + + Mat hOutTs = output[0].reshape(1, numSamplesTotal); + hOutTs = hOutTs.colRange(i * hOutTs.cols / numDirs, (i + 1) * hOutTs.cols / numDirs); + Mat cOutTs = Mat(); + + int tsStart, tsEnd, tsInc; + if (i == 1) { + tsStart = numTimeStamps - 1; + tsEnd = -1; + tsInc = -1; + } + else { + tsStart = 0; + tsEnd = numTimeStamps; + tsInc = 1; + } + for (int ts = tsStart; ts != tsEnd; ts += tsInc) + { + Range curRowRange(ts * numSamples, (ts + 1) * numSamples); + Mat xCurr = xTs.rowRange(curRowRange); + + // calculate r_t = sigmoid(x * Wx_r + h_(t-1) * Wh_r + b_r) + // calculate z_t = sigmoid(x * Wx_z + h_(t-1) * Wh_z + b_z) + gemm(xCurr, wx_rz, 1, gates, 0, gates, GEMM_2_T); // x * Wx_rz + gemm(hInternal, wh_rz, 1, gates, 1, gates, GEMM_2_T); // + h_(t-1) * Wh_rz + gemm(dummyOnes, b_rz, 1, gates, 1, gates); // + b_rz + sigmoid(gates, gates); // sigmoid() + + Mat z = gates.colRange(0, gates.cols / 2); + Mat r = gates.colRange(gates.cols / 2, gates.cols); + + // calculate n_t = tanh(r (*) (h_(t-1) * Wh_n + b_hn) + x * Wx_n + b_in) + gemm(hInternal, wh_n, 1, n_t, 0, n_t, GEMM_2_T); // h_(t-1) * Wh_n + gemm(dummyOnes, b_hn, 1, n_t, 1, n_t); // + b_hn + multiply(r, n_t, n_t); // r (*) (h_(t-1) * Wh_n + b_hn) + + gemm(xCurr, wx_n, 1, n_t, 1, n_t, GEMM_2_T); // + x * Wx_n + gemm(dummyOnes, b_in, 1, n_t, 1, n_t); // + b_in + tanh(n_t, n_t); // tanh() + + //compute next h_t = z (*) h_(t-1) + (1 - z) (*) n_t + multiply(z, hInternal, hInternal); // z (*) h_{t-1} + subtract(ones, z, z); // 1 - z + multiply(z, n_t, z); // (1 - z) * n + add(z, hInternal, hInternal); // z (*) h_(t-1) + (1 - z) (*) n_t + + //save results in output blobs + hInternal.copyTo(hOutTs.rowRange(curRowRange)); + } + } + } +}; + +Ptr GRULayer::create(const LayerParams ¶ms) { + return Ptr(new GRULayerImpl(params)); +} + } } diff --git a/modules/dnn/src/onnx/onnx_importer.cpp b/modules/dnn/src/onnx/onnx_importer.cpp index 2573d783d8..dee7e128fa 100644 --- a/modules/dnn/src/onnx/onnx_importer.cpp +++ b/modules/dnn/src/onnx/onnx_importer.cpp @@ -145,6 +145,7 @@ private: void parseNeg (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); void parseConstant (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); void parseLSTM (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseGRU (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); void parseImageScaler (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); void parseClip (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); void parseLeakyRelu (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); @@ -582,6 +583,7 @@ const std::set& ONNXImporter::getSupportedTypes() "Neg", "Constant", "LSTM", + "GRU", "ImageScaler", "Clip", "LeakyRelu", @@ -1239,6 +1241,46 @@ void ONNXImporter::parseLSTM(LayerParams& layerParams, const opencv_onnx::NodePr addLayer(layerParams, node_proto); } +void ONNXImporter::parseGRU(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto_) +{ + opencv_onnx::NodeProto node_proto = node_proto_; + LayerParams gruParams = layerParams; + gruParams.name += "/gru"; + + // https://pytorch.org/docs/stable/generated/torch.nn.GRU.html?highlight=gru# + CV_Assert(node_proto.input_size() == 6); + Mat Wx = getBlob(node_proto, 1); + Mat Wh = getBlob(node_proto, 2); + Mat b = getBlob(node_proto, 3); + Mat h0 = getBlob(node_proto, 5); + + Wx = Wx.reshape(1, Wx.size[0] * Wx.size[1]); + Wh = Wh.reshape(1, Wh.size[0] * Wh.size[1]); + h0 = h0.reshape(1, h0.size[0] * h0.size[1]); + b = b.reshape(1, b.size[0]); + + gruParams.blobs.resize(4); + gruParams.blobs[0] = Wh; + gruParams.blobs[1] = Wx; + gruParams.blobs[2] = b; + gruParams.blobs[3] = h0; + gruParams.set("bidirectional", gruParams.get("direction", "") == "bidirectional"); + + node_proto.set_output(0, gruParams.name); // set different name so output shapes will be registered on that name + addLayer(gruParams, node_proto); + + MatShape gruShape = outShapes[node_proto.output(0)]; + + // Add fake 1 as it is done in ONNX + gruShape.insert(gruShape.begin() + 1, 1); + + layerParams.type = "Reshape"; + layerParams.set("dim", DictValue::arrayInt(&gruShape[0], gruShape.size())); + node_proto.set_input(0, gruParams.name); // redirect input to GRU + node_proto.set_output(0, layerParams.name); // keep origin GRU's name + addLayer(layerParams, node_proto); +} + void ONNXImporter::parseImageScaler(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) { const float scale = layerParams.has("scale") ? layerParams.get("scale") : 1.0f; @@ -2358,6 +2400,7 @@ const ONNXImporter::DispatchMap ONNXImporter::buildDispatchMap() dispatch["Neg"] = &ONNXImporter::parseNeg; dispatch["Constant"] = &ONNXImporter::parseConstant; dispatch["LSTM"] = &ONNXImporter::parseLSTM; + dispatch["GRU"] = &ONNXImporter::parseGRU; dispatch["ImageScaler"] = &ONNXImporter::parseImageScaler; dispatch["Clip"] = &ONNXImporter::parseClip; dispatch["LeakyRelu"] = &ONNXImporter::parseLeakyRelu; diff --git a/modules/dnn/test/test_layers.cpp b/modules/dnn/test/test_layers.cpp index 1383c59e28..04d5fa6355 100644 --- a/modules/dnn/test/test_layers.cpp +++ b/modules/dnn/test/test_layers.cpp @@ -596,6 +596,35 @@ TEST(Layer_LSTM_Test_Accuracy_with_, HiddenParams) normAssert(h_t_reference, outputs[0]); } +TEST(Layer_GRU_Test_Accuracy_with_, Pytorch) +{ + Mat Wx = blobFromNPY(_tf("gru.W.npy")); + Mat Wh = blobFromNPY(_tf("gru.R.npy")); + Mat b = blobFromNPY(_tf("gru.B.npy")); + Mat h0 = blobFromNPY(_tf("gru.h0.npy")); + + Wx = Wx.reshape(1, Wx.size[0] * Wx.size[1]); + Wh = Wh.reshape(1, Wh.size[0] * Wh.size[1]); + h0 = h0.reshape(1, h0.size[0] * h0.size[1]); + b = b.reshape(1, b.size[0]); + + LayerParams gruParams; + gruParams.blobs.resize(4); + gruParams.blobs[0] = Wh; + gruParams.blobs[1] = Wx; + gruParams.blobs[2] = b; + gruParams.blobs[3] = h0; + gruParams.set("bidirectional", false); + Ptr layer = GRULayer::create(gruParams); + + Mat inp = blobFromNPY(_tf("gru.input.npy")); + std::vector inputs(1, inp), outputs; + runLayer(layer, inputs, outputs); + + Mat h_t_reference = blobFromNPY(_tf("gru.output.npy")); + normAssert(h_t_reference, outputs[0]); +} + TEST(Layer_RNN_Test_Accuracy_with_, CaffeRecurrent) { Ptr layer = RNNLayer::create(LayerParams()); diff --git a/modules/dnn/test/test_onnx_importer.cpp b/modules/dnn/test/test_onnx_importer.cpp index 8bfd864955..0e252cca83 100644 --- a/modules/dnn/test/test_onnx_importer.cpp +++ b/modules/dnn/test/test_onnx_importer.cpp @@ -720,6 +720,16 @@ TEST_P(Test_ONNX_layers, LSTM_hidden_bidirectional) testONNXModels("hidden_lstm_bi", npy, 0, 0, false, false); } +TEST_P(Test_ONNX_layers, GRU) +{ + testONNXModels("gru", npy, 0, 0, false, false); +} + +TEST_P(Test_ONNX_layers, GRU_bidirectional) +{ + testONNXModels("gru_bi", npy, 0, 0, false, false); +} + TEST_P(Test_ONNX_layers, Pad2d_Unfused) { testONNXModels("ReflectionPad2d"); From 2a177052de55c85554194a2464a91e6e09c7f768 Mon Sep 17 00:00:00 2001 From: SamFC10 Date: Mon, 9 Aug 2021 12:08:55 +0530 Subject: [PATCH 105/262] fix bug in prior-box variances --- .../dnn/src/layers/detection_output_layer.cpp | 22 +++++++++++-------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/modules/dnn/src/layers/detection_output_layer.cpp b/modules/dnn/src/layers/detection_output_layer.cpp index 8374d74293..614b3a6462 100644 --- a/modules/dnn/src/layers/detection_output_layer.cpp +++ b/modules/dnn/src/layers/detection_output_layer.cpp @@ -456,7 +456,7 @@ public: // Retrieve all prior bboxes std::vector priorBBoxes; std::vector > priorVariances; - GetPriorBBoxes(priorData, numPriors, _bboxesNormalized, priorBBoxes, priorVariances); + GetPriorBBoxes(priorData, numPriors, _bboxesNormalized, _varianceEncodedInTarget, priorBBoxes, priorVariances); // Decode all loc predictions to bboxes util::NormalizedBBox clipBounds; @@ -750,7 +750,7 @@ public: CV_Assert(prior_bboxes.size() == prior_variances.size()); CV_Assert(prior_bboxes.size() == bboxes.size()); size_t num_bboxes = prior_bboxes.size(); - CV_Assert(num_bboxes == 0 || prior_variances[0].size() == 4); + CV_Assert(num_bboxes == 0 || prior_variances[0].size() == 4 || variance_encoded_in_target); decode_bboxes.clear(); decode_bboxes.resize(num_bboxes); if(variance_encoded_in_target) { @@ -802,12 +802,13 @@ public: } // Get prior bounding boxes from prior_data - // prior_data: 1 x 2 x num_priors * 4 x 1 blob. + // prior_data: 1 x 1 x num_priors * 4 x 1 blob or 1 x 2 x num_priors * 4 x 1 blob. // num_priors: number of priors. // prior_bboxes: stores all the prior bboxes in the format of util::NormalizedBBox. // prior_variances: stores all the variances needed by prior bboxes. static void GetPriorBBoxes(const float* priorData, const int& numPriors, - bool normalized_bbox, std::vector& priorBBoxes, + bool normalized_bbox, bool variance_encoded_in_target, + std::vector& priorBBoxes, std::vector >& priorVariances) { priorBBoxes.clear(); priorBBoxes.resize(numPriors); @@ -823,13 +824,16 @@ public: bbox.set_size(BBoxSize(bbox, normalized_bbox)); } - for (int i = 0; i < numPriors; ++i) + if (!variance_encoded_in_target) { - int startIdx = (numPriors + i) * 4; - // not needed here: priorVariances[i].clear(); - for (int j = 0; j < 4; ++j) + for (int i = 0; i < numPriors; ++i) { - priorVariances[i].push_back(priorData[startIdx + j]); + int startIdx = (numPriors + i) * 4; + // not needed here: priorVariances[i].clear(); + for (int j = 0; j < 4; ++j) + { + priorVariances[i].push_back(priorData[startIdx + j]); + } } } } From 739ff84732f6385a43b32b37d875921b9073d009 Mon Sep 17 00:00:00 2001 From: Smirnov Egor Date: Mon, 9 Aug 2021 13:28:33 +0300 Subject: [PATCH 106/262] add Max layer to TFImporter --- modules/dnn/src/tensorflow/tf_importer.cpp | 26 +++++++++++++++++----- modules/dnn/test/test_tf_importer.cpp | 22 ++++++++++++++++++ 2 files changed, 43 insertions(+), 5 deletions(-) diff --git a/modules/dnn/src/tensorflow/tf_importer.cpp b/modules/dnn/src/tensorflow/tf_importer.cpp index fa33211a50..ca9d7c5e21 100644 --- a/modules/dnn/src/tensorflow/tf_importer.cpp +++ b/modules/dnn/src/tensorflow/tf_importer.cpp @@ -647,7 +647,7 @@ const TFImporter::DispatchMap TFImporter::buildDispatchMap() dispatch["PriorBox"] = &TFImporter::parsePriorBox; dispatch["Softmax"] = &TFImporter::parseSoftmax; dispatch["CropAndResize"] = &TFImporter::parseCropAndResize; - dispatch["Mean"] = dispatch["Sum"] = &TFImporter::parseMean; + dispatch["Mean"] = dispatch["Sum"] = dispatch["Max"] = &TFImporter::parseMean; dispatch["Pack"] = &TFImporter::parsePack; dispatch["ClipByValue"] = &TFImporter::parseClipByValue; dispatch["LeakyRelu"] = &TFImporter::parseLeakyRelu; @@ -657,6 +657,7 @@ const TFImporter::DispatchMap TFImporter::buildDispatchMap() return dispatch; } +// "Conv2D" "SpaceToBatchND" "DepthwiseConv2dNative" "Pad" "MirrorPad" "Conv3D" void TFImporter::parseConvolution(tensorflow::GraphDef& net, const tensorflow::NodeDef& layer_, LayerParams& layerParams) { tensorflow::NodeDef layer = layer_; @@ -876,6 +877,7 @@ void TFImporter::parseConvolution(tensorflow::GraphDef& net, const tensorflow::N data_layouts[name] = DATA_LAYOUT_NHWC; } +// "BiasAdd" "Add" "AddV2" "Sub" "AddN" void TFImporter::parseBias(tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams) { const std::string& name = layer.name(); @@ -1087,6 +1089,7 @@ void TFImporter::parseReshape(tensorflow::GraphDef& net, const tensorflow::NodeD } } +// "Flatten" "Squeeze" void TFImporter::parseFlatten(tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams) { const std::string& name = layer.name(); @@ -1245,6 +1248,7 @@ void TFImporter::parseLrn(tensorflow::GraphDef& net, const tensorflow::NodeDef& connectToAllBlobs(layer_id, dstNet, parsePin(layer.input(0)), id, num_inputs); } +// "Concat" "ConcatV2" void TFImporter::parseConcat(tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams) { const std::string& name = layer.name(); @@ -1295,6 +1299,7 @@ void TFImporter::parseConcat(tensorflow::GraphDef& net, const tensorflow::NodeDe } } +// "MaxPool" "MaxPool3D" void TFImporter::parseMaxPool(tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams) { const std::string& name = layer.name(); @@ -1316,6 +1321,7 @@ void TFImporter::parseMaxPool(tensorflow::GraphDef& net, const tensorflow::NodeD connectToAllBlobs(layer_id, dstNet, parsePin(inputName), id, num_inputs); } +// "AvgPool" "AvgPool3D" void TFImporter::parseAvgPool(tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams) { const std::string& name = layer.name(); @@ -1502,6 +1508,7 @@ void TFImporter::parseStridedSlice(tensorflow::GraphDef& net, const tensorflow:: connect(layer_id, dstNet, parsePin(layer.input(0)), id, 0); } +// "Mul" "RealDiv" void TFImporter::parseMul(tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams) { const std::string& name = layer.name(); @@ -1659,6 +1666,7 @@ void TFImporter::parseMul(tensorflow::GraphDef& net, const tensorflow::NodeDef& } } +// "FusedBatchNorm" "FusedBatchNormV3" void TFImporter::parseFusedBatchNorm(tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams) { // op: "FusedBatchNorm" @@ -1918,6 +1926,7 @@ void TFImporter::parseBlockLSTM(tensorflow::GraphDef& net, const tensorflow::Nod data_layouts[name] = DATA_LAYOUT_UNKNOWN; } +// "ResizeNearestNeighbor" "ResizeBilinear" "FusedResizeAndPadConv2D" void TFImporter::parseResize(tensorflow::GraphDef& net, const tensorflow::NodeDef& layer_, LayerParams& layerParams) { tensorflow::NodeDef layer = layer_; @@ -2106,6 +2115,7 @@ void TFImporter::parseCropAndResize(tensorflow::GraphDef& net, const tensorflow: connect(layer_id, dstNet, parsePin(layer.input(1)), id, 1); } +// "Mean" "Sum" "Max" void TFImporter::parseMean(tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams) { // Computes the mean of elements across dimensions of a tensor. @@ -2124,7 +2134,12 @@ void TFImporter::parseMean(tensorflow::GraphDef& net, const tensorflow::NodeDef& const std::string& name = layer.name(); const std::string& type = layer.op(); const int num_inputs = layer.input_size(); + std::string pool_type = cv::toLowerCase(type); + if (pool_type == "mean") + { + pool_type = "ave"; + } CV_CheckGT(num_inputs, 0, ""); Mat indices = getTensorContent(getConstBlob(layer, value_id, 1)); @@ -2161,7 +2176,7 @@ void TFImporter::parseMean(tensorflow::GraphDef& net, const tensorflow::NodeDef& LayerParams avgLp; std::string avgName = name + "/avg"; CV_Assert(layer_id.find(avgName) == layer_id.end()); - avgLp.set("pool", type == "Mean" ? "ave" : "sum"); + avgLp.set("pool", pool_type); // pooling kernel H x 1 avgLp.set("global_pooling_h", true); avgLp.set("kernel_w", 1); @@ -2202,7 +2217,7 @@ void TFImporter::parseMean(tensorflow::GraphDef& net, const tensorflow::NodeDef& int axis = toNCHW(indices.at(0)); if (axis == 2 || axis == 3) { - layerParams.set("pool", type == "Mean" ? "ave" : "sum"); + layerParams.set("pool", pool_type); layerParams.set(axis == 2 ? "kernel_w" : "kernel_h", 1); layerParams.set(axis == 2 ? "global_pooling_h" : "global_pooling_w", true); int id = dstNet.addLayer(name, "Pooling", layerParams); @@ -2234,7 +2249,7 @@ void TFImporter::parseMean(tensorflow::GraphDef& net, const tensorflow::NodeDef& Pin inpId = parsePin(layer.input(0)); addPermuteLayer(order, name + "/nhwc", inpId); - layerParams.set("pool", type == "Mean" ? "ave" : "sum"); + layerParams.set("pool", pool_type); layerParams.set("kernel_h", 1); layerParams.set("global_pooling_w", true); int id = dstNet.addLayer(name, "Pooling", layerParams); @@ -2264,7 +2279,7 @@ void TFImporter::parseMean(tensorflow::GraphDef& net, const tensorflow::NodeDef& if (indices.total() != 2 || indices.at(0) != 1 || indices.at(1) != 2) CV_Error(Error::StsNotImplemented, "Unsupported mode of reduce_mean or reduce_sum operation."); - layerParams.set("pool", type == "Mean" ? "ave" : "sum"); + layerParams.set("pool", pool_type); layerParams.set("global_pooling", true); int id = dstNet.addLayer(name, "Pooling", layerParams); layer_id[name] = id; @@ -2368,6 +2383,7 @@ void TFImporter::parseLeakyRelu(tensorflow::GraphDef& net, const tensorflow::Nod connectToAllBlobs(layer_id, dstNet, parsePin(layer.input(0)), id, num_inputs); } +// "Abs" "Tanh" "Sigmoid" "Relu" "Elu" "Exp" "Identity" "Relu6" void TFImporter::parseActivation(tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams) { const std::string& name = layer.name(); diff --git a/modules/dnn/test/test_tf_importer.cpp b/modules/dnn/test/test_tf_importer.cpp index 4f7840f9e4..68d6e88a66 100644 --- a/modules/dnn/test/test_tf_importer.cpp +++ b/modules/dnn/test/test_tf_importer.cpp @@ -128,6 +128,13 @@ TEST_P(Test_TensorFlow_layers, reduce_mean) runTensorFlowNet("global_pool_by_axis"); } +TEST_P(Test_TensorFlow_layers, reduce_max) +{ + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_NN_BUILDER); + runTensorFlowNet("max_pool_by_axis"); +} + TEST_P(Test_TensorFlow_layers, reduce_sum) { if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019) @@ -135,11 +142,21 @@ TEST_P(Test_TensorFlow_layers, reduce_sum) runTensorFlowNet("sum_pool_by_axis"); } +TEST_P(Test_TensorFlow_layers, reduce_max_channel) +{ + runTensorFlowNet("reduce_max_channel"); +} + TEST_P(Test_TensorFlow_layers, reduce_sum_channel) { runTensorFlowNet("reduce_sum_channel"); } +TEST_P(Test_TensorFlow_layers, reduce_max_channel_keep_dims) +{ + runTensorFlowNet("reduce_max_channel", false, 0.0, 0.0, false, "_keep_dims"); +} + TEST_P(Test_TensorFlow_layers, reduce_sum_channel_keep_dims) { runTensorFlowNet("reduce_sum_channel", false, 0.0, 0.0, false, "_keep_dims"); @@ -386,6 +403,11 @@ TEST_P(Test_TensorFlow_layers, pooling_reduce_mean) runTensorFlowNet("reduce_mean"); // an average pooling over all spatial dimensions. } +TEST_P(Test_TensorFlow_layers, pooling_reduce_max) +{ + runTensorFlowNet("reduce_max"); // a MAX pooling over all spatial dimensions. +} + TEST_P(Test_TensorFlow_layers, pooling_reduce_sum) { runTensorFlowNet("reduce_sum"); // a SUM pooling over all spatial dimensions. From 21d0f4075141568f7bf6616c5f8e79c06987283e Mon Sep 17 00:00:00 2001 From: Saikat Nanda Date: Mon, 9 Aug 2021 21:41:00 -0400 Subject: [PATCH 107/262] Fix YUV indexes + YUV Planner detection condition --- modules/videoio/src/cap_android_camera.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/modules/videoio/src/cap_android_camera.cpp b/modules/videoio/src/cap_android_camera.cpp index b369a12a68..5952b6f08c 100644 --- a/modules/videoio/src/cap_android_camera.cpp +++ b/modules/videoio/src/cap_android_camera.cpp @@ -304,8 +304,8 @@ public: AImage_getPlaneRowStride(image.get(), 0, &yStride); AImage_getPlaneRowStride(image.get(), 1, &uvStride); AImage_getPlaneData(image.get(), 0, &yPixel, &yLen); - AImage_getPlaneData(image.get(), 1, &vPixel, &vLen); - AImage_getPlaneData(image.get(), 2, &uPixel, &uLen); + AImage_getPlaneData(image.get(), 1, &uPixel, &uLen); + AImage_getPlaneData(image.get(), 2, &vPixel, &vLen); AImage_getPlanePixelStride(image.get(), 1, &uvPixelStride); if ( (uvPixelStride == 2) && (vPixel == uPixel + 1) && (yLen == frameWidth * frameHeight) && (uLen == ((yLen / 2) - 1)) && (vLen == uLen) ) { @@ -313,7 +313,7 @@ public: if (fourCC == FOURCC_UNKNOWN) { fourCC = FOURCC_NV21; } - } else if ( (uvPixelStride == 1) && (vPixel = uPixel + uLen) && (yLen == frameWidth * frameHeight) && (uLen == yLen / 4) && (vLen == uLen) ) { + } else if ( (uvPixelStride == 1) && (vPixel == uPixel + uLen) && (yLen == frameWidth * frameHeight) && (uLen == yLen / 4) && (vLen == uLen) ) { colorFormat = COLOR_FormatYUV420Planar; if (fourCC == FOURCC_UNKNOWN) { fourCC = FOURCC_YV12; From 992b47b9916f9dbdfee16ed1a59ba64cda0779bb Mon Sep 17 00:00:00 2001 From: AleksandrPanov Date: Tue, 10 Aug 2021 18:53:28 +0300 Subject: [PATCH 108/262] add 19769 and 19769_lightweight tests --- modules/imgproc/test/test_convhull.cpp | 73 ++++++++++++++++++++++++++ 1 file changed, 73 insertions(+) diff --git a/modules/imgproc/test/test_convhull.cpp b/modules/imgproc/test/test_convhull.cpp index f1d739b0e0..dee3769762 100644 --- a/modules/imgproc/test/test_convhull.cpp +++ b/modules/imgproc/test/test_convhull.cpp @@ -2384,5 +2384,78 @@ TEST(Imgproc_minAreaRect, reproducer_18157) EXPECT_TRUE(checkMinAreaRect(rr, contour)) << rr.center << " " << rr.size << " " << rr.angle; } +TEST(Imgproc_minAreaRect, reproducer_19769_lightweight) +{ + const int N = 23; + float pts_[N][2] = { + {1325, 732}, {1248, 808}, {582, 1510}, {586, 1524}, + {595, 1541}, {599, 1547}, {789, 1745}, {829, 1786}, + {997, 1958}, {1116, 2074}, {1207, 2066}, {1216, 2058}, + {1231, 2044}, {1265, 2011}, {2036, 1254}, {2100, 1191}, + {2169, 1123}, {2315, 979}, {2395, 900}, {2438, 787}, + {2434, 782}, {2416, 762}, {2266, 610} + }; + Mat contour(N, 1, CV_32FC2, (void*)pts_); + + RotatedRect rr = cv::minAreaRect(contour); + + EXPECT_TRUE(checkMinAreaRect(rr, contour)) << rr.center << " " << rr.size << " " << rr.angle; +} + +TEST(Imgproc_minAreaRect, reproducer_19769) +{ + const int N = 169; + float pts_[N][2] = { + {1854, 227}, {1850, 228}, {1847, 229}, {1835, 235}, + {1832, 237}, {1829, 239}, {1825, 242}, {1818, 248}, + {1807, 258}, {1759, 306}, {1712, 351}, {1708, 356}, + {1658, 404}, {1655, 408}, {1602, 459}, {1599, 463}, + {1542, 518}, {1477, 582}, {1402, 656}, {1325, 732}, + {1248, 808}, {1161, 894}, {1157, 898}, {1155, 900}, + {1068, 986}, {1060, 995}, {1058, 997}, {957, 1097}, + {956, 1097}, {814, 1238}, {810, 1242}, {805, 1248}, + {610, 1442}, {603, 1450}, {599, 1455}, {596, 1459}, + {594, 1462}, {592, 1465}, {590, 1470}, {588, 1472}, + {586, 1476}, {586, 1478}, {584, 1481}, {583, 1485}, + {582, 1490}, {582, 1510}, {583, 1515}, {584, 1518}, + {585, 1521}, {586, 1524}, {593, 1538}, {595, 1541}, + {597, 1544}, {599, 1547}, {603, 1552}, {609, 1559}, + {623, 1574}, {645, 1597}, {677, 1630}, {713, 1667}, + {753, 1707}, {789, 1744}, {789, 1745}, {829, 1786}, + {871, 1828}, {909, 1867}, {909, 1868}, {950, 1910}, + {953, 1912}, {997, 1958}, {1047, 2009}, {1094, 2056}, + {1105, 2066}, {1110, 2070}, {1113, 2072}, {1116, 2074}, + {1119, 2076}, {1122, 2077}, {1124, 2079}, {1130, 2082}, + {1133, 2083}, {1136, 2084}, {1139, 2085}, {1142, 2086}, + {1148, 2087}, {1166, 2087}, {1170, 2086}, {1174, 2085}, + {1177, 2084}, {1180, 2083}, {1188, 2079}, {1190, 2077}, + {1193, 2076}, {1196, 2074}, {1199, 2072}, {1202, 2070}, + {1207, 2066}, {1216, 2058}, {1231, 2044}, {1265, 2011}, + {1314, 1962}, {1360, 1917}, {1361, 1917}, {1408, 1871}, + {1457, 1822}, {1508, 1773}, {1512, 1768}, {1560, 1722}, + {1617, 1665}, {1671, 1613}, {1730, 1554}, {1784, 1502}, + {1786, 1500}, {1787, 1498}, {1846, 1440}, {1850, 1437}, + {1908, 1380}, {1974, 1314}, {2034, 1256}, {2036, 1254}, + {2100, 1191}, {2169, 1123}, {2242, 1051}, {2315, 979}, + {2395, 900}, {2426, 869}, {2435, 859}, {2438, 855}, + {2440, 852}, {2442, 849}, {2443, 846}, {2445, 844}, + {2446, 842}, {2446, 840}, {2448, 837}, {2449, 834}, + {2450, 829}, {2450, 814}, {2449, 809}, {2448, 806}, + {2447, 803}, {2442, 793}, {2440, 790}, {2438, 787}, + {2434, 782}, {2428, 775}, {2416, 762}, {2411, 758}, + {2342, 688}, {2340, 686}, {2338, 684}, {2266, 610}, + {2260, 605}, {2170, 513}, {2075, 417}, {2073, 415}, + {2069, 412}, {1955, 297}, {1955, 296}, {1913, 254}, + {1904, 246}, {1897, 240}, {1894, 238}, {1891, 236}, + {1888, 234}, {1880, 230}, {1877, 229}, {1874, 228}, + {1870, 227} + }; + Mat contour(N, 1, CV_32FC2, (void*)pts_); + + RotatedRect rr = cv::minAreaRect(contour); + + EXPECT_TRUE(checkMinAreaRect(rr, contour)) << rr.center << " " << rr.size << " " << rr.angle; +} + }} // namespace /* End of file. */ From aaca4987c9ffe12e9e486d40ae264859caed89df Mon Sep 17 00:00:00 2001 From: HAN Liutong Date: Wed, 11 Aug 2021 06:16:03 +0800 Subject: [PATCH 109/262] Merge pull request #20287 from hanliutong:dev-rvv-0.10 Optimization of DNN using native RISC-V vector intrinsics. * Use RVV to optimize fastGEMM (FP32) in DNN. * Use RVV to optimize fastGEMM1T in DNN. * Use RVV to optimize fastConv in DNN. * Use RVV to optimize fastDepthwiseConv in DNN. * Vectorize tails using vl. * Use "vl" instead of scalar to handle small block in fastConv. * Fix memory access out of bound in "fastGEMM1T". * Remove setvl. * Remove useless initialization. * Use loop unrolling to handle tail part instead of switch. --- modules/dnn/CMakeLists.txt | 2 +- modules/dnn/src/layers/convolution_layer.cpp | 25 +- .../dnn/src/layers/fully_connected_layer.cpp | 9 +- modules/dnn/src/layers/layers_common.simd.hpp | 549 ++++++++++++++++++ 4 files changed, 582 insertions(+), 3 deletions(-) diff --git a/modules/dnn/CMakeLists.txt b/modules/dnn/CMakeLists.txt index 4c8129cbda..70f9a5a73e 100644 --- a/modules/dnn/CMakeLists.txt +++ b/modules/dnn/CMakeLists.txt @@ -8,7 +8,7 @@ endif() set(the_description "Deep neural network module. It allows to load models from different frameworks and to make forward pass") -ocv_add_dispatched_file_force_all("layers/layers_common" AVX AVX2 AVX512_SKX) +ocv_add_dispatched_file_force_all("layers/layers_common" AVX AVX2 AVX512_SKX RVV) ocv_add_module(dnn opencv_core opencv_imgproc WRAP python java objc js) diff --git a/modules/dnn/src/layers/convolution_layer.cpp b/modules/dnn/src/layers/convolution_layer.cpp index fb57f26511..68c543be24 100644 --- a/modules/dnn/src/layers/convolution_layer.cpp +++ b/modules/dnn/src/layers/convolution_layer.cpp @@ -914,11 +914,12 @@ public: bool useAVX; bool useAVX2; bool useAVX512; + bool useRVV; int blk_size_cn; ParallelConv() : input_(0), weights_(0), output_(0), ngroups_(0), nstripes_(0), - biasvec_(0), reluslope_(0), activ_(0), is1x1_(false), useAVX(false), useAVX2(false), useAVX512(false) + biasvec_(0), reluslope_(0), activ_(0), is1x1_(false), useAVX(false), useAVX2(false), useAVX512(false), useRVV(false) , blk_size_cn(0) {} @@ -976,6 +977,7 @@ public: p.useAVX = checkHardwareSupport(CPU_AVX) && isConv2D; p.useAVX2 = checkHardwareSupport(CPU_AVX2) && isConv2D; p.useAVX512 = CV_CPU_HAS_SUPPORT_AVX512_SKX && isConv2D; + p.useRVV = checkHardwareSupport(CPU_RVV) && isConv2D; int kernel_d = isConv3D? kernel_size[0] : 1; int kernel_h = isConv1D? 1 : kernel_size[kernel_size.size() - 2]; @@ -1176,6 +1178,13 @@ public: stride_h, stride_w, dilation_h, dilation_w, pad_t, pad_l, biasptr, relu, inptr_, height, width, outptr_, out_d, outH, outW); else + #endif + #if CV_TRY_RVV + if(useRVV) + opt_RVV::fastDepthwiseConv(wptr, kernel_h, kernel_w, + stride_h, stride_w, dilation_h, dilation_w, pad_t, pad_l, + biasptr, relu, inptr_, height, width, outptr_, out_d, outH, outW); + else #endif { const float w00_ = wptr[0], w01_ = wptr[1], w02_ = wptr[2], @@ -1546,6 +1555,12 @@ public: opt_AVX::fastConv(wptr, wstep, biasptr, rowbuf0, data_out0 + ofs0, outShape, bsz, vsz, vsz_a, relu, cn0 == 0); else + #endif + #if CV_TRY_RVV + if(useRVV) + opt_RVV::fastConv(wptr, wstep, biasptr, rowbuf0, data_out0 + ofs0, + outShape, bsz, vsz, vsz_a, relu, cn0 == 0); + else #endif for( int i = 0; i < outCn; i += 2 ) { @@ -2297,6 +2312,7 @@ public: useAVX = checkHardwareSupport(CPU_AVX); useAVX2 = checkHardwareSupport(CPU_AVX2); useAVX512 = CV_CPU_HAS_SUPPORT_AVX512_SKX; + useRVV = checkHardwareSupport(CPU_RVV); } void operator()(const Range& range_) const CV_OVERRIDE @@ -2328,6 +2344,12 @@ public: if( useAVX ) opt_AVX::fastGEMM( aptr, astep, bptr, bstep, cptr, cstep, mmax, kmax, nmax ); else + #endif + #if CV_TRY_RVV + if( useRVV ) { + opt_RVV::fastGEMM( aptr, astep, bptr, bstep, cptr, cstep, mmax, kmax, nmax ); + } + else #endif for( m = 0; m < mmax; m += 2 ) { @@ -2427,6 +2449,7 @@ public: bool useAVX; bool useAVX2; bool useAVX512; + bool useRVV; }; class Col2ImInvoker : public cv::ParallelLoopBody diff --git a/modules/dnn/src/layers/fully_connected_layer.cpp b/modules/dnn/src/layers/fully_connected_layer.cpp index d9c1fa65c1..529f3c04fd 100644 --- a/modules/dnn/src/layers/fully_connected_layer.cpp +++ b/modules/dnn/src/layers/fully_connected_layer.cpp @@ -168,7 +168,7 @@ public: class FullyConnected : public ParallelLoopBody { public: - FullyConnected() : srcMat(0), weights(0), biasMat(0), activ(0), dstMat(0), nstripes(0), useAVX(false), useAVX2(false), useAVX512(false) {} + FullyConnected() : srcMat(0), weights(0), biasMat(0), activ(0), dstMat(0), nstripes(0), useAVX(false), useAVX2(false), useAVX512(false), useRVV(false) {} static void run(const Mat& srcMat, const Mat& weights, const Mat& biasMat, Mat& dstMat, const ActivationLayer* activ, int nstripes) @@ -191,6 +191,7 @@ public: p.useAVX = checkHardwareSupport(CPU_AVX); p.useAVX2 = checkHardwareSupport(CPU_AVX2); p.useAVX512 = CV_CPU_HAS_SUPPORT_AVX512_SKX; + p.useRVV = checkHardwareSupport(CPU_RVV); parallel_for_(Range(0, nstripes), p, nstripes); } @@ -239,6 +240,11 @@ public: if( useAVX ) opt_AVX::fastGEMM1T( sptr, wptr, wstep, biasptr, dptr, nw, vecsize); else + #endif + #if CV_TRY_RVV + if( useRVV ) + opt_RVV::fastGEMM1T( sptr, wptr, wstep, biasptr, dptr, nw, vecsize); + else #endif { int i = 0; @@ -293,6 +299,7 @@ public: bool useAVX; bool useAVX2; bool useAVX512; + bool useRVV; }; #ifdef HAVE_OPENCL diff --git a/modules/dnn/src/layers/layers_common.simd.hpp b/modules/dnn/src/layers/layers_common.simd.hpp index 706695a7b2..762e22e54d 100644 --- a/modules/dnn/src/layers/layers_common.simd.hpp +++ b/modules/dnn/src/layers/layers_common.simd.hpp @@ -737,5 +737,554 @@ void fastGEMM( const float* aptr, size_t astep, const float* bptr, #endif // CV_CPU_OPTIMIZATION_DECLARATIONS_ONLY +#if !defined(CV_CPU_OPTIMIZATION_DECLARATIONS_ONLY) && CV_RVV + +void fastGEMM( const float* aptr, size_t astep, const float* bptr, + size_t bstep, float* cptr, size_t cstep, + int ma, int na, int nb ) +{ + int n = 0; + size_t vl = 8; + size_t mvl0 = 8; + size_t mvl1 = 8; + for( ; n < nb; n += 16 ) + { + if ( n + 16 > nb) { + mvl0 = nb - n; + mvl1 = (nb - n -8) > 0 ? (nb - n -8) : 0; + } + + for( int m = 0; m < ma; m += 4 ) + { + const float* aptr0 = aptr + astep*m; + const float* aptr1 = aptr + astep*std::min(m+1, ma-1); + const float* aptr2 = aptr + astep*std::min(m+2, ma-1); + const float* aptr3 = aptr + astep*std::min(m+3, ma-1); + + float* cptr0 = cptr + cstep*m; + float* cptr1 = cptr + cstep*std::min(m+1, ma-1); + float* cptr2 = cptr + cstep*std::min(m+2, ma-1); + float* cptr3 = cptr + cstep*std::min(m+3, ma-1); + + vfloat32m2_t d00 = vfmv_v_f_f32m2(0, vl), d01 = vfmv_v_f_f32m2(0, vl); + vfloat32m2_t d10 = vfmv_v_f_f32m2(0, vl), d11 = vfmv_v_f_f32m2(0, vl); + vfloat32m2_t d20 = vfmv_v_f_f32m2(0, vl), d21 = vfmv_v_f_f32m2(0, vl); + vfloat32m2_t d30 = vfmv_v_f_f32m2(0, vl), d31 = vfmv_v_f_f32m2(0, vl); + + for( int k = 0; k < na; k++ ) + { + vfloat32m2_t a0 = vfmv_v_f_f32m2(aptr0[k], vl); + vfloat32m2_t a1 = vfmv_v_f_f32m2(aptr1[k], vl); + vfloat32m2_t a2 = vfmv_v_f_f32m2(aptr2[k], vl); + vfloat32m2_t a3 = vfmv_v_f_f32m2(aptr3[k], vl); + vfloat32m2_t b0 = vle32_v_f32m2(bptr + k*bstep + n, mvl0); + vfloat32m2_t b1 = vle32_v_f32m2(bptr + k*bstep + n + 8, mvl1); + d00 = vfmacc_vv_f32m2(d00, a0, b0, mvl0); + d01 = vfmacc_vv_f32m2(d01, a0, b1, mvl1); + d10 = vfmacc_vv_f32m2(d10, a1, b0, mvl0); + d11 = vfmacc_vv_f32m2(d11, a1, b1, mvl1); + d20 = vfmacc_vv_f32m2(d20, a2, b0, mvl0); + d21 = vfmacc_vv_f32m2(d21, a2, b1, mvl1); + d30 = vfmacc_vv_f32m2(d30, a3, b0, mvl0); + d31 = vfmacc_vv_f32m2(d31, a3, b1, mvl1); + } + vse32_v_f32m2(cptr0 + n, d00, mvl0); + vse32_v_f32m2(cptr1 + n, d10, mvl0); + vse32_v_f32m2(cptr2 + n, d20, mvl0); + vse32_v_f32m2(cptr3 + n, d30, mvl0); + vse32_v_f32m2(cptr0 + n + 8, d01, mvl1); + vse32_v_f32m2(cptr1 + n + 8, d11, mvl1); + vse32_v_f32m2(cptr2 + n + 8, d21, mvl1); + vse32_v_f32m2(cptr3 + n + 8, d31, mvl1); + } + } +} + +void fastGEMM1T( const float* vec, const float* weights, + size_t wstep, const float* bias, + float* dst, int nvecs, int vecsize ) +{ + int i = 0; + size_t vl = 8; + for( ; i <= nvecs - 8; i += 8 ) + { + const float* wptr = weights + i*wstep; + vfloat32m2_t vs0 = vfmv_v_f_f32m2(0, vl), vs1 = vfmv_v_f_f32m2(0, vl), + vs2 = vfmv_v_f_f32m2(0, vl), vs3 = vfmv_v_f_f32m2(0, vl), + vs4 = vfmv_v_f_f32m2(0, vl), vs5 = vfmv_v_f_f32m2(0, vl), + vs6 = vfmv_v_f_f32m2(0, vl), vs7 = vfmv_v_f_f32m2(0, vl); + + for( int k = 0; k < vecsize; k += 8, wptr += 8 ) + { + vfloat32m2_t v = vle32_v_f32m2(vec + k, vl); + + vs0 = vfmacc_vv_f32m2(vs0, vle32_v_f32m2(wptr, vl), v, vl); + vs1 = vfmacc_vv_f32m2(vs1, vle32_v_f32m2(wptr + wstep, vl), v, vl); + vs2 = vfmacc_vv_f32m2(vs2, vle32_v_f32m2(wptr + wstep*2, vl), v, vl); + vs3 = vfmacc_vv_f32m2(vs3, vle32_v_f32m2(wptr + wstep*3, vl), v, vl); + vs4 = vfmacc_vv_f32m2(vs4, vle32_v_f32m2(wptr + wstep*4, vl), v, vl); + vs5 = vfmacc_vv_f32m2(vs5, vle32_v_f32m2(wptr + wstep*5, vl), v, vl); + vs6 = vfmacc_vv_f32m2(vs6, vle32_v_f32m2(wptr + wstep*6, vl), v, vl); + vs7 = vfmacc_vv_f32m2(vs7, vle32_v_f32m2(wptr + wstep*7, vl), v, vl); + } + + // Calculate the sum of each vector + vfloat32m1_t zero = vfmv_v_f_f32m1(0, vl); + vfloat32m1_t temp0 = vfredsum_vs_f32m2_f32m1(temp0, vs0, zero, vl); + vfloat32m1_t temp1 = vfredsum_vs_f32m2_f32m1(temp1, vs1, zero, vl); + vfloat32m1_t temp2 = vfredsum_vs_f32m2_f32m1(temp2, vs2, zero, vl); + vfloat32m1_t temp3 = vfredsum_vs_f32m2_f32m1(temp3, vs3, zero, vl); + vfloat32m1_t temp4 = vfredsum_vs_f32m2_f32m1(temp4, vs4, zero, vl); + vfloat32m1_t temp5 = vfredsum_vs_f32m2_f32m1(temp5, vs5, zero, vl); + vfloat32m1_t temp6 = vfredsum_vs_f32m2_f32m1(temp6, vs6, zero, vl); + vfloat32m1_t temp7 = vfredsum_vs_f32m2_f32m1(temp7, vs7, zero, vl); + float32_t sum[8]; + sum[0] = vfmv_f_s_f32m1_f32(temp0); + sum[1] = vfmv_f_s_f32m1_f32(temp1); + sum[2] = vfmv_f_s_f32m1_f32(temp2); + sum[3] = vfmv_f_s_f32m1_f32(temp3); + sum[4] = vfmv_f_s_f32m1_f32(temp4); + sum[5] = vfmv_f_s_f32m1_f32(temp5); + sum[6] = vfmv_f_s_f32m1_f32(temp6); + sum[7] = vfmv_f_s_f32m1_f32(temp7); + vfloat32m2_t s0 = vfadd_vv_f32m2(vle32_v_f32m2(sum, vl), vle32_v_f32m2(bias + i, vl), vl); + vse32_v_f32m2(dst + i, s0, vl); + } + int mvl = nvecs - i; + if (mvl > 0) + { + const float* wptr = weights + i*wstep; + vfloat32m2_t vs0 = vfmv_v_f_f32m2(0, vl), vs1 = vfmv_v_f_f32m2(0, vl), + vs2 = vfmv_v_f_f32m2(0, vl), vs3 = vfmv_v_f_f32m2(0, vl), + vs4 = vfmv_v_f_f32m2(0, vl), vs5 = vfmv_v_f_f32m2(0, vl), + vs6 = vfmv_v_f_f32m2(0, vl), vs7 = vfmv_v_f_f32m2(0, vl); + int k = 0; + for( ; k <= vecsize - 8; k += 8, wptr += 8 ) + { + vfloat32m2_t v = vle32_v_f32m2(vec + k, vl); + vs0 = vfmacc_vv_f32m2(vs0, vle32_v_f32m2(wptr, vl), v, vl); + vs1 = vfmacc_vv_f32m2(vs1, vle32_v_f32m2(wptr + wstep*std::min(1, mvl-1), vl), v, vl); + vs2 = vfmacc_vv_f32m2(vs2, vle32_v_f32m2(wptr + wstep*std::min(2, mvl-1), vl), v, vl); + vs3 = vfmacc_vv_f32m2(vs3, vle32_v_f32m2(wptr + wstep*std::min(3, mvl-1), vl), v, vl); + vs4 = vfmacc_vv_f32m2(vs4, vle32_v_f32m2(wptr + wstep*std::min(4, mvl-1), vl), v, vl); + vs5 = vfmacc_vv_f32m2(vs5, vle32_v_f32m2(wptr + wstep*std::min(5, mvl-1), vl), v, vl); + vs6 = vfmacc_vv_f32m2(vs6, vle32_v_f32m2(wptr + wstep*std::min(6, mvl-1), vl), v, vl); + } + int kvl = vecsize - k; + if (kvl > 0) { + vfloat32m2_t v = vle32_v_f32m2(vec + k, kvl); + vs0 = vfmacc_vv_f32m2(vs0, vle32_v_f32m2(wptr, kvl), v, kvl); + vs1 = vfmacc_vv_f32m2(vs1, vle32_v_f32m2(wptr + wstep*std::min(1, mvl-1), kvl), v, kvl); + vs2 = vfmacc_vv_f32m2(vs2, vle32_v_f32m2(wptr + wstep*std::min(2, mvl-1), kvl), v, kvl); + vs3 = vfmacc_vv_f32m2(vs3, vle32_v_f32m2(wptr + wstep*std::min(3, mvl-1), kvl), v, kvl); + vs4 = vfmacc_vv_f32m2(vs4, vle32_v_f32m2(wptr + wstep*std::min(4, mvl-1), kvl), v, kvl); + vs5 = vfmacc_vv_f32m2(vs5, vle32_v_f32m2(wptr + wstep*std::min(5, mvl-1), kvl), v, kvl); + vs6 = vfmacc_vv_f32m2(vs6, vle32_v_f32m2(wptr + wstep*std::min(6, mvl-1), kvl), v, kvl); + } + // Calculate the sum of each vector + vfloat32m1_t zero = vfmv_v_f_f32m1(0, vl); + vfloat32m1_t temp0 = vfmv_v_f_f32m1(0, 4), temp1 = vfmv_v_f_f32m1(0, 4), + temp2 = vfmv_v_f_f32m1(0, 4), temp3 = vfmv_v_f_f32m1(0, 4), + temp4 = vfmv_v_f_f32m1(0, 4), temp5 = vfmv_v_f_f32m1(0, 4), + temp6 = vfmv_v_f_f32m1(0, 4), temp7 = vfmv_v_f_f32m1(0, 4); + temp0 = vfredsum_vs_f32m2_f32m1(temp0, vs0, zero, vl); + temp1 = vfredsum_vs_f32m2_f32m1(temp1, vs1, zero, vl); + temp2 = vfredsum_vs_f32m2_f32m1(temp2, vs2, zero, vl); + temp3 = vfredsum_vs_f32m2_f32m1(temp3, vs3, zero, vl); + temp4 = vfredsum_vs_f32m2_f32m1(temp4, vs4, zero, vl); + temp5 = vfredsum_vs_f32m2_f32m1(temp5, vs5, zero, vl); + temp6 = vfredsum_vs_f32m2_f32m1(temp6, vs6, zero, vl); + temp7 = vfredsum_vs_f32m2_f32m1(temp7, vs7, zero, vl); + + float32_t sum[8]; + sum[0] = vfmv_f_s_f32m1_f32(temp0); + sum[1] = vfmv_f_s_f32m1_f32(temp1); + sum[2] = vfmv_f_s_f32m1_f32(temp2); + sum[3] = vfmv_f_s_f32m1_f32(temp3); + sum[4] = vfmv_f_s_f32m1_f32(temp4); + sum[5] = vfmv_f_s_f32m1_f32(temp5); + sum[6] = vfmv_f_s_f32m1_f32(temp6); + sum[7] = vfmv_f_s_f32m1_f32(temp7); + + vfloat32m2_t s0 = vfadd_vv_f32m2(vle32_v_f32m2(sum, mvl), vle32_v_f32m2(bias + i, mvl), mvl); + vse32_v_f32m2(dst + i, s0, mvl); + } +} + +enum { FASCONV_BASE_VECSZ = 4 }; // TODO: Large base size. +void fastConv( const float* weights, size_t wstep, const float* bias, + const float* rowbuf, float* output, const int* outShape, + int blockSize, int vecsize, int vecsize_aligned, + const float* relu, bool initOutput ) +{ + int vl = 4; + int outCn = outShape[1]; + size_t outPlaneSize = outShape[2]*outShape[3]; + float r0 = 1.f, r1 = 1.f, r2 = 1.f; + vfloat32m1_t vr0 = vfmv_v_f_f32m1(1, vl), vr1 = vfmv_v_f_f32m1(1, vl), vr2 = vfmv_v_f_f32m1(1, vl); + int maskbuf[FASCONV_BASE_VECSZ] = {0}; + int rsz = blockSize % FASCONV_BASE_VECSZ; + for( int i = 0; i < rsz; i++ ) + maskbuf[FASCONV_BASE_VECSZ - i - 1] = -1; + vint32m1_t vmaskbuf = vle32_v_i32m1(maskbuf ,vl); + vbool32_t mask = vmslt_vx_i32m1_b32(vmaskbuf, 0, vl); // mask for tail + // now compute dot product of the weights + // and im2row-transformed part of the tensor + for( int i = 0; i < outCn; i += 3 ) + { + const float* wptr0 = weights + i*wstep; + const float* wptr1 = wptr0 + wstep; + const float* wptr2 = wptr1 + wstep; + float* outptr0 = output + i*outPlaneSize; + float* outptr1 = outptr0 + outPlaneSize; + float* outptr2 = outptr1 + outPlaneSize; + float bias0 = bias[i], bias1 = bias[i+1], bias2 = bias[i+2]; + + if( i+2 >= outCn ) + { + wptr2 = wptr1; + outptr2 = outptr1; + bias2 = bias1; + if( i+1 >= outCn ) + { + wptr2 = wptr1 = wptr0; + outptr2 = outptr1 = outptr0; + bias2 = bias1 = bias0; + } + } + + if( relu ) + { + r0 = relu[i]; r1 = relu[i+1]; r2 = relu[i+2]; + if( i+2 >= outCn ) + { + r2 = r1; + if( i+1 >= outCn ) + r2 = r1 = r0; + } + vr0 = vfmv_v_f_f32m1(r0, vl); + vr1 = vfmv_v_f_f32m1(r1, vl); + vr2 = vfmv_v_f_f32m1(r2, vl); + } + + int j = 0; + for( ; j < blockSize; j += FASCONV_BASE_VECSZ ) + { + bool tail = false; + if (j + FASCONV_BASE_VECSZ > blockSize) + { + if (j == 0) { + vl = blockSize; + } + else { + j = blockSize - FASCONV_BASE_VECSZ; + tail = true; + } + } + int k = 0; + const float* rptr = rowbuf + j*vecsize_aligned; + int vlm2 = 8; + vfloat32m2_t vs00 = vfmv_v_f_f32m2(0, vlm2), vs01 = vfmv_v_f_f32m2(0, vlm2), + vs02 = vfmv_v_f_f32m2(0, vlm2), vs03 = vfmv_v_f_f32m2(0, vlm2), + vs10 = vfmv_v_f_f32m2(0, vlm2), vs11 = vfmv_v_f_f32m2(0, vlm2), + vs12 = vfmv_v_f_f32m2(0, vlm2), vs13 = vfmv_v_f_f32m2(0, vlm2), + vs20 = vfmv_v_f_f32m2(0, vlm2), vs21 = vfmv_v_f_f32m2(0, vlm2), + vs22 = vfmv_v_f_f32m2(0, vlm2), vs23 = vfmv_v_f_f32m2(0, vlm2); + + for (; k < vecsize; k += 8, rptr += 8 ) + { + if (k+8 >= vecsize) { + vlm2 = vecsize - k; + } + vfloat32m2_t w0 = vle32_v_f32m2(wptr0 + k, vlm2); + vfloat32m2_t w1 = vle32_v_f32m2(wptr1 + k, vlm2); + vfloat32m2_t w2 = vle32_v_f32m2(wptr2 + k, vlm2); + vfloat32m2_t r0 = vle32_v_f32m2(rptr, vlm2); + + vs00 = vfmacc_vv_f32m2(vs00, w0, r0, vlm2); + vs10 = vfmacc_vv_f32m2(vs10, w1, r0, vlm2); + vs20 = vfmacc_vv_f32m2(vs20, w2, r0, vlm2); + + r0 = vle32_v_f32m2(rptr + vecsize_aligned, vlm2); + vs01 = vfmacc_vv_f32m2(vs01, w0, r0, vlm2); + vs11 = vfmacc_vv_f32m2(vs11, w1, r0, vlm2); + vs21 = vfmacc_vv_f32m2(vs21, w2, r0, vlm2); + + r0 = vle32_v_f32m2(rptr + vecsize_aligned*2, vlm2); + vs02 = vfmacc_vv_f32m2(vs02, w0, r0, vlm2); + vs12 = vfmacc_vv_f32m2(vs12, w1, r0, vlm2); + vs22 = vfmacc_vv_f32m2(vs22, w2, r0, vlm2); + + r0 = vle32_v_f32m2(rptr + vecsize_aligned*3, vlm2); + vs03 = vfmacc_vv_f32m2(vs03, w0, r0, vlm2); + vs13 = vfmacc_vv_f32m2(vs13, w1, r0, vlm2); + vs23 = vfmacc_vv_f32m2(vs23, w2, r0, vlm2); + } + vfloat32m1_t s0, s1, s2; + + if( initOutput ) + { + s0 = vfmv_v_f_f32m1(bias0, vl); + s1 = vfmv_v_f_f32m1(bias1, vl); + s2 = vfmv_v_f_f32m1(bias2, vl); + } + else + { + s0 = vle32_v_f32m1(outptr0 + j, vl); + s1 = vle32_v_f32m1(outptr1 + j, vl); + s2 = vle32_v_f32m1(outptr2 + j, vl); + } + // compute sum of each vs + vfloat32m1_t zero = vfmv_v_f_f32m1(0, vl); + vfloat32m1_t temp00 = vfredsum_vs_f32m2_f32m1(temp00, vs00, zero, 8); + vfloat32m1_t temp01 = vfredsum_vs_f32m2_f32m1(temp01, vs01, zero, 8); + vfloat32m1_t temp02 = vfredsum_vs_f32m2_f32m1(temp02, vs02, zero, 8); + vfloat32m1_t temp03 = vfredsum_vs_f32m2_f32m1(temp03, vs03, zero, 8); + vfloat32m1_t temp10 = vfredsum_vs_f32m2_f32m1(temp10, vs10, zero, 8); + vfloat32m1_t temp11 = vfredsum_vs_f32m2_f32m1(temp11, vs11, zero, 8); + vfloat32m1_t temp12 = vfredsum_vs_f32m2_f32m1(temp12, vs12, zero, 8); + vfloat32m1_t temp13 = vfredsum_vs_f32m2_f32m1(temp13, vs13, zero, 8); + vfloat32m1_t temp20 = vfredsum_vs_f32m2_f32m1(temp20, vs20, zero, 8); + vfloat32m1_t temp21 = vfredsum_vs_f32m2_f32m1(temp21, vs21, zero, 8); + vfloat32m1_t temp22 = vfredsum_vs_f32m2_f32m1(temp22, vs22, zero, 8); + vfloat32m1_t temp23 = vfredsum_vs_f32m2_f32m1(temp23, vs23, zero, 8); + float32_t sum0[4], sum1[4], sum2[4]; + sum0[0] = vfmv_f_s_f32m1_f32(temp00); + sum0[1] = vfmv_f_s_f32m1_f32(temp01); + sum0[2] = vfmv_f_s_f32m1_f32(temp02); + sum0[3] = vfmv_f_s_f32m1_f32(temp03); + sum1[0] = vfmv_f_s_f32m1_f32(temp10); + sum1[1] = vfmv_f_s_f32m1_f32(temp11); + sum1[2] = vfmv_f_s_f32m1_f32(temp12); + sum1[3] = vfmv_f_s_f32m1_f32(temp13); + sum2[0] = vfmv_f_s_f32m1_f32(temp20); + sum2[1] = vfmv_f_s_f32m1_f32(temp21); + sum2[2] = vfmv_f_s_f32m1_f32(temp22); + sum2[3] = vfmv_f_s_f32m1_f32(temp23); + + s0 = vfadd_vv_f32m1(vle32_v_f32m1(sum0, vl), s0, vl); + s1 = vfadd_vv_f32m1(vle32_v_f32m1(sum1, vl), s1, vl); + s2 = vfadd_vv_f32m1(vle32_v_f32m1(sum2, vl), s2, vl); + + + if( relu ) + { + vbool32_t m0 = vmfgt_vf_f32m1_b32(s0, 0, vl); + vbool32_t m1 = vmfgt_vf_f32m1_b32(s1, 0, vl); + vbool32_t m2 = vmfgt_vf_f32m1_b32(s2, 0, vl); + s0 = vmerge_vvm_f32m1(m0, vfmul_vv_f32m1(s0, vr0, vl), s0, vl); + s1 = vmerge_vvm_f32m1(m1, vfmul_vv_f32m1(s1, vr1, vl), s1, vl); + s2 = vmerge_vvm_f32m1(m2, vfmul_vv_f32m1(s2, vr2, vl), s2, vl); + } + + if( tail ) + { + s0 = vmerge_vvm_f32m1(mask, vle32_v_f32m1(outptr0 + j, vl), s0, vl); + s1 = vmerge_vvm_f32m1(mask, vle32_v_f32m1(outptr1 + j, vl), s1, vl); + s2 = vmerge_vvm_f32m1(mask, vle32_v_f32m1(outptr2 + j, vl), s2, vl); + } + + vse32_v_f32m1(outptr0 + j, s0, vl); + vse32_v_f32m1(outptr1 + j, s1, vl); + vse32_v_f32m1(outptr2 + j, s2, vl); + } + } +} + +/* +Example for load_deinterleave: + input: ptr[16] = {1,2,3, ... ,14,15,16} + output: a = {1, 3, 5, 7, 9, 11, 13, 15} + output: b = {2, 4, 6, 8,10, 12, 14, 16} +*/ +static inline void vfloat32m2_load_deinterleave(const float* ptr, vfloat32m2_t& a, vfloat32m2_t& b) +{ + int vl = 8; + uint32_t masks[] = {1,1,1,1,0,0,0,0}; + vuint32m2_t vm = vle32_v_u32m2(masks,vl); + vbool16_t mask01 = vmseq_vx_u32m2_b16 (vm, 0, vl); + vbool16_t mask10 = vmseq_vx_u32m2_b16 (vm, 1, vl); + vfloat32m2_t ta = vle32_v_f32m2(ptr, vl), tb = vle32_v_f32m2(ptr+8, vl); + uint idx[] = {0,2,4,6,1,3,5,7}; + uint idxa[] = {0,0,0,0,0,1,2,3}, idxb[] = {4,5,6,7,0,0,0,0}; + vuint32m2_t vidxa = vle32_v_u32m2(idxa, 8), vidxb = vle32_v_u32m2(idxb, 8); + vuint32m2_t vidx = vle32_v_u32m2(idx, 8); + vfloat32m2_t high = vfmv_v_f_f32m2(0, 8), low = vfmv_v_f_f32m2(0, 8); + high = vrgather_vv_f32m2(ta, vidx, 8); + low = vrgather_vv_f32m2(tb, vidx, 8); + a = vrgather_vv_f32m2_m(mask01, high, low, vidxa, 8); + b = vrgather_vv_f32m2_m(mask10, low, high, vidxb, 8); +} + +void fastDepthwiseConv( const float* wptr, + int kernel_h, int kernel_w, + int stride_h, int stride_w, + int dilation_h, int dilation_w, + int pad_t, int pad_l, + const float* biasptr, const float* relu, + const float* inptr_, + int height, int width, + float* outptr_, + int out_d, int outH, int outW ) +{ + int vl = 8; + const float w00_ = wptr[0], w01_ = wptr[1], w02_ = wptr[2], + w10 = wptr[3], w11 = wptr[4], w12 = wptr[5], + w20_ = wptr[6], w21_ = wptr[7], w22_ = wptr[8]; + int outW1 = std::min(outW, (width - dilation_w*(kernel_w - 1) + pad_l)/stride_w); + float relu_coeff = relu ? relu[out_d] : 1.f, bias = biasptr[out_d]; + + for (int out_i = 0; out_i < outH; out_i++) + { + int in_i = out_i * stride_h - pad_t, out_j = 0; + const float* imgptr0 = inptr_ + in_i*width; + const float* imgptr1 = imgptr0 + dilation_h*width; + const float* imgptr2 = imgptr0 + (dilation_h*2)*width; + float out, w00 = w00_, w01 = w01_, w02 = w02_; + float w20 = w20_, w21 = w21_, w22 = w22_; + if (in_i < 0) + { + w00 = w01 = w02 = 0.f; + imgptr0 = imgptr1; + } + else if (in_i + dilation_h*(kernel_h-1) >= height) + { + w20 = w21 = w22 = 0.f; + imgptr2 = imgptr1; + } + float* outptr = outptr_ + out_i*outW; + if (pad_l > 0) + { + out = imgptr0[0]*w01 + imgptr0[dilation_w]*w02 + + imgptr1[0]*w11 + imgptr1[dilation_w]*w12 + + imgptr2[0]*w21 + imgptr2[dilation_w]*w22 + bias; + if (relu) + out = out > 0.f ? out : out*relu_coeff; + outptr[0] = out; + out_j = 1; + } + + if (stride_w == 1 || (stride_w == 2 && dilation_w == 1)) + { + const int VECSZ = 8; + vfloat32m2_t vw00 = vfmv_v_f_f32m2(w00, vl), vw01 = vfmv_v_f_f32m2(w01, vl), vw02 = vfmv_v_f_f32m2(w02, vl), + vw10 = vfmv_v_f_f32m2(w10, vl), vw11 = vfmv_v_f_f32m2(w11, vl), vw12 = vfmv_v_f_f32m2(w12, vl), + vw20 = vfmv_v_f_f32m2(w20, vl), vw21 = vfmv_v_f_f32m2(w21, vl), vw22 = vfmv_v_f_f32m2(w22, vl); + vfloat32m2_t vbias = vfmv_v_f_f32m2(bias, vl), vrc = vfmv_v_f_f32m2(relu_coeff, vl); + + if( stride_w == 1 ) + for( ; out_j < outW1; out_j += VECSZ ) + { + if (out_j + VECSZ > outW1 && out_j > pad_l) + out_j = outW1 - VECSZ; + int in_j = out_j * stride_w - pad_l; + vfloat32m2_t v00 = vle32_v_f32m2(imgptr0 + in_j, vl), + v01 = vle32_v_f32m2(imgptr0 + in_j + dilation_w, vl), + v02 = vle32_v_f32m2(imgptr0 + in_j + dilation_w*2, vl), + v10 = vle32_v_f32m2(imgptr1 + in_j, vl), + v11 = vle32_v_f32m2(imgptr1 + in_j + dilation_w, vl), + v12 = vle32_v_f32m2(imgptr1 + in_j + dilation_w*2, vl), + v20 = vle32_v_f32m2(imgptr2 + in_j, vl), + v21 = vle32_v_f32m2(imgptr2 + in_j + dilation_w, vl), + v22 = vle32_v_f32m2(imgptr2 + in_j + dilation_w*2, vl); + + vfloat32m2_t vout0 = vfmacc_vv_f32m2(vbias, v00, vw00, vl); + vfloat32m2_t vout1 = vfmul_vv_f32m2(v01, vw01, vl); + vfloat32m2_t vout2 = vfmul_vv_f32m2(v02, vw02, vl); + + vout0 = vfmacc_vv_f32m2(vout0, v10, vw10, vl); + vout1 = vfmacc_vv_f32m2(vout1, v11, vw11, vl); + vout2 = vfmacc_vv_f32m2(vout2, v12, vw12, vl); + + vout0 = vfmacc_vv_f32m2(vout0, v20, vw20, vl); + vout1 = vfmacc_vv_f32m2(vout1, v21, vw21, vl); + vout2 = vfmacc_vv_f32m2(vout2, v22, vw22, vl); + + vout0 = vfadd_vv_f32m2(vfadd_vv_f32m2(vout0, vout1, vl), vout2, vl); + if (relu) + { + vbool16_t m = vmfgt_vf_f32m2_b16(vout0, 0, vl); + vout0 = vmerge_vvm_f32m2(m, vfmul_vv_f32m2(vout0, vrc, vl), vout0, vl); + } + vse32_v_f32m2(outptr + out_j, vout0, vl); + } + else + for( ; out_j < outW1; out_j += VECSZ ) + { + if (out_j + VECSZ > outW1 && out_j > pad_l) + out_j = outW1 - VECSZ; + int in_j = out_j * stride_w - pad_l; + vfloat32m2_t v00, v01, v02, v10, v11, v12, v20, v21, v22, unused; + vfloat32m2_load_deinterleave(imgptr0 + in_j, v00, v01); + vfloat32m2_load_deinterleave(imgptr0 + in_j + 2, v02, unused); + vfloat32m2_load_deinterleave(imgptr1 + in_j, v10, v11); + vfloat32m2_load_deinterleave(imgptr1 + in_j + 2, v12, unused); + vfloat32m2_load_deinterleave(imgptr2 + in_j, v20, v21); + vfloat32m2_load_deinterleave(imgptr2 + in_j + 2, v22, unused); + + vfloat32m2_t vout0 = vfmacc_vv_f32m2(vbias, v00, vw00, vl); + vfloat32m2_t vout1 = vfmul_vv_f32m2(v01, vw01, vl); + vfloat32m2_t vout2 = vfmul_vv_f32m2(v02, vw02, vl); + + vout0 = vfmacc_vv_f32m2(vout0, v10, vw10, vl); + vout1 = vfmacc_vv_f32m2(vout1, v11, vw11, vl); + vout2 = vfmacc_vv_f32m2(vout2, v12, vw12, vl); + + vout0 = vfmacc_vv_f32m2(vout0, v20, vw20, vl); + vout1 = vfmacc_vv_f32m2(vout1, v21, vw21, vl); + vout2 = vfmacc_vv_f32m2(vout2, v22, vw22, vl); + + vout0 = vfadd_vv_f32m2(vfadd_vv_f32m2(vout0, vout1, vl), vout2, vl); + if (relu) + { + vbool16_t m = vmfgt_vf_f32m2_b16(vout0, 0, vl); + vout0 = vmerge_vvm_f32m2(m, vfmul_vv_f32m2(vout0, vrc, vl), vout0, vl); + } + vse32_v_f32m2(outptr + out_j, vout0, vl); + } + } + + for (; out_j < outW1; out_j++) + { + int in_j = out_j * stride_w - pad_l; + out = imgptr0[in_j]*w00 + imgptr0[in_j + dilation_w]*w01 + imgptr0[in_j + dilation_w*2]*w02 + + imgptr1[in_j]*w10 + imgptr1[in_j + dilation_w]*w11 + imgptr1[in_j + dilation_w*2]*w12 + + imgptr2[in_j]*w20 + imgptr2[in_j + dilation_w]*w21 + imgptr2[in_j + dilation_w*2]*w22 + bias; + if (relu) + out = out > 0.f ? out : out*relu_coeff; + outptr[out_j] = out; + } + + for (; out_j < outW; out_j++ ) + { + int in_j0 = out_j * stride_w - pad_l, in_j1 = in_j0 + dilation_w, in_j2 = in_j0 + dilation_w*2; + float s0 = 1.f, s1 = 1.f, s2 = 1.f; + if (in_j0 >= width) + { + in_j0 = 0; + s0 = 0.f; + } + if (in_j1 >= width) + { + in_j1 = 0; + s1 = 0.f; + } + if (in_j2 >= width) + { + in_j2 = 0; + s2 = 0.f; + } + out = imgptr0[in_j0]*w00*s0 + imgptr0[in_j1]*w01*s1 + imgptr0[in_j2]*w02*s2 + + imgptr1[in_j0]*w10*s0 + imgptr1[in_j1]*w11*s1 + imgptr1[in_j2]*w12*s2 + + imgptr2[in_j0]*w20*s0 + imgptr2[in_j1]*w21*s1 + imgptr2[in_j2]*w22*s2 + bias; + if (relu) + out = out > 0.f ? out : out*relu_coeff; + outptr[out_j] = out; + } + } +} + +#endif // CV_RVV + CV_CPU_OPTIMIZATION_NAMESPACE_END }} // namespace From 7c73e28a6d5ceae9ffac5a3f6c771a8efb7bb35e Mon Sep 17 00:00:00 2001 From: Hamdi Sahloul Date: Mon, 26 Jul 2021 19:03:13 +0900 Subject: [PATCH 110/262] Improves FLANN's heap allocations by a memory pool --- modules/flann/include/opencv2/flann/heap.h | 173 +++++++++++++----- .../flann/hierarchical_clustering_index.h | 6 +- .../include/opencv2/flann/kdtree_index.h | 9 +- .../include/opencv2/flann/kmeans_index.h | 8 +- 4 files changed, 133 insertions(+), 63 deletions(-) diff --git a/modules/flann/include/opencv2/flann/heap.h b/modules/flann/include/opencv2/flann/heap.h index ee1c682cfe..8cace20449 100644 --- a/modules/flann/include/opencv2/flann/heap.h +++ b/modules/flann/include/opencv2/flann/heap.h @@ -36,9 +36,21 @@ #include #include +#include + namespace cvflann { +// TODO: Define x > y operator and use std::greater instead +template +struct greater +{ + bool operator()(const T& x, const T& y) const + { + return y < x; + } +}; + /** * Priority Queue Implementation * @@ -49,117 +61,180 @@ namespace cvflann template class Heap { - /** * Storage array for the heap. * Type T must be comparable. */ std::vector heap; - int length; - - /** - * Number of element in the heap - */ - int count; - - - public: /** - * Constructor. + * \brief Constructs a heap with a pre-allocated capacity * - * Params: - * sz = heap size + * \param capacity heap maximum capacity */ - - Heap(int sz) + Heap(const int capacity) { - length = sz; - heap.reserve(length); - count = 0; + reserve(capacity); + } + + /** + * \brief Move-constructs a heap from an external vector + * + * \param vec external vector + */ + Heap(std::vector&& vec) + : heap(std::move(vec)) + { + std::make_heap(heap.begin(), heap.end(), greater()); } /** * - * Returns: heap size + * \returns heap size */ - int size() + int size() const { - return count; + return (int)heap.size(); } /** - * Tests if the heap is empty * - * Returns: true is heap empty, false otherwise + * \returns heap capacity + */ + int capacity() const + { + return (int)heap.capacity(); + } + + /** + * \brief Tests if the heap is empty + * + * \returns true is heap empty, false otherwise */ bool empty() { - return size()==0; + return heap.empty(); } /** - * Clears the heap. + * \brief Clears the heap. */ void clear() { heap.clear(); - count = 0; } - struct CompareT + /** + * \brief Sets the heap maximum capacity. + * + * \param capacity heap maximum capacity + */ + void reserve(const int capacity) { - bool operator()(const T& t_1, const T& t_2) const - { - return t_2 < t_1; - } - }; + heap.reserve(capacity); + } /** - * Insert a new element in the heap. + * \brief Inserts a new element in the heap. * * We select the next empty leaf node, and then keep moving any larger * parents down until the right location is found to store this element. * - * Params: - * value = the new element to be inserted in the heap + * \param value the new element to be inserted in the heap */ void insert(T value) { /* If heap is full, then return without adding this element. */ - if (count == length) { + if (size() == capacity()) { return; } heap.push_back(value); - static CompareT compareT; - std::push_heap(heap.begin(), heap.end(), compareT); - ++count; + std::push_heap(heap.begin(), heap.end(), greater()); } - - /** - * Returns the node of minimum value from the heap (top of the heap). + * \brief Returns the node of minimum value from the heap (top of the heap). * - * Params: - * value = out parameter used to return the min element - * Returns: false if heap empty + * \param[out] value parameter used to return the min element + * \returns false if heap empty */ bool popMin(T& value) { - if (count == 0) { + if (empty()) { return false; } value = heap[0]; - static CompareT compareT; - std::pop_heap(heap.begin(), heap.end(), compareT); + std::pop_heap(heap.begin(), heap.end(), greater()); heap.pop_back(); - --count; return true; /* Return old last node. */ } + + /** + * \brief Returns a shared heap for the given memory pool ID. + * + * It constructs the heap if it does not already exists. + * + * \param poolId a user-chosen hashable ID for identifying the heap. + * For thread-safe operations, using current thread ID is a good choice. + * \param capacity heap maximum capacity + * \param iterThreshold remove heaps that were not reused for more than specified iterations count + * if iterThreshold value is less 2, it will be internally adjusted to twice the number of CPU threads + * \returns pointer to the heap + */ + template + static cv::Ptr> getPooledInstance( + const HashableT& poolId, const int capacity, int iterThreshold = 0) + { + static cv::Mutex mutex; + const cv::AutoLock lock(mutex); + + struct HeapMapValueType { + cv::Ptr> heapPtr; + int iterCounter; + }; + typedef std::unordered_map HeapMapType; + + static HeapMapType heapsPool; + typename HeapMapType::iterator heapIt = heapsPool.find(poolId); + + if (heapIt == heapsPool.end()) + { + // Construct the heap as it does not already exists + HeapMapValueType heapAndTimePair = {cv::makePtr>(capacity), 0}; + const std::pair& emplaceResult = heapsPool.emplace(poolId, std::move(heapAndTimePair)); + CV_CheckEQ(static_cast(emplaceResult.second), 1, "Failed to insert the heap into its memory pool"); + heapIt = emplaceResult.first; + } + else + { + CV_CheckEQ(heapIt->second.heapPtr.use_count(), 1, "Cannot modify a heap that is currently accessed by another caller"); + heapIt->second.heapPtr->clear(); + heapIt->second.heapPtr->reserve(capacity); + heapIt->second.iterCounter = 0; + } + + if (iterThreshold <= 1) { + iterThreshold = 2 * cv::getNumThreads(); + } + + // Remove heaps that were not reused for more than given iterThreshold + typename HeapMapType::iterator cleanupIt = heapsPool.begin(); + while (cleanupIt != heapsPool.end()) + { + if (cleanupIt->second.iterCounter++ > iterThreshold) + { + CV_Assert(cleanupIt != heapIt); + cleanupIt = heapsPool.erase(cleanupIt); + continue; + } + ++cleanupIt; + } + + return heapIt->second.heapPtr; + } }; } diff --git a/modules/flann/include/opencv2/flann/hierarchical_clustering_index.h b/modules/flann/include/opencv2/flann/hierarchical_clustering_index.h index 2d39d4f0f6..60662e7714 100644 --- a/modules/flann/include/opencv2/flann/hierarchical_clustering_index.h +++ b/modules/flann/include/opencv2/flann/hierarchical_clustering_index.h @@ -532,7 +532,7 @@ public: const bool explore_all_trees = get_param(searchParams,"explore_all_trees",false); // Priority queue storing intermediate branches in the best-bin-first search - Heap* heap = new Heap((int)size_); + const cv::Ptr>& heap = Heap::getPooledInstance(cv::utils::getThreadID(), (int)size_); std::vector checked(size_,false); int checks = 0; @@ -548,8 +548,6 @@ public: findNN(node, result, vec, checks, maxChecks, heap, checked, false); } - delete heap; - CV_Assert(result.full()); } @@ -742,7 +740,7 @@ private: void findNN(NodePtr node, ResultSet& result, const ElementType* vec, int& checks, int maxChecks, - Heap* heap, std::vector& checked, bool explore_all_trees = false) + const cv::Ptr>& heap, std::vector& checked, bool explore_all_trees = false) { if (node->childs==NULL) { if (!explore_all_trees && (checks>=maxChecks) && result.full()) { diff --git a/modules/flann/include/opencv2/flann/kdtree_index.h b/modules/flann/include/opencv2/flann/kdtree_index.h index 603fdbd421..8245f7db79 100644 --- a/modules/flann/include/opencv2/flann/kdtree_index.h +++ b/modules/flann/include/opencv2/flann/kdtree_index.h @@ -445,11 +445,12 @@ private: { int i; BranchSt branch; - int checkCount = 0; - Heap* heap = new Heap((int)size_); DynamicBitset checked(size_); + // Priority queue storing intermediate branches in the best-bin-first search + const cv::Ptr>& heap = Heap::getPooledInstance(cv::utils::getThreadID(), (int)size_); + /* Search once through each tree down to root. */ for (i = 0; i < trees_; ++i) { searchLevel(result, vec, tree_roots_[i], 0, checkCount, maxCheck, @@ -464,8 +465,6 @@ private: epsError, heap, checked, false); } - delete heap; - CV_Assert(result.full()); } @@ -476,7 +475,7 @@ private: * at least "mindistsq". */ void searchLevel(ResultSet& result_set, const ElementType* vec, NodePtr node, DistanceType mindist, int& checkCount, int maxCheck, - float epsError, Heap* heap, DynamicBitset& checked, bool explore_all_trees = false) + float epsError, const cv::Ptr>& heap, DynamicBitset& checked, bool explore_all_trees = false) { if (result_set.worstDist()* heap = new Heap((int)size_); + const cv::Ptr>& heap = Heap::getPooledInstance(cv::utils::getThreadID(), (int)size_); int checks = 0; for (int i=0; i& result, const ElementType* vec, int& checks, int maxChecks, - Heap* heap) + const cv::Ptr>& heap) { // Ignore those clusters that are too far away { @@ -1577,7 +1575,7 @@ private: * distances = array with the distances to each child node. * Returns: */ - int exploreNodeBranches(KMeansNodePtr node, const ElementType* q, DistanceType* domain_distances, Heap* heap) + int exploreNodeBranches(KMeansNodePtr node, const ElementType* q, DistanceType* domain_distances, const cv::Ptr>& heap) { int best_index = 0; From 9d61c181434a6903fa15e4915b9fffed65ebcae8 Mon Sep 17 00:00:00 2001 From: utibenkei Date: Sun, 8 Aug 2021 01:08:31 +0900 Subject: [PATCH 111/262] fix testSaveLoad --- modules/ml/misc/java/test/MLTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/ml/misc/java/test/MLTest.java b/modules/ml/misc/java/test/MLTest.java index 2b08543a84..504805dffa 100644 --- a/modules/ml/misc/java/test/MLTest.java +++ b/modules/ml/misc/java/test/MLTest.java @@ -36,7 +36,7 @@ public class MLTest extends OpenCVTestCase { String filename = OpenCVTestRunner.getTempFileName("yml"); saved.save(filename); SVM loaded = SVM.load(filename); - assertTrue(saved.isTrained()); + assertTrue(loaded.isTrained()); } } From 8199967b3189fb9aa711afc4e815cd13f312b7ae Mon Sep 17 00:00:00 2001 From: AleksandrPanov Date: Wed, 11 Aug 2021 19:08:52 +0300 Subject: [PATCH 112/262] fix choose minimum angle in rotatingCalipers --- modules/imgproc/src/rotcalipers.cpp | 55 ++++++++++++++++++----------- 1 file changed, 34 insertions(+), 21 deletions(-) diff --git a/modules/imgproc/src/rotcalipers.cpp b/modules/imgproc/src/rotcalipers.cpp index 527f71a247..e3d81c7e0c 100644 --- a/modules/imgproc/src/rotcalipers.cpp +++ b/modules/imgproc/src/rotcalipers.cpp @@ -88,6 +88,32 @@ enum { CALIPERS_MAXHEIGHT=0, CALIPERS_MINAREARECT=1, CALIPERS_MAXDIST=2 }; // Notes: //F*/ +static void rotate90CCW(const cv::Point2f& in, cv::Point2f &out) +{ + out.x = -in.y; + out.y = in.x; +} + +static void rotate90CW(const cv::Point2f& in, cv::Point2f &out) +{ + out.x = in.y; + out.y = -in.x; +} + +static void rotate180(const cv::Point2f& in, cv::Point2f &out) +{ + out.x = -in.x; + out.y = -in.y; +} + +/* return true if first vector is to the right (clockwise) of the second */ +static bool firstVecIsRight(const cv::Point2f& vec1, const cv::Point2f &vec2) +{ + cv::Point2f tmp; + rotate90CW(vec1, tmp); + return tmp.x * vec2.x + tmp.y * vec2.y < 0; +} + /* we will use usual cartesian coordinates */ static void rotatingCalipers( const Point2f* points, int n, int mode, float* out ) { @@ -100,6 +126,7 @@ static void rotatingCalipers( const Point2f* points, int n, int mode, float* out Point2f* vect = (Point2f*)(inv_vect_length + n); int left = 0, bottom = 0, right = 0, top = 0; int seq[4] = { -1, -1, -1, -1 }; + Point2f rot_vect[4]; /* rotating calipers sides will always have coordinates (a,b) (-b,a) (-a,-b) (b, -a) @@ -179,32 +206,18 @@ static void rotatingCalipers( const Point2f* points, int n, int mode, float* out /* all of edges will be checked while rotating calipers by 90 degrees */ for( k = 0; k < n; k++ ) { - /* sinus of minimal angle */ - /*float sinus;*/ - - /* compute cosine of angle between calipers side and polygon edge */ - /* dp - dot product */ - float dp[4] = { - +base_a * vect[seq[0]].x + base_b * vect[seq[0]].y, - -base_b * vect[seq[1]].x + base_a * vect[seq[1]].y, - -base_a * vect[seq[2]].x - base_b * vect[seq[2]].y, - +base_b * vect[seq[3]].x - base_a * vect[seq[3]].y, - }; - - float maxcos = dp[0] * inv_vect_length[seq[0]]; - /* number of calipers edges, that has minimal angle with edge */ int main_element = 0; - /* choose minimal angle */ - for ( i = 1; i < 4; ++i ) + /* choose minimum angle between calipers side and polygon edge by dot product sign */ + rot_vect[0] = vect[seq[0]]; + rotate90CW(vect[seq[1]], rot_vect[1]); + rotate180(vect[seq[2]], rot_vect[2]); + rotate90CCW(vect[seq[3]], rot_vect[3]); + for (i = 1; i < 4; i++) { - float cosalpha = dp[i] * inv_vect_length[seq[i]]; - if (cosalpha > maxcos) - { + if (firstVecIsRight(rot_vect[i], rot_vect[main_element])) main_element = i; - maxcos = cosalpha; - } } /*rotate calipers*/ From 4d63a89fa6611a69c1999328375f39afde1b1696 Mon Sep 17 00:00:00 2001 From: Daniel Playfair Cal Date: Thu, 12 Aug 2021 03:58:08 +1000 Subject: [PATCH 113/262] Merge pull request #20536 from hedgepigdaniel:fix/ocl-context-create-ownership docs(core/ocl): clarify ownership of arguments passed into OpenCL related functions * docs(core/ocl): clarify ownership in OpenCLExecutionContext::create Although it is technically true that OpenCLExecutionContext::create calls `clRetainContext` on its context argument, it is misleading because it does not increase the reference count overall. Clarify that the ownership of one reference of the passed context and device is taken. * docs(core/ocl): document ownership transfer in ocl::Device::fromHandle --- modules/core/include/opencv2/core/ocl.hpp | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/modules/core/include/opencv2/core/ocl.hpp b/modules/core/include/opencv2/core/ocl.hpp index f9cc9e019a..03666df517 100644 --- a/modules/core/include/opencv2/core/ocl.hpp +++ b/modules/core/include/opencv2/core/ocl.hpp @@ -235,7 +235,11 @@ public: /** * @param d OpenCL handle (cl_device_id). clRetainDevice() is called on success. - */ + * + * @note Ownership of the passed device is passed to OpenCV on success. + * The caller should additionally call `clRetainDevice` on it if it intends + * to continue using the device. + */ static Device fromHandle(void* d); struct Impl; @@ -826,11 +830,13 @@ public: OpenCLExecutionContext cloneWithNewQueue() const; /** @brief Creates OpenCL execution context - * OpenCV will check if available OpenCL platform has platformName name, then assign context to - * OpenCV and call `clRetainContext` function. The deviceID device will be used as target device and - * new command queue will be created. + * OpenCV will check if available OpenCL platform has platformName name, + * then assign context to OpenCV. + * The deviceID device will be used as target device and a new command queue will be created. * - * @note Lifetime of passed handles is transferred to OpenCV wrappers on success + * @note On success, ownership of one reference of the context and device is taken. + * The caller should additionally call `clRetainContext` and/or `clRetainDevice` + * to increase the reference count if it wishes to continue using them. * * @param platformName name of OpenCL platform to attach, this string is used to check if platform is available to OpenCV at runtime * @param platformID ID of platform attached context was created for (cl_platform_id) From 25cd7c7c509826e42214d00ca4e73ac096abd382 Mon Sep 17 00:00:00 2001 From: AleksandrPanov Date: Thu, 12 Aug 2021 14:40:40 +0300 Subject: [PATCH 114/262] add note about Python's dsize to doc --- modules/imgproc/include/opencv2/imgproc.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/imgproc/include/opencv2/imgproc.hpp b/modules/imgproc/include/opencv2/imgproc.hpp index 5e66b14e3b..f7583c1926 100644 --- a/modules/imgproc/include/opencv2/imgproc.hpp +++ b/modules/imgproc/include/opencv2/imgproc.hpp @@ -2223,7 +2223,7 @@ enlarge an image, it will generally look best with c#INTER_CUBIC (slow) or #INTE @param src input image. @param dst output image; it has the size dsize (when it is non-zero) or the size computed from src.size(), fx, and fy; the type of dst is the same as of src. -@param dsize output image size; if it equals zero, it is computed as: +@param dsize output image size; if it equals zero (`None` in Python), it is computed as: \f[\texttt{dsize = Size(round(fx*src.cols), round(fy*src.rows))}\f] Either dsize or both fx and fy must be non-zero. @param fx scale factor along the horizontal axis; when it equals 0, it is computed as From 955cf35d5f890b85baa12b254a325b98880813d0 Mon Sep 17 00:00:00 2001 From: JIANG Yichen Date: Mon, 9 Aug 2021 13:46:11 +0800 Subject: [PATCH 115/262] Implement ctc prefix beam search decode for TextRecognitionModel. The algorithm is based on Hannun's paper: First-Pass Large Vocabulary Continuous Speech Recognition using Bi-Directional Recurrent DNNs --- .../dnn_text_spotting.markdown | 5 + modules/dnn/include/opencv2/dnn/dnn.hpp | 13 +- modules/dnn/src/math_utils.hpp | 83 ++++++ modules/dnn/src/model.cpp | 254 +++++++++++++++--- modules/dnn/test/test_model.cpp | 19 ++ 5 files changed, 335 insertions(+), 39 deletions(-) create mode 100644 modules/dnn/src/math_utils.hpp diff --git a/doc/tutorials/dnn/dnn_text_spotting/dnn_text_spotting.markdown b/doc/tutorials/dnn/dnn_text_spotting/dnn_text_spotting.markdown index 5f28b6ce7a..b0be2627b2 100644 --- a/doc/tutorials/dnn/dnn_text_spotting/dnn_text_spotting.markdown +++ b/doc/tutorials/dnn/dnn_text_spotting/dnn_text_spotting.markdown @@ -26,6 +26,11 @@ Before recognition, you should `setVocabulary` and `setDecodeType`. - `T` is the sequence length - `B` is the batch size (only support `B=1` in inference) - and `Dim` is the length of vocabulary +1('Blank' of CTC is at the index=0 of Dim). +- "CTC-prefix-beam-search", the output of the text recognition model should be a probability matrix same with "CTC-greedy". + - The algorithm is proposed at Hannun's [paper](https://arxiv.org/abs/1408.2873). + - `setDecodeOptsCTCPrefixBeamSearch` could be used to control the beam size in search step. + - To futher optimize for big vocabulary, a new option `vocPruneSize` is introduced to avoid iterate the whole vocbulary + but only the number of `vocPruneSize` tokens with top probabilty. @ref cv::dnn::TextRecognitionModel::recognize() is the main function for text recognition. - The input image should be a cropped text image or an image with `roiRects` diff --git a/modules/dnn/include/opencv2/dnn/dnn.hpp b/modules/dnn/include/opencv2/dnn/dnn.hpp index 255b41de88..a498039f65 100644 --- a/modules/dnn/include/opencv2/dnn/dnn.hpp +++ b/modules/dnn/include/opencv2/dnn/dnn.hpp @@ -1373,7 +1373,9 @@ public: /** * @brief Set the decoding method of translating the network output into string - * @param[in] decodeType The decoding method of translating the network output into string: {'CTC-greedy': greedy decoding for the output of CTC-based methods} + * @param[in] decodeType The decoding method of translating the network output into string, currently supported type: + * - `"CTC-greedy"` greedy decoding for the output of CTC-based methods + * - `"CTC-prefix-beam-search"` Prefix beam search decoding for the output of CTC-based methods */ CV_WRAP TextRecognitionModel& setDecodeType(const std::string& decodeType); @@ -1385,6 +1387,15 @@ public: CV_WRAP const std::string& getDecodeType() const; + /** + * @brief Set the decoding method options for `"CTC-prefix-beam-search"` decode usage + * @param[in] beamSize Beam size for search + * @param[in] vocPruneSize Parameter to optimize big vocabulary search, + * only take top @p vocPruneSize tokens in each search step, @p vocPruneSize <= 0 stands for disable this prune. + */ + CV_WRAP + TextRecognitionModel& setDecodeOptsCTCPrefixBeamSearch(int beamSize, int vocPruneSize = 0); + /** * @brief Set the vocabulary for recognition. * @param[in] vocabulary the associated vocabulary of the network. diff --git a/modules/dnn/src/math_utils.hpp b/modules/dnn/src/math_utils.hpp new file mode 100644 index 0000000000..19ee474c73 --- /dev/null +++ b/modules/dnn/src/math_utils.hpp @@ -0,0 +1,83 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. + +// Code is borrowed from https://github.com/kaldi-asr/kaldi/blob/master/src/base/kaldi-math.h + +// base/kaldi-math.h + +// Copyright 2009-2011 Ondrej Glembek; Microsoft Corporation; Yanmin Qian; +// Jan Silovsky; Saarland University +// +// See ../../COPYING for clarification regarding multiple authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED +// WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABLITY OR NON-INFRINGEMENT. +// See the Apache 2 License for the specific language governing permissions and +// limitations under the License. + +#ifndef __OPENCV_DNN_MATH_UTILS_HPP__ +#define __OPENCV_DNN_MATH_UTILS_HPP__ + +#ifdef OS_QNX +#include +#else +#include +#endif + +#include + +#ifndef FLT_EPSILON +#define FLT_EPSILON 1.19209290e-7f +#endif + +namespace cv { namespace dnn { + +const float kNegativeInfinity = -std::numeric_limits::infinity(); + +const float kMinLogDiffFloat = std::log(FLT_EPSILON); + +#if !defined(_MSC_VER) || (_MSC_VER >= 1700) +inline float Log1p(float x) { return log1pf(x); } +#else +inline float Log1p(float x) { + const float cutoff = 1.0e-07; + if (x < cutoff) + return x - 2 * x * x; + else + return Log(1.0 + x); +} +#endif + +inline float Exp(float x) { return expf(x); } + +inline float LogAdd(float x, float y) { + float diff; + if (x < y) { + diff = x - y; + x = y; + } else { + diff = y - x; + } + // diff is negative. x is now the larger one. + + if (diff >= kMinLogDiffFloat) { + float res; + res = x + Log1p(Exp(diff)); + return res; + } else { + return x; // return the larger one. + } +} + +}} // namespace + +#endif // __OPENCV_DNN_MATH_UTILS_HPP__ diff --git a/modules/dnn/src/model.cpp b/modules/dnn/src/model.cpp index 0af8223a7f..bc8709d22e 100644 --- a/modules/dnn/src/model.cpp +++ b/modules/dnn/src/model.cpp @@ -3,8 +3,10 @@ // of this distribution and at http://opencv.org/license.html. #include "precomp.hpp" +#include "math_utils.hpp" #include #include +#include #include #include @@ -552,6 +554,9 @@ struct TextRecognitionModel_Impl : public Model::Impl std::string decodeType; std::vector vocabulary; + int beamSize = 10; + int vocPruneSize = 0; + TextRecognitionModel_Impl() { CV_TRACE_FUNCTION(); @@ -575,6 +580,13 @@ struct TextRecognitionModel_Impl : public Model::Impl decodeType = type; } + inline + void setDecodeOptsCTCPrefixBeamSearch(int beam, int vocPrune) + { + beamSize = beam; + vocPruneSize = vocPrune; + } + virtual std::string decode(const Mat& prediction) { @@ -586,44 +598,10 @@ struct TextRecognitionModel_Impl : public Model::Impl CV_Error(Error::StsBadArg, "TextRecognitionModel: vocabulary is not specified"); std::string decodeSeq; - if (decodeType == "CTC-greedy") - { - CV_CheckEQ(prediction.dims, 3, ""); - CV_CheckType(prediction.type(), CV_32FC1, ""); - const int vocLength = (int)(vocabulary.size()); - CV_CheckLE(prediction.size[1], vocLength, ""); - bool ctcFlag = true; - int lastLoc = 0; - for (int i = 0; i < prediction.size[0]; i++) - { - const float* pred = prediction.ptr(i); - int maxLoc = 0; - float maxScore = pred[0]; - for (int j = 1; j < vocLength + 1; j++) - { - float score = pred[j]; - if (maxScore < score) - { - maxScore = score; - maxLoc = j; - } - } - - if (maxLoc > 0) - { - std::string currentChar = vocabulary.at(maxLoc - 1); - if (maxLoc != lastLoc || ctcFlag) - { - lastLoc = maxLoc; - decodeSeq += currentChar; - ctcFlag = false; - } - } - else - { - ctcFlag = true; - } - } + if (decodeType == "CTC-greedy") { + decodeSeq = ctcGreedyDecode(prediction); + } else if (decodeType == "CTC-prefix-beam-search") { + decodeSeq = ctcPrefixBeamSearchDecode(prediction); } else if (decodeType.length() == 0) { CV_Error(Error::StsBadArg, "Please set decodeType"); } else { @@ -633,6 +611,200 @@ struct TextRecognitionModel_Impl : public Model::Impl return decodeSeq; } + virtual + std::string ctcGreedyDecode(const Mat& prediction) + { + std::string decodeSeq; + CV_CheckEQ(prediction.dims, 3, ""); + CV_CheckType(prediction.type(), CV_32FC1, ""); + const int vocLength = (int)(vocabulary.size()); + CV_CheckLE(prediction.size[1], vocLength, ""); + bool ctcFlag = true; + int lastLoc = 0; + for (int i = 0; i < prediction.size[0]; i++) + { + const float* pred = prediction.ptr(i); + int maxLoc = 0; + float maxScore = pred[0]; + for (int j = 1; j < vocLength + 1; j++) + { + float score = pred[j]; + if (maxScore < score) + { + maxScore = score; + maxLoc = j; + } + } + + if (maxLoc > 0) + { + std::string currentChar = vocabulary.at(maxLoc - 1); + if (maxLoc != lastLoc || ctcFlag) + { + lastLoc = maxLoc; + decodeSeq += currentChar; + ctcFlag = false; + } + } + else + { + ctcFlag = true; + } + } + return decodeSeq; + } + + struct PrefixScore + { + // blank ending score + float pB; + // none blank ending score + float pNB; + + PrefixScore() : pB(kNegativeInfinity), pNB(kNegativeInfinity) + { + + } + PrefixScore(float pB, float pNB) : pB(pB), pNB(pNB) + { + + } + }; + + struct PrefixHash + { + size_t operator()(const std::vector& prefix) const + { + // BKDR hash + unsigned int seed = 131; + size_t hash = 0; + for (size_t i = 0; i < prefix.size(); i++) + { + hash = hash * seed + prefix[i]; + } + return hash; + } + }; + + static + std::vector> TopK( + const float* predictions, int length, int k) + { + std::vector> results; + // No prune. + if (k <= 0) + { + for (int i = 0; i < length; ++i) + { + results.emplace_back(predictions[i], i); + } + return results; + } + + for (int i = 0; i < k; ++i) + { + results.emplace_back(predictions[i], i); + } + std::make_heap(results.begin(), results.end(), std::greater>{}); + + for (int i = k; i < length; ++i) + { + if (predictions[i] > results.front().first) + { + std::pop_heap(results.begin(), results.end(), std::greater>{}); + results.pop_back(); + results.emplace_back(predictions[i], i); + std::push_heap(results.begin(), results.end(), std::greater>{}); + } + } + return results; + } + + static inline + bool PrefixScoreCompare( + const std::pair, PrefixScore>& a, + const std::pair, PrefixScore>& b) + { + float probA = LogAdd(a.second.pB, a.second.pNB); + float probB = LogAdd(b.second.pB, b.second.pNB); + return probA > probB; + } + + virtual + std::string ctcPrefixBeamSearchDecode(const Mat& prediction) { + // CTC prefix beam seach decode. + // For more detail, refer to: + // https://distill.pub/2017/ctc/#inference + // https://gist.github.com/awni/56369a90d03953e370f3964c826ed4b0i + using Beam = std::vector, PrefixScore>>; + using BeamInDict = std::unordered_map, PrefixScore, PrefixHash>; + + CV_CheckType(prediction.type(), CV_32FC1, ""); + CV_CheckEQ(prediction.dims, 3, ""); + CV_CheckEQ(prediction.size[1], 1, ""); + CV_CheckEQ(prediction.size[2], (int)vocabulary.size() + 1, ""); // Length add 1 for ctc blank + + std::string decodeSeq; + Beam beam = {std::make_pair(std::vector(), PrefixScore(0.0, kNegativeInfinity))}; + for (int i = 0; i < prediction.size[0]; i++) + { + // Loop over time + BeamInDict nextBeam; + const float* pred = prediction.ptr(i); + std::vector> topkPreds = + TopK(pred, vocabulary.size() + 1, vocPruneSize); + for (const auto& each : topkPreds) + { + // Loop over vocabulary + float prob = each.first; + int token = each.second; + for (const auto& it : beam) + { + const std::vector& prefix = it.first; + const PrefixScore& prefixScore = it.second; + if (token == 0) // 0 stands for ctc blank + { + PrefixScore& nextScore = nextBeam[prefix]; + nextScore.pB = LogAdd(nextScore.pB, + LogAdd(prefixScore.pB + prob, prefixScore.pNB + prob)); + continue; + } + + std::vector nPrefix(prefix); + nPrefix.push_back(token); + PrefixScore& nextScore = nextBeam[nPrefix]; + if (prefix.size() > 0 && token == prefix.back()) + { + nextScore.pNB = LogAdd(nextScore.pNB, prefixScore.pB + prob); + PrefixScore& mScore = nextBeam[prefix]; + mScore.pNB = LogAdd(mScore.pNB, prefixScore.pNB + prob); + } + else + { + nextScore.pNB = LogAdd(nextScore.pNB, + LogAdd(prefixScore.pB + prob, prefixScore.pNB + prob)); + } + } + } + // Beam prune + Beam newBeam(nextBeam.begin(), nextBeam.end()); + int newBeamSize = std::min(static_cast(newBeam.size()), beamSize); + std::nth_element(newBeam.begin(), newBeam.begin() + newBeamSize, + newBeam.end(), PrefixScoreCompare); + newBeam.resize(newBeamSize); + std::sort(newBeam.begin(), newBeam.end(), PrefixScoreCompare); + beam = std::move(newBeam); + } + + CV_Assert(!beam.empty()); + for (int token : beam[0].first) + { + CV_Check(token, token > 0 && token <= vocabulary.size(), ""); + decodeSeq += vocabulary.at(token - 1); + } + return decodeSeq; + } + virtual std::string recognize(InputArray frame) { @@ -698,6 +870,12 @@ const std::string& TextRecognitionModel::getDecodeType() const return TextRecognitionModel_Impl::from(impl).decodeType; } +TextRecognitionModel& TextRecognitionModel::setDecodeOptsCTCPrefixBeamSearch(int beamSize, int vocPruneSize) +{ + TextRecognitionModel_Impl::from(impl).setDecodeOptsCTCPrefixBeamSearch(beamSize, vocPruneSize); + return *this; +} + TextRecognitionModel& TextRecognitionModel::setVocabulary(const std::vector& inputVoc) { TextRecognitionModel_Impl::from(impl).setVocabulary(inputVoc); diff --git a/modules/dnn/test/test_model.cpp b/modules/dnn/test/test_model.cpp index f7befa9937..6ac9702c69 100644 --- a/modules/dnn/test/test_model.cpp +++ b/modules/dnn/test/test_model.cpp @@ -615,6 +615,25 @@ TEST_P(Test_Model, TextRecognition) testTextRecognitionModel(weightPath, "", imgPath, seq, decodeType, vocabulary, size, mean, scale); } +TEST_P(Test_Model, TextRecognitionWithCTCPrefixBeamSearch) +{ + if (target == DNN_TARGET_OPENCL_FP16) + applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16); + + std::string imgPath = _tf("text_rec_test.png"); + std::string weightPath = _tf("onnx/models/crnn.onnx", false); + std::string seq = "welcome"; + + Size size{100, 32}; + double scale = 1.0 / 127.5; + Scalar mean = Scalar(127.5); + std::string decodeType = "CTC-prefix-beam-search"; + std::vector vocabulary = {"0","1","2","3","4","5","6","7","8","9", + "a","b","c","d","e","f","g","h","i","j","k","l","m","n","o","p","q","r","s","t","u","v","w","x","y","z"}; + + testTextRecognitionModel(weightPath, "", imgPath, seq, decodeType, vocabulary, size, mean, scale); +} + TEST_P(Test_Model, TextDetectionByDB) { if (target == DNN_TARGET_OPENCL_FP16) From 4300bb2e1f5eadd9b6eb1244ab2ed0250c2418b2 Mon Sep 17 00:00:00 2001 From: Iyad Ahmed Date: Thu, 12 Aug 2021 16:51:02 +0000 Subject: [PATCH 116/262] Merge pull request #20541 from iyadahmed:video_capture_timeout_prop * VideoCapture timeout set/get * Common formatting for enum values * Fix enum values wrongly in videoio.hpp * Define timeout enum values in public api and align with master --- modules/videoio/include/opencv2/videoio.hpp | 2 ++ modules/videoio/src/cap_ffmpeg_api.hpp | 4 ++- modules/videoio/src/cap_ffmpeg_impl.hpp | 33 +++++++++++++++++---- 3 files changed, 32 insertions(+), 7 deletions(-) diff --git a/modules/videoio/include/opencv2/videoio.hpp b/modules/videoio/include/opencv2/videoio.hpp index ba9c18bd97..aa247dd84e 100644 --- a/modules/videoio/include/opencv2/videoio.hpp +++ b/modules/videoio/include/opencv2/videoio.hpp @@ -179,6 +179,8 @@ enum VideoCaptureProperties { CAP_PROP_BITRATE =47, //!< (read-only) Video bitrate in kbits/s CAP_PROP_ORIENTATION_META=48, //!< (read-only) Frame rotation defined by stream meta (applicable for FFmpeg back-end only) CAP_PROP_ORIENTATION_AUTO=49, //!< if true - rotates output frames of CvCapture considering video file's metadata (applicable for FFmpeg back-end only) (https://github.com/opencv/opencv/issues/15499) + CAP_PROP_OPEN_TIMEOUT_MSEC=53, + CAP_PROP_READ_TIMEOUT_MSEC=54, #ifndef CV_DOXYGEN CV__CAP_PROP_LATEST #endif diff --git a/modules/videoio/src/cap_ffmpeg_api.hpp b/modules/videoio/src/cap_ffmpeg_api.hpp index 984d36f23c..e618765539 100644 --- a/modules/videoio/src/cap_ffmpeg_api.hpp +++ b/modules/videoio/src/cap_ffmpeg_api.hpp @@ -30,7 +30,9 @@ enum CV_FFMPEG_CAP_PROP_CODEC_PIXEL_FORMAT=46, CV_FFMPEG_CAP_PROP_BITRATE=47, CV_FFMPEG_CAP_PROP_ORIENTATION_META=48, - CV_FFMPEG_CAP_PROP_ORIENTATION_AUTO=49 + CV_FFMPEG_CAP_PROP_ORIENTATION_AUTO=49, + CV_FFMPEG_CAP_PROP_OPEN_TIMEOUT_MSEC=53, + CV_FFMPEG_CAP_PROP_READ_TIMEOUT_MSEC=54 }; typedef struct CvCapture_FFMPEG CvCapture_FFMPEG; diff --git a/modules/videoio/src/cap_ffmpeg_impl.hpp b/modules/videoio/src/cap_ffmpeg_impl.hpp index 4164ab941c..937d348215 100644 --- a/modules/videoio/src/cap_ffmpeg_impl.hpp +++ b/modules/videoio/src/cap_ffmpeg_impl.hpp @@ -186,8 +186,8 @@ extern "C" { #endif #if USE_AV_INTERRUPT_CALLBACK -#define LIBAVFORMAT_INTERRUPT_OPEN_TIMEOUT_MS 30000 -#define LIBAVFORMAT_INTERRUPT_READ_TIMEOUT_MS 30000 +#define LIBAVFORMAT_INTERRUPT_OPEN_DEFAULT_TIMEOUT_MS 30000 +#define LIBAVFORMAT_INTERRUPT_READ_DEFAULT_TIMEOUT_MS 30000 #ifdef _WIN32 // http://stackoverflow.com/questions/5404277/porting-clock-gettime-to-windows @@ -534,6 +534,8 @@ struct CvCapture_FFMPEG AVDictionary *dict; #endif #if USE_AV_INTERRUPT_CALLBACK + int open_timeout_ms; + int read_timeout_ms; AVInterruptCallbackMetadata interrupt_metadata; #endif @@ -568,6 +570,11 @@ void CvCapture_FFMPEG::init() frame_number = 0; eps_zero = 0.000025; +#if USE_AV_INTERRUPT_CALLBACK + open_timeout_ms = LIBAVFORMAT_INTERRUPT_OPEN_DEFAULT_TIMEOUT_MS; + read_timeout_ms = LIBAVFORMAT_INTERRUPT_READ_DEFAULT_TIMEOUT_MS; +#endif + rotation_angle = 0; #if (LIBAVFORMAT_BUILD >= CALC_FFMPEG_VERSION(52, 111, 0)) @@ -923,7 +930,7 @@ bool CvCapture_FFMPEG::open( const char* _filename ) #if USE_AV_INTERRUPT_CALLBACK /* interrupt callback */ - interrupt_metadata.timeout_after_ms = LIBAVFORMAT_INTERRUPT_OPEN_TIMEOUT_MS; + interrupt_metadata.timeout_after_ms = open_timeout_ms; get_monotonic_time(&interrupt_metadata.value); ic = avformat_alloc_context(); @@ -1227,7 +1234,7 @@ bool CvCapture_FFMPEG::grabFrame() #if USE_AV_INTERRUPT_CALLBACK // activate interrupt callback get_monotonic_time(&interrupt_metadata.value); - interrupt_metadata.timeout_after_ms = LIBAVFORMAT_INTERRUPT_READ_TIMEOUT_MS; + interrupt_metadata.timeout_after_ms = read_timeout_ms; #endif // get the next frame @@ -1483,6 +1490,12 @@ double CvCapture_FFMPEG::getProperty( int property_id ) const #else return 0; #endif +#if USE_AV_INTERRUPT_CALLBACK + case CV_FFMPEG_CAP_PROP_OPEN_TIMEOUT_MSEC: + return static_cast(open_timeout_ms); + case CV_FFMPEG_CAP_PROP_READ_TIMEOUT_MSEC: + return static_cast(read_timeout_ms); +#endif // USE_AV_INTERRUPT_CALLBACK default: break; } @@ -1677,6 +1690,14 @@ bool CvCapture_FFMPEG::setProperty( int property_id, double value ) return false; #endif break; +#if USE_AV_INTERRUPT_CALLBACK + case CV_FFMPEG_CAP_PROP_OPEN_TIMEOUT_MSEC: + open_timeout_ms = (int)value; + break; + case CV_FFMPEG_CAP_PROP_READ_TIMEOUT_MSEC: + read_timeout_ms = (int)value; + break; +#endif // USE_AV_INTERRUPT_CALLBACK default: return false; } @@ -3114,7 +3135,7 @@ bool InputMediaStream_FFMPEG::open(const char* fileName, int* codec, int* chroma #if USE_AV_INTERRUPT_CALLBACK /* interrupt callback */ - interrupt_metadata.timeout_after_ms = LIBAVFORMAT_INTERRUPT_OPEN_TIMEOUT_MS; + interrupt_metadata.timeout_after_ms = LIBAVFORMAT_INTERRUPT_OPEN_DEFAULT_TIMEOUT_MS; get_monotonic_time(&interrupt_metadata.value); ctx_ = avformat_alloc_context(); @@ -3241,7 +3262,7 @@ bool InputMediaStream_FFMPEG::read(unsigned char** data, int* size, int* endOfFi #if USE_AV_INTERRUPT_CALLBACK // activate interrupt callback get_monotonic_time(&interrupt_metadata.value); - interrupt_metadata.timeout_after_ms = LIBAVFORMAT_INTERRUPT_READ_TIMEOUT_MS; + interrupt_metadata.timeout_after_ms = LIBAVFORMAT_INTERRUPT_READ_DEFAULT_TIMEOUT_MS; #endif // free last packet if exist From cfb36443fb02586aac34cb14f6f67b551e29cb68 Mon Sep 17 00:00:00 2001 From: Julia Bareeva <34717687+JulieBar@users.noreply.github.com> Date: Fri, 13 Aug 2021 15:41:00 +0300 Subject: [PATCH 117/262] Merge pull request #20506 from JulieBar:lstm_activations * Support activations(Sigmoid, Tanh) for LSTM * fix warning --- modules/dnn/src/layers/recurrent_layers.cpp | 49 ++++++++++++++++++--- modules/dnn/src/onnx/onnx_importer.cpp | 38 +++++++++++++--- modules/dnn/test/test_onnx_importer.cpp | 5 +++ 3 files changed, 79 insertions(+), 13 deletions(-) diff --git a/modules/dnn/src/layers/recurrent_layers.cpp b/modules/dnn/src/layers/recurrent_layers.cpp index a6715aefca..9088c13390 100644 --- a/modules/dnn/src/layers/recurrent_layers.cpp +++ b/modules/dnn/src/layers/recurrent_layers.cpp @@ -80,12 +80,31 @@ static void sigmoid(const Mat &src, Mat &dst) cv::pow(1 + dst, -1, dst); } +typedef void (*ActivationFunction)(const Mat &src, Mat &dst); +static ActivationFunction get_activation_function(const String& activation) { + // most used activations for PyTorch and TF : Tanh, Sigmoid + // if you need to support more optional activations use std::map instead + if (activation == "Tanh") + { + return tanh; + } + else if (activation == "Sigmoid") + { + return sigmoid; + } + else + { + CV_Error(Error::StsNotImplemented, + cv::format("Activation function [%s] for layer LSTM is not supported", activation.c_str())); + } +} + class LSTMLayerImpl CV_FINAL : public LSTMLayer { int numTimeStamps, numSamples; bool allocated; - MatShape outTailShape; //shape of single output sample + MatShape outTailShape; //shape of single output sample MatShape outTsShape; //shape of N output samples bool useTimestampDim; @@ -95,6 +114,10 @@ class LSTMLayerImpl CV_FINAL : public LSTMLayer bool reverse; // If true, go in negative direction along the time axis bool bidirectional; // If true, produces both forward and reversed directions along time axis + ActivationFunction f_activation; + ActivationFunction g_activation; + ActivationFunction h_activation; + public: LSTMLayerImpl(const LayerParams& params) @@ -145,6 +168,20 @@ public: reverse = params.get("reverse", false); CV_Assert(!reverse || !bidirectional); + // read activations + DictValue activations = params.get("activations", ""); + if (activations.size() == 1) // if activations wasn't specified use default + { + f_activation = sigmoid; + g_activation = tanh; + h_activation = tanh; + } else { + CV_Assert(activations.size() == 3); + f_activation = get_activation_function(activations.getStringValue(0)); + g_activation = get_activation_function(activations.getStringValue(1)); + h_activation = get_activation_function(activations.getStringValue(2)); + } + allocated = false; outTailShape.clear(); } @@ -339,15 +376,15 @@ public: Mat gatesIF = gates.colRange(0, 2*numOut); gemm(cInternal, blobs[5], 1, gateI, 1, gateI); gemm(cInternal, blobs[6], 1, gateF, 1, gateF); - sigmoid(gatesIF, gatesIF); + f_activation(gatesIF, gatesIF); } else { Mat gatesIFO = gates.colRange(0, 3*numOut); - sigmoid(gatesIFO, gatesIFO); + f_activation(gatesIFO, gatesIFO); } - tanh(gateG, gateG); + g_activation(gateG, gateG); //compute c_t multiply(gateF, cInternal, gateF); // f_t (*) c_{t-1} @@ -362,11 +399,11 @@ public: if (usePeephole) { gemm(cInternal, blobs[7], 1, gateO, 1, gateO); - sigmoid(gateO, gateO); + f_activation(gateO, gateO); } //compute h_t - tanh(cInternal, hInternal); + h_activation(cInternal, hInternal); multiply(gateO, hInternal, hInternal); //save results in output blobs diff --git a/modules/dnn/src/onnx/onnx_importer.cpp b/modules/dnn/src/onnx/onnx_importer.cpp index b833b2ea44..32b56278bd 100644 --- a/modules/dnn/src/onnx/onnx_importer.cpp +++ b/modules/dnn/src/onnx/onnx_importer.cpp @@ -244,6 +244,10 @@ static DictValue parse(const ::google::protobuf::RepeatedField< ::google::protob return DictValue::arrayInt(&dst[0], src.size()); } +static DictValue parseStr(const ::google::protobuf::RepeatedPtrField< ::std::string>& src) { + return DictValue::arrayString(src.begin(), static_cast(src.size())); +} + LayerParams ONNXImporter::getLayerParams(const opencv_onnx::NodeProto& node_proto) { LayerParams lp; @@ -301,6 +305,10 @@ LayerParams ONNXImporter::getLayerParams(const opencv_onnx::NodeProto& node_prot CV_Assert(attribute_proto.ints_size() == 1 || attribute_proto.ints_size() == 2 || attribute_proto.ints_size() == 3); lp.set("dilation", parse(attribute_proto.ints())); } + else if(attribute_name == "activations" && node_proto.op_type() == "LSTM") + { + lp.set(attribute_name, parseStr(attribute_proto.strings())); + } else if (attribute_proto.has_i()) { ::google::protobuf::int64 src = attribute_proto.i(); @@ -997,18 +1005,32 @@ void ONNXImporter::parseLSTM(LayerParams& layerParams, const opencv_onnx::NodePr lstmParams.name += "/lstm"; // https://pytorch.org/docs/stable/nn.html#lstm - CV_Assert(node_proto.input_size() == 7); + CV_Assert(node_proto.input_size() >= 7); Mat Wx = getBlob(node_proto, 1); Mat Wh = getBlob(node_proto, 2); Mat b = getBlob(node_proto, 3); - Mat h0 = getBlob(node_proto, 5); - Mat c0 = getBlob(node_proto, 6); - - b = b.reshape(1, b.size[0]); const int numHidden = lstmParams.get("hidden_size"); const int numDirs = Wx.size[0]; // Is 1 for forward only and 2 for bidirectional LSTM. const int numFeatures = Wx.size[2]; + + Mat h0, c0; + if (!node_proto.input(5).empty()) { + h0 = getBlob(node_proto, 5); + h0 = h0.reshape(1, h0.size[0] * h0.size[1]); + } else { + // initial_h attribute can be empty in case of keras2onnx producer. fill it with zeros + h0 = Mat::zeros(numDirs * numFeatures, numHidden, CV_32FC1); + } + if (!node_proto.input(6).empty()) { + c0 = getBlob(node_proto, 6); + c0 = c0.reshape(1, c0.size[0] * c0.size[1]); + } else { + // initial_c attribute can be empty in case of keras2onnx producer. fill it with zeros + c0 = Mat::zeros(numDirs * numFeatures, numHidden, CV_32FC1); + } + + b = b.reshape(1, b.size[0]); Mat bx = b.colRange(0, b.cols / 2); Mat bh = b.colRange(b.cols / 2, b.cols); b = bx + bh; @@ -1036,8 +1058,7 @@ void ONNXImporter::parseLSTM(LayerParams& layerParams, const opencv_onnx::NodePr } Wx = Wx.reshape(1, Wx.size[0] * Wx.size[1]); Wh = Wh.reshape(1, Wh.size[0] * Wh.size[1]); - h0 = h0.reshape(1, h0.size[0] * h0.size[1]); - c0 = c0.reshape(1, c0.size[0] * c0.size[1]); + lstmParams.blobs.resize(5); lstmParams.blobs[0] = Wh; @@ -1045,6 +1066,9 @@ void ONNXImporter::parseLSTM(LayerParams& layerParams, const opencv_onnx::NodePr lstmParams.blobs[2] = b; lstmParams.blobs[3] = h0; lstmParams.blobs[4] = c0; + + // read direction attribute + lstmParams.set("reverse", lstmParams.get("direction", "") == "reverse"); lstmParams.set("bidirectional", lstmParams.get("direction", "") == "bidirectional"); node_proto.set_output(0, lstmParams.name); // set different name so output shapes will be registered on that name diff --git a/modules/dnn/test/test_onnx_importer.cpp b/modules/dnn/test/test_onnx_importer.cpp index 05f77730af..a446a37c79 100644 --- a/modules/dnn/test/test_onnx_importer.cpp +++ b/modules/dnn/test/test_onnx_importer.cpp @@ -665,6 +665,11 @@ TEST_P(Test_ONNX_layers, Split_EltwiseMax) testONNXModels("split_max"); } +TEST_P(Test_ONNX_layers, LSTM_Activations) +{ + testONNXModels("lstm_cntk_tanh", pb, 0, 0, false, false); +} + TEST_P(Test_ONNX_layers, LSTM) { testONNXModels("lstm", npy, 0, 0, false, false); From 917cd13ce24cd89770b78769d7bc9729277ff4c8 Mon Sep 17 00:00:00 2001 From: Iyad Ahmed Date: Fri, 13 Aug 2021 20:12:05 +0000 Subject: [PATCH 118/262] Merge pull request #20549 from iyadahmed:video_capture_timeout_set_get * VideoCapture add open/read timeout params to FFMPEG backend * Fix wrong enum name * Fix wrong enum name --- modules/videoio/include/opencv2/videoio.hpp | 2 ++ modules/videoio/src/cap_ffmpeg_impl.hpp | 25 +++++++++++++++++---- 2 files changed, 23 insertions(+), 4 deletions(-) diff --git a/modules/videoio/include/opencv2/videoio.hpp b/modules/videoio/include/opencv2/videoio.hpp index 348448bda7..16016e4b8e 100644 --- a/modules/videoio/include/opencv2/videoio.hpp +++ b/modules/videoio/include/opencv2/videoio.hpp @@ -186,6 +186,8 @@ enum VideoCaptureProperties { CAP_PROP_HW_ACCELERATION=50, //!< (**open-only**) Hardware acceleration type (see #VideoAccelerationType). Setting supported only via `params` parameter in cv::VideoCapture constructor / .open() method. Default value is backend-specific. CAP_PROP_HW_DEVICE =51, //!< (**open-only**) Hardware device index (select GPU if multiple available). Device enumeration is acceleration type specific. CAP_PROP_HW_ACCELERATION_USE_OPENCL=52, //!< (**open-only**) If non-zero, create new OpenCL context and bind it to current thread. The OpenCL context created with Video Acceleration context attached it (if not attached yet) for optimized GPU data copy between HW accelerated decoder and cv::UMat. + CAP_PROP_OPEN_TIMEOUT_MSEC=53, //!< (**open-only**) timeout in milliseconds for opening a video capture (applicable for FFmpeg back-end only) + CAP_PROP_READ_TIMEOUT_MSEC=54, //!< (**open-only**) timeout in milliseconds for reading from a video capture (applicable for FFmpeg back-end only) #ifndef CV_DOXYGEN CV__CAP_PROP_LATEST #endif diff --git a/modules/videoio/src/cap_ffmpeg_impl.hpp b/modules/videoio/src/cap_ffmpeg_impl.hpp index 1e73cb8fc8..9ec75501d0 100644 --- a/modules/videoio/src/cap_ffmpeg_impl.hpp +++ b/modules/videoio/src/cap_ffmpeg_impl.hpp @@ -183,8 +183,8 @@ extern "C" { #endif #if USE_AV_INTERRUPT_CALLBACK -#define LIBAVFORMAT_INTERRUPT_OPEN_TIMEOUT_MS 30000 -#define LIBAVFORMAT_INTERRUPT_READ_TIMEOUT_MS 30000 +#define LIBAVFORMAT_INTERRUPT_OPEN_DEFAULT_TIMEOUT_MS 30000 +#define LIBAVFORMAT_INTERRUPT_READ_DEFAULT_TIMEOUT_MS 30000 #ifdef _WIN32 // http://stackoverflow.com/questions/5404277/porting-clock-gettime-to-windows @@ -523,6 +523,8 @@ struct CvCapture_FFMPEG AVDictionary *dict; #if USE_AV_INTERRUPT_CALLBACK + int open_timeout; + int read_timeout; AVInterruptCallbackMetadata interrupt_metadata; #endif @@ -569,6 +571,11 @@ void CvCapture_FFMPEG::init() #endif dict = NULL; +#if USE_AV_INTERRUPT_CALLBACK + open_timeout = LIBAVFORMAT_INTERRUPT_OPEN_DEFAULT_TIMEOUT_MS; + read_timeout = LIBAVFORMAT_INTERRUPT_READ_DEFAULT_TIMEOUT_MS; +#endif + rawMode = false; rawModeInitialized = false; memset(&packet_filtered, 0, sizeof(packet_filtered)); @@ -928,6 +935,16 @@ bool CvCapture_FFMPEG::open(const char* _filename, const VideoCaptureParameters& if (params.has(CAP_PROP_HW_ACCELERATION_USE_OPENCL)) { use_opencl = params.get(CAP_PROP_HW_ACCELERATION_USE_OPENCL); } +#if USE_AV_INTERRUPT_CALLBACK + if (params.has(CAP_PROP_OPEN_TIMEOUT_MSEC)) + { + open_timeout = params.get(CAP_PROP_OPEN_TIMEOUT_MSEC); + } + if (params.has(CAP_PROP_READ_TIMEOUT_MSEC)) + { + read_timeout = params.get(CAP_PROP_READ_TIMEOUT_MSEC); + } +#endif if (params.warnUnusedParameters()) { CV_LOG_ERROR(NULL, "VIDEOIO/FFMPEG: unsupported parameters in .open(), see logger INFO channel for details. Bailout"); @@ -937,7 +954,7 @@ bool CvCapture_FFMPEG::open(const char* _filename, const VideoCaptureParameters& #if USE_AV_INTERRUPT_CALLBACK /* interrupt callback */ - interrupt_metadata.timeout_after_ms = LIBAVFORMAT_INTERRUPT_OPEN_TIMEOUT_MS; + interrupt_metadata.timeout_after_ms = open_timeout; get_monotonic_time(&interrupt_metadata.value); ic = avformat_alloc_context(); @@ -1282,7 +1299,7 @@ bool CvCapture_FFMPEG::grabFrame() #if USE_AV_INTERRUPT_CALLBACK // activate interrupt callback get_monotonic_time(&interrupt_metadata.value); - interrupt_metadata.timeout_after_ms = LIBAVFORMAT_INTERRUPT_READ_TIMEOUT_MS; + interrupt_metadata.timeout_after_ms = read_timeout; #endif #if USE_AV_SEND_FRAME_API From 9ef41f68fbec10b41948f10d86a2f8f8b8cff56a Mon Sep 17 00:00:00 2001 From: Smirnov Egor Date: Mon, 16 Aug 2021 15:44:54 +0300 Subject: [PATCH 119/262] fix Split partial sum --- modules/dnn/src/onnx/onnx_importer.cpp | 2 +- modules/dnn/test/test_onnx_importer.cpp | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/modules/dnn/src/onnx/onnx_importer.cpp b/modules/dnn/src/onnx/onnx_importer.cpp index 32b56278bd..6da2c5edf6 100644 --- a/modules/dnn/src/onnx/onnx_importer.cpp +++ b/modules/dnn/src/onnx/onnx_importer.cpp @@ -851,7 +851,7 @@ void ONNXImporter::parseSplit(LayerParams& layerParams, const opencv_onnx::NodeP std::vector slicePoints(numSplits - 1, splits.get(0)); for (int i = 1; i < splits.size() - 1; ++i) { - slicePoints[i] = slicePoints[i - 1] + splits.get(i - 1); + slicePoints[i] = slicePoints[i - 1] + splits.get(i); } layerParams.set("slice_point", DictValue::arrayInt(&slicePoints[0], slicePoints.size())); } diff --git a/modules/dnn/test/test_onnx_importer.cpp b/modules/dnn/test/test_onnx_importer.cpp index a446a37c79..983f72d6d6 100644 --- a/modules/dnn/test/test_onnx_importer.cpp +++ b/modules/dnn/test/test_onnx_importer.cpp @@ -617,6 +617,7 @@ TEST_P(Test_ONNX_layers, Split) testONNXModels("split_2"); testONNXModels("split_3"); testONNXModels("split_4"); + testONNXModels("split_sizes"); } TEST_P(Test_ONNX_layers, Slice) From 8dcec034ed6b52b2074573a56573883698e446b0 Mon Sep 17 00:00:00 2001 From: zyp Date: Mon, 16 Aug 2021 18:20:10 +0200 Subject: [PATCH 120/262] Merge pull request #18694 from zyp:gstreamer_gray16 * videoio/gstreamer: Add support for GRAY16_LE. * videoio/gstreamer: added BGRA/BGRx support Co-authored-by: Maksim Shabunin --- modules/videoio/src/cap_gstreamer.cpp | 23 +++++++++++++++++++++-- modules/videoio/test/test_gstreamer.cpp | 12 ++++++++++++ 2 files changed, 33 insertions(+), 2 deletions(-) diff --git a/modules/videoio/src/cap_gstreamer.cpp b/modules/videoio/src/cap_gstreamer.cpp index 60ecf6611a..e040a22cb0 100644 --- a/modules/videoio/src/cap_gstreamer.cpp +++ b/modules/videoio/src/cap_gstreamer.cpp @@ -475,8 +475,9 @@ bool GStreamerCapture::retrieveFrame(int, OutputArray dst) // video/x-raw, format=I420 -> 8bit, 1 channel (height is 1.5x larger than true height) // video/x-bayer -> 8bit, 1 channel // image/jpeg -> 8bit, mjpeg: buffer_size x 1 x 1 + // video/x-raw, format=GRAY16_LE (BE) -> 16 bit, 1 channel + // video/x-raw, format={BGRA, RGBA, BGRx, RGBx} -> 8bit, 4 channels // bayer data is never decoded, the user is responsible for that - // everything is 8 bit, so we just test the caps for bit depth Size sz = Size(frame_width, frame_height); guint n_planes = GST_VIDEO_INFO_N_PLANES(&info); if (name == "video/x-raw") @@ -507,6 +508,24 @@ bool GStreamerCapture::retrieveFrame(int, OutputArray dst) src.copyTo(dst); return true; } + else if (format == "GRAY16_LE" || format == "GRAY16_BE") + { + CV_CheckEQ((int)n_planes, 1, ""); + size_t step = GST_VIDEO_INFO_PLANE_STRIDE(&info, 0); + CV_CheckGE(step, (size_t)frame_width, ""); + Mat src(sz, CV_16UC1, map_info.data + GST_VIDEO_INFO_PLANE_OFFSET(&info, 0), step); + src.copyTo(dst); + return true; + } + else if (format == "BGRA" || format == "RGBA" || format == "BGRX" || format == "RGBX") + { + CV_CheckEQ((int)n_planes, 1, ""); + size_t step = GST_VIDEO_INFO_PLANE_STRIDE(&info, 0); + CV_CheckGE(step, (size_t)frame_width, ""); + Mat src(sz, CV_8UC4, map_info.data + GST_VIDEO_INFO_PLANE_OFFSET(&info, 0), step); + src.copyTo(dst); + return true; + } else if (format == "UYVY" || format == "YUY2" || format == "YVYU") { CV_CheckEQ((int)n_planes, 1, ""); @@ -1008,7 +1027,7 @@ bool GStreamerCapture::open(const String &filename_, const cv::VideoCaptureParam sink_pad.attach(gst_element_get_static_pad(sink, "sink")); peer_caps.attach(gst_pad_peer_query_caps(sink_pad, NULL)); if (!gst_caps_can_intersect(caps, peer_caps)) { - caps.attach(gst_caps_from_string("video/x-raw, format=(string){UYVY,YUY2,YVYU,NV12,NV21,YV12,I420}")); + caps.attach(gst_caps_from_string("video/x-raw, format=(string){UYVY,YUY2,YVYU,NV12,NV21,YV12,I420,BGRA,RGBA,BGRx,RGBx,GRAY16_LE,GRAY16_BE}")); CV_Assert(caps); } } diff --git a/modules/videoio/test/test_gstreamer.cpp b/modules/videoio/test/test_gstreamer.cpp index ca100367b1..207f6de50b 100644 --- a/modules/videoio/test/test_gstreamer.cpp +++ b/modules/videoio/test/test_gstreamer.cpp @@ -35,6 +35,10 @@ TEST_P(videoio_gstreamer, read_check) cvtColor(decode_frame, rgb_frame, convertToRGB); cvtColor(rgb_frame, gray_frame, COLOR_RGB2GRAY); + if (gray_frame.depth() == CV_16U) + { + gray_frame.convertTo(gray_frame, CV_8U, 255.0/65535); + } vector circles; HoughCircles(gray_frame, circles, HOUGH_GRADIENT, 1, gray_frame.rows/16, 100, 30, 1, 30 ); @@ -58,6 +62,10 @@ TEST_P(videoio_gstreamer, read_check) static const Param test_data[] = { make_tuple("video/x-raw, format=BGR" , Size(640, 480), Size(640, 480), COLOR_BGR2RGB), + make_tuple("video/x-raw, format=BGRA" , Size(640, 480), Size(640, 480), COLOR_BGRA2RGB), + make_tuple("video/x-raw, format=RGBA" , Size(640, 480), Size(640, 480), COLOR_RGBA2RGB), + make_tuple("video/x-raw, format=BGRx" , Size(640, 480), Size(640, 480), COLOR_BGRA2RGB), + make_tuple("video/x-raw, format=RGBx" , Size(640, 480), Size(640, 480), COLOR_RGBA2RGB), make_tuple("video/x-raw, format=GRAY8", Size(640, 480), Size(640, 480), COLOR_GRAY2RGB), make_tuple("video/x-raw, format=UYVY" , Size(640, 480), Size(640, 480), COLOR_YUV2RGB_UYVY), make_tuple("video/x-raw, format=YUY2" , Size(640, 480), Size(640, 480), COLOR_YUV2RGB_YUY2), @@ -76,6 +84,10 @@ static const Param test_data[] = { make_tuple("video/x-raw, format=NV21" , Size(322, 242), Size(322, 363), COLOR_YUV2RGB_NV21), make_tuple("video/x-raw, format=YV12" , Size(322, 242), Size(322, 363), COLOR_YUV2RGB_YV12), make_tuple("video/x-raw, format=I420" , Size(322, 242), Size(322, 363), COLOR_YUV2RGB_I420), + + // 16 bit + make_tuple("video/x-raw, format=GRAY16_LE", Size(640, 480), Size(640, 480), COLOR_GRAY2RGB), + make_tuple("video/x-raw, format=GRAY16_BE", Size(640, 480), Size(640, 480), COLOR_GRAY2RGB), }; INSTANTIATE_TEST_CASE_P(videoio, videoio_gstreamer, testing::ValuesIn(test_data)); From a50dec88d5e39a24438f9215a2fa4ee4cc6ae3ab Mon Sep 17 00:00:00 2001 From: rogday Date: Tue, 17 Aug 2021 18:23:27 +0300 Subject: [PATCH 121/262] Merge pull request #20547 from rogday:gdb_pretty_printer * add gdb rpetty printer for cv::Mat * address review comments --- .../linux_gcc_cmake/linux_gcc_cmake.markdown | 2 +- .../images/example.png | Bin 0 -> 35727 bytes .../linux_gdb_pretty_printer.markdown | 38 ++++ .../linux_install/linux_install.markdown | 2 +- .../table_of_content_introduction.markdown | 1 + samples/gdb/gdbinit | 23 ++ samples/gdb/mat_pretty_printer.py | 212 ++++++++++++++++++ 7 files changed, 276 insertions(+), 2 deletions(-) create mode 100644 doc/tutorials/introduction/linux_gdb_pretty_printer/images/example.png create mode 100644 doc/tutorials/introduction/linux_gdb_pretty_printer/linux_gdb_pretty_printer.markdown create mode 100644 samples/gdb/gdbinit create mode 100644 samples/gdb/mat_pretty_printer.py diff --git a/doc/tutorials/introduction/linux_gcc_cmake/linux_gcc_cmake.markdown b/doc/tutorials/introduction/linux_gcc_cmake/linux_gcc_cmake.markdown index eb59fea209..ee3f1eb7f9 100644 --- a/doc/tutorials/introduction/linux_gcc_cmake/linux_gcc_cmake.markdown +++ b/doc/tutorials/introduction/linux_gcc_cmake/linux_gcc_cmake.markdown @@ -1,7 +1,7 @@ Using OpenCV with gcc and CMake {#tutorial_linux_gcc_cmake} =============================== -@prev_tutorial{tutorial_linux_install} +@prev_tutorial{tutorial_linux_gdb_pretty_printer} @next_tutorial{tutorial_linux_eclipse} | | | diff --git a/doc/tutorials/introduction/linux_gdb_pretty_printer/images/example.png b/doc/tutorials/introduction/linux_gdb_pretty_printer/images/example.png new file mode 100644 index 0000000000000000000000000000000000000000..0ec673dcc21d74eb626b3f9b7a627f029c781873 GIT binary patch literal 35727 zcmbrlW0+-Iuq~LCwr$&XW~FUb+N`u~+qP}nwr$(4&N}Da+kN|~-`DYD`kp5u7SFF0RZp;#D#t;xnx{qIy-1Cdmi&y zA4v-j=OA?g0fX0s3L^`L){(2%8IdJnH=7a-N1+TmW$a(4g=x0RE1jVmkLa#J21nl^}d$n#8@U_4Y~&IuYU z&FwrjQd8%U)UZJx%_>wfDK{JV=m3}E(zWPg(3PSt-5M7(*wS!Lix?}Xf`Du|6>_sO zM1Rht1kHd0+a4+y3on~6;&EQA@@ug*fnn>OmQRDU;-fmA^QqGy4JRzLy4y-$0BwKw z)f;G(6efmaEx)dwQcAfvu&R|63;8LbT}`8{zZwrHtQLJ9spzUa(#~^S#m6K9kZ)r`I`(cn zT<}P^MNX8ej`;3ki#85SCv0BrID&+oN{H+=MEntxO~GV{=;FwtWP=PQcp{aOu|T1b-QMq-|)kGaV3}-4{ z!`Om~kQl4^1W1A~AtxwUWCo;|n7F*tQo!qpQ)sta|G+`UE-liZYJ;W2f)nF6a?;$K zT%>XAzbo&C)xb9-E$Za2ZzToJL`~*gwudl`YF4-$1#Qz_lADiIW>eWYSSF9zf?Da; zN2amEk;*sbk%w4+ACz0Gp@&Q!aO^+dw-&)aJ7c{%XZKc#n{LWw^zh! zA?I!p&t9dTb0xN?_ZH~wtnB?5!OE(GOF|hS;whQvbo?U%xCSs~GbL!jR=WXg%I%hgKMR zik_mTg*P(zleb%%fLZ@=Fm(K>vbk5N+&0$-Fl4krnqToJvPL*A8m1$NJacRyl zT{9k{`#S2cb@LhiSa3qy-1!h^>wxxpsSeW!8`Zif|6n8i0dNXkhM{cl1#$YLStKHn zGK`P`X;y#A`ww*>?*^KlVxl+?&Ut$Z zVu7V-nyiZu*F+5$Dj_4qNuNZKG3|4nD+qQ!tEE8kIrH6>l4FUo=-nv^*&`rD+Ea

Q(%%|WTqhQs~!=|nUv!kcA~{jVq#P>iWFDX^L^Gw>VMb4%l#d#%X-bh74JL( z4j+4>3;Zk0U_oq^|EX980&kuHrWgYqq8ezT62`}_ky5YSVHE+X1k^c)>yVW4DRZMM zX7xKUxP*%@^8BGo!7FeOjDsa?U9Q}ri&`#ZYBn?pu8m=wZwm7{_nd8o=uxjU!4^qx zAW@JcRyZXCJ*Kfxu6g85th>NLn{6OVsAQCFo@E+9^@ zh{;`r<`v-*A`(cBI509f{X`h2aM3mtT#^v^DLq|^9WXzGjXv>!by9#UJ~j!f@uB9+ z-ETz&==3rJX7a&T_a!8hlvO8y9zNXz0Y#sqlvpAndwnTFL?A;LBzURNS{XXqvc-wp z)#Y;bRD@g~p$7>q*WeB7REKbW zWJ+hUIa16TEa(3QzaJ1`KlYDc4z>U8M6#T(UheOt{J4BIHjqdd^w?&sLu(6p1C*D* z!5$hKf4P=!+K6j9G-|j|kqFrfC-`qN@&>0?<0ZH_3kJAjqBtP%Y-XxtS){IS;n&(# zqwO+4>2Q(XPPqlrI6{&VZ^VnV8R0|LLJ*OCAY9F3ursn+Q_3#xK1c%RaeDm9=>Cxp z9lO$Z$`?J~vzOW=hAg&34-^pLaa}Xijzze!jNMxi4<1NzA@e$C&SjDuh*t=@s*1X| znHI2aQE-py?f$)nzJq#}m^2<2?NnrE)xgSgOm^k|=*R0SmyO1C1^l1$FLLY7go)~^ zXYr|jce6l9YRY+3*3V`kH`|a-VR=jEh-VQZDve#7M+5~yH_)Uo2&8`&C#-Nt2^Rmm zD^)c+i_>qII+{ac4f{&D)h?D1nfi%v(FuppPME(VxM3BPI)?aXEbtXSc0&>WUe1rN z24p93d~DqW^H0Cvz|A;)NF@I)c>)ax`Okj|2>s`^1bM)0IGNgL$K*%p(F8QvclzrS zRae%|#5iphw=q@99N$gZK-p+me!iS`hW-|Kh_0?wM*8J49kuBpwLCfu_smc?qyWq$ zW{j1%c!+Fxg3jR?aYf?HxE$N;MRqRziOtO;bpX1S0v`0>F&6&e0;;wANuHyDF{(0o z6@&~Lm7T}uazEZ`TOilinF_erZzGGwQ+0!QTGx7g8eO~oiS7O6kAD50{~U9_)P29W zJZxRk*vyxQ2hxn!92nB}bwWStM$P#teE|;?X|&iO$u^|()MA~6q8gpGF~%to$L-Jr zs|QD&tRg^nZe4>mCgmkff}zOM5UKSkW;H>c+f)aTu1tz*!?ZCkoXsV$mtI|{VE-%< z76DZ)^N4%=!f%6OR+ZoBhSA@#P3C5ST6idYP)KGVL~!EFu(CN^!q?_;7(S++T5i@1 zYoS756xh0=FnO{F?n?pYx%s&Q1m>vq|yg7%06qh|oA*%KaN!6C_Kan|@ub)@KgQS;m3y zVO$@bSiEyEdA*VeKDP4>gR!FTEzC~adq3-BbR{^nKH=nA*m-4>$$y@Rx_B$&ZGYWS zL06UGf%s{JpBG^$C@Q+@UVnCcj8OptHor|znRR)eka$H5O}^7v7%GZ$z{$$qCG)UN zhFJIn9vJfce8=~n(m18(fe@dYVTLiNi#*VMo=_GSm|tqzu1HXt7=kHG@icgN1ZFr! zmfu_dtgsD@N$ggwd0bxMP@AnL-iBnYL_pJ9l zsb#$w8xtm;T5McTOB9uu=oj+_5YCp`Y9nN|oUy%gVY4@QyYoaOVm{Qj!H?X#pjM#P zeI72S5F!W8l>-l%eCH01s*t|k9}T9eeF-w-@dj?!>Cea#b(IDhEQYl&>5Nu$@yycB`jWJQKxZX`!X} z3Bpyij3C=)^OzShxD`)TdPtZEih{zt%f~I*%PyR^jk19G<-9{Ve|sw6nT$ZN3>d>B zDi_$0!oX5wsuaw_J`@FT9%)#a1-ba6-H{<*Ip3SCsFN83MJ>cs*7*PikjUqWlVwf= zb9gT*m~AGiYL4iVU=LGofF}h(yjU8RVfB$w&Lg&9m-rDjXTKNDGxJeQa&Es>chdBy z`C8@P1$&gecGIlS#MPUnokCmcgFv5WURg=~wWEihODCk_76pEgv&0~?<7;G`-Stsu zFy$;<$5gk+A9AGofFq0Qli6~3`{sG9^1QqzL_tX^s0LFv7{}Hf&W$M{f#0ep9$)DW zQ}4DmO9n^w(TnxGf0ChLeyq_4d)?_^<6I8&yzj_HjeqnaotsfF&u?*F{|@0iWp_;b zA~BaN@B}Cd^Yqxeog{v5H^ri#jcIHtIDD7KZ$W9bBBX=|zw5{Q@)^eS-YW}SSR51* zxBTrD<4x{x6Nn~be=_$h0h+zq93G54?E26U9_-9LaeSzP%e4~j&Z?tr#gbh?Wvmev zt2Q#8o2AkKRIK1YRUs=|q;4=m zJm}2-7XtH6%%AbpgLqCbgdyDp*eiLrbIcd zq+ps+AJp4@^s`(87xSm#-m#}Ca9AAiRwJe4gBYt>B$(G!Xbhk6KVX7eqaZB9en0!f zMnFG19H^4(KkTI&HYGbX41iIM!Uy$D#l(`2qLD7+6ES#iv8#i~ek3+6d&yi6Xo{q+ zWv-7HrzB@-kkcEVg-?Pv+Mrc-%9l}^mn3MIWj}}0RyGBBY;>%qr33)##yTrQh{~Zg z;1mVDp%vM--xBGuru!Hj?;gkMzWdbFXV1U8n{q5luWd~aAjpqD=JVTF*l7v+R29bj z;=(i;CQG3m)iVIaPJ?2TsGGHnvbEe0rgq&aU7T*FeW@`_zTF+VETFW$ zdCi4K%b-n|gFaK-?IHHb3Yt)w!77mr6x0SteZ&~u!Pt?~w}QBNxEtQA7oqf;!iam7 zs*Rk)u(6rS&dL(cr|lyMMvI}CoHEHD?XBPRj3{S3w>)=ozSOdm;F(k;6R{zi$)XzB z7*dqomUPl2(!zE*!g%7^+l-$%<|?*mgf80~^R9w8wdeHP@r3a( ziHQd-Yc<1jB^6Vmb`pb@4qS)y6A0n_J3Znd1!_>sLTlSweHC#0gMmuVu| ztx!6dHEOw8CP9PYa*(r;!;Bs0g&R8MQ4P@)W2Fu zm#z}GX&^c+dlD@3D+sYVKB1>e&w<~8l}Vb!e5gt3onp6Er?0x6T%DKX<*fpCS$}oj z+D#!8brzQ$e?t(t`x-Cnhwn^R5Vda zhz|{4${hrh7>dgfy6d2-weiWxtIYHCY4-OQHQ_Cyi8Va^dyY$2xnn!x!yg2qY45P{ z-61?}btfNZd4IX7x`Kpx`t8>eHXRn-DL*3vG-Y5@DuHb`4!6`{#kp?3>DPU3dOX;0 zX<^P-lQq11Wzo_RdzYj$6vwmPt?JhI7{V7wrTxw^S7BY5*n#mi%;!+d>mC;F%iZf0 ze~OOM`dX*&rD*iIY(3pcy9GdAVH{P+%*fC&?U{ORdow;eJ>6A(Y=1RY=D{1oJ}kc~ zNL;U9i7G|)y4^&G2(O-gAkP|QPbT46%J8UzhLqXPpl4E4QWsNj2|y}tyQQ6>ep6q^ zd3czvn!;+zPidAP)1;G6&Bz9j;Ac$MuhZ4#F8TkwbP@WqbPlxS(W=6rP|! z&SiFLr<*2=c za-yG+SB##(y%IV3Q1=GT?!f`iUQYpt7sq{BW6#OoP)pR)j578X6_oBYA9!5-w6$vG z>Ts)4om$v(Ra*qoXD>J3DBzT1FhMRGfB=toQ0%DyrYA-s4Mf1|IZSaV zc1nels8MM-G5`7`<93I5SxyyvA8gHrRQntnWBuiX?3xG-e`PqFkF&ABMA~G%&!8O& zLi2?{6fK-Zl1TQ$Yu`^2={t~5!P+VVIB zEo-b!?+Sx7!Q3E!YpOgy^{9*Xu*?PHO5?QnP{4ts@uFgy@XWM$okt^w zA-+N>^z^It#eV}OAoW2crzawiiVTGq!h;tNlOjc`CXJtUkZVXc+F!mp4Dlq7DsQh- z#@8mZvXOq@xVR5!%&_T3}$A2@=LQYzaXpjqIwv}BwXmunU2Zr1`s2TZ(4 zARuPWkzUG_+c>-#2Hw^ll`uG4OZd$_d4%WKyUiUdGrSqPP`$pixKdMvnKv2}oxhIW zclo};k*WfhBugDTp8cjtl3X-D%Z+oQ{clkjrndF}4IIu0+Iyy`NK;dG$Yo*)mBz`Gl4lq0C z?(zQNs%1hD#>gn?HqSt@sKLf|hIBo%9}!vlMM!3TmXHSK_7hw15X5hq9p8rVk8ry5 z%T>fo9GN7ocT`x$T!Grz6kT^zEp0)mgfomB#}%Q=HbTfsr#B!U%O3E6{?KaTaFP6+ z<>}W_`XV%W5>B{T{A-uK`_iJ7_BSD`k!xv z4ZW%y-AUImCDFCKcm|$44NV*`k+M8PTUv=3Z?4OSBg_^U(H z$V5Dyw?N~ni@d%&Q~Pb9B^D$(RVa!EMA#NiYPwptV2`nFrO? zA!rKdnV)u-m{=wiZ6TMeNUj`a4y&?EyK)?K%XK{SemrC#+d^C>NN-DvA(I2*`37Du zG{D1et{e^VVBw4}aH6=*r=A|xmLSKiy#dl-PrCHGPt>|J4Dtwk2AiLTM{N4z%yW1l z{KQh|^q4T3r3*>=+QF%JW|ov>X67MPtZY9(9#Tn|6Nj=*JM;p8q=xayI50wINWvB4 z;_#33-{#7V@SEE*B4Uu*jq@9-(BDga3wfA-`S8QXz)-ImgwF@`Gi~|Ds;mv6L4}d` zqQgt~X#4vC*82{Q^H4FJpPf@MOU-$e0S%)llgdr~$~xb3s(#v3=#Y{TY+4LMK-(3{ z&ZY3LFiZ@!-;AhqTaJFa{#KxJ)8Mf@$ zZiUows-Ob}aiV}tG6-FWd=@2e={f%-9qoyFYU7rZA0iD|#udN08m!WBhj!!%q=Dbs zI|Jms4XiO`Hx3B>FqKarFBoBj_dP+5pT|6GeJy}_B4#gSi!MM`aDX)M?2MRNES(nC z=&QpTCYDi+_?W=1gPBVGj5Wk^Wlc?4k2GnDhFMKqFc!t&-js>{)z>shvVg|L!9b6* zolj64n+05UJht;fGC$z);ujOE|MS#NyOxqh&z%T2gEw-l!*K1ZNqI=K?BO}xG;~B)f+XuFLIjiln|D?mm z3w4$eT55#dQHKBJGxP0E6|voS)@a%dx}Cj581_5I!IjpvX43Kbp1-v()x>*<;`vu| zk=pOSCs+76XmV#rwIcX26?EKh6IWp{%Q7eR^&Rf@Kl8MWL>4;QOy3W)E-;dFN&fCk2@-J z@JKgsVU_}=aT=SYtT0bqcpViC$4F&0Qoj`Yu+jM9)648l9kFB)VI$bH6A-sfN=hJ2 z>`^^?agJ95H1tSx_2#auhnVzIIpj=&(1{Li%RA;ypegm(M6on8Q1=UD=7d@!MZ|m= zyxfui%euU_cJzz|#QJ1>f31FGOX@tc@Yvm347gYVA`Ds@rRDhem zs2wL+Bzn;% z$d{cd=eyAD(!#?%_$r?;0*-jc!@(DA@1Wy0x<7ag*&LHQRi2g`0AKTT^R;L9F=4Ck zv6m^qpf&IM=y8?_ELJ&lJ9{&uTawqxT8cErI$%x)6pxPgb-`K>51fcu^g916fY4@T z=A6BgV-Rl$yg9hr`_MygT33+c^E5Ca(&1On5X^hl=D8Zo%vfXjKRpyTSdA8_EaAfw}B= zbo7HY4C&8H#IZ#nW_aei;WR)KlPx4`S?50|76(J`^~xmfN9bw^Ead&ZY&YH#?-4PI zAN@kl>bND!?P5t1OAz(e&Pik%l&??$PrdhiG362P^S!E}&&{1z#FDbmk^ zgyTDXe@5Y+_h*Z7j7dzh)H>0aZL?2c?A{Ic3+1+j4mP> z?>zoF$QEs`b+_PIOn25}F%9^{B@N=lxS@IDJ8AC;oq8v)dbB@M`g8Wjc?`0Pf=4-$ z%S?iSK=^|_c>2C?8rJS^&Ta|Xk`+S1eD(Rf5$e7j_CC@Y8~+vuMP>E-P=ys(bG-J{ zXRBWaM7mz9O_r^liJ1BiEr8I}%}Z&mYx$BPZl`|!OnPWy*l&$d=geKXXXOfs4v?B8 zOMR2rY{g*C*0RPQe(UKCkXNe-y*IV-4vaD+nV+!H9NJ6Toy3zjauir3a}y2|IdW zO&9A>gN=uj3eqHpJ#%@1-jJJja9}oYf;=nWAc3#FT~by$M;(lVdb+Ir0~EHnmU45R z?!>a)8<`tA(O<8(G~M}`VfD?CxUF${YvvCoQS+WunS#{t#1V;(jVuSl+z!o8wMQ%G z7NM=N&wqNkC^XKzUa8Hl=^JPebaRV*x)`C=H%PmrjiWWG z*&{HQi>f>p%%lsLd1Z8Htb`=U7|MJd{0u5 zr$_6`D4GuYpzmsJU5z>0^wJC3f}(bNu(hi6JsDlrPNNYIB4add9RW>!#CAEIfqPpZ zF-u=fb2eS)qOhbN*duXa^4;l>ol~I(y>LG?XZuS$5)d?DY^K)eyyM`77*l@q3kS=> zAfFbtkr>EKZLT>Jo<_2ScwDLyPe8wy44l>h)g_YDM0K|=5*Xe(^Zn}bXt2(8uI*1Q zL<*`_jh7;kFy{dhW^osSD9^{uOcG4U215{&x6mVkB@VTd3( z6?@aTQ4wi-+*1lv=wIm71}Rlp@Y`&L`i zi8wvpS3>bpuNrx@*m9MQVR?9w4}cUy6*Bw3DY4T-Qps#n-Dc=UX%Zg|J(4>{D=S3S=HqiCURmE|#^|CmXgD4A{P7uH062Oc&Mwj7uKCZAGs zmy+TI1J8EapU_pE1R1Y2bflWP)1z^%iY0;GE(4slr{@m?58xb`jvVGhc(E)|Q=t74 z#Oq;{CdWY2F**ohESXIt9-dol=vd83;^s;jO0X}jc83khd5G4=s@W105ij;nfoK}2 z(Z}OobOkX$e6@LPYZ6sZa-wxWo$fLp9X(o1BE0V=NMSf%s#3<*C6RMY#}EhZ)YPr@wk9fC2i&h5Qd5iU@F>gw;z?gZGQ>xucVs5^-i%qOgn%Vm1 zDWYe;f4u-Ozd?a_2$R}sVse{M?oZklf8EBo=CdJ9|GFj1XF`?v>agV9E-$zJFRWf9 z(cyDD^Hufi@Qah8_0A44sL1tOU1FL^=ppDjMUoFBgZe$wB2V91Tke$6DG3Q`X>R7f zG)_dT<3+`ID?>SOyd=dmsEIq5OlAc2_()y0udn#XB$gUl$>`hb^8-AXNVNcB7=KY$j9Pl5xnDxB7RH09 z;i-N9d7Ltp!T^KBkktUR&Y5Qr@A8+&eE1;nz;{XYd!0D#H6h3cIbfYW^9G@Y6Q0tg zSIUkKJk5ZqPVlj?_SF_eUQpEB2@q#bvcJB^k{pQ9hv4K~6MOSCyrgUEIV`y^van96 zIU=n8CcU$pJE_Yy?#s2)t}=2j0&$mt>kknRi!A&@_~(xZQlg1y z%eYANM|g2&X>KVDLy~f-jC11&G&7rQfOoR`)F{TJOZG+g7Ft_uxIX`L`HpdPE z8AOsWmA|!ucqN89W;~uv0HVK&eumyAwTF_8o!Jl^V6$WRvbYL2M>!&J6j*#Q+Nz zLEl;*F)Z)$y7Gb;uL7X>QR5jP+O%{V>8{|aXNl;pM7WwEwX)R*r)lqnM%Rt~bnkm6 zz*yY=+J=Td;LE_4ZLTJ&Z5ZNhUtzoHs(_GP*}!;m%T#(cI!RlI5V%}~mu-Gus?OtXlSRtY2Le|z#o=MY4CGbo5wRscI zKOCZ6Lh&o%!BBF$y`e9^+FdMlB2>r<4u#A-Py6D&HON_m>keqwUz6yszw=)RIq&X{ z$ECqUb*u|tm#_Cr|0MONMllo-A{HTI&2Fi3aN5i};iG)Zr!0nO=<=4FP_;dfMfvDn zBwu2@Lb7G4|Bzf^@bq6}Ki-F)32Z4V`Y0>ffO>Qs%`ti3m08$?RGo2B=jS_2YS%|c z%(d~xXYBrzsdT)??aaz}Al#BS9HJDym2!Vwd^Pa!B&-DlLOOPRGmVj5K)$tcq*amD zo8THS;|b9E?Qv%suw72Id)o&1FveW$E_44F4q=5vKW=la-j%R1AsTM+iVT^*MIcjg z8yVKR{<2{E;^9fqE#D2>i8+Y$VCymAZ^0@}`1e*TKb^11r9QEzGLtCE{fhcYhXC_x z`_MSUv(t7^szm0_;mRxFsLnfR#342dkhSZ?=}u>@6(ffE%um+81ZfAwh=F-a>Y$SS zK5)-%?Vi9R{d}8)sypZ9oApB5g25G}eaz=oaC`NRwQ9TV=3t)Uv;0$b7~GyIXdW$l zf4l4Y$X5%-a!o5(9%eV&VUE$%FjSoXug0q~#%a?%+ok~uco~@rWxXlxnzKLmilt(r zW2r#dX1WMEC_1x%?)G=)i>O*oQnh|i|s-D#xYowO^3+@1W(^_ewJQ4ndqb-a7 zU(jfh(9-DuRGkEIrd1%3N|iDXXa#D2WT~V~Us9}t;Mupv%x^tZ!^<(tYCFDMqPxjJ zpSr_j*owF4{9r%$9bH`zLUp=Qw;$W>$;t~%r@`UwJlM^4TS-G@@8~hzut{EBOXg=# zRO~$24a#LsFLzJxd_M$EW1SN;GV5i5$hrO>bv&c>bP72h=x2x7eMq|aX)&w&Pws}! zunq?(U^4cHPHTG?{nj>W%hw(gm*O`@@B7X95XaoV_zq*&6o5Zie90j8kxgEC3zBt$ zK#u`4#yaf0m)~I3?W=~6srP}ceEZ=*Wl25`Ij6cYbJr)pGRE4knY2ev1IyEwcx*{E zCSQlMP}2<$&#fU;Y1viV`;^4X^@3V3o+`0>Zg=pzqnoG0qv+*k;CYWfGm;NeO#snPdvNOCRIrCuD`Px-f3<8R@pkQ<4Q&WV4fKJ{Y3Brz! z1uxULczm2aJ}dgF-#)8KTNe>B`xPPgM_B)hB4QN;{a)fby(2X67b@l^X4wcN)(SL` zZU-3{|GvqWBhsgAIGi{G3mOO1D9XZ_jTF3$W~C}A4*Q6Je~E`Gi0|+Q*^otiW~)n0 zuGe|=P7>e=oxVw)i<+wS5r z-0z`nby(|f&%lr9+PdgbSC#UcLPVc{F4*nYt3-9MnXcqKZAg{&=lk{f;@0DVL-4#vEZhS;sG*mGaWr;5=h=+m>uc&F0-jJM8nJa#IvwltV!?(qJNCT@Mgx#@V`_6>%Yj~S`gVA5)j zi_4zF@dD;cUsLZ#T*RurroOhuKUkTFQ;O9*?Osf_R(Gb)1*hNPF@n)i{7^iFF}#Fw zoQ!HUqyX4P7J6>+7(b_qbkaN(MJ*8SX1n{PXQEIIYWi!D?xcEUJpO$tK32b13|{qG zUEA{zo=5SPsvg8Ma&!QW*__BQ+)+gL5Vl1mJ<&18vmXdP6ti!@@Ykg z7?(t5^PTARceDDGySv&7*KR%qZPHS>UaqYh893RxZN$Nm7&fM~5}czV^&CUBwq&*J z@_41KSTAo+`i81}+ig4jaau#cSk3We6VOxt8@bCw9UvvF9d7pKR*#Af-yg4~Kw0ae z5yYYR5g#uW*JA_eLQ}gC9S3LvlEWKQGa?=7xtAFYAodygj6~7;*(vV^=lvcrPdyM> z%dK8_e_Ah&jq!9noX0^F?j~C#qANC6C%2*%U1I)sQ@mPvE4~ES-B2zmi_;YvjI~yk6o?2pQU_S&Ww;l0=Od@aE>$B z0ur=)moD31d`L$4zPgd5Xr`t#yEMzJ3Qtx?$pW_X>3t`5VooTX}?(n8V7-PM}2q97vhWnXlv!(FS>YZ`Bai z?G#_j+>O<04iDsVIbW)Hh7vbpCf=9O;xDg{@`Kk;=X|`layOsv9b5@s1veu&G8w1krt-*=0CQ}hVd8!txZ6qa%hjQ#1+5_>TD-}L9rMF&NxL&Wx zj=1eRI}Wyc>mwIEMp#EwHS5j%V7=D3e6U^yZBjad(?&4N53AQYnVyFYkon|>uehP6 zpMZbNaDRHNGO3Q@o3un*D<9WR&J5ezN3*bxe$Cdp4o&dp`($1_K2M)d4^GSMPD0UO zue7IbTaS>zR~_;M*?v9rtt9#Z=#~ZQcsK(aY{1Ih3_x`xJ*e5=jWUw6dxkHSyMnRL zGd6Qtr_AyrIjbfrr18oO7o_HeIjRW|Ww38Xs)C^F5v~kGjoC>AXE;cc#F0ix|36hB zd6L}EKABTb;nXoqM8&SRaf)};pKHUf167oilVt3jE*#n-ydX#Do1)r>F2j4Abt_HH zp~%k&U$0!&^i_bxZgN>}X<$vPab92GR715;-+}q}!0%d4SrdA~;sONR)1Q&R5YaHY zBNO5s0E@kP!4l$DwC4c-3PCt}XI*-ki!7T8dU{UCf#P8EWrF3=_A;md%ubag`B$bG z=oXAu>46H?R!W-h&q@S5lVFheMNqL0;fF75gRc*7+u8A!Y`5PgJ?QBI6Bems+tNBD z^B8!UCpadU4S(&Eg}R)A^JgvEy{5mpCe_evN~FmcFy`eE1zcgm*3rHLUz+1;I1+Gnjz6qm z>gOp<161~tnlGcsn>NnlsSU-L282M4k|Lo)qx;yf+X<)HKr6rm4a^N(y8qwOh(-Ix zBukU+&Pk=CVuM8GB7X*lmwj0n%Fd3LY(AowDEHORf#DYCy6AZ8jV|@2z z#gEylL5xl{DSXzn@0`vV@jFpqELa!oer9M;Ll*a5we*xAZ3^aX(f=x~MTlbgZ7A!T zh(`EoIW)Q*P0i2(>d#cm;Ys#-o;$ex2AN;KRbEwVJ5(0FV#O4j950gm$&3V>cyC9u z&P20fuW%#XVDA*kWKOURuEWJR)~tB%;w=b-w_1mzb;Hut=J#yy+5tH)-=m9RouG6O5?xd4_1H8>-H4@3Cr`#%K_w%g5MLR8xo1|2q8)% z!6|~Ok5VEXw_)7V{GLK7usXPf+mL-4k@tF>*Vg=rV(`k-@h8j0sssbs1gn~NeJY)U z<-%nLXX8E#4qy5csai^EQYtm;UxMLS{67c=^#2vX@cv&2hB@l^{0lgxy6T23UJV~2 zn)#dom+{R3BqPTbMr2q=_A0yA&LC^^2yY%*#b+_5NLHwpWP8tw4Cnvk2~ur(+N0LO z5HTTcIz!7>&?q<}%{>EDIoDjaINYW2@P{%&!geA7RH$5gPB6>Fs7!b2K_IvJT*AEN z%U_qSD8Eb&(X&tPv?#{@c`8a+J}-uFyyoERfKwqHQjBaMhYA1zrC2tfku7?`7xxIY zTm9~Sgm@yy0+X;daXu#-p*+rE)t_{H-KT3e#l&+B2keJ000fM#@>wphnDc&a-^HsFT-|-2X`ixWrko;4p*Yy` za+U57Ib}Kjf{^N%CnSU*M~SA!ExF;E+U!-ByZ6T><7|u@z24wiKw6z(cDWkAcR4vW zHnugtbw8t|F@Cpm7UJXmj9n;fSw;pyU4eBZ+!KkrsXX!cYIMJ5Gn$DeCV!k@_hJ`h z`l6h@2k$#ku4_e9G>pLm1=#B_c)qg~1t}b5@VfZ5pW0_W=4?w1p6xB_J7aVSJEjgtTg_jukVra~)%YB4 zaoH}y3J>>x)Tmy5+H*y($lL@6&c-PD*q)c|*)G@BOiP;NwMh_XDd$bo<<997b1eF^ zq=4wJfRdG*R2Z(avKUm+t7oQ;-FeJSX^Q|V7|jV8P>|E)*AhBfRGC_5gVT|1x zI`WT3IKk{iLz7oZIdZffx_#5HX^aPRUg2Q0-0Ru$*g(=I#6SUYbl{8(o($ujt5-RWIEktV`pevBPe71wq~~GPNc@B#v4?!L(E;9mLOTGVVBX` z51>u9)j^#s@cY9f3EmBbD1!7p92nbyd_ym?d7s}AzDFt#@$t|Tmi97IcBPvEMjJz` zjZ^3OO8OKfd%2W2pahY0h&!@qvUh^P0f4IPtoxUTtBN<3O%lAbD!vL*`{9A?M$Zg9 zCo+Z$Ji^Vq(V`k_BZ`nh20x;7cn723aK_oQ5ye(s*5f&|HkAg8Sykhx3?az&Biwja zjNQtHzKimi88TgJouMY;3>fJ94Fgn-MY)-O3LI}Ud5@*uB1ZzLJUcWR{plKixtw>{ z`_I`45%RKm{f0ekd@M->Jas`&ZVc{~pVnknc!0A%(oZ{TV=OzT20tb5y5)Tfu9nfl z7arnTn{LVVjk7~CK=N&@UPNo{k)%7i8|l1r2!jqaMW0TAd5-Ak@c2-kLdGfZ z6a^w1>N`mMoogUT`zuc9L-FBbciV+Vw7u^?v;YRCZ(d=VUa5G0+LOVaoohb%sW2I2 zR<4VNi+<=^EGmeIh#WNut$1I5+RbQ*7%!rmu5^Yo%AyT}F4Mo+b&gu^WL*YEWOMr~ zn_E{$eA@(!?Yd~*mqAWXi9F5x3Sq3_Fa4_pweDJzZz`uPsGjhqhrD?nAdNmu4IBjK_Un^lj=)&I5IHf0yB3YsRWWA|d+M>Ol?3 zB))IOUo(aokxBSF({G!=jL9TVO^*^NRa-|S!HVGQj(9V_-=OruGX(|-d!(OpOSBgy z%#kBE{$i~{0RCy)m<4C4F+Q?^c@;A2s>4VA`!4;@^~f(z0Uo?ncb{&#hx@|k3aN_P zgvkA;CE1x&Gs~*t4!Pce*7)&>Zumak0BHqgAdDBh&lY&~A z;3p=rPezO$7Su(obPJOnDUXlhCFiqb#)R?q1%aSlQE6M)B^jQ*59!!Q10D&`TqZ*-Bnnogq#KMEyW$9`c_K*P>j zbljQ@lX7DM@sEbG>9vvz>+ABTO;BR(ZrSr%M9m%ypYc zLJ*QD1ps*b57q=dxe*jL%Z zs8vdyUn(Q>sz*09KO|C#iSHY7#=1|6xKjQFU21KemnRkytJQZqgEH4qly!y^mpeWh z$R#l2(?pwMlL2#^3%>sA=i;F7!C5b+PGvm%E&IT}9Rbtz6ZZfGUVGMPE3El~xvewq z(aFf-Py|P^?viGj>UcIh&k*qB3kA`Vn}-5Oma{(}Kh8{zz~5lOHtEkSb>tMb*7+F% z=aq@B>$w|@Q`Y1%U*&&sd6GM^-@GX6Cjf|f%c^N3q&rw`Mdn{X9JoiBZ$_&;NxN_W zdcw)ncUMz6C?}_-DQ!K}`vU#Z!qj7gF}Ov$u!N?6ZbxW!%1uf|D&z%+GgX9xNy*42 zzHR9{D~XN){Sr2n>dIZYlRb$h0gcL@b#CL5ROS~ARVxchNv~Zyy)keVZQm#kFVh{4 z)K3Ho;ab>$vwj1i`?E11^9G=^K!Y>95T^<4ZzQOU2Xn~J{)+s*VL!o;pWgk^3sHwckQZm3 zsXnESx0n6L?5s~RR0r5G7srCtirn5){RJNQhnnS*+;)x_*AsR{q?4XaP3fDhSd6y- z!@+d7G5tS@dZwykZaU4`e7v4%t|v-WR~>zIn*(dgO1$U2xVKSuyF1;}iK#o9?xdYO ztE!@TK;0t^;%|E(wmE+C$*K(Q-~XZhbvWpOO{uNs+Z`i#4Z}#v|8<@r9wbwvZcyka zMukS+6#mpo^*i&ZSR&lPiGoR-=u^JUBXgb$*4Xi?dZ<*9pDNsCYz(y`4G*&~Kw&LW;KF3pvvqQt03T@2`T1p?CDBeVSE{z^g9ElAb;rYvY<}QaL7Mdd+!U53u0#G zCfh1ZTUYa1_9gs9BBVy4MJe)p;V?j)4zQC>MX6Ed^`k$k>jlgm|&j}EyVUhG-)pu=lvZlFPqh~wicQkhE7Y|T8p)G7 zpF_k?P@z}c$8(JGKzGVdJi6)6b^vSjQ;yI{R@t zifBFq>vNsX&zsm3RkZI9rVPL`q3zv9^hJ z@U|Lqk7A@DQL*x#!90V6ifV=;pQfX{o`Iei*xR0ZeU<1L;uJ({$$O~JA~N$AEJhLn zWJt|zx48A@cx11ArYEvoy;^(tc-Dl%+7~)Q=KDo7OZ+XIf*hN843h~ z`Wdbyf&x~y1_wz>s4tR!V8p~?DD#ZwT$yQhTZMdy*X|* zGL;gxv>ynAyWMNJq($G)9Eu=TbzMk!XsB!!VzpkFDyF?oh8&odAzYzHOCgeK-^7v% z(BC(9QsqkYb<@&@-QD7h8+^`4S;T0O`)YFLi@Tjy|E?3c`rV2 zuTTl*K&2n82`7u2Y` z$IJQ8?Z^Tm{;wcoFpuYtY4RDGji1_tU4x7+prl}un8If$S2LA{#RcVcNQMUf^QPdC zTKf$X2Q5O{Fi{hcQ`j!?bC<QIBDino|g)H`(2FB*~$r zKE$uup-U$fos?J(kepiNwoUnSTJ8^UP&b$<6cB=fum7P5)+^1eVvRXaf^X*d5;NeS zURBlLm0E#>~q~e>%vp)XWjIlsbFYX^L{|ZJ9Srqq6k^qk=aZXN4 zU!QxAsLj$yj>tT&g`Ho zoUn(>l3%OVpTr320KCWM=DD=Voi0znVNY3l?xEELLto!DIX-o5iEFG z){noN>SZZy3)kyjb#{W-Wo~Mwvj9*|CRr>ypAhjn-YBylc!#Ik&?a@hclxbSt8(OH zdNU3uGsA$^`Z6ed=}(u|16Ia3KVWXgl_;N3A+=8IYEKL$3&_w*t$y~z9R+b29+dei zB_c=v&@HDo$@nocIkkto5C&@49tdE98GOE>{`m0r|m&uR?^i`ylx!PD!7&QruoLHW( zNCcP!X-+i0Xx>bY%5-1p7gYJLa_RyK)ewz1(zL0*_^MjwRVClqv}$~Xt%~kt0V-)M zxpJk6rvs(4)1gfK1u@r65Md@VVSl+_Wvb4}N(&)XPf)M#GUXBxa_i*ism73)Nl+|$ zm8(<0EYRc9xhfJ1sujZEB=JZB4rc-yoSJnz2YaB@elOn91NO;H6+;5jWk9Ai2@exZ zxtVDk3UXBPp_oGAcv8uhXmj3N1&?VrWz&*ENyFJ|j5Oxt$njQYx%F!>d60Bbt~MKf zZD4V!!((CFctc8mVA?q~emG`zUJ(m?Zz~`Ft_=U!=PpI$dif`4{yQ~%E5ld`yI$aXuE$%ST>eQvHc$3QY{QY>I4^JL4qAjYr( zN4VJmQuh)x&I&UFOafYdU8iuA+LTe{Iwt6DcTE z$BTCB9-;HiH_9`fG0>32m@<$s_EKi^qZs_qAp>8$D6CM4#-irfd>q8#ApQ$%77x*- zsB!3%KM^7Z#@`ze8`g<1g|dY}f4_Cp?9#+6)ys^RYvn(ltonqPCwo;z{-mb>s(rOp zJ-*%RQv(Y0d|xCsqS$VQ&9D_5V{bf&V#e%On`6W08n-CL?9PKtV`Y%_f&R{)hl$}! z!#;TVOzrJG-^S-5`J#v z(L@!Yrdp}q(Aedb)BIshW0calFeYjiGgcT=@2$>X9!qoBUtZE&;4^hXI}g$?7Equs zWbfkhcaOkK=nE1BL7*5J2W8+Rp{uh3P@+}I^_HJ z5_G>@UN03@{z)UJ>h~x>etufC3q`5P%G4l>WBOmHAu!MXLJi}eZ`f3BCq;l+buqe2 z3kzG4Ckxq;A{%VO1ch~ob<5^G4$)t5g-I?ME(Nk4n{XG+>1@r$cnXIo3tid>CUgzJ>Z6k0HBbv1sn}H&Ud6={ zDJv}+x)8+igDoNvNzx3lJTRj+A7T7m&X_EG8Jm!{C^P9IJ$z5s_N6S!j6jkGfq;lZ znkvG^GLlc7_);?-MkZMPAX!1Rn#>a0@YOkSjuoJfQ`d##!u3wPTu@Nz6^SBJc)?~% zqF`x|Yxu2o(nRAg4eGi49XD5Syst!fzRxSxHAa)2H@}?w$ z{3TD&C=Xd#k5!Qzps4D7^1UnOfI|1HU${*r?@J{17RL6~z`uA+Va$A@wBF2Mm$D;5 zmd}fJl#8mjkD?tkB>N!umI%D_) zBA2$f)sWxRW??8SL8VV&Ong(QeTb~ZmS0>_kYlnc=^IDloeRH;2s+I2QCc;8QiKK? zq-MC(mGQ+eq7HJog|X?lg&-@k#Y@qDpNUq1@VM>K%?+F}0jrm*|3o*&t&M}7_yg*e zbx5uu%S%~*3hz9nUB!r?+s~sJozeRaW{HJ_I_=TR&-7`{?X%rS&uBBLJ-a!^J45hw z*%XZ+`Fr}t^vora{|yH!5@nt?gce3J|NFF_jvA31K}=p%K~UQ^k}}4Jvrb*)C|Woy zuSdu%3e;*&a7Tn2DF{(g8I?%<-5I>sse6pPA%VAK2h+ zjzt$emRNe;FG6Rwrg2tb6J71<1S>@;S(%HhI)(SHo8cN@rYRfd8Yn^=W8w#-RJz&} zK54E`6y#We;o!XN#&9%W5y~*R@ce=CFfC#HO-D^bOx!RvvgyVK%r9E7Dt4vt)&L2{ zqi?h=_zzD!n&OZCMxOO{V4L>I{{Mm%eMpuJuK>Pz=--kCkMn;=B1^dd`Tbs(YCP%* zDH6li^~6)o<&RV3b*)5PDo|T-5l|?W`o+$br*Ol31@cl~kds%*8sZ@Q_slbD%14bg zH=(XTJT?&v?z_C@)`Vy+bPZ*vMIZ5Y=utjAPkE@3Eoh91hy8INPD7|V@uB+y4d<$RYbAcs2?;!H0S*P3T<2Ww9RRR;<>+=5oXu)0j?&2z9RknFi#KCms z{69jNI%ji}%Y|7r{RoIHaUeRSOti54u?WtrrTIKoUuJWTJVF ziA~H37&!UM-upK8()zF&SmO-XN=ErQL*I#&?g?T!uCy`9?Z4%VTAeLypHWIg3@c(+ z`Mi>-7Vf*G6N8D&&nqZsNB9N9)m8$mX2y(MgUVXa6C0>NfLYd?=M(XBpEj+@&@&Ky z#4%A{l%*}p84Z<8#e+M)JAq-Zd&x?-%_=|g&trV4bzy6#-;75mJ9<_euQVs@s$;qo ztDr`pqd9*DN%hUxbhIJ4@DGS|;bvM9naxe^#~rw{h(;0~m8thrmMvU3$(E@|X5LR2 zSvGaIPmdZyO^4R;q=LQ;cX<3Z7wV=U4Fy6juj}q!Y||34i3>KC>!Tj+OJ*gMT=Ae= z8Lgv}oani8aoXzU{7|oH&*+fyA-7Ymuc4RouG-`|)1tQj#6T;&p2ccWEi*d{$7`jq ztBvT(7!}Uu6zrBmTs>`&g@b7jhY3nz-ou_mjzxi~+#YIv_G@!2qnY{Kb9}x$53O6! zHfy*}1Ur4>)5F_Ya$f&lTk=X_gx&3!55JIYT%!k-3VXp{D>{nP1;v_+NAC9ai36Fe zmRAI~>n1PgfcyX>`J)U81=(G~Ao`bdnR=Bb>&Y5WOn8-o#OzEj-xGtB)s*>|Qoa(O z>mAf)%Inc*u-2&Z*anGLkS7+`dKqi6ZHy)+-c$T2Wl2>7W`w))StSr@P}C*!5&duX zKXZ&jXOkDdXh<&W=hoRG1!)TH&P~pavYzg&5(x1g^st=}RgYrZ52{ef@j_g=yLSE+ znToe(*qWqS-6MoQ-IbMG7jd%m%o2;XW|kgwmE@TCjNUmb9xd8G4f77qXB7+M-zki1 zm*EJG8+v{kt}{et?Idk?k1^p)7Mv2)3wVlaBOwnvgP zi-?Ur=bL9vMbLNgF}FR^fBl1UEg#{u+pnAj{vo*{OQS0Q56Ty7CPeI<0ut(f4)^bh+}015gi9>3)@H3O@LgP6bM!z3W(bAUq}%53oe z?@4Y^q+WhASj|a})*9l-70xGBJV}A`FNhj9+dj zt82l>wYkQ1Yeu~heE0hx{)sG*He90a2g%#hZ~MLFP8ziFSyt}86!=R5a&Ers(&Xs; zrlfXz4$d{_WNcjS-NisdV}oanDRA?!J#UIZ^c~%xN5}r0a6worNP}rIPZ~pC70s?YalY zHZZY3ZQ%U}KRrUdJg_-F=XxfXME?yqv)y;rp=r{3PyBz=WfvdRyC~7ed%|T(Jl^Bo z+0K|d+C|cAcFstbqn5dNldd(5$tx|`G*@3k!IbA=eSAc{<0a8&ou-WLpPtDsCJPmo zoIUiKeaj1q-KRgZZNhK1b$BM@P`8yN%&FOKn~tEGv66)H`KLq|f6%x5qA=UX=b|cI zu$W2cJHj~x-~7mop7iKVwmCjQUN7+pd40KOtJ#US-_S}Zn>l5X^t=o$G_X^saLuW( zA2+iuFJxB@VN4*|!!K$d=5P@B|A98|rXMAV{Tpo766y_^&nWI#k;>cYvoY7dPBswm zDq-hk!DUfMPhb7L82S5+Ic394u1p>h)8Nz?C6*|_cS=jpL7UQ~Al;HN(!w&7dcQXf zKHz_V&2RL0H@b=rWN5^Gz7kcL7) z+pR9lIYtTTVvJ5GsQU*1=m7o&dUvnlZ&}nW?hRZ1f|_yXRInf)f1w1A!-8Ii+Fh|M zkDZk&q3a)|WJ?ofbhwjfzUk*#EafJCZ{!N>YKi9E?g;5v(DbyRk@>wYSl%4l%Rf^X^{59 zvLP0hvsS&|Xn_MI<*1jjjxy9_2{NDN#n%))I;Dxzr`=1xCtow{n8J{ex~lurLH2-qIgxII4~`^ zW9}!W@r6X_W)%5Mc;HXrXlSk7ld#jAmxzmGtt*Mc9n+PwZLFc+1Ef+K z>OaHdnqyEK667AAj11gFk$(h8fc&osa{S^fiU2@hqSbU-OBK*mEc{{APDnR`?{GvK zwsggFV#eCP+3oi{N;74D?%~(*K1_s(O|3hLVs z+Zou#dIb@0FJ=yu?Tacp*4HT}M2@4l#7~{)l{EbIEfXpDY~!gNZVkD* zF6f+>(ezcMe=xl1M0#N2W@tGF<`kEvO_32tJCxX>T^k%HqwSTv+0w?Xx6%9yif;yX z<&0Yi)qowCX9owfyRt4Qs!NrsPnJl^$Oc?qq-QctR$>mS%TJ5O0<0JJY#w)nf2+ojuS5LPs%e5J%@5P!ugt!~JLTW7bMl5jnG#fGJ1U+Hos ziv8bUyc+YNIcZuVx+-5`YrbE455c1?RuF55Kmia+pJSo2!aBe$LSAC<9u)LGBKoW` zalm;%n1Xcb)a238v36+k4%E8Q$waidk(GGJUn6%b^Rmk9)6N zY2T*jZN-x!5JJPutCQP+YC z(xiVPkYvrVs4ZVm3vc{pWD0zh?Gp^W1gZ2cyo^@CPoY=qF<*n^Donm(Up~7-GmIb@ z)9uUH`%khHtQlH`-uI$*FBR4$9|nvkhUXCjuO@}q22WiPlPwdOw9R#z)ZRxN64i^J zOs?VgLf}c)NZ24!UgO}!M9}P~xCkgH79hbCs4NN=)GxH2Y+HLq09^{k0&FbuY*Ukr zxl!x{GB~ufSvc8P3L1k9s;P`A<6!%Ve5yiX_~6%}W1P}ML^%lcU{CKk6uqV1eB3Jw z4mwo54J@^b9a8!KAPP)C{*ppDgcK6FfW|2;65D|%$@6?Q&}MIuMIb`xrHjuBLbYXw>sK0;emO%v@x6i^w?+;70~ zZ%&P_f;||v%DjBiVi}|w_Zz$@@6pl}^qvukaVe`&pk%ZgdZRP^Wm8;sPnQl*Z#I_? zr9de@Z$7cgcy6fIGx^!9@<<69?Z~CF4c(!kauT57@NuqIt;ejWlRM<`0)sAKvL*>~ zciud&!#Z+1>IR=I2T%X85jLtMJgnXR+%U}z`}GPf<5BM$AMEj{$YGbM3tw|sVFEb5 z)bvvc?-?g2>&cGm=I(2pzL<4HCq8y5SmC0nJ~FWlDD(3}62L?+x1D2`_v~o$0=yJ%Bk8Wlx4*{%#d~AuZD!f`C#X( znTiY=d*s!QZKsI|4g>tos66H+4215ELt{=Bm9CZB`~o1Fj(!yxR|2FUFT>vFS=NABhYD=Et(}<43a&u9P<&2IoqH1(Y zE{rmuni*=TJX}RcF7Xg$Z>BX~98N_ADky52nK*LFAyi#>F zYoI2D{4NbzPJ+0^8ze>axw+MFP;zv7HOPr4eUjvcN^KT2a|u*-kE}w|YS6=pAS#L^ ziUF8>4Sy-Lfz*g`Q*Ib+n3F{Hcm~wPcFwZ!z>;gP(Pm~jb*tbRkQ9*)>}x*20fPa6 zDv^FqRke~u1}>nRrR0q9YIxqM;u+2N9k!Ai)7_Ig&d-Ym+4~t}7pCWpbyY>sIl_x2 z1m|;v6nAaUy;XAdmhQL)dRPK1u9e@y!q?aC;=h=G0W>K9fFzp3mOI8jn7ePKR9K)6 z(u6de|Dcc7@$w2LQ;gjt@DS%$z?P^>0l^G=9DAm_AM1v>YinA?6j^t#8dkoN*wZ+#{D$t{U0G^tT%_e4#D8a z$5;F^u4QO$8KVb_ZM@GhKdmkGEs9e!-D=@hN^03u>xL_f7UHLU?{VN#MuZp19#bFK zXBx%)1;kxTA)lw z?rYz+fc{U)iHJdJq(v8Cqp{_dMhpY{5MNEoFQ~*~tmY-Uty$N3F`+P{p7?|SK2Mo_ ztpY?wJR*mbr1bbBcc^{qOm(KFszvtcXgb`yQIW7AfTVO%P0PFqsVhL_l#?tC*UiE( zW8X#)uf_SWLe4DbvUJlK`Pn6rqBYs=RcbA^c>xUp4(b(U`CmhYa{R+T(T?JOZ*}J6 zZxuK;)K7p8U*HnNM+S>CW^L$}kMH6Bks;5JZ2Et-6G<6@_9>9fmV_A%4S)Zhi4Ph% z1qGNgK&=XBv>Kn*HFpj3M5(UX5TD}RdUHIniiL4hXdNmJY zjM>Vo)Y2=AzHM)AgW*JhcN1*hfNTO6CH+05zST zue2_?0Aj0di(P47I?B_g$Y^uXbXyu@@&8%S;rZVM9nz|0%H6#d-tm-*^Hmnx?D+`X z#01#}#?fF25C%Err+%1@q9=>FepIDNb!IBHXR578qF8Z#a1W)ii?5YiqAwW!O%>!|W0Ef~MNeb)0C@W)_ zmQ3fgi3qLV5#+2T9G*1**m%FSJ5gGA9fghEY@L97dO6_`4O)ZVc9zel72m$nEzh3F& zUYF-Lvya4YFLZ}P)VaVry8VEC-}KVZspqIGBIssMX-!p_kW23Ep`u+A3lAmE6y+#*A!b!{56~$k2IZ3et56nR| z2E$B!mL-itf0_P_?UqwjUa3R(*-YeZ?F>boiRINBu@ zCiAx%5qA^C<@$}C<`@!M0;;3|obLjV!AmWz`)y$#qTE&)T*|^4Dk&xwI_twp%^=_l zD{6#L?vqxN4>k5KWkeo4kBUuJn38ao7QUr#d0!Y}jwMBdKo|uOivv`aK_mrre{d5{ z)Vz};-*|A+Jmz>%SOep00bSM(?=rk*TC}H+;rW07-Soa&f11m4xO=P{pVz%VllDBI z=asN{<^DRy*BPW51&o)gMw$D_Y(?0cci|Z&AACB^mdK^IFx|FB?v3y@|JN z#l{}Q9Lr@*FI(>j5@`xCGz~7P*)7Fh_m2(ejRz3wKsIb4r6$R%hW!pEQb6&)v4mtq zvL>lIC#Bhhsp!U}xlTO;UUPE?ZNu>M*zmA+f0Voo4wm6mGWQ4Cw{w&_N(22G=UVr7 z_41(l-xxb`);u&46mJz_$En_(A7jw5r{Rs9`)~dAOg=wxFlwV1W;VjT?c@E@tfs=m z>6$WXBiGs6!>I{RFZV012VcgeS-2NAMBp$5MnO8^$O9+|%=T?$&s?1qQFGR`#H(XT zGb5SK{^(_#a$w^BoOZG2d@=Y#mQ6Znq!&4#)`c3gPQjQBD5EPiqogUJ9)+pQDow*3 zQht?sr%5M-3fn#>O&q;c7Sj7-U>r@nkVH8dC#taJqA?aKhy&r_=P^r7 z6B7G2ryfOvW_8MAlgns8N>Y+L-CvZ`R~NM^NOL0%aBKuv_4W55E-0)Wsl++q9Up&@ z-K+?%w_fYxLe?a?=(t9?l$hYrq#@w>_^ZuPbHwQ?w8o1uJg?C|JM9s?v273gtkELC zyJB#CT6{WM{^IPHP9j24nrn2!*K}xt82xc|8-eq=DL*Au*-&R=qAgh~|EPgavLYED z$qdDcyA!7k=3g!Ty7h`mKK;1gIz*^~06F-B?b<-m56}49zdIg{NIO#|Bhc@)xj?nTc{HBlP}OHt`jC2 zeem%$HvLbz0J7w)-=6q73I>!^imJL?X?KW?5eVvkSL}xK*OwTiMUFVXeck`4c1<+& zRO@GC-y)BsbG|j$nZ)POY}JLDxzE@Tn{H=MT1MG?Fu@+)m9f~O*kZ9sqIL;JhdObZ zkr68hFlvSd#rQKuhtZ%1aTwN`3Eqb#)6UKaypPm1(R%c#{xs~sgtjbC|JEvHcU1+q zR`a6t(Nt~9MwSZ`f+KT!)B#-+Z-(9sau;NQ=)YtW7%f3_AUQFO3}HIj97tJwrw;rr zY#D!2x%mE3_`ba^z#$hx-)7K%N+sZG*uQ2R#Hj=SM``cpjZC6ISCH(!l>i zS#q;rI9%AAG=#s(?GlrmGd!Hg(b?bJz1iq1I9E)8QoOf@+gOwl6b4LJ!nhtVj{ZEQ z=*+HwMwFN`VrKcwEIFryT{{1e71jt~jjl z3mb2DJj6K`886pGNm6tc%cy=I=WUXnoq@yOg>28sL}Kvk@z$AjnfkQd(F#LK^NT?C zYQjFx!`~|7_*#D6eR?yI)Z_V({*%uK>BK(w{fvCIyE_TF`--&5+NtQo=P>7FTF2`0 z{wMC&VHoS$_~*#Viuu&P8|c~UvqS%C-V;?j(RTUyDO0hJtuu+s2&WvvDigU2$MKGl zad(`frz=epvrvgZdgjM2vW>DfNW(8n<0;h%%!!S+F}SEoXDlb;`VP@|0axBK!_{5P z(ns!Hjb*_fK_d2nFuiJl{MQh??52hoBW#PeP0gH-C?^cz;zPj1p2yDsJkXsu{C^Gb z`Oq>Xs1XVA2bh#)D{VO{hX~i%*yLe+{7%@gq#PJdSPK$>448vl#)sjHf}Y-KgH_zF zKIaeDBNW#;So{(=B)fRK8|#7(nYbihJ8Wo9`Gq4O!y?r$yoY~~MEFYPfo3bX7Ff6+ zOf$kd-W0bYd0Kkv%Kn|{Z+es=(!rCjEUk%E&5ZLocb);&FJqA=i4hSIKvZzk|Etd& zm*TG_fGduxFKo<2I6groa{XFZQAvcAI35F^?7AA9q1k%)Ys+E|3M%II$t(I4q_TE= z@LY(jHErc|5QVX9IWo1Z;%F2V=~fhk?0&*LFPBxj*_&r&b4wzF!!;8{cTb+y6&ag&{D>afoGLAn|1;Ix)5%Xuu9T!jdK#eEx}}iasuDuzkL>tj>~;(K zQy`C)6!Wdz8(eU`s_QBuQDY)}Jk$?sSeiNqJCtPU3kzzZ%QM57cC2GEKi*j1JF^ht zw#;8k!#%zkA68WticmPG&Xu}di%1Y|RopLSA*DCn)Au!Swu?MUbGFmoROHe3=ewjh z$HsytS|x9=i!73*At~Nh14iZf`_K|Ej330qqw_5fEa5}2(7k}EmjCoqyBmN`uSt8a&ikj4H@Pu z+Ij@Kaf3N1@(7}Y02;8MzQoNFQc{@c+aA|AdC_P;yWf8P#CN)F-@^MeCYo?+p!+Xu zEcs7xHEEVifI^tPusCqiq6vhwt=T~PE*wfQC*Iowa-%W*Z6E!%foIp3!G+#78fs_v zIceT(F5veNf*@DUQaUF-SBJsCS25eupDr0{P{HNZLtO2DJVKgNxcfq7Utf<`EbP?{UHi{Z7pvNr!cU8nZ5pbQ z^+uS~G6w?#kSiEi#Y22~`vPY3d8EKu@iL~I6Az&C=#BjmjJQB#w7^EedSj#@wiKW^ z7X!aB$iwO5iO!qns;#LntDRNX-m>S>lMU5%)Keft>%Vd^r*X2z?=u^0-ZPHhr|%Cq zM>bs`c#?$VZMK@zlVbJ*M3+jCEl|JKW+zm>0Jhss^hxpzsV#S?*N?WrNfQG7v>Gr& z=~tzgna7xphg|8}17({VZ%)BEe+bFk{23$`wI~YSpenyM82F zX@jH4q2Uq|$M{P{jY@t*hN#TGrytCG#`QO@XX1Qa*}0>`_LMF5Bi&D=;uP!fZJ{O^ysvomtuyV&;rD-S^+)Mc|6Ql5AWcCWS5PaLWULqf zCK`?HInA*Mu}JmP?XZcNJN}ymk|Ul++_Kki^sy2!+1t%yx{Z2z!*%ay$CHlNcSt%} zGC=g@*Zs!D=9yoeA}%1c3v~r_0ndEA{jctiNx@bA65u4S?;9hTsvf zjLM}616Q6|wL>*@Aw`b&NbbjlpoNKg}52+TJ~A-ZMCKzLp<+J7Fqja(cj~J@7#gvUb7Z=O^nx z+jwbRv`<8+Pj=W?AoAk5N7Qtf>|SG2v(IVBB$1q;pGcxKiU0?mjY;6>kYNzvWMPQbqI9;~fM!8>O)Z&sSmB@dmD z-qGyRGsivLwWWw%9O;c=*>WH82rdwn-4*xQwQ`(R&;H8mLk4%gT>+;V@D$8^ovm>* z-W$5^lJUYs4)mw`Qw2YtoaBd;W4z{-mZIaC6j8t?Qm!(W;@2t%BHC|_V?TY5Jrq|z*{WOo+4Ic=jA{)CK=g@+x(Z%YN1cV}x zAc)OvYY9Kgjexv{iF3m2kgGd9QdCeD64{W{OEW9|ZX@(9EwEI1jL~#XL%ughO-d~Z zWh*uZIc4f@yCThE2X{Ud@4}=*-x_n5@iESLUz7l^ozs)irah@0Sw~e@t zDP#c%!a%ivhkI@)!oTfu#Uqe@*AR7JNcn$j!X-X|#J}|bmRKm#KNkZ-%7+8aZ>sML zeSa08AZFLa# z7^*I8bz_`Z&C5uf^HHW@X*1H%B3o~2;8mK?b;!NEv(qogepNv4PL(tc3;5skf|hngqpU587Y6=$xt`^*Ub2c zEnHF%sF8XvQ^t`(Q6_NCD+Sk{msZQ-EcUr8v-CN3S;fR1KROCbm?1_2+R5c9#%#tlK(Ere9()dWHg01O69 z%nW+TcvclTH#yJj*S%L^Q9H*+A3Qtn(EuB-f|Fg}2(PyRR}~8tkM?a#LG)QXk(b_AyTx^uv% z%|v|ScVdF>Bzk(q=}42o@?HzDOqY9eOhtrZw84M@3;ivW3^8I@yqY|t0;R>eWCCXo zwM|P{L9FJgl$9Ns^LIu-dG28UGg4d!<4*=aw)VT=CcAZ^U$(p);9!cUoer zYp%H#cfw3w?}_s8=VNkkE69V4jz0($5Gx{=|_lth$T52R~K2`@TpAfdOMXvqCWaddDaarNS25k=vf z7t717+^ZPuSi`!zojZ&N_EChn6^uzn?z%Ia^0@Mp{5wi? z8Yl)*1dK*9P=~CX0Dx6QtvvrIe$P*0)0-$D#RhZVumLY+2?IZY?T5=iO@)_t0IVj> z+cW?q`f+|}UzrWOzK%KL@p5PsI-}r42GkM)B#9$6Ew?bzKM6`w;(qc}b);`F0b(F$ z-aE!fzv0O1rq*6yNjq&3HMENeaGDrju(THI_oO~i5H63V<}($l9Ik{VzRmE(_;1c zm@(|A?GBhuF^-R`<`@heH0e)%%ptub$xZi3rE`HO%JsKjr@H=`2BIvi{V0Wfm;?tA zHFx+xt6ghdaTM0gF4V9!C>#BO_hb$R{&&=swpNT30!+2XJ-Ffu%$CS6WwX(o92>sv zC3fjz$0LV8QK#+Uf>vmQ%dP#fO#_EHJ3}#@iM`)?f%J2AuCE0Dqz8eccNskfr>cmI z>gTv?cfvbu75m*Jdw+!+-oLHacm+l9K+~`iQeH82%Z(y&)_Cnxoih^lw|Qi2+R70A zj`SWY|GLJaM$6c{h@3j6vRK)CBJ*jXzQ;HT^xyY<^N%Ygcb+?|^|zUoY;0N(0z45% z5+7m2Qum;c`e!9|q${*##B6ZjjxSlq+!^!10?8TEIda}r1hw(^PCb^3q2(=@hGS}A z?shwy-ro!{-#rKV#enRg8hoAL0RFqxjS_U#-_=dUi{}+98kjwvGyng`T$ z6N7(*$eOijaak3jWGQ-#8}+0#G~8eZ42%pR^w9TY)Us^#jL?+cbaO>=b-sIVn*XkC zhzgFCK1w1?fmh%$C-~*QbXQ6x+=_}*TC1lR|9lMeesyQzX=uM6wG^n``Jnb&zM&`` zjECRI8fg0tRFMx4E(AzD3g=yIzbd4|l->OS%p& z{sn(0Q)VMh5IG z13S58IYF~*zgadq#Hv-{D*aUGtb*TkWF1uM3oaLhO85JCCx$)L)H{D0l3cwZlqf7v z&{`Hs+MI5pL(i@ih%3*>T2O|e#Tuop$Y1y&{rh}~XhdIYb8~>|{M$h;c_PJ@y`Zq1 za-T>4NeXYW)#mbkSn$5A{-uHU_eZWMkqB8ys4y}q6J;Zc6bk}GI%W?AO^un#u*9iq z$e57`eK**JGMk$cw~Ips{r`;n#HOb7vX`%Y)jpK$JF@!RY;6m7wddEqxPU?=ZF8p6 zly%EHtnMqBN*{@d=-Ro2OIJ?O&d$l`#qoQe6uOG`PpObJkWg5v8!_EA-nSvDZP~)J z-#2dIF*;*lThhAv?k2k)U?}}Q56l`5*LAD>4W8k>ddBW5##rx#QJXvbcX|A)3n#)2`M1`TI=Uyqno? z@qOBQD)ZmdrPnP@=Zl_tE&ue!w#a>7E=kXs=dWkl+y61EtMWT$1kdSg*=l)d96bsol zHEiQ`v!fg9)Za~MtU0VB+;H>a8RP1#6I?rfdipm^S=-`V`c`J^!HM0;F*CnSx-rYW z*lf&#ZD&D#91gyF!6Z+m<`@Lygz<2pOCfikksM%dOT0< zk4*M|yV&5ZZMEgsH=F0o@hbN@n84s0t#v?R!*kII>7O2!?CN;swr1zikdWNvs)dc+ z>t37foM#^VP?a0pO0GWlj3e-* z%B*FH1~VBAQX(!)>P%hdrTpe$`(MAB{P!D*mn@xi>4U=j>aE>+ya^rm+vK19zCHWL zl>-*{=geC+TlY;tM4}p#mz}ljf|q7x7CHhQO^*`N_cawC=sq8J%53tnpZ`|>DXE#6 zxa2`--N!!~&$h4&|46^|CM+b!6?l`f>I^>D%X2n3VAiQE`E!K1AI^-La%GXC6Q^-& zs_TyD%Q)08+q8Qszv>OR@lTP@e$k?g%@*hXPMZIxE+bVr5|qHzwre^~Ihb;6##HB= z5{BlhZaFm)!F3m{^DP!na4!;Vt8#zFEYfgp#bwv$-^{*EOZj|Oj(OQKaY8~ybz1m=>Yl*3Z!C#s&8Ih3@#W;FJG_57eX^pHcj`(PhVKCh zoy{*LyQC_2>}&w;>;Jp&YKY^}8JtHJTxk-`Yg)cGct#{i{}33 t+w^~8e{sb#jr;FoHc7<-^ZJhTKlXCQ)k|&%Ov+^d0#8>zmvv4FO#q(g3$OqH literal 0 HcmV?d00001 diff --git a/doc/tutorials/introduction/linux_gdb_pretty_printer/linux_gdb_pretty_printer.markdown b/doc/tutorials/introduction/linux_gdb_pretty_printer/linux_gdb_pretty_printer.markdown new file mode 100644 index 0000000000..9d64469920 --- /dev/null +++ b/doc/tutorials/introduction/linux_gdb_pretty_printer/linux_gdb_pretty_printer.markdown @@ -0,0 +1,38 @@ +Using OpenCV with gdb-powered IDEs {#tutorial_linux_gdb_pretty_printer} +===================== + +@prev_tutorial{tutorial_linux_install} +@next_tutorial{tutorial_linux_gcc_cmake} + +| | | +| -: | :- | +| Original author | Egor Smirnov | +| Compatibility | OpenCV >= 4.0 | + +@tableofcontents + +# Capabilities {#tutorial_linux_gdb_pretty_printer_capabilities} + +This pretty-printer can show element type, `is_continuous`, `is_submatrix` flags and (possibly truncated) matrix. It is known to work in Clion, VS Code and gdb. + +![Clion example](images/example.png) + + +# Installation {#tutorial_linux_gdb_pretty_printer_installation} + +Move into `opencv/samples/gdb/`. Place `mat_pretty_printer.py` in a convinient place, rename `gdbinit` to `.gdbinit` and move it into your home folder. Change 'source' line of `.gdbinit` to point to your `mat_pretty_printer.py` path. + +In order to check version of python bundled with your gdb, use the following commands from the gdb shell: + + python + import sys + print(sys.version_info) + end + +If the version of python 3 installed in your system doesn't match the version in gdb, create a new virtual environment with the exact same version, install `numpy` and change the path to python3 in `.gdbinit` accordingly. + + +# Usage {#tutorial_linux_gdb_pretty_printer_usage} + +The fields in a debugger prefixed with `view_` are pseudo-fields added for convinience, the rest are left as is. +If you feel that the number of elements in truncated view is too low, you can edit `mat_pretty_printer.py` - `np.set_printoptions` controlls everything matrix display-related. diff --git a/doc/tutorials/introduction/linux_install/linux_install.markdown b/doc/tutorials/introduction/linux_install/linux_install.markdown index 5083fac282..e69f6ea707 100644 --- a/doc/tutorials/introduction/linux_install/linux_install.markdown +++ b/doc/tutorials/introduction/linux_install/linux_install.markdown @@ -1,7 +1,7 @@ Installation in Linux {#tutorial_linux_install} ===================== -@next_tutorial{tutorial_linux_gcc_cmake} +@next_tutorial{tutorial_linux_gdb_pretty_printer} | | | | -: | :- | diff --git a/doc/tutorials/introduction/table_of_content_introduction.markdown b/doc/tutorials/introduction/table_of_content_introduction.markdown index d1f2aa3ca3..8fa89d7d7f 100644 --- a/doc/tutorials/introduction/table_of_content_introduction.markdown +++ b/doc/tutorials/introduction/table_of_content_introduction.markdown @@ -6,6 +6,7 @@ Introduction to OpenCV {#tutorial_table_of_content_introduction} ##### Linux - @subpage tutorial_linux_install +- @subpage tutorial_linux_gdb_pretty_printer - @subpage tutorial_linux_gcc_cmake - @subpage tutorial_linux_eclipse diff --git a/samples/gdb/gdbinit b/samples/gdb/gdbinit new file mode 100644 index 0000000000..228e8f7023 --- /dev/null +++ b/samples/gdb/gdbinit @@ -0,0 +1,23 @@ +set auto-load local-gdbinit on +set print elements 0 +add-auto-load-safe-path / + +python +# Update GDB's Python paths with the `sys.path` values of the local +# Python installation, whether that is brew'ed Python, a virtualenv, +# or another system python. + +# Convert GDB to interpret in Python + +import os, subprocess, sys + +# Execute a Python using the user's shell and pull out the sys.path (for site-packages) +paths = subprocess.check_output('/usr/bin/python3 -c "import os,sys;print(os.linesep.join(sys.path).strip())"',shell=True).decode("utf-8").split() + +# Extend GDB's Python's search path +sys.path.extend(paths) + +end + + +source /your/path/to/mat_pretty_printer.py diff --git a/samples/gdb/mat_pretty_printer.py b/samples/gdb/mat_pretty_printer.py new file mode 100644 index 0000000000..e6ad2cbde2 --- /dev/null +++ b/samples/gdb/mat_pretty_printer.py @@ -0,0 +1,212 @@ +import gdb +import numpy as np +from enum import Enum + +np.set_printoptions(suppress=True) # prevent numpy exponential notation on print, default False +# np.set_printoptions(threshold=sys.maxsize) + + +def conv(obj, t): + return gdb.parse_and_eval(f'({t})({obj})') + + +def booli(obj): + return conv(str(obj).lower(), 'bool') + + +def stri(obj): + s = f'"{obj}"' + return conv(s.translate(s.maketrans('\n', ' ')), 'char*') + + +class MagicValues(Enum): + MAGIC_VAL = 0x42FF0000 + AUTO_STEP = 0 + CONTINUOUS_FLAG = 1 << 14 + SUBMATRIX_FLAG = 1 << 15 + + +class MagicMasks(Enum): + MAGIC_MASK = 0xFFFF0000 + TYPE_MASK = 0x00000FFF + DEPTH_MASK = 7 + + +class Depth(Enum): + CV_8U = 0 + CV_8S = 1 + CV_16U = 2 + CV_16S = 3 + CV_32S = 4 + CV_32F = 5 + CV_64F = 6 + CV_16F = 7 + + +def create_enum(n): + def make_type(depth, cn): + return depth.value + ((cn - 1) << 3) + defs = [(f'{depth.name}C{i}', make_type(depth, i)) for depth in Depth for i in range(1, n + 1)] + return Enum('Type', defs) + + +Type = create_enum(512) + + +class Flags: + def depth(self): + return Depth(self.flags & MagicMasks.DEPTH_MASK.value) + + def dtype(self): + depth = self.depth() + ret = None + + if depth == Depth.CV_8U: + ret = (np.uint8, 'uint8_t') + elif depth == Depth.CV_8S: + ret = (np.int8, 'int8_t') + elif depth == Depth.CV_16U: + ret = (np.uint16, 'uint16_t') + elif depth == Depth.CV_16S: + ret = (np.int16, 'int16_t') + elif depth == Depth.CV_32S: + ret = (np.int32, 'int32_t') + elif depth == Depth.CV_32F: + ret = (np.float32, 'float') + elif depth == Depth.CV_64F: + ret = (np.float64, 'double') + elif depth == Depth.CV_16F: + ret = (np.float16, 'float16') + + return ret + + def type(self): + return Type(self.flags & MagicMasks.TYPE_MASK.value) + + def channels(self): + return ((self.flags & (511 << 3)) >> 3) + 1 + + def is_continuous(self): + return (self.flags & MagicValues.CONTINUOUS_FLAG.value) != 0 + + def is_submatrix(self): + return (self.flags & MagicValues.SUBMATRIX_FLAG.value) != 0 + + def __init__(self, flags): + self.flags = flags + + def __iter__(self): + return iter({ + 'type': stri(self.type().name), + 'is_continuous': booli(self.is_continuous()), + 'is_submatrix': booli(self.is_submatrix()) + }.items()) + + +class Size: + def __init__(self, ptr): + self.ptr = ptr + + def dims(self): + return int((self.ptr - 1).dereference()) + + def to_numpy(self): + return np.array([int(self.ptr[i]) for i in range(self.dims())], dtype=np.int64) + + def __iter__(self): + return iter({'size': stri(self.to_numpy())}.items()) + + +class Mat: + def __init__(self, m, size, flags): + (dtype, ctype) = flags.dtype() + elsize = np.dtype(dtype).itemsize + + ptr = m['data'] + dataptr = int(ptr) + length = (int(m['dataend']) - dataptr) // elsize + start = (int(m['datastart']) - dataptr) // elsize + + if length == 0: + self.mat = np.array([]) + self.view = self.mat + return + + if dtype != np.float16: + ctype = gdb.lookup_type(ctype) + ptr = ptr.cast(ctype.array(length - 1).pointer()).dereference() + self.mat = np.array([ptr[i] for i in range(length)], dtype=dtype) + else: + u16 = gdb.lookup_type('uint16_t') + ptr = ptr.cast(u16.array(length - 1).pointer()).dereference() + self.mat = np.array([ptr[i] for i in range(length)], dtype=np.uint16) + self.mat = self.mat.view(np.float16) + + steps = np.asarray([int(m['step']['p'][i]) for i in range(size.dims())], dtype=np.int64) + self.view = np.lib.stride_tricks.as_strided(self.mat[start:], shape=size.to_numpy(), strides=steps) + + def __iter__(self): + return iter({'data': stri(self.view)}.items()) + + +class MatPrinter: + """Print a cv::Mat""" + + def __init__(self, mat): + self.mat = mat + + def views(self): + m = self.mat + + flags = Flags(int(m['flags'])) + size = Size(m['size']['p']) + data = Mat(m, size, flags) + + for x in [flags, size, data]: + for k, v in x: + yield 'view_' + k, v + + def real(self): + m = self.mat + + for field in m.type.fields(): + k = field.name + v = m[k] + yield k, v + + # TODO: add an enum in interface.h with all cv::Mat element types and use that instead + # yield 'test', gdb.parse_and_eval(f'(cv::MatTypes)0') + + def children(self): # TODO: hide real members under new child somehow + yield from self.views() + yield from self.real() + + +def get_type(val): + # Get the type. + vtype = val.type + + # If it points to a reference, get the reference. + if vtype.code == gdb.TYPE_CODE_REF: + vtype = vtype.target() + + # Get the unqualified type, stripped of typedefs. + vtype = vtype.unqualified().strip_typedefs() + + # Get the type name. + typename = vtype.tag + + return typename + + +def mat_printer(val): + typename = get_type(val) + + if typename is None: + return None + + if str(typename) == 'cv::Mat': + return MatPrinter(val) + + +gdb.pretty_printers.append(mat_printer) From 210bfaf8d68b556697a90aba54729418a1f29290 Mon Sep 17 00:00:00 2001 From: thezane <10068531+thezane@users.noreply.github.com> Date: Tue, 17 Aug 2021 13:09:25 -0400 Subject: [PATCH 122/262] Merge pull request #20483 from thezane:support-cumsum-layer-for-onnx * Support cumsum layer for onnx * Add unit tests * Address review comments --- .../dnn/include/opencv2/dnn/all_layers.hpp | 9 ++ modules/dnn/src/init.cpp | 1 + modules/dnn/src/layers/cumsum_layer.cpp | 131 ++++++++++++++++++ modules/dnn/src/onnx/onnx_importer.cpp | 22 ++- modules/dnn/test/test_onnx_importer.cpp | 9 ++ 5 files changed, 171 insertions(+), 1 deletion(-) create mode 100644 modules/dnn/src/layers/cumsum_layer.cpp diff --git a/modules/dnn/include/opencv2/dnn/all_layers.hpp b/modules/dnn/include/opencv2/dnn/all_layers.hpp index 9c96c5a5f1..794bfeedda 100644 --- a/modules/dnn/include/opencv2/dnn/all_layers.hpp +++ b/modules/dnn/include/opencv2/dnn/all_layers.hpp @@ -723,6 +723,15 @@ CV__DNN_INLINE_NS_BEGIN static Ptr create(const LayerParams& params); }; + class CV_EXPORTS CumSumLayer : public Layer + { + public: + int exclusive; + int reverse; + + static Ptr create(const LayerParams& params); + }; + //! @} //! @} CV__DNN_INLINE_NS_END diff --git a/modules/dnn/src/init.cpp b/modules/dnn/src/init.cpp index ebd887999b..1916aa0ec9 100644 --- a/modules/dnn/src/init.cpp +++ b/modules/dnn/src/init.cpp @@ -140,6 +140,7 @@ void initializeLayerFactory() CV_DNN_REGISTER_LAYER_CLASS(LSTM, LSTMLayer); CV_DNN_REGISTER_LAYER_CLASS(GRU, GRULayer); + CV_DNN_REGISTER_LAYER_CLASS(CumSum, CumSumLayer); } CV__DNN_INLINE_NS_END diff --git a/modules/dnn/src/layers/cumsum_layer.cpp b/modules/dnn/src/layers/cumsum_layer.cpp new file mode 100644 index 0000000000..9c70f306d4 --- /dev/null +++ b/modules/dnn/src/layers/cumsum_layer.cpp @@ -0,0 +1,131 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. + +#include "../precomp.hpp" +#include "layers_common.hpp" + +#include + +namespace cv +{ +namespace dnn +{ + +class CumSumLayerImpl CV_FINAL : public CumSumLayer +{ +public: + CumSumLayerImpl(const LayerParams ¶ms) + { + axis_raw = params.get("axis", 0); + exclusive_raw = params.get("exclusive", 0); + reverse_raw = params.get("reverse", 0); + setParamsFrom(params); + } + + bool getMemoryShapes(const std::vector &inputs, + const int requiredOutputs, + std::vector &outputs, + std::vector &internals) const CV_OVERRIDE + { + Layer::getMemoryShapes(inputs, requiredOutputs, outputs, internals); + return true; + } + + void forward(InputArrayOfArrays inputs_arr, OutputArrayOfArrays outputs_arr, OutputArrayOfArrays internals_arr) CV_OVERRIDE + { + CV_TRACE_FUNCTION(); + CV_TRACE_ARG_VALUE(name, "name", name.c_str()); + + if (inputs_arr.depth() == CV_16S) + { + forward_fallback(inputs_arr, outputs_arr, internals_arr); + return; + } + + std::vector inputs, outputs, internals; + inputs_arr.getMatVector(inputs); + outputs_arr.getMatVector(outputs); + + // Get x tensor. + const auto &src_mat = inputs[0]; + const auto *src_ptr = src_mat.ptr(); + + // Get axis. + const int axis = normalize_axis(axis_raw, src_mat.dims); + + // Get y tensor. + auto &dst_mat = outputs[0]; + src_mat.copyTo(dst_mat); + auto *dst_ptr = dst_mat.ptr(); + + // Get flags. + const auto exclusive = exclusive_raw == 1; + const auto reverse = reverse_raw == 1; + + // Get parameters to iterate outer dimension. + const size_t outer_size = src_mat.total(0, axis); + const size_t outer_step_length = src_mat.total(axis); + + // Get parameters to iterate inner dimension. + const size_t inner_size = src_mat.size[axis]; + + if (!inner_size) + return; + + const size_t inner_step_length = src_mat.total(axis + 1); + const int inner_step = (reverse ? -1 : 1) * inner_step_length; + const int inner_start = reverse ? inner_size - 1 : 0; + const int inner_stop = reverse ? -1 : inner_size; + const int inner_delta = reverse ? -1 : 1; + + // Get parameters to populate channels. + const size_t num_channels = src_mat.total(axis + 1); + + for (size_t outer_dim = 0; outer_dim < outer_size; outer_dim++) + { + const size_t outer_offset = outer_dim * outer_step_length; + size_t src_offset = outer_offset + inner_start * inner_step_length; + + // Populate first element of inner dimension. + for (size_t channel = 0; channel < num_channels; channel++) + { + if (exclusive) + { + dst_ptr[src_offset + channel] = 0.0f; + } + else + { + dst_ptr[src_offset + channel] = src_ptr[src_offset + channel]; + src_offset += inner_step; + } + } + + // Populate remaining elements of inner dimension. + for (int inner_dim = inner_start + inner_delta; inner_dim != inner_stop; inner_dim += inner_delta) + { + const size_t dst_offset = outer_offset + inner_dim * inner_step_length; + + for (size_t channel = 0; channel < num_channels; channel++) + { + const size_t previous_dst_offset = dst_offset - inner_step; + dst_ptr[dst_offset + channel] = dst_ptr[previous_dst_offset + channel] + + src_ptr[src_offset + channel]; + src_offset += inner_step; + } + } + } + } + + int axis_raw; + int exclusive_raw; + int reverse_raw; +}; + +Ptr CumSumLayer::create(const LayerParams& params) +{ + return Ptr(new CumSumLayerImpl(params)); +} + +} +} diff --git a/modules/dnn/src/onnx/onnx_importer.cpp b/modules/dnn/src/onnx/onnx_importer.cpp index 946623fd40..3379ea3a0b 100644 --- a/modules/dnn/src/onnx/onnx_importer.cpp +++ b/modules/dnn/src/onnx/onnx_importer.cpp @@ -177,6 +177,7 @@ private: void parseUpsample (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); void parseSoftMax (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); void parseDetectionOutput (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseCumSum (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); void parseCustom (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); const DispatchMap dispatch; @@ -641,7 +642,8 @@ const std::set& ONNXImporter::getSupportedTypes() "Dropout", "Identity", "Crop", - "Normalize" + "Normalize", + "CumSum" }; return layerTypes; } @@ -2399,6 +2401,23 @@ void ONNXImporter::parseDetectionOutput(LayerParams& layerParams, const opencv_o addLayer(layerParams, node_proto); } +void ONNXImporter::parseCumSum(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +{ + layerParams.type = "CumSum"; + + // Get axis. + const std::string& input1 = node_proto.input(1); + + if (constBlobs.find(input1) != constBlobs.end()) + { + Mat axis_blob = getBlob(input1); + CV_Assert(axis_blob.total() == 1u); + layerParams.set("axis", axis_blob.at(0)); + } + + addLayer(layerParams, node_proto); +} + void ONNXImporter::parseCustom(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) { for (int j = 0; j < node_proto.input_size(); j++) { @@ -2456,6 +2475,7 @@ const ONNXImporter::DispatchMap ONNXImporter::buildDispatchMap() dispatch["Upsample"] = &ONNXImporter::parseUpsample; dispatch["SoftMax"] = dispatch["LogSoftmax"] = &ONNXImporter::parseSoftMax; dispatch["DetectionOutput"] = &ONNXImporter::parseDetectionOutput; + dispatch["CumSum"] = &ONNXImporter::parseCumSum; dispatch["Custom"] = &ONNXImporter::parseCustom; return dispatch; diff --git a/modules/dnn/test/test_onnx_importer.cpp b/modules/dnn/test/test_onnx_importer.cpp index 26e5fb632c..737a184cbd 100644 --- a/modules/dnn/test/test_onnx_importer.cpp +++ b/modules/dnn/test/test_onnx_importer.cpp @@ -1362,6 +1362,15 @@ TEST_P(Test_ONNX_nets, Resnet34_kinetics) expectNoFallbacksFromIE(net); } +TEST_P(Test_ONNX_layers, CumSum) +{ + testONNXModels("cumsum_1d_exclusive_1"); + testONNXModels("cumsum_1d_reverse"); + testONNXModels("cumsum_1d_exclusive_1_reverse"); + testONNXModels("cumsum_2d_dim_1"); + testONNXModels("cumsum_3d_dim_2"); +} + INSTANTIATE_TEST_CASE_P(/**/, Test_ONNX_nets, dnnBackendsAndTargets()); }} // namespace From 46fb88c76fbe8d4b7ff3cc8eb53e5842753f6c3e Mon Sep 17 00:00:00 2001 From: Sergey Ivanov Date: Tue, 17 Aug 2021 20:11:22 +0300 Subject: [PATCH 123/262] Merge pull request #20546 from sivanov-work:initial_vpl_source G-API: oneVPL (simplification) source base commit * oneVPL source initial * Fix compilation * Fix compilation path * Fix NO VPL compile * Fix unused vars * Fix unused vars in example * Simplify oneVPL search: no custom path & download * Fix standalone GAPI * Apply comments --- modules/gapi/CMakeLists.txt | 15 ++ modules/gapi/cmake/init.cmake | 7 + modules/gapi/cmake/standalone.cmake | 7 + .../gapi/streaming/onevpl/onevpl_source.hpp | 44 +++ .../gapi/samples/onevpl_infer_single_roi.cpp | 254 ++++++++++++++++++ .../src/streaming/onevpl/onevpl_source.cpp | 48 ++++ .../streaming/onevpl/onevpl_source_priv.cpp | 63 +++++ .../streaming/onevpl/onevpl_source_priv.hpp | 62 +++++ 8 files changed, 500 insertions(+) create mode 100644 modules/gapi/include/opencv2/gapi/streaming/onevpl/onevpl_source.hpp create mode 100644 modules/gapi/samples/onevpl_infer_single_roi.cpp create mode 100644 modules/gapi/src/streaming/onevpl/onevpl_source.cpp create mode 100644 modules/gapi/src/streaming/onevpl/onevpl_source_priv.cpp create mode 100644 modules/gapi/src/streaming/onevpl/onevpl_source_priv.hpp diff --git a/modules/gapi/CMakeLists.txt b/modules/gapi/CMakeLists.txt index c5046e8be6..69c0aaaae8 100644 --- a/modules/gapi/CMakeLists.txt +++ b/modules/gapi/CMakeLists.txt @@ -163,6 +163,10 @@ set(gapi_srcs src/backends/ie/bindings_ie.cpp src/backends/python/gpythonbackend.cpp + # Streaming source + src/streaming/onevpl/onevpl_source.cpp + src/streaming/onevpl/onevpl_source_priv.cpp + # Utils (ITT tracing) src/utils/itt.cpp ) @@ -234,6 +238,17 @@ if(HAVE_PLAIDML) ocv_target_include_directories(${the_module} SYSTEM PRIVATE ${PLAIDML_INCLUDE_DIRS}) endif() +if(HAVE_GAPI_ONEVPL) + if(TARGET opencv_test_gapi) + ocv_target_compile_definitions(opencv_test_gapi PRIVATE -DHAVE_ONEVPL) + ocv_target_link_libraries(opencv_test_gapi PRIVATE ${VPL_IMPORTED_TARGETS}) + endif() + ocv_target_compile_definitions(${the_module} PRIVATE -DHAVE_ONEVPL) + ocv_target_link_libraries(${the_module} PRIVATE ${VPL_IMPORTED_TARGETS}) + if(HAVE_D3D11 AND HAVE_OPENCL) + ocv_target_include_directories(${the_module} SYSTEM PRIVATE ${OPENCL_INCLUDE_DIRS}) + endif() +endif() if(WIN32) # Required for htonl/ntohl on Windows diff --git a/modules/gapi/cmake/init.cmake b/modules/gapi/cmake/init.cmake index 4c25c75f55..1c464328ca 100644 --- a/modules/gapi/cmake/init.cmake +++ b/modules/gapi/cmake/init.cmake @@ -32,3 +32,10 @@ if(WITH_PLAIDML) set(HAVE_PLAIDML TRUE) endif() endif() + +if(WITH_GAPI_ONEVPL) + find_package(VPL) + if(VPL_FOUND) + set(HAVE_GAPI_ONEVPL TRUE) + endif() +endif() diff --git a/modules/gapi/cmake/standalone.cmake b/modules/gapi/cmake/standalone.cmake index d08eda1be5..f81c1c8a85 100644 --- a/modules/gapi/cmake/standalone.cmake +++ b/modules/gapi/cmake/standalone.cmake @@ -6,6 +6,13 @@ if (NOT TARGET ade ) find_package(ade 0.1.0 REQUIRED) endif() +if (WITH_GAPI_ONEVPL) + find_package(VPL) + if(VPL_FOUND) + set(HAVE_GAPI_ONEVPL TRUE) + endif() +endif() + set(FLUID_TARGET fluid) set(FLUID_ROOT "${CMAKE_CURRENT_LIST_DIR}/../") diff --git a/modules/gapi/include/opencv2/gapi/streaming/onevpl/onevpl_source.hpp b/modules/gapi/include/opencv2/gapi/streaming/onevpl/onevpl_source.hpp new file mode 100644 index 0000000000..fec8c73dff --- /dev/null +++ b/modules/gapi/include/opencv2/gapi/streaming/onevpl/onevpl_source.hpp @@ -0,0 +1,44 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2021 Intel Corporation + +#ifndef OPENCV_GAPI_STREAMING_ONEVPL_ONEVPL_SOURCE_HPP +#define OPENCV_GAPI_STREAMING_ONEVPL_ONEVPL_SOURCE_HPP + +#include +#include +#include + +namespace cv { +namespace gapi { +namespace wip { + +class GAPI_EXPORTS OneVPLSource : public IStreamSource +{ +public: + struct Priv; + + explicit OneVPLSource(const std::string& filePath); + ~OneVPLSource() override; + + bool pull(cv::gapi::wip::Data& data) override; + GMetaArg descr_of() const override; + +private: + explicit OneVPLSource(std::unique_ptr&& impl); + std::unique_ptr m_priv; +}; + +template +GAPI_EXPORTS_W cv::Ptr inline make_vpl_src(const std::string& filePath, Args&&... args) +{ + return make_src(filePath, std::forward(args)...); +} + +} // namespace wip +} // namespace gapi +} // namespace cv + +#endif // OPENCV_GAPI_STREAMING_ONEVPL_ONEVPL_SOURCE_HPP diff --git a/modules/gapi/samples/onevpl_infer_single_roi.cpp b/modules/gapi/samples/onevpl_infer_single_roi.cpp new file mode 100644 index 0000000000..8a7efafabf --- /dev/null +++ b/modules/gapi/samples/onevpl_infer_single_roi.cpp @@ -0,0 +1,254 @@ +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include // CommandLineParser + +const std::string about = + "This is an OpenCV-based version of oneVPLSource decoder example"; +const std::string keys = + "{ h help | | Print this help message }" + "{ input | | Path to the input demultiplexed video file }" + "{ output | | Path to the output RAW video file. Use .avi extension }" + "{ facem | face-detection-adas-0001.xml | Path to OpenVINO IE face detection model (.xml) }" + "{ faced | CPU | Target device for face detection model (e.g. CPU, GPU, VPU, ...) }"; + +namespace { +std::string get_weights_path(const std::string &model_path) { + const auto EXT_LEN = 4u; + const auto sz = model_path.size(); + CV_Assert(sz > EXT_LEN); + + auto ext = model_path.substr(sz - EXT_LEN); + std::transform(ext.begin(), ext.end(), ext.begin(), [](unsigned char c){ + return static_cast(std::tolower(c)); + }); + CV_Assert(ext == ".xml"); + return model_path.substr(0u, sz - EXT_LEN) + ".bin"; +} +} // anonymous namespace + +namespace custom { +G_API_NET(FaceDetector, , "face-detector"); + +using GDetections = cv::GArray; +using GRect = cv::GOpaque; +using GSize = cv::GOpaque; +using GPrims = cv::GArray; + +G_API_OP(LocateROI, , "sample.custom.locate-roi") { + static cv::GOpaqueDesc outMeta(const cv::GOpaqueDesc &) { + return cv::empty_gopaque_desc(); + } +}; + +G_API_OP(ParseSSD, , "sample.custom.parse-ssd") { + static cv::GArrayDesc outMeta(const cv::GMatDesc &, const cv::GOpaqueDesc &, const cv::GOpaqueDesc &) { + return cv::empty_array_desc(); + } +}; + +G_API_OP(BBoxes, , "sample.custom.b-boxes") { + static cv::GArrayDesc outMeta(const cv::GArrayDesc &, const cv::GOpaqueDesc &) { + return cv::empty_array_desc(); + } +}; + +GAPI_OCV_KERNEL(OCVLocateROI, LocateROI) { + // This is the place where we can run extra analytics + // on the input image frame and select the ROI (region + // of interest) where we want to detect our objects (or + // run any other inference). + // + // Currently it doesn't do anything intelligent, + // but only crops the input image to square (this is + // the most convenient aspect ratio for detectors to use) + + static void run(const cv::Size& in_size, cv::Rect &out_rect) { + + // Identify the central point & square size (- some padding) + const auto center = cv::Point{in_size.width/2, in_size.height/2}; + auto sqside = std::min(in_size.width, in_size.height); + + // Now build the central square ROI + out_rect = cv::Rect{ center.x - sqside/2 + , center.y - sqside/2 + , sqside + , sqside + }; + } +}; + +GAPI_OCV_KERNEL(OCVParseSSD, ParseSSD) { + static void run(const cv::Mat &in_ssd_result, + const cv::Rect &in_roi, + const cv::Size &in_parent_size, + std::vector &out_objects) { + const auto &in_ssd_dims = in_ssd_result.size; + CV_Assert(in_ssd_dims.dims() == 4u); + + const int MAX_PROPOSALS = in_ssd_dims[2]; + const int OBJECT_SIZE = in_ssd_dims[3]; + CV_Assert(OBJECT_SIZE == 7); // fixed SSD object size + + const cv::Size up_roi = in_roi.size(); + const cv::Rect surface({0,0}, in_parent_size); + + out_objects.clear(); + + const float *data = in_ssd_result.ptr(); + for (int i = 0; i < MAX_PROPOSALS; i++) { + const float image_id = data[i * OBJECT_SIZE + 0]; + const float label = data[i * OBJECT_SIZE + 1]; + const float confidence = data[i * OBJECT_SIZE + 2]; + const float rc_left = data[i * OBJECT_SIZE + 3]; + const float rc_top = data[i * OBJECT_SIZE + 4]; + const float rc_right = data[i * OBJECT_SIZE + 5]; + const float rc_bottom = data[i * OBJECT_SIZE + 6]; + (void) label; // unused + + if (image_id < 0.f) { + break; // marks end-of-detections + } + if (confidence < 0.5f) { + continue; // skip objects with low confidence + } + + // map relative coordinates to the original image scale + // taking the ROI into account + cv::Rect rc; + rc.x = static_cast(rc_left * up_roi.width); + rc.y = static_cast(rc_top * up_roi.height); + rc.width = static_cast(rc_right * up_roi.width) - rc.x; + rc.height = static_cast(rc_bottom * up_roi.height) - rc.y; + rc.x += in_roi.x; + rc.y += in_roi.y; + out_objects.emplace_back(rc & surface); + } + } +}; + +GAPI_OCV_KERNEL(OCVBBoxes, BBoxes) { + // This kernel converts the rectangles into G-API's + // rendering primitives + static void run(const std::vector &in_face_rcs, + const cv::Rect &in_roi, + std::vector &out_prims) { + out_prims.clear(); + const auto cvt = [](const cv::Rect &rc, const cv::Scalar &clr) { + return cv::gapi::wip::draw::Rect(rc, clr, 2); + }; + out_prims.emplace_back(cvt(in_roi, CV_RGB(0,255,255))); // cyan + for (auto &&rc : in_face_rcs) { + out_prims.emplace_back(cvt(rc, CV_RGB(0,255,0))); // green + } + } +}; + +} // namespace custom + +int main(int argc, char *argv[]) { + + cv::CommandLineParser cmd(argc, argv, keys); + cmd.about(about); + if (cmd.has("help")) { + cmd.printMessage(); + return 0; + } + + // get file name + std::string file_path = cmd.get("input"); + const std::string output = cmd.get("output"); + const auto face_model_path = cmd.get("facem"); + + // check ouput file extension + if (!output.empty()) { + auto ext = output.find_last_of("."); + if (ext == std::string::npos || (output.substr(ext + 1) != "avi")) { + std::cerr << "Output file should have *.avi extension for output video" << std::endl; + return -1; + } + } + + auto face_net = cv::gapi::ie::Params { + face_model_path, // path to topology IR + get_weights_path(face_model_path), // path to weights + cmd.get("faced"), // device specifier + }; + auto kernels = cv::gapi::kernels + < custom::OCVLocateROI + , custom::OCVParseSSD + , custom::OCVBBoxes>(); + auto networks = cv::gapi::networks(face_net); + + // Create source + cv::Ptr cap; + try { + cap = cv::gapi::wip::make_vpl_src(file_path); + std::cout << "oneVPL source desription: " << cap->descr_of() << std::endl; + } catch (const std::exception& ex) { + std::cerr << "Cannot create source: " << ex.what() << std::endl; + return -1; + } + + cv::GMetaArg descr = cap->descr_of(); + auto frame_descr = cv::util::get(descr); + + // Now build the graph + cv::GFrame in; + auto size = cv::gapi::streaming::size(in); + auto roi = custom::LocateROI::on(size); + auto blob = cv::gapi::infer(roi, in); + auto rcs = custom::ParseSSD::on(blob, roi, size); + auto out_frame = cv::gapi::wip::draw::renderFrame(in, custom::BBoxes::on(rcs, roi)); + auto out = cv::gapi::streaming::BGR(out_frame); + + cv::GStreamingCompiled pipeline; + try { + pipeline = cv::GComputation(cv::GIn(in), cv::GOut(out)) + .compileStreaming(cv::compile_args(kernels, networks)); + } catch (const std::exception& ex) { + std::cerr << "Exception occured during pipeline construction: " << ex.what() << std::endl; + return -1; + } + // The execution part + + // TODO USE may set pool size from outside and set queue_capacity size, + // compile arg: cv::gapi::streaming::queue_capacity + pipeline.setSource(std::move(cap)); + pipeline.start(); + + int framesCount = 0; + cv::TickMeter t; + cv::VideoWriter writer; + if (!output.empty() && !writer.isOpened()) { + const auto sz = cv::Size{frame_descr.size.width, frame_descr.size.height}; + writer.open(output, cv::VideoWriter::fourcc('M','J','P','G'), 25.0, sz); + CV_Assert(writer.isOpened()); + } + + cv::Mat outMat; + t.start(); + while (pipeline.pull(cv::gout(outMat))) { + cv::imshow("Out", outMat); + cv::waitKey(1); + if (!output.empty()) { + writer << outMat; + } + framesCount++; + } + t.stop(); + std::cout << "Elapsed time: " << t.getTimeSec() << std::endl; + std::cout << "FPS: " << framesCount / t.getTimeSec() << std::endl; + std::cout << "framesCount: " << framesCount << std::endl; + + return 0; +} diff --git a/modules/gapi/src/streaming/onevpl/onevpl_source.cpp b/modules/gapi/src/streaming/onevpl/onevpl_source.cpp new file mode 100644 index 0000000000..988986f6d9 --- /dev/null +++ b/modules/gapi/src/streaming/onevpl/onevpl_source.cpp @@ -0,0 +1,48 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2021 Intel Corporation + +#include + +#include "streaming/onevpl/onevpl_source_priv.hpp" + +namespace cv { +namespace gapi { +namespace wip { + +#ifdef HAVE_ONEVPL +OneVPLSource::OneVPLSource(const std::string& filePath) : + OneVPLSource(std::unique_ptr(new OneVPLSource::Priv(filePath))) { + + if (filePath.empty()) { + util::throw_error(std::logic_error("Cannot create 'OneVPLSource' on empty source file name")); + } +} +#else +OneVPLSource::OneVPLSource(const std::string&) { + GAPI_Assert(false && "Unsupported: G-API compiled without `WITH_GAPI_ONEVPL=ON`"); +} +#endif + +OneVPLSource::OneVPLSource(std::unique_ptr&& impl) : + IStreamSource(), + m_priv(std::move(impl)) { +} + +OneVPLSource::~OneVPLSource() { +} + +bool OneVPLSource::pull(cv::gapi::wip::Data& data) +{ + return m_priv->pull(data); +} + +GMetaArg OneVPLSource::descr_of() const +{ + return m_priv->descr_of(); +} +} // namespace wip +} // namespace gapi +} // namespace cv diff --git a/modules/gapi/src/streaming/onevpl/onevpl_source_priv.cpp b/modules/gapi/src/streaming/onevpl/onevpl_source_priv.cpp new file mode 100644 index 0000000000..5c4e8e6941 --- /dev/null +++ b/modules/gapi/src/streaming/onevpl/onevpl_source_priv.cpp @@ -0,0 +1,63 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2021 Intel Corporation + +#include +#include + +#include "streaming/onevpl/onevpl_source_priv.hpp" +#include "logger.hpp" + +#ifndef HAVE_ONEVPL +namespace cv { +namespace gapi { +namespace wip { +bool OneVPLSource::Priv::pull(cv::gapi::wip::Data&) { + return true; +} +GMetaArg OneVPLSource::Priv::descr_of() const { + return {}; +} +} // namespace wip +} // namespace gapi +} // namespace cv + +#else // HAVE_ONEVPL + +namespace cv { +namespace gapi { +namespace wip { +OneVPLSource::Priv::Priv() : + mfx_handle(MFXLoad()) +{ + GAPI_LOG_INFO(nullptr, "Initialized MFX handle: " << mfx_handle); + description_is_valid = false; +} + +OneVPLSource::Priv::Priv(const std::string&) : + OneVPLSource::Priv() +{ +} + +OneVPLSource::Priv::~Priv() +{ + GAPI_LOG_INFO(nullptr, "Unload MFX handle: " << mfx_handle); + MFXUnload(mfx_handle); +} + +bool OneVPLSource::Priv::pull(cv::gapi::wip::Data&) +{ + return false; +} + +GMetaArg OneVPLSource::Priv::descr_of() const +{ + return {}; +} +} // namespace wip +} // namespace gapi +} // namespace cv + +#endif // HAVE_ONEVPL diff --git a/modules/gapi/src/streaming/onevpl/onevpl_source_priv.hpp b/modules/gapi/src/streaming/onevpl/onevpl_source_priv.hpp new file mode 100644 index 0000000000..b139add993 --- /dev/null +++ b/modules/gapi/src/streaming/onevpl/onevpl_source_priv.hpp @@ -0,0 +1,62 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2021 Intel Corporation + +#ifndef OPENCV_GAPI_STREAMING_ONEVPL_ONEVPL_SOURCE_PRIV_HPP +#define OPENCV_GAPI_STREAMING_ONEVPL_ONEVPL_SOURCE_PRIV_HPP + +#include + +#include +#include + +#include +#include +#include + +#ifdef HAVE_ONEVPL +#if (MFX_VERSION >= 2000) +#include +#endif // MFX_VERSION + +#include + +#include + +namespace cv { +namespace gapi { +namespace wip { + +struct OneVPLSource::Priv +{ + explicit Priv(const std::string& file_path); + ~Priv(); + + bool pull(cv::gapi::wip::Data& data); + GMetaArg descr_of() const; +private: + Priv(); + mfxLoader mfx_handle; + bool description_is_valid; +}; +} // namespace wip +} // namespace gapi +} // namespace cv + +#else // HAVE_ONEVPL + +namespace cv { +namespace gapi { +namespace wip { +struct OneVPLSource::Priv final +{ + bool pull(cv::gapi::wip::Data&); + GMetaArg descr_of() const; +}; +} // namespace wip +} // namespace gapi +} // namespace cv +#endif // HAVE_ONEVPL +#endif // OPENCV_GAPI_STREAMING_ONEVPL_ONEVPL_SOURCE_PRIV_HPP From 95919051e0470237dd508a2086d8e519f5e091bf Mon Sep 17 00:00:00 2001 From: Anatoliy Talamanov Date: Wed, 18 Aug 2021 10:42:32 +0300 Subject: [PATCH 124/262] Merge pull request #20528 from TolyaTalamanov:at/fix-overwrite-blob-precision-bug [G-API] Prohibit setPrecision & preprocessing for importedNetworks * Prohibit preprocessing for imported networks * Fix typo input_names -> output_names * Move setBlob logic to separate function * Change comment --- modules/gapi/src/backends/ie/giebackend.cpp | 192 +++++++++++------- .../src/backends/ie/giebackend/giewrapper.cpp | 18 -- .../src/backends/ie/giebackend/giewrapper.hpp | 3 - 3 files changed, 120 insertions(+), 93 deletions(-) diff --git a/modules/gapi/src/backends/ie/giebackend.cpp b/modules/gapi/src/backends/ie/giebackend.cpp index fc9fc502ef..007f0db7af 100644 --- a/modules/gapi/src/backends/ie/giebackend.cpp +++ b/modules/gapi/src/backends/ie/giebackend.cpp @@ -243,10 +243,6 @@ struct IEUnit { this_plugin = cv::gimpl::ie::wrap::getPlugin(params); this_plugin.SetConfig(params.config); this_network = cv::gimpl::ie::wrap::importNetwork(this_plugin, params, rctx); - // FIXME: ICNNetwork returns InputsDataMap/OutputsDataMap, - // but ExecutableNetwork returns ConstInputsDataMap/ConstOutputsDataMap - inputs = cv::gimpl::ie::wrap::toInputsDataMap(this_network.GetInputsInfo()); - outputs = cv::gimpl::ie::wrap::toOutputsDataMap(this_network.GetOutputsInfo()); if (!params.reshape_table.empty() || !params.layer_names_to_reshape.empty()) { GAPI_LOG_WARNING(NULL, "Reshape isn't supported for imported network"); } @@ -270,10 +266,18 @@ struct IEUnit { + params.model_path)); } if (params.num_in == 1u && params.input_names.empty()) { - params.input_names = { inputs.begin()->first }; + if (params.kind == cv::gapi::ie::detail::ParamDesc::Kind::Load) { + params.input_names = { inputs.begin()->first }; + } else { + params.input_names = { this_network.GetInputsInfo().begin()->first }; + } } if (params.num_out == 1u && params.output_names.empty()) { - params.output_names = { outputs.begin()->first }; + if (params.kind == cv::gapi::ie::detail::ParamDesc::Kind::Load) { + params.output_names = { outputs.begin()->first }; + } else { + params.output_names = { this_network.GetOutputsInfo().begin()->first }; + } } if (!params.reshape_table.empty()) { GAPI_Assert((params.reshape_table.size() + params.layer_names_to_reshape.size()) <= @@ -533,6 +537,24 @@ inline IE::Blob::Ptr extractBlob(IECallContext& ctx, std::size_t i) { } GAPI_Assert(false); } + + +static void setBlob(InferenceEngine::InferRequest& req, + cv::gapi::ie::detail::ParamDesc::Kind kind, + const std::string& layer_name, + IE::Blob::Ptr blob) { + // NB: In case importNetwork preprocessing must be + // passed as SetBlob argument. + if (kind == cv::gapi::ie::detail::ParamDesc::Kind::Load) { + req.SetBlob(layer_name, blob); + } else { + GAPI_Assert(kind == cv::gapi::ie::detail::ParamDesc::Kind::Import); + IE::PreProcessInfo info; + info.setResizeAlgorithm(IE::RESIZE_BILINEAR); + req.SetBlob(layer_name, blob, info); + } +} + } // anonymous namespace std::vector cv::gimpl::ie::IECompiled::createInferRequests() { @@ -891,25 +913,30 @@ struct Infer: public cv::detail::KernelTag { // meta order. GAPI_Assert(uu.params.input_names.size() == in_metas.size() && "Known input layers count doesn't match input meta count"); - for (auto &&it : ade::util::zip(ade::util::toRange(uu.params.input_names), - ade::util::toRange(in_metas))) { - const auto &input_name = std::get<0>(it); - auto &&ii = uu.inputs.at(input_name); - const auto & mm = std::get<1>(it); - configureInputInfo(ii, mm); - if (uu.params.layer_names_to_reshape.find(input_name) != - uu.params.layer_names_to_reshape.end()) { - configureInputReshapeByImage(ii, mm, input_reshape_table); + // NB: Configuring input precision and network reshape must be done + // only in the loadNetwork case. + if (uu.params.kind == cv::gapi::ie::detail::ParamDesc::Kind::Load) { + for (auto &&it : ade::util::zip(ade::util::toRange(uu.params.input_names), + ade::util::toRange(in_metas))) { + const auto &input_name = std::get<0>(it); + auto &&ii = uu.inputs.at(input_name); + const auto & mm = std::get<1>(it); + + configureInputInfo(ii, mm); + if (uu.params.layer_names_to_reshape.find(input_name) != + uu.params.layer_names_to_reshape.end()) { + configureInputReshapeByImage(ii, mm, input_reshape_table); + } + ii->getPreProcess().setResizeAlgorithm(IE::RESIZE_BILINEAR); } - ii->getPreProcess().setResizeAlgorithm(IE::RESIZE_BILINEAR); - } - // FIXME: This isn't the best place to call reshape function. - // Сorrect solution would be to do this in compile() method of network, - // but now input meta isn't passed to compile() method. - if (!input_reshape_table.empty()) { - const_cast(&uu.net)->reshape(input_reshape_table); + // FIXME: This isn't the best place to call reshape function. + // Сorrect solution would be to do this in compile() method of network, + // but now input meta isn't passed to compile() method. + if (!input_reshape_table.empty()) { + const_cast(&uu.net)->reshape(input_reshape_table); + } } // FIXME: It would be nice here to have an exact number of network's @@ -941,7 +968,10 @@ struct Infer: public cv::detail::KernelTag { // and redirect our data producers to this memory // (A memory dialog comes to the picture again) IE::Blob::Ptr this_blob = extractBlob(*ctx, i); - req.SetBlob(ctx->uu.params.input_names[i], this_blob); + setBlob(req, + ctx->uu.params.kind, + ctx->uu.params.input_names[i], + this_blob); } // FIXME: Should it be done by kernel ? // What about to do that in RequestPool ? @@ -973,22 +1003,26 @@ struct InferROI: public cv::detail::KernelTag { GAPI_Assert(1u == uu.params.input_names.size()); GAPI_Assert(2u == in_metas.size()); - // 0th is ROI, 1st is input image - const auto &input_name = uu.params.input_names.at(0); - auto &&ii = uu.inputs.at(input_name); - auto &&mm = in_metas.at(1u); - configureInputInfo(ii, mm); - if (uu.params.layer_names_to_reshape.find(input_name) != - uu.params.layer_names_to_reshape.end()) { - configureInputReshapeByImage(ii, mm, input_reshape_table); - } - ii->getPreProcess().setResizeAlgorithm(IE::RESIZE_BILINEAR); + // NB: Configuring input precision and network reshape must be done + // only in the loadNetwork case. + if (uu.params.kind == cv::gapi::ie::detail::ParamDesc::Kind::Load) { + // 0th is ROI, 1st is input image + const auto &input_name = uu.params.input_names.at(0); + auto &&ii = uu.inputs.at(input_name); + auto &&mm = in_metas.at(1u); + configureInputInfo(ii, mm); + if (uu.params.layer_names_to_reshape.find(input_name) != + uu.params.layer_names_to_reshape.end()) { + configureInputReshapeByImage(ii, mm, input_reshape_table); + } + ii->getPreProcess().setResizeAlgorithm(IE::RESIZE_BILINEAR); - // FIXME: This isn't the best place to call reshape function. - // Сorrect solution would be to do this in compile() method of network, - // but now input meta isn't passed to compile() method. - if (!input_reshape_table.empty()) { - const_cast(&uu.net)->reshape(input_reshape_table); + // FIXME: This isn't the best place to call reshape function. + // Сorrect solution would be to do this in compile() method of network, + // but now input meta isn't passed to compile() method. + if (!input_reshape_table.empty()) { + const_cast(&uu.net)->reshape(input_reshape_table); + } } // FIXME: It would be nice here to have an exact number of network's @@ -1017,10 +1051,11 @@ struct InferROI: public cv::detail::KernelTag { auto&& this_roi = ctx->inArg(0).rref(); IE::Blob::Ptr this_blob = extractBlob(*ctx, 1); - - req.SetBlob(*(ctx->uu.params.input_names.begin()), - IE::make_shared_blob(this_blob, toIE(this_roi))); - + setBlob(req, + ctx->uu.params.kind, + *(ctx->uu.params.input_names.begin()), + IE::make_shared_blob(this_blob, + toIE(this_roi))); // FIXME: Should it be done by kernel ? // What about to do that in RequestPool ? req.StartAsync(); @@ -1055,23 +1090,27 @@ struct InferList: public cv::detail::KernelTag { GAPI_Assert(uu.params.input_names.size() == (in_metas.size() - 1u) && "Known input layers count doesn't match input meta count"); - std::size_t idx = 1u; - for (auto &&input_name : uu.params.input_names) { - auto &&ii = uu.inputs.at(input_name); - const auto & mm = in_metas[idx++]; - configureInputInfo(ii, mm); - if (uu.params.layer_names_to_reshape.find(input_name) != - uu.params.layer_names_to_reshape.end()) { - configureInputReshapeByImage(ii, mm, input_reshape_table); + // NB: Configuring input precision and network reshape must be done + // only in the loadNetwork case. + if (uu.params.kind == cv::gapi::ie::detail::ParamDesc::Kind::Load) { + std::size_t idx = 1u; + for (auto &&input_name : uu.params.input_names) { + auto &&ii = uu.inputs.at(input_name); + const auto & mm = in_metas[idx++]; + configureInputInfo(ii, mm); + if (uu.params.layer_names_to_reshape.find(input_name) != + uu.params.layer_names_to_reshape.end()) { + configureInputReshapeByImage(ii, mm, input_reshape_table); + } + ii->getPreProcess().setResizeAlgorithm(IE::RESIZE_BILINEAR); } - ii->getPreProcess().setResizeAlgorithm(IE::RESIZE_BILINEAR); - } - // FIXME: This isn't the best place to call reshape function. - // Сorrect solution would be to do this in compile() method of network, - // but now input meta isn't passed to compile() method. - if (!input_reshape_table.empty()) { - const_cast(&uu.net)->reshape(input_reshape_table); + // FIXME: This isn't the best place to call reshape function. + // Сorrect solution would be to do this in compile() method of network, + // but now input meta isn't passed to compile() method. + if (!input_reshape_table.empty()) { + const_cast(&uu.net)->reshape(input_reshape_table); + } } // roi-list version is much easier at the moment. @@ -1117,7 +1156,10 @@ struct InferList: public cv::detail::KernelTag { cv::gimpl::ie::RequestPool::Task { [ctx, rc, this_blob](InferenceEngine::InferRequest &req) { IE::Blob::Ptr roi_blob = IE::make_shared_blob(this_blob, toIE(rc)); - req.SetBlob(ctx->uu.params.input_names[0u], roi_blob); + setBlob(req, + ctx->uu.params.kind, + ctx->uu.params.input_names[0u], + roi_blob); req.StartAsync(); }, std::bind(callback, std::placeholders::_1, pos) @@ -1191,19 +1233,23 @@ struct InferList2: public cv::detail::KernelTag { && "Non-array inputs are not supported"); if (op.k.inKinds[idx] == cv::detail::OpaqueKind::CV_RECT) { - // This is a cv::Rect -- configure the IE preprocessing - configureInputInfo(ii, mm_0); - if (uu.params.layer_names_to_reshape.find(input_name) != - uu.params.layer_names_to_reshape.end()) { - configureInputReshapeByImage(ii, mm_0, input_reshape_table); - } - ii->getPreProcess().setResizeAlgorithm(IE::RESIZE_BILINEAR); + // NB: Configuring input precision and network reshape must be done + // only in the loadNetwork case. + if (uu.params.kind == cv::gapi::ie::detail::ParamDesc::Kind::Load) { + // This is a cv::Rect -- configure the IE preprocessing + configureInputInfo(ii, mm_0); + if (uu.params.layer_names_to_reshape.find(input_name) != + uu.params.layer_names_to_reshape.end()) { + configureInputReshapeByImage(ii, mm_0, input_reshape_table); + } + ii->getPreProcess().setResizeAlgorithm(IE::RESIZE_BILINEAR); - // FIXME: This isn't the best place to call reshape function. - // Сorrect solution would be to do this in compile() method of network, - // but now input meta isn't passed to compile() method. - if (!input_reshape_table.empty()) { - const_cast(&uu.net)->reshape(input_reshape_table); + // FIXME: This isn't the best place to call reshape function. + // Сorrect solution would be to do this in compile() method of network, + // but now input meta isn't passed to compile() method. + if (!input_reshape_table.empty()) { + const_cast(&uu.net)->reshape(input_reshape_table); + } } } else { // This is a cv::GMat (equals to: cv::Mat) @@ -1268,8 +1314,10 @@ struct InferList2: public cv::detail::KernelTag { GAPI_Assert(false && "Only Rect and Mat types are supported for infer list 2!"); } - - req.SetBlob(ctx->uu.params.input_names[in_idx], this_blob); + setBlob(req, + ctx->uu.params.kind, + ctx->uu.params.input_names[in_idx], + this_blob); } req.StartAsync(); }, diff --git a/modules/gapi/src/backends/ie/giebackend/giewrapper.cpp b/modules/gapi/src/backends/ie/giebackend/giewrapper.cpp index d4ec806e48..1f9721dbf4 100644 --- a/modules/gapi/src/backends/ie/giebackend/giewrapper.cpp +++ b/modules/gapi/src/backends/ie/giebackend/giewrapper.cpp @@ -22,24 +22,6 @@ namespace IE = InferenceEngine; namespace giewrap = cv::gimpl::ie::wrap; using GIEParam = cv::gapi::ie::detail::ParamDesc; -IE::InputsDataMap giewrap::toInputsDataMap (const IE::ConstInputsDataMap& inputs) { - IE::InputsDataMap transformed; - auto convert = [](const std::pair& p) { - return std::make_pair(p.first, std::const_pointer_cast(p.second)); - }; - std::transform(inputs.begin(), inputs.end(), std::inserter(transformed, transformed.end()), convert); - return transformed; -} - -IE::OutputsDataMap giewrap::toOutputsDataMap (const IE::ConstOutputsDataMap& outputs) { - IE::OutputsDataMap transformed; - auto convert = [](const std::pair& p) { - return std::make_pair(p.first, std::const_pointer_cast(p.second)); - }; - std::transform(outputs.begin(), outputs.end(), std::inserter(transformed, transformed.end()), convert); - return transformed; -} - #if INF_ENGINE_RELEASE < 2020000000 // < 2020.1 // Load extensions (taken from DNN module) std::vector giewrap::getExtensions(const GIEParam& params) { diff --git a/modules/gapi/src/backends/ie/giebackend/giewrapper.hpp b/modules/gapi/src/backends/ie/giebackend/giewrapper.hpp index 7e67cb8989..2e4bac1270 100644 --- a/modules/gapi/src/backends/ie/giebackend/giewrapper.hpp +++ b/modules/gapi/src/backends/ie/giebackend/giewrapper.hpp @@ -29,9 +29,6 @@ namespace wrap { GAPI_EXPORTS std::vector getExtensions(const GIEParam& params); GAPI_EXPORTS IE::CNNNetwork readNetwork(const GIEParam& params); -IE::InputsDataMap toInputsDataMap (const IE::ConstInputsDataMap& inputs); -IE::OutputsDataMap toOutputsDataMap(const IE::ConstOutputsDataMap& outputs); - #if INF_ENGINE_RELEASE < 2019020000 // < 2019.R2 using Plugin = IE::InferencePlugin; GAPI_EXPORTS IE::InferencePlugin getPlugin(const GIEParam& params); From 03b989251d30efff3e90e0b5e02bae1e2c4d825f Mon Sep 17 00:00:00 2001 From: Anatoliy Talamanov Date: Wed, 18 Aug 2021 18:12:27 +0300 Subject: [PATCH 125/262] Check adapter in executor --- modules/gapi/src/executor/gexecutor.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/gapi/src/executor/gexecutor.cpp b/modules/gapi/src/executor/gexecutor.cpp index 6f313197ba..9b51e70d5d 100644 --- a/modules/gapi/src/executor/gexecutor.cpp +++ b/modules/gapi/src/executor/gexecutor.cpp @@ -159,8 +159,8 @@ void writeBackExec(const Mag& mag, const RcDesc &rc, GRunArgP &g_arg) // a real copy (add a pass to StreamingBackend?) auto& out_mat = *util::get(g_arg); const auto& rmat = mag.template slot().at(rc.id); - auto mag_data = rmat.get()->data(); - if (out_mat.data != mag_data) { + auto* adapter = rmat.get(); + if (adapter != nullptr && out_mat.data != adapter->data()) { auto view = rmat.access(RMat::Access::R); asMat(view).copyTo(out_mat); } From fe625a558e0c9ab56f781c47bb0db25079368681 Mon Sep 17 00:00:00 2001 From: Smirnov Egor Date: Wed, 18 Aug 2021 18:37:35 +0300 Subject: [PATCH 126/262] fix hasDynamicShapes for batch_size and fix axis selection in Scale layer --- modules/dnn/src/onnx/onnx_importer.cpp | 13 ++++++++++++- modules/dnn/test/test_onnx_importer.cpp | 2 ++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/modules/dnn/src/onnx/onnx_importer.cpp b/modules/dnn/src/onnx/onnx_importer.cpp index 32b56278bd..dd84e3b7cd 100644 --- a/modules/dnn/src/onnx/onnx_importer.cpp +++ b/modules/dnn/src/onnx/onnx_importer.cpp @@ -475,7 +475,8 @@ void ONNXImporter::populateNet() for (int j = 0; j < inpShape.size(); ++j) { inpShape[j] = tensorShape.dim(j).dim_value(); - if (!tensorShape.dim(j).dim_param().empty()) + // NHW, NCHW(NHWC), NCDHW(NDHWC); do not set this flag if only N is dynamic + if (!tensorShape.dim(j).dim_param().empty() && !(j == 0 && inpShape.size() >= 3)) hasDynamicShapes = true; } if (!inpShape.empty() && !hasDynamicShapes) @@ -1407,6 +1408,16 @@ void ONNXImporter::parseMul(LayerParams& layerParams, const opencv_onnx::NodePro //Replace input to Power node_proto.set_input(1, powerParams.name); } + + const MatShape& broadShape = outShapes[node_proto.input(1)]; + const size_t outShapeSize = outShapes[node_proto.input(0)].size(); + const size_t diff = outShapeSize - broadShape.size(); + + size_t axis; + for (axis = diff; axis < broadShape.size() && broadShape[axis - diff] == 1; ++axis) {} + + CV_Assert(axis != outShapeSize); + layerParams.set("axis", static_cast(axis)); layerParams.type = "Scale"; } addLayer(layerParams, node_proto); diff --git a/modules/dnn/test/test_onnx_importer.cpp b/modules/dnn/test/test_onnx_importer.cpp index a446a37c79..d13b39064e 100644 --- a/modules/dnn/test/test_onnx_importer.cpp +++ b/modules/dnn/test/test_onnx_importer.cpp @@ -284,6 +284,7 @@ TEST_P(Test_ONNX_layers, Scale) if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_NN_BUILDER); testONNXModels("scale"); + testONNXModels("scale_broadcast", npy, 0, 0, false, true, 3); } TEST_P(Test_ONNX_layers, ReduceMean3D) @@ -825,6 +826,7 @@ TEST_P(Test_ONNX_layers, DynamicAxes) testONNXModels("resize_opset11_torch1.6_dynamic_axes"); testONNXModels("average_pooling_dynamic_axes"); testONNXModels("maxpooling_sigmoid_dynamic_axes"); + testONNXModels("dynamic_batch"); } TEST_P(Test_ONNX_layers, MaxPool1d) From fa90e14b0606b76a4efc9739c4ea87e659c2aab8 Mon Sep 17 00:00:00 2001 From: SamFC10 Date: Thu, 19 Aug 2021 09:56:47 +0530 Subject: [PATCH 127/262] int8 layers and 8-bit quantization support --- modules/dnn/CMakeLists.txt | 1 + .../dnn/include/opencv2/dnn/all_layers.hpp | 87 +- modules/dnn/include/opencv2/dnn/dnn.hpp | 46 + modules/dnn/src/dnn.cpp | 383 +++++- modules/dnn/src/init.cpp | 38 + .../dnn/src/int8layers/batch_norm_layer.cpp | 178 +++ .../dnn/src/int8layers/convolution_layer.cpp | 1136 +++++++++++++++ .../dnn/src/int8layers/elementwise_layers.cpp | 190 +++ modules/dnn/src/int8layers/eltwise_layer.cpp | 577 ++++++++ .../src/int8layers/fully_connected_layer.cpp | 266 ++++ modules/dnn/src/int8layers/layers_common.hpp | 41 + .../dnn/src/int8layers/layers_common.simd.hpp | 637 +++++++++ modules/dnn/src/int8layers/pooling_layer.cpp | 595 ++++++++ .../int8layers/quantize_dequantize_layer.cpp | 157 +++ modules/dnn/src/int8layers/scale_layer.cpp | 211 +++ modules/dnn/src/int8layers/softmax_layer.cpp | 176 +++ modules/dnn/src/layers/batch_norm_layer.cpp | 12 + modules/dnn/src/layers/blank_layer.cpp | 5 + modules/dnn/src/layers/concat_layer.cpp | 32 +- modules/dnn/src/layers/const_layer.cpp | 9 + modules/dnn/src/layers/convolution_layer.cpp | 42 + modules/dnn/src/layers/elementwise_layers.cpp | 179 +++ modules/dnn/src/layers/eltwise_layer.cpp | 31 + modules/dnn/src/layers/flatten_layer.cpp | 5 + .../dnn/src/layers/fully_connected_layer.cpp | 39 + modules/dnn/src/layers/padding_layer.cpp | 12 + modules/dnn/src/layers/permute_layer.cpp | 65 +- modules/dnn/src/layers/pooling_layer.cpp | 17 + modules/dnn/src/layers/reorg_layer.cpp | 5 + modules/dnn/src/layers/reshape_layer.cpp | 5 + modules/dnn/src/layers/scale_layer.cpp | 8 + .../dnn/src/layers/shuffle_channel_layer.cpp | 6 + modules/dnn/src/layers/slice_layer.cpp | 30 +- modules/dnn/src/layers/softmax_layer.cpp | 16 + modules/dnn/src/layers/split_layer.cpp | 11 + modules/dnn/test/test_int8_layers.cpp | 1220 +++++++++++++++++ 36 files changed, 6401 insertions(+), 67 deletions(-) create mode 100644 modules/dnn/src/int8layers/batch_norm_layer.cpp create mode 100644 modules/dnn/src/int8layers/convolution_layer.cpp create mode 100644 modules/dnn/src/int8layers/elementwise_layers.cpp create mode 100644 modules/dnn/src/int8layers/eltwise_layer.cpp create mode 100644 modules/dnn/src/int8layers/fully_connected_layer.cpp create mode 100644 modules/dnn/src/int8layers/layers_common.hpp create mode 100644 modules/dnn/src/int8layers/layers_common.simd.hpp create mode 100644 modules/dnn/src/int8layers/pooling_layer.cpp create mode 100644 modules/dnn/src/int8layers/quantize_dequantize_layer.cpp create mode 100644 modules/dnn/src/int8layers/scale_layer.cpp create mode 100644 modules/dnn/src/int8layers/softmax_layer.cpp create mode 100644 modules/dnn/test/test_int8_layers.cpp diff --git a/modules/dnn/CMakeLists.txt b/modules/dnn/CMakeLists.txt index 70f9a5a73e..c1236c4653 100644 --- a/modules/dnn/CMakeLists.txt +++ b/modules/dnn/CMakeLists.txt @@ -9,6 +9,7 @@ endif() set(the_description "Deep neural network module. It allows to load models from different frameworks and to make forward pass") ocv_add_dispatched_file_force_all("layers/layers_common" AVX AVX2 AVX512_SKX RVV) +ocv_add_dispatched_file_force_all("int8layers/layers_common" AVX2 AVX512_SKX) ocv_add_module(dnn opencv_core opencv_imgproc WRAP python java objc js) diff --git a/modules/dnn/include/opencv2/dnn/all_layers.hpp b/modules/dnn/include/opencv2/dnn/all_layers.hpp index 794bfeedda..fbe16850d4 100644 --- a/modules/dnn/include/opencv2/dnn/all_layers.hpp +++ b/modules/dnn/include/opencv2/dnn/all_layers.hpp @@ -258,6 +258,14 @@ CV__DNN_INLINE_NS_BEGIN static Ptr create(const LayerParams& params); }; + class CV_EXPORTS ConvolutionLayerInt8 : public BaseConvolutionLayer + { + public: + int input_zp, output_zp; + float output_sc; + static Ptr create(const LayerParams& params); + }; + class CV_EXPORTS DeconvolutionLayer : public BaseConvolutionLayer { public: @@ -300,6 +308,13 @@ CV__DNN_INLINE_NS_BEGIN static Ptr create(const LayerParams& params); }; + class CV_EXPORTS PoolingLayerInt8 : public PoolingLayer + { + public: + int input_zp, output_zp; + static Ptr create(const LayerParams& params); + }; + class CV_EXPORTS SoftmaxLayer : public Layer { public: @@ -308,6 +323,14 @@ CV__DNN_INLINE_NS_BEGIN static Ptr create(const LayerParams& params); }; + class CV_EXPORTS SoftmaxLayerInt8 : public SoftmaxLayer + { + public: + float output_sc; + int output_zp; + static Ptr create(const LayerParams& params); + }; + class CV_EXPORTS InnerProductLayer : public Layer { public: @@ -315,6 +338,13 @@ CV__DNN_INLINE_NS_BEGIN static Ptr create(const LayerParams& params); }; + class CV_EXPORTS InnerProductLayerInt8 : public InnerProductLayer + { + public: + int output_zp; + static Ptr create(const LayerParams& params); + }; + class CV_EXPORTS MVNLayer : public Layer { public: @@ -341,6 +371,22 @@ CV__DNN_INLINE_NS_BEGIN static Ptr create(const LayerParams ¶ms); }; + class CV_EXPORTS QuantizeLayer : public Layer + { + public: + float scale; + int zeropoint; + static Ptr create(const LayerParams ¶ms); + }; + + class CV_EXPORTS DequantizeLayer : public Layer + { + public: + float scale; + int zeropoint; + static Ptr create(const LayerParams ¶ms); + }; + class CV_EXPORTS ConcatLayer : public Layer { public: @@ -352,6 +398,7 @@ CV__DNN_INLINE_NS_BEGIN * Details: https://github.com/torch/nn/blob/master/doc/containers.md#depthconcat */ bool padding; + int paddingValue; static Ptr create(const LayerParams ¶ms); }; @@ -459,7 +506,11 @@ CV__DNN_INLINE_NS_BEGIN { public: virtual void forwardSlice(const float* src, float* dst, int len, - size_t outPlaneSize, int cn0, int cn1) const = 0; + size_t outPlaneSize, int cn0, int cn1) const {}; + virtual void forwardSlice(const int* src, const int* lut, int* dst, int len, + size_t outPlaneSize, int cn0, int cn1) const {}; + virtual void forwardSlice(const int8_t* src, const int8_t* lut, int8_t* dst, int len, + size_t outPlaneSize, int cn0, int cn1) const {}; }; class CV_EXPORTS ReLULayer : public ActivationLayer @@ -542,6 +593,12 @@ CV__DNN_INLINE_NS_BEGIN static Ptr create(const LayerParams ¶ms); }; + class CV_EXPORTS ActivationLayerInt8 : public ActivationLayer + { + public: + static Ptr create(const LayerParams ¶ms); + }; + /* Layers used in semantic segmentation */ class CV_EXPORTS CropLayer : public Layer @@ -563,6 +620,12 @@ CV__DNN_INLINE_NS_BEGIN static Ptr create(const LayerParams ¶ms); }; + class CV_EXPORTS EltwiseLayerInt8 : public Layer + { + public: + static Ptr create(const LayerParams ¶ms); + }; + class CV_EXPORTS BatchNormLayer : public ActivationLayer { public: @@ -572,6 +635,14 @@ CV__DNN_INLINE_NS_BEGIN static Ptr create(const LayerParams ¶ms); }; + class CV_EXPORTS BatchNormLayerInt8 : public BatchNormLayer + { + public: + float input_sc, output_sc; + int input_zp, output_zp; + static Ptr create(const LayerParams ¶ms); + }; + class CV_EXPORTS MaxUnpoolLayer : public Layer { public: @@ -591,12 +662,26 @@ CV__DNN_INLINE_NS_BEGIN static Ptr create(const LayerParams& params); }; + class CV_EXPORTS ScaleLayerInt8 : public ScaleLayer + { + public: + float output_sc; + int output_zp; + static Ptr create(const LayerParams ¶ms); + }; + class CV_EXPORTS ShiftLayer : public Layer { public: static Ptr create(const LayerParams& params); }; + class CV_EXPORTS ShiftLayerInt8 : public Layer + { + public: + static Ptr create(const LayerParams& params); + }; + class CV_EXPORTS DataAugmentationLayer : public Layer { public: diff --git a/modules/dnn/include/opencv2/dnn/dnn.hpp b/modules/dnn/include/opencv2/dnn/dnn.hpp index a498039f65..bf1670051a 100644 --- a/modules/dnn/include/opencv2/dnn/dnn.hpp +++ b/modules/dnn/include/opencv2/dnn/dnn.hpp @@ -235,6 +235,15 @@ CV__DNN_INLINE_NS_BEGIN */ virtual void forward(InputArrayOfArrays inputs, OutputArrayOfArrays outputs, OutputArrayOfArrays internals); + /** @brief Tries to quantize the given layer and compute the quantization parameters required for fixed point implementation. + * @param[in] scales input and output scales. + * @param[in] zeropoints input and output zeropoints. + * @param[out] params Quantized parameters required for fixed point implementation of that layer. + * @returns True if layer can be quantized. + */ + virtual bool tryQuantize(const std::vector > &scales, + const std::vector > &zeropoints, LayerParams& params); + /** @brief Given the @p input blobs, computes the output @p blobs. * @param[in] inputs the input blobs. * @param[out] outputs allocated output blobs, which will store results of the computation. @@ -368,6 +377,16 @@ CV__DNN_INLINE_NS_BEGIN */ virtual void getScaleShift(Mat& scale, Mat& shift) const; + /** + * @brief Returns scale and zeropoint of layers + * @param[out] scale Output scale + * @param[out] zeropoint Output zeropoint + * + * By default, @p scale is 1 and @p zeropoint is 0. + */ + virtual void getScaleZeropoint(float& scale, int& zeropoint) const; + + /** * @brief "Deattaches" all the layers, attached to particular layer. */ @@ -453,13 +472,21 @@ CV__DNN_INLINE_NS_BEGIN /** @brief Adds new layer to the net. * @param name unique name of the adding layer. * @param type typename of the adding layer (type must be registered in LayerRegister). + * @param dtype datatype of output blobs. * @param params parameters which will be used to initialize the creating layer. * @returns unique identifier of created layer, or -1 if a failure will happen. */ + int addLayer(const String &name, const String &type, const int &dtype, LayerParams ¶ms); + + /** @overload Datatype of output blobs set to default CV_32F */ int addLayer(const String &name, const String &type, LayerParams ¶ms); + /** @brief Adds new layer and connects its first input to the first output of previously added layer. * @see addLayer() */ + int addLayerToPrev(const String &name, const String &type, const int &dtype, LayerParams ¶ms); + + /** @overload */ int addLayerToPrev(const String &name, const String &type, LayerParams ¶ms); /** @brief Converts string name of the layer to the integer identifier. @@ -551,6 +578,25 @@ CV__DNN_INLINE_NS_BEGIN CV_WRAP_AS(forwardAndRetrieve) void forward(CV_OUT std::vector >& outputBlobs, const std::vector& outBlobNames); + /** @brief Returns a quantized Net from a floating-point Net. + * @param calibData Calibration data to compute the quantization parameters. + * @param inputsDtype Datatype of quantized net's inputs. Can be CV_32F or CV_8S. + * @param outputsDtype Datatype of quantized net's outputs. Can be CV_32F or CV_8S. + */ + CV_WRAP Net quantize(InputArrayOfArrays calibData, int inputsDtype, int outputsDtype); + + /** @brief Returns input scale and zeropoint for a quantized Net. + * @param scales output parameter for returning input scales. + * @param zeropoints output parameter for returning input zeropoints. + */ + CV_WRAP void getInputDetails(CV_OUT std::vector& scales, CV_OUT std::vector& zeropoints) const; + + /** @brief Returns output scale and zeropoint for a quantized Net. + * @param scales output parameter for returning output scales. + * @param zeropoints output parameter for returning output zeropoints. + */ + CV_WRAP void getOutputDetails(CV_OUT std::vector& scales, CV_OUT std::vector& zeropoints) const; + /** * @brief Compile Halide layers. * @param[in] scheduler Path to YAML file with scheduling directives. diff --git a/modules/dnn/src/dnn.cpp b/modules/dnn/src/dnn.cpp index 52a5fcba28..492ad166d0 100644 --- a/modules/dnn/src/dnn.cpp +++ b/modules/dnn/src/dnn.cpp @@ -574,9 +574,9 @@ struct LayerPin struct LayerData { - LayerData() : id(-1), skip(false), flag(0) {} - LayerData(int _id, const String &_name, const String &_type, LayerParams &_params) - : id(_id), name(_name), type(_type), params(_params), skip(false), flag(0) + LayerData() : id(-1), dtype(CV_32F), skip(false), flag(0) {} + LayerData(int _id, const String &_name, const String &_type, const int &_dtype, LayerParams &_params) + : id(_id), name(_name), type(_type), dtype(_dtype), params(_params), skip(false), flag(0) { CV_TRACE_FUNCTION(); @@ -588,6 +588,7 @@ struct LayerData int id; String name; String type; + int dtype; // Datatype of output blobs. LayerParams params; std::vector inputBlobsId; @@ -944,7 +945,7 @@ public: } } - void reuseOrCreate(const MatShape& shape, const LayerPin& lp, Mat& dst, bool use_half) + void reuseOrCreate(const MatShape& shape, const LayerPin& lp, Mat& dst, const int& dtype) { if (!DNN_DISABLE_MEMORY_OPTIMIZATIONS) { @@ -966,7 +967,8 @@ public: { Mat& unusedBlob = hostIt->second; if (unusedBlob.total() >= targetTotal && - unusedBlob.total() < bestBlobTotal) + unusedBlob.total() < bestBlobTotal && + unusedBlob.type() == dtype) { bestBlobPin = hostIt->first; bestBlob = unusedBlob; @@ -985,14 +987,13 @@ public: { // if dst already has been allocated with total(shape) elements, // it won't be recreated and pointer of dst.data remains the same. - dst.create(shape, use_half ? CV_16S : CV_32F); + dst.create(shape, dtype); addHost(lp, dst); } } void allocateBlobsForLayer(LayerData &ld, const LayerShapes& layerShapes, - std::vector& pinsForInternalBlobs, - bool use_half = false) + std::vector& pinsForInternalBlobs) { CV_TRACE_FUNCTION(); @@ -1063,7 +1064,7 @@ public: reuse(ld.inputBlobsId[0], blobPin); } else - reuseOrCreate(shapes[index], blobPin, *blobs[index], use_half); + reuseOrCreate(shapes[index], blobPin, *blobs[index], ld.dtype); } } } @@ -1193,6 +1194,7 @@ struct Net::Impl : public detail::NetImplBase lastLayerId = 0; netWasAllocated = false; + netWasQuantized = false; fusion = true; isAsync = false; preferableBackend = DNN_BACKEND_DEFAULT; @@ -1217,6 +1219,7 @@ struct Net::Impl : public detail::NetImplBase int lastLayerId; bool netWasAllocated; + bool netWasQuantized; bool fusion; bool isAsync; std::vector layersTimings; @@ -1372,7 +1375,7 @@ struct Net::Impl : public detail::NetImplBase currLayer->unsetAttached(); } - + netWasAllocated = false; layersTimings.clear(); } @@ -2541,10 +2544,11 @@ struct Net::Impl : public detail::NetImplBase CV_Assert(layerShapesIt != layersShapes.end()); + if (preferableBackend == DNN_BACKEND_OPENCV && preferableTarget == DNN_TARGET_OPENCL_FP16 && ld.dtype == CV_32F) + ld.dtype = CV_16S; + std::vector pinsForInternalBlobs; - blobManager.allocateBlobsForLayer(ld, layerShapesIt->second, pinsForInternalBlobs, - preferableBackend == DNN_BACKEND_OPENCV && - preferableTarget == DNN_TARGET_OPENCL_FP16); + blobManager.allocateBlobsForLayer(ld, layerShapesIt->second, pinsForInternalBlobs); ld.outputBlobsWrappers.resize(ld.outputBlobs.size()); for (int i = 0; i < ld.outputBlobs.size(); ++i) ld.outputBlobsWrappers[i] = wrap(ld.outputBlobs[i]); @@ -3148,7 +3152,8 @@ struct Net::Impl : public detail::NetImplBase Mat& inp = layers[0].outputBlobs[i]; CV_Assert(inp.total()); if (preferableBackend == DNN_BACKEND_OPENCV && - preferableTarget == DNN_TARGET_OPENCL_FP16) + preferableTarget == DNN_TARGET_OPENCL_FP16 && + layers[0].dtype == CV_32F) { layers[0].outputBlobs[i].create(inp.dims, inp.size, CV_16S); } @@ -3458,6 +3463,25 @@ struct Net::Impl : public detail::NetImplBase #endif } + void getQuantizationParams(const Mat& src, std::vector& scales, std::vector& zeropoints) + { + const int qmin = -128; // INT8_MIN + const int qmax = 127; // INT8_MAX + + double rmin, rmax, sc, zp; + cv::minMaxIdx(src, &rmin, &rmax); + + // 0 must be present in the range [rmin, rmax] + rmin = std::min(rmin, 0.0); + rmax = std::max(rmax, 0.0); + + sc = (rmax == rmin) ? 1.0 : (rmax - rmin)/(qmax - qmin); + zp = qmin - (rmin/sc); + + scales.push_back((float)sc); + zeropoints.push_back((int)std::round(zp)); + } + void getLayerShapesRecursively(int id, LayersShapesMap& inOutShapes) { std::vector& inputLayerIds = layers[id].inputBlobsId; @@ -3588,7 +3612,8 @@ struct Net::Impl : public detail::NetImplBase Mat& inp = layers[0].outputBlobs[i]; CV_Assert(inp.total()); if (preferableBackend == DNN_BACKEND_OPENCV && - preferableTarget == DNN_TARGET_OPENCL_FP16) + preferableTarget == DNN_TARGET_OPENCL_FP16 && + layers[0].dtype == CV_32F) { layers[0].outputBlobs[i].create(inp.dims, inp.size, CV_16S); } @@ -3614,7 +3639,7 @@ struct Net::Impl : public detail::NetImplBase const MatShape& shape = layersShapes[inputLayerId].out[inputLayerIds[i].oid]; layersShapes[layerId].in.push_back(shape); } - it->second.layerInstance->updateMemoryShapes(layersShapes[layerId].in); + it->second.getLayerInstance()->updateMemoryShapes(layersShapes[layerId].in); } } } @@ -4019,7 +4044,7 @@ Net::~Net() { } -int Net::addLayer(const String &name, const String &type, LayerParams ¶ms) +int Net::addLayer(const String &name, const String &type, const int &dtype, LayerParams ¶ms) { CV_TRACE_FUNCTION(); @@ -4042,23 +4067,35 @@ int Net::addLayer(const String &name, const String &type, LayerParams ¶ms) id = ++impl->lastLayerId; impl->layerNameToId.insert(std::make_pair(name, id)); - impl->layers.insert(std::make_pair(id, LayerData(id, name, type, params))); + impl->layers.insert(std::make_pair(id, LayerData(id, name, type, dtype, params))); if (params.get("has_dynamic_shapes", false)) impl->hasDynamicShapes = true; return id; } -int Net::addLayerToPrev(const String &name, const String &type, LayerParams ¶ms) +int Net::addLayer(const String &name, const String &type, LayerParams ¶ms) +{ + CV_TRACE_FUNCTION(); + return addLayer(name, type, CV_32F, params); +} + +int Net::addLayerToPrev(const String &name, const String &type, const int &dtype, LayerParams ¶ms) { CV_TRACE_FUNCTION(); int prvLid = impl->lastLayerId; - int newLid = this->addLayer(name, type, params); + int newLid = this->addLayer(name, type, dtype, params); this->connect(prvLid, 0, newLid, 0); return newLid; } +int Net::addLayerToPrev(const String &name, const String &type, LayerParams ¶ms) +{ + CV_TRACE_FUNCTION(); + return addLayerToPrev(name, type, CV_32F, params); +} + void Net::connect(int outLayerId, int outNum, int inpLayerId, int inpNum) { CV_TRACE_FUNCTION(); @@ -4169,16 +4206,19 @@ void Net::forward(OutputArrayOfArrays outputBlobs, const String& outputName) ld.outputBlobsWrappers[i]->copyToHost(); } } - if (ld.outputBlobs[0].depth() == CV_32F) + if (ld.outputBlobs[0].depth() == CV_16S) { - std::vector & outputvec = *(std::vector *)outputBlobs.getObj(); - outputvec = ld.outputBlobs; - } else { std::vector & outputvec = *(std::vector *)outputBlobs.getObj(); outputvec.resize(ld.outputBlobs.size()); for (int i = 0; i < outputvec.size(); i++) convertFp16(ld.outputBlobs[i], outputvec[i]); } + else + { + // Output depth can be CV_32F or CV_8S + std::vector & outputvec = *(std::vector *)outputBlobs.getObj(); + outputvec = ld.outputBlobs; + } } else if (outputBlobs.isUMatVector()) { @@ -4264,11 +4304,277 @@ void Net::forward(std::vector >& outputBlobs, } } +Net Net::quantize(InputArrayOfArrays calibData, int inputsDtype, int outputsDtype) +{ + CV_TRACE_FUNCTION(); + + // Net can be quantized only once. + if (impl->netWasQuantized) + CV_Error(Error::StsBadArg, "Cannot quantize a quantized net"); + + CV_CheckType(inputsDtype, inputsDtype == CV_32F || inputsDtype == CV_8S, "Input depth should be CV_32F or CV_8S"); + CV_CheckType(outputsDtype, outputsDtype == CV_32F || outputsDtype == CV_8S, "Output depth should be CV_32F or CV_8S"); + + bool originalFusion = impl->fusion; + int prefBackend = impl->preferableBackend; + int prefTarget = impl->preferableTarget; + + // Disable fusions and use CPU backend to quantize net + setPreferableBackend(DNN_BACKEND_OPENCV); + setPreferableTarget(DNN_TARGET_CPU); + enableFusion(false); + + if (calibData.isMat()) + { + setInput(calibData.getMat()); + } + else if (calibData.isMatVector()) + { + std::vector calibDataVec; + calibData.getMatVector(calibDataVec); + + std::vector inpNames = impl->netInputLayer->outNames; + CV_CheckEQ(calibDataVec.size(), inpNames.size(), "Calibration data size should be equal to number of inputs"); + for (int i = 0; i < calibDataVec.size(); i++) + setInput(calibDataVec[i], inpNames[i]); + } + + std::vector outNames = getUnconnectedOutLayersNames(); + std::vector pins; + for (int i = 0; i < outNames.size(); i++) + pins.push_back(impl->getPinByAlias(outNames[i])); + impl->setUpNet(pins); + + // Compute scales and zeropoints for all the layers + std::vector > scales; + std::vector > zeropoints; + for (Impl::MapIdToLayerData::iterator it = impl->layers.begin(); it != impl->layers.end(); it++) + { + LayerData& ld = it->second; + if (!ld.skip) + { + Ptr layer = ld.layerInstance; + std::vector inps(ld.inputBlobs.size()); + for (int i = 0; i < ld.inputBlobs.size(); ++i) + inps[i] = *ld.inputBlobs[i]; + layer->forward(inps, ld.outputBlobs, ld.internals); + } + + std::vector sc; + std::vector zp; + if (ld.type == "TanH") + { + sc.push_back(1.f/128); + zp.push_back(0); + } + else if (ld.type == "Sigmoid" || ld.type == "Softmax" || ld.type == "SoftMax") + { + if (ld.params.get("log_softmax", false)) + { + sc.push_back(16.f/256); + zp.push_back(127); + } + else + { + sc.push_back(1.f/256); + zp.push_back(-128); + } + } + else if (ld.type == "Split" || ld.type == "Slice" || ld.type == "Crop") + { + std::vector inp_sc; std::vector inp_zp; + impl->getQuantizationParams(*ld.inputBlobs[0], inp_sc, inp_zp); + sc.assign(ld.outputBlobs.size(), inp_sc[0]); + zp.assign(ld.outputBlobs.size(), inp_zp[0]); + } + else + { + for (int i = 0; i < ld.outputBlobs.size(); i++) + impl->getQuantizationParams(ld.outputBlobs[i], sc, zp); + } + scales.push_back(sc); + zeropoints.push_back(zp); + } + + // For some layers, the input and output scales/zeropoints must be equal so that rescaling of inputs + // is not needed during quantized inference. We start from the last layer and modify the layer's input scales/zeropoints + // TODO : Need a different approach. Current solution fails when 2 such layers have the same input layer + for (Impl::MapIdToLayerData::reverse_iterator it = impl->layers.rbegin(); it != impl->layers.rend(); ++it) + { + LayerData& ld = it->second; + // Layers with multiple outputs. Number of outputs is equal to number of inputs + if (ld.type == "Blank" || ld.type == "Dropout" || ld.type == "Identity" || ld.type == "Silence" || + ld.type == "Flatten" || ld.type == "Padding" || ld.type == "Permute" || ld.type == "Reshape" || + ld.type == "ReLU6" || ld.type == "Reorg" || ld.type == "ShuffleChannel" || + (ld.type == "ReLU" && !ld.params.get("negative_slope", 0.f)) /* ReLU with negative slope 0 */) + { + for (int i = 0; i < ld.outputBlobs.size(); i++) + { + LayerPin &pin = ld.inputBlobsId[i]; + scales[pin.lid][pin.oid] = scales[ld.id][i]; + zeropoints[pin.lid][pin.oid] = zeropoints[ld.id][i]; + } + } + // Layers with multiple inputs and single output. + else if ((ld.type == "Pooling" && toLowerCase(ld.params.get("pool", "max")) == "max") /* Max Pooling */ || + (ld.type == "Eltwise" && toLowerCase(ld.params.get("operation", "sum")) == "max") /* Elementwise max */ || + ld.type == "Concat") + { + for (int i = 0; i < ld.inputBlobsId.size(); i++) + { + LayerPin &pin = ld.inputBlobsId[i]; + scales[pin.lid][pin.oid] = scales[ld.id][0]; + zeropoints[pin.lid][pin.oid] = zeropoints[ld.id][0]; + } + } + } + + // Create a new Net and add quantized layers to it. + Net dstNet; + dstNet.impl->netWasQuantized = true; + dstNet.setInputsNames(impl->netInputLayer->outNames); + dstNet.setPreferableBackend(prefBackend); + dstNet.setPreferableTarget(prefTarget); + dstNet.enableFusion(originalFusion); + + for (Impl::MapIdToLayerData::iterator it = impl->layers.begin(); it != impl->layers.end(); it++) + { + LayerData ld = it->second; + if (ld.id == 0) + { + LayerData &quantInpLd = dstNet.impl->layers[0]; + quantInpLd.dtype = inputsDtype; + quantInpLd.params.set("scales", DictValue::arrayReal(scales[0].data(), scales[0].size())); + quantInpLd.params.set("zeropoints", DictValue::arrayInt(zeropoints[0].data(), zeropoints[0].size())); + continue; + } + + std::vector inpPins = ld.inputBlobsId; + // Fill input and output scales/zeropoints for the layer + std::vector > inp_out_sc(2); + std::vector > inp_out_zp(2); + for (int i = 0; i < inpPins.size(); i++) + { + LayerPin &pin = inpPins[i]; + inp_out_sc[0].push_back(scales[pin.lid][pin.oid]); + inp_out_zp[0].push_back(zeropoints[pin.lid][pin.oid]); + } + inp_out_sc[1] = scales[ld.id]; + inp_out_zp[1] = zeropoints[ld.id]; + + // Quantize layer + Ptr layer = ld.layerInstance; + if (layer->tryQuantize(inp_out_sc, inp_out_zp, ld.params)) + { + ld.type += "Int8"; + ld.dtype = CV_8S; + } + ld.params.set("scales", DictValue::arrayReal(inp_out_sc[1].data(), inp_out_sc[1].size())); + ld.params.set("zeropoints", DictValue::arrayInt(inp_out_zp[1].data(), inp_out_zp[1].size())); + + // Check and add quantize/dequantize node before layer + for (int i = 0; i < inpPins.size(); i++) + { + LayerPin &pin = inpPins[i]; + LayerData &inpLd = dstNet.impl->getLayerData(impl->getLayerName(pin.lid)); + pin.lid = inpLd.id; + if (inpLd.dtype != ld.dtype) + { + String layerName = (inpLd.dtype == CV_32F && ld.dtype == CV_8S) ? cv::format("quantize/%s/%d", inpLd.name.c_str(), pin.oid) + : cv::format("dequantize/%s/%d", inpLd.name.c_str(), pin.oid); + // Check if quantize/dequantize node for the input layer already exists + if (dstNet.impl->getLayerId(layerName) >= 0) + { + pin.lid = dstNet.impl->getLayerId(layerName); + pin.oid = 0; + } + else + { + LayerParams lp; + lp.set("scales", inp_out_sc[0][i]); + lp.set("zeropoints", inp_out_zp[0][i]); + lp.name = layerName; + lp.type = (inpLd.dtype == CV_32F && ld.dtype == CV_8S) ? "Quantize" : "Dequantize"; + int newLid = dstNet.addLayer(lp.name, lp.type, ld.dtype, lp); + dstNet.connect(pin.lid, pin.oid, newLid, 0); + pin.lid = newLid; pin.oid = 0; + } + } + } + + // Add quantized layer to Net and connect to its inputs. + int newLid = dstNet.addLayer(ld.name, ld.type, ld.dtype, ld.params); + for( int i = 0; i < inpPins.size(); i++ ) + dstNet.connect(inpPins[i].lid, inpPins[i].oid, newLid, i); + + // If the layer is a output layer, add quantize/dequantize node after it based on output's data type. + if (ld.requiredOutputs.size() == 0 && ld.dtype != outputsDtype) + { + LayerParams lp; + lp.set("scales", inp_out_sc[1][0]); + lp.set("zeropoints", inp_out_zp[1][0]); + lp.name = ((ld.dtype == CV_32F && outputsDtype == CV_8S) ? "quantize/" : "dequantize/") + ld.name; + lp.type = (ld.dtype == CV_32F && outputsDtype == CV_8S) ? "Quantize" : "Dequantize"; + dstNet.addLayerToPrev(lp.name, lp.type, outputsDtype, lp); + } + } + // Restore FP32 Net's backend, target and fusion + setPreferableBackend(prefBackend); + setPreferableTarget(prefTarget); + enableFusion(originalFusion); + return dstNet; +} + +void Net::getInputDetails(std::vector& scales, std::vector& zeropoints) const +{ + if (!impl->netWasQuantized) + CV_Error(Error::StsBadFunc, "Net isn't quantized"); + + LayerParams &lp = impl->layers[0].params; + DictValue sc = lp.get("scales"); + DictValue zp = lp.get("zeropoints"); + + for (int i = 0; i < sc.size(); i++) + { + scales.push_back(sc.get(i)); + zeropoints.push_back(zp.get(i)); + } +} + +void Net::getOutputDetails(std::vector& scales, std::vector& zeropoints) const +{ + if (!impl->netWasQuantized) + CV_Error(Error::StsBadFunc, "Net isn't quantized"); + + std::vector outLayerIds = getUnconnectedOutLayers(); + for (auto &lid : outLayerIds) + { + LayerParams &lp = impl->layers[lid].params; + DictValue sc = lp.get("scales"); + DictValue zp = lp.get("zeropoints"); + + for (int i = 0; i < sc.size(); i++) + { + scales.push_back(sc.get(i)); + zeropoints.push_back(zp.get(i)); + } + } +} + void Net::setPreferableBackend(int backendId) { CV_TRACE_FUNCTION(); CV_TRACE_ARG(backendId); + if (backendId == DNN_BACKEND_DEFAULT) + backendId = (Backend)PARAM_DNN_BACKEND_DEFAULT; + + if (impl->netWasQuantized && backendId != DNN_BACKEND_OPENCV) + { + CV_LOG_WARNING(NULL, "DNN: Only default backend supports quantized networks"); + backendId = DNN_BACKEND_OPENCV; + } + #ifdef HAVE_INF_ENGINE if (backendId == DNN_BACKEND_INFERENCE_ENGINE) backendId = getInferenceEngineBackendTypeParam(); @@ -4277,7 +4583,6 @@ void Net::setPreferableBackend(int backendId) if( impl->preferableBackend != backendId ) { impl->preferableBackend = backendId; - impl->netWasAllocated = false; impl->clear(); } } @@ -4287,6 +4592,13 @@ void Net::setPreferableTarget(int targetId) CV_TRACE_FUNCTION(); CV_TRACE_ARG(targetId); + if (impl->netWasQuantized && targetId != DNN_TARGET_CPU && + targetId != DNN_TARGET_OPENCL && targetId != DNN_TARGET_OPENCL_FP16) + { + CV_LOG_WARNING(NULL, "DNN: Only CPU and OpenCL/OpenCL FP16 target is supported by quantized networks"); + targetId = DNN_TARGET_CPU; + } + if( impl->preferableTarget != targetId ) { impl->preferableTarget = targetId; @@ -4306,7 +4618,6 @@ void Net::setPreferableTarget(int targetId) impl->preferableTarget = DNN_TARGET_OPENCL; #endif } - impl->netWasAllocated = false; impl->clear(); } } @@ -4935,9 +5246,10 @@ void Net::getMemoryConsumption(const int layerId, ShapesVec inLayerShapes, outLayerShapes; getLayerShapes(netInputShapes, layerId, inLayerShapes, outLayerShapes); + size_t elemSize = (impl->netWasQuantized) ? sizeof(char) : sizeof(float); for(int i = 0; i < outLayerShapes.size(); i++) { - blobs += total(outLayerShapes[i]) * sizeof(float); + blobs += total(outLayerShapes[i]) * elemSize; } } @@ -4986,7 +5298,7 @@ void Net::getMemoryConsumption(const std::vector& netInputShapes, std::vector > inLayerShapes, outLayerShapes; getLayersShapes(netInputShapes, layerIds, inLayerShapes, outLayerShapes); - + size_t elemSize = (impl->netWasQuantized) ? sizeof(char) : sizeof(float); for(int i = 0; i < layerIds.size(); i++) { int w = 0, b = 0; @@ -5001,7 +5313,7 @@ void Net::getMemoryConsumption(const std::vector& netInputShapes, for(int j = 0; j < outLayerShapes[i].size(); j++) { - b += total(outLayerShapes[i][j]) * sizeof(float); + b += total(outLayerShapes[i][j]) * elemSize; } weights.push_back(w); @@ -5021,7 +5333,6 @@ void Net::enableFusion(bool fusion) if( impl->fusion != fusion ) { impl->fusion = fusion; - impl->netWasAllocated = false; impl->clear(); } } @@ -5195,6 +5506,12 @@ void Layer::getScaleShift(Mat& scale, Mat& shift) const shift = Mat(); } +void Layer::getScaleZeropoint(float& scale, int& zeropoint) const +{ + scale = 1.f; + zeropoint = 0; +} + void Layer::unsetAttached() { setActivation(Ptr()); @@ -5321,6 +5638,12 @@ void Layer::run(const std::vector &inputs, std::vector &outputs, std:: this->forward(inputs, outputs, internals); } +bool Layer::tryQuantize(const std::vector > &scales, + const std::vector > &zeropoints, LayerParams& params) +{ + return false; +} + Layer::~Layer() {} bool Layer::getMemoryShapes(const std::vector &inputs, diff --git a/modules/dnn/src/init.cpp b/modules/dnn/src/init.cpp index 1916aa0ec9..9d8a3783a2 100644 --- a/modules/dnn/src/init.cpp +++ b/modules/dnn/src/init.cpp @@ -141,6 +141,44 @@ void initializeLayerFactory() CV_DNN_REGISTER_LAYER_CLASS(LSTM, LSTMLayer); CV_DNN_REGISTER_LAYER_CLASS(GRU, GRULayer); CV_DNN_REGISTER_LAYER_CLASS(CumSum, CumSumLayer); + + CV_DNN_REGISTER_LAYER_CLASS(Quantize, QuantizeLayer); + CV_DNN_REGISTER_LAYER_CLASS(Dequantize, DequantizeLayer); + CV_DNN_REGISTER_LAYER_CLASS(ConvolutionInt8, ConvolutionLayerInt8); + CV_DNN_REGISTER_LAYER_CLASS(InnerProductInt8, InnerProductLayerInt8); + CV_DNN_REGISTER_LAYER_CLASS(PoolingInt8, PoolingLayerInt8); + CV_DNN_REGISTER_LAYER_CLASS(EltwiseInt8, EltwiseLayerInt8); + CV_DNN_REGISTER_LAYER_CLASS(BatchNormInt8, BatchNormLayerInt8); + CV_DNN_REGISTER_LAYER_CLASS(ScaleInt8, ScaleLayerInt8); + CV_DNN_REGISTER_LAYER_CLASS(ShiftInt8, ShiftLayerInt8); + + CV_DNN_REGISTER_LAYER_CLASS(ReLUInt8, ActivationLayerInt8); + CV_DNN_REGISTER_LAYER_CLASS(ReLU6Int8, ActivationLayerInt8); + CV_DNN_REGISTER_LAYER_CLASS(SigmoidInt8, ActivationLayerInt8); + CV_DNN_REGISTER_LAYER_CLASS(TanHInt8, ActivationLayerInt8); + CV_DNN_REGISTER_LAYER_CLASS(SwishInt8, ActivationLayerInt8); + CV_DNN_REGISTER_LAYER_CLASS(MishInt8, ActivationLayerInt8); + CV_DNN_REGISTER_LAYER_CLASS(ELUInt8, ActivationLayerInt8); + CV_DNN_REGISTER_LAYER_CLASS(BNLLInt8, ActivationLayerInt8); + CV_DNN_REGISTER_LAYER_CLASS(AbsValInt8, ActivationLayerInt8); + CV_DNN_REGISTER_LAYER_CLASS(SoftmaxInt8, SoftmaxLayerInt8); + CV_DNN_REGISTER_LAYER_CLASS(SoftMaxInt8, SoftmaxLayerInt8); + + CV_DNN_REGISTER_LAYER_CLASS(ConcatInt8, ConcatLayer); + CV_DNN_REGISTER_LAYER_CLASS(FlattenInt8, FlattenLayer); + CV_DNN_REGISTER_LAYER_CLASS(PaddingInt8, PaddingLayer); + CV_DNN_REGISTER_LAYER_CLASS(BlankInt8, BlankLayer); + CV_DNN_REGISTER_LAYER_CLASS(DropoutInt8, BlankLayer); + CV_DNN_REGISTER_LAYER_CLASS(IdentityInt8, BlankLayer); + CV_DNN_REGISTER_LAYER_CLASS(SilenceInt8, BlankLayer); + CV_DNN_REGISTER_LAYER_CLASS(ConstInt8, ConstLayer); + CV_DNN_REGISTER_LAYER_CLASS(ReshapeInt8, ReshapeLayer); + CV_DNN_REGISTER_LAYER_CLASS(SplitInt8, SplitLayer); + CV_DNN_REGISTER_LAYER_CLASS(SliceInt8, SliceLayer); + CV_DNN_REGISTER_LAYER_CLASS(CropInt8, CropLayer); + CV_DNN_REGISTER_LAYER_CLASS(PermuteInt8, PermuteLayer); + CV_DNN_REGISTER_LAYER_CLASS(ReorgInt8, ReorgLayer); + CV_DNN_REGISTER_LAYER_CLASS(ShuffleChannelInt8, ShuffleChannelLayer); } CV__DNN_INLINE_NS_END diff --git a/modules/dnn/src/int8layers/batch_norm_layer.cpp b/modules/dnn/src/int8layers/batch_norm_layer.cpp new file mode 100644 index 0000000000..c5b8c3d9e9 --- /dev/null +++ b/modules/dnn/src/int8layers/batch_norm_layer.cpp @@ -0,0 +1,178 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. + +#include "../precomp.hpp" +#include "layers_common.hpp" +#include + +namespace cv +{ +namespace dnn +{ + +class BatchNormLayerInt8Impl CV_FINAL : public BatchNormLayerInt8 +{ +public: + Mat origin_weights, origin_bias; + Mat weights_, bias_; + mutable int dims; + + BatchNormLayerInt8Impl(const LayerParams& params) + : dims(-1) + { + setParamsFrom(params); + useGlobalStats = params.get("use_global_stats", true); + input_sc = params.get("input_scale"); + input_zp = params.get("input_zeropoint"); + output_sc = params.get("scales"); + output_zp = params.get("zeropoints"); + + CV_Assert(blobs.size() == 2); + size_t n = blobs[0].total(); + CV_Assert(blobs[1].total() == n && + blobs[0].isContinuous() && blobs[1].isContinuous() && + blobs[0].type() == CV_32F && blobs[1].type() == CV_32F); + + origin_weights = blobs[0]; + origin_bias = blobs[1]; + } + + virtual void finalize(InputArrayOfArrays, OutputArrayOfArrays) CV_OVERRIDE + { + origin_weights.convertTo(weights_, CV_32F, input_sc/output_sc); + addWeighted(origin_bias, 1.0/output_sc, weights_, -input_zp, output_zp, bias_, CV_32F); + } + + void getScaleShift(Mat& scale, Mat& shift) const CV_OVERRIDE + { + scale = origin_weights; + shift = origin_bias; + } + + void getScaleZeropoint(float& scale, int& zeropoint) const CV_OVERRIDE + { + scale = output_sc; + zeropoint = output_zp; + } + + virtual bool tryFuse(Ptr& top) CV_OVERRIDE + { + Mat w_, b_; + top->getScaleShift(w_, b_); + if (w_.empty() && b_.empty()) + return false; + + const int numChannels = weights_.total(); + const int numFusedWeights = w_.total(); + const int numFusedBias = b_.total(); + + if ((numFusedWeights != numChannels && numFusedWeights != 1 && !w_.empty()) || + (numFusedBias != numChannels && numFusedBias != 1 && !b_.empty())) + return false; + + float new_sc; + int new_zp; + top->getScaleZeropoint(new_sc, new_zp); + + Mat w = numFusedWeights == 1 ? Mat(1, numChannels, CV_32F, Scalar(w_.at(0))) : + (w_.empty() ? Mat::ones(1, numChannels, CV_32F) : w_.reshape(1, 1)); + + Mat b = numFusedBias == 1 ? Mat(1, numChannels, CV_32F, Scalar(b_.at(0))) : + (b_.empty() ? Mat::zeros(1, numChannels, CV_32F) : b_.reshape(1, 1)); + + weights_ = Mat(); bias_ = Mat(); + multiply(origin_weights, w, weights_, input_sc/new_sc, CV_32F); + multiply(origin_bias, w, bias_); + add(bias_, b, bias_); + addWeighted(bias_, 1.0/new_sc, weights_, -input_zp, new_zp, bias_, CV_32F); + return true; + } + + bool getMemoryShapes(const std::vector &inputs, + const int requiredOutputs, + std::vector &outputs, + std::vector &internals) const CV_OVERRIDE + { + dims = inputs[0].size(); + if (!useGlobalStats && inputs[0][0] != 1) + CV_Error(Error::StsNotImplemented, "Batch normalization in training mode with batch size > 1"); + Layer::getMemoryShapes(inputs, requiredOutputs, outputs, internals); + return true; + } + + virtual bool supportBackend(int backendId) CV_OVERRIDE + { + return backendId == DNN_BACKEND_OPENCV; + } + + bool setActivation(const Ptr& layer) CV_OVERRIDE + { + Ptr activ_int8 = layer.dynamicCast(); + if (!activ_int8.empty()) + { + return activ_int8->blobs.empty(); + } + return false; + } + + void forward(InputArrayOfArrays inputs_arr, OutputArrayOfArrays outputs_arr, OutputArrayOfArrays internals_arr) CV_OVERRIDE + { + CV_TRACE_FUNCTION(); + CV_TRACE_ARG_VALUE(name, "name", name.c_str()); + + std::vector inputs, outputs; + inputs_arr.getMatVector(inputs); + outputs_arr.getMatVector(outputs); + + CV_Assert(blobs.size() == 2); + CV_Assert(inputs.size() == 1); + + Mat &inpBlob = inputs[0]; + int planeSize = 1; + for (size_t i = 2; i < inpBlob.dims; i++) { + planeSize *= inpBlob.size[i]; + } + + for (size_t ii = 0; ii < outputs.size(); ii++) + { + Mat &outBlob = outputs[ii]; + + for(int num = 0; num < outBlob.size[0]; num++) + { + for (int n = 0; n < outBlob.size[1]; n++) + { + float w = weights_.at(n); + float b = bias_.at(n); + Mat inpBlobPlane(1, planeSize, CV_8S, inpBlob.ptr(num, n)); + Mat outBlobPlane(1, planeSize, CV_8S, outBlob.ptr(num, n)); + inpBlobPlane.convertTo(outBlobPlane, CV_8S, w, b); + } + } + } + } + + virtual int64 getFLOPS(const std::vector &inputs, + const std::vector &outputs) const CV_OVERRIDE + { + CV_UNUSED(outputs); // suppress unused variable warning + + int64 flops = 0; + for(int i = 0; i < inputs.size(); i++) + { + flops += 3*total(inputs[i]); + } + return flops; + } + +private: + bool useGlobalStats; +}; + +Ptr BatchNormLayerInt8::create(const LayerParams& params) +{ + return Ptr(new BatchNormLayerInt8Impl(params)); +} + +} // namespace dnn +} // namespace cv diff --git a/modules/dnn/src/int8layers/convolution_layer.cpp b/modules/dnn/src/int8layers/convolution_layer.cpp new file mode 100644 index 0000000000..05749885c0 --- /dev/null +++ b/modules/dnn/src/int8layers/convolution_layer.cpp @@ -0,0 +1,1136 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. + +#include "../precomp.hpp" +#include "layers_common.hpp" + +#include + +#include "opencv2/core/hal/hal.hpp" +#include "opencv2/core/hal/intrin.hpp" +#include +#include + +namespace cv +{ +namespace dnn +{ + +#if CV_SIMD +static inline void v_expand_mul_add(const v_int8x16& a, const v_int8x16& b, + v_int32x4& out0, v_int32x4& out1, v_int32x4& out2, v_int32x4& out3) +{ + v_int16x8 a0, a1, b0, b1; + v_expand(a, a0, a1); + v_expand(b, b0, b1); + + v_int32x4 t0, t1; + v_mul_expand(a0, b0, t0, t1); + out0 += t0; out1 += t1; + + v_mul_expand(a1, b1, t0, t1); + out2 += t0; out3 += t1; +} +#endif + +class BaseConvolutionLayerInt8Impl : public ConvolutionLayerInt8 +{ +public: + BaseConvolutionLayerInt8Impl(const LayerParams ¶ms) + { + setParamsFrom(params); + getConvolutionKernelParams(params, kernel_size, pads_begin, pads_end, strides, dilations, padMode, adjust_pads); + + numOutput = params.get("num_output"); + int ngroups = params.get("group", 1); + CV_Assert(numOutput % ngroups == 0); + + input_zp = params.get("input_zeropoint"); + output_zp = params.get("zeropoints"); + output_sc = params.get("scales"); + + if (kernel_size.size() == 2) { + kernel = Size(kernel_size[1], kernel_size[0]); + stride = Size(strides[1], strides[0]); + for (int i = 0; i < pads_begin.size(); i++) { + if (pads_begin[i] != pads_end[i]) + CV_Error(Error::StsNotImplemented, "Unsupported asymmetric padding in convolution layer"); + } + pad = Size(pads_begin[1], pads_begin[0]); + dilation = Size(dilations[1], dilations[0]); + + adjustPad.height = adjust_pads[0]; + adjustPad.width = adjust_pads[1]; + } + + for (int i = 0; i < adjust_pads.size(); i++) { + CV_Assert(adjust_pads[i] < strides[i]); + } + } + + virtual void finalize(InputArrayOfArrays inputs_arr, OutputArrayOfArrays outputs_arr) CV_OVERRIDE + { + std::vector inputs, outputs; + inputs_arr.getMatVector(inputs); + outputs_arr.getMatVector(outputs); + + // blobs[0] - Weights (INT8) + // blobs[1] - Biases (INT32) + // blobs[2] - Multipliers for convolution output stage (FP32) + CV_Assert(!inputs.empty() && blobs.size() == 3); + MatSize weightShape = blobs[0].size; + + CV_Assert(inputs[0].dims == outputs[0].dims); + if (weightShape.dims() == 3) + { + kernel_size.assign(1, kernel_size[0]); + strides.assign(1, strides[0]); + dilations.assign(1, dilations[0]); + pads_begin.assign(1, pads_begin[0]); + pads_end.assign(1, pads_end[0]); + } + CV_Assert(weightShape.dims() == kernel_size.size() + 2); + for (int i = 0; i < kernel_size.size(); i++) { + CV_Assert(weightShape[i + 2] == kernel_size[i]); + } + + const Mat &input = inputs[0]; + CV_Assert(((input.dims == 3 && kernel_size.size() == 1) || input.dims == 4 || input.dims == 5) && input.type() == CV_8S); + for (size_t i = 0; i < outputs.size(); i++) + { + CV_Assert(inputs[i].type() == input.type()); + CV_Assert(((input.dims == 3 && kernel_size.size() == 1) || inputs[i].dims == 4 || inputs[i].dims == 5) && inputs[i].size[1] == input.size[1]); + for (int j = 0; j < inputs[i].dims; j++) { + CV_Assert(inputs[i].size[j] == input.size[j]); + } + } + + std::vector inpShape; + std::vector outShape; + for (int i = 2; i < inputs[0].dims; i++) { + inpShape.push_back(inputs[0].size[i]); + outShape.push_back(outputs[0].size[i]); + } + getConvPoolPaddings(inpShape, kernel_size, strides, padMode, pads_begin, pads_end); + if (pads_begin.size() == 2) { + for (int i = 0; i < pads_begin.size(); i++) { + if (pads_begin[i] != pads_end[i]) + CV_Error(Error::StsNotImplemented, "Unsupported asymmetric padding in convolution layer"); + } + pad = Size(pads_begin[1], pads_begin[0]); + } + } + + virtual MatShape computeColRowShape(const MatShape &inpShape, const MatShape &outShape) const = 0; + bool is1x1() const + { + return (kernel.height == 1 && kernel.width == 1) && + (stride.height == 1 && stride.width == 1) && + (dilation.height == 1 && dilation.width == 1); + } + + virtual bool tryFuse(Ptr& top) CV_OVERRIDE + { + Mat w, b; + top->getScaleShift(w, b); + if (w.empty() && b.empty()) + return false; + + CV_Assert((w.empty() || w.type() == CV_32F) && + (b.empty() || b.type() == CV_32F)); + + float new_sc; + int new_zp; + top->getScaleZeropoint(new_sc, new_zp); + fuseWeights(w, b, new_sc); + output_sc = new_sc; + output_zp = new_zp; + return true; + } + + virtual void fuseWeights(const Mat& w_, const Mat& b_, const float& new_sc) = 0; +}; + +//TODO: simultaneously convolution and bias addition for cache optimization +class ConvolutionLayerInt8Impl CV_FINAL : public BaseConvolutionLayerInt8Impl +{ +public: + enum { VEC_ALIGN = 32, DFT_TYPE = CV_8S }; + Mat weightsMat; + std::vector biasvec; + Mat outputMultiplier; + Mat activationLUT; + Ptr activ; + + ConvolutionLayerInt8Impl(const LayerParams ¶ms) : BaseConvolutionLayerInt8Impl(params){} + + MatShape computeColRowShape(const MatShape &inpShape, const MatShape &outShape) const CV_OVERRIDE + { + CV_Assert(!blobs.empty()); + int dims = inpShape.size(); + int inpD = dims == 5 ? inpShape[2] : 1; + int inpH = inpShape[dims - 2]; + int inpW = inpShape.back(); + int inpGroupCn = blobs[0].size[1]; + int ksize = inpGroupCn * std::accumulate(kernel_size.begin(), kernel_size.end(), + 1, std::multiplies()); + return shape(inpD * inpH * inpW, ksize); + } + + virtual bool supportBackend(int backendId) CV_OVERRIDE + { + size_t ksize = kernel_size.size(); + // Only default backend and Conv1D/Conv2D/Conv3D are supported + return backendId == DNN_BACKEND_OPENCV && ksize >= 1 && ksize <= 3; + } + + bool getMemoryShapes(const std::vector &inputs, + const int requiredOutputs, + std::vector &outputs, + std::vector &internals) const CV_OVERRIDE + { + CV_Assert(!blobs.empty()); + const int* weightShape = blobs[0].size.p; + CV_Assert(blobs[1].total() == (size_t)weightShape[0]); + + internals.clear(); + + CV_Assert(inputs.size() != 0); + std::vector inpShape(inputs[0].begin() + 2, inputs[0].end()); + + int outCn = weightShape[0]; + std::vector outShape; + outShape.push_back(inputs[0][0]); + outShape.push_back(outCn); + + int inpCn = inputs[0][1]; + if (padMode.empty()) + { + for (int i = 0; i < inpShape.size(); i++) + outShape.push_back((inpShape[i] + pads_begin[i] + pads_end[i] - dilations[i] * (kernel_size[i] - 1) - 1) / strides[i] + 1); + } + else + { + getConvPoolOutParams(inpShape, kernel_size, strides, padMode, dilations, outShape); + } + + int ngroups = inpCn / weightShape[1]; + if (ngroups == 0 || ngroups * weightShape[1] != inpCn) + CV_Error(Error::StsError, format("Number of input channels should " + "be multiple of %d but got %d", weightShape[1], inpCn)); + CV_Assert(ngroups > 0 && inpCn % ngroups == 0 && outCn % ngroups == 0); + + outputs.resize(1, outShape); + + return false; + } + + virtual void finalize(InputArrayOfArrays inputs_arr, OutputArrayOfArrays outputs_arr) CV_OVERRIDE + { + BaseConvolutionLayerInt8Impl::finalize(inputs_arr, outputs_arr); + + std::vector inputs; + inputs_arr.getMatVector(inputs); + // prepare weightsMat where each row is aligned and has enough zero padding on the right to + // use vectorized (i.e. with intrinsics) loops without tail processing + Mat wm = blobs[0].reshape(1, numOutput); + if( wm.step1() % VEC_ALIGN != 0 ) + { + int newcols = (int)alignSize(wm.step1(), VEC_ALIGN); + Mat wm_buffer = Mat(numOutput, newcols, wm.type()); + Mat wm_padding = wm_buffer.colRange(wm.cols, newcols); + wm_padding.setTo(Scalar::all(0)); + Mat wm_aligned = wm_buffer.colRange(0, wm.cols); + wm.copyTo(wm_aligned); + wm = wm_aligned; + } + weightsMat = wm; + + Mat biasMat = blobs[1]; + biasvec.resize(numOutput+2); + for(int i = 0; i < numOutput; i++ ) + biasvec[i] = biasMat.at(i); + + outputMultiplier = blobs[2]; + } + + bool setActivation(const Ptr& layer) CV_OVERRIDE + { + Ptr activ_int8 = layer.dynamicCast(); + if (!activ_int8.empty()) + { + activ = activ_int8; + if (!activ_int8->blobs.empty()) + activ_int8->blobs[0].convertTo(activationLUT, CV_32S); + return true; + } + return false; + } + + virtual bool tryFuse(Ptr& top) CV_OVERRIDE + { + return BaseConvolutionLayerInt8Impl::tryFuse(top); + } + + void fuseWeights(const Mat& w_, const Mat& b_, const float& new_sc) CV_OVERRIDE + { + const int outCn = weightsMat.size[0]; + Mat w = w_.total() == 1 ? Mat(1, outCn, CV_32F, Scalar(w_.at(0))) : w_; + Mat b = b_.total() == 1 ? Mat(1, outCn, CV_32F, Scalar(b_.at(0))) : b_; + CV_Assert_N(!weightsMat.empty(), biasvec.size() == outCn + 2, + w.empty() || outCn == w.total(), b.empty() || outCn == b.total()); + + for (int i = 0; i < outCn; ++i) + { + float off = outputMultiplier.at(i) * output_sc; + if (!w.empty()) + off *= w.at(i); + + if (!b.empty()) + biasvec[i] += (int)std::round(b.at(i)/off); + + outputMultiplier.at(i) = off/new_sc; + } + biasvec[outCn] = biasvec[outCn+1] = biasvec[outCn-1]; + } + + class ParallelConv : public cv::ParallelLoopBody + { + public: + enum { BLK_SIZE = 32, BLK_SIZE_CN = 64 }; + + const Mat* input_; + const Mat* weights_; + Mat* output_; + int outShape[4]; // used only for conv2d + std::vector kernel_size, pads_begin, pads_end, strides, dilations; + int ngroups_, nstripes_; + std::vector ofstab_; + const std::vector* biasvec_; + const Mat* activLUT_; + const ActivationLayerInt8* activ_; + bool is1x1_; + bool useAVX2; + bool useAVX512; + int blk_size_cn; + int inpZp, outZp; + const float* multiplier; + + ParallelConv() + : input_(0), weights_(0), output_(0), ngroups_(0), nstripes_(0), + biasvec_(0), activLUT_(0), activ_(0), is1x1_(false), useAVX2(false), useAVX512(false) + , blk_size_cn(0), inpZp(0), outZp(0), multiplier(0) + {} + + static void run( const Mat& input, Mat& output, const Mat& weights, const Mat& multipliers, + const std::vector& biasvec, const Mat& activLUT, + const std::vector& kernel_size, const std::vector& strides, + const std::vector& pads_begin, const std::vector& pads_end, + const std::vector& dilations, + const ActivationLayerInt8* activ, int ngroups, int nstripes, int inp_Zp, int out_Zp) + { + size_t karea = std::accumulate(kernel_size.begin(), kernel_size.end(), + 1, std::multiplies()); + bool isConv1D = input.dims == 3; + bool isConv2D = input.dims == 4; + bool isConv3D = input.dims == 5; + CV_CheckEQ(static_cast(kernel_size.size()), input.dims - 2, ""); + CV_Assert_N(input.dims == output.dims, + input.size[0] == output.size[0], + weights.rows == output.size[1], + weights.cols == (input.size[1]/ngroups)*karea, + input.type() == CV_8SC1, + output.type() == CV_32SC1, + input.type() == weights.type(), + input.isContinuous(), + output.isContinuous(), + biasvec.size() == (size_t)output.size[1]+2); + CV_Check(weights.step1(), weights.step1() % VEC_ALIGN == 0, ""); + ParallelConv p; + + p.input_ = &input; + p.weights_ = &weights; + p.output_ = &output; + int max_ind = isConv1D? 3: 4; + for( int i = 0; i < max_ind; i++ ) p.outShape[i] = output.size[i]; + p.outShape[1] /= ngroups; + + p.kernel_size = kernel_size; p.strides = strides; p.dilations = dilations; + p.pads_begin = pads_begin; p.pads_end = pads_end; + + p.ngroups_ = ngroups; + p.nstripes_ = nstripes; + + int inpCnAll = input.size[1]; + int depth = (input.dims == 5) ? input.size[2] : 1; + int width = input.size[input.dims - 1]; + int height = isConv1D? 1 : input.size[input.dims - 2]; + int inpCn = inpCnAll / ngroups; + + p.is1x1_ = (isConv2D && kernel_size[0] == 1 && kernel_size[1] == 1 && + pads_begin[0] == 0 && pads_begin[1] == 0) || + (isConv1D && pads_begin[0] == 0 && kernel_size[0] == 1); + + p.useAVX2 = checkHardwareSupport(CPU_AVX2) && isConv2D; + p.useAVX512 = CV_CPU_HAS_SUPPORT_AVX512_SKX && isConv2D; + + int kernel_d = isConv3D? kernel_size[0] : 1; + int kernel_h = isConv1D? 1 : kernel_size[kernel_size.size() - 2]; + int kernel_w = kernel_size.back(); + + int blk_size_cn0 = cvCeil(1600./(kernel_w*kernel_h)); + int ncn = 32; + while (ncn*2 < blk_size_cn0 && ncn < inpCn) + ncn *= 2; + ncn = std::min(ncn, inpCn); + p.blk_size_cn = ncn; + + int dil_d = isConv3D? dilations[0] : 1; + int dil_h = isConv1D? 1 : dilations[dilations.size() - 2]; + int dil_w = dilations.back(); + + p.inpZp = inp_Zp; + p.outZp = out_Zp; + p.multiplier = multipliers.ptr(0); + + p.ofstab_.resize(karea * ncn); + int* ofstab = &p.ofstab_[0]; + + if (isConv1D) + { + for( int k = 0; k < ncn; k++ ) + for( int k_c = 0; k_c < kernel_w; k_c++ ) + ofstab[k*kernel_w + k_c] = k*width + k_c*dil_w; + } + else if (isConv2D) + { + for( int k = 0; k < ncn; k++ ) + for( int k_r = 0; k_r < kernel_h; k_r++ ) + for( int k_c = 0; k_c < kernel_w; k_c++ ) + ofstab[(k*kernel_h + k_r)*kernel_w + k_c] = + (k*height + k_r*dil_h)*width + k_c*dil_w; + } + else + { + for( int k = 0; k < ncn; k++ ) + for (int k_d = 0; k_d < kernel_d; k_d++) + for( int k_r = 0; k_r < kernel_h; k_r++ ) + for( int k_c = 0; k_c < kernel_w; k_c++ ) + ofstab[(k*kernel_d*kernel_h + k_d*kernel_h + k_r)*kernel_w + k_c] = + (k*depth*height + k_d*dil_d*height + k_r*dil_h)*width + k_c*dil_w; + } + + p.biasvec_ = &biasvec; + p.activLUT_ = &activLUT; + p.activ_ = !activLUT.empty() ? activ : 0; + + parallel_for_(Range(0, nstripes), p, nstripes); + } + + virtual void operator ()(const Range &r0) const CV_OVERRIDE + { + const int valign = ConvolutionLayerInt8Impl::VEC_ALIGN; + int ngroups = ngroups_, batchSize = input_->size[0]*ngroups; + bool isConv1D = input_->dims == 3; + bool isConv2D = input_->dims == 4; + bool isConv3D = input_->dims == 5; + + int outW = output_->size[output_->dims - 1]; + int outH = isConv1D? 1 : output_->size[output_->dims - 2]; + int outCn = output_->size[1]/ngroups; + + int depth = isConv3D? input_->size[2] : 1; + int height = isConv1D? 1 : input_->size[input_->dims - 2]; + int width = input_->size[input_->dims - 1]; + int inpCn = input_->size[1]/ngroups; + + const int nstripes = nstripes_; + + int kernel_d = isConv3D? kernel_size[0] : 1; + int kernel_h = isConv1D? 1 : kernel_size[kernel_size.size() - 2]; + int kernel_w = kernel_size.back(); + int karea = kernel_w*kernel_h*kernel_d; + + int pad_d = isConv3D? pads_begin[0] : 0; + int pad_t = isConv1D? 0 : pads_begin[pads_begin.size() - 2]; + int pad_l = pads_begin.back(); + + int stride_d = isConv3D? strides[0] : 0; + int stride_h = isConv1D? 0 : strides[strides.size() - 2]; + int stride_w = strides.back(); + + int dilation_d = isConv3D? dilations[0] : 1; + int dilation_h = isConv1D? 1 : dilations[dilations.size() - 2]; + int dilation_w = dilations.back(); + + int i, j, k, d; + int inpPlaneSize = (int)input_->total(2); + int outPlaneSize = (int)output_->total(2); + bool is1x1 = is1x1_; + + int stripesPerSample; + int stripeSize; + Range r = r0; + bool depthWiseConvolution = !is1x1 && isConv2D && ngroups > 1 && inpCn == 1 && + outCn == 1 && kernel_d == 1 && dilation_d == 1 && stride_d == 0 && pad_d == 0 && + width >= 16 + dilation_w*(kernel_w - 1); + // for now only 3x3 depth-wise convolutions are supported + depthWiseConvolution = depthWiseConvolution && kernel_w == 3 && kernel_h == 3 && + // computing at most 1 pixel from each side can involve padding + max(stride_w, dilation_w) >= pad_l && max(stride_h, dilation_h) >= pad_t && + pad_l <= 1 && pad_t <= 1; + + if( !depthWiseConvolution && nstripes >= batchSize*2 ) + { + stripesPerSample = nstripes/batchSize; + stripeSize = (int)alignSize((outPlaneSize + stripesPerSample - 1)/stripesPerSample, 8); + stripeSize = std::min(stripeSize, outPlaneSize); + } + else + { + stripesPerSample = 1; + int samplesPerStripe = std::max((batchSize + nstripes - 1)/nstripes, 1); + r.start *= samplesPerStripe; + r.end *= samplesPerStripe; + stripeSize = outPlaneSize; + } + + const int8_t* data_inp0_ = input_->ptr(); + const int* ofstab = &ofstab_[0]; + const int8_t* wptr_orig_ = weights_->ptr(); + size_t wstep = weights_->step1(); + const int* biasptr_ = &biasvec_->at(0); + const int* lutptr_ = !activLUT_->empty() ? activLUT_->ptr() : 0; + int* data_out0_ = output_->ptr(); + AutoBuffer rowbuf0_; + int8_t* rowbuf0 = 0; + bool use_rowbuf = !depthWiseConvolution; + int blk_size = depthWiseConvolution ? outPlaneSize : min((int)BLK_SIZE, stripeSize); + + // im2row buffer is not used for depth-wise convolution + if(use_rowbuf) + { + size_t rowbufsz = alignSize(karea*blk_size_cn, valign)*min((int)BLK_SIZE, blk_size); + //printf("karea=%d, blk_size_cn=%d, rowbufsz=%d, stripeSize=%d\n", karea, blk_size_cn, (int)rowbufsz, stripeSize); + rowbuf0_.allocate(rowbufsz + valign); + rowbuf0 = alignPtr(rowbuf0_.data(), (int)(valign*sizeof(int8_t))); + // we clear the buffer once; ultimately, it lets us to avoid + // tail processing after running the unrolled/vectorized loop. + // the main idea is to make sure that the tail (a.k.a. padding) of each row + // (i.e. the elements with indices between vsz=karea*ncn and vsz_a) + // does not contain NaNs or Infs. Because the padding in the weights + // matrix is explicitly initialized with 0's, we handle all other + // cases nicely, i.e. we can skip expliciting re-initialization + // of the padding - we just retain elements from the previous iteration + // of the loop over channels (cn0). + memset(rowbuf0, (int8_t)inpZp, rowbufsz*sizeof(rowbuf0[0]) ); + } + + for( int stripe = r.start; stripe < r.end; stripe++ ) + { + int subsampleIdx = stripe/stripesPerSample; + if( subsampleIdx >= batchSize ) + break; + int stripeStart = (int)((stripe - subsampleIdx*stripesPerSample)*stripeSize); + int stripeEnd = (int)std::min(stripeStart + stripeSize, outPlaneSize); + const int8_t* data_inp0 = data_inp0_ + subsampleIdx*inpPlaneSize*inpCn; + int* data_out0 = data_out0_ + subsampleIdx*outPlaneSize*outCn; + int startOutCn = (subsampleIdx % ngroups)*outCn; + const int8_t* wptr_orig = wptr_orig_ + wstep*startOutCn; + const int* biasptr = biasptr_ + startOutCn; + const float* multptr = multiplier + startOutCn; + + for( int cn0 = 0; cn0 < inpCn; cn0 += blk_size_cn ) + { + int cn1 = std::min(cn0 + blk_size_cn, inpCn); + int ncn = cn1 - cn0, vsz = karea*ncn; + int vsz_a = (int)alignSize(vsz, valign); + const int8_t* wptr = wptr_orig + cn0*karea; + + for( int ofs0 = stripeStart; ofs0 < stripeEnd; ofs0 += blk_size ) + { + int ofs, ofs1 = std::min(ofs0 + blk_size, stripeEnd); + int bsz = ofs1 - ofs0; + + int out_d = ofs0 / (outH * outW); + int out_i = (ofs0 - out_d * outH * outW) / outW; + int out_j = ofs0 % outW; + + if (depthWiseConvolution) + { + CV_Assert(out_i == 0 && out_j == 0); + int in_d = out_d * stride_d - pad_d; + const int8_t* inptr_ = data_inp0 + (cn0*depth*height + in_d*height)*width; + int* outptr_ = data_out0 + ofs0; + + #if CV_TRY_AVX2 + if(useAVX2) + opt_AVX2::fastDepthwiseConv(wptr, kernel_h, kernel_w, + stride_h, stride_w, dilation_h, dilation_w, pad_t, pad_l, + biasptr, multptr, inptr_, height, width, outptr_, out_d, outH, outW, inpZp, outZp); + else + #endif + { + const int8_t w00_ = wptr[0], w01_ = wptr[1], w02_ = wptr[2], + w10 = wptr[3], w11 = wptr[4], w12 = wptr[5], + w20_ = wptr[6], w21_ = wptr[7], w22_ = wptr[8]; + int outW1 = min(outW, (width - dilation_w*(kernel_w - 1) + pad_l)/stride_w); + int bias = biasptr[out_d], biasCopy; + float mult = multptr[out_d]; + + for (int out_i = 0; out_i < outH; out_i++) + { + int in_i = out_i * stride_h - pad_t, out_j = 0; + const int8_t* imgptr0 = inptr_ + in_i*width; + const int8_t* imgptr1 = imgptr0 + dilation_h*width; + const int8_t* imgptr2 = imgptr0 + (dilation_h*2)*width; + int8_t w00 = w00_, w01 = w01_, w02 = w02_; + int8_t w20 = w20_, w21 = w21_, w22 = w22_; + int out, out1; + // Bias has a fused offset component. bias = bias_quantized - input_zeropoint*sum_of_weights. + // In some cases below, certain weights are not used for convolution or set to zero. + // So we create a copy of bias at the start and remove the weight's components as necessary. + biasCopy = bias; + + if (in_i < 0) + { + biasCopy += inpZp * (w00 + w01 + w02); + w00 = w01 = w02 = 0; + imgptr0 = imgptr1; + } + else if (in_i + dilation_h*(kernel_h-1) >= height) + { + biasCopy += inpZp * (w20 + w21 + w22); + w20 = w21 = w22 = 0; + imgptr2 = imgptr1; + } + int* outptr = outptr_ + out_i*outW; + if (pad_l > 0) + { + out = (int)imgptr0[0]*w01 + (int)imgptr0[dilation_w]*w02 + + (int)imgptr1[0]*w11 + (int)imgptr1[dilation_w]*w12 + + (int)imgptr2[0]*w21 + (int)imgptr2[dilation_w]*w22 + + biasCopy + inpZp*(w00 + w10 + w20); + out1 = outZp + (int)std::round(out*mult); + outptr[0] = std::min(std::max(out1, -128), 127); + out_j = 1; + } + #if CV_SIMD + if( stride_w == 1 ) + { + const int out_delta = 16; + v_int8x16 vw00 = v_setall_s8(w00), vw01 = v_setall_s8(w01), vw02 = v_setall_s8(w02), + vw10 = v_setall_s8(w10), vw11 = v_setall_s8(w11), vw12 = v_setall_s8(w12), + vw20 = v_setall_s8(w20), vw21 = v_setall_s8(w21), vw22 = v_setall_s8(w22); + v_int32x4 vout0, vout1, vout2, vout3, vbias = v_setall_s32(biasCopy), voutzp = v_setall_s32(outZp), + outmin = v_setall_s32(-128), outmax = v_setall_s32(127); + v_float32x4 vmult = v_setall_f32(mult); + for( ; out_j < outW1; out_j += out_delta ) + { + if (out_j + out_delta > outW1) + { + if (out_j <= pad_l) + break; + out_j = outW1 - out_delta; + } + int in_j = out_j * stride_w - pad_l; + v_int8x16 v00 = v_load(imgptr0 + in_j), + v01 = v_load(imgptr0 + in_j + dilation_w), + v02 = v_load(imgptr0 + in_j + dilation_w*2), + v10 = v_load(imgptr1 + in_j), + v11 = v_load(imgptr1 + in_j + dilation_w), + v12 = v_load(imgptr1 + in_j + dilation_w*2), + v20 = v_load(imgptr2 + in_j), + v21 = v_load(imgptr2 + in_j + dilation_w), + v22 = v_load(imgptr2 + in_j + dilation_w*2); + + vout0 = vout1 = vout2 = vout3 = vbias; + v_expand_mul_add(v00, vw00, vout0, vout1, vout2, vout3); + v_expand_mul_add(v01, vw01, vout0, vout1, vout2, vout3); + v_expand_mul_add(v02, vw02, vout0, vout1, vout2, vout3); + v_expand_mul_add(v10, vw10, vout0, vout1, vout2, vout3); + v_expand_mul_add(v11, vw11, vout0, vout1, vout2, vout3); + v_expand_mul_add(v12, vw12, vout0, vout1, vout2, vout3); + v_expand_mul_add(v20, vw20, vout0, vout1, vout2, vout3); + v_expand_mul_add(v21, vw21, vout0, vout1, vout2, vout3); + v_expand_mul_add(v22, vw22, vout0, vout1, vout2, vout3); + + vout0 = voutzp + v_round(v_cvt_f32(vout0)*vmult); + vout1 = voutzp + v_round(v_cvt_f32(vout1)*vmult); + vout2 = voutzp + v_round(v_cvt_f32(vout2)*vmult); + vout3 = voutzp + v_round(v_cvt_f32(vout3)*vmult); + + vout0 = v_min(v_max(vout0, outmin), outmax); + vout1 = v_min(v_max(vout1, outmin), outmax); + vout2 = v_min(v_max(vout2, outmin), outmax); + vout3 = v_min(v_max(vout3, outmin), outmax); + + v_store(outptr + out_j, vout0); + v_store(outptr + out_j + 4, vout1); + v_store(outptr + out_j + 8, vout2); + v_store(outptr + out_j + 12, vout3); + } + } + #endif + for (; out_j < outW1; out_j++) + { + int in_j = out_j * stride_w - pad_l; + out = (int)imgptr0[in_j]*w00 + (int)imgptr0[in_j + dilation_w]*w01 + (int)imgptr0[in_j + dilation_w*2]*w02 + + (int)imgptr1[in_j]*w10 + (int)imgptr1[in_j + dilation_w]*w11 + (int)imgptr1[in_j + dilation_w*2]*w12 + + (int)imgptr2[in_j]*w20 + (int)imgptr2[in_j + dilation_w]*w21 + (int)imgptr2[in_j + dilation_w*2]*w22 + biasCopy; + out1 = outZp + (int)std::round(out*mult); + outptr[out_j] = std::min(std::max(out1, -128), 127); + } + + for (; out_j < outW; out_j++ ) + { + int in_j0 = out_j * stride_w - pad_l, in_j1 = in_j0 + dilation_w, in_j2 = in_j0 + dilation_w*2; + int s0 = 1, s1 = 1, s2 = 1; + if (in_j0 >= width) + { + in_j0 = 0; + s0 = 0; + biasCopy += inpZp*(w00 + w10 + w20); + } + if (in_j1 >= width) + { + in_j1 = 0; + s1 = 0; + biasCopy += inpZp*(w01 + w11 + w21); + } + if (in_j2 >= width) + { + in_j2 = 0; + s2 = 0; + biasCopy += inpZp*(w02 + w12 + w22); + } + out = (int)imgptr0[in_j0]*w00*s0 + (int)imgptr0[in_j1]*w01*s1 + (int)imgptr0[in_j2]*w02*s2 + + (int)imgptr1[in_j0]*w10*s0 + (int)imgptr1[in_j1]*w11*s1 + (int)imgptr1[in_j2]*w12*s2 + + (int)imgptr2[in_j0]*w20*s0 + (int)imgptr2[in_j1]*w21*s1 + (int)imgptr2[in_j2]*w22*s2 + biasCopy; + out1 = outZp + (int)std::round(out*mult); + outptr[out_j] = std::min(std::max(out1, -128), 127); + } + } + } + continue; + } + // do im2row for a part of input tensor + int8_t* rowbuf = rowbuf0; + + if (isConv1D) + { + for( ofs = ofs0; ofs < ofs1; out_j = 0, ++out_i ) + { + int delta = std::min(ofs1 - ofs, outW - out_j); + int out_j1 = out_j + delta; + + int in_j = out_j * stride_w - pad_l; + const int8_t* imgptr = data_inp0 + cn0*width + in_j; + ofs += delta; + + // do im2row for a part of input tensor + if( is1x1 ) + { + for( ; out_j < out_j1; out_j++, rowbuf += vsz_a, imgptr += stride_w ) + { + for( k = 0; k < vsz; k++ ) + rowbuf[k] = imgptr[k*inpPlaneSize]; + } + } + else + { + for( ; out_j < out_j1; out_j++, rowbuf += vsz_a, imgptr += stride_w, in_j += stride_w ) + { + // this condition should be true for most of the tensor elements, i.e. + // most of the time the kernel aperture is inside the tensor X-Y plane. + if( out_j + 2 <= out_j1 && 0 <= in_j && in_j + stride_w*2 <= width - (kernel_w-1)*dilation_w ) + { + for( k = 0; k < vsz; k++ ) + { + int k1 = ofstab[k]; + int8_t v0 = imgptr[k1]; + int8_t v1 = imgptr[k1 + stride_w]; + rowbuf[k] = v0; + rowbuf[k+vsz_a] = v1; + } + out_j++; + rowbuf += vsz_a; + imgptr += stride_w; + in_j += stride_w; + } + else + { + int i0 = std::max(0, (-in_j + dilation_w-1)/dilation_w); + int i1 = std::min(kernel_w, (width - in_j + dilation_w-1)/dilation_w); + + // here some non-continuous sub-row of the row will not be + // filled from the tensor; we need to make sure that the uncovered + // elements are explicitly set to 0's. the easiest way is to + // set all the elements to 0's before the loop. + memset(rowbuf, (int8_t)inpZp, vsz*sizeof(rowbuf[0])); + for( k = 0; k < ncn; k++ ) + { + for( i = i0; i < i1; i++ ) + { + int imgofs = k*width + i*dilation_w; + rowbuf[k*kernel_w + i] = imgptr[imgofs]; + } + } + } + } + } + } + } + else if (isConv2D) + { + if( is1x1 && stride_w == 1 && stride_h == 1 ) + { + const int8_t* imgptr = data_inp0 + (cn0*height + out_i)*width + out_j; + for( int j = 0; j < bsz; j++, rowbuf += vsz_a ) + { + if( j + 4 <= bsz ) + { + k = 0; + for( ; k < vsz; k++ ) + { + const int8_t* inp = imgptr + j + k*inpPlaneSize; + int8_t v0 = inp[0], v1 = inp[1], v2 = inp[2], v3 = inp[3]; + rowbuf[k] = v0; + rowbuf[k + vsz_a] = v1; + rowbuf[k + vsz_a*2] = v2; + rowbuf[k + vsz_a*3] = v3; + } + j += 3; + rowbuf += vsz_a*3; + } + else + { + for( k = 0; k < vsz; k++ ) + { + rowbuf[k] = imgptr[j + k*inpPlaneSize]; + } + } + } + } + else + for( ofs = ofs0; ofs < ofs1; out_j = 0, ++out_i ) + { + int delta = std::min(ofs1 - ofs, outW - out_j); + int out_j1 = out_j + delta; + + int in_i = out_i * stride_h - pad_t; + int in_j = out_j * stride_w - pad_l; + const int8_t* imgptr = data_inp0 + (cn0*height + in_i)*width + in_j; + ofs += delta; + + // do im2row for a part of input tensor + if( is1x1 ) + { + for( ; out_j < out_j1; out_j++, rowbuf += vsz_a, imgptr += stride_w ) + { + for( k = 0; k < vsz; k++ ) + rowbuf[k] = imgptr[k*inpPlaneSize]; + } + } + else + { + bool ok_i = 0 <= in_i && in_i < height - (kernel_h-1)*dilation_h; + int i0 = std::max(0, (-in_i + dilation_h-1)/dilation_h); + int i1 = std::min(kernel_h, (height - in_i + dilation_h-1)/dilation_h); + + for( ; out_j < out_j1; out_j++, rowbuf += vsz_a, imgptr += stride_w, in_j += stride_w ) + { + // this condition should be true for most of the tensor elements, i.e. + // most of the time the kernel aperture is inside the tensor X-Y plane. + if( ok_i && out_j + 2 <= out_j1 && 0 <= in_j && in_j + stride_w*2 <= width - (kernel_w-1)*dilation_w ) + { + for( k = 0; k < vsz; k++ ) + { + int k1 = ofstab[k]; + int8_t v0 = imgptr[k1]; + int8_t v1 = imgptr[k1 + stride_w]; + rowbuf[k] = v0; + rowbuf[k+vsz_a] = v1; + } + out_j++; + rowbuf += vsz_a; + imgptr += stride_w; + in_j += stride_w; + } + else + { + int j0 = std::max(0, (-in_j + dilation_w-1)/dilation_w); + int j1 = std::min(kernel_w, (width - in_j + dilation_w-1)/dilation_w); + + // here some non-continuous sub-row of the row will not be + // filled from the tensor; we need to make sure that the uncovered + // elements are explicitly set to 0's. the easiest way is to + // set all the elements to 0's before the loop. + memset(rowbuf, (int8_t)inpZp, vsz*sizeof(rowbuf[0])); + for( k = 0; k < ncn; k++ ) + { + for( i = i0; i < i1; i++ ) + { + for( j = j0; j < j1; j++ ) + { + int imgofs = k*(width*height) + i*(dilation_h*width) + j*dilation_w; + rowbuf[(k*kernel_h + i)*kernel_w + j] = imgptr[imgofs]; + } + } + } + } + } + } + } + } + else + { + for( ofs = ofs0; ofs < ofs1; out_d += (out_i + 1) / outH, out_i = (out_i + 1) % outH, out_j = 0 ) + { + int delta = std::min(ofs1 - ofs, outW - out_j); + int out_j1 = out_j + delta; + + int in_d = out_d * stride_d - pad_d; + int in_i = out_i * stride_h - pad_t; + int in_j = out_j * stride_w - pad_l; + const int8_t* imgptr = data_inp0 + (cn0*depth*height + in_d*height + in_i)*width + in_j; + ofs += delta; + + int d0 = std::max(0, (-in_d + dilation_d - 1) / dilation_d); + int d1 = std::min(kernel_d, (depth - in_d + dilation_d - 1) / dilation_d); + + int i0 = std::max(0, (-in_i + dilation_h-1)/dilation_h); + int i1 = std::min(kernel_h, (height - in_i + dilation_h-1)/dilation_h); + + for( ; out_j < out_j1; out_j++, rowbuf += vsz_a, imgptr += stride_w, in_j += stride_w ) + { + int j0 = std::max(0, (-in_j + dilation_w-1)/dilation_w); + int j1 = std::min(kernel_w, (width - in_j + dilation_w-1)/dilation_w); + + // here some non-continuous sub-row of the row will not be + // filled from the tensor; we need to make sure that the uncovered + // elements are explicitly set to 0's. the easiest way is to + // set all the elements to 0's before the loop. + memset(rowbuf, (int8_t)inpZp, vsz*sizeof(rowbuf[0])); + for( k = 0; k < ncn; k++ ) + { + for ( d = d0; d < d1; d++) + { + for( i = i0; i < i1; i++ ) + { + for( j = j0; j < j1; j++ ) + { + int imgofs = k*(depth*width*height) + d*dilation_d*width*height + i*(dilation_h*width) + j*dilation_w; + rowbuf[(k*kernel_d*kernel_h + d*kernel_h + i)*kernel_w + j] = imgptr[imgofs]; + } + } + } + } + } + } + } + // now compute dot product of the weights + // and im2row-transformed part of the tensor + #if CV_TRY_AVX512_SKX + if(useAVX512) + opt_AVX2::fastConv(wptr, wstep, biasptr, rowbuf0, data_out0 + ofs0, + outShape, bsz, vsz, vsz_a, outZp, multptr, cn0 == 0, cn1 == inpCn); + else + #endif + #if CV_TRY_AVX2 + if(useAVX2) + opt_AVX2::fastConv(wptr, wstep, biasptr, rowbuf0, data_out0 + ofs0, + outShape, bsz, vsz, vsz_a, outZp, multptr, cn0 == 0, cn1 == inpCn); + else + #endif + for( int i = 0; i < outCn; i += 2 ) + { + const int8_t* wptr0 = wptr + i*wstep; + const int8_t* wptr1 = wptr0 + wstep; + int* outptr0 = data_out0 + ofs0 + i*outPlaneSize; + int* outptr1 = outptr0 + outPlaneSize; + int bias0 = biasptr[i], bias1 = biasptr[i+1]; + float mult0 = multptr[i], mult1 = multptr[i+1]; + + if( i+1 >= outCn ) + { + wptr1 = wptr0; + outptr1 = outptr0; + bias1 = bias0; + mult1 = mult0; + } + int j = 0; + #if CV_SIMD128 + v_int32x4 voutzp = v_setall_s32(outZp), outmin = v_setall_s32(-128), outmax = v_setall_s32(127); + v_float32x4 vmult0 = v_setall_f32(mult0), vmult1 = v_setall_f32(mult1); + for( ; j <= bsz - 4; j += 4 ) + { + const int8_t* rptr = rowbuf0 + j*vsz_a; + v_int32x4 s0, s1; + + if( cn0 == 0 ) + { + s0 = v_setall_s32(bias0); + s1 = v_setall_s32(bias1); + } + else + { + s0 = v_load(outptr0 + j); + s1 = v_load(outptr1 + j); + } + + v_int32x4 vs00 = v_setzero_s32(), vs01 = v_setzero_s32(), + vs02 = v_setzero_s32(), vs03 = v_setzero_s32(), + vs10 = v_setzero_s32(), vs11 = v_setzero_s32(), + vs12 = v_setzero_s32(), vs13 = v_setzero_s32(); + for( k = 0; k < vsz; k += 16, rptr += 16 ) + { + v_int8x16 w0 = v_load_aligned(wptr0 + k); + v_int8x16 w1 = v_load_aligned(wptr1 + k); + v_int8x16 r0 = v_load_aligned(rptr); + v_int8x16 r1 = v_load_aligned(rptr + vsz_a); + v_int8x16 r2 = v_load_aligned(rptr + vsz_a*2); + v_int8x16 r3 = v_load_aligned(rptr + vsz_a*3); + + vs00 = v_dotprod_expand_fast(w0, r0, vs00); + vs01 = v_dotprod_expand_fast(w0, r1, vs01); + vs02 = v_dotprod_expand_fast(w0, r2, vs02); + vs03 = v_dotprod_expand_fast(w0, r3, vs03); + + vs10 = v_dotprod_expand_fast(w1, r0, vs10); + vs11 = v_dotprod_expand_fast(w1, r1, vs11); + vs12 = v_dotprod_expand_fast(w1, r2, vs12); + vs13 = v_dotprod_expand_fast(w1, r3, vs13); + } + s0 += v_int32x4(v_reduce_sum(vs00), v_reduce_sum(vs01), v_reduce_sum(vs02), v_reduce_sum(vs03)); + s1 += v_int32x4(v_reduce_sum(vs10), v_reduce_sum(vs11), v_reduce_sum(vs12), v_reduce_sum(vs13)); + if( cn1 == inpCn ) + { + s0 = voutzp + v_round(v_cvt_f32(s0)*vmult0); + s1 = voutzp + v_round(v_cvt_f32(s1)*vmult1); + + s0 = v_min(v_max(s0, outmin), outmax); + s1 = v_min(v_max(s1, outmin), outmax); + } + v_store(outptr0 + j, s0); + v_store(outptr1 + j, s1); + } + #endif + for( ; j < bsz; j++ ) + { + const int8_t* rptr = rowbuf0 + j*vsz_a; + int s00, s10; + + if( cn0 == 0 ) + { + s00 = bias0; + s10 = bias1; + } + else + { + s00 = outptr0[j]; + s10 = outptr1[j]; + } + + for( k = 0; k < vsz; k++ ) + { + int8_t r0 = rptr[k]; + s00 += (int)wptr0[k] * r0; + s10 += (int)wptr1[k] * r0; + } + if( cn1 == inpCn ) + { + int out0 = outZp + (int)std::round(s00*mult0); + int out1 = outZp + (int)std::round(s10*mult1); + + s00 = std::min(std::max(out0, -128), 127); + s10 = std::min(std::max(out1, -128), 127); + } + + outptr0[j] = s00; + outptr1[j] = s10; + } + } + } + } + if( activ_ ) + activ_->forwardSlice(data_out0 + stripeStart, lutptr_, + data_out0 + stripeStart, (int)(stripeEnd - stripeStart), + outPlaneSize, startOutCn, startOutCn + outCn); + } + } + }; + + void forward(InputArrayOfArrays inputs_arr, OutputArrayOfArrays outputs_arr, OutputArrayOfArrays internals_arr) CV_OVERRIDE + { + CV_TRACE_FUNCTION(); + CV_TRACE_ARG_VALUE(name, "name", name.c_str()); + +#if CV_SSE3 + uint32_t ftzMode = _MM_GET_FLUSH_ZERO_MODE(); + uint32_t dazMode = _MM_GET_DENORMALS_ZERO_MODE(); + _MM_SET_FLUSH_ZERO_MODE(_MM_FLUSH_ZERO_ON); + _MM_SET_DENORMALS_ZERO_MODE(_MM_DENORMALS_ZERO_ON); +#endif + + std::vector inputs, outputs; + inputs_arr.getMatVector(inputs); + outputs_arr.getMatVector(outputs); + + /*if (inputs[0].dims > 3) { + printf("conv %s: input (%d x %d x %d x %d), kernel (%d x %d), pad (%d x %d), stride (%d x %d), dilation (%d x %d)\n", + name.c_str(), inputs[0].size[0], inputs[0].size[1], inputs[0].size[2], inputs[0].size[3], + kernel.width, kernel.height, pad.width, pad.height, + stride.width, stride.height, dilation.width, dilation.height); + } + else { + printf("conv %s: input (%d x %d x %d), kernel (%d x %d), pad (%d x %d), stride (%d x %d), dilation (%d x %d)\n", + name.c_str(), inputs[0].size[0], inputs[0].size[1], inputs[0].size[2], + kernel.width, kernel.height, pad.width, pad.height, + stride.width, stride.height, dilation.width, dilation.height); + }*/ + + int inpGroupCn = blobs[0].size[1]; + CV_Assert_N(inputs.size() == (size_t)1, inputs[0].size[1] % inpGroupCn == 0, + outputs.size() == 1, inputs[0].data != outputs[0].data); + + int ngroups = inputs[0].size[1] / inpGroupCn; + CV_Assert(outputs[0].size[1] % ngroups == 0); + + int nstripes = std::max(getNumThreads(), 1); + Mat outputInt32 = Mat(shape(outputs[0]), CV_32S); + + ParallelConv::run(inputs[0], outputInt32, weightsMat, outputMultiplier, biasvec, activationLUT, kernel_size, strides, + pads_begin, pads_end, dilations, activ.get(), ngroups, nstripes, input_zp, output_zp); + + outputInt32.convertTo(outputs[0], CV_8S); + +#if CV_SSE3 + _MM_SET_FLUSH_ZERO_MODE(ftzMode); + _MM_SET_DENORMALS_ZERO_MODE(dazMode); +#endif + } + + virtual int64 getFLOPS(const std::vector &inputs, + const std::vector &outputs) const CV_OVERRIDE + { + CV_Assert(inputs.size() == outputs.size()); + + int64 flops = 0; + int karea = std::accumulate(kernel_size.begin(), kernel_size.end(), 1, std::multiplies()); + for (int i = 0; i < outputs.size(); i++) + { + flops += total(outputs[i])*(CV_BIG_INT(2)*karea*inputs[i][1] + 1); + } + return flops; + } +}; + +Ptr ConvolutionLayerInt8::create(const LayerParams ¶ms) +{ + return Ptr(new ConvolutionLayerInt8Impl(params)); +} + +} +} diff --git a/modules/dnn/src/int8layers/elementwise_layers.cpp b/modules/dnn/src/int8layers/elementwise_layers.cpp new file mode 100644 index 0000000000..75118b6bc1 --- /dev/null +++ b/modules/dnn/src/int8layers/elementwise_layers.cpp @@ -0,0 +1,190 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. + +#include "../precomp.hpp" +#include "layers_common.hpp" + +#include +#include + +namespace cv +{ +namespace dnn +{ + +class ActivationLayerInt8Impl CV_FINAL : public ActivationLayerInt8 +{ +public: + ActivationLayerInt8Impl(const LayerParams ¶ms) + { + setParamsFrom(params); + activationLUT = !blobs.empty() ? blobs[0] : Mat(); + } + + virtual bool supportBackend(int backendId) CV_OVERRIDE + { + return backendId == DNN_BACKEND_OPENCV; + } + + bool getMemoryShapes(const std::vector &inputs, + const int requiredOutputs, + std::vector &outputs, + std::vector &internals) const CV_OVERRIDE + { + Layer::getMemoryShapes(inputs, requiredOutputs, outputs, internals); + return true; + } + + class Activation : public cv::ParallelLoopBody + { + public: + const Mat* src; + const Mat* lut; + Mat* dst; + int nstripes; + + Activation() : src(0), lut(0), dst(0), nstripes(0){} + + static void run(const Mat& src, const Mat& lut, Mat& dst, int nstripes) + { + Activation p; + + p.src = &src; + p.lut = &lut; + p.dst = &dst; + p.nstripes = nstripes; + + parallel_for_(Range(0, nstripes), p, nstripes); + } + + void operator()(const Range &r) const CV_OVERRIDE + { + const int8_t* table = lut->ptr(); + int nsamples = 1, outCn = 1; + size_t planeSize = 1; + + if (src->dims > 1) + { + nsamples = src->size[0]; + outCn = src->size[1]; + } + else + outCn = src->size[0]; + + for (int i = 2; i < src->dims; ++i) + planeSize *= src->size[i]; + + size_t stripeSize = (planeSize + nstripes - 1)/nstripes; + size_t stripeStart = r.start*stripeSize; + size_t stripeEnd = std::min(r.end*stripeSize, planeSize); + int len = (int)(stripeEnd - stripeStart); + + for( int i = 0; i < nsamples; i++ ) + { + const int8_t* srcptr = src->ptr(i) + stripeStart; + int8_t* dstptr = dst->ptr(i) + stripeStart; + for( int cn = 0; cn < outCn; cn++, srcptr += planeSize, dstptr += planeSize ) + { + int i = 0; +#if CV_SIMD128 + for( ; i <= len - 16; i += 16 ) + { + v_int8x16 out(table[srcptr[i] + 128], table[srcptr[i+1] + 128], table[srcptr[i+2] + 128], table[srcptr[i+3] + 128], + table[srcptr[i+4] + 128], table[srcptr[i+5] + 128], table[srcptr[i+6] + 128], table[srcptr[i+7] + 128], + table[srcptr[i+8] + 128], table[srcptr[i+9] + 128], table[srcptr[i+10] + 128], table[srcptr[i+11] + 128], + table[srcptr[i+12] + 128], table[srcptr[i+13] + 128], table[srcptr[i+14] + 128], table[srcptr[i+15] + 128]); + v_store(dstptr + i, out); + } +#endif + for( ; i < len; i++ ) + { + dstptr[i] = table[srcptr[i] + 128]; + } + } + } + } + }; + + void forward(InputArrayOfArrays inputs_arr, OutputArrayOfArrays outputs_arr, OutputArrayOfArrays internals_arr) CV_OVERRIDE + { + CV_TRACE_FUNCTION(); + + std::vector inputs, outputs; + inputs_arr.getMatVector(inputs); + outputs_arr.getMatVector(outputs); + + for (size_t i = 0; i < inputs.size(); i++) + { + const Mat &src = inputs[i]; + if (!activationLUT.empty()) + { + const int nstripes = getNumThreads(); + Mat &dst = outputs[i]; + CV_Assert(src.size == dst.size && src.type() == dst.type() && + src.isContinuous() && dst.isContinuous() && src.type() == CV_8S); + + Activation::run(src, activationLUT, dst, nstripes); + } + else + { + src.copyTo(outputs[i]); + } + } + } + + void forwardSlice(const int8_t* src, const int8_t* lut, int8_t* dst, int len, size_t planeSize, int cn0, int cn1) const CV_OVERRIDE + { + for( int cn = cn0; cn < cn1; cn++, src += planeSize, dst += planeSize ) + { + int i = 0; +#if CV_SIMD128 + for( ; i <= len - 16; i += 16 ) + { + v_int8x16 out(lut[src[i] + 128], lut[src[i+1] + 128], lut[src[i+2] + 128], lut[src[i+3] + 128], + lut[src[i+4] + 128], lut[src[i+5] + 128], lut[src[i+6] + 128], lut[src[i+7] + 128], + lut[src[i+8] + 128], lut[src[i+9] + 128], lut[src[i+10] + 128], lut[src[i+11] + 128], + lut[src[i+12] + 128], lut[src[i+13] + 128], lut[src[i+14] + 128], lut[src[i+15] + 128]); + v_store(dst + i, out); + } +#endif + for( ; i < len; i++ ) + dst[i] = lut[src[i] + 128]; + } + } + + void forwardSlice(const int* src, const int* lut, int* dst, int len, size_t planeSize, int cn0, int cn1) const CV_OVERRIDE + { + for( int cn = cn0; cn < cn1; cn++, src += planeSize, dst += planeSize ) + { + int i = 0; +#if CV_SIMD128 + for( ; i <= len - 16; i += 16 ) + { + v_int32x4 out0(lut[src[i] + 128], lut[src[i+1] + 128], lut[src[i+2] + 128], lut[src[i+3] + 128]); + v_int32x4 out1(lut[src[i+4] + 128], lut[src[i+5] + 128], lut[src[i+6] + 128], lut[src[i+7] + 128]); + v_int32x4 out2(lut[src[i+8] + 128], lut[src[i+9] + 128], lut[src[i+10] + 128], lut[src[i+11] + 128]); + v_int32x4 out3(lut[src[i+12] + 128], lut[src[i+13] + 128], lut[src[i+14] + 128], lut[src[i+15] + 128]); + + v_store(dst + i, out0); + v_store(dst + i + 4, out1); + v_store(dst + i + 8, out2); + v_store(dst + i + 12, out3); + } +#endif + for( ; i < len; i++ ) + dst[i] = lut[src[i] + 128]; + } + + } + + Mat activationLUT; +}; + +Ptr ActivationLayerInt8::create(const LayerParams& params) +{ + return Ptr(new ActivationLayerInt8Impl(params)); +} + +} +} diff --git a/modules/dnn/src/int8layers/eltwise_layer.cpp b/modules/dnn/src/int8layers/eltwise_layer.cpp new file mode 100644 index 0000000000..be7a32b1ef --- /dev/null +++ b/modules/dnn/src/int8layers/eltwise_layer.cpp @@ -0,0 +1,577 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. + +#include "../precomp.hpp" +#include "layers_common.hpp" +#include + +namespace cv +{ +namespace dnn +{ + +class EltwiseLayerInt8Impl CV_FINAL : public EltwiseLayerInt8 +{ +public: + enum EltwiseOp + { + PROD = 0, + SUM = 1, + MAX = 2 + } op; + std::vector coeffs; + std::vector zeropoints; + + enum OutputChannelsMode + { + ELTWISE_CHANNNELS_SAME = 0, //!< number of channels from inputs must be the same and equal to output's number of channels + ELTWISE_CHANNNELS_INPUT_0, //!< number of channels from inputs may be different, + //!< output's number of channels is equal to number of channels of first input + //!< number of channels of other inputs should not be greater than number of channels of first input + ELTWISE_CHANNNELS_INPUT_0_TRUNCATE, //!< number of channels from inputs may be different, + //!< output's number of channels is equal to number of channels of first input + //!< there is restriction on number of channels of other inputs + //!< extra channels of other inputs is ignored + ELTWISE_CHANNNELS_USE_MAX, //!< number of channels from inputs may be different, + //!< output's number of channels is equal to maximal number of input channels + //!< @note supported operation: `SUM` + } channelsModeInput; + + + mutable OutputChannelsMode channelsMode; //!< "optimized" channels mode (switch to ELTWISE_CHANNNELS_SAME if number of input channels are equal) + mutable /*size_t*/int outputChannels; + + EltwiseLayerInt8Impl(const LayerParams& params) + : outputChannels(0) + { + setParamsFrom(params); + offset = params.get("offset", 0.f); + hasVecInput = false; + op = SUM; + if (params.has("operation")) + { + String operation = toLowerCase(params.get("operation")); + if (operation == "prod") + op = PROD; + else if (operation == "sum") + op = SUM; + else if (operation == "max") + op = MAX; + else + CV_Error(cv::Error::StsBadArg, "Unknown operation type \"" + operation + "\""); + } + + if (params.has("coeff")) + { + DictValue paramCoeff = params.get("coeff"); + int i, n = paramCoeff.size(); + coeffs.resize(n); + for (i = 0; i < n; i++) + { + coeffs[i] = paramCoeff.get(i); + } + } + + if (params.has("input_zeropoints")) + { + DictValue zp = params.get("input_zeropoints"); + int i, n = zp.size(); + zeropoints.resize(n); + for (i = 0; i < n; i++) + { + zeropoints[i] = zp.get(i); + } + } + + channelsModeInput = ELTWISE_CHANNNELS_SAME; + if (params.has("output_channels_mode")) + { + String v = toLowerCase(params.get("output_channels_mode")); + if (v == "same") + { + channelsModeInput = ELTWISE_CHANNNELS_SAME; + } + else if (v == "input_0") + { + channelsModeInput = ELTWISE_CHANNNELS_INPUT_0; + } + else if (v == "input_0_truncate") + { + channelsModeInput = ELTWISE_CHANNNELS_INPUT_0_TRUNCATE; + } + else if (v == "max_input_channels") + { + channelsModeInput = ELTWISE_CHANNNELS_USE_MAX; + if (op != SUM) + CV_Error(cv::Error::StsBadArg, "[" + type + "]:(" + name + ") 'max' channels mode is limited to SUM operation only"); + } + else + CV_Error(cv::Error::StsBadArg, "[" + type + "]:(" + name + ") unknown channels mode: \"" + v + "\""); + } + channelsMode = channelsModeInput; + + // TODO Must have checks for other unknown options + } + + virtual bool supportBackend(int backendId) CV_OVERRIDE + { + return backendId == DNN_BACKEND_OPENCV; + } + + bool getMemoryShapes(const std::vector &inputs, + const int requiredOutputs, + std::vector &outputs, + std::vector &internals) const CV_OVERRIDE + { + CV_Assert(inputs.size() >= 2); + CV_Assert(inputs[0].size() >= 2); + CV_Assert(coeffs.size() == 0 || coeffs.size() == inputs.size()); + CV_Assert(op == SUM || op == PROD || coeffs.size() == 0); + + int dims = inputs[0].size(); + // Number of channels in output shape is determined by the first input tensor. + bool variableChannels = false; + int numChannels = inputs[0][1]; + for (size_t i = 1; i < inputs.size(); i++) + { + CV_Assert(inputs[0][0] == inputs[i][0]); // batch sizes are equal + + int input_channels = inputs[i][1]; + if (numChannels != input_channels) + variableChannels = true; + + if (channelsModeInput == ELTWISE_CHANNNELS_SAME) + { + CV_Assert(numChannels == input_channels); + } + else if (channelsModeInput == ELTWISE_CHANNNELS_INPUT_0) + { + CV_Assert(numChannels >= input_channels); + } + else if (channelsModeInput == ELTWISE_CHANNNELS_INPUT_0_TRUNCATE) + { + // nothing to check + } + else if (channelsModeInput == ELTWISE_CHANNNELS_USE_MAX) + { + numChannels = std::max(numChannels, input_channels); + } + else + { + CV_Assert(0 && "Internal error"); + } + } + + channelsMode = variableChannels ? channelsModeInput : ELTWISE_CHANNNELS_SAME; + outputChannels = numChannels; + + outputs.assign(1, inputs[0]); + outputs[0][1] = numChannels; + + if (dims > 2) + { + size_t vecIdx = 0; + bool isVecFound = false; + for (size_t i = 0; i < inputs.size(); i++) + { + bool allOnes = isAllOnes(inputs[i], 2, dims); + if (!allOnes && !isVecFound) + { + vecIdx = i; + isVecFound = true; + } + + if (!allOnes && i != vecIdx) + { + for (size_t j = 2; j < dims; j++) + { + CV_Assert(inputs[vecIdx][j] == inputs[i][j]); + } + } + } + + if (channelsModeInput == ELTWISE_CHANNNELS_SAME && isVecFound) + { + for (size_t j = 2; j < dims; j++) + { + outputs[0][j] = inputs[vecIdx][j]; + } + } + } + + return false; + } + + void finalize(InputArrayOfArrays inputs_arr, OutputArrayOfArrays) CV_OVERRIDE + { + std::vector inputs; + inputs_arr.getMatVector(inputs); + + for (size_t i = 0; i < inputs.size(); i++) + { + MatShape inpShape = shape(inputs[i].size); + if (isAllOnes(inpShape, 2, inputs[i].dims)) + { + hasVecInput = true; + return; + } + } + } + + class EltwiseInvoker : public ParallelLoopBody + { + EltwiseLayerInt8Impl& self; + std::vector srcs; + std::vector srcNumChannels; + int nsrcs; + Mat* dst; + Mat* buf; + std::vector coeffs; + std::vector zeropoints; + int nstripes; + const Mat* activLUT; + const ActivationLayerInt8* activ; + int channels; + size_t planeSize; + float offset; + + EltwiseInvoker(EltwiseLayerInt8Impl& self_) + : self(self_) + , nsrcs(0), dst(0), buf(0), nstripes(0), activ(0), channels(0) + , planeSize(0), offset(0) + {} + + public: + static void run(EltwiseLayerInt8Impl& self, + const Mat* srcs, int nsrcs, Mat& buf, Mat& dst, + int nstripes, float offset) + { + const EltwiseOp op = self.op; + CV_Check(dst.dims, 1 < dst.dims && dst.dims <= 5, ""); CV_CheckTypeEQ(dst.type(), CV_8SC1, ""); CV_Assert(dst.isContinuous()); + CV_Assert(self.coeffs.empty() || self.coeffs.size() == (size_t)nsrcs); + CV_CheckGE(nsrcs, 2, ""); + + CV_Assert(self.outputChannels == dst.size[1]); + + EltwiseInvoker p(self); + p.srcs.resize(nsrcs); + p.srcNumChannels.resize(nsrcs); + p.coeffs = self.coeffs; // can be sorted + p.zeropoints = self.zeropoints; + + bool sortInputs = false; + for( int i = 0; i < nsrcs; i++ ) + { + p.srcs[i] = &srcs[i]; + CV_CheckEQ(srcs[i].dims, dst.dims, ""); + CV_Assert(srcs[i].isContinuous()); + CV_Assert(srcs[i].type() == dst.type()); + p.srcNumChannels[i] = (srcs[i].dims >= 4) ? srcs[i].size[1] : 1; + + if (self.channelsMode == ELTWISE_CHANNNELS_SAME) + { + CV_Assert(srcs[i].size == dst.size); + } + else if (self.channelsMode == ELTWISE_CHANNNELS_INPUT_0) + { + if (i == 0) + CV_Assert(srcs[0].size == dst.size); + CV_Assert(self.outputChannels >= p.srcNumChannels[i]); + sortInputs = true; + } + else if (self.channelsMode == ELTWISE_CHANNNELS_INPUT_0_TRUNCATE) + { + if (i == 0) + CV_Assert(srcs[0].size == dst.size); + sortInputs = true; + } + else if (self.channelsMode == ELTWISE_CHANNNELS_USE_MAX) + { + CV_Assert(op == SUM); + CV_Assert(self.outputChannels >= p.srcNumChannels[i]); + sortInputs = true; + } + else + { + CV_Assert(0 && "Internal error"); + } + + if (sortInputs) + { + // Sort srcs and coefficients in the desc order by number of channels + for (int j = i; j >= 1; j--) + { + if (std::min(self.outputChannels, p.srcs[j - 1]->size[1]) < std::min(self.outputChannels, p.srcs[j]->size[1])) + { + std::swap(p.srcs[j - 1], p.srcs[j]); + std::swap(p.srcNumChannels[j - 1], p.srcNumChannels[j]); + if (!p.coeffs.empty()) + std::swap(p.coeffs[j - 1], p.coeffs[j]); + if (!p.zeropoints.empty()) + std::swap(p.zeropoints[j - 1], p.zeropoints[j]); + } + else + break; + } + } + } + + p.nsrcs = nsrcs; + p.dst = &dst; + p.buf = &buf; + p.nstripes = nstripes; + p.offset = offset; + p.channels = (dst.dims >= 4 ? dst.size[1] : 1); + + p.planeSize = dst.total(dst.dims >= 4 ? 2 : 1); + CV_CheckEQ(dst.total(), dst.size[0] * p.channels * p.planeSize, ""); + p.activLUT = &self.activationLUT; + p.activ = !self.activationLUT.empty() ? self.activ.get() : 0; + + parallel_for_(Range(0, nstripes), p, nstripes); + } + + void operator()(const Range& r) const CV_OVERRIDE + { + const EltwiseOp op = self.op; + size_t total = dst->size[0]*planeSize; + size_t stripeSize = (total + nstripes - 1)/nstripes; + size_t stripeStart = r.start*stripeSize; + size_t stripeEnd = std::min(r.end*stripeSize, total); + const float* coeffsptr = !coeffs.empty() ? &coeffs[0] : 0; + const int* zeropointsptr = !zeropoints.empty() ? &zeropoints[0] : 0; + const int8_t* lutptr = !activLUT->empty() ? activLUT->ptr() : 0; + int8_t* dstptr0 = dst->ptr(); + float* bufptr0 = buf->ptr(); + int blockSize0 = 1 << 12; + + for (size_t ofs = stripeStart; ofs < stripeEnd; ) + { + int sampleIdx = (int)(ofs / planeSize); + int delta = (int)ofs - sampleIdx * planeSize; + int blockSize = std::min(blockSize0, std::min((int)(stripeEnd - ofs), (int)planeSize - delta)); + if( blockSize <= 0 ) + break; + ofs += blockSize; + + for (int c = 0; c < channels; c++) + { + size_t dstIdx = delta + (sampleIdx*channels + c)*planeSize; + int8_t* dstptr = dstptr0 + dstIdx; + float* bufptr = bufptr0 + dstIdx; + + // process first two inputs + { + const int8_t* srcptr0 = srcs[0]->ptr() + dstIdx; + + const int inputIdx = 1; + int src1_channels = srcNumChannels[inputIdx]; + if (c >= src1_channels) + { + // no data from second input + if (!coeffsptr) + { + for (int j = 0; j < blockSize; j++) + { + dstptr[j] = srcptr0[j]; + } + } + else + { + float c0 = coeffsptr[0]; + int z0 = op == PROD ? zeropointsptr[0] : 0; + for (int j = 0; j < blockSize; j++) + { + bufptr[j] = c0 * (srcptr0[j] - z0); + } + } + } + else + { + size_t srcIdx = delta + (sampleIdx * src1_channels + c) * planeSize; + const int8_t* srcptrI = srcs[inputIdx]->ptr() + srcIdx; + + if (op == PROD) + { + float c0 = coeffsptr[0]; + float c1 = coeffsptr[1]; + int z0 = zeropointsptr[0]; + int z1 = zeropointsptr[1]; + for (int j = 0; j < blockSize; j++) + { + bufptr[j] = (c0*(srcptr0[j] - z0)) * (c1*(srcptrI[j] - z1)); + } + } + else if (op == MAX) + { + for (int j = 0; j < blockSize; j++) + { + dstptr[j] = std::max(srcptr0[j], srcptrI[j]); + } + } + else if (op == SUM) + { + float c0 = coeffsptr[0]; + float c1 = coeffsptr[1]; + for (int j = 0; j < blockSize; j++) + { + bufptr[j] = c0*srcptr0[j] + c1*srcptrI[j]; + } + } + else + CV_Error(Error::StsInternal, ""); + } + } + + // aggregate other inputs (3+) + for (size_t inputIdx = 2; inputIdx < nsrcs; inputIdx++) + { + int srcI_channels = srcNumChannels[inputIdx]; + if (c >= srcI_channels) + continue; // no data from second input + size_t srcIdx = delta + (sampleIdx * srcI_channels + c) * planeSize; + const int8_t* srcptrI = srcs[inputIdx]->ptr() + srcIdx; + + if (op == PROD) + { + float cI = coeffsptr[inputIdx]; + int zI = zeropointsptr[inputIdx]; + for (int j = 0; j < blockSize; j++) + { + bufptr[j] *= cI*(srcptrI[j] - zI); + } + } + else if (op == MAX) + { + for (int j = 0; j < blockSize; j++) + { + dstptr[j] = std::max(dstptr[j], srcptrI[j]); + } + } + else if (op == SUM) + { + float cI = coeffsptr[inputIdx]; + for (int j = 0; j < blockSize; j++) + { + bufptr[j] += cI * srcptrI[j]; + } + } + else + CV_Error(Error::StsInternal, ""); + } + + // add offset and saturate cast to int8 + if (op == SUM || op == PROD) + { + for (int j = 0; j < blockSize; j++) + { + dstptr[j] = saturate_cast(std::round(bufptr[j] + offset)); + } + } + } + if( activ ) + { + int8_t* ptr = dstptr0 + delta + sampleIdx*channels*planeSize; + activ->forwardSlice(ptr, lutptr, ptr, blockSize, planeSize, 0, channels); + } + } + } + }; + + void forward(InputArrayOfArrays inputs_arr, OutputArrayOfArrays outputs_arr, OutputArrayOfArrays internals_arr) CV_OVERRIDE + { + CV_TRACE_FUNCTION(); + CV_TRACE_ARG_VALUE(name, "name", name.c_str()); + + std::vector inputs, outputs; + inputs_arr.getMatVector(inputs); + outputs_arr.getMatVector(outputs); + + CV_Assert(outputs.size() == 1); + const int nstripes = getNumThreads(); + + if (channelsModeInput == ELTWISE_CHANNNELS_SAME && inputs[0].dims > 2) + { + for (size_t i = 0; i < inputs.size(); i++) + { + MatShape inpShape = shape(inputs[i].size); + bool allOnes = isAllOnes(inpShape, 2, inputs[i].dims); + + if (allOnes) + { + Mat tmpInput = inputs[i]; + MatShape outShape = shape(outputs[0].size); + size_t xSize = outShape[2]; + for (size_t j = 3; j < outShape.size(); j++) + xSize *= outShape[j]; + + int dimVec[3] = {outShape[0], outShape[1], (int) xSize}; + std::vector matSizesVec(&dimVec[0], &dimVec[0] + 3); + inputs[i] = Mat(matSizesVec, tmpInput.type()); + + std::vector idx(outShape.size(), 0); + std::vector outIdx(inpShape.size(), 0); + + for (size_t j = 0; j < outShape[0]; j++) + { + outIdx[0] = idx[0] = j; + for(size_t k = 0; k < outShape[1]; k++) + { + outIdx[1] = idx[1] = k; + for (size_t x = 0; x < xSize; x++) + { + outIdx[2] = x; + inputs[i].at(outIdx.data()) = tmpInput.at(idx.data()); + } + } + } + inputs[i] = inputs[i].reshape(0, outShape); + } + } + } + + Mat buf = Mat(shape(outputs[0]), CV_32F); // to store intermediate results + EltwiseInvoker::run(*this, &inputs[0], (int)inputs.size(), buf, outputs[0], nstripes, offset); + } + + virtual int64 getFLOPS(const std::vector &inputs, + const std::vector &outputs) const CV_OVERRIDE + { + CV_UNUSED(outputs); // suppress unused variable warning + CV_Assert(inputs.size()); + + // FIXIT: handle inputs with different number of channels + long flops = inputs.size() * total(inputs[0]); + + return flops; + } + + bool setActivation(const Ptr& layer) CV_OVERRIDE + { + Ptr activ_int8 = layer.dynamicCast(); + if (!activ_int8.empty()) + { + activ = activ_int8; + if (!activ_int8->blobs.empty()) + activationLUT = activ_int8->blobs[0]; + return true; + } + return false; + } + + Mat activationLUT; + Ptr activ; + +private: + bool hasVecInput; + float offset; +}; + +Ptr EltwiseLayerInt8::create(const LayerParams& params) +{ + return Ptr(new EltwiseLayerInt8Impl(params)); +} + +} +} diff --git a/modules/dnn/src/int8layers/fully_connected_layer.cpp b/modules/dnn/src/int8layers/fully_connected_layer.cpp new file mode 100644 index 0000000000..83da677a47 --- /dev/null +++ b/modules/dnn/src/int8layers/fully_connected_layer.cpp @@ -0,0 +1,266 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. + +#include "../precomp.hpp" +#include "layers_common.hpp" + +#include + +namespace cv +{ +namespace dnn +{ + +class FullyConnectedLayerInt8Impl CV_FINAL : public InnerProductLayerInt8 +{ +public: + enum { VEC_ALIGN = 32 }; + FullyConnectedLayerInt8Impl(const LayerParams& params) + { + setParamsFrom(params); + output_zp = params.get("zeropoints"); + axis = params.get("axis", 1); + if (blobs.size() == 3) + { + // blobs[0] - Weights + // blobs[1] - Bias fused with offset + // blobs[2] - Multipliers for output stage + int numOutput = params.get("num_output"); + int innerSize = (int)blobs[0].total() / numOutput; + + CV_Assert(blobs[0].dims >= 2 && (size_t)(innerSize * numOutput) == blobs[0].total()); + CV_Assert((size_t)numOutput == blobs[1].total()); + + weightsMat = blobs[0] = blobs[0].reshape(1, numOutput); + int vecsize = weightsMat.cols; + if (vecsize % VEC_ALIGN != 0) + { + int vecsize_aligned = (int)alignSize(vecsize, VEC_ALIGN); + Mat weightsBuf(weightsMat.rows, vecsize_aligned, weightsMat.type()); + Mat wpadding = weightsBuf.colRange(vecsize, vecsize_aligned); + wpadding.setTo(Scalar::all(0)); + weightsMat = weightsBuf.colRange(0, vecsize); + blobs[0].copyTo(weightsMat); + } + biasMat = blobs[1] = blobs[1].reshape(1, 1); + outputMultiplier = blobs[2]; + } + } + + bool getMemoryShapes(const std::vector &inputs, + const int requiredOutputs, + std::vector &outputs, + std::vector &) const CV_OVERRIDE + { + int numOutput, cAxis; + CV_CheckEQ(inputs.size(), (size_t)1, ""); + CV_CheckEQ(blobs[0].dims, 2, ""); + numOutput = blobs[0].size[0]; + CV_Assert((size_t)numOutput == blobs[1].total()); + cAxis = normalize_axis(axis, inputs[0]); + + MatShape outShape(cAxis + 1); + for (int i = 0; i < cAxis; ++i) + outShape[i] = inputs[0][i]; + outShape.back() = numOutput; + + outputs.resize(1, outShape); + return false; + } + + virtual bool supportBackend(int backendId) CV_OVERRIDE + { + return backendId == DNN_BACKEND_OPENCV; + } + + virtual bool setActivation(const Ptr& layer) CV_OVERRIDE + { + Ptr activ_int8 = layer.dynamicCast(); + if (!activ_int8.empty()) + { + activ = activ_int8; + if (!activ_int8->blobs.empty()) + activ_int8->blobs[0].convertTo(activationLUT, CV_32S); + return true; + } + return false; + } + + class FullyConnected : public ParallelLoopBody + { + public: + FullyConnected() : srcMat(0), weights(0), biasMat(0), outputMultiplier(0), activationLUT(0), activ(0), + dstMat(0), nstripes(0), outZp(0), useAVX2(false), useAVX512(false) {} + + static void run(const Mat& srcMat, const Mat& weights, const Mat& biasMat, const Mat& outputMultiplier, + const Mat& activationLUT, Mat& dstMat, const ActivationLayerInt8* activ, int nstripes, int outZp) + { + CV_Assert( srcMat.dims == 2 && srcMat.cols == weights.cols && + dstMat.rows == srcMat.rows && dstMat.cols == weights.rows && + srcMat.type() == weights.type() && srcMat.type() == CV_8S && + dstMat.type() == CV_32S && biasMat.type() == CV_32S && + biasMat.isContinuous() && (int)biasMat.total() == dstMat.cols ); + + FullyConnected p; + + p.srcMat = &srcMat; + p.weights = &weights; + p.biasMat = &biasMat; + p.outputMultiplier = &outputMultiplier; + p.activationLUT = &activationLUT; + p.dstMat = &dstMat; + p.nstripes = nstripes; + p.outZp = outZp; + p.activ = !activationLUT.empty() ? activ : 0; + p.useAVX2 = checkHardwareSupport(CPU_AVX2); + p.useAVX512 = CV_CPU_HAS_SUPPORT_AVX512_SKX; + + parallel_for_(Range(0, nstripes), p, nstripes); + } + + void operator()(const Range& r) const CV_OVERRIDE + { + int valign = FullyConnectedLayerInt8Impl::VEC_ALIGN; + int nsamples = srcMat->rows; + int nw0 = weights->rows; + int k, vecsize = srcMat->cols; + int vecsize_aligned = (int)alignSize(vecsize, VEC_ALIGN); + size_t total = (size_t)nsamples*nw0; + size_t stripeSize = (total + nstripes - 1)/nstripes; + size_t stripeStart = r.start*stripeSize; + size_t stripeEnd = r.end == nstripes ? total : std::min(r.end*stripeSize, total); + size_t wstep = weights->step1(); + AutoBuffer srcbuf(vecsize_aligned + valign); + int8_t* sptr = alignPtr(srcbuf.data(), (int)(valign*sizeof(int8_t))); + const int* lutptr = !activationLUT->empty() ? activationLUT->ptr() : 0; + + for( k = vecsize; k < vecsize_aligned; k++ ) + sptr[k] = 0; + + for( size_t ofs = stripeStart; ofs < stripeEnd; ) + { + int sampleIdx = (int)(ofs / nw0); + int delta = (int)(ofs - (size_t)sampleIdx*nw0); + const int8_t* sptr_ = srcMat->ptr(sampleIdx); + const int8_t* wptr = weights->ptr(delta); + int* dptr = dstMat->ptr(sampleIdx) + delta; + const int* biasptr = biasMat->ptr() + delta; + const float* multptr = outputMultiplier->ptr() + delta; + int nw = std::min(nw0 - delta, (int)(stripeEnd - ofs)); + + memcpy(sptr, sptr_, vecsize*sizeof(sptr[0])); + #if CV_TRY_AVX512_SKX + if( useAVX512 ) + opt_AVX512_SKX::fastGEMM1T( sptr, wptr, wstep, biasptr, multptr, dptr, nw, vecsize, outZp ); + else + #endif + #if CV_TRY_AVX2 + if( useAVX2 ) + opt_AVX2::fastGEMM1T( sptr, wptr, wstep, biasptr, multptr, dptr, nw, vecsize, outZp ); + else + #endif + { + int i = 0; + #if CV_SIMD + for( ; i <= nw - 4; i += 4, wptr += 4*wstep ) + { + v_int32x4 vs0 = v_setzero_s32(), vs1 = v_setzero_s32(), + vs2 = v_setzero_s32(), vs3 = v_setzero_s32(); + v_int32x4 outzp = v_setall_s32(outZp), outmin = v_setall_s32(-128), outmax = v_setall_s32(127); + v_int32x4 s = v_load(biasptr + i); + v_float32x4 mult = v_load(multptr + i); + + for( k = 0; k < vecsize; k += 16 ) + { + v_int8x16 v = v_load_aligned(sptr + k); + vs0 = v_dotprod_expand_fast(v, v_load_aligned(wptr + k), vs0); + vs1 = v_dotprod_expand_fast(v, v_load_aligned(wptr + wstep + k), vs1); + vs2 = v_dotprod_expand_fast(v, v_load_aligned(wptr + wstep*2 + k), vs2); + vs3 = v_dotprod_expand_fast(v, v_load_aligned(wptr + wstep*3 + k), vs3); + } + + s += v_int32x4(v_reduce_sum(vs0), v_reduce_sum(vs1), v_reduce_sum(vs2), v_reduce_sum(vs3)); + v_int32x4 out = outzp + v_round(v_cvt_f32(s)*mult); + v_store(dptr + i, v_min(v_max(out, outmin), outmax)); + } + #endif + + for( ; i < nw; i++, wptr += wstep ) + { + int s0 = biasptr[i]; + float mult0 = multptr[i]; + + for( k = 0; k < vecsize; k++ ) + { + int8_t v = sptr[k]; + s0 += (int)v*wptr[k]; + } + int out0 = outZp + (int)std::round(s0*mult0); + dptr[i] = std::min(std::max(out0, -128), 127); + } + } + + if(activ) + activ->forwardSlice(dptr, lutptr, dptr, 1, 1, delta, delta + nw); + + ofs += nw; + } + } + + const Mat *srcMat, *weights, *biasMat, *outputMultiplier, *activationLUT; + const ActivationLayerInt8* activ; + Mat* dstMat; + int nstripes, outZp; + bool useAVX2; + bool useAVX512; + }; + + void forward(InputArrayOfArrays inputs_arr, OutputArrayOfArrays outputs_arr, OutputArrayOfArrays internals_arr) CV_OVERRIDE + { + CV_TRACE_FUNCTION(); + CV_TRACE_ARG_VALUE(name, "name", name.c_str()); + + std::vector input, output; + inputs_arr.getMatVector(input); + outputs_arr.getMatVector(output); + + int axisCan = normalize_axis(axis, input[0].dims); + int outerSize = input[0].total(0, axisCan); + Mat srcMat = input[0].reshape(1, outerSize); + + Mat dstMat = output[0].reshape(1, outerSize); + Mat dstMatInt32= Mat(shape(dstMat), CV_32S); + + const int nstripes = getNumThreads(); + FullyConnected::run(srcMat, weightsMat, biasMat, outputMultiplier, activationLUT, dstMatInt32, activ.get(), nstripes, output_zp); + dstMatInt32.convertTo(dstMat, CV_8S); + } + + virtual int64 getFLOPS(const std::vector &inputs, + const std::vector &outputs) const CV_OVERRIDE + { + CV_UNUSED(inputs); // suppress unused variable warning + long flops = 0; + + int innerSize = blobs[0].size[1]; + for(int i = 0; i < outputs.size(); i++) + { + flops += CV_BIG_INT(3)*innerSize*total(outputs[i]); + } + + return flops; + + } + + Mat weightsMat, biasMat, outputMultiplier, activationLUT; + Ptr activ; +}; + +Ptr InnerProductLayerInt8::create(const LayerParams& params) +{ + return Ptr(new FullyConnectedLayerInt8Impl(params)); +} + +} +} diff --git a/modules/dnn/src/int8layers/layers_common.hpp b/modules/dnn/src/int8layers/layers_common.hpp new file mode 100644 index 0000000000..cb185a9eda --- /dev/null +++ b/modules/dnn/src/int8layers/layers_common.hpp @@ -0,0 +1,41 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. + +#ifndef __OPENCV_DNN_LAYERS_LAYERS_COMMON_HPP__ +#define __OPENCV_DNN_LAYERS_LAYERS_COMMON_HPP__ +#include +#include + +#define CV_CPU_OPTIMIZATION_DECLARATIONS_ONLY +// dispatched AVX/AVX2 optimizations +#include "./layers_common.simd.hpp" +#include "int8layers/layers_common.simd_declarations.hpp" +#undef CV_CPU_OPTIMIZATION_DECLARATIONS_ONLY + +#ifdef HAVE_OPENCL +#include "../ocl4dnn/include/ocl4dnn.hpp" +#endif + +namespace cv +{ +namespace dnn +{ +void getConvolutionKernelParams(const LayerParams ¶ms, std::vector& kernel, std::vector& pads_begin, + std::vector& pads_end, std::vector& strides, std::vector& dilations, + cv::String &padMode, std::vector& adjust_pads); + +void getPoolingKernelParams(const LayerParams ¶ms, std::vector& kernel, std::vector& globalPooling, + std::vector& pads_begin, std::vector& pads_end, std::vector& strides, cv::String &padMode); + +void getConvPoolOutParams(const std::vector& inp, const std::vector& kernel, + const std::vector& stride, const String &padMode, + const std::vector& dilation, std::vector& out); + + void getConvPoolPaddings(const std::vector& inp, const std::vector& kernel, + const std::vector& strides, const String &padMode, + std::vector& pads_begin, std::vector& pads_end); +} +} + +#endif diff --git a/modules/dnn/src/int8layers/layers_common.simd.hpp b/modules/dnn/src/int8layers/layers_common.simd.hpp new file mode 100644 index 0000000000..bf6149e5c9 --- /dev/null +++ b/modules/dnn/src/int8layers/layers_common.simd.hpp @@ -0,0 +1,637 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. + +#include "opencv2/core/hal/intrin.hpp" + +namespace cv { +namespace dnn { +CV_CPU_OPTIMIZATION_NAMESPACE_BEGIN + +void fastConv( const int8_t* weights, size_t wstep, const int* bias, + const int8_t* rowbuf, int* output, const int* outShape, + int blockSize, int vecsize, int vecsize_aligned, int outZp, + const float* multiplier, bool initOutput, bool finalOutput ); +void fastDepthwiseConv( const int8_t* wptr, + int kernel_h, int kernel_w, + int stride_h, int stride_w, + int dilation_h, int dilation_w, + int pad_t, int pad_l, + const int* biasptr, const float* multptr, + const int8_t* inptr_, + int height, int width, + int* outptr_, + int out_d, int outH, int outW, + int inpZp, int outZp ); +void fastGEMM1T( const int8_t* vec, const int8_t* weights, + size_t wstep, const int* bias, const float* multiplier, + int* dst, int nvecs, int vecsize, int outZp ); + +#if !defined(CV_CPU_OPTIMIZATION_DECLARATIONS_ONLY) && CV_AVX2 +#define OPENCV_FMADD_EPI8(_Tpvec, func) \ + inline _Tpvec _##func##_fmaddepi8_epi32(const _Tpvec& a, const _Tpvec& b, const _Tpvec& c) \ + { \ + _Tpvec even_a = _##func##_srai_epi16(_##func##_bslli_epi128(a, 1), 8); \ + _Tpvec odd_a = _##func##_srai_epi16(a, 8); \ + \ + _Tpvec even_b = _##func##_srai_epi16(_##func##_bslli_epi128(b, 1), 8); \ + _Tpvec odd_b = _##func##_srai_epi16(b, 8); \ + \ + _Tpvec prod0 = _##func##_madd_epi16(even_a, even_b); \ + _Tpvec prod1 = _##func##_madd_epi16(odd_a, odd_b); \ + return _##func##_add_epi32(_##func##_add_epi32(prod0, prod1), c); \ + } +OPENCV_FMADD_EPI8(__m256i, mm256) +//OPENCV_FMADD_EPI8(__m512i, mm512) + +enum { FASCONV_BASE_VECSZ = 4 }; + +void fastConv( const int8_t* weights, size_t wstep, const int* bias, + const int8_t* rowbuf, int* output, const int* outShape, + int blockSize, int vecsize, int vecsize_aligned, int outZp, + const float* multiplier, bool initOutput, bool finalOutput ) +{ + int outCn = outShape[1]; + size_t outPlaneSize = outShape[2]*outShape[3]; + int CV_DECL_ALIGNED(16) maskbuf[FASCONV_BASE_VECSZ] = {0}; + int rsz = blockSize % FASCONV_BASE_VECSZ; + for( int i = 0; i < rsz; i++ ) + maskbuf[FASCONV_BASE_VECSZ - i - 1] = -1; + __m128 mask = _mm_loadu_ps((const float*)maskbuf); + + // now compute dot product of the weights + // and im2row-transformed part of the tensor + for( int i = 0; i < outCn; i += 3 ) + { + const int8_t* wptr0 = weights + i*wstep; + const int8_t* wptr1 = wptr0 + wstep; + const int8_t* wptr2 = wptr1 + wstep; + int* outptr0 = output + i*outPlaneSize; + int* outptr1 = outptr0 + outPlaneSize; + int* outptr2 = outptr1 + outPlaneSize; + int bias0 = bias[i], bias1 = bias[i+1], bias2 = bias[i+2]; + float mult0 = multiplier[i], mult1 = multiplier[i+1], mult2 = multiplier[i+2]; + + if( i+2 >= outCn ) + { + wptr2 = wptr1; + outptr2 = outptr1; + bias2 = bias1; + mult2 = mult1; + + if( i+1 >= outCn ) + { + wptr2 = wptr1 = wptr0; + outptr2 = outptr1 = outptr0; + bias2 = bias1 = bias0; + mult2 = mult1 = mult0; + } + } + int j = 0; + for( ; j < blockSize; j += FASCONV_BASE_VECSZ ) + { + bool tail = false; + if (j + FASCONV_BASE_VECSZ > blockSize) + { + if (j == 0) + break; + j = blockSize - FASCONV_BASE_VECSZ; + tail = true; + } + int k = 0; + const int8_t* rptr = rowbuf + j*vecsize_aligned; + + __m256i vs00 = _mm256_setzero_si256(), vs01 = _mm256_setzero_si256(), + vs02 = _mm256_setzero_si256(), vs03 = _mm256_setzero_si256(), + vs10 = _mm256_setzero_si256(), vs11 = _mm256_setzero_si256(), + vs12 = _mm256_setzero_si256(), vs13 = _mm256_setzero_si256(), + vs20 = _mm256_setzero_si256(), vs21 = _mm256_setzero_si256(), + vs22 = _mm256_setzero_si256(), vs23 = _mm256_setzero_si256(); + + /* TODO : Fix AVX-512 path. Segmentation fault in Conv2D Tests. +#if CV_AVX512_SKX // AVX512VL is necessary to avoid register spilling + if (vecsize >= 64) + { + __m512i vs00_5 = _mm512_setzero_si512(), vs01_5 = _mm512_setzero_si512(), + vs02_5 = _mm512_setzero_si512(), vs03_5 = _mm512_setzero_si512(), + vs10_5 = _mm512_setzero_si512(), vs11_5 = _mm512_setzero_si512(), + vs12_5 = _mm512_setzero_si512(), vs13_5 = _mm512_setzero_si512(), + vs20_5 = _mm512_setzero_si512(), vs21_5 = _mm512_setzero_si512(), + vs22_5 = _mm512_setzero_si512(), vs23_5 = _mm512_setzero_si512(); + + for (; k <= vecsize - 64; k += 64, rptr += 64) + { + __m512i w0 = _mm512_load_si512(wptr0 + k); + __m512i w1 = _mm512_load_si512(wptr1 + k); + __m512i w2 = _mm512_load_si512(wptr2 + k); + __m512i r0 = _mm512_load_si512(rptr); + + vs00_5 = _mm512_fmaddepi8_epi32(w0, r0, vs00_5); + vs10_5 = _mm512_fmaddepi8_epi32(w1, r0, vs10_5); + vs20_5 = _mm512_fmaddepi8_epi32(w2, r0, vs20_5); + + r0 = _mm512_load_si512(rptr + vecsize_aligned); + vs01_5 = _mm512_fmaddepi8_epi32(w0, r0, vs01_5); + vs11_5 = _mm512_fmaddepi8_epi32(w1, r0, vs11_5); + vs21_5 = _mm512_fmaddepi8_epi32(w2, r0, vs21_5); + + r0 = _mm512_load_si512(rptr + vecsize_aligned*2); + vs02_5 = _mm512_fmaddepi8_epi32(w0, r0, vs02_5); + vs12_5 = _mm512_fmaddepi8_epi32(w1, r0, vs12_5); + vs22_5 = _mm512_fmaddepi8_epi32(w2, r0, vs22_5); + + r0 = _mm512_load_si512(rptr + vecsize_aligned*3); + vs03_5 = _mm512_fmaddepi8_epi32(w0, r0, vs03_5); + vs13_5 = _mm512_fmaddepi8_epi32(w1, r0, vs13_5); + vs23_5 = _mm512_fmaddepi8_epi32(w2, r0, vs23_5); + } + + // now fold the 512 bit accumulator vectors into 256 bit vectors so that the AVX2 code can finish + // the tail of the vector + + vs00 = _mm256_add_epi32( _mm512_extracti32x8_epi32(vs00_5, 0), _mm512_extracti32x8_epi32(vs00_5, 1)); + vs10 = _mm256_add_epi32( _mm512_extracti32x8_epi32(vs10_5, 0), _mm512_extracti32x8_epi32(vs10_5, 1)); + vs20 = _mm256_add_epi32( _mm512_extracti32x8_epi32(vs20_5, 0), _mm512_extracti32x8_epi32(vs20_5, 1)); + + vs01 = _mm256_add_epi32( _mm512_extracti32x8_epi32(vs01_5, 0), _mm512_extracti32x8_epi32(vs01_5, 1)); + vs11 = _mm256_add_epi32( _mm512_extracti32x8_epi32(vs11_5, 0), _mm512_extracti32x8_epi32(vs11_5, 1)); + vs21 = _mm256_add_epi32( _mm512_extracti32x8_epi32(vs21_5, 0), _mm512_extracti32x8_epi32(vs21_5, 1)); + + vs02 = _mm256_add_epi32( _mm512_extracti32x8_epi32(vs02_5, 0), _mm512_extracti32x8_epi32(vs02_5, 1)); + vs12 = _mm256_add_epi32( _mm512_extracti32x8_epi32(vs12_5, 0), _mm512_extracti32x8_epi32(vs12_5, 1)); + vs22 = _mm256_add_epi32( _mm512_extracti32x8_epi32(vs22_5, 0), _mm512_extracti32x8_epi32(vs22_5, 1)); + + vs03 = _mm256_add_epi32( _mm512_extracti32x8_epi32(vs03_5, 0), _mm512_extracti32x8_epi32(vs03_5, 1)); + vs13 = _mm256_add_epi32( _mm512_extracti32x8_epi32(vs13_5, 0), _mm512_extracti32x8_epi32(vs13_5, 1)); + vs23 = _mm256_add_epi32( _mm512_extracti32x8_epi32(vs23_5, 0), _mm512_extracti32x8_epi32(vs23_5, 1)); + } +#endif + */ + for (; k < vecsize; k += 32, rptr += 32 ) + { + __m256i w0 = _mm256_load_si256((const __m256i*)(wptr0 + k)); + __m256i w1 = _mm256_load_si256((const __m256i*)(wptr1 + k)); + __m256i w2 = _mm256_load_si256((const __m256i*)(wptr2 + k)); + __m256i r0 = _mm256_load_si256((const __m256i*)rptr); + + vs00 = _mm256_fmaddepi8_epi32(w0, r0, vs00); + vs10 = _mm256_fmaddepi8_epi32(w1, r0, vs10); + vs20 = _mm256_fmaddepi8_epi32(w2, r0, vs20); + + r0 = _mm256_load_si256((const __m256i*)(rptr + vecsize_aligned)); + vs01 = _mm256_fmaddepi8_epi32(w0, r0, vs01); + vs11 = _mm256_fmaddepi8_epi32(w1, r0, vs11); + vs21 = _mm256_fmaddepi8_epi32(w2, r0, vs21); + + r0 = _mm256_load_si256((const __m256i*)(rptr + vecsize_aligned*2)); + vs02 = _mm256_fmaddepi8_epi32(w0, r0, vs02); + vs12 = _mm256_fmaddepi8_epi32(w1, r0, vs12); + vs22 = _mm256_fmaddepi8_epi32(w2, r0, vs22); + + r0 = _mm256_load_si256((const __m256i*)(rptr + vecsize_aligned*3)); + vs03 = _mm256_fmaddepi8_epi32(w0, r0, vs03); + vs13 = _mm256_fmaddepi8_epi32(w1, r0, vs13); + vs23 = _mm256_fmaddepi8_epi32(w2, r0, vs23); + } + + __m256i t0 = _mm256_hadd_epi32(_mm256_hadd_epi32(vs00, vs01), _mm256_hadd_epi32(vs02, vs03)); + __m256i t1 = _mm256_hadd_epi32(_mm256_hadd_epi32(vs10, vs11), _mm256_hadd_epi32(vs12, vs13)); + __m256i t2 = _mm256_hadd_epi32(_mm256_hadd_epi32(vs20, vs21), _mm256_hadd_epi32(vs22, vs23)); + + t0 = _mm256_add_epi32(t0, _mm256_permute2x128_si256(t0, t0, 1)); + t1 = _mm256_add_epi32(t1, _mm256_permute2x128_si256(t1, t1, 1)); + t2 = _mm256_add_epi32(t2, _mm256_permute2x128_si256(t2, t2, 1)); + + __m128i s0, s1, s2; + + if( initOutput ) + { + s0 = _mm_set1_epi32(bias0); + s1 = _mm_set1_epi32(bias1); + s2 = _mm_set1_epi32(bias2); + } + else + { + s0 = _mm_loadu_si128((__m128i*)(outptr0 + j)); + s1 = _mm_loadu_si128((__m128i*)(outptr1 + j)); + s2 = _mm_loadu_si128((__m128i*)(outptr2 + j)); + } + + s0 = _mm_add_epi32(s0, _mm256_castsi256_si128(t0)); + s1 = _mm_add_epi32(s1, _mm256_castsi256_si128(t1)); + s2 = _mm_add_epi32(s2, _mm256_castsi256_si128(t2)); + + if( finalOutput ) + { + __m128i voutzp = _mm_set1_epi32(outZp); + __m128i outmin = _mm_set1_epi32(-128), outmax = _mm_set1_epi32(127); + s0 = _mm_add_epi32(voutzp, _mm_cvtps_epi32(_mm_mul_ps(_mm_cvtepi32_ps(s0), _mm_set1_ps(mult0)))); + s1 = _mm_add_epi32(voutzp, _mm_cvtps_epi32(_mm_mul_ps(_mm_cvtepi32_ps(s1), _mm_set1_ps(mult1)))); + s2 = _mm_add_epi32(voutzp, _mm_cvtps_epi32(_mm_mul_ps(_mm_cvtepi32_ps(s2), _mm_set1_ps(mult2)))); + + s0 = _mm_min_epi32(_mm_max_epi32(s0, outmin), outmax); + s1 = _mm_min_epi32(_mm_max_epi32(s1, outmin), outmax); + s2 = _mm_min_epi32(_mm_max_epi32(s2, outmin), outmax); + } + if( tail ) + { + s0 = _mm_castps_si128(_mm_blendv_ps(_mm_loadu_ps((const float*)outptr0 + j), _mm_castsi128_ps(s0), mask)); + s1 = _mm_castps_si128(_mm_blendv_ps(_mm_loadu_ps((const float*)outptr1 + j), _mm_castsi128_ps(s1), mask)); + s2 = _mm_castps_si128(_mm_blendv_ps(_mm_loadu_ps((const float*)outptr2 + j), _mm_castsi128_ps(s2), mask)); + } + _mm_storeu_si128((__m128i*)(outptr0 + j), s0); + _mm_storeu_si128((__m128i*)(outptr1 + j), s1); + _mm_storeu_si128((__m128i*)(outptr2 + j), s2); + } + + for( ; j <= blockSize - 2; j += 2 ) + { + const int8_t* rptr0 = rowbuf + j*vecsize_aligned; + const int8_t* rptr1 = rowbuf + (j+1)*vecsize_aligned; + int s00, s01, s10, s11, s20, s21; + + if( initOutput ) + { + s00 = s01 = bias0; + s10 = s11 = bias1; + s20 = s21 = bias2; + } + else + { + s00 = outptr0[j]; s01 = outptr0[j+1]; + s10 = outptr1[j]; s11 = outptr1[j+1]; + s20 = outptr2[j]; s21 = outptr2[j+1]; + } + + for( int k = 0; k < vecsize; k++ ) + { + int8_t w0 = wptr0[k], w1 = wptr1[k], w2 = wptr2[k]; + int8_t r = rptr0[k]; + s00 += (int)w0*r; s10 += (int)w1*r; s20 += (int)w2*r; + r = rptr1[k]; + s01 += (int)w0*r; s11 += (int)w1*r; s21 += (int)w2*r; + } + + if( finalOutput ) + { + s00 = std::min(std::max(outZp + (int)std::round(s00*mult0), -128), 127); + s01 = std::min(std::max(outZp + (int)std::round(s01*mult0), -128), 127); + s10 = std::min(std::max(outZp + (int)std::round(s10*mult1), -128), 127); + s11 = std::min(std::max(outZp + (int)std::round(s11*mult1), -128), 127); + s20 = std::min(std::max(outZp + (int)std::round(s20*mult2), -128), 127); + s21 = std::min(std::max(outZp + (int)std::round(s21*mult2), -128), 127); + } + outptr0[j] = s00; + outptr0[j+1] = s01; + outptr1[j] = s10; + outptr1[j+1] = s11; + outptr2[j] = s20; + outptr2[j+1] = s21; + } + + for( ; j < blockSize; j++ ) + { + const int8_t* rptr0 = rowbuf + j*vecsize_aligned; + int s00, s10, s20; + + if( initOutput ) + { + s00 = bias0; + s10 = bias1; + s20 = bias2; + } + else + { + s00 = outptr0[j]; + s10 = outptr1[j]; + s20 = outptr2[j]; + } + + for( int k = 0; k < vecsize; k++ ) + { + int8_t w0 = wptr0[k], w1 = wptr1[k], w2 = wptr2[k]; + int8_t r = rptr0[k]; + s00 += (int)w0*r; s10 += (int)w1*r; s20 += (int)w2*r; + } + + if( finalOutput ) + { + s00 = std::min(std::max(outZp + (int)std::round(s00*mult0), -128), 127); + s10 = std::min(std::max(outZp + (int)std::round(s10*mult1), -128), 127); + s20 = std::min(std::max(outZp + (int)std::round(s20*mult2), -128), 127); + } + outptr0[j] = s00; + outptr1[j] = s10; + outptr2[j] = s20; + } + } + _mm256_zeroupper(); +} + +static inline void _mm256_expand_mul_add(const __m256i& a, const __m256i& b, + __m256i& out0, __m256i& out1, __m256i& out2, __m256i& out3) +{ + __m256i a0 = _mm256_cvtepi8_epi16(_mm256_castsi256_si128(a)); + __m256i a1 = _mm256_cvtepi8_epi16(_mm256_extracti128_si256(a, 1)); + + __m256i b0 = _mm256_cvtepi8_epi16(_mm256_castsi256_si128(b)); + __m256i b1 = _mm256_cvtepi8_epi16(_mm256_extracti128_si256(b, 1)); + + __m256i a0b0 = _mm256_mullo_epi16(a0, b0); + __m256i a1b1 = _mm256_mullo_epi16(a1, b1); + + out0 = _mm256_add_epi32(out0, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(a0b0))); + out1 = _mm256_add_epi32(out1, _mm256_cvtepi16_epi32(_mm256_extracti128_si256(a0b0, 1))); + out2 = _mm256_add_epi32(out2, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(a1b1))); + out3 = _mm256_add_epi32(out3, _mm256_cvtepi16_epi32(_mm256_extracti128_si256(a1b1, 1))); +} + +static inline void _mm256_load_deinterleave(const int8_t* ptr, __m256i& a, __m256i& b) +{ + __m256i t0 = _mm256_loadu_si256((const __m256i*)ptr); + __m256i t1 = _mm256_loadu_si256((const __m256i*)(ptr + 32)); + + const __m256i sh = _mm256_setr_epi8(0, 2, 4, 6, 8, 10, 12, 14, 1, 3, 5, 7, 9, 11, 13, 15, + 0, 2, 4, 6, 8, 10, 12, 14, 1, 3, 5, 7, 9, 11, 13, 15); + __m256i p0 = _mm256_shuffle_epi8(t0, sh); + __m256i p1 = _mm256_shuffle_epi8(t1, sh); + __m256i lo = _mm256_permute2x128_si256(p0, p1, 0 + 2*16); + __m256i hi = _mm256_permute2x128_si256(p0, p1, 1 + 3*16); + a = _mm256_unpacklo_epi64(lo, hi); + b = _mm256_unpackhi_epi64(lo, hi); +} + +void fastDepthwiseConv( const int8_t* wptr, + int kernel_h, int kernel_w, + int stride_h, int stride_w, + int dilation_h, int dilation_w, + int pad_t, int pad_l, + const int* biasptr, const float* multptr, + const int8_t* inptr_, + int height, int width, + int* outptr_, + int out_d, int outH, int outW, + int inpZp, int outZp) +{ + const int8_t w00_ = wptr[0], w01_ = wptr[1], w02_ = wptr[2], + w10 = wptr[3], w11 = wptr[4], w12 = wptr[5], + w20_ = wptr[6], w21_ = wptr[7], w22_ = wptr[8]; + int outW1 = min(outW, (width - dilation_w*(kernel_w - 1) + pad_l)/stride_w); + float mult = multptr[out_d]; + int bias = biasptr[out_d]; + int biasCopy; + + for (int out_i = 0; out_i < outH; out_i++) + { + int in_i = out_i * stride_h - pad_t, out_j = 0; + const int8_t* imgptr0 = inptr_ + in_i*width; + const int8_t* imgptr1 = imgptr0 + dilation_h*width; + const int8_t* imgptr2 = imgptr0 + (dilation_h*2)*width; + int8_t w00 = w00_, w01 = w01_, w02 = w02_; + int8_t w20 = w20_, w21 = w21_, w22 = w22_; + int out; + biasCopy = bias; + if (in_i < 0) + { + biasCopy += inpZp * (w00 + w01 + w02); + w00 = w01 = w02 = 0; + imgptr0 = imgptr1; + } + else if (in_i + dilation_h*(kernel_h-1) >= height) + { + biasCopy += inpZp * (w20 + w21 + w22); + w20 = w21 = w22 = 0; + imgptr2 = imgptr1; + } + int* outptr = outptr_ + out_i*outW; + if (pad_l > 0) + { + out = (int)imgptr0[0]*w01 + (int)imgptr0[dilation_w]*w02 + + (int)imgptr1[0]*w11 + (int)imgptr1[dilation_w]*w12 + + (int)imgptr2[0]*w21 + (int)imgptr2[dilation_w]*w22 + + biasCopy + inpZp*(w00 + w10 + w20); + outptr[0] = std::min(std::max(outZp + (int)std::round(out*mult), -128), 127); + out_j = 1; + } + + if (stride_w == 1 || (stride_w == 2 && dilation_w == 1)) + { + const int VECSZ = 32; + __m256i vw00 = _mm256_set1_epi8(w00), vw01 = _mm256_set1_epi8(w01), vw02 = _mm256_set1_epi8(w02), + vw10 = _mm256_set1_epi8(w10), vw11 = _mm256_set1_epi8(w11), vw12 = _mm256_set1_epi8(w12), + vw20 = _mm256_set1_epi8(w20), vw21 = _mm256_set1_epi8(w21), vw22 = _mm256_set1_epi8(w22); + __m256i vbias = _mm256_set1_epi32(biasCopy), voutzp = _mm256_set1_epi32(outZp), + outmin = _mm256_set1_epi32(-128), outmax = _mm256_set1_epi32(127); + __m256 vmult = _mm256_set1_ps(mult); + __m256i vout0, vout1, vout2, vout3; + + if( stride_w == 1 ) + { + for( ; out_j < outW1; out_j += VECSZ ) + { + if (out_j + VECSZ > outW1) + { + if (out_j <= pad_l) + break; + out_j = outW1 - VECSZ; + } + int in_j = out_j * stride_w - pad_l; + __m256i v00 = _mm256_loadu_si256((const __m256i*)(imgptr0 + in_j)), + v01 = _mm256_loadu_si256((const __m256i*)(imgptr0 + in_j + dilation_w)), + v02 = _mm256_loadu_si256((const __m256i*)(imgptr0 + in_j + dilation_w*2)), + v10 = _mm256_loadu_si256((const __m256i*)(imgptr1 + in_j)), + v11 = _mm256_loadu_si256((const __m256i*)(imgptr1 + in_j + dilation_w)), + v12 = _mm256_loadu_si256((const __m256i*)(imgptr1 + in_j + dilation_w*2)), + v20 = _mm256_loadu_si256((const __m256i*)(imgptr2 + in_j)), + v21 = _mm256_loadu_si256((const __m256i*)(imgptr2 + in_j + dilation_w)), + v22 = _mm256_loadu_si256((const __m256i*)(imgptr2 + in_j + dilation_w*2)); + + vout0 = vout1 = vout2 = vout3 = vbias; + _mm256_expand_mul_add(v00, vw00, vout0, vout1, vout2, vout3); + _mm256_expand_mul_add(v01, vw01, vout0, vout1, vout2, vout3); + _mm256_expand_mul_add(v02, vw02, vout0, vout1, vout2, vout3); + _mm256_expand_mul_add(v10, vw10, vout0, vout1, vout2, vout3); + _mm256_expand_mul_add(v11, vw11, vout0, vout1, vout2, vout3); + _mm256_expand_mul_add(v12, vw12, vout0, vout1, vout2, vout3); + _mm256_expand_mul_add(v20, vw20, vout0, vout1, vout2, vout3); + _mm256_expand_mul_add(v21, vw21, vout0, vout1, vout2, vout3); + _mm256_expand_mul_add(v22, vw22, vout0, vout1, vout2, vout3); + + vout0 = _mm256_add_epi32(voutzp, _mm256_cvtps_epi32(_mm256_mul_ps(_mm256_cvtepi32_ps(vout0), vmult))); + vout1 = _mm256_add_epi32(voutzp, _mm256_cvtps_epi32(_mm256_mul_ps(_mm256_cvtepi32_ps(vout1), vmult))); + vout2 = _mm256_add_epi32(voutzp, _mm256_cvtps_epi32(_mm256_mul_ps(_mm256_cvtepi32_ps(vout2), vmult))); + vout3 = _mm256_add_epi32(voutzp, _mm256_cvtps_epi32(_mm256_mul_ps(_mm256_cvtepi32_ps(vout3), vmult))); + + vout0 = _mm256_min_epi32(_mm256_max_epi32(vout0, outmin), outmax); + vout1 = _mm256_min_epi32(_mm256_max_epi32(vout1, outmin), outmax); + vout2 = _mm256_min_epi32(_mm256_max_epi32(vout2, outmin), outmax); + vout3 = _mm256_min_epi32(_mm256_max_epi32(vout3, outmin), outmax); + + _mm256_storeu_si256((__m256i*)(outptr + out_j), vout0); + _mm256_storeu_si256((__m256i*)(outptr + out_j + 8), vout1); + _mm256_storeu_si256((__m256i*)(outptr + out_j + 16), vout2); + _mm256_storeu_si256((__m256i*)(outptr + out_j + 24), vout3); + } + } + else + { + for( ; out_j < outW1; out_j += VECSZ ) + { + if (out_j + VECSZ > outW1) + { + if (out_j <= pad_l) + break; + out_j = outW1 - VECSZ; + } + int in_j = out_j * stride_w - pad_l; + __m256i v00, v01, v02, v10, v11, v12, v20, v21, v22, unused; + _mm256_load_deinterleave(imgptr0 + in_j, v00, v01); + _mm256_load_deinterleave(imgptr0 + in_j + 2, v02, unused); + _mm256_load_deinterleave(imgptr1 + in_j, v10, v11); + _mm256_load_deinterleave(imgptr1 + in_j + 2, v12, unused); + _mm256_load_deinterleave(imgptr2 + in_j, v20, v21); + _mm256_load_deinterleave(imgptr2 + in_j + 2, v22, unused); + + vout0 = vout1 = vout2 = vout3 = vbias; + _mm256_expand_mul_add(v00, vw00, vout0, vout1, vout2, vout3); + _mm256_expand_mul_add(v01, vw01, vout0, vout1, vout2, vout3); + _mm256_expand_mul_add(v02, vw02, vout0, vout1, vout2, vout3); + _mm256_expand_mul_add(v10, vw10, vout0, vout1, vout2, vout3); + _mm256_expand_mul_add(v11, vw11, vout0, vout1, vout2, vout3); + _mm256_expand_mul_add(v12, vw12, vout0, vout1, vout2, vout3); + _mm256_expand_mul_add(v20, vw20, vout0, vout1, vout2, vout3); + _mm256_expand_mul_add(v21, vw21, vout0, vout1, vout2, vout3); + _mm256_expand_mul_add(v22, vw22, vout0, vout1, vout2, vout3); + + vout0 = _mm256_add_epi32(voutzp, _mm256_cvtps_epi32(_mm256_mul_ps(_mm256_cvtepi32_ps(vout0), vmult))); + vout1 = _mm256_add_epi32(voutzp, _mm256_cvtps_epi32(_mm256_mul_ps(_mm256_cvtepi32_ps(vout1), vmult))); + vout2 = _mm256_add_epi32(voutzp, _mm256_cvtps_epi32(_mm256_mul_ps(_mm256_cvtepi32_ps(vout2), vmult))); + vout3 = _mm256_add_epi32(voutzp, _mm256_cvtps_epi32(_mm256_mul_ps(_mm256_cvtepi32_ps(vout3), vmult))); + + vout0 = _mm256_min_epi32(_mm256_max_epi32(vout0, outmin), outmax); + vout1 = _mm256_min_epi32(_mm256_max_epi32(vout1, outmin), outmax); + vout2 = _mm256_min_epi32(_mm256_max_epi32(vout2, outmin), outmax); + vout3 = _mm256_min_epi32(_mm256_max_epi32(vout3, outmin), outmax); + + _mm256_storeu_si256((__m256i*)(outptr + out_j), vout0); + _mm256_storeu_si256((__m256i*)(outptr + out_j + 8), vout1); + _mm256_storeu_si256((__m256i*)(outptr + out_j + 16), vout2); + _mm256_storeu_si256((__m256i*)(outptr + out_j + 24), vout3); + } + } + } + + for (; out_j < outW1; out_j++) + { + int in_j = out_j * stride_w - pad_l; + out = (int)imgptr0[in_j]*w00 + (int)imgptr0[in_j + dilation_w]*w01 + (int)imgptr0[in_j + dilation_w*2]*w02 + + (int)imgptr1[in_j]*w10 + (int)imgptr1[in_j + dilation_w]*w11 + (int)imgptr1[in_j + dilation_w*2]*w12 + + (int)imgptr2[in_j]*w20 + (int)imgptr2[in_j + dilation_w]*w21 + (int)imgptr2[in_j + dilation_w*2]*w22 + biasCopy; + outptr[out_j] = std::min(std::max(outZp + (int)std::round(out*mult), -128), 127); + } + + for (; out_j < outW; out_j++ ) + { + int in_j0 = out_j * stride_w - pad_l, in_j1 = in_j0 + dilation_w, in_j2 = in_j0 + dilation_w*2; + int s0 = 1, s1 = 1, s2 = 1; + if (in_j0 >= width) + { + in_j0 = 0; + s0 = 0; + biasCopy += inpZp*(w00 + w10 + w20); + } + if (in_j1 >= width) + { + in_j1 = 0; + s1 = 0; + biasCopy += inpZp*(w01 + w11 + w21); + } + if (in_j2 >= width) + { + in_j2 = 0; + s2 = 0; + biasCopy += inpZp*(w02 + w12 + w22); + } + out = (int)imgptr0[in_j0]*w00*s0 + (int)imgptr0[in_j1]*w01*s1 + (int)imgptr0[in_j2]*w02*s2 + + (int)imgptr1[in_j0]*w10*s0 + (int)imgptr1[in_j1]*w11*s1 + (int)imgptr1[in_j2]*w12*s2 + + (int)imgptr2[in_j0]*w20*s0 + (int)imgptr2[in_j1]*w21*s1 + (int)imgptr2[in_j2]*w22*s2 + biasCopy; + outptr[out_j] = std::min(std::max(outZp + (int)std::round(out*mult), -128), 127); + } + } + _mm256_zeroupper(); +} + +// dst = vec * weights^t + bias +void fastGEMM1T( const int8_t* vec, const int8_t* weights, + size_t wstep, const int* bias, const float* multiplier, + int* dst, int nvecs, int vecsize, int outZp ) +{ + int i = 0; + + for( ; i <= nvecs - 8; i += 8 ) + { + const int8_t* wptr = weights + i*wstep; + __m256i vs0 = _mm256_setzero_si256(), vs1 = _mm256_setzero_si256(), + vs2 = _mm256_setzero_si256(), vs3 = _mm256_setzero_si256(), + vs4 = _mm256_setzero_si256(), vs5 = _mm256_setzero_si256(), + vs6 = _mm256_setzero_si256(), vs7 = _mm256_setzero_si256(); + + __m128i voutzp = _mm_set1_epi32(outZp); + __m128i outmin = _mm_set1_epi32(-128), outmax = _mm_set1_epi32(127); + + for( int k = 0; k < vecsize; k += 32, wptr += 32 ) + { + __m256i v = _mm256_load_si256((const __m256i*)(vec + k)); + + vs0 = _mm256_fmaddepi8_epi32(_mm256_load_si256((const __m256i*)wptr), v, vs0); + vs1 = _mm256_fmaddepi8_epi32(_mm256_load_si256((const __m256i*)(wptr + wstep)), v, vs1); + vs2 = _mm256_fmaddepi8_epi32(_mm256_load_si256((const __m256i*)(wptr + wstep*2)), v, vs2); + vs3 = _mm256_fmaddepi8_epi32(_mm256_load_si256((const __m256i*)(wptr + wstep*3)), v, vs3); + vs4 = _mm256_fmaddepi8_epi32(_mm256_load_si256((const __m256i*)(wptr + wstep*4)), v, vs4); + vs5 = _mm256_fmaddepi8_epi32(_mm256_load_si256((const __m256i*)(wptr + wstep*5)), v, vs5); + vs6 = _mm256_fmaddepi8_epi32(_mm256_load_si256((const __m256i*)(wptr + wstep*6)), v, vs6); + vs7 = _mm256_fmaddepi8_epi32(_mm256_load_si256((const __m256i*)(wptr + wstep*7)), v, vs7); + } + + __m256i s0 = _mm256_hadd_epi32(_mm256_hadd_epi32(vs0, vs1), _mm256_hadd_epi32(vs2, vs3)); + __m256i s1 = _mm256_hadd_epi32(_mm256_hadd_epi32(vs4, vs5), _mm256_hadd_epi32(vs6, vs7)); + + s0 = _mm256_add_epi32(s0, _mm256_permute2x128_si256(s0, s0, 1)); + s1 = _mm256_add_epi32(s1, _mm256_permute2x128_si256(s1, s1, 1)); + + __m128i t0 = _mm_add_epi32(_mm256_castsi256_si128(s0), _mm_loadu_si128((__m128i*)(bias + i))); + __m128i t1 = _mm_add_epi32(_mm256_castsi256_si128(s1), _mm_loadu_si128((__m128i*)(bias + i + 4))); + + t0 = _mm_add_epi32(voutzp, _mm_cvtps_epi32(_mm_mul_ps(_mm_cvtepi32_ps(t0), _mm_loadu_ps(multiplier + i)))); + t1 = _mm_add_epi32(voutzp, _mm_cvtps_epi32(_mm_mul_ps(_mm_cvtepi32_ps(t1), _mm_loadu_ps(multiplier + i + 4)))); + + t0 = _mm_min_epi32(_mm_max_epi32(t0, outmin), outmax); + t1 = _mm_min_epi32(_mm_max_epi32(t1, outmin), outmax); + + _mm_storeu_si128((__m128i*)(dst + i), t0); + _mm_storeu_si128((__m128i*)(dst + i + 4), t1); + } + + for( ; i < nvecs; i++ ) + { + const int8_t* wptr = weights + i*wstep; + __m256i vs0 = _mm256_setzero_si256(); + + for( int k = 0; k < vecsize; k += 32, wptr += 32 ) + { + __m256i v = _mm256_load_si256((const __m256i*)(vec + k)); + vs0 = _mm256_fmaddepi8_epi32(_mm256_load_si256((const __m256i*)wptr), v, vs0); + } + + __m256i s0 = _mm256_hadd_epi32(_mm256_hadd_epi32(vs0, vs0), vs0); + s0 = _mm256_add_epi32(s0, _mm256_permute2x128_si256(s0, s0, 1)); + int temp = _mm_extract_epi32(_mm256_castsi256_si128(s0), 0); + dst[i] = outZp + (int)std::round((temp + bias[i]) * multiplier[i]); + } + + _mm256_zeroupper(); +} +#endif // CV_CPU_OPTIMIZATION_DECLARATIONS_ONLY + +CV_CPU_OPTIMIZATION_NAMESPACE_END +}} // namespace diff --git a/modules/dnn/src/int8layers/pooling_layer.cpp b/modules/dnn/src/int8layers/pooling_layer.cpp new file mode 100644 index 0000000000..20a0486a46 --- /dev/null +++ b/modules/dnn/src/int8layers/pooling_layer.cpp @@ -0,0 +1,595 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. + +#include "../precomp.hpp" +#include "layers_common.hpp" +#include "opencv2/core/hal/intrin.hpp" + +#include +#include +#include +using std::max; +using std::min; + +namespace cv +{ +namespace dnn +{ + +class PoolingLayerInt8Impl CV_FINAL : public PoolingLayerInt8 +{ +public: + PoolingLayerInt8Impl(const LayerParams& params) + { + computeMaxIdx = false; + globalPooling = false; + isGlobalPooling = std::vector(3, false); + output_zp = params.get("zeropoints"); + input_zp = params.get("input_zeropoint", 0); + multiplier = params.get("multiplier", 1.f); + + hasDynamicShapes = params.get("has_dynamic_shapes", false); + shapesInitialized = !hasDynamicShapes; + + if (params.has("pool") || params.has("kernel_size") || + params.has("kernel_w") || params.has("kernel_h")) + { + String pool = toLowerCase(params.get("pool", "max")); + if (pool == "max") + type = MAX; + else if (pool == "ave") + type = AVE; + else if (pool == "sum") + type = SUM; + else + CV_Error(Error::StsBadArg, "Unknown pooling type \"" + pool + "\""); + + getPoolingKernelParams(params, kernel_size, isGlobalPooling, pads_begin, pads_end, strides, padMode); + globalPooling = isGlobalPooling[0] || isGlobalPooling[1] || isGlobalPooling[2]; + } + else + CV_Error(Error::StsBadArg, "Cannot determine pooling type"); + setParamsFrom(params); + ceilMode = params.get("ceil_mode", true); + spatialScale = params.get("spatial_scale", 1); + avePoolPaddedArea = params.get("ave_pool_padded_area", true); + } + + void finalize(InputArrayOfArrays inputs_arr, OutputArrayOfArrays outputs_arr) CV_OVERRIDE + { + std::vector inputs, outputs; + inputs_arr.getMatVector(inputs); + outputs_arr.getMatVector(outputs); + + CV_Assert(!inputs.empty()); + CV_Assert(outputs.size() == 1); + + std::vector inp; + std::vector out; + for (int i = 2; i < inputs[0].dims; i++) { + inp.push_back(inputs[0].size[i]); + out.push_back(outputs[0].size[i]); + } + if (globalPooling) { + std::vector finalKernel; + for (int i = 0; i < inp.size(); i++) { + int idx = isGlobalPooling.size() - inp.size() + i; + finalKernel.push_back(isGlobalPooling[idx] ? inp[i] : kernel_size[idx]); + } + kernel_size = finalKernel; + } + + getConvPoolPaddings(inp, kernel_size, strides, padMode, pads_begin, pads_end); + + if (inputs[0].dims == 3) + { + // Pool1D + kernel_size.assign(1, kernel_size[0]); + strides.assign(1, strides[0]); + pads_begin.assign(1, pads_begin[0]); + pads_end.assign(1, pads_end[0]); + } + } + + virtual bool supportBackend(int backendId) CV_OVERRIDE + { + if (backendId == DNN_BACKEND_OPENCV) + { + if (kernel_size.size() == 3) + return preferableTarget == DNN_TARGET_CPU; + if (kernel_size.size() <= 2) + return true; + else + return false; + } + return false; + } + + bool setActivation(const Ptr& layer) CV_OVERRIDE + { + Ptr activ_int8 = layer.dynamicCast(); + if (!activ_int8.empty()) + { + return activ_int8->blobs.empty(); + } + return false; + } + + void forward(InputArrayOfArrays inputs_arr, OutputArrayOfArrays outputs_arr, OutputArrayOfArrays internals_arr) CV_OVERRIDE + { + CV_TRACE_FUNCTION(); + CV_TRACE_ARG_VALUE(name, "name", name.c_str()); + + std::vector inputs, outputs; + inputs_arr.getMatVector(inputs); + outputs_arr.getMatVector(outputs); + + switch (type) + { + case MAX: + { + CV_Assert_N(inputs.size() == 1, outputs.size() == 1); + maxPooling(inputs[0], outputs[0]); + break; + } + case AVE: case SUM: + CV_Assert_N(inputs.size() == 1, outputs.size() == 1); + avePooling(inputs[0], outputs[0]); + break; + default: + CV_Error(Error::StsNotImplemented, "Not implemented"); + break; + } + } + + class PoolingInvoker : public ParallelLoopBody + { + public: + const Mat* src, *rois; + Mat *dst; + int pad_l, pad_t, pad_r, pad_b; + bool avePoolPaddedArea; + int nstripes, inpZp, outZp; + std::vector ofsbuf; + int poolingType; + float multiplier; + float spatialScale; + + std::vector pads_begin, pads_end; + std::vector kernel_size; + std::vector strides; + + PoolingInvoker() : src(0), rois(0), dst(0), pad_l(0), pad_t(0), pad_r(0), pad_b(0), + avePoolPaddedArea(false), nstripes(0), inpZp(0), outZp(0), + poolingType(MAX), multiplier(1), spatialScale(0){} + + static void run(const Mat& src, const Mat& rois, Mat& dst, + std::vector kernel_size, std::vector strides, + std::vector pads_begin, std::vector pads_end, + bool avePoolPaddedArea, int poolingType, float spatialScale, + float multiplier, int inpZp, int outZp, int nstripes) + { + CV_Assert_N( + src.isContinuous(), dst.isContinuous(), + src.type() == CV_8S, src.type() == dst.type(), + src.dims == 3 || src.dims == 4 || src.dims == 5, dst.dims == 3 || dst.dims == 4 || dst.dims == 5, + src.size[0] == dst.size[0], src.size[1] == dst.size[1], rois.empty()); + + PoolingInvoker p; + + bool isPool1D = src.dims == 3; + bool isPool3D = src.dims == 5; + + p.src = &src; + p.rois = &rois; + p.dst = &dst; + + p.kernel_size = kernel_size; + p.strides = strides; + p.pads_begin = pads_begin; + p.pads_end = pads_end; + + p.pad_l = pads_begin.back(); + p.pad_t = isPool1D ? 0 : pads_begin[pads_begin.size() - 2]; + p.pad_r = pads_end.back(); + p.pad_b = isPool1D ? 0 : pads_end[pads_end.size() - 2]; + + p.avePoolPaddedArea = avePoolPaddedArea; + p.nstripes = nstripes; + p.inpZp = inpZp; + p.outZp = outZp; + p.poolingType = poolingType; + p.spatialScale = spatialScale; + p.multiplier = multiplier; + + int height = isPool1D ? 1 : src.size[src.dims - 2]; + int width = src.size[src.dims - 1]; + + int kernel_d = isPool3D ? kernel_size[0] : 1; + int kernel_h = isPool1D ? 1 : kernel_size[kernel_size.size() - 2]; + int kernel_w = kernel_size.back(); + + p.ofsbuf.resize(kernel_d * kernel_h * kernel_w); + for (int i = 0; i < kernel_d; ++i) { + for (int j = 0; j < kernel_h; ++j) { + for (int k = 0; k < kernel_w; ++k) { + p.ofsbuf[i * kernel_h * kernel_w + j * kernel_w + k] = width * height * i + width * j + k; + } + } + } + + parallel_for_(Range(0, nstripes), p, nstripes); + } + + void operator()(const Range& r) const CV_OVERRIDE + { + int channels = dst->size[1]; + + bool isPool3D = src->dims == 5; + bool isPool2D = src->dims == 4; + bool isPool1D = src->dims == 3; + int depth = isPool3D? dst->size[2] : 1; + int height = isPool1D? 1 : dst->size[dst->dims - 2]; + int width = dst->size[dst->dims - 1]; + + int inp_depth = isPool3D? src->size[2] : 1; + int inp_height = isPool1D? 1 : src->size[src->dims - 2]; + int inp_width = src->size[src->dims - 1]; + + size_t total = dst->total(); + size_t stripeSize = (total + nstripes - 1)/nstripes; + size_t stripeStart = r.start*stripeSize; + size_t stripeEnd = std::min(r.end*stripeSize, total); + + int kernel_d = isPool3D? kernel_size[0] : 1; + int kernel_h = isPool1D? 1 : kernel_size[kernel_size.size() - 2]; + int kernel_w = kernel_size.back(); + + int stride_d = isPool3D? strides[0] : 0; + int stride_h = isPool1D? 1 :strides[strides.size() - 2]; + int stride_w = strides.back(); + +#if CV_SIMD128 + const int* ofsptr = (const int*)&ofsbuf[0]; + if (poolingType == MAX && !ofsptr) + CV_Error(Error::StsBadArg, "ofsbuf should be initialized in this mode"); +#endif + + for( size_t ofs0 = stripeStart; ofs0 < stripeEnd; ) + { + size_t ofs = ofs0; + int x0 = (int)(ofs % width); + ofs /= width; + int y0 = (int)(ofs % height); + ofs /= height; + + int d0 = (int)(ofs % depth); + ofs /= depth; + + int c = (int)(ofs % channels); + int n = (int)(ofs / channels); + int ystart, yend; + int dstart = 0, dend = 1; + + const int8_t *srcData = 0; + int pad_d_begin = (pads_begin.size() == 3) ? pads_begin[0] : 0; + dstart = d0 * stride_d - pad_d_begin; + dend = min(dstart + kernel_d, (int)(inp_depth + pads_end[0])); + + ystart = y0 * stride_h - pad_t; + yend = min(ystart + kernel_h, inp_height + pad_b); + srcData = src->ptr(n, c); + + int ddelta = dend - dstart; + dstart = max(dstart, 0); + dend = min(dend, inp_depth); + int ydelta = yend - ystart; + ystart = max(ystart, 0); + yend = min(yend, inp_height); + int8_t *dstData = &dst->ptr(n, c, d0)[y0 * width]; + + int delta = std::min((int)(stripeEnd - ofs0), width - x0); + ofs0 += delta; + int x1 = x0 + delta; + + if( poolingType == MAX ) + for( ; x0 < x1; x0++ ) + { + int xstart = x0 * stride_w - pad_l; + int xend = min(xstart + kernel_w, inp_width); + xstart = max(xstart, 0); + if (xstart >= xend || ystart >= yend) + { + dstData[x0] = (int8_t)outZp; + continue; + } +#if CV_SIMD128 + if( isPool2D && xstart > 0 && x0 + 15 < x1 && (x0 + 15) * stride_w - pad_l + kernel_w < inp_width ) + { + v_int8x16 max_val0 = v_setall_s8(-128); + if( yend - ystart == kernel_h ) + { + const int8_t* srcData1 = srcData + ystart*inp_width + xstart; + if( stride_w == 1 ) + for (int k = 0; k < kernel_w*kernel_h; k++) + { + int index = ofsptr[k]; + v_int8x16 v0 = v_load(srcData1 + index); + max_val0 = v_max(max_val0, v0); + } + else if( stride_w == 2 ) + for (int k = 0; k < kernel_w*kernel_h; k++) + { + int index = ofsptr[k]; + v_int8x16 v0, dummy; + v_load_deinterleave(srcData1 + index, v0, dummy); + max_val0 = v_max(max_val0, v0); + } + else + for (int k = 0; k < kernel_w*kernel_h; k++) + { + int index = ofsptr[k]; + v_int8x16 v0(srcData1[index], srcData1[index + stride_w], + srcData1[index + stride_w*2], srcData1[index + stride_w*3], + srcData1[index + stride_w*4], srcData1[index + stride_w*5], + srcData1[index + stride_w*6], srcData1[index + stride_w*7], + srcData1[index + stride_w*8], srcData1[index + stride_w*9], + srcData1[index + stride_w*10], srcData1[index + stride_w*11], + srcData1[index + stride_w*12], srcData1[index + stride_w*13], + srcData1[index + stride_w*14], srcData1[index + stride_w*15]); + max_val0 = v_max(max_val0, v0); + } + } + else + { + for (int y = ystart; y < yend; ++y) + { + for (int x = xstart; x < xend; ++x) + { + const int index = y * inp_width + x; + v_int8x16 v0(srcData[index], srcData[index + stride_w], + srcData[index + stride_w*2], srcData[index + stride_w*3], + srcData[index + stride_w*4], srcData[index + stride_w*5], + srcData[index + stride_w*6], srcData[index + stride_w*7], + srcData[index + stride_w*8], srcData[index + stride_w*9], + srcData[index + stride_w*10], srcData[index + stride_w*11], + srcData[index + stride_w*12], srcData[index + stride_w*13], + srcData[index + stride_w*14], srcData[index + stride_w*15]); + max_val0 = v_max(max_val0, v0); + } + } + } + v_store(dstData + x0, max_val0); + x0 += 15; + } + else +#else + CV_UNUSED(isPool2D); +#endif + if( isPool1D ) + { + const int8_t* first = srcData + xstart; + const int8_t* last = srcData + xend; + const int8_t* max_elem = std::max_element(first, last); + if (max_elem != last) + dstData[x0] = *max_elem; + } + else + { + int8_t max_val = -128; + for (int d = dstart; d < dend; ++d) { + for (int y = ystart; y < yend; ++y) { + for (int x = xstart; x < xend; ++x) { + const int index = d * inp_width * inp_height + y * inp_width + x; + int8_t val = srcData[index]; + max_val = std::max(max_val, val); + } + } + } + dstData[x0] = max_val; + } + } + else if (poolingType == AVE || poolingType == SUM) + { + for( ; x0 < x1; ++x0) + { + int xstart = x0 * stride_w - pad_l; + int xend = min(xstart + kernel_w, inp_width + pad_r); + int xdelta = xend - xstart; + xstart = max(xstart, 0); + xend = min(xend, inp_width); + + int real_kernel_area = (dend - dstart) * (yend - ystart) * (xend - xstart); + int padded_kernel_area = xdelta * ydelta * ddelta; + int kernel_area = avePoolPaddedArea ? padded_kernel_area : real_kernel_area; + + int bias = (avePoolPaddedArea ? (padded_kernel_area - real_kernel_area) * inpZp : 0) + - (inpZp * kernel_area); + float inv_kernel_area = poolingType == AVE ? multiplier / kernel_area : multiplier; +#if CV_SIMD128 + if( isPool2D && xstart > 0 && x0 + 15 < x1 && (x0 + 15) * stride_w - pad_l + kernel_w < inp_width ) + { + v_int32x4 sum_val0 = v_setall_s32(bias), sum_val1 = v_setall_s32(bias), + sum_val2 = v_setall_s32(bias), sum_val3 = v_setall_s32(bias), + voutzp = v_setall_s32(outZp); + v_float32x4 ikarea = v_setall_f32(inv_kernel_area); + + for (int y = ystart; y < yend; ++y) + { + for (int x = xstart; x < xend; ++x) + { + const int index = y * inp_width + x; + v_int32x4 v0((int)srcData[index], (int)srcData[index + stride_w], + (int)srcData[index + stride_w*2], (int)srcData[index + stride_w*3]); + v_int32x4 v1((int)srcData[index + stride_w*4], (int)srcData[index + stride_w*5], + (int)srcData[index + stride_w*6], (int)srcData[index + stride_w*7]); + v_int32x4 v2((int)srcData[index + stride_w*8], (int)srcData[index + stride_w*9], + (int)srcData[index + stride_w*10], (int)srcData[index + stride_w*11]); + v_int32x4 v3((int)srcData[index + stride_w*12], (int)srcData[index + stride_w*13], + (int)srcData[index + stride_w*14], (int)srcData[index + stride_w*15]); + sum_val0 += v0; + sum_val1 += v1; + sum_val2 += v2; + sum_val3 += v3; + } + } + + sum_val0 = v_round(v_cvt_f32(sum_val0)*ikarea) + voutzp; + sum_val1 = v_round(v_cvt_f32(sum_val1)*ikarea) + voutzp; + sum_val2 = v_round(v_cvt_f32(sum_val2)*ikarea) + voutzp; + sum_val3 = v_round(v_cvt_f32(sum_val3)*ikarea) + voutzp; + + v_store(dstData + x0, v_pack(v_pack(sum_val0, sum_val1), v_pack(sum_val2, sum_val3))); + x0 += 15; + } + else +#endif + if( isPool1D ) + { + const int8_t* first = srcData + xstart; + const int8_t* last = srcData + xend; + int sum_val = bias + std::accumulate(first, last, 0); + dstData[x0] = saturate_cast(outZp + std::round(sum_val*inv_kernel_area)); + } + else + { + int sum_val = bias; + for (int d = dstart; d < dend; ++d) { + for (int y = ystart; y < yend; ++y) { + for (int x = xstart; x < xend; ++x) { + const int index = d * inp_width * inp_height + y * inp_width + x; + int8_t val = srcData[index]; + sum_val += (int)val; + } + } + } + dstData[x0] = saturate_cast(outZp + std::round(sum_val*inv_kernel_area)); + } + } + } + } + } + }; + + void maxPooling(Mat &src, Mat &dst) + { + const int nstripes = getNumThreads(); + Mat rois; + PoolingInvoker::run(src, rois, dst, kernel_size, strides, pads_begin, pads_end, avePoolPaddedArea, type, + spatialScale, multiplier, input_zp, output_zp, nstripes); + } + + void avePooling(Mat &src, Mat &dst) + { + const int nstripes = getNumThreads(); + Mat rois; + PoolingInvoker::run(src, rois, dst, kernel_size, strides, pads_begin, pads_end, avePoolPaddedArea, type, + spatialScale, multiplier, input_zp, output_zp, nstripes); + } + + bool getMemoryShapes(const std::vector &inputs, + const int requiredOutputs, + std::vector &outputs, + std::vector &internals) const CV_OVERRIDE + { + CV_Assert(inputs.size() != 0); + + bool isPool1D = inputs[0].size() == 3; + std::vector inpShape(inputs[0].begin() + 2, inputs[0].end()); + std::vector outShape(inputs[0].begin(), inputs[0].begin() + 2); + + std::vector local_kernel; + if (globalPooling) { + for (int i = 0; i < inpShape.size(); i++) { + int idx = isGlobalPooling.size() - inpShape.size() + i; + local_kernel.push_back(isGlobalPooling[idx] ? inpShape[i] : kernel_size[idx]); + } + } else { + local_kernel = kernel_size; + } + + if (hasDynamicShapes && !shapesInitialized) + { + //Just copy input shapes for width and height to prevent errors on loading stage + for (int i = 0; i < inpShape.size(); i++) + outShape.push_back(inpShape[i]); + } + else if (padMode.empty()) + { + int addedDims = isPool1D? inpShape.size() : local_kernel.size(); + for (int i = 0; i < addedDims; i++) { + float dst = (float) (inpShape[i] + pads_begin[i] + pads_end[i] - local_kernel[i]) / strides[i]; + outShape.push_back(1 + (ceilMode ? ceil(dst) : floor(dst))); + } + + // If we have padding, ensure that the last pooling starts strictly + // inside the image (instead of at the padding); otherwise clip the last. + for (int i = 0; i < addedDims; i++) { + if (pads_end[i] && (outShape[2 + i] - 1) * strides[i] >= inpShape[i] + pads_end[i]) { + --outShape[2 + i]; + CV_Assert((outShape[2 + i] - 1) * strides[i] < inpShape[i] + pads_end[i]); + } + } + } + else { + getConvPoolOutParams(inpShape, local_kernel, strides, padMode, + std::vector(local_kernel.size(), 1), outShape); + } + + outputs.assign(1, outShape); + return false; + } + + bool updateMemoryShapes(const std::vector &inputs) CV_OVERRIDE + { + int dims = inputs[0].size(); + CV_Assert(inputs[0][dims - 1] > 0 && inputs[0][dims - 2] > 0); + shapesInitialized = true; + return true; + } + + virtual int64 getFLOPS(const std::vector &inputs, + const std::vector &outputs) const CV_OVERRIDE + { + CV_UNUSED(inputs); // suppress unused variable warning + long flops = 0; + bool isPool1D = inputs[0].size() == 3; + size_t karea = std::accumulate(kernel_size.begin(), isPool1D? kernel_size.begin() + 1 : kernel_size.end(), + 1, std::multiplies()); + for(int i = 0; i < outputs.size(); i++) + { + if (type == MAX) + { + if (i%2 == 0) + flops += total(outputs[i])*karea; + } + else + { + flops += total(outputs[i])*(karea + 1); + } + } + return flops; + } +private: + enum Type + { + MAX, + AVE, + STOCHASTIC, + SUM, + ROI, // RoI pooling, https://arxiv.org/pdf/1504.08083.pdf + PSROI // Position-sensitive RoI pooling, https://arxiv.org/pdf/1605.06409.pdf + }; + bool hasDynamicShapes; + bool shapesInitialized; + float multiplier; +}; + +Ptr PoolingLayerInt8::create(const LayerParams& params) +{ + return Ptr(new PoolingLayerInt8Impl(params)); +} + +} +} diff --git a/modules/dnn/src/int8layers/quantize_dequantize_layer.cpp b/modules/dnn/src/int8layers/quantize_dequantize_layer.cpp new file mode 100644 index 0000000000..2ddb76a0e8 --- /dev/null +++ b/modules/dnn/src/int8layers/quantize_dequantize_layer.cpp @@ -0,0 +1,157 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. + +#include "../precomp.hpp" +#include "layers_common.hpp" + +namespace cv +{ +namespace dnn +{ + +class QuantizeLayerImpl CV_FINAL : public QuantizeLayer +{ +public: + QuantizeLayerImpl(const LayerParams& params) + { + scale = params.get("scales", 1.0f); + zeropoint = params.get("zeropoints", 0); + setParamsFrom(params); + } + + virtual bool supportBackend(int backendId) CV_OVERRIDE + { + return backendId == DNN_BACKEND_OPENCV; + } + + bool getMemoryShapes(const std::vector &inputs, + const int requiredOutputs, + std::vector &outputs, + std::vector &internals) const CV_OVERRIDE + { + CV_Assert(inputs.size() == 1); + Layer::getMemoryShapes(inputs, requiredOutputs, outputs, internals); + return false; + } + + virtual void finalize(InputArrayOfArrays inputs_arr, OutputArrayOfArrays outputs_arr) CV_OVERRIDE + { + std::vector inputs, outputs; + inputs_arr.getMatVector(inputs); + outputs_arr.getMatVector(outputs); + } + +#ifdef HAVE_OPENCL + bool forward_ocl(InputArrayOfArrays inputs_, OutputArrayOfArrays outputs_, OutputArrayOfArrays internals_) + { + std::vector inputs, outputs; + inputs_.getUMatVector(inputs); + outputs_.getUMatVector(outputs); + + if (inputs_.depth() == CV_16S) + { + UMat inputFp32(shape(inputs[0]), CV_32F); + convertFp16(inputs[0], inputFp32); + inputFp32.copyTo(inputs[0]); + } + + inputs[0].convertTo(outputs[0], CV_8S, 1.f/scale, zeropoint); + return true; + } +#endif + + void forward(InputArrayOfArrays inputs_arr, OutputArrayOfArrays outputs_arr, OutputArrayOfArrays internals_arr) CV_OVERRIDE + { + CV_TRACE_FUNCTION(); + CV_TRACE_ARG_VALUE(name, "name", name.c_str()); + + CV_OCL_RUN(IS_DNN_OPENCL_TARGET(preferableTarget), + forward_ocl(inputs_arr, outputs_arr, internals_arr)) + + std::vector inputs, outputs; + inputs_arr.getMatVector(inputs); + outputs_arr.getMatVector(outputs); + + inputs[0].convertTo(outputs[0], CV_8S, 1.f/scale, zeropoint); + } +}; + +class DequantizeLayerImpl CV_FINAL : public DequantizeLayer +{ +public: + DequantizeLayerImpl(const LayerParams& params) + { + scale = params.get("scales", 1.0f); + zeropoint = params.get("zeropoints", 0); + setParamsFrom(params); + } + + virtual bool supportBackend(int backendId) CV_OVERRIDE + { + return backendId == DNN_BACKEND_OPENCV; + } + + bool getMemoryShapes(const std::vector &inputs, + const int requiredOutputs, + std::vector &outputs, + std::vector &internals) const CV_OVERRIDE + { + CV_Assert(inputs.size() == 1); + Layer::getMemoryShapes(inputs, requiredOutputs, outputs, internals); + return false; + } + + virtual void finalize(InputArrayOfArrays inputs_arr, OutputArrayOfArrays outputs_arr) CV_OVERRIDE + { + std::vector inputs, outputs; + inputs_arr.getMatVector(inputs); + outputs_arr.getMatVector(outputs); + } + +#ifdef HAVE_OPENCL + bool forward_ocl(InputArrayOfArrays inputs_, OutputArrayOfArrays outputs_, OutputArrayOfArrays internals_) + { + std::vector inputs, outputs; + inputs_.getUMatVector(inputs); + outputs_.getUMatVector(outputs); + + UMat outputFp32(shape(outputs[0]), CV_32F); + inputs[0].convertTo(outputFp32, CV_32F, scale, -(scale*zeropoint)); + + if (outputs_.depth() == CV_16S) + convertFp16(outputFp32, outputs[0]); + else + outputFp32.copyTo(outputs[0]); + return true; + } +#endif + + void forward(InputArrayOfArrays inputs_arr, OutputArrayOfArrays outputs_arr, OutputArrayOfArrays internals_arr) CV_OVERRIDE + { + CV_TRACE_FUNCTION(); + CV_TRACE_ARG_VALUE(name, "name", name.c_str()); + + CV_OCL_RUN(IS_DNN_OPENCL_TARGET(preferableTarget), + forward_ocl(inputs_arr, outputs_arr, internals_arr)) + + std::vector inputs, outputs; + inputs_arr.getMatVector(inputs); + outputs_arr.getMatVector(outputs); + + inputs[0].convertTo(outputs[0], CV_32F, scale, -(scale*zeropoint)); + } +}; + +Ptr QuantizeLayer::create(const LayerParams& params) +{ + return Ptr(new QuantizeLayerImpl(params)); +} + +Ptr DequantizeLayer::create(const LayerParams& params) +{ + return Ptr(new DequantizeLayerImpl(params)); +} + +} +} diff --git a/modules/dnn/src/int8layers/scale_layer.cpp b/modules/dnn/src/int8layers/scale_layer.cpp new file mode 100644 index 0000000000..d7f676d047 --- /dev/null +++ b/modules/dnn/src/int8layers/scale_layer.cpp @@ -0,0 +1,211 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. + +#include "../precomp.hpp" +#include "layers_common.hpp" +#include +#include + +namespace cv +{ +namespace dnn +{ + +class ScaleLayerInt8Impl CV_FINAL : public ScaleLayerInt8 +{ +public: + Mat weights, bias; + ScaleLayerInt8Impl(const LayerParams& params) + { + setParamsFrom(params); + hasBias = params.get("bias_term", false); + axis = params.get("axis", 1); + hasWeights = false; + + output_sc = params.get("scales"); + output_zp = params.get("zeropoints"); + + DictValue inpSc = params.get("input_scales"); + DictValue inpZp = params.get("input_zeropoints"); + + for (int i = 0; i < inpSc.size(); i++) + { + inp_sc.push_back(inpSc.get(i)); + inp_zp.push_back(inpZp.get(i)); + } + } + + bool getMemoryShapes(const std::vector &inputs, + const int requiredOutputs, + std::vector &outputs, + std::vector &internals) const CV_OVERRIDE + { + outputs.assign(1, inputs[0]); + return true; + } + + virtual void finalize(InputArrayOfArrays inputs_arr, OutputArrayOfArrays) CV_OVERRIDE + { + std::vector inputs; + inputs_arr.getMatVector(inputs); + hasWeights = blobs.size() == 2 || (blobs.size() <= 1 && !hasBias); + CV_Assert((inputs.size() == 2 && blobs.empty()) || blobs.size() == (int)hasWeights + (int)hasBias); + + if (!blobs.empty()) + { + Mat w = hasWeights ? blobs[0] : Mat::ones(blobs[0].size(), CV_32F); + Mat b = hasBias ? blobs.back() : Mat::zeros(blobs.back().size(), CV_32F); + + w = w.reshape(1, 1); + b = b.reshape(1, 1); + + w.convertTo(weights, CV_32F, inp_sc[0]/output_sc); + addWeighted(b, 1.0/output_sc, weights, -inp_zp[0], output_zp, bias, CV_32F); + } + else + { + // initialized during forward() + weights = Mat(); bias = Mat(); + } + } + + virtual bool supportBackend(int backendId) CV_OVERRIDE + { + return backendId == DNN_BACKEND_OPENCV; + } + + bool setActivation(const Ptr& layer) CV_OVERRIDE + { + Ptr activ_int8 = layer.dynamicCast(); + if (!activ_int8.empty()) + { + return activ_int8->blobs.empty(); + } + return false; + } + + void forward(InputArrayOfArrays inputs_arr, OutputArrayOfArrays outputs_arr, OutputArrayOfArrays internals_arr) CV_OVERRIDE + { + CV_TRACE_FUNCTION(); + CV_TRACE_ARG_VALUE(name, "name", name.c_str()); + + std::vector inputs, outputs; + inputs_arr.getMatVector(inputs); + outputs_arr.getMatVector(outputs); + + Mat &inpBlob = inputs[0]; + Mat &outBlob = outputs[0]; + + if (blobs.empty()) + { + CV_Assert(inp_sc.size() == 2 && inp_zp.size() == 2); + Mat inp_dequantized, w, b; + inputs[1].reshape(1, 1).convertTo(inp_dequantized, CV_32F, inp_sc[1], -(inp_sc[1]*inp_zp[1])); + w = hasWeights ? inp_dequantized : Mat::ones(inp_dequantized.size(), CV_32F); + b = hasBias ? inp_dequantized : Mat::zeros(inp_dequantized.size(), CV_32F); + + w.convertTo(weights, CV_32F, inp_sc[0]/output_sc); + addWeighted(b, 1.0/output_sc, weights, -inp_zp[0], output_zp, bias, CV_32F); + } + + MatShape inpShape = shape(inpBlob); + const int numWeights = weights.total(); + CV_Assert(numWeights != 0); + CV_CheckEQ(weights.total(), bias.total(), "Incompatible weights/bias blobs"); + + int endAxis; + for (endAxis = axis + 1; endAxis <= inpBlob.dims; ++endAxis) + { + if (total(inpShape, axis, endAxis) == numWeights) + break; + } + CV_Assert(total(inpShape, axis, endAxis) == numWeights); + CV_CheckTypeEQ(inpBlob.type(), CV_8SC1, ""); CV_CheckTypeEQ(outBlob.type(), CV_8SC1, ""); + + int numSlices = total(inpShape, 0, axis); + int8_t* inpData = (int8_t*)inpBlob.data; + int8_t* outData = (int8_t*)outBlob.data; + + if (endAxis != inpBlob.dims) + { + float* weightsData = (float*)weights.data; + float* biasesData = (float*)bias.data; + int spatialSize = total(inpShape, endAxis); // spatialSize != 1 + for (int i = 0; i < numSlices; ++i) + { + for (int j = 0; j < numWeights; ++j) + { + float w = weightsData[j]; + float b = biasesData[j]; + Mat inpSlice(1, spatialSize, CV_8S, inpData); + Mat outSlice(1, spatialSize, CV_8S, outData); + inpSlice.convertTo(outSlice, CV_8S, w, b); + inpData += spatialSize; + outData += spatialSize; + } + } + } + else + { + for (int i = 0; i < numSlices; ++i) + { + Mat inpSlice(1, numWeights, CV_8S, inpData); + Mat outSlice(1, numWeights, CV_8S, outData); + + multiply(inpSlice, weights, outSlice, 1.0, CV_8S); + add(outSlice, bias, outSlice, Mat(), CV_8S); + + inpData += numWeights; + outData += numWeights; + } + } + } + + void getScaleShift(Mat& scale, Mat& shift) const CV_OVERRIDE + { + scale = (hasWeights && !blobs.empty()) ? blobs[0] : Mat(); + shift = (hasBias && !blobs.empty()) ? blobs.back() : Mat(); + } + + void getScaleZeropoint(float& scale, int& zeropoint) const CV_OVERRIDE + { + scale = output_sc; + zeropoint = output_zp; + } + + virtual int64 getFLOPS(const std::vector &inputs, + const std::vector &outputs) const CV_OVERRIDE + { + CV_UNUSED(outputs); // suppress unused variable warning + long flops = 0; + for(int i = 0; i < inputs.size(); i++) + { + flops += 2*total(inputs[i]); + } + return flops; + } + +private: + bool hasWeights; + std::vector inp_sc; + std::vector inp_zp; +}; + + +Ptr ScaleLayerInt8::create(const LayerParams& params) +{ + return Ptr(new ScaleLayerInt8Impl(params)); +} + +Ptr ShiftLayerInt8::create(const LayerParams& params) +{ + LayerParams scaleParams = params; + scaleParams.type = "ScaleInt8"; + scaleParams.set("bias_term", true); + scaleParams.set("axis", 0); + return Ptr(new ScaleLayerInt8Impl(scaleParams)); +} + +} // namespace dnn +} // namespace cv diff --git a/modules/dnn/src/int8layers/softmax_layer.cpp b/modules/dnn/src/int8layers/softmax_layer.cpp new file mode 100644 index 0000000000..7e3c82bc21 --- /dev/null +++ b/modules/dnn/src/int8layers/softmax_layer.cpp @@ -0,0 +1,176 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. + +#include "../precomp.hpp" +#include "layers_common.hpp" + +#include +#include + +namespace cv +{ +namespace dnn +{ + +class SoftMaxLayerInt8Impl CV_FINAL : public SoftmaxLayerInt8 +{ +public: + + SoftMaxLayerInt8Impl(const LayerParams& params) + { + axisRaw = params.get("axis", 1); + logSoftMax = params.get("log_softmax", false); + output_sc = params.get("scales"); + output_zp = params.get("zeropoints"); + setParamsFrom(params); + } + + bool getMemoryShapes(const std::vector &inputs, + const int requiredOutputs, + std::vector &outputs, + std::vector &internals) const CV_OVERRIDE + { + bool inplace = Layer::getMemoryShapes(inputs, requiredOutputs, outputs, internals); + MatShape shape = inputs[0]; + int cAxis = normalize_axis(axisRaw, shape.size()); + shape[cAxis] = 1; + internals.assign(1, shape); + return inplace; + } + + virtual bool supportBackend(int backendId) CV_OVERRIDE + { + return backendId == DNN_BACKEND_OPENCV; + } + + virtual bool tryFuse(Ptr& top) CV_OVERRIDE + { + Ptr dequantize_layer = top.dynamicCast(); + return !dequantize_layer.empty() && preferableTarget != DNN_TARGET_OPENCL_FP16; + } + + void forward(InputArrayOfArrays inputs_arr, OutputArrayOfArrays outputs_arr, OutputArrayOfArrays internals_arr) CV_OVERRIDE + { + CV_TRACE_FUNCTION(); + CV_TRACE_ARG_VALUE(name, "name", name.c_str()); + + std::vector inputs, outputs, internals; + inputs_arr.getMatVector(inputs); + outputs_arr.getMatVector(outputs); + internals_arr.getMatVector(internals); + + const Mat &src = inputs[0]; + Mat &dst = outputs[0]; + + int axis = normalize_axis(axisRaw, src.dims); + size_t outerSize = src.total(0, axis), channels = src.size[axis], + innerSize = src.total(axis + 1); + + CV_Assert(src.type() == CV_8S && (dst.type() == CV_8S || dst.type() == CV_32F)); + CV_Assert(src.isContinuous() && dst.isContinuous()); + + size_t outerStep = src.total(axis); + size_t cnStep = src.total(axis + 1); + const int8_t *srcPtr = src.ptr(); + const float *expPtr = blobs[0].ptr(); + + if (dst.type() == CV_32F) + { + float *dstPtr = dst.ptr(); + for (size_t outerDim = 0; outerDim < outerSize; outerDim++) + { + size_t srcOffset = outerDim * outerStep; + std::vector expSum(innerSize, 0.f); + + // sum exp along axis + for (size_t cnDim = 0; cnDim < channels; cnDim++) + { + const int offset = srcOffset + cnDim * cnStep; + for (size_t i = 0; i < innerSize; i++) + expSum[i] += expPtr[srcPtr[offset + i] + 128]; + } + + // divide by computed sum + for (size_t cnDim = 0; cnDim < channels; cnDim++) + { + const int offset = srcOffset + cnDim * cnStep; + for (size_t i = 0; i < innerSize; i++) + dstPtr[offset + i] = expPtr[srcPtr[offset + i] + 128]/expSum[i]; + } + + if (logSoftMax) + { + for (size_t cnDim = 0; cnDim < channels; cnDim++) + { + const int offset = srcOffset + cnDim * cnStep; + for (size_t i = 0; i < innerSize; i++) + dstPtr[offset + i] = log(dstPtr[offset + i]); + } + } + } + } + else + { + const float inv_scale = 1.f/output_sc; + int8_t *dstPtr = dst.ptr(); + for (size_t outerDim = 0; outerDim < outerSize; outerDim++) + { + size_t srcOffset = outerDim * outerStep; + std::vector expSum(innerSize, 0.f); + + // sum exp along axis + for (size_t cnDim = 0; cnDim < channels; cnDim++) + { + const int offset = srcOffset + cnDim * cnStep; + for (size_t i = 0; i < innerSize; i++) + expSum[i] += expPtr[srcPtr[offset + i] + 128]; + } + + // divide by computed sum and quantize to int8 + if (logSoftMax) + { + for (size_t cnDim = 0; cnDim < channels; cnDim++) + { + const int offset = srcOffset + cnDim * cnStep; + for (size_t i = 0; i < innerSize; i++) + dstPtr[offset + i] = saturate_cast(output_zp + std::round(inv_scale*log(expPtr[srcPtr[offset + i] + 128]/expSum[i]))); + } + } + else + { + for (size_t cnDim = 0; cnDim < channels; cnDim++) + { + const int offset = srcOffset + cnDim * cnStep; + for (size_t i = 0; i < innerSize; i++) + dstPtr[offset + i] = saturate_cast(output_zp + std::round(inv_scale*(expPtr[srcPtr[offset + i] + 128]/expSum[i]))); + } + } + } + } + } + + int64 getFLOPS(const std::vector &inputs, + const std::vector &outputs) const CV_OVERRIDE + { + CV_UNUSED(outputs); // suppress unused variable warning + int64 flops = 0; + + for (int i = 0; i < inputs.size(); i++) + { + flops += 4*total(inputs[i]); + } + + return flops; + } + + int axisRaw; +}; + +Ptr SoftmaxLayerInt8::create(const LayerParams& params) +{ + return Ptr(new SoftMaxLayerInt8Impl(params)); +} + +} +} diff --git a/modules/dnn/src/layers/batch_norm_layer.cpp b/modules/dnn/src/layers/batch_norm_layer.cpp index e28c964689..49804c5c13 100644 --- a/modules/dnn/src/layers/batch_norm_layer.cpp +++ b/modules/dnn/src/layers/batch_norm_layer.cpp @@ -409,6 +409,18 @@ public: } #endif // HAVE_DNN_NGRAPH + virtual bool tryQuantize(const std::vector > &scales, + const std::vector > &zeropoints, LayerParams& params) CV_OVERRIDE + { + params.set("input_scale", scales[0][0]); + params.set("input_zeropoint", zeropoints[0][0]); + + params.blobs.clear(); + params.blobs.push_back(origin_weights); + params.blobs.push_back(origin_bias); + return true; + } + virtual int64 getFLOPS(const std::vector &inputs, const std::vector &outputs) const CV_OVERRIDE { diff --git a/modules/dnn/src/layers/blank_layer.cpp b/modules/dnn/src/layers/blank_layer.cpp index 5f93b45886..59548a9c0c 100644 --- a/modules/dnn/src/layers/blank_layer.cpp +++ b/modules/dnn/src/layers/blank_layer.cpp @@ -166,6 +166,11 @@ public: } #endif + virtual bool tryQuantize(const std::vector > &scales, + const std::vector > &zeropoints, LayerParams& params) CV_OVERRIDE + { + return true; + } }; Ptr BlankLayer::create(const LayerParams& params) diff --git a/modules/dnn/src/layers/concat_layer.cpp b/modules/dnn/src/layers/concat_layer.cpp index a950c56167..536114fcd7 100644 --- a/modules/dnn/src/layers/concat_layer.cpp +++ b/modules/dnn/src/layers/concat_layer.cpp @@ -70,6 +70,7 @@ public: setParamsFrom(params); axis = params.get("axis", 1); padding = params.get("padding", false); + paddingValue = params.get("padding_value", 0); } virtual bool getMemoryShapes(const std::vector &inputs, @@ -119,13 +120,14 @@ public: (backendId == DNN_BACKEND_VKCOM && haveVulkan() && !padding); } + template class ChannelConcatInvoker : public ParallelLoopBody { public: std::vector* inputs; Mat* output; int nstripes; - std::vector chptrs; + std::vector chptrs; static void run(std::vector& inputs, Mat& output, int nstripes) { @@ -139,14 +141,14 @@ public: for( i = 0; i < ninputs; i++ ) { Mat& inp = inputs[i]; - CV_Assert( inp.isContinuous() && (inp.type() == CV_32F || inp.type() == CV_16S) && + CV_Assert( inp.isContinuous() && (inp.type() == CV_32F || inp.type() == CV_16S || inp.type() == CV_8S) && inp.dims == 4 && inp.size[0] == output.size[0] && inp.size[2] == output.size[2] && inp.size[3] == output.size[3] ); nchannels += inp.size[1]; } CV_Assert( nchannels == output.size[1] ); - CV_Assert( output.isContinuous() && (output.type() == CV_32F || output.type() == CV_16S) ); + CV_Assert( output.isContinuous() && (output.type() == CV_32F || output.type() == CV_16S || output.type() == CV_8S) ); cc.chptrs.resize(nchannels*batchsz); @@ -157,7 +159,7 @@ public: for( int j = 0; j < batchsz; j++ ) for( int k = 0; k < inp.size[1]; k++ ) { - const float* ptr = inp.ptr(j, k); + const T* ptr = inp.ptr(j, k); cc.chptrs[ofs + j*nchannels + k] = ptr; } ofs += inp.size[1]; @@ -176,8 +178,8 @@ public: size_t stripeSize = (total + nstripes - 1)/nstripes; size_t stripeStart = r.start*stripeSize; size_t stripeEnd = std::min(total, r.end*stripeSize); - const float** ptrs = (const float**)&chptrs[0]; - float* outptr = output->ptr(); + const T** ptrs = (const T**)&chptrs[0]; + T* outptr = output->ptr(); size_t blockSize0 = 1 << 16; for( size_t ofs0 = stripeStart; ofs0 < stripeEnd; ) @@ -248,7 +250,8 @@ public: CV_TRACE_FUNCTION(); CV_TRACE_ARG_VALUE(name, "name", name.c_str()); - CV_OCL_RUN(IS_DNN_OPENCL_TARGET(preferableTarget), + CV_OCL_RUN(IS_DNN_OPENCL_TARGET(preferableTarget) && + inputs_arr.depth() != CV_8S, forward_ocl(inputs_arr, outputs_arr, internals_arr)) std::vector inputs, outputs; @@ -259,12 +262,15 @@ public: Mat& outMat = outputs[0]; if (padding) - outMat.setTo(0); + outMat.setTo(paddingValue); if( cAxis == 1 && outMat.dims == 4 && !padding) { int nstripes = getNumThreads(); - ChannelConcatInvoker::run(inputs, outMat, nstripes); + if (outMat.type() == CV_8S) + ChannelConcatInvoker::run(inputs, outMat, nstripes); + else + ChannelConcatInvoker::run(inputs, outMat, nstripes); } else { @@ -394,6 +400,14 @@ public: return Ptr(new InfEngineNgraphNode(concat)); } #endif // HAVE_DNN_NGRAPH + + virtual bool tryQuantize(const std::vector > &scales, + const std::vector > &zeropoints, LayerParams& params) CV_OVERRIDE + { + if (padding) + params.set("padding_value", zeropoints[1][0]); + return true; + } }; Ptr ConcatLayer::create(const LayerParams& params) diff --git a/modules/dnn/src/layers/const_layer.cpp b/modules/dnn/src/layers/const_layer.cpp index bbea3e3f2c..18f190b36b 100644 --- a/modules/dnn/src/layers/const_layer.cpp +++ b/modules/dnn/src/layers/const_layer.cpp @@ -112,6 +112,15 @@ public: } #endif + virtual bool tryQuantize(const std::vector > &scales, + const std::vector > &zeropoints, LayerParams& params) CV_OVERRIDE + { + Mat quantizedBlob; + blobs[0].convertTo(quantizedBlob, CV_8S, 1.f/scales[1][0], zeropoints[1][0]); + params.blobs.clear(); + params.blobs.push_back(quantizedBlob); + return true; + } }; Ptr ConstLayer::create(const LayerParams& params) diff --git a/modules/dnn/src/layers/convolution_layer.cpp b/modules/dnn/src/layers/convolution_layer.cpp index 68c543be24..ec2904ee69 100644 --- a/modules/dnn/src/layers/convolution_layer.cpp +++ b/modules/dnn/src/layers/convolution_layer.cpp @@ -2083,6 +2083,48 @@ public: } #endif + virtual bool tryQuantize(const std::vector > &scales, + const std::vector > &zeropoints, LayerParams& params) CV_OVERRIDE + { + // References - https://arxiv.org/pdf/1712.05877.pdf + + // Quantized convolution with variable weights is not supported. + if (blobs.empty()) + return false; + + float inputScale = scales[0][0], outputScale = scales[1][0]; + int inputZp = zeropoints[0][0]; + params.set("input_zeropoint", inputZp); + + Mat weightsQuantized(weightsMat.rows, weightsMat.cols, CV_8S); + Mat biasQuantized(1, numOutput, CV_32S); + Mat outputMultiplier(1, numOutput, CV_32F); + double realMin, realMax, weightsScale; + + for( int i = 0; i < numOutput; i++ ) + { + // Quantize weights + cv::minMaxIdx(weightsMat.row(i), &realMin, &realMax); + realMin = std::min(realMin, 0.0); + realMax = std::max(realMax, 0.0); + weightsScale = (realMax == realMin) ? 1.0 : std::max(-realMin, realMax)/127; + weightsMat.row(i).convertTo(weightsQuantized.row(i), CV_8S, 1.f/weightsScale); + + // Quantize biases + float biasScale = inputScale * weightsScale; + biasQuantized.at(i) = (int)std::round(biasvec[i]/biasScale) - inputZp*(cv::sum(weightsQuantized.row(i))[0]); + + // Store multiplier + outputMultiplier.at(i) = biasScale / outputScale; + } + + params.blobs.clear(); + params.blobs.push_back(weightsQuantized.reshape(1, shape(blobs[0]))); + params.blobs.push_back(biasQuantized); + params.blobs.push_back(outputMultiplier); + return true; + } + virtual int64 getFLOPS(const std::vector &inputs, const std::vector &outputs) const CV_OVERRIDE { diff --git a/modules/dnn/src/layers/elementwise_layers.cpp b/modules/dnn/src/layers/elementwise_layers.cpp index 9bb5be342f..6dc1813c8b 100644 --- a/modules/dnn/src/layers/elementwise_layers.cpp +++ b/modules/dnn/src/layers/elementwise_layers.cpp @@ -255,6 +255,12 @@ public: } #endif + virtual bool tryQuantize(const std::vector > &scales, + const std::vector > &zeropoints, LayerParams& params) CV_OVERRIDE + { + return func.tryQuantize(scales, zeropoints, params); + } + virtual int64 getFLOPS(const std::vector &inputs, const std::vector &outputs) const CV_OVERRIDE { @@ -288,6 +294,8 @@ struct BaseFunctor bool tryFuse(Ptr&) { return false; } void getScaleShift(Mat&, Mat&) const {} + + bool tryQuantize(const std::vector>&, const std::vector>&, LayerParams&) { return false; } }; struct ReLUFunctor : public BaseFunctor @@ -436,6 +444,29 @@ struct ReLUFunctor : public BaseFunctor } #endif // HAVE_VULKAN + bool tryQuantize(const std::vector > &scales, + const std::vector > &zeropoints, LayerParams& params) + { + if (slope != 0.f) + { + float inpScale = scales[0][0], outScale = scales[1][0]; + int inpZp = zeropoints[0][0], outZp = zeropoints[1][0]; + + Mat lookUpTable(1, 256, CV_8S); + int8_t* table = lookUpTable.ptr(); + for (int i = -128; i < 128; i++) + { + float x = inpScale*(i - inpZp); + float y = x >= 0.f ? x : slope*x; + int quantized = outZp + (int)std::round(y/outScale); + table[i+128] = saturate_cast(quantized); + } + params.blobs.clear(); + params.blobs.push_back(lookUpTable); + } + return true; + } + int64 getFLOPSPerElement() const { return 1; } }; @@ -559,6 +590,12 @@ struct ReLU6Functor : public BaseFunctor } #endif // HAVE_VULKAN + bool tryQuantize(const std::vector > &scales, + const std::vector > &zeropoints, LayerParams& params) + { + return true; + } + int64 getFLOPSPerElement() const { return 2; } }; @@ -651,6 +688,26 @@ struct TanHFunctor : public BaseFunctor } #endif // HAVE_VULKAN + bool tryQuantize(const std::vector > &scales, + const std::vector > &zeropoints, LayerParams& params) + { + float inpScale = scales[0][0], outScale = scales[1][0]; + int inpZp = zeropoints[0][0], outZp = zeropoints[1][0]; + + Mat lookUpTable(1, 256, CV_8S); + int8_t* table = lookUpTable.ptr(); + for (int i = -128; i < 128; i++) + { + float x = inpScale*(i - inpZp); + float y = tanh(x); + int quantized = outZp + (int)std::round(y/outScale); + table[i+128] = saturate_cast(quantized); + } + params.blobs.clear(); + params.blobs.push_back(lookUpTable); + return true; + } + int64 getFLOPSPerElement() const { return 1; } }; @@ -743,6 +800,26 @@ struct SwishFunctor : public BaseFunctor } #endif // HAVE_VULKAN + bool tryQuantize(const std::vector > &scales, + const std::vector > &zeropoints, LayerParams& params) + { + float inpScale = scales[0][0], outScale = scales[1][0]; + int inpZp = zeropoints[0][0], outZp = zeropoints[1][0]; + + Mat lookUpTable(1, 256, CV_8S); + int8_t* table = lookUpTable.ptr(); + for (int i = -128; i < 128; i++) + { + float x = inpScale*(i - inpZp); + float y = x / (1.0f + exp(-x)); + int quantized = outZp + (int)std::round(y/outScale); + table[i+128] = saturate_cast(quantized); + } + params.blobs.clear(); + params.blobs.push_back(lookUpTable); + return true; + } + int64 getFLOPSPerElement() const { return 3; } }; @@ -848,6 +925,28 @@ struct MishFunctor : public BaseFunctor } #endif // HAVE_VULKAN + bool tryQuantize(const std::vector > &scales, + const std::vector > &zeropoints, LayerParams& params) + { + float inpScale = scales[0][0], outScale = scales[1][0]; + int inpZp = zeropoints[0][0], outZp = zeropoints[1][0]; + + Mat lookUpTable(1, 256, CV_8S); + int8_t* table = lookUpTable.ptr(); + for (int i = -128; i < 128; i++) + { + float x = inpScale*(i - inpZp); + float eX = exp(x); + float n = (eX + 2) * eX; + float y = (x * n) / (n + 2); + int quantized = outZp + (int)std::round(y/outScale); + table[i+128] = saturate_cast(quantized); + } + params.blobs.clear(); + params.blobs.push_back(lookUpTable); + return true; + } + int64 getFLOPSPerElement() const { return 3; } }; @@ -940,6 +1039,26 @@ struct SigmoidFunctor : public BaseFunctor } #endif // HAVE_VULKAN + bool tryQuantize(const std::vector > &scales, + const std::vector > &zeropoints, LayerParams& params) + { + float inpScale = scales[0][0], outScale = scales[1][0]; + int inpZp = zeropoints[0][0], outZp = zeropoints[1][0]; + + Mat lookUpTable(1, 256, CV_8S); + int8_t* table = lookUpTable.ptr(); + for (int i = -128; i < 128; i++) + { + float x = inpScale*(i - inpZp); + float y = 1.f/(1.f + exp(-x)); + int quantized = outZp + (int)std::round(y/outScale); + table[i+128] = saturate_cast(quantized); + } + params.blobs.clear(); + params.blobs.push_back(lookUpTable); + return true; + } + int64 getFLOPSPerElement() const { return 3; } }; @@ -1032,6 +1151,26 @@ struct ELUFunctor : public BaseFunctor } #endif // HAVE_VULKAN + bool tryQuantize(const std::vector > &scales, + const std::vector > &zeropoints, LayerParams& params) + { + float inpScale = scales[0][0], outScale = scales[1][0]; + int inpZp = zeropoints[0][0], outZp = zeropoints[1][0]; + + Mat lookUpTable(1, 256, CV_8S); + int8_t* table = lookUpTable.ptr(); + for (int i = -128; i < 128; i++) + { + float x = inpScale*(i - inpZp); + float y = x >= 0.f ? x : exp(x) - 1; + int quantized = outZp + (int)std::round(y/outScale); + table[i+128] = saturate_cast(quantized); + } + params.blobs.clear(); + params.blobs.push_back(lookUpTable); + return true; + } + int64 getFLOPSPerElement() const { return 2; } }; @@ -1130,6 +1269,26 @@ struct AbsValFunctor : public BaseFunctor } #endif // HAVE_VULKAN + bool tryQuantize(const std::vector > &scales, + const std::vector > &zeropoints, LayerParams& params) + { + float inpScale = scales[0][0], outScale = scales[1][0]; + int inpZp = zeropoints[0][0], outZp = zeropoints[1][0]; + + Mat lookUpTable(1, 256, CV_8S); + int8_t* table = lookUpTable.ptr(); + for (int i = -128; i < 128; i++) + { + float x = inpScale*(i - inpZp); + float y = abs(x); + int quantized = outZp + (int)std::round(y/outScale); + table[i+128] = saturate_cast(quantized); + } + params.blobs.clear(); + params.blobs.push_back(lookUpTable); + return true; + } + int64 getFLOPSPerElement() const { return 1; } }; @@ -1223,6 +1382,26 @@ struct BNLLFunctor : public BaseFunctor } #endif // HAVE_VULKAN + bool tryQuantize(const std::vector > &scales, + const std::vector > &zeropoints, LayerParams& params) + { + float inpScale = scales[0][0], outScale = scales[1][0]; + int inpZp = zeropoints[0][0], outZp = zeropoints[1][0]; + + Mat lookUpTable(1, 256, CV_8S); + int8_t* table = lookUpTable.ptr(); + for (int i = -128; i < 128; i++) + { + float x = inpScale*(i - inpZp); + float y = x > 0 ? x + log(1. + exp(-x)) : log(1. + exp(x)); + int quantized = outZp + (int)std::round(y/outScale); + table[i+128] = saturate_cast(quantized); + } + params.blobs.clear(); + params.blobs.push_back(lookUpTable); + return true; + } + int64 getFLOPSPerElement() const { return 5; } }; diff --git a/modules/dnn/src/layers/eltwise_layer.cpp b/modules/dnn/src/layers/eltwise_layer.cpp index a337c48d9e..860560213d 100644 --- a/modules/dnn/src/layers/eltwise_layer.cpp +++ b/modules/dnn/src/layers/eltwise_layer.cpp @@ -864,6 +864,37 @@ public: } #endif // HAVE_DNN_NGRAPH + virtual bool tryQuantize(const std::vector > &scales, + const std::vector > &zeropoints, LayerParams& params) CV_OVERRIDE + { + if (op == SUM) + { + std::vector newCoeffs; + float offset = zeropoints[1][0]; + float out_sc = scales[1][0]; + for (int i = 0; i < scales[0].size(); i++) + { + float coeff = coeffs.empty() ? 1.f : coeffs[i]; + float newcoeff = (scales[0][i] * coeff) / out_sc; + newCoeffs.push_back(newcoeff); + offset -= (newcoeff * zeropoints[0][i]); + } + params.set("coeff", DictValue::arrayReal(newCoeffs.data(), newCoeffs.size())); + params.set("offset", offset); + return true; + } + else if (op == PROD) + { + std::vector newCoeffs = scales[0]; + newCoeffs[0] /= scales[1][0]; + params.set("coeff", DictValue::arrayReal(newCoeffs.data(), newCoeffs.size())); + params.set("offset", zeropoints[1][0]); + params.set("input_zeropoints", DictValue::arrayInt(zeropoints[0].data(), zeropoints[0].size())); + return true; + } + return op == MAX; + } + virtual int64 getFLOPS(const std::vector &inputs, const std::vector &outputs) const CV_OVERRIDE { diff --git a/modules/dnn/src/layers/flatten_layer.cpp b/modules/dnn/src/layers/flatten_layer.cpp index 7cf01a14fa..8ff862fab0 100644 --- a/modules/dnn/src/layers/flatten_layer.cpp +++ b/modules/dnn/src/layers/flatten_layer.cpp @@ -227,6 +227,11 @@ virtual Ptr initNgraph(const std::vector >& inp } #endif + virtual bool tryQuantize(const std::vector > &scales, + const std::vector > &zeropoints, LayerParams& params) CV_OVERRIDE + { + return true; + } int _startAxis; int _endAxis; diff --git a/modules/dnn/src/layers/fully_connected_layer.cpp b/modules/dnn/src/layers/fully_connected_layer.cpp index 529f3c04fd..28ea7f347f 100644 --- a/modules/dnn/src/layers/fully_connected_layer.cpp +++ b/modules/dnn/src/layers/fully_connected_layer.cpp @@ -618,6 +618,45 @@ public: } #endif // HAVE_DNN_NGRAPH + virtual bool tryQuantize(const std::vector > &scales, + const std::vector > &zeropoints, LayerParams& params) CV_OVERRIDE + { + if (blobs.empty()) + return false; + + int numOutput = blobs[0].size[0]; + float inputScale = scales[0][0], outputScale = scales[1][0]; + int inputZp = zeropoints[0][0]; + + Mat weightsQuantized(weightsMat.rows, weightsMat.cols, CV_8S); + Mat biasQuantized(1, numOutput, CV_32S); + Mat outputMultiplier(1, numOutput, CV_32F); + + double realMin, realMax, weightsScale; + for( int i = 0; i < numOutput; i++ ) + { + // Quantize weights + cv::minMaxIdx(weightsMat.row(i), &realMin, &realMax); + realMin = std::min(realMin, 0.0); + realMax = std::max(realMax, 0.0); + weightsScale = (realMax == realMin) ? 1.0 : std::max(-realMin, realMax)/127; + weightsMat.row(i).convertTo(weightsQuantized.row(i), CV_8S, 1.f/weightsScale); + + // Quantize biases + float biasScale = inputScale * weightsScale; + biasQuantized.at(i) = (int)std::round(biasMat.at(i)/biasScale) - inputZp*(cv::sum(weightsQuantized.row(i))[0]); + + // Store multiplier + outputMultiplier.at(i) = biasScale / outputScale; + } + + params.blobs.clear(); + params.blobs.push_back(weightsQuantized.reshape(1, shape(blobs[0]))); + params.blobs.push_back(biasQuantized); + params.blobs.push_back(outputMultiplier); + return true; + } + virtual int64 getFLOPS(const std::vector &inputs, const std::vector &outputs) const CV_OVERRIDE { diff --git a/modules/dnn/src/layers/padding_layer.cpp b/modules/dnn/src/layers/padding_layer.cpp index d182568795..c1979ce701 100644 --- a/modules/dnn/src/layers/padding_layer.cpp +++ b/modules/dnn/src/layers/padding_layer.cpp @@ -134,6 +134,8 @@ public: cv::convertFp16(paddingValue_fp32, paddingValue_fp16); outputs[0].setTo(paddingValue_fp16[0]); } + else if (inputs_arr.depth() == CV_8S) + outputs[0].setTo(saturate_cast(paddingValue)); else outputs[0].setTo(paddingValue); inputs[0].copyTo(outputs[0](dstRanges)); @@ -264,6 +266,16 @@ public: } #endif + virtual bool tryQuantize(const std::vector > &scales, + const std::vector > &zeropoints, LayerParams& params) CV_OVERRIDE + { + float outputScale = scales[1][0]; + int outputZp = zeropoints[1][0]; + float padValue = outputZp + std::round(params.get("value", 0)/outputScale); + params.set("value", padValue); + return true; + } + private: std::vector > paddings; // Pairs pad before, pad after. std::vector dstRanges; diff --git a/modules/dnn/src/layers/permute_layer.cpp b/modules/dnn/src/layers/permute_layer.cpp index c525c3f82f..77c2469c05 100644 --- a/modules/dnn/src/layers/permute_layer.cpp +++ b/modules/dnn/src/layers/permute_layer.cpp @@ -194,6 +194,7 @@ public: #endif } + template class PermuteInvoker : public ParallelLoopBody { public: @@ -229,7 +230,7 @@ public: size_t stripeStart = r.start*stripeSize; size_t stripeEnd = std::min(r.end*stripeSize, orows); - const size_t esz = sizeof(float); + const size_t esz = sizeof(T); size_t ostep0 = out->step[0]/esz, ostep1 = out->step[1]/esz, ostep2 = out->step[2]/esz; const size_t* ord = &order->at(0); size_t istep0 = inp->step[ord[0]]/esz, istep1 = inp->step[ord[1]]/esz, @@ -241,13 +242,13 @@ public: int i1 = (int)(val % n1); int i0 = (int)(val / n1); - const float* inptr_orig = inp->ptr(); - float* outptr_orig = out->ptr(); + const T* inptr_orig = inp->ptr(); + T* outptr_orig = out->ptr(); for( size_t ofs = stripeStart; ofs < stripeEnd; ofs++ ) { - const float* inptr = inptr_orig + i0*istep0 + i1*istep1 + i2*istep2; - float* outptr = outptr_orig + i0*ostep0 + i1*ostep1 + i2*ostep2; + const T* inptr = inptr_orig + i0*istep0 + i1*istep1 + i2*istep2; + T* outptr = outptr_orig + i0*ostep0 + i1*ostep1 + i2*ostep2; for( int i3 = 0; i3 < n3; i3++ ) outptr[i3] = inptr[i3*istep3]; @@ -321,7 +322,8 @@ public: CV_TRACE_FUNCTION(); CV_TRACE_ARG_VALUE(name, "name", name.c_str()); - CV_OCL_RUN(IS_DNN_OPENCL_TARGET(preferableTarget), + CV_OCL_RUN(IS_DNN_OPENCL_TARGET(preferableTarget) && + inputs_arr.depth() != CV_8S, forward_ocl(inputs_arr, outputs_arr, internals_arr)) if (inputs_arr.depth() == CV_16S) @@ -365,24 +367,48 @@ public: if( numAxes == 4 ) { int nstripes = getNumThreads(); - PermuteInvoker::run(inp, out, _order, nstripes); + if (inp.type() == CV_8S) + PermuteInvoker::run(inp, out, _order, nstripes); + else + PermuteInvoker::run(inp, out, _order, nstripes); } else { - const float *srcData = inp.ptr(); - float *dstData = out.ptr(); - - for (i = 0; i < count; ++i) + if (inp.type() == CV_8S) { - size_t oldPosition = 0; - size_t newPosition = i; + const int8_t *srcData = inp.ptr(); + int8_t *dstData = out.ptr(); - for (j = 0; j < numAxes; ++j) + for (i = 0; i < count; ++i) { - oldPosition += (newPosition / newStride[j]) * oldStride[order[j]]; - newPosition %= newStride[j]; + size_t oldPosition = 0; + size_t newPosition = i; + + for (j = 0; j < numAxes; ++j) + { + oldPosition += (newPosition / newStride[j]) * oldStride[order[j]]; + newPosition %= newStride[j]; + } + dstData[i] = srcData[oldPosition]; + } + } + else + { + const float *srcData = inp.ptr(); + float *dstData = out.ptr(); + + for (i = 0; i < count; ++i) + { + size_t oldPosition = 0; + size_t newPosition = i; + + for (j = 0; j < numAxes; ++j) + { + oldPosition += (newPosition / newStride[j]) * oldStride[order[j]]; + newPosition %= newStride[j]; + } + dstData[i] = srcData[oldPosition]; } - dstData[i] = srcData[oldPosition]; } } } @@ -436,6 +462,11 @@ public: } #endif // HAVE_VULKAN + virtual bool tryQuantize(const std::vector > &scales, + const std::vector > &zeropoints, LayerParams& params) CV_OVERRIDE + { + return true; + } size_t _count; std::vector _order; diff --git a/modules/dnn/src/layers/pooling_layer.cpp b/modules/dnn/src/layers/pooling_layer.cpp index b8e2cfdf8f..7653e53668 100644 --- a/modules/dnn/src/layers/pooling_layer.cpp +++ b/modules/dnn/src/layers/pooling_layer.cpp @@ -1327,6 +1327,23 @@ public: return true; } + virtual bool tryQuantize(const std::vector > &scales, + const std::vector > &zeropoints, LayerParams& params) CV_OVERRIDE + { + if (type == MAX && !computeMaxIdx) + { + return true; + } + else if (type == AVE || type == SUM) + { + float multiplier = scales[0][0] / scales[1][0]; + params.set("multiplier", multiplier); + params.set("input_zeropoint", zeropoints[0][0]); + return true; + } + return false; + } + virtual int64 getFLOPS(const std::vector &inputs, const std::vector &outputs) const CV_OVERRIDE { diff --git a/modules/dnn/src/layers/reorg_layer.cpp b/modules/dnn/src/layers/reorg_layer.cpp index da1c61adac..797df4819d 100644 --- a/modules/dnn/src/layers/reorg_layer.cpp +++ b/modules/dnn/src/layers/reorg_layer.cpp @@ -231,6 +231,11 @@ public: } #endif + virtual bool tryQuantize(const std::vector > &scales, + const std::vector > &zeropoints, LayerParams& params) CV_OVERRIDE + { + return true; + } virtual int64 getFLOPS(const std::vector &inputs, const std::vector &outputs) const CV_OVERRIDE diff --git a/modules/dnn/src/layers/reshape_layer.cpp b/modules/dnn/src/layers/reshape_layer.cpp index ab8f41c7b6..4c10d155c8 100644 --- a/modules/dnn/src/layers/reshape_layer.cpp +++ b/modules/dnn/src/layers/reshape_layer.cpp @@ -343,6 +343,11 @@ public: } #endif + virtual bool tryQuantize(const std::vector > &scales, + const std::vector > &zeropoints, LayerParams& params) CV_OVERRIDE + { + return true; + } private: std::vector outShapes; diff --git a/modules/dnn/src/layers/scale_layer.cpp b/modules/dnn/src/layers/scale_layer.cpp index a5c268214e..001db24a2d 100644 --- a/modules/dnn/src/layers/scale_layer.cpp +++ b/modules/dnn/src/layers/scale_layer.cpp @@ -344,6 +344,14 @@ public: shift = (hasBias && !blobs.empty()) ? blobs.back() : Mat(); } + virtual bool tryQuantize(const std::vector > &scales, + const std::vector > &zeropoints, LayerParams& params) CV_OVERRIDE + { + params.set("input_scales", DictValue::arrayReal(scales[0].data(), scales[0].size())); + params.set("input_zeropoints", DictValue::arrayInt(zeropoints[0].data(), zeropoints[0].size())); + return true; + } + virtual int64 getFLOPS(const std::vector &inputs, const std::vector &outputs) const CV_OVERRIDE { diff --git a/modules/dnn/src/layers/shuffle_channel_layer.cpp b/modules/dnn/src/layers/shuffle_channel_layer.cpp index 6db74d1abd..2a698d270f 100644 --- a/modules/dnn/src/layers/shuffle_channel_layer.cpp +++ b/modules/dnn/src/layers/shuffle_channel_layer.cpp @@ -147,6 +147,12 @@ public: } #endif + virtual bool tryQuantize(const std::vector > &scales, + const std::vector > &zeropoints, LayerParams& params) CV_OVERRIDE + { + return true; + } + private: Ptr permute; std::vector permuteInpShape, permuteOutShape; diff --git a/modules/dnn/src/layers/slice_layer.cpp b/modules/dnn/src/layers/slice_layer.cpp index 54e2340387..9efd95cf48 100644 --- a/modules/dnn/src/layers/slice_layer.cpp +++ b/modules/dnn/src/layers/slice_layer.cpp @@ -531,7 +531,12 @@ public: { std::vector inpIdx(dimsNum, 0); std::vector outIdx(dimsNum, 0); - getSliceRecursive(inpMat, inpIdx, finalSliceRanges[i], sliceSteps[i], 0, dimsNum, outputs[i], outIdx); + if (inpMat.type() == CV_16S) + getSliceRecursive(inpMat, inpIdx, finalSliceRanges[i], sliceSteps[i], 0, dimsNum, outputs[i], outIdx); + else if (inpMat.type() == CV_8S) + getSliceRecursive(inpMat, inpIdx, finalSliceRanges[i], sliceSteps[i], 0, dimsNum, outputs[i], outIdx); + else + getSliceRecursive(inpMat, inpIdx, finalSliceRanges[i], sliceSteps[i], 0, dimsNum, outputs[i], outIdx); } } } @@ -647,8 +652,20 @@ public: } #endif + virtual bool tryQuantize(const std::vector > &scales, + const std::vector > &zeropoints, LayerParams& params) CV_OVERRIDE + { + const int numOutputs = scales[1].size(); + for (int i = 0; i < numOutputs; i++) + { + if (scales[1][i] != scales[0][0]) + return false; + } + return true; + } private: + template void getSliceRecursive(const Mat &inpMat, std::vector &inpIdx, const std::vector &sliceRanges, const std::vector &sliceSteps, int dim, int dimsNum, @@ -658,8 +675,6 @@ private: int end = sliceRanges[dim].end; int step = !sliceSteps.empty() ? sliceSteps[dim] : 1; - const bool is32F = inpMat.depth() == CV_32F; - // TODO optimization is required (for 2D tail case at least) for (int k = begin, j = 0; k < end; k += step, j++) { @@ -667,14 +682,9 @@ private: outIdx[dim] = j; if (dim + 1 < dimsNum) - getSliceRecursive(inpMat, inpIdx, sliceRanges, sliceSteps, dim + 1, dimsNum, outputs, outIdx); + getSliceRecursive(inpMat, inpIdx, sliceRanges, sliceSteps, dim + 1, dimsNum, outputs, outIdx); else - { - if (is32F) - outputs.at(outIdx.data()) = inpMat.at(inpIdx.data()); - else - outputs.at(outIdx.data()) = inpMat.at(inpIdx.data()); // 16F emulation - } + outputs.at(outIdx.data()) = inpMat.at(inpIdx.data()); } } diff --git a/modules/dnn/src/layers/softmax_layer.cpp b/modules/dnn/src/layers/softmax_layer.cpp index 546c1017ad..e937e98f8c 100644 --- a/modules/dnn/src/layers/softmax_layer.cpp +++ b/modules/dnn/src/layers/softmax_layer.cpp @@ -374,6 +374,22 @@ public: } #endif // HAVE_DNN_NGRAPH + virtual bool tryQuantize(const std::vector > &scales, + const std::vector > &zeropoints, LayerParams& params) CV_OVERRIDE + { + float inpScale = scales[0][0]; + Mat lookUpTable(1, 256, CV_32F); + float* table = lookUpTable.ptr(); + for (int i = -128; i < 128; i++) + { + float x = inpScale*(i - 127); // ensures exp(x) is always between (0, 1) + table[i+128] = std::exp(x); + } + params.blobs.clear(); + params.blobs.push_back(lookUpTable); + return true; + } + int64 getFLOPS(const std::vector &inputs, const std::vector &outputs) const CV_OVERRIDE { diff --git a/modules/dnn/src/layers/split_layer.cpp b/modules/dnn/src/layers/split_layer.cpp index b025d5ff1e..2a44176152 100644 --- a/modules/dnn/src/layers/split_layer.cpp +++ b/modules/dnn/src/layers/split_layer.cpp @@ -117,6 +117,17 @@ public: } #endif + virtual bool tryQuantize(const std::vector > &scales, + const std::vector > &zeropoints, LayerParams& params) CV_OVERRIDE + { + const int numOutputs = scales[1].size(); + for (int i = 0; i < numOutputs; i++) + { + if (scales[1][i] != scales[0][0]) + return false; + } + return true; + } }; Ptr SplitLayer::create(const LayerParams& params) diff --git a/modules/dnn/test/test_int8_layers.cpp b/modules/dnn/test/test_int8_layers.cpp new file mode 100644 index 0000000000..1fcb1d0dba --- /dev/null +++ b/modules/dnn/test/test_int8_layers.cpp @@ -0,0 +1,1220 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. + +#include "test_precomp.hpp" +#include "npy_blob.hpp" +#include +#include +namespace opencv_test { namespace { + +template +static std::string _tf(TString filename) +{ + return (getOpenCVExtraDir() + "dnn/") + filename; +} + +class Test_Int8_layers : public DNNTestLayer +{ +public: + void testLayer(const String& basename, const String& importer, double l1, double lInf, + int numInps = 1, int numOuts = 1, bool useCaffeModel = false, + bool useCommonInputBlob = true, bool hasText = false) + { + CV_Assert_N(numInps >= 1, numInps <= 10, numOuts >= 1, numOuts <= 10); + std::vector inps(numInps), inps_int8(numInps); + std::vector refs(numOuts), outs_int8(numOuts), outs_dequantized(numOuts); + std::vector inputScale, outputScale; + std::vector inputZp, outputZp; + String inpPath, outPath; + Net net, qnet; + + if (importer == "Caffe") + { + String prototxt = _tf("layers/" + basename + ".prototxt"); + String caffemodel = _tf("layers/" + basename + ".caffemodel"); + net = readNetFromCaffe(prototxt, useCaffeModel ? caffemodel : String()); + + inpPath = _tf("layers/" + (useCommonInputBlob ? "blob" : basename + ".input")); + outPath = _tf("layers/" + basename); + } + else if (importer == "TensorFlow") + { + String netPath = _tf("tensorflow/" + basename + "_net.pb"); + String netConfig = hasText ? _tf("tensorflow/" + basename + "_net.pbtxt") : ""; + net = readNetFromTensorflow(netPath, netConfig); + + inpPath = _tf("tensorflow/" + basename + "_in"); + outPath = _tf("tensorflow/" + basename + "_out"); + } + else if (importer == "ONNX") + { + String onnxmodel = _tf("onnx/models/" + basename + ".onnx"); + net = readNetFromONNX(onnxmodel); + + inpPath = _tf("onnx/data/input_" + basename); + outPath = _tf("onnx/data/output_" + basename); + } + ASSERT_FALSE(net.empty()); + net.setPreferableBackend(backend); + net.setPreferableTarget(target); + + for (int i = 0; i < numInps; i++) + inps[i] = blobFromNPY(inpPath + ((numInps > 1) ? cv::format("_%d.npy", i) : ".npy")); + + for (int i = 0; i < numOuts; i++) + refs[i] = blobFromNPY(outPath + ((numOuts > 1) ? cv::format("_%d.npy", i) : ".npy")); + + qnet = net.quantize(inps, CV_8S, CV_8S); + qnet.getInputDetails(inputScale, inputZp); + qnet.getOutputDetails(outputScale, outputZp); + + // Quantize inputs to int8 + // int8_value = float_value/scale + zero-point + for (int i = 0; i < numInps; i++) + { + inps[i].convertTo(inps_int8[i], CV_8S, 1.f/inputScale[i], inputZp[i]); + String inp_name = numInps > 1 ? (importer == "Caffe" ? cv::format("input_%d", i) : cv::format("%d", i)) : ""; + qnet.setInput(inps_int8[i], inp_name); + } + qnet.forward(outs_int8); + + // Dequantize outputs and compare with reference outputs + // float_value = scale*(int8_value - zero-point) + for (int i = 0; i < numOuts; i++) + { + outs_int8[i].convertTo(outs_dequantized[i], CV_32F, outputScale[i], -(outputScale[i] * outputZp[i])); + normAssert(refs[i], outs_dequantized[i], "", l1, lInf); + } + } +}; + +TEST_P(Test_Int8_layers, Convolution1D) +{ + testLayer("conv1d", "ONNX", 0.00302, 0.00909); + testLayer("conv1d_bias", "ONNX", 0.00306, 0.00948); +} + +TEST_P(Test_Int8_layers, Convolution2D) +{ + testLayer("layer_convolution", "Caffe", 0.0174, 0.0758, 1, 1, true); + testLayer("single_conv", "TensorFlow", 0.00413, 0.02201); + testLayer("depthwise_conv2d", "TensorFlow", 0.0388, 0.169); + testLayer("atrous_conv2d_valid", "TensorFlow", 0.0193, 0.0633); + testLayer("atrous_conv2d_same", "TensorFlow", 0.0185, 0.1322); + testLayer("keras_atrous_conv2d_same", "TensorFlow", 0.0056, 0.0244); + testLayer("convolution", "ONNX", 0.0052, 0.01516); + testLayer("two_convolution", "ONNX", 0.00295, 0.00840); +} + +TEST_P(Test_Int8_layers, Convolution3D) +{ + testLayer("conv3d", "TensorFlow", 0.00734, 0.02434); + testLayer("conv3d", "ONNX", 0.00353, 0.00941); + testLayer("conv3d_bias", "ONNX", 0.00129, 0.00249); +} + +TEST_P(Test_Int8_layers, Flatten) +{ + testLayer("flatten", "TensorFlow", 0.0036, 0.0069, 1, 1, false, true, true); + testLayer("unfused_flatten", "TensorFlow", 0.0014, 0.0028); + testLayer("unfused_flatten_unknown_batch", "TensorFlow", 0.0043, 0.0051); +} + +TEST_P(Test_Int8_layers, Padding) +{ + testLayer("padding_valid", "TensorFlow", 0.0026, 0.0064); + testLayer("padding_same", "TensorFlow", 0.0081, 0.032); + testLayer("spatial_padding", "TensorFlow", 0.0078, 0.028); + testLayer("mirror_pad", "TensorFlow", 0.0064, 0.013); + testLayer("pad_and_concat", "TensorFlow", 0.0021, 0.0098); + testLayer("padding", "ONNX", 0.0005, 0.0069); + testLayer("ReflectionPad2d", "ONNX", 0.00062, 0.0018); + testLayer("ZeroPad2d", "ONNX", 0.00037, 0.0018); +} + +TEST_P(Test_Int8_layers, AvePooling) +{ + testLayer("layer_pooling_ave", "Caffe", 0.0021, 0.0075); + testLayer("ave_pool_same", "TensorFlow", 0.00153, 0.0041); + testLayer("average_pooling_1d", "ONNX", 0.002, 0.0048); + testLayer("average_pooling", "ONNX", 0.0014, 0.0032); + testLayer("average_pooling_dynamic_axes", "ONNX", 0.0014, 0.006); + + if (target != DNN_TARGET_CPU) + throw SkipTestException("Only CPU is supported"); + testLayer("ave_pool3d", "TensorFlow", 0.00175, 0.0047); + testLayer("ave_pool3d", "ONNX", 0.00063, 0.0016); +} + +TEST_P(Test_Int8_layers, MaxPooling) +{ + testLayer("pool_conv_1d", "ONNX", 0.0006, 0.0015); + if (target != DNN_TARGET_CPU) + throw SkipTestException("Only CPU is supported"); + testLayer("pool_conv_3d", "ONNX", 0.0033, 0.0124); + + /* All the below tests have MaxPooling as last layer, so computeMaxIdx is set to true + which is not supported by int8 maxpooling + testLayer("layer_pooling_max", "Caffe", 0.0021, 0.004); + testLayer("max_pool_even", "TensorFlow", 0.0048, 0.0139); + testLayer("max_pool_odd_valid", "TensorFlow", 0.0043, 0.012); + testLayer("conv_pool_nchw", "TensorFlow", 0.007, 0.025); + testLayer("max_pool3d", "TensorFlow", 0.0025, 0.0058); + testLayer("maxpooling_1d", "ONNX", 0.0018, 0.0037); + testLayer("two_maxpooling_1d", "ONNX", 0.0037, 0.0052); + testLayer("maxpooling", "ONNX", 0.0034, 0.0065); + testLayer("two_maxpooling", "ONNX", 0.0025, 0.0052); + testLayer("max_pool3d", "ONNX", 0.0028, 0.0069);*/ +} + +TEST_P(Test_Int8_layers, Reduce) +{ + testLayer("reduce_mean", "TensorFlow", 0.0005, 0.0014); + testLayer("reduce_mean", "ONNX", 0.00062, 0.0014); + testLayer("reduce_mean_axis1", "ONNX", 0.00032, 0.0007); + testLayer("reduce_mean_axis2", "ONNX", 0.00033, 0.001); + + testLayer("reduce_sum", "TensorFlow", 0.015, 0.031); + testLayer("reduce_sum_channel", "TensorFlow", 0.008, 0.019); + testLayer("sum_pool_by_axis", "TensorFlow", 0.012, 0.032); + testLayer("reduce_sum", "ONNX", 0.0025, 0.0048); + + testLayer("reduce_max", "ONNX", 0, 0); + testLayer("reduce_max_axis_0", "ONNX", 0.0042, 0.007); + testLayer("reduce_max_axis_1", "ONNX", 0.0018, 0.0036); + + if (target != DNN_TARGET_CPU) + throw SkipTestException("Only CPU is supported"); + testLayer("reduce_mean3d", "ONNX", 0.00048, 0.0016); +} + +TEST_P(Test_Int8_layers, ReLU) +{ + testLayer("layer_relu", "Caffe", 0.0005, 0.002); + testLayer("ReLU", "ONNX", 0.0012, 0.0047); +} + +TEST_P(Test_Int8_layers, LeakyReLU) +{ + testLayer("leaky_relu", "TensorFlow", 0.0002, 0.0004); +} + +TEST_P(Test_Int8_layers, ReLU6) +{ + testLayer("keras_relu6", "TensorFlow", 0.0018, 0.0062); + testLayer("keras_relu6", "TensorFlow", 0.0018, 0.0062, 1, 1, false, true, true); + testLayer("clip_by_value", "TensorFlow", 0.0009, 0.002); + testLayer("clip", "ONNX", 0.00006, 0.00037); +} + +TEST_P(Test_Int8_layers, Sigmoid) +{ + testLayer("maxpooling_sigmoid", "ONNX", 0.0011, 0.0032); + testLayer("maxpooling_sigmoid_dynamic_axes", "ONNX", 0.0011, 0.0032); + testLayer("maxpooling_sigmoid_1d", "ONNX", 0.0011, 0.0037); +} + +TEST_P(Test_Int8_layers, Mish) +{ + testLayer("mish", "ONNX", 0.0015, 0.0025); +} + +TEST_P(Test_Int8_layers, Softmax) +{ + testLayer("layer_softmax", "Caffe", 0.0011, 0.0036); + testLayer("keras_softmax", "TensorFlow", 0.00093, 0.0027); + testLayer("slim_softmax", "TensorFlow", 0.0016, 0.0034); + testLayer("slim_softmax_v2", "TensorFlow", 0.0029, 0.017); + testLayer("softmax", "ONNX", 0.0016, 0.0028); + testLayer("log_softmax", "ONNX", 0.014, 0.025); + testLayer("softmax_unfused", "ONNX", 0.0009, 0.0021); +} + +TEST_P(Test_Int8_layers, Concat) +{ + testLayer("layer_concat_shared_input", "Caffe", 0.0076, 0.029, 1, 1, true, false); + testLayer("concat_axis_1", "TensorFlow", 0.0056, 0.017); + testLayer("keras_pad_concat", "TensorFlow", 0.0032, 0.0089); + testLayer("concat_3d", "TensorFlow", 0.005, 0.014); + testLayer("concatenation", "ONNX", 0.0032, 0.009); +} + +TEST_P(Test_Int8_layers, BatchNorm) +{ + testLayer("layer_batch_norm", "Caffe", 0.0061, 0.019, 1, 1, true); + testLayer("fused_batch_norm", "TensorFlow", 0.0063, 0.02); + testLayer("batch_norm_text", "TensorFlow", 0.0048, 0.013, 1, 1, false, true, true); + testLayer("unfused_batch_norm", "TensorFlow", 0.0076, 0.019); + testLayer("fused_batch_norm_no_gamma", "TensorFlow", 0.0067, 0.015); + testLayer("unfused_batch_norm_no_gamma", "TensorFlow", 0.0123, 0.044); + testLayer("switch_identity", "TensorFlow", 0.0035, 0.011); + testLayer("batch_norm3d", "TensorFlow", 0.0077, 0.02); + testLayer("batch_norm", "ONNX", 0.0012, 0.0049); + testLayer("batch_norm_3d", "ONNX", 0.0039, 0.012); + testLayer("frozenBatchNorm2d", "ONNX", 0.001, 0.0018); + testLayer("batch_norm_subgraph", "ONNX", 0.0049, 0.0098); +} + +TEST_P(Test_Int8_layers, Scale) +{ + testLayer("batch_norm", "TensorFlow", 0.0028, 0.0098); + testLayer("scale", "ONNX", 0.0025, 0.0071); + testLayer("expand_hw", "ONNX", 0.0012, 0.0012); + testLayer("flatten_const", "ONNX", 0.0024, 0.0048); +} + +TEST_P(Test_Int8_layers, InnerProduct) +{ + testLayer("layer_inner_product", "Caffe", 0.005, 0.02, 1, 1, true); + testLayer("matmul", "TensorFlow", 0.0061, 0.019); + testLayer("nhwc_transpose_reshape_matmul", "TensorFlow", 0.0009, 0.0091); + testLayer("nhwc_reshape_matmul", "TensorFlow", 0.03, 0.071); + testLayer("matmul_layout", "TensorFlow", 0.035, 0.06); + testLayer("tf2_dense", "TensorFlow", 0, 0); + testLayer("matmul_add", "ONNX", 0.041, 0.082); + testLayer("linear", "ONNX", 0.0018, 0.0029); + testLayer("constant", "ONNX", 0.00021, 0.0006); + testLayer("lin_with_constant", "ONNX", 0.0011, 0.0016); +} + +TEST_P(Test_Int8_layers, Reshape) +{ + testLayer("reshape_layer", "TensorFlow", 0.0032, 0.0082); + testLayer("reshape_nchw", "TensorFlow", 0.0089, 0.029); + testLayer("reshape_conv", "TensorFlow", 0.035, 0.054); + testLayer("reshape_reduce", "TensorFlow", 0.0042, 0.0078); + testLayer("reshape_as_shape", "TensorFlow", 0.0014, 0.0028); + testLayer("reshape_no_reorder", "TensorFlow", 0.0014, 0.0028); + testLayer("shift_reshape_no_reorder", "TensorFlow", 0.0063, 0.014); + testLayer("dynamic_reshape", "ONNX", 0.0047, 0.0079); + testLayer("dynamic_reshape_opset_11", "ONNX", 0.0048, 0.0081); + testLayer("flatten_by_prod", "ONNX", 0.0048, 0.0081); + testLayer("squeeze", "ONNX", 0.0048, 0.0081); + testLayer("unsqueeze", "ONNX", 0.0033, 0.0053); + testLayer("squeeze_and_conv_dynamic_axes", "ONNX", 0.0054, 0.0154); + testLayer("unsqueeze_and_conv_dynamic_axes", "ONNX", 0.0037, 0.0151); +} + +TEST_P(Test_Int8_layers, Permute) +{ + testLayer("tf2_permute_nhwc_ncwh", "TensorFlow", 0.0028, 0.006); + testLayer("transpose", "ONNX", 0.0015, 0.0046); +} + +TEST_P(Test_Int8_layers, Identity) +{ + testLayer("expand_batch", "ONNX", 0.0027, 0.0036); + testLayer("expand_channels", "ONNX", 0.0013, 0.0019); + testLayer("expand_neg_batch", "ONNX", 0.00071, 0.0019); +} + +TEST_P(Test_Int8_layers, Slice) +{ + testLayer("split", "TensorFlow", 0.0033, 0.0056); + testLayer("slice_4d", "TensorFlow", 0.003, 0.0073); + testLayer("strided_slice", "TensorFlow", 0.008, 0.0142); + testLayer("slice", "ONNX", 0.0046, 0.0077); + testLayer("slice_dynamic_axes", "ONNX", 0.0039, 0.0084); + testLayer("slice_opset_11_steps_2d", "ONNX", 0.0052, 0.0124); + testLayer("slice_opset_11_steps_3d", "ONNX", 0.0068, 0.014); + testLayer("slice_opset_11_steps_4d", "ONNX", 0.0041, 0.008); + testLayer("slice_opset_11_steps_5d", "ONNX", 0.0085, 0.021); +} + +TEST_P(Test_Int8_layers, Dropout) +{ + testLayer("layer_dropout", "Caffe", 0.0021, 0.004); + testLayer("dropout", "ONNX", 0.0029, 0.004); +} + +TEST_P(Test_Int8_layers, Eltwise) +{ + testLayer("layer_eltwise", "Caffe", 0.062, 0.15); + testLayer("conv_2_inps", "Caffe", 0.0086, 0.0232, 2, 1, true, false); + testLayer("eltwise_sub", "TensorFlow", 0.015, 0.047); + testLayer("eltwise_add_vec", "TensorFlow", 0.037, 0.21); // tflite 0.0095, 0.0365 + testLayer("eltwise_mul_vec", "TensorFlow", 0.173, 1.14); // tflite 0.0028, 0.017 + testLayer("channel_broadcast", "TensorFlow", 0.0025, 0.0063); + testLayer("split_equals", "TensorFlow", 0.02, 0.065); + testLayer("mul", "ONNX", 0.0039, 0.014); + testLayer("split_max", "ONNX", 0.004, 0.012); +} + +INSTANTIATE_TEST_CASE_P(/**/, Test_Int8_layers, dnnBackendsAndTargets()); + +class Test_Int8_nets : public DNNTestLayer +{ +public: + void testClassificationNet(Net baseNet, const Mat& blob, const Mat& ref, double l1, double lInf) + { + Net qnet = baseNet.quantize(blob, CV_32F, CV_32F); + qnet.setPreferableBackend(backend); + qnet.setPreferableTarget(target); + + qnet.setInput(blob); + Mat out = qnet.forward(); + normAssert(ref, out, "", l1, lInf); + } + + void testDetectionNet(Net baseNet, const Mat& blob, const Mat& ref, + double confThreshold, double scoreDiff, double iouDiff) + { + Net qnet = baseNet.quantize(blob, CV_32F, CV_32F); + qnet.setPreferableBackend(backend); + qnet.setPreferableTarget(target); + + qnet.setInput(blob); + Mat out = qnet.forward(); + normAssertDetections(ref, out, "", confThreshold, scoreDiff, iouDiff); + } + + void testFaster(Net baseNet, const Mat& ref, double confThreshold, double scoreDiff, double iouDiff) + { + Mat inp = imread(_tf("dog416.png")); + resize(inp, inp, Size(800, 600)); + Mat blob = blobFromImage(inp, 1.0, Size(), Scalar(102.9801, 115.9465, 122.7717), false, false); + Mat imInfo = (Mat_(1, 3) << inp.rows, inp.cols, 1.6f); + + Net qnet = baseNet.quantize(std::vector{blob, imInfo}, CV_32F, CV_32F); + qnet.setPreferableBackend(backend); + qnet.setPreferableTarget(target); + + qnet.setInput(blob, "data"); + qnet.setInput(imInfo, "im_info"); + Mat out = qnet.forward(); + normAssertDetections(ref, out, "", confThreshold, scoreDiff, iouDiff); + } + + void testONNXNet(const String& basename, double l1, double lInf, bool useSoftmax = false) + { + String onnxmodel = findDataFile("dnn/onnx/models/" + basename + ".onnx", false); + + Mat blob = readTensorFromONNX(findDataFile("dnn/onnx/data/input_" + basename + ".pb")); + Mat ref = readTensorFromONNX(findDataFile("dnn/onnx/data/output_" + basename + ".pb")); + Net baseNet = readNetFromONNX(onnxmodel); + baseNet.setPreferableBackend(backend); + baseNet.setPreferableTarget(target); + + Net qnet = baseNet.quantize(blob, CV_32F, CV_32F); + qnet.setInput(blob); + Mat out = qnet.forward(); + + if (useSoftmax) + { + LayerParams lp; + Net netSoftmax; + netSoftmax.addLayerToPrev("softmaxLayer", "Softmax", lp); + netSoftmax.setPreferableBackend(DNN_BACKEND_OPENCV); + + netSoftmax.setInput(out); + out = netSoftmax.forward(); + + netSoftmax.setInput(ref); + ref = netSoftmax.forward(); + } + + normAssert(ref, out, "", l1, lInf); + } + + void testDarknetModel(const std::string& cfg, const std::string& weights, + const cv::Mat& ref, double scoreDiff, double iouDiff, + float confThreshold = 0.24, float nmsThreshold = 0.4) + { + CV_Assert(ref.cols == 7); + std::vector > refClassIds; + std::vector > refScores; + std::vector > refBoxes; + for (int i = 0; i < ref.rows; ++i) + { + int batchId = static_cast(ref.at(i, 0)); + int classId = static_cast(ref.at(i, 1)); + float score = ref.at(i, 2); + float left = ref.at(i, 3); + float top = ref.at(i, 4); + float right = ref.at(i, 5); + float bottom = ref.at(i, 6); + Rect2d box(left, top, right - left, bottom - top); + if (batchId >= refClassIds.size()) + { + refClassIds.resize(batchId + 1); + refScores.resize(batchId + 1); + refBoxes.resize(batchId + 1); + } + refClassIds[batchId].push_back(classId); + refScores[batchId].push_back(score); + refBoxes[batchId].push_back(box); + } + + Mat img1 = imread(_tf("dog416.png")); + Mat img2 = imread(_tf("street.png")); + std::vector samples(2); + samples[0] = img1; samples[1] = img2; + + // determine test type, whether batch or single img + int batch_size = refClassIds.size(); + CV_Assert(batch_size == 1 || batch_size == 2); + samples.resize(batch_size); + + Mat inp = blobFromImages(samples, 1.0/255, Size(416, 416), Scalar(), true, false); + + Net baseNet = readNetFromDarknet(findDataFile("dnn/" + cfg), findDataFile("dnn/" + weights, false)); + Net qnet = baseNet.quantize(inp, CV_32F, CV_32F); + qnet.setPreferableBackend(backend); + qnet.setPreferableTarget(target); + qnet.setInput(inp); + std::vector outs; + qnet.forward(outs, qnet.getUnconnectedOutLayersNames()); + + for (int b = 0; b < batch_size; ++b) + { + std::vector classIds; + std::vector confidences; + std::vector boxes; + for (int i = 0; i < outs.size(); ++i) + { + Mat out; + if (batch_size > 1){ + // get the sample slice from 3D matrix (batch, box, classes+5) + Range ranges[3] = {Range(b, b+1), Range::all(), Range::all()}; + out = outs[i](ranges).reshape(1, outs[i].size[1]); + }else{ + out = outs[i]; + } + for (int j = 0; j < out.rows; ++j) + { + Mat scores = out.row(j).colRange(5, out.cols); + double confidence; + Point maxLoc; + minMaxLoc(scores, 0, &confidence, 0, &maxLoc); + + if (confidence > confThreshold) { + float* detection = out.ptr(j); + double centerX = detection[0]; + double centerY = detection[1]; + double width = detection[2]; + double height = detection[3]; + boxes.push_back(Rect2d(centerX - 0.5 * width, centerY - 0.5 * height, + width, height)); + confidences.push_back(confidence); + classIds.push_back(maxLoc.x); + } + } + } + + // here we need NMS of boxes + std::vector indices; + NMSBoxes(boxes, confidences, confThreshold, nmsThreshold, indices); + + std::vector nms_classIds; + std::vector nms_confidences; + std::vector nms_boxes; + + for (size_t i = 0; i < indices.size(); ++i) + { + int idx = indices[i]; + Rect2d box = boxes[idx]; + float conf = confidences[idx]; + int class_id = classIds[idx]; + nms_boxes.push_back(box); + nms_confidences.push_back(conf); + nms_classIds.push_back(class_id); + } + + if (cvIsNaN(iouDiff)) + { + if (b == 0) + std::cout << "Skip accuracy checks" << std::endl; + continue; + } + + normAssertDetections(refClassIds[b], refScores[b], refBoxes[b], nms_classIds, nms_confidences, nms_boxes, + format("batch size %d, sample %d\n", batch_size, b).c_str(), confThreshold, scoreDiff, iouDiff); + } + } +}; + +TEST_P(Test_Int8_nets, AlexNet) +{ +#if defined(OPENCV_32BIT_CONFIGURATION) && defined(HAVE_OPENCL) + applyTestTag(CV_TEST_TAG_MEMORY_2GB); +#else + applyTestTag(target == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_512MB : CV_TEST_TAG_MEMORY_1GB); +#endif + if (backend != DNN_BACKEND_OPENCV) + throw SkipTestException("Only OpenCV backend is supported"); + + Net net = readNetFromCaffe(findDataFile("dnn/bvlc_alexnet.prototxt"), + findDataFile("dnn/bvlc_alexnet.caffemodel", false)); + + Mat inp = imread(_tf("grace_hopper_227.png")); + Mat blob = blobFromImage(inp, 1.0, Size(227, 227), Scalar(), false); + Mat ref = blobFromNPY(_tf("caffe_alexnet_prob.npy")); + + float l1 = 1e-4, lInf = 0.003; + testClassificationNet(net, blob, ref, l1, lInf); +} + +TEST_P(Test_Int8_nets, GoogLeNet) +{ + Net net = readNetFromCaffe(findDataFile("dnn/bvlc_googlenet.prototxt"), + findDataFile("dnn/bvlc_googlenet.caffemodel", false)); + + std::vector inpMats; + inpMats.push_back( imread(_tf("googlenet_0.png")) ); + inpMats.push_back( imread(_tf("googlenet_1.png")) ); + Mat blob = blobFromImages(inpMats, 1.0, Size(224, 224), Scalar(), false); + Mat ref = blobFromNPY(_tf("googlenet_prob.npy")); + + float l1 = 2e-4, lInf = 0.06; + testClassificationNet(net, blob, ref, l1, lInf); +} + +TEST_P(Test_Int8_nets, ResNet50) +{ + applyTestTag(target == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_512MB : CV_TEST_TAG_MEMORY_1GB); + if (backend != DNN_BACKEND_OPENCV) + throw SkipTestException("Only OpenCV backend is supported"); + + Net net = readNetFromCaffe(findDataFile("dnn/ResNet-50-deploy.prototxt"), + findDataFile("dnn/ResNet-50-model.caffemodel", false)); + + Mat inp = imread(_tf("googlenet_0.png")); + Mat blob = blobFromImage(inp, 1.0, Size(224, 224), Scalar(), false); + Mat ref = blobFromNPY(_tf("resnet50_prob.npy")); + + float l1 = 3e-4, lInf = 0.035; + testClassificationNet(net, blob, ref, l1, lInf); +} + +TEST_P(Test_Int8_nets, DenseNet121) +{ + applyTestTag(CV_TEST_TAG_MEMORY_512MB); + + Net net = readNetFromCaffe(findDataFile("dnn/DenseNet_121.prototxt", false), + findDataFile("dnn/DenseNet_121.caffemodel", false)); + + Mat inp = imread(_tf("dog416.png")); + Mat blob = blobFromImage(inp, 1.0 / 255.0, Size(224, 224), Scalar(), true, true); + Mat ref = blobFromNPY(_tf("densenet_121_output.npy")); + + float l1 = 0.76, lInf = 3.31; // seems wrong + testClassificationNet(net, blob, ref, l1, lInf); +} + +TEST_P(Test_Int8_nets, SqueezeNet_v1_1) +{ + if(target == DNN_TARGET_OPENCL_FP16) + applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16); + + Net net = readNetFromCaffe(findDataFile("dnn/squeezenet_v1.1.prototxt"), + findDataFile("dnn/squeezenet_v1.1.caffemodel", false)); + + Mat inp = imread(_tf("googlenet_0.png")); + Mat blob = blobFromImage(inp, 1.0, Size(227, 227), Scalar(), false, true); + Mat ref = blobFromNPY(_tf("squeezenet_v1.1_prob.npy")); + + float l1 = 3e-4, lInf = 0.056; + testClassificationNet(net, blob, ref, l1, lInf); +} + +TEST_P(Test_Int8_nets, CaffeNet) +{ +#if defined(OPENCV_32BIT_CONFIGURATION) && (defined(HAVE_OPENCL) || defined(_WIN32)) + applyTestTag(CV_TEST_TAG_MEMORY_2GB); +#else + applyTestTag(target == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_512MB : CV_TEST_TAG_MEMORY_1GB); +#endif + +#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2019030000) + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 && target == DNN_TARGET_MYRIAD + && getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_X) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD_X, CV_TEST_TAG_DNN_SKIP_IE_NN_BUILDER, CV_TEST_TAG_DNN_SKIP_IE_VERSION); +#endif + float l1 = 4e-5, lInf = 0.0025; + testONNXNet("caffenet", l1, lInf); +} + +TEST_P(Test_Int8_nets, RCNN_ILSVRC13) +{ +#if defined(OPENCV_32BIT_CONFIGURATION) && (defined(HAVE_OPENCL) || defined(_WIN32)) + applyTestTag(CV_TEST_TAG_MEMORY_2GB); +#else + applyTestTag(target == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_512MB : CV_TEST_TAG_MEMORY_1GB); +#endif + +#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2019030000) + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 && target == DNN_TARGET_MYRIAD + && getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_X) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD_X, CV_TEST_TAG_DNN_SKIP_IE_NN_BUILDER, CV_TEST_TAG_DNN_SKIP_IE_VERSION); +#endif + float l1 = 0.02, lInf = 0.042; + testONNXNet("rcnn_ilsvrc13", l1, lInf); +} + +TEST_P(Test_Int8_nets, Inception_v2) +{ + testONNXNet("inception_v2", default_l1, default_lInf, true); +} + +TEST_P(Test_Int8_nets, MobileNet_v2) +{ + testONNXNet("mobilenetv2", default_l1, default_lInf, true); +} + +TEST_P(Test_Int8_nets, Shufflenet) +{ + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019) + { + if (target == DNN_TARGET_OPENCL_FP16) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16, CV_TEST_TAG_DNN_SKIP_IE_NN_BUILDER); + if (target == DNN_TARGET_OPENCL) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL, CV_TEST_TAG_DNN_SKIP_IE_NN_BUILDER); + if (target == DNN_TARGET_MYRIAD) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD, CV_TEST_TAG_DNN_SKIP_IE_NN_BUILDER); + } + testONNXNet("shufflenet", default_l1, default_lInf); +} + +TEST_P(Test_Int8_nets, MobileNet_SSD) +{ + Net net = readNetFromCaffe(findDataFile("dnn/MobileNetSSD_deploy.prototxt", false), + findDataFile("dnn/MobileNetSSD_deploy.caffemodel", false)); + + Mat inp = imread(_tf("street.png")); + Mat blob = blobFromImage(inp, 1.0 / 127.5, Size(300, 300), Scalar(127.5, 127.5, 127.5), false); + Mat ref = blobFromNPY(_tf("mobilenet_ssd_caffe_out.npy")); + + float confThreshold = FLT_MIN, scoreDiff = 0.059, iouDiff = 0.11; + testDetectionNet(net, blob, ref, confThreshold, scoreDiff, iouDiff); +} + +TEST_P(Test_Int8_nets, MobileNet_v1_SSD) +{ + Net net = readNetFromTensorflow(findDataFile("dnn/ssd_mobilenet_v1_coco_2017_11_17.pb", false), + findDataFile("dnn/ssd_mobilenet_v1_coco_2017_11_17.pbtxt")); + + Mat inp = imread(_tf("dog416.png")); + Mat blob = blobFromImage(inp, 1.0, Size(300, 300), Scalar(), true, false); + Mat ref = blobFromNPY(_tf("tensorflow/ssd_mobilenet_v1_coco_2017_11_17.detection_out.npy")); + + float confThreshold = 0.5, scoreDiff = 0.034, iouDiff = 0.13; + testDetectionNet(net, blob, ref, confThreshold, scoreDiff, iouDiff); +} + +TEST_P(Test_Int8_nets, MobileNet_v1_SSD_PPN) +{ +#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2018050000) + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 && (target == DNN_TARGET_OPENCL || target == DNN_TARGET_OPENCL_FP16)) + applyTestTag(target == DNN_TARGET_OPENCL ? CV_TEST_TAG_DNN_SKIP_IE_OPENCL : CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16, + CV_TEST_TAG_DNN_SKIP_IE_NN_BUILDER, CV_TEST_TAG_DNN_SKIP_IE_VERSION); +#endif + + Net net = readNetFromTensorflow(findDataFile("dnn/ssd_mobilenet_v1_ppn_coco.pb", false), + findDataFile("dnn/ssd_mobilenet_v1_ppn_coco.pbtxt")); + + Mat inp = imread(_tf("dog416.png")); + Mat blob = blobFromImage(inp, 1.0, Size(300, 300), Scalar(), true, false); + Mat ref = blobFromNPY(_tf("tensorflow/ssd_mobilenet_v1_ppn_coco.detection_out.npy")); + + float confThreshold = 0.51, scoreDiff = 0.04, iouDiff = 0.06; + testDetectionNet(net, blob, ref, confThreshold, scoreDiff, iouDiff); +} + +TEST_P(Test_Int8_nets, Inception_v2_SSD) +{ + applyTestTag(target == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_512MB : CV_TEST_TAG_MEMORY_1GB); +#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_LE(2019010000) + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 && target == DNN_TARGET_MYRIAD && + getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_X) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD_X, CV_TEST_TAG_DNN_SKIP_IE_NN_BUILDER, CV_TEST_TAG_DNN_SKIP_IE_VERSION); +#endif + + Net net = readNetFromTensorflow(findDataFile("dnn/ssd_inception_v2_coco_2017_11_17.pb", false), + findDataFile("dnn/ssd_inception_v2_coco_2017_11_17.pbtxt")); + + Mat inp = imread(_tf("street.png")); + Mat blob = blobFromImage(inp, 1.0, Size(300, 300), Scalar(), true, false); + Mat ref = (Mat_(5, 7) << 0, 1, 0.90176028, 0.19872092, 0.36311883, 0.26461923, 0.63498729, + 0, 3, 0.93569964, 0.64865261, 0.45906419, 0.80675775, 0.65708131, + 0, 3, 0.75838411, 0.44668293, 0.45907149, 0.49459291, 0.52197015, + 0, 10, 0.95932811, 0.38349164, 0.32528657, 0.40387636, 0.39165527, + 0, 10, 0.93973452, 0.66561931, 0.37841269, 0.68074018, 0.42907384); + + float confThreshold = 0.5, scoreDiff = 0.0114, iouDiff = 0.22; + testDetectionNet(net, blob, ref, confThreshold, scoreDiff, iouDiff); +} + +TEST_P(Test_Int8_nets, opencv_face_detector) +{ + Net net = readNetFromCaffe(findDataFile("dnn/opencv_face_detector.prototxt"), + findDataFile("dnn/opencv_face_detector.caffemodel", false)); + + Mat inp = imread(findDataFile("gpu/lbpcascade/er.png")); + Mat blob = blobFromImage(inp, 1.0, Size(), Scalar(104.0, 177.0, 123.0), false, false); + Mat ref = (Mat_(6, 7) << 0, 1, 0.99520785, 0.80997437, 0.16379407, 0.87996572, 0.26685631, + 0, 1, 0.9934696, 0.2831718, 0.50738752, 0.345781, 0.5985168, + 0, 1, 0.99096733, 0.13629119, 0.24892329, 0.19756334, 0.3310290, + 0, 1, 0.98977017, 0.23901358, 0.09084064, 0.29902688, 0.1769477, + 0, 1, 0.97203469, 0.67965847, 0.06876482, 0.73999709, 0.1513494, + 0, 1, 0.95097077, 0.51901293, 0.45863652, 0.5777427, 0.5347801); + + float confThreshold = 0.5, scoreDiff = 0.002, iouDiff = 0.21; + testDetectionNet(net, blob, ref, confThreshold, scoreDiff, iouDiff); +} + +TEST_P(Test_Int8_nets, EfficientDet) +{ + if (target != DNN_TARGET_CPU) + { + if (target == DNN_TARGET_OPENCL_FP16) applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16); + if (target == DNN_TARGET_OPENCL) applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL); + if (target == DNN_TARGET_MYRIAD) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD); + } + Net net = readNetFromTensorflow(findDataFile("dnn/efficientdet-d0.pb", false), + findDataFile("dnn/efficientdet-d0.pbtxt")); + + Mat inp = imread(_tf("dog416.png")); + Mat blob = blobFromImage(inp, 1.0/255, Size(512, 512), Scalar(123.675, 116.28, 103.53)); + Mat ref = (Mat_(3, 7) << 0, 1, 0.8437444, 0.153996080160141, 0.20534580945968628, 0.7463544607162476, 0.7414066195487976, + 0, 17, 0.8245924, 0.16657517850399017, 0.3996818959712982, 0.4111558794975281, 0.9306337833404541, + 0, 7, 0.8039304, 0.6118435263633728, 0.13175517320632935, 0.9065558314323425, 0.2943994700908661); + + float confThreshold = 0.65, scoreDiff = 0.17, iouDiff = 0.18; + testDetectionNet(net, blob, ref, confThreshold, scoreDiff, iouDiff); +} + +TEST_P(Test_Int8_nets, FasterRCNN_resnet50) +{ + applyTestTag( + (target == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_1GB : CV_TEST_TAG_MEMORY_2GB), + CV_TEST_TAG_LONG, + CV_TEST_TAG_DEBUG_VERYLONG + ); + +#ifdef INF_ENGINE_RELEASE + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 && + (INF_ENGINE_VER_MAJOR_LT(2019020000) || target != DNN_TARGET_CPU)) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_NN_BUILDER, CV_TEST_TAG_DNN_SKIP_IE_VERSION); + + if (INF_ENGINE_VER_MAJOR_GT(2019030000) && + backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target == DNN_TARGET_MYRIAD) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD, CV_TEST_TAG_DNN_SKIP_IE_NGRAPH); +#endif + + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target == DNN_TARGET_OPENCL_FP16) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16, CV_TEST_TAG_DNN_SKIP_IE_NGRAPH); + + if (backend == DNN_BACKEND_OPENCV && target == DNN_TARGET_OPENCL_FP16) + applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16); + + if (backend == DNN_BACKEND_CUDA && target == DNN_TARGET_CUDA_FP16) + applyTestTag(CV_TEST_TAG_DNN_SKIP_CUDA_FP16); + + Net net = readNetFromTensorflow(findDataFile("dnn/faster_rcnn_resnet50_coco_2018_01_28.pb", false), + findDataFile("dnn/faster_rcnn_resnet50_coco_2018_01_28.pbtxt")); + + Mat inp = imread(_tf("dog416.png")); + Mat blob = blobFromImage(inp, 1.0, Size(800, 600), Scalar(), true, false); + Mat ref = blobFromNPY(_tf("tensorflow/faster_rcnn_resnet50_coco_2018_01_28.detection_out.npy")); + + float confThreshold = 0.5, scoreDiff = 0.025, iouDiff = 0.15; + testDetectionNet(net, blob, ref, confThreshold, scoreDiff, iouDiff); +} + +TEST_P(Test_Int8_nets, FasterRCNN_inceptionv2) +{ + applyTestTag( + (target == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_1GB : CV_TEST_TAG_MEMORY_2GB), + CV_TEST_TAG_LONG, + CV_TEST_TAG_DEBUG_VERYLONG + ); + +#ifdef INF_ENGINE_RELEASE + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 && + (INF_ENGINE_VER_MAJOR_LT(2019020000) || target != DNN_TARGET_CPU)) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_NN_BUILDER, CV_TEST_TAG_DNN_SKIP_IE_VERSION); + + if (INF_ENGINE_VER_MAJOR_GT(2019030000) && + backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target == DNN_TARGET_MYRIAD) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD, CV_TEST_TAG_DNN_SKIP_IE_NGRAPH); +#endif + + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target == DNN_TARGET_OPENCL_FP16) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16, CV_TEST_TAG_DNN_SKIP_IE_NGRAPH); + + if (backend == DNN_BACKEND_OPENCV && target == DNN_TARGET_OPENCL_FP16) + applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16); + + if (backend == DNN_BACKEND_CUDA && target == DNN_TARGET_CUDA_FP16) + applyTestTag(CV_TEST_TAG_DNN_SKIP_CUDA_FP16); + + Net net = readNetFromTensorflow(findDataFile("dnn/faster_rcnn_inception_v2_coco_2018_01_28.pb", false), + findDataFile("dnn/faster_rcnn_inception_v2_coco_2018_01_28.pbtxt")); + + Mat inp = imread(_tf("dog416.png")); + Mat blob = blobFromImage(inp, 1.0, Size(800, 600), Scalar(), true, false); + Mat ref = blobFromNPY(_tf("tensorflow/faster_rcnn_inception_v2_coco_2018_01_28.detection_out.npy")); + + float confThreshold = 0.5, scoreDiff = 0.21, iouDiff = 0.1; + testDetectionNet(net, blob, ref, confThreshold, scoreDiff, iouDiff); +} + +TEST_P(Test_Int8_nets, FasterRCNN_vgg16) +{ + applyTestTag( +#if defined(OPENCV_32BIT_CONFIGURATION) && defined(HAVE_OPENCL) + CV_TEST_TAG_MEMORY_2GB, +#else + (target == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_1GB : CV_TEST_TAG_MEMORY_2GB), +#endif + CV_TEST_TAG_LONG, + CV_TEST_TAG_DEBUG_VERYLONG + ); + +#if defined(INF_ENGINE_RELEASE) + if ((backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 || backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) && (target == DNN_TARGET_OPENCL || target == DNN_TARGET_OPENCL_FP16)) + applyTestTag(target == DNN_TARGET_OPENCL ? CV_TEST_TAG_DNN_SKIP_IE_OPENCL : CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16); + + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_NGRAPH, CV_TEST_TAG_DNN_SKIP_IE_VERSION); + + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 && target == DNN_TARGET_MYRIAD) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD); +#endif + + Net net = readNetFromCaffe(findDataFile("dnn/faster_rcnn_vgg16.prototxt"), + findDataFile("dnn/VGG16_faster_rcnn_final.caffemodel", false)); + + Mat ref = (Mat_(3, 7) << 0, 2, 0.949398, 99.2454, 210.141, 601.205, 462.849, + 0, 7, 0.997022, 481.841, 92.3218, 722.685, 175.953, + 0, 12, 0.993028, 133.221, 189.377, 350.994, 563.166); + + float confThreshold = 0.8, scoreDiff = 0.024, iouDiff = 0.35; + testFaster(net, ref, confThreshold, scoreDiff, iouDiff); +} + +TEST_P(Test_Int8_nets, FasterRCNN_zf) +{ + applyTestTag( +#if defined(OPENCV_32BIT_CONFIGURATION) && defined(HAVE_OPENCL) + CV_TEST_TAG_MEMORY_2GB, +#else + (target == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_512MB : CV_TEST_TAG_MEMORY_1GB), +#endif + CV_TEST_TAG_DEBUG_LONG + ); + + if ((backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 || + backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) && target == DNN_TARGET_OPENCL_FP16) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16); + + if ((backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 || + backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) && target == DNN_TARGET_MYRIAD) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD); + + if (target == DNN_TARGET_CUDA_FP16) + applyTestTag(CV_TEST_TAG_DNN_SKIP_CUDA_FP16); + + Net net = readNetFromCaffe(findDataFile("dnn/faster_rcnn_zf.prototxt"), + findDataFile("dnn/ZF_faster_rcnn_final.caffemodel", false)); + + Mat ref = (Mat_(3, 7) << 0, 2, 0.90121, 120.407, 115.83, 570.586, 528.395, + 0, 7, 0.988779, 469.849, 75.1756, 718.64, 186.762, + 0, 12, 0.967198, 138.588, 206.843, 329.766, 553.176); + + float confThreshold = 0.8, scoreDiff = 0.021, iouDiff = 0.1; + testFaster(net, ref, confThreshold, scoreDiff, iouDiff); +} + +TEST_P(Test_Int8_nets, RFCN) +{ + applyTestTag( + (target == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_512MB : CV_TEST_TAG_MEMORY_2GB), + CV_TEST_TAG_LONG, + CV_TEST_TAG_DEBUG_VERYLONG + ); + + if ((backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 || + backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) && target == DNN_TARGET_OPENCL_FP16) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16); + + if ((backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 || + backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) && target == DNN_TARGET_MYRIAD) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD); + + Net net = readNetFromCaffe(findDataFile("dnn/rfcn_pascal_voc_resnet50.prototxt"), + findDataFile("dnn/resnet50_rfcn_final.caffemodel", false)); + + Mat ref = (Mat_(2, 7) << 0, 7, 0.991359, 491.822, 81.1668, 702.573, 178.234, + 0, 12, 0.94786, 132.093, 223.903, 338.077, 566.16); + + float confThreshold = 0.8, scoreDiff = 0.017, iouDiff = 0.11; + testFaster(net, ref, confThreshold, scoreDiff, iouDiff); +} + +TEST_P(Test_Int8_nets, YoloVoc) +{ + applyTestTag( +#if defined(OPENCV_32BIT_CONFIGURATION) && defined(HAVE_OPENCL) + CV_TEST_TAG_MEMORY_2GB, +#else + CV_TEST_TAG_MEMORY_1GB, +#endif + CV_TEST_TAG_LONG + ); + +#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2020040000) + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target == DNN_TARGET_OPENCL) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL, CV_TEST_TAG_DNN_SKIP_IE_VERSION); + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target == DNN_TARGET_OPENCL_FP16) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16, CV_TEST_TAG_DNN_SKIP_IE_VERSION); +#endif +#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_GE(2019010000) + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 && target == DNN_TARGET_OPENCL_FP16) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16); +#endif +#if defined(INF_ENGINE_RELEASE) + if ((backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 || backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) && + target == DNN_TARGET_MYRIAD && getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_X) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD_X); +#endif + + Mat ref = (Mat_(6, 7) << 0, 6, 0.750469f, 0.577374f, 0.127391f, 0.902949f, 0.300809f, + 0, 1, 0.780879f, 0.270762f, 0.264102f, 0.732475f, 0.745412f, + 0, 11, 0.901615f, 0.1386f, 0.338509f, 0.421337f, 0.938789f, + 1, 14, 0.623813f, 0.183179f, 0.381921f, 0.247726f, 0.625847f, + 1, 6, 0.667770f, 0.446555f, 0.453578f, 0.499986f, 0.519167f, + 1, 6, 0.844947f, 0.637058f, 0.460398f, 0.828508f, 0.66427f); + + std::string config_file = "yolo-voc.cfg"; + std::string weights_file = "yolo-voc.weights"; + + double scoreDiff = 0.1, iouDiff = 0.3; + { + SCOPED_TRACE("batch size 1"); + testDarknetModel(config_file, weights_file, ref.rowRange(0, 3), scoreDiff, iouDiff); + } + + { + SCOPED_TRACE("batch size 2"); + testDarknetModel(config_file, weights_file, ref, scoreDiff, iouDiff); + } +} + +TEST_P(Test_Int8_nets, TinyYoloVoc) +{ + applyTestTag(CV_TEST_TAG_MEMORY_512MB); + +#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2020040000) + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target == DNN_TARGET_OPENCL) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL, CV_TEST_TAG_DNN_SKIP_IE_VERSION); + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target == DNN_TARGET_OPENCL_FP16) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16, CV_TEST_TAG_DNN_SKIP_IE_VERSION); +#endif +#if defined(INF_ENGINE_RELEASE) + if ((backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 || backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) && + target == DNN_TARGET_MYRIAD && getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_X) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD_X); +#endif + + Mat ref = (Mat_(4, 7) << 0, 6, 0.761967f, 0.579042f, 0.159161f, 0.894482f, 0.31994f, + 0, 11, 0.780595f, 0.129696f, 0.386467f, 0.445275f, 0.920994f, + 1, 6, 0.651450f, 0.460526f, 0.458019f, 0.522527f, 0.5341f, + 1, 6, 0.928758f, 0.651024f, 0.463539f, 0.823784f, 0.654998f); + + std::string config_file = "tiny-yolo-voc.cfg"; + std::string weights_file = "tiny-yolo-voc.weights"; + + double scoreDiff = 0.043, iouDiff = 0.12; + { + SCOPED_TRACE("batch size 1"); + testDarknetModel(config_file, weights_file, ref.rowRange(0, 2), scoreDiff, iouDiff); + } + + { + SCOPED_TRACE("batch size 2"); + testDarknetModel(config_file, weights_file, ref, scoreDiff, iouDiff); + } +} + +TEST_P(Test_Int8_nets, YOLOv3) +{ + applyTestTag(CV_TEST_TAG_LONG, (target == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_1GB : CV_TEST_TAG_MEMORY_2GB)); + +#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2020040000) + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target == DNN_TARGET_OPENCL) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL, CV_TEST_TAG_DNN_SKIP_IE_VERSION); + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target == DNN_TARGET_OPENCL_FP16) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16, CV_TEST_TAG_DNN_SKIP_IE_VERSION); +#endif + + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target == DNN_TARGET_MYRIAD) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD, CV_TEST_TAG_DNN_SKIP_IE_NGRAPH); + + const int N0 = 3; + const int N1 = 6; + static const float ref_[/* (N0 + N1) * 7 */] = { +0, 16, 0.998836f, 0.160024f, 0.389964f, 0.417885f, 0.943716f, +0, 1, 0.987908f, 0.150913f, 0.221933f, 0.742255f, 0.746261f, +0, 7, 0.952983f, 0.614621f, 0.150257f, 0.901368f, 0.289251f, + +1, 2, 0.997412f, 0.647584f, 0.459939f, 0.821037f, 0.663947f, +1, 2, 0.989633f, 0.450719f, 0.463353f, 0.496306f, 0.522258f, +1, 0, 0.980053f, 0.195856f, 0.378454f, 0.258626f, 0.629257f, +1, 9, 0.785341f, 0.665503f, 0.373543f, 0.688893f, 0.439244f, +1, 9, 0.733275f, 0.376029f, 0.315694f, 0.401776f, 0.395165f, +1, 9, 0.384815f, 0.659824f, 0.372389f, 0.673927f, 0.429412f, + }; + Mat ref(N0 + N1, 7, CV_32FC1, (void*)ref_); + + std::string config_file = "yolov3.cfg"; + std::string weights_file = "yolov3.weights"; + + double scoreDiff = 0.08, iouDiff = 0.21, confThreshold = 0.25; + { + SCOPED_TRACE("batch size 1"); + testDarknetModel(config_file, weights_file, ref.rowRange(0, N0), scoreDiff, iouDiff, confThreshold); + } + +#if defined(INF_ENGINE_RELEASE) + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019) + { + if (target == DNN_TARGET_OPENCL) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL, CV_TEST_TAG_DNN_SKIP_IE_VERSION); + else if (target == DNN_TARGET_OPENCL_FP16 && INF_ENGINE_VER_MAJOR_LE(202010000)) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16, CV_TEST_TAG_DNN_SKIP_IE_VERSION); + else if (target == DNN_TARGET_MYRIAD && + getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_X) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD_X); + } +#endif + + { + SCOPED_TRACE("batch size 2"); + testDarknetModel(config_file, weights_file, ref, scoreDiff, iouDiff, confThreshold); + } +} + +TEST_P(Test_Int8_nets, YOLOv4) +{ + applyTestTag(CV_TEST_TAG_LONG, (target == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_1GB : CV_TEST_TAG_MEMORY_2GB)); + +#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2020040000) + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target == DNN_TARGET_OPENCL) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL, CV_TEST_TAG_DNN_SKIP_IE_VERSION); + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target == DNN_TARGET_OPENCL_FP16) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16, CV_TEST_TAG_DNN_SKIP_IE_VERSION); +#endif +#if defined(INF_ENGINE_RELEASE) + if (target == DNN_TARGET_MYRIAD) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD, CV_TEST_TAG_DNN_SKIP_IE_VERSION); +#endif + + const int N0 = 3; + const int N1 = 7; + static const float ref_[/* (N0 + N1) * 7 */] = { +0, 16, 0.992194f, 0.172375f, 0.402458f, 0.403918f, 0.932801f, +0, 1, 0.988326f, 0.166708f, 0.228236f, 0.737208f, 0.735803f, +0, 7, 0.94639f, 0.602523f, 0.130399f, 0.901623f, 0.298452f, + +1, 2, 0.99761f, 0.646556f, 0.45985f, 0.816041f, 0.659067f, +1, 0, 0.988913f, 0.201726f, 0.360282f, 0.266181f, 0.631728f, +1, 2, 0.98233f, 0.452007f, 0.462217f, 0.495612f, 0.521687f, +1, 9, 0.919195f, 0.374642f, 0.316524f, 0.398126f, 0.393714f, +1, 9, 0.856303f, 0.666842f, 0.372215f, 0.685539f, 0.44141f, +1, 9, 0.313516f, 0.656791f, 0.374734f, 0.671959f, 0.438371f, +1, 9, 0.256625f, 0.940232f, 0.326931f, 0.967586f, 0.374002f, + }; + Mat ref(N0 + N1, 7, CV_32FC1, (void*)ref_); + + std::string config_file = "yolov4.cfg"; + std::string weights_file = "yolov4.weights"; + double scoreDiff = 0.1, iouDiff = 0.17; + { + SCOPED_TRACE("batch size 1"); + testDarknetModel(config_file, weights_file, ref.rowRange(0, N0), scoreDiff, iouDiff); + } + + { + SCOPED_TRACE("batch size 2"); + +#if defined(INF_ENGINE_RELEASE) + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019) + { + if (target == DNN_TARGET_OPENCL) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL, CV_TEST_TAG_DNN_SKIP_IE_VERSION); + else if (target == DNN_TARGET_OPENCL_FP16 && INF_ENGINE_VER_MAJOR_LE(202010000)) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16, CV_TEST_TAG_DNN_SKIP_IE_VERSION); + else if (target == DNN_TARGET_MYRIAD && + getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_X) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD_X); + } +#endif + + testDarknetModel(config_file, weights_file, ref, scoreDiff, iouDiff); + } +} + +TEST_P(Test_Int8_nets, YOLOv4_tiny) +{ + applyTestTag( + target == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_512MB : CV_TEST_TAG_MEMORY_1GB + ); + +#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_GE(2021010000) + if (target == DNN_TARGET_MYRIAD) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD, CV_TEST_TAG_DNN_SKIP_IE_VERSION); +#endif + + const float confThreshold = 0.6; + + const int N0 = 2; + const int N1 = 3; + static const float ref_[/* (N0 + N1) * 7 */] = { +0, 7, 0.85935f, 0.593484f, 0.141211f, 0.920356f, 0.291593f, +0, 16, 0.795188f, 0.169207f, 0.386886f, 0.423753f, 0.933004f, + +1, 2, 0.996832f, 0.653802f, 0.464573f, 0.815193f, 0.653292f, +1, 2, 0.963325f, 0.451151f, 0.458915f, 0.496255f, 0.52241f, +1, 0, 0.926244f, 0.194851f, 0.361743f, 0.260277f, 0.632364f, + }; + Mat ref(N0 + N1, 7, CV_32FC1, (void*)ref_); + + std::string config_file = "yolov4-tiny.cfg"; + std::string weights_file = "yolov4-tiny.weights"; + double scoreDiff = 0.12; + double iouDiff = target == DNN_TARGET_OPENCL_FP16 ? 0.2 : 0.082; + +#if defined(INF_ENGINE_RELEASE) + if (target == DNN_TARGET_MYRIAD) // bad accuracy + iouDiff = std::numeric_limits::quiet_NaN(); + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 && target == DNN_TARGET_OPENCL) + iouDiff = std::numeric_limits::quiet_NaN(); + if ((backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 || + backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) && target == DNN_TARGET_OPENCL_FP16) + iouDiff = std::numeric_limits::quiet_NaN(); +#endif + + { + SCOPED_TRACE("batch size 1"); + testDarknetModel(config_file, weights_file, ref.rowRange(0, N0), scoreDiff, iouDiff, confThreshold); + } + + /* bad accuracy on second image + { + SCOPED_TRACE("batch size 2"); + testDarknetModel(config_file, weights_file, ref, scoreDiff, iouDiff, confThreshold); + } + */ + +#if defined(INF_ENGINE_RELEASE) + if (target == DNN_TARGET_MYRIAD) // bad accuracy + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD, CV_TEST_TAG_DNN_SKIP_IE_VERSION); + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 && target == DNN_TARGET_OPENCL) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL, CV_TEST_TAG_DNN_SKIP_IE_VERSION); + if ((backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 || + backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) && target == DNN_TARGET_OPENCL_FP16) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16, CV_TEST_TAG_DNN_SKIP_IE_VERSION); +#endif +} + +INSTANTIATE_TEST_CASE_P(/**/, Test_Int8_nets, dnnBackendsAndTargets()); +}} // namespace From 9cfa84313c5833d7295fcf57be93d5d2aaadfd88 Mon Sep 17 00:00:00 2001 From: Vincent Rabaud Date: Sat, 10 Jul 2021 00:21:52 +0200 Subject: [PATCH 128/262] Use the one argument version of SetTotalBytesLimit. The two argument versions has been deprecated, cf https://developers.google.com/protocol-buffers/docs/reference/cpp/google.protobuf.io.coded_stream --- modules/dnn/src/caffe/caffe_io.cpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/modules/dnn/src/caffe/caffe_io.cpp b/modules/dnn/src/caffe/caffe_io.cpp index 2fc4d84f46..ebecf95eea 100644 --- a/modules/dnn/src/caffe/caffe_io.cpp +++ b/modules/dnn/src/caffe/caffe_io.cpp @@ -92,6 +92,7 @@ #ifdef HAVE_PROTOBUF #include #include +#include #include #include @@ -1111,7 +1112,11 @@ static const int kProtoReadBytesLimit = INT_MAX; // Max size of 2 GB minus 1 by bool ReadProtoFromBinary(ZeroCopyInputStream* input, Message *proto) { CodedInputStream coded_input(input); +#if GOOGLE_PROTOBUF_VERSION >= 3006000 + coded_input.SetTotalBytesLimit(kProtoReadBytesLimit); +#else coded_input.SetTotalBytesLimit(kProtoReadBytesLimit, 536870912); +#endif return proto->ParseFromCodedStream(&coded_input); } From c08897cd106dbeca97c4591a8af088c563fe1444 Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Thu, 19 Aug 2021 20:06:41 +0000 Subject: [PATCH 129/262] cmake: handle empty CVPY_SUFFIX --- modules/python/common.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/python/common.cmake b/modules/python/common.cmake index 6a438fd1a2..ebbb2e2f65 100644 --- a/modules/python/common.cmake +++ b/modules/python/common.cmake @@ -86,7 +86,7 @@ set_target_properties(${the_module} PROPERTIES ARCHIVE_OUTPUT_NAME ${the_module} # prevent name conflict for python2/3 outputs PREFIX "" OUTPUT_NAME cv2 - SUFFIX ${CVPY_SUFFIX}) + SUFFIX "${CVPY_SUFFIX}") if(ENABLE_SOLUTION_FOLDERS) set_target_properties(${the_module} PROPERTIES FOLDER "bindings") From d6306f8ccbcd214dd806a8fed739eff03978ae40 Mon Sep 17 00:00:00 2001 From: Alexander Panov Date: Fri, 20 Aug 2021 13:57:05 +0300 Subject: [PATCH 130/262] Merge pull request #20564 from AleksandrPanov:update_kalman_sample Update kalman sample * updated view and comments, fixed dims * updated view and comments, added statePost --- samples/cpp/kalman.cpp | 58 ++++++++++++++---------- samples/python/kalman.py | 97 ++++++++++++++++++++-------------------- 2 files changed, 83 insertions(+), 72 deletions(-) diff --git a/samples/cpp/kalman.cpp b/samples/cpp/kalman.cpp index 501a749124..daf0ba5a71 100644 --- a/samples/cpp/kalman.cpp +++ b/samples/cpp/kalman.cpp @@ -1,6 +1,6 @@ #include "opencv2/video/tracking.hpp" #include "opencv2/highgui.hpp" - +#include "opencv2/core/cvdef.h" #include using namespace cv; @@ -14,15 +14,19 @@ static void help() { printf( "\nExample of c calls to OpenCV's Kalman filter.\n" " Tracking of rotating point.\n" -" Rotation speed is constant.\n" +" Point moves in a circle and is characterized by a 1D state.\n" +" state_k+1 = state_k + speed + process_noise N(0, 1e-5)\n" +" The speed is constant.\n" " Both state and measurements vectors are 1D (a point angle),\n" -" Measurement is the real point angle + gaussian noise.\n" -" The real and the estimated points are connected with yellow line segment,\n" -" the real and the measured points are connected with red line segment.\n" +" Measurement is the real state + gaussian noise N(0, 1e-1).\n" +" The real and the measured points are connected with red line segment,\n" +" the real and the estimated points are connected with yellow line segment,\n" +" the real and the corrected estimated points are connected with green line segment.\n" " (if Kalman filter works correctly,\n" -" the yellow segment should be shorter than the red one).\n" +" the yellow segment should be shorter than the red one and\n" +" the green segment should be shorter than the yellow one)." "\n" -" Pressing any key (except ESC) will reset the tracking with a different speed.\n" +" Pressing any key (except ESC) will reset the tracking.\n" " Pressing ESC will stop the program.\n" ); } @@ -39,7 +43,9 @@ int main(int, char**) for(;;) { - randn( state, Scalar::all(0), Scalar::all(0.1) ); + img = Scalar::all(0); + state.at(0) = 0.0f; + state.at(1) = 2.f * (float)CV_PI / 6; KF.transitionMatrix = (Mat_(2, 2) << 1, 1, 0, 1); setIdentity(KF.measurementMatrix); @@ -60,36 +66,40 @@ int main(int, char**) double predictAngle = prediction.at(0); Point predictPt = calcPoint(center, R, predictAngle); - randn( measurement, Scalar::all(0), Scalar::all(KF.measurementNoiseCov.at(0))); - // generate measurement + randn( measurement, Scalar::all(0), Scalar::all(KF.measurementNoiseCov.at(0))); measurement += KF.measurementMatrix*state; double measAngle = measurement.at(0); Point measPt = calcPoint(center, R, measAngle); + // correct the state estimates based on measurements + // updates statePost & errorCovPost + KF.correct(measurement); + double improvedAngle = KF.statePost.at(0); + Point improvedPt = calcPoint(center, R, improvedAngle); + // plot points - #define drawCross( center, color, d ) \ - line( img, Point( center.x - d, center.y - d ), \ - Point( center.x + d, center.y + d ), color, 1, LINE_AA, 0); \ - line( img, Point( center.x + d, center.y - d ), \ - Point( center.x - d, center.y + d ), color, 1, LINE_AA, 0 ) + img = img * 0.2; + drawMarker(img, measPt, Scalar(0, 0, 255), cv::MARKER_SQUARE, 5, 2); + drawMarker(img, predictPt, Scalar(0, 255, 255), cv::MARKER_SQUARE, 5, 2); + drawMarker(img, improvedPt, Scalar(0, 255, 0), cv::MARKER_SQUARE, 5, 2); + drawMarker(img, statePt, Scalar(255, 255, 255), cv::MARKER_STAR, 10, 1); + // forecast one step + Mat test = Mat(KF.transitionMatrix*KF.statePost); + drawMarker(img, calcPoint(center, R, Mat(KF.transitionMatrix*KF.statePost).at(0)), + Scalar(255, 255, 0), cv::MARKER_SQUARE, 12, 1); - img = Scalar::all(0); - drawCross( statePt, Scalar(255,255,255), 3 ); - drawCross( measPt, Scalar(0,0,255), 3 ); - drawCross( predictPt, Scalar(0,255,0), 3 ); - line( img, statePt, measPt, Scalar(0,0,255), 3, LINE_AA, 0 ); - line( img, statePt, predictPt, Scalar(0,255,255), 3, LINE_AA, 0 ); + line( img, statePt, measPt, Scalar(0,0,255), 1, LINE_AA, 0 ); + line( img, statePt, predictPt, Scalar(0,255,255), 1, LINE_AA, 0 ); + line( img, statePt, improvedPt, Scalar(0,255,0), 1, LINE_AA, 0 ); - if(theRNG().uniform(0,4) != 0) - KF.correct(measurement); randn( processNoise, Scalar(0), Scalar::all(sqrt(KF.processNoiseCov.at(0, 0)))); state = KF.transitionMatrix*state + processNoise; imshow( "Kalman", img ); - code = (char)waitKey(100); + code = (char)waitKey(1000); if( code > 0 ) break; diff --git a/samples/python/kalman.py b/samples/python/kalman.py index 654e3de3da..cf152a8700 100755 --- a/samples/python/kalman.py +++ b/samples/python/kalman.py @@ -1,14 +1,18 @@ #!/usr/bin/env python """ Tracking of rotating point. - Rotation speed is constant. + Point moves in a circle and is characterized by a 1D state. + state_k+1 = state_k + speed + process_noise N(0, 1e-5) + The speed is constant. Both state and measurements vectors are 1D (a point angle), - Measurement is the real point angle + gaussian noise. - The real and the estimated points are connected with yellow line segment, - the real and the measured points are connected with red line segment. + Measurement is the real state + gaussian noise N(0, 1e-1). + The real and the measured points are connected with red line segment, + the real and the estimated points are connected with yellow line segment, + the real and the corrected estimated points are connected with green line segment. (if Kalman filter works correctly, - the yellow segment should be shorter than the red one). - Pressing any key (except ESC) will reset the tracking with a different speed. + the yellow segment should be shorter than the red one and + the green segment should be shorter than the yellow one). + Pressing any key (except ESC) will reset the tracking. Pressing ESC will stop the program. """ # Python 2/3 compatibility @@ -21,8 +25,7 @@ if PY3: import numpy as np import cv2 as cv -from math import cos, sin, sqrt -import numpy as np +from math import cos, sin, sqrt, pi def main(): img_height = 500 @@ -30,64 +33,62 @@ def main(): kalman = cv.KalmanFilter(2, 1, 0) code = long(-1) - - cv.namedWindow("Kalman") - + num_circle_steps = 12 while True: - state = 0.1 * np.random.randn(2, 1) - - kalman.transitionMatrix = np.array([[1., 1.], [0., 1.]]) - kalman.measurementMatrix = 1. * np.ones((1, 2)) - kalman.processNoiseCov = 1e-5 * np.eye(2) - kalman.measurementNoiseCov = 1e-1 * np.ones((1, 1)) - kalman.errorCovPost = 1. * np.ones((2, 2)) - kalman.statePost = 0.1 * np.random.randn(2, 1) + img = np.zeros((img_height, img_width, 3), np.uint8) + state = np.array([[0.0],[(2 * pi) / num_circle_steps]]) # start state + kalman.transitionMatrix = np.array([[1., 1.], [0., 1.]]) # F. input + kalman.measurementMatrix = 1. * np.eye(1, 2) # H. input + kalman.processNoiseCov = 1e-5 * np.eye(2) # Q. input + kalman.measurementNoiseCov = 1e-1 * np.ones((1, 1)) # R. input + kalman.errorCovPost = 1. * np.eye(2, 2) # P._k|k KF state var + kalman.statePost = 0.1 * np.random.randn(2, 1) # x^_k|k KF state var while True: def calc_point(angle): - return (np.around(img_width/2 + img_width/3*cos(angle), 0).astype(int), - np.around(img_height/2 - img_width/3*sin(angle), 1).astype(int)) - + return (np.around(img_width / 2. + img_width / 3.0 * cos(angle), 0).astype(int), + np.around(img_height / 2. - img_width / 3.0 * sin(angle), 1).astype(int)) + img = img * 1e-3 state_angle = state[0, 0] state_pt = calc_point(state_angle) - + # advance Kalman filter to next timestep + # updates statePre, statePost, errorCovPre, errorCovPost + # k-> k+1, x'(k) = A*x(k) + # P'(k) = temp1*At + Q prediction = kalman.predict() - predict_angle = prediction[0, 0] - predict_pt = calc_point(predict_angle) - - measurement = kalman.measurementNoiseCov * np.random.randn(1, 1) + predict_pt = calc_point(prediction[0, 0]) # equivalent to calc_point(kalman.statePre[0,0]) # generate measurement + measurement = kalman.measurementNoiseCov * np.random.randn(1, 1) measurement = np.dot(kalman.measurementMatrix, state) + measurement measurement_angle = measurement[0, 0] measurement_pt = calc_point(measurement_angle) - # plot points - def draw_cross(center, color, d): - cv.line(img, - (center[0] - d, center[1] - d), (center[0] + d, center[1] + d), - color, 1, cv.LINE_AA, 0) - cv.line(img, - (center[0] + d, center[1] - d), (center[0] - d, center[1] + d), - color, 1, cv.LINE_AA, 0) - - img = np.zeros((img_height, img_width, 3), np.uint8) - draw_cross(np.int32(state_pt), (255, 255, 255), 3) - draw_cross(np.int32(measurement_pt), (0, 0, 255), 3) - draw_cross(np.int32(predict_pt), (0, 255, 0), 3) - - cv.line(img, state_pt, measurement_pt, (0, 0, 255), 3, cv.LINE_AA, 0) - cv.line(img, state_pt, predict_pt, (0, 255, 255), 3, cv.LINE_AA, 0) - + # correct the state estimates based on measurements + # updates statePost & errorCovPost kalman.correct(measurement) + improved_pt = calc_point(kalman.statePost[0, 0]) - process_noise = sqrt(kalman.processNoiseCov[0,0]) * np.random.randn(2, 1) - state = np.dot(kalman.transitionMatrix, state) + process_noise + # plot points + cv.drawMarker(img, measurement_pt, (0, 0, 255), cv.MARKER_SQUARE, 5, 2) + cv.drawMarker(img, predict_pt, (0, 255, 255), cv.MARKER_SQUARE, 5, 2) + cv.drawMarker(img, improved_pt, (0, 255, 0), cv.MARKER_SQUARE, 5, 2) + cv.drawMarker(img, state_pt, (255, 255, 255), cv.MARKER_STAR, 10, 1) + # forecast one step + cv.drawMarker(img, calc_point(np.dot(kalman.transitionMatrix, kalman.statePost)[0, 0]), + (255, 255, 0), cv.MARKER_SQUARE, 12, 1) + + cv.line(img, state_pt, measurement_pt, (0, 0, 255), 1, cv.LINE_AA, 0) # red measurement error + cv.line(img, state_pt, predict_pt, (0, 255, 255), 1, cv.LINE_AA, 0) # yellow pre-meas error + cv.line(img, state_pt, improved_pt, (0, 255, 0), 1, cv.LINE_AA, 0) # green post-meas error + + # update the real process + process_noise = sqrt(kalman.processNoiseCov[0, 0]) * np.random.randn(2, 1) + state = np.dot(kalman.transitionMatrix, state) + process_noise # x_k+1 = F x_k + w_k cv.imshow("Kalman", img) - - code = cv.waitKey(100) + code = cv.waitKey(1000) if code != -1: break From 6801dd043de53030a6f8e36893580951a20dd842 Mon Sep 17 00:00:00 2001 From: rogday Date: Fri, 20 Aug 2021 17:43:47 +0300 Subject: [PATCH 131/262] Merge pull request #20494 from rogday:onnx_diagnostic_fix fix ONNXImporter diagnostic mode layer registration issue * fix layer registration, thread unsafe access and align the behavior of DNN_DIAGNOSTICS_RUN between onnx and tf importers * move skipModelInput * print all missing layers * address TF issue --- apps/model-diagnostics/model_diagnostics.cpp | 2 + .../include/opencv2/dnn/layer_reg.private.hpp | 8 +- .../include/opencv2/dnn/utils/debug_utils.hpp | 24 ++ modules/dnn/src/debug_utils.cpp | 91 ++++++ modules/dnn/src/dnn.cpp | 18 +- modules/dnn/src/dnn_common.hpp | 43 +++ modules/dnn/src/onnx/onnx_importer.cpp | 308 ++++++------------ modules/dnn/src/tensorflow/tf_importer.cpp | 103 +++--- modules/dnn/test/test_tf_importer.cpp | 3 + 9 files changed, 324 insertions(+), 276 deletions(-) create mode 100644 modules/dnn/include/opencv2/dnn/utils/debug_utils.hpp create mode 100644 modules/dnn/src/debug_utils.cpp diff --git a/apps/model-diagnostics/model_diagnostics.cpp b/apps/model-diagnostics/model_diagnostics.cpp index d3934577ae..6970c85071 100644 --- a/apps/model-diagnostics/model_diagnostics.cpp +++ b/apps/model-diagnostics/model_diagnostics.cpp @@ -4,6 +4,7 @@ USAGE: **************************************************/ #include #include +#include #include @@ -57,6 +58,7 @@ int main( int argc, const char** argv ) CV_Assert(!model.empty()); enableModelDiagnostics(true); + skipModelImport(true); redirectError(diagnosticsErrorCallback, NULL); Net ocvNet = readNet(model, config, frameworkId); diff --git a/modules/dnn/include/opencv2/dnn/layer_reg.private.hpp b/modules/dnn/include/opencv2/dnn/layer_reg.private.hpp index 46a58f09bc..e944644f8f 100644 --- a/modules/dnn/include/opencv2/dnn/layer_reg.private.hpp +++ b/modules/dnn/include/opencv2/dnn/layer_reg.private.hpp @@ -12,10 +12,16 @@ CV__DNN_INLINE_NS_BEGIN //! @addtogroup dnn //! @{ -//! Register layer types of DNN model. typedef std::map > LayerFactory_Impl; + +//! Register layer types of DNN model. +//! +//! @note In order to thread-safely access the factory, see getLayerFactoryMutex() function. LayerFactory_Impl& getLayerFactoryImpl(); +//! Get the mutex guarding @ref LayerFactory_Impl, see getLayerFactoryImpl() function. +Mutex& getLayerFactoryMutex(); + //! @} CV__DNN_INLINE_NS_END } diff --git a/modules/dnn/include/opencv2/dnn/utils/debug_utils.hpp b/modules/dnn/include/opencv2/dnn/utils/debug_utils.hpp new file mode 100644 index 0000000000..71dd3ab8d6 --- /dev/null +++ b/modules/dnn/include/opencv2/dnn/utils/debug_utils.hpp @@ -0,0 +1,24 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. + +#ifndef OPENCV_DNN_UTILS_DEBUG_UTILS_HPP +#define OPENCV_DNN_UTILS_DEBUG_UTILS_HPP + +#include "../dnn.hpp" + +namespace cv { namespace dnn { +CV__DNN_INLINE_NS_BEGIN + +/** + * @brief Skip model import after diagnostic run in readNet() functions. + * @param[in] skip Indicates whether to skip the import. + * + * This is an internal OpenCV function not intended for users. + */ +CV_EXPORTS void skipModelImport(bool skip); + +CV__DNN_INLINE_NS_END +}} // namespace + +#endif // OPENCV_DNN_UTILS_DEBUG_UTILS_HPP diff --git a/modules/dnn/src/debug_utils.cpp b/modules/dnn/src/debug_utils.cpp new file mode 100644 index 0000000000..d951205bd8 --- /dev/null +++ b/modules/dnn/src/debug_utils.cpp @@ -0,0 +1,91 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. + +#include "precomp.hpp" + +#include + +#include +#include +#include + +namespace cv { namespace dnn { +CV__DNN_INLINE_NS_BEGIN + +bool DNN_DIAGNOSTICS_RUN = false; +bool DNN_SKIP_REAL_IMPORT = false; + +void enableModelDiagnostics(bool isDiagnosticsMode) +{ + DNN_DIAGNOSTICS_RUN = isDiagnosticsMode; + + if (DNN_DIAGNOSTICS_RUN) + { + detail::NotImplemented::Register(); + } + else + { + detail::NotImplemented::unRegister(); + } +} + +void skipModelImport(bool skip) +{ + DNN_SKIP_REAL_IMPORT = skip; +} + +void detail::LayerHandler::addMissing(const std::string& name, const std::string& type) +{ + cv::AutoLock lock(getLayerFactoryMutex()); + auto& registeredLayers = getLayerFactoryImpl(); + + // If we didn't add it, but can create it, it's custom and not missing. + if (layers.find(type) == layers.end() && registeredLayers.find(type) != registeredLayers.end()) + { + return; + } + + layers[type].insert(name); +} + +bool detail::LayerHandler::contains(const std::string& type) const +{ + return layers.find(type) != layers.end(); +} + +void detail::LayerHandler::printMissing() +{ + if (layers.empty()) + { + return; + } + + std::stringstream ss; + ss << "DNN: Not supported types:\n"; + for (const auto& type_names : layers) + { + const auto& type = type_names.first; + ss << "Type='" << type << "', affected nodes:\n["; + for (const auto& name : type_names.second) + { + ss << "'" << name << "', "; + } + ss.seekp(-2, std::ios_base::end); + ss << "]\n"; + } + CV_LOG_ERROR(NULL, ss.str()); +} + +LayerParams detail::LayerHandler::getNotImplementedParams(const std::string& name, const std::string& op) +{ + LayerParams lp; + lp.name = name; + lp.type = "NotImplemented"; + lp.set("type", op); + + return lp; +} + +CV__DNN_INLINE_NS_END +}} // namespace diff --git a/modules/dnn/src/dnn.cpp b/modules/dnn/src/dnn.cpp index 492ad166d0..4e38b0374f 100644 --- a/modules/dnn/src/dnn.cpp +++ b/modules/dnn/src/dnn.cpp @@ -94,22 +94,6 @@ static bool DNN_CHECK_NAN_INF = utils::getConfigurationParameterBool("OPENCV_DNN static bool DNN_CHECK_NAN_INF_DUMP = utils::getConfigurationParameterBool("OPENCV_DNN_CHECK_NAN_INF_DUMP", false); static bool DNN_CHECK_NAN_INF_RAISE_ERROR = utils::getConfigurationParameterBool("OPENCV_DNN_CHECK_NAN_INF_RAISE_ERROR", false); -bool DNN_DIAGNOSTICS_RUN = false; - -void enableModelDiagnostics(bool isDiagnosticsMode) -{ - DNN_DIAGNOSTICS_RUN = isDiagnosticsMode; - - if (DNN_DIAGNOSTICS_RUN) - { - detail::NotImplemented::Register(); - } - else - { - detail::NotImplemented::unRegister(); - } -} - using std::vector; using std::map; using std::make_pair; @@ -5662,7 +5646,7 @@ bool Layer::updateMemoryShapes(const std::vector &inputs) } ////////////////////////////////////////////////////////////////////////// -static Mutex& getLayerFactoryMutex() +Mutex& getLayerFactoryMutex() { static Mutex* volatile instance = NULL; if (instance == NULL) diff --git a/modules/dnn/src/dnn_common.hpp b/modules/dnn/src/dnn_common.hpp index 591be88079..3c68322e09 100644 --- a/modules/dnn/src/dnn_common.hpp +++ b/modules/dnn/src/dnn_common.hpp @@ -5,6 +5,9 @@ #ifndef __OPENCV_DNN_COMMON_HPP__ #define __OPENCV_DNN_COMMON_HPP__ +#include +#include + #include namespace cv { namespace dnn { @@ -13,6 +16,9 @@ CV__DNN_INLINE_NS_BEGIN Mutex& getInitializationMutex(); void initializeLayerFactory(); +extern bool DNN_DIAGNOSTICS_RUN; +extern bool DNN_SKIP_REAL_IMPORT; + namespace detail { #define CALL_MEMBER_FN(object, ptrToMemFn) ((object).*(ptrToMemFn)) @@ -25,6 +31,43 @@ public: static void unRegister(); }; +template +Net readNet(Args&& ... args) +{ + Net net; + Importer importer(net, std::forward(args)...); + return net; +} + +template +Net readNetDiagnostic(Args&& ... args) +{ + Net maybeDebugNet = readNet(std::forward(args)...); + if (DNN_DIAGNOSTICS_RUN && !DNN_SKIP_REAL_IMPORT) + { + // if we just imported the net in diagnostic mode, disable it and import again + enableModelDiagnostics(false); + Net releaseNet = readNet(std::forward(args)...); + enableModelDiagnostics(true); + return releaseNet; + } + return maybeDebugNet; +} + +class LayerHandler +{ +public: + void addMissing(const std::string& name, const std::string& type); + bool contains(const std::string& type) const; + void printMissing(); + +protected: + LayerParams getNotImplementedParams(const std::string& name, const std::string& op); + +private: + std::unordered_map> layers; +}; + struct NetImplBase { const int networkId; // network global identifier diff --git a/modules/dnn/src/onnx/onnx_importer.cpp b/modules/dnn/src/onnx/onnx_importer.cpp index 3379ea3a0b..8fc179037c 100644 --- a/modules/dnn/src/onnx/onnx_importer.cpp +++ b/modules/dnn/src/onnx/onnx_importer.cpp @@ -41,6 +41,8 @@ CV__DNN_INLINE_NS_BEGIN extern bool DNN_DIAGNOSTICS_RUN; +class ONNXLayerHandler; + class ONNXImporter { opencv_onnx::ModelProto model_proto; @@ -61,60 +63,16 @@ class ONNXImporter void addConstant(const std::string& name, const Mat& blob); void addLayer(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); - static const std::set& getSupportedTypes(); public: - - ONNXImporter(Net& net, const char *onnxFile) - : dstNet(net), utilNet(), dispatch(buildDispatchMap()) - { - hasDynamicShapes = false; - CV_Assert(onnxFile); - CV_LOG_DEBUG(NULL, "DNN/ONNX: processing ONNX model from file: " << onnxFile); - - std::fstream input(onnxFile, std::ios::in | std::ios::binary); - if (!input) - { - CV_Error(Error::StsBadArg, cv::format("Can't read ONNX file: %s", onnxFile)); - } - - if (!model_proto.ParseFromIstream(&input)) - { - CV_Error(Error::StsUnsupportedFormat, cv::format("Failed to parse ONNX model: %s", onnxFile)); - } - - populateNet(); - } - - ONNXImporter(Net& net, const char* buffer, size_t sizeBuffer) - : dstNet(net), utilNet(), dispatch(buildDispatchMap()) - { - hasDynamicShapes = false; - CV_LOG_DEBUG(NULL, "DNN/ONNX: processing in-memory ONNX model (" << sizeBuffer << " bytes)"); - - struct _Buf : public std::streambuf - { - _Buf(const char* buffer, size_t sizeBuffer) - { - char* p = const_cast(buffer); - setg(p, p, p + sizeBuffer); - } - }; - - _Buf buf(buffer, sizeBuffer); - std::istream input(&buf); - - if (!model_proto.ParseFromIstream(&input)) - CV_Error(Error::StsUnsupportedFormat, "Failed to parse onnx model from in-memory byte array."); - - populateNet(); - } + ONNXImporter(Net& net, const char *onnxFile); + ONNXImporter(Net& net, const char* buffer, size_t sizeBuffer); void populateNet(); protected: + std::unique_ptr layerHandler; Net& dstNet; - Net utilNet; opencv_onnx::GraphProto graph_proto; std::string framework_name; @@ -131,9 +89,13 @@ protected: void handleNode(const opencv_onnx::NodeProto& node_proto); private: + friend class ONNXLayerHandler; typedef void (ONNXImporter::*ONNXImporterNodeParser)(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); typedef std::map DispatchMap; + const DispatchMap dispatch; + static const DispatchMap buildDispatchMap(); + void parseMaxPool (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); void parseAveragePool (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); void parseReduce (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); @@ -178,12 +140,84 @@ private: void parseSoftMax (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); void parseDetectionOutput (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); void parseCumSum (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); - void parseCustom (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); - const DispatchMap dispatch; - static const DispatchMap buildDispatchMap(); + void parseCustomLayer (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); }; +class ONNXLayerHandler : public detail::LayerHandler +{ +public: + explicit ONNXLayerHandler(ONNXImporter* importer_); + + void fillRegistry(const opencv_onnx::GraphProto& net); + +protected: + ONNXImporter* importer; +}; + +ONNXLayerHandler::ONNXLayerHandler(ONNXImporter* importer_) : importer(importer_){} + +void ONNXLayerHandler::fillRegistry(const opencv_onnx::GraphProto &net) +{ + int layersSize = net.node_size(); + for (int li = 0; li < layersSize; li++) { + const opencv_onnx::NodeProto &node_proto = net.node(li); + const std::string& name = node_proto.output(0); + const std::string& type = node_proto.op_type(); + if (importer->dispatch.find(type) == importer->dispatch.end()) + { + addMissing(name, type); + } + } + printMissing(); +} + +ONNXImporter::ONNXImporter(Net& net, const char *onnxFile) + : layerHandler(DNN_DIAGNOSTICS_RUN ? new ONNXLayerHandler(this) : nullptr), + dstNet(net), dispatch(buildDispatchMap()) +{ + hasDynamicShapes = false; + CV_Assert(onnxFile); + CV_LOG_DEBUG(NULL, "DNN/ONNX: processing ONNX model from file: " << onnxFile); + + std::fstream input(onnxFile, std::ios::in | std::ios::binary); + if (!input) + { + CV_Error(Error::StsBadArg, cv::format("Can't read ONNX file: %s", onnxFile)); + } + + if (!model_proto.ParseFromIstream(&input)) + { + CV_Error(Error::StsUnsupportedFormat, cv::format("Failed to parse ONNX model: %s", onnxFile)); + } + + populateNet(); +} + +ONNXImporter::ONNXImporter(Net& net, const char* buffer, size_t sizeBuffer) + : layerHandler(DNN_DIAGNOSTICS_RUN ? new ONNXLayerHandler(this) : nullptr), dstNet(net), dispatch(buildDispatchMap()) +{ + hasDynamicShapes = false; + CV_LOG_DEBUG(NULL, "DNN/ONNX: processing in-memory ONNX model (" << sizeBuffer << " bytes)"); + + struct _Buf : public std::streambuf + { + _Buf(const char* buffer, size_t sizeBuffer) + { + char* p = const_cast(buffer); + setg(p, p, p + sizeBuffer); + } + }; + + _Buf buf(buffer, sizeBuffer); + std::istream input(&buf); + + if (!model_proto.ParseFromIstream(&input)) + CV_Error(Error::StsUnsupportedFormat, "Failed to parse onnx model from in-memory byte array."); + + populateNet(); +} + inline void replaceLayerParam(LayerParams& layerParams, const String& oldKey, const String& newKey) { if (layerParams.has(oldKey)) { @@ -422,11 +456,7 @@ Mat ONNXImporter::getBlob(const std::string& input_name) void ONNXImporter::addLayer(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) { - int id; - if (DNN_DIAGNOSTICS_RUN) - id = utilNet.addLayer(layerParams.name, layerParams.type, layerParams); - else - id = dstNet.addLayer(layerParams.name, layerParams.type, layerParams); + int id = dstNet.addLayer(layerParams.name, layerParams.type, layerParams); for (int i = 0; i < node_proto.output_size(); ++i) { layer_id.insert(std::make_pair(node_proto.output(i), LayerInfo(id, i))); @@ -439,10 +469,7 @@ void ONNXImporter::addLayer(LayerParams& layerParams, const std::string& input_name = node_proto.input(j); IterLayerId_t layerId = layer_id.find(input_name); if (layerId != layer_id.end()) { - if (DNN_DIAGNOSTICS_RUN) - utilNet.connect(layerId->second.layerId, layerId->second.outputId, id, inpNum); - else - dstNet.connect(layerId->second.layerId, layerId->second.outputId, id, inpNum); + dstNet.connect(layerId->second.layerId, layerId->second.outputId, id, inpNum); ++inpNum; // Collect input shapes. IterShape_t shapeIt = outShapes.find(input_name); @@ -451,11 +478,7 @@ void ONNXImporter::addLayer(LayerParams& layerParams, } } // Compute shape of output blob for this layer. - Ptr layer; - if (DNN_DIAGNOSTICS_RUN) - layer = utilNet.getLayer(id); - else - layer = dstNet.getLayer(id); // FIXIT: avoid instantiation of layers during the import stage + Ptr layer = dstNet.getLayer(id); // FIXIT: avoid instantiation of layers during the import stage layer->getMemoryShapes(layerInpShapes, 0, layerOutShapes, layerInternalShapes); for (int i = 0; i < node_proto.output_size() && i < (int)layerOutShapes.size(); ++i) { @@ -532,35 +555,11 @@ void ONNXImporter::populateNet() layer_id.insert(std::make_pair(name, LayerInfo(0, netInputs.size() - 1))); } } - utilNet.setInputsNames(netInputs); dstNet.setInputsNames(netInputs); if (DNN_DIAGNOSTICS_RUN) { - auto &supportedTypes = getSupportedTypes(); - for (int li = 0; li < layersSize; li++) { - const opencv_onnx::NodeProto &node_proto = graph_proto.node(li); - std::string name = node_proto.output(0); - std::string layer_type = node_proto.op_type(); - auto registered = supportedTypes.find(layer_type); - if (registered == supportedTypes.end()) { - CV_LOG_ERROR(NULL, "DNN/ONNX: NOTE: Potential problem with creating node " << name<< " with type " << layer_type << ".\n Type " - << layer_type << " IS NOT SUPPORTED!\n" - ); - } - } - auto oldConstBlobs = constBlobs; - auto oldOutShapes = outShapes; - auto oldLayerId = layer_id; CV_LOG_INFO(NULL, "DNN/ONNX: start diagnostic run!"); - for (int li = 0; li < layersSize; li++) { - const opencv_onnx::NodeProto &node_proto = graph_proto.node(li); - handleNode(node_proto); - } - CV_LOG_INFO(NULL, "DNN/ONNX: diagnostic run completed!"); - constBlobs = oldConstBlobs; - outShapes = oldOutShapes; - layer_id = oldLayerId; - enableModelDiagnostics(false); + layerHandler->fillRegistry(graph_proto); } for(int li = 0; li < layersSize; li++) @@ -569,83 +568,7 @@ void ONNXImporter::populateNet() handleNode(node_proto); } - CV_LOG_DEBUG(NULL, "DNN/ONNX: import completed!"); -} - -const std::set& ONNXImporter::getSupportedTypes() -{ - static const std::set layerTypes = { - "MaxPool", - "AveragePool", - "GlobalAveragePool", - "GlobalMaxPool", - "ReduceMean", - "ReduceSum", - "ReduceMax", - "Slice", - "Split", - "Add", - "Sum", - "Sub", - "Pow", - "Max", - "Neg", - "Constant", - "LSTM", - "GRU", - "ImageScaler", - "Clip", - "LeakyRelu", - "Relu", - "Elu", - "Tanh", - "PRelu", - "LRN", - "InstanceNormalization", - "BatchNormalization", - "Gemm", - "MatMul", - "Mul", - "Div", - "Conv", - "ConvTranspose", - "Transpose", - "Squeeze", - "Flatten", - "Unsqueeze", - "Expand", - "Reshape", - "Pad", - "Shape", - "Cast", - "ConstantOfShape", - "ConstantFill", - "Gather", - "Concat", - "Resize", - "Upsample", - "SoftMax", - "Softmax", - "LogSoftmax", - "DetectionOutput", - "Interp", - "CropAndResize", - "ROIPooling", - "PSROIPooling", - "ChannelsPReLU", - "Sigmoid", - "Swish", - "Mish", - "AbsVal", - "BNLL", - "MaxUnpool", - "Dropout", - "Identity", - "Crop", - "Normalize", - "CumSum" - }; - return layerTypes; + CV_LOG_DEBUG(NULL, (DNN_DIAGNOSTICS_RUN ? "DNN/ONNX: diagnostic run completed!" : "DNN/ONNX: import completed!")); } void ONNXImporter::handleNode(const opencv_onnx::NodeProto& node_proto) @@ -673,7 +596,7 @@ void ONNXImporter::handleNode(const opencv_onnx::NodeProto& node_proto) } else { - parseCustom(layerParams, node_proto); + parseCustomLayer(layerParams, node_proto); } } catch (const cv::Exception& e) @@ -683,6 +606,7 @@ void ONNXImporter::handleNode(const opencv_onnx::NodeProto& node_proto) CV_LOG_ERROR(NULL, "DNN/ONNX: Potential problem during processing node with " << node_proto.input_size() << " inputs and " << node_proto.output_size() << " outputs: " << cv::format("[%s]:(%s)", layer_type.c_str(), name.c_str()) << "\n" << e.msg ); + cv::AutoLock lock(getLayerFactoryMutex()); auto registeredLayers = getLayerFactoryImpl(); if (registeredLayers.find(layerParams.type) != registeredLayers.end()) { @@ -1068,11 +992,7 @@ void ONNXImporter::parseBias(LayerParams& layerParams, const opencv_onnx::NodePr constParams.name = layerParams.name + "/const"; constParams.type = "Const"; constParams.blobs.push_back((isSub ? -1 : 1) * blob); - int id; - if (DNN_DIAGNOSTICS_RUN) - id = utilNet.addLayer(constParams.name, constParams.type, constParams); - else - id = dstNet.addLayer(constParams.name, constParams.type, constParams); + int id = dstNet.addLayer(constParams.name, constParams.type, constParams); layer_id.insert(std::make_pair(constParams.name, LayerInfo(id, 0))); outShapes[constParams.name] = shape(blob); @@ -1117,19 +1037,12 @@ void ONNXImporter::parseBias(LayerParams& layerParams, const opencv_onnx::NodePr powerParams.type = "Power"; powerParams.set("scale", -1); - int id; //Create Power layer - if (DNN_DIAGNOSTICS_RUN) - id = utilNet.addLayer(powerParams.name, powerParams.type, powerParams); - else - id = dstNet.addLayer(powerParams.name, powerParams.type, powerParams); + int id = dstNet.addLayer(powerParams.name, powerParams.type, powerParams); //Connect to input IterLayerId_t layerId = layer_id.find(node_proto.input(1)); CV_Assert(layerId != layer_id.end()); - if (DNN_DIAGNOSTICS_RUN) - utilNet.connect(layerId->second.layerId, layerId->second.outputId, id, 0); - else - dstNet.connect(layerId->second.layerId, layerId->second.outputId, id, 0); + dstNet.connect(layerId->second.layerId, layerId->second.outputId, id, 0); //Add shape layer_id.insert(std::make_pair(powerParams.name, LayerInfo(id, 0))); outShapes[powerParams.name] = outShapes[node_proto.input(1)]; @@ -1404,18 +1317,11 @@ void ONNXImporter::parseInstanceNormalization(LayerParams& layerParams, const op layerParams.erase("epsilon"); //Create MVN layer - int id; - if (DNN_DIAGNOSTICS_RUN) - id = utilNet.addLayer(mvnParams.name, mvnParams.type, mvnParams); - else - id = dstNet.addLayer(mvnParams.name, mvnParams.type, mvnParams); + int id = dstNet.addLayer(mvnParams.name, mvnParams.type, mvnParams); //Connect to input IterLayerId_t layerId = layer_id.find(node_proto.input(0)); CV_Assert(layerId != layer_id.end()); - if (DNN_DIAGNOSTICS_RUN) - utilNet.connect(layerId->second.layerId, layerId->second.outputId, id, 0); - else - dstNet.connect(layerId->second.layerId, layerId->second.outputId, id, 0); + dstNet.connect(layerId->second.layerId, layerId->second.outputId, id, 0); //Add shape layer_id.insert(std::make_pair(mvnParams.name, LayerInfo(id, 0))); outShapes[mvnParams.name] = outShapes[node_proto.input(0)]; @@ -1621,19 +1527,12 @@ void ONNXImporter::parseMul(LayerParams& layerParams, const opencv_onnx::NodePro powerParams.type = "Power"; powerParams.set("power", -1); - int id; //Create Power layer - if (DNN_DIAGNOSTICS_RUN) - id = utilNet.addLayer(powerParams.name, powerParams.type, powerParams); - else - id = dstNet.addLayer(powerParams.name, powerParams.type, powerParams); + int id = dstNet.addLayer(powerParams.name, powerParams.type, powerParams); //Connect to input IterLayerId_t layerId = layer_id.find(node_proto.input(1)); CV_Assert(layerId != layer_id.end()); - if (DNN_DIAGNOSTICS_RUN) - utilNet.connect(layerId->second.layerId, layerId->second.outputId, id, 0); - else - dstNet.connect(layerId->second.layerId, layerId->second.outputId, id, 0); + dstNet.connect(layerId->second.layerId, layerId->second.outputId, id, 0); //Add shape layer_id.insert(std::make_pair(powerParams.name, LayerInfo(id, 0))); outShapes[powerParams.name] = outShapes[node_proto.input(1)]; @@ -2418,7 +2317,7 @@ void ONNXImporter::parseCumSum(LayerParams& layerParams, const opencv_onnx::Node addLayer(layerParams, node_proto); } -void ONNXImporter::parseCustom(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +void ONNXImporter::parseCustomLayer(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) { for (int j = 0; j < node_proto.input_size(); j++) { if (layer_id.find(node_proto.input(j)) == layer_id.end()) @@ -2476,23 +2375,18 @@ const ONNXImporter::DispatchMap ONNXImporter::buildDispatchMap() dispatch["SoftMax"] = dispatch["LogSoftmax"] = &ONNXImporter::parseSoftMax; dispatch["DetectionOutput"] = &ONNXImporter::parseDetectionOutput; dispatch["CumSum"] = &ONNXImporter::parseCumSum; - dispatch["Custom"] = &ONNXImporter::parseCustom; return dispatch; } Net readNetFromONNX(const String& onnxFile) { - Net net; - ONNXImporter onnxImporter(net, onnxFile.c_str()); - return net; + return detail::readNetDiagnostic(onnxFile.c_str()); } Net readNetFromONNX(const char* buffer, size_t sizeBuffer) { - Net net; - ONNXImporter onnxImporter(net, buffer, sizeBuffer); - return net; + return detail::readNetDiagnostic(buffer, sizeBuffer); } Net readNetFromONNX(const std::vector& buffer) diff --git a/modules/dnn/src/tensorflow/tf_importer.cpp b/modules/dnn/src/tensorflow/tf_importer.cpp index de7ec3dfcc..f87988d0a1 100644 --- a/modules/dnn/src/tensorflow/tf_importer.cpp +++ b/modules/dnn/src/tensorflow/tf_importer.cpp @@ -507,7 +507,7 @@ void ExcludeLayer(tensorflow::GraphDef& net, const int layer_index, const int in net.mutable_node()->DeleteSubrange(layer_index, 1); } -class LayerHandler; +class TFLayerHandler; class TFImporter { @@ -516,8 +516,7 @@ public: TFImporter(Net& net, const char *dataModel, size_t lenModel, const char *dataConfig = NULL, size_t lenConfig = 0); protected: - std::unique_ptr layerHandler; - std::unique_ptr utilNet; + std::unique_ptr layerHandler; Net& dstNet; void populateNet(); @@ -559,7 +558,7 @@ private: void addPermuteLayer(const int* order, const std::string& permName, Pin& inpId); void setPadding(LayerParams &layerParams, const tensorflow::NodeDef &layer, std::string& inputName, float value = 0.); - friend class LayerHandler; + friend class TFLayerHandler; typedef void (TFImporter::*TFImporterNodeParser)(tensorflow::GraphDef&, const tensorflow::NodeDef&, LayerParams&); typedef std::map DispatchMap; @@ -625,18 +624,17 @@ void TFImporter::setPadding(LayerParams &layerParams, const tensorflow::NodeDef layerParams.set("pad_mode", "VALID"); } -class LayerHandler +class TFLayerHandler : public detail::LayerHandler { public: - LayerHandler(TFImporter* importer_); - ~LayerHandler() = default; + explicit TFLayerHandler(TFImporter* importer_); - bool handleMissing(const opencv_tensorflow::NodeDef& layer); - void handleFailed(const opencv_tensorflow::NodeDef& layer); + void fillRegistry(const tensorflow::GraphDef& net); + bool handleMissing(const tensorflow::NodeDef& layer); + void handleFailed(const tensorflow::NodeDef& layer); -private: +protected: TFImporter* importer; - std::set layers; }; const TFImporter::DispatchMap TFImporter::buildDispatchMap() @@ -2471,9 +2469,8 @@ void TFImporter::parseCustomLayer(tensorflow::GraphDef& net, const tensorflow::N } TFImporter::TFImporter(Net& net, const char *model, const char *config) - : layerHandler(DNN_DIAGNOSTICS_RUN ? new LayerHandler(this) : nullptr), - utilNet(DNN_DIAGNOSTICS_RUN ? new Net : nullptr), - dstNet(DNN_DIAGNOSTICS_RUN ? *utilNet : net), dispatch(buildDispatchMap()) + : layerHandler(DNN_DIAGNOSTICS_RUN ? new TFLayerHandler(this) : nullptr), + dstNet(net), dispatch(buildDispatchMap()) { if (model && model[0]) { @@ -2494,9 +2491,8 @@ TFImporter::TFImporter( const char *dataModel, size_t lenModel, const char *dataConfig, size_t lenConfig ) - : layerHandler(DNN_DIAGNOSTICS_RUN ? new LayerHandler(this) : nullptr), - utilNet(DNN_DIAGNOSTICS_RUN ? new Net : nullptr), - dstNet(DNN_DIAGNOSTICS_RUN ? *utilNet : net), dispatch(buildDispatchMap()) + : layerHandler(DNN_DIAGNOSTICS_RUN ? new TFLayerHandler(this) : nullptr), + dstNet(net), dispatch(buildDispatchMap()) { if (dataModel != NULL && lenModel > 0) { @@ -2855,6 +2851,11 @@ void TFImporter::populateNet() addConstNodes(netBin, value_id, layers_to_ignore); addConstNodes(netTxt, value_id, layers_to_ignore); + if (DNN_DIAGNOSTICS_RUN) { + CV_LOG_INFO(NULL, "DNN/TF: start diagnostic run!"); + layerHandler->fillRegistry(net); + } + for (int li = 0; li < layersSize; li++) { const tensorflow::NodeDef& layer = net.node(li); @@ -2873,7 +2874,7 @@ void TFImporter::populateNet() CV_Assert(!netInputsNames[i].empty()); } dstNet.setInputsNames(netInputsNames); - CV_LOG_DEBUG(NULL, "DNN/TF: ===================== Import completed ====================="); + CV_LOG_DEBUG(NULL, (DNN_DIAGNOSTICS_RUN? "DNN/TF: diagnostic run completed!" : "DNN/TF: import completed!")); } void TFImporter::addPermuteLayer(const int* order, const std::string& permName, Pin& inpId) @@ -2933,60 +2934,60 @@ void TFImporter::parseNode(const tensorflow::NodeDef& layer) } } -LayerHandler::LayerHandler(TFImporter* importer_) : importer(importer_) {} +TFLayerHandler::TFLayerHandler(TFImporter* importer_) : importer(importer_) {} -void LayerHandler::handleFailed(const opencv_tensorflow::NodeDef& layer) +void TFLayerHandler::fillRegistry(const tensorflow::GraphDef& net) { - LayerParams lp; - lp.name = layer.name(); - lp.type = "NotImplemented"; - lp.set("type", layer.op()); + for (int li = 0; li < net.node_size(); li++) { + const tensorflow::NodeDef& layer = net.node(li); + + const std::string& name = layer.name(); + const std::string& type = layer.op(); + if (importer->dispatch.find(type) == importer->dispatch.end()) + { + addMissing(name, type); + } + } + printMissing(); +}; + +bool TFLayerHandler::handleMissing(const tensorflow::NodeDef& layer) +{ + bool unsupported = contains(layer.op()); + + if (unsupported) + { + handleFailed(layer); + } + + return unsupported; +} + +void TFLayerHandler::handleFailed(const tensorflow::NodeDef& layer) +{ + LayerParams lp = getNotImplementedParams(layer.name(), layer.op()); // the layer will be created or its params and type will be replaced - int id = importer->dstNet.addLayer(lp.name, "NotImplemented", lp); + int id = importer->dstNet.addLayer(lp.name, lp.type, lp); if (id != -1) // internal layer failure before the call to addLayer() { importer->layer_id[lp.name] = id; } } -bool LayerHandler::handleMissing(const opencv_tensorflow::NodeDef& layer) -{ - LayerParams lp; - // If we didn't add it, but can create it, it's custom and not missing. - if (layers.find(layer.op()) == layers.end() && LayerFactory::createLayerInstance(layer.op(), lp)) - { - return false; - } - - if (layers.insert(layer.op()).second) - { - CV_LOG_ERROR(NULL, "DNN/TF: Node='" << layer.name() << "' of type='"<< layer.op() - << "' is not supported. This error won't be displayed again."); - } - - handleFailed(layer); - - return true; -} - } // namespace #endif //HAVE_PROTOBUF Net readNetFromTensorflow(const String &model, const String &config) { - Net net; - TFImporter importer(net, model.c_str(), config.c_str()); - return net; + return detail::readNetDiagnostic(model.c_str(), config.c_str()); } Net readNetFromTensorflow(const char* bufferModel, size_t lenModel, const char* bufferConfig, size_t lenConfig) { - Net net; - TFImporter importer(net, bufferModel, lenModel, bufferConfig, lenConfig); - return net; + return detail::readNetDiagnostic(bufferModel, lenModel, bufferConfig, lenConfig); } Net readNetFromTensorflow(const std::vector& bufferModel, const std::vector& bufferConfig) diff --git a/modules/dnn/test/test_tf_importer.cpp b/modules/dnn/test/test_tf_importer.cpp index 3f33f16774..3d53ced0a4 100644 --- a/modules/dnn/test/test_tf_importer.cpp +++ b/modules/dnn/test/test_tf_importer.cpp @@ -13,6 +13,7 @@ Test for Tensorflow models loading #include "npy_blob.hpp" #include // CV_DNN_REGISTER_LAYER_CLASS +#include namespace opencv_test { @@ -605,11 +606,13 @@ public: Test_TensorFlow_diagnostics() { enableModelDiagnostics(true); + skipModelImport(true); } ~Test_TensorFlow_diagnostics() { enableModelDiagnostics(false); + skipModelImport(false); } void runFailingTensorFlowNet(const std::string& prefix, bool hasText = false) From f28e4b86fbc6bf361c2e6c7a36fd96c552a2aab7 Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Sat, 21 Aug 2021 16:04:13 +0000 Subject: [PATCH 132/262] dnn(ocl): fix top initialization in verifyResult --- modules/dnn/src/ocl4dnn/src/ocl4dnn_conv_spatial.cpp | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/modules/dnn/src/ocl4dnn/src/ocl4dnn_conv_spatial.cpp b/modules/dnn/src/ocl4dnn/src/ocl4dnn_conv_spatial.cpp index fd98919343..ef7c380c1b 100644 --- a/modules/dnn/src/ocl4dnn/src/ocl4dnn_conv_spatial.cpp +++ b/modules/dnn/src/ocl4dnn/src/ocl4dnn_conv_spatial.cpp @@ -1257,8 +1257,11 @@ bool OCL4DNNConvSpatial::verifyResult(const UMat &bottom, else if (config->tested) return false; - int32_t sz[4] = {numImages, num_output_, output_h_, output_w_}; - top.zeros(4, sz, (use_half_) ? CV_16SC1 : CV_32FC1); + //int32_t sz[4] = {numImages, num_output_, output_h_, output_w_}; + CV_CheckEQ(top.total(), (size_t)numImages * num_output_ * output_h_ * output_w_, ""); + CV_CheckTypeEQ(top.type(), (use_half_) ? CV_16SC1 : CV_32FC1, ""); + top.setTo(Scalar::all(0)); + bool saved_tuned = tuned_; tuned_ = false; convolve(bottom, top, weight, bias, numImages, config); From aa5c4945d602452737c32168021847ee1593d914 Mon Sep 17 00:00:00 2001 From: Anatoliy Talamanov Date: Mon, 23 Aug 2021 16:52:49 +0300 Subject: [PATCH 133/262] Fix GExecutor WriteBackExec --- modules/gapi/src/executor/gexecutor.cpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/modules/gapi/src/executor/gexecutor.cpp b/modules/gapi/src/executor/gexecutor.cpp index 9b51e70d5d..2ca675c7df 100644 --- a/modules/gapi/src/executor/gexecutor.cpp +++ b/modules/gapi/src/executor/gexecutor.cpp @@ -157,10 +157,14 @@ void writeBackExec(const Mag& mag, const RcDesc &rc, GRunArgP &g_arg) // FIXME: // Rework, find a better way to check if there should be // a real copy (add a pass to StreamingBackend?) + // NB: In case RMat adapter not equal to "RMatAdapter" need to + // copy data back to the host as well. + // FIXME: Rename "RMatAdapter" to "OpenCVAdapter". auto& out_mat = *util::get(g_arg); const auto& rmat = mag.template slot().at(rc.id); auto* adapter = rmat.get(); - if (adapter != nullptr && out_mat.data != adapter->data()) { + if ((adapter != nullptr && out_mat.data != adapter->data()) || + (adapter == nullptr)) { auto view = rmat.access(RMat::Access::R); asMat(view).copyTo(out_mat); } From 350562919cdda295fafad9d94a47479441736d28 Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Mon, 23 Aug 2021 18:10:46 +0000 Subject: [PATCH 134/262] highgui(win32): avoid using of stalled iterator --- modules/highgui/src/window_w32.cpp | 42 ++++++++++++++++-------------- 1 file changed, 23 insertions(+), 19 deletions(-) diff --git a/modules/highgui/src/window_w32.cpp b/modules/highgui/src/window_w32.cpp index 716af1094c..43a19baf89 100644 --- a/modules/highgui/src/window_w32.cpp +++ b/modules/highgui/src/window_w32.cpp @@ -278,6 +278,21 @@ std::shared_ptr icvFindWindowByName(const char* name) return icvFindWindowByName(std::string(name)); } +// Mutex must be locked +static +std::shared_ptr icvFindWindowByHandle(HWND hwnd) +{ + auto& g_windows = getWindowsList(); + for (auto it = g_windows.begin(); it != g_windows.end(); ++it) + { + auto window = *it; + if (!window) + continue; + if (window->hwnd == hwnd || window->frame == hwnd) + return window; + } + return std::shared_ptr(); +} static LRESULT CALLBACK HighGUIProc(HWND hwnd, UINT uMsg, WPARAM wParam, LPARAM lParam); @@ -2187,22 +2202,15 @@ static void showSaveDialog(CvWindow& window) */ static bool handleMessage(MSG& message, int& keyCode) { - // whether we have to call translate and dispatch yet - // otherwise the message was handled specifically - bool is_processed = false; - - AutoLock lock(getWindowMutex()); - auto& g_windows = getWindowsList(); - for (auto it = g_windows.begin(); it != g_windows.end() && !is_processed; ++it) + std::shared_ptr window_; + { + AutoLock lock(getWindowMutex()); + window_ = icvFindWindowByHandle(message.hwnd); + } + if (window_) { - auto window_ = *it; - if (!window_) - continue; CvWindow& window = *window_; - if (!(window.hwnd == message.hwnd || window.frame == message.hwnd)) - continue; - is_processed = true; switch (message.message) { case WM_DESTROY: @@ -2215,7 +2223,6 @@ static bool handleMessage(MSG& message, int& keyCode) case WM_SYSKEYDOWN: if (message.wParam == VK_F10) { - is_processed = true; keyCode = (int)(message.wParam << 16); return true; } @@ -2245,7 +2252,6 @@ static bool handleMessage(MSG& message, int& keyCode) message.wParam == VK_PRIOR || message.wParam == VK_NEXT) { DispatchMessage(&message); - is_processed = true; keyCode = (int)(message.wParam << 16); return true; } @@ -2254,18 +2260,16 @@ static bool handleMessage(MSG& message, int& keyCode) default: DispatchMessage(&message); - is_processed = true; break; } } - - if (!is_processed) + else { TranslateMessage(&message); DispatchMessage(&message); } - return false; // no value to return, keep processing + return false; // no keyCode to return, keep processing } /* From 2f31763335ee4e53cb1a66c9298976ec00b5c160 Mon Sep 17 00:00:00 2001 From: HAN Liutong Date: Tue, 24 Aug 2021 11:42:19 +0800 Subject: [PATCH 135/262] fix v_reduce_sum --- modules/core/include/opencv2/core/hal/intrin_rvv.hpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/modules/core/include/opencv2/core/hal/intrin_rvv.hpp b/modules/core/include/opencv2/core/hal/intrin_rvv.hpp index 5b33193781..55e0d88f59 100644 --- a/modules/core/include/opencv2/core/hal/intrin_rvv.hpp +++ b/modules/core/include/opencv2/core/hal/intrin_rvv.hpp @@ -1399,8 +1399,8 @@ OPENCV_HAL_IMPL_RVV_REDUCE_SUM(v_uint16x8, v_uint32x4, vuint32m1_t, unsigned, u1 OPENCV_HAL_IMPL_RVV_REDUCE_SUM(v_int16x8, v_int32x4, vint32m1_t, int, i16, i32, 8, wredsum) OPENCV_HAL_IMPL_RVV_REDUCE_SUM(v_uint32x4, v_uint64x2, vuint64m1_t, unsigned, u32, u64, 4, wredsumu) OPENCV_HAL_IMPL_RVV_REDUCE_SUM(v_int32x4, v_int64x2, vint64m1_t, int, i32, i64, 4, wredsum) -OPENCV_HAL_IMPL_RVV_REDUCE_SUM(v_uint64x2, v_uint64x2, vuint64m1_t, uint64, u64, u64, 4, redsum) -OPENCV_HAL_IMPL_RVV_REDUCE_SUM(v_int64x2, v_int64x2, vint64m1_t, int64, i64, i64, 4, redsum) +OPENCV_HAL_IMPL_RVV_REDUCE_SUM(v_uint64x2, v_uint64x2, vuint64m1_t, uint64, u64, u64, 2, redsum) +OPENCV_HAL_IMPL_RVV_REDUCE_SUM(v_int64x2, v_int64x2, vint64m1_t, int64, i64, i64, 2, redsum) #define OPENCV_HAL_IMPL_RVV_REDUCE_SUM_FP(_Tpvec, _wTpvec, _nwTpvec, scalartype, suffix, wsuffix, vl, red) \ inline scalartype v_reduce_sum(const _Tpvec& a) \ @@ -1411,9 +1411,9 @@ inline scalartype v_reduce_sum(const _Tpvec& a) \ return (scalartype)(_wTpvec(res).get0()); \ } -OPENCV_HAL_IMPL_RVV_REDUCE_SUM_FP(v_float32x4, v_float32x4, vfloat32m1_t, float, f32, f32, 8, fredsum) +OPENCV_HAL_IMPL_RVV_REDUCE_SUM_FP(v_float32x4, v_float32x4, vfloat32m1_t, float, f32, f32, 4, fredsum) #if CV_SIMD128_64F -OPENCV_HAL_IMPL_RVV_REDUCE_SUM_FP(v_float64x2, v_float64x2, vfloat64m1_t, double, f64, f64, 4, fredsum) +OPENCV_HAL_IMPL_RVV_REDUCE_SUM_FP(v_float64x2, v_float64x2, vfloat64m1_t, double, f64, f64, 2, fredsum) #endif From fdaa6ff9e35c5d035b208582db1cc78de5c46d8a Mon Sep 17 00:00:00 2001 From: Nicolai Behmann Date: Tue, 24 Aug 2021 11:31:54 +0200 Subject: [PATCH 136/262] Merge pull request #20475 from nibeh:patch-1 * Added exposure and gain props, maximized pixel clk * removed pixel clock maximization pixel clock maximization is not suitable for all use cases, so I removed it from PR. --- modules/videoio/src/cap_ueye.cpp | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/modules/videoio/src/cap_ueye.cpp b/modules/videoio/src/cap_ueye.cpp index eadba15546..efc2cb990e 100644 --- a/modules/videoio/src/cap_ueye.cpp +++ b/modules/videoio/src/cap_ueye.cpp @@ -171,6 +171,13 @@ double VideoCapture_uEye::getProperty(int property_id) const case CAP_PROP_FPS: value = fps; break; + case CAP_PROP_EXPOSURE: + ASSERT_UEYE(is_Exposure(cam_id, IS_EXPOSURE_CMD_GET_EXPOSURE, (void*)&value, sizeof(value))); + break; + case CAP_PROP_GAIN: + auto gain = is_SetHWGainFactor(cam_id, IS_GET_MASTER_GAIN_FACTOR, 100); + value = static_cast(gain)/100.0; + break; } return value; } @@ -201,6 +208,12 @@ bool VideoCapture_uEye::setProperty(int property_id, double value) break; ASSERT_UEYE(is_SetFrameRate(cam_id, value, &fps)); break; + case CAP_PROP_EXPOSURE: + ASSERT_UEYE(is_Exposure(cam_id, IS_EXPOSURE_CMD_SET_EXPOSURE, (void*)&value, sizeof(value))); + break; + case CAP_PROP_GAIN: + is_SetHWGainFactor(cam_id, IS_SET_MASTER_GAIN_FACTOR, static_cast(value)); + break; } if(set_format) { From 5ad6ff239bed8d9bfbd4e176a49ea1213989d5a6 Mon Sep 17 00:00:00 2001 From: Anatoliy Talamanov Date: Tue, 24 Aug 2021 12:37:50 +0300 Subject: [PATCH 137/262] Merge pull request #20555 from TolyaTalamanov:at/fix-compileStreaming-bug [G-API] Extend compileStreaming to support different overloads * Make different overloads * Order python compileStreaming overloads * Fix compileStreaming bug * Replace gin -> descr_of * Set error message * Fix review comments * Use macros for pyopencv_to GMetaArgs * Use GAPI_PROP_RW * Not split Prims python stuff --- .../include/opencv2/gapi/gcomputation.hpp | 10 +-- .../gapi/include/opencv2/gapi/gstreaming.hpp | 4 +- .../gapi/misc/python/package/gapi/__init__.py | 2 + modules/gapi/misc/python/pyopencv_gapi.hpp | 70 +++++++++++-------- modules/gapi/misc/python/shadow_gapi.hpp | 5 +- .../misc/python/test/test_gapi_streaming.py | 44 ++++++++++++ 6 files changed, 98 insertions(+), 37 deletions(-) diff --git a/modules/gapi/include/opencv2/gapi/gcomputation.hpp b/modules/gapi/include/opencv2/gapi/gcomputation.hpp index a3566fb495..7e07a587e1 100644 --- a/modules/gapi/include/opencv2/gapi/gcomputation.hpp +++ b/modules/gapi/include/opencv2/gapi/gcomputation.hpp @@ -437,11 +437,7 @@ public: * * @sa @ref gapi_compile_args */ - GStreamingCompiled compileStreaming(GMetaArgs &&in_metas, GCompileArgs &&args = {}); - - /// @private -- Exclude this function from OpenCV documentation - GAPI_WRAP GStreamingCompiled compileStreaming(const cv::detail::ExtractMetaCallback &callback, - GCompileArgs &&args = {}); + GAPI_WRAP GStreamingCompiled compileStreaming(GMetaArgs &&in_metas, GCompileArgs &&args = {}); /** * @brief Compile the computation for streaming mode. @@ -464,6 +460,10 @@ public: */ GAPI_WRAP GStreamingCompiled compileStreaming(GCompileArgs &&args = {}); + /// @private -- Exclude this function from OpenCV documentation + GAPI_WRAP GStreamingCompiled compileStreaming(const cv::detail::ExtractMetaCallback &callback, + GCompileArgs &&args = {}); + // 2. Direct metadata version /** * @overload diff --git a/modules/gapi/include/opencv2/gapi/gstreaming.hpp b/modules/gapi/include/opencv2/gapi/gstreaming.hpp index 50abe69f87..0e7d2a847f 100644 --- a/modules/gapi/include/opencv2/gapi/gstreaming.hpp +++ b/modules/gapi/include/opencv2/gapi/gstreaming.hpp @@ -396,9 +396,11 @@ namespace streaming { * In the streaming mode the pipeline steps are connected with queues * and this compile argument controls every queue's size. */ -struct GAPI_EXPORTS queue_capacity +struct GAPI_EXPORTS_W_SIMPLE queue_capacity { + GAPI_WRAP explicit queue_capacity(size_t cap = 1) : capacity(cap) { }; + GAPI_PROP_RW size_t capacity; }; /** @} */ diff --git a/modules/gapi/misc/python/package/gapi/__init__.py b/modules/gapi/misc/python/package/gapi/__init__.py index dc874f0b0c..b1326712fc 100644 --- a/modules/gapi/misc/python/package/gapi/__init__.py +++ b/modules/gapi/misc/python/package/gapi/__init__.py @@ -295,3 +295,5 @@ cv.gapi.wip.draw.Line = cv.gapi_wip_draw_Line cv.gapi.wip.draw.Mosaic = cv.gapi_wip_draw_Mosaic cv.gapi.wip.draw.Image = cv.gapi_wip_draw_Image cv.gapi.wip.draw.Poly = cv.gapi_wip_draw_Poly + +cv.gapi.streaming.queue_capacity = cv.gapi_streaming_queue_capacity diff --git a/modules/gapi/misc/python/pyopencv_gapi.hpp b/modules/gapi/misc/python/pyopencv_gapi.hpp index d378a91b5f..64f8277740 100644 --- a/modules/gapi/misc/python/pyopencv_gapi.hpp +++ b/modules/gapi/misc/python/pyopencv_gapi.hpp @@ -11,13 +11,14 @@ #include // NB: Python wrapper replaces :: with _ for classes -using gapi_GKernelPackage = cv::gapi::GKernelPackage; -using gapi_GNetPackage = cv::gapi::GNetPackage; -using gapi_ie_PyParams = cv::gapi::ie::PyParams; -using gapi_wip_IStreamSource_Ptr = cv::Ptr; -using detail_ExtractArgsCallback = cv::detail::ExtractArgsCallback; -using detail_ExtractMetaCallback = cv::detail::ExtractMetaCallback; -using vector_GNetParam = std::vector; +using gapi_GKernelPackage = cv::gapi::GKernelPackage; +using gapi_GNetPackage = cv::gapi::GNetPackage; +using gapi_ie_PyParams = cv::gapi::ie::PyParams; +using gapi_wip_IStreamSource_Ptr = cv::Ptr; +using detail_ExtractArgsCallback = cv::detail::ExtractArgsCallback; +using detail_ExtractMetaCallback = cv::detail::ExtractMetaCallback; +using vector_GNetParam = std::vector; +using gapi_streaming_queue_capacity = cv::gapi::streaming::queue_capacity; // NB: Python wrapper generate T_U for T // This behavior is only observed for inputs @@ -159,7 +160,7 @@ PyObject* pyopencv_from(const cv::gapi::wip::draw::Prims& value) } template<> -bool pyopencv_to(PyObject* obj, cv::gapi::wip::draw::Prim& value, const ArgInfo& info) +bool pyopencv_to(PyObject* obj, cv::gapi::wip::draw::Prim& value, const ArgInfo&) { #define TRY_EXTRACT(Prim) \ if (PyObject_TypeCheck(obj, reinterpret_cast(pyopencv_gapi_wip_draw_##Prim##_TypePtr))) \ @@ -175,6 +176,7 @@ bool pyopencv_to(PyObject* obj, cv::gapi::wip::draw::Prim& value, const ArgInfo& TRY_EXTRACT(Mosaic) TRY_EXTRACT(Image) TRY_EXTRACT(Poly) +#undef TRY_EXTRACT failmsg("Unsupported primitive type"); return false; @@ -186,6 +188,34 @@ bool pyopencv_to(PyObject* obj, cv::gapi::wip::draw::Prims& value, const ArgInfo return pyopencv_to_generic_vec(obj, value, info); } +template <> +bool pyopencv_to(PyObject* obj, cv::GMetaArg& value, const ArgInfo&) +{ +#define TRY_EXTRACT(Meta) \ + if (PyObject_TypeCheck(obj, \ + reinterpret_cast(pyopencv_##Meta##_TypePtr))) \ + { \ + value = reinterpret_cast(obj)->v; \ + return true; \ + } \ + + TRY_EXTRACT(GMatDesc) + TRY_EXTRACT(GScalarDesc) + TRY_EXTRACT(GArrayDesc) + TRY_EXTRACT(GOpaqueDesc) +#undef TRY_EXTRACT + + failmsg("Unsupported cv::GMetaArg type"); + return false; +} + +template <> +bool pyopencv_to(PyObject* obj, cv::GMetaArgs& value, const ArgInfo& info) +{ + return pyopencv_to_generic_vec(obj, value, info); +} + + template<> PyObject* pyopencv_from(const cv::GArg& value) { @@ -707,30 +737,12 @@ static cv::GRunArgs run_py_kernel(cv::detail::PyObjectHolder kernel, static GMetaArg get_meta_arg(PyObject* obj) { - if (PyObject_TypeCheck(obj, - reinterpret_cast(pyopencv_GMatDesc_TypePtr))) - { - return cv::GMetaArg{reinterpret_cast(obj)->v}; - } - else if (PyObject_TypeCheck(obj, - reinterpret_cast(pyopencv_GScalarDesc_TypePtr))) - { - return cv::GMetaArg{reinterpret_cast(obj)->v}; - } - else if (PyObject_TypeCheck(obj, - reinterpret_cast(pyopencv_GArrayDesc_TypePtr))) - { - return cv::GMetaArg{reinterpret_cast(obj)->v}; - } - else if (PyObject_TypeCheck(obj, - reinterpret_cast(pyopencv_GOpaqueDesc_TypePtr))) - { - return cv::GMetaArg{reinterpret_cast(obj)->v}; - } - else + cv::GMetaArg arg; + if (!pyopencv_to(obj, arg, ArgInfo("arg", false))) { util::throw_error(std::logic_error("Unsupported output meta type")); } + return arg; } static cv::GMetaArgs get_meta_args(PyObject* tuple) diff --git a/modules/gapi/misc/python/shadow_gapi.hpp b/modules/gapi/misc/python/shadow_gapi.hpp index 0b489dde0f..33a0e0fde1 100644 --- a/modules/gapi/misc/python/shadow_gapi.hpp +++ b/modules/gapi/misc/python/shadow_gapi.hpp @@ -5,8 +5,9 @@ namespace cv { struct GAPI_EXPORTS_W_SIMPLE GCompileArg { - GAPI_WRAP GCompileArg(gapi::GKernelPackage pkg); - GAPI_WRAP GCompileArg(gapi::GNetPackage pkg); + GAPI_WRAP GCompileArg(gapi::GKernelPackage arg); + GAPI_WRAP GCompileArg(gapi::GNetPackage arg); + GAPI_WRAP GCompileArg(gapi::streaming::queue_capacity arg); }; class GAPI_EXPORTS_W_SIMPLE GInferInputs diff --git a/modules/gapi/misc/python/test/test_gapi_streaming.py b/modules/gapi/misc/python/test/test_gapi_streaming.py index 7ede1b5cf3..d7914c5157 100644 --- a/modules/gapi/misc/python/test/test_gapi_streaming.py +++ b/modules/gapi/misc/python/test/test_gapi_streaming.py @@ -261,6 +261,7 @@ try: if curr_frame_number == max_num_frames: break + def test_desync(self): path = self.find_file('cv/video/768x576.avi', [os.environ['OPENCV_TEST_DATA_PATH']]) @@ -307,6 +308,49 @@ try: self.assertLess(0, none_counter) + def test_compile_streaming_empty(self): + g_in = cv.GMat() + comp = cv.GComputation(g_in, cv.gapi.medianBlur(g_in, 3)) + comp.compileStreaming() + + + def test_compile_streaming_args(self): + g_in = cv.GMat() + comp = cv.GComputation(g_in, cv.gapi.medianBlur(g_in, 3)) + comp.compileStreaming(cv.gapi.compile_args(cv.gapi.streaming.queue_capacity(1))) + + + def test_compile_streaming_descr_of(self): + g_in = cv.GMat() + comp = cv.GComputation(g_in, cv.gapi.medianBlur(g_in, 3)) + img = np.zeros((3,300,300), dtype=np.float32) + comp.compileStreaming(cv.gapi.descr_of(img)) + + + def test_compile_streaming_descr_of_and_args(self): + g_in = cv.GMat() + comp = cv.GComputation(g_in, cv.gapi.medianBlur(g_in, 3)) + img = np.zeros((3,300,300), dtype=np.float32) + comp.compileStreaming(cv.gapi.descr_of(img), + cv.gapi.compile_args(cv.gapi.streaming.queue_capacity(1))) + + + def test_compile_streaming_meta(self): + g_in = cv.GMat() + comp = cv.GComputation(g_in, cv.gapi.medianBlur(g_in, 3)) + img = np.zeros((3,300,300), dtype=np.float32) + comp.compileStreaming([cv.GMatDesc(cv.CV_8U, 3, (300, 300))]) + + + def test_compile_streaming_meta_and_args(self): + g_in = cv.GMat() + comp = cv.GComputation(g_in, cv.gapi.medianBlur(g_in, 3)) + img = np.zeros((3,300,300), dtype=np.float32) + comp.compileStreaming([cv.GMatDesc(cv.CV_8U, 3, (300, 300))], + cv.gapi.compile_args(cv.gapi.streaming.queue_capacity(1))) + + + except unittest.SkipTest as e: message = str(e) From 65ef82a94674950d3861a172d32a2cce53f015cc Mon Sep 17 00:00:00 2001 From: Sergey Ivanov Date: Tue, 24 Aug 2021 15:41:57 +0300 Subject: [PATCH 138/262] Merge pull request #20570 from sivanov-work:vpl_source_data_adapter G-API: oneVPL (simplification) Add data adapter & Cfg params * Add cfg_param & data_provider * Fix compilation after rebase * Apply some comments * Apply default ctor outside class definition comment * Apply cfg param in source * Fix compilation: add virtual dtor * Move cfg_params in regular gapi src list * Fix compilation: add export.hpp * Add errno.h * Add errno.h * Apply namespace comment * Add several Doxygen & rename cfg_param * Fix build * Update Doxygen docs for onevpl * Fix typo --- modules/gapi/CMakeLists.txt | 7 +- .../gapi/streaming/onevpl/cfg_params.hpp | 88 +++++++++++ .../onevpl/data_provider_interface.hpp | 74 ++++++++++ .../gapi/streaming/onevpl/onevpl_source.hpp | 44 ------ .../opencv2/gapi/streaming/onevpl/source.hpp | 64 ++++++++ .../gapi/samples/onevpl_infer_single_roi.cpp | 51 ++++++- .../gapi/src/streaming/onevpl/cfg_params.cpp | 137 ++++++++++++++++++ .../data_provider_interface_exception.cpp | 22 +++ .../streaming/onevpl/file_data_provider.cpp | 47 ++++++ .../streaming/onevpl/file_data_provider.hpp | 33 +++++ .../src/streaming/onevpl/onevpl_source.cpp | 48 ------ modules/gapi/src/streaming/onevpl/source.cpp | 58 ++++++++ ...onevpl_source_priv.cpp => source_priv.cpp} | 22 +-- ...onevpl_source_priv.hpp => source_priv.hpp} | 13 +- .../test/streaming/gapi_streaming_tests.cpp | 70 +++++++++ 15 files changed, 665 insertions(+), 113 deletions(-) create mode 100644 modules/gapi/include/opencv2/gapi/streaming/onevpl/cfg_params.hpp create mode 100644 modules/gapi/include/opencv2/gapi/streaming/onevpl/data_provider_interface.hpp delete mode 100644 modules/gapi/include/opencv2/gapi/streaming/onevpl/onevpl_source.hpp create mode 100644 modules/gapi/include/opencv2/gapi/streaming/onevpl/source.hpp create mode 100644 modules/gapi/src/streaming/onevpl/cfg_params.cpp create mode 100644 modules/gapi/src/streaming/onevpl/data_provider_interface_exception.cpp create mode 100644 modules/gapi/src/streaming/onevpl/file_data_provider.cpp create mode 100644 modules/gapi/src/streaming/onevpl/file_data_provider.hpp delete mode 100644 modules/gapi/src/streaming/onevpl/onevpl_source.cpp create mode 100644 modules/gapi/src/streaming/onevpl/source.cpp rename modules/gapi/src/streaming/onevpl/{onevpl_source_priv.cpp => source_priv.cpp} (66%) rename modules/gapi/src/streaming/onevpl/{onevpl_source_priv.hpp => source_priv.hpp} (80%) diff --git a/modules/gapi/CMakeLists.txt b/modules/gapi/CMakeLists.txt index 69c0aaaae8..08dee2f9af 100644 --- a/modules/gapi/CMakeLists.txt +++ b/modules/gapi/CMakeLists.txt @@ -164,8 +164,11 @@ set(gapi_srcs src/backends/python/gpythonbackend.cpp # Streaming source - src/streaming/onevpl/onevpl_source.cpp - src/streaming/onevpl/onevpl_source_priv.cpp + src/streaming/onevpl/source.cpp + src/streaming/onevpl/source_priv.cpp + src/streaming/onevpl/file_data_provider.cpp + src/streaming/onevpl/cfg_params.cpp + src/streaming/onevpl/data_provider_interface_exception.cpp # Utils (ITT tracing) src/utils/itt.cpp diff --git a/modules/gapi/include/opencv2/gapi/streaming/onevpl/cfg_params.hpp b/modules/gapi/include/opencv2/gapi/streaming/onevpl/cfg_params.hpp new file mode 100644 index 0000000000..9dc5ead7d7 --- /dev/null +++ b/modules/gapi/include/opencv2/gapi/streaming/onevpl/cfg_params.hpp @@ -0,0 +1,88 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2021 Intel Corporation + +#ifndef OPENCV_GAPI_STREAMING_ONEVPL_CFG_PARAMS_HPP +#define OPENCV_GAPI_STREAMING_ONEVPL_CFG_PARAMS_HPP + +#include +#include +#include + +#include +#include + +namespace cv { +namespace gapi { +namespace wip { +namespace onevpl { + +/** + * @brief Public class is using for creation of onevpl::GSource instances. + * + * Class members availaible through methods @ref CfgParam::get_name() and @ref CfgParam::get_value() are used by + * onevpl::GSource inner logic to create or find oneVPL particular implementation + * (software/hardware, specific API version and etc.). + * + * @note Because oneVPL may provide several implementations which are satisfying with multiple (or single one) @ref CfgParam + * criteria therefore it is possible to configure `preferred` parameters. This kind of CfgParams are created + * using `is_major = false` argument in @ref CfgParam::create method and are not used by creating oneVPL particular implementations. + * Instead they fill out a "score table" to select preferrable implementation from available list. Implementation are satisfying + * with most of these optional params would be chosen. + * If no one optional CfgParam params were present then first of available oneVPL implementation would be applied. + * Please get on https://spec.oneapi.io/versions/latest/elements/oneVPL/source/API_ref/VPL_disp_api_func.html?highlight=mfxcreateconfig#mfxsetconfigfilterproperty + * for using OneVPL configuration. In this schema `mfxU8 *name` represents @ref CfgParam::get_name() and + * `mfxVariant value` is @ref CfgParam::get_value() + */ +struct GAPI_EXPORTS CfgParam { + using name_t = std::string; + using value_t = cv::util::variant; + + /** + * Create onevp::GSource configuration parameter. + * + *@param name name of parameter. + *@param value value of parameter. + *@param is_major TRUE if parameter MUST be provided by OneVPL inner implementation, FALSE for optional (for resolve multiple available implementations). + * + */ + template + static CfgParam create(const std::string& name, ValueType&& value, bool is_major = true) { + CfgParam param(name, CfgParam::value_t(std::forward(value)), is_major); + return param; + } + + struct Priv; + + const name_t& get_name() const; + const value_t& get_value() const; + bool is_major() const; + bool operator==(const CfgParam& rhs) const; + bool operator< (const CfgParam& rhs) const; + bool operator!=(const CfgParam& rhs) const; + + CfgParam& operator=(const CfgParam& src); + CfgParam& operator=(CfgParam&& src); + CfgParam(const CfgParam& src); + CfgParam(CfgParam&& src); + ~CfgParam(); +private: + CfgParam(const std::string& param_name, value_t&& param_value, bool is_major_param); + std::shared_ptr m_priv; +}; + +} //namespace onevpl +} // namespace wip +} // namespace gapi +} // namespace cv + +#endif // OPENCV_GAPI_STREAMING_ONEVPL_CFG_PARAMS_HPP diff --git a/modules/gapi/include/opencv2/gapi/streaming/onevpl/data_provider_interface.hpp b/modules/gapi/include/opencv2/gapi/streaming/onevpl/data_provider_interface.hpp new file mode 100644 index 0000000000..2c299520f7 --- /dev/null +++ b/modules/gapi/include/opencv2/gapi/streaming/onevpl/data_provider_interface.hpp @@ -0,0 +1,74 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2021 Intel Corporation + +#ifndef GAPI_STREAMING_ONEVPL_ONEVPL_DATA_PROVIDER_INTERFACE_HPP +#define GAPI_STREAMING_ONEVPL_ONEVPL_DATA_PROVIDER_INTERFACE_HPP +#include +#include + +#include // GAPI_EXPORTS + +namespace cv { +namespace gapi { +namespace wip { +namespace onevpl { + +struct GAPI_EXPORTS DataProviderException : public std::exception { + virtual ~DataProviderException() {}; +}; + +struct GAPI_EXPORTS DataProviderSystemErrorException : public DataProviderException { + DataProviderSystemErrorException(int error_code, const std::string& desription = std::string()); + virtual ~DataProviderSystemErrorException(); + virtual const char* what() const noexcept override; + +private: + std::string reason; +}; + +/** + * @brief Public interface allows to customize extraction of video stream data + * used by onevpl::GSource instead of reading stream from file (by default). + * + * Interface implementation constructor MUST provide consistency and creates fully operable object. + * If error happened implementation MUST throw `DataProviderException` kind exceptions + * + * @note Interface implementation MUST manage stream and other constructed resources by itself to avoid any kind of leak. + * For simple interface implementation example please see `StreamDataProvider` in `tests/streaming/gapi_streaming_tests.cpp` + */ +struct GAPI_EXPORTS IDataProvider { + using Ptr = std::shared_ptr; + + virtual ~IDataProvider() {}; + + /** + * The function is used by onevpl::GSource to extract binary data stream from @ref IDataProvider + * implementation. + * + * It MUST throw `DataProviderException` kind exceptions in fail cases. + * It MUST return 0 in EOF which considered as not-fail case. + * + * @param out_data_bytes_size the available capacity of out_data buffer. + * @param out_data the output consumer buffer with capacity out_data_bytes_size. + * @return fetched bytes count. + */ + virtual size_t fetch_data(size_t out_data_bytes_size, void* out_data) = 0; + + /** + * The function is used by onevpl::GSource to check more binary data availability. + * + * It MUST return TRUE in case of EOF and NO_THROW exceptions. + * + * @return boolean value which detects end of stream + */ + virtual bool empty() const = 0; +}; +} // namespace onevpl +} // namespace wip +} // namespace gapi +} // namespace cv + +#endif // GAPI_STREAMING_ONEVPL_ONEVPL_DATA_PROVIDER_INTERFACE_HPP diff --git a/modules/gapi/include/opencv2/gapi/streaming/onevpl/onevpl_source.hpp b/modules/gapi/include/opencv2/gapi/streaming/onevpl/onevpl_source.hpp deleted file mode 100644 index fec8c73dff..0000000000 --- a/modules/gapi/include/opencv2/gapi/streaming/onevpl/onevpl_source.hpp +++ /dev/null @@ -1,44 +0,0 @@ -// This file is part of OpenCV project. -// It is subject to the license terms in the LICENSE file found in the top-level directory -// of this distribution and at http://opencv.org/license.html. -// -// Copyright (C) 2021 Intel Corporation - -#ifndef OPENCV_GAPI_STREAMING_ONEVPL_ONEVPL_SOURCE_HPP -#define OPENCV_GAPI_STREAMING_ONEVPL_ONEVPL_SOURCE_HPP - -#include -#include -#include - -namespace cv { -namespace gapi { -namespace wip { - -class GAPI_EXPORTS OneVPLSource : public IStreamSource -{ -public: - struct Priv; - - explicit OneVPLSource(const std::string& filePath); - ~OneVPLSource() override; - - bool pull(cv::gapi::wip::Data& data) override; - GMetaArg descr_of() const override; - -private: - explicit OneVPLSource(std::unique_ptr&& impl); - std::unique_ptr m_priv; -}; - -template -GAPI_EXPORTS_W cv::Ptr inline make_vpl_src(const std::string& filePath, Args&&... args) -{ - return make_src(filePath, std::forward(args)...); -} - -} // namespace wip -} // namespace gapi -} // namespace cv - -#endif // OPENCV_GAPI_STREAMING_ONEVPL_ONEVPL_SOURCE_HPP diff --git a/modules/gapi/include/opencv2/gapi/streaming/onevpl/source.hpp b/modules/gapi/include/opencv2/gapi/streaming/onevpl/source.hpp new file mode 100644 index 0000000000..b7bf4f2ffb --- /dev/null +++ b/modules/gapi/include/opencv2/gapi/streaming/onevpl/source.hpp @@ -0,0 +1,64 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2021 Intel Corporation + +#ifndef OPENCV_GAPI_STREAMING_ONEVPL_ONEVPL_SOURCE_HPP +#define OPENCV_GAPI_STREAMING_ONEVPL_ONEVPL_SOURCE_HPP + +#include +#include +#include +#include +#include + +namespace cv { +namespace gapi { +namespace wip { +namespace onevpl { +using CfgParams = std::vector; + +/** + * @brief G-API streaming source based on OneVPL implementation. + * + * This class implements IStreamSource interface. + * Its constructor takes source file path (in usual way) or @ref onevpl::IDataProvider + * interface implementation (for not file-based sources). It also allows to pass-through + * oneVPL configuration parameters by using several @ref onevpl::CfgParam. + * + * @note stream sources are passed to G-API via shared pointers, so + * please gapi::make_onevpl_src<> to create objects and ptr() to pass a + * GSource to cv::gin(). + */ +class GAPI_EXPORTS GSource : public IStreamSource +{ +public: + struct Priv; + + GSource(const std::string& filePath, + const CfgParams& cfg_params = CfgParams{}); + GSource(std::shared_ptr source, + const CfgParams& cfg_params = CfgParams{}); + ~GSource() override; + + bool pull(cv::gapi::wip::Data& data) override; + GMetaArg descr_of() const override; + +private: + explicit GSource(std::unique_ptr&& impl); + std::unique_ptr m_priv; +}; +} // namespace onevpl + +template +GAPI_EXPORTS_W cv::Ptr inline make_onevpl_src(Args&&... args) +{ + return make_src(std::forward(args)...); +} + +} // namespace wip +} // namespace gapi +} // namespace cv + +#endif // OPENCV_GAPI_STREAMING_ONEVPL_ONEVPL_SOURCE_HPP diff --git a/modules/gapi/samples/onevpl_infer_single_roi.cpp b/modules/gapi/samples/onevpl_infer_single_roi.cpp index 8a7efafabf..9a3abdf392 100644 --- a/modules/gapi/samples/onevpl_infer_single_roi.cpp +++ b/modules/gapi/samples/onevpl_infer_single_roi.cpp @@ -9,7 +9,7 @@ #include #include #include -#include +#include #include // CommandLineParser const std::string about = @@ -19,7 +19,8 @@ const std::string keys = "{ input | | Path to the input demultiplexed video file }" "{ output | | Path to the output RAW video file. Use .avi extension }" "{ facem | face-detection-adas-0001.xml | Path to OpenVINO IE face detection model (.xml) }" - "{ faced | CPU | Target device for face detection model (e.g. CPU, GPU, VPU, ...) }"; + "{ cfg_params | :;: | Semicolon separated list of oneVPL mfxVariants which is used for configuring source (see `MFXSetConfigFilterProperty` by https://spec.oneapi.io/versions/latest/elements/oneVPL/source/index.html) }"; + namespace { std::string get_weights_path(const std::string &model_path) { @@ -155,6 +156,10 @@ GAPI_OCV_KERNEL(OCVBBoxes, BBoxes) { } // namespace custom +namespace cfg { +typename cv::gapi::wip::onevpl::CfgParam create_from_string(const std::string &line); +} + int main(int argc, char *argv[]) { cv::CommandLineParser cmd(argc, argv, keys); @@ -178,10 +183,22 @@ int main(int argc, char *argv[]) { } } + // get oneVPL cfg params from cmd + std::stringstream params_list(cmd.get("cfg_params")); + std::vector source_cfgs; + try { + std::string line; + while (std::getline(params_list, line, ';')) { + source_cfgs.push_back(cfg::create_from_string(line)); + } + } catch (const std::exception& ex) { + std::cerr << "Invalid cfg parameter: " << ex.what() << std::endl; + return -1; + } + auto face_net = cv::gapi::ie::Params { face_model_path, // path to topology IR - get_weights_path(face_model_path), // path to weights - cmd.get("faced"), // device specifier + get_weights_path(face_model_path) // path to weights }; auto kernels = cv::gapi::kernels < custom::OCVLocateROI @@ -192,7 +209,7 @@ int main(int argc, char *argv[]) { // Create source cv::Ptr cap; try { - cap = cv::gapi::wip::make_vpl_src(file_path); + cap = cv::gapi::wip::make_onevpl_src(file_path, source_cfgs); std::cout << "oneVPL source desription: " << cap->descr_of() << std::endl; } catch (const std::exception& ex) { std::cerr << "Cannot create source: " << ex.what() << std::endl; @@ -213,7 +230,7 @@ int main(int argc, char *argv[]) { cv::GStreamingCompiled pipeline; try { - pipeline = cv::GComputation(cv::GIn(in), cv::GOut(out)) + pipeline = cv::GComputation(cv::GIn(in), cv::GOut(out)) .compileStreaming(cv::compile_args(kernels, networks)); } catch (const std::exception& ex) { std::cerr << "Exception occured during pipeline construction: " << ex.what() << std::endl; @@ -252,3 +269,25 @@ int main(int argc, char *argv[]) { return 0; } + + +namespace cfg { +typename cv::gapi::wip::onevpl::CfgParam create_from_string(const std::string &line) { + using namespace cv::gapi::wip; + + if (line.empty()) { + throw std::runtime_error("Cannot parse CfgParam from emply line"); + } + + std::string::size_type name_endline_pos = line.find(':'); + if (name_endline_pos == std::string::npos) { + throw std::runtime_error("Cannot parse CfgParam from: " + line + + "\nExpected separator \":\""); + } + + std::string name = line.substr(0, name_endline_pos); + std::string value = line.substr(name_endline_pos + 1); + + return cv::gapi::wip::onevpl::CfgParam::create(name, value); +} +} diff --git a/modules/gapi/src/streaming/onevpl/cfg_params.cpp b/modules/gapi/src/streaming/onevpl/cfg_params.cpp new file mode 100644 index 0000000000..456725508f --- /dev/null +++ b/modules/gapi/src/streaming/onevpl/cfg_params.cpp @@ -0,0 +1,137 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2021 Intel Corporation + +#include + +#include + +namespace cv { +namespace gapi { +namespace wip { +namespace onevpl { + +namespace util { +struct variant_comparator : cv::util::static_visitor { + variant_comparator(const CfgParam::value_t& rhs_value) : + rhs(rhs_value) {} + + template + bool visit(const ValueType& lhs) const { + return lhs < cv::util::get(rhs); + } +private: + const CfgParam::value_t& rhs; +}; +} // namespace util + +struct CfgParam::Priv { + Priv(const std::string& param_name, CfgParam::value_t&& param_value, bool is_major_param) : + name(param_name), value(std::forward(param_value)), major_flag(is_major_param) { + } + + const CfgParam::name_t& get_name_impl() const { + return name; + } + + const CfgParam::value_t& get_value_impl() const { + return value; + } + + bool is_major_impl() const { + return major_flag; + } + + // comparison implementation + bool operator< (const Priv& rhs) const { + // implement default pair comparison + if (get_name_impl() < rhs.get_name_impl()) { + return true; + } else if (get_name_impl() > rhs.get_name_impl()) { + return false; + } + + //TODO implement operator < for cv::util::variant + const CfgParam::value_t& lvar = get_value_impl(); + const CfgParam::value_t& rvar = rhs.get_value_impl(); + if (lvar.index() < rvar.index()) { + return true; + } else if (lvar.index() > rvar.index()) { + return false; + } + + util::variant_comparator comp(rvar); + return cv::util::visit(comp, lvar); + } + + bool operator==(const Priv& rhs) const { + return (get_name_impl() == rhs.get_name_impl()) + && (get_value_impl() == rhs.get_value_impl()); + } + + bool operator!=(const Priv& rhs) const { + return !(*this == rhs); + } + + CfgParam::name_t name; + CfgParam::value_t value; + bool major_flag; +}; + +CfgParam::CfgParam (const std::string& param_name, value_t&& param_value, bool is_major_param) : + m_priv(new Priv(param_name, std::move(param_value), is_major_param)) { +} + +CfgParam::~CfgParam() = default; + +CfgParam& CfgParam::operator=(const CfgParam& src) { + if (this != &src) { + m_priv = src.m_priv; + } + return *this; +} + +CfgParam& CfgParam::operator=(CfgParam&& src) { + if (this != &src) { + m_priv = std::move(src.m_priv); + } + return *this; +} + +CfgParam::CfgParam(const CfgParam& src) : + m_priv(src.m_priv) { +} + +CfgParam::CfgParam(CfgParam&& src) : + m_priv(std::move(src.m_priv)) { +} + +const CfgParam::name_t& CfgParam::get_name() const { + return m_priv->get_name_impl(); +} + +const CfgParam::value_t& CfgParam::get_value() const { + return m_priv->get_value_impl(); +} + +bool CfgParam::is_major() const { + return m_priv->is_major_impl(); +} + +bool CfgParam::operator< (const CfgParam& rhs) const { + return *m_priv < *rhs.m_priv; +} + +bool CfgParam::operator==(const CfgParam& rhs) const { + return *m_priv == *rhs.m_priv; +} + +bool CfgParam::operator!=(const CfgParam& rhs) const { + return *m_priv != *rhs.m_priv; +} +} // namespace onevpl +} // namespace wip +} // namespace gapi +} // namespace cv diff --git a/modules/gapi/src/streaming/onevpl/data_provider_interface_exception.cpp b/modules/gapi/src/streaming/onevpl/data_provider_interface_exception.cpp new file mode 100644 index 0000000000..feaae7a727 --- /dev/null +++ b/modules/gapi/src/streaming/onevpl/data_provider_interface_exception.cpp @@ -0,0 +1,22 @@ +#include +#include + +#include + +namespace cv { +namespace gapi { +namespace wip { +namespace onevpl { +DataProviderSystemErrorException::DataProviderSystemErrorException(int error_code, const std::string& desription) { + reason = desription + ", error: " + std::to_string(error_code) + ", desctiption: " + strerror(error_code); +} + +DataProviderSystemErrorException::~DataProviderSystemErrorException() = default; + +const char* DataProviderSystemErrorException::what() const noexcept { + return reason.c_str(); +} +} // namespace onevpl +} // namespace wip +} // namespace gapi +} // namespace cv diff --git a/modules/gapi/src/streaming/onevpl/file_data_provider.cpp b/modules/gapi/src/streaming/onevpl/file_data_provider.cpp new file mode 100644 index 0000000000..2a3df07c51 --- /dev/null +++ b/modules/gapi/src/streaming/onevpl/file_data_provider.cpp @@ -0,0 +1,47 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2021 Intel Corporation +#include + +#include "streaming/onevpl/file_data_provider.hpp" + +namespace cv { +namespace gapi { +namespace wip { +namespace onevpl { + +FileDataProvider::FileDataProvider(const std::string& file_path) : + source_handle(fopen(file_path.c_str(), "rb"), &fclose) { + if (!source_handle) { + throw DataProviderSystemErrorException(errno, + "FileDataProvider: cannot open source file: " + file_path); + } +} + +FileDataProvider::~FileDataProvider() = default; + +size_t FileDataProvider::fetch_data(size_t out_data_bytes_size, void* out_data) { + if (empty()) { + return 0; + } + + size_t ret = fread(out_data, 1, out_data_bytes_size, source_handle.get()); + if (ret == 0) { + if (feof(source_handle.get())) { + source_handle.reset(); + } else { + throw DataProviderSystemErrorException (errno, "FileDataProvider::fetch_data error read"); + } + } + return ret; +} + +bool FileDataProvider::empty() const { + return !source_handle; +} +} // namespace onevpl +} // namespace wip +} // namespace gapi +} // namespace cv diff --git a/modules/gapi/src/streaming/onevpl/file_data_provider.hpp b/modules/gapi/src/streaming/onevpl/file_data_provider.hpp new file mode 100644 index 0000000000..22348290be --- /dev/null +++ b/modules/gapi/src/streaming/onevpl/file_data_provider.hpp @@ -0,0 +1,33 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2021 Intel Corporation + +#ifndef GAPI_STREAMING_ONEVPL_ONEVPL_FILE_DATA_PROVIDER_HPP +#define GAPI_STREAMING_ONEVPL_ONEVPL_FILE_DATA_PROVIDER_HPP +#include + +#include + +namespace cv { +namespace gapi { +namespace wip { +namespace onevpl { +struct FileDataProvider : public IDataProvider { + + using file_ptr = std::unique_ptr; + FileDataProvider(const std::string& file_path); + ~FileDataProvider(); + + size_t fetch_data(size_t out_data_bytes_size, void* out_data) override; + bool empty() const override; +private: + file_ptr source_handle; +}; +} // namespace onevpl +} // namespace wip +} // namespace gapi +} // namespace cv + +#endif // GAPI_STREAMING_ONEVPL_ONEVPL_FILE_DATA_PROVIDER_HPP diff --git a/modules/gapi/src/streaming/onevpl/onevpl_source.cpp b/modules/gapi/src/streaming/onevpl/onevpl_source.cpp deleted file mode 100644 index 988986f6d9..0000000000 --- a/modules/gapi/src/streaming/onevpl/onevpl_source.cpp +++ /dev/null @@ -1,48 +0,0 @@ -// This file is part of OpenCV project. -// It is subject to the license terms in the LICENSE file found in the top-level directory -// of this distribution and at http://opencv.org/license.html. -// -// Copyright (C) 2021 Intel Corporation - -#include - -#include "streaming/onevpl/onevpl_source_priv.hpp" - -namespace cv { -namespace gapi { -namespace wip { - -#ifdef HAVE_ONEVPL -OneVPLSource::OneVPLSource(const std::string& filePath) : - OneVPLSource(std::unique_ptr(new OneVPLSource::Priv(filePath))) { - - if (filePath.empty()) { - util::throw_error(std::logic_error("Cannot create 'OneVPLSource' on empty source file name")); - } -} -#else -OneVPLSource::OneVPLSource(const std::string&) { - GAPI_Assert(false && "Unsupported: G-API compiled without `WITH_GAPI_ONEVPL=ON`"); -} -#endif - -OneVPLSource::OneVPLSource(std::unique_ptr&& impl) : - IStreamSource(), - m_priv(std::move(impl)) { -} - -OneVPLSource::~OneVPLSource() { -} - -bool OneVPLSource::pull(cv::gapi::wip::Data& data) -{ - return m_priv->pull(data); -} - -GMetaArg OneVPLSource::descr_of() const -{ - return m_priv->descr_of(); -} -} // namespace wip -} // namespace gapi -} // namespace cv diff --git a/modules/gapi/src/streaming/onevpl/source.cpp b/modules/gapi/src/streaming/onevpl/source.cpp new file mode 100644 index 0000000000..0decb1358b --- /dev/null +++ b/modules/gapi/src/streaming/onevpl/source.cpp @@ -0,0 +1,58 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2021 Intel Corporation + +#include + +#include "streaming/onevpl/source_priv.hpp" +#include "streaming/onevpl/file_data_provider.hpp" +namespace cv { +namespace gapi { +namespace wip { +namespace onevpl { + +#ifdef HAVE_ONEVPL +GSource::GSource(const std::string& filePath, const CfgParams& cfg_params) : + GSource(std::unique_ptr(new GSource::Priv(std::make_shared(filePath), + cfg_params))) { + + if (filePath.empty()) { + util::throw_error(std::logic_error("Cannot create 'GSource' on empty source file name")); + } +} + +GSource::GSource(std::shared_ptr source, const CfgParams& cfg_params) : + GSource(std::unique_ptr(new GSource::Priv(source, cfg_params))) { +} +#else +GSource::GSource(const std::string&, const CfgParams&) { + GAPI_Assert(false && "Unsupported: G-API compiled without `WITH_GAPI_ONEVPL=ON`"); +} + +GSource::GSource(std::shared_ptr, const CfgParams&) { + GAPI_Assert(false && "Unsupported: G-API compiled without `WITH_GAPI_ONEVPL=ON`"); +} +#endif + +GSource::GSource(std::unique_ptr&& impl) : + IStreamSource(), + m_priv(std::move(impl)) { +} + +GSource::~GSource() = default; + +bool GSource::pull(cv::gapi::wip::Data& data) +{ + return m_priv->pull(data); +} + +GMetaArg GSource::descr_of() const +{ + return m_priv->descr_of(); +} +} // namespace onevpl +} // namespace wip +} // namespace gapi +} // namespace cv diff --git a/modules/gapi/src/streaming/onevpl/onevpl_source_priv.cpp b/modules/gapi/src/streaming/onevpl/source_priv.cpp similarity index 66% rename from modules/gapi/src/streaming/onevpl/onevpl_source_priv.cpp rename to modules/gapi/src/streaming/onevpl/source_priv.cpp index 5c4e8e6941..ed838d6adc 100644 --- a/modules/gapi/src/streaming/onevpl/onevpl_source_priv.cpp +++ b/modules/gapi/src/streaming/onevpl/source_priv.cpp @@ -7,19 +7,21 @@ #include #include -#include "streaming/onevpl/onevpl_source_priv.hpp" +#include "streaming/onevpl/source_priv.hpp" #include "logger.hpp" #ifndef HAVE_ONEVPL namespace cv { namespace gapi { namespace wip { -bool OneVPLSource::Priv::pull(cv::gapi::wip::Data&) { +namespace onevpl { +bool GSource::Priv::pull(cv::gapi::wip::Data&) { return true; } -GMetaArg OneVPLSource::Priv::descr_of() const { +GMetaArg GSource::Priv::descr_of() const { return {}; } +} // namespace onevpl } // namespace wip } // namespace gapi } // namespace cv @@ -29,33 +31,35 @@ GMetaArg OneVPLSource::Priv::descr_of() const { namespace cv { namespace gapi { namespace wip { -OneVPLSource::Priv::Priv() : +namespace onevpl { +GSource::Priv::Priv() : mfx_handle(MFXLoad()) { GAPI_LOG_INFO(nullptr, "Initialized MFX handle: " << mfx_handle); description_is_valid = false; } -OneVPLSource::Priv::Priv(const std::string&) : - OneVPLSource::Priv() +GSource::Priv::Priv(std::shared_ptr, const std::vector&) : + GSource::Priv() { } -OneVPLSource::Priv::~Priv() +GSource::Priv::~Priv() { GAPI_LOG_INFO(nullptr, "Unload MFX handle: " << mfx_handle); MFXUnload(mfx_handle); } -bool OneVPLSource::Priv::pull(cv::gapi::wip::Data&) +bool GSource::Priv::pull(cv::gapi::wip::Data&) { return false; } -GMetaArg OneVPLSource::Priv::descr_of() const +GMetaArg GSource::Priv::descr_of() const { return {}; } +} // namespace onevpl } // namespace wip } // namespace gapi } // namespace cv diff --git a/modules/gapi/src/streaming/onevpl/onevpl_source_priv.hpp b/modules/gapi/src/streaming/onevpl/source_priv.hpp similarity index 80% rename from modules/gapi/src/streaming/onevpl/onevpl_source_priv.hpp rename to modules/gapi/src/streaming/onevpl/source_priv.hpp index b139add993..c278934253 100644 --- a/modules/gapi/src/streaming/onevpl/onevpl_source_priv.hpp +++ b/modules/gapi/src/streaming/onevpl/source_priv.hpp @@ -14,7 +14,7 @@ #include #include -#include +#include #ifdef HAVE_ONEVPL #if (MFX_VERSION >= 2000) @@ -28,10 +28,12 @@ namespace cv { namespace gapi { namespace wip { +namespace onevpl { -struct OneVPLSource::Priv +struct GSource::Priv { - explicit Priv(const std::string& file_path); + explicit Priv(std::shared_ptr provider, + const std::vector& params); ~Priv(); bool pull(cv::gapi::wip::Data& data); @@ -41,6 +43,7 @@ private: mfxLoader mfx_handle; bool description_is_valid; }; +} // namespace onevpl } // namespace wip } // namespace gapi } // namespace cv @@ -50,11 +53,13 @@ private: namespace cv { namespace gapi { namespace wip { -struct OneVPLSource::Priv final +namespace onevpl { +struct GSource::Priv final { bool pull(cv::gapi::wip::Data&); GMetaArg descr_of() const; }; +} // namespace onevpl } // namespace wip } // namespace gapi } // namespace cv diff --git a/modules/gapi/test/streaming/gapi_streaming_tests.cpp b/modules/gapi/test/streaming/gapi_streaming_tests.cpp index 5386d1736f..76027a56c6 100644 --- a/modules/gapi/test/streaming/gapi_streaming_tests.cpp +++ b/modules/gapi/test/streaming/gapi_streaming_tests.cpp @@ -25,6 +25,17 @@ #include #include +#include + +#ifdef HAVE_ONEVPL + +#if (MFX_VERSION >= 2000) +#include +#endif + +#include +#endif // HAVE_ONEVPL + namespace opencv_test { namespace @@ -273,6 +284,22 @@ void checkPullOverload(const cv::Mat& ref, EXPECT_EQ(0., cv::norm(ref, out_mat, cv::NORM_INF)); } +struct StreamDataProvider : public cv::gapi::wip::onevpl::IDataProvider { + + StreamDataProvider(std::istream& in) : data_stream (in) { + EXPECT_TRUE(in); + } + + size_t fetch_data(size_t out_data_size, void* out_data_buf) override { + data_stream.read(reinterpret_cast(out_data_buf), out_data_size); + return data_stream.gcount(); + } + bool empty() const override { + return data_stream.eof() || data_stream.bad(); + } +private: + std::istream& data_stream; +}; } // anonymous namespace TEST_P(GAPI_Streaming, SmokeTest_ConstInput_GMat) @@ -2214,4 +2241,47 @@ TEST(GAPI_Streaming, TestPythonAPI) cc.stop(); } +#ifdef HAVE_ONEVPL +const unsigned char hevc_header[] = { + 0x00, 0x00, 0x00, 0x01, 0x40, 0x01, 0x0C, 0x06, 0xFF, 0xFF, 0x01, 0x40, 0x00, + 0x00, 0x03, 0x00, 0x80, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x78, 0x00, + 0x00, 0x04, 0x02, 0x10, 0x30, 0x00, 0x00, 0x03, 0x00, 0x10, 0x00, 0x00, 0x03, + 0x01, 0xE5, 0x00, 0x00, 0x00, 0x01, 0x42, 0x01, 0x06, 0x01, 0x40, 0x00, 0x00, + 0x03, 0x00, 0x80, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x78, 0x00, 0x00, + 0xA0, 0x10, 0x20, 0x61, 0x63, 0x41, 0x00, 0x86, 0x49, 0x1B, 0x2B, 0x20, 0x00, + 0x00, 0x00, 0x01, 0x44, 0x01, 0xC0, 0x71, 0xC0, 0xD9, 0x20, 0x00, 0x00, 0x00, + 0x01, 0x26, 0x01, 0xAF, 0x0C +}; +TEST(OneVPL_Source, Init) +{ + using CfgParam = cv::gapi::wip::onevpl::CfgParam; + + std::vector src_params; + src_params.push_back(CfgParam::create("mfxImplDescription.Impl", + MFX_IMPL_TYPE_HARDWARE)); + src_params.push_back(CfgParam::create("mfxImplDescription.AccelerationMode", + MFX_ACCEL_MODE_VIA_D3D11, false)); + src_params.push_back(CfgParam::create("mfxImplDescription.mfxDecoderDescription.decoder.CodecID", + MFX_CODEC_HEVC)); + std::stringstream stream(std::ios_base::in | std::ios_base::out | std::ios_base::binary); + EXPECT_TRUE(stream.write(reinterpret_cast(const_cast(hevc_header)), + sizeof(hevc_header))); + std::shared_ptr stream_data_provider = std::make_shared(stream); + + cv::Ptr cap; + bool cap_created = false; + try { + cap = cv::gapi::wip::make_onevpl_src(stream_data_provider, src_params); + cap_created = true; + } catch (const std::exception&) { + } + ASSERT_TRUE(cap_created); + + cv::gapi::wip::Data out; + while (cap->pull(out)) { + (void)out; + } + EXPECT_TRUE(stream_data_provider->empty()); +} +#endif } // namespace opencv_test From 6f4160c0140c8ef88d777ba2ee5a605bf345de64 Mon Sep 17 00:00:00 2001 From: Alexander Panov Date: Tue, 24 Aug 2021 22:29:40 +0300 Subject: [PATCH 139/262] Merge pull request #20584 from AleksandrPanov:fix_HoughCircles_ALT_dimensions Fix hough circles alt dimensions * fix OutputArray _circles dimensions * add houghcircles_alt test * fix warnings * add shape assert * change asserts --- modules/imgproc/src/hough.cpp | 6 ++- modules/python/test/test_houghcircles.py | 47 ++++++++++++++++++++++++ 2 files changed, 51 insertions(+), 2 deletions(-) diff --git a/modules/imgproc/src/hough.cpp b/modules/imgproc/src/hough.cpp index 79f34521e8..5c78c94983 100644 --- a/modules/imgproc/src/hough.cpp +++ b/modules/imgproc/src/hough.cpp @@ -2301,14 +2301,16 @@ static void HoughCircles( InputArray _image, OutputArray _circles, std::vector cw(ncircles); for( i = 0; i < ncircles; i++ ) cw[i] = GetCircle4f(circles[i]); - Mat(cw).copyTo(_circles); + if (ncircles > 0) + Mat(1, (int)ncircles, cv::traits::Type::value, &cw[0]).copyTo(_circles); } else if( type == CV_32FC3 ) { std::vector cwow(ncircles); for( i = 0; i < ncircles; i++ ) cwow[i] = GetCircle(circles[i]); - Mat(cwow).copyTo(_circles); + if (ncircles > 0) + Mat(1, (int)ncircles, cv::traits::Type::value, &cwow[0]).copyTo(_circles); } else CV_Error(Error::StsError, "Internal error"); diff --git a/modules/python/test/test_houghcircles.py b/modules/python/test/test_houghcircles.py index 90cd184139..2cfc6c9d68 100644 --- a/modules/python/test/test_houghcircles.py +++ b/modules/python/test/test_houghcircles.py @@ -80,5 +80,52 @@ class houghcircles_test(NewOpenCVTests): self.assertLess(float(len(circles) - matches_counter) / len(circles), .75) + def test_houghcircles_alt(self): + + fn = "samples/data/board.jpg" + + src = self.get_sample(fn, 1) + img = cv.cvtColor(src, cv.COLOR_BGR2GRAY) + img = cv.medianBlur(img, 5) + + circles = cv.HoughCircles(img, cv.HOUGH_GRADIENT_ALT, 1, 10, np.array([]), 300, 0.9, 1, 30) + + self.assertEqual(circles.shape, (1, 18, 3)) + + circles = circles[0] + + testCircles = [[38, 181, 17.6], + [99.7, 166, 13.12], + [142.7, 160, 13.52], + [223.6, 110, 8.62], + [79.1, 206.7, 8.62], + [47.5, 351.6, 11.64], + [189.5, 354.4, 11.64], + [189.8, 298.9, 10.64], + [189.5, 252.4, 14.62], + [252.5, 393.4, 15.62], + [602.9, 467.5, 11.42], + [222, 210.4, 9.12], + [263.1, 216.7, 9.12], + [359.8, 222.6, 9.12], + [518.9, 120.9, 9.12], + [413.8, 113.4, 9.12], + [489, 127.2, 9.12], + [448.4, 121.3, 9.12], + [384.6, 128.9, 8.62]] + + matches_counter = 0 + + for i in range(len(testCircles)): + for j in range(len(circles)): + + tstCircle = circleApproximation(testCircles[i]) + circle = circleApproximation(circles[j]) + if convContoursIntersectiponRate(tstCircle, circle) > 0.6: + matches_counter += 1 + + self.assertGreater(float(matches_counter) / len(testCircles), .5) + self.assertLess(float(len(circles) - matches_counter) / len(circles), .75) + if __name__ == '__main__': NewOpenCVTests.bootstrap() From 9bda96d39e7d63da97ef38a0dcb04ff9d0670a01 Mon Sep 17 00:00:00 2001 From: Dale Phurrough Date: Tue, 24 Aug 2021 20:06:36 +0200 Subject: [PATCH 140/262] add test case --- modules/core/test/test_umat.cpp | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/modules/core/test/test_umat.cpp b/modules/core/test/test_umat.cpp index c2bd6eceba..c323d17c06 100644 --- a/modules/core/test/test_umat.cpp +++ b/modules/core/test/test_umat.cpp @@ -1419,4 +1419,37 @@ TEST(UMat, resize_Mat_issue_13577) cv::ocl::setUseOpenCL(useOCL); // restore state } +TEST(UMat, exceptions_refcounts_issue_20594) +{ + if (!cv::ocl::useOpenCL()) + { + // skip test, difficult to create exception scenario without OpenCL + std::cout << "OpenCL is not enabled. Skip test" << std::endl; + return; + } + + UMat umat1(10, 10, CV_8UC1); + EXPECT_EQ(0, umat1.u->refcount); + + // cause exception in underlying allocator + void* const original_handle = umat1.u->handle; + umat1.u->handle = NULL; + try + { + Mat mat1 = umat1.getMat(ACCESS_RW); + } + catch (...) + { + // nothing + } + + // check for correct refcount, and no change of intentional bad handle + EXPECT_EQ(0, umat1.u->refcount); + EXPECT_EQ(NULL, umat1.u->handle); + + // reset UMat to good state + umat1.u->refcount = 0; + umat1.u->handle = original_handle; +} + } } // namespace opencv_test::ocl From 54a9e009703567c03d8fccbee7bb43f945b396a1 Mon Sep 17 00:00:00 2001 From: Dale Phurrough Date: Tue, 24 Aug 2021 18:56:25 +0200 Subject: [PATCH 141/262] fix opencv/opencv#20594 - exception handling with refcounts --- modules/core/src/matrix.cpp | 5 ++- modules/core/src/ocl.cpp | 26 +++++++++++++-- modules/core/src/umatrix.cpp | 63 +++++++++++++++++++++++------------- 3 files changed, 66 insertions(+), 28 deletions(-) diff --git a/modules/core/src/matrix.cpp b/modules/core/src/matrix.cpp index 4db7c7a89e..1deff6b450 100644 --- a/modules/core/src/matrix.cpp +++ b/modules/core/src/matrix.cpp @@ -749,18 +749,17 @@ Mat::Mat(const Mat& m, const Rect& roi) data += roi.x*esz; CV_Assert( 0 <= roi.x && 0 <= roi.width && roi.x + roi.width <= m.cols && 0 <= roi.y && 0 <= roi.height && roi.y + roi.height <= m.rows ); - if( u ) - CV_XADD(&u->refcount, 1); if( roi.width < m.cols || roi.height < m.rows ) flags |= SUBMATRIX_FLAG; step[0] = m.step[0]; step[1] = esz; updateContinuityFlag(); + addref(); if( rows <= 0 || cols <= 0 ) { - release(); rows = cols = 0; + release(); } } diff --git a/modules/core/src/ocl.cpp b/modules/core/src/ocl.cpp index 8b1377eec6..2d85a32366 100644 --- a/modules/core/src/ocl.cpp +++ b/modules/core/src/ocl.cpp @@ -2774,19 +2774,33 @@ struct Kernel::Impl void cleanupUMats() { + bool exceptionOccurred = false; for( int i = 0; i < MAX_ARRS; i++ ) + { if( u[i] ) { if( CV_XADD(&u[i]->urefcount, -1) == 1 ) { u[i]->flags |= UMatData::ASYNC_CLEANUP; - u[i]->currAllocator->deallocate(u[i]); + try + { + u[i]->currAllocator->deallocate(u[i]); + } + catch(const std::exception& exc) + { + // limited by legacy before C++11, therefore log and + // remember some exception occurred to throw below + CV_LOG_ERROR(NULL, "OCL: Unexpected C++ exception in OpenCL Kernel::Impl::cleanupUMats(): " << exc.what()); + exceptionOccurred = true; + } } u[i] = 0; } + } nu = 0; haveTempDstUMats = false; haveTempSrcUMats = false; + CV_Assert(!exceptionOccurred); } void addUMat(const UMat& m, bool dst) @@ -2817,8 +2831,16 @@ struct Kernel::Impl void finit(cl_event e) { CV_UNUSED(e); - cleanupUMats(); isInProgress = false; + try + { + cleanupUMats(); + } + catch(...) + { + release(); + throw; + } release(); } diff --git a/modules/core/src/umatrix.cpp b/modules/core/src/umatrix.cpp index 94f828ba60..da01f3785f 100644 --- a/modules/core/src/umatrix.cpp +++ b/modules/core/src/umatrix.cpp @@ -540,13 +540,26 @@ UMat Mat::getUMat(int accessFlags, UMatUsageFlags usageFlags) const CV_XADD(&(u->refcount), 1); CV_XADD(&(u->urefcount), 1); } - hdr.flags = flags; - setSize(hdr, dims, size.p, step.p); - finalizeHdr(hdr); - hdr.u = new_u; - hdr.offset = 0; //data - datastart; - hdr.addref(); - return hdr; + try + { + hdr.flags = flags; + setSize(hdr, dims, size.p, step.p); + finalizeHdr(hdr); + hdr.u = new_u; + hdr.offset = 0; //data - datastart; + hdr.addref(); + return hdr; + } + catch(...) + { + if (u != NULL) + { + CV_XADD(&(u->refcount), -1); + CV_XADD(&(u->urefcount), -1); + } + new_u->currAllocator->deallocate(new_u); + throw; + } } void UMat::create(int d, const int* _sizes, int _type, UMatUsageFlags _usageFlags) @@ -692,18 +705,17 @@ UMat::UMat(const UMat& m, const Rect& roi) offset += roi.x*esz; CV_Assert( 0 <= roi.x && 0 <= roi.width && roi.x + roi.width <= m.cols && 0 <= roi.y && 0 <= roi.height && roi.y + roi.height <= m.rows ); - if( u ) - CV_XADD(&(u->urefcount), 1); if( roi.width < m.cols || roi.height < m.rows ) flags |= SUBMATRIX_FLAG; step[0] = m.step[0]; step[1] = esz; updateContinuityFlag(); + addref(); if( rows <= 0 || cols <= 0 ) { - release(); rows = cols = 0; + release(); } } @@ -969,24 +981,29 @@ Mat UMat::getMat(int accessFlags) const // TODO Support ACCESS_READ (ACCESS_WRITE) without unnecessary data transfers accessFlags |= ACCESS_RW; UMatDataAutoLock autolock(u); - if(CV_XADD(&u->refcount, 1) == 0) - u->currAllocator->map(u, accessFlags); - if (u->data != 0) + try { - Mat hdr(dims, size.p, type(), u->data + offset, step.p); - hdr.flags = flags; - hdr.u = u; - hdr.datastart = u->data; - hdr.data = u->data + offset; - hdr.datalimit = hdr.dataend = u->data + u->size; - return hdr; + if(CV_XADD(&u->refcount, 1) == 0) + u->currAllocator->map(u, accessFlags); + if (u->data != 0) + { + Mat hdr(dims, size.p, type(), u->data + offset, step.p); + hdr.flags = flags; + hdr.u = u; + hdr.datastart = u->data; + hdr.data = u->data + offset; + hdr.datalimit = hdr.dataend = u->data + u->size; + return hdr; + } } - else + catch(...) { CV_XADD(&u->refcount, -1); - CV_Assert(u->data != 0 && "Error mapping of UMat to host memory."); - return Mat(); + throw; } + CV_XADD(&u->refcount, -1); + CV_Assert(u->data != 0 && "Error mapping of UMat to host memory."); + return Mat(); } void* UMat::handle(int accessFlags) const From 38d0063c367b2697b57ab0809530889cc2e20fd7 Mon Sep 17 00:00:00 2001 From: Vincent Rabaud Date: Thu, 26 Aug 2021 09:46:25 +0200 Subject: [PATCH 142/262] Do not use deprecated ReleaseCleared in protobuf library. This is to make code work with protobuf arenas for memory management (ReleaseCleared is incompatible). The cleaning of the memory is also simpler. --- modules/dnn/src/caffe/caffe_importer.cpp | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/modules/dnn/src/caffe/caffe_importer.cpp b/modules/dnn/src/caffe/caffe_importer.cpp index 8673be03fb..2d615c448a 100644 --- a/modules/dnn/src/caffe/caffe_importer.cpp +++ b/modules/dnn/src/caffe/caffe_importer.cpp @@ -306,16 +306,13 @@ public: caffe::LayerParameter* binLayer = netBinary.mutable_layer(li); const int numBlobs = binLayer->blobs_size(); + std::vector blobs(numBlobs); + binLayer->mutable_blobs()->ExtractSubrange(0, numBlobs, blobs.data()); layerParams.blobs.resize(numBlobs); for (int bi = 0; bi < numBlobs; bi++) { - blobFromProto(binLayer->blobs(bi), layerParams.blobs[bi]); - } - binLayer->clear_blobs(); - CV_Assert(numBlobs == binLayer->blobs().ClearedCount()); - for (int bi = 0; bi < numBlobs; bi++) - { - delete binLayer->mutable_blobs()->ReleaseCleared(); + blobFromProto(*blobs[bi], layerParams.blobs[bi]); + delete blobs[bi]; } } From 8ee33ca5511c186c81f0e2e9b705e5b0c5cc1935 Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Sat, 28 Aug 2021 01:28:34 +0000 Subject: [PATCH 143/262] java(test): avoid deprecation warning - 'new Byte' => 'Byte.valueOf' --- .../org/opencv/test/utils/ConvertersTest.java | 28 +++++++++---------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/modules/java/test/common_test/src/org/opencv/test/utils/ConvertersTest.java b/modules/java/test/common_test/src/org/opencv/test/utils/ConvertersTest.java index 54d2736c46..88d75dcdfb 100644 --- a/modules/java/test/common_test/src/org/opencv/test/utils/ConvertersTest.java +++ b/modules/java/test/common_test/src/org/opencv/test/utils/ConvertersTest.java @@ -28,9 +28,9 @@ public class ConvertersTest extends OpenCVTestCase { byte value1 = 2; byte value2 = 4; byte value3 = 3; - truth.add(new Byte(value1)); - truth.add(new Byte(value2)); - truth.add(new Byte(value3)); + truth.add(Byte.valueOf(value1)); + truth.add(Byte.valueOf(value2)); + truth.add(Byte.valueOf(value3)); assertEquals(truth, bs); } @@ -248,9 +248,9 @@ public class ConvertersTest extends OpenCVTestCase { byte value1 = 2; byte value2 = 4; byte value3 = 3; - truth.add(new Byte(value1)); - truth.add(new Byte(value2)); - truth.add(new Byte(value3)); + truth.add(Byte.valueOf(value1)); + truth.add(Byte.valueOf(value2)); + truth.add(Byte.valueOf(value3)); assertEquals(truth, bs); } @@ -276,10 +276,10 @@ public class ConvertersTest extends OpenCVTestCase { byte value2 = 2; byte value3 = 3; byte value4 = 4; - bytes.add(new Byte(value1)); - bytes.add(new Byte(value2)); - bytes.add(new Byte(value3)); - bytes.add(new Byte(value4)); + bytes.add(Byte.valueOf(value1)); + bytes.add(Byte.valueOf(value2)); + bytes.add(Byte.valueOf(value3)); + bytes.add(Byte.valueOf(value4)); dst = Converters.vector_char_to_Mat(bytes); truth = new Mat(4, 1, CvType.CV_8SC1); @@ -499,10 +499,10 @@ public class ConvertersTest extends OpenCVTestCase { byte value2 = 2; byte value3 = 3; byte value4 = 4; - bytes.add(new Byte(value1)); - bytes.add(new Byte(value2)); - bytes.add(new Byte(value3)); - bytes.add(new Byte(value4)); + bytes.add(Byte.valueOf(value1)); + bytes.add(Byte.valueOf(value2)); + bytes.add(Byte.valueOf(value3)); + bytes.add(Byte.valueOf(value4)); dst = Converters.vector_uchar_to_Mat(bytes); truth = new Mat(4, 1, CvType.CV_8UC1); From 076587425e66a1dd66d06e9f274acd6f2eda2a3f Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Sat, 28 Aug 2021 17:11:26 +0000 Subject: [PATCH 144/262] build: eliminate build warnings --- modules/features2d/src/kaze/KAZEFeatures.cpp | 63 ++++++++++--------- modules/photo/src/contrast_preserve.hpp | 4 +- modules/superres/src/btv_l1.cpp | 2 +- .../TrackingMotion/cornerDetector_Demo.cpp | 2 +- 4 files changed, 39 insertions(+), 32 deletions(-) diff --git a/modules/features2d/src/kaze/KAZEFeatures.cpp b/modules/features2d/src/kaze/KAZEFeatures.cpp index 58f0937f00..ab591d417c 100644 --- a/modules/features2d/src/kaze/KAZEFeatures.cpp +++ b/modules/features2d/src/kaze/KAZEFeatures.cpp @@ -311,7 +311,7 @@ private: void KAZEFeatures::Determinant_Hessian(std::vector& kpts) { int level = 0; - float dist = 0.0, smax = 3.0; + float smax = 3.0; int npoints = 0, id_repeated = 0; int left_x = 0, right_x = 0, up_y = 0, down_y = 0; bool is_extremum = false, is_repeated = false, is_out = false; @@ -338,17 +338,24 @@ void KAZEFeatures::Determinant_Hessian(std::vector& kpts) for (int j = 0; j < (int)kpts_par_[i].size(); j++) { level = i + 1; + const TEvolution& evolution_level = evolution_[level]; + is_extremum = true; is_repeated = false; is_out = false; - // Check in case we have the same point as maxima in previous evolution levels - for (int ik = 0; ik < (int)kpts.size(); ik++) { - if (kpts[ik].class_id == level || kpts[ik].class_id == level + 1 || kpts[ik].class_id == level - 1) { - dist = pow(kpts_par_[i][j].pt.x - kpts[ik].pt.x, 2) + pow(kpts_par_[i][j].pt.y - kpts[ik].pt.y, 2); + const KeyPoint& kpts_par_ij = kpts_par_[i][j]; - if (dist < evolution_[level].sigma_size*evolution_[level].sigma_size) { - if (kpts_par_[i][j].response > kpts[ik].response) { + // Check in case we have the same point as maxima in previous evolution levels + for (int ik = 0; ik < (int)kpts.size(); ik++) + { + const KeyPoint& kpts_ik = kpts[ik]; + if (kpts_ik.class_id == level || kpts_ik.class_id == level + 1 || kpts_ik.class_id == level - 1) { + Point2f diff = kpts_par_ij.pt - kpts_ik.pt; + float dist = diff.dot(diff); + + if (dist < evolution_level.sigma_size*evolution_level.sigma_size) { + if (kpts_par_ij.response > kpts_ik.response) { id_repeated = ik; is_repeated = true; } @@ -363,23 +370,23 @@ void KAZEFeatures::Determinant_Hessian(std::vector& kpts) if (is_extremum == true) { // Check that the point is under the image limits for the descriptor computation - left_x = cvRound(kpts_par_[i][j].pt.x - smax*kpts_par_[i][j].size); - right_x = cvRound(kpts_par_[i][j].pt.x + smax*kpts_par_[i][j].size); - up_y = cvRound(kpts_par_[i][j].pt.y - smax*kpts_par_[i][j].size); - down_y = cvRound(kpts_par_[i][j].pt.y + smax*kpts_par_[i][j].size); + left_x = cvRound(kpts_par_ij.pt.x - smax*kpts_par_ij.size); + right_x = cvRound(kpts_par_ij.pt.x + smax*kpts_par_ij.size); + up_y = cvRound(kpts_par_ij.pt.y - smax*kpts_par_ij.size); + down_y = cvRound(kpts_par_ij.pt.y + smax*kpts_par_ij.size); - if (left_x < 0 || right_x >= evolution_[level].Ldet.cols || - up_y < 0 || down_y >= evolution_[level].Ldet.rows) { + if (left_x < 0 || right_x >= evolution_level.Ldet.cols || + up_y < 0 || down_y >= evolution_level.Ldet.rows) { is_out = true; } if (is_out == false) { if (is_repeated == false) { - kpts.push_back(kpts_par_[i][j]); + kpts.push_back(kpts_par_ij); npoints++; } else { - kpts[id_repeated] = kpts_par_[i][j]; + kpts[id_repeated] = kpts_par_ij; } } } @@ -513,16 +520,16 @@ public: if (options_.upright) { kpts[i].angle = 0.0; - if (options_.extended) + if (options_.extended) Get_KAZE_Upright_Descriptor_128(kpts[i], desc.ptr((int)i)); else Get_KAZE_Upright_Descriptor_64(kpts[i], desc.ptr((int)i)); } else { - KAZEFeatures::Compute_Main_Orientation(kpts[i], evolution, options_); + KAZEFeatures::Compute_Main_Orientation(kpts[i], evolution, options_); - if (options_.extended) + if (options_.extended) Get_KAZE_Descriptor_128(kpts[i], desc.ptr((int)i)); else Get_KAZE_Descriptor_64(kpts[i], desc.ptr((int)i)); @@ -712,26 +719,26 @@ void KAZE_Descriptor_Invoker::Get_KAZE_Upright_Descriptor_64(const KeyPoint &kpt y1 = (int)(sample_y - 0.5f); x1 = (int)(sample_x - 0.5f); - checkDescriptorLimits(x1, y1, options_.img_width, options_.img_height); + checkDescriptorLimits(x1, y1, options_.img_width, options_.img_height); y2 = (int)(sample_y + 0.5f); x2 = (int)(sample_x + 0.5f); - checkDescriptorLimits(x2, y2, options_.img_width, options_.img_height); + checkDescriptorLimits(x2, y2, options_.img_width, options_.img_height); fx = sample_x - x1; fy = sample_y - y1; - res1 = *(evolution[level].Lx.ptr(y1)+x1); - res2 = *(evolution[level].Lx.ptr(y1)+x2); - res3 = *(evolution[level].Lx.ptr(y2)+x1); - res4 = *(evolution[level].Lx.ptr(y2)+x2); + res1 = *(evolution[level].Lx.ptr(y1)+x1); + res2 = *(evolution[level].Lx.ptr(y1)+x2); + res3 = *(evolution[level].Lx.ptr(y2)+x1); + res4 = *(evolution[level].Lx.ptr(y2)+x2); rx = (1.0f - fx)*(1.0f - fy)*res1 + fx*(1.0f - fy)*res2 + (1.0f - fx)*fy*res3 + fx*fy*res4; - res1 = *(evolution[level].Ly.ptr(y1)+x1); - res2 = *(evolution[level].Ly.ptr(y1)+x2); - res3 = *(evolution[level].Ly.ptr(y2)+x1); - res4 = *(evolution[level].Ly.ptr(y2)+x2); + res1 = *(evolution[level].Ly.ptr(y1)+x1); + res2 = *(evolution[level].Ly.ptr(y1)+x2); + res3 = *(evolution[level].Ly.ptr(y2)+x1); + res4 = *(evolution[level].Ly.ptr(y2)+x2); ry = (1.0f - fx)*(1.0f - fy)*res1 + fx*(1.0f - fy)*res2 + (1.0f - fx)*fy*res3 + fx*fy*res4; rx = gauss_s1*rx; diff --git a/modules/photo/src/contrast_preserve.hpp b/modules/photo/src/contrast_preserve.hpp index 5681779fc9..4079272e99 100644 --- a/modules/photo/src/contrast_preserve.hpp +++ b/modules/photo/src/contrast_preserve.hpp @@ -358,9 +358,9 @@ void Decolor::grayImContruct(vector &wei, const Mat &img, Mat &Gray) co { for(int i = 0;i(i,j)=Gray.at(i,j) + + Gray.at(i,j)=static_cast(Gray.at(i,j) + static_cast(wei[kk])*pow(rgb_channel[2].at(i,j),r)*pow(rgb_channel[1].at(i,j),g)* - pow(rgb_channel[0].at(i,j),b); + pow(rgb_channel[0].at(i,j),b)); kk=kk+1; } diff --git a/modules/superres/src/btv_l1.cpp b/modules/superres/src/btv_l1.cpp index cf2ca58ceb..26c75a94ce 100644 --- a/modules/superres/src/btv_l1.cpp +++ b/modules/superres/src/btv_l1.cpp @@ -356,7 +356,7 @@ namespace for (int m = 0, ind = 0; m <= ksize; ++m) { for (int l = ksize; l + m >= 0; --l, ++ind) - btvWeights[ind] = pow(alpha_f, std::abs(m) + std::abs(l)); + btvWeights[ind] = static_cast(pow(alpha_f, std::abs(m) + std::abs(l))); } } diff --git a/samples/cpp/tutorial_code/TrackingMotion/cornerDetector_Demo.cpp b/samples/cpp/tutorial_code/TrackingMotion/cornerDetector_Demo.cpp index a43024e557..e2835531e1 100644 --- a/samples/cpp/tutorial_code/TrackingMotion/cornerDetector_Demo.cpp +++ b/samples/cpp/tutorial_code/TrackingMotion/cornerDetector_Demo.cpp @@ -62,7 +62,7 @@ int main( int argc, char** argv ) { float lambda_1 = myHarris_dst.at(i, j)[0]; float lambda_2 = myHarris_dst.at(i, j)[1]; - Mc.at(i, j) = lambda_1*lambda_2 - 0.04f*pow( ( lambda_1 + lambda_2 ), 2 ); + Mc.at(i, j) = lambda_1*lambda_2 - 0.04f*((lambda_1 + lambda_2) * (lambda_1 + lambda_2)); } } From 3995deaf76d0c988d315630ace46eb01dba37174 Mon Sep 17 00:00:00 2001 From: Dale Phurrough Date: Sat, 28 Aug 2021 01:12:44 +0200 Subject: [PATCH 145/262] fix opencv/opencv#20544 nodiscard for msvc/gcc - includes workaround for preprocessor non-compliance - enable attribute syntax checking in msvc --- cmake/OpenCVCompilerOptions.cmake | 3 ++ modules/core/include/opencv2/core/cvdef.h | 39 ++++++++++++-- modules/core/include/opencv2/core/mat.hpp | 62 +++++++++++----------- modules/core/include/opencv2/core/matx.hpp | 14 ++--- 4 files changed, 77 insertions(+), 41 deletions(-) diff --git a/cmake/OpenCVCompilerOptions.cmake b/cmake/OpenCVCompilerOptions.cmake index 303d4f451e..d444085ea3 100644 --- a/cmake/OpenCVCompilerOptions.cmake +++ b/cmake/OpenCVCompilerOptions.cmake @@ -400,6 +400,9 @@ if(MSVC) endif() endif() + # Enable [[attribute]] syntax checking to prevent silent failure: "attribute is ignored in this syntactic position" + add_extra_compiler_option("/w15240") + if(NOT ENABLE_NOISY_WARNINGS) ocv_warnings_disable(CMAKE_CXX_FLAGS /wd4127) # conditional expression is constant ocv_warnings_disable(CMAKE_CXX_FLAGS /wd4251) # class 'std::XXX' needs to have dll-interface to be used by clients of YYY diff --git a/modules/core/include/opencv2/core/cvdef.h b/modules/core/include/opencv2/core/cvdef.h index a4b560b4e1..6011b2a931 100644 --- a/modules/core/include/opencv2/core/cvdef.h +++ b/modules/core/include/opencv2/core/cvdef.h @@ -575,14 +575,47 @@ Cv64suf; # endif #endif +/****************************************************************************************\ +* CV_NODISCARD_STD attribute (C++17) * +* encourages the compiler to issue a warning if the return value is discarded * +\****************************************************************************************/ +#ifndef CV_NODISCARD_STD +# ifndef __has_cpp_attribute +// workaround preprocessor non-compliance https://reviews.llvm.org/D57851 +# define __has_cpp_attribute(__x) 0 +# endif +# if __has_cpp_attribute(nodiscard) +# define CV_NODISCARD_STD [[nodiscard]] +# elif __cplusplus >= 201703L +// available when compiler is C++17 compliant +# define CV_NODISCARD_STD [[nodiscard]] +# elif defined(_MSC_VER) && _MSC_VER >= 1911 && _MSVC_LANG >= 201703L +// available with VS2017 v15.3+ with /std:c++17 or higher; works on functions and classes +# define CV_NODISCARD_STD [[nodiscard]] +# elif defined(__GNUC__) && (((__GNUC__ * 100) + __GNUC_MINOR__) >= 700) && (__cplusplus >= 201103L) +// available with GCC 7.0+; works on functions, works or silently fails on classes +# define CV_NODISCARD_STD [[nodiscard]] +# elif defined(__GNUC__) && (((__GNUC__ * 100) + __GNUC_MINOR__) >= 408) && (__cplusplus >= 201103L) +// available with GCC 4.8+ but it usually does nothing and can fail noisily -- therefore not used +// define CV_NODISCARD_STD [[gnu::warn_unused_result]] +# endif +#endif +#ifndef CV_NODISCARD_STD +# define CV_NODISCARD_STD /* nothing by default */ +#endif + /****************************************************************************************\ -* CV_NODISCARD attribute * -* encourages the compiler to issue a warning if the return value is discarded (C++17) * +* CV_NODISCARD attribute (deprecated, GCC only) * +* DONT USE: use instead the standard CV_NODISCARD_STD macro above * +* this legacy method silently fails to issue warning until some version * +* after gcc 6.3.0. Yet with gcc 7+ you can use the above standard method * +* which makes this method useless. Don't use it. * +* @deprecated use instead CV_NODISCARD_STD * \****************************************************************************************/ #ifndef CV_NODISCARD # if defined(__GNUC__) -# define CV_NODISCARD __attribute__((__warn_unused_result__)) // at least available with GCC 3.4 +# define CV_NODISCARD __attribute__((__warn_unused_result__)) # elif defined(__clang__) && defined(__has_attribute) # if __has_attribute(__warn_unused_result__) # define CV_NODISCARD __attribute__((__warn_unused_result__)) diff --git a/modules/core/include/opencv2/core/mat.hpp b/modules/core/include/opencv2/core/mat.hpp index d0ce61e123..fdcb8fc817 100644 --- a/modules/core/include/opencv2/core/mat.hpp +++ b/modules/core/include/opencv2/core/mat.hpp @@ -1204,14 +1204,14 @@ public: The method creates a square diagonal matrix from specified main diagonal. @param d One-dimensional matrix that represents the main diagonal. */ - static Mat diag(const Mat& d); + CV_NODISCARD_STD static Mat diag(const Mat& d); /** @brief Creates a full copy of the array and the underlying data. The method creates a full copy of the array. The original step[] is not taken into account. So, the array copy is a continuous array occupying total()*elemSize() bytes. */ - Mat clone() const CV_NODISCARD; + CV_NODISCARD_STD Mat clone() const; /** @brief Copies the matrix to another one. @@ -1375,20 +1375,20 @@ public: @param cols Number of columns. @param type Created matrix type. */ - static MatExpr zeros(int rows, int cols, int type); + CV_NODISCARD_STD static MatExpr zeros(int rows, int cols, int type); /** @overload @param size Alternative to the matrix size specification Size(cols, rows) . @param type Created matrix type. */ - static MatExpr zeros(Size size, int type); + CV_NODISCARD_STD static MatExpr zeros(Size size, int type); /** @overload @param ndims Array dimensionality. @param sz Array of integers specifying the array shape. @param type Created matrix type. */ - static MatExpr zeros(int ndims, const int* sz, int type); + CV_NODISCARD_STD static MatExpr zeros(int ndims, const int* sz, int type); /** @brief Returns an array of all 1's of the specified size and type. @@ -1406,20 +1406,20 @@ public: @param cols Number of columns. @param type Created matrix type. */ - static MatExpr ones(int rows, int cols, int type); + CV_NODISCARD_STD static MatExpr ones(int rows, int cols, int type); /** @overload @param size Alternative to the matrix size specification Size(cols, rows) . @param type Created matrix type. */ - static MatExpr ones(Size size, int type); + CV_NODISCARD_STD static MatExpr ones(Size size, int type); /** @overload @param ndims Array dimensionality. @param sz Array of integers specifying the array shape. @param type Created matrix type. */ - static MatExpr ones(int ndims, const int* sz, int type); + CV_NODISCARD_STD static MatExpr ones(int ndims, const int* sz, int type); /** @brief Returns an identity matrix of the specified size and type. @@ -1435,13 +1435,13 @@ public: @param cols Number of columns. @param type Created matrix type. */ - static MatExpr eye(int rows, int cols, int type); + CV_NODISCARD_STD static MatExpr eye(int rows, int cols, int type); /** @overload @param size Alternative matrix size specification as Size(cols, rows) . @param type Created matrix type. */ - static MatExpr eye(Size size, int type); + CV_NODISCARD_STD static MatExpr eye(Size size, int type); /** @brief Allocates new array data if needed. @@ -2302,7 +2302,7 @@ public: Mat_ row(int y) const; Mat_ col(int x) const; Mat_ diag(int d=0) const; - Mat_ clone() const CV_NODISCARD; + CV_NODISCARD_STD Mat_ clone() const; //! overridden forms of Mat::elemSize() etc. size_t elemSize() const; @@ -2315,14 +2315,14 @@ public: size_t stepT(int i=0) const; //! overridden forms of Mat::zeros() etc. Data type is omitted, of course - static MatExpr zeros(int rows, int cols); - static MatExpr zeros(Size size); - static MatExpr zeros(int _ndims, const int* _sizes); - static MatExpr ones(int rows, int cols); - static MatExpr ones(Size size); - static MatExpr ones(int _ndims, const int* _sizes); - static MatExpr eye(int rows, int cols); - static MatExpr eye(Size size); + CV_NODISCARD_STD static MatExpr zeros(int rows, int cols); + CV_NODISCARD_STD static MatExpr zeros(Size size); + CV_NODISCARD_STD static MatExpr zeros(int _ndims, const int* _sizes); + CV_NODISCARD_STD static MatExpr ones(int rows, int cols); + CV_NODISCARD_STD static MatExpr ones(Size size); + CV_NODISCARD_STD static MatExpr ones(int _ndims, const int* _sizes); + CV_NODISCARD_STD static MatExpr eye(int rows, int cols); + CV_NODISCARD_STD static MatExpr eye(Size size); //! some more overridden methods Mat_& adjustROI( int dtop, int dbottom, int dleft, int dright ); @@ -2469,10 +2469,10 @@ public: //! <0 - a diagonal from the lower half) UMat diag(int d=0) const; //! constructs a square diagonal matrix which main diagonal is vector "d" - static UMat diag(const UMat& d); + CV_NODISCARD_STD static UMat diag(const UMat& d); //! returns deep copy of the matrix, i.e. the data is copied - UMat clone() const CV_NODISCARD; + CV_NODISCARD_STD UMat clone() const; //! copies the matrix content to "m". // It calls m.create(this->size(), this->type()). void copyTo( OutputArray m ) const; @@ -2503,14 +2503,14 @@ public: double dot(InputArray m) const; //! Matlab-style matrix initialization - static UMat zeros(int rows, int cols, int type); - static UMat zeros(Size size, int type); - static UMat zeros(int ndims, const int* sz, int type); - static UMat ones(int rows, int cols, int type); - static UMat ones(Size size, int type); - static UMat ones(int ndims, const int* sz, int type); - static UMat eye(int rows, int cols, int type); - static UMat eye(Size size, int type); + CV_NODISCARD_STD static UMat zeros(int rows, int cols, int type); + CV_NODISCARD_STD static UMat zeros(Size size, int type); + CV_NODISCARD_STD static UMat zeros(int ndims, const int* sz, int type); + CV_NODISCARD_STD static UMat ones(int rows, int cols, int type); + CV_NODISCARD_STD static UMat ones(Size size, int type); + CV_NODISCARD_STD static UMat ones(int ndims, const int* sz, int type); + CV_NODISCARD_STD static UMat eye(int rows, int cols, int type); + CV_NODISCARD_STD static UMat eye(Size size, int type); //! allocates new matrix data unless the matrix already has specified size and type. // previous data is unreferenced if needed. @@ -2767,7 +2767,7 @@ public: SparseMat& operator = (const Mat& m); //! creates full copy of the matrix - SparseMat clone() const CV_NODISCARD; + CV_NODISCARD_STD SparseMat clone() const; //! copies all the data to the destination matrix. All the previous content of m is erased void copyTo( SparseMat& m ) const; @@ -3004,7 +3004,7 @@ public: SparseMat_& operator = (const Mat& m); //! makes full copy of the matrix. All the elements are duplicated - SparseMat_ clone() const CV_NODISCARD; + CV_NODISCARD_STD SparseMat_ clone() const; //! equivalent to cv::SparseMat::create(dims, _sizes, DataType<_Tp>::type) void create(int dims, const int* _sizes); //! converts sparse matrix to the old-style CvSparseMat. All the elements are copied diff --git a/modules/core/include/opencv2/core/matx.hpp b/modules/core/include/opencv2/core/matx.hpp index 733f675192..f25c8bce57 100644 --- a/modules/core/include/opencv2/core/matx.hpp +++ b/modules/core/include/opencv2/core/matx.hpp @@ -146,22 +146,22 @@ public: Matx(std::initializer_list<_Tp>); //!< initialize from an initializer list #endif - static Matx all(_Tp alpha); - static Matx zeros(); - static Matx ones(); - static Matx eye(); - static Matx diag(const diag_type& d); + CV_NODISCARD_STD static Matx all(_Tp alpha); + CV_NODISCARD_STD static Matx zeros(); + CV_NODISCARD_STD static Matx ones(); + CV_NODISCARD_STD static Matx eye(); + CV_NODISCARD_STD static Matx diag(const diag_type& d); /** @brief Generates uniformly distributed random numbers @param a Range boundary. @param b The other range boundary (boundaries don't have to be ordered, the lower boundary is inclusive, the upper one is exclusive). */ - static Matx randu(_Tp a, _Tp b); + CV_NODISCARD_STD static Matx randu(_Tp a, _Tp b); /** @brief Generates normally distributed random numbers @param a Mean value. @param b Standard deviation. */ - static Matx randn(_Tp a, _Tp b); + CV_NODISCARD_STD static Matx randn(_Tp a, _Tp b); //! dot product computed with the default precision _Tp dot(const Matx<_Tp, m, n>& v) const; From 2ed5cba110552f8cfdc4925fd627f5a63187b76f Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Sun, 29 Aug 2021 09:17:44 +0000 Subject: [PATCH 146/262] build: eliminate build warnings --- modules/calib3d/src/chessboard.cpp | 8 ++++---- modules/gapi/test/streaming/gapi_streaming_tests.cpp | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/modules/calib3d/src/chessboard.cpp b/modules/calib3d/src/chessboard.cpp index 18e2605f53..1801a1cbf4 100644 --- a/modules/calib3d/src/chessboard.cpp +++ b/modules/calib3d/src/chessboard.cpp @@ -1707,10 +1707,10 @@ void Chessboard::Board::normalizeOrientation(bool bblack) iter_bottom_left.getCell()->empty() || iter_bottom_right.getCell()->empty()) return; - float d1 = pow(top_left->top_left->x,2)+pow(top_left->top_left->y,2); - float d2 = pow((*iter_top_right)->x,2)+pow((*iter_top_right)->y,2); - float d3 = pow((*iter_bottom_left)->x,2)+pow((*iter_bottom_left)->y,2); - float d4 = pow((*iter_bottom_right)->x,2)+pow((*iter_bottom_right)->y,2); + float d1 = top_left->top_left->dot(*top_left->top_left); + float d2 = (*iter_top_right)->dot(*(*iter_top_right)); + float d3 = (*iter_bottom_left)->dot(*(*iter_bottom_left)); + float d4 = (*iter_bottom_right)->dot(*(*iter_bottom_right)); if(d2 <= d1 && d2 <= d3 && d2 <= d4) // top left is top right rotateLeft(); else if(d3 <= d1 && d3 <= d2 && d3 <= d4) // top left is bottom left diff --git a/modules/gapi/test/streaming/gapi_streaming_tests.cpp b/modules/gapi/test/streaming/gapi_streaming_tests.cpp index 76027a56c6..369a0c069f 100644 --- a/modules/gapi/test/streaming/gapi_streaming_tests.cpp +++ b/modules/gapi/test/streaming/gapi_streaming_tests.cpp @@ -292,7 +292,7 @@ struct StreamDataProvider : public cv::gapi::wip::onevpl::IDataProvider { size_t fetch_data(size_t out_data_size, void* out_data_buf) override { data_stream.read(reinterpret_cast(out_data_buf), out_data_size); - return data_stream.gcount(); + return (size_t)data_stream.gcount(); } bool empty() const override { return data_stream.eof() || data_stream.bad(); From f25951c412dec1329292136a171bf876b9cb7fbd Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Mon, 30 Aug 2021 11:46:14 +0000 Subject: [PATCH 147/262] core(ocl): handle NULL in dumpValue() debug call - NULL is used for allocation of workgroup local variables --- modules/core/src/ocl.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/modules/core/src/ocl.cpp b/modules/core/src/ocl.cpp index 2d85a32366..0c14e7f3e0 100644 --- a/modules/core/src/ocl.cpp +++ b/modules/core/src/ocl.cpp @@ -2981,6 +2981,8 @@ bool Kernel::empty() const static cv::String dumpValue(size_t sz, const void* p) { + if (!p) + return "NULL"; if (sz == 4) return cv::format("%d / %uu / 0x%08x / %g", *(int*)p, *(int*)p, *(int*)p, *(float*)p); if (sz == 8) From ae6fabc6fe7e372808d3b8aab08398c87074fb7d Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Mon, 30 Aug 2021 20:40:14 +0000 Subject: [PATCH 148/262] dnn(ocl): drop CL_KERNEL_PREFERRED_WORK_GROUP_SIZE_MULTIPLE check - it is a hint and it should not block kernel execution --- .../src/ocl4dnn/src/ocl4dnn_conv_spatial.cpp | 38 +++---------------- 1 file changed, 6 insertions(+), 32 deletions(-) diff --git a/modules/dnn/src/ocl4dnn/src/ocl4dnn_conv_spatial.cpp b/modules/dnn/src/ocl4dnn/src/ocl4dnn_conv_spatial.cpp index ef7c380c1b..517a663e46 100644 --- a/modules/dnn/src/ocl4dnn/src/ocl4dnn_conv_spatial.cpp +++ b/modules/dnn/src/ocl4dnn/src/ocl4dnn_conv_spatial.cpp @@ -1437,26 +1437,13 @@ bool OCL4DNNConvSpatial::createGEMMLikeConvKernel(int32_t blockM, ocl::Program program = compileKernel(); if (program.ptr()) { - size_t workgroupSize_used; ocl::Kernel kernel(kernel_name_.c_str(), program); if (kernel.empty()) return false; - workgroupSize_used = kernel.preferedWorkGroupSizeMultiple(); - if (workgroupSize_used != simd_size) - { - std::cerr << "OpenCV(ocl4dnn): The OpenCL compiler chose a simd size (" << workgroupSize_used << ") that " << std::endl; - std::cerr << " does not equal the size (" << simd_size << ") kernel source required." << std::endl; - std::cerr << " Skip this kernel " << kernel_name_ << std::endl; - unloadProgram(kernel_name_); - return false; - } - else - { - kernelQueue.push_back(makePtr(kernel_name_, &global_size[0], &local_size[0], &workItemOutput[0], - true, KERNEL_TYPE_GEMM_LIKE)); - return true; - } + kernelQueue.push_back(makePtr(kernel_name_, &global_size[0], &local_size[0], &workItemOutput[0], + true, KERNEL_TYPE_GEMM_LIKE)); + return true; } else return false; @@ -1502,26 +1489,13 @@ bool OCL4DNNConvSpatial::createIDLFKernel(int32_t blockWidth, ocl::Program program = compileKernel(); if (program.ptr()) { - size_t workgroupSize_used; ocl::Kernel kernel(kernel_name_.c_str(), program); if (kernel.empty()) return false; - workgroupSize_used = kernel.preferedWorkGroupSizeMultiple(); - if (workgroupSize_used != simd_size) - { - std::cerr << "OpenCV(ocl4dnn): The OpenCL compiler chose a simd size (" << workgroupSize_used << ") that " << std::endl; - std::cerr << " does not equal the size (" << simd_size << ") kernel source required." << std::endl; - std::cerr << " Skip this kernel " << kernel_name_ << std::endl; - unloadProgram(kernel_name_); - return false; - } - else - { - kernelQueue.push_back(makePtr(kernel_name_, &global_size[0], &local_size[0], &workItemOutput[0], - true, KERNEL_TYPE_INTEL_IDLF)); - return true; - } + kernelQueue.push_back(makePtr(kernel_name_, &global_size[0], &local_size[0], &workItemOutput[0], + true, KERNEL_TYPE_INTEL_IDLF)); + return true; } else return false; From 16b95145439c509b9adb8600e0ff6dd83845f208 Mon Sep 17 00:00:00 2001 From: Vadim Levin Date: Fri, 27 Aug 2021 15:01:09 +0300 Subject: [PATCH 149/262] feat: update conversion logic for `std::vector` in Python bindings `PyObject*` to `std::vector` conversion logic: - If user passed Numpy Array - If array is planar and T is a primitive type (doesn't require constructor call) that matches with the element type of array, then copy element one by one with the respect of the step between array elements. If compiler is lucky (or brave enough) copy loop can be vectorized. For classes that require constructor calls this path is not possible, because we can't begin an object lifetime without hacks. - Otherwise fall-back to general case - Otherwise - execute the general case: If PyObject* corresponds to Sequence protocol - iterate over the sequence elements and invoke the appropriate `pyopencv_to` function. `std::vector` to `PyObject*` conversion logic: - If `std::vector` is empty - return empty tuple. - If `T` has a corresponding `Mat` `DataType` than return Numpy array instance of the matching `dtype` e.g. `std::vector` is returned as `np.ndarray` of shape `Nx4` and `dtype=int`. This branch helps to optimize further evaluations in user code. - Otherwise - execute the general case: Construct a tuple of length N = `std::vector::size` and insert elements one by one. Unnecessary functions were removed and code was rearranged to allow compiler select the appropriate conversion function specialization. --- .../include/opencv2/core/bindings_utils.hpp | 47 ++ modules/core/src/bindings_utils.cpp | 47 ++ modules/python/src2/cv2.cpp | 493 ++++++++---------- modules/python/test/test_legacy.py | 9 +- modules/python/test/test_misc.py | 103 ++++ 5 files changed, 432 insertions(+), 267 deletions(-) diff --git a/modules/core/include/opencv2/core/bindings_utils.hpp b/modules/core/include/opencv2/core/bindings_utils.hpp index a3f83d9c2c..c53511f88f 100644 --- a/modules/core/include/opencv2/core/bindings_utils.hpp +++ b/modules/core/include/opencv2/core/bindings_utils.hpp @@ -122,6 +122,53 @@ String testReservedKeywordConversion(int positional_argument, int lambda = 2, in return format("arg=%d, lambda=%d, from=%d", positional_argument, lambda, from); } +CV_EXPORTS_W String dumpVectorOfInt(const std::vector& vec); + +CV_EXPORTS_W String dumpVectorOfDouble(const std::vector& vec); + +CV_EXPORTS_W String dumpVectorOfRect(const std::vector& vec); + +CV_WRAP static inline +void generateVectorOfRect(size_t len, CV_OUT std::vector& vec) +{ + vec.resize(len); + if (len > 0) + { + RNG rng(12345); + Mat tmp(static_cast(len), 1, CV_32SC4); + rng.fill(tmp, RNG::UNIFORM, 10, 20); + tmp.copyTo(vec); + } +} + +CV_WRAP static inline +void generateVectorOfInt(size_t len, CV_OUT std::vector& vec) +{ + vec.resize(len); + if (len > 0) + { + RNG rng(554433); + Mat tmp(static_cast(len), 1, CV_32SC1); + rng.fill(tmp, RNG::UNIFORM, -10, 10); + tmp.copyTo(vec); + } +} + +CV_WRAP static inline +void generateVectorOfMat(size_t len, int rows, int cols, int dtype, CV_OUT std::vector& vec) +{ + vec.resize(len); + if (len > 0) + { + RNG rng(65431); + for (size_t i = 0; i < len; ++i) + { + vec[i].create(rows, cols, dtype); + rng.fill(vec[i], RNG::UNIFORM, 0, 10); + } + } +} + CV_WRAP static inline void testRaiseGeneralException() { diff --git a/modules/core/src/bindings_utils.cpp b/modules/core/src/bindings_utils.cpp index 050b7247f8..ea5b82ac7d 100644 --- a/modules/core/src/bindings_utils.cpp +++ b/modules/core/src/bindings_utils.cpp @@ -5,6 +5,7 @@ #include "precomp.hpp" #include "opencv2/core/bindings_utils.hpp" #include +#include namespace cv { namespace utils { @@ -208,4 +209,50 @@ CV_EXPORTS_W String dumpInputOutputArrayOfArrays(InputOutputArrayOfArrays argume return ss.str(); } +static inline std::ostream& operator<<(std::ostream& os, const cv::Rect& rect) +{ + return os << "[x=" << rect.x << ", y=" << rect.y << ", w=" << rect.width << ", h=" << rect.height << ']'; +} + +template +static inline String dumpVector(const std::vector& vec, Formatter format) +{ + std::ostringstream oss("[", std::ios::ate); + if (!vec.empty()) + { + oss << format << vec[0]; + for (std::size_t i = 1; i < vec.size(); ++i) + { + oss << ", " << format << vec[i]; + } + } + oss << "]"; + return oss.str(); +} + +static inline std::ostream& noFormat(std::ostream& os) +{ + return os; +} + +static inline std::ostream& floatFormat(std::ostream& os) +{ + return os << std::fixed << std::setprecision(2); +} + +String dumpVectorOfInt(const std::vector& vec) +{ + return dumpVector(vec, &noFormat); +} + +String dumpVectorOfDouble(const std::vector& vec) +{ + return dumpVector(vec, &floatFormat); +} + +String dumpVectorOfRect(const std::vector& vec) +{ + return dumpVector(vec, &noFormat); +} + }} // namespace diff --git a/modules/python/src2/cv2.cpp b/modules/python/src2/cv2.cpp index 6c5e6463d2..ff57978dc5 100644 --- a/modules/python/src2/cv2.cpp +++ b/modules/python/src2/cv2.cpp @@ -493,6 +493,33 @@ bool parseSequence(PyObject* obj, RefWrapper (&value)[N], const ArgInfo& info } } // namespace +namespace traits { +template +struct BooleanConstant +{ + static const bool value = Value; + typedef BooleanConstant type; +}; + +typedef BooleanConstant TrueType; +typedef BooleanConstant FalseType; + +template +struct VoidType { + typedef void type; +}; + +template +struct IsRepresentableAsMatDataType : FalseType +{ +}; + +template +struct IsRepresentableAsMatDataType::channel_type>::type> : TrueType +{ +}; +} // namespace traits + typedef std::vector vector_uchar; typedef std::vector vector_char; typedef std::vector vector_int; @@ -1042,6 +1069,30 @@ bool pyopencv_to(PyObject* obj, uchar& value, const ArgInfo& info) return ivalue != -1 || !PyErr_Occurred(); } +template<> +bool pyopencv_to(PyObject* obj, char& value, const ArgInfo& info) +{ + if (!obj || obj == Py_None) + { + return true; + } + if (isBool(obj)) + { + failmsg("Argument '%s' must be an integer, not bool", info.name); + return false; + } + if (PyArray_IsIntegerScalar(obj)) + { + value = saturate_cast(PyArray_PyIntAsInt(obj)); + } + else + { + failmsg("Argument '%s' is required to be an integer", info.name); + return false; + } + return !CV_HAS_CONVERSION_ERROR(value); +} + template<> PyObject* pyopencv_from(const double& value) { @@ -1454,277 +1505,12 @@ PyObject* pyopencv_from(const Point3d& p) return Py_BuildValue("(ddd)", p.x, p.y, p.z); } -template struct pyopencvVecConverter -{ - typedef typename DataType<_Tp>::channel_type _Cp; - static inline bool copyOneItem(PyObject *obj, size_t start, int channels, _Cp * data) - { - for(size_t j = 0; (int)j < channels; j++ ) - { - SafeSeqItem sub_item_wrap(obj, start + j); - PyObject* item_ij = sub_item_wrap.item; - if( PyInt_Check(item_ij)) - { - int v = (int)PyInt_AsLong(item_ij); - if( v == -1 && PyErr_Occurred() ) - return false; - data[j] = saturate_cast<_Cp>(v); - } - else if( PyLong_Check(item_ij)) - { - int v = (int)PyLong_AsLong(item_ij); - if( v == -1 && PyErr_Occurred() ) - return false; - data[j] = saturate_cast<_Cp>(v); - } - else if( PyFloat_Check(item_ij)) - { - double v = PyFloat_AsDouble(item_ij); - if( PyErr_Occurred() ) - return false; - data[j] = saturate_cast<_Cp>(v); - } - else - return false; - } - return true; - } - static bool to(PyObject* obj, std::vector<_Tp>& value, const ArgInfo& info) - { - if(!obj || obj == Py_None) - return true; - if (PyArray_Check(obj)) - { - Mat m; - pyopencv_to(obj, m, info); - m.copyTo(value); - return true; - } - else if (PySequence_Check(obj)) - { - const int type = traits::Type<_Tp>::value; - const int depth = CV_MAT_DEPTH(type), channels = CV_MAT_CN(type); - size_t i, n = PySequence_Size(obj); - value.resize(n); - for (i = 0; i < n; i++ ) - { - SafeSeqItem item_wrap(obj, i); - PyObject* item = item_wrap.item; - _Cp* data = (_Cp*)&value[i]; - - if( channels == 2 && PyComplex_Check(item) ) - { - data[0] = saturate_cast<_Cp>(PyComplex_RealAsDouble(item)); - data[1] = saturate_cast<_Cp>(PyComplex_ImagAsDouble(item)); - } - else if( channels > 1 ) - { - if( PyArray_Check(item)) - { - Mat src; - pyopencv_to(item, src, info); - if( src.dims != 2 || src.channels() != 1 || - ((src.cols != 1 || src.rows != channels) && - (src.cols != channels || src.rows != 1))) - break; - Mat dst(src.rows, src.cols, depth, data); - src.convertTo(dst, type); - if( dst.data != (uchar*)data ) - break; - } - else if (PySequence_Check(item)) - { - if (!copyOneItem(item, 0, channels, data)) - break; - } - else - { - break; - } - } - else if (channels == 1) - { - if (!copyOneItem(obj, i, channels, data)) - break; - } - else - { - break; - } - } - return i == n; - } - return false; - } - - static PyObject* from(const std::vector<_Tp>& value) - { - if(value.empty()) - return PyTuple_New(0); - int type = traits::Type<_Tp>::value; - int depth = CV_MAT_DEPTH(type), channels = CV_MAT_CN(type); - Mat src((int)value.size(), channels, depth, (uchar*)&value[0]); - return pyopencv_from(src); - } -}; - -template -bool pyopencv_to(PyObject* obj, std::vector<_Tp>& value, const ArgInfo& info) -{ - return pyopencvVecConverter<_Tp>::to(obj, value, info); -} - -template -PyObject* pyopencv_from(const std::vector<_Tp>& value) -{ - return pyopencvVecConverter<_Tp>::from(value); -} - -template static inline bool pyopencv_to_generic_vec(PyObject* obj, std::vector<_Tp>& value, const ArgInfo& info) -{ - if(!obj || obj == Py_None) - return true; - if (!PySequence_Check(obj)) - return false; - size_t n = PySequence_Size(obj); - value.resize(n); - for(size_t i = 0; i < n; i++ ) - { - SafeSeqItem item_wrap(obj, i); - if(!pyopencv_to(item_wrap.item, value[i], info)) - return false; - } - return true; -} - -template static inline PyObject* pyopencv_from_generic_vec(const std::vector<_Tp>& value) -{ - int i, n = (int)value.size(); - PyObject* seq = PyList_New(n); - for( i = 0; i < n; i++ ) - { - PyObject* item = pyopencv_from(value[i]); - if(!item) - break; - PyList_SetItem(seq, i, item); - } - if( i < n ) - { - Py_DECREF(seq); - return 0; - } - return seq; -} - template<> PyObject* pyopencv_from(const std::pair& src) { return Py_BuildValue("(id)", src.first, src.second); } -template struct pyopencvVecConverter > -{ - static bool to(PyObject* obj, std::vector >& value, const ArgInfo& info) - { - return pyopencv_to_generic_vec(obj, value, info); - } - - static PyObject* from(const std::vector >& value) - { - return pyopencv_from_generic_vec(value); - } -}; - -template struct pyopencvVecConverter > -{ - static bool to(PyObject* obj, std::vector >& value, const ArgInfo& info) - { - return pyopencv_to_generic_vec(obj, value, info); - } - - static PyObject* from(const std::vector >& value) - { - return pyopencv_from_generic_vec(value); - } -}; - -template<> struct pyopencvVecConverter -{ - static bool to(PyObject* obj, std::vector& value, const ArgInfo& info) - { - return pyopencv_to_generic_vec(obj, value, info); - } - - static PyObject* from(const std::vector& value) - { - return pyopencv_from_generic_vec(value); - } -}; - -template<> struct pyopencvVecConverter -{ - static bool to(PyObject* obj, std::vector& value, const ArgInfo& info) - { - return pyopencv_to_generic_vec(obj, value, info); - } - - static PyObject* from(const std::vector& value) - { - return pyopencv_from_generic_vec(value); - } -}; - -template<> struct pyopencvVecConverter -{ - static bool to(PyObject* obj, std::vector& value, const ArgInfo& info) - { - return pyopencv_to_generic_vec(obj, value, info); - } - - static PyObject* from(const std::vector& value) - { - return pyopencv_from_generic_vec(value); - } -}; - -template<> struct pyopencvVecConverter -{ - static bool to(PyObject* obj, std::vector& value, const ArgInfo& info) - { - return pyopencv_to_generic_vec(obj, value, info); - } - - static PyObject* from(const std::vector& value) - { - return pyopencv_from_generic_vec(value); - } -}; - -template<> struct pyopencvVecConverter -{ - static bool to(PyObject* obj, std::vector& value, const ArgInfo& info) - { - return pyopencv_to_generic_vec(obj, value, info); - } - - static PyObject* from(const std::vector& value) - { - return pyopencv_from_generic_vec(value); - } -}; - -template<> struct pyopencvVecConverter -{ - static bool to(PyObject* obj, std::vector& value, const ArgInfo& info) - { - return pyopencv_to_generic_vec(obj, value, info); - } - static PyObject* from(const std::vector& value) - { - return pyopencv_from_generic_vec(value); - } -}; - template<> bool pyopencv_to(PyObject* obj, TermCriteria& dst, const ArgInfo& info) { @@ -1852,6 +1638,183 @@ PyObject* pyopencv_from(const Moments& m) "nu30", m.nu30, "nu21", m.nu21, "nu12", m.nu12, "nu03", m.nu03); } +template +struct pyopencvVecConverter; + +template +bool pyopencv_to(PyObject* obj, std::vector& value, const ArgInfo& info) +{ + if (!obj || obj == Py_None) + { + return true; + } + return pyopencvVecConverter::to(obj, value, info); +} + +template +PyObject* pyopencv_from(const std::vector& value) +{ + return pyopencvVecConverter::from(value); +} + +template +static bool pyopencv_to_generic_vec(PyObject* obj, std::vector& value, const ArgInfo& info) +{ + if (!obj || obj == Py_None) + { + return true; + } + if (!PySequence_Check(obj)) + { + failmsg("Can't parse '%s'. Input argument doesn't provide sequence protocol", info.name); + return false; + } + const size_t n = static_cast(PySequence_Size(obj)); + value.resize(n); + for (size_t i = 0; i < n; i++) + { + SafeSeqItem item_wrap(obj, i); + if (!pyopencv_to(item_wrap.item, value[i], info)) + { + failmsg("Can't parse '%s'. Sequence item with index %lu has a wrong type", info.name, i); + return false; + } + } + return true; +} + +template +static PyObject* pyopencv_from_generic_vec(const std::vector& value) +{ + Py_ssize_t n = static_cast(value.size()); + PySafeObject seq(PyTuple_New(n)); + for (Py_ssize_t i = 0; i < n; i++) + { + PyObject* item = pyopencv_from(value[i]); + // If item can't be assigned - PyTuple_SetItem raises exception and returns -1. + if (!item || PyTuple_SetItem(seq, i, item) == -1) + { + return NULL; + } + } + return seq.release(); +} + +template +struct pyopencvVecConverter +{ + typedef typename std::vector::iterator VecIt; + + static bool to(PyObject* obj, std::vector& value, const ArgInfo& info) + { + if (!PyArray_Check(obj)) + { + return pyopencv_to_generic_vec(obj, value, info); + } + // If user passed an array it is possible to make faster conversions in several cases + PyArrayObject* array_obj = reinterpret_cast(obj); + const NPY_TYPES target_type = asNumpyType(); + const NPY_TYPES source_type = static_cast(PyArray_TYPE(array_obj)); + if (target_type == NPY_OBJECT) + { + // Non-planar arrays representing objects (e.g. array of N Rect is an array of shape Nx4) have NPY_OBJECT + // as their target type. + return pyopencv_to_generic_vec(obj, value, info); + } + if (PyArray_NDIM(array_obj) > 1) + { + failmsg("Can't parse %dD array as '%s' vector argument", PyArray_NDIM(array_obj), info.name); + return false; + } + if (target_type != source_type) + { + // Source type requires conversion + // Allowed conversions for target type is handled in the corresponding pyopencv_to function + return pyopencv_to_generic_vec(obj, value, info); + } + // For all other cases, all array data can be directly copied to std::vector data + // Simple `memcpy` is not possible because NumPy array can reference a slice of the bigger array: + // ``` + // arr = np.ones((8, 4, 5), dtype=np.int32) + // convertible_to_vector_of_int = arr[:, 0, 1] + // ``` + value.resize(static_cast(PyArray_SIZE(array_obj))); + const npy_intp item_step = PyArray_STRIDE(array_obj, 0) / PyArray_ITEMSIZE(array_obj); + const Tp* data_ptr = static_cast(PyArray_DATA(array_obj)); + for (VecIt it = value.begin(); it != value.end(); ++it, data_ptr += item_step) { + *it = *data_ptr; + } + return true; + } + + static PyObject* from(const std::vector& value) + { + if (value.empty()) + { + return PyTuple_New(0); + } + return from(value, ::traits::IsRepresentableAsMatDataType()); + } + +private: + static PyObject* from(const std::vector& value, ::traits::FalseType) + { + // Underlying type is not representable as Mat Data Type + return pyopencv_from_generic_vec(value); + } + + static PyObject* from(const std::vector& value, ::traits::TrueType) + { + // Underlying type is representable as Mat Data Type, so faster return type is available + typedef DataType DType; + typedef typename DType::channel_type UnderlyingArrayType; + + // If Mat is always exposed as NumPy array this code path can be reduced to the following snipped: + // Mat src(value); + // PyObject* array = pyopencv_from(src); + // return PyArray_Squeeze(reinterpret_cast(array)); + // This puts unnecessary restrictions on Mat object those might be avoided without losing the performance. + // Moreover, this version is a bit faster, because it doesn't create temporary objects with reference counting. + + const NPY_TYPES target_type = asNumpyType(); + const int cols = DType::channels; + PyObject* array; + if (cols == 1) + { + npy_intp dims = static_cast(value.size()); + array = PyArray_SimpleNew(1, &dims, target_type); + } + else + { + npy_intp dims[2] = {static_cast(value.size()), cols}; + array = PyArray_SimpleNew(2, dims, target_type); + } + if(!array) + { + // NumPy arrays with shape (N, 1) and (N) are not equal, so correct error message should distinguish + // them too. + String shape; + if (cols > 1) + { + shape = cv::format("(%d x %d)", static_cast(value.size()), cols); + } + else + { + shape = cv::format("(%d)", static_cast(value.size())); + } + CV_Error_(Error::StsError, ("Can't allocate NumPy array for vector with dtype=%d shape=%s", + static_cast(target_type), static_cast(value.size()), shape.c_str())); + } + // Fill the array + PyArrayObject* array_obj = reinterpret_cast(array); + UnderlyingArrayType* array_data = static_cast(PyArray_DATA(array_obj)); + // if Tp is representable as Mat DataType, so the following cast is pretty safe... + const UnderlyingArrayType* value_data = reinterpret_cast(value.data()); + memcpy(array_data, value_data, sizeof(UnderlyingArrayType) * value.size() * static_cast(cols)); + return array; + } +}; + static int OnError(int status, const char *func_name, const char *err_msg, const char *file_name, int line, void *userdata) { PyGILState_STATE gstate; diff --git a/modules/python/test/test_legacy.py b/modules/python/test/test_legacy.py index ab0a8bdc35..e550ab73c8 100644 --- a/modules/python/test/test_legacy.py +++ b/modules/python/test/test_legacy.py @@ -20,8 +20,13 @@ class Hackathon244Tests(NewOpenCVTests): flag, ajpg = cv.imencode("img_q90.jpg", a, [cv.IMWRITE_JPEG_QUALITY, 90]) self.assertEqual(flag, True) self.assertEqual(ajpg.dtype, np.uint8) - self.assertGreater(ajpg.shape[0], 1) - self.assertEqual(ajpg.shape[1], 1) + self.assertTrue(isinstance(ajpg, np.ndarray), "imencode returned buffer of wrong type: {}".format(type(ajpg))) + self.assertEqual(len(ajpg.shape), 1, "imencode returned buffer with wrong shape: {}".format(ajpg.shape)) + self.assertGreaterEqual(len(ajpg), 1, "imencode length of the returned buffer should be at least 1") + self.assertLessEqual( + len(ajpg), a.size, + "imencode length of the returned buffer shouldn't exceed number of elements in original image" + ) def test_projectPoints(self): objpt = np.float64([[1,2,3]]) diff --git a/modules/python/test/test_misc.py b/modules/python/test/test_misc.py index 4d435a46b6..c992c9450d 100644 --- a/modules/python/test/test_misc.py +++ b/modules/python/test/test_misc.py @@ -480,6 +480,109 @@ class Arguments(NewOpenCVTests): cv.utils.testReservedKeywordConversion(20, lambda_=-4, from_=12), format_str.format(20, -4, 12) ) + def test_parse_vector_int_convertible(self): + np.random.seed(123098765) + try_to_convert = partial(self._try_to_convert, cv.utils.dumpVectorOfInt) + arr = np.random.randint(-20, 20, 40).astype(np.int32).reshape(10, 2, 2) + int_min, int_max = get_limits(ctypes.c_int) + for convertible in ((int_min, 1, 2, 3, int_max), [40, 50], tuple(), + np.array([int_min, -10, 24, int_max], dtype=np.int32), + np.array([10, 230, 12], dtype=np.uint8), arr[:, 0, 1],): + expected = "[" + ", ".join(map(str, convertible)) + "]" + actual = try_to_convert(convertible) + self.assertEqual(expected, actual, + msg=get_conversion_error_msg(convertible, expected, actual)) + + def test_parse_vector_int_not_convertible(self): + np.random.seed(123098765) + arr = np.random.randint(-20, 20, 40).astype(np.float).reshape(10, 2, 2) + int_min, int_max = get_limits(ctypes.c_int) + test_dict = {1: 2, 3: 10, 10: 20} + for not_convertible in ((int_min, 1, 2.5, 3, int_max), [True, 50], 'test', test_dict, + reversed([1, 2, 3]), + np.array([int_min, -10, 24, [1, 2]], dtype=np.object), + np.array([[1, 2], [3, 4]]), arr[:, 0, 1],): + with self.assertRaises(TypeError, msg=get_no_exception_msg(not_convertible)): + _ = cv.utils.dumpVectorOfInt(not_convertible) + + def test_parse_vector_double_convertible(self): + np.random.seed(1230965) + try_to_convert = partial(self._try_to_convert, cv.utils.dumpVectorOfDouble) + arr = np.random.randint(-20, 20, 40).astype(np.int32).reshape(10, 2, 2) + for convertible in ((1, 2.12, 3.5), [40, 50], tuple(), + np.array([-10, 24], dtype=np.int32), + np.array([-12.5, 1.4], dtype=np.double), + np.array([10, 230, 12], dtype=np.float), arr[:, 0, 1], ): + expected = "[" + ", ".join(map(lambda v: "{:.2f}".format(v), convertible)) + "]" + actual = try_to_convert(convertible) + self.assertEqual(expected, actual, + msg=get_conversion_error_msg(convertible, expected, actual)) + + def test_parse_vector_double_not_convertible(self): + test_dict = {1: 2, 3: 10, 10: 20} + for not_convertible in (('t', 'e', 's', 't'), [True, 50.55], 'test', test_dict, + np.array([-10.1, 24.5, [1, 2]], dtype=np.object), + np.array([[1, 2], [3, 4]]),): + with self.assertRaises(TypeError, msg=get_no_exception_msg(not_convertible)): + _ = cv.utils.dumpVectorOfDouble(not_convertible) + + def test_parse_vector_rect_convertible(self): + np.random.seed(1238765) + try_to_convert = partial(self._try_to_convert, cv.utils.dumpVectorOfRect) + arr_of_rect_int32 = np.random.randint(5, 20, 4 * 3).astype(np.int32).reshape(3, 4) + arr_of_rect_cast = np.random.randint(10, 40, 4 * 5).astype(np.uint8).reshape(5, 4) + for convertible in (((1, 2, 3, 4), (10, -20, 30, 10)), arr_of_rect_int32, arr_of_rect_cast, + arr_of_rect_int32.astype(np.int8), [[5, 3, 1, 4]], + ((np.int8(4), np.uint8(10), np.int(32), np.int16(55)),)): + expected = "[" + ", ".join(map(lambda v: "[x={}, y={}, w={}, h={}]".format(*v), convertible)) + "]" + actual = try_to_convert(convertible) + self.assertEqual(expected, actual, + msg=get_conversion_error_msg(convertible, expected, actual)) + + def test_parse_vector_rect_not_convertible(self): + np.random.seed(1238765) + arr = np.random.randint(5, 20, 4 * 3).astype(np.float).reshape(3, 4) + for not_convertible in (((1, 2, 3, 4), (10.5, -20, 30.1, 10)), arr, + [[5, 3, 1, 4], []], + ((np.float(4), np.uint8(10), np.int(32), np.int16(55)),)): + with self.assertRaises(TypeError, msg=get_no_exception_msg(not_convertible)): + _ = cv.utils.dumpVectorOfRect(not_convertible) + + def test_vector_general_return(self): + expected_number_of_mats = 5 + expected_shape = (10, 10, 3) + expected_type = np.uint8 + mats = cv.utils.generateVectorOfMat(5, 10, 10, cv.CV_8UC3) + self.assertTrue(isinstance(mats, tuple), + "Vector of Mats objects should be returned as tuple. Got: {}".format(type(mats))) + self.assertEqual(len(mats), expected_number_of_mats, "Returned array has wrong length") + for mat in mats: + self.assertEqual(mat.shape, expected_shape, "Returned Mat has wrong shape") + self.assertEqual(mat.dtype, expected_type, "Returned Mat has wrong elements type") + empty_mats = cv.utils.generateVectorOfMat(0, 10, 10, cv.CV_32FC1) + self.assertTrue(isinstance(empty_mats, tuple), + "Empty vector should be returned as empty tuple. Got: {}".format(type(mats))) + self.assertEqual(len(empty_mats), 0, "Vector of size 0 should be returned as tuple of length 0") + + def test_vector_fast_return(self): + expected_shape = (5, 4) + rects = cv.utils.generateVectorOfRect(expected_shape[0]) + self.assertTrue(isinstance(rects, np.ndarray), + "Vector of rectangles should be returned as numpy array. Got: {}".format(type(rects))) + self.assertEqual(rects.dtype, np.int32, "Vector of rectangles has wrong elements type") + self.assertEqual(rects.shape, expected_shape, "Vector of rectangles has wrong shape") + empty_rects = cv.utils.generateVectorOfRect(0) + self.assertTrue(isinstance(empty_rects, tuple), + "Empty vector should be returned as empty tuple. Got: {}".format(type(empty_rects))) + self.assertEqual(len(empty_rects), 0, "Vector of size 0 should be returned as tuple of length 0") + + expected_shape = (10,) + ints = cv.utils.generateVectorOfInt(expected_shape[0]) + self.assertTrue(isinstance(ints, np.ndarray), + "Vector of integers should be returned as numpy array. Got: {}".format(type(ints))) + self.assertEqual(ints.dtype, np.int32, "Vector of integers has wrong elements type") + self.assertEqual(ints.shape, expected_shape, "Vector of integers has wrong shape.") + class SamplesFindFile(NewOpenCVTests): From edc442afdb559c7a8d731b86637839b5f224de35 Mon Sep 17 00:00:00 2001 From: WJJ1995 Date: Wed, 1 Sep 2021 18:10:05 +0800 Subject: [PATCH 150/262] Merge pull request #20511 from wjj19950828:add_humanseg_support_0806 * support PPSeg model for dnn module * fixed README for CI * add test case * fixed bug * deal with comments * rm dnn_model_runner * update test case * fixed bug for testcase * update testcase --- modules/dnn/src/onnx/onnx_importer.cpp | 2 +- modules/dnn/test/test_onnx_importer.cpp | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/modules/dnn/src/onnx/onnx_importer.cpp b/modules/dnn/src/onnx/onnx_importer.cpp index 6da2c5edf6..41ff6c9b1e 100644 --- a/modules/dnn/src/onnx/onnx_importer.cpp +++ b/modules/dnn/src/onnx/onnx_importer.cpp @@ -2054,7 +2054,7 @@ void ONNXImporter::parseResize(LayerParams& layerParams, const opencv_onnx::Node layerParams.set("align_corners", interp_mode == "align_corners"); if (layerParams.get("mode") == "linear") { - layerParams.set("mode", interp_mode == "pytorch_half_pixel" ? + layerParams.set("mode", interp_mode == "pytorch_half_pixel" || interp_mode == "half_pixel" ? "opencv_linear" : "bilinear"); } } diff --git a/modules/dnn/test/test_onnx_importer.cpp b/modules/dnn/test/test_onnx_importer.cpp index 983f72d6d6..07a0290b9b 100644 --- a/modules/dnn/test/test_onnx_importer.cpp +++ b/modules/dnn/test/test_onnx_importer.cpp @@ -552,6 +552,11 @@ TEST_P(Test_ONNX_layers, DynamicResize) testONNXModels("dynamic_resize_scale_11", npy, 0, 0, false, true, 2); } +TEST_P(Test_ONNX_layers, Resize_HumanSeg) +{ + testONNXModels("resize_humanseg"); +} + TEST_P(Test_ONNX_layers, Div) { const String model = _tf("models/div.onnx"); From 390957fec4601b58982c3d15130485f531d22ba5 Mon Sep 17 00:00:00 2001 From: Vadim Levin Date: Thu, 2 Sep 2021 10:32:17 +0300 Subject: [PATCH 151/262] fix: NumPy array allocation error message in vector conversion --- modules/python/src2/cv2.cpp | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/modules/python/src2/cv2.cpp b/modules/python/src2/cv2.cpp index ff57978dc5..67c4166799 100644 --- a/modules/python/src2/cv2.cpp +++ b/modules/python/src2/cv2.cpp @@ -1778,7 +1778,7 @@ private: const NPY_TYPES target_type = asNumpyType(); const int cols = DType::channels; - PyObject* array; + PyObject* array = NULL; if (cols == 1) { npy_intp dims = static_cast(value.size()); @@ -1796,14 +1796,16 @@ private: String shape; if (cols > 1) { - shape = cv::format("(%d x %d)", static_cast(value.size()), cols); + shape = format("(%d x %d)", static_cast(value.size()), cols); } else { - shape = cv::format("(%d)", static_cast(value.size())); + shape = format("(%d)", static_cast(value.size())); } - CV_Error_(Error::StsError, ("Can't allocate NumPy array for vector with dtype=%d shape=%s", - static_cast(target_type), static_cast(value.size()), shape.c_str())); + const String error_message = format("Can't allocate NumPy array for vector with dtype=%d and shape=%s", + static_cast(target_type), shape.c_str()); + emit_failmsg(PyExc_MemoryError, error_message.c_str()); + return array; } // Fill the array PyArrayObject* array_obj = reinterpret_cast(array); From d0e612dc368d657ca56101879dbc58c46a4dbb9b Mon Sep 17 00:00:00 2001 From: rogday Date: Fri, 3 Sep 2021 15:32:29 +0300 Subject: [PATCH 152/262] Merge pull request #20647 from rogday:resize_concat_optimization Fix resize+concat optimization * fix resize+concat optimization * add comment and fix indentation --- modules/dnn/src/layers/resize_layer.cpp | 7 +++++++ modules/dnn/test/test_tf_importer.cpp | 5 +++++ 2 files changed, 12 insertions(+) diff --git a/modules/dnn/src/layers/resize_layer.cpp b/modules/dnn/src/layers/resize_layer.cpp index 40c7351984..8b7d802ab2 100644 --- a/modules/dnn/src/layers/resize_layer.cpp +++ b/modules/dnn/src/layers/resize_layer.cpp @@ -111,7 +111,14 @@ public: internals_arr.getMatVector(internals); if (outHeight == inputs[0].size[2] && outWidth == inputs[0].size[3]) + { + // outputs[0] = inputs[0] doesn't work due to BlobManager optimizations + if (inputs[0].data != outputs[0].data) + { + inputs[0].copyTo(outputs[0]); + } return; + } Mat& inp = inputs[0]; Mat& out = outputs[0]; diff --git a/modules/dnn/test/test_tf_importer.cpp b/modules/dnn/test/test_tf_importer.cpp index 68d6e88a66..1a2b976eb8 100644 --- a/modules/dnn/test/test_tf_importer.cpp +++ b/modules/dnn/test/test_tf_importer.cpp @@ -1125,6 +1125,11 @@ TEST_P(Test_TensorFlow_layers, resize_bilinear_down) runTensorFlowNet("resize_bilinear_down"); } +TEST_P(Test_TensorFlow_layers, resize_concat_optimization) +{ + runTensorFlowNet("resize_concat_optimization"); +} + TEST_P(Test_TensorFlow_layers, tf2_dense) { runTensorFlowNet("tf2_dense"); From 407adc7061c9d2126a5d27c53ff76b56a705f3e2 Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Sat, 4 Sep 2021 04:35:00 +0000 Subject: [PATCH 153/262] dnn(ocl): fix buffer offsets in IDLF kernel - drop CreateSubBuffer - fix FUSED_CONV_ELTWISE mode --- modules/dnn/src/ocl4dnn/include/ocl4dnn.hpp | 2 +- .../src/ocl4dnn/src/ocl4dnn_conv_spatial.cpp | 86 +++++-------------- modules/dnn/src/opencl/conv_layer_spatial.cl | 25 ++++-- 3 files changed, 41 insertions(+), 72 deletions(-) diff --git a/modules/dnn/src/ocl4dnn/include/ocl4dnn.hpp b/modules/dnn/src/ocl4dnn/include/ocl4dnn.hpp index 7bb277d102..d6fb83becb 100644 --- a/modules/dnn/src/ocl4dnn/include/ocl4dnn.hpp +++ b/modules/dnn/src/ocl4dnn/include/ocl4dnn.hpp @@ -269,7 +269,7 @@ class OCL4DNNConvSpatial void generate_idlf_tuneritems(std::vector< cv::Ptr > &tunerItems, int blockM, int blockK, int simd_size); void setFusionDefine(ocl4dnnFusedActiv_t fused_activ, bool fused_eltwise); - void setFusionArg(ocl4dnnFusedActiv_t fused_activ, bool fused_eltwise, ocl::Kernel &kernel, cl_uint &argIdx); + void setFusionArg(ocl4dnnFusedActiv_t fused_activ, bool fused_eltwise, int fused_eltwise_offset, ocl::Kernel &kernel, cl_uint &argIdx); int32_t group_; bool bias_term_; diff --git a/modules/dnn/src/ocl4dnn/src/ocl4dnn_conv_spatial.cpp b/modules/dnn/src/ocl4dnn/src/ocl4dnn_conv_spatial.cpp index 517a663e46..b4477ebfc4 100644 --- a/modules/dnn/src/ocl4dnn/src/ocl4dnn_conv_spatial.cpp +++ b/modules/dnn/src/ocl4dnn/src/ocl4dnn_conv_spatial.cpp @@ -270,17 +270,21 @@ void OCL4DNNConvSpatial::setFusionDefine(ocl4dnnFusedActiv_t fused_activ, } template -void OCL4DNNConvSpatial::setFusionArg(ocl4dnnFusedActiv_t fused_activ, bool fused_eltwise, ocl::Kernel &kernel, cl_uint &argIdx) +void OCL4DNNConvSpatial::setFusionArg(ocl4dnnFusedActiv_t fused_activ, bool fused_eltwise, int fused_eltwise_offset, ocl::Kernel &kernel, cl_uint &argIdx) { if (fused_eltwise) - kernel.set(argIdx++, (cl_mem)bottom_data2_.handle(ACCESS_READ)); + { + kernel.set(argIdx++, ocl::KernelArg::PtrReadOnly(bottom_data2_)); + if (fused_eltwise_offset >= 0) + kernel.set(argIdx++, fused_eltwise_offset); + } switch (fused_activ) { case OCL4DNN_CONV_FUSED_ACTIV_RELU: kernel.set(argIdx++, (float)negative_slope_); break; case OCL4DNN_CONV_FUSED_ACTIV_PRELU: - kernel.set(argIdx++, (cl_mem)negative_slope_umat_.handle(ACCESS_READ)); + kernel.set(argIdx++, ocl::KernelArg::PtrReadOnly(negative_slope_umat_)); break; case OCL4DNN_CONV_FUSED_ACTIV_POWER: kernel.set(argIdx++, (float)power_); @@ -895,10 +899,12 @@ bool OCL4DNNConvSpatial::convolve(const UMat &bottom, UMat &top, if (config->kernelType == KERNEL_TYPE_INTEL_IDLF) { if (!swizzleWeight(weight, config->workItem_output[2], false)) return false; +#if 0 size_t total_bottom_size = bottom_dim_ * numImages; size_t total_kernel_size = kernel_h_ * kernel_w_ * channels_ * M_; size_t total_bias_size = M_ * group_; size_t total_top_size = top_dim_ * numImages; +#endif for (int32_t g = 0; g < group_; ++g) { bias_offset = M_ * g; int32_t image_offset = width_ * height_ * (channels_ / group_) * g; @@ -910,72 +916,22 @@ bool OCL4DNNConvSpatial::convolve(const UMat &bottom, UMat &top, return false; cl_uint argIdx = 0; - setFusionArg(fused_activ_, fused_eltwise_, kernel, argIdx); + setFusionArg(fused_activ_, fused_eltwise_, output_image_offset, kernel, argIdx); - UMat img_buffer; - if (image_offset) - { - CreateSubBuffer(bottom, img_buffer, image_offset, - total_bottom_size - image_offset, false); - if (img_buffer.empty()) - return false; + kernel.set(argIdx++, ocl::KernelArg::PtrReadOnly(bottom)); + kernel.set(argIdx++, image_offset); - kernel.set(argIdx++, ocl::KernelArg::PtrReadOnly(img_buffer)); - } - else - { - kernel.set(argIdx++, ocl::KernelArg::PtrReadOnly(bottom)); - } + kernel.set(argIdx++, ocl::KernelArg::PtrReadOnly(swizzled_weights_umat)); + kernel.set(argIdx++, kernel_offset); - UMat kernel_buffer; - if (kernel_offset) - { - CreateSubBuffer(swizzled_weights_umat, kernel_buffer, kernel_offset, - total_kernel_size - kernel_offset, false); - if (kernel_buffer.empty()) - return false; - - kernel.set(argIdx++, ocl::KernelArg::PtrReadOnly(kernel_buffer)); - } - else - { - kernel.set(argIdx++, ocl::KernelArg::PtrReadOnly(swizzled_weights_umat)); - } - - UMat bias_buffer; if (bias_term_) { - if (bias_offset) - { - CreateSubBuffer(bias, bias_buffer, bias_offset, - total_bias_size - bias_offset, false); - if (bias_buffer.empty()) - return false; - - kernel.set(argIdx++, ocl::KernelArg::PtrReadOnly(bias_buffer)); - } - else - { - kernel.set(argIdx++, ocl::KernelArg::PtrReadOnly(bias)); - } + kernel.set(argIdx++, ocl::KernelArg::PtrReadOnly(bias)); + kernel.set(argIdx++, bias_offset); } - UMat out_buffer; - if (output_image_offset) - { - CreateSubBuffer(top, out_buffer, output_image_offset, - total_top_size - output_image_offset, true); - if (out_buffer.empty()) - return false; - - kernel.set(argIdx++, ocl::KernelArg::PtrWriteOnly(out_buffer)); - kernel.set(argIdx++, (int)(out_buffer.offset / element_size)); - } - else - { - kernel.set(argIdx++, ocl::KernelArg::PtrWriteOnly(top)); - kernel.set(argIdx++, (int)(top.offset / element_size)); - } + kernel.set(argIdx++, ocl::KernelArg::PtrWriteOnly(top)); + kernel.set(argIdx++, (int)(top.offset / element_size) + output_image_offset); kernel.set(argIdx++, (uint16_t)width_); kernel.set(argIdx++, (uint16_t)height_); @@ -1005,7 +961,7 @@ bool OCL4DNNConvSpatial::convolve(const UMat &bottom, UMat &top, return false; cl_uint argIdx = 0; - setFusionArg(fused_activ_, fused_eltwise_, kernel, argIdx); + setFusionArg(fused_activ_, fused_eltwise_, -1, kernel, argIdx); UMat img_buffer; if (image_offset) @@ -1112,7 +1068,7 @@ bool OCL4DNNConvSpatial::convolve(const UMat &bottom, UMat &top, return false; cl_uint argIdx = 0; - setFusionArg(fused_activ_, fused_eltwise_, kernel, argIdx); + setFusionArg(fused_activ_, fused_eltwise_, -1, kernel, argIdx); kernel.set(argIdx++, ocl::KernelArg::PtrReadOnly(bottom)); kernel.set(argIdx++, ocl::KernelArg::PtrReadOnly(weight)); if (bias_term_) @@ -1152,7 +1108,7 @@ bool OCL4DNNConvSpatial::convolve(const UMat &bottom, UMat &top, return false; cl_uint argIdx = 0; - setFusionArg(fused_activ_, fused_eltwise_, kernel, argIdx); + setFusionArg(fused_activ_, fused_eltwise_, -1, kernel, argIdx); kernel.set(argIdx++, ocl::KernelArg::PtrReadOnly(bottom)); kernel.set(argIdx++, image_offset); kernel.set(argIdx++, ocl::KernelArg::PtrReadOnly(weight)); diff --git a/modules/dnn/src/opencl/conv_layer_spatial.cl b/modules/dnn/src/opencl/conv_layer_spatial.cl index 236e8d029a..55015557a0 100644 --- a/modules/dnn/src/opencl/conv_layer_spatial.cl +++ b/modules/dnn/src/opencl/conv_layer_spatial.cl @@ -74,18 +74,22 @@ (_dst_)[(_offset_)] = ACTIVATION_RELU_FUNCTION(_x_, _channel_); \ } while(0) #define ELTWISE_DATA_ARG __global Dtype* eltwise_data, +#define ELTWISE_DATA_ARG_WITH_OFFSET __global Dtype* eltwise_ptr, int eltwise_offset, #else #define ACTIVATION_FUNCTION(_dst_, _offset_, _data_, _channel_) do { \ const Dtype _x_ = (_data_); \ (_dst_)[(_offset_)] = ACTIVATION_RELU_FUNCTION(_x_, _channel_); \ } while(0) #define ELTWISE_DATA_ARG +#define ELTWISE_DATA_ARG_WITH_OFFSET #endif #if APPLY_BIAS #define BIAS_KERNEL_ARG __global Dtype * biases_base, +#define BIAS_KERNEL_ARG_WITH_OFFSET __global Dtype * biases_base_ptr, int biases_base_offset, #else #define BIAS_KERNEL_ARG +#define BIAS_KERNEL_ARG_WITH_OFFSET #endif #define __CAT(x, y) x##y @@ -223,19 +227,28 @@ __attribute__((reqd_work_group_size(1, 1, SIMD_SIZE))) __attribute__((intel_reqd_sub_group_size(SIMD_SIZE))) __kernel void convolve_simd( - ELTWISE_DATA_ARG + ELTWISE_DATA_ARG_WITH_OFFSET FUSED_ARG - __global Dtype* inputs, - __global Dtype* weights, - BIAS_KERNEL_ARG - __global Dtype* outputs_base, - const int outputs_offset, + __global Dtype* inputs_ptr, const int inputs_offset, + __global Dtype* weights_ptr, const int weights_offset, + BIAS_KERNEL_ARG_WITH_OFFSET + __global Dtype* outputs_base, const int outputs_offset, const ushort input_width, const ushort input_height, const ushort output_width, const ushort output_height) { + __global Dtype* inputs = inputs_ptr + inputs_offset; + __global Dtype* weights = weights_ptr + weights_offset; +#if APPLY_BIAS + __global Dtype* biases_base = biases_base_ptr + biases_base_offset; +#endif + __global Dtype* outputs = outputs_base + outputs_offset; +#ifdef FUSED_CONV_ELTWISE + __global Dtype* eltwise_data = eltwise_ptr + eltwise_offset; +#endif + unsigned int oc = get_global_id(0) * OUT_BLOCK_WIDTH; // oc = Output Column unsigned int or = get_global_id(1) * OUT_BLOCK_HEIGHT; // or = Output Row unsigned int fm = get_global_id(2); // fm = Feature Map = od = Output Depth From aaff1256088ce892f977c73ff01817c2e389ac93 Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Sat, 4 Sep 2021 01:34:02 +0000 Subject: [PATCH 154/262] core(ocl): debug capabilities --- modules/core/src/ocl.cpp | 31 ++++++++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/modules/core/src/ocl.cpp b/modules/core/src/ocl.cpp index 0c14e7f3e0..daf4fcd280 100644 --- a/modules/core/src/ocl.cpp +++ b/modules/core/src/ocl.cpp @@ -76,8 +76,11 @@ #undef CV__ALLOCATOR_STATS_LOG #define CV_OPENCL_ALWAYS_SHOW_BUILD_LOG 0 +#define CV_OPENCL_SHOW_BUILD_OPTIONS 0 +#define CV_OPENCL_SHOW_BUILD_KERNELS 0 #define CV_OPENCL_SHOW_RUN_KERNELS 0 +#define CV_OPENCL_SYNC_RUN_KERNELS 0 #define CV_OPENCL_TRACE_CHECK 0 #define CV_OPENCL_VALIDATE_BINARY_PROGRAMS 1 @@ -2983,6 +2986,8 @@ static cv::String dumpValue(size_t sz, const void* p) { if (!p) return "NULL"; + if (sz == 2) + return cv::format("%d / %uu / 0x%04x", *(short*)p, *(unsigned short*)p, *(short*)p); if (sz == 4) return cv::format("%d / %uu / 0x%08x / %g", *(int*)p, *(int*)p, *(int*)p, *(float*)p); if (sz == 8) @@ -3195,6 +3200,10 @@ bool Kernel::Impl::run(int dims, size_t globalsize[], size_t localsize[], return false; // OpenCV 5.0: raise error } +#if CV_OPENCL_SYNC_RUN_KERNELS + sync = true; +#endif + cl_command_queue qq = getQueue(q); if (haveTempDstUMats) sync = true; @@ -3625,7 +3634,28 @@ struct Program::Impl if (!param_buildExtraOptions.empty()) buildflags = joinBuildOptions(buildflags, param_buildExtraOptions); } +#if CV_OPENCL_SHOW_BUILD_OPTIONS + CV_LOG_INFO(NULL, "OpenCL program '" << sourceModule_ << "/" << sourceName_ << "' options:" << buildflags); +#endif compile(ctx, src_, errmsg); +#if CV_OPENCL_SHOW_BUILD_KERNELS + if (handle) + { + size_t retsz = 0; + char kernels_buffer[4096] = {0}; + cl_int result = clGetProgramInfo(handle, CL_PROGRAM_KERNEL_NAMES, sizeof(kernels_buffer), &kernels_buffer[0], &retsz); + CV_OCL_DBG_CHECK_RESULT(result, cv::format("clGetProgramInfo(CL_PROGRAM_KERNEL_NAMES: %s/%s)", sourceModule_.c_str(), sourceName_.c_str()).c_str()); + if (result == CL_SUCCESS && retsz < sizeof(kernels_buffer)) + { + kernels_buffer[retsz] = 0; + CV_LOG_INFO(NULL, "OpenCL program '" << sourceModule_ << "/" << sourceName_ << "' kernels: '" << kernels_buffer << "'"); + } + else + { + CV_LOG_ERROR(NULL, "OpenCL program '" << sourceModule_ << "/" << sourceName_ << "' can't retrieve kernel names!"); + } + } +#endif } bool compile(const Context& ctx, const ProgramSource::Impl* src_, String& errmsg) @@ -3857,7 +3887,6 @@ struct Program::Impl CV_LOG_INFO(NULL, result << ": Kernels='" << kernels_buffer << "'"); } #endif - } return handle != NULL; } From 5b2c0168340a8b4e67def8f52885ab21b4bbcadc Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Fri, 3 Sep 2021 02:38:53 +0000 Subject: [PATCH 155/262] dnn(ocl): avoid out of buffer access in copyWeightsSwizzled --- modules/dnn/src/opencl/conv_spatial_helper.cl | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/modules/dnn/src/opencl/conv_spatial_helper.cl b/modules/dnn/src/opencl/conv_spatial_helper.cl index 33d9db57c8..660d085956 100644 --- a/modules/dnn/src/opencl/conv_spatial_helper.cl +++ b/modules/dnn/src/opencl/conv_spatial_helper.cl @@ -62,8 +62,8 @@ __kernel void TEMPLATE(copyWeightsSwizzled, Dtype) //Original location //Output location - int outputSublayer = channels / swizzleFactor; - int outputSublayerIndex = channels % swizzleFactor; + //int outputSublayer = channels / swizzleFactor; + //int outputSublayerIndex = channels % swizzleFactor; int filter = sX / (kernel_w*kernel_h*channels); int kernel_X = sX % kernel_w; @@ -73,6 +73,10 @@ __kernel void TEMPLATE(copyWeightsSwizzled, Dtype) int FP = filter / swizzleFactor; int F1 = filter % swizzleFactor; - weightOut[FP*(kernel_w*kernel_h*channels*swizzleFactor) + kernel_C*(kernel_w*kernel_h*swizzleFactor) + kernel_Y*(kernel_w*swizzleFactor) + kernel_X*swizzleFactor + F1] - = weightIn[filter*(kernel_w*kernel_h*channels) + kernel_C*(kernel_w*kernel_h) + kernel_Y*kernel_w + kernel_X]; + int idxOut = FP*(kernel_w*kernel_h*channels*swizzleFactor) + kernel_C*(kernel_w*kernel_h*swizzleFactor) + kernel_Y*(kernel_w*swizzleFactor) + kernel_X*swizzleFactor + F1; + int idxIn = filter*(kernel_w*kernel_h*channels) + kernel_C*(kernel_w*kernel_h) + kernel_Y*kernel_w + kernel_X; + + // idxIn is not valid if (filter >= outputs) - no data for these elements. Output alignment gaps are filled by zeros + Dtype v = (filter < outputs) ? weightIn[idxIn] : (Dtype)0; + weightOut[idxOut] = v; } From 5578ad5e14fe1fcf4d7171f74c665db2a578187f Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Sat, 4 Sep 2021 16:27:51 +0000 Subject: [PATCH 156/262] dnn(ocl): fix automatic globalsize adjusting - if kernel code doesn't support that --- modules/core/include/opencv2/core/ocl.hpp | 20 +++++++++++++++++-- modules/core/src/ocl.cpp | 8 ++++++++ modules/dnn/src/layers/batch_norm_layer.cpp | 2 +- modules/dnn/src/layers/mvn_layer.cpp | 2 +- modules/dnn/src/layers/slice_layer.cpp | 2 +- .../dnn/src/ocl4dnn/src/math_functions.cpp | 1 + .../src/ocl4dnn/src/ocl4dnn_conv_spatial.cpp | 19 +++++++++--------- .../dnn/src/ocl4dnn/src/ocl4dnn_softmax.cpp | 2 +- modules/dnn/src/opencl/gemm_image.cl | 10 ++++++++++ 9 files changed, 50 insertions(+), 16 deletions(-) diff --git a/modules/core/include/opencv2/core/ocl.hpp b/modules/core/include/opencv2/core/ocl.hpp index f03de180fc..b51b39359d 100644 --- a/modules/core/include/opencv2/core/ocl.hpp +++ b/modules/core/include/opencv2/core/ocl.hpp @@ -562,7 +562,9 @@ public: i = set(i, a6); i = set(i, a7); i = set(i, a8); i = set(i, a9); i = set(i, a10); i = set(i, a11); i = set(i, a12); i = set(i, a13); i = set(i, a14); set(i, a15); return *this; } - /** @brief Run the OpenCL kernel. + + /** @brief Run the OpenCL kernel (globalsize value may be adjusted) + @param dims the work problem dimensions. It is the length of globalsize and localsize. It can be either 1, 2 or 3. @param globalsize work items for each dimension. It is not the final globalsize passed to OpenCL. Each dimension will be adjusted to the nearest integer divisible by the corresponding @@ -571,12 +573,26 @@ public: @param localsize work-group size for each dimension. @param sync specify whether to wait for OpenCL computation to finish before return. @param q command queue + + @note Use run_() if your kernel code doesn't support adjusted globalsize. */ bool run(int dims, size_t globalsize[], size_t localsize[], bool sync, const Queue& q=Queue()); + + /** @brief Run the OpenCL kernel + * + * @param dims the work problem dimensions. It is the length of globalsize and localsize. It can be either 1, 2 or 3. + * @param globalsize work items for each dimension. This value is passed to OpenCL without changes. + * @param localsize work-group size for each dimension. + * @param sync specify whether to wait for OpenCL computation to finish before return. + * @param q command queue + */ + bool run_(int dims, size_t globalsize[], size_t localsize[], bool sync, const Queue& q=Queue()); + bool runTask(bool sync, const Queue& q=Queue()); - /** @brief Similar to synchronized run() call with returning of kernel execution time + /** @brief Similar to synchronized run_() call with returning of kernel execution time + * * Separate OpenCL command queue may be used (with CL_QUEUE_PROFILING_ENABLE) * @return Execution time in nanoseconds or negative number on error */ diff --git a/modules/core/src/ocl.cpp b/modules/core/src/ocl.cpp index daf4fcd280..a550c1d91a 100644 --- a/modules/core/src/ocl.cpp +++ b/modules/core/src/ocl.cpp @@ -3160,6 +3160,14 @@ bool Kernel::run(int dims, size_t _globalsize[], size_t _localsize[], } +bool Kernel::run_(int dims, size_t _globalsize[], size_t _localsize[], + bool sync, const Queue& q) +{ + CV_Assert(p); + return p->run(dims, _globalsize, _localsize, sync, NULL, q); +} + + static bool isRaiseErrorOnReuseAsyncKernel() { static bool initialized = false; diff --git a/modules/dnn/src/layers/batch_norm_layer.cpp b/modules/dnn/src/layers/batch_norm_layer.cpp index 42676c7938..dcb4005975 100644 --- a/modules/dnn/src/layers/batch_norm_layer.cpp +++ b/modules/dnn/src/layers/batch_norm_layer.cpp @@ -231,7 +231,7 @@ public: kernel.set(4, ocl::KernelArg::PtrReadOnly(umat_weight)); kernel.set(5, ocl::KernelArg::PtrReadOnly(umat_bias)); kernel.set(6, ocl::KernelArg::PtrWriteOnly(dst)); - bool ret = kernel.run(2, global, NULL, false); + bool ret = kernel.run_(2, global, NULL, false); if (!ret) return false; } diff --git a/modules/dnn/src/layers/mvn_layer.cpp b/modules/dnn/src/layers/mvn_layer.cpp index 8f06216df1..de2b0d5690 100644 --- a/modules/dnn/src/layers/mvn_layer.cpp +++ b/modules/dnn/src/layers/mvn_layer.cpp @@ -191,7 +191,7 @@ public: k1.set(argId++, ocl::KernelArg::PtrReadOnly(bnorm_weight)); k1.set(argId++, ocl::KernelArg::PtrReadOnly(bnorm_bias)); k1.set(argId++, ocl::KernelArg::PtrWriteOnly(outMat)); - ret = k1.run(1, globalsize, localsize, false); + ret = k1.run_(1, globalsize, localsize, false); if (!ret) return false; } diff --git a/modules/dnn/src/layers/slice_layer.cpp b/modules/dnn/src/layers/slice_layer.cpp index 507964edf9..16f1958879 100644 --- a/modules/dnn/src/layers/slice_layer.cpp +++ b/modules/dnn/src/layers/slice_layer.cpp @@ -482,7 +482,7 @@ public: ocl::KernelArg::PtrReadOnly(input), ocl::KernelArg::PtrWriteOnly(output) ) - .run(2, (size_t*)ocl.global_size, (size_t*)ocl.local_size, false); + .run_(2, (size_t*)ocl.global_size, (size_t*)ocl.local_size, false); if (!ret) return false; } // for outputs.size() diff --git a/modules/dnn/src/ocl4dnn/src/math_functions.cpp b/modules/dnn/src/ocl4dnn/src/math_functions.cpp index 855a21e08f..c924d66b12 100644 --- a/modules/dnn/src/ocl4dnn/src/math_functions.cpp +++ b/modules/dnn/src/ocl4dnn/src/math_functions.cpp @@ -116,6 +116,7 @@ ocl::Image2D ocl4dnnGEMMCopyBufferToImage(UMat buffer, int offset, .args( ocl::KernelArg::PtrReadOnly(buffer), image, offset, + padded_width, padded_height, width, height, ld) .run(2, global_copy, NULL, false); diff --git a/modules/dnn/src/ocl4dnn/src/ocl4dnn_conv_spatial.cpp b/modules/dnn/src/ocl4dnn/src/ocl4dnn_conv_spatial.cpp index b4477ebfc4..3b73da801c 100644 --- a/modules/dnn/src/ocl4dnn/src/ocl4dnn_conv_spatial.cpp +++ b/modules/dnn/src/ocl4dnn/src/ocl4dnn_conv_spatial.cpp @@ -769,12 +769,11 @@ bool OCL4DNNConvSpatial::swizzleWeight(const UMat &weight, swizzled_factor ); - size_t global_work_size_copy[3] = { - (size_t) (alignSize(num_output_, swizzled_factor) * channels * kernel_w_ * kernel_h_), 1, 1 }; + size_t global_work_size_copy[1] = { (size_t)(alignSize(num_output_, swizzled_factor) * channels * kernel_w_ * kernel_h_) }; - if (!oclk_copy_weight.run(3, global_work_size_copy, NULL, false)) + if (!oclk_copy_weight.run_(1, global_work_size_copy, NULL, false)) { - std::cout << "Swizzle kernel run failed." << std::endl; + CV_LOG_ERROR(NULL, "DNN/OpenCL: Swizzle kernel run failed"); return false; } } else { @@ -937,7 +936,7 @@ bool OCL4DNNConvSpatial::convolve(const UMat &bottom, UMat &top, kernel.set(argIdx++, (uint16_t)height_); kernel.set(argIdx++, (uint16_t)output_w_); kernel.set(argIdx++, (uint16_t)output_h_); - if (!kernel.run(3, config->global_work_size, config->local_work_size, false)) + if (!kernel.run_(3, config->global_work_size, config->local_work_size, false)) { std::cout << "IDLF kernel run failed." << std::endl; return false; @@ -1056,7 +1055,7 @@ bool OCL4DNNConvSpatial::convolve(const UMat &bottom, UMat &top, gy = alignSize(gy, blockK); size_t global_size[3] = { gx, gy, config->global_work_size[2] }; - if (!kernel.run(3, global_size, config->local_work_size, false)) + if (!kernel.run_(3, global_size, config->local_work_size, false)) { std::cout << "GEMM like kernel run failed." << std::endl; return false; @@ -1085,9 +1084,9 @@ bool OCL4DNNConvSpatial::convolve(const UMat &bottom, UMat &top, global_size[1] = output_h_; global_size[2] = num_output_ * num_; - if (!kernel.run(3, global_size, NULL, false)) + if (!kernel.run_(3, global_size, NULL, false)) { - std::cout << "DWCONV kernel run failed." << std::endl; + CV_LOG_ERROR(NULL, "DNN/OpenCL: DWCONV kernel run failed"); return false; } } else { @@ -1127,11 +1126,11 @@ bool OCL4DNNConvSpatial::convolve(const UMat &bottom, UMat &top, kernel.set(argIdx++, (uint16_t)output_h_); kernel.set(argIdx++, (uint16_t)pad_w_); kernel.set(argIdx++, (uint16_t)pad_h_); - if (!kernel.run(3, config->global_work_size, + if (!kernel.run_(3, config->global_work_size, (config->use_null_local) ? NULL : config->local_work_size, false)) { - std::cout << "Basic kernel run failed." << std::endl; + CV_LOG_ERROR(NULL, "DNN/OpenCL: Basic kernel run failed"); return false; } } diff --git a/modules/dnn/src/ocl4dnn/src/ocl4dnn_softmax.cpp b/modules/dnn/src/ocl4dnn/src/ocl4dnn_softmax.cpp index 78576711a7..7b32189fdc 100644 --- a/modules/dnn/src/ocl4dnn/src/ocl4dnn_softmax.cpp +++ b/modules/dnn/src/ocl4dnn/src/ocl4dnn_softmax.cpp @@ -127,7 +127,7 @@ bool OCL4DNNSoftmax::Forward(const UMat& bottom, UMat& top) oclk_softmax_forward_kernel.set(argIdx++, ocl::KernelArg::PtrReadOnly(bottom)); oclk_softmax_forward_kernel.set(argIdx++, ocl::KernelArg::PtrWriteOnly(top)); } - ret = oclk_softmax_forward_kernel.run(3, global_size, local_size, false); + ret = oclk_softmax_forward_kernel.run_(3, global_size, local_size, false); } return ret; } diff --git a/modules/dnn/src/opencl/gemm_image.cl b/modules/dnn/src/opencl/gemm_image.cl index 710637a093..f6e0020d82 100644 --- a/modules/dnn/src/opencl/gemm_image.cl +++ b/modules/dnn/src/opencl/gemm_image.cl @@ -954,6 +954,10 @@ __kernel void TEMPLATE(gemm_buffer_copy_image_transpose, Dtype)( { const int gidx = get_global_id(0); const int gidy = get_global_id(1); + + if (gidx >= width || gidy >= height) + return; + int2 coord_dst = (int2)(gidx, gidy); __global Dtype* A_off = A + offA; Dtype srcA = A_off[gidy * ldA + gidx]; @@ -968,12 +972,18 @@ __kernel void TEMPLATE(gemm_buffer_copy_image_no_transpose, Dtype)( __global Dtype* A, __write_only image2d_t ImA, int offA, + int padded_width, + int padded_height, int width, int height, int ldA) { const int gidx = get_global_id(0); const int gidy = get_global_id(1); + + if (gidx >= padded_width || gidy >= padded_height) + return; + int2 coord_dst = (int2)(gidx, gidy); #if TYPE == TYPE_HALF if (gidx >= width || gidy >= height) { From 36cc43170d9e1e244d72c25d69cef8d716599a10 Mon Sep 17 00:00:00 2001 From: Zhuo Zhang Date: Mon, 6 Sep 2021 12:03:59 +0800 Subject: [PATCH 157/262] docs: fix image path for py_fast doc --- doc/py_tutorials/py_feature2d/py_fast/py_fast.markdown | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/doc/py_tutorials/py_feature2d/py_fast/py_fast.markdown b/doc/py_tutorials/py_feature2d/py_fast/py_fast.markdown index 4ce61370bd..b1b8a81ca8 100644 --- a/doc/py_tutorials/py_feature2d/py_fast/py_fast.markdown +++ b/doc/py_tutorials/py_feature2d/py_fast/py_fast.markdown @@ -98,7 +98,7 @@ import numpy as np import cv2 as cv from matplotlib import pyplot as plt -img = cv.imread('simple.jpg',0) +img = cv.imread('blox.jpg',0) # `/samples/data/blox.jpg` # Initiate FAST object with default values fast = cv.FastFeatureDetector_create() @@ -113,17 +113,17 @@ print( "nonmaxSuppression:{}".format(fast.getNonmaxSuppression()) ) print( "neighborhood: {}".format(fast.getType()) ) print( "Total Keypoints with nonmaxSuppression: {}".format(len(kp)) ) -cv.imwrite('fast_true.png',img2) +cv.imwrite('fast_true.png', img2) # Disable nonmaxSuppression fast.setNonmaxSuppression(0) -kp = fast.detect(img,None) +kp = fast.detect(img, None) print( "Total Keypoints without nonmaxSuppression: {}".format(len(kp)) ) img3 = cv.drawKeypoints(img, kp, None, color=(255,0,0)) -cv.imwrite('fast_false.png',img3) +cv.imwrite('fast_false.png', img3) @endcode See the results. First image shows FAST with nonmaxSuppression and second one without nonmaxSuppression: From 35e824c28710ca13b2770dc9d547bfc352293dc5 Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Mon, 6 Sep 2021 05:51:37 +0300 Subject: [PATCH 158/262] dnn(ocl): fix out of bound access in GEMM-like kernels - dropped usage of CreateSubBuffer() - buffers lifetime management issue - fixed elementwise offset - avoid out of bounds read access --- .../src/ocl4dnn/src/ocl4dnn_conv_spatial.cpp | 73 ++++------------- modules/dnn/src/opencl/conv_layer_spatial.cl | 79 ++++++++++++++++--- 2 files changed, 80 insertions(+), 72 deletions(-) diff --git a/modules/dnn/src/ocl4dnn/src/ocl4dnn_conv_spatial.cpp b/modules/dnn/src/ocl4dnn/src/ocl4dnn_conv_spatial.cpp index 3b73da801c..5eee1da4a0 100644 --- a/modules/dnn/src/ocl4dnn/src/ocl4dnn_conv_spatial.cpp +++ b/modules/dnn/src/ocl4dnn/src/ocl4dnn_conv_spatial.cpp @@ -945,9 +945,11 @@ bool OCL4DNNConvSpatial::convolve(const UMat &bottom, UMat &top, } else if (config->kernelType == KERNEL_TYPE_GEMM_LIKE) { if (!swizzleWeight(weight, config->workItem_output[1], true)) return false; +#if 0 size_t total_bottom_size = bottom_dim_ * numImages; size_t total_kernel_size = kernel_h_ * kernel_w_ * channels_ * M_; size_t total_bias_size = M_ * group_; +#endif size_t total_top_size = top_dim_ * numImages; for (int32_t g = 0; g < group_; ++g) { bias_offset = M_ * g; @@ -960,72 +962,25 @@ bool OCL4DNNConvSpatial::convolve(const UMat &bottom, UMat &top, return false; cl_uint argIdx = 0; - setFusionArg(fused_activ_, fused_eltwise_, -1, kernel, argIdx); + setFusionArg(fused_activ_, fused_eltwise_, output_image_offset, kernel, argIdx); - UMat img_buffer; - if (image_offset) - { - CreateSubBuffer(bottom, img_buffer, image_offset, - total_bottom_size - image_offset, false); - if (img_buffer.empty()) - return false; + kernel.set(argIdx++, ocl::KernelArg::PtrReadOnly(bottom)); + kernel.set(argIdx++, (int)image_offset); + kernel.set(argIdx++, (int)(bottom.total() - image_offset)); - kernel.set(argIdx++, ocl::KernelArg::PtrReadOnly(img_buffer)); - } - else - { - kernel.set(argIdx++, ocl::KernelArg::PtrReadOnly(bottom)); - } + kernel.set(argIdx++, ocl::KernelArg::PtrReadOnly(swizzled_weights_umat)); + kernel.set(argIdx++, (int)kernel_offset); + kernel.set(argIdx++, (int)(swizzled_weights_umat.total() - kernel_offset)); - UMat kernel_buffer; - if (kernel_offset) - { - CreateSubBuffer(swizzled_weights_umat, kernel_buffer, kernel_offset, - total_kernel_size - kernel_offset, false); - if (kernel_buffer.empty()) - return false; - - kernel.set(argIdx++, ocl::KernelArg::PtrReadOnly(kernel_buffer)); - } - else - { - kernel.set(argIdx++, ocl::KernelArg::PtrReadOnly(swizzled_weights_umat)); - } - - UMat bias_buffer; if (bias_term_) { - if (bias_offset) - { - CreateSubBuffer(bias, bias_buffer, bias_offset, - total_bias_size - bias_offset, false); - if (bias_buffer.empty()) - return false; - - kernel.set(argIdx++, ocl::KernelArg::PtrReadOnly(bias_buffer)); - } - else - { - kernel.set(argIdx++, ocl::KernelArg::PtrReadOnly(bias)); - } + kernel.set(argIdx++, ocl::KernelArg::PtrReadOnly(bias)); + kernel.set(argIdx++, (int)bias_offset); } - UMat out_buffer; - if (output_image_offset) - { - CreateSubBuffer(top, out_buffer, output_image_offset, - total_top_size - output_image_offset, true); - if (out_buffer.empty()) - return false; - - kernel.set(argIdx++, ocl::KernelArg::PtrWriteOnly(out_buffer)); - kernel.set(argIdx++, (int)(out_buffer.offset / element_size)); - } - else - { - kernel.set(argIdx++, ocl::KernelArg::PtrWriteOnly(top)); - kernel.set(argIdx++, (int)(top.offset / element_size)); - } + kernel.set(argIdx++, ocl::KernelArg::PtrWriteOnly(top)); + kernel.set(argIdx++, (int)(top.offset / element_size) + output_image_offset); + kernel.set(argIdx++, (int)total_top_size - (int)(top.offset / element_size)); kernel.set(argIdx++, (uint16_t)width_); kernel.set(argIdx++, (uint16_t)height_); diff --git a/modules/dnn/src/opencl/conv_layer_spatial.cl b/modules/dnn/src/opencl/conv_layer_spatial.cl index 55015557a0..e7bbacd4c4 100644 --- a/modules/dnn/src/opencl/conv_layer_spatial.cl +++ b/modules/dnn/src/opencl/conv_layer_spatial.cl @@ -401,13 +401,12 @@ typedef struct half0 { half s0; } half0; //never used but makes compiler happy. #define ROW_PITCH input_width #define GEMM_LIKE_KERNEL_ARGS \ - ELTWISE_DATA_ARG \ + ELTWISE_DATA_ARG_WITH_OFFSET \ FUSED_ARG \ - const __global Dtype *src0, \ - const __global Dtype *src1, \ - BIAS_KERNEL_ARG \ - __global Dtype *dst_base, \ - const int dst_offset, \ + const __global Dtype *src0_ptr, const unsigned int src0_offset, const unsigned int src0_limit, \ + const __global Dtype *src1_ptr, const unsigned int src1_offset, const unsigned int src1_limit, \ + BIAS_KERNEL_ARG_WITH_OFFSET \ + __global Dtype *dst_base, const unsigned int dst_offset, const unsigned int dst_limit, \ const ushort input_width, \ const ushort input_height, \ const ushort output_width, \ @@ -437,7 +436,17 @@ typedef struct half0 { half s0; } half0; //never used but makes compiler happy. __attribute__((intel_reqd_sub_group_size(8))) __kernel void Conv_Interleaved(GEMM_LIKE_KERNEL_ARGS) { + const __global Dtype *src0 = src0_ptr + src0_offset; + const __global Dtype *src1 = src1_ptr + src1_offset; +#if APPLY_BIAS + __global Dtype* biases_base = biases_base_ptr + biases_base_offset; +#endif + __global Dtype *dst = dst_base + dst_offset; +#ifdef FUSED_CONV_ELTWISE + __global Dtype* eltwise_data = eltwise_ptr + eltwise_offset; +#endif + const int group_x = get_group_id(0); const int group_y = get_group_id(1); const int global_x = get_global_id(0); @@ -460,6 +469,14 @@ __kernel void Conv_Interleaved(GEMM_LIKE_KERNEL_ARGS) } typedef CAT( Dtype, KERNEL_WIDTH ) Dtype_t; +// U_GEMM_LIKE_CONV_k11x11_cn3_g1_s4x4_d1x1_b1_in240x240_p0x0_num1_M96_activ1_eltwise0_FP32_5_1_8_32_SIMD8 doesn't run properly (src0_read out of bounds) +// Test: DNNTestNetwork.AlexNet/0 (to run all kernels use OPENCV_OCL4DNN_FORCE_AUTO_TUNING=1) +#if 0 // INPUT_PAD_W == 0 && INPUT_PAD_H == 0 && DILATION_X == 1 && DILATION_Y == 1 && INPUT_PAD_BOTTOM == 0 && INPUT_PAD_RIGHT == 0 + #define OPTIMIZE_READ 1 +#else + #define OPTIMIZE_READ 0 +#endif + // True for all threads if filter_width is multiple of TILE_N // else, true for all but right-most column of threads. if( TILE_N_LAST == 0 || global_x < WIDTH1 / TILE_N ) @@ -476,7 +493,7 @@ __kernel void Conv_Interleaved(GEMM_LIKE_KERNEL_ARGS) // atile is M rows x K columns. int curr_x = ( global_y % output_width ) * STRIDE_X; int curr_y = ( global_y / output_width ) * STRIDE_Y; -#if INPUT_PAD_H != 0 || INPUT_PAD_W != 0 || DILATION_X != 1 || DILATION_Y != 1 || INPUT_PAD_BOTTOM != 0 || INPUT_PAD_RIGHT != 0 +#if !OPTIMIZE_READ int saved_y = curr_y; #endif const __global Dtype *src0_read = src0 @@ -496,7 +513,7 @@ __kernel void Conv_Interleaved(GEMM_LIKE_KERNEL_ARGS) do { int patch_row = 0; -#if INPUT_PAD_H != 0 || INPUT_PAD_W != 0 || DILATION_X != 1 || DILATION_Y != 1 || INPUT_PAD_BOTTOM != 0 || INPUT_PAD_RIGHT != 0 +#if !OPTIMIZE_READ curr_y = saved_y; #endif @@ -514,11 +531,17 @@ __kernel void Conv_Interleaved(GEMM_LIKE_KERNEL_ARGS) // ... const bool kernel_width_is_odd = KERNEL_WIDTH % 2 == 1; -#if INPUT_PAD_W == 0 && INPUT_PAD_H == 0 && DILATION_X == 1 && DILATION_Y == 1 && INPUT_PAD_BOTTOM == 0 && INPUT_PAD_RIGHT == 0 +#if OPTIMIZE_READ #if KERNEL_WIDTH == 3 Dtype_t blockA00 = vload3(0, src0_read); Dtype* pblockA00 = (Dtype*)(&blockA00); #else + #if 0 // debug + if ((int)(src0_read - src0) >= src0_limit - KERNEL_WIDTH) + { + printf("CATCH: src0_read-src0: %d limit=%d curr_y,curr_x=%d,%d\n", (int)(src0_read - src0), src0_limit, curr_y, curr_x); + } + #endif Dtype_t blockA00 = ( (const __global Dtype_t*)src0_read )[ 0 ]; Dtype* pblockA00 = (Dtype*)(&blockA00); #endif @@ -639,7 +662,7 @@ __kernel void Conv_Interleaved(GEMM_LIKE_KERNEL_ARGS) // atile is M rows x K columns. int curr_x = ( global_y % output_width ) * STRIDE_X; int curr_y = ( global_y / output_width ) * STRIDE_Y; -#if INPUT_PAD_H != 0 || INPUT_PAD_W != 0 || DILATION_X != 1 || DILATION_Y != 1 || INPUT_PAD_BOTTOM != 0 || INPUT_PAD_RIGHT != 0 +#if !OPTIMIZE_READ int saved_y = curr_y; #endif const __global Dtype *src0_read = src0 @@ -659,14 +682,14 @@ __kernel void Conv_Interleaved(GEMM_LIKE_KERNEL_ARGS) do { int patch_row = 0; -#if INPUT_PAD_H != 0 || INPUT_PAD_W != 0 || DILATION_X != 1 || DILATION_Y != 1 || INPUT_PAD_BOTTOM != 0 || INPUT_PAD_RIGHT != 0 +#if !OPTIMIZE_READ curr_y = saved_y; #endif do { // Load atile and interleaved btile. const bool kernel_width_is_odd = KERNEL_WIDTH % 2 == 1; -#if INPUT_PAD_W == 0 && INPUT_PAD_H == 0 && DILATION_X == 1 && DILATION_Y == 1 && INPUT_PAD_BOTTOM == 0 && INPUT_PAD_RIGHT == 0 +#if OPTIMIZE_READ Dtype_t blockA00 = ( (const __global Dtype_t*)src0_read )[ 0 ]; Dtype* pblockA00 = (Dtype*)(&blockA00); #else @@ -803,7 +826,7 @@ __kernel void Conv_Interleaved(GEMM_LIKE_KERNEL_ARGS) } } } -#endif +#endif // TILE_N_LAST > 0 } #endif #ifdef GEMM_LIKE_CONV_32_2 @@ -826,7 +849,17 @@ __kernel void Conv_Interleaved(GEMM_LIKE_KERNEL_ARGS) __attribute__((intel_reqd_sub_group_size(8))) __kernel void Conv_Interleaved(GEMM_LIKE_KERNEL_ARGS) { + const __global Dtype *src0 = src0_ptr + src0_offset; + const __global Dtype *src1 = src1_ptr + src1_offset; +#if APPLY_BIAS + __global Dtype* biases_base = biases_base_ptr + biases_base_offset; +#endif + __global Dtype *dst = dst_base + dst_offset; +#ifdef FUSED_CONV_ELTWISE + __global Dtype* eltwise_data = eltwise_ptr + eltwise_offset; +#endif + const int group_x = get_group_id(0); const int group_y = get_group_id(1); const int global_x = get_global_id(0); @@ -1388,7 +1421,17 @@ __kernel void Conv_Interleaved(GEMM_LIKE_KERNEL_ARGS) __attribute__((intel_reqd_sub_group_size(16))) __kernel void Conv_Interleaved(GEMM_LIKE_KERNEL_ARGS) { + const __global Dtype *src0 = src0_ptr + src0_offset; + const __global Dtype *src1 = src1_ptr + src1_offset; +#if APPLY_BIAS + __global Dtype* biases_base = biases_base_ptr + biases_base_offset; +#endif + __global Dtype *dst = dst_base + dst_offset; +#ifdef FUSED_CONV_ELTWISE + __global Dtype* eltwise_data = eltwise_ptr + eltwise_offset; +#endif + const int group_x = get_group_id(0); const int group_y = get_group_id(1); const int global_x = get_global_id(0); @@ -1574,7 +1617,17 @@ __kernel void Conv_Interleaved(GEMM_LIKE_KERNEL_ARGS) __attribute__((intel_reqd_sub_group_size(16))) __kernel void Conv_Interleaved(GEMM_LIKE_KERNEL_ARGS) { + const __global Dtype *src0 = src0_ptr + src0_offset; + const __global Dtype *src1 = src1_ptr + src1_offset; +#if APPLY_BIAS + __global Dtype* biases_base = biases_base_ptr + biases_base_offset; +#endif + __global Dtype *dst = dst_base + dst_offset; +#ifdef FUSED_CONV_ELTWISE + __global Dtype* eltwise_data = eltwise_ptr + eltwise_offset; +#endif + const int group_x = get_group_id(0); const int group_y = get_group_id(1); const int global_x = get_global_id(0); From 0c13d34ade785d1d84da1532634294ff4dbc5fd5 Mon Sep 17 00:00:00 2001 From: Ruben Gonzalez <56379722+rgonzalezfluendo@users.noreply.github.com> Date: Tue, 7 Sep 2021 12:02:25 +0200 Subject: [PATCH 159/262] imgcodecs(doc): Sync imread_ docuemtation with the implemetation. Documentation was desynchronized in commit 11eafca3e2a4cbc62f1309d25db0ea3ed9a6ea8e --- modules/imgcodecs/src/loadsave.cpp | 4 ---- 1 file changed, 4 deletions(-) diff --git a/modules/imgcodecs/src/loadsave.cpp b/modules/imgcodecs/src/loadsave.cpp index 28d8ff285b..f5ebb8a7bc 100644 --- a/modules/imgcodecs/src/loadsave.cpp +++ b/modules/imgcodecs/src/loadsave.cpp @@ -377,10 +377,6 @@ static void ApplyExifOrientation(ExifEntry_t orientationTag, Mat& img) * * @param[in] filename File to load * @param[in] flags Flags - * @param[in] hdrtype { LOAD_CVMAT=0, - * LOAD_IMAGE=1, - * LOAD_MAT=2 - * } * @param[in] mat Reference to C++ Mat object (If LOAD_MAT) * */ From c703f1eed6c3da4f14b37cc151e03463bb4ff54f Mon Sep 17 00:00:00 2001 From: Ilya Lavrenov Date: Thu, 9 Sep 2021 10:56:34 +0300 Subject: [PATCH 160/262] Merge pull request #20673 from ilya-lavrenov:print-openvino-information * Support of OpenVINO interface libraries * cmake: rename and move ocv_get_imported_target to OpenCVUtils.cmake --- CMakeLists.txt | 3 ++- cmake/OpenCVUtils.cmake | 12 ++++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 41a77f0467..e5d809fefc 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1526,6 +1526,7 @@ if(WITH_INF_ENGINE OR INF_ENGINE_TARGET) if(INF_ENGINE_TARGET) list(GET INF_ENGINE_TARGET 0 ie_target) set(__msg "YES (${INF_ENGINE_RELEASE} / ${INF_ENGINE_VERSION})") + ocv_get_imported_target(ie_target "${ie_target}") get_target_property(_lib ${ie_target} IMPORTED_LOCATION) get_target_property(_lib_imp_rel ${ie_target} IMPORTED_IMPLIB_RELEASE) get_target_property(_lib_imp_dbg ${ie_target} IMPORTED_IMPLIB_DEBUG) @@ -1549,7 +1550,7 @@ if(WITH_INF_ENGINE OR INF_ENGINE_TARGET) endif() if(WITH_NGRAPH OR HAVE_NGRAPH) if(HAVE_NGRAPH) - set(__target ngraph::ngraph) + ocv_get_imported_target(__target ngraph::ngraph) set(__msg "YES (${ngraph_VERSION})") get_target_property(_lib ${__target} IMPORTED_LOCATION) get_target_property(_lib_imp_rel ${__target} IMPORTED_IMPLIB_RELEASE) diff --git a/cmake/OpenCVUtils.cmake b/cmake/OpenCVUtils.cmake index 932eb039b1..d7097fdc30 100644 --- a/cmake/OpenCVUtils.cmake +++ b/cmake/OpenCVUtils.cmake @@ -1619,6 +1619,18 @@ function(ocv_add_external_target name inc link def) endif() endfunction() +# Returns the first non-interface target +function(ocv_get_imported_target imported interface) + set(__result "${interface}") + get_target_property(__type "${__result}" TYPE) + if(__type STREQUAL "INTERFACE_LIBRARY") + get_target_property(__libs "${__result}" INTERFACE_LINK_LIBRARIES) + list(GET __libs 0 __interface) + ocv_get_imported_target(__result "${__interface}") + endif() + set(${imported} "${__result}" PARENT_SCOPE) +endfunction() + macro(ocv_get_libname var_name) get_filename_component(__libname "${ARGN}" NAME) From 4807cd8a6eeaa494a76df435679ccc3b9e6545e6 Mon Sep 17 00:00:00 2001 From: rogday Date: Thu, 9 Sep 2021 14:41:40 +0300 Subject: [PATCH 161/262] Merge pull request #20605 from rogday:split_slice_shenanigans Add Normalize subgraph, fix Slice, Mul and Expand * Add Normalize subgraph, support for starts<0 and axis<0 in Slice, Mul broadcasting in the middle and fix Expand's unsqueeze * remove todos * remove range-based for loop * address review comments * change >> to > > in template * fix indexation * fix expand that does nothing --- modules/dnn/src/layers/slice_layer.cpp | 59 ++++++-- .../dnn/src/onnx/onnx_graph_simplifier.cpp | 22 +++ modules/dnn/src/onnx/onnx_importer.cpp | 129 ++++++++++++++---- modules/dnn/test/test_onnx_importer.cpp | 6 + 4 files changed, 173 insertions(+), 43 deletions(-) diff --git a/modules/dnn/src/layers/slice_layer.cpp b/modules/dnn/src/layers/slice_layer.cpp index 16f1958879..a470772813 100644 --- a/modules/dnn/src/layers/slice_layer.cpp +++ b/modules/dnn/src/layers/slice_layer.cpp @@ -58,6 +58,31 @@ namespace cv namespace dnn { +void sliceRangesFromShape(const MatShape& inpShape, int& axis, std::vector >& sliceRanges) +{ + CV_Assert(inpShape.size() > 0); + bool axisNeg = (axis < 0); + axis = (axis + static_cast(inpShape.size())) % inpShape.size(); + int n = inpShape[axis]; + + for (size_t i = 0; i < sliceRanges.size(); ++i){ + std::vector& ranges = sliceRanges[i]; + if (axisNeg) + { + ranges.insert(ranges.begin(), axis, Range::all()); + } + Range& range = ranges.back(); + + if (range.start >= 0) + { + continue; + } + + CV_Assert(n != 0); + range.start = (n + range.start) % n; + } +} + class SliceLayerImpl : public SliceLayer { public: @@ -69,20 +94,22 @@ public: num_split = params.get("num_split", 0); hasDynamicShapes = params.get("has_dynamic_shapes", false); shapesInitialized = !hasDynamicShapes; + if (params.has("slice_point")) { CV_Assert(!params.has("begin") && !params.has("size") && !params.has("end")); const DictValue &indicesValue = params.get("slice_point"); + int size = axis > 0 ? axis + 1 : 1; sliceRanges.resize(indicesValue.size() + 1, - std::vector(axis + 1, Range::all())); + std::vector(size, Range::all())); int prevSlice = 0; for (int i = 0; i < indicesValue.size(); ++i) { - sliceRanges[i][axis].start = prevSlice; - sliceRanges[i][axis].end = indicesValue.get(i); - prevSlice = sliceRanges[i][axis].end; + sliceRanges[i][size - 1].start = prevSlice; + sliceRanges[i][size - 1].end = indicesValue.get(i); + prevSlice = sliceRanges[i][size - 1].end; } - sliceRanges.back()[axis].start = prevSlice; + sliceRanges.back()[size - 1].start = prevSlice; } else if (params.has("begin")) { @@ -97,7 +124,6 @@ public: { int start = begins.get(i); int sizeOrEnd = sizesOrEnds.get(i); // It may be negative to reverse indexation. - CV_Assert(start >= 0); sliceRanges[0][i].start = start; if (params.has("size")) @@ -154,16 +180,20 @@ public: CV_Assert(inputs.size() == 1); MatShape inpShape = inputs[0]; - if (!sliceRanges.empty()) + int axis_rw = axis; + std::vector > sliceRanges_rw = sliceRanges; + sliceRangesFromShape(inpShape, axis_rw, sliceRanges_rw); + + if (!sliceRanges_rw.empty()) { - outputs.resize(sliceRanges.size(), inpShape); + outputs.resize(sliceRanges_rw.size(), inpShape); for (int i = 0; i < outputs.size(); ++i) { - CV_Assert(sliceRanges[i].size() <= inpShape.size()); - for (int j = 0; j < sliceRanges[i].size(); ++j) + CV_Assert(sliceRanges_rw[i].size() <= inpShape.size()); + for (int j = 0; j < sliceRanges_rw[i].size(); ++j) { if (shapesInitialized || inpShape[j] > 0) - outputs[i][j] = normalize_axis_range(sliceRanges[i][j], inpShape[j]).size(); + outputs[i][j] = normalize_axis_range(sliceRanges_rw[i][j], inpShape[j]).size(); if (!sliceSteps.empty() && (i < sliceSteps.size()) && (j < sliceSteps[i].size()) && (sliceSteps[i][j] > 1)) outputs[i][j] = (outputs[i][j] + sliceSteps[i][j] - 1) / sliceSteps[i][j]; @@ -172,10 +202,10 @@ public: } else // Divide input blob on equal parts by axis. { - CV_Assert(0 <= axis && axis < inpShape.size()); + CV_Assert(0 <= axis_rw && axis_rw < inpShape.size()); int splits = num_split ? num_split : requiredOutputs; - CV_Assert(splits > 0 && inpShape[axis] % splits == 0); - inpShape[axis] /= splits; + CV_Assert(splits > 0 && inpShape[axis_rw] % splits == 0); + inpShape[axis_rw] /= splits; outputs.resize(splits, inpShape); } return false; @@ -200,6 +230,7 @@ public: CV_Assert(inputs.size() == 1); const MatSize& inpShape = inputs[0].size; + sliceRangesFromShape(shape(inputs[0]), axis, sliceRanges); finalSliceRanges = sliceRanges; if (sliceRanges.empty()) diff --git a/modules/dnn/src/onnx/onnx_graph_simplifier.cpp b/modules/dnn/src/onnx/onnx_graph_simplifier.cpp index b81ccf106c..76937e08f3 100644 --- a/modules/dnn/src/onnx/onnx_graph_simplifier.cpp +++ b/modules/dnn/src/onnx/onnx_graph_simplifier.cpp @@ -231,6 +231,27 @@ public: } }; +class NormalizeSubgraph2_2 : public NormalizeSubgraphBase +{ +public: + NormalizeSubgraph2_2() + { + int input = addNodeToMatch(""); + int norm = addNodeToMatch("ReduceL2", input); + + int min = addNodeToMatch(""); + int max = addNodeToMatch(""); + int clip = addNodeToMatch("Clip", norm, min, max); + + int shape = addNodeToMatch(""); + int expand = addNodeToMatch("Expand", clip, shape); + + addNodeToMatch("Div", input, expand); + + setFusedNode("Normalize", input); + } +}; + class NormalizeSubgraph3 : public NormalizeSubgraphBase { public: @@ -555,6 +576,7 @@ void simplifySubgraphs(opencv_onnx::GraphProto& net) subgraphs.push_back(makePtr()); subgraphs.push_back(makePtr()); subgraphs.push_back(makePtr()); + subgraphs.push_back(makePtr()); subgraphs.push_back(makePtr()); subgraphs.push_back(makePtr()); subgraphs.push_back(makePtr()); diff --git a/modules/dnn/src/onnx/onnx_importer.cpp b/modules/dnn/src/onnx/onnx_importer.cpp index 6583d6cf62..955c79c0fa 100644 --- a/modules/dnn/src/onnx/onnx_importer.cpp +++ b/modules/dnn/src/onnx/onnx_importer.cpp @@ -59,6 +59,8 @@ class ONNXImporter void addLayer(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void expandMid(const std::string& prefix, opencv_onnx::NodeProto& node_proto, + const std::string& input, size_t n); public: ONNXImporter(Net& net, const char *onnxFile) @@ -427,6 +429,37 @@ void ONNXImporter::addLayer(LayerParams& layerParams, } } +/** @brief Make N copies of input layer and set them as input to node_proto. + * @param prefix prefix of new layers' names + * @param node_proto node which will contain all copies as inputs + * @param input name of the node to copy + * @param n number of copies + */ +void ONNXImporter::expandMid(const std::string& prefix, opencv_onnx::NodeProto& node_proto, + const std::string& input, size_t n) +{ + std::vector input_names; + input_names.reserve(n); + for (size_t j = 0; j < n; j++) + { + LayerParams copyLP; + copyLP.name = format("%s/copy_%d", prefix.c_str(), j); + copyLP.type = "Identity"; + CV_Assert((layer_id.find(copyLP.name) == layer_id.end()) && + "Couldn't copy the node: generated name already exists in the graph."); + input_names.push_back(copyLP.name); + + node_proto.set_input(0, input); + node_proto.set_output(0, copyLP.name); + addLayer(copyLP, node_proto); + } + node_proto.clear_input(); + for (size_t i = 0; i < input_names.size(); i++) + { + node_proto.add_input(input_names[i]); + } +} + void ONNXImporter::addConstant(const std::string& name, const Mat& blob) { constBlobs.insert(std::make_pair(name, blob)); @@ -1288,6 +1321,37 @@ void ONNXImporter::parseMatMul(LayerParams& layerParams, const opencv_onnx::Node addLayer(layerParams, node_proto); } +void findBroadAxis(const MatShape& broadShape, const MatShape& outShape, size_t& axis, int& broadAxis) +{ + const size_t diff = outShape.size() - broadShape.size(); + + // find the first non-one element of the broadcasting shape + axis = 0; + for (; axis < broadShape.size() && broadShape[axis] == 1; ++axis) {} + + // find the last non-one element of the broadcasting shape + size_t endAxis = broadShape.size(); + for (; endAxis > axis && broadShape[endAxis - 1] == 1; --endAxis) {} + + // find one between axis and endAxis - as it needs to be broadcasted, + // dimensions from the left of axis and from the right of endAxis will be handled by Scale layer + broadAxis = -1; + for (size_t i = axis; i < endAxis; ++i) + { + size_t outAxis = i + diff; + if (outShape[outAxis] == broadShape[i]) + { + continue; + } + + // ensure we need to broadcast only 1 dimension in the middle + CV_Assert(broadShape[i] == 1 && broadAxis == -1); + broadAxis = static_cast(outAxis); + } + + axis += diff; +} + // "Mul" "Div" void ONNXImporter::parseMul(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto_) { @@ -1410,13 +1474,31 @@ void ONNXImporter::parseMul(LayerParams& layerParams, const opencv_onnx::NodePro } const MatShape& broadShape = outShapes[node_proto.input(1)]; - const size_t outShapeSize = outShapes[node_proto.input(0)].size(); - const size_t diff = outShapeSize - broadShape.size(); + const MatShape& outShape = outShapes[node_proto.input(0)]; - size_t axis; - for (axis = diff; axis < broadShape.size() && broadShape[axis - diff] == 1; ++axis) {} + size_t axis = 0; + int broadAxis = -1; + findBroadAxis(broadShape, outShape, axis, broadAxis); - CV_Assert(axis != outShapeSize); + // if there is a one dimension in the middle that should be broadcasted, broadcast it + if (broadAxis != -1) + { + opencv_onnx::NodeProto concat_node_proto = node_proto; + const std::string& input1 = concat_node_proto.input(1); + + expandMid(layerParams.name, concat_node_proto, input1, outShape[broadAxis]); + + LayerParams concatLP; + concatLP.name = layerParams.name + "/concat"; + concatLP.set("axis", broadAxis); + concatLP.type = "Concat"; + concat_node_proto.set_output(0, concatLP.name); + + addLayer(concatLP, concat_node_proto); + node_proto.set_input(1, concatLP.name); + } + + CV_Assert(axis != outShape.size()); layerParams.set("axis", static_cast(axis)); layerParams.type = "Scale"; } @@ -1685,12 +1767,11 @@ void ONNXImporter::parseExpand(LayerParams& layerParams, const opencv_onnx::Node // Unsqueeze and repeat along new axis if (targetShape.size() == inpShape.size() + 1) { + inpShape.insert(inpShape.begin(), targetShape.size() - inpShape.size(), 1); for (int i = 0; i < targetShape.size(); i++) { - if (targetShape[i] == -1 && i < inpShape.size()) + if (abs(targetShape[i]) == 1) targetShape[i] = inpShape[i]; - else if (i < inpShape.size() && targetShape[i] != inpShape[i]) - inpShape.insert(inpShape.begin() + i, 1); } if (haveVariables) { @@ -1710,14 +1791,19 @@ void ONNXImporter::parseExpand(LayerParams& layerParams, const opencv_onnx::Node CV_CheckEQ(inpShape.size(), targetShape.size(), "Unsupported Expand op with different dims"); std::vector broadcast_axes; + // shapes aren't right-aligned here because targetShape.size() == inpShape.size() for (int i = 0; i < targetShape.size(); i++) { if (targetShape[i] != inpShape[i]) { if (inpShape[i] == 1) + { broadcast_axes.push_back(i); - else + } + else if (targetShape[i] != 1) + { CV_Error(Error::StsError, format("Could not be broadcast by axis: %d", i)); + } } } @@ -1756,31 +1842,16 @@ void ONNXImporter::parseExpand(LayerParams& layerParams, const opencv_onnx::Node } else if (broadcast_axes.size() == 1 && broadcast_axes[0] <= 1) { - String base_name = layerParams.name + "/copy_"; - std::vector input_names; - for (int j = 0; j < targetShape[broadcast_axes[0]]; j++) - { - std::ostringstream ss; - ss << j; - LayerParams copyLP; - copyLP.name = base_name + ss.str(); - copyLP.type = "Identity"; - CV_Assert(layer_id.find(copyLP.name) == layer_id.end()); - input_names.push_back(copyLP.name); + expandMid(layerParams.name, node_proto, srcName, targetShape[broadcast_axes[0]]); - node_proto.set_input(0, srcName); - node_proto.set_output(0, copyLP.name); - addLayer(copyLP, node_proto); - } - node_proto.clear_input(); - for (int i = 0; i < input_names.size(); i++) - { - node_proto.add_input(input_names[i]); - } layerParams.set("axis", broadcast_axes[0]); layerParams.type = "Concat"; node_proto.set_output(0, layerParams.name); } + else if (broadcast_axes.empty()) + { + layerParams.type = "Identity"; + } else CV_Error(Error::StsNotImplemented, "Unsupported Expand op"); addLayer(layerParams, node_proto); diff --git a/modules/dnn/test/test_onnx_importer.cpp b/modules/dnn/test/test_onnx_importer.cpp index bd96729be1..7b94e02d0a 100644 --- a/modules/dnn/test/test_onnx_importer.cpp +++ b/modules/dnn/test/test_onnx_importer.cpp @@ -285,6 +285,7 @@ TEST_P(Test_ONNX_layers, Scale) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_NN_BUILDER); testONNXModels("scale"); testONNXModels("scale_broadcast", npy, 0, 0, false, true, 3); + testONNXModels("scale_broadcast_mid", npy, 0, 0, false, true, 2); } TEST_P(Test_ONNX_layers, ReduceMean3D) @@ -471,6 +472,8 @@ TEST_P(Test_ONNX_layers, MatMulAdd) TEST_P(Test_ONNX_layers, Expand) { + testONNXModels("expand"); + testONNXModels("expand_identity"); testONNXModels("expand_batch"); testONNXModels("expand_channels"); testONNXModels("expand_neg_batch"); @@ -611,6 +614,7 @@ TEST_P(Test_ONNX_layers, ReduceL2) testONNXModels("reduceL2"); testONNXModels("reduceL2_subgraph"); testONNXModels("reduceL2_subgraph_2"); + testONNXModels("reduceL2_subgraph2_2"); } TEST_P(Test_ONNX_layers, Split) @@ -624,6 +628,7 @@ TEST_P(Test_ONNX_layers, Split) testONNXModels("split_3"); testONNXModels("split_4"); testONNXModels("split_sizes"); + testONNXModels("split_neg_axis"); } TEST_P(Test_ONNX_layers, Slice) @@ -632,6 +637,7 @@ TEST_P(Test_ONNX_layers, Slice) testONNXModels("slice", npy, 0, 0, false, false); #else testONNXModels("slice"); + testONNXModels("slice_neg_starts"); testONNXModels("slice_opset_11"); #endif } From 068f33cfdf57ae5009d3e564a37a83bfefda9b22 Mon Sep 17 00:00:00 2001 From: Dale Phurrough Date: Thu, 9 Sep 2021 15:20:45 +0200 Subject: [PATCH 162/262] add nodiscard to features2d clone funcs --- .../features2d/include/opencv2/features2d.hpp | 8 ++++---- modules/python/src2/hdr_parser.py | 17 ++++++++++++----- 2 files changed, 16 insertions(+), 9 deletions(-) diff --git a/modules/features2d/include/opencv2/features2d.hpp b/modules/features2d/include/opencv2/features2d.hpp index cff09170c5..bf193599e1 100644 --- a/modules/features2d/include/opencv2/features2d.hpp +++ b/modules/features2d/include/opencv2/features2d.hpp @@ -1082,7 +1082,7 @@ public: that is, copies both parameters and train data. If emptyTrainData is true, the method creates an object copy with the current parameters but with empty train data. */ - CV_WRAP virtual Ptr clone( bool emptyTrainData=false ) const = 0; + CV_WRAP CV_NODISCARD_STD virtual Ptr clone( bool emptyTrainData=false ) const = 0; /** @brief Creates a descriptor matcher of a given type with the default parameters (using default constructor). @@ -1142,7 +1142,7 @@ protected: static bool isPossibleMatch( InputArray mask, int queryIdx, int trainIdx ); static bool isMaskedOut( InputArrayOfArrays masks, int queryIdx ); - static Mat clone_op( Mat m ) { return m.clone(); } + CV_NODISCARD_STD static Mat clone_op( Mat m ) { return m.clone(); } void checkMasks( InputArrayOfArrays masks, int queryDescriptorsCount ) const; //! Collection of descriptors from train images. @@ -1183,7 +1183,7 @@ public: */ CV_WRAP static Ptr create( int normType=NORM_L2, bool crossCheck=false ) ; - virtual Ptr clone( bool emptyTrainData=false ) const CV_OVERRIDE; + CV_NODISCARD_STD virtual Ptr clone( bool emptyTrainData=false ) const CV_OVERRIDE; protected: virtual void knnMatchImpl( InputArray queryDescriptors, std::vector >& matches, int k, InputArrayOfArrays masks=noArray(), bool compactResult=false ) CV_OVERRIDE; @@ -1222,7 +1222,7 @@ public: CV_WRAP static Ptr create(); - virtual Ptr clone( bool emptyTrainData=false ) const CV_OVERRIDE; + CV_NODISCARD_STD virtual Ptr clone( bool emptyTrainData=false ) const CV_OVERRIDE; protected: static void convertToDMatches( const DescriptorCollection& descriptors, const Mat& indices, const Mat& distances, diff --git a/modules/python/src2/hdr_parser.py b/modules/python/src2/hdr_parser.py index 749a9033ee..951dfe11c3 100755 --- a/modules/python/src2/hdr_parser.py +++ b/modules/python/src2/hdr_parser.py @@ -432,11 +432,18 @@ class CppHeaderParser(object): # filter off some common prefixes, which are meaningless for Python wrappers. # note that we do not strip "static" prefix, which does matter; # it means class methods, not instance methods - decl_str = self.batch_replace(decl_str, [("static inline", ""), ("inline", ""), ("explicit ", ""), - ("CV_EXPORTS_W", ""), ("CV_EXPORTS", ""), ("CV_CDECL", ""), - ("CV_WRAP ", " "), ("CV_INLINE", ""), - ("CV_DEPRECATED", ""), ("CV_DEPRECATED_EXTERNAL", "")]).strip() - + decl_str = self.batch_replace(decl_str, [("static inline", ""), + ("inline", ""), + ("explicit ", ""), + ("CV_EXPORTS_W", ""), + ("CV_EXPORTS", ""), + ("CV_CDECL", ""), + ("CV_WRAP ", " "), + ("CV_INLINE", ""), + ("CV_DEPRECATED", ""), + ("CV_DEPRECATED_EXTERNAL", ""), + ("CV_NODISCARD_STD", ""), + ("CV_NODISCARD", "")]).strip() if decl_str.strip().startswith('virtual'): virtual_method = True From c7e0888982a4d95b17ed46c7f8c4f1b3c3d81a95 Mon Sep 17 00:00:00 2001 From: fortemSteve <89345810+fortemSteve@users.noreply.github.com> Date: Thu, 9 Sep 2021 15:23:49 -0600 Subject: [PATCH 163/262] Merge pull request #20591 from fortemSteve:ffmpeg_get_stream_open_time Add CAP_PROP_STREAM_OPEN_TIME * Added CAP_PROP_STREAM_OPEN_TIME to videoio module - can be used to query the time at which the stream was opened, in seconds since Jan 1 1970 (midnight, UTC). Useful for RTSP and other live video where absolute timestamps are needed. Only applicable to ffmpeg backends * use nanoseconds instead of seconds to mark the stream open time, and change the cap prop name to CAP_PROP_STREAM_OPEN_TIME_NSEC * use microseconds for CAP_PROP_STREAM_OPEN_TIME (nanoseconds rolls over too soon, and milliseconds/seconds requires a division) * fix whitespace issue --- modules/videoio/include/opencv2/videoio.hpp | 1 + modules/videoio/src/cap_ffmpeg_impl.hpp | 3 +++ 2 files changed, 4 insertions(+) diff --git a/modules/videoio/include/opencv2/videoio.hpp b/modules/videoio/include/opencv2/videoio.hpp index 16016e4b8e..3276d0d5e4 100644 --- a/modules/videoio/include/opencv2/videoio.hpp +++ b/modules/videoio/include/opencv2/videoio.hpp @@ -188,6 +188,7 @@ enum VideoCaptureProperties { CAP_PROP_HW_ACCELERATION_USE_OPENCL=52, //!< (**open-only**) If non-zero, create new OpenCL context and bind it to current thread. The OpenCL context created with Video Acceleration context attached it (if not attached yet) for optimized GPU data copy between HW accelerated decoder and cv::UMat. CAP_PROP_OPEN_TIMEOUT_MSEC=53, //!< (**open-only**) timeout in milliseconds for opening a video capture (applicable for FFmpeg back-end only) CAP_PROP_READ_TIMEOUT_MSEC=54, //!< (**open-only**) timeout in milliseconds for reading from a video capture (applicable for FFmpeg back-end only) + CAP_PROP_STREAM_OPEN_TIME_USEC =55, //(use_opencl); #endif // USE_AV_HW_CODECS + case CAP_PROP_STREAM_OPEN_TIME_USEC: + //ic->start_time_realtime is in microseconds + return ((double)ic->start_time_realtime); default: break; } From d31b93b513df40f1549127836168a0fb17ca3cec Mon Sep 17 00:00:00 2001 From: rogday Date: Fri, 10 Sep 2021 14:07:16 +0300 Subject: [PATCH 164/262] Merge pull request #20674 from rogday:prelu_slope Fix PReLU negative slope access pattern * fix prelu negative slope access pattern * change begin() to ptr() --- modules/dnn/src/layers/elementwise_layers.cpp | 2 +- modules/dnn/test/test_onnx_importer.cpp | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/modules/dnn/src/layers/elementwise_layers.cpp b/modules/dnn/src/layers/elementwise_layers.cpp index e6cf714bff..6e7fa43a70 100644 --- a/modules/dnn/src/layers/elementwise_layers.cpp +++ b/modules/dnn/src/layers/elementwise_layers.cpp @@ -1542,7 +1542,7 @@ Ptr ChannelsPReLULayer::create(const LayerParams& params) if (params.blobs[0].total() == 1) { LayerParams reluParams = params; - reluParams.set("negative_slope", params.blobs[0].at(0)); + reluParams.set("negative_slope", *params.blobs[0].ptr()); return ReLULayer::create(reluParams); } Ptr l(new ElementWiseLayer(ChannelsPReLUFunctor(params.blobs[0]))); diff --git a/modules/dnn/test/test_onnx_importer.cpp b/modules/dnn/test/test_onnx_importer.cpp index 7b94e02d0a..f55510ec7b 100644 --- a/modules/dnn/test/test_onnx_importer.cpp +++ b/modules/dnn/test/test_onnx_importer.cpp @@ -250,6 +250,11 @@ TEST_P(Test_ONNX_layers, ReLU) testONNXModels("ReLU"); } +TEST_P(Test_ONNX_layers, PReLU) +{ + testONNXModels("PReLU_slope"); +} + TEST_P(Test_ONNX_layers, Clip) { testONNXModels("clip", npy); From e3f4f874c5214397a41befa6735f31382e7c39d2 Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Fri, 10 Sep 2021 15:00:11 +0300 Subject: [PATCH 165/262] Merge pull request #20670 from alalek:core_ocl_fix_intel_gpu_gemm_requirements core(OpenCL): fix intel_gpu_gemm kernel requirements * core(ocl): fix intel_gpu_gemm integration - allow bailout to generic OpenCL kernel * core(ocl): avoid failures of generic OpenCL gemm kernel * core(ocl): define alignment requirements of intel_gpu_gemm kernels --- modules/core/src/intel_gpu_gemm.inl.hpp | 71 +++++++---- modules/core/src/matmul.dispatch.cpp | 159 +++++++++++++----------- modules/core/test/ocl/test_gemm.cpp | 34 ++--- 3 files changed, 156 insertions(+), 108 deletions(-) diff --git a/modules/core/src/intel_gpu_gemm.inl.hpp b/modules/core/src/intel_gpu_gemm.inl.hpp index fbd567b949..fa66856f5e 100644 --- a/modules/core/src/intel_gpu_gemm.inl.hpp +++ b/modules/core/src/intel_gpu_gemm.inl.hpp @@ -24,11 +24,6 @@ #ifdef HAVE_OPENCL -#include -#include "opencl_kernels_core.hpp" -#include "opencv2/core/opencl/runtime/opencl_clamdblas.hpp" -#include "opencv2/core/opencl/runtime/opencl_core.hpp" - namespace cv { @@ -37,52 +32,79 @@ static bool intel_gpu_gemm( UMat B, Size sizeB, UMat D, Size sizeD, double alpha, double beta, - bool atrans, bool btrans) + bool atrans, bool btrans, + bool& isPropagatedC2D +) { CV_UNUSED(sizeB); int M = sizeD.height, N = sizeD.width, K = ((atrans)? sizeA.height : sizeA.width); - std::string kernelName; - bool ret = true; + if (M < 4 || N < 4 || K < 4) // vload4 + return false; - size_t lx = 8, ly = 4; - size_t dx = 4, dy = 8; + CV_LOG_VERBOSE(NULL, 0, "M=" << M << " N=" << N << " K=" << K); + + std::string kernelName; + + unsigned int lx = 8, ly = 4; + unsigned int dx = 4, dy = 8; if(!atrans && !btrans) { - if (M % 32 == 0 && N % 32 == 0 && K % 16 == 0) { kernelName = "intelblas_gemm_buffer_NN_sp"; } else { + if (M % 2 != 0) + return false; + // vload4(0, dst_write0) - 4 cols + // multiply by lx: 8 + if (N % (4*8) != 0) + return false; kernelName = "intelblas_gemm_buffer_NN"; } } else if(atrans && !btrans) { + if (M % 32 != 0) + return false; + if (N % 32 != 0) + return false; kernelName = "intelblas_gemm_buffer_TN"; } else if(!atrans && btrans) { + if (M % 128 != 0) + return false; + if (N % 8 != 0) + return false; + if (K % 512 != 0) + return false; kernelName = "intelblas_gemm_buffer_NT"; ly = 16; dx = 1; } else { + if (M % 32 != 0) + return false; + if (N % 32 != 0) + return false; + if (K % 16 != 0) + return false; kernelName = "intelblas_gemm_buffer_TT"; } - const size_t gx = (size_t)(N + dx - 1) / dx; - const size_t gy = (size_t)(M + dy - 1) / dy; + CV_LOG_DEBUG(NULL, "kernel: " << kernelName << " (M=" << M << " N=" << N << " K=" << K << ")"); + + const size_t gx = divUp((size_t)N, dx); + const size_t gy = divUp((size_t)M, dy); size_t local[] = {lx, ly, 1}; - size_t global[] = {(gx + lx - 1) / lx * lx, (gy + ly - 1) / ly * ly, 1}; - - int stride = (M * N < 1024 * 1024) ? 10000000 : 256; + size_t global[] = {roundUp(gx, lx), roundUp(gy, ly), 1}; ocl::Queue q; String errmsg; @@ -110,10 +132,13 @@ static bool intel_gpu_gemm( (int)(D.step / sizeof(float)) ); - ret = k.run(2, global, local, false, q); + bool ret = k.run(2, global, local, false, q); + return ret; } else { + int stride = (M * N < 1024 * 1024) ? 10000000 : 256; + for(int start_index = 0; start_index < K; start_index += stride) { ocl::Kernel k(kernelName.c_str(), program); @@ -132,12 +157,16 @@ static bool intel_gpu_gemm( (int) start_index, // 14 start_index stride); - ret = k.run(2, global, local, false, q); - if (!ret) return ret; + bool ret = k.run(2, global, local, false, q); + if (!ret) + { + if (start_index != 0) + isPropagatedC2D = false; // D array content is changed, need to rewrite + return false; + } } + return true; } - - return ret; } } // namespace cv diff --git a/modules/core/src/matmul.dispatch.cpp b/modules/core/src/matmul.dispatch.cpp index e81064ec16..a7447330fc 100644 --- a/modules/core/src/matmul.dispatch.cpp +++ b/modules/core/src/matmul.dispatch.cpp @@ -42,6 +42,8 @@ //M*/ #include "precomp.hpp" +#include + #include "opencl_kernels_core.hpp" #include "opencv2/core/opencl/runtime/opencl_clamdblas.hpp" #include "opencv2/core/opencl/runtime/opencl_core.hpp" @@ -155,10 +157,12 @@ static bool ocl_gemm_amdblas( InputArray matA, InputArray matB, double alpha, static bool ocl_gemm( InputArray matA, InputArray matB, double alpha, InputArray matC, double beta, OutputArray matD, int flags ) { - int depth = matA.depth(), cn = matA.channels(); - int type = CV_MAKETYPE(depth, cn); + int type = matA.type(); + int depth = CV_MAT_DEPTH(type); + int cn = CV_MAT_CN(type); - CV_Assert_N( type == matB.type(), (type == CV_32FC1 || type == CV_64FC1 || type == CV_32FC2 || type == CV_64FC2) ); + CV_CheckTypeEQ(type, matB.type(), ""); + CV_CheckType(type, type == CV_32FC1 || type == CV_64FC1 || type == CV_32FC2 || type == CV_64FC2, ""); const ocl::Device & dev = ocl::Device::getDefault(); bool doubleSupport = dev.doubleFPConfig() > 0; @@ -170,88 +174,103 @@ static bool ocl_gemm( InputArray matA, InputArray matB, double alpha, Size sizeA = matA.size(), sizeB = matB.size(), sizeC = haveC ? matC.size() : Size(0, 0); bool atrans = (flags & GEMM_1_T) != 0, btrans = (flags & GEMM_2_T) != 0, ctrans = (flags & GEMM_3_T) != 0; - CV_Assert( !haveC || matC.type() == type ); + if (haveC) + CV_CheckTypeEQ(type, matC.type(), ""); + + Size sizeD(((btrans) ? sizeB.height : sizeB.width), + ((atrans) ? sizeA.width : sizeA.height)); + + if (atrans) + sizeA = Size(sizeA.height, sizeA.width); + if (btrans) + sizeB = Size(sizeB.height, sizeB.width); + if (haveC && ctrans) + sizeC = Size(sizeC.height, sizeC.width); + + CV_CheckEQ(sizeA.width, sizeB.height, ""); + if (haveC) + CV_CheckEQ(sizeC, sizeD, ""); + + UMat A = matA.getUMat(); + UMat B = matB.getUMat(); - Size sizeD(((btrans)? sizeB.height : sizeB.width), - ((atrans)? sizeA.width : sizeA.height)); matD.create(sizeD, type); + UMat D = matD.getUMat(); - UMat A = matA.getUMat(), B = matB.getUMat(), D = matD.getUMat(); + bool isPropagatedC2D = false; // D content is updated with C / C.t() - - if (!dev.intelSubgroupsSupport() || (depth == CV_64F) || cn != 1) - { - String opts; - - if (atrans) - sizeA = Size(sizeA.height, sizeA.width); - if (btrans) - sizeB = Size(sizeB.height, sizeB.width); - if (haveC && ctrans) - sizeC = Size(sizeC.height, sizeC.width); - - CV_Assert( sizeA.width == sizeB.height && (!haveC || sizeC == sizeD) ); - - int max_wg_size = (int)dev.maxWorkGroupSize(); - int block_size = (max_wg_size / (32*cn) < 32) ? (max_wg_size / (16*cn) < 16) ? (max_wg_size / (8*cn) < 8) ? 1 : 8 : 16 : 32; - - if (atrans) - A = A.t(); - - if (btrans) - B = B.t(); - - if (haveC) - ctrans ? transpose(matC, D) : matC.copyTo(D); - - int vectorWidths[] = { 4, 4, 2, 2, 1, 4, cn, -1 }; - int kercn = ocl::checkOptimalVectorWidth(vectorWidths, B, D); - - opts += format(" -D T=%s -D T1=%s -D WT=%s -D cn=%d -D kercn=%d -D LOCAL_SIZE=%d%s%s%s", - ocl::typeToStr(type), ocl::typeToStr(depth), ocl::typeToStr(CV_MAKETYPE(depth, kercn)), - cn, kercn, block_size, - (sizeA.width % block_size !=0) ? " -D NO_MULT" : "", - haveC ? " -D HAVE_C" : "", - doubleSupport ? " -D DOUBLE_SUPPORT" : ""); - - ocl::Kernel k("gemm", cv::ocl::core::gemm_oclsrc, opts); - if (k.empty()) - return false; - - if (depth == CV_64F) - k.args(ocl::KernelArg::ReadOnlyNoSize(A), - ocl::KernelArg::ReadOnlyNoSize(B, cn, kercn), - ocl::KernelArg::ReadWrite(D, cn, kercn), - sizeA.width, alpha, beta); - else - k.args(ocl::KernelArg::ReadOnlyNoSize(A), - ocl::KernelArg::ReadOnlyNoSize(B, cn, kercn), - ocl::KernelArg::ReadWrite(D, cn, kercn), - sizeA.width, (float)alpha, (float)beta); - - size_t globalsize[2] = { (size_t)sizeD.width * cn / kercn, (size_t)sizeD.height}; - size_t localsize[2] = { (size_t)block_size, (size_t)block_size}; - - return k.run(2, globalsize, block_size!=1 ? localsize : NULL, false); - } - else + if (dev.intelSubgroupsSupport() && (depth == CV_32F) && cn == 1) { if (haveC && beta != 0.0) { ctrans ? transpose(matC, D) : matC.copyTo(D); + isPropagatedC2D = true; } else { beta = 0.0; } - return intel_gpu_gemm(A, sizeA, - B, sizeB, - D, sizeD, - alpha, - beta, - atrans, btrans); + bool res = intel_gpu_gemm(A, matA.size(), + B, matB.size(), + D, sizeD, + alpha, + beta, + atrans, btrans, + isPropagatedC2D); + if (res) + return true; + // fallback on generic OpenCL code } + + if (sizeD.width < 8 || sizeD.height < 8) + return false; + + String opts; + + int wg_size = (int)dev.maxWorkGroupSize(); + int sizeDmin = std::min(sizeD.width, sizeD.height); + wg_size = std::min(wg_size, sizeDmin * sizeDmin); + int block_size = (wg_size / (32*cn) < 32) ? (wg_size / (16*cn) < 16) ? (wg_size / (8*cn) < 8) ? 1 : 8 : 16 : 32; + + if (atrans) + A = A.t(); + + if (btrans) + B = B.t(); + + if (haveC && !isPropagatedC2D) + ctrans ? transpose(matC, D) : matC.copyTo(D); + + int vectorWidths[] = { 4, 4, 2, 2, 1, 4, cn, -1 }; + int kercn = ocl::checkOptimalVectorWidth(vectorWidths, B, D); + + opts += format(" -D T=%s -D T1=%s -D WT=%s -D cn=%d -D kercn=%d -D LOCAL_SIZE=%d%s%s%s", + ocl::typeToStr(type), ocl::typeToStr(depth), ocl::typeToStr(CV_MAKETYPE(depth, kercn)), + cn, kercn, block_size, + (sizeA.width % block_size !=0) ? " -D NO_MULT" : "", + haveC ? " -D HAVE_C" : "", + doubleSupport ? " -D DOUBLE_SUPPORT" : ""); + + ocl::Kernel k("gemm", cv::ocl::core::gemm_oclsrc, opts); + if (k.empty()) + return false; + + if (depth == CV_64F) + k.args(ocl::KernelArg::ReadOnlyNoSize(A), + ocl::KernelArg::ReadOnlyNoSize(B, cn, kercn), + ocl::KernelArg::ReadWrite(D, cn, kercn), + sizeA.width, alpha, beta); + else + k.args(ocl::KernelArg::ReadOnlyNoSize(A), + ocl::KernelArg::ReadOnlyNoSize(B, cn, kercn), + ocl::KernelArg::ReadWrite(D, cn, kercn), + sizeA.width, (float)alpha, (float)beta); + + size_t globalsize[2] = { (size_t)sizeD.width * cn / kercn, (size_t)sizeD.height}; + size_t localsize[2] = { (size_t)block_size, (size_t)block_size}; + + return k.run(2, globalsize, block_size !=1 ? localsize : NULL, false); } #endif diff --git a/modules/core/test/ocl/test_gemm.cpp b/modules/core/test/ocl/test_gemm.cpp index 825b506780..cb7cb0be1a 100644 --- a/modules/core/test/ocl/test_gemm.cpp +++ b/modules/core/test/ocl/test_gemm.cpp @@ -67,6 +67,8 @@ PARAM_TEST_CASE(Gemm, double alpha, beta; + int M, N, K; + TEST_DECLARE_INPUT_PARAMETER(A); TEST_DECLARE_INPUT_PARAMETER(B); TEST_DECLARE_INPUT_PARAMETER(C); @@ -90,30 +92,27 @@ PARAM_TEST_CASE(Gemm, void generateTestData() { - // set minimum size to 20, since testing less sizes doesn't make sense - Size ARoiSize = randomSize(20, MAX_VALUE); + M = (int)randomDoubleLog(1, 100); + N = (int)randomDoubleLog(1, 100); + K = (int)randomDoubleLog(1, 1200); + + M = roundUp(M, 1); + N = roundUp(N, 1); + K = roundUp(K, 1); + + Size ARoiSize = (atrans) ? Size(M, K) : Size(K, M); Border ABorder = randomBorder(0, use_roi ? MAX_VALUE : 0); randomSubMat(A, A_roi, ARoiSize, ABorder, type, -11, 11); - if (atrans) - ARoiSize = Size(ARoiSize.height, ARoiSize.width); - - Size BRoiSize = randomSize(20, MAX_VALUE); - if (btrans) - BRoiSize.width = ARoiSize.width; - else - BRoiSize.height = ARoiSize.width; - + Size BRoiSize = (btrans) ? Size(K, N) : Size(N, K); Border BBorder = randomBorder(0, use_roi ? MAX_VALUE : 0); randomSubMat(B, B_roi, BRoiSize, BBorder, type, -11, 11); - if (btrans) - BRoiSize = Size(BRoiSize.height, BRoiSize.width); - - Size DRoiSize = Size(BRoiSize.width, ARoiSize.height), CRoiSizeT(DRoiSize.height, DRoiSize.width); + Size CRoiSize = (ctrans) ? Size(M, N) : Size(N, M); Border CBorder = randomBorder(0, use_roi ? MAX_VALUE : 0); - randomSubMat(C, C_roi, ctrans ? CRoiSizeT : DRoiSize, CBorder, type, -11, 11); + randomSubMat(C, C_roi, CRoiSize, CBorder, type, -11, 11); + Size DRoiSize = Size(N, M); Border DBorder = randomBorder(0, use_roi ? MAX_VALUE : 0); randomSubMat(D, D_roi, DRoiSize, DBorder, type, -11, 11); @@ -132,11 +131,12 @@ OCL_TEST_P(Gemm, Accuracy) for (int i = 0; i < test_loop_times; ++i) { generateTestData(); + SCOPED_TRACE(cv::format("i=%d: M=%d N=%d K=%d", i, M, N, K)); OCL_OFF(cv::gemm(A_roi, B_roi, alpha, C_roi, beta, D_roi, flags)); OCL_ON(cv::gemm(uA_roi, uB_roi, alpha, uC_roi, beta, uD_roi, flags)); - double eps = D_roi.size().area() * 1e-4; + double eps = D_roi.size().area() * (1e-5 * K); OCL_EXPECT_MATS_NEAR(D, eps); } } From 9b4ecc96f63d64a07ac043ad06fa44a1fd02b18b Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Tue, 7 Sep 2021 04:39:28 +0000 Subject: [PATCH 166/262] core(ocl): buffer bounds in intelblas_gemm_buffer_NT --- modules/core/src/intel_gpu_gemm.inl.hpp | 6 +- modules/core/src/opencl/intel_gemm.cl | 180 +++++++++++------------- 2 files changed, 83 insertions(+), 103 deletions(-) diff --git a/modules/core/src/intel_gpu_gemm.inl.hpp b/modules/core/src/intel_gpu_gemm.inl.hpp index fa66856f5e..28cc4ab9b9 100644 --- a/modules/core/src/intel_gpu_gemm.inl.hpp +++ b/modules/core/src/intel_gpu_gemm.inl.hpp @@ -77,11 +77,7 @@ static bool intel_gpu_gemm( } else if(!atrans && btrans) { - if (M % 128 != 0) - return false; - if (N % 8 != 0) - return false; - if (K % 512 != 0) + if (K % 4 != 0) return false; kernelName = "intelblas_gemm_buffer_NT"; ly = 16; diff --git a/modules/core/src/opencl/intel_gemm.cl b/modules/core/src/opencl/intel_gemm.cl index 6cea8d7efd..53ae790779 100644 --- a/modules/core/src/opencl/intel_gemm.cl +++ b/modules/core/src/opencl/intel_gemm.cl @@ -392,6 +392,15 @@ __kernel void intelblas_gemm_buffer_NN( #define TILE_N 8 #define SLM_BLOCK 512 +/* + A K B.t() K D N + ----------- ----------- ----------- + | | | | | | + M | | x N | | => M | | + | | | | | | + ----------- ----------- ----------- +*/ + __attribute__((reqd_work_group_size(8, LWG_HEIGHT, 1))) __kernel void intelblas_gemm_buffer_NT( const __global float *src0, int off0, @@ -422,59 +431,79 @@ __kernel void intelblas_gemm_buffer_NT( float8 dot06 = 0.f; float8 dot07 = 0.f; - float4 brow0; - float4 brow1; - float4 brow2; - float4 brow3; - float4 brow4; - float4 brow5; - float4 brow6; - float4 brow7; + const int dst_row = (global_y * TILE_M); + __global float *dst_write0 = dst + global_x + dst_row * ldC + offd; - __global float *dst_write0 = dst + local_x * VEC_SIZE + ( group_x * TILE_N ) + ( group_y * LWG_HEIGHT * TILE_M + local_y * TILE_M) * ldC + offd; + const __global float *src0_read00 = src0 + off0; + const int a_row_base = global_y * TILE_M; + const int a_col_base = local_x * (TILE_K / 8); // <= TILE_K - 4 - const __global float *src0_read = src0 + local_x * ( TILE_K / 8 ) + ( group_y * LWG_HEIGHT * TILE_M + local_y * TILE_M ) * ldA + off0; - - const __global float *src1_read0 = src1 + ( group_x * TILE_N ) * ldB + off1; + const __global float *src1_read00 = src1 + off1; + const int b_row_base = (group_x * TILE_N); + //const int b_col_base = 0; __local float slm_brow[8 * SLM_BLOCK]; - __local float* slm_brow0; int local_index = mad24(local_y, 8, local_x) * 4; - int w; - for(int b_tile = 0; b_tile < K; b_tile += SLM_BLOCK) { + int w = 0; + for (int b_tile = 0; b_tile < K; b_tile += SLM_BLOCK) + { +#define UPDATE_BROW(_row) \ + { \ + float4 brow; \ + int b_row = b_row_base + _row; \ + int b_col = b_tile + local_index; \ + if (b_row < N && b_col <= K - 4 /*vload4*/) \ + brow = vload4(0, src1_read00 + mad24(b_row, ldB, b_col)); \ + else \ + brow = (float4)0; \ + vstore4(brow, 0, slm_brow + mad24(_row, SLM_BLOCK, local_index)); \ + } + barrier(CLK_LOCAL_MEM_FENCE); - vstore4(vload4(0, src1_read0 + mad24(0, ldB, local_index)), 0, slm_brow + mad24(0, SLM_BLOCK, local_index)); - vstore4(vload4(0, src1_read0 + mad24(1, ldB, local_index)), 0, slm_brow + mad24(1, SLM_BLOCK, local_index)); - vstore4(vload4(0, src1_read0 + mad24(2, ldB, local_index)), 0, slm_brow + mad24(2, SLM_BLOCK, local_index)); - vstore4(vload4(0, src1_read0 + mad24(3, ldB, local_index)), 0, slm_brow + mad24(3, SLM_BLOCK, local_index)); - vstore4(vload4(0, src1_read0 + mad24(4, ldB, local_index)), 0, slm_brow + mad24(4, SLM_BLOCK, local_index)); - vstore4(vload4(0, src1_read0 + mad24(5, ldB, local_index)), 0, slm_brow + mad24(5, SLM_BLOCK, local_index)); - vstore4(vload4(0, src1_read0 + mad24(6, ldB, local_index)), 0, slm_brow + mad24(6, SLM_BLOCK, local_index)); - vstore4(vload4(0, src1_read0 + mad24(7, ldB, local_index)), 0, slm_brow + mad24(7, SLM_BLOCK, local_index)); + UPDATE_BROW(0); + UPDATE_BROW(1); + UPDATE_BROW(2); + UPDATE_BROW(3); + UPDATE_BROW(4); + UPDATE_BROW(5); + UPDATE_BROW(6); + UPDATE_BROW(7); barrier(CLK_LOCAL_MEM_FENCE); +#undef UPDATE_BROW - slm_brow0 = slm_brow + local_x * (TILE_K / 8); - w = b_tile; - int end_w = min(b_tile + SLM_BLOCK, K); - while( w + TILE_K <= end_w ) { - float4 arow; + for (int k_tile_offset = 0; k_tile_offset < SLM_BLOCK; k_tile_offset += TILE_K) + { + int a_col = a_col_base + b_tile + k_tile_offset; - brow0 = vload4(0, slm_brow0 + 0 * SLM_BLOCK); - brow1 = vload4(0, slm_brow0 + 1 * SLM_BLOCK); - brow2 = vload4(0, slm_brow0 + 2 * SLM_BLOCK); - brow3 = vload4(0, slm_brow0 + 3 * SLM_BLOCK); - brow4 = vload4(0, slm_brow0 + 4 * SLM_BLOCK); - brow5 = vload4(0, slm_brow0 + 5 * SLM_BLOCK); - brow6 = vload4(0, slm_brow0 + 6 * SLM_BLOCK); - brow7 = vload4(0, slm_brow0 + 7 * SLM_BLOCK); + if (a_col > K - 4 /*vload4*/) + break; -#define MM_DOT_PRODUCT(_row,_dot) \ - arow = vload4(0, src0_read + _row * ldA); \ - _dot = mad( (float8)(arow.x), (float8)(brow0.x, brow1.x, brow2.x, brow3.x, brow4.x, brow5.x, brow6.x, brow7.x), _dot ); \ - _dot = mad( (float8)(arow.y), (float8)(brow0.y, brow1.y, brow2.y, brow3.y, brow4.y, brow5.y, brow6.y, brow7.y), _dot ); \ - _dot = mad( (float8)(arow.z), (float8)(brow0.z, brow1.z, brow2.z, brow3.z, brow4.z, brow5.z, brow6.z, brow7.z), _dot ); \ - _dot = mad( (float8)(arow.w), (float8)(brow0.w, brow1.w, brow2.w, brow3.w, brow4.w, brow5.w, brow6.w, brow7.w), _dot ); + int slm_brow_col = a_col_base + k_tile_offset; // <= SLM_BLOCK - 4 +#define READ_SLM_BROW(_row) \ + float4 brow##_row = vload4(0, slm_brow + mad24(_row, SLM_BLOCK, slm_brow_col)); + + READ_SLM_BROW(0); + READ_SLM_BROW(1); + READ_SLM_BROW(2); + READ_SLM_BROW(3); + READ_SLM_BROW(4); + READ_SLM_BROW(5); + READ_SLM_BROW(6); + READ_SLM_BROW(7); +#undef READ_SLM_BROW + +#define MM_DOT_PRODUCT(_row,_dot) \ + { \ + int a_row = a_row_base + _row; \ + if (a_row < M) { \ + float4 arow = vload4(0, src0_read00 + mad24(a_row, ldA, a_col)); \ + _dot = mad( (float8)(arow.x), (float8)(brow0.x, brow1.x, brow2.x, brow3.x, brow4.x, brow5.x, brow6.x, brow7.x), _dot ); \ + _dot = mad( (float8)(arow.y), (float8)(brow0.y, brow1.y, brow2.y, brow3.y, brow4.y, brow5.y, brow6.y, brow7.y), _dot ); \ + _dot = mad( (float8)(arow.z), (float8)(brow0.z, brow1.z, brow2.z, brow3.z, brow4.z, brow5.z, brow6.z, brow7.z), _dot ); \ + _dot = mad( (float8)(arow.w), (float8)(brow0.w, brow1.w, brow2.w, brow3.w, brow4.w, brow5.w, brow6.w, brow7.w), _dot ); \ + } \ + } MM_DOT_PRODUCT(0,dot00); MM_DOT_PRODUCT(1,dot01); @@ -485,53 +514,7 @@ __kernel void intelblas_gemm_buffer_NT( MM_DOT_PRODUCT(6,dot06); MM_DOT_PRODUCT(7,dot07); #undef MM_DOT_PRODUCT - - src0_read += TILE_K; - slm_brow0 += TILE_K; - w += TILE_K; } - src1_read0 += SLM_BLOCK; - } - - if(w < K) { - float4 arow; - -#define READ_BROW(_brow,_row) \ - _brow = vload4(0, slm_brow0 + _row * SLM_BLOCK); \ - _brow.x = (mad24(local_x, 4, w) < K) ? _brow.x : 0.0f; \ - _brow.y = (mad24(local_x, 4, w + 1) < K) ? _brow.y : 0.0f; \ - _brow.z = (mad24(local_x, 4, w + 2) < K) ? _brow.z : 0.0f; \ - _brow.w = (mad24(local_x, 4, w + 3) < K) ? _brow.w : 0.0f; - - READ_BROW(brow0,0); - READ_BROW(brow1,1); - READ_BROW(brow2,2); - READ_BROW(brow3,3); - READ_BROW(brow4,4); - READ_BROW(brow5,5); - READ_BROW(brow6,6); - READ_BROW(brow7,7); - -#define MM_DOT_PRODUCT(_row,_dot) \ - arow = vload4(0, src0_read + _row * ldA); \ - arow.x = (mad24(local_x, 4, w) < K) ? arow.x : 0.0f; \ - arow.y = (mad24(local_x, 4, w + 1) < K) ? arow.y : 0.0f; \ - arow.z = (mad24(local_x, 4, w + 2) < K) ? arow.z : 0.0f; \ - arow.w = (mad24(local_x, 4, w + 3) < K) ? arow.w : 0.0f; \ - _dot = mad( (float8)(arow.x), (float8)(brow0.x, brow1.x, brow2.x, brow3.x, brow4.x, brow5.x, brow6.x, brow7.x), _dot ); \ - _dot = mad( (float8)(arow.y), (float8)(brow0.y, brow1.y, brow2.y, brow3.y, brow4.y, brow5.y, brow6.y, brow7.y), _dot ); \ - _dot = mad( (float8)(arow.z), (float8)(brow0.z, brow1.z, brow2.z, brow3.z, brow4.z, brow5.z, brow6.z, brow7.z), _dot ); \ - _dot = mad( (float8)(arow.w), (float8)(brow0.w, brow1.w, brow2.w, brow3.w, brow4.w, brow5.w, brow6.w, brow7.w), _dot ); - - MM_DOT_PRODUCT(0,dot00); - MM_DOT_PRODUCT(1,dot01); - MM_DOT_PRODUCT(2,dot02); - MM_DOT_PRODUCT(3,dot03); - MM_DOT_PRODUCT(4,dot04); - MM_DOT_PRODUCT(5,dot05); - MM_DOT_PRODUCT(6,dot06); - MM_DOT_PRODUCT(7,dot07); -#undef MM_DOT_PRODUCT } #define REDUCE(_dot) \ @@ -572,21 +555,22 @@ __kernel void intelblas_gemm_buffer_NT( output = (local_x == 5) ? _dot.s5 : output; \ output = (local_x == 6) ? _dot.s6 : output; \ output = (local_x == 7) ? _dot.s7 : output; \ - if (beta != 0.0) \ + if (beta != 0.0f) \ dst_write0[0] = mad(output, (float)alpha, ((float)beta * dst_write0[0])); \ else \ dst_write0[0] = output * (float)alpha; \ dst_write0 += ldC; - if(global_x < N && global_y * 8 < M) { - OUTPUT(dot00); - if(mad24(global_y, 8, 1) < M) { OUTPUT(dot01); } - if(mad24(global_y, 8, 2) < M) { OUTPUT(dot02); } - if(mad24(global_y, 8, 3) < M) { OUTPUT(dot03); } - if(mad24(global_y, 8, 4) < M) { OUTPUT(dot04); } - if(mad24(global_y, 8, 5) < M) { OUTPUT(dot05); } - if(mad24(global_y, 8, 6) < M) { OUTPUT(dot06); } - if(mad24(global_y, 8, 7) < M) { OUTPUT(dot07); } + if (global_x < N && dst_row < M) + { + /*if (dst_row + 0 < M)*/ { OUTPUT(dot00); } + if (dst_row + 1 < M) { OUTPUT(dot01); } + if (dst_row + 2 < M) { OUTPUT(dot02); } + if (dst_row + 3 < M) { OUTPUT(dot03); } + if (dst_row + 4 < M) { OUTPUT(dot04); } + if (dst_row + 5 < M) { OUTPUT(dot05); } + if (dst_row + 6 < M) { OUTPUT(dot06); } + if (dst_row + 7 < M) { OUTPUT(dot07); } } #undef OUTPUT } From 1aacb9bb1513315267b42ecdc1bb05eb3b38a38b Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Fri, 10 Sep 2021 12:42:28 +0000 Subject: [PATCH 167/262] dnn(perf): update convolution tests --- modules/dnn/perf/perf_convolution.cpp | 232 +++++++++++++++++++++++++- 1 file changed, 226 insertions(+), 6 deletions(-) diff --git a/modules/dnn/perf/perf_convolution.cpp b/modules/dnn/perf/perf_convolution.cpp index c2a3a66ab9..bb890c6a00 100644 --- a/modules/dnn/perf/perf_convolution.cpp +++ b/modules/dnn/perf/perf_convolution.cpp @@ -26,25 +26,90 @@ struct ConvParam_t { double declared_flops; }; // Details: #12142 +// Last update: 2021-09 static const ConvParam_t testConvolutionConfigs[] = { + /* GFLOPS 3.398 x 20 = 67.956 */ {{7, 7}, {{1, 128, 46, 46}}, 128, 1, {1, 1}, {1, 1}, {3, 3}, {0, 0}, "", true, 3397788160.}, + /* GFLOPS 16.987 x 3 = 50.962 */ {{5, 5}, {{1, 1152, 16, 16}}, 1152, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 16987226112.}, + /* GFLOPS 23.122 x 2 = 46.244 */ {{5, 5}, {{1, 672, 32, 32}}, 672, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 23121788928.}, + /* GFLOPS 9.987 x 3 = 29.960 */ {{3, 3}, {{1, 256, 92, 92}}, 256, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 9986707456.}, + /* GFLOPS 1.595 x 16 = 25.524 */ {{3, 3}, {{1, 256, 26, 26}}, 512, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", false, 1595230208.}, + /* GFLOPS 4.566 x 5 = 22.828 */ {{7, 7}, {{1, 172, 46, 46}}, 128, 1, {1, 1}, {1, 1}, {3, 3}, {0, 0}, "", true, 4565684736.}, + /* GFLOPS 1.596 x 14 = 22.338 */ {{3, 3}, {{1, 128, 52, 52}}, 256, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", false, 1595576320.}, + /* GFLOPS 1.595 x 12 = 19.141 */ {{3, 3}, {{1, 512, 13, 13}}, 1024, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", false, 1595057152.}, + /* GFLOPS 6.814 x 2 = 13.629 */ {{3, 3}, {{1, 512, 38, 38}}, 512, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 6814386176.}, + /* GFLOPS 6.637 x 2 = 13.274 */ {{3, 3}, {{1, 256, 75, 75}}, 256, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 6636960000.}, + /* GFLOPS 11.797 x 1 = 11.797 */ {{5, 5}, {{1, 240, 64, 64}}, 240, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 11797463040.}, + /* GFLOPS 11.797 x 1 = 11.797 */ {{5, 5}, {{1, 480, 32, 32}}, 480, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 11796971520.}, + /* GFLOPS 10.701 x 1 = 10.701 */ {{3, 3}, {{1, 512, 38, 38}}, 804, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 10700715792.}, /* GFLOPS 10.087 x 1 = 10.087 */ {{3, 3}, {{1, 576, 38, 50}}, 512, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 10086963200.}, + /* GFLOPS 9.993 x 1 = 9.993 */ {{3, 3}, {{1, 64, 368, 368}}, 64, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 9993207808.}, + /* GFLOPS 9.989 x 1 = 9.989 */ {{3, 3}, {{1, 128, 184, 184}}, 128, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 9988874240.}, + /* GFLOPS 9.986 x 1 = 9.986 */ {{3, 3}, {{1, 512, 46, 46}}, 512, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 9985624064.}, /* GFLOPS 1.704 x 5 = 8.518 */ {{3, 3}, {{1, 512, 19, 19}}, 512, 512, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 1703596544.}, /* GFLOPS 1.704 x 5 = 8.518 */ {{3, 3}, {{1, 512, 19, 19}}, 512, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 1703596544.}, + /* GFLOPS 4.247 x 2 = 8.494 */ {{3, 3}, {{1, 480, 32, 32}}, 480, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 4247224320.}, + /* GFLOPS 8.025 x 1 = 8.025 */ {{3, 3}, {{1, 1024, 19, 19}}, 1206, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 8025101478.}, + /* GFLOPS 0.798 x 9 = 7.180 */ {{3, 3}, {{1, 128, 52, 52}}, 128, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", false, 797788160.}, + /* GFLOPS 0.798 x 9 = 7.179 */ {{3, 3}, {{1, 256, 26, 26}}, 256, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", false, 797615104.}, + /* GFLOPS 6.641 x 1 = 6.641 */ {{3, 3}, {{1, 64, 300, 300}}, 64, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 6641280000.}, /* GFLOPS 6.641 x 1 = 6.641 */ {{3, 3}, {{1, 64, 150, 200}}, 192, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 6641280000.}, + /* GFLOPS 6.638 x 1 = 6.638 */ {{3, 3}, {{1, 128, 150, 150}}, 128, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 6638400000.}, + /* GFLOPS 6.118 x 1 = 6.118 */ {{3, 3}, {{1, 144, 128, 128}}, 144, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 6117654528.}, + /* GFLOPS 6.116 x 1 = 6.116 */ {{3, 3}, {{1, 1152, 16, 16}}, 1152, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 6115590144.}, + /* GFLOPS 5.780 x 1 = 5.780 */ {{5, 5}, {{1, 672, 32, 32}}, 672, 1, {2, 2}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 5780447232.}, + /* GFLOPS 1.704 x 3 = 5.111 */ {{3, 3}, {{1, 512, 19, 19}}, 512, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 1703596544.}, + /* GFLOPS 4.997 x 1 = 4.997 */ {{3, 3}, {{1, 64, 184, 184}}, 128, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 4996603904.}, + /* GFLOPS 4.994 x 1 = 4.994 */ {{3, 3}, {{1, 128, 92, 92}}, 256, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 4994437120.}, + /* GFLOPS 4.993 x 1 = 4.993 */ {{3, 3}, {{1, 256, 46, 46}}, 512, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 4993353728.}, + /* GFLOPS 4.993 x 1 = 4.993 */ {{3, 3}, {{1, 512, 46, 46}}, 256, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 4992812032.}, /* GFLOPS 1.659 x 3 = 4.977 */ {{3, 3}, {{1, 960, 10, 10}}, 960, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 1658976000.}, /* GFLOPS 2.156 x 2 = 4.312 */ {{3, 3}, {{1, 576, 19, 19}}, 576, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 2156088384.}, + /* GFLOPS 4.247 x 1 = 4.247 */ {{5, 5}, {{1, 144, 128, 128}}, 144, 1, {2, 2}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 4247322624.}, + /* GFLOPS 0.798 x 5 = 3.988 */ {{3, 3}, {{1, 512, 13, 13}}, 512, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", false, 797528576.}, /* GFLOPS 0.958 x 4 = 3.833 */ {{3, 3}, {{1, 384, 19, 19}}, 384, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 958307712.}, + /* GFLOPS 0.624 x 6 = 3.746 */ {{3, 3}, {{1, 128, 46, 46}}, 128, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 624304640.}, + /* GFLOPS 3.408 x 1 = 3.408 */ {{3, 3}, {{1, 256, 38, 38}}, 512, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 3407562752.}, + /* GFLOPS 3.407 x 1 = 3.407 */ {{3, 3}, {{1, 512, 19, 19}}, 1024, 1, {1, 1}, {6, 6}, {6, 6}, {0, 0}, "", true, 3407193088.}, + /* GFLOPS 0.177 x 19 = 3.370 */ {{1, 1}, {{1, 512, 26, 26}}, 256, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 177382400.}, + /* GFLOPS 0.302 x 11 = 3.325 */ {{3, 3}, {{1, 64, 64, 64}}, 64, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 302252032.}, + /* GFLOPS 3.321 x 1 = 3.321 */ {{3, 3}, {{1, 64, 150, 150}}, 128, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 3320640000.}, /* GFLOPS 0.830 x 4 = 3.321 */ {{3, 3}, {{1, 64, 75, 100}}, 96, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 830160000.}, + /* GFLOPS 3.319 x 1 = 3.319 */ {{3, 3}, {{1, 128, 75, 75}}, 256, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 3319200000.}, + /* GFLOPS 1.598 x 2 = 3.195 */ {{3, 3}, {{1, 32, 416, 416}}, 64, 1, {2, 2}, {1, 1}, {1, 1}, {0, 0}, "", false, 1597652992.}, + /* GFLOPS 1.598 x 2 = 3.195 */ {{3, 3}, {{1, 32, 208, 208}}, 64, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", false, 1597652992.}, + /* GFLOPS 1.596 x 2 = 3.193 */ {{3, 3}, {{1, 64, 208, 208}}, 128, 1, {2, 2}, {1, 1}, {1, 1}, {0, 0}, "", false, 1596268544.}, + /* GFLOPS 1.596 x 2 = 3.193 */ {{3, 3}, {{1, 64, 104, 104}}, 128, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", false, 1596268544.}, + /* GFLOPS 1.596 x 2 = 3.191 */ {{3, 3}, {{1, 128, 104, 104}}, 256, 1, {2, 2}, {1, 1}, {1, 1}, {0, 0}, "", false, 1595576320.}, + /* GFLOPS 1.595 x 2 = 3.190 */ {{3, 3}, {{1, 256, 52, 52}}, 512, 1, {2, 2}, {1, 1}, {1, 1}, {0, 0}, "", false, 1595230208.}, + /* GFLOPS 1.595 x 2 = 3.190 */ {{3, 3}, {{1, 512, 26, 26}}, 1024, 1, {2, 2}, {1, 1}, {1, 1}, {0, 0}, "", false, 1595057152.}, + /* GFLOPS 0.178 x 16 = 2.841 */ {{1, 1}, {{1, 256, 52, 52}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 177555456.}, + /* GFLOPS 2.719 x 1 = 2.719 */ {{3, 3}, {{1, 96, 256, 256}}, 96, 1, {2, 2}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 2719481856.}, + /* GFLOPS 0.177 x 15 = 2.659 */ {{1, 1}, {{1, 1024, 13, 13}}, 512, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 177295872.}, /* GFLOPS 1.245 x 2 = 2.490 */ {{3, 3}, {{1, 96, 75, 100}}, 96, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 1244880000.}, + /* GFLOPS 0.798 x 3 = 2.394 */ {{3, 3}, {{1, 64, 104, 104}}, 64, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", false, 798134272.}, + /* GFLOPS 0.472 x 5 = 2.360 */ {{3, 3}, {{1, 256, 20, 20}}, 256, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 471961600.}, + /* GFLOPS 2.255 x 1 = 2.255 */ {{3, 3}, {{1, 128, 80, 100}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 2255285760.}, + /* GFLOPS 2.153 x 1 = 2.153 */ {{3, 3}, {{1, 128, 78, 98}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 2152611840.}, /* GFLOPS 2.100 x 1 = 2.100 */ {{3, 3}, {{1, 144, 75, 75}}, 144, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 2100330000.}, + /* GFLOPS 2.052 x 1 = 2.052 */ {{3, 3}, {{1, 128, 76, 96}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 2052298240.}, /* GFLOPS 1.022 x 2 = 2.044 */ {{3, 3}, {{1, 576, 19, 19}}, 273, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 1021896057.}, + /* GFLOPS 1.995 x 1 = 1.995 */ {{9, 9}, {{1, 3, 320, 400}}, 32, 1, {1, 1}, {1, 1}, {4, 4}, {0, 0}, "", true, 1994752000.}, + /* GFLOPS 1.954 x 1 = 1.954 */ {{3, 3}, {{1, 128, 74, 94}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 1954344960.}, /* GFLOPS 0.958 x 2 = 1.917 */ {{3, 3}, {{1, 192, 38, 38}}, 192, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 958446336.}, /* GFLOPS 1.888 x 1 = 1.888 */ {{3, 3}, {{1, 1024, 10, 10}}, 1024, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 1887539200.}, /* GFLOPS 1.888 x 1 = 1.888 */ {{3, 3}, {{1, 1024, 10, 10}}, 1024, 1024, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 1887539200.}, + /* GFLOPS 1.859 x 1 = 1.859 */ {{3, 3}, {{1, 128, 72, 92}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 1858752000.}, + /* GFLOPS 1.766 x 1 = 1.766 */ {{3, 3}, {{1, 128, 70, 90}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 1765519360.}, /* GFLOPS 1.704 x 1 = 1.704 */ {{3, 3}, {{1, 256, 38, 38}}, 256, 256, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 1703781376.}, /* GFLOPS 1.704 x 1 = 1.704 */ {{3, 3}, {{1, 256, 38, 38}}, 256, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 1703781376.}, + /* GFLOPS 1.675 x 1 = 1.675 */ {{3, 3}, {{1, 128, 68, 88}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 1674647040.}, /* GFLOPS 1.660 x 1 = 1.660 */ {{3, 3}, {{1, 128, 75, 75}}, 128, 128, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 1659600000.}, /* GFLOPS 1.660 x 1 = 1.660 */ {{3, 3}, {{1, 128, 75, 75}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 1659600000.}, + /* GFLOPS 1.586 x 1 = 1.586 */ {{3, 3}, {{1, 128, 66, 86}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 1586135040.}, + /* GFLOPS 1.500 x 1 = 1.500 */ {{3, 3}, {{1, 128, 64, 84}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 1499983360.}, + /* GFLOPS 1.416 x 1 = 1.416 */ {{3, 3}, {{1, 128, 62, 82}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 1416192000.}, + /* GFLOPS 0.472 x 3 = 1.416 */ {{3, 3}, {{1, 128, 40, 40}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 472064000.}, + /* GFLOPS 0.472 x 3 = 1.416 */ {{3, 3}, {{1, 512, 10, 10}}, 512, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 471910400.}, /* GFLOPS 0.280 x 5 = 1.402 */ {{1, 1}, {{1, 576, 38, 50}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 280409600.}, /* GFLOPS 0.701 x 2 = 1.401 */ {{3, 3}, {{1, 128, 38, 50}}, 160, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 700720000.}, /* GFLOPS 0.231 x 6 = 1.388 */ {{3, 3}, {{1, 128, 56, 56}}, 32, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", false, 231311360.}, @@ -53,20 +118,39 @@ static const ConvParam_t testConvolutionConfigs[] = { /* GFLOPS 0.420 x 3 = 1.261 */ {{3, 3}, {{1, 96, 38, 50}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 420492800.}, /* GFLOPS 1.261 x 1 = 1.261 */ {{3, 3}, {{1, 192, 38, 50}}, 192, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 1261113600.}, /* GFLOPS 1.258 x 1 = 1.258 */ {{3, 3}, {{1, 1280, 10, 10}}, 546, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 1258038600.}, + /* GFLOPS 1.248 x 1 = 1.248 */ {{3, 3}, {{1, 256, 46, 46}}, 128, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 1248338432.}, /* GFLOPS 1.245 x 1 = 1.245 */ {{3, 3}, {{1, 64, 75, 75}}, 192, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 1245240000.}, + /* GFLOPS 1.210 x 1 = 1.210 */ {{3, 3}, {{1, 32, 256, 256}}, 32, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 1210056704.}, + /* GFLOPS 1.196 x 1 = 1.196 */ {{3, 3}, {{1, 384, 26, 26}}, 256, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", false, 1196336128.}, + /* GFLOPS 1.195 x 1 = 1.195 */ {{9, 9}, {{1, 32, 240, 320}}, 3, 1, {1, 1}, {1, 1}, {4, 4}, {0, 0}, "", true, 1194624000.}, + /* GFLOPS 1.182 x 1 = 1.182 */ {{3, 3}, {{1, 32, 320, 400}}, 64, 1, {2, 2}, {1, 1}, {1, 1}, {0, 0}, "", true, 1181696000.}, + /* GFLOPS 1.181 x 1 = 1.181 */ {{3, 3}, {{1, 64, 160, 200}}, 128, 1, {2, 2}, {1, 1}, {1, 1}, {0, 0}, "", true, 1180672000.}, /* GFLOPS 0.561 x 2 = 1.121 */ {{3, 3}, {{1, 128, 38, 50}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 560576000.}, + /* GFLOPS 1.112 x 1 = 1.112 */ {{3, 3}, {{1, 512, 10, 10}}, 1206, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 1111570200.}, + /* GFLOPS 0.357 x 3 = 1.072 */ {{1, 1}, {{1, 64, 208, 208}}, 64, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 357187584.}, + /* GFLOPS 1.062 x 1 = 1.062 */ {{3, 3}, {{1, 240, 64, 64}}, 240, 1, {2, 2}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 1061928960.}, + /* GFLOPS 0.076 x 14 = 1.058 */ {{3, 3}, {{1, 64, 32, 32}}, 64, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 75563008.}, /* GFLOPS 1.051 x 1 = 1.051 */ {{3, 3}, {{1, 160, 38, 50}}, 192, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 1050988800.}, + /* GFLOPS 0.210 x 5 = 1.051 */ {{1, 1}, {{1, 256, 20, 20}}, 1024, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 210124800.}, + /* GFLOPS 0.210 x 5 = 1.049 */ {{1, 1}, {{1, 1024, 20, 20}}, 256, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 209817600.}, /* GFLOPS 1.006 x 1 = 1.006 */ {{3, 3}, {{1, 1024, 10, 10}}, 546, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 1006441800.}, /* GFLOPS 0.246 x 4 = 0.985 */ {{1, 1}, {{1, 256, 75, 100}}, 64, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 246240000.}, /* GFLOPS 0.189 x 5 = 0.947 */ {{1, 1}, {{1, 512, 19, 19}}, 512, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 189452800.}, /* GFLOPS 0.189 x 5 = 0.947 */ {{1, 1}, {{1, 512, 19, 19}}, 512, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 189452800.}, + /* GFLOPS 0.472 x 2 = 0.945 */ {{3, 3}, {{1, 64, 80, 80}}, 64, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 472268800.}, /* GFLOPS 0.934 x 1 = 0.934 */ {{3, 3}, {{1, 96, 150, 150}}, 96, 1, {2, 2}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 933660000.}, /* GFLOPS 0.231 x 4 = 0.925 */ {{3, 3}, {{1, 128, 28, 28}}, 128, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", false, 231311360.}, /* GFLOPS 0.896 x 1 = 0.896 */ {{5, 5}, {{1, 96, 27, 27}}, 256, 2, {1, 1}, {1, 1}, {2, 2}, {0, 0}, "", true, 895981824.}, + /* GFLOPS 0.089 x 10 = 0.890 */ {{1, 1}, {{1, 128, 52, 52}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 88950784.}, + /* GFLOPS 0.089 x 10 = 0.888 */ {{1, 1}, {{1, 256, 26, 26}}, 256, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 88777728.}, /* GFLOPS 0.876 x 1 = 0.876 */ {{3, 3}, {{1, 160, 38, 50}}, 160, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 875824000.}, /* GFLOPS 0.850 x 1 = 0.850 */ {{7, 7}, {{1, 3, 600, 800}}, 24, 1, {2, 2}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 849600000.}, /* GFLOPS 0.841 x 1 = 0.841 */ {{3, 3}, {{1, 128, 38, 50}}, 192, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 840864000.}, /* GFLOPS 0.415 x 2 = 0.831 */ {{3, 3}, {{1, 32, 150, 150}}, 32, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 415440000.}, + /* GFLOPS 0.757 x 1 = 0.757 */ {{1, 1}, {{1, 1024, 19, 19}}, 1024, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 757441536.}, + /* GFLOPS 0.712 x 1 = 0.712 */ {{1, 1}, {{1, 128, 208, 208}}, 64, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 711606272.}, + /* GFLOPS 0.178 x 4 = 0.712 */ {{1, 1}, {{1, 128, 104, 104}}, 64, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 177901568.}, + /* GFLOPS 0.354 x 2 = 0.707 */ {{1, 1}, {{1, 256, 52, 52}}, 255, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 353723760.}, /* GFLOPS 0.351 x 2 = 0.701 */ {{1, 1}, {{1, 576, 38, 50}}, 160, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 350512000.}, /* GFLOPS 0.701 x 1 = 0.701 */ {{3, 3}, {{1, 128, 75, 100}}, 160, 1, {2, 2}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 700720000.}, /* GFLOPS 0.694 x 1 = 0.694 */ {{3, 3}, {{1, 64, 56, 56}}, 192, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 694235136.}, @@ -75,19 +159,31 @@ static const ConvParam_t testConvolutionConfigs[] = { /* GFLOPS 0.058 x 12 = 0.694 */ {{3, 3}, {{1, 128, 28, 28}}, 32, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", false, 57827840.}, /* GFLOPS 0.231 x 3 = 0.694 */ {{3, 3}, {{1, 512, 7, 7}}, 512, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", false, 231236096.}, /* GFLOPS 0.160 x 4 = 0.639 */ {{3, 3}, {{1, 64, 38, 38}}, 96, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 159833472.}, + /* GFLOPS 0.211 x 3 = 0.634 */ {{1, 1}, {{1, 64, 80, 80}}, 256, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 211353600.}, + /* GFLOPS 0.211 x 3 = 0.632 */ {{1, 1}, {{1, 128, 40, 40}}, 512, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 210534400.}, + /* GFLOPS 0.210 x 3 = 0.630 */ {{1, 1}, {{1, 512, 40, 40}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 209920000.}, + /* GFLOPS 0.210 x 3 = 0.630 */ {{1, 1}, {{1, 512, 10, 10}}, 2048, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 209920000.}, /* GFLOPS 0.103 x 6 = 0.618 */ {{1, 1}, {{1, 256, 14, 14}}, 1024, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 102961152.}, /* GFLOPS 0.615 x 1 = 0.615 */ {{1, 1}, {{1, 320, 75, 100}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 615360000.}, + /* GFLOPS 0.305 x 2 = 0.609 */ {{3, 3}, {{1, 3, 416, 416}}, 32, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", false, 304578560.}, /* GFLOPS 0.597 x 1 = 0.597 */ {{3, 3}, {{1, 576, 19, 19}}, 576, 1, {2, 2}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 597254400.}, + /* GFLOPS 0.278 x 2 = 0.557 */ {{1, 1}, {{1, 128, 46, 46}}, 512, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 278431744.}, /* GFLOPS 0.185 x 3 = 0.554 */ {{1, 1}, {{1, 192, 75, 100}}, 64, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 184800000.}, /* GFLOPS 0.553 x 1 = 0.553 */ {{3, 3}, {{1, 64, 75, 100}}, 64, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 553440000.}, /* GFLOPS 0.539 x 1 = 0.539 */ {{3, 3}, {{1, 144, 75, 75}}, 144, 1, {2, 2}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 539178048.}, /* GFLOPS 0.103 x 5 = 0.514 */ {{1, 1}, {{1, 1024, 14, 14}}, 256, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 102810624.}, /* GFLOPS 0.491 x 1 = 0.491 */ {{1, 1}, {{1, 576, 38, 50}}, 224, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 490716800.}, + /* GFLOPS 0.483 x 1 = 0.483 */ {{7, 7}, {{1, 3, 320, 320}}, 64, 1, {2, 2}, {1, 1}, {3, 3}, {0, 0}, "", false, 483328000.}, /* GFLOPS 0.240 x 2 = 0.479 */ {{3, 3}, {{1, 96, 38, 38}}, 96, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 239680896.}, + /* GFLOPS 0.477 x 1 = 0.477 */ {{3, 3}, {{1, 3, 368, 368}}, 64, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 476692480.}, /* GFLOPS 0.237 x 2 = 0.474 */ {{7, 7}, {{1, 3, 224, 224}}, 64, 1, {2, 2}, {1, 1}, {3, 3}, {0, 0}, "", true, 236830720.}, /* GFLOPS 0.472 x 1 = 0.472 */ {{3, 3}, {{1, 512, 19, 19}}, 512, 512, {2, 2}, {1, 1}, {1, 1}, {0, 0}, "", true, 471910400.}, /* GFLOPS 0.472 x 1 = 0.472 */ {{3, 3}, {{1, 512, 19, 19}}, 512, 1, {2, 2}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 471910400.}, + /* GFLOPS 0.155 x 3 = 0.464 */ {{1, 1}, {{1, 112, 32, 32}}, 672, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 154828800.}, + /* GFLOPS 0.114 x 4 = 0.454 */ {{1, 1}, {{1, 192, 16, 16}}, 1152, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 113541120.}, /* GFLOPS 0.449 x 1 = 0.449 */ {{3, 3}, {{1, 384, 13, 13}}, 384, 2, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 448626048.}, + /* GFLOPS 0.089 x 5 = 0.443 */ {{1, 1}, {{1, 512, 13, 13}}, 512, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 88691200.}, + /* GFLOPS 0.428 x 1 = 0.428 */ {{1, 1}, {{1, 64, 64, 64}}, 810, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "VALID", true, 427991040.}, /* GFLOPS 0.426 x 1 = 0.426 */ {{3, 3}, {{1, 128, 75, 75}}, 128, 128, {2, 2}, {1, 1}, {1, 1}, {0, 0}, "", true, 426037760.}, /* GFLOPS 0.426 x 1 = 0.426 */ {{3, 3}, {{1, 128, 75, 75}}, 128, 1, {2, 2}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 426037760.}, /* GFLOPS 0.426 x 1 = 0.426 */ {{3, 3}, {{1, 128, 38, 38}}, 128, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", false, 426037760.}, @@ -95,46 +191,81 @@ static const ConvParam_t testConvolutionConfigs[] = { /* GFLOPS 0.426 x 1 = 0.426 */ {{3, 3}, {{1, 256, 38, 38}}, 256, 1, {2, 2}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 425945344.}, /* GFLOPS 0.426 x 1 = 0.426 */ {{3, 3}, {{1, 256, 19, 19}}, 256, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", false, 425945344.}, /* GFLOPS 0.421 x 1 = 0.421 */ {{1, 1}, {{1, 576, 38, 50}}, 192, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 420614400.}, + /* GFLOPS 0.420 x 1 = 0.420 */ {{1, 1}, {{1, 256, 40, 40}}, 512, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 420249600.}, + /* GFLOPS 0.210 x 2 = 0.420 */ {{1, 1}, {{1, 256, 80, 80}}, 64, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 210124800.}, + /* GFLOPS 0.420 x 1 = 0.420 */ {{1, 1}, {{1, 512, 20, 20}}, 1024, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 419840000.}, + /* GFLOPS 0.420 x 1 = 0.420 */ {{1, 1}, {{1, 1024, 10, 10}}, 2048, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 419635200.}, + /* GFLOPS 0.210 x 2 = 0.420 */ {{1, 1}, {{1, 2048, 10, 10}}, 512, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 209766400.}, /* GFLOPS 0.415 x 1 = 0.415 */ {{3, 3}, {{1, 32, 150, 150}}, 32, 32, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 415440000.}, /* GFLOPS 0.415 x 1 = 0.415 */ {{3, 3}, {{1, 64, 150, 150}}, 64, 64, {2, 2}, {1, 1}, {1, 1}, {0, 0}, "", true, 415080000.}, /* GFLOPS 0.415 x 1 = 0.415 */ {{3, 3}, {{1, 64, 150, 150}}, 64, 1, {2, 2}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 415080000.}, /* GFLOPS 0.104 x 4 = 0.414 */ {{1, 1}, {{1, 64, 56, 56}}, 256, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 103563264.}, /* GFLOPS 0.103 x 4 = 0.413 */ {{1, 1}, {{1, 128, 28, 28}}, 512, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 103161856.}, + /* GFLOPS 0.399 x 1 = 0.399 */ {{3, 3}, {{1, 32, 208, 208}}, 64, 1, {2, 2}, {1, 1}, {1, 1}, {0, 0}, "", false, 399413248.}, + /* GFLOPS 0.200 x 2 = 0.399 */ {{3, 3}, {{1, 32, 104, 104}}, 32, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", false, 199706624.}, + /* GFLOPS 0.200 x 2 = 0.399 */ {{3, 3}, {{1, 64, 52, 52}}, 64, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", false, 199533568.}, + /* GFLOPS 0.399 x 1 = 0.399 */ {{3, 3}, {{1, 128, 52, 52}}, 256, 1, {2, 2}, {1, 1}, {1, 1}, {0, 0}, "", false, 398894080.}, + /* GFLOPS 0.199 x 2 = 0.399 */ {{3, 3}, {{1, 128, 26, 26}}, 128, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", false, 199447040.}, + /* GFLOPS 0.399 x 1 = 0.399 */ {{3, 3}, {{1, 256, 26, 26}}, 512, 1, {2, 2}, {1, 1}, {1, 1}, {0, 0}, "", false, 398807552.}, + /* GFLOPS 0.399 x 1 = 0.399 */ {{3, 3}, {{1, 256, 13, 13}}, 512, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", false, 398807552.}, /* GFLOPS 0.376 x 1 = 0.376 */ {{1, 1}, {{1, 24, 300, 400}}, 64, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "VALID", true, 376320000.}, + /* GFLOPS 0.179 x 2 = 0.357 */ {{1, 1}, {{1, 64, 208, 208}}, 32, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 178593792.}, + /* GFLOPS 0.089 x 4 = 0.357 */ {{1, 1}, {{1, 64, 104, 104}}, 64, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 89296896.}, + /* GFLOPS 0.356 x 1 = 0.356 */ {{1, 1}, {{1, 128, 104, 104}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 355803136.}, + /* GFLOPS 0.355 x 1 = 0.355 */ {{1, 1}, {{1, 256, 52, 52}}, 256, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 355110912.}, + /* GFLOPS 0.355 x 1 = 0.355 */ {{1, 1}, {{1, 512, 26, 26}}, 512, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 354764800.}, + /* GFLOPS 0.355 x 1 = 0.355 */ {{1, 1}, {{1, 1024, 13, 13}}, 1024, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 354591744.}, + /* GFLOPS 0.355 x 1 = 0.355 */ {{1, 1}, {{1, 2048, 13, 13}}, 512, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 354505216.}, + /* GFLOPS 0.177 x 2 = 0.353 */ {{1, 1}, {{1, 512, 26, 26}}, 255, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 176689500.}, + /* GFLOPS 0.070 x 5 = 0.348 */ {{1, 1}, {{1, 128, 46, 46}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 69607936.}, /* GFLOPS 0.347 x 1 = 0.347 */ {{3, 3}, {{1, 128, 28, 28}}, 192, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 346967040.}, /* GFLOPS 0.347 x 1 = 0.347 */ {{3, 3}, {{1, 128, 28, 28}}, 192, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 346967040.}, /* GFLOPS 0.014 x 24 = 0.347 */ {{3, 3}, {{1, 128, 14, 14}}, 32, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", false, 14456960.}, + /* GFLOPS 0.113 x 3 = 0.340 */ {{1, 1}, {{1, 1152, 16, 16}}, 192, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 113295360.}, /* GFLOPS 0.053 x 6 = 0.320 */ {{1, 1}, {{1, 576, 19, 19}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 53277824.}, /* GFLOPS 0.319 x 1 = 0.319 */ {{3, 3}, {{1, 192, 19, 19}}, 256, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 319482112.}, + /* GFLOPS 0.317 x 1 = 0.317 */ {{3, 3}, {{1, 3, 300, 300}}, 64, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 316800000.}, /* GFLOPS 0.315 x 1 = 0.315 */ {{3, 3}, {{1, 96, 75, 100}}, 96, 1, {2, 2}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 315369600.}, /* GFLOPS 0.103 x 3 = 0.309 */ {{1, 1}, {{1, 512, 7, 7}}, 2048, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 102860800.}, /* GFLOPS 0.103 x 3 = 0.309 */ {{1, 1}, {{1, 512, 28, 28}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 102860800.}, + /* GFLOPS 0.154 x 2 = 0.309 */ {{1, 1}, {{1, 672, 32, 32}}, 112, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 154255360.}, /* GFLOPS 0.308 x 1 = 0.308 */ {{1, 1}, {{1, 320, 75, 100}}, 64, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 307680000.}, + /* GFLOPS 0.034 x 9 = 0.304 */ {{1, 1}, {{1, 64, 64, 64}}, 64, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "VALID", true, 33816576.}, /* GFLOPS 0.299 x 1 = 0.299 */ {{3, 3}, {{1, 256, 13, 13}}, 384, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 299105664.}, /* GFLOPS 0.299 x 1 = 0.299 */ {{3, 3}, {{1, 384, 13, 13}}, 256, 2, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 299084032.}, /* GFLOPS 0.017 x 17 = 0.290 */ {{1, 1}, {{1, 32, 32, 64}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 17039360.}, /* GFLOPS 0.017 x 16 = 0.269 */ {{1, 1}, {{1, 128, 32, 64}}, 32, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 16842752.}, /* GFLOPS 0.133 x 2 = 0.266 */ {{3, 3}, {{1, 128, 19, 19}}, 160, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 133136800.}, + /* GFLOPS 0.266 x 1 = 0.266 */ {{1, 1}, {{1, 384, 52, 52}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 266160128.}, + /* GFLOPS 0.266 x 1 = 0.266 */ {{1, 1}, {{1, 768, 26, 26}}, 256, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 265987072.}, /* GFLOPS 0.038 x 7 = 0.265 */ {{3, 3}, {{1, 16, 64, 128}}, 16, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 37879808.}, + /* GFLOPS 0.019 x 14 = 0.264 */ {{3, 3}, {{1, 64, 16, 16}}, 64, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 18890752.}, + /* GFLOPS 0.262 x 1 = 0.262 */ {{1, 1}, {{1, 2560, 20, 20}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 262195200.}, /* GFLOPS 0.126 x 2 = 0.252 */ {{3, 3}, {{1, 512, 5, 5}}, 546, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 125812050.}, /* GFLOPS 0.248 x 1 = 0.248 */ {{1, 1}, {{1, 64, 150, 200}}, 64, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 247680000.}, /* GFLOPS 0.040 x 6 = 0.240 */ {{1, 1}, {{1, 576, 19, 19}}, 96, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 39958368.}, /* GFLOPS 0.080 x 3 = 0.240 */ {{3, 3}, {{1, 96, 19, 19}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 79893632.}, /* GFLOPS 0.240 x 1 = 0.240 */ {{3, 3}, {{1, 192, 38, 38}}, 192, 1, {2, 2}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 239611584.}, /* GFLOPS 0.240 x 1 = 0.240 */ {{3, 3}, {{1, 192, 19, 19}}, 192, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 239611584.}, + /* GFLOPS 0.079 x 3 = 0.237 */ {{1, 1}, {{1, 80, 32, 32}}, 480, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 79134720.}, /* GFLOPS 0.237 x 1 = 0.237 */ {{7, 7}, {{1, 3, 224, 224}}, 64, 1, {2, 2}, {1, 1}, {3, 3}, {0, 0}, "", false, 236830720.}, /* GFLOPS 0.237 x 1 = 0.237 */ {{7, 7}, {{1, 3, 224, 224}}, 64, 1, {2, 2}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 236830720.}, + /* GFLOPS 0.118 x 2 = 0.236 */ {{3, 3}, {{1, 32, 80, 80}}, 32, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 118169600.}, + /* GFLOPS 0.236 x 1 = 0.236 */ {{3, 3}, {{1, 256, 19, 19}}, 512, 1, {2, 2}, {1, 1}, {1, 1}, {0, 0}, "", true, 235980800.}, + /* GFLOPS 0.116 x 2 = 0.231 */ {{1, 1}, {{1, 24, 128, 128}}, 144, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 115605504.}, /* GFLOPS 0.111 x 2 = 0.221 */ {{3, 3}, {{1, 192, 10, 10}}, 320, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 110624000.}, /* GFLOPS 0.213 x 1 = 0.213 */ {{3, 3}, {{1, 128, 38, 38}}, 256, 1, {2, 2}, {1, 1}, {1, 1}, {0, 0}, "", false, 213018880.}, /* GFLOPS 0.213 x 1 = 0.213 */ {{3, 3}, {{1, 128, 19, 19}}, 256, 1, {1, 1}, {2, 2}, {2, 2}, {0, 0}, "", false, 213018880.}, /* GFLOPS 0.107 x 2 = 0.213 */ {{3, 3}, {{1, 128, 19, 19}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 106509440.}, /* GFLOPS 0.213 x 1 = 0.213 */ {{3, 3}, {{1, 256, 19, 19}}, 128, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", false, 212972672.}, + /* GFLOPS 0.213 x 1 = 0.213 */ {{3, 3}, {{1, 512, 38, 38}}, 16, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 212949568.}, /* GFLOPS 0.212 x 1 = 0.212 */ {{7, 7}, {{1, 3, 300, 300}}, 32, 1, {2, 2}, {1, 1}, {3, 3}, {0, 0}, "", true, 212400000.}, /* GFLOPS 0.211 x 1 = 0.211 */ {{11, 11}, {{1, 3, 227, 227}}, 96, 1, {4, 4}, {1, 1}, {0, 0}, {0, 0}, "", true, 211120800.}, /* GFLOPS 0.210 x 1 = 0.210 */ {{3, 3}, {{1, 64, 38, 50}}, 96, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 210307200.}, /* GFLOPS 0.210 x 1 = 0.210 */ {{1, 1}, {{1, 1024, 10, 10}}, 1024, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 209817600.}, /* GFLOPS 0.210 x 1 = 0.210 */ {{1, 1}, {{1, 1024, 10, 10}}, 1024, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 209817600.}, /* GFLOPS 0.104 x 2 = 0.208 */ {{3, 3}, {{1, 32, 75, 75}}, 32, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", false, 103860000.}, + /* GFLOPS 0.208 x 1 = 0.208 */ {{1, 1}, {{1, 16, 256, 256}}, 96, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 207618048.}, /* GFLOPS 0.206 x 1 = 0.206 */ {{1, 1}, {{1, 256, 56, 56}}, 512, 1, {2, 2}, {1, 1}, {0, 0}, {0, 0}, "", false, 205922304.}, /* GFLOPS 0.206 x 1 = 0.206 */ {{1, 1}, {{1, 256, 56, 56}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 205922304.}, /* GFLOPS 0.103 x 2 = 0.206 */ {{1, 1}, {{1, 256, 56, 56}}, 64, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 102961152.}, @@ -148,27 +279,35 @@ static const ConvParam_t testConvolutionConfigs[] = { /* GFLOPS 0.190 x 1 = 0.190 */ {{1, 1}, {{1, 256, 38, 38}}, 256, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 189637632.}, /* GFLOPS 0.190 x 1 = 0.190 */ {{1, 1}, {{1, 256, 38, 38}}, 256, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 189637632.}, /* GFLOPS 0.047 x 4 = 0.190 */ {{1, 1}, {{1, 256, 38, 38}}, 64, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 47409408.}, + /* GFLOPS 0.189 x 1 = 0.189 */ {{1, 1}, {{1, 1024, 19, 19}}, 256, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 189360384.}, /* GFLOPS 0.038 x 5 = 0.189 */ {{3, 3}, {{1, 32, 32, 64}}, 32, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 37814272.}, + /* GFLOPS 0.189 x 1 = 0.189 */ {{1, 1}, {{1, 1152, 16, 16}}, 320, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 188825600.}, /* GFLOPS 0.185 x 1 = 0.185 */ {{1, 1}, {{1, 128, 75, 75}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 185040000.}, /* GFLOPS 0.185 x 1 = 0.185 */ {{1, 1}, {{1, 128, 75, 75}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 185040000.}, /* GFLOPS 0.181 x 1 = 0.181 */ {{3, 3}, {{1, 160, 14, 14}}, 320, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 180696320.}, /* GFLOPS 0.181 x 1 = 0.181 */ {{3, 3}, {{1, 160, 14, 14}}, 320, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 180696320.}, /* GFLOPS 0.090 x 2 = 0.181 */ {{3, 3}, {{1, 224, 10, 10}}, 224, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 90339200.}, /* GFLOPS 0.180 x 1 = 0.180 */ {{1, 1}, {{1, 224, 56, 56}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 180232192.}, + /* GFLOPS 0.088 x 2 = 0.177 */ {{1, 1}, {{1, 1024, 13, 13}}, 255, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 88301655.}, /* GFLOPS 0.174 x 1 = 0.174 */ {{3, 3}, {{1, 96, 28, 28}}, 128, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 173508608.}, /* GFLOPS 0.174 x 1 = 0.174 */ {{3, 3}, {{1, 96, 28, 28}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 173508608.}, /* GFLOPS 0.166 x 1 = 0.166 */ {{3, 3}, {{1, 160, 19, 19}}, 160, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 166406560.}, /* GFLOPS 0.080 x 2 = 0.160 */ {{1, 1}, {{1, 576, 19, 19}}, 192, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 79916736.}, /* GFLOPS 0.160 x 1 = 0.160 */ {{3, 3}, {{1, 128, 19, 19}}, 192, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 159764160.}, + /* GFLOPS 0.160 x 1 = 0.160 */ {{3, 3}, {{1, 1024, 19, 19}}, 24, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 159703512.}, /* GFLOPS 0.159 x 1 = 0.159 */ {{7, 7}, {{1, 3, 300, 300}}, 24, 1, {2, 2}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 159300000.}, + /* GFLOPS 0.080 x 2 = 0.159 */ {{1, 1}, {{1, 40, 64, 64}}, 240, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 79626240.}, + /* GFLOPS 0.079 x 2 = 0.157 */ {{1, 1}, {{1, 480, 32, 32}}, 80, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 78725120.}, /* GFLOPS 0.155 x 1 = 0.155 */ {{1, 1}, {{1, 192, 56, 56}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 154542080.}, /* GFLOPS 0.146 x 1 = 0.146 */ {{3, 3}, {{1, 144, 14, 14}}, 288, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 146369664.}, /* GFLOPS 0.146 x 1 = 0.146 */ {{3, 3}, {{1, 144, 14, 14}}, 288, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 146369664.}, /* GFLOPS 0.072 x 2 = 0.144 */ {{1, 1}, {{1, 1024, 10, 10}}, 352, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 72124800.}, /* GFLOPS 0.140 x 1 = 0.140 */ {{1, 1}, {{1, 576, 38, 50}}, 64, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 140204800.}, + /* GFLOPS 0.139 x 1 = 0.139 */ {{3, 3}, {{1, 256, 5, 5}}, 1206, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 138961350.}, /* GFLOPS 0.017 x 8 = 0.138 */ {{1, 1}, {{1, 16, 64, 128}}, 64, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 17301504.}, /* GFLOPS 0.067 x 2 = 0.133 */ {{1, 1}, {{1, 576, 19, 19}}, 160, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 66597280.}, /* GFLOPS 0.133 x 1 = 0.133 */ {{3, 3}, {{1, 128, 38, 38}}, 160, 1, {2, 2}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 133136800.}, + /* GFLOPS 0.044 x 3 = 0.133 */ {{1, 1}, {{1, 512, 13, 13}}, 256, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 44345600.}, /* GFLOPS 0.129 x 1 = 0.129 */ {{1, 1}, {{1, 160, 56, 56}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 128851968.}, /* GFLOPS 0.128 x 1 = 0.128 */ {{3, 3}, {{1, 64, 24, 24}}, 192, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 127512576.}, /* GFLOPS 0.120 x 1 = 0.120 */ {{5, 5}, {{1, 32, 28, 28}}, 96, 1, {1, 1}, {1, 1}, {2, 2}, {0, 0}, "", true, 120497664.}, @@ -176,22 +315,35 @@ static const ConvParam_t testConvolutionConfigs[] = { /* GFLOPS 0.040 x 3 = 0.120 */ {{1, 1}, {{1, 96, 19, 19}}, 576, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 40131648.}, /* GFLOPS 0.118 x 1 = 0.118 */ {{1, 1}, {{1, 320, 38, 38}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 118477312.}, /* GFLOPS 0.017 x 7 = 0.118 */ {{1, 1}, {{1, 64, 64, 128}}, 16, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 16908288.}, + /* GFLOPS 0.118 x 1 = 0.118 */ {{3, 3}, {{1, 64, 80, 80}}, 64, 1, {2, 2}, {1, 1}, {1, 1}, {0, 0}, "", false, 118067200.}, + /* GFLOPS 0.118 x 1 = 0.118 */ {{3, 3}, {{1, 64, 40, 40}}, 64, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 118067200.}, /* GFLOPS 0.039 x 3 = 0.118 */ {{1, 1}, {{1, 1024, 10, 10}}, 192, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 39340800.}, + /* GFLOPS 0.118 x 1 = 0.118 */ {{3, 3}, {{1, 128, 40, 40}}, 128, 1, {2, 2}, {1, 1}, {1, 1}, {0, 0}, "", false, 118016000.}, + /* GFLOPS 0.118 x 1 = 0.118 */ {{3, 3}, {{1, 128, 20, 20}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 118016000.}, + /* GFLOPS 0.118 x 1 = 0.118 */ {{3, 3}, {{1, 256, 20, 20}}, 256, 1, {2, 2}, {1, 1}, {1, 1}, {0, 0}, "", false, 117990400.}, /* GFLOPS 0.118 x 1 = 0.118 */ {{3, 3}, {{1, 256, 19, 19}}, 256, 1, {2, 2}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 117990400.}, /* GFLOPS 0.058 x 2 = 0.116 */ {{3, 3}, {{1, 16, 56, 56}}, 64, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 58003456.}, /* GFLOPS 0.058 x 2 = 0.116 */ {{3, 3}, {{1, 32, 28, 28}}, 128, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 57903104.}, /* GFLOPS 0.058 x 2 = 0.116 */ {{3, 3}, {{1, 64, 14, 14}}, 256, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 57852928.}, /* GFLOPS 0.116 x 1 = 0.116 */ {{3, 3}, {{1, 128, 14, 14}}, 256, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 115655680.}, /* GFLOPS 0.116 x 1 = 0.116 */ {{3, 3}, {{1, 128, 14, 14}}, 256, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 115655680.}, + /* GFLOPS 0.115 x 1 = 0.115 */ {{3, 3}, {{1, 3, 512, 512}}, 32, 1, {2, 2}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 115343360.}, + /* GFLOPS 0.114 x 1 = 0.114 */ {{1, 1}, {{1, 144, 128, 128}}, 24, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 113639424.}, /* GFLOPS 0.112 x 1 = 0.112 */ {{1, 1}, {{1, 1024, 10, 10}}, 546, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 111875400.}, + /* GFLOPS 0.110 x 1 = 0.110 */ {{1, 1}, {{1, 480, 32, 32}}, 112, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 110215168.}, + /* GFLOPS 0.107 x 1 = 0.107 */ {{1, 1}, {{1, 64, 32, 32}}, 810, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "VALID", true, 106997760.}, /* GFLOPS 0.036 x 3 = 0.107 */ {{1, 1}, {{1, 192, 38, 38}}, 64, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 35580160.}, /* GFLOPS 0.107 x 1 = 0.107 */ {{3, 3}, {{1, 32, 75, 75}}, 128, 1, {2, 2}, {1, 1}, {1, 1}, {0, 0}, "", false, 106648064.}, /* GFLOPS 0.107 x 1 = 0.107 */ {{3, 3}, {{1, 64, 38, 38}}, 64, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 106555648.}, + /* GFLOPS 0.105 x 1 = 0.105 */ {{1, 1}, {{1, 256, 40, 40}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 105062400.}, + /* GFLOPS 0.105 x 1 = 0.105 */ {{1, 1}, {{1, 512, 20, 20}}, 256, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 104960000.}, /* GFLOPS 0.105 x 1 = 0.105 */ {{1, 1}, {{1, 512, 10, 10}}, 1024, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 104960000.}, /* GFLOPS 0.105 x 1 = 0.105 */ {{1, 1}, {{1, 512, 10, 10}}, 1024, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 104960000.}, + /* GFLOPS 0.105 x 1 = 0.105 */ {{1, 1}, {{1, 1024, 10, 10}}, 512, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 104908800.}, /* GFLOPS 0.103 x 1 = 0.103 */ {{1, 1}, {{1, 128, 56, 56}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 103161856.}, /* GFLOPS 0.051 x 2 = 0.103 */ {{1, 1}, {{1, 256, 28, 28}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 51480576.}, /* GFLOPS 0.051 x 2 = 0.103 */ {{1, 1}, {{1, 256, 28, 28}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 51480576.}, + /* GFLOPS 0.008 x 12 = 0.101 */ {{1, 1}, {{1, 64, 32, 32}}, 64, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "VALID", true, 8454144.}, /* GFLOPS 0.101 x 1 = 0.101 */ {{1, 1}, {{1, 512, 19, 19}}, 273, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 101016825.}, /* GFLOPS 0.096 x 1 = 0.096 */ {{1, 1}, {{1, 480, 28, 28}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 96438272.}, /* GFLOPS 0.095 x 1 = 0.095 */ {{1, 1}, {{1, 128, 38, 38}}, 256, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 95003648.}, @@ -208,8 +360,10 @@ static const ConvParam_t testConvolutionConfigs[] = { /* GFLOPS 0.092 x 1 = 0.092 */ {{1, 1}, {{1, 192, 75, 100}}, 32, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 92400000.}, /* GFLOPS 0.090 x 1 = 0.090 */ {{1, 1}, {{1, 448, 28, 28}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 90015744.}, /* GFLOPS 0.045 x 2 = 0.090 */ {{3, 3}, {{1, 576, 19, 19}}, 12, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 44918508.}, + /* GFLOPS 0.044 x 2 = 0.089 */ {{1, 1}, {{1, 256, 26, 26}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 44388864.}, /* GFLOPS 0.089 x 1 = 0.089 */ {{3, 3}, {{1, 112, 14, 14}}, 224, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 88554368.}, /* GFLOPS 0.089 x 1 = 0.089 */ {{3, 3}, {{1, 112, 14, 14}}, 224, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 88554368.}, + /* GFLOPS 0.088 x 1 = 0.088 */ {{1, 1}, {{1, 256, 26, 26}}, 255, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 88430940.}, /* GFLOPS 0.021 x 4 = 0.084 */ {{5, 1}, {{1, 32, 32, 64}}, 32, 1, {1, 1}, {1, 1}, {2, 0}, {0, 0}, "", false, 21037056.}, /* GFLOPS 0.021 x 4 = 0.084 */ {{1, 5}, {{1, 32, 32, 64}}, 32, 1, {1, 1}, {1, 1}, {0, 2}, {0, 0}, "", true, 21037056.}, /* GFLOPS 0.084 x 1 = 0.084 */ {{1, 1}, {{1, 416, 28, 28}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 83593216.}, @@ -217,9 +371,13 @@ static const ConvParam_t testConvolutionConfigs[] = { /* GFLOPS 0.040 x 2 = 0.080 */ {{1, 1}, {{1, 576, 19, 19}}, 96, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 39958368.}, /* GFLOPS 0.040 x 2 = 0.079 */ {{1, 1}, {{1, 24, 75, 75}}, 144, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 39690000.}, /* GFLOPS 0.040 x 2 = 0.079 */ {{3, 3}, {{1, 3, 300, 300}}, 32, 1, {2, 2}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 39600000.}, + /* GFLOPS 0.079 x 1 = 0.079 */ {{1, 1}, {{1, 240, 64, 64}}, 40, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 78807040.}, + /* GFLOPS 0.079 x 1 = 0.079 */ {{1, 1}, {{1, 384, 40, 40}}, 64, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 78745600.}, /* GFLOPS 0.077 x 1 = 0.077 */ {{1, 1}, {{1, 96, 56, 56}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 77471744.}, /* GFLOPS 0.077 x 1 = 0.077 */ {{3, 3}, {{1, 192, 10, 10}}, 224, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 77436800.}, /* GFLOPS 0.077 x 1 = 0.077 */ {{1, 1}, {{1, 384, 28, 28}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 77170688.}, + /* GFLOPS 0.076 x 1 = 0.076 */ {{3, 3}, {{1, 3, 416, 416}}, 32, 1, {2, 2}, {1, 1}, {1, 1}, {0, 0}, "", false, 76144640.}, + /* GFLOPS 0.076 x 1 = 0.076 */ {{1, 1}, {{1, 96, 128, 128}}, 24, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 75890688.}, /* GFLOPS 0.038 x 2 = 0.076 */ {{3, 3}, {{1, 32, 32, 64}}, 32, 1, {1, 1}, {8, 8}, {8, 8}, {0, 0}, "", true, 37814272.}, /* GFLOPS 0.038 x 2 = 0.076 */ {{3, 3}, {{1, 32, 32, 64}}, 32, 1, {1, 1}, {4, 4}, {4, 4}, {0, 0}, "", true, 37814272.}, /* GFLOPS 0.038 x 2 = 0.076 */ {{3, 3}, {{1, 32, 32, 64}}, 32, 1, {1, 1}, {2, 2}, {2, 2}, {0, 0}, "", true, 37814272.}, @@ -230,6 +388,9 @@ static const ConvParam_t testConvolutionConfigs[] = { /* GFLOPS 0.071 x 1 = 0.071 */ {{1, 1}, {{1, 24, 150, 150}}, 64, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "VALID", true, 70560000.}, /* GFLOPS 0.070 x 1 = 0.070 */ {{3, 3}, {{1, 96, 14, 14}}, 208, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 70487872.}, /* GFLOPS 0.069 x 1 = 0.069 */ {{3, 3}, {{1, 96, 14, 14}}, 204, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 69132336.}, + /* GFLOPS 0.068 x 1 = 0.068 */ {{1, 1}, {{1, 32, 256, 256}}, 16, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 68157440.}, + /* GFLOPS 0.005 x 14 = 0.066 */ {{3, 3}, {{1, 64, 8, 8}}, 64, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 4722688.}, + /* GFLOPS 0.066 x 1 = 0.066 */ {{1, 1}, {{1, 672, 16, 16}}, 192, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 66109440.}, /* GFLOPS 0.066 x 1 = 0.066 */ {{1, 1}, {{1, 1280, 10, 10}}, 256, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 65561600.}, /* GFLOPS 0.033 x 2 = 0.065 */ {{3, 3}, {{1, 48, 14, 14}}, 192, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 32551680.}, /* GFLOPS 0.065 x 1 = 0.065 */ {{3, 3}, {{1, 192, 7, 7}}, 384, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 65046912.}, @@ -239,6 +400,7 @@ static const ConvParam_t testConvolutionConfigs[] = { /* GFLOPS 0.032 x 2 = 0.064 */ {{3, 3}, {{1, 96, 12, 12}}, 128, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 31868928.}, /* GFLOPS 0.061 x 1 = 0.061 */ {{1, 1}, {{1, 960, 10, 10}}, 320, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 61472000.}, /* GFLOPS 0.031 x 2 = 0.061 */ {{1, 1}, {{1, 960, 10, 10}}, 160, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 30736000.}, + /* GFLOPS 0.061 x 1 = 0.061 */ {{1, 1}, {{1, 512, 46, 46}}, 28, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 60729200.}, /* GFLOPS 0.060 x 1 = 0.060 */ {{3, 3}, {{1, 96, 38, 38}}, 96, 1, {2, 2}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 59920224.}, /* GFLOPS 0.059 x 1 = 0.059 */ {{1, 1}, {{1, 320, 38, 38}}, 64, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 59238656.}, /* GFLOPS 0.059 x 1 = 0.059 */ {{3, 3}, {{1, 128, 19, 19}}, 256, 1, {2, 2}, {1, 1}, {1, 1}, {0, 0}, "", true, 59008000.}, @@ -253,6 +415,11 @@ static const ConvParam_t testConvolutionConfigs[] = { /* GFLOPS 0.053 x 1 = 0.053 */ {{3, 3}, {{1, 128, 38, 38}}, 16, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 53254720.}, /* GFLOPS 0.053 x 1 = 0.053 */ {{1, 1}, {{1, 528, 14, 14}}, 256, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 53036032.}, /* GFLOPS 0.053 x 1 = 0.053 */ {{1, 1}, {{1, 528, 14, 14}}, 256, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 53036032.}, + /* GFLOPS 0.053 x 1 = 0.053 */ {{1, 1}, {{1, 64, 80, 80}}, 64, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 52838400.}, + /* GFLOPS 0.053 x 1 = 0.053 */ {{1, 1}, {{1, 64, 40, 40}}, 256, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 52838400.}, + /* GFLOPS 0.053 x 1 = 0.053 */ {{1, 1}, {{1, 128, 80, 80}}, 32, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 52633600.}, + /* GFLOPS 0.053 x 1 = 0.053 */ {{1, 1}, {{1, 128, 20, 20}}, 512, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 52633600.}, + /* GFLOPS 0.053 x 1 = 0.053 */ {{1, 1}, {{1, 256, 10, 10}}, 1024, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 52531200.}, /* GFLOPS 0.052 x 1 = 0.052 */ {{1, 1}, {{1, 1024, 10, 10}}, 256, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 52454400.}, /* GFLOPS 0.052 x 1 = 0.052 */ {{1, 1}, {{1, 1024, 10, 10}}, 256, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 52454400.}, /* GFLOPS 0.052 x 1 = 0.052 */ {{1, 1}, {{1, 1024, 10, 10}}, 256, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 52454400.}, @@ -268,6 +435,7 @@ static const ConvParam_t testConvolutionConfigs[] = { /* GFLOPS 0.050 x 1 = 0.050 */ {{1, 1}, {{1, 992, 14, 14}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 49799680.}, /* GFLOPS 0.048 x 1 = 0.048 */ {{1, 1}, {{1, 960, 14, 14}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 48194048.}, /* GFLOPS 0.047 x 1 = 0.047 */ {{1, 1}, {{1, 256, 19, 19}}, 256, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 47409408.}, + /* GFLOPS 0.047 x 1 = 0.047 */ {{1, 1}, {{1, 144, 64, 64}}, 40, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 47349760.}, /* GFLOPS 0.047 x 1 = 0.047 */ {{1, 1}, {{1, 512, 38, 50}}, 24, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 46740000.}, /* GFLOPS 0.047 x 1 = 0.047 */ {{1, 1}, {{1, 928, 14, 14}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 46588416.}, /* GFLOPS 0.046 x 1 = 0.046 */ {{1, 1}, {{1, 64, 75, 75}}, 64, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 46440000.}, @@ -280,6 +448,7 @@ static const ConvParam_t testConvolutionConfigs[] = { /* GFLOPS 0.045 x 1 = 0.045 */ {{3, 3}, {{1, 3, 227, 227}}, 64, 1, {2, 2}, {1, 1}, {0, 0}, {0, 0}, "", true, 44946880.}, /* GFLOPS 0.044 x 1 = 0.044 */ {{3, 3}, {{1, 128, 19, 19}}, 192, 1, {2, 2}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 44256000.}, /* GFLOPS 0.044 x 1 = 0.044 */ {{3, 3}, {{1, 1024, 10, 10}}, 24, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 44239200.}, + /* GFLOPS 0.044 x 1 = 0.044 */ {{1, 1}, {{1, 512, 13, 13}}, 255, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 44172375.}, /* GFLOPS 0.043 x 1 = 0.043 */ {{7, 7}, {{1, 3, 96, 96}}, 64, 1, {2, 2}, {1, 1}, {3, 3}, {0, 0}, "", true, 43499520.}, /* GFLOPS 0.043 x 1 = 0.043 */ {{1, 1}, {{1, 864, 14, 14}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 43377152.}, /* GFLOPS 0.042 x 1 = 0.042 */ {{1, 1}, {{1, 832, 14, 14}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 41771520.}, @@ -289,6 +458,7 @@ static const ConvParam_t testConvolutionConfigs[] = { /* GFLOPS 0.040 x 1 = 0.040 */ {{3, 3}, {{1, 64, 19, 19}}, 96, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 39958368.}, /* GFLOPS 0.040 x 1 = 0.040 */ {{3, 3}, {{1, 256, 19, 19}}, 24, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 39932376.}, /* GFLOPS 0.040 x 1 = 0.040 */ {{3, 3}, {{1, 3, 300, 300}}, 32, 1, {2, 2}, {1, 1}, {1, 1}, {0, 0}, "", true, 39600000.}, + /* GFLOPS 0.039 x 1 = 0.039 */ {{1, 1}, {{1, 240, 32, 32}}, 80, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 39403520.}, /* GFLOPS 0.039 x 1 = 0.039 */ {{1, 1}, {{1, 144, 75, 75}}, 24, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 39015000.}, /* GFLOPS 0.039 x 1 = 0.039 */ {{1, 1}, {{1, 192, 28, 28}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 38635520.}, /* GFLOPS 0.039 x 1 = 0.039 */ {{1, 1}, {{1, 768, 14, 14}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 38560256.}, @@ -297,9 +467,11 @@ static const ConvParam_t testConvolutionConfigs[] = { /* GFLOPS 0.036 x 1 = 0.036 */ {{1, 1}, {{1, 480, 14, 14}}, 192, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 36164352.}, /* GFLOPS 0.018 x 2 = 0.036 */ {{1, 1}, {{1, 192, 38, 38}}, 32, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 17790080.}, /* GFLOPS 0.035 x 1 = 0.035 */ {{1, 1}, {{1, 704, 14, 14}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 35348992.}, + /* GFLOPS 0.035 x 1 = 0.035 */ {{1, 1}, {{1, 512, 46, 46}}, 16, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 34702400.}, /* GFLOPS 0.034 x 1 = 0.034 */ {{1, 1}, {{1, 672, 14, 14}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 33743360.}, /* GFLOPS 0.034 x 1 = 0.034 */ {{1, 1}, {{1, 128, 32, 64}}, 64, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 33685504.}, /* GFLOPS 0.034 x 1 = 0.034 */ {{2, 2}, {{1, 64, 64, 128}}, 32, 1, {2, 2}, {1, 1}, {0, 0}, {0, 0}, "", false, 33619968.}, + /* GFLOPS 0.033 x 1 = 0.033 */ {{3, 3}, {{1, 256, 3, 3}}, 804, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 33350724.}, /* GFLOPS 0.033 x 1 = 0.033 */ {{1, 1}, {{1, 528, 14, 14}}, 160, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 33147520.}, /* GFLOPS 0.033 x 1 = 0.033 */ {{1, 1}, {{1, 528, 14, 14}}, 160, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 33147520.}, /* GFLOPS 0.033 x 1 = 0.033 */ {{1, 1}, {{1, 1024, 10, 10}}, 160, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 32784000.}, @@ -307,24 +479,29 @@ static const ConvParam_t testConvolutionConfigs[] = { /* GFLOPS 0.032 x 1 = 0.032 */ {{1, 1}, {{1, 512, 14, 14}}, 160, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 32144000.}, /* GFLOPS 0.032 x 1 = 0.032 */ {{1, 1}, {{1, 640, 14, 14}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 32137728.}, /* GFLOPS 0.032 x 1 = 0.032 */ {{1, 1}, {{1, 508, 14, 14}}, 160, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 31893120.}, + /* GFLOPS 0.011 x 3 = 0.032 */ {{1, 1}, {{1, 320, 16, 16}}, 64, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 10502144.}, /* GFLOPS 0.031 x 1 = 0.031 */ {{1, 1}, {{1, 832, 7, 7}}, 384, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 31328640.}, /* GFLOPS 0.031 x 1 = 0.031 */ {{1, 1}, {{1, 832, 7, 7}}, 384, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 31328640.}, /* GFLOPS 0.031 x 1 = 0.031 */ {{1, 1}, {{1, 608, 14, 14}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 30532096.}, + /* GFLOPS 0.015 x 2 = 0.030 */ {{1, 1}, {{1, 128, 46, 46}}, 28, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 15226736.}, /* GFLOPS 0.015 x 2 = 0.030 */ {{5, 5}, {{1, 24, 14, 14}}, 64, 1, {1, 1}, {1, 1}, {2, 2}, {0, 0}, "", true, 15065344.}, /* GFLOPS 0.015 x 2 = 0.030 */ {{5, 5}, {{1, 24, 14, 14}}, 64, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 15065344.}, /* GFLOPS 0.015 x 2 = 0.030 */ {{5, 5}, {{1, 48, 7, 7}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 15059072.}, /* GFLOPS 0.029 x 1 = 0.029 */ {{3, 3}, {{1, 256, 10, 10}}, 256, 1, {2, 2}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 29497600.}, + /* GFLOPS 0.015 x 2 = 0.029 */ {{1, 1}, {{1, 112, 32, 32}}, 64, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 14745600.}, /* GFLOPS 0.029 x 1 = 0.029 */ {{1, 1}, {{1, 192, 28, 28}}, 96, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 28976640.}, /* GFLOPS 0.029 x 1 = 0.029 */ {{1, 1}, {{1, 192, 28, 28}}, 96, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 28976640.}, /* GFLOPS 0.029 x 1 = 0.029 */ {{1, 1}, {{1, 512, 14, 14}}, 144, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 28929600.}, /* GFLOPS 0.029 x 1 = 0.029 */ {{1, 1}, {{1, 512, 14, 14}}, 144, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 28929600.}, /* GFLOPS 0.029 x 1 = 0.029 */ {{1, 1}, {{1, 576, 14, 14}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 28926464.}, /* GFLOPS 0.027 x 1 = 0.027 */ {{1, 1}, {{1, 544, 14, 14}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 27320832.}, + /* GFLOPS 0.027 x 1 = 0.027 */ {{1, 1}, {{1, 64, 16, 16}}, 810, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "VALID", true, 26749440.}, /* GFLOPS 0.027 x 1 = 0.027 */ {{1, 1}, {{1, 384, 19, 19}}, 96, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 26650464.}, /* GFLOPS 0.027 x 1 = 0.027 */ {{1, 1}, {{1, 576, 19, 19}}, 64, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 26638912.}, /* GFLOPS 0.027 x 1 = 0.027 */ {{3, 3}, {{1, 128, 38, 38}}, 8, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 26627360.}, /* GFLOPS 0.027 x 1 = 0.027 */ {{1, 1}, {{1, 528, 14, 14}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 26518016.}, /* GFLOPS 0.027 x 1 = 0.027 */ {{1, 1}, {{1, 528, 14, 14}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 26518016.}, + /* GFLOPS 0.009 x 3 = 0.026 */ {{1, 1}, {{1, 128, 46, 46}}, 16, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 8700992.}, /* GFLOPS 0.026 x 1 = 0.026 */ {{1, 1}, {{1, 96, 75, 75}}, 24, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 26055000.}, /* GFLOPS 0.026 x 1 = 0.026 */ {{1, 1}, {{1, 64, 56, 56}}, 64, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "VALID", true, 25890816.}, /* GFLOPS 0.026 x 1 = 0.026 */ {{1, 1}, {{1, 64, 56, 56}}, 64, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 25890816.}, @@ -336,6 +513,7 @@ static const ConvParam_t testConvolutionConfigs[] = { /* GFLOPS 0.013 x 2 = 0.026 */ {{1, 1}, {{1, 256, 28, 28}}, 32, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 12870144.}, /* GFLOPS 0.026 x 1 = 0.026 */ {{1, 1}, {{1, 512, 14, 14}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 25715200.}, /* GFLOPS 0.013 x 2 = 0.026 */ {{1, 1}, {{1, 512, 14, 14}}, 64, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 12857600.}, + /* GFLOPS 0.002 x 12 = 0.025 */ {{1, 1}, {{1, 64, 16, 16}}, 64, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "VALID", true, 2113536.}, /* GFLOPS 0.024 x 1 = 0.024 */ {{1, 1}, {{1, 480, 14, 14}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 24109568.}, /* GFLOPS 0.024 x 1 = 0.024 */ {{1, 1}, {{1, 128, 38, 38}}, 256, 1, {2, 2}, {1, 1}, {0, 0}, {0, 0}, "", false, 23750912.}, /* GFLOPS 0.024 x 1 = 0.024 */ {{1, 1}, {{1, 256, 19, 19}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 23704704.}, @@ -345,7 +523,9 @@ static const ConvParam_t testConvolutionConfigs[] = { /* GFLOPS 0.023 x 1 = 0.023 */ {{1, 1}, {{1, 448, 14, 14}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 22503936.}, /* GFLOPS 0.023 x 1 = 0.023 */ {{1, 1}, {{1, 512, 14, 14}}, 112, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 22500800.}, /* GFLOPS 0.022 x 1 = 0.022 */ {{1, 1}, {{1, 508, 14, 14}}, 112, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 22325184.}, + /* GFLOPS 0.022 x 1 = 0.022 */ {{3, 3}, {{1, 512, 10, 10}}, 24, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 22120800.}, /* GFLOPS 0.021 x 1 = 0.021 */ {{3, 3}, {{1, 128, 12, 12}}, 256, 1, {2, 2}, {1, 1}, {1, 1}, {0, 0}, "", true, 21242880.}, + /* GFLOPS 0.021 x 1 = 0.021 */ {{1, 1}, {{1, 40, 64, 64}}, 64, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 21233664.}, /* GFLOPS 0.021 x 1 = 0.021 */ {{1, 1}, {{1, 416, 14, 14}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 20898304.}, /* GFLOPS 0.021 x 1 = 0.021 */ {{1, 1}, {{1, 832, 7, 7}}, 256, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 20885760.}, /* GFLOPS 0.021 x 1 = 0.021 */ {{1, 1}, {{1, 832, 7, 7}}, 256, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 20885760.}, @@ -360,6 +540,7 @@ static const ConvParam_t testConvolutionConfigs[] = { /* GFLOPS 0.019 x 1 = 0.019 */ {{1, 1}, {{1, 192, 28, 28}}, 64, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 19317760.}, /* GFLOPS 0.019 x 1 = 0.019 */ {{1, 1}, {{1, 192, 28, 28}}, 64, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 19317760.}, /* GFLOPS 0.019 x 1 = 0.019 */ {{1, 1}, {{1, 384, 14, 14}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 19292672.}, + /* GFLOPS 0.019 x 1 = 0.019 */ {{1, 1}, {{1, 64, 64, 64}}, 36, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "VALID", true, 19021824.}, /* GFLOPS 0.018 x 1 = 0.018 */ {{1, 1}, {{1, 576, 10, 10}}, 160, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 18448000.}, /* GFLOPS 0.018 x 1 = 0.018 */ {{1, 1}, {{1, 480, 14, 14}}, 96, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 18082176.}, /* GFLOPS 0.018 x 1 = 0.018 */ {{1, 1}, {{1, 480, 14, 14}}, 96, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 18082176.}, @@ -371,13 +552,16 @@ static const ConvParam_t testConvolutionConfigs[] = { /* GFLOPS 0.016 x 1 = 0.016 */ {{1, 1}, {{1, 832, 7, 7}}, 192, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 15664320.}, /* GFLOPS 0.015 x 1 = 0.015 */ {{5, 5}, {{1, 48, 7, 7}}, 128, 1, {1, 1}, {1, 1}, {2, 2}, {0, 0}, "", true, 15059072.}, /* GFLOPS 0.015 x 1 = 0.015 */ {{5, 5}, {{1, 32, 12, 12}}, 64, 1, {1, 1}, {1, 1}, {2, 2}, {0, 0}, "", true, 14754816.}, + /* GFLOPS 0.015 x 1 = 0.015 */ {{3, 3}, {{1, 128, 10, 10}}, 256, 1, {2, 2}, {1, 1}, {1, 1}, {0, 0}, "", true, 14752000.}, /* GFLOPS 0.014 x 1 = 0.014 */ {{1, 1}, {{1, 288, 14, 14}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 14475776.}, /* GFLOPS 0.014 x 1 = 0.014 */ {{1, 1}, {{1, 512, 5, 5}}, 546, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 13991250.}, /* GFLOPS 0.013 x 1 = 0.013 */ {{1, 1}, {{1, 144, 38, 38}}, 32, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 13354112.}, /* GFLOPS 0.007 x 2 = 0.013 */ {{1, 1}, {{1, 16, 56, 56}}, 64, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 6623232.}, + /* GFLOPS 0.013 x 1 = 0.013 */ {{1, 1}, {{1, 512, 10, 10}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 13120000.}, /* GFLOPS 0.013 x 1 = 0.013 */ {{1, 1}, {{1, 832, 7, 7}}, 160, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 13053600.}, /* GFLOPS 0.013 x 1 = 0.013 */ {{1, 1}, {{1, 832, 7, 7}}, 160, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 13053600.}, /* GFLOPS 0.007 x 2 = 0.013 */ {{1, 1}, {{1, 32, 28, 28}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 6522880.}, + /* GFLOPS 0.001 x 11 = 0.013 */ {{3, 3}, {{1, 64, 4, 4}}, 64, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 1180672.}, /* GFLOPS 0.006 x 2 = 0.013 */ {{1, 1}, {{1, 64, 14, 14}}, 256, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 6472704.}, /* GFLOPS 0.013 x 1 = 0.013 */ {{1, 1}, {{1, 128, 56, 56}}, 16, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 12895232.}, /* GFLOPS 0.013 x 1 = 0.013 */ {{1, 1}, {{1, 256, 28, 28}}, 32, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 12870144.}, @@ -394,6 +578,7 @@ static const ConvParam_t testConvolutionConfigs[] = { /* GFLOPS 0.012 x 1 = 0.012 */ {{1, 1}, {{1, 640, 6, 6}}, 256, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 11805696.}, /* GFLOPS 0.012 x 1 = 0.012 */ {{1, 1}, {{1, 928, 7, 7}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 11647104.}, /* GFLOPS 0.011 x 1 = 0.011 */ {{1, 1}, {{1, 896, 7, 7}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 11245696.}, + /* GFLOPS 0.011 x 1 = 0.011 */ {{1, 1}, {{1, 256, 13, 13}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 11097216.}, /* GFLOPS 0.011 x 1 = 0.011 */ {{3, 3}, {{1, 256, 10, 10}}, 24, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 11061600.}, /* GFLOPS 0.006 x 2 = 0.011 */ {{3, 3}, {{1, 512, 5, 5}}, 24, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 5530200.}, /* GFLOPS 0.011 x 1 = 0.011 */ {{1, 1}, {{1, 864, 7, 7}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 10844288.}, @@ -417,13 +602,13 @@ static const ConvParam_t testConvolutionConfigs[] = { /* GFLOPS 0.008 x 1 = 0.008 */ {{1, 1}, {{1, 608, 7, 7}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 7633024.}, /* GFLOPS 0.008 x 1 = 0.008 */ {{5, 5}, {{1, 16, 14, 14}}, 48, 1, {1, 1}, {1, 1}, {2, 2}, {0, 0}, "", true, 7535808.}, /* GFLOPS 0.008 x 1 = 0.008 */ {{5, 5}, {{1, 16, 14, 14}}, 48, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 7535808.}, - /* GFLOPS 0.004 x 2 = 0.007 */ {{3, 3}, {{1, 64, 5, 5}}, 128, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 3689600.}, /* GFLOPS 0.007 x 1 = 0.007 */ {{1, 1}, {{1, 640, 6, 6}}, 160, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 7378560.}, /* GFLOPS 0.004 x 2 = 0.007 */ {{1, 1}, {{1, 48, 14, 14}}, 192, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 3650304.}, /* GFLOPS 0.007 x 1 = 0.007 */ {{1, 1}, {{1, 384, 14, 14}}, 48, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 7234752.}, /* GFLOPS 0.007 x 1 = 0.007 */ {{1, 1}, {{1, 576, 7, 7}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 7231616.}, /* GFLOPS 0.007 x 1 = 0.007 */ {{1, 1}, {{1, 256, 12, 12}}, 96, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 7091712.}, /* GFLOPS 0.007 x 1 = 0.007 */ {{1, 1}, {{1, 544, 7, 7}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 6830208.}, + /* GFLOPS 0.007 x 1 = 0.007 */ {{1, 1}, {{1, 64, 8, 8}}, 810, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "VALID", true, 6687360.}, /* GFLOPS 0.007 x 1 = 0.007 */ {{3, 3}, {{1, 160, 6, 6}}, 256, 1, {2, 2}, {1, 1}, {1, 1}, {0, 0}, "", true, 6637824.}, /* GFLOPS 0.007 x 1 = 0.007 */ {{1, 1}, {{1, 528, 14, 14}}, 32, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 6629504.}, /* GFLOPS 0.007 x 1 = 0.007 */ {{1, 1}, {{1, 528, 14, 14}}, 32, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 6629504.}, @@ -434,11 +619,13 @@ static const ConvParam_t testConvolutionConfigs[] = { /* GFLOPS 0.006 x 1 = 0.006 */ {{1, 1}, {{1, 512, 7, 7}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 6428800.}, /* GFLOPS 0.006 x 1 = 0.006 */ {{1, 1}, {{1, 512, 14, 14}}, 32, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 6428800.}, /* GFLOPS 0.006 x 1 = 0.006 */ {{1, 1}, {{1, 512, 14, 14}}, 32, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 6428800.}, + /* GFLOPS 0.001 x 12 = 0.006 */ {{1, 1}, {{1, 64, 8, 8}}, 64, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "VALID", true, 528384.}, /* GFLOPS 0.006 x 1 = 0.006 */ {{3, 3}, {{1, 256, 10, 10}}, 12, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 5530800.}, /* GFLOPS 0.005 x 1 = 0.005 */ {{1, 1}, {{1, 192, 12, 12}}, 96, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 5322240.}, /* GFLOPS 0.005 x 1 = 0.005 */ {{3, 3}, {{1, 128, 5, 5}}, 256, 1, {2, 2}, {1, 1}, {1, 1}, {0, 0}, "", true, 5310720.}, /* GFLOPS 0.005 x 1 = 0.005 */ {{3, 3}, {{1, 128, 5, 5}}, 256, 1, {2, 2}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 5310720.}, /* GFLOPS 0.005 x 1 = 0.005 */ {{3, 3}, {{1, 128, 5, 5}}, 256, 1, {2, 2}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 5310720.}, + /* GFLOPS 0.005 x 1 = 0.005 */ {{3, 3}, {{1, 128, 5, 5}}, 256, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 5310720.}, /* GFLOPS 0.005 x 1 = 0.005 */ {{1, 1}, {{1, 1024, 10, 10}}, 24, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 4917600.}, /* GFLOPS 0.005 x 1 = 0.005 */ {{1, 1}, {{1, 1024, 10, 10}}, 24, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 4917600.}, /* GFLOPS 0.005 x 1 = 0.005 */ {{1, 1}, {{1, 192, 28, 28}}, 16, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 4829440.}, @@ -446,6 +633,7 @@ static const ConvParam_t testConvolutionConfigs[] = { /* GFLOPS 0.005 x 1 = 0.005 */ {{1, 1}, {{1, 256, 14, 14}}, 48, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 4826304.}, /* GFLOPS 0.005 x 1 = 0.005 */ {{1, 1}, {{1, 512, 14, 14}}, 24, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 4821600.}, /* GFLOPS 0.005 x 1 = 0.005 */ {{1, 1}, {{1, 508, 14, 14}}, 24, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 4783968.}, + /* GFLOPS 0.005 x 1 = 0.005 */ {{1, 1}, {{1, 64, 32, 32}}, 36, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "VALID", true, 4755456.}, /* GFLOPS 0.005 x 1 = 0.005 */ {{1, 1}, {{1, 64, 24, 24}}, 64, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 4755456.}, /* GFLOPS 0.005 x 1 = 0.005 */ {{1, 1}, {{1, 256, 12, 12}}, 64, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 4727808.}, /* GFLOPS 0.005 x 1 = 0.005 */ {{1, 1}, {{1, 1024, 3, 3}}, 256, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 4720896.}, @@ -455,6 +643,7 @@ static const ConvParam_t testConvolutionConfigs[] = { /* GFLOPS 0.004 x 1 = 0.004 */ {{1, 1}, {{1, 16, 128, 256}}, 4, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 4325376.}, /* GFLOPS 0.004 x 1 = 0.004 */ {{1, 1}, {{1, 64, 64, 128}}, 4, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 4227072.}, /* GFLOPS 0.004 x 1 = 0.004 */ {{1, 1}, {{1, 832, 7, 7}}, 48, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 3916080.}, + /* GFLOPS 0.004 x 1 = 0.004 */ {{3, 3}, {{1, 256, 1, 1}}, 804, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 3705636.}, /* GFLOPS 0.004 x 1 = 0.004 */ {{5, 5}, {{1, 16, 12, 12}}, 32, 1, {1, 1}, {1, 1}, {2, 2}, {0, 0}, "", true, 3691008.}, /* GFLOPS 0.004 x 1 = 0.004 */ {{3, 3}, {{1, 64, 10, 10}}, 128, 1, {2, 2}, {1, 1}, {1, 1}, {0, 0}, "", true, 3689600.}, /* GFLOPS 0.004 x 1 = 0.004 */ {{5, 5}, {{1, 32, 6, 6}}, 64, 1, {1, 1}, {1, 1}, {2, 2}, {0, 0}, "", true, 3688704.}, @@ -470,6 +659,7 @@ static const ConvParam_t testConvolutionConfigs[] = { /* GFLOPS 0.003 x 1 = 0.003 */ {{1, 1}, {{1, 480, 14, 14}}, 16, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 3013696.}, /* GFLOPS 0.003 x 1 = 0.003 */ {{1, 1}, {{1, 320, 12, 12}}, 32, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 2953728.}, /* GFLOPS 0.003 x 1 = 0.003 */ {{1, 1}, {{1, 640, 6, 6}}, 64, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 2951424.}, + /* GFLOPS 0.003 x 1 = 0.003 */ {{3, 3}, {{1, 256, 5, 5}}, 24, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 2765400.}, /* GFLOPS 0.003 x 1 = 0.003 */ {{3, 3}, {{1, 128, 5, 5}}, 128, 1, {2, 2}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 2655360.}, /* GFLOPS 0.003 x 1 = 0.003 */ {{1, 1}, {{1, 832, 7, 7}}, 32, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 2610720.}, /* GFLOPS 0.003 x 1 = 0.003 */ {{1, 1}, {{1, 256, 3, 3}}, 546, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 2520882.}, @@ -482,32 +672,46 @@ static const ConvParam_t testConvolutionConfigs[] = { /* GFLOPS 0.002 x 1 = 0.002 */ {{1, 1}, {{1, 508, 4, 4}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 2082816.}, /* GFLOPS 0.002 x 1 = 0.002 */ {{1, 1}, {{1, 1024, 1, 1}}, 1000, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 2049000.}, /* GFLOPS 0.001 x 2 = 0.002 */ {{3, 3}, {{1, 256, 3, 3}}, 24, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 995544.}, - /* GFLOPS 0.001 x 2 = 0.002 */ {{3, 3}, {{1, 128, 5, 5}}, 16, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 922000.}, /* GFLOPS 0.002 x 1 = 0.002 */ {{1, 1}, {{1, 1024, 3, 3}}, 96, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 1770336.}, + /* GFLOPS 0.002 x 1 = 0.002 */ {{1, 1}, {{1, 64, 4, 4}}, 810, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "VALID", true, 1671840.}, + /* GFLOPS 0.002 x 1 = 0.002 */ {{1, 1}, {{1, 32, 80, 80}}, 4, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 1664000.}, + /* GFLOPS 0.002 x 1 = 0.002 */ {{1, 1}, {{1, 256, 5, 5}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 1641600.}, /* GFLOPS 0.001 x 1 = 0.001 */ {{1, 1}, {{1, 640, 6, 6}}, 32, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 1475712.}, /* GFLOPS 0.001 x 1 = 0.001 */ {{3, 3}, {{1, 128, 5, 5}}, 24, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 1383000.}, + /* GFLOPS 0.001 x 1 = 0.001 */ {{3, 3}, {{1, 64, 5, 5}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 1328256.}, /* GFLOPS 0.001 x 1 = 0.001 */ {{1, 1}, {{1, 736, 3, 3}}, 96, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 1272672.}, + /* GFLOPS 0.001 x 1 = 0.001 */ {{1, 1}, {{1, 64, 16, 16}}, 36, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "VALID", true, 1188864.}, + /* GFLOPS 0.000 x 9 = 0.001 */ {{1, 1}, {{1, 64, 4, 4}}, 64, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "VALID", true, 132096.}, + /* GFLOPS 0.001 x 2 = 0.001 */ {{1, 1}, {{1, 256, 3, 3}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 590976.}, /* GFLOPS 0.001 x 2 = 0.001 */ {{1, 1}, {{1, 256, 3, 3}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 590976.}, /* GFLOPS 0.001 x 1 = 0.001 */ {{3, 3}, {{1, 128, 3, 3}}, 128, 1, {2, 2}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 1180160.}, /* GFLOPS 0.001 x 1 = 0.001 */ {{1, 1}, {{1, 256, 2, 2}}, 546, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 1120392.}, - /* GFLOPS 0.000 x 2 = 0.001 */ {{3, 3}, {{1, 128, 5, 5}}, 8, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 461000.}, /* GFLOPS 0.001 x 1 = 0.001 */ {{1, 1}, {{1, 192, 12, 12}}, 16, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 887040.}, /* GFLOPS 0.000 x 2 = 0.001 */ {{3, 3}, {{1, 256, 2, 2}}, 24, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 442464.}, - /* GFLOPS 0.000 x 2 = 0.001 */ {{1, 1}, {{1, 128, 5, 5}}, 64, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 411200.}, + /* GFLOPS 0.000 x 2 = 0.001 */ {{1, 1}, {{1, 32, 80, 80}}, 1, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 416000.}, /* GFLOPS 0.001 x 1 = 0.001 */ {{3, 3}, {{1, 128, 5, 5}}, 12, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 691500.}, + /* GFLOPS 0.001 x 1 = 0.001 */ {{3, 3}, {{1, 256, 3, 3}}, 16, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 663696.}, /* GFLOPS 0.001 x 1 = 0.001 */ {{1, 1}, {{1, 640, 2, 2}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 655872.}, /* GFLOPS 0.001 x 1 = 0.001 */ {{1, 1}, {{1, 512, 5, 5}}, 24, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 615000.}, /* GFLOPS 0.001 x 1 = 0.001 */ {{1, 1}, {{1, 512, 5, 5}}, 24, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 615000.}, /* GFLOPS 0.001 x 1 = 0.001 */ {{1, 1}, {{1, 128, 3, 3}}, 256, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 592128.}, - /* GFLOPS 0.001 x 1 = 0.001 */ {{1, 1}, {{1, 256, 3, 3}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 590976.}, /* GFLOPS 0.001 x 1 = 0.001 */ {{1, 1}, {{1, 256, 3, 3}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 590976.}, + /* GFLOPS 0.001 x 1 = 0.001 */ {{3, 3}, {{1, 128, 3, 3}}, 256, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 590080.}, /* GFLOPS 0.001 x 1 = 0.001 */ {{1, 1}, {{1, 256, 3, 3}}, 126, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 581742.}, /* GFLOPS 0.001 x 1 = 0.001 */ {{1, 1}, {{1, 256, 4, 4}}, 64, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 525312.}, + /* GFLOPS 0.000 x 4 = 0.000 */ {{1, 1}, {{1, 48, 1, 1}}, 1152, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 111744.}, + /* GFLOPS 0.000 x 4 = 0.000 */ {{1, 1}, {{1, 1152, 1, 1}}, 48, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 110640.}, + /* GFLOPS 0.000 x 1 = 0.000 */ {{1, 1}, {{1, 128, 5, 5}}, 64, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 411200.}, + /* GFLOPS 0.000 x 1 = 0.000 */ {{3, 3}, {{1, 128, 3, 3}}, 16, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 331920.}, /* GFLOPS 0.000 x 1 = 0.000 */ {{1, 1}, {{1, 192, 5, 5}}, 32, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 308000.}, + /* GFLOPS 0.000 x 1 = 0.000 */ {{1, 1}, {{1, 64, 8, 8}}, 36, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "VALID", true, 297216.}, /* GFLOPS 0.000 x 1 = 0.000 */ {{1, 1}, {{1, 128, 2, 2}}, 256, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 263168.}, /* GFLOPS 0.000 x 2 = 0.000 */ {{1, 1}, {{1, 256, 2, 2}}, 64, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 131328.}, /* GFLOPS 0.000 x 1 = 0.000 */ {{1, 1}, {{1, 256, 2, 2}}, 126, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 258552.}, /* GFLOPS 0.000 x 1 = 0.000 */ {{1, 1}, {{1, 1024, 1, 1}}, 96, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 196704.}, + /* GFLOPS 0.000 x 1 = 0.000 */ {{3, 3}, {{1, 128, 3, 3}}, 8, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 165960.}, + /* GFLOPS 0.000 x 1 = 0.000 */ {{1, 1}, {{1, 128, 3, 3}}, 64, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 148032.}, + /* GFLOPS 0.000 x 1 = 0.000 */ {{3, 3}, {{1, 64, 3, 3}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 147584.}, /* GFLOPS 0.000 x 1 = 0.000 */ {{3, 3}, {{1, 64, 2, 2}}, 128, 1, {2, 2}, {1, 1}, {1, 1}, {0, 0}, "", true, 147584.}, /* GFLOPS 0.000 x 1 = 0.000 */ {{3, 3}, {{1, 64, 2, 2}}, 128, 1, {2, 2}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 147584.}, /* GFLOPS 0.000 x 1 = 0.000 */ {{3, 3}, {{1, 64, 2, 2}}, 128, 1, {2, 2}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 147584.}, @@ -515,16 +719,32 @@ static const ConvParam_t testConvolutionConfigs[] = { /* GFLOPS 0.000 x 1 = 0.000 */ {{1, 1}, {{1, 128, 1, 1}}, 546, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 140322.}, /* GFLOPS 0.000 x 1 = 0.000 */ {{1, 1}, {{1, 256, 2, 2}}, 64, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 131328.}, /* GFLOPS 0.000 x 1 = 0.000 */ {{1, 1}, {{1, 256, 2, 2}}, 64, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 131328.}, + /* GFLOPS 0.000 x 3 = 0.000 */ {{1, 1}, {{1, 28, 1, 1}}, 672, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 38304.}, + /* GFLOPS 0.000 x 3 = 0.000 */ {{1, 1}, {{1, 672, 1, 1}}, 28, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 37660.}, /* GFLOPS 0.000 x 1 = 0.000 */ {{1, 1}, {{1, 256, 3, 3}}, 24, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 110808.}, /* GFLOPS 0.000 x 1 = 0.000 */ {{1, 1}, {{1, 256, 3, 3}}, 24, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 110808.}, /* GFLOPS 0.000 x 2 = 0.000 */ {{3, 3}, {{1, 128, 1, 1}}, 24, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 55320.}, + /* GFLOPS 0.000 x 1 = 0.000 */ {{1, 1}, {{1, 64, 4, 4}}, 36, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "VALID", true, 74304.}, /* GFLOPS 0.000 x 1 = 0.000 */ {{3, 3}, {{1, 64, 2, 2}}, 64, 1, {2, 2}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 73792.}, + /* GFLOPS 0.000 x 1 = 0.000 */ {{3, 3}, {{1, 256, 1, 1}}, 16, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 73744.}, + /* GFLOPS 0.000 x 3 = 0.000 */ {{1, 1}, {{1, 20, 1, 1}}, 480, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 19680.}, + /* GFLOPS 0.000 x 3 = 0.000 */ {{1, 1}, {{1, 480, 1, 1}}, 20, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 19220.}, /* GFLOPS 0.000 x 1 = 0.000 */ {{1, 1}, {{1, 256, 2, 2}}, 24, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 49248.}, /* GFLOPS 0.000 x 1 = 0.000 */ {{1, 1}, {{1, 256, 2, 2}}, 24, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 49248.}, + /* GFLOPS 0.000 x 1 = 0.000 */ {{3, 3}, {{1, 128, 1, 1}}, 16, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 36880.}, /* GFLOPS 0.000 x 1 = 0.000 */ {{1, 1}, {{1, 128, 1, 1}}, 126, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 32382.}, + /* GFLOPS 0.000 x 1 = 0.000 */ {{3, 3}, {{1, 128, 1, 1}}, 8, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 18440.}, /* GFLOPS 0.000 x 1 = 0.000 */ {{1, 1}, {{1, 64, 1, 1}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 16512.}, + /* GFLOPS 0.000 x 2 = 0.000 */ {{1, 1}, {{1, 10, 1, 1}}, 240, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 5040.}, + /* GFLOPS 0.000 x 2 = 0.000 */ {{1, 1}, {{1, 240, 1, 1}}, 10, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 4810.}, /* GFLOPS 0.000 x 1 = 0.000 */ {{1, 1}, {{1, 128, 1, 1}}, 24, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 6168.}, - /* GFLOPS 0.000 x 1 = 0.000 */ {{1, 1}, {{1, 128, 1, 1}}, 24, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 6168.} + /* GFLOPS 0.000 x 1 = 0.000 */ {{1, 1}, {{1, 128, 1, 1}}, 24, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 6168.}, + /* GFLOPS 0.000 x 2 = 0.000 */ {{1, 1}, {{1, 6, 1, 1}}, 144, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 1872.}, + /* GFLOPS 0.000 x 2 = 0.000 */ {{1, 1}, {{1, 144, 1, 1}}, 6, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 1734.}, + /* GFLOPS 0.000 x 1 = 0.000 */ {{1, 1}, {{1, 4, 1, 1}}, 96, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 864.}, + /* GFLOPS 0.000 x 1 = 0.000 */ {{1, 1}, {{1, 96, 1, 1}}, 4, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 772.}, + /* GFLOPS 0.000 x 1 = 0.000 */ {{1, 1}, {{1, 8, 1, 1}}, 32, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 544.}, + /* GFLOPS 0.000 x 1 = 0.000 */ {{1, 1}, {{1, 32, 1, 1}}, 8, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 520.} }; struct ConvParamID { From de1a45987964d466a13b3f9ba2e1e59b3be38cca Mon Sep 17 00:00:00 2001 From: Dale Phurrough Date: Fri, 10 Sep 2021 17:59:56 +0200 Subject: [PATCH 168/262] fix opencv/opencv#20613 * copy 4.x selectOpenCLDevice() -- it is compatible * filter platforms rather than trying only first matching * this works on 3.4 and 4.x master --- modules/core/src/ocl.cpp | 59 ++++++++++++++++++++-------------------- 1 file changed, 30 insertions(+), 29 deletions(-) diff --git a/modules/core/src/ocl.cpp b/modules/core/src/ocl.cpp index a550c1d91a..e93e3094be 100644 --- a/modules/core/src/ocl.cpp +++ b/modules/core/src/ocl.cpp @@ -1670,7 +1670,7 @@ static bool parseOpenCLDeviceConfiguration(const std::string& configurationStr, split(configurationStr, ':', parts); if (parts.size() > 3) { - std::cerr << "ERROR: Invalid configuration string for OpenCL device" << std::endl; + CV_LOG_ERROR(NULL, "OpenCL: Invalid configuration string for OpenCL device: " << configurationStr); return false; } if (parts.size() > 2) @@ -1687,22 +1687,20 @@ static bool parseOpenCLDeviceConfiguration(const std::string& configurationStr, } #if defined WINRT || defined _WIN32_WCE -static cl_device_id selectOpenCLDevice() +static cl_device_id selectOpenCLDevice(const char* configuration = NULL) { + CV_UNUSED(configuration) return NULL; } #else -// std::tolower is int->int -static char char_tolower(char ch) -{ - return (char)std::tolower((int)ch); -} -static cl_device_id selectOpenCLDevice() +static cl_device_id selectOpenCLDevice(const char* configuration = NULL) { std::string platform, deviceName; std::vector deviceTypes; - const char* configuration = getenv("OPENCV_OPENCL_DEVICE"); + if (!configuration) + configuration = getenv("OPENCV_OPENCL_DEVICE"); + if (configuration && (strcmp(configuration, "disabled") == 0 || !parseOpenCLDeviceConfiguration(std::string(configuration), platform, deviceTypes, deviceName) @@ -1747,22 +1745,24 @@ static cl_device_id selectOpenCLDevice() platforms.resize(numPlatforms); } - int selectedPlatform = -1; if (platform.length() > 0) { - for (size_t i = 0; i < platforms.size(); i++) + for (std::vector::iterator currentPlatform = platforms.begin(); currentPlatform != platforms.end();) { std::string name; - CV_OCL_DBG_CHECK(getStringInfo(clGetPlatformInfo, platforms[i], CL_PLATFORM_NAME, name)); + CV_OCL_DBG_CHECK(getStringInfo(clGetPlatformInfo, *currentPlatform, CL_PLATFORM_NAME, name)); if (name.find(platform) != std::string::npos) { - selectedPlatform = (int)i; - break; + ++currentPlatform; + } + else + { + currentPlatform = platforms.erase(currentPlatform); } } - if (selectedPlatform == -1) + if (platforms.size() == 0) { - std::cerr << "ERROR: Can't find OpenCL platform by name: " << platform << std::endl; + CV_LOG_ERROR(NULL, "OpenCL: Can't find OpenCL platform by name: " << platform); goto not_found; } } @@ -1781,7 +1781,7 @@ static cl_device_id selectOpenCLDevice() { int deviceType = 0; std::string tempStrDeviceType = deviceTypes[t]; - std::transform(tempStrDeviceType.begin(), tempStrDeviceType.end(), tempStrDeviceType.begin(), char_tolower); + std::transform(tempStrDeviceType.begin(), tempStrDeviceType.end(), tempStrDeviceType.begin(), details::char_tolower); if (tempStrDeviceType == "gpu" || tempStrDeviceType == "dgpu" || tempStrDeviceType == "igpu") deviceType = Device::TYPE_GPU; @@ -1793,17 +1793,15 @@ static cl_device_id selectOpenCLDevice() deviceType = Device::TYPE_ALL; else { - std::cerr << "ERROR: Unsupported device type for OpenCL device (GPU, CPU, ACCELERATOR): " << deviceTypes[t] << std::endl; + CV_LOG_ERROR(NULL, "OpenCL: Unsupported device type for OpenCL device (GPU, CPU, ACCELERATOR): " << deviceTypes[t]); goto not_found; } - std::vector devices; // TODO Use clReleaseDevice to cleanup - for (int i = selectedPlatform >= 0 ? selectedPlatform : 0; - (selectedPlatform >= 0 ? i == selectedPlatform : true) && (i < (int)platforms.size()); - i++) + std::vector devices; + for (std::vector::iterator currentPlatform = platforms.begin(); currentPlatform != platforms.end(); ++currentPlatform) { cl_uint count = 0; - cl_int status = clGetDeviceIDs(platforms[i], deviceType, 0, NULL, &count); + cl_int status = clGetDeviceIDs(*currentPlatform, deviceType, 0, NULL, &count); if (!(status == CL_SUCCESS || status == CL_DEVICE_NOT_FOUND)) { CV_OCL_DBG_CHECK_RESULT(status, "clGetDeviceIDs get count"); @@ -1812,7 +1810,7 @@ static cl_device_id selectOpenCLDevice() continue; size_t base = devices.size(); devices.resize(base + count); - status = clGetDeviceIDs(platforms[i], deviceType, count, &devices[base], &count); + status = clGetDeviceIDs(*currentPlatform, deviceType, count, &devices[base], &count); if (!(status == CL_SUCCESS || status == CL_DEVICE_NOT_FOUND)) { CV_OCL_DBG_CHECK_RESULT(status, "clGetDeviceIDs get IDs"); @@ -1844,13 +1842,16 @@ not_found: if (!configuration) return NULL; // suppress messages on stderr - std::cerr << "ERROR: Requested OpenCL device not found, check configuration: " << configuration << std::endl - << " Platform: " << (platform.length() == 0 ? "any" : platform) << std::endl - << " Device types: "; + std::ostringstream msg; + msg << "ERROR: Requested OpenCL device not found, check configuration: '" << configuration << "'" << std::endl + << " Platform: " << (platform.length() == 0 ? "any" : platform) << std::endl + << " Device types:"; for (size_t t = 0; t < deviceTypes.size(); t++) - std::cerr << deviceTypes[t] << " "; + msg << ' ' << deviceTypes[t]; - std::cerr << std::endl << " Device name: " << (deviceName.length() == 0 ? "any" : deviceName) << std::endl; + msg << std::endl << " Device name: " << (deviceName.length() == 0 ? "any" : deviceName); + + CV_LOG_ERROR(NULL, msg.str()); return NULL; } #endif From aa7ba0bc1a99357beed7f4624de353f55ceccb20 Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Fri, 10 Sep 2021 20:58:37 +0000 Subject: [PATCH 169/262] build: winpack_dldt with dldt 2021.4.1 --- cmake/OpenCVDetectInferenceEngine.cmake | 4 +- ...-dldt-disable-multidevice-autoplugin.patch | 16 ++ ...20210630-dldt-disable-unused-targets.patch | 219 ++++++++++++++++++ .../2021.4.1/20210630-dldt-pdb.patch | 15 ++ .../2021.4.1/20210630-dldt-vs-version.patch | 16 ++ .../winpack_dldt/2021.4.1/build.config.py | 1 + .../winpack_dldt/2021.4.1/patch.config.py | 4 + .../winpack_dldt/2021.4.1/sysroot.config.py | 56 +++++ platforms/winpack_dldt/build_package.py | 5 +- 9 files changed, 332 insertions(+), 4 deletions(-) create mode 100644 platforms/winpack_dldt/2021.4.1/20210630-dldt-disable-multidevice-autoplugin.patch create mode 100644 platforms/winpack_dldt/2021.4.1/20210630-dldt-disable-unused-targets.patch create mode 100644 platforms/winpack_dldt/2021.4.1/20210630-dldt-pdb.patch create mode 100644 platforms/winpack_dldt/2021.4.1/20210630-dldt-vs-version.patch create mode 100644 platforms/winpack_dldt/2021.4.1/build.config.py create mode 100644 platforms/winpack_dldt/2021.4.1/patch.config.py create mode 100644 platforms/winpack_dldt/2021.4.1/sysroot.config.py diff --git a/cmake/OpenCVDetectInferenceEngine.cmake b/cmake/OpenCVDetectInferenceEngine.cmake index 6308d1b424..41951b710a 100644 --- a/cmake/OpenCVDetectInferenceEngine.cmake +++ b/cmake/OpenCVDetectInferenceEngine.cmake @@ -147,8 +147,8 @@ if(INF_ENGINE_TARGET) endif() endif() if(NOT INF_ENGINE_RELEASE AND NOT INF_ENGINE_RELEASE_INIT) - message(WARNING "InferenceEngine version has not been set, 2021.4 will be used by default. Set INF_ENGINE_RELEASE variable if you experience build errors.") - set(INF_ENGINE_RELEASE_INIT "2021040000") + message(WARNING "InferenceEngine version has not been set, 2021.4.1 will be used by default. Set INF_ENGINE_RELEASE variable if you experience build errors.") + set(INF_ENGINE_RELEASE_INIT "2021040100") elseif(DEFINED INF_ENGINE_RELEASE) set(INF_ENGINE_RELEASE_INIT "${INF_ENGINE_RELEASE}") endif() diff --git a/platforms/winpack_dldt/2021.4.1/20210630-dldt-disable-multidevice-autoplugin.patch b/platforms/winpack_dldt/2021.4.1/20210630-dldt-disable-multidevice-autoplugin.patch new file mode 100644 index 0000000000..f1e7487442 --- /dev/null +++ b/platforms/winpack_dldt/2021.4.1/20210630-dldt-disable-multidevice-autoplugin.patch @@ -0,0 +1,16 @@ +diff --git a/inference-engine/src/CMakeLists.txt b/inference-engine/src/CMakeLists.txt +index 0ba0dd78..7d34e7cb 100644 +--- a/inference-engine/src/CMakeLists.txt ++++ b/inference-engine/src/CMakeLists.txt +@@ -26,9 +26,9 @@ endif() + + add_subdirectory(hetero_plugin) + +-add_subdirectory(auto_plugin) ++#add_subdirectory(auto_plugin) + +-add_subdirectory(multi_device) ++#add_subdirectory(multi_device) + + add_subdirectory(transformations) + diff --git a/platforms/winpack_dldt/2021.4.1/20210630-dldt-disable-unused-targets.patch b/platforms/winpack_dldt/2021.4.1/20210630-dldt-disable-unused-targets.patch new file mode 100644 index 0000000000..9d44cdadc6 --- /dev/null +++ b/platforms/winpack_dldt/2021.4.1/20210630-dldt-disable-unused-targets.patch @@ -0,0 +1,219 @@ +diff --git a/cmake/developer_package/add_ie_target.cmake b/cmake/developer_package/add_ie_target.cmake +index d49f16a4d..2726ca787 100644 +--- a/cmake/developer_package/add_ie_target.cmake ++++ b/cmake/developer_package/add_ie_target.cmake +@@ -92,7 +92,7 @@ function(addIeTarget) + if (ARG_TYPE STREQUAL EXECUTABLE) + add_executable(${ARG_NAME} ${all_sources}) + elseif(ARG_TYPE STREQUAL STATIC OR ARG_TYPE STREQUAL SHARED) +- add_library(${ARG_NAME} ${ARG_TYPE} ${all_sources}) ++ add_library(${ARG_NAME} ${ARG_TYPE} EXCLUDE_FROM_ALL ${all_sources}) + else() + message(SEND_ERROR "Invalid target type ${ARG_TYPE} specified for target name ${ARG_NAME}") + endif() +diff --git a/inference-engine/CMakeLists.txt b/inference-engine/CMakeLists.txt +index 1ac7fd8bf..df7091e51 100644 +--- a/inference-engine/CMakeLists.txt ++++ b/inference-engine/CMakeLists.txt +@@ -39,7 +39,7 @@ if(ENABLE_TESTS) + add_subdirectory(tests) + endif() + +-add_subdirectory(tools) ++#add_subdirectory(tools) + + function(ie_build_samples) + # samples should be build with the same flags as from OpenVINO package, +@@ -58,7 +58,7 @@ endfunction() + + # gflags and format_reader targets are kept inside of samples directory and + # they must be built even if samples build is disabled (required for tests and tools). +-ie_build_samples() ++#ie_build_samples() + + if(ENABLE_PYTHON) + add_subdirectory(ie_bridges/python) +@@ -142,7 +142,7 @@ endif() + # Developer package + # + +-openvino_developer_export_targets(COMPONENT openvino_common TARGETS format_reader gflags ie_samples_utils) ++#openvino_developer_export_targets(COMPONENT openvino_common TARGETS format_reader gflags ie_samples_utils) + + # for Template plugin + if(NGRAPH_INTERPRETER_ENABLE) +@@ -166,7 +166,7 @@ function(ie_generate_dev_package_config) + @ONLY) + endfunction() + +-ie_generate_dev_package_config() ++#ie_generate_dev_package_config() + + # + # Coverage +diff --git a/inference-engine/src/inference_engine/CMakeLists.txt b/inference-engine/src/inference_engine/CMakeLists.txt +index e8ed1a5c4..1fc9fc3ff 100644 +--- a/inference-engine/src/inference_engine/CMakeLists.txt ++++ b/inference-engine/src/inference_engine/CMakeLists.txt +@@ -110,7 +110,7 @@ add_cpplint_target(${TARGET_NAME}_plugin_api_cpplint FOR_SOURCES ${plugin_api_sr + + # Create object library + +-add_library(${TARGET_NAME}_obj OBJECT ++add_library(${TARGET_NAME}_obj OBJECT EXCLUDE_FROM_ALL + ${LIBRARY_SRC} + ${LIBRARY_HEADERS} + ${PUBLIC_HEADERS}) +@@ -181,7 +181,7 @@ ie_add_api_validator_post_build_step(TARGET ${TARGET_NAME}) + + # Static library used for unit tests which are always built + +-add_library(${TARGET_NAME}_s STATIC ++add_library(${TARGET_NAME}_s STATIC EXCLUDE_FROM_ALL + $ + $ + ${IE_STATIC_DEPENDENT_FILES}) +diff --git a/inference-engine/src/legacy_api/CMakeLists.txt b/inference-engine/src/legacy_api/CMakeLists.txt +index 8eae82bd2..e0e6745b1 100644 +--- a/inference-engine/src/legacy_api/CMakeLists.txt ++++ b/inference-engine/src/legacy_api/CMakeLists.txt +@@ -26,7 +26,7 @@ endif() + + file(TOUCH ${CMAKE_CURRENT_BINARY_DIR}/dummy.cpp) + +-add_library(${TARGET_NAME}_obj OBJECT ++add_library(${TARGET_NAME}_obj OBJECT EXCLUDE_FROM_ALL + ${LIBRARY_SRC} + ${PUBLIC_HEADERS}) + +diff --git a/inference-engine/src/mkldnn_plugin/CMakeLists.txt b/inference-engine/src/mkldnn_plugin/CMakeLists.txt +index fe57b29dd..07831e2fb 100644 +--- a/inference-engine/src/mkldnn_plugin/CMakeLists.txt ++++ b/inference-engine/src/mkldnn_plugin/CMakeLists.txt +@@ -67,7 +67,7 @@ ie_add_api_validator_post_build_step(TARGET ${TARGET_NAME}) + + # add test object library + +-add_library(${TARGET_NAME}_obj OBJECT ${SOURCES} ${HEADERS}) ++add_library(${TARGET_NAME}_obj OBJECT EXCLUDE_FROM_ALL ${SOURCES} ${HEADERS}) + target_link_libraries(${TARGET_NAME}_obj PUBLIC mkldnn) + + target_include_directories(${TARGET_NAME}_obj PRIVATE $ +diff --git a/inference-engine/src/preprocessing/CMakeLists.txt b/inference-engine/src/preprocessing/CMakeLists.txt +index f9548339d..ef962145a 100644 +--- a/inference-engine/src/preprocessing/CMakeLists.txt ++++ b/inference-engine/src/preprocessing/CMakeLists.txt +@@ -101,7 +101,7 @@ endif() + + # Create object library + +-add_library(${TARGET_NAME}_obj OBJECT ++add_library(${TARGET_NAME}_obj OBJECT EXCLUDE_FROM_ALL + ${LIBRARY_SRC} + ${LIBRARY_HEADERS}) + +@@ -153,7 +153,7 @@ ie_add_api_validator_post_build_step(TARGET ${TARGET_NAME}) + + # Static library used for unit tests which are always built + +-add_library(${TARGET_NAME}_s STATIC ++add_library(${TARGET_NAME}_s STATIC EXCLUDE_FROM_ALL + $) + + set_ie_threading_interface_for(${TARGET_NAME}_s) +diff --git a/inference-engine/src/vpu/common/CMakeLists.txt b/inference-engine/src/vpu/common/CMakeLists.txt +index 249e47c28..4ddf63049 100644 +--- a/inference-engine/src/vpu/common/CMakeLists.txt ++++ b/inference-engine/src/vpu/common/CMakeLists.txt +@@ -5,7 +5,7 @@ + file(GLOB_RECURSE SOURCES *.cpp *.hpp *.h) + + function(add_common_target TARGET_NAME STATIC_IE) +- add_library(${TARGET_NAME} STATIC ${SOURCES}) ++ add_library(${TARGET_NAME} STATIC EXCLUDE_FROM_ALL ${SOURCES}) + + ie_faster_build(${TARGET_NAME} + UNITY +@@ -60,7 +60,7 @@ add_common_target("vpu_common_lib" FALSE) + + # Unit tests support for graph transformer + if(WIN32) +- add_common_target("vpu_common_lib_test_static" TRUE) ++ #add_common_target("vpu_common_lib_test_static" TRUE) + else() + add_library("vpu_common_lib_test_static" ALIAS "vpu_common_lib") + endif() +diff --git a/inference-engine/src/vpu/graph_transformer/CMakeLists.txt b/inference-engine/src/vpu/graph_transformer/CMakeLists.txt +index bc73ab5b1..b4c1547fc 100644 +--- a/inference-engine/src/vpu/graph_transformer/CMakeLists.txt ++++ b/inference-engine/src/vpu/graph_transformer/CMakeLists.txt +@@ -5,7 +5,7 @@ + file(GLOB_RECURSE SOURCES *.cpp *.hpp *.h *.inc) + + function(add_graph_transformer_target TARGET_NAME STATIC_IE) +- add_library(${TARGET_NAME} STATIC ${SOURCES}) ++ add_library(${TARGET_NAME} STATIC EXCLUDE_FROM_ALL ${SOURCES}) + + set_ie_threading_interface_for(${TARGET_NAME}) + +@@ -70,7 +70,7 @@ add_graph_transformer_target("vpu_graph_transformer" FALSE) + + # Unit tests support for graph transformer + if(WIN32) +- add_graph_transformer_target("vpu_graph_transformer_test_static" TRUE) ++ #add_graph_transformer_target("vpu_graph_transformer_test_static" TRUE) + else() + add_library("vpu_graph_transformer_test_static" ALIAS "vpu_graph_transformer") + endif() +diff --git a/inference-engine/thirdparty/pugixml/CMakeLists.txt b/inference-engine/thirdparty/pugixml/CMakeLists.txt +index 8bcb2801a..f7e031c01 100644 +--- a/inference-engine/thirdparty/pugixml/CMakeLists.txt ++++ b/inference-engine/thirdparty/pugixml/CMakeLists.txt +@@ -41,7 +41,7 @@ if(BUILD_SHARED_LIBS) + else() + add_library(pugixml STATIC ${SOURCES}) + if (MSVC) +- add_library(pugixml_mt STATIC ${SOURCES}) ++ #add_library(pugixml_mt STATIC ${SOURCES}) + #if (WIN32) + # set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} /MT") + # set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} /MTd") +diff --git a/ngraph/core/builder/CMakeLists.txt b/ngraph/core/builder/CMakeLists.txt +index ff5c381e7..2797ec9ab 100644 +--- a/ngraph/core/builder/CMakeLists.txt ++++ b/ngraph/core/builder/CMakeLists.txt +@@ -16,7 +16,7 @@ source_group("src" FILES ${LIBRARY_SRC}) + source_group("include" FILES ${PUBLIC_HEADERS}) + + # Create shared library +-add_library(${TARGET_NAME} STATIC ${LIBRARY_SRC} ${PUBLIC_HEADERS}) ++add_library(${TARGET_NAME} STATIC EXCLUDE_FROM_ALL ${LIBRARY_SRC} ${PUBLIC_HEADERS}) + + if(COMMAND ie_faster_build) + ie_faster_build(${TARGET_NAME} +diff --git a/ngraph/core/reference/CMakeLists.txt b/ngraph/core/reference/CMakeLists.txt +index ef4a764ab..f6d3172e2 100644 +--- a/ngraph/core/reference/CMakeLists.txt ++++ b/ngraph/core/reference/CMakeLists.txt +@@ -16,7 +16,7 @@ source_group("src" FILES ${LIBRARY_SRC}) + source_group("include" FILES ${PUBLIC_HEADERS}) + + # Create shared library +-add_library(${TARGET_NAME} STATIC ${LIBRARY_SRC} ${PUBLIC_HEADERS}) ++add_library(${TARGET_NAME} STATIC EXCLUDE_FROM_ALL ${LIBRARY_SRC} ${PUBLIC_HEADERS}) + + if(COMMAND ie_faster_build) + ie_faster_build(${TARGET_NAME} +diff --git a/openvino/itt/CMakeLists.txt b/openvino/itt/CMakeLists.txt +index e9f880b8c..c63f4df63 100644 +--- a/openvino/itt/CMakeLists.txt ++++ b/openvino/itt/CMakeLists.txt +@@ -6,7 +6,7 @@ set(TARGET_NAME itt) + + file(GLOB_RECURSE SOURCES "src/*.cpp" "src/*.hpp") + +-add_library(${TARGET_NAME} STATIC ${SOURCES}) ++add_library(${TARGET_NAME} STATIC EXCLUDE_FROM_ALL ${SOURCES}) + + add_library(openvino::itt ALIAS ${TARGET_NAME}) + diff --git a/platforms/winpack_dldt/2021.4.1/20210630-dldt-pdb.patch b/platforms/winpack_dldt/2021.4.1/20210630-dldt-pdb.patch new file mode 100644 index 0000000000..65e6f84dc8 --- /dev/null +++ b/platforms/winpack_dldt/2021.4.1/20210630-dldt-pdb.patch @@ -0,0 +1,15 @@ +iff --git a/CMakeLists.txt b/CMakeLists.txt +index e0706a72e..9a053b1e4 100644 +--- a/CMakeLists.txt ++++ b/CMakeLists.txt +@@ -6,6 +6,10 @@ cmake_minimum_required(VERSION 3.13) + + project(OpenVINO) + ++set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} /Zi /FS") ++set(CMAKE_SHARED_LINKER_FLAGS_RELEASE "${CMAKE_SHARED_LINKER_FLAGS_RELEASE} /DEBUG /OPT:REF /OPT:ICF") ++set(CMAKE_MODULE_LINKER_FLAGS_RELEASE "${CMAKE_SHARED_LINKER_FLAGS_RELEASE} /DEBUG /OPT:REF /OPT:ICF") ++ + set(OpenVINO_MAIN_SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}) + set(IE_MAIN_SOURCE_DIR ${OpenVINO_MAIN_SOURCE_DIR}/inference-engine) + diff --git a/platforms/winpack_dldt/2021.4.1/20210630-dldt-vs-version.patch b/platforms/winpack_dldt/2021.4.1/20210630-dldt-vs-version.patch new file mode 100644 index 0000000000..36b0068775 --- /dev/null +++ b/platforms/winpack_dldt/2021.4.1/20210630-dldt-vs-version.patch @@ -0,0 +1,16 @@ +diff --git a/cmake/developer_package/vs_version/vs_version.cmake b/cmake/developer_package/vs_version/vs_version.cmake +index 14d4c0e1e..6a44f73b9 100644 +--- a/cmake/developer_package/vs_version/vs_version.cmake ++++ b/cmake/developer_package/vs_version/vs_version.cmake +@@ -8,9 +8,9 @@ set(IE_VS_VER_FILEVERSION_STR "${IE_VERSION_MAJOR}.${IE_VERSION_MINOR}.${IE_VERS + + set(IE_VS_VER_COMPANY_NAME_STR "Intel Corporation") + set(IE_VS_VER_PRODUCTVERSION_STR "${CI_BUILD_NUMBER}") +-set(IE_VS_VER_PRODUCTNAME_STR "OpenVINO toolkit") ++set(IE_VS_VER_PRODUCTNAME_STR "OpenVINO toolkit (for OpenCV Windows package)") + set(IE_VS_VER_COPYRIGHT_STR "Copyright (C) 2018-2021, Intel Corporation") +-set(IE_VS_VER_COMMENTS_STR "https://docs.openvinotoolkit.org/") ++set(IE_VS_VER_COMMENTS_STR "https://github.com/opencv/opencv/wiki/Intel%27s-Deep-Learning-Inference-Engine-backend") + + # + # ie_add_vs_version_file(NAME diff --git a/platforms/winpack_dldt/2021.4.1/build.config.py b/platforms/winpack_dldt/2021.4.1/build.config.py new file mode 100644 index 0000000000..a643c17928 --- /dev/null +++ b/platforms/winpack_dldt/2021.4.1/build.config.py @@ -0,0 +1 @@ +os.environ['CI_BUILD_NUMBER'] = '2021.4.1-opencv_winpack_dldt' diff --git a/platforms/winpack_dldt/2021.4.1/patch.config.py b/platforms/winpack_dldt/2021.4.1/patch.config.py new file mode 100644 index 0000000000..7f8715aae2 --- /dev/null +++ b/platforms/winpack_dldt/2021.4.1/patch.config.py @@ -0,0 +1,4 @@ +applyPatch('20210630-dldt-disable-unused-targets.patch') +applyPatch('20210630-dldt-pdb.patch') +applyPatch('20210630-dldt-disable-multidevice-autoplugin.patch') +applyPatch('20210630-dldt-vs-version.patch') diff --git a/platforms/winpack_dldt/2021.4.1/sysroot.config.py b/platforms/winpack_dldt/2021.4.1/sysroot.config.py new file mode 100644 index 0000000000..fa4281107d --- /dev/null +++ b/platforms/winpack_dldt/2021.4.1/sysroot.config.py @@ -0,0 +1,56 @@ +sysroot_bin_dir = prepare_dir(self.sysrootdir / 'bin') +copytree(self.build_dir / 'install', self.sysrootdir / 'ngraph') +#rm_one(self.sysrootdir / 'ngraph' / 'lib' / 'ngraph.dll') + +build_config = 'Release' if not self.config.build_debug else 'Debug' +build_bin_dir = self.build_dir / 'bin' / 'intel64' / build_config + +def copy_bin(name): + global build_bin_dir, sysroot_bin_dir + copytree(build_bin_dir / name, sysroot_bin_dir / name) + +dll_suffix = 'd' if self.config.build_debug else '' +def copy_dll(name): + global copy_bin, dll_suffix + copy_bin(name + dll_suffix + '.dll') + copy_bin(name + dll_suffix + '.pdb') + +copy_bin('cache.json') +copy_dll('clDNNPlugin') +copy_dll('HeteroPlugin') +copy_dll('inference_engine') +copy_dll('inference_engine_ir_reader') +#copy_dll('inference_engine_ir_v7_reader') +copy_dll('inference_engine_legacy') +copy_dll('inference_engine_transformations') # runtime +copy_dll('inference_engine_lp_transformations') # runtime +#copy_dll('inference_engine_preproc') # runtime +copy_dll('MKLDNNPlugin') # runtime +copy_dll('myriadPlugin') # runtime +#copy_dll('MultiDevicePlugin') # runtime, not used +copy_dll('ngraph') +copy_bin('plugins.xml') +copy_bin('pcie-ma2x8x.elf') +copy_bin('usb-ma2x8x.mvcmd') + +copytree(self.srcdir / 'inference-engine' / 'temp' / 'tbb' / 'bin', sysroot_bin_dir) +copytree(self.srcdir / 'inference-engine' / 'temp' / 'tbb', self.sysrootdir / 'tbb') + +sysroot_ie_dir = prepare_dir(self.sysrootdir / 'deployment_tools' / 'inference_engine') +sysroot_ie_lib_dir = prepare_dir(sysroot_ie_dir / 'lib' / 'intel64') + +copytree(self.srcdir / 'inference-engine' / 'include', sysroot_ie_dir / 'include') +if not self.config.build_debug: + copytree(build_bin_dir / 'ngraph.lib', sysroot_ie_lib_dir / 'ngraph.lib') + copytree(build_bin_dir / 'inference_engine.lib', sysroot_ie_lib_dir / 'inference_engine.lib') + copytree(build_bin_dir / 'inference_engine_ir_reader.lib', sysroot_ie_lib_dir / 'inference_engine_ir_reader.lib') + copytree(build_bin_dir / 'inference_engine_legacy.lib', sysroot_ie_lib_dir / 'inference_engine_legacy.lib') +else: + copytree(build_bin_dir / 'ngraphd.lib', sysroot_ie_lib_dir / 'ngraphd.lib') + copytree(build_bin_dir / 'inference_engined.lib', sysroot_ie_lib_dir / 'inference_engined.lib') + copytree(build_bin_dir / 'inference_engine_ir_readerd.lib', sysroot_ie_lib_dir / 'inference_engine_ir_readerd.lib') + copytree(build_bin_dir / 'inference_engine_legacyd.lib', sysroot_ie_lib_dir / 'inference_engine_legacyd.lib') + +sysroot_license_dir = prepare_dir(self.sysrootdir / 'etc' / 'licenses') +copytree(self.srcdir / 'LICENSE', sysroot_license_dir / 'dldt-LICENSE') +copytree(self.sysrootdir / 'tbb/LICENSE', sysroot_license_dir / 'tbb-LICENSE') diff --git a/platforms/winpack_dldt/build_package.py b/platforms/winpack_dldt/build_package.py index bd4355e1cd..0194323930 100644 --- a/platforms/winpack_dldt/build_package.py +++ b/platforms/winpack_dldt/build_package.py @@ -469,7 +469,8 @@ class Builder: def main(): dldt_src_url = 'https://github.com/openvinotoolkit/openvino' - dldt_src_commit = '2021.4' + dldt_src_commit = '2021.4.1' + dldt_config = None dldt_release = None build_cache_dir_default = os.environ.get('BUILD_CACHE_DIR', '.build_cache') @@ -503,7 +504,7 @@ def main(): parser.add_argument('--dldt_reference_dir', help='DLDT reference git repository (optional)') parser.add_argument('--dldt_src_dir', help='DLDT custom source repository (skip git checkout and patching, use for TESTING only)') - parser.add_argument('--dldt_config', help='Specify DLDT build configuration (defaults to evaluate from DLDT commit/branch)') + parser.add_argument('--dldt_config', default=dldt_config, help='Specify DLDT build configuration (defaults to evaluate from DLDT commit/branch)') parser.add_argument('--override_patch_hashsum', default='', help='(script debug mode)') From 51b03b87e6f9827cbee3feb6dd6dc385c4b77297 Mon Sep 17 00:00:00 2001 From: Zihao Mu Date: Fri, 10 Sep 2021 18:15:22 +0800 Subject: [PATCH 170/262] BiasAdd could load Const from second place. --- modules/dnn/src/tensorflow/tf_importer.cpp | 7 ++++++- modules/dnn/test/test_tf_importer.cpp | 12 ++++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/modules/dnn/src/tensorflow/tf_importer.cpp b/modules/dnn/src/tensorflow/tf_importer.cpp index ca9d7c5e21..521c8ce4c3 100644 --- a/modules/dnn/src/tensorflow/tf_importer.cpp +++ b/modules/dnn/src/tensorflow/tf_importer.cpp @@ -914,7 +914,12 @@ void TFImporter::parseBias(tensorflow::GraphDef& net, const tensorflow::NodeDef& layer_id[name] = id; // one input only - connect(layer_id, dstNet, parsePin(layer.input(0)), id, 0); + Pin inp0 = parsePin(layer.input(0)); + if (layer_id.find(inp0.name) != layer_id.end()) + // First operand is a constant. + connect(layer_id, dstNet, parsePin(layer.input(0)), id, 0); + else + connect(layer_id, dstNet, parsePin(layer.input(1)), id, 0); } else { diff --git a/modules/dnn/test/test_tf_importer.cpp b/modules/dnn/test/test_tf_importer.cpp index 1a2b976eb8..cdf3794bf1 100644 --- a/modules/dnn/test/test_tf_importer.cpp +++ b/modules/dnn/test/test_tf_importer.cpp @@ -559,6 +559,18 @@ TEST_P(Test_TensorFlow_layers, l2_normalize) runTensorFlowNet("l2_normalize"); } +TEST_P(Test_TensorFlow_layers, BiasAdd) +{ +#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_GE(2019010000) + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 && target == DNN_TARGET_MYRIAD + && getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_X + ) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD_X, CV_TEST_TAG_DNN_SKIP_IE_NN_BUILDER, CV_TEST_TAG_DNN_SKIP_IE_VERSION); +#endif + + runTensorFlowNet("bias_add_1"); +} + // TODO: fix it and add to l2_normalize TEST_P(Test_TensorFlow_layers, l2_normalize_3d) { From 9d1e8b1e1ddef6b591d9cce41b625a0c8fb992a5 Mon Sep 17 00:00:00 2001 From: Suleyman TURKMEN Date: Sat, 11 Sep 2021 22:58:45 +0300 Subject: [PATCH 171/262] Update convexhull.cpp --- samples/cpp/convexhull.cpp | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/samples/cpp/convexhull.cpp b/samples/cpp/convexhull.cpp index bb34be2469..d839b8061f 100644 --- a/samples/cpp/convexhull.cpp +++ b/samples/cpp/convexhull.cpp @@ -38,23 +38,14 @@ int main( int argc, char** argv ) points.push_back(pt); } - vector hull; - convexHull(Mat(points), hull, true); + vector hull; + convexHull(points, hull, true); img = Scalar::all(0); for( i = 0; i < count; i++ ) circle(img, points[i], 3, Scalar(0, 0, 255), FILLED, LINE_AA); - int hullcount = (int)hull.size(); - Point pt0 = points[hull[hullcount-1]]; - - for( i = 0; i < hullcount; i++ ) - { - Point pt = points[hull[i]]; - line(img, pt0, pt, Scalar(0, 255, 0), 1,LINE_AA); - pt0 = pt; - } - + polylines(img, hull, true, Scalar(0, 255, 0), 1, LINE_AA); imshow("hull", img); char key = (char)waitKey(); From 6e66a9222a071892ea9f1c9c7951fdcc85e08f7b Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Sat, 11 Sep 2021 22:26:52 +0000 Subject: [PATCH 172/262] dnn(onnx): fix format specifier --- modules/dnn/src/onnx/onnx_importer.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/dnn/src/onnx/onnx_importer.cpp b/modules/dnn/src/onnx/onnx_importer.cpp index 955c79c0fa..2cd36dc94d 100644 --- a/modules/dnn/src/onnx/onnx_importer.cpp +++ b/modules/dnn/src/onnx/onnx_importer.cpp @@ -443,7 +443,7 @@ void ONNXImporter::expandMid(const std::string& prefix, opencv_onnx::NodeProto& for (size_t j = 0; j < n; j++) { LayerParams copyLP; - copyLP.name = format("%s/copy_%d", prefix.c_str(), j); + copyLP.name = format("%s/copy_%d", prefix.c_str(), (int)j); copyLP.type = "Identity"; CV_Assert((layer_id.find(copyLP.name) == layer_id.end()) && "Couldn't copy the node: generated name already exists in the graph."); From 50462dcdc6d0f1cccae49f0fa54ccddd2a0e4a6c Mon Sep 17 00:00:00 2001 From: YashasSamaga Date: Mon, 13 Sep 2021 20:44:33 +0530 Subject: [PATCH 173/262] fix effrank assert to allow input effrank <= output effrank --- modules/dnn/src/cuda4dnn/primitives/padding.hpp | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/modules/dnn/src/cuda4dnn/primitives/padding.hpp b/modules/dnn/src/cuda4dnn/primitives/padding.hpp index cbf75131a1..ce2a5c3c47 100644 --- a/modules/dnn/src/cuda4dnn/primitives/padding.hpp +++ b/modules/dnn/src/cuda4dnn/primitives/padding.hpp @@ -52,14 +52,19 @@ namespace cv { namespace dnn { namespace cuda4dnn { auto output_wrapper = outputs[0].dynamicCast(); auto output = output_wrapper->getSpan(); - auto effective_rank = get_effective_rank(input); - CV_Assert(get_effective_rank(input) == get_effective_rank(output)); - /* suppose we require padding for the first spatial axis (H in NCHW or D in NCDHW) * * there could be a case where the batch axis, channel axis, and the first spatial axis are all one * this would result in effective rank being less than the number of axes requiring padding */ + /* the effective rank of the input may be smaller than the effective rank of the output but the converse is never true + * input: [1, 1, 1, 3]; effective rank = 1 + * output: [1, 1, 3, 3]; effective rank = 2 + * + * hence, we use the effective rank of the output tensor for the padding operation + */ + auto effective_rank = get_effective_rank(output); + CV_Assert(get_effective_rank(input) <= effective_rank); effective_rank = std::max(effective_rank, dstRanges.size()); for (int i = effective_rank - dstRanges.size(); i < effective_rank; i++) From 1618c963e4c89a816dfa6bfebb3e94764b357d61 Mon Sep 17 00:00:00 2001 From: Alexander Panov Date: Mon, 13 Sep 2021 19:27:00 +0300 Subject: [PATCH 174/262] Merge pull request #20676 from AleksandrPanov:delete_createConvexHull_convertTo * deleted dublicated createConvexHull and convertTo * replaced checkVector(2) with points.empty() --- .../imgproc/src/min_enclosing_triangle.cpp | 22 ++----------------- 1 file changed, 2 insertions(+), 20 deletions(-) diff --git a/modules/imgproc/src/min_enclosing_triangle.cpp b/modules/imgproc/src/min_enclosing_triangle.cpp index 4853a755d9..2651871412 100644 --- a/modules/imgproc/src/min_enclosing_triangle.cpp +++ b/modules/imgproc/src/min_enclosing_triangle.cpp @@ -129,7 +129,6 @@ static bool areOnTheSameSideOfLine(const cv::Point2f &p1, const cv::Point2f &p2, static double areaOfTriangle(const cv::Point2f &a, const cv::Point2f &b, const cv::Point2f &c); -static void createConvexHull(cv::InputArray points, std::vector &polygon); static double distanceBtwPoints(const cv::Point2f &a, const cv::Point2f &b); @@ -319,29 +318,12 @@ namespace minEnclosingTriangle { static void findMinEnclosingTriangle(cv::InputArray points, CV_OUT cv::OutputArray triangle, CV_OUT double &area) { std::vector resultingTriangle, polygon; - - createConvexHull(points, polygon); + CV_Assert(!points.empty()); + convexHull(points, polygon, true, true); findMinEnclosingTriangle(polygon, resultingTriangle, area); cv::Mat(resultingTriangle).copyTo(triangle); } -//! Create the convex hull of the given set of points -/*! -* @param points The provided set of points -* @param polygon The polygon representing the convex hull of the points -*/ -static void createConvexHull(cv::InputArray points, std::vector &polygon) { - cv::Mat pointsMat = points.getMat(); - std::vector pointsVector; - - CV_Assert((pointsMat.checkVector(2) > 0) && - ((pointsMat.depth() == CV_32F) || (pointsMat.depth() == CV_32S))); - - pointsMat.convertTo(pointsVector, CV_32F); - - convexHull(pointsVector, polygon, true, true); -} - //! Find the minimum enclosing triangle and its area /*! * The overall complexity of the algorithm is theta(n) where "n" represents the number From 3385d386481962fb4b0619780c87fbf1c78d5c81 Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Tue, 14 Sep 2021 04:15:35 +0000 Subject: [PATCH 175/262] cmake: fix handling of INF_ENGINE_RELEASE - default value should be handled earlier --- cmake/OpenCVDetectInferenceEngine.cmake | 27 +++++++++++++------------ 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/cmake/OpenCVDetectInferenceEngine.cmake b/cmake/OpenCVDetectInferenceEngine.cmake index 41951b710a..35640fa719 100644 --- a/cmake/OpenCVDetectInferenceEngine.cmake +++ b/cmake/OpenCVDetectInferenceEngine.cmake @@ -105,6 +105,20 @@ if(InferenceEngine_FOUND) message(STATUS "Detected InferenceEngine: cmake package (${InferenceEngine_VERSION})") endif() +if(DEFINED InferenceEngine_VERSION) + message(STATUS "InferenceEngine: ${InferenceEngine_VERSION}") + if(NOT INF_ENGINE_RELEASE AND NOT (InferenceEngine_VERSION VERSION_LESS "2021.4")) + math(EXPR INF_ENGINE_RELEASE_INIT "${InferenceEngine_VERSION_MAJOR} * 1000000 + ${InferenceEngine_VERSION_MINOR} * 10000 + ${InferenceEngine_VERSION_PATCH} * 100") + endif() +endif() +if(NOT INF_ENGINE_RELEASE AND NOT INF_ENGINE_RELEASE_INIT) + message(STATUS "WARNING: InferenceEngine version has not been set, 2021.4.1 will be used by default. Set INF_ENGINE_RELEASE variable if you experience build errors.") + set(INF_ENGINE_RELEASE_INIT "2021040100") +elseif(DEFINED INF_ENGINE_RELEASE) + set(INF_ENGINE_RELEASE_INIT "${INF_ENGINE_RELEASE}") +endif() +set(INF_ENGINE_RELEASE "${INF_ENGINE_RELEASE_INIT}" CACHE STRING "Force IE version, should be in form YYYYAABBCC (e.g. 2020.1.0.2 -> 2020010002)") + if(NOT INF_ENGINE_TARGET AND INF_ENGINE_LIB_DIRS AND INF_ENGINE_INCLUDE_DIRS) find_path(ie_custom_inc "inference_engine.hpp" PATHS "${INF_ENGINE_INCLUDE_DIRS}" NO_DEFAULT_PATH) if(CMAKE_BUILD_TYPE STREQUAL "Debug") @@ -140,19 +154,6 @@ endif() # Add more features to the target if(INF_ENGINE_TARGET) - if(DEFINED InferenceEngine_VERSION) - message(STATUS "InferenceEngine: ${InferenceEngine_VERSION}") - if(NOT INF_ENGINE_RELEASE AND NOT (InferenceEngine_VERSION VERSION_LESS "2021.4")) - math(EXPR INF_ENGINE_RELEASE_INIT "${InferenceEngine_VERSION_MAJOR} * 1000000 + ${InferenceEngine_VERSION_MINOR} * 10000 + ${InferenceEngine_VERSION_PATCH} * 100") - endif() - endif() - if(NOT INF_ENGINE_RELEASE AND NOT INF_ENGINE_RELEASE_INIT) - message(WARNING "InferenceEngine version has not been set, 2021.4.1 will be used by default. Set INF_ENGINE_RELEASE variable if you experience build errors.") - set(INF_ENGINE_RELEASE_INIT "2021040100") - elseif(DEFINED INF_ENGINE_RELEASE) - set(INF_ENGINE_RELEASE_INIT "${INF_ENGINE_RELEASE}") - endif() - set(INF_ENGINE_RELEASE "${INF_ENGINE_RELEASE_INIT}" CACHE STRING "Force IE version, should be in form YYYYAABBCC (e.g. 2020.1.0.2 -> 2020010002)") set_target_properties(${INF_ENGINE_TARGET} PROPERTIES INTERFACE_COMPILE_DEFINITIONS "HAVE_INF_ENGINE=1;INF_ENGINE_RELEASE=${INF_ENGINE_RELEASE}" ) From c410d7a97d03f2749d737b9e9232cd738fe09ed2 Mon Sep 17 00:00:00 2001 From: rogday Date: Tue, 14 Sep 2021 20:49:49 +0300 Subject: [PATCH 176/262] Merge pull request #20671 from rogday:yolov4x-mish Add support for YOLOv4x-mish * backport to 3.4 for supporting yolov4x-mish * add YOLOv4x-mish test * address review comments Co-authored-by: Guo Xu --- modules/dnn/src/darknet/darknet_io.cpp | 6 +- modules/dnn/src/layers/region_layer.cpp | 88 +++++++++++++++------- modules/dnn/test/test_darknet_importer.cpp | 72 ++++++++++++++++++ 3 files changed, 135 insertions(+), 31 deletions(-) diff --git a/modules/dnn/src/darknet/darknet_io.cpp b/modules/dnn/src/darknet/darknet_io.cpp index 4915538ff7..99715df829 100644 --- a/modules/dnn/src/darknet/darknet_io.cpp +++ b/modules/dnn/src/darknet/darknet_io.cpp @@ -470,7 +470,7 @@ namespace cv { fused_layer_names.push_back(last_layer); } - void setYolo(int classes, const std::vector& mask, const std::vector& anchors, float thresh, float nms_threshold, float scale_x_y) + void setYolo(int classes, const std::vector& mask, const std::vector& anchors, float thresh, float nms_threshold, float scale_x_y, int new_coords) { cv::dnn::LayerParams region_param; region_param.name = "Region-name"; @@ -484,6 +484,7 @@ namespace cv { region_param.set("thresh", thresh); region_param.set("nms_threshold", nms_threshold); region_param.set("scale_x_y", scale_x_y); + region_param.set("new_coords", new_coords); std::vector usedAnchors(numAnchors * 2); for (int i = 0; i < numAnchors; ++i) @@ -882,6 +883,7 @@ namespace cv { float thresh = getParam(layer_params, "thresh", 0.2); float nms_threshold = getParam(layer_params, "nms_threshold", 0.0); float scale_x_y = getParam(layer_params, "scale_x_y", 1.0); + int new_coords = getParam(layer_params, "new_coords", 0); std::string anchors_values = getParam(layer_params, "anchors", std::string()); CV_Assert(!anchors_values.empty()); @@ -894,7 +896,7 @@ namespace cv { CV_Assert(classes > 0 && num_of_anchors > 0 && (num_of_anchors * 2) == anchors_vec.size()); setParams.setPermute(false); - setParams.setYolo(classes, mask_vec, anchors_vec, thresh, nms_threshold, scale_x_y); + setParams.setYolo(classes, mask_vec, anchors_vec, thresh, nms_threshold, scale_x_y, new_coords); } else { CV_Error(cv::Error::StsParseError, "Unknown layer type: " + layer_type); diff --git a/modules/dnn/src/layers/region_layer.cpp b/modules/dnn/src/layers/region_layer.cpp index 4a8cb724d6..578b0d7dec 100644 --- a/modules/dnn/src/layers/region_layer.cpp +++ b/modules/dnn/src/layers/region_layer.cpp @@ -64,6 +64,7 @@ class RegionLayerImpl CV_FINAL : public RegionLayer public: int coords, classes, anchors, classfix; float thresh, nmsThreshold, scale_x_y; + int new_coords; bool useSoftmax, useLogistic; #ifdef HAVE_OPENCL UMat blob_umat; @@ -83,6 +84,7 @@ public: useLogistic = params.get("logistic", false); nmsThreshold = params.get("nms_threshold", 0.4); scale_x_y = params.get("scale_x_y", 1.0); // Yolov4 + new_coords = params.get("new_coords", 0); // Yolov4x-mish CV_Assert(nmsThreshold >= 0.); CV_Assert(coords == 4); @@ -113,7 +115,7 @@ public: { #ifdef HAVE_DNN_NGRAPH if (backendId == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) - return INF_ENGINE_VER_MAJOR_GE(INF_ENGINE_RELEASE_2020_2) && preferableTarget != DNN_TARGET_MYRIAD; + return INF_ENGINE_VER_MAJOR_GE(INF_ENGINE_RELEASE_2020_2) && preferableTarget != DNN_TARGET_MYRIAD && new_coords == 0; #endif return backendId == DNN_BACKEND_OPENCV; } @@ -259,26 +261,28 @@ public: const float *srcData = inpBlob.ptr(); float *dstData = outBlob.ptr(); - // logistic activation for t0, for each grid cell (X x Y x Anchor-index) - for (int i = 0; i < batch_size*rows*cols*anchors; ++i) { - int index = cell_size*i; - float x = srcData[index + 4]; - dstData[index + 4] = logistic_activate(x); // logistic activation - } - - if (useSoftmax) { // Yolo v2 + if (new_coords == 0) { + // logistic activation for t0, for each grid cell (X x Y x Anchor-index) for (int i = 0; i < batch_size*rows*cols*anchors; ++i) { int index = cell_size*i; - softmax_activate(srcData + index + 5, classes, 1, dstData + index + 5); + float x = srcData[index + 4]; + dstData[index + 4] = logistic_activate(x); // logistic activation } - } - else if (useLogistic) { // Yolo v3 - for (int i = 0; i < batch_size*rows*cols*anchors; ++i){ - int index = cell_size*i; - const float* input = srcData + index + 5; - float* output = dstData + index + 5; - for (int c = 0; c < classes; ++c) - output[c] = logistic_activate(input[c]); + + if (useSoftmax) { // Yolo v2 + for (int i = 0; i < batch_size*rows*cols*anchors; ++i) { + int index = cell_size*i; + softmax_activate(srcData + index + 5, classes, 1, dstData + index + 5); + } + } + else if (useLogistic) { // Yolo v3 + for (int i = 0; i < batch_size*rows*cols*anchors; ++i){ + int index = cell_size*i; + const float* input = srcData + index + 5; + float* output = dstData + index + 5; + for (int c = 0; c < classes; ++c) + output[c] = logistic_activate(input[c]); + } } } for (int b = 0; b < batch_size; ++b) @@ -290,20 +294,46 @@ public: int index = (y*cols + x)*anchors + a; // index for each grid-cell & anchor int p_index = index_sample_offset + index * cell_size + 4; float scale = dstData[p_index]; - if (classfix == -1 && scale < .5) scale = 0; // if(t0 < 0.5) t0 = 0; + if (classfix == -1 && scale < .5) + { + scale = 0; // if(t0 < 0.5) t0 = 0; + } int box_index = index_sample_offset + index * cell_size; - float x_tmp = (logistic_activate(srcData[box_index + 0]) - 0.5f) * scale_x_y + 0.5f; - float y_tmp = (logistic_activate(srcData[box_index + 1]) - 0.5f) * scale_x_y + 0.5f; - dstData[box_index + 0] = (x + x_tmp) / cols; - dstData[box_index + 1] = (y + y_tmp) / rows; - dstData[box_index + 2] = exp(srcData[box_index + 2]) * biasData[2 * a] / wNorm; - dstData[box_index + 3] = exp(srcData[box_index + 3]) * biasData[2 * a + 1] / hNorm; + if (new_coords == 1) { + float x_tmp = (srcData[box_index + 0] - 0.5f) * scale_x_y + 0.5f; + float y_tmp = (srcData[box_index + 1] - 0.5f) * scale_x_y + 0.5f; + dstData[box_index + 0] = (x + x_tmp) / cols; + dstData[box_index + 1] = (y + y_tmp) / rows; + dstData[box_index + 2] = (srcData[box_index + 2]) * (srcData[box_index + 2]) * 4 * biasData[2 * a] / wNorm; + dstData[box_index + 3] = (srcData[box_index + 3]) * (srcData[box_index + 3]) * 4 * biasData[2 * a + 1] / hNorm; - int class_index = index_sample_offset + index * cell_size + 5; - for (int j = 0; j < classes; ++j) { - float prob = scale*dstData[class_index + j]; // prob = IoU(box, object) = t0 * class-probability - dstData[class_index + j] = (prob > thresh) ? prob : 0; // if (IoU < threshold) IoU = 0; + scale = srcData[p_index]; + if (classfix == -1 && scale < thresh) + { + scale = 0; // if(t0 < 0.5) t0 = 0; + } + + int class_index = index_sample_offset + index * cell_size + 5; + for (int j = 0; j < classes; ++j) { + float prob = scale*srcData[class_index + j]; // prob = IoU(box, object) = t0 * class-probability + dstData[class_index + j] = (prob > thresh) ? prob : 0; // if (IoU < threshold) IoU = 0; + } + } + else + { + float x_tmp = (logistic_activate(srcData[box_index + 0]) - 0.5f) * scale_x_y + 0.5f; + float y_tmp = (logistic_activate(srcData[box_index + 1]) - 0.5f) * scale_x_y + 0.5f; + dstData[box_index + 0] = (x + x_tmp) / cols; + dstData[box_index + 1] = (y + y_tmp) / rows; + dstData[box_index + 2] = exp(srcData[box_index + 2]) * biasData[2 * a] / wNorm; + dstData[box_index + 3] = exp(srcData[box_index + 3]) * biasData[2 * a + 1] / hNorm; + + int class_index = index_sample_offset + index * cell_size + 5; + for (int j = 0; j < classes; ++j) { + float prob = scale*dstData[class_index + j]; // prob = IoU(box, object) = t0 * class-probability + dstData[class_index + j] = (prob > thresh) ? prob : 0; // if (IoU < threshold) IoU = 0; + } } } if (nmsThreshold > 0) { diff --git a/modules/dnn/test/test_darknet_importer.cpp b/modules/dnn/test/test_darknet_importer.cpp index 9983d99ef5..ea700573e6 100644 --- a/modules/dnn/test/test_darknet_importer.cpp +++ b/modules/dnn/test/test_darknet_importer.cpp @@ -681,6 +681,78 @@ TEST_P(Test_Darknet_nets, YOLOv4_tiny) #endif } +TEST_P(Test_Darknet_nets, YOLOv4x_mish) +{ + applyTestTag(CV_TEST_TAG_LONG, (target == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_1GB : CV_TEST_TAG_MEMORY_2GB)); + +#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2020040000) // nGraph compilation failure + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target == DNN_TARGET_OPENCL) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL, CV_TEST_TAG_DNN_SKIP_IE_VERSION); + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target == DNN_TARGET_OPENCL_FP16) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16, CV_TEST_TAG_DNN_SKIP_IE_VERSION); +#endif +#if defined(INF_ENGINE_RELEASE) + if (target == DNN_TARGET_MYRIAD) // NC_OUT_OF_MEMORY + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD, CV_TEST_TAG_DNN_SKIP_IE_VERSION); +#endif + + // batchId, classId, confidence, left, top, right, bottom + const int N0 = 3; + const int N1 = 5; + static const float ref_[/* (N0 + N1) * 7 */] = { +0, 16, 0.925536f, 0.17188f, 0.386832f, 0.406138f, 0.941696f, +0, 1, 0.912028f, 0.162125f, 0.208863f, 0.741316f, 0.729332f, +0, 7, 0.841018f, 0.608953f, 0.128653f, 0.900692f, 0.295657f, + +1, 2, 0.925697f, 0.650438f, 0.458118f, 0.813927f, 0.661775f, +1, 0, 0.882156f, 0.203644f, 0.365763f, 0.265473f, 0.632195f, +1, 2, 0.848857f, 0.451044f, 0.462997f, 0.496629f, 0.522719f, +1, 9, 0.736015f, 0.374503f, 0.316029f, 0.399358f, 0.392883f, +1, 9, 0.727129f, 0.662469f, 0.373687f, 0.687877f, 0.441335f, + }; + Mat ref(N0 + N1, 7, CV_32FC1, (void*)ref_); + + double scoreDiff = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.006 : 8e-5; + double iouDiff = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.042 : 3e-4; + + std::string config_file = "yolov4x-mish.cfg"; + std::string weights_file = "yolov4x-mish.weights"; + +#if defined(INF_ENGINE_RELEASE) + if ((backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 || + backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) && target == DNN_TARGET_MYRIAD && + getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_X) + { + scoreDiff = 0.04; + iouDiff = 0.2; + } +#endif + + { + SCOPED_TRACE("batch size 1"); + testDarknetModel(config_file, weights_file, ref.rowRange(0, N0), scoreDiff, iouDiff); + } + + { + SCOPED_TRACE("batch size 2"); + +#if defined(INF_ENGINE_RELEASE) + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019) + { + if (target == DNN_TARGET_OPENCL) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL, CV_TEST_TAG_DNN_SKIP_IE_VERSION); + else if (target == DNN_TARGET_OPENCL_FP16 && INF_ENGINE_VER_MAJOR_LE(202010000)) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16, CV_TEST_TAG_DNN_SKIP_IE_VERSION); + else if (target == DNN_TARGET_MYRIAD && + getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_X) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD_X); + } +#endif + + testDarknetModel(config_file, weights_file, ref, scoreDiff, iouDiff); + } +} + INSTANTIATE_TEST_CASE_P(/**/, Test_Darknet_nets, dnnBackendsAndTargets()); From 41a2eb5245583403f4a26ca24ec7ba736d2cca5e Mon Sep 17 00:00:00 2001 From: Rohit Sutradhar <40035473+r0hit2005@users.noreply.github.com> Date: Wed, 15 Sep 2021 22:20:49 +0530 Subject: [PATCH 177/262] Merge pull request #20361 from r0hit2005:master Tutorial for parallel_for_ and Universal Intrinsic (GSoC '21) * New parallel_for tutorial * Universal Intrinsics Draft Tutorial * Added draft of universal intrinsic tutorial * * Added final markdown for parallel_for_new * Added first half of universal intrinsic tutorial * Fixed warnings in documentation and sample code for parallel_for_new tutorial * Restored original parallel_for_ tutorial and table_of_content_core * Minor changes * Added demonstration of 1-D vectorized convolution * * Added 2-D convolution implementation and tutorial * Minor changes in vectorized implementation of 1-D and 2-D convolution * Minor changes to univ_intrin tutorial. Added new tutorials to the table of contents * Minor changes * Removed variable sized array initializations * Fixed conversion warnings * Added doxygen references, minor fixes * Added jpg image for parallel_for_ doc --- .../file_input_output_with_xml_yml.markdown | 2 +- ...ow_to_use_OpenCV_parallel_for_new.markdown | 166 +++++++++ .../images/convolution-example-matrix.gif | Bin 0 -> 150600 bytes .../images/resimg.jpg | Bin 0 -> 65566 bytes .../core/table_of_content_core.markdown | 3 +- .../core/univ_intrin/univ_intrin.markdown | 334 ++++++++++++++++++ .../how_to_use_OpenCV_parallel_for_new.cpp | 332 +++++++++++++++++ .../core/univ_intrin/univ_intrin.cpp | 230 ++++++++++++ 8 files changed, 1065 insertions(+), 2 deletions(-) create mode 100644 doc/tutorials/core/how_to_use_OpenCV_parallel_for_new/how_to_use_OpenCV_parallel_for_new.markdown create mode 100644 doc/tutorials/core/how_to_use_OpenCV_parallel_for_new/images/convolution-example-matrix.gif create mode 100644 doc/tutorials/core/how_to_use_OpenCV_parallel_for_new/images/resimg.jpg create mode 100644 doc/tutorials/core/univ_intrin/univ_intrin.markdown create mode 100644 samples/cpp/tutorial_code/core/how_to_use_OpenCV_parallel_for_/how_to_use_OpenCV_parallel_for_new.cpp create mode 100644 samples/cpp/tutorial_code/core/univ_intrin/univ_intrin.cpp diff --git a/doc/tutorials/core/file_input_output_with_xml_yml/file_input_output_with_xml_yml.markdown b/doc/tutorials/core/file_input_output_with_xml_yml/file_input_output_with_xml_yml.markdown index da060cf27d..46838dd99b 100644 --- a/doc/tutorials/core/file_input_output_with_xml_yml/file_input_output_with_xml_yml.markdown +++ b/doc/tutorials/core/file_input_output_with_xml_yml/file_input_output_with_xml_yml.markdown @@ -4,7 +4,7 @@ File Input and Output using XML and YAML files {#tutorial_file_input_output_with @tableofcontents @prev_tutorial{tutorial_discrete_fourier_transform} -@next_tutorial{tutorial_how_to_use_OpenCV_parallel_for_} +@next_tutorial{tutorial_how_to_use_OpenCV_parallel_for_new} | | | | -: | :- | diff --git a/doc/tutorials/core/how_to_use_OpenCV_parallel_for_new/how_to_use_OpenCV_parallel_for_new.markdown b/doc/tutorials/core/how_to_use_OpenCV_parallel_for_new/how_to_use_OpenCV_parallel_for_new.markdown new file mode 100644 index 0000000000..57334e2b9b --- /dev/null +++ b/doc/tutorials/core/how_to_use_OpenCV_parallel_for_new/how_to_use_OpenCV_parallel_for_new.markdown @@ -0,0 +1,166 @@ +How to use the OpenCV parallel_for_ to parallelize your code {#tutorial_how_to_use_OpenCV_parallel_for_new} +================================================================== + +@tableofcontents + +@prev_tutorial{tutorial_file_input_output_with_xml_yml} +@next_tutorial{tutorial_univ_intrin} + +| | | +| -: | :- | +| Compatibility | OpenCV >= 3.0 | + +Goal +---- + +The goal of this tutorial is to demonstrate the use of the OpenCV `parallel_for_` framework to easily parallelize your code. To illustrate the concept, we will write a program to perform convolution operation over an image. +The full tutorial code is [here](https://github.com/opencv/opencv/tree/master/samples/cpp/tutorial_code/how_to_use_OpenCV_parallel_for_/how_to_use_OpenCV_parallel_for_new.cpp). + +Precondition +---- + +### Parallel Frameworks +The first precondition is to have OpenCV built with a parallel framework. +In OpenCV 4.5, the following parallel frameworks are available in that order: + +* Intel Threading Building Blocks (3rdparty library, should be explicitly enabled) +* OpenMP (integrated to compiler, should be explicitly enabled) +* APPLE GCD (system wide, used automatically (APPLE only)) +* Windows RT concurrency (system wide, used automatically (Windows RT only)) +* Windows concurrency (part of runtime, used automatically (Windows only - MSVC++ >= 10)) +* Pthreads + +As you can see, several parallel frameworks can be used in the OpenCV library. Some parallel libraries are third party libraries and have to be explicitly enabled in CMake before building, while others are automatically available with the platform (e.g. APPLE GCD). + +### Race Conditions +Race conditions occur when more than one thread try to write *or* read and write to a particular memory location simultaneously. +Based on that, we can broadly classify algorithms into two categories:- +1. Algorithms in which only a single thread writes data to a particular memory location. + * In *convolution*, for example, even though multiple threads may read from a pixel at a particular time, only a single thread *writes* to a particular pixel. + +2. Algorithms in which multiple threads may write to a single memory location. + * Finding contours, features, etc. Such algorithms may require each thread to add data to a global variable simultaneously. For example, when detecting features, each thread will add features of their respective parts of the image to a common vector, thus creating a race condition. + +Convolution +----------- + +We will use the example of performing a convolution to demonstrate the use of `parallel_for_` to parallelize the computation. This is an example of an algorithm which does not lead to a race condition. + +Theory +------ +Convolution is a simple mathematical operation widely used in image processing. Here, we slide a smaller matrix, called the *kernel*, over an image and a sum of the product of pixel values and corresponding values in the kernel gives us the value of the particular pixel in the output (called the anchor point of the kernel). Based on the values in the kernel, we get different results. +In the example below, we use a 3x3 kernel (anchored at its center) and convolve over a 5x5 matrix to produce a 3x3 matrix. The size of the output can be altered by padding the input with suitable values. +![Convolution Animation](images/convolution-example-matrix.gif) + +For more information about different kernels and what they do, look [here](https://en.wikipedia.org/wiki/Kernel_(image_processing)) + +For the purpose of this tutorial, we will implement the simplest form of the function which takes a grayscale image (1 channel) and an odd length square kernel and produces an output image. +The operation will not be performed in-place. +@note We can store a few of the relevant pixels temporarily to make sure we use the original values during the convolution and then do it in-place. However, the purpose of this tutorial is to introduce parallel_for_ function and an inplace implementation may be too complicated. + +Pseudocode +----------- + + InputImage src, OutputImage dst, kernel(size n) + makeborder(src, n/2) + for each pixel (i, j) strictly inside borders, do: + { + value := 0 + for k := -n/2 to n/2, do: + for l := -n/2 to n/2, do: + value += kernel[n/2 + k][n/2 + l]*src[i + k][j + l] + + dst[i][j] := value + } + +For an *n-sized kernel*, we will add a border of size *n/2* to handle edge cases. +We then run two loops to move along the kernel and add the products to sum + +Implementation +-------------- + +### Sequential implementation + +@snippet how_to_use_OpenCV_parallel_for_new.cpp convolution-sequential + +We first make an output matrix(dst) with the same size as src and add borders to the src image(to handle edge cases). +@snippet how_to_use_OpenCV_parallel_for_new.cpp convolution-make-borders + +We then sequentially iterate over the pixels in the src image and compute the value over the kernel and the neighbouring pixel values. +We then fill value to the corresponding pixel in the dst image. +@snippet how_to_use_OpenCV_parallel_for_new.cpp convolution-kernel-loop + +### Parallel implementation + +When looking at the sequential implementation, we can notice that each pixel depends on multiple neighbouring pixels but only one pixel is edited at a time. Thus, to optimize the computation, we can split the image into stripes and parallely perform convolution on each, by exploiting the multi-core architecture of modern processor. The OpenCV @ref cv::parallel_for_ framework automatically decides how to split the computation efficiently and does most of the work for us. + +@note Although values of a pixel in a particular stripe may depend on pixel values outside the stripe, these are only read only operations and hence will not cause undefined behaviour. + + +We first declare a custom class that inherits from @ref cv::ParallelLoopBody and override the `virtual void operator ()(const cv::Range& range) const`. +@snippet how_to_use_OpenCV_parallel_for_new.cpp convolution-parallel + +The range in the `operator ()` represents the subset of values that will be treated by an individual thread. Based on the requirement, there may be different ways of splitting the range which in turn changes the computation. + +For example, we can either +1. Split the entire traversal of the image and obtain the [row, col] coordinate in the following way (as shown in the above code): + + @snippet how_to_use_OpenCV_parallel_for_new.cpp overload-full + + We would then call the parallel_for_ function in the following way: + @snippet how_to_use_OpenCV_parallel_for_new.cpp convolution-parallel-function +
+ +2. Split the rows and compute for each row: + + @snippet how_to_use_OpenCV_parallel_for_new.cpp overload-row-split + + In this case, we call the parallel_for_ function with a different range: + @snippet how_to_use_OpenCV_parallel_for_new.cpp convolution-parallel-function-row + +@note In our case, both implementations perform similarly. Some cases may allow better memory access patterns or other performance benefits. + +To set the number of threads, you can use: @ref cv::setNumThreads. You can also specify the number of splitting using the nstripes parameter in @ref cv::parallel_for_. For instance, if your processor has 4 threads, setting `cv::setNumThreads(2)` or setting `nstripes=2` should be the same as by default it will use all the processor threads available but will split the workload only on two threads. + +@note C++ 11 standard allows to simplify the parallel implementation by get rid of the `parallelConvolution` class and replacing it with lambda expression: + +@snippet how_to_use_OpenCV_parallel_for_new.cpp convolution-parallel-cxx11 + +Results +----------- + +The resulting time taken for execution of the two implementations on a +* *512x512 input* with a *5x5 kernel*: + + This program shows how to use the OpenCV parallel_for_ function and + compares the performance of the sequential and parallel implementations for a + convolution operation + Usage: + ./a.out [image_path -- default lena.jpg] + + Sequential Implementation: 0.0953564s + Parallel Implementation: 0.0246762s + Parallel Implementation(Row Split): 0.0248722s + +
+ +* *512x512 input with a 3x3 kernel* + + This program shows how to use the OpenCV parallel_for_ function and + compares the performance of the sequential and parallel implementations for a + convolution operation + Usage: + ./a.out [image_path -- default lena.jpg] + + Sequential Implementation: 0.0301325s + Parallel Implementation: 0.0117053s + Parallel Implementation(Row Split): 0.0117894s + +The performance of the parallel implementation depends on the type of CPU you have. For instance, on 4 cores - 8 threads CPU, runtime may be 6x to 7x faster than a sequential implementation. There are many factors to explain why we do not achieve a speed-up of 8x: + * the overhead to create and manage the threads, + * background processes running in parallel, + * the difference between 4 hardware cores with 2 logical threads for each core and 8 hardware cores. + +In the tutorial, we used a horizontal gradient filter(as shown in the animation above), which produces an image highlighting the vertical edges. + +![result image](images/resimg.jpg) \ No newline at end of file diff --git a/doc/tutorials/core/how_to_use_OpenCV_parallel_for_new/images/convolution-example-matrix.gif b/doc/tutorials/core/how_to_use_OpenCV_parallel_for_new/images/convolution-example-matrix.gif new file mode 100644 index 0000000000000000000000000000000000000000..070208eb3da505fe6d69aabefdded607633b5153 GIT binary patch literal 150600 zcmX8aWl&pdqXpmucXuo9aB!!%YjJmXFHk7Ko#O6L+}(@2yBBvT?vR^1-}^hm%x10i zWV1RqM9r$B^@9Osc{vHel=LUpyPo3YNPt0w13NesJMpe&gbBW$f%^1ptkF zc=^gN$-~Gaz$z-yMFjl*3u5Xom#)Svi0&Y6%n1-+lq%InqHO=ol#Q0IN6e2IoLnF|9X17dO9_~y?=3W@%Z#Kwfg}D4ZU`D zv2k>EaQ~W9-vC5J_Kz-H*t>^9rdm5VJbHPR)v%IO)H!}Q1m6wIY3M*9lZ>oCj4xk| zt-LIq9xWZ60bsBJP(&lEJM)`&(>wRRVG(65Bb(=+LrZrf%WIFfs|{Vl%V*bwG`s`T zXM{9#Q@b6%{o$Hg-YI#KGX;;Kanl;KbVE;^OA!=EBDM`pDqh z)#S&&`NOlr*T-XgY*;81EJQRy1RQ*70z@LqLuTa)g8+O%{-r{a?7kik6UMArV8`C8z#PPE$u-juR5MA}_b9o}IR?j;_Ub zNY;?(4PD$I4L3LV2I+vcv!ks?xZQVeXB!QdA2I){gilxm1O(r(bU!~oNOgZECI%ZD zLrM}786BCL7MWTWm0A@Y9UT)F`^(EcF*_Slp5$=<UVZPS@TF)OI!J`=6yOinLF>FnCze0@1H({bP`fX2q2JAL$)wHeKEYe3Nd7O{Q=?$#DuY#!||m{ z$Yvp`K-vqDVPfSKVj9E*h>>L*$v-d4P;`YHt$4&dl-YY;)7x_8Wu<$b{$jAw3_5 zRDEIdc46-UA`*leNdFht9w5psO}8zrokE~nIy!}j2O)0d>>45-1R{t4tEY#MbwdnX zJ9>f`2oV6Xm-WMUh-?rFHg2vUBZ9!VdG-l`ar5>G;uwTSh3;_f}2L$1>)0MM_R|wAMclQt*|L69_!59Q0 z2+I(tA#_6w{2$#A%pqt)T!paq@bLNoLG64LBFWS9BLsN}?Emu|A|7PgZ>PuqTl&Z2 z6og!eqL7zEcm&^#{LgTR(h#2!zyL5JiELedUoZkTqrqrh!9X|$m3+2rec@0v0jJ&O zXnoPhFLH?}61j%ru_Su+3WKqR-xFzU=HuCNjU`iAeC}tPV~wRVd7{Bcr1DK=b49XA zjE3V)|A3&8$rRh_b_XJ`nT#ge>i0+DsT6V*+Zzri(mCz7C)*p3XYwVY$&@;p zP8Z75D~+Z)n$K72%qMb{I$JI`+T72#r#f5z{p}4#{-WH~cC$a4#AH0()qZz8TcnVu z+}#2B?KRr}o$l^@`nNL>4gR9i)Ae$9vQTL})6@O-biF^3r}C%gYw~yvjxrmX0R9Y9KW!N~gDw z7|viyUkDm5PMM0}TDIAb<@$s8Un)7DXnQemTgPvz8{qz=X!F~ zA>;5fe;xih8HNQ>`jaOR}BpfF%VUjErSPc9aaP zE-*V*9`Ro}$15WS!HnlsP0M!Y)eTUfF^ZOJB&M2PxbxxK&NIe~`qA5jaORn^afk9I zNv6xDMJ@a4CKy?Xv*t~&4M$*Qj~p}AekyTQ`##YR*7kGS=*W)C;b^BX0WIT`EVFx+ zwXJWDSqxAK*Gw)wKx}inmWz-Ll48*0gcId2QRbUL91#Q8np-1>`eBlR3D-eFX9M7TblK#v|Alpn7RO`cv+bO^c-Eo~i-i#kZongJvpK+s zOV!Daw1E5R`VSJ~eXbgJP`RJ_sn1kHd6K2_(J1;pVV%PY_d+=p1LNR)ogJNLwb;B@ zh2KfP@pe)LBg^&g3DPtM(TdaF0OUnxb;CsbSvFGA-1*ILf;8 zrrKP?sZY(gTQUVBFCGA-qg-Pwg{WS#`9h)Dm6u{mAg)S<}Z z%B!-Qsjhbt7-+gfgSJf~r zC==+)$qjl~6Cg`jf~@VR)vQ@-lm@H`oL>Y~s@sk%toZGZc_Q(ILX0TM8iurj@nUv= zbFjE2yC^_>$NY=CYz=<0TciNMn)zQzD``A=ei%U_plzq}(^Ec?@OhUFCsNrkH5R^^ zBuo}t6+|)W6!LbVBmx~7_ZTo8)C{f*S>960A^wKSORW!=CtVC%2iQnl63Ci|7c3*{MsTZ z<2{;ExHsUbuYu$ha;?smd%ID(n=2Q;U<|vROsMeT_3=zmp*(dQow?QDojhv%A_~dt z^!Xe6Op(`CCF#d!H8AzGVvzYtf3NC^Jm~RxTnhAs0Gj#*mK{N8~n zEheR+!0Sdvp}fYU68+2ZZh zdz7I>+sF=dBB%_;4}KD8tM8m8n7n9S(o!hyc+v?az{%3Ua&~7W-DO)-9ayj?pg}v> zaHO=`o;RSoxnlIgQHlmLjDBl3yJM&BuY*G86K&{+*+tt$38$#*g=QvzVuc;GzdH4t zuRASG_@!v}@l`$LDu|-uZ`v{%9(zqZ4*~V)g^7uJX3MzOMLlNQMVeEmxpAml(|5vi zzL~4ZXt+HW9|xy>U45Q@GB1zLsXpHT{V?J#UamYxtDlubuK(hcx|*wv|Y|_-uy<9pzF0VkH-*@5f2m7#%Tyh<=eNkkXhE~pp-gMvhRKQn!6(Timk@a1N z=>NUU%?i6YFDm-nMA1jDMUj_IUM2Z(L?`_%6w$!E$o<-u(L2bwHu=f~@#?n_la8)U5$E<-C2D z7Z9WZ<*d_5xE)`bN3yi*~k>Xgz>l>0#J&3pAHY1^OftM4;!+iyKY4Yl3Z0V?l% z?&iMaCcJ;oyJ7*H3ARA`7a;J;Q1u%9r?CV7q;Pl3TyiaP(If5S$GBlIv=XTf_l6q9Gl!?f z#|-u>2qg0kCPVm%91$#(9!RnpENTO~=k+b6bynpSL)nM_F6tt^8X|lgB17+7$?K#s zD#3E%^~VcekTTe0C6q!ZxU&`Aj4w=VG`LSD%tj~7sw~WwAVgvm+jiMnl_T6uHr(?> z-pf1O*9Pz-J>35!Jb+KjFCsh_CL-)PJfcii+l@A6HG&#^7!gkpY0^)tUf@a7M`=tD z!~hj0vKr>l7W|qYoOcpgZ1V$$E$VlA)VNFt4MJ#vh4&g>v@}9=gHANGv@D}eXa|C8 zXi{|7b95&G$X6`bAU%>}$p6oYnDqn8vD!qgS=)H&7EI}E>U zlcdi4R0p4=xMa80w9s!izjADnrEHV;@X^0Vra`O5G87~a(t0*W1T}NVed4F66Mk0> z1pakFJgiHJmqSbB&q&tINM%U9Hc4$rOKr?Ya(u~f%wS2gauBCWi$;w1UIYEeNP}ZQ zuhdPeI|a#>V}l4imD-Ji5S>WJ^dk{9)m@_u5z!KSG6r?CFWEA()-v+TvnSWGr^m8k zcIdx&;nuEYF4^YPN9I%!B(yPj?+xd0bB2C`y+Q#tDdH4AtbDQuY_l(XGRE4o%X^V# z81nA<^P1ZoOHO~4y~LdLqQm**fHQKyr)l)Ac{(BfH^KS4a&c6p-l%6uOl!GVb{YS4 z^YDoB2r>)q+6&;21Q*-$-U-pZ(C2?W$tx#JrWS~8%)lmn&1o;s+R3o%Q$hNzAdimKR>;Txd*Y$+?RyEw zdRE3ksbgvS>*;S}#scsvvUz5iMMbtZc-{RAQ5b>_k`@LOhAL2G7DZk@mVvR{mN|H>vZA7JCub|(eFn2molpW;_wuLzz z!LCtJUV2{I9q}P+IYr~FXzTf$cJ7t(Wz~#mMnokIzST7tNd*4ZRmi?o6x!(;gk=&l|gIO3KYqy+EC3hFp{cBJ+GC$L{+}J^=C+>SMpfbj72CJ*1yNA_2qT3(kkdZ zi>fM6PhZeRt{Vi28ZT_C;q~ic!GJsqyOP(D>Q=@qRG0E>(Ry;C_}q&6%cy!pohnec zqne8`REY~r8r;W&4C{~?{Hvb4i5-TngNB{SHy2p%7JX-#P)k6I8FDovQ}s1DoRtg8 zKNoaorbah~f?A^Dg;_X9lSba?Iz;-AJ~9@|3z5OKOt5k1&~dR3!#T!0tRlA0Q$rFGP@!Cc850eVqd5y~A-TN;|SQpMIj z+r|Wn0075|jOxCT%T(&ME4+~h7XfFi28}cepr}KM z*9(g$ZkJ^WNo2yJ#nGb#dujR*@$YO{=bX2bPY9$;NH1|`A}=|djzaY&>m(k2+jg_W zYU!vD=*^W z`X*@K+rL8H<7eI{E7tZ)8`^Okab6!9*1PFgpp-%}cEeV$0GVc5(xDXAnC~%F|3|aN zL#wHUii@gL{onv9I1G(%GN@&V<^uAxZ}T_*84Q7F_b<9$fd0^+T%*&wB`-1DLNttT zbe-viK#-L}%PRX+WzCdoxr1UoGD>~SL~)!#(_m(Kd3h7}oREJVoDLk#*RFvt1Dan% z<=I5P1u~Bk4Zt4bvQ4}@_LR|9bi$IW0_uLWG(+1;K6K1KPQ3}HF{}+Hby#I6Cibuc(WOj7>`-~(=Xc)DOw3L69 z38MD3aOSLH7kU_h+$8m^Di~vpt-qTr@Su=CXiNT{B92VckQGr{s|sIRo;(-?T|K7p*_X}IHIMdE49ce=<8CWwE;%R7$Dx5}$q zLAi>X4?5^ccq!`vV80InSVMc>O4Khp0W(s4TI&pQ%UaFFa3BL|%R6*^0FA*cDwZf< z&96lc1xG}GW^HAbH)F1WzlX(l4$iS(L$)OB1|(hCYzm0h%IK(DsIM%^i8#_W1;!=u*^rF zoc}}{xT+f_c8#oFFtXX=SwY)y%XN;EE?0Q@EY=ZKcewglXF1WTGfW>Q-X>sN!r5P0 z;fRJl|L?9}CX4<&Af~f_YpfrzN|#>^xq&x!SQVHuou=I)v%^h|4JNGFRWe-L+rm8P zZm+>vXZ7W^m~B$V(-WLc*2lIJ$ZsyWR>`q%4wp%>rCliWQ{nbvu@l{gJIYi1D*)JVyd;Nt(?G8PAD3c54^EZI`#~kaTo{t6osa9!9_ghA-Pb z!oUo-5n6|q4kvXR(_iP7wC~E$+hytR*D4LW?tKfcVTsvev3UIss}(WU*4~^QB%{V+ zuuHGOCW6j8tUdpNwBDhq#FkXwA&tPu-`H-QpKEYR{JH+^$c_?O$k0MG(4$dIKG`~r z&Eu9+5)x8sLaN$8(Tzg{K8h`g0Etuiu7*R^lQ890o=Dh*PN;yd094kQ6@j3{IPQ+0 zDI&2={_Uqg+f&`oU6MJK+^O}#E$%=sYQ~ZfqmQLlj$NLivzp8Q2vB!OtB>Qkq3h4> zgSY>H&&~(7DsoX!I$jmx;!f_~4}M3lPbhOw{mhlfI#p!3)Q`L7x49&ya4^iDoZDHs=E~nsWPU9(bh*_Nn_TMCJll(7k=ztgj?hx)(TKWG`2Lv)`0rls$BX3H zH4*#Ep3yUF>-}9vt$F8T^vv_EQJLq=zj*XhjP6Dh%@@DZhp)9Sp5rfsyD!1;ug09u zB?3heV@~wZ$SxwjCnR+me_oAa9&*bcHpd>k4X=^@dsI;Rkc)mJD0nMRa@GWAs&~yi zb|2kSJHLKJy_+E2{i-b(yZTor!aMursmb|eJN8A#=|cj%h-+Bc`S)W61_OX0Oq5SlDyCV^SNh5&bQn@swiN|d=ISP4X8A~SRM$N{tBA3h%oMADR*ejCGZaH17 zyKtnG&F?wyaLuyL=O(rR#6^m;`Bp5EM0;62&cIamO?@g;Z*obqQlnXZYyCk}P6it> zid5rqMmO9-7FCs-{A}eJS4|iG;5u7LCrKNwVI771{*wJ1EjA3M2Fpr z-~atREaN^rTY7dqsO|ZA3q5}CegfsBQp&1rBs3J$43>%!etY!a=nR0b?FI^ZEG^wl zTD)+1EEMg|{8G;R^q;J;{AXRA2M$;j^0gI)a-OJ8-HDS|WpE`3Da}zU49w4z2Kg}Z zqi}k#pdAzR|Ksog6kOIrDa1Gk-9F2)3*u;N%t?6~4c15znku?x*s+SvNIt0j_%||h z@P;89bwP|J`$bF*t9LEk1FHz=lZ;jBWYWKr36H;s#a9v%`MV8eptu4DSy!$%@hGB8 zrGvE8Gk7&H*lNa zUis}VV(2(%x#b|wr?J0ulml-X+svr@50(iXz58>lUv+O^cA}ubjGJb8Ii~ z+CIFWZM!%rG^^EC;H5TUdEUpihB%RBzA+C4o^}hc^01U;sw?7y8MeY!wWd~max49h zyCt7YAbI`ohMxmiFM2$3AsI5}6uAVuzgODx34gyuX6}_+DuOdlOHx=vb)tVwGnlJ{ zg{7|74Y9smIB!U7IM~Y>yyF=-FXMMwHom*5zxjK2>E_iyskheaz8u>3yp+N*c6~oSK-$^Q*bH6%d}z)y z90=rzHAeTWQXd&vAyDG|n`gy%?d1PJfne-s|H)4Y0J5$G1bn_dfdc?=s-O=hoKF;Q zQ5ZbcKtux*AkniZJO^74RxvpYx=tu#M(|I#Lb%1uW3Uj$a3%FZ5t*BBOZPVymC&u| ztzI7QF84BYp#vu4B@*cYA~GvaVgnF`R=PO(*%jdZc7TMopp%@40S&H~6l>&&(+uED zFknsyRmU#Hpyp~g;7x&IJ=}#J#2#WQIgaq3WIvXgrLmlc8Ew>x)YuMN_|JrWk{dM{ z!OseN!~1V7@98C9pl*K!!4N(QNJfZZsU<`j9O&4r$jZVcCd`z{@-%qMDOzFSx)f#M z@s`W!*z83)hwSTH&<-E7&|BLm?jy2O$mzUJg#H$qtG$h26WZoT^V-};u6`cpS+9ub zqec!-9~C#ln*-)u7;-Lg*Pw{9C+9k3;Bw0VP~7NTSb_Ur{Om5-3Tz-C(s+vTqgNuc5zw$;<-` zHHoC`*l9JNY33!|mU8pQ^E&u0SolR2RBg^`%3#jbZx@$||E}~2^3{kr;wEz+(i0Mi znkysP zyY|KpdfHuQ?B&DfRHia^s{{S16wr>$$#%$ER`PgF7xd=@?M;kL70rs{Z1A2ykcVd#LlCr+%U%6Cd07Ri}!b;nv03;j}3VEd#8_P zIcN0;{YJCZto}m>s#OZ%J>N)Q1S7Ve4{sVswc0slG}kXs%sXb?>18XIB&!i?Ch#L_ zopw=dJ~>Tl_a(eTNr5Y=M>oi*bh14QV1k+m2}R@F+>**SQn%25>wCinFHPPb&NAqm zgS|m;BS$b7ahGx}q?R&){PVoof3sYIfebMgy#Ztyuvsp7Sqpr9saC1)TDJG-ABqpeBPZbTn-dtO|zCU zRb8>Ypx#}y?^@%wqkvDRzQvO^IZU@5Zerh$b7O0lpKQtO-?*`3=VdKOpj^#91<@bW#`33(>S$a8k zIrhOeP*^cpx!RF_*jPZ9Y_lajf%6o)Ol+dLcHwy$G8cquxZz(GTEv1;&FbJyMb4(W z>cw};cD9dqo0gFB#|e?NVU_DPcBI7v5700G1~OTH7Hxq&ciqmGf7y@AK3YBCf3nLZ z0pY311JULIiY;Q?NE{e}$RQYV)M5$<^z^=%5$PT`xx6UM1bs-p0QgZjjRk&_K8)#e z^Q1{X0iARJkS&hM>B|5j0bd~D1QyXyo3A4K%4|ni1Z-LQGuq; ztfZG*@dBVZUnd4NB3V5E4I@~XKG|s&)>)zlE3(t4AWXrfr*XX#SQegJ58BslXS(0LO$1q2Ph9;DP8MnM3w!RlNx354PhtLMaIrzgL&?-Caaj*%eRLg4*2 zA%@Z`D5&1tdk%ZH0c$bZ`*oqbk0}Ih5J(sw$N+#TffAu=N|D${e3odTJK!ImK%O+O zUf3v3)#ob=OfA}n=GFnh`~60W;W6zBtYh{~YLKUfibK-Ge;fsc>Aj_3y2v;#()#p*>7GFVj6mI{;FW>o zD~?dgIw>%Mg@m^Q;x%!Qod+nUxMXu8j3Ytvf$PM?A$32T z#ZGnbN{d=2A%`;p+zE7tDeZF)h7HLkH9=7IleyVXAaLnQi|O+BAOexpm3a^a8sPbC zfF{Kx*7^i&BB90-f@(qpra%>r_*zxmf-u;Fu%cwdl~U28nifQZ|OQ>4t8jeMGikbh^%&FRQQ zW7ka8cz4IRK?1K*6KML5qBgsjlU(qfUv@=r6b%Q!qN*g2&Zr=VFZ=^Qpc-gCsNlgK z38p3o30{bjWBfL&qU|13WM&I;B1y@x$j$WouX{p{&b%|P7z$0b)g^}eABjAeSYjLn zbs{Dv(_OwbSpIqLcXASg#c0zfDBV&Sb{bSgCnj~q8m=qnL*{7DGDpwHJcl$yQgx~^1&5fU`y((U3i@*2cLz9; zW36{Y*0Wpn#27g0hUiG8Z!IDPV30isW}(`GU>=XXOPXooZ^eA0<^XwURK&P%u1gZl{1}i^1)Nq*PU~JU+2m;rB1pmj9 z#$6}#6#-_^0yu;tXYg(D;I6tChgZY?%lU<}e15Hnl$M(R+_w-d8_$`F>}9a51g%&R z34X+6I8DUj251~plgg3}w<9}NbJ=UFG!gg*nxpdvOPZx**w0wX%-@BG+-0%QT5#nH zG(R{_7YMH}be!7PGO(&OVCMakbtFQSdozR3$hB8DXDmE>VMT!nq?(D`Sfwn&cra@S z-ht^!lPmqjRH9Si4nd86CG``4{HEfjk9sf6)#)bn9L=a)L!D4dokP8E^^Or?_FCgm zN&(mgQ0jV=mc_|55;`$zIvm9omI{hVu~snCV77&a_>HrZSz;KimvQ~DSR~wsns>+> z(>Fa@l&L5~y^N-LOV5=NSXK8>gKIZFki7E7*00SXgH2e2sp^f+2X*ictYJ6}vqZGc zj6;1S_vV}VmV(0OQO?#ztii^-p`qh)cJ;>8$JP$F)^Vut{6j6-!(4!n;SG1ilZIjJ zzE(kZ%#g75_3di7JHzRa@OCUl$HP|gn?b&CJ=@)e{)^GV7RO?W);$Zsk>Ot!zrSjl zJ9Y~do7F56DP>GAOBl_}mmg~({*r!c8=p(i`s8|uU$^@mRSr`$9xittr`DegHIXIz z(lZSZDvbp$X^Cn~iN5~YpEtg`)c+K2{5KU2P~N%|-b8bffL+Fei`#A+?0(6SK7Yt} zq|{*u%QYL?L7B$*DZCtxM1V28b{#8%6}AZFl?H88gM&89aikA+Hr>6|+oN?gMX34f zE=xvSvrpe+LVA=TYHkPhZOgmt9%H~b8>ep=aGVZgY+w5A-w zhfQ#JCQF8_W+X$KvEwmQowi2Q z(9+{cgLAS_N!mv$IS`yO_gD-elin%#FpVBZV~IY&?PLN+rDO&doJ3CBM8;+M?ZA-E z*~D+3>^Gl~^le6?YA7{x!KFh}?H$|N9|k0#u^e&n)!1fDS2I;;4)s?Dm{>VLSx;{r zGz*})x8$%&(-lp%$9A9*s9Pl*MHjl+bp!VDw#|JXt5kWi+@|aj_)D_9b~1jpDc8{^ zkul}B>|k#h^{Jo0xZ9MZo(VAqnzo;m7|oRYJjuSZE%_+ky*wA8;`-yYnkp^ZYjo24 z$!UO1miuWpUR`W!<}eD!q%eQ}#?%muYUiePPU>}9?|R;@yl#rajT?z^T-kW_tSBy#&L(tX^+eaA^G5~`(j}6 z;=3^KX&s$DIP}djA%p>`>XVt{OUk2l;8b~~=(vuEH zJO$W1cF(jHZ*cn>M@7&9i7)}X0m_2V0c~hc-S=8f@aS3yPiqL^l0vJKqR&(FBMuGa zt9tqitA>|E%85B zV8Wqp$?cZ^p8U2!eCJqhGOo|_=CwL?a_kY9s>r}NwO&W|t|Ze#a5e-)tnopfG3J=mEk;6V6G?HB7|xe;5| z|4yj(WBGqxb7I}&JT2hiBO4K{as6jy{A#A|cU!FKS={GN1NV;x^B)kLqi~#TkT$D` z-|!V3O$~aE6@16U3QJW6giQyUY2ElS-1TjDRU6IG)H<9@7gPX# zb_UlweF=l*wIaUaL&ES1#zTU9A{*QCF+^*eeKH#%pD`@U3$E5nN=E87mX+_1lipvi z2xRX%TJOLouOl|^X3{4&W6us6D1rJh@EOX(f2NaU??E{lAY~(aaqVvJb9+BtEC0XW zJ371)MPBA&?r%y&19~1@{@%z3><8BxgrU1_vq9gT539g?o|Ck{G(waWg)#33bDj~ivp3neuM1l+zow)o?+pjky_-i^GLs+0B1$E! zrga`^t2I|rxLf=8sucW_j)2$zHDa{1*V>kqbz$$05ru9reGhLG|8n1d#!ht6l6eM! zO;(k?%QL~qFPb8jZ6jWwT0fQm6R_<%SL)wK%zqz$|J-$k`&MfADXr9b;?=(PzL^H zOCP9PesF*BJ4kk)NHeH{k8nJI3!)T% zmfFGyIc0m!9mpj!a`sT}oX}8-<(thX0C#$oD10|pIV?GQDbqP^(HQkk=qXizpX`oj z_Y8|=YNcw`cAktw-mfigmuC-5t98bW%h>ia%cxDW2)umt`n9Q0Un_@ z(3FheyHvzv)o7t{YIQ`mE4!Jyt_feER}oiAOz*xd*Wyujd?PI3n(MbX3v(JpJ>AT3 zUAr|L*uMr(aXhAE#o%~cv#(7k-hVA&^BCzo7Uljk_85%+;aQ@V=m+)5oRx zXn$ETn$JFl*dP%MN`)=3u%u=_)t-LXp@5jWmGBYHK^v~M(xMU16~+k`WiG3-6w`&F zd0v+n%^Q)$!z$A@3f>r(6n1VH$$8uMxc$_r*u3CJ#~UkU5SsjA7c7|`OOLmbJcHib zZ6{vw&GVbHKCFa|Z4ar{`UX+f*1FQb^dnJqlcYF%73ZnV5z$a4rNV(d8`=#~{2c3~ z0d{$cm?DpMOcU5`Y&h}J)qxh{`ddo=81qIG;v4f)^I9Sz?c7$FHfGkr5v|jYla4gH zf`|Fa0VVM{1#+dQ&#q>5n5rVpUKk04;7m`y{zy|c2_ZT#Ley3Kqw zt{GR<^Hik!gI{pt{KXi9VYHH<4=?`d#Oh-?3H5f5a-X_L6{7q#(h_+U8`dadNPyWK zkzn@3Fa~3jW6=e1K^K-{E~pM;VYXp;vCP_a25aMTQh+Ea|Jl!TM#J$&^!;ZP;cIX( z3xBv=HP{ueu@W5YTL%h-_m*@EW->hzf8Zfnu8XC@(gH}I$WZ)d$Eh?U*$qSZ&8arw zu_{kVjp!}9Dx84zyDbL?H+_gn+{GKq!XmW`uBFbOJqCvJkbu-uTD62KHc|pGqUZfb z4bv8kKoyw8O3w?LzN|O5jS;sCiI5>WQVH;*KG;�kLhmq;l_lVQTe|l!cnr@9ed> z$V^wFRdzGfBqD?6Uz7VGb*EMqViYWRVJ2N ze`9LF6ZYrfhB;K`nnXNYE>nBE{s+IWHGQ|F5|`#e=Egw@PiG{nQkZ&n);qOi>A*Bf zY;tiq^}g07;#`tvKT%5;Au?w9Ea;3ats;S8Xg-~&lYdbzVL=fPLOxTl-WV%mmcpro zrxq%ApMYIWFWD6~S8k_aNa9G&Cz!dE%aT$~Csn#8UqO)^^uWyMf4uu>0>lojDEI^y z|K48gEw^L=ln7o>jlM3ckEGY(_9m+-{a3osd-+gcs5Hx!C;yFGH>5(--@&i#c+S-+ zQIuS`s;5hKxzWG5@qCkcO$%{+ASI$28Uyk%D1i_hGAkH1;sf`uWr{?&EwxkYipfTT z3#7RDh@o3JR`{^!&KOVSs@1eoMCUpTUzM=J^BuI9i2HUJWs~zSqjVNdbx(7evzb{W zs^VQ>tR74dI#?KOTST`hujdQ=9fyd`EZe({RJsJ>Q~cMEH&$u6aSksH9Lw#i_lTa_ z0E$;q*pmjOx^NxyDOZ-T?*-qCcsnaLS;MFnw`}lk(?4=Dm@j3&8eZ}yTs^RXzx`#* zHs8Y;0w=QtkBgZ=<3AH47+nRGrbiYlwv5civc(jc?Y4VF_FMXI|B1@pMKx;~_&!xG zaxS-rweZyH-RR~JoV`Z4LLihl=^96TVe&;CZ&dGUT_~n&je?4w*el7Yr?!2Te?DK15syy1JQo$m{qM5=N}+}8zp`8ha;lJyKepNsj@e0l z-es&UKeiP39Y;J}??w;u4!5V*hfuxkygfdTKu_D+FlFx@COt+fT}ej=JMCUg@lEbB z+-0~6iHTJBv{LN+a%U49+f;?0l~UG&4B#Ej8|}dMeLT7cI#BdS+L~`+IM5K?klgqdNW?RUUg%^ z|9<`x+-a{#={+9kFu@nv?M{Cl0(TRA=*Y+(0P%;_du{?{y7nM{c7s``4fnqN)mYe? zb^}$sFMYQjIaROyP1D=?P@~Ozp^g2vGGp}ERNLL@XbkJ&(R@b-?)syGSnk>|!a`j7Rzwcv9=*AI>0 z^-cJIkFI|+U44K3)@%PffBW_MarXy2mi9;gWv&TD;{rnT#laH#gMDZ6 zIS)pe99GK^N;Dt9N(v=D0W>3rYFdZe(?qa}<90ED%_T=FA@@AR>Tce-?9N9XCQrPN zK$*@*-6ls_&PTg4L7OMTISEqF&O<_@z{I22@F0iznh)nh3}^wXVhfo5he@8QEAdHK zfNMm7XE%nu{`Yta6^mmBKZqiqWmmrslf#07sHA|Xs7p+|~R!XYE9lQ=z;^{Axw}gbt6gS*^%Lo*- zPJ5=plrysw@1dsjT7@)GnUrpnj3s7gK4t?6g~v*igTOD$kMWFW#w^=rTlJI_<9p1* zW{anL%sFP0js|y{#?3Z+>}Ov%_9!_iRj6}`{(Gck+{{b)W)AwcL%43>fnm<+W&Y1F zk0Hp+QO?|4XP>X5utpIYeS;Ax30TE!roo49g(|gqI8y z+a@MjjX|Bo#5fqj#sJ~dQ!COE^Enygx4=kzRaVN}&o3_`GUz884J7gIgKp~OkPpQ? znG-HF`ZiwV#RZZ!l2Yif&>-J`;fIFb-0eODh-?jg6FAhEHdpLFcsS67N-ZpJHkL>< zp+L)$d|cCuB6~Dl`)!(u=mOCBX^FHs3Ef4l(R84Vc4*`_s#cL=^z}E3-Jv=Hjd4RT zJ&f}5Sm4y^q?rIL27vTxLu(U z8VEo%i%KLJY79rAUMz~rMQhw}bjK3L+9c|HB#Po4NFqz4p+T#0(l3N}d?$7!uVslM zM2Arf5F|QQCqJ@`I!r28c}zYWZ(r5_+}@~9|L$Az8cL@Jx4*?r3%O=3I}0PMVz*BT z*XbjUmIK323Cm2VEt9|>M05y30E|~!e?2-&Ga94yL&vYZU%eziR_4KsM>1x|GFH|s z<61aeKpYDjoXIpFX&^enJVnc)_*^eu)9!UjUzl52S+RjjNE#n)ilx`aH=Z&fBTHE# zdUfhSiv}QWgqUR^P;fj@&X?xb=fLn>!cQV_DQ#{a2ZK58MIb(mjo>=n8H2cyGAQM& z9|nQ`T-R3J;=oQkFunz(fL0#Lvkwo%{q>wWKQ>G}I%;)g=0M#iOS0&-SFE3@<*6#+U1HDIu4#?+gUk8&PH%# zFPi2MoY1S=9Kz6_6*8JvuO-uy5BOaj$H5&1GL{wSRSL=5%THVXq-2Quq(exD=H(cw zCn}+oWHS3&S-H0dzlKBt14TAyA5#O?$zfGW;U6%pXRzx#94tl=89VRVKKnk;=!S6i zx@w3*e&V@}rO9J&Oh%gK214DjvRDVjmKLh~3#T>mcp6OT&h? zRCcNnKR4m)7HZe*;rYHY?BM|HADHU3CdHM;Xzv;7OTH17sZN!YHeoOpTQOm{^tM@5 znw}Zs$vU7k0hacx-irZZNQd(!y)sw82G&#IsK8P(n-HGfZ!OkhB^R#K^b;Ir{>@=^ z%az@6!@~|Lr=%vIAoScTyMStj(!pQTsfD8s(rDM!r{zWF>qk@X10>}~>WjxRT7B># z%!4HrhzAl1>l^!LCoe2Oco%R)$eD4P7E`eru%W^cBg$ffqX@OW+_fz%Z|x!xfcb?D z=eu6vXw#sr=kc^SY3*u$#A~Gq!eT`-xj@5ZMh0sFrMWzhji1xi!nRb2xV7U>c+YPhl}1)LsAi{5X}r~@d^7j7_=0ii^f zUQl&|PBv$XHeG>`@BUCG166ZQs)lRPU~R4+YfLqBwASg>>vE5_vl0bhiDN9(j6zp# z55KXBJ+fAo+u6#ohGme2m8iKO&>g{eQy*s5GOlC%>mP-GcGjC;^xX++Sic4fguUn5 zl+cJK6aCA(y@dA<>lp|Cr?Ny*@jj|oZzul1dMm%$s&xZ5v9Em;is4k-7)0GD9s`qh zjXZE&+44c|Lfrr@GQ932H)zm6BjTr>Q>eh>|6(U>yh08=Aq2pof~z*;h;k0eTGp9-)l zvk^a*+M3It@y{`M8VEQWDBY{@McgutX5i`Ldal%=2mYB?m65E)K$DBn@4J${u|>bu zqqZ^7-PzTL^eM+iu;RK8`mf;Co#++xqKEC^@KQ2waPT!!Y=V$BE7%!{St2g3X(SMB zu7aHYkG6a2&Mey21svO|*tTukwr$(CZQHi3N>XuBv2B05);Vpp{R{TEoHwJjc{BSQ z4G4jf-hebgUTZ0xiEPiUc|-Db zcGH=%w!fnU@rs-^{ase+pu|C2vGJWqTZJ$^&xE!h`W_r9Cd9FNgCqmWNgtA=D_ljj zbKb#b!x>WP}X#+5|LX(KwmP_9`&$2jpx!XJze6 zZX}y*)Rk^^g#2ZWj%4B+5|M)f)N^->yQriexYeY*JLmQRA z&;x2q4%UWeN4m}nb~+n(iPJXGk``TQHjAwVr3u|>1d(QhJ=IYAV|hlWkUC5nhGhMY z=V-@F`zvGBm%g+bt8fl<)>@|qBbUz`&x=c+6cgcJP689T#Hx*YR}>B~MAp?O(?6gl z0k%J7rNz@iHu_YK{v6B@p6cpGDcN@XSFiEpZ`0goaFws#YtM2RZI;3A^2{9>tvnxWMlke!^BoR^tBjJRNXL- z>CASY-dvlz=F9Q#`o&g>IQxl+TuHA7`Yc3|fAiGSFN)*`*FijE0To=H*Tx+XCI}VG z@$~2F>`<*;&+;>71m1-#rVyBf)~p?BGEuG`YBCt?d_^{AJD7#MT9#%jGm|KpYMMM`tib4KCb`4wsFvmCo$X zl_dswG#+gAAC@z(`33F=${{M5v_&&-wI&`;DGb%#J_#W`2?0#Ut#4qnB&h*j(Gq|j zba|iZg4QIHo;HcXc8d1ZpAXj%vJe31CD}<$Ocj@|tXVN#{q4(vJ=3JH-M)JV*9xUO z1@H5&38*A>+ze0NDNo)wM}9!(giU$7i31XS{VE34GafIr2vsS8jRS&*n!nd!@otxW z?hx$0QuEz30Mw?1;*i`vfcZEDqXgNMV}j>kawINqnMV!BK##Uh$E_vMs~&pv8P${z z<*KtqqWck|+@0`Se3%pc48*4UO%t`p)O`e3sN1fCaWs+ZVk7pAeQ?bLn3gnji@|wG zVGsGKkQ|gB0fv7cvVWhzf1iT?fPw$;62BEPqj@u*K9e4brfh&DgmEPGsTokE0pn{% zGIDHDb>iZ9G{VPI>DY!5s8?ui!{`6^_3wKT^%`mWLMg`@Dd?dv!8vEh^~g{B&5*VX z0}j{#IYRl=;o;q*pL2Q5!$7|r!2&JMfVzypv&O(P%QI$AKf8VazV$%x8F!+`q{ivY zTZ+I_N%3opz-xoRCxhAt4cm*u)Mp1ktN?K1p-8aX|1dX@elPINQSz@h|37P4KwKi= zEkZyWz^PsG{GTT?0)k&6Cu9*xY#$NK?7qR|#v(B=Vo_9+C&m-@_~^y= z!;Qp}?BapEzrmBuL`(%Gi!V7-%;j=8oK9A{QY;h-1cD&Yxl;j_N+l9WJ>B6Ky-#%Cu% zKzhaTqzuuQ;z_67Q$^rP|9+eXBB55q=pHz)7Do_;;FLxI#obXsK)h%uajgNepy#ab z+}(5h*{Y=tz-O|ycl=?3f{R3YXri!8(cvwN&4hHxv0~Tawo6kKMX}1#ebdp=B}?Rt zkW94pOLLMRqk|w!B*VH+L=C~LN=aL>mGu&lqQk>YSkq{Zf~dkSmqRV~%udq+!0@Vz zLKV1^(Qw<^P}FUuhjl@F#nGzDmyJL-62s&`Mm-6;yei6447iLlL$laz3gpmoYJb$U z?9dzSwb3NGYAc0`!Y~KB9XLi@yW(Xa_qkgd!*E61!qMJh_E{DaKYtJL5sI3Zy~-9g z5?P|kF=?66J+NqM48dz`J54=LMxx@O!0QF^hs9(AEa{Se4~(Pna%_>|G%gi$fV* z+TNn2SCmp2^us-^DwnG*_`ILvG_Pebg!`E$k-8yAL@@DV2R#`N$!;Vb-Wyse?k=md zG6zCKJu{n-Pv}@)MLNTJY>%_9b=5RQk5kpyyC|0QuaiuAD$6W;s_WfyRMu{^l;?GA z#<$$cOoE9e7=|)kbGUy7jy%i0an64GeuBtDY6o(!1(}bw&aVDk4l#bEdMokgU3!4b z{QN)x_64Q2%C)xAoQEVq?`2~`O*FNvDpbT2u!cdC@bWhZQU7f}iedNdFiAoG{W!~E z_x-deLI2~ts$tg~X`oO4^SbMH_w#lL(ctTTib24cD}R#6?s3E6uir+h$m8YfdBfkI z(rQYrfzS8bzknKk=fc;ogg|iL0!oP2&|x|x52CJLr*N1;(8{;Wl_?697bK8BKtfWe zPkWFvugJZAW1^_*cu-22!`D?ZQtQ+CFh-an8)l;NX2~j$kE*O;xtA==Py2{O8nwPs z)6|9gIJx*xqGa?;7I}pbw3MVUR#@cvh`pMiS(##V>2c2R$nfYLnBq()w(`noa>(PE z;%w}-!8Tgk*!!3hs!c|SSw2O$_n8uW&o@a4kXMHUz*v$(2+4_ItVcx1S(0M($w_gp zM20mYbWT|^dN;`#eNV>>L0Gaz2q~FktS3xKS+ZvIDOq!_CoDx+a#r#w*?(G3*qXBB z?Ceu=cAic+hOp$F;!|>uSx>osXUV(Or{rC`p7QKtDR|AN#o4Ep;60y72w^KH#iy2% zv7JjPWhe?T!WmvP5{^V59V_WnJQj_~ z@P90!=oEd84mWYV*8P)E;#T9VNNEYO`yXKNWijmpx_qG_R_~eZ_y0*KEdP~IDpZUA zNhn2r0x|e?n$73ShI(^abKPwCYYiqN`E)t%sasLbm*f9mLV*M)LH)0UqTkcyvbDf8 zcU}(L_I#;*dh}oxs*v|z3B}iwStPfTp*dga|4Aqr!}@$=Pmhn61pzVyIDW$`9R)sM z2%39AmwpE!9uBmYyCF!?YlsBbdqF#4nDS+K@LRu;_apGbGZ!LBgMWqj8kbfcMANpP z9mH@^f*i)Ou4al5F@IPb#`ET{6Gjq)V3m6CXlCyJlTgZ{5*ZlZ_Xs6RDfUu%m8~Mv zIDR@H({=SX$}$Z5JCETF(zTAGhrZU2vTeDc4{)W%sVo$6y19<>1oz&Bi7CU+E3yK@ zvktR@+6~W&!pb=Z;xs?Thk!2pU_N!Vl}Krty=Hk-8b?OIp}a-0riM z*4536)>Uzty1I~bO)Ih<;(yx8NV?69ceWz-SF$l<=rq+!=imUGEJ^ScDnzJ!+;Xh)%^qlvhCd@rjBhh zZjvPpQ!rwOE>jJBL9Js4mJg3}10cAzWuEx&EYmRa3~tMeC@LRQKlv?e{do6B?$x~F z?RLxZLAvhKABJB{tgBii-E2!M>9Wt`ToQKfT`CEAw2Qisbk{ZZ9Nw(0dL7r+bM$+1 zZW|V_fE}w%>%&?1+SbA4$Z6ND@3XaN4PK9(=;c}uxKsk=t_9!EIpq zLni|t&TV^|xz>>Uh|KqsMuYYDaW%wzPPh&AMG~|Qt8w5CJ^o|127kuZ_{k;4+>=sc=Zowdo?If_ozUXo-4@MR zz<3kRL8g%}g%2(pN|4|VFL9*1%FYMAQrISMi66Z$!Ibf&*wpqdVinRTD_GL7J*7|} zDigd*?L8)Osm3fRX0&51Fl-l={<(Qs9~&hxOn{*TM<`1i()hjog-tSvx(fsW zEehgxWR1c6}3sL~b=c?n;DRBtx>LBSoH(mowkcVE1lN0Skue zh&~u#MHtaZk=o`h_6DF4WX>XaXzLHwv6tm!Pn{BOOBPVYNa7T-NrEST9A@J}%R_E{FXTq9(DY2-AAIAn?S=luGiE zhrUrQB&^E&zE9(!^R8Do%|qetQ-+DFMFr<-KbJ>cUXV0$pR3CKd2TU~+ESH4UXue!#gkisAjAG#T7=erIa7AS5Zpnp90N(__vzez&ELumv6m$ulFy`HAKG( z%4v|(869Yz*_Z097X^8692i~)jJPjBX(QL?pfx2Hfc$0jeky$!!Dss#G-YJpaOA09 zv!hdr6=+3^1`4+(c(;MJPHkwEBy51*;*Pd>QvLU}{pAxeYo2$t3_vKmrn$bMmWPnw zvfs|R3($pIjz(^GH4<+oPtfgS$NA_J_bX2yu>u$DpiQN^#xDKd(xHg#vf{Zr^+udK zFG5;#z8sS%egERA3wp2d&}YpD$Fty?XN#0%B9M4+kReNpG{kqEGec3_>N1_(Ot;wMB%6=obmwBa%DW-~cQeVD!dG!fomc|By;sHvm4>Ff5l zoX+%xzDL*AwLJ+^?;fTD6y`x3QdmUJ{7?PU_3X7}az^ff5a8H-d|(M8zUS+h!2@pA z0zYEuEBJ(Sp=;i4&xuuA;mFF*vrd-&K}O?ihvrn8y>R5Ioag%-DA0SQLjJirMEF8& zg4<4)C@Fsn+7-nDv`#?J3+$Aq5kQ+bb07vn?E456LZd5`WI zBUC@c-n~Qr*LsSNE;z{U5gh)f$Ot_}O2pUGm3efa*1lh|lf5S#3-9xYyEg@hd?$Pg zU>%_PPo)ZcJ9G|jBQtH)2@<7?WAmOiShr_{WxVQ}j<3+Ug3mfY{&!&uUNfkGx5uUU}yAiT*305&stwfjiha9kRa}JAiFaCB)+t!26lPz(>Kt*Fyu~ z`z|2h^H2fs5h3(Jv_+fzy5RRO>9e9$;`hbQ3Bnm@i|Y855jfJ$D;|u;+<{WGk3!dh zjHC-w^lLImwlN6rEa*8R0M-cJb5ZnnQZRLAa3_%_jk+k~x(M4-Fl!|g zJ!S}VWC%-T2-leizh;PFX9zE5Xn-iL@FDBRlIuVn(l1b`B;5Ki!B9_Xe5SJ%#++<|yr0z0%A zq2A=$p5#`{8C_cGCgd1hnHgQ(87OSZh0W8YrHt2%Z35!Yyyy_XZo*B2<8Mhq?3)Bd#2n4mq z8Goc1f8rT`1_jlb>UlNETp$RlSA^H|78aHnbD@#&G6_4s7Ng8bw3Qx(WfD6v8Gb~b zh!Q~!$aK!ijBsRv_&W*u3n>YSFbORS2-Yy^S5^{sR1$7ilHpxqXJR}tGzf`QGA?2K z=1}C8nnUZEe|TohTV_HbM}qof!rWWT58#&%ms((Dc`Ukn)V!wU1{3hPATa8Al4Mq@ zv=uO>mDh@8G6i%R8C9BU7I2ne@?>gk9F%d|S~9a#0I;IR2d6jdxz_+viUG7|9b_UU zdn`9JFFzI)ZGMu1jYU{>;VrXQ_VdQfC$XJk6FRf_F9 zONyp-!fGV)qsysyRO>|S&3Iz|cpQ3FM&UV-FLY+9RyLktc7;}UXqBh2bEW}wdNh~+ z;6&zoWF&M^Wa4{zAA5%GNDfj)Vgi?!hgYgh7XqAPZqa!XVN~{PR`&dR?gF%PEtNz) zSNhXlGP$9@%$-lPR8AXK4%S)b5a3<4yUNgsD&rK|k%KB#>pl)9F?SNW0E;U-cq4al zDz}C!?{PS-*%IyHE&Kj7Zw<=>)H)9sCLhx>2k$lJ&#x@YvmB&nn-4B0x2b&1{rpj? z0(hhXFR6l;_oCORG=burFRoOuYJUYq?`^3_ijQRPiacv zh_%A#!o=c8tP};tO4Say3&36RA_pXtb-)31>0xlWRd)$@cF7%c1k6U28cW=*ROxtD zE`E1~mvy$Rw_jj(-n0fpjT5{wwwtA3`V*9gZY4r4BcwVtyqb4eZFF7>Ruze6Il8w) z29|SjRe3vE5!ZCpn6_>qb+yL_kW^PL*&YOGAn-aj2vK5PMRYc2b@lhVeyeq!-UsYH zCA|7(-f?wU@3il1BR2I#aVuq^8CR9ido4(X&@#AfsY^INw|-`}{6{p`@9rWTtZEn? zpylWa32z|p?z)$(YG7C@dFs;AjG8qd2nnEaqv;@~sha6U_rkNL0q$B8>2ez{+TA@U za`W20afHS+9l%-+93TU7%h60mr6p~umWCKXQ%t2xy=n8HTvE_hdk%sg==#h z)L;?@60GH5Bhp5rj8^rgbiBr5(<4xmurglv` zOfB_C{XQqeEq4utP2`JoO-B(X2lwWK2%z} zVqlHRuZEtZn%9penrH~MO$dcX=w|E=*BpO8nS_8#nlChk;2y_A75P3RXoL@DCQyA% zLX)4uw0?6N9zZJ2&xoQ$n0zj`THq*Q){7IQPq%Cr>3q zS+_6}{KH~hd39-(R8!vxfz2pIgbz(rU{4gfn_+kRQe(#xc6e0$J*04fWtn07Lup|)awZ;8m!CD_hUi_%j$ zeIe+tm+GH&-Q3SoSe&+k7>N8q?1sCns+pqI?lyZtpo%kiLL~SKPw)fG_ATqdi4Q1T zR-q1_`X8!c&c^oHyW;z4xJT*^2yOrR=;)a}_uJ_qBDiK!*%F!=8gfl>^X!!F5s_^4 zJ~wh<5kRQg4r*t-j2(k=r_vtOA6KJf$*v%Gs(W;tMaIc%n{K{6L=YgLaU{2=&wiEK z+;^Wk(ZV_vBrs5*rei`dA;Bp_-X0SgcmB+o_OPC@nRaHN@-JAJDV_;PX>Ye)Y)4!V zGT0J^u3ol{-ltO7+eY13SN#wj3s^PtqCl9$v`GdGCXYldpFe_rtJG|^2o`}|r>0eB zS_Vr+Fjup!obOW*cU>ygFd}8pWi^dewMlBdZQnSM5tqI-8h7y!XBCol}Q04X$!(b$Y5H#>2 z#x53JlCLxzef7+gciT5I9JkUDA3qB2LY;>v=&YW8N!rn$hp9r`HgP(jirAGxti`>v zm5WM!cCajw3YS1HSyFFPp?A{J@tloOT>K%OSNJ-zL1VU3QBNs zOE_xJO5?W7*!68=rDVnh@j$`snt+DYu4V6)On~3O zO7-;Fl-vDnq; z>Q%Dbfb<8rhI_cr7VMp`W@kH^*%Y4WuqiKd#129o5gfaZWJWq&Ig3H%o3DBIPW746 zJ-A&sx##6%zwUntwte{BH=0b)Yhdhu#l_u*Z{jYsxJew)Up|!I#MgOeFufDDvqJt` zUdh5O$b+y^d^SJ`f~JToTQGlC8F576xn*R~*oFCRdK?+SLgEbLY^Xx6arYG$w(Tm? z^xFRBb!+zk|HRZw3~rp#wf_;8)@9$0I5;!{Jwn5ssF&M2gPe^gnyaWWZB(tb>|!rT>>7;hu)MEYyT2LF;tKdnJn|M!R{%V@$HP7PiaO!S$_+@6U+)kh| z4K^2cVCRy)uC#l_&3`81?g4!2)<|w^bob41;gRt2@zSeAF`?7fAs_UzHv9b^3rt6DF z4Pg+d{H2&p#iLAYtoBElPKUYNM9;sGxgN%(M5%&Qg{7R2Wipwx#$`iCPofF9;M*pH zTB6cCLqi|7rB*8C3nU_*kFyJ@Z9gZNWG??J`6Gn<*SZK)I z@c(}6E<*I!J-a-606|-1-Lz#vDq=L`VQgPRb9EF71Gi4R=wSI?Y_mquEO`3g9rwGKX!Nj4*S}T8p7~UCHs19*Wk* z0imsfH$Fe~GV$>fEULhRvj-gnA{|}+mJkKD0?=}LG)uCKa%Ouf$xAEMJV>E3&Z#bw zm#nZ3$%%Zk4#`V^|5*v%JS=6U+i_J9AreY;o0WJiQB&7Vx@#8G&2eSQ zXcgqJED=jx-21w`Fsxor63&cJv9D28)uE3x%g_nuI*D0@78%}tG28e+t|9WH(#e7no?=6z}nrXbX}D|HnbAj z=J%WvRT!%-ha4<}FnpSK)bFL;bmgX(FP(JURt@}(r~l{#IK#D=6^u#;m8TKicb@&! z-;9L3l9dkgNT8}W#3dkj9m7KWc_{eE;>+J8$#kl0tUaQEKyZlaKWozIAb@g5JHD_k9F6F;lcJXjAjZU zcp{j;!|o)(RdH^Jj|Nt-7Ks%sbC777qj1~9&>x~fQ1c=oDgPb1sgx3TU z92x)WdlO>}cZyNAEk>X66lY1k$8lUF0^AS>p_Vs>QjTUH>uX}wZy-6E*qG#+n~w(F zAteUkl8OmtVw2nnLBpArLjE1BEQqZQN3&H$UUwpgQ)Xl}tu!U4c%K?FV?=``H6x*z z5K79eOz`(T#8twai28FM-!fZ*KQKO_7Tp-Z?;A}17{MHdBa>pjV1-G^&PAY$E|LqDUIgEnJ%xPyBk$3^ly;nZ#${2( z=EkpLg(SB`AL5nh>Caj!jBB2bvY|KgHVm& zbZt{1vyjWQVl`z;KkRIiwv$Y2G~kEos&Ke^*i2EMN(9_S`lw$LyvivAz2tkTL;A@5 zr&Y0zKJ;TtyLSGpRu^2;qzh%YpNP|N{r9(-=|&`C1MpkFmp%2v)ihc<0;_vS%mH-nsF2+~;>o{jXpf^5@N&V7c?A0^G}A z%V>pT5X%5hm)T`1n)wa1!1kW=k75JqfaiTE|3{&K_*UN4-y_vg7nd`;qAYC%S#k!Ae)9fweo#san2({I1okg9jq zRsyTWDNaoYs|(Vw>9Th>!X#}JF1*sammeC9l^YZ`U|>uN64AQ*Oc*bK>yLc})qCG* z1*s~0h^2SN(V_o43SD?^;|D2_a#6580+H)LlY(#Lk z#KsuF3^p}|bJX6V?#@BN*Qoz%hAA)#V~JwSeGl)q_5h?`74U!wtAn9UW5MUoAjAw0 zHcG*WRL!|WI9njI+}Ku-fWblJ6Lj%oD)T3ozd{K1*}>J2fDw0j9IWFxE1^rbMODIG z5FaUO2305bF$`CAVYm||h`?aq^e4p^42BvKxWXBj$(bPHm!JY?xa**RdND`)BaM$s zS2`@|aY#OOAhCU*052xlwQz2L4(Q7}C`;y`=p^VkXk5^J#AgwN-QJKLA+HJQKz(Pa zC};S;&QV8~aDwep$(-E#kzO&&0R_)Kv08l2W&9jf$8e;qq#ZT~k^TwD0IC4IcSJ;~f;bmK;11NOGgSIuND-`) zc)br#rmKp1$eU(VSf)_iTTIPc6!HpMx))p;DIDLFDtLpRMREX3dBR*#fEf$ZQ?tNc za7@`!*t>Rux)%v+dsx$U%ztQn35t8sr`YriB0`I^R}s`vs?WKZ{Uy4H&@&vcUp2yI z2m!As{xw()f945#FTNHH3_ScGt!oq*95xUb>2WEQrhl(qvB$dy5m+6v+cB-VuL2x@Km zYuLWr2rfs-IvVm-7s1Qrw;a!eyRRMYv)HRf;Inq3`6$t!*DJ$+&s&Ws%@P*{ZXA%WmM z)Qua)RXKy5+GW@!MjwlPNZbrfC}P|g1pLK1QaEWL74|6t=_C}9u)+u!Y8mY3%};vl z<4T-jWko#9p|{%}7tRgM^_`uBsa4&ZF`WRbiFdUvcEk)G zLlx7>icg8{B`*#y_f4Jg0cOb@8U*1}TNLoufi;Ad)zk!cQUYJs6tAcMQh%c{F0f}m7L)ig>^<21mjEZ z#0~+aY`Yg~i`S*k-9#kvts3Bk*B}Hpw}dc7;$~(8XOdH~RGbII7f+oYm2rF*oNs63 zcIOy_P49D3mC1%anJ6GJVV>s9PP4YaY^nr&aEcYqVmu>FgG4O25@YGfWgy8c?L}w{ zh7Z*SX%YT|Ho(c_7`9@m`xrrn}#bA~w4Z{jTXDM<7Wf~zp6n{TQ363{&k*ZF(fekeDECM3qKWn3O zFoPXCiN$tERF{DL1XrkiXtv%4k}!PwO^*sJA8Cyt6{_Bzl>3!u}%Z%zQneZ?(O7cA*gu5}8oOd?DO zLHYBBODY#!WB8>ag(a>O@MJoRb>xwg8=nqPRn3Te?FRRI>-^JF-0hZ!kN%u}jdv`< z2iqdsdlon3^7mcxe*~#}VhVgbk&^Qc$meR>Iw@4!g;~dHpKS!aT&gyZE=g@|bbD^( z?}%mPttq?Bki_tj*R)^^%OH>-G?UdKk}>J+s5!tXF@Qmi&&o@5@;KLW_0fV?lrhxr zL_0+eD^fzptcMuiK~DEH2pr261PUkYgxefzN{NhZ2!qR3$~4q1>ldl^$Zciq#AN%a zt)?mZ;jwG={1E`F4LGi8gKh2MV>0MtY8qSa;$sAml5Sie;kwRl)R<{)aI-Gwq5f8x zSt;A#blvK(+Yy}_anIeM1#U{8(V?8ui^*J;*iO#ESt?%ig zg{QsZr@zyyburg1R<`gqR=b(D{L{8mc)Z1+s*R~*@T^BY^GzQT!}YfFmml=csa=9% z%gkpTZr7Z_f+sy>ev;sk;>eSJ?7AKV0^%Xx-aO;qeK|u^bJZiUR`~oRS4aIecI^uZ{QHA`Z)1A z>52!(NqQ9Jc-M6Nr(cFaWaY1V>nQAuq^orR5R8<0@b=?A8adBkOm;Qy`s_b_2f|JI z`*mA02l@aIV@3!~1oS;%d`6O!MplT1r!nGd>UszuAAMQ-R{+{_TE%_9%MP^yo}uTNy!|H>hp z%KOj6#T41YpC}>hQuKr=ac{w67z=}J_SqeqBh1u=>FKN<$eEh@t5@FXu~XECU*6-F zk(>oX06j@rhUr@}S)UpI;Xvdu&knKxC>EXN5FL5f(`miT2MwRrN19uW4;Pym+S*fi zKAThFSu7BXr=6Yp(_2NNoH;f9%Kj&zh+F3NpXaVxc~240v$IlF&IiypSm1ZMx-%t) zvnJtNHJVyC`kI*Rnx)pWr_~$6_Mb9^oZE^YLvAoMtnQA;o;2WJt`sR7656z~(_tYM zwuEmKhji)0UpB&-(b=p2mbGTCQS#%D_jt=F8#n8E8E&WpuC%XCk!LXIH|BFc&%HcY z*S7Il7AJV2q)ZHI%(tE82i}ZojNPRU7c)2}3Mq*%j88w7NanZMt)6zakx98YtTy0n zPZ%?|g>$pTrX=ONccKE2j zeaF!HEZ6?tb@(YKc)xJ~LcF6KbO8J~01G&RS)6^-EcRo9f)IDjSU(2BO|N zHe1HjPrYDJt;#d4klPQYcMT3#`JG+)2|M{eULxP1yrMf{29RF6Uqe;2tk>(;w%@^a z*fEITA^`3WKyOg5><9^*iKFFl1&O`W#{wXri|m~+67CC9oK4G|Tc~XdHgB*s8gAn4 z?%1*67RX-corOsr?4Rra%o*_e_D(3~&dK-}452oh0jTH}$h4=Fbbp;)E*{SNj5NXR zp5Y(WmhRN@PlM`QB8(re7N^ijw8a`gxfbjx=3Q`qKNcOh$et!KyyD;!xRBhI2@@QV z6+BAWSw9#*sSw=%6@KO#c&y96piFm6C^CkLU=q3|5zaWB96#gGaGQ#L(x-6qX@8(8 zaDozu<-&ji>AkO`vX%vdc|haZdU0h#eBL{8Gh%qQ{Y#}D@o3-h=uj{KsjxOc@%f(Lyg#hplbgfor}P5ql1_LvV&EWwUp2;QCoVWCPax>*6SP|uAE656 z@Yj*&SKyy#z;)SILIr3qzz+lUCnex%pTj?O;jUFIL(gi`xyNGy>|YvoJ|#3UpNzXtY{W*WOH}!)tQk< zT5Q?58<(Ovc~B>9OJcxJM@jG~{XLq_2*^b}0b?jwa=3}*|Nf9UUGL~|&0o7%?{M2x zN*<-?-T5VEmO|F>T-*Po_B%(zSuNu0-ECk|?zcd=&s|&DZs2(n#h(DJyo$m=aBj&} z+*v8kBB=nr6HT*d=*3J!OnGyxNQBZAY>TEac5K3OsuC>Q`Zg#W&yk(GMZ)MSqj9o3 zQzOK?nCgBrHwW6QkH?A^E@T<$P(DIGs7oX9`-T}(wLOf z0OKqpf$Y&F$ybYp{>I^5WcWj)Qej%UZCwQUWr5@rZ0^DD={e6*#1y$Y-ccBt$k<(# zp&0sSMYJ(ROqgXv+TK}E)k&O#RqEAM(qmD!EZ=)n z1v#H`M5FCaFb~%uwp=s+LNE4NhTRvd+_(Di-kwE3IPRXeWd!k(#!=IO<-ky9f$4IU zKrzzVj#8SV2%|_N+M@)6@0?6xtxU``eRvv~rj6afxXcQJ$+jvUBKfU1-b10rfCT#5 zO!sh=b?hKDY;%7XNuEXZp_4o{>OWy@=8a2YSuUX}j|H?eNi6tPscBCxWx#PYA)Z#V%6WU-C zW;pm0NhAfds40}M)0YQv3_mr~=*IAnWCfJ)E+<2Qeq23{Tyt?&8r!<$)@RsIAO1yq zL($o1Vf;PXXUY5#7`kVU;^*q8kHO1sC(g50FTi*Xnh`VxB0v_eHST4$Iv))&IdgiI zY|sqb#8n|CvT<=fCbVzWVGIiGN6&&cZ->_g77rguEdl{YX&Ixt&9J+%fS8UXgyo8h zljOReNh-N^MwMT&rxcD;H}G0b;%MfiSvAVE2k_f3+BGS2^j)KaoiaVu&g*cNv8dgeF8A$i$&86%&% z37_*#hcV2e?E)Ef$R7P0day#I?8%vXAP52h+BNa^AwJ5vMC~Udi}iir(M-OYAZ@7% zjss0P&SYh;u*iu9uCXTAU_=}U_oW_u^n0qzYAMcYY84{JxmF<&8K4t;wT^pUDKn#} z5YBW|vLm^eC9rE3;fGp42-z{IgFW?ca9QyI>G`N6KT7WpYmJzhcZ7|o;` zJp>zk9Fmjh4`(KZbRltyrECdR5u1ENY*-LEKVl0_*5=WXzN`6*ew03G$+Nh;WMzWl$ zZEA_C)Htrbdy7NlN^0>V@rC8|n2NEPVzvpm!~DARrXM}?3#s<=ea_j zDQ%(v1U>N=jx{Q#^-34>be6oDvM5=#YW8{psr=&^?hhd%TQrKz25UqIqn9!;J5rFE zn3Zmdd3a7)OS$16aveJen_)p$m|<$Yy{HhlDoFx^?z|=?QHiw#x?f{ z=T@5Snpe^13C9k z4i3}ZIEf86yVzb^RuN{0>rtn%y4qn|eI#q(ZOawMJq_aKN3T{YvDV|sh3@xN{Hih< zgkBRGWa|G5K5VUmA=&cvAD*kUrW<@p@^Tk?wb!(?*@3DOv@&D7< zcb#F%;I!dp{*s*AcirTyv$cxB(5km{>lfl)x5S><%M=ak7_kM65a2mNLhr;R(!G7! z&uPV6Z=dPZiVSaQ#?p5_IK;S8LjPQ)vV+cg*A**9H&^GPmz*;=WSr5+ z@8i*TnKS7mX=%UE{)02a%Rn*ju={dk)OQp541Ta@=+QD&c)Ud9vr$#jKZyA2daV6@ z(+$7;d+wV)^4HrI0M`TMrtiUNrDy;8Ck9aHX#D8^3cdH->r7$(vyyb`1$mnPL9y0* zuc_qILC5g9@fQuQMBybDz2cKABp8bP?R$?R`g5cn6tidHpM=uzbN}UJFZwM-02;wiYt0n(4v;)vuyU^N# za5jXPjsXrYghGk>L$^Skio{Ndh_(m>uD-`6M*!Eo1Ie!gS-uIUtPcNEP!&gr_)>`I zjD00$z&QE`{$c|BM7I-d{{XoN1sfnV{Y{7_UWA6BgoJM-Kq-xCEQbzm*nSP96q$xz zoP?mdk9@O-mj6#eF{*BqrfD<~R$MFx(|DFa9T??C5tJa9Z;!jQ1++L^%FBflwgk+Q*Q1r(yv792&|2KhCM~`M0=jsT^ZKIlq*51{67CyP(gYIvre?nulCSux0QskvbFoj~?rW}>S)Puva-G)-N!E~XeQi&n7qad6@ zW%{Zkv}a;;s-Uc@pa9Ne5#}=G6H~@!)7JBnm#nhv=i;BB63Nv;TBtlqOp%HsG0rhz z{a&-8++n%!W3u2ynoO{hq!3Ec^z=$$HR?Fca1yO%JnffK$l~K$T5_o9(#0Lq(#&9u z=uzrfSxHgT(oSJIMZhp^Xt`o1h(&Oy$Z!w^94zaeyj4EAH90v2H+-8K0Yor(omPwt zHj!>B@xTzOsq_g@v?(p2xkRhETsIjr61j4)D4XW8xw*O2O0e!vxL&ZRjXxNz=#blz zApI=xy6mYT0y%9VIr)8=ZF!N2e~I`wq4+_$c0Gm9F1dm`0sX%LT*fE@a0y2Gd33qL z0m30nf@MxGWo${ol!Ye&`G)XU6jY3qteGIRy=Gp<6>LE|61`?@i-WYhmJ&z7RA7|M zfhs}9#_r`7?aDs{{wkIhQB6-sincW5kGPoKz4#LG-zv9=yG;tQc`G0m?qwx z8Zu~>DaZmynu2l4`dM7e?LQ~cNlGZ|l$|dI20^V&WIfUhjq9=+* zVepj08E`{4oaq@fnNSKKp`F2x!6-_;Q`dmk^;M)6sHvLfqfpiX;6r>?BPj9+{z$#V8C6jP}{F1OWkzTqstgyYT5(nAAMg z({6TI0rJCULPG`C0%1z4S00d};w87l0f-)IgKCLoo`xp$r&D4pNK&i2=;eo20|=dJ zhE^+Tl0%|q=xQ4OSgoRA@I>OCa6@jky>H@>XrmJ#KV0f@W|en>L~*78sq&A2 z?q^rt>lx$HP;n9iR9XR_`IOKuk3fam@N&+TY|B*=r(2CTwj#al5f6E4&dsxB!iXtuCZj5 zJUp#<(AwWRV~1hFCrAwf!J2z=m+)@IA~tLUH?36Y33a&+Dmbue&0aW(!Z%PVyE5ot zCZ()iWk`Cjc)pHHTgZ%Sb{qV8SyX zEo43aZ+jlB)wwMPwAb!_8bn)>J>@a@;7G0RsNmv<1 z;Tz9Oa_aK^TH}wA1IHaQea0=BxR=7q%n9xkRqvKi*~B6cORoL4Vy3^Um|^vZ@9iLN}R5LS8R5BH)WFd zKKgXRiuYI4<}wGiLL&GR#+eEeZ7Tz@*;=hrt^&o9svldpDsvU9QfvH%WSDmDboX&m zI_27?I38aj>l*lhEBJ#0Hai04Uk~MBzv~Gu=!XJm9Gp1woPvR(_PIL$w=8UB_HL+< zgZR}6c`m%diqimcCOK`7fsdcKY7+VC619x?7G>}4cq2F>;J93K;IqO(4=X+b?l*)OBuj~vLTZdoifX{hHyf=({`Ii$`m|vS&g7;;wagi3j44yu2X`Wll0pKp3B zSSPI}dgU#8L6W&j-Z-!KIzmEhH(*1rqAKQyE0;4*pJzH|5S^^AYtXqkyk4C#2za3b zh5sm>wTmvfZtt`Y`K)I<_9!;4%bS_kcs)Y8Z|AfrBKkjS@8#P6@wLyoe@?*0o~&1< z?0&;R$oj>$*T9@NB|rQINMYVhA}NDbVAfV(wbMIDAt$&&cDdu1uK#p!^7^@>`@gw6 zeub^u)+~}brJ;wUY4)>e#_-4I9i{_(g3@`AhNRFXyhk{3p(p5Gn!3wRe6EwUjk743 zXS`ddH*pGeM{Fhv%WG>|yMDGWovREv063;j_ja0Yi#w&nn&^dU32Zm0w^Kc{TfK2V zxxDLqJYxK>S3Ka3xrBG$c9tOG*tx{Ey?<^_>$+YQpLM{n>gah8I2IM=9BV8QPQ9%}l-J|a7G^*22Y)^5 zof;dCa(j#~G=JIf!Sc_dty;Kf)G&AuVM2qsX4xwE5YEAf3USTi zrx0Vtg&Gr9%r%i?NRcB+mNa=1WlEJRS+-mW(&b5*F=^J!IMb#`oH=>+^!c;rPLw!z z3I%!;X;P(2nKpI$6lzqdQ-KQ zwsYy$wVM_1UA=pC@b&vr?q9)!2@6J>w=mzsi5WM3$T6^E$&)EpwtN|LOQM-MclP`l zbm+&3J95QJ8fj$IqFJ|g{Tg=7iLjtqs|C15HCd}~l2fG$*3h`Jg7Y37QP%HQW=(VD zr|NiY>C>rK*E?&@oKv0vq+*F3dh>`AM$JKn%zWVEx;&9Gt`M4eQJOj}Mmc}9b^ZJK z_fKui6JH8ar4x%V+UgbhpgGW?gb0i!D_OLAkGEO4a}T525{hMxRmSVYlhdL@2$%Wh z@DIflRb0^|4!b!BmkGrK377ch3vWYDc9RJIzk^KCWGDhXX%97iG;t-N_?T02wYW&E z=E4*6W0A@#t(?w?8YXxw#%gLvhZJiTsu9Q0VDZG2E>TNxmV|mVsJuA%L1w{;OgUvl zS%}oclo2~j5`ZP=_~;Z24XTYf#5yr07xXB+64FQ|%?yqo8;q~G`P35e&-^&t@gP?^ zU1g4M4CyVsi@k%f|SZnAt7Y=nb%@{+!ls3Urq$MZTY-K!jj#y$8_og3N6_(z5 z?fr_MZm_|m-GgN1b4G2|9oQgp%!8%>n_iJzXh>vr3YW)$v%RPm5W9Kh6oEmkPds_= z{TSqs%X$+IAt%!1PE*e7Xk8j8hO#(<-UVqp4aa~byCvx)#T1)cCD9jq)ab-Re$WDO zOM|fKccEe@Ql-#Becskrk*&TOYuS`-u!i^s#fxBPzkTe}F+yd!;8M><+r(ns>~lSB z)r4_f36Jzq-Qw`hn(w~-W)PaissUg}oCy>q(VOt7bHlF>!g*C&)c#UCZXYG`)rHgc zQX+u0Tr}gU5A5~t(oM%2=j9FpFlm5uMGU878HZMsHtC*SRCH;;Pu7Rwc5)QG3*uA2 z&DnKm$l^%^^z`PPx7V0XV{ocv1<81c_C1?0#^fr%?^7U@x<)TmlaWE3mGya9^P~p zFCf_ugBetjK+^H3pfp5z9u&@$KuEqBX%K}eRN)F)*g_Y+5QZ_7;S6b5LmS=@hdI>Y z4tdx^AN~-CK@{Q;iC9DC!atPGx_z@Un+eZYM>|G^eG-eGLL~GGu`ThAffVFiCa@@w zkV=b#ROBM-lCf%h1ZMzx1SBIVSxM(qD_WU^#TXB@GjnC-N7>Plwk%mnQ>Fx4dUR68 zvZc3LSW9xuI^`^BIioGf%xEFh$n+2bxmx}bm?NZ!As-?NV~{bJ$yBEMzQG4D&2f^; zl;$AG=e0R}0*lks<~C=-j}medoZ*b7ILSFnWR{aL0RaFZ`2+=I0R8|h2mmbrpaLKP zhX4W+00J8T0}B8N3jhiM0s{mB3mXCpA_5E(1QHYn6bA_o8wvpc3Kba&6(RiPsJ6MPuC1=S)3dj@v%JN%!nL@@b}wP!O!u+&f~()>cq*(#LU^m(&fa{?8VgA#?|x2*7?TT+s51V z$Jp%0+WN_^sL8go$*7M5O`pVk*%hct|)!NI}?aSK8%i8VB+VsoZ-OJ(K&E4Y7 z;rz|w$j;#P&f??F<>=Ag{LD|}s(%0+k*t)sc>E+qw_Sxpt+3L>O>eJfD#M+Rz1^5XH>;`8<6 zyuIV+*yHltiG8S(#`Ah z_3Qin?8U|G%**WL$n5Rx?atBd)YI_B$MNO8^3cxn&e8Mc#`Mh0^wQGw>&x`?^!4l5 z_4WGp>csZ(%=Yrt_VU{H^y>Eh{`dIx_{+`u&(ZqR)cV=l`swHT?Zx`@+xzt5{O-&A z@6!I>;r{Z{{{Hs=^3DJA$N%}w|N7Mb{M-NioJq5$ z&6_xL>fFh*r_Y~2g9;r=w5ZXeNRujE%CxD|r%(!vDk;FZ^8nAFKZ_1cdbH`%r%|U?O`3v$*RTP|j!nC^?c2C>>)y?~x9{Bz z1Sr8oaBt@0$d?wtB&@ls=g^}|pH97c^ahrwWp8G^yLal(UXlM1FbR_R^PAA8SI@q^ z`#7iEoI0@nz5V<69pK+z|9!FegM9LZ1IZO=AXCd}r^|N?zS1Ch4mK!Zgxw(!$b}gC z5uk<~cIe@UAch#gh9Q<{;)xxONFs_Xwg|w1Fvd941I{f7%Y+o(nB$H*J~x0TJ(d^b zkUSP?q>VrF*W!}&$@fQpOGfEolTbcsWtCD^`647{{U3FwsoyqPH@n1(9qrgQ$O$El=_`YEb)5_;&Xu$KP>X{5B)YU`~@PAX|Ajk1}l zr-E8)>aV~O%jdAHu4-(ubwr|}tFcBa?MAcSsw=KUUTZC{K(47C8=!*A?6Rr~8!oxU zmOCo4=(Y>(wD88;P_^6ETQ9xZ+Usq)cJd`+_9e!r>yeC8B@%1%ld*WGP)$2?D3{Gmuz#Z zD);R0%P?nLA3!iV-376q@@BYqSCw!upYVz&Nhn_pnz-ONszGT1v!U-IF zkU|L`2ysa9M|SU;045}He76qH9!D50$OZurZ?w|6?s@P#!~wE-f$~;lIsT2R3M$Az z0IP6{*fGx!q|=@R<$^CNAb>PFcps&#lLFdMzQeHcN=Z8->HWe?{-K zazFuS0wg0j2vef+nOgx`7&tS9Y+=!wfq_mKj93C!auQQaP~ZaG$3}6{j1oJfojPyH zfmRB$Y2S2195}$o61Z>#eMq4*p~?SGs{oA!1NeaIYBjVFKJI+YP(ud?K!q#TGI=Ag zUo6YXOckQ>-i3|D4yv*)UL_{S0YXUN)WrG*p8~+J!?yI0L{sSK(d}gVngx$d~sh|x2AgJ*S+*9Y8m6y>o1vIT?UMCTpJ`&PK|IwJ< z;u=HhVbV!$OlO|HK+ZTc(Fm0~r{@IF0u0^KS#`|aKIzKYqP*-gNMLE}5?QOHnlBt8 zOawO5ltp0y^FPykD@EOv**gF9;E|vBp8=RK)X55TK{2I3T;uFIQXH~1+8TMuE*^N^F%YqK~YHmukyvX*~xWUD6my72`sjFUEI zT5zrvQx5Cr-kG8;ySe`-T_zeWm)v23YLb^_2AlzthXyS7Ysha7v`ONu*Dnh#%OW)p)G^WYwNzbGjSAVt zu7=v06|GJ{qgpY>NeioshSI^}j?KlIHLxvu=bPF3)`$5-WX!VbiZ$C&p)NJChfOSD z1J2UO&UVBek_?S3tCHa)(4jzyR!lX);Rp(41jpVCr!&3=y$i4FGT=e;ci8;qIM2Dxcg}O1GoTzb?|IOLPV{N$ zqu=|0M=*X~^rk!g=tjSQyGO2Hl*bRiB_}tdTkf@&Q#su>TX)QZ!gO#v{p><7yV1Qt zkG9JL?M`1iKO#U5xVzo%Z!bpy_ziG0iQDScDYvM-{$j8XOzdO#NjvB+cY8Sg@pq3r zrr%G@4olI2mSAZPkhLq-XW~tt1JGVb-~j*m$Kh_*J%Okf~yPhXCaJAw7-3G z(3>j5zlZ<$$4~yxm*4!34}bB|kB{@K|NQWezx<8=eS{d&hza=3YWB~6kN5xotpPv< zW&jgbfKo(&2WWr_NL~myfDV{|54eC5IDrvJffaax1UMZ2Rus=r8Ov6G6wz%UsC#wC ze(Ps?qc?vlxPtM}b`oHMC#ZtQM|H|qgQj4ED1T+Ag)lgOGsqChr)2vAIXrlSbz>o~ zS1$Uo2Bc<$n9_!@XMKDY2L>Q=QMeQhR~jGJhki%^fEb8^IEaK;h=%BY`d5HTz=TeL zc`5%GgY5TykQj-R7=vN>ZQ)U3XV?qafrGAd4nMetXCQ=ya{y_!hS=v546q8;(2B13 zim(`qvN(&EunLfZ3b>eyy10wH*o(a=H@~JN@iawR;fPr1g_ih;l_-45xQts!f5=C2 zn|O;m$b+GXilVqG&ZlQc2!6^i57iKk+AxmNAdcmDj_8<<>c|c3hz#zycr%v?zBrHc zIFAHSV$ugE-RMtKh>XeDjLw*O?;wc*IeC|;5MzinRYiuKriN>nDGo`8#aEh9=j~5Aa1?hX8_lyD=knsON zkN~-p>BorwCUDNCdP#PTD>rQqxo1TwDli!pC25Wz`IJxzm0^;Sqp*@zd6ih1l_$9z zDkqVfu#{6_jK?^Wo~M&zsd?gHlWIwcWNCs2nGgwSXPfA0k49T?7&4?NUs%zU=@6BE z*_VG=lA@rMRY{nJd6-;zlwTQ5GTD~Q$djB$3~0HQDQK3IX^DIXl>6n0bY@~niIiMt zaEh5Wd#RBf`Io2(l2cijtl65N@S2C|npa7b+{l{v*)0|*4npqZ;BN?F7S)GD; zpe%}@%ZYFeT6w|X1RWp&5x@hI>7Z^op~4A&P*9^CAOn|Rb`NTq!5{+_V52)Qq(z#U z^@&m(N|$v>X8Aa!qeG&}5}u6+peQ<^1o{h9iJ-BWm*ubO2 zXO_gkrk=3^{&1mjx|5d>0enhvY$u^R8jYG6a2|SSSSm?*#+)G9Jf;76qF(x_kQ$XM zDyCtYm1O#vS}KK>mj!@`qfQWUTY#ZGYN2N74}2N~a~g4|3aC2@kb4-auL`HkD5!&K zsL3fgRhpWje4408kk_(qMVQnnxK-#3Y)TUsY!XJTqdge0Hie+1gk2m zG%0y+dX_=J8AOVtj7Oen>UlfB03qOg@KB>caDp1D5sSlNUUsXn@S<}#qKoP|_ez@o z>8R>3ti-CUsZgnux|Kq*pPBj;;diR#kO@qxo8^gf^Vh8;fUbMkuv%!AoLT@wnhAUg z1x-q;))ECokYDjyD`ar8aPU$jD`)X_1V+$ePxfd$l>|p{1zrEiOuckdFBz=s0F_>_ z1sMsBkt(Uh%ALs|24cVogdhn^JFo#uus1ttI!kHFaUIYjN6ZK9U zMPL->oHkPh3WNf^T1q^74gKnod$6@w(6h%dwAt{Ks|gIu>IEB@wa41DPHPIz`YX2u zVFnkY>uIVJ2d*8En_$b9ytf6ay0Cp3s^ZF%o;z_Jzy)}kg>f4((u7rX^rUi`x1h1N zeY=@~GDkD^Rc~mQqEZGZ@EYU;N6_P|)8j>zahMewv^`&J5azpu&%D_8DvU`(G zAOi=Cqh58R4-BdwOM|v@OAK|lcuOl%wE$h<1tDBbJ;<`@vbYyQ1Ftc-i3K9QRJ@m! ztNf`>fcpZCYe(Q44UiiRTR^-F@C4*rzScRZ^4JNfPy~Qz1+y>%fS9xb`?T{rDq(QC zpA^08qPmZ_rv)Gb{dS~V{I=^Cei^F(53mCIunl4>1E2cCM0&BKm!y5Vrbi?T{l>l> z5W8v_yg!Le_s1E-QD-dELBGYpp2;dcB?Zbuw>tkM!*(KG*y{={BwUQVXf!l{0(4`J zyRX8^x*ia5P7uU9T)vxItZSOkBC5o|r(C=U`e4UsTYO>+ z#|$6?e|mEjE3Jos~8+9W&m2$M993BSKMR*6VL))0F*>&35pv~ z_=sbwvbbQd1a~#blnjvpPy?L|1Hq`5njE=0oDQG*16DA;{FkXs6SPz`)w zqd*MGp-i*@%g>;@m0QfTqbmxm94g-R0`d4x~;#-n>5-1BrxFVsnJ-d#}ZQ{3ng8g zK>^gA()I!e0o~HR)>VHEAkbZH{_{x109`Q4*y+LqMl}Ke#Xf>NI8zF(J0YE3dfAz6 z-k8hTE9uW*s@|(?H^q`&H-tY$NZ(tR&`L3`wvC<*j*RX(nM0S$pZDZV4w(>+FZf3q zE8D@1(o-lMTc>4)%snS#U<5YMV!s3dAdceSbh^+}R7w@KB3>*iu7KVZ<3MeldtlT! zp3lCxaT;I_H@M+H9=(h7J&gZEJ4VhVtL>Wx{;(1)qkh|D7q!wpKwcC-fFEvVfC9qS5{FZ^vI)r)0-41S8xM1u<7P?12*fcdESmT-sk_i z)TzM43Xrr&OaMw8)ubIyMM$-aUTuvY=@1H;mF|U94!f8Mve~g6KyC@KwFG2UvL`dhL=PiM>5HpoMjLt=FTGNt|gdp_tk?L;;IEX<=7kFFYIZj=0+k zOtKC^_pS|lZUI2+uh{>Y3&M`RT`h>oPVn7sEC*jO36B&?PU$*Ie+wV+?pO42I`Kym z!gOG~$7?H9Km>NX+=WbOI@SXxuDw<)Fi{T#EKIm9q6JL=WQr!^Izi(gxdm7-)G~j* z{r&?pPy|jq+RI)zQJ?iA6YY-KlSps$c^C9T&y!bPtF2crRb=s1&#SJk_#sZEUTHWj zpVT5btU9jjV_Mpns`n00p>MtUoCoo{sq{|oYiJ@`qc5c*&+dKmt4PE8c|YDlO!@j8 zzlE-tKYtXU|Brot;JZ(g+CGhiTKHvpGJax<#8RQ zWq7OL&*tP05Ss${F@&d}9)k!0Eo7Kb;lqXpAu6gvaF8DW{WNam*wN!hkRe5mBwG_O9v+XgSm}T&zB`{wg@`3=*$o^US-|db;py^ zs?Bt{t~UQ|+}E->J885iQQtp-|0asmbZOKSlQR_X+WGUVZsAT}E1kOB>+$f()0$O# zcJN^7+SOa0@nXm64Zl*4zWsZ_0FW!MT;Klk<)lyJ-``pPasK*?grL=+Gw?u;V#{s8 zU>LlMxZ!w-aKfOB6A-}8Qj-cn4mtELJP@q|5j+v&8gZ_=N;(Wa7S&s^MHt7g4%rHtR1MD%+Jon`DMk}!tG^BPUNU0l` zSW^G=Hbxh9G&d!e1f@wzpDe?&9r1&vkLJEfaZpf0I&(`d)im|gQAt&GRaRRqbxf~5 zboEqPO>D3NbjUl8&RE)Wlh+pQ%ydUThuyE(KTiuZR8XnW$WV4#YD@eDk$;UwYRqfM0+E&NqaA zqX_5Vk$wgCJvkSC6S5jHOx6%&$;GTq_<2T65lqO*CQE`DYX1xf$qMgK5hgUCm=}X{H@sT4AS~CJf?DGJgM= zGmNj6IJK;|_F5@(YaTLWX{kN(WJy!jDVbX=bopzoi*ci4v6;=P%rtY30B556UdBwm z{RVnG6?IYiaH$hlyjR2^Ui@*$5jT(Ox8v3tw5~1JTG;tC*PQOUu_W7UBp*~eWw)#D z+;h+Z0X=j8oEobL+F0_;uJh{NZ1KEf9ZveaVlYeDeuVKz;VxcmI9(<8MCz`VXidfcx{;e}4n+ z&%k~q)I%7b>Yk8zhPT@d4uRY7SpsX7I|qKncUUppu8w!S;$2X3iCZ4?F31110daF;Svx6e6h%imr z&2M6g64(Q0IlXy8Z<=#!#4KiX_^8XX*+dk+r02G75=?HYaAOx?XFFNLOfk_VirncN zb_gmKfOct;39Dv9H+e}62qJmbv|!YFVE`2RP@gS}${8RK&W!@;m2U{C)-Dx*Dh|<~ z?!+ZL7i79$HVL041=~syw8%)tYM?ny=!q5zJZi+Dq7NnBA$D;Gi^6A{9#kPYnF=7M zK9v@#vFUS0=h9rZ1a0+1*-UAwoPJi-E{@D8S95$2|rU#u; z6;LbjWVMUc)vo7Isbbq?ja`yVms=%lga{z65B@2!rUgl4)tc7q*cME1nQU#Jam~zX zR<6NC&t@lQ$~Wu3R~8_E^IUWklA{;9yo0*i(Aa#=@4M8 zzRjjZ&x_uCZWvJeJn?7WODq**#l9gf8BMCY2>%kI8A$)W2~X@vPs8@uKS3TVf~Bb7 z28&mMGdAxtC(N|;ewjTL=4gg7Y+=s+SVtW;@Kr~43~K@M$WLZllkL32DBCC$XV4Rs z3kPG|0otHy!ZL5UOvD>U7{YNqGn#?KV@HEh%~DD+cin8HXnDW_Td;*K=Aj$2u@=c< zm;w8W@LvG?61(O!0s_cvno76%po11Pd994hBo-QX4yJ5dALi>iX)n@ihV-yGEZWZc zSyUj_FQWL61MxXwHgG0gFPS`)TadtgKj;K7@N8m7mw^IVJjF;48*A3A0t{i`!@9}P zZg{g>4PL+_udP~+Tj#pg)ya3R4^6;C+t{IEeoX(4b^NtSKN{Ja3)KMg=&?&b8+_6p za;WE=6C$YZ2V&4R4l|MJY^|E4en^4?wg@Fhqx&kQR^z|Ga3vS>l!>&yvX-};G++lS z=f7U~)fydN)FhmFY<`UwTHpdW)Z1>%eu)#z_k=iC`{LRxwHGK}mS`rrYohRg0;qvzZuW--NU#OaV`18I z201VZKmsKbLVYtxa>xtciqRaP1}z}>?I-`z`PPsf^lFU5*=_eoT?g9j1$N=ImNJ>CYa0S!fjaw_6Zp8T12@mG0r-1? z-b*2svkDr}wh8b65!?U`AVCuhLEag_e#@KW962$IuyB3_aLEA>OfWf$vrWE)WXz(&mtSmAV z03t93M_@-_*(@r&g2t1X05HG!!w)|h1N>0}Px?9#xW{9Wm>v*-ajVBB+z=r#y6L+` z@G*e^3YAOaB?$;d)e^DR%ftU)NJeZcu{_9(7_grsyb~oPMGK?Fst|;D8^d>#w|Lvd zn8k;1FvoLLkCG!uDqw}3!>~_?LJpw3 ztkjS!M1d~4zsn4|8UX)Tyd1tVdO81>gpQg-sI@Dr;+!G@GzrCQ1ZKFV-HSdB@Fbs%oG^$^b+o>B zd>*F=qAW_q()5gc2j2yb_c%1i~wl0K{C*4r$F&Fw2T` zkOq8$wp0QQ_!rz14ju|T0pZO^>%gjT0lnk{#M1HBfgz7zdIEJ;TfIGBZ0hI7$Rcu~Oq z6s*=m2SQ-E^&x;$D^LU77Ct-(Bg7sCr3xi50TyrpE4|Vbz)~(Pff?Y-{5&F=Jc|n5 zGQq^TP8`CvY)=0e`ANq>2OI2yH03!T+Cmnc$IfIf4FLlBk*H!|0uohE$tVLrO(X8h z&e)oa(-a9|;DZm~zc6qCPx?{miU}Uzfjy{}`vNtYpu;-TmLkx$8L)zwdQyTo0%-i4 zMHH0bTvbphQ?nXV<2=&~l^QYlqQ?xXFM7o4EDXH+!|9YL_8b5?MTAoPnf~K9GVy)v-NkKQuCe%5$zkfy|8v%T^6j*et+dES&~i#?}HR1U0tx8?fV; zQpEyHj3xLYx4cSnG}h~D9?cYh#Dm6EMFV!#3=J94 z>C^%aupj@+eAcDVf-k5(Gnl><2sH)`B(VVNNGU@|Y-CP*jn$W>$(ThG zaNG!mu|X_?0e}5h#OPR8!=f?Vlw%FQw-Gn@J6aSFr~CMz@jP1m@qvRZmd0Z|tOQv~ zYgyPBOCnnoQ7gvU>#DfSEub=7kj*_RwwBI7TVr89fVvmRSi zv4T~#o7tmLTb!&YKi%2IBsfiKv_6!@g^=1(5?mMUIc165(Obi<<=S(-wyy=-r_vZ7 zA>04Xusv=(E z4P9*eOS%F+TVc|+r3>+)!qrwT(KQn`gK_e1zodc zJB1?O)RBl$EV$=)Z!+$PmSmXR_8kl^~7+3*El z14iEv6~ZrbV8?69=!M@G7LDm`C;QDR{51zc;9&hNjy;QElJZ`g0$twqYYdKqlQ?RLkR3lf6WMPh$&Dx#?nm65=8j-3LSB@H=D0SYTvZ z;|q)5wxt)UR63sHTV^^6ej%^zWna$S2=wDTEzLNt-alAx#JGzT_0xI|K;XxUTB7HXor4ih(>3~b-w>%c+ONt zR#XN*PQ2*Fq-K3qxPyjRCSFh}w&fd6SGe5P%@tvUK|{X`nw^U2z>yU)nKhYy>9cs1 znie2j!D*kqX`o()o+f1_wnPF z?cZx;X;AjA1A1q74xGO+2NO=iwL0K=Mq@>0Jh~d@SeELLzP$63%8;I!be6SB#wtuk zX;#?cKQ1tq{stZoxp5$oX{c!eUhMyO4r|=bt-)Ny zf;3d)K5pbrZsjKD85V$zHaQ!ZtdRg;~qDD{A82fS`E(RuBK{) z{$$?{?wQmn-tO)7esB05C<%yfnNw@#>`BRT=z$txnh229nucXR?-}Ct5ylaA~4e;LXOd{1)&yDR2*8FZ~X2125^UUT}+i?forL z*4xi??Ch!&Ba7L@=GCpwnRd7z#REs0%Qe|=Tc5zyE?6h_7CI8D4 z2Jwf&@f_#z4yE#<8gcdVajXS$zix1?>x3Mxz^6X)2v_p673KdKr!gqMB>wib0MBwA zS2Qbc3@vY2E{{V7Byv^Y2NGlCe0JE9EOQbb;3YH;oz_dxH^aCAp zuW?q~3v>jhFA4ukXC7Td2eV*roK*ksH*fSau5)a}Y8U7LA!WS@`2iVFfEj>-PZjgn zssg`FE|MSDL&w{`_ZC-I?5cXjV^Hf{GFUv)OT1gDdLEx@l6UkNOp z&GxZ&bS>!ZwRO|zI=}rWKEw1(-*rn%Z}p}%8E<%qk9hy_*lUm}_l&-AJcV}y*LZb5 z^^TYKM@bi8>}%@kW!ro}D;Q7(2y<+AG;MdkG)8jPW_XEbzMQFfi3e_+|3-@U_|sKn zc9-U3C$6Elcx4wvIBtn8AQ*o+OB5$YudTY3&xMnRIz2Z=2DSmP<7BZ@c@3;LxgM~=7(z@F2p33K24(=Wrp!h!Pu0tZ307 z19bg1a_m^|+{OaBT#<^^$fPG&yI8p*14fV~lDJCJtZDP+tDGYT=)?)_kR+F;hN<*v z)Tn@wGLOnU*ApdEoFZ`^GDZ}a%R4huhAP_iYt))NxrVK}6)f7Y1w68C>-H_&xN_%$ z?AK5lx4e3_A*-g0j|B-^q7hDK`0(Dui5vgUQiV!WV@{|D5MYrDQ?de;BRlR4*)!a0{U8`#135ZACu3XnY}ofdY>+zju&g z^XT}SPs`rKj+07?1Op0iAfmMvS@Pw_(`?|qyXFsOEOs&Y4JDOO?0St-+Bqe!JkjP2{$2y71nbXUxi(xPB$HT5!hgg1%sG~drdXU zEuTH60eg{AHbZDA#u%DNr0IsnYODqDnj5m^7)T+8+$dyfyP*bvg-I&u8+6h|*QArp zK^diS*3E@rfv#`|z3b0V#)+I?R};DbAtNEVA&(JTuK!%0=!oVN6f}yFLqytxWLRl!Rtt zjFg^!MC-&aeP;R~2$*)h>(e1fsMkd-=PZ#l!SX>J6ImPxU>VtA@5?gVS2Fsr1(C9| zj>C?g81XG)%&;c`KY+?{s2-c_YHohmz*>+Wo1Ajnjk`r0&cD+9a^%z{J9*{LsT>#2 zEUom+CtcJwpu1O>PR4plv}B(U;HfbCht2d*qNqh`!WaBE)#-y*B|G_hgqt75=%xVTnMFbn=vc>O@KQy1j8P5wD65d&5Qn^xB^&b*M1}xsnHU))Gzp2s z$LY@!G>}soIaxX%Xa+OWGNA9YxxlXtAe-WBSH3tnGz6v+gGZU-6zJqdSjGo=$UGP= zZ8@S9R_brMl;J-&Hp5^FvyXNt=E+cY&&DD0kjiW(5hK^oV&M-iuydq2p~w`Y)#L-3 z5e7&xNxVq9Vr(;En(bcsx{dzKgOm6i0FS~%QwUI{?TqC@4VF)RPLBbs-E~u3-=E;| zZrt4o?wZD(;K3nCaCf($fdqGVr*U_8cM0wgAh<(t>*f2KnVp^5+1mYMpTezM_ntcE zyxyO>5-O3i)NpWBsK^nAr6@K(LePGq*{5&&uX&ylCB~-d{aWNaOo;Frp$7r^9Wp&V z;P$k59JeTp>F?6Vh|&8p9O4k)QYj~QoETx}>}s1b#ot4!KU9=TBwkOHNyL>4DU%$7 zg{;ZW!j^P}xbm=)7){G@R>ZZFSTJy?8an>Wx~!UppFQ?MH4tgIvw~c$eOAv#80at5 zj2a9C!+o@^mOu<{wX&;6`V2H!J#Y=pUAWBDH`1B|Sk2P=83SZ?-*v{nB#~klTDruk z&rsXDD8r^|Q?D>4RCo=`>AxO(Y^C)&#jzXa>ai^8BCpY<_cjzSNZKXoP^_PGG*ib@ zIs7DCMnrONL13|T-V-p`SGjHNWd|iCHja3E_UcMJq@d3dtE#8~4LJ$Kcv+-%?$XCm zE>LlTC{DLf5>gTunV}$I3-nuSPkT{!X1VmfPdcx6$!ZW5Ta_5%EnMW{+9?S~e>%A@ zv_2Zz(B((a3*`chZc|0_U0d-Et}#at(66yCpxFWa;>%h-e)HSy!xdv%uhiHf94Y4GbiEd9&X26x4udYReT5>|pw zkZ*ifY)4AlPKT4;oHF-x53p1R8F|>?;`y>HoQiPb^OH}qmynN0Ph6)xJR2j7SZr87 zQVoT5-R9~X(o%g)pW$V#YMAJ-o|54Ywa;!S{4JXm%>5Z|uUo#R5`cRy*Ux<6`7vp8Gy0-wMM z-Jw$CF7yU>>acJ6{5l!O-N#3)pX2ufWHJT%ICD(#=L6Zv&BZ}A-60NERpUV&zlEt z_q}9qSAQql7V|vo>6p)M1C}n9Ss$KNqb^S8y5HoU3J`*(h2K}ah-W%dLV}`2g*knL zIWIC=H@7g?S&LA~Nv}eNx1wMeV?xLz$x4~{ zYeo*g7=}~@hH4{65feuq0?VXrB2AGY?Ss+Iw}@x}P(z0G6uziX6luQXm_<<-*2+*_ z?=>IDHsJDKIi4oe3kltkEJQr>T)g<-cy1K<`nmYQxdhSM`0+-B=|;d3#m@*EI8C|4 zT}C}z+r10q$87y5DjAsO8=B0y8sssc9tv`}JaX1Cbkc42jcug7Xo@m?=QJWRS(Ogd z?Oui*@~?R`iWHR5L?n6oSi+QFO!H_NDRg^((;`l&MHnzNj#@#JMcRn+ZSdF8^0*aZ)7- zd+N_<;QB8GV%$_xl+*%l?-T!ag{o4hXY)2zU^t5wWK|?(%L>G^3lvs&<@s}j*mvHP z{prPY8lsFFYjcVlck9=wRoqMyJbSUo{jfL-n=$sJq872{0gMZjH(G=}%6=5HTKIxA zYEp%2Qwi|)dzz$@GPH%-Dkceq@OW*X#h*gbOsW40%_MiI!`rLj)y*k)Ve#F{$jkoL zM%x!KUh~gH#KNKJV++!mEtFH#3NTE;ehR^ZGsjt3(A6*eB8+O}W^VK&mM#tgq?q<2 z=-D-%O0e#vTriq&jRJ>Rplip6%!n;Wc78I{rEIH?Xi z_DJjys>k9Q3>C^MU~0RPJ18D-icKfCnAx*`$YP6v7}6Nbt`K<2y8JMGW}p~KMTE0W zej*H=Z&1@vq4Peb{FNzg!X4^vMMvcn0@nj{Ax(3ZnuK|X@|9RIPYyIP&$Zn?I0GjV zSj{TInQIRf2Z~X&0wfU)MUWg7zET{RW=C5Y#R&3QD1b3()gsLn&HPeD5iOJOTWB;G z4?Wk%={x{Xmgau(TY>xZ=!|hH;9{%QB87rs6kjy7Z*u`q3<}Cv8mo|8C}yBzG96fX z_6}f_gIWTz?Qv~1W|1n1b~}QZC0vLN%$}a|fTGDnk#);9S5V8v>pIMN{L^Sw8oDoG z%`T0DP`Ke#nsRRS>Kd)?9&~Tm7udnlRbar6S@a| z#ZaVTX*D?|O+LvFOQ)%i6{QHkWVS9zvy$=Nqv109p?R#uPT?~$eg}#xouW_=1ma5c zV(c;|C;MUvBFJ&)W}7qCqGh|fY52a-$3IELVOi)_4kuO>(|h53SH?hUG1q*N_xG}p z!7Xo2HcYUS`60-V=${nlPn3U6%_+u231;%lJ61`>YR^72QG}$ed|(9oly|Kf1X+j3 zh0;PtFWUYHtK+sVcI~w#E$yX6=nF5(p*W`A@Cy-I^E)jcG$)J69`InP;G;bl>ay-+ zrIyiW9<9<>)Gy-c1E;r9QmNPuCNoc@Z;a;AkI#N?^FM_&G<{tc|{oo%K{wHH0$7kISpv`WQ=vY<}1rGqtas8AIWg$ zC3d@CK6cAhcH}ZF+LXJ8p0*271#3hf);8@_qwLlfD;3M~rt?@Bb}F}`^VdlYXBI0~ zuFrm>vi=E=-8??qY1K((*Dox)7gl2)j<=4WxvmQCAdmfF_vI{WA6Obtj3v5ENYv)JG6BPYv`Z5X|Qq z7$p#_Mh&b12+pDg&H)7PUIYIVgb-4L5CcL?sX@#EAr;pkRfCY5YmmD^D8n@mwUBPc8N*Hk{A3I7)1|8nw6v zY%Y+_~oE~w3Pg#IO{gRR{??On{AtW}-9N4yJa&)_hx zptN^E?GusrBR($J!5}y+EG#J{EV(#5sXQ_|D$3m{A5Q`e ztlGMN_;iL(aZ^i4d_-yEcxlr}`8%I3iL2_Et7{*sZ)|GpuWuUu(=;~KGBnaUw9@fk z1odgx>{a*7UhmXZ-{fxJ)X6}5+Q7u}za#3=(V>Oq|8Ug*j;Kd!ivNYE_eS3l_59g? zI_m!^sMr3*r?+k<-yQY#>pzJ4T~O~Hy$kA@osEAG^*f(lznb#`}=o6{m!Q!kN5vmP@fGx|2KU47IHoDzwznQ{|ld%&6KXq8;l^}u>SwXr?q4L z3!g6idIHaEw-mF#*_I{Od_L94bd|k^tiU?2_X9ct#TjdU{dPJe01KCB(Vl6^tl!K4|cZSUu^Y9K<>{Cp9|!shsyu4(jPU= z5o&Uja=th4rR*;8bQ=i4z8&5+8*pCqeim^>piP~3-O3yu3xE=%-wwpN?lbhmH80r? zCb-^-{P7`}ekX87mv<+O(*Hr$i@xb_D;%DY{!iqm(w3bl%she~y6TQps_5IZ2ddat z#Fgz>3HwkHN*V5>-2@^ykw1xQo2GdYnr;@R$+~sUW+^Q8h(!_H(VoUY^`g=uIfHbo zu?)u{M57G*F)KP|suEmhPfRdNyV$IT9(9wrsowEUx${(mrFezmDeajB+5$Ttd|&zns>^qtA1P`TUPYI zl}}eR93P)l=~HUi77a#wRpj(ZF|qIuaJL`TcN*(3)cs<&sVtuhMz(4EJ%wo2I%F)z z+BWJuZV%oWIyq|F+fS1#y*QTn$QsRga@P6=weHXrJZ4jq{;Vy>NC{0h=h(CQP4FjZ-(D za`^A=chpw~BKZstV^n_zPnj2af2NgZ(&dbFhadSHlvu}!izV-rO`G!oF zf*xX6MNn&V$xmyiHFUNNMoU!K0uoU*S3&Qf?t+JSGd!CALN)71NMqtn77gM>lFg+ zq%|wwd+tk;#~b3(KRY})W3xBWu9wP<(8wKs$k@WjvpH9?&Gr4RsgG#ab1?rIP6lV2 z1b{mHQ>-!8U*IKonZbeXJRNcb% z-8>~HhHyps^D;*i^eKxY$VMsKkSiiWw51tjWVWGq*GMkj@0wx!*s_XV!^WcPf$9*w z`uQ}Li`P$C%ILr4%&@M?IY8d}ws2%%Ht-H>P1wosoRnQs70|{OIg#{c-1C`y@y&ya zS)SsUBFeH+e|c01aH0dFO!Dfb2!a?qW>Dy<39%bNune4+5nnnIXrPt-DH3};>7VoI zYwJoVQZXWekY->^6QsG_90HMRHb<#f=vUUg$dJ!4u>`J64J@WAcSvKj0f>d@=k6<%?Yxv4v@Z zXB25QGckM_CV91v(JDB#ZsQ3dowi;VIex3EC_=nlP#~b*Pwfx1?wG$|7v-`r%)-H4 zh%)DP$_f3?>C4&Dp}MQJfuY~}SfMe3n$$lkUY)3qGBo}w3h$dCip3>jT7i0k5k6OjWtP`iUm~-o*1k3~ z<;G>1^%W>vqGAkyrg5|uI+Gb-^7VyAOU>8JAtLASh=*o)=+(8eUg&x<4LPSip`^kS z^XgL5L3YV9_8H6af4r^b+g`y!6c8IiYBDy|g=4xqRo?)@GgjJKQLgkuinzhWB(`SiO<%jF?wVOj?JS(W>nz|k*He7Uv3vE@8HT!V zrN=yX3^{2!(!OsSnPhR()7<#O-PEiRP-SR7KvXa7-d-tL?KQKuaod)Hb0%c(EZ)v> z6|EtfHZ7~$vmN_aF{ZKl(n0IDob*}pLKi~UM+1~R6j(Beo=-G2fsl2JlGd;f+@co?ABhYzQ)T}M=E>Px+?3e-fYMpc4Enz9m!cAYTUzj+uI zNcKhWN7Z#cRmy z_B#~k(Iz$2zY)fHJ(Sg1o!KL(&myBUL2^(3^?UMf@%Fr_G8t3XBy5A)A+yzY-PMKa zVAl#%J%IDX(_)MHO)^8)iN5&LQiDE7TzTUFuYI}mFH6(owCOQ2L}R5IyUAwk1k+kW zhpF-IT4<=&&VKx9Z40fr<&478JMww`P>QE|Zt~ndM`!MprMQ0S`@wh7*yZ7=s)SsH z0n@+jV+2$aGW;FZhRx4g=ZY=mOO=j+R2}PR4lM(Ns1Cu?)qYSTEkit0SJB^J{*a@$ zj(k?Uj?c*4&4cvws%G?GCtob@{Yq2##w&=`8A>one;3Q)ySNwj{BnS8!8;v-_AT`D z%i(&}Lyetz=sTZ2QjTexFGh2vP~SmuRSqOAnYyh~(gVaFmMje`-ib?}T24;!V5dzw zj6><_k4s0bI|(P+e$P0RhBU!kN8dXkXI+SEJXtiE-t`f`U4|-k>@XNTKw5BT#Aj z(;oUHU;5Ct_%nO>qagfXk@jcj4G00a{cUov|3(6nYBU7%bMZ1@4G<{Ciz=ZRDCH3- zZ5ep1Y;*Y;8Tb&`2nhOe_>(`?m+ZlxF*PWunH|H@hFID!?k;Gn$j?~Q8?lxeK{~*i zAi(C(KQBn#K95~u0}H_*nu3|a>)^C6C?1QPj4uP2782triO;`nrDDf zR3Ms2F}y+XykY8=A==Bqx`!d%&0#UL-d8aSr5=oO923u^G!yDe^ z?ngi$Wk?@vc<85L8CH52UNaK4`4HtwA911LFrpGJ%Vs`Y6LFe~cD@|_t0#1TH)hZy z@=iLY#4&P|K05Uv@{5uC%(8nyXw<1xFi<9HTO^#?+;w6E9_=v}!zyH(APz?>?u9oF zZ!`waDj@OJ)5SQV1tEs=G5YE-A|W*9J~f`vD&~1PmXI*u&5{Ij*?z<~7Rf3q17aCV zY#EHu8%wkjwQddCnl&HWE z&rgt~%9r{`5cDLP!a|Usl$*3&lw7&^-EcH%b|f{~E!p%aReCs~s3p>!(BGLa<^Aqu zC}D_anDWp4kdud$A1l$<^r?YbF-%9vS%h)no@PRHHY2exBw!OEWt3@;Z-$7l?JGcb zU&P#!gw%I{+~Xx^n3*|}3{8>2DwCMQmqDqSmG7A)6{cMfl1@>YeoBx^2YF22)k+Pq z%EHmgP@woWO8A{J7B$5)L4U<@a2ooP6qK`X8aj{7Wf*#DO=crS8hUeVF+sX$Sk}s@ zH$pntJS4c`L9DpdTCO#HD=nh(D19$2Ufn8bP9{Uk(@N|8LhN)#8v{^BIqI)x6hpeB z0S53r;&RK7C$g2HSDIIwj9ToT$M=*8(-sBi6_>e^`)L)7G8SCu`Q>;vWmzOw&Kl)p z8m+shVB0GGk5=JV*;L{(*SHk;`N!O3?o2a07>@7+I6@eHZp(deCZ=-o`IciMgT3H1 zfU?G*9WkG)%wCAGfI&J1S-ZeT7GArpP^8TQonJ`yDXWyQP=&ux1EP%({S*<9R`_A8 zFflyoE5*0*73hl^#Hq)O0gbf%;8=56C^I*&ALQUO^H>U^R8}Nd@RPmgYOFjtjN!X^ z(8|*7fpr1EwQH9r@Ov)vm8)&Rt&GM|{DDyc!wNizQ;5lsnrd1&mnf{F&99;v)WA0N zp#8lph_KF88$;{_RBaQXjDg@xdP-x7GV+Rxv!-DbYJ74_v$2BmAXY-+i-1AGoa;1u zkshx}kmc(tfX=J9`mTtpjDMgtv*0+$o~WGAiU-9}IrK5jaM}q25aQ2YkanNO4FC|~ zLwyD4ct25VIPwq%xsmRf>if$3-HOdxhpw!`w+F(WZWU}mjw_W~>gwa<$1h6=ty8}) z`=@O|!;6G~8LH)a9rZdm1&;1M}4SA?hX-(i3pT@67cIc$LWR!o*JfO_oOtKIsxcZazQF2Sas?e!Lq0c1QmD;61UOt z4;fBL60oCV&AH&XcT-J2{pAW9Fh*38^i&=!BE$uO;WC5Ko_Nyc0|_z@ScNj>Hd~^w zFuKGbzpvi2e_O+B3qZ?r!9_pRi=5;W62eK8J8gwD9PwHfm2)!&ql_^?$E2k-TiKU0 z<}rcI%)_gJ$W1Wctx{!)&(eSsVB=&RVD+&(+L|}XIKWy*vH2;S!WZzNu+K#xI^v*8 zft($70d`iP#3w;w3M}Hi%kmqiWtyMyqYVlTzlst@_lRNrhHUrF7$U++iM||6#&~Ns zQ;8=rAdkxJgKUp1q5Y$U(34@N_+7Tja}xRlDkk%go_Se`c5I4UHCJLO<5#r1QR`M4 zD-#_IGh}$&j1Gp=?wW@*Ph@VK9;p7YPJRfpry9A0?nKopgXE$K{AURO2;9{`J<9JZ zvBohFoZh1N2!jWJQ3soFW|UkDbpPgWP5F)%o!Nkm#zX+h*4`o-m}= zDPrC}+M_&fh7@8x+6u*a3}rFa#!X~n7Fpz;`F@MJW9st<0YJy+h31BJC!2`zZ%hHp zh$8+YPgLa3`$dcD`$jOc7&EsYV6M(HZL_{y@Yi?E&D=jV>Xsyj8P* z`5rS7mPa_d8QGyvaeNj~0mE2{1KS{FBSaD9R3OPP9Qe8CNmnLPMG)~MHJl1uE)Yhx zd>*RGTO8bxelIw7hMZddQX$4Ct?V}rqlBGIuYM-jy%e0hXWHYl} zl;E^#V#(rXEH{hrm;(r($qG-Vv2_Et12zIDEkG!%y8a6d$Qa*Y@|hBMCqDh99uX;} zdoDmc=ujT37eTv7@a{y$?OVV!%)a?Tg=fR&ZL|MQUhR>G7>z`kf>svn76p{T}HZ& z=RVo_FV~jmn?SR9&%tWPDML8sstn8NghuRfMA>cyGYY<5;E;Dwaretf9xE75TdiRq z&p&)vt!N$l6TZ+zFb%~r?!4B?hS4@^<54H%=B~LWX;cGq)aJaANlcm8T(CHHkD2JI+63h4_N04)Lql!Fp}{++hjxryTM`2!(CI4i5_FekgWvVD=) z>X=lhZ<%+0rnSQsnd@iAvMs5Iw!nP}B zWtOT))^xgum5V{hVHA3nzY!a7NWy6Np7OI(?vH8B6rDby6lwxi*UQo+^;6XWF!kl# zQADNnl}n!ur@1>lh{zLrJrXG zpJHG-gCAHolgqxh}HF5!=3G>sU9eS!vo>YM(ImLLS&Ly&Bicx1ipVMR4kic*@jO4@dtSQS z&UE~gW-LGEWHJ&*)}7A0!fpyx&+4$h?#afaS%bt$qqlk^ZjL2hWxl~rJ&;yw)0U~X zb_by!^t&ZroWw3&o9hjtBWeA5=2T@K_mQWn+Qa4Wy(jg9SVP%ivjg#BD*E;Eu5Ys) zfAVbQ<#8x&rutsq=i&PUERAkN2~oyL#Qont4j2;&r)y9+m&sc8*C%aPFygW8+|l6| zq6j%A_8*SuVdVN6XiXLQP}BYl3x$lyISz>TP&r9pE1gz#j4iud;X)fEVsc_oG{iML zzLYwABK>|-oAnLEXWk`-025mTjFDid=U~Qk7;xEnIFn*@!U-WRXPjpzV`Un6*P87p z*z%xK1{-#Ck=6PE`jf;rsbWr90~i=jot*Xsk~X*ijIX z6iA{xyNG+i?4{2ip9zXj{(T8o6+XD>xf6?v&Mq-2nLgM~fV9Rszcat%B3T;DoPU?|izb_9*mYfSVbkkpRS8Cn{&= z0msSyiC`V@IXmJoeukWSPPdfwn|_*&{r4G%b;ZAOt-XCP2c_7nX@+${G))fOG^6;J zKcnnntxB*pyi?5a-8+OD`4&XKA~QC>I8PW%>iPkBFJ4@x#qoW<&0>Q{0Q2b5dcF%f z?ys(ksFr$&x>N?l7AYsmRV?eZ-h>KGt`4jYda!gWZMAV4q?Wme@GYJPw_God#pOTO zy$UJ1emozCHNSQuur7MNMAozBqyZ^W32pBXQ%zEPA4%ITIPgkdpUbLkbsnmmo6~;& zw%dUM2#$AtU=IZJDEc9r8lHZj7g!Lj@x%5ehh_2Tg`-stAf^_n6nud6qWQ-LP*UeY zi+S{6MT3IqO}C+J)lpwIMyz-9w;v~pxvG7le8{nNuJelP2&Cxcx-(WVqNIB`FlWT! zP-e(>@jH8jFvG{7p%r@^0e@J2g(yn^D82o7LB=uP<-`G;1H3~gNo=9qM57C~9^w^) zPSjGf+h_{2;tcyUrkgq@j)%bmr7#XZWhdgv5nDgea+^BvhUJiwj7kf=oyEl#n9^J+ z%V;%-#i#6=(g{7v$c_EdwG=dAs9fSzedp5yTd-u$%ZY@2S5aIOMkID&!wgVYN&RfQ z6wmM<4ZILzd5A?;w-N^#(8^Ut1+(c;!Glh(5t;hNKUDvKyy$`YOeljJ!9B6B#Mz~Y4dB;h$;j#5!l*M z-C&mtqzEGOiNi_!R`zTPb~=78IAWbT>=ZC{IxC%a`ll(59GDNKLXTP{@*@7=nM`ak zttTa}RE0uTGltQk;pxz*qTFcWgID3Fi`IF!D{2gxV2EkMuYqCrbC7LnE}TNk+&r#uW4e8~MOTdAITc*&p|CZ5BK z)W}6Jd5AWs{?&kGp=uzFP6%0Gq5lrufDyLB$P#HWOK=VX)_tW_%;(0^6`7aBB4M^R zFV%>duPY&2JJe9n;qG-?WY~Kj%BjQj9>AI{^9nFGIl0#C*>Nl92`V>(E?fyM!%Oz) zOy(5+F5XpDFWs!k`X%B?V~tr{U<-9|QVE7pecejg+Gr87eC zER0}%>ndKOPX)TlflAzHd+7S}Q0T{ytZ~GB?&Fa+$f%_iF%C{e*Frf0-qnu`BK5M0 zFga0#q!P*?bQWlWaY&%A!Lc!7w)dO!spZ1w-0UBSq%@U3lWm!00E+ ze6{_QO0hb32ZdrbqTOy}28*f*49W&D^7*Dsu#!4rHQHn<69*1WNVxj1c2e>~nwqbw zIEpJ2*eHGupbK4sE72Q`Bxq8iWe-0+bY-*YchFx}AjPmoK$}w3y-tiN3o*9IMOyfPMtZheP^u@0~&?zrpuBVdO`6>~uWQ z!XPW>hL25YhM~O(+=c;+(W@H?reet4(@Gk8@vF-VTZ+v524Yx6H=sl?jD;tXRV=3L?9Xzg6z8twbfUv zE>S_Cn*}5ud0qeh{-7JsU>ym(dBS?cSZkr96O`G!0xE>8b*}wXYli<^1pYN6@C3+_(4N?>=QOfB;*3!}E5 zV&i74qSOgsLlMb`Zh%Bc3mNe?KqvMe9Csx>(LMO(Pz8#jWODZP)u}LuI+1sxNkc*S zg&PE#n;Ae~pv8=lJ2;lbQ;`V_`JmMBIBP`grAoIY@9+`{^HODu-f2$S`!g2 z$M`HzA3^94ZD>GCevd11ISNW08}h(98@x7yKpZym-buBSE~S(jkMKahKOr39g!l(+ zJo(H3W*pxhXqd#-E~&qM$1arkjDy79q_h=D5~{(PyS`=~l2aJ|P(8kr8G~k{L&3|f zlh_p4;4J-tdI;BzwEgj7Kfw_w&ws7ZM9e|}7r;HURr%O6fWCYIrc4oT>0VH$N3 zr4K^!r4E!WsU0GDz=3y!vh$MGwnbq9kuZP(v1*^Nu*H94!x_aUdqX1{B?62CsZ4aG zP34DxEq*DQJ<9wt3%!1@S~5v;O&cej@Qyh(RLI(Kz(s5RSV0MArZG`Uh63tP*8o7H zw_mZE*v3PKk)!{79F(ejV-hiU*~@1owMNaC%%up}zz!5_dS9XvIHC5*G`Wvh<&$B7 zz+d9x5R`H`W{e(S2WrhtE93IuOk1_GGGG6E!ho)Pb$?5GVzFaj2>WA8(2WA-C6s7| ze0@Th^Lp$=L3yI;XX!k@gcm?B6|qbn6cIgeab-*}Ocq~EOw3Rr?1a{4m(NeFyGWE= zV_lFWHE@VHF}xlZCLTZ{lG3^!s!HE!+bb8X1~+>;Ui!he8)kwpY)U#~YBS*z2PbMV zkt`l@7y8raDQCr%ciA;e_FsvR8x#~;$6S)FUq5D2W(wCjFWp9s%Pm;7;M*?-;+!TN zfuQN-tRXb@5r~Z%j}o$=zLV?h{y9OR4y;%|;VdB5pkiZBAhR?SUZ?zF=**t-{88$R z-j`ppBN9Li$*SEMqY}~MH5QKvB}6sZ`ePQ_J=$-nP&71hEwde1tFjK)eFd7(7_t9w zYZz43KMm-arvU~Sigu!NVjw7m3_nH!mLy-*F2gYbdZM;6)!B#EV8cb{S>EeY&inx+ zDOa}7LA0#OS!9WUOwMDnENP_Qlmsrn9G$2M0F(Jb$>*rME0D{R>xz^ml~sIl8P$l| zw4rdzaMjPSu_EOBFEzAL+pa0(V{@~0@yC1p<9fH3YeCS$uA(c8Q%Xk0e(73u`xqZae>%s)blv9!n(-c z8cAd}un2L*SypH8=A`d*y54^?PuE*HgDQ3^iJ=ay%N)hSw5}Iw ztX?E*aNC$XXr}uo3GpEDH=}@oovr@voPqJq?>Clipd!i{&4<2nNmcc&;Jo#)x*B+M zRQ;^2L`O0K7^b~KgSS;(G;V?MpfBk5t?n5srpQL(7{9N4QYrTph?CJNN*Caxx9)v* zmgVZ8`~=C-`luIiUpklZqx2I_wm&>?3lJJ%2pc1eSMcW3!o4v-@a(sjKp}TQ+ci!o zZ1!tRT}BZs99FN|__ld-C~Tm`XudbYKnjx%nw>`3KQZYss)jq;S%vt)JEc*(WoIS) zUz=I>cgDF&g@lJ~5`TY^id1r|IOj5=C)vZPGIjOvEwVGxj@Wf|HqZzb66MhTjIc#` zPN=px!5mzRy1Dz~*aR7Ej|OEMiq`A~w3{Wii>q%oV742Y3O}Jj7niyJ1>>uk;qsUK z_x~WZ*;f~ zrW>GO;S0p9(}rR&iqB?K)gtdBf6eQ(HOqSBBv?$ zbFk@3y?$tyq1u#Xl%jQ^zIBoPv3jkvaI|Hq{h>~_wMm>+bltIPy`}2Zk=VXz8d39n@ikH`k>ld`KK%2HHla?xYV3|#)y-lTS8WO^$RKXEhdL5&A zwy#^Zl~*~(8_KN+{NQG-6`_ru*&nd*iKEPLpAMj@B<_q zIAxlFIhGWSFKcq%7N_5PF1*Z2P|;+6R&{*3X@7x6(}GY;M{=>nZw6i%{OthT9*yy8G8@QbK~yaDzSK;{Ufb3Eg?gi>;Z`E&`ZaS8YH z63LnZF~l7(1BEqo15Ig~guC5jU zx#RpI*{+kymgsl>@syJk@y?(klPRr36|HYV)%vKt(f!xuhkd7RHv1OD5WJ-e`rigP zN!JYB&TI_9PgBeI9X)CKS}JlDU(3$CU{7Ft4j^dvmjSR&L6uR8!LoEpPDlFl*q^VQ zitKmn*Ya4C&uN{)s}9}X^w1o3bsML-;ZtPHvw89D&Q`BaRSsEy{S7^L`DAgu@ap9B z^w*2{8@I43_oo}Efzx=;sT_tYv+TcbQm)B&W{Im;b^I1E0k@SyH;B(`BHg!S3ASPy zf2W^r0=W=>AULgWiHG1aElwSq{I)`?c9}RcV_M2G*0y|adxRw4cGMi1E`B3lx>fwS zs+nv?gYWuCi=-M;{Jdydowb-Cx5$&@=C|qsq%~(Bb~T@I3laOvvy)E4s)_!2*?~;v zeFU-WC3WjrKP4IS=a}$7x*DgV5LdELA_W@=CZ3CD^SylWeb1iTlA|9Qy{foI0~-kH z-`uuAcj5LBTC{o~g^vgg^Zl*s>I(h{pW$%7KKqzeJmA^gk%J!v{*8{? z=-bX&7yod(r`yRm-BAG`|Gm5!-pZyt#o4;`@-j#X8iG=DN$vOe(+fLrU&2Cf4Vt!{hil; zIZ?0(4|%E6xX=rqo^}C}w*cBXpCEd$_AEEPoi`)=r%uw8l212YsntGd2Co7a##ZN!jPnAfmE zt$Iu$y(bC!;pReJ+I%w&gP@zEiAwD{3NF=}IrmN9GyhT-h=r+GVd+E(^vCCt3UrI5 z8q^@?!K2m$lZ_=ZYSh^tUDHgYvTJd-aI=tbyKO{Po{Y{ZK_lu!H?D=v9dZ=N#^C|J z+ThOXE7{-@fT)sL)zy`FqqN6PG-}F}h3wi6ron<%Gud2fA$FPam6k9FA|+V5qBgge zN63g6g#ZqCY*t%M^EvlBJP3#V=>yyDNQ}*F0^K9~{zM{&%lSl$s8F99^qKlVu;S)? znZ~mMsxKE|N6F*VqFg)Y`FfLqDilil*4kR1YjcyG<&E?fbqbT|)L4a!=B$vEJL?1J z?sQc{)=)}KdnR+SY4WeTtbZWgP7JHvudaPfrOcvPBIPZ{$|qRqbu78XPny%)Lz0-E zaXh3PT^f{AsOvD7R$bR*c#VuQJl(d<@QhowO_Q7+wk>i|gFjH&C3>VYyR^*0;=V82 zRs#lW2iF|Ca{b5&?6r(dB0*46RFRC_1R62Ic{C!CMccw4MR|r#ai9>Rlb20Jz;F=(wL9J>X~o zA`5LNOfOH%l&~o1oe(1Sw%q&}FSvBePSmkvba;ds1|+pzm5fNKKG=dtwGwbCfCynb zuS6)m4IesVvG9FMpksS2jH``NIo`ZS#McLL_cHY37j~XP2^79W5%nJa3=u*ZYX+LU z1=-~dcjMrLAXxLdVwM(J(a&URgr_{}8M{@jRXd!9w?8Wh#*U`GgZ$gt- zb*?*pPSlF)$-z8bkJF>i1{O(D&q|KaAGTFzeRk{1HxST#3qYl@<-e}<7uNTfP*Vij zqJ};o*)|Nt2UbX?LdXX&y0L1+r^Osfxz7z~QExeYI7!Us&(jtfvG0yRLle-PeXdK8 zxc8$*^PCNb^MxF7_l*d@JGVO$w-~}g&(G_RAYEm00JghTZZT>AHriinN!v2X00soZ z@`KnAYY49n{&z<}j&@UXebNvsj~oJKs^q6{%(;xn6D!jn`aURllLMIPzrOb)m`}8- zY2|dnFPOI)im%WVtg!}v?sEhaPTK-RmWloj?6AJBkSoFL;={X?lw3%{ec$+Nw9&_t zaG)nEZM2MYW}nLTW*3f-#%m0S52bwF4*qEe%!CV+;A50MmmPlu)~gh^?bbwFbnGNy z(JBKtF8lB;c>fR-1~h*tLe<%IQX+fVj>(1j6{spsL2gqk2K(OY{-~yba-l9NVz1Q?7@Tf2_HgpS>E>in+J8f1puT2_g-Xd6rYW3zMtQq&~I&mgP4D4E929XKu zl5hLf^G!Yfqg@8C4kM^jcr1a9!r01s1Q5~cTr7B2%TvL>2q_JtClAZbJ00lXB_AR11vhK`|XH=%U z52nUn3|Rb)e@UEx6((X*LRly&ShsUFyl11QGgLNH2kQ@I#|MA&`tLJ_Vjy@CmswD= zoc5~XVR;FxO}Ye;%570i=aRRCZit-HwbyMGh-UciJOdqlyXg!2tAiZidy0Eg4~jxk zu`P1w@}Z|+Z9yc9C}sq{LZebVBDbPwzfCv&xfrX)ZJtLhqc#QxIqaGq3B;$T);2I*GzH7S%$={mp4?Aaa-ZsKH6l`6SW>L5zv@mV!zeYYwof6&@ z@N!CP@XX_G{Q3&FihYmbk5naDz!9Lsc&$`ZN6Ac@X}jz-RX0{X3X(K!2IL6Ej`sheo{X+I>&npfkluVU(M z)x7fDFUe&?VDaw{zjX^8{J71s{glA`?}lz=t0iOl$6qCll`(QWEGEz##4)PcT=E4* za?kHP<$R6_o|0Sdg^~M-H{U1yOl}YyqYVpXTX>|1Zvva{okedQMjmxa@f|$#!Ya3H zhG(|MO3!;)DxG|ziVyOmSDOt`#B(DWb^_ZT^w_)p0?Mg&&(QKl^kkit<^tEH4P3i& zZJlWPboV)`r$TrH7NEa|?Jw&;&3LmO=R}lOh|@i-{kFeJ%0W2;xIEckk=_=|!R=mh zdoDON-sYokuIvB2-oAyms#SXSh5m^(S!#dp0xl4Em}%khMO?DYSdPNJ`Gpn1*2 zbnPezbuN6#@glLJ#E#GDkkz-F()^jZ>t4>)UHt~G8gI_(C>A&*P`aE!`gC0rGm1U` z;Hw2&_^9P3e3Sq2^+2`zX|Y@Q4p{kmW@CW3#Vc%lll|u^q5EYQo{#;q*5Ge__v?Of z=W~BH%wwkXXbbixe#n7z#cMC$d&s(&I?MG0@q&^b(Q>` zPa`~RJvkX=IVth^8sQ~Fv#a$GWE-LVFhWP7;GfXL!lA zWKmM)OFo7?(V@|f;$9vcMI}(p6)O^E#NDOZYv$12-iK&`P++a>EwD6pa3+p+CtgiuR~- zi-llzm}s~7aY{vq&F{ubeV>tx$2^ch7Fab>Vam!0V917b>;i+PMJ>r zfzGxWUA;LkaW1<$<>#?7JyA&E*)sjU1Ij{3$$LuA?xg8e;U7v+vyr^|}Ib8s?RW!bJJ+-YOwWGm^^lZ5^F-_O^a(lsXCq67E zk98Nm3iE7zXYvYn0~&iW3x{@dH`*f?pCeloVK?eY>uhR|d>S9>Be(2Awjdhcz9WVX z8b7X7kJ$ac+L$Xkl$`A+Ih~*?8^>IuqFiiAkBtoHAe1)SXb(9BfOnYT?&~X|RZL|e#)ID`H z+ff`$O3VdqLV8wUzES+tLV^Kd+;U}fHaXvIWik`kIuW@lE?yPUF^#rPXR6E?J8=)L5?E^=jrKsLiEe4}G|1t7@>rNoQt z+XR*HRSQ{Iox#QbexKlXp*3+>ZQ((GKfuO!0)5w`FApe1H$ygGq?fXyxl^HGk$`Af zovtdUZ@Q*&tF#tjk<)Nkui=A55#TfBqkXI|)%?yeqIMsCZ8Q`2!u zV`7sXv^9u#Yt6QP+5%0t-x~Ryp+(v0fT?vSwq>e3 z!RA482wv%{N11u4F?E2#qciN8s!1;{^i&wg>naD?91T9#C?YUU5L+6*t%bweW+GI# zuFk;YX}y`=>V8gz$+*T zQKyOs=g)m$ z8PsJ`H7%^c$g;LC`%8q3QHmafxe6X+JI3gU-U^#NZ^UxQBS@*qPe3WGnq_4=dn@&b zH;>2?R?9w_l%-y4o1e?9{Xso48bI|^l=uLs)~0GFB@3YqBaC$=dZ=i9Cg~`HV}bll zIOMnW{xzK&ADQ;^MOS?2hN@wi^@WHAb6;@C$*EZ~l!i(+ebx{#SGM5=$7me~0PUu& zyd}JRc;SX+QOP$5<^iB{WX=nor9}>c9m2GCFJ`Hwre$GOhH9S;#PQ@su;7BA3zXm) zo$99!SkU)h!krM~&uZ1@&lRtjw+1{<&Qc}6|88cPB2Ju%^e)ByLu z!cKlDZV$cmb|E2tKD}6*3Ib;hU0=4PTdeh54ec63dpGPpH)7xXJc}1eL95QwEGNty zM8j;dvKHyXVfk_>-C8Aobp&jH}7Oj$w98#XGQh~SE!5sI-d%*S*iGo`>7E&24CG1R& zi`v$8TBDnEw~GNc`Hfb&l2*kObDhImtTztj2QiC!6Q#>m6%tNW8cr7+clo6SWvw~w z!+A=Vd9|fhRl_zV(>6`VyZ(JDHEwryNltB7PMwT49aGo;XT{F>Ua?>Ot=I_w?*cf~ ze^u;=A0QDaAdxWsgMj~=u%i&XJK#tNF#lDt<70iG!uuZtoRa2W1pMEG{jUQ~L;H?^ zGrTA4!lKN7D|RJz4r+FOF>YSozX-U5G@qolz`qr{q=dMzVKNcjn)C{U7+ddgwp#dbUqRa!h4=$9uV6-8Nm@ zP+Q--)L5O;+%wYB-qAYT+CIMDIr^9X?wncZncjcrzyI3b?*jNc|GoHo;GO^O$a?3$ zhiCUkrdR%!>)*~s|CZ~2`S0=XYvZds|M1^;?*jNl^FQ(W^3CMR)70|We+2Nq<@)W{ zjjgX6=l{Up+yBYdcW(cd>w6~)n`eLd@6DUP<@*0->#g5b55KLi|FysOHvgCZK0N(? za`^vUt{<))y#8;z{w{!@9{i2h4{rYU>%aatUf;Za_@93LzvK1)vA_3j?)JyK_Ky$# z#_Oy9^51{{ldx}gAKe}Njo1Gz*N<2J^52jDC4ldB|3$$6D`EeC3*bk6e-ZGr#din% zuK@mQ^Zx3;5b*!siv8jr1pMY50spsRANyYf{N>kQ2mJqnfZw4}>%Al3Vm5amIzOHy z6Nwl+W}E4Wraeg~VzKBK>HiA?PTiHn?Yi#Pp;j@M&*%*C%KP64c+nn~2-z)a|t!^(^V!DBM1RNL$frujS)%bHL0-Z)v9uM*z z0mqffXUuXr_?k*YQkDAUpNf5nQCY@>PODO_Ky#+6?bk{rk@OJf2L9Pbht7}N16ib? z&4G}Xw~swMP6#!&G=>fzJX*G)#Q!4TeC~^L))nis|5og-;PHI@o~N6`s1n1uJ`d2B zu=TG(4FP@r+X*SFx z^1sZx`SFf`)9%XdYe*YK6_;8S`(e~i{G=wx#w(0D$7b1$<2kIPh~}t1-b-lxy-JZN zjv%w2Bu&jr94kiuRFb4XY+Vwsu4#SXrp#Amp8U~)*DS*zoUSa>%mv>(POH4?Fx$*r zr#!=g9!o2EpOkN%{xG(9@8Jk#UUIw?9m%BH$-9c+DC+nW4LU;Jg4pe9Sc-sY&@ zdzt>M(f_dPs3?ZAhLLgFr@H#%{5`>WtD_*Iamv^-9evZL2fs;sjI5j)kq0fCTuWDb zv_nZBHT^|*F3+#?p4&Iri>MVgIhZb3KE#c_ry+*&wrOkbF*cG|;(>36FISTS z4@l`rCIiYtuta{Kp?Q({+H-F%<$)DgX(L;Mc_mDhrsBCb77+l%04Z!RheebGLOd@3 z5(k~I_+~)x%Q=^tq-YSpmg@A-6v&4~{3@8jCYplNuK3+6)`KqUk#QX>2J|JFnC(@@@|$itQ-qjy@vJfc5P~02Wiu(69l@^z3-FK-BqPN4->fjF-rx@FtL#lcEm8Qs zaMCd2-)0-5UO4-0Kvs zs5J*0|H;8Kht=ATs-|wf8$*~8(vmNoGNBN7Bv1{s9V>%MHb>4tEL|3ER-;B;gb^TC z{*@r5i;AuvBLFeL2si+B6(+QAgpZpgzzq$-BD-Ob|2P!F&xFi9H4z5gU7qXS^jrU6 zAsch0gP7JE1SA5;2z}Ta4`IDdOLJ92R7n)q6Zq;4jTU^LZA zG7KR!hcBR%mgT_=DpjXste9ly?qo@>qF6+6JsGwNIOR;8F8y#Q`Bg+6-EhD(&pwo8 z2$@PXOqH+L0|9?}y;n{1+qC86h$eVk@r_xadpHk1TvP0azN-6>STJnIA=EHDV-}uQ zFzWO1s4othbR@-=cqg4ujBHK7GZI}>GY2172dOA3Yg1(OAgBMOu=qZNNrf=z#bA8tkXWpfCIixK zFT&PMaMtse5b8&Y;C7C?m$}@?M19B#p`1;-u#Ncq8A8Xir(JKR%6kJ-61Q=HM7akmnZ^2<<_vxo($ zQ9hhA_9yat#ZHRS6v@sQjFRv;LJjj99ZS!Esye*|IUw0*C8r7}q;)8e9>lXVxaXO= zH!h6QoDe3zPe%AOAsH|bpEa03PO&zr6z`IfBLGh&_B5qBcAfeY9m2iwu`9XsH~nq5 zD(g6p8c(}MLd*0a*U}z%63MAWV4v~GnuB-NExN7*X_edBdvz33XIG(nCB4Hi0ec7KSw)(9wXS}<3O z4u-l0-k#bTLwe60eE)1DEOUJsM7DEId9T>(Ih&Y+KN($Pv0ueM{J|OEhFJZx{W$v< z0l)P8)BOX0+A#qBbQMgfw+)9nZ6P?KSB*5n5nvGICy_Vllf}Qaci&dQK5Gl_^s@K6 zNe||oq>qLi|R8MG3#of zM0fz}(l&(lVi3*mZ;T7eJNBv1xoXL0N#zS)J%s_9scGbK;;Q#5nB~4XMgJ^H_&um? z0<6 zGx*SJQqLXxL1V?Iw1wDZ#&qijU(0939QUz=Q1~DZz)y0;67l^@{#|`S@7Yp5%zB$K z9w776nPILjY_C2+7L>KGP2wFDOjeRpZVC$1AAaJa9=1c_9&*xC8kETrd$&Fy(v2 zP8&Q%>hEO%-y|IH@iDj(5WVZW~d}jX#SxeZ&IjhW*9P0m|tQfnf_Ju*VGBBFQMyLiI90uP1sC{oI!Sgdpi8Xb2;(7PZn z1?9X^i}xd1m`Jz?^r@5|C)wD#G3K60Ejsz*M)}5(`Mn@`a$Uq`v|=0D8;-4`LT6Jjkh7zN%Q8KI7b)DEj0* zHp@!i$-Q4oTj+59Bn$j8iWF&C@lX{AQ;l$F1fS4Z_!^OZi~T_nEDUAguIyR`t$PKd zCtRf`>@n<_=@kJbga+OA#beVtY`Y37mk)aEAx}7q*#}GbEtN%2Hs>6z=6wGBDN@)C zpxl%%oKG^=2n1)9op1M?jhPK4!-B-*17|e=a?~YS*EA1SH$iI5(&|j>SuPRjLKR3f zzB@^iUr(T6$u5=w>O=)9bV(Ltah|Gy^a>r(07VnkRW7)Y0G0;e`--Jnk#iLkKFE>6 zsp>Jii1h&RGmBirLG@lI%1UBm$7$L3LRdr%XvCZvi_BCzp2m17x!J_>4*XgbJOM?x z5^{`+xlU+em{RPW1PD4*9nUQMG00t`%5|IEtuANEU&i`imbxIH>|Dt!j?nSP{J$$21t1pNY{l6;p=3IagC`wF&2T~gJujm#WnT|?`6f4b}E)Cf310Q-kRG^=j-RQV3E ze!uH#gqX$>mIA~(fIOhbzq7{fv5p|IoU=xbspI{-R82&;Dyo$^5QcsAAp&g_uoa?4 z7NSx~=F30yG{UHi{@G|~VHw)4@W*fb7_pzRUgG7AAOuLE`!~u=7=nj< z+m8~}ah{kN+ns7pa&4>;kvcn))*Dl9yry4j`VdFQ z>|>Q+8Ny2nVUZai47=Mfa>gMC-e-r~zve{lWkS(E0QJ>*abf?0ewGk^~5yd=_8HWK@i4Hx1WT6m2X#48=^c< zh1G`fGO7hh%`gfSO=L`DJcLYI*S&(D0B+gC8d{~q9!mkgOrN>o=jroivPL!@2_`8e zKYmwcgdBA|X{ZxclLUQL_4509qZ&Nj&qM^@kkjhI(ukZ0;mYt;Qv=o(30-k`jD&&$ z*FSYkuZH+F14}O1g=mn3cX3W0^q!vkdcVOljCa{7(;z8#wh2ZXyrPWpC}YKH2s2P6Ku$k1lm$<9HIOtkdfwr^YuD$-;>Zr z-3B@<(3O<{t50yULHlbRVj3@H8Qu_4_(b8pM$<%b$d3N`&gT=9)+=d7YSVrz4RxN^ zvJI+Vivy8YSzi4_!X0vQS1-VulRx|`Dl?t86cVpC8O+7-Y(CG~^FqrQeRr~52R{dx zUP1B{Lj8u7AIq)^Uriyd`GoKhpt4j>7qyY1Rm-%y;RUf7z`KdDv-}mUhbFpRilEpZ zX={>Ti^r)INc0`uXG`mKF?eQ6J8Ef@Z%vfXnyzLIxohMy4^q(RhXB-%GorL+WH|nV zxkbVlvV2%r<%VClUw}^Q+pCM{)liKLfyI;q)x2Sv2R*DzP?)U{=3$(+=P@V-o9G6q zL%uz|vOSQXLOkLi((Uy~fnC<9f}g#mJifcU*@u@sYpCA=ou%C&Y2S^FU~XXm9~;$} z0ZDJKdx(t+XdvivL@!vd0_al}wC@`*G@z)RP^-pyIkK(W(JvLnprCiP$nm3^Vr7v)w=CHvb; zj`0;u7pMDml@%Bte4yaGo}wF(H%!r>-xpwqh~qc1pvC4PAM4&NobUJBRg?M0jkqVM zKp6FTjqmOU#@f(mvyFO+?e?_L0)Wm*+j3KfjC-aTRsR{-r$}3ei!PX7Elep4#6Cwv zm+ebE#-GZzb*`Ym4*M!yzKVTkHX&9SvqesM`Cp!h#ePNAGF{W5=1C;%p4MNx^rg3d zI)k|csb#N23Eth{=6~b7?e7ip+#H;I9oN6SYc^WsOt{0hj-W9_p;o#+@V#g2yw<#A z;!K!V2j8+PePIFLuO3_pG}IVg-q`&)Hx;@#$=|hUaFBA`mg^sxr@mA|ZBSwE0>3;O z#y-r|Kllo@2cS*_)7`7c-uqhL(-hnn1wI}e-Zx3y6v>`dF-N)!Jwz)$4MCrKkez2F zY+|9_6Qbti7upKd90@8Eih!Zb9$RjiQ3sf~D}K7Pe4d->W;2VNA_G$C@xHrweWyM?4cr4!ar9QN zj7LJz|9rpK`n@-jNcC%vCf?a>I*U{5P-~!2B^1Fko|*`~M6#<&qkms)sqHig7G=~Q z*_R0%yx8gLd$_Uj;OH)`VV)!Fm-oARG=%|dE^k|?VO~~HW}vUO&T3~nm*!ospyX5# zlt9CLdu?`mazl47&zsNvaJn*AVfe%O#X4}~>z5xt`b{k8d0-pnwRNPqp`1+)INCZk z9Qc>5T(1Vx$V;z8vsk9+L{Nx!>jcom&S2km+9`Ynyqtdm$L@*GR*?9`ArYfg%M{k6x#q&J%}LFVDZ;e-eAm>i=*h#y(Ajx0jK z3!@T642|gA`grw~92y=pB;-uBNuSwGVGF| zkDSb0hpCs#Nca6@El>yR2Qh_rg?^Q60AK*F;N#CA?mn;m1f^|2%bLfV_i;%^fX``- z)W&mLlvaTEMYm7@yp0}Bx>5uQ*N>YULJ!T`g&Bl6!*4$WKiK>pXQuH?MjYrjTmPNp z-!hN|>k28Zce1G*9xL&;V&7@&ASLwSWN|ms751Q}5kaz04n`w2g2he-r$6gQ`<+l2 zAIV1+&-<16mkejNQA_uuomQ!VAvfPHSnJ;wSlGEC6@H>Th>@m~N6J!|RQp&UjOn8> z$>{CI~<&xHeG^H83?j1yiongTyaGNm7i43AC*>*N+nHDl871Ch1NcXl8M zbqEopcVi=KF&(m#q)3~7#}%ZEemIauux^_VOAR5T6-yNp3P(s}l0#yA7#A~rYP6Qv zUn5VVozz&bj&3E-j_;U|Ho3P!dS_vRz1CVex~{p4Ka>%sVUDYT4W=bW_bnL8WQaeH zEwcQ?I{G?+FD)c!lqll7!X$-B&GJ5lA+R{hdax$$3tXb`bUt$Edzvy!ZO9HCDyKu9 ziW)gd$tU}LC{*Fy?{esv-jYLW;Utslale!$%WpzMO(_FMkClRmU6r*j4r4+qT0)bS z@{v|G!irCwc}Qe73MM-j%fdC@9jV-Y{+n{v^+E3L2YMBXseX4LRSAPe;-0MRqR{?Q z#TOZ|y8Lwv3pRHgdVgv85QOZ~)`fCM#!5x~)cKnAgPH?68$O2hW@loJVjlidO=g$z zs(6`dge?X`vY(3<4y^?S`V1LE*|j2DtvWr*Ma6C+O}e48;99F*dSyh!7g>RAw${uW8N<2<%A5*zBU3)}67Fto z>AiKv>~c&`p~l)SYto6V;BN;P}y-!q)mK^#w|PV)$jLNeu> zcIafdCFtl4-zNgwl zyXbz?_3UU2`@kIpGr1PLVuXTo83IW_dwa3$+Ak4qN`km{5AVqzw&M_QVEO|k5V1Iu z?z<02nUX>ViG=fQ&jOwiNSRbqG1`Gpmpa>h@$F_H1y-ZYbUG9ekRc#2sNrLq{IF80#u2bdG2mm} z%t#n;Cy+=73nc(nFbu1?GUtfyH5>c1Qe7@s`L`_Z78??SPRWN07S1vkm_YS7c$&~v zN2MK3t}nc!PTRAmWA}3BnN`~9k3;x&{=Ys&lC5-ai#}e9^Ak6K=GjbvNjA&K5>jL7 zDWn*f_k*z}B)003=|stECsr&X&s}Qq%$lDWPvq3*r{oV>9sP(nvmJQpeOH5$BxlTT z4{m!aNr#rb05C$R12c{n_vknvE``M}Qn5^^V8ot>ZsrM=xg2A$z?CH^L5hw};BpI6 zs9hqHdKJ8%E39R*{p?*xc z=zB<}0T6xU>X)-T&yR|Aj`AR5Te-&24;lg~EU|^&uJ1L^`AZH4%g#=r8c)^BG=#3I zpT+w@3RFBYz0FEu{S}57is(E9VIjxPkdF^$(iIRbwh;FB2s&;cOK=ak5Lvko4L8lm zwbypgl5q}h+FcY9>#@+tjX51NAl@~hjK8UfdJx->*ljkB2j`p00Dy@&l741qq$Zwu z&1N3BBf+w$^$-W&K7029v|M}6gh8tYyE!{2rLb2kv-=C&IlqSbL8Yj{S*Td3Q55A1I6v}m=O-*%lC=PaiQ>kJVp$* zz=I<25`x^NtY8DSWc6|55n|IqLb*4uwg%@mscGlSdAZ!_G}4L|jlAX_VYl z(Cw;GlP9ygdm=V=qLTq$7!Q){su`tF`YD;s@Cl+XbsQlDo32zC?)WQ}ALjU8*%1YR z22ad#lf}eZEEE#PXf@YSoYA2b;7G|HOfNjK2q0mkU=x=jQjuNZ&+`(eHtk{#+30#k z9K4-CCFKwLlOzcy;*o1b>h^OQ@iXm`Nkrd$86n8cBD8DG9Ry{z2C4PTo-T2j+zytcu6dOCCa-Xchk-o6 zOIL|OhbE=ATcuqRwVv9Vy&1D9&=EQ;i=I`~ahB7J39-S>C8aL%bR{6T)b(SnBRKWz zk$81jEgWHEP^Q^r@-H+mlPEe+PpZH?2QPID6!GQlXx}jQsu&=LFo(csI>}^0OjRCv zzVvjuKeAK_0vY&cS{5`kY@-~eg4XuR*rIB~T(2GcZBMZ@L;*o&erbU>k;Ac(u+VmWO{4q?c+Rx+b;@m{&^+w|Pe z6kv35!NcUi-P~neS?Phu*?1lLvSE3$>uFN2P=PkMC~?vB+6i?5p~`0oT80&4ETKNv zNw$+9Q@;=zG)YqPAq&w6O<`awUB0&jrPZc79I`?e`ggvwx*}7oQ2kFDYTA+u+~-Iu zYzyCZla)K&7~;|xct?w(h>}QsgpdY+tP@>85MjT- z<r8l)GN6@7!~GlgP$dT|Q6(!6o2G~O!^HZs+8Ee! zp3-Pu-nOp(4+e?7dhG#eQymKfpXgNb>E>4qeL;kclH%<()~(&JFv`MNqR`DVv#o^H zEp*UUb=SsczZ405&}9hdK5sOKbCL~x`f_-P&Wy?&%YbJD^eDe5&QZIdzFlc#AWmz9 z`FRz1x0NtNAN6XuhoGO3C%35Ihj9?PlNM?FaX9sy5}JArc9P~!{KxJ0v|)+ORlqi~ z1$OkS-doM40M_dCCt6sJpLOm#dgPigH|q^lb#V2|cZlg&eJTA=@;PZ+{qDTZZ8Jij zR!oK(Gx>Q5EoiPWb}-3|Amfdp&6+Nfw}%!O_LltVV2{gV&)dz6 z=FgA@*S5--ECz%ueS0i-&#r)?MynN^RsgHAg1BgVC9B{e*6q)pT66Kk3C{3?5o|+B z3nk0;gU^Dnh>f8tjb^x#Nvy8%25$#3@Zr@ZEKCLr<`=7M`9?`<2e!*`n&F39ntPJj zmOg?%N#f0>z8>yGsy21$1u+#N%$vDrD%>ZdkADmu^WS%vIt+On(YZDOM=7?thcj3> z9QwywrYY?CS=1Hylij!-;W)lo1mwZidC`>!STzfW&)ix0hGVB*?8a>U zJeECFFIThMK+btPP7nEp%4LI8WG)ka3=7vAgmMgWurWfBs?N8m={uEk0$kngT*gWW z5g)`zGX09HY%Dh_6x@?4u_%`OCP8hTI$-JFY!QE7VP~OQJxS0rX4|y(YvT{W+0!q8 zRd93NcpkA?0m{Ay9YyPGdr@EGn8z>5U|V`Mm=qM|^71{tH%x(i#?%9wWW}SxW?M#l z`$Z=EB|-ad%N;QJOjFy(g__|?lBXpo))P48Nqr~l7ZWQU*}3Ue;QcyN`B}4o6#L3h zD-yblp$iIhb%(?Fysou^)!!W(Iv0oa7yAzQv&uB)JilJf9WI&3U7^jj)9uD=Z80@% z(rxHM@X`0)&gJTDe#T$oR-G7I(C)#QJ@VAgCXVdtGlwZOy_xwRgXl9C~gyr=NS5q5| zslp$ASR$rxaAD|oVdTzV9~Hd$UM@i1cj-Qgm?2baVxz8JTk>O!aLT^7gv@THLm_qOxR0r-CU z67D*kW?e(&QcCRnV_SCSkw+O9Y10@#KR!$O$hu{=400u_qSk`Xa!>M-TC~6UtAVv# zzneVnlvcxK>`%9dsT%>ItZe~zleuG!AG8t?huTtF23q|C(QerFcl5pPdgC@gqW5(8 zdq%gbk}CSyiB9)cYj{y*zRHm*N!4=d7JjhCd6kCtroH#c zx&HJ@>3s^Pyp#f5RtMy!d|$Z|1jTHLp*~Jl_BCbk0M4KiCOO%iUXu|SQh*dmU3d^V2hG&Z&eCaYmI)+hFV5O628`m;!j zS^XZL8%dKjj5)&r=BI&abx#%(HRpS9wG1QAl_4NXbUkeIq}lLm0{0gzkR}sDTNz9p zfqe`k?YyPN9{v0s=hPa{YIBu~w@+8Ll2VpGTL9-#T>IjXNEJFq`M zsP|CMrZ&lAT^EKvGukq=^no-qLB=D~G+sFz-x8gg%{ZK-S9MIz_*Q}}>Zig4d^CK3 zhYWj7RY##s{UA3*9KTFrEbH`E`vxtNG>aZT9|G^b(l#=SnAp0^H=>k+LMbUP2Rs-^ z*;rv!6O&&}S`-=j%X=gsV*+S!d8To)js@uRwfGLBd1qD(gL8B^X)J3xw7HR!QI)Zo zuYpcPS{COrSrM_bG+w?R{j*Al`|L@jo~{q?iErC^CSmzaB`_CC{w)1Nr5N#%9&Onv zMqV;Z*?MUeuDQ4|jwBKp18!ve0#GK1O|!T(^?7|e>*Y1VLgf*?+puw5k;+KRt`E+V z-oy{0GMA}X)3PLTjUc=gmubDC;9Ky6y=BUImIRhhJYjtlLtk2Y+b#~AwKm(E&n1W_ zIvc4%6a^0h&8~*=qOl2n7Ud<4u*p!=s;tdrL&vq^ zVDamPbjb*1eq`LG24O55B^jUhuw>`5u5#)`SH*vZ@|36qa6ja_*)xa(tED6(8f$*J zj5rMgUUqOt1Lp0VSA+IA*v3atlzAql(eoeG3^yX>tlY-DN5s;!Xe2+g&7FOeqDV{? znU#H%XOfm1=HOTcfLM?gRV}T2jw*3qEq_IG5on#J2SPU-I`c7kjd34#I6CFSRJA7p z=j_qozlqR{iYd%**@#s45#ZsjSsLQkuPx_OUH$ZJW8HgL&iA z5f9Pd7xt|wSbu~f{)1=9cpHV}e6F^%M=n)o;WY>W{f=uosYqbS2HP4yAp_rVipdh8 z5H&?1AgCnyhA8JnW3NQ1gkhT&u$x31N{CD+XaVf)EYnUK7cUG9ldACPNMsE=d(J+a6zM6Nm|oBQhPIV8%2QEk-gpg5Xf6b?)h ziYEqn+fX>>6{innzq4g>0yw;Y7zDxtU-44(2>mnbVHhH3f{!#|=Lw}(_&5zr)W|q^ zl(|kgb%wht%6&8lsea%Sh4^U#l17oNP_uqdL1S*dUz z!GPjOSbWXloU*jK-s?LbF?iWr|{L45e0gG6-3_Gf((^{3Z== zF4VT*3WOWdT9Mo1g_SI`r=N%2r#8zv_n4Yl#7%yfTG?aDT8Ph|y?d1J5+&p_)Rs(f zOGvYpw|%5W3|rH2*8-7Gb8;A~I}ny%b&}Y9f#OSNJrju3{Up^Xjx(blZTRs-PzHs` zvQ4CFOX5V90c#-&<^q}U_1Ihs6?+o^Tg{rH$W1<>M{531j%rr>Tn=d_;-f<}!r<=t5E@LA$VmWa_> z-uqg%BU#k@-D+hzi|vQ&5VWhKpOEd5s$$nFIsUCU>edbd%$d?K3Qy}e%qd>q0)1Po zXH-TA1h!-4jHf|^PlL|7OM}s50?N#cMMm$5pg=+szS`hT?!_g;*c@cM8kn%J_oge3G$C-jI03b?$4{2zrBAXJ`vi@hxE3V=XFXd89d%78eI40)1R!EolRqQ zJdWWuy2SU#w!3bp%sRf^psOOR@8x5ZpuX{H7f^qPw7SIbeEroR=MAmWHlyV!zwFuY z$ltv7((dTIGj8a8riQ~mLL&H9R|I|m`|a?)3f)5NB$ICXy(*NV6VI3}+(8%IuSr4j zk^YFA{4tvUlBIjs_M_`$9a=JFS;U}YRqJ~e(d*9vG?LWFu!#PA%%K2oHy zLZn$j}dY)DcZLc=FU6HdNr7QstuvYiAeQe;A(T}0o$OW#G#P`=9%38KTnG8x>Z+udbV zANY8`%kXT%2t&b0Zt}%mn^M*U(_`m}Rr0yc@DHac)hr&)X%VgF9)<3n-D=E95eBz| zDYwracfcOTxfstPEKj~EFH;^BnhD=K0)8LN5>G~bHB3A;&Pon8Wn3d!LYp>1WGWS9GZW*Z6qhU&7c_&4B_Q?5Qu}EtC`myjx+lbBaLoHt zN_LMLV@)KVQnn8U#lSS~5g^xBst5&f#Ul2JaS+O7^%H{GrHZjoI*QP}*}nd+^s||; zgUP3IN>u|hRi-juB=WQ*9m&(upi~lN3@R!?1Ahr%jPb$Ks5o8g1c&T_>;kr0`g_H0 zj@W4|bG@%jjr_TqM6bF)YMEHyLz>CUJTc5%(PT-b_I9a~OqMLShc>h{ zuA-LvXaMagV)N#3n)?F>2O3Y465Y2$R?Q(#gNmR?h{U8L%3@2OT}_$gWN#m1&#?+W z@&m8xip%dbn9pee;h0{)0=GlUeyR$8i+%IiiXdXz6W${bQxPOc+t&eTO?w>D7Y5X} z^p&-EWu&DEAPe@WjEE-f!G!V3w zkIPFZKf&HL%iudnv(ZdTugVIa%RD`f7p+PTr=ygv%weL-Ew>8FugcrD-jb+VX|;|# zIZ+=w@u{Z^*`X`KIdy%adm^(brmxO3vrgip&myQURkY~?oy3<{W#7|8Inak8R2$Bm zguc<1#oMHr*c1Y*OEPW9y{kW0o(d<^*A7+J5YTB)+pvup=vASW=0jHs(!^NNCp1@A zj?p&-RMwT#mo$$xnIP6YD>i4FHAylQb&=taLZE*LYDqKLe12 zAsVl%XEbcvb!*lJrP2K~&7YH7XMg2&fyoHWkf93=)EZ?A~lEl7EPSY=<3a zN9q|9yNcwZAPPZ2eHRQ?g{xu;)I}ONsNQyi@r3BP5%`l3_{Se+rbZL+ljA&28k-rH zo~gd842liZf*gL?;MKlpC)MfNi<;EQ1)Ndioh8UHrdLY_{2$itx~a`a-}`+Mf=h9C zcPnm%;!=u3aWC#}#oda#ySvlkUfkW?t$1#JYpuO!-+Rva@OPfFA^)HaWv-Oz^p?@WwwK_Yel^1G1J#5U~*## z^}9RMP6qGGhPVzX9?p)gOBrTMPhBpg`Ks8BOK`ByVQ8!SH6xBK7XJLQDe4KjNVB}B z^KCjme>5$)IFIwVrw4(%WM9()gpQ^?ssVq=eA2o;cQN@Y>C9`4FUi}C%UHQH)%c`d zR7&i*=U6=tOkjkjnO$AXN_+==sDV2mo%3c$6ZB_^t%h2-(3Yynd0le$w>S@8@#$to z{0)u0*cDQnu%1U#vV00p-hCU*iuRG}j2UB>5z-B9$#|qWimqLb!RWWa=Zl(Qw`6~A z;nm&7=jXx*gjQ$O7Ph{2*vE2bD)9hfR-AlG6t)hp%)n+NEZ!|f?wLmyCR@6~38=L& zuG&0i!$k%NZZX!~YvjB#%@ZrVsTD!e>_D=FYxVlZ--RBUK{n*<*ZU!QQ<- z%%ZmNev>s%ab1|XJn>ZZJiH+=2&LM(&=j^)Mo_2GGrTzZE> zI8KGYTLDP@0t!dIW~CxNhnOVp=ZKTfMptQk4T?n$*96QVYjiMh!2u zj*`V3US78}F%9)?w+l%ea<$s3jRIKPNADze-ZA~cAPev~n4)c9UZKvDA{*;zE-S;Dpv zy4*P|np1m5is-Ac)qJDQ2B+>pqwYnc&Wf=%q|xdnR}U%%la^``$z_pJ^b?N=A{keR zORf=Alks1$4L+CLFA0;cO{TV7-(6XojKAO0Hlccr7nns%n0azpXf;_Bb6Gau7v!*- zg>acC->W0YTeWl995mU8I{i1IhX5Gi09=3^(8YgJ@Q45?bO1c!dky}d6g(oqdkP*3 z2?HAaUnzJj+J8Fm$OQjJ^k{_a|04zej_9HAfB-yXFdh}{e(CEla3x zZmsX=ZR}}mes}Z(!>t3$o&Q4gom12QarD#gj{ds)-;Vy@5%}TZ|3u(tPyZ|bzA~_U zH#ocaFG~M3)KED*vHxCxA6|I+52Bwx8~q;z_|flGuwCniNG(9{8NBmzWc|~?@rI{AI&fSzasj@qrIh}zWFu2=V0R!^2!Pu^B<{+FXa-#)+IzFPc$aP)VN|K;eP{^RIZ{&Dn2 z5C7`G|4&N){Ce8oe>ygL`v1kz-~Hq0SN?x<^zVrN_2l1<9u5L{g(s9k)#(nnpC(fo zvdirao+FkCOtH%o5C0dUx6dDpCjAsnDE+??z15>t(VwVRn`5LSmc{XG;Q$08nST&H zj61!~NJHsNu|mE~rVJLoYJo<*#o9>Yi6SL*PdE|3uhmk$=}ewZd6KnOv+aywsBAN{ zcDvif{@f_{g*Ih!2LiEN%XG%n$h5LDl#jL6@9IscMd59> zIvLGOUW$=NwK{5Ba+DflY;MsI{ZA|(R!mchANOP zpNYq@H(G8fIqso6ezuwW0kZk!*E^y=BoZ2mWbW`h4KZz>K=~o$bar=Uo6M~D!|xUV zL%Zp#^*F!b_2^!_;RL_lOzOBMsG$>p{#j}>2-mcPOcq9scH|#KuN^|}p+U|-nN_kK z&KRDu?Z=dbo9jqAB(ct zua78mxc1Etb3^kQigRe|1`m@1+u#oJ8OCu-Q^HKkj*8%z=u3*TCFto2W5bVei!zGJ z%5rm<(GSuhTOi~NOp!f&N9D}}$tM9x=f{?DBbD!n-lQqKJjSJk@1(X!ZJMD}@{N9_ zVe5Us+_-3A#+dG?scLER-M5vg8RfH)WLm!f22Wo}T=2D!0fonxwm z*f110Mwg+#VAw=7*t^ie0`K4If)E^eNqe#B3AXLp(P5dK{1T-7e6<8FHElxeD{8Qz z=@{B;jI~l4uSU96-9>?&Csku1{7`4xoD1uh>n;qP^sKyPSuDOYTS%m|-iF9p4qTG6<$WEzNRl}fmeR3B- z9fE{#StQBQDP#7F7__XhfCM|i+o}7pX1sJ?(Dc$jhH;2HRPa7b*h>c)-{3L&@%DBB zqL{4TH^H;cl*hIJ-$X&^P;lv&Z-1@>l#7n(fc0NCiQy$R>jKMfHr9t(JKet{`qwEc z955*uZdWg~!oT`^qyhNpIrEUvnXuR!6={z^1tOo+&@T7^n>&GQiFmUS^X;=bFA{ky zgiETA+=9;GIzJTwC?X^v#tP)r&U;LOf~Iqe1+n&NM>df{cluqrxtDK#pv488zbb?} z(C6WQEGZ;$vu0zPH7M)>1&&g&DqQnvJ~b?o&$reBlv_w05FUVrfxRS<2&12@EdvrL zUu4m=yNRG_^Jme>Tqp_jcd~%&i#4cjNV0}=AOJK+m7O$@m6&FL`;%l_oRjADGTJy? z9|u}kWH|;QO#&Qm^%)w9Wc)65GH3txgD(JXGwT4R3#>dcB@X>S&Y%$6q276&W>r2- zTgyE>m>BAsDB2CLHrb_FR>{r^CyN@VAgttXubDsVi}BXtXZ$|3eSaiL(!WDwt)7$( zRi_EKH;(ukIphK-U$lV!3ZLn&EyYL0OfNk4<&MEDnMSqxhxD4IKRht?mlM)?wQ$H= z`@0Gp`AeAiR0uTxpt&I$C-WnGM#v+=>&u=+W}eKv%9p!T8I#nf-Pu2hMRVdN zQRPMBnq$qT$OiEBCnYZ8vfmyV)r8~A^}huzR4A!dSl5;5ii)WdIwbxKh&qkNZqBq|(A9Nl5|US`;H9*BoF&!<-(4mD)V;Qe?^mK{B(|k|N!B8sG*Qxc7)hpJc2Chc^#11^KQZRNR)0 za9{a`_~VRi!tg(IquQ`S4gA#i5o_te8D|Zosn|mCbR&7qkq{uSLAfXk=@{ayc7WGb z!7Lx{GVQk$G?m+N@M;<0iDnI9l8d(8eHb9ZW(##i+9iQX1vctjnLe(l0_rs?CW306 zY1&cVX+5#V_;q6AmYH9s_2~4S6^^A5cC%O@$;43fQfhMhrc=R6187IF~Qg zn5NDA+2c^n?2--}_G~yo^|Q80vFv<{ zd~=m!#ktGO(|oIoYk@1%Pubql-%e^M7&l!PJ|DLKh3H@35xr;+KOAi@3HXVBcOI9_@Ff-qJIiV+L#B-D{t$`gX)t_a=u1W;%?_7m3ak zZ$YZ_SbgGYzWwbkNtl)1^y~BT%G-U*gtx0xblCdM+uvWXLIX6LrZW?-=QU031q&IL z$shGD;&p}c26c2c4zHE|_V^rnkH4OKLY~WFUmyp~AKtD%k-xlGu^;|qJI~~=<6kaF zyd1DZ{gZ-!u$6d=>4wZlwCU+Ag*PSySQ7u;?2{pnl$w_)8G9{f!=Jpnb&hq(^^G8V(a zB|}0UJUleQJ%&QEFGF%`!YfL`D|vkV;5@^kFZ#D8gB`nXo>v86PgVdPDL9IRtn@d3tx2$DR~IrJcQpm3@-pg zeBg;PHw#_DHET^V?OBY78ZwDpjMn7}!?KTbRn<&2iyB*uSyPX)9`dgXjiwHbg360p zqYZ>FjeX)a*rkm|eT>~Riyonk0uTE)mjqod8kxXEUQGI3(4r6zTaXO$*N~)LCqebt#uX6@oi9=bV8h*t~44;RK)^W@3eD@~hS%5ii{sNzb&q6UB7OaZ3_b?_i}x~20srToFmzmYa-;mLoj zff^Roa;F0~yssj@c_+5yxIrV*Jh>wdI_B`?W=ZDG9i=MK$DSvh*&+>jIU{hq+!X(zEfK&rNO<4vkF8A}svKLCj zrZJHx){0yhTDh&%VJHV%8XAujfNEz&H3^*?mia&rV7)9qG%v&*$!*j}_Ur;;+d-4? zl{d)b$qqr|lVZmeAhouoRk&x-m+;nURB$e)-@EILJRvMt6!XbMVkbdgL4zIeGc**6 zC!V68yP)+T>d>KhRWQqhda)n(Wt0$(xs7Y9#vSD)i+Ra~mElmKw%E%uWnh9S0MesY zLi+admd@k~B3`lnd9JzZctx|6K$PbiRQ^0UTo`IQ7f#ZT?P*0jlF(z+0Gk{bqi?bF zBPP~eqDZ7A0%8ThBiZ0(82(A9z>_*28NI(ImE@@(j_->Ni@*W=Em5Qm9Ft(qSUnn9 zGjL74QlKK-q};Pn!%)1OwnCdMk1o0LvAe`3mG>56?TOIb!3Er2GFtbu}$)lq1VsO3~lE(ATcM|_7~*l#D)E`+$% zOvNT$4s%}3frn0zhgXmq(?CNiMO?Cgh_YJvq1w*0YTzU%l53|NY~2A{OsB8ju4SFM zEGY7%qBHE*#_KYnXo94{ z|EfVeia;?r&azYKOltLcItI6tx4*&lsVEgZv!*m@qPR={`c~2UQm)^$1o8^|eRcQi z-DFvCQP5ml8P=MyT-CpSm`o#ZxMQ~>QiUc51p02;gYr7HH?TGf6?d0R+1B9Ic4ExID~ z(X?ne0H?k$S$z23a5)=~JM~#P8|s9cSF=Luho(noe*L;vIa3a;(Vk%d3w?cchuIKo zKj6<~hAiQV@#iZHTmxKp)PP_nMQYKLK*-=t(p@}wq_otr* z>#rcqU$?2PnoT2$mmp$=)^crA#*Dtj5wkDCAYy$`L}tBeSV=S$C$pIRSCxTy4;dkl zoPv`E17A| z)lj5W`P4JC|73@6U|@nkJa=?iEbB(cRKBwNhN%4R-C4c}WPK-a{kNnK;bZ7UH8vNbCpGQ)aF>I}Tb3Hn{fZdlQ2#p-W_(?j^fcCy1h zX45C1)1Y4$=ZlBN6#Lm+sZ#9FVzJ1pS&T~ z^udj`<5_K1D90E-Zb@x3@qf2dxKf8i4w-!`r$PvAkIo~@CJ>0ZnJ`}lWFL~!UD~^7 z-m>3X>D*$hmgYY8n(NgBOP<%4o)e~BpJtu#ttGKk-O9V(wQSwVRNRHrCh~1eP7SLd zZCuhK_+)h6hsOLRz1XKAuDuev`!tL{<#oZmaT7IgM}rgQ^}*YJB2Lr&K?m>HilngI z=H7SpSb^luk>yd?{jX*9Uvk^ijIYm)F^?}O_X+Mt-^|r(^qw$d?%A`m!b!rTN&eQX zKRK~nmPrg z%n<(|5bAz=R8I8!pA>vSpyitObitn)7s%PlLXU-E+ikJ!X15B|nbDC4;)} zsO3+2pJXi0Kg|k!jOuz*YPS)AQRN%JT(X*$?pdWC+oyK*aF9S%_rN$8MWt$uiQGXK zL*+Gnn4~B4#CQ5BayMpaE73YqS}Sq6-m2I^b)tb})a5-}! zJ!<3UF!BW*Q6Kd{o`a18>ci^wC>WYhVRWp>^Ga@Bf3)T^;!+0_NZSq3e*6OgFMR?y z=lYtLU9})ungi-g1{OwuR>fZwO%;*=lpt`veW&S(vy;+PjknWcY;#UCQqr_J0NOVJ zX(9UDj_MH&ncxWsu)U`1#d7W5tGFH_Qv$uEW>nRdLOu0He97iGoz}?uB5E`_Z`tQnP91D zIJ>BMp|x`A`CP5xEcj4NbIN`!~EJEi;g`60b`9ca?fC(cN@VLnnFP-7z)!HzV*o)risk4bLnmi2BJ) zQW!gRIgvQy>?nBY)_mJU#0uhKl>Hbbe^iL9W{aDw9t=Bf+jllEQ9D5pv0F1W&LHix z2^FEMVb{dJ?s75Df1r9w)TY|?^X++~3lZW;(f3THZO8bXsl`&nwq({{oT2#rgJ7G; z06_En{dWrfcH#qI(`go3%G-Hfi<|TgNlvmSOP}=bvS7l>J@a6K^Fgu)=AMPWFhSU>aveP||J zkp+|Y6#T9}tl$GgWTsR;kd$%@B$fgWtS#~RzWk*|Tk4mgV35Tj;`m(6JWp?{EpH#x zECQ7C?^grFP?51gk6@|33(Ods(V3M}6#M#iGG7ed0z<4y4IcDHTi~}2L!8dgekjuD zh?R6zoc)7dlDgd>g@7%59sdfvf!g4ofIb}i5>Y=a8gsLTjionru_i*aFdxhaQdoDK zls}bNPTGbPMD7SGf=Y2e3l_qC6yd4m9pe62ZrofB(R{T)O8D??l~&YTMkPy0`1Hh> zLHTh+xs2RyeP_qHXS;zbshZBAU`x@;t@mqNOj4t^!sVM8jwpIu=&%5#CfuZGd`7r; zY6Afbf{3&QWLhq~yCd0r1OU{Co8s6E)Z^|HBL~36rp%kC5jgS+h4dd`bQB!W_zZvP zuf*`O5&in{s%Ojyn>}YwdbcF9Selkg$vq-g>GNJ;XT_E>${<~#n77(gsL!?ilhYUe z4ZL36TXb(cs3IQTo+%u>nQUv|q0AuHBxT-797-k9Rd_Be;Ca;@KTknl+eo>0G1j!K)6tp zxo-xuMk zYAcBO!cQrey8Gdx(vjBXQakLHS`g#cPb-==lvvyziSUH+&}g7a#teemoTJx{pfMC+7rp|0uPS*MNYYWn_&h1V-mv$_YJM;F2(<%&* zAfT28EIuZSgTCPtUCQ>SHCwCSh`PcqvK_m~$;$|yOs#kg&5u8e+Fc@W!5^%CDK=I} zw;CI2>?L=jDAAiI^KZWKEp%Z$$68>V#zI37n(2ZrUGTS{8QuJ}_PuL@Ciu4`1jvT= zAmKGX;agQLz^%yhJ(-JbBkKV^xSuXEtwL=HwK22zf7z&Bg^R?g<2D-+HA>y2Ka_csP;0OnM(rUdK zw{1N#s*E}@T!lW*9^27anEL*#91Pidw@ZKa-b&g)kf@93S>4RI6^BU2Dbi`n@evTd`t1 z&#V@`W9igb{11gH`1u^k@-K)d2SorU;S$FxBX@ar;V$1Y-2CDjZ6aAO10+vW9!NK` zkR1o5KoGx8lJOl^-&5Z5hB@OIe z1i#;Ks%$T$OtIToPl}G#vFir=kVT!DL&ffa%wgb`mcE}77=k>gSj&%*3AQJr4^1y;ji{yCsCGVAz77`*Xx)K z_hE%b_f>3{`e1^VZagl5`2j`|7*Y)&8t(CylJhbw6)F(Pzbhms{Mszj#UG*NQvh=) zq*y3Q1m25ud_Wx*;REe^?|yv4Wl{njo3;CeB=#2KpF1uZKF$U_dP=|#t>P!T+M+!IWrkUqKLXMeBsg z^MwHgR4XN>eFWFl0l}d;sCKk0_5O>}L|ArE?3x+mZkec>0QyZnj8a-dIM7TO@Bst! zdB^{2YhX^gz-9NRjKu(F3K4~=4%gnsqD?{^DPm|T;9(uNPq!b0Dgg)&j{o6^OrYNc z{{Rr!1t3p~3a$1c?$);J*R8tqOB?~wytqioIIgy_StEV1tm=$scwQBG;d6kiMk0t2 zyo~V$Xl6MSn~m?ovG2kqJ)RYZmN~~W-2e^$z&b{3w=U99zZq$e`f{Qs8R19t_kVLh zB(dG-zQ2LrNJv$PaEDs65itdKK|C!zvJY*Gma&dTA!evGf;qow;JOM|7`7fTId@TQ zBAK}u(UjAunS`iKRX7~(_&I>3Nm0M8$A?>FaAy zpc7-j{PhvmZc{L+zE7yPk72Zlk%EEF9sv8`C+jbIH(UfkF+!q&6Bm#|gNg(4!woyb zv$3(}8ukV9kPc+l^*8esFWCFjZ3E}Xdd^BqFi^?p7^@^F^Mq@AzA85IdkHeF_9%aZ z=J1uY!t5$?L;y*3X}|?*#usbkV-WC1Yh(Z`0*4vd2NQ1_;(NICJ);&3>qbiX1GHhk zvUNAMlTeordNRT$Df{|Uh7b&QN?An+Q?$~TH<_3FVt{4MwwtibNKM^_=(d_nz=`Ra za;G~rngwvW4gTg90KkK|kw7gJ-LPs#-YkaSC}aFLGE%&Xv0mlShv)FYbby57AJAvN zrCb4R54HlUW-H@1YpaonFzAm7A9b(m@T7YFu1G@+L%SSTE6iI11NnLO#CVu-N4hN{ zh!=rmvEvdc@{K1UII%4O1+r_1Vn+$%w1+%mmhyi|LtHq>KPmYiMvkkaf=Gw_>nVRF z1IHF(@s3uyFUm#e{eBG~fpVOLXzAtUeLn+p`qMDk_E`rtX8T_R6j*ZolW)bTXCr5f zqxuRyali#HU*R+FhFLTK09N@RBN=2j85Ln+4CF!ZmrTM-D7EboUunttA^8PQ00(aS z^<@Gkt5o(X5VfO{_)#(;ti{_3OhFYyvKNtbJgQp*8^e%*WGwQ* zgc5m}c_!LlA~59|B<))%`}w5>WbN!marFP31d4U>3x#%+o~#5Cv;gP8esp<71en3W zpa23mRZS~efw$P@sIm;lyc?!;X#0Dq)K$E3U7h%Li?1re+>#_4r0n&;*pBap{!J@Y`kZ~ zh~gZrwTML1kqaiYUvDyQze43R@;C0(6clI(jiI31TfL|jB(^# z$jag7bT{##;hlx|dHDtA{x;#kbrf81+6#KYnDR*fxP3whYH7^Ql*#8=F29la>?YoV zE^jx(Mk(xvV+SnF)GQYS;&(0MPpTuG)>0~id^07l7lYMl`RkeZDH>&ck)+ zRh{dC$0{XyR86FCFrn4bL|_$G{ReI&x%{ZsXVV1Y>P-?ao$@ zZ+k=Yqf**GP5eEfzKva`P-Ntgn+eChV#%W>BA;nbal!yaXWhb6R>6|TWi$(%wJ9d7 zE6#LlI{a##H61BhiL7JkAYG(If3}qQr70~I>m2?{|EU+PF6bz;L|{dvPOj$fw?z#- zXedgt>@Z6!)AVr$sM*WJ7pKMVJzY$b9aCN33cj ziAYx~Se3HJL{34DRG6s$=a|wU>lP6bKff=`VWK|s9rc1D(Eeo+vv(bxO%mH#ZHyzT z1@uXvMTvD>ndmtSlHt6ikEaM7X^`)2gb%|^Fkkhymd(F!KnlHi;UFUC0IjJ3VqgLw zkb;JvcZ@TUhDF4GcP_GVBGX*Gu;iDz5(xG~3Lt+Py>ePf%Y!qdeDdMENpRJ~$te;zgQ|Mk;0Yf$&CCQN+&P<2`3 zm#(#snf?3zfurhRHEmzbWrt_otgIs6nz+_RcdzBuz@GTK!#wu-T}Y6RkV}=}cljtI zAM-#BfsiV*2%jiL1-=Js+^Bss{IU_2{PF}fNp(T<6o$h-`Xf2Hqr~&dqM{1PDs%nF zL;L)r);4qO(St}7UY>SSOJ?Yt=%c&dW2>7jEwy7i2}?%<3sA5{F!78544Z4x@p-DH z@tc_sym_~OR72+h4h8n~-SKM1j>6l)Jc8K|KC@cRV>@ZHW}nkuC~2YAMU8n!2cLh_n2Ibnfv8ys^@?STF>!GU3rAnfgmX?z=Z5NpKez}E6Cf5d)gUZT4x z4e~qAobBM$Vnz1S!_hV@gwWoR=YV-er_QYSAL>C<3L(UJI+S`S%dY3nrEE_IM+5vx=Hx=_l7S4M%zcMj^ zaf`1mQ5u=98`U?m5<0HTxVR(d?WePxe5#zTnyyGMblg=e8+UITfW!p@*$2T6IJPd| zZSN!HzGDwwF#kBrqmO1wzZT}@=OP-5{;D!q8=8yB>_sMN(OS>-Esq@%eH#>Kax)!33 zb036drrG(>l6w<(PZBZs(C7^yOyX6MG&UrD92&WfiADh&-CH!@z@g8i#-@ep+S=(Fu6eft<)ZJNvCkw1dyw^Et=Fig_m}U z%TK&sE#qfR7f%Fn9;IBbvLsh_IDbuZ9;f4jeB9hNq%^ef2McG@kK_CkXXEo1O#f&~Iu9rv}8Y zbDzCKsDC&pzi|y8WtfH8r$`jeK0z5#PJj3`-*c9B^ZF9Z!wzCr_Si#h_G+hTVSEy-6X<>iI0G$eL7fnK`GYWSK;28#I|(J!F` zWPY+*Uke+{2S4;6Af*qHI%H*ekn|=T&cw&sslt&H_|WGsFfQg1dW_l z)eqwXb8~Yf5{!R=3(GA3GV6BW2FLvjOZatFT8erQN_yDHWx<=waQEEv0Zl>^{31I{@!VSD@cjJ^VKow_X1u@ zE`Q&sp+ssnHl{0YE+!fP#+9`$_O=S8T& z_%aP^~LA2UO4qPpg%x$Uoiz&;OzT{D4pxSIe`W;aFBg}L#|NGRMRZF)Sz0$ zU9t=!3crqT9C1C&@61(MBEq0G`L*t@WOWrK1A%>bZ zP`iFQ#5?6cl8Ajfl+~F$JvS$S`g#FBl=wamZ+hIgq-ia!fVOIj+Ju5@na-dakx$01 zv}ng#MvcX6I3z2MB!x-}2tFjWr7a%!L`YCnbMJw#l>mh{dN0#6_eOtd*{>;pFQ=}? zkgU+Gm8EMxt;gTKr_LgCtymc>FKeS|oR)YF9cI#tfLJ%eb2r=dBF7_^7OCIX3=*Sj zTxPPJ|5}kN&dze))g0bU2mO3o$GU6~hFAZLZr`WBN5&2jbNs`p+* zF~wtOtYsiSiQ{A%Y+@>2?PuM&ucz`%&u&_+W4t5z!<z&f{C@xFA`t_xy>iXfrTIZNCvP|Nh zDAz28Y?Ox!U-B#^JkQ^R0#r1Ad1+fpLrzNEmZP>EW6m37-QBNrtE#6m1FCQae-x9& zVM$=x```o`7&;$~%=9(hG*`)A)!Zo4dM?$DRzdDtS|_|m#RdQ{0vRA()NR8f8oQ*6yZ_4Et&DaZ7OhE$T?*BmHJqfEm?kO99o92(Mprl1 z1-=l=*L>p{2XqqK5jO?^z+UElSl)5LNs@QIQVzX(S>uB7P;^0E*cLi5cr7S zg|HnP{TcVt55+18Q*G0i0<9ax)lqGo){U^cV+xTO)kPxobZ6g#!1&x3t|V^YK}#Pl%t0}vH_ zwIm#S&QDI>e<$|xSQcicwfAOLT+3XQ2MM?C+s4hY=T{@GA;-n-0{En|(6V0QG(||! zBnL=x!zh9KSXZ0kl>N03NMVY0@}*wjhJ?56$&NpgZKfs5;M1}5CzNHW2R6}-?)8K7 z$P+znaQRvq3lQ%KcQ^EX-(ULg#_rt4ydE}}`yw>RnQ7*kKZIa5LC0Yv!}YL6x$`%D zAp;gZ_2K&P`nO=+q*SsR65Dk_k-Pw4p)q_h9^mDZ+*n!XD53Dgd?nEh<58WEKBG8{ zKzYOX!Uk&79lsq>9d20#9T*Rilks9ndmzB4B8iqDUde?hDM#*jPmH4NH$ z@3ik;H2@tmH%IYMG2sr0HBu1)1@a1F^==!DQ>BD|L=a`3B$FUGb}%M2!J>f~@`C|i z6?;ONqxqC{tFh+F&@mC-WI2YRcFdy1e!*04SCN;hjn!FDK0tiDYh|7F6?_EsQ2$Qu z&1#XM*3?P-^iejmF0GNcp^}L)lXkRmYWhd{;^&H8zA2Vy5g;@z1>wL06pUI=wS@}& z7qo!@fH5hG*H4NufkSCSr!ClzmF78FM(RMT?wX)kO)Hc1a!H{1hxh0WMWPG`u18Ot zawD{PR5@@qT&=MIUj;>s)c>$d#qD^mg11j|Wn@>a8F3v+lW1p!?pg)?4^frGC25cs z{0y{qLWyG{2CP2&R-4y*QP6VfLEG#tESmrHb7G*S2txQx*UVCd7q9rnliHo5Q1q7S z_sEg8Ww$0>?@K?Qpxt_ZXDc*jgcXE`CW4In9^832yJ`&p9;#h3iO$vaEj&mlg~!C( z^K7H392x-*0HR`cY}S4W{wACN0t;PSSZMF|Z}NB?fvwGc2qZttD=}Y`($8vG>29Cm z<-qH*2Ve=z1N(QH;HUsUancB;x9Xbh^DF^uu+X%TQ_v$C`k~5!gNnLp;K z)99rt#3f5J$(0FlXpX%UO)I0n>RDkhWnFN(M6#la(^9|gU;PuU;U|J;SRa~NenKCI zD!72Uc^=(HE#X|6wbDR#)Ro^~7PwdWVJEDdjm@W*NugG?ppxA*{i5swKIm-upI^;5 ztXYW(5*#Y{a1>at7}I=xHT=p9w=(p4rDXMUZ>tHexcf3vR)KdU+vRVlG4kio-op*L z)FRqs5z~`A!A&BSaSqkVoA)-s(McoBR7lq4EPDH8@TZrlgFCw>$r^yqcclJr=2x&r zgjOj@MJYP7C%Qs+c?(RO2IO0$0#`k%bknstU ziv*Jg+C_hEzHJDW*A~Nioh(V+IRxs?{6oogQng24l{{+Wvct{bokPNK)H#0H>dnv* zH(-72X?`I+WcJqe%U5cK9PWqNe;(ChJcMFS^+j3~MlB(@uhh+s6_H;{jU`Kxk4k{u#@HE7l`;oj9;Qv~ z>2(tkzCdekldbLHZd_euQZ!_ri|aC;kwBEpk02|rL?}&GI0}2Z*Jz2Gk6ACRqz{{`7+k~%^w|w2tH|M7D8XC3JO}_3(tY>T z5I}BX&?>4&Bsz8YO20nGw+NUt(mq84WP~*Y&~S}8*%ljNy>o7o*B0ZJem`9v-gP!) zZX#&t%L7kN2xc#NNAA#^4$(M4XQmQ#A}Q_p7U4EAISkQeW*$PGA$?UI@7W^hue>^B zL$Yx)WTABm7+H$_Z3tU-+O2+++#=OEN{fLZb)5ma!}c$GvaKQgjHqq0B=UsPJdW!u z68U`2JVR2h46e5A{aYNHF`ZOsGNiZU=~k_IQaPr!4IWl<7Q+FiZA1R>FqSt`bo?Fb z4oH;XNbbk&)C|0B^mB@85hD?fZm`BiH14hlx$)p+B2(B>__yvV84+UtNmi#Ow87z9XDlIhqN9J<`v<*2EsKDWtf*qznSg7}l4P`z! zK;D6p^HLNq4PH8lGQf#aF|9zN$oQz=pw3HCmOP$%nGDk-PcB$pl}}6lNnh>0zGtFY6B5F$;8tr{bRAEX{|6(@@VYj3MH+Ex@ ztYQx?N_0vpTjwGiZ&RV*B9-ay-f^ZR_U|cp8s&!XI{?2M4&V+?fS>>n*h7LU%!5LqJW}YLWy&5->-=j1wX+}~O)9wRJq1s@ zV^!kiQ{=>0;w2%gvTZ_Jgy*tNJB}quoMsxGwzGT`=+aj5lb9~<{%hoW3f^`YfsMWN zERa@CzEoOlL0>F)EQ*4cPqx{v6z*&aiMli;nl6>R->LAx0~DWDbd*+C>ZUo%{;eQ` zedck>JafQ2ef%i%yfky0F8lZ>yX=UBteeW$gfoFIkDC77?I(;64C0bS@s%;d85r0l zC~F=U@)i1<(igiNX9!yq2bYBu<>w`pm9A6f_mCwNS>#t~vtg+h9OvUnABYQ-I#J_% z%_^%pFSDpCJJ_PAwDEUdii#hgFC(&Cc{r|?FRMi%E4`*~kT}UI4BzB9k>CLgmC~Dm zc0$O@n_Voa^p`7qPL#1NYy4Ao3`!cR7~1(V>-f~mm9|^C`ie9mCy>aDPWJa;GQ;ot zlL!Zf%6ZG0UzSec@6_tV(ZA?7Gh?p*KnRnNI) zZ*aws=3wu*N#C~gu*d13MPh#$D(7^WcKaFovaDwL&L%%wPw0Z@qnVa0 zG#5YbQF5Ktp`R@~oGpzwhp)Y;mIo~QYlG4(7}0t~Gw^ZxY{p^(N||ZdLu`qOX(cVo+W1>3kYy^a z3U-4v&vb2Ce;gQ4{c=CAYp?ktV`aVB?ZKghpa;P#k(h(U1i&~EgDp>3c;^6%U8pzc zDBtb#%CI;ntCQSULJaIc{A`2`tjOP)C10kI6a2+RY7sn(#`ll#C2J*4Rhi8&<*%#9 zE!aq6js`G$;1#+Mtlk&1=>*)iCCGNn2}$oi3YflWP@7jjdx(`(u;Ea$oYUKjDX_A_ zV}r(O;eOjOFWQkLI|MsiEfVEQ1hJ9Ful~&fHK(S(H93f4ssp9`*mHVWySsOatk?E~ znMPA+A*}GQ05la*RwV2XUN-r0b#$L@d~9eD!10s|=i<9`tk#!|=~iMHbX;G742~ad zJJ>@#0{`}eU;nyV`}~n#-E4jhfWz8H9|K_>q}_$NsD2UxBiQN+R1<&}hOt)x$*C(I z8umLVxghm|i>&vvxQ_gvob=x8F_ZzBk;x*K4q_GOC4#_29st_y4cE=7#Ob+&XEh?K z1?L6}q1r6ej{~L|dw?_$6AZ+m?AFf&$f#c+5-^GV<>-cH6+2_)EN+lF3HIP%f4g45 z!)X*(NWzFg16i$!oLGzKHt_K`eErTTd9QY7Pa+giFOgD9rN*Fv^YJe3PQAULsm>Pb zBg?4^TH+8u^B_uX!C4-(h3o4m6=4se`#)seV{}}7!|(CQ#J17cHkvfHo5r?nHMVWr zw$m7mZQD-MIk~RqUiWj(I`eks?d-kQ{_pSSr-&TMZi8}%TjV0(6a)zN2YY7|H^YG> zqgGw#Fb3C2qTw_EZ&|m!qHL;N)^!LF2ep_Te<_@H(LlC7d9$ykyh@e3GOKNQPku7Y zr90BRU$=BMc}B9@ep2YPpDA!%9-h+mDzlcLw*k3WeB-qJ;zm2vqG`owip=Gp_0)yO z<*3klPtEmRkJAj=&AGC1RH@YsKH24`x%<7iYvHZ6(G&I0R!_1y_eOFjJ6o^LXX{!n zA2lw|k=E(0Rw59i_j2n`H$>muC$C&CyY1(nU)pGsTM1}i9MRpK@VNsdUJ4-EL>0O5 zG`LyTZR|2#U!>bYf5U{#kA*m$1YUFTBVUA;y8jG%p+0wQY~YS;f2{#D2viJcOek1H99Rsh zf4BimTm(2MWCFJTS_2pa|Fs6damlIhC~)v7xQG};iI`bGp#d^_PI7Eia!M*%N-SC~ z9$IcmW*$XWT2yv+O7^c}pVk1kC?~I&B(InnzpU{$rSGDA_@Wa3aszT|Qa{wCewfLq z*(yl0E33<^Xc_;@4Y+<<1KJjWy7rFxs@z7_uAkz7rK^XfYly9r?I$W| zw*#$d1OGq+XTL`O0}Tu}Kiw_tN>r^6BRC>A~{(+)79L%IVZU%E127>e0gL z+5GCKFL3zhA7bG4a(&^G7`R_QI$J;cyMFilKi0tJ?aJou{qE)Z-uZusfxY|3|0)9q zBOM1PhX)UDhbyCp7w`Y|1&&_d{!0v;&rO}5ET7*$o_|^c{|_-Ra&d8SIX!#1Fn=>V zayK)5KRtVYH2g3*`7k&4aMJy7KJjod_i%mp@bK`d3_PB%KR&+w3k|G1J)AxL{rmj# z_%bo|dUo*kd-iR9;ca#8?Q-b-e*OJn@6Xl5pR2jQTf2Yn=l=eE1l>)7?uS9IQ=qp6 z(4TeC|D_B7{{4Jf1FxUfKwVycAj~O+c3IpuV+bOdaF(P}!B8X)o9e=7ec?zf{#5Y) zZw+M8G*C)gu-lwXjyaVo1h)9XT}u694M-;bYYog9Nu@F%(sGB;|V4OA}I>W4-U z%CuCiHk!{@EK0dlueTcPk7vuY)@V~e-kojo`Ppsv`htTK(WluP42FUmNlv!07>>k} zNaWCe+R_wBFrJMQ?TyF3^TM(64kl76=gYoV>P~euTS6z;@<&{DwAd_7Xh-APceY;7 zPwhzJeRR6p!8Ik~bLDusogB{;%H;OMaa*WN)LSu4^SwV^Z}dlko^?Alp3nE^6Gwb~ zyWbwb%O-LadVAC$iig^NGU@u2eEcaV-LY=?*!$uK=Ige@1VibL=ZnZ|p67$1YQ7uH zq=9YVhv!MU>wD=#T@V0@Hs5n4u`SaD!)ZC$i)6+U-SDIGG~bWrE=ehf;IV8qiWP*S zDT?JlvM>%6r%We~WZ^tLNSvEeFG}E*mokZw)Geoo`KCFhm#Sq+14uGXu`tzE4k<68 z)2}Hv&a_2JFHV)1qd7`;Qj@l%5DwO?1aq_Ja|-))u4)HRaR`v8);p)po(wQP-v(%{3W8VL7HXL7oVY~;*K|` zWq0oGBD_cQ{HpUFv9qQl$Pga0`*fJ5(!Oeom#MLx<@qufx~kX#&BHY&pxxzJHu&&BJtO8c_Bh=zW_5VI zssP6{m`;Y*1NK(}a3t0ZfHRl}SElX;Oa!ysftZ5TPTjTX_a7RCQ?uw&;+^FEN2+7`uIM$NqpN zIeyBlK6uO0fqR{s5blG}E>r3NybuNnBoR$EP@jD(>|32Yi<9nRS6(L>5LO=b0vf&)xwOg_hYj z0HJt2q__>!yTMPvI2U!xuU@rnE*xZqh+}*-26UMeRc0&L?sKQskRx_x!6(MuCLcGv zvsR}nD)`Gv!$q287;h*`=-yG1OI<;1U(Ex2kOupz%a|wwY%65IDD4^1ar7USH5;{T zp5Pi#%HZzM(Nh%sUC@o5$YV(c4Cn(vpm3wWtz5)oy%{zOJN5-fUCizfw+9P%DCnI< znGZY_j{pytl=vK1J};AL89#n|R>LB_ORE9QgxO+Rf2nc7-eAk*+=udzl1gNUa60v1 zB%w{wG_Qp7m^{sK@Qe~1v8J|14zT6o^!34NdrC7RTENll_<@(4%-U_yV;h~2L8}0Q zhO-e0coENQ%|nFaGOW}-Ix4=WJI%NBKRXVr}#-Wo0~LtR_C%#E8FX<;dnq$m>V4i9{q-8*r&A z-)GN=BvWGnX-PYYqS{(7NFxe_#8%z>VPVDJxwe7b{_3K9rvA>~Z)!9nn>YM*% z>&OyS47U6O_t7OZOhR@KE%W6UHKPOg0O}q_=gSb2E=!b|?7sH}8QyhRpDDHSZsFT< zSI*0In5Q*A@=xjE&9d9CNm+*E`|e|krmWGqUC2~2uj98fp?s>c2E_0hqIxZ)j)T@B zWOPeIkCIMFi<3tjo$a70rItJHovcHy`PXUp6t4$8l>Qi}<^og%TX|7}50wPqNnglUN1;3T5Tf)G5 z>q44>xuZ`tuUD5^hdHX9qR(~TJEr9SHrILZTPunLL4%vGNzx2nm^xKkPPjh#=Q7y9 zs;#dr54$$A7=5|+6v{UQ=X~=z;DwJ8$}Sw%`(5}r=Y^j(#<_sZXCC9H zcg|1S>y=_&Ip1c?a-N_^Qn^i`CJno!Ip1iDq^*bF|FH(l_8$kse0Yt$!`1sfp4w7} zHk@kTx84OXh5^ALPlA8;z_(sT6?*Q}Rw>%RK5_5Quh{V9%-`}(x&y@>6h zCGK~k>a#%QBb|z}rDl0X;keQ25sK;CFyf2l?tjs0c;D(zu;SmwMdmy6)1NEgj|4Qi zdjPehH$eRha0Lmx%?vgS8H(GPO+66L%y|Xcf8EUy@x(8e!gIFXutv=nXT?{n&9yHz zh^Q^--i?Sf4B8DPn1nl6Q9W2SEkKz%fKKuY!>GG3b|9m<(F$`2`I9ZrQvei95bA6o ze34$rqTbM=hdqwI zkfhn?9dW}xFqYadzRf-9pY~lBEld=(%pjXu6GJknC@s`8Efnu_uA3XKN*#{)B`63c zGE_1+FfHtEHrN~{JmJW~*gP;*Jv>c4bgwvYamYsqDq?t)QkN^J)IDMeENliR`X_Z* z15V_kWaNr@aEl~iTUfYZnsMJuWbAWfLRxtGQgl~YxXy66;4tMFPQ)Rr>BLG@Nm;0O zTf_^s_lQ~aDtF{ZSpY*o%zYo6%nTT+J=iytPj1s;feI2z1B^coTFTYW6vwAZ5gc|2 zfZ85;)#3-!ACmJD_-oYQhFZT+GWIdd|94t!g=93yJ(|fPdYzjvs|7}+51RrcXcDa! zG6;q6%{~rS1Kc{#WbefGb;rFbE*{YpEKxn2$ODDgL(=Xf#Kt^|V$4yACSoeomD(b< z^TvywCi+WT>>rp!mhdpJv3RvOY}j@P_z+0=)8Lq8NN64ifiY#*_OLCnCwlp&AS2ON#Q|&oF7m_neHF9n(k`ydZD(B##{`#6=>~4x;UlY|D_MuVHeusCNmFbq7;?0X{|5XlqCB=ENl3aV(alUIuI97@rU)I3V+VpW4*R+y zCf^kQBM_iOD>dY;6nip@z#bugK}UmX#tf-WSf8ieAWJ@|zPl`aHMys?L)Sk?J)%5Iasyq_nfRS-U0I*lLz zH!mws8u0j9Do5SO)(IOg2!*FujPwhdoez3LGjcn^s?$D>Lor5ED-)qSMP(wzfZLFz zv=DQ|g2FxyO|+R)uu0%7=^pocPot1gX3Gu)(ASyM!dS3(LAV=^Er%WQFGQ6EE!Cw_ ziKmt^FJR-0_d!*-K?4x4uFCJ`3BLhw$hmIAA=kY zk8T#1&u5n4=?rC_$cPeiYF|{QQIphSMgG+BrO$BuCgVw_qBW9CdGSXD20~|~g8Cg- zOHsY?IeQfZGdx}LBrBAeXeC6YVwMQ(7@g&BzT{V^{1YlTv~*~+8Gr(Dn~p}Co>Vpc zBXHWIAV?oN49{Io5GsZlJT-HWg|`rAz#_%6NjVQ(v9l;DGzlIGT<0c{k)T=m+{_J^ zS6?yX8z8D!8f6)MAn8T{b}d(roL3>w`p;OWq)EL1WIr910y&3_as=5QMWxMq8J4^+ z77+tyaRc<$VE$TF5-U|3>m1!?{ak*3l45>Iwuef|A2i#3ti8$FTWP$`d>t zk}Ughr80|T93;aF>oR*)5CUPV>MnzRgcq@5{L!4-(Oze(tNk;KMOTj=$$@!ZM|LJ* zdhW5W1WhT*)#4{;cJAT5F>>%cf=N3%((WHpdbx*n`021m0m1n}Vt;}?^(F+19 zA5=J~;)dN#$d!Io!ma1d|Ki~kQ6Iod*zT{dGa7LssFeTC4mG7h`6qK0zkz{ZblMux zZ7nEl-IXJ2G{QQXJ+6^weTt42W(PztownLO`1CsiCWirQyp$mDFMRFy%<&vWPXAjD z`$%qCKipD>TI9uC@jh_GagN%I{(-llrFzI-(TQi+k*_0*Uif|oYE_fkbJ)~_g*W=R ziUqS-vCCcHWGKsIeh>>9^PD=_+}V*GCt;b^zq8ktE2I|W>>&gI5TbR8-UoSkBZ7w3 zux}47=-Ct9e9WDqlfj3>lL51_ic+}z>YP?9N;VQYVnQTxK_-{JW!c^9>k*7}1$3Ep zwwnxmvcazWV@VtKPW2hy-Rt`>D*|#uf|tK{PMe7qWZQ51&+n{R@)X9c7V)xR;3-5P zC`QcReRu~1GT5euWSG#qaHc@flf^#1KV|32`PZ-_Ee2f;lveH&f$hUMV0}>{UeAY~- zstC${c~$^E6btIRM_YWL?|kNy>2uz)n|ivpX^2(=HN;%vAT?2{oNy||`yf~tcQYsy zl`d!5Y6vy-S5#(Z))^GF0IR{P2D>`Gsxy;Dgo8m@4Iji1uW*g@h-D27WdVYak~4$p z=Rdz@E_)K@blKG8l&rU(Z1bw%wegse;f5mpokbSj9auf6_&7*dh2=)rBiR~Fuv&k( z#5I*aVs6{>Cp=1A-b)3!9dy0!d~M6LyUOgQIbQMF#o5~Wk<(C*Ql`Fjm?4uL{BoS# zzim=8AF$ONq-$ZGlgy(V6|{O3ufvoeaJ*XC?dp3xcQ@eSwv zcaCqR%_)P2jF;_Yj&4fcH11}MUlG#9s%>Uik7nFf<_?kOdykGc-#K;o(wsogg6?JJ z`(gLhIYifGqUZHD#M`p0Yn-R+sGbYHG^Px*tA?wn*_tbvveV3~E4bbp1D5L^!V5lz z+m+YLj`U0``Kwxx0K>HJl}$wcsqPIF6Ey6%U)MDS@#j-pntSxWZDqapB(afbf5wPb z(_#ILV)}hFMRYT+{Rp(1OGdhT8O0a)ddL4W`VIBk6|B!_YZ=%+mhbUo!c-n#VTAFU zjt=S`H};loTZZ80y}a*fr=rOw~95>3}@FMrb+aLNdP3!AESjIj9qD^`)%WZzTqvxO5 zh`m=dzm5qQBVRn^{=6!vee}43x{SGrB4#$!!zMVs*~l{zntChIeXxZxkHvCq%Y9u^ zXk<5hXL@{C10m0;#6A*kCrf{OBXXaDR`|0{`(PpUd;AO7#r7XMU6UC-vkdEt>YVqn zt&iG+m|Y~4G6*Dc^_eArRt-`+L@ExMT z)DQDSW%{%RSat(rNn~;m)Ey4P(m52mqCD=8Bit)vD0SBGhhd81D7=PwwX9ZZbw)}U zyjnmU>!WZGtXO9Yb~_CL$k_55FIDnma)^HDQOG3%UPb6=qTRejrz_27$F|4Q&r;r_ z^xUn;6&4St_pV-6ZZ+?X+VEw?$lnCK^-(Xjx9vEq+v5FD05m{&_8UP6bQSwPNPOoT zAt;h0d*MW=u$uumhDpUiM7{@L>a>F;IPW>wgxF6l|;{f;h(14lW|jpwI%SZ;ySe5bz}75K6v!Tp85W}YaDq91~+ zalO$d|0f=xVRo? zY`wT!qIU|OP}TOroLG#*h}`EP3ukyzH%?3z(f9&pe^n=$mh7r&0uQb)YgDeIx^5YG zihvcQ#4Rg`%;e!T&8N^|Pb`tMAN51%`H%C0oLNyU zK=QBl*LM*Nw$>x9isV2*1B5+>ce?zo((a zB%S;UQfMedw;{(|eH3x#4^G?fJ*jOMus6y}ow|c%%XDySg#ib|f)MpfAeI-N8*fu| z16Z+@9;l2PpP&pqc*OgGzpDx!P4)%|HaLQ?CDe9abU9$(nj`mVP&~LZ3}Ak&@S7c? z7|gB{f2CI|rj(Wsly{bZDyQzA2p3hSIUxNbY5t3G_$pGLoa}hVRD_-OAj%@h;KRO5 z^x;AgfBJ@G8$}&fS0#XJ6m}0qNQrR@QaSE6kgTgpy9>ZDj0hG&b5MX_s1xW>8qZ&hK1bf^Lxo(V*@wdGUngmUVGt%UC$o z2V;Z!<;O2(r1M~kAbnbfpa+=C z7u1iIi#4X3_74jd>njGy$0k^%8?)EhE16H9Wqd)P`WC(*XP1InM8dJxI_WXTj#VGh zDRTJj2f2htM*?AelnJjh>U<9vOKx-9vHWBE*ZX)oFJVLl@Q+jFM+g=AUy2hbyo&i~ zbpiNK`e(l|c(GNJh(##XMJa0GVj3m$fvE0&5W;bxoTV)qy;SHa1MR2DO$2l0|$#G@RXH@i z1FoxbZ5TeW%YGAP-rNSUAtm9aA8>_>;&nJZJMIyB7G*6@$LLQ z=8kIlEniIHNQZt5=?C#R9;qc6plRoU!HNrlGj4F!U|R_zaH0efw+RjI5U8VABU&4r zFBQd@hjgeKn4yOGea>eR+CPss;Fkel-p`pkPOR~$lQ`yQtd2j|m!c@t`ykY*&z%y_ zMb74eD!W<25p}<^3qt+S5FG1|GvKcyt{t5-5t9Oi-*p|Q>i+nmm18WWY;_iK)+#ej z4NYhKZW1NpQ4o3o)j2f|rIct=*usIOR9=?^v;foEJ77Qzx^qM}R!p5B8!4S*tl{vcgKDJ8wJ#Eg)FwA^NaW@0(afdoow!QY?Jg2l3kr0n5^ys zUen1Tqn2t4C$KvS6v5Eha1@V_u{yFlLdT&D(Q- zqThlj>!-zGcp4>T+BWO`+GPAeFY~f~X%yJ$F!gmkAdv$P0a?O{uMWaQ(}f`?d(6AN zK2|Jmsb7D}&SJZ8F`zTxO`mx*@hf9tm)dz+F){E0r&~L)GF}*%e*-)gjKD+zXPc?l z$;<}ywR4>cPJmOayaVA$dW=xwQsWyt`Ux>V&!gwN)jT?Y#8r@G^}rP%zZW)oB9)RP zteY2C)?LzRW2<^taP5RZE?UgkUEpFN7?m}U%e`+FzxOCU=8mnGIlSayJ^D>Dbn%Tn zB%!UkjP^|g7JD>;ai%x;)3Jf)05OU%G_W?yt%t5u_?*BGztf`Lp;5SJt#97{1TGWiu`io zPr^McCMh0PAo330OjST_eBn*)KgHq0WAO$f!vb*5i?H~eK03>lY%Ac+X>R;knh>>n6hdN}ppH#%I$!9ic51WuiXM#lj_$^JuP03kDE!h!(Qo&ZaoSpJH?01C$K zJkRy3=!0aB40@~KFSKe}LFO_p8w`=QGFp>>&nKi!VsRB=XX13_gF1Q8%Ie`fveayu z%t@VK66A%PMgggjKw59nqSG3Ws_}HEB&@FR5NFA~xhQmfNeg273J3w)ch0$+rd~2G z;;3?>EhUpmbij)F3dYYeW> zJx)EAM};$#k06q39LXx`k1W{LC(`$IW5k>Ug31^G2y3tQ&Wp+qu*Q#Dn(sp-NZXnp z=@}IiArCSs{Nm^B2jWWR?wcz#|ZG$!cz%~?wp)BT9?sYlZ}s( zNTin1dz;SI99WR$ESwdSt7ys)?aB-qW^ihQo9)PX=q);!1fmFX%9b%&Gyo8~XoW|% zSwesErBLs1)oC|!YKd~Yk8MG)$`$~D2Ec`b5#Ou;w4xx1x^5=@Odc5wA%Y+-6fxw< zV9U<@l0gAGl%dU;^wXf|q#52r(k36CP(ewSOl~0g1qBJ3!ju76-b}`0J5D+T7e<*s zI*u}FFc_wwGDR~OoHx@mP8o%JgGmFtsv;}hkA?yWy2-4s@|^+H6kQ+8axHTrf^~hH zO(R!qxmoYYgyraJ)sy#1; zJvwHygdqKF%cmppx7pA|YlPsr^79ta7%;Xx{O6z*f)EP|9O4X+;ilDfqXg!Fixo%l7rAOB0ou=dKU{lZ#Y4IE18GsI zQ3%Ne5DN-e?hVg3vC#+VrC=5RuOth$aejR1&wpU}P^xE}yZa?NnFTO1G&+|^A!qrO@B zMNdoK>QI{iUWZ+AnJ*=5lSgECL*_v=(0DotZ!wx$`?7e4vKqXa6&LSqn{3-9)Q zr~DO%mZzc4?Pkrg4<9fFGviWcGaF4MQ@_Jc@2^#}w~I!x*w|h2%7++XIpogNs3w?c z2Q(2ZFks=*2k{zHZdph8!#G zFaNIa(5^meaH7qw2Pq$pt{P$gmx2yva!12pp2hV?Lq&p$&Vr8rMlwX7YaPUhCFcJxH5MDJXP0Q{;DvH_efO8yhf2I?@`{d$~ z=&lDHuLthw`;Jp4Is#nX<$k0K*j{j^a!X{;)Ist!^A$xHT7O{#Z8@H7wwGuRsep!} z!*!POY-+xInoo6=(Ij)xYK$s~SVU&v@}U@nM`{o9L>leZ2FEZyW+yME?T`D2mZoM{ z#^8L1!loud>L$Jd-PW8Ls3nIQTc#eI-yqra9X1au=5?if4ZQCPFG$BghOWoUUvW)1 zV@+%`9+FK}9?gfrEoev&A~}tJnjTKTo!s~zlI!lF3z$y@Oe-aYr{$eygoNiv{?6J8 zhvk8T_Tc&c^-zHGhj)wFJG138vC%>IQLLqI2@yu}6+uoRLczKsnk#y4^(wSBDdPoKo(tbOI@xOk>pVexzeL7jAXQFNM~V)7EeL(tPBWQ-7_Mp-?6@ ztT`q!ki|2*1Z<*+%q5PjdarV4vbXzlwjKF)u%0ZPp?-foGLtmF0C~hH%(&wC2sZ3I zSx<%N?YC_DD4u;e#DZBkHxeRQl$SMwzi2VYKie`*YsW=P)F9y0;T6BwSh`FwZC|C- zx<@&!0Sr#K>cLZ1A0iz-%hMe#{WxA~Uy#@RsM-FLW1HuAxlFXF_o4^lI9Y|dUdOVl zlegYC*Fb`5yJtj}^)>Z0>17B$>c}$_O@Jdfu|R&dC}7q^0o|Z7IiQEer8)u7Zf`K{ z91xWpuwxx?V;v~Uz#GKv*SAK)Ozm;E>=N!S=aTIZzS!8Vb@YcB~TG&WMq^YhOYfR6Sd*3ZNCpufD1B%!87!y1s((3UZl zeld=Ra)(q*H|iz0fL_bAC3=EuM-ml#W<9exqI;Ow>rs(A&fFcb(rWIXXTBjtkr0euT?fP*4wC&^E72<4_@}REP@9Z|ADN&)NP$hp&b2C&=W44tVeWU-)={u(j zE%5n+)I!(HnhM)ps&`(a*YecjVSJ`uoO=2EL`!DPMe=Z`$Yn{@3MrfNnK#(;0ntUY?*jb^1$88g(Z$AagwQEeaH z>JCHpY#rigg8#zOYiTF&l49YOO1lnbt6YmT&K$0 ziM-aJ?>rpz78^kk-fHjOl%8Zg+il1mLCxRIK<7#c@9X(5rQ5vpGZnMiLsIfO`ixQ{ z^TmDI>;r#t2$%!Qo|P2HZYpdR`*}L=yda5r!4+K>K;e z?y4i+MRvCNCazms%pGp~@7s}wdZkCuVP8D%B>t7Og^5D#i!U*g9*A^?}cHi+24jV?_(}Ixo&pKwCk}1{wOhOP2l;n{?$r$ z+JS`nv<5zSzc&t|)pSxuy|a51;7M|?=#Y9on6g=RQO}9reM+~#N}46$S0fs> z&W4)D@kX-wTBMCExP%#%4=tHiNegeI#Lb&emiF_9ujh&1ZKV`F5NUpyq@wzSqe&30 zN}GC_vaVYiu|wGRH4C+oh!?RvsKAtg1Nm(0@Ipq%QLY)J?qYZx5o7#^+>86(!5M0 z7-Z?^4iEUku!PIDxY*yR{}mx2Q2`N^9VH;oajdgB|JF46amg!#aBo5d*T-%U{?F%_ zMQv7vy^(R;4RZ?$yO2X!3>baV9MW?WSQeEk0XK;vUn+2fU}C1hfxI_`^=M29|1QBs z?W3K?Q~6%Dfz!%^Xqrbs1$7oLy>IsSM;N(SQ*$P6WjAYlKjigB#qj)k1e0x2b{>8% z@oJ!}1LS=Z+siK$k~nDDRTDk_lQnZay^po%xru@s2Ufhi9F)v!iE{2-A@h4$@eU#eh6EvwOpw@`swwJ#%KMyecAn8h1jh#@ zDDV4v@QqR&NyC7)Imr+--c8^^eRTNhV9rz9vuKBVt>_NAJ}o6A7ywTk9%d%5{Gzmc z3L~FN(_IdnECrXvb65`AD}l%z!EjFprE5V>#;g-GjNe z5LD(PKMIEn8|5Q1LOG%nWJi}Ghr^SAMFNx_kyB5P$KQpr6{30G&BXeq9rGm|%r($@ zNI{|ygJ&8eQ$@PsTVEMKlPMa#G0zUV4p@7s>x*DR0i)2^1!2skLLc-|Na`;W3OlND zXueru|CS2>w%inCfCtlTkDBDpZ}|m4eIxcZM3Rr)NrK#t!;ytH!(E&ZR5Km!Z&TwG zAX*^;oY&|thH}h8sMaLA<{4lpo-v~-KOmv?kHmpe$)j&RmBMVDDFxCMX*v}P@Xmn2 z#gpntgNjTTcj3h1ZIM$WDh!$2EC==NcQe^<}8l~}YHH@%+l*|7heeI%E{noHp6EnaM zj!0*F43l18f~8=3DIK)pH=fk*=UGLNDRC^VMi#N4=*}-87iul0+WOQ;Mlac@0!8_#N+>$V4eh7y(t|D4O|fjbYe7#}@9F}Wvh)UM zmaNXX$`(vA;Cp!KQGd7+jd9oTV+qaHBIbupUY2Pm^9^-yV$+dFOW)umvUlu`rZc+C zf{=QG=j?J&yC6*H(GcpCyQfb&)_xg%XQ9)3&RJ?2AxW=cj}M+qY3XTsxd}i`()(a- z@2w5Kq-q*p1Yxy>Q1+-N+yx!x)YqZ!XY3-CPSq%Q+d8|o1)QDTjL#4VQU3%& zve^p9z#g5-EJbnSEyp^XT;38pBLs<%hK>k?*g3lsqdnrS25L+vLc|F*{%u)m25+&n zF4oqM%sUo%4ls#&(Pn#^ZWy}z|nBWL->|Cbpd*Ed^UhWKn){Iq~$$i4FIO3_yrPD%`c7?@SQk?7}0uy)40FoEq_WuPKzgEwJf*0WA@~oE1iO z`49V=*2Q`peA=d`yUMNtv2YD`3Q_J#p67Q*S6wteKYIN(jSgS-N<(aMdBVplM$n-j zRy0@_T*SxZ+&F&uJbkrow*6BYF0s!( zxux0LFaEig`FfBo+xh&NSD3ipUG(>G^9n@?n&XjRWAb2Ac3@+sVRg4aVEUluyiXS3 z1%nu7PUNu~|G9eNDOg7*i{NoH2O%N?Qg;W%G-A&s@dlOfwp;pR5u~q-5mH^02PaK} z8L69L2D%G496?4W2V??QMl{_CI^mk! zIs1&lX#s#Dp*=`Cl|CUxauL^T-+YUB1IXK9 z#g|{noYKjqS!bmR4t_KxNY{|d4wK6U{NlTik{&;h>p771-3Qk_kW(Vx0vy)v=8Hq+ zibEwxL=-8X?Q^moDzgToFc+xEQ~WTTCQ+qOH8oKc@aK*z5UVi!nqv6oQ>-9N#=o?$ zBtP@LhC+)d3@wTj1{REl2Mp$RfXCiMO%qikpjfd&LG;a-vzTJf;Xnh+^tPG8kk`bJ z(Nx!W1Ng@T#ZWE$@$hrAYXa=ZAg$QI_9)h*h@XMA_oRw6QooO zIb@++08nEpO;g$!>RV&s=lM`tTv6IR9U)HKQvF7yG@GUb;%M1WmTJCH zI*XS&mb&?ty5V8D$543;BaUZLd0LuyRvddan32+gQ7{K{ z%2W9)nfZ|VS>%-X&ZU`T_(xtItKm`kbzrUt7SUS?fl zVx3bKb9JKGa1zNX7}a8K7B(Cty)?^U&Q59`h6SRCJUr zO0=a(cFUZJlE(xl(I ziFpxF>(h-!XQm&NrjsB7b8A1(54Y;py@=u6&bXB$(AZ%e$V_7-GoQjAfTFCbGV@zh z%jmIUoJ=Q4rX3!pFW}vF5kNq@(5`Z^47aI@UCVO!h>v>hv+7ZI_7n0tq8#FsiVhij z+bru^OPv(A$qd0E$cn|jkdO4JdcoBG%}5t}LqA4axkmveaWjh(7dx*?AHUP*AwNAo z5V0_9liH(Y04unJXI^`l9rH`h_w9n9CTqibc?MW9P+twc0(8O&Y(g7}L(o`fq%qT| z+K^|o>%s4t{QYxOu=nV~G|LZA1`>pNtX}3F#?JbUy_)S2O|#dH0`-~RBWRBIsQqca zg7a6p*rPc-x(bI(iCMUS?-k1AOwBV@p644O29RJTa@_hpBxd z+|VJE?KnZz^Z*$;769xWbZu5`wj&I-MOpqlP>11U6b4C;YKIDeTsxg%Ha_WxfcHJZ zfhAjP{9Tp&@1EaL2ORt=ZJdKm7mGigz4@~|3t?o*%W%5e#}5cTC{%8G)Vw}$dH^bl z!&u4rWx3QB5wM+M08HqO0vt4u#Nis3vHFLd+3qz4wIe0fENmauUWGMbXhRUxF&;$y z9|%D#tJ}B+yR*F;CW`MQ61DbqH8DedU=ru?qZp7{j&}|z51P-;$9)`np^B=NcXF-HI+-lWgU+(g9EPcjdglx(e3iP}`35pZh6+!$<{LV0 z43=0Vrtnl+KkXeI10IdkI845~x^$FUv{I4ha9UQnTDE>dcZXJ;RJzi8=F2TE=KI!P zeq*o~-yS?26N7&f?AK;Yuv>3&__o1t_vFCD<;2zM^o{G}*D)!Dn@&Q4vl^GPS*xoZ zm)o-{jsBU7TdR94mq+Te3IqgzKgrsbCU zC%>orpZs3rGr#8%<=_>Q;uTZ4FYi}-?-`Q%m)}R^ z{mb5mCKZPzroP(y$kgwz{5~$y?N#6>q!*?ZmZSxHr4==26s2aC^<>r5{uhVe)L;I+ zyrO-kvT3}wETOKcrLMiNzNfx^aO@w7KiD$xZ;Jod;lFD9$-|DR%g*tCDE`=X&tHl^ z-1}>xcWURqQ~bqOivRG6@Q3E_hv$xmm+nVqw*QB~zaHIK9o=~O>+lkN&?n{8xnk_-};&2ZLI7usXNL@(+aW3vZl}SOgx6)vv*tt$qiRkKz9Z zgfCS`)f>w-z4N=Yu2@+2eE<@%^nXJ5b$&wM2%-)-ia!-={u|*Jbd>!?_>9xlX47S# zmt*MF8XdNL|04XAHs_c(3nNY6*MMLDBK&4n!=6xd-2Pb2*W1TXypEKnIhl_5;**vJ zHvL8Tq9croQby&<#VY0Eqh78@^HmxGIe`&3wF}M8J$iCXHs$OvF34*ii1vvkyCSgl zt1>}Xd&6qCZa3bL{iB6C>v8#XFI(01{17OqLdTc8)78;({qa@Yg4?nZDtm>WzW0Dr z*u^#duYyqMp;prh1dL;`u8jT#e%*fyaYEZ3rT>=H7gJr*(D$u$iJ><_XUazKGcoOE zC}napmKUi!LY@cyL&~NvZ3jYrBzF2=gujcoMZ;OUXB6`uj&9ph0L^kc4&?=5`<3Nd zngIF4>2?x-*U%cr$qS}Y#S0q^?W7ox9PA`F3rX#ySrjeKrW!U5nI@QE;nT!B^lR-U zo0XOBWq(2IE6RXyr;boNTWRgby2F*x(uXlyk^1_G(;pNtJ|9?QL@_(jr6w~k9u(>O zB9)~kCR-hrz76B2_bjWTHz}$jhf43~et&K6l{N2LS(VpxmswUgEwv6)ciaqGhkG8D z9oJfrw3XF0MtbE|_DyM?R4>>e(bfHS^QNfJG4s-8TueTqZEj==uWEtljY!pRcrl!{ z!&iBqhEJx)F$JEtA{BRNahIXE+>S|G*IrXsoSWa2F&umT8Dgt$fLuQ2wQX6oo%a(h z&sy=nY(7@?6+*t4I#NkzKOud@ zY1Nu5lm))F}C{Uaeb;*sU$q`fM@9m>?aij=z6iUFnu_d1d%=wwIPtlG z{KIkKH3lSX^)^3^=d4{Oiys4mQNgm-l)?$1W@8S4ul~@oQI!IN@S(F}9-xHqb$cEU z%=F>xFeSz3v7zs?@5F`RcjybH7E%v@syb^pxHz0W_Jnh>Cp9-+JPb;wsNP2hV{FCB z#TIsEp%W2spXpJT4S0)HOC{Q&j)8%Q|K(dNo_a6fF-9-@*15Jam8$2@iJs2vJ=v)S0o zc4Hk2e68V215#n4?LsYe^C>e1sNY$9507czjHM2403QReX079}zwvfY-zz0hz1gxw zF_gMCtc@`APk^WH1eTV>hGnJ~FtZF$Ri1cv-9 zYPJCTJ-jK@-$xzoO~Fx-5Z`V?(;qgEB}#I;FHLRP_l%(j*BM^dp%608cP_5SQ)__y zrp#xZ6Y2*;`j$`y3>Q0EnfBVuhTH%dy3 z_ys`nGEg~>O8khRQl_gk%PKZITZK-6Q|-UVQ~Y3E1?ijpEcL+Zj{2wcHT$g7l~Aki z%32+pHDiFBq9MNHTAL_#Yy?BFICg_r$IfIU4jt_L0f$@@y;EVXkwHBnGFzWYP-&!M z`Jn^a&8=E7dXlC)Qor1<-smA_I|xD6Sb6=c#%-Qt^dIQ(e{O8KR$|VI6qt}%-zOrE ztAY(9EP)SKX$}dpqb!xt%j5;q4f39Ss`vgjT#M_Uw_~?4T+VPT-$%bY`rVi7Qcu|L zwqgd$-hPKr<{^6+G0GC+kKMiwCvew=3tL`mUe1!_i;YW{c?dZa{E%Kyb4woEV00P*VOf zlA;>T#b&$b520ypPQ^TiMCh=i+=aQPE}EA|%ZR~?Q|bWYE<259-vb_dT9S5uoAnOm1z$(^aBcE!?>6t+{X}TdRp!-&*?XFY$!O{jo|*O- zF$sdHROY(utY2&R$qTg`z0;whkCtVy>3V~GXZ?&m2kLDPv(?d@<;HSH+D%?_jbpAA zJkKV&H+)>GMS<>8k4J%MgbmA^E<9`GDbGt9pPD;DstV;!taKiiCOaA%n<|fNoFA9h zlN-Fy;{$+{XdL_UP1iG{r*3VJE9cQo?M&DWFn;;e8kRD367hPb4EcqwXic4#CntV{ zPgAfT@47r!EcKxe3*3>-z0*NEUva!Qu|w`uU@KYQ2`sOXX=rvAuh_;jWUf;~6W#PaMRro^j=I33%z{^FfL^}~P8!$FfM*udJZB`A< zwW6uzDjT;cBp++tclN2bq$$eK5_!I%uH$u+?DND@bk^I)e1hh2@l$v7sMW`p+fH-w z^C;mb*JEPr+Pk*Pgz={xfuHviUwvtf)zQk60GbuO=^#Gq$BLO5Nq>k(Dx7;I=IZ4kSJ|2sSahS1P(kBF6b zVXb&!(IsDaG{aI-!@8Tpx)DOa$?>92cEYDDqV6mt9v6`6 z5C|Glqn12SVZuku-^^*t9&SNhpGgn1SZ`d(gz{N)oKzC3GN=qTR=0 zq=jLJc>)Izgw_oxFidI8(m(~A>dudvYzFc-smfUnL2IRT`@dZOghL99w4O%upMbUrAE zDW!^uor7TsbOe*Zair@;LJp}XlIV&}iO!AjZw)N&92_nHuKW{#rbQz>sZ<(7>oPnM zZbYNmP)aKvp=myvtVMjFRO~S%S=%E$8G-#3L}7n2c-DqAaV$CMbv1oh zbvXd6Hb80aK*R&AdxRM4ev8t13#%oBeC*&|xR}L?kUdBSXGE60Jrsf9k^QOELn;>B zqX>UsnV2=064#I54a|fkgSlvlRHw-azJbvlGWwHKKtyfXvxpjAn)G}P?6mym%I7d1 zhLP-~z#f=Jz9TNNCp+ztAU^d00t6XgSY1j8saM;z^{Cua66vnXbYQxz6ejczWaYlr zSJM&dTws0D^RztDT?t%%)}w3ITl&%c7IF1sqbcDN$P;-L9Hi?C5sYG8OptPgyA=gsK3#xWl(#ut zD0QMhqD9vqaPo>9_tBc?6^Q!%1DE^->B5YJK%zV_I(6xs^wBB+fDAmmIV)nccH{?s zSG!K!3P8L!5Pa$BBynhEmjm;j^?R;YA!=hv@nL>b(Rck|@D=^{NB|r@9eCirAfxQN zju(bhFK(SzT2~NdEP*7JwOivKdO$D5%9iB!6s^_(g%&cgD7A7k6*~!07{munk--Y) z<}xqD%1QoMD=qN#C)hexms@(mi|8wuS!{+`3?le`LY{u0kHUZh4a^JDGr(&V^&;Ff18 z7?NYH6})j5Yy1`v=Ip_b6eoe47tUSq@M@X06E7 zEgU&!9gCS`o{h@=6vnadC8nzhttbtRL2uSU=AWG3$Y|=53%{6{x;n`yvC6Pcw7or2 z#WpEMmuQA*0ppy*s6r=dy@#8N7DFpMTlZPBuU%jYyGl|7N-+SiZ^Y`ej#_rSYzt)+ zS|?2L?(0n$Tw{t78LH#9CO}BsXeYydv2KRO%SNyn-{TRJkJ*I_XAI|-i-?z9OowBX z{%{V47ZVAn`1tYL47W2n^YVTAVJ+}^fj2fR_G=hfN5o^+E<=7tI7-K1M|?Q^#|-ep zB}i1}r^9^kl)Et2twGnJ`Tco(dj!i)vM5i8 zLVX#zL4$z*jI77w=T<*Z9VPFHj$61XH%g%jT6NLSC3Kb~WKCx|<4jHa?jQO1J!=f< zAGCV%9R_`xI!d7ToP6bV?`0!5((}JV;DvrCS4;I~r+u$j&(M_uP#u)XN;bk08U|<; zC|mWd^v4%SgNlCJewJ;Tau$F43*To>eAQ8~yS!YYTXC48X)eDBmk|E>y74U%i#tH{ z#DsGeS=zr%&a9=Pm1dAidN9apF!T#DA5lRyBz?8H(ggy85ewWrhIwt;B{BAjd;k2t z3zjeMRU{lXVSr_-2C|dEf+LfVQ!Hi9frLUJ`JB=kfgCIw(JrfY1#F?7<#xaF5s4|; z^7^h!01ZAeX6UEfUNgebUJ$?Y1kTAo^K7>tBQQXA6wWhtuKj1ekOJkpJN@VEgndmu zJebwP%5CWTAwP$)nB(-+iY&Q`w)6M4x;PV=t-Xo6sHcN{FsLks*m#T5QAhb;r2yo* z_`CQq};bnNsV89Qfg2rimYBB#K!uA{gM6d`X`#Mh>7- zR2e1ovVMhHg|aItGMnxL&|JxelAG?Y`ze^R4`6LmI~y+TMZ^WifnsLHDgV|vYPoFg zasUf`9ayN&DQlYRAw8gT-m#_Us}Js|r0Za6q;_k@`Q5Nw9iWQyD>6Iv(Jtss!!HaU z?MrCB%1*DuC{eemIUIq(N3CmL%%m9b>+r$RbbBRt8A5o{m2RFOQ7SsOyrjS+}3ycm7}bQjHK;?VE<=}qO`ENRjtvaw)Au8zTFdHS{hq31vZ5yS{N1l0`3}bL{p7ZuNRo`96E=kR zd$NoNpAc3&U$zzw^d=EW(&-S>nfK2n_8T>u^S?c6>|DJ`x?%J!hIrA=xJ=JTdJ)s((JRK->+nJ04)(8j;7H>{uU{-#fB1tJXeJ zS~xIHJGnePnIJm7lRIQDgM^=^4+%!^3F00I&Qxb_o#-_p9tED0*fi;#P|cmskeut* zWaZlJ=dzy8|Bgu3J%ut~=&_yC7JrM4E+89UK*i0%V5zMdIA8HOxj;Saiaz@djl5W~ z;U+@gJEYq?(Yq8Gy{dUUoj#CExofL6ek(^5ldlQnFRuP-c5VK#$!z?jrt){U`5L3|uA@Q#H6J5l4xP)_p`h=? z%!2VBZ|`$RA3p5e=jlHTmZoIr#|)@?Nqjt0xjCSFbbEk5y{)<#>b#cwzB`zGk7Dpg z`&(f6^P?V}iJ-pO!Qr1Wg%ej!rihr!-*!jUq#?EF_xe?cWqkLK9TBbehrglrw>tz+ z-Djvl$(`LFUpAng-i;5C)dysSqoAal7*goWN3_k=YmOz?m$Bz=;uLpujOTw4{f!SC-z5{V3x6X@b* zJA6=aa2O|$W);2(2Z5-x-^?rKOX{HVrmN1Xl>9~bNW+I}-#%oh_btsb%$ET>qp4)n zE2b1IfAX7_&)LppIqr-V65dL{AbMPBuP@J^t^S0-!q3SF`5E?xy=7tUTyO*p0w@)V zbQdp8f59OU2{%=pnos1u?U>=I+qIlIN80YOT>4|}$RyQMl;y<*V z8gxdj*Ph_0(G|iXp{Fb4ygyl}@KrGD-aZfZll|Tcp9#C!M0dnmaMaWN<-x#r^Y4OM zK%v1;qaW4QUVNVJ|Ll>HnQMNNy;-N|6b8}73V~5&9faQ_4fVvleylL|8#}D27C_;6 zZ_zkoRaePruZdkFie(<&Df-6HE2ldv(1H;H)Jt>}C$O$(>nG&EHjp6kzP^wk3kXz_ zB#+>9lBA4Lb&?`-cDa-So7or;=ym1sQ$`Pp)NftqDHytj zdSzLbXNm+?G{}{l9LHO(nU!4wz!aY(v8$RIOHgO4m~mlIqB3SDtb(p6@-!Z^_w$rS zjWkcX7G>JXDMp=F9Ro9~=o+OFatz>%M_l5I%K3Rq?2@Rc89_zY?fo>Z?Zm{D&n0=B` z-!ld^q-q*PCDrSj3cyWh3E<;+_BhrS&S`tfx}lodemv&J)i+7P-}pv!hbt1}td64$ z5bR6S%KAEYKg+|7_ONdpCg@>SWFFLuuhV1bxE)8F4ZYI{Z9%y*s%q{`7u*ALk{uEoYn8^B(g~FGfEie@l@6S6v zdAA+!WN?LF!--h+`?puWZv~6107&QVg~nt(cA`-1=G-&RW9x6f;Tk$8F#^;POx~%l z7YYcNCIfA);t#HGWe_ zxd@LY-SqLHf%eX8$5!y4#D7eL8NV*_aodLnWjjU&%aHANHY4%&Pcn%lZYGgu#0iPX!oDHwOqAIe_M9Qklcp0W-_$}Vdrrug6M5cqHy z?Q3LZ3BO7i zm!k+QIV1($fJ^MN%>r)2qH>B`-B)@!Yay5}^#+`=q`XAB?gcH?R$dJM^$Q4Wts;u{ z40dr>;y8ORD@@Oqb@;DDCLa~b7|DqM(8?|fdOfioent;h7`46;^#%{sP^V-tDS!ci zAx$ZEVi(yfu1ilEPHI8qvg*Jfbj2*ZX{b@HsV~4<*Li<9<+3qqFR;**qhNmIK7d80 zB-jAHTt!&iU64}oml6Ks2cuIP5msqf#c6FVR6`JoC7HeFNNng;atg9)_uC`&a|q@{ zK{+l4L!(d|rue!k;*&)`R{L=1LRBQU)H53HwYUCEELV^jaiPMBbpPsuTP7B_sqE3i z@SDgYVlP$G3}b!XRVA&JM#y@yfiN5;R1*9Z^*4a#b^8TSkUfcQGdPH!k3~zUHoPh<)@Bu_ ziH+Kso`R8AEn0qb(}g6UD|O?(}0 zkRB8;aOjUfzqNA4{tK*VC7Xa6{_cj&zm>H55<#UN&3PZc)A-~%(Xv}#Ik35u=E;fH zn%$06t5~}mouYrhy3N- zpP0%I?QS>=f2b5RxE4i&pJSK_MwW)dN!a)Odi3WTYH#A>et}ip>Cb#4#F~oc<3bmZmCr%>kiE z@KtTaTJekDxgi@*gDkgg_T;G!ICsgqw%}`d$Ar%NvqlWCNd*%Efl2#Vs0M7unK>AS zHwkL-K9`?}KoS!W?>-2*mmjK8eUzkoO)SUNn!m5fP#qzue~*bMWrUDI#s4pbNpU|fI?4W(v{EQ?y`ZpY)AwyH$p z{b*gwyWheM?Not7SA|cz%0D}9h(5MigrkNH5o?l zC*79ljrLJw00Qek^JOA8CsHIKgvQ~6bUW+s@IGH}2qyr}AT{%ckuah`)dodJ5e1;9 zybC(dAf^&1!7kJ{)afVNDXQ_i@=F}2w2I{CPpH%tWw$T^{RVAcQpH2ILXR2naT5G_ z4Z!N>6bcWFSi(JlkMzcg-I#}MNNFD*5=5mKAq1Tt{Dj;2iQuR(dSOdC)s{Ll5d#vO z2gC!(-GyObh`{JJ?a;gTRS3YWjNlHd;e~+ApvC$Iijvhjs?hr%$m?|u#{3=}L4>sZ z{L3An1#uNgPN&75b^M_fdXdwKAA!eg`QWI=i|#6VabIKxYD(!gZ`9EII9iV8G?ZFj z4nH)_pEX{;T!f_GE(H>l_(c-(IcDO!Fs9SxaDQ?ULZIk>Qs*5Q$G410SUn2m zB48#}?2weiD|*xF9Ej8Z#h;`zq6o|@vH1yCw+ffCMr1gK+T(-U^+?GP(K%znad>Bs zu#n;a2H;YH6kyBP}4&OA#V=qLQ%lR9xq*H`Xx|jW{Brz1H@tVqY#_IVpXQj z^1I2Dug^ya67grzY}!OQ_I=Bt z9Ya}Z{iIBn64_qm@@X(W(j!e85Zg}ODhCPlrxeCQ2pTpA2J;%Q=j7L6t5Um+5NKET z(?{Wj&kPEI0?Bi#@G*Y%A1Wz4t9nv^YuP{~l*CKY3c+y%3|d7AVairc24pQ~DB&>sy2SIJ;||)L0AqHIg(K;tRCIB2Ky? z7xchK?@_V`6@ZP9rCPdimWrvAAMqj=38PxIOL@GErt3BY{AIwmAPuTr(I7xJ_-s95 z+ZQYDs;bvQkSRh=u%fifv6uj>mV`R%y3*<B(`@=>vTE4+WO`MC()iV)poUCSN zthVM*jsGB;K>f^)|9R@IeutLMO=XD0!;}XT;;|{$$*9(;n*MTW&lMNOZ?zwYR8rnE zI-8?wza>|18;TJ{7L$G$MCR(lRk0%6M9(7`%$j}%!Wc#hseqW*xuaHdTc%y*3|4(M z%%t?$@=MWtNdd@p0bIz^M2`UJ^Y`%OZZA#jv^!?n* zC5d8flDxUKCcOpY+q?duTmp_G?ya#(KZj~ZaY1>YypNt395^FPS8f&m$yA^BMv5+0lu{4YK z{rhWNB=x?f#@QRutH1Nb50M&Y3ahML1HjpKR+o+kwlC}Ru{fRLMqOw&@c{N%a+_Xx zn?CuTE<2k(UmF(WquaEVsH)?JP!;k1%?9=LZ}NMHetQ{>#|5$>#RnlxX3NddO5gl<9b}!{W zxvDm1#I5EUkHqGzpTleyiwX;bHLQYKENvCJn)&H~u1*U_}c-EboW&r~t3O!~Qx zcXbibwhoF;**tA$BkZ$T;o|^jDb2s0Dr}EXSy!TM+VCK4wpg(G4siAk@G)`VM8;gx zgCw|k#P1GBALEfz9MHxc&<7$3eH<`J9pP@<@1pJSP?jGW&)UYU^@$0dZ`w}PdmIFh zKkHc#j5}I!>alpGudZ5LiQBO++bDEw<$XTebkiEX+4|bPG@ZS7KeqA7&h|TSvo6{( zpx-el`vT~`Da{PINTg_+ZLgv)giXM3YP!@12{wxS@Ui)_%Iv?iOC!#3I>a{X!2)_$*UE-PrLUMwg{s^LV8^AfKws&VzJs6zM5brI5An0 z78`}DQ3#zA%dPQGSC@f+#^_Bm9%e~W6N|5|@r8d5?GA%~)Y$Af*-qX_Uf!k`89V+w z&eVH-1iujxay_!U_3*u><%x0B)Pv#|UhNj%IaA5XkGat9xk2_!70-73NRb#6+DsBuzNY$aCLrrC0>fX(@~A46b=JvT2N9G^Z*A2=4Uak=fmFp>a5bC_wN;D5du?4`GlnlBJI5j?G@C;2A=_2rU1(iaa(d=X$XH3Z!WnM&j~D z42;T&8N*4)Nd^r_)?imHO0!JHe)pM+`!2`NA$ulU5x7Z{B#!$(;Vx8!Q5pY$LqO@Jd={i%@Bt1QKsS=%YkjIjt|D~zM;b>)Yk zx*-^|+HJ#_+N_yOD9V2hzM$ZT^ir1lRO(h>;$(>wdUZPxlZaxpmm(l>f!KSuel}bp zBV#W~4Gu4fQd>Qrx<;D-!jZ<-Ur}-qW}frs*%3a*CzA~&_iPf8RO!%;O{}ASQ|F8< z*r}ejx%m@$D7#q@L)CXwc+V8JZ*Cr9dgGyYrJ=e0jpVGUxn*T^xswP}@@xQy&LP20 zLvi|)W;f{BPo;}P*3o9F2(zRauCO{Mp6R+{nC{-5an#ANe3OQai%b{$nBk<#?v7rz zUTgk51M!atxb3XJ>g97m!ypq3@Ce{BOsy?wQ=6-be9&a?8&BbilbTL@WG(7*RU5Nk zNVm7rg3V8EaX&PImz-%X2DFqQOyh-vU|oxccLI3c+S!BY?~8TGRz^rqTOhelty)Be{ze{TKtOH_oVo-1;_~>& z?k&X!Lv@U4Z z8j0jeoD|OtFd6A7!FIGO$omk&Gp%yNG_woEqvPY7hv_ zA~m>9U;-qF*91~C%OT1#wI7|awv5kXU)+Y=xb@@+R;Xv^;2>ZrAvAC4p#@4X2c@tH&u^u#6pRkFIs`Rwr%dZ zk4%amO-**cdKV(KzIC4;V zLkT+t?w6@GLCs2qszo%Oc0zF>8h#<6!cg~6FqWhg9O56i^JuZnE;x_(LX3TR7J&Sp zm}(7&i%#Dh0BtgZUB&a;?3M|Zft>wSBP+iyOvc-`Qjn;AhD?D>D`J;EOW%#4N7N=- z%Iqw&-wkWc?Hz1DNDn};td{Z#JUc* zR%7Z*pECt?kKD~ALaxSW8-3-XND}GMN+bv336Hu`9Dy~^FUR@ciMOF;-hvZuy^rc% zw$e09@1=-oke+Nz>Sxkcs=7GB=*@L?Zb%)F+b7rL`IVpu( z4CcD*y~IvnQnPJJ8m^`BteXMeyNzhQl=+{&&J>gFmMrcq6-Ug{862nclSiIFuTGPy z!pHT5q)i-ZlC=+U^9$>3jP+gAKTe4s4zC2HrV-aaL8C$#PMy9D+6>6w)TXpwYuL3d zvpjmf>&pzObbnsi`E^UWo30f5`qeZj*4q$7ydit;hpZ>WZB0YG?A$gaK)M`cAu)hB#w0O9NjDH z8?raT^MmHhxaiEZK`~W|L?3 zL!}DX=oeUdy{y>)u5|{^$zUzWpkkB&k=hWE92)pUHI7)cUbJZ7L8t$%Bm#rr+uLEX z?HaN?TQOfMl(*4_d-(4fc@1t04f{$q2pTtF1lQ4b6Uf5!4+aedfdnWbVwmZMI`4BE zI1N|B$lSzo`y6rckuqO=-aT$_#ISB|9%w5Ikr&3C17rgR& zrsM*h%m7g#6Vc&KzA57NAZ3`r-*Sg0VTC3ibJkJSH^lun6q${bcHevqH^FG3RCzE_ zp&#R~*KL<8lv6iluo{8~C#%gZse#ZLRE0ExcSbyk6}3&o_=n$nQN{ps$h+Zn8cqAC zOg}1bs>~IB7X0NAN~JeBp?5=t^+5HJbvxmFM;}yVNKzzXDE?lO%7~lVP;l2+5TcM= zsO6-s-CY>dPK9D+rlYKHinH}O7-}XHUi2LglTJ9FtBBgFjM}=Y$hvXYT6!xb7XE$5 zkh$BcsnI$Mh{CLu3b&g|lAekrj@l8YI2H`#-H*fIrE>P#!iY3;<}R-LRb-e|=&DWG z{byH@r0}<7yxU=j`RcAKd!%D>v1j()N97_9Z5k3BQ#Z2>0!M0l>fau7G~PKhzPrV~ zUYTAm>0e*W^L_=pI~9MKTK~dt$PE(2M+4|f zQ=>~QZuU~x5rSKhzsyNUi_!Ucm84gdW{j0)pD(6$8{;1%q@B`zJ1tIIwVaqdQ0yqK z+RsZeD^2#@_6^$)y(!HXv~tWI%L5&Hu|mH4(8xL#KL`QDr;J0END5!@3!>>u1$VxI z=yPwV^U6|+kdk!6O;VG~D#pqjvdZ53(uVF9*J`p^q1B9GBhifB&_B&keAOi&^PAkvQ*Nv z1edpUSf3~|KoGN9=Ac8`^wzpYhvaA^KS6lyFASY8!_D#KRpbnUbJp*O7`v+&@^PY@ z;23*5%G)^9Ntu)4lsSUNG+6}m3&}LssBDPas4vgv*`cBbr*g)Z z5lPz*3#AKFm&qDT69^D^J>75VL9d26h*u`hv;ty8 z!b(_25vIi}#}&a8;;U4;G{abzL*8VJ{$o`LJGQoD^Tpu^n-akH!FomBt~rlvd)GFF zywVr2nB$WhqXvzNJ+Ue}9L7mZx!xbe=s4LR!)Nw8Y*J!A7BAmH>%>g-!(U{c&!a-d z$%savf2hRWgA=+0+QrR=a9tp`Qf7}WA%FCmx9$}BnN2m)&-_PZKT8bQB)2i>zi~i* zio7(qKBm0Bad-=w?~3ZHX=L&(g)dS=y65FcbUxi zO?G?>>5TT$<^m3Z02Oewi*-og(1C*x|5h1flQljz^rXjH0K(*2w~fdKoRX8{0TbYR zfy9_1oF|O(2dZy`S*!{xZu7df4ssUOH&9_nA~SFvJ5kRNPFv|t9VAo8vXy=EH773KCu;VyW05~Lw(&QP+0IUpLG*7m zsaDl!G&u25tZN-1);24XYzZ)HN8Xq#xf}k`D5SNXj_#nXmK=q*gtVFn)xwvV8kY8A z{VcD0;H5xRaRiZwv<@$0VIFY`5rWhBLXM+zfw{zb_jmy8PD?>*>rC}tbE!DwHq2R5 zC%}8*3H~&S@!G&uEz}vIE+$Drcf!7Jw|R3Ia`=WBN1pQ*jdaHG@^z)wHWSf>QH@u{ zInOt2$umw%ky<>11Sy(}RS#j{RQ0BsFx+GX9gi5?JJ+d)3%AWuvc!5y=C=y^4WCJv zHcDCe6Ta_Tz3YAn8&-GyWz?X>QK-H}twFIgRi)4($i<>w{WF{EGmMG$-kNp~XJ3Qs z*5xrZc5RB8oz^^;;nl`x7N@RmSJ?y`)vpbLWz6~PPKF9@?a3Et)^?iMR2>}+q~w{?2-P$7bc+wmLVuKO617E?Q8a{V;2k z<8U(;a65FnHb3Kbny0s8;Z*&8)0fPxGx9pM#N%40X0LGw<#)H8cX3Q`rTQLXqS55B zwdABj>tfwho6YlnfZH;r$vb4i6}o7T%%lI2$^E0VC;E-|8BglYtrYTY3sw`f^wy$= z+v1YPzdYgVym>lxqd6vTd3v>Id5S+llZdmNzqv<2>aHbRX^8b!NC59sC~sIwGeb;s zSaEZB$X&Q{aq!POTfBmZ5XCn_<_M!6CzPTe-wX3^ctood$1U+jqTkEFwbT>vdAeY= z#Itw`v$rH9(8lk1@M9K`2zMvac*ZK;C)l^dI(w#g^QDGhWQ+0DV_Bwt<4Z4YNnaBe z0{pus@C;x?0B{3xKxcpu00;{JgA9Nn{NFVJMDl;t1YrMrO#uJ3CV)ZoziI+_1aF9_ z{+$wdZ3<9w<5SVR)&xiyM9D}PUYi1xxR{hQw3JL-^we+Zx%im)WZCI4IX;MTa#H^j z6yWCjr1shr_#{B|PfdV9OitsqCLpD%DWh)tPfb8h^|OhVsk@e$<$tXSc$jHPSiRN+ z+=K0%ZS6h6UV{R!5dp8@P_K~mFA@1ZvC%NvYwh$8Ke+32pRTC)63@VL|ENyK0TN7v*`*%vz|as_UP==(CpIE?9#tF0ta(T|IrcnCn9jL^V$)ZKRW(Dr34O_{&P)W=bx0o z)y?nS<=;n3EB`+`0)Osz76*4u9{*cM;Nb4&UnznA=m;D=KL59rz<=%tT<(rsjZI$d z_Wa*c0)PH5DS^Agt%vc+hv}Jz#ifUnzQ?QO$Lr0f(~+mssh9s_O5o`T`n&~wm2v;; zdaViEzt#lKb-MztC*G+JI^^~QPmxFm{!Hc9l7-F-MJ6{5Kg|l`Sa)pKJW~V zJ@fbQ!^^+?{gi9)^FL$H&p-eEU-Q~<5mJ~5hX*<+;)o=cXcB}VhN9w%EINl`iY>-?A&fHCs3DCu*7zchHWpywk3a?) zKOX=HO79tk6nN;WBDl1@e$Ba~8B>7$TXW~pU)OjhY+mtHpMVVF`<$>Wo&e0XM# zS8nO%n{b8&CYeQ!xh9(K(b=Dwcdp2%o@~Y`=%9oiH7B5W_SvJNi>k?Ji+|qPW}%c; zYH3M_BI;tEnjV=ckCAo?>7!3xYU-(|KJ=-mp>~qytBAJ$3aY4Ik~(FoxaMl=s;|(Z z2%x+sqG_;>78@n8vD#WJg1R>A?3>8y>7%jK-XW~3zryOJowOzz-?z8S+O3h3K5Oo| zB}PkUtIhCY;B8Eb(Wk7qQER2LEHaDkzWgevE{3auHmqjoXm?sNIPzL>igVtVaK8{o zOkS|wjm?AaS=!hGZ+?ix$1guh4Z|S&>Q#Tc^oL?_J(>OVs#AVku$D<&%V#MI znhgwOrXB>&7jW2K)#gXR3W8)#V31(1I;I*h+@%E5qh5HP00Q4J&3&&64*)^%8r)H* zgcD!^;8akYrdjZYICRLIWGI4@A<#z6_=4g8o)AL+$j5LQ_<|3&WuyrDEeBpJVx#QE zIlvWw3k=Mm7Pknd*hOK9iVGu>QkJ>_tPPAQ#(*rH+OGI^pPg#6CDIERn}*8yE!9!l2a#a%nV{7@lw+&=HYvI^rW`04T*e zk@A?s1Ct4(s5w~vF>j%)keQ-ZFejq2WMUMcy)Zeq-d!fyWBIN<{S~l(J)~i$n%K%xYO&|*D@K(i*}~5C6qps0Wk)-x&vey0)_j#g}PM+t|~_mZ~HJk~tpJTO$^CKD(6g5lF5M&Cad_#P+qc>qfF%~dw9k&?uL`)!R0DX`O98zvX*76 zk1K;Y%x6w>m#N$U;x>2xTt5afc!~TqBOh6@>bM4$>#=1%tJ%+gKC_v{4Cp``TFrwl zw4x8q=r;HGAaUMIen~@T2}OET_chLbaYfrszr@oJOZ1{gP3lq?TGXaawW`O_WK^&E z)q+m%LM}}iTHB`5MeG@_AG+&e>g&;i1WKsUY>#3a``E}%wyIsNY*??F+0Bl&wBxa8 zM^7Yrwj7MMw|zWrd;6r_{eoLU@7*{gI`_1uu``OwP z33$R2h~Z3I+JMpj#E~En@NnVy_rWlaxy->$^P1cI<~SEjOLxxmzDSrv8_#&P87*{0 zU%cq{UbsL;F0ur8?W-oY7dk}FF{np9P^!G>Ir*XV;51+q$Ot<%#!hyzm;LN$H#^!Z z;bC1EqZDwDyWHna_qy9%I)7lqB^t4Jz4P7ge+Rta2SND4_r32E07tq=+;OA_9qCYh zJmlxmb*ckTtf|fXtxGP|M^Rlw9`469MvslsYlHO9NWJP;&-&J%o*1xKN82j_3%lDs z7PqfE0ruC2K=l6iz~95+h3|Xf8~^x+|Fwf84|(P{pXigXJS9mrt*al1^VU~5YO`-} z#V=nOx!8vP>0dv7@|*wsA0z>5zWtSAiBte(UFc8n}TR$O#?DejjKGBIpV6 zw}13!f+(1R_-BIs#~}X)W{p>I1L$#k27NO~X!<673Mglu)q{v+UqFaLn$>l{;bE?K zgcq2Er$-zk7=ljtgiTm`v9W?wScQ3Tg;;2XC#ZidxF9!(abK7XVn}SG7KZ$$gKfoh z*(7An^?>=LhCQNg7l(v&sD$X3fl%mydKiU#_w?}fXlXsvZ!Ka2!p1Eh79P2Y=V86_<0dXfulHzrihAt2!(&hiiEg~g7}K)fp0XX zVq++YMn{db_=u-wjWw8b2dEHK$BW-JXPKBUZ?+~x_!~wzjHI}ZqR52C_>Q6wkDs85 z%&3g@=#1w9ZN-oT9ROfF01Vd%iPiXJ&9)D~&;0Xnb<*w}cF#|koV0shF4)M$zF zRgSa*hv?*s;5bI(Xbb1)8|kQy?AVUA2am^?k|Riu_1KbucnSDu9|1i|M&v{mIVz-lr%Vr$hHq&5CIJk1eJi6bIA%y>2HR{4KG&( z;_z}!P?V*niy&iDt2B-x*O3btWO9gh>u{BJ=#KJ830t{`l_`(O=#uiM1}!O;6XOkM z){h|&VnINc0Xdje=9W4sW#LXh_lF(ji>{mlV|-H1bNAEOgU?tD1;() zn7>Fcj`B(-HA!3R+a_8kTzxkfk}`S*Nt4a8cr|-7S&p>(m!`FHF2;y zhM8GmwMImhIrY^YbhJkJgkQpm91)sDpcjg(*A8Bw1>q-p)TxA8xq)E-2qp>$Z5@yZ1O;LTh`^c52%cWokc(gzu~`5zV4m9e4o7F3`H*F;$z${|q-q(T=jo%2MhPtt zH4@35y_ssi>0_$#Gzkz;dDJ4Y(^BeVb#k>L3bhs@@KWHxCsVLUw$nOBRU}+cLgTX> zn)93&*rpP21?(^k)exddIHGiVQmAyID9Mt__y@XU05A%cG+Knc$pr)dmZNx?p0$~3 z`j7mkVM6*hvHd038!%@zep;mc*OE9Xfs%;}A{YiHP;)adHP?xqupp?dC<%W@ z1p)v703ZivV68FgiYDS}Z4h91d7}})n^6jDKdNYLpaVScu9g~`Bmk-vxt2-buI{=B zU*L~FFav1`sRBuuGD1Kmw0)XaYZbZ*n8XDa`U#WurHT+VAQTD;MWzQkBi>W6hEp-j zB{J9s05pIwSyv`<`k3q(muCS5#cHf~Xq}T8k3w)#e(A3S5C$p#8mL}bt(VXRR;sOm zr?Z6^t~QFJ?*)-Mnq?WsZ1j1PE1+USDg$9Zu^p{kRVa-LGM=Esi#@Bx*&vZOeP zB6^Z9%cKMl0&n!GL%@}NxT4i61{eT1f0K8HKnLHtiiwJjpjmB4O9}h1wfDekNsEh1 z+W;~^i7h4t0LG&ZiHq|ZV4pez@HM0W`(*;_I%42Efb&ah`XqmQ0u-<{MKd3aWMh_?|tBxCm)Ox6wiwEip z01)s51Z#NqxVa*^mIWc3DR4CVpi0Ajc~fL~h> zT>`j05C#iS01_6QA{0z=Avc+}Lqg+9BMY&#fI4W=Q)as|eZxY4O9LdMx5fml6i5h! zPz{8@y@Kn#OX#vMOTNaS!_vBvzw(tz(+4jAIPj~C^4l9q%dabj%=)&gDt zlv0qSh{kA7`vp&1k^IR#iEAyUDxk4L19fCeaHXpQ`%h8zuooaE;r2FEwVko%HH>=?{uwP>-B zqN<3uyTC|knvSTNQo5uEoT>lI#iWWEpo2w{gr(0ry$%}*c~TY>oUjVo9ks(q7R*~2 zivTR32_l@ft#t>|YYPhG3y+`>amvSjOvB)Jtm+891TX_DNdPEd%qsfBn9ImG009`# z%|5WmasbJSthqvqi^YHi5xD~SiL{!Wo&YQbEzkinP{l2lh)N1;Ob`McU;<45z>=7| zykM*KWEK=q0&&Y%7DG2q1Vyh*oH=4k4tlYW^&HVN0ITB!VT{J^vPUdnGXy+7d0)t(%78MF+Bo&Jpx65xsHs}_NUXrH_45`e5+Zy zNNSXvI-B+^c|mQ(qz2GRVAVYU1Pawh6Y8olf=U1LKw_OQdAqOB)T*4VN6W-bt-Ml~ z-P(5)!4ovn=jYbVe3e>Bvn3j{HEXy%OoBQ6l6WB4UuoEkaf?%1i|09j#qh}m2+&{U zMR}Z7)zNJ`1z^t0#%+vKAN5RbB+;1vw#%|3LulKadW^Ax#SF!~*`g%dCo2Jfe9|pz zcVuk>cKHF6dDp)E(scmSb6|hM-I?VqWvaW_MK_bkZJWkzgI%1SdJWV6 zZNve7;DV;V`q`;E4v9xi6`8UHvt+lUF$F_#1VxV4WhAvjo#{#4C}GZ=RN-8mF{n+jO$Ptm0rc=;V6!Uo-d@1=z@&jZGP;??&^`Q z7|-tGmrlU7zLdAV>6H<1o*3%9p6y*8jNER+BWjt4`|ZE&=>5%|Q9`P(r?5e(%k<9Op4&Xd}?to_C9DmOPfAG=1@KjCmwcPL@ zsfqF)US3}B@$Cr}Z;aUgk?|OxzA~!u{8^(v-Oe9xiy<$iZf5e*?cGC9?L=?w5AW-z zUg}Km^7mfs$9VBH5488F?3Zrvf{FDFS!M`t^w#ds*!}gKep^RRS@iDmEB^8^zv^@z z?o^M3=kNyqju>A^;EV0?1!&YeZ}(n?mlx1-nlAK3U-sFJ@++?*N$-Wq5S?hhrxXwM zPUv*~{`PL4e*h2_z_G*$Idh!f`4Z{*pbuc1ANr&pV6hueEw}52pN1*F3W@J8HT>4H zPv&VKeoqhi94PgbpSfKS3=3fOKacKC9P$M}{ORuZ>-Oh@e{$^p-J;I=%|Gw{2X;#j z-xGhcxS#L$4GQ4@Uh^-R3~n&@GPC!x-q^wK4L5fFv6lRo82t$A{_q0-6aD;*+47CA zdbgkbKT$NT}nkKRLv5hYHfSkdA|j2SgjZ14_PGl0Z-09M0N1iWx1|@2e)xLcNHaaYDN)@W7PNhPn zI<>0Rt5~fz&8pSw*REOv*oq3aQ`xg=(XM6t6hPRvapkI|duuG*x^wl?Rl3ygU%-JK z-e_Qv6fb7n*zse>3J6G^T-owv%$YSO;4HwifzJR!k0xCjL1+x1kD6PDFbbKnXYk`=n_pksef;vR?a!}&egQy?;UpGQ3N-LQ1QS$H z!2#k3rjI=?;UmI=DzxxIg~(U{If&Nt@I!ip`)x!LaT9U35{WBLMHYFAfWwS>DB!dJ zW*nf#8*Ri$ZvBw{Q6mm!+i!@R+B$HG!$R(RJ%}FSuWKzlUDZ-B zHso{Gzq};#RxxqakE=D^^tH`7C7ke1VT&~ukA(2lvqM^&&FIh;2Zc6KM5m<|P~$cV z_EBy#@%CF%!X0nYZh47ht3T3J7eZv)b$4BO-<5aX?ueZ?-h1<{H+hHlRS7VOVAgEx3L!QXmYf~h-ny1P@kaBCrIT`U@^#kkAKmQe?F4h39C$6bZ* zy#v?#Tty1^wq93_HrtUmrha;H#fdfGSjZz+8bY;SUK>QLwf6k;u5|%B?1_Ud`|Q&% zw>-kkH+NPyw@aS8?viGwebHxg{#$U~d*7Y+ri3xv_uq-H`4?4NRegEpQ|DuH=bM*) z`lAJqgnI0!f4q@~TtB#T(8JHwYtqX%|7+8u&z{cKy`R0vsP>1?6APRl>Lm)oSV-^fz5hF;! z6B@A&O6#5q`2;=>f)I4%J0A%Xls+Xcu!&B5U)zR4Ke@d_j9|nTPRK|i`>{uNJIu=| zOjtZN#t|w#%wiJ#U_Cq95sySeT^@h9MJ{rVgHiM#6)i@IT~H!GhpYvG;#5TjC1PHF zM2G_07f27ekQ?~eWVL*=Node!jP{@#DNSjcxrNeWyp~=%5f?=lw};pLP;$` z4VUZNW977nNiIQ=9mC|BGo;u-5kk_67faj{ow)`%Ky#89x+F0B4Z?_4b6bQLu;7JjgwvC~DF!DO0fmcpbepB*W*BiZ&WpBT z85&*Xh|&l%NDXeC1q6>fi3iVzUhbaqq~uO>s#AV`NSO55#+QZ>P-0f2pi>l`G7-5< zhx!tA)eNd@tR+%z1fc_mA;D4ZfmLa-w50Z61qlw*02!#1q-FF?6D|;e!>E;7=!9ua zZ8}St))J>btsW$%DOL6X_Lo&vPka2?j!|%+0$Py8FaWyL&m|N8p<$T8WJ2(RL@ttq zQKb_RP7}@SQI)WOf!!BHiG{azzyya$SvkeZO|v$@1R-FpXl5+jb+ zmguDxq2*m`*iPfJ6I5u!C(x$n)79Y;y3wuNK1Umvp$aq&{ZI@Op7B^h9+Z(NxC~|+ zX+_RnNJBMJMbn5VTJ?E-W~>u3SoSlWRN=fB2??t>bi?+1#a(n3o6R@7fGY)TC;I z5Ii`HcM%I?WHyzN0YbuLeh`KaOh&VtmE^`eizC1C(Sq>*O^*wXG@~y(5t}8lC`RAm z)&^L)ej8Q^7Gz)oLNu7SSxhP5_4%uW*q0d2nK`aATwvQpdd_F9FgNhRHyo5+fYlEwML((Tpmz z&N{)90}y73W^&4aTEuEY@Mf2Cd9CVD$Acq0R~C(;vJi%~i(WZ0u#h1KD!@Pv#E@4U z&P53@5P=GOFbQPO(M~=XLA8OP2Yylaxk+B$5?o;IE|5*nf3(*NM)2(G4N=g8Zk7`s z0H9y@chLh)LIq5qg?HOH(vs#hBPC;kMb2mjJ1*h>iTB`{8o+?P;-)64P3@pmucx%L z-UmoT8HB=s+02BR-AXI*ZUK)~!n1=yT(w!PcvB4>#odAfcti$h=M}lbqd{k8B!W9D z_d-@kO=wc!H0CC8x<>6ODK7^jCC82tqc-x!l)V}sdFAOs)a``-QD_Xx|IIA1Od z4|;9VmV#B4>R<(2A7Eg2jMTM+vTy`s08Wt;CC;!6iwbUJgdyCB0x`8I%WvfDnioI` zIiD%tJbk%-ARGvD2fz^kJ6+L$fC@W+fB;7SK)H4EnF_rJfD7Ip^x)4*v0Q*dVWeRR z7CZp!@;+=Dybfu8qmKY8DErL`PBFF*Le$vSgp*(Y-s?(y)gwLYnJnHmcXxonGC8nN zQ0w@R>!a~AYJw=qIeDRot-^3KZ(uVExID?rJe5f~ruYiZ=mB#{y}Y?AzcHLBxRE>H znbQNH(#tcCVSuJn7ti=T#zDSZQk*XEj3uy?9S{wm%OgGmHPLXo+v5|eD}W$dfYpMs zMC+pJTd;kLqOgmFvFpA9K!LI&vVob9*^2=uT)~BaJGwi*=CQl;rT;% z!l@OrAeKTZk) z9WcfqSOjCVGJ0r%0_Xtx14ETiMjklECV+-Yh%GMACdHe>LrDT;)B#oqf*&BjU274# zA}&1~9-LXW2UMr3m@`WNf*|{VV@s~RB8a6E0NazQ>7u>abHsIlLCc|n(@+A%GCn1v z#6Ykv5NSReNdsSK4MqDw3D7tHLQ;ky)BpnbfN(Lp8-N9cok;3z% z!l`n&Crbk<%dLDcD+HuE9XPo&OFT2%!6tx%MPLOuv8~ox893AhdgwamBT8TEu;!ou z3Mc>@z#j^Pjw6_W8hHYE6r6mFrdYwmaKlBQF)=)x!IkvK<^nYu(1e6k#KBTN8hHR1 zu$mWp80)iuM>DEXWGY8O5W>)aMIePL;J=DdLg%{0zd9nAJd!A@nk|{Q5ve#V^Sc6D zJdTsK-D*D@>A#HwK-&;QXRJ&XNk=(r4jZrmtOS8esY(I@1H5}jT4=o;YRR+tN`4$j zhTJRj`?eaX0frMa=xV|Ln_=7*JOwatx_TxAeczj zln(Q|19sfQ=rKrvbW7)gt|d@G2Wde82uok`sT4$jYlBD^00RJBNS@M|?~JdTlZ|&n zhLFjw_#gzuV5}HRMNH^RKhY6BK*b0xSTlExS+Y5V);$Be?)jW*F0b z+ydF0O|fjbth|o@rwc5_*~d8egJ8JI1L#Mj(<#&$13HaM${+$y@v{w04_|--Fmnh( zD1oaBQH8;kY9xeQ5XG+>9~E^`ixGm=nt>`{C<2kuf+zwfoFJ9E!q?EGC1ui5JtLGL z)zM_Yl?w|@OMoc&$j|%84#bLMQ#g+V(B&KeGv&Z{`c3~)&=>Th;B>)sc>-a5&nf~a(WGey(oF+UJ^0Vn*_H(*G{LD1aOgf0L95OmP$!jLYA z*fM|vto*tEf_+e{sa7AHw~U0W&{4amf>Dv}6;35qOmfU3#Y}WPCzU;;bwxQYjm-f1 zgikPnAel?56wuxggAA}aC==6})mfxz#3o6BbUTA*Lb`ukk{#HyaVvzVRLEyVp+Fs1 z;#$;>4WA+G%kLXmueF$xHCS=AT8Vhs%Us!X#jUip&vuOqgmgIy7y?A2)j*t!)I)$1 zFw>x|)yqjx21pGE0Nj3XFFi^YI!L#p^?{zl06)Fc<*CsWfB@gjwy9mnswLZz;aKU+ zT8(_9knP$@{o0W=p~@|>P{mxoFx#CxRhKmH3AHaoDE~q8c66vz#aI(O5yc)`3#0@S3!R0^QHu&eAnO9~xWKh0HmeAGA$Z z)x{Q8l}A!&2B0-A^8Hfa5lcg)TvGu~!{w{T{at&xT+~e%=k!?U?S+ioTG0jBu+7c; zC9P=eUhh4;DgEB~Tw6sM-)Li_3Zyd-`^`yIvE%*S3%*_nvfulyOU{ifOVHGUOH6}R zH|@pT@Lga7ejC?i-IpLu&U@e+x?Ks*i>{>KhOA)u{TRp9V0IheXNj=<{a{Ju*8e?X zS4IGSZT4MP?~ zWk@}$u1ir*Xk_WVYNlFio+0C-J$+!+4hAm&rmSOP zmd15HXO;70EuLq5473vVV+sywYT9Rh?q`YERn3Hq#9--`erd+o44Iy3ntqZivFXtO zMTtgXD8A^Q@afeBYRz-t-SueNr3#KdY68kbFCOFut)QkRR=1mjKNjQ*5@Vj$=Byp) z7q#U8zFtZeRh1S`vMy`0K5Mj2Yqef$wr*?wduO9&=AMyRUp*d5S?F*z=?mFn3`Xh9 z-DX7oSa1eE_OMwyzHH0}4v@Hjw!p~s&4H)L+pqU(ZJ^F!4^q3&Sx8i?XuNLp)RHW$A)Z7cAJ$l z2k!0doY3e7%-KEm9U5M4ssK=?mS%2a0K=~CK4KaYw(jO~16k>8l$LEoW@IEDZ`&^H z*WT&vjp$&Gn^#+J7zz&KE{@ctTcy@3^UYm)uJ8T1?d&!eDwz@x4{;H9xe+gM6BiPkPVt)t zK^1TD6<3m3L_`;#ahsl3#7=Mp*V=+UXz~^typC$cL~ovs@88&P4X1FHc#-K#=Fb*o zx_)x~W<9>{@5CHG0xxi*xpFMuo&z5(9M5qOc4QHbq#rj~2v3;*$c`J4dl6w?;Wsn# zQ*s%&cJ4YWYC5M`NTJODzh|meZG?CW3^nPg(d{n>bFHE6F?aB*K6BaNZDY3ZG*@6; zL-Ld0ORE2YPMshj# z>?R+vC?6b74`Wo;U{CjTQD^1-oo!t9+^!z75jJfg4{{-|WDGxGN|*IYUvg$i>W|KK zyWVx_;x_5-t6pDpLf2eUH>e%w@w79CMppDlhPq~Vb|NnnX;0N>hau^k^E;O|Y(E}7 z@8e(h^=}7v*v9cg*6RE~^fhj9^L9xphoCM`zj%*#wVn6>Hh*EF#&kKd_G`!OCF6FD zXP$m9U965{aVL1NyJcfn<8)8>O=R?kfA=DHd7-fM(!O`*rg;Zc_PqZ0MeKHg2zh4* z?DuE^4=_=Z7i@z50pFT|Geq`dUl9IwZGjj*>V|4|Z+B5`d6;LVho^QczI7+p^qXJy z!VPq3{&=Yg_QQHGqVL~wr)7~O49mF2lUJsFkEclPy`e$cb>C}O-*ByWZ#P%xz#seu zm~V@}c$?pE13hg}=i$bmWVC;n^TbP_5A)C+%|}bvPIaC`l7;7bMmTl#&2O&MH`yDA5%-g-Vs8Rcb@AHG5VVuS99v z(sf(+uHCYk?4s?vckN%lc=--K3t*7Oi)b+aLJ}xx4jna;C2umsSn*|x0vacDlo9fo z$BSKIPF(tQpv|2#gZBKHGV6h{J98Xs`!?>}9#uZWrfpq1XY8D5;X#OygbG@60T2HB zIrQkxmFWc)i<7BRm|zhoz`|H1V+9DHxSrkm`R(b~N0RslwPihTMjVL)qv3ng@N4Vt z@Bja9D~$t|JLBkZK>JRS8AHO-%$bUKUY77?VsvNl4&@56F;3h&*Vx zAxj0&Ggftg6-EnTFUsg%jDoe*%rBkHCF6{Yz!#rmKhAcMYO=Xj+8CkOE z8 z{!2AzdiN_p4NN<|b!w;e?)&mxJa25*f@>d=wPk1f{WR2Y@okve_Cl^TWt#hTDH^cr z$#!eQJ%62w(OowXN8WG$b)Vb&KT^armJf?vY;qe4;DM^bg90L~bD9d^#xzAReK3e( z4m6obXm*w%jHiR%0mJQbM;HytB?F~1Vd`9vK6%oWhz5OL@1yMOZZ?sM|fllb!UsgOhI-^fCU-c7bjhbFpY}I0SpNl-!mhhrQ8@PtM~hByrfsZ1prcc3J#`H*XE{NEC{G`8lTPm1N56B6Wy zg6;r9J@boO-+K5OrvxJ?z>bAeoM4jR z6@#D-Z=lmE7%Yhf!7zaWsPGm79e^-1$(JzJC7J~YgDubzI}zk8gbLlAHo>3?4lLkq zGu-75c{m0l2ytnuSVF%{3Q~>C(u{f#K>%F9Kae8vn_f$%be7qUD4tJ>RKyN?!oWw< z5#R^xvnKu0Qq0fP06)V7m`&YCum~x|f}|?d6{pH6bvkaSTJL6qw7Jxb%@$ecG`Fn(OCO-gpx)m;)KbJuX-)moeWig1W)2(85ZE z+)lMMgK%9P%$Si_xuS6rkzifxLU)HKj0c6^$r{R2RG3T_lQ{kP{vRK?Xt4CtE@HR!rJpJT`#uT@3?R zot=WN9)y9*(3@TxYK>nYsG5LBtVYfp7yuA>WrDMOwCiz@0b&SnW}|iq2A3F1pS3U( zd+N;UXc)F>0&%>uDPp9NxXph$aXcrbl#$63QkK)P2U|eSgDG;W=pMlTsyt26D33Zw-vbn-8%FL+|&a&yw|Szb<1-j?2!9ZP0Hr*$x(it`bFDmeNKCS z0PSs=*ZkWshB>2kP9VKM4CjLPkkA46RzW1A2W)}^DpYq1RHu6KG0M{y#9L(lQzm)g zW>9$Zp1N|(G!~bvo@6&E-fIHmG6ibhxY;*?_baWP?PoK&ol7o?+{c3MvEM9I6F+<0 z5WeuB0&DS&AKiB(fBBChfbh9jZ_BHW0v_z35GdboQVQarAn@hyRPgD_pQa27Cqms3 za`1%L?IswPZR`nb!qETTn8W{1?QO3<-s`^q{r7$S4awczAJ_n%A9YUiIbT#H-~m2g z^npZ+Fj<*h-(Mw}S`)#Xq_Sb^%LSxbpQzZ5~S zDA?$&83Fo7{^_4|(3$`3;O<=$00v;R)L^g~9?vP@JTRaUB4GpiL<4~T4biy;7pT$@ zsFDrrKn`?Z=W*XzL_rV001ecR*V#5dxjx8KJo`;#NK2N03l5gqM_oni!(q zBvvB7^$cE-;2n-xG1cKGu2CF1R@0;(+8v@uRKUl991iXuAPQn2a-Ju$pepKNJSn0s z%7eA!Vi7vx1N|Z}UYd|;q6OxKF(%^}5~7xsqGDj(DDslipkdvm;w?sE__*RL`XTNG zBFVX(Db=DaUgK>j;S%cC8)|jYH@aOnDx`Tp5*hBJMBa%Z3Z$T@BQUaKI}RhHIN{>$Tgq4< zLmlIkF(W=k57?PxNU0>1SfqCm!z2~tLAIhy)+936Bu(nzLNcU7^5jl3aqGWm{Z{lWmey9J42Xqq;-t<}7pX6o7JbS5KohU99x5M)NnmTf2Lx#)ia-+&q@jqYZR zjzo*K`5DlIG`p0bh15Re%bpl(OlS#;KLcDV>I=+F9O+qUi`x zqGB>BT+yO`M(NeS-<#S=n{MZf)~TW{Dx-dlB=)JBMCwOQ-vAU`l3pq@T4kZ`r>64f zp+adK3P7WdDyf$0P5CLNLgY}=W2zb>LxJii8o;2!YBYYTR_Z~enyRkuDzDx~VTmP? z5>>DQkFXAFkID|Q4w1qsE3-DMvpy@dMys?6(X>{pvm$|9^=7YbE4S_{2BNB~W}>}; zD|&@vKhi3dnUSrcD@Cj;yV7R2#;d&E=!v%fYGFL8y-wdmJd$1&WufwknZ25m%4(;o zsln3FfnnynE-b^wX}yx`1ww2k>Kg(StdzRzKXNLfHY~?>tbu;3#Lmpo>FK@t>xu%{ zto|!SI*-aeRib(<%*O2fJuJl1Y|UCMjACr9wrmN?tk3@JE!r%&x(>FYtlxG z{;SR=?aKl!)J82hn=U5)P}9tQk$7lC(&MQb(*Zoa_xd4 zZO)D@+{SGZ32jxPZOM9V+w$yV>McXat>6AF*@i5SoUPq5CZW3RWFfAgDsBLRE#N*b z9&QrrtT4)EdHpzzOL^^ZS1P%>?+yru4M79nUlh1@BS|H%53mf?eNwvW2EdG z2&~pUk4v3v>JqHVY3=JRZ}WyP$4cT9C;<|juMsH0nbj?iC;<%|8HsM6M6w~6C2b8k z4HekS66kOD-mZ|eA#C0u_=+z9%c~@^0J6@G5sa(BO)Lm>2c!;er;^U;tzc#m7z(m2 z_dX9w0l*HOqS*~_0grI6N@4}ukx5_HFS-0bzRZ~%Ce{c469kyHRoK^cjn?IJIH zjGYH}v6{iF5SKBxl29iNKoaBWd5vyi6!GjluZqS71p`k);cZKe-OCV7#&xm9!O#b% zp&1Wy%=#OgJf78BqSARsoKF?S6nA4+DYq1+r@rvM7)5 z6S9Rw6{`q99;$-ujg%Q2Y@Q=0%>$cSUCr>ywNwUAoatWgGvbiznNSLR9ZiigG0!Ur zl|Tld&KS1qxX~IF#pC3PCFm#zm4HPIdMhxWQH z89y{k7pT9{02jc_J4+q(_DZoLvdz5kMc@FnzNHF6BQIZs6~w>`Xq3Kq87L<%&^+Bo zqn|L#;he^_RZ}SmeSl$YQWGP<4cs&N;l&XQ!Kfj@6%9d6gCss1^%DR=S>J>Uw3_wm za6SvsLf5lOjWcG@%UHKHN0q=EPV(ph@l_Z0AsX@_Q?wE{Fs;C{MPIaHJwm^DT@_og z%3U%K3xM!QCRbYj!Y4CyLmRee*C-KB;0N`z8_S4e3q$H;VV~66TsK_zo}k@u^R0IF z2Eeu;k2Y`1^rYen%dGEWo~RhG3~N93GX97akkJSVus)XGa&xiQz9tz-cl`A>c7LZF zML@m_v{w9 zf%j+s2C!@Pw}ziM-U)bEcevS_wD;bxgiB-bVmOM|IPF3B-QM>pfVfk~xbgZpjoY}8 z=N;8DHjX3HCXa9La|5}F54n@ul!{+8fv>kpf;T1m_zyq1mQQVw54d`Z_eq2KEjRg= zm$_$^GK&lEmt(n=A2^VkIh;FcmwWk#gLa9F`PQ`ge+GG+$9a^8>zpg2^p3bzCi#Wi zdB*m+q63NsPMWbk`c6Z-q)$4f$EBrTI;Lm(q-QBOEV`%bhBWYl6C6_&bh>Yb=cwDM zsB>u>`9i9HI;?~GF@?I9f@!SZIoJ7-mnY5)KL literal 0 HcmV?d00001 diff --git a/doc/tutorials/core/how_to_use_OpenCV_parallel_for_new/images/resimg.jpg b/doc/tutorials/core/how_to_use_OpenCV_parallel_for_new/images/resimg.jpg new file mode 100644 index 0000000000000000000000000000000000000000..96885c3c51f4fdc55728acadfd99eb3a32654344 GIT binary patch literal 65566 zcmdqIbx>Se*EiUNAVGr%Pl5$aaEAcF-L-LdcMAjv9^73TCpa_~+})vZcXyjk?tSlb z-*;xds+p>9{+I(*r`hMMz4!99e!Y2~eO>{)mJksa0l>fj05H%W!1EJ;RLI84*51g$ z!N~BfzS~=KTO$*Cdn1Ft2cH)J-vBTFx!~d9;NcMwpcf*_tKSz2777{)^cM>Q`!)J& zYzz`S92`87_r!$6?`bF~Xjqt7zI@Sw`G@}H|GNpE+W{DeFlDd_urTidFEC(WF<_oM z0mM)yUc&z7<)7;%96SQd3s}Tg&+`Dpzn)y5{PXnx@dr544rS!^AHwQ*)Q71q!zl)z zXV+swyzS#WtI8GR-(}S;YiDZIieZ zb}ePJBnh*hE+S;bTnm_~;|Y9>IXfJoWG*R_Ro%w#n2g+GJpcd@BHT?^yWc5*ZGv3T z9+|AdZ*`|si5(fc0+AKc$rk`PecEG{#ViE65JBz^(`sib>D?`sL0D2C z3;R^A{RnrvrHij_bA8HI%FdT~Er2uCff(M#!A$svaNpVPt*_A+4hZ}19J-B~Ao?IgLh zBPj3bE6he~0*xYa{?zpCcG!U-;{+k`_&V_RVwngf1%H@x;UqTt1A?d5*Jh}4&)D9L zmz^T%t%@A&yE`rj6Yv&zxgC@-Ywno`H#-gqmYB7%g_KY7krhmD@`GA@ziyN#$eupb zOBZ`5PR=sdIvwUWb2c=I#Mq?OthS^i4d=68@Y-k}Ph`w;j{*Sb)B<|7$6`(-M|ld{ z4EL7{MAMwy%}r&hrToadUCsTj24Iq=ilplGt)*tg^+}Rv0BoNQ3kB=7a8>^dux56c zBC2`f*k5b6Wa69L0ugvW;1$@^@Ny=K?YLaF{2D5962a^Q$fF*M<5!s@B<1V9?q>Iv z2nBIMzvj5lJ7qkOlJK?W#i|Y`qNeJ$+5Qo2k1yy{p65p4)~nnlR}1m83z)qHIRqy? z+HM06<7E@xgq*h0W%JvbwH5liKQAwn&jA3~UrtKXEBkPH>3ExM=51LD;gHzkr~$49 zaLF;yOfw=47;KTh)~wO3k7@?kd`xbSjMsXtxR zpGO|SLY8d$IF#^k$dr6PT=vSHy4GqT;s$oWho0kPpk%yew~ma~QFb|M!m;?)7P46y zaV)n#rraTxR$sp`36k1m!+G@jvy4TOa#J>zcvI=3gMX)hizrpecTm;frS0wn?Zcs* zR#6t8L6kdnPh`x+?qJgL8M6K{R3!{`8{E?S%wA}zBU;0QfYbY2%b=!d*48Of)IWG-_skMh|-}nlGh!IPX+-;`Z;z zCg!)SCEV8aAzw_Xf(+4|mA4RL__Quo6R0n>ZTv(z3$DjxiWlNe=BTV)cL#K&+{%To z!lh?aB^!sEWrCF``8t+#0RT_%!KFRDzr5@bzUD_&?Rw6m; z4MVsb(Enb-(Wh=n7n^LnLV1ettKL-mm$bz(-8yCNB8A9TsgTKaNAxABb8n(%MjFil zO};DGzE+0nNv90uZYLdGkvA;I0R>%W&hb3XZ&XkCMIw3YQr4hpr0hCF zyNN0z&rVKUJ>s~}ZZS}V8gsr`#!MD1*bKLI`9bM~G1V*hcYwW(l_)j&z@I^iW0^3{ zaH*tIbw|PbL)0MV?l=s0?LzANltMf~#80s2x>o$u%6>U9YxT#UzjK4M{i>1jo89cq zGd7Xjeuaa}DJgx!x!PcYhVvPtXzvQz^p;Q}*4fFPS&!dgD_MLKw>b9Wh>gT~UWL`f zm`Nuu(VKU+Q)M)(&OX^BQ`<#c;MbS>w;C_f1}0)~h%Qz`*&d*N^07e|XsVP5(P`R@ zXIw-#snNnL)oRSOOKJr1jGV*CI(6vt3^}0~@avU-g1U*>iD$i*-jlnd3XSF-xaHM= z(d(UWZm?TNiX(#cIU5HP8x2#D6lZa+SrlJ)@cEMMXTlSFeuwv_OT2j;k~HQq#f$8^ z5rkm~={owbY|Q|Eg!%===eUra@EHpQ1AJJ*N5AIJP_co!tfr4&pU1S9&a7Pzi7zx6 zS7^02gb(*M<(IC_g!5!h*}FxvG$5tS5Bh9}v#c8FWV(Iayy~lL87b(%w?8yePOk6O z8z(X&jM8VyW1BZ~__D5YDKQM9o}^fizT|=?li$nar#VfZW1Pnf#7B5&GS-7kH+6fj z=kDrT4|dPKgcubo-GT<6($1Fl)i+u~qiJ_rI2|f&enBnmxKiFLNomi#E-y=mm9IBt z4ShdvN^!PtNR#hQW;-@*LDbex8l(hGIL&-*e;j{IWYwAf3(v*gYEDrC)yq%d#=vB5 zu7T#4745w9f)5*IS*Pfkrm2Vwv{7>(8FLYYJe4fQowqA~sG}^ta_fZb2BI8qKcit~NCr7Oe3_KAyc$CddXS=&~ z!8lxHt__!xc{Zz|)|i6}b$KRUbsg@8cMIMDzt_2zV5lmJ@GwilZV0}QDfQ~tndgFx z=n1CS6Exf*j9hP`Wdt^Y+LpCF3=UYgpjFA>6K}a_FDP{h_uM(P;uo2lrIFV3=3b)+ zt!z$2D@4(AR5_PVvq>OXbB9xv0T^R7)G~DO9qK}^S-2`tTW}t=ZG1NrF<*HpxmeuP z4|tj&=}}**SUjk=Gco+xvO$$ z$XAo1=pmA?DACvccs@--aAjpQ>UI%!%V8czX{3r;2@TPD)D}c^^N|(|SSA=-tnoO~ zJ)-#@xV6p_fhGu|u2U0l7T|^lR7|4MZqj7b?lIh;Mbg=03Q1Z_=flp;)4B|^+NIV! zP+5d|m+*MF@>W()N}z7^Q}Q0Mcnf67`iHyTEg#ETz|z1mTjl37*+JowTcF8gy=@v( z^&D&H#ZsQgM$vcyfxFYz-E4?qFxi8Ky7ld4IsS~5^FoB()%)LN!KS1}hJx5yi+zmP z+c8c>Co{77sZ6S;sx0myqiEnlc9=^J_6v&buBET%#A)6%vgO!brCo$FpcOaf&9mCJ z_K`x?qyiJSd6bJ?HjoZQpG6okyPZsk{$PX;igR_r1~F%fNf9&EZ0I0VkkPx?ciBSt z=V6bqo)W^Oe&x;$rlHgAr>@jD&GRE3@+&5NJe;2+a8u_U81EB0qJLq~W#Y*8cJ)A-u|xL3B} z!ZefVNQPk;tcfy?Lt41T@bNKOd8P3PgEi{Ur+Aczikx>^UXq*70Oi&k&TLJqLi^4# zOK1+)t*;2R;wt>oMHdLfFPrL3&e8sQK9TA{>A_!N)GjxAim~hNwyl`%Zk@t)2(=Oj zl(`t{Jd$9qNVmDg08%H1qE|z!%FBwARhmc=Hw_(XIXG$Ka9bwibsHqN`yJYdAgnt2 z>lo=@XC?zn!Eh=yvSjk)gT~K*90jm&KG$bp4y8ShWpGD$i~|38ljnNae(F&wlh^k@ z)z63Mj^>Ov4+}J*py0kuEsMrL1K`*ZQ|Jqk!?})f&ixuaF~zg)Mh#hxKeJX66(SHZ9j2O;<0Cf`ZB=-S>P|K$cHgeS{AC;K zr-HAAp%vTdB&tCEo#TNux8gQq=BFv`zHi_{5!{V_! z-t7iy7vy9~&9suGbjJXr88`CB&3%ysGfX7&GWj#%uQ}Wu~Wk43Jn?{jK;q|JZ`DL%lU8i_6f+ zXyW9&XMbf|shL&n{+wsG(R+39QPeYlVZFLMWjV9aVT_4VR-tf;uQIp*8kyY6>NPDp zYJRXZj zL)iF((z#Tr08K?6rl`_|u;?7G-@@Bw7N)q?ph7H1Vch*4x!;*X3dx$_)S!bry)H@u ztnzBZlE~Rk>>HP(Tdirr7cBBpE;h5Y>GzBL%1q_Os?dl|N}#wv;{W5*)FrpOcO4Op zQ=o>KIUnzR)9g6A>ZbLp0`~^qz+_1t6S-yuH@r;O_nA%Qdp?fPtee=!(jUruT1nVd z)~mO2li%>;i{q4_7mA@ek0iPz&6lIjk*vx|VyC#->Ykb6-_bAi(qFW2bH|K}bKe9Z zSdqN+F3U)cbj%50N_g6n!(G@qFdI~ zeR2|0be$r?5*Tl{8l1v9e6U`%zCTelf{r9oiYy!Sk{)Pz@7Hu{88Cc$^$Dsce{&VY zSj~NEJW*_1VY7ZcE}&SQO+OGpF0~sll1rnqgi3HYp(;X!F6rih#t*c43r&l%caa0$ zkx7+u*p-(uCF7#IFI#9OKZp(wMQF(kxHo{Ccv`rGT3nWKRmmZmt&IqS5M`u>^kD))! zokiwu9V}TE-KbSyFQ{3ReBb;Mjy5C3HtDud?4Q!Oa9$NTB1vPJMnK$5vX6a++1Xs~ zqYVRXR`I;Nunx_0zfv@q=kY8XxJcIE%%!1B1*9^DB_bbHMSviSZ1+pmveLJY0gcca z=BD8&l0_#*C4+g2He-G~(gjn+v<)o|qMTQx?J})r>{e(%tge91r?y2Sd47zi>!ezX_|<0LcJ}@EZAh)8cL_-yU3la{W2!hUHgUSIaw& z5bX_=$enVrc-LKYxTvTJDdXD%*RFOs?$xs|08s$$XleQrys=h}6NHqT6n7miP-N+h zbmDaC%D^U>rfkmMa_!i&)6Coaa9d4z8dNSraP4646V=S&)?Q}dSMQPZ3>b)`3oYti z@L2i`tu+_&3dOH$C3<(kMph0WiG^gVLxO75^Za^1+<*l;nl$EasF!_Cq-sV9-tUf# z;p3NAbXPufGYgyhMK}r@PIc79*T~YFGK+Om^bRgjVp{vpfR8SQBb5)kqP^PqTc1*n zAuDdQo9&I(N&C({7Mo&5^dNY^LYAs^b48orw(9nEldJlpsy4=Sk66=5@gv@` zE;KBy*W@T(wiaHwG_{H)r~Kggh52UKd9p{#Knf;9#BCs}YMw>Ls+rtwq`dCO@0jGp z=&3LrYte(K&1SFO=z`!{(qg;6+$!8~kmKEOw6Xin-S&*dS{7m$O68b1F+@^-^@Bv> z)?;`D1a?P@mz`JPyZ&xQQ#5*5*3D#KX`(3iQNyb|UztImk7p_><$xFpeEn~^e4wBr z_J6|@|19$dOMLPAf6J2ppHB+=zu*z}|3^G3yavF)zJPs!2#@ya1soJ6hQi%1U|(Wj zzGH+Fc#ZY;J(Iki9Xzw3!q+HlT(VEBijL7)6_r(E<2d9jLi+X&{%r<+6iTZnr=PzV zX50F<$NY^b!|=a&2AtdMYD6;e%Ri|8b9evqx7>UUiJ^CPtGMhI^V1`Zbd^HeXy~L5 z&S6CD6Ae1_{aU1v(4gM~esycBm!w@DdL=jYUZST$HY%Q4d^tDe6LCIxcm+?i7mKP% z(EEo&a5>Y*-wT0TLx0b7XI;Fu1|{@PDY~g|7CqI;-)rjqZ}*}WhQA2@i}3&MUhMe( z_e811|9v75&BLDwK{QXlCz4;N(RKa3lJvr*(e&DCo!x_?bC>wpZ(8C~#Vyf2{*cFy zf8Xilp?}j}G|MKD{^p@Z_zvSPa;;TTNiz#NK!bE^Xss7Kr2yl>DqdIo=9)bul%eA$ ztIquIEzkeWqjT{e;drP0y)}^epRK6w{`CSj+o3w!A1TPIg0sEJPrd0HQ_7k$5Nq>9 z`m#f^lcE}i6PeRr;Z|^$PBu4M>3q+-Eq@qTsGYnC(Mf&=4E}p;>KTxb3w_*|j(;d< z)v53><8A2qXxKn$O}Hz;tz3+!!?Et?#GKsVU> zZ$26omj2#VBXswU=Q_6ubymgDY$N{PKgKk%&inpfN$+1{x9pu#Tg(|J=Z>7op#fa?h75;irGsE_;$7TPe z)1bgc_3rlcgf0Ga`V_f6Fo!t_0nveKN507N9sD~VBaoioA+Cfx2PTF-!A1+tgkC-*l* z%sVO<9HKN2?5e5)It;LVG?*y^x$%b;lhkj>&CC=#bz`fue!WI2D_2|R-FvE8$!KTx zOj|a!_hh@F$=I~IN86}#AEyEm`&@802+16KWOJU#F!B)hv+EO- z=15T(GZsj*4FpzJUE9T#`ylxFAP(~19Ni1Wtk$*3d6u^7LHrv;$3?qHWm}y+Mm3xF zI8kiS%*#XPY>{s61l#5KS_s@-hTT(|ibCSyk=#9#S$RGAkUe3A9+#W=9=N6{%Hc!; z+%(>SkiFJxH11xKxs86*@0$(h85(&pXpVDBL5yuy^uR@1d?GKsLEO8AzJq)mskKmzpNefy0vEi+{0&rbk|M_WGN)N2 z@Vo7zPCz7`?K`2jQN97HGE^b`wca!*?76HOnRI3H+OIsx*!$A1f~Y!d{4#Gfy*uhZ zoqN<9OwxJ;UuXN0ze~?E(vR@Xv09XD^_4oNO>2dR+@R$y4ISNgbGA zTBqU=;CQqSX4g$oZs-0zSZB|1E0*cxeh5iLagC*Ym#`n zEb^sBeksm$6JZ0k{faYIc1dYS1fJO0$9kv2Jjd}C)xC8`3}7B|O!SFV&j9x! znrz@QoWNP_V-O`PZ~fa=9TWrB`{>9f1;M$rcTvBXK0wYFW^u_ZtVuVuwkHEEXv;c& zxuF!I+2AQbtRSA5`FRe6JtD5XO{-5T8=PKRnXlLg3aoyzS&sMm0E>+g=bB5oM_%d5cpWY^;oczq;q*$hC8~A3pkx)rV)mX4L_w)Dv~0sMs0VzD^_G$&wAn9mWt!^&U`*FcLvsb1Ik$ z>0J(*%RbTfTu)3UwW0J0TV+q_dB<(@37BryoH@quo{@_a>s6CmMd6TC9`hwsH$tvh zsdKspq1T;j#pdBs^}nj%rHO@n=^eg6@>7Xc$JN1`uCynp=-1E~dJxIPUFDaeG^s2T zU=)sq@rzaFZHi9dqKW@#rK(hIVf`l0l7;(qK4aL#bhRq@PWk?|^JcZPWUgC!@?w^j zf52Lo;$8{}PW4OX0VdXbVLlqZZ9uiTYr#!^y2B4D$zrmkNW@f=R&f25SBZZc=T^L)vBb-6@F(x5_(yE{BM2HwhKIr@yQW1=o-Q4IR z+={CB3GA|Zg)a-s=BO=WkI~$JAD18ovW5-HaW04RhRJ$5ULy6$m!9y;5p1gF5NwVo zRK$-s*K2cpL69}Ad@js7!)DzqL1S6=aL@kELLG>l)r)L3x-=ypxvf+NB?Ad&jMF4r zqE*n~GwvqS+fL%J_`QK#TPup~<~j^tz}2%HYtV3@48IJEe?3Ytnn+((Z1gnx4A^}J z{IUL`-`39^J)|1b4=Gx~uAU4h<}~QOZ2bY4>V?)o?!{iPelwK^>Nq9ft4Ca z-}l~g`sc6V&gba*>=LkfgLuE!Zl`H>kcy+vEkkNn5G&edZBs63}=StoM^7lkRCm*Uf%%~Syl>0v)y*+Mg@%50FxWFzH4(R z(qlRPg{ZyiI?j1m0V3pyoBMKIL~p%l@l+&kz?_tiJTTL$dt*lJpFJdHg2>-F(Mzt{ zj&4Grrl1nQa~_g8S2e?<_r_ppt*~$vWP<~1Em9`Py+C5o&ZR$_od(t(7V_J#5hcv~ z!2*XZ{WVb|acJA$@Q-XV%75oarDwqJ;sE~W&E*G2bQc=L&Z-{#ou#8AO{1H=AUR!t zvpry@)EZzO;BV6^v-y2~01az+uQ~pBRkbC_URLFaIP~rkcXW2+Jl8ZD`N4o6TfmH- zjIoVW>Rqht9xAXf%rS_bJ*|xpEwWUYdydtI!)Ik zhylIO47uj1+87?A#-)(LzDpy{c&_oOs!)>wwLHE1#BMv+Ak{tfHD0SdXQ^W4R#v=c z?Q8O! z+K#Zty5)@Lj|DYrQg}T|YC-Cp-)rsbPkb5kLiIh1I~(|TRytiZva4q6yPn1MUPg7) zTBypsves?W(Xi^1@Q<^|w6^&`^K=)*{yv0UrvP=V_k~dm#2(fG)syI?Kieo zg=&nV!|R7eMC}hqsu@-jME5@j?xhS1cLd#>=W4H0(!#h5I zZ;RSB^ktyXf!9{iTa*+^FLr7_QWZI}Bi~YKAVF}VHFcrY`T?0pb;}xVL1{*eUW?GM zWz0K5wa(-FwMrO=4PR82oN7THZd2+iEH@usDyQZa)LcL=sBUFD!rjq}34x)5*%TXl7*HkeZ4cyt?KKY&7MqXEV#|plS z4wRkg7fjnuWG+NzzR0|fs8-wRyj{0P5U#k=?6FQm8Qn|TBLv=$(~a!ebe0?J-vr=( zZEQkt8tM*#+V(X)c`sn;n~8ioy^uDp?LS}fybzj5$mS0!#btYQOcmiAelDu6iF|2n zg<9fs^>dzoPkadJCT)1hN#lK^N>c`0&osbb9>7eRZlzTSPcvl5nF@hxl5@0JQbm3>rI3v6R);3_*s{GLaSrU zmcro0HJk9&kyJ7#qpq{nB;}fSdUZEC8S}LNXU?sv&$PCHqI|1!p6PO zgjr3-jhi{QudK*8gpAFZ9}>yJOJ8Mo#8mp2xm=9Q{I<0MKBeWKn+v_h1?aZVfaaY& zH-wyBjXs}3YajD=6fJ`~6spmHy8POqb)CEe1a;M}B<@P-USqIx|B&dA$&+G|fAV@- zwR-4_MI?GtPb-JYDjRFaWJMy8y61QJM^ogbMTs6V%aGT-tIlC1z~uk{YTv}3;L-4v zS#O-(#wA^;*F&<}w@MT@5oWgo1$~bpp~E+0zDF>;LuWSgGR3Cz{ssq!sXUg;rBd8? zw55i2B&8f^Lg=*rv^akL$|R{cyqJ+vS?!yZX(7QmLv~16wQvJred|ge(?%OdPL5Pj zi9>T62|{lwqA4NgDg8cg4nhxdTtjm3y=keBS%w*5p>QCpa|m?QMRMUq&}SU+JDz**36(siV}RRH$l5VS*{C z$w|oHV|uFW%1M;vKE~b@nm!v$aR)686Z~rb>bv=(N#3UCI`QCp z5g}xIOfv+(I`xR)TUbP3pDn{}3~d%*%wc*YyZ~`DK{DwWxkoV?;|^`nS@xP_Ib3@j zeZJAchg#6qv~2v+UV*78V|0pjXsjIE$*(Z$+@A}tsg}m5uNY?=yJXK*<^0*XzajqC z`P+O@51CQe0&Jg!dszwG@~4^JF+4nT@GVC7EwLY40>|Y?1E!Dj6m=(I{a1<_?Y%ia z4;932fx$-&nl#~2trvl#)&!<0-B2GE8-)X^(|c*vC=fPmPt8_VT+k!96ScyP{B8aB z!brw{lp~Es2(&b_+|AsF=@!eds6X#oYbuAwM3^j+_E zs!rtvHC*SU7wetg9vnC53;C&MgRH4_zEN4R_Z~1hXihuH8ZfBP@5a^6E_bHr+J>93D?4Y2x7Qj<`yGv%m5>LF5gPzklJ1xX(l! zW!1%uh)f&a7*UALh0fFHlWG4@UE!CGpo|!|Uw7HjhIUTuNQxiw>)nOLyb|$q_?#Z5H9i7LJ$XBInxd4)d zbi>DY!{V#@oq_L*NG61;3TJtz5(fg$!ysOWdZ0ny+h)Di%P%R-BkXo7&j20(rX{?o z+^O4CW9U`_Yl~v6lOGnDo;70nf)8FP)-xa-2?;Uf)x3wuT2%g^>9mOxT9=Lv+gO3) znNdR+0c+0Z*Gs+)Fjye{#Q|ttN`QXkwR(LcW6p;Zj&T}*hLP>3D9uetlD@GX!ef^v zB9VU>`g+$i`BP-6_KMTp$XRJk*Ig{E#oEUqJ^ zk0@0d%yxrWM`iI+oT4#=)0+b5*uUBb73*Ujir##3NVh(3=I4XSoBm7_OiCohBU%?Z3hI1#BnVvk5??^3|$abCBLfb+_hN+hi#UAAo~CHA3cl#Ev(0!KY^#71uMv34~Tp38SdYO~h^r=PmQ*N0_1fFifDe3hkFsHBa-86l2|Txd;64)^q# zLJTen&W3S~ff;ZlBVI}v>uC^(iP|$D&Mx=+K$QmzuR)<04BXI!Jj4gCD9EHs8c;I- z#n)2{Hvj1_uA;@0`=MIHNnIDS4k7mMMA_L3nQGAsjdQEGbI=VGmRC|hv*@b-@bW1+h zQ7E$h#=);NT|D2JJ;rbe;a$B`k=jjt`E+4dz^w<_Rl+ks>e1jYd4h+!QVlF@Z%bcw^cm~YezoL|F}S8`En1eaMTrav%@^;P zopg_U{B~worNQ-LR+)QQHn>B4-P5~99v@W0NwhlLH7hs>$&iQ;MT7Gc%gB1tn60?JW-V4f*2bgTVGw2YXkkzj#-tPv`?ZXnPe#Hm zH|fZ;oWTYD9%LRe*8D(l1_f&V%Ir5+Mc&boewl_D>qs`pLgBsmsPdZlEBM*_L#hoU zMYaU&B?y*-{XwDiXa@0d&*_I{J> zFiNArKnJgJ&fyT5jE6akZ*0v8Ew`Oav4$yZmoB=OnS$WrCqtD9e@0IO3;mieD)X3hnEy@OPY(lrJu=cuo)?seJ+2l((6I z?UP3o{)y@Ec;K0Dr~Kf3kKgta++VP(GQYSm2u?(IBQi{S)kb6yOb4E1V3e9uZ%~eJ zaRKx0BO+zz_rN;HZ1;3y*-QIHijz63LQSi;$FXr*qv z;KLx#HM+CQ7pD(ugz@KvinW7w@~L#4hj(#i zTo)FL=G#UlE#!uj6S_NZXPkT;8O<}=1jyvRDT;~uteuHI<3JONdGGbtxu^%I4i zuQkf3a`s36LfE*K;`NR&EK=b>mODQDSm>=Rg3d>1*;VidSzg&KsD=U-AlJ?%++v^# z7Y$oOBuBbcj53Q~Y0EkcT&qO9agRqD)(De$L|L!9a+qVd-O@wMr!WX3wf1%+YdXTL zh2cKy>XHRt8V=D5QGz4~Q%;?J`+VP;%*JkR?cuKb!FLiG(|A0yR4Vl`rr&j?OM(O~ z6SwLeWC84m*f^f2cEY)Zi^2N_jf=LIHOqnpRz)QA$+)6SrNKky;q_z|io(9DJTGZ! z<0``UbRNn6)pvgELPbYrl^(_;E8G~L=u_uWs&GuVS3YG+C-@-L_5&da0I{<(Zab$>dsT(W)97wh_aLL%%Op9y~iWLOz8zr%DpCIh! z57iq`@QA-8=6D@UW7tMgY7@Jb7JSs#V7)2Q!*ed^F#BQd`+1^p>~PmR7meAW8;Mk+ zWm5K5d)uK~r!AEvY{^kpTCx9*3&?k-xfv0pV<#_wD~AYj z$sNkf29csz-oFa7tZ})~S(BLSYFO(vy>`b%B_`-V%$fTNRG()VZrzcGWHQzqNX6HRx6L2JV> za!!ykH-2@|ekg$ugkv@doyUvxpY{hBv?)HCMQf@tGZ{Kr5Bzc4Tqf(V z;b9nc4b}^-zBo=MRQ_ zWLjiHpu}aXMCCe(06P@W|MeuqbOE)#55HA$Iz97XsG%hT$HHOyV2)c{$>c}JqJ?*M{$xUAPx@``H(!O=|DP9?SJy%=a*sO$mA zq3U0^u0{*`K}1tEMVs+5lyr$q$28Fx>y_YWzhDRI(_PgGu#rc7sj zOL>#mCzi7%Gh!|U-Jp1h&RUAc*oRifzc7+Ned`fS77^oxUuai7Nv_k%6fP0HM-bbU zqIzt5aLIe){%t9AwO(|=hztlvJEB9r1-=)KgPuL-_7FFSyv7TYR6B&l*| zaQ~e6wZx=2TN|ecik2a_Q#wRtth$kyI~AZi2$Yjyb7&-XQbfmKo~y@}J}r%1u)@F? zA?70$*N;DP{&*37-^0pZTxhn7XzI%D@vjtmD3N~}Ld-!UBEHBn#Yh)C&d!8LQiDpi z2jeC1v-Ydto`eeWF6piEz5G-;#tKH)#hoIx5Xs9rDhCa$

WLJ-eb_7`wu{Jk4&m zCgo-tl{dfu*eL76{gR>&u5-w#yh9Cf(rCCZM0u~M8f_IMiCJ9J71Qth!6r5Nq5bAMbyf+C8cl~pJNtPBB z+7DV&?+JsDLK24+86$xOko(VK%z@URF1Kii^VDvuguWXg|+?;HPQQ`H>& z-D)fU?%aq1>!e3ddWuXwU)zo#Q@@{^=(cMyt?%%HX|Re~*M^O7kCP=rYZ)!`sB&Rr z9FJP|D=u>$_sCiEcQE}LzEh356elTrRGH+?$aZ|NcahH?W+im2-+R2A79)x!FJZI; zEJL_KLdTLz7V9QsHH-|;(L5I&&o>P}`+|CK=#fBhKxa1i>GiV2SkmR9AVrtDqPk2{(%qa;jgc^ z@3c~t8F|cuWRSv<6PJ*;q+xsUfNANR|r=x_7xHrr58#r$dG z8L$*usK_2(U~byz#}k^Z^eG9o-_pw1-_O6h@m7|qL&vZ1NoD!x5zLq!G-G7%G&T^+ z!lH{3pM<}l*z}9WRi9}D#|BieUL3GjylkxDO!|q=2%1UGx5hWmTW1u;jLCBZ;rSQqu*k^TQwW9Jn5EX{#O6a=v4ZFZKtkg_t&h0CCf$WZNaROqaFJ* z0LhHuwCko81*KIX6BH7iaZzI-0GLn5x}Ujx27FK)p_eL8HqrN}S*5DeMr;6K> z(NSEbsogk-c~3xgB*l5vpctXAZzkCDb(4^+C~N06XgBtf>Pyb}Whn*d8DKu|kHZ}f zX8T|UsWo7B=npe@qY&&MYs3=4uto;TBSU-V|2Ssw=Li>MCR=$&5{$_c_3DxEVN;J# zwG}*Yi@4Uc!+^EVHyjiW{|&c}*ZAozzp_Il0vkFnj^Jt-bnB6dNWfF@i_UUOgS~}agCU4rZl?3j-G>?*uO9B;(55r~U z4xE)JxD}O!jA&#}_d#W{ob~ceGUT<$5(BPV2~i4JGb$2S5gp{}ihjFjr{K0a#;M*n znXw;#yBF(rYVq7Bu7&9H-abJ_8)$r^~FciKMhy z*E)WFI8p;X2nb|KOH12#rk9l`XF8jgQ_GslD9d!g zNgr6I!rn#V!JWomzXn~;&!Sh)fQ2cH z^JlO z#ZIxFIglX%a%(4d=?*QLtq_-6Fwy()8CI8>LNO}Qp>#)H7AuKdx~q`mZgL_4=5@X4 zA$YEuq|dCt2bjzq`(DDENpuz8G0t$Qw=h~W#=8b^jnCZ<8l}uXX=Y{;7e&kJ4~DE1 zr;V7{yD6V}AAGn}G~Oye#}_AT^&3FgC(=2kWNHf+{h`dw*thr#0hKC@f8aO155>Qr z-zE8)?2H7YQDC4NluydNiKu;5x=zx~-k?0-o{}C)7?2*0HLr<tHkbQ{V8LiLX|MbSzAS zW#XbgGO<2;Oat?x>5?VBKjM=I^r=hRo$E%H@Xt}P zI$`;$437_Ih&18YEu11C4r5fV6l!u0->?4M8^PgW*n`f2N4!QNkUX~C*Mhd|8CXpa z`3!|H&PqF*^ItUJjM;JD4qHdHv;&8i`bTzeCOsI({nG+3s~~Fl(pc6n<}dTJwcqpk zqZ!%=*IOA$EVNM{%D?pkBZh*y9rmcJ!%=xO#gwW?jPZ#s85`IYnppF?#GV+B?54i( z<-M&N(&U`UB5mKqRx4hziZ2A8N#t6q3h@b3g}I!aK~Kvu{5dVBImSi>)g|uJue5O& z_w2_aLJQBv$+7tU8zRmzmg?(oc2l~b$hU7G)_bV0E~TRm-+1*!y&>Ekz_fmMVA6i< z6oz-LJ4h9;Dq5%hrk0PUmT{qVUx`SuNI%5h6;xd_ea9E4D7NR`X9oMVeP16^OtV0T z?O1VBpyqMPt(k?qjJ!3@mZEjcD_~iKq0T&5-X@Lg+Ltj2e=ITYAvAZ1XS%jxifRDA zf)z;gnGh5?mC0Ln=X2U`ai=<3#(Maz!`mR@qXb!EkGg=ZAvQ8d0EsuL@0#RqTFuBVO`q5br(~y9w z!o=>>vNI>;aW=|(;^g$`^ukG{2O;94Gm8g$=q85TXhLQUiL(UdR={BJl)#%cNFwJ~ zF}E%&{1U>~%jK|+&K|xBHk#B+tK42R@bjXr)!!D8c>D^xa$e;`iLayoOKreH2;DB6 zOa@eEbEi?N5(wcQU9I1oKn)YqVgzpEeL>SoM~3JHSs<5&AdJWnvIoj)qVm$iC!CKF zy|drb^ro#qc?wftYBGo8BP+&?4KRzR6qBkb6>Cl48=sK5MR5jZYSX^0jbpj?yFxus zK-0Am^#A>okIp76!SL%C=Y%#4LgV)3@m#L(y*ce$RT=WMK5o#gacjLHTn9q4kW~{r z3SJ+2(~{zoF26Dz8v!bEhVYm5g){TRI(hgan>D@4$qCH@bi`jDe=yvNPCQ7O-6qZA z?h24NU@0F_+cLe-b3ssrf5NkG`mb{2>NclAK5WtxhZr-=b=!`U#axkh>eldAk@OyE z1`GD~X&I)*Jdo|CM}>k86~0$Q9F-lN7?0WrE?R&UQec*X5nRf(Z(FoTOr)0B7PeE~ zRVmR|1)JkT-FL=&%+$2lxhQ6R=6)_c8X#dwx+9lvcdQvBQR!HTKqzV(Ap8*B{NpqS z+lkp3q^z{QQzBEf5nGtcEU}%oiPBGi{bc9i?l(upIX68;=pD;RC6`Gw)|Yo0bnC`P zuvwXTs|Y>Rh>q+o@Od8vm|S20i=&iI4|=zaQz?oM>=AQ+H+(TCM1_5&wD4hxZmG5` zWob1pRmqCS26uJbDyWzuiZg}(t-rKZR!#~7o&k4wV37uhFCsbW7Y06q+0w@b?~Cu& z{|8m?{8(4?ZhNPVZQD-69Vd;sW81bHHf(I$wrv|b)(#stW|O}8p7Wl2&s~4O`eDsE z*BtYCK4VOYTa~bjN)Pefjk4I=t{LNnqUJx!{~!j-k>$qDTTB+*$8esWawZe@dDryQ zUG;XenQ}m?t|bB20;D<+FsS=6yCR38qtbVyd^0V}ZC7GJL_DL+m{e>+nvM;KmZ&BJ zLQ7sJcjUcKc>{VKvxzU~r}7Zl_g~LwcBBvZ?_M!OPq+R6R0M6fKT02C(+KZ*i=7+} zp)P+I{Ez;B_Z=*qSW(U|YsGY_q#pK4HQTMxjWmjOi#6wjFHRYgVLPhuOJ6!6N6ZI3mW+mon33X>CODqsv}~W zq`0Ebly$f#PdYhoj(vaA?OUxUsdwA*n_7C|DsgA1k}pIz{DW{-OLZ4lWi?#8KkR!B zr!G(v8&mHAJu)X`J3Z)y$mx@E(3-%01RDQH@7dVJzv?WobDq=Iq^@5Wm3GX&n<14^ ztM?%TwgQUcFjmckcz3MSQ(QDaLWQbXZ4MAtUp=l4fpaTGiyAqX^1q#p9%H`)r0Ra! zV5z{FW9U3?6+|peN)^{s(2Wrj(QX-8LAH#kw0Yy`8R+b{6==cFbf;&W*Tc6xGAs3^ zk-jMNgh?-~ES*VtQaLJ~l4u6?L41O3o_|pLMA@XjM&t{M4G2ZbSHgOidpJ4?Qo)De zL)?xe3wL{}YRr<8OLRZ9bF}K&V)MqCbCrMTnp?J) z?fihAyJpmj@iNqM%SuYgI<;qQ_;eiuQp$JBH49erO>Q{l_ywKSfVURaaJwku4j z_Q@g9(mKHVuordpY8|44A9L_jI#aabY0sC_mV_r3KlV>-b?2}0OCsDJ86;Y_p*l0w z0M*#5RY$^ri$jbOiBYQ76GP|4=ABzsc9k=yU6uE2b}$^3r0Lk77x5+M3@P+`7Yv5PCmXcJUChApXZ zDf;~}=n~5gsriT5dB~msyW9vO-xI8ha|bxy0v%L?YqL1<1dFBwaxa~ogs6N$HHz%Hf) zTi+~xtdrvWRqDS{?$5to>XjIuh$>PKv=n}78Qfj1q7vb*>mDDr#v`uE z-Z-4N1F=L0lVO$QnAN)D38STMj2$XIpGEK~6m9+JZ&a?4-r~i5lEtlJqMfQx3Cfa8 zE$4bZSqCc2h4Fv|)d@lvw!D%Zm+Zgv<8lY1VxbH?rW<~ zy1eK8VcE7*1uHHGU6zvCv6#GCAL3hwS1OB5y;+y)Io+L5aqFb_?rDiM`jkjY7<)GI zADRO~71MB+N4d{LIuzH_7E5np1$|ThK`dap$Zikf{U!=}#J6y1=k%z9sgH?}BrNOv zBrq4JlU$tC{ycpz)ee0W^-T{avp8H6QKryKXE30&AXy2W#vp3#-FX4 z$8^=TC3~&Ignj!r`@P`*fIf$kK;38UMFbRp%7%^Pe#+)+ayC$aU0M09xY5`w>BnrJ zNZ(-VR*bvt+JG^nHpItTV@g~4kGdIS_94!q30DVxOrqm1N=rR7AJN<;DP`8<6SFuU z!?MMS2GAQpLgAXh?=%mZ-Y*(@qZ?mcIuC$I#RJ7PuEdxv5INZPM3S@`%7#(a$SJ?H zwgLu}5NGf{$TG#wEIqO(;2y}GO}cncf>&y#A*+(9Tn$$Lz%MPScX#?KcmE-vr!#lt zW?C1Ce zU`B0%7!yi)Gp5xq;#Nq^Gb;t$_jxY=PoKPTvE`L^Y0o7(flh53eSqV-Ei2(%zSNPx zEHB^j-l8mHN!s9QZw;GKKFz##?z1B(rn(^btw4U|t)kCJ?an7yd=+P+R#fWyDkk&o z@l$yiTL9xEFb>VM%TGdG_8g!!>udvawXBTS#elz%WRS1gZFNuoE$00MdAoAGN)KCc zk-jW`zGy-pTD^(Ba@2nC9|RA>Or9W8_Kk;w3kvk7494iKvUpzC2Td_?cSw|vMw=6f za1}RUtfRP=`rU(R`cs-)vrM0O<^+x0vR{CcwoKiG@2!R>q@P|A0z$h^*H7cFzB8Ge_bSQH)72x(u=xkjuvphwiqh> zI)c2DSPkdg`_p_SBW|5JJ(fFah`Rf@MaUJ%#C6eha;BqA>BC+B7|O+6ZyW6B_3GR; zYdnoDZ_k;x$cxn{B{|uSy$Ydi%f)fQF%e|tTxhyJ_l)t zUypd$gZ@=1?;ihwJj;joN*)lMyL2B4ihW~gtVYNA!p@}BGv?HXOwE)6b3wq6nO^aU zMhm$l%?7AAvrx5fRhel9?f(BeW$zHBUdb=C1>SZW(c+U>_(44$kFX}zwsH%Ixhv9qog+OoG{}noSt~O*o{ruqwv+>_ z5M22xH7_uH6?4N`n&5ETswr9Uo=;vV00p(@MmXKL?zCr{v*N{QxnJbzV#_vus=rdd z%>#iOZ2GXS_)7b~?JE56YBdMJHLHoqI#RJaF!!mmkqj|qBG?k`*}mte#Sbuk|R;T2-65^<^AM*k)9t3n>ytY^Ey<;{L z+dfCc#a|VzRT*byDnng19pIrBaL`QnE%;g889UzmzQ6th=7R{ngm;9pA3mqs0r<~k z&G`fGYnOZYvl9OUFOmKSUfRy1AH}PgR?nVnU01m$8~{XFr}lJPJo!O?$w|stu$rgi zT^iddos~^k{=hO5%vSBZLf|#Fy=Q(EML#4$4p-twH{~3!*d1By?zZ&XtZn`78y6Gu zTz25-tjYT3^7N?1@kv3|J#g{o$r0AfJ3GE8WHdU{D9^>g2@lVgLf5hJvyW+)HC3Q{ z2p%xA>~xhGeM>{pzLPRZon8ZhW8D>u4{Dj}{TRY(mg6sf-ejx~;h(!HK#;1$O|>3x z%TD95ui)hAM7vsi7~a;Exvb9)sC)+Xs6@%${FSTQsf;^~iQJfbU&hY(b|ckZcx8LL zT#cX*LOSW+i@Zhp0=@vR*E_dB&Jfg;DUX3Vs&DMZoluNAR)hZw@a*{Xuvz6?O?Nst zSN?<882rxyZPu0fdn~>H(#SbGw``m}*p0tc-MyM~nmuw8^L_w2DRbWy5Gy2r9>fg`wFNB{XdJT|)5>_I*T<&U{>1#SBB(O7CN=R!^ zj!4B(zKd{{mR?`33C8HXq1a_#mlN7(=?*G0&>SSx0K?fTIvFawTCvH8pLqVrH^zU0 znYh7yt>eP;|IfgU04m}e1#D!f}`gb4A z8m-zKY#9wft@>bd-`L6fmA?!}%!lJBragAq%#nPWcY=8|@Kn4yIOnzx4@@&s!g=q>S{VT1ZHW7hKZ+7^`&b=++8al_KVjkawkmaf9|ZQ9fyBySGagl?I7y zTkE(Lyz*|XD{z)XvmFU@O*`RF%y512C+F;gB5C4?M|R8({+!u!F+kX31<}bN54V{b z6yDmwlIr;5j@1AFr_IHG-fRKDTR>fV#y^i0+lNb;39Ge}qd>s!W0)Ev#aQWKFwSUD z@!Q3yC4LW%CV6S)7lZ-kKx8idXtiIDO|}l**ma1PUU}#-uJ}oJiXjW_-d1*92zgdg zp(xdd#zsBRS|EL zzastDAQizVK5n~ZcSJ97`ymLmEb6qOszjTQI+(y*us7?^tTtm~#UF*~Z)3FIVN2mC zNab%=$eER-o*kGj*>d4l`eZTs}wS#Ciol%FtGh~K>{4s1X&SZ=m&`pKf!WdI2`Ru9jmTat}DM6>y zT|wfdV2ITp(OL^WL(7g84%rp9URP7|sFdqHV#6b@qx7G&kS)Z%oI86GszOh1S$>b! z&>929q}|2kuELvSea|Ya!{7N5{DG06N)|JfaZjX_miyYQx(UGcs#^etxOTHNI&rk9znGAAeHlvC*HOGd^B+O^WwGq_qF zrAS|YtF$PLe}mqRvYxy3^!q$Jtp2Bo@X4?RMZeYS>iIXXUisQPfeFfMVey3B9w+>( z7!nN_4?Vh#{xj_}TYjF)<3-FdTI9^bY*Wb65-D?db#8~s9#BHH;kiU}R}QI5YF+Wg z$}kb{@3sqHyLb}HaI)({)inLmG)BwurY#=xm;HQ;Te&tOb(MXkfvYt=pc4~75b4kwEI`IfJ1 zF}&Dr{WDGz0>tENFrDm}0`?|ojVWRo*gqONH#x=ae4*)@*f#x~e_FGJezL2Bm&19Y zf>LT*Q;=Ofh=r=sxxB&NMo17Zo$=_%{xb<{O_4?`kH6emO5%n+~Wx7H{J3 zqz?XoxKN6#OUtci5;<$yfEW5@Dl7LSUMm5NcOve)6}){?P>*(Ktb~bd*o_tPjn2Pl zxK*mof)AOa3R!R^pO)F$x?%%)wo(Wklm#}CVORcdm8y5}xLsd`N0+*pBSQIJ3R=Ti zW~++SD`TLP1a#@Mf++qQvfYn<;VsJ5hbm03;_*^_#m<{hE>O2% zcfetU1V*c4z=vgP4c{e0{zX65)D@JzMKSKeJctIAr!Nb(85>n@|J0#L#EIm*X-b2H zjHouKrp8b?lTB7Vt7hKr`HKp)bO)bk#S=^v)IfL*baL%>j~8?5d~(3e_W$_Y|5Fmi zUqrOd8j@!&d4{Q+@fbw`9BZ(@&sytB3=w^{3>0xo)~j4)u3nY!EQTEBbOxP`c6}Ja zKc}$e?x=H~Sj`94%oE)aB*g+@UpfV9gQ;KhkI#}jHl-7ey7TuD4&pS2U!Za_bLm_p zjl9g0IsHb;Q`NqhvPdxOz?B*DfwOMHGK|40dCzH*6m&GOJN1-!V%BxSHD<2wQtYB9 zqtW(nY4eiI&Wun9&fSKxbJs6uK}LO8Qnn_^2Wkhr_H-9x`>y$O`WhDc(P^A30VD!W z+8@CZ;4yCFg@DhxVejw-;jJ#$$P2VKVSD#r@fMmo3X*B|B~}^E`?moQRwmnax?%zU zqm%E!1B2<6bMuSjBmi8&HqJNro?EbRwhikN9bU@3wU=g7?c^!HeIT~h+q{xSNOvmv z52A#f3?TSvB+uHA} zE=oeG&QckJB^6O;-mfJ7Rpe_YJm{vr8k5{S_T(IV_cEFK^%$hmRs8BPMQYzMWeJX- zlMt4aLC(m-5a)IG`H3R+Lg4mdABgp9t?Y#4V}3>9g&w zwfhf3s;N(BtxG3vG74EvcryXqiKtHX#}%LWrDiMGv%@VWBuesVC$!h^_jLM6Y@-TXal@{MBg+FAm~ z7DGcmNgcilB3Estog^3UDhOwXKAAQJ4;sMwCTj+jgDG2dn9Ti^St?4G%!K$9&2^kN zdK)=-x7%y(fQ5nW68^Bd*?O15UGU3HWimOd~kNV#( zR@na_Q(azM>*`;2u+6`gpW|Bd-6m*5jvs3aA3p}rp62yqDqkT!ZaZgT<%7uywxOj{`Z;i z{qr>s|B1n3M1%PH^$R2e13ypCiT+1c|t!P(Hn~ zG8P&)bD%+UxC;j5X{4(aB`K?M)_w|7_bScnHT zXZ1Uuueqag-}Ek@oZplDW{lB1|0aJ(-Anypv)jJcBCjeeAZ=zDRWC9&mwLx_=DeVt zK@bn}8#}D?kB%0=%HENl9fZC)iF)G45Fr!!{`Y{uWRE9FueM5c;BZz>=3R5K7J3m* z(jACTv|F(FzVHtM_-eg4S-Yz$6c*$562W=TG$jnhm=xOigz+lB%X7MGxg=t(Pk1C< z<4)RX#gv&8J4T71b2w0%)4~8w_#NO@#?mv+#LCSS1Oxh;t|+mVWu=2C_6BlL6OB3z zv+J01Ssr>wO!NRwh~&58SePcxG^{gyG~`th6*(53*FQg;oXJ_?noVN413hRKe>vAF zh&EnpR5p^`{gBc4^PndmnI_#ber4oBP_ap)a3mTXr?4w`d2eI$KEq3EpMOvqsmI%xXA`ksXU!RN}!xY=5txFR-AU4t4-%}Ujw&$gJMXI7NhH@1+%z63cFP>%>!#TDm*ylqGDxXg>6x2UP_W*>Qc^N_wTvLInIf=|KLsNo ztW-nR);NkWMjTOk9PTayD1fk9pIRFqKD``8vmH|4O%T?$=1q~j_75VkxLXjZ2?tP7 z%hz(Vh^%usW=BMxDgEdvEXi)aXe^|yF8ag}>TPS`QY8#I)Tg%Ek+QbKYGpn~5$7K} zNnlbvbrE@1yp#ryPo$iJo@Cbe@xm%AChtPKa0?VOEd(}+U$JDvBSo{oR^H6#Vj!s? zu)pu@TUy$WF|qM}s}T22Sw5HQzuAy1fSEYdiNEDRxs@*NhLYD|i|f0r;PYnui&lCx zg|*%j0Ta#t`tsMj^s&%+prGrZ18$U_Yr%=!Sh*lRCQQn zh??c`DLaFlZXnafQJ5wk^pys(N>hqEJX;z{$5$m;_|@%^P?qnaoOYFsOk z?fAM=+>qjMrG~3>g}`>vG+66JHNIU(yZrEGv$5>ga!yh>e2s$zW-b+Wo<3XLH3$N5 zGOT2^X9bspf^u^p zRb;gi&RUoytyf@h#3e6R1_e$SZFI7uP))F8J3l@O;t?#apn`(Ef^u-UToMqfKA{XJ1-CtxcFRVh_&qTrMn3>G36EYrt4#Nx;tp zVCGl7FOqET$_#sU(k-dD4?u86`KQ!OvNz+R}-K`xPgQ4SYD8DZWLpbi(ns(^MXWk^0{E zpvhb6X*r!jWv3{&7EBVG3YbUE7>d!<3Ma?z7@YF%fHcXP@qLYTvH@C$vQElQPm104 z$%|I@OngDO)pp|4I*SrCcQi=7l#!qwjoc6=`ah1A$2 z!leKKdx3J`wEWyBIX-y{oQ8pw?8?JMYM#KF{%Yb5C)$u3zWC!LO4zxe*wB4}Cs!Fi zchuXK?j60(nQ``@)xN_n=`+e;0i8weZxon_CkBHst)1_dY;v{R;f%luuP;A)s>8*gMwiU%gQtLYaUwPpu_T%~{|2MQoO|S?y;3fc^l9`B4!+TYRvoPp!X zk&!QkE^-#X#J&^@@~M~rGOu=09mQc1o)+Vph{%JZ8Gzg!y(p3Vb_ioRnV{KiLEYQU zv?&fh_+hE&kstzSC~m+=k>6o+bzbY-dCD-DDDD4Pwe)MvZZQR(P%h$DOhs2~qM4kU z_=Pa*$H^9UwcKC7eT^RSZ!Vt>2G$?B!P27iUaMOk_NXfoHq!L_Q+(0m8|8)+qBc+1 z^lKm$FuOqs@wX6?BUzEebYCNHc+N`-x+)W|XkX$j%=*{pxzudQ!`fd&QTwz5OtA&| zp>lwsOw|aG0~-PXi9rawPi#`4N*CGay!o17_D5o2rt~(;yCr9RXq8s#?s6Q3 zJx8rRD((AP0RE}W@X8@QM`#sw!=BzvYb;HuFa#13%3px%#^k|%LY7Ac#j*kLH-uSDN4OPDfbfQF? zAJ~JI3WrL&RJzt%YqI<*-IR8;ZT%0!N9Ehx;nKbMCK9`}iC)}aEjqXeKz|zqNyNcY zjbT$A>8@LA<}0x~csr)UjpCT2*Oath*#ETu@|16}g;x6nA!YzPWEUEbC>Nl$LImAy z;lQS|h>n+sSx1p?u4wW-Tj|ju+(Z$|xS-c9dMaTC-Ad zk?F6+WTHHgIs{}qiDWrvFR$p^*Zvd75UO1~5zcfpoM{6qUS8i7u3v1HzN9D_d%(vY z64rm(Ivs;Y-2_XiY1wrxneJ0R%wB@?mqJI>qgb{ES<-{uJ2dClnH;8A`;NA*-@YoL z8ycANnmK})pYkLZ_=NEwm7OqJ`9M7ED^f*)z$Gfq5b=ij&V}*8Kxp$YVcUb^gFZnu zC_Pi%iYd~@>;+c>$h4>Op^8-hlP_1Q3$j1VJuc5obZsC+2D-7s7lLR&V#~#gfPNKTp2S$q5O<=@uf?BQ{UR+-D8$ z>8bggHSIJXM-iP)`|bb^_?GgpT`HOmh9Z89hx9fAtkOf~N5UTd)YFuv+7oiQaXS-_ zje9h+$e6?qj$gR-Tvz~d&+S{z-lo87$Ll;a*zaw^a!jiP=TL1SUF6@+81jPIcNP3^ zxOjv|&eX6+(V5p;Z*8UlJ#pbV5giGlpOGyxIm0BnyfSm9)q5s86w86Gr}l6V;eSpj%aTt)7_WriFqo9u@f z^G$P4X<2WEK7$D43stb1i;XLOTdk##lX6}JRW43R7EZy?s*5$ln+@ zLvg!)NGvB^Ndjk(o_9F=>jG{iXm1m?c)TqO-7A9F4{-BEag5gh$CH#}^YAdZgCKUosV;5+05Z zX8ZvbFF<Qfo=U`SfE^clla8uC)Xxg%H5)=*yxUi%hqCP$3>^u~{3dpI%CR5=9C`NZ zS2uw}Z8G`QfDoB!OS=MS7TX4DfbC&w(tv}&Z+KOH=v#Xm5j3-Td_oV)@6{NuF;~gU zY+~>qkdjPZKF>6cgdLa~s|L~?0@du=0{ts0yomwT@~JW zh|eKsHk6!w$Ux0MLfa5`rXGz?t9tDrFV7M@qOFQfDxi0S@LDrdf7_946DL72QT{ss zpw|{}j+Z20 zm@W#qP2UA63tX7C{>A{^a#7be6U=R82kdSVuq(R3`Xar;~97j7fp!l*po;Q&ouCT&6Fwh8tQD1X-^ z>awliSc&S+oN|?GqHPz}2T1l@C}F+Rvaxbs=1hX2l3NDXH`p zcq5kuN)fUkvP4vE*W!Nux1bazJz4fsBJ?60#*mwTE0?dYzeyai*lNDv8kA?Uv+HBH zP?rfjOyjOpT~95_t2F90(*TMW;E}0pjK))1zHhDMiML#jUCfOsbx>Cktk^qD>yX!} z7mx-rfEC10!k)h3cND(e53j?D>Teor63(hO0M*s>q9$l+dxj z8}zOpT+E2bQS1*`X{{5w(9*d_YvT9RlK$dg*|qU7ZPl(caxe7+%!86?P17z|Qornc z%0he-bhfpsbmhWf?NH+Y3Z?p(SEcMnX?;cFSCmKPwL9Rw4z`v=GnLREp=JMJDRPUg z;tE8*rJ{j2(~!5Xd{+K^0}XV_iqAel*IKS;Iu}L)jS9kPjtMgDR7du`BZb%H@6mh5 zIyYiah-UTeqHG3F&m6QpOw^ei%~9=2om8dOyQ#cDZa{_J6GtH|2>2JLb*7BpNtT+j zB|Wq&5n3fWMs7n|v!^AD$;rcc(X5Ldc+iF@*Wp9Hwa1w%fQiL`6d@N#5Kfh-VP(6G z8BXA^4O+qLzw^YP;Z>v$)7#{Za}{( z>X7&Jkp0QzB^1hbClS^qgjq|yON2_+RQdfyR1VwB3`>+9;dE09t3WhN&|rDE5~M(- zVk^*CGHkHMaxbeN^Q7)qsFOhevf;ZNd>>9NF75h&et5}(gXYnXADuqo?1lO84Uc(Y zaM_s`w=H(sm#)8A)@&KpADWZ<<5jm5ggjK``T-9{Pq@*O9oRohM4Jn@)zWn4SFX6W{ zA*=Oe(1q~fSJS967yjXRy?<-_nz89@#l8U1k_$HXPZY>c-*D8A;0n%6%y|yiu>(}O zok?ZYf|olfpOnvtK($m3JXA0|kyGoGk(IV);k!Wlog>(iZ5~2JO~A&u zaEiG_F|#_xA1jp}$u;)Sm=#|PC-_5Z9k@8C3Ox8wdt2-V5#@7fc_V3JnwmkTB&Bz; zaI=;sn(urgu|*3MMPjiD1(2^J;u=f8Pgbr3W2m(A1X;T>bXBChA85Ew$^69}grMVj zZjr-pfsX_;P{-VCY-Pw16U?D0?8!^`cu_;34R|?E>Z?x;f{OoZ=v%NDyhx!5$q*hn zF5yEx$oj}GS@m{=p{OYVF@*(PL_{o-LKlq-lbO=Ot9wW3MVOrY1yB z{)t+Lf>}W38$sj3S^`^2d)y9X(?ap`A8;@6jy!%rRl+rmH0Jn93i~R}(cTk_aH(Xc ziY2P`T8vpMw0nvuX5n^lP}Qy@qZ3BspQL$6^Zdial1&@iHb3tBIJBCrGfP*p)9Vk^tt!W zdVP@zVWMQdsubo!nELXG4Ht%8qRO<*hF=e&Fw#nhh9fy*&D{hLh)o)oYxKI zT1i)(Ru{%~%@gCuasV^pM=~7G~f@FZS z$&&zTDibMJ%3^W2cBsCVA70y4fJ!=YWB|SoQX@Y|M~v!_;fLtHy}r;r_IAtE1gr;N ze3B!LbaU^Z4nll0UVXC2`NFjgf9SigZqQ^`rjYm2qPUK8T)N;J7w0MF8NOzh#$2T6 z(CuxeK&r)CK(HvqbkE8#zA z&HLI;#=oZOhc+H6&c0r$&O0}>pB(y=NX(5!t;I)G4G5?Er7$R*-;@3MSv{z+29J|u z^NmQDW)Z`rbXlNuEmC97OdX%gv39O3ukawbj;PI)`>Qb3QpnR@cN68H7f9BaeX30^ zcb$aU5qyjer!EvS-Npk78!jpK_io#s@^R^owPf2G*~X(Zh$5e}SQk9hk3nO=vWgGR zA}8J%Y)nXU4Xi0?nfShpv-pnQF>};yxweTD?h>qAP_B15NK9Wza3sp-q%%OFI1fN4 zsD?|ZprK&nPG!AZM z=5$sO7yZaT$vfn9eHcsndUM+FXIEja+HFR~so~o?7O|G4m40Bk_-tha%f(GfOhFKK zUD?njJqsafKseTN*|tBB2oUL(5K52A_!mx+Ju+j$FoUzv<^ld`-kX&ux;VwH(n|ZG ztXlWm1l2ap25JIRcBWfwQ~PeGvdy)OEO9h8ZdB6?aYV|7Ir7q0?#ml8b7RR`kj{j* zO;PO*Qwqmin`8mYuyC*3;C_UO@zNI-8?-yr7JvT?R7RQ4E#S>YMoW0}N zBQ%rFV5^HDTA>w3`*+6vL+sd?zJ-@uZTnehdmd8U5=adkwxvwZbpp--6$3Pbty{+C zZPS+5;YP2`1$i&Vhx@yHc6BDT-;gF^ezQ^!-whHeHF!0$y!Ygl1t8_h(E+CI>8x|# z+|7$R9Y36*$g2NAfMZe$8pn4%IHP}upO?WJE3IjxumBjJL>HOB9wjdX5&gM?RE1_) zco)_km*B(_#{Tlu>gpd9@+GJgP?TK^Ip_#aN8J2K6*)9o`e;EC%$)TlFbrJU9@u15 zU>@ziXoYDeW=Mfnn!#me@6m~iRN%2hjUqcI&Trp;nZOs!hOBMUY{n==Jo8}x-fLfC z9hKSeiM?%JuldSrmn7Me;27{VDD#=Rn@%!yTz{Oal(ii6GXiqd+fIfqH|kh=75YJQ zGLJNn8|#Cy21Luu_=w-?=!}Sb^cGQ1^QUJ#%Ya##2b0kvd6hXzkj`XT?1)1YE0($S z5SysX6uWJ3zB1J{B!7Q0V}ik3W}t*km{L?1%Z0id>H5?LzfCDRw%|062;R=Eb&baK zV!CdaX2JNAExRuI0)>@3{D$k%1$j4b1Jv+VxZx2Lh-dxji<%2`3EUd?3RXRROk~QL zv~7hpmnGtPLbqJlvh&HPFokNdc0ne(r3KhARv6NfV9!PUDfgB* z0~X(@g{lc*(?a;RW@Z_rDh|5rEAj$spO7Q6sU|p;@H^*+Ihe#Yj@N_s zO|^Vtb;5|lR%#}NWYXd&RPL?=_l=p_8ifrH!9Qk8DLClmls6^l9Q8)fwWGmvE@X=< z-Plglofw>Iy0A*Yz-3o2wjQMD8Sq!7=t z@>jYoQLgM<-?~a_Rg*--q?)wxm@BdA%&7KiWDd0QeQ0}5{}n?>09EvZ%jSH+t3Wa4 z(YTL<_1krJ6QIn}exq%?Bjl(vSlU@3Io?Cq?>3N;FaT#|+ zH{LiG=~iW&B{m^6+6lG{pr(s~=3*tzR2YH$=>=D14%`n{Q3elAL58k$GS_bnNgFU9 zFB~2mF_x!`S^`!a`$mhFjzw_ z4u4rg^#q>Ep$E#>3~zQvt|)!8AI#r6ZRH3WA025S-qoCEFk1VbkN@{_bxB2D*i1cl zx3(Rrc4-R^NSq1wJ-3Uy+isM)<`oobrIAL&pGa@h^^#Na{dy)tv#5Cj^W^jFIXj}g<<&6{2@Tv(M{ zi6ZD&g<412aju9r(`cMQX~SZ9)0CO9!&ZA@xQJ*G9Sad~>AfX0X?GAKCTS;bCdq)i zudEMc7hZ_ym+vFhIQK9a(BvHUG{d&a7TZKQywq4w}wEdK36G`bBC*&kggzxCK zHyJ8{1^X`=q1>Ei3JKzL-h#M`#&F0NwjaqxO|L2}a{SCQU9qxr{~(5;lK41fL*)FX zlmCFZ9?)_N4+CLQg}`o1`&=@67HFxC;x8n9h+xoNV2)g;Ps1)0wFuP>@Q3$1!k*o& z@~97%Tp?R(ksL>t{wq0;O11P(rMNvcosi+PosPlH3&s5o)KJM6PjbI7vy1kQ7o2L@ z9<3J6VE^$bA8h!Gz8j21)q2pO+mb6vMC}0P#o!{=iYmoJYiLl8sR&0Ii|XVw1Ldf7 zZG~@>IhXE7NPCj3z`+G4*A`Er-60=S($t^ka6we-kOuhU;URQY7CO)NHJAZHT7Nj=AjCv9wU8xS~;%1$OdFD(VDPiDA=e>F%L zsxF?c>sD{f0?(c=vgBi4V+SfzkO1wiFzo{yO9wTf2(o_K_^lxr{Rdz4v!o9gmkZ1B zBUOO9eLA(Wd}%(rNeV2tjp|o*QhqFSBd~-i3YZA^9vx?4!>8E{aFI#{cRK8g>nYiE z5T(Q;?*T34WBA)~1M|XK^O~q@;MS#ETs2D2?y(Uhzc459aM3sK+bv-)K4iw=da2fL zT1^3&NXSaSVr)h2dHLyA@YbUfnA$tHG^@V?f%IpI)rh4Ag-i7lk)gMc+IfWGFrat>DM zrX$nO@sQmvv-Z*cfX~ZRI2_{o6_8Y=%f?Ssf{mNWSX@v?F`3IO@F?XQCFrMPRTAxE zC-R_cPNE30g4BNe+UkyXyy5Z?+xO;Rdn475(n`dTVM(8#{#?zaQBss`UVcu@CwklC z`g6{F>^J(GP;wdkt-22cKbOkzPpA;%ADfL&wTaGw7izEJ4(t<}8#T|8G9?`edMxHwsfOdJ7|BN*f7wMNZU6Gc&{QO6Oblvs7126E zV+9=0oW?H}8h3KMr%D|w_R)BahwmT{^0tQU1q4xYxTk$Nan&*N`mJaYL}-AIB3R$!t$V{!QN zH$CLYcU!3q=9Y=<5SwkrHIGU>LvP{?;1El!W#C^+RXev3#9Fc^(v%0PtgYSg4Qljz zXnQ~_U9YKI++OsB{tLh9kHWJe<_KiV1o{We;p`1X=VE8x37?|^EA51UVUn4kQ6lLx z``*^ubt(qG0At-qu+_(S0;_An^(?`dKpGc0UkptCKL}ZBLant-v+I^4WL2iRbJ{odhDT`eLsg(cFsOK7=$@1JAc6HO?le{5P%3bX&3(V^O-ov2_j3q3 z+b8h&o}kO&wRqs1rgCKIVC5|zxw$6xI zc41b@p$T_LvAOjJD5ywzRv>C}%*vdqAYOuC$j1cvwHx){Cd>!VfQ0%K{&|#7f3B7JBu0OK z-;wnsX{Ir}-kuX(qD>iODxr3vXAPiL*g49mtTodwrNXlk!)}WzI>!Z@e{Rb8A4J|S zQJct;KP3Mk#6$i+0G&W$zsF7`Z8t-erVWJIQKpdxEzZ(gzQVp4M4u?sD%~6Pp`@~9 zZp;n$SMvGHRAUCUvtv{a&VT72LN_McIMCF^wz7!qE&9Yn{qkjLV_joWMXd&OCuCbD zGI88aJdJrmxRHor@R@<3Zz*{n>6#lsNflCCQnC9NoV}$QjX#?N&SYXx;JHZlV_(+L z!L&kRSt`p!#}fRCbo6;fb&RcryuHtKbex$Rry_OgBvvToa_=v0c^2>7oXk|jtPF-O zU&J;Zio66?nMh7vB2ASBa(=D-z(~!n&4oK`r#YAe{xV#me!|F z++-%CuU`F&dFBoc4dYb1q8F-uwj+lF^p^Xy|7NU>(i7rSLm+)b?nMbpHURSAgbRpOEw;3&f2sV%Mb98Na?wLqfa(CXKNn zDe=)QkHGRsY7mC$8W5XK1zuU5dJTzDD~d6jgko5QWYcp~LbagQhD7`FY9c#O-3{5Ade^W=Q>2Df50uf<%i@Rnp~#b>(9bGtu^ zZ{d(7U9?rQC0SwSinCw4McLIux)yZ>#;h3;DW25mTV*<559pe}*8)XMI8W9IgB!G4 z`7g+^B?jM+Yc`JoAYNRcyX08PB&p_7K*-VS{{RBH8i`K2A^!k^UG6)`M0j-3?na;C z3xL+5O3Ju>)r$ghhQjvuCPP>*6KbLD1-P+3#XHNv5N75SHRl3tc_$pW9vB(lg9bsr z$&8N(Jf5{z`LWJOT&{Js*>zUlf!DgNm%~?Ac$n~*eDQM#EHVD->HG<6!1L*3Pyre z$n>s&PSv)-TNK%n`B#||q#cD+v5bB8qf}CVf?mYUA?sFMP z4CNyX*)WuBkI=HVJ;OVTWZlI%6qbnX7Df#L2$qF7!V@_RqO>xr9=@h`^_BW{inJAw zoL*P^QH>QUcy)M%$xzuF6 zb+l-ZOdVoXDVE=WSaaqS{EYOjX358QMX?ZVwqQajd~6IlAMby zn2_#bG>G?sQ!-m3YkU!5D!m&awVRgmDKTg?ecZ37BG(MVIV~rST*$Vz!4E(pn5Lk+|5EsCsTji>24#yF&E^hW!YvBQ=XsUg&i0U?RD% z$rioH9Zs}E^pBQ$83hvMQbu-v0^33aNO&D9u_XEqF6v%{pEu;!>}qcyi*D5SW{W=o zj|rn9XhMM+>pCh+)H!x-1*nCmY>ozgt?0zGVKnD0#4pj%vSQ#;(xY6~y9U+d_~J-! zcrG02UZQ`XFf28jl&P1y9GX;)O*lW|=W00kusq;PzdY9@a5&r-Op@sPyy1tQc8;=*nFFPHGBUd_ipS^0#Yq-_544^iY zZ8Yev+#2Jgu{Y)klXCeG+Ax@%-b9F%ipCmmBLKV`*^NWoV^I@)&41EdZp3n8Ym#VW znKE3N#^Z)m`FKGn5ul~1LhjN<*nNooB&6mhCdc2S4&R$bPmbd>?-hN8 z5@{E`TX`c|pzolS3nSZDlWIx#{{TnoKikX*pO0GY@FnefQM4o%NoXriu(CH$ErW6w zUmgB=f;-_MZj(+5zxt4=o>hCmb#$&t`4-NJd2XQY7){!K%55~(P}=tRO4&{9CM6GG z{{V&Eqzk}{en*2Iz00-0c=Ah)hJz~VcnWH>(;J^i6*X2^WxH?hTJ@lz-@q$gKa<-R zIl-QzBAZk>V-3UsAP7yRwK_Q17grRg#GB-?=KlZ~z5ScFgv6G)v`mwqj zEMmkQw(M1ex(aPO(g*4Qu)at{j_@*ryY>Au*+R-sA1JfaAxTS3IYs)A-gs7{UD)6y z$C?U0w&+2dO4=z$tO+0Gq-6~5<%%W(KIygkNVmv@^ao;nGmpvkEReVpCI*8tk(t|lHsl@|B7-i^wQ)LYcK z<43^q#-Row5Ire7icRK7dBWdeE^8VJ(AAJ>`@N(+(OGaqO|#hgO2SX82kKzJgjaGs z>H`ak5gDW(EnEH(+!YcY91y!?kHqdV!0uf(ATG){P{NRjO=OnVQ2UMU-tv3_?KsGl z3q!KdK;V&7r*4Uvc*U5gMM8wF>_mMS^NuLCWAsXmQauC@YSU1Q~b5G1s3sHMmYplc{ za+s=dd(NoUvM++yc0$U#crwn{5v1#a0ty#sGD6a*owVqO+?6e$3!CJtG8aMr#@l`hLQF zCnz613=-7tvJ)?GC-PZL{{X?F;xt1-+;R9bc!AqycC}=rq=!l-XUJv0pX)|nR-F14 z{{XS{LgIH+9gi_k-aM!B$usvx2`Rr9u`5m^{6$ofL@xz#-;#J<&@+aRJAi)zaGfr> z2;#>dYGOiAlJBsz2MN64+_1&H{{U?|l4Qzg)A5E!Ig!N2R(N_yw7~th62_yJmR7A)_(}QT6PsV(VXmzqoK19x;?V@d^ zXd7(uOz=cWXJ6z}-`XxfN@Te#sZhGOm5I&W{AFg7Q(F7OkeM+hZO+@aAE^v#6sY)z z5UkCaML{arW`tyI?#FYLUWMO*YD^;^?Ybo-PvCZ*NP_5L`ZQU%ED_YAPXwMM%@w<2 zxZcz85n8_z*4{P@UBS$rhPjb6*O98(fsMtA57##yAFPtKnlHLtF%iL?#%?|Z7j%{T zVlW-hLMEzqK+@SKsy7sNU5ZPuk+Ks{$w8}APvFsV7M2z&<*f+hCbuZi{7bcS?BHL~C))2QRUjoTc`+NO*A(=)ekeWSF=^%(4rpBSb+?;4{ zggM%<`4Ci38;WRByL^UlG2KwpY;{_++-}54{nF?lkdleaFe^(z8#f^J*o4TA zQ34eq7>&utPmvDQ8rO3Rz{JKKuH;&D+{7cjNZb}NvTP8Wtwfi2*l3pqrM(h}5|<>k zIXH$X2H0ay)Xhj59mvz#`6YcsO8IbNDZxo=2|^-TJf|L_AppeZH!#^^GJB){071N-KURbr z{{R^J^weZT8*a*yMJ5E+_pp;%RM4VlJDuchJo=5ptJM?c&jN7@-aC$;-K%3bH&jKC zYe>6_cO{|nAqo^wBsDRv%wm!yp*U##;seO=&oT3GI->@K?uJqvU7E=&M32~duFR1( z>%Aup5G@ec#*D6`FfTfxEqX({k;9kR-HFOfQ6?`8R-}*7sFaLQYbIz2(n|FSl%fnt zV+GORL`v&XX`~+Tm?OX^PupSB;{@lRYp*AD{ZCQNw*sM0=9u96EkWr{8;Q!1eg*!G z-GzL&`e0)oYW$l$Q-;#UF)xW%vgufM{JuGOfn~IH395-b_;HCs>`A(!o3nC--wjAhG_8YZV6X?VT6P%1{hZ zC^19z?qs;E>;`To{{UetecjdmFzok>)E#aFyeg#EID8*p>0-EWrI^om^xN^s5Mgy^ zua&N|q*=C_jLM91uQ`$9*K+tWIrTuFd zB3_J6g8^~9V^kdKYi0PiBquAuJTI@(l>nP#_~!)Lt9doz?;B0K+W3y~v2IR{N_2j3 zAcL_$y!Op58>BW#HN_{aY;=haDf|X*2)wB6zGoIdlO}j8(o$Fn5qYbTZXL>4Y$YRte z7rXJvmX*{FR#8CrB`yYnU=6AJN+^vJ$FT$Yb+au<^tzxg{o@R&xmp?@Z5 zF0A;RX3df+c}tRXu@l|R>gs{>877St{S&aPU|YKGxDq@VWDD5Pa$n04K~zt?MAMBl z()iw4nQG!XJ}w1HC~Ejz15-p_jDe%AqWZ>yy3^x$Av+1IijNEdy}1?Y4qBx%gLuqK z{{U;?90Yumya1qVn$|O1=r!LH9utWE{C-0yZN#E#CdzB^jpF5Nx$}%ltu>3w#O=te zhZq`p=V3U9$=0yzZ?^G@Q`+&xLn{qv4NYk_=8%w%ja+wV?cM5A4qd=^uyogb;Oi2U zMYYw|HVr2|55D~ws49{lDa_5KL)h41q<|%F596%R7g}D!dhxB{1x-f1>l<_=hwl{u z-!S8-1K~+IJdV z;2P)rQ?@&`=U>5^53&0%=}3;Gua;c8TvYAewTY?d5y`^Z6J}mApieb(W23zJZ|N~; zdNJ0Y1CX~tG#I;@X0xikjEtu{M+!!*m_(Ux)2*5hK!{SgWp!SKbQ8xlDj=RyVJB8NHD8(94=~r=$C~^*wk@JYed$ zaxE-7{mfAIFU;5Jsi{}+L-eKqrl4)(4Mp8yz<&NQydq6}Ty}NKnp$-n0^xz8>rSg2 zTp|ud@>h)4{iACA{{UE1=DIM1<@U~Za^39=<;!k0t9AT2yMl|KZ}Ett9_{Dbi>d(K zakXH1;olA*cmtKZnl;*(gRZy&Yc{&rYgaW0n^^ul+ylzq-+O6@h;AL9AC+*ZzH2s{ z`@Lh#LvMGBtl5ds>6{J>?|Z?o4LMa#)qE}rW}b=%f_LWxiP;7`&mLwwB3%x+&cxHk zqrUf&@G2Yb6Am~6k^cbPa;32cY{04{KqWUr<3(HrzHl>xX}wA|;tCxm^_vUe5&^-8T8SX{fg8%x$N~C* z+0~wm+&!F5I|rP09~#NzF+9vFTOdC1<>w)_pTlq{07n5Iro6mfjxb)S9a=ZM*uenW z>+75e6c(lH1AIbl;c-v*OssC;Wz^JHoSA;Lg}~F-_Q?*71m~63B?p1Mb`AO7N0`&S zR%DHK!(M(dh?t7fm1ntE1H0?eqt6HEj*y=@D5@}*jrqjQib5|$K)CZ0&N>G)Q%e_( zFrKl9Y0k|r&P!CU2C)FQ)yktl(NFzh83LPHyqd!kork7%fGAMCMhTk}!KPx}8luVH ziSvoef_#)SgL1+%t}(-Wc6BkZ)zGT3(Vi{c#e^o!eDSvwhl=CT@{UkYruByEQ@!tf z^@+6WcPz(kv%`+YiDlkXwa4slCSDK7#ZJQ`VTm-pOtlzFzZ-7&z^J;O94~EQqTvyE zn$2o$$DHdr{{R`=8R@9ys|z1Hge$1kz^~Jq296YSwyCLrY zf1vDL<-GynjtD}ecFX`kp)e>2h>NUC677C4J1vjfbAV1@I^9PUPaYNaLjhH#qVOw| z?h>L&xF3z;LN+DC%HA(VF&#CsXCNH%e%xn`*Bd~jxHY^qVWa$c#u0^wTz0rv#He}K z>Y!db!h|lI-E)HlJ&!zTaUxaYPt`D1h^zTxEwhThJN+aDkx<2cFx9?1*eoLu-mt5Q z5GFae9aFLRZhVOLBN_hE{6MDV&E=6XfqcgfmA9MooG9ZELyP`0IFxj4(p)4F`IDC+ z=S_g`w7+?!gLq6qD`L+o%{}@t~`QI2YZNA?) ziKjGZ=yEYRIviDA_kL~yQ@u^)SJEGH;7)rMaZEw9)5C+e2OeU2Mi6o-hM<`Mr@R#H1=ZJu#{U5EIN)fVZ_$uR z>z}g>eh7%*kPTq_5@xLAZ{$Bt4nqA3gEs#F8311CGk~R1m>`jOZQ(IMo}w^YQ3 zv2hj&L}qGGalZcm-CS~8G{~E2)p8>aW%`&38%ZaP{{VPe+BSPQ%36ziJdP;e?PA(j z$9P2=X!DNawnd=~l08;9y^Y5ha6FmG7Dn~=B+AO;(9Zt=?rK#AuGx6}lM?tT#%i(@ zEl?kqj2bOa1F;8f8mrt$s zcZpX}rwC!45Rg|yCU0cC9(Hq)sb+-Q>%`58BgA>|^tf{H`9EOF4Baew^fF)@ft7PP z&7uq8(I|Y=0z$Sn^FZ+~%s$$7Cxkb_)^m}kH z8%HQnl~khZd>ZF?jD?2YJBQDVrlE23qhRe?PmXp)V z3Rj{NeBh|#$6IH0zsogQ2-5-DQ-^v9P*$oxK||25rx7D!esKs;Xm>h?%YtiZV8J20 z;e7B;cQfqD0VbE_!f0Jl_{M{r@e{43#AD~+?*oxRr zjjzyPq%y@|ztPGnC=5IU{$iFQmzT);ab1bb7mw!wzKF}@!o0ZS7I^?^ zIj3XOn!z7;XmUA1j?=FJCUPu@92h{5KepVK&4drv=qL_ePZ}{o1*GGQMlPv-oGUwI z<5|hiJ@cD-qJVo0j+#H5OC#}dpj~*x9uPTKH=nn@A9asAzum#_l}UioEh*@laH-Ki z?^_-9aOwgpASuX}<^KR{zX%ZAaSu=h7d0|TPk3`rVRUcK9cUynD|bJ(2bBu*ZuzFd4Ns;W`DKa2(8)x8+mYN!XBeE$IMI%uz% zv-kXCpc!-sD}2rS-b$|x>s}>!`Y<#H&qH1B2$~K9=c52eRhJvm`EcC;l{J_XpSJ`# z7pE=S&BG?xcWbP$4lbN(_{~vtv6f+rgY(8Mu=zIQa$W4>St$CP)Yr}(Non(OP?|Pt z-dT|3<@0bguFmjkKH~lE1t4iCQTvJ!rVAW8TwE zPmSQfn;rs5uR6dS!VB0szyXM@o(Lu_8sW0oh^+0_N^aL@z2bRix?BJ(Q{i!9K1pbp z1l9QB)Oj~B3uG~PiR8#9o!I)sYC}c+2)p7xv4s#w1?Ks|;d(Eo&#RTUg)g09MOxlW zHS+K9V-%VfI0a~3#uzTcy>37rJ9z;qJ{;JC(;#>cN15(9IF_riW~M$>*3 z$T2h>wCVose}rSXUo6m zj8#ji`7yEfV(5sx^3NLi$R|HjTX&j8>a_Fo(Q{)RSIxeRV;DK@`N=?y0(E_VSTr`_ zpg`r>*^wD+pqpwJ1!$on_+!Ub<>G{M@eBb7(A|F@_X&^KpSzpY9ZSRcI54lS9e2R? zdt6otDeK>!k0G3bE8U#21ciPs41IQRVlgnCb8!%R4It@SaicwFWHy&Orw^6`o_ zu$%e6jNIMSA2{gSxN8O=Q-=%7InfeYYXxqQNZ|pMrkpo@+^AO0W9t!0>3HqQS#sTQ zVg{)cAdFIOk#LCAIbZJ)oLc)F=P(~%miW&zGMipBd0bNhLA)FObBGVx7x=MVxi41G z#wc_9@@Y9_Dgoyx_~{6xiC_un&e1Tn~IS zFGg+bmA*9MVJ!;y5tsn5ICE3Z7P8W<$>%jzO$~FqkZzYNfBbKZv=r^(xF|nocwAJ} zBRW=SFt;cg3}7#JoT+E14NHkoz8_cxjZUBMG?5jRRfiGW-$cs&^ZVuY4#(X>zo8Nx z^7L$B;I{h4?mil|wSPcDIRY84D0T6>ig0lYh5^OaRGo2&ym?n4vJVIAfT7rNl*Tnu z+5Z4pHk#MH@%3>+h5&FA4crLI8kPL(DRnLUyLSHoxw0pqy@|60vtBT^(Qc=Z!X}#E zoQ=Cxs`fnOLMGf0wn|=I1&wEP0`-Jd|U~%LwZS$kPP4u=QtH< zxNBdG5zCA`V2QM=%Kre3so%sf1gabe$CPpU48y?eROawr08hzU{3q$iBh%z|W3B8> z8Ay||U{`F(;9~0?@465T54ReNL$8N}z!(hFHOEF2Qa5p++Gdox9)kCfiPrANAfSAoYM>shV?6Q?2k~ zj|476-)1@nxw4Nw7pOT7BUmg)kvTs?GSC%MoEj>MCsTJ2Py^FL&%R7V%3$&9NkV+G-ydU_mt?^Nrz5;etc&pJ}WO^N6yeK<{2$V&+?&1XA;qpb(;ddd^!6(V>qs_i-~W#*s60++gE1 z*;XXT zCls!`)-Km&cW*ZUM@bwXjU zy%hWg9k-BUN_rPx8uOLknhU1^pw%1?LXh^n7_XN|@H#VCczNeWHU(WCvaYr^^x62p zn#HyDrZ;P?7#JE!^NUxe@AZHq{8;N^&3%&@)dfGT7)%X3mqb?c>l&)_caxr`JVMF& zVT)Uw;%zPB`j~YCUAc4c8pgoIH9ecb9RN)BZg)*)__`=J^@K80*$-YEtvvK=X@@u} z@d3L=2|$fsCSs}i*M&@8;cyEgHd9I0bZ`Mdq#Yk8tQS*EO69}o1D`X2J}?vp9ugp! z05WZHJ8N3wTFsqZY;-=xVbDkOvvvN1Hc`tQ`aT~202nTtMd+C-UuF0`=F;756@?4;8(Y{qofIi$Sw@0_=*}eY& zwfM*^Ht*lwP)LA>>|-i+G#QAWL~oxN4=bO3aNS`i1(6=P*^T!jHM`bA5l!?|cZf0N zHYI9*SR@BSdF`z@pTLJ{<>w)-Xok-5S4eSh9N|ztR7A3Q4PY$_$nYK$o2)`|CoT9g z5Jy~)drcUQLbHSZV$y?3AoOrl?1BBRhe*?{>}{{XYbLKQ{a z{;>f-alPLdnGJFk*~H40%VH3C~=E(?;~*Lae(%-L1; zWR_GT!GJW{1bevjh(PP&xp2B5J%Kmb#ARC0uJv$Nx#AhQwMKyd0Pw)Z=$-E%sRj>N zMO9L7a^nO94-)~vVUh>a$%`>R9lrIg(~{Ngb17&Zop z&{bSyl>vmy=^yeL2w z0PokvC%6PWM;NH8_%9b0kfXPUNq{5R6!zS^bhqor`KC$J(g-gmB{q$iMd)(g% ztQ?0u8kbWOoaT?4amT#fS??M;k@qJUA)wF-*kCrDk>kdPIgNoWvf8yu+yQqAJZ}O3Duq{03=K$B*E@OY_{Ja!r7!Jg7*fl*t!Y2D zFG6h`F4kE(gWWTD0_mo@^oCFZ)agomAH$X@c#A0YJkBv73B>n*n=u|@zkWaFBXtA6 zke3Q5U{5k#%_vmn&j$aaz9fm5eTR;A6=Q8$#=hQxq4dH*yON z=;o8Q;q^gP$ zHPTbV;<7f9b`S{b1Fc2MF1RmV@syLdc23t3DTbr`{9<($Xy2g!069qnFe$a~?B#ar&U zq*Nno=3`M!?_6((Il9g0+;9v)l0(k^aT#C312o=;2~N7f5RwWntC0md>xzFl6im@6 z(catuPk88L=bF4x&8vl`R7K|4d(JxUzU9j#S_ZHkpo0b=YsRrU_ru;PdO$lxPIG=z z-~2*jUJDVratnAax#mapRoD^KwaW|m%0H0GB@djl=NQ1WxYy1kOLla|7K>kCM=v#y ze6khsn>Zsp1I#el#?AwPs7|Zej~;VVQ(|bpE;Ep!0tD%jFmED+hqu7QD8xLq!O>mj#IqlfXktgI zxvuWz5KVZgGWCSoZgtF}0!H?wCn2=)LqBV zjAjG!PlMToe?Xv1)W$zWp~BtlJD5F)4i<;^gFu9QkCE2#jYJ#l5>wSMtHh3zrnwX~ zt;HK&m?=Ub(1(q4F(Y8B5S$95>)?xst?(ck< zkfA_omkDKwPcCpjdWi2?&A9KIym{mDT#ou8Yv*nZPB*`YSpMR>{{S0o=4Rrc+>__O z)G$5Bvry=`wmY@)tVXMmIO*0E!g3#+Jdj48d}QS8aCq)ingn|WSWZfJ}r$z?;% z1_gn8HGW(Qe{@l|+V5g0I<;ClrGE|r>0v-A9gW~`pxCT}PaX;q|1A=mdmCM$Cmy0zyJ{Q{5WU-0K9+3+ZD3Z#qtn8 zjGzxA{Riv1`TOK?h-*xEoQLBUsuH=QW4~Ca(PL+sn`^tCHhg>r0JVATrlU^DYN?JgB2icdJ>nZVFO+zLG$(0zyM zYbi^nn1}mcz;OspnDf`Xbr1wJoF?bzAbj5*oa$DF{&6(ys%-LLO|5X9Y5JA`;NUkD* zEeeR-Wk!>QMHcm%s25<@JmpChy?^64@|HJmGtJ83VFE9eZzaT?uvy!Tl5MRXP&YE0 zM4aXzjD+I5-a{>Y_gr(dU>}^|2bBqSMrS77A3ttrN2#aFUj_)G`T@tN$xjM*uII)Q z6tVnAw~30>@|*j4%@9BuLeKA6!hnwRlwF4LNq4_|p0Lvf=UW4dmsSFRWs2rqI-16M zU**18gUR_~xj&(;*qbqeI7K@hWjX{Nw2t37d&u6rZ#A%a7(d6(ZGFA_$Y-JqIt)k_ zOu`Bta(oUbMKmGbPdm5Cm4WAq!anU@vAk;|i+RI|-`)tXh}z@L(mFuca1R*)*#2)o z+|^tJN>VFStKKlD6>>sH>f(Z0^cbElK{3-dgi>{r!@zP8@)$jmkEMfton<5{*R**{ z&pA<9_xBvV&fcQ1@-{G5J9xGgzL#0Bu90Zoyt=^dg9@wuLR4S%>Y$f`hIdAl`+(#b(vDo5iK9rl+*xGBnpk9EUz-$}#XQXDkj zS@FJBGGTSudFty<2qR|8;lDm|HSkd8o4_$e6&@AZ=Gxu11#%HCn>iU>`SRenLqJCy zyarQF!8AuMapAlrHbr8OkiWlK-Vg_^oG-k6yjl%9!mT7Vr?U>sz+G2|j9kY|A?8<4 zjN}v~0w24LLtq_U4p(}y4v1@oT__Ik2KcNHj)KA!5bp#IF?Dnyj|GL+W###8F^&zuxy*%QWO_3_Km>&_MoT77>j&aowmjSkP2cqr`n z*#`}tm^X~M8ZI)NdBh2$)(8%`*kJ?TBIBV6*`TOn%uZjlv+c$*_x}KuR;pc_r?;~e zV9h7#sQI`Mhn(SrChcA?#%|Si+km?C{M@MeL9SCbX`{b|z$-w5$>SEO2(bX=M#>O9{EtmI39 zl83x}b>I{mPbPBSNBSHXGnZEuQz2USIb|hwpK!s*0WJW~CcQuN>y+*&po6_~p8L@vPXZHdM}H zoH|9DCi%^w7h%-j&n_~?sI!50-<&Ed;hS`!dtwAaM*B7sG(EFq;4o&57rr1K*G4{< z(G&(w5SVqm7vnCfh8_GKu@OP+1Go3cASyeZH@6N5K;_E*u!j?6)TBOKB$U;&)s?J- zM@}PYtQnvs0X82Va=Nu(m2#c&tJXK$O}o9({NdKLEGF|?WML%a3oC9#A3&O%2b>9} ziMbtMhNa{$92=K8D$QQk_8Cdt=Q?BRonX0g(4f~aiW}pPOld(R{+@6@0)}AM+Oq=>c2hbkPhdK`^0jELw0}j6zMA) zd7Qa9_TK)VIPT3K<5%u*bfV~bq9!6QgZjN18qws=9`GH4Kf5QsR&NQYeBh(KP5ev{ z-Bc%od9>Oy(i(p_2@zrrXqD%b@iRof(p)c*7nR32l)SH_yUa4Mwwyp4dxi)MH(1NR ziHtIe=Yf#*gS>(VKqWeI>LYIAlsm30{8v#W{-!#Y7q3!gG)DIu$nZ?vYO32H`Gn!t z93ja^z)sgS6#NjLLSPJ5k&@BftVTHaTq+2So~{ghHr)Ap=F6-TdNih-4rYdAU>N5K9gt=Zr$mLzB-q zdRV#Xe;j0V7~Tbh@6HoJ!#t5Wn8^}IPWc$)F+}UXJMo>^FD}11&QQ>>c*h7WX~Of# zjkJUa`a`y5!f0$1b1_^6eXSbB6?0%}ZTno;B~dt#JU!rxD!Cv0n{v_$0Y8+D$91@y ziLV1_qa#zo#7k02VaOcV}(-#6c&I)#>RTT!?A}*7ThGxRjcyzo3XEif@b% zVUzRD@e#WhcT95g?Kx`BhZegd-Et7%CXTV7`??+e9ZS4WtDvU>V6zjWfV@5oRpsLj zQ&q$yu@(7zM|f0Iq*<12uVx*Kr=;YZU_GLpHa`~;j0!hAJwG~p<&u-INhj{dS;vb; zzHCnhCtT}m;&7SZTol1Kv?(53E;LF7c5UxH?eeZXBsXo5&L^`EA&&4lcnfEGx7p5g zVLR3Mr}x1?dlsYf(BKd$01CKRlZ{o+Y^Zt(HNp9uxQH@q)G+bH`^J<+k@>;GX&cFQ zWo6_sCWO-S>nYjX8vJ6#s1qFrksNm+xxDEQoz&gP{{XhC2I0POH9I5P1?C1acu(lY zyvTA-c)6BpIZBhj@vDR1w=;V9aVVub%LNgzyaf7Fj4N95A`a#fM|yA9o#FGNvr({d zgsv|D(*44Yb7*InI)`*(cZ^p)OJzN~e9r>)+hXHPh5AgAXd?6r$*72hN z01ej%IT&}Pt*%@;tBNtapBM*voZ5+5jL9I0$=v5$;xKI5q!MrjStDf8>CP$ujStkH zST~dwF9#T%O`N9OW@%AU)WR5olax;mO!1;E9U^j$2rv+uf7Ztbh|~fsAN*6nq!=n{ zCkLM{TBBZi>l34_0GYY_$c8%R!OgrVBD%%gkoa+N<`vooBj)R$Tx4qQadD*aaqxHi z`SWu3zvW&?sO8VSTtbEry2>m(}eUkUidt7}4y4ZiTnH9#Oc zdPg^RN~jbC4%==oGKltX!?!ixzWW7y6f*hJWedKO8n>l1ZuZ^hB2gVHSBb*7hfq60 za(lowr~w@BG&cs{2Sbd^on)0Y%frYpGQ2m?E`KJmw}a(s$@xR=D0qT>lz`(3vpX-UV8T0U@rx=ZjmE+bEk=8fP*Zr_+(D^XQ9 zJ!5ILhWK5Ya8e|LG6b%-xb3%mV2(6W0FN(uaqOI8uJ0U7aKVRm!m{{Ki=)ZJ$eWcs zH}RIX>uX=UZYcp7*>{D~?#FD~EmbwSw-*|Dtic1iNwQn~V4h5LL)`I!F$FHK1AuV0 za2*#LrLB~*y*F7sDurnHV^DxQaA4Rbnu)&gA8WUy4td2OX2VwpmtJg6uftY$X0gae3`@wpE#9UyYj350ME`DbdbN^b91!l=a|C(07JfU zAV@gg6ao0wb@u%|o-sjLf;M>RD9;u|DQ|v74DruU>(P zpo+SZ3>tu{YBHTJ0ZQgd60u2p3xsu$jrW&m-r0tb zZQNU}weDrq3{cmRb6&TrM{V$4aA-{tX3%o^!^Bq61m#|st*BGves%oi8;C0eui5J! zJl%tQw>b2wu+*8b*8Aj%x?_$Hb{xb<$uf(Og83i3g10Wu0eUjz3NE%3=PeL?u*Eel zC+8}qEmxQS05V8-P4hPA4IUP9Xx1}E>Jqy5lGO7Ibq@mQvl)leA#Mp)* z_kaN6vlInhA2=S};Q~5gUD(JOdmMN6DCYyD(2XCKYpp~YbY1z57QMqn+wY}v{*$-b zUbKE(3AVnBwelds*BCzEu63Hx3ugg(H-*SR_yLj-q`g(xm7GYmAEN&N2Ir`d7~j_( zZm?k?Q26EcxWn*jb51`5#YZLVU3|D3@DcbJewR}!Y~k1GSz&9(4kwT&A^_%yUTRIo z;kKCtP^03`KqKEFm{$sf!J$9{%3}7dm ztp*?_-#BZY2kmojj9^|>Ia}3wn8B0+ z*0>|qDol$MJQvI>j{pfI?0D7@FSv;|J-!zewvkgWw;qL%AtmE96cHfz9A@RT5&f(3}pjQCJ@4ey5+5SaTR-A|r4mb`7KVPeS zOV)7g9L`C%!IP_vQ|AkHnot9czaOBd4j+nP09LPxV0XPy>`;yP!z8q2N!d2eu;s0? zi~J@%F2O&PE*6n6q;vJ=g%ad?mW$4?arr&OML&|_6^6<@vU(<4J?<(K%0WFDQ%ZNb zyiXrm;*FH;Yf|)ekatxd#zVO-jjQl=SjE`(vz1=ls+uN|^^_F|Vm?vh0<`O&)09S# zYi!M`@T^UoK!^}^Om<*f*b5J41&ek~Pz&))2Zm%iUcH$D)6U{({(;5;;1c_cuza5%u_goCIc$pBh5>F(FYC@j1k%y?*ob zJM7c@xGhZ0|_S+l6b^mo3wm$E&$IKF;8ES$EDM=V9tK9cGtS_Z>)E5!^WtK z7VgAbbd1rU(^YM)>l*dl+BZIDIC}720b+Zi=77S9r4Dz#OgCU_Z!j_PX7R@21wj>w ze*=UlIRy=a6$C+q`A@r>m>`|ObboG)G*Y!uPkLW@QpGo)tg0PPZG^sX8{}9a^x`n~ z3EwV^mKJz=?+8c+%Y{+nl^+=~%UY`N&J0Eu5O*9`RKT9?k6#xHiA!oI=AHOV_6~ln zFsntVOJR&DtE`cHZyG_XNR_L`o!xQTGTO1q8sxzYZ<`N$VaRk`hF5PAY|Wz zSRIa^%l{0F9sOIl`U`ykTj?(Cf-p;49L(PLv|AfYXa;LzR(OUTp=KXt^wf3 z0#?W!4;H7+0b)}cV9y3P1!uDGjwyz=f83f&%xQpKB#{au^G`!2V8CKP{b=K(0tXY| zt)Dm*OYDnd!u6NPZ9oe|;1eE{R8Txa6|uGv;JMWo1Yp4h=UA(=RmcS8J)PX@1HwB( zuK_c5LXBEaA#QR}7}VTVGZ1s7e1*nqqMR27O%eY9h(Y*p+C@n4bg#pQ7fSfU%|Y4V z#cx)>M+Pp1!}}Or{*|3*{`i#-0~HM85H>`41IdX)SPSUqC;h0C7W%sxm8n62hWCu6 z#VgN?`(}c>gfLt%boM{S18M@saARyKIX65l=VntdEgRaL;x^m1Da!n~2NIajIt#2U zWl^9w6I%sH)eH@!n}Vn=!Hx(W!3~4Aw-r0;BOv?2<8!+W-?Ml_c@7{C=Mg#Kia3?* zlSOqqFsBArTRh@mNK=8Y{^HI=R5|f*x}fuEQw_q>8rN3^BQy_xsWECC^Uo}E%+Ot# zmr7UbdP6w z8gF|{aS?kOG=3i&Cc-u&LE+yf83e&p7?YQJG7>#vLV(5K_8#)JC^c2v`Nj2AG~^fb zd7e3?)yi~4`C6EYm|se9e5_O}Z&z0PV#p=Et^E{%E>w9Q)?D`vQwEZoaBmg?iGlTP zio8$dV9KUxUFpN>>kPNT^+Nn(dtgUfpPbx-T#R*f!<2@|_+BwSClCyWlWXCV^t-cw z`*3ERo-$dj%6u8R9G61J4#_P%X#Vh$5yrgb{{Y(?uFV8XgAGg8@vjvj>37~88f8&9 z<51TJp7MzBb&5Mklie_6pbf*uzT5{w5IEd6l9CI7= zHMNJ67f3oC(^}}l$x2M!yc>zSIC?@AqPJK)ExQhmCJFMmVFGZ{>|?Xi1qcbn>027o z=)-Vd1(#MC2_y=6XHWLie{S(rb4gH)M71ACay>-$yGcJ0IX*+ctjS}$b9_=a7mmhzwmQosh6yuI_dydRtbo+EQ&ZW0HC zb{|>Dmsc#?ux<;pi=#&Vas*U#@?XoGN(6+oA&;QCSk~w^-(%@?dNZq^*7_c?E{&D7 zb@(_81Y;6i0D8ED&~Jr_;N3$h5-DvOHOD#2fnkmMLo8s*?b=2CJ4d&pe;~*2+YqiV zVAlA+0u9sx(fPT*=X)b#m*NYjg04Ctg>$0{^UyN;axq$}yYsK_Fa9hwX6LK{$wPSj z1SG~cwUgs_c|f==LDz#Dr;x=*WE|ug&3t4H*i-8%K!vjbY&yU0Zt6g8_?_a!&cWqc ziND_%kuQMrvlX&V7JA2|U648L;^wf3#BW8=7IT2{v<~LD_l4>gCGYZN2P{>dTZ0#N zVLl(mx#g*~B~*TxfDc<5-IuSP}1%lR)=fYP6_qD%vFbl}zp3M*S@tO)+y<2waq z-xvp!knmgslWgSE(J>H^^)pZ>es32gVbISwhIg*f>Cj|Au^A_8{{S#od_J^@;#blu z$_Tl$U)MMus0-sqYhMI-l&y}~^UBeAGaz{8sWI6fA z(SHy1fi2(olwnhRZpC#>VMl6@&-8ynQt^_5!d7#w%t;2MDGOlb?`wPWyif}(y=7={ zR>Wa=Jh-sCY$>4f0nQ&s$AQ^P+Ec~I`9QOfCi^hN-}_KF2LOJC z+Mt2^t`tOo+m0aQ-m)O~guBlUQ!DVfw^ld9&2fp(cPZ;4=t5@E2_+L(e-jjIF?IVw zI3WQ`d=JtCn_1WRfsbC_FIE{r(@DZ@fvK?Gh%fHowrWc%yE!=%S`*yLH}c6I#5Ba< zt>X&Ri)6%GY&Q7Ct<4Xc4B`I(b8>K_t4%swNo3n9E1rfBKOdZhE+rbZ9-&@Oz@v@n{ z0lnkmbfpCti-N~EAhOUtae%}{=X;y)H)2OtKL$0~WBg|b)#X%KLrlGM*x6; zuU-xljISS1;<&46xR^eY@$fEMVJPH?A^WDXyQ-1ToZ16>%0+hLPPeCcTe^+89vXgd ze+l@wvHnDunK#xEa>D+u=APodu5O!t< z7%xe`k;IrN#u9?OI%A_PD`nGpT#bZ~h`5nYh*cO!f-q_r1iUvY;cR1WNQn=EYt zFw1hwOrcHp;KJ4rZ=Cg~7K}-5*Q|DxW-hxS;j508@v`y1*Asq|*-l-KJV*}V{B6wu za^<{C5vF|p_(i239O9yMZk4P&+<82~Iy_Bb8>>a8ApWV7fVV-~Xv<(oeq1}%C5ILH#avYF)t)}F^_n-& z&+Ko9HAr%=+GRTE2DBVvJnnvR#Am6<2xiUCf_Hz38A;WUD}F9;Jf^(reuWY7vj8c& zW8rc|5|q^1AKoT%7Pbalx*2d!bC$1V4+obPV!^Zrf&{`-w++lG)Vtj{WJX@vM{+*y zDGb%pgq|8Wo68*?AHV|uk#bLF0ymHe)=!Le+K}J0h+wEnk@LaM@y@nWZNmCCa6~~0 z1Oer599$EmNpf=z5&46^kMQ7bbeJfTDi4~!c(xD(JXnqz0m%h73lzkNcf6@E_ZRfa&`%(k~bFJ(d3BeAE)ONdw1O0twGz3>IiB@Z})4 zIP#KVDR{sKw)E>-&uq`mckRivB9>iPc+OFfK&n8TVIAHDhpaK^3FKNSTY+XWG zFh5i?Y#*Nhc|Zr+^0?g133JRm@@Bm-qId?_{5Gmy<{G-|0;D65Fzte-$Thana^diT z8%|&JEw3lNe6`@4Dr-_1DKS!3A`^PHqrEvsty7+ig3df72wLPhGQ@)XhLZ z8|cl!q2u&bpN#ktYVSJ>-Qrh;!r!PS0C0e0fYA5)DsTL)8-$}1OS!}+?YsbhrF@A= z@AZeFF;RxckWa$q>x!PG;PT)trY`F9F;e{p1yFU7NzR!hJX%KsM5KcFQs9@9q_jFU>8X_#;90WT)O}&Y5xGeGhucO2DJNf4SxjN-^qyOXEa)I zXGxMpZ!Q98_S&!5IV77ip)q*C*q#gsdTnuWwx?VG@oVjjhGhT->w;-}P)^Z~M|%&q z@xZ|lW(4|o-f0DcTWr7>4Hhob{dY4wbLczx42hoawj)YD$Cdhf<{E0I3(5I}4R62KYN1K_$}*u0qMhSVRYm*Fe0Rt2-m8Yer`oRPl`!frq!}-Y z*mtarhki#7Gj1&8YrvrQ$isTUiDZ5`%r8PAwF>-0oB+)YqV`Lc6zdgb9`R%z2yy%l z&x0twmyQi}mB`W)*li{x&-x%Vu;3cXRja#d5f^MjorI!o}zDPCGAlknQ_{3u<_<)jx3H}UQWL9uH?(dYjG+s&BpOA05<(NaT8$Dj)lenW&M?8xIRP9=NYm>@sFnq zNzNKBb#u4SbIgnKKnb#5XYp z%`$TvhyaC^Wu!0R_F>>;JQi7O00TWX87v~d&jno$3h6f9$+1j5P$t66> z?uc~a7n-0`)&dnu5Y9(~oGF}y?1#nY)*HrFO56B2FrFp>;BRa?Ieqt=gI!(+Oj#so z@I#DkRTsUldDsK;%=Tgr82B|!qjyg^s2Qa+vl#P(5OJ?YsCs>2^Obk2%MLIwz^rGW z%ZQkIECcAmoGP$*Ihds~OQvj}4nCH3t33Yz25EA$@(jKu*)_9Q1GZCxmc3l)$Whq% zlLZKF(syjxgM&>0CoWiNzm{Bd7xtI|A6)zI0ibg9;5T{v8#T^6Esrwn{{ZF40K4%` ze~9phGzJGZzB6uRtzO^Muq}3ZFcEZp4X>{_9oZMROS11JP&OE#-~oW2OICktiU12X z0)yGgvb@ZE28R4%W@QS2sjH01*p{xd zqp>qDQOo1=z6DVeq675~S4C~k+G5h)&Qts*IcfJ~PtpL!+I?*?Z40`X5QUD@!G$AH zi8yar3}8`TTgP|}xOWT#a|%*|0Aqom*9Xta4ib#xn~P;Rfd}51 zA{%R{-Z;mI0n4B`+RXT`>wLr4#n~DR-{{T6qdeuCX=NC|11+x!;DG08t zV_XuHk<~dmhp?TC<+u!}8DeH4E|Ik*55s`04}%naISXtHY9E@gq0lym7w{K2+B}ew z6ADP9qBMQRFtaASH2m?(P(#M}XD?8#y&>9C$D!P5ep6t;_#XxXx{(T?l+53x-*`?Q=3F&@%Bsx672Kw4~x-3D^YiQ~JX|3%%>k1V;dvEb}?H29A?Z zvbf3`XhL^auNa0e9t<3=w!FJAZ&+MHmT%_}4UdEIo(+E6aev{s{{R}_{PPMb&2I=I zXg=Au^f@VVqNFi+%`C0BKTN~6!%N&>Zz}*5^Gw<6Tt1Aw4V|T8o*{75``T&A?o`bX zKMa68BR`jz9W&OzPODgyFgmaT6%UGfUGa~flHXU5!T}Vw`1h|mGEE@g0HKtbCj1=BhsqJ5>~hJl>- zG4Q7ZDCzha$-g2G2NnooQNP|s0g)^pR`r3> zHC3RB`fyi14;SPFxcj#l<6Iv&a%!7CXN8@S9xmanZ+LBxWZZhnk=UE;G~v+TQ1c}? z#VFxu-F!IK@pxdNcl39IQKN%vCM4@uka)yurG8uM0e-SXoQjSL+oq{F+ld8{0R$O zEVM#+#R;K~0{;N|Zy6X<{vmzhMYRKxzgar-7x8cgKFK`UnnA7at^WYb-l#sj?=*K* zc;gFDtDgBnIqtS))wUZb*&Nr_q9?8w}$WAIPwDPGJA)URXn7P>=up^kJ@vmwT=LDAf{hZyT z!40`IyDkDfwNiR%kuEg}u}?oY9*EH|5=8k7-$iQbs{Yt3hZ<)q@MYA(M%&hFdDm^< z<;wYgwhe$zfZK$o_PWzq&tdRcymtjx)ltQ-9*m?2wcjoDy1@ipjCZ!Nid&Z-qzh7`Be59!)cq#H>6WH>~Go03q+Z3ir?-)&x?pUygD4(?5*Z9dnwV z^F$7DRHt|ffy@5p#F=+u9UqKybKw5~UUAm!;%`oSinuq&zIDbEv=Q=tSo%2xE}5eT zMImu@b;o8KLDWiMxf-Aq>Dq=9*6u1Xy!6V%RUx~R@`NOiM03!#$zoVY|#=?sxro4hWUH0-zolrRD=p_1%P zjkNQ|Dmh#Wvv^^#*sSxDp1ngX6UVE`bPE?XLGf0>Yhi#%DxxZr+MH z-Q>w-VclQu+`$V%J4O8B)}aT4ZG7i_eKOW@!Ui1#Y5Br?LuKn9OhyR7DayH7vb+3c z8YT=IG9@N`Nlao_86z$9VFRTTgL!Y9s?i2bDkg%0UyRLT218IwdimW z#EY5_!s`?o7M<@@!A?z;cCfA>jcnWf;j6uGA~o^HIiV1s$H#c>_*8W>-3uXfR4el9 z^Z@1H8vK}9r8q|#6xx)JGYp4i=6|(Pko)Iva{%GAVUHV$9$XgB2g!m^e6AEsq!q6Q zBrAzY=hNb06g-F<3GQN|3#k<3{9GdYwPUf4vXVG-->gzlAR$-g*^L>3D0I=A4}xt* zpHmtYrvTdDDqvJW46;e6_p;Yx87_K{H>4Z#0yG5G8r>jDJ0noF++TSiO% zUy%_$T6G_@0v|DcoI;aD&MjW1CyZ}zqiz)(cFmNXc@Bdbd>q}09yDHW6AMhxuKT!r zvKuxMugEgYh~gK&*uw%}B(+|))-#(sk)F`Cx;e*SJ4BZK%ScY5u z$Xpb4s&Tw+{w8o)>E0zyt4Vq;5HJ0-C=5kAcXC~-vs!&(wOsA<&MnnnZQ`g+(^uQG z5h_rN%li1lMMHPC*WQo3R0C1*pcpb^vaK|SGZYf(pewrR#t3S#f{KluLoCA6?*9Pt z=Aw%;jWuEA!d~}qqqF40vb9XMO@CPhO|X`r)-`MiY z5Wny_!;F4hyf{J$q!sT4sXev*Ik^u!?i2V77I5euHZ#wqY~Kb10d}uu9_Z-=3hXyK<$9IAkPj#nEv0xn*|dmLqv8=sR$CE!~kp&d+) zZokDFlH|3tM?-v~KF!0m51AphzV0Mn2L>a8n4+^;H;zt8SCOX7I9+6s?Na>W1~yir z-?hR$sLKjG^Ng+jrciyl3_*4BO?I5YuS1Y4w()4*DaJ7~jYn*6Xs0-YRYW7BvxX7H zd(~#0m>{LGMPWE8$|USvBld$i7M`-nPlmELtPA-^8%b|FMyJL)6JkN|;KDLn!l3;I zb#&qpQ+*jLuM3F8dmQWh3!QA-`8aTdxo}3E3zES*C!yTLE(?78->d>s)dzevo)-t| znPvWN3JZ$Qz<*iJi(jlz3aTD7kLMr~$Qt#_j)zNqI}8hN0Jvr-J{$^mxII9gBuqJE z$3a6q(IquL@j4Q}6ypp1p}R~pz3tt@E(%G~_`}$bEl;VNSHb*b-MAWCOC98E4jT|Y zcHiR#!wo};g;81|bj}H?1->~hBwdsRgw~0P6BGvGBNU#d`vcjC0^D$SaGp$3ag)!T zBMjhU^ALMQ$}MOyFSsJ&Ite0s<1`T@X2)z27hFMia4rBab{t`azD;d7CJ+r1VZbLD zOBV_n`TNc#ZPm~?$T>y$!ky@a<5;2x23&*4dYB3-(XMv%cHz+AYIbw1sa|yK@?^ZX za<3TUE?+43hIctH?9xhdGjR|I%hjai-gvfESas!ovdhcg{lMwX~ zlX*5gp~i7k;yPDdcNh;@UqUKu(sUlnKVm((lX%-7LBd^T80Z9wlqk6f(1 z=y$<#ev-w7i|iQM+1p)cd{rFKR02rS1E)MQn{@vG7vk!V^z*z<4)G}3M8}J>V`JkB zT^!DKgZE5!00H(^4&p)Qau3qnCYWf)vHt+f*;PC5AClmRd6^G1P~hpqYP-{hw#Fv0 zRHat0`1qJGVi+kXGhm;G1=PxW{dpH~PYKH-8Fd z%_JZ?QM;Al5q?H3c!|E2zoUm$0QS1c+)Q_AKPCv-OE=y|wTf>H)DE&XzwY$vV-ria zc+FdexRVg5&pVmCWN7(v%r=L=GZ33DUSO*j3B-iTW96R+SBTtIT64+ zAV_6cT2REAIvkech+}ajZkd6ZdJsBS1eJ z!iTim-t&nbOm2(8fG@8<iv6}?EGK^oO=oq=h8BReFz&=PS%^z<96g*gLI40yjSu3Cp-&s0* zf41vjoq~3qVs`3u*^AAKvEDYj!I%$sV!Ne%=KeEGD-syEHRXG0z;s{==o3?V;|M`V z5jN9}1`2cybQu#UtGXy;YO)p7bnC|Pnwr3cTt34OQ&^iFVl6zPvYf7qydL}-A*@nkhQY1}7muRvNx;(p92{g3L+Chn3^=JT!*AnxC?UU`?=?}; z{Ob`^rt;-->+!AraeR;=&$snlJ+7kT2bs=TnWA7kBcRJ<=cbp76sfM=>Sm(pW65s) z9rKhO&&#YwZH4o#J`5ecz8aYs9AQS%-mn^LI~Oa}RJwV=UBaH0J`}_yq8`@-&KD`o zaFI3npXXIIZWqTm?Zf%3sGQwkyjO60d}RedKtg(43t82OJ~Jiz#IIHzGgISm$jF2| z2P=SP?|onsAu_)VaCUb?6?F8lxeqkn*M5vDQX+1k{rvyNKA0`m#Fg1a@n+^TUnaB$as7C3W zuC}@i8TuJDf5BW58i0%Ba^j8}8lM5)R}^^gJN@j$18_9%+;~hH9N#~=gGsI))#l?2 zHNsNWKWv>%1`neR9!=r6YLn*SpC_Cxa;m)V4F}xx=Q&gCc)*g~zj-i4ss7Yj!0Xn- zc#%6PJiTByblu}guhET~m)9;1?J?a(bi%; z>A{PF3%VhR@Wkwnz>c}+CnTd%Uj{_d34RO)=@aqOKWtDZL7V1(SOD|}H>}luuNNNM zysODKyjuXSznp?J3*X};2!j0(XF+z(vyf5Lw-D9mRNH_ zH;mD0ALf-vk^=Y_R|>X4e8IR&yEC9vubI|ReBw&neVDsuI#%E9ae@&Kz-r*CFJrr< z=D5{HzWyi|lmHwHMUnb*f_flrfKl^uRRxnrk(MEhi8za-M|+MW)>DYrMCBl04Xcsv zF9?IKF(h-%)O7(bOuNC95SGEe$Qg5XCH7%=&Uv1U4PBLAG5vmkfK zBZx;31^D9y4=7?cP}V_s{p)OMH^Fq`K~bc>d^1D|1LwR>2~)zj5H_c%Z_XM1+Vau` zKqGm3!>pl^>S@oMLkFkrfmJ!kv^5`?CbCDD-)x7$gJccuP?~2VwK#k<-f7i9Pb0sq zcSO~qz0BJ^pM*N);N8L76z$U!;r=l5(ZD7$Tm1w&Hmb z#6!Wp@qj^{dGms4FKu`=hq8~({{R` zkmNsn#0E#RqyugPZF~Vy_?%p9TEusz3;<}VfKCU>;|~Vthve%lBzyO2&sCp{cLI$g z@ZrK9=3c&vh8p!$^ufGXOgii)F-kEq?j2aA6tNK;}1o6e0c9H)8MOiPN!1BKsW|6%b zgM^*WV5ZxVe}Wu{hWue>@{aFEj8fqj@Q5&t&qcXn-YyAgJb8O32s4EGaa)6!CPx|( zsTu)uS^!Twj>~NPm??6iunO*gzAO8%>X?wTwg?lT71-kI4PDyl^Q;ho5vA{aVF2bP z@nE?f-M@1+4&dQD7gG?jIQQoWrP9-)cH*0FfZrItG-~02D2Ux^rb@@ZmB|k5)}p2m zZ9`M*?a9M5J4uV}6mvuo%jisy>+|C-08sW!IfC-*irm<;e3v4v*bCdvA{?E!&M}i$ zqii|KIP3gm@~c%|Zu507B+A_S@M}Jw&_J5IF#0;_t{>&jMo66g(d0J>z7?zUjVi|X z3VU3x*HX?kB<{ z;@j38rM)2S=ayv2qP*>_AchC?Pw1`*A#v2IPK#d#7^}Auq2SZu!AcDj9lF*=0OtN( zew^0-0FWgQya7z)4Bu_A`Y?VQF+|rT#YMccEb!S}(|7@blXKPtExlvT6hM?_{7VicY*K+O;d`bqkC z6$Ji>$LDxt0CfGht!DA=r+x=DO5KI$og4~X1g)wwSp$~?!R|~6kP`JPsesSo9?#(& zV9AMAHQ;#uSzW~36K)-jY;FAyj2P%`bT=Gm5k3o(T%qEH2YJ9!*=qOWSz!v%N@N%q zXbA(0tl}ABdr9+;f`Y6{b^K(NH{%K`d)6o@(TQ03)^1-INee@BH`bi)JY5Xh-7wqN{{Tn%#0-VU8p&W$JYF+(a3DAG<%~IK54(u^ijB)C@FJY<4~$A^ zqO94@I#X570Z?u0=NNL9q|h63C~TGg03A%_5?TfC0T5t@GUsPR`u_g_&_S1DejEeV z8FZTG+k)$<=mZ~yaf2=?O`zyJ;)rNqZ4Vw~$c7M_Ou@AAu>hyz6AVyOUFEDFlZul~ z6PUc>Q(`gUj}NB|gZ{IzaxI<}PCrJBfDTphoyow7 zs`wToysBPRbAi*G6XK=wFfH>nf*^2%;uwPQNj@2MoG81s)*O@kwivh3kA%W4hsZ5| z7Go%J5d3cbb6m7r%@P<^QK9q8^aiAa?x@D6kBtvChU>HgNe{egphx*D^@>C-ydH)E zW4>2*Ocdz(_Ta(;3m+n1y#D}mWd8t9JM5Foi?sxXYMrt;z!p&6P1q(eWrq#bH`|YF z72&e1!%)u0z{7K?c-^@(ViMcaf|!MKy%+J0N~)=8H~7k18)z9H83_TLs(5*ZjyX%I zb7$bV#pVJPA#S5EcAAj$txTh}ABPDMv=v7NOv2HKUV6ZMfDQxStc2$EH8W|7GMeHv z;W)}A%Gh@_CB>~G2i^P4myk>lsqc~q^rJ({{VBD7QMf|Iz`;ZC}3fi zd~e?(U5HE3%-#;?G)!57?PEu7GQ>f!!Rw^+*^4Cbx?V#!fV@BMQ-C;PV-2v?!~naa z&GUmdYp&(lfK^+3d2R!a*WSO+`v_nY5g*gDQvRD?8y<4MhiSqpik)B_S4{HK{w7$) z$CY+Jdxck_I%-Tkq=4zr`&!PG#xKR8j$P@=*ZZsh0YRZR=L-v=F;R0TKe%v|aNZdt zJF!d%mevmS<;S?K>BkEg4ZxIVjENl+sQ2dZ;tth%tA@8c;85n*+Pt;J zAIj#`yb$8!)`lMDB??9BH-Xxpk{CnpXyg|L0|FiuFf7o6WO{hUv?y=^v!@SRgx>9* zsg0>hQH62O#1a#DWNAD>q}@6k8d7mOhm_flnIopaK3a*9P(n?*gXPv};doely5ld! zP+;nFidW)PxwSqiMVM+{Bu& zm%php7QBpD{{Ry;zZ3@9#`4HoH7&K{*ZITRsM2StupN+7%@4M}Ln{iL{{ZF|zyfWX z=^Dk{Qd|TciIBRso;Xje8?tYcczsMAMO^(y`oS?(dV162Tv;Mp0VD2_fp?%w@U94Z z!c6y;mv}kjN312GOZ%;~E9W06FvWz;Z@jE%JBU+${O!tSMX>z}*?8V+CE%yaE++xz z_oq1Eom_wq5dF;C^k4z(){XO;^T$ulNOIi%G92Aszwx{~xFQSL(Ek9B^pH`>KWT*7 z>wmsSBF%ZlV^Gl9-7uvR)Uej;C*=X)G618FK|^w=XnFFaZWN^h6S(O7;Pf@`z(yZ7 zU_k@l#9jPcbvyq6ZFE{R3hgjK1g*Ydyg|sJ)%(D;tt9kgWdJ48b+acYL*T$D5XsgH z&Z2m;U3Z2m0;+A=l>B8tLA9W}^>H=ghsb^jf>sMHy%SP*-HIBktJ8)!n#};U3-arsMj>| zk-k>^ykSmDlf{=f)ZC7N$TbdJ-~Vpx^mxs9`oC`!P`BSrPA_(0olI zAHH`GM+0B$C)gX;DT1&lu8coeCJ^J;gg+Z&+eHx`2fkRPKncc}JQlLYC)!&axLq!f zp@&&=#k4?sg3E@|U`?p;_y#0zHz3w`wftl>OYL?iq;4cp!$7;!mZq_>32QjlI1U@I z{p1IBUNJ?lIz2drUV?lR0$yyy1!=VTw>f-m@XD!RDZbowCk7a5Kx?#u}(4$9plaszDKd9Wt}u@*el~V8nRzz1dK|s>FoxYp)nWa zg-MH`u_m`!~hf#0RR91000000000000000p#Rzc2mt~C0Y3mk-~Yq_ k6cGUc000000000000000005x>+5iXv0|5a)07KvZ*>sBET>t<8 literal 0 HcmV?d00001 diff --git a/doc/tutorials/core/table_of_content_core.markdown b/doc/tutorials/core/table_of_content_core.markdown index 4cd77fcdfc..ff142b7f9c 100644 --- a/doc/tutorials/core/table_of_content_core.markdown +++ b/doc/tutorials/core/table_of_content_core.markdown @@ -9,4 +9,5 @@ The Core Functionality (core module) {#tutorial_table_of_content_core} - @subpage tutorial_basic_linear_transform - @subpage tutorial_discrete_fourier_transform - @subpage tutorial_file_input_output_with_xml_yml -- @subpage tutorial_how_to_use_OpenCV_parallel_for_ +- @subpage tutorial_how_to_use_OpenCV_parallel_for_new +- @subpage tutorial_univ_intrin diff --git a/doc/tutorials/core/univ_intrin/univ_intrin.markdown b/doc/tutorials/core/univ_intrin/univ_intrin.markdown new file mode 100644 index 0000000000..7e6a8a7cc8 --- /dev/null +++ b/doc/tutorials/core/univ_intrin/univ_intrin.markdown @@ -0,0 +1,334 @@ +Vectorizing your code using Universal Intrinsics {#tutorial_univ_intrin} +================================================================== + +@tableofcontents + +@prev_tutorial{tutorial_how_to_use_OpenCV_parallel_for_new} + +| | | +| -: | :- | +| Compatibility | OpenCV >= 3.0 | + +Goal +---- + +The goal of this tutorial is to provide a guide to using the @ref core_hal_intrin feature to vectorize your C++ code for a faster runtime. +We'll briefly look into _SIMD intrinsics_ and how to work with wide _registers_, followed by a tutorial on the basic operations using wide registers. + +Theory +------ + +In this section, we will briefly look into a few concepts to better help understand the functionality. + +### Intrinsics +Intrinsics are functions which are separately handled by the compiler. These functions are often optimized to perform in the most efficient ways possible and hence run faster than normal implementations. However, since these functions depend on the compiler, it makes it difficult to write portable applications. + +### SIMD +SIMD stands for **Single Instruction, Multiple Data**. SIMD Intrinsics allow the processor to vectorize calculations. The data is stored in what are known as *registers*. A *register* may be *128-bits*, *256-bits* or *512-bits* wide. Each *register* stores **multiple values** of the **same data type**. The size of the register and the size of each value determines the number of values stored in total. + +Depending on what *Instruction Sets* your CPU supports, you may be able to use the different registers. To learn more, look [here](https://en.wikipedia.org/wiki/Instruction_set_architecture) + +Universal Intrinsics +-------------------- + +OpenCVs universal intrinsics provides an abstraction to SIMD vectorization methods and allows the user to use intrinsics without the need to write system specific code. + +OpenCV Universal Intrinsics support the following instruction sets: +* *128 bit* registers of various types support is implemented for a wide range of architectures including + * x86(SSE/SSE2/SSE4.2), + * ARM(NEON), + * PowerPC(VSX), + * MIPS(MSA). +* *256 bit* registers are supported on x86(AVX2) and +* *512 bit* registers are supported on x86(AVX512) + +**We will now introduce the available structures and functions:** +* Register structures +* Load and store +* Mathematical Operations +* Reduce and Mask + +### Register Structures + +The Universal Intrinsics set implements every register as a structure based on the particular SIMD register. +All types contain the `nlanes` enumeration which gives the exact number of values that the type can hold. This eliminates the need to hardcode the number of values during implementations. + +@note Each register structure is under the `cv` namespace. + +There are **two types** of registers: + +* **Variable sized registers**: These structures do not have a fixed size and their exact bit length is deduced during compilation, based on the available SIMD capabilities. Consequently, the value of the `nlanes` enum is determined in compile time. +
+ + Each structure follows the following convention: + + v_[type of value][size of each value in bits] + + For instance, **v_uint8 holds 8-bit unsigned integers** and **v_float32 holds 32-bit floating point values**. We then declare a register like we would declare any object in C++ + + Based on the available SIMD instruction set, a particular register will hold different number of values. + For example: If your computer supports a maximum of 256bit registers, + * *v_uint8* will hold 32 8-bit unsigned integers + * *v_float64* will hold 4 64-bit floats (doubles) + + v_uint8 a; // a is a register supporting uint8(char) data + int n = a.nlanes; // n holds 32 + + Available data type and sizes: + |Type|Size in bits| + |-:|:-| + |uint| 8, 16, 32, 64| + |int | 8, 16, 32, 64| + |float | 32, 64| + +* **Constant sized registers**: These structures have a fixed bit size and hold a constant number of values. We need to know what SIMD instruction set is supported by the system and select compatible registers. Use these only if exact bit length is necessary. +
+ + Each structure follows the convention: + + v_[type of value][size of each value in bits]x[number of values] + + Suppose we want to store + * 32-bit(*size in bits*) signed integers in a **128 bit register**. Since the register size is already known, we can find out the *number of data points in register* (*128/32 = 4*): + + v_int32x8 reg1 // holds 8 32-bit signed integers. + + * 64-bit floats in 512 bit register: + + v_float64x8 reg2 // reg2.nlanes = 8 + +### Load and Store operations + +Now that we know how registers work, let us look at the functions used for filling these registers with values. + +* **Load**: Load functions allow you to *load* values into a register. + * *Constructors* - When declaring a register structure, we can either provide a memory address from where the register will pick up contiguous values, or provide the values explicitly as multiple arguments (Explicit multiple arguments is available only for Constant Sized Registers): + + float ptr[32] = {1, 2, 3 ..., 32}; // ptr is a pointer to a contiguous memory block of 32 floats + + // Variable Sized Registers // + int x = v_float32().nlanes; // set x as the number of values the register can hold + + v_float32 reg1(ptr); // reg1 stores first x values according to the maximum register size available. + v_float32 reg2(ptr + x); // reg stores the next x values + + // Constant Sized Registers // + v_float32x4 reg1(ptr); // reg1 stores the first 4 floats (1, 2, 3, 4) + v_float32x4 reg2(ptr + 4); // reg2 stores the next 4 floats (5, 6, 7, 8) + + // Or we can explicitly write down the values. + v_float32x4(1, 2, 3, 4); + +
+ * *Load Function* - We can use the load method and provide the memory address of the data: + + float ptr[32] = {1, 2, 3, ..., 32}; + v_float32 reg_var; + reg_var = vx_load(ptr); // loads values from ptr[0] upto ptr[reg_var.nlanes - 1] + + v_float32x4 reg_128; + reg_128 = v_load(ptr); // loads values from ptr[0] upto ptr[3] + + v_float32x8 reg_256; + reg_256 = v256_load(ptr); // loads values from ptr[0] upto ptr[7] + + v_float32x16 reg_512; + reg_512 = v512_load(ptr); // loads values from ptr[0] upto ptr[15] + + @note The load function assumes data is unaligned. If your data is aligned, you may use the `vx_load_aligned()` function. +
+ +* **Store**: Store functions allow you to *store* the values from a register into a particular memory location. + * To store values from a register into a memory location, you may use the *v_store()* function: + + float ptr[4]; + v_store(ptr, reg); // store the first 128 bits(interpreted as 4x32-bit floats) of reg into ptr. +
+@note Ensure **ptr** has the same type as register. You can also cast the register into the proper type before carrying out operations. Simply typecasting the pointer to a particular type will lead wrong interpretation of data. + +### Binary and Unary Operators + +The universal intrinsics set provides element wise binary and unary operations. + +* **Arithmetics**: We can add, subtract, multiply and divide two registers element-wise. The registers must be of the same width and hold the same type. To multiply two registers, for example: + + v_float32 a, b; // {a1, ..., an}, {b1, ..., bn} + v_float32 c; + c = a + b // {a1 + b1, ..., an + bn} + c = a * b; // {a1 * b1, ..., an * bn} + +
+ +* **Bitwise Logic and Shifts**: We can left shift or right shift the bits of each element of the register. We can also apply bitwise &, |, ^ and ~ operators between two registers element-wise: + + v_int32 as; // {a1, ..., an} + v_int32 al = as << 2; // {a1 << 2, ..., an << 2} + v_int32 bl = as >> 2; // {a1 >> 2, ..., an >> 2} + + v_int32 a, b; + v_int32 a_and_b = a & b; // {a1 & b1, ..., an & bn} + +
+ +* **Comparison Operators**: We can compare values between two registers using the <, >, <= , >=, == and != operators. Since each register contains multiple values, we don't get a single bool for these operations. Instead, for true values, all bits are converted to one (0xff for 8 bits, 0xffff for 16 bits, etc), while false values return bits converted to zero. + + // let us consider the following code is run in a 128-bit register + v_uint8 a; // a = {0, 1, 2, ..., 15} + v_uint8 b; // b = {15, 14, 13, ..., 0} + + v_uint8 c = a < b; + + /* + let us look at the first 4 values in binary + + a = |00000000|00000001|00000010|00000011| + b = |00001111|00001110|00001101|00001100| + c = |11111111|11111111|11111111|11111111| + + If we store the values of c and print them as integers, we will get 255 for true values and 0 for false values. + */ + --- + // In a computer supporting 256-bit registers + v_int32 a; // a = {1, 2, 3, 4, 5, 6, 7, 8} + v_int32 b; // b = {8, 7, 6, 5, 4, 3, 2, 1} + + v_int32 c = (a < b); // c = {-1, -1, -1, -1, 0, 0, 0, 0} + + /* + The true values are 0xffffffff, which in signed 32-bit integer representation is equal to -1. + */ +
+ +* **Min/Max operations**: We can use the *v_min()* and *v_max()* functions to return registers containing element-wise min, or max, of the two registers: + + v_int32 a; // {a1, ..., an} + v_int32 b; // {b1, ..., bn} + + v_int32 mn = v_min(a, b); // {min(a1, b1), ..., min(an, bn)} + v_int32 mx = v_max(a, b); // {max(a1, b1), ..., max(an, bn)} +
+ +@note Comparison and Min/Max operators are not available for 64 bit integers. Bitwise shift and logic operators are available only for integer values. Bitwise shift is available only for 16, 32 and 64 bit registers. + +### Reduce and Mask + +* **Reduce Operations**: The *v_reduce_min()*, *v_reduce_max()* and *v_reduce_sum()* return a single value denoting the min, max or sum of the entire register: + + v_int32 a; // a = {a1, ..., a4} + int mn = v_reduce_min(a); // mn = min(a1, ..., an) + int sum = v_reduce_sum(a); // sum = a1 + ... + an +
+ +* **Mask Operations**: Mask operations allow us to replicate conditionals in wide registers. These include: + * *v_check_all()* - Returns a bool, which is true if all the values in the register are less than zero. + * *v_check_any()* - Returns a bool, which is true if any value in the register is less than zero. + * *v_select()* - Returns a register, which blends two registers, based on a mask. + + v_uint8 a; // {a1, .., an} + v_uint8 b; // {b1, ..., bn} + + v_int32x4 mask: // {0xff, 0, 0, 0xff, ..., 0xff, 0} + + v_uint8 Res = v_select(mask, a, b) // {a1, b2, b3, a4, ..., an-1, bn} + + /* + "Res" will contain the value from "a" if mask is true (all bits set to 1), + and value from "b" if mask is false (all bits set to 0) + + We can use comparison operators to generate mask and v_select to obtain results based on conditionals. + It is common to set all values of b to 0. Thus, v_select will give values of "a" or 0 based on the mask. + */ + +## Demonstration +In the following section, we will vectorize a simple convolution function for single channel and compare the results to a scalar implementation. +@note Not all algorithms are improved by manual vectorization. In fact, in certain cases, the compiler may *autovectorize* the code, thus producing faster results for scalar implementations. + +You may learn more about convolution from the previous tutorial. We use the same naive implementation from the previous tutorial and compare it to the vectorized version. + +The full tutorial code is [here](https://github.com/opencv/opencv/tree/master/samples/cpp/tutorial_code/univ_intrin/univ_intrin.cpp). + +### Vectorizing Convolution + +We will first implement a 1-D convolution and then vectorize it. The 2-D vectorized convolution will perform 1-D convolution across the rows to produce the correct results. + +#### 1-D Convolution: Scalar +@snippet univ_intrin.cpp convolution-1D-scalar + +1. We first set up variables and make a border on both sides of the src matrix, to take care of edge cases. + @snippet univ_intrin.cpp convolution-1D-border + +2. For the main loop, we select an index *i* and offset it on both sides along with the kernel, using the k variable. We store the value in *value* and add it to the *dst* matrix. + @snippet univ_intrin.cpp convolution-1D-scalar-main + +#### 1-D Convolution: Vector + +We will now look at the vectorized version of 1-D convolution. +@snippet univ_intrin.cpp convolution-1D-vector + +1. In our case, the kernel is a float. Since the kernel's datatype is the largest, we convert src to float32, forming *src_32*. We also make a border like we did for the naive case. + @snippet univ_intrin.cpp convolution-1D-convert + +2. Now, for each column in the *kernel*, we calculate the scalar product of the value with all *window* vectors of length `step`. We add these values to the already stored values in ans + @snippet univ_intrin.cpp convolution-1D-main + + * We declare a pointer to the src_32 and kernel and run a loop for each kernel element + @snippet univ_intrin.cpp convolution-1D-main-h1 + + * We load a register with the current kernel element. A window is shifted from *0* to *len - step* and its product with the kernel_wide array is added to the values stored in *ans*. We store the values back into *ans* + @snippet univ_intrin.cpp convolution-1D-main-h2 + + * Since the length might not be divisible by steps, we take care of the remaining values directly. The number of *tail* values will always be less than *step* and will not affect the performance significantly. We store all the values to *ans* which is a float pointer. We can also directly store them in a `Mat` object + @snippet univ_intrin.cpp convolution-1D-main-h3 + + * Here is an iterative example: + + For example: + kernel: {k1, k2, k3} + src: ...|a1|a2|a3|a4|... + + + iter1: + for each idx i in (0, len), 'step' idx at a time + kernel_wide: |k1|k1|k1|k1| + window: |a0|a1|a2|a3| + ans: ...| 0| 0| 0| 0|... + sum = ans + window * kernel_wide + = |a0 * k1|a1 * k1|a2 * k1|a3 * k1| + + iter2: + kernel_wide: |k2|k2|k2|k2| + window: |a1|a2|a3|a4| + ans: ...|a0 * k1|a1 * k1|a2 * k1|a3 * k1|... + sum = ans + window * kernel_wide + = |a0 * k1 + a1 * k2|a1 * k1 + a2 * k2|a2 * k1 + a3 * k2|a3 * k1 + a4 * k2| + + iter3: + kernel_wide: |k3|k3|k3|k3| + window: |a2|a3|a4|a5| + ans: ...|a0 * k1 + a1 * k2|a1 * k1 + a2 * k2|a2 * k1 + a3 * k2|a3 * k1 + a4 * k2|... + sum = sum + window * kernel_wide + = |a0*k1 + a1*k2 + a2*k3|a1*k1 + a2*k2 + a3*k3|a2*k1 + a3*k2 + a4*k3|a3*k1 + a4*k2 + a5*k3| + + +@note The function parameters also include *row*, *rowk* and *len*. These values are used when using the function as an intermediate step of 2-D convolution + +#### 2-D Convolution + +Suppose our kernel has *ksize* rows. To compute the values for a particular row, we compute the 1-D convolution of the previous *ksize/2* and the next *ksize/2* rows, with the corresponding kernel row. The final values is simply the sum of the individual 1-D convolutions +@snippet univ_intrin.cpp convolution-2D + +1. We first initialize variables and make a border above and below the *src* matrix. The left and right sides are handled by the 1-D convolution function. + @snippet univ_intrin.cpp convolution-2D-init + +2. For each row, we calculate the 1-D convolution of the rows above and below it. we then add the values to the *dst* matrix. + @snippet univ_intrin.cpp convolution-2D-main + +3. We finally convert the *dst* matrix to a *8-bit* `unsigned char` matrix + @snippet univ_intrin.cpp convolution-2D-conv + +Results +------- + +In the tutorial, we used a horizontal gradient kernel. We obtain the same output image for both methods. + +Improvement in runtime varies and will depend on the SIMD capabilities available in your CPU. diff --git a/samples/cpp/tutorial_code/core/how_to_use_OpenCV_parallel_for_/how_to_use_OpenCV_parallel_for_new.cpp b/samples/cpp/tutorial_code/core/how_to_use_OpenCV_parallel_for_/how_to_use_OpenCV_parallel_for_new.cpp new file mode 100644 index 0000000000..cfa9d22b0d --- /dev/null +++ b/samples/cpp/tutorial_code/core/how_to_use_OpenCV_parallel_for_/how_to_use_OpenCV_parallel_for_new.cpp @@ -0,0 +1,332 @@ +#include +#include +#include +#include +#include + +using namespace std; +using namespace cv; + +namespace +{ +//! [convolution-sequential] +void conv_seq(Mat src, Mat &dst, Mat kernel) +{ + //![convolution-make-borders] + int rows = src.rows, cols = src.cols; + dst = Mat(rows, cols, src.type()); + + // Taking care of edge values + // Make border = kernel.rows / 2; + + int sz = kernel.rows / 2; + copyMakeBorder(src, src, sz, sz, sz, sz, BORDER_REPLICATE); + //![convolution-make-borders] + + //! [convolution-kernel-loop] + for (int i = 0; i < rows; i++) + { + uchar *dptr = dst.ptr(i); + for (int j = 0; j < cols; j++) + { + double value = 0; + + for (int k = -sz; k <= sz; k++) + { + // slightly faster results when we create a ptr due to more efficient memory access. + uchar *sptr = src.ptr(i + sz + k); + for (int l = -sz; l <= sz; l++) + { + value += kernel.ptr(k + sz)[l + sz] * sptr[j + sz + l]; + } + } + dptr[j] = saturate_cast(value); + } + } + //! [convolution-kernel-loop] +} +//! [convolution-sequential] + +#ifdef CV_CXX11 +void conv_parallel(Mat src, Mat &dst, Mat kernel) +{ + int rows = src.rows, cols = src.cols; + + dst = Mat(rows, cols, CV_8UC1, Scalar(0)); + + // Taking care of edge values + // Make border = kernel.rows / 2; + + int sz = kernel.rows / 2; + copyMakeBorder(src, src, sz, sz, sz, sz, BORDER_REPLICATE); + + //! [convolution-parallel-cxx11] + parallel_for_(Range(0, rows * cols), [&](const Range &range) + { + for (int r = range.start; r < range.end; r++) + { + int i = r / cols, j = r % cols; + + double value = 0; + for (int k = -sz; k <= sz; k++) + { + uchar *sptr = src.ptr(i + sz + k); + for (int l = -sz; l <= sz; l++) + { + value += kernel.ptr(k + sz)[l + sz] * sptr[j + sz + l]; + } + } + dst.ptr(i)[j] = saturate_cast(value); + } + }); + //! [convolution-parallel-cxx11] +} + +void conv_parallel_row_split(Mat src, Mat &dst, Mat kernel) +{ + int rows = src.rows, cols = src.cols; + + dst = Mat(rows, cols, CV_8UC1, Scalar(0)); + + // Taking care of edge values + // Make border = kernel.rows / 2; + + int sz = kernel.rows / 2; + copyMakeBorder(src, src, sz, sz, sz, sz, BORDER_REPLICATE); + + //! [convolution-parallel-cxx11-row-split] + parallel_for_(Range(0, rows), [&](const Range &range) + { + for (int i = range.start; i < range.end; i++) + { + + uchar *dptr = dst.ptr(i); + for (int j = 0; j < cols; j++) + { + double value = 0; + for (int k = -sz; k <= sz; k++) + { + uchar *sptr = src.ptr(i + sz + k); + for (int l = -sz; l <= sz; l++) + { + value += kernel.ptr(k + sz)[l + sz] * sptr[j + sz + l]; + } + } + dptr[j] = saturate_cast(value); + } + } + }); + //! [convolution-parallel-cxx11-row-split] +} +#else + +//! [convolution-parallel] +class parallelConvolution : public ParallelLoopBody +{ +private: + Mat m_src, &m_dst; + Mat m_kernel; + int sz; + +public: + parallelConvolution(Mat src, Mat &dst, Mat kernel) + : m_src(src), m_dst(dst), m_kernel(kernel) + { + sz = kernel.rows / 2; + } + + //! [overload-full] + virtual void operator()(const Range &range) const CV_OVERRIDE + { + for (int r = range.start; r < range.end; r++) + { + int i = r / m_src.cols, j = r % m_src.cols; + + double value = 0; + for (int k = -sz; k <= sz; k++) + { + uchar *sptr = m_src.ptr(i + sz + k); + for (int l = -sz; l <= sz; l++) + { + value += m_kernel.ptr(k + sz)[l + sz] * sptr[j + sz + l]; + } + } + m_dst.ptr(i)[j] = saturate_cast(value); + } + } + //! [overload-full] +}; +//! [convolution-parallel] + +void conv_parallel(Mat src, Mat &dst, Mat kernel) +{ + int rows = src.rows, cols = src.cols; + + dst = Mat(rows, cols, CV_8UC1, Scalar(0)); + + // Taking care of edge values + // Make border = kernel.rows / 2; + + int sz = kernel.rows / 2; + copyMakeBorder(src, src, sz, sz, sz, sz, BORDER_REPLICATE); + + //! [convolution-parallel-function] + parallelConvolution obj(src, dst, kernel); + parallel_for_(Range(0, rows * cols), obj); + //! [convolution-parallel-function] +} + +//! [conv-parallel-row-split] +class parallelConvolutionRowSplit : public ParallelLoopBody +{ +private: + Mat m_src, &m_dst; + Mat m_kernel; + int sz; + +public: + parallelConvolutionRowSplit(Mat src, Mat &dst, Mat kernel) + : m_src(src), m_dst(dst), m_kernel(kernel) + { + sz = kernel.rows / 2; + } + + //! [overload-row-split] + virtual void operator()(const Range &range) const CV_OVERRIDE + { + for (int i = range.start; i < range.end; i++) + { + + uchar *dptr = dst.ptr(i); + for (int j = 0; j < cols; j++) + { + double value = 0; + for (int k = -sz; k <= sz; k++) + { + uchar *sptr = src.ptr(i + sz + k); + for (int l = -sz; l <= sz; l++) + { + value += kernel.ptr(k + sz)[l + sz] * sptr[j + sz + l]; + } + } + dptr[j] = saturate_cast(value); + } + } + } + //! [overload-row-split] +}; +//! [conv-parallel-row-split] + +void conv_parallel_row_split(Mat src, Mat &dst, Mat kernel) +{ + int rows = src.rows, cols = src.cols; + + dst = Mat(rows, cols, CV_8UC1, Scalar(0)); + + // Taking care of edge values + // Make border = kernel.rows / 2; + + int sz = kernel.rows / 2; + copyMakeBorder(src, src, sz, sz, sz, sz, BORDER_REPLICATE); + + //! [convolution-parallel-function-row] + parallelConvolutionRowSplit obj(src, dst, kernel); + parallel_for_(Range(0, rows), obj); + //! [convolution-parallel-function-row] +} + +#endif + +static void help(char *progName) +{ + cout << endl + << " This program shows how to use the OpenCV parallel_for_ function and \n" + << " compares the performance of the sequential and parallel implementations for a \n" + << " convolution operation\n" + << " Usage:\n " + << progName << " [image_path -- default lena.jpg] " << endl + << endl; +} +} + +int main(int argc, char *argv[]) +{ + + help(argv[0]); + const char *filepath = argc >= 2 ? argv[1] : "../../../../data/lena.jpg"; + + Mat src, dst, kernel; + src = imread(filepath, IMREAD_GRAYSCALE); + + if (src.empty()) + { + cerr << "Can't open [" << filepath << "]" << endl; + return EXIT_FAILURE; + } + namedWindow("Input", 1); + namedWindow("Output1", 1); + namedWindow("Output2", 1); + namedWindow("Output3", 1); + imshow("Input", src); + + kernel = (Mat_(3, 3) << 1, 0, -1, + 1, 0, -1, + 1, 0, -1); + + /* + Uncomment the kernels you want to use or write your own kernels to test out + performance. + */ + + /* + kernel = (Mat_(5, 5) << 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1); + kernel /= 100; + */ + + /* + kernel = (Mat_(3, 3) << 1, 1, 1, + 0, 0, 0, + -1, -1, -1); + + */ + + double t = (double)getTickCount(); + + conv_seq(src, dst, kernel); + + t = ((double)getTickCount() - t) / getTickFrequency(); + cout << " Sequential implementation: " << t << "s" << endl; + + imshow("Output1", dst); + waitKey(0); + + t = (double)getTickCount(); + + conv_parallel(src, dst, kernel); + + t = ((double)getTickCount() - t) / getTickFrequency(); + cout << " Parallel Implementation: " << t << "s" << endl; + + imshow("Output2", dst); + waitKey(0); + + t = (double)getTickCount(); + + conv_parallel_row_split(src, dst, kernel); + + t = ((double)getTickCount() - t) / getTickFrequency(); + cout << " Parallel Implementation(Row Split): " << t << "s" << endl + << endl; + + imshow("Output3", dst); + waitKey(0); + + // imwrite("src.png", src); + // imwrite("dst.png", dst); + + return 0; +} \ No newline at end of file diff --git a/samples/cpp/tutorial_code/core/univ_intrin/univ_intrin.cpp b/samples/cpp/tutorial_code/core/univ_intrin/univ_intrin.cpp new file mode 100644 index 0000000000..9be4170d7b --- /dev/null +++ b/samples/cpp/tutorial_code/core/univ_intrin/univ_intrin.cpp @@ -0,0 +1,230 @@ +#include +#include +#include +#include +#include +#include + +using namespace std; +using namespace cv; + +const int N = 100005, K = 2000; + +namespace +{ + +void conv_seq(Mat src, Mat &dst, Mat kernel) +{ + int rows = src.rows, cols = src.cols; + dst = Mat(rows, cols, CV_8UC1); + + int sz = kernel.rows / 2; + copyMakeBorder(src, src, sz, sz, sz, sz, BORDER_REPLICATE); + for (int i = 0; i < rows; i++) + { + uchar *dptr = dst.ptr(i); + for (int j = 0; j < cols; j++) + { + float value = 0; + + for (int k = -sz; k <= sz; k++) + { + // slightly faster results when we create a ptr due to more efficient memory access. + uchar *sptr = src.ptr(i + sz + k); + for (int l = -sz; l <= sz; l++) + { + value += kernel.ptr(k + sz)[l + sz] * sptr[j + sz + l]; + } + } + dptr[j] = saturate_cast(value); + } + } +} + +//! [convolution-1D-scalar] +void conv1d(Mat src, Mat &dst, Mat kernel) +{ + + //! [convolution-1D-border] + int len = src.cols; + dst = Mat(1, len, CV_8UC1); + + int sz = kernel.cols / 2; + copyMakeBorder(src, src, 0, 0, sz, sz, BORDER_REPLICATE); + //! [convolution-1D-border] + + //! [convolution-1D-scalar-main] + for (int i = 0; i < len; i++) + { + double value = 0; + for (int k = -sz; k <= sz; k++) + value += src.ptr(0)[i + k + sz] * kernel.ptr(0)[k + sz]; + + dst.ptr(0)[i] = saturate_cast(value); + } + //! [convolution-1D-scalar-main] +} +//! [convolution-1D-scalar] + +//! [convolution-1D-vector] +void conv1dsimd(Mat src, Mat kernel, float *ans, int row = 0, int rowk = 0, int len = -1) +{ + if (len == -1) + len = src.cols; + + //! [convolution-1D-convert] + Mat src_32, kernel_32; + + const int alpha = 1; + src.convertTo(src_32, CV_32FC1, alpha); + + int ksize = kernel.cols, sz = kernel.cols / 2; + copyMakeBorder(src_32, src_32, 0, 0, sz, sz, BORDER_REPLICATE); + //! [convolution-1D-convert] + + + //! [convolution-1D-main] + //! [convolution-1D-main-h1] + int step = v_float32().nlanes; + float *sptr = src_32.ptr(row), *kptr = kernel.ptr(rowk); + for (int k = 0; k < ksize; k++) + { + //! [convolution-1D-main-h1] + //! [convolution-1D-main-h2] + v_float32 kernel_wide = vx_setall_f32(kptr[k]); + int i; + for (i = 0; i + step < len; i += step) + { + v_float32 window = vx_load(sptr + i + k); + v_float32 sum = vx_load(ans + i) + kernel_wide * window; + v_store(ans + i, sum); + } + //! [convolution-1D-main-h2] + + //! [convolution-1D-main-h3] + for (; i < len; i++) + { + *(ans + i) += sptr[i + k]*kptr[k]; + } + //! [convolution-1D-main-h3] + } + //! [convolution-1D-main] +} +//! [convolution-1D-vector] + +//! [convolution-2D] +void convolute_simd(Mat src, Mat &dst, Mat kernel) +{ + //! [convolution-2D-init] + int rows = src.rows, cols = src.cols; + int ksize = kernel.rows, sz = ksize / 2; + dst = Mat(rows, cols, CV_32FC1); + + copyMakeBorder(src, src, sz, sz, 0, 0, BORDER_REPLICATE); + + int step = v_float32().nlanes; + //! [convolution-2D-init] + + //! [convolution-2D-main] + for (int i = 0; i < rows; i++) + { + for (int k = 0; k < ksize; k++) + { + float ans[N] = {0}; + conv1dsimd(src, kernel, ans, i + k, k, cols); + int j; + for (j = 0; j + step < cols; j += step) + { + v_float32 sum = vx_load(&dst.ptr(i)[j]) + vx_load(&ans[j]); + v_store(&dst.ptr(i)[j], sum); + } + + for (; j < cols; j++) + dst.ptr(i)[j] += ans[j]; + } + } + //! [convolution-2D-main] + + //! [convolution-2D-conv] + const int alpha = 1; + dst.convertTo(dst, CV_8UC1, alpha); + //! [convolution-2D-conv] +} +//! [convolution-2D] + +static void help(char *progName) +{ + cout << endl + << " This program shows how to use the OpenCV parallel_for_ function and \n" + << " compares the performance of the sequential and parallel implementations for a \n" + << " convolution operation\n" + << " Usage:\n " + << progName << " [image_path -- default lena.jpg] " << endl + << endl; +} +} + +int main(int argc, char *argv[]) +{ + + // 1-D Convolution // + Mat vsrc(1, N, CV_8UC1), k(1, K, CV_32FC1), vdst; + RNG rng(time(0)); + rng.RNG::fill(vsrc, RNG::UNIFORM, Scalar(0), Scalar(255)); + rng.RNG::fill(k, RNG::UNIFORM, Scalar(-50), Scalar(50)); + + double t = (double)getTickCount(); + conv1d(vsrc, vdst, k); + t = ((double)getTickCount() - t) / getTickFrequency(); + cout << " Sequential 1-D convolution implementation: " << t << "s" << endl; + + t = (double)getTickCount(); + float ans[N] = {0}; + conv1dsimd(vsrc, k, ans); + t = ((double)getTickCount() - t) / getTickFrequency(); + cout << " Vectorized 1-D convolution implementation: " << t << "s" << endl; + + // 2-D Convolution // + help(argv[0]); + + const char *filepath = argc >= 2 ? argv[1] : "../../../../data/lena.jpg"; + + Mat src, dst1, dst2, kernel; + src = imread(filepath, IMREAD_GRAYSCALE); + + if (src.empty()) + { + cerr << "Can't open [" << filepath << "]" << endl; + return EXIT_FAILURE; + } + namedWindow("Input", 1); + namedWindow("Output", 1); + imshow("Input", src); + + kernel = (Mat_(3, 3) << 1, 0, -1, + 2, 0, -2, + 1, 0, -1); + + t = (double)getTickCount(); + + conv_seq(src, dst1, kernel); + + t = ((double)getTickCount() - t) / getTickFrequency(); + cout << " Sequential 2-D convolution implementation: " << t << "s" << endl; + + imshow("Output", dst1); + waitKey(0); + + t = (double)getTickCount(); + + convolute_simd(src, dst2, kernel); + + t = ((double)getTickCount() - t) / getTickFrequency(); + cout << " Vectorized 2-D convolution implementation: " << t << "s" << endl + << endl; + + imshow("Output", dst2); + waitKey(0); + + return 0; +} \ No newline at end of file From 9c5d7716e273284d4973eefdef079436c7caaa9d Mon Sep 17 00:00:00 2001 From: SamFC10 Date: Fri, 17 Sep 2021 17:40:57 +0530 Subject: [PATCH 178/262] fix for unsqueeze opset version 13 --- modules/dnn/src/onnx/onnx_importer.cpp | 12 ++++++++++-- modules/dnn/test/test_onnx_importer.cpp | 1 + 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/modules/dnn/src/onnx/onnx_importer.cpp b/modules/dnn/src/onnx/onnx_importer.cpp index 2cd36dc94d..5343c05361 100644 --- a/modules/dnn/src/onnx/onnx_importer.cpp +++ b/modules/dnn/src/onnx/onnx_importer.cpp @@ -1693,8 +1693,16 @@ void ONNXImporter::parseFlatten(LayerParams& layerParams, const opencv_onnx::Nod void ONNXImporter::parseUnsqueeze(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) { - CV_Assert(node_proto.input_size() == 1); - DictValue axes = layerParams.get("axes"); + CV_Assert(node_proto.input_size() == 1 || node_proto.input_size() == 2); + DictValue axes; + if (node_proto.input_size() == 2) + { + Mat blob = getBlob(node_proto, 1); + axes = DictValue::arrayInt(blob.ptr(), blob.total()); + } + else + axes = layerParams.get("axes"); + if (constBlobs.find(node_proto.input(0)) != constBlobs.end()) { // Constant input. diff --git a/modules/dnn/test/test_onnx_importer.cpp b/modules/dnn/test/test_onnx_importer.cpp index f55510ec7b..b4ecd4601c 100644 --- a/modules/dnn/test/test_onnx_importer.cpp +++ b/modules/dnn/test/test_onnx_importer.cpp @@ -605,6 +605,7 @@ TEST_P(Test_ONNX_layers, DynamicReshape) TEST_P(Test_ONNX_layers, Reshape) { testONNXModels("unsqueeze"); + testONNXModels("unsqueeze_opset_13"); } TEST_P(Test_ONNX_layers, Squeeze) From 3c89a28a0612c3f44d9b44fbe28e4bf0ded564d2 Mon Sep 17 00:00:00 2001 From: Vadim Levin Date: Sat, 18 Sep 2021 10:02:55 +0300 Subject: [PATCH 179/262] Merge pull request #20611 from VadimLevin:dev/vlevin/pure-python-modules * feat: OpenCV extension with pure Python modules * feat: cv2 is now a Python package instead of extension module Python package cv2 now can handle both Python and C extension modules properly without additional "subfolders" like "_extra_py_code". * feat: can call native function from its reimplementation in Python --- .../include/opencv2/core/bindings_utils.hpp | 6 ++ .../misc/python/package/utils/__init__.py | 14 ++++ modules/python/common.cmake | 47 +++++++---- modules/python/package/cv2/__init__.py | 83 +++++++++++++++---- .../package/cv2/_extra_py_code/__init__.py | 53 ------------ .../package/extra_modules/misc/__init__.py | 1 + .../package/extra_modules/misc/version.py | 5 ++ modules/python/python3/CMakeLists.txt | 14 ++++ modules/python/python_loader.cmake | 1 - modules/python/test/test_misc.py | 31 +++++++ 10 files changed, 170 insertions(+), 85 deletions(-) create mode 100644 modules/core/misc/python/package/utils/__init__.py delete mode 100644 modules/python/package/cv2/_extra_py_code/__init__.py create mode 100644 modules/python/package/extra_modules/misc/__init__.py create mode 100644 modules/python/package/extra_modules/misc/version.py diff --git a/modules/core/include/opencv2/core/bindings_utils.hpp b/modules/core/include/opencv2/core/bindings_utils.hpp index a3f45eb0c7..f9c73dd4ff 100644 --- a/modules/core/include/opencv2/core/bindings_utils.hpp +++ b/modules/core/include/opencv2/core/bindings_utils.hpp @@ -116,6 +116,12 @@ String dumpRange(const Range& argument) } } +CV_WRAP static inline +int testOverwriteNativeMethod(int argument) +{ + return argument; +} + CV_WRAP static inline String testReservedKeywordConversion(int positional_argument, int lambda = 2, int from = 3) { diff --git a/modules/core/misc/python/package/utils/__init__.py b/modules/core/misc/python/package/utils/__init__.py new file mode 100644 index 0000000000..49cd40ba2d --- /dev/null +++ b/modules/core/misc/python/package/utils/__init__.py @@ -0,0 +1,14 @@ +from collections import namedtuple + +import cv2 + + +NativeMethodPatchedResult = namedtuple("NativeMethodPatchedResult", + ("py", "native")) + + +def testOverwriteNativeMethod(arg): + return NativeMethodPatchedResult( + arg + 1, + cv2.utils._native.testOverwriteNativeMethod(arg) + ) diff --git a/modules/python/common.cmake b/modules/python/common.cmake index cedf071434..251b78c6cb 100644 --- a/modules/python/common.cmake +++ b/modules/python/common.cmake @@ -1,6 +1,31 @@ # This file is included from a subdirectory set(PYTHON_SOURCE_DIR "${CMAKE_CURRENT_LIST_DIR}") +function(ocv_add_python_files_from_path search_path) + file(GLOB_RECURSE extra_py_files + RELATIVE "${search_path}" + # Plain Python code + "${search_path}/*.py" + # Type annotations + "${search_path}/*.pyi" + ) + message(DEBUG "Extra Py files for ${search_path}: ${extra_py_files}") + if(extra_py_files) + list(SORT extra_py_files) + foreach(filename ${extra_py_files}) + get_filename_component(module "${filename}" DIRECTORY) + if(NOT ${module} IN_LIST extra_modules) + list(APPEND extra_modules ${module}) + endif() + configure_file("${search_path}/${filename}" "${__loader_path}/cv2/${filename}" COPYONLY) + install(FILES "${search_path}/${filename}" DESTINATION "${OPENCV_PYTHON_INSTALL_PATH}/cv2/${module}/" COMPONENT python) + endforeach() + message(STATUS "Found ${extra_modules} Python modules from ${search_path}") + else() + message(WARNING "Can't add Python files and modules from ${module_path}. There is no .py or .pyi files") + endif() +endfunction() + ocv_add_module(${MODULE_NAME} BINDINGS PRIVATE_REQUIRED opencv_python_bindings_generator) include_directories(SYSTEM @@ -224,23 +249,15 @@ if(NOT OPENCV_SKIP_PYTHON_LOADER) if (";${OPENCV_MODULE_${m}_WRAPPERS};" MATCHES ";python;" AND HAVE_${m} AND EXISTS "${OPENCV_MODULE_${m}_LOCATION}/misc/python/package" ) - set(__base "${OPENCV_MODULE_${m}_LOCATION}/misc/python/package") - file(GLOB_RECURSE extra_py_files - RELATIVE "${__base}" - "${__base}/**/*.py" - ) - if(extra_py_files) - list(SORT extra_py_files) - foreach(f ${extra_py_files}) - get_filename_component(__dir "${f}" DIRECTORY) - configure_file("${__base}/${f}" "${__loader_path}/cv2/_extra_py_code/${f}" COPYONLY) - install(FILES "${__base}/${f}" DESTINATION "${OPENCV_PYTHON_INSTALL_PATH}/cv2/_extra_py_code/${__dir}/" COMPONENT python) - endforeach() - else() - message(WARNING "Module ${m} has no .py files in misc/python/package") - endif() + ocv_add_python_files_from_path("${OPENCV_MODULE_${m}_LOCATION}/misc/python/package") endif() endforeach(m) + + if(NOT "${OCV_PYTHON_EXTRA_MODULES_PATH}" STREQUAL "") + foreach(extra_ocv_py_modules_path ${OCV_PYTHON_EXTRA_MODULES_PATH}) + ocv_add_python_files_from_path(${extra_ocv_py_modules_path}) + endforeach() + endif() endif() # NOT OPENCV_SKIP_PYTHON_LOADER unset(PYTHON_SRC_DIR) diff --git a/modules/python/package/cv2/__init__.py b/modules/python/package/cv2/__init__.py index 27e7edd083..80e2612276 100644 --- a/modules/python/package/cv2/__init__.py +++ b/modules/python/package/cv2/__init__.py @@ -2,6 +2,7 @@ OpenCV Python binary extension loader ''' import os +import importlib import sys __all__ = [] @@ -15,17 +16,54 @@ except ImportError: print(' pip install numpy') raise - -py_code_loader = None -if sys.version_info[:2] >= (3, 0): - try: - from . import _extra_py_code as py_code_loader - except: - pass - # TODO # is_x64 = sys.maxsize > 2**32 + +def __load_extra_py_code_for_module(base, name, enable_debug_print=False): + module_name = "{}.{}".format(__name__, name) + export_module_name = "{}.{}".format(base, name) + native_module = sys.modules.pop(module_name, None) + try: + py_module = importlib.import_module(module_name) + except ImportError as err: + if enable_debug_print: + print("Can't load Python code for module:", module_name, + ". Reason:", err) + # Extension doesn't contain extra py code + return False + + if not hasattr(base, name): + setattr(sys.modules[base], name, py_module) + sys.modules[export_module_name] = py_module + # If it is C extension module it is already loaded by cv2 package + if native_module: + setattr(py_module, "_native", native_module) + for k, v in filter(lambda kv: not hasattr(py_module, kv[0]), + native_module.__dict__.items()): + if enable_debug_print: print(' symbol: {} = {}'.format(k, v)) + setattr(py_module, k, v) + return True + + +def __collect_extra_submodules(enable_debug_print=False): + def modules_filter(module): + return all(( + # module is not internal + not module.startswith("_"), + # it is not a file + os.path.isdir(os.path.join(_extra_submodules_init_path, module)) + )) + if sys.version_info[0] < 3: + if enable_debug_print: + print("Extra submodules is loaded only for Python 3") + return [] + + __INIT_FILE_PATH = os.path.abspath(__file__) + _extra_submodules_init_path = os.path.dirname(__INIT_FILE_PATH) + return filter(modules_filter, os.listdir(_extra_submodules_init_path)) + + def bootstrap(): import sys @@ -107,23 +145,36 @@ def bootstrap(): # amending of LD_LIBRARY_PATH works for sub-processes only os.environ['LD_LIBRARY_PATH'] = ':'.join(l_vars['BINARIES_PATHS']) + ':' + os.environ.get('LD_LIBRARY_PATH', '') - if DEBUG: print('OpenCV loader: replacing cv2 module') - del sys.modules['cv2'] - import cv2 + if DEBUG: print("Relink everything from native cv2 module to cv2 package") + + py_module = sys.modules.pop("cv2") + + native_module = importlib.import_module("cv2") + + sys.modules["cv2"] = py_module + setattr(py_module, "_native", native_module) + + for item_name, item in filter(lambda kv: kv[0] not in ("__file__", "__loader__", "__spec__", + "__name__", "__package__"), + native_module.__dict__.items()): + if item_name not in g_vars: + g_vars[item_name] = item sys.path = save_sys_path # multiprocessing should start from bootstrap code (https://github.com/opencv/opencv/issues/18502) try: - import sys del sys.OpenCV_LOADER - except: - pass + except Exception as e: + if DEBUG: + print("Exception during delete OpenCV_LOADER:", e) if DEBUG: print('OpenCV loader: binary extension... OK') - if py_code_loader: - py_code_loader.init('cv2') + for submodule in __collect_extra_submodules(DEBUG): + if __load_extra_py_code_for_module("cv2", submodule, DEBUG): + if DEBUG: print("Extra Python code for", submodule, "is loaded") if DEBUG: print('OpenCV loader: DONE') + bootstrap() diff --git a/modules/python/package/cv2/_extra_py_code/__init__.py b/modules/python/package/cv2/_extra_py_code/__init__.py deleted file mode 100644 index be84566825..0000000000 --- a/modules/python/package/cv2/_extra_py_code/__init__.py +++ /dev/null @@ -1,53 +0,0 @@ -import sys -import importlib - -__all__ = ['init'] - - -DEBUG = False -if hasattr(sys, 'OpenCV_LOADER_DEBUG'): - DEBUG = True - - -def _load_py_code(base, name): - try: - m = importlib.import_module(__name__ + name) - except ImportError: - return # extension doesn't exist? - - if DEBUG: print('OpenCV loader: added python code extension for: ' + name) - - if hasattr(m, '__all__'): - export_members = { k : getattr(m, k) for k in m.__all__ } - else: - export_members = m.__dict__ - - for k, v in export_members.items(): - if k.startswith('_'): # skip internals - continue - if isinstance(v, type(sys)): # don't bring modules - continue - if DEBUG: print(' symbol: {} = {}'.format(k, v)) - setattr(sys.modules[base + name ], k, v) - - del sys.modules[__name__ + name] - - -# TODO: listdir -def init(base): - _load_py_code(base, '.cv2') # special case - prefix = base - prefix_len = len(prefix) - - modules = [ m for m in sys.modules.keys() if m.startswith(prefix) ] - for m in modules: - m2 = m[prefix_len:] # strip prefix - if len(m2) == 0: - continue - if m2.startswith('._'): # skip internals - continue - if m2.startswith('.load_config_'): # skip helper files - continue - _load_py_code(base, m2) - - del sys.modules[__name__] diff --git a/modules/python/package/extra_modules/misc/__init__.py b/modules/python/package/extra_modules/misc/__init__.py new file mode 100644 index 0000000000..3e0559d3d4 --- /dev/null +++ b/modules/python/package/extra_modules/misc/__init__.py @@ -0,0 +1 @@ +from .version import get_ocv_version diff --git a/modules/python/package/extra_modules/misc/version.py b/modules/python/package/extra_modules/misc/version.py new file mode 100644 index 0000000000..d34705872e --- /dev/null +++ b/modules/python/package/extra_modules/misc/version.py @@ -0,0 +1,5 @@ +import cv2 + + +def get_ocv_version(): + return getattr(cv2, "__version__", "unavailable") diff --git a/modules/python/python3/CMakeLists.txt b/modules/python/python3/CMakeLists.txt index d95af21e04..0bb401f2ec 100644 --- a/modules/python/python3/CMakeLists.txt +++ b/modules/python/python3/CMakeLists.txt @@ -15,9 +15,23 @@ set(the_description "The python3 bindings") set(MODULE_NAME python3) set(MODULE_INSTALL_SUBDIR python3) +set(_ocv_extra_modules_path ${CMAKE_CURRENT_LIST_DIR}/../package/extra_modules) +set(_old_ocv_python_extra_modules_path ${OCV_PYTHON_EXTRA_MODULES_PATH}) + +if("${OCV_PYTHON_EXTRA_MODULES_PATH}" STREQUAL "") + set(OCV_PYTHON_EXTRA_MODULES_PATH ${_ocv_extra_modules_path}) +else() + list(APPEND OCV_PYTHON_EXTRA_MODULES_PATH ${_ocv_extra_modules_path}) +endif() + +unset(_ocv_extra_modules_path) + set(PYTHON PYTHON3) include(../common.cmake) +set(OCV_PYTHON_EXTRA_MODULES_PATH ${_old_ocv_python_extra_modules_path}) + +unset(_old_ocv_python_extra_modules_path) unset(MODULE_NAME) unset(MODULE_INSTALL_SUBDIR) diff --git a/modules/python/python_loader.cmake b/modules/python/python_loader.cmake index 677111b061..a872ffd763 100644 --- a/modules/python/python_loader.cmake +++ b/modules/python/python_loader.cmake @@ -25,7 +25,6 @@ endif() set(PYTHON_LOADER_FILES "setup.py" "cv2/__init__.py" "cv2/load_config_py2.py" "cv2/load_config_py3.py" - "cv2/_extra_py_code/__init__.py" ) foreach(fname ${PYTHON_LOADER_FILES}) get_filename_component(__dir "${fname}" DIRECTORY) diff --git a/modules/python/test/test_misc.py b/modules/python/test/test_misc.py index 111b1427b4..76803992dc 100644 --- a/modules/python/test/test_misc.py +++ b/modules/python/test/test_misc.py @@ -4,6 +4,12 @@ from __future__ import print_function import ctypes from functools import partial from collections import namedtuple +import sys + +if sys.version_info[0] < 3: + from collections import Sequence +else: + from collections.abc import Sequence import numpy as np import cv2 as cv @@ -585,6 +591,31 @@ class Arguments(NewOpenCVTests): self.assertEqual(ints.shape, expected_shape, "Vector of integers has wrong shape.") +class CanUsePurePythonModuleFunction(NewOpenCVTests): + def test_can_get_ocv_version(self): + import sys + if sys.version_info[0] < 3: + raise unittest.SkipTest('Python 2.x is not supported') + + self.assertEqual(cv.misc.get_ocv_version(), cv.__version__, + "Can't get package version using Python misc module") + + def test_native_method_can_be_patched(self): + import sys + + if sys.version_info[0] < 3: + raise unittest.SkipTest('Python 2.x is not supported') + + res = cv.utils.testOverwriteNativeMethod(10) + self.assertTrue(isinstance(res, Sequence), + msg="Overwritten method should return sequence. " + "Got: {} of type {}".format(res, type(res))) + self.assertSequenceEqual(res, (11, 10), + msg="Failed to overwrite native method") + res = cv.utils._native.testOverwriteNativeMethod(123) + self.assertEqual(res, 123, msg="Failed to call native method implementation") + + class SamplesFindFile(NewOpenCVTests): def test_ExistedFile(self): From ba8f9d8620de9ff6cd0b81ab422a4d5d08f5b4e0 Mon Sep 17 00:00:00 2001 From: Sergey Ivanov Date: Mon, 20 Sep 2021 16:28:32 +0300 Subject: [PATCH 180/262] Merge pull request #20601 from sivanov-work:surf_bk_test G-API: oneVPL (simplification) added surface & frame adapter * added surface & frame adapter * Add FrameAdapter unut tests * Fix compilation after rebase * Fix compilation tests * Apply some review comments * Fix compile warning * Revert back CV_Func usage * Apply some comments --- modules/gapi/CMakeLists.txt | 2 + .../accelerators/accel_policy_interface.hpp | 59 ++++++ .../surface/cpu_frame_adapter.cpp | 117 +++++++++++ .../surface/cpu_frame_adapter.hpp | 41 ++++ .../onevpl/accelerators/surface/surface.cpp | 76 +++++++ .../onevpl/accelerators/surface/surface.hpp | 102 ++++++++++ .../gapi_streaming_vpl_core_test.cpp | 185 ++++++++++++++++++ 7 files changed, 582 insertions(+) create mode 100644 modules/gapi/src/streaming/onevpl/accelerators/accel_policy_interface.hpp create mode 100644 modules/gapi/src/streaming/onevpl/accelerators/surface/cpu_frame_adapter.cpp create mode 100644 modules/gapi/src/streaming/onevpl/accelerators/surface/cpu_frame_adapter.hpp create mode 100644 modules/gapi/src/streaming/onevpl/accelerators/surface/surface.cpp create mode 100644 modules/gapi/src/streaming/onevpl/accelerators/surface/surface.hpp create mode 100644 modules/gapi/test/streaming/gapi_streaming_vpl_core_test.cpp diff --git a/modules/gapi/CMakeLists.txt b/modules/gapi/CMakeLists.txt index 08dee2f9af..56d051cf0f 100644 --- a/modules/gapi/CMakeLists.txt +++ b/modules/gapi/CMakeLists.txt @@ -169,6 +169,8 @@ set(gapi_srcs src/streaming/onevpl/file_data_provider.cpp src/streaming/onevpl/cfg_params.cpp src/streaming/onevpl/data_provider_interface_exception.cpp + src/streaming/onevpl/accelerators/surface/cpu_frame_adapter.cpp + src/streaming/onevpl/accelerators/surface/surface.cpp # Utils (ITT tracing) src/utils/itt.cpp diff --git a/modules/gapi/src/streaming/onevpl/accelerators/accel_policy_interface.hpp b/modules/gapi/src/streaming/onevpl/accelerators/accel_policy_interface.hpp new file mode 100644 index 0000000000..87b1246d25 --- /dev/null +++ b/modules/gapi/src/streaming/onevpl/accelerators/accel_policy_interface.hpp @@ -0,0 +1,59 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2021 Intel Corporation + +#ifndef GAPI_STREAMING_ONEVPL_ACCELERATORS_ACCEL_POLICY_INTERFACE_HPP +#define GAPI_STREAMING_ONEVPL_ACCELERATORS_ACCEL_POLICY_INTERFACE_HPP + +#include +#include +#include + +#include + +#ifdef HAVE_ONEVPL +#include + +namespace cv { +namespace gapi { +namespace wip { + +class Surface; +struct VPLAccelerationPolicy +{ + virtual ~VPLAccelerationPolicy() {} + + using pool_key_t = void*; + + using session_t = mfxSession; + using surface_t = Surface; + using surface_ptr_t = std::shared_ptr; + using surface_weak_ptr_t = std::weak_ptr; + using surface_ptr_ctr_t = std::function out_buf_ptr, + size_t out_buf_ptr_offset, + size_t out_buf_ptr_size)>; + + virtual void init(session_t session) = 0; + virtual void deinit(session_t session) = 0; + + // Limitation: cannot give guarantee in succesful memory realloccation + // for existing workspace in existing pool (see realloc) + // thus it is not implemented, + // PLEASE provide initial memory area large enough + virtual pool_key_t create_surface_pool(size_t pool_size, size_t surface_size_bytes, surface_ptr_ctr_t creator) = 0; + + virtual surface_weak_ptr_t get_free_surface(pool_key_t key) = 0; + virtual size_t get_free_surface_count(pool_key_t key) const = 0; + virtual size_t get_surface_count(pool_key_t key) const = 0; + + virtual cv::MediaFrame::AdapterPtr create_frame_adapter(pool_key_t key, + mfxFrameSurface1* surface) = 0; +}; +} // namespace wip +} // namespace gapi +} // namespace cv + +#endif // HAVE_ONEVPL +#endif // GAPI_STREAMING_ONEVPL_ACCELERATORS_ACCEL_POLICY_INTERFACE_HPP diff --git a/modules/gapi/src/streaming/onevpl/accelerators/surface/cpu_frame_adapter.cpp b/modules/gapi/src/streaming/onevpl/accelerators/surface/cpu_frame_adapter.cpp new file mode 100644 index 0000000000..3fd959ca8b --- /dev/null +++ b/modules/gapi/src/streaming/onevpl/accelerators/surface/cpu_frame_adapter.cpp @@ -0,0 +1,117 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2021 Intel Corporation + +#include "streaming/onevpl/accelerators/surface/cpu_frame_adapter.hpp" +#include "streaming/onevpl/accelerators/surface/surface.hpp" +#include "logger.hpp" + +#ifdef HAVE_ONEVPL + +#if (MFX_VERSION >= 2000) +#include +#endif + +#include + +namespace cv { +namespace gapi { +namespace wip { + +VPLMediaFrameCPUAdapter::VPLMediaFrameCPUAdapter(std::shared_ptr surface): + parent_surface_ptr(surface) { + + GAPI_Assert(parent_surface_ptr && "Surface is nullptr"); + parent_surface_ptr->obtain_lock(); + + GAPI_LOG_DEBUG(nullptr, "surface: " << parent_surface_ptr->get_handle() << + ", w: " << parent_surface_ptr->get_info().Width << + ", h: " << parent_surface_ptr->get_info().Height << + ", p: " << parent_surface_ptr->get_data().Pitch); +} + +VPLMediaFrameCPUAdapter::~VPLMediaFrameCPUAdapter() { + + // Each VPLMediaFrameCPUAdapter releases mfx surface counter + // The last VPLMediaFrameCPUAdapter releases shared Surface pointer + // The last surface pointer releases workspace memory + parent_surface_ptr->release_lock(); +} + +cv::GFrameDesc VPLMediaFrameCPUAdapter::meta() const { + GFrameDesc desc; + const Surface::info_t& info = parent_surface_ptr->get_info(); + switch(info.FourCC) + { + case MFX_FOURCC_I420: + throw std::runtime_error("MediaFrame doesn't support I420 type"); + break; + case MFX_FOURCC_NV12: + desc.fmt = MediaFormat::NV12; + break; + default: + throw std::runtime_error("MediaFrame unknown 'fmt' type: " + std::to_string(info.FourCC)); + } + + desc.size = cv::Size{info.Width, info.Height}; + return desc; +} + +MediaFrame::View VPLMediaFrameCPUAdapter::access(MediaFrame::Access) { + const Surface::data_t& data = parent_surface_ptr->get_data(); + const Surface::info_t& info = parent_surface_ptr->get_info(); + using stride_t = typename cv::MediaFrame::View::Strides::value_type; + + stride_t pitch = static_cast(data.Pitch); + switch(info.FourCC) { + case MFX_FOURCC_I420: + { + cv::MediaFrame::View::Ptrs pp = { + data.Y, + data.U, + data.V, + nullptr + }; + cv::MediaFrame::View::Strides ss = { + pitch, + pitch / 2, + pitch / 2, 0u + }; + return cv::MediaFrame::View(std::move(pp), std::move(ss)); + } + case MFX_FOURCC_NV12: + { + cv::MediaFrame::View::Ptrs pp = { + data.Y, + data.UV, nullptr, nullptr + }; + cv::MediaFrame::View::Strides ss = { + pitch, + pitch, 0u, 0u + }; + return cv::MediaFrame::View(std::move(pp), std::move(ss)); + } + break; + default: + throw std::runtime_error("MediaFrame unknown 'fmt' type: " + std::to_string(info.FourCC)); + } +} + +cv::util::any VPLMediaFrameCPUAdapter::blobParams() const { + GAPI_Assert("VPLMediaFrameCPUAdapter::blobParams() is not implemented"); + return {}; +} + +void VPLMediaFrameCPUAdapter::serialize(cv::gapi::s11n::IOStream&) { + GAPI_Assert("VPLMediaFrameCPUAdapter::serialize() is not implemented"); +} + +void VPLMediaFrameCPUAdapter::deserialize(cv::gapi::s11n::IIStream&) { + GAPI_Assert("VPLMediaFrameCPUAdapter::deserialize() is not implemented"); +} +} // namespace wip +} // namespace gapi +} // namespace cv +#endif // HAVE_ONEVPL diff --git a/modules/gapi/src/streaming/onevpl/accelerators/surface/cpu_frame_adapter.hpp b/modules/gapi/src/streaming/onevpl/accelerators/surface/cpu_frame_adapter.hpp new file mode 100644 index 0000000000..16111dadaf --- /dev/null +++ b/modules/gapi/src/streaming/onevpl/accelerators/surface/cpu_frame_adapter.hpp @@ -0,0 +1,41 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2021 Intel Corporation + +#ifndef GAPI_STREAMING_ONEVPL_ACCELERATORS_SURFACE_CPU_FRAME_ADAPTER_HPP +#define GAPI_STREAMING_ONEVPL_ACCELERATORS_SURFACE_CPU_FRAME_ADAPTER_HPP +#include + +#include +#include "opencv2/gapi/own/exports.hpp" // GAPI_EXPORTS + +#ifdef HAVE_ONEVPL + +namespace cv { +namespace gapi { +namespace wip { + +class Surface; +class VPLMediaFrameCPUAdapter : public cv::MediaFrame::IAdapter { +public: + // GAPI_EXPORTS for tests + GAPI_EXPORTS explicit VPLMediaFrameCPUAdapter(std::shared_ptr assoc_surface); + GAPI_EXPORTS ~VPLMediaFrameCPUAdapter(); + cv::GFrameDesc meta() const override; + MediaFrame::View access(MediaFrame::Access) override; + + // The default implementation does nothing + cv::util::any blobParams() const override; + void serialize(cv::gapi::s11n::IOStream&) override; + void deserialize(cv::gapi::s11n::IIStream&) override; +private: + std::shared_ptr parent_surface_ptr; +}; +} // namespace wip +} // namespace gapi +} // namespace cv + +#endif // HAVE_ONEVPL +#endif // GAPI_STREAMING_ONEVPL_ACCELERATORS_SURFACE_CPU_FRAME_ADAPTER_HPP diff --git a/modules/gapi/src/streaming/onevpl/accelerators/surface/surface.cpp b/modules/gapi/src/streaming/onevpl/accelerators/surface/surface.cpp new file mode 100644 index 0000000000..5d315412e1 --- /dev/null +++ b/modules/gapi/src/streaming/onevpl/accelerators/surface/surface.cpp @@ -0,0 +1,76 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2021 Intel Corporation + +#include "streaming/onevpl/accelerators/surface/surface.hpp" +#include "logger.hpp" + +#ifdef HAVE_ONEVPL + +namespace cv { +namespace gapi { +namespace wip { + +Surface::Surface(std::unique_ptr&& surf, std::shared_ptr associated_memory) : + workspace_memory_ptr(associated_memory), + mfx_surface(std::move(surf)), + mirrored_locked_count() { + + GAPI_Assert(mfx_surface && "Surface is nullptr"); + mirrored_locked_count.store(mfx_surface->Data.Locked); + GAPI_LOG_DEBUG(nullptr, "create surface: " << mfx_surface << + ", locked count: " << mfx_surface->Data.Locked); +} + +Surface::~Surface() { + GAPI_LOG_DEBUG(nullptr, "destroy surface: " << mfx_surface << + ", worspace memory counter: " << workspace_memory_ptr.use_count()); +} + +std::shared_ptr Surface::create_surface(std::unique_ptr&& surf, + std::shared_ptr accociated_memory) { + surface_ptr_t ret {new Surface(std::move(surf), accociated_memory)}; + return ret; +} + +Surface::handle_t* Surface::get_handle() const { + return mfx_surface.get(); +} + +const Surface::info_t& Surface::get_info() const { + return mfx_surface->Info; +} + +const Surface::data_t& Surface::get_data() const { + return mfx_surface->Data; +} + +size_t Surface::get_locks_count() const { + return mirrored_locked_count.load(); +} + +size_t Surface::obtain_lock() { + size_t locked_count = mirrored_locked_count.fetch_add(1); + GAPI_Assert(locked_count < std::numeric_limits::max() && "Too many references "); + mfx_surface->Data.Locked = static_cast(locked_count + 1); + GAPI_LOG_DEBUG(nullptr, "surface: " << mfx_surface.get() << + ", locked times: " << locked_count + 1); + return locked_count; // return preceding value +} + +size_t Surface::release_lock() { + size_t locked_count = mirrored_locked_count.fetch_sub(1); + GAPI_Assert(locked_count < std::numeric_limits::max() && "Too many references "); + GAPI_Assert(locked_count && "Surface lock counter is invalid"); + mfx_surface->Data.Locked = static_cast(locked_count - 1); + GAPI_LOG_DEBUG(nullptr, "surface: " << mfx_surface.get() << + ", locked times: " << locked_count - 1); + return locked_count; // return preceding value +} +} // namespace wip +} // namespace gapi +} // namespace cv + +#endif // HAVE_ONEVPL diff --git a/modules/gapi/src/streaming/onevpl/accelerators/surface/surface.hpp b/modules/gapi/src/streaming/onevpl/accelerators/surface/surface.hpp new file mode 100644 index 0000000000..3fc30099cf --- /dev/null +++ b/modules/gapi/src/streaming/onevpl/accelerators/surface/surface.hpp @@ -0,0 +1,102 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2021 Intel Corporation + +#ifndef GAPI_STREAMING_ONEVPL_ACCELERATORS_SURFACE_HPP +#define GAPI_STREAMING_ONEVPL_ACCELERATORS_SURFACE_HPP + +#include +#include + +#include "opencv2/gapi/own/exports.hpp" // GAPI_EXPORTS + +#ifdef HAVE_ONEVPL +#if (MFX_VERSION >= 2000) +#include +#endif + +#include + + +namespace cv { +namespace gapi { +namespace wip { + +/** + * @brief Inner class for managing oneVPL surface through interface `mfxFrameSurface1`. + * + * Surface has no own memory and shares accelerator allocated memory using reference counter semantics. + * So it lives till a last memory consumer (surface/accelerator/media frame) lives. + * This approach allows to support different scenarious in releasing allocated memory + * + * VPL surface `mfxFrameSurface1` support Lock-Free semantics and application MUST NOT operate with a + * surface in locked state. But VPL inner counter is not threadsafe so it would be failed in any concurrent scenario. + * std::atomic counter introduced in a way to overcome this problem. + * But only few scenarious for concurrency are supported here because it is not assumed to implement entire Surface in + * for a fully multithread approach. + * Supported concurrency scenarios deal only with transaction pair: @ref Surface::get_locks_count() against + * @ref Surface::release_lock() - which may be called from different threads. On the other hand @ref Surface::get_locks_count() against + * @ref Surface::obtain_lock() happens in single thread only. Surface doesn't support shared ownership that + * because it doesn't require thread safe guarantee between transactions: + * - @ref Surface::obtain_lock() against @ref Surface::obtain_lock() + * - @ref Surface::obtain_lock() against @ref Surface::release_lock() + * - @ref Surface::release_lock() against @ref Surface::release_lock() + */ +class Surface { + using handle_t = mfxFrameSurface1; + + std::shared_ptr workspace_memory_ptr; + std::unique_ptr mfx_surface; + std::atomic mirrored_locked_count; +public: + using info_t = mfxFrameInfo; + using data_t = mfxFrameData; + + // GAPI_EXPORTS for tests + GAPI_EXPORTS static std::shared_ptr create_surface(std::unique_ptr&& surf, + std::shared_ptr accociated_memory); + GAPI_EXPORTS ~Surface(); + + GAPI_EXPORTS handle_t* get_handle() const; + GAPI_EXPORTS const info_t& get_info() const; + GAPI_EXPORTS const data_t& get_data() const; + + /** + * Extract value thread-safe lock counter (see @ref Surface description). + * It's usual situation that counter may be instantly decreased in other thread after this method called. + * We need instantaneous value. This method syncronized in inter-threading way with @ref Surface::release_lock() + * + * @return fetched locks count. + */ + GAPI_EXPORTS size_t get_locks_count() const; + + /** + * Atomically increase value of thread-safe lock counter (see @ref Surface description). + * This method is single-threaded happens-after @ref Surface::get_locks_count() and + * multi-threaded happens-before @ref Surface::release_lock() + * + * @return locks count just before its increasing. + */ + GAPI_EXPORTS size_t obtain_lock(); + + /** + * Atomically decrease value of thread-safe lock counter (see @ref Surface description). + * This method is synchronized with @ref Surface::get_locks_count() and + * multi-threaded happens-after @ref Surface::obtain_lock() + * + * @return locks count just before its decreasing. + */ + GAPI_EXPORTS size_t release_lock(); +private: + Surface(std::unique_ptr&& surf, std::shared_ptr accociated_memory); +}; + +using surface_ptr_t = std::shared_ptr; +using surface_weak_ptr_t = std::weak_ptr; +} // namespace wip +} // namespace gapi +} // namespace cv +#endif // HAVE_ONEVPL +#endif // GAPI_STREAMING_ONEVPL_ACCELERATORS_SURFACE_HPP diff --git a/modules/gapi/test/streaming/gapi_streaming_vpl_core_test.cpp b/modules/gapi/test/streaming/gapi_streaming_vpl_core_test.cpp new file mode 100644 index 0000000000..be905029ba --- /dev/null +++ b/modules/gapi/test/streaming/gapi_streaming_vpl_core_test.cpp @@ -0,0 +1,185 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2021 Intel Corporation + + +#include "../test_precomp.hpp" + +#include "../common/gapi_tests_common.hpp" + +#include + +#include +#include + +#include +#include +#include + +#include +#include + +#include +#include +#include + +#include + +#ifdef HAVE_ONEVPL +#include "streaming/onevpl/accelerators/surface/surface.hpp" +#include "streaming/onevpl/accelerators/surface/cpu_frame_adapter.hpp" + +namespace opencv_test +{ +namespace +{ +TEST(OneVPL_Source_Surface, InitSurface) +{ + using namespace cv::gapi::wip; + + // create raw MFX handle + std::unique_ptr handle(new mfxFrameSurface1{}); + mfxFrameSurface1 *mfx_core_handle = handle.get(); + + // create preallocate surface memory: empty for test + std::shared_ptr associated_memory {}; + auto surf = Surface::create_surface(std::move(handle), associated_memory); + + // check self consistency + EXPECT_EQ(reinterpret_cast(surf->get_handle()), + reinterpret_cast(mfx_core_handle)); + EXPECT_EQ(surf->get_locks_count(), 0); + EXPECT_EQ(surf->obtain_lock(), 0); + EXPECT_EQ(surf->get_locks_count(), 1); + EXPECT_EQ(surf->release_lock(), 1); + EXPECT_EQ(surf->get_locks_count(), 0); +} + +TEST(OneVPL_Source_Surface, ConcurrentLock) +{ + using namespace cv::gapi::wip; + + // create raw MFX handle + std::unique_ptr handle(new mfxFrameSurface1{}); + + // create preallocate surface memory: empty for test + std::shared_ptr associated_memory {}; + auto surf = Surface::create_surface(std::move(handle), associated_memory); + + // check self consistency + EXPECT_EQ(surf->get_locks_count(), 0); + + // MFX internal limitation: do not exceede U16 range + // so I16 is using here + int16_t lock_counter = std::numeric_limits::max() - 1; + std::promise barrier; + std::future sync = barrier.get_future(); + + + std::thread worker_thread([&barrier, surf, lock_counter] () { + barrier.set_value(); + + // concurrent lock + for (int16_t i = 0; i < lock_counter; i ++) { + surf->obtain_lock(); + } + }); + sync.wait(); + + // concurrent lock + for (int16_t i = 0; i < lock_counter; i ++) { + surf->obtain_lock(); + } + + worker_thread.join(); + EXPECT_EQ(surf->get_locks_count(), lock_counter * 2); +} + +TEST(OneVPL_Source_Surface, MemoryLifeTime) +{ + using namespace cv::gapi::wip; + + // create preallocate surface memory + std::unique_ptr preallocated_memory_ptr(new char); + std::shared_ptr associated_memory (preallocated_memory_ptr.get(), + [&preallocated_memory_ptr] (void* ptr) { + EXPECT_TRUE(preallocated_memory_ptr); + EXPECT_EQ(preallocated_memory_ptr.get(), ptr); + preallocated_memory_ptr.reset(); + }); + + // generate surfaces + constexpr size_t surface_num = 10000; + std::vector> surfaces(surface_num); + std::generate(surfaces.begin(), surfaces.end(), [surface_num, associated_memory](){ + std::unique_ptr handle(new mfxFrameSurface1{}); + return Surface::create_surface(std::move(handle), associated_memory); + }); + + // destroy surfaces + { + std::thread deleter_thread([&surfaces]() { + surfaces.clear(); + }); + deleter_thread.join(); + } + + // workspace memory must be alive + EXPECT_EQ(surfaces.size(), 0); + EXPECT_TRUE(associated_memory != nullptr); + EXPECT_TRUE(preallocated_memory_ptr.get() != nullptr); + + // generate surfaces again + 1 + constexpr size_t surface_num_plus_one = 10001; + surfaces.resize(surface_num_plus_one); + std::generate(surfaces.begin(), surfaces.end(), [surface_num_plus_one, associated_memory](){ + std::unique_ptr handle(new mfxFrameSurface1{}); + return Surface::create_surface(std::move(handle), associated_memory); + }); + + // remember one surface + std::shared_ptr last_surface = surfaces.back(); + + // destroy another surfaces + surfaces.clear(); + + // destroy associated_memory + associated_memory.reset(); + + // workspace memory must be still alive + EXPECT_EQ(surfaces.size(), 0); + EXPECT_TRUE(associated_memory == nullptr); + EXPECT_TRUE(preallocated_memory_ptr.get() != nullptr); + + // destroy last surface + last_surface.reset(); + + // workspace memory must be freed + EXPECT_TRUE(preallocated_memory_ptr.get() == nullptr); +} + +TEST(OneVPL_Source_CPUFrameAdapter, InitFrameAdapter) +{ + using namespace cv::gapi::wip; + + // create raw MFX handle + std::unique_ptr handle(new mfxFrameSurface1{}); + + // create preallocate surface memory: empty for test + std::shared_ptr associated_memory {}; + auto surf = Surface::create_surface(std::move(handle), associated_memory); + + // check consistency + EXPECT_EQ(surf->get_locks_count(), 0); + + { + VPLMediaFrameCPUAdapter adapter(surf); + EXPECT_EQ(surf->get_locks_count(), 1); + } + EXPECT_EQ(surf->get_locks_count(), 0); +} +} +} // namespace opencv_test +#endif // HAVE_ONEVPL From b3cc8289956b70e36383d614bb1f98b5eca5be69 Mon Sep 17 00:00:00 2001 From: Giles Payne Date: Mon, 20 Sep 2021 22:44:13 +0900 Subject: [PATCH 181/262] Fix put/get functions for non-contiguous Mat on iOS/macOS --- modules/core/misc/objc/common/Mat.mm | 95 ++++++++++++++--------- modules/core/misc/objc/test/MatTest.swift | 22 +++++- 2 files changed, 78 insertions(+), 39 deletions(-) diff --git a/modules/core/misc/objc/common/Mat.mm b/modules/core/misc/objc/common/Mat.mm index 045bd8393e..dc2316c87b 100644 --- a/modules/core/misc/objc/common/Mat.mm +++ b/modules/core/misc/objc/common/Mat.mm @@ -13,25 +13,28 @@ #import "CvType.h" #import "CVObjcUtil.h" -// return true if we have reached the final index -static bool incIdx(cv::Mat* mat, std::vector& indices) { - for (int dim = mat->dims-1; dim>=0; dim--) { - indices[dim] = (indices[dim] + 1) % mat->size[dim]; - if (indices[dim] != 0) { - return false; - } +static int idx2Offset(cv::Mat* mat, std::vector& indices) { + int offset = indices[0]; + for (int dim=1; dim < mat->dims; dim++) { + offset = offset*mat->size[dim] + indices[dim]; + } + return offset; +} + +static void offset2Idx(cv::Mat* mat, size_t offset, std::vector& indices) { + for (int dim=mat->dims-1; dim>=0; dim--) { + indices[dim] = offset % mat->size[dim]; + offset = (offset - indices[dim]) / mat->size[dim]; } - return true; } // returns true if final index was reached -static bool updateIdx(cv::Mat* mat, std::vector& indices, int inc) { - for (int index = 0; index < inc; index++) { - if (incIdx(mat, indices)) { - return true; - } - } - return false; +static bool updateIdx(cv::Mat* mat, std::vector& indices, size_t inc) { + size_t currentOffset = idx2Offset(mat, indices); + size_t newOffset = currentOffset + inc; + bool reachedEnd = newOffset>=(size_t)mat->total(); + offset2Idx(mat, reachedEnd?0:newOffset, indices); + return reachedEnd; } @implementation Mat { @@ -724,22 +727,31 @@ template int getData(NSArray* indices, cv::Mat* mat, int } int arrayAvailable = count; + size_t countBytes = count * sizeof(T); + size_t remainingBytes = (size_t)(mat->total() - idx2Offset(mat, tempIndices))*mat->elemSize(); + countBytes = (countBytes>remainingBytes)?remainingBytes:countBytes; + int result = (int)countBytes; int matAvailable = getMatAvailable(mat, tempIndices); int available = MIN(arrayAvailable, matAvailable); - int result = (int)(available * mat->elemSize() / mat->channels()); if (mat->isContinuous()) { memcpy(tBuffer, mat->ptr(tempIndices.data()), available * sizeof(T)); } else { - int copyOffset = 0; - int copyCount = MIN((mat->size[mat->dims - 1] - tempIndices[mat->dims - 1]) * mat->channels(), available); - while (available > 0) { - memcpy(tBuffer + copyOffset, mat->ptr(tempIndices.data()), copyCount * sizeof(T)); - if (updateIdx(mat, tempIndices, copyCount / mat->channels())) { - break; - } - available -= copyCount; - copyOffset += copyCount * sizeof(T); - copyCount = MIN(mat->size[mat->dims-1] * mat->channels(), available); + char* buff = (char*)tBuffer; + size_t blockSize = mat->size[mat->dims-1] * mat->elemSize(); + size_t firstPartialBlockSize = (mat->size[mat->dims-1] - tempIndices[mat->dims-1]) * mat->step[mat->dims-1]; + for (int dim=mat->dims-2; dim>=0 && blockSize == mat->step[dim]; dim--) { + blockSize *= mat->size[dim]; + firstPartialBlockSize += (mat->size[dim] - (tempIndices[dim]+1)) * mat->step[dim]; + } + size_t copyCount = (countBytesptr(tempIndices.data()); + while(countBytes>0) { + memcpy(buff, data, copyCount); + updateIdx(mat, tempIndices, copyCount / mat->elemSize()); + countBytes -= copyCount; + buff += copyCount; + copyCount = countBytesptr(tempIndices.data()); } } return result; @@ -817,22 +829,31 @@ template int putData(NSArray* indices, cv::Mat* mat, int } int arrayAvailable = count; + size_t countBytes = count * sizeof(T); + size_t remainingBytes = (size_t)(mat->total() - idx2Offset(mat, tempIndices))*mat->elemSize(); + countBytes = (countBytes>remainingBytes)?remainingBytes:countBytes; + int result = (int)countBytes; int matAvailable = getMatAvailable(mat, tempIndices); int available = MIN(arrayAvailable, matAvailable); - int result = (int)(available * mat->elemSize() / mat->channels()); if (mat->isContinuous()) { memcpy(mat->ptr(tempIndices.data()), tBuffer, available * sizeof(T)); } else { - int copyOffset = 0; - int copyCount = MIN((mat->size[mat->dims - 1] - tempIndices[mat->dims - 1]) * mat->channels(), available); - while (available > 0) { - memcpy(mat->ptr(tempIndices.data()), tBuffer + copyOffset, copyCount * sizeof(T)); - if (updateIdx(mat, tempIndices, copyCount / mat->channels())) { - break; - } - available -= copyCount; - copyOffset += copyCount * sizeof(T); - copyCount = MIN(mat->size[mat->dims-1] * (int)mat->channels(), available); + char* buff = (char*)tBuffer; + size_t blockSize = mat->size[mat->dims-1] * mat->elemSize(); + size_t firstPartialBlockSize = (mat->size[mat->dims-1] - tempIndices[mat->dims-1]) * mat->step[mat->dims-1]; + for (int dim=mat->dims-2; dim>=0 && blockSize == mat->step[dim]; dim--) { + blockSize *= mat->size[dim]; + firstPartialBlockSize += (mat->size[dim] - (tempIndices[dim]+1)) * mat->step[dim]; + } + size_t copyCount = (countBytesptr(tempIndices.data()); + while(countBytes>0){ + memcpy(data, buff, copyCount); + updateIdx(mat, tempIndices, copyCount / mat->elemSize()); + countBytes -= copyCount; + buff += copyCount; + copyCount = countBytesptr(tempIndices.data()); } } return result; diff --git a/modules/core/misc/objc/test/MatTest.swift b/modules/core/misc/objc/test/MatTest.swift index 8a513505cc..2dcfedf41f 100644 --- a/modules/core/misc/objc/test/MatTest.swift +++ b/modules/core/misc/objc/test/MatTest.swift @@ -431,7 +431,7 @@ class MatTests: OpenCVTestCase { // whole Mat var bytesNum = try m.get(row: 1, col: 1, data: &buff) - XCTAssertEqual(12, bytesNum); + XCTAssertEqual(12, bytesNum) XCTAssert(buff == [110, 111, 120, 121, 130, 131]) // sub-Mat @@ -442,8 +442,26 @@ class MatTests: OpenCVTestCase { XCTAssert(buff00 == [230, 231, 240, 241]) var buff11 = [Int16](repeating: 0, count: 4) bytesNum = try sm.get(row: 1, col: 1, data: &buff11) - XCTAssertEqual(4, bytesNum); + XCTAssertEqual(4, bytesNum) XCTAssert(buff11 == [340, 341, 0, 0]) + + let m2 = Mat(sizes: [5, 6, 8], type: CvType.CV_16S) + let data:[Int16] = (0.. Date: Sun, 15 Aug 2021 20:33:06 +0000 Subject: [PATCH 182/262] python: cv.Mat wrapper over numpy.ndarray --- .../py_bindings_basics.markdown | 8 ++ .../python/package/mat_wrapper/__init__.py | 33 +++++ modules/python/src2/cv2.cpp | 88 +++++++++--- modules/python/test/test_mat.py | 131 ++++++++++++++++++ modules/python/test/tests_common.py | 1 + 5 files changed, 244 insertions(+), 17 deletions(-) create mode 100644 modules/core/misc/python/package/mat_wrapper/__init__.py create mode 100644 modules/python/test/test_mat.py diff --git a/doc/py_tutorials/py_bindings/py_bindings_basics/py_bindings_basics.markdown b/doc/py_tutorials/py_bindings/py_bindings_basics/py_bindings_basics.markdown index 2c5eccd0d6..001952deca 100644 --- a/doc/py_tutorials/py_bindings/py_bindings_basics/py_bindings_basics.markdown +++ b/doc/py_tutorials/py_bindings/py_bindings_basics/py_bindings_basics.markdown @@ -60,6 +60,14 @@ of C++. So this is the basic version of how OpenCV-Python bindings are generated. +@note There is no 1:1 mapping of numpy.ndarray on cv::Mat. For example, cv::Mat has channels field, +which is emulated as last dimension of numpy.ndarray and implicitly converted. +However, such implicit conversion has problem with passing of 3D numpy arrays into C++ code +(the last dimension is implicitly reinterpreted as number of channels). +Refer to the [issue](https://github.com/opencv/opencv/issues/19091) for workarounds if you need to process 3D arrays or ND-arrays with channels. +OpenCV 4.5.4+ has `cv.Mat` wrapper derived from `numpy.ndarray` to explicitly handle the channels behavior. + + How to extend new modules to Python? ------------------------------------ diff --git a/modules/core/misc/python/package/mat_wrapper/__init__.py b/modules/core/misc/python/package/mat_wrapper/__init__.py new file mode 100644 index 0000000000..7309c32b01 --- /dev/null +++ b/modules/core/misc/python/package/mat_wrapper/__init__.py @@ -0,0 +1,33 @@ +__all__ = [] + +import sys +import numpy as np +import cv2 as cv + +# NumPy documentation: https://numpy.org/doc/stable/user/basics.subclassing.html + +class Mat(np.ndarray): + ''' + cv.Mat wrapper for numpy array. + + Stores extra metadata information how to interpret and process of numpy array for underlying C++ code. + ''' + + def __new__(cls, arr, **kwargs): + obj = arr.view(Mat) + return obj + + def __init__(self, arr, **kwargs): + self.wrap_channels = kwargs.pop('wrap_channels', getattr(arr, 'wrap_channels', False)) + if len(kwargs) > 0: + raise TypeError('Unknown parameters: {}'.format(repr(kwargs))) + + def __array_finalize__(self, obj): + if obj is None: + return + self.wrap_channels = getattr(obj, 'wrap_channels', None) + + +Mat.__module__ = cv.__name__ +cv.Mat = Mat +cv._registerMatType(Mat) diff --git a/modules/python/src2/cv2.cpp b/modules/python/src2/cv2.cpp index e97f17470c..6231fde67f 100644 --- a/modules/python/src2/cv2.cpp +++ b/modules/python/src2/cv2.cpp @@ -49,6 +49,8 @@ static PyObject* opencv_error = NULL; +static PyTypeObject* pyopencv_Mat_TypePtr = nullptr; + class ArgInfo { public: @@ -638,10 +640,20 @@ static bool isBool(PyObject* obj) CV_NOEXCEPT return PyArray_IsScalar(obj, Bool) || PyBool_Check(obj); } +template +static std::string pycv_dumpArray(const T* arr, int n) +{ + std::ostringstream out; + out << "["; + for (int i = 0; i < n; ++i) + out << " " << arr[i]; + out << " ]"; + return out.str(); +} + // special case, when the converter needs full ArgInfo structure static bool pyopencv_to(PyObject* o, Mat& m, const ArgInfo& info) { - bool allowND = true; if(!o || o == Py_None) { if( !m.data ) @@ -727,12 +739,29 @@ static bool pyopencv_to(PyObject* o, Mat& m, const ArgInfo& info) return false; } - int size[CV_MAX_DIM+1]; - size_t step[CV_MAX_DIM+1]; size_t elemsize = CV_ELEM_SIZE1(type); const npy_intp* _sizes = PyArray_DIMS(oarr); const npy_intp* _strides = PyArray_STRIDES(oarr); + + CV_LOG_DEBUG(NULL, "Incoming ndarray '" << info.name << "': ndims=" << ndims << " _sizes=" << pycv_dumpArray(_sizes, ndims) << " _strides=" << pycv_dumpArray(_strides, ndims)); + bool ismultichannel = ndims == 3 && _sizes[2] <= CV_CN_MAX; + if (pyopencv_Mat_TypePtr && PyObject_TypeCheck(o, pyopencv_Mat_TypePtr)) + { + bool wrapChannels = false; + PyObject* pyobj_wrap_channels = PyObject_GetAttrString(o, "wrap_channels"); + if (pyobj_wrap_channels) + { + if (!pyopencv_to_safe(pyobj_wrap_channels, wrapChannels, ArgInfo("cv.Mat.wrap_channels", 0))) + { + // TODO extra message + Py_DECREF(pyobj_wrap_channels); + return false; + } + Py_DECREF(pyobj_wrap_channels); + } + ismultichannel = wrapChannels && ndims >= 1; + } for( int i = ndims-1; i >= 0 && !needcopy; i-- ) { @@ -746,14 +775,26 @@ static bool pyopencv_to(PyObject* o, Mat& m, const ArgInfo& info) needcopy = true; } - if( ismultichannel && _strides[1] != (npy_intp)elemsize*_sizes[2] ) - needcopy = true; + if (ismultichannel) + { + int channels = ndims >= 1 ? (int)_sizes[ndims - 1] : 1; + if (channels > CV_CN_MAX) + { + failmsg("%s unable to wrap channels, too high (%d > CV_CN_MAX=%d)", info.name, (int)channels, (int)CV_CN_MAX); + return false; + } + ndims--; + type |= CV_MAKETYPE(0, channels); + + if (ndims >= 1 && _strides[ndims - 1] != (npy_intp)elemsize*_sizes[ndims]) + needcopy = true; + } if (needcopy) { if (info.outputarg) { - failmsg("Layout of the output array %s is incompatible with cv::Mat (step[ndims-1] != elemsize or step[1] != elemsize*nchannels)", info.name); + failmsg("Layout of the output array %s is incompatible with cv::Mat", info.name); return false; } @@ -769,6 +810,9 @@ static bool pyopencv_to(PyObject* o, Mat& m, const ArgInfo& info) _strides = PyArray_STRIDES(oarr); } + int size[CV_MAX_DIM+1] = {}; + size_t step[CV_MAX_DIM+1] = {}; + // Normalize strides in case NPY_RELAXED_STRIDES is set size_t default_step = elemsize; for ( int i = ndims - 1; i >= 0; --i ) @@ -787,23 +831,16 @@ static bool pyopencv_to(PyObject* o, Mat& m, const ArgInfo& info) } // handle degenerate case + // FIXIT: Don't force 1D for Scalars if( ndims == 0) { size[ndims] = 1; step[ndims] = elemsize; ndims++; } - if( ismultichannel ) - { - ndims--; - type |= CV_MAKETYPE(0, size[2]); - } - - if( ndims > 2 && !allowND ) - { - failmsg("%s has more than 2 dimensions", info.name); - return false; - } +#if 1 + CV_LOG_DEBUG(NULL, "Construct Mat: ndims=" << ndims << " size=" << pycv_dumpArray(size, ndims) << " step=" << pycv_dumpArray(step, ndims) << " type=" << cv::typeToString(type)); +#endif m = Mat(ndims, size, type, PyArray_DATA(oarr), step); m.u = g_numpyAllocator.allocate(o, ndims, size, type, step); @@ -2183,7 +2220,24 @@ static int convert_to_char(PyObject *o, char *dst, const ArgInfo& info) #include "pyopencv_generated_types_content.h" #include "pyopencv_generated_funcs.h" +static PyObject* pycvRegisterMatType(PyObject *self, PyObject *value) +{ + CV_LOG_DEBUG(NULL, cv::format("pycvRegisterMatType %p %p\n", self, value)); + + if (0 == PyType_Check(value)) + { + PyErr_SetString(PyExc_TypeError, "Type argument is expected"); + return NULL; + } + + Py_INCREF(value); + pyopencv_Mat_TypePtr = (PyTypeObject*)value; + + Py_RETURN_NONE; +} + static PyMethodDef special_methods[] = { + {"_registerMatType", (PyCFunction)(pycvRegisterMatType), METH_O, "_registerMatType(cv.Mat) -> None (Internal)"}, {"redirectError", CV_PY_FN_WITH_KW(pycvRedirectError), "redirectError(onError) -> None"}, #ifdef HAVE_OPENCV_HIGHGUI {"createTrackbar", (PyCFunction)pycvCreateTrackbar, METH_VARARGS, "createTrackbar(trackbarName, windowName, value, count, onChange) -> None"}, diff --git a/modules/python/test/test_mat.py b/modules/python/test/test_mat.py new file mode 100644 index 0000000000..72614fda36 --- /dev/null +++ b/modules/python/test/test_mat.py @@ -0,0 +1,131 @@ +#!/usr/bin/env python +from __future__ import print_function + +import numpy as np +import cv2 as cv + +import os +import sys +import unittest + +from tests_common import NewOpenCVTests + +try: + if sys.version_info[:2] < (3, 0): + raise unittest.SkipTest('Python 2.x is not supported') + + + class MatTest(NewOpenCVTests): + + def test_mat_construct(self): + data = np.random.random([10, 10, 3]) + + #print(np.ndarray.__dictoffset__) # 0 + #print(cv.Mat.__dictoffset__) # 88 (> 0) + #print(cv.Mat) # + #print(cv.Mat.__base__) # + + mat_data0 = cv.Mat(data) + assert isinstance(mat_data0, cv.Mat) + assert isinstance(mat_data0, np.ndarray) + self.assertEqual(mat_data0.wrap_channels, False) + res0 = cv.utils.dumpInputArray(mat_data0) + self.assertEqual(res0, "InputArray: empty()=false kind=0x00010000 flags=0x01010000 total(-1)=300 dims(-1)=3 size(-1)=[10 10 3] type(-1)=CV_64FC1") + + mat_data1 = cv.Mat(data, wrap_channels=True) + assert isinstance(mat_data1, cv.Mat) + assert isinstance(mat_data1, np.ndarray) + self.assertEqual(mat_data1.wrap_channels, True) + res1 = cv.utils.dumpInputArray(mat_data1) + self.assertEqual(res1, "InputArray: empty()=false kind=0x00010000 flags=0x01010000 total(-1)=100 dims(-1)=2 size(-1)=10x10 type(-1)=CV_64FC3") + + mat_data2 = cv.Mat(mat_data1) + assert isinstance(mat_data2, cv.Mat) + assert isinstance(mat_data2, np.ndarray) + self.assertEqual(mat_data2.wrap_channels, True) # fail if __array_finalize__ doesn't work + res2 = cv.utils.dumpInputArray(mat_data2) + self.assertEqual(res2, "InputArray: empty()=false kind=0x00010000 flags=0x01010000 total(-1)=100 dims(-1)=2 size(-1)=10x10 type(-1)=CV_64FC3") + + + def test_mat_construct_4d(self): + data = np.random.random([5, 10, 10, 3]) + + mat_data0 = cv.Mat(data) + assert isinstance(mat_data0, cv.Mat) + assert isinstance(mat_data0, np.ndarray) + self.assertEqual(mat_data0.wrap_channels, False) + res0 = cv.utils.dumpInputArray(mat_data0) + self.assertEqual(res0, "InputArray: empty()=false kind=0x00010000 flags=0x01010000 total(-1)=1500 dims(-1)=4 size(-1)=[5 10 10 3] type(-1)=CV_64FC1") + + mat_data1 = cv.Mat(data, wrap_channels=True) + assert isinstance(mat_data1, cv.Mat) + assert isinstance(mat_data1, np.ndarray) + self.assertEqual(mat_data1.wrap_channels, True) + res1 = cv.utils.dumpInputArray(mat_data1) + self.assertEqual(res1, "InputArray: empty()=false kind=0x00010000 flags=0x01010000 total(-1)=500 dims(-1)=3 size(-1)=[5 10 10] type(-1)=CV_64FC3") + + mat_data2 = cv.Mat(mat_data1) + assert isinstance(mat_data2, cv.Mat) + assert isinstance(mat_data2, np.ndarray) + self.assertEqual(mat_data2.wrap_channels, True) # __array_finalize__ doesn't work + res2 = cv.utils.dumpInputArray(mat_data2) + self.assertEqual(res2, "InputArray: empty()=false kind=0x00010000 flags=0x01010000 total(-1)=500 dims(-1)=3 size(-1)=[5 10 10] type(-1)=CV_64FC3") + + + def test_mat_wrap_channels_fail(self): + data = np.random.random([2, 3, 4, 520]) + + mat_data0 = cv.Mat(data) + assert isinstance(mat_data0, cv.Mat) + assert isinstance(mat_data0, np.ndarray) + self.assertEqual(mat_data0.wrap_channels, False) + res0 = cv.utils.dumpInputArray(mat_data0) + self.assertEqual(res0, "InputArray: empty()=false kind=0x00010000 flags=0x01010000 total(-1)=12480 dims(-1)=4 size(-1)=[2 3 4 520] type(-1)=CV_64FC1") + + with self.assertRaises(cv.error): + mat_data1 = cv.Mat(data, wrap_channels=True) # argument unable to wrap channels, too high (520 > CV_CN_MAX=512) + res1 = cv.utils.dumpInputArray(mat_data1) + print(mat_data1.__dict__) + print(res1) + + + def test_ufuncs(self): + data = np.arange(10) + mat_data = cv.Mat(data) + mat_data2 = 2 * mat_data + self.assertEqual(type(mat_data2), cv.Mat) + np.testing.assert_equal(2 * data, 2 * mat_data) + + + def test_comparison(self): + # Undefined behavior, do NOT use that. + # Behavior may be changed in the future + + data = np.ones((10, 10, 3)) + mat_wrapped = cv.Mat(data, wrap_channels=True) + mat_simple = cv.Mat(data) + np.testing.assert_equal(mat_wrapped, mat_simple) # ???: wrap_channels is not checked for now + np.testing.assert_equal(data, mat_simple) + np.testing.assert_equal(data, mat_wrapped) + + #self.assertEqual(mat_wrapped, mat_simple) # ??? + #self.assertTrue(mat_wrapped == mat_simple) # ??? + #self.assertTrue((mat_wrapped == mat_simple).all()) + + +except unittest.SkipTest as e: + + message = str(e) + + class TestSkip(unittest.TestCase): + def setUp(self): + self.skipTest('Skip tests: ' + message) + + def test_skip(): + pass + + pass + + +if __name__ == '__main__': + NewOpenCVTests.bootstrap() diff --git a/modules/python/test/tests_common.py b/modules/python/test/tests_common.py index a938a8e2cb..2245722939 100644 --- a/modules/python/test/tests_common.py +++ b/modules/python/test/tests_common.py @@ -10,6 +10,7 @@ import random import argparse import numpy as np +#sys.OpenCV_LOADER_DEBUG = True import cv2 as cv # Python 3 moved urlopen to urllib.requests From f7b4b750d8930b5bb6696cea6d609dc70a0597db Mon Sep 17 00:00:00 2001 From: mikael Date: Tue, 21 Sep 2021 19:46:33 +0200 Subject: [PATCH 183/262] Detect FP16 on FreeBSD aarch64 --- modules/core/src/system.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/core/src/system.cpp b/modules/core/src/system.cpp index d8b8f67559..df9e8a0ce7 100644 --- a/modules/core/src/system.cpp +++ b/modules/core/src/system.cpp @@ -533,7 +533,7 @@ struct HWFeatures } #endif // CV_CPUID_X86 - #if defined __ANDROID__ || defined __linux__ + #if defined __ANDROID__ || defined __linux__ || defined __FreeBSD__ #ifdef __aarch64__ have[CV_CPU_NEON] = true; have[CV_CPU_FP16] = true; @@ -559,7 +559,7 @@ struct HWFeatures CV_LOG_INFO(NULL, "- FP16 instructions is NOT enabled via build flags"); #endif #endif - #elif defined __arm__ + #elif defined __arm__ && !defined __FreeBSD__ int cpufile = open("/proc/self/auxv", O_RDONLY); if (cpufile >= 0) From 38b9ec7a18e95140a1f34d7b1c52f4fe74c14459 Mon Sep 17 00:00:00 2001 From: rogday Date: Wed, 22 Sep 2021 15:17:37 +0300 Subject: [PATCH 184/262] Merge pull request #20682 from rogday:min * Add Min layer to CPU, OpenCL, Halide, Inference Engine, NGraph and CUDA * fix indentation * add min to fusion and halide tests; fix doc --- .../dnn/include/opencv2/dnn/all_layers.hpp | 2 +- modules/dnn/src/cuda/eltwise_ops.cu | 7 ++++ modules/dnn/src/cuda/functors.hpp | 15 ++++++++ .../dnn/src/cuda4dnn/kernels/eltwise_ops.hpp | 3 ++ .../dnn/src/cuda4dnn/primitives/eltwise.hpp | 5 ++- modules/dnn/src/layers/eltwise_layer.cpp | 34 ++++++++++++++++++- modules/dnn/src/onnx/onnx_importer.cpp | 10 +++--- modules/dnn/test/test_halide_layers.cpp | 2 +- modules/dnn/test/test_layers.cpp | 2 +- modules/dnn/test/test_onnx_importer.cpp | 5 +++ 10 files changed, 76 insertions(+), 9 deletions(-) diff --git a/modules/dnn/include/opencv2/dnn/all_layers.hpp b/modules/dnn/include/opencv2/dnn/all_layers.hpp index fbe16850d4..cfe6595d78 100644 --- a/modules/dnn/include/opencv2/dnn/all_layers.hpp +++ b/modules/dnn/include/opencv2/dnn/all_layers.hpp @@ -610,7 +610,7 @@ CV__DNN_INLINE_NS_BEGIN /** @brief Element wise operation on inputs Extra optional parameters: - - "operation" as string. Values are "sum" (default), "prod", "max", "div" + - "operation" as string. Values are "sum" (default), "prod", "max", "div", "min" - "coeff" as float array. Specify weights of inputs for SUM operation - "output_channels_mode" as string. Values are "same" (default, all input must have the same layout), "input_0", "input_0_truncate", "max_input_channels" */ diff --git a/modules/dnn/src/cuda/eltwise_ops.cu b/modules/dnn/src/cuda/eltwise_ops.cu index b24801531f..109c3fbdc9 100644 --- a/modules/dnn/src/cuda/eltwise_ops.cu +++ b/modules/dnn/src/cuda/eltwise_ops.cu @@ -74,6 +74,11 @@ void eltwise_max_2(const Stream& stream, Span output, View x, View y) { eltwise_op>(stream, output, x, y); } +template +void eltwise_min_2(const Stream& stream, Span output, View x, View y) { + eltwise_op>(stream, output, x, y); +} + template void eltwise_sum_2(const Stream& stream, Span output, View x, View y) { eltwise_op>(stream, output, x, y); @@ -100,11 +105,13 @@ void eltwise_div_2(const Stream& stream, Span output, View x, View y) { template void eltwise_sum_coeff_2(const Stream&, Span<__half>, __half, View<__half>, __half, View<__half>); template void eltwise_sum_2(const Stream& stream, Span<__half> output, View<__half> x, View<__half> y); template void eltwise_max_2(const Stream& stream, Span<__half> output, View<__half> x, View<__half> y); + template void eltwise_min_2(const Stream& stream, Span<__half> output, View<__half> x, View<__half> y); #endif template void eltwise_div_2(const Stream& stream, Span output, View x, View y); template void eltwise_prod_2(const Stream& stream, Span output, View x, View y); template void eltwise_sum_coeff_2(const Stream&, Span, float, View, float, View); template void eltwise_sum_2(const Stream& stream, Span output, View x, View y); template void eltwise_max_2(const Stream& stream, Span output, View x, View y); + template void eltwise_min_2(const Stream& stream, Span output, View x, View y); }}}} /* namespace cv::dnn::cuda4dnn::kernels */ diff --git a/modules/dnn/src/cuda/functors.hpp b/modules/dnn/src/cuda/functors.hpp index 1c29de0426..f01a07c77e 100644 --- a/modules/dnn/src/cuda/functors.hpp +++ b/modules/dnn/src/cuda/functors.hpp @@ -262,6 +262,21 @@ struct MaxFunctor { } }; +template +struct MinFunctor { + struct Params { + CUDA4DNN_HOST_DEVICE Params() { } + }; + + CUDA4DNN_DEVICE MinFunctor() { } + CUDA4DNN_DEVICE MinFunctor(const Params& params) { } + + CUDA4DNN_DEVICE T operator()(T x, T y) { + using csl::device::min; + return min(x, y); + } +}; + template struct SumFunctor { struct Params { diff --git a/modules/dnn/src/cuda4dnn/kernels/eltwise_ops.hpp b/modules/dnn/src/cuda4dnn/kernels/eltwise_ops.hpp index 092b1571af..096ba04e48 100644 --- a/modules/dnn/src/cuda4dnn/kernels/eltwise_ops.hpp +++ b/modules/dnn/src/cuda4dnn/kernels/eltwise_ops.hpp @@ -15,6 +15,9 @@ namespace cv { namespace dnn { namespace cuda4dnn { namespace kernels { template void eltwise_max_2(const csl::Stream& stream, csl::Span output, csl::View x, csl::View y); + template + void eltwise_min_2(const csl::Stream& stream, csl::Span output, csl::View x, csl::View y); + template void eltwise_sum_2(const csl::Stream& stream, csl::Span output, csl::View x, csl::View y); diff --git a/modules/dnn/src/cuda4dnn/primitives/eltwise.hpp b/modules/dnn/src/cuda4dnn/primitives/eltwise.hpp index adcd7ab3f8..b46f0d870f 100644 --- a/modules/dnn/src/cuda4dnn/primitives/eltwise.hpp +++ b/modules/dnn/src/cuda4dnn/primitives/eltwise.hpp @@ -25,7 +25,8 @@ namespace cv { namespace dnn { namespace cuda4dnn { MAX, SUM, PRODUCT, - DIV + DIV, + MIN, }; class EltwiseOpBase : public CUDABackendNode { @@ -78,6 +79,7 @@ namespace cv { namespace dnn { namespace cuda4dnn { switch (op) { case EltwiseOpType::MAX: kernels::eltwise_max_2(stream, output, input_x, input_y); break; + case EltwiseOpType::MIN: kernels::eltwise_min_2(stream, output, input_x, input_y); break; case EltwiseOpType::PRODUCT: kernels::eltwise_prod_2(stream, output, input_x, input_y); break; case EltwiseOpType::DIV: kernels::eltwise_div_2(stream, output, input_x, input_y); break; case EltwiseOpType::SUM: @@ -104,6 +106,7 @@ namespace cv { namespace dnn { namespace cuda4dnn { switch (op) { case EltwiseOpType::MAX: kernels::eltwise_max_2(stream, output, output, input); break; + case EltwiseOpType::MIN: kernels::eltwise_min_2(stream, output, output, input); break; case EltwiseOpType::PRODUCT: kernels::eltwise_prod_2(stream, output, output, input); break; case EltwiseOpType::DIV: kernels::eltwise_div_2(stream, output, output, input); break; case EltwiseOpType::SUM: diff --git a/modules/dnn/src/layers/eltwise_layer.cpp b/modules/dnn/src/layers/eltwise_layer.cpp index 860560213d..2c473ff412 100644 --- a/modules/dnn/src/layers/eltwise_layer.cpp +++ b/modules/dnn/src/layers/eltwise_layer.cpp @@ -71,7 +71,8 @@ public: PROD = 0, SUM = 1, MAX = 2, - DIV = 3 + DIV = 3, + MIN = 4, } op; std::vector coeffs; @@ -109,6 +110,8 @@ public: op = SUM; else if (operation == "max") op = MAX; + else if (operation == "min") + op = MIN; else if (operation == "div") op = DIV; else @@ -470,6 +473,13 @@ public: dstptr[j] = std::max(srcptr0[j], srcptrI[j]); } } + else if (op == MIN) + { + for (int j = 0; j < blockSize; j++) + { + dstptr[j] = std::min(srcptr0[j], srcptrI[j]); + } + } else if (op == SUM) { if (!coeffsptr || (coeffsptr[0] == 1.0f && coeffsptr[1] == 1.0f)) @@ -524,6 +534,13 @@ public: dstptr[j] = std::max(dstptr[j], srcptrI[j]); } } + else if (op == MIN) + { + for (int j = 0; j < blockSize; j++) + { + dstptr[j] = std::min(dstptr[j], srcptrI[j]); + } + } else if (op == SUM) { if (!coeffsptr || coeffsptr[inputIdx] == 1.0f) @@ -641,6 +658,11 @@ public: for (int i = 2; i < inputs.size(); ++i) max(inputs[i], outputs[0], outputs[0]); break; + case MIN: + min(inputs[0], inputs[1], outputs[0]); + for (int i = 2; i < inputs.size(); ++i) + min(inputs[i], outputs[0], outputs[0]); + break; default: return false; } @@ -745,6 +767,7 @@ public: auto op_ = [this] { switch (op) { case MAX: return cuda4dnn::EltwiseOpType::MAX; + case MIN: return cuda4dnn::EltwiseOpType::MIN; case SUM: return cuda4dnn::EltwiseOpType::SUM; case PROD: return cuda4dnn::EltwiseOpType::PRODUCT; case DIV: return cuda4dnn::EltwiseOpType::DIV; @@ -799,6 +822,12 @@ public: for (int i = 2; i < inputBuffers.size(); ++i) topExpr = max(topExpr, inputBuffers[i](x, y, c, n)); break; + case MIN: + topExpr = min(inputBuffers[0](x, y, c, n), + inputBuffers[1](x, y, c, n)); + for (int i = 2; i < inputBuffers.size(); ++i) + topExpr = min(topExpr, inputBuffers[i](x, y, c, n)); + break; default: return Ptr(); } @@ -823,6 +852,8 @@ public: ieLayer.setEltwiseType(InferenceEngine::Builder::EltwiseLayer::EltwiseType::DIV); else if (op == MAX) ieLayer.setEltwiseType(InferenceEngine::Builder::EltwiseLayer::EltwiseType::MAX); + else if (op == MIN) + ieLayer.setEltwiseType(InferenceEngine::Builder::EltwiseLayer::EltwiseType::MIN); else CV_Error(Error::StsNotImplemented, "Unsupported eltwise operation"); @@ -857,6 +888,7 @@ public: case PROD: curr_node = std::make_shared(curr_node, next_node); break; case DIV: curr_node = std::make_shared(curr_node, next_node); break; case MAX: curr_node = std::make_shared(curr_node, next_node); break; + case MIN: curr_node = std::make_shared(curr_node, next_node); break; default: CV_Error(Error::StsNotImplemented, "Unsupported eltwise operation"); } } diff --git a/modules/dnn/src/onnx/onnx_importer.cpp b/modules/dnn/src/onnx/onnx_importer.cpp index 82ab806225..9cc81768c3 100644 --- a/modules/dnn/src/onnx/onnx_importer.cpp +++ b/modules/dnn/src/onnx/onnx_importer.cpp @@ -105,7 +105,7 @@ private: void parseSplit (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); void parseBias (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); void parsePow (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); - void parseMax (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseMinMax (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); void parseNeg (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); void parseConstant (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); void parseLSTM (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); @@ -1105,10 +1105,12 @@ void ONNXImporter::parsePow(LayerParams& layerParams, const opencv_onnx::NodePro addLayer(layerParams, node_proto); } -void ONNXImporter::parseMax(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +// "Min" "Max" +void ONNXImporter::parseMinMax(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) { + const std::string& layer_type = node_proto.op_type(); layerParams.type = "Eltwise"; - layerParams.set("operation", "max"); + layerParams.set("operation", layer_type == "Max" ? "max" : "min"); addLayer(layerParams, node_proto); } @@ -2421,7 +2423,7 @@ const ONNXImporter::DispatchMap ONNXImporter::buildDispatchMap() dispatch["Split"] = &ONNXImporter::parseSplit; dispatch["Add"] = dispatch["Sum"] = dispatch["Sub"] = &ONNXImporter::parseBias; dispatch["Pow"] = &ONNXImporter::parsePow; - dispatch["Max"] = &ONNXImporter::parseMax; + dispatch["Min"] = dispatch["Max"] = &ONNXImporter::parseMinMax; dispatch["Neg"] = &ONNXImporter::parseNeg; dispatch["Constant"] = &ONNXImporter::parseConstant; dispatch["LSTM"] = &ONNXImporter::parseLSTM; diff --git a/modules/dnn/test/test_halide_layers.cpp b/modules/dnn/test/test_halide_layers.cpp index 165ee4d67b..a6c1b7d0f8 100644 --- a/modules/dnn/test/test_halide_layers.cpp +++ b/modules/dnn/test/test_halide_layers.cpp @@ -893,7 +893,7 @@ TEST_P(Eltwise, Accuracy) INSTANTIATE_TEST_CASE_P(Layer_Test_Halide, Eltwise, Combine( /*input size*/ Values(Vec3i(1, 4, 5), Vec3i(2, 8, 6)), -/*operation*/ Values("prod", "sum", "div", "max"), +/*operation*/ Values("prod", "sum", "div", "max", "min"), /*num convs*/ Values(1, 2, 3), /*weighted(for sum only)*/ Bool(), dnnBackendsAndTargetsWithHalide() diff --git a/modules/dnn/test/test_layers.cpp b/modules/dnn/test/test_layers.cpp index 04d5fa6355..ee0629b853 100644 --- a/modules/dnn/test/test_layers.cpp +++ b/modules/dnn/test/test_layers.cpp @@ -2340,7 +2340,7 @@ public: static testing::internal::ParamGenerator eltwiseOpList() { // TODO: automate list generation - return Values("sum", "max", "prod", "div"); + return Values("sum", "max", "min", "prod", "div"); } static testing::internal::ParamGenerator activationLayersList() diff --git a/modules/dnn/test/test_onnx_importer.cpp b/modules/dnn/test/test_onnx_importer.cpp index 103582fbdd..ca3c29fbaf 100644 --- a/modules/dnn/test/test_onnx_importer.cpp +++ b/modules/dnn/test/test_onnx_importer.cpp @@ -301,6 +301,11 @@ TEST_P(Test_ONNX_layers, ReduceMax) testONNXModels("reduce_max_axis_1"); } +TEST_P(Test_ONNX_layers, Min) +{ + testONNXModels("min", npy, 0, 0, false, true, 2); +} + TEST_P(Test_ONNX_layers, Scale) { if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019) From 86a51015b1b5e1d2c555aff6abdf0f50d4ed3f5a Mon Sep 17 00:00:00 2001 From: Amir Tulegenov Date: Wed, 22 Sep 2021 20:42:32 +0600 Subject: [PATCH 185/262] Merge pull request #19554 from amirtu:OCV-215_cvtColorTwoPlane_wrong_output_when_Y_Plane_Mat_has_step different paddings in cvtColorTwoPlane() for biplane YUV420 * Different paddings support in cvtColorTwoPlane() for biplane YUV420 * Build fix for dispatch case. * Resoted old behaviour for y.step==uv.step to exclude perf regressions. Co-authored-by: amir.tulegenov Co-authored-by: Alexander Smorkalov --- 3rdparty/carotene/hal/tegra_hal.hpp | 39 ++++++++------- 3rdparty/openvx/hal/openvx_hal.cpp | 14 ++++-- 3rdparty/openvx/hal/openvx_hal.hpp | 3 ++ .../include/opencv2/imgproc/hal/hal.hpp | 5 ++ modules/imgproc/src/color_yuv.dispatch.cpp | 40 +++++++++++---- modules/imgproc/src/color_yuv.simd.hpp | 49 ++++++++----------- modules/imgproc/src/hal_replacement.hpp | 35 +++++++++++++ modules/imgproc/test/test_color.cpp | 34 +++++++++---- 8 files changed, 149 insertions(+), 70 deletions(-) diff --git a/3rdparty/carotene/hal/tegra_hal.hpp b/3rdparty/carotene/hal/tegra_hal.hpp index 2dfa30b619..7fcca975fe 100644 --- a/3rdparty/carotene/hal/tegra_hal.hpp +++ b/3rdparty/carotene/hal/tegra_hal.hpp @@ -1778,30 +1778,30 @@ TegraCvtColor_Invoker(bgrx2hsvf, bgrx2hsv, src_data + static_cast(range. : CV_HAL_ERROR_NOT_IMPLEMENTED \ ) -#define TEGRA_CVT2PYUVTOBGR(src_data, src_step, dst_data, dst_step, dst_width, dst_height, dcn, swapBlue, uIdx) \ +#define TEGRA_CVT2PYUVTOBGR_EX(y_data, y_step, uv_data, uv_step, dst_data, dst_step, dst_width, dst_height, dcn, swapBlue, uIdx) \ ( \ CAROTENE_NS::isSupportedConfiguration() ? \ dcn == 3 ? \ uIdx == 0 ? \ (swapBlue ? \ CAROTENE_NS::yuv420i2rgb(CAROTENE_NS::Size2D(dst_width, dst_height), \ - src_data, src_step, \ - src_data + src_step * dst_height, src_step, \ + y_data, y_step, \ + uv_data, uv_step, \ dst_data, dst_step) : \ CAROTENE_NS::yuv420i2bgr(CAROTENE_NS::Size2D(dst_width, dst_height), \ - src_data, src_step, \ - src_data + src_step * dst_height, src_step, \ + y_data, y_step, \ + uv_data, uv_step, \ dst_data, dst_step)), \ CV_HAL_ERROR_OK : \ uIdx == 1 ? \ (swapBlue ? \ CAROTENE_NS::yuv420sp2rgb(CAROTENE_NS::Size2D(dst_width, dst_height), \ - src_data, src_step, \ - src_data + src_step * dst_height, src_step, \ + y_data, y_step, \ + uv_data, uv_step, \ dst_data, dst_step) : \ CAROTENE_NS::yuv420sp2bgr(CAROTENE_NS::Size2D(dst_width, dst_height), \ - src_data, src_step, \ - src_data + src_step * dst_height, src_step, \ + y_data, y_step, \ + uv_data, uv_step, \ dst_data, dst_step)), \ CV_HAL_ERROR_OK : \ CV_HAL_ERROR_NOT_IMPLEMENTED : \ @@ -1809,29 +1809,32 @@ TegraCvtColor_Invoker(bgrx2hsvf, bgrx2hsv, src_data + static_cast(range. uIdx == 0 ? \ (swapBlue ? \ CAROTENE_NS::yuv420i2rgbx(CAROTENE_NS::Size2D(dst_width, dst_height), \ - src_data, src_step, \ - src_data + src_step * dst_height, src_step, \ + y_data, y_step, \ + uv_data, uv_step, \ dst_data, dst_step) : \ CAROTENE_NS::yuv420i2bgrx(CAROTENE_NS::Size2D(dst_width, dst_height), \ - src_data, src_step, \ - src_data + src_step * dst_height, src_step, \ + y_data, y_step, \ + uv_data, uv_step, \ dst_data, dst_step)), \ CV_HAL_ERROR_OK : \ uIdx == 1 ? \ (swapBlue ? \ CAROTENE_NS::yuv420sp2rgbx(CAROTENE_NS::Size2D(dst_width, dst_height), \ - src_data, src_step, \ - src_data + src_step * dst_height, src_step, \ + y_data, y_step, \ + uv_data, uv_step, \ dst_data, dst_step) : \ CAROTENE_NS::yuv420sp2bgrx(CAROTENE_NS::Size2D(dst_width, dst_height), \ - src_data, src_step, \ - src_data + src_step * dst_height, src_step, \ + y_data, y_step, \ + uv_data, uv_step, \ dst_data, dst_step)), \ CV_HAL_ERROR_OK : \ CV_HAL_ERROR_NOT_IMPLEMENTED : \ CV_HAL_ERROR_NOT_IMPLEMENTED \ : CV_HAL_ERROR_NOT_IMPLEMENTED \ ) +#define TEGRA_CVT2PYUVTOBGR(src_data, src_step, dst_data, dst_step, dst_width, dst_height, dcn, swapBlue, uIdx) \ + TEGRA_CVT2PYUVTOBGR_EX(src_data, src_step, src_data + src_step * dst_height, src_step, dst_data, dst_step, \ + dst_width, dst_height, dcn, swapBlue, uIdx); #undef cv_hal_cvtBGRtoBGR #define cv_hal_cvtBGRtoBGR TEGRA_CVTBGRTOBGR @@ -1847,6 +1850,8 @@ TegraCvtColor_Invoker(bgrx2hsvf, bgrx2hsv, src_data + static_cast(range. #define cv_hal_cvtBGRtoHSV TEGRA_CVTBGRTOHSV #undef cv_hal_cvtTwoPlaneYUVtoBGR #define cv_hal_cvtTwoPlaneYUVtoBGR TEGRA_CVT2PYUVTOBGR +#undef cv_hal_cvtTwoPlaneYUVtoBGREx +#define cv_hal_cvtTwoPlaneYUVtoBGREx TEGRA_CVT2PYUVTOBGR_EX #endif // OPENCV_IMGPROC_HAL_INTERFACE_H diff --git a/3rdparty/openvx/hal/openvx_hal.cpp b/3rdparty/openvx/hal/openvx_hal.cpp index 53a2711d52..4987482392 100644 --- a/3rdparty/openvx/hal/openvx_hal.cpp +++ b/3rdparty/openvx/hal/openvx_hal.cpp @@ -923,6 +923,11 @@ int ovx_hal_cvtGraytoBGR(const uchar * a, size_t astep, uchar * b, size_t bstep, } int ovx_hal_cvtTwoPlaneYUVtoBGR(const uchar * a, size_t astep, uchar * b, size_t bstep, int w, int h, int bcn, bool swapBlue, int uIdx) +{ + return ovx_hal_cvtTwoPlaneYUVtoBGREx(a, astep, a + h * astep, astep, b, bstep, w, h, bcn, swapBlue, uIdx); +} + +int ovx_hal_cvtTwoPlaneYUVtoBGREx(const uchar * a, size_t astep, const uchar * b, size_t bstep, uchar * c, size_t cstep, int w, int h, int bcn, bool swapBlue, int uIdx) { if (skipSmallImages(w, h)) return CV_HAL_ERROR_NOT_IMPLEMENTED; @@ -933,8 +938,7 @@ int ovx_hal_cvtTwoPlaneYUVtoBGR(const uchar * a, size_t astep, uchar * b, size_t if (w & 1 || h & 1) // It's not described in spec but sample implementation unable to convert odd sized images return CV_HAL_ERROR_NOT_IMPLEMENTED; - refineStep(w, h, uIdx ? VX_DF_IMAGE_NV21 : VX_DF_IMAGE_NV12, astep); - refineStep(w, h, bcn == 3 ? VX_DF_IMAGE_RGB : VX_DF_IMAGE_RGBX, bstep); + try { ivx::Context ctx = getOpenVXHALContext(); @@ -943,8 +947,8 @@ int ovx_hal_cvtTwoPlaneYUVtoBGR(const uchar * a, size_t astep, uchar * b, size_t std::vector ptrs; addr.push_back(ivx::Image::createAddressing(w, h, 1, (vx_int32)astep)); ptrs.push_back((void*)a); - addr.push_back(ivx::Image::createAddressing(w / 2, h / 2, 2, (vx_int32)astep)); - ptrs.push_back((void*)(a + h * astep)); + addr.push_back(ivx::Image::createAddressing(w / 2, h / 2, 2, (vx_int32)bstep)); + ptrs.push_back((void*)b); vxImage ia = ivx::Image::createFromHandle(ctx, uIdx ? VX_DF_IMAGE_NV21 : VX_DF_IMAGE_NV12, addr, ptrs); @@ -952,7 +956,7 @@ int ovx_hal_cvtTwoPlaneYUVtoBGR(const uchar * a, size_t astep, uchar * b, size_t return CV_HAL_ERROR_NOT_IMPLEMENTED; // OpenCV store NV12/NV21 as RANGE_RESTRICTED while OpenVX expect RANGE_FULL vxImage ib = ivx::Image::createFromHandle(ctx, bcn == 3 ? VX_DF_IMAGE_RGB : VX_DF_IMAGE_RGBX, - ivx::Image::createAddressing(w, h, bcn, (vx_int32)bstep), b); + ivx::Image::createAddressing(w, h, bcn, (vx_int32)cstep), c); ivx::IVX_CHECK_STATUS(vxuColorConvert(ctx, ia, ib)); } catch (ivx::RuntimeError & e) diff --git a/3rdparty/openvx/hal/openvx_hal.hpp b/3rdparty/openvx/hal/openvx_hal.hpp index c94cde3158..8b5a22f7de 100644 --- a/3rdparty/openvx/hal/openvx_hal.hpp +++ b/3rdparty/openvx/hal/openvx_hal.hpp @@ -49,6 +49,7 @@ int ovx_hal_morph(cvhalFilter2D *filter_context, uchar *a, size_t astep, uchar * int ovx_hal_cvtBGRtoBGR(const uchar * a, size_t astep, uchar * b, size_t bstep, int w, int h, int depth, int acn, int bcn, bool swapBlue); int ovx_hal_cvtGraytoBGR(const uchar * a, size_t astep, uchar * b, size_t bstep, int w, int h, int depth, int bcn); int ovx_hal_cvtTwoPlaneYUVtoBGR(const uchar * a, size_t astep, uchar * b, size_t bstep, int w, int h, int bcn, bool swapBlue, int uIdx); +int ovx_hal_cvtTwoPlaneYUVtoBGREx(const uchar * a, size_t astep, const uchar * b, size_t bstep, uchar * c, size_t cstep, int w, int h, int bcn, bool swapBlue, int uIdx); int ovx_hal_cvtThreePlaneYUVtoBGR(const uchar * a, size_t astep, uchar * b, size_t bstep, int w, int h, int bcn, bool swapBlue, int uIdx); int ovx_hal_cvtBGRtoThreePlaneYUV(const uchar * a, size_t astep, uchar * b, size_t bstep, int w, int h, int acn, bool swapBlue, int uIdx); int ovx_hal_cvtOnePlaneYUVtoBGR(const uchar * a, size_t astep, uchar * b, size_t bstep, int w, int h, int bcn, bool swapBlue, int uIdx, int ycn); @@ -130,6 +131,8 @@ int ovx_hal_integral(int depth, int sdepth, int, const uchar * a, size_t astep, #define cv_hal_cvtGraytoBGR ovx_hal_cvtGraytoBGR #undef cv_hal_cvtTwoPlaneYUVtoBGR #define cv_hal_cvtTwoPlaneYUVtoBGR ovx_hal_cvtTwoPlaneYUVtoBGR +#undef cv_hal_cvtTwoPlaneYUVtoBGREx +#define cv_hal_cvtTwoPlaneYUVtoBGREx ovx_hal_cvtTwoPlaneYUVtoBGREx #undef cv_hal_cvtThreePlaneYUVtoBGR #define cv_hal_cvtThreePlaneYUVtoBGR ovx_hal_cvtThreePlaneYUVtoBGR #undef cv_hal_cvtBGRtoThreePlaneYUV diff --git a/modules/imgproc/include/opencv2/imgproc/hal/hal.hpp b/modules/imgproc/include/opencv2/imgproc/hal/hal.hpp index a435fd6b85..9e5728d0c6 100644 --- a/modules/imgproc/include/opencv2/imgproc/hal/hal.hpp +++ b/modules/imgproc/include/opencv2/imgproc/hal/hal.hpp @@ -198,6 +198,11 @@ CV_EXPORTS void cvtTwoPlaneYUVtoBGR(const uchar * y_data, const uchar * uv_data, int dst_width, int dst_height, int dcn, bool swapBlue, int uIdx); +CV_EXPORTS void cvtTwoPlaneYUVtoBGR(const uchar * y_data, size_t y_step, const uchar * uv_data, size_t uv_step, + uchar * dst_data, size_t dst_step, + int dst_width, int dst_height, + int dcn, bool swapBlue, int uIdx); + CV_EXPORTS void cvtThreePlaneYUVtoBGR(const uchar * src_data, size_t src_step, uchar * dst_data, size_t dst_step, int dst_width, int dst_height, diff --git a/modules/imgproc/src/color_yuv.dispatch.cpp b/modules/imgproc/src/color_yuv.dispatch.cpp index 18b2096680..cac4fa1b41 100644 --- a/modules/imgproc/src/color_yuv.dispatch.cpp +++ b/modules/imgproc/src/color_yuv.dispatch.cpp @@ -124,8 +124,9 @@ void cvtTwoPlaneYUVtoBGR(const uchar * src_data, size_t src_step, CALL_HAL(cvtTwoPlaneYUVtoBGR, cv_hal_cvtTwoPlaneYUVtoBGR, src_data, src_step, dst_data, dst_step, dst_width, dst_height, dcn, swapBlue, uIdx); - CV_CPU_DISPATCH(cvtTwoPlaneYUVtoBGR, (src_data, src_step, dst_data, dst_step, dst_width, dst_height, dcn, swapBlue, uIdx), - CV_CPU_DISPATCH_MODES_ALL); + cvtTwoPlaneYUVtoBGR( + src_data, src_step, src_data + src_step * dst_height, src_step, dst_data, dst_step, + dst_width, dst_height, dcn, swapBlue, uIdx); } void cvtTwoPlaneYUVtoBGR(const uchar * y_data, const uchar * uv_data, size_t src_step, @@ -135,7 +136,20 @@ void cvtTwoPlaneYUVtoBGR(const uchar * y_data, const uchar * uv_data, size_t src { CV_INSTRUMENT_REGION(); - CV_CPU_DISPATCH(cvtTwoPlaneYUVtoBGR, (y_data, uv_data, src_step, dst_data, dst_step, dst_width, dst_height, dcn, swapBlue, uIdx), + cvtTwoPlaneYUVtoBGR(y_data, src_step, uv_data, src_step, dst_data, dst_step, dst_width, dst_height, dcn, swapBlue, uIdx); +} + +void cvtTwoPlaneYUVtoBGR(const uchar * y_data, size_t y_step, const uchar * uv_data, size_t uv_step, + uchar * dst_data, size_t dst_step, + int dst_width, int dst_height, + int dcn, bool swapBlue, int uIdx) +{ + CV_INSTRUMENT_REGION(); + + CALL_HAL(cvtTwoPlaneYUVtoBGREx, cv_hal_cvtTwoPlaneYUVtoBGREx, + y_data, y_step, uv_data, uv_step, dst_data, dst_step, dst_width, dst_height, dcn, swapBlue, uIdx); + + CV_CPU_DISPATCH(cvtTwoPlaneYUVtoBGR, (y_data, y_step, uv_data, uv_step, dst_data, dst_step, dst_width, dst_height, dcn, swapBlue, uIdx), CV_CPU_DISPATCH_MODES_ALL); } @@ -172,7 +186,8 @@ void cvtBGRtoTwoPlaneYUV(const uchar * src_data, size_t src_step, { CV_INSTRUMENT_REGION(); - // TODO: add hal replacement method + CALL_HAL(cvtBGRtoTwoPlaneYUV, cv_hal_cvtBGRtoTwoPlaneYUV, + src_data, src_step, y_data, dst_step, uv_data, dst_step, width, height, scn, swapBlue, uIdx); CV_CPU_DISPATCH(cvtBGRtoTwoPlaneYUV, (src_data, src_step, y_data, uv_data, dst_step, width, height, scn, swapBlue, uIdx), CV_CPU_DISPATCH_MODES_ALL); @@ -406,14 +421,21 @@ void cvtColorTwoPlaneYUV2BGRpair( InputArray _ysrc, InputArray _uvsrc, OutputArr Mat ysrc = _ysrc.getMat(), uvsrc = _uvsrc.getMat(); - CV_CheckEQ(ysrc.step, uvsrc.step, ""); - _dst.create( ysz, CV_MAKETYPE(depth, dcn)); Mat dst = _dst.getMat(); - hal::cvtTwoPlaneYUVtoBGR(ysrc.data, uvsrc.data, ysrc.step, - dst.data, dst.step, dst.cols, dst.rows, - dcn, swapb, uidx); + if(ysrc.step == uvsrc.step) + { + hal::cvtTwoPlaneYUVtoBGR(ysrc.data, uvsrc.data, ysrc.step, + dst.data, dst.step, dst.cols, dst.rows, + dcn, swapb, uidx); + } + else + { + hal::cvtTwoPlaneYUVtoBGR(ysrc.data, ysrc.step, uvsrc.data, uvsrc.step, + dst.data, dst.step, dst.cols, dst.rows, + dcn, swapb, uidx); + } } } // namespace cv diff --git a/modules/imgproc/src/color_yuv.simd.hpp b/modules/imgproc/src/color_yuv.simd.hpp index 076d1a4bd5..196a03b995 100644 --- a/modules/imgproc/src/color_yuv.simd.hpp +++ b/modules/imgproc/src/color_yuv.simd.hpp @@ -17,11 +17,7 @@ void cvtYUVtoBGR(const uchar * src_data, size_t src_step, uchar * dst_data, size_t dst_step, int width, int height, int depth, int dcn, bool swapBlue, bool isCbCr); -void cvtTwoPlaneYUVtoBGR(const uchar * src_data, size_t src_step, - uchar * dst_data, size_t dst_step, - int dst_width, int dst_height, - int dcn, bool swapBlue, int uIdx); -void cvtTwoPlaneYUVtoBGR(const uchar * y_data, const uchar * uv_data, size_t src_step, +void cvtTwoPlaneYUVtoBGR(const uchar * y_data, size_t y_step, const uchar * uv_data, size_t uv_step, uchar * dst_data, size_t dst_step, int dst_width, int dst_height, int dcn, bool swapBlue, int uIdx); @@ -1177,24 +1173,28 @@ struct YUV420sp2RGB8Invoker : ParallelLoopBody uchar * dst_data; size_t dst_step; int width; - const uchar* my1, *muv; - size_t stride; + const uchar* my1; + size_t my1_step; + const uchar* muv; + size_t muv_step; - YUV420sp2RGB8Invoker(uchar * _dst_data, size_t _dst_step, int _dst_width, size_t _stride, const uchar* _y1, const uchar* _uv) - : dst_data(_dst_data), dst_step(_dst_step), width(_dst_width), my1(_y1), muv(_uv), stride(_stride) {} + YUV420sp2RGB8Invoker(uchar * _dst_data, size_t _dst_step, int _dst_width, + const uchar* _y1, size_t _y1_step, const uchar* _uv, size_t _uv_step) : + dst_data(_dst_data), dst_step(_dst_step), width(_dst_width), + my1(_y1), my1_step(_y1_step), muv(_uv), muv_step(_uv_step) {} void operator()(const Range& range) const CV_OVERRIDE { const int rangeBegin = range.start * 2; const int rangeEnd = range.end * 2; - const uchar* y1 = my1 + rangeBegin * stride, *uv = muv + rangeBegin * stride / 2; + const uchar* y1 = my1 + rangeBegin * my1_step, *uv = muv + rangeBegin * muv_step / 2; - for (int j = rangeBegin; j < rangeEnd; j += 2, y1 += stride * 2, uv += stride) + for (int j = rangeBegin; j < rangeEnd; j += 2, y1 += my1_step * 2, uv += muv_step) { uchar* row1 = dst_data + dst_step * j; uchar* row2 = dst_data + dst_step * (j + 1); - const uchar* y2 = y1 + stride; + const uchar* y2 = y1 + my1_step; int i = 0; #if CV_SIMD @@ -1395,9 +1395,10 @@ struct YUV420p2RGB8Invoker : ParallelLoopBody #define MIN_SIZE_FOR_PARALLEL_YUV420_CONVERSION (320*240) template -inline void cvtYUV420sp2RGB(uchar * dst_data, size_t dst_step, int dst_width, int dst_height, size_t _stride, const uchar* _y1, const uchar* _uv) +inline void cvtYUV420sp2RGB(uchar * dst_data, size_t dst_step, int dst_width, int dst_height, + const uchar* _y1, size_t _y1_step, const uchar* _uv, size_t _uv_step) { - YUV420sp2RGB8Invoker converter(dst_data, dst_step, dst_width, _stride, _y1, _uv); + YUV420sp2RGB8Invoker converter(dst_data, dst_step, dst_width, _y1, _y1_step, _uv, _uv_step); if (dst_width * dst_height >= MIN_SIZE_FOR_PARALLEL_YUV420_CONVERSION) parallel_for_(Range(0, dst_height/2), converter); else @@ -1817,26 +1818,16 @@ void cvtYUVtoBGR(const uchar * src_data, size_t src_step, CvtColorLoop(src_data, src_step, dst_data, dst_step, width, height, YCrCb2RGB_f(dcn, blueIdx, isCbCr)); } -void cvtTwoPlaneYUVtoBGR(const uchar * src_data, size_t src_step, - uchar * dst_data, size_t dst_step, - int dst_width, int dst_height, - int dcn, bool swapBlue, int uIdx) -{ - CV_INSTRUMENT_REGION(); - - const uchar* uv = src_data + src_step * static_cast(dst_height); - cvtTwoPlaneYUVtoBGR(src_data, uv, src_step, dst_data, dst_step, dst_width, dst_height, dcn, swapBlue, uIdx); -} - typedef void (*cvt_2plane_yuv_ptr_t)(uchar * /* dst_data*/, size_t /* dst_step */, int /* dst_width */, int /* dst_height */, - size_t /* _stride */, const uchar* /* _y1 */, - const uchar* /* _uv */); + size_t /* _y1_step */, + const uchar* /* _uv */, + size_t /* _uv_step */); -void cvtTwoPlaneYUVtoBGR(const uchar * y_data, const uchar * uv_data, size_t src_step, +void cvtTwoPlaneYUVtoBGR(const uchar * y_data, size_t y_step, const uchar * uv_data, size_t uv_step, uchar * dst_data, size_t dst_step, int dst_width, int dst_height, int dcn, bool swapBlue, int uIdx) @@ -1859,7 +1850,7 @@ void cvtTwoPlaneYUVtoBGR(const uchar * y_data, const uchar * uv_data, size_t src default: CV_Error( CV_StsBadFlag, "Unknown/unsupported color conversion code" ); break; }; - cvtPtr(dst_data, dst_step, dst_width, dst_height, src_step, y_data, uv_data); + cvtPtr(dst_data, dst_step, dst_width, dst_height, y_data, y_step, uv_data, uv_step); } typedef void (*cvt_3plane_yuv_ptr_t)(uchar * /* dst_data */, diff --git a/modules/imgproc/src/hal_replacement.hpp b/modules/imgproc/src/hal_replacement.hpp index d6b1d78273..3368093c56 100644 --- a/modules/imgproc/src/hal_replacement.hpp +++ b/modules/imgproc/src/hal_replacement.hpp @@ -498,6 +498,39 @@ inline int hal_ni_cvtLabtoBGR(const uchar * src_data, size_t src_step, uchar * d */ inline int hal_ni_cvtTwoPlaneYUVtoBGR(const uchar * src_data, size_t src_step, uchar * dst_data, size_t dst_step, int dst_width, int dst_height, int dcn, bool swapBlue, int uIdx) { return CV_HAL_ERROR_NOT_IMPLEMENTED; } +/** + @brief Extended version of hal_cvtTwoPlaneYUVtoBGR. + @param y_data,y_step source image data and step (Y-plane) + @param uv_data,uv_step source image data and step (UV-plane) + @param dst_data,dst_step destination image data and step + @param dst_width,dst_height destination image size + @param dcn destination image channels (3 or 4) + @param swapBlue if set to true B and R destination channels will be swapped (write RGB) + @param uIdx U-channel index in the interleaved U/V plane (0 or 1) + Convert from YUV (YUV420sp (or NV12/NV21) - Y plane followed by interleaved U/V plane) to BGR, RGB, BGRA or RGBA. + Only for CV_8U. + */ +inline int hal_ni_cvtTwoPlaneYUVtoBGREx(const uchar * y_data, size_t y_step, const uchar * uv_data, size_t uv_step, + uchar * dst_data, size_t dst_step, int dst_width, int dst_height, + int dcn, bool swapBlue, int uIdx) { return CV_HAL_ERROR_NOT_IMPLEMENTED; } + +/** + @brief hal_cvtBGRtoTwoPlaneYUV + @param src_data,src_step source image data and step + @param y_data,y_step destination image data and step (Y-plane) + @param uv_data,uv_step destination image data and step (UV-plane) + @param width,height image size + @param scn source image channels (3 or 4) + @param swapBlue if set to true B and R source channels will be swapped (treat as RGB) + @param uIdx U-channel plane index (0 or 1) + Convert from BGR, RGB, BGRA or RGBA to YUV (YUV420sp (or NV12/NV21) - Y plane followed by interleaved U/V plane). + Only for CV_8U. + */ +inline int hal_ni_cvtBGRtoTwoPlaneYUV(const uchar * src_data, size_t src_step, + uchar * y_data, size_t y_step, uchar * uv_data, size_t uv_step, + int width, int height, + int scn, bool swapBlue, int uIdx) { return CV_HAL_ERROR_NOT_IMPLEMENTED; } + /** @brief hal_cvtThreePlaneYUVtoBGR @param src_data,src_step source image data and step @@ -576,6 +609,8 @@ inline int hal_ni_cvtMultipliedRGBAtoRGBA(const uchar * src_data, size_t src_ste #define cv_hal_cvtBGRtoLab hal_ni_cvtBGRtoLab #define cv_hal_cvtLabtoBGR hal_ni_cvtLabtoBGR #define cv_hal_cvtTwoPlaneYUVtoBGR hal_ni_cvtTwoPlaneYUVtoBGR +#define cv_hal_cvtTwoPlaneYUVtoBGREx hal_ni_cvtTwoPlaneYUVtoBGREx +#define cv_hal_cvtBGRtoTwoPlaneYUV hal_ni_cvtBGRtoTwoPlaneYUV #define cv_hal_cvtThreePlaneYUVtoBGR hal_ni_cvtThreePlaneYUVtoBGR #define cv_hal_cvtBGRtoThreePlaneYUV hal_ni_cvtBGRtoThreePlaneYUV #define cv_hal_cvtOnePlaneYUVtoBGR hal_ni_cvtOnePlaneYUVtoBGR diff --git a/modules/imgproc/test/test_color.cpp b/modules/imgproc/test/test_color.cpp index 450cf57923..204203d053 100644 --- a/modules/imgproc/test/test_color.cpp +++ b/modules/imgproc/test/test_color.cpp @@ -3072,20 +3072,34 @@ TEST(ImgProc_RGB2YUV, regression_13668) EXPECT_EQ(res, ref); } -TEST(ImgProc_cvtColorTwoPlane, missing_check_17036) // test can be removed if required feature is implemented +TEST(ImgProc_cvtColorTwoPlane, y_plane_padding_differs_from_uv_plane_padding_17036) { - std::vector y_data(700 * 480); - std::vector uv_data(640 * 240); + RNG &rng = theRNG(); - Mat y_plane_padding(480, 640, CV_8UC1, y_data.data(), 700); // with stride - Mat uv_plane(240, 320, CV_8UC2, uv_data.data()); + std::vector y_reference(640 * 480); + std::vector uv_reference(640 * 240); + std::vector y_padded(700 * 480); + std::vector uv_padded(700 * 240); - Mat result; + Mat y_reference_mat(480, 640, CV_8UC1, y_reference.data()); + Mat uv_reference_mat(240, 320, CV_8UC2, uv_reference.data()); + Mat y_padded_mat(480, 640, CV_8UC1, y_padded.data(), 700); + Mat uv_padded_mat(240, 320, CV_8UC2, uv_padded.data(), 700); - EXPECT_THROW( - cvtColorTwoPlane(y_plane_padding, uv_plane, result, COLOR_YUV2RGB_NV21); - , cv::Exception - ); + rng.fill(y_reference_mat, RNG::UNIFORM, 16, 235 + 1); + rng.fill(uv_reference_mat, RNG::UNIFORM, 16, 240 + 1); + + y_reference_mat.copyTo(y_padded_mat(Rect(0, 0, y_reference_mat.cols, y_reference_mat.rows))); + uv_reference_mat.copyTo(uv_padded_mat(Rect(0, 0, uv_reference_mat.cols, uv_reference_mat.rows))); + + Mat rgb_reference_mat, rgb_y_padded_mat, rgb_uv_padded_mat; + + cvtColorTwoPlane(y_reference_mat, uv_reference_mat, rgb_reference_mat, COLOR_YUV2RGB_NV21); + cvtColorTwoPlane(y_padded_mat, uv_reference_mat, rgb_y_padded_mat, COLOR_YUV2RGB_NV21); + cvtColorTwoPlane(y_reference_mat, uv_padded_mat, rgb_uv_padded_mat, COLOR_YUV2RGB_NV21); + + EXPECT_DOUBLE_EQ(cvtest::norm(rgb_reference_mat, rgb_y_padded_mat, NORM_INF), .0); + EXPECT_DOUBLE_EQ(cvtest::norm(rgb_reference_mat, rgb_uv_padded_mat, NORM_INF), .0); } From 54386c82fd5ca3079f60d1758bb1d25651a329c9 Mon Sep 17 00:00:00 2001 From: Sergey Ivanov Date: Thu, 23 Sep 2021 14:34:30 +0300 Subject: [PATCH 186/262] Merge pull request #20727 from sivanov-work:merge_vpl_accel_impl G-API: oneVPL (simplification) added CPU, DX11(fallback CPU) accels & surface pool * Add CPU, DX11(fallback CPU) accels & surface pool * Fix build for surface_pool * Apply some comments * Fix indentation --- modules/gapi/CMakeLists.txt | 6 + .../onevpl/accelerators/accel_policy_cpu.cpp | 225 ++++++++++++++++++ .../onevpl/accelerators/accel_policy_cpu.hpp | 55 +++++ .../onevpl/accelerators/accel_policy_dx11.cpp | 114 +++++++++ .../onevpl/accelerators/accel_policy_dx11.hpp | 67 ++++++ .../accelerators/surface/surface_pool.cpp | 69 ++++++ .../accelerators/surface/surface_pool.hpp | 45 ++++ .../gapi_streaming_vpl_core_test.cpp | 161 ++++++++++++- 8 files changed, 741 insertions(+), 1 deletion(-) create mode 100644 modules/gapi/src/streaming/onevpl/accelerators/accel_policy_cpu.cpp create mode 100644 modules/gapi/src/streaming/onevpl/accelerators/accel_policy_cpu.hpp create mode 100644 modules/gapi/src/streaming/onevpl/accelerators/accel_policy_dx11.cpp create mode 100644 modules/gapi/src/streaming/onevpl/accelerators/accel_policy_dx11.hpp create mode 100644 modules/gapi/src/streaming/onevpl/accelerators/surface/surface_pool.cpp create mode 100644 modules/gapi/src/streaming/onevpl/accelerators/surface/surface_pool.hpp diff --git a/modules/gapi/CMakeLists.txt b/modules/gapi/CMakeLists.txt index 56d051cf0f..380f1d7d84 100644 --- a/modules/gapi/CMakeLists.txt +++ b/modules/gapi/CMakeLists.txt @@ -171,6 +171,9 @@ set(gapi_srcs src/streaming/onevpl/data_provider_interface_exception.cpp src/streaming/onevpl/accelerators/surface/cpu_frame_adapter.cpp src/streaming/onevpl/accelerators/surface/surface.cpp + src/streaming/onevpl/accelerators/surface/surface_pool.cpp + src/streaming/onevpl/accelerators/accel_policy_cpu.cpp + src/streaming/onevpl/accelerators/accel_policy_dx11.cpp # Utils (ITT tracing) src/utils/itt.cpp @@ -247,6 +250,9 @@ if(HAVE_GAPI_ONEVPL) if(TARGET opencv_test_gapi) ocv_target_compile_definitions(opencv_test_gapi PRIVATE -DHAVE_ONEVPL) ocv_target_link_libraries(opencv_test_gapi PRIVATE ${VPL_IMPORTED_TARGETS}) + if(HAVE_D3D11 AND HAVE_OPENCL) + ocv_target_include_directories(opencv_test_gapi SYSTEM PRIVATE ${OPENCL_INCLUDE_DIRS}) + endif() endif() ocv_target_compile_definitions(${the_module} PRIVATE -DHAVE_ONEVPL) ocv_target_link_libraries(${the_module} PRIVATE ${VPL_IMPORTED_TARGETS}) diff --git a/modules/gapi/src/streaming/onevpl/accelerators/accel_policy_cpu.cpp b/modules/gapi/src/streaming/onevpl/accelerators/accel_policy_cpu.cpp new file mode 100644 index 0000000000..0a5f68ae4e --- /dev/null +++ b/modules/gapi/src/streaming/onevpl/accelerators/accel_policy_cpu.cpp @@ -0,0 +1,225 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2021 Intel Corporation + +#ifdef HAVE_ONEVPL +#include +#include + +#include "streaming/onevpl/accelerators/accel_policy_cpu.hpp" +#include "streaming/onevpl/accelerators/surface/cpu_frame_adapter.hpp" +#include "streaming/onevpl/accelerators/surface/surface.hpp" +#include "logger.hpp" + +#ifdef _WIN32 + #include + #include +#endif +namespace cv { +namespace gapi { +namespace wip { + +VPLCPUAccelerationPolicy::VPLCPUAccelerationPolicy() { + GAPI_LOG_INFO(nullptr, "created"); +} + +VPLCPUAccelerationPolicy::~VPLCPUAccelerationPolicy() { + for (auto& pair : pool_table) { + pair.second.clear(); + // do not free key here: last surface will release it + } + GAPI_LOG_INFO(nullptr, "destroyed"); +} + +void VPLCPUAccelerationPolicy::init(session_t session) { + GAPI_LOG_INFO(nullptr, "initialize session: " << session); +} + +void VPLCPUAccelerationPolicy::deinit(session_t session) { + GAPI_LOG_INFO(nullptr, "deinitialize session: " << session); +} + +VPLCPUAccelerationPolicy::pool_key_t +VPLCPUAccelerationPolicy::create_surface_pool(size_t pool_size, size_t surface_size_bytes, + surface_ptr_ctr_t creator) { + GAPI_LOG_DEBUG(nullptr, "pool size: " << pool_size << ", surface size bytes: " << surface_size_bytes); + + // create empty pool + pool_t pool; + pool.reserve(pool_size); + + // allocate workplace memory area + size_t preallocated_raw_bytes = pool_size * surface_size_bytes; + size_t page_size_bytes = 4 * 1024; + void *preallocated_pool_memory_ptr = nullptr; + +#ifdef _WIN32 + SYSTEM_INFO sysInfo; + GetSystemInfo(&sysInfo); + page_size_bytes = sysInfo.dwPageSize; + + GAPI_LOG_DEBUG(nullptr, "page size: " << page_size_bytes << ", preallocated_raw_bytes: " << preallocated_raw_bytes); + preallocated_pool_memory_ptr = _aligned_malloc(preallocated_raw_bytes, page_size_bytes); +#else + GAPI_Assert(false && "Compatibility is not tested for systems differ than \"_WIN32\". " + "Please feel free to set it up under OPENCV contribution policy"); +#endif + + if (!preallocated_pool_memory_ptr) { + throw std::runtime_error("VPLCPUAccelerationPolicy::create_surface_pool - failed: not enough memory." + "Requested surface count: " + std::to_string(pool_size) + + ", surface bytes: " + std::to_string(surface_size_bytes)); + } + + // fill pool with surfaces + std::shared_ptr workspace_mem_owner (preallocated_pool_memory_ptr, [] (void *ptr){ + GAPI_LOG_INFO(nullptr, "Free workspace memory: " << ptr); +#ifdef _WIN32 + _aligned_free(ptr); + GAPI_LOG_INFO(nullptr, "Released workspace memory: " << ptr); + ptr = nullptr; +#else + GAPI_Assert(false && "Not implemented for systems differ than \"_WIN32\". " + "Please feel free to set it up under OPENCV contribution policy"); +#endif + + }); + size_t i = 0; + try { + for (; i < pool_size; i++) { + size_t preallocated_mem_offset = static_cast(i) * surface_size_bytes; + + surface_ptr_t surf_ptr = creator(workspace_mem_owner, + preallocated_mem_offset, + preallocated_raw_bytes); + pool.push_back(std::move(surf_ptr)); + } + } catch (const std::exception& ex) { + throw std::runtime_error(std::string("VPLCPUAccelerationPolicy::create_surface_pool - ") + + "cannot construct surface index: " + std::to_string(i) + ", error: " + + ex.what() + + "Requested surface count: " + std::to_string(pool_size) + + ", surface bytes: " + std::to_string(surface_size_bytes)); + } + + // remember pool by key + GAPI_LOG_INFO(nullptr, "New pool allocated, key: " << preallocated_pool_memory_ptr << + ", surface count: " << pool.size() << + ", surface size bytes: " << surface_size_bytes); + if (!pool_table.emplace(preallocated_pool_memory_ptr, std::move(pool)).second) { + GAPI_LOG_WARNING(nullptr, "Cannot insert pool, table size: " + std::to_string(pool_table.size()) << + ", key: " << preallocated_pool_memory_ptr << " exists"); + GAPI_Assert(false && "Cannot create pool in VPLCPUAccelerationPolicy"); + } + + return preallocated_pool_memory_ptr; +} + +VPLCPUAccelerationPolicy::surface_weak_ptr_t VPLCPUAccelerationPolicy::get_free_surface(pool_key_t key) { + auto pool_it = pool_table.find(key); + if (pool_it == pool_table.end()) { + throw std::runtime_error("VPLCPUAccelerationPolicy::get_free_surface - " + "key is not found, table size: " + + std::to_string(pool_table.size())); + } + + pool_t& requested_pool = pool_it->second; +#ifdef TEST_PERF + return requested_pool.find_free(); +#else // TEST_PERF + auto it = + std::find_if(requested_pool.begin(), requested_pool.end(), + [](const surface_ptr_t& val) { + GAPI_DbgAssert(val && "Pool contains empty surface"); + return !val->get_locks_count(); + }); + + // Limitation realloc pool might be a future extension + if (it == requested_pool.end()) { + std::stringstream ss; + ss << "cannot get free surface from pool, key: " << key << ", size: " << requested_pool.size(); + const std::string& str = ss.str(); + GAPI_LOG_WARNING(nullptr, str); + throw std::runtime_error(std::string(__FUNCTION__) + " - " + str); + } + + return *it; +#endif // TEST_PERF +} + +size_t VPLCPUAccelerationPolicy::get_free_surface_count(pool_key_t key) const { + auto pool_it = pool_table.find(key); + if (pool_it == pool_table.end()) { + GAPI_LOG_WARNING(nullptr, "key is not found: " << key << + ", table size: " << pool_table.size()); + return 0; + } +#ifdef TEST_PERF + return 0; +#else // TEST_PERF + const pool_t& requested_pool = pool_it->second; + size_t free_surf_count = + std::count_if(requested_pool.begin(), requested_pool.end(), + [](const surface_ptr_t& val) { + GAPI_Assert(val && "Pool contains empty surface"); + return !val->get_locks_count(); + }); + return free_surf_count; +#endif // TEST_PERF +} + +size_t VPLCPUAccelerationPolicy::get_surface_count(pool_key_t key) const { + auto pool_it = pool_table.find(key); + if (pool_it == pool_table.end()) { + GAPI_LOG_DEBUG(nullptr, "key is not found: " << key << + ", table size: " << pool_table.size()); + return 0; + } +#ifdef TEST_PERF + return 0; +#else // TEST_PERF + return pool_it->second.size(); +#endif // TEST_PERF +} + +cv::MediaFrame::AdapterPtr VPLCPUAccelerationPolicy::create_frame_adapter(pool_key_t key, + mfxFrameSurface1* surface) { + auto pool_it = pool_table.find(key); + if (pool_it == pool_table.end()) { + std::stringstream ss; + ss << "key is not found: " << key << ", table size: " << pool_table.size(); + const std::string& str = ss.str(); + GAPI_LOG_WARNING(nullptr, str); + throw std::runtime_error(std::string(__FUNCTION__) + " - " + str); + } + + pool_t& requested_pool = pool_it->second; +#ifdef TEST_PERF + return cv::MediaFrame::AdapterPtr{new VPLMediaFrameCPUAdapter(requested_pool.find_by_handle(surface))}; +#else // TEST_PERF + auto it = + std::find_if(requested_pool.begin(), requested_pool.end(), + [surface](const surface_ptr_t& val) { + GAPI_DbgAssert(val && "Pool contains empty surface"); + return val->get_handle() == surface; + }); + + // Limitation realloc pool might be a future extension + if (it == requested_pool.end()) { + std::stringstream ss; + ss << "cannot get requested surface from pool, key: " << key << ", surf: " + << surface << ", pool size: " << requested_pool.size(); + const std::string& str = ss.str(); + GAPI_LOG_WARNING(nullptr, str); + throw std::runtime_error(std::string(__FUNCTION__) + " - " + str); + } + + return cv::MediaFrame::AdapterPtr{new VPLMediaFrameCPUAdapter(*it)}; +#endif // TEST_PERF +} +} // namespace wip +} // namespace gapi +} // namespace cv +#endif // HAVE_ONEVPL diff --git a/modules/gapi/src/streaming/onevpl/accelerators/accel_policy_cpu.hpp b/modules/gapi/src/streaming/onevpl/accelerators/accel_policy_cpu.hpp new file mode 100644 index 0000000000..cfe3057315 --- /dev/null +++ b/modules/gapi/src/streaming/onevpl/accelerators/accel_policy_cpu.hpp @@ -0,0 +1,55 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2021 Intel Corporation + +#ifndef GAPI_STREAMING_ONEVPL_ACCELERATORS_ACCEL_POLICY_CPU_HPP +#define GAPI_STREAMING_ONEVPL_ACCELERATORS_ACCEL_POLICY_CPU_HPP + +#include +#include + +#include "opencv2/gapi/own/exports.hpp" // GAPI_EXPORTS + +#ifdef HAVE_ONEVPL +#include +#include "streaming/onevpl/accelerators/accel_policy_interface.hpp" +#ifdef TEST_PERF +#include "streaming/onevpl/accelerators/surface/surface_pool.hpp" +#endif // TEST_PERF + +namespace cv { +namespace gapi { +namespace wip { + +// GAPI_EXPORTS for tests +struct GAPI_EXPORTS VPLCPUAccelerationPolicy final : public VPLAccelerationPolicy +{ + VPLCPUAccelerationPolicy(); + ~VPLCPUAccelerationPolicy(); +#ifdef TEST_PERF + using pool_t = CachedPool; +#else // TEST_PERF + using pool_t = std::vector; +#endif // TEST_PERF + + void init(session_t session) override; + void deinit(session_t session) override; + pool_key_t create_surface_pool(size_t pool_size, size_t surface_size_bytes, surface_ptr_ctr_t creator) override; + surface_weak_ptr_t get_free_surface(pool_key_t key) override; + size_t get_free_surface_count(pool_key_t key) const override; + size_t get_surface_count(pool_key_t key) const override; + + cv::MediaFrame::AdapterPtr create_frame_adapter(pool_key_t key, + mfxFrameSurface1* surface) override; + +private: + std::map pool_table; +}; +} // namespace wip +} // namespace gapi +} // namespace cv + +#endif // HAVE_ONEVPL +#endif // GAPI_STREAMING_ONEVPL_ACCELERATORS_ACCEL_POLICY_CPU_HPP diff --git a/modules/gapi/src/streaming/onevpl/accelerators/accel_policy_dx11.cpp b/modules/gapi/src/streaming/onevpl/accelerators/accel_policy_dx11.cpp new file mode 100644 index 0000000000..348b864a15 --- /dev/null +++ b/modules/gapi/src/streaming/onevpl/accelerators/accel_policy_dx11.cpp @@ -0,0 +1,114 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2021 Intel Corporation + +#ifdef HAVE_ONEVPL +#include "streaming/onevpl/accelerators/accel_policy_dx11.hpp" +#include "streaming/onevpl/accelerators/surface/cpu_frame_adapter.hpp" +#include "streaming/onevpl/accelerators/surface/surface.hpp" +#include "logger.hpp" + +#ifdef HAVE_DIRECTX +#ifdef HAVE_D3D11 +#pragma comment(lib,"d3d11.lib") + +#define D3D11_NO_HELPERS +#include +#include +#include +#include "opencv2/core/directx.hpp" + +#ifdef HAVE_OPENCL +#include +#endif + +namespace cv { +namespace gapi { +namespace wip { +VPLDX11AccelerationPolicy::VPLDX11AccelerationPolicy() +{ +#ifdef CPU_ACCEL_ADAPTER + adapter.reset(new VPLCPUAccelerationPolicy); +#endif +} + +VPLDX11AccelerationPolicy::~VPLDX11AccelerationPolicy() +{ + if (hw_handle) + { + GAPI_LOG_INFO(nullptr, "VPLDX11AccelerationPolicy release ID3D11Device"); + hw_handle->Release(); + } +} + +void VPLDX11AccelerationPolicy::init(session_t session) { + mfxStatus sts = MFXVideoCORE_GetHandle(session, MFX_HANDLE_D3D11_DEVICE, reinterpret_cast(&hw_handle)); + if (sts != MFX_ERR_NONE) + { + throw std::logic_error("Cannot create VPLDX11AccelerationPolicy, MFXVideoCORE_GetHandle error"); + } + + GAPI_LOG_INFO(nullptr, "VPLDX11AccelerationPolicy initialized, session: " << session); +} + +void VPLDX11AccelerationPolicy::deinit(session_t session) { + GAPI_LOG_INFO(nullptr, "deinitialize session: " << session); +} + +VPLDX11AccelerationPolicy::pool_key_t +VPLDX11AccelerationPolicy::create_surface_pool(size_t pool_size, size_t surface_size_bytes, + surface_ptr_ctr_t creator) { + GAPI_LOG_DEBUG(nullptr, "pool size: " << pool_size << ", surface size bytes: " << surface_size_bytes); + +#ifdef CPU_ACCEL_ADAPTER + return adapter->create_surface_pool(pool_size, surface_size_bytes, creator); +#endif + (void)pool_size; + (void)surface_size_bytes; + (void)creator; + throw std::runtime_error("VPLDX11AccelerationPolicy::create_surface_pool() is not implemented"); +} + +VPLDX11AccelerationPolicy::surface_weak_ptr_t VPLDX11AccelerationPolicy::get_free_surface(pool_key_t key) +{ +#ifdef CPU_ACCEL_ADAPTER + return adapter->get_free_surface(key); +#endif + (void)key; + throw std::runtime_error("VPLDX11AccelerationPolicy::get_free_surface() is not implemented"); +} + +size_t VPLDX11AccelerationPolicy::get_free_surface_count(pool_key_t key) const { +#ifdef CPU_ACCEL_ADAPTER + return adapter->get_free_surface_count(key); +#endif + (void)key; + throw std::runtime_error("get_free_surface_count() is not implemented"); +} + +size_t VPLDX11AccelerationPolicy::get_surface_count(pool_key_t key) const { +#ifdef CPU_ACCEL_ADAPTER + return adapter->get_surface_count(key); +#endif + (void)key; + throw std::runtime_error("VPLDX11AccelerationPolicy::get_surface_count() is not implemented"); +} + +cv::MediaFrame::AdapterPtr VPLDX11AccelerationPolicy::create_frame_adapter(pool_key_t key, + mfxFrameSurface1* surface) { + +#ifdef CPU_ACCEL_ADAPTER + return adapter->create_frame_adapter(key, surface); +#endif + (void)key; + (void)surface; + throw std::runtime_error("VPLDX11AccelerationPolicy::create_frame_adapter() is not implemented"); +} +} // namespace wip +} // namespace gapi +} // namespace cv +#endif // HAVE_D3D11 +#endif // HAVE_DIRECTX +#endif // HAVE_ONEVPL diff --git a/modules/gapi/src/streaming/onevpl/accelerators/accel_policy_dx11.hpp b/modules/gapi/src/streaming/onevpl/accelerators/accel_policy_dx11.hpp new file mode 100644 index 0000000000..04970432c5 --- /dev/null +++ b/modules/gapi/src/streaming/onevpl/accelerators/accel_policy_dx11.hpp @@ -0,0 +1,67 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2021 Intel Corporation + +#ifndef GAPI_STREAMING_ONEVPL_ACCELERATORS_ACCEL_POLICY_DX11_HPP +#define GAPI_STREAMING_ONEVPL_ACCELERATORS_ACCEL_POLICY_DX11_HPP + +#include "opencv2/gapi/own/exports.hpp" // GAPI_EXPORTS +//TODO Remove the next MACRO right after DX11 implementation +#define CPU_ACCEL_ADAPTER + +#ifdef HAVE_ONEVPL +#include +#include "streaming/onevpl/accelerators/accel_policy_interface.hpp" + +#ifdef CPU_ACCEL_ADAPTER +#include "streaming/onevpl/accelerators/accel_policy_cpu.hpp" +#endif + +#ifdef HAVE_DIRECTX +#ifdef HAVE_D3D11 +#define D3D11_NO_HELPERS +#include +#include +#include "opencv2/core/directx.hpp" +#ifdef HAVE_OPENCL +#include +#endif + +namespace cv { +namespace gapi { +namespace wip { + +// GAPI_EXPORTS for tests +struct GAPI_EXPORTS VPLDX11AccelerationPolicy final: public VPLAccelerationPolicy +{ + // GAPI_EXPORTS for tests + VPLDX11AccelerationPolicy(); + ~VPLDX11AccelerationPolicy(); + + void init(session_t session) override; + void deinit(session_t session) override; + pool_key_t create_surface_pool(size_t pool_size, size_t surface_size_bytes, surface_ptr_ctr_t creator) override; + surface_weak_ptr_t get_free_surface(pool_key_t key) override; + size_t get_free_surface_count(pool_key_t key) const override; + size_t get_surface_count(pool_key_t key) const override; + + cv::MediaFrame::AdapterPtr create_frame_adapter(pool_key_t key, + mfxFrameSurface1* surface) override; + +private: + ID3D11Device *hw_handle; + +#ifdef CPU_ACCEL_ADAPTER + std::unique_ptr adapter; +#endif +}; +} // namespace wip +} // namespace gapi +} // namespace cv +#endif // HAVE_D3D11 +#endif // HAVE_DIRECTX + +#endif // HAVE_ONEVPL +#endif // GAPI_STREAMING_ONEVPL_ACCELERATORS_ACCEL_POLICY_DX11_HPP diff --git a/modules/gapi/src/streaming/onevpl/accelerators/surface/surface_pool.cpp b/modules/gapi/src/streaming/onevpl/accelerators/surface/surface_pool.cpp new file mode 100644 index 0000000000..8ead7965b4 --- /dev/null +++ b/modules/gapi/src/streaming/onevpl/accelerators/surface/surface_pool.cpp @@ -0,0 +1,69 @@ +#include "streaming/onevpl/accelerators/surface/surface_pool.hpp" +#include "streaming/onevpl/accelerators/surface/surface.hpp" +#include "logger.hpp" + +#ifdef HAVE_ONEVPL + +namespace cv { +namespace gapi { +namespace wip { + +void CachedPool::reserve(size_t size) { + surfaces.reserve(size); +} + +size_t CachedPool::size() const { + return surfaces.size(); +} + +void CachedPool::clear() { + surfaces.clear(); + next_free_it = surfaces.begin(); + cache.clear(); +} + +void CachedPool::push_back(surface_ptr_t &&surf) { + cache.insert(std::make_pair(surf->get_handle(), surf)); + surfaces.push_back(std::move(surf)); + next_free_it = surfaces.begin(); +} + +CachedPool::surface_ptr_t CachedPool::find_free() { + auto it = + std::find_if(next_free_it, surfaces.end(), + [](const surface_ptr_t& val) { + GAPI_DbgAssert(val && "Pool contains empty surface"); + return !val->get_locks_count(); + }); + + // Limitation realloc pool might be a future extension + if (it == surfaces.end()) { + it = std::find_if(surfaces.begin(), next_free_it, + [](const surface_ptr_t& val) { + GAPI_DbgAssert(val && "Pool contains empty surface"); + return !val->get_locks_count(); + }); + if (it == next_free_it) { + std::stringstream ss; + ss << "cannot get free surface from pool, size: " << surfaces.size(); + const std::string& str = ss.str(); + GAPI_LOG_WARNING(nullptr, str); + throw std::runtime_error(std::string(__FUNCTION__) + " - " + str); + } + } + + next_free_it = it; + ++next_free_it; + + return *it; +} + +CachedPool::surface_ptr_t CachedPool::find_by_handle(mfxFrameSurface1* handle) { + auto it = cache.find(handle); + GAPI_Assert(it != cache.end() && "Cannot find cached surface from pool. Data corruption is possible"); + return it->second; +} +} // namespace wip +} // namespace gapi +} // namespace cv +#endif // HAVE_ONEVPL diff --git a/modules/gapi/src/streaming/onevpl/accelerators/surface/surface_pool.hpp b/modules/gapi/src/streaming/onevpl/accelerators/surface/surface_pool.hpp new file mode 100644 index 0000000000..2059e9b27e --- /dev/null +++ b/modules/gapi/src/streaming/onevpl/accelerators/surface/surface_pool.hpp @@ -0,0 +1,45 @@ +#ifndef GAPI_STREAMING_ONEVPL_SURFACE_SURFACE_POOL_HPP +#define GAPI_STREAMING_ONEVPL_SURFACE_SURFACE_POOL_HPP + +#include +#include +#include + +#include "opencv2/gapi/own/exports.hpp" // GAPI_EXPORTS + +#ifdef HAVE_ONEVPL +#if (MFX_VERSION >= 2000) +#include +#endif + +#include + +namespace cv { +namespace gapi { +namespace wip { + +class Surface; +// GAPI_EXPORTS for tests +class GAPI_EXPORTS CachedPool { +public: + using surface_ptr_t = std::shared_ptr; + using surface_container_t = std::vector; + using free_surface_iterator_t = typename surface_container_t::iterator; + using cached_surface_container_t = std::map; + + void push_back(surface_ptr_t &&surf); + void reserve(size_t size); + size_t size() const; + void clear(); + surface_ptr_t find_free(); + surface_ptr_t find_by_handle(mfxFrameSurface1* handle); +private: + surface_container_t surfaces; + free_surface_iterator_t next_free_it; + cached_surface_container_t cache; +}; +} // namespace wip +} // namespace gapi +} // namespace cv +#endif // HAVE_ONEVPL +#endif // GAPI_STREAMING_ONEVPL_SURFACE_SURFACE_POOL_HPP diff --git a/modules/gapi/test/streaming/gapi_streaming_vpl_core_test.cpp b/modules/gapi/test/streaming/gapi_streaming_vpl_core_test.cpp index be905029ba..5a36a2befb 100644 --- a/modules/gapi/test/streaming/gapi_streaming_vpl_core_test.cpp +++ b/modules/gapi/test/streaming/gapi_streaming_vpl_core_test.cpp @@ -9,6 +9,7 @@ #include "../common/gapi_tests_common.hpp" +#include #include #include @@ -30,11 +31,18 @@ #ifdef HAVE_ONEVPL #include "streaming/onevpl/accelerators/surface/surface.hpp" #include "streaming/onevpl/accelerators/surface/cpu_frame_adapter.hpp" +#include "streaming/onevpl/accelerators/accel_policy_cpu.hpp" namespace opencv_test { namespace { +cv::gapi::wip::surface_ptr_t create_test_surface(std::shared_ptr out_buf_ptr, + size_t, size_t) { + std::unique_ptr handle(new mfxFrameSurface1{}); + return cv::gapi::wip::Surface::create_surface(std::move(handle), out_buf_ptr); +} + TEST(OneVPL_Source_Surface, InitSurface) { using namespace cv::gapi::wip; @@ -160,7 +168,7 @@ TEST(OneVPL_Source_Surface, MemoryLifeTime) EXPECT_TRUE(preallocated_memory_ptr.get() == nullptr); } -TEST(OneVPL_Source_CPUFrameAdapter, InitFrameAdapter) +TEST(OneVPL_Source_CPU_FrameAdapter, InitFrameAdapter) { using namespace cv::gapi::wip; @@ -180,6 +188,157 @@ TEST(OneVPL_Source_CPUFrameAdapter, InitFrameAdapter) } EXPECT_EQ(surf->get_locks_count(), 0); } + +TEST(OneVPL_Source_CPU_Accelerator, InitDestroy) +{ + using cv::gapi::wip::VPLCPUAccelerationPolicy; + using cv::gapi::wip::VPLAccelerationPolicy; + + auto acceleration_policy = std::make_shared(); + + size_t surface_count = 10; + size_t surface_size_bytes = 1024; + size_t pool_count = 3; + std::vector pool_export_keys; + pool_export_keys.reserve(pool_count); + + // create several pools + for (size_t i = 0; i < pool_count; i++) + { + VPLAccelerationPolicy::pool_key_t key = + acceleration_policy->create_surface_pool(surface_count, + surface_size_bytes, + create_test_surface); + // check consistency + EXPECT_EQ(acceleration_policy->get_surface_count(key), surface_count); + EXPECT_EQ(acceleration_policy->get_free_surface_count(key), surface_count); + + pool_export_keys.push_back(key); + } + + EXPECT_NO_THROW(acceleration_policy.reset()); +} + +TEST(OneVPL_Source_CPU_Accelerator, PoolProduceConsume) +{ + using cv::gapi::wip::VPLCPUAccelerationPolicy; + using cv::gapi::wip::VPLAccelerationPolicy; + using cv::gapi::wip::Surface; + + auto acceleration_policy = std::make_shared(); + + size_t surface_count = 10; + size_t surface_size_bytes = 1024; + + VPLAccelerationPolicy::pool_key_t key = + acceleration_policy->create_surface_pool(surface_count, + surface_size_bytes, + create_test_surface); + // check consistency + EXPECT_EQ(acceleration_policy->get_surface_count(key), surface_count); + EXPECT_EQ(acceleration_policy->get_free_surface_count(key), surface_count); + + // consume available surfaces + std::vector> surfaces; + surfaces.reserve(surface_count); + for (size_t i = 0; i < surface_count; i++) { + std::shared_ptr surf = acceleration_policy->get_free_surface(key).lock(); + EXPECT_TRUE(surf.get() != nullptr); + EXPECT_EQ(surf->obtain_lock(), 0); + surfaces.push_back(std::move(surf)); + } + + // check consistency (no free surfaces) + EXPECT_EQ(acceleration_policy->get_surface_count(key), surface_count); + EXPECT_EQ(acceleration_policy->get_free_surface_count(key), 0); + + // fail consume non-free surfaces + for (size_t i = 0; i < surface_count; i++) { + EXPECT_THROW(acceleration_policy->get_free_surface(key), std::runtime_error); + } + + // release surfaces + for (auto& surf : surfaces) { + EXPECT_EQ(surf->release_lock(), 1); + } + surfaces.clear(); + + // check consistency + EXPECT_EQ(acceleration_policy->get_surface_count(key), surface_count); + EXPECT_EQ(acceleration_policy->get_free_surface_count(key), surface_count); + + //check availability after release + for (size_t i = 0; i < surface_count; i++) { + std::shared_ptr surf = acceleration_policy->get_free_surface(key).lock(); + EXPECT_TRUE(surf.get() != nullptr); + EXPECT_EQ(surf->obtain_lock(), 0); + } +} + +TEST(OneVPL_Source_CPU_Accelerator, PoolProduceConcurrentConsume) +{ + using cv::gapi::wip::VPLCPUAccelerationPolicy; + using cv::gapi::wip::VPLAccelerationPolicy; + using cv::gapi::wip::Surface; + + auto acceleration_policy = std::make_shared(); + + size_t surface_count = 10; + size_t surface_size_bytes = 1024; + + VPLAccelerationPolicy::pool_key_t key = + acceleration_policy->create_surface_pool(surface_count, + surface_size_bytes, + create_test_surface); + + // check consistency + EXPECT_EQ(acceleration_policy->get_surface_count(key), surface_count); + EXPECT_EQ(acceleration_policy->get_free_surface_count(key), surface_count); + + // consume available surfaces + std::vector> surfaces; + surfaces.reserve(surface_count); + for (size_t i = 0; i < surface_count; i++) { + std::shared_ptr surf = acceleration_policy->get_free_surface(key).lock(); + EXPECT_TRUE(surf.get() != nullptr); + EXPECT_EQ(surf->obtain_lock(), 0); + surfaces.push_back(std::move(surf)); + } + + std::promise launch_promise; + std::future sync = launch_promise.get_future(); + std::promise surface_released_promise; + std::future released_result = surface_released_promise.get_future(); + std::thread worker_thread([&launch_promise, &surface_released_promise, &surfaces] () { + launch_promise.set_value(); + + // concurrent release surfaces + size_t surfaces_count = surfaces.size(); + for (auto& surf : surfaces) { + EXPECT_EQ(surf->release_lock(), 1); + std::this_thread::sleep_for(std::chrono::seconds(1)); + } + surfaces.clear(); + + surface_released_promise.set_value(surfaces_count); + }); + sync.wait(); + + // check free surface concurrently + std::future_status status; + size_t free_surface_count = 0; + size_t free_surface_count_prev = 0; + do { + status = released_result.wait_for(std::chrono::seconds(1)); + free_surface_count = acceleration_policy->get_free_surface_count(key); + EXPECT_TRUE(free_surface_count >= free_surface_count_prev); + free_surface_count_prev = free_surface_count; + } while (status != std::future_status::ready); + std::cerr<< "Ready" << std::endl; + free_surface_count = acceleration_policy->get_free_surface_count(key); + worker_thread.join(); + EXPECT_TRUE(free_surface_count >= free_surface_count_prev); +} } } // namespace opencv_test #endif // HAVE_ONEVPL From 499d8adb75f18113c1fd5ec36cbfbe5fac9b993a Mon Sep 17 00:00:00 2001 From: Anatoliy Talamanov Date: Thu, 23 Sep 2021 22:59:40 +0300 Subject: [PATCH 187/262] Merge pull request #20705 from TolyaTalamanov:at/handle-reshape-in-gexecutor G-API: Handle reshape for generic case in GExecutor * Handle reshape for generic case for GExecutor * Add initResources * Add tests * Refactor reshape method --- modules/gapi/src/executor/gexecutor.cpp | 17 +- .../test/internal/gapi_int_executor_tests.cpp | 217 ++++++++++++++++++ 2 files changed, 230 insertions(+), 4 deletions(-) diff --git a/modules/gapi/src/executor/gexecutor.cpp b/modules/gapi/src/executor/gexecutor.cpp index 2ca675c7df..ad9d380b4e 100644 --- a/modules/gapi/src/executor/gexecutor.cpp +++ b/modules/gapi/src/executor/gexecutor.cpp @@ -7,8 +7,6 @@ #include "precomp.hpp" -#include - #include #include @@ -411,7 +409,8 @@ bool cv::gimpl::GExecutor::canReshape() const { // FIXME: Introduce proper reshaping support on GExecutor level // for all cases! - return (m_ops.size() == 1) && m_ops[0].isl_exec->canReshape(); + return std::all_of(m_ops.begin(), m_ops.end(), + [](const OpDesc& op) { return op.isl_exec->canReshape(); }); } void cv::gimpl::GExecutor::reshape(const GMetaArgs& inMetas, const GCompileArgs& args) @@ -421,7 +420,17 @@ void cv::gimpl::GExecutor::reshape(const GMetaArgs& inMetas, const GCompileArgs& ade::passes::PassContext ctx{g}; passes::initMeta(ctx, inMetas); passes::inferMeta(ctx, true); - m_ops[0].isl_exec->reshape(g, args); + + // NB: Before reshape islands need to re-init resources for every slot. + for (auto slot : m_slots) + { + initResource(slot.slot_nh, slot.data_nh); + } + + for (auto& op : m_ops) + { + op.isl_exec->reshape(g, args); + } } void cv::gimpl::GExecutor::prepareForNewStream() diff --git a/modules/gapi/test/internal/gapi_int_executor_tests.cpp b/modules/gapi/test/internal/gapi_int_executor_tests.cpp index 04afa1fda8..90a338a3d0 100644 --- a/modules/gapi/test/internal/gapi_int_executor_tests.cpp +++ b/modules/gapi/test/internal/gapi_int_executor_tests.cpp @@ -6,10 +6,158 @@ #include "../test_precomp.hpp" +#include "../gapi_mock_kernels.hpp" namespace opencv_test { +namespace +{ + +class GMockExecutable final: public cv::gimpl::GIslandExecutable +{ + virtual inline bool canReshape() const override { + return m_priv->m_can_reshape; + } + + virtual void reshape(ade::Graph&, const GCompileArgs&) override + { + m_priv->m_reshape_counter++; + } + virtual void handleNewStream() override { } + virtual void run(std::vector&&, std::vector&&) { } + virtual bool allocatesOutputs() const override + { + return true; + } + + virtual cv::RMat allocate(const cv::GMatDesc&) const override + { + m_priv->m_allocate_counter++; + return cv::RMat(); + } + + // NB: GMockBackendImpl creates new unique_ptr + // on every compile call. Need to share counters between instances in order + // to validate it in tests. + struct Priv + { + bool m_can_reshape; + int m_reshape_counter; + int m_allocate_counter; + }; + + std::shared_ptr m_priv; + +public: + GMockExecutable(bool can_reshape = true) + : m_priv(new Priv{can_reshape, 0, 0}) + { + }; + + void setReshape(bool can_reshape) { m_priv->m_can_reshape = can_reshape; } + + int getReshapeCounter() const { return m_priv->m_reshape_counter; } + int getAllocateCounter() const { return m_priv->m_allocate_counter; } +}; + +class GMockBackendImpl final: public cv::gapi::GBackend::Priv +{ + virtual void unpackKernel(ade::Graph &, + const ade::NodeHandle &, + const cv::GKernelImpl &) override { } + + virtual EPtr compile(const ade::Graph &, + const cv::GCompileArgs &, + const std::vector &) const override + { + ++m_compile_counter; + return EPtr{new GMockExecutable(m_exec)}; + } + + mutable int m_compile_counter = 0; + GMockExecutable m_exec; + + virtual bool controlsMerge() const override { + return true; + } + + virtual bool allowsMerge(const cv::gimpl::GIslandModel::Graph &, + const ade::NodeHandle &, + const ade::NodeHandle &, + const ade::NodeHandle &) const override { + return false; + } + +public: + GMockBackendImpl(const GMockExecutable& exec) : m_exec(exec) { }; + int getCompileCounter() const { return m_compile_counter; } +}; + +class GMockFunctor : public gapi::cpu::GOCVFunctor +{ +public: + GMockFunctor(cv::gapi::GBackend backend, + const char* id, + const Meta &meta, + const Impl& impl) + : gapi::cpu::GOCVFunctor(id, meta, impl), m_backend(backend) + { + } + + cv::gapi::GBackend backend() const override { return m_backend; } + +private: + cv::gapi::GBackend m_backend; +}; + +template +GMockFunctor mock_kernel(const cv::gapi::GBackend& backend, Callable c) +{ + using P = cv::detail::OCVCallHelper; + return GMockFunctor{ backend + , K::id() + , &K::getOutMeta + , std::bind(&P::callFunctor, std::placeholders::_1, c) + }; +} + +void dummyFooImpl(const cv::Mat&, cv::Mat&) { }; +void dummyBarImpl(const cv::Mat&, const cv::Mat&, cv::Mat&) { }; + +struct GExecutorReshapeTest: public ::testing::Test +{ + GExecutorReshapeTest() + : comp([](){ + cv::GMat in; + cv::GMat out = I::Bar::on(I::Foo::on(in), in); + return cv::GComputation(in, out); + }) + { + backend_impl1 = std::make_shared(island1); + backend1 = cv::gapi::GBackend{backend_impl1}; + backend_impl2 = std::make_shared(island2); + backend2 = cv::gapi::GBackend{backend_impl2}; + auto kernel1 = mock_kernel(backend1, dummyFooImpl); + auto kernel2 = mock_kernel(backend2, dummyBarImpl); + pkg = cv::gapi::kernels(kernel1, kernel2); + in_mat1 = cv::Mat::eye(32, 32, CV_8UC1); + in_mat2 = cv::Mat::eye(64, 64, CV_8UC1); + } + + cv::GComputation comp; + GMockExecutable island1; + std::shared_ptr backend_impl1; + cv::gapi::GBackend backend1; + GMockExecutable island2; + std::shared_ptr backend_impl2; + cv::gapi::GBackend backend2; + cv::gapi::GKernelPackage pkg; + cv::Mat in_mat1, in_mat2, out_mat;; +}; + +} // anonymous namespace + // FIXME: avoid code duplication // The below graph and config is taken from ComplexIslands test suite TEST(GExecutor, SmokeTest) @@ -77,6 +225,75 @@ TEST(GExecutor, SmokeTest) // with breakdown worked) } +TEST_F(GExecutorReshapeTest, ReshapeInsteadOfRecompile) +{ + // NB: Initial state + EXPECT_EQ(0, backend_impl1->getCompileCounter()); + EXPECT_EQ(0, backend_impl2->getCompileCounter()); + EXPECT_EQ(0, island1.getReshapeCounter()); + EXPECT_EQ(0, island2.getReshapeCounter()); + + // NB: First compilation. + comp.apply(cv::gin(in_mat1), cv::gout(out_mat), cv::compile_args(pkg)); + EXPECT_EQ(1, backend_impl1->getCompileCounter()); + EXPECT_EQ(1, backend_impl2->getCompileCounter()); + EXPECT_EQ(0, island1.getReshapeCounter()); + EXPECT_EQ(0, island2.getReshapeCounter()); + + // NB: GMockBackendImpl implements "reshape" method, + // so it won't be recompiled if the meta is changed. + comp.apply(cv::gin(in_mat2), cv::gout(out_mat), cv::compile_args(pkg)); + EXPECT_EQ(1, backend_impl1->getCompileCounter()); + EXPECT_EQ(1, backend_impl2->getCompileCounter()); + EXPECT_EQ(1, island1.getReshapeCounter()); + EXPECT_EQ(1, island2.getReshapeCounter()); +} + +TEST_F(GExecutorReshapeTest, OneBackendNotReshapable) +{ + // NB: Make first island not reshapable + island1.setReshape(false); + + // NB: Initial state + EXPECT_EQ(0, backend_impl1->getCompileCounter()); + EXPECT_EQ(0, island1.getReshapeCounter()); + EXPECT_EQ(0, backend_impl2->getCompileCounter()); + EXPECT_EQ(0, island2.getReshapeCounter()); + + // NB: First compilation. + comp.apply(cv::gin(in_mat1), cv::gout(out_mat), cv::compile_args(pkg)); + EXPECT_EQ(1, backend_impl1->getCompileCounter()); + EXPECT_EQ(1, backend_impl2->getCompileCounter()); + EXPECT_EQ(0, island1.getReshapeCounter()); + EXPECT_EQ(0, island2.getReshapeCounter()); + + // NB: Since one of islands isn't reshapable + // the entire graph isn't reshapable as well. + comp.apply(cv::gin(in_mat2), cv::gout(out_mat), cv::compile_args(pkg)); + EXPECT_EQ(2, backend_impl1->getCompileCounter()); + EXPECT_EQ(2, backend_impl2->getCompileCounter()); + EXPECT_EQ(0, island1.getReshapeCounter()); + EXPECT_EQ(0, island2.getReshapeCounter()); +} + +TEST_F(GExecutorReshapeTest, ReshapeCallAllocate) +{ + // NB: Initial state + EXPECT_EQ(0, island1.getAllocateCounter()); + EXPECT_EQ(0, island1.getReshapeCounter()); + + // NB: First compilation. + comp.apply(cv::gin(in_mat1), cv::gout(out_mat), cv::compile_args(pkg)); + EXPECT_EQ(1, island1.getAllocateCounter()); + EXPECT_EQ(0, island1.getReshapeCounter()); + + // NB: The entire graph is reshapable, so it won't be recompiled, but reshaped. + // Check that reshape call "allocate" to reallocate buffers. + comp.apply(cv::gin(in_mat2), cv::gout(out_mat), cv::compile_args(pkg)); + EXPECT_EQ(2, island1.getAllocateCounter()); + EXPECT_EQ(1, island1.getReshapeCounter()); +} + // FIXME: Add explicit tests on GMat/GScalar/GArray being connectors // between executed islands From f9bd83c8547054357b0ecc55dadda787e5422dad Mon Sep 17 00:00:00 2001 From: Tsukasa Sugiura Date: Fri, 24 Sep 2021 15:35:42 +0900 Subject: [PATCH 188/262] fix cast in text detection sample --- samples/dnn/text_detection.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/samples/dnn/text_detection.py b/samples/dnn/text_detection.py index 7014a80148..22c4f6b5ec 100644 --- a/samples/dnn/text_detection.py +++ b/samples/dnn/text_detection.py @@ -214,8 +214,8 @@ def main(): 0.5, (255, 0, 0)) for j in range(4): - p1 = (vertices[j][0], vertices[j][1]) - p2 = (vertices[(j + 1) % 4][0], vertices[(j + 1) % 4][1]) + p1 = (int(vertices[j][0]), int(vertices[j][1])) + p2 = (int(vertices[(j + 1) % 4][0]), int(vertices[(j + 1) % 4][1])) cv.line(frame, p1, p2, (0, 255, 0), 1) # Put efficiency information From 3673b45437e4c1e0490c0c48ba76439c4f135361 Mon Sep 17 00:00:00 2001 From: Ruslan Garnov Date: Fri, 24 Sep 2021 14:25:57 +0300 Subject: [PATCH 189/262] Added desync RMats and MediaFrames support --- .../gapi/include/opencv2/gapi/gstreaming.hpp | 12 +++- .../include/opencv2/gapi/streaming/desync.hpp | 5 +- modules/gapi/src/api/kernels_streaming.cpp | 7 +- .../gapi/src/executor/gstreamingexecutor.cpp | 5 +- .../test/streaming/gapi_streaming_tests.cpp | 68 ++++++++++++++++++- 5 files changed, 90 insertions(+), 7 deletions(-) diff --git a/modules/gapi/include/opencv2/gapi/gstreaming.hpp b/modules/gapi/include/opencv2/gapi/gstreaming.hpp index 0e7d2a847f..890eb584fb 100644 --- a/modules/gapi/include/opencv2/gapi/gstreaming.hpp +++ b/modules/gapi/include/opencv2/gapi/gstreaming.hpp @@ -2,7 +2,7 @@ // It is subject to the license terms in the LICENSE file found in the top-level directory // of this distribution and at http://opencv.org/license.html. // -// Copyright (C) 2018 Intel Corporation +// Copyright (C) 2018-2021 Intel Corporation #ifndef OPENCV_GAPI_GSTREAMING_COMPILED_HPP @@ -65,6 +65,7 @@ using OptionalOpaqueRef = OptRef; using GOptRunArgP = util::variant< optional*, optional*, + optional*, optional*, cv::detail::OptionalVectorRef, cv::detail::OptionalOpaqueRef @@ -74,6 +75,7 @@ using GOptRunArgsP = std::vector; using GOptRunArg = util::variant< optional, optional, + optional, optional, optional, optional @@ -95,6 +97,14 @@ template<> inline GOptRunArgP wrap_opt_arg(optional &m) { return GOptRunArgP{&m}; } +template<> inline GOptRunArgP wrap_opt_arg(optional &m) { + return GOptRunArgP{&m}; +} + +template<> inline GOptRunArgP wrap_opt_arg(optional &f) { + return GOptRunArgP{&f}; +} + template<> inline GOptRunArgP wrap_opt_arg(optional &s) { return GOptRunArgP{&s}; } diff --git a/modules/gapi/include/opencv2/gapi/streaming/desync.hpp b/modules/gapi/include/opencv2/gapi/streaming/desync.hpp index 86de279fe9..1ed6e24b49 100644 --- a/modules/gapi/include/opencv2/gapi/streaming/desync.hpp +++ b/modules/gapi/include/opencv2/gapi/streaming/desync.hpp @@ -2,7 +2,7 @@ // It is subject to the license terms in the LICENSE file found in the top-level directory // of this distribution and at http://opencv.org/license.html. // -// Copyright (C) 2020 Intel Corporation +// Copyright (C) 2020-2021 Intel Corporation #ifndef OPENCV_GAPI_GSTREAMING_DESYNC_HPP @@ -73,9 +73,10 @@ G desync(const G &g) { * which produces an array of cv::util::optional<> objects. * * @note This feature is highly experimental now and is currently - * limited to a single GMat argument only. + * limited to a single GMat/GFrame argument only. */ GAPI_EXPORTS GMat desync(const GMat &g); +GAPI_EXPORTS GFrame desync(const GFrame &f); } // namespace streaming } // namespace gapi diff --git a/modules/gapi/src/api/kernels_streaming.cpp b/modules/gapi/src/api/kernels_streaming.cpp index c88fbede5b..2c50551f4e 100644 --- a/modules/gapi/src/api/kernels_streaming.cpp +++ b/modules/gapi/src/api/kernels_streaming.cpp @@ -2,7 +2,7 @@ // It is subject to the license terms in the LICENSE file found in the top-level directory // of this distribution and at http://opencv.org/license.html. // -// Copyright (C) 2020 Intel Corporation +// Copyright (C) 2020-2021 Intel Corporation #include "precomp.hpp" @@ -75,6 +75,11 @@ cv::GMat cv::gapi::streaming::desync(const cv::GMat &g) { // object will feed both branches of the streaming executable. } +// All notes from the above desync(GMat) are also applicable here +cv::GFrame cv::gapi::streaming::desync(const cv::GFrame &f) { + return cv::gapi::copy(detail::desync(f)); +} + cv::GMat cv::gapi::streaming::BGR(const cv::GFrame& in) { return cv::gapi::streaming::GBGR::on(in); } diff --git a/modules/gapi/src/executor/gstreamingexecutor.cpp b/modules/gapi/src/executor/gstreamingexecutor.cpp index 27049aef63..2379e3e164 100644 --- a/modules/gapi/src/executor/gstreamingexecutor.cpp +++ b/modules/gapi/src/executor/gstreamingexecutor.cpp @@ -186,8 +186,9 @@ void sync_data(cv::gimpl::stream::Result &r, cv::GOptRunArgsP &outputs) // FIXME: this conversion should be unified switch (out_obj.index()) { - HANDLE_CASE(cv::Scalar); break; - HANDLE_CASE(cv::RMat); break; + HANDLE_CASE(cv::Scalar); break; + HANDLE_CASE(cv::RMat); break; + HANDLE_CASE(cv::MediaFrame); break; case T::index_of*>(): { // Mat: special handling. diff --git a/modules/gapi/test/streaming/gapi_streaming_tests.cpp b/modules/gapi/test/streaming/gapi_streaming_tests.cpp index 369a0c069f..4f8b2563ac 100644 --- a/modules/gapi/test/streaming/gapi_streaming_tests.cpp +++ b/modules/gapi/test/streaming/gapi_streaming_tests.cpp @@ -2,7 +2,7 @@ // It is subject to the license terms in the LICENSE file found in the top-level directory // of this distribution and at http://opencv.org/license.html. // -// Copyright (C) 2019-2020 Intel Corporation +// Copyright (C) 2019-2021 Intel Corporation #include "../test_precomp.hpp" @@ -2284,4 +2284,70 @@ TEST(OneVPL_Source, Init) EXPECT_TRUE(stream_data_provider->empty()); } #endif + +TEST(GAPI_Streaming, TestDesyncRMat) { + cv::GMat in; + auto blurred = cv::gapi::blur(in, cv::Size{3,3}); + auto desynced = cv::gapi::streaming::desync(blurred); + auto out = in - blurred; + auto pipe = cv::GComputation(cv::GIn(in), cv::GOut(desynced, out)).compileStreaming(); + + cv::Size sz(32,32); + cv::Mat in_mat(sz, CV_8UC3); + cv::randu(in_mat, cv::Scalar::all(0), cv::Scalar(255)); + pipe.setSource(cv::gin(in_mat)); + pipe.start(); + + cv::optional out_desync; + cv::optional out_rmat; + while (true) { + // Initially it throwed "bad variant access" since there was + // no RMat handling in wrap_opt_arg + EXPECT_NO_THROW(pipe.pull(cv::gout(out_desync, out_rmat))); + if (out_rmat) break; + } +} + +G_API_OP(GTestBlur, , "test.blur") { + static GFrameDesc outMeta(GFrameDesc d) { return d; } +}; +GAPI_OCV_KERNEL(GOcvTestBlur, GTestBlur) { + static void run(const cv::MediaFrame& in, cv::MediaFrame& out) { + auto d = in.desc(); + GAPI_Assert(d.fmt == cv::MediaFormat::BGR); + auto view = in.access(cv::MediaFrame::Access::R); + cv::Mat mat(d.size, CV_8UC3, view.ptr[0]); + cv::Mat blurred; + cv::blur(mat, blurred, cv::Size{3,3}); + out = cv::MediaFrame::Create(blurred); + } +}; + +TEST(GAPI_Streaming, TestDesyncMediaFrame) { + initTestDataPath(); + cv::GFrame in; + auto blurred = GTestBlur::on(in); + auto desynced = cv::gapi::streaming::desync(blurred); + auto out = GTestBlur::on(blurred); + auto pipe = cv::GComputation(cv::GIn(in), cv::GOut(desynced, out)) + .compileStreaming(cv::compile_args(cv::gapi::kernels())); + + std::string filepath = findDataFile("cv/video/768x576.avi"); + try { + pipe.setSource(filepath); + } catch(...) { + throw SkipTestException("Video file can not be opened"); + } + pipe.start(); + + cv::optional out_desync; + cv::optional out_frame; + while (true) { + // Initially it throwed "bad variant access" since there was + // no MediaFrame handling in wrap_opt_arg + EXPECT_NO_THROW(pipe.pull(cv::gout(out_desync, out_frame))); + if (out_frame) break; + } +} + } // namespace opencv_test From 91ff45fbde176ff749a7d4c1b265c73370e2ea7e Mon Sep 17 00:00:00 2001 From: easonycwang Date: Fri, 24 Sep 2021 17:44:58 +0800 Subject: [PATCH 190/262] Tile: This submission is used to improve the performance of the inpaint algorithm for 3 channels images(RGB or BGR). Reason: The original algorithm implementation did not consider the cache hits. The loop of channels is outside the core loop, so the perfmance is not very good. Moving the channel loop inside the core loop can significantly improve cache hits, thereby improving performance. Performance: 360P, about >= 30% improvement iphone8P: 5.52ms -> 3.75ms iphone6s: 14.04ms -> 9.15ms --- modules/photo/src/inpaint.cpp | 90 ++++++++++++++++++++--------------- 1 file changed, 51 insertions(+), 39 deletions(-) diff --git a/modules/photo/src/inpaint.cpp b/modules/photo/src/inpaint.cpp index 14c178e248..74e006588b 100644 --- a/modules/photo/src/inpaint.cpp +++ b/modules/photo/src/inpaint.cpp @@ -308,50 +308,58 @@ icvTeleaInpaintFMM(const CvMat *f, CvMat *t, CvMat *out, int range, CvPriorityQu FastMarching_solve(i+1,j,i,j+1,f,t)); CV_MAT_ELEM(*t,float,i,j) = dist; + cv::Point2f gradT[3]; for (color=0; color<=2; color++) { - cv::Point2f gradI,gradT,r; - float Ia=0,Jx=0,Jy=0,s=1.0e-20f,w,dst,lev,dir,sat; - if (CV_MAT_ELEM(*f,uchar,i,j+1)!=INSIDE) { if (CV_MAT_ELEM(*f,uchar,i,j-1)!=INSIDE) { - gradT.x=(float)((CV_MAT_ELEM(*t,float,i,j+1)-CV_MAT_ELEM(*t,float,i,j-1)))*0.5f; + gradT[color].x=(float)((CV_MAT_ELEM(*t,float,i,j+1)-CV_MAT_ELEM(*t,float,i,j-1)))*0.5f; } else { - gradT.x=(float)((CV_MAT_ELEM(*t,float,i,j+1)-CV_MAT_ELEM(*t,float,i,j))); + gradT[color].x=(float)((CV_MAT_ELEM(*t,float,i,j+1)-CV_MAT_ELEM(*t,float,i,j))); } } else { if (CV_MAT_ELEM(*f,uchar,i,j-1)!=INSIDE) { - gradT.x=(float)((CV_MAT_ELEM(*t,float,i,j)-CV_MAT_ELEM(*t,float,i,j-1))); + gradT[color].x=(float)((CV_MAT_ELEM(*t,float,i,j)-CV_MAT_ELEM(*t,float,i,j-1))); } else { - gradT.x=0; + gradT[color].x=0; } } if (CV_MAT_ELEM(*f,uchar,i+1,j)!=INSIDE) { if (CV_MAT_ELEM(*f,uchar,i-1,j)!=INSIDE) { - gradT.y=(float)((CV_MAT_ELEM(*t,float,i+1,j)-CV_MAT_ELEM(*t,float,i-1,j)))*0.5f; + gradT[color].y=(float)((CV_MAT_ELEM(*t,float,i+1,j)-CV_MAT_ELEM(*t,float,i-1,j)))*0.5f; } else { - gradT.y=(float)((CV_MAT_ELEM(*t,float,i+1,j)-CV_MAT_ELEM(*t,float,i,j))); + gradT[color].y=(float)((CV_MAT_ELEM(*t,float,i+1,j)-CV_MAT_ELEM(*t,float,i,j))); } } else { if (CV_MAT_ELEM(*f,uchar,i-1,j)!=INSIDE) { - gradT.y=(float)((CV_MAT_ELEM(*t,float,i,j)-CV_MAT_ELEM(*t,float,i-1,j))); + gradT[color].y=(float)((CV_MAT_ELEM(*t,float,i,j)-CV_MAT_ELEM(*t,float,i-1,j))); } else { - gradT.y=0; + gradT[color].y=0; } } - for (k=i-range; k<=i+range; k++) { - int km=k-1+(k==1),kp=k-1-(k==t->rows-2); - for (l=j-range; l<=j+range; l++) { - int lm=l-1+(l==1),lp=l-1-(l==t->cols-2); - if (k>0&&l>0&&krows-1&&lcols-1) { - if ((CV_MAT_ELEM(*f,uchar,k,l)!=INSIDE)&& - ((l-j)*(l-j)+(k-i)*(k-i)<=range*range)) { + } + + cv::Point2f gradI,r; + float Jx[3] = {0,0,0}; + float Jy[3] = {0,0,0}; + float Ia[3] = {0,0,0}; + float s[3] = {1.0e-20f,1.0e-20f,1.0e-20f}; + float w,dst,lev,dir,sat; + + for (k=i-range; k<=i+range; k++) { + int km=k-1+(k==1),kp=k-1-(k==t->rows-2); + for (l=j-range; l<=j+range; l++) { + int lm=l-1+(l==1),lp=l-1-(l==t->cols-2); + if (k>0&&l>0&&krows-1&&lcols-1) { + if ((CV_MAT_ELEM(*f,uchar,k,l)!=INSIDE)&& + ((l-j)*(l-j)+(k-i)*(k-i)<=range*range)) { + for (color=0; color<=2; color++) { r.y = (float)(i-k); r.x = (float)(j-l); dst = (float)(1./(VectorLength(r)*sqrt((double)VectorLength(r)))); lev = (float)(1./(1+fabs(CV_MAT_ELEM(*t,float,k,l)-CV_MAT_ELEM(*t,float,i,j)))); - dir=VectorScalMult(r,gradT); + dir=VectorScalMult(r,gradT[color]); if (fabs(dir)<=0.01) dir=0.000001f; w = (float)fabs(dst*lev*dir); @@ -381,18 +389,18 @@ icvTeleaInpaintFMM(const CvMat *f, CvMat *t, CvMat *out, int range, CvPriorityQu gradI.y=0; } } - Ia += (float)w * (float)(CV_MAT_3COLOR_ELEM(*out,uchar,km,lm,color)); - Jx -= (float)w * (float)(gradI.x*r.x); - Jy -= (float)w * (float)(gradI.y*r.y); - s += w; + Ia[color] += (float)w * (float)(CV_MAT_3COLOR_ELEM(*out,uchar,km,lm,color)); + Jx[color] -= (float)w * (float)(gradI.x*r.x); + Jy[color] -= (float)w * (float)(gradI.y*r.y); + s[color] += w; } } } } - sat = (float)((Ia/s+(Jx+Jy)/(sqrt(Jx*Jx+Jy*Jy)+1.0e-20f)+0.5f)); - { + } + for (color=0; color<=2; color++) { + sat = (float)((Ia[color]/s[color]+(Jx[color]+Jy[color])/(sqrt(Jx[color]*Jx[color]+Jy[color]*Jy[color])+1.0e-20f)+0.5f)); CV_MAT_3COLOR_ELEM(*out,uchar,i-1,j-1,color) = cv::saturate_cast(sat); - } } CV_MAT_ELEM(*f,uchar,i,j) = BAND; @@ -540,17 +548,19 @@ icvNSInpaintFMM(const CvMat *f, CvMat *t, CvMat *out, int range, CvPriorityQueue FastMarching_solve(i+1,j,i,j+1,f,t)); CV_MAT_ELEM(*t,float,i,j) = dist; - for (color=0; color<=2; color++) { - cv::Point2f gradI,r; - float Ia=0,s=1.0e-20f,w,dst,dir; + cv::Point2f gradI,r; + float Ia[3]={0,0,0}; + float s[3]={1.0e-20f,1.0e-20f,1.0e-20f}; + float w,dst,dir; - for (k=i-range; k<=i+range; k++) { - int km=k-1+(k==1),kp=k-1-(k==f->rows-2); - for (l=j-range; l<=j+range; l++) { - int lm=l-1+(l==1),lp=l-1-(l==f->cols-2); - if (k>0&&l>0&&krows-1&&lcols-1) { - if ((CV_MAT_ELEM(*f,uchar,k,l)!=INSIDE)&& - ((l-j)*(l-j)+(k-i)*(k-i)<=range*range)) { + for (k=i-range; k<=i+range; k++) { + int km=k-1+(k==1),kp=k-1-(k==f->rows-2); + for (l=j-range; l<=j+range; l++) { + int lm=l-1+(l==1),lp=l-1-(l==f->cols-2); + if (k>0&&l>0&&krows-1&&lcols-1) { + if ((CV_MAT_ELEM(*f,uchar,k,l)!=INSIDE)&& + ((l-j)*(l-j)+(k-i)*(k-i)<=range*range)) { + for (color=0; color<=2; color++) { r.y=(float)(k-i); r.x=(float)(l-j); @@ -594,13 +604,15 @@ icvNSInpaintFMM(const CvMat *f, CvMat *t, CvMat *out, int range, CvPriorityQueue dir = (float)fabs(VectorScalMult(r,gradI)/sqrt(VectorLength(r)*VectorLength(gradI))); } w = dst*dir; - Ia += (float)w * (float)(CV_MAT_3COLOR_ELEM(*out,uchar,km,lm,color)); - s += w; + Ia[color] += (float)w * (float)(CV_MAT_3COLOR_ELEM(*out,uchar,km,lm,color)); + s[color] += w; } } } } - CV_MAT_3COLOR_ELEM(*out,uchar,i-1,j-1,color) = cv::saturate_cast((double)Ia/s); + } + for (color=0; color<=2; color++) { + CV_MAT_3COLOR_ELEM(*out,uchar,i-1,j-1,color) = cv::saturate_cast((double)Ia[color]/s[color]); } CV_MAT_ELEM(*f,uchar,i,j) = BAND; From 9e835e8edb62b42ca53d887be4c124fed142dfcd Mon Sep 17 00:00:00 2001 From: thezane <10068531+thezane@users.noreply.github.com> Date: Sat, 25 Sep 2021 13:42:12 -0400 Subject: [PATCH 191/262] Merge pull request #20636 from thezane:recoverPoseFromDifferentCameras Recover pose from different cameras (version 2) * add recoverPose for two different cameras * Address review comments from original PR * Address new review comments * Rename private api Co-authored-by: tompollok Co-authored-by: Zane --- modules/calib3d/include/opencv2/calib3d.hpp | 70 +++++++++++++++++++ modules/calib3d/src/five-point.cpp | 70 ++++++++++++++----- .../calib3d/test/test_cameracalibration.cpp | 48 +++++++++++-- 3 files changed, 162 insertions(+), 26 deletions(-) diff --git a/modules/calib3d/include/opencv2/calib3d.hpp b/modules/calib3d/include/opencv2/calib3d.hpp index 4928df65d1..37c5e7089a 100644 --- a/modules/calib3d/include/opencv2/calib3d.hpp +++ b/modules/calib3d/include/opencv2/calib3d.hpp @@ -2842,6 +2842,76 @@ unit length. */ CV_EXPORTS_W void decomposeEssentialMat( InputArray E, OutputArray R1, OutputArray R2, OutputArray t ); +/** @brief Recovers the relative camera rotation and the translation from corresponding points in two images from two different cameras, using cheirality check. Returns the number of +inliers that pass the check. + +@param points1 Array of N 2D points from the first image. The point coordinates should be +floating-point (single or double precision). +@param points2 Array of the second image points of the same size and format as points1 . +@param cameraMatrix1 Input/output camera matrix for the first camera, the same as in +@ref calibrateCamera. Furthermore, for the stereo case, additional flags may be used, see below. +@param distCoeffs1 Input/output vector of distortion coefficients, the same as in +@ref calibrateCamera. +@param cameraMatrix2 Input/output camera matrix for the first camera, the same as in +@ref calibrateCamera. Furthermore, for the stereo case, additional flags may be used, see below. +@param distCoeffs2 Input/output vector of distortion coefficients, the same as in +@ref calibrateCamera. +@param E The output essential matrix. +@param R Output rotation matrix. Together with the translation vector, this matrix makes up a tuple +that performs a change of basis from the first camera's coordinate system to the second camera's +coordinate system. Note that, in general, t can not be used for this tuple, see the parameter +described below. +@param t Output translation vector. This vector is obtained by @ref decomposeEssentialMat and +therefore is only known up to scale, i.e. t is the direction of the translation vector and has unit +length. +@param method Method for computing an essential matrix. +- @ref RANSAC for the RANSAC algorithm. +- @ref LMEDS for the LMedS algorithm. +@param prob Parameter used for the RANSAC or LMedS methods only. It specifies a desirable level of +confidence (probability) that the estimated matrix is correct. +@param threshold Parameter used for RANSAC. It is the maximum distance from a point to an epipolar +line in pixels, beyond which the point is considered an outlier and is not used for computing the +final fundamental matrix. It can be set to something like 1-3, depending on the accuracy of the +point localization, image resolution, and the image noise. +@param mask Input/output mask for inliers in points1 and points2. If it is not empty, then it marks +inliers in points1 and points2 for then given essential matrix E. Only these inliers will be used to +recover pose. In the output mask only inliers which pass the cheirality check. + +This function decomposes an essential matrix using @ref decomposeEssentialMat and then verifies +possible pose hypotheses by doing cheirality check. The cheirality check means that the +triangulated 3D points should have positive depth. Some details can be found in @cite Nister03. + +This function can be used to process the output E and mask from @ref findEssentialMat. In this +scenario, points1 and points2 are the same input for findEssentialMat.: +@code + // Example. Estimation of fundamental matrix using the RANSAC algorithm + int point_count = 100; + vector points1(point_count); + vector points2(point_count); + + // initialize the points here ... + for( int i = 0; i < point_count; i++ ) + { + points1[i] = ...; + points2[i] = ...; + } + + // Input: camera calibration of both cameras, for example using intrinsic chessboard calibration. + Mat cameraMatrix1, distCoeffs1, cameraMatrix2, distCoeffs2; + + // Output: Essential matrix, relative rotation and relative translation. + Mat E, R, t, mask; + + recoverPose(points1, points2, cameraMatrix1, distCoeffs1, cameraMatrix2, distCoeffs2, E, R, t, mask); +@endcode + */ +CV_EXPORTS_W int recoverPose( InputArray points1, InputArray points2, + InputArray cameraMatrix1, InputArray distCoeffs1, + InputArray cameraMatrix2, InputArray distCoeffs2, + OutputArray E, OutputArray R, OutputArray t, + int method = cv::RANSAC, double prob = 0.999, double threshold = 1.0, + InputOutputArray mask = noArray()); + /** @brief Recovers the relative camera rotation and the translation from an estimated essential matrix and the corresponding points in two images, using cheirality check. Returns the number of inliers that pass the check. diff --git a/modules/calib3d/src/five-point.cpp b/modules/calib3d/src/five-point.cpp index d005248535..d947334142 100644 --- a/modules/calib3d/src/five-point.cpp +++ b/modules/calib3d/src/five-point.cpp @@ -401,6 +401,29 @@ protected: } }; +// Find essential matrix given undistorted points and two cameras. +static Mat findEssentialMat_( InputArray _points1, InputArray _points2, + InputArray cameraMatrix1, InputArray cameraMatrix2, + int method, double prob, double threshold, OutputArray _mask) +{ + // Scale the points back. We use "arithmetic mean" between the supplied two camera matrices. + // Thanks to such 2-stage procedure RANSAC threshold still makes sense, because the undistorted + // and rescaled points have a similar value range to the original ones. + Mat _pointsTransformed1, _pointsTransformed2; + Mat cm1 = cameraMatrix1.getMat(), cm2 = cameraMatrix2.getMat(), cm0; + Mat(cm1 + cm2).convertTo(cm0, CV_64F, 0.5); + CV_Assert(cm0.rows == 3 && cm0.cols == 3); + CV_Assert(std::abs(cm0.at(2, 0)) < 1e-3 && + std::abs(cm0.at(2, 1)) < 1e-3 && + std::abs(cm0.at(2, 2) - 1.) < 1e-3); + Mat affine = cm0.rowRange(0, 2); + + transform(_points1, _pointsTransformed1, affine); + transform(_points2, _pointsTransformed2, affine); + + return findEssentialMat(_pointsTransformed1, _pointsTransformed2, cm0, method, prob, threshold, _mask); +} + } // Input should be a vector of n 2D points or a Nx2 matrix @@ -489,25 +512,10 @@ cv::Mat cv::findEssentialMat( InputArray _points1, InputArray _points2, CV_INSTRUMENT_REGION(); // Undistort image points, bring them to 3x3 identity "camera matrix" - Mat _pointsUntistorted1, _pointsUntistorted2; - undistortPoints(_points1, _pointsUntistorted1, cameraMatrix1, distCoeffs1); - undistortPoints(_points2, _pointsUntistorted2, cameraMatrix2, distCoeffs2); - - // Scale the points back. We use "arithmetic mean" between the supplied two camera matrices. - // Thanks to such 2-stage procedure RANSAC threshold still makes sense, because the undistorted - // and rescaled points have a similar value range to the original ones. - Mat cm1 = cameraMatrix1.getMat(), cm2 = cameraMatrix2.getMat(), cm0; - Mat(cm1 + cm2).convertTo(cm0, CV_64F, 0.5); - CV_Assert(cm0.rows == 3 && cm0.cols == 3); - CV_Assert(std::abs(cm0.at(2, 0)) < 1e-3 && - std::abs(cm0.at(2, 1)) < 1e-3 && - std::abs(cm0.at(2, 2) - 1.) < 1e-3); - Mat affine = cm0.rowRange(0, 2); - - transform(_pointsUntistorted1, _pointsUntistorted1, affine); - transform(_pointsUntistorted2, _pointsUntistorted2, affine); - - return findEssentialMat(_pointsUntistorted1, _pointsUntistorted2, cm0, method, prob, threshold, _mask); + Mat _pointsUndistorted1, _pointsUndistorted2; + undistortPoints(_points1, _pointsUndistorted1, cameraMatrix1, distCoeffs1); + undistortPoints(_points2, _pointsUndistorted2, cameraMatrix2, distCoeffs2); + return findEssentialMat_(_pointsUndistorted1, _pointsUndistorted2, cameraMatrix1, cameraMatrix2, method, prob, threshold, _mask); } cv::Mat cv::findEssentialMat( InputArray points1, InputArray points2, @@ -524,6 +532,30 @@ cv::Mat cv::findEssentialMat( InputArray points1, InputArray points2, } +int cv::recoverPose( InputArray _points1, InputArray _points2, + InputArray cameraMatrix1, InputArray distCoeffs1, + InputArray cameraMatrix2, InputArray distCoeffs2, + OutputArray E, OutputArray R, OutputArray t, + int method, double prob, double threshold, + InputOutputArray _mask) +{ + CV_INSTRUMENT_REGION(); + + // Undistort image points, bring them to 3x3 identity "camera matrix" + Mat _pointsUndistorted1, _pointsUndistorted2; + undistortPoints(_points1, _pointsUndistorted1, cameraMatrix1, distCoeffs1); + undistortPoints(_points2, _pointsUndistorted2, cameraMatrix2, distCoeffs2); + + // Get essential matrix. + Mat _E = findEssentialMat_(_pointsUndistorted1, _pointsUndistorted2, cameraMatrix1, cameraMatrix2, + method, prob, threshold, _mask); + CV_Assert(_E.cols == 3 && _E.rows == 3); + E.create(3, 3, _E.type()); + _E.copyTo(E); + + return recoverPose(_E, _pointsUndistorted1, _pointsUndistorted2, Mat::eye(3,3, CV_64F), R, t, _mask); +} + int cv::recoverPose( InputArray E, InputArray _points1, InputArray _points2, InputArray _cameraMatrix, OutputArray _R, OutputArray _t, double distanceThresh, InputOutputArray _mask, OutputArray triangulatedPoints) diff --git a/modules/calib3d/test/test_cameracalibration.cpp b/modules/calib3d/test/test_cameracalibration.cpp index 9f3663e8cc..10350527fa 100644 --- a/modules/calib3d/test/test_cameracalibration.cpp +++ b/modules/calib3d/test/test_cameracalibration.cpp @@ -2142,7 +2142,17 @@ TEST(CV_RecoverPoseTest, regression_15341) // camera matrix with both focal lengths = 1, and principal point = (0, 0) const Mat cameraMatrix = Mat::eye(3, 3, CV_64F); - const Mat zeroDistCoeffs = Mat::zeros(1, 5, CV_64F); + + // camera matrix with focal lengths 0.5 and 0.6 respectively and principal point = (100, 200) + double cameraMatrix2Data[] = { 0.5, 0, 100, + 0, 0.6, 200, + 0, 0, 1 }; + const Mat cameraMatrix2( 3, 3, CV_64F, cameraMatrix2Data ); + + // zero and nonzero distortion coefficients + double nonZeroDistCoeffsData[] = { 0.01, 0.0001, 0, 0, 1e-04, 0.2, 0.02, 0.0002 }; // k1, k2, p1, p2, k3, k4, k5, k6 + vector distCoeffsList = {Mat::zeros(1, 5, CV_64F), Mat{1, 8, CV_64F, nonZeroDistCoeffsData}}; + const auto &zeroDistCoeffs = distCoeffsList[0]; int Inliers = 0; @@ -2158,14 +2168,26 @@ TEST(CV_RecoverPoseTest, regression_15341) // Estimation of fundamental matrix using the RANSAC algorithm Mat E, E2, R, t; + + // Check pose when camera matrices are different. + for (const auto &distCoeffs: distCoeffsList) + { + E = findEssentialMat(points1, points2, cameraMatrix, distCoeffs, cameraMatrix2, distCoeffs, RANSAC, 0.999, 1.0, mask); + recoverPose(points1, points2, cameraMatrix, distCoeffs, cameraMatrix2, distCoeffs, E2, R, t, RANSAC, 0.999, 1.0, mask); + EXPECT_LT(cv::norm(E, E2, NORM_INF), 1e-4) << + "Two big difference between the same essential matrices computed using different functions with different cameras, testcase " << testcase; + EXPECT_EQ(0, (int)mask[13]) << "Detecting outliers in function failed with different cameras, testcase " << testcase; + } + + // Check pose when camera matrices are the same. E = findEssentialMat(points1, points2, cameraMatrix, RANSAC, 0.999, 1.0, mask); E2 = findEssentialMat(points1, points2, cameraMatrix, zeroDistCoeffs, cameraMatrix, zeroDistCoeffs, RANSAC, 0.999, 1.0, mask); EXPECT_LT(cv::norm(E, E2, NORM_INF), 1e-4) << - "Two big difference between the same essential matrices computed using different functions, testcase " << testcase; - EXPECT_EQ(0, (int)mask[13]) << "Detecting outliers in function findEssentialMat failed, testcase " << testcase; + "Two big difference between the same essential matrices computed using different functions with same cameras, testcase " << testcase; + EXPECT_EQ(0, (int)mask[13]) << "Detecting outliers in function findEssentialMat failed with same cameras, testcase " << testcase; points2[12] = Point2f(0.0f, 0.0f); // provoke another outlier detection for recover Pose Inliers = recoverPose(E, points1, points2, cameraMatrix, R, t, mask); - EXPECT_EQ(0, (int)mask[12]) << "Detecting outliers in function failed, testcase " << testcase; + EXPECT_EQ(0, (int)mask[12]) << "Detecting outliers in function failed with same cameras, testcase " << testcase; } else // testcase with mat input data { @@ -2185,14 +2207,26 @@ TEST(CV_RecoverPoseTest, regression_15341) // Estimation of fundamental matrix using the RANSAC algorithm Mat E, E2, R, t; + + // Check pose when camera matrices are different. + for (const auto &distCoeffs: distCoeffsList) + { + E = findEssentialMat(points1, points2, cameraMatrix, distCoeffs, cameraMatrix2, distCoeffs, RANSAC, 0.999, 1.0, mask); + recoverPose(points1, points2, cameraMatrix, distCoeffs, cameraMatrix2, distCoeffs, E2, R, t, RANSAC, 0.999, 1.0, mask); + EXPECT_LT(cv::norm(E, E2, NORM_INF), 1e-4) << + "Two big difference between the same essential matrices computed using different functions with different cameras, testcase " << testcase; + EXPECT_EQ(0, (int)mask.at(13)) << "Detecting outliers in function failed with different cameras, testcase " << testcase; + } + + // Check pose when camera matrices are the same. E = findEssentialMat(points1, points2, cameraMatrix, RANSAC, 0.999, 1.0, mask); E2 = findEssentialMat(points1, points2, cameraMatrix, zeroDistCoeffs, cameraMatrix, zeroDistCoeffs, RANSAC, 0.999, 1.0, mask); EXPECT_LT(cv::norm(E, E2, NORM_INF), 1e-4) << - "Two big difference between the same essential matrices computed using different functions, testcase " << testcase; - EXPECT_EQ(0, (int)mask.at(13)) << "Detecting outliers in function findEssentialMat failed, testcase " << testcase; + "Two big difference between the same essential matrices computed using different functions with same cameras, testcase " << testcase; + EXPECT_EQ(0, (int)mask.at(13)) << "Detecting outliers in function findEssentialMat failed with same cameras, testcase " << testcase; points2.at(12) = Point2f(0.0f, 0.0f); // provoke an outlier detection Inliers = recoverPose(E, points1, points2, cameraMatrix, R, t, mask); - EXPECT_EQ(0, (int)mask.at(12)) << "Detecting outliers in function failed, testcase " << testcase; + EXPECT_EQ(0, (int)mask.at(12)) << "Detecting outliers in function failed with same cameras, testcase " << testcase; } EXPECT_EQ(Inliers, point_count - invalid_point_count) << "Number of inliers differs from expected number of inliers, testcase " << testcase; From 236c64a17d4c9e1eac3fb25044bdfeceba1d7463 Mon Sep 17 00:00:00 2001 From: Nicholas Ho <88894303+Nicholas-Ho-arm@users.noreply.github.com> Date: Sat, 25 Sep 2021 18:43:33 +0100 Subject: [PATCH 192/262] Merge pull request #20712 from Nicholas-Ho-arm:3.4_RowVec_8u32f * Add RowVec_8u32f * Fix build errors in Linux x64 Debug and armeabi-v7a * Reformat code to make it more clean and conventional * Optimise with vx_load_expand_q() --- modules/imgproc/src/filter.simd.hpp | 47 ++++++++++++++++++++++++++++- 1 file changed, 46 insertions(+), 1 deletion(-) diff --git a/modules/imgproc/src/filter.simd.hpp b/modules/imgproc/src/filter.simd.hpp index bed6f83071..94dbce060c 100644 --- a/modules/imgproc/src/filter.simd.hpp +++ b/modules/imgproc/src/filter.simd.hpp @@ -465,6 +465,49 @@ struct RowVec_8u32s bool smallValues; }; +struct RowVec_8u32f +{ + RowVec_8u32f() {} + RowVec_8u32f( const Mat& _kernel ) : kernel(_kernel) {} + + int operator()(const uchar* _src, uchar* _dst, int width, int cn) const + { + CV_INSTRUMENT_REGION(); + + int i = 0, k, _ksize = kernel.rows + kernel.cols - 1; + float* dst = (float*)_dst; + const float* _kx = kernel.ptr(); + width *= cn; + for( ; i <= width - v_uint8::nlanes; i += v_uint8::nlanes ) + { + v_float32 s0 = vx_setzero_f32(); + v_float32 s1 = vx_setzero_f32(); + v_float32 s2 = vx_setzero_f32(); + v_float32 s3 = vx_setzero_f32(); + k = 0; + for( ; k < _ksize ; k++ ) + { + v_float32 f = vx_setall_f32(_kx[k]); + const uchar* src = (const uchar*)_src + i + k * cn; + v_float32 vs_ll = v_cvt_f32(v_reinterpret_as_s32(vx_load_expand_q(src))); + v_float32 vs_lh = v_cvt_f32(v_reinterpret_as_s32(vx_load_expand_q(src + v_float32::nlanes))); + v_float32 vs_hl = v_cvt_f32(v_reinterpret_as_s32(vx_load_expand_q(src + 2*v_float32::nlanes))); + v_float32 vs_hh = v_cvt_f32(v_reinterpret_as_s32(vx_load_expand_q(src + 3*v_float32::nlanes))); + s0 = v_muladd(vs_ll, f, s0); + s1 = v_muladd(vs_lh, f, s1); + s2 = v_muladd(vs_hl, f, s2); + s3 = v_muladd(vs_hh, f, s3); + } + v_store(dst + i, s0); + v_store(dst + i + v_float32::nlanes, s1); + v_store(dst + i + 2*v_float32::nlanes, s2); + v_store(dst + i + 3*v_float32::nlanes, s3); + } + return i; + } + + Mat kernel; +}; struct SymmRowSmallVec_8u32s { @@ -2292,6 +2335,7 @@ struct FilterVec_32f #else typedef RowNoVec RowVec_8u32s; +typedef RowNoVec RowVec_8u32f; typedef RowNoVec RowVec_16s32f; typedef RowNoVec RowVec_32f; typedef SymmRowSmallNoVec SymmRowSmallVec_8u32s; @@ -2899,7 +2943,8 @@ Ptr getLinearRowFilter( return makePtr > (kernel, anchor, RowVec_8u32s(kernel)); if( sdepth == CV_8U && ddepth == CV_32F ) - return makePtr >(kernel, anchor); + return makePtr > + (kernel, anchor, RowVec_8u32f(kernel)); if( sdepth == CV_8U && ddepth == CV_64F ) return makePtr >(kernel, anchor); if( sdepth == CV_16U && ddepth == CV_32F ) From fdc8ed8d05d6f073dab21a6f0fdb3f20953738be Mon Sep 17 00:00:00 2001 From: Suleyman TURKMEN Date: Sat, 25 Sep 2021 09:31:44 +0300 Subject: [PATCH 193/262] Update perf_bgfg_mog2.cpp, perf_bgfg_knn.cpp --- modules/video/perf/opencl/perf_bgfg_knn.cpp | 6 +++--- modules/video/perf/opencl/perf_bgfg_mog2.cpp | 6 +++--- modules/video/perf/perf_bgfg_knn.cpp | 6 +++--- modules/video/perf/perf_bgfg_mog2.cpp | 6 +++--- 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/modules/video/perf/opencl/perf_bgfg_knn.cpp b/modules/video/perf/opencl/perf_bgfg_knn.cpp index 30419af422..c96ba6881a 100644 --- a/modules/video/perf/opencl/perf_bgfg_knn.cpp +++ b/modules/video/perf/opencl/perf_bgfg_knn.cpp @@ -20,7 +20,7 @@ typedef TestBaseWithParam KNN_GetBackgroundImage; using namespace opencv_test; -OCL_PERF_TEST_P(KNN_Apply, KNN, Combine(Values("gpu/video/768x576.avi", "gpu/video/1920x1080.avi"), Values(1,3))) +OCL_PERF_TEST_P(KNN_Apply, KNN, Combine(Values("cv/video/768x576.avi", "cv/video/1920x1080.avi"), Values(1,3))) { VideoKNNParamType params = GetParam(); @@ -51,8 +51,8 @@ OCL_PERF_TEST_P(KNN_Apply, KNN, Combine(Values("gpu/video/768x576.avi", "gpu/vid } OCL_PERF_TEST_P(KNN_GetBackgroundImage, KNN, Values( - std::make_pair("gpu/video/768x576.avi", 5), - std::make_pair("gpu/video/1920x1080.avi", 5))) + std::make_pair("cv/video/768x576.avi", 5), + std::make_pair("cv/video/1920x1080.avi", 5))) { VideoKNNParamType params = GetParam(); diff --git a/modules/video/perf/opencl/perf_bgfg_mog2.cpp b/modules/video/perf/opencl/perf_bgfg_mog2.cpp index 9952be79f6..b127a50d8d 100644 --- a/modules/video/perf/opencl/perf_bgfg_mog2.cpp +++ b/modules/video/perf/opencl/perf_bgfg_mog2.cpp @@ -20,7 +20,7 @@ typedef TestBaseWithParam MOG2_GetBackgroundImage; using namespace opencv_test; -OCL_PERF_TEST_P(MOG2_Apply, Mog2, Combine(Values("gpu/video/768x576.avi", "gpu/video/1920x1080.avi"), Values(1,3))) +OCL_PERF_TEST_P(MOG2_Apply, Mog2, Combine(Values("cv/video/768x576.avi", "cv/video/1920x1080.avi"), Values(1,3))) { VideoMOG2ParamType params = GetParam(); @@ -51,8 +51,8 @@ OCL_PERF_TEST_P(MOG2_Apply, Mog2, Combine(Values("gpu/video/768x576.avi", "gpu/v } OCL_PERF_TEST_P(MOG2_GetBackgroundImage, Mog2, Values( - std::make_pair("gpu/video/768x576.avi", 5), - std::make_pair("gpu/video/1920x1080.avi", 5))) + std::make_pair("cv/video/768x576.avi", 5), + std::make_pair("cv/video/1920x1080.avi", 5))) { VideoMOG2ParamType params = GetParam(); diff --git a/modules/video/perf/perf_bgfg_knn.cpp b/modules/video/perf/perf_bgfg_knn.cpp index d9ead09fd9..5d4f3bc400 100644 --- a/modules/video/perf/perf_bgfg_knn.cpp +++ b/modules/video/perf/perf_bgfg_knn.cpp @@ -15,7 +15,7 @@ typedef tuple VideoKNNParamType; typedef TestBaseWithParam KNN_Apply; typedef TestBaseWithParam KNN_GetBackgroundImage; -PERF_TEST_P(KNN_Apply, KNN, Combine(Values("gpu/video/768x576.avi", "gpu/video/1920x1080.avi"), Values(1,3))) +PERF_TEST_P(KNN_Apply, KNN, Combine(Values("cv/video/768x576.avi", "cv/video/1920x1080.avi"), Values(1,3))) { VideoKNNParamType params = GetParam(); @@ -46,8 +46,8 @@ PERF_TEST_P(KNN_Apply, KNN, Combine(Values("gpu/video/768x576.avi", "gpu/video/1 } PERF_TEST_P(KNN_GetBackgroundImage, KNN, Values( - std::make_pair("gpu/video/768x576.avi", 5), - std::make_pair("gpu/video/1920x1080.avi", 5))) + std::make_pair("cv/video/768x576.avi", 5), + std::make_pair("cv/video/1920x1080.avi", 5))) { VideoKNNParamType params = GetParam(); diff --git a/modules/video/perf/perf_bgfg_mog2.cpp b/modules/video/perf/perf_bgfg_mog2.cpp index 92e5d0283f..b932d6a22c 100644 --- a/modules/video/perf/perf_bgfg_mog2.cpp +++ b/modules/video/perf/perf_bgfg_mog2.cpp @@ -15,7 +15,7 @@ typedef tuple VideoMOG2ParamType; typedef TestBaseWithParam MOG2_Apply; typedef TestBaseWithParam MOG2_GetBackgroundImage; -PERF_TEST_P(MOG2_Apply, Mog2, Combine(Values("gpu/video/768x576.avi", "gpu/video/1920x1080.avi"), Values(1,3))) +PERF_TEST_P(MOG2_Apply, Mog2, Combine(Values("cv/video/768x576.avi", "cv/video/1920x1080.avi"), Values(1,3))) { VideoMOG2ParamType params = GetParam(); @@ -46,8 +46,8 @@ PERF_TEST_P(MOG2_Apply, Mog2, Combine(Values("gpu/video/768x576.avi", "gpu/video } PERF_TEST_P(MOG2_GetBackgroundImage, Mog2, Values( - std::make_pair("gpu/video/768x576.avi", 5), - std::make_pair("gpu/video/1920x1080.avi", 5))) + std::make_pair("cv/video/768x576.avi", 5), + std::make_pair("cv/video/1920x1080.avi", 5))) { VideoMOG2ParamType params = GetParam(); From 2cc14bd0fb2b898d7667c01bd24754dc9df0d5bd Mon Sep 17 00:00:00 2001 From: Alexander Smorkalov Date: Mon, 27 Sep 2021 10:53:23 +0300 Subject: [PATCH 194/262] Verbose output for errors found by header parser. --- modules/python/src2/hdr_parser.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/modules/python/src2/hdr_parser.py b/modules/python/src2/hdr_parser.py index 951dfe11c3..80cdea339f 100755 --- a/modules/python/src2/hdr_parser.py +++ b/modules/python/src2/hdr_parser.py @@ -53,13 +53,13 @@ class CppHeaderParser(object): def get_macro_arg(self, arg_str, npos): npos2 = npos3 = arg_str.find("(", npos) if npos2 < 0: - print("Error: no arguments for the macro at %d" % (self.lineno,)) + print("Error: no arguments for the macro at %s:%d" % (self.hname, self.lineno)) sys.exit(-1) balance = 1 while 1: t, npos3 = self.find_next_token(arg_str, ['(', ')'], npos3+1) if npos3 < 0: - print("Error: no matching ')' in the macro call at %d" % (self.lineno,)) + print("Error: no matching ')' in the macro call at %s:%d" % (self.hname, self.lineno)) sys.exit(-1) if t == '(': balance += 1 @@ -161,7 +161,7 @@ class CppHeaderParser(object): angle_stack.append(0) elif w == "," or w == '>': if not angle_stack: - print("Error at %d: argument contains ',' or '>' not within template arguments" % (self.lineno,)) + print("Error at %s:%d: argument contains ',' or '>' not within template arguments" % (self.hname, self.lineno)) sys.exit(-1) if w == ",": arg_type += "_and_" @@ -191,7 +191,7 @@ class CppHeaderParser(object): p1 = arg_name.find("[") p2 = arg_name.find("]",p1+1) if p2 < 0: - print("Error at %d: no closing ]" % (self.lineno,)) + print("Error at %s:%d: no closing ]" % (self.hname, self.lineno)) sys.exit(-1) counter_str = arg_name[p1+1:p2].strip() if counter_str == "": From 8fa8d471af43f62c60ea13c2ae1274390e34768b Mon Sep 17 00:00:00 2001 From: WJJ1995 Date: Tue, 28 Sep 2021 05:59:09 +0800 Subject: [PATCH 195/262] Merge pull request #20290 from wjj19950828:add_paddle_humanseg_demo Add paddle humanseg demo * fixed onnx resize op bug * add humanseg demo for PaddlePaddle sample * update README.md and flake8 format * update func name * update README.md for enviroment setup * update README.md in the way install paddle2onnx * update README.md * update README.md * add paddleseg in requirements.txt * deal with comments * replace picture --- .../dnn_conversion/paddlepaddle/README.md | 69 +++++++++-- .../paddlepaddle/data/result_test_human.jpg | Bin 0 -> 62199 bytes .../paddlepaddle/paddle_humanseg.py | 112 ++++++++++++++++++ .../paddlepaddle/paddle_resnet50.py | 18 +-- .../dnn_conversion/requirements.txt | 1 + 5 files changed, 183 insertions(+), 17 deletions(-) create mode 100644 samples/dnn/dnn_model_runner/dnn_conversion/paddlepaddle/data/result_test_human.jpg create mode 100644 samples/dnn/dnn_model_runner/dnn_conversion/paddlepaddle/paddle_humanseg.py diff --git a/samples/dnn/dnn_model_runner/dnn_conversion/paddlepaddle/README.md b/samples/dnn/dnn_model_runner/dnn_conversion/paddlepaddle/README.md index 7aba491c9d..1b1a28767c 100644 --- a/samples/dnn/dnn_model_runner/dnn_conversion/paddlepaddle/README.md +++ b/samples/dnn/dnn_model_runner/dnn_conversion/paddlepaddle/README.md @@ -1,6 +1,6 @@ -# Run PaddlePaddle model by OpenCV +# Run PaddlePaddle model using OpenCV -This tutorial shows how to run PaddlePaddle model by opencv. +These two demonstrations show how to inference PaddlePaddle model using OpenCV. ## Environment Setup @@ -10,16 +10,69 @@ pip install paddlehub pip install paddle2onnx ``` -## Run PaddlePaddle model demo +## 1. Run PaddlePaddle ResNet50 using OpenCV -Run the example code as below, +### Run PaddlePaddle model demo + +Run the code sample as follows: ```shell python paddle_resnet50.py ``` -there are 3 part of this execution +There are three parts to the process: -- 1. Export PaddlePaddle ResNet50 model to onnx format; -- 2. Use `cv2.dnn.readNetFromONNX` load model file; -- 3. Preprocess image file and do inference. +1. Export PaddlePaddle ResNet50 model to onnx format. +2. Use `cv2.dnn.readNetFromONNX` to load the model file. +3. Preprocess image file and do the inference. + +## 2. Run PaddleSeg Portrait Segmentation using OpenCV + +### Convert to ONNX Model + +#### 1. Get Paddle Inference model + +For more details, please refer to [PaddleSeg](https://github.com/PaddlePaddle/PaddleSeg/blob/release/2.1/contrib/HumanSeg/README.md). + +```shell +wget https://x2paddle.bj.bcebos.com/inference/models/humanseg_hrnet18_small_v1.zip +unzip humanseg_hrnet18_small_v1.zip +``` + +Notes: + +* The exported model must have a fixed input shape, as dynamic is not supported at this moment. + +#### 2. Convert to ONNX model using paddle2onnx + +To convert the model, use the following command: + +``` +paddle2onnx --model_dir humanseg_hrnet18_small_v1 \ + --model_filename model.pdmodel \ + --params_filename model.pdiparams \ + --opset_version 11 \ + --save_file humanseg_hrnet18_tiny.onnx +``` + +The converted model can be found in the current directory by the name `humanseg_hrnet18_tiny.onnx` . + +### Run PaddleSeg Portrait Segmentation demo + +Run the code sample as follows: + +```shell +python paddle_humanseg.py +``` + +There are three parts to the process: + +1. Use `cv2.dnn.readNetFromONNX` to load the model file. +2. Preprocess image file and do inference. +3. Postprocess image file and visualize. + +The resulting file can be found at `data/result_test_human.jpg` . + +### Portrait segmentation visualization + + diff --git a/samples/dnn/dnn_model_runner/dnn_conversion/paddlepaddle/data/result_test_human.jpg b/samples/dnn/dnn_model_runner/dnn_conversion/paddlepaddle/data/result_test_human.jpg new file mode 100644 index 0000000000000000000000000000000000000000..652b03f3b464ab432133061e780e7519c90c4ff7 GIT binary patch literal 62199 zcmbTdWl$VX)IPefc#uFKxCIL=5+JxHB)Ej&PLN>1-Q5=t5Nrv-Z8s1$xH}0D++7w} zY=MP^0QvKK->Q4R+z+inx3xnbl047`sru>&Hq~mJXTdyQ3PON0RUJJ2jJfl zKmmY_^nub+QFWK?uaY+QUoddBC>tn4p2x!;OQO3TVCDy!;$G{Ap0HZ`~O^!D`+ z3=R#COioSD%+Ad(EN*OWZSU+N_Vy1@XXh8_%PY+F&40MC0Kor+^-%vm;v#>*h5g?g z;QxmU3)}bM1d`+6vI^o+ywb-1;Qr*9P&fhQ>$IY}ZbCNU_oq}=9uq`Q*@8RL&tn7!|NbJMCf zSR8ece+;LnTljKx^R~A*er*78OJQ_L1grsjkUocWEsew`x-S*WD%pR|j)9rb+lZEc{mZLA!uH{|8irA>=r zNui7{ zQBpDKD!-dH4@~(yZa4VT`~wgU?wC`- z1oEo1nh>N~R5x;jle+sVY!kWBA|KrXerAP-|dfSO04)i@Cah z4tA|Rp$=0!5n_K%Fzfll7zLQO3Q<^i|zReD2av_NW5b>$*h;F%7k%SFi+IMvifdW#P--{R51{ z4VqGgAk9oOLW=S6o=~$F!x=1|gBIuaRAiS8S3oBQh$JNTcDGuS7SA2*OnnaWpymy>UG zr+wn#j5bgwM+gNx&`O={IBg2J`n~pes`m$_{!((2OpAeCXw8yHti14=oa%WBtqRBX zq1FC@rue?0b|t^FCRjs!Ng)8asoLwfZ3-N$bsw$nfLAqu`|DT6jekj+P2i`ykp|rO zR9=|)M66>+J${8gF?*k3$6yT4S>0=~&sx!~W|B-e3VKv2oAGQZi$^$o@Y9^s25t7? zx(bsrLxg{AC~wJXkg7ZmdCK|t^}gXiO{N;_FA{uGr-K0Yd-f$qUzPk6<1@0vR@X8y zQ)>w}*}KpJFN54_m$U6J$+G(*gP6;$BhgwSSw(-@h*l0+`vcDbiuV#AlZrzGo)8|H zICQRem0y`f&w%#B7y(SCEoPUvK07Kotggv7f@Z zTPCVmD~4xMOzyhr)%1yZ3q#8ts84f`qBA7Kky;bZE&_SKGl~wK@){zvy=s$7NDJh~ zF>}Qjyh-XF%}Cl~VTF_*##Gct9L)uleoTR1)S`~2Ni{r-8>4m`KNm%16l%T={4u{! zV~LBi1u5J%;MxqNr`hKCM$0w@>C|-G8=hXyy|~NSp1q+=TgD@LHLh-*aG@0&U@g+4 z)PR-NyH$qV-6iu_*TV}h1!sO94HR&38E31P~ zg3C+Zgnf0E=wj>~1ARe>{MPQX&Y&6dybm2mn5?o*m%+)nb-))4h=lV_8ceYlakt{D zE2J5BIWSkFvcMop`W2-|T;?Dbdi)>2yqyrP*EALJoR+O(tI_y+yf=h17c{}1EH9S~ znr`FX9ET4SEjI$8PRX@mT%PqwntFEAQ!_!b-pqn4q$+5rg~{pYO8IC*TI65_OnG9) zHJ^bv@LU5W8x!#?=xeZ|loh;KTj`bF*X{fi-pM1)orJe6$7n>Ym!vs3DX2M)Aw|pe z9CwnW0;$ zW;OrM%-RD|b7jk`OQHp5TT)7mG|&L3s**Vqxw~_FGFyMZ?JSSA{yHfdVSHK1S}DYF3hijNEHeQ5h+XQ;)h0> ztTLc@v^;G2y4WIb43hkWbt3#`Y^9UKAcXs7!9S}mlUle2q0~T5WO7{;iV5J;yQe(~3N38FL z>U|l_){Gn8g+SLHxcwv058UO)so{b+qj z<$-dl8`5@LjE>97ee9#kf|wIIM`F(VjbmCb{SLo8cl(+FKF9n6q?wPde+cA1+7fIysh~(y+A_Vb z`US8y0I$g3fv=K?n$>>*^gGD4vUsRR4Ct_rycZc)1y_#jV!*E}pM0LSG2FC|M?$FO zOdtH1W1q19o2?88B(pS>waLnJj9~Ekq9$*M(PKmgS#^AquVSIy`)+$DBI6G!E^D(s z5Gb2JBWi%}tLY97b3oOxKOGAeUb{_jSPm3&SmTj6B7J)>g)a3=fEmSCy*P;$2=7-*=>uD>e3?&0(W&MW}>2biraZsryk8BAV_=Blbw zSy*98Nb$`s!_}B1L;{-F8=#W)=W+3Im?o%llGiypgEs*v zU_J2mEzgLzgptnIEv+xxMW_2wJpJT45P}nZuB6&$3tTvXp2pcN?Jn|1jyI|;cLnRGt85{!xs&Dop&+Ytf{o|yn_N$-l6?cCmdzl&KeW$zBkQ>XEcNHt~SmyTuN9J1ec&L{0~oIdR2UStSUsFpVO+;9Ay-#W@nJm4M; z2EBI_lkoqkxS%&Cyp(1p5G&GLs~1!v(Z$z&QAKASH~AY_#Kw@Z+A=EElU72XOQFK( ze%tO;@67@!quk(+b;9Vhsh3H-Y_nnPE7@*uhTI(pif;1B<3vGBkPEJT@8Z;(4>szn2};Pv#9S3-f*7IfMGAsLK-rnP+5_&Fes(lK*!8}tONz6n zL){i}ahUs!yD|06kRjNm>w}wrjYGd?l@y0TRtji&aIV-0+Nk*TOw7{jA3$zH{ZC)H zwt@3Jv*~V;WIlztP(vimmR#&ZcgBlv-VKXhuwVyZX4Mv&<5EG#JbFx?_f@!v7iA!uh4&~i;!J_mv^|KOq zFYbW*5z^pgCs|oSoOL31_?S!>G-6OW_&qNUEnApDIo#Xog~#X2)k z3}OrDd?j|Yb#rt!R+3hx4>0UD)?DG{Uv6yD7ou3eA@jL5NCZEnMpB9#_r2-W5t=}x)TyUbyn2dCM44wWz?d@ZeE6H7=(<_-i4gDPF_-Bw>ReNMyK zs>aMBkRb#KSxY0eLzEUt%) zK_m(k5HdN>#g=>A#^!B}!;_9XG{`>p(f!Nj{s#j#U&-?5C^jG6hEqo5BDZCLFIDuW z1=jbZVtYk09i%y&w62^3qPdJU+3ZS*iPW$h4pYHzf+hY!HJ1jxEm?tntkel$8{+F= zG*SrRe0qB_Bz^_``+8z>PK$^VfU8Y?I_@@>2F9zkI5K3NQeJ;dD)Tm2-!`_^gp8WY zvxT};^Or&%D*5TEyd)q5zTHxs(xB=g*tGQ%G})$B8C)hI&yq?OaF%4{PjT>ZK-M~` zw>HpJ__eNEtqKE`o&WNNw3D0c{DRAxCVJRh6HiN=eT)74-oLh!)^unw60w{d;AuLu zl;zy^ciUisO){f}nPloMi{G{cQ%vhW0A7$n3`jR((+Z47h-Pc|ANj76)OWr=e*F(X z_oIC9s?lPxFe zKCg4=tN>BjJ_RWssIle!<_;6*sytGEx3~0Na{Y-(Vlim2)0O5#_vGa{UB-_Ne{}it z26A7XL1Cp5!K9(<_Ff#M zP{Sn<^_V;W!IU{{>KKp{w4%hyRwOYF4|o$os}`@(J3y+%MSJU*65qx_{th*O1qrnuFwv3&%qyrf&nN8jA182qkN}Q532dE>_a&_Xip80cjc3aW$sy|5P2e%T= zW?OSdzZ6?DmnwSQiTR2pQ8*aUuz-?DM=OSJ(%B+)kqv(=kEEX`a% zh;Z&%DE`WUm@lB3;VwAPphThEd}W?yk#|Rhi{X6%&6@>-{4JYz?IBY#zxX)IG*Z0xTg;DH(kRf_3WY_>NUx?=vSx-#6s`kYVfjVWK9! zs`SZ*6g1T)R&KPs1Oy*5UT?wu@S}oWy}ZgZ`YuLV4iD_?{Z+?T6>`M|tkq|A@Hnk^ z(GZfO!)VS1iY*b0f%?%WBx-gcOVgiu1tF1>9hox?^DZ@N*o2_`P@XfgIMJsl_$X1vHK3rI_ zvGIP!hz6M5H7373d#w6dTgbMX2WIUZV{r#-VU?%VG*Ibm-d2hLFMn`-zqSYYLGgYJVQ_n2h(Vp*+PS!$(D z(D0Iir}kv6GwT{5vm1mA?tc{zYs>rjd)b2kQR-#97Qfy2lhaDEU)ow^=bi86@wH^2kR(S#J9}DcvelvVjN)?9Rt_P|5%xIH<}pI? zd+%jGw(gN1gFks=Sk6GrR*oa2gnMpPf>EDyrx$+CxTHU>X_AeaFw?qaQ6`seN)`p9 z-g?}bnl?}FBE#p;lpnt2(f}`%V0!B{gGcX}Kot2rn*L5oy-|$_<7Zw*OQgeP-;!7 z_OYg!iXz*(=*@8k64mV^36^~3!Pz=K0kP$H8|ie3@piqLm)D>8RY%9PjGfzxlzW+B zrS|En10?MWM{D>Z&!*Td1FYG^ciSxt+I7mE0d}scD2s>(`q0k9?FH690MSKWfuc!; z4|Uhz>%ch@TmOO!TSvR>pm)Ab1LXYvTwh_btp$@B^-l(&e#{7wqle%c`jgYY;1L#t zOkq&a!lBK;q<`U{+W?s zM)>eQfGm7{QS@t8qg>k-KI9=RGH$78{h`~&PwB9?QnJ9|60TH9N6h4O9e^5^O>bd#G- zwy6T_*qW00+6?8un5%Da@jdPl`{RZctBq$QtP%r(H@Z`p(l^A=+vH+7)nA0(*c)>= zq<&DTe!8RnlEe7@WccR|qKq9Ug%!7U68*B>(oK#m-*}TsiLKk+=A>Xsw%>`f&wW2oLL6=_B{{Y`%OgHa|S^*=W zJhKUHlfng)c8y%I>8r~M+ko~s{AX=)m+_})XhzlB3X`w;L3mvQ9%LAUcDX(clB1Z% zCreW_gH-2hN5D;iqjoQXr5*feM)3z0)y(g$PVjHbHNk$vRUu;FwEL(AA96V3Bs`&* z(!+T@gVKTeGx_9S zZsFRgQkPrhNW+h|Dlf{*C4f}Lr3Wl8288^nGJLrmnv40qgYouPMfw z9!yshE<~Cd5Az{i*i$93%nTyd(?Us8xLc_^u0h)gjE?>7H3 zc-&qnpp)5Q1ogHMS(6iW3kFSuFl*LPC3STTh(9>XZG%VFwxUI$?efTBe7NsD^+b6z z)o1Hc$@u37W1oZ_tE2+&6qgR^6(ZbILNd2QDR&d}(!y{U5X_~%bhU@Gf~Zarbxip3hpB{CQfhS~UhD5Y6NkfF@S7HS#Z6#9y@pkZXwR`yUctnuH0 zDDxAP$mGpR$C!}v{%TtbvfKCU{%V9P-2>nBCSkAqxT~kE-bYAUC@2wb<)*?g_cKU( z7K{rN?`D&LF-mHTMtvP>k}I$E>UfG`Gg6P+UbhCyA!akR0z`>)EGSpz;!+kFo&|K{Y^_U>retJDJPK!7=y)q0%4PHwiEnQUcI1)dtqILNTu z=pyuIdq#LUgb@Q>1>qQR0RfgDYcPX-Ph*A=Mz7W%b|X1Xcgrh`-tLSTDdNfT!$S?4 zXVH&5QnP3oh%cc^Zzawd)A~gyYwQkgmR6TL7F2LjNj>v7bBf4>)-Byz*`!gkKLjoc zRQriDipKo8q&>AJyhW4sKCMRxH8XfYqmOF|t92GRJePx>@U^!`Gz^g{ir?thqT8sI zZHUCjY5t24-5Ae)KHRY0A zw%DXe+vRR%w$2rJCF_PI)*=gQ17fPTKu&pf3M?F-qY)QS>`lt%X>jG;A-Z<|{Ht}or6E+6pg66;Bolg0>VA$h)ICihV0Ieu8V(bcQMRo_QGC%wooMo<=Tcs@i*pDXs3&^_=488yp| z-R)HsB$g;KTaFmPWOTIchS7z6WZ$+ka_RP}cIAmS z(6y$~aa2CjitE7Y0K@y-V*m(1e##WwL((_4Jbm9f^>NAV z%!Ov_Ms`R1=jUVM>mh!-ZF733x4D|ttoEc6T*LroM6LI$a>j94hx2Pjt*{!tiz7ZZ z0JZ@;f?-bnO3;PgFu)tiGu8LlcnQ}?nZ=T1B|z_v|K?n;iOD42Wq42y(QbVRU9wZR zvo8sdO>y}6sh*O;01Ar26m2v##b|N+ON>bw*nX?+^NI8%4_~`IHwxi<`{KJB8RNJ} z!OQEbKYK7cfLS9^58UUjTgjEZ6!asvB%`{3!CyDJ^#3Y=81`R8mP~#&u}^1bbDlW)Y#J@bqP{ zPM>=#%^z^OJXFlMA?to4*?*V~QuvSu%CfhQZq}rqX_EM^GJ_AEH{yCW+3h*yd|(zC z8%#FMl1P&4{h<^@J8bSb1Wjug+5+Qe;4{X;O=m9(>u{U2Bg**(;o3hlPYj<20o+Ie z9$&G!4CaZSkL*oOdL=4$y~WA6{yUpoyCG#$F9gw&sAy>Ckw4hL;5>G`Ea&~k%&(#- z>EQ<@f-_Xvk1HmL&>5s{bW%uj9(hbIv5HH`@#&p%6vkgHTrN5IlID!*Kb6?uzXQ$j z-^w{un3ueN{vp-W4kiQoJ{crXHgEhqVS<_s8hV3X)70daEzi?mK4Ke$7g8TcniV~n zAEpGqkRFOV8nozX68`F!HxO|6fzS4t{|rtUwXM9S(CY{{nQ+-K2-&p2X`v=6xfnOB zPCqbS|K+MUuO?oop!*x;>}rd?3$vt z>l$*br7501Isd8{PMnV+^Kwj*O$ zGKOlH>2BZDq14X}myoqgd&+?ym3yf^6%>62C~Cb^VFQY<`^P50)q*J~on@);o>6TL z@izg94H@o8J%WQniAKe6Sm5w@R1fUPb$;fjy?PNG?C)ZB- zLUS&5V=IIBQLZuHCT^9Aj8{g1xb`lGsV!n7R6in2xWYVM5tB-hiQav^Co}cS-iRr+ zEnfJPrRHBYrIkgy!g{uAL-XmmO>`4ri?$1}HR44EzFEmCW$bFVBnp?Vr}$k(z~w^F zZTXc#FjWi2l(x>isO*>v_DNr&ndQQsX9!R=x|yL@)*-nM)MLZeI8J=frNbc<4%k5 zEoum>tw%6UhsL&tYHl|}o=E@NNcegYELk4-^4>`ksk67YsTbg=y--LoTI^UeZ}Rbq zUW>WZVp3YBieJn`a7>Es?~B`)f0iR=-SY#!{R2d+c9JyUY7|;00z%y%WKPH`tz42!|+c!NfKE0AR zo6Hlh%SVjR+{}rn8W$>MZ+*X5|D08xvoE^#2{6pMKh9j<_zwWhFh0HYDTN0 zp3`9;?t*y+b2K2AnW7n2$R`TI6V(|6=7)&vzt7$IA)Q-SE}8urBRO-sp3X1Pjbl)1 zFKN==^8GzoP^j^`S92EVyAA*WGc_pjGrxGrlS6AM8_{;?yxG7L<#29q?Kh~N!|JcX z_8>V)Azz9FRlfJXxq^Lly>JSCOJw?7s#9_o?i#(yl`(B`uxB;-u=C7 z;7n}-DJN5%FgN;yv&4H)3FJRJ*_+$!vy4O_Vv^s_!T;(jv8sh;;?S=Mc#xKo_1+a&4|Wf1}pxD@pNcVvm{(cuqV7v z2uzaT(;S)9o8u(%r#;y><7s}D)3OwD=lr*7QUXWM-kS^*WJr%5JO zCiR@cKOc2gX?}gUTa8PXr6lKe@)2VFQwb_pV2_dPI#`T&`IS6BY(})PyoYxrS^@x( z+RW}2dt~PUCSX{!$aL*WE0C$aR7p#Vk1~0%#eVAfbbKuj0`l9Nj|vP|I0oxP)Pjdgs#dO9IWAW!xa&mRcbvB zY((lExz>EOJqK_c74#L)nZ;-kRKdT5;QO49W{4vmGTN_D$m#GTI1!; z7k@>j$0?pEF{F(n^`wo0IlFm3^@6tzLQ8H*+Z(3tCtb1F?OZeF(3{PjPcf>2UVi+n z=x$(2m)$MOjEA8Hp&_16Fnc0Ka+a|_Zv1CTAX2^NH`#Zj`aeLFLwJ|M#h`JcL(xlH zmDdl$-59_@A7!G6Fe-OUOMIy$KDad?INRbH6IX6xl71_Q9-q4XYryx_k4tyvGkxf1 zOAp=iYl9-cM;p71A3R_n?p{)P896##2l&fhwFbGOG6_A{m+s%$jwe*gIi{aFJgMJN zB*RO%7y~hk#oFPdAM4`yZq<69+MkZvQ}Uz)oUIXKE`LAU)E<`@3q(uLeW30GX#*CX z@82D4_CgIod7B*L@ArOV!oQl}*ePO9ZYx!HC}?HvPg z4i0VxI3hNS&#U|rp~nn*#T;rD6UI_ zQtU3`rk#nFb*K7l3rfX(vZgt)@DE}`A$qjHk~NS=n84F#kC29T7I3GQq>|@pUQ)f5 z!>!@b7V7E6oE=JWO})6i=wd~8&x{pNVBM5K$j<=&pEKW3;x&Avy69}%iBOXe`)i6N zjk8)6?ZRxR#J*{0GDj`}v!e_04;nTpM;=6X2eCD2*XsKRAZq{x^2y}5cw>S30D#AY z`dC9;IH!07gCax7&>!Q6g%8QKC&shaW1%73hAwDPK3Vi<1`Y`{!M7|lx)BJ@PbD~e zvC8Ild*Ja4BI^tap|Ft-dD`GyW$twUNKSC^*d&M1nBgoS->%)x+-$jL$(sJv!2`!8 zi$;bvP#^ZA1TI}rQv2p&lebIUh4&nbJ0&j8u6V*{bwFZCD(Oo63R>A$FtaL(A#s_I zcDeD{j=v;{rQlM!2Afh^P+3bsRT9~Uaq7SVSZ}Syq@J)!{`%#CMj&UBOhXn+Di)9M zpz(EY4-@BOQ=Y6Le4GsP>LScBJKS(3R@2d)((YX@)dqc;IVRb;v?295F!IDg3R9~t znOMXBI_qLEBoi6>ILOK65?nf{&?Nz*pFJ5U@-t?6K{z1Wd0(|gruirzl<^1Yr4Ae zG}Cq*W$*ykdVWWFm-fzQCcKu7EcG;h+Rzj@n{3CmdyK!+?}B7vOI9ACz-N@&tZ#^w zTYPQ*LelHxKD5-_VEC3wa&;-6oBi z-r)D#bN#44)vVo2r1xXGRDRDV;Ju~r4A?rFX82eL@MxUPorQtiTx9bfAl@J#rku3A z%FhF&T*2?%qy76?i%R{HUfFy^*X)!q;(%`r*SuDrpJq?hk#5xGptCh*g{Wsz@pF zt+3Y={Dd!dEOq9fn#-Q)q23?CVU6hKkdeT^QN5F***U>V%OqhJtAPA1`>k|8n)T#w zm3}|31^TS??zUO@Z;A8I@H=6zx*7erqcR9myUf^5m;;h$kzE4 zbc{K|;_=Qb{hK^h!`&nb1;)Nax+Bjpc{XtVDVaq8Y6)y!b3Sd$2baE4Q~MHU{ixC# z*Q04j=03A%W5Y+}dDbHK8STSRSrKeANqWOz{{VJc^T_@VNz~iy-W29n4tTjO#Ep$t zb;dE7KvjJXBO4qT(ILSAwkW=iacD#Np~GqQ26fCykjaJy-p8q`xV6xmPQcaFb@FW zOkNrbyruH@^{ote<5uZYsitZ}7sH*fnOsgoG&$`QE~d7?wflYd z_oBZ3Byi>O8Zw~4zFdi3JF}>~&}D9r>%BH~Mrq^>%JUoCIvxc1ozeq-ZK}zWsjoc0 zho4j^T;$oa(F0-?H*h*gd=XF4#wz@@3eVMg#z3@1`Y^G42 zleK-%l5qV@o2RuiL$>xB>$M7?Q5KE};X?g29~;JbkY%QN2;VuCOm&b9`!k~xjz_qR4H0yQ-5NjKS<*LH4PP&N@xqxC>zHk**s=H6}*>2h@4&A zKkHz*7OMw@wrrD4*6ho2&ZJ%b1K69|b)GCy=@!#Dz2;m|G5fJ{t5&%``XuCnffyWm zgG_Y&(U!dAH~wC7PNv++H=9OGjEqx4YPk?${N93`nRZOJ`DR8!bSzge8alY5MpU#_ z+06yFa`cURMI_I1e&}!m`Q|TeU8aSZ<(v7{`jo4Ae}GJS`kYp`K0`8;z+7tOS;e>( z!n~OHN>ubgtTs=`JoTVN&k5pBUOm=f`_5!^9Li>%Dbn}e=%wu-k^K1isGMb$<8ytl zOFPD`P71}L?tI|>&2ICGm|icg5k6O0fSw$)qtnx`v)Wj_>B@2&x-Xp~rYbCzhaNV0 zBy_!=A$AA8wyb(g+s5m#cN4tK`$9yI?Rn*6}Oa%CM$-m{QpW zgYgs84~rI8Art#XFF7e~y(4gN&VZrc+08VrNn~muWM;zxU%w!%hIRO4*rw9xm+?p#Fd zKvd z{ZbI$sa@=$H2-0TUjJ(LZg3gujb&>?Vb5>fEQxUd$yb0Z@99Mn-|4x;bt~PKKNeP6H8_B~UOyOmIEyg<$wFFkP#6&Rl3ZNU&VRsl?1*`Op;hxi%wAct|#)DX=qz zE+)FBzt&6{wYsRBV&`c`v&zn>vfSmvST~n&n?k)Y<{2AR{;I0Z#P~pK`{HlOxD;*p z!1|6Koa0Ka^`7u14Taym%802hTDE86pJbVK=C}4vUffsXGz>n(mY{i0aEwP1Wc~MT zXg5eiDscfW7j8WFSrVFAvy4R<3rM3Z$ujrDIv+<7?cOtmto;K#mDVBFcI#pb4XRS~XyV`icz%9h zAj0zv^1?P{*)}BWd*3?HLsiPA%}z>louFf5S3d%B-i;Pe?uU?-qpz5AZ&JDi;+e_3 zyUYBLJ=wXRqU-2?(b#VfzE9_VzHSpd88D~kupyOMPtx3>ck$o}@AT(xnKvxr0^TgX zy?DH0yi!4(RS$jtnoJ=L8F^&xTzlWQhA+S>$Q8v^$4|;x*O5jj!GP<51P2D>{ypSM zSn%4+;)(ita_P+BEaJKne4%oCDOG8GI*G|JID%a>XLm?VLY*JErsgv#KCr#gb5wNX zqLXA?vz^Kzoc{pbf^DpuU-X%Y{1CcRr5@ee)6T`sNy;o(aea0&nkvVVbIDHs0H;bR z;MrOcMC?5XV|*sQ`H_5-WLv0?AsXw0o8KS{daIlZ>1OKLOVgsA-uJw5(pcuG&ZV8c zEa$90$~~ICy(QSmS`CGB69~&v8I*Ub#x(zBzMm+>K9u{}=6!mP_{-)qRWnZ}yP$G^ z4fHY^WU_(=eyk6oT2~v@Wv`ca^OrcW&{88;Y?IhmQSp~55S)%d42b=RR)0aufOS1* zwmy8!^$4{6WR3Cf=C|3PM~tN%695{fLWE2wPj)vox044ieNt_5fs^u>kiw2zR(Ys1 z=&jwZ@1YE)th=V4H-_7$8z+JPQE@c^SgpZ?Y0% zD0~M3@!{P%*o6S4U(Y>=j11#(bK`G8{?@_qOt?^}RQ>D#?X}<4nR8B(^r_ zJ&L4o{!ng%Io+PGe5%Mc<}HhA83#u=M~rYfo(j_zUQ>0u`R;D-sBOzxHdaiR0hjo7 zyB{=9=QJh$qz%h@!`V~0wPDsvF^2Zgb=@Aynl#8WvM!W**vXAZ_rq@Kj1C0^>7c}V zo>@Fv7<`_f!qP#ZUiBs{9gj8)Fo1<$hqKjmZ~WWT!P>F&MD1LbY~dHVxoOL><^5Ql zV{U9>dE3)$>ZgzFdv2s`hZd$4`{Rg8!c@3Y3AWQx%I|4!$+P<*qH^TYYNbmR`ftBa z&clwa->~>6D-uW5Iai5-!_j?uD$zz*$f)nx5VE0_qfv&gXBAG%b zq0++CZn`WSW3mqap3|nZ8LUsAS2>cUkSba!^@=1Ndnciio!iR)APTy{IXQKjSrlCK} z_TQPOMMjK-Meh;fqzi)~@V!+ahH-15Eif2`zv&xy@$0Xe$rfh{#Zq0?GG5O>>6d7P z1BX0&80lN_y(8D0R@HYd<2qYTcc7kChK}aesl%eCnVsBF(X+=B!I{ZaKG+1;ZnBo1 zshWz45_vGfg4a7xm*Ff$Hm`aLl`Te%0I_6gD&WDK@m$O<#nQO+d4-S*1?QQ3qNP-8 zaydjwPlLIaiL>LU|4|z#l--@xzwD`Ny$BArQCv@$GETbW+{rEI9`>h8>42hb2jwZ} z-#Ot^KV#I|uI0-a10;)emXd8-lp7cnF2)f30q8#_=w3I_vtG~hbW!Z?Izm_QdM9Fq zyc{ZvM-1Kl_5d}mV@b&PAx*LtS6Y_x)25nXf4(2M(A|zy>qqV0tZP96MUe_OeM!B8T{-k%dU$=^^ zj0ct!B<#rd2n0A#zJT zaTWahmN6hiruwI;87Fmqz;CEFg?MQx%E-(U(tds0UO_un9t~$%+KvmK!(rk%iiveg z$7+b|9+NXQ_`0;TWNIb&ctEB_2QS5u(Zqfr;BRQ!yVbJk{pKIk>J#jyF5vvIfZS<1 z;m?1+FgD$LcRG{}eivQUDmQM)5@ZsMOpCmk_#L2on#2~`0~*Gy`2Gmu^rZWt-cxEefXiMN3@)cq4i&)_2=qYIc#+hgQCGs7C4q~mp?;EaQ`*Jq*G-rJ&viMw%woPYZ3 z=(X?{hu9D-5a-neF}x3<$MA~$wiBz2Drhh#C zRqZl(Cq&(YYKI*T(TcBg;Efj?TCb-;%||1#+4Y#7DX7mWgn|g;7|(B|aysNPO*sNH z)1VdfP1k^Q7bHETeEO(0QN9{@UQT|)89Z~a4ASH|s?}3Iayn$B{K`S1zKL&zei_Nz zr^E-Q+9~pw-X%ZT^Y(wWnh5W%?8K5j2_$Fviuxx=_$6l*nuN6g0iFd^{h##f71LoX z6dj=QN2Pm(j8=vb5LL!2+oM7X9;XAF=Fsr%ci_T>#k?dEeJe-99|+doASH_=^~kSb zpGCH~P(p%1sjW09q5+ivtp1}-ROU$A>oer#Z$r(lJ`q^U3Ega@k?GBO@5G;i7LwV@ zKE?>zdZvG+eJL%JY1IZXgP%&~Ja=tkH1`rJ{nMVDR`a0;8(79O_pW|({88{E-V)U% z0yE^XJ9-Qn^POHrLcvcy=b*2oKWYeUpjn+&a0#!PbwvcR+;P0M@Snx7`KElmo%)E7g1h;+Sl#lo#g&40=`- zV&xNDJk>gvXK&-JZrVF$D0$jQ72$p;)~CL|WZrNI2OM$!b6wBG4-`dl1DK`cjAJL( zyvF6-=4nXkbDvsNY4RcBD*IVny`<`zj+-X^I{H{xup~z0ZpTcHhv;k3{vCW|@kWJv z5!U24(83SduogKj*pPCp7bQjy%1$y#1xY0NZIp#)!yE(Jx~&~ch1z`xj3x}_Mc&GXO`=6mdvv2KAb*LiKX4j3$M%c1USQfw>h9D9- z9Y#lb`k%so6g2DQj@MkYK3t#$kc_Y3M;-ba{B`i}h;<(h-bH<5bkR*2%yK7GIL1f5 z3Ff|n_(l6}c(cRCJv+qnCDcWZ$H-4DcITbVf^rDY9k{Q>xQD~cc2?bd73$Kp?e>?d zS~YH}-L2oquY#+GSBJeH#ebRlT-r61Ln&bH%1v()19b!$k9vEy)GLOOj7c`uDW;GRDUyk&lG zyd!ZBg>Kw5l1&A-nQtxt;KjXGGDjtlh53H(B%fOzDDd=B28&c!qihtx1bdO6TGPAM z{5x}Cvfb)SBn3)O&mL0(vT#4xX8dto5zVuVs|`8j_2mgAb!Vm5c>FJ}jWbGEsUq;7M2)qp?DHw7`?+RL~hz2BmxxV5PimRTuryaaisXsWz>EscuLv= zawIyOlPEa#hBxEikf`B|g)pUCV1SkOm$%Do#BP0R(Y^J!=n8 z(KO5W+5XId$jZLa&!;1&<6o@cJXpZIJY9SnlWTa+%GO;RW#;$mwwnHj9v{rAQBzf+ zCiPZLD?ghtV)$#JU3nH(zaIV@$t}aGA|Df#JC986h;8@IF^%OH>A0+_E>9oQz3=0+F9k z1`oK!XJ{TA(&e1XELo075=4+OCE_Xx1RUUlw3F0__av&`C{Tr1&kKjqZl?LK^N%Ow z&pNJ4pSp^l?&6B!gx#Je}(PsL&2t_6Cp8_k)vh*0A!HGi0{eo+W=L)TflFn z-kn~y*ty=5b?mJy$P}RIUtX!FHo>!7U?j*rr!~(>0##ij?_!6jP*1a}0FQ1P$C6nHfWd>^Q-)vU-GCP_gaa|4hF;~WE= zo^y`Eyaz+@7scIsQ=d}M{9*9=_e+@^tdnVht{^0ks>>$evIBxZB=rCsfPI6*nmcHo z7x6cYd{wB)JbGn?)R7yAQZ{K!Mk{#ss8s~9&HxxJymLlvjgDJ^=XU08CFa|*{LaiZ zI+bh2EpPaz|`a{7YeZ;|nGGb z#L5;ar6xWG0opJL86Ytk$9{20FRgyV99L5!Tfi{#6N4cG7#Zw4pTn+drSFa9dnrzb zWX}w7kY*!nNq|mSjsPB{n)-~N89W|iMz?0Jyq^C6f%8r9H$pbjR_>D@mr(&Ev=gFozLf7k0-@f!q&t>a*axM zakcF&?!T=%8rt;Qo{uCic?+ypF^O#?kZm%=%C3*MB&aCpz#hyi8hiOwC1LZO&_$lPb-QC>N$_&)1M z)Lt2E=2(_zSR|G*42lji4gfr3r>%Nf2N&V7c(!3i_OD3=GxZS_AX8iA^tbN8& zhldj9}nI62711!!C8f-nhQfM&CNU2N@aM?YLw*r&}K zoz5;Tqr|==TsUSYJxRxd_ygSfo1!udLGoTXwnlJsVDqtpYV(7lYGl++>W3cHy=~1 zu=s;-<%CG^$BgvH6|JW0Z+$AZ;e+k%n)BGS>)k$87ciW0$r!GeN4XK;atG!9d;WDY zM@AlPO&#U6^6UVgetT8hO-&haNaLxlK{ZW;65YpRR&DjHU;;Sp>CgWFUa?!A^lHUN zQ?ZWzCCS_mx$ZqGwyl2|w=54noyBwi0NaFs1f56YP~B?#;b2=Fae>qPKl;>J+T>2O z8nw*bPWdhGjM2Bu4^006^{c!1U-2^6O<^6xY(YE$)p64q>;C}PuPW82cbS-a{&hcw zbq0d$D`W%E=luTwpRH*bxaxURglb9IpHpeS7rZ{O2V&dt!ue+){c%uD@kc|I%#xl@ zAQUJ1SB&Y}*^NLl1{`{E{(r)+F0!+})FUU<{*}6vWMS<2C8_FCd{EI4B!`BS=!~1x`BXElfk6v>vYW%rE4>^e~rEn4Ug?H z*Sfc_(y!WlVDP-C3hEip*B{Ef4^h$XB9y#G803s~sqAj1g#t*Y^Q>xfMe1onojJ>M z=oo%1cx;^gyaWFC1Kibx_?_Xhq>IG=0KW14Yr$>xB~mipz3STPfDOU$IPcf#?NfJS zrdWDi&vB39hlci&Z&N)-&yFfsekFKrVY}_hSHJH+`t{*2tSMY&eQFnlCdSvxJ;F`jVR2&i)1ab3m{{ZTxQf)}AJqS)WZ2AZO7C(oWg11qSLF6#O{b-lR z+5>VflNsc1ALCvZZ>j(W<{h!xrfpU=0Ouneu~E*=-X4i-Gq5KQ=4ZwBL)md5mW0X9Jwq z$UYUb)vbV7u1HWuIXrt;T9-QgtYL-Ya0fzbt~rXPl4cJCN^{dK zT~>qQ`+L~H5`&(&HSrXG5d2M}&K2Fr-PrnnI`mJ6e;sF7Om`BnVbnKX4R%t;#%)~2 zrV3Gw&!=_&02JF<+8IQgW1efpJ}>-j)3nPuQSVi74szqtxc>l%zqB+uew8$m-T=E! zM&Vx~{7Lw|ulV;@kejz#P~;?P#BS%QtFMZTW1-0!QoW9c<3Gh2JaMWurG$l|4ofg3 z=Le7Rt|C7V-Ccam8=s)-?_Dp$j|h!oMUr(5o^#31QrYf5#R^HvVps2%sIw7Q-V!p7S301;l+DLhQ5YWtvh`kmh2!I8SD>F@UEiE z!$Mni1vx!RjMHzG8PlTX&K4~mec_@`eT8mmHdl}VnA~81cs&0ApU%786TnF#hA;vT zU@N1K;rwZ80^4(p^#1_sR#LARvqgoRp2wW6lT3qp^N#+%j~%Kzg}Ap&FJ8wdr&{hc z9|<+OGYIj4$II9xg+te+d}xi;;lN`J3D*Vf;ClH&7H)N8B}9A3fnSq*Kb<$x#dVDJ4Od#*FRrM zl{E)gw;rRmde@te$ugYN{pyOJowe<{)a0v#!`A*JQ$B<6&+Vb(%@0o+Md-WKVNgEJ z=4zf)kz+h>5jICo3jK4^y)VPRwhfnwtPYib;w^hjyhxa}#+__0EuAt3Ss!CBQII>>;m;1@DO(JkDJ$xm zl76a9?d0RWy$)~eN)^?qw;#vz{{Vn`kH>F_SDqpk-xz#7uC3FfMPuR%xL{--X+#SZ z!^XswS)))HPC$}2!6d1r@DIe#5BTOE?2E9L+K5cOscfu+AATZ!@IIYKzB5>!2=Lai zV{s*pqv78xLGfEjVRLdV!GYdZP^FoE0&Z*)@zApHVHjt{0amfG!fB*yk0Iys6o~1Ue zsf|fBC?U7|5}mzlLyK32Y~dGn_mc zy}iQc{sa`@{uTL!Y2$G?nbfON@om{zw{F_M(EI9md`=FMrA83tR^98e*GqQabLmfq zzqBvFuM&7NKZoBLB>R2MfOOM!Ed{>PGQ}z!k^l=EZXolxg(MtffKLH?@VnvuuLN^j-`HC@&P}EAI_I_l zWp2leX0$Z#4EWPRw^Xq4MB0sFj1k zB;;;oT&T`L$S1xlw9&t4{{RvG&en8*&S7BD|~O{{Y255%`)bZ9VRrQORaZ@x}Ir(!7)ie{}Y!LmUzSAOblZ$E{*(zB2fAlW(e7K&0n}BRzd9 zjM4PZk9w{6dwn_d`*HrVOUcLG9Gs%CBoW6-?5ynk3*ZZLsOtX!*>K&SS>yR|an3SJ zAE9DBvTO97Huk+b9>Ebn%B?tG`v7DWtM8?hKC1cT{UW58Z964`3^mYO>V z9Yy{04JtgPNMjsvM$Ut91Ovx$fsA;5t@}XuPR%#PcXsGo0z{2~J$M9>$Kh49zm8rP z)Nhq7CceG7lj8}ot|fjgd;9Qsw-N1OHH4xwDVh@m-aUcEW%iY zjJCJsjk#BS@An>)sr)LrOASj_@aKrM?M~ZEfo8gB1hC2FmQ~&sSl4h2z#I~KR|Dcp zWov$Ux%x3uAD9+5ubEH>s?7iu8T|Y6Z;+DP=Uv?EeCX!2he{4umxUDn$e2Olpr<~oIm((hW;M@GQy)Bgb1 zUrl(<(%MK_Q9;N!;=Ci`CV=r?wDAsi9!ICWX+p2Hj;0uje#&|ryI)~;$m1s;#oC3x zXolg)9N-?c3Td*tZgGtC2DY@%1nSz1Alg~GWE^35HN0VS309ijlvh<^I5{VSNcx!N zer%o&2V?xJWxgADngu3JDewb1(Djhe$TFw3N7Iyg@<0Av#o|&j|Dm5C^+OgDe z&UpilY0a)Y7a;-U`}OCK(ydwe1H=&VC85Y1VSyh*gG|(X3*tdIku>H#e~4qZy%B&q z+LvtnP*eqT*YN!+@Ambr%tR6n?ttSpf_xp~RT(;apQ8`)p?o#sbsTB35&$Ie&+F-1 zDLCk4=XZ2pxm3B%Kse}6HB#eDgoCpHjDiUEsqHk+5X8Y|)1)IfB!ixQl{;ygz0LyZ z(&KyaQHpM9gdW_>WMtuV6 zB2-4%9g5UVqF<0o+d0VOs3-YUsM}&)No>m+$BHGphv#xnZhL=|RUp*aww>e>F~O=j zhLx#8;Z3BxdaX9r%TQtof${DMsFYiJl^U)qLtQnEqYTHV=kNULL#zTgWB3}WCX;Z; zujLRvovDLNxKV){WOXC8K2Z9^ueoY%S!6;tbsn8R$E9Ul-KPF~_2ac#8f@TgDtdRQ zF7&xdZYX`vy&R~YYTn`{@KJzAxw#MD2aD|A*Pd{43(L618e=3!aspkT>lSpk-Q&riJPr2{Y{QFm{ zd_C|jlE%B>8RI$i#ct{)8@73#%2Jh(&xJl2HZer9!a>OE{(bA&EOh&Gr;MoM&{kH3 z;j2g_MT#&wbgrc7rWgt;k~Z=W)3tFsvwEI+<0v3iV5&LDuQu^lmTk{H z2dTw(zY{e$>_mv?ZZp)^l4{-|TaCzHoRN;FHPJ~m9Z_$t!nrEpx94iVJOrY4N$FZX9EKFs zqggl&-2VWa*2?`)K53qd@Y_Q2(I#Z%@;J?U_JM5)v=O=u*zN`maNZ1oB8onp0T>|m z@925wxUT;IPl7^YXqXUv_^v!eQ{3;tR8dVcOG)u|zu>odG&`(D=Q3MKmWj>?3{>&B zXNDY(#8(UB4-0%H@sYUIFWLM#sc>YL$8?cgG0x&*$IH+km4{yDyIoSzlF!VPbF_BP z{{X9{c_+m9ZevqCFS8lx-`D*A0Q#%I%y>&G&QqOexXbyX?DxN#eRofD&&=rI@$Z(R z`L3U*;Bj6e(jd^T#kH2Km+>@#BWq}6&nE*QsK;J3s09*@m=(FaZk3s2K!sUK9P3{snwt@fMk+{3`Khh_s8zE~Xldo%N(5NbIA9r8iNS zBUNHb`F>L|ZOlGmGHYgU+NyUVo&J7ewz964ja4qC9%`Dd9+2UYiIf%pxfsn%_ z?#KjGvH10^_-6LN_$%S}hqR40+DnNpG~`+4OaA~qBNU4}%Ogn1<&vRQlrC~Ia7XC; zFOhL_$$9DOerR*t=q`_KZ@}b>F2c}h<@_7H{7DEQ6skHvA^ zXt3VL43foc+Qyv?#@UFL^>YfkyU*X*@4G&ft zjB#5@Bgt_v+mAANWN8Xu3?h)DB#=o2LEuk`{weUEid)9IN?NtVk14o@Sqn!pNJ(^H zr#L4BV1ft-ITg*sxepLg%ibdyH_X>8Zq}al?Cs>Yzf;uUn5@SgT80(cT#c_^cm5dq zRQ|-jv;P2%J|JA;4l#`NhN}k z%*Q0Xr+xKx4;Ag!Buj8T(;sjb>M(8)Hm$BNrcABFsDkK+NU{29`)sCF3a-OFEoBp zlT)LLs~U3A$=d7I-8Vc>QvI2=JwL?LMXUJY!fieo&z#oq%#*g$f}t5U2IC)ijPuV* z^>2i~8a@g9B-U+o7<@RM9BBuTFJy>G<__jdj0}^>IUsfb007I@FX3fm4Z-L-^Xc2) zpU`&TJWb)-ZEfU^@CS|fD#L<*&N_NmwI2>Lj9X5%YL{MdO8xE1et|9?!ctd+rTBl5 z^sk6MHh7=I5?lC_!+#PqKN4u>;IcNCa2o0u_Z+LHK43a!eL7&*v-~&xpgcM86G(-j zzKxuq-T?!gdVfm%@X=q!9}KR3-Qdl4VtHmaw7i99iPJgScLGP$9xJ2Lzi2y6H&RVf z-$uF6wFCvPJc$$RQxM<4S+W}h91+;`>t7YbTmhM7N!7ys@|F60Nq>25<#oI0k3PO% zTL~vP>mSGFeckah;a|gFjeifXJ{)-e078N6CjS7BJKK0sZY(6~B#H5rJ&5Q>1ad+A z#r>wgWxt6(voDO=!^IX4E{}I}FZLd@8!3+RB|H<1?R*eQkO=L?eXa0|_N?&6v1cBU zt!oPv#{d|vP@z6bH%g>SCtw~9RLaVhyoA%RCjlpu_T z3<&oIxp-5J^DZ96a$;MlMZbABcD&J;pB`AE% zFg${JKj-{w*t|2ZTMIDDD99eQ(tL6M0D@(B-@!M{;%|Uga%);CW@)W1rvCs-w8*(^lb_+6~t6}D)7u6gUB^kv^ z4(Rm#cj3;39-0_is3mipf1KBbd|U8Uv>Tq?04Ojq*k}I$u4~$L&x(2!+E@v63EPfH zuOac5iS2c}vuO)wE&XfJtm7CqXGuBDBa6{I1h=F+rg-4jQ>S<;EifxX4@?dRO6WW< z;p8^z+lF?EyQnOhdK@5bIp-XESC1T5HA^16@O-qnmgE{QhvK~fOlzL}S6`xhG1D)Q z3F8_11JC1JCYR$WEHBIqgeG&GjN|gJOYo15tYwXS$#H?yu&-8(6IVQ%ji&BqYySWY zw0juKMGBG!KOn9y_eQX_xiGvcc{sv~?|f%=!;w6R-2*>!>ig>5v9kd#&s@q6H zVF9I+lyZ?sKFWU>0Fn9zA0(f z8x*-xaezl(YV>KmZKvF6M{KK|xxpUaO5Q0rlg-NYJZHuK01YfGFFfes7zYe7$N9xB zi{V{6ZsBT#bR&_+<@Bu|h<-7%gLH;P;GO}j9}oOP(xX6Tmuc#LUY^yAtrlw+&W$Zg z!CGVF{gpTzasL3;s&;-5(;57~vo3RhK>q;iRm+cvdK`#%IVagq^2JuN@z#a*)jnK-_XbKHo~Sd+LpkTe)s{8UD5D+GX==+HaY-g5x=_A)CXJ*#U4Zm=(bw174T# zzg3F*$XYYu)C_~c{{TPdwQ(rIqw_jBr%fDh#d{0QG5q4>jO3o!?OYUEt2~=av6_{cGC1 z4|8s|$~Xi8o^#K7@J|n3n+t_G$Oi_!FTwsVf*qzwc;k)L&slRu$n2p}DQbIeiFs-y zS8NPox=Ai=#gk_k#xY(+2ac@OGaTP{1oQs@)~nW^6s=~3pptQs#tG-GJPcPoq=OkG zqjYoM9c_{0mgWLT!8rUqE9b`4rSq9(Cp(V>V!f;4Uy0$ridYfDZU=4%@5Ok<#+M>Y zOM&weRYnduBl(YS)K{TdK{aE~grxT?T-_3-gLds%SCUTevu>{l`uf)?bE5>Dl!#+IV~q1t%%!l>nr2j2UoSDY zeLB=y%DWOQ~fevwub*G;UxgZ!pq2Q3`I^}6ejWJg3z(vY zN5k|WkHZz~T0XxVR_iEmcm}?3k4nGOW)~J@p;M3sde?KH_}cOdHHzXw8An3Lt|}^1 zRShW4RJTX6>wYV>n@!8eG6ziiSCD*7@r|3>yfI0*o&e7^nc|;_7mWwm?cXKF0R-1C z`wsyxqmNQ+mcxXFMNtTb!!cOxkkQYjyK1=syrd5l07% zt>d+4J7d3(BHmo|$vH5$vH4Ftj@^Z^iO}QbQZw`iq55NxdLG8P4GZkkS

+*(@=Z zBtoAtU`9B|0FQ3I+>F`N{5N%EM7{*^OmM@<%nyYo0LHbQ`$f*6+0Gre!1Snrt#W?~nF+u6<5GrorO>01?QB^7F#R%Goek z?{9!|Gr$U|kWZ#_U#{?mI}WlsP}b`G`?K+GIi)I`OIm4JJ6rBYSZ1^q?>zI~`9Ngm z_Cb^A03+J1==x30rD(I^Z6Y}<n&SIf@ubyY%Z=5_O#6)k(Ip~OzmF1jz?S? z$=3EmRrU~Ur%9&I21P^|_>dbn5zO${Xhya4wNL-FS zzs|Vdjea4v)BG8IJ-Y>oZsSDf0GxyPACP~MU#;SDsNS2K=KbX-_f9|Nei@Tuo6Tq_ zt9#4yDc*RVJDEbn*eBno6@N(6;?%a2AUVc4$K_r*VetwhI7Xav-~2tR(mVy?l$`?o zpy7x(&lwrV9=w|RiuBYYc%$~LHBNc^$3@~V30uW#%-G3Zn6Df0{nVNj(URIm$j;zK z06g)>QT=~2?K;M+w>HNzgM~eF$F+Fx#EEULZp;dsnT`|=NhgeR(y_xgdo!mHq04iY zw)oYl**@F57_QX0W_cuGAUMVqyK*zg^{+_yN&8~z{uO5O^$)SEWqi198KvGbNdb{Y zF~?pqMR`E*<)rHGBqtUkGOKahaMLfm9AKm`|>w4FPns7%AX57-L zOO|i#MYQ+rtbDbL)8u^hY`+asPEl5ub^TlLKSIA}@7njm{x83@veoq&E##Uo@J}>* zcVuK_`Vo)In)*)DR=c(k-dSGD51s}ljc^zrR~>!5`;6D*SB2)f@aglkso5lX7tM{~ zQl(d?Cy~h`jAOB{sQ&O(|T9bU~Nq4vAkGB%rQ@C}X`%dAVTZW!PAq1Yp zog~T4}fE?K)+h zwxE{_6f6clb!Pr8-z#B&#(P)M-a7C-rkA8#UTRu%g9!pi#6U1P0~k2L$*-FHSMb+W z@mGqS?xvCLmUsIsZ5o2ANWeJjz#Y#@{e!~npNK2s-X@Nib#*)ScF`>!skpbconFQ` zOiZU&-CpW?`RTXcW6aNtwemK$uQ3GmE!!2b;XP&EVI|GUE>BWx*EAp5b56a1UTZA( zB!kUqTYk&hWvsj7wAs!=larsWIsB{kikM38-DviyR;L+U%(Wa!zxMX1YlJ1-_qbT_HU<9l!e3$au5D8jYpz+3g?^sa>qAoDw+CB;cO-HO=tkKs(0wxyag zxrfYfO>!F7!F!u+MENC`amO9H_O6*ytwGtIW1MEjT==(BzK_bD_^>@eKHVz<{@M$Y zU9d^&bDHY({{V%z$aXn%+me4u(}&>PmjGp04EpA!%bRgDk>q!}TZV;DSswW)XEi_LQ2z{*iqjia2Na&Ryy@ca|J zx#x2bPZ*^6lDZw^)a12a6Y9FGo^9>fC(|dIj?-J392RnM(BywA?qA>zrW7(r_`%@y z2BH4|g#)TE+OFrGa6ixaH8zvn6yl6cKVC^VcJ{~MfBk=jUlT(q+>Qn?LGM>v;C`*A z$L2JWbjZhg&WFK%AddI!sIC_{2M0e&mYUrc3Z7=IkBW7jHX^o{SpJy)RZm*+zl*M| z+7`SePhVVmb*+mZh29`wyTf~N+t#fA0K%%)T;-!~zo(^A=SsoNO`cV#t?ZK-r)FYK zNj<8Qzg&a$uTjbq~g}2()twiy*e4~1hR5V zAK{{`TX=g~wu{P)F&W6@(NUmW_IWg0Yn>m?FfQ%6$QE94z{>xY!pOgd5aQ-OqG?Hs^y`$p|lY`gRxN!968R~c9XHHPlbJ{fzF8fU_E95et zdhqWVSS6OXA;9F42=wN?7SF}6E|Q4M2gnCrhaXDwpNLo9b@E9b4o(gaLtToF!%+8h%8&4q- zq^*B`9_MmPZb4|=5yvfRi<&Q3Ff)0)dII!;LdfO#UeG&XrI6lWZOK_}NVr6yb& zPUgOq;oF;xY==DhR_FX8bEE=RhmJVxdHgGP!j?heKow6PEeit6Fg66V#Rc)%cj zbkuv7RA5R#C=FqM_lA zUOm^XUD^nnmgAgpSza5r1O{M8&p76_w=4spgnAC+58 zIy_*<(DTJ;O=skg>W2rDpY!SXEXjBTz>p9dLVhubR!WyhFu2ZXa7F3M1Dqa}lcMXZtZBBI^mgOU2W0RS1gr9_K7i)6-qJ4+!Xit{NF-mH>gC8XM;JK? z80(Y4Bz7Qr=kabWp-z@&szYhTGt}TLMOwK-r+v%f6 zmQLc9(tkaVkK_6_&s40H;A#z z894-idmRHng3C>dLeXbKE1~;M)J#Z;PB&qXC6DnrJDA`M3<1pPs|BsJIzEYQ9%oXh z@RF*?4?9m)g>p$_gCRNYG2Ljg+G#onSuZzCgsIt=s!pgqn>(hVBe?9dyH0>_Z-)@nn#j+>>R_sacPCu1PuUUDvUuTYOii8;g z@tp7lOQ0l{I#=3mvFA;hBrJaJc^N-SgIU#*%Sw`H(7Up+On^xsXE^DB#sTU`9A>^B zC&kMHno+Kv(qHg@27bq%U@2x91|tn6q@eBUmDl2tgR5LMf-i^klZ&gBCRqpkvO&RP z@1K+pT>k)evTyte;qMk{Z{gi_K!Ia2`NR|DxoxQA9vO12kVbgmsphsceKO=}@2lTT zsXf>YB+TC|NCyZBx1}T#X}VUU=6$K+jzAQGk*O>}$mqaGN8?}6 z*^D8G!AVKmnQ8ib?_>E72QOiZPkTLnTc0QVMbq^U1$h4eRPf#WDHOJM@-Enq$X9VH z%6q9?5$r{F{{Rd1`C3GhNls21aly|$Pbc-R@Ak>~cy;dq>Aw>ooBc0Jw^Y1A7$#Vq zkwX#>RdRh!8_*1Rhr`biMQwQ;Qob>P{!Mn|xYe0u>zQ)O4Z=$IvyHmhUh2>1Yuz8X zQk_3%8kC>9(rNh~`G2WRC5b2rz$B0|G1DjUHRT>W)!>T!85m@?NdBMC^RBL67~30A zBXP?0Bxk3+d6&dJOiy|xOpG7BfKEDa4;AV|4xz1$Wm(Iqx29|McPwO zl6w7XVk^5A0HY~591i~g=N0DqHm0$VW+bRnfUE&0@dv*?{Wz~k_z`$ zsi{cPn~k+ET9^Y5GXNf>uOMcujY8T7Qdk~y^yjT?O`+So1je9Xbj4%q3j(*A@r-4T2!r`f5 z94W;m%rfzV`(o`7OkL30hgN=7#uSoD+*BiB9&B)`gv90h}XsVCUnv-oJ_Lbn< zF#<7&V`%MMhl~6zX1by(#yM_A1}o64ZxCtgh97Wu_O1iPy1JcpQ+6=@KhOE%zIQi* zjNXS{9p1&O9|KuPs>U&c+ZDa0cnZ*{{o3Q)RGI@rb8fB=Bns%P5mx{&PPO4Eqp`uW z9#4nX%^uMR@6I~c5Ag%Qkii?`5y(6huVqOi36?!8i1?9l34oJ>+Zm!t$j_CgcqBd$ zjde1poaA)vU6+RZ5fP9(rx+badYeYkzTK(rMF*3|<6W#8Q`xG5xE{dqT*|V!Zp9rx zz`Awe^4c&z3e&atQKz;6+dyGGw%w#Q3M- zO{_Mdk`d|Bxf@@F*6*pNARND3b+2UcR;gyzSZBdKc@@fOS5P*q8Qd|yj+ms}Z>UBx zNNhg|ti`f7kOmYUm0IuNwX^_(5cC_Rd$gBa%|S23?$+1-+m$~YJUH7>}w z^E1aE;f!%HL{owO$fCUlSS1c|f<19XE@W$De7_fhyj^W?wc$D2?t9fm_R zIbg^xAm;06*u_zNOK=AKD1Sj~K=|;~AvZv@1(E$W&2}Zq(%#E_rTzss8{9 zU&WD~tr$JW9Mjj}--`BU$j>0GqF47Ihn*wF*e z2OLnNbzyTo&zj}3B*my@-({%+&F_KRl@OxL&nzn^y9ib6Kqp0Vt zdC!P+XraH)l)RpxQnj=+i<8uc!5SvH9>!>L1~b8~>d#ERP#$Oq$QzC;Tj6Gj4eV&s zs3hYgjOR7z*Io^>j@!&gmykPks1-p|s$t<}6GrbhwHE1Y$%CsgpRoju7#g-<<2 zd2W^7jPIeCP5>Q2;=Y=*JSraz>#dAi4_xHttRD?{gr^q3fH=YHUhQM>_Swcn0M@my z!<$_H0LQ2#j^di5HPqGSZS0SfyhGs)eV=)>01TXSRCM1DYsn~4ZNTn3SJt=R2<&dU zzy}_m{c5Xk@HWjQ-Xa+P0CtBB4t0$8`K)~37l(EERBg25-2VXes@z@@@e+;Z%Yn~R zUrt$k9kDS4Qojf%p47Rs+F;v@o$=VcKp}m%sBT4E7HVUUxmNLey2s@s4byMuVwkC zc!u&$SyXgTIswm0(a|S;Yg279RwCBXmTR1J2dby$GJgu?-(O8C+ThOMu25hDxwtGq z^*ufL6|vzfE34bc^c$CW{?Y{cgT~lrkc^1I>HttlJa!%PUtPs8jw2wPBCg}_dRd=6 zz_p$miYQ6FCft7Y{&qU4b)&7^n|MeT(4XEAN6vc>rBKx_TJl$n#9T1JMLhXn;3z!- zW+No<0LkE1jMiFRjiZ#6ZEc`CTt+f8(*T^3PfmxESy0af&E(4xks7jz*eU(m0rv;1 zFeG4P@=53`@g&-5pVKw&qsu3`ld0%FCclBM{7tFDZ47J3NwFMC21|Va=ngZ1)SBdi zM}K-H@Wzch*ukIlsm|A(i2%9JIbsz+86W@v8Lrn>isQu>6KZ}TaR!qFhwS#szzLa7 z3pZi`1Od~G$89wy^3mFllF5uH}#sBgv7u z2=2pzK*$PLu@%4KPl5I~T7?L8I2nW;judieHF0UX~w?T~XfBN-$+u$v%;54@Dfzyin2P>o7yIUU+ zMd1B*2L$QTfsx4v^Y;G$>({3IHG@)))IlY?ft+NHqP~*8_y=T>m0K`4?4qgJd>Mut zovUsE&Ik+F>rRu6%Y{aoJhJ#)+gfEHc0Wv${&}u5P`cD&yb@f&w`+zQ)DgkR&Orom z$UJ)HzMr4r9kSXmFpQD?+Tc8I@V#x=l!pVc#}%aHr1d=~nMqLOS{_9Mi5aGf#X_o( z(V-_K@y2t5&PH>Lci>isgXFrg)*zlhcyJG6U0tVx{>>b)n_P!na-`!2IUkljgwjdj zd95Z{BZq;5!rbVK$j7+m>Q+o(9}tU~$);a&yO8;5>g5 zTHUOQPZ`cVtJpuYzjjk^L0);47@82zoTDwOPwe@@p- zmO?ddLi7T-4;RIq#jS*s4q<)LWd)IxWYjMnq>cHl_3qhLpEl68L$7nsen)aUnY4S;GjWct_eD}1` zGrlrcvD2H<(U3FKpf$mKNVuJ$WnIUf)#$6IM-q;l`kKntJT+;5{<1&QwGybDNs~`w z&Abe=CEzkbLEY5-YtWb^w&7WShw-Z#M}q8a$Z-%D=yp0#V->WS$q>B;u2F$ziOdN@f^<@b+b_-W$dW>PnkBpfj4M>Xr(Z;2q(EtVqVBv;9v8}a3{ z>Y-BSZsrG;>t5mT+g%rT5q|WLae#TJN>b4?n-HL?Hyw{#({v=gA+~4mt)%d^yh*vE z9Xeu<3Yh-PrGV%wV)pGJ7|7srUYseX6N$Y}UsklXvCC{;J-Svs$BXQA(nCt49kW(^ zZ?8cVE<zo0QqBllY~!a;YlF77JG4p9b}Db8QG#0|0P)*R8(Ru?)kIwdqC*QMx?I zUM|LM)%-Ag#DE8>rY^T?i}%;ulU&!0Z{E~5mnJcsR}*{U+vrCV5$G$Dqq8)P+C7@f zTeFe5k$*n59=oqvEwcpLPSc!Mj%hwWx{hG7&%rqBS{ME_62wZ))Oyv$u3MEG%2qS{ zdGSGXT|>!YF}oS*-nIM>@nY8aB1qs9j;6fh#db@t+%&Qb;G9-2krnozb>+h$03I`3 zs}UZ4R(I9HIMP!`zJA#g+DY)2!k$0e8XY>}EbsPpTYHF?_lMVHkjH`O?T)qcW~T&N zCy67~Zj(0mA0)&afZbKI(1LJt=t-}1@ZHt##Qy+?dcVVe6Kc>h+*?ByogK#2#Hx2k z=vW6-&s>fx!L){(SI{Q#j;#?}%K_w(Aj1>~5x3MI-JeV^t$r(v!d0h=#AR5Qt8P$w z+U;{`bi4iOpPJ$!Un|U3A#WLL+RtD3bGp-D()>rKM;X-+NgOx{9DuMqjzHu9XuGhr zwhas11}wRG21Z99{c(ejt$Fr|{h^^>!Ko`WYi}s~Qt~heBXRWvXCu@Tz#_YCVP}@g zSuE7G#HS-^;4gf7ag&<)9E!j8W_Jfuqj7&d572XI$3Ee@oUyc$QuVgK=2o+}g6C03 zo;G-+Q*=Qm9fE>-9s(YD&o$NQH=@VI7FPh3-yD)jjPaG)dwS##tz%mKjw``rj2KHB zB9d|dSk4FL3H-e&^IwbjlJZD{Jn%&t$GDBS1&IW9I0O;xU$Qzh>tQEKTJx<>#_-sh zxJ+d_@X^7hzFM6Is4g`NYilHOs+X-KtTTWC&PnO?_adQ5kJu-kz{p^XpWc}p_mVNd zRLS6ya99DxKqt0doqhJ3J=fPFXN!j$Nl!_798$xl>EGD z_@l&@wi;}cY8T0E0+#aGn-20f9COJ7ILA3S&o#n}-xK(X(nvH|8%MKQnkeLa8Rj|p znTK?& zn_8AzslMB(v;{;2w6Y{*fI~M_PT3?5fSe3hp!jRzUZcu<0$HGv*HH@4 z7EnkkRmRVgI8r$$*1WI6f7tqDCf3eY5nN(PB@>WtPColKcQ84~1dcP1NxuoYd zV-%trc;hQJ00qg&DxedPKp8!GBi^`iykU)9y_!o;>$&P$2BT-LG?Lp}JZg@B37G+Wu6HN|<#-^R4nWQ_GuUX_eXA%|L7MQ5 zLRocfV$SB)-r^`U{759Wo#K;#Mn=v;jGTjv=bG&N8}Vthdw8t%`{{3_zy?!jv6b~C zt2Xao6nmUk>HI6l*=9Ef?DI;hooMxOcArbDT_@Gwm20P$qnl(EC{(@`b1kiHwBGmf z^71`4_eQi-`>IAO0{2d~x0J}Sf!l#vIxmjADdLEb={mekB!ek$a7s*hCt>Dz8 z5UhF6UMfv%RJlv0jacCO)LLLOT%?j5ob)2Fzr6J)rNq&`9$H)|GC&(6JdTyK{{RZ3 zPLO2VFXB4aN8$CfSka|$GuzU-7+X(bBiu4aJu9-EH6?R5Bbd=VA8n;t7-Eb72NAdU!ltJR9bg#5!`U?2XTcZ5~4I z=Q*!bvV=9dagb}rJ{;dc9@ba%?~3)E4^)mwMo8`6s;)K|x}4G2r9xDerh`Jb8D7=G zd`*cy#VBFa8t$jNL7Z0w@e$rzM*#E4#w()-D8{mFm{s^$=AIdhoA->cz(3Ztd|PZ6 zM{q`RIL{SVK(;pegBD(W>r2HmB(_@zKQBOfROeOc)OrsO8ATp(plQ)TsCkZnWwVki z)VwpP+Cgl)WCIn!MQFjxD&XfJVvCJp=F&xq8A#_G5;I>ZDOnwe^|azsHB@* zvw^cV2+zH9z6yd{`&D-&k~ul+S{@|RStAGsW82!fy}!L;t)g5s|^gavnR@Z*5*jS2+WYeXF*N?nI5BiGLAdTW5~m zesFo+UL&jc>gqX3M?AS8=Z~dD;x826+(_}wA?kWpEqyyq2`3%+Bi6bc6BJ~(Xx^L0 zlCVBxgpaR(&ZV;Pq6Q4d8Sk3nTXS`OI}W%R{{RYV%+N3;h{t`^lY=%>is*XAo#N=` z0nzYHU43reNb@v6#Gs9ZUR`&-O89a8b8&>h`w>U-10o3OLnWlKIReXjk&qMU-L?g?% zs?}3xwngHP-H)I3G*_8hc+iN3S^ogwCW_q8+C=&Q{@dw6n+J=p08O}LA)tqkSrpGP19QTj>Qnqrmh~wqYe_U5RrhHU} z*~6QXHuWB#j((ryS2yC%6e@7_ zMD4T@k>B;HRO1v?S<0&1`zKEE1Q#p2a6Ed|KNRb5M{2}4`=nQqcu(TWeT^iNT=d8% zf^pK9#$Oa|#gTtF5=TOR`qi~JW^&0kaNim=hLZL1H*!YS2LSZwcs+PE=1r|?QMnP_ zHwV_YJXx;KtlhCKo@)W&%VduA6Zf6X;qIej!u~GQbo=$0?ii>%f^)}1*QZ+gOWt8Iz3j>jn$I`l=hrTSj@V2`Z*pY%OVb?T8YOYMzHmB*I zfpwT}wCLSr2jv*89d`c!De}qNui;-md?EPJX+E0{FBFZS{p@}f-fDjnt!^#LM?P>p zYtW5o#x^+|UCQ4Obu|z^WXK$Kubh4>>uVIQt0~DO5s_Yz@f+e4eoTbSIvfIe*UrBZ zykGX|-xHkVkLV& zxLVFswcPV@Fj1?f$FunI=_Q1;asmiF>&`VfK;75BQ(BiEFHKI|DU&$^y>t52=fKc8<$tK;{-L!*@_UZc9 zHSy!Z*8UXnjGqxL?Yz5d+jPBXQ~1Q}1d)P2+Gz;%03I=1$AW$(MRv9OR1LH<3#d=d#-S2RvU5`jF8H>9OH4Xh2yNh5ju1+)FF9NdbIY7wY0vv z_D%ec10O@{39ei7CcQg;Cmu9=O;R7V+uR8QE<>Q=NZ10bJ#qj)nDwZ?w{=ehS<7dC z;kVSS{H>PP$tev4XBkqY5O4_q5DE z8gtbr7XJMo$^QV6z<76G(6zg8`Z~&u1c0BlM9M+N2;2b-Fi1S)1JHG=aoF9q+i^O& z#=B!rw1fa>rb6x`+dVs0&x(92EIOPwP-^kp#~fv2fWdG34{cJJJ$ zsrXM>)pextM~Zw!B&}^K$!lteR7Z|><2V3gk~5qRYpC({&)F_#wuB$wwg6F|l#IK5 z2qAO+*{OU>t69RMOtY{zS4>82pA=N&yr$;YG4=7mQo=%|EIMt=S15Pu7~? znA$fEgXUIKkUEot(z0o-&$Gj3bZbRBn=#p3YYe(vHqp)JHr0z@A&*{j`RBGXT_1ux zQ>@tBwx8lJ65QR|+#E)3nPfJ^p3DaFy|I?S&ZBiG{8<%z}0Y#?WWDG^mkk7 z=jd&0HaeV?(sA1E-!_h)#Be-Uzi7!3NbDml%aQ@$kQm?)K?EEScqDUB+FmW76CQAD z)xI45%zqJepAW9BuK11(FVBINEjIGm=XQ;UNjAC|^Zn3rGuVz3UHyrECS2IIwWaIY zRlHHl<^7-c#uLEg$Pz~xJr`&lL1WUtW5_&4!D67P=3l#-`$;}m`MAHsM((sn=NXsw zej60&(x{sG_uBse(mc#~%`DDM&HyBJ&MN+e@dhiq#fff)cp2STpAkQ2e~aD|K>DVE zEXwliS*!z0nZY3LQMGs-2?TMFE6?m1d@Fk-*SdtcjrK^sM3_d-HkM!p;(90~p1{}N z;5dse$0YFeViHV#HpHb4N zr;Jwa12&m-mr<$`OfQxI4oOpnP&<+U9qXFB@k(6k#zQH_dgi*bI@KXfCl;<4S~8`H zbm}WfZjQr2)T7dMHH9VlbAg_juT$`jtg_&lI^(^3_cw?ZJIL+?gMvA)O7ItqVwN>l zlN~dYS>=hH)A8oFFE5$UqPJ1UKGn`@bNM%D z&Hdqmc&St7jqY>Rha}oPZ{d_c+oGO@MmX(VhO3~%cN(_Pa6LNLoP0U)U8E4l6S&Cg zGhU5r;+rXA7;t|OzW7qKFz1PIr)uX`WNH|*X zKZ+N~^G@n`AoV|;dKg=&Yq6YS_iHWcSAy_peQ|742fzl#T}=;F3A#@&5olYL2Yy1bELY?|>@aEZdn4BmXS0Idb{{SApm2LR*!QuQlJgFZdGDjWw=DDxg z?@~*RN?S)fw4VN%uV(QMjF);8(QF?w9;4f}d@gO6)WYU*k4IzCrAbbe`kyCFaA1+6 zU~;77XT3CEoaI4MM+XD)tG+qYA4~BS)uS&P@&rTRvNmqyh##eLkn~72zH{@pPBB zGAEpKxD0ScIn8&!6ZKZ|q}wKX;CKH3J*&jEo5q&;i#C2^N;XSFIHJlAG!|pddB>(H zh6OwjNIfunzwDwn*xII@O3HUoRuD z^sPIsA_YU`zsx(HhwD^W1(B2>8$ca<@$j5hODP&XtK7_aCqY#Mmp#6tH0SkYXpQ6`040qYdJLQjJX9# z#z3xv!l`L3(2>9=wrMujr6sM4egW6@&kfA6H<+>HVC3VD->rJKo$(3ewfW{Ck%N=P zanf7B(Qb+q2OU5hpKN+o9K=t&%{p|`KrZbI3V+$mE~HE z>*^CTJ78q+d;V2t#Cqh{SuE!nKmBuAHu9Dmv&}VOB#EuAXR)8pw|%4@zO~qBe;2hI zI0O<)_jtu}7|QM4$vrVpNQWeX0QJXeQi^+`l$>@t-xB!x>t7yH%G*d8&m8yZo->|% z_2W3G{43&{y$bClSx(Y<0Q9b2CS*NoG;D z9wqT`dlUQNjz}5c9AiJH>0TM(9}(PIW-m1GI*jK6x9@yYJoctC`8esH)}l#Vl^sma zA9%8PE*?1MCzHtjb;!>&*JCou$ByLHty+IJM`;MjKD_5OLeR05QB$>-l_N^9ND2-Q1a?1KZD=)Pqwvp( zZ?vnW#KbR3_HTy%DNSy8G|Q`;B=Rh7;{>QA0zf=uXFU2>&k|~U5~=|{qrc%@XTy&g z-07E)TwBc1%M^jt!3X9CAo4%Pv9C&1s7{*QroQw|O;(59o-4i=_OjSpMkNjANb3ZM z7?KGJ11<&>5C$=lI{{vDJ|kVolSSaa2x=Z6y@o&DMWk69fe-`$(@LPEV}J?CAoX5r z&b%S;)5Sj&G_5mL@h8L$O8FXhOaA~m<)k+t&moHdouSNORIUIUOCAFe_rHg4ycO^V z_G!~T5BQ(M!pa#Dr@gSXjFQ0Id5B$sUzmjes3plp85lME#m8B`BA#HW(xJ~dKW6pO za<_SOZ*39g@YOuiDZx&SIi#mIs#k67`BUxae7oYG*~j*ey3w?s7<^mSbZ-af`r9eB zvx87F-z+YLgyckzfS|76rAZ_Wxjb>VAG0@wJT2lR)I25Qokv*mn=N&yX+LEw{c?e4 zX5)exh$kc!$6rp?d`GPKlJsgCuBR;Qu&it6F=K!L$0U$F26z|%@mTtWhP0{Y=$ep6 zQJtJ_V7EJtKob7K4}Gn`bEuztQa!OAUo9LkglhKGEO)= zXB-kcN7c1sY|!|J!s*exdwFF(X1QC7UErnU%l@c?!p;EP zm^~x_VEy1nHG7+4u=Mb=RbeN~*{}F#^ERw>4KGHBrkwzdNhI|pk8(*lC*0M0)i3-x z>zotF{{TPElTOuI{@`iPFoJYl&u=RN+kd>IaLdj^55xj$<=uqI7-!{ik+E_J$6x15 zRP{UJ>DQtAq^xh|0gx1S+ME0g+<^V+(8O7E_Q;3F}<-*M}vDmWBKF;C9CqpQ-7y z=+XVM?(tJU!?B*5rM{gH#UB{N8tz~u(d^{u3LcX%Jy?!Z ze+a?qMm|CPpYQDT-y2xk>-vh@*_JOYyO0*;R|Fs($s}a;!8otDbT5fEFKHV^I)nfz zQNUg}1JGi)eVq&(6&h;E9rg3yX@Q0wh1Y-U%=L)wFDzkOt%5`!d)RlafA~nW<+Ckr zU-yyiaIL`y83&$!8sk>e%3d+i^rAo*!aC(gJaCqZ5_OCbi#rr?_BjZ+|D&FbQ*y}!C%Y8;@ z_eCJ;Bban_`tmx1*!mIH^xZ<+WyESh7#IVtDRi6Fw3+S0B#{t82kN;UJ;$|l)1^`k zRD`V2wHlQ$l@&@&rna&0UZdeSG>K=>uC(!Cd2MiFGg~4FZKjR+0f_04M>*q>PM&l09zIMp9Jw{XG;?t6>0lQQoH4syp*lF zzQ1#^>v2TqTOx*QLR__DYrg#&efy&`T=;3@3l$oMj$zg8eBFZL!TxYU=a^d$ouHlr z0gkn`;F#{AMO-i=0EQSN*!mu|M@{iwui@*s@AN%JOL?N&Rg?l4=O8kk7=Ud=?jrtIZ{hinsGG^8WxYxV|o~BO!d#&q{*&;^Ih%EKeOhc&%+4LeqRlu1N>499Kzo`#Ahmj@Ef?bT}`T1}iVvFJh2@G0zRiIpZ9k zO6A7m@f_aDlDoHOujT${wTI2HSk?QQiTP~)=c(d8B9c^c=m^JJr={wI%jK8ko_XfC z{BhykM@74q%TckG-gvSkf_RED{XqbZ{Y7z>w&-<)`!mV_MJ{AeK7OPC3BVb^B;?~H z4B&I%aZX99x;4Gee+P~BFtm?G(rpdRu#jMqpyNK(o#K5R<5I|kkT@c?{1qIM#-c;U z2h05{V^q@Z<^*j7k(|`WGLo&zwm9oz9ZHuxs>5B>bjzKOxM1;)_15UW5N^Q3lj+V) zH&yV&26H0E#(gP~X%@EYv_?1`tLf;*HDIJ`7j$WAJ};W;+^A#7C)c%kmyhpcc%*;F zCz|bV6;d)4M?+qJ@djPC(g1J|sIIz6DQFDw{{RqQH`<(*#v46z`2PSZ=dW)c``tPa zdC%6gwfVo&WD4gyk6&!o1^l3?A-^NRu9-2#5M1h#DEXY6bnDZlW?$dC5~a^@593l_ zKH^Cyj(Xy=E~U>0wPM?5k+JAsv(})t{wvzA~}U5fa(0fyoaLH_{iucmZwiB=kpg%pt7sU-k7KGpF2Pk^spyeiXIJFwLw6w^HK z;#b3m)O=_z7A&KZ008yR{{UXPPYnDMGutcq1ZSO~bgsWbv$NN4(&B6$aya*|Uhu|& zrlq2wP2=N0uO&x9bm2hBi!wNbtu*fvj)I`zQKe2@11 zCD6rV5=YIp-T>8AKty75&@W2U)4Ug|K`1V#1A;qaAO5QP1O64=j(CI)c=>bqREyvX zB7f8lI(iB})+%eEFB@~jB=B9_v2GdUej>6iJPD}_qaz6)qUjk}-u&O`+a7J^-)7GAR0jRLSUtxvLdGB9D z{{X_jvj-*tf_eZc$>19SoJIP3(D;s$=ug)5*!j{whq|s#*h>M|3(#b9?an#(_pR+S z!1v8)5CRAUdUmgF{{V%5WsLbF@)YR&4{Z-I#0R!2UlGHi8pc`P$AfDB01x#Bh^Pdd zRXsn!TIHOAL=DGIwe&Oi7TR89h!c(uC_VwS>*Wr8s9Z%3!+6Jij~upbVmnVEp<;I6 z99Bj5g|%%(-Q7+bw%?)e(DB>(SGx>%|}I9X`PEvTDc8 zm){KaIYVM2kV)y*s@wcD)|K)jZoP3|Pd9+<+m9r4>$FsV@UQH<+o!qOI#PT|4#N8D z?0n$A4)s!?fJq~YQSjGOVZt0R=Yxv+Ja`J#NBW_T{8PW-1GVTO1M5QKN^hXPxo(He zQhYSkHxJz$o;rSiN_0OB^;IAgJmc$MN`JxyYDNSj9P|{c;0r)8_H_fU>{0%*Q@+9V z#@X}kpYZEZfaDCE@=aQQ0P2dTEApK2j-tKoJQHZA9!SS=#XR^b0=s;mG3dSj0If{8 zl0Am8j_1v{UkMQ9J;9EnI(2hoPww%5OwBUII z{`u+6CB(2=SWg(=L+51pd8$S*6^|G{;Ze!(<5GM%Mk5_@*1nG(2(-W+Ko~u7_|i9l ztj`7|IsX7=n&Mb?53RfGeAy?%T~G`K;j_z23M&d`^aq|*`E9J(avVU~M z1Gr+Ywegoj@z$*OI={pXHHat5i0UzfT%3{yK*=YP22E(_U$FPbpW24;d};AFUH;9n zwna;A8%&2b+GIEZViGahtOha^NXHiSjguQYfs_RUSgk~!o? zBLsU9ivE1eaqlq2&kvKl45^S65y! zp4WUxJbz$<9ur4)1!h9=zYa~o#Kz!li{|hYSQ?A=KDvuAL{(MOz{GH zm5`P{K;ph_`1kNj#$N{y+q_c@M@|xLk649=*}*3OGaMM$@Bm;kaul{RO{TgY?JgwA z^I9&SH!FQ5r~CuuUli)TA^3TywugBLywj2QS}{_F1HsHqlF5VEk~;Ctbsik}0D=hTJ-Tt$qWFSV^G>+_(r{8WEJO=R;YXK(e=3NobE(M(cYAE!zb%P;aXQiE zl@#UOUw^pfJZJF+H<7gO56&^jlHo@f?g#k(RnJ*o=@CojT|+g&Jh7Rw3H87~&o$}3 z1^t&{)b9*FF!9`0Xb`bWSaKhbHt&$G)6e%$Ls9sr_EzyOi7b=&N5uLzw+T`fIWFV3 zm~qsHZJc{HPo;EKr#v-(VNxwi(f)R^^7y>N6JHeNS4Y~~97KA?mvMTxV&lqK#v@Rt zj(Us?1O651UIP82XYl+ZP1W^dG(ZJOnU3H)2FMu$*R6Qor=#j#9@OX2wYwRuuI;2L zJ-o_8BMzX0{z&$&ui=NnJzK}Ok!re}658l-N$@b~t!)AY;5w$wEOk+=p6m10qVJq|O2`hGlDvc>yPcoSam zwZ*&mo+zaelM}HA2N~*5wRrxi;_nJxXrJ(oL-79qPZlsErHm3m8zcbd1yG?1d+>SZ z*0|X`IpL!(n;(a)kdvR5+$itH00Z={oF@!sxR|+43YV0ncGvy7rEi^?+d{TI$KFYL zzw){1x-aeDCXXDJTIYnWuI=D`u<90a0Ilheaf9?H=}W5q-BRh6&*Cea%{n*_%>{;J zGqK3y7yzEUjy-W+aSfflvUzrzMx3Y)cK-lnz;bm0#D%qh=kLqA4S>CaT?h68K4b5tiCL zcT1nBTT7&tFmwL^O$Ot~(xV>$HArHP`@=sJ^yv}vn{918hfHuk)wAct#q5!yQ|*z^geQ~Us9brIJNzJ%XXUm)vK+<7cQ*gx0(X;f-wDnIv1AnDr}p;*2T7GNS}8szD#?HxjuDLCt*i;_nef4uJL$ z&AI1aDMGj22RR4SWcTzH()=&@>#k`YE4aV#uZZN+Axqg(+RENPJ!c_IV6;pbSdiQj z2~xl)QO$UKRh;8;yl49Zgp`%sX0K;^yGbtjQpEF&;ME(gNZQKQ*ReEGqki%!P^&5QAdG|4t#_Xf z{{U=lHuf7xe0$*8Rw-_bZF{8wBQfNlFicc{aHQdhAdnX$C$ECz;1BH~rmml5;<%;N zWsTV)xwf7wtvwa;QGCFp{pMkf%0jkx6~Xx|KOV}SEi85s(}ne%WzM}?qw?wVxxZq*%io$a5)&Ld_@P)rHejgi{VzR zoa{Ks!RcKuggh(aO*w_L%Hl9V0B7m>SGo@cSdu}Abl?F}N8qa{n<#*KQl5QEO3hpr zHLisphVi|GXl^r}2|oV-=QZlOM~Q9Xja0DU_Tsq99X8(31@iHaZk2f^jd)PQ-#DtJ z#4v`)79~B-@jO{-tAi(AXs!?KZoCGfxBmcG>67i^G(rf-U$54jw&9Ne55u)caJ&)F z^c0|sWG@4$uh0FnvBEKJ)HX=#*NRC%QWaP0=}$uB?hD7|O7U;bip_`n^PMP=R*C`qzrTY zKf;Gr$V$a=Y61CIB7ttCkW;6i=B6WQ0FJ!kfEfIk10M91MPawlXtY6$4%}j#Em3lF zk(^|7sY8K`^x~3Egq&j?bJm4bTHu->XaY9UKQ5H3XghwDVIvNwdY@eT`cvTvxRIQ8 zC2{^VDeH7LnN6&~U_$<4g|rNC2Y!`qLZOLO`i{A!SmPsc0CUbw4ySQlHdp&Hg&-;F zIQ1hx{d!Sogl$2SR<@}HhreoQWGqu}9N^>Lg*9yknNZsN-2epV2a1qdFIHc3kye;T zoQ^)c)3zZSbII@2W|y>$^#txv7RXP^58+Sy5|ViOW~~VE!2td=frGZy9sPRJRS>Vx zs%(Mi0Ozk13k)6?BeAQ(HB>L0;Qmx8Ry{xi=~SeOy^5b>034y?t}EzY_$T+ne*^fV z#QNWeynAbJJ)V@7;?myfLnB-{murwl7^15X2;qvb!B9to6;YC-+b6YqSL|h|K{m6i zcyjwv`z@5Q#c@2~;j)3$Anlee06F6%*O8viAK5iImY(namlF!gGNjY;*XEC=wf_Je z$Esa_Vt8uVEi|iPvfADh3NSO4KBSYLIrSvu)p4fUt+L;V(WL5A*0Algh_)lyu{*iR zW6<^{wQR3dV=zjeD&PU@{&QcPxw*;RdIuz6v@F4WXzC0syT7>4(EU2rjkU^KMrE3O zDaS+8{N}v3;-~Eu@RQ+n(bKgn`-|;OKr(6iQY?=KK_JLkl~;_M0$2hG01!_UKW)#4 znuKx->s?;nVglR6Bry4G2?Lh%I6RVaFaYbGD@ektUhG?bM{o9aE#cj6N6X#leGJ|& zwJ{Mi+1uAAsTII{U-7qvl|<+yFRWQkqW z`CxEdaCZPW&Swux6N;_up&PTzPnu~*cd_DH#r?*!ajAG~!xGJNY}(J0rNt7NBxt{Q zHlP5O2j$>%o_m__d46&&1`RtMgdWda!4Qky84{Ihw(nz z__U)4{hQs()u&{=F5bI!QlnnIMeX*Eo8JEb@IS2xE-rNmBQ~o$MQy;2&K6~cB?J{! zKz6E(3=V*BI#p3YSpTz97I=Osrc*PnX&>Qv}q6sbPR4PeMiO$0NwX-~XFo5v@7z;OkOMd))70}?#T1jc4t+EJ zdr~|Lu$`cNYq7@qT-!q}Z2`^)=|ftoB&U#Xq4&B%u^jn~EqbaPnw+F8o z{{TPdig>aFXB|gc&@if>nLBr<6kPBD7~`L66nlp%GXBmwh40(zL9)g-4D}~J&a^~h z`@Zx`2`W`oenj@`Pm~%(J6r8e#xo6;r#uUt#bT9^XV(DKcRCM%{6Qd(#eM|Sq>ab# z?=3BicTT`?{$jApXBj>Ds|#qshWPlWq$;4A;KgNFELwijD(?EN9=&d@sfAo63d`O| z;k;GwKf@+fzVME(cw>f8=EqcGT0xRZtEnR$xe9UIn&Zxg;|~LP>f6R%9Mgr2iFPHj zx41?~%YtyKcm+c)0R&@?MlwCAt;k|_`j4mQPlYYmJQvn6+*?ecE?|*H9RXG#9;$Mq z3@||ef(Wl08N*G|ZiPuX_LI75+g{10`?mc%amy`?VR~)f%=P~O0sL0b{x*0@El1+7 zhqO3#``IIgx`cF;m^jC8eLh>E2QI4jf65tK~k;BAb>$242%q8pgFI~Gb|QumDHRa;-!Ce>-TEc z%`=@f2_;ckqwH^t-w$-(jNc6P&kfnfZxzJ1;cX=`aD`%76r2vFl~2A%&nG_t>N-8f znW#;tTmj}=%+X7?`@yhr?V9@A!G9lgIW*bii&T;egqkhUJAe<$yz|ryl5x%n>Ugi2 zd~G~eo-@_&Aw#xZQsKuK;K;;(I{M!ZF}R5;*hy&>?|I$XKfxo=!_-ofihh5g&i#mt z40q{A*;$VO=C2r(V1@mAP`ge@QGv#5^)#F6b#mOK_Js0!)NO7Qa!EC!#xgJzlk4kH zvXV|0z%i)JX>C&VSlDGgLe_9-6Vz)&DZpXHN3PyyCoxhN)p%oizC7q z!0mya)YMky+ zdUvTJor%a+$m&g5mrmTowgCsRr50XNjGj+ZLTx)RvuuQ2u^UQqJAeABKerLo9xCE# zdzfU8oMdrB=v?8OB%fZ?r8fRSt1B(oE4*+q-kU7jeq|%n=Bz%LsNPyO54rx7qe^YS zu>+p}0EJ4NknC3%MqiW?J-cF-IeFw0!5km+#a&$-D-W1-%cV&pker_}0c|XqIfS;Pf=HXx?$ilRWg@(?3d0RExHwYD+<_ zcZ0NZ&tF4%u*QocY8%}0znE-*%4AbVAt!7kfmmq~>&#!JOH7?D# z1Hd=}ueP4p+8dGVI{yHZLg~#J%M9b!ucc2s+5(u8Q3h$h)vQ*MX$S*p;QG{Y%^(4ZJQ6Wi#+4+C=57EJ#yflB zn3`J_*arit^rM=3c>}vDwcW!E4Dsk`Bc5%?V_=_py(XOzB}06}gW8roD+B;}Jn=>5 z#jjGFlTLGhKs;Ap@SDa~-VeIEn(jyQCh~+)tYwR^VoLB291gy{E1xzG6o4=tM<=CQ ziqcD2&OT|MrDI^bY9>z+n)#~442Q~jX7X-PgF_;_f38JZ)lYH*i<)q(pQB#>nzkXVpH zgTWw^kT85|&Myb+b2E5{OpeC*#?0Ezo;h`V2Q}rN7Zb<65%`IB zAB!)I(>1JS!ueFHPY?w`0Ote|f&n-@^dO(l{Yn@2F z^tUpgb|1V41-kL{9Wji03eE6@k$9WLaM;T_2pZ*KR)4wJ7Ho5cP^9_+YcG`0<373Q zKj9>{p81WGWI$MmM8IS&a5r`Yj!rlPoM2b${3C?G;qc0HOP*T%wmyH0bIev@jC5(X zsl!B;#l0yPN-G`wacVYYw@s&BGYDniBswkE_n^m z=hqeTxQbGtiIo>CxZ3?5yE}R7-8P+@vG({(ULPAzo|L{_SE)g!fA4!>oa4SKNEj8% zWR6D%fm27M{_o9@l08TA%`jPpaune7$QbtRS#!xg=Ud!@ONIdL!NENG)1tm9AI!Kt zzO`j^;SmNwpQ-Eqf2BF}v~!Vw0-R&|)bmRIVzT*_o_5A@_4cUR?sXmb&w99;atU@T z0!N{z{{UvApDjx8dguC8UZkFZ%HG8>a99FI@bF3LR${fbhfQw~c%#HNnl6TFWIdjX{YjIF3bSoKpc`z0meuGoN-t`75@Nd z{{R|%Uj3qVFARJiu)ox_%?QC~9;#30y3!aG-1??@#;vXDHp1R}!Wuh!c(NEn6j?YK87xW38N%d)>)tT< zk>C%8no!m}b>fRXF4&S5OS#nyj0SM3PUk+#2|kta^z%BHekGg9>bmr=X!FAS-}dk7 ztbHfdF!7~nK_kHQ@7UkP{v`0x)b(hz-wkTz0gp|GRhIhIjyYC#iCPj4In2Y{3i00; z_(R2B0`UaC8u4zEEtTXSF^)_S>Pf?^0z)am!vaCU=OBUVzZyT_r`{0oUC_~f82mTX zb?eK3e$97y!rloo1`4PRBY~U{LL7oKkZWhbemGCtgWyi19<|`nqxj>)l5USqkr`P_ z0?1Q-7)3g(C=5srnaS7*`zpC+eHuSl%rO4NN$B?9WaDg2~0(_q#ZUr_O`&t0u}dPyzg@;>I>y1I9XI*AzmfM#PgvF@DSd=Od@4Dbd}QJbM0>XZt!W zq_lu>j=$2G_H4h(zz)Ei{{WuVZlu!OKFL;Vjj#Chrs~YTAc!`!WVj$~wG0-{P-`G*~sg6T%O1ZFLZMC0nRKWj*j*5#NF{>M2HrOjNDO z%-5B~sryH@&IZxfO`0p6EK6+h#If#%=1i~y8=G?+vEwGCH!@2*OEiq`>`ObGmM5tv z*1FFVd=L1O;H@<3ekAaH)xEm)ol3~5lg8qNh5T>{KJ|m5_-9epbt`=zN`Omsbuz_o z5MaC5oB#_D21x^x=}}JxuyW*$K1Tek{{YMtB~L`GXUxh@M&9wWY{M?xkWPN56v1R-mu!O^j0}!X^{oA@ ztn?!7q9v69!#6|J(ml*$B;CmPE(PgVTZQPnHucRUPqtM-yw3EYHj@0u!61ZSp4sN#x_bRQydGRE2JxcRs|RfT&w zcC>N=KqbH(KV2D^CQ$zS~$-;8|QrEZU;^? zlkM+X#yuH&*p;QX&c!%$o;L7BJ)YJ}Q9O~4l##_OS`sb1a!;??tkOsnK>MryjpRCR1)s+Q;~ z<=2KFvx45@(_XWba;}Q`!}h=&o}=@q>~zC?@WXqmT0kT(^GHYo2abcTanyUz@a;>N zK(a+Eh~NQ)l4m@wF-*3VC1JRp2nRgl{A%6BhuOTyq0}tiHGD@aNJu@w$m$3jd(mqg zO6E6sLzHdcunIBIC{$kf=wnaR6g-Y+RDq7TyT4I&%GweRGAj+6XoRyIp@-> zS!nu(p&H01v^WQHo0QLPPv?w!(`C^$2x4+(v`C3o1kAk%BoWU;gZPRa4KzctDcD<< zjLCZ^&7Pq2{{ZXpPH5qBVCQommC4|Je@b+=H#&M5E_|K430z1Ro-#NDl1_Vi3Ur!2 zt8*F~JJ^~t61?Lm3}ELt$?b#rk5Nn5xoKe}n9nu?BETiXjtI%YsS+sFN&862I4hik zPaYWYEH?rPH1%lInLf}ZjDoq~j(X#_UTAG6RDuI@4Y*}_3M=dk2?HPkGC3GH?U700 zVhpc)=qakK}5-jNET$_>WhqoUH`PCEP<3z{h$XM~#HOZaB%# zf5M*}TA!NAEu1pQQ(=gb81w|=85H!g)x6BfYRTF6LUWH$JN4;Va$j^#x72|eNs+Ev zLIU7}&peOUnz0lT2)AUmSAai1^Y~Ps*?P3F#~r=1x?DRfWJYjE=b^?sV09pjQ<^<$ zH&m9+U>|^5)qw=^GB7^Btz2lh>eDLLWC|{f1SDiJ_M}m8_QpcEBd&NH9DY21I+9_j z!7_P~vhm9#XAQ=3KQGdqWpAqAN|tufyN8!*lMy2w&jXMKPta$zN{7F*`-N*Vw3jzg zL$E?mIo!QJ&nAVwl@LhqFa&Ty_5T1ls~0xfrHs#MbO>B&RSgzR=sm>kG1dL>3uNfbmCr#u6|>a!}yCADWH0?QuNG31=!HV0q@ap4wclrMd%u}NM@a+bz;!(C0Gi(>UTDl^Icr5b|K`EPF0kggOP)g&;x;=TI}V~yh*3&u*c$Ct6O(#Sl-z# z2#onfkIqV|FTpq@5^y=duIJ%L?EPvIwCv9$3_(&#M3vat&V_lO5g%W2f)g>&n3<=9?A{E zPWI+jR$FU!ll69QXLriJvkz8sidF2T_EHZm1m#k8i^@^^uL&HSXEf z6&J0{nPg}s2^b^;g2emMHS`fmix_{KjN>4j@-z8^j@$K*J!zaS;6nK@i9}ZmjbYJ*d%Er`EoXHu z(y$Hjt4wzSFp;#5NdydJiuu+ZBlbPin%`RZh4GJ9@g}vc+lRVu5L%n7t7i&9fg+7t z8DBXTF;5ZHU6U@{F$1>UcMSzhSH2hyM3KoF$8M+`I3WQgm=S=) z@C#rJn)WEQ1dco`r1PBTl6`ou5%~T4cziSX9pg#98rw5j9(YjegQ8h_z`-ofKYJTY^n0TF}j zv&Bz>UbXCMrA!O5*Tq}Gd5n%CTXbCt@OfFXA%W5N7C&(@kQj_v5fr&4q4Ni~h!URpiA zSb?~P1YnGh4u?$CJj)p{auXrrY@VlSQ=eSd+<0a? zLj6sA=lgGX7G>3Rj~!|7{Ij96ydYo(Q@IH}GC^VVQC~-&VckqJs~zI*cH-g9PnmQ* z-@u+Mwein`bl(+Ok>%gnq@_n87>}8Lp@9e7S2_Da_!KzSSq0k`pzSnWl+eBOsB6?7HA>RXTwvZ_k1u7amHgU5BP6t+8=Z-RUuZh~Hi6y_f@gIhCn}ol-j^*vT=oI+cQREy^`%jatCHm6qAy2uaS(6hdAKVto(JY z==Txa+X#@y6bl$+Ibjl&QGj#EBoaB~;~i-e#+To|mg`Hkxj`&*2reg{K^vyxtQn=< z(TF2BBOGLAv-X%}qf8rygP-C4r6#Q0%0Uu{KXBJf;VAd)vSl##$uxpyjq z*N_JQ@tmAs6JiU^QaS$5apbxH%p_z5&&n8p2nVTLk(0^b6dV1!p$eKjy!)kVxtS9N-KT zp<$#@ist?~BoLsERFISkVglM=Leec+-PdTN4U9`T&#g!Rlp-~!5GH? zWMtqBf;xi3xfi}#2hMyk1XRO3ZUcoh~$itK*=PENZJTwHxY=GmpVqoS^{W zatH?`0hP}jB`Eg+WREoG4%)m36>$ufmv2vSE3NdWXcK)@BJvnyeWHBwg6TG+c`YskVpe37(9+8G0Krj zlPdYKsq$uEeozPmf(}kd7~R-lfs=^Z%IZi-YnY^PTYH_%9YEv}!N55H3=$Lq1_o*{ zj9uEfQL;(BNu|{F&|az&PNVml-bTmwtrOY8s)HBl}}AvklC#lBI|^JRFAU>T(FjPz!tg zei%}Ftxb?&9ZQD<& zV_-s@90mbM13AwpXyEZd*R{oxPqui1@I+XLLvtqd0B}I+bAkpi4{i-dEOr)=A(98! zA|$i6caea~7>r;8jQ7CCc@-G9S#~IzGbA~bI1ROdIW7nQj(QS6KEMoGjFY-=@fka7 zdgOj$wyQKTsxUGO)^bz;N`P=dKA->zBRIhtvUsCkHum~NdbQoW@FS#66u914=jOv? zmdgM}a0U)WF@gnzcTmNp-4iGQ!=WXW%8lMx2LmLIhoC1MaVo)WB(Xx5a!0plArPr> zGC?H>1Lna1o&y7tKs9lji*2`eur^KZ%ZQ?$Mhl&-BW0Hg0SAy|1b{~% z61eoKR?^1mNv

1Y>k#dlm{*l?3MmjFK=x2Ot6oB0qTb=6^O#u1^n&d{_)t5^9SK zOw2ZzLjuGw`G`A!BXQspfshYMLE=9Ys;k?0ox;X87MRX{#oSUGYTXdGGZ(CTGBX$hbTL#D&j76cNuY+aXL=dasJEH7}o7i|k@m z5}k*7fIt8eN|03az~}%Y83o)n)>f+0OomoXmmoF(45fj=#^MQVj0_L}!NzGpsMybb zwvfdmLmMQd<0Ue~=Fd481P}nmKm#C}rOgT6<4={}v8f)v;w4gsWh}@^Edd~7CAk~` z6;4ScV+Xk)pT@o*yMeD+teIm4SQ|bD2Lm924gl;22Q`{rI|eeFNtOwZlN6D#1fc|i zcW^LR0&$GbLw^Ody0XXi%bS_mHry#V++@Z<4&O5Tqa2@@SDBq-Yhdu2Q*-Cj;k;6IyOUOH zOWF6a%Z{w@^c#$Ce=DWy`gxxB;r{@G9vCu9_P-YBk=~fWmcz`J>Q=_$q?QS^WBaNX zIp-%eHP)Z-!^48o$HksHxsLX9KV`RzNo$vv8@B+fA^MXIlxBtV0;&=Zsg83%$u<2=@1i2fveCe)(-*u2z7+N~N!f_1hfw;(Fv zMh4-GspJ9)1mM@@Pb|sQT4{a%0Ps(jh9T4VPgl|Y8fjW(oErV5%vbh>$XTP(OVf22H^fUUQCY^tuvirb=3{_YIRp&jt}*6568O*X zp6kT1c%M=Dp|03lNnsK|(Adn<7R&xKlpX~Lh4a`Q}}RitRGk6^2*{g1;PS`b_xa^j?D6RFeGG> zYlOVE0^uz*Ulv-(9sHnfVq|^gK)@I{%Wwh42>_5eIG+N1QQ;2;SsxR8M)4KDoXW{< zrrW{2pi-p%@nQrIr$L^1;8M8pr-8067WU;HW;=}WEUkwGU}pm$4azbI2OMA%U!?H0 zd~PCzT3D)1&~fFHQde50li8(ZveNpV=36?_griY>HnyME)<;ukrd{0K6teNAn5w{G z(l!B(TXp~&umA%%9r1yJbYt**Sxbq0f2B03<_RH{790q2~I z05VQV&PUaIJThOyy}a4zw0YOV$V()#|^BRioF>s#^9ve7hA2IzETi+W!E?p91&?!!P1b9e9q? z%EBTDmf`tk9Dsmjo%qNK3vrM*uLSsm{{RIy@z;!!=STPh;jyRLTeuJ6O<*Bl#1H~2 zA1w*wjm60T@DCO8_r_bV9sEo2UDu0m^)%ODXK5_##mR}Sq2;`>8D#|E4D8D}$tQ8H zW5N1Qj(#9`h8-#?g@iW5ubFzL8+%0~kf<|~M>qhIa87wOzYK5>4rP#zFENs;)~!E= zKQ6D=Pb1T-j;T{fG?Ds^`y2dp@khk(f_57B#7_|}p{!eK7f?xlh>@k(SqWB9ilBgw zqk-w3PHX3{_$Y^j{AuE^k5NhC9}a5QV)sV1l6iF7V=BidnDGGkBV!R9e3O6x$?67& zz@HuXGxmGwz<@v z-ZtlGI6X?>l14saymjX!j1gZmlU2mz{ApVsmr}JEb0u}kxn*samIb#K5Yv~(uhQI?SImsuJf=DESGFC4bn@4!lEiKV0HB;??*tsdXA*jxuvVv`F7KHeM@&BO$%yr#?KlemSsaP zmV^)FNE{9W4wkWmy z)SADS{1J6CEc}@#CA9^O+dEm|A&JgOIUtUwpl`ZK0~|>`-j8*gd;Ls#lHp>J;mF1T z3>T;z^(UU02D&Rhi~1`k+3!3vrOJi8q>2_@vMFGtM-8|!0|NvQ039<{f8k<$E}AB= z)jSyjvRy3Ftakyxku!|A`IummNN<=nGC=~hhFyYss;lOhtdYrBTG(1Vs|KZM2$Y;W zF|wnKs054xFirr*K_}@~VY_`U?mM`qcDfEEf>@QXvVaIVI3xpsk-#|_Jx<-vi9QFk zh2KECj@SkY!d4`p0~rOdI+92u1Oc2k7ObZD6RFw7_B*@lLmY9b5!^W>f^bRdKsX$N zPZd*XhL@ms;rYqnQAZqWj2Hf+0Of>1fio7mn(;lkX5sg%11&71Rrlq*UKIQPYGy7UlM#f_=l+2U92RgsIXm^DUru1!T&STq|-&0D+D$ zNgmm!Us~#RF=|?6>{Hv%7?LTXaLVdNK?HRNt$f4rf5yKU{A;L5;Y+U&S$&Sh@adnm!D#L~$jAa)6eXrCJ9JK2iY7MoSH+1mnJN z3FM%sgsSP&r1?C%{{WFO#8amxlF;{mj6VUqDe;oXKCy82caW~wuPxLTDoOcR9vOKg z4CEe!b6#Jme$Af^{tM|c`0wIhj3m{dum($gHal?&-N*;Zj5jf0dXbPY2|NNSEob(^ z_?=}3*k$k?rLES(zi^Ss!621f_9{+E#sEDIN00c+;!lhGd8wZec+d;l1K^x$vsF5jDSI>!*K=a7SYRU!Gi5`!I6UC;{ftVBo3!ICm>hPPJ-sU zTbx~eN6QJGMOfJbd0}N41&IKhfN((|a6llI2d*$kw&v+A=-6!3yPr00}&dfWUyn zo`;+j!OtRH6y1R70%R2(MtJ9fHV(t!}rn+F1uxY#w%C{W$B_@%#Z-?(VD|$CGP! zv;q{{L%m4J!Q7yR;~b1}jx&-OC=!!W)3Z z#PVRPTltv4yRtdS^b|t^Kri z7e8h)mL>&}aF_>jf-}!N{v34eMHQzkTpohzRhIVp7Hu{;z(!r&k* zRPA?^w$exiutqXK$G1_=a#34GF)1WTXFj2IEJ^YwwhN7dug1>;`{BPj80l0rz^fJo!1$?x2rwH3Q-5-Qu7 zlgn7&YI%)_-H4qKciYh9uTH;hlDBJ5>t}qEpu-kO%_<fAMSrMVSl_;Hrgagn`~akYrvr|^f2sVrAhhatO)@c< zs}GbC1MpP!-TuhN20b&1D5{>474;uBr+eoFOMpafOu>U`3D#z3=DhKZ8rLI zcP5plv$LdOm8X=bCwK!pMo-JQWak^W^rDKN`FK)F!zR~t31?Tjw~3}#-tJtI*#&;~ z0m$kJ&OaU!8=X>EW4%kDzHDcDF94kN#xd6&y=bDgNjYp!QI-?xV%)1GxepUDB?lWo z91LTjAD81(t+mW{Q%^m>X;BP&3guraa>u7)qZsw))S{}(nP@MxI(v;yNM37qfnpA$ z%GgFX9&mDcj0OJ7k_UtI4u-!4eg%)x&4LjVqEk3{t zk^mSy&t9B=lbNofzqMaH-Zz-U%M^k}B}%?CjN|Vq_2BjA0HU7vPq2THdJFFqUBH(x z_@ebzY`H^l-9rqF_83fe&vDZg?k{{vVcU++9^zkOxE8 zu17$428t`w=9QVL9CvZf%OMV@7>MU_QGj;kvOpOe1KXU`v5O5snPfL4NR1&^ln_A# zArjSYe^NM3Yd8?a%6f>(A1bAUQ$uO#&+E{jUC)ZLclnUrOt zMnJ)vB;|=Y7#spQ7#^mIDaLY7ag6@R&@~9uG%_(oY?npA1mQ>=@&_2tVb`^k@J5EM z9f#QCf8Pjl=)W=Dg2dxIo=ETQMHQl)q@CDZvZd9Ih^W$ASYWKXGr&1PyWfvN_;3KH zUwD5{{SkfVlbeS#zqMok5AJiqPB0{F354x?5?6)NP$@te6x@N zBL&7sAY-V&2exsHH~taZS#9OHj^aQSj?mnkfyh3ir|ZQORJPh6Ww(bT3fi0ANZ=1K zg8(-Gc91~po!-25$3qdeHc}yiQ?qKShDG2Ua5&(8C#HIy3MxRIF7Ba-ELSY0R3vhD=n=WnRMJwfgcXri`ts?%$g+h9b| zc$>vh$0g)?+}|XHSi9~l%2m1%jAcQ`sK-j&i{n>}^!rGjb)&6OtC7cG}91)CT z9AJ=nG*Mj?vDms(e3N|!_cbiOIWDLqk5lm66G9bE!k`u`z~r3w4TIEVX9RIudM>x% z4;WlM?Ee4{?M2LYhEVSslE(mUEsT@GoMRarV*t@beKrRXR}j*w@9LNN0yMQxhW-rk z4wE#W*ik^T%qN;yu(B5_z?^~49WqWbaf-v#{ss79$nxQ6(IaNww;wS)0iTx~V1bMR zF_A?T>(;>F>T4(tLBN#o6aayN=?lj0}ie$B$ z%DZ%HhG*VNgSY@e03F%RbHE<-QC~ToVDQ*@Cr&mgC1VaasUJl0HTWdm8Hn&G~e3{Eb`qFqW}g3 WC>&sJ&Iho-=cN=;TP=#Rx&PTnl3ZZ` literal 0 HcmV?d00001 diff --git a/samples/dnn/dnn_model_runner/dnn_conversion/paddlepaddle/paddle_humanseg.py b/samples/dnn/dnn_model_runner/dnn_conversion/paddlepaddle/paddle_humanseg.py new file mode 100644 index 0000000000..e2ef62eade --- /dev/null +++ b/samples/dnn/dnn_model_runner/dnn_conversion/paddlepaddle/paddle_humanseg.py @@ -0,0 +1,112 @@ +import os +import paddlehub.vision.transforms as T +import numpy as np +import cv2 as cv + + +def get_color_map_list(num_classes): + """ + Returns the color map for visualizing the segmentation mask, + which can support arbitrary number of classes. + + Args: + num_classes (int): Number of classes. + + Returns: + (list). The color map. + """ + + num_classes += 1 + color_map = num_classes * [0, 0, 0] + for i in range(0, num_classes): + j = 0 + lab = i + while lab: + color_map[i * 3] |= (((lab >> 0) & 1) << (7 - j)) + color_map[i * 3 + 1] |= (((lab >> 1) & 1) << (7 - j)) + color_map[i * 3 + 2] |= (((lab >> 2) & 1) << (7 - j)) + j += 1 + lab >>= 3 + color_map = color_map[3:] + return color_map + + +def visualize(image, result, save_dir=None, weight=0.6): + """ + Convert predict result to color image, and save added image. + + Args: + image (str): The path of origin image. + result (np.ndarray): The predict result of image. + save_dir (str): The directory for saving visual image. Default: None. + weight (float): The image weight of visual image, and the result weight is (1 - weight). Default: 0.6 + + Returns: + vis_result (np.ndarray): If `save_dir` is None, return the visualized result. + """ + + color_map = get_color_map_list(256) + color_map = [color_map[i:i + 3] for i in range(0, len(color_map), 3)] + color_map = np.array(color_map).astype("uint8") + # Use OpenCV LUT for color mapping + c1 = cv.LUT(result, color_map[:, 0]) + c2 = cv.LUT(result, color_map[:, 1]) + c3 = cv.LUT(result, color_map[:, 2]) + pseudo_img = np.dstack((c1, c2, c3)) + + im = cv.imread(image) + vis_result = cv.addWeighted(im, weight, pseudo_img, 1 - weight, 0) + + if save_dir is not None: + if not os.path.exists(save_dir): + os.makedirs(save_dir) + image_name = os.path.split(image)[-1] + out_path = os.path.join(save_dir, image_name) + cv.imwrite(out_path, vis_result) + else: + return vis_result + + +def preprocess(image_path): + ''' preprocess input image file to np.ndarray + + Args: + image_path(str): Path of input image file + + Returns: + ProcessedImage(numpy.ndarray): A numpy.ndarray + variable which shape is (1, 3, 192, 192) + ''' + transforms = T.Compose([ + T.Resize((192, 192)), + T.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5]) + ], + to_rgb=True) + return np.expand_dims(transforms(image_path), axis=0) + + +if __name__ == '__main__': + img_path = "../../../../data/messi5.jpg" + # load PPSeg Model use cv.dnn + net = cv.dnn.readNetFromONNX('humanseg_hrnet18_tiny.onnx') + # read and preprocess image file + im = preprocess(img_path) + # inference + net.setInput(im) + result = net.forward(['save_infer_model/scale_0.tmp_1']) + # post process + image = cv.imread(img_path) + r, c, _ = image.shape + result = np.argmax(result[0], axis=1).astype(np.uint8) + result = cv.resize(result[0, :, :], + dsize=(c, r), + interpolation=cv.INTER_NEAREST) + + print("grid_image.shape is: ", result.shape) + folder_path = "data" + if not os.path.exists(folder_path): + os.makedirs(folder_path) + file_path = os.path.join(folder_path, '%s.jpg' % "result_test_human") + result_color = visualize(img_path, result) + cv.imwrite(file_path, result_color) + print('%s saved' % file_path) diff --git a/samples/dnn/dnn_model_runner/dnn_conversion/paddlepaddle/paddle_resnet50.py b/samples/dnn/dnn_model_runner/dnn_conversion/paddlepaddle/paddle_resnet50.py index b95ce917e6..90c0d26e15 100644 --- a/samples/dnn/dnn_model_runner/dnn_conversion/paddlepaddle/paddle_resnet50.py +++ b/samples/dnn/dnn_model_runner/dnn_conversion/paddlepaddle/paddle_resnet50.py @@ -16,15 +16,15 @@ def preprocess(image_path): variable which shape is (1, 3, 224, 224) ''' transforms = T.Compose([ - T.Resize((256, 256)), - T.CenterCrop(224), - T.Normalize(mean=[0.485, 0.456, 0.406], - std=[0.229, 0.224, 0.225])], - to_rgb=True) + T.Resize((256, 256)), + T.CenterCrop(224), + T.Normalize(mean=[0.485, 0.456, 0.406], + std=[0.229, 0.224, 0.225])], + to_rgb=True) return np.expand_dims(transforms(image_path), axis=0) -def export_onnx_mobilenetv2(save_path): +def export_onnx_resnet50(save_path): ''' export PaddlePaddle model to ONNX format Args: @@ -35,7 +35,7 @@ def export_onnx_mobilenetv2(save_path): ''' model = hub.Module(name="resnet50_vd_imagenet_ssld") input_spec = paddle.static.InputSpec( - [1, 3, 224, 224], "float32", "image") + [1, 3, 224, 224], "float32", "image") paddle.onnx.export(model, save_path, input_spec=[input_spec], opset_version=10) @@ -45,9 +45,9 @@ if __name__ == '__main__': save_path = './resnet50' image_file = './data/cat.jpg' labels = open('./data/labels.txt').read().strip().split('\n') - model = export_onnx_mobilenetv2(save_path) + model = export_onnx_resnet50(save_path) - # load mobilenetv2 use cv.dnn + # load resnet50 use cv.dnn net = cv.dnn.readNetFromONNX(save_path + '.onnx') # read and preprocess image file im = preprocess(image_file) diff --git a/samples/dnn/dnn_model_runner/dnn_conversion/requirements.txt b/samples/dnn/dnn_model_runner/dnn_conversion/requirements.txt index eb217e27df..6887c2ab2c 100644 --- a/samples/dnn/dnn_model_runner/dnn_conversion/requirements.txt +++ b/samples/dnn/dnn_model_runner/dnn_conversion/requirements.txt @@ -12,3 +12,4 @@ paddlepaddle>=2.0.0 paddlepaddle-gpu>=2.0.0 paddlehub>=2.1.0 paddle2onnx>=0.5.1 +paddleseg>=2.0.0 \ No newline at end of file From 982745fb839ee27dc0e3b9ab45b9b3e9111593bf Mon Sep 17 00:00:00 2001 From: Alexander Panov Date: Tue, 28 Sep 2021 10:30:07 +0300 Subject: [PATCH 196/262] Merge pull request #20735 from AleksandrPanov:radon_checkerboard generate radon checkerboard * added _make_round_rect * added round rect to make_checkerboard_pattern * added markers * update docs * removed links to findChessboardCornersSB() and added checks to markers --- doc/pattern_tools/gen_pattern.py | 97 +++++++++++++++++-- .../camera_calibration_pattern.markdown | 4 + modules/calib3d/include/opencv2/calib3d.hpp | 4 + 3 files changed, 98 insertions(+), 7 deletions(-) diff --git a/doc/pattern_tools/gen_pattern.py b/doc/pattern_tools/gen_pattern.py index a6ffc7ca7e..83ed115d36 100755 --- a/doc/pattern_tools/gen_pattern.py +++ b/doc/pattern_tools/gen_pattern.py @@ -6,13 +6,14 @@ python gen_pattern.py -o out.svg -r 11 -c 8 -T circles -s 20.0 -R 5.0 -u mm -w 2 -o, --output - output file (default out.svg) -r, --rows - pattern rows (default 11) -c, --columns - pattern columns (default 8) --T, --type - type of pattern, circles, acircles, checkerboard (default circles) +-T, --type - type of pattern, circles, acircles, checkerboard, radon_checkerboard (default circles) -s, --square_size - size of squares in pattern (default 20.0) -R, --radius_rate - circles_radius = square_size/radius_rate (default 5.0) -u, --units - mm, inches, px, m (default mm) -w, --page_width - page width in units (default 216) -h, --page_height - page height in units (default 279) -a, --page_size - page size (default A4), supersedes -h -w arguments +-m, --markers - list of cells with markers for the radon checkerboard -H, --help - show help """ @@ -22,7 +23,7 @@ from svgfig import * class PatternMaker: - def __init__(self, cols, rows, output, units, square_size, radius_rate, page_width, page_height): + def __init__(self, cols, rows, output, units, square_size, radius_rate, page_width, page_height, markers): self.cols = cols self.rows = rows self.output = output @@ -31,6 +32,7 @@ class PatternMaker: self.radius_rate = radius_rate self.width = page_width self.height = page_height + self.markers = markers self.g = SVG("g") # the svg group container def make_circles_pattern(self): @@ -70,6 +72,74 @@ class PatternMaker: height=spacing, fill="black", stroke="none") self.g.append(square) + @staticmethod + def _make_round_rect(x, y, diam, corners=("right", "right", "right", "right")): + rad = diam / 2 + cw_point = ((0, 0), (diam, 0), (diam, diam), (0, diam)) + mid_cw_point = ((0, rad), (rad, 0), (diam, rad), (rad, diam)) + res_str = "M{},{} ".format(x + mid_cw_point[0][0], y + mid_cw_point[0][1]) + n = len(cw_point) + for i in range(n): + if corners[i] == "right": + res_str += "L{},{} L{},{} ".format(x + cw_point[i][0], y + cw_point[i][1], + x + mid_cw_point[(i + 1) % n][0], y + mid_cw_point[(i + 1) % n][1]) + elif corners[i] == "round": + res_str += "A{},{} 0,0,1 {},{} ".format(rad, rad, x + mid_cw_point[(i + 1) % n][0], + y + mid_cw_point[(i + 1) % n][1]) + else: + raise TypeError("unknown corner type") + return res_str + + def _get_type(self, x, y): + corners = ["right", "right", "right", "right"] + is_inside = True + if x == 0: + corners[0] = "round" + corners[3] = "round" + is_inside = False + if y == 0: + corners[0] = "round" + corners[1] = "round" + is_inside = False + if x == self.cols - 1: + corners[1] = "round" + corners[2] = "round" + is_inside = False + if y == self.rows - 1: + corners[2] = "round" + corners[3] = "round" + is_inside = False + return corners, is_inside + + def make_radon_checkerboard_pattern(self): + spacing = self.square_size + xspacing = (self.width - self.cols * self.square_size) / 2.0 + yspacing = (self.height - self.rows * self.square_size) / 2.0 + for x in range(0, self.cols): + for y in range(0, self.rows): + if x % 2 == y % 2: + corner_types, is_inside = self._get_type(x, y) + if is_inside: + square = SVG("rect", x=x * spacing + xspacing, y=y * spacing + yspacing, width=spacing, + height=spacing, fill="black", stroke="none") + else: + square = SVG("path", d=self._make_round_rect(x * spacing + xspacing, y * spacing + yspacing, + spacing, corner_types), fill="black", stroke="none") + self.g.append(square) + if self.markers is not None: + r = self.square_size * 0.17 + pattern_width = ((self.cols - 1.0) * spacing) + (2.0 * r) + pattern_height = ((self.rows - 1.0) * spacing) + (2.0 * r) + x_spacing = (self.width - pattern_width) / 2.0 + y_spacing = (self.height - pattern_height) / 2.0 + for x, y in self.markers: + color = "black" + if x % 2 == y % 2: + color = "white" + dot = SVG("circle", cx=(x * spacing) + x_spacing + r, + cy=(y * spacing) + y_spacing + r, r=r, fill=color, stroke="none") + self.g.append(dot) + def save(self): c = canvas(self.g, width="%d%s" % (self.width, self.units), height="%d%s" % (self.height, self.units), viewBox="0 0 %d %d" % (self.width, self.height)) @@ -85,7 +155,7 @@ def main(): type=int) parser.add_argument("-r", "--rows", help="pattern rows", default="11", action="store", dest="rows", type=int) parser.add_argument("-T", "--type", help="type of pattern", default="circles", action="store", dest="p_type", - choices=["circles", "acircles", "checkerboard"]) + choices=["circles", "acircles", "checkerboard", "radon_checkerboard"]) parser.add_argument("-u", "--units", help="length unit", default="mm", action="store", dest="units", choices=["mm", "inches", "px", "m"]) parser.add_argument("-s", "--square_size", help="size of squares in pattern", default="20.0", action="store", @@ -96,8 +166,12 @@ def main(): dest="page_width", type=float) parser.add_argument("-h", "--page_height", help="page height in units", default=argparse.SUPPRESS, action="store", dest="page_height", type=float) - parser.add_argument("-a", "--page_size", help="page size, superseded if -h and -w are set", default="A4", action="store", - dest="page_size", choices=["A0", "A1", "A2", "A3", "A4", "A5"]) + parser.add_argument("-a", "--page_size", help="page size, superseded if -h and -w are set", default="A4", + action="store", dest="page_size", choices=["A0", "A1", "A2", "A3", "A4", "A5"]) + parser.add_argument("-m", "--markers", help="list of cells with markers for the radon checkerboard. Marker " + "coordinates as list of numbers: -m 1 2 3 4 means markers in cells " + "[1, 2] and [3, 4]", + action="store", dest="markers", nargs="+", type=int) args = parser.parse_args() show_help = args.show_help @@ -121,10 +195,19 @@ def main(): "A5": [148, 210]} page_width = page_sizes[page_size][0] page_height = page_sizes[page_size][1] - pm = PatternMaker(columns, rows, output, units, square_size, radius_rate, page_width, page_height) + if len(args.markers) % 2 == 1: + raise ValueError("The length of the markers array={} must be even".format(len(args.markers))) + markers = set() + for x, y in zip(args.markers[::2], args.markers[1::2]): + if x in range(0, columns) and y in range(0, rows): + markers.add((x, y)) + else: + raise ValueError("The marker {},{} is outside the checkerboard".format(x, y)) + + pm = PatternMaker(columns, rows, output, units, square_size, radius_rate, page_width, page_height, markers) # dict for easy lookup of pattern type mp = {"circles": pm.make_circles_pattern, "acircles": pm.make_acircles_pattern, - "checkerboard": pm.make_checkerboard_pattern} + "checkerboard": pm.make_checkerboard_pattern, "radon_checkerboard": pm.make_radon_checkerboard_pattern} mp[p_type]() # this should save pattern to output pm.save() diff --git a/doc/tutorials/calib3d/camera_calibration_pattern/camera_calibration_pattern.markdown b/doc/tutorials/calib3d/camera_calibration_pattern/camera_calibration_pattern.markdown index c87f9f95f8..b13f09756b 100644 --- a/doc/tutorials/calib3d/camera_calibration_pattern/camera_calibration_pattern.markdown +++ b/doc/tutorials/calib3d/camera_calibration_pattern/camera_calibration_pattern.markdown @@ -36,6 +36,10 @@ create a circle board pattern in file acircleboard.svg with 7 rows, 5 columns an python gen_pattern.py -o acircleboard.svg --rows 7 --columns 5 --type acircles --square_size 10 --radius_rate 2 +create a radon checkerboard for findChessboardCornersSB() with markers in (7 4), (7 5), (8 5) cells: + + python gen_pattern.py -o radon_checkerboard.svg --rows 10 --columns 15 --type radon_checkerboard -s 12.1 -m 7 4 7 5 8 5 + If you want to change unit use -u option (mm inches, px, m) If you want to change page size use -w and -h options diff --git a/modules/calib3d/include/opencv2/calib3d.hpp b/modules/calib3d/include/opencv2/calib3d.hpp index 37c5e7089a..198e405f6d 100644 --- a/modules/calib3d/include/opencv2/calib3d.hpp +++ b/modules/calib3d/include/opencv2/calib3d.hpp @@ -1489,6 +1489,8 @@ Sample usage of detecting and drawing chessboard corners: : the board to make the detection more robust in various environments. Otherwise, if there is no border and the background is dark, the outer black squares cannot be segmented properly and so the square grouping and ordering algorithm fails. + +Use gen_pattern.py (@ref tutorial_camera_calibration_pattern) to create checkerboard. */ CV_EXPORTS_W bool findChessboardCorners( InputArray image, Size patternSize, OutputArray corners, int flags = CALIB_CB_ADAPTIVE_THRESH + CALIB_CB_NORMALIZE_IMAGE ); @@ -1545,6 +1547,8 @@ transformation it is beneficial to use round corners for the field corners which are located on the outside of the board. The following figure illustrates a sample checkerboard optimized for the detection. However, any other checkerboard can be used as well. + +Use gen_pattern.py (@ref tutorial_camera_calibration_pattern) to create checkerboard. ![Checkerboard](pics/checkerboard_radon.png) */ CV_EXPORTS_AS(findChessboardCornersSBWithMeta) From 27df98721137959ac91436e3a59b06c915d5ee82 Mon Sep 17 00:00:00 2001 From: Giles Payne Date: Tue, 28 Sep 2021 20:31:07 +0900 Subject: [PATCH 197/262] Fix bug in initializers Rect2f(Point2f,Point2f) and Rect2d(Point2d,Point2d) --- modules/core/misc/objc/common/Rect2d.mm | 8 ++++---- modules/core/misc/objc/common/Rect2f.mm | 8 ++++---- modules/core/misc/objc/test/RectTest.swift | 24 ++++++++++++++++++++++ 3 files changed, 32 insertions(+), 8 deletions(-) diff --git a/modules/core/misc/objc/common/Rect2d.mm b/modules/core/misc/objc/common/Rect2d.mm index e0aa8b2844..977294d66c 100644 --- a/modules/core/misc/objc/common/Rect2d.mm +++ b/modules/core/misc/objc/common/Rect2d.mm @@ -64,10 +64,10 @@ } - (instancetype)initWithPoint:(Point2d*)point1 point:(Point2d*)point2 { - int x = (point1.x < point2.x ? point1.x : point2.x); - int y = (point1.y < point2.y ? point1.y : point2.y); - int width = (point1.x > point2.x ? point1.x : point2.x) - x; - int height = (point1.y > point2.y ? point1.y : point2.y) - y; + double x = (point1.x < point2.x ? point1.x : point2.x); + double y = (point1.y < point2.y ? point1.y : point2.y); + double width = (point1.x > point2.x ? point1.x : point2.x) - x; + double height = (point1.y > point2.y ? point1.y : point2.y) - y; return [self initWithX:x y:y width:width height:height]; } diff --git a/modules/core/misc/objc/common/Rect2f.mm b/modules/core/misc/objc/common/Rect2f.mm index 243154fe1f..2d6c326cc6 100644 --- a/modules/core/misc/objc/common/Rect2f.mm +++ b/modules/core/misc/objc/common/Rect2f.mm @@ -64,10 +64,10 @@ } - (instancetype)initWithPoint:(Point2f*)point1 point:(Point2f*)point2 { - int x = (point1.x < point2.x ? point1.x : point2.x); - int y = (point1.y < point2.y ? point1.y : point2.y); - int width = (point1.x > point2.x ? point1.x : point2.x) - x; - int height = (point1.y > point2.y ? point1.y : point2.y) - y; + float x = (point1.x < point2.x ? point1.x : point2.x); + float y = (point1.y < point2.y ? point1.y : point2.y); + float width = (point1.x > point2.x ? point1.x : point2.x) - x; + float height = (point1.y > point2.y ? point1.y : point2.y) - y; return [self initWithX:x y:y width:width height:height]; } diff --git a/modules/core/misc/objc/test/RectTest.swift b/modules/core/misc/objc/test/RectTest.swift index d8a4b4d987..4b152e0946 100644 --- a/modules/core/misc/objc/test/RectTest.swift +++ b/modules/core/misc/objc/test/RectTest.swift @@ -101,6 +101,30 @@ class RectTest: OpenCVTestCase { XCTAssertEqual(1, r.height); } + func testRect2fPointPoint() { + let p1 = Point2f(x:4.3, y:4.1) + let p2 = Point2f(x:2.7, y:3.9) + + let r = Rect2f(point: p1, point: p2) + XCTAssertNotNil(r); + XCTAssertEqual(2.7, r.x); + XCTAssertEqual(3.9, r.y); + XCTAssertEqual(1.6, r.width, accuracy: OpenCVTestCase.FEPS); + XCTAssertEqual(0.2, r.height, accuracy: OpenCVTestCase.FEPS); + } + + func testRect2dPointPoint() { + let p1 = Point2d(x:4.7879839, y:4.9922311) + let p2 = Point2d(x:2.1213123, y:3.1122129) + + let r = Rect2d(point: p1, point: p2) + XCTAssertNotNil(r); + XCTAssertEqual(2.1213123, r.x); + XCTAssertEqual(3.1122129, r.y); + XCTAssertEqual(2.6666716, r.width, accuracy: OpenCVTestCase.EPS); + XCTAssertEqual(1.8800182, r.height, accuracy: OpenCVTestCase.EPS); + } + func testRectPointSize() { let p1 = Point(x: 4, y: 4) let sz = Size(width: 3, height: 1) From b25ad12f1aab0c37afa41a729db85bf97730cedc Mon Sep 17 00:00:00 2001 From: Dmitriy Fishman Date: Tue, 28 Sep 2021 15:29:47 +0300 Subject: [PATCH 198/262] Update video_input_psnr_ssim.markdown --- .../video-input-psnr-ssim/video_input_psnr_ssim.markdown | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/tutorials/videoio/video-input-psnr-ssim/video_input_psnr_ssim.markdown b/doc/tutorials/videoio/video-input-psnr-ssim/video_input_psnr_ssim.markdown index ffd4d0213e..7bc8561347 100644 --- a/doc/tutorials/videoio/video-input-psnr-ssim/video_input_psnr_ssim.markdown +++ b/doc/tutorials/videoio/video-input-psnr-ssim/video_input_psnr_ssim.markdown @@ -180,7 +180,7 @@ implementation below. This will return a similarity index for each channel of the image. This value is between zero and one, where one corresponds to perfect fit. Unfortunately, the many Gaussian blurring is quite -costly, so while the PSNR may work in a real time like environment (24 frame per second) this will +costly, so while the PSNR may work in a real time like environment (24 frames per second) this will take significantly more than to accomplish similar performance results. Therefore, the source code presented at the start of the tutorial will perform the PSNR measurement From 2f83c3b689423c74a296473057333cef20673109 Mon Sep 17 00:00:00 2001 From: Cavendish-Koo <1020057909@qq.com> Date: Tue, 28 Sep 2021 21:18:07 +0800 Subject: [PATCH 199/262] fix the bug of HoughlinesSDIV --- modules/imgproc/src/hough.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/modules/imgproc/src/hough.cpp b/modules/imgproc/src/hough.cpp index bbb6dfcbdb..e66be8b4b7 100644 --- a/modules/imgproc/src/hough.cpp +++ b/modules/imgproc/src/hough.cpp @@ -435,12 +435,14 @@ HoughLinesSDiv( InputArray image, OutputArray lines, int type, } } + int pos = (int)(lst.size() - 1); + if( pos >= 0 && lst[pos].rho < 0 ) + lst.pop_back(); + lines.create((int)lst.size(), 1, type); Mat _lines = lines.getMat(); for( size_t idx = 0; idx < lst.size(); idx++ ) { - if( lst[idx].rho < 0 ) - continue; if (type == CV_32FC2) { _lines.at((int)idx) = Vec2f(lst[idx].rho, lst[idx].theta); From c1148c4ea6c800730e1bb31fa3032f3fbe4c8c4a Mon Sep 17 00:00:00 2001 From: Sergey Ivanov Date: Tue, 28 Sep 2021 18:02:21 +0300 Subject: [PATCH 200/262] Merge pull request #20739 from sivanov-work:merge_base_decode G-API: oneVPL (simplification) Add simple decode pipeline * Add simple decode pipeline & add onevpl namespace * Address some review comments * Add compilation guard --- modules/gapi/CMakeLists.txt | 4 + .../onevpl/accelerators/accel_policy_cpu.cpp | 2 + .../onevpl/accelerators/accel_policy_cpu.hpp | 2 + .../onevpl/accelerators/accel_policy_dx11.cpp | 7 +- .../onevpl/accelerators/accel_policy_dx11.hpp | 2 + .../accelerators/accel_policy_interface.hpp | 2 + .../surface/cpu_frame_adapter.cpp | 2 + .../surface/cpu_frame_adapter.hpp | 2 + .../onevpl/accelerators/surface/surface.cpp | 2 + .../onevpl/accelerators/surface/surface.hpp | 2 + .../accelerators/surface/surface_pool.cpp | 2 + .../accelerators/surface/surface_pool.hpp | 2 + .../engine/decode/decode_engine_legacy.cpp | 310 ++++++++++++++++++ .../engine/decode/decode_engine_legacy.hpp | 48 +++ .../onevpl/engine/decode/decode_session.cpp | 78 +++++ .../onevpl/engine/decode/decode_session.hpp | 60 ++++ .../onevpl/engine/engine_session.cpp | 33 ++ .../onevpl/engine/engine_session.hpp | 49 +++ .../onevpl/engine/processing_engine_base.cpp | 134 ++++++++ .../onevpl/engine/processing_engine_base.hpp | 96 ++++++ modules/gapi/src/streaming/onevpl/utils.hpp | 38 +++ .../gapi_streaming_vpl_core_test.cpp | 130 +++++++- 22 files changed, 992 insertions(+), 15 deletions(-) create mode 100644 modules/gapi/src/streaming/onevpl/engine/decode/decode_engine_legacy.cpp create mode 100644 modules/gapi/src/streaming/onevpl/engine/decode/decode_engine_legacy.hpp create mode 100644 modules/gapi/src/streaming/onevpl/engine/decode/decode_session.cpp create mode 100644 modules/gapi/src/streaming/onevpl/engine/decode/decode_session.hpp create mode 100644 modules/gapi/src/streaming/onevpl/engine/engine_session.cpp create mode 100644 modules/gapi/src/streaming/onevpl/engine/engine_session.hpp create mode 100644 modules/gapi/src/streaming/onevpl/engine/processing_engine_base.cpp create mode 100644 modules/gapi/src/streaming/onevpl/engine/processing_engine_base.hpp create mode 100644 modules/gapi/src/streaming/onevpl/utils.hpp diff --git a/modules/gapi/CMakeLists.txt b/modules/gapi/CMakeLists.txt index 380f1d7d84..5fd108c0ec 100644 --- a/modules/gapi/CMakeLists.txt +++ b/modules/gapi/CMakeLists.txt @@ -174,6 +174,10 @@ set(gapi_srcs src/streaming/onevpl/accelerators/surface/surface_pool.cpp src/streaming/onevpl/accelerators/accel_policy_cpu.cpp src/streaming/onevpl/accelerators/accel_policy_dx11.cpp + src/streaming/onevpl/engine/engine_session.cpp + src/streaming/onevpl/engine/processing_engine_base.cpp + src/streaming/onevpl/engine/decode/decode_engine_legacy.cpp + src/streaming/onevpl/engine/decode/decode_session.cpp # Utils (ITT tracing) src/utils/itt.cpp diff --git a/modules/gapi/src/streaming/onevpl/accelerators/accel_policy_cpu.cpp b/modules/gapi/src/streaming/onevpl/accelerators/accel_policy_cpu.cpp index 0a5f68ae4e..75cb2fdb38 100644 --- a/modules/gapi/src/streaming/onevpl/accelerators/accel_policy_cpu.cpp +++ b/modules/gapi/src/streaming/onevpl/accelerators/accel_policy_cpu.cpp @@ -20,6 +20,7 @@ namespace cv { namespace gapi { namespace wip { +namespace onevpl { VPLCPUAccelerationPolicy::VPLCPUAccelerationPolicy() { GAPI_LOG_INFO(nullptr, "created"); @@ -219,6 +220,7 @@ cv::MediaFrame::AdapterPtr VPLCPUAccelerationPolicy::create_frame_adapter(pool_k return cv::MediaFrame::AdapterPtr{new VPLMediaFrameCPUAdapter(*it)}; #endif // TEST_PERF } +} // namespace onevpl } // namespace wip } // namespace gapi } // namespace cv diff --git a/modules/gapi/src/streaming/onevpl/accelerators/accel_policy_cpu.hpp b/modules/gapi/src/streaming/onevpl/accelerators/accel_policy_cpu.hpp index cfe3057315..dc5a9347b5 100644 --- a/modules/gapi/src/streaming/onevpl/accelerators/accel_policy_cpu.hpp +++ b/modules/gapi/src/streaming/onevpl/accelerators/accel_policy_cpu.hpp @@ -22,6 +22,7 @@ namespace cv { namespace gapi { namespace wip { +namespace onevpl { // GAPI_EXPORTS for tests struct GAPI_EXPORTS VPLCPUAccelerationPolicy final : public VPLAccelerationPolicy @@ -47,6 +48,7 @@ struct GAPI_EXPORTS VPLCPUAccelerationPolicy final : public VPLAccelerationPolic private: std::map pool_table; }; +} // namespace onevpl } // namespace wip } // namespace gapi } // namespace cv diff --git a/modules/gapi/src/streaming/onevpl/accelerators/accel_policy_dx11.cpp b/modules/gapi/src/streaming/onevpl/accelerators/accel_policy_dx11.cpp index 348b864a15..cb27df8661 100644 --- a/modules/gapi/src/streaming/onevpl/accelerators/accel_policy_dx11.cpp +++ b/modules/gapi/src/streaming/onevpl/accelerators/accel_policy_dx11.cpp @@ -8,6 +8,7 @@ #include "streaming/onevpl/accelerators/accel_policy_dx11.hpp" #include "streaming/onevpl/accelerators/surface/cpu_frame_adapter.hpp" #include "streaming/onevpl/accelerators/surface/surface.hpp" +#include "streaming/onevpl/utils.hpp" #include "logger.hpp" #ifdef HAVE_DIRECTX @@ -27,6 +28,8 @@ namespace cv { namespace gapi { namespace wip { +namespace onevpl { + VPLDX11AccelerationPolicy::VPLDX11AccelerationPolicy() { #ifdef CPU_ACCEL_ADAPTER @@ -47,7 +50,8 @@ void VPLDX11AccelerationPolicy::init(session_t session) { mfxStatus sts = MFXVideoCORE_GetHandle(session, MFX_HANDLE_D3D11_DEVICE, reinterpret_cast(&hw_handle)); if (sts != MFX_ERR_NONE) { - throw std::logic_error("Cannot create VPLDX11AccelerationPolicy, MFXVideoCORE_GetHandle error"); + throw std::logic_error("Cannot create VPLDX11AccelerationPolicy, MFXVideoCORE_GetHandle error: " + + mfxstatus_to_string(sts)); } GAPI_LOG_INFO(nullptr, "VPLDX11AccelerationPolicy initialized, session: " << session); @@ -106,6 +110,7 @@ cv::MediaFrame::AdapterPtr VPLDX11AccelerationPolicy::create_frame_adapter(pool_ (void)surface; throw std::runtime_error("VPLDX11AccelerationPolicy::create_frame_adapter() is not implemented"); } +} // namespace onevpl } // namespace wip } // namespace gapi } // namespace cv diff --git a/modules/gapi/src/streaming/onevpl/accelerators/accel_policy_dx11.hpp b/modules/gapi/src/streaming/onevpl/accelerators/accel_policy_dx11.hpp index 04970432c5..a875f57085 100644 --- a/modules/gapi/src/streaming/onevpl/accelerators/accel_policy_dx11.hpp +++ b/modules/gapi/src/streaming/onevpl/accelerators/accel_policy_dx11.hpp @@ -32,6 +32,7 @@ namespace cv { namespace gapi { namespace wip { +namespace onevpl { // GAPI_EXPORTS for tests struct GAPI_EXPORTS VPLDX11AccelerationPolicy final: public VPLAccelerationPolicy @@ -57,6 +58,7 @@ private: std::unique_ptr adapter; #endif }; +} // namespace onevpl } // namespace wip } // namespace gapi } // namespace cv diff --git a/modules/gapi/src/streaming/onevpl/accelerators/accel_policy_interface.hpp b/modules/gapi/src/streaming/onevpl/accelerators/accel_policy_interface.hpp index 87b1246d25..31ee91535c 100644 --- a/modules/gapi/src/streaming/onevpl/accelerators/accel_policy_interface.hpp +++ b/modules/gapi/src/streaming/onevpl/accelerators/accel_policy_interface.hpp @@ -19,6 +19,7 @@ namespace cv { namespace gapi { namespace wip { +namespace onevpl { class Surface; struct VPLAccelerationPolicy @@ -51,6 +52,7 @@ struct VPLAccelerationPolicy virtual cv::MediaFrame::AdapterPtr create_frame_adapter(pool_key_t key, mfxFrameSurface1* surface) = 0; }; +} // namespace onevpl } // namespace wip } // namespace gapi } // namespace cv diff --git a/modules/gapi/src/streaming/onevpl/accelerators/surface/cpu_frame_adapter.cpp b/modules/gapi/src/streaming/onevpl/accelerators/surface/cpu_frame_adapter.cpp index 3fd959ca8b..d3020ab168 100644 --- a/modules/gapi/src/streaming/onevpl/accelerators/surface/cpu_frame_adapter.cpp +++ b/modules/gapi/src/streaming/onevpl/accelerators/surface/cpu_frame_adapter.cpp @@ -19,6 +19,7 @@ namespace cv { namespace gapi { namespace wip { +namespace onevpl { VPLMediaFrameCPUAdapter::VPLMediaFrameCPUAdapter(std::shared_ptr surface): parent_surface_ptr(surface) { @@ -111,6 +112,7 @@ void VPLMediaFrameCPUAdapter::serialize(cv::gapi::s11n::IOStream&) { void VPLMediaFrameCPUAdapter::deserialize(cv::gapi::s11n::IIStream&) { GAPI_Assert("VPLMediaFrameCPUAdapter::deserialize() is not implemented"); } +} // namespace onevpl } // namespace wip } // namespace gapi } // namespace cv diff --git a/modules/gapi/src/streaming/onevpl/accelerators/surface/cpu_frame_adapter.hpp b/modules/gapi/src/streaming/onevpl/accelerators/surface/cpu_frame_adapter.hpp index 16111dadaf..04a9bdc275 100644 --- a/modules/gapi/src/streaming/onevpl/accelerators/surface/cpu_frame_adapter.hpp +++ b/modules/gapi/src/streaming/onevpl/accelerators/surface/cpu_frame_adapter.hpp @@ -16,6 +16,7 @@ namespace cv { namespace gapi { namespace wip { +namespace onevpl { class Surface; class VPLMediaFrameCPUAdapter : public cv::MediaFrame::IAdapter { @@ -33,6 +34,7 @@ public: private: std::shared_ptr parent_surface_ptr; }; +} // namespace onevpl } // namespace wip } // namespace gapi } // namespace cv diff --git a/modules/gapi/src/streaming/onevpl/accelerators/surface/surface.cpp b/modules/gapi/src/streaming/onevpl/accelerators/surface/surface.cpp index 5d315412e1..3f5fd00305 100644 --- a/modules/gapi/src/streaming/onevpl/accelerators/surface/surface.cpp +++ b/modules/gapi/src/streaming/onevpl/accelerators/surface/surface.cpp @@ -12,6 +12,7 @@ namespace cv { namespace gapi { namespace wip { +namespace onevpl { Surface::Surface(std::unique_ptr&& surf, std::shared_ptr associated_memory) : workspace_memory_ptr(associated_memory), @@ -69,6 +70,7 @@ size_t Surface::release_lock() { ", locked times: " << locked_count - 1); return locked_count; // return preceding value } +} // namespace onevpl } // namespace wip } // namespace gapi } // namespace cv diff --git a/modules/gapi/src/streaming/onevpl/accelerators/surface/surface.hpp b/modules/gapi/src/streaming/onevpl/accelerators/surface/surface.hpp index 3fc30099cf..828e5cb1c7 100644 --- a/modules/gapi/src/streaming/onevpl/accelerators/surface/surface.hpp +++ b/modules/gapi/src/streaming/onevpl/accelerators/surface/surface.hpp @@ -23,6 +23,7 @@ namespace cv { namespace gapi { namespace wip { +namespace onevpl { /** * @brief Inner class for managing oneVPL surface through interface `mfxFrameSurface1`. @@ -95,6 +96,7 @@ private: using surface_ptr_t = std::shared_ptr; using surface_weak_ptr_t = std::weak_ptr; +} // namespace onevpl } // namespace wip } // namespace gapi } // namespace cv diff --git a/modules/gapi/src/streaming/onevpl/accelerators/surface/surface_pool.cpp b/modules/gapi/src/streaming/onevpl/accelerators/surface/surface_pool.cpp index 8ead7965b4..729b37f7ea 100644 --- a/modules/gapi/src/streaming/onevpl/accelerators/surface/surface_pool.cpp +++ b/modules/gapi/src/streaming/onevpl/accelerators/surface/surface_pool.cpp @@ -7,6 +7,7 @@ namespace cv { namespace gapi { namespace wip { +namespace onevpl { void CachedPool::reserve(size_t size) { surfaces.reserve(size); @@ -63,6 +64,7 @@ CachedPool::surface_ptr_t CachedPool::find_by_handle(mfxFrameSurface1* handle) { GAPI_Assert(it != cache.end() && "Cannot find cached surface from pool. Data corruption is possible"); return it->second; } +} // namespace onevpl } // namespace wip } // namespace gapi } // namespace cv diff --git a/modules/gapi/src/streaming/onevpl/accelerators/surface/surface_pool.hpp b/modules/gapi/src/streaming/onevpl/accelerators/surface/surface_pool.hpp index 2059e9b27e..029fa350db 100644 --- a/modules/gapi/src/streaming/onevpl/accelerators/surface/surface_pool.hpp +++ b/modules/gapi/src/streaming/onevpl/accelerators/surface/surface_pool.hpp @@ -17,6 +17,7 @@ namespace cv { namespace gapi { namespace wip { +namespace onevpl { class Surface; // GAPI_EXPORTS for tests @@ -38,6 +39,7 @@ private: free_surface_iterator_t next_free_it; cached_surface_container_t cache; }; +} // namespace onevpl } // namespace wip } // namespace gapi } // namespace cv diff --git a/modules/gapi/src/streaming/onevpl/engine/decode/decode_engine_legacy.cpp b/modules/gapi/src/streaming/onevpl/engine/decode/decode_engine_legacy.cpp new file mode 100644 index 0000000000..ad4a4eca8e --- /dev/null +++ b/modules/gapi/src/streaming/onevpl/engine/decode/decode_engine_legacy.cpp @@ -0,0 +1,310 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2021 Intel Corporation + +#ifdef HAVE_ONEVPL + +#include +#include + +#include + +#include "streaming/onevpl/engine/decode/decode_engine_legacy.hpp" +#include "streaming/onevpl/engine/decode/decode_session.hpp" +#include "streaming/onevpl/accelerators/accel_policy_interface.hpp" +#include "streaming/onevpl/accelerators/surface/surface.hpp" +#include "streaming/onevpl/utils.hpp" +#include "logger.hpp" + + +namespace cv { +namespace gapi { +namespace wip { +namespace onevpl { +/* UTILS */ +mfxU32 GetSurfaceSize(mfxU32 FourCC, mfxU32 width, mfxU32 height) { + mfxU32 nbytes = 0; + + mfxU32 half_width = width / 2; + mfxU32 half_height = height / 2; + switch (FourCC) { + case MFX_FOURCC_I420: + case MFX_FOURCC_NV12: + nbytes = width * height + 2 * half_width * half_height; + break; + case MFX_FOURCC_I010: + case MFX_FOURCC_P010: + nbytes = width * height + 2 * half_width * half_height; + nbytes *= 2; + break; + case MFX_FOURCC_RGB4: + nbytes = width * height * 4; + break; + default: + GAPI_LOG_WARNING(nullptr, "Unsupported FourCC requested: " << FourCC); + GAPI_Assert(false && "Unsupported FourCC requested"); + break; + } + return nbytes; +} + +surface_ptr_t create_surface_RGB4(mfxFrameInfo frameInfo, + std::shared_ptr out_buf_ptr, + size_t out_buf_ptr_offset, + size_t out_buf_size) +{ + mfxU8* buf = reinterpret_cast(out_buf_ptr.get()); + mfxU16 surfW = frameInfo.Width * 4; + mfxU16 surfH = frameInfo.Height; + (void)surfH; + + // TODO more intelligent check + if (out_buf_size <= out_buf_ptr_offset) { + throw std::runtime_error(std::string("Insufficient buffer size: ") + + std::to_string(out_buf_size) + ", buffer offset: " + + std::to_string(out_buf_ptr_offset) + + ", expected surface width: " + std::to_string(surfW) + + ", height: " + std::to_string(surfH)); + } + + std::unique_ptr handle(new mfxFrameSurface1); + memset(handle.get(), 0, sizeof(mfxFrameSurface1)); + + handle->Info = frameInfo; + handle->Data.B = buf + out_buf_ptr_offset; + handle->Data.G = handle->Data.B + 1; + handle->Data.R = handle->Data.B + 2; + handle->Data.A = handle->Data.B + 3; + handle->Data.Pitch = surfW; + + return Surface::create_surface(std::move(handle), out_buf_ptr); +} + +surface_ptr_t create_surface_other(mfxFrameInfo frameInfo, + std::shared_ptr out_buf_ptr, + size_t out_buf_ptr_offset, + size_t out_buf_size) +{ + mfxU8* buf = reinterpret_cast(out_buf_ptr.get()); + mfxU16 surfH = frameInfo.Height; + mfxU16 surfW = (frameInfo.FourCC == MFX_FOURCC_P010) ? frameInfo.Width * 2 : frameInfo.Width; + + // TODO more intelligent check + if (out_buf_size <= + out_buf_ptr_offset + (surfW * surfH) + ((surfW / 2) * (surfH / 2))) { + throw std::runtime_error(std::string("Insufficient buffer size: ") + + std::to_string(out_buf_size) + ", buffer offset: " + + std::to_string(out_buf_ptr_offset) + + ", expected surface width: " + std::to_string(surfW) + + ", height: " + std::to_string(surfH)); + } + + std::unique_ptr handle(new mfxFrameSurface1); + memset(handle.get(), 0, sizeof(mfxFrameSurface1)); + + handle->Info = frameInfo; + handle->Data.Y = buf + out_buf_ptr_offset; + handle->Data.U = buf + out_buf_ptr_offset + (surfW * surfH); + handle->Data.V = handle->Data.U + ((surfW / 2) * (surfH / 2)); + handle->Data.Pitch = surfW; + + return Surface::create_surface(std::move(handle), out_buf_ptr); +} + +VPLLegacyDecodeEngine::VPLLegacyDecodeEngine(std::unique_ptr&& accel) + : ProcessingEngineBase(std::move(accel)) { + + GAPI_LOG_INFO(nullptr, "Create Legacy Decode Engine"); + create_pipeline( + // 1) Read File + [this] (EngineSession& sess) -> ExecutionStatus + { + LegacyDecodeSession &my_sess = static_cast(sess); + my_sess.last_status = ReadEncodedStream(my_sess.stream, my_sess.data_provider); + if (my_sess.last_status != MFX_ERR_NONE) { + my_sess.data_provider.reset(); //close source + } + return ExecutionStatus::Continue; + }, + // 2) enqueue ASYNC decode + [this] (EngineSession& sess) -> ExecutionStatus + { + LegacyDecodeSession &my_sess = static_cast(sess); + + my_sess.last_status = + MFXVideoDECODE_DecodeFrameAsync(my_sess.session, + my_sess.last_status == MFX_ERR_NONE + ? &my_sess.stream + : nullptr, /* No more data to read, start decode draining mode*/ + my_sess.procesing_surface_ptr.lock()->get_handle(), + &my_sess.output_surface_ptr, + &my_sess.sync); + return ExecutionStatus::Continue; + }, + // 3) Wait for ASYNC decode result + [this] (EngineSession& sess) -> ExecutionStatus + { + if (sess.last_status == MFX_ERR_NONE) // Got 1 decoded frame + { + do { + //TODO try to extract TIMESTAMP + sess.last_status = MFXVideoCORE_SyncOperation(sess.session, sess.sync, 100); + if (MFX_ERR_NONE == sess.last_status) { + + LegacyDecodeSession& my_sess = static_cast(sess); + on_frame_ready(my_sess); + } + } while (sess.last_status == MFX_WRN_IN_EXECUTION); + } + return ExecutionStatus::Continue; + }, + // 4) Falls back on generic status procesing + [this] (EngineSession& sess) -> ExecutionStatus + { + return this->process_error(sess.last_status, static_cast(sess)); + } + ); +} + +void VPLLegacyDecodeEngine::initialize_session(mfxSession mfx_session, + DecoderParams&& decoder_param, + std::shared_ptr provider) +{ + mfxFrameAllocRequest decRequest = {}; + // Query number required surfaces for decoder + MFXVideoDECODE_QueryIOSurf(mfx_session, &decoder_param.param, &decRequest); + + // External (application) allocation of decode surfaces + GAPI_LOG_DEBUG(nullptr, "Query IOSurf for session: " << mfx_session << + ", mfxFrameAllocRequest.NumFrameSuggested: " << decRequest.NumFrameSuggested << + ", mfxFrameAllocRequest.Type: " << decRequest.Type); + + mfxU32 singleSurfaceSize = GetSurfaceSize(decoder_param.param.mfx.FrameInfo.FourCC, + decoder_param.param.mfx.FrameInfo.Width, + decoder_param.param.mfx.FrameInfo.Height); + if (!singleSurfaceSize) { + throw std::runtime_error("Cannot determine surface size for: fourCC" + + std::to_string(decoder_param.param.mfx.FrameInfo.FourCC) + + ", width: " + std::to_string(decoder_param.param.mfx.FrameInfo.Width) + + ", height: " + std::to_string(decoder_param.param.mfx.FrameInfo.Height)); + } + + const auto &frameInfo = decoder_param.param.mfx.FrameInfo; + auto surface_creator = + [&frameInfo] (std::shared_ptr out_buf_ptr, size_t out_buf_ptr_offset, + size_t out_buf_size) -> surface_ptr_t { + return (frameInfo.FourCC == MFX_FOURCC_RGB4) ? + create_surface_RGB4(frameInfo, out_buf_ptr, out_buf_ptr_offset, + out_buf_size) : + create_surface_other(frameInfo, out_buf_ptr, out_buf_ptr_offset, + out_buf_size);}; + + //TODO Configure preallocation size (how many frames we can hold) + const size_t preallocated_frames_count = 30; + VPLAccelerationPolicy::pool_key_t decode_pool_key = + acceleration_policy->create_surface_pool(decRequest.NumFrameSuggested * preallocated_frames_count, + singleSurfaceSize, + surface_creator); + + // create session + std::shared_ptr sess_ptr = + register_session(mfx_session, + std::move(decoder_param), + provider); + + sess_ptr->init_surface_pool(decode_pool_key); + // prepare working decode surface + sess_ptr->swap_surface(*this); +} + +ProcessingEngineBase::ExecutionStatus VPLLegacyDecodeEngine::execute_op(operation_t& op, EngineSession& sess) { + return op(sess); +} + +void VPLLegacyDecodeEngine::on_frame_ready(LegacyDecodeSession& sess) +{ + GAPI_LOG_DEBUG(nullptr, "[" << sess.session << "], frame ready"); + + // manage memory ownership rely on acceleration policy + auto frame_adapter = acceleration_policy->create_frame_adapter(sess.decoder_pool_id, + sess.output_surface_ptr); + ready_frames.emplace(cv::MediaFrame(std::move(frame_adapter)), sess.generate_frame_meta()); +} + +ProcessingEngineBase::ExecutionStatus VPLLegacyDecodeEngine::process_error(mfxStatus status, LegacyDecodeSession& sess) +{ + GAPI_LOG_DEBUG(nullptr, "status: " << mfxstatus_to_string(status)); + + switch (status) { + case MFX_ERR_NONE: + return ExecutionStatus::Continue; + case MFX_ERR_MORE_DATA: // The function requires more bitstream at input before decoding can proceed + if (!sess.data_provider || sess.data_provider->empty()) { + // No more data to drain from decoder, start encode draining mode + return ExecutionStatus::Processed; + } + else + return ExecutionStatus::Continue; // read more data + break; + case MFX_ERR_MORE_SURFACE: + { + // The function requires more frame surface at output before decoding can proceed. + // This applies to external memory allocations and should not be expected for + // a simple internal allocation case like this + try { + sess.swap_surface(*this); + return ExecutionStatus::Continue; + } catch (const std::exception& ex) { + GAPI_LOG_WARNING(nullptr, "[" << sess.session << "] error: " << ex.what()); + } + break; + } + case MFX_ERR_DEVICE_LOST: + // For non-CPU implementations, + // Cleanup if device is lost + GAPI_DbgAssert(false && "VPLLegacyDecodeEngine::process_error - " + "MFX_ERR_DEVICE_LOST is not processed"); + break; + case MFX_WRN_DEVICE_BUSY: + // For non-CPU implementations, + // Wait a few milliseconds then try again + GAPI_DbgAssert(false && "VPLLegacyDecodeEngine::process_error - " + "MFX_WRN_DEVICE_BUSY is not processed"); + break; + case MFX_WRN_VIDEO_PARAM_CHANGED: + // The decoder detected a new sequence header in the bitstream. + // Video parameters may have changed. + // In external memory allocation case, might need to reallocate the output surface + GAPI_DbgAssert(false && "VPLLegacyDecodeEngine::process_error - " + "MFX_WRN_VIDEO_PARAM_CHANGED is not processed"); + break; + case MFX_ERR_INCOMPATIBLE_VIDEO_PARAM: + // The function detected that video parameters provided by the application + // are incompatible with initialization parameters. + // The application should close the component and then reinitialize it + GAPI_DbgAssert(false && "VPLLegacyDecodeEngine::process_error - " + "MFX_ERR_INCOMPATIBLE_VIDEO_PARAM is not processed"); + break; + case MFX_ERR_REALLOC_SURFACE: + // Bigger surface_work required. May be returned only if + // mfxInfoMFX::EnableReallocRequest was set to ON during initialization. + // This applies to external memory allocations and should not be expected for + // a simple internal allocation case like this + GAPI_DbgAssert(false && "VPLLegacyDecodeEngine::process_error - " + "MFX_ERR_REALLOC_SURFACE is not processed"); + break; + default: + GAPI_LOG_WARNING(nullptr, "Unknown status code: " << mfxstatus_to_string(status)); + break; + } + + return ExecutionStatus::Failed; +} + +} // namespace onevpl +} // namespace wip +} // namespace gapi +} // namespace cv +#endif // HAVE_ONEVPL diff --git a/modules/gapi/src/streaming/onevpl/engine/decode/decode_engine_legacy.hpp b/modules/gapi/src/streaming/onevpl/engine/decode/decode_engine_legacy.hpp new file mode 100644 index 0000000000..5db54c3199 --- /dev/null +++ b/modules/gapi/src/streaming/onevpl/engine/decode/decode_engine_legacy.hpp @@ -0,0 +1,48 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2021 Intel Corporation + +#ifndef GAPI_STREAMING_ONVPL_DECODE_ENGINE_LEGACY_HPP +#define GAPI_STREAMING_ONVPL_DECODE_ENGINE_LEGACY_HPP +#include +#include + +#include "streaming/onevpl/engine/processing_engine_base.hpp" + +#ifdef HAVE_ONEVPL +#if (MFX_VERSION >= 2000) + #include +#endif +#include + +namespace cv { +namespace gapi { +namespace wip { +namespace onevpl { + +class LegacyDecodeSession; +struct DecoderParams; +struct IDataProvider; +struct VPLAccelerationPolicy; + +class VPLLegacyDecodeEngine : public ProcessingEngineBase { +public: + + VPLLegacyDecodeEngine(std::unique_ptr&& accel); + void initialize_session(mfxSession mfx_session, DecoderParams&& decoder_param, + std::shared_ptr provider) override; + +private: + ExecutionStatus execute_op(operation_t& op, EngineSession& sess) override; + ExecutionStatus process_error(mfxStatus status, LegacyDecodeSession& sess); + + void on_frame_ready(LegacyDecodeSession& sess); +}; +} // namespace onevpl +} // namespace wip +} // namespace gapi +} // namespace cv +#endif // HAVE_ONEVPL +#endif // GAPI_STREAMING_ONVPL_DECODE_ENGINE_LEGACY_HPP diff --git a/modules/gapi/src/streaming/onevpl/engine/decode/decode_session.cpp b/modules/gapi/src/streaming/onevpl/engine/decode/decode_session.cpp new file mode 100644 index 0000000000..3468869e15 --- /dev/null +++ b/modules/gapi/src/streaming/onevpl/engine/decode/decode_session.cpp @@ -0,0 +1,78 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2021 Intel Corporation + +#ifdef HAVE_ONEVPL + +#include +#include + +#include "streaming/onevpl/engine/decode/decode_session.hpp" +#include "streaming/onevpl/engine/decode/decode_engine_legacy.hpp" +#include "streaming/onevpl/accelerators/accel_policy_interface.hpp" +#include "streaming/onevpl/accelerators/surface/surface.hpp" +#include "streaming/onevpl/utils.hpp" + +#include "logger.hpp" +namespace cv { +namespace gapi { +namespace wip { +namespace onevpl { +LegacyDecodeSession::LegacyDecodeSession(mfxSession sess, + DecoderParams&& decoder_param, + std::shared_ptr provider) : + EngineSession(sess, std::move(decoder_param.stream)), + mfx_decoder_param(std::move(decoder_param.param)), + data_provider(std::move(provider)), + procesing_surface_ptr(), + output_surface_ptr(), + decoded_frames_count() +{ +} + +LegacyDecodeSession::~LegacyDecodeSession() +{ + GAPI_LOG_INFO(nullptr, "Close Decode for session: " << session); + MFXVideoDECODE_Close(session); +} + +void LegacyDecodeSession::swap_surface(VPLLegacyDecodeEngine& engine) { + VPLAccelerationPolicy* acceleration_policy = engine.get_accel(); + GAPI_Assert(acceleration_policy && "Empty acceleration_policy"); + auto old_locked = procesing_surface_ptr.lock(); + try { + auto cand = acceleration_policy->get_free_surface(decoder_pool_id).lock(); + + GAPI_LOG_DEBUG(nullptr, "[" << session << "] swap surface" + ", old: " << (old_locked ? old_locked->get_handle() : nullptr) << + ", new: "<< cand->get_handle()); + + procesing_surface_ptr = cand; + } catch (const std::exception& ex) { + GAPI_LOG_WARNING(nullptr, "[" << session << "] error: " << ex.what() << + "Abort"); + } +} + +void LegacyDecodeSession::init_surface_pool(VPLAccelerationPolicy::pool_key_t key) { + GAPI_Assert(key && "Init decode pull with empty key"); + decoder_pool_id = key; +} + +Data::Meta LegacyDecodeSession::generate_frame_meta() { + const auto now = std::chrono::system_clock::now(); + const auto dur = std::chrono::duration_cast + (now.time_since_epoch()); + Data::Meta meta { + {cv::gapi::streaming::meta_tag::timestamp, int64_t{dur.count()} }, + {cv::gapi::streaming::meta_tag::seq_id, int64_t{decoded_frames_count++}} + }; + return meta; +} +} // namespace onevpl +} // namespace wip +} // namespace gapi +} // namespace cv +#endif // HAVE_ONEVPL diff --git a/modules/gapi/src/streaming/onevpl/engine/decode/decode_session.hpp b/modules/gapi/src/streaming/onevpl/engine/decode/decode_session.hpp new file mode 100644 index 0000000000..46b1decc81 --- /dev/null +++ b/modules/gapi/src/streaming/onevpl/engine/decode/decode_session.hpp @@ -0,0 +1,60 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2021 Intel Corporation + +#ifndef GAPI_STREAMING_ONVPL_ENGINE_DECODE_DECODE_SESSION_HPP +#define GAPI_STREAMING_ONVPL_ENGINE_DECODE_DECODE_SESSION_HPP +#include +#include + +#include + +#include "streaming/onevpl/engine/engine_session.hpp" +#include "streaming/onevpl/accelerators/accel_policy_interface.hpp" +#ifdef HAVE_ONEVPL +#if (MFX_VERSION >= 2000) + #include +#endif +#include + +namespace cv { +namespace gapi { +namespace wip { +namespace onevpl { + +struct IDataProvider; +class Surface; +struct VPLAccelerationPolicy; + +class LegacyDecodeSession : public EngineSession { +public: + friend class VPLLegacyDecodeEngine; + + LegacyDecodeSession(mfxSession sess, DecoderParams&& decoder_param, std::shared_ptr provider); + ~LegacyDecodeSession(); + using EngineSession::EngineSession; + + void swap_surface(VPLLegacyDecodeEngine& engine); + void init_surface_pool(VPLAccelerationPolicy::pool_key_t key); + + mfxVideoParam mfx_decoder_param; + std::shared_ptr data_provider; + + Data::Meta generate_frame_meta(); +private: + VPLAccelerationPolicy::pool_key_t decoder_pool_id; + mfxFrameAllocRequest request; + + std::weak_ptr procesing_surface_ptr; + mfxFrameSurface1* output_surface_ptr; + + int64_t decoded_frames_count; +}; +} // namespace onevpl +} // namespace wip +} // namespace gapi +} // namespace cv +#endif // HAVE_ONEVPL +#endif // GAPI_STREAMING_ONVPL_ENGINE_DECODE_DECODE_SESSION_HPP diff --git a/modules/gapi/src/streaming/onevpl/engine/engine_session.cpp b/modules/gapi/src/streaming/onevpl/engine/engine_session.cpp new file mode 100644 index 0000000000..9f8028361a --- /dev/null +++ b/modules/gapi/src/streaming/onevpl/engine/engine_session.cpp @@ -0,0 +1,33 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2021 Intel Corporation +#ifdef HAVE_ONEVPL + +#include "streaming/onevpl/engine/engine_session.hpp" +#include "streaming/onevpl/utils.hpp" +#include "logger.hpp" + +namespace cv { +namespace gapi { +namespace wip { +namespace onevpl { + +EngineSession::EngineSession(mfxSession sess, mfxBitstream&& str) : + session(sess), stream(std::move(str)) {} +EngineSession::~EngineSession() +{ + GAPI_LOG_INFO(nullptr, "Close session: " << session); + MFXClose(session); +} + +std::string EngineSession::error_code_to_str() const +{ + return mfxstatus_to_string(last_status); +} +} // namespace onevpl +} // namespace wip +} // namespace gapi +} // namespace cv +#endif // HAVE_ONEVPL diff --git a/modules/gapi/src/streaming/onevpl/engine/engine_session.hpp b/modules/gapi/src/streaming/onevpl/engine/engine_session.hpp new file mode 100644 index 0000000000..d8f2f6b312 --- /dev/null +++ b/modules/gapi/src/streaming/onevpl/engine/engine_session.hpp @@ -0,0 +1,49 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2021 Intel Corporation + +#ifndef GAPI_STREAMING_ONEVPL_ENGINE_ENGINE_SESSION_HPP +#define GAPI_STREAMING_ONEVPL_ENGINE_ENGINE_SESSION_HPP + +#include +#include +#include +#include +#include +#include + +#include "opencv2/gapi/own/exports.hpp" // GAPI_EXPORTS + +#ifdef HAVE_ONEVPL +#include + +namespace cv { +namespace gapi { +namespace wip { +namespace onevpl { + +// GAPI_EXPORTS for tests +struct GAPI_EXPORTS DecoderParams { + mfxBitstream stream; + mfxVideoParam param; +}; + +struct GAPI_EXPORTS EngineSession { + mfxSession session; + mfxBitstream stream; + mfxSyncPoint sync; + mfxStatus last_status; + + EngineSession(mfxSession sess, mfxBitstream&& str); + std::string error_code_to_str() const; + virtual ~EngineSession(); +}; +} // namespace onevpl +} // namespace wip +} // namespace gapi +} // namespace cv + +#endif // HAVE_ONEVPL +#endif // GAPI_STREAMING_ONEVPL_ENGINE_ENGINE_SESSION_HPP diff --git a/modules/gapi/src/streaming/onevpl/engine/processing_engine_base.cpp b/modules/gapi/src/streaming/onevpl/engine/processing_engine_base.cpp new file mode 100644 index 0000000000..3161b1627e --- /dev/null +++ b/modules/gapi/src/streaming/onevpl/engine/processing_engine_base.cpp @@ -0,0 +1,134 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2021 Intel Corporation + +#ifdef HAVE_ONEVPL + +#include + +#include +#include "streaming/onevpl/engine/processing_engine_base.hpp" +#include "streaming/onevpl/accelerators/accel_policy_interface.hpp" +#include "logger.hpp" + +namespace cv { +namespace gapi { +namespace wip { +namespace onevpl { + +ProcessingEngineBase::ProcessingEngineBase(std::unique_ptr&& accel) : + acceleration_policy(std::move(accel)) { +} + +ProcessingEngineBase::~ProcessingEngineBase() { + GAPI_LOG_INFO(nullptr, "destroyed"); +} + +ProcessingEngineBase::ExecutionStatus ProcessingEngineBase::process(mfxSession session) { + auto sess_it = sessions.find(session); + if (sess_it == sessions.end()) { + return ExecutionStatus::SessionNotFound; + } + + session_ptr processing_session = sess_it->second; + ExecutionData& exec_data = execution_table[session]; + + GAPI_LOG_DEBUG(nullptr, "[" << session <<"] start op id: " << exec_data.op_id); + ExecutionStatus status = execute_op(pipeline.at(exec_data.op_id), *processing_session); + size_t old_op_id = exec_data.op_id++; + if (exec_data.op_id == pipeline.size()) + { + exec_data.op_id = 0; + } + GAPI_LOG_DEBUG(nullptr, "[" << session <<"] finish op id: " << old_op_id << + ", " << processing_session->error_code_to_str() << + ", " << ProcessingEngineBase::status_to_string(status) << + ", next op id: " << exec_data.op_id); + + if (status == ExecutionStatus::Failed) { + + GAPI_LOG_WARNING(nullptr, "Operation for session: " << session << + ", " << ProcessingEngineBase::status_to_string(status) << + " - remove it"); + sessions.erase(sess_it); + execution_table.erase(session); + } + + if (status == ExecutionStatus::Processed) { + sessions.erase(sess_it); + execution_table.erase(session); + } + + return status; +} + +const char* ProcessingEngineBase::status_to_string(ExecutionStatus status) +{ + switch(status) { + case ExecutionStatus::Continue: return "CONTINUE"; + case ExecutionStatus::Processed: return "PROCESSED"; + case ExecutionStatus::SessionNotFound: return "NOT_FOUND_SESSION"; + case ExecutionStatus::Failed: return "FAILED"; + default: + return "UNKNOWN"; + } +} + +ProcessingEngineBase::ExecutionStatus ProcessingEngineBase::execute_op(operation_t& op, EngineSession& sess) +{ + return op(sess); +} + +size_t ProcessingEngineBase::get_ready_frames_count() const +{ + return ready_frames.size(); +} + +void ProcessingEngineBase::get_frame(Data &data) +{ + data = ready_frames.front(); + ready_frames.pop(); +} + +const VPLAccelerationPolicy* ProcessingEngineBase::get_accel() const { + return acceleration_policy.get(); +} + +VPLAccelerationPolicy* ProcessingEngineBase::get_accel() { + return const_cast(static_cast(this)->get_accel()); +} + + +// Read encoded stream from file +mfxStatus ReadEncodedStream(mfxBitstream &bs, std::shared_ptr& data_provider) { + + if (!data_provider) { + return MFX_ERR_MORE_DATA; + } + + mfxU8 *p0 = bs.Data; + mfxU8 *p1 = bs.Data + bs.DataOffset; + if (bs.DataOffset > bs.MaxLength - 1) { + return MFX_ERR_NOT_ENOUGH_BUFFER; + } + if (bs.DataLength + bs.DataOffset > bs.MaxLength) { + return MFX_ERR_NOT_ENOUGH_BUFFER; + } + + std::copy_n(p0, bs.DataLength, p1); + + bs.DataOffset = 0; + bs.DataLength += static_cast(data_provider->fetch_data(bs.MaxLength - bs.DataLength, + bs.Data + bs.DataLength)); + if (bs.DataLength == 0) + return MFX_ERR_MORE_DATA; + + return MFX_ERR_NONE; +} +} // namespace onevpl +} // namespace wip +} // namespace gapi +} // namespace cv +#endif // HAVE_ONEVPL diff --git a/modules/gapi/src/streaming/onevpl/engine/processing_engine_base.hpp b/modules/gapi/src/streaming/onevpl/engine/processing_engine_base.hpp new file mode 100644 index 0000000000..7d4869ac66 --- /dev/null +++ b/modules/gapi/src/streaming/onevpl/engine/processing_engine_base.hpp @@ -0,0 +1,96 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2021 Intel Corporation + +#ifndef GAPI_STREAMING_ONEVPL_ENGINE_PROCESSING_ENGINE_BASE_HPP +#define GAPI_STREAMING_ONEVPL_ENGINE_PROCESSING_ENGINE_BASE_HPP + +#include +#include "streaming/onevpl/engine/engine_session.hpp" +#include "opencv2/gapi/own/exports.hpp" // GAPI_EXPORTS + +namespace cv { +namespace gapi { +namespace wip { +namespace onevpl { + +struct VPLAccelerationPolicy; +struct IDataProvider; + +// GAPI_EXPORTS for tests +class GAPI_EXPORTS ProcessingEngineBase { +public: + enum class ExecutionStatus { + Continue, + Processed, + SessionNotFound, + Failed + }; + struct ExecutionData { + size_t op_id = 0; + }; + + using file_ptr = std::unique_ptr; + + using session_ptr = std::shared_ptr; + using SessionsTable = std::map; + using ExecutionDataTable = std::map; + + using frame_t = cv::gapi::wip::Data; + using frames_container_t = std::queue; + using operation_t = std::function; + + static const char * status_to_string(ExecutionStatus); + + ProcessingEngineBase(std::unique_ptr&& accel); + virtual ~ProcessingEngineBase(); + + virtual void initialize_session(mfxSession mfx_session, + DecoderParams&& decoder_param, + std::shared_ptr provider) = 0; + + ExecutionStatus process(mfxSession session); + size_t get_ready_frames_count() const; + void get_frame(Data &data); + + const VPLAccelerationPolicy* get_accel() const; + VPLAccelerationPolicy* get_accel(); +protected: + SessionsTable sessions; + frames_container_t ready_frames; + ExecutionDataTable execution_table; + + std::vector pipeline; + std::unique_ptr acceleration_policy; + + virtual ExecutionStatus execute_op(operation_t& op, EngineSession& sess); + + template + void create_pipeline(Ops&&...ops) + { + GAPI_DbgAssert(pipeline.empty() && "Pipeline must be empty"); + std::vector({std::forward(ops)...}).swap(pipeline); + } + + template + std::shared_ptr register_session(mfxSession key, + SessionArgs&& ...args) + { + auto sess_impl = std::make_shared(key, + std::forward(args)...); + sessions.emplace(key, sess_impl); + execution_table.emplace(key, ExecutionData{}); + return sess_impl; + } +}; + + +mfxStatus ReadEncodedStream(mfxBitstream &bs, std::shared_ptr& data_provider); +} // namespace onevpl +} // namespace wip +} // namespace gapi +} // namespace cv + +#endif // GAPI_STREAMING_ONEVPL_ENGINE_PROCESSING_ENGINE_BASE_HPP diff --git a/modules/gapi/src/streaming/onevpl/utils.hpp b/modules/gapi/src/streaming/onevpl/utils.hpp new file mode 100644 index 0000000000..0512c4f687 --- /dev/null +++ b/modules/gapi/src/streaming/onevpl/utils.hpp @@ -0,0 +1,38 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2021 Intel Corporation + +#ifndef GAPI_STREAMING_ONEVPL_ONEVPL_UTILS_HPP +#define GAPI_STREAMING_ONEVPL_ONEVPL_UTILS_HPP + +#ifdef HAVE_ONEVPL +#if (MFX_VERSION >= 2000) +#include +#endif // MFX_VERSION + +#include +#include + +#include +#include + +#include + + +namespace cv { +namespace gapi { +namespace wip { +namespace onevpl { + +inline std::string mfxstatus_to_string(mfxStatus) { + return "UNKNOWN"; +} + +} // namespace onevpl +} // namespace wip +} // namespace gapi +} // namespace cv +#endif // HAVE_ONEVPL +#endif // GAPI_STREAMING_ONEVPL_ONEVPL_UTILS_HPP diff --git a/modules/gapi/test/streaming/gapi_streaming_vpl_core_test.cpp b/modules/gapi/test/streaming/gapi_streaming_vpl_core_test.cpp index 5a36a2befb..0b8822b366 100644 --- a/modules/gapi/test/streaming/gapi_streaming_vpl_core_test.cpp +++ b/modules/gapi/test/streaming/gapi_streaming_vpl_core_test.cpp @@ -12,6 +12,7 @@ #include #include +#include #include #include @@ -29,23 +30,88 @@ #include #ifdef HAVE_ONEVPL +#include + #include "streaming/onevpl/accelerators/surface/surface.hpp" #include "streaming/onevpl/accelerators/surface/cpu_frame_adapter.hpp" #include "streaming/onevpl/accelerators/accel_policy_cpu.hpp" +#include "streaming/onevpl/engine/processing_engine_base.hpp" +#include "streaming/onevpl/engine/engine_session.hpp" namespace opencv_test { namespace { -cv::gapi::wip::surface_ptr_t create_test_surface(std::shared_ptr out_buf_ptr, + +struct EmptyDataProvider : public cv::gapi::wip::onevpl::IDataProvider { + + size_t fetch_data(size_t, void*) override { + return 0; + } + bool empty() const override { + return true; + } +}; + +struct TestProcessingSession : public cv::gapi::wip::onevpl::EngineSession { + TestProcessingSession(mfxSession mfx_session) : + EngineSession(mfx_session, {}) { + } +}; + +struct TestProcessingEngine: public cv::gapi::wip::onevpl::ProcessingEngineBase { + + size_t pipeline_stage_num = 0; + + TestProcessingEngine(std::unique_ptr&& accel) : + cv::gapi::wip::onevpl::ProcessingEngineBase(std::move(accel)) { + using cv::gapi::wip::onevpl::EngineSession; + create_pipeline( + // 0) + [this] (EngineSession&) -> ExecutionStatus + { + pipeline_stage_num = 0; + return ExecutionStatus::Continue; + }, + // 1) + [this] (EngineSession&) -> ExecutionStatus + { + pipeline_stage_num = 1; + return ExecutionStatus::Continue; + }, + // 2) + [this] (EngineSession&) -> ExecutionStatus + { + pipeline_stage_num = 2; + return ExecutionStatus::Continue; + }, + // 3) + [this] (EngineSession&) -> ExecutionStatus + { + pipeline_stage_num = 3; + ready_frames.emplace(cv::MediaFrame()); + return ExecutionStatus::Processed; + } + ); + } + + void initialize_session(mfxSession mfx_session, + cv::gapi::wip::onevpl::DecoderParams&&, + std::shared_ptr) override { + + register_session(mfx_session); + } +}; + +cv::gapi::wip::onevpl::surface_ptr_t create_test_surface(std::shared_ptr out_buf_ptr, size_t, size_t) { std::unique_ptr handle(new mfxFrameSurface1{}); - return cv::gapi::wip::Surface::create_surface(std::move(handle), out_buf_ptr); + return cv::gapi::wip::onevpl::Surface::create_surface(std::move(handle), out_buf_ptr); } TEST(OneVPL_Source_Surface, InitSurface) { - using namespace cv::gapi::wip; + using namespace cv::gapi::wip::onevpl; // create raw MFX handle std::unique_ptr handle(new mfxFrameSurface1{}); @@ -67,7 +133,7 @@ TEST(OneVPL_Source_Surface, InitSurface) TEST(OneVPL_Source_Surface, ConcurrentLock) { - using namespace cv::gapi::wip; + using namespace cv::gapi::wip::onevpl; // create raw MFX handle std::unique_ptr handle(new mfxFrameSurface1{}); @@ -107,7 +173,7 @@ TEST(OneVPL_Source_Surface, ConcurrentLock) TEST(OneVPL_Source_Surface, MemoryLifeTime) { - using namespace cv::gapi::wip; + using namespace cv::gapi::wip::onevpl; // create preallocate surface memory std::unique_ptr preallocated_memory_ptr(new char); @@ -170,7 +236,7 @@ TEST(OneVPL_Source_Surface, MemoryLifeTime) TEST(OneVPL_Source_CPU_FrameAdapter, InitFrameAdapter) { - using namespace cv::gapi::wip; + using namespace cv::gapi::wip::onevpl; // create raw MFX handle std::unique_ptr handle(new mfxFrameSurface1{}); @@ -191,8 +257,8 @@ TEST(OneVPL_Source_CPU_FrameAdapter, InitFrameAdapter) TEST(OneVPL_Source_CPU_Accelerator, InitDestroy) { - using cv::gapi::wip::VPLCPUAccelerationPolicy; - using cv::gapi::wip::VPLAccelerationPolicy; + using cv::gapi::wip::onevpl::VPLCPUAccelerationPolicy; + using cv::gapi::wip::onevpl::VPLAccelerationPolicy; auto acceleration_policy = std::make_shared(); @@ -221,9 +287,9 @@ TEST(OneVPL_Source_CPU_Accelerator, InitDestroy) TEST(OneVPL_Source_CPU_Accelerator, PoolProduceConsume) { - using cv::gapi::wip::VPLCPUAccelerationPolicy; - using cv::gapi::wip::VPLAccelerationPolicy; - using cv::gapi::wip::Surface; + using cv::gapi::wip::onevpl::VPLCPUAccelerationPolicy; + using cv::gapi::wip::onevpl::VPLAccelerationPolicy; + using cv::gapi::wip::onevpl::Surface; auto acceleration_policy = std::make_shared(); @@ -277,9 +343,9 @@ TEST(OneVPL_Source_CPU_Accelerator, PoolProduceConsume) TEST(OneVPL_Source_CPU_Accelerator, PoolProduceConcurrentConsume) { - using cv::gapi::wip::VPLCPUAccelerationPolicy; - using cv::gapi::wip::VPLAccelerationPolicy; - using cv::gapi::wip::Surface; + using cv::gapi::wip::onevpl::VPLCPUAccelerationPolicy; + using cv::gapi::wip::onevpl::VPLAccelerationPolicy; + using cv::gapi::wip::onevpl::Surface; auto acceleration_policy = std::make_shared(); @@ -339,6 +405,42 @@ TEST(OneVPL_Source_CPU_Accelerator, PoolProduceConcurrentConsume) worker_thread.join(); EXPECT_TRUE(free_surface_count >= free_surface_count_prev); } + +TEST(OneVPL_Source_ProcessingEngine, Init) +{ + using namespace cv::gapi::wip::onevpl; + std::unique_ptr accel; + TestProcessingEngine engine(std::move(accel)); + + mfxSession mfx_session{}; + engine.initialize_session(mfx_session, DecoderParams{}, std::shared_ptr{}); + + EXPECT_EQ(engine.get_ready_frames_count(), 0); + ProcessingEngineBase::ExecutionStatus ret = engine.process(mfx_session); + EXPECT_EQ(ret, ProcessingEngineBase::ExecutionStatus::Continue); + EXPECT_EQ(engine.pipeline_stage_num, 0); + + ret = engine.process(mfx_session); + EXPECT_EQ(ret, ProcessingEngineBase::ExecutionStatus::Continue); + EXPECT_EQ(engine.pipeline_stage_num, 1); + + ret = engine.process(mfx_session); + EXPECT_EQ(ret, ProcessingEngineBase::ExecutionStatus::Continue); + EXPECT_EQ(engine.pipeline_stage_num, 2); + + ret = engine.process(mfx_session); + EXPECT_EQ(ret, ProcessingEngineBase::ExecutionStatus::Processed); + EXPECT_EQ(engine.pipeline_stage_num, 3); + EXPECT_EQ(engine.get_ready_frames_count(), 1); + + ret = engine.process(mfx_session); + EXPECT_EQ(ret, ProcessingEngineBase::ExecutionStatus::SessionNotFound); + EXPECT_EQ(engine.pipeline_stage_num, 3); + EXPECT_EQ(engine.get_ready_frames_count(), 1); + + cv::gapi::wip::Data frame; + engine.get_frame(frame); +} } } // namespace opencv_test #endif // HAVE_ONEVPL From 5865af7f6e76c501b3c016b83686c49036deb726 Mon Sep 17 00:00:00 2001 From: Smirnov Alexey Date: Wed, 29 Sep 2021 12:13:13 +0300 Subject: [PATCH 201/262] Add more parameters to render fixture --- .../gapi/test/common/gapi_tests_helpers.hpp | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/modules/gapi/test/common/gapi_tests_helpers.hpp b/modules/gapi/test/common/gapi_tests_helpers.hpp index ce0c90442d..3f4e1ee0b3 100644 --- a/modules/gapi/test/common/gapi_tests_helpers.hpp +++ b/modules/gapi/test/common/gapi_tests_helpers.hpp @@ -50,6 +50,18 @@ namespace opencv_test __TUPLE_PARAM_TYPE(index) param_name = getSpecificParam(); \ __WRAP_VAARGS(__DEFINE_PARAMS_IMPL7(index+1, __VA_ARGS__)) +#define __DEFINE_PARAMS_IMPL9(index, param_name, ...) \ + __TUPLE_PARAM_TYPE(index) param_name = getSpecificParam(); \ + __WRAP_VAARGS(__DEFINE_PARAMS_IMPL8(index+1, __VA_ARGS__)) + +#define __DEFINE_PARAMS_IMPL10(index, param_name, ...) \ + __TUPLE_PARAM_TYPE(index) param_name = getSpecificParam(); \ + __WRAP_VAARGS(__DEFINE_PARAMS_IMPL9(index+1, __VA_ARGS__)) + +#define __DEFINE_PARAMS_IMPL11(index, param_name, ...) \ + __TUPLE_PARAM_TYPE(index) param_name = getSpecificParam(); \ + __WRAP_VAARGS(__DEFINE_PARAMS_IMPL10(index+1, __VA_ARGS__)) + // user interface to define member variables of specified names #define DEFINE_SPECIFIC_PARAMS_0() @@ -76,6 +88,15 @@ namespace opencv_test #define DEFINE_SPECIFIC_PARAMS_8(...) \ __WRAP_VAARGS(__DEFINE_PARAMS_IMPL8(0, __VA_ARGS__)) + +#define DEFINE_SPECIFIC_PARAMS_9(...) \ + __WRAP_VAARGS(__DEFINE_PARAMS_IMPL9(0, __VA_ARGS__)) + +#define DEFINE_SPECIFIC_PARAMS_10(...) \ + __WRAP_VAARGS(__DEFINE_PARAMS_IMPL10(0, __VA_ARGS__)) + +#define DEFINE_SPECIFIC_PARAMS_11(...) \ + __WRAP_VAARGS(__DEFINE_PARAMS_IMPL11(0, __VA_ARGS__)) } // namespace opencv_test #endif //OPENCV_GAPI_TESTS_HELPERS_HPP From 846317ef372e37332fca8804c76cfa612cdf66e8 Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Wed, 29 Sep 2021 13:16:46 +0000 Subject: [PATCH 202/262] dnn(ocl): fix conv BASIC workgroup --- .../dnn/src/ocl4dnn/src/ocl4dnn_conv_spatial.cpp | 13 ++++++++++--- modules/dnn/src/opencl/conv_layer_spatial.cl | 12 ++++++++---- 2 files changed, 18 insertions(+), 7 deletions(-) diff --git a/modules/dnn/src/ocl4dnn/src/ocl4dnn_conv_spatial.cpp b/modules/dnn/src/ocl4dnn/src/ocl4dnn_conv_spatial.cpp index 5eee1da4a0..a1164273ac 100644 --- a/modules/dnn/src/ocl4dnn/src/ocl4dnn_conv_spatial.cpp +++ b/modules/dnn/src/ocl4dnn/src/ocl4dnn_conv_spatial.cpp @@ -1081,9 +1081,16 @@ bool OCL4DNNConvSpatial::convolve(const UMat &bottom, UMat &top, kernel.set(argIdx++, (uint16_t)output_h_); kernel.set(argIdx++, (uint16_t)pad_w_); kernel.set(argIdx++, (uint16_t)pad_h_); - if (!kernel.run_(3, config->global_work_size, - (config->use_null_local) ? NULL : config->local_work_size, - false)) + + size_t wgs = kernel.workGroupSize(); + if (!wgs) + { + CV_LOG_ERROR(NULL, "DNN/OpenCL: Can't query workGroupSize of Basic kernel"); + return false; + } + size_t lws[1] = { wgs }; + size_t gws[1] = { roundUp((size_t)output_w_ * output_h_ * M_, (unsigned)lws[0]) }; + if (!kernel.run_(1, gws, lws, false)) { CV_LOG_ERROR(NULL, "DNN/OpenCL: Basic kernel run failed"); return false; diff --git a/modules/dnn/src/opencl/conv_layer_spatial.cl b/modules/dnn/src/opencl/conv_layer_spatial.cl index e7bbacd4c4..455f0ed7ea 100644 --- a/modules/dnn/src/opencl/conv_layer_spatial.cl +++ b/modules/dnn/src/opencl/conv_layer_spatial.cl @@ -158,10 +158,14 @@ __kernel void ConvolveBasic( ) { __global Dtype* convolved_image = convolved_image_base + convolved_image_base_offset; - const int outputX = get_global_id(0); - const int outputY = get_global_id(1); - const int kernelNum = get_global_id(2) * ZPAR; - if (outputX < output_width && outputY < output_height) + const int out_idx = get_global_id(0); // 1D task layout: [output_width * output_height * OUTPUT_Z] + const int plane_size = output_width * output_height; + const int out_plane_idx = out_idx % plane_size; + const int outputZ = out_idx / plane_size; + const int outputY = out_plane_idx / output_width; + const int outputX = out_plane_idx % output_width; + const int kernelNum = outputZ * ZPAR; + if (kernelNum < OUTPUT_Z) { Dtype sum[ZPAR]; for (int kern = 0; kern < ZPAR; kern++) From 9a8552e8aec3b314cc947b16db8ea182d691b322 Mon Sep 17 00:00:00 2001 From: Suleyman TURKMEN Date: Sat, 25 Sep 2021 09:31:44 +0300 Subject: [PATCH 203/262] Update perf_bgfg_mog2.cpp, perf_bgfg_knn.cpp --- modules/video/perf/opencl/perf_bgfg_knn.cpp | 6 +++--- modules/video/perf/opencl/perf_bgfg_mog2.cpp | 6 +++--- modules/video/perf/perf_bgfg_knn.cpp | 6 +++--- modules/video/perf/perf_bgfg_mog2.cpp | 6 +++--- 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/modules/video/perf/opencl/perf_bgfg_knn.cpp b/modules/video/perf/opencl/perf_bgfg_knn.cpp index 833db66d86..6e95e52b2f 100644 --- a/modules/video/perf/opencl/perf_bgfg_knn.cpp +++ b/modules/video/perf/opencl/perf_bgfg_knn.cpp @@ -19,7 +19,7 @@ typedef TestBaseWithParam KNN_GetBackgroundImage; using namespace opencv_test; -OCL_PERF_TEST_P(KNN_Apply, KNN, Combine(Values("gpu/video/768x576.avi", "gpu/video/1920x1080.avi"), Values(1,3))) +OCL_PERF_TEST_P(KNN_Apply, KNN, Combine(Values("cv/video/768x576.avi", "cv/video/1920x1080.avi"), Values(1,3))) { VideoKNNParamType params = GetParam(); @@ -51,8 +51,8 @@ OCL_PERF_TEST_P(KNN_Apply, KNN, Combine(Values("gpu/video/768x576.avi", "gpu/vid } OCL_PERF_TEST_P(KNN_GetBackgroundImage, KNN, Values( - std::make_pair("gpu/video/768x576.avi", 5), - std::make_pair("gpu/video/1920x1080.avi", 5))) + std::make_pair("cv/video/768x576.avi", 5), + std::make_pair("cv/video/1920x1080.avi", 5))) { VideoKNNParamType params = GetParam(); diff --git a/modules/video/perf/opencl/perf_bgfg_mog2.cpp b/modules/video/perf/opencl/perf_bgfg_mog2.cpp index 8e5f095257..3de03fa1ee 100644 --- a/modules/video/perf/opencl/perf_bgfg_mog2.cpp +++ b/modules/video/perf/opencl/perf_bgfg_mog2.cpp @@ -19,7 +19,7 @@ typedef TestBaseWithParam MOG2_GetBackgroundImage; using namespace opencv_test; -OCL_PERF_TEST_P(MOG2_Apply, Mog2, Combine(Values("gpu/video/768x576.avi", "gpu/video/1920x1080.avi"), Values(1,3))) +OCL_PERF_TEST_P(MOG2_Apply, Mog2, Combine(Values("cv/video/768x576.avi", "cv/video/1920x1080.avi"), Values(1,3))) { VideoMOG2ParamType params = GetParam(); @@ -51,8 +51,8 @@ OCL_PERF_TEST_P(MOG2_Apply, Mog2, Combine(Values("gpu/video/768x576.avi", "gpu/v } OCL_PERF_TEST_P(MOG2_GetBackgroundImage, Mog2, Values( - std::make_pair("gpu/video/768x576.avi", 5), - std::make_pair("gpu/video/1920x1080.avi", 5))) + std::make_pair("cv/video/768x576.avi", 5), + std::make_pair("cv/video/1920x1080.avi", 5))) { VideoMOG2ParamType params = GetParam(); diff --git a/modules/video/perf/perf_bgfg_knn.cpp b/modules/video/perf/perf_bgfg_knn.cpp index 23bb1fe0a5..42d6368fd3 100644 --- a/modules/video/perf/perf_bgfg_knn.cpp +++ b/modules/video/perf/perf_bgfg_knn.cpp @@ -14,7 +14,7 @@ typedef tuple VideoKNNParamType; typedef TestBaseWithParam KNN_Apply; typedef TestBaseWithParam KNN_GetBackgroundImage; -PERF_TEST_P(KNN_Apply, KNN, Combine(Values("gpu/video/768x576.avi", "gpu/video/1920x1080.avi"), Values(1,3))) +PERF_TEST_P(KNN_Apply, KNN, Combine(Values("cv/video/768x576.avi", "cv/video/1920x1080.avi"), Values(1,3))) { VideoKNNParamType params = GetParam(); @@ -46,8 +46,8 @@ PERF_TEST_P(KNN_Apply, KNN, Combine(Values("gpu/video/768x576.avi", "gpu/video/1 } PERF_TEST_P(KNN_GetBackgroundImage, KNN, Values( - std::make_pair("gpu/video/768x576.avi", 5), - std::make_pair("gpu/video/1920x1080.avi", 5))) + std::make_pair("cv/video/768x576.avi", 5), + std::make_pair("cv/video/1920x1080.avi", 5))) { VideoKNNParamType params = GetParam(); diff --git a/modules/video/perf/perf_bgfg_mog2.cpp b/modules/video/perf/perf_bgfg_mog2.cpp index f911a9c74e..610138f85c 100644 --- a/modules/video/perf/perf_bgfg_mog2.cpp +++ b/modules/video/perf/perf_bgfg_mog2.cpp @@ -14,7 +14,7 @@ typedef tuple VideoMOG2ParamType; typedef TestBaseWithParam MOG2_Apply; typedef TestBaseWithParam MOG2_GetBackgroundImage; -PERF_TEST_P(MOG2_Apply, Mog2, Combine(Values("gpu/video/768x576.avi", "gpu/video/1920x1080.avi"), Values(1,3))) +PERF_TEST_P(MOG2_Apply, Mog2, Combine(Values("cv/video/768x576.avi", "cv/video/1920x1080.avi"), Values(1,3))) { VideoMOG2ParamType params = GetParam(); @@ -46,8 +46,8 @@ PERF_TEST_P(MOG2_Apply, Mog2, Combine(Values("gpu/video/768x576.avi", "gpu/video } PERF_TEST_P(MOG2_GetBackgroundImage, Mog2, Values( - std::make_pair("gpu/video/768x576.avi", 5), - std::make_pair("gpu/video/1920x1080.avi", 5))) + std::make_pair("cv/video/768x576.avi", 5), + std::make_pair("cv/video/1920x1080.avi", 5))) { VideoMOG2ParamType params = GetParam(); From f8f6cd6ef5d2b06845e19ed44f3865a51f1a1162 Mon Sep 17 00:00:00 2001 From: Suleyman TURKMEN Date: Tue, 28 Sep 2021 21:23:26 +0300 Subject: [PATCH 204/262] Update OpenCVDetectVTK.cmake --- cmake/OpenCVDetectVTK.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/OpenCVDetectVTK.cmake b/cmake/OpenCVDetectVTK.cmake index 57c154475c..b1a58ca085 100644 --- a/cmake/OpenCVDetectVTK.cmake +++ b/cmake/OpenCVDetectVTK.cmake @@ -1,7 +1,7 @@ if(NOT VTK_FOUND) find_package(VTK QUIET NAMES vtk VTK) if(VTK_FOUND) - if(VTK_VERSION VERSION_EQUAL "9") # VTK 9.0 + if(NOT (VTK_VERSION VERSION_LESS "9.0.0") AND (VTK_VERSION VERSION_LESS "10.0.0")) # VTK 9.x find_package(VTK 9 QUIET NAMES vtk COMPONENTS FiltersExtraction FiltersSources From 9b768727080b3279c244ad595115b1d5126d32ed Mon Sep 17 00:00:00 2001 From: Suleyman TURKMEN Date: Fri, 1 Oct 2021 16:23:16 +0300 Subject: [PATCH 205/262] restore LSD --- modules/imgproc/include/opencv2/imgproc.hpp | 10 +- modules/imgproc/src/lsd.cpp | 983 +++++++++++++++++++- modules/imgproc/test/test_lsd.cpp | 4 - samples/cpp/lsd_lines.cpp | 73 ++ 4 files changed, 1055 insertions(+), 15 deletions(-) create mode 100644 samples/cpp/lsd_lines.cpp diff --git a/modules/imgproc/include/opencv2/imgproc.hpp b/modules/imgproc/include/opencv2/imgproc.hpp index f7583c1926..855cbaf159 100644 --- a/modules/imgproc/include/opencv2/imgproc.hpp +++ b/modules/imgproc/include/opencv2/imgproc.hpp @@ -1220,12 +1220,14 @@ protected: //! @addtogroup imgproc_feature //! @{ +/** @example samples/cpp/lsd_lines.cpp +An example using the LineSegmentDetector +\image html building_lsd.png "Sample output image" width=434 height=300 +*/ + /** @brief Line segment detector class following the algorithm described at @cite Rafael12 . - -@note Implementation has been removed due original code license conflict - */ class CV_EXPORTS_W LineSegmentDetector : public Algorithm { @@ -1288,8 +1290,6 @@ to edit those, as to tailor it for their own application. @param log_eps Detection threshold: -log10(NFA) \> log_eps. Used only when advance refinement is chosen. @param density_th Minimal density of aligned region points in the enclosing rectangle. @param n_bins Number of bins in pseudo-ordering of gradient modulus. - -@note Implementation has been removed due original code license conflict */ CV_EXPORTS_W Ptr createLineSegmentDetector( int refine = LSD_REFINE_STD, double scale = 0.8, diff --git a/modules/imgproc/src/lsd.cpp b/modules/imgproc/src/lsd.cpp index 1ec984d290..d06759c2bb 100644 --- a/modules/imgproc/src/lsd.cpp +++ b/modules/imgproc/src/lsd.cpp @@ -42,7 +42,123 @@ #include "precomp.hpp" #include -namespace cv { +///////////////////////////////////////////////////////////////////////////////////////// +// Default LSD parameters +// SIGMA_SCALE 0.6 - Sigma for Gaussian filter is computed as sigma = sigma_scale/scale. +// QUANT 2.0 - Bound to the quantization error on the gradient norm. +// ANG_TH 22.5 - Gradient angle tolerance in degrees. +// LOG_EPS 0.0 - Detection threshold: -log10(NFA) > log_eps +// DENSITY_TH 0.7 - Minimal density of region points in rectangle. +// N_BINS 1024 - Number of bins in pseudo-ordering of gradient modulus. + +#define M_3_2_PI (3 * CV_PI) / 2 // 3/2 pi +#define M_2__PI (2 * CV_PI) // 2 pi + +#ifndef M_LN10 +#define M_LN10 2.30258509299404568402 +#endif + +#define NOTDEF double(-1024.0) // Label for pixels with undefined gradient. + +#define NOTUSED 0 // Label for pixels not used in yet. +#define USED 1 // Label for pixels already used in detection. + +#define RELATIVE_ERROR_FACTOR 100.0 + +const double DEG_TO_RADS = CV_PI / 180; + +#define log_gamma(x) ((x)>15.0?log_gamma_windschitl(x):log_gamma_lanczos(x)) + +struct edge +{ + cv::Point p; + bool taken; +}; + +///////////////////////////////////////////////////////////////////////////////////////// + +inline double distSq(const double x1, const double y1, + const double x2, const double y2) +{ + return (x2 - x1)*(x2 - x1) + (y2 - y1)*(y2 - y1); +} + +inline double dist(const double x1, const double y1, + const double x2, const double y2) +{ + return sqrt(distSq(x1, y1, x2, y2)); +} + +// Signed angle difference +inline double angle_diff_signed(const double& a, const double& b) +{ + double diff = a - b; + while(diff <= -CV_PI) diff += M_2__PI; + while(diff > CV_PI) diff -= M_2__PI; + return diff; +} + +// Absolute value angle difference +inline double angle_diff(const double& a, const double& b) +{ + return std::fabs(angle_diff_signed(a, b)); +} + +// Compare doubles by relative error. +inline bool double_equal(const double& a, const double& b) +{ + // trivial case + if(a == b) return true; + + double abs_diff = fabs(a - b); + double aa = fabs(a); + double bb = fabs(b); + double abs_max = (aa > bb)? aa : bb; + + if(abs_max < DBL_MIN) abs_max = DBL_MIN; + + return (abs_diff / abs_max) <= (RELATIVE_ERROR_FACTOR * DBL_EPSILON); +} + +inline bool AsmallerB_XoverY(const edge& a, const edge& b) +{ + if (a.p.x == b.p.x) return a.p.y < b.p.y; + else return a.p.x < b.p.x; +} + +/** + * Computes the natural logarithm of the absolute value of + * the gamma function of x using Windschitl method. + * See http://www.rskey.org/gamma.htm + */ +inline double log_gamma_windschitl(const double& x) +{ + return 0.918938533204673 + (x-0.5)*log(x) - x + + 0.5*x*log(x*sinh(1/x) + 1/(810.0*pow(x, 6.0))); +} + +/** + * Computes the natural logarithm of the absolute value of + * the gamma function of x using the Lanczos approximation. + * See http://www.rskey.org/gamma.htm + */ +inline double log_gamma_lanczos(const double& x) +{ + static double q[7] = { 75122.6331530, 80916.6278952, 36308.2951477, + 8687.24529705, 1168.92649479, 83.8676043424, + 2.50662827511 }; + double a = (x + 0.5) * log(x + 5.5) - (x + 5.5); + double b = 0; + for(int n = 0; n < 7; ++n) + { + a -= log(x + double(n)); + b += q[n] * pow(x, double(n)); + } + return a + log(b); +} +/////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +namespace cv{ class LineSegmentDetectorImpl CV_FINAL : public LineSegmentDetector { @@ -113,7 +229,164 @@ public: int compareSegments(const Size& size, InputArray lines1, InputArray lines2, InputOutputArray _image = noArray()) CV_OVERRIDE; private: + Mat image; + Mat scaled_image; + Mat_ angles; // in rads + Mat_ modgrad; + Mat_ used; + + int img_width; + int img_height; + double LOG_NT; + + bool w_needed; + bool p_needed; + bool n_needed; + + const double SCALE; + const int doRefine; + const double SIGMA_SCALE; + const double QUANT; + const double ANG_TH; + const double LOG_EPS; + const double DENSITY_TH; + const int N_BINS; + + struct RegionPoint { + int x; + int y; + uchar* used; + double angle; + double modgrad; + }; + + struct normPoint + { + Point2i p; + int norm; + }; + + std::vector ordered_points; + + struct rect + { + double x1, y1, x2, y2; // first and second point of the line segment + double width; // rectangle width + double x, y; // center of the rectangle + double theta; // angle + double dx,dy; // (dx,dy) is vector oriented as the line segment + double prec; // tolerance angle + double p; // probability of a point with angle within 'prec' + }; + LineSegmentDetectorImpl& operator= (const LineSegmentDetectorImpl&); // to quiet MSVC + +/** + * Detect lines in the whole input image. + * + * @param lines Return: A vector of Vec4f elements specifying the beginning and ending point of a line. + * Where Vec4f is (x1, y1, x2, y2), point 1 is the start, point 2 - end. + * Returned lines are strictly oriented depending on the gradient. + * @param widths Return: Vector of widths of the regions, where the lines are found. E.g. Width of line. + * @param precisions Return: Vector of precisions with which the lines are found. + * @param nfas Return: Vector containing number of false alarms in the line region, with precision of 10%. + * The bigger the value, logarithmically better the detection. + * * -1 corresponds to 10 mean false alarms + * * 0 corresponds to 1 mean false alarm + * * 1 corresponds to 0.1 mean false alarms + */ + void flsd(std::vector& lines, + std::vector& widths, std::vector& precisions, + std::vector& nfas); + +/** + * Finds the angles and the gradients of the image. Generates a list of pseudo ordered points. + * + * @param threshold The minimum value of the angle that is considered defined, otherwise NOTDEF + * @param n_bins The number of bins with which gradients are ordered by, using bucket sort. + * @param ordered_points Return: Vector of coordinate points that are pseudo ordered by magnitude. + * Pixels would be ordered by norm value, up to a precision given by max_grad/n_bins. + */ + void ll_angle(const double& threshold, const unsigned int& n_bins); + +/** + * Grow a region starting from point s with a defined precision, + * returning the containing points size and the angle of the gradients. + * + * @param s Starting point for the region. + * @param reg Return: Vector of points, that are part of the region + * @param reg_angle Return: The mean angle of the region. + * @param prec The precision by which each region angle should be aligned to the mean. + */ + void region_grow(const Point2i& s, std::vector& reg, + double& reg_angle, const double& prec); + +/** + * Finds the bounding rotated rectangle of a region. + * + * @param reg The region of points, from which the rectangle to be constructed from. + * @param reg_angle The mean angle of the region. + * @param prec The precision by which points were found. + * @param p Probability of a point with angle within 'prec'. + * @param rec Return: The generated rectangle. + */ + void region2rect(const std::vector& reg, const double reg_angle, + const double prec, const double p, rect& rec) const; + +/** + * Compute region's angle as the principal inertia axis of the region. + * @return Regions angle. + */ + double get_theta(const std::vector& reg, const double& x, + const double& y, const double& reg_angle, const double& prec) const; + +/** + * An estimation of the angle tolerance is performed by the standard deviation of the angle at points + * near the region's starting point. Then, a new region is grown starting from the same point, but using the + * estimated angle tolerance. If this fails to produce a rectangle with the right density of region points, + * 'reduce_region_radius' is called to try to satisfy this condition. + */ + bool refine(std::vector& reg, double reg_angle, + const double prec, double p, rect& rec, const double& density_th); + +/** + * Reduce the region size, by elimination the points far from the starting point, until that leads to + * rectangle with the right density of region points or to discard the region if too small. + */ + bool reduce_region_radius(std::vector& reg, double reg_angle, + const double prec, double p, rect& rec, double density, const double& density_th); + +/** + * Try some rectangles variations to improve NFA value. Only if the rectangle is not meaningful (i.e., log_nfa <= log_eps). + * @return The new NFA value. + */ + double rect_improve(rect& rec) const; + +/** + * Calculates the number of correctly aligned points within the rectangle. + * @return The new NFA value. + */ + double rect_nfa(const rect& rec) const; + +/** + * Computes the NFA values based on the total number of points, points that agree. + * n, k, p are the binomial parameters. + * @return The new NFA value. + */ + double nfa(const int& n, const int& k, const double& p) const; + +/** + * Is the point at place 'address' aligned to angle theta, up to precision 'prec'? + * @return Whether the point is aligned. + */ + bool isAligned(int x, int y, const double& theta, const double& prec) const; + +public: + // Compare norm + static inline bool compare_norm( const normPoint& n1, const normPoint& n2 ) + { + return (n1.norm > n2.norm); + } }; ///////////////////////////////////////////////////////////////////////////////////////// @@ -131,12 +404,13 @@ CV_EXPORTS Ptr createLineSegmentDetector( LineSegmentDetectorImpl::LineSegmentDetectorImpl(int _refine, double _scale, double _sigma_scale, double _quant, double _ang_th, double _log_eps, double _density_th, int _n_bins) + : img_width(0), img_height(0), LOG_NT(0), w_needed(false), p_needed(false), n_needed(false), + SCALE(_scale), doRefine(_refine), SIGMA_SCALE(_sigma_scale), QUANT(_quant), + ANG_TH(_ang_th), LOG_EPS(_log_eps), DENSITY_TH(_density_th), N_BINS(_n_bins) { CV_Assert(_scale > 0 && _sigma_scale > 0 && _quant >= 0 && _ang_th > 0 && _ang_th < 180 && _density_th >= 0 && _density_th < 1 && _n_bins > 0); - CV_UNUSED(_refine); CV_UNUSED(_log_eps); - CV_Error(Error::StsNotImplemented, "Implementation has been removed due original code license issues"); } void LineSegmentDetectorImpl::detect(InputArray _image, OutputArray _lines, @@ -144,11 +418,708 @@ void LineSegmentDetectorImpl::detect(InputArray _image, OutputArray _lines, { CV_INSTRUMENT_REGION(); - CV_UNUSED(_image); CV_UNUSED(_lines); - CV_UNUSED(_width); CV_UNUSED(_prec); CV_UNUSED(_nfa); - CV_Error(Error::StsNotImplemented, "Implementation has been removed due original code license issues"); + image = _image.getMat(); + CV_Assert(!image.empty() && image.type() == CV_8UC1); + + std::vector lines; + std::vector w, p, n; + w_needed = _width.needed(); + p_needed = _prec.needed(); + if (doRefine < LSD_REFINE_ADV) + n_needed = false; + else + n_needed = _nfa.needed(); + + flsd(lines, w, p, n); + + Mat(lines).copyTo(_lines); + if(w_needed) Mat(w).copyTo(_width); + if(p_needed) Mat(p).copyTo(_prec); + if(n_needed) Mat(n).copyTo(_nfa); + + // Clear used structures + ordered_points.clear(); } +void LineSegmentDetectorImpl::flsd(std::vector& lines, + std::vector& widths, std::vector& precisions, + std::vector& nfas) +{ + // Angle tolerance + const double prec = CV_PI * ANG_TH / 180; + const double p = ANG_TH / 180; + const double rho = QUANT / sin(prec); // gradient magnitude threshold + + if(SCALE != 1) + { + Mat gaussian_img; + const double sigma = (SCALE < 1)?(SIGMA_SCALE / SCALE):(SIGMA_SCALE); + const double sprec = 3; + const unsigned int h = (unsigned int)(ceil(sigma * sqrt(2 * sprec * log(10.0)))); + Size ksize(1 + 2 * h, 1 + 2 * h); // kernel size + GaussianBlur(image, gaussian_img, ksize, sigma); + // Scale image to needed size + resize(gaussian_img, scaled_image, Size(), SCALE, SCALE, INTER_LINEAR_EXACT); + ll_angle(rho, N_BINS); + } + else + { + scaled_image = image; + ll_angle(rho, N_BINS); + } + + LOG_NT = 5 * (log10(double(img_width)) + log10(double(img_height))) / 2 + log10(11.0); + const size_t min_reg_size = size_t(-LOG_NT/log10(p)); // minimal number of points in region that can give a meaningful event + + // // Initialize region only when needed + // Mat region = Mat::zeros(scaled_image.size(), CV_8UC1); + used = Mat_::zeros(scaled_image.size()); // zeros = NOTUSED + std::vector reg; + + // Search for line segments + for(size_t i = 0, points_size = ordered_points.size(); i < points_size; ++i) + { + const Point2i& point = ordered_points[i].p; + if((used.at(point) == NOTUSED) && (angles.at(point) != NOTDEF)) + { + double reg_angle; + region_grow(ordered_points[i].p, reg, reg_angle, prec); + + // Ignore small regions + if(reg.size() < min_reg_size) { continue; } + + // Construct rectangular approximation for the region + rect rec; + region2rect(reg, reg_angle, prec, p, rec); + + double log_nfa = -1; + if(doRefine > LSD_REFINE_NONE) + { + // At least REFINE_STANDARD lvl. + if(!refine(reg, reg_angle, prec, p, rec, DENSITY_TH)) { continue; } + + if(doRefine >= LSD_REFINE_ADV) + { + // Compute NFA + log_nfa = rect_improve(rec); + if(log_nfa <= LOG_EPS) { continue; } + } + } + // Found new line + + // Add the offset + rec.x1 += 0.5; rec.y1 += 0.5; + rec.x2 += 0.5; rec.y2 += 0.5; + + // scale the result values if a sub-sampling was performed + if(SCALE != 1) + { + rec.x1 /= SCALE; rec.y1 /= SCALE; + rec.x2 /= SCALE; rec.y2 /= SCALE; + rec.width /= SCALE; + } + + //Store the relevant data + lines.push_back(Vec4f(float(rec.x1), float(rec.y1), float(rec.x2), float(rec.y2))); + if(w_needed) widths.push_back(rec.width); + if(p_needed) precisions.push_back(rec.p); + if(n_needed && doRefine >= LSD_REFINE_ADV) nfas.push_back(log_nfa); + } + } +} + +void LineSegmentDetectorImpl::ll_angle(const double& threshold, + const unsigned int& n_bins) +{ + //Initialize data + angles = Mat_(scaled_image.size()); + modgrad = Mat_(scaled_image.size()); + + img_width = scaled_image.cols; + img_height = scaled_image.rows; + + // Undefined the down and right boundaries + angles.row(img_height - 1).setTo(NOTDEF); + angles.col(img_width - 1).setTo(NOTDEF); + + // Computing gradient for remaining pixels + double max_grad = -1; + for(int y = 0; y < img_height - 1; ++y) + { + const uchar* scaled_image_row = scaled_image.ptr(y); + const uchar* next_scaled_image_row = scaled_image.ptr(y+1); + double* angles_row = angles.ptr(y); + double* modgrad_row = modgrad.ptr(y); + for(int x = 0; x < img_width-1; ++x) + { + int DA = next_scaled_image_row[x + 1] - scaled_image_row[x]; + int BC = scaled_image_row[x + 1] - next_scaled_image_row[x]; + int gx = DA + BC; // gradient x component + int gy = DA - BC; // gradient y component + double norm = std::sqrt((gx * gx + gy * gy) / 4.0); // gradient norm + + modgrad_row[x] = norm; // store gradient + + if (norm <= threshold) // norm too small, gradient no defined + { + angles_row[x] = NOTDEF; + } + else + { + angles_row[x] = fastAtan2(float(gx), float(-gy)) * DEG_TO_RADS; // gradient angle computation + if (norm > max_grad) { max_grad = norm; } + } + + } + } + + // Compute histogram of gradient values + double bin_coef = (max_grad > 0) ? double(n_bins - 1) / max_grad : 0; // If all image is smooth, max_grad <= 0 + for(int y = 0; y < img_height - 1; ++y) + { + const double* modgrad_row = modgrad.ptr(y); + for(int x = 0; x < img_width - 1; ++x) + { + normPoint _point; + int i = int(modgrad_row[x] * bin_coef); + _point.p = Point(x, y); + _point.norm = i; + ordered_points.push_back(_point); + } + } + + // Sort + std::sort(ordered_points.begin(), ordered_points.end(), compare_norm); +} + +void LineSegmentDetectorImpl::region_grow(const Point2i& s, std::vector& reg, + double& reg_angle, const double& prec) +{ + reg.clear(); + + // Point to this region + RegionPoint seed; + seed.x = s.x; + seed.y = s.y; + seed.used = &used.at(s); + reg_angle = angles.at(s); + seed.angle = reg_angle; + seed.modgrad = modgrad.at(s); + reg.push_back(seed); + + float sumdx = float(std::cos(reg_angle)); + float sumdy = float(std::sin(reg_angle)); + *seed.used = USED; + + //Try neighboring regions + for (size_t i = 0;i(yy); + const double* angles_row = angles.ptr(yy); + const double* modgrad_row = modgrad.ptr(yy); + for(int xx = xx_min; xx <= xx_max; ++xx) + { + uchar& is_used = used_row[xx]; + if(is_used != USED && + (isAligned(xx, yy, reg_angle, prec))) + { + const double& angle = angles_row[xx]; + // Add point + is_used = USED; + RegionPoint region_point; + region_point.x = xx; + region_point.y = yy; + region_point.used = &is_used; + region_point.modgrad = modgrad_row[xx]; + region_point.angle = angle; + reg.push_back(region_point); + + // Update region's angle + sumdx += cos(float(angle)); + sumdy += sin(float(angle)); + // reg_angle is used in the isAligned, so it needs to be updates? + reg_angle = fastAtan2(sumdy, sumdx) * DEG_TO_RADS; + } + } + } + } +} + +void LineSegmentDetectorImpl::region2rect(const std::vector& reg, + const double reg_angle, const double prec, const double p, rect& rec) const +{ + double x = 0, y = 0, sum = 0; + for(size_t i = 0; i < reg.size(); ++i) + { + const RegionPoint& pnt = reg[i]; + const double& weight = pnt.modgrad; + x += double(pnt.x) * weight; + y += double(pnt.y) * weight; + sum += weight; + } + + // Weighted sum must differ from 0 + CV_Assert(sum > 0); + + x /= sum; + y /= sum; + + double theta = get_theta(reg, x, y, reg_angle, prec); + + // Find length and width + double dx = cos(theta); + double dy = sin(theta); + double l_min = 0, l_max = 0, w_min = 0, w_max = 0; + + for(size_t i = 0; i < reg.size(); ++i) + { + double regdx = double(reg[i].x) - x; + double regdy = double(reg[i].y) - y; + + double l = regdx * dx + regdy * dy; + double w = -regdx * dy + regdy * dx; + + if(l > l_max) l_max = l; + else if(l < l_min) l_min = l; + if(w > w_max) w_max = w; + else if(w < w_min) w_min = w; + } + + // Store values + rec.x1 = x + l_min * dx; + rec.y1 = y + l_min * dy; + rec.x2 = x + l_max * dx; + rec.y2 = y + l_max * dy; + rec.width = w_max - w_min; + rec.x = x; + rec.y = y; + rec.theta = theta; + rec.dx = dx; + rec.dy = dy; + rec.prec = prec; + rec.p = p; + + // Min width of 1 pixel + if(rec.width < 1.0) rec.width = 1.0; +} + +double LineSegmentDetectorImpl::get_theta(const std::vector& reg, const double& x, + const double& y, const double& reg_angle, const double& prec) const +{ + double Ixx = 0.0; + double Iyy = 0.0; + double Ixy = 0.0; + + // Compute inertia matrix + for(size_t i = 0; i < reg.size(); ++i) + { + const double& regx = reg[i].x; + const double& regy = reg[i].y; + const double& weight = reg[i].modgrad; + double dx = regx - x; + double dy = regy - y; + Ixx += dy * dy * weight; + Iyy += dx * dx * weight; + Ixy -= dx * dy * weight; + } + + // Check if inertia matrix is null + CV_Assert(!(double_equal(Ixx, 0) && double_equal(Iyy, 0) && double_equal(Ixy, 0))); + + // Compute smallest eigenvalue + double lambda = 0.5 * (Ixx + Iyy - sqrt((Ixx - Iyy) * (Ixx - Iyy) + 4.0 * Ixy * Ixy)); + + // Compute angle + double theta = (fabs(Ixx)>fabs(Iyy))? + double(fastAtan2(float(lambda - Ixx), float(Ixy))): + double(fastAtan2(float(Ixy), float(lambda - Iyy))); // in degs + theta *= DEG_TO_RADS; + + // Correct angle by 180 deg if necessary + if(angle_diff(theta, reg_angle) > prec) { theta += CV_PI; } + + return theta; +} + +bool LineSegmentDetectorImpl::refine(std::vector& reg, double reg_angle, + const double prec, double p, rect& rec, const double& density_th) +{ + double density = double(reg.size()) / (dist(rec.x1, rec.y1, rec.x2, rec.y2) * rec.width); + + if (density >= density_th) { return true; } + + // Try to reduce angle tolerance + double xc = double(reg[0].x); + double yc = double(reg[0].y); + const double& ang_c = reg[0].angle; + double sum = 0, s_sum = 0; + int n = 0; + + for (size_t i = 0; i < reg.size(); ++i) + { + *(reg[i].used) = NOTUSED; + if (dist(xc, yc, reg[i].x, reg[i].y) < rec.width) + { + const double& angle = reg[i].angle; + double ang_d = angle_diff_signed(angle, ang_c); + sum += ang_d; + s_sum += ang_d * ang_d; + ++n; + } + } + CV_Assert(n > 0); + double mean_angle = sum / double(n); + // 2 * standard deviation + double tau = 2.0 * sqrt((s_sum - 2.0 * mean_angle * sum) / double(n) + mean_angle * mean_angle); + + // Try new region + region_grow(Point(reg[0].x, reg[0].y), reg, reg_angle, tau); + + if (reg.size() < 2) { return false; } + + region2rect(reg, reg_angle, prec, p, rec); + density = double(reg.size()) / (dist(rec.x1, rec.y1, rec.x2, rec.y2) * rec.width); + + if (density < density_th) + { + return reduce_region_radius(reg, reg_angle, prec, p, rec, density, density_th); + } + else + { + return true; + } +} + +bool LineSegmentDetectorImpl::reduce_region_radius(std::vector& reg, double reg_angle, + const double prec, double p, rect& rec, double density, const double& density_th) +{ + // Compute region's radius + double xc = double(reg[0].x); + double yc = double(reg[0].y); + double radSq1 = distSq(xc, yc, rec.x1, rec.y1); + double radSq2 = distSq(xc, yc, rec.x2, rec.y2); + double radSq = radSq1 > radSq2 ? radSq1 : radSq2; + + while(density < density_th) + { + radSq *= 0.75*0.75; // Reduce region's radius to 75% of its value + // Remove points from the region and update 'used' map + for (size_t i = 0; i < reg.size(); ++i) + { + if(distSq(xc, yc, double(reg[i].x), double(reg[i].y)) > radSq) + { + // Remove point from the region + *(reg[i].used) = NOTUSED; + std::swap(reg[i], reg[reg.size() - 1]); + reg.pop_back(); + --i; // To avoid skipping one point + } + } + + if(reg.size() < 2) { return false; } + + // Re-compute rectangle + region2rect(reg ,reg_angle, prec, p, rec); + + // Re-compute region points density + density = double(reg.size()) / + (dist(rec.x1, rec.y1, rec.x2, rec.y2) * rec.width); + } + + return true; +} + +double LineSegmentDetectorImpl::rect_improve(rect& rec) const +{ + double delta = 0.5; + double delta_2 = delta / 2.0; + + double log_nfa = rect_nfa(rec); + + if(log_nfa > LOG_EPS) return log_nfa; // Good rectangle + + // Try to improve + // Finer precision + rect r = rect(rec); // Copy + for(int n = 0; n < 5; ++n) + { + r.p /= 2; + r.prec = r.p * CV_PI; + double log_nfa_new = rect_nfa(r); + if(log_nfa_new > log_nfa) + { + log_nfa = log_nfa_new; + rec = rect(r); + } + } + if(log_nfa > LOG_EPS) return log_nfa; + + // Try to reduce width + r = rect(rec); + for(unsigned int n = 0; n < 5; ++n) + { + if((r.width - delta) >= 0.5) + { + r.width -= delta; + double log_nfa_new = rect_nfa(r); + if(log_nfa_new > log_nfa) + { + rec = rect(r); + log_nfa = log_nfa_new; + } + } + } + if(log_nfa > LOG_EPS) return log_nfa; + + // Try to reduce one side of rectangle + r = rect(rec); + for(unsigned int n = 0; n < 5; ++n) + { + if((r.width - delta) >= 0.5) + { + r.x1 += -r.dy * delta_2; + r.y1 += r.dx * delta_2; + r.x2 += -r.dy * delta_2; + r.y2 += r.dx * delta_2; + r.width -= delta; + double log_nfa_new = rect_nfa(r); + if(log_nfa_new > log_nfa) + { + rec = rect(r); + log_nfa = log_nfa_new; + } + } + } + if(log_nfa > LOG_EPS) return log_nfa; + + // Try to reduce other side of rectangle + r = rect(rec); + for(unsigned int n = 0; n < 5; ++n) + { + if((r.width - delta) >= 0.5) + { + r.x1 -= -r.dy * delta_2; + r.y1 -= r.dx * delta_2; + r.x2 -= -r.dy * delta_2; + r.y2 -= r.dx * delta_2; + r.width -= delta; + double log_nfa_new = rect_nfa(r); + if(log_nfa_new > log_nfa) + { + rec = rect(r); + log_nfa = log_nfa_new; + } + } + } + if(log_nfa > LOG_EPS) return log_nfa; + + // Try finer precision + r = rect(rec); + for(unsigned int n = 0; n < 5; ++n) + { + if((r.width - delta) >= 0.5) + { + r.p /= 2; + r.prec = r.p * CV_PI; + double log_nfa_new = rect_nfa(r); + if(log_nfa_new > log_nfa) + { + rec = rect(r); + log_nfa = log_nfa_new; + } + } + } + + return log_nfa; +} + +double LineSegmentDetectorImpl::rect_nfa(const rect& rec) const +{ + int total_pts = 0, alg_pts = 0; + double half_width = rec.width / 2.0; + double dyhw = rec.dy * half_width; + double dxhw = rec.dx * half_width; + + edge ordered_x[4]; + edge* min_y = &ordered_x[0]; + edge* max_y = &ordered_x[0]; // Will be used for loop range + + ordered_x[0].p.x = int(rec.x1 - dyhw); ordered_x[0].p.y = int(rec.y1 + dxhw); ordered_x[0].taken = false; + ordered_x[1].p.x = int(rec.x2 - dyhw); ordered_x[1].p.y = int(rec.y2 + dxhw); ordered_x[1].taken = false; + ordered_x[2].p.x = int(rec.x2 + dyhw); ordered_x[2].p.y = int(rec.y2 - dxhw); ordered_x[2].taken = false; + ordered_x[3].p.x = int(rec.x1 + dyhw); ordered_x[3].p.y = int(rec.y1 - dxhw); ordered_x[3].taken = false; + + std::sort(ordered_x, ordered_x + 4, AsmallerB_XoverY); + + // Find min y. And mark as taken. find max y. + for(unsigned int i = 1; i < 4; ++i) + { + if(min_y->p.y > ordered_x[i].p.y) {min_y = &ordered_x[i]; } + if(max_y->p.y < ordered_x[i].p.y) {max_y = &ordered_x[i]; } + } + min_y->taken = true; + + // Find leftmost untaken point; + edge* leftmost = 0; + for(unsigned int i = 0; i < 4; ++i) + { + if(!ordered_x[i].taken) + { + if(!leftmost) // if uninitialized + { + leftmost = &ordered_x[i]; + } + else if (leftmost->p.x > ordered_x[i].p.x) + { + leftmost = &ordered_x[i]; + } + } + } + CV_Assert(leftmost != NULL); + leftmost->taken = true; + + // Find rightmost untaken point; + edge* rightmost = 0; + for(unsigned int i = 0; i < 4; ++i) + { + if(!ordered_x[i].taken) + { + if(!rightmost) // if uninitialized + { + rightmost = &ordered_x[i]; + } + else if (rightmost->p.x < ordered_x[i].p.x) + { + rightmost = &ordered_x[i]; + } + } + } + CV_Assert(rightmost != NULL); + rightmost->taken = true; + + // Find last untaken point; + edge* tailp = 0; + for(unsigned int i = 0; i < 4; ++i) + { + if(!ordered_x[i].taken) + { + if(!tailp) // if uninitialized + { + tailp = &ordered_x[i]; + } + else if (tailp->p.x > ordered_x[i].p.x) + { + tailp = &ordered_x[i]; + } + } + } + CV_Assert(tailp != NULL); + tailp->taken = true; + + double flstep = (min_y->p.y != leftmost->p.y) ? + (min_y->p.x - leftmost->p.x) / (min_y->p.y - leftmost->p.y) : 0; //first left step + double slstep = (leftmost->p.y != tailp->p.x) ? + (leftmost->p.x - tailp->p.x) / (leftmost->p.y - tailp->p.x) : 0; //second left step + + double frstep = (min_y->p.y != rightmost->p.y) ? + (min_y->p.x - rightmost->p.x) / (min_y->p.y - rightmost->p.y) : 0; //first right step + double srstep = (rightmost->p.y != tailp->p.x) ? + (rightmost->p.x - tailp->p.x) / (rightmost->p.y - tailp->p.x) : 0; //second right step + + double lstep = flstep, rstep = frstep; + + double left_x = min_y->p.x, right_x = min_y->p.x; + + // Loop around all points in the region and count those that are aligned. + int min_iter = min_y->p.y; + int max_iter = max_y->p.y; + for(int y = min_iter; y <= max_iter; ++y) + { + if (y < 0 || y >= img_height) continue; + + for(int x = int(left_x); x <= int(right_x); ++x) + { + if (x < 0 || x >= img_width) continue; + + ++total_pts; + if(isAligned(x, y, rec.theta, rec.prec)) + { + ++alg_pts; + } + } + + if(y >= leftmost->p.y) { lstep = slstep; } + if(y >= rightmost->p.y) { rstep = srstep; } + + left_x += lstep; + right_x += rstep; + } + + return nfa(total_pts, alg_pts, rec.p); +} + +double LineSegmentDetectorImpl::nfa(const int& n, const int& k, const double& p) const +{ + // Trivial cases + if(n == 0 || k == 0) { return -LOG_NT; } + if(n == k) { return -LOG_NT - double(n) * log10(p); } + + double p_term = p / (1 - p); + + double log1term = (double(n) + 1) - log_gamma(double(k) + 1) + - log_gamma(double(n-k) + 1) + + double(k) * log(p) + double(n-k) * log(1.0 - p); + double term = exp(log1term); + + if(double_equal(term, 0)) + { + if(k > n * p) return -log1term / M_LN10 - LOG_NT; + else return -LOG_NT; + } + + // Compute more terms if needed + double bin_tail = term; + double tolerance = 0.1; // an error of 10% in the result is accepted + for(int i = k + 1; i <= n; ++i) + { + double bin_term = double(n - i + 1) / double(i); + double mult_term = bin_term * p_term; + term *= mult_term; + bin_tail += term; + if(bin_term < 1) + { + double err = term * ((1 - pow(mult_term, double(n-i+1))) / (1 - mult_term) - 1); + if(err < tolerance * fabs(-log10(bin_tail) - LOG_NT) * bin_tail) break; + } + + } + return -log10(bin_tail) - LOG_NT; +} + +inline bool LineSegmentDetectorImpl::isAligned(int x, int y, const double& theta, const double& prec) const +{ + if(x < 0 || y < 0 || x >= angles.cols || y >= angles.rows) { return false; } + const double& a = angles.at(y, x); + if(a == NOTDEF) { return false; } + + // It is assumed that 'theta' and 'a' are in the range [-pi,pi] + double n_theta = theta - a; + if(n_theta < 0) { n_theta = -n_theta; } + if(n_theta > M_3_2_PI) + { + n_theta -= M_2__PI; + if(n_theta < 0) n_theta = -n_theta; + } + + return n_theta <= prec; +} + + void LineSegmentDetectorImpl::drawSegments(InputOutputArray _image, InputArray lines) { CV_INSTRUMENT_REGION(); diff --git a/modules/imgproc/test/test_lsd.cpp b/modules/imgproc/test/test_lsd.cpp index e162a3c6f4..43d00b4928 100644 --- a/modules/imgproc/test/test_lsd.cpp +++ b/modules/imgproc/test/test_lsd.cpp @@ -5,8 +5,6 @@ namespace opencv_test { namespace { -#if 0 // LSD implementation has been removed due original code license issues - const Size img_size(640, 480); const int LSD_TEST_SEED = 0x134679; const int EPOCHS = 20; @@ -404,6 +402,4 @@ TEST_F(Imgproc_LSD_Common, compareSegmentsVec4i) ASSERT_EQ(result2, 11); } -#endif - }} // namespace diff --git a/samples/cpp/lsd_lines.cpp b/samples/cpp/lsd_lines.cpp new file mode 100644 index 0000000000..3feed9cbc2 --- /dev/null +++ b/samples/cpp/lsd_lines.cpp @@ -0,0 +1,73 @@ +#include "opencv2/imgproc.hpp" +#include "opencv2/imgcodecs.hpp" +#include "opencv2/highgui.hpp" +#include + +using namespace std; +using namespace cv; + +int main(int argc, char** argv) +{ + cv::CommandLineParser parser(argc, argv, + "{input i|building.jpg|input image}" + "{refine r|false|if true use LSD_REFINE_STD method, if false use LSD_REFINE_NONE method}" + "{canny c|false|use Canny edge detector}" + "{overlay o|false|show result on input image}" + "{help h|false|show help message}"); + + if (parser.get("help")) + { + parser.printMessage(); + return 0; + } + + parser.printMessage(); + + String filename = samples::findFile(parser.get("input")); + bool useRefine = parser.get("refine"); + bool useCanny = parser.get("canny"); + bool overlay = parser.get("overlay"); + + Mat image = imread(filename, IMREAD_GRAYSCALE); + + if( image.empty() ) + { + cout << "Unable to load " << filename; + return 1; + } + + imshow("Source Image", image); + + if (useCanny) + { + Canny(image, image, 50, 200, 3); // Apply Canny edge detector + } + + // Create and LSD detector with standard or no refinement. + Ptr ls = useRefine ? createLineSegmentDetector(LSD_REFINE_STD) : createLineSegmentDetector(LSD_REFINE_NONE); + + double start = double(getTickCount()); + vector lines_std; + + // Detect the lines + ls->detect(image, lines_std); + + double duration_ms = (double(getTickCount()) - start) * 1000 / getTickFrequency(); + std::cout << "It took " << duration_ms << " ms." << std::endl; + + // Show found lines + if (!overlay || useCanny) + { + image = Scalar(0, 0, 0); + } + + ls->drawSegments(image, lines_std); + + String window_name = useRefine ? "Result - standard refinement" : "Result - no refinement"; + window_name += useCanny ? " - Canny edge detector used" : ""; + + imshow(window_name, image); + + waitKey(); + return 0; +} From 6d83a73858b2b1103e103ad0d4c51b147cab41db Mon Sep 17 00:00:00 2001 From: Anatoliy Talamanov Date: Fri, 1 Oct 2021 13:07:55 +0300 Subject: [PATCH 206/262] Fix bugs with hanging frames --- modules/gapi/src/backends/ie/giebackend.cpp | 6 +- .../gapi/src/executor/gstreamingexecutor.cpp | 4 + .../gapi/test/infer/gapi_infer_ie_test.cpp | 135 ++++++++++++++++++ 3 files changed, 144 insertions(+), 1 deletion(-) diff --git a/modules/gapi/src/backends/ie/giebackend.cpp b/modules/gapi/src/backends/ie/giebackend.cpp index 007f0db7af..03584c9561 100644 --- a/modules/gapi/src/backends/ie/giebackend.cpp +++ b/modules/gapi/src/backends/ie/giebackend.cpp @@ -630,6 +630,11 @@ void cv::gimpl::ie::RequestPool::callback(cv::gimpl::ie::RequestPool::Task task, InferenceEngine::InferRequest& request, size_t id) { task.callback(request); + // NB: IE::InferRequest keeps the callback until the new one is set. + // Since user's callback might keep resources that should be released, + // need to destroy its after execution. + // Let's set the empty one to cause the destruction of a callback. + request.SetCompletionCallback([](){}); m_idle_ids.push(id); } @@ -831,7 +836,6 @@ static void PostOutputs(InferenceEngine::InferRequest &request, auto output = ctx->output(i); ctx->out.meta(output, ctx->input(0).meta); ctx->out.post(std::move(output)); - } } diff --git a/modules/gapi/src/executor/gstreamingexecutor.cpp b/modules/gapi/src/executor/gstreamingexecutor.cpp index 2379e3e164..bc43db8a37 100644 --- a/modules/gapi/src/executor/gstreamingexecutor.cpp +++ b/modules/gapi/src/executor/gstreamingexecutor.cpp @@ -347,6 +347,10 @@ bool QueueReader::getInputVector(std::vector &in_queues, cv::GRunArgs &in_constants, cv::GRunArgs &isl_inputs) { + // NB: Need to release resources from the previous step, to fetch new ones. + // On some systems it might be impossible to allocate new memory + // until the old one is released. + m_cmd.clear(); // NOTE: in order to maintain the GRunArg's underlying object // lifetime, keep the whole cmd vector (of size == # of inputs) // in memory. diff --git a/modules/gapi/test/infer/gapi_infer_ie_test.cpp b/modules/gapi/test/infer/gapi_infer_ie_test.cpp index e33f165ae1..b7ea891b81 100644 --- a/modules/gapi/test/infer/gapi_infer_ie_test.cpp +++ b/modules/gapi/test/infer/gapi_infer_ie_test.cpp @@ -9,6 +9,8 @@ #ifdef HAVE_INF_ENGINE #include +#include +#include #include @@ -2052,6 +2054,139 @@ TEST(IEFrameAdapter, blobParams) EXPECT_EQ(expected, actual); } +namespace +{ + +struct Sync { + std::mutex m; + std::condition_variable cv; + int counter = 0; +}; + +class GMockMediaAdapter final: public cv::MediaFrame::IAdapter { +public: + explicit GMockMediaAdapter(cv::Mat m, Sync& sync) + : m_mat(m), m_sync(sync) { + } + + cv::GFrameDesc meta() const override { + return cv::GFrameDesc{cv::MediaFormat::BGR, m_mat.size()}; + } + + cv::MediaFrame::View access(cv::MediaFrame::Access) override { + cv::MediaFrame::View::Ptrs pp = { m_mat.ptr(), nullptr, nullptr, nullptr }; + cv::MediaFrame::View::Strides ss = { m_mat.step, 0u, 0u, 0u }; + return cv::MediaFrame::View(std::move(pp), std::move(ss)); + } + + ~GMockMediaAdapter() { + { + std::lock_guard lk{m_sync.m}; + m_sync.counter--; + } + m_sync.cv.notify_one(); + } + +private: + cv::Mat m_mat; + Sync& m_sync; +}; + +// NB: This source is needed to simulate real +// cases where the memory resources are limited. +// GMockSource(int limit) - accept the number of MediaFrames that +// the source can produce until resources are over. +class GMockSource : public cv::gapi::wip::IStreamSource { +public: + explicit GMockSource(int limit) + : m_limit(limit), m_mat(cv::Size(1920, 1080), CV_8UC3) { + cv::randu(m_mat, cv::Scalar::all(0), cv::Scalar::all(255)); + } + + bool pull(cv::gapi::wip::Data& data) { + std::unique_lock lk(m_sync.m); + m_sync.counter++; + // NB: Can't produce new frames until old ones are released. + m_sync.cv.wait(lk, [this]{return m_sync.counter <= m_limit;}); + + data = cv::MediaFrame::Create(m_mat, m_sync); + return true; + } + + GMetaArg descr_of() const override { + return GMetaArg{cv::GFrameDesc{cv::MediaFormat::BGR, m_mat.size()}}; + } + +private: + int m_limit; + cv::Mat m_mat; + Sync m_sync; +}; + +struct LimitedSourceInfer: public ::testing::Test { + using AGInfo = std::tuple; + G_API_NET(AgeGender, , "test-age-gender"); + + LimitedSourceInfer() + : comp([](){ + cv::GFrame in; + cv::GMat age, gender; + std::tie(age, gender) = cv::gapi::infer(in); + return cv::GComputation(cv::GIn(in), cv::GOut(age, gender)); + }) { + initDLDTDataPath(); + } + + GStreamingCompiled compileStreaming(int nireq) { + cv::gapi::ie::detail::ParamDesc params; + params.model_path = findDataFile(SUBDIR + "age-gender-recognition-retail-0013.xml"); + params.weights_path = findDataFile(SUBDIR + "age-gender-recognition-retail-0013.bin"); + params.device_id = "CPU"; + + auto pp = cv::gapi::ie::Params { + params.model_path, params.weights_path, params.device_id } + .cfgOutputLayers({ "age_conv3", "prob" }) + .cfgNumRequests(nireq); + + return comp.compileStreaming(cv::compile_args(cv::gapi::networks(pp))); + } + + void run(const int max_frames, const int limit, const int nireq) { + auto pipeline = compileStreaming(nireq); + pipeline.setSource(limit); + pipeline.start(); + + int num_frames = 0; + while (num_frames != max_frames && + pipeline.pull(cv::gout(out_age, out_gender))) { + ++num_frames; + } + } + + cv::GComputation comp; + cv::Mat out_age, out_gender; +}; + +} // anonymous namespace + +TEST_F(LimitedSourceInfer, ReleaseFrame) +{ + constexpr int max_frames = 50; + constexpr int resources_limit = 1; + constexpr int nireq = 1; + + run(max_frames, resources_limit, nireq); +} + +TEST_F(LimitedSourceInfer, ReleaseFrameAsync) +{ + constexpr int max_frames = 50; + constexpr int resources_limit = 4; + constexpr int nireq = 8; + + run(max_frames, resources_limit, nireq); +} + } // namespace opencv_test #endif // HAVE_INF_ENGINE From 1e74f5850bec99979584e0e37eeeab8c133be3a9 Mon Sep 17 00:00:00 2001 From: Tomoaki Teshima Date: Fri, 1 Oct 2021 23:17:02 +0900 Subject: [PATCH 207/262] suppress GaussianBlur to generate empty images * sharp Gaussian kernel causes over flow and ends up in blank image --- modules/cudafilters/test/test_filters.cpp | 4 ++-- modules/imgproc/src/smooth.simd.hpp | 5 ++++- modules/imgproc/test/test_smooth_bitexact.cpp | 11 +++++++++++ 3 files changed, 17 insertions(+), 3 deletions(-) diff --git a/modules/cudafilters/test/test_filters.cpp b/modules/cudafilters/test/test_filters.cpp index a86d8c9fff..bb235dad09 100644 --- a/modules/cudafilters/test/test_filters.cpp +++ b/modules/cudafilters/test/test_filters.cpp @@ -445,8 +445,8 @@ PARAM_TEST_CASE(GaussianBlur, cv::cuda::DeviceInfo, cv::Size, MatDepth, Channels CUDA_TEST_P(GaussianBlur, Accuracy) { cv::Mat src = randomMat(size, type); - double sigma1 = randomDouble(0.1, 1.0); - double sigma2 = randomDouble(0.1, 1.0); + double sigma1 = randomDouble(0.0, 1.0); + double sigma2 = randomDouble(0.0, 1.0); cv::Ptr gauss = cv::cuda::createGaussianFilter(src.type(), -1, ksize, sigma1, sigma2, borderType); diff --git a/modules/imgproc/src/smooth.simd.hpp b/modules/imgproc/src/smooth.simd.hpp index 3a39765c71..62ff31ac94 100644 --- a/modules/imgproc/src/smooth.simd.hpp +++ b/modules/imgproc/src/smooth.simd.hpp @@ -1958,7 +1958,10 @@ public: } else if (kxlen % 2 == 1) { - hlineSmoothFunc = hlineSmoothONa_yzy_a; + if (kx[(kxlen - 1)/ 2] == FT::one()) + hlineSmoothFunc = hlineSmooth1N1; + else + hlineSmoothFunc = hlineSmoothONa_yzy_a; for (int i = 0; i < kxlen / 2; i++) if (!(kx[i] == kx[kxlen - 1 - i])) { diff --git a/modules/imgproc/test/test_smooth_bitexact.cpp b/modules/imgproc/test/test_smooth_bitexact.cpp index 246f1df798..d4ae2af833 100644 --- a/modules/imgproc/test/test_smooth_bitexact.cpp +++ b/modules/imgproc/test/test_smooth_bitexact.cpp @@ -249,4 +249,15 @@ TEST(GaussianBlur_Bitexact, regression_9863) checkGaussianBlur_8Uvs32F(src8u, src32f, 151, 30); } +TEST(GaussianBlur_Bitexact, overflow_20792) +{ + Mat src(128, 128, CV_16UC1, Scalar(255)); + Mat dst; + double sigma = theRNG().uniform(0.0, 0.2); // a peaky kernel + GaussianBlur(src, dst, Size(7, 7), sigma, 0.9); + int count = (int)countNonZero(dst); + int nintyPercent = (int)(src.rows*src.cols * 0.9); + EXPECT_GT(count, nintyPercent); +} + }} // namespace From f977d10a19e1da13532ea038ac8d7e2cf882040f Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Fri, 1 Oct 2021 18:51:01 +0000 Subject: [PATCH 208/262] dnn(ocl): fix conv DWCONV workgroup --- .../dnn/src/ocl4dnn/src/ocl4dnn_conv_spatial.cpp | 15 +++++++++------ modules/dnn/src/opencl/conv_layer_spatial.cl | 11 +++++++---- 2 files changed, 16 insertions(+), 10 deletions(-) diff --git a/modules/dnn/src/ocl4dnn/src/ocl4dnn_conv_spatial.cpp b/modules/dnn/src/ocl4dnn/src/ocl4dnn_conv_spatial.cpp index a1164273ac..45bd249e5d 100644 --- a/modules/dnn/src/ocl4dnn/src/ocl4dnn_conv_spatial.cpp +++ b/modules/dnn/src/ocl4dnn/src/ocl4dnn_conv_spatial.cpp @@ -1034,12 +1034,15 @@ bool OCL4DNNConvSpatial::convolve(const UMat &bottom, UMat &top, kernel.set(argIdx++, (uint16_t)output_w_); kernel.set(argIdx++, (uint16_t)output_h_); - size_t global_size[3]; - global_size[0] = output_w_; - global_size[1] = output_h_; - global_size[2] = num_output_ * num_; - - if (!kernel.run_(3, global_size, NULL, false)) + size_t wgs = kernel.workGroupSize(); + if (!wgs) + { + CV_LOG_ERROR(NULL, "DNN/OpenCL: Can't query workGroupSize of DWCONV kernel"); + return false; + } + size_t lws[1] = { wgs }; + size_t gws[1] = { roundUp((size_t)output_w_ * output_h_ * num_output_ * num_, (unsigned)lws[0]) }; + if (!kernel.run_(1, gws, lws, false)) { CV_LOG_ERROR(NULL, "DNN/OpenCL: DWCONV kernel run failed"); return false; diff --git a/modules/dnn/src/opencl/conv_layer_spatial.cl b/modules/dnn/src/opencl/conv_layer_spatial.cl index 455f0ed7ea..eb5d354020 100644 --- a/modules/dnn/src/opencl/conv_layer_spatial.cl +++ b/modules/dnn/src/opencl/conv_layer_spatial.cl @@ -1850,10 +1850,13 @@ __kernel void DWCONV( const ushort output_width, const ushort output_height) { __global Dtype* convolved_image = convolved_image_base + convolved_image_offset; - const int outputX = get_global_id(0); - const int outputY = get_global_id(1); - const int outputZ = get_global_id(2); - if(outputX < output_width && outputY < output_height) + const int out_idx = get_global_id(0); // 1D task layout: [output_width * output_height * OUTPUT_Z] + const int plane_size = output_width * output_height; + const int out_plane_idx = out_idx % plane_size; + const int outputZ = out_idx / plane_size; + const int outputY = out_plane_idx / output_width; + const int outputX = out_plane_idx % output_width; + if (outputZ < OUTPUT_Z) { Dtype sum = 0.; From 4d87f6025eb4be3b2f2a27fb542b5c46fce60193 Mon Sep 17 00:00:00 2001 From: shengyu Date: Wed, 29 Sep 2021 21:02:40 +0800 Subject: [PATCH 209/262] remove redundant semicolons --- modules/gapi/include/opencv2/gapi/own/types.hpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/modules/gapi/include/opencv2/gapi/own/types.hpp b/modules/gapi/include/opencv2/gapi/own/types.hpp index 53bb867e32..6bd6880982 100644 --- a/modules/gapi/include/opencv2/gapi/own/types.hpp +++ b/modules/gapi/include/opencv2/gapi/own/types.hpp @@ -27,7 +27,7 @@ class Point { public: Point() = default; - Point(int _x, int _y) : x(_x), y(_y) {}; + Point(int _x, int _y) : x(_x), y(_y) {} int x = 0; int y = 0; @@ -37,7 +37,7 @@ class Point2f { public: Point2f() = default; - Point2f(float _x, float _y) : x(_x), y(_y) {}; + Point2f(float _x, float _y) : x(_x), y(_y) {} float x = 0.f; float y = 0.f; @@ -47,9 +47,9 @@ class Rect { public: Rect() = default; - Rect(int _x, int _y, int _width, int _height) : x(_x), y(_y), width(_width), height(_height) {}; + Rect(int _x, int _y, int _width, int _height) : x(_x), y(_y), width(_width), height(_height) {} #if !defined(GAPI_STANDALONE) - Rect(const cv::Rect& other) : x(other.x), y(other.y), width(other.width), height(other.height) {}; + Rect(const cv::Rect& other) : x(other.x), y(other.y), width(other.width), height(other.height) {} inline Rect& operator=(const cv::Rect& other) { x = other.x; @@ -104,9 +104,9 @@ class Size { public: Size() = default; - Size(int _width, int _height) : width(_width), height(_height) {}; + Size(int _width, int _height) : width(_width), height(_height) {} #if !defined(GAPI_STANDALONE) - Size(const cv::Size& other) : width(other.width), height(other.height) {}; + Size(const cv::Size& other) : width(other.width), height(other.height) {} inline Size& operator=(const cv::Size& rhs) { width = rhs.width; From 259c39a63aa0be0d1bcda6bc6741548b24ecd13d Mon Sep 17 00:00:00 2001 From: Suleyman TURKMEN Date: Sat, 2 Oct 2021 13:32:11 +0300 Subject: [PATCH 210/262] additional changes --- modules/imgproc/include/opencv2/imgproc.hpp | 7 +++++-- modules/imgproc/src/lsd.cpp | 4 ++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/modules/imgproc/include/opencv2/imgproc.hpp b/modules/imgproc/include/opencv2/imgproc.hpp index 855cbaf159..576bebc21b 100644 --- a/modules/imgproc/include/opencv2/imgproc.hpp +++ b/modules/imgproc/include/opencv2/imgproc.hpp @@ -1228,6 +1228,9 @@ An example using the LineSegmentDetector /** @brief Line segment detector class following the algorithm described at @cite Rafael12 . + +@note Implementation has been removed from OpenCV version 3.4.6 to 3.4.15 and version 4.1.0 to 4.5.3 due original code license conflict. +restored again after [Computation of a NFA](https://github.com/rafael-grompone-von-gioi/binomial_nfa) code published under the MIT license. */ class CV_EXPORTS_W LineSegmentDetector : public Algorithm { @@ -1241,8 +1244,8 @@ public: @param image A grayscale (CV_8UC1) input image. If only a roi needs to be selected, use: `lsd_ptr-\>detect(image(roi), lines, ...); lines += Scalar(roi.x, roi.y, roi.x, roi.y);` - @param lines A vector of Vec4i or Vec4f elements specifying the beginning and ending point of a line. Where - Vec4i/Vec4f is (x1, y1, x2, y2), point 1 is the start, point 2 - end. Returned lines are strictly + @param lines A vector of Vec4f elements specifying the beginning and ending point of a line. Where + Vec4f is (x1, y1, x2, y2), point 1 is the start, point 2 - end. Returned lines are strictly oriented depending on the gradient. @param width Vector of widths of the regions, where the lines are found. E.g. Width of line. @param prec Vector of precisions with which the lines are found. diff --git a/modules/imgproc/src/lsd.cpp b/modules/imgproc/src/lsd.cpp index d06759c2bb..c4a77ee265 100644 --- a/modules/imgproc/src/lsd.cpp +++ b/modules/imgproc/src/lsd.cpp @@ -191,8 +191,8 @@ public: * If only a roi needs to be selected, use * lsd_ptr->detect(image(roi), ..., lines); * lines += Scalar(roi.x, roi.y, roi.x, roi.y); - * @param _lines Return: A vector of Vec4i or Vec4f elements specifying the beginning and ending point of a line. - * Where Vec4i/Vec4f is (x1, y1, x2, y2), point 1 is the start, point 2 - end. + * @param _lines Return: A vector of Vec4f elements specifying the beginning and ending point of a line. + * Where Vec4f is (x1, y1, x2, y2), point 1 is the start, point 2 - end. * Returned lines are strictly oriented depending on the gradient. * @param width Return: Vector of widths of the regions, where the lines are found. E.g. Width of line. * @param prec Return: Vector of precisions with which the lines are found. From ef53a9229f4f666459480b2366a4680921d45e5a Mon Sep 17 00:00:00 2001 From: Jannik Bamberger Date: Sat, 2 Oct 2021 00:04:23 +0200 Subject: [PATCH 211/262] Automatically set the correct OpenCV version in build.gradle Automatically sets the correct OpenCV version in the CMAKE example contained in the build.gradle file of the Android SDK. --- modules/java/android_sdk/build.gradle.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/java/android_sdk/build.gradle.in b/modules/java/android_sdk/build.gradle.in index 6d317fcd39..14a7b2c6fe 100644 --- a/modules/java/android_sdk/build.gradle.in +++ b/modules/java/android_sdk/build.gradle.in @@ -58,7 +58,7 @@ // // - Use find_package() in app/CMakeLists.txt: // -// find_package(OpenCV 3.4 REQUIRED java) +// find_package(OpenCV @OPENCV_VERSION_MAJOR@.@OPENCV_VERSION_MINOR@ REQUIRED java) // ... // target_link_libraries(native-lib ${OpenCV_LIBRARIES}) // From de5b6386e01d2e6ea622913627b8776c73d6ada3 Mon Sep 17 00:00:00 2001 From: Jonas Vautherin Date: Sat, 2 Oct 2021 03:07:25 +0200 Subject: [PATCH 212/262] Fix gst_initializer Use the return value of gst_init_check instead of testing the error pointer --- modules/videoio/src/cap_gstreamer.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/modules/videoio/src/cap_gstreamer.cpp b/modules/videoio/src/cap_gstreamer.cpp index 6bad528909..d8390691ff 100644 --- a/modules/videoio/src/cap_gstreamer.cpp +++ b/modules/videoio/src/cap_gstreamer.cpp @@ -216,10 +216,10 @@ private: call_deinit = utils::getConfigurationParameterBool("OPENCV_VIDEOIO_GSTREAMER_CALL_DEINIT", false); GSafePtr err; - gst_init_check(NULL, NULL, err.getRef()); - if (err) + gboolean gst_init_res = gst_init_check(NULL, NULL, err.getRef()); + if (!gst_init_res) { - CV_WARN("Can't initialize GStreamer: " << err->message); + CV_WARN("Can't initialize GStreamer: " << (err ? err->message : "")); isFailed = true; return; } From 87ebf2e50bc468356493dcc6860c482c902ad69e Mon Sep 17 00:00:00 2001 From: SamFC10 Date: Sun, 3 Oct 2021 15:16:01 +0530 Subject: [PATCH 213/262] fix illegal memory access in int8 convolution --- .../dnn/src/int8layers/convolution_layer.cpp | 26 ++++++++++++------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/modules/dnn/src/int8layers/convolution_layer.cpp b/modules/dnn/src/int8layers/convolution_layer.cpp index 05749885c0..ea29610222 100644 --- a/modules/dnn/src/int8layers/convolution_layer.cpp +++ b/modules/dnn/src/int8layers/convolution_layer.cpp @@ -159,7 +159,7 @@ public: enum { VEC_ALIGN = 32, DFT_TYPE = CV_8S }; Mat weightsMat; std::vector biasvec; - Mat outputMultiplier; + std::vector outputMultiplier; Mat activationLUT; Ptr activ; @@ -249,10 +249,14 @@ public: Mat biasMat = blobs[1]; biasvec.resize(numOutput+2); - for(int i = 0; i < numOutput; i++ ) - biasvec[i] = biasMat.at(i); - outputMultiplier = blobs[2]; + Mat outMult = blobs[2]; + outputMultiplier.resize(numOutput+2); + for(int i = 0; i < numOutput; i++ ) + { + biasvec[i] = biasMat.at(i); + outputMultiplier[i] = outMult.at(i); + } } bool setActivation(const Ptr& layer) CV_OVERRIDE @@ -283,16 +287,17 @@ public: for (int i = 0; i < outCn; ++i) { - float off = outputMultiplier.at(i) * output_sc; + float off = outputMultiplier[i] * output_sc; if (!w.empty()) off *= w.at(i); if (!b.empty()) biasvec[i] += (int)std::round(b.at(i)/off); - outputMultiplier.at(i) = off/new_sc; + outputMultiplier[i] = off/new_sc; } biasvec[outCn] = biasvec[outCn+1] = biasvec[outCn-1]; + outputMultiplier[outCn] = outputMultiplier[outCn+1] = outputMultiplier[outCn-1]; } class ParallelConv : public cv::ParallelLoopBody @@ -315,7 +320,7 @@ public: bool useAVX512; int blk_size_cn; int inpZp, outZp; - const float* multiplier; + const std::vector* multiplier; ParallelConv() : input_(0), weights_(0), output_(0), ngroups_(0), nstripes_(0), @@ -323,7 +328,7 @@ public: , blk_size_cn(0), inpZp(0), outZp(0), multiplier(0) {} - static void run( const Mat& input, Mat& output, const Mat& weights, const Mat& multipliers, + static void run( const Mat& input, Mat& output, const Mat& weights, const std::vector& multipliers, const std::vector& biasvec, const Mat& activLUT, const std::vector& kernel_size, const std::vector& strides, const std::vector& pads_begin, const std::vector& pads_end, @@ -392,7 +397,7 @@ public: p.inpZp = inp_Zp; p.outZp = out_Zp; - p.multiplier = multipliers.ptr(0); + p.multiplier = &multipliers; p.ofstab_.resize(karea * ncn); int* ofstab = &p.ofstab_[0]; @@ -501,6 +506,7 @@ public: const int8_t* wptr_orig_ = weights_->ptr(); size_t wstep = weights_->step1(); const int* biasptr_ = &biasvec_->at(0); + const float* multptr_ = &multiplier->at(0); const int* lutptr_ = !activLUT_->empty() ? activLUT_->ptr() : 0; int* data_out0_ = output_->ptr(); AutoBuffer rowbuf0_; @@ -539,7 +545,7 @@ public: int startOutCn = (subsampleIdx % ngroups)*outCn; const int8_t* wptr_orig = wptr_orig_ + wstep*startOutCn; const int* biasptr = biasptr_ + startOutCn; - const float* multptr = multiplier + startOutCn; + const float* multptr = multptr_ + startOutCn; for( int cn0 = 0; cn0 < inpCn; cn0 += blk_size_cn ) { From 505dde09de0b245bb9958d42673671761a690be1 Mon Sep 17 00:00:00 2001 From: YashasSamaga Date: Mon, 4 Oct 2021 12:38:45 +0530 Subject: [PATCH 214/262] support broadcasting in eltwise ops --- modules/dnn/src/cuda/eltwise_ops.cu | 273 ++++++++++++++++-- modules/dnn/src/cuda/kernel_dispatcher.hpp | 18 ++ .../dnn/src/cuda4dnn/kernels/eltwise_ops.hpp | 14 +- modules/dnn/src/dnn.cpp | 14 +- 4 files changed, 283 insertions(+), 36 deletions(-) diff --git a/modules/dnn/src/cuda/eltwise_ops.cu b/modules/dnn/src/cuda/eltwise_ops.cu index 109c3fbdc9..db4daa4718 100644 --- a/modules/dnn/src/cuda/eltwise_ops.cu +++ b/modules/dnn/src/cuda/eltwise_ops.cu @@ -5,13 +5,16 @@ #include #include +#include "array.hpp" #include "functors.hpp" #include "grid_stride_range.hpp" #include "execution.hpp" #include "vector_traits.hpp" +#include "kernel_dispatcher.hpp" #include "../cuda4dnn/csl/stream.hpp" #include "../cuda4dnn/csl/span.hpp" +#include "../cuda4dnn/csl/tensor.hpp" #include @@ -40,6 +43,32 @@ namespace raw { v_store(output_vPtr[i], vec_x); } } + + template + __global__ void eltwise_op_bcast( + Span output, array out_strides, + View x, array x_strides, array x_bcast, + View y, array y_strides, array y_bcast, + const typename EltwiseOp::Params params) { + EltwiseOp eltwise_op(params); + + for (auto i : grid_stride_range(output.size())) { + index_type out_index = i / out_strides[0]; + index_type x_index = x_bcast[0] ? 0 : out_index * x_strides[0]; + index_type y_index = y_bcast[0] ? 0 : out_index * y_strides[0]; + + for (int j = 1; j < Rank; j++) + { + out_index = (i % out_strides[j - 1]) / out_strides[j]; + if (!x_bcast[j]) + x_index += out_index * x_strides[j]; + if (!y_bcast[j]) + y_index += out_index * y_strides[j]; + } + + output[i] = eltwise_op(x[x_index], y[y_index]); + } + } } template static @@ -55,63 +84,251 @@ void launch_vectorized_eltwise_op(const Stream& stream, Span output, View launch_kernel(kernel, policy, output, x, y, params); } -template static -void eltwise_op(const Stream& stream, Span output, View x, View y, const typename EltwiseOp::Params& params = {}) { - CV_Assert(x.size() == y.size()); - CV_Assert(x.size() == output.size()); +template static +void launch_eltwise_op_bcast( + const Stream& stream, + Span output, const std::vector& outStride, + View x, const std::vector& inStride1, const std::vector& inBcast1, + View y, const std::vector& inStride2, const std::vector& inBcast2, + const typename EltwiseOp::Params& params) +{ + CV_Assert(outStride.size() == Rank); + CV_Assert(inStride1.size() == Rank); + CV_Assert(inStride2.size() == Rank); + CV_Assert(inBcast1.size() == Rank); + CV_Assert(inBcast2.size() == Rank); - if (is_fully_aligned(output, 4) && is_fully_aligned(x, 4) && is_fully_aligned(y, 4)) { - launch_vectorized_eltwise_op(stream, output, x, y, params); - } else if (is_fully_aligned(output, 2) && is_fully_aligned(x, 2) && is_fully_aligned(y, 2)) { - launch_vectorized_eltwise_op(stream, output, x, y, params); - } else { - launch_vectorized_eltwise_op(stream, output, x, y, params); + array outStride_k, inStride1_k, inStride2_k; + outStride_k.assign(std::begin(outStride), std::end(outStride)); + inStride1_k.assign(std::begin(inStride1), std::end(inStride1)); + inStride2_k.assign(std::begin(inStride2), std::end(inStride2)); + + array inBcast1_k, inBcast2_k; + inBcast1_k.assign(std::begin(inBcast1), std::end(inBcast1)); + inBcast2_k.assign(std::begin(inBcast2), std::end(inBcast2)); + + auto kernel = raw::eltwise_op_bcast; + auto policy = make_policy(kernel, output.size(), 0, stream); + launch_kernel(kernel, policy, output, outStride_k, x, inStride1_k, inBcast1_k, y, inStride2_k, inBcast2_k, params); +} + +GENERATE_KERNEL_DISPATCHER_2TP(eltwise_op_bcast_dispatcher, launch_eltwise_op_bcast); + +template static +void eltwise_op(const Stream& stream, TensorSpan output, TensorView x, TensorView y, const typename EltwiseOp::Params& params = {}) { + if (is_shape_same(output, x) && is_shape_same(output, y)) + { + /* no broadcasting; use fast path */ + CV_Assert(x.size() == y.size()); + CV_Assert(x.size() == output.size()); + + if (is_fully_aligned(output, 4) && is_fully_aligned(x, 4) && is_fully_aligned(y, 4)) { + launch_vectorized_eltwise_op(stream, output, x, y, params); + } else if (is_fully_aligned(output, 2) && is_fully_aligned(x, 2) && is_fully_aligned(y, 2)) { + launch_vectorized_eltwise_op(stream, output, x, y, params); + } else { + launch_vectorized_eltwise_op(stream, output, x, y, params); + } + } + else + { + CV_Assert(is_shape_compatible(output, x)); + CV_Assert(is_shape_compatible(output, y)); + + /* matching singleton axes in both input tensors can be eliminated + * + * Reasoning: + * ---------- + * Singleton axes do not contribute towards address calculation. They are redundant + * unless there is broadcasting. If both input tensors have singleton axis at a + * specified position, there is no broadcasting on that axis. + * + * Example: + * --------- + * x: [1, 256, 32, 32] -> [256, 32, 32] + * y: [1, 256, 1, 1] -> [256, 1, 1] + */ + for (int r = 0; r < output.rank(); r++) + { + while (x.get_axis_size(r) == 1 && y.get_axis_size(r) == 1) { + CV_Assert(output.get_axis_size(r) == 1); + + x.squeeze(r); + y.squeeze(r); + output.squeeze(r); + } + } + + auto inShape1 = x.shape_as_vector(); + auto inShape2 = y.shape_as_vector(); + auto outShape = output.shape_as_vector(); + + /* contiguous axes that do not broadcast can be merged into one axis + * + * Example: + * --------- + * x: [32, 8, 8] -> [32, 64] + * y: [1, 8, 8] -> [1, 64] + */ + for (int i = 0; i < inShape1.size(); i++) { + /* check if axis `i` requires any broadcasting */ + if (inShape1[i] == inShape2[i]) { + /* loop invariant: `i` is the first axis in the contiguous axis sequence */ + + int j = i + 1; /* `j` is the axis which we will attempt to merge */ + while (j < inShape1.size() && inShape1[j] == inShape2[j]) { + CV_Assert(outShape[j] == inShape1[j]); + + /* `j` axis is also used fully; merge `i` and `j` */ + auto new_size = inShape1[i] * inShape1[j]; + inShape1[i] = new_size; + inShape2[i] = new_size; + + /* delete axis `j` */ + inShape1.erase(std::begin(inShape1) + j); + inShape2.erase(std::begin(inShape2) + j); + outShape.erase(std::begin(outShape) + j); + + /* optimizations should not break the invariants */ + CV_Assert(inShape1.size() == outShape.size()); + CV_Assert(inShape2.size() == outShape.size()); + CV_Assert(inShape1[i] == outShape[i]); + CV_Assert(inShape2[i] == outShape[i]); + } + } + } + + /* contiguous broadcasting axes on the same tensor can be merged into one axis + * + * Example: + * --------- + * x: [256, 8, 8] -> [256, 64] + * y: [256, 1, 1] -> [256, 1] + */ + for (int i = 0; i < inShape1.size(); i++) { + /* check if axis `i` requires any broadcasting in tensor 1 */ + if (inShape1[i] == 1 && inShape2[i] != 1) { + /* loop invariant: `i` is the first axis in the contiguous axis sequence */ + + int j = i + 1; /* `j` is the axis which we will attempt to merge */ + while (j < inShape1.size() && inShape1[j] == 1 && inShape2[j] != 1) { + CV_Assert(outShape[j] == inShape2[j]); + + /* `j` axis is also used fully; merge `i` and `j` */ + inShape1[i] = 1; + inShape2[i] = inShape2[i] * inShape2[j]; + outShape[i] = inShape2[i]; + + /* delete axis `j` */ + inShape1.erase(std::begin(inShape1) + j); + inShape2.erase(std::begin(inShape2) + j); + outShape.erase(std::begin(outShape) + j); + + /* optimizations should not break the invariants */ + CV_Assert(inShape1.size() == outShape.size()); + CV_Assert(inShape2.size() == outShape.size()); + CV_Assert(inShape1[i] == 1); + CV_Assert(inShape2[i] == outShape[i]); + } + } + + /* check if axis `i` requires any broadcasting in tensor 2 */ + if (inShape1[i] != 1 && inShape2[i] == 1) { + /* loop invariant: `i` is the first axis in the contiguous axis sequence */ + + int j = i + 1; /* `j` is the axis which we will attempt to merge */ + while (j < inShape1.size() && inShape1[j] != 1 && inShape2[j] == 1) { + CV_Assert(outShape[j] == inShape1[j]); + + /* `j` axis is also used fully; merge `i` and `j` */ + inShape1[i] = inShape1[i] * inShape1[j]; + inShape2[i] = 1; + outShape[i] = inShape1[i]; + + /* delete axis `j` */ + inShape1.erase(std::begin(inShape1) + j); + inShape2.erase(std::begin(inShape2) + j); + outShape.erase(std::begin(outShape) + j); + + /* optimizations should not break the invariants */ + CV_Assert(inShape1.size() == outShape.size()); + CV_Assert(inShape2.size() == outShape.size()); + CV_Assert(inShape1[i] == outShape[i]); + CV_Assert(inShape2[i] == 1); + } + } + } + + auto rank = outShape.size(); + + std::vector inStride1(rank), inStride2(rank), outStride(rank); + inStride1.back() = 1; + inStride2.back() = 1; + outStride.back() = 1; + /* garbage, ..., garbage, 1 */ + + std::copy(std::begin(inShape1) + 1, std::end(inShape1), std::begin(inStride1)); + std::copy(std::begin(inShape2) + 1, std::end(inShape2), std::begin(inStride2)); + std::copy(std::begin(outShape) + 1, std::end(outShape), std::begin(outStride)); + /* dim[0], dim[1], ..., dim[-1], 1 */ + + std::partial_sum(inStride1.rbegin(), inStride1.rend(), inStride1.rbegin(), std::multiplies()); + std::partial_sum(inStride2.rbegin(), inStride2.rend(), inStride2.rbegin(), std::multiplies()); + std::partial_sum(outStride.rbegin(), outStride.rend(), outStride.rbegin(), std::multiplies()); + /* stride[0], stride[1], ..., stride[-2], 1 */ + + std::vector inBcast1(rank), inBcast2(rank); + std::transform(std::begin(inShape1), std::end(inShape1), std::begin(inBcast1), [](std::size_t sz) { return sz == 1; }); + std::transform(std::begin(inShape2), std::end(inShape2), std::begin(inBcast2), [](std::size_t sz) { return sz == 1; }); + + CV_Assert(1 <= rank && rank <= CSL_MAX_TENSOR_RANK); + eltwise_op_bcast_dispatcher(rank, stream, output, outStride, x, inStride1, inBcast1, y, inStride2, inBcast2, params); } } template -void eltwise_max_2(const Stream& stream, Span output, View x, View y) { +void eltwise_max_2(const Stream& stream, TensorSpan output, TensorView x, TensorView y) { eltwise_op>(stream, output, x, y); } template -void eltwise_min_2(const Stream& stream, Span output, View x, View y) { +void eltwise_min_2(const Stream& stream, TensorSpan output, TensorView x, TensorView y) { eltwise_op>(stream, output, x, y); } template -void eltwise_sum_2(const Stream& stream, Span output, View x, View y) { +void eltwise_sum_2(const Stream& stream, TensorSpan output, TensorView x, TensorView y) { eltwise_op>(stream, output, x, y); } template -void eltwise_sum_coeff_2(const Stream& stream, Span output, T coeff_x, View x, T coeff_y, View y) { +void eltwise_sum_coeff_2(const Stream& stream, TensorSpan output, T coeff_x, TensorView x, T coeff_y, TensorView y) { eltwise_op>(stream, output, x, y, {coeff_x, coeff_y}); } template -void eltwise_prod_2(const Stream& stream, Span output, View x, View y) { +void eltwise_prod_2(const Stream& stream, TensorSpan output, TensorView x, TensorView y) { eltwise_op>(stream, output, x, y); } template -void eltwise_div_2(const Stream& stream, Span output, View x, View y) { +void eltwise_div_2(const Stream& stream, TensorSpan output, TensorView x, TensorView y) { eltwise_op>(stream, output, x, y); } #if !defined(__CUDA_ARCH__) || (__CUDA_ARCH__ >= 530) - template void eltwise_div_2(const Stream& stream, Span<__half> output, View<__half> x, View<__half> y); - template void eltwise_prod_2(const Stream& stream, Span<__half> output, View<__half> x, View<__half> y); - template void eltwise_sum_coeff_2(const Stream&, Span<__half>, __half, View<__half>, __half, View<__half>); - template void eltwise_sum_2(const Stream& stream, Span<__half> output, View<__half> x, View<__half> y); - template void eltwise_max_2(const Stream& stream, Span<__half> output, View<__half> x, View<__half> y); - template void eltwise_min_2(const Stream& stream, Span<__half> output, View<__half> x, View<__half> y); + template void eltwise_div_2(const Stream& stream, TensorSpan<__half> output, TensorView<__half> x, TensorView<__half> y); + template void eltwise_prod_2(const Stream& stream, TensorSpan<__half> output, TensorView<__half> x, TensorView<__half> y); + template void eltwise_sum_coeff_2(const Stream&, TensorSpan<__half>, __half, TensorView<__half>, __half, TensorView<__half>); + template void eltwise_sum_2(const Stream& stream, TensorSpan<__half> output, TensorView<__half> x, TensorView<__half> y); + template void eltwise_max_2(const Stream& stream, TensorSpan<__half> output, TensorView<__half> x, TensorView<__half> y); + template void eltwise_min_2(const Stream& stream, TensorSpan<__half> output, TensorView<__half> x, TensorView<__half> y); #endif - template void eltwise_div_2(const Stream& stream, Span output, View x, View y); - template void eltwise_prod_2(const Stream& stream, Span output, View x, View y); - template void eltwise_sum_coeff_2(const Stream&, Span, float, View, float, View); - template void eltwise_sum_2(const Stream& stream, Span output, View x, View y); - template void eltwise_max_2(const Stream& stream, Span output, View x, View y); - template void eltwise_min_2(const Stream& stream, Span output, View x, View y); + template void eltwise_div_2(const Stream& stream, TensorSpan output, TensorView x, TensorView y); + template void eltwise_prod_2(const Stream& stream, TensorSpan output, TensorView x, TensorView y); + template void eltwise_sum_coeff_2(const Stream&, TensorSpan, float, TensorView, float, TensorView); + template void eltwise_sum_2(const Stream& stream, TensorSpan output, TensorView x, TensorView y); + template void eltwise_max_2(const Stream& stream, TensorSpan output, TensorView x, TensorView y); + template void eltwise_min_2(const Stream& stream, TensorSpan output, TensorView x, TensorView y); }}}} /* namespace cv::dnn::cuda4dnn::kernels */ diff --git a/modules/dnn/src/cuda/kernel_dispatcher.hpp b/modules/dnn/src/cuda/kernel_dispatcher.hpp index 6eff8344d3..b0fc658850 100644 --- a/modules/dnn/src/cuda/kernel_dispatcher.hpp +++ b/modules/dnn/src/cuda/kernel_dispatcher.hpp @@ -73,4 +73,22 @@ name(selector, std::forward(args)...); \ } +// Same as GENERATE_KERNEL_DISPATCHER but takes two class template parameters T and TP1 instead of just T +#define GENERATE_KERNEL_DISPATCHER_2TP(name,func); \ + template static \ + typename std::enable_if \ + ::type name(int selector, Args&& ...args) { \ + if(selector == start) \ + func(std::forward(args)...); \ + } \ + \ + template static \ + typename std::enable_if \ + ::type name(int selector, Args&& ...args) { \ + if(selector == start) \ + func(std::forward(args)...); \ + else \ + name(selector, std::forward(args)...); \ + } + #endif /* OPENCV_DNN_SRC_CUDA_KERNEL_DISPATCHER_HPP */ diff --git a/modules/dnn/src/cuda4dnn/kernels/eltwise_ops.hpp b/modules/dnn/src/cuda4dnn/kernels/eltwise_ops.hpp index 096ba04e48..0e44372fee 100644 --- a/modules/dnn/src/cuda4dnn/kernels/eltwise_ops.hpp +++ b/modules/dnn/src/cuda4dnn/kernels/eltwise_ops.hpp @@ -6,29 +6,29 @@ #define OPENCV_DNN_SRC_CUDA4DNN_KERNELS_ELTWISE_OPS_HPP #include "../csl/stream.hpp" -#include "../csl/span.hpp" +#include "../csl/tensor.hpp" #include namespace cv { namespace dnn { namespace cuda4dnn { namespace kernels { template - void eltwise_max_2(const csl::Stream& stream, csl::Span output, csl::View x, csl::View y); + void eltwise_max_2(const csl::Stream& stream, csl::TensorSpan output, csl::TensorView x, csl::TensorView y); template - void eltwise_min_2(const csl::Stream& stream, csl::Span output, csl::View x, csl::View y); + void eltwise_min_2(const csl::Stream& stream, csl::TensorSpan output, csl::TensorView x, csl::TensorView y); template - void eltwise_sum_2(const csl::Stream& stream, csl::Span output, csl::View x, csl::View y); + void eltwise_sum_2(const csl::Stream& stream, csl::TensorSpan output, csl::TensorView x, csl::TensorView y); template - void eltwise_sum_coeff_2(const csl::Stream& stream, csl::Span output, T coeff_x, csl::View x, T coeff_y, csl::View y); + void eltwise_sum_coeff_2(const csl::Stream& stream, csl::TensorSpan output, T coeff_x, csl::TensorView x, T coeff_y, csl::TensorView y); template - void eltwise_prod_2(const csl::Stream& stream, csl::Span output, csl::View x, csl::View y); + void eltwise_prod_2(const csl::Stream& stream, csl::TensorSpan output, csl::TensorView x, csl::TensorView y); template - void eltwise_div_2(const csl::Stream& stream, csl::Span output, csl::View x, csl::View y); + void eltwise_div_2(const csl::Stream& stream, csl::TensorSpan output, csl::TensorView x, csl::TensorView y); }}}} /* namespace cv::dnn::cuda4dnn::kernels */ diff --git a/modules/dnn/src/dnn.cpp b/modules/dnn/src/dnn.cpp index 4e38b0374f..c8fc7721d4 100644 --- a/modules/dnn/src/dnn.cpp +++ b/modules/dnn/src/dnn.cpp @@ -2710,7 +2710,19 @@ struct Net::Impl : public detail::NetImplBase // we create a temporary backend node for eltwise layer to obtain the eltwise configuration cuda4dnn::csl::CSLContext context; // assume that initCUDA and EltwiseOp do not use the context during init const auto node = nextData->layerInstance->initCUDA(&context, nextData->inputBlobsWrappers, nextData->outputBlobsWrappers); - const auto eltwiseNode = node.dynamicCast(); + auto eltwiseNode = node.dynamicCast(); + + // broadcasting not supported in fused ops + auto required_shape = shape(nextData->outputBlobs[0]); + for (int i = 0; i < nextData->inputBlobs.size(); i++) + { + if (shape(*nextData->inputBlobs[i]) != required_shape) + { + eltwiseNode.reset(); + break; + } + } + // CUDA backend uses EltwiseOp when all operands have the same number of channels; otherwise, ShortcutOp is used. // Hence, a successful cast to EltwiseOp implies that the number of channels is same in all operand tensors. if (eltwiseNode.empty() || eltwiseNode->op != cuda4dnn::EltwiseOpType::SUM || !eltwiseNode->coeffs.empty()) From 62414e3073eeec874305e858b297ed8dba5a1b89 Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Mon, 4 Oct 2021 10:46:32 +0000 Subject: [PATCH 215/262] core(parallel): suppress TSAN warning --- modules/core/src/parallel_impl.cpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/modules/core/src/parallel_impl.cpp b/modules/core/src/parallel_impl.cpp index 7f30c9d5f3..1d9690b04c 100644 --- a/modules/core/src/parallel_impl.cpp +++ b/modules/core/src/parallel_impl.cpp @@ -615,6 +615,11 @@ void ThreadPool::run(const Range& range, const ParallelLoopBody& body, double ns { WorkerThread& thread = *(threads[i].get()); if ( +#if defined(__clang__) && defined(__has_feature) +#if __has_feature(thread_sanitizer) + 1 || // Robust workaround to avoid data race warning of `thread.job` +#endif +#endif #if !defined(CV_USE_GLOBAL_WORKERS_COND_VAR) thread.isActive || #endif From d20c9bde7e427962edd824deae7d63f629d8d871 Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Mon, 4 Oct 2021 09:40:16 +0000 Subject: [PATCH 216/262] core(TLS): force TlsAbstraction initialization before main() --- modules/core/src/system.cpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/modules/core/src/system.cpp b/modules/core/src/system.cpp index df9e8a0ce7..e2284b0415 100644 --- a/modules/core/src/system.cpp +++ b/modules/core/src/system.cpp @@ -1834,6 +1834,12 @@ static void WINAPI opencv_fls_destructor(void* pData) #endif // CV_USE_FLS #endif // _WIN32 +static TlsAbstraction* const g_force_initialization_of_TlsAbstraction +#if defined __GNUC__ + __attribute__((unused)) +#endif + = getTlsAbstraction(); + } // namespace details using namespace details; From d94d469c867b9d5c595b43b67243bfbd6d311d8c Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Mon, 4 Oct 2021 16:27:05 +0000 Subject: [PATCH 217/262] valgrind: update suppressions --- platforms/scripts/valgrind.supp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/platforms/scripts/valgrind.supp b/platforms/scripts/valgrind.supp index 600c61fad0..9229828446 100644 --- a/platforms/scripts/valgrind.supp +++ b/platforms/scripts/valgrind.supp @@ -51,7 +51,7 @@ OpenCV-TLS-getTlsStorage Memcheck:Leak ... - fun:_ZN2cvL13getTlsStorageEv + fun:_ZN2cv*L13getTlsStorageEv } { @@ -159,7 +159,7 @@ OpenCV-DNN-getLayerFactoryMutex Memcheck:Leak ... - fun:_ZN2cv3dnn*L20getLayerFactoryMutexEv + fun:_ZN2cv3dnn*20getLayerFactoryMutexEv } { From 9085b933d81ffd12fbdabffe1ffc69a33580e631 Mon Sep 17 00:00:00 2001 From: Zihao Mu Date: Tue, 5 Oct 2021 00:37:38 +0800 Subject: [PATCH 218/262] Merge pull request #20702 from zihaomu:tf_expand_dim_layer Add ExpandDims layer of tf_importer.cpp * Add ExpandDims to tf_importer. * add -1 expand test case. * Support different dimensions of input. * Compatible with 5-dimensional NDHWC data * Code align * support 3-dim input. * 3-dim bug fixed. * fixing error of code format. --- modules/dnn/src/tensorflow/tf_importer.cpp | 134 ++++++++++++++++++++- modules/dnn/test/test_tf_importer.cpp | 13 ++ 2 files changed, 144 insertions(+), 3 deletions(-) diff --git a/modules/dnn/src/tensorflow/tf_importer.cpp b/modules/dnn/src/tensorflow/tf_importer.cpp index 813407dbd8..9a9cd4692f 100644 --- a/modules/dnn/src/tensorflow/tf_importer.cpp +++ b/modules/dnn/src/tensorflow/tf_importer.cpp @@ -555,7 +555,7 @@ protected: std::map layer_id; private: - void addPermuteLayer(const int* order, const std::string& permName, Pin& inpId); + void addPermuteLayer(const int* order, const std::string& permName, Pin& inpId, int orderSize = 4); void setPadding(LayerParams &layerParams, const tensorflow::NodeDef &layer, std::string& inputName, float value = 0.); friend class TFLayerHandler; @@ -595,6 +595,7 @@ private: void parseClipByValue (tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams); void parseLeakyRelu (tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams); void parseActivation (tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams); + void parseExpandDims (tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams); void parseCustomLayer (tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams); }; @@ -672,6 +673,7 @@ const TFImporter::DispatchMap TFImporter::buildDispatchMap() dispatch["LeakyRelu"] = &TFImporter::parseLeakyRelu; dispatch["Abs"] = dispatch["Tanh"] = dispatch["Sigmoid"] = dispatch["Relu"] = dispatch["Elu"] = dispatch["Exp"] = dispatch["Identity"] = dispatch["Relu6"] = &TFImporter::parseActivation; + dispatch["ExpandDims"] = &TFImporter::parseExpandDims; return dispatch; } @@ -1113,6 +1115,123 @@ void TFImporter::parseReshape(tensorflow::GraphDef& net, const tensorflow::NodeD } } +void TFImporter::parseExpandDims(tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams) +{ + const std::string& name = layer.name(); + const int num_inputs = layer.input_size(); + + CV_Assert(!netInputShapes.empty()); + + CV_CheckGT(num_inputs, 0, ""); + Pin inpId = parsePin(layer.input(0)); + DataLayout inpLayout = getDataLayout(layer.input(0), data_layouts); + + // Get input shape + std::vector inShape_, outShape_; + int inpIdindex = layer_id.find(inpId.name)->second; + + dstNet.getLayerShapes(netInputShapes, inpIdindex, inShape_, outShape_); + MatShape inpShape = outShape_[0]; + std::vector outShape = inpShape; + + int outShapeSize = outShape.size(); + + CV_Assert(inpShape.size() >= 1); + // 2nd blob is dims tensor + int axis = getConstBlob(layer, value_id, 1).int_val().Get(0); + + // Convert negative numbers to positive numbers, axis can be in range [-(D+1), D]. + if(axis < 0) + { + axis = inpShape.size() + axis + 1; + } + + CV_Assert(0 <= axis && axis <= inpShape.size()); + + // After ExpendDims, 3-dim data will become 4-dim data, and OpenCV retains 4-dim data as NCHW data layout. + // Convert OpenCV's NHC to NCH first. + if(outShapeSize == 3) + { + // If axis equal to outShapeSize, that mean we expand in Channel dimmension, and do not add permuteLayer. + if(axis != outShapeSize) + { + int order[] = {0, 2, 1}; // From OpenCV's NHC to NCH. + addPermuteLayer(order, name + "/nch", inpId, 3); + + std::swap(outShape[1], outShape[2]); + } + axis = (axis != 0)?(axis % outShapeSize + 1):2; + } + + if(inpShape.size() == 4) + { + if(axis == inpShape.size()) + { + int order[] = {0, 2, 3, 1}; // From OpenCV's NCHW to NHWC. + addPermuteLayer(order, name + "/nhwc", inpId); + + // Convert shape From OpenCV's NCHW to NHWC. + if(inpLayout == DATA_LAYOUT_NHWC) + { + std::swap(outShape[1], outShape[2]); + std::swap(outShape[2], outShape[3]); + } + } + if(inpLayout == DATA_LAYOUT_NHWC || inpLayout == DATA_LAYOUT_NCHW) + { + // toNCHW + axis = (axis != 0)?(axis % outShapeSize + 1):0; + } + } + + // After ExpendDims, 5-dim data will become 6-dim data, and OpenCV retains 6-dim data as original data layout. + // Convert OpenCV's NCDHW to NDHWC first. + if (inpShape.size() == 5 && (inpLayout == DATA_LAYOUT_NDHWC || inpLayout == DATA_LAYOUT_UNKNOWN)) + { + int order[] = {0, 2, 3, 4, 1}; // From OpenCV's NCDHW to NDHWC. + addPermuteLayer(order, name + "/ndhwc", inpId, 5); + + // Convert shape From OpenCV's NCDHW to NDHWC. + if(inpLayout == DATA_LAYOUT_NDHWC) + { + std::swap(outShape[1], outShape[2]); + std::swap(outShape[2], outShape[3]); + std::swap(outShape[3], outShape[4]); + } + } + + outShape.insert(outShape.begin() + axis, 1); + outShapeSize += 1; + + // From OpenCV's NCDHW to NDHWC. + if((inpLayout != DATA_LAYOUT_NHWC && inpLayout != DATA_LAYOUT_NCHW) && outShapeSize == 5) + { + for(int i = 1; i < outShapeSize - 1; i++) + { + std::swap(outShape[outShapeSize - i - 1], outShape[outShapeSize - i]); + } + } + + layerParams.set("dim", DictValue::arrayInt(&outShape[0], outShape.size())); + int id = dstNet.addLayer(name, "Reshape", layerParams); + layer_id[name] = id; + + connect(layer_id, dstNet, inpId, id, 0); + + if(outShapeSize == 5) + { + data_layouts[name] = DATA_LAYOUT_NDHWC; + } + else if(outShapeSize == 4) + { + data_layouts[name] = DATA_LAYOUT_NCHW; + } + else + { + data_layouts[name] = inpLayout; + } +} + // "Flatten" "Squeeze" void TFImporter::parseFlatten(tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams) { @@ -1419,6 +1538,15 @@ void TFImporter::parsePlaceholder(tensorflow::GraphDef& net, const tensorflow::N if (dims[0] == -1) // It's OK to have undetermined batch size dims[0] = 1; } + + if (dims.size() == 5 && predictedLayout == DATA_LAYOUT_NDHWC) + { + std::swap(dims[3], dims[4]); // NDHWC->NDHCW + std::swap(dims[2], dims[3]); // NDHCW->NDCHW + std::swap(dims[1], dims[2]); // NDCHW->NCDHW + if (dims[0] == -1) // It's OK to have undetermined batch size + dims[0] = 1; + } bool hasNeg = false; for (int i = 0; i < dims.size() && !hasNeg; ++i) { @@ -2882,10 +3010,10 @@ void TFImporter::populateNet() CV_LOG_DEBUG(NULL, (DNN_DIAGNOSTICS_RUN? "DNN/TF: diagnostic run completed!" : "DNN/TF: import completed!")); } -void TFImporter::addPermuteLayer(const int* order, const std::string& permName, Pin& inpId) +void TFImporter::addPermuteLayer(const int* order, const std::string& permName, Pin& inpId, int orderSize) { LayerParams permLP; - permLP.set("order", DictValue::arrayInt(order, 4)); + permLP.set("order", DictValue::arrayInt(order, orderSize)); CV_Assert(layer_id.find(permName) == layer_id.end()); int permId = dstNet.addLayer(permName, "Permute", permLP); layer_id[permName] = permId; diff --git a/modules/dnn/test/test_tf_importer.cpp b/modules/dnn/test/test_tf_importer.cpp index eec1a628f1..8e7f969480 100644 --- a/modules/dnn/test/test_tf_importer.cpp +++ b/modules/dnn/test/test_tf_importer.cpp @@ -593,6 +593,19 @@ TEST_P(Test_TensorFlow_layers, BiasAdd) runTensorFlowNet("bias_add_1"); } +TEST_P(Test_TensorFlow_layers, ExpandDims) +{ +#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_GE(2019010000) + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 && target == DNN_TARGET_MYRIAD + && getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_X + ) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD_X, CV_TEST_TAG_DNN_SKIP_IE_NN_BUILDER, CV_TEST_TAG_DNN_SKIP_IE_VERSION); +#endif + + runTensorFlowNet("expand_dims_1"); + runTensorFlowNet("expand_dims_2"); +} + // TODO: fix it and add to l2_normalize TEST_P(Test_TensorFlow_layers, l2_normalize_3d) { From 24f43e7ae91feda546f884dbba1ff858e1c1ed25 Mon Sep 17 00:00:00 2001 From: xhawk18 Date: Tue, 5 Oct 2021 00:44:57 +0800 Subject: [PATCH 219/262] Merge pull request #20183 from xhawk18:3.4 * improve compatibility for qt 6. * cmake(highgui): rework Qt dependency support * cmake(highgui): workaround Qt5Config.cmake "components" bug Co-authored-by: Alexander Alekhin --- CMakeLists.txt | 15 ++++--- cmake/OpenCVDetectVTK.cmake | 29 +++--------- cmake/OpenCVFindLibsGUI.cmake | 59 +++++++++++++++++------- modules/highgui/CMakeLists.txt | 75 +++++++++++++++++-------------- modules/highgui/src/window_QT.cpp | 52 ++++++++++++++++++--- 5 files changed, 145 insertions(+), 85 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 94ef43fcb4..d9f98bf23e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1232,12 +1232,17 @@ status("") status(" GUI: ") if(WITH_QT OR HAVE_QT) - if(HAVE_QT5) - status(" QT:" "YES (ver ${Qt5Core_VERSION_STRING})") - status(" QT OpenGL support:" HAVE_QT_OPENGL THEN "YES (${Qt5OpenGL_LIBRARIES} ${Qt5OpenGL_VERSION_STRING})" ELSE NO) - elseif(HAVE_QT) + if(HAVE_QT) status(" QT:" "YES (ver ${QT_VERSION_MAJOR}.${QT_VERSION_MINOR}.${QT_VERSION_PATCH} ${QT_EDITION})") - status(" QT OpenGL support:" HAVE_QT_OPENGL THEN "YES (${QT_QTOPENGL_LIBRARY})" ELSE NO) + if(HAVE_QT_OPENGL) + if(Qt${QT_VERSION_MAJOR}OpenGL_LIBRARIES) + status(" QT OpenGL support:" HAVE_QT_OPENGL THEN "YES (${Qt${QT_VERSION_MAJOR}OpenGL_LIBRARIES} ${Qt${QT_VERSION_MAJOR}OpenGL_VERSION_STRING})" ELSE NO) + else() + status(" QT OpenGL support:" HAVE_QT_OPENGL THEN "YES (${QT_QTOPENGL_LIBRARY})" ELSE NO) + endif() + else() + status(" QT OpenGL support:" "NO") + endif() else() status(" QT:" "NO") endif() diff --git a/cmake/OpenCVDetectVTK.cmake b/cmake/OpenCVDetectVTK.cmake index 14956886e3..94ab735592 100644 --- a/cmake/OpenCVDetectVTK.cmake +++ b/cmake/OpenCVDetectVTK.cmake @@ -26,31 +26,14 @@ if(VTK_VERSION VERSION_LESS "5.8.0") endif() # Different Qt versions can't be linked together -if(HAVE_QT5 AND VTK_VERSION VERSION_LESS "6.0.0") - if(VTK_USE_QT) - message(STATUS "VTK support is disabled. Incompatible combination: OpenCV + Qt5 and VTK ver.${VTK_VERSION} + Qt4") - endif() -endif() - -# Different Qt versions can't be linked together. VTK 6.0.0 doesn't provide a way to get Qt version it was linked with -if(HAVE_QT5 AND VTK_VERSION VERSION_EQUAL "6.0.0" AND NOT DEFINED FORCE_VTK) - message(STATUS "VTK support is disabled. Possible incompatible combination: OpenCV+Qt5, and VTK ver.${VTK_VERSION} with Qt4") - message(STATUS "If it is known that VTK was compiled without Qt4, please define '-DFORCE_VTK=TRUE' flag in CMake") +if((HAVE_QT AND VTK_USE_QT) + AND NOT DEFINED FORCE_VTK # deprecated + AND NOT DEFINED OPENCV_FORCE_VTK +) + message(STATUS "VTK support is disabled. Possible incompatible combination: OpenCV+Qt, and VTK ver.${VTK_VERSION} with Qt") + message(STATUS "If it is known that VTK was compiled without Qt4, please define '-DOPENCV_FORCE_VTK=TRUE' flag in CMake") return() endif() -# Different Qt versions can't be linked together -if(HAVE_QT AND VTK_VERSION VERSION_GREATER "6.0.0" AND NOT ${VTK_QT_VERSION} STREQUAL "") - if(HAVE_QT5 AND ${VTK_QT_VERSION} EQUAL "4") - message(STATUS "VTK support is disabled. Incompatible combination: OpenCV + Qt5 and VTK ver.${VTK_VERSION} + Qt4") - return() - endif() - - if(NOT HAVE_QT5 AND ${VTK_QT_VERSION} EQUAL "5") - message(STATUS "VTK support is disabled. Incompatible combination: OpenCV + Qt4 and VTK ver.${VTK_VERSION} + Qt5") - return() - endif() -endif() - set(HAVE_VTK ON) message(STATUS "Found VTK ${VTK_VERSION} (${VTK_USE_FILE})") diff --git a/cmake/OpenCVFindLibsGUI.cmake b/cmake/OpenCVFindLibsGUI.cmake index 82f7e0ff12..5e7db6c199 100644 --- a/cmake/OpenCVFindLibsGUI.cmake +++ b/cmake/OpenCVFindLibsGUI.cmake @@ -11,25 +11,50 @@ if(WITH_WIN32UI) CMAKE_FLAGS "-DLINK_LIBRARIES:STRING=user32;gdi32") endif() -# --- QT4 --- -ocv_clear_vars(HAVE_QT HAVE_QT5) -if(WITH_QT) - if(NOT WITH_QT EQUAL 4) - find_package(Qt5 COMPONENTS Core Gui Widgets Test Concurrent REQUIRED NO_MODULE) - if(Qt5_FOUND) - set(HAVE_QT5 ON) - set(HAVE_QT ON) - find_package(Qt5 COMPONENTS OpenGL QUIET) - if(Qt5OpenGL_FOUND) - set(QT_QTOPENGL_FOUND ON) - endif() - endif() +macro(ocv_find_package_Qt4) + find_package(Qt4 COMPONENTS QtCore QtGui QtTest ${ARGN}) + if(QT4_FOUND) + set(QT_FOUND 1) + ocv_assert(QT_VERSION_MAJOR EQUAL 4) endif() +endmacro() - if(NOT HAVE_QT) - find_package(Qt4 REQUIRED QtCore QtGui QtTest) - if(QT4_FOUND) - set(HAVE_QT TRUE) +macro(ocv_find_package_Qt OCV_QT_VER) + find_package(Qt${OCV_QT_VER} COMPONENTS Core Gui Widgets Test Concurrent ${ARGN} NO_MODULE) + if(Qt${OCV_QT_VER}_FOUND) + set(QT_FOUND 1) + set(QT_VERSION "${Qt${OCV_QT_VER}_VERSION}") + set(QT_VERSION_MAJOR "${Qt${OCV_QT_VER}_VERSION_MAJOR}") + set(QT_VERSION_MINOR "${Qt${OCV_QT_VER}_VERSION_MINOR}") + set(QT_VERSION_PATCH "${Qt${OCV_QT_VER}_VERSION_PATCH}") + set(QT_VERSION_TWEAK "${Qt${OCV_QT_VER}_VERSION_TWEAK}") + set(QT_VERSION_COUNT "${Qt${OCV_QT_VER}_VERSION_COUNT}") + endif() +endmacro() + +# --- QT4 --- +if(WITH_QT) + if(NOT WITH_QT GREATER 0) + # BUG: Qt5Config.cmake script can't handle components properly: find_package(QT NAMES Qt6 Qt5 REQUIRED NO_MODULE COMPONENTS Core Gui Widgets Test Concurrent) + ocv_find_package_Qt(6 QUIET) + if(NOT QT_FOUND) + ocv_find_package_Qt(5 QUIET) + endif() + if(NOT QT_FOUND) + ocv_find_package_Qt4(QUIET) + endif() + elseif(WITH_QT EQUAL 4) + ocv_find_package_Qt4(REQUIRED) + else() # WITH_QT= + ocv_find_package_Qt("${WITH_QT}" REQUIRED) + endif() + if(QT_FOUND) + set(HAVE_QT ON) + if(QT_VERSION_MAJOR GREATER 4) + find_package(Qt${QT_VERSION_MAJOR} COMPONENTS OpenGL QUIET) + if(Qt${QT_VERSION_MAJOR}OpenGL_FOUND) + set(QT_QTOPENGL_FOUND ON) # HAVE_QT_OPENGL is defined below + endif() endif() endif() endif() diff --git a/modules/highgui/CMakeLists.txt b/modules/highgui/CMakeLists.txt index 296f4cab2b..6e69961099 100644 --- a/modules/highgui/CMakeLists.txt +++ b/modules/highgui/CMakeLists.txt @@ -41,46 +41,55 @@ file(GLOB highgui_ext_hdrs # Removing WinRT API headers by default list(REMOVE_ITEM highgui_ext_hdrs "${CMAKE_CURRENT_LIST_DIR}/include/opencv2/${name}/highgui_winrt.hpp") -if(HAVE_QT5) - # "Automoc" doesn't work properly with opencv_world build, use QT5_WRAP_CPP() directly - #set(CMAKE_AUTOMOC ON) +if(HAVE_QT) + if(QT_VERSION_MAJOR GREATER 4) + # "Automoc" doesn't work properly with opencv_world build, use QT_WRAP_CPP() directly + #set(CMAKE_AUTOMOC ON) - set(CMAKE_INCLUDE_CURRENT_DIR ON) + set(CMAKE_INCLUDE_CURRENT_DIR ON) - QT5_ADD_RESOURCES(_RCC_OUTFILES ${CMAKE_CURRENT_LIST_DIR}/src/window_QT.qrc) - QT5_WRAP_CPP(_MOC_OUTFILES ${CMAKE_CURRENT_LIST_DIR}/src/window_QT.h) - list(APPEND highgui_srcs - ${CMAKE_CURRENT_LIST_DIR}/src/window_QT.cpp - ${CMAKE_CURRENT_LIST_DIR}/src/window_QT.h - ${_MOC_OUTFILES} - ${_RCC_OUTFILES}) + if(QT_VERSION_MAJOR EQUAL 6) + QT6_ADD_RESOURCES(_RCC_OUTFILES ${CMAKE_CURRENT_LIST_DIR}/src/window_QT.qrc) + QT6_WRAP_CPP(_MOC_OUTFILES ${CMAKE_CURRENT_LIST_DIR}/src/window_QT.h) + elseif(QT_VERSION_MAJOR EQUAL 5) + QT5_ADD_RESOURCES(_RCC_OUTFILES ${CMAKE_CURRENT_LIST_DIR}/src/window_QT.qrc) + QT5_WRAP_CPP(_MOC_OUTFILES ${CMAKE_CURRENT_LIST_DIR}/src/window_QT.h) + else() + message(FATAL_ERROR "Unsuported QT version: ${QT_VERSION_MAJOR}") + endif() - foreach(dt5_dep Core Gui Widgets Test Concurrent) - add_definitions(${Qt5${dt5_dep}_DEFINITIONS}) - include_directories(${Qt5${dt5_dep}_INCLUDE_DIRS}) - list(APPEND HIGHGUI_LIBRARIES ${Qt5${dt5_dep}_LIBRARIES}) - endforeach() + list(APPEND highgui_srcs + ${CMAKE_CURRENT_LIST_DIR}/src/window_QT.cpp + ${CMAKE_CURRENT_LIST_DIR}/src/window_QT.h + ${_MOC_OUTFILES} + ${_RCC_OUTFILES}) - if(HAVE_QT_OPENGL) - add_definitions(${Qt5OpenGL_DEFINITIONS}) - include_directories(${Qt5OpenGL_INCLUDE_DIRS}) - list(APPEND HIGHGUI_LIBRARIES ${Qt5OpenGL_LIBRARIES}) - endif() + set(qt_deps Core Gui Widgets Test Concurrent) + if(HAVE_QT_OPENGL) + list(APPEND qt_deps OpenGL) + endif() -elseif(HAVE_QT) - if (HAVE_QT_OPENGL) - set(QT_USE_QTOPENGL TRUE) - endif() - include(${QT_USE_FILE}) + foreach(dt_dep ${qt_deps}) + add_definitions(${Qt${QT_VERSION_MAJOR}${dt_dep}_DEFINITIONS}) + include_directories(${Qt${QT_VERSION_MAJOR}${dt_dep}_INCLUDE_DIRS}) + list(APPEND HIGHGUI_LIBRARIES ${Qt${QT_VERSION_MAJOR}${dt_dep}_LIBRARIES}) + endforeach() + else() + ocv_assert(QT_VERSION_MAJOR EQUAL 4) + if (HAVE_QT_OPENGL) + set(QT_USE_QTOPENGL TRUE) + endif() + include(${QT_USE_FILE}) - QT4_ADD_RESOURCES(_RCC_OUTFILES ${CMAKE_CURRENT_LIST_DIR}/src/window_QT.qrc) - QT4_WRAP_CPP(_MOC_OUTFILES ${CMAKE_CURRENT_LIST_DIR}/src/window_QT.h) + QT4_ADD_RESOURCES(_RCC_OUTFILES ${CMAKE_CURRENT_LIST_DIR}/src/window_QT.qrc) + QT4_WRAP_CPP(_MOC_OUTFILES ${CMAKE_CURRENT_LIST_DIR}/src/window_QT.h) - list(APPEND HIGHGUI_LIBRARIES ${QT_LIBRARIES}) - list(APPEND highgui_srcs ${CMAKE_CURRENT_LIST_DIR}/src/window_QT.cpp ${_MOC_OUTFILES} ${_RCC_OUTFILES}) - ocv_check_flag_support(CXX -Wno-missing-declarations _have_flag "") - if(${_have_flag}) - set_source_files_properties(${_RCC_OUTFILES} PROPERTIES COMPILE_FLAGS -Wno-missing-declarations) + list(APPEND HIGHGUI_LIBRARIES ${QT_LIBRARIES}) + list(APPEND highgui_srcs ${CMAKE_CURRENT_LIST_DIR}/src/window_QT.cpp ${_MOC_OUTFILES} ${_RCC_OUTFILES}) + ocv_check_flag_support(CXX -Wno-missing-declarations _have_flag "") + if(${_have_flag}) + set_source_files_properties(${_RCC_OUTFILES} PROPERTIES COMPILE_FLAGS -Wno-missing-declarations) + endif() endif() elseif(WINRT) if(NOT WINRT_8_0) diff --git a/modules/highgui/src/window_QT.cpp b/modules/highgui/src/window_QT.cpp index 8dff03117e..e7ee43cf5e 100644 --- a/modules/highgui/src/window_QT.cpp +++ b/modules/highgui/src/window_QT.cpp @@ -64,6 +64,38 @@ #endif +#if QT_VERSION >= QT_VERSION_CHECK(5, 15, 0) +#define Qt_MiddleButton Qt::MiddleButton +inline Qt::Orientation wheelEventOrientation(QWheelEvent *we) { + if (std::abs(we->angleDelta().x()) < std::abs(we->angleDelta().y())) + return Qt::Vertical; + else + return Qt::Horizontal; +} +inline int wheelEventDelta(QWheelEvent *we) { + if(wheelEventOrientation(we) == Qt::Vertical) + return we->angleDelta().y(); + else + return we->angleDelta().x(); +} +inline QPoint wheelEventPos(QWheelEvent *we) { + return we->position().toPoint(); +} +#else +#define Qt_MiddleButton Qt::MidButton +inline Qt::Orientation wheelEventOrientation(QWheelEvent *we) { + return we->orientation(); +} +inline int wheelEventDelta(QWheelEvent *we) { + return we->delta(); +} +inline QPoint wheelEventPos(QWheelEvent *we) { + return we->pos(); +} + +#endif + + //Static and global first static GuiReceiver *guiMainThread = NULL; static int parameterSystemC = 1; @@ -1579,7 +1611,9 @@ CvWinProperties::CvWinProperties(QString name_paraWindow, QObject* /*parent*/) myLayout->setObjectName(QString::fromUtf8("boxLayout")); myLayout->setContentsMargins(0, 0, 0, 0); myLayout->setSpacing(0); +#if QT_VERSION < QT_VERSION_CHECK(5, 13, 0) myLayout->setMargin(0); +#endif myLayout->setSizeConstraint(QLayout::SetFixedSize); setLayout(myLayout); @@ -1957,7 +1991,9 @@ void CvWindow::createBarLayout() myBarLayout->setObjectName(QString::fromUtf8("barLayout")); myBarLayout->setContentsMargins(0, 0, 0, 0); myBarLayout->setSpacing(0); +#if QT_VERSION < QT_VERSION_CHECK(5, 13, 0) myBarLayout->setMargin(0); +#endif } @@ -1967,7 +2003,9 @@ void CvWindow::createGlobalLayout() myGlobalLayout->setObjectName(QString::fromUtf8("boxLayout")); myGlobalLayout->setContentsMargins(0, 0, 0, 0); myGlobalLayout->setSpacing(0); +#if QT_VERSION < QT_VERSION_CHECK(5, 13, 0) myGlobalLayout->setMargin(0); +#endif setMinimumSize(1, 1); if (param_flags == CV_WINDOW_AUTOSIZE) @@ -2205,7 +2243,7 @@ void CvWindow::icvLoadControlPanel() } if (t->type == type_CvButtonbar) { - int subsize = settings.beginReadArray(QString("buttonbar")+i); + int subsize = settings.beginReadArray(QString("buttonbar%1").arg(i)); if ( subsize == ((CvButtonbar*)t)->layout()->count() ) icvLoadButtonbar((CvButtonbar*)t,&settings); @@ -2236,7 +2274,7 @@ void CvWindow::icvSaveControlPanel() } if (t->type == type_CvButtonbar) { - settings.beginWriteArray(QString("buttonbar")+i); + settings.beginWriteArray(QString("buttonbar%1").arg(i)); icvSaveButtonbar((CvButtonbar*)t,&settings); settings.endArray(); } @@ -2396,14 +2434,14 @@ void OCVViewPort::icvmouseHandler(QMouseEvent* evnt, type_mouse_event category, flags |= CV_EVENT_FLAG_LBUTTON; if(buttons & Qt::RightButton) flags |= CV_EVENT_FLAG_RBUTTON; - if(buttons & Qt::MidButton) + if(buttons & Qt_MiddleButton) flags |= CV_EVENT_FLAG_MBUTTON; if (cv_event == -1) { if (category == mouse_wheel) { QWheelEvent *we = (QWheelEvent *) evnt; - cv_event = ((we->orientation() == Qt::Vertical) ? CV_EVENT_MOUSEWHEEL : CV_EVENT_MOUSEHWHEEL); - flags |= (we->delta() & 0xffff)<<16; + cv_event = ((wheelEventOrientation(we) == Qt::Vertical) ? CV_EVENT_MOUSEWHEEL : CV_EVENT_MOUSEHWHEEL); + flags |= (wheelEventDelta(we) & 0xffff)<<16; return; } switch(evnt->button()) @@ -2416,7 +2454,7 @@ void OCVViewPort::icvmouseHandler(QMouseEvent* evnt, type_mouse_event category, cv_event = tableMouseButtons[category][1]; flags |= CV_EVENT_FLAG_RBUTTON; break; - case Qt::MidButton: + case Qt_MiddleButton: cv_event = tableMouseButtons[category][2]; flags |= CV_EVENT_FLAG_MBUTTON; break; @@ -2772,7 +2810,7 @@ void DefaultViewPort::wheelEvent(QWheelEvent* evnt) { icvmouseEvent((QMouseEvent *)evnt, mouse_wheel); - scaleView(evnt->delta() / 240.0, evnt->pos()); + scaleView(wheelEventDelta(evnt) / 240.0, wheelEventPos(evnt)); viewport()->update(); QWidget::wheelEvent(evnt); From f11f2bfb56e4bbeb6a05ec4112adc375ea8fc515 Mon Sep 17 00:00:00 2001 From: keroiber <39693294+keroiber@users.noreply.github.com> Date: Mon, 4 Oct 2021 18:51:49 +0200 Subject: [PATCH 220/262] Merge pull request #20743 from keroiber:prefix_js_function_bindings_with_namespace * Prefix global javascript functions with sub-namespaces * js: handle 'namespace_prefix_override', update filtering - avoid functions override with same name but different namespace Co-authored-by: Alexander Alekhin --- modules/js/generator/CMakeLists.txt | 1 + modules/js/generator/embindgen.py | 51 +++++++++++++++++++++++------ platforms/js/opencv_js.config.py | 24 ++++++++++++-- 3 files changed, 63 insertions(+), 13 deletions(-) diff --git a/modules/js/generator/CMakeLists.txt b/modules/js/generator/CMakeLists.txt index 7a53429651..3d66df154f 100644 --- a/modules/js/generator/CMakeLists.txt +++ b/modules/js/generator/CMakeLists.txt @@ -60,6 +60,7 @@ add_custom_command( ${JS_SOURCE_DIR}/src/core_bindings.cpp ${CMAKE_CURRENT_SOURCE_DIR}/embindgen.py ${CMAKE_CURRENT_SOURCE_DIR}/templates.py + "${OPENCV_JS_WHITELIST_FILE}" ${scripts_hdr_parser} #(not needed - generated by CMake) ${CMAKE_CURRENT_BINARY_DIR}/headers.txt ${opencv_hdrs} diff --git a/modules/js/generator/embindgen.py b/modules/js/generator/embindgen.py index dc7a001df1..7eb20c9861 100644 --- a/modules/js/generator/embindgen.py +++ b/modules/js/generator/embindgen.py @@ -104,6 +104,9 @@ def makeWhiteList(module_list): return wl white_list = None +namespace_prefix_override = { + 'dnn' : '' +} # Features to be exported export_enums = False @@ -271,6 +274,8 @@ class FuncVariant(object): class FuncInfo(object): def __init__(self, class_name, name, cname, namespace, isconstructor): + self.name_id = '_'.join([namespace] + ([class_name] if class_name else []) + [name]) # unique id for dict key + self.class_name = class_name self.name = name self.cname = cname @@ -295,9 +300,9 @@ class JSWrapperGenerator(object): self.bindings = [] self.wrapper_funcs = [] - self.classes = {} + self.classes = {} # FIXIT 'classes' should belong to 'namespaces' self.namespaces = {} - self.enums = {} + self.enums = {} # FIXIT 'enums' should belong to 'namespaces' self.parser = hdr_parser.CppHeaderParser() self.class_idx = 0 @@ -419,7 +424,8 @@ class JSWrapperGenerator(object): else: func_map = self.namespaces.setdefault(namespace, Namespace()).funcs - func = func_map.setdefault(name, FuncInfo(class_name, name, cpp_name, namespace, is_constructor)) + fi = FuncInfo(class_name, name, cpp_name, namespace, is_constructor) + func = func_map.setdefault(fi.name_id, fi) variant = FuncVariant(class_name, name, decl, is_constructor, is_class_method, is_const_method, is_virtual_method, is_pure_virtual_method, ref_return, const_return) @@ -430,7 +436,7 @@ class JSWrapperGenerator(object): f.write(buf.getvalue()) f.close() - def gen_function_binding_with_wrapper(self, func, class_info): + def gen_function_binding_with_wrapper(self, func, ns_name, class_info): binding_text = None wrapper_func_text = None @@ -488,8 +494,23 @@ class JSWrapperGenerator(object): # Wrapper function - wrap_func_name = (func.class_name+"_" if class_info != None else "") + func.name.split("::")[-1] + "_wrapper" - js_func_name = func.name + if ns_name != None and ns_name != "cv": + ns_parts = ns_name.split(".") + if ns_parts[0] == "cv": + ns_parts = ns_parts[1:] + ns_part = "_".join(ns_parts) + "_" + ns_id = '_'.join(ns_parts) + ns_prefix = namespace_prefix_override.get(ns_id, ns_id) + if ns_prefix: + ns_prefix = ns_prefix + '_' + else: + ns_prefix = '' + if class_info == None: + js_func_name = ns_prefix + func.name + wrap_func_name = js_func_name + "_wrapper" + else: + wrap_func_name = ns_prefix + func.class_name + "_" + func.name + "_wrapper" + js_func_name = func.name # TODO: Name functions based wrap directives or based on arguments list if index > 0: @@ -740,12 +761,22 @@ class JSWrapperGenerator(object): # step 2: generate bindings # Global functions for ns_name, ns in sorted(self.namespaces.items()): - if ns_name.split('.')[0] != 'cv': + ns_parts = ns_name.split('.') + if ns_parts[0] != 'cv': + print('Ignore namespace: {}'.format(ns_name)) continue - for name, func in sorted(ns.funcs.items()): + else: + ns_parts = ns_parts[1:] + ns_id = '_'.join(ns_parts) + ns_prefix = namespace_prefix_override.get(ns_id, ns_id) + for name_id, func in sorted(ns.funcs.items()): + name = func.name + if ns_prefix: + name = ns_prefix + '_' + name if name in ignore_list: continue if not name in white_list['']: + #print('Not in whitelist: "{}" from ns={}'.format(name, ns_name)) continue ext_cnst = False @@ -769,7 +800,7 @@ class JSWrapperGenerator(object): continue if with_wrapped_functions: - binding, wrapper = self.gen_function_binding_with_wrapper(func, class_info=None) + binding, wrapper = self.gen_function_binding_with_wrapper(func, ns_name, class_info=None) self.bindings += binding self.wrapper_funcs += wrapper else: @@ -802,7 +833,7 @@ class JSWrapperGenerator(object): class_bindings.append(constructor_template.substitute(signature=', '.join(args))) else: if with_wrapped_functions and (len(method.variants) > 1 or len(method.variants[0].args)>0 or "String" in method.variants[0].rettype): - binding, wrapper = self.gen_function_binding_with_wrapper(method, class_info=class_info) + binding, wrapper = self.gen_function_binding_with_wrapper(method, None, class_info=class_info) self.wrapper_funcs = self.wrapper_funcs + wrapper class_bindings = class_bindings + binding else: diff --git a/platforms/js/opencv_js.config.py b/platforms/js/opencv_js.config.py index 3e09805cd3..05d2883b78 100644 --- a/platforms/js/opencv_js.config.py +++ b/platforms/js/opencv_js.config.py @@ -55,8 +55,26 @@ features2d = {'Feature2D': ['detect', 'compute', 'detectAndCompute', 'descriptor 'BFMatcher': ['isMaskSupported', 'create'], '': ['drawKeypoints', 'drawMatches', 'drawMatchesKnn']} -calib3d = {'': ['findHomography', 'calibrateCameraExtended', 'drawFrameAxes', 'estimateAffine2D', \ - 'getDefaultNewCameraMatrix', 'initUndistortRectifyMap', 'Rodrigues', \ - 'solvePnP', 'solvePnPRansac', 'solvePnPRefineLM']} +calib3d = { + '': [ + 'findHomography', + 'calibrateCameraExtended', + 'drawFrameAxes', + 'estimateAffine2D', + 'getDefaultNewCameraMatrix', + 'initUndistortRectifyMap', + 'Rodrigues', + 'solvePnP', + 'solvePnPRansac', + 'solvePnPRefineLM', + 'projectPoints', + + # cv::fisheye namespace + 'fisheye_initUndistortRectifyMap', + 'fisheye_projectPoints', + ], +} white_list = makeWhiteList([core, imgproc, objdetect, video, dnn, features2d, calib3d]) + +# namespace_prefix_override['dnn'] = '' # compatibility stuff (enabled by default) From cce78cc5e21d9e72d04504864dfb06afdebe56cb Mon Sep 17 00:00:00 2001 From: Jebastin Nadar Date: Mon, 4 Oct 2021 23:37:38 +0530 Subject: [PATCH 221/262] Merge pull request #20535 from SamFC10:onnx-q dnn : int8 quantized layers support in onnx importer * added quantized layers support in onnx importer * added more cases in eltwise node, some more checks * added tests for quantized nodes * relax thresholds for failed tests, address review comments * refactoring based on review comments * added support for unsupported cases and pre-quantized resnet50 test * relax thresholds due to int8 resize layer --- .../dnn/include/opencv2/dnn/all_layers.hpp | 7 + modules/dnn/src/dnn.cpp | 5 +- modules/dnn/src/init.cpp | 2 + ...ntize_layer.cpp => quantization_utils.cpp} | 53 ++ modules/dnn/src/layers/resize_layer.cpp | 156 ++++-- .../dnn/src/onnx/onnx_graph_simplifier.cpp | 21 +- modules/dnn/src/onnx/onnx_importer.cpp | 486 +++++++++++++++++- modules/dnn/test/test_int8_layers.cpp | 8 +- modules/dnn/test/test_onnx_importer.cpp | 111 ++++ 9 files changed, 795 insertions(+), 54 deletions(-) rename modules/dnn/src/int8layers/{quantize_dequantize_layer.cpp => quantization_utils.cpp} (73%) diff --git a/modules/dnn/include/opencv2/dnn/all_layers.hpp b/modules/dnn/include/opencv2/dnn/all_layers.hpp index cfe6595d78..9fde7ad4c4 100644 --- a/modules/dnn/include/opencv2/dnn/all_layers.hpp +++ b/modules/dnn/include/opencv2/dnn/all_layers.hpp @@ -387,6 +387,13 @@ CV__DNN_INLINE_NS_BEGIN static Ptr create(const LayerParams ¶ms); }; + class CV_EXPORTS RequantizeLayer : public Layer + { + public: + float scale, shift; + static Ptr create(const LayerParams ¶ms); + }; + class CV_EXPORTS ConcatLayer : public Layer { public: diff --git a/modules/dnn/src/dnn.cpp b/modules/dnn/src/dnn.cpp index 4e38b0374f..24daeb249f 100644 --- a/modules/dnn/src/dnn.cpp +++ b/modules/dnn/src/dnn.cpp @@ -4055,6 +4055,9 @@ int Net::addLayer(const String &name, const String &type, const int &dtype, Laye if (params.get("has_dynamic_shapes", false)) impl->hasDynamicShapes = true; + if (dtype == CV_8S) + impl->netWasQuantized = true; + return id; } @@ -4389,7 +4392,7 @@ Net Net::quantize(InputArrayOfArrays calibData, int inputsDtype, int outputsDtyp // Layers with multiple outputs. Number of outputs is equal to number of inputs if (ld.type == "Blank" || ld.type == "Dropout" || ld.type == "Identity" || ld.type == "Silence" || ld.type == "Flatten" || ld.type == "Padding" || ld.type == "Permute" || ld.type == "Reshape" || - ld.type == "ReLU6" || ld.type == "Reorg" || ld.type == "ShuffleChannel" || + ld.type == "ReLU6" || ld.type == "Reorg" || ld.type == "ShuffleChannel" || ld.type == "Resize" || (ld.type == "ReLU" && !ld.params.get("negative_slope", 0.f)) /* ReLU with negative slope 0 */) { for (int i = 0; i < ld.outputBlobs.size(); i++) diff --git a/modules/dnn/src/init.cpp b/modules/dnn/src/init.cpp index 9d8a3783a2..123cb170b7 100644 --- a/modules/dnn/src/init.cpp +++ b/modules/dnn/src/init.cpp @@ -144,6 +144,7 @@ void initializeLayerFactory() CV_DNN_REGISTER_LAYER_CLASS(Quantize, QuantizeLayer); CV_DNN_REGISTER_LAYER_CLASS(Dequantize, DequantizeLayer); + CV_DNN_REGISTER_LAYER_CLASS(Requantize, RequantizeLayer); CV_DNN_REGISTER_LAYER_CLASS(ConvolutionInt8, ConvolutionLayerInt8); CV_DNN_REGISTER_LAYER_CLASS(InnerProductInt8, InnerProductLayerInt8); CV_DNN_REGISTER_LAYER_CLASS(PoolingInt8, PoolingLayerInt8); @@ -173,6 +174,7 @@ void initializeLayerFactory() CV_DNN_REGISTER_LAYER_CLASS(SilenceInt8, BlankLayer); CV_DNN_REGISTER_LAYER_CLASS(ConstInt8, ConstLayer); CV_DNN_REGISTER_LAYER_CLASS(ReshapeInt8, ReshapeLayer); + CV_DNN_REGISTER_LAYER_CLASS(ResizeInt8, ResizeLayer); CV_DNN_REGISTER_LAYER_CLASS(SplitInt8, SplitLayer); CV_DNN_REGISTER_LAYER_CLASS(SliceInt8, SliceLayer); CV_DNN_REGISTER_LAYER_CLASS(CropInt8, CropLayer); diff --git a/modules/dnn/src/int8layers/quantize_dequantize_layer.cpp b/modules/dnn/src/int8layers/quantization_utils.cpp similarity index 73% rename from modules/dnn/src/int8layers/quantize_dequantize_layer.cpp rename to modules/dnn/src/int8layers/quantization_utils.cpp index 2ddb76a0e8..0346f147ba 100644 --- a/modules/dnn/src/int8layers/quantize_dequantize_layer.cpp +++ b/modules/dnn/src/int8layers/quantization_utils.cpp @@ -10,6 +10,7 @@ namespace cv namespace dnn { +// Quantize FP32/FP16 Inputs to INT8 class QuantizeLayerImpl CV_FINAL : public QuantizeLayer { public: @@ -77,6 +78,7 @@ public: } }; +// Dequantize INT8 Inputs to FP32/FP16 class DequantizeLayerImpl CV_FINAL : public DequantizeLayer { public: @@ -143,6 +145,52 @@ public: } }; +// Rescale/Requantize INT8 Inputs from (scale1, zeropoint1) to (scale2, zeropoint2) +class RequantizeLayerImpl CV_FINAL : public RequantizeLayer +{ +public: + RequantizeLayerImpl(const LayerParams& params) + { + scale = params.get("scale", 1.f); + shift = params.get("shift", 0.f); + setParamsFrom(params); + } + + virtual bool supportBackend(int backendId) CV_OVERRIDE + { + return backendId == DNN_BACKEND_OPENCV; + } + + bool getMemoryShapes(const std::vector &inputs, + const int requiredOutputs, + std::vector &outputs, + std::vector &internals) const CV_OVERRIDE + { + CV_Assert(inputs.size() == 1); + Layer::getMemoryShapes(inputs, requiredOutputs, outputs, internals); + return false; + } + + virtual void finalize(InputArrayOfArrays inputs_arr, OutputArrayOfArrays outputs_arr) CV_OVERRIDE + { + std::vector inputs, outputs; + inputs_arr.getMatVector(inputs); + outputs_arr.getMatVector(outputs); + } + + void forward(InputArrayOfArrays inputs_arr, OutputArrayOfArrays outputs_arr, OutputArrayOfArrays internals_arr) CV_OVERRIDE + { + CV_TRACE_FUNCTION(); + CV_TRACE_ARG_VALUE(name, "name", name.c_str()); + + std::vector inputs, outputs; + inputs_arr.getMatVector(inputs); + outputs_arr.getMatVector(outputs); + + inputs[0].convertTo(outputs[0], CV_8S, scale, shift); + } +}; + Ptr QuantizeLayer::create(const LayerParams& params) { return Ptr(new QuantizeLayerImpl(params)); @@ -153,5 +201,10 @@ Ptr DequantizeLayer::create(const LayerParams& params) return Ptr(new DequantizeLayerImpl(params)); } +Ptr RequantizeLayer::create(const LayerParams& params) +{ + return Ptr(new RequantizeLayerImpl(params)); +} + } } diff --git a/modules/dnn/src/layers/resize_layer.cpp b/modules/dnn/src/layers/resize_layer.cpp index f24c823807..47ff0719fe 100644 --- a/modules/dnn/src/layers/resize_layer.cpp +++ b/modules/dnn/src/layers/resize_layer.cpp @@ -49,6 +49,8 @@ public: alignCorners = params.get("align_corners", false); halfPixelCenters = params.get("half_pixel_centers", false); + if (interpolation == "opencv_linear") + halfPixelCenters = true; } bool getMemoryShapes(const std::vector &inputs, @@ -131,8 +133,11 @@ public: Mat& inp = inputs[0]; Mat& out = outputs[0]; - if ((interpolation == "nearest" && !alignCorners && !halfPixelCenters) || interpolation == "opencv_linear" || (interpolation == "bilinear" && halfPixelCenters)) + int depth = inp.depth(); + if ((interpolation == "nearest" && !alignCorners && !halfPixelCenters) || (interpolation == "opencv_linear" && depth != CV_8S) || + (interpolation == "bilinear" && halfPixelCenters && depth != CV_8S)) { + // INTER_LINEAR Resize mode does not support INT8 inputs InterpolationFlags mode = interpolation == "nearest" ? INTER_NEAREST : INTER_LINEAR; for (size_t n = 0; n < inputs[0].size[0]; ++n) { @@ -164,34 +169,66 @@ public: widthOffset = 0.5f * scaleWidth; } - for (int y = 0; y < outHeight; ++y) + if (depth == CV_8S) { - float input_y = y * scaleHeight + heightOffset; - int y0 = halfPixelCenters ? std::floor(input_y) : lroundf(input_y); - y0 = std::min(y0, inpHeight - 1); - - const float* inpData_row = inpPlanes.ptr(y0); - - for (int x = 0; x < outWidth; ++x) + for (int y = 0; y < outHeight; ++y) { - float input_x = x * scaleWidth + widthOffset; - int x0 = halfPixelCenters ? std::floor(input_x) : lroundf(input_x); - x0 = std::min(x0, inpWidth - 1); + float input_y = y * scaleHeight + heightOffset; + int y0 = halfPixelCenters ? std::floor(input_y) : lroundf(input_y); + y0 = std::min(y0, inpHeight - 1); - float* outData = outPlanes.ptr(y, x); - const float* inpData_row_c = inpData_row; + const int8_t* inpData_row = inpPlanes.ptr(y0); - for (int c = 0; c < numPlanes; ++c) + for (int x = 0; x < outWidth; ++x) { - *outData = inpData_row_c[x0]; + float input_x = x * scaleWidth + widthOffset; + int x0 = halfPixelCenters ? std::floor(input_x) : lroundf(input_x); + x0 = std::min(x0, inpWidth - 1); - inpData_row_c += inpSpatialSize; - outData += outSpatialSize; + int8_t* outData = outPlanes.ptr(y, x); + const int8_t* inpData_row_c = inpData_row; + + for (int c = 0; c < numPlanes; ++c) + { + *outData = inpData_row_c[x0]; + + inpData_row_c += inpSpatialSize; + outData += outSpatialSize; + } + } + } + } + else + { + for (int y = 0; y < outHeight; ++y) + { + float input_y = y * scaleHeight + heightOffset; + int y0 = halfPixelCenters ? std::floor(input_y) : lroundf(input_y); + y0 = std::min(y0, inpHeight - 1); + + const float* inpData_row = inpPlanes.ptr(y0); + + for (int x = 0; x < outWidth; ++x) + { + float input_x = x * scaleWidth + widthOffset; + int x0 = halfPixelCenters ? std::floor(input_x) : lroundf(input_x); + x0 = std::min(x0, inpWidth - 1); + + float* outData = outPlanes.ptr(y, x); + const float* inpData_row_c = inpData_row; + + for (int c = 0; c < numPlanes; ++c) + { + *outData = inpData_row_c[x0]; + + inpData_row_c += inpSpatialSize; + outData += outSpatialSize; + } } } } } - else if (interpolation == "bilinear") + else if (interpolation == "bilinear" || interpolation == "opencv_linear") { const int inpHeight = inp.size[2]; const int inpWidth = inp.size[3]; @@ -202,31 +239,65 @@ public: Mat inpPlanes = inp.reshape(1, numPlanes * inpHeight); Mat outPlanes = out.reshape(1, numPlanes * outHeight); - for (int y = 0; y < outHeight; ++y) + if (depth == CV_8S) { - float input_y = y * scaleHeight; - int y0 = static_cast(input_y); - const float* inpData_row0 = inpPlanes.ptr(y0); - const float* inpData_row1 = inpPlanes.ptr(std::min(y0 + 1, inpHeight - 1)); - for (int x = 0; x < outWidth; ++x) + for (int y = 0; y < outHeight; ++y) { - float input_x = x * scaleWidth; - int x0 = static_cast(input_x); - int x1 = std::min(x0 + 1, inpWidth - 1); - - float* outData = outPlanes.ptr(y, x); - const float* inpData_row0_c = inpData_row0; - const float* inpData_row1_c = inpData_row1; - for (int c = 0; c < numPlanes; ++c) + float input_y = halfPixelCenters ? std::max((y + 0.5f) * scaleHeight - 0.5f, 0.0f) : y * scaleHeight; + int y0 = static_cast(input_y); + const int8_t* inpData_row0 = inpPlanes.ptr(y0); + const int8_t* inpData_row1 = inpPlanes.ptr(std::min(y0 + 1, inpHeight - 1)); + for (int x = 0; x < outWidth; ++x) { - *outData = inpData_row0_c[x0] + - (input_y - y0) * (inpData_row1_c[x0] - inpData_row0_c[x0]) + - (input_x - x0) * (inpData_row0_c[x1] - inpData_row0_c[x0] + - (input_y - y0) * (inpData_row1_c[x1] - inpData_row0_c[x1] - inpData_row1_c[x0] + inpData_row0_c[x0])); + float input_x = halfPixelCenters ? std::max((x + 0.5f) * scaleWidth - 0.5f, 0.0f) : x * scaleWidth; + int x0 = static_cast(input_x); + int x1 = std::min(x0 + 1, inpWidth - 1); - inpData_row0_c += inpSpatialSize; - inpData_row1_c += inpSpatialSize; - outData += outSpatialSize; + int8_t* outData = outPlanes.ptr(y, x); + const int8_t* inpData_row0_c = inpData_row0; + const int8_t* inpData_row1_c = inpData_row1; + for (int c = 0; c < numPlanes; ++c) + { + *outData = static_cast(inpData_row0_c[x0] + + (input_y - y0) * (inpData_row1_c[x0] - inpData_row0_c[x0]) + + (input_x - x0) * (inpData_row0_c[x1] - inpData_row0_c[x0] + + (input_y - y0) * (inpData_row1_c[x1] - inpData_row0_c[x1] - inpData_row1_c[x0] + inpData_row0_c[x0]))); + + inpData_row0_c += inpSpatialSize; + inpData_row1_c += inpSpatialSize; + outData += outSpatialSize; + } + } + } + } + else + { + for (int y = 0; y < outHeight; ++y) + { + float input_y = y * scaleHeight; + int y0 = static_cast(input_y); + const float* inpData_row0 = inpPlanes.ptr(y0); + const float* inpData_row1 = inpPlanes.ptr(std::min(y0 + 1, inpHeight - 1)); + for (int x = 0; x < outWidth; ++x) + { + float input_x = x * scaleWidth; + int x0 = static_cast(input_x); + int x1 = std::min(x0 + 1, inpWidth - 1); + + float* outData = outPlanes.ptr(y, x); + const float* inpData_row0_c = inpData_row0; + const float* inpData_row1_c = inpData_row1; + for (int c = 0; c < numPlanes; ++c) + { + *outData = inpData_row0_c[x0] + + (input_y - y0) * (inpData_row1_c[x0] - inpData_row0_c[x0]) + + (input_x - x0) * (inpData_row0_c[x1] - inpData_row0_c[x0] + + (input_y - y0) * (inpData_row1_c[x1] - inpData_row0_c[x1] - inpData_row1_c[x0] + inpData_row0_c[x0])); + + inpData_row0_c += inpSpatialSize; + inpData_row1_c += inpSpatialSize; + outData += outSpatialSize; + } } } } @@ -363,6 +434,11 @@ public: } #endif + virtual bool tryQuantize(const std::vector > &scales, + const std::vector > &zeropoints, LayerParams& params) CV_OVERRIDE + { + return true; + } protected: int outWidth, outHeight; diff --git a/modules/dnn/src/onnx/onnx_graph_simplifier.cpp b/modules/dnn/src/onnx/onnx_graph_simplifier.cpp index 7caa5ed54d..c4ebcd1c63 100644 --- a/modules/dnn/src/onnx/onnx_graph_simplifier.cpp +++ b/modules/dnn/src/onnx/onnx_graph_simplifier.cpp @@ -594,7 +594,8 @@ void simplifySubgraphs(opencv_onnx::GraphProto& net) Mat getMatFromTensor(opencv_onnx::TensorProto& tensor_proto) { if (tensor_proto.raw_data().empty() && tensor_proto.float_data().empty() && - tensor_proto.double_data().empty() && tensor_proto.int64_data().empty()) + tensor_proto.double_data().empty() && tensor_proto.int64_data().empty() && + tensor_proto.int32_data().empty()) return Mat(); opencv_onnx::TensorProto_DataType datatype = tensor_proto.data_type(); @@ -663,6 +664,24 @@ Mat getMatFromTensor(opencv_onnx::TensorProto& tensor_proto) convertInt64ToInt32(src, dst, blob.total()); } } + else if (datatype == opencv_onnx::TensorProto_DataType_INT8 || + datatype == opencv_onnx::TensorProto_DataType_UINT8) + { + // TODO : Add support for uint8 weights and acitvations. For now, converting uint8 tensors to int8. + int offset = datatype == opencv_onnx::TensorProto_DataType_INT8 ? 0 : -128; + int depth = datatype == opencv_onnx::TensorProto_DataType_INT8 ? CV_8S : CV_8U; + + if (!tensor_proto.int32_data().empty()) + { + const ::google::protobuf::RepeatedField field = tensor_proto.int32_data(); + Mat(sizes, CV_32SC1, (void*)field.data()).convertTo(blob, CV_8S, 1.0, offset); + } + else + { + char* val = const_cast(tensor_proto.raw_data().c_str()); + Mat(sizes, depth, val).convertTo(blob, CV_8S, 1.0, offset); + } + } else { std::string errorMsg = "Unsupported data type: " + diff --git a/modules/dnn/src/onnx/onnx_importer.cpp b/modules/dnn/src/onnx/onnx_importer.cpp index 9ef947c645..7c230f28c8 100644 --- a/modules/dnn/src/onnx/onnx_importer.cpp +++ b/modules/dnn/src/onnx/onnx_importer.cpp @@ -63,6 +63,8 @@ class ONNXImporter void addConstant(const std::string& name, const Mat& blob); void addLayer(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void handleQuantizedNode(LayerParams& layerParams, + const opencv_onnx::NodeProto& node_proto); void expandMid(const std::string& prefix, opencv_onnx::NodeProto& node_proto, const std::string& input, size_t n); @@ -142,6 +144,14 @@ private: void parseSoftMax (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); void parseDetectionOutput (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); void parseCumSum (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseQuantDequant (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseQConv (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseQMatMul (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseQEltwise (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseQLeakyRelu (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseQSigmoid (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseQAvgPool (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseQConcat (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); void parseCustomLayer (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); }; @@ -242,7 +252,7 @@ void runLayer(LayerParams& params, const std::vector& inputs, CV_Assert((bool)layer); std::vector inpShapes(inputs.size()); - int ddepth = CV_32F; + int ddepth = params.get("depth", CV_32F); for (size_t i = 0; i < inputs.size(); ++i) { inpShapes[i] = shape(inputs[i]); @@ -458,7 +468,8 @@ Mat ONNXImporter::getBlob(const std::string& input_name) void ONNXImporter::addLayer(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) { - int id = dstNet.addLayer(layerParams.name, layerParams.type, layerParams); + int depth = layerParams.get("depth", CV_32F); + int id = dstNet.addLayer(layerParams.name, layerParams.type, depth, layerParams); for (int i = 0; i < node_proto.output_size(); ++i) { layer_id.insert(std::make_pair(node_proto.output(i), LayerInfo(id, i))); @@ -525,6 +536,51 @@ void ONNXImporter::addConstant(const std::string& name, const Mat& blob) outShapes.insert(std::make_pair(name, shape(blob))); } +void ONNXImporter::handleQuantizedNode(LayerParams& layerParams, + const opencv_onnx::NodeProto& node_proto) +{ + // Quantized nodes have output names ending with 'quantized' + std::string outName = node_proto.output(0); + int len = outName.length(); + if (len <= 9) + return; + + if (outName.substr(len - 9) == "quantized") + { + outName = outName.substr(0, len - 9); + Mat scale, zeropoint; + + if (constBlobs.find(outName + "scale") != constBlobs.end() && + constBlobs.find(outName + "zero_point") != constBlobs.end()) + { + scale = getBlob(outName + "scale"); + zeropoint = getBlob(outName + "zero_point"); + } + else + { + std::string inpName = node_proto.input(0); + inpName = inpName.substr(0, inpName.length() - 9); + scale = getBlob(inpName + "scale"); + zeropoint = getBlob(inpName + "zero_point"); + + for (int i = 0; i < node_proto.output_size(); i++) + { + std::string out = node_proto.output(i); + out = out.substr(0, out.length() - 9); + addConstant(out + "scale", scale); + addConstant(out + "zero_point", zeropoint); + } + } + + if (scale.total() != 1 || zeropoint.total() != 1) + CV_Error(Error::StsNotImplemented, "Per-channel scales/zeropoints are not supported"); + + layerParams.set("depth", CV_8S); + layerParams.set("scales", DictValue::arrayReal(scale.ptr(), 1)); + layerParams.set("zeropoints", DictValue::arrayInt(zeropoint.ptr(), 1)); + } +} + void ONNXImporter::populateNet() { CV_Assert(model_proto.has_graph()); @@ -623,6 +679,8 @@ void ONNXImporter::handleNode(const opencv_onnx::NodeProto& node_proto) layerParams.type = layer_type; layerParams.set("has_dynamic_shapes", hasDynamicShapes); + handleQuantizedNode(layerParams, node_proto); + DispatchMap::const_iterator iter = dispatch.find(layer_type); if (iter != dispatch.end()) { @@ -684,7 +742,8 @@ void ONNXImporter::handleNode(const opencv_onnx::NodeProto& node_proto) void ONNXImporter::parseMaxPool(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) { - layerParams.type = "Pooling"; + int depth = layerParams.get("depth", CV_32F); + layerParams.type = (depth == CV_8S) ? "PoolingInt8" : "Pooling"; layerParams.set("pool", "MAX"); layerParams.set("ceil_mode", layerParams.has("pad_mode")); addLayer(layerParams, node_proto); @@ -988,7 +1047,8 @@ void ONNXImporter::parseSplit(LayerParams& layerParams, const opencv_onnx::NodeP { layerParams.set("num_split", node_proto.output_size()); } - layerParams.type = "Slice"; + int depth = layerParams.get("depth", CV_32F); + layerParams.type = (depth == CV_8S) ? "SliceInt8" : "Slice"; addLayer(layerParams, node_proto); } @@ -1743,7 +1803,8 @@ void ONNXImporter::parseConvTranspose(LayerParams& layerParams, const opencv_onn void ONNXImporter::parseTranspose(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) { - layerParams.type = "Permute"; + int depth = layerParams.get("depth", CV_32F); + layerParams.type = (depth == CV_8S) ? "PermuteInt8" : "Permute"; replaceLayerParam(layerParams, "perm", "order"); CV_Assert(node_proto.input_size() == 1); @@ -1807,6 +1868,8 @@ void ONNXImporter::parseSqueeze(LayerParams& layerParams, const opencv_onnx::Nod addConstant(layerParams.name, out); return; } + int depth = layerParams.get("depth", CV_32F); + layerParams.type += (depth == CV_8S) ? "Int8" : ""; addLayer(layerParams, node_proto); } @@ -1862,12 +1925,14 @@ void ONNXImporter::parseUnsqueeze(LayerParams& layerParams, const opencv_onnx::N if (axes.size() != 1) CV_Error(Error::StsNotImplemented, "Multidimensional unsqueeze"); + int depth = layerParams.get("depth", CV_32F); + MatShape inpShape = outShapes[node_proto.input(0)]; int axis = axes.getIntValue(0); CV_Assert(0 <= axis && axis <= inpShape.size()); std::vector outShape = inpShape; outShape.insert(outShape.begin() + axis, 1); - layerParams.type = "Reshape"; + layerParams.type = (depth == CV_8S) ? "ReshapeInt8" : "Reshape"; layerParams.set("dim", DictValue::arrayInt(&outShape[0], outShape.size())); if (hasDynamicShapes) { @@ -2004,6 +2069,8 @@ void ONNXImporter::parseExpand(LayerParams& layerParams, const opencv_onnx::Node void ONNXImporter::parseReshape(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) { CV_Assert(node_proto.input_size() == 2 || layerParams.has("shape")); + int depth = layerParams.get("depth", CV_32F); + layerParams.type += (depth == CV_8S) ? "Int8" : ""; if (node_proto.input_size() == 2) { Mat blob = getBlob(node_proto, 1); @@ -2038,7 +2105,8 @@ void ONNXImporter::parseReshape(LayerParams& layerParams, const opencv_onnx::Nod void ONNXImporter::parsePad(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) { - layerParams.type = "Padding"; + int depth = layerParams.get("depth", CV_32F); + layerParams.type = (depth == CV_8S) ? "PaddingInt8" : "Padding"; replaceLayerParam(layerParams, "mode", "type"); if (node_proto.input_size() == 3 || node_proto.input_size() == 2) { @@ -2051,7 +2119,8 @@ void ONNXImporter::parsePad(LayerParams& layerParams, const opencv_onnx::NodePro if (node_proto.input_size() == 3) { Mat value = getBlob(node_proto, 2); - layerParams.set("value", value.ptr()[0]); + float padValue = (depth == CV_8S) ? (float)value.ptr()[0] : value.ptr()[0]; + layerParams.set("value", padValue); } } addLayer(layerParams, node_proto); @@ -2270,6 +2339,9 @@ void ONNXImporter::parseResize(LayerParams& layerParams, const opencv_onnx::Node for (int i = 1; i < node_proto.input_size(); i++) CV_Assert(layer_id.find(node_proto.input(i)) == layer_id.end()); + int depth = layerParams.get("depth", CV_32F); + layerParams.type += (depth == CV_8S) ? "Int8" : ""; + if (layerParams.has("coordinate_transformation_mode")) { String interp_mode = layerParams.get("coordinate_transformation_mode"); @@ -2419,6 +2491,396 @@ void ONNXImporter::parseCustomLayer(LayerParams& layerParams, const opencv_onnx: addLayer(layerParams, node_proto); } +void ONNXImporter::parseQuantDequant(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +{ + CV_Assert(node_proto.input_size() == 3); + layerParams.type = (node_proto.op_type() == "QuantizeLinear") ? "Quantize" : "Dequantize"; + + if (node_proto.op_type() == "DequantizeLinear") + { + Mat scale = getBlob(node_proto, 1); + Mat zeropoint = getBlob(node_proto, 2); + + layerParams.set("scales", DictValue::arrayReal(scale.ptr(), 1)); + layerParams.set("zeropoints", DictValue::arrayInt(zeropoint.ptr(), 1)); + } + addLayer(layerParams, node_proto); +} + +void ONNXImporter::parseQConv(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +{ + int ninputs = node_proto.input_size(); + CV_Assert(ninputs == 8 || ninputs == 9); + + Mat inp_sc = getBlob(node_proto, 1); + Mat inp_zp = getBlob(node_proto, 2); + + Mat weights = getBlob(node_proto, 3); + int outCn = weights.size[0]; + Mat w_scale = getBlob(node_proto, 4); + CV_Assert(w_scale.total() == 1 || w_scale.total() == outCn); + Mat wt_sc = (w_scale.total() == outCn) ? w_scale : Mat(1, outCn, CV_32F, Scalar(w_scale.at(0))); + + Mat out_sc = getBlob(node_proto, 6); + Mat bias = (ninputs == 9) ? getBlob(node_proto, 8) : Mat::zeros(1, outCn, CV_32S); + + Mat weights_2d = weights.reshape(1, outCn); + Mat biasFused(1, outCn, CV_32S); + Mat outputMultiplier(1, outCn, CV_32F); + for (int i = 0; i < outCn; i++) + { + biasFused.at(i) = bias.at(i) - inp_zp.at(0)*(cv::sum(weights_2d.row(i))[0]); + outputMultiplier.at(i) = (inp_sc.at(0) * wt_sc.at(i)) / out_sc.at(0); + } + + layerParams.type = "ConvolutionInt8"; + layerParams.set("num_output", outCn); + layerParams.set("input_zeropoint", inp_zp.at(0)); + layerParams.blobs.push_back(weights); + layerParams.blobs.push_back(biasFused); + layerParams.blobs.push_back(outputMultiplier); + addLayer(layerParams, node_proto); +} + +void ONNXImporter::parseQMatMul(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +{ + int ninputs = node_proto.input_size(); + CV_Assert(ninputs == 8); + + if (constBlobs.find(node_proto.input(3)) == constBlobs.end()) + CV_Error(Error::StsNotImplemented, "Variable weights is not supported"); + + int firstInpDims = outShapes[node_proto.input(0)].size(); + + Mat inp_sc = getBlob(node_proto, 1); + Mat inp_zp = getBlob(node_proto, 2); + + Mat weights = getBlob(node_proto, 3).t(); + int outCn = weights.size[0]; + int secondInpDims = weights.dims; + + Mat w_scale = getBlob(node_proto, 4); + CV_Assert(w_scale.total() == 1 || w_scale.total() == outCn); + Mat wt_sc = (w_scale.total() == outCn) ? w_scale : Mat(1, outCn, CV_32F, Scalar(w_scale.at(0))); + Mat out_sc = getBlob(node_proto, 6); + + Mat bias(1, outCn, CV_32S); + Mat outputMultiplier(1, outCn, CV_32F); + for (int i = 0; i < outCn; i++) + { + bias.at(i) = -inp_zp.at(0)*(cv::sum(weights.row(i))[0]); + outputMultiplier.at(i) = (inp_sc.at(0) * wt_sc.at(i)) / out_sc.at(0); + } + + layerParams.type = "InnerProductInt8"; + layerParams.set("num_output", outCn); + layerParams.set("axis", firstInpDims - secondInpDims + 1); + layerParams.blobs.push_back(weights); + layerParams.blobs.push_back(bias); + layerParams.blobs.push_back(outputMultiplier); + addLayer(layerParams, node_proto); +} + +void ONNXImporter::parseQEltwise(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto_) +{ + opencv_onnx::NodeProto node_proto = node_proto_; + CV_Assert(node_proto.input_size() == 8); + std::string op = (node_proto.op_type() == "QLinearAdd") ? "sum" : "prod"; + int constId = -1; + for (int i = 0; i < 4; i += 3) + { + if (constBlobs.find(node_proto.input(i)) != constBlobs.end()) + constId = i; + } + + Mat inp_0_sc = getBlob(node_proto, 1); + Mat inp_0_zp = getBlob(node_proto, 2); + + Mat inp_1_sc = getBlob(node_proto, 4); + Mat inp_1_zp = getBlob(node_proto, 5); + + // Set 2nd input as the const input + if (constId == 0) + { + cv::swap(inp_0_sc, inp_1_sc); + cv::swap(inp_0_zp, inp_1_zp); + } + + float out_sc = getBlob(node_proto, 6).at(0); + int8_t out_zp = getBlob(node_proto, 7).at(0); + + std::vector inp_scales = {inp_0_sc.at(0), inp_1_sc.at(0)}; + std::vector inp_zps = {inp_0_zp.at(0), inp_1_zp.at(0)}; + + std::vector coeffs; + float offset; + if (op == "sum") + { + coeffs = {inp_scales[0]/out_sc, inp_scales[1]/out_sc}; + offset = out_zp - coeffs[0]*inp_zps[0] - coeffs[1]*inp_zps[1]; + } + else + { + coeffs = {inp_scales[0]/out_sc, inp_scales[1]}; + offset = out_zp; + } + + if (constId != -1) + { + Mat blob = getBlob(node_proto, constId); + if (blob.total() == 1) + { + float val = inp_scales[1] * (blob.at(0) - inp_zps[1]); + float scale = inp_scales[0] / out_sc; + if (op == "prod") + scale *= val; + + float shift = out_zp - scale*inp_zps[0]; + if (op == "sum") + shift += (val/out_sc); + + LayerParams rescaleParams; + rescaleParams.name = layerParams.name; + rescaleParams.type = "Requantize"; + rescaleParams.set("depth", CV_8S); + rescaleParams.set("scale", scale); + rescaleParams.set("shift", shift); + addLayer(rescaleParams, node_proto); + return; + } + else + { + MatShape inpShape = outShapes[node_proto.input(3 - constId)]; + if (blob.dims == 2) + blob = blob.t(); + + if (shape(blob) == inpShape) + { + LayerParams constParams; + constParams.name = layerParams.name + "/const"; + constParams.type = "ConstInt8"; + constParams.set("depth", CV_8S); + constParams.set("scales", DictValue::arrayReal(inp_1_sc.ptr(), 1)); + constParams.set("zeropoints", DictValue::arrayInt(inp_1_zp.ptr(), 1)); + constParams.blobs.push_back(blob); + + int id = dstNet.addLayer(constParams.name, constParams.type, CV_8S, constParams); + layer_id.insert(std::make_pair(constParams.name, LayerInfo(id, 0))); + outShapes[constParams.name] = shape(blob); + node_proto.set_input(constId, constParams.name); + + layerParams.type = "EltwiseInt8"; + layerParams.set("operation", op); + layerParams.set("coeff", DictValue::arrayReal(coeffs.data(), coeffs.size())); + layerParams.set("offset", offset); + } + else + { + layerParams.type = "ScaleInt8"; + layerParams.set("bias_term", op == "sum"); + int axis = 1; + for (int i = 0; i < graph_proto.initializer_size(); i++) + { + opencv_onnx::TensorProto tensor_proto = graph_proto.initializer(i); + if (tensor_proto.name() == node_proto.input(constId)) + { + axis = inpShape.size() - tensor_proto.dims_size(); + break; + } + } + layerParams.set("axis", axis); + blob = blob.reshape(1, 1); + Mat blob_dequantized; + blob.convertTo(blob_dequantized, CV_32F, inp_scales[1], -(inp_scales[1] * inp_zps[1])); + layerParams.blobs.push_back(blob_dequantized); + layerParams.set("input_scales", DictValue::arrayReal(inp_scales.data(), inp_scales.size())); + } + } + } + else if (outShapes[node_proto.input(0)] == outShapes[node_proto.input(3)]) + { + layerParams.type = "EltwiseInt8"; + layerParams.set("operation", op); + layerParams.set("coeff", DictValue::arrayReal(coeffs.data(), coeffs.size())); + layerParams.set("offset", offset); + } + else + { + layerParams.type = "ScaleInt8"; + layerParams.set("bias_term", op == "sum"); + layerParams.set("input_scales", DictValue::arrayReal(inp_scales.data(), inp_scales.size())); + } + + layerParams.set("input_zeropoints", DictValue::arrayInt(inp_zps.data(), inp_zps.size())); + addLayer(layerParams, node_proto); +} + +void ONNXImporter::parseQLeakyRelu(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +{ + CV_Assert(node_proto.input_size() == 5); + + float slope = layerParams.get("alpha"); + float inp_sc = getBlob(node_proto, 1).at(0); + int8_t inp_zp = getBlob(node_proto, 2).at(0); + float out_sc = getBlob(node_proto, 3).at(0); + int8_t out_zp = getBlob(node_proto, 4).at(0); + + Mat lookUpTable(1, 256, CV_8S); + int8_t* table = lookUpTable.ptr(); + for (int i = -128; i < 128; i++) + { + float x = inp_sc*(i - inp_zp); + float y = x >= 0.f ? x : slope*x; + int quantized = out_zp + cvRound(y/out_sc); + table[i+128] = saturate_cast(quantized); + } + + layerParams.type = "ReLUInt8"; + layerParams.blobs.push_back(lookUpTable); + addLayer(layerParams, node_proto); +} + +void ONNXImporter::parseQSigmoid(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +{ + CV_Assert(node_proto.input_size() == 5); + + float inp_sc = getBlob(node_proto, 1).at(0); + int8_t inp_zp = getBlob(node_proto, 2).at(0); + float out_sc = getBlob(node_proto, 3).at(0); + int8_t out_zp = getBlob(node_proto, 4).at(0); + + Mat lookUpTable(1, 256, CV_8S); + int8_t* table = lookUpTable.ptr(); + for (int i = -128; i < 128; i++) + { + float x = inp_sc*(i - inp_zp); + float y = 1.f/(1.f + std::exp(-x)); + int quantized = out_zp + cvRound(y/out_sc); + table[i+128] = saturate_cast(quantized); + } + + layerParams.type = "SigmoidInt8"; + layerParams.blobs.push_back(lookUpTable); + addLayer(layerParams, node_proto); +} + +void ONNXImporter::parseQAvgPool(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +{ + CV_Assert(node_proto.input_size() == 5); + float inp_sc = getBlob(node_proto, 1).at(0); + int8_t inp_zp = getBlob(node_proto, 2).at(0); + float out_sc = getBlob(node_proto, 3).at(0); + + layerParams.type = "PoolingInt8"; + layerParams.set("pool", "ave"); + layerParams.set("global_pooling", node_proto.op_type() == "QLinearGlobalAveragePool"); + layerParams.set("multiplier", inp_sc/out_sc); + layerParams.set("input_zeropoint", inp_zp); + addLayer(layerParams, node_proto); +} + +void ONNXImporter::parseQConcat(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto_) +{ + opencv_onnx::NodeProto node_proto = node_proto_; + layerParams.type = "ConcatInt8"; + int num_inputs = node_proto.input_size(); + + float out_scale = getBlob(node_proto, 0).at(0); + int out_zp = getBlob(node_proto, 1).at(0); + + for (int i = 2; i < num_inputs; i += 3) + { + float inp_scale = getBlob(node_proto, i + 1).at(0); + int inp_zp = getBlob(node_proto, i + 2).at(0); + + if (inp_scale != out_scale || inp_zp != out_zp) + { + float scale = inp_scale/out_scale; + float shift = out_zp - scale*inp_zp; + + if (constBlobs.find(node_proto.input(i)) != constBlobs.end()) + { + Mat blob = getBlob(node_proto, i); + Mat blob_rescaled; + blob.convertTo(blob_rescaled, CV_8S, scale, shift); + constBlobs[node_proto.input(i)] = blob_rescaled; + } + else + { + LayerParams rescaleParams; + rescaleParams.name = node_proto.input(i) + "/rescale"; + rescaleParams.type = "Requantize"; + rescaleParams.set("depth", CV_8S); + rescaleParams.set("scale", scale); + rescaleParams.set("shift", shift); + + opencv_onnx::NodeProto proto; + proto.add_input(node_proto.input(i)); + proto.add_output(rescaleParams.name); + addLayer(rescaleParams, proto); + node_proto.set_input(i, rescaleParams.name); + } + } + } + + bool hasVariableInps = false; + for (int i = 2; i < num_inputs; i += 3) + { + if (layer_id.find(node_proto.input(i)) != layer_id.end()) + { + hasVariableInps = true; + break; + } + } + + if (!hasVariableInps) + { + std::vector inputs, concatenated; + MatShape inputShape; + for (size_t i = 2; i < num_inputs; i += 3) + { + Mat blob = getBlob(node_proto, i); + if (blob.size.dims() > inputShape.size()) + { + inputShape = shape(blob); + } + inputs.push_back(blob); + } + + int axis = layerParams.get("axis", 1); + for (size_t i = 0; i < inputs.size(); ++i) + { + MatShape targetShape = inputShape; + targetShape[axis] = shape(inputs[i])[axis]; + CV_CheckEQ(total(targetShape), total(shape(inputs[i])), ""); + inputs[i] = inputs[i].reshape(0, targetShape); + } + runLayer(layerParams, inputs, concatenated); + CV_Assert(concatenated.size() == 1); + addConstant(layerParams.name, concatenated[0]); + return; + } + else + { + for (int i = 2; i < num_inputs; i += 3) + { + if (constBlobs.find(node_proto.input(i)) != constBlobs.end()) + { + LayerParams constParams; + constParams.name = node_proto.input(i); + constParams.type = "ConstInt8"; + constParams.blobs.push_back(getBlob(node_proto, i)); + constParams.set("depth", CV_8S); + + opencv_onnx::NodeProto proto; + proto.add_output(constParams.name); + addLayer(constParams, proto); + } + } + } + addLayer(layerParams, node_proto); +} + const ONNXImporter::DispatchMap ONNXImporter::buildDispatchMap() { DispatchMap dispatch; @@ -2468,6 +2930,14 @@ const ONNXImporter::DispatchMap ONNXImporter::buildDispatchMap() dispatch["SoftMax"] = dispatch["LogSoftmax"] = &ONNXImporter::parseSoftMax; dispatch["DetectionOutput"] = &ONNXImporter::parseDetectionOutput; dispatch["CumSum"] = &ONNXImporter::parseCumSum; + dispatch["QuantizeLinear"] = dispatch["DequantizeLinear"] = &ONNXImporter::parseQuantDequant; + dispatch["QLinearConv"] = &ONNXImporter::parseQConv; + dispatch["QLinearMatMul"] = &ONNXImporter::parseQMatMul; + dispatch["QLinearAdd"] = dispatch["QLinearMul"] = &ONNXImporter::parseQEltwise; + dispatch["QLinearLeakyRelu"] = &ONNXImporter::parseQLeakyRelu; + dispatch["QLinearSigmoid"] = &ONNXImporter::parseQSigmoid; + dispatch["QLinearAveragePool"] = dispatch["QLinearGlobalAveragePool"] = &ONNXImporter::parseQAvgPool; + dispatch["QLinearConcat"] = &ONNXImporter::parseQConcat; return dispatch; } diff --git a/modules/dnn/test/test_int8_layers.cpp b/modules/dnn/test/test_int8_layers.cpp index 1fcb1d0dba..5e6c05c716 100644 --- a/modules/dnn/test/test_int8_layers.cpp +++ b/modules/dnn/test/test_int8_layers.cpp @@ -583,7 +583,7 @@ TEST_P(Test_Int8_nets, ResNet50) Mat blob = blobFromImage(inp, 1.0, Size(224, 224), Scalar(), false); Mat ref = blobFromNPY(_tf("resnet50_prob.npy")); - float l1 = 3e-4, lInf = 0.035; + float l1 = 3e-4, lInf = 0.04; testClassificationNet(net, blob, ref, l1, lInf); } @@ -714,7 +714,7 @@ TEST_P(Test_Int8_nets, MobileNet_v1_SSD_PPN) Mat blob = blobFromImage(inp, 1.0, Size(300, 300), Scalar(), true, false); Mat ref = blobFromNPY(_tf("tensorflow/ssd_mobilenet_v1_ppn_coco.detection_out.npy")); - float confThreshold = 0.51, scoreDiff = 0.04, iouDiff = 0.06; + float confThreshold = 0.51, scoreDiff = 0.05, iouDiff = 0.06; testDetectionNet(net, blob, ref, confThreshold, scoreDiff, iouDiff); } @@ -815,7 +815,7 @@ TEST_P(Test_Int8_nets, FasterRCNN_resnet50) Mat blob = blobFromImage(inp, 1.0, Size(800, 600), Scalar(), true, false); Mat ref = blobFromNPY(_tf("tensorflow/faster_rcnn_resnet50_coco_2018_01_28.detection_out.npy")); - float confThreshold = 0.5, scoreDiff = 0.025, iouDiff = 0.15; + float confThreshold = 0.5, scoreDiff = 0.05, iouDiff = 0.15; testDetectionNet(net, blob, ref, confThreshold, scoreDiff, iouDiff); } @@ -1127,7 +1127,7 @@ TEST_P(Test_Int8_nets, YOLOv4) std::string config_file = "yolov4.cfg"; std::string weights_file = "yolov4.weights"; - double scoreDiff = 0.1, iouDiff = 0.17; + double scoreDiff = 0.15, iouDiff = 0.2; { SCOPED_TRACE("batch size 1"); testDarknetModel(config_file, weights_file, ref.rowRange(0, N0), scoreDiff, iouDiff); diff --git a/modules/dnn/test/test_onnx_importer.cpp b/modules/dnn/test/test_onnx_importer.cpp index 4334da2ad6..5d324b8aac 100644 --- a/modules/dnn/test/test_onnx_importer.cpp +++ b/modules/dnn/test/test_onnx_importer.cpp @@ -991,6 +991,112 @@ TEST_P(Test_ONNX_layers, ConvResizePool1d) testONNXModels("conv_resize_pool_1d"); } +TEST_P(Test_ONNX_layers, Quantized_Convolution) +{ + testONNXModels("quantized_conv_uint8_weights", npy, 0.004, 0.02); + testONNXModels("quantized_conv_int8_weights", npy, 0.03, 0.5); + testONNXModels("quantized_conv_per_channel_weights", npy, 0.06, 0.4); +} + +TEST_P(Test_ONNX_layers, Quantized_MatMul) +{ + testONNXModels("quantized_matmul_uint8_weights", npy, 0.005, 0.007); + testONNXModels("quantized_matmul_int8_weights", npy, 0.06, 0.2); + testONNXModels("quantized_matmul_per_channel_weights", npy, 0.06, 0.22); +} + +TEST_P(Test_ONNX_layers, Quantized_MatMul_Variable_Weights) +{ + // Unsupported + EXPECT_THROW( + { + testONNXModels("quantized_matmul_variable_inputs"); + }, cv::Exception); +} + +TEST_P(Test_ONNX_layers, Quantized_Eltwise) +{ + testONNXModels("quantized_eltwise"); +} + +TEST_P(Test_ONNX_layers, Quantized_Eltwise_Scalar) +{ + testONNXModels("quantized_eltwise_scalar"); +} + +TEST_P(Test_ONNX_layers, Quantized_Eltwise_Broadcast) +{ + testONNXModels("quantized_eltwise_broadcast"); +} + +TEST_P(Test_ONNX_layers, Quantized_LeakyReLU) +{ + testONNXModels("quantized_leaky_relu"); +} + +TEST_P(Test_ONNX_layers, Quantized_Sigmoid) +{ + testONNXModels("quantized_sigmoid"); +} + +TEST_P(Test_ONNX_layers, Quantized_MaxPool) +{ + testONNXModels("quantized_maxpool"); +} + +TEST_P(Test_ONNX_layers, Quantized_AvgPool) +{ + testONNXModels("quantized_avgpool"); +} + +TEST_P(Test_ONNX_layers, Quantized_Split) +{ + testONNXModels("quantized_split"); +} + +TEST_P(Test_ONNX_layers, Quantized_Pad) +{ + testONNXModels("quantized_padding"); +} + +TEST_P(Test_ONNX_layers, Quantized_Reshape) +{ + testONNXModels("quantized_reshape"); +} + +TEST_P(Test_ONNX_layers, Quantized_Transpose) +{ + testONNXModels("quantized_transpose"); +} + +TEST_P(Test_ONNX_layers, Quantized_Squeeze) +{ + testONNXModels("quantized_squeeze"); +} + +TEST_P(Test_ONNX_layers, Quantized_Unsqueeze) +{ + testONNXModels("quantized_unsqueeze"); +} + +TEST_P(Test_ONNX_layers, Quantized_Resize) +{ + testONNXModels("quantized_resize_nearest"); + testONNXModels("quantized_resize_bilinear", npy, 2e-4, 0.003); + testONNXModels("quantized_resize_bilinear_align", npy, 3e-4, 0.003); +} + +TEST_P(Test_ONNX_layers, Quantized_Concat) +{ + testONNXModels("quantized_concat"); + testONNXModels("quantized_concat_const_blob"); +} + +TEST_P(Test_ONNX_layers, Quantized_Constant) +{ + testONNXModels("quantized_constant", npy, 0.002, 0.008); +} + INSTANTIATE_TEST_CASE_P(/*nothing*/, Test_ONNX_layers, dnnBackendsAndTargets()); class Test_ONNX_nets : public Test_ONNX_layers @@ -1127,6 +1233,11 @@ TEST_P(Test_ONNX_nets, ResNet50v1) testONNXModels("resnet50v1", pb, default_l1, default_lInf, true, target != DNN_TARGET_MYRIAD); } +TEST_P(Test_ONNX_nets, ResNet50_Int8) +{ + testONNXModels("resnet50_int8", pb, default_l1, default_lInf, true); +} + TEST_P(Test_ONNX_nets, ResNet101_DUC_HDC) { applyTestTag(CV_TEST_TAG_VERYLONG); From 4938765eb3f9808ae4e08a006bb4e88eedc55a86 Mon Sep 17 00:00:00 2001 From: Shivanshu Tyagi Date: Mon, 4 Oct 2021 23:48:02 +0530 Subject: [PATCH 222/262] Merge pull request #20291 from spazewalker:master speech recognition sample * speech recognition sample added.(initial commit) * fixed typos, removed plt * trailing whitespaces removed * masking removed and using opencv for displaying spectrogram * description added * requested changes and add opencl fp16 target * parenthesis and halide removed * workaround 3d matrix issue * handle multi channel audio support for multiple files at once * suggested changes fix whitespaces --- samples/dnn/speech_recognition.py | 506 ++++++++++++++++++++++++++++++ 1 file changed, 506 insertions(+) create mode 100644 samples/dnn/speech_recognition.py diff --git a/samples/dnn/speech_recognition.py b/samples/dnn/speech_recognition.py new file mode 100644 index 0000000000..025607edab --- /dev/null +++ b/samples/dnn/speech_recognition.py @@ -0,0 +1,506 @@ +import numpy as np +import cv2 as cv +import argparse +import os +import soundfile as sf # Temporary import to load audio files + +''' + You can download the converted onnx model from https://drive.google.com/drive/folders/1wLtxyao4ItAg8tt4Sb63zt6qXzhcQoR6?usp=sharing + or convert the model yourself. + + You can get the original pre-trained Jasper model from NVIDIA : https://ngc.nvidia.com/catalog/models/nvidia:jasper_pyt_onnx_fp16_amp/files + Download and unzip : `$ wget --content-disposition https://api.ngc.nvidia.com/v2/models/nvidia/jasper_pyt_onnx_fp16_amp/versions/20.10.0/zip -O jasper_pyt_onnx_fp16_amp_20.10.0.zip && unzip -o ./jasper_pyt_onnx_fp16_amp_20.10.0.zip && unzip -o ./jasper_pyt_onnx_fp16_amp.zip` + + you can get the script to convert the model here : https://gist.github.com/spazewalker/507f1529e19aea7e8417f6e935851a01 + + You can convert the model using the following steps: + 1. Import onnx and load the original model + ``` + import onnx + model = onnx.load("./jasper-onnx/1/model.onnx") + ``` + + 3. Change data type of input layer + ``` + inp = model.graph.input[0] + model.graph.input.remove(inp) + inp.type.tensor_type.elem_type = 1 + model.graph.input.insert(0,inp) + ``` + + 4. Change the data type of output layer + ``` + out = model.graph.output[0] + model.graph.output.remove(out) + out.type.tensor_type.elem_type = 1 + model.graph.output.insert(0,out) + ``` + + 5. Change the data type of every initializer and cast it's values from FP16 to FP32 + ``` + for i,init in enumerate(model.graph.initializer): + model.graph.initializer.remove(init) + init.data_type = 1 + init.raw_data = np.frombuffer(init.raw_data, count=np.product(init.dims), dtype=np.float16).astype(np.float32).tobytes() + model.graph.initializer.insert(i,init) + ``` + + 6. Add an additional reshape node to handle the inconsistant input from python and c++ of openCV. + see https://github.com/opencv/opencv/issues/19091 + Make & insert a new node with 'Reshape' operation & required initializer + ``` + tensor = numpy_helper.from_array(np.array([0,64,-1]),name='shape_reshape') + model.graph.initializer.insert(0,tensor) + node = onnx.helper.make_node(op_type='Reshape',inputs=['input__0','shape_reshape'], outputs=['input_reshaped'], name='reshape__0') + model.graph.node.insert(0,node) + model.graph.node[1].input[0] = 'input_reshaped' + ``` + + 7. Finally save the model + ``` + with open('jasper_dynamic_input_float.onnx','wb') as f: + onnx.save_model(model,f) + ``` + + Original Repo : https://github.com/NVIDIA/DeepLearningExamples/tree/master/PyTorch/SpeechRecognition/Jasper + ''' + +class FilterbankFeatures: + def __init__(self, + sample_rate=16000, window_size=0.02, window_stride=0.01, + n_fft=512, preemph=0.97, n_filt=64, lowfreq=0, + highfreq=None, log=True, dither=1e-5): + ''' + Initializes pre-processing class. Default values are the values used by the Jasper + architecture for pre-processing. For more details, refer to the paper here: + https://arxiv.org/abs/1904.03288 + ''' + self.win_length = int(sample_rate * window_size) # frame size + self.hop_length = int(sample_rate * window_stride) # stride + self.n_fft = n_fft or 2 ** np.ceil(np.log2(self.win_length)) + self.log = log + self.dither = dither + self.n_filt = n_filt + self.preemph = preemph + highfreq = highfreq or sample_rate / 2 + self.window_tensor = np.hanning(self.win_length) + + self.filterbanks = self.mel(sample_rate, self.n_fft, n_mels=n_filt, fmin=lowfreq, fmax=highfreq) + self.filterbanks.dtype=np.float32 + self.filterbanks = np.expand_dims(self.filterbanks,0) + + def normalize_batch(self, x, seq_len): + ''' + Normalizes the features. + ''' + x_mean = np.zeros((seq_len.shape[0], x.shape[1]), dtype=x.dtype) + x_std = np.zeros((seq_len.shape[0], x.shape[1]), dtype=x.dtype) + for i in range(x.shape[0]): + x_mean[i, :] = np.mean(x[i, :, :seq_len[i]],axis=1) + x_std[i, :] = np.std(x[i, :, :seq_len[i]],axis=1) + # make sure x_std is not zero + x_std += 1e-10 + return (x - np.expand_dims(x_mean,2)) / np.expand_dims(x_std,2) + + def calculate_features(self, x, seq_len): + ''' + Calculates filterbank features. + args: + x : mono channel audio + seq_len : length of the audio sample + returns: + x : filterbank features + ''' + dtype = x.dtype + + seq_len = np.ceil(seq_len / self.hop_length) + seq_len = np.array(seq_len,dtype=np.int32) + + # dither + if self.dither > 0: + x += self.dither * np.random.randn(*x.shape) + + # do preemphasis + if self.preemph is not None: + x = np.concatenate( + (np.expand_dims(x[0],-1), x[1:] - self.preemph * x[:-1]), axis=0) + + # Short Time Fourier Transform + x = self.stft(x, n_fft=self.n_fft, hop_length=self.hop_length, + win_length=self.win_length, + fft_window=self.window_tensor) + + # get power spectrum + x = (x**2).sum(-1) + + # dot with filterbank energies + x = np.matmul(np.array(self.filterbanks,dtype=x.dtype), x) + + # log features if required + if self.log: + x = np.log(x + 1e-20) + + # normalize if required + x = self.normalize_batch(x, seq_len).astype(dtype) + return x + + # Mel Frequency calculation + def hz_to_mel(self, frequencies): + ''' + Converts frequencies from hz to mel scale. Input can be a number or a vector. + ''' + frequencies = np.asanyarray(frequencies) + + f_min = 0.0 + f_sp = 200.0 / 3 + + mels = (frequencies - f_min) / f_sp + + # Fill in the log-scale part + min_log_hz = 1000.0 # beginning of log region (Hz) + min_log_mel = (min_log_hz - f_min) / f_sp # same (Mels) + logstep = np.log(6.4) / 27.0 # step size for log region + + if frequencies.ndim: + # If we have array data, vectorize + log_t = frequencies >= min_log_hz + mels[log_t] = min_log_mel + np.log(frequencies[log_t] / min_log_hz) / logstep + elif frequencies >= min_log_hz: + # If we have scalar data, directly + mels = min_log_mel + np.log(frequencies / min_log_hz) / logstep + return mels + + def mel_to_hz(self, mels): + ''' + Converts frequencies from mel to hz scale. Input can be a number or a vector. + ''' + mels = np.asanyarray(mels) + + # Fill in the linear scale + f_min = 0.0 + f_sp = 200.0 / 3 + freqs = f_min + f_sp * mels + + # And now the nonlinear scale + min_log_hz = 1000.0 # beginning of log region (Hz) + min_log_mel = (min_log_hz - f_min) / f_sp # same (Mels) + logstep = np.log(6.4) / 27.0 # step size for log region + + if mels.ndim: + # If we have vector data, vectorize + log_t = mels >= min_log_mel + freqs[log_t] = min_log_hz * np.exp(logstep * (mels[log_t] - min_log_mel)) + elif mels >= min_log_mel: + # If we have scalar data, check directly + freqs = min_log_hz * np.exp(logstep * (mels - min_log_mel)) + + return freqs + + def mel_frequencies(self, n_mels=128, fmin=0.0, fmax=11025.0): + ''' + Calculates n mel frequencies between 2 frequencies + args: + n_mels : number of bands + fmin : min frequency + fmax : max frequency + returns: + mels : vector of mel frequencies + ''' + # 'Center freqs' of mel bands - uniformly spaced between limits + min_mel = self.hz_to_mel(fmin) + max_mel = self.hz_to_mel(fmax) + + mels = np.linspace(min_mel, max_mel, n_mels) + + return self.mel_to_hz(mels) + + def mel(self, sr, n_fft, n_mels=128, fmin=0.0, fmax=None, dtype=np.float32): + ''' + Generates mel filterbank + args: + sr : Sampling rate + n_fft : number of FFT components + n_mels : number of Mel bands to generate + fmin : lowest frequency (in Hz) + fmax : highest frequency (in Hz). sr/2.0 if None + dtype : the data type of the output basis. + returns: + mels : Mel transform matrix + ''' + # default Max freq = half of sampling rate + if fmax is None: + fmax = float(sr) / 2 + + # Initialize the weights + n_mels = int(n_mels) + weights = np.zeros((n_mels, int(1 + n_fft // 2)), dtype=dtype) + + # Center freqs of each FFT bin + fftfreqs = np.linspace(0, float(sr) / 2, int(1 + n_fft // 2), endpoint=True) + + # 'Center freqs' of mel bands - uniformly spaced between limits + mel_f = self.mel_frequencies(n_mels + 2, fmin=fmin, fmax=fmax) + + fdiff = np.diff(mel_f) + ramps = np.subtract.outer(mel_f, fftfreqs) + + for i in range(n_mels): + # lower and upper slopes for all bins + lower = -ramps[i] / fdiff[i] + upper = ramps[i + 2] / fdiff[i + 1] + + # .. then intersect them with each other and zero + weights[i] = np.maximum(0, np.minimum(lower, upper)) + + # Using Slaney-style mel which is scaled to be approx constant energy per channel + enorm = 2.0 / (mel_f[2 : n_mels + 2] - mel_f[:n_mels]) + weights *= enorm[:, np.newaxis] + return weights + + # STFT preperation + def pad_window_center(self, data, size, axis=-1, **kwargs): + ''' + Centers the data and pads. + args: + data : Vector to be padded and centered + size : Length to pad data + axis : Axis along which to pad and center the data + kwargs : arguments passed to np.pad + return : centered and padded data + ''' + kwargs.setdefault("mode", "constant") + n = data.shape[axis] + lpad = int((size - n) // 2) + lengths = [(0, 0)] * data.ndim + lengths[axis] = (lpad, int(size - n - lpad)) + if lpad < 0: + raise Exception( + ("Target size ({:d}) must be at least input size ({:d})").format(size, n) + ) + return np.pad(data, lengths, **kwargs) + + def frame(self, x, frame_length, hop_length): + ''' + Slices a data array into (overlapping) frames. + args: + x : array to frame + frame_length : length of frame + hop_length : Number of steps to advance between frames + return : A framed view of `x` + ''' + if x.shape[-1] < frame_length: + raise Exception( + "Input is too short (n={:d})" + " for frame_length={:d}".format(x.shape[-1], frame_length) + ) + x = np.asfortranarray(x) + n_frames = 1 + (x.shape[-1] - frame_length) // hop_length + strides = np.asarray(x.strides) + new_stride = np.prod(strides[strides > 0] // x.itemsize) * x.itemsize + shape = list(x.shape)[:-1] + [frame_length, n_frames] + strides = list(strides) + [hop_length * new_stride] + return np.lib.stride_tricks.as_strided(x, shape=shape, strides=strides) + + def dtype_r2c(self, d, default=np.complex64): + ''' + Find the complex numpy dtype corresponding to a real dtype. + args: + d : The real-valued dtype to convert to complex. + default : The default complex target type, if `d` does not match a known dtype + return : The complex dtype + ''' + mapping = { + np.dtype(np.float32): np.complex64, + np.dtype(np.float64): np.complex128, + } + dt = np.dtype(d) + if dt.kind == "c": + return dt + return np.dtype(mapping.get(dt, default)) + + def stft(self, y, n_fft, hop_length=None, win_length=None, fft_window=None, pad_mode='reflect', return_complex=False): + ''' + Short Time Fourier Transform. The STFT represents a signal in the time-frequency + domain by computing discrete Fourier transforms (DFT) over short overlapping windows. + args: + y : input signal + n_fft : length of the windowed signal after padding with zeros. + hop_length : number of audio samples between adjacent STFT columns. + win_length : Each frame of audio is windowed by window of length win_length and + then padded with zeros to match n_fft + fft_window : a vector or array of length `n_fft` having values computed by a + window function + pad_mode : mode while padding the singnal + return_complex : returns array with complex data type if `True` + return : Matrix of short-term Fourier transform coefficients. + ''' + if win_length is None: + win_length = n_fft + if hop_length is None: + hop_length = int(win_length // 4) + if y.ndim!=1: + raise Exception(f'Invalid input shape. Only Mono Channeled audio supported. Input must have shape (Audio,). Got {y.shape}') + + # Pad the window out to n_fft size + fft_window = self.pad_window_center(fft_window, n_fft) + + # Reshape so that the window can be broadcast + fft_window = fft_window.reshape((-1, 1)) + + # Pad the time series so that frames are centered + y = np.pad(y, int(n_fft // 2), mode=pad_mode) + + # Window the time series. + y_frames = self.frame(y, frame_length=n_fft, hop_length=hop_length) + + # Convert data type to complex + dtype = self.dtype_r2c(y.dtype) + + # Pre-allocate the STFT matrix + stft_matrix = np.empty( (int(1 + n_fft // 2), y_frames.shape[-1]), dtype=dtype, order="F") + + stft_matrix = np.fft.rfft( fft_window * y_frames, axis=0) + return stft_matrix if return_complex==True else np.stack((stft_matrix.real,stft_matrix.imag),axis=-1) + +class Decoder: + ''' + Used for decoding the output of jasper model. + ''' + def __init__(self): + labels=[' ','a','b','c','d','e','f','g','h','i','j','k','l','m','n','o','p','q','r','s','t','u','v','w','x','y','z',"'"] + self.labels_map = {i: label for i,label in enumerate(labels)} + self.blank_id = 28 + + def decode(self,x): + """ + Takes output of Jasper model and performs ctc decoding algorithm to + remove duplicates and special symbol. Returns prediction + """ + x = np.argmax(x,axis=-1) + hypotheses = [] + prediction = x.tolist() + # CTC decoding procedure + decoded_prediction = [] + previous = self.blank_id + for p in prediction: + if (p != previous or previous == self.blank_id) and p != self.blank_id: + decoded_prediction.append(p) + previous = p + hypothesis = ''.join([self.labels_map[c] for c in decoded_prediction]) + hypotheses.append(hypothesis) + return hypotheses + +def predict(features, net, decoder): + ''' + Passes the features through the Jasper model and decodes the output to english transcripts. + args: + features : input features, calculated using FilterbankFeatures class + net : Jasper model dnn.net object + decoder : Decoder object + return : Predicted text + ''' + # This is a workaround https://github.com/opencv/opencv/issues/19091 + # expanding 1 dimentions allows us to pass it to the network + # from python. This should be resolved in the future. + features = np.expand_dims(features,axis=3) + + # make prediction + net.setInput(features) + output = net.forward() + + # decode output to transcript + prediction = decoder.decode(output.squeeze(0)) + return prediction[0] + +if __name__ == '__main__': + + # Computation backends supported by layers + backends = (cv.dnn.DNN_BACKEND_DEFAULT, cv.dnn.DNN_BACKEND_INFERENCE_ENGINE, cv.dnn.DNN_BACKEND_OPENCV) + # Target Devices for computation + targets = (cv.dnn.DNN_TARGET_CPU, cv.dnn.DNN_TARGET_OPENCL, cv.dnn.DNN_TARGET_OPENCL_FP16) + + parser = argparse.ArgumentParser(description='This script runs Jasper Speech recognition model', + formatter_class=argparse.ArgumentDefaultsHelpFormatter) + parser.add_argument('--input_audio', type=str, required=True, help='Path to input audio file. OR Path to a txt file with relative path to multiple audio files in different lines') + parser.add_argument('--show_spectrogram', action='store_true', help='Whether to show a spectrogram of the input audio.') + parser.add_argument('--model', type=str, default='jasper.onnx', help='Path to the onnx file of Jasper. default="jasper.onnx"') + parser.add_argument('--output', type=str, help='Path to file where recognized audio transcript must be saved. Leave this to print on console.') + parser.add_argument('--backend', choices=backends, default=cv.dnn.DNN_BACKEND_DEFAULT, type=int, + help='Select a computation backend: ' + "%d: automatically (by default) " + "%d: OpenVINO Inference Engine " + "%d: OpenCV Implementation " % backends) + parser.add_argument('--target', choices=targets, default=cv.dnn.DNN_TARGET_CPU, type=int, + help='Select a target device: ' + "%d: CPU target (by default) " + "%d: OpenCL " + "%d: OpenCL FP16 " % targets) + + args, _ = parser.parse_known_args() + + if args.input_audio and not os.path.isfile(args.input_audio): + raise OSError("Input audio file does not exist") + if not os.path.isfile(args.model): + raise OSError("Jasper model file does not exist") + if args.input_audio.endswith('.txt'): + with open(args.input_audio) as f: + content = f.readlines() + content = [x.strip() for x in content] + audio_file_paths = content + for audio_file_path in audio_file_paths: + if not os.path.isfile(audio_file_path): + raise OSError("Audio file({audio_file_path}) does not exist") + else: + audio_file_paths = [args.input_audio] + audio_file_paths = [os.path.abspath(x) for x in audio_file_paths] + + # Read audio Files + features = [] + try: + for audio_file_path in audio_file_paths: + audio = sf.read(audio_file_path) + # If audio is stereo, just take one channel. + X = audio[0] if audio[0].ndim==1 else audio[0][:,0] + features.append(X) + except: + raise Exception(f"Soundfile cannot read {args.input_audio}. Try a different format") + + # Get Filterbank Features + feature_extractor = FilterbankFeatures() + for i in range(len(features)): + X = features[i] + seq_len = np.array([X.shape[0]], dtype=np.int32) + features[i] = feature_extractor.calculate_features(x=X, seq_len=seq_len) + + # Load Network + net = cv.dnn.readNetFromONNX(args.model) + net.setPreferableBackend(args.backend) + net.setPreferableTarget(args.target) + + # Show spectogram if required + if args.show_spectrogram and not args.input_audio.endswith('.txt'): + img = cv.normalize(src=features[0][0], dst=None, alpha=0, beta=255, norm_type=cv.NORM_MINMAX, dtype=cv.CV_8U) + img = cv.applyColorMap(img, cv.COLORMAP_JET) + cv.imshow('spectogram', img) + cv.waitKey(0) + + # Initialize decoder + decoder = Decoder() + + # Make prediction + prediction = [] + print("Predicting...") + for feature in features: + print(f"\rAudio file {len(prediction)+1}/{len(features)}", end='') + prediction.append(predict(feature, net, decoder)) + print("") + + # save transcript if required + if args.output: + with open(args.output,'w') as f: + for pred in prediction: + f.write(pred+'\n') + print("Transcript was written to {}".format(args.output)) + else: + print(prediction) + cv.destroyAllWindows() From ebef84e9ea8492d15e02ef622e652af29a632eff Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Mon, 4 Oct 2021 20:47:07 +0000 Subject: [PATCH 223/262] pre: OpenCV 3.4.16 (version++) --- .../cross_referencing/tutorial_cross_referencing.markdown | 4 ++-- modules/core/include/opencv2/core/version.hpp | 4 ++-- modules/dnn/include/opencv2/dnn/dnn.hpp | 4 ++-- modules/python/package/setup.py | 2 +- platforms/android/build_sdk.py | 2 +- platforms/android/service/readme.txt | 2 +- platforms/maven/opencv-it/pom.xml | 2 +- platforms/maven/opencv/pom.xml | 2 +- platforms/maven/pom.xml | 2 +- 9 files changed, 12 insertions(+), 12 deletions(-) diff --git a/doc/tutorials/introduction/cross_referencing/tutorial_cross_referencing.markdown b/doc/tutorials/introduction/cross_referencing/tutorial_cross_referencing.markdown index ac347cbca9..27dfc5588a 100644 --- a/doc/tutorials/introduction/cross_referencing/tutorial_cross_referencing.markdown +++ b/doc/tutorials/introduction/cross_referencing/tutorial_cross_referencing.markdown @@ -39,14 +39,14 @@ Open your Doxyfile using your favorite text editor and search for the key `TAGFILES`. Change it as follows: @code -TAGFILES = ./docs/doxygen-tags/opencv.tag=http://docs.opencv.org/3.4.15 +TAGFILES = ./docs/doxygen-tags/opencv.tag=http://docs.opencv.org/3.4.16 @endcode If you had other definitions already, you can append the line using a `\`: @code TAGFILES = ./docs/doxygen-tags/libstdc++.tag=https://gcc.gnu.org/onlinedocs/libstdc++/latest-doxygen \ - ./docs/doxygen-tags/opencv.tag=http://docs.opencv.org/3.4.15 + ./docs/doxygen-tags/opencv.tag=http://docs.opencv.org/3.4.16 @endcode Doxygen can now use the information from the tag file to link to the OpenCV diff --git a/modules/core/include/opencv2/core/version.hpp b/modules/core/include/opencv2/core/version.hpp index b80af3e713..dc8a3e603c 100644 --- a/modules/core/include/opencv2/core/version.hpp +++ b/modules/core/include/opencv2/core/version.hpp @@ -7,8 +7,8 @@ #define CV_VERSION_MAJOR 3 #define CV_VERSION_MINOR 4 -#define CV_VERSION_REVISION 15 -#define CV_VERSION_STATUS "-dev" +#define CV_VERSION_REVISION 16 +#define CV_VERSION_STATUS "-pre" #define CVAUX_STR_EXP(__A) #__A #define CVAUX_STR(__A) CVAUX_STR_EXP(__A) diff --git a/modules/dnn/include/opencv2/dnn/dnn.hpp b/modules/dnn/include/opencv2/dnn/dnn.hpp index 524dc587e6..a2a48b7f3e 100644 --- a/modules/dnn/include/opencv2/dnn/dnn.hpp +++ b/modules/dnn/include/opencv2/dnn/dnn.hpp @@ -47,9 +47,9 @@ #include "opencv2/core/async.hpp" #if !defined CV_DOXYGEN && !defined CV_STATIC_ANALYSIS && !defined CV_DNN_DONT_ADD_EXPERIMENTAL_NS -#define CV__DNN_EXPERIMENTAL_NS_BEGIN namespace experimental_dnn_34_v22 { +#define CV__DNN_EXPERIMENTAL_NS_BEGIN namespace experimental_dnn_34_v23 { #define CV__DNN_EXPERIMENTAL_NS_END } -namespace cv { namespace dnn { namespace experimental_dnn_34_v22 { } using namespace experimental_dnn_34_v22; }} +namespace cv { namespace dnn { namespace experimental_dnn_34_v23 { } using namespace experimental_dnn_34_v23; }} #else #define CV__DNN_EXPERIMENTAL_NS_BEGIN #define CV__DNN_EXPERIMENTAL_NS_END diff --git a/modules/python/package/setup.py b/modules/python/package/setup.py index 7c015770e6..47a0629dc7 100644 --- a/modules/python/package/setup.py +++ b/modules/python/package/setup.py @@ -9,7 +9,7 @@ def main(): os.chdir(SCRIPT_DIR) package_name = 'opencv' - package_version = os.environ.get('OPENCV_VERSION', '3.4.15') # TODO + package_version = os.environ.get('OPENCV_VERSION', '3.4.16') # TODO long_description = 'Open Source Computer Vision Library Python bindings' # TODO diff --git a/platforms/android/build_sdk.py b/platforms/android/build_sdk.py index cbde665611..54d6f8fb4a 100755 --- a/platforms/android/build_sdk.py +++ b/platforms/android/build_sdk.py @@ -269,7 +269,7 @@ class Builder: # Add extra data apkxmldest = check_dir(os.path.join(apkdest, "res", "xml"), create=True) apklibdest = check_dir(os.path.join(apkdest, "libs", abi.name), create=True) - for ver, d in self.extra_packs + [("3.4.15", os.path.join(self.libdest, "lib"))]: + for ver, d in self.extra_packs + [("3.4.16", os.path.join(self.libdest, "lib"))]: r = ET.Element("library", attrib={"version": ver}) log.info("Adding libraries from %s", d) diff --git a/platforms/android/service/readme.txt b/platforms/android/service/readme.txt index a10138c1a6..c3d3244a18 100644 --- a/platforms/android/service/readme.txt +++ b/platforms/android/service/readme.txt @@ -12,7 +12,7 @@ manually using adb tool: adb install /apk/OpenCV__Manager__.apk -Example: OpenCV_3.4.15-dev_Manager_3.49_armeabi-v7a.apk +Example: OpenCV_3.4.16-dev_Manager_3.49_armeabi-v7a.apk Use the list of platforms below to determine proper OpenCV Manager package for your device: diff --git a/platforms/maven/opencv-it/pom.xml b/platforms/maven/opencv-it/pom.xml index bef5fe28aa..039f951160 100644 --- a/platforms/maven/opencv-it/pom.xml +++ b/platforms/maven/opencv-it/pom.xml @@ -4,7 +4,7 @@ org.opencv opencv-parent - 3.4.15 + 3.4.16 org.opencv opencv-it diff --git a/platforms/maven/opencv/pom.xml b/platforms/maven/opencv/pom.xml index 6b11c7a0fa..64f3f5656f 100644 --- a/platforms/maven/opencv/pom.xml +++ b/platforms/maven/opencv/pom.xml @@ -4,7 +4,7 @@ org.opencv opencv-parent - 3.4.15 + 3.4.16 org.opencv opencv diff --git a/platforms/maven/pom.xml b/platforms/maven/pom.xml index 69e5772489..f3fadc106f 100644 --- a/platforms/maven/pom.xml +++ b/platforms/maven/pom.xml @@ -3,7 +3,7 @@ 4.0.0 org.opencv opencv-parent - 3.4.15 + 3.4.16 pom OpenCV Parent POM From 3e6f27522b1c5a57a680cb553a2ef42945268a5f Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Mon, 4 Oct 2021 22:35:47 +0000 Subject: [PATCH 224/262] pre: OpenCV 4.5.4 (version++) --- .../cross_referencing/tutorial_cross_referencing.markdown | 4 ++-- modules/core/include/opencv2/core/version.hpp | 4 ++-- modules/dnn/include/opencv2/dnn/version.hpp | 2 +- modules/python/package/setup.py | 2 +- platforms/maven/opencv-it/pom.xml | 2 +- platforms/maven/opencv/pom.xml | 2 +- platforms/maven/pom.xml | 2 +- 7 files changed, 9 insertions(+), 9 deletions(-) diff --git a/doc/tutorials/introduction/cross_referencing/tutorial_cross_referencing.markdown b/doc/tutorials/introduction/cross_referencing/tutorial_cross_referencing.markdown index 7493560635..da3047aa87 100644 --- a/doc/tutorials/introduction/cross_referencing/tutorial_cross_referencing.markdown +++ b/doc/tutorials/introduction/cross_referencing/tutorial_cross_referencing.markdown @@ -46,14 +46,14 @@ Open your Doxyfile using your favorite text editor and search for the key `TAGFILES`. Change it as follows: @code -TAGFILES = ./docs/doxygen-tags/opencv.tag=http://docs.opencv.org/4.5.3 +TAGFILES = ./docs/doxygen-tags/opencv.tag=http://docs.opencv.org/4.5.4 @endcode If you had other definitions already, you can append the line using a `\`: @code TAGFILES = ./docs/doxygen-tags/libstdc++.tag=https://gcc.gnu.org/onlinedocs/libstdc++/latest-doxygen \ - ./docs/doxygen-tags/opencv.tag=http://docs.opencv.org/4.5.3 + ./docs/doxygen-tags/opencv.tag=http://docs.opencv.org/4.5.4 @endcode Doxygen can now use the information from the tag file to link to the OpenCV diff --git a/modules/core/include/opencv2/core/version.hpp b/modules/core/include/opencv2/core/version.hpp index f627d71472..4aaf30ce5e 100644 --- a/modules/core/include/opencv2/core/version.hpp +++ b/modules/core/include/opencv2/core/version.hpp @@ -7,8 +7,8 @@ #define CV_VERSION_MAJOR 4 #define CV_VERSION_MINOR 5 -#define CV_VERSION_REVISION 3 -#define CV_VERSION_STATUS "-dev" +#define CV_VERSION_REVISION 4 +#define CV_VERSION_STATUS "-pre" #define CVAUX_STR_EXP(__A) #__A #define CVAUX_STR(__A) CVAUX_STR_EXP(__A) diff --git a/modules/dnn/include/opencv2/dnn/version.hpp b/modules/dnn/include/opencv2/dnn/version.hpp index 0efc9c0237..63e70d2d6e 100644 --- a/modules/dnn/include/opencv2/dnn/version.hpp +++ b/modules/dnn/include/opencv2/dnn/version.hpp @@ -6,7 +6,7 @@ #define OPENCV_DNN_VERSION_HPP /// Use with major OpenCV version only. -#define OPENCV_DNN_API_VERSION 20210608 +#define OPENCV_DNN_API_VERSION 20211004 #if !defined CV_DOXYGEN && !defined CV_STATIC_ANALYSIS && !defined CV_DNN_DONT_ADD_INLINE_NS #define CV__DNN_INLINE_NS __CV_CAT(dnn4_v, OPENCV_DNN_API_VERSION) diff --git a/modules/python/package/setup.py b/modules/python/package/setup.py index cb585aa80a..030523182a 100644 --- a/modules/python/package/setup.py +++ b/modules/python/package/setup.py @@ -9,7 +9,7 @@ def main(): os.chdir(SCRIPT_DIR) package_name = 'opencv' - package_version = os.environ.get('OPENCV_VERSION', '4.5.3') # TODO + package_version = os.environ.get('OPENCV_VERSION', '4.5.4') # TODO long_description = 'Open Source Computer Vision Library Python bindings' # TODO diff --git a/platforms/maven/opencv-it/pom.xml b/platforms/maven/opencv-it/pom.xml index 1166eee2fb..0209abc3a1 100644 --- a/platforms/maven/opencv-it/pom.xml +++ b/platforms/maven/opencv-it/pom.xml @@ -4,7 +4,7 @@ org.opencv opencv-parent - 4.5.3 + 4.5.4 org.opencv opencv-it diff --git a/platforms/maven/opencv/pom.xml b/platforms/maven/opencv/pom.xml index 4ac2c2b7f5..3551ffc09f 100644 --- a/platforms/maven/opencv/pom.xml +++ b/platforms/maven/opencv/pom.xml @@ -4,7 +4,7 @@ org.opencv opencv-parent - 4.5.3 + 4.5.4 org.opencv opencv diff --git a/platforms/maven/pom.xml b/platforms/maven/pom.xml index 72f4f82b1c..66aa22c8ec 100644 --- a/platforms/maven/pom.xml +++ b/platforms/maven/pom.xml @@ -3,7 +3,7 @@ 4.0.0 org.opencv opencv-parent - 4.5.3 + 4.5.4 pom OpenCV Parent POM From c54abde1bd775788dd2a49c4f6cbb44afd9924f3 Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Tue, 5 Oct 2021 10:09:27 +0300 Subject: [PATCH 225/262] ffmpeg/3.4: update FFmpeg wrapper 2021.10 - FFmpeg 3.4.8 (no changes) --- 3rdparty/ffmpeg/ffmpeg.cmake | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/3rdparty/ffmpeg/ffmpeg.cmake b/3rdparty/ffmpeg/ffmpeg.cmake index a1ada4eeaa..336bf0d227 100644 --- a/3rdparty/ffmpeg/ffmpeg.cmake +++ b/3rdparty/ffmpeg/ffmpeg.cmake @@ -1,8 +1,8 @@ -# Binaries branch name: ffmpeg/3.4_20210302 -# Binaries were created for OpenCV: 2ab1f3f166fccc3a01497209cc01c5cea44ff201 -ocv_update(FFMPEG_BINARIES_COMMIT "e99214251d9f3cde7c48abd46b2259bddc9885b6") -ocv_update(FFMPEG_FILE_HASH_BIN32 "fad5ada9be36120bba8966709e7953a8") -ocv_update(FFMPEG_FILE_HASH_BIN64 "650e2272728491923e566f784f79cfef") +# Binaries branch name: ffmpeg/3.4_20211005 +# Binaries were created for OpenCV: 95c1d2a8872b222f32bd88db9f1efcbd9f70a9cf +ocv_update(FFMPEG_BINARIES_COMMIT "0bf6c0753d435d2c82c03c48db0c6e18ac79976c") +ocv_update(FFMPEG_FILE_HASH_BIN32 "55c25bbc13e4a12d4339b70d3b76987f") +ocv_update(FFMPEG_FILE_HASH_BIN64 "67caee9231c6843483b4de9815d6526e") ocv_update(FFMPEG_FILE_HASH_CMAKE "3b90f67f4b429e77d3da36698cef700c") function(download_win_ffmpeg script_var) From 19a880bb918122168f656804d43993b81e7b18ae Mon Sep 17 00:00:00 2001 From: Giles Payne Date: Tue, 5 Oct 2021 20:16:06 +0900 Subject: [PATCH 226/262] Simple matrix multiplication for Mat in iOS/Android --- modules/core/misc/java/src/java/core+Mat.java | 22 ++++++++++++++-- .../core/misc/java/src/java/core+MatMatMul.kt | 3 +++ modules/core/misc/java/test/MatTest.java | 10 +++++++ modules/core/misc/objc/common/Mat.h | 10 +++++++ modules/core/misc/objc/common/Mat.mm | 5 ++++ modules/core/misc/objc/common/MatExt.swift | 6 +++++ modules/core/misc/objc/test/MatTest.swift | 10 +++++++ modules/java/generator/src/cpp/Mat.cpp | 26 +++++++++++++++++++ 8 files changed, 90 insertions(+), 2 deletions(-) create mode 100644 modules/core/misc/java/src/java/core+MatMatMul.kt diff --git a/modules/core/misc/java/src/java/core+Mat.java b/modules/core/misc/java/src/java/core+Mat.java index 5fcc727738..2d9ee314a3 100644 --- a/modules/core/misc/java/src/java/core+Mat.java +++ b/modules/core/misc/java/src/java/core+Mat.java @@ -466,16 +466,32 @@ public class Mat { // C++: Mat Mat::mul(Mat m, double scale = 1) // - // javadoc: Mat::mul(m, scale) + /** + * Element-wise multiplication with scale factor + * @param m operand with with which to perform element-wise multiplication + * @param scale scale factor + */ public Mat mul(Mat m, double scale) { return new Mat(n_mul(nativeObj, m.nativeObj, scale)); } - // javadoc: Mat::mul(m) + /** + * Element-wise multiplication + * @param m operand with with which to perform element-wise multiplication + */ public Mat mul(Mat m) { return new Mat(n_mul(nativeObj, m.nativeObj)); } + /** + * Matrix multiplication + * @param m operand with with which to perform matrix multiplication + * @see Core#gemm(Mat, Mat, double, Mat, double, Mat, int) + */ + public Mat matMul(Mat m) { + return new Mat(n_matMul(nativeObj, m.nativeObj)); + } + // // C++: static Mat Mat::ones(int rows, int cols, int type) // @@ -1732,6 +1748,8 @@ public class Mat { private static native long n_mul(long nativeObj, long m_nativeObj); + private static native long n_matMul(long nativeObj, long m_nativeObj); + // C++: static Mat Mat::ones(int rows, int cols, int type) private static native long n_ones(int rows, int cols, int type); diff --git a/modules/core/misc/java/src/java/core+MatMatMul.kt b/modules/core/misc/java/src/java/core+MatMatMul.kt new file mode 100644 index 0000000000..3dcb6bde79 --- /dev/null +++ b/modules/core/misc/java/src/java/core+MatMatMul.kt @@ -0,0 +1,3 @@ +package org.opencv.core + +operator fun Mat.times(other: Mat): Mat = this.matMul(other) diff --git a/modules/core/misc/java/test/MatTest.java b/modules/core/misc/java/test/MatTest.java index 3075dba16b..323440b2eb 100644 --- a/modules/core/misc/java/test/MatTest.java +++ b/modules/core/misc/java/test/MatTest.java @@ -686,6 +686,16 @@ public class MatTest extends OpenCVTestCase { assertMatEqual(truth, dst, EPS); } + public void testMatMulMat() { + Mat m1 = new Mat(2, 2, CvType.CV_32F, new Scalar(2)); + Mat m2 = new Mat(2, 2, CvType.CV_32F, new Scalar(3)); + + dst = m1.matMul(m2); + + truth = new Mat(2, 2, CvType.CV_32F, new Scalar(12)); + assertMatEqual(truth, dst, EPS); + } + public void testOnesIntIntInt() { dst = Mat.ones(matSize, matSize, CvType.CV_32F); diff --git a/modules/core/misc/objc/common/Mat.h b/modules/core/misc/objc/common/Mat.h index fd1dce27ba..b5e868306c 100644 --- a/modules/core/misc/objc/common/Mat.h +++ b/modules/core/misc/objc/common/Mat.h @@ -114,7 +114,17 @@ CV_EXPORTS @interface Mat : NSObject - (BOOL)isSubmatrix; - (void)locateROI:(Size2i*)wholeSize ofs:(Point2i*)offset NS_SWIFT_NAME(locateROI(wholeSize:offset:)); - (Mat*)mul:(Mat*)mat scale:(double)scale; +/** + Performs element-wise multiplication + @param mat operand with with which to perform element-wise multiplication +*/ - (Mat*)mul:(Mat*)mat; +/** + Performs matrix multiplication + @param mat operand with with which to perform matrix multiplication + @see `Core.gemm(...)` +*/ +- (Mat*)matMul:(Mat*)mat; + (Mat*)ones:(int)rows cols:(int)cols type:(int)type NS_SWIFT_NAME(ones(rows:cols:type:)); + (Mat*)ones:(Size2i*)size type:(int)type NS_SWIFT_NAME(ones(size:type:)); + (Mat*)onesEx:(NSArray*)sizes type:(int)type NS_SWIFT_NAME(ones(sizes:type:)); diff --git a/modules/core/misc/objc/common/Mat.mm b/modules/core/misc/objc/common/Mat.mm index dc2316c87b..ab5e1de6f9 100644 --- a/modules/core/misc/objc/common/Mat.mm +++ b/modules/core/misc/objc/common/Mat.mm @@ -372,6 +372,11 @@ static bool updateIdx(cv::Mat* mat, std::vector& indices, size_t inc) { return [[Mat alloc] initWithNativeMat:new cv::Mat(_nativePtr->mul(*(cv::Mat*)mat.nativePtr))]; } +- (Mat*)matMul:(Mat*)mat { + cv::Mat temp = self.nativeRef * mat.nativeRef; + return [Mat fromNative:temp]; +} + + (Mat*)ones:(int)rows cols:(int)cols type:(int)type { return [[Mat alloc] initWithNativeMat:new cv::Mat(cv::Mat::ones(rows, cols, type))]; } diff --git a/modules/core/misc/objc/common/MatExt.swift b/modules/core/misc/objc/common/MatExt.swift index a6ba548599..f0f4446682 100644 --- a/modules/core/misc/objc/common/MatExt.swift +++ b/modules/core/misc/objc/common/MatExt.swift @@ -715,3 +715,9 @@ public extension Mat { return MatAt(mat: self, indices: indices) } } + +public extension Mat { + static func *(lhs:Mat, rhs: Mat) -> Mat { + return lhs.matMul(rhs) + } +} diff --git a/modules/core/misc/objc/test/MatTest.swift b/modules/core/misc/objc/test/MatTest.swift index 2dcfedf41f..87f99fb84a 100644 --- a/modules/core/misc/objc/test/MatTest.swift +++ b/modules/core/misc/objc/test/MatTest.swift @@ -683,6 +683,16 @@ class MatTests: OpenCVTestCase { try assertMatEqual(truth!, dst, OpenCVTestCase.EPS) } + func testMatMulMat() throws { + let m1 = Mat(rows: 2, cols: 2, type: CvType.CV_32F, scalar: Scalar(2)) + let m2 = Mat(rows: 2, cols: 2, type: CvType.CV_32F, scalar: Scalar(3)) + + dst = m1.matMul(m2) + + truth = Mat(rows: 2, cols: 2, type: CvType.CV_32F, scalar: Scalar(12)) + try assertMatEqual(truth!, dst, OpenCVTestCase.EPS) + } + func testOnesIntIntInt() throws { dst = Mat.ones(rows: OpenCVTestCase.matSize, cols: OpenCVTestCase.matSize, type: CvType.CV_32F) diff --git a/modules/java/generator/src/cpp/Mat.cpp b/modules/java/generator/src/cpp/Mat.cpp index d59fe4a506..f43a4c6ed5 100644 --- a/modules/java/generator/src/cpp/Mat.cpp +++ b/modules/java/generator/src/cpp/Mat.cpp @@ -1372,6 +1372,32 @@ JNIEXPORT jlong JNICALL Java_org_opencv_core_Mat_n_1mul__JJ } +// +// Mat Mat Mat::matMul(Mat m) +// + +JNIEXPORT jlong JNICALL Java_org_opencv_core_Mat_n_1matMul__JJ + (JNIEnv* env, jclass, jlong self, jlong m_nativeObj); + +JNIEXPORT jlong JNICALL Java_org_opencv_core_Mat_n_1matMul__JJ + (JNIEnv* env, jclass, jlong self, jlong m_nativeObj) +{ + static const char method_name[] = "Mat::n_1matMul__JJ()"; + try { + LOGD("%s", method_name); + Mat* me = (Mat*) self; //TODO: check for NULL + Mat& m = *((Mat*)m_nativeObj); + Mat _retval_ = (*me) * m; + return (jlong) new Mat(_retval_); + } catch(const std::exception &e) { + throwJavaException(env, &e, method_name); + } catch (...) { + throwJavaException(env, 0, method_name); + } + + return 0; +} + // // static Mat Mat::ones(int rows, int cols, int type) From b7a7119b1fc44a64d22b59410482b4351cd6dcdb Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Tue, 5 Oct 2021 14:32:39 +0300 Subject: [PATCH 227/262] ffmpeg/4.x: update FFmpeg wrapper 2021.10 - FFmpeg 4.4 (no changes) --- 3rdparty/ffmpeg/ffmpeg.cmake | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/3rdparty/ffmpeg/ffmpeg.cmake b/3rdparty/ffmpeg/ffmpeg.cmake index 764e4b19d8..2eea658178 100644 --- a/3rdparty/ffmpeg/ffmpeg.cmake +++ b/3rdparty/ffmpeg/ffmpeg.cmake @@ -1,8 +1,8 @@ -# Binaries branch name: ffmpeg/master_20210608 -# Binaries were created for OpenCV: eaa9228a4fdfb9c2465aea65a50ce2d16b55dce0 -ocv_update(FFMPEG_BINARIES_COMMIT "213fcd5d4897319a83207406036c4a5957fba010") -ocv_update(FFMPEG_FILE_HASH_BIN32 "bab661341c30862fa88627130219c0a5") -ocv_update(FFMPEG_FILE_HASH_BIN64 "ac99f9767a83103c31709628af685924") +# Binaries branch name: ffmpeg/master_20211005 +# Binaries were created for OpenCV: 672399c751c431bbe52818b33fd3ca17b51e0e16 +ocv_update(FFMPEG_BINARIES_COMMIT "40b4666d1aa374205fd61373496e15d92ecd5313") +ocv_update(FFMPEG_FILE_HASH_BIN32 "c2f9a897d464a2dce2286f8067ad9d90") +ocv_update(FFMPEG_FILE_HASH_BIN64 "878a4e8fe5a4d68f18c9cdde543b9ead") ocv_update(FFMPEG_FILE_HASH_CMAKE "8862c87496e2e8c375965e1277dee1c7") function(download_win_ffmpeg script_var) From e5fb50476c14d0f096ae2b93b2b4be578710a43f Mon Sep 17 00:00:00 2001 From: HAN Liutong Date: Tue, 5 Oct 2021 23:35:00 +0800 Subject: [PATCH 228/262] Merge pull request #20521 from hanliutong:dev-rvv-multiVLEN Make the implementation of optimization in DNN adjustable to different vector sizes with RVV intrinsics. * Update fastGEMM for multi VLEN. * Update fastGEMM1T for multi VLEN. * Update fastDepthwiseConv for multi VLEN. * Update fastConv for multi VLEN. * Replace malloc with cv::AutoBuffer. --- modules/dnn/src/layers/layers_common.simd.hpp | 604 ++++++++++-------- 1 file changed, 335 insertions(+), 269 deletions(-) diff --git a/modules/dnn/src/layers/layers_common.simd.hpp b/modules/dnn/src/layers/layers_common.simd.hpp index 762e22e54d..0a077e4631 100644 --- a/modules/dnn/src/layers/layers_common.simd.hpp +++ b/modules/dnn/src/layers/layers_common.simd.hpp @@ -744,58 +744,66 @@ void fastGEMM( const float* aptr, size_t astep, const float* bptr, int ma, int na, int nb ) { int n = 0; - size_t vl = 8; - size_t mvl0 = 8; - size_t mvl1 = 8; - for( ; n < nb; n += 16 ) + int vl = vsetvlmax_e32m4(); + int mvl = vl; + for( ; n < nb; n += vl ) { - if ( n + 16 > nb) { - mvl0 = nb - n; - mvl1 = (nb - n -8) > 0 ? (nb - n -8) : 0; + if ( n + vl > nb) { + mvl = nb - n; } - for( int m = 0; m < ma; m += 4 ) + for( int m = 0; m < ma; m += 7 ) { const float* aptr0 = aptr + astep*m; const float* aptr1 = aptr + astep*std::min(m+1, ma-1); const float* aptr2 = aptr + astep*std::min(m+2, ma-1); const float* aptr3 = aptr + astep*std::min(m+3, ma-1); + const float* aptr4 = aptr + astep*std::min(m+4, ma-1); + const float* aptr5 = aptr + astep*std::min(m+5, ma-1); + const float* aptr6 = aptr + astep*std::min(m+6, ma-1); float* cptr0 = cptr + cstep*m; float* cptr1 = cptr + cstep*std::min(m+1, ma-1); float* cptr2 = cptr + cstep*std::min(m+2, ma-1); float* cptr3 = cptr + cstep*std::min(m+3, ma-1); + float* cptr4 = cptr + cstep*std::min(m+4, ma-1); + float* cptr5 = cptr + cstep*std::min(m+5, ma-1); + float* cptr6 = cptr + cstep*std::min(m+6, ma-1); - vfloat32m2_t d00 = vfmv_v_f_f32m2(0, vl), d01 = vfmv_v_f_f32m2(0, vl); - vfloat32m2_t d10 = vfmv_v_f_f32m2(0, vl), d11 = vfmv_v_f_f32m2(0, vl); - vfloat32m2_t d20 = vfmv_v_f_f32m2(0, vl), d21 = vfmv_v_f_f32m2(0, vl); - vfloat32m2_t d30 = vfmv_v_f_f32m2(0, vl), d31 = vfmv_v_f_f32m2(0, vl); + vfloat32m4_t d0 = vfmv_v_f_f32m4(0, vl); + vfloat32m4_t d1 = vfmv_v_f_f32m4(0, vl); + vfloat32m4_t d2 = vfmv_v_f_f32m4(0, vl); + vfloat32m4_t d3 = vfmv_v_f_f32m4(0, vl); + vfloat32m4_t d4 = vfmv_v_f_f32m4(0, vl); + vfloat32m4_t d5 = vfmv_v_f_f32m4(0, vl); + vfloat32m4_t d6 = vfmv_v_f_f32m4(0, vl); for( int k = 0; k < na; k++ ) { - vfloat32m2_t a0 = vfmv_v_f_f32m2(aptr0[k], vl); - vfloat32m2_t a1 = vfmv_v_f_f32m2(aptr1[k], vl); - vfloat32m2_t a2 = vfmv_v_f_f32m2(aptr2[k], vl); - vfloat32m2_t a3 = vfmv_v_f_f32m2(aptr3[k], vl); - vfloat32m2_t b0 = vle32_v_f32m2(bptr + k*bstep + n, mvl0); - vfloat32m2_t b1 = vle32_v_f32m2(bptr + k*bstep + n + 8, mvl1); - d00 = vfmacc_vv_f32m2(d00, a0, b0, mvl0); - d01 = vfmacc_vv_f32m2(d01, a0, b1, mvl1); - d10 = vfmacc_vv_f32m2(d10, a1, b0, mvl0); - d11 = vfmacc_vv_f32m2(d11, a1, b1, mvl1); - d20 = vfmacc_vv_f32m2(d20, a2, b0, mvl0); - d21 = vfmacc_vv_f32m2(d21, a2, b1, mvl1); - d30 = vfmacc_vv_f32m2(d30, a3, b0, mvl0); - d31 = vfmacc_vv_f32m2(d31, a3, b1, mvl1); + float32_t a0 = aptr0[k]; + float32_t a1 = aptr1[k]; + float32_t a2 = aptr2[k]; + float32_t a3 = aptr3[k]; + float32_t a4 = aptr4[k]; + float32_t a5 = aptr5[k]; + float32_t a6 = aptr6[k]; + + vfloat32m4_t b = vle32_v_f32m4(bptr + k*bstep + n, mvl); + d0 = vfmacc_vf_f32m4(d0, a0, b, mvl); + d1 = vfmacc_vf_f32m4(d1, a1, b, mvl); + d2 = vfmacc_vf_f32m4(d2, a2, b, mvl); + d3 = vfmacc_vf_f32m4(d3, a3, b, mvl); + d4 = vfmacc_vf_f32m4(d4, a4, b, mvl); + d5 = vfmacc_vf_f32m4(d5, a5, b, mvl); + d6 = vfmacc_vf_f32m4(d6, a6, b, mvl); } - vse32_v_f32m2(cptr0 + n, d00, mvl0); - vse32_v_f32m2(cptr1 + n, d10, mvl0); - vse32_v_f32m2(cptr2 + n, d20, mvl0); - vse32_v_f32m2(cptr3 + n, d30, mvl0); - vse32_v_f32m2(cptr0 + n + 8, d01, mvl1); - vse32_v_f32m2(cptr1 + n + 8, d11, mvl1); - vse32_v_f32m2(cptr2 + n + 8, d21, mvl1); - vse32_v_f32m2(cptr3 + n + 8, d31, mvl1); + vse32_v_f32m4(cptr0 + n, d0, mvl); + vse32_v_f32m4(cptr1 + n, d1, mvl); + vse32_v_f32m4(cptr2 + n, d2, mvl); + vse32_v_f32m4(cptr3 + n, d3, mvl); + vse32_v_f32m4(cptr4 + n, d4, mvl); + vse32_v_f32m4(cptr5 + n, d5, mvl); + vse32_v_f32m4(cptr6 + n, d6, mvl); } } } @@ -804,71 +812,108 @@ void fastGEMM1T( const float* vec, const float* weights, size_t wstep, const float* bias, float* dst, int nvecs, int vecsize ) { + int vlm2 = vsetvlmax_e32m2(); int i = 0; - size_t vl = 8; - for( ; i <= nvecs - 8; i += 8 ) + for( ; i <= nvecs - 15; i += 15 ) { const float* wptr = weights + i*wstep; - vfloat32m2_t vs0 = vfmv_v_f_f32m2(0, vl), vs1 = vfmv_v_f_f32m2(0, vl), - vs2 = vfmv_v_f_f32m2(0, vl), vs3 = vfmv_v_f_f32m2(0, vl), - vs4 = vfmv_v_f_f32m2(0, vl), vs5 = vfmv_v_f_f32m2(0, vl), - vs6 = vfmv_v_f_f32m2(0, vl), vs7 = vfmv_v_f_f32m2(0, vl); - - for( int k = 0; k < vecsize; k += 8, wptr += 8 ) + vfloat32m2_t + vs0 = vfmv_v_f_f32m2(0, vlm2), vs1 = vfmv_v_f_f32m2(0, vlm2), vs2 = vfmv_v_f_f32m2(0, vlm2), + vs3 = vfmv_v_f_f32m2(0, vlm2), vs4 = vfmv_v_f_f32m2(0, vlm2), vs5 = vfmv_v_f_f32m2(0, vlm2), + vs6 = vfmv_v_f_f32m2(0, vlm2), vs7 = vfmv_v_f_f32m2(0, vlm2), vs8 = vfmv_v_f_f32m2(0, vlm2), + vs9 = vfmv_v_f_f32m2(0, vlm2), vs10 = vfmv_v_f_f32m2(0, vlm2), vs11 = vfmv_v_f_f32m2(0, vlm2), + vs12 = vfmv_v_f_f32m2(0, vlm2), vs13 = vfmv_v_f_f32m2(0, vlm2), vs14 = vfmv_v_f_f32m2(0, vlm2); + int k = 0; + for( ; k < vecsize - vlm2; k += vlm2, wptr += vlm2 ) { - vfloat32m2_t v = vle32_v_f32m2(vec + k, vl); + vfloat32m2_t v = vle32_v_f32m2(vec + k, vlm2); - vs0 = vfmacc_vv_f32m2(vs0, vle32_v_f32m2(wptr, vl), v, vl); - vs1 = vfmacc_vv_f32m2(vs1, vle32_v_f32m2(wptr + wstep, vl), v, vl); - vs2 = vfmacc_vv_f32m2(vs2, vle32_v_f32m2(wptr + wstep*2, vl), v, vl); - vs3 = vfmacc_vv_f32m2(vs3, vle32_v_f32m2(wptr + wstep*3, vl), v, vl); - vs4 = vfmacc_vv_f32m2(vs4, vle32_v_f32m2(wptr + wstep*4, vl), v, vl); - vs5 = vfmacc_vv_f32m2(vs5, vle32_v_f32m2(wptr + wstep*5, vl), v, vl); - vs6 = vfmacc_vv_f32m2(vs6, vle32_v_f32m2(wptr + wstep*6, vl), v, vl); - vs7 = vfmacc_vv_f32m2(vs7, vle32_v_f32m2(wptr + wstep*7, vl), v, vl); + vs0 = vfmacc_vv_f32m2(vs0, vle32_v_f32m2(wptr, vlm2), v, vlm2); + vs1 = vfmacc_vv_f32m2(vs1, vle32_v_f32m2(wptr + wstep, vlm2), v, vlm2); + vs2 = vfmacc_vv_f32m2(vs2, vle32_v_f32m2(wptr + wstep*2, vlm2), v, vlm2); + vs3 = vfmacc_vv_f32m2(vs3, vle32_v_f32m2(wptr + wstep*3, vlm2), v, vlm2); + vs4 = vfmacc_vv_f32m2(vs4, vle32_v_f32m2(wptr + wstep*4, vlm2), v, vlm2); + vs5 = vfmacc_vv_f32m2(vs5, vle32_v_f32m2(wptr + wstep*5, vlm2), v, vlm2); + vs6 = vfmacc_vv_f32m2(vs6, vle32_v_f32m2(wptr + wstep*6, vlm2), v, vlm2); + vs7 = vfmacc_vv_f32m2(vs7, vle32_v_f32m2(wptr + wstep*7, vlm2), v, vlm2); + vs8 = vfmacc_vv_f32m2(vs8, vle32_v_f32m2(wptr + wstep*8, vlm2), v, vlm2); + vs9 = vfmacc_vv_f32m2(vs9, vle32_v_f32m2(wptr + wstep*9, vlm2), v, vlm2); + vs10 = vfmacc_vv_f32m2(vs10, vle32_v_f32m2(wptr + wstep*10, vlm2), v, vlm2); + vs11 = vfmacc_vv_f32m2(vs11, vle32_v_f32m2(wptr + wstep*11, vlm2), v, vlm2); + vs12 = vfmacc_vv_f32m2(vs12, vle32_v_f32m2(wptr + wstep*12, vlm2), v, vlm2); + vs13 = vfmacc_vv_f32m2(vs13, vle32_v_f32m2(wptr + wstep*13, vlm2), v, vlm2); + vs14 = vfmacc_vv_f32m2(vs14, vle32_v_f32m2(wptr + wstep*14, vlm2), v, vlm2); + } + int kvl = vecsize - k; + if (kvl > 0) { + vfloat32m2_t v = vle32_v_f32m2(vec + k, kvl); + vs0 = vfmacc_vv_f32m2(vs0, vle32_v_f32m2(wptr, kvl), v, kvl); + vs1 = vfmacc_vv_f32m2(vs1, vle32_v_f32m2(wptr + wstep*1, kvl), v, kvl); + vs2 = vfmacc_vv_f32m2(vs2, vle32_v_f32m2(wptr + wstep*2, kvl), v, kvl); + vs3 = vfmacc_vv_f32m2(vs3, vle32_v_f32m2(wptr + wstep*3, kvl), v, kvl); + vs4 = vfmacc_vv_f32m2(vs4, vle32_v_f32m2(wptr + wstep*4, kvl), v, kvl); + vs5 = vfmacc_vv_f32m2(vs5, vle32_v_f32m2(wptr + wstep*5, kvl), v, kvl); + vs6 = vfmacc_vv_f32m2(vs6, vle32_v_f32m2(wptr + wstep*6, kvl), v, kvl); + vs7 = vfmacc_vv_f32m2(vs7, vle32_v_f32m2(wptr + wstep*7, kvl), v, kvl); + vs8 = vfmacc_vv_f32m2(vs8, vle32_v_f32m2(wptr + wstep*8, kvl), v, kvl); + vs9 = vfmacc_vv_f32m2(vs9, vle32_v_f32m2(wptr + wstep*9, kvl), v, kvl); + vs10 = vfmacc_vv_f32m2(vs10, vle32_v_f32m2(wptr + wstep*10, kvl), v, kvl); + vs11 = vfmacc_vv_f32m2(vs11, vle32_v_f32m2(wptr + wstep*11, kvl), v, kvl); + vs12 = vfmacc_vv_f32m2(vs12, vle32_v_f32m2(wptr + wstep*12, kvl), v, kvl); + vs13 = vfmacc_vv_f32m2(vs13, vle32_v_f32m2(wptr + wstep*13, kvl), v, kvl); + vs14 = vfmacc_vv_f32m2(vs14, vle32_v_f32m2(wptr + wstep*14, kvl), v, kvl); } // Calculate the sum of each vector - vfloat32m1_t zero = vfmv_v_f_f32m1(0, vl); - vfloat32m1_t temp0 = vfredsum_vs_f32m2_f32m1(temp0, vs0, zero, vl); - vfloat32m1_t temp1 = vfredsum_vs_f32m2_f32m1(temp1, vs1, zero, vl); - vfloat32m1_t temp2 = vfredsum_vs_f32m2_f32m1(temp2, vs2, zero, vl); - vfloat32m1_t temp3 = vfredsum_vs_f32m2_f32m1(temp3, vs3, zero, vl); - vfloat32m1_t temp4 = vfredsum_vs_f32m2_f32m1(temp4, vs4, zero, vl); - vfloat32m1_t temp5 = vfredsum_vs_f32m2_f32m1(temp5, vs5, zero, vl); - vfloat32m1_t temp6 = vfredsum_vs_f32m2_f32m1(temp6, vs6, zero, vl); - vfloat32m1_t temp7 = vfredsum_vs_f32m2_f32m1(temp7, vs7, zero, vl); - float32_t sum[8]; - sum[0] = vfmv_f_s_f32m1_f32(temp0); - sum[1] = vfmv_f_s_f32m1_f32(temp1); - sum[2] = vfmv_f_s_f32m1_f32(temp2); - sum[3] = vfmv_f_s_f32m1_f32(temp3); - sum[4] = vfmv_f_s_f32m1_f32(temp4); - sum[5] = vfmv_f_s_f32m1_f32(temp5); - sum[6] = vfmv_f_s_f32m1_f32(temp6); - sum[7] = vfmv_f_s_f32m1_f32(temp7); - vfloat32m2_t s0 = vfadd_vv_f32m2(vle32_v_f32m2(sum, vl), vle32_v_f32m2(bias + i, vl), vl); - vse32_v_f32m2(dst + i, s0, vl); + float32_t sum[15]; + vfloat32m1_t zero = vfmv_v_f_f32m1(0, vlm2); + sum[0] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m2_f32m1(zero, vs0, zero, vlm2)); + sum[1] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m2_f32m1(zero, vs1, zero, vlm2)); + sum[2] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m2_f32m1(zero, vs2, zero, vlm2)); + sum[3] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m2_f32m1(zero, vs3, zero, vlm2)); + sum[4] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m2_f32m1(zero, vs4, zero, vlm2)); + sum[5] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m2_f32m1(zero, vs5, zero, vlm2)); + sum[6] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m2_f32m1(zero, vs6, zero, vlm2)); + sum[7] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m2_f32m1(zero, vs7, zero, vlm2)); + sum[8] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m2_f32m1(zero, vs8, zero, vlm2)); + sum[9] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m2_f32m1(zero, vs9, zero, vlm2)); + sum[10] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m2_f32m1(zero, vs10, zero, vlm2)); + sum[11] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m2_f32m1(zero, vs11, zero, vlm2)); + sum[12] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m2_f32m1(zero, vs12, zero, vlm2)); + sum[13] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m2_f32m1(zero, vs13, zero, vlm2)); + sum[14] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m2_f32m1(zero, vs14, zero, vlm2)); + + vfloat32m4_t s0 = vfadd_vv_f32m4(vle32_v_f32m4(sum, 15), vle32_v_f32m4(bias + i, 15), 15); + vse32_v_f32m4(dst + i, s0, 15); } int mvl = nvecs - i; if (mvl > 0) { const float* wptr = weights + i*wstep; - vfloat32m2_t vs0 = vfmv_v_f_f32m2(0, vl), vs1 = vfmv_v_f_f32m2(0, vl), - vs2 = vfmv_v_f_f32m2(0, vl), vs3 = vfmv_v_f_f32m2(0, vl), - vs4 = vfmv_v_f_f32m2(0, vl), vs5 = vfmv_v_f_f32m2(0, vl), - vs6 = vfmv_v_f_f32m2(0, vl), vs7 = vfmv_v_f_f32m2(0, vl); + vfloat32m2_t + vs0 = vfmv_v_f_f32m2(0, vlm2), vs1 = vfmv_v_f_f32m2(0, vlm2), vs2 = vfmv_v_f_f32m2(0, vlm2), + vs3 = vfmv_v_f_f32m2(0, vlm2), vs4 = vfmv_v_f_f32m2(0, vlm2), vs5 = vfmv_v_f_f32m2(0, vlm2), + vs6 = vfmv_v_f_f32m2(0, vlm2), vs7 = vfmv_v_f_f32m2(0, vlm2), vs8 = vfmv_v_f_f32m2(0, vlm2), + vs9 = vfmv_v_f_f32m2(0, vlm2), vs10 = vfmv_v_f_f32m2(0, vlm2), vs11 = vfmv_v_f_f32m2(0, vlm2), + vs12 = vfmv_v_f_f32m2(0, vlm2), vs13 = vfmv_v_f_f32m2(0, vlm2); int k = 0; - for( ; k <= vecsize - 8; k += 8, wptr += 8 ) + for( ; k <= vecsize - vlm2; k += vlm2, wptr += vlm2 ) { - vfloat32m2_t v = vle32_v_f32m2(vec + k, vl); - vs0 = vfmacc_vv_f32m2(vs0, vle32_v_f32m2(wptr, vl), v, vl); - vs1 = vfmacc_vv_f32m2(vs1, vle32_v_f32m2(wptr + wstep*std::min(1, mvl-1), vl), v, vl); - vs2 = vfmacc_vv_f32m2(vs2, vle32_v_f32m2(wptr + wstep*std::min(2, mvl-1), vl), v, vl); - vs3 = vfmacc_vv_f32m2(vs3, vle32_v_f32m2(wptr + wstep*std::min(3, mvl-1), vl), v, vl); - vs4 = vfmacc_vv_f32m2(vs4, vle32_v_f32m2(wptr + wstep*std::min(4, mvl-1), vl), v, vl); - vs5 = vfmacc_vv_f32m2(vs5, vle32_v_f32m2(wptr + wstep*std::min(5, mvl-1), vl), v, vl); - vs6 = vfmacc_vv_f32m2(vs6, vle32_v_f32m2(wptr + wstep*std::min(6, mvl-1), vl), v, vl); + vfloat32m2_t v = vle32_v_f32m2(vec + k, vlm2); + vs0 = vfmacc_vv_f32m2(vs0, vle32_v_f32m2(wptr, vlm2), v, vlm2); + vs1 = vfmacc_vv_f32m2(vs1, vle32_v_f32m2(wptr + wstep*std::min(1, mvl-1), vlm2), v, vlm2); + vs2 = vfmacc_vv_f32m2(vs2, vle32_v_f32m2(wptr + wstep*std::min(2, mvl-1), vlm2), v, vlm2); + vs3 = vfmacc_vv_f32m2(vs3, vle32_v_f32m2(wptr + wstep*std::min(3, mvl-1), vlm2), v, vlm2); + vs4 = vfmacc_vv_f32m2(vs4, vle32_v_f32m2(wptr + wstep*std::min(4, mvl-1), vlm2), v, vlm2); + vs5 = vfmacc_vv_f32m2(vs5, vle32_v_f32m2(wptr + wstep*std::min(5, mvl-1), vlm2), v, vlm2); + vs6 = vfmacc_vv_f32m2(vs6, vle32_v_f32m2(wptr + wstep*std::min(6, mvl-1), vlm2), v, vlm2); + vs7 = vfmacc_vv_f32m2(vs7, vle32_v_f32m2(wptr + wstep*std::min(7, mvl-1), vlm2), v, vlm2); + vs8 = vfmacc_vv_f32m2(vs8, vle32_v_f32m2(wptr + wstep*std::min(8, mvl-1), vlm2), v, vlm2); + vs9 = vfmacc_vv_f32m2(vs9, vle32_v_f32m2(wptr + wstep*std::min(9, mvl-1), vlm2), v, vlm2); + vs10 = vfmacc_vv_f32m2(vs10, vle32_v_f32m2(wptr + wstep*std::min(10, mvl-1), vlm2), v, vlm2); + vs11 = vfmacc_vv_f32m2(vs11, vle32_v_f32m2(wptr + wstep*std::min(11, mvl-1), vlm2), v, vlm2); + vs12 = vfmacc_vv_f32m2(vs12, vle32_v_f32m2(wptr + wstep*std::min(12, mvl-1), vlm2), v, vlm2); + vs13 = vfmacc_vv_f32m2(vs13, vle32_v_f32m2(wptr + wstep*std::min(13, mvl-1), vlm2), v, vlm2); } int kvl = vecsize - k; if (kvl > 0) { @@ -880,54 +925,47 @@ void fastGEMM1T( const float* vec, const float* weights, vs4 = vfmacc_vv_f32m2(vs4, vle32_v_f32m2(wptr + wstep*std::min(4, mvl-1), kvl), v, kvl); vs5 = vfmacc_vv_f32m2(vs5, vle32_v_f32m2(wptr + wstep*std::min(5, mvl-1), kvl), v, kvl); vs6 = vfmacc_vv_f32m2(vs6, vle32_v_f32m2(wptr + wstep*std::min(6, mvl-1), kvl), v, kvl); + vs7 = vfmacc_vv_f32m2(vs7, vle32_v_f32m2(wptr + wstep*std::min(7, mvl-1), kvl), v, kvl); + vs8 = vfmacc_vv_f32m2(vs8, vle32_v_f32m2(wptr + wstep*std::min(8, mvl-1), kvl), v, kvl); + vs9 = vfmacc_vv_f32m2(vs9, vle32_v_f32m2(wptr + wstep*std::min(9, mvl-1), kvl), v, kvl); + vs10 = vfmacc_vv_f32m2(vs10, vle32_v_f32m2(wptr + wstep*std::min(10, mvl-1), kvl), v, kvl); + vs11 = vfmacc_vv_f32m2(vs11, vle32_v_f32m2(wptr + wstep*std::min(11, mvl-1), kvl), v, kvl); + vs12 = vfmacc_vv_f32m2(vs12, vle32_v_f32m2(wptr + wstep*std::min(12, mvl-1), kvl), v, kvl); + vs13 = vfmacc_vv_f32m2(vs13, vle32_v_f32m2(wptr + wstep*std::min(13, mvl-1), kvl), v, kvl); } // Calculate the sum of each vector - vfloat32m1_t zero = vfmv_v_f_f32m1(0, vl); - vfloat32m1_t temp0 = vfmv_v_f_f32m1(0, 4), temp1 = vfmv_v_f_f32m1(0, 4), - temp2 = vfmv_v_f_f32m1(0, 4), temp3 = vfmv_v_f_f32m1(0, 4), - temp4 = vfmv_v_f_f32m1(0, 4), temp5 = vfmv_v_f_f32m1(0, 4), - temp6 = vfmv_v_f_f32m1(0, 4), temp7 = vfmv_v_f_f32m1(0, 4); - temp0 = vfredsum_vs_f32m2_f32m1(temp0, vs0, zero, vl); - temp1 = vfredsum_vs_f32m2_f32m1(temp1, vs1, zero, vl); - temp2 = vfredsum_vs_f32m2_f32m1(temp2, vs2, zero, vl); - temp3 = vfredsum_vs_f32m2_f32m1(temp3, vs3, zero, vl); - temp4 = vfredsum_vs_f32m2_f32m1(temp4, vs4, zero, vl); - temp5 = vfredsum_vs_f32m2_f32m1(temp5, vs5, zero, vl); - temp6 = vfredsum_vs_f32m2_f32m1(temp6, vs6, zero, vl); - temp7 = vfredsum_vs_f32m2_f32m1(temp7, vs7, zero, vl); + float32_t sum[14]; + vfloat32m1_t zero = vfmv_v_f_f32m1(0, vlm2); + sum[0] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m2_f32m1(zero, vs0, zero, vlm2)); + sum[1] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m2_f32m1(zero, vs1, zero, vlm2)); + sum[2] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m2_f32m1(zero, vs2, zero, vlm2)); + sum[3] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m2_f32m1(zero, vs3, zero, vlm2)); + sum[4] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m2_f32m1(zero, vs4, zero, vlm2)); + sum[5] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m2_f32m1(zero, vs5, zero, vlm2)); + sum[6] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m2_f32m1(zero, vs6, zero, vlm2)); + sum[7] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m2_f32m1(zero, vs7, zero, vlm2)); + sum[8] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m2_f32m1(zero, vs8, zero, vlm2)); + sum[9] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m2_f32m1(zero, vs9, zero, vlm2)); + sum[10] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m2_f32m1(zero, vs10, zero, vlm2)); + sum[11] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m2_f32m1(zero, vs11, zero, vlm2)); + sum[12] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m2_f32m1(zero, vs12, zero, vlm2)); + sum[13] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m2_f32m1(zero, vs13, zero, vlm2)); - float32_t sum[8]; - sum[0] = vfmv_f_s_f32m1_f32(temp0); - sum[1] = vfmv_f_s_f32m1_f32(temp1); - sum[2] = vfmv_f_s_f32m1_f32(temp2); - sum[3] = vfmv_f_s_f32m1_f32(temp3); - sum[4] = vfmv_f_s_f32m1_f32(temp4); - sum[5] = vfmv_f_s_f32m1_f32(temp5); - sum[6] = vfmv_f_s_f32m1_f32(temp6); - sum[7] = vfmv_f_s_f32m1_f32(temp7); - - vfloat32m2_t s0 = vfadd_vv_f32m2(vle32_v_f32m2(sum, mvl), vle32_v_f32m2(bias + i, mvl), mvl); - vse32_v_f32m2(dst + i, s0, mvl); + vfloat32m4_t s0 = vfadd_vv_f32m4(vle32_v_f32m4(sum, mvl), vle32_v_f32m4(bias + i, mvl), mvl); + vse32_v_f32m4(dst + i, s0, mvl); } } -enum { FASCONV_BASE_VECSZ = 4 }; // TODO: Large base size. +enum { FASCONV_BASE_VECSZ = 8 }; void fastConv( const float* weights, size_t wstep, const float* bias, const float* rowbuf, float* output, const int* outShape, int blockSize, int vecsize, int vecsize_aligned, const float* relu, bool initOutput ) { - int vl = 4; + int vl = FASCONV_BASE_VECSZ; + int vlm1Max = vsetvlmax_e32m1(); int outCn = outShape[1]; size_t outPlaneSize = outShape[2]*outShape[3]; - float r0 = 1.f, r1 = 1.f, r2 = 1.f; - vfloat32m1_t vr0 = vfmv_v_f_f32m1(1, vl), vr1 = vfmv_v_f_f32m1(1, vl), vr2 = vfmv_v_f_f32m1(1, vl); - int maskbuf[FASCONV_BASE_VECSZ] = {0}; - int rsz = blockSize % FASCONV_BASE_VECSZ; - for( int i = 0; i < rsz; i++ ) - maskbuf[FASCONV_BASE_VECSZ - i - 1] = -1; - vint32m1_t vmaskbuf = vle32_v_i32m1(maskbuf ,vl); - vbool32_t mask = vmslt_vx_i32m1_b32(vmaskbuf, 0, vl); // mask for tail // now compute dot product of the weights // and im2row-transformed part of the tensor for( int i = 0; i < outCn; i += 3 ) @@ -953,20 +991,6 @@ void fastConv( const float* weights, size_t wstep, const float* bias, } } - if( relu ) - { - r0 = relu[i]; r1 = relu[i+1]; r2 = relu[i+2]; - if( i+2 >= outCn ) - { - r2 = r1; - if( i+1 >= outCn ) - r2 = r1 = r0; - } - vr0 = vfmv_v_f_f32m1(r0, vl); - vr1 = vfmv_v_f_f32m1(r1, vl); - vr2 = vfmv_v_f_f32m1(r2, vl); - } - int j = 0; for( ; j < blockSize; j += FASCONV_BASE_VECSZ ) { @@ -983,110 +1007,152 @@ void fastConv( const float* weights, size_t wstep, const float* bias, } int k = 0; const float* rptr = rowbuf + j*vecsize_aligned; - int vlm2 = 8; - vfloat32m2_t vs00 = vfmv_v_f_f32m2(0, vlm2), vs01 = vfmv_v_f_f32m2(0, vlm2), - vs02 = vfmv_v_f_f32m2(0, vlm2), vs03 = vfmv_v_f_f32m2(0, vlm2), - vs10 = vfmv_v_f_f32m2(0, vlm2), vs11 = vfmv_v_f_f32m2(0, vlm2), - vs12 = vfmv_v_f_f32m2(0, vlm2), vs13 = vfmv_v_f_f32m2(0, vlm2), - vs20 = vfmv_v_f_f32m2(0, vlm2), vs21 = vfmv_v_f_f32m2(0, vlm2), - vs22 = vfmv_v_f_f32m2(0, vlm2), vs23 = vfmv_v_f_f32m2(0, vlm2); + int vlm1 = vsetvlmax_e32m1(); + vfloat32m1_t + vs00 = vfmv_v_f_f32m1(0, vlm1), vs10 = vfmv_v_f_f32m1(0, vlm1), vs20 = vfmv_v_f_f32m1(0, vlm1), + vs01 = vfmv_v_f_f32m1(0, vlm1), vs11 = vfmv_v_f_f32m1(0, vlm1), vs21 = vfmv_v_f_f32m1(0, vlm1), + vs02 = vfmv_v_f_f32m1(0, vlm1), vs12 = vfmv_v_f_f32m1(0, vlm1), vs22 = vfmv_v_f_f32m1(0, vlm1), + vs03 = vfmv_v_f_f32m1(0, vlm1), vs13 = vfmv_v_f_f32m1(0, vlm1), vs23 = vfmv_v_f_f32m1(0, vlm1), + vs04 = vfmv_v_f_f32m1(0, vlm1), vs14 = vfmv_v_f_f32m1(0, vlm1), vs24 = vfmv_v_f_f32m1(0, vlm1), + vs05 = vfmv_v_f_f32m1(0, vlm1), vs15 = vfmv_v_f_f32m1(0, vlm1), vs25 = vfmv_v_f_f32m1(0, vlm1), + vs06 = vfmv_v_f_f32m1(0, vlm1), vs16 = vfmv_v_f_f32m1(0, vlm1), vs26 = vfmv_v_f_f32m1(0, vlm1), + vs07 = vfmv_v_f_f32m1(0, vlm1), vs17 = vfmv_v_f_f32m1(0, vlm1), vs27 = vfmv_v_f_f32m1(0, vlm1); - for (; k < vecsize; k += 8, rptr += 8 ) + for (; k < vecsize; k += vlm1, rptr += vlm1 ) { - if (k+8 >= vecsize) { - vlm2 = vecsize - k; + if (k + vlm1 >= vecsize) { + vlm1 = vecsize - k; } - vfloat32m2_t w0 = vle32_v_f32m2(wptr0 + k, vlm2); - vfloat32m2_t w1 = vle32_v_f32m2(wptr1 + k, vlm2); - vfloat32m2_t w2 = vle32_v_f32m2(wptr2 + k, vlm2); - vfloat32m2_t r0 = vle32_v_f32m2(rptr, vlm2); + vfloat32m1_t w0 = vle32_v_f32m1(wptr0 + k, vlm1); + vfloat32m1_t w1 = vle32_v_f32m1(wptr1 + k, vlm1); + vfloat32m1_t w2 = vle32_v_f32m1(wptr2 + k, vlm1); + vfloat32m1_t r0 = vle32_v_f32m1(rptr, vlm1); - vs00 = vfmacc_vv_f32m2(vs00, w0, r0, vlm2); - vs10 = vfmacc_vv_f32m2(vs10, w1, r0, vlm2); - vs20 = vfmacc_vv_f32m2(vs20, w2, r0, vlm2); + vs00 = vfmacc_vv_f32m1(vs00, w0, r0, vlm1); + vs10 = vfmacc_vv_f32m1(vs10, w1, r0, vlm1); + vs20 = vfmacc_vv_f32m1(vs20, w2, r0, vlm1); - r0 = vle32_v_f32m2(rptr + vecsize_aligned, vlm2); - vs01 = vfmacc_vv_f32m2(vs01, w0, r0, vlm2); - vs11 = vfmacc_vv_f32m2(vs11, w1, r0, vlm2); - vs21 = vfmacc_vv_f32m2(vs21, w2, r0, vlm2); + r0 = vle32_v_f32m1(rptr + vecsize_aligned, vlm1); + vs01 = vfmacc_vv_f32m1(vs01, w0, r0, vlm1); + vs11 = vfmacc_vv_f32m1(vs11, w1, r0, vlm1); + vs21 = vfmacc_vv_f32m1(vs21, w2, r0, vlm1); - r0 = vle32_v_f32m2(rptr + vecsize_aligned*2, vlm2); - vs02 = vfmacc_vv_f32m2(vs02, w0, r0, vlm2); - vs12 = vfmacc_vv_f32m2(vs12, w1, r0, vlm2); - vs22 = vfmacc_vv_f32m2(vs22, w2, r0, vlm2); + r0 = vle32_v_f32m1(rptr + vecsize_aligned*2, vlm1); + vs02 = vfmacc_vv_f32m1(vs02, w0, r0, vlm1); + vs12 = vfmacc_vv_f32m1(vs12, w1, r0, vlm1); + vs22 = vfmacc_vv_f32m1(vs22, w2, r0, vlm1); - r0 = vle32_v_f32m2(rptr + vecsize_aligned*3, vlm2); - vs03 = vfmacc_vv_f32m2(vs03, w0, r0, vlm2); - vs13 = vfmacc_vv_f32m2(vs13, w1, r0, vlm2); - vs23 = vfmacc_vv_f32m2(vs23, w2, r0, vlm2); + r0 = vle32_v_f32m1(rptr + vecsize_aligned*3, vlm1); + vs03 = vfmacc_vv_f32m1(vs03, w0, r0, vlm1); + vs13 = vfmacc_vv_f32m1(vs13, w1, r0, vlm1); + vs23 = vfmacc_vv_f32m1(vs23, w2, r0, vlm1); + + r0 = vle32_v_f32m1(rptr + vecsize_aligned*4, vlm1); + vs04 = vfmacc_vv_f32m1(vs04, w0, r0, vlm1); + vs14 = vfmacc_vv_f32m1(vs14, w1, r0, vlm1); + vs24 = vfmacc_vv_f32m1(vs24, w2, r0, vlm1); + + r0 = vle32_v_f32m1(rptr + vecsize_aligned*5, vlm1); + vs05 = vfmacc_vv_f32m1(vs05, w0, r0, vlm1); + vs15 = vfmacc_vv_f32m1(vs15, w1, r0, vlm1); + vs25 = vfmacc_vv_f32m1(vs25, w2, r0, vlm1); + + r0 = vle32_v_f32m1(rptr + vecsize_aligned*6, vlm1); + vs06 = vfmacc_vv_f32m1(vs06, w0, r0, vlm1); + vs16 = vfmacc_vv_f32m1(vs16, w1, r0, vlm1); + vs26 = vfmacc_vv_f32m1(vs26, w2, r0, vlm1); + + r0 = vle32_v_f32m1(rptr + vecsize_aligned*7, vlm1); + vs07 = vfmacc_vv_f32m1(vs07, w0, r0, vlm1); + vs17 = vfmacc_vv_f32m1(vs17, w1, r0, vlm1); + vs27 = vfmacc_vv_f32m1(vs27, w2, r0, vlm1); } - vfloat32m1_t s0, s1, s2; + // compute sum of each vs + vfloat32m1_t zero = vfmv_v_f_f32m1(0, vlm1Max); + // vl is required here to be at least FASCONV_BASE_VECSZ, aka 8. + float32_t sum0[FASCONV_BASE_VECSZ], sum1[FASCONV_BASE_VECSZ], sum2[FASCONV_BASE_VECSZ]; + sum0[0] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m1_f32m1(zero, vs00, zero, vlm1Max)); + sum0[1] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m1_f32m1(zero, vs01, zero, vlm1Max)); + sum0[2] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m1_f32m1(zero, vs02, zero, vlm1Max)); + sum0[3] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m1_f32m1(zero, vs03, zero, vlm1Max)); + sum0[4] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m1_f32m1(zero, vs04, zero, vlm1Max)); + sum0[5] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m1_f32m1(zero, vs05, zero, vlm1Max)); + sum0[6] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m1_f32m1(zero, vs06, zero, vlm1Max)); + sum0[7] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m1_f32m1(zero, vs07, zero, vlm1Max)); + sum1[0] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m1_f32m1(zero, vs10, zero, vlm1Max)); + sum1[1] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m1_f32m1(zero, vs11, zero, vlm1Max)); + sum1[2] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m1_f32m1(zero, vs12, zero, vlm1Max)); + sum1[3] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m1_f32m1(zero, vs13, zero, vlm1Max)); + sum1[4] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m1_f32m1(zero, vs14, zero, vlm1Max)); + sum1[5] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m1_f32m1(zero, vs15, zero, vlm1Max)); + sum1[6] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m1_f32m1(zero, vs16, zero, vlm1Max)); + sum1[7] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m1_f32m1(zero, vs17, zero, vlm1Max)); + sum2[0] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m1_f32m1(zero, vs20, zero, vlm1Max)); + sum2[1] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m1_f32m1(zero, vs21, zero, vlm1Max)); + sum2[2] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m1_f32m1(zero, vs22, zero, vlm1Max)); + sum2[3] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m1_f32m1(zero, vs23, zero, vlm1Max)); + sum2[4] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m1_f32m1(zero, vs24, zero, vlm1Max)); + sum2[5] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m1_f32m1(zero, vs25, zero, vlm1Max)); + sum2[6] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m1_f32m1(zero, vs26, zero, vlm1Max)); + sum2[7] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m1_f32m1(zero, vs27, zero, vlm1Max)); + + // if VLEN = 128, so LMUL = 2 for vl = 8. + // otherwise, VLEN >=256, we only use fist 8 element of the vReg. + vfloat32m2_t s0, s1, s2; if( initOutput ) { - s0 = vfmv_v_f_f32m1(bias0, vl); - s1 = vfmv_v_f_f32m1(bias1, vl); - s2 = vfmv_v_f_f32m1(bias2, vl); + s0 = vfmv_v_f_f32m2(bias0, vl); + s1 = vfmv_v_f_f32m2(bias1, vl); + s2 = vfmv_v_f_f32m2(bias2, vl); } else { - s0 = vle32_v_f32m1(outptr0 + j, vl); - s1 = vle32_v_f32m1(outptr1 + j, vl); - s2 = vle32_v_f32m1(outptr2 + j, vl); + s0 = vle32_v_f32m2(outptr0 + j, vl); + s1 = vle32_v_f32m2(outptr1 + j, vl); + s2 = vle32_v_f32m2(outptr2 + j, vl); } - // compute sum of each vs - vfloat32m1_t zero = vfmv_v_f_f32m1(0, vl); - vfloat32m1_t temp00 = vfredsum_vs_f32m2_f32m1(temp00, vs00, zero, 8); - vfloat32m1_t temp01 = vfredsum_vs_f32m2_f32m1(temp01, vs01, zero, 8); - vfloat32m1_t temp02 = vfredsum_vs_f32m2_f32m1(temp02, vs02, zero, 8); - vfloat32m1_t temp03 = vfredsum_vs_f32m2_f32m1(temp03, vs03, zero, 8); - vfloat32m1_t temp10 = vfredsum_vs_f32m2_f32m1(temp10, vs10, zero, 8); - vfloat32m1_t temp11 = vfredsum_vs_f32m2_f32m1(temp11, vs11, zero, 8); - vfloat32m1_t temp12 = vfredsum_vs_f32m2_f32m1(temp12, vs12, zero, 8); - vfloat32m1_t temp13 = vfredsum_vs_f32m2_f32m1(temp13, vs13, zero, 8); - vfloat32m1_t temp20 = vfredsum_vs_f32m2_f32m1(temp20, vs20, zero, 8); - vfloat32m1_t temp21 = vfredsum_vs_f32m2_f32m1(temp21, vs21, zero, 8); - vfloat32m1_t temp22 = vfredsum_vs_f32m2_f32m1(temp22, vs22, zero, 8); - vfloat32m1_t temp23 = vfredsum_vs_f32m2_f32m1(temp23, vs23, zero, 8); - float32_t sum0[4], sum1[4], sum2[4]; - sum0[0] = vfmv_f_s_f32m1_f32(temp00); - sum0[1] = vfmv_f_s_f32m1_f32(temp01); - sum0[2] = vfmv_f_s_f32m1_f32(temp02); - sum0[3] = vfmv_f_s_f32m1_f32(temp03); - sum1[0] = vfmv_f_s_f32m1_f32(temp10); - sum1[1] = vfmv_f_s_f32m1_f32(temp11); - sum1[2] = vfmv_f_s_f32m1_f32(temp12); - sum1[3] = vfmv_f_s_f32m1_f32(temp13); - sum2[0] = vfmv_f_s_f32m1_f32(temp20); - sum2[1] = vfmv_f_s_f32m1_f32(temp21); - sum2[2] = vfmv_f_s_f32m1_f32(temp22); - sum2[3] = vfmv_f_s_f32m1_f32(temp23); - - s0 = vfadd_vv_f32m1(vle32_v_f32m1(sum0, vl), s0, vl); - s1 = vfadd_vv_f32m1(vle32_v_f32m1(sum1, vl), s1, vl); - s2 = vfadd_vv_f32m1(vle32_v_f32m1(sum2, vl), s2, vl); - + s0 = vfadd_vv_f32m2(vle32_v_f32m2(sum0, vl), s0, vl); + s1 = vfadd_vv_f32m2(vle32_v_f32m2(sum1, vl), s1, vl); + s2 = vfadd_vv_f32m2(vle32_v_f32m2(sum2, vl), s2, vl); if( relu ) { - vbool32_t m0 = vmfgt_vf_f32m1_b32(s0, 0, vl); - vbool32_t m1 = vmfgt_vf_f32m1_b32(s1, 0, vl); - vbool32_t m2 = vmfgt_vf_f32m1_b32(s2, 0, vl); - s0 = vmerge_vvm_f32m1(m0, vfmul_vv_f32m1(s0, vr0, vl), s0, vl); - s1 = vmerge_vvm_f32m1(m1, vfmul_vv_f32m1(s1, vr1, vl), s1, vl); - s2 = vmerge_vvm_f32m1(m2, vfmul_vv_f32m1(s2, vr2, vl), s2, vl); + vfloat32m2_t vr0 = vfmv_v_f_f32m2(1, vl), vr1 = vfmv_v_f_f32m2(1, vl), vr2 = vfmv_v_f_f32m2(1, vl); + float r0 = relu[i], r1 = relu[i+1], r2 = relu[i+2]; + if( i+2 >= outCn ) + { + r2 = r1; + if( i+1 >= outCn ) + r2 = r1 = r0; + } + vr0 = vfmv_v_f_f32m2(r0, vl); + vr1 = vfmv_v_f_f32m2(r1, vl); + vr2 = vfmv_v_f_f32m2(r2, vl); + vbool16_t m0 = vmfgt_vf_f32m2_b16(s0, 0, vl); + vbool16_t m1 = vmfgt_vf_f32m2_b16(s1, 0, vl); + vbool16_t m2 = vmfgt_vf_f32m2_b16(s2, 0, vl); + s0 = vmerge_vvm_f32m2(m0, vfmul_vv_f32m2(s0, vr0, vl), s0, vl); + s1 = vmerge_vvm_f32m2(m1, vfmul_vv_f32m2(s1, vr1, vl), s1, vl); + s2 = vmerge_vvm_f32m2(m2, vfmul_vv_f32m2(s2, vr2, vl), s2, vl); } if( tail ) { - s0 = vmerge_vvm_f32m1(mask, vle32_v_f32m1(outptr0 + j, vl), s0, vl); - s1 = vmerge_vvm_f32m1(mask, vle32_v_f32m1(outptr1 + j, vl), s1, vl); - s2 = vmerge_vvm_f32m1(mask, vle32_v_f32m1(outptr2 + j, vl), s2, vl); + int maskbuf[FASCONV_BASE_VECSZ] = {0}; + int rsz = blockSize % FASCONV_BASE_VECSZ; + for( int i = 0; i < rsz; i++ ) + maskbuf[FASCONV_BASE_VECSZ - i - 1] = -1; + vint32m2_t vmaskbuf = vle32_v_i32m2(maskbuf ,vl); + vbool16_t mask = vmslt_vx_i32m2_b16(vmaskbuf, 0, vl); // mask for tail + s0 = vmerge_vvm_f32m2(mask, vle32_v_f32m2(outptr0 + j, vl), s0, vl); + s1 = vmerge_vvm_f32m2(mask, vle32_v_f32m2(outptr1 + j, vl), s1, vl); + s2 = vmerge_vvm_f32m2(mask, vle32_v_f32m2(outptr2 + j, vl), s2, vl); } - vse32_v_f32m1(outptr0 + j, s0, vl); - vse32_v_f32m1(outptr1 + j, s1, vl); - vse32_v_f32m1(outptr2 + j, s2, vl); + vse32_v_f32m2(outptr0 + j, s0, vl); + vse32_v_f32m2(outptr1 + j, s1, vl); + vse32_v_f32m2(outptr2 + j, s2, vl); } } } @@ -1097,23 +1163,27 @@ Example for load_deinterleave: output: a = {1, 3, 5, 7, 9, 11, 13, 15} output: b = {2, 4, 6, 8,10, 12, 14, 16} */ -static inline void vfloat32m2_load_deinterleave(const float* ptr, vfloat32m2_t& a, vfloat32m2_t& b) +static inline void vfloat32m2_load_deinterleave(const float* ptr, vfloat32m2_t& a, vfloat32m2_t& b, int vl) { - int vl = 8; - uint32_t masks[] = {1,1,1,1,0,0,0,0}; - vuint32m2_t vm = vle32_v_u32m2(masks,vl); - vbool16_t mask01 = vmseq_vx_u32m2_b16 (vm, 0, vl); - vbool16_t mask10 = vmseq_vx_u32m2_b16 (vm, 1, vl); - vfloat32m2_t ta = vle32_v_f32m2(ptr, vl), tb = vle32_v_f32m2(ptr+8, vl); - uint idx[] = {0,2,4,6,1,3,5,7}; - uint idxa[] = {0,0,0,0,0,1,2,3}, idxb[] = {4,5,6,7,0,0,0,0}; - vuint32m2_t vidxa = vle32_v_u32m2(idxa, 8), vidxb = vle32_v_u32m2(idxb, 8); - vuint32m2_t vidx = vle32_v_u32m2(idx, 8); - vfloat32m2_t high = vfmv_v_f_f32m2(0, 8), low = vfmv_v_f_f32m2(0, 8); - high = vrgather_vv_f32m2(ta, vidx, 8); - low = vrgather_vv_f32m2(tb, vidx, 8); - a = vrgather_vv_f32m2_m(mask01, high, low, vidxa, 8); - b = vrgather_vv_f32m2_m(mask10, low, high, vidxb, 8); + vuint64m4_t mask = vmv_v_x_u64m4(1,vl*2); + vuint32m4_t mask_re = vreinterpret_v_u64m4_u32m4(mask); + vbool8_t mask0 = vmseq_vx_u32m4_b8 (mask_re, 1, vl*2); + vbool8_t mask1 = vmseq_vx_u32m4_b8 (mask_re, 0, vl*2); + vfloat32m4_t tempa = vundefined_f32m4(), tempb = vundefined_f32m4(); + vfloat32m4_t vw = vle32_v_f32m4(ptr, vl*2); + tempa = vcompress_vm_f32m4(mask0, tempa, vw, vl*2); + tempb = vcompress_vm_f32m4(mask1, tempb, vw, vl*2); + /* The following instructions have not to be supported by the GNU toolchain. + So we temporarily use store and load instead. + // a = vlmul_trunc_v_f32m4_f32m2(tempa); + // b = vlmul_trunc_v_f32m4_f32m2(tempb); + */ + cv::AutoBuffer cvBuffer(sizeof(float32_t)*vl*2); + float* buffer = (float*)cvBuffer.data(); + vse32_v_f32m4(buffer, tempa, vl); + a = vle32_v_f32m2(buffer, vl); + vse32_v_f32m4(buffer, tempb, vl); + b = vle32_v_f32m2(buffer, vl); } void fastDepthwiseConv( const float* wptr, @@ -1127,7 +1197,7 @@ void fastDepthwiseConv( const float* wptr, float* outptr_, int out_d, int outH, int outW ) { - int vl = 8; + int vl = vsetvlmax_e32m2(); const float w00_ = wptr[0], w01_ = wptr[1], w02_ = wptr[2], w10 = wptr[3], w11 = wptr[4], w12 = wptr[5], w20_ = wptr[6], w21_ = wptr[7], w22_ = wptr[8]; @@ -1166,17 +1236,11 @@ void fastDepthwiseConv( const float* wptr, if (stride_w == 1 || (stride_w == 2 && dilation_w == 1)) { - const int VECSZ = 8; - vfloat32m2_t vw00 = vfmv_v_f_f32m2(w00, vl), vw01 = vfmv_v_f_f32m2(w01, vl), vw02 = vfmv_v_f_f32m2(w02, vl), - vw10 = vfmv_v_f_f32m2(w10, vl), vw11 = vfmv_v_f_f32m2(w11, vl), vw12 = vfmv_v_f_f32m2(w12, vl), - vw20 = vfmv_v_f_f32m2(w20, vl), vw21 = vfmv_v_f_f32m2(w21, vl), vw22 = vfmv_v_f_f32m2(w22, vl); - vfloat32m2_t vbias = vfmv_v_f_f32m2(bias, vl), vrc = vfmv_v_f_f32m2(relu_coeff, vl); - if( stride_w == 1 ) - for( ; out_j < outW1; out_j += VECSZ ) + for( ; out_j < outW1; out_j += vl ) { - if (out_j + VECSZ > outW1 && out_j > pad_l) - out_j = outW1 - VECSZ; + if (out_j + vl > outW1) + vl = outW1 - out_j; int in_j = out_j * stride_w - pad_l; vfloat32m2_t v00 = vle32_v_f32m2(imgptr0 + in_j, vl), v01 = vle32_v_f32m2(imgptr0 + in_j + dilation_w, vl), @@ -1188,57 +1252,59 @@ void fastDepthwiseConv( const float* wptr, v21 = vle32_v_f32m2(imgptr2 + in_j + dilation_w, vl), v22 = vle32_v_f32m2(imgptr2 + in_j + dilation_w*2, vl); - vfloat32m2_t vout0 = vfmacc_vv_f32m2(vbias, v00, vw00, vl); - vfloat32m2_t vout1 = vfmul_vv_f32m2(v01, vw01, vl); - vfloat32m2_t vout2 = vfmul_vv_f32m2(v02, vw02, vl); + vfloat32m2_t vout0 = vfmul_vf_f32m2(v00, w00, vl); + vfloat32m2_t vout1 = vfmul_vf_f32m2(v01, w01, vl); + vfloat32m2_t vout2 = vfmul_vf_f32m2(v02, w02, vl); + vout0 = vfadd_vf_f32m2(vout0, bias, vl); - vout0 = vfmacc_vv_f32m2(vout0, v10, vw10, vl); - vout1 = vfmacc_vv_f32m2(vout1, v11, vw11, vl); - vout2 = vfmacc_vv_f32m2(vout2, v12, vw12, vl); + vout0 = vfmacc_vf_f32m2(vout0, w10, v10, vl); + vout1 = vfmacc_vf_f32m2(vout1, w11, v11, vl); + vout2 = vfmacc_vf_f32m2(vout2, w12, v12, vl); - vout0 = vfmacc_vv_f32m2(vout0, v20, vw20, vl); - vout1 = vfmacc_vv_f32m2(vout1, v21, vw21, vl); - vout2 = vfmacc_vv_f32m2(vout2, v22, vw22, vl); + vout0 = vfmacc_vf_f32m2(vout0, w20, v20, vl); + vout1 = vfmacc_vf_f32m2(vout1, w21, v21, vl); + vout2 = vfmacc_vf_f32m2(vout2, w22, v22, vl); vout0 = vfadd_vv_f32m2(vfadd_vv_f32m2(vout0, vout1, vl), vout2, vl); if (relu) { vbool16_t m = vmfgt_vf_f32m2_b16(vout0, 0, vl); - vout0 = vmerge_vvm_f32m2(m, vfmul_vv_f32m2(vout0, vrc, vl), vout0, vl); + vout0 = vmerge_vvm_f32m2(m, vfmul_vf_f32m2(vout0, relu_coeff, vl), vout0, vl); } vse32_v_f32m2(outptr + out_j, vout0, vl); } - else - for( ; out_j < outW1; out_j += VECSZ ) + else //stride_w == 2 && dilation_w == 1 + for( ; out_j < outW1; out_j += vl ) { - if (out_j + VECSZ > outW1 && out_j > pad_l) - out_j = outW1 - VECSZ; + if (out_j + vl > outW1) + vl = outW1 - out_j; int in_j = out_j * stride_w - pad_l; vfloat32m2_t v00, v01, v02, v10, v11, v12, v20, v21, v22, unused; - vfloat32m2_load_deinterleave(imgptr0 + in_j, v00, v01); - vfloat32m2_load_deinterleave(imgptr0 + in_j + 2, v02, unused); - vfloat32m2_load_deinterleave(imgptr1 + in_j, v10, v11); - vfloat32m2_load_deinterleave(imgptr1 + in_j + 2, v12, unused); - vfloat32m2_load_deinterleave(imgptr2 + in_j, v20, v21); - vfloat32m2_load_deinterleave(imgptr2 + in_j + 2, v22, unused); + vfloat32m2_load_deinterleave(imgptr0 + in_j, v00, v01, vl); + vfloat32m2_load_deinterleave(imgptr0 + in_j + 2, v02, unused, vl); + vfloat32m2_load_deinterleave(imgptr1 + in_j, v10, v11, vl); + vfloat32m2_load_deinterleave(imgptr1 + in_j + 2, v12, unused, vl); + vfloat32m2_load_deinterleave(imgptr2 + in_j, v20, v21, vl); + vfloat32m2_load_deinterleave(imgptr2 + in_j + 2, v22, unused, vl); - vfloat32m2_t vout0 = vfmacc_vv_f32m2(vbias, v00, vw00, vl); - vfloat32m2_t vout1 = vfmul_vv_f32m2(v01, vw01, vl); - vfloat32m2_t vout2 = vfmul_vv_f32m2(v02, vw02, vl); + vfloat32m2_t vout0 = vfmul_vf_f32m2(v00, w00, vl); + vfloat32m2_t vout1 = vfmul_vf_f32m2(v01, w01, vl); + vfloat32m2_t vout2 = vfmul_vf_f32m2(v02, w02, vl); + vout0 = vfadd_vf_f32m2(vout0, bias, vl); - vout0 = vfmacc_vv_f32m2(vout0, v10, vw10, vl); - vout1 = vfmacc_vv_f32m2(vout1, v11, vw11, vl); - vout2 = vfmacc_vv_f32m2(vout2, v12, vw12, vl); + vout0 = vfmacc_vf_f32m2(vout0, w10, v10, vl); + vout1 = vfmacc_vf_f32m2(vout1, w11, v11, vl); + vout2 = vfmacc_vf_f32m2(vout2, w12, v12, vl); - vout0 = vfmacc_vv_f32m2(vout0, v20, vw20, vl); - vout1 = vfmacc_vv_f32m2(vout1, v21, vw21, vl); - vout2 = vfmacc_vv_f32m2(vout2, v22, vw22, vl); + vout0 = vfmacc_vf_f32m2(vout0, w20, v20, vl); + vout1 = vfmacc_vf_f32m2(vout1, w21, v21, vl); + vout2 = vfmacc_vf_f32m2(vout2, w22, v22, vl); vout0 = vfadd_vv_f32m2(vfadd_vv_f32m2(vout0, vout1, vl), vout2, vl); if (relu) { vbool16_t m = vmfgt_vf_f32m2_b16(vout0, 0, vl); - vout0 = vmerge_vvm_f32m2(m, vfmul_vv_f32m2(vout0, vrc, vl), vout0, vl); + vout0 = vmerge_vvm_f32m2(m, vfmul_vf_f32m2(vout0, relu_coeff, vl), vout0, vl); } vse32_v_f32m2(outptr + out_j, vout0, vl); } From 646924fce8422647b42bfaa5e7b2a83cda7ecf6d Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Tue, 5 Oct 2021 23:21:27 +0000 Subject: [PATCH 229/262] dnn(pytest/test_input_3d): reload model between switching targets --- modules/dnn/misc/python/test/test_dnn.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/modules/dnn/misc/python/test/test_dnn.py b/modules/dnn/misc/python/test/test_dnn.py index f7bfc01119..9ea93d3fe1 100644 --- a/modules/dnn/misc/python/test/test_dnn.py +++ b/modules/dnn/misc/python/test/test_dnn.py @@ -323,20 +323,22 @@ class dnn_test(NewOpenCVTests): raise unittest.SkipTest("Missing DNN test files (dnn/onnx/data/{input/output}_hidden_lstm.npy). " "Verify OPENCV_DNN_TEST_DATA_PATH configuration parameter.") - net = cv.dnn.readNet(model) input = np.load(input_file) # we have to expand the shape of input tensor because Python bindings cut 3D tensors to 2D # it should be fixed in future. see : https://github.com/opencv/opencv/issues/19091 # please remove `expand_dims` after that input = np.expand_dims(input, axis=3) gold_output = np.load(output_file) - net.setInput(input) for backend, target in self.dnnBackendsAndTargets: printParams(backend, target) + net = cv.dnn.readNet(model) + net.setPreferableBackend(backend) net.setPreferableTarget(target) + + net.setInput(input) real_output = net.forward() normAssert(self, real_output, gold_output, "", getDefaultThreshold(target)) From 755e0143fb8c40fee082b8343a2b3ce383c47a0b Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Wed, 6 Oct 2021 15:05:45 +0300 Subject: [PATCH 230/262] Merge pull request #20815 from alalek:fix_20649_revert_19859 features2d: repair SimpleBlobDetector * features2d: revert code change by PR #19859 Reverted commit 76860933f0a31c0abd1b26c1f11b25972cda031e * features2d: check SimpleBlobDetector parameters consistency --- modules/features2d/src/blobdetector.cpp | 23 +++++++++++++------ modules/features2d/test/test_blobdetector.cpp | 1 + 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/modules/features2d/src/blobdetector.cpp b/modules/features2d/src/blobdetector.cpp index c2215cd57c..fb79831807 100644 --- a/modules/features2d/src/blobdetector.cpp +++ b/modules/features2d/src/blobdetector.cpp @@ -44,6 +44,8 @@ #include #include +#include + // Requires CMake flag: DEBUG_opencv_features2d=ON //#define DEBUG_BLOB_DETECTOR @@ -317,6 +319,19 @@ void SimpleBlobDetectorImpl::detect(InputArray image, std::vector& CV_Error(Error::StsUnsupportedFormat, "Blob detector only supports 8-bit images!"); } + CV_CheckGT(params.thresholdStep, 0.0f, ""); + if (params.minThreshold + params.thresholdStep >= params.maxThreshold) + { + // https://github.com/opencv/opencv/issues/6667 + CV_LOG_ONCE_INFO(NULL, "SimpleBlobDetector: params.minDistBetweenBlobs is ignored for case with single threshold"); +#if 0 // OpenCV 5.0 + CV_CheckEQ(params.minRepeatability, 1u, "Incompatible parameters for case with single threshold"); +#else + if (params.minRepeatability != 1) + CV_LOG_WARNING(NULL, "SimpleBlobDetector: params.minRepeatability=" << params.minRepeatability << " is incompatible for case with single threshold. Empty result is expected."); +#endif + } + std::vector < std::vector

> centers; for (double thresh = params.minThreshold; thresh < params.maxThreshold; thresh += params.thresholdStep) { @@ -325,19 +340,13 @@ void SimpleBlobDetectorImpl::detect(InputArray image, std::vector& std::vector < Center > curCenters; findBlobs(grayscaleImage, binarizedImage, curCenters); - if(params.maxThreshold - params.minThreshold <= params.thresholdStep) { - // if the difference between min and max threshold is less than the threshold step - // we're only going to enter the loop once, so we need to add curCenters - // to ensure we still use minDistBetweenBlobs - centers.push_back(curCenters); - } std::vector < std::vector
> newCenters; for (size_t i = 0; i < curCenters.size(); i++) { bool isNew = true; for (size_t j = 0; j < centers.size(); j++) { - double dist = norm(centers[j][centers[j].size() / 2 ].location - curCenters[i].location); + double dist = norm(centers[j][ centers[j].size() / 2 ].location - curCenters[i].location); isNew = dist >= params.minDistBetweenBlobs && dist >= centers[j][ centers[j].size() / 2 ].radius && dist >= curCenters[i].radius; if (!isNew) { diff --git a/modules/features2d/test/test_blobdetector.cpp b/modules/features2d/test/test_blobdetector.cpp index 56b7145862..c6ed09e6a2 100644 --- a/modules/features2d/test/test_blobdetector.cpp +++ b/modules/features2d/test/test_blobdetector.cpp @@ -12,6 +12,7 @@ TEST(Features2d_BlobDetector, bug_6667) SimpleBlobDetector::Params params; params.minThreshold = 250; params.maxThreshold = 260; + params.minRepeatability = 1; // https://github.com/opencv/opencv/issues/6667 std::vector keypoints; Ptr detector = SimpleBlobDetector::create(params); From eb56ca3b0d7dec26603f45b9d9e63fc186e27eaa Mon Sep 17 00:00:00 2001 From: Giles Payne Date: Wed, 6 Oct 2021 21:06:30 +0900 Subject: [PATCH 231/262] Fix build on older Xcode versions --- modules/imgcodecs/misc/objc/ios/Mat+QuickLook.mm | 4 ++++ modules/imgcodecs/misc/objc/macosx/Mat+QuickLook.mm | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/modules/imgcodecs/misc/objc/ios/Mat+QuickLook.mm b/modules/imgcodecs/misc/objc/ios/Mat+QuickLook.mm index 7bfee07eb1..84cf25879b 100644 --- a/modules/imgcodecs/misc/objc/ios/Mat+QuickLook.mm +++ b/modules/imgcodecs/misc/objc/ios/Mat+QuickLook.mm @@ -22,11 +22,15 @@ static UIFont* getBodoni72() { } static UIFont* getAnySerif() { +#if defined(__IPHONE_OS_VERSION_MAX_ALLOWED) && __IPHONE_OS_VERSION_MAX_ALLOWED >= 130000 if (@available(iOS 13.0, *)) { return [UIFont fontWithDescriptor:[[UIFontDescriptor preferredFontDescriptorWithTextStyle:UIFontTextStyleBody] fontDescriptorWithDesign:UIFontDescriptorSystemDesignSerif] size:SIZE]; } else { return nil; } +#else + return nil; +#endif } static UIFont* getSystemFont() { diff --git a/modules/imgcodecs/misc/objc/macosx/Mat+QuickLook.mm b/modules/imgcodecs/misc/objc/macosx/Mat+QuickLook.mm index 6775f81780..bdc5eda811 100644 --- a/modules/imgcodecs/misc/objc/macosx/Mat+QuickLook.mm +++ b/modules/imgcodecs/misc/objc/macosx/Mat+QuickLook.mm @@ -22,11 +22,15 @@ static NSFont* getBodoni72() { } static NSFont* getAnySerif() { +#if defined(__MAC_OS_X_VERSION_MAX_ALLOWED) && __MAC_OS_X_VERSION_MAX_ALLOWED >= 110000 if (@available(macOS 11.0, *)) { return [NSFont fontWithDescriptor:[[NSFontDescriptor preferredFontDescriptorForTextStyle:NSFontTextStyleBody options:@{}] fontDescriptorWithDesign:NSFontDescriptorSystemDesignSerif] size:SIZE]; } else { return nil; } +#else + return nil; +#endif } static NSFont* getSystemFont() { From a3d7811f240c19c2c07640be45dcf16a2b947248 Mon Sep 17 00:00:00 2001 From: Oliver Kuckertz Date: Wed, 6 Oct 2021 18:41:05 +0200 Subject: [PATCH 232/262] Merge pull request #20725 from mologie:fix-dnn-tf-on-arm * dnn: fix unaligned memory access crash on armv7 The getTensorContent function would return a Mat pointing to some member of a Protobuf-encoded message. Protobuf does not make any alignment guarantees, which results in a crash on armv7 when loading models while bit 2 is set in /proc/cpu/alignment (or the relevant kernel feature for alignment compatibility is disabled). Any read attempt from the previously unaligned data member would send SIGBUS. As workaround, this commit makes an aligned copy via existing clone functionality in getTensorContent. The unsafe copy=false option is removed. Unfortunately, a rather crude hack in PReLUSubgraph in fact writes(!) to the Protobuf message. We limit ourselves to fixing the alignment issues in this commit, and add getTensorContentRefUnaligned to cover the write case with a safe memcpy. A FIXME marks the issue. * dnn: reduce amount of .clone() calls * dnn: update FIXME comment Co-authored-by: Alexander Alekhin --- .../src/tensorflow/tf_graph_simplifier.cpp | 41 +++++++++++++++++-- .../src/tensorflow/tf_graph_simplifier.hpp | 2 +- modules/dnn/src/tensorflow/tf_importer.cpp | 4 ++ 3 files changed, 42 insertions(+), 5 deletions(-) diff --git a/modules/dnn/src/tensorflow/tf_graph_simplifier.cpp b/modules/dnn/src/tensorflow/tf_graph_simplifier.cpp index 354fef0297..54395504c7 100644 --- a/modules/dnn/src/tensorflow/tf_graph_simplifier.cpp +++ b/modules/dnn/src/tensorflow/tf_graph_simplifier.cpp @@ -19,6 +19,16 @@ CV__DNN_EXPERIMENTAL_NS_BEGIN using ::google::protobuf::RepeatedField; using ::google::protobuf::MapPair; +static Mat getTensorContentRef_(const tensorflow::TensorProto& tensor); +static inline +bool isAlignedMat(const Mat& m) +{ + int depth = m.depth(); + int alignment = CV_ELEM_SIZE1(depth); + return (((size_t)m.data) & (alignment - 1)) == 0; +} + + class TFNodeWrapper : public ImportNodeWrapper { public: @@ -719,8 +729,19 @@ public: { if (!negativeScales) { - Mat scales = getTensorContent(inputNodes[1]->attr().at("value").tensor(), /*copy*/false); - scales *= -1; + Mat scalesRef = getTensorContentRef_(inputNodes[1]->attr().at("value").tensor()); + // FIXME: This breaks the const guarantees of tensor() by writing to scalesRef + if (isAlignedMat(scalesRef)) + { + scalesRef *= -1; + } + else + { + Mat scales = scalesRef.clone() * -1; + CV_Assert(scalesRef.isContinuous()); + CV_Assert(scales.isContinuous()); + memcpy(scalesRef.data, scales.data, scales.total() * scales.elemSize()); + } } } @@ -832,7 +853,8 @@ void RemoveIdentityOps(tensorflow::GraphDef& net) } } -Mat getTensorContent(const tensorflow::TensorProto &tensor, bool copy) +// NB: returned Mat::data pointer may be unaligned +Mat getTensorContentRef_(const tensorflow::TensorProto& tensor) { const std::string& content = tensor.tensor_content(); Mat m; @@ -904,7 +926,18 @@ Mat getTensorContent(const tensorflow::TensorProto &tensor, bool copy) CV_Error(Error::StsError, "Tensor's data type is not supported"); break; } - return copy ? m.clone() : m; + + return m; +} + +Mat getTensorContent(const tensorflow::TensorProto& tensor, bool forceCopy) +{ + // If necessary clone m to have aligned data pointer + Mat m = getTensorContentRef_(tensor); + if (forceCopy || !isAlignedMat(m)) + return m.clone(); + else + return m; } void releaseTensor(tensorflow::TensorProto* tensor) diff --git a/modules/dnn/src/tensorflow/tf_graph_simplifier.hpp b/modules/dnn/src/tensorflow/tf_graph_simplifier.hpp index 55f36cdb44..4f3dfa870d 100644 --- a/modules/dnn/src/tensorflow/tf_graph_simplifier.hpp +++ b/modules/dnn/src/tensorflow/tf_graph_simplifier.hpp @@ -21,7 +21,7 @@ void RemoveIdentityOps(tensorflow::GraphDef& net); void simplifySubgraphs(tensorflow::GraphDef& net); -Mat getTensorContent(const tensorflow::TensorProto &tensor, bool copy = true); +Mat getTensorContent(const tensorflow::TensorProto& tensor, bool forceCopy = true); void releaseTensor(tensorflow::TensorProto* tensor); diff --git a/modules/dnn/src/tensorflow/tf_importer.cpp b/modules/dnn/src/tensorflow/tf_importer.cpp index 521c8ce4c3..16c5c16308 100644 --- a/modules/dnn/src/tensorflow/tf_importer.cpp +++ b/modules/dnn/src/tensorflow/tf_importer.cpp @@ -122,8 +122,10 @@ void parseTensor(const tensorflow::TensorProto &tensor, Mat &dstBlob) } dstBlob.create(shape, CV_32F); + CV_Assert(dstBlob.isContinuous()); Mat tensorContent = getTensorContent(tensor, /*no copy*/false); + CV_Assert(tensorContent.isContinuous()); int size = tensorContent.total(); CV_Assert(size == (int)dstBlob.total()); @@ -2522,8 +2524,10 @@ void TFImporter::kernelFromTensor(const tensorflow::TensorProto &tensor, Mat &ds out_c = shape[0]; input_c = shape[1]; dstBlob.create(shape, CV_32F); + CV_Assert(dstBlob.isContinuous()); Mat tensorContent = getTensorContent(tensor, /*no copy*/false); + CV_Assert(tensorContent.isContinuous()); int size = tensorContent.total(); CV_Assert(size == (int)dstBlob.total()); From c6a6f39d29aec1a8f355abe8435b38d15b7f43ba Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Wed, 6 Oct 2021 16:49:02 +0000 Subject: [PATCH 233/262] android: drop sourceCompatibility/targetCompatibility options --- modules/java/android_sdk/android_gradle_lib/build.gradle | 4 ---- modules/java/android_sdk/build.gradle.in | 4 ---- 2 files changed, 8 deletions(-) diff --git a/modules/java/android_sdk/android_gradle_lib/build.gradle b/modules/java/android_sdk/android_gradle_lib/build.gradle index 3966c9def6..41cdb9e5e1 100644 --- a/modules/java/android_sdk/android_gradle_lib/build.gradle +++ b/modules/java/android_sdk/android_gradle_lib/build.gradle @@ -36,10 +36,6 @@ android { proguardFiles getDefaultProguardFile('proguard-android.txt'), 'proguard-rules.txt' } } - compileOptions { - sourceCompatibility JavaVersion.VERSION_1_6 - targetCompatibility JavaVersion.VERSION_1_6 - } sourceSets { main { diff --git a/modules/java/android_sdk/build.gradle.in b/modules/java/android_sdk/build.gradle.in index 4268231873..90f0e88fd7 100644 --- a/modules/java/android_sdk/build.gradle.in +++ b/modules/java/android_sdk/build.gradle.in @@ -128,10 +128,6 @@ android { proguardFiles getDefaultProguardFile('proguard-android.txt'), 'proguard-rules.txt' } } - compileOptions { - sourceCompatibility JavaVersion.VERSION_1_6 - targetCompatibility JavaVersion.VERSION_1_6 - } sourceSets { main { From c49cfefe88633d73aed53061bb7b345445c74546 Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Wed, 6 Oct 2021 17:25:32 +0000 Subject: [PATCH 234/262] videoio: fix plugins handling if no filesystem available --- modules/videoio/CMakeLists.txt | 7 ++- modules/videoio/src/cap.cpp | 12 ++++- modules/videoio/src/videoio_registry.cpp | 62 +++++++++++++----------- 3 files changed, 50 insertions(+), 31 deletions(-) diff --git a/modules/videoio/CMakeLists.txt b/modules/videoio/CMakeLists.txt index 534fcf0e37..98127d1c98 100644 --- a/modules/videoio/CMakeLists.txt +++ b/modules/videoio/CMakeLists.txt @@ -1,5 +1,10 @@ +set(VIDEOIO_ENABLE_PLUGINS_DEFAULT ON) +if(EMSCRIPTEN OR IOS OR WINRT) + set(VIDEOIO_ENABLE_PLUGINS_DEFAULT OFF) +endif() + set(VIDEOIO_PLUGIN_LIST "" CACHE STRING "List of videoio backends to be compiled as plugins (ffmpeg, gstreamer, mfx, msmf or special value 'all')") -set(VIDEOIO_ENABLE_PLUGINS "ON" CACHE BOOL "Allow building and using of videoio plugins") +set(VIDEOIO_ENABLE_PLUGINS "${VIDEOIO_ENABLE_PLUGINS_DEFAULT}" CACHE BOOL "Allow building and using of videoio plugins") mark_as_advanced(VIDEOIO_PLUGIN_LIST VIDEOIO_ENABLE_PLUGINS) string(REPLACE "," ";" VIDEOIO_PLUGIN_LIST "${VIDEOIO_PLUGIN_LIST}") # support comma-separated list (,) too diff --git a/modules/videoio/src/cap.cpp b/modules/videoio/src/cap.cpp index f8967514ed..fa958e2c8f 100644 --- a/modules/videoio/src/cap.cpp +++ b/modules/videoio/src/cap.cpp @@ -122,7 +122,11 @@ bool VideoCapture::open(const String& filename, int apiPreference, const std::ve const VideoBackendInfo& info = backends[i]; if (apiPreference == CAP_ANY || apiPreference == info.id) { - + if (!info.backendFactory) + { + CV_LOG_DEBUG(NULL, "VIDEOIO(" << info.name << "): factory is not available (plugins require filesystem support)"); + continue; + } CV_CAPTURE_LOG_DEBUG(NULL, cv::format("VIDEOIO(%s): trying capture filename='%s' ...", info.name, filename.c_str())); @@ -232,10 +236,14 @@ bool VideoCapture::open(int cameraNum, int apiPreference, const std::vector const VideoBackendInfo& info = backends[i]; if (apiPreference == CAP_ANY || apiPreference == info.id) { + if (!info.backendFactory) + { + CV_LOG_DEBUG(NULL, "VIDEOIO(" << info.name << "): factory is not available (plugins require filesystem support)"); + continue; + } CV_CAPTURE_LOG_DEBUG(NULL, cv::format("VIDEOIO(%s): trying capture cameraNum=%d ...", info.name, cameraNum)); - CV_Assert(!info.backendFactory.empty()); const Ptr backend = info.backendFactory->getBackend(); if (!backend.empty()) diff --git a/modules/videoio/src/videoio_registry.cpp b/modules/videoio/src/videoio_registry.cpp index 61e3ac0724..6cd32edc27 100644 --- a/modules/videoio/src/videoio_registry.cpp +++ b/modules/videoio/src/videoio_registry.cpp @@ -8,6 +8,8 @@ #include "opencv2/videoio/registry.hpp" +#include "opencv2/core/utils/filesystem.private.hpp" // OPENCV_HAVE_FILESYSTEM_SUPPORT + #include "cap_librealsense.hpp" #include "cap_dshow.hpp" @@ -34,15 +36,19 @@ namespace cv { namespace { +#if OPENCV_HAVE_FILESYSTEM_SUPPORT && defined(ENABLE_PLUGINS) #define DECLARE_DYNAMIC_BACKEND(cap, name, mode) \ { \ cap, (BackendMode)(mode), 1000, name, createPluginBackendFactory(cap, name) \ -} +}, +#else +#define DECLARE_DYNAMIC_BACKEND(cap, name, mode) /* nothing */ +#endif #define DECLARE_STATIC_BACKEND(cap, name, mode, createCaptureFile, createCaptureCamera, createWriter) \ { \ cap, (BackendMode)(mode), 1000, name, createBackendFactory(createCaptureFile, createCaptureCamera, createWriter) \ -} +}, /** Ordering guidelines: - modern optimized, multi-platform libraries: ffmpeg, gstreamer, Media SDK @@ -55,90 +61,90 @@ namespace { static const struct VideoBackendInfo builtin_backends[] = { #ifdef HAVE_FFMPEG - DECLARE_STATIC_BACKEND(CAP_FFMPEG, "FFMPEG", MODE_CAPTURE_BY_FILENAME | MODE_WRITER, cvCreateFileCapture_FFMPEG_proxy, 0, cvCreateVideoWriter_FFMPEG_proxy), + DECLARE_STATIC_BACKEND(CAP_FFMPEG, "FFMPEG", MODE_CAPTURE_BY_FILENAME | MODE_WRITER, cvCreateFileCapture_FFMPEG_proxy, 0, cvCreateVideoWriter_FFMPEG_proxy) #elif defined(ENABLE_PLUGINS) || defined(HAVE_FFMPEG_WRAPPER) - DECLARE_DYNAMIC_BACKEND(CAP_FFMPEG, "FFMPEG", MODE_CAPTURE_BY_FILENAME | MODE_WRITER), + DECLARE_DYNAMIC_BACKEND(CAP_FFMPEG, "FFMPEG", MODE_CAPTURE_BY_FILENAME | MODE_WRITER) #endif #ifdef HAVE_GSTREAMER - DECLARE_STATIC_BACKEND(CAP_GSTREAMER, "GSTREAMER", MODE_CAPTURE_ALL | MODE_WRITER, createGStreamerCapture_file, createGStreamerCapture_cam, create_GStreamer_writer), + DECLARE_STATIC_BACKEND(CAP_GSTREAMER, "GSTREAMER", MODE_CAPTURE_ALL | MODE_WRITER, createGStreamerCapture_file, createGStreamerCapture_cam, create_GStreamer_writer) #elif defined(ENABLE_PLUGINS) - DECLARE_DYNAMIC_BACKEND(CAP_GSTREAMER, "GSTREAMER", MODE_CAPTURE_ALL | MODE_WRITER), + DECLARE_DYNAMIC_BACKEND(CAP_GSTREAMER, "GSTREAMER", MODE_CAPTURE_ALL | MODE_WRITER) #endif #ifdef HAVE_MFX // Media SDK - DECLARE_STATIC_BACKEND(CAP_INTEL_MFX, "INTEL_MFX", MODE_CAPTURE_BY_FILENAME | MODE_WRITER, create_MFX_capture, 0, create_MFX_writer), + DECLARE_STATIC_BACKEND(CAP_INTEL_MFX, "INTEL_MFX", MODE_CAPTURE_BY_FILENAME | MODE_WRITER, create_MFX_capture, 0, create_MFX_writer) #elif defined(ENABLE_PLUGINS) - DECLARE_DYNAMIC_BACKEND(CAP_INTEL_MFX, "INTEL_MFX", MODE_CAPTURE_BY_FILENAME | MODE_WRITER), + DECLARE_DYNAMIC_BACKEND(CAP_INTEL_MFX, "INTEL_MFX", MODE_CAPTURE_BY_FILENAME | MODE_WRITER) #endif // Apple platform #ifdef HAVE_AVFOUNDATION - DECLARE_STATIC_BACKEND(CAP_AVFOUNDATION, "AVFOUNDATION", MODE_CAPTURE_ALL | MODE_WRITER, create_AVFoundation_capture_file, create_AVFoundation_capture_cam, create_AVFoundation_writer), + DECLARE_STATIC_BACKEND(CAP_AVFOUNDATION, "AVFOUNDATION", MODE_CAPTURE_ALL | MODE_WRITER, create_AVFoundation_capture_file, create_AVFoundation_capture_cam, create_AVFoundation_writer) #endif // Windows #ifdef WINRT_VIDEO - DECLARE_STATIC_BACKEND(CAP_WINRT, "WINRT", MODE_CAPTURE_BY_INDEX, 0, create_WRT_capture, 0), + DECLARE_STATIC_BACKEND(CAP_WINRT, "WINRT", MODE_CAPTURE_BY_INDEX, 0, create_WRT_capture, 0) #endif #ifdef HAVE_MSMF - DECLARE_STATIC_BACKEND(CAP_MSMF, "MSMF", MODE_CAPTURE_ALL | MODE_WRITER, cvCreateCapture_MSMF, cvCreateCapture_MSMF, cvCreateVideoWriter_MSMF), + DECLARE_STATIC_BACKEND(CAP_MSMF, "MSMF", MODE_CAPTURE_ALL | MODE_WRITER, cvCreateCapture_MSMF, cvCreateCapture_MSMF, cvCreateVideoWriter_MSMF) #elif defined(ENABLE_PLUGINS) && defined(_WIN32) - DECLARE_DYNAMIC_BACKEND(CAP_MSMF, "MSMF", MODE_CAPTURE_ALL | MODE_WRITER), + DECLARE_DYNAMIC_BACKEND(CAP_MSMF, "MSMF", MODE_CAPTURE_ALL | MODE_WRITER) #endif #ifdef HAVE_DSHOW - DECLARE_STATIC_BACKEND(CAP_DSHOW, "DSHOW", MODE_CAPTURE_BY_INDEX, 0, create_DShow_capture, 0), + DECLARE_STATIC_BACKEND(CAP_DSHOW, "DSHOW", MODE_CAPTURE_BY_INDEX, 0, create_DShow_capture, 0) #endif // Linux, some Unix #if defined HAVE_CAMV4L2 - DECLARE_STATIC_BACKEND(CAP_V4L2, "V4L2", MODE_CAPTURE_ALL, create_V4L_capture_file, create_V4L_capture_cam, 0), + DECLARE_STATIC_BACKEND(CAP_V4L2, "V4L2", MODE_CAPTURE_ALL, create_V4L_capture_file, create_V4L_capture_cam, 0) #elif defined HAVE_VIDEOIO - DECLARE_STATIC_BACKEND(CAP_V4L, "V4L_BSD", MODE_CAPTURE_ALL, create_V4L_capture_file, create_V4L_capture_cam, 0), + DECLARE_STATIC_BACKEND(CAP_V4L, "V4L_BSD", MODE_CAPTURE_ALL, create_V4L_capture_file, create_V4L_capture_cam, 0) #endif // RGB-D universal #ifdef HAVE_OPENNI2 - DECLARE_STATIC_BACKEND(CAP_OPENNI2, "OPENNI2", MODE_CAPTURE_ALL, create_OpenNI2_capture_file, create_OpenNI2_capture_cam, 0), + DECLARE_STATIC_BACKEND(CAP_OPENNI2, "OPENNI2", MODE_CAPTURE_ALL, create_OpenNI2_capture_file, create_OpenNI2_capture_cam, 0) #endif #ifdef HAVE_LIBREALSENSE - DECLARE_STATIC_BACKEND(CAP_REALSENSE, "INTEL_REALSENSE", MODE_CAPTURE_BY_INDEX, 0, create_RealSense_capture, 0), + DECLARE_STATIC_BACKEND(CAP_REALSENSE, "INTEL_REALSENSE", MODE_CAPTURE_BY_INDEX, 0, create_RealSense_capture, 0) #endif // OpenCV file-based only - DECLARE_STATIC_BACKEND(CAP_IMAGES, "CV_IMAGES", MODE_CAPTURE_BY_FILENAME | MODE_WRITER, create_Images_capture, 0, create_Images_writer), - DECLARE_STATIC_BACKEND(CAP_OPENCV_MJPEG, "CV_MJPEG", MODE_CAPTURE_BY_FILENAME | MODE_WRITER, createMotionJpegCapture, 0, createMotionJpegWriter), + DECLARE_STATIC_BACKEND(CAP_IMAGES, "CV_IMAGES", MODE_CAPTURE_BY_FILENAME | MODE_WRITER, create_Images_capture, 0, create_Images_writer) + DECLARE_STATIC_BACKEND(CAP_OPENCV_MJPEG, "CV_MJPEG", MODE_CAPTURE_BY_FILENAME | MODE_WRITER, createMotionJpegCapture, 0, createMotionJpegWriter) // special interfaces / stereo cameras / other SDKs #if defined(HAVE_DC1394_2) - DECLARE_STATIC_BACKEND(CAP_FIREWIRE, "FIREWIRE", MODE_CAPTURE_BY_INDEX, 0, create_DC1394_capture, 0), + DECLARE_STATIC_BACKEND(CAP_FIREWIRE, "FIREWIRE", MODE_CAPTURE_BY_INDEX, 0, create_DC1394_capture, 0) #endif // GigE #ifdef HAVE_PVAPI - DECLARE_STATIC_BACKEND(CAP_PVAPI, "PVAPI", MODE_CAPTURE_BY_INDEX, 0, create_PvAPI_capture, 0), + DECLARE_STATIC_BACKEND(CAP_PVAPI, "PVAPI", MODE_CAPTURE_BY_INDEX, 0, create_PvAPI_capture, 0) #endif #ifdef HAVE_XIMEA - DECLARE_STATIC_BACKEND(CAP_XIAPI, "XIMEA", MODE_CAPTURE_ALL, create_XIMEA_capture_file, create_XIMEA_capture_cam, 0), + DECLARE_STATIC_BACKEND(CAP_XIAPI, "XIMEA", MODE_CAPTURE_ALL, create_XIMEA_capture_file, create_XIMEA_capture_cam, 0) #endif #ifdef HAVE_ARAVIS_API - DECLARE_STATIC_BACKEND(CAP_ARAVIS, "ARAVIS", MODE_CAPTURE_BY_INDEX, 0, create_Aravis_capture, 0), + DECLARE_STATIC_BACKEND(CAP_ARAVIS, "ARAVIS", MODE_CAPTURE_BY_INDEX, 0, create_Aravis_capture, 0) #endif #ifdef HAVE_UEYE // uEye - DECLARE_STATIC_BACKEND(CAP_UEYE, "UEYE", MODE_CAPTURE_BY_INDEX, 0, create_ueye_camera, 0), + DECLARE_STATIC_BACKEND(CAP_UEYE, "UEYE", MODE_CAPTURE_BY_INDEX, 0, create_ueye_camera, 0) #elif defined(ENABLE_PLUGINS) - DECLARE_DYNAMIC_BACKEND(CAP_UEYE, "UEYE", MODE_CAPTURE_BY_INDEX), + DECLARE_DYNAMIC_BACKEND(CAP_UEYE, "UEYE", MODE_CAPTURE_BY_INDEX) #endif #ifdef HAVE_GPHOTO2 - DECLARE_STATIC_BACKEND(CAP_GPHOTO2, "GPHOTO2", MODE_CAPTURE_ALL, createGPhoto2Capture, createGPhoto2Capture, 0), + DECLARE_STATIC_BACKEND(CAP_GPHOTO2, "GPHOTO2", MODE_CAPTURE_ALL, createGPhoto2Capture, createGPhoto2Capture, 0) #endif #ifdef HAVE_XINE - DECLARE_STATIC_BACKEND(CAP_XINE, "XINE", MODE_CAPTURE_BY_FILENAME, createXINECapture, 0, 0), + DECLARE_STATIC_BACKEND(CAP_XINE, "XINE", MODE_CAPTURE_BY_FILENAME, createXINECapture, 0, 0) #endif #if defined(HAVE_ANDROID_MEDIANDK) || defined(HAVE_ANDROID_NATIVE_CAMERA) DECLARE_STATIC_BACKEND(CAP_ANDROID, "ANDROID_NATIVE", @@ -163,7 +169,7 @@ static const struct VideoBackendInfo builtin_backends[] = #else 0, #endif - 0), + 0) #endif // dropped backends: MIL, TYZX }; From 2221dcc9f297072a0772d752c91397fcc7a6cdc3 Mon Sep 17 00:00:00 2001 From: Smirnov Egor Date: Tue, 5 Oct 2021 16:46:37 +0300 Subject: [PATCH 235/262] add SoftNMS implementation --- modules/dnn/include/opencv2/dnn/dnn.hpp | 33 +++++++++++ modules/dnn/src/nms.cpp | 77 +++++++++++++++++++++++++ modules/dnn/test/test_nms.cpp | 37 ++++++++++++ 3 files changed, 147 insertions(+) diff --git a/modules/dnn/include/opencv2/dnn/dnn.hpp b/modules/dnn/include/opencv2/dnn/dnn.hpp index bf1670051a..8b4e5e21d4 100644 --- a/modules/dnn/include/opencv2/dnn/dnn.hpp +++ b/modules/dnn/include/opencv2/dnn/dnn.hpp @@ -1130,6 +1130,39 @@ CV__DNN_INLINE_NS_BEGIN CV_OUT std::vector& indices, const float eta = 1.f, const int top_k = 0); + /** + * @brief Enum of Soft NMS methods. + * @see softNMSBoxes + */ + enum class SoftNMSMethod + { + SOFTNMS_LINEAR = 1, + SOFTNMS_GAUSSIAN = 2 + }; + + /** @brief Performs soft non maximum suppression given boxes and corresponding scores. + * Reference: https://arxiv.org/abs/1704.04503 + * @param bboxes a set of bounding boxes to apply Soft NMS. + * @param scores a set of corresponding confidences. + * @param updated_scores a set of corresponding updated confidences. + * @param score_threshold a threshold used to filter boxes by score. + * @param nms_threshold a threshold used in non maximum suppression. + * @param indices the kept indices of bboxes after NMS. + * @param top_k keep at most @p top_k picked indices. + * @param sigma parameter of Gaussian weighting. + * @param method Gaussian or linear. + * @see SoftNMSMethod + */ + CV_EXPORTS_W void softNMSBoxes(const std::vector& bboxes, + const std::vector& scores, + CV_OUT std::vector& updated_scores, + const float score_threshold, + const float nms_threshold, + CV_OUT std::vector& indices, + size_t top_k = 0, + const float sigma = 0.5, + SoftNMSMethod method = SoftNMSMethod::SOFTNMS_GAUSSIAN); + /** @brief This class is presented high-level API for neural networks. * diff --git a/modules/dnn/src/nms.cpp b/modules/dnn/src/nms.cpp index 26489dc7e9..d321cfed88 100644 --- a/modules/dnn/src/nms.cpp +++ b/modules/dnn/src/nms.cpp @@ -58,6 +58,83 @@ void NMSBoxes(const std::vector& bboxes, const std::vector& NMSFast_(bboxes, scores, score_threshold, nms_threshold, eta, top_k, indices, rotatedRectIOU); } +void softNMSBoxes(const std::vector& bboxes, + const std::vector& scores, + std::vector& updated_scores, + const float score_threshold, + const float nms_threshold, + std::vector& indices, + size_t top_k, + const float sigma, + SoftNMSMethod method) +{ + CV_Assert_N(bboxes.size() == scores.size(), score_threshold >= 0, + nms_threshold >= 0, sigma >= 0); + + indices.clear(); + updated_scores.clear(); + + std::vector > score_index_vec(scores.size()); + for (size_t i = 0; i < scores.size(); i++) + { + score_index_vec[i].first = scores[i]; + score_index_vec[i].second = i; + } + + const auto score_cmp = [](const std::pair& a, const std::pair& b) + { + return a.first == b.first ? a.second > b.second : a.first < b.first; + }; + + top_k = top_k == 0 ? scores.size() : std::min(top_k, scores.size()); + ptrdiff_t start = 0; + while (indices.size() < top_k) + { + auto it = std::max_element(score_index_vec.begin() + start, score_index_vec.end(), score_cmp); + + float bscore = it->first; + size_t bidx = it->second; + + if (bscore < score_threshold) + { + break; + } + + indices.push_back(static_cast(bidx)); + updated_scores.push_back(bscore); + std::swap(score_index_vec[start], *it); // first start elements are chosen + + for (size_t i = start + 1; i < scores.size(); ++i) + { + float& bscore_i = score_index_vec[i].first; + const size_t bidx_i = score_index_vec[i].second; + + if (bscore_i < score_threshold) + { + continue; + } + + float overlap = rectOverlap(bboxes[bidx], bboxes[bidx_i]); + + switch (method) + { + case SoftNMSMethod::SOFTNMS_LINEAR: + if (overlap > nms_threshold) + { + bscore_i *= 1.f - overlap; + } + break; + case SoftNMSMethod::SOFTNMS_GAUSSIAN: + bscore_i *= exp(-(overlap * overlap) / sigma); + break; + default: + CV_Error(Error::StsBadArg, "Not supported SoftNMS method."); + } + } + ++start; + } +} + CV__DNN_INLINE_NS_END }// dnn }// cv diff --git a/modules/dnn/test/test_nms.cpp b/modules/dnn/test/test_nms.cpp index 53cb3500ec..5ba240287c 100644 --- a/modules/dnn/test/test_nms.cpp +++ b/modules/dnn/test/test_nms.cpp @@ -37,4 +37,41 @@ TEST(NMS, Accuracy) ASSERT_EQ(indices[i], ref_indices[i]); } +TEST(SoftNMS, Accuracy) +{ + //reference results are obtained using TF v2.7 tf.image.non_max_suppression_with_scores + std::string dataPath = findDataFile("dnn/soft_nms_reference.yml"); + FileStorage fs(dataPath, FileStorage::READ); + + std::vector bboxes; + std::vector scores; + std::vector ref_indices; + std::vector ref_updated_scores; + + fs["boxes"] >> bboxes; + fs["probs"] >> scores; + fs["indices"] >> ref_indices; + fs["updated_scores"] >> ref_updated_scores; + + std::vector updated_scores; + const float score_thresh = .01f; + const float nms_thresh = .5f; + std::vector indices; + const size_t top_k = 0; + const float sigma = 1.; // sigma in TF is being multiplied by 2, so 0.5 should be passed there + cv::dnn::softNMSBoxes(bboxes, scores, updated_scores, score_thresh, nms_thresh, indices, top_k, sigma); + + ASSERT_EQ(ref_indices.size(), indices.size()); + for(size_t i = 0; i < indices.size(); i++) + { + ASSERT_EQ(indices[i], ref_indices[i]); + } + + ASSERT_EQ(ref_updated_scores.size(), updated_scores.size()); + for(size_t i = 0; i < updated_scores.size(); i++) + { + EXPECT_NEAR(updated_scores[i], ref_updated_scores[i], 1e-7); + } +} + }} // namespace From bdaa6a191005498f429142e3ddac039e589b3c83 Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Wed, 6 Oct 2021 19:09:22 +0000 Subject: [PATCH 236/262] highgui: repair Qt backend --- modules/highgui/CMakeLists.txt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/modules/highgui/CMakeLists.txt b/modules/highgui/CMakeLists.txt index 3ef1535d2d..c5541241a4 100644 --- a/modules/highgui/CMakeLists.txt +++ b/modules/highgui/CMakeLists.txt @@ -51,6 +51,8 @@ set(OPENCV_HIGHGUI_BUILTIN_BACKEND "") if(HAVE_QT) set(OPENCV_HIGHGUI_BUILTIN_BACKEND "QT${QT_VERSION_MAJOR}") + add_definitions(-DHAVE_QT) + if(QT_VERSION_MAJOR GREATER 4) # "Automoc" doesn't work properly with opencv_world build, use QT_WRAP_CPP() directly #set(CMAKE_AUTOMOC ON) @@ -75,6 +77,7 @@ if(HAVE_QT) set(qt_deps Core Gui Widgets Test Concurrent) if(HAVE_QT_OPENGL) + add_definitions(-DHAVE_QT_OPENGL) list(APPEND qt_deps OpenGL) endif() From 94e92cd6c0250bd091ffea99f7ab036e6292d7a3 Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Wed, 6 Oct 2021 21:27:18 +0000 Subject: [PATCH 237/262] dnn(ocl): skip int8 tests due to memory access issues --- modules/dnn/test/test_int8_layers.cpp | 131 ++++++++++++++++++++++++++ 1 file changed, 131 insertions(+) diff --git a/modules/dnn/test/test_int8_layers.cpp b/modules/dnn/test/test_int8_layers.cpp index 5e6c05c716..85c30a4271 100644 --- a/modules/dnn/test/test_int8_layers.cpp +++ b/modules/dnn/test/test_int8_layers.cpp @@ -544,6 +544,12 @@ TEST_P(Test_Int8_nets, AlexNet) if (backend != DNN_BACKEND_OPENCV) throw SkipTestException("Only OpenCV backend is supported"); + if (target == DNN_TARGET_OPENCL_FP16 && !ocl::Device::getDefault().isIntel()) + applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16); + if (target == DNN_TARGET_OPENCL && !ocl::Device::getDefault().isIntel()) + applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL); + + Net net = readNetFromCaffe(findDataFile("dnn/bvlc_alexnet.prototxt"), findDataFile("dnn/bvlc_alexnet.caffemodel", false)); @@ -557,6 +563,11 @@ TEST_P(Test_Int8_nets, AlexNet) TEST_P(Test_Int8_nets, GoogLeNet) { + if (target == DNN_TARGET_OPENCL_FP16 && !ocl::Device::getDefault().isIntel()) + applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16); + if (target == DNN_TARGET_OPENCL && !ocl::Device::getDefault().isIntel()) + applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL); + Net net = readNetFromCaffe(findDataFile("dnn/bvlc_googlenet.prototxt"), findDataFile("dnn/bvlc_googlenet.caffemodel", false)); @@ -576,6 +587,11 @@ TEST_P(Test_Int8_nets, ResNet50) if (backend != DNN_BACKEND_OPENCV) throw SkipTestException("Only OpenCV backend is supported"); + if (target == DNN_TARGET_OPENCL_FP16 && !ocl::Device::getDefault().isIntel()) + applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16); + if (target == DNN_TARGET_OPENCL && !ocl::Device::getDefault().isIntel()) + applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL); + Net net = readNetFromCaffe(findDataFile("dnn/ResNet-50-deploy.prototxt"), findDataFile("dnn/ResNet-50-model.caffemodel", false)); @@ -591,6 +607,11 @@ TEST_P(Test_Int8_nets, DenseNet121) { applyTestTag(CV_TEST_TAG_MEMORY_512MB); + if (target == DNN_TARGET_OPENCL_FP16 && !ocl::Device::getDefault().isIntel()) + applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16); + if (target == DNN_TARGET_OPENCL && !ocl::Device::getDefault().isIntel()) + applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL); + Net net = readNetFromCaffe(findDataFile("dnn/DenseNet_121.prototxt", false), findDataFile("dnn/DenseNet_121.caffemodel", false)); @@ -607,6 +628,11 @@ TEST_P(Test_Int8_nets, SqueezeNet_v1_1) if(target == DNN_TARGET_OPENCL_FP16) applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16); + if (target == DNN_TARGET_OPENCL_FP16 && !ocl::Device::getDefault().isIntel()) + applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16); + if (target == DNN_TARGET_OPENCL && !ocl::Device::getDefault().isIntel()) + applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL); + Net net = readNetFromCaffe(findDataFile("dnn/squeezenet_v1.1.prototxt"), findDataFile("dnn/squeezenet_v1.1.caffemodel", false)); @@ -626,6 +652,11 @@ TEST_P(Test_Int8_nets, CaffeNet) applyTestTag(target == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_512MB : CV_TEST_TAG_MEMORY_1GB); #endif + if (target == DNN_TARGET_OPENCL_FP16 && !ocl::Device::getDefault().isIntel()) + applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16); + if (target == DNN_TARGET_OPENCL && !ocl::Device::getDefault().isIntel()) + applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL); + #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2019030000) if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 && target == DNN_TARGET_MYRIAD && getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_X) @@ -643,6 +674,11 @@ TEST_P(Test_Int8_nets, RCNN_ILSVRC13) applyTestTag(target == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_512MB : CV_TEST_TAG_MEMORY_1GB); #endif + if (target == DNN_TARGET_OPENCL_FP16 && !ocl::Device::getDefault().isIntel()) + applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16); + if (target == DNN_TARGET_OPENCL && !ocl::Device::getDefault().isIntel()) + applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL); + #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2019030000) if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 && target == DNN_TARGET_MYRIAD && getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_X) @@ -654,16 +690,31 @@ TEST_P(Test_Int8_nets, RCNN_ILSVRC13) TEST_P(Test_Int8_nets, Inception_v2) { + if (target == DNN_TARGET_OPENCL_FP16 && !ocl::Device::getDefault().isIntel()) + applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16); + if (target == DNN_TARGET_OPENCL && !ocl::Device::getDefault().isIntel()) + applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL); + testONNXNet("inception_v2", default_l1, default_lInf, true); } TEST_P(Test_Int8_nets, MobileNet_v2) { + if (target == DNN_TARGET_OPENCL_FP16 && !ocl::Device::getDefault().isIntel()) + applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16); + if (target == DNN_TARGET_OPENCL && !ocl::Device::getDefault().isIntel()) + applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL); + testONNXNet("mobilenetv2", default_l1, default_lInf, true); } TEST_P(Test_Int8_nets, Shufflenet) { + if (target == DNN_TARGET_OPENCL_FP16 && !ocl::Device::getDefault().isIntel()) + applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16); + if (target == DNN_TARGET_OPENCL && !ocl::Device::getDefault().isIntel()) + applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL); + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019) { if (target == DNN_TARGET_OPENCL_FP16) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16, CV_TEST_TAG_DNN_SKIP_IE_NN_BUILDER); @@ -675,6 +726,11 @@ TEST_P(Test_Int8_nets, Shufflenet) TEST_P(Test_Int8_nets, MobileNet_SSD) { + if (target == DNN_TARGET_OPENCL_FP16 && !ocl::Device::getDefault().isIntel()) + applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16); + if (target == DNN_TARGET_OPENCL && !ocl::Device::getDefault().isIntel()) + applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL); + Net net = readNetFromCaffe(findDataFile("dnn/MobileNetSSD_deploy.prototxt", false), findDataFile("dnn/MobileNetSSD_deploy.caffemodel", false)); @@ -688,6 +744,11 @@ TEST_P(Test_Int8_nets, MobileNet_SSD) TEST_P(Test_Int8_nets, MobileNet_v1_SSD) { + if (target == DNN_TARGET_OPENCL_FP16 && !ocl::Device::getDefault().isIntel()) + applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16); + if (target == DNN_TARGET_OPENCL && !ocl::Device::getDefault().isIntel()) + applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL); + Net net = readNetFromTensorflow(findDataFile("dnn/ssd_mobilenet_v1_coco_2017_11_17.pb", false), findDataFile("dnn/ssd_mobilenet_v1_coco_2017_11_17.pbtxt")); @@ -701,6 +762,11 @@ TEST_P(Test_Int8_nets, MobileNet_v1_SSD) TEST_P(Test_Int8_nets, MobileNet_v1_SSD_PPN) { + if (target == DNN_TARGET_OPENCL_FP16 && !ocl::Device::getDefault().isIntel()) + applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16); + if (target == DNN_TARGET_OPENCL && !ocl::Device::getDefault().isIntel()) + applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL); + #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2018050000) if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 && (target == DNN_TARGET_OPENCL || target == DNN_TARGET_OPENCL_FP16)) applyTestTag(target == DNN_TARGET_OPENCL ? CV_TEST_TAG_DNN_SKIP_IE_OPENCL : CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16, @@ -720,6 +786,11 @@ TEST_P(Test_Int8_nets, MobileNet_v1_SSD_PPN) TEST_P(Test_Int8_nets, Inception_v2_SSD) { + if (target == DNN_TARGET_OPENCL_FP16 && !ocl::Device::getDefault().isIntel()) + applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16); + if (target == DNN_TARGET_OPENCL && !ocl::Device::getDefault().isIntel()) + applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL); + applyTestTag(target == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_512MB : CV_TEST_TAG_MEMORY_1GB); #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_LE(2019010000) if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 && target == DNN_TARGET_MYRIAD && @@ -744,6 +815,11 @@ TEST_P(Test_Int8_nets, Inception_v2_SSD) TEST_P(Test_Int8_nets, opencv_face_detector) { + if (target == DNN_TARGET_OPENCL_FP16 && !ocl::Device::getDefault().isIntel()) + applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16); + if (target == DNN_TARGET_OPENCL && !ocl::Device::getDefault().isIntel()) + applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL); + Net net = readNetFromCaffe(findDataFile("dnn/opencv_face_detector.prototxt"), findDataFile("dnn/opencv_face_detector.caffemodel", false)); @@ -762,6 +838,11 @@ TEST_P(Test_Int8_nets, opencv_face_detector) TEST_P(Test_Int8_nets, EfficientDet) { + if (target == DNN_TARGET_OPENCL_FP16 && !ocl::Device::getDefault().isIntel()) + applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16); + if (target == DNN_TARGET_OPENCL && !ocl::Device::getDefault().isIntel()) + applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL); + if (target != DNN_TARGET_CPU) { if (target == DNN_TARGET_OPENCL_FP16) applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16); @@ -789,6 +870,11 @@ TEST_P(Test_Int8_nets, FasterRCNN_resnet50) CV_TEST_TAG_DEBUG_VERYLONG ); + if (target == DNN_TARGET_OPENCL_FP16 && !ocl::Device::getDefault().isIntel()) + applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16); + if (target == DNN_TARGET_OPENCL && !ocl::Device::getDefault().isIntel()) + applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL); + #ifdef INF_ENGINE_RELEASE if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 && (INF_ENGINE_VER_MAJOR_LT(2019020000) || target != DNN_TARGET_CPU)) @@ -827,6 +913,11 @@ TEST_P(Test_Int8_nets, FasterRCNN_inceptionv2) CV_TEST_TAG_DEBUG_VERYLONG ); + if (target == DNN_TARGET_OPENCL_FP16 && !ocl::Device::getDefault().isIntel()) + applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16); + if (target == DNN_TARGET_OPENCL && !ocl::Device::getDefault().isIntel()) + applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL); + #ifdef INF_ENGINE_RELEASE if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 && (INF_ENGINE_VER_MAJOR_LT(2019020000) || target != DNN_TARGET_CPU)) @@ -869,6 +960,11 @@ TEST_P(Test_Int8_nets, FasterRCNN_vgg16) CV_TEST_TAG_DEBUG_VERYLONG ); + if (target == DNN_TARGET_OPENCL_FP16 && !ocl::Device::getDefault().isIntel()) + applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16); + if (target == DNN_TARGET_OPENCL && !ocl::Device::getDefault().isIntel()) + applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL); + #if defined(INF_ENGINE_RELEASE) if ((backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 || backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) && (target == DNN_TARGET_OPENCL || target == DNN_TARGET_OPENCL_FP16)) applyTestTag(target == DNN_TARGET_OPENCL ? CV_TEST_TAG_DNN_SKIP_IE_OPENCL : CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16); @@ -902,6 +998,11 @@ TEST_P(Test_Int8_nets, FasterRCNN_zf) CV_TEST_TAG_DEBUG_LONG ); + if (target == DNN_TARGET_OPENCL_FP16 && !ocl::Device::getDefault().isIntel()) + applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16); + if (target == DNN_TARGET_OPENCL && !ocl::Device::getDefault().isIntel()) + applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL); + if ((backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 || backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) && target == DNN_TARGET_OPENCL_FP16) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16); @@ -932,6 +1033,11 @@ TEST_P(Test_Int8_nets, RFCN) CV_TEST_TAG_DEBUG_VERYLONG ); + if (target == DNN_TARGET_OPENCL_FP16 && !ocl::Device::getDefault().isIntel()) + applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16); + if (target == DNN_TARGET_OPENCL && !ocl::Device::getDefault().isIntel()) + applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL); + if ((backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 || backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) && target == DNN_TARGET_OPENCL_FP16) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16); @@ -961,6 +1067,11 @@ TEST_P(Test_Int8_nets, YoloVoc) CV_TEST_TAG_LONG ); + if (target == DNN_TARGET_OPENCL_FP16 && !ocl::Device::getDefault().isIntel()) + applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16); + if (target == DNN_TARGET_OPENCL && !ocl::Device::getDefault().isIntel()) + applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL); + #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2020040000) if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target == DNN_TARGET_OPENCL) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL, CV_TEST_TAG_DNN_SKIP_IE_VERSION); @@ -1003,6 +1114,11 @@ TEST_P(Test_Int8_nets, TinyYoloVoc) { applyTestTag(CV_TEST_TAG_MEMORY_512MB); + if (target == DNN_TARGET_OPENCL_FP16 && !ocl::Device::getDefault().isIntel()) + applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16); + if (target == DNN_TARGET_OPENCL && !ocl::Device::getDefault().isIntel()) + applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL); + #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2020040000) if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target == DNN_TARGET_OPENCL) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL, CV_TEST_TAG_DNN_SKIP_IE_VERSION); @@ -1039,6 +1155,11 @@ TEST_P(Test_Int8_nets, YOLOv3) { applyTestTag(CV_TEST_TAG_LONG, (target == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_1GB : CV_TEST_TAG_MEMORY_2GB)); + if (target == DNN_TARGET_OPENCL_FP16 && !ocl::Device::getDefault().isIntel()) + applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16); + if (target == DNN_TARGET_OPENCL && !ocl::Device::getDefault().isIntel()) + applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL); + #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2020040000) if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target == DNN_TARGET_OPENCL) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL, CV_TEST_TAG_DNN_SKIP_IE_VERSION); @@ -1097,6 +1218,11 @@ TEST_P(Test_Int8_nets, YOLOv4) { applyTestTag(CV_TEST_TAG_LONG, (target == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_1GB : CV_TEST_TAG_MEMORY_2GB)); + if (target == DNN_TARGET_OPENCL_FP16 && !ocl::Device::getDefault().isIntel()) + applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16); + if (target == DNN_TARGET_OPENCL && !ocl::Device::getDefault().isIntel()) + applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL); + #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2020040000) if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target == DNN_TARGET_OPENCL) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL, CV_TEST_TAG_DNN_SKIP_IE_VERSION); @@ -1159,6 +1285,11 @@ TEST_P(Test_Int8_nets, YOLOv4_tiny) target == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_512MB : CV_TEST_TAG_MEMORY_1GB ); + if (target == DNN_TARGET_OPENCL_FP16 && !ocl::Device::getDefault().isIntel()) + applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16); + if (target == DNN_TARGET_OPENCL && !ocl::Device::getDefault().isIntel()) + applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL); + #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_GE(2021010000) if (target == DNN_TARGET_MYRIAD) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD, CV_TEST_TAG_DNN_SKIP_IE_VERSION); From eab2b9dc09570b141adeba3814f196a4efd8adb2 Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Wed, 6 Oct 2021 18:43:12 +0000 Subject: [PATCH 238/262] core: ensure is_trivially_copyable for simple types --- modules/core/include/opencv2/core/types.hpp | 126 +++----------------- modules/core/test/test_misc.cpp | 23 ++++ 2 files changed, 39 insertions(+), 110 deletions(-) diff --git a/modules/core/include/opencv2/core/types.hpp b/modules/core/include/opencv2/core/types.hpp index 3f0131da8c..5cdddab35d 100644 --- a/modules/core/include/opencv2/core/types.hpp +++ b/modules/core/include/opencv2/core/types.hpp @@ -162,13 +162,13 @@ public: //! default constructor Point_(); Point_(_Tp _x, _Tp _y); - Point_(const Point_& pt); - Point_(Point_&& pt) CV_NOEXCEPT; + Point_(const Point_& pt) = default; + Point_(Point_&& pt) CV_NOEXCEPT = default; Point_(const Size_<_Tp>& sz); Point_(const Vec<_Tp, 2>& v); - Point_& operator = (const Point_& pt); - Point_& operator = (Point_&& pt) CV_NOEXCEPT; + Point_& operator = (const Point_& pt) = default; + Point_& operator = (Point_&& pt) CV_NOEXCEPT = default; //! conversion to another data type template operator Point_<_Tp2>() const; @@ -244,13 +244,13 @@ public: //! default constructor Point3_(); Point3_(_Tp _x, _Tp _y, _Tp _z); - Point3_(const Point3_& pt); - Point3_(Point3_&& pt) CV_NOEXCEPT; + Point3_(const Point3_& pt) = default; + Point3_(Point3_&& pt) CV_NOEXCEPT = default; explicit Point3_(const Point_<_Tp>& pt); Point3_(const Vec<_Tp, 3>& v); - Point3_& operator = (const Point3_& pt); - Point3_& operator = (Point3_&& pt) CV_NOEXCEPT; + Point3_& operator = (const Point3_& pt) = default; + Point3_& operator = (Point3_&& pt) CV_NOEXCEPT = default; //! conversion to another data type template operator Point3_<_Tp2>() const; //! conversion to cv::Vec<> @@ -320,12 +320,12 @@ public: //! default constructor Size_(); Size_(_Tp _width, _Tp _height); - Size_(const Size_& sz); - Size_(Size_&& sz) CV_NOEXCEPT; + Size_(const Size_& sz) = default; + Size_(Size_&& sz) CV_NOEXCEPT = default; Size_(const Point_<_Tp>& pt); - Size_& operator = (const Size_& sz); - Size_& operator = (Size_&& sz) CV_NOEXCEPT; + Size_& operator = (const Size_& sz) = default; + Size_& operator = (Size_&& sz) CV_NOEXCEPT = default; //! the area (width*height) _Tp area() const; //! aspect ratio (width/height) @@ -425,13 +425,13 @@ public: //! default constructor Rect_(); Rect_(_Tp _x, _Tp _y, _Tp _width, _Tp _height); - Rect_(const Rect_& r); - Rect_(Rect_&& r) CV_NOEXCEPT; + Rect_(const Rect_& r) = default; + Rect_(Rect_&& r) CV_NOEXCEPT = default; Rect_(const Point_<_Tp>& org, const Size_<_Tp>& sz); Rect_(const Point_<_Tp>& pt1, const Point_<_Tp>& pt2); - Rect_& operator = ( const Rect_& r ); - Rect_& operator = ( Rect_&& r ) CV_NOEXCEPT; + Rect_& operator = (const Rect_& r) = default; + Rect_& operator = (Rect_&& r) CV_NOEXCEPT = default; //! the top-left corner Point_<_Tp> tl() const; //! the bottom-right corner @@ -1164,14 +1164,6 @@ template inline Point_<_Tp>::Point_(_Tp _x, _Tp _y) : x(_x), y(_y) {} -template inline -Point_<_Tp>::Point_(const Point_& pt) - : x(pt.x), y(pt.y) {} - -template inline -Point_<_Tp>::Point_(Point_&& pt) CV_NOEXCEPT - : x(std::move(pt.x)), y(std::move(pt.y)) {} - template inline Point_<_Tp>::Point_(const Size_<_Tp>& sz) : x(sz.width), y(sz.height) {} @@ -1180,20 +1172,6 @@ template inline Point_<_Tp>::Point_(const Vec<_Tp,2>& v) : x(v[0]), y(v[1]) {} -template inline -Point_<_Tp>& Point_<_Tp>::operator = (const Point_& pt) -{ - x = pt.x; y = pt.y; - return *this; -} - -template inline -Point_<_Tp>& Point_<_Tp>::operator = (Point_&& pt) CV_NOEXCEPT -{ - x = std::move(pt.x); y = std::move(pt.y); - return *this; -} - template template inline Point_<_Tp>::operator Point_<_Tp2>() const { @@ -1431,14 +1409,6 @@ template inline Point3_<_Tp>::Point3_(_Tp _x, _Tp _y, _Tp _z) : x(_x), y(_y), z(_z) {} -template inline -Point3_<_Tp>::Point3_(const Point3_& pt) - : x(pt.x), y(pt.y), z(pt.z) {} - -template inline -Point3_<_Tp>::Point3_(Point3_&& pt) CV_NOEXCEPT - : x(std::move(pt.x)), y(std::move(pt.y)), z(std::move(pt.z)) {} - template inline Point3_<_Tp>::Point3_(const Point_<_Tp>& pt) : x(pt.x), y(pt.y), z(_Tp()) {} @@ -1459,20 +1429,6 @@ Point3_<_Tp>::operator Vec<_Tp, 3>() const return Vec<_Tp, 3>(x, y, z); } -template inline -Point3_<_Tp>& Point3_<_Tp>::operator = (const Point3_& pt) -{ - x = pt.x; y = pt.y; z = pt.z; - return *this; -} - -template inline -Point3_<_Tp>& Point3_<_Tp>::operator = (Point3_&& pt) CV_NOEXCEPT -{ - x = std::move(pt.x); y = std::move(pt.y); z = std::move(pt.z); - return *this; -} - template inline _Tp Point3_<_Tp>::dot(const Point3_& pt) const { @@ -1685,14 +1641,6 @@ template inline Size_<_Tp>::Size_(_Tp _width, _Tp _height) : width(_width), height(_height) {} -template inline -Size_<_Tp>::Size_(const Size_& sz) - : width(sz.width), height(sz.height) {} - -template inline -Size_<_Tp>::Size_(Size_&& sz) CV_NOEXCEPT - : width(std::move(sz.width)), height(std::move(sz.height)) {} - template inline Size_<_Tp>::Size_(const Point_<_Tp>& pt) : width(pt.x), height(pt.y) {} @@ -1703,20 +1651,6 @@ Size_<_Tp>::operator Size_<_Tp2>() const return Size_<_Tp2>(saturate_cast<_Tp2>(width), saturate_cast<_Tp2>(height)); } -template inline -Size_<_Tp>& Size_<_Tp>::operator = (const Size_<_Tp>& sz) -{ - width = sz.width; height = sz.height; - return *this; -} - -template inline -Size_<_Tp>& Size_<_Tp>::operator = (Size_<_Tp>&& sz) CV_NOEXCEPT -{ - width = std::move(sz.width); height = std::move(sz.height); - return *this; -} - template inline _Tp Size_<_Tp>::area() const { @@ -1827,14 +1761,6 @@ template inline Rect_<_Tp>::Rect_(_Tp _x, _Tp _y, _Tp _width, _Tp _height) : x(_x), y(_y), width(_width), height(_height) {} -template inline -Rect_<_Tp>::Rect_(const Rect_<_Tp>& r) - : x(r.x), y(r.y), width(r.width), height(r.height) {} - -template inline -Rect_<_Tp>::Rect_(Rect_<_Tp>&& r) CV_NOEXCEPT - : x(std::move(r.x)), y(std::move(r.y)), width(std::move(r.width)), height(std::move(r.height)) {} - template inline Rect_<_Tp>::Rect_(const Point_<_Tp>& org, const Size_<_Tp>& sz) : x(org.x), y(org.y), width(sz.width), height(sz.height) {} @@ -1848,26 +1774,6 @@ Rect_<_Tp>::Rect_(const Point_<_Tp>& pt1, const Point_<_Tp>& pt2) height = std::max(pt1.y, pt2.y) - y; } -template inline -Rect_<_Tp>& Rect_<_Tp>::operator = ( const Rect_<_Tp>& r ) -{ - x = r.x; - y = r.y; - width = r.width; - height = r.height; - return *this; -} - -template inline -Rect_<_Tp>& Rect_<_Tp>::operator = ( Rect_<_Tp>&& r ) CV_NOEXCEPT -{ - x = std::move(r.x); - y = std::move(r.y); - width = std::move(r.width); - height = std::move(r.height); - return *this; -} - template inline Point_<_Tp> Rect_<_Tp>::tl() const { diff --git a/modules/core/test/test_misc.cpp b/modules/core/test/test_misc.cpp index 67d0a53995..55615b0d5f 100644 --- a/modules/core/test/test_misc.cpp +++ b/modules/core/test/test_misc.cpp @@ -798,4 +798,27 @@ TEST(Core_Allocation, alignedAllocation) } } + +#if !(defined(__GNUC__) && __GNUC__ < 5) // GCC 4.8 emits: 'is_trivially_copyable' is not a member of 'std' +TEST(Core_Types, trivially_copyable) +{ + EXPECT_TRUE(std::is_trivially_copyable::value); + EXPECT_TRUE(std::is_trivially_copyable::value); + EXPECT_TRUE(std::is_trivially_copyable::value); + EXPECT_TRUE(std::is_trivially_copyable::value); + EXPECT_TRUE(std::is_trivially_copyable::value); + EXPECT_TRUE(std::is_trivially_copyable::value); + EXPECT_TRUE(std::is_trivially_copyable::value); + //EXPECT_TRUE(std::is_trivially_copyable::value); // derived from Vec (Matx) +} + +TEST(Core_Types, trivially_copyable_extra) +{ + EXPECT_TRUE(std::is_trivially_copyable::value); + EXPECT_TRUE(std::is_trivially_copyable::value); + EXPECT_TRUE(std::is_trivially_copyable::value); + EXPECT_TRUE(std::is_trivially_copyable::value); +} +#endif + }} // namespace From 04b40ff2213ad4810e65feac41e798724dde0d83 Mon Sep 17 00:00:00 2001 From: Stanislaw Halik Date: Mon, 4 Oct 2021 14:42:32 +0200 Subject: [PATCH 239/262] add new supported MSVC version --- cmake/OpenCVDetectCXXCompiler.cmake | 2 ++ cmake/templates/OpenCVConfig.root-WIN32.cmake.in | 14 ++++++++++++++ 2 files changed, 16 insertions(+) diff --git a/cmake/OpenCVDetectCXXCompiler.cmake b/cmake/OpenCVDetectCXXCompiler.cmake index f9fc1189ce..24eda01f18 100644 --- a/cmake/OpenCVDetectCXXCompiler.cmake +++ b/cmake/OpenCVDetectCXXCompiler.cmake @@ -169,6 +169,8 @@ elseif(MSVC) set(OpenCV_RUNTIME vc15) elseif(MSVC_VERSION MATCHES "^192[0-9]$") set(OpenCV_RUNTIME vc16) + elseif(MSVC_VERSION MATCHES "^193[0-9]$") + set(OpenCV_RUNTIME vc17) else() message(WARNING "OpenCV does not recognize MSVC_VERSION \"${MSVC_VERSION}\". Cannot set OpenCV_RUNTIME") endif() diff --git a/cmake/templates/OpenCVConfig.root-WIN32.cmake.in b/cmake/templates/OpenCVConfig.root-WIN32.cmake.in index 5da438a9ee..b0f254ebe8 100644 --- a/cmake/templates/OpenCVConfig.root-WIN32.cmake.in +++ b/cmake/templates/OpenCVConfig.root-WIN32.cmake.in @@ -137,6 +137,20 @@ elseif(MSVC) set(OpenCV_RUNTIME vc14) # selecting previous compatible runtime version endif() endif() + elseif(MSVC_VERSION MATCHES "^193[0-9]$") + set(OpenCV_RUNTIME vc17) + check_one_config(has_VS2022) + if(NOT has_VS2022) + set(OpenCV_RUNTIME vc16) + check_one_config(has_VS2019) + if(NOT has_VS2019) + set(OpenCV_RUNTIME vc15) # selecting previous compatible runtime version + check_one_config(has_VS2017) + if(NOT has_VS2017) + set(OpenCV_RUNTIME vc14) # selecting previous compatible runtime version + endif() + endif() + endif() endif() elseif(MINGW) set(OpenCV_RUNTIME mingw) From d023f316ac281d287a230816a4bac7bc13d1a9de Mon Sep 17 00:00:00 2001 From: Stanislaw Halik Date: Mon, 4 Oct 2021 14:42:08 +0200 Subject: [PATCH 240/262] fix OPENCV_DISABLE_THREAD_SUPPORT Message was, CMake Error at CMakeLists.txt:1475 (message): Not all parallel frameworks have been disabled (using Concurrency). --- cmake/OpenCVFindFrameworks.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/OpenCVFindFrameworks.cmake b/cmake/OpenCVFindFrameworks.cmake index 9d673f2697..19f6d66340 100644 --- a/cmake/OpenCVFindFrameworks.cmake +++ b/cmake/OpenCVFindFrameworks.cmake @@ -17,7 +17,7 @@ else() endif() # --- Concurrency --- -if(MSVC AND NOT HAVE_TBB) +if(MSVC AND NOT HAVE_TBB AND NOT OPENCV_DISABLE_THREAD_SUPPORT) set(_fname "${CMAKE_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/CMakeTmp/concurrencytest.cpp") file(WRITE "${_fname}" "#if _MSC_VER < 1600\n#error\n#endif\nint main() { return 0; }\n") try_compile(HAVE_CONCURRENCY "${CMAKE_BINARY_DIR}" "${_fname}") From dfc94c58f0567446c14aa217d81efb5ca42b5aa8 Mon Sep 17 00:00:00 2001 From: Alexander Panov Date: Fri, 8 Oct 2021 01:46:25 +0300 Subject: [PATCH 241/262] Merge pull request #20823 from AleksandrPanov:fix_orb_integer_overflow Fix ORB integer overflow * set size_t step to fix integer overflow in ptr0 offset * added issue_537 test * minor fix tags, points * added size_t_step and offset to remove mixed unsigned and signed operations * features2d: update ORB checks Co-authored-by: Alexander Alekhin --- modules/features2d/src/orb.cpp | 11 ++++++++--- modules/features2d/test/test_orb.cpp | 26 ++++++++++++++++++++++++++ 2 files changed, 34 insertions(+), 3 deletions(-) diff --git a/modules/features2d/src/orb.cpp b/modules/features2d/src/orb.cpp index 39ebec7f20..85045557b6 100644 --- a/modules/features2d/src/orb.cpp +++ b/modules/features2d/src/orb.cpp @@ -131,12 +131,17 @@ static void HarrisResponses(const Mat& img, const std::vector& layerinfo, std::vector& pts, int blockSize, float harris_k) { - CV_Assert( img.type() == CV_8UC1 && blockSize*blockSize <= 2048 ); + CV_CheckTypeEQ(img.type(), CV_8UC1, ""); + CV_CheckGT(blockSize, 0, ""); + CV_CheckLE(blockSize*blockSize, 2048, ""); size_t ptidx, ptsize = pts.size(); const uchar* ptr00 = img.ptr(); - int step = (int)(img.step/img.elemSize1()); + size_t size_t_step = img.step; + CV_CheckLE(size_t_step * blockSize + blockSize + 1, (size_t)INT_MAX, ""); // ofs computation, step+1 + int step = static_cast(size_t_step); + int r = blockSize/2; float scale = 1.f/((1 << 2) * blockSize * 255.f); @@ -154,7 +159,7 @@ HarrisResponses(const Mat& img, const std::vector& layerinfo, int y0 = cvRound(pts[ptidx].pt.y); int z = pts[ptidx].octave; - const uchar* ptr0 = ptr00 + (y0 - r + layerinfo[z].y)*step + x0 - r + layerinfo[z].x; + const uchar* ptr0 = ptr00 + (y0 - r + layerinfo[z].y)*size_t_step + (x0 - r + layerinfo[z].x); int a = 0, b = 0, c = 0; for( int k = 0; k < blockSize*blockSize; k++ ) diff --git a/modules/features2d/test/test_orb.cpp b/modules/features2d/test/test_orb.cpp index 0ffaa626d7..b37d4708d6 100644 --- a/modules/features2d/test/test_orb.cpp +++ b/modules/features2d/test/test_orb.cpp @@ -141,5 +141,31 @@ TEST(Features2D_ORB, regression_16197) ASSERT_NO_THROW(orbPtr->detectAndCompute(img, noArray(), kps, fv)); } +// https://github.com/opencv/opencv-python/issues/537 +BIGDATA_TEST(Features2D_ORB, regression_opencv_python_537) // memory usage: ~3 Gb +{ + applyTestTag( + CV_TEST_TAG_LONG, + CV_TEST_TAG_DEBUG_VERYLONG, + CV_TEST_TAG_MEMORY_6GB + ); + + const int width = 25000; + const int height = 25000; + Mat img(Size(width, height), CV_8UC1, Scalar::all(0)); + + const int border = 23, num_lines = 23; + for (int i = 0; i < num_lines; i++) + { + cv::Point2i point1(border + i * 100, border + i * 100); + cv::Point2i point2(width - border - i * 100, height - border * i * 100); + cv::line(img, point1, point2, 255, 1, LINE_AA); + } + + Ptr orbPtr = ORB::create(31); + std::vector kps; + Mat fv; + ASSERT_NO_THROW(orbPtr->detectAndCompute(img, noArray(), kps, fv)); +} }} // namespace From 724e04e979f6706680ad359037ea536dc057d7e2 Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Thu, 7 Oct 2021 17:58:19 +0000 Subject: [PATCH 242/262] dnn(ocl4dnn): add extra checks to convolution layer - prevent running code over unsupported/non-tested configurations - prevent integer div by zero --- modules/dnn/src/layers/convolution_layer.cpp | 20 +++++++++++++++++++ .../src/ocl4dnn/src/ocl4dnn_conv_spatial.cpp | 1 + 2 files changed, 21 insertions(+) diff --git a/modules/dnn/src/layers/convolution_layer.cpp b/modules/dnn/src/layers/convolution_layer.cpp index edbd2baefb..1236ff5783 100644 --- a/modules/dnn/src/layers/convolution_layer.cpp +++ b/modules/dnn/src/layers/convolution_layer.cpp @@ -46,6 +46,7 @@ #include "../op_inf_engine.hpp" #include "../ie_ngraph.hpp" +#include #include #include "opencv2/core/hal/hal.hpp" @@ -1494,7 +1495,26 @@ public: config.pad = pad; config.stride = stride; config.dilation = dilation; + if (inputs[0].dims != 4 && inputs[0].dims != umat_blobs[0].dims) + { + static bool bypassCheck = utils::getConfigurationParameterBool("OPENCV_OCL4DNN_CONVOLUTION_IGNORE_INPUT_DIMS_4_CHECK", false); + if (!bypassCheck) + { + CV_LOG_ERROR(NULL, "DNN/OpenCL: Unsupported configuration: inputs[0].dims=" << inputs[0].dims << " umat_blobs[0].dims=" << umat_blobs[0].dims + << ". Consider reporting complete reproducer to https://github.com/opencv/opencv/issues/20833." + << " You can skip this check temporary through OPENCV_OCL4DNN_CONVOLUTION_IGNORE_INPUT_DIMS_4_CHECK=1" + ); + return false; + } + } config.group = inputs[0].size[1] / umat_blobs[0].size[1]; + if (config.group < 1) // config.group == 0 causes div by zero in ocl4dnn code + { + CV_LOG_WARNING(NULL, "DNN/OpenCL: Unsupported config.group=" << config.group + << ". Consider reporting complete reproducer to https://github.com/opencv/opencv/issues/20833" + ); + return false; + } config.bias_term = umat_blobs.size() == 2; config.use_half = use_half; diff --git a/modules/dnn/src/ocl4dnn/src/ocl4dnn_conv_spatial.cpp b/modules/dnn/src/ocl4dnn/src/ocl4dnn_conv_spatial.cpp index 45bd249e5d..022312aa1f 100644 --- a/modules/dnn/src/ocl4dnn/src/ocl4dnn_conv_spatial.cpp +++ b/modules/dnn/src/ocl4dnn/src/ocl4dnn_conv_spatial.cpp @@ -167,6 +167,7 @@ OCL4DNNConvSpatial::OCL4DNNConvSpatial(OCL4DNNConvConfig config) channels_ = config.in_shape[dims - spatial_dims - 1]; num_output_ = config.out_shape[dims - spatial_dims - 1]; group_ = config.group; + CV_CheckGT(group_, 0, ""); // avoid div by zero below fused_activ_ = OCL4DNN_CONV_FUSED_ACTIV_NONE; fused_eltwise_ = false; From 27545dcc86c11aca4a5c8fff0ef14f839d9d8869 Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Thu, 7 Oct 2021 23:27:10 +0000 Subject: [PATCH 243/262] core: add __NetBSD__ build fix in parallel.cpp --- modules/core/src/parallel.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/core/src/parallel.cpp b/modules/core/src/parallel.cpp index cfff4cea4b..d1454457b7 100644 --- a/modules/core/src/parallel.cpp +++ b/modules/core/src/parallel.cpp @@ -54,8 +54,8 @@ #endif #if defined __unix__ || defined __APPLE__ || defined __GLIBC__ \ - || defined __HAIKU__ || defined __EMSCRIPTEN__ || defined __FreeBSD__ \ - || defined __OpenBSD__ + || defined __HAIKU__ || defined __EMSCRIPTEN__ \ + || defined __FreeBSD__ || defined __NetBSD__ || defined __OpenBSD__ #include #include #include From 8c2dd5fb9a4e568466f92db5d954d5ddc351bc9d Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Fri, 8 Oct 2021 00:12:14 +0000 Subject: [PATCH 244/262] dnn(ocl4dnn): cleanup dead code, improve logging --- modules/dnn/src/ocl4dnn/include/ocl4dnn.hpp | 2 - .../src/ocl4dnn/src/ocl4dnn_conv_spatial.cpp | 100 ++++++------------ modules/dnn/src/opencl/conv_layer_spatial.cl | 39 ++----- 3 files changed, 42 insertions(+), 99 deletions(-) diff --git a/modules/dnn/src/ocl4dnn/include/ocl4dnn.hpp b/modules/dnn/src/ocl4dnn/include/ocl4dnn.hpp index d6fb83becb..bf5fba71a1 100644 --- a/modules/dnn/src/ocl4dnn/include/ocl4dnn.hpp +++ b/modules/dnn/src/ocl4dnn/include/ocl4dnn.hpp @@ -222,8 +222,6 @@ class OCL4DNNConvSpatial bool createDWConvKernel(int32_t blockWidth, int32_t blockHeight, int32_t blockDepth); - void CreateSubBuffer(const UMat& buffer, UMat& sub_buffer, - int32_t offset, int32_t size, bool write_only); bool convolve(const UMat &bottom, UMat &top, const UMat &weight, const UMat &bias, int32_t numImages, diff --git a/modules/dnn/src/ocl4dnn/src/ocl4dnn_conv_spatial.cpp b/modules/dnn/src/ocl4dnn/src/ocl4dnn_conv_spatial.cpp index 45bd249e5d..6c468fc7da 100644 --- a/modules/dnn/src/ocl4dnn/src/ocl4dnn_conv_spatial.cpp +++ b/modules/dnn/src/ocl4dnn/src/ocl4dnn_conv_spatial.cpp @@ -218,14 +218,7 @@ OCL4DNNConvSpatial::OCL4DNNConvSpatial(OCL4DNNConvConfig config) #endif if (!use_cache_path_) { - static int warn_ = 0; - if (!warn_) - { - std::cerr - << "OpenCV(ocl4dnn): Kernel configuration cache directory doesn't exist: " << cache_path_ << std::endl - << std::endl; - warn_ = true; - } + CV_LOG_ONCE_ERROR(NULL, "OpenCV(ocl4dnn): Kernel configuration cache directory doesn't exist: " << cache_path_); } } @@ -418,7 +411,6 @@ void OCL4DNNConvSpatial::setupKernelDetails(int32_t kernelType, addDef("CHANNELS", channels_ / group_); addDef("APPLY_BIAS", bias_term_); addDef("OUTPUT_Z", M_); - addDef("ZPAR", 1); setFusionDefine(fused_activ_, fused_eltwise_); src_ = cv::ocl::dnn::conv_layer_spatial_oclsrc; @@ -672,8 +664,7 @@ void interleaveMatrix(Dtype* mem_dst, const Dtype *mem, int r, int c, int interleavedRows, int nonInterleavedRows, int blockWidth, int rowAlignment ) { - CHECK_EQ(interleavedRows % 2, 0) << - "interleaveMatrix only supports even values for interleavedRows."; + CV_Check(interleavedRows, interleavedRows % 2 == 0, "interleaveMatrix only supports even values for interleavedRows."); size_t memSize = r * c * sizeof(float); size_t dstSize = memSize * @@ -685,9 +676,12 @@ void interleaveMatrix(Dtype* mem_dst, const Dtype *mem, const int yStride = c * 2; const Dtype *pSrc = mem; Dtype* pDst = mem_dst; - for (int y = 0; y < r;) { - for (int rows = 0; rows < interleavedRows; rows += 2) { - if ( y >= r ) break; + for (int y = 0; y < r;) + { + for (int rows = 0; rows < interleavedRows; rows += 2) + { + if (y >= r) + break; if ((c % xStride) == 0) { for (int x = 0; x < c / xStride; x++) { memcpy(pDst + x * xStride * 2, // NOLINT @@ -712,11 +706,14 @@ void interleaveMatrix(Dtype* mem_dst, const Dtype *mem, y += 2; } - for (int rows = 0; rows < nonInterleavedRows; rows++) { - if (y >= r) break; + for (int rows = 0; rows < nonInterleavedRows; rows++) + { + if (y >= r) + break; const int stride = rowAlignment; int remaining = c; - for (int x = 0; x < c; x += stride) { + for (int x = 0; x < c; x += stride) + { if (remaining >= stride) { memcpy(pDst + x * 2, pSrc + x, stride * sizeof(Dtype)); // NOLINT remaining -=stride; @@ -852,34 +849,6 @@ bool OCL4DNNConvSpatial::createBasicKernel(int32_t blockWidth, return false; } -template<> -void OCL4DNNConvSpatial::CreateSubBuffer(const UMat& buffer, UMat& sub_buffer, - int32_t offset, int32_t size, bool write_only) -{ - cl_mem sub_mem; - cl_buffer_region region; - cl_int err; - size_t element_size = (use_half_) ? sizeof(short) : sizeof(float); - - region.origin = offset * element_size + buffer.offset; - region.size = size * element_size; - sub_mem = clCreateSubBuffer((cl_mem)buffer.handle(ACCESS_READ), - write_only ? CL_MEM_WRITE_ONLY : CL_MEM_READ_ONLY, - CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); - if (err) - { - std::cout << "Failed to create sub buffer." << std::endl; - return; - } - - int step = element_size, rows = size, cols = 1; - ocl::convertFromBuffer(sub_mem, step, rows, cols, - (use_half_) ? CV_16SC1 : CV_32FC1, sub_buffer); - - //decrease ocl mem refcount - clReleaseMemObject(sub_mem); -} - template<> bool OCL4DNNConvSpatial::convolve(const UMat &bottom, UMat &top, const UMat &weight, const UMat &bias, @@ -938,7 +907,7 @@ bool OCL4DNNConvSpatial::convolve(const UMat &bottom, UMat &top, kernel.set(argIdx++, (uint16_t)output_h_); if (!kernel.run_(3, config->global_work_size, config->local_work_size, false)) { - std::cout << "IDLF kernel run failed." << std::endl; + CV_LOG_ERROR(NULL, "DNN/OpenCL: IDLF kernel run failed"); return false; } } @@ -1012,7 +981,7 @@ bool OCL4DNNConvSpatial::convolve(const UMat &bottom, UMat &top, if (!kernel.run_(3, global_size, config->local_work_size, false)) { - std::cout << "GEMM like kernel run failed." << std::endl; + CV_LOG_ERROR(NULL, "DNN/OpenCL: GEMM like kernel run failed"); return false; } } @@ -1115,14 +1084,9 @@ float OCL4DNNConvSpatial::timedConvolve(const UMat &bottom, UMat &top, { queue = cv::ocl::Queue::getDefault(); } - catch (const cv::Exception&) + catch (const std::exception& e) { - static int warn_ = 0; - if (!warn_) - { - std::cout << "OpenCV(ocl4dnn): Can't get OpenCL default queue for auto-tuning." << std::endl; - warn_ = true; - } + CV_LOG_ONCE_ERROR(NULL, "OpenCV(ocl4dnn): Can't get OpenCL default queue for auto-tuning: " << e.what()); return 1e6; } @@ -1326,9 +1290,9 @@ ocl::Program OCL4DNNConvSpatial::compileKernel() phash.insert(std::pair(kernel_name_, program)); if (!program.ptr()) { - std::cout << "Failed to compile kernel: " << kernel_name_ - << ", buildflags: " << options - << ", errmsg: " << errmsg << std::endl; + CV_LOG_WARNING(NULL, "DNN/OpenCL: Failed to compile kernel: " << kernel_name_ + << ", buildflags: '" << options << "', errmsg: '" << errmsg << "'" + ); } return program; } @@ -1754,7 +1718,8 @@ void OCL4DNNConvSpatial::setupConvolution(const UMat &bottom, fastestTime = kernelQueue[x]->executionTime; } } - if (fastestKernel < 0) break; + if (fastestKernel < 0) + break; // Test fastest kernel bool verified = verifyResult(bottom, top, weight, bias, numImages, kernelQueue[fastestKernel], verifyTop); if (verified == true) { @@ -1913,17 +1878,18 @@ bool OCL4DNNConvSpatial::setupKernelByConfig(int x, int y, int z, int typ { if (z == 1) z = 16; - CHECK_EQ(z == 16 || z == 8, true) << "invalid SIMD size" << std::endl; + CV_Check(z, z == 16 || z == 8, "DNN/OpenCL: IDLF - invalid SIMD size"); } kernelQueue.clear(); createConvolutionKernel(type, x, y, z); - if (kernelQueue.size() != 1) { - std::cerr << "Failed setup kernel by config:" + if (kernelQueue.size() != 1) + { + CV_LOG_ERROR(NULL, "DNN/OpenCL: Failed setup kernel by config: " << " x = " << x << " y = " << y << " z = " << z << " type = " << type - << std::endl; + ); return false; } bestKernelConfig = kernelQueue[0]; @@ -1955,13 +1921,9 @@ bool OCL4DNNConvSpatial::loadTunedConfig() { if (cache_path_.empty()) { - static int warn_ = 0; - if (!warn_) - { - std::cout << "OpenCV(ocl4dnn): consider to specify kernel configuration cache directory " << std::endl - << " via OPENCV_OCL4DNN_CONFIG_PATH parameter." << std::endl; - warn_ = true; - } + CV_LOG_ONCE_WARNING(NULL, "OpenCV(ocl4dnn): consider to specify kernel configuration cache directory " + "through OPENCV_OCL4DNN_CONFIG_PATH parameter." + ); } return false; } diff --git a/modules/dnn/src/opencl/conv_layer_spatial.cl b/modules/dnn/src/opencl/conv_layer_spatial.cl index eb5d354020..c9ddacfb8e 100644 --- a/modules/dnn/src/opencl/conv_layer_spatial.cl +++ b/modules/dnn/src/opencl/conv_layer_spatial.cl @@ -161,23 +161,15 @@ __kernel void ConvolveBasic( const int out_idx = get_global_id(0); // 1D task layout: [output_width * output_height * OUTPUT_Z] const int plane_size = output_width * output_height; const int out_plane_idx = out_idx % plane_size; - const int outputZ = out_idx / plane_size; + const int outputZ = out_idx / plane_size; // kernelNum const int outputY = out_plane_idx / output_width; const int outputX = out_plane_idx % output_width; - const int kernelNum = outputZ * ZPAR; - if (kernelNum < OUTPUT_Z) + if (outputZ < OUTPUT_Z) { - Dtype sum[ZPAR]; - for (int kern = 0; kern < ZPAR; kern++) - { - sum[kern] = 0.0f; - } + Dtype sum = 0.0f; const int org_y = outputY * STRIDE_Y - pad_h; const int org_x = outputX * STRIDE_X - pad_w; - const int currentKernelOffset = kernel_offset + kernelNum*KERNEL_HEIGHT*KERNEL_WIDTH*CHANNELS; -#if APPLY_BIAS - const int biasIndex = bias_offset + kernelNum; -#endif + const int currentKernelOffset = kernel_offset + outputZ*KERNEL_HEIGHT*KERNEL_WIDTH*CHANNELS; const int local_image_offset = org_y * input_width + org_x; const int imageSize = input_width * input_height; __global Dtype* image_dataPtr = (image_data + (image_offset + local_image_offset)); @@ -186,17 +178,13 @@ __kernel void ConvolveBasic( { for (int y = 0; y < KERNEL_HEIGHT; y++) { + int y_ = org_y + y * DILATION_Y; for (int x = 0; x < KERNEL_WIDTH; x++) { - int y_ = org_y + y * DILATION_Y; int x_ = org_x + x * DILATION_X; - if (!(y_ >= 0 && y_ < input_height && x_ >= 0 && x_ < input_width)) + if (y_ >= 0 && y_ < input_height && x_ >= 0 && x_ < input_width) { - continue; - } - for (int kern = 0; kern < ZPAR; kern++) - { - sum[kern] += image_dataPtr[x * DILATION_X] * kernel_dataPtr[kern*KERNEL_HEIGHT*KERNEL_WIDTH*CHANNELS + x]; + sum = mad(image_dataPtr[x * DILATION_X], kernel_dataPtr[x], sum); } } image_dataPtr += input_width * DILATION_Y; @@ -205,18 +193,13 @@ __kernel void ConvolveBasic( image_dataPtr += imageSize - input_width*KERNEL_HEIGHT*DILATION_Y; } - for (int kern = 0; kern < ZPAR; kern++) - { - if (kernelNum + kern < OUTPUT_Z) - { - int offset = convolved_image_offset + (kernelNum+kern)*output_height*output_width + outputY*output_width + outputX; + int offset = convolved_image_offset + out_idx; #if APPLY_BIAS - ACTIVATION_FUNCTION(convolved_image, offset, sum[kern] + bias[biasIndex + kern], biasIndex + kern); + int biasIndex = bias_offset + outputZ; + ACTIVATION_FUNCTION(convolved_image, offset, sum + bias[biasIndex], biasIndex); #else - ACTIVATION_FUNCTION(convolved_image, offset, sum[kern], kernelNum + kern); + ACTIVATION_FUNCTION(convolved_image, offset, sum, outputZ); #endif - } - } } } From e75387f029fe6dbfe3b67837046c1983dfe19971 Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Thu, 7 Oct 2021 15:32:18 +0000 Subject: [PATCH 245/262] core: fix compilation of copy ctors/assignment operators with GCC 4.x --- modules/core/include/opencv2/core/types.hpp | 37 +++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/modules/core/include/opencv2/core/types.hpp b/modules/core/include/opencv2/core/types.hpp index 5cdddab35d..1e9a8b629c 100644 --- a/modules/core/include/opencv2/core/types.hpp +++ b/modules/core/include/opencv2/core/types.hpp @@ -162,13 +162,23 @@ public: //! default constructor Point_(); Point_(_Tp _x, _Tp _y); +#if (defined(__GNUC__) && __GNUC__ < 5) // GCC 4.x bug. Details: https://github.com/opencv/opencv/pull/20837 + Point_(const Point_& pt); + Point_(Point_&& pt) CV_NOEXCEPT = default; +#elif OPENCV_ABI_COMPATIBILITY < 500 Point_(const Point_& pt) = default; Point_(Point_&& pt) CV_NOEXCEPT = default; +#endif Point_(const Size_<_Tp>& sz); Point_(const Vec<_Tp, 2>& v); +#if (defined(__GNUC__) && __GNUC__ < 5) // GCC 4.x bug. Details: https://github.com/opencv/opencv/pull/20837 + Point_& operator = (const Point_& pt); + Point_& operator = (Point_&& pt) CV_NOEXCEPT = default; +#elif OPENCV_ABI_COMPATIBILITY < 500 Point_& operator = (const Point_& pt) = default; Point_& operator = (Point_&& pt) CV_NOEXCEPT = default; +#endif //! conversion to another data type template operator Point_<_Tp2>() const; @@ -244,13 +254,17 @@ public: //! default constructor Point3_(); Point3_(_Tp _x, _Tp _y, _Tp _z); +#if OPENCV_ABI_COMPATIBILITY < 500 Point3_(const Point3_& pt) = default; Point3_(Point3_&& pt) CV_NOEXCEPT = default; +#endif explicit Point3_(const Point_<_Tp>& pt); Point3_(const Vec<_Tp, 3>& v); +#if OPENCV_ABI_COMPATIBILITY < 500 Point3_& operator = (const Point3_& pt) = default; Point3_& operator = (Point3_&& pt) CV_NOEXCEPT = default; +#endif //! conversion to another data type template operator Point3_<_Tp2>() const; //! conversion to cv::Vec<> @@ -320,12 +334,16 @@ public: //! default constructor Size_(); Size_(_Tp _width, _Tp _height); +#if OPENCV_ABI_COMPATIBILITY < 500 Size_(const Size_& sz) = default; Size_(Size_&& sz) CV_NOEXCEPT = default; +#endif Size_(const Point_<_Tp>& pt); +#if OPENCV_ABI_COMPATIBILITY < 500 Size_& operator = (const Size_& sz) = default; Size_& operator = (Size_&& sz) CV_NOEXCEPT = default; +#endif //! the area (width*height) _Tp area() const; //! aspect ratio (width/height) @@ -425,13 +443,17 @@ public: //! default constructor Rect_(); Rect_(_Tp _x, _Tp _y, _Tp _width, _Tp _height); +#if OPENCV_ABI_COMPATIBILITY < 500 Rect_(const Rect_& r) = default; Rect_(Rect_&& r) CV_NOEXCEPT = default; +#endif Rect_(const Point_<_Tp>& org, const Size_<_Tp>& sz); Rect_(const Point_<_Tp>& pt1, const Point_<_Tp>& pt2); +#if OPENCV_ABI_COMPATIBILITY < 500 Rect_& operator = (const Rect_& r) = default; Rect_& operator = (Rect_&& r) CV_NOEXCEPT = default; +#endif //! the top-left corner Point_<_Tp> tl() const; //! the bottom-right corner @@ -1164,6 +1186,12 @@ template inline Point_<_Tp>::Point_(_Tp _x, _Tp _y) : x(_x), y(_y) {} +#if (defined(__GNUC__) && __GNUC__ < 5) // GCC 4.x bug. Details: https://github.com/opencv/opencv/pull/20837 +template inline +Point_<_Tp>::Point_(const Point_& pt) + : x(pt.x), y(pt.y) {} +#endif + template inline Point_<_Tp>::Point_(const Size_<_Tp>& sz) : x(sz.width), y(sz.height) {} @@ -1172,6 +1200,15 @@ template inline Point_<_Tp>::Point_(const Vec<_Tp,2>& v) : x(v[0]), y(v[1]) {} +#if (defined(__GNUC__) && __GNUC__ < 5) // GCC 4.x bug. Details: https://github.com/opencv/opencv/pull/20837 +template inline +Point_<_Tp>& Point_<_Tp>::operator = (const Point_& pt) +{ + x = pt.x; y = pt.y; + return *this; +} +#endif + template template inline Point_<_Tp>::operator Point_<_Tp2>() const { From 9537a909f7fbcca5e9596846e29cae5e7eb1c86a Mon Sep 17 00:00:00 2001 From: Jonas Vautherin Date: Fri, 8 Oct 2021 04:07:04 +0200 Subject: [PATCH 246/262] Merge pull request #20801 from JonasVautherin:fix-gst-error-handling * Fix gst error handling * Use the return value instead of the error, which gives no guarantee of being NULL in case of error * Test err pointer before accessing it * Remove unreachable code * videoio(gstreamer): restore check in writer code Co-authored-by: Alexander Alekhin --- modules/videoio/src/cap_gstreamer.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/modules/videoio/src/cap_gstreamer.cpp b/modules/videoio/src/cap_gstreamer.cpp index d8390691ff..dd6387d892 100644 --- a/modules/videoio/src/cap_gstreamer.cpp +++ b/modules/videoio/src/cap_gstreamer.cpp @@ -761,7 +761,7 @@ bool GStreamerCapture::open(const String &filename_) } else { - CV_WARN("Error opening file: " << filename << " (" << err->message << ")"); + CV_WARN("Error opening file: " << filename << " (" << (err ? err->message : "") << ")"); return false; } } @@ -769,9 +769,9 @@ bool GStreamerCapture::open(const String &filename_) { GSafePtr err; uridecodebin.attach(gst_parse_launch(filename, err.getRef())); - if (err) + if (!uridecodebin) { - CV_WARN("Error opening bin: " << err->message); + CV_WARN("Error opening bin: " << (err ? err->message : "")); return false; } manualpipeline = true; @@ -1979,7 +1979,7 @@ void handleMessage(GstElement * pipeline) gst_message_parse_error(msg, err.getRef(), debug.getRef()); GSafePtr name; name.attach(gst_element_get_name(GST_MESSAGE_SRC (msg))); CV_WARN("Embedded video playback halted; module " << name.get() << - " reported: " << err->message); + " reported: " << (err ? err->message : "")); CV_LOG_DEBUG(NULL, "GStreamer debug: " << debug.get()); gst_element_set_state(GST_ELEMENT(pipeline), GST_STATE_NULL); From 4985311d46f8e073646de3d2abff9ff9a8c7dd05 Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Fri, 8 Oct 2021 04:36:58 +0300 Subject: [PATCH 247/262] core(tls): avoid destruction of TlsAbstraction singleton --- modules/core/src/system.cpp | 135 ++++++++++++++++++++++-------------- 1 file changed, 83 insertions(+), 52 deletions(-) diff --git a/modules/core/src/system.cpp b/modules/core/src/system.cpp index e2284b0415..27142a4034 100644 --- a/modules/core/src/system.cpp +++ b/modules/core/src/system.cpp @@ -53,6 +53,18 @@ #include #include +#ifndef OPENCV_WITH_THREAD_SANITIZER + #if defined(__clang__) && defined(__has_feature) + #if __has_feature(thread_sanitizer) + #define OPENCV_WITH_THREAD_SANITIZER 1 + #include // assume C++11 + #endif + #endif +#endif +#ifndef OPENCV_WITH_THREAD_SANITIZER + #define OPENCV_WITH_THREAD_SANITIZER 0 +#endif + namespace cv { static void _initSystem() @@ -1426,74 +1438,75 @@ namespace details { #endif #endif -template -class DisposedSingletonMark -{ -private: - static bool mark; -protected: - DisposedSingletonMark() {} - ~DisposedSingletonMark() - { - mark = true; - } -public: - static bool isDisposed() { return mark; } -}; - // TLS platform abstraction layer -class TlsAbstraction : public DisposedSingletonMark +class TlsAbstraction { public: TlsAbstraction(); - ~TlsAbstraction(); - void* getData() const + ~TlsAbstraction() { - if (isDisposed()) // guard: static initialization order fiasco - return NULL; - return getData_(); - } - void setData(void *pData) - { - if (isDisposed()) // guard: static initialization order fiasco - return; - return setData_(pData); + // TlsAbstraction singleton should not be released + // There is no reliable way to avoid problems caused by static initialization order fiasco + // NB: Do NOT use logging here + fprintf(stderr, "OpenCV FATAL: TlsAbstraction::~TlsAbstraction() call is not expected\n"); + fflush(stderr); } + void* getData() const; + void setData(void *pData); + + void releaseSystemResources(); + private: - void* getData_() const; - void setData_(void *pData); #ifdef _WIN32 #ifndef WINRT DWORD tlsKey; + bool disposed; #endif #else // _WIN32 pthread_key_t tlsKey; +#if OPENCV_WITH_THREAD_SANITIZER + std::atomic disposed; +#else + bool disposed; +#endif #endif }; -template<> bool DisposedSingletonMark::mark = false; - -static TlsAbstraction& getTlsAbstraction_() +class TlsAbstractionReleaseGuard { - static TlsAbstraction g_tls; // disposed in atexit() handlers (required for unregistering our callbacks) - return g_tls; -} + TlsAbstraction& tls_; +public: + TlsAbstractionReleaseGuard(TlsAbstraction& tls) : tls_(tls) + { + /* nothing */ + } + ~TlsAbstractionReleaseGuard() + { + tls_.releaseSystemResources(); + } +}; + +// TODO use reference static TlsAbstraction* getTlsAbstraction() { #ifdef CV_CXX11 - static TlsAbstraction* instance = &getTlsAbstraction_(); + static TlsAbstraction *g_tls = new TlsAbstraction(); // memory leak is intended here to avoid disposing of TLS container + static TlsAbstractionReleaseGuard g_tlsReleaseGuard(*g_tls); #else - static TlsAbstraction* volatile instance = NULL; - if (instance == NULL) + static TlsAbstraction* volatile g_tls = NULL; + if (g_tls == NULL) { cv::AutoLock lock(cv::getInitializationMutex()); - if (instance == NULL) - instance = &getTlsAbstraction_(); + if (g_tls == NULL) + { + g_tls = new TlsAbstraction(); + static TlsAbstractionReleaseGuard g_tlsReleaseGuard(*g_tls); + } } #endif - return DisposedSingletonMark::isDisposed() ? NULL : instance; + return g_tls; } @@ -1501,12 +1514,15 @@ static TlsAbstraction* getTlsAbstraction() #ifdef WINRT static __declspec( thread ) void* tlsData = NULL; // using C++11 thread attribute for local thread data TlsAbstraction::TlsAbstraction() {} -TlsAbstraction::~TlsAbstraction() {} -void* TlsAbstraction::getData_() const +void TlsAbstraction::releaseSystemResources() +{ + cv::__termination = true; // DllMain is missing in static builds +} +void* TlsAbstraction::getData() const { return tlsData; } -void TlsAbstraction::setData_(void *pData) +void TlsAbstraction::setData(void *pData) { tlsData = pData; } @@ -1515,6 +1531,7 @@ void TlsAbstraction::setData_(void *pData) static void NTAPI opencv_fls_destructor(void* pData); #endif // CV_USE_FLS TlsAbstraction::TlsAbstraction() + : disposed(false) { #ifndef CV_USE_FLS tlsKey = TlsAlloc(); @@ -1523,8 +1540,10 @@ TlsAbstraction::TlsAbstraction() #endif // CV_USE_FLS CV_Assert(tlsKey != TLS_OUT_OF_INDEXES); } -TlsAbstraction::~TlsAbstraction() +void TlsAbstraction::releaseSystemResources() { + cv::__termination = true; // DllMain is missing in static builds + disposed = true; #ifndef CV_USE_FLS TlsFree(tlsKey); #else // CV_USE_FLS @@ -1532,16 +1551,20 @@ TlsAbstraction::~TlsAbstraction() #endif // CV_USE_FLS tlsKey = TLS_OUT_OF_INDEXES; } -void* TlsAbstraction::getData_() const +void* TlsAbstraction::getData() const { + if (disposed) + return NULL; #ifndef CV_USE_FLS return TlsGetValue(tlsKey); #else // CV_USE_FLS return FlsGetValue(tlsKey); #endif // CV_USE_FLS } -void TlsAbstraction::setData_(void *pData) +void TlsAbstraction::setData(void *pData) { + if (disposed) + return; // no-op #ifndef CV_USE_FLS CV_Assert(TlsSetValue(tlsKey, pData) == TRUE); #else // CV_USE_FLS @@ -1552,11 +1575,14 @@ void TlsAbstraction::setData_(void *pData) #else // _WIN32 static void opencv_tls_destructor(void* pData); TlsAbstraction::TlsAbstraction() + : disposed(false) { CV_Assert(pthread_key_create(&tlsKey, opencv_tls_destructor) == 0); } -TlsAbstraction::~TlsAbstraction() +void TlsAbstraction::releaseSystemResources() { + cv::__termination = true; // DllMain is missing in static builds + disposed = true; if (pthread_key_delete(tlsKey) != 0) { // Don't use logging here @@ -1564,12 +1590,16 @@ TlsAbstraction::~TlsAbstraction() fflush(stderr); } } -void* TlsAbstraction::getData_() const +void* TlsAbstraction::getData() const { + if (disposed) + return NULL; return pthread_getspecific(tlsKey); } -void TlsAbstraction::setData_(void *pData) +void TlsAbstraction::setData(void *pData) { + if (disposed) + return; // no-op CV_Assert(pthread_setspecific(tlsKey, pData) == 0); } #endif @@ -1597,6 +1627,7 @@ public: TlsStorage() : tlsSlotsSize(0) { + (void)getTlsAbstraction(); // ensure singeton initialization (for correct order of atexit calls) tlsSlots.reserve(32); threads.reserve(32); g_isTlsStorageInitialized = true; @@ -1834,11 +1865,11 @@ static void WINAPI opencv_fls_destructor(void* pData) #endif // CV_USE_FLS #endif // _WIN32 -static TlsAbstraction* const g_force_initialization_of_TlsAbstraction +static TlsStorage* const g_force_initialization_of_TlsStorage #if defined __GNUC__ __attribute__((unused)) #endif - = getTlsAbstraction(); + = &getTlsStorage(); } // namespace details using namespace details; From 9c84749e2c28d0744376e515a159c3cf30bb43c4 Mon Sep 17 00:00:00 2001 From: Smirnov Egor Date: Wed, 6 Oct 2021 16:09:20 +0300 Subject: [PATCH 248/262] backport YOLOv4x-mish new_coords CUDA implementation --- modules/dnn/src/cuda/region.cu | 78 +++++++++++++------ modules/dnn/src/cuda4dnn/kernels/region.hpp | 2 +- .../dnn/src/cuda4dnn/primitives/region.hpp | 6 +- modules/dnn/src/layers/region_layer.cpp | 5 +- modules/dnn/test/test_darknet_importer.cpp | 10 ++- 5 files changed, 73 insertions(+), 28 deletions(-) diff --git a/modules/dnn/src/cuda/region.cu b/modules/dnn/src/cuda/region.cu index 06b44abe9c..3700a93d99 100644 --- a/modules/dnn/src/cuda/region.cu +++ b/modules/dnn/src/cuda/region.cu @@ -31,7 +31,7 @@ namespace cv { namespace dnn { namespace cuda4dnn { namespace kernels { size_type boxes_per_cell, size_type box_size, size_type rows, size_type cols, T scale_x_y, size_type height_norm, size_type width_norm, - T object_prob_cutoff) + T object_prob_cutoff, bool new_coords) { using vector2_type = get_vector_type_t; auto bias_vPtr = vector2_type::get_pointer(bias.data()); @@ -47,22 +47,43 @@ namespace cv { namespace dnn { namespace cuda4dnn { namespace kernels { const auto y = (box_index % batch_inner_size) / row_inner_size; const auto x = (box_index % row_inner_size) / col_inner_size; - using device::fast_sigmoid; - const auto tmp_x = (fast_sigmoid(input[box_offset + 0]) - static_cast(0.5)) * scale_x_y + static_cast(0.5); - const auto tmp_y = (fast_sigmoid(input[box_offset + 1]) - static_cast(0.5)) * scale_x_y + static_cast(0.5); - output[box_offset + 0] = (T(x) + tmp_x) / T(cols); - output[box_offset + 1] = (T(y) + tmp_y) / T(rows); + /* When new_coords is true, we shouldn't use logistic activation again */ + T objectness_prob; + if (new_coords) + { + const auto tmp_x = (input[box_offset + 0] - static_cast(0.5)) * scale_x_y + static_cast(0.5); + const auto tmp_y = (input[box_offset + 1] - static_cast(0.5)) * scale_x_y + static_cast(0.5); - vector2_type bias_xy; - v_load(bias_xy, bias_vPtr[box_of_the_cell]); + output[box_offset + 0] = fast_divide_ftz(static_cast(x) + tmp_x, static_cast(cols)); + output[box_offset + 1] = fast_divide_ftz(static_cast(y) + tmp_y, static_cast(rows)); - using device::fast_exp; - output[box_offset + 2] = fast_exp(input[box_offset + 2]) * bias_xy.data[0] / T(width_norm); - output[box_offset + 3] = fast_exp(input[box_offset + 3]) * bias_xy.data[1] / T(height_norm); + vector2_type bias_xy; + v_load(bias_xy, bias_vPtr[box_of_the_cell]); - /* squash objectness score into a probability */ - using device::fast_sigmoid; - T objectness_prob = fast_sigmoid(input[box_offset + 4]); + output[box_offset + 2] = input[box_offset + 2] * input[box_offset + 2] * + static_cast(4) * bias_xy.data[0] / static_cast(width_norm); + output[box_offset + 3] = input[box_offset + 3] * input[box_offset + 3] * + static_cast(4) * bias_xy.data[1] / static_cast(height_norm); + + objectness_prob = input[box_offset + 4]; + } + else + { + const auto tmp_x = (fast_sigmoid(input[box_offset + 0]) - static_cast(0.5)) * scale_x_y + static_cast(0.5); + const auto tmp_y = (fast_sigmoid(input[box_offset + 1]) - static_cast(0.5)) * scale_x_y + static_cast(0.5); + + output[box_offset + 0] = fast_divide_ftz(static_cast(x) + tmp_x, static_cast(cols)); + output[box_offset + 1] = fast_divide_ftz(static_cast(y) + tmp_y, static_cast(rows)); + + vector2_type bias_xy; + v_load(bias_xy, bias_vPtr[box_of_the_cell]); + + output[box_offset + 2] = fast_exp(input[box_offset + 2]) * bias_xy.data[0] / static_cast(width_norm); + output[box_offset + 3] = fast_exp(input[box_offset + 3]) * bias_xy.data[1] / static_cast(height_norm); + + /* squash objectness score into a probability */ + objectness_prob = fast_sigmoid(input[box_offset + 4]); + } /* ignore prediction if the objectness probability is less than the cutoff */ if (objectness_prob < object_prob_cutoff) @@ -73,7 +94,8 @@ namespace cv { namespace dnn { namespace cuda4dnn { namespace kernels { } template - __global__ void region_sigmoid_class_score(Span output, View input, T class_prob_cutoff, size_type box_size) + __global__ void region_sigmoid_class_score(Span output, View input, T class_prob_cutoff, + size_type box_size, bool new_coords) { for (auto idx : grid_stride_range(output.size())) { const index_type box_no = idx / box_size; @@ -92,9 +114,20 @@ namespace cv { namespace dnn { namespace cuda4dnn { namespace kernels { * * to obtain the actual class probability, we multiply the conditional probability * with the object probability + * + * when new_coords is true, we shouldn't use logistic activation again. */ - using device::fast_sigmoid; - auto actual_class_prob = objectness_prob * fast_sigmoid(input[idx]); + + T actual_class_prob; + if (new_coords) + { + actual_class_prob = objectness_prob * input[idx]; + } + else + { + actual_class_prob = objectness_prob * fast_sigmoid(input[idx]); + } + if (actual_class_prob <= class_prob_cutoff) actual_class_prob = T(0); output[idx] = actual_class_prob; @@ -147,7 +180,8 @@ namespace cv { namespace dnn { namespace cuda4dnn { namespace kernels { std::size_t boxes_per_cell, std::size_t box_size, std::size_t rows, std::size_t cols, T scale_x_y, std::size_t height_norm, std::size_t width_norm, - bool if_true_sigmoid_else_softmax /* true = sigmoid, false = softmax */) + bool if_true_sigmoid_else_softmax, /* true = sigmoid, false = softmax */ + bool new_coords) { CV_Assert(output.size() == input.size()); CV_Assert(output.size() % box_size == 0); @@ -158,12 +192,12 @@ namespace cv { namespace dnn { namespace cuda4dnn { namespace kernels { launch_kernel(box_kernel, box_policy, output, input, bias, boxes_per_cell, box_size, rows, cols, scale_x_y, height_norm, width_norm, - object_prob_cutoff); + object_prob_cutoff, new_coords); if (if_true_sigmoid_else_softmax) { auto kernel_score = raw::region_sigmoid_class_score; auto policy_score = make_policy(kernel_score, output.size(), 0, stream); - launch_kernel(kernel_score, policy_score, output, input, class_prob_cutoff, box_size); + launch_kernel(kernel_score, policy_score, output, input, class_prob_cutoff, box_size, new_coords); } else { auto kernel_score = raw::region_softmax_class_score; auto policy_score = make_policy(kernel_score, output.size(), 0, stream); @@ -173,10 +207,10 @@ namespace cv { namespace dnn { namespace cuda4dnn { namespace kernels { #if !defined(__CUDA_ARCH__) || (__CUDA_ARCH__ >= 530) template void region(const Stream&, Span<__half>, View<__half>, View<__half>, - __half, __half, std::size_t, std::size_t, std::size_t, std::size_t, __half, std::size_t, std::size_t, bool); + __half, __half, std::size_t, std::size_t, std::size_t, std::size_t, __half, std::size_t, std::size_t, bool, bool); #endif template void region(const Stream&, Span, View, View, - float, float, std::size_t, std::size_t, std::size_t, std::size_t, float, std::size_t, std::size_t, bool); + float, float, std::size_t, std::size_t, std::size_t, std::size_t, float, std::size_t, std::size_t, bool, bool); }}}} /* namespace cv::dnn::cuda4dnn::kernels */ diff --git a/modules/dnn/src/cuda4dnn/kernels/region.hpp b/modules/dnn/src/cuda4dnn/kernels/region.hpp index 87742d2f81..b815fb11c9 100644 --- a/modules/dnn/src/cuda4dnn/kernels/region.hpp +++ b/modules/dnn/src/cuda4dnn/kernels/region.hpp @@ -18,7 +18,7 @@ namespace cv { namespace dnn { namespace cuda4dnn { namespace kernels { std::size_t boxes_per_cell, std::size_t box_size, std::size_t rows, std::size_t cols, T scale_x_y, std::size_t height_norm, std::size_t width_norm, - bool if_true_sigmoid_else_softmax); + bool if_true_sigmoid_else_softmax, bool new_coords); }}}} /* namespace cv::dnn::cuda4dnn::kernels */ diff --git a/modules/dnn/src/cuda4dnn/primitives/region.hpp b/modules/dnn/src/cuda4dnn/primitives/region.hpp index 7813a47bc7..d22d44214e 100644 --- a/modules/dnn/src/cuda4dnn/primitives/region.hpp +++ b/modules/dnn/src/cuda4dnn/primitives/region.hpp @@ -60,6 +60,7 @@ namespace cv { namespace dnn { namespace cuda4dnn { T class_prob_cutoff; T nms_iou_threshold; + bool new_coords; }; template @@ -87,6 +88,7 @@ namespace cv { namespace dnn { namespace cuda4dnn { class_prob_cutoff = config.class_prob_cutoff; nms_iou_threshold = config.nms_iou_threshold; + new_coords = config.new_coords; } void forward( @@ -115,7 +117,8 @@ namespace cv { namespace dnn { namespace cuda4dnn { boxes_per_cell, cell_box_size, rows, cols, scale_x_y, height_norm, width_norm, - if_true_sigmoid_else_softmax + if_true_sigmoid_else_softmax, + new_coords ); if (nms_iou_threshold > 0) { @@ -176,6 +179,7 @@ namespace cv { namespace dnn { namespace cuda4dnn { T object_prob_cutoff, class_prob_cutoff; T nms_iou_threshold; + bool new_coords; }; }}} /* namespace cv::dnn::cuda4dnn */ diff --git a/modules/dnn/src/layers/region_layer.cpp b/modules/dnn/src/layers/region_layer.cpp index 242e5e6f88..73ed53974f 100644 --- a/modules/dnn/src/layers/region_layer.cpp +++ b/modules/dnn/src/layers/region_layer.cpp @@ -125,7 +125,7 @@ public: #endif #ifdef HAVE_CUDA if (backendId == DNN_BACKEND_CUDA) - return new_coords == 0; + return true; #endif return backendId == DNN_BACKEND_OPENCV; } @@ -437,11 +437,12 @@ public: config.scale_x_y = scale_x_y; - config.object_prob_cutoff = (classfix == -1) ? 0.5 : 0.0; + config.object_prob_cutoff = (classfix == -1) ? thresh : 0.f; config.class_prob_cutoff = thresh; config.nms_iou_threshold = nmsThreshold; + config.new_coords = (new_coords == 1); return make_cuda_node(preferableTarget, std::move(context->stream), blobs[0], config); } #endif diff --git a/modules/dnn/test/test_darknet_importer.cpp b/modules/dnn/test/test_darknet_importer.cpp index c85981a8bc..d822efa889 100644 --- a/modules/dnn/test/test_darknet_importer.cpp +++ b/modules/dnn/test/test_darknet_importer.cpp @@ -745,8 +745,14 @@ TEST_P(Test_Darknet_nets, YOLOv4x_mish) }; Mat ref(N0 + N1, 7, CV_32FC1, (void*)ref_); - double scoreDiff = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.006 : 8e-5; - double iouDiff = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.042 : 3e-4; + double scoreDiff = 8e-5; + double iouDiff = 3e-4; + + if (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD || target == DNN_TARGET_CUDA_FP16) + { + scoreDiff = 0.006; + iouDiff = 0.042; + } std::string config_file = "yolov4x-mish.cfg"; std::string weights_file = "yolov4x-mish.weights"; From b1cf5501233405de3ea5926d1d688e421b337458 Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Fri, 8 Oct 2021 18:31:56 +0000 Subject: [PATCH 249/262] release: OpenCV 3.4.16 --- modules/core/include/opencv2/core/version.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/core/include/opencv2/core/version.hpp b/modules/core/include/opencv2/core/version.hpp index dc8a3e603c..d05f5219f0 100644 --- a/modules/core/include/opencv2/core/version.hpp +++ b/modules/core/include/opencv2/core/version.hpp @@ -8,7 +8,7 @@ #define CV_VERSION_MAJOR 3 #define CV_VERSION_MINOR 4 #define CV_VERSION_REVISION 16 -#define CV_VERSION_STATUS "-pre" +#define CV_VERSION_STATUS "" #define CVAUX_STR_EXP(__A) #__A #define CVAUX_STR(__A) CVAUX_STR_EXP(__A) From 34d359fe035a92d48a399c6e6975c77513bd5139 Mon Sep 17 00:00:00 2001 From: Yuantao Feng Date: Sat, 9 Oct 2021 03:13:49 +0800 Subject: [PATCH 250/262] Merge pull request #20422 from fengyuentau:dnn_face Add DNN-based face detection and face recognition into modules/objdetect * Add DNN-based face detector impl and interface * Add a sample for DNN-based face detector * add recog * add notes * move samples from samples/cpp to samples/dnn * add documentation for dnn_face * add set/get methods for input size, nms & score threshold and topk * remove the DNN prefix from the face detector and face recognizer * remove default values in the constructor of impl * regenerate priors after setting input size * two filenames for readnet * Update face.hpp * Update face_recognize.cpp * Update face_match.cpp * Update face.hpp * Update face_recognize.cpp * Update face_match.cpp * Update face_recognize.cpp * Update dnn_face.markdown * Update dnn_face.markdown * Update face.hpp * Update dnn_face.markdown * add regression test for face detection * remove underscore prefix; fix warnings * add reference & acknowledgement for face detection * Update dnn_face.markdown * Update dnn_face.markdown * Update ts.hpp * Update test_face.cpp * Update face_match.cpp * fix a compile error for python interface; add python examples for face detection and recognition * Major changes for Vadim's comments: * Replace class name FaceDetector with FaceDetectorYN in related failes * Declare local mat before loop in modules/objdetect/src/face_detect.cpp * Make input image and save flag optional in samples/dnn/face_detect(.cpp, .py) * Add camera support in samples/dnn/face_detect(.cpp, .py) * correct file paths for regression test * fix convertion warnings; remove extra spaces * update face_recog * Update dnn_face.markdown * Fix warnings and errors for the default CI reports: * Remove trailing white spaces and extra new lines. * Fix convertion warnings for windows and iOS. * Add braces around initialization of subobjects. * Fix warnings and errors for the default CI systems: * Add prefix 'FR_' for each value name in enum DisType to solve the redefinition error for iOS compilation; Modify other code accordingly * Add bookmark '#tutorial_dnn_face' to solve warnings from doxygen * Correct documentations to solve warnings from doxygen * update FaceRecognizerSF * Fix the error for CI to find ONNX models correctly * add suffix f to float assignments * add backend & target options for initializing face recognizer * add checkeq for checking input size and preset size * update test and threshold * changes in response to alalek's comments: * fix typos in samples/dnn/face_match.py * import numpy before importing cv2 * add documentation to .setInputSize() * remove extra include in face_recognize.cpp * fix some bugs * Update dnn_face.markdown * update thresholds; remove useless code * add time suffix to YuNet filename in test * objdetect: update test code --- doc/tutorials/dnn/dnn_face/dnn_face.markdown | 95 ++++++ .../dnn_text_spotting.markdown | 2 +- .../dnn/table_of_content_dnn.markdown | 1 + modules/objdetect/CMakeLists.txt | 2 +- .../objdetect/include/opencv2/objdetect.hpp | 1 + .../include/opencv2/objdetect/face.hpp | 125 ++++++++ modules/objdetect/src/face_detect.cpp | 288 ++++++++++++++++++ modules/objdetect/src/face_recognize.cpp | 182 +++++++++++ modules/objdetect/test/test_face.cpp | 219 +++++++++++++ modules/objdetect/test/test_main.cpp | 17 +- modules/ts/include/opencv2/ts.hpp | 1 + samples/dnn/CMakeLists.txt | 1 + samples/dnn/face_detect.cpp | 132 ++++++++ samples/dnn/face_detect.py | 101 ++++++ samples/dnn/face_match.cpp | 103 +++++++ samples/dnn/face_match.py | 57 ++++ samples/dnn/results/audrybt1.jpg | Bin 0 -> 47680 bytes 17 files changed, 1324 insertions(+), 3 deletions(-) create mode 100644 doc/tutorials/dnn/dnn_face/dnn_face.markdown create mode 100644 modules/objdetect/include/opencv2/objdetect/face.hpp create mode 100644 modules/objdetect/src/face_detect.cpp create mode 100644 modules/objdetect/src/face_recognize.cpp create mode 100644 modules/objdetect/test/test_face.cpp create mode 100644 samples/dnn/face_detect.cpp create mode 100644 samples/dnn/face_detect.py create mode 100644 samples/dnn/face_match.cpp create mode 100644 samples/dnn/face_match.py create mode 100644 samples/dnn/results/audrybt1.jpg diff --git a/doc/tutorials/dnn/dnn_face/dnn_face.markdown b/doc/tutorials/dnn/dnn_face/dnn_face.markdown new file mode 100644 index 0000000000..e5092b8b92 --- /dev/null +++ b/doc/tutorials/dnn/dnn_face/dnn_face.markdown @@ -0,0 +1,95 @@ +# DNN-based Face Detection And Recognition {#tutorial_dnn_face} + +@tableofcontents + +@prev_tutorial{tutorial_dnn_text_spotting} +@next_tutorial{pytorch_cls_tutorial_dnn_conversion} + +| | | +| -: | :- | +| Original Author | Chengrui Wang, Yuantao Feng | +| Compatibility | OpenCV >= 4.5.1 | + +## Introduction + +In this section, we introduce the DNN-based module for face detection and face recognition. Models can be obtained in [Models](#Models). The usage of `FaceDetectorYN` and `FaceRecognizer` are presented in [Usage](#Usage). + +## Models + +There are two models (ONNX format) pre-trained and required for this module: +- [Face Detection](https://github.com/ShiqiYu/libfacedetection.train/tree/master/tasks/task1/onnx): + - Size: 337KB + - Results on WIDER Face Val set: 0.830(easy), 0.824(medium), 0.708(hard) +- [Face Recognition](https://drive.google.com/file/d/1ClK9WiB492c5OZFKveF3XiHCejoOxINW/view?usp=sharing) + - Size: 36.9MB + - Results: + + | Database | Accuracy | Threshold (normL2) | Threshold (cosine) | + | -------- | -------- | ------------------ | ------------------ | + | LFW | 99.60% | 1.128 | 0.363 | + | CALFW | 93.95% | 1.149 | 0.340 | + | CPLFW | 91.05% | 1.204 | 0.275 | + | AgeDB-30 | 94.90% | 1.202 | 0.277 | + | CFP-FP | 94.80% | 1.253 | 0.212 | + +## Usage + +### DNNFaceDetector + +```cpp +// Initialize FaceDetectorYN +Ptr faceDetector = FaceDetectorYN::create(onnx_path, "", image.size(), score_thresh, nms_thresh, top_k); + +// Forward +Mat faces; +faceDetector->detect(image, faces); +``` + +The detection output `faces` is a two-dimension array of type CV_32F, whose rows are the detected face instances, columns are the location of a face and 5 facial landmarks. The format of each row is as follows: + +``` +x1, y1, w, h, x_re, y_re, x_le, y_le, x_nt, y_nt, x_rcm, y_rcm, x_lcm, y_lcm +``` +, where `x1, y1, w, h` are the top-left coordinates, width and height of the face bounding box, `{x, y}_{re, le, nt, rcm, lcm}` stands for the coordinates of right eye, left eye, nose tip, the right corner and left corner of the mouth respectively. + + +### Face Recognition + +Following Face Detection, run codes below to extract face feature from facial image. + +```cpp +// Initialize FaceRecognizer with model path (cv::String) +Ptr faceRecognizer = FaceRecognizer::create(model_path, ""); + +// Aligning and cropping facial image through the first face of faces detected by dnn_face::DNNFaceDetector +Mat aligned_face; +faceRecognizer->alignCrop(image, faces.row(0), aligned_face); + +// Run feature extraction with given aligned_face (cv::Mat) +Mat feature; +faceRecognizer->feature(aligned_face, feature); +feature = feature.clone(); +``` + +After obtaining face features *feature1* and *feature2* of two facial images, run codes below to calculate the identity discrepancy between the two faces. + +```cpp +// Calculating the discrepancy between two face features by using cosine distance. +double cos_score = faceRecognizer->match(feature1, feature2, FaceRecognizer::DisType::COSINE); +// Calculating the discrepancy between two face features by using normL2 distance. +double L2_score = faceRecognizer->match(feature1, feature2, FaceRecognizer::DisType::NORM_L2); +``` + +For example, two faces have same identity if the cosine distance is greater than or equal to 0.363, or the normL2 distance is less than or equal to 1.128. + +## Reference: + +- https://github.com/ShiqiYu/libfacedetection +- https://github.com/ShiqiYu/libfacedetection.train +- https://github.com/zhongyy/SFace + +## Acknowledgement + +Thanks [Professor Shiqi Yu](https://github.com/ShiqiYu/) and [Yuantao Feng](https://github.com/fengyuentau) for training and providing the face detection model. + +Thanks [Professor Deng](http://www.whdeng.cn/), [PhD Candidate Zhong](https://github.com/zhongyy/) and [Master Candidate Wang](https://github.com/crywang/) for training and providing the face recognition model. diff --git a/doc/tutorials/dnn/dnn_text_spotting/dnn_text_spotting.markdown b/doc/tutorials/dnn/dnn_text_spotting/dnn_text_spotting.markdown index b0be2627b2..5c465941ca 100644 --- a/doc/tutorials/dnn/dnn_text_spotting/dnn_text_spotting.markdown +++ b/doc/tutorials/dnn/dnn_text_spotting/dnn_text_spotting.markdown @@ -3,7 +3,7 @@ @tableofcontents @prev_tutorial{tutorial_dnn_OCR} -@next_tutorial{pytorch_cls_tutorial_dnn_conversion} +@next_tutorial{tutorial_dnn_face} | | | | -: | :- | diff --git a/doc/tutorials/dnn/table_of_content_dnn.markdown b/doc/tutorials/dnn/table_of_content_dnn.markdown index 0d5e43ee11..3f74826dac 100644 --- a/doc/tutorials/dnn/table_of_content_dnn.markdown +++ b/doc/tutorials/dnn/table_of_content_dnn.markdown @@ -10,6 +10,7 @@ Deep Neural Networks (dnn module) {#tutorial_table_of_content_dnn} - @subpage tutorial_dnn_custom_layers - @subpage tutorial_dnn_OCR - @subpage tutorial_dnn_text_spotting +- @subpage tutorial_dnn_face #### PyTorch models with OpenCV In this section you will find the guides, which describe how to run classification, segmentation and detection PyTorch DNN models with OpenCV. diff --git a/modules/objdetect/CMakeLists.txt b/modules/objdetect/CMakeLists.txt index 3fa0c5d33b..f4d5b22b74 100644 --- a/modules/objdetect/CMakeLists.txt +++ b/modules/objdetect/CMakeLists.txt @@ -1,5 +1,5 @@ set(the_description "Object Detection") -ocv_define_module(objdetect opencv_core opencv_imgproc opencv_calib3d WRAP java objc python js) +ocv_define_module(objdetect opencv_core opencv_imgproc opencv_calib3d opencv_dnn WRAP java objc python js) if(HAVE_QUIRC) get_property(QUIRC_INCLUDE GLOBAL PROPERTY QUIRC_INCLUDE_DIR) diff --git a/modules/objdetect/include/opencv2/objdetect.hpp b/modules/objdetect/include/opencv2/objdetect.hpp index eaee1290ce..59dca7399b 100644 --- a/modules/objdetect/include/opencv2/objdetect.hpp +++ b/modules/objdetect/include/opencv2/objdetect.hpp @@ -768,5 +768,6 @@ protected: } #include "opencv2/objdetect/detection_based_tracker.hpp" +#include "opencv2/objdetect/face.hpp" #endif diff --git a/modules/objdetect/include/opencv2/objdetect/face.hpp b/modules/objdetect/include/opencv2/objdetect/face.hpp new file mode 100644 index 0000000000..f2429c5f31 --- /dev/null +++ b/modules/objdetect/include/opencv2/objdetect/face.hpp @@ -0,0 +1,125 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. + +#ifndef OPENCV_OBJDETECT_FACE_HPP +#define OPENCV_OBJDETECT_FACE_HPP + +#include + +/** @defgroup dnn_face DNN-based face detection and recognition + */ + +namespace cv +{ + +/** @brief DNN-based face detector, model download link: https://github.com/ShiqiYu/libfacedetection.train/tree/master/tasks/task1/onnx. + */ +class CV_EXPORTS_W FaceDetectorYN +{ +public: + virtual ~FaceDetectorYN() {}; + + /** @brief Set the size for the network input, which overwrites the input size of creating model. Call this method when the size of input image does not match the input size when creating model + * + * @param input_size the size of the input image + */ + CV_WRAP virtual void setInputSize(const Size& input_size) = 0; + + CV_WRAP virtual Size getInputSize() = 0; + + /** @brief Set the score threshold to filter out bounding boxes of score less than the given value + * + * @param score_threshold threshold for filtering out bounding boxes + */ + CV_WRAP virtual void setScoreThreshold(float score_threshold) = 0; + + CV_WRAP virtual float getScoreThreshold() = 0; + + /** @brief Set the Non-maximum-suppression threshold to suppress bounding boxes that have IoU greater than the given value + * + * @param nms_threshold threshold for NMS operation + */ + CV_WRAP virtual void setNMSThreshold(float nms_threshold) = 0; + + CV_WRAP virtual float getNMSThreshold() = 0; + + /** @brief Set the number of bounding boxes preserved before NMS + * + * @param top_k the number of bounding boxes to preserve from top rank based on score + */ + CV_WRAP virtual void setTopK(int top_k) = 0; + + CV_WRAP virtual int getTopK() = 0; + + /** @brief A simple interface to detect face from given image + * + * @param image an image to detect + * @param faces detection results stored in a cv::Mat + */ + CV_WRAP virtual int detect(InputArray image, OutputArray faces) = 0; + + /** @brief Creates an instance of this class with given parameters + * + * @param model the path to the requested model + * @param config the path to the config file for compability, which is not requested for ONNX models + * @param input_size the size of the input image + * @param score_threshold the threshold to filter out bounding boxes of score smaller than the given value + * @param nms_threshold the threshold to suppress bounding boxes of IoU bigger than the given value + * @param top_k keep top K bboxes before NMS + * @param backend_id the id of backend + * @param target_id the id of target device + */ + CV_WRAP static Ptr create(const String& model, + const String& config, + const Size& input_size, + float score_threshold = 0.9f, + float nms_threshold = 0.3f, + int top_k = 5000, + int backend_id = 0, + int target_id = 0); +}; + +/** @brief DNN-based face recognizer, model download link: https://drive.google.com/file/d/1ClK9WiB492c5OZFKveF3XiHCejoOxINW/view. + */ +class CV_EXPORTS_W FaceRecognizerSF +{ +public: + virtual ~FaceRecognizerSF() {}; + + /** @brief Definition of distance used for calculating the distance between two face features + */ + enum DisType { FR_COSINE=0, FR_NORM_L2=1 }; + + /** @brief Aligning image to put face on the standard position + * @param src_img input image + * @param face_box the detection result used for indicate face in input image + * @param aligned_img output aligned image + */ + CV_WRAP virtual void alignCrop(InputArray src_img, InputArray face_box, OutputArray aligned_img) const = 0; + + /** @brief Extracting face feature from aligned image + * @param aligned_img input aligned image + * @param face_feature output face feature + */ + CV_WRAP virtual void feature(InputArray aligned_img, OutputArray face_feature) = 0; + + /** @brief Calculating the distance between two face features + * @param _face_feature1 the first input feature + * @param _face_feature2 the second input feature of the same size and the same type as _face_feature1 + * @param dis_type defining the similarity with optional values "FR_OSINE" or "FR_NORM_L2" + */ + CV_WRAP virtual double match(InputArray _face_feature1, InputArray _face_feature2, int dis_type = FaceRecognizerSF::FR_COSINE) const = 0; + + /** @brief Creates an instance of this class with given parameters + * @param model the path of the onnx model used for face recognition + * @param config the path to the config file for compability, which is not requested for ONNX models + * @param backend_id the id of backend + * @param target_id the id of target device + */ + CV_WRAP static Ptr create(const String& model, const String& config, int backend_id = 0, int target_id = 0); +}; + +} // namespace cv + +#endif diff --git a/modules/objdetect/src/face_detect.cpp b/modules/objdetect/src/face_detect.cpp new file mode 100644 index 0000000000..4095745b7e --- /dev/null +++ b/modules/objdetect/src/face_detect.cpp @@ -0,0 +1,288 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. + +#include "precomp.hpp" + +#include "opencv2/imgproc.hpp" +#include "opencv2/core.hpp" +#include "opencv2/dnn.hpp" + +#include + +namespace cv +{ + +class FaceDetectorYNImpl : public FaceDetectorYN +{ +public: + FaceDetectorYNImpl(const String& model, + const String& config, + const Size& input_size, + float score_threshold, + float nms_threshold, + int top_k, + int backend_id, + int target_id) + { + net = dnn::readNet(model, config); + CV_Assert(!net.empty()); + + net.setPreferableBackend(backend_id); + net.setPreferableTarget(target_id); + + inputW = input_size.width; + inputH = input_size.height; + + scoreThreshold = score_threshold; + nmsThreshold = nms_threshold; + topK = top_k; + + generatePriors(); + } + + void setInputSize(const Size& input_size) override + { + inputW = input_size.width; + inputH = input_size.height; + generatePriors(); + } + + Size getInputSize() override + { + Size input_size; + input_size.width = inputW; + input_size.height = inputH; + return input_size; + } + + void setScoreThreshold(float score_threshold) override + { + scoreThreshold = score_threshold; + } + + float getScoreThreshold() override + { + return scoreThreshold; + } + + void setNMSThreshold(float nms_threshold) override + { + nmsThreshold = nms_threshold; + } + + float getNMSThreshold() override + { + return nmsThreshold; + } + + void setTopK(int top_k) override + { + topK = top_k; + } + + int getTopK() override + { + return topK; + } + + int detect(InputArray input_image, OutputArray faces) override + { + // TODO: more checkings should be done? + if (input_image.empty()) + { + return 0; + } + CV_CheckEQ(input_image.size(), Size(inputW, inputH), "Size does not match. Call setInputSize(size) if input size does not match the preset size"); + + // Build blob from input image + Mat input_blob = dnn::blobFromImage(input_image); + + // Forward + std::vector output_names = { "loc", "conf", "iou" }; + std::vector output_blobs; + net.setInput(input_blob); + net.forward(output_blobs, output_names); + + // Post process + Mat results = postProcess(output_blobs); + results.convertTo(faces, CV_32FC1); + return 1; + } +private: + void generatePriors() + { + // Calculate shapes of different scales according to the shape of input image + Size feature_map_2nd = { + int(int((inputW+1)/2)/2), int(int((inputH+1)/2)/2) + }; + Size feature_map_3rd = { + int(feature_map_2nd.width/2), int(feature_map_2nd.height/2) + }; + Size feature_map_4th = { + int(feature_map_3rd.width/2), int(feature_map_3rd.height/2) + }; + Size feature_map_5th = { + int(feature_map_4th.width/2), int(feature_map_4th.height/2) + }; + Size feature_map_6th = { + int(feature_map_5th.width/2), int(feature_map_5th.height/2) + }; + + std::vector feature_map_sizes; + feature_map_sizes.push_back(feature_map_3rd); + feature_map_sizes.push_back(feature_map_4th); + feature_map_sizes.push_back(feature_map_5th); + feature_map_sizes.push_back(feature_map_6th); + + // Fixed params for generating priors + const std::vector> min_sizes = { + {10.0f, 16.0f, 24.0f}, + {32.0f, 48.0f}, + {64.0f, 96.0f}, + {128.0f, 192.0f, 256.0f} + }; + const std::vector steps = { 8, 16, 32, 64 }; + + // Generate priors + priors.clear(); + for (size_t i = 0; i < feature_map_sizes.size(); ++i) + { + Size feature_map_size = feature_map_sizes[i]; + std::vector min_size = min_sizes[i]; + + for (int _h = 0; _h < feature_map_size.height; ++_h) + { + for (int _w = 0; _w < feature_map_size.width; ++_w) + { + for (size_t j = 0; j < min_size.size(); ++j) + { + float s_kx = min_size[j] / inputW; + float s_ky = min_size[j] / inputH; + + float cx = (_w + 0.5f) * steps[i] / inputW; + float cy = (_h + 0.5f) * steps[i] / inputH; + + Rect2f prior = { cx, cy, s_kx, s_ky }; + priors.push_back(prior); + } + } + } + } + } + + Mat postProcess(const std::vector& output_blobs) + { + // Extract from output_blobs + Mat loc = output_blobs[0]; + Mat conf = output_blobs[1]; + Mat iou = output_blobs[2]; + + // Decode from deltas and priors + const std::vector variance = {0.1f, 0.2f}; + float* loc_v = (float*)(loc.data); + float* conf_v = (float*)(conf.data); + float* iou_v = (float*)(iou.data); + Mat faces; + // (tl_x, tl_y, w, h, re_x, re_y, le_x, le_y, nt_x, nt_y, rcm_x, rcm_y, lcm_x, lcm_y, score) + // 'tl': top left point of the bounding box + // 're': right eye, 'le': left eye + // 'nt': nose tip + // 'rcm': right corner of mouth, 'lcm': left corner of mouth + Mat face(1, 15, CV_32FC1); + for (size_t i = 0; i < priors.size(); ++i) { + // Get score + float clsScore = conf_v[i*2+1]; + float iouScore = iou_v[i]; + // Clamp + if (iouScore < 0.f) { + iouScore = 0.f; + } + else if (iouScore > 1.f) { + iouScore = 1.f; + } + float score = std::sqrt(clsScore * iouScore); + face.at(0, 14) = score; + + // Get bounding box + float cx = (priors[i].x + loc_v[i*14+0] * variance[0] * priors[i].width) * inputW; + float cy = (priors[i].y + loc_v[i*14+1] * variance[0] * priors[i].height) * inputH; + float w = priors[i].width * exp(loc_v[i*14+2] * variance[0]) * inputW; + float h = priors[i].height * exp(loc_v[i*14+3] * variance[1]) * inputH; + float x1 = cx - w / 2; + float y1 = cy - h / 2; + face.at(0, 0) = x1; + face.at(0, 1) = y1; + face.at(0, 2) = w; + face.at(0, 3) = h; + + // Get landmarks + face.at(0, 4) = (priors[i].x + loc_v[i*14+ 4] * variance[0] * priors[i].width) * inputW; // right eye, x + face.at(0, 5) = (priors[i].y + loc_v[i*14+ 5] * variance[0] * priors[i].height) * inputH; // right eye, y + face.at(0, 6) = (priors[i].x + loc_v[i*14+ 6] * variance[0] * priors[i].width) * inputW; // left eye, x + face.at(0, 7) = (priors[i].y + loc_v[i*14+ 7] * variance[0] * priors[i].height) * inputH; // left eye, y + face.at(0, 8) = (priors[i].x + loc_v[i*14+ 8] * variance[0] * priors[i].width) * inputW; // nose tip, x + face.at(0, 9) = (priors[i].y + loc_v[i*14+ 9] * variance[0] * priors[i].height) * inputH; // nose tip, y + face.at(0, 10) = (priors[i].x + loc_v[i*14+10] * variance[0] * priors[i].width) * inputW; // right corner of mouth, x + face.at(0, 11) = (priors[i].y + loc_v[i*14+11] * variance[0] * priors[i].height) * inputH; // right corner of mouth, y + face.at(0, 12) = (priors[i].x + loc_v[i*14+12] * variance[0] * priors[i].width) * inputW; // left corner of mouth, x + face.at(0, 13) = (priors[i].y + loc_v[i*14+13] * variance[0] * priors[i].height) * inputH; // left corner of mouth, y + + faces.push_back(face); + } + + if (faces.rows > 1) + { + // Retrieve boxes and scores + std::vector faceBoxes; + std::vector faceScores; + for (int rIdx = 0; rIdx < faces.rows; rIdx++) + { + faceBoxes.push_back(Rect2i(int(faces.at(rIdx, 0)), + int(faces.at(rIdx, 1)), + int(faces.at(rIdx, 2)), + int(faces.at(rIdx, 3)))); + faceScores.push_back(faces.at(rIdx, 14)); + } + + std::vector keepIdx; + dnn::NMSBoxes(faceBoxes, faceScores, scoreThreshold, nmsThreshold, keepIdx, 1.f, topK); + + // Get NMS results + Mat nms_faces; + for (int idx: keepIdx) + { + nms_faces.push_back(faces.row(idx)); + } + return nms_faces; + } + else + { + return faces; + } + } +private: + dnn::Net net; + + int inputW; + int inputH; + float scoreThreshold; + float nmsThreshold; + int topK; + + std::vector priors; +}; + +Ptr FaceDetectorYN::create(const String& model, + const String& config, + const Size& input_size, + const float score_threshold, + const float nms_threshold, + const int top_k, + const int backend_id, + const int target_id) +{ + return makePtr(model, config, input_size, score_threshold, nms_threshold, top_k, backend_id, target_id); +} + +} // namespace cv diff --git a/modules/objdetect/src/face_recognize.cpp b/modules/objdetect/src/face_recognize.cpp new file mode 100644 index 0000000000..6550a13b4b --- /dev/null +++ b/modules/objdetect/src/face_recognize.cpp @@ -0,0 +1,182 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. + +#include "precomp.hpp" + +#include "opencv2/dnn.hpp" + +#include + +namespace cv +{ + +class FaceRecognizerSFImpl : public FaceRecognizerSF +{ +public: + FaceRecognizerSFImpl(const String& model, const String& config, int backend_id, int target_id) + { + net = dnn::readNet(model, config); + CV_Assert(!net.empty()); + + net.setPreferableBackend(backend_id); + net.setPreferableTarget(target_id); + }; + void alignCrop(InputArray _src_img, InputArray _face_mat, OutputArray _aligned_img) const override + { + Mat face_mat = _face_mat.getMat(); + float src_point[5][2]; + for (int row = 0; row < 5; ++row) + { + for(int col = 0; col < 2; ++col) + { + src_point[row][col] = face_mat.at(0, row*2+col+4); + } + } + Mat warp_mat = getSimilarityTransformMatrix(src_point); + warpAffine(_src_img, _aligned_img, warp_mat, Size(112, 112), INTER_LINEAR); + }; + void feature(InputArray _aligned_img, OutputArray _face_feature) override + { + Mat inputBolb = dnn::blobFromImage(_aligned_img, 1, Size(112, 112), Scalar(0, 0, 0), true, false); + net.setInput(inputBolb); + net.forward(_face_feature); + }; + double match(InputArray _face_feature1, InputArray _face_feature2, int dis_type) const override + { + Mat face_feature1 = _face_feature1.getMat(), face_feature2 = _face_feature2.getMat(); + face_feature1 /= norm(face_feature1); + face_feature2 /= norm(face_feature2); + + if(dis_type == DisType::FR_COSINE){ + return sum(face_feature1.mul(face_feature2))[0]; + }else if(dis_type == DisType::FR_NORM_L2){ + return norm(face_feature1, face_feature2); + }else{ + throw std::invalid_argument("invalid parameter " + std::to_string(dis_type)); + } + + }; + +private: + Mat getSimilarityTransformMatrix(float src[5][2]) const { + float dst[5][2] = { {38.2946f, 51.6963f}, {73.5318f, 51.5014f}, {56.0252f, 71.7366f}, {41.5493f, 92.3655f}, {70.7299f, 92.2041f} }; + float avg0 = (src[0][0] + src[1][0] + src[2][0] + src[3][0] + src[4][0]) / 5; + float avg1 = (src[0][1] + src[1][1] + src[2][1] + src[3][1] + src[4][1]) / 5; + //Compute mean of src and dst. + float src_mean[2] = { avg0, avg1 }; + float dst_mean[2] = { 56.0262f, 71.9008f }; + //Subtract mean from src and dst. + float src_demean[5][2]; + for (int i = 0; i < 2; i++) + { + for (int j = 0; j < 5; j++) + { + src_demean[j][i] = src[j][i] - src_mean[i]; + } + } + float dst_demean[5][2]; + for (int i = 0; i < 2; i++) + { + for (int j = 0; j < 5; j++) + { + dst_demean[j][i] = dst[j][i] - dst_mean[i]; + } + } + double A00 = 0.0, A01 = 0.0, A10 = 0.0, A11 = 0.0; + for (int i = 0; i < 5; i++) + A00 += dst_demean[i][0] * src_demean[i][0]; + A00 = A00 / 5; + for (int i = 0; i < 5; i++) + A01 += dst_demean[i][0] * src_demean[i][1]; + A01 = A01 / 5; + for (int i = 0; i < 5; i++) + A10 += dst_demean[i][1] * src_demean[i][0]; + A10 = A10 / 5; + for (int i = 0; i < 5; i++) + A11 += dst_demean[i][1] * src_demean[i][1]; + A11 = A11 / 5; + Mat A = (Mat_(2, 2) << A00, A01, A10, A11); + double d[2] = { 1.0, 1.0 }; + double detA = A00 * A11 - A01 * A10; + if (detA < 0) + d[1] = -1; + double T[3][3] = { {1.0, 0.0, 0.0}, {0.0, 1.0, 0.0}, {0.0, 0.0, 1.0} }; + Mat s, u, vt, v; + SVD::compute(A, s, u, vt); + double smax = s.ptr(0)[0]>s.ptr(1)[0] ? s.ptr(0)[0] : s.ptr(1)[0]; + double tol = smax * 2 * FLT_MIN; + int rank = 0; + if (s.ptr(0)[0]>tol) + rank += 1; + if (s.ptr(1)[0]>tol) + rank += 1; + double arr_u[2][2] = { {u.ptr(0)[0], u.ptr(0)[1]}, {u.ptr(1)[0], u.ptr(1)[1]} }; + double arr_vt[2][2] = { {vt.ptr(0)[0], vt.ptr(0)[1]}, {vt.ptr(1)[0], vt.ptr(1)[1]} }; + double det_u = arr_u[0][0] * arr_u[1][1] - arr_u[0][1] * arr_u[1][0]; + double det_vt = arr_vt[0][0] * arr_vt[1][1] - arr_vt[0][1] * arr_vt[1][0]; + if (rank == 1) + { + if ((det_u*det_vt) > 0) + { + Mat uvt = u*vt; + T[0][0] = uvt.ptr(0)[0]; + T[0][1] = uvt.ptr(0)[1]; + T[1][0] = uvt.ptr(1)[0]; + T[1][1] = uvt.ptr(1)[1]; + } + else + { + double temp = d[1]; + d[1] = -1; + Mat D = (Mat_(2, 2) << d[0], 0.0, 0.0, d[1]); + Mat Dvt = D*vt; + Mat uDvt = u*Dvt; + T[0][0] = uDvt.ptr(0)[0]; + T[0][1] = uDvt.ptr(0)[1]; + T[1][0] = uDvt.ptr(1)[0]; + T[1][1] = uDvt.ptr(1)[1]; + d[1] = temp; + } + } + else + { + Mat D = (Mat_(2, 2) << d[0], 0.0, 0.0, d[1]); + Mat Dvt = D*vt; + Mat uDvt = u*Dvt; + T[0][0] = uDvt.ptr(0)[0]; + T[0][1] = uDvt.ptr(0)[1]; + T[1][0] = uDvt.ptr(1)[0]; + T[1][1] = uDvt.ptr(1)[1]; + } + double var1 = 0.0; + for (int i = 0; i < 5; i++) + var1 += src_demean[i][0] * src_demean[i][0]; + var1 = var1 / 5; + double var2 = 0.0; + for (int i = 0; i < 5; i++) + var2 += src_demean[i][1] * src_demean[i][1]; + var2 = var2 / 5; + double scale = 1.0 / (var1 + var2)* (s.ptr(0)[0] * d[0] + s.ptr(1)[0] * d[1]); + double TS[2]; + TS[0] = T[0][0] * src_mean[0] + T[0][1] * src_mean[1]; + TS[1] = T[1][0] * src_mean[0] + T[1][1] * src_mean[1]; + T[0][2] = dst_mean[0] - scale*TS[0]; + T[1][2] = dst_mean[1] - scale*TS[1]; + T[0][0] *= scale; + T[0][1] *= scale; + T[1][0] *= scale; + T[1][1] *= scale; + Mat transform_mat = (Mat_(2, 3) << T[0][0], T[0][1], T[0][2], T[1][0], T[1][1], T[1][2]); + return transform_mat; + } +private: + dnn::Net net; +}; + +Ptr FaceRecognizerSF::create(const String& model, const String& config, int backend_id, int target_id) +{ + return makePtr(model, config, backend_id, target_id); +} + +} // namespace cv diff --git a/modules/objdetect/test/test_face.cpp b/modules/objdetect/test/test_face.cpp new file mode 100644 index 0000000000..2e944c50df --- /dev/null +++ b/modules/objdetect/test/test_face.cpp @@ -0,0 +1,219 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. + +#include "test_precomp.hpp" + +namespace opencv_test { namespace { + +// label format: +// image_name +// num_face +// face_1 +// face_.. +// face_num +std::map blobFromTXT(const std::string& path, int numCoords) +{ + std::ifstream ifs(path.c_str()); + CV_Assert(ifs.is_open()); + + std::map gt; + + Mat faces; + int faceNum = -1; + int faceCount = 0; + for (std::string line, key; getline(ifs, line); ) + { + std::istringstream iss(line); + if (line.find(".png") != std::string::npos) + { + // Get filename + iss >> key; + } + else if (line.find(" ") == std::string::npos) + { + // Get the number of faces + iss >> faceNum; + } + else + { + // Get faces + Mat face(1, numCoords, CV_32FC1); + for (int j = 0; j < numCoords; j++) + { + iss >> face.at(0, j); + } + faces.push_back(face); + faceCount++; + } + + if (faceCount == faceNum) + { + // Store faces + gt[key] = faces; + + faces.release(); + faceNum = -1; + faceCount = 0; + } + } + + return gt; +} + +TEST(Objdetect_face_detection, regression) +{ + // Pre-set params + float scoreThreshold = 0.7f; + float matchThreshold = 0.9f; + float l2disThreshold = 5.0f; + int numLM = 5; + int numCoords = 4 + 2 * numLM; + + // Load ground truth labels + std::map gt = blobFromTXT(findDataFile("dnn_face/detection/cascades_labels.txt"), numCoords); + // for (auto item: gt) + // { + // std::cout << item.first << " " << item.second.size() << std::endl; + // } + + // Initialize detector + std::string model = findDataFile("dnn/onnx/models/yunet-202109.onnx", false); + Ptr faceDetector = FaceDetectorYN::create(model, "", Size(300, 300)); + faceDetector->setScoreThreshold(0.7f); + + // Detect and match + for (auto item: gt) + { + std::string imagePath = findDataFile("cascadeandhog/images/" + item.first); + Mat image = imread(imagePath); + + // Set input size + faceDetector->setInputSize(image.size()); + + // Run detection + Mat faces; + faceDetector->detect(image, faces); + // std::cout << item.first << " " << item.second.rows << " " << faces.rows << std::endl; + + // Match bboxes and landmarks + std::vector matchedItem(item.second.rows, false); + for (int i = 0; i < faces.rows; i++) + { + if (faces.at(i, numCoords) < scoreThreshold) + continue; + + bool boxMatched = false; + std::vector lmMatched(numLM, false); + cv::Rect2f resBox(faces.at(i, 0), faces.at(i, 1), faces.at(i, 2), faces.at(i, 3)); + for (int j = 0; j < item.second.rows && !boxMatched; j++) + { + if (matchedItem[j]) + continue; + + // Retrieve bbox and compare IoU + cv::Rect2f gtBox(item.second.at(j, 0), item.second.at(j, 1), item.second.at(j, 2), item.second.at(j, 3)); + double interArea = (resBox & gtBox).area(); + double iou = interArea / (resBox.area() + gtBox.area() - interArea); + if (iou >= matchThreshold) + { + boxMatched = true; + matchedItem[j] = true; + } + + // Match landmarks if bbox is matched + if (!boxMatched) + continue; + for (int lmIdx = 0; lmIdx < numLM; lmIdx++) + { + float gtX = item.second.at(j, 4 + 2 * lmIdx); + float gtY = item.second.at(j, 4 + 2 * lmIdx + 1); + float resX = faces.at(i, 4 + 2 * lmIdx); + float resY = faces.at(i, 4 + 2 * lmIdx + 1); + float l2dis = cv::sqrt((gtX - resX) * (gtX - resX) + (gtY - resY) * (gtY - resY)); + + if (l2dis <= l2disThreshold) + { + lmMatched[lmIdx] = true; + } + } + } + EXPECT_TRUE(boxMatched) << "In image " << item.first << ", cannot match resBox " << resBox << " with any ground truth."; + if (boxMatched) + { + EXPECT_TRUE(std::all_of(lmMatched.begin(), lmMatched.end(), [](bool v) { return v; })) << "In image " << item.first << ", resBox " << resBox << " matched but its landmarks failed to match."; + } + } + } +} + +TEST(Objdetect_face_recognition, regression) +{ + // Pre-set params + float score_thresh = 0.9f; + float nms_thresh = 0.3f; + double cosine_similar_thresh = 0.363; + double l2norm_similar_thresh = 1.128; + + // Load ground truth labels + std::ifstream ifs(findDataFile("dnn_face/recognition/cascades_label.txt").c_str()); + CV_Assert(ifs.is_open()); + + std::set fSet; + std::map featureMap; + std::map, int> gtMap; + + + for (std::string line, key; getline(ifs, line);) + { + std::string fname1, fname2; + int label; + std::istringstream iss(line); + iss>>fname1>>fname2>>label; + // std::cout< faceDetector = FaceDetectorYN::create(detect_model, "", Size(150, 150), score_thresh, nms_thresh); + + std::string recog_model = findDataFile("dnn/onnx/models/face_recognizer_fast.onnx", false); + Ptr faceRecognizer = FaceRecognizerSF::create(recog_model, ""); + + // Detect and match + for (auto fname: fSet) + { + std::string imagePath = findDataFile("dnn_face/recognition/" + fname); + Mat image = imread(imagePath); + + Mat faces; + faceDetector->detect(image, faces); + + Mat aligned_face; + faceRecognizer->alignCrop(image, faces.row(0), aligned_face); + + Mat feature; + faceRecognizer->feature(aligned_face, feature); + + featureMap[fname] = feature.clone(); + } + + for (auto item: gtMap) + { + Mat feature1 = featureMap[item.first.first]; + Mat feature2 = featureMap[item.first.second]; + int label = item.second; + + double cos_score = faceRecognizer->match(feature1, feature2, FaceRecognizerSF::DisType::FR_COSINE); + double L2_score = faceRecognizer->match(feature1, feature2, FaceRecognizerSF::DisType::FR_NORM_L2); + + EXPECT_TRUE(label == 0 ? cos_score <= cosine_similar_thresh : cos_score > cosine_similar_thresh) << "Cosine match result of images " << item.first.first << " and " << item.first.second << " is different from ground truth (score: "<< cos_score <<";Thresh: "<< cosine_similar_thresh <<")."; + EXPECT_TRUE(label == 0 ? L2_score > l2norm_similar_thresh : L2_score <= l2norm_similar_thresh) << "L2norm match result of images " << item.first.first << " and " << item.first.second << " is different from ground truth (score: "<< L2_score <<";Thresh: "<< l2norm_similar_thresh <<")."; + } +} + +}} // namespace diff --git a/modules/objdetect/test/test_main.cpp b/modules/objdetect/test/test_main.cpp index 93e4d2860e..4031f0522b 100644 --- a/modules/objdetect/test/test_main.cpp +++ b/modules/objdetect/test/test_main.cpp @@ -7,4 +7,19 @@ #include #endif -CV_TEST_MAIN("cv") +static +void initTests() +{ +#ifdef HAVE_OPENCV_DNN + const char* extraTestDataPath = +#ifdef WINRT + NULL; +#else + getenv("OPENCV_DNN_TEST_DATA_PATH"); +#endif + if (extraTestDataPath) + cvtest::addDataSearchPath(extraTestDataPath); +#endif // HAVE_OPENCV_DNN +} + +CV_TEST_MAIN("cv", initTests()) diff --git a/modules/ts/include/opencv2/ts.hpp b/modules/ts/include/opencv2/ts.hpp index 394bc6e0fa..2e7a241d8e 100644 --- a/modules/ts/include/opencv2/ts.hpp +++ b/modules/ts/include/opencv2/ts.hpp @@ -37,6 +37,7 @@ #include #include #include +#include #ifndef OPENCV_32BIT_CONFIGURATION diff --git a/samples/dnn/CMakeLists.txt b/samples/dnn/CMakeLists.txt index 209fbb586c..9a1aeed339 100644 --- a/samples/dnn/CMakeLists.txt +++ b/samples/dnn/CMakeLists.txt @@ -4,6 +4,7 @@ set(OPENCV_DNN_SAMPLES_REQUIRED_DEPS opencv_core opencv_imgproc opencv_dnn + opencv_objdetect opencv_video opencv_imgcodecs opencv_videoio diff --git a/samples/dnn/face_detect.cpp b/samples/dnn/face_detect.cpp new file mode 100644 index 0000000000..8d91a10968 --- /dev/null +++ b/samples/dnn/face_detect.cpp @@ -0,0 +1,132 @@ +#include +#include +#include +#include + +#include + +using namespace cv; +using namespace std; + +static Mat visualize(Mat input, Mat faces, int thickness=2) +{ + Mat output = input.clone(); + for (int i = 0; i < faces.rows; i++) + { + // Print results + cout << "Face " << i + << ", top-left coordinates: (" << faces.at(i, 0) << ", " << faces.at(i, 1) << "), " + << "box width: " << faces.at(i, 2) << ", box height: " << faces.at(i, 3) << ", " + << "score: " << faces.at(i, 14) << "\n"; + + // Draw bounding box + rectangle(output, Rect2i(int(faces.at(i, 0)), int(faces.at(i, 1)), int(faces.at(i, 2)), int(faces.at(i, 3))), Scalar(0, 255, 0), thickness); + // Draw landmarks + circle(output, Point2i(int(faces.at(i, 4)), int(faces.at(i, 5))), 2, Scalar(255, 0, 0), thickness); + circle(output, Point2i(int(faces.at(i, 6)), int(faces.at(i, 7))), 2, Scalar( 0, 0, 255), thickness); + circle(output, Point2i(int(faces.at(i, 8)), int(faces.at(i, 9))), 2, Scalar( 0, 255, 0), thickness); + circle(output, Point2i(int(faces.at(i, 10)), int(faces.at(i, 11))), 2, Scalar(255, 0, 255), thickness); + circle(output, Point2i(int(faces.at(i, 12)), int(faces.at(i, 13))), 2, Scalar( 0, 255, 255), thickness); + } + return output; +} + +int main(int argc, char ** argv) +{ + CommandLineParser parser(argc, argv, + "{help h | | Print this message.}" + "{input i | | Path to the input image. Omit for detecting on default camera.}" + "{model m | yunet.onnx | Path to the model. Download yunet.onnx in https://github.com/ShiqiYu/libfacedetection.train/tree/master/tasks/task1/onnx.}" + "{score_threshold | 0.9 | Filter out faces of score < score_threshold.}" + "{nms_threshold | 0.3 | Suppress bounding boxes of iou >= nms_threshold.}" + "{top_k | 5000 | Keep top_k bounding boxes before NMS.}" + "{save s | false | Set true to save results. This flag is invalid when using camera.}" + "{vis v | true | Set true to open a window for result visualization. This flag is invalid when using camera.}" + ); + if (argc == 1 || parser.has("help")) + { + parser.printMessage(); + return -1; + } + + String modelPath = parser.get("model"); + + float scoreThreshold = parser.get("score_threshold"); + float nmsThreshold = parser.get("nms_threshold"); + int topK = parser.get("top_k"); + + bool save = parser.get("save"); + bool vis = parser.get("vis"); + + // Initialize FaceDetectorYN + Ptr detector = FaceDetectorYN::create(modelPath, "", Size(320, 320), scoreThreshold, nmsThreshold, topK); + + // If input is an image + if (parser.has("input")) + { + String input = parser.get("input"); + Mat image = imread(input); + + // Set input size before inference + detector->setInputSize(image.size()); + + // Inference + Mat faces; + detector->detect(image, faces); + + // Draw results on the input image + Mat result = visualize(image, faces); + + // Save results if save is true + if(save) + { + cout << "Results saved to result.jpg\n"; + imwrite("result.jpg", result); + } + + // Visualize results + if (vis) + { + namedWindow(input, WINDOW_AUTOSIZE); + imshow(input, result); + waitKey(0); + } + } + else + { + int deviceId = 0; + VideoCapture cap; + cap.open(deviceId, CAP_ANY); + int frameWidth = int(cap.get(CAP_PROP_FRAME_WIDTH)); + int frameHeight = int(cap.get(CAP_PROP_FRAME_HEIGHT)); + detector->setInputSize(Size(frameWidth, frameHeight)); + + Mat frame; + TickMeter tm; + String msg = "FPS: "; + while(waitKey(1) < 0) // Press any key to exit + { + // Get frame + if (!cap.read(frame)) + { + cerr << "No frames grabbed!\n"; + break; + } + + // Inference + Mat faces; + tm.start(); + detector->detect(frame, faces); + tm.stop(); + + // Draw results on the input image + Mat result = visualize(frame, faces); + putText(result, msg + to_string(tm.getFPS()), Point(0, 15), FONT_HERSHEY_SIMPLEX, 0.5, Scalar(0, 255, 0)); + + // Visualize results + imshow("Live", result); + + tm.reset(); + } + } +} \ No newline at end of file diff --git a/samples/dnn/face_detect.py b/samples/dnn/face_detect.py new file mode 100644 index 0000000000..65069d6590 --- /dev/null +++ b/samples/dnn/face_detect.py @@ -0,0 +1,101 @@ +import argparse + +import numpy as np +import cv2 as cv + +def str2bool(v): + if v.lower() in ['on', 'yes', 'true', 'y', 't']: + return True + elif v.lower() in ['off', 'no', 'false', 'n', 'f']: + return False + else: + raise NotImplementedError + +parser = argparse.ArgumentParser() +parser.add_argument('--input', '-i', type=str, help='Path to the input image.') +parser.add_argument('--model', '-m', type=str, default='yunet.onnx', help='Path to the model. Download the model at https://github.com/ShiqiYu/libfacedetection.train/tree/master/tasks/task1/onnx.') +parser.add_argument('--score_threshold', type=float, default=0.9, help='Filtering out faces of score < score_threshold.') +parser.add_argument('--nms_threshold', type=float, default=0.3, help='Suppress bounding boxes of iou >= nms_threshold.') +parser.add_argument('--top_k', type=int, default=5000, help='Keep top_k bounding boxes before NMS.') +parser.add_argument('--save', '-s', type=str2bool, default=False, help='Set true to save results. This flag is invalid when using camera.') +parser.add_argument('--vis', '-v', type=str2bool, default=True, help='Set true to open a window for result visualization. This flag is invalid when using camera.') +args = parser.parse_args() + +def visualize(input, faces, thickness=2): + output = input.copy() + if faces[1] is not None: + for idx, face in enumerate(faces[1]): + print('Face {}, top-left coordinates: ({:.0f}, {:.0f}), box width: {:.0f}, box height {:.0f}, score: {:.2f}'.format(idx, face[0], face[1], face[2], face[3], face[-1])) + + coords = face[:-1].astype(np.int32) + cv.rectangle(output, (coords[0], coords[1]), (coords[0]+coords[2], coords[1]+coords[3]), (0, 255, 0), 2) + cv.circle(output, (coords[4], coords[5]), 2, (255, 0, 0), 2) + cv.circle(output, (coords[6], coords[7]), 2, (0, 0, 255), 2) + cv.circle(output, (coords[8], coords[9]), 2, (0, 255, 0), 2) + cv.circle(output, (coords[10], coords[11]), 2, (255, 0, 255), 2) + cv.circle(output, (coords[12], coords[13]), 2, (0, 255, 255), 2) + return output + +if __name__ == '__main__': + + # Instantiate FaceDetectorYN + detector = cv.FaceDetectorYN.create( + args.model, + "", + (320, 320), + args.score_threshold, + args.nms_threshold, + args.top_k + ) + + # If input is an image + if args.input is not None: + image = cv.imread(args.input) + + # Set input size before inference + detector.setInputSize((image.shape[1], image.shape[0])) + + # Inference + faces = detector.detect(image) + + # Draw results on the input image + result = visualize(image, faces) + + # Save results if save is true + if args.save: + print('Resutls saved to result.jpg\n') + cv.imwrite('result.jpg', result) + + # Visualize results in a new window + if args.vis: + cv.namedWindow(args.input, cv.WINDOW_AUTOSIZE) + cv.imshow(args.input, result) + cv.waitKey(0) + else: # Omit input to call default camera + deviceId = 0 + cap = cv.VideoCapture(deviceId) + frameWidth = int(cap.get(cv.CAP_PROP_FRAME_WIDTH)) + frameHeight = int(cap.get(cv.CAP_PROP_FRAME_HEIGHT)) + detector.setInputSize([frameWidth, frameHeight]) + + tm = cv.TickMeter() + while cv.waitKey(1) < 0: + hasFrame, frame = cap.read() + if not hasFrame: + print('No frames grabbed!') + break + + # Inference + tm.start() + faces = detector.detect(frame) # faces is a tuple + tm.stop() + + # Draw results on the input image + frame = visualize(frame, faces) + + cv.putText(frame, 'FPS: {}'.format(tm.getFPS()), (0, 15), cv.FONT_HERSHEY_SIMPLEX, 0.5, (0, 255, 0)) + + # Visualize results in a new Window + cv.imshow('Live', frame) + + tm.reset() \ No newline at end of file diff --git a/samples/dnn/face_match.cpp b/samples/dnn/face_match.cpp new file mode 100644 index 0000000000..f24134b890 --- /dev/null +++ b/samples/dnn/face_match.cpp @@ -0,0 +1,103 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. + +#include "opencv2/dnn.hpp" +#include "opencv2/imgproc.hpp" +#include "opencv2/highgui.hpp" + +#include + +#include "opencv2/objdetect.hpp" + + +using namespace cv; +using namespace std; + + +int main(int argc, char ** argv) +{ + if (argc != 5) + { + std::cerr << "Usage " << argv[0] << ": " + << " " + << " " + << "" + << "\n"; + return -1; + } + + String det_onnx_path = argv[1]; + String reg_onnx_path = argv[2]; + String image1_path = argv[3]; + String image2_path = argv[4]; + std::cout< faceDetector; + + faceDetector = FaceDetectorYN::create(det_onnx_path, "", image1.size(), score_thresh, nms_thresh, top_k); + Mat faces_1; + faceDetector->detect(image1, faces_1); + if (faces_1.rows < 1) + { + std::cerr << "Cannot find a face in " << image1_path << "\n"; + return -1; + } + + faceDetector = FaceDetectorYN::create(det_onnx_path, "", image2.size(), score_thresh, nms_thresh, top_k); + Mat faces_2; + faceDetector->detect(image2, faces_2); + if (faces_2.rows < 1) + { + std::cerr << "Cannot find a face in " << image2_path << "\n"; + return -1; + } + + // Initialize FaceRecognizerSF + Ptr faceRecognizer = FaceRecognizerSF::create(reg_onnx_path, ""); + + + Mat aligned_face1, aligned_face2; + faceRecognizer->alignCrop(image1, faces_1.row(0), aligned_face1); + faceRecognizer->alignCrop(image2, faces_2.row(0), aligned_face2); + + Mat feature1, feature2; + faceRecognizer->feature(aligned_face1, feature1); + feature1 = feature1.clone(); + faceRecognizer->feature(aligned_face2, feature2); + feature2 = feature2.clone(); + + double cos_score = faceRecognizer->match(feature1, feature2, FaceRecognizerSF::DisType::FR_COSINE); + double L2_score = faceRecognizer->match(feature1, feature2, FaceRecognizerSF::DisType::FR_NORM_L2); + + if(cos_score >= cosine_similar_thresh) + { + std::cout << "They have the same identity;"; + } + else + { + std::cout << "They have different identities;"; + } + std::cout << " Cosine Similarity: " << cos_score << ", threshold: " << cosine_similar_thresh << ". (higher value means higher similarity, max 1.0)\n"; + + if(L2_score <= l2norm_similar_thresh) + { + std::cout << "They have the same identity;"; + } + else + { + std::cout << "They have different identities."; + } + std::cout << " NormL2 Distance: " << L2_score << ", threshold: " << l2norm_similar_thresh << ". (lower value means higher similarity, min 0.0)\n"; + + return 0; +} diff --git a/samples/dnn/face_match.py b/samples/dnn/face_match.py new file mode 100644 index 0000000000..b36c9f6367 --- /dev/null +++ b/samples/dnn/face_match.py @@ -0,0 +1,57 @@ +import argparse + +import numpy as np +import cv2 as cv + +parser = argparse.ArgumentParser() +parser.add_argument('--input1', '-i1', type=str, help='Path to the input image1.') +parser.add_argument('--input2', '-i2', type=str, help='Path to the input image2.') +parser.add_argument('--face_detection_model', '-fd', type=str, help='Path to the face detection model. Download the model at https://github.com/ShiqiYu/libfacedetection.train/tree/master/tasks/task1/onnx.') +parser.add_argument('--face_recognition_model', '-fr', type=str, help='Path to the face recognition model. Download the model at https://drive.google.com/file/d/1ClK9WiB492c5OZFKveF3XiHCejoOxINW/view.') +args = parser.parse_args() + +# Read the input image +img1 = cv.imread(args.input1) +img2 = cv.imread(args.input2) + +# Instantiate face detector and recognizer +detector = cv.FaceDetectorYN.create( + args.face_detection_model, + "", + (img1.shape[1], img1.shape[0]) +) +recognizer = cv.FaceRecognizerSF.create( + args.face_recognition_model, + "" +) + +# Detect face +detector.setInputSize((img1.shape[1], img1.shape[0])) +face1 = detector.detect(img1) +detector.setInputSize((img2.shape[1], img2.shape[0])) +face2 = detector.detect(img2) +assert face1[1].shape[0] > 0, 'Cannot find a face in {}'.format(args.input1) +assert face2[1].shape[0] > 0, 'Cannot find a face in {}'.format(args.input2) + +# Align faces +face1_align = recognizer.alignCrop(img1, face1[1][0]) +face2_align = recognizer.alignCrop(img2, face2[1][0]) + +# Extract features +face1_feature = recognizer.faceFeature(face1_align) +face2_feature = recognizer.faceFeature(face2_align) + +# Calculate distance (0: cosine, 1: L2) +cosine_similarity_threshold = 0.363 +cosine_score = recognizer.faceMatch(face1_feature, face2_feature, 0) +msg = 'different identities' +if cosine_score >= cosine_similarity_threshold: + msg = 'the same identity' +print('They have {}. Cosine Similarity: {}, threshold: {} (higher value means higher similarity, max 1.0).'.format(msg, cosine_score, cosine_similarity_threshold)) + +l2_similarity_threshold = 1.128 +l2_score = recognizer.faceMatch(face1_feature, face2_feature, 1) +msg = 'different identities' +if l2_score <= l2_similarity_threshold: + msg = 'the same identity' +print('They have {}. NormL2 Distance: {}, threshold: {} (lower value means higher similarity, min 0.0).'.format(msg, l2_score, l2_similarity_threshold)) \ No newline at end of file diff --git a/samples/dnn/results/audrybt1.jpg b/samples/dnn/results/audrybt1.jpg new file mode 100644 index 0000000000000000000000000000000000000000..5829f82309cf904f7eddbe42ae555798ab3acff6 GIT binary patch literal 47680 zcmbTdbx<8m^foxSTae&x!GgO5f+x7U`^DWQxI4iK?(QVGh2VaX;4T-xL6`UUeYN|~ z*4FO!RCmqP^h}+n`qVl7oOAkh`E?V(mY0^324G+S0OsuiURME002b!I_TLEmUxSDH zZ$yBHhl58%Kt%kXj)a1Yh=hWKh=`1ajDq@Kd%HtNLq-4Z;lH2!uTNnS;NTEYkr0vo zXUhMl;k6gQL5A6dA%ugW24Hbu;Ba7G2LXyVIT8Lxyf9X$i%2PSSFUOs*SLCMcj(lWAg@@ncDnp)aAx@P7UmR8m_wytjO9-dy_KB3>j z!XqN1qLWin)6z3Czi0g@DlRE4E3c?*Y-(<4ZENr7>>n5$8Xg%P`#n21zp%KpyaL|Z z-r3#TKR7%(zP!4=xxKrG{(JZj7YqRRzhJ$M|F^hs-f+SGw+2Z6;evtneCu#H@CZ~K zi0?kBAelJhQga3)<4GhHHuRy;aH&G@Ou=q~?*_CLt}-vJBx|Ap-T1onU7 z0t4u9FmHqnHN1r7kChUYRr zJC59oi{?Z6B)?HsVp8OyF(Bu9s^gfs?w@J(K&$yhZyd>O3TI$yaX=xhxiUatG>OfaTrvTpU1c>$?bM6(GV_ z;&@k${MdVt=sfSUKp1En7>R4kmsrdT!Y|ZL3+sT@(T@Li8Jxe^-6nrI860CE_;CIE zS9_?R<^<*nk&JOsDa?_&RhJh&nMS~Ty&z9HMwpYL&gIA@tY|aLg|5{Z?)GD^XGyy~ z->AtG%(ImCD-d?EWAxzp3iL>HXZNcXs8oyzeYw&DSFHDp9zkxpkgs% zAqI&szr62>^9Q{G7o){uWZjO#7;Qc0Qr>eX4#-zqgd;+o2Tx2rcX8v2DWm@weK0wN zrjg15`|mI7o`qh4zlAbS9|K8zR^M)u486d+zA&mH6S311-#3zNbMwge3K(`DJeuxu7*~xFkzU@E%9XFRdW;1r%EI*+9f*@T4%V$A zykL7VD?=kLDTV|MQ;Lo>Cq$la*Ys2gM>K_R>L$d%xG$K0AzGnN7JOkZa8P(?9{4>+ zuIjDm2yPo3bB6*5Nf;Ze$Djv2uy3FTqOSDLPZK&&{xV{O>0Utt$@u1|DTXal{hhc5 zsth$13!z8P_WHLFb;tqnJl%qT$F~y_!g?ah@>V5+Na7^CbZGKV`{v2I8Vl>?fP-JM zyFpkRRTNNx@tuorPwxw!bfE)}#CE!Yq3$;-CUwh9<75Np#u6cwTF?5KYXZZFit^7I zj~@g3T*NHvz~CF}zi7YMj>ap|q;i{U6qI~zQ1eiXxY;0OK+?z`p(I0$O*F@EtuF|W z`_t8(R{%p#q3B4?#<6(WR+DLFZJ*MBCK02t9Q7hNd{mgM1yc6(KKl-T_7{ml>B~xf zQ(%Mx$Rv52(NEvat3JJy`*T|^KD(LzkI>+8!IW53;Q2%ej5M6GDdlOid>&IFqL!VY&Pj#z@FC=O# zEj$V-$a(4wjc<8SB53stcwrM;ep8chQ9lIikc~kzgo$lB%yNj2Ex92)h z;$1yy6Z`vv=-}I!44R+<(zs@rBcnIvv-{u5rvcL2=}C}E$=2)g(xdbaS(h`E5e^ZP z-z->MrLv0j82|8WTNib!@PXTt%>p^pH*@3<$LPo}XK|9wgfoRp(M4|p42g};@!Qz2 zI&3>6i72c037W!zp4TZF&t(Dh+D5$<6%sQ_A7oHWYscncVPkV}QqEGQiLcR~H7_)} zyj3ky!I>{3d*T{}psp(@5lJ6d=~Hvwt;CZ0{KodiA#KJ6XwYOkvid6yN6;*@71G{@ zw)0#K;4@kAsQDybAO4BH1w<78$7jb0dchA9gWjUl*qM+!}6z};bw^(zp?^{2vOTiaYRwt4)wux^SyO6~jHo zf#Z&yC|{*q({dVB_FDTy>ATXSH0v!=Sw#2;JykX9_ZpG`pP1<5{H;X)@inJlV^43HOh zUh!ie_X61H8qe|p$5d6K;JyL3lZVTPxuMynTH3FrBFj@JFM;wvR&=#H;WQjGpp6N0 zG*&rYp@_0h3Fqn}Eu$FpsEn9q0o^dr#X>l~wr14*$K4g)sIs9#Vo!0u$oz>7w5#0j zKvdtclP|9(3{N`RJ|ZA&8YGrTfeUS9?!kk0DML%_$_>53?=D+^j^lAmP7Yh8;^f3< zJFngJ4kaMj1>(?mbY3O|H6PX$&h3c$2gPOT6-re8scLgF>}rU8b~apj1rRzRlfj!) z&<{&PtP}&Hq!DsW*&1yfcG@&BhH@ZNZ~Bp{(OH3`ai67(fIk$trO*#MHC)?MA>Z0M z!@>)a6bcpWNeyW*)x#(_^FHa{!}S0Zp902>AnJ}##(IE|(OUh5j?{psb7QN&-8 z=2rmS&lQr?Hn-c<7IJ*I$Ud|BS;uc|eB|l?CTdKK4~eYD_7&J%&(+d+1;Uw^{D?dI z=DWqVE+RW?@ zXMQiv)wg;1x5lvg0#CoCHi2fufzB_a0r$TC)5uc7pJ-tF%%OQ4Bz-O0^{YMEtEOm$ z7dEb16f{1(*MB3x4}U9ANt|AM--*yIO0jg$?qRb;XbbMKl000t$z_FQU7Y=OV8jsYiM?9NMc(c zH{)Lo|Cn2BhQ3i6M8;cYg=;e++maHy;&uRP#nk4e-P+)j!yxOunQ(I~UH zv9T}IWv#kSi$WTl`yFO9H8DW4iwLQKRFid^sCb^*Kv#P&`T-x#WM!X*rs(&;FZo(6 z;8bABnj6DRS?AYZva~eM10?G5Xyv0#3LJzk0|@8jp$7kGZ+YdN9B$0kSD;~+(<<7AvUEyDLDT7L5cdJ zo4@7ate$7Oo>k8(Vfb`al~j9UP|eJIo47cY+=H?x>x`Q}_3?;utDUz=dltJntpd00 z!EWDHD;0cz88^(duC0sBofg480l7ij3f|y7R!pPvs84!e zcb&J?&6$C{KF%G5$6L&uSUr)J%z4(<^V@s#MnKrD03%yOe&H# z1ot%8o)h>EiLoYsF8S=wkFIuUDY}sFG-qh{p6f(ZBW$w4)8Vd3Z~XCy#zG@CoT9g~ z&mTtMzWOi5VW&k#&}7EPD2H2xXJR#H*Sp3syz>Zje6O_*B>01*Jnt&rj15vq@k8Qe zc#{8Jw!a{kdtDc{C0f(#JL)|dEZU~y-+bg9cUOMkw$cp;!=R* z2(B+|6xMxZnV@z0lKf9xXgJb?8E(4DZ~2xd*fQDBEB#_9Yj;@eqkB)-b|Qh-7=QXE z8#{_wGu$MM$RXUUAF^ra&_m483z;vW14vbe{L9CzU6{n8L}2rP^zOpIVUv)}n}%ls zDx)3luhVSf7`8CaTPjBzr-)A!zLpo1gByK>>d_ci9^j>&}A`UOe_*P6(K^!%i z+Z*h2dw(D{o4~0U;5yoOP(z0YsO9O<0zy^*OkS^=bpad$;RI6`1?)^&&@mtvcIr>9`Ud{71LBA+2+SC3Y*@h1%IWN!sVF$ihw<5KKlV8eY-pbZ6- z#d9za=gG3UHKYGQSXbpyWFV1^f%t1EPQrQRJGs$gjjvXO7W11-u(8JeYvytHR&<8U&U16d_ zUh+Kq{&|>1PdJ~~_~6;fFN2R<6>}_@*+BTKjiUO`WpAi+dSITMJ>SO$@OUvuE%8{T z(=`ezR(K=q#<#0$|3Re2^tgwk#GQMyz;BcDC>q{Jm^os5t!{3V6g7Lo(D~a9fvZbJ zii4?EJt=R1HMII{0F)>~V6yJz|GZEIAC6Z`&XH$mpE|i*bk{ z(SD~k@+2nQJLCfqJPke;Wq|yC9@}U zmi?&o()S8raWjj(@gRx-e6JXznb|(0FYU8;os&^H)CT^w0t(Kd9pRsdXtmY?Au?|) z?FkEl)Jq8Ju}XlSc(V$qURjePgNuUx&ueo^6sxRBQAs6Ps-)5lYzr<0p&>pCVyiSS zC`XSU0~Yfe?;~Fby!Zyd40mh1oL>u%@2b&T@5f1yi!D{rVC9e);BmG8LgYy%jQW4U zKn);|TC;3hIetV9c73#;>ADxEj~X&}UrzQldxf784(*1jQZ%+|*?&_-3~w_@G@nR2 zR~eSbwh-ec-Y}FjNgVM#!rnU>;Qk@nUC_F$_6|{4I zCC?K_dtJL2;y*0l08CBNPBr}r`kU?#v%Na6fR9eS3{1qy)5&ehJ+=IwOOy7mA4;2J zxhsALC|hNB62w!Z@6@4=DZ`M3j*4caKSjeldhhQ<#!pR&-m z*7sc34DYKae)44hyU%&1MlOf?C`ne9&?#zYCpL(CEJz<}Q@(t3Bs^j)XK8T6`HiVs zz44zf_63u#6++|?U~ObGW2aEDJOJChxJj$3KWV0B*9T8t!sneSyjk8?{md}9%xfW4Z-H@LsE zkyx{+jl=y>A!#`5`dnflLn1?TGSx?rAGlr4L(;_`SF(PZ-g+h+zby_Y?=;?@W?ydX zz4M%;S$#X$GDkIMq?o#EfMb)ELh+c=54i%Fm0Xn&a48U%zM^h`bs~$ks(7eL(ep`odWvp=v8=k4!DgF}! z5TX$QZ&o;D@E!D#`>-~TC~FTSudh?NV^4|CR*gJLnOppD#mNY#(L8Q#Pt2;M#!h=& z<0$OJE7xZl>^zsVLpFNISZ(y~nA0u^{Hg4#FIBj8SXCqPdL%y+VMqyd(#N}U%ZKBV_fo^6;0xn>3F}ox$MEfR6NYrdLoF&dHdf`z zssCvPkb2NFcK7psSHhr;A>W#39HaECygGiNm@iC)3VqI+b&KbJl5=g(_UDBAULxKx zAVBf)H;Tb#9wM{WpeXg?MKhU@9^^h$-BwK($Fbt?Fb&5!4C4ImzgXnW8=`|2z`@IG zFk{iv$xCx!V8UfanQVA#rN9$HBB>3qHbH`gcPC@ z#8+c|--|8Ov&6FUb|>a<0YE%izaoXo9mqjJu#I zxSU||cQSZiI0W_kSE9F= zRb}82`cwENUw;qollpUg0NoKePQBiEwj;icXUb8NZGsK{&4)spm9Sm?#NylDQ|A7c zk8eJ?S!^IM#;5F?u+5`@pBSWNKcRxu+R26s!uGH&D#6sz%R-f79$p*I`2N*US?*~HMr}v~GDROx?$`KGOJelV z4C@3!^oRm|1vK7xSw8X!I6Rm)!>k6XyQDD-awxqfXS9eFEZrV#O#D5bl7lU{6cpH} z`HS&M5o%U**a+3@WtTB=`(u8oSyVFQjnjUden^j*R4-+A&8BVTxMnJvdie?roqg=> z!7l6IeFg0Ez%!9mP+VVx~mLncr#5L+*v5?}&|Yeff)2ohGnMpVO8v@JjtDwVgr$vgOEtii=&*Xr+80oKP$wtOEy9as9qN17}i zt@n+N+GNWAT+CgRFPL0&%<5=RFw=aJ#tlAIn z_W_~W98u|EE7(7-7>q}f#o>`G*-ADu!LCLdOo5Ja&1Y`D89Y*s=qtI+Vd9UG#KG1W zf?_()Dr+5VHMkl~66&C=L`yI-9`D8t5*_~gwWBdX-7;BWQ7;mev-g;GnQk+;E*&H1 zIyfiK_T{^Gl>*47d_#t3XnZ{*a{T(7a3yo86$M=_p{w^ogSGkK&z4Au&>e|Ms@FejY!ys zA1K!#_t8Azi`+S@-&hb;bb738P3Ce=Tygn`WXklV+JIx(2*aHZvs@rL`t(G1z&Y6n zV_>%cNrYiUEe7x1SYeD}S8JACBy}X0Ue80DG?U`od}T7fz)lYyOvVnHuL1~vuXB#g zGUDt*E53h&9Jb3~#jq#(W_Le3EGJb;;}Fvem_~el zp1H)#ejl}6|A?5zZN72rBCcsuWb_@jmJ>Tan{|||V>Cj~-&bdv@=~$R${7UDWRalI z;zZY;7(~OFG?$DyGrdE*gO3#}tni%VTrNpaLbvul-C4dh3h@vGN;OGv%#AcI5>6I^ zi_W9E6pIW$d=e^ouDnq1Xp3I`@+iI*>g8?av8RHMDPyE42OA`qANC&7+ESrCne8x8 z(H<$TV(KEGlK*p*wX5CP0L&@QA=! z^bsTQBmU*y354_iK7HqlJP+Og7?q~{N9 zvz`3ri2wkunna!oxPdX$=p{6apwOj@7hiEs$l$*#NflUqPqCQGRC`wc8>MeOL`iAj zCm>aShTxiW#YlNT%cpoNOfXIfo+Vp8S|tsjeN@|@1CyJ7_XLT!P9n`6iQ^-#J@vle z>us}c_w+w|p_ji5bnBZ5h$;j|Rapn;Ftv&PsCbeLri7Yp<9^%#VV;I?O`T)?oA`fl z=l>kHJx^~vw_kyk9|zm?!P{rjC?<2~4B2v8L)|Gd&R3IOTWQq##tGY#UPD-D0gJi^ zy++>z{qJLI-y6(35=xaU>y6h7QcBVYXKPa`55&IH34$YHzR5U!G_xu9!<(%>@AcYO zNKNO}55X8Li1pn07yXaqE)~Mp=lD&GU}^Jy@MOHppWuOep8KmjpY~o;+%D5)6~3`J za|kjGdP%)H`ynL28-XR8Hr*3N0kvHDE*f*zZbf>P`^?u>r^4lLTA5i@uWk;9Oa#=$ zPYPuUzJ58-j!;Oh&VdTPZ}WIWUCh-z#AC>|8P{%(ac;CSs2I$G#ril^7H%)eCp)3@k3^PSqV7 zmSrL`M0NlU_G8czasFx9<@Lx0^TLy6U5TI6B|0;GE-50J=$vxf+_ptK4iXi89=t`e zvh*v^BvFlt>p14smuN|txlfc4fp;`LGEqKTq08Z_o;v5;omw>UyE&a zIAUOE8+v(eGM$4Vnz1as=gJ!)B9Vr2Y)LRj&{;a#CZK`5BEnD4p_iQ_)vAFGh zQ?V6uAqX-QN*My29^70_@31G}Q3x{g79%UJ(AxupC zSsYQyBeZM{Wn6^96Y<6Kw*BVwUvoc{(phvTauYhq6~zVZDWN+LC)QJIYJg@oZ;mms zRFbhsmA}_Lsgz2YVP-Z?!B*C!@W(CivL}WggpaP~Z=b|k2C)onf7KiwJa+AKE&WbRVsJC{3wcUGoq9ny(rU-gl;6k;q>4%V&G|r^`vTp; z$4k*)`IBsWe~$(#sb<&xj)#Oih-f(l2e8xBk@3SH+j#O`+nRz|>3uyT&zfaR(eJj3 z8D-5@gXG-~R)Y{(mIws>uqQwkj?Q-x15!>j*9|{WOam8g6GQk%$pke5vFYPqflJPp z{=9uOl28{lI;(g?tW4DFFuHk68AA@`j<9NX?ULDFi&?352-&^ILzZ-*1xN_K^<(GW zeLH628ee%-kEZ`kPIC$|e}xOb8ZTrxC+O`Hb{`unK(f|jt0Quk1EYO|mtaV76`fdi zH+G6UE<=JV`+5QyRW=L7NYj%gtP409GtS}I&Cmd3P;Jdf<4gzJ?&nX@%3 z8;I>#hvx$;=)xqAnv8pAKP3 z7&UF!4+#6IKeT@BY z&%B*wGJcNi3KLh#(n-#XhB3P51fU**0v}6x zeAfDW%d_C=+Ea5YqDfAsalPHqGX}{K;Eud`&WR?<+`A8OzsP}Eb3f71DpbIgP`9lb zmMiTlt-f0_Lsuk@=fV)gPut?&UaWj8L~yTNZxN0e$VYw42G${4Cwh6l#ZR%DnSGf} zH$IDMK9^)R?A1g){!wE1>Gwt(5ilSxDO>G+9Ca^?IIGa8IyQ^?ncZ2GsLeIF)~jDK z<5%%6gT7ke&ip7X#kTZ+ls5XGzi7@}P0E-UVZ3bgG_?&+t?CVJkY^nXI&S%k^)D`| zN?N%WDN&EpHjWUwTw1Y=?5QIKH;qoUFFArl8YLQvfA^Vbq8Gc=^oo_~9*skBnR_i$T~SOq@GDjKo$gwW)noO4 zY=Tum216~^f>(9uSN2VUg;`4Y(RG>;TxDmMJhsDZPEIK**|z7Lb_a*@d-Dz&lQ26S zB}BtA@VAKFMRy&~&7$xxgMp|^+ZQ9R04^D`g1-baW5{RobUA~de(I2^;B=u+z3@5c zvjOAztp}$QkLFOFZi7dWr>`wbX9fQp(UnKz>|cTF$*;e8McF;_WGQ#>??w*Gsu%oq zxih}zF~En%#uFyYyHzB6nG2bH*M4AzQ?XTHd2oYt>n;2+R+~2~0lfc4H$UE{Un$sC zMU`0Na9`0#C9Y9Y3O==Sx~vGG#X1R8v@rjheL%EM$XY@pgJ!$1BMJi(`57Uy{KrD( z3vJ;CL-rt(t|Tyj%4wcU$P9X-erV7jc0tlSh0j!|ba!KG>-?F*njo!;ciZg0L`5`; zl)Phc0WyIHWrVJ0#kxruW#PcTL@y5L#p%oB3S&GeeFyHf^wa19Ab*y66EzAp=jGR|d}U zI~E`Pck3d($*jg55&?cJ4u~#A-umPtvF6H%Pe^hbj+sUf7NyxTVnhqNs zE^Pyln9d@fjC6sxuNt^MMaaEcMZ!#GH2E@pwJ4y_d{{&4-8=Y6uDAb}i;!9UChTH1 z*{>Q$7_MQ4D5WHv*tG9Dt1JVA1|vz&vq<)#+;lhDdNzZX)vA*8_hB5gCco`zuRDGn zr|+asGR22{Y$4OwkNV>}0295J&OwM2L|8#&&Ybc$(HmYPPLZ*u`H+XsTdH-@G`e@> zy7J}BC%0pVEOJU72;aVXdGtuJGk;84>q8%_S42C%=1NEbI8%}_)o0X1^PTRT`^sfL z<`YJoA>@qx;?$$@5-w*=iO&zigV__Tn0NS*GO$5E|JxeuH`A*#f6~t8!P#3Ia*fd1s|}y?wDg>wNe=o%L~;xH$~{FDcx^Tsliw1-4wEZw zsnP#Ubw|mA0>s1--!5**2>eBiWp`Syc4n4`e|^CTLxzD%7qAR)oh>!Ou+?SDX9x$G zyi7*XYc%xOrFt_YOZbg>A9$cJ(9|+5c7c4^5*&Sa6ns~+HTB#Zwdj#+X&qU0>Rlc^ z&ERj~A-4WT12bb~wRI&YcAmau?M-~H*!^oxEAR-#}zWEUiF#hi@Uf?$M#l^?BK)lje@jE&$q7r*6?TLU_()|>dCUayh6cCr znIo`B%N?K+WgOu({>=3-(3hLGa8fcF?#UN}3bOUF^lbikko!b21({P^Q^)utwQ#`D z^Cdt!NK9OmB(SePu)%1MadUk~PYA8z;x}tvW3qO)Q{-+^mQ3zvI0R*^ji9eXuZ=J5RT>lxU%^U9C>l>P4Ea7EbdIY2+zlNVj)RA|L3*32}`y zWiI{$BizHMn7d^t+_nL}u)F7_qg29xvsWW7!erq#v)2#bgmj4r^FMy03mPHy%Hr@C z9XxYhW_Rb$3BM=_QihG*Drx+(3CDs`KG774p#C8w|5V5R zT+_4pGrzv>bZ+ZR@Ir3hy^aVPstLJmEi1LY6Z9Rdv+;JY3Y|(fE=-JteeWy*9lR%h-<7eui!RIRNERXdt(&4h2Y5@;BA~qj zAu)dLZ)4PxiDP6wJ`a~&v9@DG{|jMj;&X<48?93jwAdc@id;KL^tdu%z6Z)fGkEt9 zak_FT)|>dH#?@>m)3NxQ|7Oa{2yxqVwg*L60I}Aw)f4ge%iwP2M`P&0=E9$(lAahp zg0`ghegy4}IrdUym}VWB4^R2}K)|KW5oZ$$gJ~vZkl-tDTY0ut1G1v6RNyRRo}<0H ztFR%1t5Ah`m_{PPh|A%$^fI)eiGIJIMqgydId9eqCwx)&@n@oBim0{7P}@V zxGl!svw`-6AdZpLhr}RI^}^9@`74?Fk9lj%q;`XLSZ+hQ!Gn+j&Jl?)E3*mR{{O2SzC4k1fqj7WAZ`mRn2L?%a z&-XIpUa<+u?Z=3Vw1FtAr%ccBdA%eciDe>9IUKmL$nHs`OrKOXG_#Yur$t|qR{%%; zV;$B*cg;|PX>CH%RS`DIzl=h-@Y}-kS~q^vg@dp%uIB9XUE0#k0%p(-aCYdNKC&1@ zH`m;LgU1JU-xjA%D&peXKsWD?HFi)fW0*I?QzrnI{JgC`amGbT1Tm^%ezHKWN|{3s zFxsNbnGq)k{m_HvZ)=6H3TuLSer4ljW$T02Hv0D0q--a~hl9iaa`69)7Z+GfX?7zf zh^X+27XqotI-8ZPa$F@!-YVQVgVVl&$98f~|D}681Apiltl1LhoUpfK zWej+Bq@0l;xhN{JDm44M? z%x8c9UF%)zXo-d_JZaNACq+3`MFe+4gauYVIb$AQtvoRnLLm2=B^scXnWD_ugVY2#X2(A>y-3_>PbbdC;Vn6?*HRenAB zk-|INVcU}3oQB(u@G*`r3qAU3-Crw5))3u>&~2fe4i~Bg$psP+4fK!P@3Cs?-kB1SGuXC+0q#FeE2N*gTRc1 zYP?NV7D=q)>7)i)S29)Q$8m|70Sia_!1`Mnc~X0D`6X@452-cU5oviC1=TpgQ`mp) z709mvJUI*vxC{GT)|H58;?thqj#>0glG8RXy;ZrCQkb@73U_J^oH5NFVr?akxw2E- z4~_;Bzr_3+hnM^#+o*1i<@2K7v)NAm;pe626TaB?U7;`iMbnALqBG2lDhab!xPXYq z!5r<)kvIk8jFjwBhNE>lmOE9*2V&VRHV>M5y<4Jo~ z6wA-BVd=@BRJ3rbMw+NO|NT+uN&CabKrOiCJU%({qm^tTieJFqU?~2iXjIk)s+fM;9Q3Sb&LjTLrF>95q z!+kVhl~32bs)|gacG_$1cEyUO(K+3cZD2dBDObbxPJ0~_wJ|3|1X_S~I?E@gEJtFY z4U1J?Qpfq66u_q@FPp zryK92>)V5C#adV$sC%W`X(nNDKEUmhgfAFlvgeRuiwicn)R5hMcdd~kjn0lc9giz! z__O+nMAjs?+nl^PighS^3ZB3dc9-*87&a#-Q}C_oZ=ZKg{?3M=t)E&lbKL%TE6{o_ zyG{kU@__)p(V-d?|3%8M!Ie{&d5ZEX~gc-B#g^S_`LjwD7& z65b((-0A@V@^5ZKfPhu}Hh+r^9lds8lo<(NT>A?6o*(2Oyk)bp>;(tayfZvq)l*iMTloa_ukwO>L2bC? zX~7WgLu`Ers3_w_qz_jh#A_dU=8jjnRWq7v3BU{NTYj5K7Z>5L8kh9njKQM z{>Jl4ob=|g#XX7bFuUvpPM*s3=ori6Ao&Wr0z_~K|%D5-o*c|&6$iW0Pj(^@E7{w73j~8c*{9Q zoYFudUQo**H2tl-SK8JDFZj^xG8;yexpPWH{@hejcsWU#t_%AtxLB-IxbrC}pV2Um zSJ4Z~?Ty-NhK2A9q_M*6wh>|bQyuVu~z`f6S_?cZtML>;>VWC zZ-nJlQX2INgxa4ZSB;o{dl3tIwoHaKrDd>>4@93v%6a95qTTwe#F{M zvvo4@=m08i?e1=6qF07ytqDOI8$Ykg(h!aI=>pLYtNr&=HAYG4@1f3 z-o`TI=H_sYk&8U;JDwTI{0FX|mRcq><#U8voIvAQl2F*l((c^gT~W(2N-vR z!qoB~?F@b`HGdEmxT}#l#g@LK)*txbjyXn~&vpynh|~+lyQkDFjpFuL%J2*{jkja0 zIjcGFa%enZu~*pB-mN?*vQN2s+m-w(EHO%6nsN+vdIbH^e^;$^zUCU62i@tBR|cJ$ zcgz;Y^Rr7930-W!8iSn2n0iyzRLZ8@d45h*#F?d>!I?-CAvsa?^`d{P7vWl}^>Xk3 zRJQhWz+|Gkiq=xde*hzBSwGnYhSt}r1>iX&(dmenidv+jTG=ML(jhd-v|?yq6BrTs za6#0ySkyJIUEiKm#Cu6UONPgO(7-=RQ%`!ubg=4S%RXKTnB==Pm2{wWS20e}5RxHJ z1~{bAjpglw$+&*&6Rk)N!v_ETqv}1%0Pjma4+C4%-rBCC+&^pnhXO`a6;a>QF%x^B zmP3knP14=AGiYvynm*C~I>x)*2~nJsw1WRMkeCtfkNZ#ZKvZb$i00g>MaGwzKOTuI zI`}vji9h~bA?`(&0S7_aiHd8EC*cNWnqR$d!ev1PNt;^EEe0fLA5JG$WD0rGHhFcCrm@2F=ZNMA>bG`J)xLEETQ zY}3$%T!f5nDFeLR&lh1Yuf&wup zv=ToUdZ~!rrr;A|TOPcK@vI^W+ zYSMTV2Y)CJU0++OJyK2>M(i8=od8tg()pAlz3k2T^nDi^IKSvb0%r^ulTQ4vfbhH) zyQ%me1yaP(quZw=0O5|;OKml0=gAuQi4bDE8@U7!U!F2bBc?Qf>U+sPYTR$x~np`Gsv znglf7v1unFZi!RqV6H~Y!x?4wK-|Ros`4X^AW9?@QJMs+#Sn%#`-@sqOwElg-I*RN zPHVXbt^Rz{z|F%~U^QVxOY6=E$q#o+4B@B=DZX!_Xk$Y>Hd~TA89R=W!HJr^C`hC0 zY4tMV1&#I13A~3sLhg;AUoSm(9_<2%FL@$XcnpXSJ~iM7I(<^D50{2Vgy;QqP~VF* zKCdDL;HfA^+ZdZdevith(di+GhFP@mrpHilpXfW?sfVE6_dmzD0i zFA-|W`W>S;C_3!EU*uHk*fL{I{U22QWmKE(6ZVY;cP(0iyBBvW#i2-{xDCT^y5kBNZL?? z(YCP_@(_eE-Y^cDsT@{T=)01`1q`;e-A6gRa%o&+U-iK={c)<59Vu!^;-)|r^rM0_ zbzR~5hPpO)uiqrNab;t*&@OfFYleEogKyzPl5Y|Y)wyoKu~DY>z)J1;;q2$T?B6RZ zW4twS=9mdMr60#edq2B9nE41~FROg2GuoixF7Amu{L0a?cBTB*`w;T9W(16kc%ym? zSXAAGWx7$|MKX1+#9{A_#_C@hL}L$c%O6qQl}rwIPkOsl!u}d9k2>z*j`gkok|GtK z)~r=A)8n>Wb;i<%v59@DJj131lJ_ae4yGiQ&6Uo^IbHD9bPcBC!YBvp7IJ-0j<)?(T_5}P`^b#XhN`8+a4pON#>L1z$PJ}+Sg-l-UoxMqkW%= zI;akQqHOPfim3q@z$o2-nM&swsEVY*0pFriQ6rPbsHtu85QWXy)_2OQmYHT1R-Jk2 z5+9!EIfjYMLRs>OL}%C#qCFH9gjr=KiY!b7>24%+mSd64H23}cd->}{?3pu}`zIE( zxwJH@loY;tYG~%`!|1=lwSO?tPQ72>$3Lp&5~?_s1v0upFSYD(OgOGb?Iz5}5-bm; z^YN*2dqXhHI=bK+buspF&&9223&I3+nQN;bZ4Rjx5>UjC#P=i^UIU;brt>vZDW%RS z%_S2}@h%Pr-)&wekfGdxKins0*43Drn>eM5W!f}p$t*yHdD-UKH&&xm&&jS z?sVMxS1qWbSm(Ji$Uly)Xz4p#9e^rp_&#Vpum4+z?N-&x$f#bdM9mp=HaCUf=hPOTZdglmPU_)7vhCgx^?E@x)jxs59=e|G1p*u0 zg;QD4dpgsqhnhl3)`J{yD;}=;;W2BP%oazwD`@`?yF2yYK~UjyqNWG2Dc6X%{UgRx zI%|K5CwbApt1mud%v5Mpe>hMFOr40$H%cN|1XRjcJ2~j@T%x`!e`1HOb)KG|cl{*L z-bqW?4Eo{0kU({DIyp4hzoFg)8N?Bbkhj;Wf$SfZdKo{$)qQTu^n#V>n_DM`3OYgxOI5LhDd`19}SKxdr2L?1_+wTei z)|chcsLb*cUm(f7?lOJofw$##ew!N=?ShyjSeM0idN%u1Q-q+IBbaP!eaRT<@ty9* zuz9*O>jY5sWE^|FYXucWgnYqo8J`r7+%z?gll@n}YRQ<*PXV-p$C2b?`PZUzYkwrT ziHWsD7q7hjQt0!Ay^+hS;oWl@trP5Ddzotu%i<_v1@im6*d)pe)s(eG>js1oGD8hg z>`F~ksnZ>$v~}L^gs1f0?A}Z+rVeK#CQ_5#y~z~R1Vt}W{d`Z*2-k1(vC*|&mz7}=dMK6JSIvt)y{HSa=e!0dS}l zni5ApimUT>Dm(U?-0fmFjS!+JKir>=7DG{pH&EOEIXr`9MvNtF(R8&Dl0GOV5U@Jb z9&6JS9$8^_!K1+TrXmVGSA)I;;Cezyf%e`sM|6D}1afC|Eczax8HsCa&rD9p;DL;`$ zOYVtH*ObeTD2qC~mg=DOI+^`M`E&KH;tuu3&#IJRd1{8wWGG@$HFO~X9%s3R5!^~D zJie7VMb4FXNa(|)4+E(uhPMUnJkPZy@dO^jXXgeX4MCqHZ`CLSO z2*$+}ntp#@GELQ6j2@ZfhcP4wmp>ZiGs2;PQYa(8(#J$mO5eztA!Vj`3;qa^0Cna%*n$ zxElpXg0gdIN3G)v)vWuI$}X)sjT9c9Lu*$@YZ8y=V4QJI>^1;}yr~uJoFoP2$TJgCH(5Hv376sxD@ku~4%5;Gdz5-5jsDEw(1mg#aCbNTCW z&sR6#5*F11H@JXg5+o$Se3!c!V)Ua(M60qoRNYxX32kX=f+60PEHhZAcTca~0thpT zo~$!95G?7Gfkw^q8NKVX#gBR7?B`<-oO_X$HIlVu0Ti)e9$v-=ka0eTI6M0l91@i# zh`1f2q7T0Jg5m!1)9a1xcnyPlR>TL())iRY?xc+H*fJiAp^j-@@Z_Z%BTHk z*OT6?*&bTYeqGCZLqg54zn={;v~dZ*80t=bH8sXHTy4(S`?Y}*kJ3HTolxbMY-*lU zxuV?Hm{=2A-XN>ewaKVdQtgbb&no8VjS=NXva+2i>z~T4s?@tR$e9f>n3D7P*-)oZuWP#-Hf1Q z#sU7Y((iDiMI@-oKF#x9Fun7q8#1}Xz7w9x6W-#OnoUsljliPi(r7#7xnK|#bhU5M z&65H-_KAB6Ee7V<$P;zN9r8+K4ps>yzsLIb%L}#b;>n+xw*AT=b3s8S(-$4dI2f`ThN|L7yT zDmqYffmGIThCq@rS_Rn!9jx3ZIXKdE*C|r8?yo+gm`;%*+_NSY-*)a!u9)#7`I7il zo_ZSqK#NP87<}yzDSXVhDEuwwtwzVIa7;_+vAA}*+*p0N*+F1xNF8emuDj61fuC>53W|A2l~#dGOE4r0p8oqskQT zx1i^)`;Jz-JX0FX2xl}UVjDHJq%mfFuCIvTU0V^isP!5NKbr$fDeQ6(eQlbgd)b=k zX^ZnSTFzDm+rt{WR$&dG=iLiKTx zM)vTn1Z}+l=}ZrGj(jZD(|z~TeB6KZFF1yej&i5ukDmTwx8QESm%->`q$`4qC8%N5 zZJ5*^MSH~MUY=g?yEdl_egTRbnPP9GgYUS@bCWVItx9GpYKmUU$ksWZ`m~`%9@c}u-m$% zG*jXVdP-DsRsSMaV95p`;ceGi$XL7`%-al?dwM0Nv6w%9*s{=RS$Fbpad5r@cQzo_ zF)HYdU6aCzQ0|@_=)|v(Tf}eHtq`_;>z5`l*3^)?4jNwcF+W=&S>HEIMWzaYP<&@3 zQ|8mwmpuIDNw&lOtM_o;D(9uUD3x>&kHCaCiRHgP`@RwuH)mq6j0og(a%$%Bq5Qwc zEQM3eqy~QvZ}9`^K**_%+^hHLgGndE7VA8uPUV-=G95g#?=-}n&;aj@N0}>=M~6Y_ zWr}Y2Y4-CPt!+(d6Q`>N>)v1NwA;-`AB=zS*r}MeG8Zs)jJIpmw|De(zHYpSVk<;b ztO=5tfdSS`_IiTKXh2)E*3a#9Ij+*|MdKC@USD}x)5sMCxsnRyaj->Gt_YC>0GQw@ zrX)-8MBU8ato|RK%@Xnn{xa-U@qftGHkoTTI#BR&3+b|taZ$ov{X~uOV}JYSI+mP? zX;qL^R@404J6KCbk%~SLV}Q7zqXJ6;DJiKnnfkP~wWc{iCZ^P%cPmfz_dS(m&3w+} ziJkb;4|q$voNQj@tDP6=6M}weV!id$7Qe3H?l5wQ*kNwdzoe%4=!-^jvkknUX|cMB z;H>d*{ip5`PU}~hmvyHH7y%2UWM9jE(f6!ATw3~f+9#HaUIVlxsHIlW(RL@oYNIy+ z2l0h$(NlmN$9n-1I{_T0lM#_HIuP8HG+Zr7cM(XUmc%3%OzVf0oIDGy%c% z7va&RUDXIJ#*uYuPn*6C_%?UzhWd-s}CAHRY%Cnq+#7ac=iilgG+|R=ASXILY^@O7PQy3 zfO7!3{oRMW51G7yyXf_^)-7VB7H-c61CX;(;1@LZTs7v+QUveDlXepTE|CV&fvQC{E z%s?vmCggwPU~&KV)M-9Lx^XODX8!{;hYd`JFTMV-nMmpn8(%tKmGGr#zEU)!-5U7% zQ89O=PN;6d@?J!HyU2g~b`B9X9Q2o8aJsJ>@6EdlkG7BJmpkaT0`%P2J6TjmH%MX8 z@|DxHLy6;2|En>MdE`h|qA~0EhQd#7KJV=KKJc^@hNOlfZADM#LUopQYl1OfZT{Dn z<4^gWrpRJDEmwg4UAN=>?M&R)qLp6VK_<8kLj3>#G)_-aXWI5@_$Bg?v*wTke z*@Gp|gfyGm<|YTm;k$F!8f?=;n4Ami2jGMZDsgklt^Ts4QJgCw zM#5aqLCG$zM<@pd%a6!NAmYc4z46fWM9@cXpH+(g^Eoa)C5<#8OBmbQC#xn~#;;=Mhi4 zVN3Dpm#gC+Yk&rp^*=q$tiCBu2j=F1;q`x(nU)v^7 z1ZBb8_S`QzO-pgmvX9~bJ-fUgU8rmCEVHsSz7{&2Wq{-Qw>-AQsi$!?sVD;*ulAcBky1d=9;bz=^CA9$Z`Kf;FUafIl|J*Dn=5JT3TLmMX}W=or)g24Y$R%(ssbQzap_FJ*_j(#BceeD76E&E zM(Xmnh=JZI#eJqg#S=^7X19Iy;IoXqs*foV(Z-3~DrG!n_9^9IgiMy7#2*~j#9*Lb zZ6z4@e~p6w?~&C(UIw?HbYphJo=E3+&zr8uLO#fwQ;-(#H=oX~CJCHzKB3edwD*({ z-AmAG^elvJfA8kc9W$!Q`zmaF{uc6baaLdUXQ~Wdci0WaHDEP*GlJDRi6kWbKXFA# zlH`?omFH=F8qIG?5Lz1`Ipr)F`NNovkc{KV$9NZDrt+G?Ufe(z`TD}t>ERwRPp_7D zM@|(US)AMRhP${n2L?msmUwUM=TTGJezq7HFhzJn@lefg6FUxxy}T1s!nW5F@0pj9 zcQt6j)<$k~;Uzh@c|q>8PqOgyB9Bgz&Z4bR$Qv&dqmet~y!O4@vsJmqx-=x;7grX7 z0q_#6nZE4cP-(bOF2;uc|gtl7SW3J8aIIj_vnWif&*+ z%H`Gi;~%q}WJXH(nu9m#I+~YC8jZ9Drc%t+IGC`waO-8>!hWGa_7k{?J}%al#XV}y zCFaoXY0j+CcxT0S8(?MIxTIQ#9#hLC0C0*sDJFKY=1H2?q*iL&G+V^Er~zWsHs4bf zB}tZ9YMdQkyFqea*SB@eC8^4;1K~bteD*aaa8{SO6sQoEu(YBsc}H zIXg?uEj%M#N9&w86rDr3um9NTzLvj|8c)2TpxQIzK_T%i93C$Zh{&9pTUaH)UucvWd*XmO)B z>06c%j|L-4;}Oprah82g!oV5KU=zF2L`V0UeGMIG)JJ_L)hrg$%}AO+#FVy)Z3~^P zHt&KM#c?^8bRF^3{R9T%SBZ^?5gG6`xy9Uhh{e?JwDqY;=71z_PN)Gp!;y?AT+tI7 zwPxLP%*oykdVanuBPh+eipk8XDCD+JOdc(HB~K~0^Qt+|R0*bvZEKjudmo?TwSR=P zYLV%u(J@AAxtz}Q5%zvjq{8~-YDKqxMsW3cx(gYIOtijuIzzQ~X(>)lmf*_mpwz{p z`UDX^;5IQ8%&281P}jRC>w-wO8RX_&?kv_QwPx8bxCFua9w9L6S?%RNr6w|cUuatO zxS`S~+4GgX9)9`0LTbd5xiw>pG=Sz(y7es>qVj9g9p%H=N=r9_Kkf;QH>b9PA_XMU z*4&K)>7GzwsoITr-(Pcs0EfsMZa0m=z>h6^1WNV@be$i+ ztX|F;Cc2;`x$E3=AJ?XxuZ5RlxgypHM-KA|aOF>a>ChRLSLUty9yfLMxgIeZmN6B6 z(>H8&!so$xIX+txMk!L#)A&tK1Kd&IO~r!WyfnXZwgI91C?6Kc)ea5(H-qu*>X1Fn zj>}*mxH!U70kycPRS1;|prVkCFy?%-KarBzo>J4?n?$T&hK~Op<>-0{5d0U}jD^At zdJ@C7f6%{f`t}t)V!4k}63R(EU_#!81s8CD*Bw~*)F(nhQRhPWgQ*eP1GHYO(CNWk zn$CImJ?2OxYmSC!?C(8d`LnsoK;XtvjL$c3@{Op4v}Iko3MPS%mc0=is5SquCIYgl z5OneMO68Ub;el*qJeoSvqXbI{`z|qvmhGCqwdF}`cg62^A;xB@w~dvT3)Bfjv%s)y zv4lIToEB6#h7usQ&QfZ-1~qEOrN#11se0A&qtZ-cuf`#oO<;nw_!on^-oBm zAV0X~RB3S5S?+g-JCKbnjv0KOe@4Add37Yx5+ZYOu_`PQ#rL%#(>@JQXsn)8`2&kF zTK}kEggmC)SW+euTbof8?pB{8 zdx;S-AEH-(e)_L29pXJI$*QCX+a4}CX~0%gs0^<#gMZziTiup{Fn_w$+*FzCX$1I) zBuF*4Xm7M8(eu8#f8VRVjSm%)r=_iJ#!N3aKVBC6*`lO?UXS{5gNln|?I|eQbPYis zWb(0EG0tN3cKCbbxWAW+|W z>OOjof6f}ht0kEX{>E;LC22;?5Bte1It9IBQfgyp6@64w9HHJlgcN?o%}8((bgQ%X0%=PxhvSjBJ;OvImVLP_h~>c?`oU0!diX}uKi77sVxd;W z^Z{O$|1D>`e4J@{^0Jf$5FkJBu{6~tth&!H|NdPcL$h7vhow>&7oTGMydv&7mFpR6 z;<%U%ln}sc*4?-}jm=h5^{^But>8hU3{LhD#<@~vu~IPGCXFnAP?L`HEwQWH@@Lz! zKxk!GVPvx%S`w|8x%>7*lclpghE=ElNitk@Uzasn77ZBETulFV(5}Y!_jRhEvxzwN z@&Nos(IR`D#}RUa>}~K#-=@vKm=^M3iGoq`+B|qT*xvE$+ql;zea-cYZchfKo8~eNg?-Ub zvh~YW#Q|jBikb!JeKvERv+nl4VMT!WUOqeBb$ElgyG%S;F~FYzu?ohp*P~WL|I7qM z-tyW!1=KQ9t!nLjGo2Ybb1dHG{zd z=i)+P*awFE|8Jy)-vJ7&;NGWW4pdWa?k@FPTFL&6lH6T`1ga#voRw`3y5elZo9Rd5 zVAQ7P^f24Hm@Sl={8(1_R@51<2ZSLC`bQc6694K--MsCl?kMa9JNyy#uVoASSyHK{b&I6wVHX~ls%j>aCS(9z7OdA4Yq-vI}vLUUTz&opu*l3VDRi<{?Y~mCp~L5 zLFpnU`d#`?T0)^o>py|E2$!Y*0M^Z|kC+8sr!&UCKACEOZ3V}!YyxEuq*N4hJ9RLZ z?wXc)>fkj@yT%M{+=t`!X?klh_{xGr1;^ofMivU}aRevf?ojS&Hz2Z*Hr|AJ?589?f6r5SG%igl+YE|D9Ig#^PQw!H_`+My& zsZF_StBkV5DF>P^-)TagpA$$*1!irI`vk6pq%k~SzV{Mm4RKDWwAJGMeBAV?OPI&g z!sa$^}c!>yqg{2ZJUUbJI(ov+rDBbPO3m^ZG)W8)kXz5u_{Trt8?B9{FQV z<5Za-oY5RV0oYmLjNQ7CxqzKb4!pwSTkq8qKi}2Kn!#Ai9P4gZZ#AFs3{P1Df@nSv zI3@|F>`yW>E&I@uYk|sYbKW@J{LLU^{|y!YI>G`+1!L`PrVRMfF{JN2?{8AhGq$!Z z*tX@~JAWx^^O04x4lux}50NKBX)gYIOdAzl6Xu`ZLb6tDnEr)R113;mZ(am&n2)>K zP`-De<3UiA?S78sb7#m?YjAN%P2iR6#jww!nzHq!$zlL>`BzF}_LQmY21ya&7n%AS zYQ`O|>9^0-01|F*NX19r4&ak?n?GmD6jgoPh>qED^PTem7@ttj9nBY+Dcq;eno2XI%r+WKx0 zD@Hn~#9!sZGU@rriZS zXWwC)*B-9P5Kfsc8M)zVg3ixJ_xFA9 zl-+C5>Q9TG2~^Z`t;(FcI%n0vxfx75h||&@r@vQ?FN6X2rXU2?3(iqluu_yldhe!q zBeD%#s{Q-i;0$}(V^fRd2?G!&tN(u-g8y>~{_p4+|K4z>Og1u>pPQ8a1{BC2^R)qE zwXuQag5|as6T!sy%(n^%zmdl?IlH@rPQ%+EmxN2Bm5O*txPWxJO+rspXsKaBXdiv! zX5u4f=;#aJ9;awkiE1WT%!B;8oq2BgPF`|$-Blezbv6AH~NKK__Pq5`$cWM*?}2EN39KS-W6Sm=S@u-GJ~Z#KIX zHpcDx##7HRx9}CWOO{IIU90e^**TV+4)v;SvAt3fu zX7jBOFpUxLT>K~@hYQwoc#;{C1IutX)FW2ge~5kjI`1L$>d9)4xnKf6GkwtOP211r zv{z1!=F+z?3Hgo|`V1Augae5Yn=_4)iPDyD?)kIsV#nhU=-T=bvL!oAb0}ShA~A*~ z(>{Y~kxrtTHWflrP~V`ew&8B_R(%xvJE~;FktcmWXa)C#ZIP$Ot_yl{Y546l=rD7M zw$|`)wL=PDZanZQL&in$#Lpb_a!bo7(>CdwkAW@|q{2=iFH<^|ri(~@xGhgTL`pmP zLWH{6K7az5l*B(9C~vB`;?rS|m?1sp>lEorfjKkjTC%puRfNeTZ};Z}b@-~3Pz40GkXctcmyZv)hor{}$&eG)40FEc@G#qzQ&HLW?h zX||KSk(q4Y0^_%^k{D$$Un=(&#f)ud$7e`?a4T1=^rzWXZuW5qsKkrq8I0er`dzUG z*ykuRzdfrSNtf6uuL6Z>CfD| zG_}0D&2~tz|K9Fk52}OIJt;@c9q0!MNZ< zQ2y%95sh=5T;(SqNN-u%*uiIe^6mVGOc@(X_)5aXhZ$*@23sd1OFShP+83`@8NoJj z=6HSz6K9<#X*XwcL+vGzcu^T-Ud^Moi9Us)eVGE$(4k25=f-3L@^;8xUe9&r{7xsa zR2{5U?5q*s;^*Py5)>lXt_6Vnbc_I#8$}FyDY(AsMVp75T4M)BE>lkeL^iSyve+f2 zmy&60Fnh&i2w0g!5AaoVbKCOqlAJw&7G{;yAaLZtvy$lPAabh~X_|wMrEKf*OQ3<3 zw>k)n@(&>oU;!pbR8joPo?+BP8x=A1I$vvbpdNNIs|31Fi=+GH6PU{AZR+DNuJnGE z`8zJRoo|WWbg?RT%YL5)cfXyJ$TEV*|8BUVWaKN@_! zpAJ$h-2}`|L27c#q)8r-4w)mRK=w0lxU=Q&quYY}aJbSgH+{6UX~-w1*2|@1p#a|h z0I1(=LPd@~|3A{D;rtd{`QrD#t=lAIA)F8_47aGVy*kdcZ+vd6;^Io)|IOK|)7QE( zSF9S6+x@9-71;pEy490ZiRyHFwDwO|T@$U|8+f72l^1j?`oy~a?R4&IO3RxT6TowP z{pIUL<%^8>@Ap=ch<4)ITZhSxfIihO+~2>z4)Ce0uI8`*n!ra(F+v_j(Zr>5zXSdm z{Rdd$ml+ZkOab@42>;2IlhJ9N+DXoJM9V^u*BoTO$>K!QI#j&_4d-=bGIglYeuFUH z;VMb?&z+@le`{qTNj`l0;_)w=5#wsDe7D+i5RM*)y{i9B*0ohCB}rh-^LEO={JfdQ zO?F%gGgpqwhxq`Fr2F)0CVQXAmB<93Z)(G%g}>>ntmb4P_X8iT1wDA3yzea0xj(u2 zlh1QF`JB{bSB?Am$}ny~+e#T0PuFt6YS|L(I-r(LN~v9qSzuUG_t)EwB$WQ`=lN`_ z{{X&?qz|rjOhJFYxfs{fn!HWty!>f34zN-qn9?qlzs<)y{wdNh+>#qC#M<~^zo8l@ zzS5DFlHbR?oFD*Mpwys``5Y8t_)Bm!t&yT34#@kbT zxMB20l2GyaK9M@d9$yPYV!ZUg%YQY+R5NT z{izGH>}pMo1BU7x)NGBqN4QK;(>dsUFK+!v<ivDHB%izga`FSN>A&Ns>JH#EHy1pS6U5IqJ6qP8@Uc^qtIN(=- zO7Fe!qMT}ZjcudZ+r0#J*Z97QZs(7=>)YPr87=r+5dM|rgonkrB{X}KQ;G0?{KsZ= zTI`=PZ3`fhvp!?tL$U59Nub@;e&rvijob#8?r)=xQ#wEf^VlPCepheu_Q)eG@+i2K ztE+KJA|I!l+vNUEfAPY!zne-dw)ppe$Yu(DKYfci%rH1UFM~ov%J6aU6||Cj@`dew ze&mH8QFu?&8_F?X!(UtbsS|;}EcrR@wp}*ag+KdF3?*AgO~MvZ!I?}3Mv1n&Hs>Eo zaeD0=@4Lx>QL~*pckw~2r!$oag(dB+vNY0936(#R5wLKTW_H-t#=VcDC%%t5YF?~8 z-$uyAHoF+ww>QLbtV-Y)9mE>2B<-k7{TfMdMEgU8ZviW+`qQ487@JT~Dy;jQD*{&D zRit^7Iwd$dG3)t8*~@z*^#25Ae%rx!R z4MoWsQOR)yRHGK(XTI93(;U(d!~y2a-}8%ok{M<9Ydc;|y-}T^aGVn8!>T(nc04I2 z1@F~*PwzXuI5}pZheQ(!C7J;89(79Ae%0@jL}9e+PlIl=^5gOezobvexJa=}HBJT2 zP=mRswr}zY5}VgdD+|7pi{nM6+%Npu-dR8_06*fYk7JMZjVOM+UKsu(@T$7X0({K$RQ! z9@63bA0Tc<{$1wQ7x(M6SX6lEm=}wVb}ipzsL{;-&zpl{Ipgp|`M_9y2b&pB|PkRGw%AwxxRTLi_$rfAg?Gdm;6p zx7aWd4ZRjn@bLH#P{f#yV^<&`|ELKyND>Dv{RjkG&KXlwwgk_?VuJ!2 zSIh$4Mhh+=T1!bQeYM(Z!^L*MMve0Z{pY#pQ!k-Fvi_M>^o7=|bu;xZLY=dcXxx}y zkKt1SO*ZhoG77$G=z*n{njLtf=fWltRS5u!JnlOxAHA;rj6nCbu#fk)Co?ts3B`Kr z)vsA~eu83#vPONKPoQNlr67_&Hz#42UF|sqELMfA18qt5-~3G|Bb&e@r(;f+U)nVY z@_~w<7r5tIQF%cmywk?0cZQv%%J&e0yo0wM-emrM?Gf6XV}_fRJ(~aw^suN8nYZR7 zMF{u)UB6_0kJUZzLUgW_%DncD*5fB*q$PWIe0W;B`@2$v}OJ7zy|kG+4~YE3IpY0I>BC;meOuR1cTH7?XaY`cLJxc za`LHd_pm%RH@3=v`pN#jgJW$=v!t6sLL16P?;3W}f9Xz7wq?6X;|J+r?$c@F*!kb4 z?D~I{NlsE@M7J5E>fOr?3gM{zppK~RsnC#8bmBDU5{cA{vc=HG3-n~NKyx!E$#}@j{fBGVwz^0|6V*&EPvU)IxHlQ+pfPuun^0(!y9wkG5|6rrP>*KEq-PZ%7kd z^LnR0TAxd>0Bk|+Dbmg6FQy0|&Z#t?bbU*c^GHH+{n(Ufr^h}nU?xD za1(tC)CO-O-en#PwzyTSvlua}H(ut7-(@2X)u}l4&RejYL{6kG!*Rs1#3%jz`e{uL zq(iX6pJcW>%CEafp4vn-e(|g@De-gin0$m9{T{JQT}%*)yIoo7bUv!kLCUH?-wLMh zgJt`V8#v^&O#9^<%fK15uQUpnNu-jmIgnTz$_Sd0FZF$vS!gAKcZQcUio|-#7duZ0 zL5sZV2GE=F1hTz6XD}3$G6RyQod4+KI$oBIk_-u2%;G-c6H^rr0W8*dIjrRNGJ3WvEwEwtvB=5CwfnIGi+2eyS-kc|Yr;U8!slX=>&u=gmxm1Gjy;b5 zdwL4^?-_7u!tU8rA~w8AZrrRj{|8MK$O*eOc?oj)hujDk!Fc@PPY+znGKQjjM{{X8 zUHlXHUHW`a*`q$n?dR9km~z>`Bdk(2*{vKCHJhX)2o9|Z$cP~cxtw;M@2m(_jLHkt zitM_578OINxHH*XKlTFn)Q~>;o5dW_XNSe@Sfm5SxNl1QrSl`uk605#Ha;TWE8;zn z!jgH=ULCg*e4xDq<~g}q+oFv4>x|_MleSAqj^3Nk4pDY?*&)m*Q1WjmXoqfepMfzFBVU!)|1!n$LfnM7DQtn|?8)H4 zZN#zehMAh38@?wMNoPOX*U8I9!r0t`#-^kEy&B#2#`bojbLjCIpPkO2A{#O4ijIF@ zFEewGFkpGl!nl4?h5dH6B;+520WHzeUjb|2^7*^P%h)` ziK(rN_~DRxEyn1!W$M{gAhh3j_{15kq{eu_MdnZF`yVciR`OtE}zy^TJC7Hih}^-%yV+f%*UV zeH5h(gr3Z=)g`;4nr^JS+DZ3(FHT9q+{{MW?=!gwq>lxL$|et081!}cax@cITw??| zsTB@M(#BW>XpuC8tQ!y1CmrqB$-U{A7hc#xja#Q(i)rhE;Z%rd|EkE!PXX6=Wwwa! z#dQnU`hM$_K{`f+VS-P_(3!sg^6!hC>B!GSG3jAwoi@9?9;OI_+5*a&En{RDi+7z6Ptn zhY?NFWCK>hV>i5dq*q*0ooWJx(mB6D^PXmbHW>&Huq>PZ)vnS@&yK`t)NQf@e&B2K zUnZ0GskWxy8mXjgQ)dcM19Ze6M`ShjJicZgeUp_PgL}(EY1y^mzKxWz-8}aLrYX~L z@8TGi(oCb#PQ51ULE)siQL%tq?cd6lukQ_qWtrd(kYI+)V0w5Gi{$$f+W9FTBPi)# z +#fdH%U%X7nC-OI%MqV8B;l0==zh=JV5f6K|o7Ie|tkX+!>+92g>>X$bYbm^`u z3EVhOFhK!a6t6$Me5YrC@TD%3&0(ys;txiUb9NYJNS7nZon$+Y&FF`e_1iKd^Yj)2 z{}perLEx2sEkiU5t0_AGLejSvX(H)d-uSkte)`Z6)s82C#YaZANdkrN)P{CgiC4e9 z$8q^i-Ale#{3aj&@8~Z|465+R$&YC|wQ+VURCKEU0b1li+#j(hV1_+( zdgKR3@8jNaCSR5VQG*Dd9nsKkn`Xjx1fP-9*Y|I!^=VCa4jb+37C52f23R+B2hlxM z{B#ydM95nBmB_Tf$j;-4r?T=a-qCg4f$0C^+`@hNF1MZakH(+omg-(>C#%C3)~1c- zbZMr*v;#F}RHIhCSwHN_A^Osdo4%_(BuDTh9W6oKyb(Az;A$CkH_zfg-TP|R) z@W*!L45$2K_y~DdYJ-vsnKNo32Z}z*FmmhbpQX$yuG6e+EB^O6C#QZoszt8j?=j_( zD_@6)%FR@$q?T#g{ciq9eT*tCY<#b&i3#8e3A~|?0{D=OTD$e44P!GkOOsxP^=mm7 zv_;yb$vtC@L?rLQ#XtIlCGH_}mD?y*;jlQU6XxE=O0*sM zxKc$>vH8c)+mTZBK!+kzE5cHY-rp+}` z?tWoApE0?5JiMZ;uo~tks@i`U!vIWq(CM5#6%Hd(+zp~Xo%c*u1G+c>@ZX;XKmu69 zKTKP1HuJ?+{OwK0QiYdue`#H#Zd`Mw3B&HiCxxmk)n*x)qaK05Uo2};#w|HR zIJ(*R?sDoJXak|t_jyCPQuhwK^f>ooLwPGneHA6e?MWF99SeoHi6GXFFie{;UX}s+ zbUzvA<+xy%fW5@uo5!Cgg;)JGPFs3hW-~N1OEz;Ul)jDXC=alFyFAM2^pWFd+?@vg zcJV9waBs66)rt_!owT1-@{L&2Bz;9=_Ff;X!$hGE=)eb1@)&fB-ABT;2STx6or@yI z%^IKQ58VWLIgKq&O_HVhA!Pt=c9x$z+V4G4jzRuwdN3`vE%UZW?ZpFFwn-m~k*G1I zRwcmoOt;z3)9e4!(^*F~`G@^~bTc}nOOP&U7>Kk|N)8DD>FzEG=@3v_Qb0f?MoBne zG!oJ=MuWgc4jFuZd(Qbi&;L93+3tJi{#@7ldS9=14HHw-lg{7%rE&iONJOg^ts_7K z1&y^KJ_wO#4+47~^A2ILzwvo)&b2uW>HJA@V$ITRw#?GQqrPs!?$kFaSRQDKHNvqQ zI(bHAF6Xtmre?me!sdwO+i;{ell)Bkd4 z1dKR1&-hc#T2kWSBvFap`(;;6ILMo{vZxgvD2P&tzxK^>-gT*4-$H<1G`hKX_`O%4f1+CK`>Jheyi(^yo9`>4Vx*IFKy#Y*qGa$A;EUb~)=+cMkP zr=0v<**sf6!tL#>V;;R@z+1o+k-@KM(fjH>4RS{;kfviwA@F}@>ykdgH9j6|JE+UE zjdJ|jH6V9Y?AdrG<83EP0?sxj8Stfk0`+O%$`iPD7bRGjpXxQX-y|o!No6%47!pf` z$Lh|{);E1<_%g(~Jw;l}heOb3N(*g>pAV7v=0K9*b6$6mb9S3Cl|B1Zhe<&|evHHF z3brF8Jt_N%KXX4wC8kfT%D}BI^4|LOoARK`=Ltp89pV(%+geuS&cml3hO_${p($=Bbs_v`=O z^(tA|dRs4CwgsJa7lamN)3R49*OO^P$>EP&qw-S8QCUyYdXqx~nl>ic&m9sJvl+cF z*}@{}(LUIM-*`5_ZF4mNW-?b1CG-+`g1Pam;4;d6i!HTdwA=39RBZrKd{}Q)`z7;E z)h^Fe*&>T0c};13VpY7F&i9e^C@X@1ECq)M=ps8-AoR^!>0+XH@P4G8U^Lfc{yj8zG^;a*FK5yLflXs?sRp$(3KINbi5t*F55~oOWQ#Hpz=Y< ztVj(Z4N;{Yr3@jJ!z*$ESstqpA7=!h2nG7z5iYz{FX>1<54kR*IC#X8vK;Ga{x-J+ zpH!)1K0BzM(hmYE5!_Ew0`m>$TpP6F*)b9FX+YuafB4z7tcyEOEJ*6tobi$Fl-{%XqaIIdLJ zDp&r{zitM<{4@BT`BQU)0o(%0FJB4j*Qj>3I_*93dh~48VJE0eK;f0^}_#9Sy-db(@;8Q;=Xl>;pC?fZb+G2lc@930hnwF4|!q|S) zvOBX;H@A9tYJH+q=3EpVpL8LtrThL2l2t9dDbYf>)-=bH*}o6ME^DSV(XVd?`y#*s zdBUgW=?gseKi>22kCEF0!uy?40R(f4gO74|WXvvBKlS$S{n7Ypl1DL<-AR-Q%?#b4 z#5peOs+5_5I=NJ<)0rNBklrZ_-Gywa$EtWdc1%W@119feAPf@2jmn)1h?4w|J%U~@TJH}%(kKFo$22On2q`5$i#(;bO}-_FWo0FI zMXv{l5d@Z-2aJv8|W*T0{7S3wbb>VY4X`E!9s{9^7jKIDnYhkV(TnoGF>~vNZ7PC&;`V zdiHd_l42w|nLb^Wvx!Zxbnwk1ZN~)+-_|s{Yi&bt@~|SbxTKJ zKWu1ATd;Q}M)M|^XkY~vBdyHs1)QYsn8hd}nAh2^NOsenVIQ|-3*&K#SzySJ)q>M( z>vX~f5At!wxADs3g$L#^n-kR|NW29p-!JX`K)pOt^H>gL;z>ZL*56HpNRS2kPdVC6 z!$XCQ{~2kH1^)d>^|1xf@=d*XE`(w?eGRuPb!V~N9DTi@2WgX)cV5o)lt0@%PQIDwmb@v3Ncr`v9ZvuT7lDra|)?VcHk)Q|9&i zV60SN=H-FlEvB|K8(B^Dm0G34S#E6b#}Qs08934L37@NdTfq-tEm2q$yWVqrvY!*b zL#YpH`VUaQlINidueSV~Ho17ZdGLC$VIhxgwSipgaqVhv`e^hr(XWh^sZFRt5ke8f&uy+BdOl)^Ge079t@jKCxP2|N z=g1Y!grAP+E45kPI;%k6!$YKp?Yw)%=3ej_aP|IS7rIG?8XM9udsC8#>JtEh7DB(K z3$IP-;q@C$No;(>ayfb8!Lgm3&DKR6#-J+iEVminF!e7ZEa|d1xBmf3T%V!{=c$N1 z;#=lYoiHd>_B7cfqOt0Wgtud%4*}~BQ;+J`!_8u!X70K>f&4F5M=|Jim;%OdbNj~suMZi|E{{9KA z)%v%nC(m{0nuW8zD>ipX$nU2Uh4vwuK3Fr(`Tq;dy^Va91hzWG$l(L2;-IDtLV4mko90XDKLg#fsXq@It+LyMk-xl|6ZeKy%E(ZFp==dFHCmk(5c>CZjnz#iDbu z$^qgw`eK&%6&=H}F)1pBb6|vjcPuB`$E5MBN1Vplo_bW#VCSXp^!J}rBv67=dK@++ zk*71B_1()w=@cf6N0a3nnbI))FCS8$u%^&BULMM^Q9wQS=Am2eObLn!|RZtVciMUCzzi;I=Yr5_s#0t{M9Cmm{qq z?;vW@=c2H#H1GB!Hi)CII9>MVt8F%C+|8I?7XIV_vTkL`r=4x>>LML^HX>%$`egm)u1ZZwhCxuEzj z$f4fc@`HS1jW0GrT(Lq9N$wrKXu+Y+$=g0WMdzv$$fhA13Jgof8+*jd-Ws2ktxfrC9CYb0#34F>+&Dkb>n6owOR zzWu!nsDsXjzcfG)B_YspSY@47w3Ks}>yh!~qmh5^ag#JwS}Zj7zTe7JaQX$ql2LCA z21D})@72waC-(?wWw0v(>&1kX@U`UW#YJ*+WzTZo{irq>c&(_pW=jYN0+Ff6tpo=R z2lY~*JeIl^Mcf^J{(R(l*JeJEc*7D6&x)o6ggW7;J7Tb!FaBkBvD=U}Y3EEU8TvFp z=qCfofDTSJEq#F~;mt~2OiTYOm@eR7-*qT*BctKr_XZm}B?v*bt{ME-nu)96_$F}m zi?I1~0IsHS8Mfxatq)EaG}5SB7YzR<$`lP#jJ6*J_C+BZ8L!oUqM|mThOkwJB%XzUUHL>?%jt5A+HoBkdA&4rfb#w^c8pYElm9e^NptZ&YEi4&|Wsj zIhHp0n~*h<2w4WC$950{BRNq63Q8faXkIIqMjUIoKFCwFlMQ4OX&NybHK2@%EOGIz zui2qYNkNB({@d%La{k{{V7vn_^kXmRI%pWw+E$Zv6&73tLH?=Otn~eA>olUBKDX%d z7Je;Q-DmO_V@5I9tR?)z{k1SD62|Ha8t!9`a*Rs80THRlJ6j8bE7&UY-CSz-RO>u; zxy+lD?0h+n;%Y4K!Lwox5``IxTpea611sZ;%r_6OaSmtDCBL*GBK>^5d)jRM9P=|C zO;C|3(`SYP)}2^eFB=e;d)<#JyQlGPK{dn#1CNTtm zfA3JUTPD3g$i?VL$AuN*!k){O68y3(F_$hUTZfp07k95P3hPgO`F3G=?p3KcjK%fF zPIGnG{({FT?^czF$eH<601r55v$VjFvL!^JW@7)js##LAEPk9zP~xJQlhwXjwb5X( z;QI69PNkO)CyusFY!|~iLGX7f!NaG~p&7b8D+$GQ%WZ?EM&LIv-{0!wB11j$P%oZc zeHEj0iEITkP(v_84Q&6-rHgKGwRPb;l!rj4GF39w52K+ox=q=04Pef<7HH4)zwmP5 zZ7~RXv~onuLe`2|<_zx>^2t^?VDm8pzt&SWDVSE5WyGC%#2hFPFg-Rw!by9cb75b* z-za&-d}gNAb|>@kXigs@`|ho&DsJ9ueMxf(JpAprXF`)jI`5ydp%VPcawP*Z0c*Jg*%?|c!IK&ktRpZcu^WrjEDc?tXAK* zc4tpVv>xF0hzrJDP)2(PvuP@pknUo{+z=?vtmM zu`jJ2c^eXqDS$p){Rc?>*O{o7Fb)OT^%ihYyAI39C6kXzo7y&d4*=Fv9|y0g!Q}5W zf?Krad)XT6k$qeD3yr9Lu*RP@un%isV>TdF-FH7Eb+?$?+$=i4`AR*PRy>u1p-ZRP>Wq~{OvPcu&~ zdO(aHe+=u0j0UB6v+3{e<2eHwkSWlvL}=36!sjTK={l)Ww87P=$o4r`M`}d5nsByl zF1uGAuHQxAs&#ky)5N6mE)--a7~#B}EkddFag=d6xzhFg3*hk~b8~7Yvu_wBi^)Ph z7VFsu6!~a<*j00ScE*)TCgeOIwyy}F%g>@V_l5EE;C>GPODAAOb+psg8I|j!#)<~b zBilExLS25}rViHc3sl9JSnW{Z!Ug(?S|djy1zQYOX`xu!QaD>gSncv*LzqwfrcUYO znY8YthL7n|Hy`j$w@b9AAjwxvQiHOS9$Z)$L9NX90iu1S%gaVQtm7Ji4g9cqSO_cu z%H-8MRVWV{^A)ieuii7GC{LfKDl{{#W)e@+1Cs-4Puq_BMD2I0<~tNyGCq}-Z5RC} z3afE7b-^WZBv6$nTZ~J;{~wliRdAk^Kkuu%)>O%EykUb^>BYwr2`%BgTIsuSbz4@n zb4=88GRG3(B1~^Of&MU&@Ur-IbXK8SJU! z3=aF)FE>uxf25|exn<>94}U3Zs(mJXV*NO0Vo$ka0JdEs4rJ>SjsR;eTsjP`CwG$K zc*{fwSx5=J%Y0&W__2J=D(-NM|yI!O7eXZ9tyy=6(Nafgb+i8%e~FCrkxjsC~x zfB%Y-_JEVT2b@+#HO#J|2n46gUpMnF?jS)z0p@>du^onSI*e|}LCfki>0faoXEPY}Q* zF@$OHF2#)V2cOR9?eP+MtShJ6CL#Ck;?4{0J7;vC{P5&<;ncwfaFm@IAe=WV_wT%H z5&rc5nj%0h*GG8zd$Ye@F*SEi@t5b1FRZ}aQcWyVYfgPcP3&#xU7IQpQr{*e4F$Z8 zY?;~VqWFq->1#RFVtH>HxwDNe8cgY&I*Cfk01K*7v)b*1t)qAM;O3n8cH@=~xxJQBbq2=N%qWvvLIV^9tN` z=Pe?3$Osu3AaeB}WdyKp=S&8&^COp#HgPA2eB%%flU(~$=_g&p;#PxIs%sNC_*eTI z9h;=q>pEXuOSpf|+N!gj@vZk209UR(e|_h_31+tMF6p2*121A;eO=LObTRA7onEaX zq3Kypm(*dY7`fkAkYghmz$O++@W>{%3gxxVzW4HpY=~k#2n2KIv;kRjN>mSHOfh6% zFrdY&yK2<)Uvv5b-D>wsS?z#So7nE&cSnkpf6Rt0)9(A?@fk@c1K1GCm$lf0Cs2+_ls`SB(X8^ovl~zC(ru|=HQcb>Y4~+#`s>SMU zn8miy-H*bK1C`GzZS=^6*5d`2-RvXpVFS@1yb(>#zi^q*?RG*Uxjp(%F6~~3`jn#Q zj_DG3TznP@zw04cJ#{<&Q&Hsg5s_~9Z$m`eYV(*H9e@P$VDjvCVv~0V(ak(cX=QlW zeX!H3s7JxZi~U_rF+0dL({t4gzcmfX_rg8g5=0;=S)LhUd>RU0*@W5i;GOxVR9DzN z<$~3Rs(jvzPuDBpT_sHMG3l``@;S`Xqm7 zl5Vhv-dN_w$E`yh15J55#dxxreFq4`A*O2;sny~VsfI)baA1mp0Tf7!y<6ce>#dYi z4S{6wWy=;7EGf zl(@)sG_hVpaS4RQ(^RA+eH;kg& zi*+aGHzk10?JIg*W9}>xXnpvh5}Pfys^?awT)a}RFC!68uMqnc&r=TPv)1fh6@o%Y z4_VTUY#f-=^7<}PY24q>w4FQX9q&o*%P=ymWpyuqMiVZG(%3%2)<`HqGH_u-e&%%W za`pnBYPTfi0oA9ezusC(Sy?cb5{cPJKD_{ha+n!I2|Y6K2Y<)Uc1^#p>(w;MUwkcO zImGcw-e9>;(ZI?+cwrObj3RQrc59kZyW&MFHor|K#htDzqQo@o1Ac0%phNMXzySca zy(PfozA1$F741VV4~>?pd1e%Q*z3L@Ckwe^Cth1oOcqB#o^hujqNU0x^bN@Aok&ZW zeKgQm;;J0Ws{0LoRXX-F-egx~4{u6)lu7PPl^O3OpHhYaAVrwvGmS;%leTyXCk^56 z%5%^?6h-5M4u8nP$PQxbDo?0DXlB zgDq7$=@YPZvO|H0wevOWsc7(ilk$0#VeVZj?u&F1Q62bQ=&RPSmtvE8 z@=t@sXK9!^rv7H7MwKM?(f2tr6q0k<{fvnV$g<&T8DLW>kCyOg9mgJ9k)4!@yP2AV zoBBYThrVsIFSCszXF*EaL|hJzB8$8}OZ%*Miv|_uE0alL^_tULVf^*W8` z9nlx%yG1hO=2qmF9;;@J#9gwO{$V-T;ls-?Z25dp)}rB(@>mY`3hN87p$#U}Mq%e9 zSEYbsi583FP?Mpw2?t<-e7WHRXmhcpzg$26DtjAq~Y?%nz@Hyc`ikaaJwQi72i=L0_!> zn`*)C5UJSwAAkuf4Z`z4R8`1**mx=KTK0l9_Z*XwD!lbK<}hwPyjpG8fM)v@XmM`)pUf22AL-%>`7Z%43{{=a8}bxbr&JXqVL1@ZjxDbu_}>%19Gs%pE~wi`6URAWWxl0j-N!FMU(<~ zT%2xkA3xTSC&-OlV%w;dC|ihlxMMd-uDL&1%%jv;l{0{U|8`B2=R<(g7N6Fp=!VX( zlPRg$pWti3-!*fQ5+lbl+cH_+wez-^n#a)!UQ{mPN7gLhZVzrS)yA9QX1*SRy$urL zy1qe7yV^E~ZljV~gojU!+gZ)}NKV~QHJF{C+p|AJ3#Tx057u;0upQi|*Z0VsbI6fz zOE$E~HqX8@@slC;g{h6V$-io&+TJ1oN)irB_ zSC)7~zmae6L2xxy+r$)To!@N44C>)KUn1I5iOC2@Sv@X#i zhAZ3zi6L&`ACGQgXDNh6#-vBt;n}g`lONqZw*h_^rcKFZDbNy;4&-%%qd@lTx_!T? zd*_AhZsK?gX;03nPr(+(r51eD#cKKdA1F(5tyN-Ai*zuP#(4)2cb>V;8tWZuNH|rv z4FbCTpx`&+mM6CNpB)_A@8%8rJ|phU;$_HWUqiL#>Q;x2Y0avBiRVgln`8MBB;xt) zS*{~K6>gp*Kju`5)jth6q{na=vw%W@WMoyYSwTp!7L6@{T(WDljm56qa6| z_`XN2{%FBx(^q)Tkm|!6tzE5YhB0F-~;kBaFU3ej~;}=U!!xkI1(AG zDX|uV2yPMkH!W>*mm+qj_Fci2PU*am&pcKkb?M*Ci#yUXJOL`Ub1ILADuaI=4?slx ze7?HK^f~`|n+$6G@Vl4XeM?h+7ROLrHd87izjVF_hX@Lyc>D#iD%(6$)|QRfn;QL~ zH^X~6j>`p#B9t`WPFlQc$aszd{koUhQfWpgJ8eNy&Q2N{ntEXf5cMWO$&)q#F7{x7 z*P~Ce4a9i_h{?0bamz#}YZ6+2(!T0M@eeg{KU;h>ySzFSz-xPWU`Hsw|Lld&XPK#= zn&Q++AOQg6j#jVHCUQZAWl`++l7*Iiu!pl$54l>9J8IPZf=7g zd$bQbs9r@<%3syaF20I&mrNjE@{2kC{g2$~vJ9v?6K)n!Qx*pP5GaoOR5EZ{{r0Ni zo^iNWF#S;UOYi3!x!V^Vr)mc{95z)Udzp*ff8gP^Nd4f$2l_t;viTAtNvGA~ntK8y z!&@+(Ri!9>fJzqH=}r||W2xcQ$TR##bxjklea)eJja^?kGguUvbz}474>D@QyYAD;jQp`5*l9tIzUutsy6@%mG#x2}{7=N5PqHR1b_??Ko(jL_$-?)AKZlM&~*Xk7A z*4E)14@vc3455R+zk40l^&g-mOO>OYv0%8~jn$>jxf?I@M}{n^UlC?qfgDlPEBgIA zSD&A7V1~0bh5C1vE=bg_@@iAL~{&YDSuz{xAd>*9H+&Pw|XsxB?7qf(` zu`@-1{qlhQGC4Y&;+o*8*@MEM8K)0WTn3YAetUYL;+5W*V^v?W$juSMHQy97u{akQ@qD@61}pQw*| z`pgUbb7DK@utdiLq4JA^3ixI~AHH6mb8z4<@k$9L26y1uRQ7v`ym?Rl16^S{D-3UM*O(KzqOf};lZ%`6#L%yJ zLefA9Z&Mm4ktns`creK9fvc-@y3?ywFN9X8b;1GEm6MU#2zpN-sHHDNW00i6oHq9| z4BTNqZ>^1hr2Pl*q3^1`kRdis^fFP3C1w-KLC2%``+bAoT)qS$(keMzKQ-3g7kh{d z#s}eb$u5uHYWk&~9uI4kxa9lb&_3*5|5QixGU|O$V=cKM7O0UecO9*&V)2#Qf|Xw^7FT3!l6h_SIcr*zR-S;T3- zzPKk6B+1kL!2jLnDiVR=M@fc7QCgtkg_C+F-MC{5U^zMs&o3DT=R$&z4jQ@fX*g5Eq6g=ABE1B8V4d<I`@n4y zLuKRtN`r`k@m~MQ+a7MV#ymq29GJC2?xbTiTjMLnMt^<)L@TguN(=#lIH<6wPrA0^ zCm0swX#{RWB;0MKx9yum^3>%rq$mrT*B|uJOL5Snm4SJm6?>)&8N`Hc#_S_)27pA9 zb>p?J3@bNj1?Pv!>MrzyIen5;yfpIpr;ghEa!gp^(6y8?Tn`l(k2s5uqB@iQ+|akh z7aeQ^{pW8dI$ZLD$$CuhPm+ z&s$`gCn@F!b*gY{_8ik*OU4ZVGJi%3hXb(xQ71&k2@B&OkmcAk`%r}gMbgresNdnz zU^wnDxvQ#=TTXz}aW^3qTix4)spe0(g_(!U<}(T`g!v5;;X*+rqaO@k*qM%yg=QpPjFplGvGYYw)JRYdM{2 z*m^%buHC>epx3wV>=h3eDeyU4Mk#(}MeBp(MwERHLJngl_@=tjk?j@2_>|$J=~}$k z`-|t&*svUSx_Kubat3)*_jXbw&^q_G1=_$O@yU)*CT^^_xB~nl{c2lgQK2VkkpiLC znZPbn>N!_*d5Kk+T|29SD+WWukA4Sdj1OyjNnZ~POv}U5e|Ag>;H|ZKh*`}!87~}+ z=X88f$hepc-o((wVPlm+&Nj<=4eEnO z!`Gd52=nwQD{q(_MJ&(&n%)M*fbTo!;6vRo#2NGQw+$nS6)mQ03rYomR7n}fMKo}1 z!PHc+KbDeMxmL&tA@)J~j0t^Xj`GZ{W)X@oFfKS!G_?)ylNhJ&$E#JMB+|V;Th@CZ zxkqrUSWhXN+HhDLT%Xs6IC~UW_)^NV;;dbb^l#@x14{&&X3ntr=>w$UV#@O`5x{iF zIke|9L3J^!J5(tCHpN43Jv(&ATig$^$=FL2IsPA@S@wjs3B!wC-l6~x24&Q)}jkEYI=BwX*u(;tbRx;uEs)C=WiL!YwJx%8<`%6z* zcUY**BN z*P;Q`l?4I-M6(dP2xHIi;T7VY#s~r2@5q;N_3Azqj z)q_Y@HHUt58{(8`-C`8rW^;aM8Cj#Wnm>LEwKlxHfXUz#{Yh zb6`_(o^i4`K+Xul8kerGFLi9i4!IL>9T#d#!`Kq?o@uf=TwWe1{43RuT+9DeF1;c$ z6TM?wR8a2+Px4IhYyo7&<*`2S7XUDrg+3>*q;{!q*Osr=zhv2uVRJE6d6Z-NmBl6X z0h4Aw(G)1bjU8eo@mcQZpl{KhyzFaE%$N(CsP%9q$bk7bO-Y=QsR#uV5*0R=Y?b@y zEeID+uQT3Om{<&m!P3aP_Yuw~+j{Mu9fwq>8oW6S>0KQtamBJ7Q4xO|%zqR*gT<6R za^EC)V2qrw=0(eHoe@FZ5bK^p?m2ohXC(;5ljqOy->tla5oj*6FmaypzUwJiL;wdl zmi_}kyHs-OHnf`1zIR3u8+6I2%HH(#FDgMr-MX;$o$?RF1jXmtY|8dl_r>8pNU3Y> zKbJb;9@UV<%R8R1tv)IXja_TKx|d3kR$~SeB<}*`z0}erX@{cldpUzswr&z~w!EO~ zULloWmuu*d7>$8wTa^xV_L0Rnxs=pS(iDOX!s-+a1Qu?U4)oE4YoTN^P^*Zk`tr5b zz<-T}LDd@xX@41%vtS*yM7C);cmRhT_P{+1=e!d8N)`5;v|+P3mi-Sv*29GeaYRPs zelikmsg*FFoQToQR!OE6)zSx=A#4EXoa;1nb3-r~{t&yt6SozW_~rY&*vUynjMu)3 zeXAY^pi>BfPoB#Q-)3HEif)~LA4PddcCR(Suo7nuy{0gG4w>gmKAVz#HIbnG@K;Tb zaKnt?W895*8GDsV8oVCM)fk8rM~9UEOlz+R2ym)(c_p*k(v_eLAH-Q@3b5;AmCY-1 z489{Bn+@XPA^0vgT@`-!6>Tuom48A8=b**BMA26bEADGa4-;&2PAfR@9^vmmcNMIT#=K;`DOKVN2JYSl=Gsb=Nfw?9M9kLAs}nO3YP#m}Y-PpQ54*Ll|7-F^PG-ZcouyxiU%%8pGAfk5r6IcuhY; zyoL>ywN%liPyX~g4PDW!GiS^ex8P&m7bG5lJ(HM>!1)hA9Ql!v*kLNP1@Hbz%`b-9 z3-v%>EyWzU2b-PW*bd~$ecFSZGp&?{`yz4}?9$e=r*Qz3lh35{M?vt8IUV8&O;=u4g!jf;Z6acu5b2+IGCOP$Jh3f-qpGpmB z2l50BuFm7QOKiV*LC8VnEW;ovozE!vw-NZsH}X_6EG9LoXd7XZWX*@-PwsgWRu^1n z@p7|d%q}W6)tq?c= zwpR$fiZvdYpxAy6cGW;RSX&$a8Y~CsMO5zOjK|HB`&%2-*aD}I{@K7m7&;nIbT4DV zv^G+>E4>Pz%UGw;R3OMU03b!(zn{9@9*o+=HVJ@8zK7{vK`+v9t>3(@A5&MS2hsx` z&a*IGi}b1-UTCievN%+R&Yv$Ge|o8%g94vD^AsuX&EpT)Kka1_ir5Iug8GcC!u#5y zPJ|E!tNGXO+feOauwB`>ciunEthY@aV6{EAYAsL%WNfDhyP^|g{nWbnjs?TJ`0zAx z_JI&(6;K9vX{*LODMR~ENe=EqE0j70PoCZY3_l9YzzBr-&mqyh_;>3;%f%& z?I%fJJs(J0*Wu zag@G~GfeEqD!i`D5VltiDZ_VgP&D^DH1%-D{4q8<*C&AiUW;od?TlxWX8J5NtZu9f z%Ze39Os!)c0Yop++@xbn4n-K~QzN>T>L=9>}EhOTP+5JokM@qZ!P5k(_d@N>@G0 zpk*BY+5?(39Dy=aiR0eU68sT!PKao=M(`?{XNFPKl-4vnZQ?d6ZrVfxu{>5g(d@bh z@AgqaU%<$4=#1q_{qq_ZNRHeHzL6?N9!E=lqW>oA!{p+IGRip*6U;K0+QlC4qbO5k zxq(kTD$G3CC<)_5>T41+*`#x@_C7x3z;Cl5J&5EDBLhzZG-@-NHJf{AbOLsNX!eIU zpS)w*xmb*eO$IceAW-PF&cfXD`_i_;Gs@K|cc%`pMQ+cbDWR9d%@fMO!OCn(9sb%T zz5Ak`r{%$g+fk`nN&W|3Ac)8b=O6OlHUCt>&JN^Dczf$Stk=^_*u?O^ks??CdVWlw zOGbPzC9*v;FHcgV-c?F7@v&asbK}1T)+GM{IL&6SLp8p{zk#qzv_i04VxWRXS%5)d zrhZU_;7Lib7YPX!Cq3`Q{8rT9j0QmEbc(*!iLKJe=hr#!w-Fw2|VuX_(^+#0L{Gju-+aD*i6Q9W8x%kiDM|*4kvn`KS z`C=2l!DU9*-apEhB@%%j7GMRKZ+^HujaT|NRDv!ITX!2*|1$lM5A6Z*b zKCsrel_}TmNOI>ouPYgt@f04zVXx7nOBN-@5r{{hJNv7>;% uGnxMYjZusI*su8Dhy5VzzA%!#+l%&y)F<`Z;atWwXe#o5qT{0eTmFCU9RAw? literal 0 HcmV?d00001 From af5615123168363ea08c70a80b9b2e9aa006ce27 Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Fri, 8 Oct 2021 19:47:07 +0000 Subject: [PATCH 251/262] highgui(docs): we don't support 32-bit integer images in imshow() --- modules/highgui/include/opencv2/highgui.hpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/modules/highgui/include/opencv2/highgui.hpp b/modules/highgui/include/opencv2/highgui.hpp index cdab0ada78..7ae2395737 100644 --- a/modules/highgui/include/opencv2/highgui.hpp +++ b/modules/highgui/include/opencv2/highgui.hpp @@ -383,10 +383,12 @@ cv::WINDOW_AUTOSIZE flag, the image is shown with its original size, however it Otherwise, the image is scaled to fit the window. The function may scale the image, depending on its depth: - If the image is 8-bit unsigned, it is displayed as is. -- If the image is 16-bit unsigned or 32-bit integer, the pixels are divided by 256. That is, the +- If the image is 16-bit unsigned, the pixels are divided by 256. That is, the value range [0,255\*256] is mapped to [0,255]. - If the image is 32-bit or 64-bit floating-point, the pixel values are multiplied by 255. That is, the value range [0,1] is mapped to [0,255]. +- 32-bit integer images are not processed anymore due to ambiguouty of required transform. + Convert to 8-bit unsigned matrix using a custom preprocessing specific to image's context. If window was created with OpenGL support, cv::imshow also support ogl::Buffer , ogl::Texture2D and cuda::GpuMat as input. From 788f330d07fe3dd229b3f2648a8090b61e555c70 Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Fri, 8 Oct 2021 20:09:43 +0000 Subject: [PATCH 252/262] js: fix build of SIMD tests --- modules/ts/src/precomp.hpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/modules/ts/src/precomp.hpp b/modules/ts/src/precomp.hpp index a9d4404e3b..2725f63429 100644 --- a/modules/ts/src/precomp.hpp +++ b/modules/ts/src/precomp.hpp @@ -1,7 +1,9 @@ #include "opencv2/ts.hpp" #include #include "opencv2/core/utility.hpp" +#if !defined(__EMSCRIPTEN__) #include "opencv2/core/private.hpp" +#endif #ifdef GTEST_LINKED_AS_SHARED_LIBRARY #error ts module should not have GTEST_LINKED_AS_SHARED_LIBRARY defined From 39ee5c5a461760b528e157960d36c567eba236f6 Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Fri, 8 Oct 2021 20:34:07 +0000 Subject: [PATCH 253/262] plugins: reverse candidates order to fetch higher versions first --- modules/core/src/parallel/plugin_parallel_wrapper.impl.hpp | 3 +++ modules/highgui/src/plugin_wrapper.impl.hpp | 3 +++ modules/videoio/src/backend_plugin.cpp | 3 +++ 3 files changed, 9 insertions(+) diff --git a/modules/core/src/parallel/plugin_parallel_wrapper.impl.hpp b/modules/core/src/parallel/plugin_parallel_wrapper.impl.hpp index a5649b60c0..b9b3105d20 100644 --- a/modules/core/src/parallel/plugin_parallel_wrapper.impl.hpp +++ b/modules/core/src/parallel/plugin_parallel_wrapper.impl.hpp @@ -223,6 +223,9 @@ std::vector getPluginCandidates(const std::string& baseName) continue; std::vector candidates; cv::glob(utils::fs::join(path, plugin_expr), candidates); + // Prefer candisates with higher versions + // TODO: implemented accurate versions-based comparator + std::sort(candidates.begin(), candidates.end(), std::greater()); CV_LOG_DEBUG(NULL, " - " << path << ": " << candidates.size()); copy(candidates.begin(), candidates.end(), back_inserter(results)); } diff --git a/modules/highgui/src/plugin_wrapper.impl.hpp b/modules/highgui/src/plugin_wrapper.impl.hpp index 97aea69098..23f0ecf6bf 100644 --- a/modules/highgui/src/plugin_wrapper.impl.hpp +++ b/modules/highgui/src/plugin_wrapper.impl.hpp @@ -224,6 +224,9 @@ std::vector getPluginCandidates(const std::string& baseName) continue; std::vector candidates; cv::glob(utils::fs::join(path, plugin_expr), candidates); + // Prefer candisates with higher versions + // TODO: implemented accurate versions-based comparator + std::sort(candidates.begin(), candidates.end(), std::greater()); CV_LOG_DEBUG(NULL, " - " << path << ": " << candidates.size()); copy(candidates.begin(), candidates.end(), back_inserter(results)); } diff --git a/modules/videoio/src/backend_plugin.cpp b/modules/videoio/src/backend_plugin.cpp index e42c98649c..039083c6ad 100644 --- a/modules/videoio/src/backend_plugin.cpp +++ b/modules/videoio/src/backend_plugin.cpp @@ -373,6 +373,9 @@ std::vector getPluginCandidates(const std::string& baseName) continue; std::vector candidates; cv::glob(utils::fs::join(path, plugin_expr), candidates); + // Prefer candisates with higher versions + // TODO: implemented accurate versions-based comparator + std::sort(candidates.begin(), candidates.end(), std::greater()); CV_LOG_INFO(NULL, " - " << path << ": " << candidates.size()); copy(candidates.begin(), candidates.end(), back_inserter(results)); } From ac57be91e12cd4021c813a1e3978746f8aa47cb8 Mon Sep 17 00:00:00 2001 From: Yaniv Hollander Date: Sat, 9 Oct 2021 11:43:50 -0400 Subject: [PATCH 254/262] Merge pull request #20721 from YanivHollander:DocFixes Doc fixes * Update imgproc.hpp * Update imgproc.hpp --- modules/imgproc/include/opencv2/imgproc.hpp | 70 ++++++++++----------- 1 file changed, 35 insertions(+), 35 deletions(-) diff --git a/modules/imgproc/include/opencv2/imgproc.hpp b/modules/imgproc/include/opencv2/imgproc.hpp index 24686228fd..9e5375ef4b 100644 --- a/modules/imgproc/include/opencv2/imgproc.hpp +++ b/modules/imgproc/include/opencv2/imgproc.hpp @@ -2745,13 +2745,6 @@ CV_EXPORTS_W void warpPolar(InputArray src, OutputArray dst, Size dsize, //! @addtogroup imgproc_misc //! @{ -/** @overload */ -CV_EXPORTS_W void integral( InputArray src, OutputArray sum, int sdepth = -1 ); - -/** @overload */ -CV_EXPORTS_AS(integral2) void integral( InputArray src, OutputArray sum, - OutputArray sqsum, int sdepth = -1, int sqdepth = -1 ); - /** @brief Calculates the integral of an image. The function calculates one or more integral images for the source image as follows: @@ -2790,6 +2783,13 @@ CV_EXPORTS_AS(integral3) void integral( InputArray src, OutputArray sum, OutputArray sqsum, OutputArray tilted, int sdepth = -1, int sqdepth = -1 ); +/** @overload */ +CV_EXPORTS_W void integral( InputArray src, OutputArray sum, int sdepth = -1 ); + +/** @overload */ +CV_EXPORTS_AS(integral2) void integral( InputArray src, OutputArray sum, + OutputArray sqsum, int sdepth = -1, int sqdepth = -1 ); + //! @} imgproc_misc //! @addtogroup imgproc_motion @@ -3472,19 +3472,6 @@ the first variant of the function and distanceType == #DIST_L1. CV_EXPORTS_W void distanceTransform( InputArray src, OutputArray dst, int distanceType, int maskSize, int dstType=CV_32F); -/** @example samples/cpp/ffilldemo.cpp -An example using the FloodFill technique -*/ - -/** @overload - -variant without `mask` parameter -*/ -CV_EXPORTS int floodFill( InputOutputArray image, - Point seedPoint, Scalar newVal, CV_OUT Rect* rect = 0, - Scalar loDiff = Scalar(), Scalar upDiff = Scalar(), - int flags = 4 ); - /** @brief Fills a connected component with the given color. The function cv::floodFill fills a connected component starting from the seed point with the specified @@ -3561,6 +3548,19 @@ CV_EXPORTS_W int floodFill( InputOutputArray image, InputOutputArray mask, Scalar loDiff = Scalar(), Scalar upDiff = Scalar(), int flags = 4 ); +/** @example samples/cpp/ffilldemo.cpp +An example using the FloodFill technique +*/ + +/** @overload + +variant without `mask` parameter +*/ +CV_EXPORTS int floodFill( InputOutputArray image, + Point seedPoint, Scalar newVal, CV_OUT Rect* rect = 0, + Scalar loDiff = Scalar(), Scalar upDiff = Scalar(), + int flags = 4 ); + //! Performs linear blending of two images: //! \f[ \texttt{dst}(i,j) = \texttt{weights1}(i,j)*\texttt{src1}(i,j) + \texttt{weights2}(i,j)*\texttt{src2}(i,j) \f] //! @param src1 It has a type of CV_8UC(n) or CV_32FC(n), where n is a positive integer. @@ -4399,7 +4399,7 @@ lines are drawn using Gaussian filtering. CV_EXPORTS_W void line(InputOutputArray img, Point pt1, Point pt2, const Scalar& color, int thickness = 1, int lineType = LINE_8, int shift = 0); -/** @brief Draws a arrow segment pointing from the first point to the second one. +/** @brief Draws an arrow segment pointing from the first point to the second one. The function cv::arrowedLine draws an arrow between pt1 and pt2 points in the image. See also #line. @@ -4529,11 +4529,6 @@ CV_EXPORTS_W void drawMarker(InputOutputArray img, Point position, const Scalar& /* END OF MARKER SECTION */ /* ----------------------------------------------------------------------------------------- */ -/** @overload */ -CV_EXPORTS void fillConvexPoly(InputOutputArray img, const Point* pts, int npts, - const Scalar& color, int lineType = LINE_8, - int shift = 0); - /** @brief Fills a convex polygon. The function cv::fillConvexPoly draws a filled convex polygon. This function is much faster than the @@ -4552,10 +4547,9 @@ CV_EXPORTS_W void fillConvexPoly(InputOutputArray img, InputArray points, int shift = 0); /** @overload */ -CV_EXPORTS void fillPoly(InputOutputArray img, const Point** pts, - const int* npts, int ncontours, - const Scalar& color, int lineType = LINE_8, int shift = 0, - Point offset = Point() ); +CV_EXPORTS void fillConvexPoly(InputOutputArray img, const Point* pts, int npts, + const Scalar& color, int lineType = LINE_8, + int shift = 0); /** @example samples/cpp/tutorial_code/ImgProc/basic_drawing/Drawing_1.cpp An example using drawing functions @@ -4580,9 +4574,10 @@ CV_EXPORTS_W void fillPoly(InputOutputArray img, InputArrayOfArrays pts, Point offset = Point() ); /** @overload */ -CV_EXPORTS void polylines(InputOutputArray img, const Point* const* pts, const int* npts, - int ncontours, bool isClosed, const Scalar& color, - int thickness = 1, int lineType = LINE_8, int shift = 0 ); +CV_EXPORTS void fillPoly(InputOutputArray img, const Point** pts, + const int* npts, int ncontours, + const Scalar& color, int lineType = LINE_8, int shift = 0, + Point offset = Point() ); /** @brief Draws several polygonal curves. @@ -4601,6 +4596,11 @@ CV_EXPORTS_W void polylines(InputOutputArray img, InputArrayOfArrays pts, bool isClosed, const Scalar& color, int thickness = 1, int lineType = LINE_8, int shift = 0 ); +/** @overload */ +CV_EXPORTS void polylines(InputOutputArray img, const Point* const* pts, const int* npts, + int ncontours, bool isClosed, const Scalar& color, + int thickness = 1, int lineType = LINE_8, int shift = 0 ); + /** @example samples/cpp/contours2.cpp An example program illustrates the use of cv::findContours and cv::drawContours \image html WindowsQtContoursOutput.png "Screenshot of the program" @@ -4633,7 +4633,7 @@ parameter is only taken into account when there is hierarchy available. @param offset Optional contour shift parameter. Shift all the drawn contours by the specified \f$\texttt{offset}=(dx,dy)\f$ . @note When thickness=#FILLED, the function is designed to handle connected components with holes correctly -even when no hierarchy date is provided. This is done by analyzing all the outlines together +even when no hierarchy data is provided. This is done by analyzing all the outlines together using even-odd rule. This may give incorrect results if you have a joint collection of separately retrieved contours. In order to solve this problem, you need to call #drawContours separately for each sub-group of contours, or iterate over the collection using contourIdx parameter. @@ -4647,7 +4647,7 @@ CV_EXPORTS_W void drawContours( InputOutputArray image, InputArrayOfArrays conto /** @brief Clips the line against the image rectangle. The function cv::clipLine calculates a part of the line segment that is entirely within the specified -rectangle. it returns false if the line segment is completely outside the rectangle. Otherwise, +rectangle. It returns false if the line segment is completely outside the rectangle. Otherwise, it returns true . @param imgSize Image size. The image rectangle is Rect(0, 0, imgSize.width, imgSize.height) . @param pt1 First line point. From 4223495e6cd67011f86b8ecd9be1fa105018f3b1 Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Sat, 9 Oct 2021 15:48:26 +0000 Subject: [PATCH 255/262] release: OpenCV 4.5.4 --- modules/core/include/opencv2/core/version.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/core/include/opencv2/core/version.hpp b/modules/core/include/opencv2/core/version.hpp index 4aaf30ce5e..2c9d0952e5 100644 --- a/modules/core/include/opencv2/core/version.hpp +++ b/modules/core/include/opencv2/core/version.hpp @@ -8,7 +8,7 @@ #define CV_VERSION_MAJOR 4 #define CV_VERSION_MINOR 5 #define CV_VERSION_REVISION 4 -#define CV_VERSION_STATUS "-pre" +#define CV_VERSION_STATUS "" #define CVAUX_STR_EXP(__A) #__A #define CVAUX_STR(__A) CVAUX_STR_EXP(__A) From a9d7b6eab7bf030b9e90e8467df00f8551915c84 Mon Sep 17 00:00:00 2001 From: Smirnov Egor Date: Mon, 11 Oct 2021 18:58:10 +0300 Subject: [PATCH 256/262] fix const - input and remove unimplemented function --- modules/dnn/src/onnx/onnx_importer.cpp | 67 +++++++++++++++++-------- modules/dnn/test/test_onnx_importer.cpp | 7 +++ 2 files changed, 52 insertions(+), 22 deletions(-) diff --git a/modules/dnn/src/onnx/onnx_importer.cpp b/modules/dnn/src/onnx/onnx_importer.cpp index 5343c05361..f497a17a7a 100644 --- a/modules/dnn/src/onnx/onnx_importer.cpp +++ b/modules/dnn/src/onnx/onnx_importer.cpp @@ -53,7 +53,6 @@ class ONNXImporter Mat getBlob(const std::string& input_name); LayerParams getLayerParams(const opencv_onnx::NodeProto& node_proto); - bool isCeilMode(const LayerParams& layerParams); void addConstant(const std::string& name, const Mat& blob); void addLayer(LayerParams& layerParams, @@ -61,6 +60,7 @@ class ONNXImporter void expandMid(const std::string& prefix, opencv_onnx::NodeProto& node_proto, const std::string& input, size_t n); + void addNegation(const LayerParams& layerParams, opencv_onnx::NodeProto& node_proto, int input_id); public: ONNXImporter(Net& net, const char *onnxFile) @@ -460,6 +460,32 @@ void ONNXImporter::expandMid(const std::string& prefix, opencv_onnx::NodeProto& } } +/** @brief Multiply one of node_proto inputs by -1 + * @param layerParams parameters of the node + * @param node_proto node which input will be replaced + * @param input_id id of input to be multiplied by -1 + */ +void ONNXImporter::addNegation(const LayerParams& layerParams, opencv_onnx::NodeProto& node_proto, int input_id) +{ + LayerParams powerParams; + powerParams.name = layerParams.name + "/neg"; + powerParams.type = "Power"; + powerParams.set("scale", -1.f); + + //Create Power layer + int id = dstNet.addLayer(powerParams.name, powerParams.type, powerParams); + //Connect to input + IterLayerId_t layerId = layer_id.find(node_proto.input(input_id)); + CV_Assert(layerId != layer_id.end()); + dstNet.connect(layerId->second.layerId, layerId->second.outputId, id, 0); + //Add shape + layer_id.insert(std::make_pair(powerParams.name, LayerInfo(id, 0))); + outShapes[powerParams.name] = outShapes[node_proto.input(input_id)]; + + //Replace input to Power + node_proto.set_input(input_id, powerParams.name); +} + void ONNXImporter::addConstant(const std::string& name, const Mat& blob) { constBlobs.insert(std::make_pair(name, blob)); @@ -918,29 +944,42 @@ void ONNXImporter::parseBias(LayerParams& layerParams, const opencv_onnx::NodePr else if (is_const_0 || is_const_1) { int const_blob_id = is_const_0 ? 0 : 1; + int input_id = 1 - const_blob_id; Mat blob = getBlob(node_proto, const_blob_id); int blob_total = blob.total(); + + const float inputScale = isSub && is_const_0 ? -1.f : 1.f; + const float constScale = isSub && is_const_1 ? -1.f : 1.f; + if (blob_total == 1) { layerParams.type = "Power"; - layerParams.set("shift", (isSub ? -1 : 1) * blob.ptr()[0]); + layerParams.set("scale", inputScale); + layerParams.set("shift", constScale * blob.ptr()[0]); } else { - MatShape inpShape = outShapes[node_proto.input(1 - const_blob_id)]; + MatShape inpShape = outShapes[node_proto.input(input_id)]; if (shape(blob) == inpShape) { LayerParams constParams; constParams.name = layerParams.name + "/const"; constParams.type = "Const"; - constParams.blobs.push_back((isSub ? -1 : 1) * blob); + constParams.blobs.push_back(blob); int id = dstNet.addLayer(constParams.name, constParams.type, constParams); layer_id.insert(std::make_pair(constParams.name, LayerInfo(id, 0))); outShapes[constParams.name] = shape(blob); layerParams.type = "Eltwise"; + float coeffs[] = {1., isSub ? -1.f : 1.f}; + layerParams.set("coeff", DictValue::arrayReal(coeffs, 2)); node_proto.set_input(const_blob_id, constParams.name); } else { + if (inputScale < 0.f) + { + addNegation(layerParams, node_proto, input_id); + } + layerParams.type = "Scale"; layerParams.set("bias_term", true); int axis = 1; @@ -955,7 +994,7 @@ void ONNXImporter::parseBias(LayerParams& layerParams, const opencv_onnx::NodePr } layerParams.set("axis", axis); blob = blob.reshape(1, 1); - layerParams.blobs.push_back((isSub ? -1 : 1) * blob); + layerParams.blobs.push_back(constScale * blob); } } } @@ -972,23 +1011,7 @@ void ONNXImporter::parseBias(LayerParams& layerParams, const opencv_onnx::NodePr { if (isSub) { - LayerParams powerParams; - powerParams.name = layerParams.name + "/neg"; - powerParams.type = "Power"; - powerParams.set("scale", -1); - - //Create Power layer - int id = dstNet.addLayer(powerParams.name, powerParams.type, powerParams); - //Connect to input - IterLayerId_t layerId = layer_id.find(node_proto.input(1)); - CV_Assert(layerId != layer_id.end()); - dstNet.connect(layerId->second.layerId, layerId->second.outputId, id, 0); - //Add shape - layer_id.insert(std::make_pair(powerParams.name, LayerInfo(id, 0))); - outShapes[powerParams.name] = outShapes[node_proto.input(1)]; - - //Replace input to Power - node_proto.set_input(1, powerParams.name); + addNegation(layerParams, node_proto, 1); } layerParams.type = "Scale"; layerParams.set("bias_term", true); diff --git a/modules/dnn/test/test_onnx_importer.cpp b/modules/dnn/test/test_onnx_importer.cpp index b4ecd4601c..daba29e547 100644 --- a/modules/dnn/test/test_onnx_importer.cpp +++ b/modules/dnn/test/test_onnx_importer.cpp @@ -931,6 +931,13 @@ TEST_P(Test_ONNX_layers, ConvResizePool1d) testONNXModels("conv_resize_pool_1d"); } +TEST_P(Test_ONNX_layers, SubFromConst) +{ + testONNXModels("sub_from_const1"); + testONNXModels("sub_from_const_eltwise"); + testONNXModels("sub_from_const_broadcast"); +} + INSTANTIATE_TEST_CASE_P(/*nothing*/, Test_ONNX_layers, dnnBackendsAndTargets()); class Test_ONNX_nets : public Test_ONNX_layers From 238dbffb48e80901a92938e8d80d0bff7de38dcc Mon Sep 17 00:00:00 2001 From: Smirnov Egor Date: Mon, 11 Oct 2021 20:59:44 +0300 Subject: [PATCH 257/262] change asserts for Sum --- modules/dnn/src/onnx/onnx_importer.cpp | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/modules/dnn/src/onnx/onnx_importer.cpp b/modules/dnn/src/onnx/onnx_importer.cpp index 5343c05361..a4484570b1 100644 --- a/modules/dnn/src/onnx/onnx_importer.cpp +++ b/modules/dnn/src/onnx/onnx_importer.cpp @@ -903,7 +903,19 @@ void ONNXImporter::parseBias(LayerParams& layerParams, const opencv_onnx::NodePr opencv_onnx::NodeProto node_proto = node_proto_; const std::string& layer_type = node_proto.op_type(); bool isSub = layer_type == "Sub"; - CV_CheckEQ(node_proto.input_size(), 2, ""); + CV_Assert((node_proto.input_size() == 2) || (layer_type == "Sum" && node_proto.input_size() > 2)); + + if (layer_type == "Sum" && node_proto.input_size() > 2) + { + for (int i = 0; i < node_proto.input_size(); ++i) + { + if (layer_id.find(node_proto.input(i)) == layer_id.end()) + { + CV_Error(Error::StsNotImplemented, "Sum of constants is not implemented for inputs > 2"); + } + } + } + bool is_const_0 = layer_id.find(node_proto.input(0)) == layer_id.end(); bool is_const_1 = layer_id.find(node_proto.input(1)) == layer_id.end(); if (is_const_0 && is_const_1) From 3d93675ff9463fd8247b482c07e19ad8a59c61ff Mon Sep 17 00:00:00 2001 From: Stanislaw Halik Date: Mon, 4 Oct 2021 14:42:51 +0200 Subject: [PATCH 258/262] fix link error on shared libs with -MT --- modules/highgui/CMakeLists.txt | 2 +- modules/imgcodecs/CMakeLists.txt | 2 +- samples/CMakeLists.txt | 6 ++++-- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/modules/highgui/CMakeLists.txt b/modules/highgui/CMakeLists.txt index 6e69961099..e48e2470ce 100644 --- a/modules/highgui/CMakeLists.txt +++ b/modules/highgui/CMakeLists.txt @@ -163,7 +163,7 @@ if(APPLE) add_apple_compiler_options(${the_module}) endif() -if(MSVC) +if(MSVC AND NOT BUILD_SHARED_LIBS AND BUILD_WITH_STATIC_CRT) set_target_properties(${the_module} PROPERTIES LINK_FLAGS "/NODEFAULTLIB:atlthunk.lib /NODEFAULTLIB:atlsd.lib /NODEFAULTLIB:libcmt.lib /DEBUG") endif() diff --git a/modules/imgcodecs/CMakeLists.txt b/modules/imgcodecs/CMakeLists.txt index 63edd45e78..10fb3278fb 100644 --- a/modules/imgcodecs/CMakeLists.txt +++ b/modules/imgcodecs/CMakeLists.txt @@ -134,7 +134,7 @@ if(APPLE) add_apple_compiler_options(${the_module}) endif() -if(MSVC) +if(MSVC AND NOT BUILD_SHARED_LIBS AND BUILD_WITH_STATIC_CRT) set_target_properties(${the_module} PROPERTIES LINK_FLAGS "/NODEFAULTLIB:atlthunk.lib /NODEFAULTLIB:atlsd.lib /NODEFAULTLIB:libcmt.lib /DEBUG") endif() diff --git a/samples/CMakeLists.txt b/samples/CMakeLists.txt index 4cc4c07d7b..87f5cfcf88 100644 --- a/samples/CMakeLists.txt +++ b/samples/CMakeLists.txt @@ -97,8 +97,10 @@ if(MSVC) endforeach(flag_var) set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} /NODEFAULTLIB:atlthunk.lib /NODEFAULTLIB:msvcrt.lib /NODEFAULTLIB:msvcrtd.lib") - set(CMAKE_EXE_LINKER_FLAGS_DEBUG "${CMAKE_EXE_LINKER_FLAGS_DEBUG} /NODEFAULTLIB:libcmt.lib") - set(CMAKE_EXE_LINKER_FLAGS_RELEASE "${CMAKE_EXE_LINKER_FLAGS_RELEASE} /NODEFAULTLIB:libcmtd.lib") + if(NOT BUILD_WITH_STATIC_CRT) + set(CMAKE_EXE_LINKER_FLAGS_DEBUG "${CMAKE_EXE_LINKER_FLAGS_DEBUG} /NODEFAULTLIB:libcmt.lib") + set(CMAKE_EXE_LINKER_FLAGS_RELEASE "${CMAKE_EXE_LINKER_FLAGS_RELEASE} /NODEFAULTLIB:libcmtd.lib") + endif() endif() endif() From 982503e9a8724465570ccab65e8e5c9656940b5f Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Wed, 13 Oct 2021 01:14:37 +0000 Subject: [PATCH 259/262] core: ensure 'int' result from CV_POPCNT_U64(x) --- modules/core/include/opencv2/core/cv_cpu_dispatch.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/core/include/opencv2/core/cv_cpu_dispatch.h b/modules/core/include/opencv2/core/cv_cpu_dispatch.h index 540fbb605c..e92798d6c9 100644 --- a/modules/core/include/opencv2/core/cv_cpu_dispatch.h +++ b/modules/core/include/opencv2/core/cv_cpu_dispatch.h @@ -55,7 +55,7 @@ # ifdef _MSC_VER # include # if defined(_M_X64) -# define CV_POPCNT_U64 _mm_popcnt_u64 +# define CV_POPCNT_U64 (int)_mm_popcnt_u64 # endif # define CV_POPCNT_U32 _mm_popcnt_u32 # else From e4a87f2f4f9276ed17551236eb4e08285e10824a Mon Sep 17 00:00:00 2001 From: icestraw Date: Wed, 13 Oct 2021 22:35:29 +0800 Subject: [PATCH 260/262] fix type error of slice indices --- doc/py_tutorials/py_imgproc/py_pyramids/py_pyramids.markdown | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/py_tutorials/py_imgproc/py_pyramids/py_pyramids.markdown b/doc/py_tutorials/py_imgproc/py_pyramids/py_pyramids.markdown index 63fde0a130..602ab29ad6 100644 --- a/doc/py_tutorials/py_imgproc/py_pyramids/py_pyramids.markdown +++ b/doc/py_tutorials/py_imgproc/py_pyramids/py_pyramids.markdown @@ -117,7 +117,7 @@ for i in range(5,0,-1): LS = [] for la,lb in zip(lpA,lpB): rows,cols,dpt = la.shape - ls = np.hstack((la[:,0:cols/2], lb[:,cols/2:])) + ls = np.hstack((la[:,0:cols//2], lb[:,cols//2:])) LS.append(ls) # now reconstruct @@ -127,7 +127,7 @@ for i in range(1,6): ls_ = cv.add(ls_, LS[i]) # image with direct connecting each half -real = np.hstack((A[:,:cols/2],B[:,cols/2:])) +real = np.hstack((A[:,:cols//2],B[:,cols//2:])) cv.imwrite('Pyramid_blending2.jpg',ls_) cv.imwrite('Direct_blending.jpg',real) From bd0732b1d03bcbc15d919d6dbe3dc45eac1b9514 Mon Sep 17 00:00:00 2001 From: Nicholas Ho <88894303+Nicholas-Ho-arm@users.noreply.github.com> Date: Fri, 15 Oct 2021 12:47:53 +0100 Subject: [PATCH 261/262] Merge pull request #20740 from Nicholas-Ho-arm:3.4_SymmColumnVec_32f8u * Add SymmColumnVec_32f8u * Fix double to float warnings --- modules/imgproc/src/filter.simd.hpp | 81 ++++++++++++++++++++++++++++- 1 file changed, 79 insertions(+), 2 deletions(-) diff --git a/modules/imgproc/src/filter.simd.hpp b/modules/imgproc/src/filter.simd.hpp index 94dbce060c..966de94c0b 100644 --- a/modules/imgproc/src/filter.simd.hpp +++ b/modules/imgproc/src/filter.simd.hpp @@ -1160,6 +1160,81 @@ struct SymmColumnVec_32s8u Mat kernel; }; +struct SymmColumnVec_32f8u +{ + SymmColumnVec_32f8u() { symmetryType = 0; delta = 0; } + SymmColumnVec_32f8u(const Mat& _kernel, int _symmetryType, int, double _delta) + { + symmetryType = _symmetryType; + kernel = _kernel; + delta = (float)_delta; + CV_Assert( (symmetryType & (KERNEL_SYMMETRICAL | KERNEL_ASYMMETRICAL)) != 0 ); + } + + int operator()(const uchar** _src, uchar* _dst, int width) const + { + CV_INSTRUMENT_REGION(); + + int _ksize = kernel.rows + kernel.cols - 1; + if( _ksize == 1 ) return 0; + const int ksize2 = _ksize / 2; + const float* ky = kernel.ptr() + ksize2; + int i = 0, k; + bool symmetrical = (symmetryType & KERNEL_SYMMETRICAL) != 0; + const float** src = (const float**)_src; + + if( symmetrical ) + { + for( ; i <= width - v_uint8::nlanes; i += v_uint8::nlanes ) + { + v_float32 v_ky0 = vx_setall_f32(ky[0]); + v_float32 v32_delta = vx_setall_f32(delta); + const float* S = src[0] + i; + v_float32 s0 = v_muladd(v_ky0, vx_load(S), v32_delta); + v_float32 s1 = v_muladd(v_ky0, vx_load(S + v_float32::nlanes), v32_delta); + v_float32 s2 = v_muladd(v_ky0, vx_load(S + 2*v_float32::nlanes), v32_delta); + v_float32 s3 = v_muladd(v_ky0, vx_load(S + 3*v_float32::nlanes), v32_delta); + for( k = 1; k <= ksize2; k++ ) + { + v_float32 v_kyk = vx_setall_f32(ky[k]); + const float* S0 = src[k] + i; + const float* S1 = src[-k] + i; + s0 = v_muladd(v_kyk, vx_load(S0) + vx_load(S1), s0); + s1 = v_muladd(v_kyk, vx_load(S0 + v_float32::nlanes) + vx_load(S1 + v_float32::nlanes), s1); + s2 = v_muladd(v_kyk, vx_load(S0 + 2*v_float32::nlanes) + vx_load(S1 + 2*v_float32::nlanes), s2); + s3 = v_muladd(v_kyk, vx_load(S0 + 3*v_float32::nlanes) + vx_load(S1 + 3*v_float32::nlanes), s3); + } + v_store(_dst + i, v_pack_u(v_pack(v_round(s0), v_round(s1)), v_pack(v_round(s2), v_round(s3)))); + } + } + else + { + for( ; i <= width - v_uint8::nlanes; i += v_uint8::nlanes ) + { + v_float32 s0 = vx_setall_f32(delta); + v_float32 s1 = vx_setall_f32(delta); + v_float32 s2 = vx_setall_f32(delta); + v_float32 s3 = vx_setall_f32(delta); + for( k = 1; k <= ksize2; k++ ) + { + v_float32 v_kyk = vx_setall_f32(ky[k]); + const float* S0 = src[k] + i; + const float* S1 = src[-k] + i; + s0 = v_muladd(v_kyk, vx_load(S0) - vx_load(S1), s0); + s1 = v_muladd(v_kyk, vx_load(S0 + v_float32::nlanes) - vx_load(S1 + v_float32::nlanes), s1); + s2 = v_muladd(v_kyk, vx_load(S0 + 2*v_float32::nlanes) - vx_load(S1 + 2*v_float32::nlanes), s2); + s3 = v_muladd(v_kyk, vx_load(S0 + 3*v_float32::nlanes) - vx_load(S1 + 3*v_float32::nlanes), s3); + } + v_store(_dst + i, v_pack_u(v_pack(v_round(s0), v_round(s1)), v_pack(v_round(s2), v_round(s3)))); + } + } + return i; + } + int symmetryType; + float delta; + Mat kernel; +}; + struct SymmColumnSmallVec_32s16s { @@ -2341,6 +2416,7 @@ typedef RowNoVec RowVec_32f; typedef SymmRowSmallNoVec SymmRowSmallVec_8u32s; typedef SymmRowSmallNoVec SymmRowSmallVec_32f; typedef ColumnNoVec SymmColumnVec_32s8u; +typedef ColumnNoVec SymmColumnVec_32f8u; typedef ColumnNoVec SymmColumnVec_32f16s; typedef ColumnNoVec SymmColumnVec_32f; typedef SymmColumnSmallNoVec SymmColumnSmallVec_32s16s; @@ -3031,8 +3107,9 @@ Ptr getLinearColumnFilter( (kernel, anchor, delta, symmetryType, FixedPtCastEx(bits), SymmColumnVec_32s8u(kernel, symmetryType, bits, delta)); if( ddepth == CV_8U && sdepth == CV_32F ) - return makePtr, ColumnNoVec> > - (kernel, anchor, delta, symmetryType); + return makePtr, SymmColumnVec_32f8u> > + (kernel, anchor, delta, symmetryType, Cast(), + SymmColumnVec_32f8u(kernel, symmetryType, 0, delta)); if( ddepth == CV_8U && sdepth == CV_64F ) return makePtr, ColumnNoVec> > (kernel, anchor, delta, symmetryType); From a5906827645e183a731ac6b7eeda0e7c8ebb0563 Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Fri, 15 Oct 2021 16:58:37 +0300 Subject: [PATCH 262/262] cmake: update installation of python extra submodules - support Python standalone builds - loader installs submodules unconditionally --- modules/python/CMakeLists.txt | 5 -- modules/python/bindings/CMakeLists.txt | 4 + modules/python/common.cmake | 40 --------- modules/python/package/cv2/__init__.py | 3 +- modules/python/python3/CMakeLists.txt | 14 ---- modules/python/python_loader.cmake | 110 +++++++++++++++++++------ 6 files changed, 89 insertions(+), 87 deletions(-) diff --git a/modules/python/CMakeLists.txt b/modules/python/CMakeLists.txt index a51acf386e..c6a9075224 100644 --- a/modules/python/CMakeLists.txt +++ b/modules/python/CMakeLists.txt @@ -20,11 +20,6 @@ add_subdirectory(bindings) add_subdirectory(test) -if(NOT OPENCV_SKIP_PYTHON_LOADER) - include("./python_loader.cmake") - message(STATUS "OpenCV Python: during development append to PYTHONPATH: ${CMAKE_BINARY_DIR}/python_loader") -endif() - if(__disable_python2) ocv_module_disable_(python2) endif() diff --git a/modules/python/bindings/CMakeLists.txt b/modules/python/bindings/CMakeLists.txt index 442107b135..b39c679615 100644 --- a/modules/python/bindings/CMakeLists.txt +++ b/modules/python/bindings/CMakeLists.txt @@ -8,6 +8,10 @@ set(OPENCV_PYTHON_BINDINGS_DIR "${CMAKE_CURRENT_BINARY_DIR}" CACHE INTERNAL "") # This file is included from a subdirectory set(PYTHON_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../") +if(NOT OPENCV_SKIP_PYTHON_LOADER) + include("${PYTHON_SOURCE_DIR}/python_loader.cmake") +endif() + # get list of modules to wrap set(OPENCV_PYTHON_MODULES) foreach(m ${OPENCV_MODULES_BUILD}) diff --git a/modules/python/common.cmake b/modules/python/common.cmake index 251b78c6cb..ebbb2e2f65 100644 --- a/modules/python/common.cmake +++ b/modules/python/common.cmake @@ -1,31 +1,6 @@ # This file is included from a subdirectory set(PYTHON_SOURCE_DIR "${CMAKE_CURRENT_LIST_DIR}") -function(ocv_add_python_files_from_path search_path) - file(GLOB_RECURSE extra_py_files - RELATIVE "${search_path}" - # Plain Python code - "${search_path}/*.py" - # Type annotations - "${search_path}/*.pyi" - ) - message(DEBUG "Extra Py files for ${search_path}: ${extra_py_files}") - if(extra_py_files) - list(SORT extra_py_files) - foreach(filename ${extra_py_files}) - get_filename_component(module "${filename}" DIRECTORY) - if(NOT ${module} IN_LIST extra_modules) - list(APPEND extra_modules ${module}) - endif() - configure_file("${search_path}/${filename}" "${__loader_path}/cv2/${filename}" COPYONLY) - install(FILES "${search_path}/${filename}" DESTINATION "${OPENCV_PYTHON_INSTALL_PATH}/cv2/${module}/" COMPONENT python) - endforeach() - message(STATUS "Found ${extra_modules} Python modules from ${search_path}") - else() - message(WARNING "Can't add Python files and modules from ${module_path}. There is no .py or .pyi files") - endif() -endfunction() - ocv_add_module(${MODULE_NAME} BINDINGS PRIVATE_REQUIRED opencv_python_bindings_generator) include_directories(SYSTEM @@ -243,21 +218,6 @@ if(NOT OPENCV_SKIP_PYTHON_LOADER) endif() configure_file("${PYTHON_SOURCE_DIR}/package/template/config-x.y.py.in" "${__python_loader_install_tmp_path}/cv2/${__target_config}" @ONLY) install(FILES "${__python_loader_install_tmp_path}/cv2/${__target_config}" DESTINATION "${OPENCV_PYTHON_INSTALL_PATH}/cv2/" COMPONENT python) - - # handle Python extra code - foreach(m ${OPENCV_MODULES_BUILD}) - if (";${OPENCV_MODULE_${m}_WRAPPERS};" MATCHES ";python;" AND HAVE_${m} - AND EXISTS "${OPENCV_MODULE_${m}_LOCATION}/misc/python/package" - ) - ocv_add_python_files_from_path("${OPENCV_MODULE_${m}_LOCATION}/misc/python/package") - endif() - endforeach(m) - - if(NOT "${OCV_PYTHON_EXTRA_MODULES_PATH}" STREQUAL "") - foreach(extra_ocv_py_modules_path ${OCV_PYTHON_EXTRA_MODULES_PATH}) - ocv_add_python_files_from_path(${extra_ocv_py_modules_path}) - endforeach() - endif() endif() # NOT OPENCV_SKIP_PYTHON_LOADER unset(PYTHON_SRC_DIR) diff --git a/modules/python/package/cv2/__init__.py b/modules/python/package/cv2/__init__.py index 80e2612276..07d1e0d21e 100644 --- a/modules/python/package/cv2/__init__.py +++ b/modules/python/package/cv2/__init__.py @@ -41,7 +41,7 @@ def __load_extra_py_code_for_module(base, name, enable_debug_print=False): setattr(py_module, "_native", native_module) for k, v in filter(lambda kv: not hasattr(py_module, kv[0]), native_module.__dict__.items()): - if enable_debug_print: print(' symbol: {} = {}'.format(k, v)) + if enable_debug_print: print(' symbol({}): {} = {}'.format(name, k, v)) setattr(py_module, k, v) return True @@ -51,6 +51,7 @@ def __collect_extra_submodules(enable_debug_print=False): return all(( # module is not internal not module.startswith("_"), + not module.startswith("python-"), # it is not a file os.path.isdir(os.path.join(_extra_submodules_init_path, module)) )) diff --git a/modules/python/python3/CMakeLists.txt b/modules/python/python3/CMakeLists.txt index 0bb401f2ec..d95af21e04 100644 --- a/modules/python/python3/CMakeLists.txt +++ b/modules/python/python3/CMakeLists.txt @@ -15,23 +15,9 @@ set(the_description "The python3 bindings") set(MODULE_NAME python3) set(MODULE_INSTALL_SUBDIR python3) -set(_ocv_extra_modules_path ${CMAKE_CURRENT_LIST_DIR}/../package/extra_modules) -set(_old_ocv_python_extra_modules_path ${OCV_PYTHON_EXTRA_MODULES_PATH}) - -if("${OCV_PYTHON_EXTRA_MODULES_PATH}" STREQUAL "") - set(OCV_PYTHON_EXTRA_MODULES_PATH ${_ocv_extra_modules_path}) -else() - list(APPEND OCV_PYTHON_EXTRA_MODULES_PATH ${_ocv_extra_modules_path}) -endif() - -unset(_ocv_extra_modules_path) - set(PYTHON PYTHON3) include(../common.cmake) -set(OCV_PYTHON_EXTRA_MODULES_PATH ${_old_ocv_python_extra_modules_path}) - -unset(_old_ocv_python_extra_modules_path) unset(MODULE_NAME) unset(MODULE_INSTALL_SUBDIR) diff --git a/modules/python/python_loader.cmake b/modules/python/python_loader.cmake index a872ffd763..670d85154c 100644 --- a/modules/python/python_loader.cmake +++ b/modules/python/python_loader.cmake @@ -22,6 +22,12 @@ else() set(CMAKE_PYTHON_EXTENSION_INSTALL_PATH_BASE "os.path.join(LOADER_DIR, 'not_installed')") endif() +if(OpenCV_FOUND) + return() # Ignore "standalone" builds of Python bindings +endif() + + + set(PYTHON_LOADER_FILES "setup.py" "cv2/__init__.py" "cv2/load_config_py2.py" "cv2/load_config_py3.py" @@ -39,34 +45,84 @@ foreach(fname ${PYTHON_LOADER_FILES}) endif() endforeach() -if(NOT OpenCV_FOUND) # Ignore "standalone" builds of Python bindings - if(WIN32) - if(CMAKE_GENERATOR MATCHES "Visual Studio") - list(APPEND CMAKE_PYTHON_BINARIES_PATH "'${EXECUTABLE_OUTPUT_PATH}/Release'") # TODO: CMAKE_BUILD_TYPE is not defined - else() - list(APPEND CMAKE_PYTHON_BINARIES_PATH "'${EXECUTABLE_OUTPUT_PATH}'") - endif() + + +if(WIN32) + if(CMAKE_GENERATOR MATCHES "Visual Studio") + list(APPEND CMAKE_PYTHON_BINARIES_PATH "'${EXECUTABLE_OUTPUT_PATH}/Release'") # TODO: CMAKE_BUILD_TYPE is not defined else() - list(APPEND CMAKE_PYTHON_BINARIES_PATH "'${LIBRARY_OUTPUT_PATH}'") + list(APPEND CMAKE_PYTHON_BINARIES_PATH "'${EXECUTABLE_OUTPUT_PATH}'") + endif() +else() + list(APPEND CMAKE_PYTHON_BINARIES_PATH "'${LIBRARY_OUTPUT_PATH}'") +endif() +string(REPLACE ";" ",\n " CMAKE_PYTHON_BINARIES_PATH "${CMAKE_PYTHON_BINARIES_PATH}") +configure_file("${PYTHON_SOURCE_DIR}/package/template/config.py.in" "${__loader_path}/cv2/config.py" @ONLY) + + + +# install +if(DEFINED OPENCV_PYTHON_INSTALL_PATH) + if(WIN32) + list(APPEND CMAKE_PYTHON_BINARIES_INSTALL_PATH "os.path.join(${CMAKE_PYTHON_EXTENSION_INSTALL_PATH_BASE}, '${OPENCV_BIN_INSTALL_PATH}')") + else() + list(APPEND CMAKE_PYTHON_BINARIES_INSTALL_PATH "os.path.join(${CMAKE_PYTHON_EXTENSION_INSTALL_PATH_BASE}, '${OPENCV_LIB_INSTALL_PATH}')") + endif() + set(CMAKE_PYTHON_BINARIES_PATH "${CMAKE_PYTHON_BINARIES_INSTALL_PATH}") + if (WIN32 AND HAVE_CUDA) + if (DEFINED CUDA_TOOLKIT_ROOT_DIR) + list(APPEND CMAKE_PYTHON_BINARIES_PATH "os.path.join(os.getenv('CUDA_PATH', '${CUDA_TOOLKIT_ROOT_DIR}'), 'bin')") + endif() endif() string(REPLACE ";" ",\n " CMAKE_PYTHON_BINARIES_PATH "${CMAKE_PYTHON_BINARIES_PATH}") - configure_file("${PYTHON_SOURCE_DIR}/package/template/config.py.in" "${__loader_path}/cv2/config.py" @ONLY) - - # install - if(DEFINED OPENCV_PYTHON_INSTALL_PATH) - if(WIN32) - list(APPEND CMAKE_PYTHON_BINARIES_INSTALL_PATH "os.path.join(${CMAKE_PYTHON_EXTENSION_INSTALL_PATH_BASE}, '${OPENCV_BIN_INSTALL_PATH}')") - else() - list(APPEND CMAKE_PYTHON_BINARIES_INSTALL_PATH "os.path.join(${CMAKE_PYTHON_EXTENSION_INSTALL_PATH_BASE}, '${OPENCV_LIB_INSTALL_PATH}')") - endif() - set(CMAKE_PYTHON_BINARIES_PATH "${CMAKE_PYTHON_BINARIES_INSTALL_PATH}") - if (WIN32 AND HAVE_CUDA) - if (DEFINED CUDA_TOOLKIT_ROOT_DIR) - list(APPEND CMAKE_PYTHON_BINARIES_PATH "os.path.join(os.getenv('CUDA_PATH', '${CUDA_TOOLKIT_ROOT_DIR}'), 'bin')") - endif() - endif() - string(REPLACE ";" ",\n " CMAKE_PYTHON_BINARIES_PATH "${CMAKE_PYTHON_BINARIES_PATH}") - configure_file("${PYTHON_SOURCE_DIR}/package/template/config.py.in" "${__python_loader_install_tmp_path}/cv2/config.py" @ONLY) - install(FILES "${__python_loader_install_tmp_path}/cv2/config.py" DESTINATION "${OPENCV_PYTHON_INSTALL_PATH}/cv2/" COMPONENT python) - endif() + configure_file("${PYTHON_SOURCE_DIR}/package/template/config.py.in" "${__python_loader_install_tmp_path}/cv2/config.py" @ONLY) + install(FILES "${__python_loader_install_tmp_path}/cv2/config.py" DESTINATION "${OPENCV_PYTHON_INSTALL_PATH}/cv2/" COMPONENT python) +endif() + + + +# +# Handle Python extra code (submodules) +# +function(ocv_add_python_files_from_path search_path) + file(GLOB_RECURSE extra_py_files + RELATIVE "${search_path}" + # Plain Python code + "${search_path}/*.py" + # Type annotations + "${search_path}/*.pyi" + ) + ocv_debug_message("Extra Py files for ${search_path}: ${extra_py_files}") + if(extra_py_files) + list(SORT extra_py_files) + foreach(filename ${extra_py_files}) + get_filename_component(module "${filename}" DIRECTORY) + if(NOT ${module} IN_LIST extra_modules) + list(APPEND extra_modules ${module}) + endif() + configure_file("${search_path}/${filename}" "${__loader_path}/cv2/${filename}" COPYONLY) + if(DEFINED OPENCV_PYTHON_INSTALL_PATH) + install(FILES "${search_path}/${filename}" DESTINATION "${OPENCV_PYTHON_INSTALL_PATH}/cv2/${module}/" COMPONENT python) + endif() + endforeach() + message(STATUS "Found '${extra_modules}' Python modules from ${search_path}") + else() + message(WARNING "Can't add Python files and modules from '${module_path}'. There is no .py or .pyi files") + endif() +endfunction() + +ocv_add_python_files_from_path("${PYTHON_SOURCE_DIR}/package/extra_modules") + +foreach(m ${OPENCV_MODULES_BUILD}) + if (";${OPENCV_MODULE_${m}_WRAPPERS};" MATCHES ";python;" AND HAVE_${m} + AND EXISTS "${OPENCV_MODULE_${m}_LOCATION}/misc/python/package" + ) + ocv_add_python_files_from_path("${OPENCV_MODULE_${m}_LOCATION}/misc/python/package") + endif() +endforeach(m) + +if(NOT "${OPENCV_PYTHON_EXTRA_MODULES_PATH}" STREQUAL "") + foreach(extra_ocv_py_modules_path ${OPENCV_PYTHON_EXTRA_MODULES_PATH}) + ocv_add_python_files_from_path(${extra_ocv_py_modules_path}) + endforeach() endif()