Merge remote-tracking branch 'upstream/3.4' into merge-3.4

modules/dnn/src/dnn.cpp

@@ -1757,6 +1757,7 @@ struct Net::Impl
 
             Ptr<InfEngineBackendNode> ieNode = node.dynamicCast<InfEngineBackendNode>();
             CV_Assert(!ieNode.empty());
+            ieNode->net->reset();
 
             for (it = layers.begin(); it != layers.end(); ++it)
             {
@@ -2058,6 +2059,7 @@ struct Net::Impl
 
             Ptr<InfEngineNgraphNode> ieNode = node.dynamicCast<InfEngineNgraphNode>();
             CV_Assert(!ieNode.empty());
+            ieNode->net->reset();
 
             for (it = layers.begin(); it != layers.end(); ++it)
             {

modules/dnn/src/ie_ngraph.cpp

@@ -780,6 +780,13 @@ void forwardNgraph(const std::vector<Ptr<BackendWrapper> >& outBlobsWrappers,
     ieNode->net->forward(outBlobsWrappers, isAsync);
 }
 
+void InfEngineNgraphNet::reset()
+{
+    allBlobs.clear();
+    infRequests.clear();
+    isInit = false;
+}
+
 void InfEngineNgraphNet::addBlobs(const std::vector<cv::Ptr<BackendWrapper> >& ptrs)
 {
     auto wrappers = ngraphWrappers(ptrs);

modules/dnn/src/ie_ngraph.hpp

@@ -52,6 +52,8 @@ public:
 
     void createNet(Target targetId);
     void setNodePtr(std::shared_ptr<ngraph::Node>* ptr);
+
+    void reset();
 private:
     void release();
     int getNumComponents();

modules/dnn/src/layers/region_layer.cpp

@@ -50,11 +50,16 @@
 #include "opencl_kernels_dnn.hpp"
 #endif
 
+#ifdef HAVE_DNN_NGRAPH
+#include "../ie_ngraph.hpp"
+#endif
+
 #ifdef HAVE_CUDA
 #include "../cuda4dnn/primitives/region.hpp"
 using namespace cv::dnn::cuda4dnn;
 #endif
 
+
 namespace cv
 {
 namespace dnn
@@ -93,12 +98,6 @@ public:
             CV_Error(cv::Error::StsNotImplemented, "Yolo9000 is not implemented");
     }
 
-    virtual bool supportBackend(int backendId) CV_OVERRIDE
-    {
-        return backendId == DNN_BACKEND_OPENCV ||
-               backendId == DNN_BACKEND_CUDA;
-    }
-
     bool getMemoryShapes(const std::vector<MatShape> &inputs,
                          const int requiredOutputs,
                          std::vector<MatShape> &outputs,
@@ -115,6 +114,19 @@ public:
         return false;
     }
 
+    virtual bool supportBackend(int backendId) CV_OVERRIDE
+    {
+#ifdef HAVE_DNN_NGRAPH
+    if (backendId == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH)
+        return INF_ENGINE_VER_MAJOR_GE(INF_ENGINE_RELEASE_2020_2) && preferableTarget != DNN_TARGET_MYRIAD;
+#endif
+#ifdef HAVE_CUDA
+        if (backendId == DNN_BACKEND_CUDA)
+            return true;
+#endif
+        return backendId == DNN_BACKEND_OPENCV;
+    }
+
     float logistic_activate(float x) { return 1.F / (1.F + exp(-x)); }
 
     void softmax_activate(const float* input, const int n, const float temp, float* output)
@@ -411,6 +423,202 @@ public:
         }
         return flops;
     }
+
+#ifdef HAVE_DNN_NGRAPH
+    virtual Ptr<BackendNode> initNgraph(const std::vector<Ptr<BackendWrapper> > &inputs,
+                                        const std::vector<Ptr<BackendNode> >& nodes) CV_OVERRIDE
+    {
+        auto& input = nodes[0].dynamicCast<InfEngineNgraphNode>()->node;
+        auto parent_shape = input->get_shape();
+        int64_t b = parent_shape[0];
+        int64_t h = parent_shape[1];
+        int64_t w = parent_shape[2];
+        int64_t c = parent_shape[3];
+
+        int64_t cols = b * h * w * anchors;
+        int64_t rows = c / anchors;
+        auto shape_node = std::make_shared<ngraph::op::Constant>(ngraph::element::i64, ngraph::Shape{2},  std::vector<int64_t>{cols, rows});
+        auto tr_axes = std::make_shared<ngraph::op::Constant>(ngraph::element::i64, ngraph::Shape{2}, std::vector<int64_t>{1, 0});
+
+        std::shared_ptr<ngraph::Node> input2d;
+        {
+            input2d = std::make_shared<ngraph::op::v1::Reshape>(input, shape_node, true);
+            input2d = std::make_shared<ngraph::op::Transpose>(input2d, tr_axes);
+        }
+
+        std::shared_ptr<ngraph::Node> region;
+        {
+            auto new_axes = std::make_shared<ngraph::op::Constant>(ngraph::element::i64, ngraph::Shape{4}, std::vector<int64_t>{0, 3, 1, 2});
+            auto tr_input = std::make_shared<ngraph::op::Transpose>(input, new_axes);
+
+            std::vector<float> anchors_vec(blobs[0].ptr<float>(), blobs[0].ptr<float>() + blobs[0].total());
+            std::vector<int64_t> mask(anchors, 1);
+            region = std::make_shared<ngraph::op::RegionYolo>(tr_input, coords, classes, anchors, useSoftmax, mask, 1, 3, anchors_vec);
+
+            auto shape_as_inp = std::make_shared<ngraph::op::Constant>(ngraph::element::i64,
+                                                                       ngraph::Shape{tr_input->get_shape().size()}, tr_input->get_shape().data());
+
+            region = std::make_shared<ngraph::op::v1::Reshape>(region, shape_as_inp, true);
+            new_axes = std::make_shared<ngraph::op::Constant>(ngraph::element::i64, ngraph::Shape{4}, std::vector<int64_t>{0, 2, 3, 1});
+            region = std::make_shared<ngraph::op::Transpose>(region, new_axes);
+
+            region = std::make_shared<ngraph::op::v1::Reshape>(region, shape_node, true);
+            region = std::make_shared<ngraph::op::Transpose>(region, tr_axes);
+        }
+
+        auto strides = std::make_shared<ngraph::op::Constant>(ngraph::element::i64, ngraph::Shape{2}, std::vector<int64_t>{1, 1});
+        std::vector<int64_t> boxes_shape{b, anchors, h, w};
+        auto shape_3d = std::make_shared<ngraph::op::Constant>(ngraph::element::i64, ngraph::Shape{boxes_shape.size()}, boxes_shape.data());
+
+        ngraph::Shape box_broad_shape{1, (size_t)anchors, (size_t)h, (size_t)w};
+
+        std::shared_ptr<ngraph::Node> box_x;
+        {
+            auto lower_bounds = std::make_shared<ngraph::op::Constant>(ngraph::element::i64, ngraph::Shape{2}, std::vector<int64_t>{0, 0});
+            auto upper_bounds = std::make_shared<ngraph::op::Constant>(ngraph::element::i64, ngraph::Shape{2}, std::vector<int64_t>{1, cols});
+            box_x = std::make_shared<ngraph::op::v1::StridedSlice>(input2d, lower_bounds, upper_bounds, strides, std::vector<int64_t>{}, std::vector<int64_t>{});
+            box_x = std::make_shared<ngraph::op::Sigmoid>(box_x);
+            box_x = std::make_shared<ngraph::op::v1::Reshape>(box_x, shape_3d, true);
+
+            std::vector<float> x_indices(w * h * anchors);
+            auto begin = x_indices.begin();
+            for (int i = 0; i < h; i++)
+            {
+                std::fill(begin + i * anchors, begin + (i + 1) * anchors, i);
+            }
+
+            for (int j = 1; j < w; j++)
+            {
+                std::copy(begin, begin + h * anchors, begin + j * h * anchors);
+            }
+            auto horiz = std::make_shared<ngraph::op::Constant>(ngraph::element::f32, box_broad_shape, x_indices.data());
+            box_x = std::make_shared<ngraph::op::v1::Add>(box_x, horiz, ngraph::op::AutoBroadcastType::NUMPY);
+
+            auto cols_node = std::make_shared<ngraph::op::Constant>(ngraph::element::f32, ngraph::Shape{1}, std::vector<float>{float(w)});
+            box_x = std::make_shared<ngraph::op::v1::Divide>(box_x, cols_node, ngraph::op::AutoBroadcastType::NUMPY);
+        }
+
+        std::shared_ptr<ngraph::Node> box_y;
+        {
+            auto lower_bounds = std::make_shared<ngraph::op::Constant>(ngraph::element::i64, ngraph::Shape{2}, std::vector<int64_t>{1, 0});
+            auto upper_bounds = std::make_shared<ngraph::op::Constant>(ngraph::element::i64, ngraph::Shape{2}, std::vector<int64_t>{2, cols});
+            box_y = std::make_shared<ngraph::op::v1::StridedSlice>(input2d, lower_bounds, upper_bounds, strides, std::vector<int64_t>{}, std::vector<int64_t>{});
+            box_y = std::make_shared<ngraph::op::Sigmoid>(box_y);
+            box_y = std::make_shared<ngraph::op::v1::Reshape>(box_y, shape_3d, true);
+
+            std::vector<float> y_indices(h * anchors);
+            for (int i = 0; i < h; i++)
+            {
+                std::fill(y_indices.begin() + i * anchors, y_indices.begin() + (i + 1) * anchors, i);
+            }
+
+            auto vert = std::make_shared<ngraph::op::Constant>(ngraph::element::f32, ngraph::Shape{1, (size_t)anchors, (size_t)h, 1}, y_indices.data());
+            box_y = std::make_shared<ngraph::op::v1::Add>(box_y, vert, ngraph::op::AutoBroadcastType::NUMPY);
+            auto rows_node = std::make_shared<ngraph::op::Constant>(ngraph::element::f32, ngraph::Shape{1}, std::vector<float>{float(h)});
+            box_y = std::make_shared<ngraph::op::v1::Divide>(box_y, rows_node, ngraph::op::AutoBroadcastType::NUMPY);
+        }
+
+        std::shared_ptr<ngraph::Node> box_w, box_h;
+        {
+            int hNorm, wNorm;
+            if (nodes.size() > 1)
+            {
+                auto node_1_shape = nodes[1].dynamicCast<InfEngineNgraphNode>()->node->get_shape();
+                hNorm = node_1_shape[2];
+                wNorm = node_1_shape[3];
+            }
+            else
+            {
+                hNorm = h;
+                wNorm = w;
+            }
+
+            std::vector<float> anchors_w(anchors), anchors_h(anchors);
+            for (size_t a = 0; a < anchors; ++a)
+            {
+                anchors_w[a] = blobs[0].at<float>(0, 2 * a) / wNorm;
+                anchors_h[a] = blobs[0].at<float>(0, 2 * a + 1) / hNorm;
+            }
+
+            std::vector<float> bias_w(w * h * anchors), bias_h(w * h * anchors);
+            for (int j = 0; j < h; j++)
+            {
+                std::copy(anchors_w.begin(), anchors_w.end(), bias_w.begin() + j * anchors);
+                std::copy(anchors_h.begin(), anchors_h.end(), bias_h.begin() + j * anchors);
+            }
+
+            for (int i = 1; i < w; i++)
+            {
+                std::copy(bias_w.begin(), bias_w.begin() + h * anchors, bias_w.begin() + i * h * anchors);
+                std::copy(bias_h.begin(), bias_h.begin() + h * anchors, bias_h.begin() + i * h * anchors);
+            }
+
+            auto lower_bounds = std::make_shared<ngraph::op::Constant>(ngraph::element::i64, ngraph::Shape{2}, std::vector<int64_t>{2, 0});
+            auto upper_bounds = std::make_shared<ngraph::op::Constant>(ngraph::element::i64, ngraph::Shape{2}, std::vector<int64_t>{3, cols});
+            box_w = std::make_shared<ngraph::op::v1::StridedSlice>(input2d, lower_bounds, upper_bounds, strides, std::vector<int64_t>{}, std::vector<int64_t>{});
+            box_w = std::make_shared<ngraph::op::v0::Exp>(box_w);
+            box_w = std::make_shared<ngraph::op::v1::Reshape>(box_w, shape_3d, true);
+            auto anchor_w_node = std::make_shared<ngraph::op::Constant>(ngraph::element::f32, box_broad_shape, bias_w.data());
+            box_w = std::make_shared<ngraph::op::v1::Multiply>(box_w, anchor_w_node, ngraph::op::AutoBroadcastType::NUMPY);
+
+            lower_bounds = std::make_shared<ngraph::op::Constant>(ngraph::element::i64, ngraph::Shape{2}, std::vector<int64_t>{3, 0});
+            upper_bounds = std::make_shared<ngraph::op::Constant>(ngraph::element::i64, ngraph::Shape{2}, std::vector<int64_t>{4, cols});
+            box_h = std::make_shared<ngraph::op::v1::StridedSlice>(input2d, lower_bounds, upper_bounds, strides, std::vector<int64_t>{}, std::vector<int64_t>{});
+            box_h = std::make_shared<ngraph::op::v0::Exp>(box_h);
+            box_h = std::make_shared<ngraph::op::v1::Reshape>(box_h, shape_3d, true);
+            auto anchor_h_node = std::make_shared<ngraph::op::Constant>(ngraph::element::f32, box_broad_shape, bias_h.data());
+            box_h = std::make_shared<ngraph::op::v1::Multiply>(box_h, anchor_h_node, ngraph::op::AutoBroadcastType::NUMPY);
+        }
+
+        std::shared_ptr<ngraph::Node> scale;
+        {
+            auto lower_bounds = std::make_shared<ngraph::op::Constant>(ngraph::element::i64, ngraph::Shape{2}, std::vector<int64_t>{4, 0});
+            auto upper_bounds = std::make_shared<ngraph::op::Constant>(ngraph::element::i64, ngraph::Shape{2}, std::vector<int64_t>{5, cols});
+            scale = std::make_shared<ngraph::op::v1::StridedSlice>(region, lower_bounds, upper_bounds, strides, std::vector<int64_t>{}, std::vector<int64_t>{});
+
+            if (classfix == -1)
+            {
+                auto thresh_node = std::make_shared<ngraph::op::Constant>(ngraph::element::f32, ngraph::Shape{1}, std::vector<float>{0.5});
+                auto mask = std::make_shared<ngraph::op::v1::Less>(scale, thresh_node);
+                auto zero_node = std::make_shared<ngraph::op::Constant>(ngraph::element::f32, mask->get_shape(), std::vector<float>(b * cols, 0));
+                scale = std::make_shared<ngraph::op::v1::Select>(mask, scale, zero_node);
+            }
+        }
+
+        std::shared_ptr<ngraph::Node> probs;
+        {
+            auto lower_bounds = std::make_shared<ngraph::op::Constant>(ngraph::element::i64, ngraph::Shape{2}, std::vector<int64_t>{5, 0});
+            auto upper_bounds = std::make_shared<ngraph::op::Constant>(ngraph::element::i64, ngraph::Shape{2}, std::vector<int64_t>{rows, cols});
+            auto classes = std::make_shared<ngraph::op::v1::StridedSlice>(region, lower_bounds, upper_bounds, strides, std::vector<int64_t>{}, std::vector<int64_t>{});
+            probs = std::make_shared<ngraph::op::v1::Multiply>(classes, scale, ngraph::op::AutoBroadcastType::NUMPY);
+
+            auto thresh_node = std::make_shared<ngraph::op::Constant>(ngraph::element::f32, ngraph::Shape{1}, &thresh);
+            auto mask = std::make_shared<ngraph::op::v1::Greater>(probs, thresh_node);
+            auto zero_node = std::make_shared<ngraph::op::Constant>(ngraph::element::f32, mask->get_shape(), std::vector<float>((rows - 5) * cols, 0));
+            probs = std::make_shared<ngraph::op::v1::Select>(mask, probs, zero_node);
+        }
+
+
+        auto concat_shape = std::make_shared<ngraph::op::Constant>(ngraph::element::i64, ngraph::Shape{2}, std::vector<int64_t>{1, cols});
+        box_x = std::make_shared<ngraph::op::v1::Reshape>(box_x, concat_shape, true);
+        box_y = std::make_shared<ngraph::op::v1::Reshape>(box_y, concat_shape, true);
+        box_w = std::make_shared<ngraph::op::v1::Reshape>(box_w, concat_shape, true);
+        box_h = std::make_shared<ngraph::op::v1::Reshape>(box_h, concat_shape, true);
+
+        ngraph::NodeVector inp_nodes{box_x, box_y, box_w, box_h, scale, probs};
+        std::shared_ptr<ngraph::Node> result = std::make_shared<ngraph::op::Concat>(inp_nodes, 0);
+        result = std::make_shared<ngraph::op::Transpose>(result, tr_axes);
+        if (b > 1)
+        {
+            std::vector<size_t> sizes = {(size_t)b, result->get_shape()[0] / b, result->get_shape()[1]};
+            auto shape_node = std::make_shared<ngraph::op::Constant>(ngraph::element::i64, ngraph::Shape{sizes.size()}, sizes.data());
+            result = std::make_shared<ngraph::op::v1::Reshape>(result, shape_node, true);
+        }
+
+        return Ptr<BackendNode>(new InfEngineNgraphNode(result));
+    }
+#endif  // HAVE_DNN_NGRAPH
+
 };
 
 Ptr<RegionLayer> RegionLayer::create(const LayerParams& params)

modules/dnn/src/onnx/onnx_importer.cpp

@@ -1397,8 +1397,7 @@ void ONNXImporter::populateNet(Net dstNet)
                 CV_Assert(layer_id.find(node_proto.input(i)) == layer_id.end());
 
             String interp_mode = layerParams.get<String>("coordinate_transformation_mode");
-            CV_Assert_N(interp_mode != "tf_crop_and_resize", interp_mode != "asymmetric",
-                        interp_mode != "tf_half_pixel_for_nn");
+            CV_Assert_N(interp_mode != "tf_crop_and_resize", interp_mode != "tf_half_pixel_for_nn");
 
             layerParams.set("align_corners", interp_mode == "align_corners");
             Mat shapes = getBlob(node_proto, constBlobs, node_proto.input_size() - 1);
@@ -1426,6 +1425,22 @@ void ONNXImporter::populateNet(Net dstNet)
         }
         else if (layer_type == "Upsample")
         {
+            //fused from Resize Subgraph
+            if (layerParams.has("coordinate_transformation_mode"))
+            {
+                String interp_mode = layerParams.get<String>("coordinate_transformation_mode");
+                CV_Assert_N(interp_mode != "tf_crop_and_resize", interp_mode != "tf_half_pixel_for_nn");
+
+                layerParams.set("align_corners", interp_mode == "align_corners");
+                if (layerParams.get<String>("mode") == "linear")
+                {
+                    layerParams.set("mode", interp_mode == "pytorch_half_pixel" ?
+                                            "opencv_linear" : "bilinear");
+                }
+            }
+            if (layerParams.get<String>("mode") == "linear" && framework_name == "pytorch")
+                layerParams.set("mode", "opencv_linear");
+
             layerParams.type = "Resize";
             if (layerParams.has("scales"))
             {
@@ -1435,22 +1450,21 @@ void ONNXImporter::populateNet(Net dstNet)
                 layerParams.set("zoom_factor_y", scales.getIntValue(2));
                 layerParams.set("zoom_factor_x", scales.getIntValue(3));
             }
-            else
+            else if (layerParams.has("height_scale") && layerParams.has("width_scale"))
             {
                 // Caffe2 layer
                 replaceLayerParam(layerParams, "height_scale", "zoom_factor_y");
                 replaceLayerParam(layerParams, "width_scale", "zoom_factor_x");
             }
-            replaceLayerParam(layerParams, "mode", "interpolation");
-
-            if (layerParams.get<String>("interpolation") == "linear" && framework_name == "pytorch") {
-                layerParams.type = "Resize";
+            else
+            {
+                // scales as input
                 Mat scales = getBlob(node_proto, constBlobs, 1);
                 CV_Assert(scales.total() == 4);
-                layerParams.set("interpolation", "opencv_linear");
                 layerParams.set("zoom_factor_y", scales.at<float>(2));
                 layerParams.set("zoom_factor_x", scales.at<float>(3));
             }
+            replaceLayerParam(layerParams, "mode", "interpolation");
         }
         else if (layer_type == "SoftMax" || layer_type == "LogSoftmax")
         {

modules/dnn/src/op_inf_engine.cpp

@@ -891,6 +891,13 @@ bool InfEngineBackendNet::isInitialized()
 #endif
 }
 
+void InfEngineBackendNet::reset()
+{
+    allBlobs.clear();
+    infRequests.clear();
+    isInit = false;
+}
+
 void InfEngineBackendNet::addBlobs(const std::vector<cv::Ptr<BackendWrapper> >& ptrs)
 {
     auto wrappers = infEngineWrappers(ptrs);

modules/dnn/src/op_inf_engine.hpp

@@ -112,6 +112,8 @@ public:
 
     void addBlobs(const std::vector<cv::Ptr<BackendWrapper> >& ptrs);
 
+    void reset();
+
 private:
     InferenceEngine::Builder::Network netBuilder;
 

modules/dnn/test/test_darknet_importer.cpp

@@ -556,6 +556,10 @@ TEST_P(Test_Darknet_layers, avgpool_softmax)
 
 TEST_P(Test_Darknet_layers, region)
 {
+#if defined(INF_ENGINE_RELEASE)
+     if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && INF_ENGINE_VER_MAJOR_GE(2020020000))
+        applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_NGRAPH, CV_TEST_TAG_DNN_SKIP_IE_VERSION);
+#endif
     testDarknetLayer("region");
 }
 

modules/dnn/test/test_misc.cpp

@@ -764,6 +764,48 @@ TEST_P(Test_Model_Optimizer, readFromBuffer)
     normAssert(ref, actual, "", 0, 0);
 }
 
+TEST_P(Test_Model_Optimizer, flexible_inputs)
+{
+    const Backend backendId = get<0>(GetParam());
+    const Target targetId = get<1>(GetParam());
+
+    const std::string& model = findDataFile("dnn/layers/layer_convolution_fp16.bin");
+    const std::string& proto = findDataFile("dnn/layers/layer_convolution_fp16.xml");
+
+    if (backendId == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019)
+        setInferenceEngineBackendType(CV_DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_API);
+    else if (backendId == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH)
+        setInferenceEngineBackendType(CV_DNN_BACKEND_INFERENCE_ENGINE_NGRAPH);
+    else
+        FAIL() << "Unknown backendId";
+
+    Net net0 = readNet(model, proto);
+    net0.setPreferableTarget(targetId);
+
+    Net net1 = readNet(model, proto);
+    net1.setPreferableTarget(targetId);
+
+    // Generate inputs.
+    int blobSize0[] = {2, 6, 75, 113};
+    Mat input0(4, &blobSize0[0], CV_32F);
+    randu(input0, 0, 255);
+
+    net0.setInput(input0);
+    Mat ref = net0.forward().clone();
+
+    int blobSize1[] = {1, 6, 10, 9};
+    Mat input1(4, &blobSize1[0], CV_32F);
+    randu(input1, 0, 255);
+
+    net1.setInput(input1);
+    Mat out = net1.forward();
+    EXPECT_NE(out.size, ref.size);
+
+    net1.setInput(input0);
+    out = net1.forward();
+    normAssert(ref, out, 0, 0);
+}
+
 INSTANTIATE_TEST_CASE_P(/**/, Test_Model_Optimizer,
     dnnBackendsAndTargetsIE()
 );

modules/dnn/test/test_onnx_importer.cpp

@@ -393,6 +393,7 @@ TEST_P(Test_ONNX_layers, ResizeUnfused)
     testONNXModels("upsample_unfused_opset9_torch1.4");
     testONNXModels("resize_nearest_unfused_opset11_torch1.4");
     testONNXModels("resize_nearest_unfused_opset11_torch1.3");
+    testONNXModels("resize_bilinear_unfused_opset11_torch1.4");
 }
 
 TEST_P(Test_ONNX_layers, MultyInputs)