Merge remote-tracking branch 'upstream/3.4' into merge-3.4

samples/dnn/mask_rcnn.py

@@ -43,7 +43,7 @@ def showLegend(classes):
         for i in range(len(classes)):
             block = legend[i * blockHeight:(i + 1) * blockHeight]
             block[:,:] = colors[i]
-            cv.putText(block, classes[i], (0, blockHeight/2), cv.FONT_HERSHEY_SIMPLEX, 0.5, (255, 255, 255))
+            cv.putText(block, classes[i], (0, blockHeight//2), cv.FONT_HERSHEY_SIMPLEX, 0.5, (255, 255, 255))
 
         cv.namedWindow('Legend', cv.WINDOW_NORMAL)
         cv.imshow('Legend', legend)

samples/dnn/object_detection.cpp

@@ -45,7 +45,7 @@ std::vector<std::string> classes;
 inline void preprocess(const Mat& frame, Net& net, Size inpSize, float scale,
                        const Scalar& mean, bool swapRB);
 
-void postprocess(Mat& frame, const std::vector<Mat>& out, Net& net);
+void postprocess(Mat& frame, const std::vector<Mat>& out, Net& net, int backend);
 
 void drawPred(int classId, float conf, int left, int top, int right, int bottom, Mat& frame);
 
@@ -148,7 +148,8 @@ int main(int argc, char** argv)
 
     // Load a model.
     Net net = readNet(modelPath, configPath, parser.get<String>("framework"));
-    net.setPreferableBackend(parser.get<int>("backend"));
+    int backend = parser.get<int>("backend");
+    net.setPreferableBackend(backend);
     net.setPreferableTarget(parser.get<int>("target"));
     std::vector<String> outNames = net.getUnconnectedOutLayersNames();
 
@@ -245,7 +246,7 @@ int main(int argc, char** argv)
         std::vector<Mat> outs = predictionsQueue.get();
         Mat frame = processedFramesQueue.get();
 
-        postprocess(frame, outs, net);
+        postprocess(frame, outs, net, backend);
 
         if (predictionsQueue.counter > 1)
         {
@@ -285,7 +286,7 @@ int main(int argc, char** argv)
         std::vector<Mat> outs;
         net.forward(outs, outNames);
 
-        postprocess(frame, outs, net);
+        postprocess(frame, outs, net, backend);
 
         // Put efficiency information.
         std::vector<double> layersTimes;
@@ -319,7 +320,7 @@ inline void preprocess(const Mat& frame, Net& net, Size inpSize, float scale,
     }
 }
 
-void postprocess(Mat& frame, const std::vector<Mat>& outs, Net& net)
+void postprocess(Mat& frame, const std::vector<Mat>& outs, Net& net, int backend)
 {
     static std::vector<int> outLayers = net.getUnconnectedOutLayers();
     static std::string outLayerType = net.getLayer(outLayers[0])->type;
@@ -396,11 +397,48 @@ void postprocess(Mat& frame, const std::vector<Mat>& outs, Net& net)
     else
         CV_Error(Error::StsNotImplemented, "Unknown output layer type: " + outLayerType);
 
-    std::vector<int> indices;
-    NMSBoxes(boxes, confidences, confThreshold, nmsThreshold, indices);
-    for (size_t i = 0; i < indices.size(); ++i)
+    // NMS is used inside Region layer only on DNN_BACKEND_OPENCV for another backends we need NMS in sample
+    // or NMS is required if number of outputs > 1
+    if (outLayers.size() > 1 || (outLayerType == "Region" && backend != DNN_BACKEND_OPENCV))
+    {
+        std::map<int, std::vector<size_t> > class2indices;
+        for (size_t i = 0; i < classIds.size(); i++)
+        {
+            if (confidences[i] >= confThreshold)
+            {
+                class2indices[classIds[i]].push_back(i);
+            }
+        }
+        std::vector<Rect> nmsBoxes;
+        std::vector<float> nmsConfidences;
+        std::vector<int> nmsClassIds;
+        for (std::map<int, std::vector<size_t> >::iterator it = class2indices.begin(); it != class2indices.end(); ++it)
+        {
+            std::vector<Rect> localBoxes;
+            std::vector<float> localConfidences;
+            std::vector<size_t> classIndices = it->second;
+            for (size_t i = 0; i < classIndices.size(); i++)
+            {
+                localBoxes.push_back(boxes[classIndices[i]]);
+                localConfidences.push_back(confidences[classIndices[i]]);
+            }
+            std::vector<int> nmsIndices;
+            NMSBoxes(localBoxes, localConfidences, confThreshold, nmsThreshold, nmsIndices);
+            for (size_t i = 0; i < nmsIndices.size(); i++)
+            {
+                size_t idx = nmsIndices[i];
+                nmsBoxes.push_back(localBoxes[idx]);
+                nmsConfidences.push_back(localConfidences[idx]);
+                nmsClassIds.push_back(it->first);
+            }
+        }
+        boxes = nmsBoxes;
+        classIds = nmsClassIds;
+        confidences = nmsConfidences;
+    }
+
+    for (size_t idx = 0; idx < boxes.size(); ++idx)
     {
-        int idx = indices[i];
         Rect box = boxes[idx];
         drawPred(classIds[idx], confidences[idx], box.x, box.y,
                  box.x + box.width, box.y + box.height, frame);

samples/dnn/object_detection.py

@@ -141,9 +141,6 @@ def postprocess(frame, outs):
         # Network produces output blob with a shape NxC where N is a number of
         # detected objects and C is a number of classes + 4 where the first 4
         # numbers are [center_x, center_y, width, height]
-        classIds = []
-        confidences = []
-        boxes = []
         for out in outs:
             for detection in out:
                 scores = detection[5:]
@@ -163,9 +160,25 @@ def postprocess(frame, outs):
         print('Unknown output layer type: ' + lastLayer.type)
         exit()
 
-    indices = cv.dnn.NMSBoxes(boxes, confidences, confThreshold, nmsThreshold)
+    # NMS is used inside Region layer only on DNN_BACKEND_OPENCV for another backends we need NMS in sample
+    # or NMS is required if number of outputs > 1
+    if len(outNames) > 1 or lastLayer.type == 'Region' and args.backend != cv.dnn.DNN_BACKEND_OPENCV:
+        indices = []
+        classIds = np.array(classIds)
+        boxes = np.array(boxes)
+        confidences = np.array(confidences)
+        unique_classes = set(classIds)
+        for cl in unique_classes:
+            class_indices = np.where(classIds == cl)[0]
+            conf = confidences[class_indices]
+            box  = boxes[class_indices].tolist()
+            nms_indices = cv.dnn.NMSBoxes(box, conf, confThreshold, nmsThreshold)
+            nms_indices = nms_indices[:, 0] if len(nms_indices) else []
+            indices.extend(class_indices[nms_indices])
+    else:
+        indices = np.arange(0, len(classIds))
+
     for i in indices:
-        i = i[0]
         box = boxes[i]
         left = box[0]
         top = box[1]

samples/dnn/segmentation.py

@@ -65,7 +65,7 @@ def showLegend(classes):
         for i in range(len(classes)):
             block = legend[i * blockHeight:(i + 1) * blockHeight]
             block[:,:] = colors[i]
-            cv.putText(block, classes[i], (0, blockHeight/2), cv.FONT_HERSHEY_SIMPLEX, 0.5, (255, 255, 255))
+            cv.putText(block, classes[i], (0, blockHeight//2), cv.FONT_HERSHEY_SIMPLEX, 0.5, (255, 255, 255))
 
         cv.namedWindow('Legend', cv.WINDOW_NORMAL)
         cv.imshow('Legend', legend)
@@ -76,7 +76,7 @@ net = cv.dnn.readNet(args.model, args.config, args.framework)
 net.setPreferableBackend(args.backend)
 net.setPreferableTarget(args.target)
 
-winName = 'Deep learning image classification in OpenCV'
+winName = 'Deep learning semantic segmentation in OpenCV'
 cv.namedWindow(winName, cv.WINDOW_NORMAL)
 
 cap = cv.VideoCapture(args.input if args.input else 0)

samples/dnn/tf_text_graph_common.py

@@ -269,7 +269,7 @@ def parseTextGraph(filePath):
 def removeIdentity(graph_def):
     identities = {}
     for node in graph_def.node:
-        if node.op == 'Identity':
+        if node.op == 'Identity' or node.op == 'IdentityN':
             identities[node.name] = node.input[0]
             graph_def.node.remove(node)
 

samples/dnn/tf_text_graph_efficientdet.py

@@ -0,0 +1,236 @@
+# This file is a part of OpenCV project.
+# It is a subject to the license terms in the LICENSE file found in the top-level directory
+# of this distribution and at http://opencv.org/license.html.
+#
+# Copyright (C) 2020, Intel Corporation, all rights reserved.
+# Third party copyrights are property of their respective owners.
+#
+# Use this script to get the text graph representation (.pbtxt) of EfficientDet
+# deep learning network trained in https://github.com/google/automl.
+# Then you can import it with a binary frozen graph (.pb) using readNetFromTensorflow() function.
+# See details and examples on the following wiki page: https://github.com/opencv/opencv/wiki/TensorFlow-Object-Detection-API
+import argparse
+import re
+from math import sqrt
+from tf_text_graph_common import *
+
+
+class AnchorGenerator:
+    def __init__(self, min_level, aspect_ratios, num_scales, anchor_scale):
+        self.min_level = min_level
+        self.aspect_ratios = aspect_ratios
+        self.anchor_scale = anchor_scale
+        self.scales = [2**(float(s) / num_scales) for s in range(num_scales)]
+
+    def get(self, layer_id):
+        widths = []
+        heights = []
+        for s in self.scales:
+            for a in self.aspect_ratios:
+                base_anchor_size = 2**(self.min_level + layer_id) * self.anchor_scale
+                heights.append(base_anchor_size * s * a[1])
+                widths.append(base_anchor_size * s * a[0])
+        return widths, heights
+
+
+def createGraph(modelPath, outputPath, min_level, aspect_ratios, num_scales,
+                anchor_scale, num_classes, image_width, image_height):
+    print('Min level: %d' % min_level)
+    print('Anchor scale: %f' % anchor_scale)
+    print('Num scales: %d' % num_scales)
+    print('Aspect ratios: %s' % str(aspect_ratios))
+    print('Number of classes: %d' % num_classes)
+    print('Input image size: %dx%d' % (image_width, image_height))
+
+    # Read the graph.
+    _inpNames = ['image_arrays']
+    outNames = ['detections']
+
+    writeTextGraph(modelPath, outputPath, outNames)
+    graph_def = parseTextGraph(outputPath)
+
+    def getUnconnectedNodes():
+        unconnected = []
+        for node in graph_def.node:
+            if node.op == 'Const':
+                continue
+            unconnected.append(node.name)
+            for inp in node.input:
+                if inp in unconnected:
+                    unconnected.remove(inp)
+        return unconnected
+
+
+    nodesToKeep = ['truediv']  # Keep preprocessing nodes
+
+    removeIdentity(graph_def)
+
+    scopesToKeep = ('image_arrays', 'efficientnet', 'resample_p6', 'resample_p7',
+                    'fpn_cells', 'class_net', 'box_net', 'Reshape', 'concat')
+
+    addConstNode('scale_w', [2.0], graph_def)
+    addConstNode('scale_h', [2.0], graph_def)
+    nodesToKeep += ['scale_w', 'scale_h']
+
+    for node in graph_def.node:
+        if re.match('efficientnet-(.*)/blocks_\d+/se/mul_1', node.name):
+            node.input[0], node.input[1] = node.input[1], node.input[0]
+
+        if re.match('fpn_cells/cell_\d+/fnode\d+/resample(.*)/nearest_upsampling/Reshape_1$', node.name):
+            node.op = 'ResizeNearestNeighbor'
+            node.input[1] = 'scale_w'
+            node.input.append('scale_h')
+
+            for inpNode in graph_def.node:
+                if inpNode.name == node.name[:node.name.rfind('_')]:
+                    node.input[0] = inpNode.input[0]
+
+        if re.match('box_net/box-predict(_\d)*/separable_conv2d$', node.name):
+            node.addAttr('loc_pred_transposed', True)
+
+        # Replace RealDiv to Mul with inversed scale for compatibility
+        if node.op == 'RealDiv':
+            for inpNode in graph_def.node:
+                if inpNode.name != node.input[1] or not 'value' in inpNode.attr:
+                    continue
+
+                tensor = inpNode.attr['value']['tensor'][0]
+                if not 'float_val' in tensor:
+                    continue
+                scale = float(inpNode.attr['value']['tensor'][0]['float_val'][0])
+
+                addConstNode(inpNode.name + '/inv', [1.0 / scale], graph_def)
+                nodesToKeep.append(inpNode.name + '/inv')
+                node.input[1] = inpNode.name + '/inv'
+                node.op = 'Mul'
+                break
+
+
+    def to_remove(name, op):
+        if name in nodesToKeep:
+            return False
+        return op == 'Const' or not name.startswith(scopesToKeep)
+
+    removeUnusedNodesAndAttrs(to_remove, graph_def)
+
+    # Attach unconnected preprocessing
+    assert(graph_def.node[1].name == 'truediv' and graph_def.node[1].op == 'RealDiv')
+    graph_def.node[1].input.insert(0, 'image_arrays')
+    graph_def.node[2].input.insert(0, 'truediv')
+
+    priors_generator = AnchorGenerator(min_level, aspect_ratios, num_scales, anchor_scale)
+    priorBoxes = []
+    for i in range(5):
+        inpName = ''
+        for node in graph_def.node:
+            if node.name == 'Reshape_%d' % (i * 2 + 1):
+                inpName = node.input[0]
+                break
+
+        priorBox = NodeDef()
+        priorBox.name = 'PriorBox_%d' % i
+        priorBox.op = 'PriorBox'
+        priorBox.input.append(inpName)
+        priorBox.input.append(graph_def.node[0].name)  # image_tensor
+
+        priorBox.addAttr('flip', False)
+        priorBox.addAttr('clip', False)
+
+        widths, heights = priors_generator.get(i)
+
+        priorBox.addAttr('width', widths)
+        priorBox.addAttr('height', heights)
+        priorBox.addAttr('variance', [1.0, 1.0, 1.0, 1.0])
+
+        graph_def.node.extend([priorBox])
+        priorBoxes.append(priorBox.name)
+
+    addConstNode('concat/axis_flatten', [-1], graph_def)
+
+    def addConcatNode(name, inputs, axisNodeName):
+        concat = NodeDef()
+        concat.name = name
+        concat.op = 'ConcatV2'
+        for inp in inputs:
+            concat.input.append(inp)
+        concat.input.append(axisNodeName)
+        graph_def.node.extend([concat])
+
+    addConcatNode('PriorBox/concat', priorBoxes, 'concat/axis_flatten')
+
+    sigmoid = NodeDef()
+    sigmoid.name = 'concat/sigmoid'
+    sigmoid.op = 'Sigmoid'
+    sigmoid.input.append('concat')
+    graph_def.node.extend([sigmoid])
+
+    addFlatten(sigmoid.name, sigmoid.name + '/Flatten', graph_def)
+    addFlatten('concat_1', 'concat_1/Flatten', graph_def)
+
+    detectionOut = NodeDef()
+    detectionOut.name = 'detection_out'
+    detectionOut.op = 'DetectionOutput'
+
+    detectionOut.input.append('concat_1/Flatten')
+    detectionOut.input.append(sigmoid.name + '/Flatten')
+    detectionOut.input.append('PriorBox/concat')
+
+    detectionOut.addAttr('num_classes', num_classes)
+    detectionOut.addAttr('share_location', True)
+    detectionOut.addAttr('background_label_id', num_classes + 1)
+    detectionOut.addAttr('nms_threshold', 0.6)
+    detectionOut.addAttr('confidence_threshold', 0.2)
+    detectionOut.addAttr('top_k', 100)
+    detectionOut.addAttr('keep_top_k', 100)
+    detectionOut.addAttr('code_type', "CENTER_SIZE")
+    graph_def.node.extend([detectionOut])
+
+    graph_def.node[0].attr['shape'] =  {
+            'shape': {
+                'dim': [
+                    {'size': -1},
+                    {'size': image_height},
+                    {'size': image_width},
+                    {'size': 3}
+                ]
+            }
+        }
+
+    while True:
+        unconnectedNodes = getUnconnectedNodes()
+        unconnectedNodes.remove(detectionOut.name)
+        if not unconnectedNodes:
+            break
+
+        for name in unconnectedNodes:
+            for i in range(len(graph_def.node)):
+                if graph_def.node[i].name == name:
+                    del graph_def.node[i]
+                    break
+
+    # Save as text
+    graph_def.save(outputPath)
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description='Run this script to get a text graph of '
+                                                 'SSD model from TensorFlow Object Detection API. '
+                                                 'Then pass it with .pb file to cv::dnn::readNetFromTensorflow function.')
+    parser.add_argument('--input', required=True, help='Path to frozen TensorFlow graph.')
+    parser.add_argument('--output', required=True, help='Path to output text graph.')
+    parser.add_argument('--min_level', default=3, type=int, help='Parameter from training config')
+    parser.add_argument('--num_scales', default=3, type=int, help='Parameter from training config')
+    parser.add_argument('--anchor_scale', default=4.0, type=float, help='Parameter from training config')
+    parser.add_argument('--aspect_ratios', default=[1.0, 1.0, 1.4, 0.7, 0.7, 1.4],
+                        nargs='+', type=float, help='Parameter from training config')
+    parser.add_argument('--num_classes', default=90, type=int, help='Number of classes to detect')
+    parser.add_argument('--width', default=512, type=int, help='Network input width')
+    parser.add_argument('--height', default=512, type=int, help='Network input height')
+    args = parser.parse_args()
+
+    ar = args.aspect_ratios
+    assert(len(ar) % 2 == 0)
+    ar = list(zip(ar[::2], ar[1::2]))
+
+    createGraph(args.input, args.output, args.min_level, ar, args.num_scales,
+                args.anchor_scale, args.num_classes, args.width, args.height)