diff --git a/modules/dnn/CMakeLists.txt b/modules/dnn/CMakeLists.txt index f1f9e13695..a76548363c 100644 --- a/modules/dnn/CMakeLists.txt +++ b/modules/dnn/CMakeLists.txt @@ -393,7 +393,7 @@ if(HAVE_PROTOBUF) ocv_target_compile_definitions(${the_module} PRIVATE "HAVE_PROTOBUF=1") if(PROTOBUF_UPDATE_FILES) - file(GLOB proto_files "${CMAKE_CURRENT_LIST_DIR}/src/tensorflow/*.proto" "${CMAKE_CURRENT_LIST_DIR}/src/caffe/opencv-caffe.proto" "${CMAKE_CURRENT_LIST_DIR}/src/onnx/opencv-onnx.proto") + file(GLOB proto_files "${CMAKE_CURRENT_LIST_DIR}/src/tensorflow/*.proto" "${CMAKE_CURRENT_LIST_DIR}/src/onnx/opencv-onnx.proto") if(CMAKE_VERSION VERSION_LESS "3.13.0") set(PROTOBUF_GENERATE_CPP_APPEND_PATH ON) # required for tensorflow protobuf_generate_cpp(fw_srcs fw_hdrs ${proto_files}) @@ -587,28 +587,6 @@ ocv_add_perf_tests(${dnn_runtime_libs} FILES Include ${perf_hdrs} ) -ocv_option(OPENCV_DNN_PERF_CAFFE "Add performance tests of Caffe framework" OFF) -ocv_option(OPENCV_DNN_PERF_CLCAFFE "Add performance tests of clCaffe framework" OFF) -if(BUILD_PERF_TESTS) - if (OPENCV_DNN_PERF_CAFFE - OR ${the_module}_PERF_CAFFE # compatibility for deprecated option - ) - find_package(Caffe QUIET) - if (Caffe_FOUND) - ocv_target_compile_definitions(opencv_perf_dnn PRIVATE "HAVE_CAFFE=1") - ocv_target_link_libraries(opencv_perf_dnn caffe) - endif() - elseif(OPENCV_DNN_PERF_CLCAFFE - OR ${the_module}_PERF_CAFFE # compatibility for deprecated option - ) - find_package(Caffe QUIET) - if (Caffe_FOUND) - ocv_target_compile_definitions(opencv_perf_dnn PRIVATE "HAVE_CLCAFFE=1") - ocv_target_link_libraries(opencv_perf_dnn caffe) - endif() - endif() -endif() - if(DNN_ENABLE_PLUGINS) ocv_target_compile_definitions(${the_module} PRIVATE ENABLE_PLUGINS) if(TARGET opencv_test_dnn) diff --git a/modules/dnn/include/opencv2/dnn/all_layers.hpp b/modules/dnn/include/opencv2/dnn/all_layers.hpp index 2e8f56add1..72568fabcd 100644 --- a/modules/dnn/include/opencv2/dnn/all_layers.hpp +++ b/modules/dnn/include/opencv2/dnn/all_layers.hpp @@ -55,7 +55,7 @@ CV__DNN_INLINE_NS_BEGIN Classes listed here, in fact, provides C++ API for creating instances of built-in layers. In addition to this way of layers instantiation, there is a more common factory API (see @ref dnnLayerFactory), it allows to create layers dynamically (by name) and register new ones. - You can use both API, but factory API is less convenient for native C++ programming and basically designed for use inside importers (see @ref readNetFromCaffe(), @ref readNetFromTensorflow()). + You can use both API, but factory API is less convenient for native C++ programming and basically designed for use inside importers (see @ref readNetFromTensorflow()). Built-in layers partially reproduce functionality of corresponding ONNX, TensorFlow and Caffe layers. In particular, the following layers and Caffe importer were tested to reproduce Caffe functionality: diff --git a/modules/dnn/include/opencv2/dnn/dnn.hpp b/modules/dnn/include/opencv2/dnn/dnn.hpp index 37fe53f7e2..7da73d3cbf 100644 --- a/modules/dnn/include/opencv2/dnn/dnn.hpp +++ b/modules/dnn/include/opencv2/dnn/dnn.hpp @@ -1090,43 +1090,6 @@ CV__DNN_INLINE_NS_BEGIN ENGINE_ORT=4 //!< Try to use ONNX Runtime wrapper (ONNX only, requires build with WITH_ONNXRUNTIME=ON). }; - /** @brief Reads a network model stored in Caffe framework's format. - * @param prototxt path to the .prototxt file with text description of the network architecture. - * @param caffeModel path to the .caffemodel file with learned network. - * @param engine select DNN engine to be used. With auto selection the new engine is used. - * Please pay attention that the new DNN does not support non-CPU back-ends for now. - * @returns Net object. - */ - CV_EXPORTS_W Net readNetFromCaffe(CV_WRAP_FILE_PATH const String &prototxt, - CV_WRAP_FILE_PATH const String &caffeModel = String(), - int engine = ENGINE_AUTO); - - /** @brief Reads a network model stored in Caffe model in memory. - * @param bufferProto buffer containing the content of the .prototxt file - * @param bufferModel buffer containing the content of the .caffemodel file - * @param engine select DNN engine to be used. With auto selection the new engine is used. - * Please pay attention that the new DNN does not support non-CPU back-ends for now. - * @returns Net object. - */ - CV_EXPORTS_W Net readNetFromCaffe(const std::vector& bufferProto, - const std::vector& bufferModel = std::vector(), - int engine = ENGINE_AUTO); - - /** @brief Reads a network model stored in Caffe model in memory. - * @details This is an overloaded member function, provided for convenience. - * It differs from the above function only in what argument(s) it accepts. - * @param bufferProto buffer containing the content of the .prototxt file - * @param lenProto length of bufferProto - * @param bufferModel buffer containing the content of the .caffemodel file - * @param lenModel length of bufferModel - * @param engine select DNN engine to be used. With auto selection the new engine is used. - * Please pay attention that the new DNN does not support non-CPU back-ends for now. - * @returns Net object. - */ - CV_EXPORTS Net readNetFromCaffe(const char *bufferProto, size_t lenProto, - const char *bufferModel = NULL, size_t lenModel = 0, - int engine = ENGINE_AUTO); - /** @brief Reads a network model stored in TensorFlow framework's format. * @param model path to the .pb file with binary protobuf description of the network architecture * @param config path to the .pbtxt file that contains text graph definition in protobuf format. @@ -1201,13 +1164,11 @@ CV__DNN_INLINE_NS_BEGIN * @brief Read deep learning network represented in one of the supported formats. * @param[in] model Binary file contains trained weights. The following file * extensions are expected for models from different frameworks: - * * `*.caffemodel` (Caffe, http://caffe.berkeleyvision.org/) * * `*.pb` (TensorFlow, https://www.tensorflow.org/) * * `*.bin` | `*.onnx` (OpenVINO, https://software.intel.com/openvino-toolkit) * * `*.onnx` (ONNX, https://onnx.ai/) * @param[in] config Text file contains network configuration. It could be a * file with the following extensions: - * * `*.prototxt` (Caffe, http://caffe.berkeleyvision.org/) * * `*.pbtxt` (TensorFlow, https://www.tensorflow.org/) * * `*.xml` (OpenVINO, https://software.intel.com/openvino-toolkit) * @param[in] framework Explicit framework name tag to determine a format. @@ -1217,7 +1178,7 @@ CV__DNN_INLINE_NS_BEGIN * @returns Net object. * * This function automatically detects an origin framework of trained model - * and calls an appropriate function such @ref readNetFromCaffe, @ref readNetFromTensorflow. + * and calls an appropriate function such @ref readNetFromTensorflow, @ref readNetFromONNX. * An order of @p model and @p config arguments does not matter. */ CV_EXPORTS_W Net readNet(CV_WRAP_FILE_PATH const String& model, @@ -1467,22 +1428,6 @@ CV__DNN_INLINE_NS_BEGIN */ CV_EXPORTS_W void imagesFromBlob(const cv::Mat& blob_, OutputArrayOfArrays images_); - /** @brief Convert all weights of Caffe network to half precision floating point. - * @param src Path to origin model from Caffe framework contains single - * precision floating point weights (usually has `.caffemodel` extension). - * @param dst Path to destination model with updated weights. - * @param layersTypes Set of layers types which parameters will be converted. - * By default, converts only Convolutional and Fully-Connected layers' - * weights. - * - * @note Shrinked model has no origin float32 weights so it can't be used - * in origin Caffe framework anymore. However the structure of data - * is taken from NVidia's Caffe fork: https://github.com/NVIDIA/caffe. - * So the resulting model may be used there. - */ - CV_EXPORTS_W void shrinkCaffeModel(CV_WRAP_FILE_PATH const String& src, CV_WRAP_FILE_PATH const String& dst, - const std::vector& layersTypes = std::vector()); - /** @brief Create a text representation for a binary network stored in protocol buffer format. * @param[in] model A path to binary network. * @param[in] output A path to output text file to be created. diff --git a/modules/dnn/misc/java/test/DnnForwardAndRetrieve.java b/modules/dnn/misc/java/test/DnnForwardAndRetrieve.java index 22c4919eac..682099a261 100644 --- a/modules/dnn/misc/java/test/DnnForwardAndRetrieve.java +++ b/modules/dnn/misc/java/test/DnnForwardAndRetrieve.java @@ -1,67 +1,71 @@ package org.opencv.test.dnn; +import java.io.File; import java.util.ArrayList; import java.util.List; import org.opencv.core.Core; import org.opencv.core.CvType; import org.opencv.core.Mat; -import org.opencv.core.MatOfByte; -import org.opencv.core.Range; import org.opencv.dnn.Dnn; import org.opencv.dnn.Net; import org.opencv.test.OpenCVTestCase; public class DnnForwardAndRetrieve extends OpenCVTestCase { + private final static String ENV_OPENCV_DNN_TEST_DATA_PATH = "OPENCV_DNN_TEST_DATA_PATH"; + private final static String ENV_OPENCV_TEST_DATA_PATH = "OPENCV_TEST_DATA_PATH"; + + private String modelFileName = ""; + + @Override + protected void setUp() throws Exception { + super.setUp(); + + String dnnTestDataPath = System.getenv(ENV_OPENCV_DNN_TEST_DATA_PATH); + String generalTestDataPath = System.getenv(ENV_OPENCV_TEST_DATA_PATH); + + File model = null; + + if (generalTestDataPath != null) { + model = new File(generalTestDataPath, "dnn/onnx/models/split_0.onnx"); + } + + if ((model == null || !model.isFile()) && dnnTestDataPath != null) { + model = new File(dnnTestDataPath, "dnn/onnx/models/split_0.onnx"); + } + + if (model == null || !model.isFile()) { + isTestCaseEnabled = false; + return; + } + + modelFileName = model.getAbsolutePath(); + } + public void testForwardAndRetrieve() { - // Create a simple Caffe prototxt with a Slice layer - String prototxt = - "input: \"data\"\n" + - "layer {\n" + - " name: \"testLayer\"\n" + - " type: \"Slice\"\n" + - " bottom: \"data\"\n" + - " top: \"firstCopy\"\n" + - " top: \"secondCopy\"\n" + - " slice_param {\n" + - " axis: 0\n" + - " slice_point: 2\n" + - " }\n" + - "}"; - - // Read network from prototxt - MatOfByte bufferProto = new MatOfByte(); - bufferProto.fromArray(prototxt.getBytes()); - MatOfByte bufferModel = new MatOfByte(); - Net net = Dnn.readNetFromCaffe(bufferProto, bufferModel, Dnn.ENGINE_CLASSIC); + // Verifies forwardAndRetrieve nested list marshalling using a small ONNX model instead of the removed Caffe importer. + Net net = Dnn.readNetFromONNX(modelFileName, Dnn.ENGINE_CLASSIC); net.setPreferableBackend(Dnn.DNN_BACKEND_OPENCV); - // Create input data - Mat inp = new Mat(4, 5, CvType.CV_32F); + // split_0.onnx declares a single 4D input named "image" of shape [1, 3, 2, 2]. + Mat inp = new Mat(new int[]{1, 3, 2, 2}, CvType.CV_32F); Core.randu(inp, -1, 1); net.setInput(inp); - // Define output names - List outNames = new ArrayList<>(); - outNames.add("testLayer"); + List outNames = net.getUnconnectedOutLayersNames(); + assertFalse("Model has no output layers", outNames.isEmpty()); - // Forward and retrieve multiple outputs + // Forward and retrieve every output blob of the requested layers. List> outBlobs = new ArrayList<>(); net.forwardAndRetrieve(outBlobs, outNames); - // Verify results - assertEquals(1, outBlobs.size()); - assertEquals(2, outBlobs.get(0).size()); - - // Compare results - Mat expectedFirst = inp.rowRange(0, 2); - Mat expectedSecond = inp.rowRange(2, 4); - - Mat actualFirst = outBlobs.get(0).get(0); - Mat actualSecond = outBlobs.get(0).get(1); - - assertEquals(0, Core.norm(expectedFirst, actualFirst, Core.NORM_INF), EPS); - assertEquals(0, Core.norm(expectedSecond, actualSecond, Core.NORM_INF), EPS); + // One entry per requested layer name, each holding at least one valid blob. + assertEquals(outNames.size(), outBlobs.size()); + for (List blobs : outBlobs) { + assertFalse(blobs.isEmpty()); + for (Mat blob : blobs) + assertFalse(blob.empty()); + } } } diff --git a/modules/dnn/misc/js/gen_dict.json b/modules/dnn/misc/js/gen_dict.json index e1b7f2ae2d..b023caafe7 100644 --- a/modules/dnn/misc/js/gen_dict.json +++ b/modules/dnn/misc/js/gen_dict.json @@ -2,7 +2,7 @@ "whitelist": { "dnn_Net": ["setInput", "forward", "setPreferableBackend","getUnconnectedOutLayersNames"], - "": ["readNetFromCaffe", "readNetFromTensorflow", "readNetFromTorch", + "": ["readNetFromTensorflow", "readNetFromTorch", "readNetFromONNX", "readNetFromTFLite", "readNet", "blobFromImage"] }, "namespace_prefix_override": diff --git a/modules/dnn/misc/objc/gen_dict.json b/modules/dnn/misc/objc/gen_dict.json index b94846bb2c..e0afda9f43 100644 --- a/modules/dnn/misc/objc/gen_dict.json +++ b/modules/dnn/misc/objc/gen_dict.json @@ -1,8 +1,6 @@ { "func_arg_fix" : { "Dnn": { - "(Net*)readNetFromCaffe:(NSString*)prototxt caffeModel:(NSString*)caffeModel engine:(int)engine" : { "readNetFromCaffe" : {"name" : "readNetFromCaffeFile"} }, - "(Net*)readNetFromCaffe:(ByteVector*)bufferProto bufferModel:(ByteVector*)bufferModel engine:(int)engine" : { "readNetFromCaffe" : {"name" : "readNetFromCaffeBuffer"} }, "(Net*)readNetFromONNX:(NSString*)onnxFile engine:(int)engine" : { "readNetFromONNX" : {"name" : "readNetFromONNXFile"} }, "(Net*)readNetFromONNX:(ByteVector*)buffer engine:(int)engine" : { "readNetFromONNX" : {"name" : "readNetFromONNXBuffer"} }, "(Net*)readNetFromTensorflow:(NSString*)model config:(NSString*)config engine:(int)engine extraOutputs:(NSArray*)extraOutputs" : { "readNetFromTensorflow" : {"name" : "readNetFromTensorflowFile"} }, diff --git a/modules/dnn/misc/python/test/test_dnn.py b/modules/dnn/misc/python/test/test_dnn.py index 006ec1aa0a..cf764e93d2 100755 --- a/modules/dnn/misc/python/test/test_dnn.py +++ b/modules/dnn/misc/python/test/test_dnn.py @@ -82,6 +82,13 @@ class dnn_test(NewOpenCVTests): g_dnnBackendsAndTargets = self.initBackendsAndTargets() self.dnnBackendsAndTargets = g_dnnBackendsAndTargets + def checkIETarget(self, backend, target): + # OpenVINO is optional; a target is usable only if its backend lists it. + try: + return target in cv.dnn.getAvailableTargets(backend) + except BaseException: + return False + def initBackendsAndTargets(self): self.dnnBackendsAndTargets = [ [cv.dnn.DNN_BACKEND_OPENCV, cv.dnn.DNN_TARGET_CPU], @@ -109,20 +116,6 @@ class dnn_test(NewOpenCVTests): os.environ['OPENCV_TEST_DATA_PATH']], required=required) - def checkIETarget(self, backend, target): - proto = self.find_dnn_file('dnn/layers/layer_convolution.prototxt') - model = self.find_dnn_file('dnn/layers/layer_convolution.caffemodel') - net = cv.dnn.readNet(proto, model, engine=cv.dnn.ENGINE_CLASSIC) - try: - net.setPreferableBackend(backend) - net.setPreferableTarget(target) - inp = np.random.standard_normal([1, 2, 10, 11]).astype(np.float32) - net.setInput(inp) - net.forward() - except BaseException: - return False - return True - def test_getAvailableTargets(self): targets = cv.dnn.getAvailableTargets(cv.dnn.DNN_BACKEND_OPENCV) self.assertTrue(cv.dnn.DNN_TARGET_CPU in targets) @@ -219,25 +212,24 @@ class dnn_test(NewOpenCVTests): def test_model(self): img_path = self.find_dnn_file("dnn/street.png") - weights = self.find_dnn_file("dnn/MobileNetSSD_deploy_19e3ec3.caffemodel", required=False) - config = self.find_dnn_file("dnn/MobileNetSSD_deploy_19e3ec3.prototxt", required=False) - if weights is None or config is None: - raise unittest.SkipTest("Missing DNN test files (dnn/MobileNetSSD_deploy_19e3ec3.{prototxt/caffemodel}). Verify OPENCV_DNN_TEST_DATA_PATH configuration parameter.") + weights = self.find_dnn_file("dnn/onnx/models/ssd_vgg16.onnx", required=False) + if weights is None: + raise unittest.SkipTest("Missing DNN test files (dnn/onnx/models/ssd_vgg16.onnx). Verify OPENCV_DNN_TEST_DATA_PATH configuration parameter.") frame = cv.imread(img_path) - model = cv.dnn_DetectionModel(weights, config) - model.setInputParams(size=(300, 300), mean=(127.5, 127.5, 127.5), scale=1.0/127.5) + model = cv.dnn_DetectionModel(weights) + model.setInputParams(size=(300, 300), mean=(0, 0, 0), scale=1.0, swapRB=False) iouDiff = 0.05 - confThreshold = 0.0001 + confThreshold = 0.3 nmsThreshold = 0 - scoreDiff = 1.1e-3 + scoreDiff = 5e-3 classIds, confidences, boxes = model.detect(frame, confThreshold, nmsThreshold) - refClassIds = (7, 15) - refConfidences = (0.9998, 0.8793) - refBoxes = ((328, 238, 85, 102), (101, 188, 34, 138)) + refClassIds = (37,) + refConfidences = (0.8196,) + refBoxes = ((331, 233, 85, 107),) normAssertDetections(self, refClassIds, refConfidences, refBoxes, classIds, confidences, boxes,confThreshold, scoreDiff, iouDiff) @@ -251,14 +243,13 @@ class dnn_test(NewOpenCVTests): def test_classification_model(self): img_path = self.find_dnn_file("dnn/googlenet_0.png") - weights = self.find_dnn_file("dnn/squeezenet_v1.1.caffemodel", required=False) - config = self.find_dnn_file("dnn/squeezenet_v1.1.prototxt") + weights = self.find_dnn_file("dnn/squeezenet_v1.1.onnx", required=False) ref = np.load(self.find_dnn_file("dnn/squeezenet_v1.1_prob.npy")) - if weights is None or config is None: - raise unittest.SkipTest("Missing DNN test files (dnn/squeezenet_v1.1.{prototxt/caffemodel}). Verify OPENCV_DNN_TEST_DATA_PATH configuration parameter.") + if weights is None: + raise unittest.SkipTest("Missing DNN test files (dnn/squeezenet_v1.1.onnx). Verify OPENCV_DNN_TEST_DATA_PATH configuration parameter.") frame = cv.imread(img_path) - model = cv.dnn_ClassificationModel(config, weights) + model = cv.dnn_ClassificationModel(weights) model.setInputSize(227, 227) model.setInputCrop(True) @@ -323,54 +314,6 @@ class dnn_test(NewOpenCVTests): normAssertDetections(self, refClassIds, refScores, refBoxes, testClassIds, testScores, testBoxes, 0.5) - def test_async(self): - # bug: https://github.com/opencv/opencv/issues/26376 - raise unittest.SkipTest("The new dnn engine does not support async inference") - - timeout = 10*1000*10**6 # in nanoseconds (10 sec) - proto = self.find_dnn_file('dnn/layers/layer_convolution.prototxt') - model = self.find_dnn_file('dnn/layers/layer_convolution.caffemodel') - if proto is None or model is None: - raise unittest.SkipTest("Missing DNN test files (dnn/layers/layer_convolution.{prototxt/caffemodel}). Verify OPENCV_DNN_TEST_DATA_PATH configuration parameter.") - - print('\n') - for backend, target in self.dnnBackendsAndTargets: - if backend != cv.dnn.DNN_BACKEND_INFERENCE_ENGINE: - continue - - printParams(backend, target) - - netSync = cv.dnn.readNet(proto, model, engine=cv.dnn.ENGINE_CLASSIC) - netSync.setPreferableBackend(backend) - netSync.setPreferableTarget(target) - - netAsync = cv.dnn.readNet(proto, model) - netAsync.setPreferableBackend(backend) - netAsync.setPreferableTarget(target) - - # Generate inputs - numInputs = 10 - inputs = [] - for _ in range(numInputs): - inputs.append(np.random.standard_normal([2, 6, 75, 113]).astype(np.float32)) - - # Run synchronously - refs = [] - for i in range(numInputs): - netSync.setInput(inputs[i]) - refs.append(netSync.forward()) - - # Run asynchronously. To make test more robust, process inputs in the reversed order. - outs = [] - for i in reversed(range(numInputs)): - netAsync.setInput(inputs[i]) - outs.insert(0, netAsync.forwardAsync()) - - for i in reversed(range(numInputs)): - ret, result = outs[i].get(timeoutNs=float(timeout)) - self.assertTrue(ret) - normAssert(self, refs[i], result, 'Index: %d' % i, 1e-10) - def test_nms(self): confs = (1, 1) rects = ((0, 0, 0.4, 0.4), (0, 0, 0.2, 0.4)) # 0.5 overlap @@ -401,51 +344,8 @@ class dnn_test(NewOpenCVTests): return [inputs[0][:,:,self.ystart:self.yend,self.xstart:self.xend]] cv.dnn_registerLayer('CropCaffe', CropLayer) - proto = ''' - name: "TestCrop" - input: "input" - input_shape - { - dim: 1 - dim: 2 - dim: 5 - dim: 5 - } - input: "roi" - input_shape - { - dim: 1 - dim: 2 - dim: 3 - dim: 3 - } - layer { - name: "Crop" - type: "CropCaffe" - bottom: "input" - bottom: "roi" - top: "Crop" - }''' - - net = cv.dnn.readNetFromCaffe(bytearray(proto.encode())) - for backend, target in self.dnnBackendsAndTargets: - if backend != cv.dnn.DNN_BACKEND_OPENCV: - continue - - printParams(backend, target) - - net.setPreferableBackend(backend) - net.setPreferableTarget(target) - src_shape = [1, 2, 5, 5] - dst_shape = [1, 2, 3, 3] - inp = np.arange(0, np.prod(src_shape), dtype=np.float32).reshape(src_shape) - roi = np.empty(dst_shape, dtype=np.float32) - net.setInput(inp, "input") - net.setInput(roi, "roi") - out = net.forward() - ref = inp[:, :, 1:4, 1:4] - normAssert(self, out, ref) + # Skipped: Requires ONNX custom layer multi-input support and Python binding fixes for Net.connect (see #26200). cv.dnn_unregisterLayer('CropCaffe') # check that dnn module can work with 3D tensor as input for network diff --git a/modules/dnn/misc/quantize_face_detector.py b/modules/dnn/misc/quantize_face_detector.py index a5baac659d..541592386a 100644 --- a/modules/dnn/misc/quantize_face_detector.py +++ b/modules/dnn/misc/quantize_face_detector.py @@ -33,7 +33,7 @@ assert(not args.quantize or not args.fp16) dtype = tf.float16 if args.fp16 else tf.float32 ################################################################################ -cvNet = cv.dnn.readNetFromCaffe(args.proto, args.model) +cvNet = cv.dnn.readNet(args.proto, args.model) def dnnLayer(name): return cvNet.getLayer(long(cvNet.getLayerId(name))) diff --git a/modules/dnn/perf/perf_caffe.cpp b/modules/dnn/perf/perf_caffe.cpp deleted file mode 100644 index eacc205294..0000000000 --- a/modules/dnn/perf/perf_caffe.cpp +++ /dev/null @@ -1,109 +0,0 @@ -// This file is part of OpenCV project. -// It is subject to the license terms in the LICENSE file found in the top-level directory -// of this distribution and at http://opencv.org/license.html. -// -// Copyright (C) 2017, Intel Corporation, all rights reserved. -// Third party copyrights are property of their respective owners. - -// Recommends run this performance test via -// ./bin/opencv_perf_dnn 2> /dev/null | grep "PERFSTAT" -A 3 -// because whole output includes Caffe's logs. -// -// Note: Be sure that interesting version of Caffe was linked. -// -// How to build Intel-Caffe with MKLDNN backend -// ============================================ -// mkdir build && cd build -// cmake -DCMAKE_BUILD_TYPE=Release \ -// -DUSE_MKLDNN_AS_DEFAULT_ENGINE=ON \ -// -DUSE_MKL2017_AS_DEFAULT_ENGINE=OFF \ -// -DCPU_ONLY=ON \ -// -DCMAKE_INSTALL_PREFIX=/usr/local .. && make -j8 -// sudo make install -// -// In case of problems with cublas_v2.h at include/caffe/util/device_alternate.hpp: add line -// #define CPU_ONLY -// before the first line -// #ifdef CPU_ONLY // CPU-only Caffe. - -#if defined(HAVE_CAFFE) || defined(HAVE_CLCAFFE) - -#include "perf_precomp.hpp" -#include -#include - -namespace opencv_test { - -static caffe::Net* initNet(std::string proto, std::string weights) -{ - proto = findDataFile(proto); - weights = findDataFile(weights, false); - -#ifdef HAVE_CLCAFFE - caffe::Caffe::set_mode(caffe::Caffe::GPU); - caffe::Caffe::SetDevice(0); - - caffe::Net* net = - new caffe::Net(proto, caffe::TEST, caffe::Caffe::GetDefaultDevice()); -#else - caffe::Caffe::set_mode(caffe::Caffe::CPU); - - caffe::Net* net = new caffe::Net(proto, caffe::TEST); -#endif - - net->CopyTrainedLayersFrom(weights); - - caffe::Blob* input = net->input_blobs()[0]; - - CV_Assert(input->num() == 1); - CV_Assert(input->channels() == 3); - - Mat inputMat(input->height(), input->width(), CV_32FC3, (char*)input->cpu_data()); - randu(inputMat, 0.0f, 1.0f); - - net->Forward(); - return net; -} - -PERF_TEST(AlexNet_caffe, CaffePerfTest) -{ - caffe::Net* net = initNet("dnn/bvlc_alexnet.prototxt", - "dnn/bvlc_alexnet.caffemodel"); - TEST_CYCLE() net->Forward(); - SANITY_CHECK_NOTHING(); -} - -PERF_TEST(GoogLeNet_caffe, CaffePerfTest) -{ - caffe::Net* net = initNet("dnn/bvlc_googlenet.prototxt", - "dnn/bvlc_googlenet.caffemodel"); - TEST_CYCLE() net->Forward(); - SANITY_CHECK_NOTHING(); -} - -PERF_TEST(ResNet50_caffe, CaffePerfTest) -{ - caffe::Net* net = initNet("dnn/ResNet-50-deploy.prototxt", - "dnn/ResNet-50-model.caffemodel"); - TEST_CYCLE() net->Forward(); - SANITY_CHECK_NOTHING(); -} - -PERF_TEST(SqueezeNet_v1_1_caffe, CaffePerfTest) -{ - caffe::Net* net = initNet("dnn/squeezenet_v1.1.prototxt", - "dnn/squeezenet_v1.1.caffemodel"); - TEST_CYCLE() net->Forward(); - SANITY_CHECK_NOTHING(); -} - -PERF_TEST(MobileNet_SSD, CaffePerfTest) -{ - caffe::Net* net = initNet("dnn/MobileNetSSD_deploy_19e3ec3.prototxt", - "dnn/MobileNetSSD_deploy_19e3ec3.caffemodel"); - TEST_CYCLE() net->Forward(); - SANITY_CHECK_NOTHING(); -} - -} // namespace -#endif // HAVE_CAFFE diff --git a/modules/dnn/perf/perf_net.cpp b/modules/dnn/perf/perf_net.cpp index ecfb4f6c86..029958dc07 100644 --- a/modules/dnn/perf/perf_net.cpp +++ b/modules/dnn/perf/perf_net.cpp @@ -9,6 +9,7 @@ #include "opencv2/core/ocl.hpp" #include "opencv2/dnn/shape_utils.hpp" +#include #include "../test/test_common.hpp" @@ -96,17 +97,17 @@ public: PERF_TEST_P_(DNNTestNetwork, AlexNet) { - processNet("dnn/bvlc_alexnet.caffemodel", "dnn/bvlc_alexnet.prototxt", cv::Size(227, 227)); + processNet("dnn/onnx/models/alexnet.onnx", "", cv::Size(227, 227)); } PERF_TEST_P_(DNNTestNetwork, GoogLeNet) { - processNet("dnn/bvlc_googlenet.caffemodel", "dnn/bvlc_googlenet.prototxt", cv::Size(224, 224)); + processNet("dnn/onnx/models/googlenet.onnx", "", cv::Size(224, 224)); } PERF_TEST_P_(DNNTestNetwork, ResNet_50) { - processNet("dnn/ResNet-50-model.caffemodel", "dnn/ResNet-50-deploy.prototxt", cv::Size(224, 224)); + processNet("dnn/onnx/models/resnet50v1.onnx", "", cv::Size(224, 224)); } PERF_TEST_P_(DNNTestNetwork, ResNet_18_v1_ONNX) @@ -131,7 +132,7 @@ PERF_TEST_P_(DNNTestNetwork, ResNet50_QDQ_ONNX) PERF_TEST_P_(DNNTestNetwork, SqueezeNet_v1_1) { - processNet("dnn/squeezenet_v1.1.caffemodel", "dnn/squeezenet_v1.1.prototxt", cv::Size(227, 227)); + processNet("dnn/onnx/models/squeezenet.onnx", "", cv::Size(227, 227)); } PERF_TEST_P_(DNNTestNetwork, Inception_5h) @@ -144,12 +145,28 @@ PERF_TEST_P_(DNNTestNetwork, SSD) { applyTestTag(CV_TEST_TAG_DEBUG_VERYLONG); - processNet("dnn/VGG_ILSVRC2016_SSD_300x300_iter_440000.caffemodel", "dnn/ssd_vgg16.prototxt", cv::Size(300, 300)); + // The Caffe-SSD specific handling lives in the new engine importer only; + // the classic importer can no longer load this model. + auto engine_forced = static_cast( + utils::getConfigurationParameterSizeT("OPENCV_FORCE_DNN_ENGINE", dnn::ENGINE_AUTO)); + if (engine_forced == dnn::ENGINE_CLASSIC) + throw SkipTestException("SSD_VGG16 is supported on the new DNN engine only"); + + processNet("dnn/onnx/models/ssd_vgg16.onnx", "", cv::Size(300, 300)); } -PERF_TEST_P_(DNNTestNetwork, MobileNet_SSD_Caffe) +PERF_TEST_P_(DNNTestNetwork, MobileNet_SSD_v1_ONNX) { - processNet("dnn/MobileNetSSD_deploy_19e3ec3.caffemodel", "dnn/MobileNetSSD_deploy_19e3ec3.prototxt", cv::Size(300, 300)); + // Dynamic-shape preprocessing in this model needs the new engine; OpenVINO uses the classic one. + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_NGRAPH); + + // This model expects a uint8 NHWC image as input. + Mat image(cv::Size(300, 300), CV_8UC3); + randu(image, 0, 255); + int imsize[] = {1, image.rows, image.cols, 3}; + Mat input(4, imsize, CV_8U, image.data); + processNet("dnn/onnx/models/ssd_mobilenet_v1_12.onnx", "", input); } PERF_TEST_P_(DNNTestNetwork, MobileNet_SSD_v1_TensorFlow) @@ -162,18 +179,9 @@ PERF_TEST_P_(DNNTestNetwork, MobileNet_SSD_v2_TensorFlow) processNet("dnn/ssd_mobilenet_v2_coco_2018_03_29.pb", "ssd_mobilenet_v2_coco_2018_03_29.pbtxt", cv::Size(300, 300)); } -PERF_TEST_P_(DNNTestNetwork, MobileNet_SSD_v1_ONNX) -{ - Mat image(cv::Size(300, 300), CV_8UC3); - randu(image, 0, 255); - int imsize[] = {1, image.rows, image.cols, 3}; - Mat input(4, imsize, CV_8U, image.data); - processNet("dnn/onnx/models/ssd_mobilenet_v1_12.onnx", "", input); -} - PERF_TEST_P_(DNNTestNetwork, DenseNet_121) { - processNet("dnn/DenseNet_121.caffemodel", "dnn/DenseNet_121.prototxt", cv::Size(224, 224)); + processNet("dnn/onnx/models/densenet121.onnx", "", cv::Size(224, 224)); } PERF_TEST_P_(DNNTestNetwork, OpenPose_pose_mpi_faster_4_stages) @@ -184,7 +192,8 @@ PERF_TEST_P_(DNNTestNetwork, OpenPose_pose_mpi_faster_4_stages) throw SkipTestException(""); // The same .caffemodel but modified .prototxt // See https://github.com/CMU-Perceptual-Computing-Lab/openpose/blob/master/src/openpose/pose/poseParameters.cpp - processNet("dnn/openpose_pose_mpi.caffemodel", "dnn/openpose_pose_mpi_faster_4_stages.prototxt", cv::Size(368, 368)); + // processNet("dnn/openpose_pose_mpi.caffemodel", "dnn/openpose_pose_mpi_faster_4_stages.prototxt", cv::Size(368, 368)); + processNet("dnn/onnx/models/openpose_pose_mpi.onnx", "", cv::Size(368, 368)); } PERF_TEST_P_(DNNTestNetwork, Inception_v2_SSD_TensorFlow) diff --git a/modules/dnn/src/caffe/caffe_importer.cpp b/modules/dnn/src/caffe/caffe_importer.cpp deleted file mode 100644 index 4fc292510a..0000000000 --- a/modules/dnn/src/caffe/caffe_importer.cpp +++ /dev/null @@ -1,828 +0,0 @@ -/*M/////////////////////////////////////////////////////////////////////////////////////// -// -// IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. -// -// By downloading, copying, installing or using the software you agree to this license. -// If you do not agree to this license, do not download, install, -// copy or use the software. -// -// -// License Agreement -// For Open Source Computer Vision Library -// -// Copyright (C) 2013, OpenCV Foundation, all rights reserved. -// Third party copyrights are property of their respective owners. -// -// Redistribution and use in source and binary forms, with or without modification, -// are permitted provided that the following conditions are met: -// -// * Redistribution's of source code must retain the above copyright notice, -// this list of conditions and the following disclaimer. -// -// * Redistribution's in binary form must reproduce the above copyright notice, -// this list of conditions and the following disclaimer in the documentation -// and/or other materials provided with the distribution. -// -// * The name of the copyright holders may not be used to endorse or promote products -// derived from this software without specific prior written permission. -// -// This software is provided by the copyright holders and contributors "as is" and -// any express or implied warranties, including, but not limited to, the implied -// warranties of merchantability and fitness for a particular purpose are disclaimed. -// In no event shall the Intel Corporation or contributors be liable for any direct, -// indirect, incidental, special, exemplary, or consequential damages -// (including, but not limited to, procurement of substitute goods or services; -// loss of use, data, or profits; or business interruption) however caused -// and on any theory of liability, whether in contract, strict liability, -// or tort (including negligence or otherwise) arising in any way out of -// the use of this software, even if advised of the possibility of such damage. -// -//M*/ - -#include "../precomp.hpp" -#include "../net_impl.hpp" - -#ifdef HAVE_PROTOBUF -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include "caffe_io.hpp" -#endif - -#include -#include - -namespace cv { -namespace dnn { -CV__DNN_INLINE_NS_BEGIN - -#ifdef HAVE_PROTOBUF -using ::google::protobuf::RepeatedFieldRef; -using ::google::protobuf::Message; -using ::google::protobuf::Descriptor; -using ::google::protobuf::FieldDescriptor; -using ::google::protobuf::Reflection; - -namespace -{ - -template -static cv::String toString(const T &v) -{ - std::ostringstream ss; - ss << v; - return ss.str(); -} - -static inline -MatShape parseBlobShape(const caffe::BlobShape& _input_shape) -{ - MatShape shape; - for (int i = 0; i < _input_shape.dim_size(); i++) - { - shape.push_back((int)_input_shape.dim(i)); - } - return shape; -} - -class CaffeImporter -{ - FPDenormalsIgnoreHintScope fp_denormals_ignore_scope; - - caffe::NetParameter net; - caffe::NetParameter netBinary; - -public: - - CaffeImporter(const char *prototxt, const char *caffeModel) - { - CV_TRACE_FUNCTION(); - - ReadNetParamsFromTextFileOrDie(prototxt, &net); - - if (caffeModel && caffeModel[0]) - ReadNetParamsFromBinaryFileOrDie(caffeModel, &netBinary); - } - - CaffeImporter(const char *dataProto, size_t lenProto, - const char *dataModel, size_t lenModel) - { - CV_TRACE_FUNCTION(); - - ReadNetParamsFromTextBufferOrDie(dataProto, lenProto, &net); - - if (dataModel != NULL && lenModel > 0) - ReadNetParamsFromBinaryBufferOrDie(dataModel, lenModel, &netBinary); - } - - void extractCustomParams(const google::protobuf::UnknownFieldSet& unknownFields, cv::dnn::LayerParams ¶ms) - { - const int numFields = unknownFields.field_count(); - for (int i = 0; i < numFields; ++i) - { - const google::protobuf::UnknownField& field = unknownFields.field(i); - CV_Assert(field.type() == google::protobuf::UnknownField::TYPE_GROUP); - CV_CheckGE(field.group().field_count(), 2, "UnknownField should have at least 2 items: name and value"); - std::string fieldName(field.group().field(0).length_delimited()); - std::string fieldValue(field.group().field(1).length_delimited()); - params.set(fieldName, fieldValue); - } - } - - void addParam(const Message &msg, const FieldDescriptor *field, cv::dnn::LayerParams ¶ms) - { - const Reflection *refl = msg.GetReflection(); - int type = field->cpp_type(); - bool isRepeated = field->is_repeated(); - const std::string name(field->name()); - - #define SET_UP_FILED(getter, arrayConstr, gtype) \ - if (isRepeated) { \ - const RepeatedFieldRef v = refl->GetRepeatedFieldRef(msg, field); \ - params.set(name, DictValue::arrayConstr(v.begin(), (int)v.size())); \ - } \ - else { \ - params.set(name, refl->getter(msg, field)); \ - } - - switch (type) - { - case FieldDescriptor::CPPTYPE_INT32: - SET_UP_FILED(GetInt32, arrayInt, ::google::protobuf::int32); - break; - case FieldDescriptor::CPPTYPE_UINT32: - SET_UP_FILED(GetUInt32, arrayInt, ::google::protobuf::uint32); - break; - case FieldDescriptor::CPPTYPE_INT64: - SET_UP_FILED(GetInt32, arrayInt, ::google::protobuf::int64); - break; - case FieldDescriptor::CPPTYPE_UINT64: - SET_UP_FILED(GetUInt32, arrayInt, ::google::protobuf::uint64); - break; - case FieldDescriptor::CPPTYPE_BOOL: - SET_UP_FILED(GetBool, arrayInt, bool); - break; - case FieldDescriptor::CPPTYPE_DOUBLE: - SET_UP_FILED(GetDouble, arrayReal, double); - break; - case FieldDescriptor::CPPTYPE_FLOAT: - SET_UP_FILED(GetFloat, arrayReal, float); - break; - case FieldDescriptor::CPPTYPE_STRING: - if (isRepeated) { - const RepeatedFieldRef v = refl->GetRepeatedFieldRef(msg, field); - params.set(name, DictValue::arrayString(v.begin(), (int)v.size())); - } - else { - params.set(name, refl->GetString(msg, field)); - } - break; - case FieldDescriptor::CPPTYPE_ENUM: - if (isRepeated) { - int size = refl->FieldSize(msg, field); - std::vector buf(size); - for (int i = 0; i < size; i++) - buf[i] = refl->GetRepeatedEnum(msg, field, i)->name(); - params.set(name, DictValue::arrayString(buf.begin(), size)); - } - else { - params.set(name, std::string(refl->GetEnum(msg, field)->name())); - } - break; - default: - CV_Error(Error::StsError, "Unknown type \"" + String(field->type_name()) + "\" in prototxt"); - } - } - - inline static bool ends_with_param(const std::string &str) - { - static const std::string _param("_param"); - return (str.size() >= _param.size()) && str.compare(str.size() - _param.size(), _param.size(), _param) == 0; - } - - template - typename std::enable_if::value, void>::type - extractLayerParams(const MSG &msg, cv::dnn::LayerParams ¶ms, bool isInternal = false) - { - const Descriptor *msgDesc = msg.GetDescriptor(); - const Reflection *msgRefl = msg.GetReflection(); - - for (int fieldId = 0; fieldId < msgDesc->field_count(); fieldId++) - { - const FieldDescriptor *fd = msgDesc->field(fieldId); - - if (!isInternal && !ends_with_param(std::string(fd->name()))) - continue; - - const google::protobuf::UnknownFieldSet& unknownFields = msgRefl->GetUnknownFields(msg); - bool hasData = fd->is_required() || - (!fd->is_repeated() && !fd->is_required() && msgRefl->HasField(msg, fd)) || - (fd->is_repeated() && msgRefl->FieldSize(msg, fd) > 0) || - !unknownFields.empty(); - if (!hasData) - continue; - - extractCustomParams(unknownFields, params); - if (fd->cpp_type() == FieldDescriptor::CPPTYPE_MESSAGE) - { - if (fd->is_repeated()) //Extract only first item! - extractLayerParams(msgRefl->GetRepeatedMessage(msg, fd, 0), params, true); - else - extractLayerParams(msgRefl->GetMessage(msg, fd), params, true); - } - else - { - addParam(msg, fd, params); - } - } - } - - template - typename std::enable_if::value, void>::type - extractLayerParams(const MSG &msg, cv::dnn::LayerParams ¶ms, bool isInternal = false) - { - CV_Error(Error::StsError, "DNN/CAFFE: do not have your message be a MessageLite"); - } - - void blobShapeFromProto(const caffe::BlobProto &pbBlob, MatShape& shape) - { - shape.clear(); - if (pbBlob.has_num() || pbBlob.has_channels() || pbBlob.has_height() || pbBlob.has_width()) - { - shape.push_back(pbBlob.num()); - shape.push_back(pbBlob.channels()); - shape.push_back(pbBlob.height()); - shape.push_back(pbBlob.width()); - } - else if (pbBlob.has_shape()) - { - shape = parseBlobShape(pbBlob.shape()); - } - else - shape.resize(1, 1); // Is a scalar. - } - - template - typename std::enable_if::value, void>::type - AssertBlobProtoIsFloat(const BLOB_PROTO &pbBlob) { - CV_DbgAssert(pbBlob.GetDescriptor()->FindFieldByLowercaseName("data")->cpp_type() == FieldDescriptor::CPPTYPE_FLOAT); - } - - template - typename std::enable_if::value, void>::type - AssertBlobProtoIsFloat(const BLOB_PROTO &pbBlob) { - CV_Error(Error::StsError, "DNN/CAFFE: do not have your message be a MessageLite"); - } - - void blobFromProto(const caffe::BlobProto &pbBlob, cv::Mat &dstBlob) - { - MatShape shape; - blobShapeFromProto(pbBlob, shape); - - dstBlob.create((int)shape.size(), &shape[0], CV_32F); - if (pbBlob.data_size()) - { - // Single precision floats. - CV_Assert(pbBlob.data_size() == (int)dstBlob.total()); - - AssertBlobProtoIsFloat(pbBlob); - Mat(dstBlob.size, CV_32F, (void*)pbBlob.data().data()).copyTo(dstBlob); - } - else - { - CV_Assert(pbBlob.has_raw_data()); - const std::string& raw_data = pbBlob.raw_data(); - if (pbBlob.raw_data_type() == caffe::FLOAT16) - { - // Half precision floats. - CV_Assert(raw_data.size() / 2 == (int)dstBlob.total()); - - Mat halfs((int)shape.size(), &shape[0], CV_16FC1, (void*)raw_data.c_str()); - halfs.convertTo(dstBlob, CV_32F); - } - else if (pbBlob.raw_data_type() == caffe::FLOAT) - { - CV_Assert(raw_data.size() / 4 == (int)dstBlob.total()); - Mat((int)shape.size(), &shape[0], CV_32FC1, (void*)raw_data.c_str()).copyTo(dstBlob); - } - else - CV_Error(Error::StsNotImplemented, "Unexpected blob data type"); - } - } - - void extractBinaryLayerParams(const caffe::LayerParameter& layer, LayerParams& layerParams) - { - const std::string &name = layer.name(); - - int li; - for (li = 0; li != netBinary.layer_size(); li++) - { - const caffe::LayerParameter& binLayer = netBinary.layer(li); - // Break if the layer name is the same and the blobs are not cleared - if (binLayer.name() == name && binLayer.blobs_size() != 0) - break; - } - - if (li == netBinary.layer_size()) - return; - - caffe::LayerParameter* binLayer = netBinary.mutable_layer(li); - const int numBlobs = binLayer->blobs_size(); - std::vector blobs(numBlobs); - binLayer->mutable_blobs()->ExtractSubrange(0, numBlobs, blobs.data()); - layerParams.blobs.resize(numBlobs); - for (int bi = 0; bi < numBlobs; bi++) - { - blobFromProto(*blobs[bi], layerParams.blobs[bi]); - delete blobs[bi]; - } - } - - Ptr addLayer(Net& dstNet, - const String& type, - const String& name, - LayerParams& layerParams, - const std::vector& inputs, - const std::vector& outputs) - { - layerParams.type = type; - layerParams.name = name; - Ptr layer = LayerFactory::createLayerInstance(type, layerParams); - if (!layer) { - CV_Error(Error::StsError, "Can't create layer " + name + " with type " + type); - return nullptr; - } - - for (const String& inputName : inputs) - layer->inputs.push_back(dstNet.getArg(inputName)); - for (const String& outputName : outputs) - layer->outputs.push_back(dstNet.getArg(outputName)); - layer->netimpl = dstNet.getImpl(); - CV_Assert(dstNet.getImpl()->dump_indent == 3); - return layer; - } - - struct BlobNote - { - BlobNote(const std::string &_name, int _layerId, int _outNum) : - name(_name), layerId(_layerId), outNum(_outNum) {} - - std::string name; - int layerId, outNum; - }; - - std::vector addedBlobs; - std::map layerCounter; - - void populateNet(Net dstNet, bool newEngine) - { - CV_TRACE_FUNCTION(); - - int layersSize = net.layer_size(); - layerCounter.clear(); - - // OLD ENGINE - if(!newEngine) - { - addedBlobs.clear(); - addedBlobs.reserve(layersSize + 1); - } - std::vector netInputs(net.input_size()); - std::vector inp_shapes; - - // NEW ENGINE - Net::Impl* netImpl = dstNet.getImpl(); - std::vector> curr_prog; - std::vector modelInputs, modelOutputs; - - { - int net_input_size = net.input_size(); - for (int inNum = 0; inNum < net_input_size; inNum++) - { - if (newEngine) - { - modelInputs.push_back(netImpl->newArg(net.input(inNum), DNN_ARG_INPUT)); - netImpl->args.at(modelInputs.back().idx).type = CV_32F; - } - else - { - addedBlobs.push_back(BlobNote(net.input(inNum), 0, inNum)); - netInputs[inNum] = net.input(inNum); - } - } - - if (net.input_dim_size() > 0) // deprecated in Caffe proto - { - int net_input_dim_size = net.input_dim_size(); - CV_Check(net_input_dim_size, net_input_dim_size % 4 == 0, ""); - CV_CheckEQ(net_input_dim_size, net_input_size * 4, ""); - for (int inp_id = 0; inp_id < net_input_size; inp_id++) - { - int dim = inp_id * 4; - MatShape shape(4); - shape[0] = net.input_dim(dim); - shape[1] = net.input_dim(dim+1); - shape[2] = net.input_dim(dim+2); - shape[3] = net.input_dim(dim+3); - if (newEngine) - netImpl->args.at(modelInputs[inp_id].idx).shape = shape; - else - inp_shapes.push_back(shape); - } - } - else if (net.input_shape_size() > 0) // deprecated in Caffe proto - { - int net_input_shape_size = net.input_shape_size(); - CV_CheckEQ(net_input_shape_size, net_input_size, ""); - for (int inp_id = 0; inp_id < net_input_shape_size; inp_id++) - { - MatShape shape = parseBlobShape(net.input_shape(inp_id)); - if (newEngine) - netImpl->args.at(modelInputs[inp_id].idx).shape = shape; - else - inp_shapes.push_back(shape); - } - } - else - { - for (int inp_id = 0; inp_id < net_input_size; inp_id++) - { - MatShape shape; // empty - if (newEngine) - netImpl->args.at(modelInputs[inp_id].idx).shape = shape; - else - inp_shapes.push_back(shape); - } - } - } - - if (newEngine && net.layer(layersSize - 1).type() == "Silence") - { - CV_LOG_WARNING(NULL, "Caffe parser: Silence layer was ignored"); - layersSize--; - } - - for (int li = 0; li < layersSize; li++) - { - const caffe::LayerParameter &layer = net.layer(li); - String name = layer.name(); - String type = layer.type(); - LayerParams layerParams; - - extractLayerParams(layer, layerParams); - extractBinaryLayerParams(layer, layerParams); - - if (newEngine && li == layersSize - 1) - { - for (int outNum = 0; outNum < layer.top_size(); outNum++) - modelOutputs.push_back(netImpl->newArg(layer.top(outNum), DNN_ARG_OUTPUT)); - } - - int repetitions = layerCounter[name]++; - if (repetitions) - name += String("_") + toString(repetitions); - - if (type == "Input") - { - for (int outNum = 0; outNum < layer.top_size(); outNum++) - { - if (newEngine) - { - modelInputs.push_back(netImpl->newArg(layer.top(outNum), DNN_ARG_INPUT)); - netImpl->args.at(modelInputs.back().idx).type = CV_32F; - } - else - { - addOutput(layer, 0, outNum); - addedBlobs.back().outNum = netInputs.size(); - netInputs.push_back(addedBlobs.back().name); - } - } - if (layer.has_input_param()) - { - const caffe::InputParameter &inputParameter = layer.input_param(); - int input_shape_size = inputParameter.shape_size(); - CV_CheckEQ(input_shape_size, layer.top_size(), ""); - for (int inp_id = 0; inp_id < input_shape_size; inp_id++) - { - MatShape shape = parseBlobShape(inputParameter.shape(inp_id)); - if (newEngine) - { - int inputIdx = modelInputs.size() - input_shape_size + inp_id; - netImpl->args.at(modelInputs[inputIdx].idx).shape = shape; - } - else - { - inp_shapes.push_back(shape); - } - } - } - continue; - } - else if (type == "BatchNorm") - { - if (!layerParams.get("use_global_stats", true)) - { - CV_Assert_N(layer.bottom_size() == 1, layer.top_size() == 1); - - LayerParams mvnParams; - mvnParams.set("eps", layerParams.get("eps", 1e-5)); - std::string mvnName = name + "/mvn"; - - int repetitions = layerCounter[mvnName]++; - if (repetitions) - mvnName += String("_") + toString(repetitions); - - if (newEngine) - { - Ptr netLayer = addLayer( - dstNet, "MVN", mvnName, mvnParams, - {layer.bottom(0)}, - {layer.top(0)}); - curr_prog.push_back(netLayer); - continue; - } - else - { - int mvnId = dstNet.addLayer(mvnName, "MVN", mvnParams); - addInput(layer.bottom(0), mvnId, 0, dstNet); - addOutput(layer, mvnId, 0); - net.mutable_layer(li)->set_bottom(0, layer.top(0)); - layerParams.blobs[0].setTo(0); // mean - layerParams.blobs[1].setTo(1); // std - } - } - } - else if (type == "Axpy") - { - CV_Assert_N(layer.bottom_size() == 3, layer.top_size() == 1); - - std::string scaleName = name + "/scale"; - int repetitions = layerCounter[scaleName]++; - if (repetitions) { - scaleName += String("_") + toString(repetitions); - } - - LayerParams scaleParams; - scaleParams.set("axis", 1); - scaleParams.set("has_bias", false); - - if (newEngine) - { - std::string intermediateTensor = scaleName + "_intermediate_output"; - Ptr netLayerScale= addLayer( - dstNet, "Scale", scaleName, scaleParams, - {layer.bottom(2), layer.bottom(0)}, - {intermediateTensor}); - curr_prog.push_back(netLayerScale); - - LayerParams eltwiseParams; - Ptr netLayerEltwise = addLayer( - dstNet, "Eltwise", name, eltwiseParams, - {intermediateTensor, layer.bottom(1)}, - {layer.top(0)}); - curr_prog.push_back(netLayerEltwise); - continue; - } - else - { - int scaleId = dstNet.addLayer(scaleName, "Scale", scaleParams); - addInput(layer.bottom(2), scaleId, 0, dstNet); - addInput(layer.bottom(0), scaleId, 1, dstNet); - addOutput(layer, scaleId, 0); - net.mutable_layer(li)->set_bottom(0, layer.top(0)); - net.mutable_layer(li)->mutable_bottom()->RemoveLast(); - type = "Eltwise"; - } - } - else if (type == "Resample") - { - CV_Assert(layer.bottom_size() == 1 || layer.bottom_size() == 2); - type = "Resize"; - String interp = toLowerCase(layerParams.get("type")); - layerParams.set("interpolation", interp == "linear" ? "bilinear" : interp); - - if (layerParams.has("factor")) - { - float factor = layerParams.get("factor"); - CV_Assert(layer.bottom_size() != 2 || factor == 1.0); - layerParams.set("zoom_factor", factor); - - if ((interp == "linear" && factor != 1.0) || - (interp == "nearest" && factor < 1.0)) - CV_Error(Error::StsNotImplemented, "Unsupported Resample mode"); - } - } - else if ("Convolution" == type) - { - CV_Assert(layer.bottom_size() == layer.top_size()); - for (int i = 0; i < layer.bottom_size(); i++) - { - if (newEngine) - { - Ptr netLayer = addLayer( - dstNet, type, layer.top(i), layerParams, - {layer.bottom(i)}, {layer.top(i)}); - curr_prog.push_back(netLayer); - } - else - { - int conv_id = dstNet.addLayer(layer.top(i), type, layerParams); - addInput(layer.bottom(i), conv_id, 0, dstNet); - addedBlobs.push_back(BlobNote(layer.top(i), conv_id, 0)); - } - } - continue; - } - else if ("ConvolutionDepthwise" == type) - { - type = "Convolution"; - } - else if (type == "Softmax"){ - // set default axis to 1 - if(!layerParams.has("axis")) - layerParams.set("axis", 1); - } - else if ("Proposal" == type) - { - if (newEngine && layer.top_size() == 1) - { - // Add unused optional second output and create the Proposal layer - std::vector layerInputs; - for (int inNum = 0; inNum < layer.bottom_size(); inNum++) - layerInputs.push_back(layer.bottom(inNum)); - Ptr netLayer = addLayer( - dstNet, type, name, layerParams, - layerInputs, {layer.top(0), name + "___output_scores"}); - curr_prog.push_back(netLayer); - continue; - } - } - else if ("Silence" == type) - { - if (newEngine) - { - CV_LOG_WARNING(NULL, "Caffe parser: Silence layer was ignored"); - continue; - } - } - - if (newEngine) - { - std::vector layerInputs, layerOutputs; - for (int inNum = 0; inNum < layer.bottom_size(); inNum++) - layerInputs.push_back(layer.bottom(inNum)); - for (int outNum = 0; outNum < layer.top_size(); outNum++) - layerOutputs.push_back(layer.top(outNum)); - - Ptr netLayer = addLayer( - dstNet, type, name, layerParams, - layerInputs, layerOutputs); - curr_prog.push_back(netLayer); - } - else - { - int id = dstNet.addLayer(name, type, layerParams); - for (int inNum = 0; inNum < layer.bottom_size(); inNum++) - addInput(layer.bottom(inNum), id, inNum, dstNet); - for (int outNum = 0; outNum < layer.top_size(); outNum++) - addOutput(layer, id, outNum); - } - } - - if (newEngine) - { - Ptr curr_graph = netImpl->newGraph(net.name(), modelInputs, true); - curr_graph->setOutputs(modelOutputs); - curr_graph->setProg(curr_prog); - - netImpl->mainGraph = curr_graph; - netImpl->modelFormat = DNN_MODEL_CAFFE; - netImpl->originalLayout = DATA_LAYOUT_NCHW; - netImpl->prepareForInference(); - } - else - { - dstNet.setInputsNames(netInputs); - - if (inp_shapes.size() > 0) - { - CV_CheckEQ(inp_shapes.size(), netInputs.size(), ""); - for (int inp_id = 0; inp_id < inp_shapes.size(); inp_id++) - dstNet.setInputShape(netInputs[inp_id], inp_shapes[inp_id]); - } - - addedBlobs.clear(); - } - } - - void addOutput(const caffe::LayerParameter &layer, int layerId, int outNum) - { - const std::string &name = layer.top(outNum); - - bool haveDups = false; - for (int idx = (int)addedBlobs.size() - 1; idx >= 0; idx--) - { - if (addedBlobs[idx].name == name) - { - haveDups = true; - break; - } - } - - if (haveDups) - { - bool isInplace = layer.bottom_size() > outNum && layer.bottom(outNum) == name; - if (!isInplace) - CV_Error(Error::StsBadArg, "Duplicate blobs produced by multiple sources"); - } - - addedBlobs.push_back(BlobNote(name, layerId, outNum)); - } - - void addInput(const std::string &name, int layerId, int inNum, Net &dstNet) - { - int idx; - for (idx = (int)addedBlobs.size() - 1; idx >= 0; idx--) - { - if (addedBlobs[idx].name == name) - break; - } - - if (idx < 0) - { - CV_Error(Error::StsObjectNotFound, "Can't find output blob \"" + name + "\""); - } - - dstNet.connect(addedBlobs[idx].layerId, addedBlobs[idx].outNum, layerId, inNum); - } -}; - -} - -Net readNetFromCaffe(const String &prototxt, - const String &caffeModel, /*= String()*/ - int engine) -{ - static const int engine_forced = (int)utils::getConfigurationParameterSizeT("OPENCV_FORCE_DNN_ENGINE", ENGINE_AUTO); - if(engine_forced != ENGINE_AUTO) - engine = engine_forced; - - CaffeImporter caffeImporter(prototxt.c_str(), caffeModel.c_str()); - Net net; - caffeImporter.populateNet(net, engine == ENGINE_NEW || engine == ENGINE_AUTO); - return net; -} - -Net readNetFromCaffe(const char *bufferProto, size_t lenProto, - const char *bufferModel, size_t lenModel, - int engine) -{ - static const int engine_forced = (int)utils::getConfigurationParameterSizeT("OPENCV_FORCE_DNN_ENGINE", ENGINE_AUTO); - if(engine_forced != ENGINE_AUTO) - engine = engine_forced; - - CaffeImporter caffeImporter(bufferProto, lenProto, bufferModel, lenModel); - Net net; - caffeImporter.populateNet(net, engine == ENGINE_NEW || engine == ENGINE_AUTO); - return net; -} - -Net readNetFromCaffe(const std::vector& bufferProto, - const std::vector& bufferModel, - int engine) -{ - const char* bufferProtoPtr = reinterpret_cast(&bufferProto[0]); - const char* bufferModelPtr = bufferModel.empty() ? NULL : - reinterpret_cast(&bufferModel[0]); - return readNetFromCaffe(bufferProtoPtr, bufferProto.size(), - bufferModelPtr, bufferModel.size(), - engine); -} - -#else // HAVE_PROTOBUF - -#define DNN_PROTOBUF_UNSUPPORTED() CV_Error(Error::StsError, "DNN/Caffe: Build OpenCV with Protobuf to import Caffe models") - -Net readNetFromCaffe(const String &, const String &, int) { - DNN_PROTOBUF_UNSUPPORTED(); -} - -Net readNetFromCaffe(const char *, size_t, const char *, size_t, int) { - DNN_PROTOBUF_UNSUPPORTED(); -} - -Net readNetFromCaffe(const std::vector&, const std::vector&, int) { - DNN_PROTOBUF_UNSUPPORTED(); -} - -#endif // HAVE_PROTOBUF - -CV__DNN_INLINE_NS_END -}} // namespace diff --git a/modules/dnn/src/caffe/caffe_shrinker.cpp b/modules/dnn/src/caffe/caffe_shrinker.cpp deleted file mode 100644 index a23ff5deb3..0000000000 --- a/modules/dnn/src/caffe/caffe_shrinker.cpp +++ /dev/null @@ -1,80 +0,0 @@ -// This file is part of OpenCV project. -// It is subject to the license terms in the LICENSE file found in the top-level directory -// of this distribution and at http://opencv.org/license.html. -// -// Copyright (C) 2017, Intel Corporation, all rights reserved. -// Third party copyrights are property of their respective owners. - -#include "../precomp.hpp" - -#ifdef HAVE_PROTOBUF -#include -#include "caffe_io.hpp" -#endif - -namespace cv { namespace dnn { -CV__DNN_INLINE_NS_BEGIN - -#ifdef HAVE_PROTOBUF - -void shrinkCaffeModel(const String& src, const String& dst, const std::vector& layersTypes) -{ - CV_TRACE_FUNCTION(); - - std::vector types(layersTypes); - if (types.empty()) - { - types.push_back("Convolution"); - types.push_back("InnerProduct"); - } - - caffe::NetParameter net; - ReadNetParamsFromBinaryFileOrDie(src.c_str(), &net); - - for (int i = 0; i < net.layer_size(); ++i) - { - caffe::LayerParameter* lp = net.mutable_layer(i); - if (std::find(types.begin(), types.end(), lp->type()) == types.end()) - { - continue; - } - for (int j = 0; j < lp->blobs_size(); ++j) - { - caffe::BlobProto* blob = lp->mutable_blobs(j); - CV_Assert(blob->data_size() != 0); // float32 array. - - Mat floats(1, blob->data_size(), CV_32FC1, (void*)blob->data().data()); - Mat halfs(1, blob->data_size(), CV_16FC1); - floats.convertTo(halfs, CV_16F); // Convert to float16. - - blob->clear_data(); // Clear float32 data. - - // Set float16 data. - blob->set_raw_data(halfs.data, halfs.total() * halfs.elemSize()); - blob->set_raw_data_type(caffe::FLOAT16); - } - } -#if GOOGLE_PROTOBUF_VERSION < 3005000 - size_t msgSize = saturate_cast(net.ByteSize()); -#else - size_t msgSize = net.ByteSizeLong(); -#endif - std::vector output(msgSize); - net.SerializeWithCachedSizesToArray(&output[0]); - - std::ofstream ofs(dst.c_str(), std::ios::binary); - ofs.write((const char*)&output[0], msgSize); - ofs.close(); -} - -#else - -void shrinkCaffeModel(const String& src, const String& dst, const std::vector& types) -{ - CV_Error(cv::Error::StsNotImplemented, "libprotobuf required to import data from Caffe models"); -} - -#endif // HAVE_PROTOBUF - -CV__DNN_INLINE_NS_END -}} // namespace diff --git a/modules/dnn/src/caffe/opencv-caffe.proto b/modules/dnn/src/caffe/opencv-caffe.proto deleted file mode 100644 index d540591f82..0000000000 --- a/modules/dnn/src/caffe/opencv-caffe.proto +++ /dev/null @@ -1,1649 +0,0 @@ -/*M/////////////////////////////////////////////////////////////////////////////////////// -//COPYRIGHT -// -//All contributions by the University of California: -//Copyright (c) 2014, The Regents of the University of California (Regents) -//All rights reserved. -// -//All other contributions: -//Copyright (c) 2014, the respective contributors -//All rights reserved. -// -//Caffe uses a shared copyright model: each contributor holds copyright over -//their contributions to Caffe. The project versioning records all such -//contribution and copyright details. If a contributor wants to further mark -//their specific copyright on a particular contribution, they should indicate -//their copyright solely in the commit message of the change when it is -//committed. -// -//LICENSE -// -//Redistribution and use in source and binary forms, with or without -//modification, are permitted provided that the following conditions are met: -// -//1. Redistributions of source code must retain the above copyright notice, this -// list of conditions and the following disclaimer. -//2. Redistributions in binary form must reproduce the above copyright notice, -// this list of conditions and the following disclaimer in the documentation -// and/or other materials provided with the distribution. -// -//THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -//ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -//WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -//DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR -//ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -//(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -//LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND -//ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -//(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -//SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -//CONTRIBUTION AGREEMENT -// -//By contributing to the BVLC/caffe repository through pull-request, comment, -//or otherwise, the contributor releases their content to the -//license and copyright terms herein. -// -//M*/ - -syntax = "proto2"; - -package opencv_caffe; - -// NVIDIA's Caffe feature is used to store fp16 weights, https://github.com/NVIDIA/caffe: -// Math and storage types -enum Type { - DOUBLE = 0; - FLOAT = 1; - FLOAT16 = 2; - INT = 3; // math not supported - UINT = 4; // math not supported -} - -// Specifies the shape (dimensions) of a Blob. -message BlobShape { - repeated int64 dim = 1 [packed = true]; -} - -message BlobProto { - optional BlobShape shape = 7; - repeated float data = 5 [packed = true]; - repeated float diff = 6 [packed = true]; - repeated double double_data = 8 [packed = true]; - repeated double double_diff = 9 [packed = true]; - - // NVIDIA's Caffe fields begin. - optional Type raw_data_type = 10; - optional bytes raw_data = 12 [packed = false]; - // NVIDIA's Caffe fields end. - - // 4D dimensions -- deprecated. Use "shape" instead. - optional int32 num = 1 [default = 0]; - optional int32 channels = 2 [default = 0]; - optional int32 height = 3 [default = 0]; - optional int32 width = 4 [default = 0]; -} - -// The BlobProtoVector is simply a way to pass multiple blobproto instances -// around. -message BlobProtoVector { - repeated BlobProto blobs = 1; -} - -message PermuteParameter { - // The new orders of the axes of data. Notice it should be with - // in the same range as the input data, and it starts from 0. - // Do not provide repeated order. - repeated uint32 order = 1; -} - -// Message that stores parameters used by NormalizeBBoxLayer -message NormalizeBBoxParameter { - optional bool across_spatial = 1 [default = true]; - // Initial value of scale. Default is 1.0 for all - optional FillerParameter scale_filler = 2; - // Whether or not scale parameters are shared across channels. - optional bool channel_shared = 3 [default = true]; - // Epsilon for not dividing by zero while normalizing variance - optional float eps = 4 [default = 1e-10]; -} - -// Message that store parameters used by PriorBoxLayer -message PriorBoxParameter { - // Encode/decode type. - enum CodeType { - CORNER = 1; - CENTER_SIZE = 2; - } - // Minimum box size (in pixels). Required! - repeated float min_size = 1; - // Maximum box size (in pixels). Required! - repeated float max_size = 2; - // Various of aspect ratios. Duplicate ratios will be ignored. - // If none is provided, we use default ratio 1. - repeated float aspect_ratio = 3; - // If true, will flip each aspect ratio. - // For example, if there is aspect ratio "r", - // we will generate aspect ratio "1.0/r" as well. - optional bool flip = 4 [default = true]; - // If true, will clip the prior so that it is within [0, 1] - optional bool clip = 5 [default = true]; - // Variance for adjusting the prior bboxes. - repeated float variance = 6; - // By default, we calculate img_height, img_width, step_x, step_y based on - // bottom[0] (feat) and bottom[1] (img). Unless these values are explicitly - // provided. - // Explicitly provide the img_size. - optional uint32 img_size = 7; - // Either img_size or img_h/img_w should be specified; not both. - optional uint32 img_h = 8; - optional uint32 img_w = 9; - // Explicitly provide the step size. - optional float step = 10; - // Either step or step_h/step_w should be specified; not both. - optional float step_h = 11; - optional float step_w = 12; - // Offset to the top left corner of each cell. - optional float offset = 13 [default = 0.5]; - // Offset to the top corner of each cell. - repeated float offset_h = 14; - // Offset to the left corner of each cell. - repeated float offset_w = 15; - // Priox boxes width (in pixels). - repeated float width = 16; - // Priox boxes height (in pixels). - repeated float height = 17; -} - -// Message that store parameters used by DetectionOutputLayer -message DetectionOutputParameter { - // Number of classes to be predicted. Required! - optional uint32 num_classes = 1; - // If true, bounding box are shared among different classes. - optional bool share_location = 2 [default = true]; - // Background label id. If there is no background class, - // set it as -1. - optional int32 background_label_id = 3 [default = 0]; - // Parameters used for non maximum suppression. - optional NonMaximumSuppressionParameter nms_param = 4; - // Parameters used for saving detection results. - optional SaveOutputParameter save_output_param = 5; - // Type of coding method for bbox. - optional PriorBoxParameter.CodeType code_type = 6 [default = CORNER]; - // If true, variance is encoded in target; otherwise we need to adjust the - // predicted offset accordingly. - optional bool variance_encoded_in_target = 8 [default = false]; - // Number of total bboxes to be kept per image after nms step. - // -1 means keeping all bboxes after nms step. - optional int32 keep_top_k = 7 [default = -1]; - // Only consider detections whose confidences are larger than a threshold. - // If not provided, consider all boxes. - optional float confidence_threshold = 9; - // If prior boxes are normalized to [0, 1] or not. - optional bool normalized_bbox = 10 [default = true]; - // OpenCV custom parameter - optional bool clip = 1000 [default = false]; -} - -message Datum { - optional int32 channels = 1; - optional int32 height = 2; - optional int32 width = 3; - // the actual image data, in bytes - optional bytes data = 4; - optional int32 label = 5; - // Optionally, the datum could also hold float data. - repeated float float_data = 6; - // If true data contains an encoded image that need to be decoded - optional bool encoded = 7 [default = false]; -} - -message FillerParameter { - // The filler type. - optional string type = 1 [default = 'constant']; - optional float value = 2 [default = 0]; // the value in constant filler - optional float min = 3 [default = 0]; // the min value in uniform filler - optional float max = 4 [default = 1]; // the max value in uniform filler - optional float mean = 5 [default = 0]; // the mean value in Gaussian filler - optional float std = 6 [default = 1]; // the std value in Gaussian filler - // The expected number of non-zero output weights for a given input in - // Gaussian filler -- the default -1 means don't perform sparsification. - optional int32 sparse = 7 [default = -1]; - // Normalize the filler variance by fan_in, fan_out, or their average. - // Applies to 'xavier' and 'msra' fillers. - enum VarianceNorm { - FAN_IN = 0; - FAN_OUT = 1; - AVERAGE = 2; - } - optional VarianceNorm variance_norm = 8 [default = FAN_IN]; -} - -message NetParameter { - optional string name = 1; // consider giving the network a name - // DEPRECATED. See InputParameter. The input blobs to the network. - repeated string input = 3; - // DEPRECATED. See InputParameter. The shape of the input blobs. - repeated BlobShape input_shape = 8; - - // 4D input dimensions -- deprecated. Use "input_shape" instead. - // If specified, for each input blob there should be four - // values specifying the num, channels, height and width of the input blob. - // Thus, there should be a total of (4 * #input) numbers. - repeated int32 input_dim = 4; - - // Whether the network will force every layer to carry out backward operation. - // If set False, then whether to carry out backward is determined - // automatically according to the net structure and learning rates. - optional bool force_backward = 5 [default = false]; - // The current "state" of the network, including the phase, level, and stage. - // Some layers may be included/excluded depending on this state and the states - // specified in the layers' include and exclude fields. - optional NetState state = 6; - - // Print debugging information about results while running Net::Forward, - // Net::Backward, and Net::Update. - optional bool debug_info = 7 [default = false]; - - // The layers that make up the net. Each of their configurations, including - // connectivity and behavior, is specified as a LayerParameter. - repeated LayerParameter layer = 100; // ID 100 so layers are printed last. - - // DEPRECATED: use 'layer' instead. - repeated V1LayerParameter layers = 2; -} - -// NOTE -// Update the next available ID when you add a new SolverParameter field. -// -// SolverParameter next available ID: 41 (last added: type) -message SolverParameter { - ////////////////////////////////////////////////////////////////////////////// - // Specifying the train and test networks - // - // Exactly one train net must be specified using one of the following fields: - // train_net_param, train_net, net_param, net - // One or more test nets may be specified using any of the following fields: - // test_net_param, test_net, net_param, net - // If more than one test net field is specified (e.g., both net and - // test_net are specified), they will be evaluated in the field order given - // above: (1) test_net_param, (2) test_net, (3) net_param/net. - // A test_iter must be specified for each test_net. - // A test_level and/or a test_stage may also be specified for each test_net. - ////////////////////////////////////////////////////////////////////////////// - - // Proto filename for the train net, possibly combined with one or more - // test nets. - optional string net = 24; - // Inline train net param, possibly combined with one or more test nets. - optional NetParameter net_param = 25; - - optional string train_net = 1; // Proto filename for the train net. - repeated string test_net = 2; // Proto filenames for the test nets. - optional NetParameter train_net_param = 21; // Inline train net params. - repeated NetParameter test_net_param = 22; // Inline test net params. - - // The states for the train/test nets. Must be unspecified or - // specified once per net. - // - // By default, all states will have solver = true; - // train_state will have phase = TRAIN, - // and all test_state's will have phase = TEST. - // Other defaults are set according to the NetState defaults. - optional NetState train_state = 26; - repeated NetState test_state = 27; - - // The number of iterations for each test net. - repeated int32 test_iter = 3; - - // The number of iterations between two testing phases. - optional int32 test_interval = 4 [default = 0]; - optional bool test_compute_loss = 19 [default = false]; - // If true, run an initial test pass before the first iteration, - // ensuring memory availability and printing the starting value of the loss. - optional bool test_initialization = 32 [default = true]; - optional float base_lr = 5; // The base learning rate - // the number of iterations between displaying info. If display = 0, no info - // will be displayed. - optional int32 display = 6; - // Display the loss averaged over the last average_loss iterations - optional int32 average_loss = 33 [default = 1]; - optional int32 max_iter = 7; // the maximum number of iterations - // accumulate gradients over `iter_size` x `batch_size` instances - optional int32 iter_size = 36 [default = 1]; - - // The learning rate decay policy. The currently implemented learning rate - // policies are as follows: - // - fixed: always return base_lr. - // - step: return base_lr * gamma ^ (floor(iter / step)) - // - exp: return base_lr * gamma ^ iter - // - inv: return base_lr * (1 + gamma * iter) ^ (- power) - // - multistep: similar to step but it allows non uniform steps defined by - // stepvalue - // - poly: the effective learning rate follows a polynomial decay, to be - // zero by the max_iter. return base_lr (1 - iter/max_iter) ^ (power) - // - sigmoid: the effective learning rate follows a sigmod decay - // return base_lr ( 1/(1 + exp(-gamma * (iter - stepsize)))) - // - // where base_lr, max_iter, gamma, step, stepvalue and power are defined - // in the solver parameter protocol buffer, and iter is the current iteration. - optional string lr_policy = 8; - optional float gamma = 9; // The parameter to compute the learning rate. - optional float power = 10; // The parameter to compute the learning rate. - optional float momentum = 11; // The momentum value. - optional float weight_decay = 12; // The weight decay. - // regularization types supported: L1 and L2 - // controlled by weight_decay - optional string regularization_type = 29 [default = "L2"]; - // the stepsize for learning rate policy "step" - optional int32 stepsize = 13; - // the stepsize for learning rate policy "multistep" - repeated int32 stepvalue = 34; - - // Set clip_gradients to >= 0 to clip parameter gradients to that L2 norm, - // whenever their actual L2 norm is larger. - optional float clip_gradients = 35 [default = -1]; - - optional int32 snapshot = 14 [default = 0]; // The snapshot interval - optional string snapshot_prefix = 15; // The prefix for the snapshot. - // whether to snapshot diff in the results or not. Snapshotting diff will help - // debugging but the final protocol buffer size will be much larger. - optional bool snapshot_diff = 16 [default = false]; - enum SnapshotFormat { - HDF5 = 0; - BINARYPROTO = 1; - } - optional SnapshotFormat snapshot_format = 37 [default = BINARYPROTO]; - // the mode solver will use: 0 for CPU and 1 for GPU. Use GPU in default. - enum SolverMode { - CPU = 0; - GPU = 1; - } - optional SolverMode solver_mode = 17 [default = GPU]; - // the device_id will that be used in GPU mode. Use device_id = 0 in default. - optional int32 device_id = 18 [default = 0]; - // If non-negative, the seed with which the Solver will initialize the Caffe - // random number generator -- useful for reproducible results. Otherwise, - // (and by default) initialize using a seed derived from the system clock. - optional int64 random_seed = 20 [default = -1]; - - // type of the solver - optional string type = 40 [default = "SGD"]; - - // numerical stability for RMSProp, AdaGrad and AdaDelta and Adam - optional float delta = 31 [default = 1e-8]; - // parameters for the Adam solver - optional float momentum2 = 39 [default = 0.999]; - - // RMSProp decay value - // MeanSquare(t) = rms_decay*MeanSquare(t-1) + (1-rms_decay)*SquareGradient(t) - optional float rms_decay = 38 [default = 0.99]; - - // If true, print information about the state of the net that may help with - // debugging learning problems. - optional bool debug_info = 23 [default = false]; - - // If false, don't save a snapshot after training finishes. - optional bool snapshot_after_train = 28 [default = true]; - - // DEPRECATED: old solver enum types, use string instead - enum SolverType { - SGD = 0; - NESTEROV = 1; - ADAGRAD = 2; - RMSPROP = 3; - ADADELTA = 4; - ADAM = 5; - } - // DEPRECATED: use type instead of solver_type - optional SolverType solver_type = 30 [default = SGD]; -} - -// A message that stores the solver snapshots -message SolverState { - optional int32 iter = 1; // The current iteration - optional string learned_net = 2; // The file that stores the learned net. - repeated BlobProto history = 3; // The history for sgd solvers - optional int32 current_step = 4 [default = 0]; // The current step for learning rate -} - -enum Phase { - TRAIN = 0; - TEST = 1; -} - -message NetState { - optional Phase phase = 1 [default = TEST]; - optional int32 level = 2 [default = 0]; - repeated string stage = 3; -} - -message NetStateRule { - // Set phase to require the NetState have a particular phase (TRAIN or TEST) - // to meet this rule. - optional Phase phase = 1; - - // Set the minimum and/or maximum levels in which the layer should be used. - // Leave undefined to meet the rule regardless of level. - optional int32 min_level = 2; - optional int32 max_level = 3; - - // Customizable sets of stages to include or exclude. - // The net must have ALL of the specified stages and NONE of the specified - // "not_stage"s to meet the rule. - // (Use multiple NetStateRules to specify conjunctions of stages.) - repeated string stage = 4; - repeated string not_stage = 5; -} - -// Specifies training parameters (multipliers on global learning constants, -// and the name and other settings used for weight sharing). -message ParamSpec { - // The names of the parameter blobs -- useful for sharing parameters among - // layers, but never required otherwise. To share a parameter between two - // layers, give it a (non-empty) name. - optional string name = 1; - - // Whether to require shared weights to have the same shape, or just the same - // count -- defaults to STRICT if unspecified. - optional DimCheckMode share_mode = 2; - enum DimCheckMode { - // STRICT (default) requires that num, channels, height, width each match. - STRICT = 0; - // PERMISSIVE requires only the count (num*channels*height*width) to match. - PERMISSIVE = 1; - } - - // The multiplier on the global learning rate for this parameter. - optional float lr_mult = 3 [default = 1.0]; - - // The multiplier on the global weight decay for this parameter. - optional float decay_mult = 4 [default = 1.0]; -} - -// NOTE -// Update the next available ID when you add a new LayerParameter field. -// -// LayerParameter next available layer-specific ID: 147 (last added: recurrent_param) -message LayerParameter { - optional string name = 1; // the layer name - optional string type = 2; // the layer type - repeated string bottom = 3; // the name of each bottom blob - repeated string top = 4; // the name of each top blob - - // The train / test phase for computation. - optional Phase phase = 10; - - // The amount of weight to assign each top blob in the objective. - // Each layer assigns a default value, usually of either 0 or 1, - // to each top blob. - repeated float loss_weight = 5; - - // Specifies training parameters (multipliers on global learning constants, - // and the name and other settings used for weight sharing). - repeated ParamSpec param = 6; - - // The blobs containing the numeric parameters of the layer. - repeated BlobProto blobs = 7; - - // Specifies whether to backpropagate to each bottom. If unspecified, - // Caffe will automatically infer whether each input needs backpropagation - // to compute parameter gradients. If set to true for some inputs, - // backpropagation to those inputs is forced; if set false for some inputs, - // backpropagation to those inputs is skipped. - // - // The size must be either 0 or equal to the number of bottoms. - repeated bool propagate_down = 11; - - // Rules controlling whether and when a layer is included in the network, - // based on the current NetState. You may specify a non-zero number of rules - // to include OR exclude, but not both. If no include or exclude rules are - // specified, the layer is always included. If the current NetState meets - // ANY (i.e., one or more) of the specified rules, the layer is - // included/excluded. - repeated NetStateRule include = 8; - repeated NetStateRule exclude = 9; - - // Parameters for data pre-processing. - optional TransformationParameter transform_param = 100; - - // Parameters shared by loss layers. - optional LossParameter loss_param = 101; - - // Layer type-specific parameters. - // - // Note: certain layers may have more than one computational engine - // for their implementation. These layers include an Engine type and - // engine parameter for selecting the implementation. - // The default for the engine is set by the ENGINE switch at compile-time. - optional AccuracyParameter accuracy_param = 102; - optional ArgMaxParameter argmax_param = 103; - optional BatchNormParameter batch_norm_param = 139; - optional BiasParameter bias_param = 141; - optional ConcatParameter concat_param = 104; - optional ContrastiveLossParameter contrastive_loss_param = 105; - optional ConvolutionParameter convolution_param = 106; - optional CropParameter crop_param = 144; - optional DataParameter data_param = 107; - optional DetectionOutputParameter detection_output_param = 147; - optional DropoutParameter dropout_param = 108; - optional DummyDataParameter dummy_data_param = 109; - optional EltwiseParameter eltwise_param = 110; - optional ELUParameter elu_param = 140; - optional EmbedParameter embed_param = 137; - optional ExpParameter exp_param = 111; - optional FlattenParameter flatten_param = 135; - optional HDF5DataParameter hdf5_data_param = 112; - optional HDF5OutputParameter hdf5_output_param = 113; - optional HingeLossParameter hinge_loss_param = 114; - optional ImageDataParameter image_data_param = 115; - optional InfogainLossParameter infogain_loss_param = 116; - optional InnerProductParameter inner_product_param = 117; - optional InputParameter input_param = 143; - optional LogParameter log_param = 134; - optional LRNParameter lrn_param = 118; - optional MemoryDataParameter memory_data_param = 119; - optional MVNParameter mvn_param = 120; - optional NormalizeBBoxParameter norm_param = 149; - optional PermuteParameter permute_param = 148; - optional ParameterParameter parameter_param = 145; - optional PoolingParameter pooling_param = 121; - optional PowerParameter power_param = 122; - optional PReLUParameter prelu_param = 131; - optional PriorBoxParameter prior_box_param = 150; - optional ProposalParameter proposal_param = 201; - optional PSROIPoolingParameter psroi_pooling_param = 10002; // https://github.com/daijifeng001/caffe-rfcn - optional PythonParameter python_param = 130; - optional RecurrentParameter recurrent_param = 146; - optional ReductionParameter reduction_param = 136; - optional ReLUParameter relu_param = 123; - optional ReshapeParameter reshape_param = 133; - optional ROIPoolingParameter roi_pooling_param = 8266711; // https://github.com/rbgirshick/caffe-fast-rcnn/tree/fast-rcnn - optional ScaleParameter scale_param = 142; - optional SigmoidParameter sigmoid_param = 124; - optional SoftmaxParameter softmax_param = 125; - optional SPPParameter spp_param = 132; - optional SliceParameter slice_param = 126; - optional TanHParameter tanh_param = 127; - optional ThresholdParameter threshold_param = 128; - optional TileParameter tile_param = 138; - optional WindowDataParameter window_data_param = 129; -} - -// Message that stores parameters used to apply transformation -// to the data layer's data -message TransformationParameter { - // For data pre-processing, we can do simple scaling and subtracting the - // data mean, if provided. Note that the mean subtraction is always carried - // out before scaling. - optional float scale = 1 [default = 1]; - // Specify if we want to randomly mirror data. - optional bool mirror = 2 [default = false]; - // Specify if we would like to randomly crop an image. - optional uint32 crop_size = 3 [default = 0]; - // mean_file and mean_value cannot be specified at the same time - optional string mean_file = 4; - // if specified can be repeated once (would subtract it from all the channels) - // or can be repeated the same number of times as channels - // (would subtract them from the corresponding channel) - repeated float mean_value = 5; - // Force the decoded image to have 3 color channels. - optional bool force_color = 6 [default = false]; - // Force the decoded image to have 1 color channels. - optional bool force_gray = 7 [default = false]; -} - -// Message that stores parameters shared by loss layers -message LossParameter { - // If specified, ignore instances with the given label. - optional int32 ignore_label = 1; - // How to normalize the loss for loss layers that aggregate across batches, - // spatial dimensions, or other dimensions. Currently only implemented in - // SoftmaxWithLoss and SigmoidCrossEntropyLoss layers. - enum NormalizationMode { - // Divide by the number of examples in the batch times spatial dimensions. - // Outputs that receive the ignore label will NOT be ignored in computing - // the normalization factor. - FULL = 0; - // Divide by the total number of output locations that do not take the - // ignore_label. If ignore_label is not set, this behaves like FULL. - VALID = 1; - // Divide by the batch size. - BATCH_SIZE = 2; - // Do not normalize the loss. - NONE = 3; - } - // For historical reasons, the default normalization for - // SigmoidCrossEntropyLoss is BATCH_SIZE and *not* VALID. - optional NormalizationMode normalization = 3 [default = VALID]; - // Deprecated. Ignored if normalization is specified. If normalization - // is not specified, then setting this to false will be equivalent to - // normalization = BATCH_SIZE to be consistent with previous behavior. - optional bool normalize = 2; -} - -// Messages that store parameters used by individual layer types follow, in -// alphabetical order. - -message AccuracyParameter { - // When computing accuracy, count as correct by comparing the true label to - // the top k scoring classes. By default, only compare to the top scoring - // class (i.e. argmax). - optional uint32 top_k = 1 [default = 1]; - - // The "label" axis of the prediction blob, whose argmax corresponds to the - // predicted label -- may be negative to index from the end (e.g., -1 for the - // last axis). For example, if axis == 1 and the predictions are - // (N x C x H x W), the label blob is expected to contain N*H*W ground truth - // labels with integer values in {0, 1, ..., C-1}. - optional int32 axis = 2 [default = 1]; - - // If specified, ignore instances with the given label. - optional int32 ignore_label = 3; -} - -message ArgMaxParameter { - // If true produce pairs (argmax, maxval) - optional bool out_max_val = 1 [default = false]; - optional uint32 top_k = 2 [default = 1]; - // The axis along which to maximise -- may be negative to index from the - // end (e.g., -1 for the last axis). - // By default ArgMaxLayer maximizes over the flattened trailing dimensions - // for each index of the first / num dimension. - optional int32 axis = 3; -} - -message ConcatParameter { - // The axis along which to concatenate -- may be negative to index from the - // end (e.g., -1 for the last axis). Other axes must have the - // same dimension for all the bottom blobs. - // By default, ConcatLayer concatenates blobs along the "channels" axis (1). - optional int32 axis = 2 [default = 1]; - - // DEPRECATED: alias for "axis" -- does not support negative indexing. - optional uint32 concat_dim = 1 [default = 1]; -} - -message BatchNormParameter { - // If false, accumulate global mean/variance values via a moving average. If - // true, use those accumulated values instead of computing mean/variance - // across the batch. - optional bool use_global_stats = 1; - // How much does the moving average decay each iteration? - optional float moving_average_fraction = 2 [default = .999]; - // Small value to add to the variance estimate so that we don't divide by - // zero. - optional float eps = 3 [default = 1e-5]; - // It true, scale and add biases. Source: https://github.com/NVIDIA/caffe/ - optional bool scale_bias = 7 [default = false]; -} - -message BiasParameter { - // The first axis of bottom[0] (the first input Blob) along which to apply - // bottom[1] (the second input Blob). May be negative to index from the end - // (e.g., -1 for the last axis). - // - // For example, if bottom[0] is 4D with shape 100x3x40x60, the output - // top[0] will have the same shape, and bottom[1] may have any of the - // following shapes (for the given value of axis): - // (axis == 0 == -4) 100; 100x3; 100x3x40; 100x3x40x60 - // (axis == 1 == -3) 3; 3x40; 3x40x60 - // (axis == 2 == -2) 40; 40x60 - // (axis == 3 == -1) 60 - // Furthermore, bottom[1] may have the empty shape (regardless of the value of - // "axis") -- a scalar bias. - optional int32 axis = 1 [default = 1]; - - // (num_axes is ignored unless just one bottom is given and the bias is - // a learned parameter of the layer. Otherwise, num_axes is determined by the - // number of axes by the second bottom.) - // The number of axes of the input (bottom[0]) covered by the bias - // parameter, or -1 to cover all axes of bottom[0] starting from `axis`. - // Set num_axes := 0, to add a zero-axis Blob: a scalar. - optional int32 num_axes = 2 [default = 1]; - - // (filler is ignored unless just one bottom is given and the bias is - // a learned parameter of the layer.) - // The initialization for the learned bias parameter. - // Default is the zero (0) initialization, resulting in the BiasLayer - // initially performing the identity operation. - optional FillerParameter filler = 3; -} - -message ContrastiveLossParameter { - // margin for dissimilar pair - optional float margin = 1 [default = 1.0]; - // The first implementation of this cost did not exactly match the cost of - // Hadsell et al 2006 -- using (margin - d^2) instead of (margin - d)^2. - // legacy_version = false (the default) uses (margin - d)^2 as proposed in the - // Hadsell paper. New models should probably use this version. - // legacy_version = true uses (margin - d^2). This is kept to support / - // reproduce existing models and results - optional bool legacy_version = 2 [default = false]; -} - -message ConvolutionParameter { - optional uint32 num_output = 1; // The number of outputs for the layer - optional bool bias_term = 2 [default = true]; // whether to have bias terms - - // Pad, kernel size, and stride are all given as a single value for equal - // dimensions in all spatial dimensions, or once per spatial dimension. - repeated uint32 pad = 3; // The padding size; defaults to 0 - repeated uint32 kernel_size = 4; // The kernel size - repeated uint32 stride = 6; // The stride; defaults to 1 - // Factor used to dilate the kernel, (implicitly) zero-filling the resulting - // holes. (Kernel dilation is sometimes referred to by its use in the - // algorithme à trous from Holschneider et al. 1987.) - repeated uint32 dilation = 18; // The dilation; defaults to 1 - - // For 2D convolution only, the *_h and *_w versions may also be used to - // specify both spatial dimensions. - optional uint32 pad_h = 9 [default = 0]; // The padding height (2D only) - optional uint32 pad_w = 10 [default = 0]; // The padding width (2D only) - optional uint32 kernel_h = 11; // The kernel height (2D only) - optional uint32 kernel_w = 12; // The kernel width (2D only) - optional uint32 stride_h = 13; // The stride height (2D only) - optional uint32 stride_w = 14; // The stride width (2D only) - - optional uint32 group = 5 [default = 1]; // The group size for group conv - - optional FillerParameter weight_filler = 7; // The filler for the weight - optional FillerParameter bias_filler = 8; // The filler for the bias - enum Engine { - DEFAULT = 0; - CAFFE = 1; - CUDNN = 2; - } - optional Engine engine = 15 [default = DEFAULT]; - - // The axis to interpret as "channels" when performing convolution. - // Preceding dimensions are treated as independent inputs; - // succeeding dimensions are treated as "spatial". - // With (N, C, H, W) inputs, and axis == 1 (the default), we perform - // N independent 2D convolutions, sliding C-channel (or (C/g)-channels, for - // groups g>1) filters across the spatial axes (H, W) of the input. - // With (N, C, D, H, W) inputs, and axis == 1, we perform - // N independent 3D convolutions, sliding (C/g)-channels - // filters across the spatial axes (D, H, W) of the input. - optional int32 axis = 16 [default = 1]; - - // Whether to force use of the general ND convolution, even if a specific - // implementation for blobs of the appropriate number of spatial dimensions - // is available. (Currently, there is only a 2D-specific convolution - // implementation; for input blobs with num_axes != 2, this option is - // ignored and the ND implementation will be used.) - optional bool force_nd_im2col = 17 [default = false]; -} - -message CropParameter { - // To crop, elements of the first bottom are selected to fit the dimensions - // of the second, reference bottom. The crop is configured by - // - the crop `axis` to pick the dimensions for cropping - // - the crop `offset` to set the shift for all/each dimension - // to align the cropped bottom with the reference bottom. - // All dimensions up to but excluding `axis` are preserved, while - // the dimensions including and trailing `axis` are cropped. - // If only one `offset` is set, then all dimensions are offset by this amount. - // Otherwise, the number of offsets must equal the number of cropped axes to - // shift the crop in each dimension accordingly. - // Note: standard dimensions are N,C,H,W so the default is a spatial crop, - // and `axis` may be negative to index from the end (e.g., -1 for the last - // axis). - optional int32 axis = 1 [default = 2]; - repeated uint32 offset = 2; -} - -message DataParameter { - enum DB { - LEVELDB = 0; - LMDB = 1; - } - // Specify the data source. - optional string source = 1; - // Specify the batch size. - optional uint32 batch_size = 4; - // The rand_skip variable is for the data layer to skip a few data points - // to avoid all asynchronous sgd clients to start at the same point. The skip - // point would be set as rand_skip * rand(0,1). Note that rand_skip should not - // be larger than the number of keys in the database. - // DEPRECATED. Each solver accesses a different subset of the database. - optional uint32 rand_skip = 7 [default = 0]; - optional DB backend = 8 [default = LEVELDB]; - // DEPRECATED. See TransformationParameter. For data pre-processing, we can do - // simple scaling and subtracting the data mean, if provided. Note that the - // mean subtraction is always carried out before scaling. - optional float scale = 2 [default = 1]; - optional string mean_file = 3; - // DEPRECATED. See TransformationParameter. Specify if we would like to randomly - // crop an image. - optional uint32 crop_size = 5 [default = 0]; - // DEPRECATED. See TransformationParameter. Specify if we want to randomly mirror - // data. - optional bool mirror = 6 [default = false]; - // Force the encoded image to have 3 color channels - optional bool force_encoded_color = 9 [default = false]; - // Prefetch queue (Number of batches to prefetch to host memory, increase if - // data access bandwidth varies). - optional uint32 prefetch = 10 [default = 4]; -} - -message NonMaximumSuppressionParameter { - // Threshold to be used in nms. - optional float nms_threshold = 1 [default = 0.3]; - // Maximum number of results to be kept. - optional int32 top_k = 2; - // Parameter for adaptive nms. - optional float eta = 3 [default = 1.0]; -} - -message SaveOutputParameter { - // Output directory. If not empty, we will save the results. - optional string output_directory = 1; - // Output name prefix. - optional string output_name_prefix = 2; - // Output format. - // VOC - PASCAL VOC output format. - // COCO - MS COCO output format. - optional string output_format = 3; - // If you want to output results, must also provide the following two files. - // Otherwise, we will ignore saving results. - // label map file. - optional string label_map_file = 4; - // A file which contains a list of names and sizes with same order - // of the input DB. The file is in the following format: - // name height width - // ... - optional string name_size_file = 5; - // Number of test images. It can be less than the lines specified in - // name_size_file. For example, when we only want to evaluate on part - // of the test images. - optional uint32 num_test_image = 6; -} - -message DropoutParameter { - optional float dropout_ratio = 1 [default = 0.5]; // dropout ratio - // Faster-RCNN framework's parameter. - // source: https://github.com/rbgirshick/caffe-fast-rcnn/tree/faster-rcnn - optional bool scale_train = 2 [default = true]; // scale train or test phase -} - -// DummyDataLayer fills any number of arbitrarily shaped blobs with random -// (or constant) data generated by "Fillers" (see "message FillerParameter"). -message DummyDataParameter { - // This layer produces N >= 1 top blobs. DummyDataParameter must specify 1 or N - // shape fields, and 0, 1 or N data_fillers. - // - // If 0 data_fillers are specified, ConstantFiller with a value of 0 is used. - // If 1 data_filler is specified, it is applied to all top blobs. If N are - // specified, the ith is applied to the ith top blob. - repeated FillerParameter data_filler = 1; - repeated BlobShape shape = 6; - - // 4D dimensions -- deprecated. Use "shape" instead. - repeated uint32 num = 2; - repeated uint32 channels = 3; - repeated uint32 height = 4; - repeated uint32 width = 5; -} - -message EltwiseParameter { - enum EltwiseOp { - PROD = 0; - SUM = 1; - MAX = 2; - } - optional EltwiseOp operation = 1 [default = SUM]; // element-wise operation - repeated float coeff = 2; // blob-wise coefficient for SUM operation - - // Whether to use an asymptotically slower (for >2 inputs) but stabler method - // of computing the gradient for the PROD operation. (No effect for SUM op.) - optional bool stable_prod_grad = 3 [default = true]; -} - -// Message that stores parameters used by ELULayer -message ELUParameter { - // Described in: - // Clevert, D.-A., Unterthiner, T., & Hochreiter, S. (2015). Fast and Accurate - // Deep Network Learning by Exponential Linear Units (ELUs). arXiv - optional float alpha = 1 [default = 1]; -} - -// Message that stores parameters used by EmbedLayer -message EmbedParameter { - optional uint32 num_output = 1; // The number of outputs for the layer - // The input is given as integers to be interpreted as one-hot - // vector indices with dimension num_input. Hence num_input should be - // 1 greater than the maximum possible input value. - optional uint32 input_dim = 2; - - optional bool bias_term = 3 [default = true]; // Whether to use a bias term - optional FillerParameter weight_filler = 4; // The filler for the weight - optional FillerParameter bias_filler = 5; // The filler for the bias - -} - -// Message that stores parameters used by ExpLayer -message ExpParameter { - // ExpLayer computes outputs y = base ^ (shift + scale * x), for base > 0. - // Or if base is set to the default (-1), base is set to e, - // so y = exp(shift + scale * x). - optional float base = 1 [default = -1.0]; - optional float scale = 2 [default = 1.0]; - optional float shift = 3 [default = 0.0]; -} - -/// Message that stores parameters used by FlattenLayer -message FlattenParameter { - // The first axis to flatten: all preceding axes are retained in the output. - // May be negative to index from the end (e.g., -1 for the last axis). - optional int32 axis = 1 [default = 1]; - - // The last axis to flatten: all following axes are retained in the output. - // May be negative to index from the end (e.g., the default -1 for the last - // axis). - optional int32 end_axis = 2 [default = -1]; -} - -// Message that stores parameters used by HDF5DataLayer -message HDF5DataParameter { - // Specify the data source. - optional string source = 1; - // Specify the batch size. - optional uint32 batch_size = 2; - - // Specify whether to shuffle the data. - // If shuffle == true, the ordering of the HDF5 files is shuffled, - // and the ordering of data within any given HDF5 file is shuffled, - // but data between different files are not interleaved; all of a file's - // data are output (in a random order) before moving onto another file. - optional bool shuffle = 3 [default = false]; -} - -message HDF5OutputParameter { - optional string file_name = 1; -} - -message HingeLossParameter { - enum Norm { - L1 = 1; - L2 = 2; - } - // Specify the Norm to use L1 or L2 - optional Norm norm = 1 [default = L1]; -} - -message ImageDataParameter { - // Specify the data source. - optional string source = 1; - // Specify the batch size. - optional uint32 batch_size = 4 [default = 1]; - // The rand_skip variable is for the data layer to skip a few data points - // to avoid all asynchronous sgd clients to start at the same point. The skip - // point would be set as rand_skip * rand(0,1). Note that rand_skip should not - // be larger than the number of keys in the database. - optional uint32 rand_skip = 7 [default = 0]; - // Whether or not ImageLayer should shuffle the list of files at every epoch. - optional bool shuffle = 8 [default = false]; - // It will also resize images if new_height or new_width are not zero. - optional uint32 new_height = 9 [default = 0]; - optional uint32 new_width = 10 [default = 0]; - // Specify if the images are color or gray - optional bool is_color = 11 [default = true]; - // DEPRECATED. See TransformationParameter. For data pre-processing, we can do - // simple scaling and subtracting the data mean, if provided. Note that the - // mean subtraction is always carried out before scaling. - optional float scale = 2 [default = 1]; - optional string mean_file = 3; - // DEPRECATED. See TransformationParameter. Specify if we would like to randomly - // crop an image. - optional uint32 crop_size = 5 [default = 0]; - // DEPRECATED. See TransformationParameter. Specify if we want to randomly mirror - // data. - optional bool mirror = 6 [default = false]; - optional string root_folder = 12 [default = ""]; -} - -message InfogainLossParameter { - // Specify the infogain matrix source. - optional string source = 1; -} - -message InnerProductParameter { - optional uint32 num_output = 1; // The number of outputs for the layer - optional bool bias_term = 2 [default = true]; // whether to have bias terms - optional FillerParameter weight_filler = 3; // The filler for the weight - optional FillerParameter bias_filler = 4; // The filler for the bias - - // The first axis to be lumped into a single inner product computation; - // all preceding axes are retained in the output. - // May be negative to index from the end (e.g., -1 for the last axis). - optional int32 axis = 5 [default = 1]; - // Specify whether to transpose the weight matrix or not. - // If transpose == true, any operations will be performed on the transpose - // of the weight matrix. The weight matrix itself is not going to be transposed - // but rather the transfer flag of operations will be toggled accordingly. - optional bool transpose = 6 [default = false]; -} - -message InputParameter { - // This layer produces N >= 1 top blob(s) to be assigned manually. - // Define N shapes to set a shape for each top. - // Define 1 shape to set the same shape for every top. - // Define no shape to defer to reshaping manually. - repeated BlobShape shape = 1; -} - -// Message that stores parameters used by LogLayer -message LogParameter { - // LogLayer computes outputs y = log_base(shift + scale * x), for base > 0. - // Or if base is set to the default (-1), base is set to e, - // so y = ln(shift + scale * x) = log_e(shift + scale * x) - optional float base = 1 [default = -1.0]; - optional float scale = 2 [default = 1.0]; - optional float shift = 3 [default = 0.0]; -} - -// Message that stores parameters used by LRNLayer -message LRNParameter { - optional uint32 local_size = 1 [default = 5]; - optional float alpha = 2 [default = 1.]; - optional float beta = 3 [default = 0.75]; - enum NormRegion { - ACROSS_CHANNELS = 0; - WITHIN_CHANNEL = 1; - } - optional NormRegion norm_region = 4 [default = ACROSS_CHANNELS]; - optional float k = 5 [default = 1.]; - enum Engine { - DEFAULT = 0; - CAFFE = 1; - CUDNN = 2; - } - optional Engine engine = 6 [default = DEFAULT]; -} - -message MemoryDataParameter { - optional uint32 batch_size = 1; - optional uint32 channels = 2; - optional uint32 height = 3; - optional uint32 width = 4; -} - -message MVNParameter { - // This parameter can be set to false to normalize mean only - optional bool normalize_variance = 1 [default = true]; - - // This parameter can be set to true to perform DNN-like MVN - optional bool across_channels = 2 [default = false]; - - // Epsilon for not dividing by zero while normalizing variance - optional float eps = 3 [default = 1e-9]; -} - -message ParameterParameter { - optional BlobShape shape = 1; -} - -message PoolingParameter { - enum PoolMethod { - MAX = 0; - AVE = 1; - STOCHASTIC = 2; - } - optional PoolMethod pool = 1 [default = MAX]; // The pooling method - // Pad, kernel size, and stride are all given as a single value for equal - // dimensions in height and width or as Y, X pairs. - optional uint32 pad = 4 [default = 0]; // The padding size (equal in Y, X) - optional uint32 pad_h = 9 [default = 0]; // The padding height - optional uint32 pad_w = 10 [default = 0]; // The padding width - optional uint32 kernel_size = 2; // The kernel size (square) - optional uint32 kernel_h = 5; // The kernel height - optional uint32 kernel_w = 6; // The kernel width - optional uint32 stride = 3 [default = 1]; // The stride (equal in Y, X) - optional uint32 stride_h = 7; // The stride height - optional uint32 stride_w = 8; // The stride width - enum Engine { - DEFAULT = 0; - CAFFE = 1; - CUDNN = 2; - } - optional Engine engine = 11 [default = DEFAULT]; - // If global_pooling then it will pool over the size of the bottom by doing - // kernel_h = bottom->height and kernel_w = bottom->width - optional bool global_pooling = 12 [default = false]; - // Specify floor/ceil mode - // source: https://github.com/BVLC/caffe/pull/3057 - optional bool ceil_mode = 13 [default = true]; -} - -message PowerParameter { - // PowerLayer computes outputs y = (shift + scale * x) ^ power. - optional float power = 1 [default = 1.0]; - optional float scale = 2 [default = 1.0]; - optional float shift = 3 [default = 0.0]; -} - -message PythonParameter { - optional string module = 1; - optional string layer = 2; - // This value is set to the attribute `param_str` of the `PythonLayer` object - // in Python before calling the `setup()` method. This could be a number, - // string, dictionary in Python dict format, JSON, etc. You may parse this - // string in `setup` method and use it in `forward` and `backward`. - optional string param_str = 3 [default = '']; - // Whether this PythonLayer is shared among worker solvers during data parallelism. - // If true, each worker solver sequentially run forward from this layer. - // This value should be set true if you are using it as a data layer. - optional bool share_in_parallel = 4 [default = false]; -} - -// Message that stores parameters used by RecurrentLayer -message RecurrentParameter { - // The dimension of the output (and usually hidden state) representation -- - // must be explicitly set to non-zero. - optional uint32 num_output = 1 [default = 0]; - - optional FillerParameter weight_filler = 2; // The filler for the weight - optional FillerParameter bias_filler = 3; // The filler for the bias - - // Whether to enable displaying debug_info in the unrolled recurrent net. - optional bool debug_info = 4 [default = false]; - - // Whether to add as additional inputs (bottoms) the initial hidden state - // blobs, and add as additional outputs (tops) the final timestep hidden state - // blobs. The number of additional bottom/top blobs required depends on the - // recurrent architecture -- e.g., 1 for RNNs, 2 for LSTMs. - optional bool expose_hidden = 5 [default = false]; -} - -// Message that stores parameters used by ReductionLayer -message ReductionParameter { - enum ReductionOp { - SUM = 1; - ASUM = 2; - SUMSQ = 3; - MEAN = 4; - } - - optional ReductionOp operation = 1 [default = SUM]; // reduction operation - - // The first axis to reduce to a scalar -- may be negative to index from the - // end (e.g., -1 for the last axis). - // (Currently, only reduction along ALL "tail" axes is supported; reduction - // of axis M through N, where N < num_axes - 1, is unsupported.) - // Suppose we have an n-axis bottom Blob with shape: - // (d0, d1, d2, ..., d(m-1), dm, d(m+1), ..., d(n-1)). - // If axis == m, the output Blob will have shape - // (d0, d1, d2, ..., d(m-1)), - // and the ReductionOp operation is performed (d0 * d1 * d2 * ... * d(m-1)) - // times, each including (dm * d(m+1) * ... * d(n-1)) individual data. - // If axis == 0 (the default), the output Blob always has the empty shape - // (count 1), performing reduction across the entire input -- - // often useful for creating new loss functions. - optional int32 axis = 2 [default = 0]; - - optional float coeff = 3 [default = 1.0]; // coefficient for output -} - -// Message that stores parameters used by ReLULayer -message ReLUParameter { - // Allow non-zero slope for negative inputs to speed up optimization - // Described in: - // Maas, A. L., Hannun, A. Y., & Ng, A. Y. (2013). Rectifier nonlinearities - // improve neural network acoustic models. In ICML Workshop on Deep Learning - // for Audio, Speech, and Language Processing. - optional float negative_slope = 1 [default = 0]; - enum Engine { - DEFAULT = 0; - CAFFE = 1; - CUDNN = 2; - } - optional Engine engine = 2 [default = DEFAULT]; -} - -message ReshapeParameter { - // Specify the output dimensions. If some of the dimensions are set to 0, - // the corresponding dimension from the bottom layer is used (unchanged). - // Exactly one dimension may be set to -1, in which case its value is - // inferred from the count of the bottom blob and the remaining dimensions. - // For example, suppose we want to reshape a 2D blob "input" with shape 2 x 8: - // - // layer { - // type: "Reshape" bottom: "input" top: "output" - // reshape_param { ... } - // } - // - // If "input" is 2D with shape 2 x 8, then the following reshape_param - // specifications are all equivalent, producing a 3D blob "output" with shape - // 2 x 2 x 4: - // - // reshape_param { shape { dim: 2 dim: 2 dim: 4 } } - // reshape_param { shape { dim: 0 dim: 2 dim: 4 } } - // reshape_param { shape { dim: 0 dim: 2 dim: -1 } } - // reshape_param { shape { dim: 0 dim:-1 dim: 4 } } - // - optional BlobShape shape = 1; - - // axis and num_axes control the portion of the bottom blob's shape that are - // replaced by (included in) the reshape. By default (axis == 0 and - // num_axes == -1), the entire bottom blob shape is included in the reshape, - // and hence the shape field must specify the entire output shape. - // - // axis may be non-zero to retain some portion of the beginning of the input - // shape (and may be negative to index from the end; e.g., -1 to begin the - // reshape after the last axis, including nothing in the reshape, - // -2 to include only the last axis, etc.). - // - // For example, suppose "input" is a 2D blob with shape 2 x 8. - // Then the following ReshapeLayer specifications are all equivalent, - // producing a blob "output" with shape 2 x 2 x 4: - // - // reshape_param { shape { dim: 2 dim: 2 dim: 4 } } - // reshape_param { shape { dim: 2 dim: 4 } axis: 1 } - // reshape_param { shape { dim: 2 dim: 4 } axis: -3 } - // - // num_axes specifies the extent of the reshape. - // If num_axes >= 0 (and axis >= 0), the reshape will be performed only on - // input axes in the range [axis, axis+num_axes]. - // num_axes may also be -1, the default, to include all remaining axes - // (starting from axis). - // - // For example, suppose "input" is a 2D blob with shape 2 x 8. - // Then the following ReshapeLayer specifications are equivalent, - // producing a blob "output" with shape 1 x 2 x 8. - // - // reshape_param { shape { dim: 1 dim: 2 dim: 8 } } - // reshape_param { shape { dim: 1 dim: 2 } num_axes: 1 } - // reshape_param { shape { dim: 1 } num_axes: 0 } - // - // On the other hand, these would produce output blob shape 2 x 1 x 8: - // - // reshape_param { shape { dim: 2 dim: 1 dim: 8 } } - // reshape_param { shape { dim: 1 } axis: 1 num_axes: 0 } - // - optional int32 axis = 2 [default = 0]; - optional int32 num_axes = 3 [default = -1]; -} - -message ScaleParameter { - // The first axis of bottom[0] (the first input Blob) along which to apply - // bottom[1] (the second input Blob). May be negative to index from the end - // (e.g., -1 for the last axis). - // - // For example, if bottom[0] is 4D with shape 100x3x40x60, the output - // top[0] will have the same shape, and bottom[1] may have any of the - // following shapes (for the given value of axis): - // (axis == 0 == -4) 100; 100x3; 100x3x40; 100x3x40x60 - // (axis == 1 == -3) 3; 3x40; 3x40x60 - // (axis == 2 == -2) 40; 40x60 - // (axis == 3 == -1) 60 - // Furthermore, bottom[1] may have the empty shape (regardless of the value of - // "axis") -- a scalar multiplier. - optional int32 axis = 1 [default = 1]; - - // (num_axes is ignored unless just one bottom is given and the scale is - // a learned parameter of the layer. Otherwise, num_axes is determined by the - // number of axes by the second bottom.) - // The number of axes of the input (bottom[0]) covered by the scale - // parameter, or -1 to cover all axes of bottom[0] starting from `axis`. - // Set num_axes := 0, to multiply with a zero-axis Blob: a scalar. - optional int32 num_axes = 2 [default = 1]; - - // (filler is ignored unless just one bottom is given and the scale is - // a learned parameter of the layer.) - // The initialization for the learned scale parameter. - // Default is the unit (1) initialization, resulting in the ScaleLayer - // initially performing the identity operation. - optional FillerParameter filler = 3; - - // Whether to also learn a bias (equivalent to a ScaleLayer+BiasLayer, but - // may be more efficient). Initialized with bias_filler (defaults to 0). - optional bool bias_term = 4 [default = false]; - optional FillerParameter bias_filler = 5; -} - -message SigmoidParameter { - enum Engine { - DEFAULT = 0; - CAFFE = 1; - CUDNN = 2; - } - optional Engine engine = 1 [default = DEFAULT]; -} - -message SliceParameter { - // The axis along which to slice -- may be negative to index from the end - // (e.g., -1 for the last axis). - // By default, SliceLayer concatenates blobs along the "channels" axis (1). - optional int32 axis = 3 [default = 1]; - repeated uint32 slice_point = 2; - - // DEPRECATED: alias for "axis" -- does not support negative indexing. - optional uint32 slice_dim = 1 [default = 1]; -} - -// Message that stores parameters used by SoftmaxLayer, SoftmaxWithLossLayer -message SoftmaxParameter { - enum Engine { - DEFAULT = 0; - CAFFE = 1; - CUDNN = 2; - } - optional Engine engine = 1 [default = DEFAULT]; - - // The axis along which to perform the softmax -- may be negative to index - // from the end (e.g., -1 for the last axis). - // Any other axes will be evaluated as independent softmaxes. - optional int32 axis = 2 [default = 1]; -} - -message TanHParameter { - enum Engine { - DEFAULT = 0; - CAFFE = 1; - CUDNN = 2; - } - optional Engine engine = 1 [default = DEFAULT]; -} - -// Message that stores parameters used by TileLayer -message TileParameter { - // The index of the axis to tile. - optional int32 axis = 1 [default = 1]; - - // The number of copies (tiles) of the blob to output. - optional int32 tiles = 2; -} - -// Message that stores parameters used by ThresholdLayer -message ThresholdParameter { - optional float threshold = 1 [default = 0]; // Strictly positive values -} - -message WindowDataParameter { - // Specify the data source. - optional string source = 1; - // For data pre-processing, we can do simple scaling and subtracting the - // data mean, if provided. Note that the mean subtraction is always carried - // out before scaling. - optional float scale = 2 [default = 1]; - optional string mean_file = 3; - // Specify the batch size. - optional uint32 batch_size = 4; - // Specify if we would like to randomly crop an image. - optional uint32 crop_size = 5 [default = 0]; - // Specify if we want to randomly mirror data. - optional bool mirror = 6 [default = false]; - // Foreground (object) overlap threshold - optional float fg_threshold = 7 [default = 0.5]; - // Background (non-object) overlap threshold - optional float bg_threshold = 8 [default = 0.5]; - // Fraction of batch that should be foreground objects - optional float fg_fraction = 9 [default = 0.25]; - // Amount of contextual padding to add around a window - // (used only by the window_data_layer) - optional uint32 context_pad = 10 [default = 0]; - // Mode for cropping out a detection window - // warp: cropped window is warped to a fixed size and aspect ratio - // square: the tightest square around the window is cropped - optional string crop_mode = 11 [default = "warp"]; - // cache_images: will load all images in memory for faster access - optional bool cache_images = 12 [default = false]; - // append root_folder to locate images - optional string root_folder = 13 [default = ""]; -} - -message SPPParameter { - enum PoolMethod { - MAX = 0; - AVE = 1; - STOCHASTIC = 2; - } - optional uint32 pyramid_height = 1; - optional PoolMethod pool = 2 [default = MAX]; // The pooling method - enum Engine { - DEFAULT = 0; - CAFFE = 1; - CUDNN = 2; - } - optional Engine engine = 6 [default = DEFAULT]; -} - -// DEPRECATED: use LayerParameter. -message V1LayerParameter { - repeated string bottom = 2; - repeated string top = 3; - optional string name = 4; - repeated NetStateRule include = 32; - repeated NetStateRule exclude = 33; - enum LayerType { - NONE = 0; - ABSVAL = 35; - ACCURACY = 1; - ARGMAX = 30; - BNLL = 2; - CONCAT = 3; - CONTRASTIVE_LOSS = 37; - CONVOLUTION = 4; - DATA = 5; - DECONVOLUTION = 39; - DROPOUT = 6; - DUMMY_DATA = 32; - EUCLIDEAN_LOSS = 7; - ELTWISE = 25; - EXP = 38; - FLATTEN = 8; - HDF5_DATA = 9; - HDF5_OUTPUT = 10; - HINGE_LOSS = 28; - IM2COL = 11; - IMAGE_DATA = 12; - INFOGAIN_LOSS = 13; - INNER_PRODUCT = 14; - LRN = 15; - MEMORY_DATA = 29; - MULTINOMIAL_LOGISTIC_LOSS = 16; - MVN = 34; - POOLING = 17; - POWER = 26; - RELU = 18; - SIGMOID = 19; - SIGMOID_CROSS_ENTROPY_LOSS = 27; - SILENCE = 36; - SOFTMAX = 20; - SOFTMAX_LOSS = 21; - SPLIT = 22; - SLICE = 33; - TANH = 23; - WINDOW_DATA = 24; - THRESHOLD = 31; - } - optional LayerType type = 5; - repeated BlobProto blobs = 6; - repeated string param = 1001; - repeated DimCheckMode blob_share_mode = 1002; - enum DimCheckMode { - STRICT = 0; - PERMISSIVE = 1; - } - repeated float blobs_lr = 7; - repeated float weight_decay = 8; - repeated float loss_weight = 35; - optional AccuracyParameter accuracy_param = 27; - optional ArgMaxParameter argmax_param = 23; - optional ConcatParameter concat_param = 9; - optional ContrastiveLossParameter contrastive_loss_param = 40; - optional ConvolutionParameter convolution_param = 10; - optional DataParameter data_param = 11; - optional DropoutParameter dropout_param = 12; - optional DummyDataParameter dummy_data_param = 26; - optional EltwiseParameter eltwise_param = 24; - optional ExpParameter exp_param = 41; - optional HDF5DataParameter hdf5_data_param = 13; - optional HDF5OutputParameter hdf5_output_param = 14; - optional HingeLossParameter hinge_loss_param = 29; - optional ImageDataParameter image_data_param = 15; - optional InfogainLossParameter infogain_loss_param = 16; - optional InnerProductParameter inner_product_param = 17; - optional LRNParameter lrn_param = 18; - optional MemoryDataParameter memory_data_param = 22; - optional MVNParameter mvn_param = 34; - optional PoolingParameter pooling_param = 19; - optional PowerParameter power_param = 21; - optional ReLUParameter relu_param = 30; - optional SigmoidParameter sigmoid_param = 38; - optional SoftmaxParameter softmax_param = 39; - optional SliceParameter slice_param = 31; - optional TanHParameter tanh_param = 37; - optional ThresholdParameter threshold_param = 25; - optional WindowDataParameter window_data_param = 20; - optional TransformationParameter transform_param = 36; - optional LossParameter loss_param = 42; - optional V0LayerParameter layer = 1; -} - -// DEPRECATED: V0LayerParameter is the old way of specifying layer parameters -// in Caffe. We keep this message type around for legacy support. -message V0LayerParameter { - optional string name = 1; // the layer name - optional string type = 2; // the string to specify the layer type - - // Parameters to specify layers with inner products. - optional uint32 num_output = 3; // The number of outputs for the layer - optional bool biasterm = 4 [default = true]; // whether to have bias terms - optional FillerParameter weight_filler = 5; // The filler for the weight - optional FillerParameter bias_filler = 6; // The filler for the bias - - optional uint32 pad = 7 [default = 0]; // The padding size - optional uint32 kernelsize = 8; // The kernel size - optional uint32 group = 9 [default = 1]; // The group size for group conv - optional uint32 stride = 10 [default = 1]; // The stride - enum PoolMethod { - MAX = 0; - AVE = 1; - STOCHASTIC = 2; - } - optional PoolMethod pool = 11 [default = MAX]; // The pooling method - optional float dropout_ratio = 12 [default = 0.5]; // dropout ratio - - optional uint32 local_size = 13 [default = 5]; // for local response norm - optional float alpha = 14 [default = 1.]; // for local response norm - optional float beta = 15 [default = 0.75]; // for local response norm - optional float k = 22 [default = 1.]; - - // For data layers, specify the data source - optional string source = 16; - // For data pre-processing, we can do simple scaling and subtracting the - // data mean, if provided. Note that the mean subtraction is always carried - // out before scaling. - optional float scale = 17 [default = 1]; - optional string meanfile = 18; - // For data layers, specify the batch size. - optional uint32 batchsize = 19; - // For data layers, specify if we would like to randomly crop an image. - optional uint32 cropsize = 20 [default = 0]; - // For data layers, specify if we want to randomly mirror data. - optional bool mirror = 21 [default = false]; - - // The blobs containing the numeric parameters of the layer - repeated BlobProto blobs = 50; - // The ratio that is multiplied on the global learning rate. If you want to - // set the learning ratio for one blob, you need to set it for all blobs. - repeated float blobs_lr = 51; - // The weight decay that is multiplied on the global weight decay. - repeated float weight_decay = 52; - - // The rand_skip variable is for the data layer to skip a few data points - // to avoid all asynchronous sgd clients to start at the same point. The skip - // point would be set as rand_skip * rand(0,1). Note that rand_skip should not - // be larger than the number of keys in the database. - optional uint32 rand_skip = 53 [default = 0]; - - // Fields related to detection (det_*) - // foreground (object) overlap threshold - optional float det_fg_threshold = 54 [default = 0.5]; - // background (non-object) overlap threshold - optional float det_bg_threshold = 55 [default = 0.5]; - // Fraction of batch that should be foreground objects - optional float det_fg_fraction = 56 [default = 0.25]; - - // optional bool OBSOLETE_can_clobber = 57 [default = true]; - - // Amount of contextual padding to add around a window - // (used only by the window_data_layer) - optional uint32 det_context_pad = 58 [default = 0]; - - // Mode for cropping out a detection window - // warp: cropped window is warped to a fixed size and aspect ratio - // square: the tightest square around the window is cropped - optional string det_crop_mode = 59 [default = "warp"]; - - // For ReshapeLayer, one needs to specify the new dimensions. - optional int32 new_num = 60 [default = 0]; - optional int32 new_channels = 61 [default = 0]; - optional int32 new_height = 62 [default = 0]; - optional int32 new_width = 63 [default = 0]; - - // Whether or not ImageLayer should shuffle the list of files at every epoch. - // It will also resize images if new_height or new_width are not zero. - optional bool shuffle_images = 64 [default = false]; - - // For ConcatLayer, one needs to specify the dimension for concatenation, and - // the other dimensions must be the same for all the bottom blobs. - // By default it will concatenate blobs along the channels dimension. - optional uint32 concat_dim = 65 [default = 1]; - - optional HDF5OutputParameter hdf5_output_param = 1001; -} - -message PReLUParameter { - // Parametric ReLU described in K. He et al, Delving Deep into Rectifiers: - // Surpassing Human-Level Performance on ImageNet Classification, 2015. - - // Initial value of a_i. Default is a_i=0.25 for all i. - optional FillerParameter filler = 1; - // Whether or not slope parameters are shared across channels. - optional bool channel_shared = 2 [default = false]; -} - -// The normalized bounding box [0, 1] w.r.t. the input image size. -message NormalizedBBox { - optional float xmin = 1; - optional float ymin = 2; - optional float xmax = 3; - optional float ymax = 4; - optional int32 label = 5; - optional bool difficult = 6; - optional float score = 7; - optional float size = 8; -} - -// origin: https://github.com/rbgirshick/caffe-fast-rcnn/tree/fast-rcnn -// Message that stores parameters used by ROIPoolingLayer -message ROIPoolingParameter { - // Pad, kernel size, and stride are all given as a single value for equal - // dimensions in height and width or as Y, X pairs. - optional uint32 pooled_h = 1 [default = 0]; // The pooled output height - optional uint32 pooled_w = 2 [default = 0]; // The pooled output width - // Multiplicative spatial scale factor to translate ROI coords from their - // input scale to the scale used when pooling - optional float spatial_scale = 3 [default = 1]; -} - -message ProposalParameter { - optional uint32 feat_stride = 1 [default = 16]; - optional uint32 base_size = 2 [default = 16]; - optional uint32 min_size = 3 [default = 16]; - repeated float ratio = 4; - repeated float scale = 5; - optional uint32 pre_nms_topn = 6 [default = 6000]; - optional uint32 post_nms_topn = 7 [default = 300]; - optional float nms_thresh = 8 [default = 0.7]; -} - -// origin: https://github.com/daijifeng001/caffe-rfcn -message PSROIPoolingParameter { - required float spatial_scale = 1; - required int32 output_dim = 2; // output channel number - required int32 group_size = 3; // equal to pooled_size -} diff --git a/modules/dnn/src/cuda4dnn/primitives/prior_box.hpp b/modules/dnn/src/cuda4dnn/primitives/prior_box.hpp index 5f48a8c5e7..f62863a282 100644 --- a/modules/dnn/src/cuda4dnn/primitives/prior_box.hpp +++ b/modules/dnn/src/cuda4dnn/primitives/prior_box.hpp @@ -95,7 +95,9 @@ namespace cv { namespace dnn { namespace cuda4dnn { const std::vector>& outputs, csl::Workspace& workspace) override { - CV_Assert(inputs.size() == 2); /* we don't need the inputs but we are given */ + // PriorBox uses layer parameters and inputs[0]/inputs[1] shapes; tensor data is never read. + // Extra inputs (e.g., in ssd_vgg16.onnx) are ignored, so require >= 2 inputs instead of exactly 2. + CV_Assert(inputs.size() >= 2); CV_Assert(outputs.size() == 1); auto output_wrapper = outputs[0].dynamicCast(); diff --git a/modules/dnn/src/dnn_read.cpp b/modules/dnn/src/dnn_read.cpp index c3fcb9e147..e7348985f2 100644 --- a/modules/dnn/src/dnn_read.cpp +++ b/modules/dnn/src/dnn_read.cpp @@ -17,11 +17,10 @@ Net readNet(const String& _model, const String& _config, const String& _framewor String config = _config; const std::string modelExt = model.substr(model.rfind('.') + 1); const std::string configExt = config.substr(config.rfind('.') + 1); - if (framework == "caffe" || modelExt == "caffemodel" || configExt == "caffemodel" || modelExt == "prototxt" || configExt == "prototxt") + if (framework == "caffe" || modelExt == "caffemodel" || configExt == "caffemodel" || + modelExt == "prototxt" || configExt == "prototxt") { - if (modelExt == "prototxt" || configExt == "caffemodel") - std::swap(model, config); - return readNetFromCaffe(config, model, engine); + CV_Error(Error::StsError, "Caffe importer has been removed. Please use ONNX-converted models or use an older OpenCV version."); } if (framework == "tensorflow" || modelExt == "pb" || configExt == "pb" || modelExt == "pbtxt" || configExt == "pbtxt") { @@ -58,10 +57,10 @@ Net readNet(const String& _framework, const std::vector& bufferModel, const std::vector& bufferConfig, int engine) { String framework = toLowerCase(_framework); + if (framework == "caffe") + CV_Error(Error::StsError, "Caffe importer has been removed. Please use ONNX-converted models or use an older OpenCV version."); if (framework == "onnx") return readNetFromONNX(bufferModel, engine); - else if (framework == "caffe") - return readNetFromCaffe(bufferConfig, bufferModel, engine); else if (framework == "tensorflow") return readNetFromTensorflow(bufferModel, bufferConfig, engine); else if (framework == "darknet") diff --git a/modules/dnn/src/layers/prior_box_layer.cpp b/modules/dnn/src/layers/prior_box_layer.cpp index c10d33a64a..8eafc13428 100644 --- a/modules/dnn/src/layers/prior_box_layer.cpp +++ b/modules/dnn/src/layers/prior_box_layer.cpp @@ -459,7 +459,9 @@ public: inputs_arr.getMatVector(inputs); outputs_arr.getMatVector(outputs); - CV_Assert(inputs.size() == 2); + // PriorBox uses layer parameters and inputs[0]/inputs[1] shapes; tensor data is never read. + // Extra inputs (e.g., in ssd_vgg16.onnx) are ignored, so require >= 2 inputs instead of exactly 2. + CV_Assert(inputs.size() >= 2); int _layerWidth = inputs[0].size[3]; int _layerHeight = inputs[0].size[2]; @@ -528,7 +530,9 @@ public: #ifdef HAVE_DNN_NGRAPH virtual Ptr initNgraph(const std::vector >& inputs, const std::vector >& nodes) CV_OVERRIDE { - CV_Assert(nodes.size() == 2); + // PriorBox uses layer parameters and inputs[0]/inputs[1] shapes; tensor data is never read. + // Extra inputs (e.g., in ssd_vgg16.onnx) are ignored, so require >= 2 inputs instead of exactly 2. + CV_Assert(nodes.size() >= 2); auto layer = nodes[0].dynamicCast()->node; auto image = nodes[1].dynamicCast()->node; auto layer_shape = std::make_shared(layer); diff --git a/modules/dnn/test/imagenet_cls_test_alexnet.py b/modules/dnn/test/imagenet_cls_test_alexnet.py deleted file mode 100644 index 30910cbb44..0000000000 --- a/modules/dnn/test/imagenet_cls_test_alexnet.py +++ /dev/null @@ -1,262 +0,0 @@ -from __future__ import print_function -from abc import ABCMeta, abstractmethod -import numpy as np -import sys -import os -import argparse -import time - -try: - import caffe -except ImportError: - raise ImportError('Can\'t find Caffe Python module. If you\'ve built it from sources without installation, ' - 'configure environment variable PYTHONPATH to "git/caffe/python" directory') -try: - import cv2 as cv -except ImportError: - raise ImportError('Can\'t find OpenCV Python module. If you\'ve built it from sources without installation, ' - 'configure environment variable PYTHONPATH to "opencv_build_dir/lib" directory (with "python3" subdirectory if required)') - -try: - xrange # Python 2 -except NameError: - xrange = range # Python 3 - - -class DataFetch(object): - imgs_dir = '' - frame_size = 0 - bgr_to_rgb = False - __metaclass__ = ABCMeta - - @abstractmethod - def preprocess(self, img): - pass - - def get_batch(self, imgs_names): - assert type(imgs_names) is list - batch = np.zeros((len(imgs_names), 3, self.frame_size, self.frame_size)).astype(np.float32) - for i in range(len(imgs_names)): - img_name = imgs_names[i] - img_file = self.imgs_dir + img_name - assert os.path.exists(img_file) - img = cv.imread(img_file, cv.IMREAD_COLOR) - min_dim = min(img.shape[-3], img.shape[-2]) - resize_ratio = self.frame_size / float(min_dim) - img = cv.resize(img, (0, 0), fx=resize_ratio, fy=resize_ratio) - cols = img.shape[1] - rows = img.shape[0] - y1 = (rows - self.frame_size) / 2 - y2 = y1 + self.frame_size - x1 = (cols - self.frame_size) / 2 - x2 = x1 + self.frame_size - img = img[y1:y2, x1:x2] - if self.bgr_to_rgb: - img = img[..., ::-1] - image_data = img[:, :, 0:3].transpose(2, 0, 1) - batch[i] = self.preprocess(image_data) - return batch - - -class MeanBlobFetch(DataFetch): - mean_blob = np.ndarray(()) - - def __init__(self, frame_size, mean_blob_path, imgs_dir): - self.imgs_dir = imgs_dir - self.frame_size = frame_size - blob = caffe.proto.caffe_pb2.BlobProto() - data = open(mean_blob_path, 'rb').read() - blob.ParseFromString(data) - self.mean_blob = np.array(caffe.io.blobproto_to_array(blob)) - start = (self.mean_blob.shape[2] - self.frame_size) / 2 - stop = start + self.frame_size - self.mean_blob = self.mean_blob[:, :, start:stop, start:stop][0] - - def preprocess(self, img): - return img - self.mean_blob - - -class MeanChannelsFetch(MeanBlobFetch): - def __init__(self, frame_size, imgs_dir): - self.imgs_dir = imgs_dir - self.frame_size = frame_size - self.mean_blob = np.ones((3, self.frame_size, self.frame_size)).astype(np.float32) - self.mean_blob[0] *= 104 - self.mean_blob[1] *= 117 - self.mean_blob[2] *= 123 - - -class MeanValueFetch(MeanBlobFetch): - def __init__(self, frame_size, imgs_dir, bgr_to_rgb): - self.imgs_dir = imgs_dir - self.frame_size = frame_size - self.mean_blob = np.ones((3, self.frame_size, self.frame_size)).astype(np.float32) - self.mean_blob *= 117 - self.bgr_to_rgb = bgr_to_rgb - - -def get_correct_answers(img_list, img_classes, net_output_blob): - correct_answers = 0 - for i in range(len(img_list)): - indexes = np.argsort(net_output_blob[i])[-5:] - correct_index = img_classes[img_list[i]] - if correct_index in indexes: - correct_answers += 1 - return correct_answers - - -class Framework(object): - in_blob_name = '' - out_blob_name = '' - - __metaclass__ = ABCMeta - - @abstractmethod - def get_name(self): - pass - - @abstractmethod - def get_output(self, input_blob): - pass - - -class CaffeModel(Framework): - net = caffe.Net - need_reshape = False - - def __init__(self, prototxt, caffemodel, in_blob_name, out_blob_name, need_reshape=False): - caffe.set_mode_cpu() - self.net = caffe.Net(prototxt, caffemodel, caffe.TEST) - self.in_blob_name = in_blob_name - self.out_blob_name = out_blob_name - self.need_reshape = need_reshape - - def get_name(self): - return 'Caffe' - - def get_output(self, input_blob): - if self.need_reshape: - self.net.blobs[self.in_blob_name].reshape(*input_blob.shape) - return self.net.forward_all(**{self.in_blob_name: input_blob})[self.out_blob_name] - - -class DnnCaffeModel(Framework): - net = object - - def __init__(self, prototxt, caffemodel, in_blob_name, out_blob_name): - self.net = cv.dnn.readNetFromCaffe(prototxt, caffemodel) - self.in_blob_name = in_blob_name - self.out_blob_name = out_blob_name - - def get_name(self): - return 'DNN' - - def get_output(self, input_blob): - self.net.setInput(input_blob, self.in_blob_name) - return self.net.forward(self.out_blob_name) - -class DNNOnnxModel(Framework): - net = object - - def __init__(self, onnx_file, in_blob_name, out_blob_name): - self.net = cv.dnn.readNetFromONNX(onnx_file) - self.in_blob_name = in_blob_name - self.out_blob_name = out_blob_name - - def get_name(self): - return 'DNN (ONNX)' - - def get_output(self, input_blob): - self.net.setInput(input_blob, self.in_blob_name) - return self.net.forward(self.out_blob_name) - - -class ClsAccEvaluation: - log = sys.stdout - img_classes = {} - batch_size = 0 - - def __init__(self, log_path, img_classes_file, batch_size): - self.log = open(log_path, 'w') - self.img_classes = self.read_classes(img_classes_file) - self.batch_size = batch_size - - @staticmethod - def read_classes(img_classes_file): - result = {} - with open(img_classes_file) as file: - for l in file.readlines(): - result[l.split()[0]] = int(l.split()[1]) - return result - - def process(self, frameworks, data_fetcher): - sorted_imgs_names = sorted(self.img_classes.keys()) - correct_answers = [0] * len(frameworks) - samples_handled = 0 - blobs_l1_diff = [0] * len(frameworks) - blobs_l1_diff_count = [0] * len(frameworks) - blobs_l_inf_diff = [sys.float_info.min] * len(frameworks) - inference_time = [0.0] * len(frameworks) - - for x in xrange(0, len(sorted_imgs_names), self.batch_size): - sublist = sorted_imgs_names[x:x + self.batch_size] - batch = data_fetcher.get_batch(sublist) - - samples_handled += len(sublist) - - frameworks_out = [] - fw_accuracy = [] - for i in range(len(frameworks)): - start = time.time() - out = frameworks[i].get_output(batch) - end = time.time() - correct_answers[i] += get_correct_answers(sublist, self.img_classes, out) - fw_accuracy.append(100 * correct_answers[i] / float(samples_handled)) - frameworks_out.append(out) - inference_time[i] += end - start - print(samples_handled, 'Accuracy for', frameworks[i].get_name() + ':', fw_accuracy[i], file=self.log) - print("Inference time, ms ", \ - frameworks[i].get_name(), inference_time[i] / samples_handled * 1000, file=self.log) - - for i in range(1, len(frameworks)): - log_str = frameworks[0].get_name() + " vs " + frameworks[i].get_name() + ':' - diff = np.abs(frameworks_out[0] - frameworks_out[i]) - l1_diff = np.sum(diff) / diff.size - print(samples_handled, "L1 difference", log_str, l1_diff, file=self.log) - blobs_l1_diff[i] += l1_diff - blobs_l1_diff_count[i] += 1 - if np.max(diff) > blobs_l_inf_diff[i]: - blobs_l_inf_diff[i] = np.max(diff) - print(samples_handled, "L_INF difference", log_str, blobs_l_inf_diff[i], file=self.log) - - self.log.flush() - - for i in range(1, len(blobs_l1_diff)): - log_str = frameworks[0].get_name() + " vs " + frameworks[i].get_name() + ':' - print('Final l1 diff', log_str, blobs_l1_diff[i] / blobs_l1_diff_count[i], file=self.log) - -if __name__ == "__main__": - parser = argparse.ArgumentParser() - parser.add_argument("--imgs_dir", help="path to ImageNet validation subset images dir, ILSVRC2012_img_val dir") - parser.add_argument("--img_cls_file", help="path to file with classes ids for images, val.txt file from this " - "archive: http://dl.caffe.berkeleyvision.org/caffe_ilsvrc12.tar.gz") - parser.add_argument("--prototxt", help="path to caffe prototxt, download it here: " - "https://github.com/BVLC/caffe/blob/master/models/bvlc_alexnet/deploy.prototxt") - parser.add_argument("--caffemodel", help="path to caffemodel file, download it here: " - "http://dl.caffe.berkeleyvision.org/bvlc_alexnet.caffemodel") - parser.add_argument("--log", help="path to logging file") - parser.add_argument("--mean", help="path to ImageNet mean blob caffe file, imagenet_mean.binaryproto file from" - "this archive: http://dl.caffe.berkeleyvision.org/caffe_ilsvrc12.tar.gz") - parser.add_argument("--batch_size", help="size of images in batch", default=1000) - parser.add_argument("--frame_size", help="size of input image", default=227) - parser.add_argument("--in_blob", help="name for input blob", default='data') - parser.add_argument("--out_blob", help="name for output blob", default='prob') - args = parser.parse_args() - - data_fetcher = MeanBlobFetch(args.frame_size, args.mean, args.imgs_dir) - - frameworks = [CaffeModel(args.prototxt, args.caffemodel, args.in_blob, args.out_blob), - DnnCaffeModel(args.prototxt, args.caffemodel, '', args.out_blob)] - - acc_eval = ClsAccEvaluation(args.log, args.img_cls_file, args.batch_size) - acc_eval.process(frameworks, data_fetcher) diff --git a/modules/dnn/test/imagenet_cls_test_googlenet.py b/modules/dnn/test/imagenet_cls_test_googlenet.py deleted file mode 100644 index 28f1abc2ff..0000000000 --- a/modules/dnn/test/imagenet_cls_test_googlenet.py +++ /dev/null @@ -1,39 +0,0 @@ -import numpy as np -import sys -import os -import argparse -from imagenet_cls_test_alexnet import MeanChannelsFetch, CaffeModel, DnnCaffeModel, ClsAccEvaluation -try: - import caffe -except ImportError: - raise ImportError('Can\'t find Caffe Python module. If you\'ve built it from sources without installation, ' - 'configure environment variable PYTHONPATH to "git/caffe/python" directory') -try: - import cv2 as cv -except ImportError: - raise ImportError('Can\'t find OpenCV Python module. If you\'ve built it from sources without installation, ' - 'configure environment variable PYTHONPATH to "opencv_build_dir/lib" directory (with "python3" subdirectory if required)') - -if __name__ == "__main__": - parser = argparse.ArgumentParser() - parser.add_argument("--imgs_dir", help="path to ImageNet validation subset images dir, ILSVRC2012_img_val dir") - parser.add_argument("--img_cls_file", help="path to file with classes ids for images, val.txt file from this " - "archive: http://dl.caffe.berkeleyvision.org/caffe_ilsvrc12.tar.gz") - parser.add_argument("--prototxt", help="path to caffe prototxt, download it here: " - "https://github.com/BVLC/caffe/blob/master/models/bvlc_alexnet/deploy.prototxt") - parser.add_argument("--caffemodel", help="path to caffemodel file, download it here: " - "http://dl.caffe.berkeleyvision.org/bvlc_alexnet.caffemodel") - parser.add_argument("--log", help="path to logging file") - parser.add_argument("--batch_size", help="size of images in batch", default=500, type=int) - parser.add_argument("--frame_size", help="size of input image", default=224, type=int) - parser.add_argument("--in_blob", help="name for input blob", default='data') - parser.add_argument("--out_blob", help="name for output blob", default='prob') - args = parser.parse_args() - - data_fetcher = MeanChannelsFetch(args.frame_size, args.imgs_dir) - - frameworks = [CaffeModel(args.prototxt, args.caffemodel, args.in_blob, args.out_blob), - DnnCaffeModel(args.prototxt, args.caffemodel, '', args.out_blob)] - - acc_eval = ClsAccEvaluation(args.log, args.img_cls_file, args.batch_size) - acc_eval.process(frameworks, data_fetcher) diff --git a/modules/dnn/test/imagenet_cls_test_inception.py b/modules/dnn/test/imagenet_cls_test_inception.py deleted file mode 100644 index dce59327d6..0000000000 --- a/modules/dnn/test/imagenet_cls_test_inception.py +++ /dev/null @@ -1,77 +0,0 @@ -import numpy as np -import sys -import os -import argparse -import tensorflow as tf -from tensorflow.python.platform import gfile -from imagenet_cls_test_alexnet import MeanValueFetch, DnnCaffeModel, Framework, ClsAccEvaluation -try: - import cv2 as cv -except ImportError: - raise ImportError('Can\'t find OpenCV Python module. If you\'ve built it from sources without installation, ' - 'configure environment variable PYTHONPATH to "opencv_build_dir/lib" directory (with "python3" subdirectory if required)') - -# If you've got an exception "Cannot load libmkl_avx.so or libmkl_def.so" or similar, try to export next variable -# before running the script: -# LD_PRELOAD=/opt/intel/mkl/lib/intel64/libmkl_core.so:/opt/intel/mkl/lib/intel64/libmkl_sequential.so - - -class TensorflowModel(Framework): - sess = tf.Session - output = tf.Graph - - def __init__(self, model_file, in_blob_name, out_blob_name): - self.in_blob_name = in_blob_name - self.sess = tf.Session() - with gfile.FastGFile(model_file, 'rb') as f: - graph_def = tf.GraphDef() - graph_def.ParseFromString(f.read()) - self.sess.graph.as_default() - tf.import_graph_def(graph_def, name='') - self.output = self.sess.graph.get_tensor_by_name(out_blob_name + ":0") - - def get_name(self): - return 'Tensorflow' - - def get_output(self, input_blob): - assert len(input_blob.shape) == 4 - batch_tf = input_blob.transpose(0, 2, 3, 1) - out = self.sess.run(self.output, - {self.in_blob_name+':0': batch_tf}) - out = out[..., 1:1001] - return out - - -class DnnTfInceptionModel(DnnCaffeModel): - net = cv.dnn.Net() - - def __init__(self, model_file, in_blob_name, out_blob_name): - self.net = cv.dnn.readNetFromTensorflow(model_file) - self.in_blob_name = in_blob_name - self.out_blob_name = out_blob_name - - def get_output(self, input_blob): - return super(DnnTfInceptionModel, self).get_output(input_blob)[..., 1:1001] - - -if __name__ == "__main__": - parser = argparse.ArgumentParser() - parser.add_argument("--imgs_dir", help="path to ImageNet validation subset images dir, ILSVRC2012_img_val dir") - parser.add_argument("--img_cls_file", help="path to file with classes ids for images, download it here:" - "https://github.com/opencv/opencv_extra/tree/5.x/testdata/dnn/img_classes_inception.txt") - parser.add_argument("--model", help="path to tensorflow model, download it here:" - "https://storage.googleapis.com/download.tensorflow.org/models/inception5h.zip") - parser.add_argument("--log", help="path to logging file") - parser.add_argument("--batch_size", help="size of images in batch", default=1) - parser.add_argument("--frame_size", help="size of input image", default=224) - parser.add_argument("--in_blob", help="name for input blob", default='input') - parser.add_argument("--out_blob", help="name for output blob", default='softmax2') - args = parser.parse_args() - - data_fetcher = MeanValueFetch(args.frame_size, args.imgs_dir, True) - - frameworks = [TensorflowModel(args.model, args.in_blob, args.out_blob), - DnnTfInceptionModel(args.model, '', args.out_blob)] - - acc_eval = ClsAccEvaluation(args.log, args.img_cls_file, args.batch_size) - acc_eval.process(frameworks, data_fetcher) diff --git a/modules/dnn/test/pascal_semsegm_test_fcn.py b/modules/dnn/test/pascal_semsegm_test_fcn.py deleted file mode 100644 index 9ece057e36..0000000000 --- a/modules/dnn/test/pascal_semsegm_test_fcn.py +++ /dev/null @@ -1,255 +0,0 @@ -from __future__ import print_function -from abc import ABCMeta, abstractmethod -import numpy as np -import sys -import argparse -import time - -from imagenet_cls_test_alexnet import CaffeModel, DNNOnnxModel -try: - import cv2 as cv -except ImportError: - raise ImportError('Can\'t find OpenCV Python module. If you\'ve built it from sources without installation, ' - 'configure environment variable PYTHONPATH to "opencv_build_dir/lib" directory (with "python3" subdirectory if required)') - - -def get_metrics(conf_mat): - pix_accuracy = np.trace(conf_mat) / np.sum(conf_mat) - t = np.sum(conf_mat, 1) - num_cl = np.count_nonzero(t) - assert num_cl - mean_accuracy = np.sum(np.nan_to_num(np.divide(np.diagonal(conf_mat), t))) / num_cl - col_sum = np.sum(conf_mat, 0) - mean_iou = np.sum( - np.nan_to_num(np.divide(np.diagonal(conf_mat), (t + col_sum - np.diagonal(conf_mat))))) / num_cl - return pix_accuracy, mean_accuracy, mean_iou - - -def eval_segm_result(net_out): - assert type(net_out) is np.ndarray - assert len(net_out.shape) == 4 - - channels_dim = 1 - y_dim = channels_dim + 1 - x_dim = y_dim + 1 - res = np.zeros(net_out.shape).astype(int) - for i in range(net_out.shape[y_dim]): - for j in range(net_out.shape[x_dim]): - max_ch = np.argmax(net_out[..., i, j]) - res[0, max_ch, i, j] = 1 - return res - - -def get_conf_mat(gt, prob): - assert type(gt) is np.ndarray - assert type(prob) is np.ndarray - - conf_mat = np.zeros((gt.shape[0], gt.shape[0])) - for ch_gt in range(conf_mat.shape[0]): - gt_channel = gt[ch_gt, ...] - for ch_pr in range(conf_mat.shape[1]): - prob_channel = prob[ch_pr, ...] - conf_mat[ch_gt][ch_pr] = np.count_nonzero(np.multiply(gt_channel, prob_channel)) - return conf_mat - - -class MeanChannelsPreproc: - def __init__(self): - pass - - @staticmethod - def process(img, framework): - image_data = None - if framework == "Caffe": - image_data = cv.dnn.blobFromImage(img, scalefactor=1.0, mean=(123.0, 117.0, 104.0), swapRB=True) - elif framework == "DNN (ONNX)": - image_data = cv.dnn.blobFromImage(img, scalefactor=0.019, mean=(123.675, 116.28, 103.53), swapRB=True) - else: - raise ValueError("Unknown framework") - return image_data - - -class DatasetImageFetch(object): - __metaclass__ = ABCMeta - data_prepoc = object - - @abstractmethod - def __iter__(self): - pass - - @abstractmethod - def next(self): - pass - - @staticmethod - def pix_to_c(pix): - return pix[0] * 256 * 256 + pix[1] * 256 + pix[2] - - @staticmethod - def color_to_gt(color_img, colors): - num_classes = len(colors) - gt = np.zeros((num_classes, color_img.shape[0], color_img.shape[1])).astype(int) - for img_y in range(color_img.shape[0]): - for img_x in range(color_img.shape[1]): - c = DatasetImageFetch.pix_to_c(color_img[img_y][img_x]) - if c in colors: - cls = colors.index(c) - gt[cls][img_y][img_x] = 1 - return gt - - -class PASCALDataFetch(DatasetImageFetch): - img_dir = '' - segm_dir = '' - names = [] - colors = [] - i = 0 - - def __init__(self, img_dir, segm_dir, names_file, segm_cls_colors, preproc): - self.img_dir = img_dir - self.segm_dir = segm_dir - self.colors = self.read_colors(segm_cls_colors) - self.data_prepoc = preproc - self.i = 0 - - with open(names_file) as f: - for l in f.readlines(): - self.names.append(l.rstrip()) - - @staticmethod - def read_colors(colors): - result = [] - for color in colors: - result.append(DatasetImageFetch.pix_to_c(color)) - return result - - def __iter__(self): - return self - - def __next__(self): - if self.i < len(self.names): - name = self.names[self.i] - self.i += 1 - segm_file = self.segm_dir + name + ".png" - img_file = self.img_dir + name + ".jpg" - gt = self.color_to_gt(cv.imread(segm_file, cv.IMREAD_COLOR)[:, :, ::-1], self.colors) - img = cv.imread(img_file, cv.IMREAD_COLOR) - img_caffe = self.data_prepoc.process(img[:, :, ::-1], "Caffe") - img_dnn = self.data_prepoc.process(img[:, :, ::-1], "DNN (ONNX)") - img_dict = { - "Caffe": img_caffe, - "DNN (ONNX)": img_dnn - } - return img_dict, gt - else: - self.i = 0 - raise StopIteration - - def get_num_classes(self): - return len(self.colors) - - -class SemSegmEvaluation: - log = sys.stdout - - def __init__(self, log_path,): - self.log = open(log_path, 'w') - - def process(self, frameworks, data_fetcher): - samples_handled = 0 - - conf_mats = [np.zeros((data_fetcher.get_num_classes(), data_fetcher.get_num_classes())) for i in range(len(frameworks))] - blobs_l1_diff = [0] * len(frameworks) - blobs_l1_diff_count = [0] * len(frameworks) - blobs_l_inf_diff = [sys.float_info.min] * len(frameworks) - inference_time = [0.0] * len(frameworks) - - for in_blob_dict, gt in data_fetcher: - frameworks_out = [] - samples_handled += 1 - for i in range(len(frameworks)): - start = time.time() - framework_name = frameworks[i].get_name() - out = frameworks[i].get_output(in_blob_dict[framework_name]) - end = time.time() - segm = eval_segm_result(out) - conf_mats[i] += get_conf_mat(gt, segm[0]) - frameworks_out.append(out) - inference_time[i] += end - start - - pix_acc, mean_acc, miou = get_metrics(conf_mats[i]) - - name = frameworks[i].get_name() - print(samples_handled, 'Pixel accuracy, %s:' % name, 100 * pix_acc, file=self.log) - print(samples_handled, 'Mean accuracy, %s:' % name, 100 * mean_acc, file=self.log) - print(samples_handled, 'Mean IOU, %s:' % name, 100 * miou, file=self.log) - print("Inference time, ms ", \ - frameworks[i].get_name(), inference_time[i] / samples_handled * 1000, file=self.log) - - for i in range(1, len(frameworks)): - log_str = frameworks[0].get_name() + " vs " + frameworks[i].get_name() + ':' - diff = np.abs(frameworks_out[0] - frameworks_out[i]) - l1_diff = np.sum(diff) / diff.size - print(samples_handled, "L1 difference", log_str, l1_diff, file=self.log) - blobs_l1_diff[i] += l1_diff - blobs_l1_diff_count[i] += 1 - if np.max(diff) > blobs_l_inf_diff[i]: - blobs_l_inf_diff[i] = np.max(diff) - print(samples_handled, "L_INF difference", log_str, blobs_l_inf_diff[i], file=self.log) - - self.log.flush() - - for i in range(1, len(blobs_l1_diff)): - log_str = frameworks[0].get_name() + " vs " + frameworks[i].get_name() + ':' - print('Final l1 diff', log_str, blobs_l1_diff[i] / blobs_l1_diff_count[i], file=self.log) - -# PASCAL VOC 2012 classes colors -colors_pascal_voc_2012 = [ - [0, 0, 0], - [128, 0, 0], - [0, 128, 0], - [128, 128, 0], - [0, 0, 128], - [128, 0, 128], - [0, 128, 128], - [128, 128, 128], - [64, 0, 0], - [192, 0, 0], - [64, 128, 0], - [192, 128, 0], - [64, 0, 128], - [192, 0, 128], - [64, 128, 128], - [192, 128, 128], - [0, 64, 0], - [128, 64, 0], - [0, 192, 0], - [128, 192, 0], - [0, 64, 128], -] - -if __name__ == "__main__": - parser = argparse.ArgumentParser() - parser.add_argument("--imgs_dir", help="path to PASCAL VOC 2012 images dir, data/VOC2012/JPEGImages") - parser.add_argument("--segm_dir", help="path to PASCAL VOC 2012 segmentation dir, data/VOC2012/SegmentationClass/") - parser.add_argument("--val_names", help="path to file with validation set image names, download it here: " - "https://github.com/shelhamer/fcn.berkeleyvision.org/blob/master/data/pascal/seg11valid.txt") - parser.add_argument("--prototxt", help="path to caffe prototxt, download it here: " - "https://github.com/opencv/opencv/blob/5.x/samples/data/dnn/fcn8s-heavy-pascal.prototxt") - parser.add_argument("--caffemodel", help="path to caffemodel file, download it here: " - "http://dl.caffe.berkeleyvision.org/fcn8s-heavy-pascal.caffemodel") - parser.add_argument("--onnxmodel", help="path to onnx model file, download it here: " - "https://github.com/onnx/models/raw/491ce05590abb7551d7fae43c067c060eeb575a6/validated/vision/object_detection_segmentation/fcn/model/fcn-resnet50-12.onnx") - parser.add_argument("--log", help="path to logging file", default='log.txt') - parser.add_argument("--in_blob", help="name for input blob", default='data') - parser.add_argument("--out_blob", help="name for output blob", default='score') - args = parser.parse_args() - - prep = MeanChannelsPreproc() - df = PASCALDataFetch(args.imgs_dir, args.segm_dir, args.val_names, colors_pascal_voc_2012, prep) - - fw = [CaffeModel(args.prototxt, args.caffemodel, args.in_blob, args.out_blob, True), - DNNOnnxModel(args.onnxmodel, args.in_blob, args.out_blob)] - - segm_eval = SemSegmEvaluation(args.log) - segm_eval.process(fw, df) diff --git a/modules/dnn/test/test_backends.cpp b/modules/dnn/test/test_backends.cpp index 31df338ec6..b47a52dec7 100644 --- a/modules/dnn/test/test_backends.cpp +++ b/modules/dnn/test/test_backends.cpp @@ -124,8 +124,11 @@ TEST_P(DNNTestNetwork, DISABLED_YOLOv8n) { TEST_P(DNNTestNetwork, AlexNet) { applyTestTag(CV_TEST_TAG_MEMORY_1GB); - processNet("dnn/bvlc_alexnet.caffemodel", "dnn/bvlc_alexnet.prototxt", - Size(227, 227), "prob"); + // Skip memory-heavy OpenCL targets on 32-bit (x86) platforms to avoid OutOfMemoryError + if (sizeof(void*) == 4 && + (target == DNN_TARGET_OPENCL || target == DNN_TARGET_OPENCL_FP16)) + throw SkipTestException("Skip memory-heavy OpenCL target on 32-bit (x86) platform"); + processNet("dnn/onnx/models/alexnet.onnx", "", Size(227, 227)); expectNoFallbacksFromIE(net); expectNoFallbacksFromCUDA(net); } @@ -137,16 +140,20 @@ TEST_P(DNNTestNetwork, ResNet_50) CV_TEST_TAG_DEBUG_VERYLONG ); - processNet("dnn/ResNet-50-model.caffemodel", "dnn/ResNet-50-deploy.prototxt", - Size(224, 224), "prob"); + double l1 = default_l1, lInf = default_lInf; + if (target == DNN_TARGET_CUDA_FP16 || target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_CPU_FP16) + { + l1 = 0.015; + lInf = 0.05; + } + processNet("dnn/onnx/models/resnet50v1.onnx", "", Size(224, 224), "", l1, lInf); expectNoFallbacksFromIE(net); expectNoFallbacksFromCUDA(net); } TEST_P(DNNTestNetwork, SqueezeNet_v1_1) { - processNet("dnn/squeezenet_v1.1.caffemodel", "dnn/squeezenet_v1.1.prototxt", - Size(227, 227), "prob"); + processNet("dnn/onnx/models/squeezenet.onnx", "", Size(227, 227)); expectNoFallbacksFromIE(net); expectNoFallbacksFromCUDA(net); } @@ -155,8 +162,7 @@ TEST_P(DNNTestNetwork, GoogLeNet) { applyTestTag(target == DNN_TARGET_CPU ? "" : CV_TEST_TAG_MEMORY_512MB); - processNet("dnn/bvlc_googlenet.caffemodel", "dnn/bvlc_googlenet.prototxt", - Size(224, 224), "prob"); + processNet("dnn/onnx/models/googlenet.onnx", "", Size(224, 224)); expectNoFallbacksFromIE(net); expectNoFallbacksFromCUDA(net); } @@ -176,60 +182,6 @@ TEST_P(DNNTestNetwork, Inception_5h) expectNoFallbacksFromCUDA(net); } -TEST_P(DNNTestNetwork, MobileNet_SSD_Caffe) -{ - applyTestTag(CV_TEST_TAG_MEMORY_512MB); - Mat sample = imread(findDataFile("dnn/street.png")); - Mat inp = blobFromImage(sample, 1.0f / 127.5, Size(300, 300), Scalar(127.5, 127.5, 127.5), false); - float scoreDiff = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD || target == DNN_TARGET_CPU_FP16) ? 1.5e-2 : 0.0; - float iouDiff = (target == DNN_TARGET_MYRIAD) ? 0.063 : 0.0; - float detectionConfThresh = (target == DNN_TARGET_MYRIAD) ? 0.262 : FLT_MIN; - processNet("dnn/MobileNetSSD_deploy_19e3ec3.caffemodel", "dnn/MobileNetSSD_deploy_19e3ec3.prototxt", - inp, "detection_out", scoreDiff, iouDiff, detectionConfThresh); - expectNoFallbacksFromIE(net); -} - -TEST_P(DNNTestNetwork, MobileNet_SSD_Caffe_Different_Width_Height) -{ -#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2022010000) - // May hang on some configurations - if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && (target == DNN_TARGET_OPENCL || target == DNN_TARGET_OPENCL_FP16)) - applyTestTag(target == DNN_TARGET_OPENCL ? CV_TEST_TAG_DNN_SKIP_IE_OPENCL : CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16, - CV_TEST_TAG_DNN_SKIP_IE_NGRAPH, CV_TEST_TAG_DNN_SKIP_IE_VERSION - ); -#elif defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2021040000) - // IE exception: Ngraph operation Transpose with name conv15_2_mbox_conf_perm has dynamic output shape on 0 port, but CPU plug-in supports only static shape - if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && (target == DNN_TARGET_OPENCL || target == DNN_TARGET_OPENCL_FP16)) - applyTestTag(target == DNN_TARGET_OPENCL ? CV_TEST_TAG_DNN_SKIP_IE_OPENCL : CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16, - CV_TEST_TAG_DNN_SKIP_IE_NGRAPH, CV_TEST_TAG_DNN_SKIP_IE_VERSION - ); - if ((backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 || backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) && - target == DNN_TARGET_MYRIAD && getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_X) - applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD_X, CV_TEST_TAG_DNN_SKIP_IE_NGRAPH, CV_TEST_TAG_DNN_SKIP_IE_VERSION); -#elif defined(INF_ENGINE_RELEASE) - if ((backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 || backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) && - target == DNN_TARGET_MYRIAD && getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_X) - applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD_X, CV_TEST_TAG_DNN_SKIP_IE_NGRAPH, CV_TEST_TAG_DNN_SKIP_IE_VERSION); -#endif - - Mat sample = imread(findDataFile("dnn/street.png")); - Mat inp = blobFromImage(sample, 1.0f / 127.5, Size(300, 560), Scalar(127.5, 127.5, 127.5), false); - float scoreDiff = 0.0, iouDiff = 0.0; - if (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD || target == DNN_TARGET_CPU_FP16) - { - scoreDiff = 0.029; - iouDiff = 0.09; - } - else if (target == DNN_TARGET_CUDA_FP16) - { - scoreDiff = 0.03; - iouDiff = 0.08; - } - processNet("dnn/MobileNetSSD_deploy_19e3ec3.caffemodel", "dnn/MobileNetSSD_deploy_19e3ec3.prototxt", - inp, "detection_out", scoreDiff, iouDiff); - expectNoFallbacksFromIE(net); -} - TEST_P(DNNTestNetwork, MobileNet_SSD_v1_TensorFlow) { applyTestTag((target == DNN_TARGET_CPU || target == DNN_TARGET_CPU_FP16) ? "" : CV_TEST_TAG_MEMORY_512MB); @@ -313,6 +265,18 @@ TEST_P(DNNTestNetwork, SSD_VGG16) CV_TEST_TAG_DEBUG_VERYLONG ); + // This converted SSD model relies on layers the OpenVINO backend can't run. + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_NGRAPH); + + auto engine_forced = static_cast( + cv::utils::getConfigurationParameterSizeT("OPENCV_FORCE_DNN_ENGINE", cv::dnn::ENGINE_AUTO)); + if (engine_forced == cv::dnn::ENGINE_CLASSIC) + { + applyTestTag(CV_TEST_TAG_DNN_SKIP_PARSER); + return; + } + Mat sample = imread(findDataFile("dnn/street.png")); Mat inp = blobFromImage(sample, 1.0f, Size(300, 300), Scalar(), false); @@ -332,8 +296,8 @@ TEST_P(DNNTestNetwork, SSD_VGG16) iouDiff = 0.13; } - processNet("dnn/VGG_ILSVRC2016_SSD_300x300_iter_440000.caffemodel", - "dnn/ssd_vgg16.prototxt", inp, "detection_out", scoreDiff, + processNet("dnn/onnx/models/ssd_vgg16.onnx", "", + inp, "detection_out", scoreDiff, iouDiff, 0.2, false); expectNoFallbacksFromIE(net); } @@ -350,7 +314,7 @@ TEST_P(DNNTestNetwork, OpenPose_pose_coco) const float l1 = (target == DNN_TARGET_MYRIAD) ? 0.009 : 0.0; const float lInf = (target == DNN_TARGET_MYRIAD) ? 0.09 : 0.0; - processNet("dnn/openpose_pose_coco.caffemodel", "dnn/openpose_pose_coco.prototxt", + processNet("dnn/onnx/models/openpose_pose_coco.onnx", "", Size(46, 46), "", l1, lInf); expectNoFallbacksFromIE(net); expectNoFallbacksFromCUDA(net); @@ -369,7 +333,7 @@ TEST_P(DNNTestNetwork, OpenPose_pose_mpi) // output range: [-0.001, 0.97] const float l1 = (target == DNN_TARGET_MYRIAD) ? 0.02 : 0.0; const float lInf = (target == DNN_TARGET_MYRIAD || target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_CPU_FP16) ? 0.2 : 0.0; - processNet("dnn/openpose_pose_mpi.caffemodel", "dnn/openpose_pose_mpi.prototxt", + processNet("dnn/onnx/models/openpose_pose_mpi.onnx", "", Size(46, 46), "", l1, lInf); expectNoFallbacksFromIE(net); expectNoFallbacksFromCUDA(net); @@ -384,9 +348,8 @@ TEST_P(DNNTestNetwork, OpenPose_pose_mpi_faster_4_stages) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD_X, CV_TEST_TAG_DNN_SKIP_IE_VERSION); #endif - // The same .caffemodel but modified .prototxt // See https://github.com/CMU-Perceptual-Computing-Lab/openpose/blob/master/src/openpose/pose/poseParameters.cpp - processNet("dnn/openpose_pose_mpi.caffemodel", "dnn/openpose_pose_mpi_faster_4_stages.prototxt", + processNet("dnn/onnx/models/openpose_pose_mpi_faster_4_stages.onnx", "", Size(46, 46)); expectNoFallbacksFromIE(net); expectNoFallbacksFromCUDA(net); @@ -456,10 +419,10 @@ TEST_P(DNNTestNetwork, DenseNet_121) } else if (target == DNN_TARGET_CUDA_FP16) { - l1 = 0.008; - lInf = 0.06; + l1 = 2e-2; + lInf = 9e-2; } - processNet("dnn/DenseNet_121.caffemodel", "dnn/DenseNet_121.prototxt", Size(224, 224), "", l1, lInf); + processNet("dnn/onnx/models/densenet121.onnx", "", Size(224, 224), "", l1, lInf); if (target != DNN_TARGET_MYRIAD || getInferenceEngineVPUType() != CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_X) expectNoFallbacksFromIE(net); expectNoFallbacksFromCUDA(net); diff --git a/modules/dnn/test/test_caffe_importer.cpp b/modules/dnn/test/test_caffe_importer.cpp index 6bc7689346..7f031bd760 100644 --- a/modules/dnn/test/test_caffe_importer.cpp +++ b/modules/dnn/test/test_caffe_importer.cpp @@ -42,6 +42,7 @@ #include "test_precomp.hpp" #include "npy_blob.hpp" #include +#include namespace opencv_test { namespace { @@ -58,8 +59,8 @@ public: double scoreDiff = 0.0, double iouDiff = 0.0) { checkBackend(); - Net net = readNetFromCaffe(findDataFile("dnn/" + proto), - findDataFile("dnn/" + model, false)); + Net net = readNet(findDataFile("dnn/" + proto), + findDataFile("dnn/" + model, false)); net.setPreferableBackend(backend); net.setPreferableTarget(target); @@ -71,7 +72,7 @@ public: Mat blob = blobFromImage(img, 1.0, Size(), Scalar(102.9801, 115.9465, 122.7717), false, false); Mat imInfo = (Mat_(1, 3) << img.rows, img.cols, 1.6f); - net.setInput(blob, "data"); + net.setInput(blob); net.setInput(imInfo, "im_info"); // Output has shape 1x1xNx7 where N - number of detections. // An every detection is a vector of values [id, classId, confidence, left, top, right, bottom] @@ -82,205 +83,6 @@ public: } }; -TEST(Test_Caffe, memory_read) -{ - const string proto = findDataFile("dnn/bvlc_googlenet.prototxt"); - const string model = findDataFile("dnn/bvlc_googlenet.caffemodel", false); - - std::vector dataProto; - readFileContent(proto, dataProto); - - std::vector dataModel; - readFileContent(model, dataModel); - - Net net = readNetFromCaffe(dataProto.data(), dataProto.size()); - net.setPreferableBackend(DNN_BACKEND_OPENCV); - ASSERT_FALSE(net.empty()); - - Net net2 = readNetFromCaffe(dataProto.data(), dataProto.size(), - dataModel.data(), dataModel.size()); - ASSERT_FALSE(net2.empty()); -} - -TEST(Test_Caffe, read_gtsrb) -{ - Net net = readNetFromCaffe(_tf("gtsrb.prototxt")); - ASSERT_FALSE(net.empty()); -} - -TEST(Test_Caffe, read_googlenet) -{ - Net net = readNetFromCaffe(_tf("bvlc_googlenet.prototxt")); - ASSERT_FALSE(net.empty()); -} - -TEST_P(Test_Caffe_nets, Axpy) -{ -#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_LT(2021040000) - if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019) - applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_NN_BUILDER); - if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) - applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_NGRAPH); -#endif - - String proto = _tf("axpy.prototxt"); - Net net = readNetFromCaffe(proto); - - checkBackend(); - net.setPreferableBackend(backend); - net.setPreferableTarget(target); - - int size[] = {1, 2, 3, 4}; - int scale_size[] = {1, 2, 1, 1}; - Mat scale(4, &scale_size[0], CV_32F); - Mat shift(4, &size[0], CV_32F); - Mat inp(4, &size[0], CV_32F); - randu(scale, -1.0f, 1.0f); - randu(shift, -1.0f, 1.0f); - randu(inp, -1.0f, 1.0f); - - net.setInput(scale, "scale"); - net.setInput(shift, "shift"); - net.setInput(inp, "data"); - - Mat out = net.forward(); - - Mat ref(4, &size[0], inp.type()); - for (int i = 0; i < inp.size[1]; i++) { - for (int h = 0; h < inp.size[2]; h++) { - for (int w = 0; w < inp.size[3]; w++) { - int idx[] = {0, i, h, w}; - int scale_idx[] = {0, i, 0, 0}; - ref.at(idx) = inp.at(idx) * scale.at(scale_idx) + - shift.at(idx); - } - } - } - float l1 = 1e-5, lInf = 1e-4; - if (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_CPU_FP16) - { - l1 = 2e-4; - lInf = 1e-3; - } - if (target == DNN_TARGET_MYRIAD) - { - l1 = 0.001; - lInf = 0.001; - } - if(target == DNN_TARGET_CUDA_FP16) - { - l1 = 0.0002; - lInf = 0.0007; - } - normAssert(ref, out, "", l1, lInf); -} - -typedef testing::TestWithParam > Reproducibility_AlexNet; -TEST_P(Reproducibility_AlexNet, Accuracy) -{ - Target targetId = get<1>(GetParam()); -#if defined(OPENCV_32BIT_CONFIGURATION) && defined(HAVE_OPENCL) - applyTestTag(CV_TEST_TAG_MEMORY_2GB); -#else - applyTestTag(targetId == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_512MB : CV_TEST_TAG_MEMORY_1GB); -#endif - ASSERT_TRUE(ocl::useOpenCL() || targetId == DNN_TARGET_CPU || targetId == DNN_TARGET_CPU_FP16); - - bool readFromMemory = get<0>(GetParam()); - Net net; - { - const string proto = findDataFile("dnn/bvlc_alexnet.prototxt"); - const string model = findDataFile("dnn/bvlc_alexnet.caffemodel", false); - if (readFromMemory) - { - std::vector dataProto; - readFileContent(proto, dataProto); - std::vector dataModel; - readFileContent(model, dataModel); - - net = readNetFromCaffe(dataProto.data(), dataProto.size(), - dataModel.data(), dataModel.size()); - } - else - net = readNetFromCaffe(proto, model); - ASSERT_FALSE(net.empty()); - } - - // Test input layer size - std::vector inLayerShapes; - std::vector outLayerShapes; - net.getLayerShapes(MatShape(), CV_32F, 0, inLayerShapes, outLayerShapes); - ASSERT_FALSE(inLayerShapes.empty()); - ASSERT_EQ(inLayerShapes[0].size(), 4); - ASSERT_EQ(inLayerShapes[0][0], 1); - ASSERT_EQ(inLayerShapes[0][1], 3); - ASSERT_EQ(inLayerShapes[0][2], 227); - ASSERT_EQ(inLayerShapes[0][3], 227); - - const float l1 = 1e-5; - const float lInf = (targetId == DNN_TARGET_OPENCL_FP16 || targetId == DNN_TARGET_CPU_FP16) ? 4e-3 : 1e-4; - - net.setPreferableBackend(DNN_BACKEND_OPENCV); - net.setPreferableTarget(targetId); - - if (targetId == DNN_TARGET_CPU_FP16) - net.enableWinograd(false); - - Mat sample = imread(_tf("grace_hopper_227.png")); - ASSERT_TRUE(!sample.empty()); - - net.setInput(blobFromImage(sample, 1.0f, Size(227, 227), Scalar(), false), "data"); - - Mat out; - // BUG: https://github.com/opencv/opencv/issues/26349 - if (net.getMainGraph()) - out = net.forward(); - else - out = net.forward("prob"); - - Mat ref = blobFromNPY(_tf("caffe_alexnet_prob.npy")); - normAssert(ref, out, "", l1, lInf); -} - -INSTANTIATE_TEST_CASE_P(/**/, Reproducibility_AlexNet, Combine(testing::Bool(), - testing::ValuesIn(getAvailableTargets(DNN_BACKEND_OPENCV)))); - -TEST(Reproducibility_FCN, Accuracy) -{ - applyTestTag(CV_TEST_TAG_LONG, CV_TEST_TAG_DEBUG_VERYLONG, CV_TEST_TAG_MEMORY_2GB); - - Net net; - { - const string proto = findDataFile("dnn/fcn8s-heavy-pascal.prototxt"); - const string model = findDataFile("dnn/fcn8s-heavy-pascal.caffemodel", false); - net = readNetFromCaffe(proto, model); - ASSERT_FALSE(net.empty()); - } - net.setPreferableBackend(DNN_BACKEND_OPENCV); - - Mat sample = imread(_tf("street.png")); - ASSERT_TRUE(!sample.empty()); - - std::vector layerIds; - std::vector weights, blobs; - net.getMemoryConsumption(shape(1,3,227,227), CV_32F, layerIds, weights, blobs); - - net.setInput(blobFromImage(sample, 1.0f, Size(500, 500), Scalar(), false), "data"); - - Mat out; - // BUG: https://github.com/opencv/opencv/issues/26349 - if (net.getMainGraph()) - out = net.forward(); - else - out = net.forward("score"); - - Mat refData = imread(_tf("caffe_fcn8s_prob.png"), IMREAD_ANYDEPTH); - int shape[] = {1, 21, 500, 500}; - Mat ref(4, shape, CV_32FC1, refData.data); - - normAssert(ref, out, "", 0.013, 0.17); -} - TEST(Reproducibility_SSD, Accuracy) { applyTestTag( @@ -288,13 +90,19 @@ TEST(Reproducibility_SSD, Accuracy) CV_TEST_TAG_DEBUG_VERYLONG ); - Net net; + // The classic engine importer no longer carries the Caffe-SSD specific + // handling (LpNormalization/DetectionOutput); this model is supported on + // the new engine only. + auto engine_forced = static_cast( + cv::utils::getConfigurationParameterSizeT("OPENCV_FORCE_DNN_ENGINE", cv::dnn::ENGINE_AUTO)); + if (engine_forced == cv::dnn::ENGINE_CLASSIC) { - const string proto = findDataFile("dnn/ssd_vgg16.prototxt"); - const string model = findDataFile("dnn/VGG_ILSVRC2016_SSD_300x300_iter_440000.caffemodel", false); - net = readNetFromCaffe(proto, model); - ASSERT_FALSE(net.empty()); + applyTestTag(CV_TEST_TAG_DNN_SKIP_PARSER); + return; } + + Net net = readNetFromONNX(findDataFile("dnn/onnx/models/ssd_vgg16.onnx", false)); + ASSERT_FALSE(net.empty()); net.setPreferableBackend(DNN_BACKEND_OPENCV); Mat sample = imread(_tf("street.png")); @@ -304,334 +112,18 @@ TEST(Reproducibility_SSD, Accuracy) cvtColor(sample, sample, COLOR_BGRA2BGR); Mat in_blob = blobFromImage(sample, 1.0f, Size(300, 300), Scalar(), false); - net.setInput(in_blob, "data"); + net.setInput(in_blob); - // BUG: https://github.com/opencv/opencv/issues/26349 - Mat out; - if(net.getMainGraph()) - out = net.forward(); - else - out = net.forward("detection_out"); + Mat out = net.forward(); Mat ref = blobFromNPY(_tf("ssd_out.npy")); - normAssertDetections(ref, out, "", 0.06); -} - -typedef testing::TestWithParam > Reproducibility_MobileNet_SSD; -TEST_P(Reproducibility_MobileNet_SSD, Accuracy) -{ - const string proto = findDataFile("dnn/MobileNetSSD_deploy_19e3ec3.prototxt", false); - const string model = findDataFile("dnn/MobileNetSSD_deploy_19e3ec3.caffemodel", false); - Net net = readNetFromCaffe(proto, model); - int backendId = get<0>(GetParam()); - int targetId = get<1>(GetParam()); - - net.setPreferableBackend(backendId); - net.setPreferableTarget(targetId); - - Mat sample = imread(_tf("street.png")); - - Mat inp = blobFromImage(sample, 1.0f / 127.5, Size(300, 300), Scalar(127.5, 127.5, 127.5), false); - net.setInput(inp); - Mat out = net.forward().clone(); - - ASSERT_EQ(out.size[2], 100); - - float scores_diff = 1e-5, boxes_iou_diff = 1e-4; - if (targetId == DNN_TARGET_OPENCL_FP16 || targetId == DNN_TARGET_MYRIAD || targetId == DNN_TARGET_CPU_FP16) - { - scores_diff = 1.5e-2; - boxes_iou_diff = 6.3e-2; - } - else if (targetId == DNN_TARGET_CUDA_FP16) - { - scores_diff = 0.015; - boxes_iou_diff = 0.07; - } - Mat ref = blobFromNPY(_tf("mobilenet_ssd_caffe_out.npy")); - normAssertDetections(ref, out, "", FLT_MIN, scores_diff, boxes_iou_diff); - - // Check that detections aren't preserved. - inp.setTo(0.0f); - net.setInput(inp); - Mat zerosOut = net.forward(); - zerosOut = zerosOut.reshape(1, zerosOut.total() / 7); - - const int numDetections = zerosOut.rows; - // TODO: fix it - if (targetId != DNN_TARGET_MYRIAD || - getInferenceEngineVPUType() != CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_X) - { - ASSERT_NE(numDetections, 0); - for (int i = 0; i < numDetections; ++i) - { - float confidence = zerosOut.ptr(i)[2]; - ASSERT_EQ(confidence, 0); - } - } - - // There is something wrong with Reshape layer in Myriad plugin. - if (backendId == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 - || backendId == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH - ) - { - if (targetId == DNN_TARGET_MYRIAD || targetId == DNN_TARGET_OPENCL_FP16) - return; - } - - // Check batching mode. - inp = blobFromImages(std::vector(2, sample), 1.0f / 127.5, Size(300, 300), Scalar(127.5, 127.5, 127.5), false); - net.setInput(inp); - Mat outBatch = net.forward(); - - // Output blob has a shape 1x1x2Nx7 where N is a number of detection for - // a single sample in batch. The first numbers of detection vectors are batch id. - // For Inference Engine backend there is -1 delimiter which points the end of detections. - const int numRealDetections = ref.size[2]; - EXPECT_EQ(outBatch.size[2], 2 * numDetections); - out = out.reshape(1, numDetections).rowRange(0, numRealDetections); - outBatch = outBatch.reshape(1, 2 * numDetections); - for (int i = 0; i < 2; ++i) - { - Mat pred = outBatch.rowRange(i * numRealDetections, (i + 1) * numRealDetections); - EXPECT_EQ(countNonZero(pred.col(0) != i), 0); - normAssert(pred.colRange(1, 7), out.colRange(1, 7)); - } -} -INSTANTIATE_TEST_CASE_P(/**/, Reproducibility_MobileNet_SSD, dnnBackendsAndTargets()); - -typedef testing::TestWithParam Reproducibility_ResNet50; -TEST_P(Reproducibility_ResNet50, Accuracy) -{ - Target targetId = GetParam(); - applyTestTag(targetId == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_512MB : CV_TEST_TAG_MEMORY_1GB); - ASSERT_TRUE(ocl::useOpenCL() || targetId == DNN_TARGET_CPU || targetId == DNN_TARGET_CPU_FP16); - - Net net = readNetFromCaffe(findDataFile("dnn/ResNet-50-deploy.prototxt"), - findDataFile("dnn/ResNet-50-model.caffemodel", false)); - - net.setPreferableBackend(DNN_BACKEND_OPENCV); - net.setPreferableTarget(targetId); - - if (targetId == DNN_TARGET_CPU_FP16) - net.enableWinograd(false); - - float l1 = (targetId == DNN_TARGET_OPENCL_FP16 || targetId == DNN_TARGET_CPU_FP16) ? 3e-5 : 1e-5; - float lInf = (targetId == DNN_TARGET_OPENCL_FP16 || targetId == DNN_TARGET_CPU_FP16) ? 6e-3 : 1e-4; - - Mat input = blobFromImage(imread(_tf("googlenet_0.png")), 1.0f, Size(224,224), Scalar(), false); - ASSERT_TRUE(!input.empty()); - - net.setInput(input); - Mat out = net.forward(); - - Mat ref = blobFromNPY(_tf("resnet50_prob.npy")); - normAssert(ref, out, "", l1, lInf); - - if (targetId == DNN_TARGET_OPENCL || targetId == DNN_TARGET_OPENCL_FP16) - { - UMat out_umat; - net.forward(out_umat); - normAssert(ref, out_umat, "out_umat", l1, lInf); - - std::vector out_umats; - net.forward(out_umats); - normAssert(ref, out_umats[0], "out_umat_vector", l1, lInf); - } -} -INSTANTIATE_TEST_CASE_P(/**/, Reproducibility_ResNet50, - testing::ValuesIn(getAvailableTargets(DNN_BACKEND_OPENCV))); - -typedef testing::TestWithParam Reproducibility_SqueezeNet_v1_1; -TEST_P(Reproducibility_SqueezeNet_v1_1, Accuracy) -{ - int targetId = GetParam(); - if(targetId == DNN_TARGET_OPENCL_FP16) - applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16); - if(targetId == DNN_TARGET_CPU_FP16) - applyTestTag(CV_TEST_TAG_DNN_SKIP_CPU_FP16); - Net net = readNetFromCaffe(findDataFile("dnn/squeezenet_v1.1.prototxt"), - findDataFile("dnn/squeezenet_v1.1.caffemodel", false)); - net.setPreferableBackend(DNN_BACKEND_OPENCV); - net.setPreferableTarget(targetId); - - Mat input = blobFromImage(imread(_tf("googlenet_0.png")), 1.0f, Size(227,227), Scalar(), false, true); - ASSERT_TRUE(!input.empty()); - - Mat out; - if (targetId == DNN_TARGET_OPENCL) - { - // Firstly set a wrong input blob and run the model to receive a wrong output. - // Then set a correct input blob to check CPU->GPU synchronization is working well. - net.setInput(input * 2.0f); - out = net.forward(); - } - net.setInput(input); - out = net.forward(); - - Mat ref = blobFromNPY(_tf("squeezenet_v1.1_prob.npy")); - normAssert(ref, out); -} -INSTANTIATE_TEST_CASE_P(/**/, Reproducibility_SqueezeNet_v1_1, - testing::ValuesIn(getAvailableTargets(DNN_BACKEND_OPENCV))); - -TEST(Reproducibility_AlexNet_fp16, Accuracy) -{ - applyTestTag(CV_TEST_TAG_MEMORY_512MB); - const float l1 = 1e-5; - const float lInf = 3e-3; - - const string proto = findDataFile("dnn/bvlc_alexnet.prototxt"); - const string model = findDataFile("dnn/bvlc_alexnet.caffemodel", false); - - shrinkCaffeModel(model, "bvlc_alexnet.caffemodel_fp16"); - Net net = readNetFromCaffe(proto, "bvlc_alexnet.caffemodel_fp16"); - net.setPreferableBackend(DNN_BACKEND_OPENCV); - - Mat sample = imread(findDataFile("dnn/grace_hopper_227.png")); - - net.setInput(blobFromImage(sample, 1.0f, Size(227, 227), Scalar())); - Mat out = net.forward(); - Mat ref = blobFromNPY(findDataFile("dnn/caffe_alexnet_prob.npy")); - normAssert(ref, out, "", l1, lInf); -} - -TEST(Reproducibility_GoogLeNet_fp16, Accuracy) -{ - const float l1 = 1e-5; - const float lInf = 3e-3; - - const string proto = findDataFile("dnn/bvlc_googlenet.prototxt"); - const string model = findDataFile("dnn/bvlc_googlenet.caffemodel", false); - - shrinkCaffeModel(model, "bvlc_googlenet.caffemodel_fp16"); - Net net = readNetFromCaffe(proto, "bvlc_googlenet.caffemodel_fp16"); - net.setPreferableBackend(DNN_BACKEND_OPENCV); - - std::vector inpMats; - inpMats.push_back( imread(_tf("googlenet_0.png")) ); - inpMats.push_back( imread(_tf("googlenet_1.png")) ); - ASSERT_TRUE(!inpMats[0].empty() && !inpMats[1].empty()); - - net.setInput(blobFromImages(inpMats, 1.0f, Size(), Scalar(), false), "data"); - - // BUG: https://github.com/opencv/opencv/issues/26349 - Mat out; - if(net.getMainGraph()) - out = net.forward(); - else - out = net.forward("prob"); - - Mat ref = blobFromNPY(_tf("googlenet_prob.npy")); - normAssert(out, ref, "", l1, lInf); -} - -// https://github.com/richzhang/colorization -TEST_P(Test_Caffe_nets, Colorization) -{ - applyTestTag( - target == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_512MB : CV_TEST_TAG_MEMORY_1GB, - CV_TEST_TAG_DEBUG_VERYLONG - ); - checkBackend(); - - Mat inp = blobFromNPY(_tf("colorization_inp.npy")); - Mat ref = blobFromNPY(_tf("colorization_out.npy")); - Mat kernel = blobFromNPY(_tf("colorization_pts_in_hull.npy")); - - const string proto = findDataFile("dnn/colorization_deploy_v2.prototxt", false); - const string model = findDataFile("dnn/colorization_release_v2.caffemodel", false); - Net net = readNetFromCaffe(proto, model); - net.setPreferableBackend(backend); - net.setPreferableTarget(target); - - // This model has bad accuracy when the FP16 and Winograd are enable at same time. - if (target == DNN_TARGET_CPU_FP16) - net.enableWinograd(false); - - net.getLayer(net.getLayerId("class8_ab"))->blobs.push_back(kernel); - net.getLayer(net.getLayerId("conv8_313_rh"))->blobs.push_back(Mat(1, 313, CV_32F, 2.606)); - - net.setInput(inp); - Mat out = net.forward(); - - // Reference output values are in range [-29.1, 69.5] - double l1 = 4e-4, lInf = 3e-3; - if (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_CPU_FP16) - { - l1 = 0.25; - lInf = 5.3; - } - else if (target == DNN_TARGET_MYRIAD) - { - l1 = (getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_X) ? 0.5 : 0.25; - lInf = (getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_X) ? 11 : 5.3; - } - else if(target == DNN_TARGET_CUDA_FP16) - { - l1 = 0.21; - lInf = 4.5; - } -#if defined(INF_ENGINE_RELEASE) - if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target == DNN_TARGET_OPENCL_FP16) - { - l1 = 0.3; lInf = 10; - } -#endif - - normAssert(out, ref, "", l1, lInf); - expectNoFallbacksFromIE(net); -} - -TEST_P(Test_Caffe_nets, DenseNet_121) -{ - applyTestTag(CV_TEST_TAG_MEMORY_512MB); - checkBackend(); - const string proto = findDataFile("dnn/DenseNet_121.prototxt", false); - const string weights = findDataFile("dnn/DenseNet_121.caffemodel", false); - - Mat inp = imread(_tf("dog416.png")); - Model model(proto, weights); - model.setInputScale(1.0 / 255).setInputSwapRB(true).setInputCrop(true); - std::vector outs; - Mat ref = blobFromNPY(_tf("densenet_121_output.npy")); - - model.setPreferableBackend(backend); - model.setPreferableTarget(target); - model.predict(inp, outs); - - // Reference is an array of 1000 values from a range [-6.16, 7.9] - float l1 = default_l1, lInf = default_lInf; - if (target == DNN_TARGET_OPENCL_FP16) - { -#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_GE(2019020000) - l1 = 0.05; lInf = 0.3; -#else - l1 = 0.017; lInf = 0.0795; -#endif - } - else if (target == DNN_TARGET_MYRIAD) - { - l1 = 0.11; lInf = 0.5; - } - else if (target == DNN_TARGET_CUDA_FP16) - { - l1 = 0.04; lInf = 0.2; - } - else if (target == DNN_TARGET_CPU_FP16) - { - l1 = 0.06; lInf = 0.3; - } - - normAssert(outs[0], ref, "", l1, lInf); - if (target != DNN_TARGET_MYRIAD || getInferenceEngineVPUType() != CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_X) - expectNoFallbacksFromIE(model.getNetwork_()); + normAssertDetections(ref, out, "", 0.06, 1e-4, 0.18); } TEST(Test_Caffe, multiple_inputs) { - const string proto = findDataFile("dnn/layers/net_input.prototxt"); - Net net = readNetFromCaffe(proto); + const string model = findDataFile("dnn/layers/net_input.onnx"); + Net net = readNetFromONNX(model); net.setPreferableBackend(DNN_BACKEND_OPENCV); Mat first_image(10, 11, CV_32FC3); @@ -656,239 +148,6 @@ TEST(Test_Caffe, multiple_inputs) normAssert(out, first_image + second_image); } -TEST(Test_Caffe, shared_weights) -{ - const string proto = findDataFile("dnn/layers/shared_weights.prototxt"); - const string model = findDataFile("dnn/layers/shared_weights.caffemodel"); - - Net net = readNetFromCaffe(proto, model); - - Mat input_1 = (Mat_(2, 2) << 0., 2., 4., 6.); - Mat input_2 = (Mat_(2, 2) << 1., 3., 5., 7.); - - Mat blob_1 = blobFromImage(input_1); - Mat blob_2 = blobFromImage(input_2); - - net.setInput(blob_1, "input_1"); - net.setInput(blob_2, "input_2"); - net.setPreferableBackend(DNN_BACKEND_OPENCV); - - Mat sum = net.forward(); - - EXPECT_EQ(sum.at(0,0), 12.); - EXPECT_EQ(sum.at(0,1), 16.); -} - -typedef testing::TestWithParam > opencv_face_detector; -TEST_P(opencv_face_detector, Accuracy) -{ - std::string proto = findDataFile("dnn/opencv_face_detector.prototxt"); - std::string model = findDataFile(get<0>(GetParam()), false); - dnn::Target targetId = (dnn::Target)(int)get<1>(GetParam()); - - if (targetId == DNN_TARGET_OPENCL_FP16) - applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16); - if (targetId == DNN_TARGET_CPU_FP16) - applyTestTag(CV_TEST_TAG_DNN_SKIP_CPU_FP16); - - Net net = readNetFromCaffe(proto, model); - Mat img = imread(findDataFile("gpu/lbpcascade/er.png")); - Mat blob = blobFromImage(img, 1.0, Size(), Scalar(104.0, 177.0, 123.0), false, false); - - net.setPreferableBackend(DNN_BACKEND_OPENCV); - net.setPreferableTarget(targetId); - - net.setInput(blob); - // Output has shape 1x1xNx7 where N - number of detections. - // An every detection is a vector of values [id, classId, confidence, left, top, right, bottom] - Mat out = net.forward(); - Mat ref = (Mat_(6, 7) << 0, 1, 0.99520785, 0.80997437, 0.16379407, 0.87996572, 0.26685631, - 0, 1, 0.9934696, 0.2831718, 0.50738752, 0.345781, 0.5985168, - 0, 1, 0.99096733, 0.13629119, 0.24892329, 0.19756334, 0.3310290, - 0, 1, 0.98977017, 0.23901358, 0.09084064, 0.29902688, 0.1769477, - 0, 1, 0.97203469, 0.67965847, 0.06876482, 0.73999709, 0.1513494, - 0, 1, 0.95097077, 0.51901293, 0.45863652, 0.5777427, 0.5347801); - normAssertDetections(ref, out, "", 0.5, 1e-4, 2e-4); -} - -// False positives bug for large faces: https://github.com/opencv/opencv/issues/15106 -TEST_P(opencv_face_detector, issue_15106) -{ - std::string proto = findDataFile("dnn/opencv_face_detector.prototxt"); - std::string model = findDataFile(get<0>(GetParam()), false); - dnn::Target targetId = (dnn::Target)(int)get<1>(GetParam()); - - if (targetId == DNN_TARGET_OPENCL_FP16) - applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16); - if (targetId == DNN_TARGET_CPU_FP16) - applyTestTag(CV_TEST_TAG_DNN_SKIP_CPU_FP16); - - Net net = readNetFromCaffe(proto, model); - Mat img = imread(findDataFile("cv/shared/lena.png")); - img = img.rowRange(img.rows / 4, 3 * img.rows / 4).colRange(img.cols / 4, 3 * img.cols / 4); - Mat blob = blobFromImage(img, 1.0, Size(300, 300), Scalar(104.0, 177.0, 123.0), false, false); - - net.setPreferableBackend(DNN_BACKEND_OPENCV); - net.setPreferableTarget(targetId); - - net.setInput(blob); - // Output has shape 1x1xNx7 where N - number of detections. - // An every detection is a vector of values [id, classId, confidence, left, top, right, bottom] - Mat out = net.forward(); - Mat ref = (Mat_(1, 7) << 0, 1, 0.9149431, 0.30424616, 0.26964942, 0.88733053, 0.99815309); - normAssertDetections(ref, out, "", 0.89, 6e-5, 1e-4); -} -INSTANTIATE_TEST_CASE_P(Test_Caffe, opencv_face_detector, - Combine( - Values("dnn/opencv_face_detector.caffemodel", - "dnn/opencv_face_detector_fp16.caffemodel"), - testing::ValuesIn(getAvailableTargets(DNN_BACKEND_OPENCV)) - ) -); - -TEST_P(Test_Caffe_nets, FasterRCNN_vgg16) -{ - applyTestTag( -#if defined(OPENCV_32BIT_CONFIGURATION) && defined(HAVE_OPENCL) - CV_TEST_TAG_MEMORY_2GB, // utilizes ~1Gb, but huge blobs may not be allocated on 32-bit systems due memory fragmentation -#else - CV_TEST_TAG_MEMORY_2GB, -#endif - CV_TEST_TAG_LONG, - CV_TEST_TAG_DEBUG_VERYLONG - ); - -#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_LT(2021040000) - if ((backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 || backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) && (target == DNN_TARGET_OPENCL || target == DNN_TARGET_OPENCL_FP16)) - applyTestTag(target == DNN_TARGET_OPENCL ? CV_TEST_TAG_DNN_SKIP_IE_OPENCL : CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16); - - if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) - applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_NGRAPH, CV_TEST_TAG_DNN_SKIP_IE_VERSION); - - if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 && target == DNN_TARGET_MYRIAD) - applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD); -#endif - -#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2021040000) - // IE exception: Ngraph operation Reshape with name rpn_cls_score_reshape has dynamic output shape on 0 port, but CPU plug-in supports only static shape - if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && (target == DNN_TARGET_OPENCL || target == DNN_TARGET_OPENCL_FP16)) - applyTestTag(target == DNN_TARGET_OPENCL ? CV_TEST_TAG_DNN_SKIP_IE_OPENCL : CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16, - CV_TEST_TAG_DNN_SKIP_IE_NGRAPH, CV_TEST_TAG_DNN_SKIP_IE_VERSION - ); - // Check 'backward_compatible_check || in_out_elements_equal' failed at core/src/op/reshape.cpp:390: - // While validating node 'v1::Reshape bbox_pred_reshape (bbox_pred[0]:f32{1,84}, Constant_241202[0]:i64{4}) -> (f32{?,?,?,?})' with friendly_name 'bbox_pred_reshape': - // Requested output shape {1,6300,4,1} is incompatible with input shape Shape{1, 84} - if (target == DNN_TARGET_MYRIAD) - applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD, CV_TEST_TAG_DNN_SKIP_IE_NGRAPH, CV_TEST_TAG_DNN_SKIP_IE_VERSION); -#endif - - double scoreDiff = 0.0012, iouDiff = 0.03; -#if defined(INF_ENGINE_RELEASE) - if (target == DNN_TARGET_MYRIAD) - applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD, CV_TEST_TAG_DNN_SKIP_IE_NGRAPH, CV_TEST_TAG_DNN_SKIP_IE_VERSION); - if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) { - iouDiff = 0.02; - if (target == DNN_TARGET_OPENCL || target == DNN_TARGET_OPENCL_FP16) { - scoreDiff = 0.04; - iouDiff = 0.06; - } - } -#endif - - static Mat ref = (Mat_(3, 7) << 0, 2, 0.949398, 99.2454, 210.141, 601.205, 462.849, - 0, 7, 0.997022, 481.841, 92.3218, 722.685, 175.953, - 0, 12, 0.993028, 133.221, 189.377, 350.994, 563.166); - testFaster("faster_rcnn_vgg16.prototxt", "VGG16_faster_rcnn_final.caffemodel", ref, scoreDiff, iouDiff); -} - -TEST_P(Test_Caffe_nets, FasterRCNN_zf) -{ - applyTestTag( -#if defined(OPENCV_32BIT_CONFIGURATION) && defined(HAVE_OPENCL) - CV_TEST_TAG_MEMORY_2GB, -#else - (target == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_512MB : CV_TEST_TAG_MEMORY_1GB), -#endif - CV_TEST_TAG_DEBUG_VERYLONG - ); -#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2021040000) - // IE exception: Ngraph operation Reshape with name rpn_cls_score_reshape has dynamic output shape on 0 port, but CPU plug-in supports only static shape - if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && (target == DNN_TARGET_OPENCL || target == DNN_TARGET_OPENCL_FP16)) - applyTestTag(target == DNN_TARGET_OPENCL ? CV_TEST_TAG_DNN_SKIP_IE_OPENCL : CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16, - CV_TEST_TAG_DNN_SKIP_IE_NGRAPH, CV_TEST_TAG_DNN_SKIP_IE_VERSION - ); -#endif - - if ((backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 || - backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) && target == DNN_TARGET_MYRIAD) - applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD); - if (target == DNN_TARGET_CUDA_FP16) - applyTestTag(CV_TEST_TAG_DNN_SKIP_CUDA_FP16); - if (target == DNN_TARGET_CPU_FP16) - applyTestTag(CV_TEST_TAG_DNN_SKIP_CPU_FP16); - static Mat ref = (Mat_(3, 7) << 0, 2, 0.90121, 120.407, 115.83, 570.586, 528.395, - 0, 7, 0.988779, 469.849, 75.1756, 718.64, 186.762, - 0, 12, 0.967198, 138.588, 206.843, 329.766, 553.176); - - double scoreDiff = 0.003, iouDiff = 0.07; - if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) { - scoreDiff = 0.02; - iouDiff = 0.13; - } - - testFaster("faster_rcnn_zf.prototxt", "ZF_faster_rcnn_final.caffemodel", ref, scoreDiff, iouDiff); -} - -TEST_P(Test_Caffe_nets, RFCN) -{ - applyTestTag( - (target == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_512MB : CV_TEST_TAG_MEMORY_2GB), - CV_TEST_TAG_LONG, - CV_TEST_TAG_DEBUG_VERYLONG - ); - - float scoreDiff = default_l1, iouDiff = default_lInf; - if (backend == DNN_BACKEND_OPENCV && (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_CPU_FP16)) - { - scoreDiff = 4e-3; - iouDiff = 8e-2; - } - if (target == DNN_TARGET_CUDA_FP16) - { - scoreDiff = 0.0034; - iouDiff = 0.12; - } - -#if defined(INF_ENGINE_RELEASE) - if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) - { - scoreDiff = 0.1f; - iouDiff = 0.2f; - } - - // Check 'backward_compatible_check || in_out_elements_equal' failed at core/src/op/reshape.cpp:427: - // While validating node 'v1::Reshape bbox_pred_reshape (ave_bbox_pred_rois[0]:f32{1,8,1,1}, Constant_388[0]:i64{4}) -> (f32{?,?,?,?})' with friendly_name 'bbox_pred_reshape': - // Requested output shape {1,300,8,1} is incompatible with input shape {1, 8, 1, 1} - if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target == DNN_TARGET_MYRIAD) - applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD, CV_TEST_TAG_DNN_SKIP_IE_NGRAPH, CV_TEST_TAG_DNN_SKIP_IE_VERSION); -#elif defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2021040000) - // Exception: Function contains several inputs and outputs with one friendly name! (HETERO bug?) - if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target != DNN_TARGET_CPU) - applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_NGRAPH, CV_TEST_TAG_DNN_SKIP_IE_VERSION); -#elif defined(INF_ENGINE_RELEASE) - if ((backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 || - backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) && target == DNN_TARGET_OPENCL_FP16) - applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16); - if ((backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 || - backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) && target == DNN_TARGET_MYRIAD) - applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD); -#endif - - static Mat ref = (Mat_(2, 7) << 0, 7, 0.991359, 491.822, 81.1668, 702.573, 178.234, - 0, 12, 0.94786, 132.093, 223.903, 338.077, 566.16); - testFaster("rfcn_pascal_voc_resnet50.prototxt", "resnet50_rfcn_final.caffemodel", ref, scoreDiff, iouDiff); -} - INSTANTIATE_TEST_CASE_P(/**/, Test_Caffe_nets, dnnBackendsAndTargets()); }} // namespace diff --git a/modules/dnn/test/test_googlenet.cpp b/modules/dnn/test/test_googlenet.cpp deleted file mode 100644 index 2868cbbf64..0000000000 --- a/modules/dnn/test/test_googlenet.cpp +++ /dev/null @@ -1,169 +0,0 @@ -/*M/////////////////////////////////////////////////////////////////////////////////////// -// -// IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. -// -// By downloading, copying, installing or using the software you agree to this license. -// If you do not agree to this license, do not download, install, -// copy or use the software. -// -// -// License Agreement -// For Open Source Computer Vision Library -// -// Copyright (C) 2013, OpenCV Foundation, all rights reserved. -// Third party copyrights are property of their respective owners. -// -// Redistribution and use in source and binary forms, with or without modification, -// are permitted provided that the following conditions are met: -// -// * Redistribution's of source code must retain the above copyright notice, -// this list of conditions and the following disclaimer. -// -// * Redistribution's in binary form must reproduce the above copyright notice, -// this list of conditions and the following disclaimer in the documentation -// and/or other materials provided with the distribution. -// -// * The name of the copyright holders may not be used to endorse or promote products -// derived from this software without specific prior written permission. -// -// This software is provided by the copyright holders and contributors "as is" and -// any express or implied warranties, including, but not limited to, the implied -// warranties of merchantability and fitness for a particular purpose are disclaimed. -// In no event shall the Intel Corporation or contributors be liable for any direct, -// indirect, incidental, special, exemplary, or consequential damages -// (including, but not limited to, procurement of substitute goods or services; -// loss of use, data, or profits; or business interruption) however caused -// and on any theory of liability, whether in contract, strict liability, -// or tort (including negligence or otherwise) arising in any way out of -// the use of this software, even if advised of the possibility of such damage. -// -//M*/ - -#include "test_precomp.hpp" -#include "npy_blob.hpp" -#include -#include - -namespace opencv_test { namespace { - -template -static std::string _tf(TString filename) -{ - return (getOpenCVExtraDir() + "/dnn/") + filename; -} - -typedef testing::TestWithParam Reproducibility_GoogLeNet; -TEST_P(Reproducibility_GoogLeNet, Batching) -{ - const int targetId = GetParam(); - if (targetId == DNN_TARGET_OPENCL_FP16) - applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16); - if (targetId == DNN_TARGET_CPU_FP16) - applyTestTag(CV_TEST_TAG_DNN_SKIP_CPU_FP16); - Net net = readNetFromCaffe(findDataFile("dnn/bvlc_googlenet.prototxt"), - findDataFile("dnn/bvlc_googlenet.caffemodel", false)); - net.setPreferableBackend(DNN_BACKEND_OPENCV); - net.setPreferableTarget(targetId); - - if (targetId == DNN_TARGET_OPENCL) - { - // Initialize network for a single image in the batch but test with batch size=2. - Mat inp = Mat(224, 224, CV_8UC3); - randu(inp, -1, 1); - net.setInput(blobFromImage(inp)); - net.forward(); - } - - std::vector inpMats; - inpMats.push_back( imread(_tf("googlenet_0.png")) ); - inpMats.push_back( imread(_tf("googlenet_1.png")) ); - ASSERT_TRUE(!inpMats[0].empty() && !inpMats[1].empty()); - - net.setInput(blobFromImages(inpMats, 1.0f, Size(), Scalar(), false), "data"); - - // BUG: https://github.com/opencv/opencv/issues/26349 - Mat out; - if(net.getMainGraph()) - out = net.forward(); - else - out = net.forward("prob"); - - Mat ref = blobFromNPY(_tf("googlenet_prob.npy")); - normAssert(out, ref); -} - -TEST_P(Reproducibility_GoogLeNet, IntermediateBlobs) -{ - const int targetId = GetParam(); - if (targetId == DNN_TARGET_OPENCL_FP16) - applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16); - if (targetId == DNN_TARGET_CPU_FP16) - applyTestTag(CV_TEST_TAG_DNN_SKIP_CPU_FP16); - // BUG: https://github.com/opencv/opencv/issues/26349 - Net net = readNetFromCaffe(findDataFile("dnn/bvlc_googlenet.prototxt"), - findDataFile("dnn/bvlc_googlenet.caffemodel", false), ENGINE_CLASSIC); - net.setPreferableBackend(DNN_BACKEND_OPENCV); - net.setPreferableTarget(targetId); - - std::vector blobsNames; - blobsNames.push_back("conv1/7x7_s2"); - blobsNames.push_back("conv1/relu_7x7"); - blobsNames.push_back("inception_4c/1x1"); - blobsNames.push_back("inception_4c/relu_1x1"); - std::vector outs; - Mat in = blobFromImage(imread(_tf("googlenet_0.png")), 1.0f, Size(), Scalar(), false); - net.setInput(in, "data"); - net.forward(outs, blobsNames); - CV_Assert(outs.size() == blobsNames.size()); - - for (size_t i = 0; i < blobsNames.size(); i++) - { - std::string filename = blobsNames[i]; - std::replace( filename.begin(), filename.end(), '/', '#'); - Mat ref = blobFromNPY(_tf("googlenet_" + filename + ".npy")); - - normAssert(outs[i], ref, "", 1E-4, 1E-2); - } -} - -TEST_P(Reproducibility_GoogLeNet, SeveralCalls) -{ - const int targetId = GetParam(); - if (targetId == DNN_TARGET_OPENCL_FP16) - applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16); - if (targetId == DNN_TARGET_CPU_FP16) - applyTestTag(CV_TEST_TAG_DNN_SKIP_CPU_FP16); - // BUG: https://github.com/opencv/opencv/issues/26349 - Net net = readNetFromCaffe(findDataFile("dnn/bvlc_googlenet.prototxt"), - findDataFile("dnn/bvlc_googlenet.caffemodel", false), ENGINE_CLASSIC); - net.setPreferableBackend(DNN_BACKEND_OPENCV); - net.setPreferableTarget(targetId); - - std::vector inpMats; - inpMats.push_back( imread(_tf("googlenet_0.png")) ); - inpMats.push_back( imread(_tf("googlenet_1.png")) ); - ASSERT_TRUE(!inpMats[0].empty() && !inpMats[1].empty()); - - net.setInput(blobFromImages(inpMats, 1.0f, Size(), Scalar(), false), "data"); - Mat out = net.forward(); - - Mat ref = blobFromNPY(_tf("googlenet_prob.npy")); - normAssert(out, ref); - - std::vector blobsNames; - blobsNames.push_back("conv1/7x7_s2"); - std::vector outs; - Mat in = blobFromImage(inpMats[0], 1.0f, Size(), Scalar(), false); - net.setInput(in, "data"); - net.forward(outs, blobsNames); - CV_Assert(outs.size() == blobsNames.size()); - - ref = blobFromNPY(_tf("googlenet_conv1#7x7_s2.npy")); - - normAssert(outs[0], ref, "", 1E-4, 1E-2); -} - -INSTANTIATE_TEST_CASE_P(/**/, Reproducibility_GoogLeNet, - testing::ValuesIn(getAvailableTargets(DNN_BACKEND_OPENCV))); - -}} // namespace diff --git a/modules/dnn/test/test_int8_layers.cpp b/modules/dnn/test/test_int8_layers.cpp index 88013f78bd..8eee0d5ccc 100644 --- a/modules/dnn/test/test_int8_layers.cpp +++ b/modules/dnn/test/test_int8_layers.cpp @@ -46,16 +46,7 @@ public: String inpPath, outPath; Net net, qnet; - if (importer == "Caffe") - { - String prototxt = _tf("layers/" + basename + ".prototxt"); - String caffemodel = _tf("layers/" + basename + ".caffemodel"); - net = readNetFromCaffe(prototxt, useCaffeModel ? caffemodel : String()); - - inpPath = _tf("layers/" + (useCommonInputBlob ? "blob" : basename + ".input")); - outPath = _tf("layers/" + basename); - } - else if (importer == "TensorFlow") + if (importer == "TensorFlow") { String netPath = _tf("tensorflow/" + basename + "_net.pb"); String netConfig = hasText ? _tf("tensorflow/" + basename + "_net.pbtxt") : ""; @@ -768,121 +759,6 @@ public: } }; -TEST_P(Test_Int8_nets, AlexNet) -{ -#if defined(OPENCV_32BIT_CONFIGURATION) && defined(HAVE_OPENCL) - applyTestTag(CV_TEST_TAG_MEMORY_2GB); -#else - applyTestTag(target == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_512MB : CV_TEST_TAG_MEMORY_1GB); -#endif - if (target == DNN_TARGET_OPENCL_FP16 && !ocl::Device::getDefault().isIntel()) - applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16); - if (target == DNN_TARGET_OPENCL && !ocl::Device::getDefault().isIntel()) - applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL); - - - Net net = readNetFromCaffe(findDataFile("dnn/bvlc_alexnet.prototxt"), - findDataFile("dnn/bvlc_alexnet.caffemodel", false)); - - Mat inp = imread(_tf("grace_hopper_227.png")); - Mat blob = blobFromImage(inp, 1.0, Size(227, 227), Scalar(), false); - Mat ref = blobFromNPY(_tf("caffe_alexnet_prob.npy")); - - float l1 = 1e-4, lInf = 0.003; - testClassificationNet(net, blob, ref, l1, lInf); -} - -TEST_P(Test_Int8_nets, GoogLeNet) -{ - if (target == DNN_TARGET_OPENCL_FP16 && !ocl::Device::getDefault().isIntel()) - applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16); - if (target == DNN_TARGET_OPENCL && !ocl::Device::getDefault().isIntel()) - applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL); - - Net net = readNetFromCaffe(findDataFile("dnn/bvlc_googlenet.prototxt"), - findDataFile("dnn/bvlc_googlenet.caffemodel", false)); - - std::vector inpMats; - inpMats.push_back( imread(_tf("googlenet_0.png")) ); - inpMats.push_back( imread(_tf("googlenet_1.png")) ); - Mat blob = blobFromImages(inpMats, 1.0, Size(224, 224), Scalar(), false); - Mat ref = blobFromNPY(_tf("googlenet_prob.npy")); - - float l1 = 2e-4, lInf = 0.07; - testClassificationNet(net, blob, ref, l1, lInf); -} - -TEST_P(Test_Int8_nets, ResNet50) -{ - applyTestTag( - target == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_512MB : CV_TEST_TAG_MEMORY_1GB, - CV_TEST_TAG_DEBUG_VERYLONG - ); - - if (target == DNN_TARGET_OPENCL_FP16 && !ocl::Device::getDefault().isIntel()) - applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16); - if (target == DNN_TARGET_OPENCL && !ocl::Device::getDefault().isIntel()) - applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL); - - Net net = readNetFromCaffe(findDataFile("dnn/ResNet-50-deploy.prototxt"), - findDataFile("dnn/ResNet-50-model.caffemodel", false)); - - Mat inp = imread(_tf("googlenet_0.png")); - Mat blob = blobFromImage(inp, 1.0, Size(224, 224), Scalar(), false); - Mat ref = blobFromNPY(_tf("resnet50_prob.npy")); - - float l1 = 3e-4, lInf = 0.05; - testClassificationNet(net, blob, ref, l1, lInf); - - { - SCOPED_TRACE("Per-tensor quantize"); - testClassificationNet(net, blob, ref, l1, lInf, false); - } -} - -TEST_P(Test_Int8_nets, DenseNet121) -{ - applyTestTag(CV_TEST_TAG_MEMORY_512MB); - - if (target == DNN_TARGET_OPENCL_FP16 && !ocl::Device::getDefault().isIntel()) - applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16); - if (target == DNN_TARGET_OPENCL && !ocl::Device::getDefault().isIntel()) - applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL); - if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) - applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_NGRAPH); - - Net net = readNetFromCaffe(findDataFile("dnn/DenseNet_121.prototxt", false), - findDataFile("dnn/DenseNet_121.caffemodel", false)); - - Mat inp = imread(_tf("dog416.png")); - Mat blob = blobFromImage(inp, 1.0 / 255.0, Size(224, 224), Scalar(), true, true); - Mat ref = blobFromNPY(_tf("densenet_121_output.npy")); - - float l1 = 0.76, lInf = 3.31; // seems wrong - testClassificationNet(net, blob, ref, l1, lInf); -} - -TEST_P(Test_Int8_nets, SqueezeNet_v1_1) -{ - if(target == DNN_TARGET_OPENCL_FP16) - applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16); - - if (target == DNN_TARGET_OPENCL_FP16 && !ocl::Device::getDefault().isIntel()) - applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16); - if (target == DNN_TARGET_OPENCL && !ocl::Device::getDefault().isIntel()) - applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL); - - Net net = readNetFromCaffe(findDataFile("dnn/squeezenet_v1.1.prototxt"), - findDataFile("dnn/squeezenet_v1.1.caffemodel", false)); - - Mat inp = imread(_tf("googlenet_0.png")); - Mat blob = blobFromImage(inp, 1.0, Size(227, 227), Scalar(), false, true); - Mat ref = blobFromNPY(_tf("squeezenet_v1.1_prob.npy")); - - float l1 = 3e-4, lInf = 0.056; - testClassificationNet(net, blob, ref, l1, lInf); -} - TEST_P(Test_Int8_nets, CaffeNet) { #if defined(OPENCV_32BIT_CONFIGURATION) && (defined(HAVE_OPENCL) || defined(_WIN32)) @@ -947,24 +823,6 @@ TEST_P(Test_Int8_nets, Shufflenet) testONNXNet("shufflenet", default_l1, default_lInf); } -TEST_P(Test_Int8_nets, MobileNet_SSD) -{ - if (target == DNN_TARGET_OPENCL_FP16 && !ocl::Device::getDefault().isIntel()) - applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16); - if (target == DNN_TARGET_OPENCL && !ocl::Device::getDefault().isIntel()) - applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL); - - Net net = readNetFromCaffe(findDataFile("dnn/MobileNetSSD_deploy_19e3ec3.prototxt", false), - findDataFile("dnn/MobileNetSSD_deploy_19e3ec3.caffemodel", false)); - - Mat inp = imread(_tf("street.png")); - Mat blob = blobFromImage(inp, 1.0 / 127.5, Size(300, 300), Scalar(127.5, 127.5, 127.5), false); - Mat ref = blobFromNPY(_tf("mobilenet_ssd_caffe_out.npy")); - - float confThreshold = FLT_MIN, scoreDiff = 0.084, iouDiff = 0.43; - testDetectionNet(net, blob, ref, confThreshold, scoreDiff, iouDiff); -} - TEST_P(Test_Int8_nets, MobileNet_v1_SSD) { if (target == DNN_TARGET_OPENCL_FP16 && !ocl::Device::getDefault().isIntel()) @@ -1025,31 +883,6 @@ TEST_P(Test_Int8_nets, Inception_v2_SSD) testDetectionNet(net, blob, ref, confThreshold, scoreDiff, iouDiff); } -TEST_P(Test_Int8_nets, opencv_face_detector) -{ - if (target == DNN_TARGET_OPENCL_FP16 && !ocl::Device::getDefault().isIntel()) - applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16); - if (target == DNN_TARGET_OPENCL && !ocl::Device::getDefault().isIntel()) - applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL); - if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) - applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_NGRAPH); - - Net net = readNetFromCaffe(findDataFile("dnn/opencv_face_detector.prototxt"), - findDataFile("dnn/opencv_face_detector.caffemodel", false)); - - Mat inp = imread(findDataFile("gpu/lbpcascade/er.png")); - Mat blob = blobFromImage(inp, 1.0, Size(), Scalar(104.0, 177.0, 123.0), false, false); - Mat ref = (Mat_(6, 7) << 0, 1, 0.99520785, 0.80997437, 0.16379407, 0.87996572, 0.26685631, - 0, 1, 0.9934696, 0.2831718, 0.50738752, 0.345781, 0.5985168, - 0, 1, 0.99096733, 0.13629119, 0.24892329, 0.19756334, 0.3310290, - 0, 1, 0.98977017, 0.23901358, 0.09084064, 0.29902688, 0.1769477, - 0, 1, 0.97203469, 0.67965847, 0.06876482, 0.73999709, 0.1513494, - 0, 1, 0.95097077, 0.51901293, 0.45863652, 0.5777427, 0.5347801); - - float confThreshold = 0.5, scoreDiff = 0.002, iouDiff = 0.4; - testDetectionNet(net, blob, ref, confThreshold, scoreDiff, iouDiff); -} - TEST_P(Test_Int8_nets, EfficientDet) { if (cvtest::skipUnstableTests) @@ -1143,91 +976,6 @@ TEST_P(Test_Int8_nets, FasterRCNN_inceptionv2) testDetectionNet(net, blob, ref, confThreshold, scoreDiff, iouDiff); } -TEST_P(Test_Int8_nets, FasterRCNN_vgg16) -{ - applyTestTag( -#if defined(OPENCV_32BIT_CONFIGURATION) && defined(HAVE_OPENCL) - CV_TEST_TAG_MEMORY_2GB, -#else - CV_TEST_TAG_MEMORY_2GB, -#endif - CV_TEST_TAG_LONG, - CV_TEST_TAG_DEBUG_VERYLONG - ); - - if (target == DNN_TARGET_OPENCL_FP16 && !ocl::Device::getDefault().isIntel()) - applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16); - if (target == DNN_TARGET_OPENCL && !ocl::Device::getDefault().isIntel()) - applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL); - if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) - applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_NGRAPH); - - Net net = readNetFromCaffe(findDataFile("dnn/faster_rcnn_vgg16.prototxt"), - findDataFile("dnn/VGG16_faster_rcnn_final.caffemodel", false)); - - Mat ref = (Mat_(3, 7) << 0, 2, 0.949398, 99.2454, 210.141, 601.205, 462.849, - 0, 7, 0.997022, 481.841, 92.3218, 722.685, 175.953, - 0, 12, 0.993028, 133.221, 189.377, 350.994, 563.166); - - float confThreshold = 0.8, scoreDiff = 0.048, iouDiff = 0.35; - testFaster(net, ref, confThreshold, scoreDiff, iouDiff); -} - -TEST_P(Test_Int8_nets, FasterRCNN_zf) -{ - applyTestTag( -#if defined(OPENCV_32BIT_CONFIGURATION) && defined(HAVE_OPENCL) - CV_TEST_TAG_MEMORY_2GB, -#else - (target == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_512MB : CV_TEST_TAG_MEMORY_1GB), -#endif - CV_TEST_TAG_DEBUG_VERYLONG - ); - - if (target == DNN_TARGET_OPENCL_FP16 && !ocl::Device::getDefault().isIntel()) - applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16); - if (target == DNN_TARGET_OPENCL && !ocl::Device::getDefault().isIntel()) - applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL); - if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) - applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_NGRAPH); - - Net net = readNetFromCaffe(findDataFile("dnn/faster_rcnn_zf.prototxt"), - findDataFile("dnn/ZF_faster_rcnn_final.caffemodel", false)); - - Mat ref = (Mat_(3, 7) << 0, 2, 0.90121, 120.407, 115.83, 570.586, 528.395, - 0, 7, 0.988779, 469.849, 75.1756, 718.64, 186.762, - 0, 12, 0.967198, 138.588, 206.843, 329.766, 553.176); - - float confThreshold = 0.8, scoreDiff = 0.021, iouDiff = 0.1; - testFaster(net, ref, confThreshold, scoreDiff, iouDiff); -} - -TEST_P(Test_Int8_nets, RFCN) -{ - applyTestTag( - (target == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_512MB : CV_TEST_TAG_MEMORY_2GB), - CV_TEST_TAG_LONG, - CV_TEST_TAG_DEBUG_VERYLONG - ); - - if (target == DNN_TARGET_OPENCL_FP16 && !ocl::Device::getDefault().isIntel()) - applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16); - if (target == DNN_TARGET_OPENCL && !ocl::Device::getDefault().isIntel()) - applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL); - - Net net = readNetFromCaffe(findDataFile("dnn/rfcn_pascal_voc_resnet50.prototxt"), - findDataFile("dnn/resnet50_rfcn_final.caffemodel", false)); - - Mat ref = (Mat_(2, 7) << 0, 7, 0.991359, 491.822, 81.1668, 702.573, 178.234, - 0, 12, 0.94786, 132.093, 223.903, 338.077, 566.16); - - float confThreshold = 0.8, scoreDiff = 0.15, iouDiff = 0.11; - if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) { - iouDiff = 0.12; - } - testFaster(net, ref, confThreshold, scoreDiff, iouDiff); -} - TEST_P(Test_Int8_nets, YOLOv3) { applyTestTag( diff --git a/modules/dnn/test/test_layers.cpp b/modules/dnn/test/test_layers.cpp index 6616aeda1b..c97e641b9d 100644 --- a/modules/dnn/test/test_layers.cpp +++ b/modules/dnn/test/test_layers.cpp @@ -63,155 +63,6 @@ static String _tf(TString filename) return (basetestdir + "dnn/layers/") + filename; } -class Test_Caffe_layers : public DNNTestLayer -{ -public: - void testLayerUsingCaffeModels(const String& basename, bool useCaffeModel = false, - bool useCommonInputBlob = true, double l1 = 0.0, double lInf = 0.0, - int numInps = 1, int numOuts = 1) - { - CV_Assert_N(numInps >= 1, numInps <= 10, numOuts >= 1, numOuts <= 10); - String prototxt = _tf(basename + ".prototxt"); - String caffemodel = _tf(basename + ".caffemodel"); - - std::vector inps, refs, outs; - - if (numInps > 1) - { - for (int i = 0; i < numInps; i++) - { - String inpfile = _tf(basename + cv::format(".input_%d.npy", i)); - inps.push_back(blobFromNPY(inpfile)); - } - } - else - { - String inpfile = (useCommonInputBlob) ? _tf("blob.npy") : _tf(basename + ".input.npy"); - inps.push_back(blobFromNPY(inpfile)); - } - - if (numOuts > 1) - { - for (int i = 0; i < numOuts; i++) - { - String outfile = _tf(basename + cv::format("_%d.npy", i)); - refs.push_back(blobFromNPY(outfile)); - } - } - else - { - String outfile = _tf(basename + ".npy"); - refs.push_back(blobFromNPY(outfile)); - } - - Net net = readNetFromCaffe(prototxt, (useCaffeModel) ? caffemodel : String()); - ASSERT_FALSE(net.empty()); - checkBackend(&inps[0], &refs[0]); - - net.setPreferableBackend(backend); - net.setPreferableTarget(target); - - String inp_name = "input"; - if (numInps > 1) - { - for (int i = 0; i < numInps; i++) - { - net.setInput(inps[i], inp_name + cv::format("_%d", i)); - } - } - else - { - net.setInput(inps.back(), inp_name); - } - - net.forward(outs); - for (int i = 0; i < refs.size(); i++) - { - normAssert(refs[i], outs[i], "", l1 ? l1 : default_l1, lInf ? lInf : default_lInf); - } - } -}; - -TEST_P(Test_Caffe_layers, Softmax) -{ - testLayerUsingCaffeModels("layer_softmax"); -} - -TEST_P(Test_Caffe_layers, LRN) -{ - double l1 = 0.0, lInf = 0.0; - // The OpenCL kernels use the native_ math functions which have - // implementation defined accuracy, so we use relaxed thresholds. See - // https://github.com/opencv/opencv/issues/9821 for more details. - if (target == DNN_TARGET_OPENCL) - { - l1 = 0.01; - lInf = 0.01; - } - testLayerUsingCaffeModels("layer_lrn_spatial", false, true, l1, lInf); - testLayerUsingCaffeModels("layer_lrn_channels", false, true, l1, lInf); -} - -TEST_P(Test_Caffe_layers, Convolution) -{ - testLayerUsingCaffeModels("layer_convolution", true); -} - -TEST_P(Test_Caffe_layers, DeConvolution) -{ - if(target == DNN_TARGET_CUDA_FP16) - applyTestTag(CV_TEST_TAG_DNN_SKIP_CUDA_FP16); - testLayerUsingCaffeModels("layer_deconvolution", true, false); -} - -TEST_P(Test_Caffe_layers, InnerProduct) -{ -#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_LT(2021040000) - if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019) - applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_NN_BUILDER); - if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) - applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_NGRAPH); -#endif -#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2021040000) - // IE exception: Ngraph operation Reshape with name Reshape_4219609 has dynamic output shape on 0 port, but CPU plug-in supports only static shape - if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && (target == DNN_TARGET_OPENCL || target == DNN_TARGET_OPENCL_FP16)) - applyTestTag(target == DNN_TARGET_OPENCL ? CV_TEST_TAG_DNN_SKIP_IE_OPENCL : CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16, - CV_TEST_TAG_DNN_SKIP_IE_NGRAPH, CV_TEST_TAG_DNN_SKIP_IE_VERSION - ); -#endif - - if (backend == DNN_BACKEND_OPENCV && target == DNN_TARGET_OPENCL_FP16) - applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16); - if (backend == DNN_BACKEND_OPENCV && target == DNN_TARGET_CPU_FP16) - applyTestTag(CV_TEST_TAG_DNN_SKIP_CPU_FP16); - - double l1 = 0.0, lInf = 0.0; - if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && (target == DNN_TARGET_OPENCL || target == DNN_TARGET_OPENCL_FP16)) - { - l1 = 5e-3; - lInf = 2e-2; - } - testLayerUsingCaffeModels("layer_inner_product", true, true, l1, lInf); -} - -TEST_P(Test_Caffe_layers, Pooling_max) -{ - testLayerUsingCaffeModels("layer_pooling_max"); -} - -TEST_P(Test_Caffe_layers, Pooling_ave) -{ - testLayerUsingCaffeModels("layer_pooling_ave"); -} - -TEST_P(Test_Caffe_layers, MVN) -{ - if(backend == DNN_BACKEND_CUDA) - applyTestTag(CV_TEST_TAG_DNN_SKIP_CUDA); /* MVN is unsupported */ - - testLayerUsingCaffeModels("layer_mvn"); -} - void testReshape(const MatShape& inputShape, const MatShape& targetShape, int axis = 0, int num_axes = -1, MatShape mask = MatShape()) @@ -259,192 +110,6 @@ TEST(Layer_Test_Reshape, Accuracy) } } -TEST_P(Test_Caffe_layers, BatchNorm) -{ - testLayerUsingCaffeModels("layer_batch_norm", true); - testLayerUsingCaffeModels("layer_batch_norm_local_stats", true, false); -} - -TEST_P(Test_Caffe_layers, ReLU) -{ - testLayerUsingCaffeModels("layer_relu"); -} - -TEST_P(Test_Caffe_layers, Dropout) -{ - testLayerUsingCaffeModels("layer_dropout"); -} - -TEST_P(Test_Caffe_layers, Concat) -{ - if (cvtest::skipUnstableTests && (backend == DNN_BACKEND_VKCOM)) - { - throw SkipTestException("Test_Caffe_layers.Concat test produces unstable result with Vulkan"); - } - -#if defined(INF_ENGINE_RELEASE) -#if INF_ENGINE_VER_MAJOR_GE(2019010000) && INF_ENGINE_VER_MAJOR_LT(2019020000) - if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 && target == DNN_TARGET_MYRIAD) - applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD, CV_TEST_TAG_DNN_SKIP_IE_NN_BUILDER, CV_TEST_TAG_DNN_SKIP_IE_VERSION); -#elif INF_ENGINE_VER_MAJOR_EQ(2019020000) - if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 && - (target == DNN_TARGET_OPENCL || target == DNN_TARGET_OPENCL_FP16)) - applyTestTag(target == DNN_TARGET_OPENCL ? CV_TEST_TAG_DNN_SKIP_IE_OPENCL : CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16, - CV_TEST_TAG_DNN_SKIP_IE_NN_BUILDER, CV_TEST_TAG_DNN_SKIP_IE_VERSION); -#endif - -#if INF_ENGINE_VER_MAJOR_LT(2021040000) - if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && - (target == DNN_TARGET_OPENCL || target == DNN_TARGET_OPENCL_FP16)) - applyTestTag(target == DNN_TARGET_OPENCL ? CV_TEST_TAG_DNN_SKIP_IE_OPENCL : CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16, - CV_TEST_TAG_DNN_SKIP_IE_NGRAPH, CV_TEST_TAG_DNN_SKIP_IE_VERSION); -#endif - -#endif - testLayerUsingCaffeModels("layer_concat"); - testLayerUsingCaffeModels("layer_concat_optim", true, false); - testLayerUsingCaffeModels("layer_concat_shared_input", true, false); -} - -TEST_P(Test_Caffe_layers, Fused_Concat) -{ - if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 && (target == DNN_TARGET_OPENCL || target == DNN_TARGET_OPENCL_FP16)) - applyTestTag(target == DNN_TARGET_OPENCL ? CV_TEST_TAG_DNN_SKIP_IE_OPENCL : CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16, - CV_TEST_TAG_DNN_SKIP_IE_NN_BUILDER, CV_TEST_TAG_DNN_SKIP_IE_VERSION); - - checkBackend(); - - // Test case - // input - // | - // v - // some_layer - // | | - // v v - // concat - Net net; - int interLayer; - { - LayerParams lp; - lp.type = "AbsVal"; - lp.name = "someLayer"; - interLayer = net.addLayerToPrev(lp.name, lp.type, lp); - } - { - LayerParams lp; - lp.set("axis", 1); - lp.type = "Concat"; - lp.name = "testConcat"; - int id = net.addLayer(lp.name, lp.type, lp); - net.connect(interLayer, 0, id, 0); - net.connect(interLayer, 0, id, 1); - } - int shape[] = {1, 2, 3, 4}; - Mat input(4, shape, CV_32F); - randu(input, 0.0f, 1.0f); // [0, 1] to make AbsVal an identity transformation. - - net.setInput(input); - net.setPreferableBackend(backend); - net.setPreferableTarget(target); - Mat out = net.forward(); - - normAssert(slice(out, Range::all(), Range(0, 2), Range::all(), Range::all()), input, "", default_l1, default_lInf); - normAssert(slice(out, Range::all(), Range(2, 4), Range::all(), Range::all()), input, "", default_l1, default_lInf); -} - -TEST_P(Test_Caffe_layers, Eltwise) -{ - if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 && target == DNN_TARGET_MYRIAD) - applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD); - testLayerUsingCaffeModels("layer_eltwise"); -} - -TEST_P(Test_Caffe_layers, PReLU) -{ - double lInf = (target == DNN_TARGET_MYRIAD || target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_CPU_FP16) ? 0.021 : 0.0; - testLayerUsingCaffeModels("layer_prelu", true, true, 0.0, lInf); -} - -// TODO: fix an unstable test case -TEST_P(Test_Caffe_layers, layer_prelu_fc) -{ - if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) - applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_NGRAPH); // TODO: fix this test for OpenVINO - - if (backend == DNN_BACKEND_OPENCV && target == DNN_TARGET_OPENCL_FP16) - applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16); - // Reference output values are in range [-0.0001, 10.3906] - double l1 = (target == DNN_TARGET_MYRIAD) ? 0.005 : 0.0; - double lInf = (target == DNN_TARGET_MYRIAD) ? 0.021 : 0.0; -#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_GE(2020040000) - if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target == DNN_TARGET_OPENCL) - { - l1 = 0.006f; lInf = 0.05f; - } - if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target == DNN_TARGET_OPENCL_FP16) - { - l1 = 0.01f; lInf = 0.05f; - } -#endif - testLayerUsingCaffeModels("layer_prelu_fc", true, false, l1, lInf); -} - -TEST_P(Test_Caffe_layers, Reshape_Split_Slice) -{ -#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_LT(2023000000) - if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019) - applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_NN_BUILDER); - if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) - applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_NGRAPH); -#endif - - Net net = readNetFromCaffe(_tf("reshape_and_slice_routines.prototxt")); - ASSERT_FALSE(net.empty()); - - net.setPreferableBackend(backend); - net.setPreferableTarget(target); - - Mat input(6, 12, CV_32F); - RNG rng(0); - rng.fill(input, RNG::UNIFORM, -1, 1); - - net.setInput(input, "input"); - Mat output; - if (net.getMainGraph()) - output = net.forward(); - else - output = net.forward("output"); - - normAssert(input, output, "", default_l1, default_lInf); -} - -TEST_P(Test_Caffe_layers, Conv_Elu) -{ -#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_RELEASE <= 2018050000 - if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 && target == DNN_TARGET_MYRIAD) - applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD, CV_TEST_TAG_DNN_SKIP_IE_VERSION); -#endif - - Net net = readNetFromTensorflow(_tf("layer_elu_model.pb")); - ASSERT_FALSE(net.empty()); - - Mat inp = blobFromNPY(_tf("layer_elu_in.npy")); - Mat ref = blobFromNPY(_tf("layer_elu_out.npy")); - - net.setInput(inp, "input"); - net.setPreferableBackend(backend); - net.setPreferableTarget(target); - Mat out = net.forward(); - - double l1 = default_l1, lInf = default_lInf; - if (target == DNN_TARGET_CUDA_FP16) - { - l1 = 0.0002; - lInf = 0.0005; - } - normAssert(ref, out, "", l1, lInf); -} - class Layer_LSTM_Test : public ::testing::Test { public: @@ -768,148 +433,6 @@ TEST(Layer_MHARoPe_Test_Accuracy_with_, Pytorch) normAssert(h_t_reference, outputs[0]); } -TEST_P(Test_Caffe_layers, Accum) -{ -#ifdef OPENCV_DNN_EXTERNAL_PROTOBUF - throw SkipTestException("Requires patched protobuf"); -#else - if (backend == DNN_BACKEND_OPENCV && target != DNN_TARGET_CPU) - applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL, CV_TEST_TAG_DNN_SKIP_OPENCL_FP16); - - testLayerUsingCaffeModels("accum", false, false, 0.0, 0.0, 2); - testLayerUsingCaffeModels("accum_ref", false, false, 0.0, 0.0, 2); -#endif -} - -TEST_P(Test_Caffe_layers, FlowWarp) -{ - if (backend == DNN_BACKEND_OPENCV && target == DNN_TARGET_OPENCL_FP16) - applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16); - - testLayerUsingCaffeModels("flow_warp", false, false, 0.0, 0.0, 2); -} - -TEST_P(Test_Caffe_layers, ChannelNorm) -{ - if (backend == DNN_BACKEND_OPENCV && target == DNN_TARGET_OPENCL_FP16) - applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16); - testLayerUsingCaffeModels("channel_norm", false, false); -} - -TEST_P(Test_Caffe_layers, DataAugmentation) -{ -#ifdef OPENCV_DNN_EXTERNAL_PROTOBUF - throw SkipTestException("Requires patched protobuf"); -#else - if (backend == DNN_BACKEND_OPENCV && target == DNN_TARGET_OPENCL_FP16) - applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16); - testLayerUsingCaffeModels("data_augmentation", true, false); - testLayerUsingCaffeModels("data_augmentation_2x1", true, false); - testLayerUsingCaffeModels("data_augmentation_8x6", true, false); -#endif -} - -TEST_P(Test_Caffe_layers, Resample) -{ -#ifdef OPENCV_DNN_EXTERNAL_PROTOBUF - throw SkipTestException("Requires patched protobuf"); -#else -#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_LT(2023000000) - if (backend != DNN_BACKEND_OPENCV) - applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_NN_BUILDER, CV_TEST_TAG_DNN_SKIP_IE_NGRAPH); -#endif - testLayerUsingCaffeModels("nearest_2inps", false, false, 0.0, 0.0, 2); - testLayerUsingCaffeModels("nearest", false, false); -#endif -} - -TEST_P(Test_Caffe_layers, Correlation) -{ -#ifdef OPENCV_DNN_EXTERNAL_PROTOBUF - throw SkipTestException("Requires patched protobuf"); -#else - if (backend == DNN_BACKEND_OPENCV && target == DNN_TARGET_OPENCL_FP16) - applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_NGRAPH, CV_TEST_TAG_DNN_SKIP_IE_NN_BUILDER, - CV_TEST_TAG_DNN_SKIP_OPENCL, CV_TEST_TAG_DNN_SKIP_OPENCL_FP16); - testLayerUsingCaffeModels("correlation", false, false, 0.0, 0.0, 2); -#endif -} - -TEST_P(Test_Caffe_layers, Convolution2Inputs) -{ - testLayerUsingCaffeModels("conv_2_inps", true, false, 0.0, 0.0, 2); -} - -TEST_P(Test_Caffe_layers, ROIPooling_Accuracy) -{ - Net net = readNetFromCaffe(_tf("net_roi_pooling.prototxt")); - ASSERT_FALSE(net.empty()); - - Mat inp = blobFromNPY(_tf("net_roi_pooling.input.npy")); - Mat rois = blobFromNPY(_tf("net_roi_pooling.rois.npy")); - Mat ref = blobFromNPY(_tf("net_roi_pooling.npy")); - - checkBackend(&inp, &ref); - - net.setPreferableBackend(backend); - net.setPreferableTarget(target); - - net.setInput(inp, "input"); - net.setInput(rois, "rois"); - - Mat out = net.forward(); - - double l1 = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 1e-3 : 1e-5; - double lInf = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 1e-3 : 1e-4; - if (target == DNN_TARGET_CUDA_FP16) - { - l1 = 2e-4; - lInf = 9e-4; - } - normAssert(out, ref, "", l1, lInf); -} - -TEST_P(Test_Caffe_layers, FasterRCNN_Proposal) -{ - if (backend == DNN_BACKEND_OPENCV && target == DNN_TARGET_OPENCL_FP16) - applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16); - if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019) - applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_NN_BUILDER); - if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) - applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_NGRAPH); - if(backend == DNN_BACKEND_CUDA) - applyTestTag(CV_TEST_TAG_DNN_SKIP_CUDA); /* Proposal layer is unsupported */ - - Net net = readNetFromCaffe(_tf("net_faster_rcnn_proposal.prototxt")); - - Mat scores = blobFromNPY(_tf("net_faster_rcnn_proposal.scores.npy")); - Mat deltas = blobFromNPY(_tf("net_faster_rcnn_proposal.deltas.npy")); - Mat imInfo = (Mat_(1, 3) << 600, 800, 1.6f); - - net.setInput(scores, "rpn_cls_prob_reshape"); - net.setInput(deltas, "rpn_bbox_pred"); - net.setInput(imInfo, "im_info"); - - std::vector outs; - net.setPreferableBackend(backend); - net.setPreferableTarget(target); - net.forward(outs); - - for (int i = 0; i < 2; ++i) - { - Mat ref = blobFromNPY(_tf(i == 0 ? "net_faster_rcnn_proposal.out_rois.npy" : - "net_faster_rcnn_proposal.out_scores.npy")); - const int numDets = ref.size[0]; - EXPECT_LE(numDets, outs[i].size[0]); - normAssert(outs[i].rowRange(0, numDets), ref); - - if (numDets < outs[i].size[0]) - { - EXPECT_EQ(countNonZero(outs[i].rowRange(numDets, outs[i].size[0])), 0); - } - } -} - typedef testing::TestWithParam > Scale_untrainable; TEST_P(Scale_untrainable, Accuracy) { @@ -1072,6 +595,8 @@ INSTANTIATE_TEST_CASE_P(Layer_Test, Crop, Combine( /*offset value*/ Values(3, 4) )); +class Test_Caffe_layers : public DNNTestLayer {}; + // Check that by default average pooling layer should not count zero padded values // into the normalization area. TEST_P(Test_Caffe_layers, Average_pooling_kernel_area) @@ -1099,32 +624,6 @@ TEST_P(Test_Caffe_layers, Average_pooling_kernel_area) normAssert(out, blobFromImage(ref)); } -TEST_P(Test_Caffe_layers, PriorBox_repeated) -{ - Net net = readNet(_tf("prior_box.prototxt")); - int inp_size[] = {1, 3, 10, 10}; - int shape_size[] = {1, 2, 3, 4}; - Mat inp(4, inp_size, CV_32F); - randu(inp, -1.0f, 1.0f); - Mat shape(4, shape_size, CV_32F); - randu(shape, -1.0f, 1.0f); - net.setInput(inp, "data"); - net.setInput(shape, "shape"); - net.setPreferableBackend(backend); - net.setPreferableTarget(target); - Mat out = net.forward(); - Mat ref = blobFromNPY(_tf("priorbox_output.npy")); - - double l1 = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 1e-3 : 1e-5; - double lInf = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 1e-3 : 1e-4; - if (target == DNN_TARGET_CUDA_FP16) - { - l1 = 7e-5; - lInf = 0.0005; - } - normAssert(out, ref, "", l1, lInf); -} - // Test PriorBoxLayer in case of no aspect ratios (just squared proposals). TEST_P(Test_Caffe_layers, PriorBox_squares) { @@ -1289,46 +788,6 @@ INSTANTIATE_TEST_CASE_P(/**/, Layer_Test_DWconv_Prelu, Combine(Values(3, 6), Val // ./ModelOptimizer -w /path/to/caffemodel -d /path/to/prototxt \ // -p FP32 -i -b ${batch_size} -o /path/to/output/folder typedef testing::TestWithParam > Layer_Test_Convolution_DLDT; -TEST_P(Layer_Test_Convolution_DLDT, Accuracy) -{ - const Backend backendId = get<0>(GetParam()); - const Target targetId = get<1>(GetParam()); - - if (backendId == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 && targetId == DNN_TARGET_MYRIAD) - applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD, CV_TEST_TAG_DNN_SKIP_IE_NN_BUILDER); - - if (backendId != DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 && backendId != DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) - throw SkipTestException("No support for async forward"); - - ASSERT_EQ(DNN_BACKEND_INFERENCE_ENGINE_NGRAPH, backendId); - - Net netDefault = readNet(_tf("layer_convolution.caffemodel"), _tf("layer_convolution.prototxt")); - Net net = readNet(_tf("layer_convolution.xml"), _tf("layer_convolution.bin")); - - Mat inp = blobFromNPY(_tf("blob.npy")); - - netDefault.setInput(inp); - netDefault.setPreferableBackend(DNN_BACKEND_OPENCV); - Mat outDefault = netDefault.forward(); - - net.setInput(inp); - net.setPreferableBackend(backendId); - net.setPreferableTarget(targetId); - - Mat out = net.forward(); - - double l1 = (targetId == DNN_TARGET_OPENCL_FP16 || targetId == DNN_TARGET_MYRIAD) ? 1.5e-3 : 1e-5; - double lInf = (targetId == DNN_TARGET_OPENCL_FP16 || targetId == DNN_TARGET_MYRIAD) ? 1.8e-2 : 1e-4; - normAssert(outDefault, out, "", l1, lInf); - - std::vector outLayers = net.getUnconnectedOutLayers(); - ASSERT_EQ(net.getLayer(outLayers[0])->name, "output"); - if (backendId == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019) - ASSERT_EQ(net.getLayer(outLayers[0])->type, "Convolution"); - else - ASSERT_EQ(net.getLayer(outLayers[0])->type, "Result"); -} - TEST_P(Layer_Test_Convolution_DLDT, multithreading) { const Backend backendId = get<0>(GetParam()); @@ -1640,43 +1099,6 @@ private: int outWidth, outHeight, zoomFactor; }; -// BUG: https://github.com/opencv/opencv/issues/26194 -// After unregistration of the custom 'Interp' the model uses the standard Resize layer. -// According to the graph, the model must produce 2 x 3 x 18 x 16 tensor with Resize layer, -// but the result is compared with 2 x 3 x 17 x 15 tensor, just like the custom 'Interp' layer produced, -// so we get the test failure. It looks like the test needs to be fixed. -TEST_P(Test_Caffe_layers, DISABLED_Interp) -{ -#ifdef OPENCV_DNN_EXTERNAL_PROTOBUF - throw SkipTestException("Requires patched protobuf"); -#else -#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2021030000) - if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target == DNN_TARGET_MYRIAD) - applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD, CV_TEST_TAG_DNN_SKIP_IE_NGRAPH); // exception -#endif - - if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 && target == DNN_TARGET_MYRIAD) - applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD); - - // Test a custom layer. - CV_DNN_REGISTER_LAYER_CLASS(Interp, CustomInterpLayer); - try - { - testLayerUsingCaffeModels("layer_interp", false, false); - } - catch (...) - { - LayerFactory::unregisterLayer("Interp"); - throw; - } - LayerFactory::unregisterLayer("Interp"); - - // Test an implemented layer. - - testLayerUsingCaffeModels("layer_interp", false, false); -#endif -} - INSTANTIATE_TEST_CASE_P(/*nothing*/, Test_Caffe_layers, dnnBackendsAndTargets()); TEST(Layer_Test_PoolingIndices, Accuracy) diff --git a/modules/dnn/test/test_misc.cpp b/modules/dnn/test/test_misc.cpp index 45072cbd90..19fd982f97 100644 --- a/modules/dnn/test/test_misc.cpp +++ b/modules/dnn/test/test_misc.cpp @@ -245,8 +245,7 @@ TEST(blobFromImagesWithParams_4ch, multi_image) TEST(readNet, Regression) { - Net net = readNet(findDataFile("dnn/squeezenet_v1.1.prototxt"), - findDataFile("dnn/squeezenet_v1.1.caffemodel", false)); + Net net = readNet(findDataFile("dnn/onnx/models/squeezenet.onnx", false)); EXPECT_FALSE(net.empty()); net = readNet(findDataFile("dnn/ssd_mobilenet_v1_coco.pbtxt"), findDataFile("dnn/ssd_mobilenet_v1_coco.pb", false)); @@ -256,9 +255,7 @@ TEST(readNet, Regression) TEST(readNet, do_not_call_setInput) // https://github.com/opencv/opencv/issues/16618 { // 1. load network - const string proto = findDataFile("dnn/squeezenet_v1.1.prototxt"); - const string model = findDataFile("dnn/squeezenet_v1.1.caffemodel", false); - Net net = readNetFromCaffe(proto, model); + Net net = readNet(findDataFile("dnn/onnx/models/squeezenet.onnx", false)); // 2. mistake: no inputs are specified through .setInput() @@ -325,13 +322,7 @@ TEST_P(dump, Regression) { const int backend = get<0>(GetParam()); const int target = get<1>(GetParam()); - Net net = readNet(findDataFile("dnn/squeezenet_v1.1.prototxt"), - findDataFile("dnn/squeezenet_v1.1.caffemodel", false)); - - if (net.getMainGraph()) - ASSERT_EQ(net.getLayer(net.getLayerId("fire2/concat"))->inputs.size(), 2); - else - ASSERT_EQ(net.getLayerInputs(net.getLayerId("fire2/concat")).size(), 2); + Net net = readNet(findDataFile("dnn/onnx/models/squeezenet.onnx", false)); int size[] = {1, 3, 227, 227}; Mat input = cv::Mat::ones(4, size, CV_32F); @@ -579,20 +570,17 @@ INSTANTIATE_TEST_CASE_P(/**/, DeprecatedForward, dnnBackendsAndTargets()); TEST(Net, forwardAndRetrieve) { - std::string prototxt = - "input: \"data\"\n" - "layer {\n" - " name: \"testLayer\"\n" - " type: \"Slice\"\n" - " bottom: \"data\"\n" - " top: \"firstCopy\"\n" - " top: \"secondCopy\"\n" - " slice_param {\n" - " axis: 0\n" - " slice_point: 2\n" - " }\n" - "}"; - Net net = readNetFromCaffe(&prototxt[0], prototxt.size()); + LayerParams lpSlice; + lpSlice.name = "testLayer"; + lpSlice.type = "Slice"; + lpSlice.set("axis", 0); + Mat slicePoint = (Mat_(1, 1) << 2); + lpSlice.set("slice_point", DictValue::arrayInt((int*)slicePoint.data, 1)); + + Net net; + int sliceId = net.addLayer(lpSlice.name, lpSlice.type, lpSlice); + net.connect(0, 0, sliceId, 0); + net.setPreferableBackend(DNN_BACKEND_OPENCV); Mat inp(4, 5, CV_32F); diff --git a/modules/dnn/test/test_model.cpp b/modules/dnn/test/test_model.cpp index 9065b462c5..89e9b722d0 100644 --- a/modules/dnn/test/test_model.cpp +++ b/modules/dnn/test/test_model.cpp @@ -21,7 +21,7 @@ static std::string _tf(TString filename, bool required = true) class Test_Model : public DNNTestLayer { public: - void testDetectModel(const std::string& weights, const std::string& cfg, + void testDetectModel(const std::string& weights, const std::string& /*cfg*/, const std::string& imgPath, const std::vector& refClassIds, const std::vector& refConfidences, const std::vector& refBoxes, @@ -29,35 +29,67 @@ public: double confThreshold = 0.24, double nmsThreshold = 0.0, const Size& size = {-1, -1}, Scalar mean = Scalar(), double scale = 1.0, bool swapRB = false, bool crop = false, - bool nmsAcrossClasses = false) + const std::vector& outNames = {"boxes", "scores", "class_idx"}) { checkBackend(); + CV_Assert(outNames.size() == 3); Mat frame = imread(imgPath); - DetectionModel model(weights, cfg); + Net net = readNet(weights); - model.setInputSize(size).setInputMean(mean).setInputScale(scale) - .setInputSwapRB(swapRB).setInputCrop(crop); - - model.setPreferableBackend(backend); - model.setPreferableTarget(target); - - model.setNmsAcrossClasses(nmsAcrossClasses); + net.setPreferableBackend(backend); + net.setPreferableTarget(target); if (target == DNN_TARGET_CPU_FP16) - model.enableWinograd(false); + net.enableWinograd(false); - std::vector classIds; + net.setInput(blobFromImage(frame, scale, size, mean, swapRB, crop, CV_32F)); + + std::vector outs; + net.forward(outs, outNames); + ASSERT_EQ(outs.size(), 3u); + + Mat outBoxes = outs[0].reshape(1, (int)outs[0].total() / 4); + const Mat& outScores = outs[1]; + const Mat& outClsIds = outs[2]; + + std::vector boxes; std::vector confidences; - std::vector boxes; + std::vector classIds; + for (int i = 0; i < outBoxes.rows; ++i) + { + float confidence = outScores.at(i); + if (confidence < confThreshold) + continue; - model.detect(frame, classIds, confidences, boxes, confThreshold, nmsThreshold); - - std::vector boxesDouble(boxes.size()); - for (int i = 0; i < boxes.size(); i++) { - boxesDouble[i] = boxes[i]; + const float* box = outBoxes.ptr(i); + boxes.emplace_back(box[0] / size.width, box[1] / size.height, + (box[2] - box[0]) / size.width, (box[3] - box[1]) / size.height); + confidences.push_back(confidence); + classIds.push_back((int)outClsIds.at(i)); } - normAssertDetections(refClassIds, refConfidences, refBoxes, classIds, - confidences, boxesDouble, "", + + if (nmsThreshold > 0) + { + std::vector keep; + NMSBoxesBatched(boxes, confidences, classIds, + (float)confThreshold, (float)nmsThreshold, keep); + + std::vector nmsBoxes; + std::vector nmsConfidences; + std::vector nmsClassIds; + for (int idx : keep) + { + nmsBoxes.push_back(boxes[idx]); + nmsConfidences.push_back(confidences[idx]); + nmsClassIds.push_back(classIds[idx]); + } + boxes = std::move(nmsBoxes); + confidences = std::move(nmsConfidences); + classIds = std::move(nmsClassIds); + } + + normAssertDetections(refClassIds, refConfidences, refBoxes, + classIds, confidences, boxes, "", confThreshold, scoreDiff, iouDiff); } @@ -282,133 +314,66 @@ TEST_P(Test_Model, Classify) std::pair ref(652, 0.641789); std::string img_path = _tf("grace_hopper_227.png"); - std::string config_file = _tf("bvlc_alexnet.prototxt"); - std::string weights_file = _tf("bvlc_alexnet.caffemodel", false); + std::string weights_file = _tf("onnx/models/alexnet.onnx", false); Size size{227, 227}; float norm = 1e-4; - testClassifyModel(weights_file, config_file, img_path, ref, norm, size); + testClassifyModel(weights_file, "", img_path, ref, norm, size); } - -TEST_P(Test_Model, DetectionOutput) +TEST_P(Test_Model, YOLOv3) { - applyTestTag(CV_TEST_TAG_DEBUG_VERYLONG); - -#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2022010000) - // Check 'backward_compatible_check || in_out_elements_equal' failed at core/src/op/reshape.cpp:427: - // While validating node 'v1::Reshape bbox_pred_reshape (ave_bbox_pred_rois[0]:f32{1,8,1,1}, Constant_388[0]:i64{4}) -> (f32{?,?,?,?})' with friendly_name 'bbox_pred_reshape': - // Requested output shape {1,300,8,1} is incompatible with input shape {1, 8, 1, 1} - if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target == DNN_TARGET_MYRIAD) - applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD, CV_TEST_TAG_DNN_SKIP_IE_NGRAPH, CV_TEST_TAG_DNN_SKIP_IE_VERSION); -#elif defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2021040000) - // Exception: Function contains several inputs and outputs with one friendly name! (HETERO bug?) - if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target != DNN_TARGET_CPU) - applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_NGRAPH, CV_TEST_TAG_DNN_SKIP_IE_VERSION); + applyTestTag( + CV_TEST_TAG_LONG, + CV_TEST_TAG_MEMORY_2GB, + CV_TEST_TAG_DEBUG_VERYLONG + ); if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target == DNN_TARGET_MYRIAD) - applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD, CV_TEST_TAG_DNN_SKIP_IE_NGRAPH, CV_TEST_TAG_DNN_SKIP_IE_VERSION); -#elif defined(INF_ENGINE_RELEASE) - // FIXIT DNN_BACKEND_INFERENCE_ENGINE is misused - if (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_OPENCL_FP16) - applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16); + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD, CV_TEST_TAG_DNN_SKIP_IE_NGRAPH); - if (target == DNN_TARGET_MYRIAD) - applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD); -#endif + // The in-graph YOLO decode (Split) is not evaluable by the OpenVINO backend. + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_NGRAPH); - std::vector refClassIds = {7, 12}; - std::vector refConfidences = {0.991359f, 0.94786f}; - std::vector refBoxes = {Rect2d(491, 81, 212, 98), - Rect2d(132, 223, 207, 344)}; + std::string model_file = _tf("yolov3.onnx", false); + std::string img_path = _tf("dog416.png"); // Matches standard YOLO inference classes - std::string img_path = _tf("dog416.png"); - std::string weights_file = _tf("resnet50_rfcn_final.caffemodel", false); - std::string config_file = _tf("rfcn_pascal_voc_resnet50.prototxt"); + // Extracted from original flat ref_ array: batchId, classId, confidence, left, top, right, bottom + std::vector refClassIds = {7, 16, 1}; + std::vector refConfidences = {0.606292f, 0.55195f, 0.433444f}; - Scalar mean = Scalar(102.9801, 115.9465, 122.7717); - Size size{800, 600}; + // Rect2d requires (x, y, width, height) format + std::vector refBoxes = { + Rect2d(0.612037f, 0.149921f, 0.910763f - 0.612037f, 0.300503f - 0.149921f), // Class 7 + Rect2d(0.170690f, 0.356024f, 0.471459f - 0.170690f, 0.877178f - 0.356024f), // Class 16 + Rect2d(0.199235f, 0.301175f, 0.753253f - 0.199235f, 0.744156f - 0.301175f) // Class 1 + }; - double scoreDiff = default_l1, iouDiff = 1e-5; - float confThreshold = 0.8; - double nmsThreshold = 0.0; - if (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_CUDA_FP16 || target == DNN_TARGET_CPU_FP16) + // Setting precision tolerances + double scoreDiff = 8e-5, iouDiff = 3e-4; + if (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD || target == DNN_TARGET_CPU_FP16) { - if (backend == DNN_BACKEND_OPENCV) - scoreDiff = 4e-3; - else - scoreDiff = 2e-2; - iouDiff = 1.8e-1; - } -#if defined(INF_ENGINE_RELEASE) - if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) - { - scoreDiff = 0.05; - iouDiff = 0.08; - } -#endif - - testDetectModel(weights_file, config_file, img_path, refClassIds, refConfidences, refBoxes, - scoreDiff, iouDiff, confThreshold, nmsThreshold, size, mean); -} - - -TEST_P(Test_Model, DetectionMobilenetSSD) -{ - Mat ref = blobFromNPY(_tf("mobilenet_ssd_caffe_out.npy")); - ref = ref.reshape(1, ref.size[2]); - - std::string img_path = _tf("street.png"); - Mat frame = imread(img_path); - int frameWidth = frame.cols; - int frameHeight = frame.rows; - - std::vector refClassIds; - std::vector refConfidences; - std::vector refBoxes; - for (int i = 0; i < ref.rows; i++) - { - refClassIds.emplace_back(ref.at(i, 1)); - refConfidences.emplace_back(ref.at(i, 2)); - int left = ref.at(i, 3) * frameWidth; - int top = ref.at(i, 4) * frameHeight; - int right = ref.at(i, 5) * frameWidth; - int bottom = ref.at(i, 6) * frameHeight; - int width = right - left + 1; - int height = bottom - top + 1; - refBoxes.emplace_back(left, top, width, height); - } - - std::string weights_file = _tf("MobileNetSSD_deploy_19e3ec3.caffemodel", false); - std::string config_file = _tf("MobileNetSSD_deploy_19e3ec3.prototxt"); - - Scalar mean = Scalar(127.5, 127.5, 127.5); - double scale = 1.0 / 127.5; - Size size{300, 300}; - - double scoreDiff = 1e-5, iouDiff = 1e-5; - if (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_CPU_FP16) - { - scoreDiff = 1.7e-2; - iouDiff = 6.91e-2; - } - else if (target == DNN_TARGET_MYRIAD) - { - scoreDiff = 0.017; - if (getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_X) - iouDiff = 0.1; + scoreDiff = 0.006; + iouDiff = 0.042; } else if (target == DNN_TARGET_CUDA_FP16) { - scoreDiff = 0.0028; - iouDiff = 1e-2; + scoreDiff = 0.04; + iouDiff = 0.03; } - float confThreshold = FLT_MIN; - double nmsThreshold = 0.0; - testDetectModel(weights_file, config_file, img_path, refClassIds, refConfidences, refBoxes, - scoreDiff, iouDiff, confThreshold, nmsThreshold, size, mean, scale); + // Model parameters + double confThreshold = 0.24; + double nmsThreshold = 0.4; + Size size{640, 640}; + double scale = 1.0 / 255.0; // Standard image scaling for YOLO models + Scalar mean = Scalar(); + bool swapRB = true; // YOLO expects RGB + + testDetectModel(model_file, "", img_path, refClassIds, refConfidences, refBoxes, + scoreDiff, iouDiff, confThreshold, nmsThreshold, size, mean, scale, swapRB); } TEST_P(Test_Model, Keypoints_pose) @@ -478,44 +443,6 @@ TEST_P(Test_Model, Keypoints_face) testKeypointsModel(weights, "", inp, exp, norm, size, mean, scale, swapRB); } -TEST_P(Test_Model, Detection_normalized) -{ - std::string img_path = _tf("grace_hopper_227.png"); - std::vector refClassIds = {15}; - std::vector refConfidences = {0.999222f}; - std::vector refBoxes = {Rect2d(0, 4, 227, 222)}; - - std::string weights_file = _tf("MobileNetSSD_deploy_19e3ec3.caffemodel", false); - std::string config_file = _tf("MobileNetSSD_deploy_19e3ec3.prototxt"); - - Scalar mean = Scalar(127.5, 127.5, 127.5); - double scale = 1.0 / 127.5; - Size size{300, 300}; - - double scoreDiff = 1e-5, iouDiff = 1e-5; - float confThreshold = FLT_MIN; - double nmsThreshold = 0.0; - if (target == DNN_TARGET_CUDA) - { - scoreDiff = 3e-4; - iouDiff = 0.018; - } - if (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD || target == DNN_TARGET_CUDA_FP16 || target == DNN_TARGET_CPU_FP16) - { - scoreDiff = 5e-3; - iouDiff = 0.09; - } -#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_GE(2020040000) - if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target == DNN_TARGET_MYRIAD) - { - scoreDiff = 0.02; - iouDiff = 0.1f; - } -#endif - testDetectModel(weights_file, config_file, img_path, refClassIds, refConfidences, refBoxes, - scoreDiff, iouDiff, confThreshold, nmsThreshold, size, mean, scale); -} - TEST_P(Test_Model, Segmentation) { applyTestTag( diff --git a/modules/objdetect/src/barcode.cpp b/modules/objdetect/src/barcode.cpp index 1b963e4242..5b38e03956 100644 --- a/modules/objdetect/src/barcode.cpp +++ b/modules/objdetect/src/barcode.cpp @@ -348,11 +348,13 @@ BarcodeDetector::BarcodeDetector(const string &prototxt_path, const string &mode Ptr p_ = new BarcodeImpl(); p = p_; p_->sr = make_shared(); - if (!prototxt_path.empty() && !model_path.empty()) + // The Super Resolution model is now a single-file ONNX network; the legacy Caffe + // prototxt argument is retained for API compatibility but is no longer used. + CV_UNUSED(prototxt_path); + if (!model_path.empty()) { - CV_Assert(utils::fs::exists(prototxt_path)); CV_Assert(utils::fs::exists(model_path)); - int res = p_->sr->init(prototxt_path, model_path); + int res = p_->sr->init(model_path); CV_Assert(res == 0); p_->use_nn_sr = true; } diff --git a/modules/objdetect/src/barcode_decoder/common/super_scale.cpp b/modules/objdetect/src/barcode_decoder/common/super_scale.cpp index 4b7099a27d..33e1db02ea 100644 --- a/modules/objdetect/src/barcode_decoder/common/super_scale.cpp +++ b/modules/objdetect/src/barcode_decoder/common/super_scale.cpp @@ -18,9 +18,10 @@ namespace barcode { constexpr static float MAX_SCALE = 4.0f; -int SuperScale::init(const std::string &proto_path, const std::string &model_path) +int SuperScale::init(const std::string &model_path) { - srnet_ = dnn::readNetFromCaffe(proto_path, model_path); + // Super Resolution is loaded from a single-file ONNX model (e.g. dnn/wechat_2021-01/sr.onnx). + srnet_ = dnn::readNetFromONNX(model_path); net_loaded_ = true; return 0; } @@ -77,9 +78,8 @@ int SuperScale::superResolutionScale(const Mat &src, Mat &dst) #else // HAVE_OPENCV_DNN -int SuperScale::init(const std::string &proto_path, const std::string &model_path) +int SuperScale::init(const std::string &model_path) { - CV_UNUSED(proto_path); CV_UNUSED(model_path); return 0; } diff --git a/modules/objdetect/src/barcode_decoder/common/super_scale.hpp b/modules/objdetect/src/barcode_decoder/common/super_scale.hpp index 024ea86e54..37d7288a55 100644 --- a/modules/objdetect/src/barcode_decoder/common/super_scale.hpp +++ b/modules/objdetect/src/barcode_decoder/common/super_scale.hpp @@ -22,7 +22,7 @@ public: ~SuperScale() = default; - int init(const std::string &proto_path, const std::string &model_path); + int init(const std::string &model_path); void processImageScale(const Mat &src, Mat &dst, float scale, const bool &use_sr, int sr_max_size = 160); diff --git a/platforms/js/opencv_js.config.py b/platforms/js/opencv_js.config.py index ced62a19a0..8fb716054b 100644 --- a/platforms/js/opencv_js.config.py +++ b/platforms/js/opencv_js.config.py @@ -171,7 +171,7 @@ video = { } dnn = {'dnn_Net': ['setInput', 'forward', 'setPreferableBackend','getUnconnectedOutLayersNames'], - '': ['readNetFromCaffe', 'readNetFromTensorflow', + '': ['readNetFromTensorflow', 'readNetFromONNX', 'readNetFromTFLite', 'readNet', 'blobFromImage']} features = {'Feature2D': ['detect', 'compute', 'detectAndCompute', 'descriptorSize', 'descriptorType', 'defaultNorm', 'empty', 'getDefaultName'], diff --git a/samples/android/mobilenet-objdetect/CMakeLists.txt b/samples/android/mobilenet-objdetect/CMakeLists.txt index 85abfae6a7..da07d46b18 100644 --- a/samples/android/mobilenet-objdetect/CMakeLists.txt +++ b/samples/android/mobilenet-objdetect/CMakeLists.txt @@ -1,24 +1,20 @@ set(sample example-mobilenet-objdetect) -ocv_download(FILENAME "mobilenet_iter_73000.caffemodel" - HASH "bbcb3b6a0afe1ec89e1288096b5b8c66" - URL - "${OPENCV_MOBILENET_SSD_WEIGHTS_URL}" - "$ENV{OPENCV_MOBILENET_SSD_WEIGHTS_URL}" - "https://raw.githubusercontent.com/chuanqi305/MobileNet-SSD/97406996b1eee2d40eb0a00ae567cf41e23369f9/mobilenet_iter_73000.caffemodel" - DESTINATION_DIR "${CMAKE_CURRENT_LIST_DIR}/res/raw" - ID OPENCV_MOBILENET_SSD_WEIGHTS - STATUS res) +# Explicitly ensure resource directories exist to guarantee smooth resource compiling +file(MAKE_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/res/raw") -ocv_download(FILENAME "deploy.prototxt" - HASH "f1978dc4fe20c680e850ce99830c5945" - URL - "${OPENCV_MOBILENET_SSD_CONFIG_URL}" - "$ENV{OPENCV_MOBILENET_SSD_CONFIG_URL}" - "https://raw.githubusercontent.com/chuanqi305/MobileNet-SSD/97406996b1eee2d40eb0a00ae567cf41e23369f9/deploy.prototxt" - DESTINATION_DIR "${CMAKE_CURRENT_LIST_DIR}/res/raw" - ID OPENCV_MOBILENET_SSD_CONFIG - STATUS res) +# The Caffe MobileNet-SSD (mobilenet_iter_73000.caffemodel + deploy.prototxt) has been +# replaced by a single-file ONNX model. The sample now loads "res/raw/mobilenet.onnx". +if((DEFINED ENV{OPENCV_MOBILENET_SSD_ONNX_URL} OR DEFINED OPENCV_MOBILENET_SSD_ONNX_URL) AND OPENCV_MOBILENET_SSD_ONNX_HASH) + ocv_download(FILENAME "mobilenet.onnx" + HASH "${OPENCV_MOBILENET_SSD_ONNX_HASH}" + URL + "${OPENCV_MOBILENET_SSD_ONNX_URL}" + "$ENV{OPENCV_MOBILENET_SSD_ONNX_URL}" + DESTINATION_DIR "${CMAKE_CURRENT_SOURCE_DIR}/res/raw" + ID OPENCV_MOBILENET_SSD_ONNX + STATUS res) +endif() add_android_project(${sample} "${CMAKE_CURRENT_SOURCE_DIR}" LIBRARY_DEPS "${OPENCV_ANDROID_LIB_DIR}" SDK_TARGET 11 "${ANDROID_SDK_TARGET}") if(TARGET ${sample}) diff --git a/samples/android/mobilenet-objdetect/src/org/opencv/samples/opencv_mobilenet/MainActivity.java b/samples/android/mobilenet-objdetect/src/org/opencv/samples/opencv_mobilenet/MainActivity.java index 6d417a5214..77e55b5cea 100644 --- a/samples/android/mobilenet-objdetect/src/org/opencv/samples/opencv_mobilenet/MainActivity.java +++ b/samples/android/mobilenet-objdetect/src/org/opencv/samples/opencv_mobilenet/MainActivity.java @@ -54,15 +54,19 @@ public class MainActivity extends CameraActivity implements CvCameraViewListener } //! [init_model_from_memory] - mModelBuffer = loadFileFromResource(R.raw.mobilenet_iter_73000); - mConfigBuffer = loadFileFromResource(R.raw.deploy); - if (mModelBuffer == null || mConfigBuffer == null) { - Log.e(TAG, "Failed to load model from resources"); - } else - Log.i(TAG, "Model files loaded successfully"); + // Load a single-file MobileNet-SSD ONNX model from res/raw/mobilenet.onnx. + // The resource is resolved by name at runtime so the sample still builds when the + // (optionally downloaded) model is absent; object detection is simply disabled then. + int modelResId = getResources().getIdentifier("mobilenet", "raw", getPackageName()); + if (modelResId != 0) + mModelBuffer = loadFileFromResource(modelResId); - net = Dnn.readNet("caffe", mModelBuffer, mConfigBuffer); - Log.i(TAG, "Network loaded successfully"); + if (mModelBuffer == null) { + Log.e(TAG, "MobileNet ONNX model (res/raw/mobilenet.onnx) not found - object detection disabled"); + } else { + net = Dnn.readNetFromONNX(mModelBuffer); + Log.i(TAG, "Network loaded successfully"); + } //! [init_model_from_memory] setContentView(R.layout.activity_main); @@ -91,8 +95,8 @@ public class MainActivity extends CameraActivity implements CvCameraViewListener if (mOpenCvCameraView != null) mOpenCvCameraView.disableView(); - mModelBuffer.release(); - mConfigBuffer.release(); + if (mModelBuffer != null) + mModelBuffer.release(); } // Load a network. @@ -107,7 +111,10 @@ public class MainActivity extends CameraActivity implements CvCameraViewListener final double MEAN_VAL = 127.5; final double THRESHOLD = 0.2; - // Get a new frame + if (net == null) { + return inputFrame.rgba(); + } + Log.d(TAG, "handle new frame!"); Mat frame = inputFrame.rgba(); Imgproc.cvtColor(frame, frame, Imgproc.COLOR_RGBA2RGB); @@ -188,7 +195,6 @@ public class MainActivity extends CameraActivity implements CvCameraViewListener "motorbike", "person", "pottedplant", "sheep", "sofa", "train", "tvmonitor"}; - private MatOfByte mConfigBuffer; private MatOfByte mModelBuffer; private Net net; private CameraBridgeViewBase mOpenCvCameraView; diff --git a/samples/dnn/optical_flow.py b/samples/dnn/optical_flow.py index efff6b7068..d299f28030 100644 --- a/samples/dnn/optical_flow.py +++ b/samples/dnn/optical_flow.py @@ -1,19 +1,14 @@ #!/usr/bin/env python ''' -This sample using FlowNet v2 and RAFT model to calculate optical flow. - -FlowNet v2 Original Paper: https://arxiv.org/abs/1612.01925. -FlowNet v2 Repo: https://github.com/lmb-freiburg/flownet2. - -Download the converted .caffemodel model from https://drive.google.com/open?id=16qvE9VNmU39NttpZwZs81Ga8VYQJDaWZ -and .prototxt from https://drive.google.com/file/d/1RyNIUsan1ZOh2hpYIH36A-jofAvJlT6a/view?usp=sharing. -Otherwise download original model from https://lmb.informatik.uni-freiburg.de/resources/binaries/flownet2/flownet2-models.tar.gz, -convert .h5 model to .caffemodel and modify original .prototxt using .prototxt from link above. +This sample uses the RAFT model to calculate optical flow. RAFT Original Paper: https://arxiv.org/pdf/2003.12039.pdf RAFT Repo: https://github.com/princeton-vl/RAFT Download the .onnx model from here https://github.com/opencv/opencv_zoo/raw/281d232cd99cd920853106d853c440edd35eb442/models/optical_flow_estimation_raft/optical_flow_estimation_raft_2023aug.onnx. + +Note: the legacy FlowNet v2 Caffe pipeline (--proto/.caffemodel) has been removed together +with the Caffe importer. Please provide a single ONNX model. ''' import argparse @@ -25,9 +20,8 @@ import cv2 as cv class OpticalFlow(object): def __init__(self, model, height, width, proto=""): if proto: - self.net = cv.dnn.readNetFromCaffe(proto, model) - else: - self.net = cv.dnn.readNet(model) + raise cv.error("Caffe support has been removed. Please provide a single ONNX model path (e.g. RAFT).") + self.net = cv.dnn.readNet(model) self.net.setPreferableBackend(cv.dnn.DNN_BACKEND_OPENCV) self.height = height self.width = width @@ -75,41 +69,18 @@ if __name__ == '__main__': parser.add_argument('-input', '-i', required=True, help='Path to input video file. Skip this argument to capture frames from a camera.') parser.add_argument('--height', default=320, type=int, help='Input height') parser.add_argument('--width', default=448, type=int, help='Input width') - parser.add_argument('--proto', '-p', default='', help='Path to prototxt.') - parser.add_argument('--model', '-m', required=True, help='Path to model.') + parser.add_argument('--model', '-m', required=True, help='Path to a single ONNX model (e.g. RAFT).') args, _ = parser.parse_known_args() if not os.path.isfile(args.model): raise OSError("Model does not exist") - if args.proto and not os.path.isfile(args.proto): - raise OSError("Prototxt does not exist") winName = 'Calculation optical flow in OpenCV' cv.namedWindow(winName, cv.WINDOW_NORMAL) cap = cv.VideoCapture(args.input if args.input else 0) hasFrame, first_frame = cap.read() - if args.proto: - divisor = 64. - var = {} - var['ADAPTED_WIDTH'] = int(np.ceil(args.width/divisor) * divisor) - var['ADAPTED_HEIGHT'] = int(np.ceil(args.height/divisor) * divisor) - var['SCALE_WIDTH'] = args.width / float(var['ADAPTED_WIDTH']) - var['SCALE_HEIGHT'] = args.height / float(var['ADAPTED_HEIGHT']) - - config = '' - proto = open(args.proto).readlines() - for line in proto: - for key, value in var.items(): - tag = "$%s$" % key - line = line.replace(tag, str(value)) - config += line - - caffemodel = open(args.model, 'rb').read() - - opt_flow = OpticalFlow(caffemodel, var['ADAPTED_HEIGHT'], var['ADAPTED_WIDTH'], bytearray(config.encode())) - else: - opt_flow = OpticalFlow(args.model, 360, 480) + opt_flow = OpticalFlow(args.model, 360, 480) while cv.waitKey(1) < 0: hasFrame, second_frame = cap.read()