From f8fb3a7f55f37c121bd171451e375dc9f0df9a60 Mon Sep 17 00:00:00 2001 From: Gursimar Singh Date: Fri, 6 Sep 2024 15:17:04 +0530 Subject: [PATCH] Merge pull request #25515 from gursimarsingh:improved_edge_detection_sample #25006 #25314 This pull request removes hed_pretrained caffe model to the SOTA dexined onnx model for edge detection. Usage of conventional methods like canny has also been added The obsolete cpp and python sample has been removed TODO: - [ ] Remove temporary hack for quantized models. Refer issue https://github.com/opencv/opencv_zoo/issues/273 ### Pull Request Readiness Checklist See details at https://github.com/opencv/opencv/wiki/How_to_contribute#making-a-good-pull-request - [x] I agree to contribute to the project under Apache 2 License. - [x] To the best of my knowledge, the proposed patch is not based on a code under GPL or another license that is incompatible with OpenCV - [x] The PR is proposed to the proper branch - [x] There is a reference to the original bug report and related work - [ ] There is accuracy test, performance test and test data in opencv_extra repository, if applicable Patch to opencv_extra has the same branch name. - [x] The feature is well documented and sample code can be built with the project CMake --- .../dnn_custom_layers/dnn_custom_layers.md | 4 +- .../dnn/src/int8layers/quantization_utils.cpp | 2 +- modules/imgproc/include/opencv2/imgproc.hpp | 2 +- samples/cpp/{ => snippets}/edge.cpp | 2 +- samples/dnn/common.hpp | 3 + samples/dnn/common.py | 62 +++-- samples/dnn/custom_layer.py | 31 +++ samples/dnn/edge_detection.cpp | 242 ++++++++++++++++++ samples/dnn/edge_detection.py | 216 ++++++++++++---- samples/dnn/models.yml | 16 ++ 10 files changed, 493 insertions(+), 87 deletions(-) rename samples/cpp/{ => snippets}/edge.cpp (99%) create mode 100644 samples/dnn/custom_layer.py create mode 100644 samples/dnn/edge_detection.cpp diff --git a/doc/tutorials/dnn/dnn_custom_layers/dnn_custom_layers.md b/doc/tutorials/dnn/dnn_custom_layers/dnn_custom_layers.md index 451005413e..319f4ab196 100644 --- a/doc/tutorials/dnn/dnn_custom_layers/dnn_custom_layers.md +++ b/doc/tutorials/dnn/dnn_custom_layers/dnn_custom_layers.md @@ -218,13 +218,13 @@ a centric one. - Create a class with `getMemoryShapes` and `forward` methods -@snippet dnn/edge_detection.py CropLayer +@snippet dnn/custom_layer.py CropLayer @note Both methods should return lists. - Register a new layer. -@snippet dnn/edge_detection.py Register +@snippet dnn/custom_layer.py Register That's it! We have replaced an implemented OpenCV's layer to a custom one. You may find a full script in the [source code](https://github.com/opencv/opencv/tree/5.x/samples/dnn/edge_detection.py). diff --git a/modules/dnn/src/int8layers/quantization_utils.cpp b/modules/dnn/src/int8layers/quantization_utils.cpp index df5b05a737..d9c64150f2 100644 --- a/modules/dnn/src/int8layers/quantization_utils.cpp +++ b/modules/dnn/src/int8layers/quantization_utils.cpp @@ -41,7 +41,7 @@ static void broadcast1D2TargetMat(Mat& data, const MatShape& targetShape, int ax static void block_repeat(InputArray src, const MatShape& srcShape, int axis, int repetitions, OutputArray dst) { CV_Assert(src.getObj() != dst.getObj()); - CV_Check(axis, axis >= 0 && axis < src.dims(), "Axis out of range"); + CV_Check(axis, axis >= 0 && (axis < src.dims() || (src.dims()==1 && axis==1)), "axis is out of range"); // (src.dims()==1 && axis==1) has been added as a temporary fix for quantized models. Refer issue https://github.com/opencv/opencv_zoo/issues/273 CV_CheckGT(repetitions, 1, "More than one repetition expected"); Mat src_mat = src.getMat(); diff --git a/modules/imgproc/include/opencv2/imgproc.hpp b/modules/imgproc/include/opencv2/imgproc.hpp index b9f3cb10de..5d73e3f578 100644 --- a/modules/imgproc/include/opencv2/imgproc.hpp +++ b/modules/imgproc/include/opencv2/imgproc.hpp @@ -1885,7 +1885,7 @@ CV_EXPORTS_W void Laplacian( InputArray src, OutputArray dst, int ddepth, //! @addtogroup imgproc_feature //! @{ -/** @example samples/cpp/edge.cpp +/** @example samples/cpp/snippets/edge.cpp This program demonstrates usage of the Canny edge detector Check @ref tutorial_canny_detector "the corresponding tutorial" for more details diff --git a/samples/cpp/edge.cpp b/samples/cpp/snippets/edge.cpp similarity index 99% rename from samples/cpp/edge.cpp rename to samples/cpp/snippets/edge.cpp index 339baf2aeb..fc5e8529e3 100644 --- a/samples/cpp/edge.cpp +++ b/samples/cpp/snippets/edge.cpp @@ -82,4 +82,4 @@ int main( int argc, const char** argv ) waitKey(0); return 0; -} +} \ No newline at end of file diff --git a/samples/dnn/common.hpp b/samples/dnn/common.hpp index 43f7472e4f..ec31e05d29 100644 --- a/samples/dnn/common.hpp +++ b/samples/dnn/common.hpp @@ -104,6 +104,9 @@ std::string findModel(const std::string& filename, const std::string& sha1) std::string modelPath = utils::fs::join(getenv("OPENCV_DOWNLOAD_CACHE_DIR"), utils::fs::join(sha1, filename)); if (utils::fs::exists(modelPath)) return modelPath; + modelPath = utils::fs::join(getenv("OPENCV_DOWNLOAD_CACHE_DIR"),filename); + if (utils::fs::exists(modelPath)) + return modelPath; } std::cout << "File " + filename + " not found! " diff --git a/samples/dnn/common.py b/samples/dnn/common.py index 78391a1d8e..8634099cbf 100644 --- a/samples/dnn/common.py +++ b/samples/dnn/common.py @@ -3,12 +3,14 @@ import os import cv2 as cv -def add_argument(zoo, parser, name, help, required=False, default=None, type=None, action=None, nargs=None): - if len(sys.argv) <= 1: +def add_argument(zoo, parser, name, help, required=False, default=None, type=None, action=None, nargs=None, alias=None): + if alias is not None: + modelName = alias + elif len(sys.argv) > 1: + modelName = sys.argv[1] + else: return - modelName = sys.argv[1] - if os.path.isfile(zoo): fs = cv.FileStorage(zoo, cv.FILE_STORAGE_READ) node = fs.getNode(modelName) @@ -50,7 +52,7 @@ def add_argument(zoo, parser, name, help, required=False, default=None, type=Non action=action, nargs=nargs, type=type) -def add_preproc_args(zoo, parser, sample): +def add_preproc_args(zoo, parser, sample, alias=None): aliases = [] if os.path.isfile(zoo): fs = cv.FileStorage(zoo, cv.FILE_STORAGE_READ) @@ -62,34 +64,34 @@ def add_preproc_args(zoo, parser, sample): parser.add_argument('alias', nargs='?', choices=aliases, help='An alias name of model to extract preprocessing parameters from models.yml file.') - add_argument(zoo, parser, 'model', required=True, + add_argument(zoo, parser, 'model', help='Path to a binary file of model contains trained weights. ' 'It could be a file with extensions .caffemodel (Caffe), ' - '.pb (TensorFlow), .weights (Darknet), .bin (OpenVINO)') + '.pb (TensorFlow), .weights (Darknet), .bin (OpenVINO)', alias=alias) add_argument(zoo, parser, 'config', help='Path to a text file of model contains network configuration. ' - 'It could be a file with extensions .prototxt (Caffe), .pbtxt or .config (TensorFlow), .cfg (Darknet), .xml (OpenVINO)') + 'It could be a file with extensions .prototxt (Caffe), .pbtxt or .config (TensorFlow), .cfg (Darknet), .xml (OpenVINO)', alias=alias) add_argument(zoo, parser, 'mean', nargs='+', type=float, default=[0, 0, 0], help='Preprocess input image by subtracting mean values. ' - 'Mean values should be in BGR order.') + 'Mean values should be in BGR order.', alias=alias) add_argument(zoo, parser, 'std', nargs='+', type=float, default=[0, 0, 0], - help='Preprocess input image by dividing on a standard deviation.') + help='Preprocess input image by dividing on a standard deviation.', alias=alias) add_argument(zoo, parser, 'scale', type=float, default=1.0, - help='Preprocess input image by multiplying on a scale factor.') + help='Preprocess input image by multiplying on a scale factor.', alias=alias) add_argument(zoo, parser, 'width', type=int, - help='Preprocess input image by resizing to a specific width.') + help='Preprocess input image by resizing to a specific width.', alias=alias) add_argument(zoo, parser, 'height', type=int, - help='Preprocess input image by resizing to a specific height.') + help='Preprocess input image by resizing to a specific height.', alias=alias) add_argument(zoo, parser, 'rgb', action='store_true', - help='Indicate that model works with RGB input images instead BGR ones.') + help='Indicate that model works with RGB input images instead BGR ones.', alias=alias) add_argument(zoo, parser, 'labels', - help='Optional path to a text file with names of labels to label detected objects.') + help='Optional path to a text file with names of labels to label detected objects.', alias=alias) add_argument(zoo, parser, 'postprocessing', type=str, - help='Post-processing kind depends on model topology.') + help='Post-processing kind depends on model topology.', alias=alias) add_argument(zoo, parser, 'background_label_id', type=int, default=-1, - help='An index of background class in predictions. If not negative, exclude such class from list of classes.') + help='An index of background class in predictions. If not negative, exclude such class from list of classes.', alias=alias) add_argument(zoo, parser, 'sha1', type=str, - help='Optional path to hashsum of downloaded model to be loaded from models.yml') + help='Optional path to hashsum of downloaded model to be loaded from models.yml', alias=alias) def findModel(filename, sha1): if filename: @@ -107,10 +109,12 @@ def findModel(filename, sha1): if os.path.exists(os.path.join(os.environ['OPENCV_DOWNLOAD_CACHE_DIR'], sha1, filename)): return os.path.join(os.environ['OPENCV_DOWNLOAD_CACHE_DIR'], sha1, filename) - print('File ' + filename + ' not found! Please specify a path to ' - 'model download directory in OPENCV_DOWNLOAD_CACHE_DIR ' - 'environment variable or pass a full path to ' + filename) - exit(0) + if os.path.exists(os.path.join(os.environ['OPENCV_DOWNLOAD_CACHE_DIR'], filename)): + return os.path.join(os.environ['OPENCV_DOWNLOAD_CACHE_DIR'], filename) + + raise FileNotFoundError('File ' + filename + ' not found! Please specify a path to ' + 'model download directory in OPENCV_DOWNLOAD_CACHE_DIR ' + 'environment variable or pass a full path to ' + filename) def findFile(filename): if filename: @@ -137,12 +141,14 @@ def findFile(filename): except KeyError: pass - print('File ' + filename + ' not found! Please specify the path to ' - '/opencv/samples/data in the OPENCV_SAMPLES_DATA_PATH environment variable, ' - 'or specify the path to opencv_extra/testdata in the OPENCV_DNN_TEST_DATA_PATH environment variable, ' - 'or specify the path to the model download cache directory in the OPENCV_DOWNLOAD_CACHE_DIR environment variable, ' - 'or pass the full path to ' + filename + '.') - exit(0) + raise FileNotFoundError( + 'File ' + filename + ' not found! Please specify the path to ' + '/opencv/samples/data in the OPENCV_SAMPLES_DATA_PATH environment variable, ' + 'or specify the path to opencv_extra/testdata in the OPENCV_DNN_TEST_DATA_PATH environment variable, ' + 'or specify the path to the model download cache directory in the OPENCV_DOWNLOAD_CACHE_DIR environment variable, ' + 'or pass the full path to ' + filename + '.' + ) + def get_backend_id(backend_name): backend_ids = { diff --git a/samples/dnn/custom_layer.py b/samples/dnn/custom_layer.py new file mode 100644 index 0000000000..d7163876fb --- /dev/null +++ b/samples/dnn/custom_layer.py @@ -0,0 +1,31 @@ +import cv2 as cv + +#! [CropLayer] +class CropLayer(object): + def __init__(self, params, blobs): + self.xstart = 0 + self.xend = 0 + self.ystart = 0 + self.yend = 0 + + # Our layer receives two inputs. We need to crop the first input blob + # to match a shape of the second one (keeping batch size and number of channels) + def getMemoryShapes(self, inputs): + inputShape, targetShape = inputs[0], inputs[1] + batchSize, numChannels = inputShape[0], inputShape[1] + height, width = targetShape[2], targetShape[3] + + self.ystart = (inputShape[2] - targetShape[2]) // 2 + self.xstart = (inputShape[3] - targetShape[3]) // 2 + self.yend = self.ystart + height + self.xend = self.xstart + width + + return [[batchSize, numChannels, height, width]] + + def forward(self, inputs): + return [inputs[0][:,:,self.ystart:self.yend,self.xstart:self.xend]] +#! [CropLayer] + +#! [Register] +cv.dnn_registerLayer('Crop', CropLayer) +#! [Register] \ No newline at end of file diff --git a/samples/dnn/edge_detection.cpp b/samples/dnn/edge_detection.cpp new file mode 100644 index 0000000000..2cd84a0b00 --- /dev/null +++ b/samples/dnn/edge_detection.cpp @@ -0,0 +1,242 @@ +#include +#include +#include +#include +#include +#include +#include + +#include "common.hpp" +// Define namespace to simplify code +using namespace cv; +using namespace cv::dnn; +using namespace std; + +int threshold1 = 0; +int threshold2 = 50; +int blurAmount = 5; + +// Function to apply sigmoid activation +static void sigmoid(Mat& input) { + exp(-input, input); // e^-input + input = 1.0 / (1.0 + input); // 1 / (1 + e^-input) +} + +static void applyCanny(const Mat& image, Mat& result) { + Mat gray; + cvtColor(image, gray, COLOR_BGR2GRAY); + Canny(gray, result, threshold1, threshold2); +} + +// Load Model +static void loadModel(const string modelPath, String backend, String target, Net &net){ + net = readNetFromONNX(modelPath); + net.setPreferableBackend(getBackendID(backend)); + net.setPreferableTarget(getTargetID(target)); +} + +static void setupCannyWindow(){ + destroyWindow("Output"); + namedWindow("Output", WINDOW_AUTOSIZE); + moveWindow("Output", 200, 50); + + createTrackbar("thrs1", "Output", &threshold1, 255, nullptr); + createTrackbar("thrs2", "Output", &threshold2, 255, nullptr); + createTrackbar("blur", "Output", &blurAmount, 20, nullptr); +} + +// Function to process the neural network output to generate edge maps +static pair postProcess(const vector& output, int height, int width) { + vector preds; + preds.reserve(output.size()); + for (const Mat &p : output) { + Mat img; + // Correctly handle 4D tensor assuming it's always in the format [1, 1, height, width] + Mat processed; + if (p.dims == 4 && p.size[0] == 1 && p.size[1] == 1) { + // Use only the spatial dimensions + processed = p.reshape(0, {p.size[2], p.size[3]}); + } else { + processed = p.clone(); + } + sigmoid(processed); + normalize(processed, img, 0, 255, NORM_MINMAX, CV_8U); + resize(img, img, Size(width, height)); // Resize to the original size + preds.push_back(img); + } + Mat fuse = preds.back(); // Last element as the fused result + // Calculate the average of the predictions + Mat ave = Mat::zeros(height, width, CV_32F); + for (Mat &pred : preds) { + Mat temp; + pred.convertTo(temp, CV_32F); + ave += temp; + } + ave /= static_cast(preds.size()); + ave.convertTo(ave, CV_8U); + return {fuse, ave}; // Return both fused and average edge maps +} + +static void applyDexined(Net &net, const Mat &image, Mat &result) { + int originalWidth = image.cols; + int originalHeight = image.rows; + vector outputs; + net.forward(outputs); + pair res = postProcess(outputs, originalHeight, originalWidth); + result = res.first; // or res.second for average edge map +} + +int main(int argc, char** argv) { + const string about = + "This sample demonstrates edge detection with dexined and canny edge detection techniques.\n\n" + "To run with canny:\n" + "\t ./example_dnn_edge_detection --input=path/to/your/input/image/or/video (don't give --input flag if want to use device camera)\n" + "With Dexined:\n" + "\t ./example_dnn_edge_detection dexined --input=path/to/your/input/image/or/video\n\n" + "For switching between deep learning based model(dexined) and canny edge detector, press space bar in case of video. In case of image, pass the argument --method for switching between dexined and canny.\n" + "Model path can also be specified using --model argument. Download it using python download_models.py dexined from dnn samples directory\n\n"; + + const string param_keys = + "{ help h | | Print help message. }" + "{ @alias | | An alias name of model to extract preprocessing parameters from models.yml file. }" + "{ zoo | ../dnn/models.yml | An optional path to file with preprocessing parameters }" + "{ input i | | Path to input image or video file. Skip this argument to capture frames from a camera.}" + "{ method | dexined | Choose method: dexined or canny. }" + "{ model | | Path to the model file for using dexined. }"; + + const string backend_keys = format( + "{ backend | default | Choose one of computation backends: " + "default: automatically (by default), " + "openvino: Intel's Deep Learning Inference Engine (https://software.intel.com/openvino-toolkit), " + "opencv: OpenCV implementation, " + "vkcom: VKCOM, " + "cuda: CUDA, " + "webnn: WebNN }"); + + const string target_keys = format( + "{ target | cpu | Choose one of target computation devices: " + "cpu: CPU target (by default), " + "opencl: OpenCL, " + "opencl_fp16: OpenCL fp16 (half-float precision), " + "vpu: VPU, " + "vulkan: Vulkan, " + "cuda: CUDA, " + "cuda_fp16: CUDA fp16 (half-float preprocess) }"); + + + string keys = param_keys + backend_keys + target_keys; + + CommandLineParser parser(argc, argv, keys); + if (parser.has("help")) + { + cout << about << endl; + parser.printMessage(); + return -1; + } + + string modelName = parser.get("@alias"); + string zooFile = parser.get("zoo"); + + const char* path = getenv("OPENCV_SAMPLES_DATA_PATH"); + if ((path != NULL) || parser.has("@alias") || (parser.get("model") != "")) { + modelName = "dexined"; + zooFile = findFile(zooFile); + } + else{ + cout<<"[WARN] set the environment variables or pass path to dexined.onnx model file using --model and models.yml file using --zoo for using dexined based edge detector. Continuing with canny edge detector\n\n"; + } + + keys += genPreprocArguments(modelName, zooFile); + + parser = CommandLineParser(argc, argv, keys); + int width = parser.get("width"); + int height = parser.get("height"); + float scale = parser.get("scale"); + Scalar mean = parser.get("mean"); + bool swapRB = parser.get("rgb"); + String backend = parser.get("backend"); + String target = parser.get("target"); + string method = parser.get("method"); + String sha1 = parser.get("sha1"); + string model = findModel(parser.get("model"), sha1); + parser.about(about); + + VideoCapture cap; + if (parser.has("input")) + cap.open(samples::findFile(parser.get("input"))); + else + cap.open(0); + + namedWindow("Input", WINDOW_AUTOSIZE); + namedWindow("Output", WINDOW_AUTOSIZE); + moveWindow("Output", 200, 0); + Net net; + Mat image; + + if (model.empty()) { + cout << "[WARN] Model file not provided, using canny instead. Pass model using --model=/path/to/dexined.onnx to use dexined model." << endl; + method = "canny"; + } + + if (method == "dexined") { + loadModel(model, backend, target, net); + } + else{ + Mat dummy = Mat::zeros(512, 512, CV_8UC3); + setupCannyWindow(); + } + cout<<"To switch between canny and dexined press space bar."<> image; + if (image.empty()) + { + cout << "Press any key to exit" << endl; + waitKey(); + break; + } + + Mat result; + int kernelSize = 2 * blurAmount + 1; + Mat blurred; + GaussianBlur(image, blurred, Size(kernelSize, kernelSize), 0); + if (method == "dexined") + { + Mat blob = blobFromImage(blurred, scale, Size(width, height), mean, swapRB, false, CV_32F); + net.setInput(blob); + applyDexined(net, image, result); + } + else if (method == "canny") + { + applyCanny(blurred, result); + } + imshow("Input", image); + imshow("Output", result); + int key = waitKey(30); + + if (key == ' ' && method == "canny") + { + if (!model.empty()){ + method = "dexined"; + if (net.empty()) + loadModel(model, backend, target, net); + destroyWindow("Output"); + namedWindow("Input", WINDOW_AUTOSIZE); + namedWindow("Output", WINDOW_AUTOSIZE); + moveWindow("Output", 200, 0); + } else { + cout << "[ERROR] Provide model file using --model to use dexined. Download model using python download_models.py dexined from dnn samples directory" << endl; + } + } + else if (key == ' ' && method == "dexined") + { + method = "canny"; + setupCannyWindow(); + } + else if (key == 27 || key == 'q') + { // Escape key to exit + break; + } + } + destroyAllWindows(); + return 0; +} \ No newline at end of file diff --git a/samples/dnn/edge_detection.py b/samples/dnn/edge_detection.py index e3eeff2e36..ab8c35b207 100644 --- a/samples/dnn/edge_detection.py +++ b/samples/dnn/edge_detection.py @@ -1,69 +1,177 @@ +''' +This sample demonstrates edge detection with dexined and canny edge detection techniques. +For switching between deep learning based model(dexined) and canny edge detector, press space bar in case of video. In case of image, pass the argument --method for switching between dexined and canny. +''' + import cv2 as cv import argparse +import numpy as np +from common import * -parser = argparse.ArgumentParser( - description='This sample shows how to define custom OpenCV deep learning layers in Python. ' - 'Holistically-Nested Edge Detection (https://arxiv.org/abs/1504.06375) neural network ' - 'is used as an example model. Find a pre-trained model at https://github.com/s9xie/hed.') -parser.add_argument('--input', help='Path to image or video. Skip to capture frames from camera') -parser.add_argument('--prototxt', help='Path to deploy.prototxt', required=True) -parser.add_argument('--caffemodel', help='Path to hed_pretrained_bsds.caffemodel', required=True) -parser.add_argument('--width', help='Resize input image to a specific width', default=500, type=int) -parser.add_argument('--height', help='Resize input image to a specific height', default=500, type=int) -args = parser.parse_args() +def get_args_parser(func_args): + backends = ("default", "openvino", "opencv", "vkcom", "cuda") + targets = ("cpu", "opencl", "opencl_fp16", "ncs2_vpu", "hddl_vpu", "vulkan", "cuda", "cuda_fp16") -#! [CropLayer] -class CropLayer(object): - def __init__(self, params, blobs): - self.xstart = 0 - self.xend = 0 - self.ystart = 0 - self.yend = 0 + parser = argparse.ArgumentParser(add_help=False) + parser.add_argument('--zoo', default=os.path.join(os.path.dirname(os.path.abspath(__file__)), 'models.yml'), + help='An optional path to file with preprocessing parameters.') + parser.add_argument('--input', help='Path to input image or video file. Skip this argument to capture frames from a camera.', default=0, required=False) + parser.add_argument('--method', help='choose method: dexined or canny', default='canny', required=False) + parser.add_argument('--backend', default="default", type=str, choices=backends, + help="Choose one of computation backends: " + "default: automatically (by default), " + "openvino: Intel's Deep Learning Inference Engine (https://software.intel.com/openvino-toolkit), " + "opencv: OpenCV implementation, " + "vkcom: VKCOM, " + "cuda: CUDA, " + "webnn: WebNN") + parser.add_argument('--target', default="cpu", type=str, choices=targets, + help="Choose one of target computation devices: " + "cpu: CPU target (by default), " + "opencl: OpenCL, " + "opencl_fp16: OpenCL fp16 (half-float precision), " + "ncs2_vpu: NCS2 VPU, " + "hddl_vpu: HDDL VPU, " + "vulkan: Vulkan, " + "cuda: CUDA, " + "cuda_fp16: CUDA fp16 (half-float preprocess)") - # Our layer receives two inputs. We need to crop the first input blob - # to match a shape of the second one (keeping batch size and number of channels) - def getMemoryShapes(self, inputs): - inputShape, targetShape = inputs[0], inputs[1] - batchSize, numChannels = inputShape[0], inputShape[1] - height, width = targetShape[2], targetShape[3] + args, _ = parser.parse_known_args() + add_preproc_args(args.zoo, parser, 'edge_detection', 'dexined') + parser = argparse.ArgumentParser(parents=[parser], + description=''' + To run: + Canny: + python edge_detection.py --input=path/to/your/input/image/or/video (don't give --input flag if want to use device camera) + Dexined: + python edge_detection.py dexined --input=path/to/your/input/image/or/video - self.ystart = (inputShape[2] - targetShape[2]) // 2 - self.xstart = (inputShape[3] - targetShape[3]) // 2 - self.yend = self.ystart + height - self.xend = self.xstart + width + "In case of video input, for switching between deep learning based model (Dexined) and Canny edge detector, press space bar. Pass as argument in case of image input." - return [[batchSize, numChannels, height, width]] + Model path can also be specified using --model argument + ''', formatter_class=argparse.RawTextHelpFormatter) + return parser.parse_args(func_args) - def forward(self, inputs): - return [inputs[0][:,:,self.ystart:self.yend,self.xstart:self.xend]] -#! [CropLayer] +threshold1 = 0 +threshold2 = 50 +blur_amount = 5 +gray = None -#! [Register] -cv.dnn_registerLayer('Crop', CropLayer) -#! [Register] +def sigmoid(x): + return 1.0 / (1.0 + np.exp(-x)) -# Load the model. -net = cv.dnn.readNet(cv.samples.findFile(args.prototxt), cv.samples.findFile(args.caffemodel)) +def post_processing(output, shape): + h, w = shape + preds = [] + for p in output: + img = sigmoid(p) + img = np.squeeze(img) + img = cv.normalize(img, None, 0, 255, cv.NORM_MINMAX, cv.CV_8U) + img = cv.resize(img, (w, h)) + preds.append(img) + fuse = preds[-1] + ave = np.array(preds, dtype=np.float32) + ave = np.uint8(np.mean(ave, axis=0)) + return fuse, ave -kWinName = 'Holistically-Nested Edge Detection' -cv.namedWindow('Input', cv.WINDOW_NORMAL) -cv.namedWindow(kWinName, cv.WINDOW_NORMAL) +def apply_canny(image): + global threshold1, threshold2, blur_amount + kernel_size = 2 * blur_amount + 1 + blurred = cv.GaussianBlur(image, (kernel_size, kernel_size), 0) + result = cv.Canny(blurred, threshold1, threshold2) + cv.imshow('Output', result) -cap = cv.VideoCapture(args.input if args.input else 0) -while cv.waitKey(1) < 0: - hasFrame, frame = cap.read() - if not hasFrame: - cv.waitKey() - break +def setupCannyWindow(image): + global gray + cv.destroyWindow('Output') + cv.namedWindow('Output', cv.WINDOW_AUTOSIZE) + cv.moveWindow('Output', 200, 50) + gray = cv.cvtColor(image, cv.COLOR_BGR2GRAY) - cv.imshow('Input', frame) + cv.createTrackbar('thrs1', 'Output', threshold1, 255, lambda value: [globals().__setitem__('threshold1', value), apply_canny(gray)]) + cv.createTrackbar('thrs2', 'Output', threshold2, 255, lambda value: [globals().__setitem__('threshold2', value), apply_canny(gray)]) + cv.createTrackbar('blur', 'Output', blur_amount, 20, lambda value: [globals().__setitem__('blur_amount', value), apply_canny(gray)]) - inp = cv.dnn.blobFromImage(frame, scalefactor=1.0, size=(args.width, args.height), - mean=(104.00698793, 116.66876762, 122.67891434), - swapRB=False, crop=False) - net.setInput(inp) +def loadModel(args): + net = cv.dnn.readNetFromONNX(args.model) + net.setPreferableBackend(get_backend_id(args.backend)) + net.setPreferableTarget(get_target_id(args.target)) + return net - out = net.forward() - out = out[0, 0] - out = cv.resize(out, (frame.shape[1], frame.shape[0])) - cv.imshow(kWinName, out) +def apply_dexined(model, image): + out = model.forward() + result,_ = post_processing(out, image.shape[:2]) + t, _ = model.getPerfProfile() + label = 'Inference time: %.2f ms' % (t * 1000.0 / cv.getTickFrequency()) + cv.putText(image, label, (0, 15), cv.FONT_HERSHEY_SIMPLEX, 0.5, (0, 0, 255)) + cv.putText(result, label, (0, 15), cv.FONT_HERSHEY_SIMPLEX, 0.5, (255, 255, 255)) + cv.imshow("Output", result) + +def main(func_args=None): + args = get_args_parser(func_args) + + cap = cv.VideoCapture(cv.samples.findFile(args.input) if args.input else 0) + if not cap.isOpened(): + print("Failed to open the input video") + exit(-1) + cv.namedWindow('Input', cv.WINDOW_AUTOSIZE) + cv.namedWindow('Output', cv.WINDOW_AUTOSIZE) + cv.moveWindow('Output', 200, 50) + + method = args.method + if os.getenv('OPENCV_SAMPLES_DATA_PATH') is not None or hasattr(args, 'model'): + try: + args.model = findModel(args.model, args.sha1) + method = 'dexined' + except: + print("[WARN] Model file not provided, using canny instead. Pass model using --model=/path/to/dexined.onnx to use dexined model.") + method = 'canny' + args.model = None + else: + print("[WARN] Model file not provided, using canny instead. Pass model using --model=/path/to/dexined.onnx to use dexined model.") + method = 'canny' + + if method == 'canny': + dummy = np.zeros((512, 512, 3), dtype="uint8") + setupCannyWindow(dummy) + net = None + if method == "dexined": + net = loadModel(args) + while cv.waitKey(1) < 0: + hasFrame, image = cap.read() + if not hasFrame: + print("Press any key to exit") + cv.waitKey(0) + break + if method == "canny": + global gray + gray = cv.cvtColor(image, cv.COLOR_BGR2GRAY) + apply_canny(gray) + elif method == "dexined": + inp = cv.dnn.blobFromImage(image, args.scale, (args.width, args.height), args.mean, swapRB=args.rgb, crop=False) + + net.setInput(inp) + apply_dexined(net, image) + + cv.imshow("Input", image) + key = cv.waitKey(30) + if key == ord(' ') and method == 'canny': + if hasattr(args, 'model') and args.model is not None: + print("model: ", args.model) + method = "dexined" + if net is None: + net = loadModel(args) + cv.destroyWindow('Output') + cv.namedWindow('Output', cv.WINDOW_AUTOSIZE) + cv.moveWindow('Output', 200, 50) + else: + print("[ERROR] Provide model file using --model to use dexined. Download model using python download_models.py dexined from dnn samples directory") + elif key == ord(' ') and method=='dexined': + method = "canny" + setupCannyWindow(image) + elif key == 27 or key == ord('q'): + break + cv.destroyAllWindows() + +if __name__ == '__main__': + main() \ No newline at end of file diff --git a/samples/dnn/models.yml b/samples/dnn/models.yml index 0c478cec31..a58eea4cef 100644 --- a/samples/dnn/models.yml +++ b/samples/dnn/models.yml @@ -288,3 +288,19 @@ u2netp: height: 320 rgb: true sample: "segmentation" + +################################################################################ +# Edge Detection models. +################################################################################ + +dexined: + load_info: + url: "https://github.com/gursimarsingh/opencv_zoo/raw/dexined_model/models/edge_detection_dexined/dexined.onnx" + sha1: "f86f2d32c3cf892771f76b5e6b629b16a66510e9" + model: "dexined.onnx" + mean: [103.5, 116.2, 123.6] + scale: 1.0 + width: 512 + height: 512 + rgb: false + sample: "edge_detection"