From e823493af181bd6082efd7200ee9c13d2a8f9e51 Mon Sep 17 00:00:00 2001 From: Gursimar Singh Date: Wed, 18 Sep 2024 18:49:46 +0530 Subject: [PATCH] Merge pull request #25710 from gursimarsingh:improved_object_detection_sample Merged yolo_detector and object detection sample #25710 Relates to #25006 This pull request merges the yolo_detector.cpp sample with the object_detector.cpp sample. It also beautifies the bounding box display on the output images ### Pull Request Readiness Checklist See details at https://github.com/opencv/opencv/wiki/How_to_contribute#making-a-good-pull-request - [x] I agree to contribute to the project under Apache 2 License. - [x] To the best of my knowledge, the proposed patch is not based on a code under GPL or another license that is incompatible with OpenCV - [x] The PR is proposed to the proper branch - [x] There is a reference to the original bug report and related work - [ ] There is accuracy test, performance test and test data in opencv_extra repository, if applicable Patch to opencv_extra has the same branch name. - [x] The feature is well documented and sample code can be built with the project CMake --- doc/tutorials/dnn/dnn_yolo/dnn_yolo.markdown | 64 +- samples/dnn/common.hpp | 27 + samples/dnn/models.yml | 56 +- samples/dnn/object_detection.cpp | 714 ++++++++++++------- samples/dnn/object_detection.py | 231 +++--- samples/dnn/yolo_detector.cpp | 382 ---------- 6 files changed, 684 insertions(+), 790 deletions(-) delete mode 100644 samples/dnn/yolo_detector.cpp diff --git a/doc/tutorials/dnn/dnn_yolo/dnn_yolo.markdown b/doc/tutorials/dnn/dnn_yolo/dnn_yolo.markdown index 8406b4746c..708873c40d 100644 --- a/doc/tutorials/dnn/dnn_yolo/dnn_yolo.markdown +++ b/doc/tutorials/dnn/dnn_yolo/dnn_yolo.markdown @@ -150,13 +150,12 @@ Once we have our ONNX graph of the model, we just simply can run with OpenCV's s 3. Run the following command: @code{.cpp} -./bin/example_dnn_yolo_detector --input= \ - --classes= \ +./bin/example_dnn_object_detection --input= \ + --labels= \ --thr= \ --nms= \ --mean= \ --scale= \ - --yolo= \ --padvalue= \ --paddingmode= \ --backend= \ @@ -166,7 +165,7 @@ Once we have our ONNX graph of the model, we just simply can run with OpenCV's s @endcode - --input: File path to your input image or video. If omitted, it will capture frames from a camera. -- --classes: File path to a text file containing class names for object detection. +- --labels: File path to a text file containing class names for object detection. - --thr: Confidence threshold for detection (e.g., 0.5). - --nms: Non-maximum suppression threshold (e.g., 0.4). - --mean: Mean normalization value (e.g., 0.0 for no mean normalization). @@ -191,43 +190,28 @@ To demonstrate how to run OpenCV YOLO samples without your own pretrained model, Run the YOLOX detector(with default values): @code{.sh} -git clone https://github.com/opencv/opencv_extra.git -cd opencv_extra/testdata/dnn -python download_models.py yolox_s_inf_decoder -cd .. -export OPENCV_TEST_DATA_PATH=$(pwd) +cd opencv/samples/dnn +export OPENCV_DOWNLOAD_CACHE_DIR= +cd ../data +export OPENCV_SAMPLES_DATA_PATH=$(pwd) +python download_models.py yolov8x --save_dir=$OPENCV_DOWNLOAD_CACHE_DIR cd -./bin/example_dnn_yolo_detector +./bin/example_dnn_object_detection yolov8x @endcode This will execute the YOLOX detector with your camera. For YOLOv8 (for instance), follow these additional steps: @code{.sh} -cd opencv_extra/testdata/dnn -python download_models.py yolov8 -cd .. -export OPENCV_TEST_DATA_PATH=$(pwd) +cd opencv/samples/dnn +export OPENCV_DOWNLOAD_CACHE_DIR= +cd ../data +export OPENCV_SAMPLES_DATA_PATH=$(pwd) +python download_models.py yolov8n --save_dir=$OPENCV_DOWNLOAD_CACHE_DIR cd - -./bin/example_dnn_yolo_detector --model=onnx/models/yolov8n.onnx --yolo=yolov8 --mean=0.0 --scale=0.003921568627 --paddingmode=2 --padvalue=144.0 --thr=0.5 --nms=0.4 --rgb=0 +./bin/example_dnn_object_detection yolov8n --model=onnx/models/yolov8n.onnx --mean=0.0 --scale=0.003921568627 --paddingmode=2 --padvalue=144.0 --thr=0.5 --nms=0.4 --rgb=0 @endcode -For YOLOv10, follow these steps: - -@code{.sh} -cd opencv_extra/testdata/dnn -python download_models.py yolov10 -cd .. -export OPENCV_TEST_DATA_PATH=$(pwd) -cd - -./bin/example_dnn_yolo_detector --model=onnx/models/yolov10s.onnx --yolo=yolov10 --width=640 --height=480 --scale=0.003921568627 --padvalue=114 -@endcode - -This will run `YOLOv10` detector on first camera found on your system. If you want to run it on a image/video file, you can use `--input` option to specify the path to the file. - - VIDEO DEMO: @youtube{NHtRlndE2cg} @@ -238,30 +222,30 @@ module this is also quite easy to achieve. Below we will outline the sample impl - Import required libraries -@snippet samples/dnn/yolo_detector.cpp includes +@snippet samples/dnn/object_detection.cpp includes - Read ONNX graph and create neural network model: -@snippet samples/dnn/yolo_detector.cpp read_net +@snippet samples/dnn/object_detection.cpp read_net - Read image and pre-process it: -@snippet samples/dnn/yolo_detector.cpp preprocess_params -@snippet samples/dnn/yolo_detector.cpp preprocess_call -@snippet samples/dnn/yolo_detector.cpp preprocess_call_func +@snippet samples/dnn/object_detection.cpp preprocess_params +@snippet samples/dnn/object_detection.cpp preprocess_call +@snippet samples/dnn/object_detection.cpp preprocess_call_func - Inference: -@snippet samples/dnn/yolo_detector.cpp forward_buffers -@snippet samples/dnn/yolo_detector.cpp forward +@snippet samples/dnn/object_detection.cpp forward_buffers +@snippet samples/dnn/object_detection.cpp forward - Post-Processing All post-processing steps are implemented in function `yoloPostProcess`. Please pay attention, that NMS step is not included into onnx graph. Sample uses OpenCV function for it. -@snippet samples/dnn/yolo_detector.cpp postprocess +@snippet samples/dnn/object_detection.cpp postprocess - Draw predicted boxes -@snippet samples/dnn/yolo_detector.cpp draw_boxes +@snippet samples/dnn/object_detection.cpp draw_boxes \ No newline at end of file diff --git a/samples/dnn/common.hpp b/samples/dnn/common.hpp index 92d0c66241..a01b1ca40f 100644 --- a/samples/dnn/common.hpp +++ b/samples/dnn/common.hpp @@ -12,6 +12,8 @@ std::string findFile(const std::string& filename); std::string findModel(const std::string& filename, const std::string& sha1); +std::vector findAliases(std::string& zooFile, const std::string& sampleType); + inline int getBackendID(const String& backend) { std::map backendIDs = { {"default", cv::dnn::DNN_BACKEND_DEFAULT}, @@ -177,8 +179,33 @@ std::string genPreprocArguments(const std::string& modelName, const std::string& modelName, zooFile)+ genArgument(prefix + "labels", "Path to a text file with names of classes to label detected objects.", modelName, zooFile)+ + genArgument(prefix + "postprocessing", "Indicate the postprocessing type of model i.e. yolov8, yolonas, etc.", + modelName, zooFile)+ genArgument(prefix + "sha1", "Optional path to hashsum of downloaded model to be loaded from models.yml", modelName, zooFile)+ genArgument(prefix + "download_sha", "Optional path to hashsum of downloaded model to be loaded from models.yml", modelName, zooFile); +} + +std::vector findAliases(std::string& zooFile, const std::string& sampleType) { + std::vector aliases; + + zooFile = findFile(zooFile); + + cv::FileStorage fs(zooFile, cv::FileStorage::READ); + + cv::FileNode root = fs.root(); + for (const auto& node : root) { + std::string alias = node.name(); + cv::FileNode sampleNode = node["sample"]; + + if (!sampleNode.empty() && sampleNode.isString()) { + std::string sampleValue = (std::string)sampleNode; + if (sampleValue == sampleType) { + aliases.push_back(alias); + } + } + } + + return aliases; } \ No newline at end of file diff --git a/samples/dnn/models.yml b/samples/dnn/models.yml index 46a988abe0..99abf22b26 100644 --- a/samples/dnn/models.yml +++ b/samples/dnn/models.yml @@ -16,9 +16,9 @@ yolov8x: width: 640 height: 640 rgb: true - classes: "object_detection_classes_yolo.txt" - background_label_id: 0 - sample: "yolo_detector" + labels: "object_detection_classes_yolo.txt" + postprocessing: "yolov8" + sample: "object_detection" yolov8s: load_info: @@ -30,11 +30,11 @@ yolov8s: width: 640 height: 640 rgb: true - classes: "object_detection_classes_yolo.txt" - background_label_id: 0 - sample: "yolo_detector" + labels: "object_detection_classes_yolo.txt" + postprocessing: "yolov8" + sample: "object_detection" -yolov8n: +yolov8: load_info: url: "https://github.com/CVHub520/X-AnyLabeling/releases/download/v0.1.0/yolov8n.onnx" sha1: "68f864475d06e2ec4037181052739f268eeac38d" @@ -44,9 +44,9 @@ yolov8n: width: 640 height: 640 rgb: true - classes: "object_detection_classes_yolo.txt" - background_label_id: 0 - sample: "yolo_detector" + labels: "object_detection_classes_yolo.txt" + postprocessing: "yolov8" + sample: "object_detection" yolov8m: load_info: @@ -58,9 +58,9 @@ yolov8m: width: 640 height: 640 rgb: true - classes: "object_detection_classes_yolo.txt" - background_label_id: 0 - sample: "yolo_detector" + labels: "object_detection_classes_yolo.txt" + postprocessing: "yolov8" + sample: "object_detection" yolov8l: load_info: @@ -72,8 +72,8 @@ yolov8l: width: 640 height: 640 rgb: true - classes: "object_detection_classes_yolo.txt" - background_label_id: 0 + labels: "object_detection_classes_yolo.txt" + postprocessing: "yolov8" sample: "yolo_detector" # YOLO4 object detection family from Darknet (https://github.com/AlexeyAB/darknet) @@ -90,7 +90,7 @@ yolov4: width: 416 height: 416 rgb: true - classes: "object_detection_classes_yolo.txt" + labels: "object_detection_classes_yolo.txt" background_label_id: 0 sample: "object_detection" @@ -105,7 +105,7 @@ yolov4-tiny: width: 416 height: 416 rgb: true - classes: "object_detection_classes_yolo.txt" + labels: "object_detection_classes_yolo.txt" background_label_id: 0 sample: "object_detection" @@ -120,7 +120,7 @@ yolov3: width: 416 height: 416 rgb: true - classes: "object_detection_classes_yolo.txt" + labels: "object_detection_classes_yolo.txt" background_label_id: 0 sample: "object_detection" @@ -135,24 +135,10 @@ tiny-yolo-voc: width: 416 height: 416 rgb: true - classes: "object_detection_classes_pascal_voc.txt" + labels: "object_detection_classes_pascal_voc.txt" background_label_id: 0 sample: "object_detection" -yolov8: - load_info: - url: "https://github.com/CVHub520/X-AnyLabeling/releases/download/v0.1.0/yolov8n.onnx" - sha1: "68f864475d06e2ec4037181052739f268eeac38d" - model: "yolov8n.onnx" - mean: [0, 0, 0] - scale: 0.00392 - width: 640 - height: 640 - rgb: true - postprocessing: "yolov8" - classes: "object_detection_classes_yolo.txt" - sample: "object_detection" - # Caffe implementation of SSD model from https://github.com/chuanqi305/MobileNet-SSD ssd_caffe: load_info: @@ -165,7 +151,7 @@ ssd_caffe: width: 300 height: 300 rgb: false - classes: "object_detection_classes_pascal_voc.txt" + labels: "object_detection_classes_pascal_voc.txt" sample: "object_detection" # TensorFlow implementation of SSD model from https://github.com/tensorflow/models/tree/master/research/object_detection @@ -183,7 +169,7 @@ ssd_tf: width: 300 height: 300 rgb: true - classes: "object_detection_classes_coco.txt" + labels: "object_detection_classes_coco.txt" sample: "object_detection" # TensorFlow implementation of Faster-RCNN model from https://github.com/tensorflow/models/tree/master/research/object_detection diff --git a/samples/dnn/object_detection.cpp b/samples/dnn/object_detection.cpp index 0baf35c41e..d20f49a9ee 100644 --- a/samples/dnn/object_detection.cpp +++ b/samples/dnn/object_detection.cpp @@ -1,68 +1,114 @@ +//![includes] #include #include #include #include +#include #include -#if defined(HAVE_THREADS) -#define USE_THREADS 1 -#endif - -#ifdef USE_THREADS #include #include #include -#endif +#include "iostream" #include "common.hpp" - -std::string param_keys = - "{ help h | | Print help message. }" - "{ @alias | | An alias name of model to extract preprocessing parameters from models.yml file. }" - "{ zoo | models.yml | An optional path to file with preprocessing parameters }" - "{ device | 0 | camera device number. }" - "{ input i | | Path to input image or video file. Skip this argument to capture frames from a camera. }" - "{ framework f | | Optional name of an origin framework of the model. Detect it automatically if it does not set. }" - "{ classes | | Optional path to a text file with names of classes to label detected objects. }" - "{ thr | .5 | Confidence threshold. }" - "{ nms | .4 | Non-maximum suppression threshold. }" - "{ async | 0 | Number of asynchronous forwards at the same time. " - "Choose 0 for synchronous mode }"; -std::string backend_keys = cv::format( - "{ backend | 0 | Choose one of computation backends: " - "%d: automatically (by default), " - "%d: Intel's Deep Learning Inference Engine (https://software.intel.com/openvino-toolkit), " - "%d: OpenCV implementation, " - "%d: VKCOM, " - "%d: CUDA }", cv::dnn::DNN_BACKEND_DEFAULT, cv::dnn::DNN_BACKEND_INFERENCE_ENGINE, cv::dnn::DNN_BACKEND_OPENCV, cv::dnn::DNN_BACKEND_VKCOM, cv::dnn::DNN_BACKEND_CUDA); -std::string target_keys = cv::format( - "{ target | 0 | Choose one of target computation devices: " - "%d: CPU target (by default), " - "%d: OpenCL, " - "%d: OpenCL fp16 (half-float precision), " - "%d: VPU, " - "%d: Vulkan, " - "%d: CUDA, " - "%d: CUDA fp16 (half-float preprocess) }", cv::dnn::DNN_TARGET_CPU, cv::dnn::DNN_TARGET_OPENCL, cv::dnn::DNN_TARGET_OPENCL_FP16, cv::dnn::DNN_TARGET_MYRIAD, cv::dnn::DNN_TARGET_VULKAN, cv::dnn::DNN_TARGET_CUDA, cv::dnn::DNN_TARGET_CUDA_FP16); -std::string keys = param_keys + backend_keys + target_keys; +//![includes] using namespace cv; using namespace dnn; +using namespace std; -float confThreshold, nmsThreshold; -std::vector classes; +const string about = + "Firstly, download required models using `download_models.py` (if not already done). Set environment variable OPENCV_DOWNLOAD_CACHE_DIR to specify where models should be downloaded. Also, point OPENCV_SAMPLES_DATA_PATH to opencv/samples/data.\n" + "To run:\n" + "\t ./example_dnn_object_detection model_name --input=path/to/your/input/image/or/video (don't give --input flag if want to use device camera)\n" + "Sample command:\n" + "\t ./example_dnn_object_detection yolov8 --input=$OPENCV_SAMPLES_DATA_PATH/baboon.jpg\n" -inline void preprocess(const Mat& frame, Net& net, Size inpSize, float scale, - const Scalar& mean, bool swapRB); + "Model path can also be specified using --model argument. "; -void postprocess(Mat& frame, const std::vector& out, Net& net, int backend); +const string param_keys = + "{ help h | | Print help message. }" + "{ @alias | | An alias name of model to extract preprocessing parameters from models.yml file. }" + "{ zoo | ../dnn/models.yml | An optional path to file with preprocessing parameters }" + "{ device | 0 | camera device number. }" + "{ input i | | Path to input image or video file. Skip this argument to capture frames from a camera. }" + "{ thr | .5 | Confidence threshold. }" + "{ nms | .4 | Non-maximum suppression threshold. }" + "{ async | 0 | Number of asynchronous forwards at the same time. " + "Choose 0 for synchronous mode }" + "{ padvalue | 114.0 | padding value. }" + "{ paddingmode | 2 | Choose one of padding modes: " + "0: resize to required input size without extra processing, " + "1: Image will be cropped after resize, " + "2: Resize image to the desired size while preserving the aspect ratio of original image }"; -void drawPred(int classId, float conf, int left, int top, int right, int bottom, Mat& frame); +const string backend_keys = format( + "{ backend | default | Choose one of computation backends: " + "default: automatically (by default), " + "openvino: Intel's Deep Learning Inference Engine (https://software.intel.com/openvino-toolkit), " + "opencv: OpenCV implementation, " + "vkcom: VKCOM, " + "cuda: CUDA, " + "webnn: WebNN }"); -void callback(int pos, void* userdata); +const string target_keys = format( + "{ target | cpu | Choose one of target computation devices: " + "cpu: CPU target (by default), " + "opencl: OpenCL, " + "opencl_fp16: OpenCL fp16 (half-float precision), " + "vpu: VPU, " + "vulkan: Vulkan, " + "cuda: CUDA, " + "cuda_fp16: CUDA fp16 (half-float preprocess) }"); + +string keys = param_keys + backend_keys + target_keys; + +float confThreshold, nmsThreshold, scale, paddingValue; +vector labels; +Scalar meanv; +bool swapRB; +int inpWidth, inpHeight; +size_t asyncNumReq = 0; +ImagePaddingMode paddingMode; +string modelName, framework; + +static void preprocess(const Mat& frame, Net& net, Size inpSize); + +static void postprocess(Mat& frame, const vector& outs, Net& net, int backend, vector& classIds, vector& confidences, vector& boxes, const string yolo_name); + +static void drawPred(vector& classIds, vector& confidences, vector& boxes, Mat& frame, FontFace& sans, int stdSize, int stdWeight, int stdImgSize, int stdThickness); + +static void callback(int pos, void* userdata); + +static Scalar getColor(int classId); + +static void yoloPostProcessing( + const vector& outs, + vector& keep_classIds, + vector& keep_confidences, + vector& keep_boxes, + float conf_threshold, + float iou_threshold, + const string& yolo_name); + +static void printAliases(string& zooFile){ + vector aliases = findAliases(zooFile, "object_detection"); + + cout<<"Alias choices: [ "; + for (auto it: aliases){ + cout<<"'"< 128 ? Scalar(0, 0, 0) : Scalar(255, 255, 255); +} -#ifdef USE_THREADS template class QueueFPS : public std::queue { @@ -112,233 +158,362 @@ private: TickMeter tm; std::mutex mutex; }; -#endif // USE_THREADS int main(int argc, char** argv) { CommandLineParser parser(argc, argv, keys); - const std::string modelName = parser.get("@alias"); - const std::string zooFile = parser.get("zoo"); + string zooFile = parser.get("zoo"); + if (!parser.has("@alias") || parser.has("help")) + { + cout << about << endl; + parser.printMessage(); + printAliases(zooFile); + return -1; + } + zooFile = findFile(zooFile); + modelName = parser.get("@alias"); keys += genPreprocArguments(modelName, zooFile); parser = CommandLineParser(argc, argv, keys); - parser.about("Use this script to run object detection deep learning networks using OpenCV."); - if (argc == 1 || parser.has("help")) + + if (!parser.has("model")) { - parser.printMessage(); - return 0; + cout << "Path to model is not provided in command line or model alias is not correct" << endl; + printAliases(zooFile); + return -1; } confThreshold = parser.get("thr"); nmsThreshold = parser.get("nms"); - float scale = parser.get("scale"); - Scalar mean = parser.get("mean"); - bool swapRB = parser.get("rgb"); - int inpWidth = parser.get("width"); - int inpHeight = parser.get("height"); - size_t asyncNumReq = parser.get("async"); - CV_Assert(parser.has("model")); - std::string modelPath = findFile(parser.get("model")); - std::string configPath = findFile(parser.get("config")); + //![preprocess_params] + scale = parser.get("scale"); + meanv = parser.get("mean"); + swapRB = parser.get("rgb"); + inpWidth = parser.get("width"); + inpHeight = parser.get("height"); + int async = parser.get("async"); + paddingValue = parser.get("padvalue"); + const string yolo_name = parser.get("postprocessing"); + paddingMode = static_cast(parser.get("paddingmode")); + //![preprocess_params] + String sha1 = parser.get("sha1"); + const string modelPath = findModel(parser.get("model"), sha1); + const string configPath = findFile(parser.get("config")); + framework = modelPath.substr(modelPath.rfind('.') + 1); - // Open file with classes names. - if (parser.has("classes")) + if (parser.has("labels")) { - std::string file = parser.get("classes"); - std::ifstream ifs(file.c_str()); + const string file = findFile(parser.get("labels")); + ifstream ifs(file.c_str()); if (!ifs.is_open()) CV_Error(Error::StsError, "File " + file + " not found"); - std::string line; - while (std::getline(ifs, line)) + string line; + while (getline(ifs, line)) { - classes.push_back(line); + labels.push_back(line); } } - - // Load a model. - Net net = readNet(modelPath, configPath, parser.get("framework")); - int backend = parser.get("backend"); + //![read_net] + Net net = readNet(modelPath, configPath); + int backend = getBackendID(parser.get("backend")); net.setPreferableBackend(backend); - net.setPreferableTarget(parser.get("target")); - std::vector outNames = net.getUnconnectedOutLayersNames(); + net.setPreferableTarget(getTargetID(parser.get("target"))); + //![read_net] // Create a window - static const std::string kWinName = "Deep learning object detection in OpenCV"; - namedWindow(kWinName, WINDOW_NORMAL); + static const string kWinName = "Deep learning object detection in OpenCV"; + namedWindow(kWinName, WINDOW_AUTOSIZE); int initialConf = (int)(confThreshold * 100); - createTrackbar("Confidence threshold, %", kWinName, &initialConf, 99, callback); + createTrackbar("Confidence threshold, %", kWinName, &initialConf, 99, callback, &net); // Open a video file or an image file or a camera stream. VideoCapture cap; - if (parser.has("input")) - cap.open(parser.get("input")); - else - cap.open(parser.get("device")); + bool openSuccess = parser.has("input") ? cap.open(parser.get("input")) : cap.open(parser.get("device")); + if (!openSuccess){ + cout << "Could not open input file or camera device" << endl; + return 0; + } -#ifdef USE_THREADS - bool process = true; + FontFace sans("sans"); - // Frames capturing thread - QueueFPS framesQueue; - std::thread framesThread([&](){ - Mat frame; - while (process) - { - cap >> frame; - if (!frame.empty()) - framesQueue.push(frame.clone()); - else - break; - } - }); + int stdSize = 15; + int stdWeight = 150; + int stdImgSize = 512; + int stdThickness = 2; + vector classIds; + vector confidences; + vector boxes; - // Frames processing thread - QueueFPS processedFramesQueue; - QueueFPS > predictionsQueue; - std::thread processingThread([&](){ - std::queue futureOutputs; - Mat blob; - while (process) - { - // Get a next frame + if (async > 0 && backend == DNN_BACKEND_INFERENCE_ENGINE){ + asyncNumReq = async; + } + + if (async != 0) { + // Threading is enabled + bool process = true; + + // Frames capturing thread + QueueFPS framesQueue; + std::thread framesThread([&]() { Mat frame; - { - if (!framesQueue.empty()) - { - frame = framesQueue.get(); - if (asyncNumReq) - { - if (futureOutputs.size() == asyncNumReq) - frame = Mat(); - } - else - framesQueue.clear(); // Skip the rest of frames - } - } - - // Process the frame - if (!frame.empty()) - { - preprocess(frame, net, Size(inpWidth, inpHeight), scale, mean, swapRB); - processedFramesQueue.push(frame); - - if (asyncNumReq) - { - futureOutputs.push(net.forwardAsync()); - } + while (process) { + cap >> frame; + if (!frame.empty()) + framesQueue.push(frame.clone()); else + break; + } + }); + + // Frames processing thread + QueueFPS processedFramesQueue; + QueueFPS> predictionsQueue; + std::thread processingThread([&]() { + std::queue futureOutputs; + Mat blob; + while (process) { + // Get the next frame + Mat frame; { - std::vector outs; - net.forward(outs, outNames); - predictionsQueue.push(outs); + if (!framesQueue.empty()) { + frame = framesQueue.get(); + if (asyncNumReq) { + if (futureOutputs.size() == asyncNumReq) + frame = Mat(); + } + } + } + + // Process the frame + if (!frame.empty()) { + preprocess(frame, net, Size(inpWidth, inpHeight)); + processedFramesQueue.push(frame); + + if (asyncNumReq) { + futureOutputs.push(net.forwardAsync()); + } else { + vector outs; + net.forward(outs, net.getUnconnectedOutLayersNames()); + predictionsQueue.push(outs); + } + } + + while (!futureOutputs.empty() && + futureOutputs.front().wait_for(std::chrono::seconds(0))) { + AsyncArray async_out = futureOutputs.front(); + futureOutputs.pop(); + Mat out; + async_out.get(out); + predictionsQueue.push({out}); } } + }); - while (!futureOutputs.empty() && - futureOutputs.front().wait_for(std::chrono::seconds(0))) - { - AsyncArray async_out = futureOutputs.front(); - futureOutputs.pop(); - Mat out; - async_out.get(out); - predictionsQueue.push({out}); + // Postprocessing and rendering loop + while (waitKey(100) < 0) { + if (predictionsQueue.empty()) + continue; + + vector outs = predictionsQueue.get(); + Mat frame = processedFramesQueue.get(); + + classIds.clear(); + confidences.clear(); + boxes.clear(); + postprocess(frame, outs, net, backend, classIds, confidences, boxes, yolo_name); + + drawPred(classIds, confidences, boxes, frame, sans, stdSize, stdWeight, stdImgSize, stdThickness); + + int imgWidth = max(frame.rows, frame.cols); + int size = static_cast((stdSize * imgWidth) / (stdImgSize * 1.5)); + int weight = static_cast((stdWeight * imgWidth) / (stdImgSize * 1.5)); + + if (predictionsQueue.counter > 1) { + string label = format("Camera: %.2f FPS", framesQueue.getFPS()); + rectangle(frame, Point(0, 0), Point(10 * size, 3 * size + size / 4), Scalar::all(255), FILLED); + putText(frame, label, Point(0, size), Scalar::all(0), sans, size, weight); + + label = format("Network: %.2f FPS", predictionsQueue.getFPS()); + putText(frame, label, Point(0, 2 * size), Scalar::all(0), sans, size, weight); + + label = format("Skipped frames: %d", framesQueue.counter - predictionsQueue.counter); + putText(frame, label, Point(0, 3 * size), Scalar::all(0), sans, size, weight); } + imshow(kWinName, frame); } - }); - // Postprocessing and rendering loop - while (waitKey(1) < 0) - { - if (predictionsQueue.empty()) - continue; + process = false; + framesThread.join(); + processingThread.join(); + } else { + if (asyncNumReq) + CV_Error(Error::StsNotImplemented, "Asynchronous forward is supported only with Inference Engine backend."); + // Threading is disabled, run synchronously + Mat frame, blob; + while (waitKey(100) < 0) { + cap >> frame; + if (frame.empty()) { + waitKey(); + break; + } + preprocess(frame, net, Size(inpWidth, inpHeight)); - std::vector outs = predictionsQueue.get(); - Mat frame = processedFramesQueue.get(); + vector outs; + net.forward(outs, net.getUnconnectedOutLayersNames()); - postprocess(frame, outs, net, backend); + classIds.clear(); + confidences.clear(); + boxes.clear(); - if (predictionsQueue.counter > 1) - { - std::string label = format("Camera: %.2f FPS", framesQueue.getFPS()); - putText(frame, label, Point(0, 15), FONT_HERSHEY_SIMPLEX, 0.5, Scalar(0, 255, 0)); + postprocess(frame, outs, net, backend, classIds, confidences, boxes, yolo_name); - label = format("Network: %.2f FPS", predictionsQueue.getFPS()); - putText(frame, label, Point(0, 30), FONT_HERSHEY_SIMPLEX, 0.5, Scalar(0, 255, 0)); + drawPred(classIds, confidences, boxes, frame, sans, stdSize, stdWeight, stdImgSize, stdThickness); - label = format("Skipped frames: %d", framesQueue.counter - predictionsQueue.counter); - putText(frame, label, Point(0, 45), FONT_HERSHEY_SIMPLEX, 0.5, Scalar(0, 255, 0)); + vector layersTimes; + int imgWidth = max(frame.rows, frame.cols); + int size = static_cast((stdSize * imgWidth) / (stdImgSize * 1.5)); + int weight = static_cast((stdWeight * imgWidth) / (stdImgSize * 1.5)); + double freq = getTickFrequency() / 1000; + double t = net.getPerfProfile(layersTimes) / freq; + string label = format("Inference time: %.2f ms", t); + putText(frame, label, Point(0, size), Scalar(0, 255, 0), sans, size, weight); + imshow(kWinName, frame); } - imshow(kWinName, frame); } - - process = false; - framesThread.join(); - processingThread.join(); - -#else // USE_THREADS - if (asyncNumReq) - CV_Error(Error::StsNotImplemented, "Asynchronous forward is supported only with Inference Engine backend."); - - // Process frames. - Mat frame, blob; - while (waitKey(1) < 0) - { - cap >> frame; - if (frame.empty()) - { - waitKey(); - break; - } - - preprocess(frame, net, Size(inpWidth, inpHeight), scale, mean, swapRB); - - std::vector outs; - net.forward(outs, outNames); - - postprocess(frame, outs, net, backend); - - // Put efficiency information. - std::vector layersTimes; - double freq = getTickFrequency() / 1000; - double t = net.getPerfProfile(layersTimes) / freq; - std::string label = format("Inference time: %.2f ms", t); - putText(frame, label, Point(0, 15), FONT_HERSHEY_SIMPLEX, 0.5, Scalar(0, 255, 0)); - - imshow(kWinName, frame); - } -#endif // USE_THREADS return 0; } -inline void preprocess(const Mat& frame, Net& net, Size inpSize, float scale, - const Scalar& mean, bool swapRB) +void preprocess(const Mat& frame, Net& net, Size inpSize) { - static Mat blob; - // Create a 4D blob from a frame. - if (inpSize.width <= 0) inpSize.width = frame.cols; - if (inpSize.height <= 0) inpSize.height = frame.rows; - blobFromImage(frame, blob, 1.0, inpSize, Scalar(), swapRB, false, CV_8U); + Size size(inpSize.width <= 0 ? frame.cols : inpSize.width, inpSize.height <= 0 ? frame.rows : inpSize.height); - // Run a model. - net.setInput(blob, "", scale, mean); - if (net.getLayer(0)->outputNameToIndex("im_info") != -1) // Faster-RCNN or R-FCN + // Prepare the blob from the image + Mat inp; + if(framework == "weights"){ // checks whether model is darknet + blobFromImage(frame, inp, scale, size, meanv, swapRB, false, CV_32F); + } + else{ + //![preprocess_call] + Image2BlobParams imgParams( + scale, + size, + meanv, + swapRB, + CV_32F, + DNN_LAYOUT_NCHW, + paddingMode, + paddingValue); + + inp = blobFromImageWithParams(frame, imgParams); + //![preprocess_call] + } + + // Set the blob as the network input + net.setInput(inp); + + // Check if the model is Faster-RCNN or R-FCN + if (net.getLayer(0)->outputNameToIndex("im_info") != -1) { - resize(frame, frame, inpSize); - Mat imInfo = (Mat_(1, 3) << inpSize.height, inpSize.width, 1.6f); + // Resize the frame and prepare imInfo + resize(frame, frame, size); + Mat imInfo = (Mat_(1, 3) << size.height, size.width, 1.6f); net.setInput(imInfo, "im_info"); } } -void postprocess(Mat& frame, const std::vector& outs, Net& net, int backend) +void yoloPostProcessing( + const vector& outs, + vector& keep_classIds, + vector& keep_confidences, + vector& keep_boxes, + float conf_threshold, + float iou_threshold, + const string& yolo_name) { - static std::vector outLayers = net.getUnconnectedOutLayers(); - static std::string outLayerType = net.getLayer(outLayers[0])->type; + // Retrieve + vector classIds; + vector confidences; + vector boxes; + + vector outs_copy = outs; + + if (yolo_name == "yolov8") + { + transposeND(outs_copy[0], {0, 2, 1}, outs_copy[0]); + } + + if (yolo_name == "yolonas") + { + // outs contains 2 elements of shape [1, 8400, 80] and [1, 8400, 4]. Concat them to get [1, 8400, 84] + Mat concat_out; + // squeeze the first dimension + outs_copy[0] = outs_copy[0].reshape(1, outs_copy[0].size[1]); + outs_copy[1] = outs_copy[1].reshape(1, outs_copy[1].size[1]); + hconcat(outs_copy[1], outs_copy[0], concat_out); + outs_copy[0] = concat_out; + // remove the second element + outs_copy.pop_back(); + // unsqueeze the first dimension + outs_copy[0] = outs_copy[0].reshape(0, vector{1, 8400, 84}); + } + + for (auto preds : outs_copy) + { + preds = preds.reshape(1, preds.size[1]); // [1, 8400, 85] -> [8400, 85] + for (int i = 0; i < preds.rows; ++i) + { + // filter out non-object + float obj_conf = (yolo_name == "yolov8" || yolo_name == "yolonas") ? 1.0f : preds.at(i, 4); + if (obj_conf < conf_threshold) + continue; + + Mat scores = preds.row(i).colRange((yolo_name == "yolov8" || yolo_name == "yolonas") ? 4 : 5, preds.cols); + double conf; + Point maxLoc; + minMaxLoc(scores, 0, &conf, 0, &maxLoc); + + conf = (yolo_name == "yolov8" || yolo_name == "yolonas") ? conf : conf * obj_conf; + if (conf < conf_threshold) + continue; + + // get bbox coords + float* det = preds.ptr(i); + double cx = det[0]; + double cy = det[1]; + double w = det[2]; + double h = det[3]; + + // [x1, y1, x2, y2] + if (yolo_name == "yolonas") { + boxes.push_back(Rect2d(cx, cy, w, h)); + } else { + boxes.push_back(Rect2d(cx - 0.5 * w, cy - 0.5 * h, + cx + 0.5 * w, cy + 0.5 * h)); + } + classIds.push_back(maxLoc.x); + confidences.push_back(static_cast(conf)); + } + } + + // NMS + vector keep_idx; + NMSBoxes(boxes, confidences, conf_threshold, iou_threshold, keep_idx); + + for (auto i : keep_idx) + { + keep_classIds.push_back(classIds[i]); + keep_confidences.push_back(confidences[i]); + keep_boxes.push_back(boxes[i]); + } +} + +void postprocess(Mat& frame, const vector& outs, Net& net, int backend, vector& classIds, vector& confidences, vector& boxes, const string yolo_name) +{ + static vector outLayers = net.getUnconnectedOutLayers(); + static string outLayerType = net.getLayer(outLayers[0])->type; - std::vector classIds; - std::vector confidences; - std::vector boxes; if (outLayerType == "DetectionOutput") { // Network produces output blob with a shape 1x1xNx7 where N is a number of @@ -405,14 +580,46 @@ void postprocess(Mat& frame, const std::vector& outs, Net& net, int backend } } } - else - CV_Error(Error::StsNotImplemented, "Unknown output layer type: " + outLayerType); + else if (outLayerType == "Identity") + { + //![forward_buffers] + vector keep_classIds; + vector keep_confidences; + vector keep_boxes; + //![forward_buffers] - // NMS is used inside Region layer only on DNN_BACKEND_OPENCV for another backends we need NMS in sample - // or NMS is required if number of outputs > 1 + //![postprocess] + yoloPostProcessing(outs, keep_classIds, keep_confidences, keep_boxes, confThreshold, nmsThreshold, yolo_name); + //![postprocess] + + for (size_t i = 0; i < keep_classIds.size(); ++i) + { + classIds.push_back(keep_classIds[i]); + confidences.push_back(keep_confidences[i]); + Rect2d box = keep_boxes[i]; + boxes.push_back(Rect(cvFloor(box.x), cvFloor(box.y), cvFloor(box.width-box.x), cvFloor(box.height-box.y))); + } + if (framework == "onnx"){ + Image2BlobParams paramNet; + paramNet.scalefactor = scale; + paramNet.size = Size(inpWidth, inpHeight); + paramNet.mean = meanv; + paramNet.swapRB = swapRB; + paramNet.paddingmode = paddingMode; + + paramNet.blobRectsToImageRects(boxes, boxes, frame.size()); + } + } + else + { + CV_Error(Error::StsNotImplemented, "Unknown output layer type: " + outLayerType); + } + + // NMS is used inside Region layer only on DNN_BACKEND_OPENCV for other backends we need NMS in sample + // or NMS is required if the number of outputs > 1 if (outLayers.size() > 1 || (outLayerType == "Region" && backend != DNN_BACKEND_OPENCV)) { - std::map > class2indices; + map > class2indices; for (size_t i = 0; i < classIds.size(); i++) { if (confidences[i] >= confThreshold) @@ -420,20 +627,20 @@ void postprocess(Mat& frame, const std::vector& outs, Net& net, int backend class2indices[classIds[i]].push_back(i); } } - std::vector nmsBoxes; - std::vector nmsConfidences; - std::vector nmsClassIds; - for (std::map >::iterator it = class2indices.begin(); it != class2indices.end(); ++it) + vector nmsBoxes; + vector nmsConfidences; + vector nmsClassIds; + for (map >::iterator it = class2indices.begin(); it != class2indices.end(); ++it) { - std::vector localBoxes; - std::vector localConfidences; - std::vector classIndices = it->second; + vector localBoxes; + vector localConfidences; + vector classIndices = it->second; for (size_t i = 0; i < classIndices.size(); i++) { localBoxes.push_back(boxes[classIndices[i]]); localConfidences.push_back(confidences[classIndices[i]]); } - std::vector nmsIndices; + vector nmsIndices; NMSBoxes(localBoxes, localConfidences, confThreshold, nmsThreshold, nmsIndices); for (size_t i = 0; i < nmsIndices.size(); i++) { @@ -447,36 +654,49 @@ void postprocess(Mat& frame, const std::vector& outs, Net& net, int backend classIds = nmsClassIds; confidences = nmsConfidences; } - - for (size_t idx = 0; idx < boxes.size(); ++idx) - { - Rect box = boxes[idx]; - drawPred(classIds[idx], confidences[idx], box.x, box.y, - box.x + box.width, box.y + box.height, frame); - } } -void drawPred(int classId, float conf, int left, int top, int right, int bottom, Mat& frame) +void drawPred(vector& classIds, vector& confidences, vector& boxes, Mat& frame, FontFace& sans, int stdSize, int stdWeight, int stdImgSize, int stdThickness) { - rectangle(frame, Point(left, top), Point(right, bottom), Scalar(0, 255, 0)); + int imgWidth = max(frame.rows, frame.cols); + int size = (stdSize*imgWidth)/stdImgSize; + int weight = (stdWeight*imgWidth)/stdImgSize; + int thickness = (stdThickness*imgWidth)/stdImgSize; - std::string label = format("%.2f", conf); - if (!classes.empty()) - { - CV_Assert(classId < (int)classes.size()); - label = classes[classId] + ": " + label; + for (size_t idx = 0; idx < boxes.size(); ++idx){ + Scalar boxColor = getColor(classIds[idx]); + int left = boxes[idx].x; + int top = boxes[idx].y; + int right = boxes[idx].x + boxes[idx].width; + int bottom = boxes[idx].y + boxes[idx].height; + rectangle(frame, Point(left, top), Point(right, bottom), boxColor, thickness); + + string label = format("%.2f", confidences[idx]); + if (!labels.empty()) + { + CV_Assert(classIds[idx] < (int)labels.size()); + label = labels[classIds[idx]] + ": " + label; + } + + Rect r = getTextSize(Size(), label, Point(), sans, size, weight); + int baseline = r.y + r.height; + Size labelSize = Size(r.width, r.height + size/4 - baseline); + + top = max(top-thickness/2, labelSize.height); + rectangle(frame, Point(left-thickness/2, top-(labelSize.height)), + Point(left + labelSize.width, top), boxColor, FILLED); + putText(frame, label, Point(left, top-size/4), getTextColor(boxColor), sans, size, weight); } - - int baseLine; - Size labelSize = getTextSize(label, FONT_HERSHEY_SIMPLEX, 0.5, 1, &baseLine); - - top = max(top, labelSize.height); - rectangle(frame, Point(left, top - labelSize.height), - Point(left + labelSize.width, top + baseLine), Scalar::all(255), FILLED); - putText(frame, label, Point(left, top), FONT_HERSHEY_SIMPLEX, 0.5, Scalar()); } void callback(int pos, void*) { confThreshold = pos * 0.01f; } + +Scalar getColor(int classId) { + int r = min((classId >> 0 & 1) * 128 + (classId >> 3 & 1) * 64 + (classId >> 6 & 1) * 32 + 80, 255); + int g = min((classId >> 1 & 1) * 128 + (classId >> 4 & 1) * 64 + (classId >> 7 & 1) * 32 + 40, 255); + int b = min((classId >> 2 & 1) * 128 + (classId >> 5 & 1) * 64 + (classId >> 8 & 1) * 32 + 40, 255); + return Scalar(b, g, r); +} diff --git a/samples/dnn/object_detection.py b/samples/dnn/object_detection.py index 208a131b83..f2171a5716 100644 --- a/samples/dnn/object_detection.py +++ b/samples/dnn/object_detection.py @@ -12,10 +12,22 @@ from tf_text_graph_common import readTextMessage from tf_text_graph_ssd import createSSDGraph from tf_text_graph_faster_rcnn import createFasterRCNNGraph -backends = (cv.dnn.DNN_BACKEND_DEFAULT, cv.dnn.DNN_BACKEND_INFERENCE_ENGINE, cv.dnn.DNN_BACKEND_OPENCV, - cv.dnn.DNN_BACKEND_VKCOM, cv.dnn.DNN_BACKEND_CUDA) -targets = (cv.dnn.DNN_TARGET_CPU, cv.dnn.DNN_TARGET_OPENCL, cv.dnn.DNN_TARGET_OPENCL_FP16, cv.dnn.DNN_TARGET_MYRIAD, cv.dnn.DNN_TARGET_HDDL, - cv.dnn.DNN_TARGET_VULKAN, cv.dnn.DNN_TARGET_CUDA, cv.dnn.DNN_TARGET_CUDA_FP16) +def help(): + print( + ''' + Firstly, download required models using `download_models.py` (if not already done). Set environment variable OPENCV_DOWNLOAD_CACHE_DIR to specify where models should be downloaded. Also, point OPENCV_SAMPLES_DATA_PATH to opencv/samples/data.\n"\n + + To run: + python object_detection.py model_name(e.g yolov8) --input=path/to/your/input/image/or/video (don't pass --input to use device camera) + + Sample command: + python object_detection.py yolov8 --input=path/to/image + Model path can also be specified using --model argument + ''' + ) + +backends = ("default", "openvino", "opencv", "vkcom", "cuda") +targets = ("cpu", "opencl", "opencl_fp16", "ncs2_vpu", "hddl_vpu", "vulkan", "cuda", "cuda_fp16") parser = argparse.ArgumentParser(add_help=False) parser.add_argument('--zoo', default=os.path.join(os.path.dirname(os.path.abspath(__file__)), 'models.yml'), @@ -30,27 +42,27 @@ parser.add_argument('--framework', choices=['caffe', 'tensorflow', 'darknet', 'd 'Detect it automatically if it does not set.') parser.add_argument('--thr', type=float, default=0.5, help='Confidence threshold') parser.add_argument('--nms', type=float, default=0.4, help='Non-maximum suppression threshold') -parser.add_argument('--backend', choices=backends, default=cv.dnn.DNN_BACKEND_DEFAULT, type=int, +parser.add_argument('--backend', default="default", type=str, choices=backends, help="Choose one of computation backends: " - "%d: automatically (by default), " - "%d: Intel's Deep Learning Inference Engine (https://software.intel.com/openvino-toolkit), " - "%d: OpenCV implementation, " - "%d: VKCOM, " - "%d: CUDA" % backends) -parser.add_argument('--target', choices=targets, default=cv.dnn.DNN_TARGET_CPU, type=int, - help='Choose one of target computation devices: ' - '%d: CPU target (by default), ' - '%d: OpenCL, ' - '%d: OpenCL fp16 (half-float precision), ' - '%d: NCS2 VPU, ' - '%d: HDDL VPU, ' - '%d: Vulkan, ' - '%d: CUDA, ' - '%d: CUDA fp16 (half-float preprocess)' % targets) + "default: automatically (by default), " + "openvino: Intel's Deep Learning Inference Engine (https://software.intel.com/openvino-toolkit), " + "opencv: OpenCV implementation, " + "vkcom: VKCOM, " + "cuda: CUDA, " + "webnn: WebNN") +parser.add_argument('--target', default="cpu", type=str, choices=targets, + help="Choose one of target computation devices: " + "cpu: CPU target (by default), " + "opencl: OpenCL, " + "opencl_fp16: OpenCL fp16 (half-float precision), " + "ncs2_vpu: NCS2 VPU, " + "hddl_vpu: HDDL VPU, " + "vulkan: Vulkan, " + "cuda: CUDA, " + "cuda_fp16: CUDA fp16 (half-float preprocess)") parser.add_argument('--async', type=int, default=0, - dest='asyncN', - help='Number of asynchronous forwards at the same time. ' - 'Choose 0 for synchronous mode') + dest='use_threads', + help='Choose 0 for synchronous mode and 1 for asynchronous mode') args, _ = parser.parse_known_args() add_preproc_args(args.zoo, parser, 'object_detection') parser = argparse.ArgumentParser(parents=[parser], @@ -58,9 +70,14 @@ parser = argparse.ArgumentParser(parents=[parser], formatter_class=argparse.ArgumentDefaultsHelpFormatter) args = parser.parse_args() -args.model = findFile(args.model) -args.config = findFile(args.config) -args.classes = findFile(args.classes) +if args.alias is None or hasattr(args, 'help'): + help() + exit(1) + +args.model = findModel(args.model, args.sha1) +if args.config is not None: + args.config = findFile(args.config) +args.labels = findFile(args.labels) # If config specified, try to load it as TensorFlow Object Detection API's pipeline. config = readTextMessage(args.config) @@ -77,40 +94,38 @@ if 'model' in config: # Load names of classes -classes = None -if args.classes: - with open(args.classes, 'rt') as f: - classes = f.read().rstrip('\n').split('\n') +labels = None +if args.labels: + with open(args.labels, 'rt') as f: + labels = f.read().rstrip('\n').split('\n') # Load a network net = cv.dnn.readNet(args.model, args.config, args.framework) -net.setPreferableBackend(args.backend) -net.setPreferableTarget(args.target) +net.setPreferableBackend(get_backend_id(args.backend)) +net.setPreferableTarget(get_target_id(args.target)) outNames = net.getUnconnectedOutLayersNames() confThreshold = args.thr nmsThreshold = args.nms +stdSize = 0.8 +stdWeight = 2 +stdImgSize = 512 +asyncN = 0 + +def get_color(class_id): + r = min((class_id >> 0 & 1) * 128 + (class_id >> 3 & 1) * 64 + (class_id >> 6 & 1) * 32 + 80, 255) + g = min((class_id >> 1 & 1) * 128 + (class_id >> 4 & 1) * 64 + (class_id >> 7 & 1) * 32 + 40, 255) + b = min((class_id >> 2 & 1) * 128 + (class_id >> 5 & 1) * 64 + (class_id >> 8 & 1) * 32 + 40, 255) + return (int(b), int(g), int(r)) + +def get_text_color(bg_color): + luminance = 0.299 * bg_color[2] + 0.587 * bg_color[1] + 0.114 * bg_color[0] + return (0, 0, 0) if luminance > 128 else (255, 255, 255) def postprocess(frame, outs): frameHeight = frame.shape[0] frameWidth = frame.shape[1] - def drawPred(classId, conf, left, top, right, bottom): - # Draw a bounding box. - cv.rectangle(frame, (left, top), (right, bottom), (0, 255, 0)) - - label = '%.2f' % conf - - # Print a label of class. - if classes: - assert(classId < len(classes)) - label = '%s: %s' % (classes[classId], label) - - labelSize, baseLine = cv.getTextSize(label, cv.FONT_HERSHEY_SIMPLEX, 0.5, 1) - top = max(top, labelSize[1]) - cv.rectangle(frame, (left, top - labelSize[1]), (left + labelSize[0], top + baseLine), (255, 255, 255), cv.FILLED) - cv.putText(frame, label, (left, top), cv.FONT_HERSHEY_SIMPLEX, 0.5, (0, 0, 0)) - layerNames = net.getLayerNames() lastLayerId = net.getLayerId(layerNames[-1]) lastLayer = net.getLayer(lastLayerId) @@ -194,17 +209,33 @@ def postprocess(frame, outs): else: indices = np.arange(0, len(classIds)) + return boxes, classIds, confidences, indices + +def drawPred(classIds, confidences, boxes, indices, fontSize, fontThickness): for i in indices: box = boxes[i] left = box[0] top = box[1] - width = box[2] - height = box[3] - drawPred(classIds[i], confidences[i], left, top, left + width, top + height) + right = box[0] + box[2] + bottom = box[1] + box[3] + bg_color = get_color(classIds[i]) + cv.rectangle(frame, (left, top), (right, bottom), bg_color, fontThickness) + + label = '%.2f' % confidences[i] + + # Print a label of class. + if labels: + assert(classIds[i] < len(labels)) + label = '%s: %s' % (labels[classIds[i]], label) + + labelSize, baseLine = cv.getTextSize(label, cv.FONT_HERSHEY_SIMPLEX, fontSize, fontThickness) + top = max(top, labelSize[1]) + cv.rectangle(frame, (int(left-fontThickness/2), top - labelSize[1]), (left + labelSize[0], top + baseLine), bg_color, cv.FILLED) + cv.putText(frame, label, (left, top-fontThickness), cv.FONT_HERSHEY_SIMPLEX, fontSize, get_text_color(bg_color), fontThickness) # Process inputs winName = 'Deep learning object detection in OpenCV' -cv.namedWindow(winName, cv.WINDOW_NORMAL) +cv.namedWindow(winName, cv.WINDOW_AUTOSIZE) def callback(pos): global confThreshold @@ -252,7 +283,7 @@ def framesThreadBody(): processedFramesQueue = queue.Queue() predictionsQueue = QueueFPS() def processingThreadBody(): - global processedFramesQueue, predictionsQueue, args, process + global processedFramesQueue, predictionsQueue, args, process, asyncN futureOutputs = [] while process: @@ -261,8 +292,8 @@ def processingThreadBody(): try: frame = framesQueue.get_nowait() - if args.asyncN: - if len(futureOutputs) == args.asyncN: + if asyncN: + if len(futureOutputs) == asyncN: frame = None # Skip the frame else: framesQueue.queue.clear() # Skip the rest of frames @@ -277,7 +308,7 @@ def processingThreadBody(): # Create a 4D blob from a frame. inpWidth = args.width if args.width else frameWidth inpHeight = args.height if args.height else frameHeight - blob = cv.dnn.blobFromImage(frame, size=(inpWidth, inpHeight), swapRB=args.rgb, ddepth=cv.CV_8U) + blob = cv.dnn.blobFromImage(frame, size=(inpWidth, inpHeight), swapRB=args.rgb, ddepth=cv.CV_32F) processedFramesQueue.put(frame) # Run a model @@ -286,7 +317,7 @@ def processingThreadBody(): frame = cv.resize(frame, (inpWidth, inpHeight)) net.setInput(np.array([[inpHeight, inpWidth, 1.6]], dtype=np.float32), 'im_info') - if args.asyncN: + if asyncN: futureOutputs.append(net.forwardAsync()) else: outs = net.forward(outNames) @@ -298,40 +329,68 @@ def processingThreadBody(): del futureOutputs[0] +if args.use_threads: + framesThread = Thread(target=framesThreadBody) + framesThread.start() -framesThread = Thread(target=framesThreadBody) -framesThread.start() + processingThread = Thread(target=processingThreadBody) + processingThread.start() -processingThread = Thread(target=processingThreadBody) -processingThread.start() + # + # Postprocessing and rendering loop + # + while cv.waitKey(1) < 0: + try: + # Request prediction first because they put after frames + outs = predictionsQueue.get_nowait() + frame = processedFramesQueue.get_nowait() + imgWidth = max(frame.shape[:2]) + fontSize = (stdSize*imgWidth)/stdImgSize + fontThickness = max(1,(stdWeight*imgWidth)//stdImgSize) -# -# Postprocessing and rendering loop -# -while cv.waitKey(1) < 0: - try: - # Request prediction first because they put after frames - outs = predictionsQueue.get_nowait() - frame = processedFramesQueue.get_nowait() + boxes, classIds, confidences, indices = postprocess(frame, outs) + drawPred(classIds, confidences, boxes, indices, fontSize, fontThickness) + fontSize = fontSize/2 + # Put efficiency information. + if predictionsQueue.counter > 1: + label = 'Camera: %.2f FPS' % (framesQueue.getFPS()) + cv.rectangle(frame, (0, 0), (int(260*fontSize), int(80*fontSize)), (255,255,255), cv.FILLED) + cv.putText(frame, label, (0, int(25*fontSize)), cv.FONT_HERSHEY_SIMPLEX, fontSize, (0, 0, 0), fontThickness) - postprocess(frame, outs) + label = 'Network: %.2f FPS' % (predictionsQueue.getFPS()) + cv.putText(frame, label, (0, int(2*25*fontSize)), cv.FONT_HERSHEY_SIMPLEX, fontSize, (0, 0, 0), fontThickness) - # Put efficiency information. - if predictionsQueue.counter > 1: - label = 'Camera: %.2f FPS' % (framesQueue.getFPS()) - cv.putText(frame, label, (0, 15), cv.FONT_HERSHEY_SIMPLEX, 0.5, (0, 255, 0)) + label = 'Skipped frames: %d' % (framesQueue.counter - predictionsQueue.counter) + cv.putText(frame, label, (0, int(3*25*fontSize)), cv.FONT_HERSHEY_SIMPLEX, fontSize, (0, 0, 0), fontThickness) - label = 'Network: %.2f FPS' % (predictionsQueue.getFPS()) - cv.putText(frame, label, (0, 30), cv.FONT_HERSHEY_SIMPLEX, 0.5, (0, 255, 0)) - - label = 'Skipped frames: %d' % (framesQueue.counter - predictionsQueue.counter) - cv.putText(frame, label, (0, 45), cv.FONT_HERSHEY_SIMPLEX, 0.5, (0, 255, 0)) - - cv.imshow(winName, frame) - except queue.Empty: - pass + cv.imshow(winName, frame) + except queue.Empty: + pass -process = False -framesThread.join() -processingThread.join() + process = False + framesThread.join() + processingThread.join() + +else: + # Non-threaded processing if --async is 0 + while cv.waitKey(1) < 0: + hasFrame, frame = cap.read() + if not hasFrame: + cv.waitKey() + break + + frameHeight = frame.shape[0] + frameWidth = frame.shape[1] + + inpWidth = args.width if args.width else frameWidth + inpHeight = args.height if args.height else frameHeight + blob = cv.dnn.blobFromImage(frame, size=(inpWidth, inpHeight), swapRB=args.rgb, ddepth=cv.CV_32F) + + net.setInput(blob, scalefactor=args.scale, mean=args.mean) + outs = net.forward(outNames) + + boxes, classIds, confidences, indices = postprocess(frame, outs) + drawPred(classIds, confidences, boxes, indices, (stdSize*max(frame.shape[:2]))/stdImgSize, (stdWeight*max(frame.shape[:2]))//stdImgSize) + + cv.imshow(winName, frame) \ No newline at end of file diff --git a/samples/dnn/yolo_detector.cpp b/samples/dnn/yolo_detector.cpp deleted file mode 100644 index bd82acff4a..0000000000 --- a/samples/dnn/yolo_detector.cpp +++ /dev/null @@ -1,382 +0,0 @@ -/** - * @file yolo_detector.cpp - * @brief Yolo Object Detection Sample - * @author OpenCV team - */ - -//![includes] -#include -#include -#include -#include -#include -#include "iostream" -#include "common.hpp" -#include -//![includes] - -using namespace cv; -using namespace cv::dnn; - -void getClasses(std::string classesFile); -void drawPrediction(int classId, float conf, int left, int top, int right, int bottom, Mat& frame); -void yoloPostProcessing( - std::vector& outs, - std::vector& keep_classIds, - std::vector& keep_confidences, - std::vector& keep_boxes, - float conf_threshold, - float iou_threshold, - const std::string& model_name, - const int nc -); - -std::vector classes; - - -std::string keys = - "{ help h | | Print help message. }" - "{ device | 0 | camera device number. }" - "{ model | onnx/models/yolox_s_inf_decoder.onnx | Default model. }" - "{ yolo | yolox | yolo model version. }" - "{ input i | | Path to input image or video file. Skip this argument to capture frames from a camera. }" - "{ classes | | Optional path to a text file with names of classes to label detected objects. }" - "{ nc | 80 | Number of classes. Default is 80 (coming from COCO dataset). }" - "{ thr | .5 | Confidence threshold. }" - "{ nms | .4 | Non-maximum suppression threshold. }" - "{ mean | 0.0 | Normalization constant. }" - "{ scale | 1.0 | Preprocess input image by multiplying on a scale factor. }" - "{ width | 640 | Preprocess input image by resizing to a specific width. }" - "{ height | 640 | Preprocess input image by resizing to a specific height. }" - "{ rgb | 1 | Indicate that model works with RGB input images instead BGR ones. }" - "{ padvalue | 114.0 | padding value. }" - "{ paddingmode | 2 | Choose one of computation backends: " - "0: resize to required input size without extra processing, " - "1: Image will be cropped after resize, " - "2: Resize image to the desired size while preserving the aspect ratio of original image }" - "{ backend | 0 | Choose one of computation backends: " - "0: automatically (by default), " - "1: Halide language (http://halide-lang.org/), " - "2: Intel's Deep Learning Inference Engine (https://software.intel.com/openvino-toolkit), " - "3: OpenCV implementation, " - "4: VKCOM, " - "5: CUDA }" - "{ target | 0 | Choose one of target computation devices: " - "0: CPU target (by default), " - "1: OpenCL, " - "2: OpenCL fp16 (half-float precision), " - "3: VPU, " - "4: Vulkan, " - "6: CUDA, " - "7: CUDA fp16 (half-float preprocess) }" - "{ async | 0 | Number of asynchronous forwards at the same time. " - "Choose 0 for synchronous mode }"; - -void getClasses(std::string classesFile) -{ - std::ifstream ifs(classesFile.c_str()); - if (!ifs.is_open()) - CV_Error(Error::StsError, "File " + classesFile + " not found"); - std::string line; - while (std::getline(ifs, line)) - classes.push_back(line); -} - -void drawPrediction(int classId, float conf, int left, int top, int right, int bottom, Mat& frame) -{ - rectangle(frame, Point(left, top), Point(right, bottom), Scalar(0, 255, 0)); - - std::string label = format("%.2f", conf); - if (!classes.empty()) - { - CV_Assert(classId < (int)classes.size()); - label = classes[classId] + ": " + label; - } - - int baseLine; - Size labelSize = getTextSize(label, FONT_HERSHEY_SIMPLEX, 0.5, 1, &baseLine); - - top = max(top, labelSize.height); - rectangle(frame, Point(left, top - labelSize.height), - Point(left + labelSize.width, top + baseLine), Scalar::all(255), FILLED); - putText(frame, label, Point(left, top), FONT_HERSHEY_SIMPLEX, 0.5, Scalar()); -} - -void yoloPostProcessing( - std::vector& outs, - std::vector& keep_classIds, - std::vector& keep_confidences, - std::vector& keep_boxes, - float conf_threshold, - float iou_threshold, - const std::string& model_name, - const int nc=80) -{ - // Retrieve - std::vector classIds; - std::vector confidences; - std::vector boxes; - - if (model_name == "yolov8" || model_name == "yolov10" || - model_name == "yolov9") - { - cv::transposeND(outs[0], {0, 2, 1}, outs[0]); - } - - if (model_name == "yolonas") - { - // outs contains 2 elemets of shape [1, 8400, 80] and [1, 8400, 4]. Concat them to get [1, 8400, 84] - Mat concat_out; - // squeeze the first dimension - outs[0] = outs[0].reshape(1, outs[0].size[1]); - outs[1] = outs[1].reshape(1, outs[1].size[1]); - cv::hconcat(outs[1], outs[0], concat_out); - outs[0] = concat_out; - // remove the second element - outs.pop_back(); - // unsqueeze the first dimension - outs[0] = outs[0].reshape(0, std::vector{1, 8400, nc + 4}); - } - - // assert if last dim is 85 or 84 - CV_CheckEQ(outs[0].dims, 3, "Invalid output shape. The shape should be [1, #anchors, 85 or 84]"); - CV_CheckEQ((outs[0].size[2] == nc + 5 || outs[0].size[2] == 80 + 4), true, "Invalid output shape: "); - - for (auto preds : outs) - { - preds = preds.reshape(1, preds.size[1]); // [1, 8400, 85] -> [8400, 85] - for (int i = 0; i < preds.rows; ++i) - { - // filter out non object - float obj_conf = (model_name == "yolov8" || model_name == "yolonas" || - model_name == "yolov9" || model_name == "yolov10") ? 1.0f : preds.at(i, 4) ; - if (obj_conf < conf_threshold) - continue; - - Mat scores = preds.row(i).colRange((model_name == "yolov8" || model_name == "yolonas" || model_name == "yolov9" || model_name == "yolov10") ? 4 : 5, preds.cols); - double conf; - Point maxLoc; - minMaxLoc(scores, 0, &conf, 0, &maxLoc); - - conf = (model_name == "yolov8" || model_name == "yolonas" || model_name == "yolov9" || model_name == "yolov10") ? conf : conf * obj_conf; - if (conf < conf_threshold) - continue; - - // get bbox coords - float* det = preds.ptr(i); - double cx = det[0]; - double cy = det[1]; - double w = det[2]; - double h = det[3]; - - // [x1, y1, x2, y2] - if (model_name == "yolonas" || model_name == "yolov10"){ - boxes.push_back(Rect2d(cx, cy, w, h)); - } else { - boxes.push_back(Rect2d(cx - 0.5 * w, cy - 0.5 * h, - cx + 0.5 * w, cy + 0.5 * h)); - } - classIds.push_back(maxLoc.x); - confidences.push_back(static_cast(conf)); - } - } - - // NMS - std::vector keep_idx; - NMSBoxes(boxes, confidences, conf_threshold, iou_threshold, keep_idx); - - for (auto i : keep_idx) - { - keep_classIds.push_back(classIds[i]); - keep_confidences.push_back(confidences[i]); - keep_boxes.push_back(boxes[i]); - } -} - -/** - * @function main - * @brief Main function - */ -int main(int argc, char** argv) -{ - CommandLineParser parser(argc, argv, keys); - parser.about("Use this script to run object detection deep learning networks using OpenCV."); - if (parser.has("help")) - { - parser.printMessage(); - return 0; - } - - CV_Assert(parser.has("model")); - CV_Assert(parser.has("yolo")); - // if model is default, use findFile to get the full path otherwise use the given path - std::string weightPath = findFile(parser.get("model")); - std::string yolo_model = parser.get("yolo"); - int nc = parser.get("nc"); - - float confThreshold = parser.get("thr"); - float nmsThreshold = parser.get("nms"); - //![preprocess_params] - float paddingValue = parser.get("padvalue"); - bool swapRB = parser.get("rgb"); - int inpWidth = parser.get("width"); - int inpHeight = parser.get("height"); - Scalar scale = parser.get("scale"); - Scalar mean = parser.get("mean"); - ImagePaddingMode paddingMode = static_cast(parser.get("paddingmode")); - //![preprocess_params] - - // check if yolo model is valid - if (yolo_model != "yolov5" && yolo_model != "yolov6" - && yolo_model != "yolov7" && yolo_model != "yolov8" - && yolo_model != "yolov10" && yolo_model !="yolov9" - && yolo_model != "yolox" && yolo_model != "yolonas") - CV_Error(Error::StsError, "Invalid yolo model: " + yolo_model); - - // get classes - if (parser.has("classes")) - { - getClasses(findFile(parser.get("classes"))); - } - - // load model - //![read_net] - Net net = readNet(weightPath); - int backend = parser.get("backend"); - net.setPreferableBackend(backend); - net.setPreferableTarget(parser.get("target")); - //![read_net] - - VideoCapture cap; - Mat img; - bool isImage = false; - bool isCamera = false; - - // Check if input is given - if (parser.has("input")) - { - String input = parser.get("input"); - // Check if the input is an image - if (input.find(".jpg") != String::npos || input.find(".png") != String::npos) - { - img = imread(findFile(input)); - if (img.empty()) - { - CV_Error(Error::StsError, "Cannot read image file: " + input); - } - isImage = true; - } - else - { - cap.open(input); - if (!cap.isOpened()) - { - CV_Error(Error::StsError, "Cannot open video " + input); - } - isCamera = true; - } - } - else - { - int cameraIndex = parser.get("device"); - cap.open(cameraIndex); - if (!cap.isOpened()) - { - CV_Error(Error::StsError, cv::format("Cannot open camera #%d", cameraIndex)); - } - isCamera = true; - } - - // image pre-processing - //![preprocess_call] - Size size(inpWidth, inpHeight); - Image2BlobParams imgParams( - scale, - size, - mean, - swapRB, - CV_32F, - DNN_LAYOUT_NCHW, - paddingMode, - paddingValue); - - // rescale boxes back to original image - Image2BlobParams paramNet; - paramNet.scalefactor = scale; - paramNet.size = size; - paramNet.mean = mean; - paramNet.swapRB = swapRB; - paramNet.paddingmode = paddingMode; - //![preprocess_call] - - //![forward_buffers] - std::vector outs; - std::vector keep_classIds; - std::vector keep_confidences; - std::vector keep_boxes; - std::vector boxes; - //![forward_buffers] - - Mat inp; - while (waitKey(1) < 0) - { - - if (isCamera) - cap >> img; - if (img.empty()) - { - std::cout << "Empty frame" << std::endl; - waitKey(); - break; - } - //![preprocess_call_func] - inp = blobFromImageWithParams(img, imgParams); - //![preprocess_call_func] - - //![forward] - net.setInput(inp); - net.forward(outs, net.getUnconnectedOutLayersNames()); - //![forward] - - //![postprocess] - yoloPostProcessing( - outs, keep_classIds, keep_confidences, keep_boxes, - confThreshold, nmsThreshold, - yolo_model, - nc); - //![postprocess] - - // covert Rect2d to Rect - //![draw_boxes] - for (auto box : keep_boxes) - { - boxes.push_back(Rect(cvFloor(box.x), cvFloor(box.y), cvFloor(box.width - box.x), cvFloor(box.height - box.y))); - } - - paramNet.blobRectsToImageRects(boxes, boxes, img.size()); - - for (size_t idx = 0; idx < boxes.size(); ++idx) - { - Rect box = boxes[idx]; - drawPrediction(keep_classIds[idx], keep_confidences[idx], box.x, box.y, - box.width + box.x, box.height + box.y, img); - } - - const std::string kWinName = "Yolo Object Detector"; - namedWindow(kWinName, WINDOW_NORMAL); - imshow(kWinName, img); - //![draw_boxes] - - outs.clear(); - keep_classIds.clear(); - keep_confidences.clear(); - keep_boxes.clear(); - boxes.clear(); - - if (isImage) - { - waitKey(); - break; - } - } -}