From d79e95c0184060453a34a31aa02f20c5667a263b Mon Sep 17 00:00:00 2001 From: Abhishek Gola Date: Fri, 19 Sep 2025 15:25:37 +0530 Subject: [PATCH] Merge pull request #27674 from abhishek-gola:nonmaxsuppression_layer_add Added nonmaxsuppression (NMS) layer to new DNN engine #27674 ### Pull Request Readiness Checklist See details at https://github.com/opencv/opencv/wiki/How_to_contribute#making-a-good-pull-request - [x] I agree to contribute to the project under Apache 2 License. - [x] To the best of my knowledge, the proposed patch is not based on a code under GPL or another license that is incompatible with OpenCV - [x] The PR is proposed to the proper branch - [x] There is a reference to the original bug report and related work - [x] There is accuracy test, performance test and test data in opencv_extra repository, if applicable Patch to opencv_extra has the same branch name. - [x] The feature is well documented and sample code can be built with the project CMake --- .../dnn/include/opencv2/dnn/all_layers.hpp | 6 + modules/dnn/src/init.cpp | 1 + .../src/layers/nonmaxsuppression_layer.cpp | 226 ++++++++++++++++++ modules/dnn/src/onnx/onnx_importer2.cpp | 8 + ...conformance_layer_filter__openvino.inl.hpp | 18 +- ...yer_filter_opencv_classic_denylist.inl.hpp | 9 + ..._conformance_layer_parser_denylist.inl.hpp | 9 - 7 files changed, 259 insertions(+), 18 deletions(-) create mode 100644 modules/dnn/src/layers/nonmaxsuppression_layer.cpp diff --git a/modules/dnn/include/opencv2/dnn/all_layers.hpp b/modules/dnn/include/opencv2/dnn/all_layers.hpp index eccf2db0ae..b4d794516b 100644 --- a/modules/dnn/include/opencv2/dnn/all_layers.hpp +++ b/modules/dnn/include/opencv2/dnn/all_layers.hpp @@ -1414,6 +1414,12 @@ CV__DNN_INLINE_NS_BEGIN static Ptr create(const LayerParams ¶ms); }; + class CV_EXPORTS NonMaxSuppressionLayer : public Layer + { + public: + static Ptr create(const LayerParams& params); + }; + class CV_EXPORTS ClipLayer : public Layer { public: static Ptr create(const LayerParams ¶ms); diff --git a/modules/dnn/src/init.cpp b/modules/dnn/src/init.cpp index 66f37eecca..94da807e87 100644 --- a/modules/dnn/src/init.cpp +++ b/modules/dnn/src/init.cpp @@ -96,6 +96,7 @@ void initializeLayerFactory() CV_DNN_REGISTER_LAYER_CLASS(Pad2, Pad2Layer); CV_DNN_REGISTER_LAYER_CLASS(NonZero, NonZeroLayer); CV_DNN_REGISTER_LAYER_CLASS(QuantizeLinear, QuantizeLinearLayer); + CV_DNN_REGISTER_LAYER_CLASS(NonMaxSuppression, NonMaxSuppressionLayer); CV_DNN_REGISTER_LAYER_CLASS(Range, RangeLayer); CV_DNN_REGISTER_LAYER_CLASS(Reshape, ReshapeLayer); CV_DNN_REGISTER_LAYER_CLASS(Reshape2, Reshape2Layer); diff --git a/modules/dnn/src/layers/nonmaxsuppression_layer.cpp b/modules/dnn/src/layers/nonmaxsuppression_layer.cpp new file mode 100644 index 0000000000..47e743913d --- /dev/null +++ b/modules/dnn/src/layers/nonmaxsuppression_layer.cpp @@ -0,0 +1,226 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// Copyright (C) 2025, BigVision LLC, all rights reserved. +// Third party copyrights are property of their respective owners. + +#include "../precomp.hpp" +#include "layers_common.hpp" +#include +#include +#include +#include + +namespace cv { +namespace dnn { + +// ONNX NonMaxSuppression (opset 10 and 11) +// Spec: https://onnx.ai/onnx/operators/onnx__NonMaxSuppression.html +// Inputs: boxes [B, N, 4], scores [B, C, N], (optional) max_output_boxes_per_class, iou_threshold, score_threshold +// Output: selected_indices [K, 3] (int64): [batch_idx, class_idx, box_idx] + +class NonMaxSuppressionLayerImpl CV_FINAL : public NonMaxSuppressionLayer +{ +public: + NonMaxSuppressionLayerImpl(const LayerParams& p) { + setParamsFrom(p); + centerPointBox = p.get("center_point_box", 0); + defaultMaxOut = p.get("default_max_output_boxes_per_class", 0); + defaultIouThr = p.get("default_iou_threshold", 0.f); + defaultScoreThr = p.get("default_score_threshold", 0.f); + } + + bool supportBackend(int backendId) CV_OVERRIDE { + return backendId == DNN_BACKEND_OPENCV; + } + + virtual bool dynamicOutputShapes() const CV_OVERRIDE { return true; } + + bool getMemoryShapes(const std::vector& inputs, const int, std::vector& outputs, std::vector&) const CV_OVERRIDE { + CV_Assert(inputs.size() >= 2); + const MatShape& boxes = inputs[0]; + const MatShape& scores = inputs[1]; + CV_Assert(boxes.size() == 3 && boxes[2] == 4); + CV_Assert(scores.size() == 3 && boxes[0] == scores[0] && boxes[1] == scores[2]); + + outputs.assign(1, MatShape({1, 3})); + return false; + } + + void getTypes(const std::vector&, const int requiredOutputs, const int requiredInternals, + std::vector& outputs, std::vector& internals) const CV_OVERRIDE { + outputs.assign(requiredOutputs, MatType(CV_64S)); + } + + void forward(InputArrayOfArrays inputs_arr, OutputArrayOfArrays outputs_arr, OutputArrayOfArrays) CV_OVERRIDE { + std::vector inputs, outputs; + + Mat boxesBlob, scoresBlob; + int maxOut = defaultMaxOut; + float iouThr = defaultIouThr; + float scoreThr = defaultScoreThr; + + auto inKind = inputs_arr.kind(); + if (inKind == _InputArray::STD_VECTOR_UMAT) { + std::vector uinputs; + inputs_arr.getUMatVector(uinputs); + CV_Assert(uinputs.size() >= 2); + boxesBlob = uinputs[0].getMat(ACCESS_READ); + scoresBlob = uinputs[1].getMat(ACCESS_READ); + if (uinputs.size() >= 3 && !uinputs[2].empty()) { + Mat tmp = uinputs[2].getMat(ACCESS_READ); + CV_Assert(tmp.channels() == 1 && tmp.total() == 1); + maxOut = tensorToScalar(tmp); + } + if (uinputs.size() >= 4 && !uinputs[3].empty()) { + Mat tmp = uinputs[3].getMat(ACCESS_READ); + CV_Assert(tmp.channels() == 1 && tmp.total() == 1); + iouThr = tensorToScalar(tmp); + } + if (uinputs.size() >= 5 && !uinputs[4].empty()) { + Mat tmp = uinputs[4].getMat(ACCESS_READ); + CV_Assert(tmp.channels() == 1 && tmp.total() == 1); + scoreThr = tensorToScalar(tmp); + } + } else { + inputs_arr.getMatVector(inputs); + CV_Assert(inputs.size() >= 2); + boxesBlob = inputs[0]; + scoresBlob = inputs[1]; + if (inputs.size() >= 3 && !inputs[2].empty()) { + CV_Assert(inputs[2].channels() == 1 && inputs[2].total() == 1); + maxOut = tensorToScalar(inputs[2]); + } + if (inputs.size() >= 4 && !inputs[3].empty()) { + CV_Assert(inputs[3].channels() == 1 && inputs[3].total() == 1); + iouThr = tensorToScalar(inputs[3]); + } + if (inputs.size() >= 5 && !inputs[4].empty()) { + CV_Assert(inputs[4].channels() == 1 && inputs[4].total() == 1); + scoreThr = tensorToScalar(inputs[4]); + } + } + + const int B = boxesBlob.size[0]; + const int C = scoresBlob.size[1]; + const int tasks = B * C; + + std::vector>> tripletsPerBC; + run(boxesBlob, scoresBlob, maxOut, iouThr, scoreThr, tripletsPerBC); + + int K = 0; + for (int t = 0; t < tasks; ++t) K += (int)tripletsPerBC[t].size(); + + Mat outMat; + auto outKind = outputs_arr.kind(); + if (outKind == _InputArray::STD_VECTOR_MAT) { + std::vector& outs = outputs_arr.getMatVecRef(); + CV_Assert(outs.size() == 1); + outs[0].create(K, 3, CV_64S); + outMat = outs[0]; + } else if (outKind == _InputArray::STD_VECTOR_UMAT) { + std::vector uouts; + outputs_arr.getUMatVector(uouts); + CV_Assert(uouts.size() == 1); + uouts[0].fit(MatShape({K, 3}), CV_64S); + outMat = uouts[0].getMat(ACCESS_WRITE); + } else { + outputs_arr.getMatRef(0).create(K, 3, CV_64S); + outMat = outputs_arr.getMatRef(0); + } + + auto* out = outMat.ptr(); + std::vector offsets(tasks + 1, 0); + for (int t = 0; t < tasks; ++t) + offsets[t + 1] = offsets[t] + (int)tripletsPerBC[t].size() * 3; + parallel_for_(Range(0, tasks), [&](const Range& r){ + for (int t = r.start; t < r.end; ++t) { + int64_t* dst = out + offsets[t]; + const auto& v = tripletsPerBC[t]; + for (const auto& trip : v) { + *dst++ = trip[0]; *dst++ = trip[1]; *dst++ = trip[2]; + } + } + }); + } + +private: + void run(const Mat& boxesBlob, const Mat& scoresBlob, int maxOut, float iouThr, float scoreThr, + std::vector>>& tripletsPerBC) const + { + const int B = boxesBlob.size[0]; + const int N = boxesBlob.size[1]; + const int C = scoresBlob.size[1]; + + const int tasks = B * C; + tripletsPerBC.clear(); + tripletsPerBC.resize(tasks); + + parallel_for_(Range(0, tasks), [&](const Range& r){ + std::vector rects; + std::vector scores; + std::vector globalIndices; + rects.reserve(N); scores.reserve(N); globalIndices.reserve(N); + std::vector rects2d; + std::vector keep; + rects2d.reserve(N); + keep.reserve(N); + + for (int taskIdx = r.start; taskIdx < r.end; ++taskIdx) { + rects.clear(); scores.clear(); globalIndices.clear(); + rects2d.clear(); keep.clear(); + const int b = taskIdx / C; + const int c = taskIdx % C; + + const float* sPtr = scoresBlob.ptr(b, c); + for (int n = 0; n < N; ++n) { + float sc = sPtr[n]; + const float* bx = boxesBlob.ptr(b, n); + Rect2f r; + if (centerPointBox == 0) { + const float y1 = bx[0], x1 = bx[1], y2 = bx[2], x2 = bx[3]; + const float x = std::min(x1, x2), y = std::min(y1, y2); + const float w = std::max(0.f, std::abs(x2 - x1)); + const float h = std::max(0.f, std::abs(y2 - y1)); + r = Rect2f(x, y, w, h); + } else { + const float yc = bx[0], xc = bx[1]; + const float h = std::max(0.f, bx[2]), w = std::max(0.f, bx[3]); + r = Rect2f(xc - 0.5f*w, yc - 0.5f*h, w, h); + } + rects.push_back(r); + scores.push_back(sc); + globalIndices.push_back(n); + } + + if (rects.empty()) + continue; + + for (const Rect2f& rf : rects) { + rects2d.emplace_back((double)rf.x, (double)rf.y, (double)rf.width, (double)rf.height); + } + NMSBoxes(rects2d, scores, /*score_threshold*/ scoreThr, /*nms_threshold*/ iouThr, keep, 1.f, 0); + if (maxOut > 0 && (int)keep.size() > maxOut) + keep.resize(maxOut); + + auto& local = tripletsPerBC[taskIdx]; + local.reserve(keep.size()); + for (int kept : keep) { + const int globalIdx = globalIndices[kept]; + local.push_back({(int64_t)b, (int64_t)c, (int64_t)globalIdx}); + } + } + }); + } + + int centerPointBox = 0; + int defaultMaxOut = 0; + float defaultIouThr = 0.f; + float defaultScoreThr = 0.f; +}; + +Ptr NonMaxSuppressionLayer::create(const LayerParams& params) { + return Ptr(new NonMaxSuppressionLayerImpl(params)); +} + +}} // namespace dnn diff --git a/modules/dnn/src/onnx/onnx_importer2.cpp b/modules/dnn/src/onnx/onnx_importer2.cpp index 93e0f01ef7..90a0f6640c 100644 --- a/modules/dnn/src/onnx/onnx_importer2.cpp +++ b/modules/dnn/src/onnx/onnx_importer2.cpp @@ -232,6 +232,7 @@ protected: void parseTranspose (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); void parseUnsqueeze (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); void parseUpsample (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseNonMaxSuprression (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); void parseTopK2 (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); void parseBitShift (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); @@ -1704,6 +1705,12 @@ void ONNXImporter2::parseGridSample(LayerParams& layerParams, const opencv_onnx: addLayer(layerParams, node_proto); } +void ONNXImporter2::parseNonMaxSuprression(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +{ + layerParams.type = "NonMaxSuprression"; + addLayer(layerParams, node_proto); +} + void ONNXImporter2::parseUpsample(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) { int n_inputs = node_proto.input_size(); @@ -2548,6 +2555,7 @@ void ONNXImporter2::buildDispatchMap_ONNX_AI(int opset_version) dispatch["GridSample"] = &ONNXImporter2::parseGridSample; dispatch["Upsample"] = &ONNXImporter2::parseUpsample; dispatch["BitShift"] = &ONNXImporter2::parseBitShift; + dispatch["NonMaxSuprression"] = &ONNXImporter2::parseNonMaxSuprression; dispatch["SoftMax"] = dispatch["Softmax"] = dispatch["LogSoftmax"] = &ONNXImporter2::parseSoftMax; dispatch["DetectionOutput"] = &ONNXImporter2::parseDetectionOutput; dispatch["CumSum"] = &ONNXImporter2::parseCumSum; diff --git a/modules/dnn/test/test_onnx_conformance_layer_filter__openvino.inl.hpp b/modules/dnn/test/test_onnx_conformance_layer_filter__openvino.inl.hpp index 322119c9df..0eaf9fa354 100644 --- a/modules/dnn/test/test_onnx_conformance_layer_filter__openvino.inl.hpp +++ b/modules/dnn/test/test_onnx_conformance_layer_filter__openvino.inl.hpp @@ -1200,23 +1200,23 @@ CASE(test_nllloss_NCd1d2d3d4d5_none_no_weight) CASE(test_nllloss_NCd1d2d3d4d5_none_no_weight_expanded) // no filter CASE(test_nonmaxsuppression_center_point_box_format) - // no filter + SKIP; CASE(test_nonmaxsuppression_flipped_coordinates) - // no filter + SKIP; CASE(test_nonmaxsuppression_identical_boxes) - // no filter + SKIP; CASE(test_nonmaxsuppression_limit_output_size) - // no filter + SKIP; CASE(test_nonmaxsuppression_single_box) - // no filter + SKIP; CASE(test_nonmaxsuppression_suppress_by_IOU) - // no filter + SKIP; CASE(test_nonmaxsuppression_suppress_by_IOU_and_scores) - // no filter + SKIP; CASE(test_nonmaxsuppression_two_batches) - // no filter + SKIP; CASE(test_nonmaxsuppression_two_classes) - // no filter + SKIP; CASE(test_nonzero_example) SKIP; CASE(test_not_2d) diff --git a/modules/dnn/test/test_onnx_conformance_layer_filter_opencv_classic_denylist.inl.hpp b/modules/dnn/test/test_onnx_conformance_layer_filter_opencv_classic_denylist.inl.hpp index 5603231545..7835cb9c5d 100644 --- a/modules/dnn/test/test_onnx_conformance_layer_filter_opencv_classic_denylist.inl.hpp +++ b/modules/dnn/test/test_onnx_conformance_layer_filter_opencv_classic_denylist.inl.hpp @@ -166,3 +166,12 @@ "test_resize_downsample_sizes_linear_pytorch_half_pixel", "test_resize_downsample_sizes_nearest_tf_half_pixel_for_nn", "test_resize_tf_crop_and_resize", +"test_nonmaxsuppression_center_point_box_format", +"test_nonmaxsuppression_flipped_coordinates", +"test_nonmaxsuppression_identical_boxes", +"test_nonmaxsuppression_limit_output_size", +"test_nonmaxsuppression_single_box", +"test_nonmaxsuppression_suppress_by_IOU", +"test_nonmaxsuppression_suppress_by_IOU_and_scores", +"test_nonmaxsuppression_two_batches", +"test_nonmaxsuppression_two_classes", diff --git a/modules/dnn/test/test_onnx_conformance_layer_parser_denylist.inl.hpp b/modules/dnn/test/test_onnx_conformance_layer_parser_denylist.inl.hpp index 7dfa6d3abe..954721d720 100644 --- a/modules/dnn/test/test_onnx_conformance_layer_parser_denylist.inl.hpp +++ b/modules/dnn/test/test_onnx_conformance_layer_parser_denylist.inl.hpp @@ -123,15 +123,6 @@ "test_nllloss_NCd1d2d3d4d5_mean_weight", // Issue:: Layer does not exist (NegativeLogLikelihoodLoss, SoftmaxCrossEntropyLoss) "test_nllloss_NCd1d2d3d4d5_mean_weight_expanded", // Issue::Wrong output "test_nllloss_NCd1d2d3d4d5_none_no_weight", // Issue:: Layer does not exist (NegativeLogLikelihoodLoss, SoftmaxCrossEntropyLoss) -"test_nonmaxsuppression_center_point_box_format", // Issue:: Layer does not exist (NonMaxSuppression)::Can't create layer "onnx_node_output_0!selected_indices" of type "NonMaxSuppression" in function 'getLayerInstance' -"test_nonmaxsuppression_flipped_coordinates", // ---- same as above --- -"test_nonmaxsuppression_identical_boxes", // ---- same as above --- -"test_nonmaxsuppression_limit_output_size", // ---- same as above --- -"test_nonmaxsuppression_single_box", // ---- same as above --- -"test_nonmaxsuppression_suppress_by_IOU", // ---- same as above --- -"test_nonmaxsuppression_suppress_by_IOU_and_scores", // ---- same as above --- -"test_nonmaxsuppression_two_batches", // ---- same as above --- -"test_nonmaxsuppression_two_classes", // ---- same as above --- "test_onehot_negative_indices", // Issue:: Layer does not exist (OneHot) :: Can't create layer "onnx_node_output_0!y" of type "OneHot" in function 'getLayerInstance' "test_onehot_with_axis", // ---- same as above --- "test_onehot_with_negative_axis", // ---- same as above ---