mirror of
https://github.com/opencv/opencv.git
synced 2026-07-29 23:33:05 +04:00
Merge pull request #27674 from abhishek-gola:nonmaxsuppression_layer_add
Added nonmaxsuppression (NMS) layer to new DNN engine #27674 ### Pull Request Readiness Checklist See details at https://github.com/opencv/opencv/wiki/How_to_contribute#making-a-good-pull-request - [x] I agree to contribute to the project under Apache 2 License. - [x] To the best of my knowledge, the proposed patch is not based on a code under GPL or another license that is incompatible with OpenCV - [x] The PR is proposed to the proper branch - [x] There is a reference to the original bug report and related work - [x] There is accuracy test, performance test and test data in opencv_extra repository, if applicable Patch to opencv_extra has the same branch name. - [x] The feature is well documented and sample code can be built with the project CMake
This commit is contained in:
Abhishek Gola
GitHub
@@ -1414,6 +1414,12 @@ CV__DNN_INLINE_NS_BEGIN
|
||||
static Ptr<CastLayer> create(const LayerParams ¶ms);
|
||||
};
|
||||
|
||||
class CV_EXPORTS NonMaxSuppressionLayer : public Layer
|
||||
{
|
||||
public:
|
||||
static Ptr<NonMaxSuppressionLayer> create(const LayerParams& params);
|
||||
};
|
||||
|
||||
class CV_EXPORTS ClipLayer : public Layer {
|
||||
public:
|
||||
static Ptr<ClipLayer> create(const LayerParams ¶ms);
|
||||
|
||||
@@ -96,6 +96,7 @@ void initializeLayerFactory()
|
||||
CV_DNN_REGISTER_LAYER_CLASS(Pad2, Pad2Layer);
|
||||
CV_DNN_REGISTER_LAYER_CLASS(NonZero, NonZeroLayer);
|
||||
CV_DNN_REGISTER_LAYER_CLASS(QuantizeLinear, QuantizeLinearLayer);
|
||||
CV_DNN_REGISTER_LAYER_CLASS(NonMaxSuppression, NonMaxSuppressionLayer);
|
||||
CV_DNN_REGISTER_LAYER_CLASS(Range, RangeLayer);
|
||||
CV_DNN_REGISTER_LAYER_CLASS(Reshape, ReshapeLayer);
|
||||
CV_DNN_REGISTER_LAYER_CLASS(Reshape2, Reshape2Layer);
|
||||
|
||||
@@ -0,0 +1,226 @@
|
||||
// This file is part of OpenCV project.
|
||||
// It is subject to the license terms in the LICENSE file found in the top-level directory
|
||||
// of this distribution and at http://opencv.org/license.html.
|
||||
// Copyright (C) 2025, BigVision LLC, all rights reserved.
|
||||
// Third party copyrights are property of their respective owners.
|
||||
|
||||
#include "../precomp.hpp"
|
||||
#include "layers_common.hpp"
|
||||
#include <opencv2/dnn/shape_utils.hpp>
|
||||
#include <limits>
|
||||
#include <numeric>
|
||||
#include <cstdint>
|
||||
|
||||
namespace cv {
|
||||
namespace dnn {
|
||||
|
||||
// ONNX NonMaxSuppression (opset 10 and 11)
|
||||
// Spec: https://onnx.ai/onnx/operators/onnx__NonMaxSuppression.html
|
||||
// Inputs: boxes [B, N, 4], scores [B, C, N], (optional) max_output_boxes_per_class, iou_threshold, score_threshold
|
||||
// Output: selected_indices [K, 3] (int64): [batch_idx, class_idx, box_idx]
|
||||
|
||||
class NonMaxSuppressionLayerImpl CV_FINAL : public NonMaxSuppressionLayer
|
||||
{
|
||||
public:
|
||||
NonMaxSuppressionLayerImpl(const LayerParams& p) {
|
||||
setParamsFrom(p);
|
||||
centerPointBox = p.get<int>("center_point_box", 0);
|
||||
defaultMaxOut = p.get<int>("default_max_output_boxes_per_class", 0);
|
||||
defaultIouThr = p.get<float>("default_iou_threshold", 0.f);
|
||||
defaultScoreThr = p.get<float>("default_score_threshold", 0.f);
|
||||
}
|
||||
|
||||
bool supportBackend(int backendId) CV_OVERRIDE {
|
||||
return backendId == DNN_BACKEND_OPENCV;
|
||||
}
|
||||
|
||||
virtual bool dynamicOutputShapes() const CV_OVERRIDE { return true; }
|
||||
|
||||
bool getMemoryShapes(const std::vector<MatShape>& inputs, const int, std::vector<MatShape>& outputs, std::vector<MatShape>&) const CV_OVERRIDE {
|
||||
CV_Assert(inputs.size() >= 2);
|
||||
const MatShape& boxes = inputs[0];
|
||||
const MatShape& scores = inputs[1];
|
||||
CV_Assert(boxes.size() == 3 && boxes[2] == 4);
|
||||
CV_Assert(scores.size() == 3 && boxes[0] == scores[0] && boxes[1] == scores[2]);
|
||||
|
||||
outputs.assign(1, MatShape({1, 3}));
|
||||
return false;
|
||||
}
|
||||
|
||||
void getTypes(const std::vector<MatType>&, const int requiredOutputs, const int requiredInternals,
|
||||
std::vector<MatType>& outputs, std::vector<MatType>& internals) const CV_OVERRIDE {
|
||||
outputs.assign(requiredOutputs, MatType(CV_64S));
|
||||
}
|
||||
|
||||
void forward(InputArrayOfArrays inputs_arr, OutputArrayOfArrays outputs_arr, OutputArrayOfArrays) CV_OVERRIDE {
|
||||
std::vector<Mat> inputs, outputs;
|
||||
|
||||
Mat boxesBlob, scoresBlob;
|
||||
int maxOut = defaultMaxOut;
|
||||
float iouThr = defaultIouThr;
|
||||
float scoreThr = defaultScoreThr;
|
||||
|
||||
auto inKind = inputs_arr.kind();
|
||||
if (inKind == _InputArray::STD_VECTOR_UMAT) {
|
||||
std::vector<UMat> uinputs;
|
||||
inputs_arr.getUMatVector(uinputs);
|
||||
CV_Assert(uinputs.size() >= 2);
|
||||
boxesBlob = uinputs[0].getMat(ACCESS_READ);
|
||||
scoresBlob = uinputs[1].getMat(ACCESS_READ);
|
||||
if (uinputs.size() >= 3 && !uinputs[2].empty()) {
|
||||
Mat tmp = uinputs[2].getMat(ACCESS_READ);
|
||||
CV_Assert(tmp.channels() == 1 && tmp.total() == 1);
|
||||
maxOut = tensorToScalar<int>(tmp);
|
||||
}
|
||||
if (uinputs.size() >= 4 && !uinputs[3].empty()) {
|
||||
Mat tmp = uinputs[3].getMat(ACCESS_READ);
|
||||
CV_Assert(tmp.channels() == 1 && tmp.total() == 1);
|
||||
iouThr = tensorToScalar<float>(tmp);
|
||||
}
|
||||
if (uinputs.size() >= 5 && !uinputs[4].empty()) {
|
||||
Mat tmp = uinputs[4].getMat(ACCESS_READ);
|
||||
CV_Assert(tmp.channels() == 1 && tmp.total() == 1);
|
||||
scoreThr = tensorToScalar<float>(tmp);
|
||||
}
|
||||
} else {
|
||||
inputs_arr.getMatVector(inputs);
|
||||
CV_Assert(inputs.size() >= 2);
|
||||
boxesBlob = inputs[0];
|
||||
scoresBlob = inputs[1];
|
||||
if (inputs.size() >= 3 && !inputs[2].empty()) {
|
||||
CV_Assert(inputs[2].channels() == 1 && inputs[2].total() == 1);
|
||||
maxOut = tensorToScalar<int>(inputs[2]);
|
||||
}
|
||||
if (inputs.size() >= 4 && !inputs[3].empty()) {
|
||||
CV_Assert(inputs[3].channels() == 1 && inputs[3].total() == 1);
|
||||
iouThr = tensorToScalar<float>(inputs[3]);
|
||||
}
|
||||
if (inputs.size() >= 5 && !inputs[4].empty()) {
|
||||
CV_Assert(inputs[4].channels() == 1 && inputs[4].total() == 1);
|
||||
scoreThr = tensorToScalar<float>(inputs[4]);
|
||||
}
|
||||
}
|
||||
|
||||
const int B = boxesBlob.size[0];
|
||||
const int C = scoresBlob.size[1];
|
||||
const int tasks = B * C;
|
||||
|
||||
std::vector<std::vector<Vec<int64_t,3>>> tripletsPerBC;
|
||||
run(boxesBlob, scoresBlob, maxOut, iouThr, scoreThr, tripletsPerBC);
|
||||
|
||||
int K = 0;
|
||||
for (int t = 0; t < tasks; ++t) K += (int)tripletsPerBC[t].size();
|
||||
|
||||
Mat outMat;
|
||||
auto outKind = outputs_arr.kind();
|
||||
if (outKind == _InputArray::STD_VECTOR_MAT) {
|
||||
std::vector<Mat>& outs = outputs_arr.getMatVecRef();
|
||||
CV_Assert(outs.size() == 1);
|
||||
outs[0].create(K, 3, CV_64S);
|
||||
outMat = outs[0];
|
||||
} else if (outKind == _InputArray::STD_VECTOR_UMAT) {
|
||||
std::vector<UMat> uouts;
|
||||
outputs_arr.getUMatVector(uouts);
|
||||
CV_Assert(uouts.size() == 1);
|
||||
uouts[0].fit(MatShape({K, 3}), CV_64S);
|
||||
outMat = uouts[0].getMat(ACCESS_WRITE);
|
||||
} else {
|
||||
outputs_arr.getMatRef(0).create(K, 3, CV_64S);
|
||||
outMat = outputs_arr.getMatRef(0);
|
||||
}
|
||||
|
||||
auto* out = outMat.ptr<int64_t>();
|
||||
std::vector<int> offsets(tasks + 1, 0);
|
||||
for (int t = 0; t < tasks; ++t)
|
||||
offsets[t + 1] = offsets[t] + (int)tripletsPerBC[t].size() * 3;
|
||||
parallel_for_(Range(0, tasks), [&](const Range& r){
|
||||
for (int t = r.start; t < r.end; ++t) {
|
||||
int64_t* dst = out + offsets[t];
|
||||
const auto& v = tripletsPerBC[t];
|
||||
for (const auto& trip : v) {
|
||||
*dst++ = trip[0]; *dst++ = trip[1]; *dst++ = trip[2];
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
private:
|
||||
void run(const Mat& boxesBlob, const Mat& scoresBlob, int maxOut, float iouThr, float scoreThr,
|
||||
std::vector<std::vector<Vec<int64_t,3>>>& tripletsPerBC) const
|
||||
{
|
||||
const int B = boxesBlob.size[0];
|
||||
const int N = boxesBlob.size[1];
|
||||
const int C = scoresBlob.size[1];
|
||||
|
||||
const int tasks = B * C;
|
||||
tripletsPerBC.clear();
|
||||
tripletsPerBC.resize(tasks);
|
||||
|
||||
parallel_for_(Range(0, tasks), [&](const Range& r){
|
||||
std::vector<Rect2f> rects;
|
||||
std::vector<float> scores;
|
||||
std::vector<int> globalIndices;
|
||||
rects.reserve(N); scores.reserve(N); globalIndices.reserve(N);
|
||||
std::vector<Rect2d> rects2d;
|
||||
std::vector<int> keep;
|
||||
rects2d.reserve(N);
|
||||
keep.reserve(N);
|
||||
|
||||
for (int taskIdx = r.start; taskIdx < r.end; ++taskIdx) {
|
||||
rects.clear(); scores.clear(); globalIndices.clear();
|
||||
rects2d.clear(); keep.clear();
|
||||
const int b = taskIdx / C;
|
||||
const int c = taskIdx % C;
|
||||
|
||||
const float* sPtr = scoresBlob.ptr<float>(b, c);
|
||||
for (int n = 0; n < N; ++n) {
|
||||
float sc = sPtr[n];
|
||||
const float* bx = boxesBlob.ptr<float>(b, n);
|
||||
Rect2f r;
|
||||
if (centerPointBox == 0) {
|
||||
const float y1 = bx[0], x1 = bx[1], y2 = bx[2], x2 = bx[3];
|
||||
const float x = std::min(x1, x2), y = std::min(y1, y2);
|
||||
const float w = std::max(0.f, std::abs(x2 - x1));
|
||||
const float h = std::max(0.f, std::abs(y2 - y1));
|
||||
r = Rect2f(x, y, w, h);
|
||||
} else {
|
||||
const float yc = bx[0], xc = bx[1];
|
||||
const float h = std::max(0.f, bx[2]), w = std::max(0.f, bx[3]);
|
||||
r = Rect2f(xc - 0.5f*w, yc - 0.5f*h, w, h);
|
||||
}
|
||||
rects.push_back(r);
|
||||
scores.push_back(sc);
|
||||
globalIndices.push_back(n);
|
||||
}
|
||||
|
||||
if (rects.empty())
|
||||
continue;
|
||||
|
||||
for (const Rect2f& rf : rects) {
|
||||
rects2d.emplace_back((double)rf.x, (double)rf.y, (double)rf.width, (double)rf.height);
|
||||
}
|
||||
NMSBoxes(rects2d, scores, /*score_threshold*/ scoreThr, /*nms_threshold*/ iouThr, keep, 1.f, 0);
|
||||
if (maxOut > 0 && (int)keep.size() > maxOut)
|
||||
keep.resize(maxOut);
|
||||
|
||||
auto& local = tripletsPerBC[taskIdx];
|
||||
local.reserve(keep.size());
|
||||
for (int kept : keep) {
|
||||
const int globalIdx = globalIndices[kept];
|
||||
local.push_back({(int64_t)b, (int64_t)c, (int64_t)globalIdx});
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
int centerPointBox = 0;
|
||||
int defaultMaxOut = 0;
|
||||
float defaultIouThr = 0.f;
|
||||
float defaultScoreThr = 0.f;
|
||||
};
|
||||
|
||||
Ptr<NonMaxSuppressionLayer> NonMaxSuppressionLayer::create(const LayerParams& params) {
|
||||
return Ptr<NonMaxSuppressionLayer>(new NonMaxSuppressionLayerImpl(params));
|
||||
}
|
||||
|
||||
}} // namespace dnn
|
||||
@@ -232,6 +232,7 @@ protected:
|
||||
void parseTranspose (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto);
|
||||
void parseUnsqueeze (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto);
|
||||
void parseUpsample (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto);
|
||||
void parseNonMaxSuprression (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto);
|
||||
void parseTopK2 (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto);
|
||||
void parseBitShift (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto);
|
||||
|
||||
@@ -1704,6 +1705,12 @@ void ONNXImporter2::parseGridSample(LayerParams& layerParams, const opencv_onnx:
|
||||
addLayer(layerParams, node_proto);
|
||||
}
|
||||
|
||||
void ONNXImporter2::parseNonMaxSuprression(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto)
|
||||
{
|
||||
layerParams.type = "NonMaxSuprression";
|
||||
addLayer(layerParams, node_proto);
|
||||
}
|
||||
|
||||
void ONNXImporter2::parseUpsample(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto)
|
||||
{
|
||||
int n_inputs = node_proto.input_size();
|
||||
@@ -2548,6 +2555,7 @@ void ONNXImporter2::buildDispatchMap_ONNX_AI(int opset_version)
|
||||
dispatch["GridSample"] = &ONNXImporter2::parseGridSample;
|
||||
dispatch["Upsample"] = &ONNXImporter2::parseUpsample;
|
||||
dispatch["BitShift"] = &ONNXImporter2::parseBitShift;
|
||||
dispatch["NonMaxSuprression"] = &ONNXImporter2::parseNonMaxSuprression;
|
||||
dispatch["SoftMax"] = dispatch["Softmax"] = dispatch["LogSoftmax"] = &ONNXImporter2::parseSoftMax;
|
||||
dispatch["DetectionOutput"] = &ONNXImporter2::parseDetectionOutput;
|
||||
dispatch["CumSum"] = &ONNXImporter2::parseCumSum;
|
||||
|
||||
@@ -1200,23 +1200,23 @@ CASE(test_nllloss_NCd1d2d3d4d5_none_no_weight)
|
||||
CASE(test_nllloss_NCd1d2d3d4d5_none_no_weight_expanded)
|
||||
// no filter
|
||||
CASE(test_nonmaxsuppression_center_point_box_format)
|
||||
// no filter
|
||||
SKIP;
|
||||
CASE(test_nonmaxsuppression_flipped_coordinates)
|
||||
// no filter
|
||||
SKIP;
|
||||
CASE(test_nonmaxsuppression_identical_boxes)
|
||||
// no filter
|
||||
SKIP;
|
||||
CASE(test_nonmaxsuppression_limit_output_size)
|
||||
// no filter
|
||||
SKIP;
|
||||
CASE(test_nonmaxsuppression_single_box)
|
||||
// no filter
|
||||
SKIP;
|
||||
CASE(test_nonmaxsuppression_suppress_by_IOU)
|
||||
// no filter
|
||||
SKIP;
|
||||
CASE(test_nonmaxsuppression_suppress_by_IOU_and_scores)
|
||||
// no filter
|
||||
SKIP;
|
||||
CASE(test_nonmaxsuppression_two_batches)
|
||||
// no filter
|
||||
SKIP;
|
||||
CASE(test_nonmaxsuppression_two_classes)
|
||||
// no filter
|
||||
SKIP;
|
||||
CASE(test_nonzero_example)
|
||||
SKIP;
|
||||
CASE(test_not_2d)
|
||||
|
||||
@@ -166,3 +166,12 @@
|
||||
"test_resize_downsample_sizes_linear_pytorch_half_pixel",
|
||||
"test_resize_downsample_sizes_nearest_tf_half_pixel_for_nn",
|
||||
"test_resize_tf_crop_and_resize",
|
||||
"test_nonmaxsuppression_center_point_box_format",
|
||||
"test_nonmaxsuppression_flipped_coordinates",
|
||||
"test_nonmaxsuppression_identical_boxes",
|
||||
"test_nonmaxsuppression_limit_output_size",
|
||||
"test_nonmaxsuppression_single_box",
|
||||
"test_nonmaxsuppression_suppress_by_IOU",
|
||||
"test_nonmaxsuppression_suppress_by_IOU_and_scores",
|
||||
"test_nonmaxsuppression_two_batches",
|
||||
"test_nonmaxsuppression_two_classes",
|
||||
|
||||
@@ -123,15 +123,6 @@
|
||||
"test_nllloss_NCd1d2d3d4d5_mean_weight", // Issue:: Layer does not exist (NegativeLogLikelihoodLoss, SoftmaxCrossEntropyLoss)
|
||||
"test_nllloss_NCd1d2d3d4d5_mean_weight_expanded", // Issue::Wrong output
|
||||
"test_nllloss_NCd1d2d3d4d5_none_no_weight", // Issue:: Layer does not exist (NegativeLogLikelihoodLoss, SoftmaxCrossEntropyLoss)
|
||||
"test_nonmaxsuppression_center_point_box_format", // Issue:: Layer does not exist (NonMaxSuppression)::Can't create layer "onnx_node_output_0!selected_indices" of type "NonMaxSuppression" in function 'getLayerInstance'
|
||||
"test_nonmaxsuppression_flipped_coordinates", // ---- same as above ---
|
||||
"test_nonmaxsuppression_identical_boxes", // ---- same as above ---
|
||||
"test_nonmaxsuppression_limit_output_size", // ---- same as above ---
|
||||
"test_nonmaxsuppression_single_box", // ---- same as above ---
|
||||
"test_nonmaxsuppression_suppress_by_IOU", // ---- same as above ---
|
||||
"test_nonmaxsuppression_suppress_by_IOU_and_scores", // ---- same as above ---
|
||||
"test_nonmaxsuppression_two_batches", // ---- same as above ---
|
||||
"test_nonmaxsuppression_two_classes", // ---- same as above ---
|
||||
"test_onehot_negative_indices", // Issue:: Layer does not exist (OneHot) :: Can't create layer "onnx_node_output_0!y" of type "OneHot" in function 'getLayerInstance'
|
||||
"test_onehot_with_axis", // ---- same as above ---
|
||||
"test_onehot_with_negative_axis", // ---- same as above ---
|
||||
|
||||
Reference in New Issue
Block a user