From b0b77e7b3239bd14ad4d18978c45254745989960 Mon Sep 17 00:00:00 2001 From: Abhishek Gola Date: Wed, 3 Jun 2026 15:41:11 +0530 Subject: [PATCH] Merge pull request #29073 from abhishek-gola:disk_feature_extractor Added DISK feature extractor support #29073 closes: https://github.com/opencv/opencv/issues/27083 Merge with: https://github.com/opencv/opencv_extra/pull/1368/ ### Pull Request Readiness Checklist See details at https://github.com/opencv/opencv/wiki/How_to_contribute#making-a-good-pull-request - [x] I agree to contribute to the project under Apache 2 License. - [x] To the best of my knowledge, the proposed patch is not based on a code under GPL or another license that is incompatible with OpenCV - [x] The PR is proposed to the proper branch - [x] There is a reference to the original bug report and related work - [x] There is accuracy test, performance test and test data in opencv_extra repository, if applicable Patch to opencv_extra has the same branch name. - [x] The feature is well documented and sample code can be built with the project CMake --- doc/opencv.bib | 9 + modules/dnn/src/layers/conv2_common.cpp | 2 +- .../dnn/src/layers/cpu_kernels/fast_norm.cpp | 47 ++-- modules/dnn/src/layers/maxpool_layer.cpp | 151 +++++++++-- modules/dnn/src/onnx/onnx_importer2.cpp | 2 +- ...conformance_layer_filter__openvino.inl.hpp | 6 + ...yer_filter_opencv_classic_denylist.inl.hpp | 3 + ..._conformance_layer_parser_denylist.inl.hpp | 3 - modules/features/CMakeLists.txt | 2 +- modules/features/include/opencv2/features.hpp | 79 ++++++ modules/features/src/disk.cpp | 234 ++++++++++++++++++ modules/features/test/npy_blob.cpp | 103 ++++++++ modules/features/test/npy_blob.hpp | 20 ++ modules/features/test/test_disk.cpp | 174 +++++++++++++ modules/features/test/test_main.cpp | 9 +- 15 files changed, 793 insertions(+), 51 deletions(-) create mode 100644 modules/features/src/disk.cpp create mode 100644 modules/features/test/npy_blob.cpp create mode 100644 modules/features/test/npy_blob.hpp create mode 100644 modules/features/test/test_disk.cpp diff --git a/doc/opencv.bib b/doc/opencv.bib index 4d0dc16f69..85040e7c7f 100644 --- a/doc/opencv.bib +++ b/doc/opencv.bib @@ -1303,6 +1303,15 @@ issn = {1042-296X}, url = {https://kmlee.gatech.edu/me6406/handeye.pdf} } +@inproceedings{Tyszkiewicz2020DISK, + author = {Tyszkiewicz, Micha{\l} and Fua, Pascal and Trulls, Eduard}, + title = {{DISK}: Learning local features with policy gradient}, + booktitle = {Advances in Neural Information Processing Systems}, + volume = {33}, + pages = {14254--14265}, + year = {2020}, + url = {https://proceedings.neurips.cc/paper/2020/hash/a42a596fc71e17828440030074d15e74-Abstract.html} +} @inproceedings{UES01, author = {Uyttendaele, Matthew and Eden, Ashley and Skeliski, R}, title = {Eliminating ghosting and exposure artifacts in image mosaics}, diff --git a/modules/dnn/src/layers/conv2_common.cpp b/modules/dnn/src/layers/conv2_common.cpp index 6e521e7ea1..6244eb618f 100644 --- a/modules/dnn/src/layers/conv2_common.cpp +++ b/modules/dnn/src/layers/conv2_common.cpp @@ -54,7 +54,7 @@ MatShape convInferShape(const MatShape& inpShape, const MatShape& wshape, const std::vector& pads, AutoPadding autoPad, bool ceilMode) { - bool blockLayout = true; + bool blockLayout = (inpShape.layout == DATA_LAYOUT_BLOCK); int ndims = inpShape.dims; size_t nspatialdims = (size_t)(ndims - 2 - int(blockLayout)); MatShape outshape = inpShape; diff --git a/modules/dnn/src/layers/cpu_kernels/fast_norm.cpp b/modules/dnn/src/layers/cpu_kernels/fast_norm.cpp index 36feb3bb1b..869fc50582 100644 --- a/modules/dnn/src/layers/cpu_kernels/fast_norm.cpp +++ b/modules/dnn/src/layers/cpu_kernels/fast_norm.cpp @@ -193,20 +193,23 @@ void fastNormChannel(const Mat &input, const Mat &scale, const Mat &bias, Mat &o const size_t outStep3 = output.step.p[3] / sizeof(float); const size_t norm_size = (size_t)H * (size_t)W; - const float inv_norm_size = 1.f / (float)norm_size; #if CV_SIMD const int VEC_SZ = VTraits::vlanes(); #endif + // Accumulators are double-precision + const double inv_norm_size_d = 1.0 / (double)norm_size; + parallel_for_(Range(0, N * C1), [&](const Range& r) { const float* inptr0 = (const float*)input.data; float* outptr0 = (float*)output.data; - AutoBuffer buf(C0 * 4); - float* sum = buf.data(); - float* sqsum = sum + C0; - float* alpha = sqsum + C0; + AutoBuffer sumBuf(C0 * 2); + double* sum = sumBuf.data(); + double* sqsum = sum + C0; + AutoBuffer abBuf(C0 * 2); + float* alpha = abBuf.data(); float* beta = alpha + C0; for (int i = r.start; i < r.end; ++i) { @@ -219,28 +222,36 @@ void fastNormChannel(const Mat &input, const Mat &scale, const Mat &bias, Mat &o float* outbase = outptr0 + n * outStep0 + c1 * outStep1; int c0 = 0; -#if CV_SIMD +#if CV_SIMD && CV_SIMD_64F + const int VEC_SZ_D = VTraits::vlanes(); + CV_DbgAssert(VEC_SZ == 2 * VEC_SZ_D); for (; c0 <= validC0 - VEC_SZ; c0 += VEC_SZ) { - v_float32 vsum = vx_setzero_f32(); - v_float32 vsqsum = vx_setzero_f32(); + v_float64 vsum_lo = vx_setzero_f64(), vsum_hi = vx_setzero_f64(); + v_float64 vsqsum_lo = vx_setzero_f64(), vsqsum_hi = vx_setzero_f64(); for (int h = 0; h < H; ++h) { const float* inrow = inbase + h * inStep2; for (int w = 0; w < W; ++w) { v_float32 v = vx_load(inrow + w * inStep3 + c0); - vsum = v_add(vsum, v); - vsqsum = v_fma(v, v, vsqsum); + v_float64 vlo = v_cvt_f64(v); + v_float64 vhi = v_cvt_f64_high(v); + vsum_lo = v_add(vsum_lo, vlo); + vsum_hi = v_add(vsum_hi, vhi); + vsqsum_lo = v_fma(vlo, vlo, vsqsum_lo); + vsqsum_hi = v_fma(vhi, vhi, vsqsum_hi); } } - vx_store(sum + c0, vsum); - vx_store(sqsum + c0, vsqsum); + vx_store(sum + c0, vsum_lo); + vx_store(sum + c0 + VEC_SZ_D, vsum_hi); + vx_store(sqsum + c0, vsqsum_lo); + vx_store(sqsum + c0 + VEC_SZ_D, vsqsum_hi); } #endif for (; c0 < validC0; ++c0) { - float s = 0.f, sq = 0.f; + double s = 0., sq = 0.; for (int h = 0; h < H; ++h) { const float* inrow = inbase + h * inStep2; for (int w = 0; w < W; ++w) { - float v = inrow[w * inStep3 + c0]; + double v = (double)inrow[w * inStep3 + c0]; s += v; sq += v * v; } @@ -250,11 +261,11 @@ void fastNormChannel(const Mat &input, const Mat &scale, const Mat &bias, Mat &o } for (int c = 0; c < validC0; ++c) { - float mean = sum[c] * inv_norm_size; - float var = std::max(0.f, sqsum[c] * inv_norm_size - mean * mean); - float inv_stdev = 1.f / std::sqrt(var + epsilon); + double mean = sum[c] * inv_norm_size_d; + double var = std::max(0., sqsum[c] * inv_norm_size_d - mean * mean); + float inv_stdev = 1.f / std::sqrt((float)var + epsilon); alpha[c] = scale_data[cbase + c] * inv_stdev; - beta[c] = bias_data[cbase + c] - alpha[c] * mean; + beta[c] = bias_data[cbase + c] - alpha[c] * (float)mean; } c0 = 0; diff --git a/modules/dnn/src/layers/maxpool_layer.cpp b/modules/dnn/src/layers/maxpool_layer.cpp index 55321678f9..ce64e7ceb0 100644 --- a/modules/dnn/src/layers/maxpool_layer.cpp +++ b/modules/dnn/src/layers/maxpool_layer.cpp @@ -376,6 +376,53 @@ static void maxPool16bf(const void* inp_, void* out_, const ConvState& cs) typedef void (*MaxPoolFunc)(const void* inp, void* out, const ConvState& cs); +// 2-output (values + ONNX-style int64 indices) NCHW scalar implementation. +static void maxPool32f_nchw_with_indices(const float* inp, float* out, int64_t* outIdx, + int N, int C, int Hi, int Wi, int H, int W, + int kH, int kW, int sH, int sW, + int padH, int padW, int dilH, int dilW) +{ + int NC = N * C; + int inHW = Hi * Wi; + int outHW = H * W; + parallel_for_(Range(0, NC), [&](const Range& r) { + for (int nc = r.start; nc < r.end; nc++) { + int c = nc % C; + const float* inp_nc = inp + nc * inHW; + float* out_nc = out + nc * outHW; + int64_t* idx_nc = outIdx + nc * outHW; + for (int yo = 0; yo < H; yo++) { + for (int xo = 0; xo < W; xo++) { + float vmax = -FLT_MAX; + int64_t idxmax = -1; + for (int ky = 0; ky < kH; ky++) { + int yi = yo * sH - padH + ky * dilH; + if ((unsigned)yi >= (unsigned)Hi) + continue; + for (int kx = 0; kx < kW; kx++) { + int xi = xo * sW - padW + kx * dilW; + if ((unsigned)xi >= (unsigned)Wi) + continue; + float v = inp_nc[yi * Wi + xi]; + if (v > vmax) { + vmax = v; + // ONNX storage_order=0: index = (c*Hi + yi)*Wi + xi + idxmax = (int64_t)(c * Hi + yi) * Wi + xi; + } + } + } + if (idxmax < 0) { + vmax = 0.f; + idxmax = (int64_t)(c * Hi + 0) * Wi + 0; + } + out_nc[yo * W + xo] = vmax; + idx_nc[yo * W + xo] = idxmax; + } + } + } + }); +} + class MaxPoolLayerImpl : public MaxPoolLayer { public: @@ -452,7 +499,10 @@ public: int ninputs = (int)inptypes.size(); CV_Assert(ninputs == 1); - outtypes.assign(1, inferType(inptypes[0])); + outtypes.clear(); + outtypes.push_back(inferType(inptypes[0])); + if (outputs.size() == 2u) + outtypes.push_back(CV_64S); // ONNX MaxPool indices temptypes.clear(); } @@ -463,13 +513,14 @@ public: std::vector &outshapes, std::vector &tempshapes) const CV_OVERRIDE { - CV_Assert(outputs.size() == 1u); + CV_Assert(outputs.size() == 1u || outputs.size() == 2u); size_t ninputs = inpshapes.size(); CV_Assert(ninputs == 1); - outshapes.assign(1, convInferShape(inpshapes[0], MatShape(), + MatShape outshape = convInferShape(inpshapes[0], MatShape(), kernel_shape, 0, strides, dilations, - pads, auto_pad, ceil_mode)); + pads, auto_pad, ceil_mode); + outshapes.assign(outputs.size(), outshape); tempshapes.clear(); return true; } @@ -480,9 +531,12 @@ public: std::vector& outputs) const CV_OVERRIDE { CV_Assert(actualInputs.size() == 1u); - desiredInputs.assign(1, DATA_LAYOUT_BLOCK); - outputs.assign(requiredOutputs, DATA_LAYOUT_BLOCK); - return getNetImpl(this)->defaultC0; + + const bool wantsIndices = requiredOutputs == 2; + const DataLayout layout = wantsIndices ? DATA_LAYOUT_NCHW : DATA_LAYOUT_BLOCK; + desiredInputs.assign(1, layout); + outputs.assign(requiredOutputs, layout); + return wantsIndices ? 0 : getNetImpl(this)->defaultC0; } void finalize(InputArrayOfArrays, OutputArrayOfArrays outputs_arr) CV_OVERRIDE @@ -493,40 +547,85 @@ public: OutputArrayOfArrays outputs_arr, OutputArrayOfArrays) CV_OVERRIDE { - size_t ninputs = inputs_arr.total(); - CV_Assert(ninputs == 1); + CV_Assert(inputs_arr.total() == 1); + const int inptype = inputs_arr.type(0); + const MatShape inpshape = inputs_arr.shape(0); - int inptype = inputs_arr.type(0); - MatShape inpshape = inputs_arr.shape(0); - MatShape outshape = convInferShape(inpshape, MatShape(), - kernel_shape, 0, strides, dilations, - pads, auto_pad, ceil_mode); - int outKind = outputs_arr.kind(); + const int outKind = outputs_arr.kind(); CV_Assert(outKind == _InputArray::STD_VECTOR_MAT || outKind == _InputArray::STD_VECTOR_UMAT); - ConvState cs; - cs.initPooling(inpshape, outshape, kernel_shape, strides, - dilations, pads, auto_pad, ceil_mode); + const size_t noutputs = outputs.size(); + CV_Assert(noutputs == 1u || noutputs == 2u); + const bool wantsIndices = (noutputs == 2u); + + if (wantsIndices) { + CV_Assert(inptype == CV_32F && "MaxPool with indices currently supports CV_32F only"); + CV_Assert(inpshape.dims == 4 && "MaxPool with indices: only 4D (N,C,H,W) inputs supported"); + CV_Assert(inpshape.layout != DATA_LAYOUT_BLOCK && + "MaxPool with indices does not run on a BLOCK-layout input"); + } + + const MatShape outshape = convInferShape(inpshape, MatShape(), + kernel_shape, 0, strides, dilations, + pads, auto_pad, ceil_mode); + Mat inp = inputs_arr.getMat(0); if (outKind == _InputArray::STD_VECTOR_MAT) { - Mat inp = inputs_arr.getMat(0); std::vector& outs = outputs_arr.getMatVecRef(); - outs.resize(1); + outs.resize(noutputs); outs[0].fit(outshape, inptype); - runOp(inp, outs[0], cs); + if (wantsIndices) + outs[1].fit(outshape, CV_64S); + runForward(inp, outs, inpshape, outshape); } else { // [TODO] more efficient OpenCL implementation - Mat inp = inputs_arr.getMat(0); std::vector& outs = outputs_arr.getUMatVecRef(); - outs.resize(1); + outs.resize(noutputs); outs[0].fit(outshape, inptype); - Mat temp(outshape, inptype); - runOp(inp, temp, cs); - temp.copyTo(outs[0]); + if (wantsIndices) + outs[1].fit(outshape, CV_64S); + + std::vector tmp(noutputs); + tmp[0].create(outshape, inptype); + if (wantsIndices) + tmp[1].create(outshape, CV_64S); + runForward(inp, tmp, inpshape, outshape); + for (size_t i = 0; i < noutputs; ++i) + tmp[i].copyTo(outs[i]); } } + void runForward(const Mat& inp, std::vector& outs, + const MatShape& inpshape, const MatShape& outshape) + { + if (outs.size() == 1u) { + ConvState cs; + cs.initPooling(inpshape, outshape, kernel_shape, strides, + dilations, pads, auto_pad, ceil_mode); + runOp(inp, outs[0], cs); + } else { + runOpWithIndices(inp, outs[0], outs[1]); + } + } + + void runOpWithIndices(const Mat& inp, Mat& outVal, Mat& outIdx) + { + const int N = inp.size[0], C = inp.size[1], Hi = inp.size[2], Wi = inp.size[3]; + const int H = outVal.size[2], W = outVal.size[3]; + const int kH = (int)kernel_shape[0]; + const int kW = (int)kernel_shape[1]; + const int sH = strides.size() > 0 ? (int)strides[0] : 1; + const int sW = strides.size() > 1 ? (int)strides[1] : sH; + const int dilH = dilations.size() > 0 ? (int)dilations[0] : 1; + const int dilW = dilations.size() > 1 ? (int)dilations[1] : dilH; + const int padH = pads.size() > 0 ? (int)pads[0] : 0; + const int padW = pads.size() > 1 ? (int)pads[1] : padH; + maxPool32f_nchw_with_indices( + inp.ptr(), outVal.ptr(), outIdx.ptr(), + N, C, Hi, Wi, H, W, kH, kW, sH, sW, padH, padW, dilH, dilW); + } + void runOp(const Mat& inp, Mat& out, const ConvState& cs) { int inptype = inp.type(); diff --git a/modules/dnn/src/onnx/onnx_importer2.cpp b/modules/dnn/src/onnx/onnx_importer2.cpp index a8212fb781..7920d6ae51 100644 --- a/modules/dnn/src/onnx/onnx_importer2.cpp +++ b/modules/dnn/src/onnx/onnx_importer2.cpp @@ -1181,7 +1181,7 @@ void ONNXImporter2::parseMaxUnpool(LayerParams& layerParams, const opencv_onnx:: void ONNXImporter2::parseMaxPool(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) { - CV_CheckEQ(node_outputs.size(), 1u, "the new engine does not support MaxPool with 2 outputs yet"); + CV_CheckLE(node_outputs.size(), 2u, "MaxPool may have at most 2 outputs (values + indices)"); layerParams.type = "MaxPool"; addLayer(layerParams, node_proto); } diff --git a/modules/dnn/test/test_onnx_conformance_layer_filter__openvino.inl.hpp b/modules/dnn/test/test_onnx_conformance_layer_filter__openvino.inl.hpp index da4b843574..a01a75d26a 100644 --- a/modules/dnn/test/test_onnx_conformance_layer_filter__openvino.inl.hpp +++ b/modules/dnn/test/test_onnx_conformance_layer_filter__openvino.inl.hpp @@ -1546,6 +1546,12 @@ CASE(test_maxpool_3d_default) #if SKIP_SET_1 SKIP_NON_CPU; #endif +CASE(test_maxpool_3d_dilations) + SKIP; +CASE(test_maxpool_3d_dilations_use_ref_impl) + SKIP; +CASE(test_maxpool_3d_dilations_use_ref_impl_large) + SKIP; CASE(test_maxpool_with_argmax_2d_precomputed_pads) SKIP; CASE(test_maxpool_with_argmax_2d_precomputed_strides) diff --git a/modules/dnn/test/test_onnx_conformance_layer_filter_opencv_classic_denylist.inl.hpp b/modules/dnn/test/test_onnx_conformance_layer_filter_opencv_classic_denylist.inl.hpp index af7b565a07..e06c00881a 100644 --- a/modules/dnn/test/test_onnx_conformance_layer_filter_opencv_classic_denylist.inl.hpp +++ b/modules/dnn/test/test_onnx_conformance_layer_filter_opencv_classic_denylist.inl.hpp @@ -753,3 +753,6 @@ "test_convtranspose_pad", "test_convtranspose_pads", "test_convtranspose_with_kernel", +"test_maxpool_3d_dilations", +"test_maxpool_3d_dilations_use_ref_impl", +"test_maxpool_3d_dilations_use_ref_impl_large", diff --git a/modules/dnn/test/test_onnx_conformance_layer_parser_denylist.inl.hpp b/modules/dnn/test/test_onnx_conformance_layer_parser_denylist.inl.hpp index 6d746bd648..75d6133672 100644 --- a/modules/dnn/test/test_onnx_conformance_layer_parser_denylist.inl.hpp +++ b/modules/dnn/test/test_onnx_conformance_layer_parser_denylist.inl.hpp @@ -329,9 +329,6 @@ "test_lppool_3d_default", "test_matmulinteger", // Issues::Layer does not exist. Can't create layer "onnx_node_output_0!Y" of type "MatMulInteger" in function 'getLayerInstance' "test_maxpool_2d_ceil_output_size_reduce_by_one", -"test_maxpool_3d_dilations", -"test_maxpool_3d_dilations_use_ref_impl", -"test_maxpool_3d_dilations_use_ref_impl_large", //shape mismatch "test_melweightmatrix", "test_momentum", // Issues::Layer does not exist. Can't create layer "onnx_node_output_0!X1_new" of type "ai.onnx.preview.training.Momentum" in function 'getLayerInstance' "test_momentum_multiple", // ---- same as above --- diff --git a/modules/features/CMakeLists.txt b/modules/features/CMakeLists.txt index c6c73ffd10..d471a92f80 100644 --- a/modules/features/CMakeLists.txt +++ b/modules/features/CMakeLists.txt @@ -6,7 +6,7 @@ set(debug_modules "") if(DEBUG_opencv_features) list(APPEND debug_modules opencv_highgui) endif() -ocv_define_module(features opencv_imgproc opencv_geometry ${debug_modules} OPTIONAL opencv_flann WRAP java objc python js) +ocv_define_module(features opencv_imgproc opencv_geometry ${debug_modules} OPTIONAL opencv_flann opencv_dnn WRAP java objc python js) ocv_install_3rdparty_licenses(mscr "${CMAKE_CURRENT_SOURCE_DIR}/3rdparty/mscr/chi_table_LICENSE.txt") ocv_install_3rdparty_licenses(annoylib "${CMAKE_CURRENT_SOURCE_DIR}/3rdparty/annoy/LICENSE") diff --git a/modules/features/include/opencv2/features.hpp b/modules/features/include/opencv2/features.hpp index a82e54e7ed..5fff55fb3b 100644 --- a/modules/features/include/opencv2/features.hpp +++ b/modules/features/include/opencv2/features.hpp @@ -669,9 +669,88 @@ public: CV_WRAP virtual void setK(double k) = 0; CV_WRAP virtual double getK() const = 0; + CV_WRAP virtual String getDefaultName() const CV_OVERRIDE; }; +#if defined(HAVE_OPENCV_DNN) || defined(CV_DOXYGEN) + +/** @brief DISK feature detector and descriptor, based on a DNN model. + +DISK (Deep Image Structure and Keypoints) is a learned local-feature pipeline that produces +keypoints and 128-D L2-normalized descriptors via a single forward pass through a fully +convolutional network. This class wraps an ONNX export of the pre-trained DISK model through +cv::dnn::Net and exposes it under the standard cv::Feature2D interface so it can be used as +a drop-in alternative to SIFT/ORB. + +The class assumes the ONNX model has a single input named `image` taking an N×3×H×W float32 +tensor in [0, 1] (RGB channel order) and three outputs named `keypoints` (N×2), `scores` (N) +and `descriptors` (N×128). + */ +class CV_EXPORTS_W DISK : public Feature2D +{ +public: + /** @brief Creates a DISK detector. + @param modelPath Path to the DISK ONNX model. + @param maxKeypoints Maximum number of keypoints to return per image. The strongest + responses (by network score) are kept; -1 keeps all detections. + @param scoreThreshold Discard keypoints with network score strictly below this value. + @param imageSize Target input size (width, height) fed to the network. Use Size() + (the default) to fall back to the network's expected fixed input + shape of 1024x1024. When overriding, both dimensions must be + positive multiples of 16, since DISK downsamples by a factor of 16. + @param backendId DNN backend identifier (see cv::dnn::Backend); 0 = DNN_BACKEND_DEFAULT. + @param targetId DNN target identifier (see cv::dnn::Target); 0 = DNN_TARGET_CPU. + */ + CV_WRAP static Ptr create(const String& modelPath, + int maxKeypoints = -1, + float scoreThreshold = 0.0f, + const Size& imageSize = Size(), + int backendId = 0, + int targetId = 0); + + /** @brief Creates a DISK detector from an in-memory model buffer. + + This overload loads the DISK ONNX model from a buffer instead of a file on disk. It is + intended for cases where the model is read from application resources (for example Android + assets) and is not available as a path on the filesystem. + + @param bufferModel A buffer containing the contents of the DISK ONNX model. + @param maxKeypoints Maximum number of keypoints to return per image. The strongest + responses (by network score) are kept; -1 keeps all detections. + @param scoreThreshold Discard keypoints with network score strictly below this value. + @param imageSize Target input size (width, height) fed to the network. Use Size() + (the default) to fall back to the network's expected fixed input + shape of 1024x1024. When overriding, both dimensions must be + positive multiples of 16, since DISK downsamples by a factor of 16. + @param backendId DNN backend identifier (see cv::dnn::Backend); 0 = DNN_BACKEND_DEFAULT. + @param targetId DNN target identifier (see cv::dnn::Target); 0 = DNN_TARGET_CPU. + + @note In C++ this is an overload of @ref create. The Python/Java/Objective-C bindings expose + it as `createFromMemory`, because Objective-C selectors are not disambiguated by argument + type and would otherwise clash with the file-path @ref create. + */ + CV_WRAP_AS(createFromMemory) static Ptr create(const std::vector& bufferModel, + int maxKeypoints = -1, + float scoreThreshold = 0.0f, + const Size& imageSize = Size(), + int backendId = 0, + int targetId = 0); + + CV_WRAP virtual void setMaxKeypoints(int maxKeypoints) = 0; + CV_WRAP virtual int getMaxKeypoints() const = 0; + + CV_WRAP virtual void setScoreThreshold(float threshold) = 0; + CV_WRAP virtual float getScoreThreshold() const = 0; + + CV_WRAP virtual void setImageSize(const Size& size) = 0; + CV_WRAP virtual Size getImageSize() const = 0; + + CV_WRAP virtual String getDefaultName() const CV_OVERRIDE; +}; + +#endif // HAVE_OPENCV_DNN || CV_DOXYGEN + /** @brief Class for extracting blobs from an image. : The class implements a simple algorithm for extracting blobs from an image: diff --git a/modules/features/src/disk.cpp b/modules/features/src/disk.cpp new file mode 100644 index 0000000000..7d652a5643 --- /dev/null +++ b/modules/features/src/disk.cpp @@ -0,0 +1,234 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// Copyright (C) 2026, BigVision LLC, all rights reserved. +// Third party copyrights are property of their respective owners. + +#include "precomp.hpp" +#include "opencv2/features.hpp" + +#ifdef HAVE_OPENCV_DNN +#include "opencv2/dnn.hpp" + +#include +#include + +namespace cv { + +using namespace dnn; + +// Default network input size used when the user does not specify one explicitly. +// Matches the fixed-shape standalone DISK ONNX export shipped in opencv_extra. +static const Size kDefaultDiskInputSize = Size(1024, 1024); + +// DISK is a fully convolutional network with a 16x downsampling stride, so user-provided +// input sizes must be positive multiples of 16. +static const int kDiskStride = 16; + +class DISK_Impl CV_FINAL : public DISK +{ +public: + DISK_Impl(const String& modelPath, int maxKeypoints, float scoreThreshold, + const Size& imageSize, int backendId, int targetId) + : maxKeypoints_(maxKeypoints), + scoreThreshold_(scoreThreshold), + imageSize_(imageSize) + { + validateImageSize(imageSize_); + initNet(readNetFromONNX(modelPath), backendId, targetId); + } + + DISK_Impl(const std::vector& bufferModel, int maxKeypoints, float scoreThreshold, + const Size& imageSize, int backendId, int targetId) + : maxKeypoints_(maxKeypoints), + scoreThreshold_(scoreThreshold), + imageSize_(imageSize) + { + validateImageSize(imageSize_); + initNet(readNetFromONNX(bufferModel), backendId, targetId); + } + + void detectAndCompute(InputArray _image, InputArray _mask, + std::vector& keypoints, + OutputArray _descriptors, + bool useProvidedKeypoints) CV_OVERRIDE + { + CV_Assert(!useProvidedKeypoints && "DISK does not support providing keypoints externally"); + + keypoints.clear(); + + Mat image = _image.getMat(); + if (image.empty()) + { + if (_descriptors.needed()) + _descriptors.release(); + return; + } + + Mat mask = _mask.getMat(); + if (!mask.empty()) + { + CV_Assert(mask.type() == CV_8UC1); + CV_Assert(mask.size() == image.size()); + } + + const Size netSize = (imageSize_.width > 0 && imageSize_.height > 0) ? imageSize_ : kDefaultDiskInputSize; + const float scaleX = static_cast(image.cols) / netSize.width; + const float scaleY = static_cast(image.rows) / netSize.height; + + Mat blob; + // 1/255 normalization, swap BGR->RGB, no mean subtraction. + blobFromImage(image, blob, 1.0 / 255.0, netSize, Scalar(), /*swapRB=*/true, /*crop=*/false); + net_.setInput(blob, "image"); + + const std::vector outNames = {"keypoints", "scores", "descriptors"}; + std::vector outs; + net_.forward(outs, outNames); + CV_Assert(outs.size() == 3); + + // DISK's ONNX export emits keypoints as int64 pixel coordinates, scores and + // descriptors as float32. Reshape each output to (N, *) for row-wise indexing. + Mat kptsBlob = outs[0].reshape(1, outs[0].size[1]); // N x 2, int64 + Mat scoresBlob = outs[1].reshape(1, outs[1].size[1]); // N x 1, float32 + Mat descBlob = outs[2].reshape(1, outs[2].size[1]); // N x D, float32 + + CV_Assert(kptsBlob.depth() == CV_64S); + CV_Assert(scoresBlob.depth() == CV_32F); + CV_Assert(descBlob.depth() == CV_32F); + + const int numFeatures = kptsBlob.rows; + CV_Assert(scoresBlob.rows == numFeatures); + CV_Assert(descBlob.rows == numFeatures); + + const int64_t* kptsData = kptsBlob.ptr(); + const float* scoresData = scoresBlob.ptr(); + + std::vector validIndices; + validIndices.reserve(numFeatures); + keypoints.reserve(numFeatures); + + for (int i = 0; i < numFeatures; ++i) + { + const float score = scoresData[i]; + if (score <= scoreThreshold_) + continue; + + const float x = static_cast(kptsData[i * 2]) * scaleX; + const float y = static_cast(kptsData[i * 2 + 1]) * scaleY; + + if (!mask.empty()) + { + const int ix = cvFloor(x); + const int iy = cvFloor(y); + if (ix < 0 || iy < 0 || ix >= mask.cols || iy >= mask.rows) + continue; + if (mask.at(iy, ix) == 0) + continue; + } + + keypoints.emplace_back(x, y, 1.0f, -1.0f, score); + validIndices.push_back(i); + } + + if (maxKeypoints_ > 0 && static_cast(keypoints.size()) > maxKeypoints_) + { + std::vector order(keypoints.size()); + std::iota(order.begin(), order.end(), 0); + std::partial_sort(order.begin(), order.begin() + maxKeypoints_, order.end(), + [&](int a, int b) { return keypoints[a].response > keypoints[b].response; }); + order.resize(maxKeypoints_); + + std::vector kept; + std::vector keptIdx; + kept.reserve(maxKeypoints_); + keptIdx.reserve(maxKeypoints_); + for (int idx : order) + { + kept.push_back(keypoints[idx]); + keptIdx.push_back(validIndices[idx]); + } + keypoints.swap(kept); + validIndices.swap(keptIdx); + } + + if (_descriptors.needed()) + { + if (validIndices.empty()) + { + _descriptors.release(); + return; + } + const int dim = descBlob.cols; + _descriptors.create(static_cast(validIndices.size()), dim, CV_32F); + Mat descriptors = _descriptors.getMat(); + for (size_t i = 0; i < validIndices.size(); ++i) + descBlob.row(validIndices[i]).copyTo(descriptors.row(static_cast(i))); + } + } + + int descriptorSize() const CV_OVERRIDE { return 128; } + int descriptorType() const CV_OVERRIDE { return CV_32F; } + int defaultNorm() const CV_OVERRIDE { return NORM_L2; } + + bool empty() const CV_OVERRIDE { return net_.empty(); } + + void setMaxKeypoints(int maxKeypoints) CV_OVERRIDE { maxKeypoints_ = maxKeypoints; } + int getMaxKeypoints() const CV_OVERRIDE { return maxKeypoints_; } + + void setScoreThreshold(float threshold) CV_OVERRIDE { scoreThreshold_ = threshold; } + float getScoreThreshold() const CV_OVERRIDE { return scoreThreshold_; } + + void setImageSize(const Size& size) CV_OVERRIDE + { + validateImageSize(size); + imageSize_ = size; + } + Size getImageSize() const CV_OVERRIDE { return imageSize_; } + + String getDefaultName() const CV_OVERRIDE { return Feature2D::getDefaultName() + ".DISK"; } + +private: + void initNet(const Net& net, int backendId, int targetId) + { + net_ = net; + net_.setPreferableBackend(backendId); + net_.setPreferableTarget(targetId); + } + + static void validateImageSize(const Size& size) + { + if (size.width == 0 && size.height == 0) + return; // use default + CV_Assert(size.width > 0 && size.height > 0); + CV_Assert(size.width % kDiskStride == 0); + CV_Assert(size.height % kDiskStride == 0); + } + + int maxKeypoints_; + float scoreThreshold_; + Size imageSize_; + Net net_; +}; + +Ptr DISK::create(const String& modelPath, int maxKeypoints, float scoreThreshold, + const Size& imageSize, int backendId, int targetId) +{ + CV_TRACE_FUNCTION(); + return makePtr(modelPath, maxKeypoints, scoreThreshold, imageSize, backendId, targetId); +} + +Ptr DISK::create(const std::vector& bufferModel, int maxKeypoints, float scoreThreshold, + const Size& imageSize, int backendId, int targetId) +{ + CV_TRACE_FUNCTION(); + return makePtr(bufferModel, maxKeypoints, scoreThreshold, imageSize, backendId, targetId); +} + +String DISK::getDefaultName() const +{ + return Feature2D::getDefaultName() + ".DISK"; +} + +} // namespace cv + +#endif // HAVE_OPENCV_DNN diff --git a/modules/features/test/npy_blob.cpp b/modules/features/test/npy_blob.cpp new file mode 100644 index 0000000000..a67db5f2e7 --- /dev/null +++ b/modules/features/test/npy_blob.cpp @@ -0,0 +1,103 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2017, Intel Corporation, all rights reserved. +// Third party copyrights are property of their respective owners. + +#include "test_precomp.hpp" +#include "npy_blob.hpp" + +namespace cv +{ + +static std::string getType(const std::string& header) +{ + std::string field = "'descr':"; + int idx = header.find(field); + CV_Assert(idx != -1); + + int from = header.find('\'', idx + field.size()) + 1; + int to = header.find('\'', from); + return header.substr(from, to - from); +} + +static std::string getFortranOrder(const std::string& header) +{ + std::string field = "'fortran_order':"; + int idx = header.find(field); + CV_Assert(idx != -1); + + int from = header.find_last_of(' ', idx + field.size()) + 1; + int to = header.find(',', from); + return header.substr(from, to - from); +} + +static std::vector getShape(const std::string& header) +{ + std::string field = "'shape':"; + int idx = header.find(field); + CV_Assert(idx != -1); + + int from = header.find('(', idx + field.size()) + 1; + int to = header.find(')', from); + + std::string shapeStr = header.substr(from, to - from); + if (shapeStr.empty()) + return std::vector(1, 1); + + // Remove all commas. + shapeStr.erase(std::remove(shapeStr.begin(), shapeStr.end(), ','), + shapeStr.end()); + + std::istringstream ss(shapeStr); + int value; + + std::vector shape; + while (ss >> value) + { + shape.push_back(value); + } + return shape; +} + +Mat blobFromNPY(const std::string& path) +{ + std::ifstream ifs(path.c_str(), std::ios::binary); + CV_Assert(ifs.is_open()); + + std::string magic(6, '*'); + ifs.read(&magic[0], magic.size()); + CV_Assert(magic == "\x93NUMPY"); + + ifs.ignore(1); // Skip major version byte. + ifs.ignore(1); // Skip minor version byte. + + unsigned short headerSize; + ifs.read((char*)&headerSize, sizeof(headerSize)); + + std::string header(headerSize, '*'); + ifs.read(&header[0], header.size()); + + // Extract data type. + int matType; + if (getType(header) == " shape = getShape(header); + + Mat blob(shape, matType); + ifs.read((char*)blob.data, blob.total() * blob.elemSize()); + CV_Assert((size_t)ifs.gcount() == blob.total() * blob.elemSize()); + + return blob; +} + +} // namespace cv diff --git a/modules/features/test/npy_blob.hpp b/modules/features/test/npy_blob.hpp new file mode 100644 index 0000000000..60a7c2671a --- /dev/null +++ b/modules/features/test/npy_blob.hpp @@ -0,0 +1,20 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2017, Intel Corporation, all rights reserved. +// Third party copyrights are property of their respective owners. + +#ifndef __OPENCV_DNN_TEST_NPY_BLOB_HPP__ +#define __OPENCV_DNN_TEST_NPY_BLOB_HPP__ + +namespace cv +{ + +// Parse serialized NumPy array by np.save(...) +// Based on specification of .npy data format. +Mat blobFromNPY(const std::string& path); + +} + +#endif diff --git a/modules/features/test/test_disk.cpp b/modules/features/test/test_disk.cpp new file mode 100644 index 0000000000..56ca784c73 --- /dev/null +++ b/modules/features/test/test_disk.cpp @@ -0,0 +1,174 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// Copyright (C) 2026, BigVision LLC, all rights reserved. +// Third party copyrights are property of their respective owners. + +#include "test_precomp.hpp" +#include "npy_blob.hpp" + +#ifdef HAVE_OPENCV_DNN + +#include "opencv2/dnn.hpp" +#include "opencv2/core/utils/configuration.private.hpp" + +namespace opencv_test { namespace { + +static void skipIfClassicDnnEngine() +{ + const auto engine = static_cast( + cv::utils::getConfigurationParameterSizeT("OPENCV_FORCE_DNN_ENGINE", cv::dnn::ENGINE_AUTO)); + if (engine == cv::dnn::ENGINE_CLASSIC) + throw SkipTestException("DISK ONNX model is not supported by the classic DNN engine"); +} + +static void testDiskRegression(const Size& imageSize, const std::string& tag) +{ + skipIfClassicDnnEngine(); + applyTestTag(CV_TEST_TAG_MEMORY_2GB); + + Mat refKpts = blobFromNPY(cvtest::findDataFile("features/disk/box_in_scene_" + tag + "_kpts.npy")); + Mat refDesc = blobFromNPY(cvtest::findDataFile("features/disk/box_in_scene_" + tag + "_desc.npy")); + if (refKpts.type() != CV_32F) + refKpts.convertTo(refKpts, CV_32F); + ASSERT_EQ(refKpts.cols, 3); + const int n = refKpts.rows; + + const std::string modelPath = cvtest::findDataFile("dnn/disk.onnx", false); + + Ptr detector; + ASSERT_NO_THROW(detector = DISK::create(modelPath, n, 0.0f, imageSize)); + ASSERT_TRUE(detector); + EXPECT_FALSE(detector->empty()); + EXPECT_EQ(detector->descriptorSize(), 128); + EXPECT_EQ(detector->descriptorType(), CV_32F); + EXPECT_EQ(detector->defaultNorm(), NORM_L2); + + Mat img = imread(cvtest::findDataFile("shared/box_in_scene.png")); + ASSERT_FALSE(img.empty()); + + std::vector keypoints; + Mat descriptors; + detector->detectAndCompute(img, noArray(), keypoints, descriptors); + + ASSERT_EQ(static_cast(keypoints.size()), n) << "keypoint count mismatch (" << tag << ")"; + ASSERT_EQ(descriptors.rows, n); + ASSERT_EQ(descriptors.cols, refDesc.cols); + ASSERT_EQ(descriptors.type(), CV_32F); + + Mat pos(n, 2, CV_32F), resp(n, 1, CV_32F); + for (int i = 0; i < n; ++i) + { + pos.at(i, 0) = keypoints[i].pt.x; + pos.at(i, 1) = keypoints[i].pt.y; + resp.at(i, 0) = keypoints[i].response; + } + + EXPECT_LE(cvtest::norm(pos, refKpts.colRange(0, 2), NORM_INF), 1e-3) + << "keypoint positions differ (" << tag << ")"; + EXPECT_LE(cvtest::norm(resp, refKpts.col(2), NORM_INF), 0.1) + << "keypoint responses differ (" << tag << ")"; + EXPECT_LE(cvtest::norm(descriptors, refDesc, NORM_INF), 1e-2) + << "descriptors differ (" << tag << ")"; +} + +TEST(Features2d_DISK, regression_default) +{ + testDiskRegression(Size(), "default"); +} + +TEST(Features2d_DISK, regression_512x384) +{ + testDiskRegression(Size(512, 384), "512x384"); +} + +TEST(Features2d_DISK, MaxKeypointsAndThreshold) +{ + skipIfClassicDnnEngine(); + applyTestTag(CV_TEST_TAG_MEMORY_2GB); + + const std::string modelPath = cvtest::findDataFile("dnn/disk.onnx", false); + + Ptr detector = DISK::create(modelPath); + ASSERT_TRUE(detector); + + Mat img = imread(cvtest::findDataFile("shared/lena.png")); + ASSERT_FALSE(img.empty()); + + std::vector baseKpts; + Mat baseDesc; + detector->detectAndCompute(img, noArray(), baseKpts, baseDesc); + ASSERT_GT(baseKpts.size(), 50u); + + const int kCap = 50; + detector->setMaxKeypoints(kCap); + EXPECT_EQ(detector->getMaxKeypoints(), kCap); + + std::vector capKpts; + Mat capDesc; + detector->detectAndCompute(img, noArray(), capKpts, capDesc); + EXPECT_EQ(capKpts.size(), static_cast(kCap)); + EXPECT_EQ(capDesc.rows, kCap); + + float minKept = std::numeric_limits::max(); + for (const KeyPoint& kp : capKpts) + minKept = std::min(minKept, kp.response); + + detector->setMaxKeypoints(-1); + detector->setScoreThreshold(minKept); + std::vector thrKpts; + detector->detectAndCompute(img, noArray(), thrKpts, noArray()); + for (const KeyPoint& kp : thrKpts) + EXPECT_GT(kp.response, minKept); +} + +TEST(Features2d_DISK, MaskSupport) +{ + skipIfClassicDnnEngine(); + applyTestTag(CV_TEST_TAG_MEMORY_2GB); + + const std::string modelPath = cvtest::findDataFile("dnn/disk.onnx", false); + + Ptr detector = DISK::create(modelPath); + Mat img = imread(cvtest::findDataFile("shared/lena.png")); + ASSERT_FALSE(img.empty()); + + Mat mask = Mat::zeros(img.size(), CV_8UC1); + const Rect roi(img.cols / 4, img.rows / 4, img.cols / 2, img.rows / 2); + mask(roi).setTo(255); + + std::vector keypoints; + Mat descriptors; + detector->detectAndCompute(img, mask, keypoints, descriptors); + + ASSERT_FALSE(keypoints.empty()); + ASSERT_EQ(descriptors.rows, static_cast(keypoints.size())); + + for (const KeyPoint& kp : keypoints) + { + EXPECT_TRUE(roi.contains(Point(cvFloor(kp.pt.x), cvFloor(kp.pt.y)))) + << "Keypoint " << kp.pt << " escaped the mask ROI " << roi; + } +} + +TEST(Features2d_DISK, InvalidImageSize) +{ + skipIfClassicDnnEngine(); + const std::string modelPath = cvtest::findDataFile("dnn/disk.onnx", false); + + EXPECT_THROW(DISK::create(modelPath, -1, 0.0f, Size(1000, 1024)), cv::Exception); + EXPECT_THROW(DISK::create(modelPath, -1, 0.0f, Size(1024, 1000)), cv::Exception); + EXPECT_THROW(DISK::create(modelPath, -1, 0.0f, Size(-16, 1024)), cv::Exception); + + Ptr detector; + ASSERT_NO_THROW(detector = DISK::create(modelPath, -1, 0.0f, Size())); + ASSERT_TRUE(detector); + + EXPECT_THROW(detector->setImageSize(Size(15, 1024)), cv::Exception); + EXPECT_NO_THROW(detector->setImageSize(Size(512, 512))); + EXPECT_EQ(detector->getImageSize(), Size(512, 512)); +} + +}} // namespace + +#endif // HAVE_OPENCV_DNN diff --git a/modules/features/test/test_main.cpp b/modules/features/test/test_main.cpp index 93e4d2860e..68c90f775e 100644 --- a/modules/features/test/test_main.cpp +++ b/modules/features/test/test_main.cpp @@ -7,4 +7,11 @@ #include #endif -CV_TEST_MAIN("cv") +static +void initTests() +{ + cvtest::addDataSearchEnv("OPENCV_DNN_TEST_DATA_PATH"); + cvtest::addDataSearchSubDirectory(""); // override "cv" prefix below to access without "../dnn" hacks +} + +CV_TEST_MAIN("cv", initTests())