From 527f01449dd4490fa107712d5547832d7ae2641d Mon Sep 17 00:00:00 2001 From: Yang Guanyuhan <1523543870@qq.com> Date: Fri, 5 Jun 2026 19:20:53 +0800 Subject: [PATCH] Merge pull request #28986 from YangGuanyuhan:ai-aliked-lightglue-pipeline [GSOC] feat: Add ALIKED feature extractor and LightGlue matcher with DNN integration #28986 ## PR Description ### Summary Integrate ALIKED and LightGlue into OpenCV's `features` module as native`Feature2D` and `DescriptorMatcher` implementations, enabling end-to-end neural feature matching within OpenCV's ecosystem. --- ### What's included #### New classes - **`cv::ALIKED`** extends `Feature2D` - CNN-based keypoint detection - 128-D descriptor extraction via ONNX Runtime - **`cv::LightGlueMatcher`** extends `DescriptorMatcher` - Deep feature matching with spatial context - Uses keypoints and image sizes during matching --- #### API design - Standard OpenCV patterns: - `detectAndCompute()` - `match()` - `knnMatch()` - Multiple factory methods: - ONNX model path - In-memory model buffer - Pre-loaded `dnn::Net` - `Params` structs use `CV_EXPORTS_W_SIMPLE` for Python/Java bindings support - Optional DNN dependency: - `HAVE_OPENCV_DNN` guards - Stub implementations throw `StsNotImplemented` --- ### Files added | File | Description | |------|-------------| | `src/feature2d_aliked.cpp` | ALIKED implementation | | `src/matchers_lightglue.cpp` | LightGlueMatcher implementation | | `src/aliked_context.hpp` | Shared internal context struct | | `test/test_aliked_lightglue.cpp` | Unit tests (9 test cases) | | `samples/cpp/example_features_aliked_lightglue.cpp` | Demo application | --- ### Files modified - `CMakeLists.txt` - Add `opencv_dnn` as optional dependency - `features.hpp` - Add ALIKED and LightGlueMatcher declarations - `precomp.hpp` - Add DNN include guard --- ### Usage ```cpp // Feature extraction Ptr aliked = ALIKED::create("aliked-n16rot-top1k-640.onnx"); vector kpts; Mat descs; aliked->detectAndCompute(image, Mat(), kpts, descs); // Feature matching Ptr lg = LightGlueMatcher::create("aliked_lightglue.onnx"); lg->setPairInfo( kpts1Mat, kpts2Mat, img1.size(), img2.size() ); vector matches; lg->match(descs1, descs2, matches); ```` please refer to samples/cpp/example_features_aliked_lightglue.cpp --- ### Test plan * Build with `BUILD_LIST=features,dnn` * Build without DNN: * Verify stubs compile * Verify `StsNotImplemented` is thrown * Run: * `ctest -R Features2d_ALIKED` * `ctest -R Features2d_LightGlueMatcher` * Run sample application with: * Real images * Real ONNX models * Verify Python/Java bindings compile and work --- ### Related Phase 1 of the "End-to-End AI Feature Extraction and LightGlue Matching Pipeline" GSoC project. Designed to be extensible to: * XFeat * SuperPoint * Other neural feature extractors ### test dependency Depends on the opencv_extra PR adding ALIKED and LightGlue test models: - [opencv_extra PR](https://github.com/opencv/opencv_extra/pull/1366) This PR adds the following ONNX models to `download_models.py`: - `aliked-n16rot-top1k-640.onnx` - `aliked_lightglue.onnx` These models are required for the `features2d` tests in the main OpenCV repository to validate the ALIKED and LightGlue feature extraction and matching pipeline. ### Pull Request Readiness Checklist See details at https://github.com/opencv/opencv/wiki/How_to_contribute#making-a-good-pull-request - [x] I agree to contribute to the project under Apache 2 License. - [x] To the best of my knowledge, the proposed patch is not based on a code under GPL or another license that is incompatible with OpenCV - [x] The PR is proposed to the proper branch - [ ] There is a reference to the original bug report and related work - [x] There is accuracy test, performance test and test data in opencv_extra repository, if applicable Patch to opencv_extra has the same branch name. - [x] The feature is well documented and sample code can be built with the project CMake --- doc/opencv.bib | 15 + modules/dnn/src/layers/cast2_layer.cpp | 38 +++ modules/dnn/src/layers/topk2_layer.cpp | 1 - modules/features/include/opencv2/features.hpp | 99 ++++++ modules/features/src/aliked_context.hpp | 21 ++ modules/features/src/feature2d_aliked.cpp | 186 ++++++++++++ modules/features/src/matchers_lightglue.cpp | 282 ++++++++++++++++++ modules/features/src/precomp.hpp | 4 + .../features/test/test_aliked_lightglue.cpp | 171 +++++++++++ modules/features/test/test_main.cpp | 1 + .../opencv2/stitching/detail/matchers.hpp | 40 +++ modules/stitching/src/matchers.cpp | 143 +++++++++ .../cpp/example_features_aliked_lightglue.cpp | 169 +++++++++++ samples/cpp/stitching_detailed.cpp | 77 ++++- 14 files changed, 1240 insertions(+), 7 deletions(-) create mode 100644 modules/features/src/aliked_context.hpp create mode 100644 modules/features/src/feature2d_aliked.cpp create mode 100644 modules/features/src/matchers_lightglue.cpp create mode 100644 modules/features/test/test_aliked_lightglue.cpp create mode 100644 samples/cpp/example_features_aliked_lightglue.cpp diff --git a/doc/opencv.bib b/doc/opencv.bib index 85040e7c7f..62c49a3fcd 100644 --- a/doc/opencv.bib +++ b/doc/opencv.bib @@ -27,6 +27,13 @@ publisher = {IEEE Computer Society}, address = {Washington, DC, USA}, } +@article{Zhao23, + author = {Zhao, Xiaoming and Wu, Xingming and Chen, Weihai and Chen, Peter C. Y. and Xu, Qingsong and Li, Zhengguo}, + title = {ALIKED: A Lighter Keypoint and Descriptor Extraction Network via Deformable Transformation}, + journal = {arXiv preprint arXiv:2304.03608}, + year = {2023}, + url = {https://arxiv.org/abs/2304.03608}, +} @inproceedings{Arandjelovic:2012:TTE:2354409.2355123, author = {Arandjelovic, Relja}, title = {Three Things Everyone Should Know to Improve Object Retrieval}, @@ -759,6 +766,14 @@ year = {2020}, url = {https://arxiv.org/abs/1911.08947} } +@inproceedings{Lindenberger23, + author = {Lindenberger, Philipp and Sarlin, Paul-Edouard and Pollefeys, Marc}, + title = {LightGlue: Local Feature Matching at Light Speed}, + booktitle = {Proceedings of the IEEE/CVF International Conference on Computer Vision (ICCV)}, + year = {2023}, + pages = {17627--17638}, + url = {https://arxiv.org/abs/2306.13643}, +} @article{Louhichi07, author = {Louhichi, H. and Fournel, T. and Lavest, J. M. and Ben Aissia, H.}, title = {Self-calibration of Scheimpflug cameras: an easy protocol}, diff --git a/modules/dnn/src/layers/cast2_layer.cpp b/modules/dnn/src/layers/cast2_layer.cpp index ba97688ae9..0e48a0e97b 100644 --- a/modules/dnn/src/layers/cast2_layer.cpp +++ b/modules/dnn/src/layers/cast2_layer.cpp @@ -21,6 +21,40 @@ namespace cv { namespace dnn { namespace { + // ONNX Cast float->int truncates toward zero; Mat::convertTo rounds. Truncate to match the spec. + template + inline void truncateToIntImpl(const Mat& src, Mat& dst) + { + const int n = (int)src.total() * src.channels(); + DT* d = dst.ptr
(); + if (src.depth() == CV_32F) + { + const float* s = src.ptr(); + for (int i = 0; i < n; ++i) + d[i] = saturate_cast
(std::trunc(s[i])); + } + else + { + const double* s = src.ptr(); + for (int i = 0; i < n; ++i) + d[i] = saturate_cast
(std::trunc(s[i])); + } + } + + inline void truncateFloatToInt(const Mat& src, Mat& dst) + { + switch (dst.depth()) + { + case CV_8U: truncateToIntImpl(src, dst); break; + case CV_8S: truncateToIntImpl(src, dst); break; + case CV_16U: truncateToIntImpl(src, dst); break; + case CV_16S: truncateToIntImpl(src, dst); break; + case CV_32S: truncateToIntImpl(src, dst); break; + case CV_64S: truncateToIntImpl(src, dst); break; + default: src.convertTo(dst, dst.depth()); break; + } + } + inline void castQuantized(const Mat& src, Mat& dst, int targetDepth) { if (targetDepth == CV_16F) @@ -297,6 +331,10 @@ public: else src.convertTo(dst, ddepth); } + else if ((sdepth == CV_32F || sdepth == CV_64F) && CV_IS_INT_TYPE(ddepth)) + { + truncateFloatToInt(src, dst); + } else { src.convertTo(dst, ddepth); diff --git a/modules/dnn/src/layers/topk2_layer.cpp b/modules/dnn/src/layers/topk2_layer.cpp index 217f8315ca..aa0cacf403 100644 --- a/modules/dnn/src/layers/topk2_layer.cpp +++ b/modules/dnn/src/layers/topk2_layer.cpp @@ -50,7 +50,6 @@ public: axis = params.get("axis", -1); largest = params.get("largest", 1) == 1; sorted = params.get("sorted", 1) == 1; - CV_CheckTrue(sorted, "TopK2: sorted == false is not supported"); if (params.has("k")) { K = params.get("k"); CV_CheckGT(K, 0, "TopK2: K needs to be a positive integer"); diff --git a/modules/features/include/opencv2/features.hpp b/modules/features/include/opencv2/features.hpp index 5fff55fb3b..55519cca94 100644 --- a/modules/features/include/opencv2/features.hpp +++ b/modules/features/include/opencv2/features.hpp @@ -50,6 +50,10 @@ #include "opencv2/flann/miniflann.hpp" #endif +#ifdef HAVE_OPENCV_DNN +#include "opencv2/dnn.hpp" +#endif + /** @defgroup features Features Framework @{ @@ -835,6 +839,46 @@ public: CV_WRAP virtual const std::vector >& getBlobContours() const = 0; }; +/** @brief ALIKED feature detector and descriptor extractor. + +ALIKED (A Lightweight Image KEYpoint Detector) is a CNN-based feature detector and descriptor +extractor, as described in @cite Zhao23 . It produces 128-dimensional float descriptors and +keypoints with sub-pixel accuracy. + +The model expects RGB input [1,3,H,W] and internally converts BGR images to RGB. +*/ +class CV_EXPORTS_W ALIKED : public Feature2D +{ +protected: + ALIKED(); +public: + virtual ~ALIKED(); + + struct CV_EXPORTS_W_SIMPLE Params + { + CV_WRAP Params(); + CV_PROP_RW Size inputSize; //!< Input image size for the network, default 640x640 + CV_PROP_RW bool normalizeDescriptors; //!< Whether to L2-normalize descriptors, default true + CV_PROP_RW int engine; //!< DNN engine type (dnn::EngineType), default ENGINE_NEW + CV_PROP_RW int backend; //!< DNN backend, default DNN_BACKEND_DEFAULT + CV_PROP_RW int target; //!< DNN target, default DNN_TARGET_CPU + }; + + /** @brief Creates ALIKED from a model file path. + @param modelPath Path to the ONNX model file. + @param params ALIKED parameters. + */ + CV_WRAP static Ptr create(const String& modelPath, const ALIKED::Params& params = ALIKED::Params()); + +#ifdef HAVE_OPENCV_DNN + /** @brief Creates ALIKED from in-memory model data. + @param modelData Buffer containing the model data. + @param params ALIKED parameters. + */ + static Ptr create(const std::vector& modelData, const ALIKED::Params& params = ALIKED::Params()); +#endif +}; + /****************************************************************************************\ * Distance * @@ -1252,6 +1296,61 @@ protected: #endif +/** @brief LightGlue feature matcher. + +LightGlue is a CNN-based feature matcher, as described in @cite Lindenberger23 . It takes +keypoint locations and descriptors from two images and directly predicts match pairs. Unlike +traditional matchers that compute descriptor distances, LightGlue uses attention mechanisms +to produce confidence scores for each potential match pair. + +The matcher extends DescriptorMatcher and supports the standard match(), knnMatch(), and +radiusMatch() interfaces. Context (keypoints and image sizes) must be provided via +setPairInfo() before matching. +*/ +class CV_EXPORTS_W LightGlueMatcher : public DescriptorMatcher +{ +protected: + LightGlueMatcher(); +public: + virtual ~LightGlueMatcher(); + + /** @brief Creates LightGlueMatcher from a model file path. + @param modelPath Path to the ONNX model file. + @param scoreThreshold Match confidence threshold. + @param backend DNN backend + @param target DNN target + */ + CV_WRAP static Ptr create(const String& modelPath, float scoreThreshold = 0.0f, int backend = 0, int target = 0); + +#ifdef HAVE_OPENCV_DNN + /** @brief Creates LightGlueMatcher from in-memory model data. + @param modelData Buffer containing the model data. + @param scoreThreshold Match confidence threshold. + @param backend DNN backend + @param target DNN target + */ + CV_WRAP_AS(createFromMemory) static Ptr create(const std::vector& modelData, float scoreThreshold = 0.0f, int backend = 0, int target = 0); +#endif + + /** @brief Sets the keypoint and image size context for the next match() call. + + This provides the spatial context that LightGlue needs in addition to descriptors. + Must be called before match()/knnMatch()/radiusMatch() unless using automatic context + from in-process ALIKED instances. + + @param queryKpts Query image keypoints (Nx2 float matrix with x,y coordinates). + @param trainKpts Train image keypoints (Nx2 float matrix with x,y coordinates). + @param queryImageSize Size of the query image (width, height). + @param trainImageSize Size of the train image (width, height). + */ + CV_WRAP virtual void setPairInfo(InputArray queryKpts, InputArray trainKpts, + Size queryImageSize = Size(), Size trainImageSize = Size()) = 0; + + /** @brief Clears stored pair context information. + */ + CV_WRAP virtual void clearPairInfo() = 0; +}; + //! @} features_match /****************************************************************************************\ diff --git a/modules/features/src/aliked_context.hpp b/modules/features/src/aliked_context.hpp new file mode 100644 index 0000000000..54c76cd382 --- /dev/null +++ b/modules/features/src/aliked_context.hpp @@ -0,0 +1,21 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. + +#ifndef OPENCV_FEATURES_ALIKED_CONTEXT_HPP +#define OPENCV_FEATURES_ALIKED_CONTEXT_HPP + +#include "opencv2/features.hpp" + +namespace cv +{ + +struct ALIKEDContext +{ + Mat normalizedKeypoints; // Nx2 float, coordinates in [-1, 1] + Size imageSize; +}; + +} // namespace cv + +#endif diff --git a/modules/features/src/feature2d_aliked.cpp b/modules/features/src/feature2d_aliked.cpp new file mode 100644 index 0000000000..1a80b00eae --- /dev/null +++ b/modules/features/src/feature2d_aliked.cpp @@ -0,0 +1,186 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. + +#include "precomp.hpp" + +#ifdef HAVE_OPENCV_DNN +#include "opencv2/dnn.hpp" +#include "aliked_context.hpp" +#endif + +namespace cv +{ + +ALIKED::ALIKED() {} +ALIKED::~ALIKED() {} + +ALIKED::Params::Params() +{ + inputSize = Size(640, 640); + normalizeDescriptors = true; +#ifdef HAVE_OPENCV_DNN + engine = dnn::ENGINE_NEW; + backend = dnn::DNN_BACKEND_DEFAULT; + target = dnn::DNN_TARGET_CPU; +#else + engine = -1; + backend = -1; + target = -1; +#endif +} + +#ifdef HAVE_OPENCV_DNN + +class ALIKEDImpl : public ALIKED +{ +public: + ALIKEDImpl(const ALIKED::Params& _params, const String& modelPath) + : params(_params) + { + net = dnn::readNet(modelPath, "", "", static_cast(params.engine)); + CV_Assert(!net.empty()); + net.setPreferableBackend(params.backend); + net.setPreferableTarget(params.target); + } + + ALIKEDImpl(const std::vector& modelData, const ALIKED::Params& _params) + : params(_params) + { + net = dnn::readNetFromONNX(modelData); + CV_Assert(!net.empty()); + net.setPreferableBackend(params.backend); + net.setPreferableTarget(params.target); + } + + void detectAndCompute(InputArray image, InputArray mask, + std::vector& keypoints, + OutputArray descriptors, + bool useProvidedKeypoints) CV_OVERRIDE; + + int descriptorSize() const CV_OVERRIDE; + int descriptorType() const CV_OVERRIDE; + int defaultNorm() const CV_OVERRIDE; + bool empty() const CV_OVERRIDE; + + const ALIKEDContext& getLastContext() const { return lastContext; } + +protected: + dnn::Net net; + ALIKED::Params params; + ALIKEDContext lastContext; + + void runNetwork(InputArray image, std::vector& keypoints, + Mat& descriptors, Mat& scores); +}; + +void ALIKEDImpl::runNetwork(InputArray _image, std::vector& keypoints, + Mat& descriptors, Mat& scores) +{ + Mat image = _image.getMat(); + Size inputSz = params.inputSize; + Size origSize = image.size(); + + // BGR->RGB conversion via swapRB=true + Mat blob = dnn::blobFromImage(image, 1.0/255.0, inputSz, Scalar(), /*swapRB=*/true, /*crop=*/false); + + net.setInput(blob, "image"); + + std::vector outNames = {"keypoints", "descriptors", "scores"}; + std::vector outputs; + net.forward(outputs, outNames); + + CV_Assert(outputs.size() == 3); + + // ORT engine drops the batch dimension, so outputs are: + // keypoints: [N, 2] (not [1, N, 2]) + // descriptors: [N, 128] (not [1, N, 128]) + // scores: [N] (not [1, N, 1]) + int N = outputs[0].rows; + + Mat normKpts = outputs[0].reshape(0, N); // Nx2 + Mat desc = outputs[1].reshape(0, N); // Nx128 + Mat scr = outputs[2].reshape(0, N); // Nx1 + + // Store normalized keypoints for LightGlue context + lastContext.normalizedKeypoints = normKpts.clone(); + lastContext.imageSize = origSize; + + // Convert normalized [-1,1] coordinates to pixel coordinates + keypoints.resize(N); + for (int i = 0; i < N; i++) + { + float nx = normKpts.at(i, 0); + float ny = normKpts.at(i, 1); + float px = (nx + 1.0f) * 0.5f * (float)origSize.width; + float py = (ny + 1.0f) * 0.5f * (float)origSize.height; + float score = scr.at(i, 0); + keypoints[i] = KeyPoint(px, py, 1.0f, -1.0f, score, 0, -1); + } + + // Optionally L2-normalize descriptors + if (params.normalizeDescriptors) + { + for (int i = 0; i < N; i++) + { + Mat row = desc.row(i); + normalize(row, row); + } + } + + descriptors = desc; + scores = scr; +} + +void ALIKEDImpl::detectAndCompute(InputArray image, InputArray mask, + std::vector& keypoints, + OutputArray descriptors, + bool useProvidedKeypoints) +{ + CV_INSTRUMENT_REGION(); + CV_UNUSED(mask); + CV_UNUSED(useProvidedKeypoints); + + if (image.empty()) + { + keypoints.clear(); + descriptors.release(); + return; + } + + Mat desc; + Mat sc; + runNetwork(image, keypoints, desc, sc); + + if (descriptors.needed()) + desc.copyTo(descriptors); +} + +int ALIKEDImpl::descriptorSize() const { return 128; } +int ALIKEDImpl::descriptorType() const { return CV_32F; } +int ALIKEDImpl::defaultNorm() const { return NORM_L2; } +bool ALIKEDImpl::empty() const { return net.empty(); } + +Ptr ALIKED::create(const String& modelPath, const ALIKED::Params& params) +{ + return makePtr(params, modelPath); +} + +Ptr ALIKED::create(const std::vector& modelData, const ALIKED::Params& params) +{ + return makePtr(modelData, params); +} + +#else // !HAVE_OPENCV_DNN + +Ptr ALIKED::create(const String& modelPath, const ALIKED::Params& params) +{ + CV_UNUSED(modelPath); + CV_UNUSED(params); + CV_Error(cv::Error::StsNotImplemented, + "ALIKED requires OpenCV built with opencv_dnn module!"); +} + +#endif // HAVE_OPENCV_DNN + +} // namespace cv diff --git a/modules/features/src/matchers_lightglue.cpp b/modules/features/src/matchers_lightglue.cpp new file mode 100644 index 0000000000..798c673362 --- /dev/null +++ b/modules/features/src/matchers_lightglue.cpp @@ -0,0 +1,282 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. + +#include "precomp.hpp" + +#ifdef HAVE_OPENCV_DNN +#include "opencv2/dnn.hpp" +#include "aliked_context.hpp" +#endif + +namespace cv +{ + +LightGlueMatcher::LightGlueMatcher() {} +LightGlueMatcher::~LightGlueMatcher() {} + +#ifdef HAVE_OPENCV_DNN + +struct LightGluePairContext +{ + Mat queryKeypoints; // Nx2 float (normalized [-1,1] or pixel) + Mat trainKeypoints; // Mx2 float + Size queryImageSize; + Size trainImageSize; + bool valid; + + LightGluePairContext() : valid(false) {} + + void clear() + { + queryKeypoints.release(); + trainKeypoints.release(); + queryImageSize = Size(); + trainImageSize = Size(); + valid = false; + } +}; + +class LightGlueMatcherImpl : public LightGlueMatcher +{ +public: + LightGlueMatcherImpl(const String& modelPath, float _scoreThreshold, int backend, int target) + { + scoreThreshold = _scoreThreshold; + net = dnn::readNet(modelPath, "", ""); + CV_Assert(!net.empty()); + net.setPreferableBackend(backend); + net.setPreferableTarget(target); + } + + LightGlueMatcherImpl(const std::vector& modelData, float _scoreThreshold, int backend, int target) + { + scoreThreshold = _scoreThreshold; + net = dnn::readNetFromONNX(modelData); + CV_Assert(!net.empty()); + net.setPreferableBackend(backend); + net.setPreferableTarget(target); + } + + // Private constructor for clone() — shares the already-loaded network + LightGlueMatcherImpl(const dnn::Net& _net, float _scoreThreshold) + : net(_net), scoreThreshold(_scoreThreshold) {}; + + // DescriptorMatcher interface + bool isMaskSupported() const CV_OVERRIDE { return false; } + Ptr clone(bool emptyTrainData) const CV_OVERRIDE; + + // LightGlueMatcher interface + void setPairInfo(InputArray queryKpts, InputArray trainKpts, + Size queryImageSize = Size(), Size trainImageSize = Size()) CV_OVERRIDE; + void clearPairInfo() CV_OVERRIDE; + +protected: + void knnMatchImpl(InputArray queryDescriptors, + std::vector>& matches, int k, + InputArrayOfArrays masks = noArray(), + bool compactResult = false) CV_OVERRIDE; + void radiusMatchImpl(InputArray queryDescriptors, + std::vector>& matches, float maxDistance, + InputArrayOfArrays masks = noArray(), + bool compactResult = false) CV_OVERRIDE; + + void lightglueMatch(const Mat& queryDesc, const Mat& trainDesc, + const Mat& queryKpts, const Mat& trainKpts, + Size queryImgSize, Size trainImgSize, + std::vector& matches); + + bool resolveContext(Mat& queryKpts, Mat& trainKpts, + Size& queryImgSize, Size& trainImgSize); + + dnn::Net net; + float scoreThreshold; + LightGluePairContext pairContext; +}; + +Ptr LightGlueMatcherImpl::clone(bool emptyTrainData) const +{ + Ptr m = makePtr(net, scoreThreshold); + // Always copy pairContext - it's matcher state, not train data + m->pairContext = pairContext; + if (!emptyTrainData) + { + m->trainDescCollection = trainDescCollection; + m->utrainDescCollection = utrainDescCollection; + } + return m; +} + +void LightGlueMatcherImpl::setPairInfo(InputArray _queryKpts, InputArray _trainKpts, + Size _queryImageSize, Size _trainImageSize) +{ + pairContext.queryKeypoints = _queryKpts.getMat().clone(); + pairContext.trainKeypoints = _trainKpts.getMat().clone(); + pairContext.queryImageSize = _queryImageSize; + pairContext.trainImageSize = _trainImageSize; + pairContext.valid = true; +} + +void LightGlueMatcherImpl::clearPairInfo() +{ + pairContext.clear(); +} + +bool LightGlueMatcherImpl::resolveContext(Mat& queryKpts, Mat& trainKpts, + Size& queryImgSize, Size& trainImgSize) +{ + if (pairContext.valid) + { + queryKpts = pairContext.queryKeypoints; + trainKpts = pairContext.trainKeypoints; + queryImgSize = pairContext.queryImageSize; + trainImgSize = pairContext.trainImageSize; + return true; + } + return false; +} + +void LightGlueMatcherImpl::lightglueMatch(const Mat& queryDesc, const Mat& trainDesc, + const Mat& queryKpts, const Mat& trainKpts, + Size queryImgSize, Size trainImgSize, + std::vector& matches) +{ + int N = queryDesc.rows; + int M = trainDesc.rows; + + // Normalize keypoints to [-1, 1] if in pixel coordinates + Mat kpts0 = queryKpts.clone(); + Mat kpts1 = trainKpts.clone(); + + if (queryImgSize.width > 0 && queryImgSize.height > 0) + { + for (int i = 0; i < kpts0.rows; i++) + { + kpts0.at(i, 0) = kpts0.at(i, 0) / (float)queryImgSize.width * 2.0f - 1.0f; + kpts0.at(i, 1) = kpts0.at(i, 1) / (float)queryImgSize.height * 2.0f - 1.0f; + } + } + if (trainImgSize.width > 0 && trainImgSize.height > 0) + { + for (int i = 0; i < kpts1.rows; i++) + { + kpts1.at(i, 0) = kpts1.at(i, 0) / (float)trainImgSize.width * 2.0f - 1.0f; + kpts1.at(i, 1) = kpts1.at(i, 1) / (float)trainImgSize.height * 2.0f - 1.0f; + } + } + + // Prepare blobs: [1, N, 2] and [1, N, D] + int descDim = queryDesc.cols; + int szK0[] = {1, N, 2}; + int szK1[] = {1, M, 2}; + int szD0[] = {1, N, descDim}; + int szD1[] = {1, M, descDim}; + Mat kpts0blob = kpts0.reshape(0, 3, szK0); + Mat kpts1blob = kpts1.reshape(0, 3, szK1); + Mat desc0blob = queryDesc.reshape(0, 3, szD0); + Mat desc1blob = trainDesc.reshape(0, 3, szD1); + + net.setInput(kpts0blob, "kpts0"); + net.setInput(kpts1blob, "kpts1"); + net.setInput(desc0blob, "desc0"); + net.setInput(desc1blob, "desc1"); + + std::vector outNames = {"matches0", "mscores0"}; + std::vector outs; + net.forward(outs, outNames); + + CV_Assert(outs.size() == 2); + + // matches0: [M, 2] int64 - pair indices (kpt0_idx, kpt1_idx) + // mscores0: [M] float32 - confidence per pair + Mat matchesMat = outs[0]; + Mat scoresMat = outs[1]; + + matches.clear(); + int nMatches = matchesMat.rows; + matches.reserve(nMatches); + + for (int i = 0; i < nMatches; i++) + { + int qIdx = (int)matchesMat.at(i, 0); + int tIdx = (int)matchesMat.at(i, 1); + if (qIdx >= 0 && tIdx >= 0 && qIdx < N && tIdx < M) + { + float score = scoresMat.at(i); + if (score >= scoreThreshold) + { + matches.push_back(DMatch(qIdx, tIdx, 1.0f - score)); + } + } + } +} + +void LightGlueMatcherImpl::knnMatchImpl(InputArray _queryDescriptors, + std::vector>& matches, + int k, InputArrayOfArrays, bool) +{ + CV_INSTRUMENT_REGION(); + + if (k != 1) + CV_Error(cv::Error::StsBadArg, "LightGlueMatcher only supports k=1"); + + Mat queryKpts, trainKpts; + Size queryImgSize, trainImgSize; + if (!resolveContext(queryKpts, trainKpts, queryImgSize, trainImgSize)) + { + CV_Error(cv::Error::StsBadArg, + "LightGlueMatcher: no valid context. Call setPairInfo() before matching."); + } + + CV_Assert(!trainDescCollection.empty()); + const Mat& trainDesc = trainDescCollection[0]; + Mat queryDesc = _queryDescriptors.getMat(); + + std::vector flatMatches; + lightglueMatch(queryDesc, trainDesc, queryKpts, trainKpts, + queryImgSize, trainImgSize, flatMatches); + + matches.clear(); + matches.resize(queryDesc.rows); + for (const auto& m : flatMatches) + { + matches[m.queryIdx].push_back(m); + } + + clearPairInfo(); +} + +void LightGlueMatcherImpl::radiusMatchImpl(InputArray, std::vector>&, + float, InputArrayOfArrays, bool) +{ + CV_Error(cv::Error::StsNotImplemented, + "radiusMatch is not supported by LightGlueMatcher. Use match() or knnMatch()."); +} + +Ptr LightGlueMatcher::create(const String& modelPath, float scoreThreshold, int backend, int target) +{ + return makePtr(modelPath, scoreThreshold, backend, target); +} + +Ptr LightGlueMatcher::create(const std::vector& modelData, + float scoreThreshold, int backend, int target) +{ + return makePtr(modelData, scoreThreshold, backend, target); +} + +#else // !HAVE_OPENCV_DNN + +Ptr LightGlueMatcher::create(const String& modelPath, + float scoreThreshold, int backend, int target) +{ + CV_UNUSED(modelPath); + CV_UNUSED(scoreThreshold); + CV_UNUSED(backend); + CV_UNUSED(target); + CV_Error(cv::Error::StsNotImplemented, + "LightGlueMatcher requires OpenCV built with opencv_dnn module!"); +} + +#endif // HAVE_OPENCV_DNN + +} // namespace cv diff --git a/modules/features/src/precomp.hpp b/modules/features/src/precomp.hpp index 007ce3a2b8..6c0b6ec087 100644 --- a/modules/features/src/precomp.hpp +++ b/modules/features/src/precomp.hpp @@ -47,6 +47,10 @@ #include "opencv2/imgproc.hpp" #include "opencv2/geometry.hpp" +#ifdef HAVE_OPENCV_DNN +#include "opencv2/dnn.hpp" +#endif + #include "opencv2/core/utility.hpp" #include "opencv2/core/private.hpp" #include "opencv2/core/ocl.hpp" diff --git a/modules/features/test/test_aliked_lightglue.cpp b/modules/features/test/test_aliked_lightglue.cpp new file mode 100644 index 0000000000..c1c6e43c63 --- /dev/null +++ b/modules/features/test/test_aliked_lightglue.cpp @@ -0,0 +1,171 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. + +#include "test_precomp.hpp" +#include "npy_blob.hpp" + +#ifdef HAVE_OPENCV_DNN + +#include "opencv2/dnn.hpp" +#include "opencv2/core/utils/configuration.private.hpp" + +namespace opencv_test { namespace { + +static void skipIfClassicDnnEngine() +{ + const auto engine = static_cast( + cv::utils::getConfigurationParameterSizeT("OPENCV_FORCE_DNN_ENGINE", cv::dnn::ENGINE_AUTO)); + if (engine == cv::dnn::ENGINE_CLASSIC) + throw SkipTestException("ALIKED/LightGlue reference outputs are generated with the new DNN engine"); +} + +TEST(Features2d_ALIKED, Regression) +{ + skipIfClassicDnnEngine(); + const std::string modelPath = cvtest::findDataFile("dnn/onnx/models/aliked-n16rot-top1k-640.onnx", false); + + Ptr aliked = ALIKED::create(modelPath); + ASSERT_FALSE(aliked.empty()); + ASSERT_FALSE(aliked->empty()); + EXPECT_EQ(aliked->descriptorSize(), 128); + EXPECT_EQ(aliked->descriptorType(), CV_32F); + EXPECT_EQ(aliked->defaultNorm(), NORM_L2); + + const std::string imgPath = cvtest::findDataFile("shared/box.png"); + Mat img = imread(imgPath); + ASSERT_FALSE(img.empty()) << "Could not load test image: " << imgPath; + + std::vector keypoints; + Mat descriptors; + aliked->detectAndCompute(img, noArray(), keypoints, descriptors); + + EXPECT_GT(keypoints.size(), 100u); + ASSERT_EQ(descriptors.rows, static_cast(keypoints.size())); + EXPECT_EQ(descriptors.cols, 128); + EXPECT_EQ(descriptors.type(), CV_32F); + + for (const KeyPoint& kp : keypoints) + { + EXPECT_GE(kp.pt.x, 0.f); + EXPECT_GE(kp.pt.y, 0.f); + EXPECT_LT(kp.pt.x, static_cast(img.cols)); + EXPECT_LT(kp.pt.y, static_cast(img.rows)); + EXPECT_GT(kp.response, 0.f); + } + + // Load ORT reference outputs (generated with same OpenCV preprocessing) + Mat refKpts = blobFromNPY(cvtest::findDataFile("dnn/aliked_keypoints_box.npy")); + Mat refDescs = blobFromNPY(cvtest::findDataFile("dnn/aliked_descriptors_box.npy")); + + // Keypoint count must match exactly + ASSERT_EQ(static_cast(keypoints.size()), refKpts.rows) + << "Keypoint count mismatch: got " << keypoints.size() + << ", expected " << refKpts.rows; + + // Compare each keypoint (normalized coords -> pixel coords) + const float origW = static_cast(img.cols); + const float origH = static_cast(img.rows); + for (int i = 0; i < refKpts.rows; i++) + { + float refX = (refKpts.at(i, 0) + 1.0f) * 0.5f * origW; + float refY = (refKpts.at(i, 1) + 1.0f) * 0.5f * origH; + EXPECT_NEAR(keypoints[i].pt.x, refX, 1e-4) << "Keypoint " << i << " x mismatch"; + EXPECT_NEAR(keypoints[i].pt.y, refY, 1e-4) << "Keypoint " << i << " y mismatch"; + } + + // Compare descriptors row by row + for (int i = 0; i < refDescs.rows; i++) + { + Mat diff = descriptors.row(i) - refDescs.row(i); + double maxDiff = cv::norm(diff, cv::NORM_INF); + EXPECT_LT(maxDiff, 1e-5) << "Descriptor " << i << " mismatch (max diff=" << maxDiff << ")"; + } +} + +TEST(Features2d_LightGlue, Regression) +{ + skipIfClassicDnnEngine(); + const std::string alikedPath = cvtest::findDataFile("dnn/onnx/models/aliked-n16rot-top1k-640.onnx", false); + const std::string lgPath = cvtest::findDataFile("dnn/onnx/models/aliked_lightglue.onnx", false); + + Ptr aliked = ALIKED::create(alikedPath); + Ptr lg = LightGlueMatcher::create(lgPath); + ASSERT_FALSE(aliked.empty()); + ASSERT_FALSE(lg.empty()); + + Mat img1 = imread(cvtest::findDataFile("shared/box.png")); + Mat img2 = imread(cvtest::findDataFile("shared/box_in_scene.png")); + ASSERT_FALSE(img1.empty()); + ASSERT_FALSE(img2.empty()); + + // Detect features on both images + std::vector kpts1, kpts2; + Mat descs1, descs2; + aliked->detectAndCompute(img1, noArray(), kpts1, descs1); + aliked->detectAndCompute(img2, noArray(), kpts2, descs2); + + ASSERT_GT(static_cast(kpts1.size()), 0); + ASSERT_GT(static_cast(kpts2.size()), 0); + + // Build keypoint matrices (pixel coordinates) + Mat kpts1Mat(static_cast(kpts1.size()), 2, CV_32F); + Mat kpts2Mat(static_cast(kpts2.size()), 2, CV_32F); + for (size_t i = 0; i < kpts1.size(); i++) + { + kpts1Mat.at(static_cast(i), 0) = kpts1[i].pt.x; + kpts1Mat.at(static_cast(i), 1) = kpts1[i].pt.y; + } + for (size_t i = 0; i < kpts2.size(); i++) + { + kpts2Mat.at(static_cast(i), 0) = kpts2[i].pt.x; + kpts2Mat.at(static_cast(i), 1) = kpts2[i].pt.y; + } + + lg->setPairInfo(kpts1Mat, kpts2Mat, img1.size(), img2.size()); + + std::vector matches; + lg->match(descs1, descs2, matches); + + ASSERT_GT(static_cast(matches.size()), 0); + for (const auto& m : matches) + { + EXPECT_GE(m.queryIdx, 0); + EXPECT_LT(m.queryIdx, static_cast(kpts1.size())); + EXPECT_GE(m.trainIdx, 0); + EXPECT_LT(m.trainIdx, static_cast(kpts2.size())); + } + + // Load ORT reference outputs + Mat refMatches = blobFromNPY(cvtest::findDataFile("dnn/lightglue_matches.npy")); + + // Match count must match exactly (same OpenCV preprocessing in both) + ASSERT_EQ(static_cast(matches.size()), refMatches.rows) + << "Match count mismatch: got " << matches.size() + << ", expected " << refMatches.rows; + + // Compare each match (index pairs should be identical) + for (int i = 0; i < refMatches.rows; i++) + { + int refQIdx = static_cast(refMatches.at(i, 0)); + int refTIdx = static_cast(refMatches.at(i, 1)); + EXPECT_EQ(matches[i].queryIdx, refQIdx) << "Match " << i << " queryIdx mismatch"; + EXPECT_EQ(matches[i].trainIdx, refTIdx) << "Match " << i << " trainIdx mismatch"; + } +} + +#else // !HAVE_OPENCV_DNN + +TEST(Features2d_ALIKED, not_available) +{ + EXPECT_THROW(ALIKED::create("dummy.onnx"), cv::Exception); +} + +TEST(Features2d_LightGlueMatcher, not_available) +{ + EXPECT_THROW(LightGlueMatcher::create("dummy.onnx"), cv::Exception); +} + +#endif // HAVE_OPENCV_DNN + +}} // namespace opencv_test diff --git a/modules/features/test/test_main.cpp b/modules/features/test/test_main.cpp index 68c90f775e..083c7149d4 100644 --- a/modules/features/test/test_main.cpp +++ b/modules/features/test/test_main.cpp @@ -12,6 +12,7 @@ void initTests() { cvtest::addDataSearchEnv("OPENCV_DNN_TEST_DATA_PATH"); cvtest::addDataSearchSubDirectory(""); // override "cv" prefix below to access without "../dnn" hacks + cvtest::addDataSearchEnv("OPENCV_DNN_TEST_DATA_PATH"); } CV_TEST_MAIN("cv", initTests()) diff --git a/modules/stitching/include/opencv2/stitching/detail/matchers.hpp b/modules/stitching/include/opencv2/stitching/detail/matchers.hpp index c8b2ba993f..661090b743 100644 --- a/modules/stitching/include/opencv2/stitching/detail/matchers.hpp +++ b/modules/stitching/include/opencv2/stitching/detail/matchers.hpp @@ -259,6 +259,46 @@ protected: bool full_affine_; }; +/** @brief Features matcher that adapts LightGlueMatcher (DescriptorMatcher) to the +stitching pipeline's FeaturesMatcher interface. + +This matcher uses DNN-based LightGlue for feature matching, requiring ALIKED-style +keypoints with spatial context for positional encoding. + +@sa cv::detail::FeaturesMatcher cv::LightGlueMatcher + */ +class CV_EXPORTS_W LightGlueFeaturesMatcher : public FeaturesMatcher +{ +public: + /** @brief Constructs a LightGlue features matcher. + + @param lgMatcher LightGlueMatcher instance for DNN-based matching + @param num_matches_thresh1 Minimum number of matches required for the 2D projective transform + estimation used in the inliers classification step + @param num_matches_thresh2 Minimum number of matches required for the 2D projective transform + re-estimation on inliers + @param matches_confidence_thresh Matching confidence threshold to take the match into account. + */ + CV_WRAP LightGlueFeaturesMatcher(Ptr lgMatcher, + int num_matches_thresh1 = 6, + int num_matches_thresh2 = 6, + double matches_confidence_thresh = 3.0); + + /** @brief Sets the LightGlue confidence threshold for filtering matches. + */ + CV_WRAP void setScoreThreshold(float thresh); + +protected: + void match(const ImageFeatures &features1, const ImageFeatures &features2, + MatchesInfo &matches_info) CV_OVERRIDE; + + Ptr lgMatcher_; + int num_matches_thresh1_; + int num_matches_thresh2_; + double matches_confidence_thresh_; + float lg_score_thresh_; +}; + //! @} stitching_match } // namespace detail diff --git a/modules/stitching/src/matchers.cpp b/modules/stitching/src/matchers.cpp index edd69820cb..e7c7616fc6 100644 --- a/modules/stitching/src/matchers.cpp +++ b/modules/stitching/src/matchers.cpp @@ -565,5 +565,148 @@ void AffineBestOf2NearestMatcher::match(const ImageFeatures &features1, const Im } +LightGlueFeaturesMatcher::LightGlueFeaturesMatcher(Ptr lgMatcher, + int num_matches_thresh1, + int num_matches_thresh2, + double matches_confidence_thresh) + : FeaturesMatcher(false), // not thread-safe: setPairInfo() mutates state + lgMatcher_(lgMatcher), + num_matches_thresh1_(num_matches_thresh1), + num_matches_thresh2_(num_matches_thresh2), + matches_confidence_thresh_(matches_confidence_thresh), + lg_score_thresh_(0.0f) +{ +} + +void LightGlueFeaturesMatcher::setScoreThreshold(float thresh) +{ + lg_score_thresh_ = thresh; +} + +void LightGlueFeaturesMatcher::match(const ImageFeatures &features1, const ImageFeatures &features2, + MatchesInfo &matches_info) +{ + matches_info.src_img_idx = features1.img_idx; + matches_info.dst_img_idx = features2.img_idx; + matches_info.confidence = 0; + + // Guard empty keypoints + if (features1.keypoints.empty() || features2.keypoints.empty()) + return; + + const int N = static_cast(features1.keypoints.size()); + const int M = static_cast(features2.keypoints.size()); + + // Build Nx2 keypoint coordinate matrices + Mat kpts1Mat(N, 2, CV_32F); + Mat kpts2Mat(M, 2, CV_32F); + for (int i = 0; i < N; i++) + { + kpts1Mat.at(i, 0) = features1.keypoints[i].pt.x; + kpts1Mat.at(i, 1) = features1.keypoints[i].pt.y; + } + for (int i = 0; i < M; i++) + { + kpts2Mat.at(i, 0) = features2.keypoints[i].pt.x; + kpts2Mat.at(i, 1) = features2.keypoints[i].pt.y; + } + + // Set pair context for LightGlue spatial reasoning + lgMatcher_->setPairInfo(kpts1Mat, kpts2Mat, features1.img_size, features2.img_size); + + // Run LightGlue matching + Mat desc1 = features1.descriptors.getMat(ACCESS_READ); + Mat desc2 = features2.descriptors.getMat(ACCESS_READ); + std::vector matches; + lgMatcher_->match(desc1, desc2, matches); + + // Filter by score threshold + if (lg_score_thresh_ > 0.0f) + { + float maxDist = 1.0f - lg_score_thresh_; + std::vector filtered; + filtered.reserve(matches.size()); + for (const auto& m : matches) + { + if (m.distance <= maxDist) + filtered.push_back(m); + } + matches.swap(filtered); + } + + // Store matches before homography estimation + matches_info.matches = matches; + + // Guard: need at least 4 matches for findHomography + if (matches.size() < 4) + return; + + // Estimate homography with centered coordinates + Mat src_points(1, static_cast(matches.size()), CV_32FC2); + Mat dst_points(1, static_cast(matches.size()), CV_32FC2); + for (size_t i = 0; i < matches.size(); ++i) + { + const DMatch& m = matches[i]; + + Point2f p = features1.keypoints[m.queryIdx].pt; + p.x -= features1.img_size.width * 0.5f; + p.y -= features1.img_size.height * 0.5f; + src_points.at(0, static_cast(i)) = p; + + p = features2.keypoints[m.trainIdx].pt; + p.x -= features2.img_size.width * 0.5f; + p.y -= features2.img_size.height * 0.5f; + dst_points.at(0, static_cast(i)) = p; + } + + matches_info.H = findHomography(src_points, dst_points, matches_info.inliers_mask, RANSAC); + if (matches_info.H.empty() || + std::abs(determinant(matches_info.H)) < std::numeric_limits::epsilon()) + return; + + // Compute inliers and confidence (Brown & Lowe formula) + matches_info.num_inliers = 0; + for (size_t i = 0; i < matches_info.inliers_mask.size(); ++i) + if (matches_info.inliers_mask[i]) + matches_info.num_inliers++; + + matches_info.confidence = matches_info.num_inliers / + (8 + 0.3 * matches_info.matches.size()); + + // Zero confidence for too-close image pairs + if (matches_info.confidence > matches_confidence_thresh_) + matches_info.confidence = 0.; + + // Refine homography on inliers if enough + if (matches_info.num_inliers >= num_matches_thresh2_) + { + src_points.create(1, matches_info.num_inliers, CV_32FC2); + dst_points.create(1, matches_info.num_inliers, CV_32FC2); + int inlier_idx = 0; + for (size_t i = 0; i < matches_info.matches.size(); ++i) + { + if (!matches_info.inliers_mask[i]) + continue; + + const DMatch& m = matches_info.matches[i]; + + Point2f p = features1.keypoints[m.queryIdx].pt; + p.x -= features1.img_size.width * 0.5f; + p.y -= features1.img_size.height * 0.5f; + src_points.at(0, inlier_idx) = p; + + p = features2.keypoints[m.trainIdx].pt; + p.x -= features2.img_size.width * 0.5f; + p.y -= features2.img_size.height * 0.5f; + dst_points.at(0, inlier_idx) = p; + + inlier_idx++; + } + + matches_info.H = findHomography(src_points, dst_points, RANSAC); + } +} + + } // namespace detail } // namespace cv diff --git a/samples/cpp/example_features_aliked_lightglue.cpp b/samples/cpp/example_features_aliked_lightglue.cpp new file mode 100644 index 0000000000..d562cbc110 --- /dev/null +++ b/samples/cpp/example_features_aliked_lightglue.cpp @@ -0,0 +1,169 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. + +// ALIKED + LightGlueMatcher usage example +// Demonstrates feature detection, extraction, and matching using ALIKED and LightGlue. + +#include +#include +#include +#include +#include + +using namespace cv; +using namespace std; + +int main(int argc, char** argv) +{ + // ---- Parse arguments ---- + String alikedModel, lightglueModel, imgPath1, imgPath2; + + if (argc >= 5) + { + imgPath1 = argv[1]; + imgPath2 = argv[2]; + alikedModel = argv[3]; + lightglueModel = argv[4]; + } +else + { + cout << "Usage: " << argv[0] << " " << endl; + cout << endl; + cout << "Example:" << endl; + cout << " " << argv[0] << " img1.jpg img2.jpg aliked-n16rot-top1k-640.onnx aliked_lightglue.onnx" << endl; + return 0; + } + + // ---- Load images ---- + Mat img1 = imread(imgPath1); + Mat img2 = imread(imgPath2); + if (img1.empty() || img2.empty()) + { + cerr << "Error: cannot load images." << endl; + return -1; + } + + // ================================================================ + // 1. Create ALIKED feature extractor + // ================================================================ + // Method A: From ONNX model file + Ptr aliked = ALIKED::create(alikedModel); + + // Method B: Customize parameters + // ALIKED::Params params; + // params.inputSize = Size(640, 640); // Network input resolution + // params.normalizeDescriptors = true; // L2-normalize descriptors + // Ptr aliked = ALIKED::create(alikedModel, params); + + // Method C: From in-memory model data + // vector modelData = readFile(alikedModel); + // Ptr aliked = ALIKED::create(modelData); + + cout << "Descriptor size: " << aliked->descriptorSize() << endl; // 128 + cout << "Descriptor type: " << aliked->descriptorType() << endl; // CV_32F + cout << "Default norm: " << aliked->defaultNorm() << endl; // NORM_L2 + + // ================================================================ + // 2. Detect keypoints and compute descriptors + // ================================================================ + vector kpts1, kpts2; + Mat descs1, descs2; + + // Method A: detect + compute in one call (recommended) + aliked->detectAndCompute(img1, Mat(), kpts1, descs1); + aliked->detectAndCompute(img2, Mat(), kpts2, descs2); + + // Method B: detect only (no descriptors) + // vector kpts; + // aliked->detect(img, kpts); + + // Method C: compute only (from existing keypoints) + // Mat descs; + // aliked->compute(img, kpts, descs); + + cout << "Image 1: " << kpts1.size() << " keypoints, descriptors " << descs1.rows << "x" << descs1.cols << endl; + cout << "Image 2: " << kpts2.size() << " keypoints, descriptors " << descs2.rows << "x" << descs2.cols << endl; + + // ================================================================ + // 3. Create LightGlueMatcher + // ================================================================ + // Method A: From ONNX model file + Ptr lg = LightGlueMatcher::create(lightglueModel); + + // Method B: Customize parameters + // LightGlueMatcher::Params lgParams; + // lgParams.scoreThreshold = 0.1f; // Filter low-confidence matches + // lgParams.disableWinograd = false; // Keep Winograd convolution + // Ptr lg = LightGlueMatcher::create(lightglueModel, lgParams); + + // Method C: From in-memory model data + // vector lgData = readFile(lightglueModel); + // Ptr lg = LightGlueMatcher::create(lgData); + + // ================================================================ + // 4. Set keypoint context for LightGlue + // ================================================================ + // LightGlue needs keypoint coordinates + image sizes for spatial reasoning. + // Build Nx2 float matrices with pixel coordinates. + + Mat kpts1Mat((int)kpts1.size(), 2, CV_32F); + Mat kpts2Mat((int)kpts2.size(), 2, CV_32F); + for (size_t i = 0; i < kpts1.size(); i++) + { + kpts1Mat.at((int)i, 0) = kpts1[i].pt.x; + kpts1Mat.at((int)i, 1) = kpts1[i].pt.y; + } + for (size_t i = 0; i < kpts2.size(); i++) + { + kpts2Mat.at((int)i, 0) = kpts2[i].pt.x; + kpts2Mat.at((int)i, 1) = kpts2[i].pt.y; + } + + // setPairInfo must be called before match()/knnMatch() + lg->setPairInfo(kpts1Mat, kpts2Mat, img1.size(), img2.size()); + + // ================================================================ + // 5. Match descriptors + // ================================================================ + + // Method A: 1-to-1 matching (returns best match per query keypoint) + vector matches; + lg->match(descs1, descs2, matches); + + cout << "1-to-1 matches: " << matches.size() << endl; + + // Method B: kNN matching (k=1 only for LightGlue) + // vector> knnMatches; + // lg->knnMatch(descs1, descs2, knnMatches, 1); + // // knnMatches[i] contains matches for query keypoint i + + // ================================================================ + // 6. Filter matches by confidence (optional) + // ================================================================ + // DMatch distance = 1.0 - confidence_score + // Lower distance = better match + + vector goodMatches; + float distanceThreshold = 0.9f; // confidence > 0.1 + for (const auto& m : matches) + { + if (m.distance < distanceThreshold) + goodMatches.push_back(m); + } + cout << "Good matches (distance < " << distanceThreshold << "): " << goodMatches.size() << endl; + + // ================================================================ + // 7. Visualize results + // ================================================================ + Mat canvas; + cv::drawMatches(img1, kpts1, img2, kpts2, goodMatches, canvas, + Scalar::all(-1), Scalar::all(-1), vector(), + DrawMatchesFlags::NOT_DRAW_SINGLE_POINTS); + + imshow("ALIKED + LightGlue Matches", canvas); + cout << "Press any key to exit..." << endl; + waitKey(0); + + return 0; +} diff --git a/samples/cpp/stitching_detailed.cpp b/samples/cpp/stitching_detailed.cpp index 784b10c647..b1779d2f6f 100644 --- a/samples/cpp/stitching_detailed.cpp +++ b/samples/cpp/stitching_detailed.cpp @@ -16,6 +16,8 @@ #include "opencv2/stitching/detail/seam_finders.hpp" #include "opencv2/stitching/detail/warpers.hpp" #include "opencv2/stitching/warpers.hpp" +#include "opencv2/features.hpp" +#include "opencv2/core/ocl.hpp" #ifdef HAVE_OPENCV_XFEATURES2D #include "opencv2/xfeatures2d.hpp" @@ -45,9 +47,10 @@ static void printUsage(char** argv) "\nMotion Estimation Flags:\n" " --work_megapix \n" " Resolution for image registration step. The default is 0.6 Mpx.\n" - " --features (surf|orb|sift|akaze)\n" + " --features (surf|orb|sift|akaze|aliked)\n" " Type of features used for images matching.\n" " The default is surf if available, orb otherwise.\n" + " When using 'aliked', requires --matcher lightglue and DNN model paths.\n" " --matcher (homography|affine)\n" " Matcher used for pairwise image matching.\n" " --estimator (homography|affine)\n" @@ -103,7 +106,14 @@ static void printUsage(char** argv) " --timelapse (as_is|crop) \n" " Output warped images separately as frames of a time lapse movie, with 'fixed_' prepended to input file names.\n" " --rangewidth \n" - " uses range_width to limit number of images to match with.\n"; + " uses range_width to limit number of images to match with.\n" + "\nDNN Feature Options (when --features aliked --matcher lightglue):\n" + " --aliked_model \n" + " Path to ALIKED ONNX model file.\n" + " --lightglue_model \n" + " Path to LightGlue ONNX model file (for ALIKED descriptors).\n" + " --lg_score_thresh \n" + " LightGlue confidence threshold. The default is 0.0 (accept all).\n"; } @@ -142,6 +152,9 @@ float blend_strength = 5; string result_name = "result.jpg"; bool timelapse = false; int range_width = -1; +String aliked_model_path; +String lightglue_model_path; +float lg_score_thresh = 0.0f; static int parseCmdArgs(int argc, char** argv) @@ -204,7 +217,7 @@ static int parseCmdArgs(int argc, char** argv) } else if (string(argv[i]) == "--matcher") { - if (string(argv[i + 1]) == "homography" || string(argv[i + 1]) == "affine") + if (string(argv[i + 1]) == "homography" || string(argv[i + 1]) == "affine" || string(argv[i + 1]) == "lightglue") matcher_type = argv[i + 1]; else { @@ -376,6 +389,21 @@ static int parseCmdArgs(int argc, char** argv) result_name = argv[i + 1]; i++; } + else if (string(argv[i]) == "--aliked_model") + { + aliked_model_path = argv[i + 1]; + i++; + } + else if (string(argv[i]) == "--lightglue_model") + { + lightglue_model_path = argv[i + 1]; + i++; + } + else if (string(argv[i]) == "--lg_score_thresh") + { + lg_score_thresh = static_cast(atof(argv[i + 1])); + i++; + } else img_names.push_back(argv[i]); } @@ -383,6 +411,19 @@ static int parseCmdArgs(int argc, char** argv) { compose_megapix = 0.6; } + + // Validate DNN options + if (features_type == "aliked" && matcher_type != "lightglue") + { + cout << "Error: --features aliked requires --matcher lightglue\n"; + return -1; + } + if (features_type == "aliked" && (aliked_model_path.empty() || lightglue_model_path.empty())) + { + cout << "Error: --features aliked requires --aliked_model and --lightglue_model\n"; + return -1; + } + return 0; } @@ -401,6 +442,11 @@ int main(int argc, char* argv[]) if (retval) return retval; + // Disable OpenCL for DNN-based features to avoid backend sync issues + bool use_aliked = (features_type == "aliked"); + if (use_aliked) + cv::ocl::setUseOpenCL(false); + // Check if have enough images int num_images = static_cast(img_names.size()); if (num_images < 2) @@ -418,7 +464,11 @@ int main(int argc, char* argv[]) #endif Ptr finder; - if (features_type == "orb") + if (use_aliked) + { + // ALIKED will be created per-image in the loop below + } + else if (features_type == "orb") { finder = ORB::create(); } @@ -488,7 +538,15 @@ int main(int argc, char* argv[]) is_seam_scale_set = true; } - computeImageFeatures(finder, img, features[i]); + if (use_aliked) + { + Ptr aliked = ALIKED::create(aliked_model_path); + computeImageFeatures(aliked, img, features[i]); + } + else + { + computeImageFeatures(finder, img, features[i]); + } features[i].img_idx = i; LOGLN("Features in image #" << i+1 << ": " << features[i].keypoints.size()); @@ -507,7 +565,14 @@ int main(int argc, char* argv[]) #endif vector pairwise_matches; Ptr matcher; - if (matcher_type == "affine") + if (use_aliked && matcher_type == "lightglue") + { + Ptr lg = LightGlueMatcher::create(lightglue_model_path); + Ptr lgMatcher = makePtr(lg); + lgMatcher->setScoreThreshold(lg_score_thresh); + matcher = lgMatcher; + } + else if (matcher_type == "affine") matcher = makePtr(false, try_cuda, match_conf); else if (range_width==-1) matcher = makePtr(try_cuda, match_conf);