mirror of
https://github.com/opencv/opencv.git
synced 2026-07-31 00:03:03 +04:00
Merge pull request #28986 from YangGuanyuhan:ai-aliked-lightglue-pipeline
[GSOC] feat: Add ALIKED feature extractor and LightGlue matcher with DNN integration #28986 ## PR Description ### Summary Integrate ALIKED and LightGlue into OpenCV's `features` module as native`Feature2D` and `DescriptorMatcher` implementations, enabling end-to-end neural feature matching within OpenCV's ecosystem. --- ### What's included #### New classes - **`cv::ALIKED`** extends `Feature2D` - CNN-based keypoint detection - 128-D descriptor extraction via ONNX Runtime - **`cv::LightGlueMatcher`** extends `DescriptorMatcher` - Deep feature matching with spatial context - Uses keypoints and image sizes during matching --- #### API design - Standard OpenCV patterns: - `detectAndCompute()` - `match()` - `knnMatch()` - Multiple factory methods: - ONNX model path - In-memory model buffer - Pre-loaded `dnn::Net` - `Params` structs use `CV_EXPORTS_W_SIMPLE` for Python/Java bindings support - Optional DNN dependency: - `HAVE_OPENCV_DNN` guards - Stub implementations throw `StsNotImplemented` --- ### Files added | File | Description | |------|-------------| | `src/feature2d_aliked.cpp` | ALIKED implementation | | `src/matchers_lightglue.cpp` | LightGlueMatcher implementation | | `src/aliked_context.hpp` | Shared internal context struct | | `test/test_aliked_lightglue.cpp` | Unit tests (9 test cases) | | `samples/cpp/example_features_aliked_lightglue.cpp` | Demo application | --- ### Files modified - `CMakeLists.txt` - Add `opencv_dnn` as optional dependency - `features.hpp` - Add ALIKED and LightGlueMatcher declarations - `precomp.hpp` - Add DNN include guard --- ### Usage ```cpp // Feature extraction Ptr<ALIKED> aliked = ALIKED::create("aliked-n16rot-top1k-640.onnx"); vector<KeyPoint> kpts; Mat descs; aliked->detectAndCompute(image, Mat(), kpts, descs); // Feature matching Ptr<LightGlueMatcher> lg = LightGlueMatcher::create("aliked_lightglue.onnx"); lg->setPairInfo( kpts1Mat, kpts2Mat, img1.size(), img2.size() ); vector<DMatch> matches; lg->match(descs1, descs2, matches); ```` please refer to samples/cpp/example_features_aliked_lightglue.cpp --- ### Test plan * Build with `BUILD_LIST=features,dnn` * Build without DNN: * Verify stubs compile * Verify `StsNotImplemented` is thrown * Run: * `ctest -R Features2d_ALIKED` * `ctest -R Features2d_LightGlueMatcher` * Run sample application with: * Real images * Real ONNX models * Verify Python/Java bindings compile and work --- ### Related Phase 1 of the "End-to-End AI Feature Extraction and LightGlue Matching Pipeline" GSoC project. Designed to be extensible to: * XFeat * SuperPoint * Other neural feature extractors ### test dependency Depends on the opencv_extra PR adding ALIKED and LightGlue test models: - [opencv_extra PR](https://github.com/opencv/opencv_extra/pull/1366) This PR adds the following ONNX models to `download_models.py`: - `aliked-n16rot-top1k-640.onnx` - `aliked_lightglue.onnx` These models are required for the `features2d` tests in the main OpenCV repository to validate the ALIKED and LightGlue feature extraction and matching pipeline. ### Pull Request Readiness Checklist See details at https://github.com/opencv/opencv/wiki/How_to_contribute#making-a-good-pull-request - [x] I agree to contribute to the project under Apache 2 License. - [x] To the best of my knowledge, the proposed patch is not based on a code under GPL or another license that is incompatible with OpenCV - [x] The PR is proposed to the proper branch - [ ] There is a reference to the original bug report and related work - [x] There is accuracy test, performance test and test data in opencv_extra repository, if applicable Patch to opencv_extra has the same branch name. - [x] The feature is well documented and sample code can be built with the project CMake
This commit is contained in:
Yang Guanyuhan
GitHub
@@ -50,6 +50,10 @@
|
||||
#include "opencv2/flann/miniflann.hpp"
|
||||
#endif
|
||||
|
||||
#ifdef HAVE_OPENCV_DNN
|
||||
#include "opencv2/dnn.hpp"
|
||||
#endif
|
||||
|
||||
/**
|
||||
@defgroup features Features Framework
|
||||
@{
|
||||
@@ -835,6 +839,46 @@ public:
|
||||
CV_WRAP virtual const std::vector<std::vector<cv::Point> >& getBlobContours() const = 0;
|
||||
};
|
||||
|
||||
/** @brief ALIKED feature detector and descriptor extractor.
|
||||
|
||||
ALIKED (A Lightweight Image KEYpoint Detector) is a CNN-based feature detector and descriptor
|
||||
extractor, as described in @cite Zhao23 . It produces 128-dimensional float descriptors and
|
||||
keypoints with sub-pixel accuracy.
|
||||
|
||||
The model expects RGB input [1,3,H,W] and internally converts BGR images to RGB.
|
||||
*/
|
||||
class CV_EXPORTS_W ALIKED : public Feature2D
|
||||
{
|
||||
protected:
|
||||
ALIKED();
|
||||
public:
|
||||
virtual ~ALIKED();
|
||||
|
||||
struct CV_EXPORTS_W_SIMPLE Params
|
||||
{
|
||||
CV_WRAP Params();
|
||||
CV_PROP_RW Size inputSize; //!< Input image size for the network, default 640x640
|
||||
CV_PROP_RW bool normalizeDescriptors; //!< Whether to L2-normalize descriptors, default true
|
||||
CV_PROP_RW int engine; //!< DNN engine type (dnn::EngineType), default ENGINE_NEW
|
||||
CV_PROP_RW int backend; //!< DNN backend, default DNN_BACKEND_DEFAULT
|
||||
CV_PROP_RW int target; //!< DNN target, default DNN_TARGET_CPU
|
||||
};
|
||||
|
||||
/** @brief Creates ALIKED from a model file path.
|
||||
@param modelPath Path to the ONNX model file.
|
||||
@param params ALIKED parameters.
|
||||
*/
|
||||
CV_WRAP static Ptr<ALIKED> create(const String& modelPath, const ALIKED::Params& params = ALIKED::Params());
|
||||
|
||||
#ifdef HAVE_OPENCV_DNN
|
||||
/** @brief Creates ALIKED from in-memory model data.
|
||||
@param modelData Buffer containing the model data.
|
||||
@param params ALIKED parameters.
|
||||
*/
|
||||
static Ptr<ALIKED> create(const std::vector<uchar>& modelData, const ALIKED::Params& params = ALIKED::Params());
|
||||
#endif
|
||||
};
|
||||
|
||||
|
||||
/****************************************************************************************\
|
||||
* Distance *
|
||||
@@ -1252,6 +1296,61 @@ protected:
|
||||
|
||||
#endif
|
||||
|
||||
/** @brief LightGlue feature matcher.
|
||||
|
||||
LightGlue is a CNN-based feature matcher, as described in @cite Lindenberger23 . It takes
|
||||
keypoint locations and descriptors from two images and directly predicts match pairs. Unlike
|
||||
traditional matchers that compute descriptor distances, LightGlue uses attention mechanisms
|
||||
to produce confidence scores for each potential match pair.
|
||||
|
||||
The matcher extends DescriptorMatcher and supports the standard match(), knnMatch(), and
|
||||
radiusMatch() interfaces. Context (keypoints and image sizes) must be provided via
|
||||
setPairInfo() before matching.
|
||||
*/
|
||||
class CV_EXPORTS_W LightGlueMatcher : public DescriptorMatcher
|
||||
{
|
||||
protected:
|
||||
LightGlueMatcher();
|
||||
public:
|
||||
virtual ~LightGlueMatcher();
|
||||
|
||||
/** @brief Creates LightGlueMatcher from a model file path.
|
||||
@param modelPath Path to the ONNX model file.
|
||||
@param scoreThreshold Match confidence threshold.
|
||||
@param backend DNN backend
|
||||
@param target DNN target
|
||||
*/
|
||||
CV_WRAP static Ptr<LightGlueMatcher> create(const String& modelPath, float scoreThreshold = 0.0f, int backend = 0, int target = 0);
|
||||
|
||||
#ifdef HAVE_OPENCV_DNN
|
||||
/** @brief Creates LightGlueMatcher from in-memory model data.
|
||||
@param modelData Buffer containing the model data.
|
||||
@param scoreThreshold Match confidence threshold.
|
||||
@param backend DNN backend
|
||||
@param target DNN target
|
||||
*/
|
||||
CV_WRAP_AS(createFromMemory) static Ptr<LightGlueMatcher> create(const std::vector<uchar>& modelData, float scoreThreshold = 0.0f, int backend = 0, int target = 0);
|
||||
#endif
|
||||
|
||||
/** @brief Sets the keypoint and image size context for the next match() call.
|
||||
|
||||
This provides the spatial context that LightGlue needs in addition to descriptors.
|
||||
Must be called before match()/knnMatch()/radiusMatch() unless using automatic context
|
||||
from in-process ALIKED instances.
|
||||
|
||||
@param queryKpts Query image keypoints (Nx2 float matrix with x,y coordinates).
|
||||
@param trainKpts Train image keypoints (Nx2 float matrix with x,y coordinates).
|
||||
@param queryImageSize Size of the query image (width, height).
|
||||
@param trainImageSize Size of the train image (width, height).
|
||||
*/
|
||||
CV_WRAP virtual void setPairInfo(InputArray queryKpts, InputArray trainKpts,
|
||||
Size queryImageSize = Size(), Size trainImageSize = Size()) = 0;
|
||||
|
||||
/** @brief Clears stored pair context information.
|
||||
*/
|
||||
CV_WRAP virtual void clearPairInfo() = 0;
|
||||
};
|
||||
|
||||
//! @} features_match
|
||||
|
||||
/****************************************************************************************\
|
||||
|
||||
Reference in New Issue
Block a user