mirror of
https://github.com/opencv/opencv.git
synced 2026-07-31 08:13:04 +04:00
Merge pull request #28986 from YangGuanyuhan:ai-aliked-lightglue-pipeline
[GSOC] feat: Add ALIKED feature extractor and LightGlue matcher with DNN integration #28986 ## PR Description ### Summary Integrate ALIKED and LightGlue into OpenCV's `features` module as native`Feature2D` and `DescriptorMatcher` implementations, enabling end-to-end neural feature matching within OpenCV's ecosystem. --- ### What's included #### New classes - **`cv::ALIKED`** extends `Feature2D` - CNN-based keypoint detection - 128-D descriptor extraction via ONNX Runtime - **`cv::LightGlueMatcher`** extends `DescriptorMatcher` - Deep feature matching with spatial context - Uses keypoints and image sizes during matching --- #### API design - Standard OpenCV patterns: - `detectAndCompute()` - `match()` - `knnMatch()` - Multiple factory methods: - ONNX model path - In-memory model buffer - Pre-loaded `dnn::Net` - `Params` structs use `CV_EXPORTS_W_SIMPLE` for Python/Java bindings support - Optional DNN dependency: - `HAVE_OPENCV_DNN` guards - Stub implementations throw `StsNotImplemented` --- ### Files added | File | Description | |------|-------------| | `src/feature2d_aliked.cpp` | ALIKED implementation | | `src/matchers_lightglue.cpp` | LightGlueMatcher implementation | | `src/aliked_context.hpp` | Shared internal context struct | | `test/test_aliked_lightglue.cpp` | Unit tests (9 test cases) | | `samples/cpp/example_features_aliked_lightglue.cpp` | Demo application | --- ### Files modified - `CMakeLists.txt` - Add `opencv_dnn` as optional dependency - `features.hpp` - Add ALIKED and LightGlueMatcher declarations - `precomp.hpp` - Add DNN include guard --- ### Usage ```cpp // Feature extraction Ptr<ALIKED> aliked = ALIKED::create("aliked-n16rot-top1k-640.onnx"); vector<KeyPoint> kpts; Mat descs; aliked->detectAndCompute(image, Mat(), kpts, descs); // Feature matching Ptr<LightGlueMatcher> lg = LightGlueMatcher::create("aliked_lightglue.onnx"); lg->setPairInfo( kpts1Mat, kpts2Mat, img1.size(), img2.size() ); vector<DMatch> matches; lg->match(descs1, descs2, matches); ```` please refer to samples/cpp/example_features_aliked_lightglue.cpp --- ### Test plan * Build with `BUILD_LIST=features,dnn` * Build without DNN: * Verify stubs compile * Verify `StsNotImplemented` is thrown * Run: * `ctest -R Features2d_ALIKED` * `ctest -R Features2d_LightGlueMatcher` * Run sample application with: * Real images * Real ONNX models * Verify Python/Java bindings compile and work --- ### Related Phase 1 of the "End-to-End AI Feature Extraction and LightGlue Matching Pipeline" GSoC project. Designed to be extensible to: * XFeat * SuperPoint * Other neural feature extractors ### test dependency Depends on the opencv_extra PR adding ALIKED and LightGlue test models: - [opencv_extra PR](https://github.com/opencv/opencv_extra/pull/1366) This PR adds the following ONNX models to `download_models.py`: - `aliked-n16rot-top1k-640.onnx` - `aliked_lightglue.onnx` These models are required for the `features2d` tests in the main OpenCV repository to validate the ALIKED and LightGlue feature extraction and matching pipeline. ### Pull Request Readiness Checklist See details at https://github.com/opencv/opencv/wiki/How_to_contribute#making-a-good-pull-request - [x] I agree to contribute to the project under Apache 2 License. - [x] To the best of my knowledge, the proposed patch is not based on a code under GPL or another license that is incompatible with OpenCV - [x] The PR is proposed to the proper branch - [ ] There is a reference to the original bug report and related work - [x] There is accuracy test, performance test and test data in opencv_extra repository, if applicable Patch to opencv_extra has the same branch name. - [x] The feature is well documented and sample code can be built with the project CMake
This commit is contained in:
Yang Guanyuhan
GitHub
@@ -259,6 +259,46 @@ protected:
|
||||
bool full_affine_;
|
||||
};
|
||||
|
||||
/** @brief Features matcher that adapts LightGlueMatcher (DescriptorMatcher) to the
|
||||
stitching pipeline's FeaturesMatcher interface.
|
||||
|
||||
This matcher uses DNN-based LightGlue for feature matching, requiring ALIKED-style
|
||||
keypoints with spatial context for positional encoding.
|
||||
|
||||
@sa cv::detail::FeaturesMatcher cv::LightGlueMatcher
|
||||
*/
|
||||
class CV_EXPORTS_W LightGlueFeaturesMatcher : public FeaturesMatcher
|
||||
{
|
||||
public:
|
||||
/** @brief Constructs a LightGlue features matcher.
|
||||
|
||||
@param lgMatcher LightGlueMatcher instance for DNN-based matching
|
||||
@param num_matches_thresh1 Minimum number of matches required for the 2D projective transform
|
||||
estimation used in the inliers classification step
|
||||
@param num_matches_thresh2 Minimum number of matches required for the 2D projective transform
|
||||
re-estimation on inliers
|
||||
@param matches_confidence_thresh Matching confidence threshold to take the match into account.
|
||||
*/
|
||||
CV_WRAP LightGlueFeaturesMatcher(Ptr<LightGlueMatcher> lgMatcher,
|
||||
int num_matches_thresh1 = 6,
|
||||
int num_matches_thresh2 = 6,
|
||||
double matches_confidence_thresh = 3.0);
|
||||
|
||||
/** @brief Sets the LightGlue confidence threshold for filtering matches.
|
||||
*/
|
||||
CV_WRAP void setScoreThreshold(float thresh);
|
||||
|
||||
protected:
|
||||
void match(const ImageFeatures &features1, const ImageFeatures &features2,
|
||||
MatchesInfo &matches_info) CV_OVERRIDE;
|
||||
|
||||
Ptr<LightGlueMatcher> lgMatcher_;
|
||||
int num_matches_thresh1_;
|
||||
int num_matches_thresh2_;
|
||||
double matches_confidence_thresh_;
|
||||
float lg_score_thresh_;
|
||||
};
|
||||
|
||||
//! @} stitching_match
|
||||
|
||||
} // namespace detail
|
||||
|
||||
@@ -565,5 +565,148 @@ void AffineBestOf2NearestMatcher::match(const ImageFeatures &features1, const Im
|
||||
}
|
||||
|
||||
|
||||
LightGlueFeaturesMatcher::LightGlueFeaturesMatcher(Ptr<LightGlueMatcher> lgMatcher,
|
||||
int num_matches_thresh1,
|
||||
int num_matches_thresh2,
|
||||
double matches_confidence_thresh)
|
||||
: FeaturesMatcher(false), // not thread-safe: setPairInfo() mutates state
|
||||
lgMatcher_(lgMatcher),
|
||||
num_matches_thresh1_(num_matches_thresh1),
|
||||
num_matches_thresh2_(num_matches_thresh2),
|
||||
matches_confidence_thresh_(matches_confidence_thresh),
|
||||
lg_score_thresh_(0.0f)
|
||||
{
|
||||
}
|
||||
|
||||
void LightGlueFeaturesMatcher::setScoreThreshold(float thresh)
|
||||
{
|
||||
lg_score_thresh_ = thresh;
|
||||
}
|
||||
|
||||
void LightGlueFeaturesMatcher::match(const ImageFeatures &features1, const ImageFeatures &features2,
|
||||
MatchesInfo &matches_info)
|
||||
{
|
||||
matches_info.src_img_idx = features1.img_idx;
|
||||
matches_info.dst_img_idx = features2.img_idx;
|
||||
matches_info.confidence = 0;
|
||||
|
||||
// Guard empty keypoints
|
||||
if (features1.keypoints.empty() || features2.keypoints.empty())
|
||||
return;
|
||||
|
||||
const int N = static_cast<int>(features1.keypoints.size());
|
||||
const int M = static_cast<int>(features2.keypoints.size());
|
||||
|
||||
// Build Nx2 keypoint coordinate matrices
|
||||
Mat kpts1Mat(N, 2, CV_32F);
|
||||
Mat kpts2Mat(M, 2, CV_32F);
|
||||
for (int i = 0; i < N; i++)
|
||||
{
|
||||
kpts1Mat.at<float>(i, 0) = features1.keypoints[i].pt.x;
|
||||
kpts1Mat.at<float>(i, 1) = features1.keypoints[i].pt.y;
|
||||
}
|
||||
for (int i = 0; i < M; i++)
|
||||
{
|
||||
kpts2Mat.at<float>(i, 0) = features2.keypoints[i].pt.x;
|
||||
kpts2Mat.at<float>(i, 1) = features2.keypoints[i].pt.y;
|
||||
}
|
||||
|
||||
// Set pair context for LightGlue spatial reasoning
|
||||
lgMatcher_->setPairInfo(kpts1Mat, kpts2Mat, features1.img_size, features2.img_size);
|
||||
|
||||
// Run LightGlue matching
|
||||
Mat desc1 = features1.descriptors.getMat(ACCESS_READ);
|
||||
Mat desc2 = features2.descriptors.getMat(ACCESS_READ);
|
||||
std::vector<DMatch> matches;
|
||||
lgMatcher_->match(desc1, desc2, matches);
|
||||
|
||||
// Filter by score threshold
|
||||
if (lg_score_thresh_ > 0.0f)
|
||||
{
|
||||
float maxDist = 1.0f - lg_score_thresh_;
|
||||
std::vector<DMatch> filtered;
|
||||
filtered.reserve(matches.size());
|
||||
for (const auto& m : matches)
|
||||
{
|
||||
if (m.distance <= maxDist)
|
||||
filtered.push_back(m);
|
||||
}
|
||||
matches.swap(filtered);
|
||||
}
|
||||
|
||||
// Store matches before homography estimation
|
||||
matches_info.matches = matches;
|
||||
|
||||
// Guard: need at least 4 matches for findHomography
|
||||
if (matches.size() < 4)
|
||||
return;
|
||||
|
||||
// Estimate homography with centered coordinates
|
||||
Mat src_points(1, static_cast<int>(matches.size()), CV_32FC2);
|
||||
Mat dst_points(1, static_cast<int>(matches.size()), CV_32FC2);
|
||||
for (size_t i = 0; i < matches.size(); ++i)
|
||||
{
|
||||
const DMatch& m = matches[i];
|
||||
|
||||
Point2f p = features1.keypoints[m.queryIdx].pt;
|
||||
p.x -= features1.img_size.width * 0.5f;
|
||||
p.y -= features1.img_size.height * 0.5f;
|
||||
src_points.at<Point2f>(0, static_cast<int>(i)) = p;
|
||||
|
||||
p = features2.keypoints[m.trainIdx].pt;
|
||||
p.x -= features2.img_size.width * 0.5f;
|
||||
p.y -= features2.img_size.height * 0.5f;
|
||||
dst_points.at<Point2f>(0, static_cast<int>(i)) = p;
|
||||
}
|
||||
|
||||
matches_info.H = findHomography(src_points, dst_points, matches_info.inliers_mask, RANSAC);
|
||||
if (matches_info.H.empty() ||
|
||||
std::abs(determinant(matches_info.H)) < std::numeric_limits<double>::epsilon())
|
||||
return;
|
||||
|
||||
// Compute inliers and confidence (Brown & Lowe formula)
|
||||
matches_info.num_inliers = 0;
|
||||
for (size_t i = 0; i < matches_info.inliers_mask.size(); ++i)
|
||||
if (matches_info.inliers_mask[i])
|
||||
matches_info.num_inliers++;
|
||||
|
||||
matches_info.confidence = matches_info.num_inliers /
|
||||
(8 + 0.3 * matches_info.matches.size());
|
||||
|
||||
// Zero confidence for too-close image pairs
|
||||
if (matches_info.confidence > matches_confidence_thresh_)
|
||||
matches_info.confidence = 0.;
|
||||
|
||||
// Refine homography on inliers if enough
|
||||
if (matches_info.num_inliers >= num_matches_thresh2_)
|
||||
{
|
||||
src_points.create(1, matches_info.num_inliers, CV_32FC2);
|
||||
dst_points.create(1, matches_info.num_inliers, CV_32FC2);
|
||||
int inlier_idx = 0;
|
||||
for (size_t i = 0; i < matches_info.matches.size(); ++i)
|
||||
{
|
||||
if (!matches_info.inliers_mask[i])
|
||||
continue;
|
||||
|
||||
const DMatch& m = matches_info.matches[i];
|
||||
|
||||
Point2f p = features1.keypoints[m.queryIdx].pt;
|
||||
p.x -= features1.img_size.width * 0.5f;
|
||||
p.y -= features1.img_size.height * 0.5f;
|
||||
src_points.at<Point2f>(0, inlier_idx) = p;
|
||||
|
||||
p = features2.keypoints[m.trainIdx].pt;
|
||||
p.x -= features2.img_size.width * 0.5f;
|
||||
p.y -= features2.img_size.height * 0.5f;
|
||||
dst_points.at<Point2f>(0, inlier_idx) = p;
|
||||
|
||||
inlier_idx++;
|
||||
}
|
||||
|
||||
matches_info.H = findHomography(src_points, dst_points, RANSAC);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
} // namespace detail
|
||||
} // namespace cv
|
||||
|
||||
Reference in New Issue
Block a user