Merge pull request #28986 from YangGuanyuhan:ai-aliked-lightglue-pipeline

[GSOC] feat: Add ALIKED feature extractor and LightGlue matcher with DNN integration #28986 ## PR Description ### Summary Integrate ALIKED and LightGlue into OpenCV's `features` module as native`Feature2D` and `DescriptorMatcher` implementations, enabling end-to-end neural feature matching within OpenCV's ecosystem. --- ### What's included #### New classes - **`cv::ALIKED`** extends `Feature2D` - CNN-based keypoint detection - 128-D descriptor extraction via ONNX Runtime - **`cv::LightGlueMatcher`** extends `DescriptorMatcher` - Deep feature matching with spatial context - Uses keypoints and image sizes during matching --- #### API design - Standard OpenCV patterns: - `detectAndCompute()` - `match()` - `knnMatch()` - Multiple factory methods: - ONNX model path - In-memory model buffer - Pre-loaded `dnn::Net` - `Params` structs use `CV_EXPORTS_W_SIMPLE` for Python/Java bindings support - Optional DNN dependency: - `HAVE_OPENCV_DNN` guards - Stub implementations throw `StsNotImplemented` --- ### Files added | File | Description | |------|-------------| | `src/feature2d_aliked.cpp` | ALIKED implementation | | `src/matchers_lightglue.cpp` | LightGlueMatcher implementation | | `src/aliked_context.hpp` | Shared internal context struct | | `test/test_aliked_lightglue.cpp` | Unit tests (9 test cases) | | `samples/cpp/example_features_aliked_lightglue.cpp` | Demo application | --- ### Files modified - `CMakeLists.txt` - Add `opencv_dnn` as optional dependency - `features.hpp` - Add ALIKED and LightGlueMatcher declarations - `precomp.hpp` - Add DNN include guard --- ### Usage ```cpp // Feature extraction Ptr<ALIKED> aliked = ALIKED::create("aliked-n16rot-top1k-640.onnx"); vector<KeyPoint> kpts; Mat descs; aliked->detectAndCompute(image, Mat(), kpts, descs); // Feature matching Ptr<LightGlueMatcher> lg = LightGlueMatcher::create("aliked_lightglue.onnx"); lg->setPairInfo( kpts1Mat, kpts2Mat, img1.size(), img2.size() ); vector<DMatch> matches; lg->match(descs1, descs2, matches); ```` please refer to samples/cpp/example_features_aliked_lightglue.cpp --- ### Test plan * Build with `BUILD_LIST=features,dnn` * Build without DNN: * Verify stubs compile * Verify `StsNotImplemented` is thrown * Run: * `ctest -R Features2d_ALIKED` * `ctest -R Features2d_LightGlueMatcher` * Run sample application with: * Real images * Real ONNX models * Verify Python/Java bindings compile and work --- ### Related Phase 1 of the "End-to-End AI Feature Extraction and LightGlue Matching Pipeline" GSoC project. Designed to be extensible to: * XFeat * SuperPoint * Other neural feature extractors ### test dependency Depends on the opencv_extra PR adding ALIKED and LightGlue test models: - [opencv_extra PR](https://github.com/opencv/opencv_extra/pull/1366) This PR adds the following ONNX models to `download_models.py`: - `aliked-n16rot-top1k-640.onnx` - `aliked_lightglue.onnx` These models are required for the `features2d` tests in the main OpenCV repository to validate the ALIKED and LightGlue feature extraction and matching pipeline. ### Pull Request Readiness Checklist See details at https://github.com/opencv/opencv/wiki/How_to_contribute#making-a-good-pull-request - [x] I agree to contribute to the project under Apache 2 License. - [x] To the best of my knowledge, the proposed patch is not based on a code under GPL or another license that is incompatible with OpenCV - [x] The PR is proposed to the proper branch - [ ] There is a reference to the original bug report and related work - [x] There is accuracy test, performance test and test data in opencv_extra repository, if applicable Patch to opencv_extra has the same branch name. - [x] The feature is well documented and sample code can be built with the project CMake
2026-07-29 15:23:05 +04:00 · 2026-06-05 19:20:53 +08:00
parent 04aee009aa
commit 527f01449d
14 changed files with 1240 additions and 7 deletions
@@ -0,0 +1,169 @@
+// This file is part of OpenCV project.
+// It is subject to the license terms in the LICENSE file found in the top-level directory
+// of this distribution and at http://opencv.org/license.html.
+
+// ALIKED + LightGlueMatcher usage example
+// Demonstrates feature detection, extraction, and matching using ALIKED and LightGlue.
+
+#include <opencv2/features.hpp>
+#include <opencv2/imgcodecs.hpp>
+#include <opencv2/imgproc.hpp>
+#include <opencv2/highgui.hpp>
+#include <iostream>
+
+using namespace cv;
+using namespace std;
+
+int main(int argc, char** argv)
+{
+    // ---- Parse arguments ----
+    String alikedModel, lightglueModel, imgPath1, imgPath2;
+
+    if (argc >= 5)
+    {
+        imgPath1 = argv[1];
+        imgPath2 = argv[2];
+        alikedModel = argv[3];
+        lightglueModel = argv[4];
+    }
+else
+    {
+        cout << "Usage: " << argv[0] << " <image1> <image2> <aliked_model> <lightglue_model>" << endl;
+        cout << endl;
+        cout << "Example:" << endl;
+        cout << "  " << argv[0] << " img1.jpg img2.jpg aliked-n16rot-top1k-640.onnx aliked_lightglue.onnx" << endl;
+        return 0;
+    }
+
+    // ---- Load images ----
+    Mat img1 = imread(imgPath1);
+    Mat img2 = imread(imgPath2);
+    if (img1.empty() || img2.empty())
+    {
+        cerr << "Error: cannot load images." << endl;
+        return -1;
+    }
+
+    // ================================================================
+    //  1. Create ALIKED feature extractor
+    // ================================================================
+    // Method A: From ONNX model file
+    Ptr<ALIKED> aliked = ALIKED::create(alikedModel);
+
+    // Method B: Customize parameters
+    // ALIKED::Params params;
+    // params.inputSize = Size(640, 640);       // Network input resolution
+    // params.normalizeDescriptors = true;       // L2-normalize descriptors
+    // Ptr<ALIKED> aliked = ALIKED::create(alikedModel, params);
+
+    // Method C: From in-memory model data
+    // vector<uchar> modelData = readFile(alikedModel);
+    // Ptr<ALIKED> aliked = ALIKED::create(modelData);
+
+    cout << "Descriptor size: " << aliked->descriptorSize() << endl;  // 128
+    cout << "Descriptor type: " << aliked->descriptorType() << endl;  // CV_32F
+    cout << "Default norm: " << aliked->defaultNorm() << endl;        // NORM_L2
+
+    // ================================================================
+    //  2. Detect keypoints and compute descriptors
+    // ================================================================
+    vector<KeyPoint> kpts1, kpts2;
+    Mat descs1, descs2;
+
+    // Method A: detect + compute in one call (recommended)
+    aliked->detectAndCompute(img1, Mat(), kpts1, descs1);
+    aliked->detectAndCompute(img2, Mat(), kpts2, descs2);
+
+    // Method B: detect only (no descriptors)
+    // vector<KeyPoint> kpts;
+    // aliked->detect(img, kpts);
+
+    // Method C: compute only (from existing keypoints)
+    // Mat descs;
+    // aliked->compute(img, kpts, descs);
+
+    cout << "Image 1: " << kpts1.size() << " keypoints, descriptors " << descs1.rows << "x" << descs1.cols << endl;
+    cout << "Image 2: " << kpts2.size() << " keypoints, descriptors " << descs2.rows << "x" << descs2.cols << endl;
+
+    // ================================================================
+    //  3. Create LightGlueMatcher
+    // ================================================================
+    // Method A: From ONNX model file
+    Ptr<LightGlueMatcher> lg = LightGlueMatcher::create(lightglueModel);
+
+    // Method B: Customize parameters
+    // LightGlueMatcher::Params lgParams;
+    // lgParams.scoreThreshold = 0.1f;    // Filter low-confidence matches
+    // lgParams.disableWinograd = false;   // Keep Winograd convolution
+    // Ptr<LightGlueMatcher> lg = LightGlueMatcher::create(lightglueModel, lgParams);
+
+    // Method C: From in-memory model data
+    // vector<uchar> lgData = readFile(lightglueModel);
+    // Ptr<LightGlueMatcher> lg = LightGlueMatcher::create(lgData);
+
+    // ================================================================
+    //  4. Set keypoint context for LightGlue
+    // ================================================================
+    // LightGlue needs keypoint coordinates + image sizes for spatial reasoning.
+    // Build Nx2 float matrices with pixel coordinates.
+
+    Mat kpts1Mat((int)kpts1.size(), 2, CV_32F);
+    Mat kpts2Mat((int)kpts2.size(), 2, CV_32F);
+    for (size_t i = 0; i < kpts1.size(); i++)
+    {
+        kpts1Mat.at<float>((int)i, 0) = kpts1[i].pt.x;
+        kpts1Mat.at<float>((int)i, 1) = kpts1[i].pt.y;
+    }
+    for (size_t i = 0; i < kpts2.size(); i++)
+    {
+        kpts2Mat.at<float>((int)i, 0) = kpts2[i].pt.x;
+        kpts2Mat.at<float>((int)i, 1) = kpts2[i].pt.y;
+    }
+
+    // setPairInfo must be called before match()/knnMatch()
+    lg->setPairInfo(kpts1Mat, kpts2Mat, img1.size(), img2.size());
+
+    // ================================================================
+    //  5. Match descriptors
+    // ================================================================
+
+    // Method A: 1-to-1 matching (returns best match per query keypoint)
+    vector<DMatch> matches;
+    lg->match(descs1, descs2, matches);
+
+    cout << "1-to-1 matches: " << matches.size() << endl;
+
+    // Method B: kNN matching (k=1 only for LightGlue)
+    // vector<vector<DMatch>> knnMatches;
+    // lg->knnMatch(descs1, descs2, knnMatches, 1);
+    // // knnMatches[i] contains matches for query keypoint i
+
+    // ================================================================
+    //  6. Filter matches by confidence (optional)
+    // ================================================================
+    // DMatch distance = 1.0 - confidence_score
+    // Lower distance = better match
+
+    vector<DMatch> goodMatches;
+    float distanceThreshold = 0.9f;  // confidence > 0.1
+    for (const auto& m : matches)
+    {
+        if (m.distance < distanceThreshold)
+            goodMatches.push_back(m);
+    }
+    cout << "Good matches (distance < " << distanceThreshold << "): " << goodMatches.size() << endl;
+
+    // ================================================================
+    //  7. Visualize results
+    // ================================================================
+    Mat canvas;
+    cv::drawMatches(img1, kpts1, img2, kpts2, goodMatches, canvas,
+                    Scalar::all(-1), Scalar::all(-1), vector<char>(),
+                    DrawMatchesFlags::NOT_DRAW_SINGLE_POINTS);
+
+    imshow("ALIKED + LightGlue Matches", canvas);
+    cout << "Press any key to exit..." << endl;
+    waitKey(0);
+
+    return 0;
+}
@@ -16,6 +16,8 @@
 #include "opencv2/stitching/detail/seam_finders.hpp"
 #include "opencv2/stitching/detail/warpers.hpp"
 #include "opencv2/stitching/warpers.hpp"
+#include "opencv2/features.hpp"
+#include "opencv2/core/ocl.hpp"

 #ifdef HAVE_OPENCV_XFEATURES2D
 #include "opencv2/xfeatures2d.hpp"
@@ -45,9 +47,10 @@ static void printUsage(char** argv)
        "\nMotion Estimation Flags:\n"
        "  --work_megapix <float>\n"
        "      Resolution for image registration step. The default is 0.6 Mpx.\n"
-        "  --features (surf|orb|sift|akaze)\n"
+        "  --features (surf|orb|sift|akaze|aliked)\n"
        "      Type of features used for images matching.\n"
        "      The default is surf if available, orb otherwise.\n"
+        "      When using 'aliked', requires --matcher lightglue and DNN model paths.\n"
        "  --matcher (homography|affine)\n"
        "      Matcher used for pairwise image matching.\n"
        "  --estimator (homography|affine)\n"
@@ -103,7 +106,14 @@ static void printUsage(char** argv)
        "  --timelapse (as_is|crop) \n"
        "      Output warped images separately as frames of a time lapse movie, with 'fixed_' prepended to input file names.\n"
        "  --rangewidth <int>\n"
-        "      uses range_width to limit number of images to match with.\n";
+        "      uses range_width to limit number of images to match with.\n"
+        "\nDNN Feature Options (when --features aliked --matcher lightglue):\n"
+        "  --aliked_model <path>\n"
+        "      Path to ALIKED ONNX model file.\n"
+        "  --lightglue_model <path>\n"
+        "      Path to LightGlue ONNX model file (for ALIKED descriptors).\n"
+        "  --lg_score_thresh <float>\n"
+        "      LightGlue confidence threshold. The default is 0.0 (accept all).\n";
 }


@@ -142,6 +152,9 @@ float blend_strength = 5;
 string result_name = "result.jpg";
 bool timelapse = false;
 int range_width = -1;
+String aliked_model_path;
+String lightglue_model_path;
+float lg_score_thresh = 0.0f;


 static int parseCmdArgs(int argc, char** argv)
@@ -204,7 +217,7 @@ static int parseCmdArgs(int argc, char** argv)
        }
        else if (string(argv[i]) == "--matcher")
        {
-            if (string(argv[i + 1]) == "homography" || string(argv[i + 1]) == "affine")
+            if (string(argv[i + 1]) == "homography" || string(argv[i + 1]) == "affine" || string(argv[i + 1]) == "lightglue")
                matcher_type = argv[i + 1];
            else
            {
@@ -376,6 +389,21 @@ static int parseCmdArgs(int argc, char** argv)
            result_name = argv[i + 1];
            i++;
        }
+        else if (string(argv[i]) == "--aliked_model")
+        {
+            aliked_model_path = argv[i + 1];
+            i++;
+        }
+        else if (string(argv[i]) == "--lightglue_model")
+        {
+            lightglue_model_path = argv[i + 1];
+            i++;
+        }
+        else if (string(argv[i]) == "--lg_score_thresh")
+        {
+            lg_score_thresh = static_cast<float>(atof(argv[i + 1]));
+            i++;
+        }
        else
            img_names.push_back(argv[i]);
    }
@@ -383,6 +411,19 @@ static int parseCmdArgs(int argc, char** argv)
    {
        compose_megapix = 0.6;
    }
+
+    // Validate DNN options
+    if (features_type == "aliked" && matcher_type != "lightglue")
+    {
+        cout << "Error: --features aliked requires --matcher lightglue\n";
+        return -1;
+    }
+    if (features_type == "aliked" && (aliked_model_path.empty() || lightglue_model_path.empty()))
+    {
+        cout << "Error: --features aliked requires --aliked_model and --lightglue_model\n";
+        return -1;
+    }
+
    return 0;
 }

@@ -401,6 +442,11 @@ int main(int argc, char* argv[])
    if (retval)
        return retval;

+    // Disable OpenCL for DNN-based features to avoid backend sync issues
+    bool use_aliked = (features_type == "aliked");
+    if (use_aliked)
+        cv::ocl::setUseOpenCL(false);
+
    // Check if have enough images
    int num_images = static_cast<int>(img_names.size());
    if (num_images < 2)
@@ -418,7 +464,11 @@ int main(int argc, char* argv[])
 #endif

    Ptr<Feature2D> finder;
-    if (features_type == "orb")
+    if (use_aliked)
+    {
+        // ALIKED will be created per-image in the loop below
+    }
+    else if (features_type == "orb")
    {
        finder = ORB::create();
    }
@@ -488,7 +538,15 @@ int main(int argc, char* argv[])
            is_seam_scale_set = true;
        }

-        computeImageFeatures(finder, img, features[i]);
+        if (use_aliked)
+        {
+            Ptr<ALIKED> aliked = ALIKED::create(aliked_model_path);
+            computeImageFeatures(aliked, img, features[i]);
+        }
+        else
+        {
+            computeImageFeatures(finder, img, features[i]);
+        }
        features[i].img_idx = i;
        LOGLN("Features in image #" << i+1 << ": " << features[i].keypoints.size());

@@ -507,7 +565,14 @@ int main(int argc, char* argv[])
 #endif
    vector<MatchesInfo> pairwise_matches;
    Ptr<FeaturesMatcher> matcher;
-    if (matcher_type == "affine")
+    if (use_aliked && matcher_type == "lightglue")
+    {
+        Ptr<LightGlueMatcher> lg = LightGlueMatcher::create(lightglue_model_path);
+        Ptr<LightGlueFeaturesMatcher> lgMatcher = makePtr<LightGlueFeaturesMatcher>(lg);
+        lgMatcher->setScoreThreshold(lg_score_thresh);
+        matcher = lgMatcher;
+    }
+    else if (matcher_type == "affine")
        matcher = makePtr<AffineBestOf2NearestMatcher>(false, try_cuda, match_conf);
    else if (range_width==-1)
        matcher = makePtr<BestOf2NearestMatcher>(try_cuda, match_conf);