mirror of
https://github.com/opencv/opencv.git
synced 2026-07-30 07:43:03 +04:00
Merge pull request #28986 from YangGuanyuhan:ai-aliked-lightglue-pipeline
[GSOC] feat: Add ALIKED feature extractor and LightGlue matcher with DNN integration #28986 ## PR Description ### Summary Integrate ALIKED and LightGlue into OpenCV's `features` module as native`Feature2D` and `DescriptorMatcher` implementations, enabling end-to-end neural feature matching within OpenCV's ecosystem. --- ### What's included #### New classes - **`cv::ALIKED`** extends `Feature2D` - CNN-based keypoint detection - 128-D descriptor extraction via ONNX Runtime - **`cv::LightGlueMatcher`** extends `DescriptorMatcher` - Deep feature matching with spatial context - Uses keypoints and image sizes during matching --- #### API design - Standard OpenCV patterns: - `detectAndCompute()` - `match()` - `knnMatch()` - Multiple factory methods: - ONNX model path - In-memory model buffer - Pre-loaded `dnn::Net` - `Params` structs use `CV_EXPORTS_W_SIMPLE` for Python/Java bindings support - Optional DNN dependency: - `HAVE_OPENCV_DNN` guards - Stub implementations throw `StsNotImplemented` --- ### Files added | File | Description | |------|-------------| | `src/feature2d_aliked.cpp` | ALIKED implementation | | `src/matchers_lightglue.cpp` | LightGlueMatcher implementation | | `src/aliked_context.hpp` | Shared internal context struct | | `test/test_aliked_lightglue.cpp` | Unit tests (9 test cases) | | `samples/cpp/example_features_aliked_lightglue.cpp` | Demo application | --- ### Files modified - `CMakeLists.txt` - Add `opencv_dnn` as optional dependency - `features.hpp` - Add ALIKED and LightGlueMatcher declarations - `precomp.hpp` - Add DNN include guard --- ### Usage ```cpp // Feature extraction Ptr<ALIKED> aliked = ALIKED::create("aliked-n16rot-top1k-640.onnx"); vector<KeyPoint> kpts; Mat descs; aliked->detectAndCompute(image, Mat(), kpts, descs); // Feature matching Ptr<LightGlueMatcher> lg = LightGlueMatcher::create("aliked_lightglue.onnx"); lg->setPairInfo( kpts1Mat, kpts2Mat, img1.size(), img2.size() ); vector<DMatch> matches; lg->match(descs1, descs2, matches); ```` please refer to samples/cpp/example_features_aliked_lightglue.cpp --- ### Test plan * Build with `BUILD_LIST=features,dnn` * Build without DNN: * Verify stubs compile * Verify `StsNotImplemented` is thrown * Run: * `ctest -R Features2d_ALIKED` * `ctest -R Features2d_LightGlueMatcher` * Run sample application with: * Real images * Real ONNX models * Verify Python/Java bindings compile and work --- ### Related Phase 1 of the "End-to-End AI Feature Extraction and LightGlue Matching Pipeline" GSoC project. Designed to be extensible to: * XFeat * SuperPoint * Other neural feature extractors ### test dependency Depends on the opencv_extra PR adding ALIKED and LightGlue test models: - [opencv_extra PR](https://github.com/opencv/opencv_extra/pull/1366) This PR adds the following ONNX models to `download_models.py`: - `aliked-n16rot-top1k-640.onnx` - `aliked_lightglue.onnx` These models are required for the `features2d` tests in the main OpenCV repository to validate the ALIKED and LightGlue feature extraction and matching pipeline. ### Pull Request Readiness Checklist See details at https://github.com/opencv/opencv/wiki/How_to_contribute#making-a-good-pull-request - [x] I agree to contribute to the project under Apache 2 License. - [x] To the best of my knowledge, the proposed patch is not based on a code under GPL or another license that is incompatible with OpenCV - [x] The PR is proposed to the proper branch - [ ] There is a reference to the original bug report and related work - [x] There is accuracy test, performance test and test data in opencv_extra repository, if applicable Patch to opencv_extra has the same branch name. - [x] The feature is well documented and sample code can be built with the project CMake
This commit is contained in:
Yang Guanyuhan
GitHub
@@ -50,6 +50,10 @@
|
||||
#include "opencv2/flann/miniflann.hpp"
|
||||
#endif
|
||||
|
||||
#ifdef HAVE_OPENCV_DNN
|
||||
#include "opencv2/dnn.hpp"
|
||||
#endif
|
||||
|
||||
/**
|
||||
@defgroup features Features Framework
|
||||
@{
|
||||
@@ -835,6 +839,46 @@ public:
|
||||
CV_WRAP virtual const std::vector<std::vector<cv::Point> >& getBlobContours() const = 0;
|
||||
};
|
||||
|
||||
/** @brief ALIKED feature detector and descriptor extractor.
|
||||
|
||||
ALIKED (A Lightweight Image KEYpoint Detector) is a CNN-based feature detector and descriptor
|
||||
extractor, as described in @cite Zhao23 . It produces 128-dimensional float descriptors and
|
||||
keypoints with sub-pixel accuracy.
|
||||
|
||||
The model expects RGB input [1,3,H,W] and internally converts BGR images to RGB.
|
||||
*/
|
||||
class CV_EXPORTS_W ALIKED : public Feature2D
|
||||
{
|
||||
protected:
|
||||
ALIKED();
|
||||
public:
|
||||
virtual ~ALIKED();
|
||||
|
||||
struct CV_EXPORTS_W_SIMPLE Params
|
||||
{
|
||||
CV_WRAP Params();
|
||||
CV_PROP_RW Size inputSize; //!< Input image size for the network, default 640x640
|
||||
CV_PROP_RW bool normalizeDescriptors; //!< Whether to L2-normalize descriptors, default true
|
||||
CV_PROP_RW int engine; //!< DNN engine type (dnn::EngineType), default ENGINE_NEW
|
||||
CV_PROP_RW int backend; //!< DNN backend, default DNN_BACKEND_DEFAULT
|
||||
CV_PROP_RW int target; //!< DNN target, default DNN_TARGET_CPU
|
||||
};
|
||||
|
||||
/** @brief Creates ALIKED from a model file path.
|
||||
@param modelPath Path to the ONNX model file.
|
||||
@param params ALIKED parameters.
|
||||
*/
|
||||
CV_WRAP static Ptr<ALIKED> create(const String& modelPath, const ALIKED::Params& params = ALIKED::Params());
|
||||
|
||||
#ifdef HAVE_OPENCV_DNN
|
||||
/** @brief Creates ALIKED from in-memory model data.
|
||||
@param modelData Buffer containing the model data.
|
||||
@param params ALIKED parameters.
|
||||
*/
|
||||
static Ptr<ALIKED> create(const std::vector<uchar>& modelData, const ALIKED::Params& params = ALIKED::Params());
|
||||
#endif
|
||||
};
|
||||
|
||||
|
||||
/****************************************************************************************\
|
||||
* Distance *
|
||||
@@ -1252,6 +1296,61 @@ protected:
|
||||
|
||||
#endif
|
||||
|
||||
/** @brief LightGlue feature matcher.
|
||||
|
||||
LightGlue is a CNN-based feature matcher, as described in @cite Lindenberger23 . It takes
|
||||
keypoint locations and descriptors from two images and directly predicts match pairs. Unlike
|
||||
traditional matchers that compute descriptor distances, LightGlue uses attention mechanisms
|
||||
to produce confidence scores for each potential match pair.
|
||||
|
||||
The matcher extends DescriptorMatcher and supports the standard match(), knnMatch(), and
|
||||
radiusMatch() interfaces. Context (keypoints and image sizes) must be provided via
|
||||
setPairInfo() before matching.
|
||||
*/
|
||||
class CV_EXPORTS_W LightGlueMatcher : public DescriptorMatcher
|
||||
{
|
||||
protected:
|
||||
LightGlueMatcher();
|
||||
public:
|
||||
virtual ~LightGlueMatcher();
|
||||
|
||||
/** @brief Creates LightGlueMatcher from a model file path.
|
||||
@param modelPath Path to the ONNX model file.
|
||||
@param scoreThreshold Match confidence threshold.
|
||||
@param backend DNN backend
|
||||
@param target DNN target
|
||||
*/
|
||||
CV_WRAP static Ptr<LightGlueMatcher> create(const String& modelPath, float scoreThreshold = 0.0f, int backend = 0, int target = 0);
|
||||
|
||||
#ifdef HAVE_OPENCV_DNN
|
||||
/** @brief Creates LightGlueMatcher from in-memory model data.
|
||||
@param modelData Buffer containing the model data.
|
||||
@param scoreThreshold Match confidence threshold.
|
||||
@param backend DNN backend
|
||||
@param target DNN target
|
||||
*/
|
||||
CV_WRAP_AS(createFromMemory) static Ptr<LightGlueMatcher> create(const std::vector<uchar>& modelData, float scoreThreshold = 0.0f, int backend = 0, int target = 0);
|
||||
#endif
|
||||
|
||||
/** @brief Sets the keypoint and image size context for the next match() call.
|
||||
|
||||
This provides the spatial context that LightGlue needs in addition to descriptors.
|
||||
Must be called before match()/knnMatch()/radiusMatch() unless using automatic context
|
||||
from in-process ALIKED instances.
|
||||
|
||||
@param queryKpts Query image keypoints (Nx2 float matrix with x,y coordinates).
|
||||
@param trainKpts Train image keypoints (Nx2 float matrix with x,y coordinates).
|
||||
@param queryImageSize Size of the query image (width, height).
|
||||
@param trainImageSize Size of the train image (width, height).
|
||||
*/
|
||||
CV_WRAP virtual void setPairInfo(InputArray queryKpts, InputArray trainKpts,
|
||||
Size queryImageSize = Size(), Size trainImageSize = Size()) = 0;
|
||||
|
||||
/** @brief Clears stored pair context information.
|
||||
*/
|
||||
CV_WRAP virtual void clearPairInfo() = 0;
|
||||
};
|
||||
|
||||
//! @} features_match
|
||||
|
||||
/****************************************************************************************\
|
||||
|
||||
@@ -0,0 +1,21 @@
|
||||
// This file is part of OpenCV project.
|
||||
// It is subject to the license terms in the LICENSE file found in the top-level directory
|
||||
// of this distribution and at http://opencv.org/license.html.
|
||||
|
||||
#ifndef OPENCV_FEATURES_ALIKED_CONTEXT_HPP
|
||||
#define OPENCV_FEATURES_ALIKED_CONTEXT_HPP
|
||||
|
||||
#include "opencv2/features.hpp"
|
||||
|
||||
namespace cv
|
||||
{
|
||||
|
||||
struct ALIKEDContext
|
||||
{
|
||||
Mat normalizedKeypoints; // Nx2 float, coordinates in [-1, 1]
|
||||
Size imageSize;
|
||||
};
|
||||
|
||||
} // namespace cv
|
||||
|
||||
#endif
|
||||
@@ -0,0 +1,186 @@
|
||||
// This file is part of OpenCV project.
|
||||
// It is subject to the license terms in the LICENSE file found in the top-level directory
|
||||
// of this distribution and at http://opencv.org/license.html.
|
||||
|
||||
#include "precomp.hpp"
|
||||
|
||||
#ifdef HAVE_OPENCV_DNN
|
||||
#include "opencv2/dnn.hpp"
|
||||
#include "aliked_context.hpp"
|
||||
#endif
|
||||
|
||||
namespace cv
|
||||
{
|
||||
|
||||
ALIKED::ALIKED() {}
|
||||
ALIKED::~ALIKED() {}
|
||||
|
||||
ALIKED::Params::Params()
|
||||
{
|
||||
inputSize = Size(640, 640);
|
||||
normalizeDescriptors = true;
|
||||
#ifdef HAVE_OPENCV_DNN
|
||||
engine = dnn::ENGINE_NEW;
|
||||
backend = dnn::DNN_BACKEND_DEFAULT;
|
||||
target = dnn::DNN_TARGET_CPU;
|
||||
#else
|
||||
engine = -1;
|
||||
backend = -1;
|
||||
target = -1;
|
||||
#endif
|
||||
}
|
||||
|
||||
#ifdef HAVE_OPENCV_DNN
|
||||
|
||||
class ALIKEDImpl : public ALIKED
|
||||
{
|
||||
public:
|
||||
ALIKEDImpl(const ALIKED::Params& _params, const String& modelPath)
|
||||
: params(_params)
|
||||
{
|
||||
net = dnn::readNet(modelPath, "", "", static_cast<dnn::EngineType>(params.engine));
|
||||
CV_Assert(!net.empty());
|
||||
net.setPreferableBackend(params.backend);
|
||||
net.setPreferableTarget(params.target);
|
||||
}
|
||||
|
||||
ALIKEDImpl(const std::vector<uchar>& modelData, const ALIKED::Params& _params)
|
||||
: params(_params)
|
||||
{
|
||||
net = dnn::readNetFromONNX(modelData);
|
||||
CV_Assert(!net.empty());
|
||||
net.setPreferableBackend(params.backend);
|
||||
net.setPreferableTarget(params.target);
|
||||
}
|
||||
|
||||
void detectAndCompute(InputArray image, InputArray mask,
|
||||
std::vector<KeyPoint>& keypoints,
|
||||
OutputArray descriptors,
|
||||
bool useProvidedKeypoints) CV_OVERRIDE;
|
||||
|
||||
int descriptorSize() const CV_OVERRIDE;
|
||||
int descriptorType() const CV_OVERRIDE;
|
||||
int defaultNorm() const CV_OVERRIDE;
|
||||
bool empty() const CV_OVERRIDE;
|
||||
|
||||
const ALIKEDContext& getLastContext() const { return lastContext; }
|
||||
|
||||
protected:
|
||||
dnn::Net net;
|
||||
ALIKED::Params params;
|
||||
ALIKEDContext lastContext;
|
||||
|
||||
void runNetwork(InputArray image, std::vector<KeyPoint>& keypoints,
|
||||
Mat& descriptors, Mat& scores);
|
||||
};
|
||||
|
||||
void ALIKEDImpl::runNetwork(InputArray _image, std::vector<KeyPoint>& keypoints,
|
||||
Mat& descriptors, Mat& scores)
|
||||
{
|
||||
Mat image = _image.getMat();
|
||||
Size inputSz = params.inputSize;
|
||||
Size origSize = image.size();
|
||||
|
||||
// BGR->RGB conversion via swapRB=true
|
||||
Mat blob = dnn::blobFromImage(image, 1.0/255.0, inputSz, Scalar(), /*swapRB=*/true, /*crop=*/false);
|
||||
|
||||
net.setInput(blob, "image");
|
||||
|
||||
std::vector<String> outNames = {"keypoints", "descriptors", "scores"};
|
||||
std::vector<Mat> outputs;
|
||||
net.forward(outputs, outNames);
|
||||
|
||||
CV_Assert(outputs.size() == 3);
|
||||
|
||||
// ORT engine drops the batch dimension, so outputs are:
|
||||
// keypoints: [N, 2] (not [1, N, 2])
|
||||
// descriptors: [N, 128] (not [1, N, 128])
|
||||
// scores: [N] (not [1, N, 1])
|
||||
int N = outputs[0].rows;
|
||||
|
||||
Mat normKpts = outputs[0].reshape(0, N); // Nx2
|
||||
Mat desc = outputs[1].reshape(0, N); // Nx128
|
||||
Mat scr = outputs[2].reshape(0, N); // Nx1
|
||||
|
||||
// Store normalized keypoints for LightGlue context
|
||||
lastContext.normalizedKeypoints = normKpts.clone();
|
||||
lastContext.imageSize = origSize;
|
||||
|
||||
// Convert normalized [-1,1] coordinates to pixel coordinates
|
||||
keypoints.resize(N);
|
||||
for (int i = 0; i < N; i++)
|
||||
{
|
||||
float nx = normKpts.at<float>(i, 0);
|
||||
float ny = normKpts.at<float>(i, 1);
|
||||
float px = (nx + 1.0f) * 0.5f * (float)origSize.width;
|
||||
float py = (ny + 1.0f) * 0.5f * (float)origSize.height;
|
||||
float score = scr.at<float>(i, 0);
|
||||
keypoints[i] = KeyPoint(px, py, 1.0f, -1.0f, score, 0, -1);
|
||||
}
|
||||
|
||||
// Optionally L2-normalize descriptors
|
||||
if (params.normalizeDescriptors)
|
||||
{
|
||||
for (int i = 0; i < N; i++)
|
||||
{
|
||||
Mat row = desc.row(i);
|
||||
normalize(row, row);
|
||||
}
|
||||
}
|
||||
|
||||
descriptors = desc;
|
||||
scores = scr;
|
||||
}
|
||||
|
||||
void ALIKEDImpl::detectAndCompute(InputArray image, InputArray mask,
|
||||
std::vector<KeyPoint>& keypoints,
|
||||
OutputArray descriptors,
|
||||
bool useProvidedKeypoints)
|
||||
{
|
||||
CV_INSTRUMENT_REGION();
|
||||
CV_UNUSED(mask);
|
||||
CV_UNUSED(useProvidedKeypoints);
|
||||
|
||||
if (image.empty())
|
||||
{
|
||||
keypoints.clear();
|
||||
descriptors.release();
|
||||
return;
|
||||
}
|
||||
|
||||
Mat desc;
|
||||
Mat sc;
|
||||
runNetwork(image, keypoints, desc, sc);
|
||||
|
||||
if (descriptors.needed())
|
||||
desc.copyTo(descriptors);
|
||||
}
|
||||
|
||||
int ALIKEDImpl::descriptorSize() const { return 128; }
|
||||
int ALIKEDImpl::descriptorType() const { return CV_32F; }
|
||||
int ALIKEDImpl::defaultNorm() const { return NORM_L2; }
|
||||
bool ALIKEDImpl::empty() const { return net.empty(); }
|
||||
|
||||
Ptr<ALIKED> ALIKED::create(const String& modelPath, const ALIKED::Params& params)
|
||||
{
|
||||
return makePtr<ALIKEDImpl>(params, modelPath);
|
||||
}
|
||||
|
||||
Ptr<ALIKED> ALIKED::create(const std::vector<uchar>& modelData, const ALIKED::Params& params)
|
||||
{
|
||||
return makePtr<ALIKEDImpl>(modelData, params);
|
||||
}
|
||||
|
||||
#else // !HAVE_OPENCV_DNN
|
||||
|
||||
Ptr<ALIKED> ALIKED::create(const String& modelPath, const ALIKED::Params& params)
|
||||
{
|
||||
CV_UNUSED(modelPath);
|
||||
CV_UNUSED(params);
|
||||
CV_Error(cv::Error::StsNotImplemented,
|
||||
"ALIKED requires OpenCV built with opencv_dnn module!");
|
||||
}
|
||||
|
||||
#endif // HAVE_OPENCV_DNN
|
||||
|
||||
} // namespace cv
|
||||
@@ -0,0 +1,282 @@
|
||||
// This file is part of OpenCV project.
|
||||
// It is subject to the license terms in the LICENSE file found in the top-level directory
|
||||
// of this distribution and at http://opencv.org/license.html.
|
||||
|
||||
#include "precomp.hpp"
|
||||
|
||||
#ifdef HAVE_OPENCV_DNN
|
||||
#include "opencv2/dnn.hpp"
|
||||
#include "aliked_context.hpp"
|
||||
#endif
|
||||
|
||||
namespace cv
|
||||
{
|
||||
|
||||
LightGlueMatcher::LightGlueMatcher() {}
|
||||
LightGlueMatcher::~LightGlueMatcher() {}
|
||||
|
||||
#ifdef HAVE_OPENCV_DNN
|
||||
|
||||
struct LightGluePairContext
|
||||
{
|
||||
Mat queryKeypoints; // Nx2 float (normalized [-1,1] or pixel)
|
||||
Mat trainKeypoints; // Mx2 float
|
||||
Size queryImageSize;
|
||||
Size trainImageSize;
|
||||
bool valid;
|
||||
|
||||
LightGluePairContext() : valid(false) {}
|
||||
|
||||
void clear()
|
||||
{
|
||||
queryKeypoints.release();
|
||||
trainKeypoints.release();
|
||||
queryImageSize = Size();
|
||||
trainImageSize = Size();
|
||||
valid = false;
|
||||
}
|
||||
};
|
||||
|
||||
class LightGlueMatcherImpl : public LightGlueMatcher
|
||||
{
|
||||
public:
|
||||
LightGlueMatcherImpl(const String& modelPath, float _scoreThreshold, int backend, int target)
|
||||
{
|
||||
scoreThreshold = _scoreThreshold;
|
||||
net = dnn::readNet(modelPath, "", "");
|
||||
CV_Assert(!net.empty());
|
||||
net.setPreferableBackend(backend);
|
||||
net.setPreferableTarget(target);
|
||||
}
|
||||
|
||||
LightGlueMatcherImpl(const std::vector<uchar>& modelData, float _scoreThreshold, int backend, int target)
|
||||
{
|
||||
scoreThreshold = _scoreThreshold;
|
||||
net = dnn::readNetFromONNX(modelData);
|
||||
CV_Assert(!net.empty());
|
||||
net.setPreferableBackend(backend);
|
||||
net.setPreferableTarget(target);
|
||||
}
|
||||
|
||||
// Private constructor for clone() — shares the already-loaded network
|
||||
LightGlueMatcherImpl(const dnn::Net& _net, float _scoreThreshold)
|
||||
: net(_net), scoreThreshold(_scoreThreshold) {};
|
||||
|
||||
// DescriptorMatcher interface
|
||||
bool isMaskSupported() const CV_OVERRIDE { return false; }
|
||||
Ptr<DescriptorMatcher> clone(bool emptyTrainData) const CV_OVERRIDE;
|
||||
|
||||
// LightGlueMatcher interface
|
||||
void setPairInfo(InputArray queryKpts, InputArray trainKpts,
|
||||
Size queryImageSize = Size(), Size trainImageSize = Size()) CV_OVERRIDE;
|
||||
void clearPairInfo() CV_OVERRIDE;
|
||||
|
||||
protected:
|
||||
void knnMatchImpl(InputArray queryDescriptors,
|
||||
std::vector<std::vector<DMatch>>& matches, int k,
|
||||
InputArrayOfArrays masks = noArray(),
|
||||
bool compactResult = false) CV_OVERRIDE;
|
||||
void radiusMatchImpl(InputArray queryDescriptors,
|
||||
std::vector<std::vector<DMatch>>& matches, float maxDistance,
|
||||
InputArrayOfArrays masks = noArray(),
|
||||
bool compactResult = false) CV_OVERRIDE;
|
||||
|
||||
void lightglueMatch(const Mat& queryDesc, const Mat& trainDesc,
|
||||
const Mat& queryKpts, const Mat& trainKpts,
|
||||
Size queryImgSize, Size trainImgSize,
|
||||
std::vector<DMatch>& matches);
|
||||
|
||||
bool resolveContext(Mat& queryKpts, Mat& trainKpts,
|
||||
Size& queryImgSize, Size& trainImgSize);
|
||||
|
||||
dnn::Net net;
|
||||
float scoreThreshold;
|
||||
LightGluePairContext pairContext;
|
||||
};
|
||||
|
||||
Ptr<DescriptorMatcher> LightGlueMatcherImpl::clone(bool emptyTrainData) const
|
||||
{
|
||||
Ptr<LightGlueMatcherImpl> m = makePtr<LightGlueMatcherImpl>(net, scoreThreshold);
|
||||
// Always copy pairContext - it's matcher state, not train data
|
||||
m->pairContext = pairContext;
|
||||
if (!emptyTrainData)
|
||||
{
|
||||
m->trainDescCollection = trainDescCollection;
|
||||
m->utrainDescCollection = utrainDescCollection;
|
||||
}
|
||||
return m;
|
||||
}
|
||||
|
||||
void LightGlueMatcherImpl::setPairInfo(InputArray _queryKpts, InputArray _trainKpts,
|
||||
Size _queryImageSize, Size _trainImageSize)
|
||||
{
|
||||
pairContext.queryKeypoints = _queryKpts.getMat().clone();
|
||||
pairContext.trainKeypoints = _trainKpts.getMat().clone();
|
||||
pairContext.queryImageSize = _queryImageSize;
|
||||
pairContext.trainImageSize = _trainImageSize;
|
||||
pairContext.valid = true;
|
||||
}
|
||||
|
||||
void LightGlueMatcherImpl::clearPairInfo()
|
||||
{
|
||||
pairContext.clear();
|
||||
}
|
||||
|
||||
bool LightGlueMatcherImpl::resolveContext(Mat& queryKpts, Mat& trainKpts,
|
||||
Size& queryImgSize, Size& trainImgSize)
|
||||
{
|
||||
if (pairContext.valid)
|
||||
{
|
||||
queryKpts = pairContext.queryKeypoints;
|
||||
trainKpts = pairContext.trainKeypoints;
|
||||
queryImgSize = pairContext.queryImageSize;
|
||||
trainImgSize = pairContext.trainImageSize;
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
void LightGlueMatcherImpl::lightglueMatch(const Mat& queryDesc, const Mat& trainDesc,
|
||||
const Mat& queryKpts, const Mat& trainKpts,
|
||||
Size queryImgSize, Size trainImgSize,
|
||||
std::vector<DMatch>& matches)
|
||||
{
|
||||
int N = queryDesc.rows;
|
||||
int M = trainDesc.rows;
|
||||
|
||||
// Normalize keypoints to [-1, 1] if in pixel coordinates
|
||||
Mat kpts0 = queryKpts.clone();
|
||||
Mat kpts1 = trainKpts.clone();
|
||||
|
||||
if (queryImgSize.width > 0 && queryImgSize.height > 0)
|
||||
{
|
||||
for (int i = 0; i < kpts0.rows; i++)
|
||||
{
|
||||
kpts0.at<float>(i, 0) = kpts0.at<float>(i, 0) / (float)queryImgSize.width * 2.0f - 1.0f;
|
||||
kpts0.at<float>(i, 1) = kpts0.at<float>(i, 1) / (float)queryImgSize.height * 2.0f - 1.0f;
|
||||
}
|
||||
}
|
||||
if (trainImgSize.width > 0 && trainImgSize.height > 0)
|
||||
{
|
||||
for (int i = 0; i < kpts1.rows; i++)
|
||||
{
|
||||
kpts1.at<float>(i, 0) = kpts1.at<float>(i, 0) / (float)trainImgSize.width * 2.0f - 1.0f;
|
||||
kpts1.at<float>(i, 1) = kpts1.at<float>(i, 1) / (float)trainImgSize.height * 2.0f - 1.0f;
|
||||
}
|
||||
}
|
||||
|
||||
// Prepare blobs: [1, N, 2] and [1, N, D]
|
||||
int descDim = queryDesc.cols;
|
||||
int szK0[] = {1, N, 2};
|
||||
int szK1[] = {1, M, 2};
|
||||
int szD0[] = {1, N, descDim};
|
||||
int szD1[] = {1, M, descDim};
|
||||
Mat kpts0blob = kpts0.reshape(0, 3, szK0);
|
||||
Mat kpts1blob = kpts1.reshape(0, 3, szK1);
|
||||
Mat desc0blob = queryDesc.reshape(0, 3, szD0);
|
||||
Mat desc1blob = trainDesc.reshape(0, 3, szD1);
|
||||
|
||||
net.setInput(kpts0blob, "kpts0");
|
||||
net.setInput(kpts1blob, "kpts1");
|
||||
net.setInput(desc0blob, "desc0");
|
||||
net.setInput(desc1blob, "desc1");
|
||||
|
||||
std::vector<String> outNames = {"matches0", "mscores0"};
|
||||
std::vector<Mat> outs;
|
||||
net.forward(outs, outNames);
|
||||
|
||||
CV_Assert(outs.size() == 2);
|
||||
|
||||
// matches0: [M, 2] int64 - pair indices (kpt0_idx, kpt1_idx)
|
||||
// mscores0: [M] float32 - confidence per pair
|
||||
Mat matchesMat = outs[0];
|
||||
Mat scoresMat = outs[1];
|
||||
|
||||
matches.clear();
|
||||
int nMatches = matchesMat.rows;
|
||||
matches.reserve(nMatches);
|
||||
|
||||
for (int i = 0; i < nMatches; i++)
|
||||
{
|
||||
int qIdx = (int)matchesMat.at<int64_t>(i, 0);
|
||||
int tIdx = (int)matchesMat.at<int64_t>(i, 1);
|
||||
if (qIdx >= 0 && tIdx >= 0 && qIdx < N && tIdx < M)
|
||||
{
|
||||
float score = scoresMat.at<float>(i);
|
||||
if (score >= scoreThreshold)
|
||||
{
|
||||
matches.push_back(DMatch(qIdx, tIdx, 1.0f - score));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void LightGlueMatcherImpl::knnMatchImpl(InputArray _queryDescriptors,
|
||||
std::vector<std::vector<DMatch>>& matches,
|
||||
int k, InputArrayOfArrays, bool)
|
||||
{
|
||||
CV_INSTRUMENT_REGION();
|
||||
|
||||
if (k != 1)
|
||||
CV_Error(cv::Error::StsBadArg, "LightGlueMatcher only supports k=1");
|
||||
|
||||
Mat queryKpts, trainKpts;
|
||||
Size queryImgSize, trainImgSize;
|
||||
if (!resolveContext(queryKpts, trainKpts, queryImgSize, trainImgSize))
|
||||
{
|
||||
CV_Error(cv::Error::StsBadArg,
|
||||
"LightGlueMatcher: no valid context. Call setPairInfo() before matching.");
|
||||
}
|
||||
|
||||
CV_Assert(!trainDescCollection.empty());
|
||||
const Mat& trainDesc = trainDescCollection[0];
|
||||
Mat queryDesc = _queryDescriptors.getMat();
|
||||
|
||||
std::vector<DMatch> flatMatches;
|
||||
lightglueMatch(queryDesc, trainDesc, queryKpts, trainKpts,
|
||||
queryImgSize, trainImgSize, flatMatches);
|
||||
|
||||
matches.clear();
|
||||
matches.resize(queryDesc.rows);
|
||||
for (const auto& m : flatMatches)
|
||||
{
|
||||
matches[m.queryIdx].push_back(m);
|
||||
}
|
||||
|
||||
clearPairInfo();
|
||||
}
|
||||
|
||||
void LightGlueMatcherImpl::radiusMatchImpl(InputArray, std::vector<std::vector<DMatch>>&,
|
||||
float, InputArrayOfArrays, bool)
|
||||
{
|
||||
CV_Error(cv::Error::StsNotImplemented,
|
||||
"radiusMatch is not supported by LightGlueMatcher. Use match() or knnMatch().");
|
||||
}
|
||||
|
||||
Ptr<LightGlueMatcher> LightGlueMatcher::create(const String& modelPath, float scoreThreshold, int backend, int target)
|
||||
{
|
||||
return makePtr<LightGlueMatcherImpl>(modelPath, scoreThreshold, backend, target);
|
||||
}
|
||||
|
||||
Ptr<LightGlueMatcher> LightGlueMatcher::create(const std::vector<uchar>& modelData,
|
||||
float scoreThreshold, int backend, int target)
|
||||
{
|
||||
return makePtr<LightGlueMatcherImpl>(modelData, scoreThreshold, backend, target);
|
||||
}
|
||||
|
||||
#else // !HAVE_OPENCV_DNN
|
||||
|
||||
Ptr<LightGlueMatcher> LightGlueMatcher::create(const String& modelPath,
|
||||
float scoreThreshold, int backend, int target)
|
||||
{
|
||||
CV_UNUSED(modelPath);
|
||||
CV_UNUSED(scoreThreshold);
|
||||
CV_UNUSED(backend);
|
||||
CV_UNUSED(target);
|
||||
CV_Error(cv::Error::StsNotImplemented,
|
||||
"LightGlueMatcher requires OpenCV built with opencv_dnn module!");
|
||||
}
|
||||
|
||||
#endif // HAVE_OPENCV_DNN
|
||||
|
||||
} // namespace cv
|
||||
@@ -47,6 +47,10 @@
|
||||
#include "opencv2/imgproc.hpp"
|
||||
#include "opencv2/geometry.hpp"
|
||||
|
||||
#ifdef HAVE_OPENCV_DNN
|
||||
#include "opencv2/dnn.hpp"
|
||||
#endif
|
||||
|
||||
#include "opencv2/core/utility.hpp"
|
||||
#include "opencv2/core/private.hpp"
|
||||
#include "opencv2/core/ocl.hpp"
|
||||
|
||||
@@ -0,0 +1,171 @@
|
||||
// This file is part of OpenCV project.
|
||||
// It is subject to the license terms in the LICENSE file found in the top-level directory
|
||||
// of this distribution and at http://opencv.org/license.html.
|
||||
|
||||
#include "test_precomp.hpp"
|
||||
#include "npy_blob.hpp"
|
||||
|
||||
#ifdef HAVE_OPENCV_DNN
|
||||
|
||||
#include "opencv2/dnn.hpp"
|
||||
#include "opencv2/core/utils/configuration.private.hpp"
|
||||
|
||||
namespace opencv_test { namespace {
|
||||
|
||||
static void skipIfClassicDnnEngine()
|
||||
{
|
||||
const auto engine = static_cast<cv::dnn::EngineType>(
|
||||
cv::utils::getConfigurationParameterSizeT("OPENCV_FORCE_DNN_ENGINE", cv::dnn::ENGINE_AUTO));
|
||||
if (engine == cv::dnn::ENGINE_CLASSIC)
|
||||
throw SkipTestException("ALIKED/LightGlue reference outputs are generated with the new DNN engine");
|
||||
}
|
||||
|
||||
TEST(Features2d_ALIKED, Regression)
|
||||
{
|
||||
skipIfClassicDnnEngine();
|
||||
const std::string modelPath = cvtest::findDataFile("dnn/onnx/models/aliked-n16rot-top1k-640.onnx", false);
|
||||
|
||||
Ptr<ALIKED> aliked = ALIKED::create(modelPath);
|
||||
ASSERT_FALSE(aliked.empty());
|
||||
ASSERT_FALSE(aliked->empty());
|
||||
EXPECT_EQ(aliked->descriptorSize(), 128);
|
||||
EXPECT_EQ(aliked->descriptorType(), CV_32F);
|
||||
EXPECT_EQ(aliked->defaultNorm(), NORM_L2);
|
||||
|
||||
const std::string imgPath = cvtest::findDataFile("shared/box.png");
|
||||
Mat img = imread(imgPath);
|
||||
ASSERT_FALSE(img.empty()) << "Could not load test image: " << imgPath;
|
||||
|
||||
std::vector<KeyPoint> keypoints;
|
||||
Mat descriptors;
|
||||
aliked->detectAndCompute(img, noArray(), keypoints, descriptors);
|
||||
|
||||
EXPECT_GT(keypoints.size(), 100u);
|
||||
ASSERT_EQ(descriptors.rows, static_cast<int>(keypoints.size()));
|
||||
EXPECT_EQ(descriptors.cols, 128);
|
||||
EXPECT_EQ(descriptors.type(), CV_32F);
|
||||
|
||||
for (const KeyPoint& kp : keypoints)
|
||||
{
|
||||
EXPECT_GE(kp.pt.x, 0.f);
|
||||
EXPECT_GE(kp.pt.y, 0.f);
|
||||
EXPECT_LT(kp.pt.x, static_cast<float>(img.cols));
|
||||
EXPECT_LT(kp.pt.y, static_cast<float>(img.rows));
|
||||
EXPECT_GT(kp.response, 0.f);
|
||||
}
|
||||
|
||||
// Load ORT reference outputs (generated with same OpenCV preprocessing)
|
||||
Mat refKpts = blobFromNPY(cvtest::findDataFile("dnn/aliked_keypoints_box.npy"));
|
||||
Mat refDescs = blobFromNPY(cvtest::findDataFile("dnn/aliked_descriptors_box.npy"));
|
||||
|
||||
// Keypoint count must match exactly
|
||||
ASSERT_EQ(static_cast<int>(keypoints.size()), refKpts.rows)
|
||||
<< "Keypoint count mismatch: got " << keypoints.size()
|
||||
<< ", expected " << refKpts.rows;
|
||||
|
||||
// Compare each keypoint (normalized coords -> pixel coords)
|
||||
const float origW = static_cast<float>(img.cols);
|
||||
const float origH = static_cast<float>(img.rows);
|
||||
for (int i = 0; i < refKpts.rows; i++)
|
||||
{
|
||||
float refX = (refKpts.at<float>(i, 0) + 1.0f) * 0.5f * origW;
|
||||
float refY = (refKpts.at<float>(i, 1) + 1.0f) * 0.5f * origH;
|
||||
EXPECT_NEAR(keypoints[i].pt.x, refX, 1e-4) << "Keypoint " << i << " x mismatch";
|
||||
EXPECT_NEAR(keypoints[i].pt.y, refY, 1e-4) << "Keypoint " << i << " y mismatch";
|
||||
}
|
||||
|
||||
// Compare descriptors row by row
|
||||
for (int i = 0; i < refDescs.rows; i++)
|
||||
{
|
||||
Mat diff = descriptors.row(i) - refDescs.row(i);
|
||||
double maxDiff = cv::norm(diff, cv::NORM_INF);
|
||||
EXPECT_LT(maxDiff, 1e-5) << "Descriptor " << i << " mismatch (max diff=" << maxDiff << ")";
|
||||
}
|
||||
}
|
||||
|
||||
TEST(Features2d_LightGlue, Regression)
|
||||
{
|
||||
skipIfClassicDnnEngine();
|
||||
const std::string alikedPath = cvtest::findDataFile("dnn/onnx/models/aliked-n16rot-top1k-640.onnx", false);
|
||||
const std::string lgPath = cvtest::findDataFile("dnn/onnx/models/aliked_lightglue.onnx", false);
|
||||
|
||||
Ptr<ALIKED> aliked = ALIKED::create(alikedPath);
|
||||
Ptr<LightGlueMatcher> lg = LightGlueMatcher::create(lgPath);
|
||||
ASSERT_FALSE(aliked.empty());
|
||||
ASSERT_FALSE(lg.empty());
|
||||
|
||||
Mat img1 = imread(cvtest::findDataFile("shared/box.png"));
|
||||
Mat img2 = imread(cvtest::findDataFile("shared/box_in_scene.png"));
|
||||
ASSERT_FALSE(img1.empty());
|
||||
ASSERT_FALSE(img2.empty());
|
||||
|
||||
// Detect features on both images
|
||||
std::vector<KeyPoint> kpts1, kpts2;
|
||||
Mat descs1, descs2;
|
||||
aliked->detectAndCompute(img1, noArray(), kpts1, descs1);
|
||||
aliked->detectAndCompute(img2, noArray(), kpts2, descs2);
|
||||
|
||||
ASSERT_GT(static_cast<int>(kpts1.size()), 0);
|
||||
ASSERT_GT(static_cast<int>(kpts2.size()), 0);
|
||||
|
||||
// Build keypoint matrices (pixel coordinates)
|
||||
Mat kpts1Mat(static_cast<int>(kpts1.size()), 2, CV_32F);
|
||||
Mat kpts2Mat(static_cast<int>(kpts2.size()), 2, CV_32F);
|
||||
for (size_t i = 0; i < kpts1.size(); i++)
|
||||
{
|
||||
kpts1Mat.at<float>(static_cast<int>(i), 0) = kpts1[i].pt.x;
|
||||
kpts1Mat.at<float>(static_cast<int>(i), 1) = kpts1[i].pt.y;
|
||||
}
|
||||
for (size_t i = 0; i < kpts2.size(); i++)
|
||||
{
|
||||
kpts2Mat.at<float>(static_cast<int>(i), 0) = kpts2[i].pt.x;
|
||||
kpts2Mat.at<float>(static_cast<int>(i), 1) = kpts2[i].pt.y;
|
||||
}
|
||||
|
||||
lg->setPairInfo(kpts1Mat, kpts2Mat, img1.size(), img2.size());
|
||||
|
||||
std::vector<DMatch> matches;
|
||||
lg->match(descs1, descs2, matches);
|
||||
|
||||
ASSERT_GT(static_cast<int>(matches.size()), 0);
|
||||
for (const auto& m : matches)
|
||||
{
|
||||
EXPECT_GE(m.queryIdx, 0);
|
||||
EXPECT_LT(m.queryIdx, static_cast<int>(kpts1.size()));
|
||||
EXPECT_GE(m.trainIdx, 0);
|
||||
EXPECT_LT(m.trainIdx, static_cast<int>(kpts2.size()));
|
||||
}
|
||||
|
||||
// Load ORT reference outputs
|
||||
Mat refMatches = blobFromNPY(cvtest::findDataFile("dnn/lightglue_matches.npy"));
|
||||
|
||||
// Match count must match exactly (same OpenCV preprocessing in both)
|
||||
ASSERT_EQ(static_cast<int>(matches.size()), refMatches.rows)
|
||||
<< "Match count mismatch: got " << matches.size()
|
||||
<< ", expected " << refMatches.rows;
|
||||
|
||||
// Compare each match (index pairs should be identical)
|
||||
for (int i = 0; i < refMatches.rows; i++)
|
||||
{
|
||||
int refQIdx = static_cast<int>(refMatches.at<int64_t>(i, 0));
|
||||
int refTIdx = static_cast<int>(refMatches.at<int64_t>(i, 1));
|
||||
EXPECT_EQ(matches[i].queryIdx, refQIdx) << "Match " << i << " queryIdx mismatch";
|
||||
EXPECT_EQ(matches[i].trainIdx, refTIdx) << "Match " << i << " trainIdx mismatch";
|
||||
}
|
||||
}
|
||||
|
||||
#else // !HAVE_OPENCV_DNN
|
||||
|
||||
TEST(Features2d_ALIKED, not_available)
|
||||
{
|
||||
EXPECT_THROW(ALIKED::create("dummy.onnx"), cv::Exception);
|
||||
}
|
||||
|
||||
TEST(Features2d_LightGlueMatcher, not_available)
|
||||
{
|
||||
EXPECT_THROW(LightGlueMatcher::create("dummy.onnx"), cv::Exception);
|
||||
}
|
||||
|
||||
#endif // HAVE_OPENCV_DNN
|
||||
|
||||
}} // namespace opencv_test
|
||||
@@ -12,6 +12,7 @@ void initTests()
|
||||
{
|
||||
cvtest::addDataSearchEnv("OPENCV_DNN_TEST_DATA_PATH");
|
||||
cvtest::addDataSearchSubDirectory(""); // override "cv" prefix below to access without "../dnn" hacks
|
||||
cvtest::addDataSearchEnv("OPENCV_DNN_TEST_DATA_PATH");
|
||||
}
|
||||
|
||||
CV_TEST_MAIN("cv", initTests())
|
||||
|
||||
Reference in New Issue
Block a user