1
0
mirror of https://github.com/opencv/opencv.git synced 2026-07-29 15:23:05 +04:00

Merge pull request #29331 from agrim-rai/slamVO

[GSOC] Added Visual Odometry (VO) for SLAM
This commit is contained in:
Abhishek Gola
2026-07-25 10:34:00 +05:30
committed by GitHub
29 changed files with 2746 additions and 2 deletions
+2
View File
@@ -832,6 +832,8 @@ macro(ocv_glob_module_sources)
"${CMAKE_CURRENT_LIST_DIR}/include/opencv2/${name}/hal/*.h"
"${CMAKE_CURRENT_LIST_DIR}/include/opencv2/${name}/utils/*.hpp"
"${CMAKE_CURRENT_LIST_DIR}/include/opencv2/${name}/utils/*.h"
"${CMAKE_CURRENT_LIST_DIR}/include/opencv2/${name}/slam/*.hpp"
"${CMAKE_CURRENT_LIST_DIR}/include/opencv2/${name}/slam/*.h"
"${CMAKE_CURRENT_LIST_DIR}/include/opencv2/${name}/legacy/*.h"
)
file(GLOB lib_hdrs_detail
@@ -998,6 +998,19 @@ public:
*/
CV_WRAP virtual bool isMaskSupported() const = 0;
/** @brief Provides keypoint and image-size context for matchers that need it (e.g. LightGlueMatcher).
Must be called before match()/knnMatch()/radiusMatch() for matchers that require this context.
Matchers that don't need it (e.g. BFMatcher, FlannBasedMatcher) ignore the call.
@param queryKpts Query image keypoints.
@param trainKpts Train image keypoints.
@param queryImageSize Size of the query image (width, height).
@param trainImageSize Size of the train image (width, height).
*/
CV_WRAP virtual void setImagePairInfo(const std::vector<KeyPoint>& queryKpts, const std::vector<KeyPoint>& trainKpts,
Size queryImageSize = Size(), Size trainImageSize = Size());
/** @brief Trains a descriptor matcher
Trains a descriptor matcher (for example, the flann index). In all methods to match, the method
@@ -1349,6 +1362,10 @@ public:
/** @brief Clears stored pair context information.
*/
CV_WRAP virtual void clearPairInfo() = 0;
/** @brief Convenience overload of setPairInfo() taking keypoints directly. */
CV_WRAP void setImagePairInfo(const std::vector<KeyPoint>& queryKpts, const std::vector<KeyPoint>& trainKpts,
Size queryImageSize = Size(), Size trainImageSize = Size()) CV_OVERRIDE;
};
//! @} features_match
+4
View File
@@ -580,6 +580,10 @@ bool DescriptorMatcher::empty() const
void DescriptorMatcher::train()
{}
void DescriptorMatcher::setImagePairInfo( const std::vector<KeyPoint>&, const std::vector<KeyPoint>&,
Size, Size )
{}
void DescriptorMatcher::match( InputArray queryDescriptors, InputArray trainDescriptors,
std::vector<DMatch>& matches, InputArray mask ) const
{
@@ -15,6 +15,26 @@ namespace cv
LightGlueMatcher::LightGlueMatcher() {}
LightGlueMatcher::~LightGlueMatcher() {}
void LightGlueMatcher::setImagePairInfo(const std::vector<KeyPoint>& queryKpts, const std::vector<KeyPoint>& trainKpts,
Size queryImageSize, Size trainImageSize)
{
Mat qk((int)queryKpts.size(), 2, CV_32F);
for (size_t i = 0; i < queryKpts.size(); ++i)
{
qk.at<float>((int)i, 0) = queryKpts[i].pt.x;
qk.at<float>((int)i, 1) = queryKpts[i].pt.y;
}
Mat tk((int)trainKpts.size(), 2, CV_32F);
for (size_t i = 0; i < trainKpts.size(); ++i)
{
tk.at<float>((int)i, 0) = trainKpts[i].pt.x;
tk.at<float>((int)i, 1) = trainKpts[i].pt.y;
}
setPairInfo(qk, tk, queryImageSize, trainImageSize);
}
#ifdef HAVE_OPENCV_DNN
struct LightGluePairContext
+2 -2
View File
@@ -4,8 +4,8 @@ set(debug_modules "")
if(DEBUG_opencv_ptcloud)
list(APPEND debug_modules opencv_highgui)
endif()
ocv_define_module(ptcloud opencv_geometry opencv_imgproc opencv_flann ${debug_modules}
WRAP java objc python js
ocv_define_module(ptcloud opencv_geometry opencv_imgproc opencv_features opencv_video ${debug_modules}
WRAP java objc python js
)
ocv_target_link_libraries(${the_module} ${LAPACK_LIBRARIES})
@@ -11,6 +11,7 @@
#include "opencv2/ptcloud/odometry.hpp"
#include "opencv2/ptcloud/odometry_frame.hpp"
#include "opencv2/ptcloud/odometry_settings.hpp"
#include "opencv2/ptcloud/slam.hpp"
/**
@defgroup ptcloud Point Cloud Processing
@@ -0,0 +1,23 @@
// This file is part of OpenCV project.
// It is subject to the license terms in the LICENSE file found in the top-level directory
// of this distribution and at http://opencv.org/license.html.
// Copyright (C) 2026, BigVision LLC, all rights reserved.
// Third party copyrights are property of their respective owners.
#ifndef OPENCV_PTCLOUD_SLAM_HPP
#define OPENCV_PTCLOUD_SLAM_HPP
/**
@defgroup slam SLAM and Visual Odometry
Monocular visual odometry pipeline. Entry point is @ref cv::slam::VisualOdometry.
Bootstraps an initial map from two-view geometry, then tracks subsequent frames
with PnP, growing the map at keyframe promotions.
*/
#include "opencv2/ptcloud/slam/types.hpp"
#include "opencv2/ptcloud/slam/map.hpp"
#include "opencv2/ptcloud/slam/odometry_params.hpp"
#include "opencv2/ptcloud/slam/visual_odometry.hpp"
#endif // OPENCV_PTCLOUD_SLAM_HPP
@@ -0,0 +1,78 @@
// This file is part of OpenCV project.
// It is subject to the license terms in the LICENSE file found in the top-level directory
// of this distribution and at http://opencv.org/license.html.
// Copyright (C) 2026, BigVision LLC, all rights reserved.
// Third party copyrights are property of their respective owners.
#ifndef OPENCV_SLAM_MAP_HPP
#define OPENCV_SLAM_MAP_HPP
#include "opencv2/ptcloud/slam/types.hpp"
#include <set>
#include <vector>
namespace cv {
namespace slam {
//! @addtogroup slam
//! @{
/** @brief Owns all persistent SLAM state (KeyFrames and MapPoints).
Pointers from addKeyframe / addMapPoint are valid until removeMapPoint / clear. */
class CV_EXPORTS Map
{
public:
Map();
~Map();
Map(const Map&) = delete;
Map& operator=(const Map&) = delete;
// Keyframes
KeyFrame* addKeyframe(KeyFrame* kf); //!< takes ownership; assigns id if < 0
KeyFrame* getKeyframe(int id) const;
const std::set<KeyFrame*>& keyframes() const;
int numKeyframes() const;
// Map points
MapPoint* addMapPoint(MapPoint* mp); //!< takes ownership; assigns id if < 0
MapPoint* getMapPoint(int id) const;
const std::set<MapPoint*>& mapPoints() const;
int numMapPoints() const;
void addObservation(KeyFrame* kf, size_t kpIdx, MapPoint* mp);
void removeObservation(KeyFrame* kf, MapPoint* mp);
void removeMapPoint(MapPoint* mp);
// Reference / current keyframes
void setRefKeyframe(KeyFrame* kf);
KeyFrame* getRefKeyframe() const;
void setCurrentKeyframe(KeyFrame* kf);
KeyFrame* getCurrentKeyframe() const;
// Trajectory
void appendPose(const Matx44d& T_cw);
const std::vector<Matx44d>& trajectory() const;
// Lifecycle
void clear();
private:
struct Impl;
Ptr<Impl> impl;
};
//! @}
}} // namespace cv::slam
#endif // OPENCV_SLAM_MAP_HPP
@@ -0,0 +1,65 @@
// This file is part of OpenCV project.
// It is subject to the license terms in the LICENSE file found in the top-level directory
// of this distribution and at http://opencv.org/license.html.
// Copyright (C) 2026, BigVision LLC, all rights reserved.
// Third party copyrights are property of their respective owners.
#ifndef OPENCV_SLAM_ODOMETRY_PARAMS_HPP
#define OPENCV_SLAM_ODOMETRY_PARAMS_HPP
#include "opencv2/core.hpp"
namespace cv {
namespace slam {
//! @addtogroup slam
//! @{
/** @brief Tunable parameters for visual odometry: initialization, tracking, keyframe selection, and local-map refinement. */
struct CV_EXPORTS_W_SIMPLE OdometryParams
{
CV_WRAP OdometryParams() {}
// Bootstrap (two-view map initialization)
CV_PROP_RW int minInitInliers = 40; //!< Minimum match/inlier count at each bootstrap stage.
CV_PROP_RW double minInitParallaxDeg = 1.5; //!< Minimum parallax (deg) to trigger initialization.
CV_PROP_RW int minInitPoints = 50; //!< Minimum triangulated points to seed the map.
CV_PROP_RW double hfRatioThresh = 0.45; //!< Homography/fundamental score ratio above which homography is chosen.
CV_PROP_RW double minGrowthParallaxDeg = 0.1; //!< Minimum parallax (deg) to triangulate new points during map growth.
CV_PROP_RW double essentialRansacThresh = 1.0; //!< RANSAC reprojection threshold (px) for essential-matrix/homography estimation.
CV_PROP_RW double essentialRansacConfidence = 0.999; //!< RANSAC confidence for essential-matrix estimation.
// Tracking (PnP)
CV_PROP_RW double pnpReprojThresh = 4.0; //!< PnP RANSAC reprojection threshold (px).
CV_PROP_RW int pnpMinInliers = 6; //!< Minimum PnP inliers to accept a pose.
CV_PROP_RW int pnpRansacIters = 500; //!< Maximum PnP RANSAC iterations.
CV_PROP_RW double pnpConfidence = 0.99; //!< PnP RANSAC confidence.
// Motion model (guided match search)
CV_PROP_RW double motionModelRadius = 15.0; //!< Guided-match search radius (px).
CV_PROP_RW double motionModelRadiusWide = 30.0; //!< Wider fallback search radius (px) when the narrow search finds too few.
CV_PROP_RW int motionModelMinMatches = 20; //!< Matches below which the wider search runs.
CV_PROP_RW double descProjThresh = 1.0; //!< Descriptor-distance cutoff for a projected map-point match.
// Optical flow fallback
CV_PROP_RW int opticalFlowMinInliers = 10; //!< Minimum correspondences for the optical-flow fallback.
// Keyframe promotion
CV_PROP_RW int kfMinFrames = 1; //!< Minimum frames since last keyframe before inserting one.
CV_PROP_RW int kfMaxFrames = 30; //!< Frames since last keyframe after which one is forced.
CV_PROP_RW double kfInlierRatio = 0.70; //!< Insert a keyframe when inliers drop below this fraction of the last keyframe's.
CV_PROP_RW int kfMinInliers = 40; //!< Absolute inlier floor: max(kfMinInliers, kfInlierRatio * lastKfInliers).
CV_PROP_RW double kfRotThreshDeg = 5.0; //!< Rotation (deg) from last keyframe that forces a new one.
CV_PROP_RW double kfTransThresh = 0.5; //!< Translation from last keyframe that forces a new one.
// Local map refinement
CV_PROP_RW int localMapTopK = 10; //!< Top co-visible keyframes forming the local map.
CV_PROP_RW int localMapNeighborK = 5; //!< Covisibility neighbors expanded per local-map keyframe.
CV_PROP_RW double localMapRadius = 7.0; //!< Reprojection search radius (px) for local-map points.
};
//! @}
}} // namespace cv::slam
#endif // OPENCV_SLAM_ODOMETRY_PARAMS_HPP
@@ -0,0 +1,71 @@
// This file is part of OpenCV project.
// It is subject to the license terms in the LICENSE file found in the top-level directory
// of this distribution and at http://opencv.org/license.html.
// Copyright (C) 2026, BigVision LLC, all rights reserved.
// Third party copyrights are property of their respective owners.
#ifndef OPENCV_SLAM_TYPES_HPP
#define OPENCV_SLAM_TYPES_HPP
#include "opencv2/core.hpp"
#include "opencv2/core/types.hpp"
#include <map>
#include <vector>
namespace cv {
namespace slam {
//! @addtogroup slam
//! @{
// Pipeline lifecycle state
enum OdometryState
{
NOT_INITIALIZED = 0,
INITIALIZING = 1,
TRACKING = 2
};
struct MapPoint;
struct KeyFrame;
// 3D landmark, owned by Map
struct CV_EXPORTS MapPoint
{
int id = -1;
Point3d pos { 0, 0, 0 };
Mat refDesc;
std::map<KeyFrame*, size_t> observations; // kf -> keypoint index
int visibleCount = 0;
int foundCount = 0;
bool bad = false; // soft-delete; check before use
};
// Keyframe: pose + keypoints + covisibility graph, owned by Map
struct CV_EXPORTS KeyFrame
{
int id = -1;
Matx44d poseCw = Matx44d::eye(); // world->camera
std::vector<KeyPoint> keypoints;
Mat descriptors;
std::vector<Point2f> undistKpts; // parallel to keypoints
Size imageSize;
std::vector<MapPoint*> mapPoints; // parallel to keypoints; null = unmatched
std::map<KeyFrame*, int> covisibility;
std::vector<std::pair<KeyFrame*, int>> orderedCovisibility; // sorted descending by count
KeyFrame* parent = nullptr;
Mat globalDesc;
};
//! @}
}} // namespace cv::slam
#endif // OPENCV_SLAM_TYPES_HPP
@@ -0,0 +1,79 @@
// This file is part of OpenCV project.
// It is subject to the license terms in the LICENSE file found in the top-level directory
// of this distribution and at http://opencv.org/license.html.
// Copyright (C) 2026, BigVision LLC, all rights reserved.
// Third party copyrights are property of their respective owners.
#ifndef OPENCV_SLAM_VISUAL_ODOMETRY_HPP
#define OPENCV_SLAM_VISUAL_ODOMETRY_HPP
#include "opencv2/core.hpp"
#include "opencv2/features.hpp"
#include "opencv2/ptcloud/slam/types.hpp"
#include "opencv2/ptcloud/slam/map.hpp"
#include "opencv2/ptcloud/slam/odometry_params.hpp"
#include <vector>
namespace cv {
namespace slam {
//! @addtogroup slam
//! @{
/** @brief Monocular visual odometry pipeline.
State machine: NOT_INITIALIZED → INITIALIZING (H/F two-view bootstrap) → TRACKING
(per-frame PnP + local-map refinement). Tracking failure rewinds to INITIALIZING.
Purely in-memory: feed images with @ref processFrame and read back the trajectory
and map with @ref getTrajectory / @ref getMap. See samples/slam for driving this
from a directory of images and writing the result to disk (e.g. COLMAP format).
*/
class CV_EXPORTS_W VisualOdometry
{
public:
virtual ~VisualOdometry();
CV_WRAP static Ptr<VisualOdometry> create(
const Ptr<Feature2D>& detector,
const Ptr<DescriptorMatcher>& matcher,
InputArray cameraMatrix,
InputArray distCoeffs = noArray(),
const OdometryParams& params = OdometryParams());
/** @brief Feed one image. Returns true if a pose was emitted. */
CV_WRAP virtual bool processFrame(InputArray image) = 0;
/** @brief Reset to NOT_INITIALIZED, clearing map and trajectory. */
CV_WRAP virtual void reset() = 0;
CV_WRAP virtual OdometryState getState() const = 0;
CV_WRAP virtual Matx44d getLastPose() const = 0;
//! @note Not exposed to Python: Map holds raw pointers / non-convertible containers.
virtual const Map& getMap() const = 0;
//! @brief Number of keyframes in the map. Convenience for callers (e.g. Python) that
//! can't use @ref getMap directly.
CV_WRAP virtual int getNumKeyframes() const = 0;
//! @brief Number of map points in the map. Convenience for callers (e.g. Python) that
//! can't use @ref getMap directly.
CV_WRAP virtual int getNumMapPoints() const = 0;
CV_WRAP virtual const std::vector<Matx44d>& getTrajectory() const = 0;
CV_WRAP virtual const OdometryParams& getParams() const = 0;
CV_WRAP virtual void setParams(const OdometryParams& params) = 0;
protected:
VisualOdometry();
};
//! @}
}} // namespace cv::slam
#endif // OPENCV_SLAM_VISUAL_ODOMETRY_HPP
+148
View File
@@ -0,0 +1,148 @@
// This file is part of OpenCV project.
// It is subject to the license terms in the LICENSE file found in the top-level directory
// of this distribution and at http://opencv.org/license.html.
// Copyright (C) 2026, BigVision LLC, all rights reserved.
// Third party copyrights are property of their respective owners.
#include "precomp.hpp"
#include <set>
#include <unordered_map>
namespace cv {
namespace slam {
struct Map::Impl
{
std::set<KeyFrame*> keyframes;
std::set<MapPoint*> mapPoints;
std::unordered_map<int, KeyFrame*> kfIndex;
std::unordered_map<int, MapPoint*> mpIndex;
KeyFrame* refKf = nullptr;
KeyFrame* currentKf = nullptr;
std::vector<Matx44d> trajectory;
int nextKfId = 0;
int nextMpId = 0;
};
Map::Map() : impl(makePtr<Impl>()) {}
Map::~Map()
{
for (KeyFrame* kf : impl->keyframes) delete kf;
for (MapPoint* mp : impl->mapPoints) delete mp;
}
// Keyframes
KeyFrame* Map::addKeyframe(KeyFrame* kf)
{
CV_Assert(kf);
if (kf->id < 0)
kf->id = impl->nextKfId++;
else if (kf->id >= impl->nextKfId)
impl->nextKfId = kf->id + 1;
impl->keyframes.insert(kf);
impl->kfIndex[kf->id] = kf;
return kf;
}
KeyFrame* Map::getKeyframe(int id) const
{
auto it = impl->kfIndex.find(id);
return (it != impl->kfIndex.end()) ? it->second : nullptr;
}
const std::set<KeyFrame*>& Map::keyframes() const { return impl->keyframes; }
int Map::numKeyframes() const { return (int)impl->keyframes.size(); }
// Map points
MapPoint* Map::addMapPoint(MapPoint* mp)
{
CV_Assert(mp);
if (mp->id < 0)
mp->id = impl->nextMpId++;
else if (mp->id >= impl->nextMpId)
impl->nextMpId = mp->id + 1;
impl->mapPoints.insert(mp);
impl->mpIndex[mp->id] = mp;
return mp;
}
MapPoint* Map::getMapPoint(int id) const
{
auto it = impl->mpIndex.find(id);
return (it != impl->mpIndex.end()) ? it->second : nullptr;
}
const std::set<MapPoint*>& Map::mapPoints() const { return impl->mapPoints; }
int Map::numMapPoints() const { return (int)impl->mapPoints.size(); }
// Observations
void Map::addObservation(KeyFrame* kf, size_t kpIdx, MapPoint* mp)
{
CV_Assert(kf && mp);
CV_Assert(kpIdx < kf->mapPoints.size());
if (kf->mapPoints[kpIdx] != nullptr) return;
kf->mapPoints[kpIdx] = mp;
mp->observations[kf] = kpIdx;
}
void Map::removeObservation(KeyFrame* kf, MapPoint* mp)
{
if (!kf || !mp) return;
auto it = mp->observations.find(kf);
if (it == mp->observations.end()) return;
size_t kpIdx = it->second;
if (kpIdx < kf->mapPoints.size())
kf->mapPoints[kpIdx] = nullptr;
mp->observations.erase(it);
}
void Map::removeMapPoint(MapPoint* mp)
{
if (!mp) return;
for (auto& [kf, kpIdx] : mp->observations)
if (kpIdx < kf->mapPoints.size())
kf->mapPoints[kpIdx] = nullptr;
impl->mapPoints.erase(mp);
impl->mpIndex.erase(mp->id);
delete mp;
}
// Reference / current keyframes
void Map::setRefKeyframe(KeyFrame* kf) { impl->refKf = kf; }
KeyFrame* Map::getRefKeyframe() const { return impl->refKf; }
void Map::setCurrentKeyframe(KeyFrame* kf) { impl->currentKf = kf; }
KeyFrame* Map::getCurrentKeyframe() const { return impl->currentKf; }
// Trajectory
void Map::appendPose(const Matx44d& T_cw) { impl->trajectory.push_back(T_cw); }
const std::vector<Matx44d>& Map::trajectory() const { return impl->trajectory; }
// Lifecycle
void Map::clear()
{
for (KeyFrame* kf : impl->keyframes) delete kf;
for (MapPoint* mp : impl->mapPoints) delete mp;
impl->keyframes.clear();
impl->mapPoints.clear();
impl->kfIndex.clear();
impl->mpIndex.clear();
impl->refKf = nullptr;
impl->currentKf = nullptr;
impl->trajectory.clear();
impl->nextKfId = 0;
impl->nextMpId = 0;
}
}} // namespace cv::slam
@@ -0,0 +1,79 @@
// This file is part of OpenCV project.
// It is subject to the license terms in the LICENSE file found in the top-level directory
// of this distribution and at http://opencv.org/license.html.
// Copyright (C) 2026, BigVision LLC, all rights reserved.
// Third party copyrights are property of their respective owners.
#ifndef OPENCV_SLAM_FRAME_HPP
#define OPENCV_SLAM_FRAME_HPP
#include "../precomp.hpp"
namespace cv {
namespace slam {
/** @brief Per-frame scratch pad. Created per incoming image, never stored in the Map. */
struct Frame
{
Mat image;
std::vector<KeyPoint> keypoints;
Mat descriptors;
std::vector<Point2f> undistKpts; //!< undistorted, parallel to keypoints
Size imageSize;
Matx44d poseCw = Matx44d::eye();
std::vector<MapPoint*> mapPoints; //!< parallel to keypoints; nullptr = unmatched
std::vector<bool> outliers; //!< parallel to keypoints; true = inlier check failed
// Grid cell size in pixels — dimensions adapt to each image resolution.
static constexpr int CELL_SIZE_PX = 20;
int gridRows = 0;
int gridCols = 0;
std::unordered_map<int, std::vector<size_t>> grid; //!< cell key -> keypoint indices
int cellKey(int row, int col) const { return row * gridCols + col; }
void buildGrid()
{
grid.clear();
if (imageSize.width <= 0 || imageSize.height <= 0) return;
gridCols = std::max(1, (imageSize.width + CELL_SIZE_PX - 1) / CELL_SIZE_PX);
gridRows = std::max(1, (imageSize.height + CELL_SIZE_PX - 1) / CELL_SIZE_PX);
for (size_t i = 0; i < undistKpts.size(); ++i)
{
int col = std::min(gridCols - 1, std::max(0, (int)(undistKpts[i].x / CELL_SIZE_PX)));
int row = std::min(gridRows - 1, std::max(0, (int)(undistKpts[i].y / CELL_SIZE_PX)));
grid[cellKey(row, col)].push_back(i);
}
}
std::vector<size_t> getKeypointsInRadius(float x, float y, float r) const
{
std::vector<size_t> result;
if (gridRows <= 0 || gridCols <= 0 || grid.empty()) return result;
const int colMin = std::max(0, (int)((x - r) / CELL_SIZE_PX));
const int colMax = std::min(gridCols-1, (int)((x + r) / CELL_SIZE_PX));
const int rowMin = std::max(0, (int)((y - r) / CELL_SIZE_PX));
const int rowMax = std::min(gridRows-1, (int)((y + r) / CELL_SIZE_PX));
const float r2 = r * r;
for (int row = rowMin; row <= rowMax; ++row)
for (int col = colMin; col <= colMax; ++col)
{
auto it = grid.find(cellKey(row, col));
if (it == grid.end()) continue;
for (size_t idx : it->second)
{
float dx = undistKpts[idx].x - x;
float dy = undistKpts[idx].y - y;
if (dx * dx + dy * dy <= r2)
result.push_back(idx);
}
}
return result;
}
};
}} // namespace cv::slam
#endif // OPENCV_SLAM_FRAME_HPP
@@ -0,0 +1,146 @@
// This file is part of OpenCV project.
// It is subject to the license terms in the LICENSE file found in the top-level directory
// of this distribution and at http://opencv.org/license.html.
// Copyright (C) 2026, BigVision LLC, all rights reserved.
// Third party copyrights are property of their respective owners.
#include "../precomp.hpp"
#include "vo_impl.hpp"
namespace cv {
namespace slam {
// Factory
VisualOdometry::VisualOdometry() = default;
VisualOdometry::~VisualOdometry() = default;
Ptr<VisualOdometry> VisualOdometry::create(
const Ptr<Feature2D>& detector,
const Ptr<DescriptorMatcher>& matcher,
InputArray cameraMatrix,
InputArray distCoeffs,
const OdometryParams& params)
{
CV_Assert(detector && "VisualOdometry::create: detector must not be null");
CV_Assert(matcher && "VisualOdometry::create: matcher must not be null");
Mat K = cameraMatrix.getMat();
CV_Assert(!K.empty() && K.rows == 3 && K.cols == 3);
Mat dist = distCoeffs.empty() ? Mat() : distCoeffs.getMat();
return makePtr<VisualOdometryImpl>(detector, matcher, K, dist, params);
}
// Constructor
VisualOdometryImpl::VisualOdometryImpl(
const Ptr<Feature2D>& detector_,
const Ptr<DescriptorMatcher>& matcher_,
const Mat& cameraMatrix,
const Mat& distCoeffs,
const OdometryParams& params_)
: detector(detector_), matcher(matcher_), params(params_)
{
cameraMatrix.convertTo(K, CV_64F);
if (!distCoeffs.empty())
distCoeffs.convertTo(dist, CV_64F);
}
// reset / processFrame
void VisualOdometryImpl::reset()
{
state = NOT_INITIALIZED;
lastPoseCw = Matx44d::eye();
refFrame = Frame();
lastKf = nullptr;
framesSinceKf = 0;
lastKfInliers = 0;
velocity = Matx44d::eye();
hasVelocity = false;
prevFrame = Frame();
hasPrevFrame = false;
lastEvent.clear();
map.clear();
}
bool VisualOdometryImpl::processFrame(InputArray image)
{
CV_INSTRUMENT_REGION();
if (image.empty()) return false;
lastEvent.clear();
Frame currentFrame;
extractFeatures(image, currentFrame);
if (currentFrame.keypoints.empty() || currentFrame.descriptors.empty()) return false;
currentFrame.mapPoints.assign(currentFrame.keypoints.size(), nullptr);
currentFrame.outliers.assign(currentFrame.keypoints.size(), false);
currentFrame.buildGrid();
switch (state)
{
case NOT_INITIALIZED:
refFrame = currentFrame;
state = INITIALIZING;
return false;
case INITIALIZING:
return bootstrap(currentFrame);
case TRACKING:
return track(currentFrame);
}
return false;
}
// Feature extraction
void VisualOdometryImpl::extractFeatures(InputArray image, Frame& out) const
{
Mat img = image.getMat();
out.imageSize = img.size();
out.keypoints.clear();
// Detect and compute on the original image (color/grey is up to the detector).
detector->detectAndCompute(img, noArray(), out.keypoints, out.descriptors);
// Store a greyscale copy for the optical-flow fallback.
if (img.channels() > 1)
cvtColor(img, out.image, COLOR_BGR2GRAY);
else
out.image = img.clone();
// Pre-compute undistorted pixel coordinates used by every stage.
if (!out.keypoints.empty())
{
std::vector<Point2f> raw;
raw.reserve(out.keypoints.size());
for (const auto& kp : out.keypoints)
raw.push_back(kp.pt);
if (!dist.empty())
undistortPoints(raw, out.undistKpts, K, dist, noArray(), K);
else
out.undistKpts = raw;
}
}
// Frame matching helper
void VisualOdometryImpl::matchFrames(
const std::vector<KeyPoint>& qKp, const Mat& qDesc, Size qSz,
const std::vector<KeyPoint>& tKp, const Mat& tDesc, Size tSz,
std::vector<DMatch>& matches) const
{
matches.clear();
if (qDesc.empty() || tDesc.empty()) return;
if (qKp.empty() || tKp.empty()) return;
matcher->setImagePairInfo(qKp, tKp, qSz, tSz);
matcher->match(qDesc, tDesc, matches);
}
}} // namespace cv::slam
@@ -0,0 +1,252 @@
// This file is part of OpenCV project.
// It is subject to the license terms in the LICENSE file found in the top-level directory
// of this distribution and at http://opencv.org/license.html.
// Copyright (C) 2026, BigVision LLC, all rights reserved.
// Third party copyrights are property of their respective owners.
#include "../precomp.hpp"
#include "vo_impl.hpp"
namespace cv {
namespace slam {
namespace {
// Returns the homography candidate index with the most cheirality-consistent triangulations.
int bestHomographyCandidate(
const std::vector<Mat>& Rs,
const std::vector<Mat>& ts,
const std::vector<Point2f>& pts1,
const std::vector<Point2f>& pts2,
const Mat& P1)
{
int bestIdx = 0;
int bestCount = -1;
const int nSol = (int)Rs.size();
for (int s = 0; s < nSol; ++s)
{
Mat Rt(3, 4, CV_64F);
Rs[s].copyTo(Rt(Rect(0, 0, 3, 3)));
ts[s].reshape(1, 3).copyTo(Rt(Rect(3, 0, 1, 3)));
Mat Klocal = P1(Rect(0, 0, 3, 3)).clone();
Mat P2_s = Klocal * Rt;
Mat pts4D;
triangulatePoints(P1, P2_s, pts1, pts2, pts4D);
int count = 0;
for (int i = 0; i < pts4D.cols; ++i)
{
float w = pts4D.at<float>(3, i);
if (std::abs(w) < 1e-9f) continue;
float X = pts4D.at<float>(0, i) / w;
float Y = pts4D.at<float>(1, i) / w;
float Z = pts4D.at<float>(2, i) / w;
float Z2 = (float)(Rs[s].at<double>(2,0)*X + Rs[s].at<double>(2,1)*Y
+ Rs[s].at<double>(2,2)*Z
+ ts[s].reshape(1,3).at<double>(2,0));
if (Z > 0 && Z2 > 0) ++count;
}
if (count > bestCount) { bestCount = count; bestIdx = s; }
}
return bestIdx;
}
} // anonymous namespace
bool VisualOdometryImpl::bootstrap(Frame& cur)
{
std::vector<DMatch> matches;
matchFrames(refFrame.keypoints, refFrame.descriptors, refFrame.imageSize,
cur.keypoints, cur.descriptors, cur.imageSize, matches);
if ((int)matches.size() < params.minInitInliers)
{
// slide ref forward so we don't keep trying against a frame that's too far
refFrame = std::move(cur);
return false;
}
std::vector<Point2f> refU, curU;
refU.reserve(matches.size());
curU.reserve(matches.size());
for (const auto& m : matches)
{
refU.push_back(refFrame.undistKpts[m.queryIdx]);
curU.push_back(cur.undistKpts[m.trainIdx]);
}
Mat maskE, maskH;
Mat E = findEssentialMat(refU, curU, K, RANSAC,
params.essentialRansacConfidence,
params.essentialRansacThresh, 1000, maskE);
Mat H = findHomography(refU, curU, RANSAC,
params.essentialRansacThresh, maskH);
const int nE = (!E.empty() && !maskE.empty()) ? countNonZero(maskE) : 0;
const int nH = (!H.empty() && !maskH.empty()) ? countNonZero(maskH) : 0;
if (nE < params.minInitInliers && nH < params.minInitInliers)
return false;
const double RH = (double)nH / ((double)nH + (double)nE + 1e-9);
Mat R, t, modelMask;
Mat P1 = K * Mat::eye(3, 4, CV_64F);
if (RH > params.hfRatioThresh && !H.empty())
{
std::vector<Mat> Rs, ts, normals;
int nSol = decomposeHomographyMat(H, K, Rs, ts, normals);
if (nSol <= 0) return false;
std::vector<Point2f> p1In, p2In;
for (size_t i = 0; i < matches.size(); ++i)
if (!maskH.empty() && maskH.at<uchar>((int)i))
{ p1In.push_back(refU[i]); p2In.push_back(curU[i]); }
if (p1In.empty()) return false;
int best = bestHomographyCandidate(Rs, ts, p1In, p2In, P1);
R = Rs[best].clone();
t = ts[best].clone();
modelMask = maskH;
}
else
{
if (E.empty() || nE < params.minInitInliers) return false;
Mat recoverMask = maskE.clone();
int nPose = recoverPose(E, refU, curU, K, R, t, recoverMask);
if (nPose < params.minInitInliers) return false;
modelMask = recoverMask;
}
std::vector<Point2f> refIn, curIn;
std::vector<int> matchIn;
for (size_t i = 0; i < matches.size(); ++i)
if (!modelMask.empty() && modelMask.at<uchar>((int)i))
{
refIn.push_back(refU[i]);
curIn.push_back(curU[i]);
matchIn.push_back((int)i);
}
if ((int)refIn.size() < params.minInitInliers)
return false;
Mat Rt(3, 4, CV_64F);
R.copyTo(Rt(Rect(0, 0, 3, 3)));
t.reshape(1, 3).copyTo(Rt(Rect(3, 0, 1, 3)));
Mat P2 = K * Rt;
Mat pts4D;
triangulatePoints(P1, P2, refIn, curIn, pts4D);
Matx44d T_ref = Matx44d::eye();
Matx44d T_cur = detail::makePose(R, t);
std::vector<Point3d> goodPts;
std::vector<int> goodMatch;
int nValid = 0;
int nPos = 0;
for (int i = 0; i < pts4D.cols; ++i)
{
double w = pts4D.at<float>(3, i);
if (std::abs(w) < 1e-9) continue;
++nValid;
double X = pts4D.at<float>(0, i) / w;
double Y = pts4D.at<float>(1, i) / w;
double Z = pts4D.at<float>(2, i) / w;
if (Z <= 0) continue;
double Z2 = R.at<double>(2,0)*X + R.at<double>(2,1)*Y
+ R.at<double>(2,2)*Z + t.reshape(1,3).at<double>(2,0);
if (Z2 <= 0) continue;
++nPos;
Point3d Xw(X, Y, Z);
if (detail::parallaxDeg(Xw, T_ref, T_cur) < params.minInitParallaxDeg)
continue;
goodPts.push_back(Xw);
goodMatch.push_back(matchIn[i]);
}
// reject degenerate decompositions (< 90% positive-depth ratio)
if (nValid > 0 && (double)nPos / nValid < 0.9)
return false;
if ((int)goodPts.size() < params.minInitPoints)
return false;
// normalize scale: set median scene depth to 1.0
std::vector<double> depths;
depths.reserve(goodPts.size());
for (const auto& p : goodPts) depths.push_back(p.z);
std::nth_element(depths.begin(), depths.begin() + depths.size()/2, depths.end());
double med = depths[depths.size()/2];
if (med < 1e-9) return false;
double scale = 1.0 / med;
for (auto& p : goodPts) { p.x *= scale; p.y *= scale; p.z *= scale; }
Mat tSc;
t.reshape(1, 3).convertTo(tSc, CV_64F);
tSc = tSc * scale;
T_cur = detail::makePose(R, tSc);
auto makeKF = [](const Frame& f) -> KeyFrame* {
KeyFrame* kf = new KeyFrame();
kf->poseCw = Matx44d::eye();
kf->keypoints = f.keypoints;
kf->descriptors = f.descriptors.clone();
kf->undistKpts = f.undistKpts;
kf->imageSize = f.imageSize;
kf->mapPoints.assign(f.keypoints.size(), nullptr);
return kf;
};
KeyFrame* kfRef = makeKF(refFrame);
kfRef->poseCw = T_ref;
KeyFrame* kfCur = makeKF(cur);
kfCur->poseCw = T_cur;
kfCur->parent = kfRef;
map.addKeyframe(kfRef);
map.addKeyframe(kfCur);
for (size_t i = 0; i < goodPts.size(); ++i)
{
MapPoint* mp = new MapPoint();
mp->pos = goodPts[i];
const DMatch& m = matches[goodMatch[i]];
mp->refDesc = refFrame.descriptors.row(m.queryIdx).clone();
map.addMapPoint(mp);
map.addObservation(kfRef, (size_t)m.queryIdx, mp);
map.addObservation(kfCur, (size_t)m.trainIdx, mp);
}
detail::updateCovisibility(kfRef);
map.setRefKeyframe(kfRef);
map.setCurrentKeyframe(kfCur);
lastKf = kfCur;
framesSinceKf = 0;
lastKfInliers = (int)goodPts.size();
lastPoseCw = T_cur;
state = TRACKING;
hasVelocity = false;
map.appendPose(T_ref);
map.appendPose(T_cur);
refFrame = Frame();
return true;
}
}} // namespace cv::slam
@@ -0,0 +1,110 @@
// This file is part of OpenCV project.
// It is subject to the license terms in the LICENSE file found in the top-level directory
// of this distribution and at http://opencv.org/license.html.
// Copyright (C) 2026, BigVision LLC, all rights reserved.
// Third party copyrights are property of their respective owners.
#ifndef OPENCV_SLAM_VO_IMPL_HPP
#define OPENCV_SLAM_VO_IMPL_HPP
#include "../precomp.hpp"
#include "frame.hpp"
#include "../optimizer/pose_optimizer.hpp"
namespace cv {
namespace slam {
/** @brief Concrete VisualOdometry implementation (pimpl target).
Stage logic is split across:
- vo_bootstrap.cpp : two-view H/F initialisation
- vo_tracking.cpp : per-frame localisation (motion model, fallback 1/2, local map)
- vo_keyframe.cpp : keyframe promotion decision + covisibility helpers
- vo_map_growth.cpp : triangulation of new map points at promotion time
- visual_odometry.cpp : factory, processFrame()
*/
class VisualOdometryImpl CV_FINAL : public VisualOdometry
{
public:
VisualOdometryImpl(const Ptr<Feature2D>& detector,
const Ptr<DescriptorMatcher>& matcher,
const Mat& cameraMatrix,
const Mat& distCoeffs,
const OdometryParams& params);
// --- VisualOdometry interface -------------------------------------------
bool processFrame(InputArray image) CV_OVERRIDE;
void reset() CV_OVERRIDE;
OdometryState getState() const CV_OVERRIDE { return state; }
Matx44d getLastPose() const CV_OVERRIDE { return lastPoseCw; }
const Map& getMap() const CV_OVERRIDE { return map; }
int getNumKeyframes() const CV_OVERRIDE { return map.numKeyframes(); }
int getNumMapPoints() const CV_OVERRIDE { return map.numMapPoints(); }
const std::vector<Matx44d>& getTrajectory() const CV_OVERRIDE { return map.trajectory(); }
const OdometryParams& getParams() const CV_OVERRIDE { return params; }
void setParams(const OdometryParams& p) CV_OVERRIDE { params = p; }
// --- Stage entry points -------------------------------------------------
bool bootstrap(Frame& cur);
bool track(Frame& cur);
bool trackWithMotionModel(Frame& cur); // motion model
bool trackWithReferenceKF(Frame& cur); // fallback 1
bool trackWithOpticalFlow(Frame& cur); // fallback 2
void trackLocalMap(Frame& cur);
bool shouldPromoteKeyframe(int nInliers, const Matx44d& T_cw, String& reason) const;
void promoteKeyframeAndGrowMap(Frame& cur);
// --- Shared helpers (visual_odometry.cpp) --------------------------------
void extractFeatures(InputArray image, Frame& out) const;
void matchFrames(const std::vector<KeyPoint>& qKp, const Mat& qDesc, Size qSz,
const std::vector<KeyPoint>& tKp, const Mat& tDesc, Size tSz,
std::vector<DMatch>& matches) const;
// --- Owned state ---------------------------------------------------------
Ptr<Feature2D> detector;
Ptr<DescriptorMatcher> matcher;
Mat K; // 3×3 CV_64F
Mat dist; // distortion coefficients (may be empty)
OdometryParams params;
OdometryState state = NOT_INITIALIZED;
Matx44d lastPoseCw = Matx44d::eye();
Frame refFrame;
KeyFrame* lastKf = nullptr;
int framesSinceKf = 0;
int lastKfInliers = 0;
Matx44d velocity = Matx44d::eye();
bool hasVelocity = false;
Frame prevFrame;
bool hasPrevFrame = false;
String lastEvent;
Map map;
};
namespace detail {
double rotationAngleDeg(const Matx44d& A_cw, const Matx44d& B_cw);
double parallaxDeg(const Point3d& X_world, const Matx44d& A_cw, const Matx44d& B_cw);
Matx34d projectionFromPose(const Matx44d& T_cw);
Matx44d makePose(const Mat& R, const Mat& t);
Point3d cameraCenterWorld(const Matx44d& T_cw);
void updateCovisibility(KeyFrame* kf);
} // namespace detail
}} // namespace cv::slam
#endif // OPENCV_SLAM_VO_IMPL_HPP
@@ -0,0 +1,145 @@
// This file is part of OpenCV project.
// It is subject to the license terms in the LICENSE file found in the top-level directory
// of this distribution and at http://opencv.org/license.html.
// Copyright (C) 2026, BigVision LLC, all rights reserved.
// Third party copyrights are property of their respective owners.
#include "../precomp.hpp"
#include "vo_impl.hpp"
namespace cv {
namespace slam {
namespace detail {
double rotationAngleDeg(const Matx44d& A, const Matx44d& B)
{
Matx33d Ra(A(0,0),A(0,1),A(0,2), A(1,0),A(1,1),A(1,2), A(2,0),A(2,1),A(2,2));
Matx33d Rb(B(0,0),B(0,1),B(0,2), B(1,0),B(1,1),B(1,2), B(2,0),B(2,1),B(2,2));
Matx33d D = Rb * Ra.t();
double tr = D(0,0) + D(1,1) + D(2,2);
double c = std::max(-1.0, std::min(1.0, (tr - 1.0) * 0.5));
return std::acos(c) * 180.0 / CV_PI;
}
Point3d cameraCenterWorld(const Matx44d& T_cw)
{
Matx33d R(T_cw(0,0),T_cw(0,1),T_cw(0,2),
T_cw(1,0),T_cw(1,1),T_cw(1,2),
T_cw(2,0),T_cw(2,1),T_cw(2,2));
Matx31d t(T_cw(0,3),T_cw(1,3),T_cw(2,3));
Matx31d C = -R.t() * t;
return Point3d(C(0),C(1),C(2));
}
double parallaxDeg(const Point3d& Xw, const Matx44d& A_cw, const Matx44d& B_cw)
{
Point3d CA = cameraCenterWorld(A_cw), CB = cameraCenterWorld(B_cw);
Point3d vA = Xw - CA, vB = Xw - CB;
double nA = std::sqrt(vA.dot(vA)), nB = std::sqrt(vB.dot(vB));
if (nA < 1e-9 || nB < 1e-9) return 0.0;
double c = std::max(-1.0, std::min(1.0, vA.dot(vB) / (nA * nB)));
return std::acos(c) * 180.0 / CV_PI;
}
Matx34d projectionFromPose(const Matx44d& T)
{
return Matx34d(T(0,0),T(0,1),T(0,2),T(0,3),
T(1,0),T(1,1),T(1,2),T(1,3),
T(2,0),T(2,1),T(2,2),T(2,3));
}
Matx44d makePose(const Mat& R, const Mat& t)
{
CV_Assert(R.rows == 3 && R.cols == 3 && t.total() == 3);
Matx44d T = Matx44d::eye();
Mat Rd, td;
R.convertTo(Rd, CV_64F);
t.reshape(1,3).convertTo(td, CV_64F);
for (int i = 0; i < 3; ++i)
{
for (int j = 0; j < 3; ++j) T(i,j) = Rd.at<double>(i,j);
T(i,3) = td.at<double>(i,0);
}
return T;
}
static void rebuildOrderedCovisibility(KeyFrame* target)
{
target->orderedCovisibility.clear();
for (auto& [other, cnt] : target->covisibility)
target->orderedCovisibility.push_back({other, cnt});
std::sort(target->orderedCovisibility.begin(),
target->orderedCovisibility.end(),
[](const auto& a, const auto& b){ return a.second > b.second; });
}
void updateCovisibility(KeyFrame* kf)
{
if (!kf) return;
kf->covisibility.clear();
for (MapPoint* mp : kf->mapPoints)
{
if (!mp || mp->bad) continue;
for (auto& [obsKf, kpIdx] : mp->observations)
{
if (obsKf == kf) continue;
kf->covisibility[obsKf]++;
obsKf->covisibility[kf]++;
}
}
rebuildOrderedCovisibility(kf);
for (auto& [nb, cnt] : kf->covisibility)
rebuildOrderedCovisibility(nb);
}
} // namespace detail
bool VisualOdometryImpl::shouldPromoteKeyframe(int nInliers, const Matx44d& T_cw,
String& reason) const
{
if (framesSinceKf <= params.kfMinFrames) return false;
if (framesSinceKf > params.kfMaxFrames)
{
reason = format("timeout(%d)", framesSinceKf);
return true;
}
if (!lastKf) return false;
// Motion-based triggers: rotation then translation (geometric, frame-count independent)
double rot = detail::rotationAngleDeg(lastKf->poseCw, T_cw);
if (rot > params.kfRotThreshDeg)
{
reason = format("rot=%.1fdeg", rot);
return true;
}
// Tracking quality drop
int minInliers = (lastKfInliers > 0)
? std::max(params.kfMinInliers, (int)(params.kfInlierRatio * lastKfInliers))
: params.kfMinInliers;
if (nInliers < minInliers)
{
reason = format("inliers=%d<%d", nInliers, minInliers);
return true;
}
Point3d cCur = detail::cameraCenterWorld(T_cw);
Point3d cLast = detail::cameraCenterWorld(lastKf->poseCw);
Point3d d = cCur - cLast;
double transDist = std::sqrt(d.dot(d));
if (transDist > params.kfTransThresh)
{
reason = format("trans=%.3f", transDist);
return true;
}
return false;
}
}} // namespace cv::slam
@@ -0,0 +1,135 @@
// This file is part of OpenCV project.
// It is subject to the license terms in the LICENSE file found in the top-level directory
// of this distribution and at http://opencv.org/license.html.
// Copyright (C) 2026, BigVision LLC, all rights reserved.
// Third party copyrights are property of their respective owners.
#include "../precomp.hpp"
#include "vo_impl.hpp"
namespace cv {
namespace slam {
namespace {
inline Point2d projectThrough(const Mat& P, double X, double Y, double Z)
{
double u = P.at<double>(0,0)*X + P.at<double>(0,1)*Y + P.at<double>(0,2)*Z + P.at<double>(0,3);
double v = P.at<double>(1,0)*X + P.at<double>(1,1)*Y + P.at<double>(1,2)*Z + P.at<double>(1,3);
double w = P.at<double>(2,0)*X + P.at<double>(2,1)*Y + P.at<double>(2,2)*Z + P.at<double>(2,3);
if (std::abs(w) < 1e-12) return Point2d(0, 0);
return Point2d(u / w, v / w);
}
inline double cameraDepth(const Matx44d& T_cw, double X, double Y, double Z)
{
return T_cw(2,0)*X + T_cw(2,1)*Y + T_cw(2,2)*Z + T_cw(2,3);
}
} // anonymous namespace
void VisualOdometryImpl::promoteKeyframeAndGrowMap(Frame& cur)
{
KeyFrame* newKf = new KeyFrame();
newKf->poseCw = cur.poseCw;
newKf->keypoints = cur.keypoints;
newKf->descriptors = cur.descriptors.clone();
newKf->undistKpts = cur.undistKpts;
newKf->imageSize = cur.imageSize;
newKf->mapPoints.assign(cur.keypoints.size(), nullptr);
newKf->parent = lastKf;
map.addKeyframe(newKf);
for (size_t i = 0; i < cur.mapPoints.size(); ++i)
{
MapPoint* mp = cur.mapPoints[i];
if (!mp || mp->bad || cur.outliers[i]) continue;
map.addObservation(newKf, i, mp);
}
std::vector<DMatch> kfToCur;
matchFrames(lastKf->keypoints, lastKf->descriptors, lastKf->imageSize,
cur.keypoints, cur.descriptors, cur.imageSize, kfToCur);
Mat Rt1(3, 4, CV_64F), Rt2(3, 4, CV_64F);
for (int i = 0; i < 3; ++i)
for (int j = 0; j < 4; ++j)
{
Rt1.at<double>(i,j) = lastKf->poseCw(i,j);
Rt2.at<double>(i,j) = cur.poseCw(i,j);
}
Mat P1 = K * Rt1;
Mat P2 = K * Rt2;
std::vector<Point2f> pts1, pts2;
std::vector<int> triMatchIdx;
for (size_t i = 0; i < kfToCur.size(); ++i)
{
const DMatch& m = kfToCur[i];
if ((size_t)m.queryIdx >= lastKf->mapPoints.size()) continue;
if (lastKf->mapPoints[m.queryIdx] != nullptr) continue;
if ((size_t)m.trainIdx >= newKf->mapPoints.size()) continue;
if (newKf->mapPoints[m.trainIdx] != nullptr) continue;
pts1.push_back(lastKf->undistKpts[m.queryIdx]);
pts2.push_back(cur.undistKpts[m.trainIdx]);
triMatchIdx.push_back((int)i);
}
if (pts1.empty())
{
detail::updateCovisibility(newKf);
map.setCurrentKeyframe(newKf);
lastKf = newKf;
lastKfInliers = 0;
framesSinceKf = 0;
return;
}
Mat pts4D;
triangulatePoints(P1, P2, pts1, pts2, pts4D);
int nNew = 0;
for (int i = 0; i < pts4D.cols; ++i)
{
double w = pts4D.at<float>(3, i);
if (std::abs(w) < 1e-9) continue;
double X = pts4D.at<float>(0, i) / w;
double Y = pts4D.at<float>(1, i) / w;
double Z = pts4D.at<float>(2, i) / w;
if (cameraDepth(lastKf->poseCw, X, Y, Z) <= 0) continue;
if (cameraDepth(cur.poseCw, X, Y, Z) <= 0) continue;
Point2d p1p = projectThrough(P1, X, Y, Z);
Point2d p2p = projectThrough(P2, X, Y, Z);
double e1 = std::hypot(p1p.x - pts1[i].x, p1p.y - pts1[i].y);
double e2 = std::hypot(p2p.x - pts2[i].x, p2p.y - pts2[i].y);
if (e1 > params.pnpReprojThresh || e2 > params.pnpReprojThresh) continue;
Point3d Xw(X, Y, Z);
if (detail::parallaxDeg(Xw, lastKf->poseCw, cur.poseCw)
< params.minGrowthParallaxDeg) continue;
MapPoint* mp = new MapPoint();
mp->pos = Xw;
const DMatch& dm = kfToCur[triMatchIdx[i]];
mp->refDesc = cur.descriptors.row(dm.trainIdx).clone();
map.addMapPoint(mp);
map.addObservation(lastKf, (size_t)dm.queryIdx, mp);
map.addObservation(newKf, (size_t)dm.trainIdx, mp);
++nNew;
}
detail::updateCovisibility(newKf);
map.setCurrentKeyframe(newKf);
lastKf = newKf;
lastKfInliers = 0;
framesSinceKf = 0;
(void)nNew;
}
}} // namespace cv::slam
@@ -0,0 +1,401 @@
// This file is part of OpenCV project.
// It is subject to the license terms in the LICENSE file found in the top-level directory
// of this distribution and at http://opencv.org/license.html.
// Copyright (C) 2026, BigVision LLC, all rights reserved.
// Third party copyrights are property of their respective owners.
#include "../precomp.hpp"
#include "vo_impl.hpp"
namespace cv {
namespace slam {
namespace {
bool projectPoint(const Matx44d& T_cw, const Mat& K,
double Xw, double Yw, double Zw,
double& u, double& v)
{
const double Xc = T_cw(0,0)*Xw + T_cw(0,1)*Yw + T_cw(0,2)*Zw + T_cw(0,3);
const double Yc = T_cw(1,0)*Xw + T_cw(1,1)*Yw + T_cw(1,2)*Zw + T_cw(1,3);
const double Zc = T_cw(2,0)*Xw + T_cw(2,1)*Yw + T_cw(2,2)*Zw + T_cw(2,3);
if (Zc <= 0.0) return false;
u = K.at<double>(0,0) * Xc / Zc + K.at<double>(0,2);
v = K.at<double>(1,1) * Yc / Zc + K.at<double>(1,2);
return true;
}
inline double descDist(const Mat& a, const Mat& b)
{
if (a.empty() || b.empty()) return std::numeric_limits<double>::max();
int normType = (a.type() == CV_8U) ? NORM_HAMMING : NORM_L2;
return norm(a, b, normType);
}
void buildLocalMapPoints(const KeyFrame* kf, int topK,
std::set<MapPoint*>& localMps)
{
if (!kf) return;
for (MapPoint* mp : kf->mapPoints)
if (mp && !mp->bad) localMps.insert(mp);
int k = 0;
for (const auto& [nbKf, cnt] : kf->orderedCovisibility)
{
if (k++ >= topK) break;
for (MapPoint* mp : nbKf->mapPoints)
if (mp && !mp->bad) localMps.insert(mp);
}
}
int runPnP(const std::vector<Point3f>& obj, const std::vector<Point2f>& img,
Frame& frame,
const Mat& K, double reprojThresh, int maxIters, double confidence,
int minInliers)
{
if ((int)obj.size() < minInliers) return -1;
Mat rvec, tvec, inlierIdx;
bool ok = solvePnPRansac(obj, img, K, Mat(),
rvec, tvec, false,
maxIters, (float)reprojThresh, confidence,
inlierIdx, SOLVEPNP_AP3P);
if (!ok || inlierIdx.rows < minInliers) return -1;
std::vector<Point3f> objIn;
std::vector<Point2f> imgIn;
objIn.reserve(inlierIdx.rows); imgIn.reserve(inlierIdx.rows);
for (int i = 0; i < inlierIdx.rows; ++i)
{
int idx = inlierIdx.at<int>(i);
objIn.push_back(obj[idx]);
imgIn.push_back(img[idx]);
}
solvePnPRefineLM(objIn, imgIn, K, Mat(), rvec, tvec);
Mat R;
Rodrigues(rvec, R);
frame.poseCw = detail::makePose(R, tvec);
return inlierIdx.rows;
}
} // anonymous namespace
// Motion model: constant-velocity pose prediction + projected map point search
bool VisualOdometryImpl::trackWithMotionModel(Frame& cur)
{
if (!lastKf) return false;
cur.poseCw = velocity * lastPoseCw;
std::set<MapPoint*> localMps;
buildLocalMapPoints(lastKf, params.localMapTopK, localMps);
std::fill(cur.mapPoints.begin(), cur.mapPoints.end(), nullptr);
std::fill(cur.outliers.begin(), cur.outliers.end(), false);
auto doSearch = [&](float radius) -> int {
int n = 0;
for (MapPoint* mp : localMps)
{
double u, v;
if (!projectPoint(cur.poseCw, K, mp->pos.x, mp->pos.y, mp->pos.z, u, v))
continue;
if (u < 0 || u >= cur.imageSize.width ||
v < 0 || v >= cur.imageSize.height) continue;
mp->visibleCount++;
auto cands = cur.getKeypointsInRadius((float)u, (float)v, radius);
double bestD = std::numeric_limits<double>::max();
size_t bestI = std::numeric_limits<size_t>::max();
for (size_t idx : cands)
{
if (cur.mapPoints[idx]) continue;
double d = descDist(mp->refDesc, cur.descriptors.row((int)idx));
if (d < bestD) { bestD = d; bestI = idx; }
}
if (bestI != std::numeric_limits<size_t>::max() &&
bestD < params.descProjThresh)
{
cur.mapPoints[bestI] = mp;
mp->foundCount++;
++n;
}
}
return n;
};
int n = doSearch((float)params.motionModelRadius);
if (n < params.motionModelMinMatches)
{
std::fill(cur.mapPoints.begin(), cur.mapPoints.end(), nullptr);
n = doSearch((float)params.motionModelRadiusWide);
}
if (n < params.pnpMinInliers) return false;
std::vector<Point3f> obj; std::vector<Point2f> img;
obj.reserve(n); img.reserve(n);
for (size_t i = 0; i < cur.mapPoints.size(); ++i)
{
if (!cur.mapPoints[i]) continue;
const MapPoint* mp = cur.mapPoints[i];
obj.push_back(Point3f((float)mp->pos.x,(float)mp->pos.y,(float)mp->pos.z));
img.push_back(cur.undistKpts[i]);
}
int nInliers = runPnP(obj, img, cur, K,
params.pnpReprojThresh,
params.pnpRansacIters,
params.pnpConfidence,
params.pnpMinInliers);
if (nInliers < 0) return false;
int nOpt = poseInlierCheck(cur, K, params.pnpReprojThresh);
return nOpt >= params.pnpMinInliers;
}
// Fallback 1: descriptor match against the reference keyframe
bool VisualOdometryImpl::trackWithReferenceKF(Frame& cur)
{
if (!lastKf) return false;
std::fill(cur.mapPoints.begin(), cur.mapPoints.end(), nullptr);
std::fill(cur.outliers.begin(), cur.outliers.end(), false);
std::vector<DMatch> matches;
matchFrames(lastKf->keypoints, lastKf->descriptors, lastKf->imageSize,
cur.keypoints, cur.descriptors, cur.imageSize, matches);
std::vector<Point3f> obj;
std::vector<Point2f> img;
std::vector<MapPoint*> corrMps;
std::vector<int> corrKp;
obj.reserve(matches.size()); img.reserve(matches.size());
corrMps.reserve(matches.size()); corrKp.reserve(matches.size());
for (const auto& m : matches)
{
if ((size_t)m.queryIdx >= lastKf->mapPoints.size()) continue;
MapPoint* mp = lastKf->mapPoints[m.queryIdx];
if (!mp || mp->bad) continue;
obj.push_back(Point3f((float)mp->pos.x,(float)mp->pos.y,(float)mp->pos.z));
img.push_back(cur.undistKpts[m.trainIdx]);
corrMps.push_back(mp);
corrKp.push_back(m.trainIdx);
}
if ((int)obj.size() < params.pnpMinInliers)
{
lastEvent = format("refKF: 2d3d=%d < %d", (int)obj.size(), params.pnpMinInliers);
return false;
}
int nInliers = runPnP(obj, img, cur, K,
params.pnpReprojThresh, params.pnpRansacIters,
params.pnpConfidence, params.pnpMinInliers);
if (nInliers < 0)
{
lastEvent = "refKF: PnP failed";
return false;
}
for (size_t k = 0; k < corrMps.size(); ++k)
{
int kpIdx = corrKp[k];
if ((size_t)kpIdx < cur.mapPoints.size() && !cur.mapPoints[kpIdx])
cur.mapPoints[kpIdx] = corrMps[k];
}
int nOpt = poseInlierCheck(cur, K, params.pnpReprojThresh);
return nOpt >= params.pnpMinInliers;
}
// Fallback 2: optical flow when descriptor match also fails
bool VisualOdometryImpl::trackWithOpticalFlow(Frame& cur)
{
if (!hasPrevFrame || prevFrame.image.empty()) return false;
const Frame& prev = prevFrame;
if (prev.keypoints.empty()) return false;
// LK runs on the raw imgs
std::vector<Point2f> prevPts;
prevPts.reserve(prev.keypoints.size());
for (const auto& kp : prev.keypoints) prevPts.push_back(kp.pt);
std::vector<Point2f> curPts;
std::vector<uchar> status;
std::vector<float> err;
calcOpticalFlowPyrLK(prev.image, cur.image, prevPts, curPts,
status, err, Size(21, 21), 3,
TermCriteria(TermCriteria::COUNT | TermCriteria::EPS, 30, 0.01));
// PnP uses K with no distortion
std::vector<Point2f> curUndist;
if (!dist.empty())
undistortPoints(curPts, curUndist, K, dist, noArray(), K);
else
curUndist = curPts;
std::vector<Point3f> obj; std::vector<Point2f> img;
obj.reserve(prevPts.size()); img.reserve(prevPts.size());
for (size_t i = 0; i < prevPts.size(); ++i)
{
if (!status[i]) continue;
if (i >= prev.mapPoints.size()) continue;
MapPoint* mp = prev.mapPoints[i];
if (!mp || mp->bad) continue;
obj.push_back(Point3f((float)mp->pos.x,(float)mp->pos.y,(float)mp->pos.z));
img.push_back(curUndist[i]);
}
if ((int)obj.size() < params.opticalFlowMinInliers)
{
lastEvent = format("optflow: corr=%d < %d", (int)obj.size(), params.opticalFlowMinInliers);
return false;
}
int nInliers = runPnP(obj, img, cur, K,
params.pnpReprojThresh, params.pnpRansacIters,
params.pnpConfidence, params.opticalFlowMinInliers);
if (nInliers < 0)
{
lastEvent = "optflow: PnP failed";
return false;
}
return true;
}
// Local map refinement: expand coverage + recheck inliers
void VisualOdometryImpl::trackLocalMap(Frame& cur)
{
if (!lastKf) return;
std::set<MapPoint*> localMps;
buildLocalMapPoints(lastKf, params.localMapTopK, localMps);
int nbK = 0;
for (const auto& [nbKf, cnt] : lastKf->orderedCovisibility)
{
if (nbK++ >= params.localMapTopK) break;
int nb2K = 0;
for (const auto& [nb2Kf, cnt2] : nbKf->orderedCovisibility)
{
if (nb2K++ >= params.localMapNeighborK) break;
for (MapPoint* mp : nb2Kf->mapPoints)
if (mp && !mp->bad) localMps.insert(mp);
}
}
std::set<MapPoint*> alreadyMatched;
for (MapPoint* mp : cur.mapPoints)
if (mp) alreadyMatched.insert(mp);
bool anyNew = false;
const float r = (float)params.localMapRadius;
for (MapPoint* mp : localMps)
{
if (alreadyMatched.count(mp)) continue;
double u, v;
if (!projectPoint(cur.poseCw, K, mp->pos.x, mp->pos.y, mp->pos.z, u, v))
continue;
if (u < 0 || u >= cur.imageSize.width ||
v < 0 || v >= cur.imageSize.height) continue;
mp->visibleCount++;
auto cands = cur.getKeypointsInRadius((float)u, (float)v, r);
double bestD = std::numeric_limits<double>::max();
size_t bestI = std::numeric_limits<size_t>::max();
for (size_t idx : cands)
{
if (cur.mapPoints[idx]) continue;
double d = descDist(mp->refDesc, cur.descriptors.row((int)idx));
if (d < bestD) { bestD = d; bestI = idx; }
}
if (bestI != std::numeric_limits<size_t>::max() &&
bestD < params.descProjThresh)
{
cur.mapPoints[bestI] = mp;
cur.outliers[bestI] = false;
mp->foundCount++;
alreadyMatched.insert(mp);
anyNew = true;
}
}
if (anyNew)
poseInlierCheck(cur, K, params.pnpReprojThresh);
}
bool VisualOdometryImpl::track(Frame& cur)
{
if (!lastKf)
{
refFrame = cur;
state = INITIALIZING;
return false;
}
// motion model tracking
bool ok = false;
if (hasVelocity)
ok = trackWithMotionModel(cur);
// fallback 1: descriptor match against reference keyframe
if (!ok)
ok = trackWithReferenceKF(cur);
// fallback 2: optical flow
if (!ok)
ok = trackWithOpticalFlow(cur);
if (!ok)
{
lastEvent = "track lost: all stages failed";
refFrame = cur;
state = INITIALIZING;
return false;
}
trackLocalMap(cur);
int nInliers = 0;
for (size_t i = 0; i < cur.mapPoints.size(); ++i)
if (cur.mapPoints[i] && !cur.outliers[i]) ++nInliers;
{
Matx44d Tcw_last_inv = lastPoseCw.inv();
velocity = cur.poseCw * Tcw_last_inv;
hasVelocity = true;
}
lastPoseCw = cur.poseCw;
map.appendPose(cur.poseCw);
++framesSinceKf;
prevFrame = cur;
hasPrevFrame = true;
String kf_reason;
if (shouldPromoteKeyframe(nInliers, cur.poseCw, kf_reason))
{
int mpBefore = map.numMapPoints();
promoteKeyframeAndGrowMap(cur);
lastPoseCw = lastKf->poseCw;
lastEvent = format("keyframe: %s, +%d mp",
kf_reason.c_str(),
map.numMapPoints() - mpBefore);
}
return true;
}
}} // namespace cv::slam
@@ -0,0 +1,49 @@
// This file is part of OpenCV project.
// It is subject to the license terms in the LICENSE file found in the top-level directory
// of this distribution and at http://opencv.org/license.html.
// Copyright (C) 2026, BigVision LLC, all rights reserved.
// Third party copyrights are property of their respective owners.
#include "../precomp.hpp"
#include "pose_optimizer.hpp"
namespace cv { namespace slam {
// pose is not optimized yet, only inlier classification is done.
// adding bundle adjustment based pose optimization in a later commit.
int poseInlierCheck(Frame& frame, const Mat& K, double reprojThresh)
{
const double fx = K.at<double>(0, 0);
const double fy = K.at<double>(1, 1);
const double cx = K.at<double>(0, 2);
const double cy = K.at<double>(1, 2);
const Matx44d& T = frame.poseCw;
int nInliers = 0;
for (size_t i = 0; i < frame.mapPoints.size(); ++i)
{
frame.outliers[i] = true;
MapPoint* mp = frame.mapPoints[i];
if (!mp || mp->bad) continue;
const double Xc = T(0,0)*mp->pos.x + T(0,1)*mp->pos.y + T(0,2)*mp->pos.z + T(0,3);
const double Yc = T(1,0)*mp->pos.x + T(1,1)*mp->pos.y + T(1,2)*mp->pos.z + T(1,3);
const double Zc = T(2,0)*mp->pos.x + T(2,1)*mp->pos.y + T(2,2)*mp->pos.z + T(2,3);
if (Zc <= 0.0) continue;
const double u = fx * Xc / Zc + cx;
const double v = fy * Yc / Zc + cy;
const double dx = u - static_cast<double>(frame.undistKpts[i].x);
const double dy = v - static_cast<double>(frame.undistKpts[i].y);
if (std::sqrt(dx * dx + dy * dy) <= reprojThresh)
{
frame.outliers[i] = false;
++nInliers;
}
}
return nInliers;
}
}} // namespace cv::slam
@@ -0,0 +1,21 @@
// This file is part of OpenCV project.
// It is subject to the license terms in the LICENSE file found in the top-level directory
// of this distribution and at http://opencv.org/license.html.
// Copyright (C) 2026, BigVision LLC, all rights reserved.
// Third party copyrights are property of their respective owners.
#ifndef OPENCV_SLAM_POSE_OPTIMIZER_HPP
#define OPENCV_SLAM_POSE_OPTIMIZER_HPP
#include "../odometry/frame.hpp"
namespace cv {
namespace slam {
// Classify map point associations as inlier/outlier by reprojection distance.
// Returns inlier count. frame.poseCw is NOT modified.
int poseInlierCheck(Frame& frame, const Mat& K, double reprojThresh);
}} // namespace cv::slam
#endif // OPENCV_SLAM_POSE_OPTIMIZER_HPP
+29
View File
@@ -0,0 +1,29 @@
// This file is part of OpenCV project.
// It is subject to the license terms in the LICENSE file found in the top-level directory
// of this distribution and at http://opencv.org/license.html.
// Copyright (C) 2026, BigVision LLC, all rights reserved.
// Third party copyrights are property of their respective owners.
#ifndef OPENCV_SLAM_PRECOMP_HPP
#define OPENCV_SLAM_PRECOMP_HPP
#include "opencv2/ptcloud/slam.hpp"
#include "opencv2/core.hpp"
#include "opencv2/core/private.hpp"
#include "opencv2/imgproc.hpp"
#include "opencv2/features.hpp"
#include "opencv2/geometry.hpp"
#include "opencv2/video/tracking.hpp"
#include <algorithm>
#include <cmath>
#include <cstdint>
#include <limits>
#include <map>
#include <set>
#include <unordered_map>
#include <unordered_set>
#include <vector>
#endif // OPENCV_SLAM_PRECOMP_HPP
+79
View File
@@ -0,0 +1,79 @@
// This file is part of OpenCV project.
// It is subject to the license terms in the LICENSE file found in the top-level directory
// of this distribution and at http://opencv.org/license.html.
// Copyright (C) 2026, BigVision LLC, all rights reserved.
// Third party copyrights are property of their respective owners.
#include "test_precomp.hpp"
namespace opencv_test { namespace {
// Allocates a keyframe with @p n keypoint slots (parallel mapPoints[] zeroed),
// as the pipeline expects before any observation is wired.
static slam::KeyFrame* newKeyFrame(int n)
{
slam::KeyFrame* kf = new slam::KeyFrame();
kf->keypoints.resize(n);
kf->mapPoints.assign(n, nullptr);
return kf;
}
TEST(SLAM_Map, registers_and_clears)
{
slam::Map map;
// addKeyframe / addMapPoint assign sequential ids and the index round-trips.
slam::KeyFrame* kf0 = map.addKeyframe(newKeyFrame(0));
slam::KeyFrame* kf1 = map.addKeyframe(newKeyFrame(0));
slam::MapPoint* mp0 = map.addMapPoint(new slam::MapPoint());
EXPECT_EQ(kf0->id, 0);
EXPECT_EQ(kf1->id, 1);
EXPECT_EQ(mp0->id, 0);
EXPECT_EQ(map.numKeyframes(), 2);
EXPECT_EQ(map.numMapPoints(), 1);
EXPECT_EQ(map.getKeyframe(1), kf1);
EXPECT_EQ(map.getMapPoint(0), mp0);
EXPECT_TRUE(map.getKeyframe(99) == nullptr);
// An explicit id must advance the auto-id counter so it cannot collide.
slam::KeyFrame* kfExplicit = new slam::KeyFrame();
kfExplicit->id = 5;
map.addKeyframe(kfExplicit);
EXPECT_EQ(map.addKeyframe(newKeyFrame(0))->id, 6);
// clear() drops all state and restarts the id counters.
map.clear();
EXPECT_EQ(map.numKeyframes(), 0);
EXPECT_EQ(map.numMapPoints(), 0);
EXPECT_EQ(map.addKeyframe(newKeyFrame(0))->id, 0);
}
TEST(SLAM_Map, wires_and_removes_observations)
{
slam::Map map;
slam::KeyFrame* kfA = map.addKeyframe(newKeyFrame(3));
slam::KeyFrame* kfB = map.addKeyframe(newKeyFrame(3));
slam::MapPoint* mp = map.addMapPoint(new slam::MapPoint());
// addObservation links both directions: keyframe slot <-> observation map.
map.addObservation(kfA, 2, mp);
map.addObservation(kfB, 0, mp);
EXPECT_EQ(kfA->mapPoints[2], mp);
EXPECT_EQ(mp->observations[kfA], 2u);
EXPECT_EQ(mp->observations.size(), 2u);
// Adding into an already-occupied slot is a no-op.
slam::MapPoint* other = map.addMapPoint(new slam::MapPoint());
map.addObservation(kfA, 2, other);
EXPECT_EQ(kfA->mapPoints[2], mp);
EXPECT_EQ(other->observations.count(kfA), 0u);
// removeMapPoint unlinks every observing keyframe and drops the point.
const int mpId = mp->id;
map.removeMapPoint(mp); // deletes mp; must not be dereferenced afterwards
EXPECT_TRUE(map.getMapPoint(mpId) == nullptr);
EXPECT_TRUE(kfA->mapPoints[2] == nullptr);
EXPECT_TRUE(kfB->mapPoints[0] == nullptr);
}
}} // namespace opencv_test
+2
View File
@@ -13,6 +13,8 @@
#include <opencv2/core/utils/logger.hpp>
#include "opencv2/ptcloud/depth.hpp"
#include "opencv2/ptcloud/odometry.hpp"
#include "opencv2/ptcloud/slam.hpp"
#include "opencv2/features.hpp"
#ifdef HAVE_OPENCL
#include <opencv2/core/ocl.hpp>
@@ -0,0 +1,251 @@
// This file is part of OpenCV project.
// It is subject to the license terms in the LICENSE file found in the top-level directory
// of this distribution and at http://opencv.org/license.html.
// Copyright (C) 2026, BigVision LLC, all rights reserved.
// Third party copyrights are property of their respective owners.
#include "test_precomp.hpp"
namespace opencv_test { namespace {
// The slam module has no committed image data, so - as in the geometry module's
// tests - the pipeline is exercised on a synthetic scene: a fixed 3D point cloud
// (generated with theRNG()) projected through a pin-hole camera that translates
// along +X. A stub Feature2D replays the projected keypoints frame by frame and
// a brute-force matcher recovers the ground-truth correspondences, so
// VisualOdometry runs end-to-end without a real detector or image files.
const int descDim = 8;
const int cloudSize = 400;
const Size imageSize(640, 480);
// Camera centres along +X: a wide first baseline for good bootstrap parallax,
// then small uniform steps suited to per-frame tracking.
static const double camCenters[] = { 0.0, 0.8, 1.1, 1.4, 1.7, 2.0 };
static Matx33d cameraMatrix()
{
return Matx33d(500, 0, 320,
0, 500, 240,
0, 0, 1);
}
// Each landmark gets a globally-unique, view-invariant descriptor: identical for
// the same 3D point in every frame (so the matcher pairs them at distance 0) and
// >= sqrt(descDim) apart for distinct points (well above descProjThresh, so
// the projection search during tracking never mismatches).
static Mat makeDescriptor(int id)
{
return Mat(1, descDim, CV_32F, Scalar((double)(id + 1)));
}
// Stub Feature2D: replays pre-computed keypoints/descriptors, one frame per
// detectAndCompute() call (processFrame() invokes the detector exactly once).
class StubDetector CV_FINAL : public Feature2D
{
public:
struct FrameFeatures { std::vector<KeyPoint> keypoints; Mat descriptors; };
std::vector<FrameFeatures> frames;
size_t next = 0;
void detectAndCompute(InputArray, InputArray,
std::vector<KeyPoint>& keypoints,
OutputArray descriptors, bool) CV_OVERRIDE
{
CV_Assert(next < frames.size());
keypoints = frames[next].keypoints;
frames[next].descriptors.copyTo(descriptors);
++next;
}
};
static std::vector<Point3f> makeCloud()
{
RNG& rng = theRNG(); // seeded per-test by the ts framework -> reproducible
std::vector<Point3f> pts(cloudSize);
for (int i = 0; i < cloudSize; i++)
pts[i] = Point3f(rng.uniform(-2.5f, 2.5f),
rng.uniform(-1.8f, 1.8f),
rng.uniform( 4.0f, 9.0f)); // varied depth -> non-planar
return pts;
}
// Projects the cloud through the camera at centre (camX, 0, 0), keeping the
// points that land inside the image. Ground-truth pose is pure translation
// (R = I), so projectPoints takes a zero rotation vector.
static StubDetector::FrameFeatures renderFrame(const std::vector<Point3f>& cloud, double camX)
{
Vec3d rvec(0, 0, 0), tvec(-camX, 0, 0); // t = -R*C, with R = I, C = (camX,0,0)
std::vector<Point2f> proj;
projectPoints(cloud, rvec, tvec, cameraMatrix(), noArray(), proj);
StubDetector::FrameFeatures ff;
std::vector<int> ids;
for (int i = 0; i < (int)cloud.size(); i++)
{
const Point2f& p = proj[i];
if (p.x < 0 || p.x >= imageSize.width ||
p.y < 0 || p.y >= imageSize.height) continue;
ff.keypoints.push_back(KeyPoint(p, 7.f));
ids.push_back(i);
}
ff.descriptors.create((int)ids.size(), descDim, CV_32F);
for (int r = 0; r < (int)ids.size(); r++)
makeDescriptor(ids[r]).copyTo(ff.descriptors.row(r));
return ff;
}
// Builds a VisualOdometry fed by a stub detector pre-loaded with @p nFrames of
// the synthetic sequence and a ground-truth brute-force matcher.
static Ptr<slam::VisualOdometry> makeOdometry(int nFrames,
const slam::OdometryParams& params = slam::OdometryParams())
{
std::vector<Point3f> cloud = makeCloud();
Ptr<StubDetector> detector = makePtr<StubDetector>();
for (int f = 0; f < nFrames; f++)
detector->frames.push_back(renderFrame(cloud, camCenters[f]));
Ptr<DescriptorMatcher> matcher = BFMatcher::create(NORM_L2, true);
return slam::VisualOdometry::create(detector, matcher,
Mat(cameraMatrix()), noArray(), params);
}
static Mat blankImage() { return Mat::zeros(imageSize, CV_8UC1); }
// Rotation magnitude (deg) of @p T's rotation block relative to identity.
static double rotationFromIdentityDeg(const Matx44d& T)
{
const double trace = T(0,0) + T(1,1) + T(2,2);
const double c = std::max(-1.0, std::min(1.0, (trace - 1.0) * 0.5));
return std::acos(c) * 180.0 / CV_PI;
}
// Camera centre in world coordinates: C = -R^T t.
static Point3d cameraCenter(const Matx44d& T)
{
Matx33d R(T(0,0),T(0,1),T(0,2), T(1,0),T(1,1),T(1,2), T(2,0),T(2,1),T(2,2));
Matx31d t(T(0,3), T(1,3), T(2,3));
Matx31d C = -R.t() * t;
return Point3d(C(0), C(1), C(2));
}
TEST(SLAM_VisualOdometry, create_validates_arguments)
{
Ptr<Feature2D> detector = makePtr<StubDetector>();
Ptr<DescriptorMatcher> matcher = BFMatcher::create(NORM_L2, true);
Mat K(cameraMatrix());
EXPECT_THROW(slam::VisualOdometry::create(Ptr<Feature2D>(), matcher, K),
cv::Exception); // null detector
EXPECT_THROW(slam::VisualOdometry::create(detector, Ptr<DescriptorMatcher>(), K),
cv::Exception); // null matcher
EXPECT_THROW(slam::VisualOdometry::create(detector, matcher, Mat()),
cv::Exception); // empty intrinsics
EXPECT_THROW(slam::VisualOdometry::create(detector, matcher, Mat::eye(2, 2, CV_64F)),
cv::Exception); // wrong-size intrinsics
}
TEST(SLAM_VisualOdometry, bootstrap_initializes_map)
{
Ptr<slam::VisualOdometry> vo = makeOdometry(2);
Mat image = blankImage();
EXPECT_FALSE(vo->processFrame(image)); // frame 0 -> INITIALIZING
EXPECT_EQ(vo->getState(), slam::INITIALIZING);
EXPECT_TRUE(vo->processFrame(image)); // frame 1 -> TRACKING
EXPECT_EQ(vo->getState(), slam::TRACKING);
const slam::Map& map = vo->getMap();
EXPECT_EQ(map.numKeyframes(), 2);
EXPECT_GE(map.numMapPoints(), 100); // OdometryParams::minInitPoints
EXPECT_EQ(vo->getTrajectory().size(), 2u);
// Reference keyframe is pinned to the world origin.
const slam::KeyFrame* kfRef = map.getKeyframe(0);
ASSERT_TRUE(kfRef != nullptr);
EXPECT_LT(cv::norm(kfRef->poseCw - Matx44d::eye()), 1e-12);
// Median scene depth is normalized to 1.
std::vector<double> depths;
depths.reserve(map.mapPoints().size());
for (slam::MapPoint* mp : map.mapPoints())
depths.push_back(mp->pos.z);
ASSERT_FALSE(depths.empty());
std::nth_element(depths.begin(), depths.begin() + depths.size() / 2, depths.end());
EXPECT_NEAR(depths[depths.size() / 2], 1.0, 1e-2);
// Second camera: ~no rotation; translation along +X up to scale.
const slam::KeyFrame* kfCur = map.getKeyframe(1);
ASSERT_TRUE(kfCur != nullptr);
EXPECT_LT(rotationFromIdentityDeg(kfCur->poseCw), 2.0);
Point3d C = cameraCenter(kfCur->poseCw);
EXPECT_GT(C.x, 0.0);
EXPECT_LT(std::abs(C.y), 0.2 * std::abs(C.x));
EXPECT_LT(std::abs(C.z), 0.2 * std::abs(C.x));
}
TEST(SLAM_VisualOdometry, tracks_after_bootstrap)
{
Ptr<slam::VisualOdometry> vo = makeOdometry(3);
Mat image = blankImage();
ASSERT_FALSE(vo->processFrame(image));
ASSERT_TRUE(vo->processFrame(image));
ASSERT_EQ(vo->getState(), slam::TRACKING);
const Point3d cBootstrap = cameraCenter(vo->getLastPose());
EXPECT_TRUE(vo->processFrame(image)); // frame 2 -> tracked by PnP
EXPECT_EQ(vo->getState(), slam::TRACKING);
EXPECT_EQ(vo->getTrajectory().size(), 3u);
const Matx44d pose = vo->getLastPose();
EXPECT_LT(rotationFromIdentityDeg(pose), 2.0);
const Point3d C = cameraCenter(pose);
EXPECT_GT(C.x, cBootstrap.x); // camera keeps advancing +X
EXPECT_LT(std::abs(C.y), 0.2 * std::abs(C.x));
EXPECT_LT(std::abs(C.z), 0.2 * std::abs(C.x));
}
TEST(SLAM_VisualOdometry, promotes_keyframes_during_tracking)
{
slam::OdometryParams params;
params.kfMaxFrames = 1; // force a keyframe promotion early in tracking
Ptr<slam::VisualOdometry> vo = makeOdometry(6, params);
Mat image = blankImage();
ASSERT_FALSE(vo->processFrame(image)); // -> INITIALIZING
ASSERT_TRUE(vo->processFrame(image)); // -> TRACKING (2 keyframes)
ASSERT_EQ(vo->getMap().numKeyframes(), 2);
for (int f = 2; f < 6; f++)
EXPECT_TRUE(vo->processFrame(image));
EXPECT_EQ(vo->getState(), slam::TRACKING);
EXPECT_GT(vo->getMap().numKeyframes(), 2); // new keyframes were promoted
const slam::KeyFrame* current = vo->getMap().getCurrentKeyframe();
ASSERT_TRUE(current != nullptr);
EXPECT_GT(current->id, 1); // current keyframe advanced
}
TEST(SLAM_VisualOdometry, reset_clears_state)
{
Ptr<slam::VisualOdometry> vo = makeOdometry(2);
Mat image = blankImage();
vo->processFrame(image);
vo->processFrame(image);
ASSERT_EQ(vo->getState(), slam::TRACKING);
ASSERT_GT(vo->getMap().numKeyframes(), 0);
vo->reset();
EXPECT_EQ(vo->getState(), slam::NOT_INITIALIZED);
EXPECT_EQ(vo->getMap().numKeyframes(), 0);
EXPECT_EQ(vo->getMap().numMapPoints(), 0);
EXPECT_TRUE(vo->getTrajectory().empty());
EXPECT_LT(cv::norm(vo->getLastPose() - Matx44d::eye()), 1e-12);
}
}} // namespace opencv_test
+2
View File
@@ -23,6 +23,7 @@ endif()
add_subdirectory(cpp)
add_subdirectory(java/tutorial_code)
add_subdirectory(dnn)
add_subdirectory(slam)
add_subdirectory(gpu)
add_subdirectory(tapi)
# HACK: CMake 4.x finds and links wrong OpenCL in 32-bit builds on Windows x64
@@ -127,6 +128,7 @@ if(WIN32)
add_subdirectory(directx)
endif()
add_subdirectory(dnn)
add_subdirectory(slam)
# add_subdirectory(gpu)
# HACK: CMake 4.x finds and links wrong OpenCL in 32-bit builds on Windows x64
if(NOT (WIN32 AND CMAKE_SIZEOF_VOID_P EQUAL 4))
+23
View File
@@ -0,0 +1,23 @@
ocv_install_example_src(slam *.cpp *.hpp CMakeLists.txt)
set(OPENCV_SLAM_SAMPLES_REQUIRED_DEPS
opencv_core
opencv_dnn
opencv_features
opencv_imgcodecs
opencv_ptcloud)
ocv_check_dependencies(${OPENCV_SLAM_SAMPLES_REQUIRED_DEPS})
if(NOT BUILD_EXAMPLES OR NOT OCV_DEPENDENCIES_FOUND)
return()
endif()
project(slam_samples)
ocv_include_modules_recurse(${OPENCV_SLAM_SAMPLES_REQUIRED_DEPS})
include_directories("${CMAKE_CURRENT_SOURCE_DIR}/../dnn")
file(GLOB_RECURSE slam_samples RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} *.cpp)
foreach(sample_filename ${slam_samples})
ocv_define_sample(tgt ${sample_filename} slam)
ocv_target_link_libraries(${tgt} PRIVATE ${OPENCV_LINKER_LIBS} ${OPENCV_SLAM_SAMPLES_REQUIRED_DEPS})
endforeach()
+320
View File
@@ -0,0 +1,320 @@
// This file is part of OpenCV project.
// It is subject to the license terms in the LICENSE file found in the top-level directory
// of this distribution and at http://opencv.org/license.html.
// Copyright (C) 2026, BigVision LLC, all rights reserved.
#include <opencv2/ptcloud.hpp>
#include <opencv2/features.hpp>
#include <opencv2/core.hpp>
#include <opencv2/core/quaternion.hpp>
#include <opencv2/imgcodecs.hpp>
#include <algorithm>
#include <fstream>
#include <iomanip>
#include <iostream>
#include <sstream>
#include "../dnn/common.hpp"
using namespace cv;
using namespace std;
namespace {
// path helper: strips the directory part off a path
String basenameOf(const String& path)
{
const size_t slash = path.find_last_of("/\\");
return (slash == String::npos) ? path : path.substr(slash + 1);
}
// reader: lists readable image files under a directory, sorted by filename
std::vector<String> listImageFiles(const String& imagesDir)
{
std::vector<String> allFiles;
glob(imagesDir, allFiles, false);
std::vector<String> imgFiles;
imgFiles.reserve(allFiles.size());
for (const auto& f : allFiles)
if (haveImageReader(f)) imgFiles.push_back(f);
std::sort(imgFiles.begin(), imgFiles.end());
return imgFiles;
}
// bookkeeping: maps each pose in vo->getTrajectory() back to its source image filename.
//
// processFrame() doesn't report which frame a pose came from, so this infers it: most frames
// add one pose, but the frame where two-view initialization completes adds two at once (the
// reference frame's pose, remembered when initialization started, plus the current frame's).
class PoseFilenameTracker
{
public:
void update(slam::OdometryState before, slam::OdometryState after,
size_t trajectorySize, const String& filename)
{
if (before == slam::NOT_INITIALIZED ||
(before == slam::TRACKING && after == slam::INITIALIZING))
refFilename_ = filename;
const size_t added = trajectorySize - prevTrajectorySize_;
if (added == 1)
filenames_.push_back(filename);
else if (added == 2)
{
filenames_.push_back(refFilename_);
filenames_.push_back(filename);
}
prevTrajectorySize_ = trajectorySize;
}
const std::vector<String>& filenames() const { return filenames_; }
private:
std::vector<String> filenames_;
size_t prevTrajectorySize_ = 0;
String refFilename_;
};
// writer: dumps camera.txt, point3d.txt and images.txt (COLMAP text format) for @p vo into
// @p outputFolder; @p poseFilenames[i] is the source image for trajectory pose i
void writeColmapFiles(const slam::VisualOdometry& vo, const Mat& K,
const std::vector<String>& poseFilenames,
const String& outputFolder)
{
cv::utils::fs::createDirectories(outputFolder);
{
std::ofstream f(cv::utils::fs::join(outputFolder, "camera.txt").c_str());
int width = 0, height = 0;
if (!vo.getMap().keyframes().empty())
{
const slam::KeyFrame* kf = *vo.getMap().keyframes().begin();
width = kf->imageSize.width;
height = kf->imageSize.height;
}
f.setf(std::ios::fixed); f.precision(4);
f << "fx " << K.at<double>(0, 0) << "\n"
<< "fy " << K.at<double>(1, 1) << "\n"
<< "cx " << K.at<double>(0, 2) << "\n"
<< "cy " << K.at<double>(1, 2) << "\n"
<< "width " << width << "\n"
<< "height " << height << "\n";
}
{
std::ofstream f(cv::utils::fs::join(outputFolder, "point3d.txt").c_str());
f << "# Map points in world coordinates.\n# Columns: id X Y Z n_observations\n";
f.setf(std::ios::scientific); f.precision(9);
for (slam::MapPoint* mp : vo.getMap().mapPoints())
{
if (!mp || mp->bad) continue;
f << mp->id << " "
<< mp->pos.x << " " << mp->pos.y << " " << mp->pos.z << " "
<< mp->observations.size() << "\n";
}
}
{
std::ofstream f(cv::utils::fs::join(outputFolder, "images.txt").c_str());
const auto& traj = vo.getTrajectory();
f << "# Image list with two lines of data per image:\n"
<< "# IMAGE_ID, QW, QX, QY, QZ, TX, TY, TZ, CAMERA_ID, NAME\n"
<< "# POINTS2D[] as (X, Y, POINT3D_ID)\n"
<< "# Number of images: " << traj.size() << ", mean observations per image: 0.0\n";
f.setf(std::ios::fixed); f.precision(6);
for (size_t i = 0; i < traj.size(); ++i)
{
const Matx44d& T = traj[i];
Matx33d R;
for (int r = 0; r < 3; ++r)
for (int c = 0; c < 3; ++c) R(r, c) = T(r, c);
const Quatd q = Quatd::createFromRotMat(R);
const String name = (i < poseFilenames.size())
? basenameOf(poseFilenames[i])
: (String("pose_") + std::to_string(i));
f << i << " " << q.w << " " << q.x << " " << q.y << " " << q.z << " "
<< T(0,3) << " " << T(1,3) << " " << T(2,3) << " " << 1 << " " << name << "\n";
}
}
}
} // namespace
const string about =
"Monocular visual odometry using ALIKED + LightGlue.\n\n"
"Runs feature detection (ALIKED) and matching (LightGlue) from ONNX models over a\n"
"directory of images, estimates the camera trajectory and writes it to --output.\n"
"To run:\n"
"\t ./example_slam_visual_odometry --aliked=aliked.onnx --lightglue=lightglue.onnx --images=./seq\n"
"Sample command (run on the GPU):\n"
"\t ./example_slam_visual_odometry --aliked=aliked.onnx --lightglue=lightglue.onnx --images=./seq --target=cuda\n";
const string param_keys =
"{ help h | | Print help message }"
"{ aliked | <none> | Path to ALIKED ONNX model }"
"{ lightglue | <none> | Path to LightGlue ONNX model }"
"{ images | <none> | Path to directory with input images }"
"{ output | vo_out | Output directory for trajectory and map }"
"{ fx | 718.856 | Camera focal length X }"
"{ fy | 718.856 | Camera focal length Y }"
"{ cx | 607.1928 | Camera principal point X }"
"{ cy | 185.2157 | Camera principal point Y }"
"{ dist | | Lens distortion coeffs, comma-separated k1,k2,p1,p2[,k3,...] (default: none) }";
const string backend_keys = format(
"{ backend | default | Choose one of computation backends: "
"default: automatically (by default), "
"openvino: Intel's Deep Learning Inference Engine (https://software.intel.com/openvino-toolkit), "
"opencv: OpenCV implementation, "
"vkcom: VKCOM, "
"cuda: CUDA, "
"webnn: WebNN }");
const string target_keys = format(
"{ target | cpu | Choose one of target computation devices: "
"cpu: CPU target (by default), "
"opencl: OpenCL, "
"opencl_fp16: OpenCL fp16 (half-float precision), "
"vpu: VPU, "
"vulkan: Vulkan, "
"cuda: CUDA, "
"cuda_fp16: CUDA fp16 (half-float preprocess) }");
string keys = param_keys + backend_keys + target_keys;
int main(int argc, char** argv)
{
CommandLineParser parser(argc, argv, keys);
parser.about(about);
if (parser.has("help"))
{
parser.printMessage();
return 0;
}
const String alikedPath = parser.get<String>("aliked");
const String lightgluePath = parser.get<String>("lightglue");
const String imagesDir = parser.get<String>("images");
const int backendId = getBackendID(parser.get<String>("backend"));
const int targetId = getTargetID(parser.get<String>("target"));
if (!parser.check() || alikedPath == "<none>" || lightgluePath == "<none>" || imagesDir == "<none>")
{
parser.printErrors();
parser.printMessage();
return 1;
}
const String outputDir = parser.get<String>("output");
const Matx33d K(parser.get<double>("fx"), 0., parser.get<double>("cx"),
0., parser.get<double>("fy"), parser.get<double>("cy"),
0., 0., 1.);
// Optional lens distortion (k1,k2,p1,p2[,k3,...]); empty means no distortion.
Mat distCoeffs;
{
std::stringstream ss(parser.get<String>("dist"));
std::vector<double> coeffs;
String tok;
while (std::getline(ss, tok, ','))
{
const size_t b = tok.find_first_not_of(" \t");
const size_t e = tok.find_last_not_of(" \t");
if (b == String::npos) continue;
coeffs.push_back(std::stod(tok.substr(b, e - b + 1)));
}
if (!coeffs.empty())
distCoeffs = Mat(coeffs, true).reshape(1, 1);
}
ALIKED::Params detParams;
detParams.inputSize = Size(640, 640);
detParams.engine = dnn::ENGINE_NEW;
detParams.backend = backendId;
detParams.target = targetId;
auto detector = ALIKED::create(alikedPath, detParams);
auto matcher = LightGlueMatcher::create(lightgluePath, 0.0f,
backendId, targetId);
slam::OdometryParams voParams;
voParams.minInitParallaxDeg = 1.5;
voParams.minInitPoints = 50;
voParams.pnpReprojThresh = 4.0;
voParams.kfMaxFrames = 30;
voParams.localMapTopK = 10;
auto vo = slam::VisualOdometry::create(
detector, matcher, Mat(K), distCoeffs, voParams);
const std::vector<String> imgFiles = listImageFiles(imagesDir);
if (imgFiles.empty())
{
std::cerr << "no images found in " << imagesDir << "\n";
return 1;
}
std::cout << "images folder : " << imagesDir << "\n"
<< "output folder : " << outputDir << "\n"
<< "total frames : " << imgFiles.size() << "\n\n";
PoseFilenameTracker poseTracker;
int nEmitted = 0;
const int nDigits = std::to_string(imgFiles.size()).size();
const int64 t0 = getTickCount();
for (size_t i = 0; i < imgFiles.size(); ++i)
{
const int64 tFrameStart = getTickCount();
Mat img = imread(imgFiles[i]);
if (img.empty())
{
std::cerr << "Frame " << std::setw(nDigits) << (i + 1) << "/" << imgFiles.size()
<< " failed to read: " << imgFiles[i] << "\n";
continue;
}
const slam::OdometryState before = vo->getState();
const bool emitted = vo->processFrame(img);
const slam::OdometryState after = vo->getState();
if (emitted) ++nEmitted;
poseTracker.update(before, after, vo->getTrajectory().size(), imgFiles[i]);
const double frameElapsed = (getTickCount() - tFrameStart) / getTickFrequency();
const double fps = frameElapsed > 0. ? 1. / frameElapsed : 0.;
std::cout << "Frame " << std::setw(nDigits) << (i + 1) << "/" << imgFiles.size()
<< " processed | fps: " << std::fixed << std::setprecision(1) << std::setw(5) << fps
<< " | emitted: " << (emitted ? "yes" : "no ")
<< " | keyframes: " << vo->getMap().numKeyframes()
<< " | map points: " << vo->getMap().numMapPoints() << "\n";
}
const double elapsed = (getTickCount() - t0) / getTickFrequency();
const double avgFps = elapsed > 0. ? imgFiles.size() / elapsed : 0.;
const bool ok = nEmitted > 0;
if (ok && !outputDir.empty())
writeColmapFiles(*vo, Mat(K), poseTracker.filenames(), outputDir);
std::cout << "\n"
<< "==================== Visual Odometry Result ====================\n"
<< "status : " << (ok ? "OK" : "FAILED") << "\n"
<< "total frames : " << imgFiles.size() << "\n"
<< "camera poses : " << vo->getTrajectory().size() << "\n"
<< "keyframes : " << vo->getMap().numKeyframes() << "\n"
<< "map points : " << vo->getMap().numMapPoints() << "\n"
<< "elapsed time : " << std::fixed << std::setprecision(2) << elapsed << " s\n"
<< "average fps : " << std::fixed << std::setprecision(2) << avgFps << "\n"
<< "output dir : " << outputDir << "\n"
<< "====================================================================\n";
return ok ? 0 : 1;
}
+192
View File
@@ -0,0 +1,192 @@
'''
Monocular visual odometry with cv.slam.VisualOdometry (ALIKED + LightGlue).
Example:
python visual_odometry.py --aliked aliked.onnx --lightglue lg.onnx --images ./seq
'''
import argparse
import glob
import os
import sys
import time
import numpy as np
import cv2 as cv
def build_K(fx, fy, cx, cy):
return np.array([[fx, 0., cx],
[0., fy, cy],
[0., 0., 1.]], dtype=np.float64)
def list_image_files(images_dir):
files = [f for f in sorted(glob.glob(os.path.join(images_dir, '*')))
if cv.haveImageReader(f)]
return files
def rotation_matrix_to_quaternion(R):
# Shepperd's method: numerically stable for all rotations.
trace = R[0, 0] + R[1, 1] + R[2, 2]
if trace > 0:
s = np.sqrt(trace + 1.0) * 2
qw = 0.25 * s
qx = (R[2, 1] - R[1, 2]) / s
qy = (R[0, 2] - R[2, 0]) / s
qz = (R[1, 0] - R[0, 1]) / s
elif R[0, 0] > R[1, 1] and R[0, 0] > R[2, 2]:
s = np.sqrt(1.0 + R[0, 0] - R[1, 1] - R[2, 2]) * 2
qw = (R[2, 1] - R[1, 2]) / s
qx = 0.25 * s
qy = (R[0, 1] + R[1, 0]) / s
qz = (R[0, 2] + R[2, 0]) / s
elif R[1, 1] > R[2, 2]:
s = np.sqrt(1.0 + R[1, 1] - R[0, 0] - R[2, 2]) * 2
qw = (R[0, 2] - R[2, 0]) / s
qx = (R[0, 1] + R[1, 0]) / s
qy = 0.25 * s
qz = (R[1, 2] + R[2, 1]) / s
else:
s = np.sqrt(1.0 + R[2, 2] - R[0, 0] - R[1, 1]) * 2
qw = (R[1, 0] - R[0, 1]) / s
qx = (R[0, 2] + R[2, 0]) / s
qy = (R[1, 2] + R[2, 1]) / s
qz = 0.25 * s
return qw, qx, qy, qz
def write_colmap_files(vo, K, image_size, pose_filenames, output_dir):
# Note: cv.slam.Map isn't wrapped for Python (only aggregate counts are,
# via getNumKeyframes/getNumMapPoints), so only camera intrinsics and the
# trajectory (images.txt) can be exported here; point3d.txt (map points)
# requires the C++ sample.
os.makedirs(output_dir, exist_ok=True)
with open(os.path.join(output_dir, 'camera.txt'), 'w') as f:
f.write(f"fx {K[0, 0]:.4f}\n")
f.write(f"fy {K[1, 1]:.4f}\n")
f.write(f"cx {K[0, 2]:.4f}\n")
f.write(f"cy {K[1, 2]:.4f}\n")
f.write(f"width {image_size[0]}\n")
f.write(f"height {image_size[1]}\n")
traj = vo.getTrajectory()
with open(os.path.join(output_dir, 'images.txt'), 'w') as f:
f.write("# Image list with two lines of data per image:\n")
f.write("# IMAGE_ID, QW, QX, QY, QZ, TX, TY, TZ, CAMERA_ID, NAME\n")
f.write("# POINTS2D[] as (X, Y, POINT3D_ID)\n")
f.write(f"# Number of images: {len(traj)}, mean observations per image: 0.0\n")
for i, T in enumerate(traj):
qw, qx, qy, qz = rotation_matrix_to_quaternion(T[:3, :3])
name = (os.path.basename(pose_filenames[i]) if i < len(pose_filenames)
else f"pose_{i}")
f.write(f"{i} {qw:.6f} {qx:.6f} {qy:.6f} {qz:.6f} "
f"{T[0,3]:.6f} {T[1,3]:.6f} {T[2,3]:.6f} 1 {name}\n")
def main():
parser = argparse.ArgumentParser(
description='Monocular visual odometry using ALIKED + LightGlue')
parser.add_argument('--aliked', required=True,
help='Path to ALIKED ONNX model')
parser.add_argument('--lightglue', required=True,
help='Path to LightGlue ONNX model')
parser.add_argument('--images', required=True,
help='Path to directory with input images')
parser.add_argument('--output', default='vo_out',
help='Output directory for trajectory and map (default: vo_out)')
parser.add_argument('--fx', type=float, default=718.856,
help='Camera focal length X (default: KITTI-00)')
parser.add_argument('--fy', type=float, default=718.856,
help='Camera focal length Y (default: KITTI-00)')
parser.add_argument('--cx', type=float, default=607.1928,
help='Camera principal point X (default: KITTI-00)')
parser.add_argument('--cy', type=float, default=185.2157,
help='Camera principal point Y (default: KITTI-00)')
parser.add_argument('--min-parallax', type=float, default=1.5,
help='Minimum initialisation parallax in degrees (default: 1.5)')
parser.add_argument('--min-points', type=int, default=50,
help='Minimum initialisation map points (default: 50)')
args = parser.parse_args()
det_params = cv.ALIKED.Params()
det_params.inputSize = (640, 640)
det_params.engine = cv.dnn.ENGINE_NEW
detector = cv.ALIKED.create(args.aliked, det_params)
matcher = cv.LightGlueMatcher.create(
args.lightglue, 0.0,
cv.dnn.DNN_BACKEND_DEFAULT,
cv.dnn.DNN_TARGET_CPU)
vo_params = cv.slam.OdometryParams()
vo_params.minInitParallaxDeg = args.min_parallax
vo_params.minInitPoints = args.min_points
K = build_K(args.fx, args.fy, args.cx, args.cy)
vo = cv.slam.VisualOdometry.create(
detector, matcher, K, np.array([]), vo_params)
image_files = list_image_files(args.images)
if not image_files:
print(f"no images found in {args.images}", file=sys.stderr)
return 1
print(f"images_folder = {args.images}")
print(f"output_folder = {args.output}")
print(f"found {len(image_files)} image(s)")
# Tracks which input image each emitted trajectory pose came from, for images.txt.
pose_filenames = []
prev_traj_len = 0
ref_filename = None
image_size = (0, 0)
n_emitted = 0
t0 = time.perf_counter()
for i, path in enumerate(image_files):
img = cv.imread(path)
if img is None:
print(f"[FRAME {i}] file={path} imread failed", file=sys.stderr)
continue
image_size = (img.shape[1], img.shape[0])
before = vo.getState()
emitted = vo.processFrame(img)
after = vo.getState()
if emitted:
n_emitted += 1
# Track which input image maps to each trajectory pose.
if before == cv.slam.NOT_INITIALIZED or \
(before == cv.slam.TRACKING and after == cv.slam.INITIALIZING):
ref_filename = path
traj_len = len(vo.getTrajectory())
added = traj_len - prev_traj_len
if added == 1:
pose_filenames.append(path)
elif added == 2:
pose_filenames.append(ref_filename)
pose_filenames.append(path)
prev_traj_len = traj_len
print(f"[FRAME {i}] file={path}"
f" emitted={'yes' if emitted else 'no'}"
f" keyframes={vo.getNumKeyframes()}"
f" map_points={vo.getNumMapPoints()}")
elapsed = time.perf_counter() - t0
ok = n_emitted > 0
if ok and args.output:
write_colmap_files(vo, K, image_size, pose_filenames, args.output)
print(f"run={'ok' if ok else 'FAILED'} frames={len(vo.getTrajectory())} elapsed={elapsed:.2f}s")
print(f"output -> {args.output}")
return 0 if ok else 1
if __name__ == '__main__':
sys.exit(main())