From 4e60decad3ae8649b5888b13db0c43cc0b2d6e5f Mon Sep 17 00:00:00 2001 From: Maria Dimashova Date: Tue, 3 Aug 2010 16:28:52 +0000 Subject: [PATCH] added GridAdaptedFeatureDetector, PyramidAdaptedFeatureDetector and funcs to draw keypoints and matches --- .../include/opencv2/features2d/features2d.hpp | 66 +++++- modules/features2d/src/descriptors.cpp | 203 ++++++++++++++---- modules/features2d/src/detectors.cpp | 122 +++++++++-- samples/cpp/descriptor_extractor_matcher.cpp | 15 +- 4 files changed, 349 insertions(+), 57 deletions(-) diff --git a/modules/features2d/include/opencv2/features2d/features2d.hpp b/modules/features2d/include/opencv2/features2d/features2d.hpp index 7fb4cabfa1..c21d6fcb36 100644 --- a/modules/features2d/include/opencv2/features2d/features2d.hpp +++ b/modules/features2d/include/opencv2/features2d/features2d.hpp @@ -1477,6 +1477,44 @@ protected: CV_EXPORTS Ptr createDetector( const string& detectorType ); +/* + * Adapts a detector to partition the source image into a grid and detect + * points in each cell. + */ +class CV_EXPORTS GridAdaptedFeatureDetector : public FeatureDetector +{ +public: + GridAdaptedFeatureDetector( const Ptr& _detector, int _maxTotalKeypoints, + int _gridRows=4, int _gridCols=4 ); + // todo read/write + +protected: + Ptr detector; + int maxTotalKeypoints; + int gridRows; + int gridCols; + + virtual void detectImpl( const Mat& image, const Mat& mask, vector& keypoints ) const; +}; + +/* + * Adapts a detector to detect points over multiple levels of a Gaussian + * pyramid. Useful for detectors that are not inherently scaled. + */ +class PyramidAdaptedFeatureDetector : public FeatureDetector +{ +public: + PyramidAdaptedFeatureDetector( const Ptr& _detector, int _levels=2 ); + + // todo read/write + +protected: + Ptr detector; + int levels; + + virtual void detectImpl( const Mat& image, const Mat& mask, vector& keypoints ) const; +}; + /****************************************************************************************\ * DescriptorExtractor * \****************************************************************************************/ @@ -2273,18 +2311,38 @@ struct CV_EXPORTS DrawMatchesFlags enum{ DEFAULT = 0, // Output image matrix will be created (Mat::create), // i.e. existing memory of output image may be reused. // Two source image, matches and single keypoints will be drawn. + // For each keypoint only the center point will be drawn (without + // the circle around keypoint with keypoint size and orientation). DRAW_OVER_OUTIMG = 1, // Output image matrix will not be created (Mat::create). // Matches will be drawn on existing content of output image. - NOT_DRAW_SINGLE_POINTS = 2 // Single keypoints will not be drawn. + NOT_DRAW_SINGLE_POINTS = 2, // Single keypoints will not be drawn. + DRAW_RICH_KEYPOINTS = 4 // For each keypoint the circle around keypoint with keypoint size and + // orientation will be drawn. }; }; +// Draw keypoints. +CV_EXPORTS void drawKeypoints( const Mat& image, const vector& keypoints, Mat& outImg, + const Scalar& color=Scalar::all(-1), int flags=DrawMatchesFlags::DEFAULT ); + // Draws matches of keypints from two images on output image. CV_EXPORTS void drawMatches( const Mat& img1, const vector& keypoints1, const Mat& img2, const vector& keypoints2, - const vector& matches, Mat& outImg, - const Scalar& matchColor = Scalar::all(-1), const Scalar& singlePointColor = Scalar::all(-1), - const vector& matchesMask = vector(), int flags = DrawMatchesFlags::DEFAULT ); + const vector& matches1to2, Mat& outImg, + const Scalar& matchColor=Scalar::all(-1), const Scalar& singlePointColor=Scalar::all(-1), + const vector& matchesMask=vector(), int flags=DrawMatchesFlags::DEFAULT ); + +CV_EXPORTS void drawMatches( const Mat& img1, const vector& keypoints1, + const Mat& img2, const vector& keypoints2, + const vector& matches1to2, Mat& outImg, + const Scalar& matchColor=Scalar::all(-1), const Scalar& singlePointColor=Scalar::all(-1), + const vector& matchesMask=vector(), int flags=DrawMatchesFlags::DEFAULT ); + +CV_EXPORTS void drawMatches( const Mat& img1, const vector& keypoints1, + const Mat& img2, const vector& keypoints2, + const vector >& matches1to2, Mat& outImg, + const Scalar& matchColor=Scalar::all(-1), const Scalar& singlePointColor=Scalar::all(-1), + const vector >& matchesMask=vector >(), int flags=DrawMatchesFlags::DEFAULT ); } diff --git a/modules/features2d/src/descriptors.cpp b/modules/features2d/src/descriptors.cpp index 29d185de30..0b44a2f2cf 100644 --- a/modules/features2d/src/descriptors.cpp +++ b/modules/features2d/src/descriptors.cpp @@ -48,6 +48,10 @@ //#define _KDTREE using namespace std; + +const int draw_shift_bits = 4; +const int draw_multiplier = 1 << draw_shift_bits; + namespace cv { @@ -69,63 +73,190 @@ Mat windowedMatchingMask( const vector& keypoints1, const vector& keypoints1, - const Mat& img2,const vector& keypoints2, - const vector& matches, Mat& outImg, - const Scalar& matchColor, const Scalar& singlePointColor, - const vector& matchesMask, int flags ) +/* + * Drawing functions + */ + +static inline void _drawKeypoint( Mat& img, const KeyPoint& p, const Scalar& color, int flags ) +{ + Point center( p.pt.x * draw_multiplier, p.pt.y * draw_multiplier ); + + if( flags & DrawMatchesFlags::DRAW_RICH_KEYPOINTS ) + { + int radius = p.size/2 * draw_multiplier; // KeyPoint::size is a diameter + + // draw the circles around keypoints with the keypoints size + circle( img, center, radius, color, 1, CV_AA, draw_shift_bits ); + + // draw orientation of the keypoint, if it is applicable + if( p.angle != -1 ) + { + float srcAngleRad = p.angle*CV_PI/180; + Point orient(cos(srcAngleRad)*radius, sin(srcAngleRad)*radius); + line( img, center, center+orient, color, 1, CV_AA, draw_shift_bits ); + } +#if 0 + else + { + // draw center with R=1 + int radius = 1 * draw_multiplier; + circle( img, center, radius, color, 1, CV_AA, draw_shift_bits ); + } +#endif + } + else + { + // draw center with R=3 + int radius = 3 * draw_multiplier; + circle( img, center, radius, color, 1, CV_AA, draw_shift_bits ); + } +} + +void drawKeypoints( const Mat& image, const vector& keypoints, Mat& outImg, + const Scalar& _color, int flags ) +{ + if( !(flags & DrawMatchesFlags::DRAW_OVER_OUTIMG) ) + cvtColor( image, outImg, CV_GRAY2BGR ); + + RNG& rng=theRNG(); + bool isRandColor = _color == Scalar::all(-1); + + for( vector::const_iterator i = keypoints.begin(), ie = keypoints.end(); i != ie; ++i ) + { + Scalar color = isRandColor ? Scalar(rng(256), rng(256), rng(256)) : _color; + _drawKeypoint( outImg, *i, color, flags ); + } +} + +static void _prepareImgAndDrawKeypoints( const Mat& img1, const vector& keypoints1, + const Mat& img2, const vector& keypoints2, + Mat& outImg, Mat& outImg1, Mat& outImg2, + const Scalar& singlePointColor, int flags ) { Size size( img1.cols + img2.cols, MAX(img1.rows, img2.rows) ); if( flags & DrawMatchesFlags::DRAW_OVER_OUTIMG ) { if( size.width > outImg.cols || size.height > outImg.rows ) CV_Error( CV_StsBadSize, "outImg has size less than need to draw img1 and img2 together" ); + outImg1 = outImg( Rect(0, 0, img1.cols, img1.rows) ); + outImg2 = outImg( Rect(img1.cols, 0, img2.cols, img2.rows) ); } else { outImg.create( size, CV_MAKETYPE(img1.depth(), 3) ); - Mat outImg1 = outImg( Rect(0, 0, img1.cols, img1.rows) ); + outImg1 = outImg( Rect(0, 0, img1.cols, img1.rows) ); + outImg2 = outImg( Rect(img1.cols, 0, img2.cols, img2.rows) ); cvtColor( img1, outImg1, CV_GRAY2RGB ); - Mat outImg2 = outImg( Rect(img1.cols, 0, img2.cols, img2.rows) ); cvtColor( img2, outImg2, CV_GRAY2RGB ); } - RNG rng; // draw keypoints if( !(flags & DrawMatchesFlags::NOT_DRAW_SINGLE_POINTS) ) { - bool isRandSinglePointColor = singlePointColor == Scalar::all(-1); - for( vector::const_iterator it = keypoints1.begin(); it < keypoints1.end(); ++it ) - { - circle( outImg, it->pt, 3, isRandSinglePointColor ? - Scalar(rng.uniform(0, 256), rng.uniform(0, 256), rng.uniform(0, 256)) : singlePointColor ); - } - for( vector::const_iterator it = keypoints2.begin(); it < keypoints2.end(); ++it ) - { - Point p = it->pt; - circle( outImg, Point(p.x+img1.cols, p.y), 3, isRandSinglePointColor ? - Scalar(rng.uniform(0, 256), rng.uniform(0, 256), rng.uniform(0, 256)) : singlePointColor ); - } - } + Mat outImg1 = outImg( Rect(0, 0, img1.cols, img1.rows) ); + drawKeypoints( outImg1, keypoints1, outImg1, singlePointColor, flags + DrawMatchesFlags::DRAW_OVER_OUTIMG ); + + Mat outImg2 = outImg( Rect(img1.cols, 0, img2.cols, img2.rows) ); + drawKeypoints( outImg2, keypoints2, outImg2, singlePointColor, flags + DrawMatchesFlags::DRAW_OVER_OUTIMG ); + } +} + +static inline void _drawMatch( Mat& outImg, Mat& outImg1, Mat& outImg2 , + const KeyPoint& kp1, const KeyPoint& kp2, const Scalar& matchColor, int flags ) +{ + RNG& rng = theRNG(); + bool isRandMatchColor = matchColor == Scalar::all(-1); + Scalar color = isRandMatchColor ? Scalar( rng(256), rng(256), rng(256) ) : matchColor; + + _drawKeypoint( outImg1, kp1, color, flags ); + _drawKeypoint( outImg2, kp2, color, flags ); + + Point2f pt1 = kp1.pt, + pt2 = kp2.pt, + dpt2 = Point2f( std::min(pt2.x+outImg1.cols, float(outImg.cols-1)), pt2.y ); + + line( outImg, Point(pt1.x*draw_multiplier, pt1.y*draw_multiplier), Point(dpt2.x*draw_multiplier, dpt2.y*draw_multiplier), + color, 1, CV_AA, draw_shift_bits ); +} + +void drawMatches( const Mat& img1, const vector& keypoints1, + const Mat& img2,const vector& keypoints2, + const vector& matches1to2, Mat& outImg, + const Scalar& matchColor, const Scalar& singlePointColor, + const vector& matchesMask, int flags ) +{ + if( matches1to2.size() != keypoints1.size() ) + CV_Error( CV_StsBadSize, "matches1to2 must have the same size as keypoints1" ); + if( !matchesMask.empty() && matchesMask.size() != matches1to2.size() ) + CV_Error( CV_StsBadSize, "matchesMask must have the same size as matches1to2" ); + + Mat outImg1, outImg2; + _prepareImgAndDrawKeypoints( img1, keypoints1, img2, keypoints2, + outImg, outImg1, outImg2, singlePointColor, flags ); // draw matches - bool isRandMatchColor = matchColor == Scalar::all(-1); - if( matches.size() != keypoints1.size() ) - CV_Error( CV_StsBadSize, "matches must have the same size as keypoints1" ); - if( !matchesMask.empty() && matchesMask.size() != keypoints1.size() ) - CV_Error( CV_StsBadSize, "mask must have the same size as keypoints1" ); - vector::const_iterator mit = matches.begin(); - for( int i1 = 0; mit != matches.end(); ++mit, i1++ ) + for( size_t i1 = 0; i1 < keypoints1.size(); i1++ ) { - if( (matchesMask.empty() || matchesMask[i1] ) && *mit >= 0 ) + int i2 = matches1to2[i1]; + if( (matchesMask.empty() || matchesMask[i1] ) && i2 >= 0 ) { - Point2f pt1 = keypoints1[i1].pt, - pt2 = keypoints2[*mit].pt, - dpt2 = Point2f( std::min(pt2.x+img1.cols, float(outImg.cols-1)), pt2.y ); - Scalar randColor( rng.uniform(0, 256), rng.uniform(0, 256), rng.uniform(0, 256) ); - circle( outImg, pt1, 3, isRandMatchColor ? randColor : matchColor ); - circle( outImg, dpt2, 3, isRandMatchColor ? randColor : matchColor ); - line( outImg, pt1, dpt2, isRandMatchColor ? randColor : matchColor ); + const KeyPoint &kp1 = keypoints1[i1], &kp2 = keypoints2[i2]; + _drawMatch( outImg, outImg1, outImg2, kp1, kp2, matchColor, flags ); + } + } +} + +void drawMatches( const Mat& img1, const vector& keypoints1, + const Mat& img2, const vector& keypoints2, + const vector& matches1to2, Mat& outImg, + const Scalar& matchColor, const Scalar& singlePointColor, + const vector& matchesMask, int flags ) +{ + if( !matchesMask.empty() && matchesMask.size() != matches1to2.size() ) + CV_Error( CV_StsBadSize, "matchesMask must have the same size as matches1to2" ); + + Mat outImg1, outImg2; + _prepareImgAndDrawKeypoints( img1, keypoints1, img2, keypoints2, + outImg, outImg1, outImg2, singlePointColor, flags ); + + // draw matches + for( size_t m = 0; m < matches1to2.size(); m++ ) + { + int i1 = matches1to2[m].indexQuery; + int i2 = matches1to2[m].indexTrain; + if( matchesMask.empty() || matchesMask[m] ) + { + const KeyPoint &kp1 = keypoints1[i1], &kp2 = keypoints2[i2]; + _drawMatch( outImg, outImg1, outImg2, kp1, kp2, matchColor, flags ); + } + } +} + +void drawMatches( const Mat& img1, const vector& keypoints1, + const Mat& img2, const vector& keypoints2, + const vector >& matches1to2, Mat& outImg, + const Scalar& matchColor, const Scalar& singlePointColor, + const vector >& matchesMask, int flags ) +{ + if( !matchesMask.empty() && matchesMask.size() != matches1to2.size() ) + CV_Error( CV_StsBadSize, "matchesMask must have the same size as matches1to2" ); + + Mat outImg1, outImg2; + _prepareImgAndDrawKeypoints( img1, keypoints1, img2, keypoints2, + outImg, outImg1, outImg2, singlePointColor, flags ); + + // draw matches + for( size_t i = 0; i < matches1to2.size(); i++ ) + { + for( size_t j = 0; j < matches1to2[i].size(); j++ ) + { + int i1 = matches1to2[i][j].indexQuery; + int i2 = matches1to2[i][j].indexTrain; + if( matchesMask.empty() || matchesMask[i][j] ) + { + const KeyPoint &kp1 = keypoints1[i1], &kp2 = keypoints2[i2]; + _drawMatch( outImg, outImg1, outImg2, kp1, kp2, matchColor, flags ); + } } } } diff --git a/modules/features2d/src/detectors.cpp b/modules/features2d/src/detectors.cpp index 6cee854697..7607252893 100644 --- a/modules/features2d/src/detectors.cpp +++ b/modules/features2d/src/detectors.cpp @@ -46,8 +46,8 @@ using namespace std; namespace cv { /* - FeatureDetector -*/ + * FeatureDetector + */ struct MaskPredicate { MaskPredicate( const Mat& _mask ) : mask(_mask) @@ -70,8 +70,8 @@ void FeatureDetector::removeInvalidPoints( const Mat& mask, vector& ke }; /* - FastFeatureDetector -*/ + * FastFeatureDetector + */ FastFeatureDetector::FastFeatureDetector( int _threshold, bool _nonmaxSuppression ) : threshold(_threshold), nonmaxSuppression(_nonmaxSuppression) {} @@ -95,8 +95,8 @@ void FastFeatureDetector::detectImpl( const Mat& image, const Mat& mask, vector< } /* - GoodFeaturesToTrackDetector -*/ + * GoodFeaturesToTrackDetector + */ GoodFeaturesToTrackDetector::GoodFeaturesToTrackDetector( int _maxCorners, double _qualityLevel, \ double _minDistance, int _blockSize, bool _useHarrisDetector, double _k ) @@ -140,8 +140,8 @@ void GoodFeaturesToTrackDetector::detectImpl( const Mat& image, const Mat& mask, } /* - MserFeatureDetector -*/ + * MserFeatureDetector + */ MserFeatureDetector::MserFeatureDetector( int delta, int minArea, int maxArea, double maxVariation, double minDiversity, int maxEvolution, double areaThreshold, @@ -204,8 +204,8 @@ void MserFeatureDetector::detectImpl( const Mat& image, const Mat& mask, vector< } /* - StarFeatureDetector -*/ + * StarFeatureDetector + */ StarFeatureDetector::StarFeatureDetector(int maxSize, int responseThreshold, int lineThresholdProjected, int lineThresholdBinarized, @@ -244,8 +244,8 @@ void StarFeatureDetector::detectImpl( const Mat& image, const Mat& mask, vector< } /* - SiftFeatureDetector -*/ + * SiftFeatureDetector + */ SiftFeatureDetector::SiftFeatureDetector(double threshold, double edgeThreshold, int nOctaves, int nOctaveLayers, int firstOctave, int angleMode) : sift(threshold, edgeThreshold, nOctaves, nOctaveLayers, firstOctave, angleMode) @@ -286,8 +286,8 @@ void SiftFeatureDetector::detectImpl( const Mat& image, const Mat& mask, } /* - SurfFeatureDetector -*/ + * SurfFeatureDetector + */ SurfFeatureDetector::SurfFeatureDetector( double hessianThreshold, int octaves, int octaveLayers) : surf(hessianThreshold, octaves, octaveLayers) {} @@ -360,4 +360,98 @@ Ptr createDetector( const string& detectorType ) return fd; } +/* + * GridAdaptedFeatureDetector + */ +GridAdaptedFeatureDetector::GridAdaptedFeatureDetector( const Ptr& _detector, + int _maxTotalKeypoints, int _gridRows, int _gridCols ) + : detector(_detector), maxTotalKeypoints(_maxTotalKeypoints), gridRows(_gridRows), gridCols(_gridCols) +{} + +struct ResponseComparator +{ + bool operator() (const KeyPoint& a, const KeyPoint& b) + { + return std::abs(a.response) > std::abs(b.response); + } +}; + +void keepStrongest( int N, vector& keypoints ) +{ + if( (int)keypoints.size() > N ) + { + vector::iterator nth = keypoints.begin() + N; + std::nth_element( keypoints.begin(), nth, keypoints.end(), ResponseComparator() ); + keypoints.erase( nth, keypoints.end() ); + } +} + +void GridAdaptedFeatureDetector::detectImpl( const Mat &image, const Mat &mask, + vector &keypoints ) const +{ + keypoints.clear(); + keypoints.reserve(maxTotalKeypoints); + + int maxPerCell = maxTotalKeypoints / (gridRows * gridCols); + for( int i = 0; i < gridRows; ++i ) + { + Range row_range((i*image.rows)/gridRows, ((i+1)*image.rows)/gridRows); + for( int j = 0; j < gridCols; ++j ) + { + Range col_range((j*image.cols)/gridCols, ((j+1)*image.cols)/gridCols); + Mat sub_image = image(row_range, col_range); + Mat sub_mask; + if( !mask.empty() ) + sub_mask = mask(row_range, col_range); + + vector sub_keypoints; + detector->detect( sub_image, sub_keypoints, sub_mask ); + keepStrongest( maxPerCell, sub_keypoints ); + for( std::vector::iterator it = sub_keypoints.begin(), end = sub_keypoints.end(); + it != end; ++it ) + { + it->pt.x += col_range.start; + it->pt.y += row_range.start; + } + + keypoints.insert( keypoints.end(), sub_keypoints.begin(), sub_keypoints.end() ); + } + } +} + +/* + * GridAdaptedFeatureDetector + */ +PyramidAdaptedFeatureDetector::PyramidAdaptedFeatureDetector( const Ptr& _detector, int _levels ) + : detector(_detector), levels(_levels) +{} + +void PyramidAdaptedFeatureDetector::detectImpl( const Mat& image, const Mat& mask, vector& keypoints ) const +{ + Mat src = image; + for( int l = 0, multiplier = 1; l <= levels; ++l, multiplier *= 2 ) + { + // Detect on current level of the pyramid + vector new_pts; + detector->detect(src, new_pts); + for( vector::iterator it = new_pts.begin(), end = new_pts.end(); it != end; ++it) + { + it->pt.x *= multiplier; + it->pt.y *= multiplier; + it->size *= multiplier; + it->octave = l; + } + removeInvalidPoints( mask, new_pts ); + keypoints.insert( keypoints.end(), new_pts.begin(), new_pts.end() ); + + // Downsample + if( l < levels ) + { + Mat dst; + pyrDown(src, dst); + src = dst; + } + } +} + } diff --git a/samples/cpp/descriptor_extractor_matcher.cpp b/samples/cpp/descriptor_extractor_matcher.cpp index 5dcb9b93b9..4407d7e0bc 100644 --- a/samples/cpp/descriptor_extractor_matcher.cpp +++ b/samples/cpp/descriptor_extractor_matcher.cpp @@ -8,6 +8,9 @@ using namespace cv; using namespace std; +#define DRAW_RICH_KEYPOINTS_MODE 0 +#define DRAW_OUTLIERS_MODE 0 + void warpPerspectiveRand( const Mat& src, Mat& dst, Mat& H, RNG& rng ) { H.create(3, 3, CV_32FC1); @@ -79,12 +82,18 @@ void doIteration( const Mat& img1, Mat& img2, bool isWarpPerspective, matchesMask[i1] = 1; } // draw inliers - drawMatches( img1, keypoints1, img2, keypoints2, matches, drawImg, CV_RGB(0, 255, 0), CV_RGB(0, 0, 255), matchesMask ); -#if 0 // draw outliers + drawMatches( img1, keypoints1, img2, keypoints2, matches, drawImg, CV_RGB(0, 255, 0), CV_RGB(0, 0, 255), matchesMask +#if DRAW_RICH_KEYPOINTS_MODE + , DrawMatchesFlags::DRAW_RICH_KEYPOINTS +#endif + ); + +#if DRAW_OUTLIERS_MODE + // draw outliers for( size_t i1 = 0; i1 < matchesMask.size(); i1++ ) matchesMask[i1] = !matchesMask[i1]; drawMatches( img1, keypoints1, img2, keypoints2, matches, drawImg, CV_RGB(0, 0, 255), CV_RGB(255, 0, 0), matchesMask, - DrawMatchesFlags::DRAW_OVER_OUTIMG | DrawMatchesFlags::NOT_DRAW_SINGLE_POINTS ) + DrawMatchesFlags::DRAW_OVER_OUTIMG | DrawMatchesFlags::NOT_DRAW_SINGLE_POINTS ); #endif } else