Added GPU version of PyrLK based global motion estimator (videostab)

modules/videostab/include/opencv2/videostab/global_motion.hpp

@@ -50,6 +50,10 @@
 #include "opencv2/features2d/features2d.hpp"
 #include "opencv2/videostab/optical_flow.hpp"
 
+#if HAVE_OPENCV_GPU
+  #include "opencv2/gpu/gpu.hpp"
+#endif
+
 namespace cv
 {
 namespace videostab
@@ -161,14 +165,43 @@ private:
     Ptr<FeatureDetector> detector_;
     Ptr<ISparseOptFlowEstimator> optFlowEstimator_;
     RansacParams ransacParams_;
+    float minInlierRatio_;
+    Size gridSize_;
+
     std::vector<uchar> status_;
     std::vector<KeyPoint> keypointsPrev_;
     std::vector<Point2f> pointsPrev_, points_;
     std::vector<Point2f> pointsPrevGood_, pointsGood_;
-    float minInlierRatio_;
-    Size gridSize_;
 };
 
+#if HAVE_OPENCV_GPU
+class CV_EXPORTS PyrLkRobustMotionEstimatorGpu : public GlobalMotionEstimatorBase
+{
+public:
+    PyrLkRobustMotionEstimatorGpu(MotionModel model = MM_AFFINE);
+
+    void setRansacParams(const RansacParams &val) { ransacParams_ = val; }
+    RansacParams ransacParams() const { return ransacParams_; }
+
+    void setMinInlierRatio(float val) { minInlierRatio_ = val; }
+    float minInlierRatio() const { return minInlierRatio_; }
+
+    virtual Mat estimate(const Mat &frame0, const Mat &frame1, bool *ok = 0);
+    Mat estimate(const gpu::GpuMat &frame0, const gpu::GpuMat &frame1, bool *ok = 0);
+
+private:
+    gpu::GoodFeaturesToTrackDetector_GPU detector_;
+    SparsePyrLkOptFlowEstimatorGpu optFlowEstimator_;
+    RansacParams ransacParams_;
+    float minInlierRatio_;
+
+    gpu::GpuMat frame0_, grayFrame0_, frame1_;
+    gpu::GpuMat pointsPrev_, points_;
+    Mat hostPointsPrev_, hostPoints_;
+    gpu::GpuMat status_;
+};
+#endif
+
 CV_EXPORTS Mat getMotion(int from, int to, const std::vector<Mat> &motions);
 
 } // namespace videostab

modules/videostab/include/opencv2/videostab/optical_flow.hpp

@@ -47,7 +47,7 @@
 #include "opencv2/opencv_modules.hpp"
 
 #if HAVE_OPENCV_GPU
-#include "opencv2/gpu/gpu.hpp"
+  #include "opencv2/gpu/gpu.hpp"
 #endif
 
 namespace cv
@@ -99,6 +99,27 @@ public:
 };
 
 #if HAVE_OPENCV_GPU
+class CV_EXPORTS SparsePyrLkOptFlowEstimatorGpu
+        : public PyrLkOptFlowEstimatorBase, public ISparseOptFlowEstimator
+{
+public:
+    SparsePyrLkOptFlowEstimatorGpu();
+
+    virtual void run(
+            InputArray frame0, InputArray frame1, InputArray points0, InputOutputArray points1,
+            OutputArray status, OutputArray errors);
+
+    void run(const gpu::GpuMat &frame0, const gpu::GpuMat &frame1, const gpu::GpuMat &points0, gpu::GpuMat &points1,
+             gpu::GpuMat &status, gpu::GpuMat &errors);
+
+    void run(const gpu::GpuMat &frame0, const gpu::GpuMat &frame1, const gpu::GpuMat &points0, gpu::GpuMat &points1,
+             gpu::GpuMat &status);
+
+private:
+    gpu::PyrLKOpticalFlow optFlowEstimator_;
+    gpu::GpuMat frame0_, frame1_, points0_, points1_, status_, errors_;
+};
+
 class CV_EXPORTS DensePyrLkOptFlowEstimatorGpu
         : public PyrLkOptFlowEstimatorBase, public IDenseOptFlowEstimator
 {
@@ -108,6 +129,7 @@ public:
     virtual void run(
             InputArray frame0, InputArray frame1, InputOutputArray flowX, InputOutputArray flowY,
             OutputArray errors);
+
 private:
     gpu::PyrLKOpticalFlow optFlowEstimator_;
     gpu::GpuMat frame0_, frame1_, flowX_, flowY_, errors_;

modules/videostab/src/global_motion.cpp

@@ -241,13 +241,11 @@ Mat estimateGlobalMotionLeastSquares(
 
 
 Mat estimateGlobalMotionRobust(
-        const vector<Point2f> &points0, const vector<Point2f> &points1, int model,
+        int npoints, const Point2f *points0, const Point2f *points1, int model,
         const RansacParams &params, float *rmse, int *ninliers)
 {
     CV_Assert(model <= MM_AFFINE);
-    CV_Assert(points0.size() == points1.size());
 
-    const int npoints = static_cast<int>(points0.size());
     const int niters = static_cast<int>(ceil(log(1 - params.prob) /
                                              log(1 - pow(1 - params.eps, params.size))));
 
@@ -382,11 +380,7 @@ PyrLkRobustMotionEstimator::PyrLkRobustMotionEstimator(MotionModel model)
     setDetector(new GoodFeaturesToTrackDetector());
     setOptFlowEstimator(new SparsePyrLkOptFlowEstimator());
     setMotionModel(model);
-
-    RansacParams ransac = RansacParams::default2dMotion(model);
-    ransac.size *= 2; // we use more points than needed, but result looks better
-    setRansacParams(ransac);
-
+    setRansacParams(RansacParams::default2dMotion(model));
     setMinInlierRatio(0.1f);
     setGridSize(Size(0,0));
 }
@@ -394,35 +388,37 @@ PyrLkRobustMotionEstimator::PyrLkRobustMotionEstimator(MotionModel model)
 
 Mat PyrLkRobustMotionEstimator::estimate(const Mat &frame0, const Mat &frame1, bool *ok)
 {
+    // find keypoints
+
     detector_->detect(frame0, keypointsPrev_);
 
     // add extra keypoints
+
     if (gridSize_.width > 0 && gridSize_.height > 0)
     {
-        float dx = (float)frame0.cols / (gridSize_.width + 1);
-        float dy = (float)frame0.rows / (gridSize_.height + 1);
+        float dx = static_cast<float>(frame0.cols) / (gridSize_.width + 1);
+        float dy = static_cast<float>(frame0.rows) / (gridSize_.height + 1);
         for (int x = 0; x < gridSize_.width; ++x)
             for (int y = 0; y < gridSize_.height; ++y)
                 keypointsPrev_.push_back(KeyPoint((x+1)*dx, (y+1)*dy, 0.f));
     }
 
-    // draw keypoints
-    /*Mat img;
-    drawKeypoints(frame0, keypointsPrev_, img);
-    imshow("frame0_keypoints", img);
-    waitKey(3);*/
+    // extract points from keypoints
 
     pointsPrev_.resize(keypointsPrev_.size());
     for (size_t i = 0; i < keypointsPrev_.size(); ++i)
         pointsPrev_[i] = keypointsPrev_[i].pt;
 
+    // find correspondences
+
     optFlowEstimator_->run(frame0, frame1, pointsPrev_, points_, status_, noArray());
 
-    size_t npoints = points_.size();
-    pointsPrevGood_.clear(); pointsPrevGood_.reserve(npoints);
-    pointsGood_.clear(); pointsGood_.reserve(npoints);
+    // leave good correspondences only
 
-    for (size_t i = 0; i < npoints; ++i)
+    pointsPrevGood_.clear(); pointsPrevGood_.reserve(points_.size());
+    pointsGood_.clear(); pointsGood_.reserve(points_.size());
+
+    for (size_t i = 0; i < points_.size(); ++i)
     {
         if (status_[i])
         {
@@ -431,24 +427,29 @@ Mat PyrLkRobustMotionEstimator::estimate(const Mat &frame0, const Mat &frame1, b
         }
     }
 
-    int ninliers;
+    size_t npoints = pointsGood_.size();
+
+    // find motion
+
+    int ninliers = 0;
     Mat_<float> M;
 
     if (motionModel_ != MM_HOMOGRAPHY)
         M = estimateGlobalMotionRobust(
-                pointsPrevGood_, pointsGood_, motionModel_, ransacParams_, 0, &ninliers);
+                npoints, &pointsPrevGood_[0], &pointsGood_[0], motionModel_,
+                ransacParams_, 0, &ninliers);
     else
     {
         vector<uchar> mask;
         M = findHomography(pointsPrevGood_, pointsGood_, mask, CV_RANSAC, ransacParams_.thresh);
-
-        ninliers = 0;
-        for (size_t i  = 0; i < pointsGood_.size(); ++i)
+        for (size_t i  = 0; i < npoints; ++i)
             if (mask[i]) ninliers++;
     }
 
+    // check if we're confident enough in estimated motion
+
     if (ok) *ok = true;
-    if (static_cast<float>(ninliers) / pointsGood_.size() < minInlierRatio_)
+    if (static_cast<float>(ninliers) / npoints < minInlierRatio_)
     {
         M = Mat::eye(3, 3, CV_32F);
         if (ok) *ok = false;
@@ -458,6 +459,85 @@ Mat PyrLkRobustMotionEstimator::estimate(const Mat &frame0, const Mat &frame1, b
 }
 
 
+#if HAVE_OPENCV_GPU
+PyrLkRobustMotionEstimatorGpu::PyrLkRobustMotionEstimatorGpu(MotionModel model)
+{
+    CV_Assert(gpu::getCudaEnabledDeviceCount() > 0);
+    setMotionModel(model);
+    setRansacParams(RansacParams::default2dMotion(model));
+    setMinInlierRatio(0.1f);
+}
+
+
+Mat PyrLkRobustMotionEstimatorGpu::estimate(const Mat &frame0, const Mat &frame1, bool *ok)
+{
+    frame0_.upload(frame0);
+    frame1_.upload(frame1);
+    return estimate(frame0_, frame1_, ok);
+}
+
+
+Mat PyrLkRobustMotionEstimatorGpu::estimate(const gpu::GpuMat &frame0, const gpu::GpuMat &frame1, bool *ok)
+{
+    // convert frame to gray if it's color
+
+    gpu::GpuMat grayFrame0;
+    if (frame0.channels() == 1)
+        grayFrame0 = frame0;
+    else
+    {
+        gpu::cvtColor(frame0_, grayFrame0_, CV_BGR2GRAY);
+        grayFrame0 = grayFrame0_;
+    }
+
+    // find keypoints
+
+    detector_(grayFrame0, pointsPrev_);
+
+    // find correspondences
+
+    optFlowEstimator_.run(frame0, frame1, pointsPrev_, points_, status_);    
+
+    // leave good correspondences only
+
+    gpu::compactPoints(pointsPrev_, points_, status_);
+
+    pointsPrev_.download(hostPointsPrev_);
+    points_.download(hostPoints_);
+
+    int npoints = hostPointsPrev_.cols;
+
+    // find motion
+
+    int ninliers = 0;
+    Mat_<float> M;
+
+    if (motionModel_ != MM_HOMOGRAPHY)
+        M = estimateGlobalMotionRobust(
+                npoints, hostPointsPrev_.ptr<Point2f>(0), hostPoints_.ptr<Point2f>(), motionModel_,
+                ransacParams_, 0, &ninliers);
+    else
+    {
+        vector<uchar> mask;
+        M = findHomography(hostPointsPrev_, hostPoints_, mask, CV_RANSAC, ransacParams_.thresh);
+        for (int i  = 0; i < npoints; ++i)
+            if (mask[i]) ninliers++;
+    }
+
+    // check if we're confident enough in estimated motion
+
+    if (ok) *ok = true;
+    if (static_cast<float>(ninliers) / npoints < minInlierRatio_)
+    {
+        M = Mat::eye(3, 3, CV_32F);
+        if (ok) *ok = false;
+    }
+
+    return M;
+}
+#endif // #if HAVE_OPENCV_GPU
+
+
 Mat getMotion(int from, int to, const vector<Mat> &motions)
 {
     Mat M = Mat::eye(3, 3, CV_32F);

modules/videostab/src/inpainting.cpp

@@ -520,7 +520,6 @@ void completeFrameAccordingToFlow(
     Mat_<uchar> flowMask_(flowMask), mask1_(mask1), mask0_(mask0);
     Mat_<float> flowX_(flowX), flowY_(flowY);
 
-    //int count = 0;
     for (int y0 = 0; y0 < frame0.rows; ++y0)
     {
         for (int x0 = 0; x0 < frame0.cols; ++x0)
@@ -535,12 +534,10 @@ void completeFrameAccordingToFlow(
                 {                    
                     frame0.at<Point3_<uchar> >(y0,x0) = frame1.at<Point3_<uchar> >(y1,x1);
                     mask0_(y0,x0) = 255;
-                    //count++;
                 }
             }
         }
     }
-    //cout << count << endl;
 }
 
 } // namespace videostab

modules/videostab/src/motion_stabilizing.cpp

@@ -261,9 +261,6 @@ void LpMotionStabilizer::stabilize(
     rowlb_.assign(nrows, -INF);
     rowub_.assign(nrows, INF);
 
-    vector<CoinShallowPackedVector> packedRows;
-    packedRows.reserve(nrows);
-
     int r = 0;
 
     // frame corners

modules/videostab/src/optical_flow.cpp

@@ -61,6 +61,53 @@ void SparsePyrLkOptFlowEstimator::run(
 
 
 #if HAVE_OPENCV_GPU
+SparsePyrLkOptFlowEstimatorGpu::SparsePyrLkOptFlowEstimatorGpu()
+{
+    CV_Assert(gpu::getCudaEnabledDeviceCount() > 0);
+}
+
+
+void SparsePyrLkOptFlowEstimatorGpu::run(
+        InputArray frame0, InputArray frame1, InputArray points0, InputOutputArray points1,
+        OutputArray status, OutputArray errors)
+{
+    frame0_.upload(frame0.getMat());
+    frame1_.upload(frame1.getMat());
+    points0_.upload(points0.getMat());
+
+    if (errors.needed())
+    {
+        run(frame0_, frame1_, points0_, points1_, status_, errors_);
+        errors_.download(errors.getMatRef());
+    }
+    else
+        run(frame0_, frame1_, points0_, points1_, status_);
+
+    points1_.download(points1.getMatRef());
+    status_.download(status.getMatRef());
+}
+
+
+void SparsePyrLkOptFlowEstimatorGpu::run(
+        const gpu::GpuMat &frame0, const gpu::GpuMat &frame1, const gpu::GpuMat &points0,
+        gpu::GpuMat &points1, gpu::GpuMat &status, gpu::GpuMat &errors)
+{
+    optFlowEstimator_.winSize = winSize_;
+    optFlowEstimator_.maxLevel = maxLevel_;
+    optFlowEstimator_.sparse(frame0, frame1, points0, points1, status, &errors);
+}
+
+
+void SparsePyrLkOptFlowEstimatorGpu::run(
+        const gpu::GpuMat &frame0, const gpu::GpuMat &frame1, const gpu::GpuMat &points0,
+        gpu::GpuMat &points1, gpu::GpuMat &status)
+{
+    optFlowEstimator_.winSize = winSize_;
+    optFlowEstimator_.maxLevel = maxLevel_;
+    optFlowEstimator_.sparse(frame0, frame1, points0, points1, status);
+}
+
+
 DensePyrLkOptFlowEstimatorGpu::DensePyrLkOptFlowEstimatorGpu()
 {
     CV_Assert(gpu::getCudaEnabledDeviceCount() > 0);
@@ -76,6 +123,7 @@ void DensePyrLkOptFlowEstimatorGpu::run(
 
     optFlowEstimator_.winSize = winSize_;
     optFlowEstimator_.maxLevel = maxLevel_;
+
     if (errors.needed())
     {
         optFlowEstimator_.dense(frame0_, frame1_, flowX_, flowY_, &errors_);
@@ -87,8 +135,7 @@ void DensePyrLkOptFlowEstimatorGpu::run(
     flowX_.download(flowX.getMatRef());
     flowY_.download(flowY.getMatRef());
 }
-#endif
-
+#endif // #if HAVE_OPENCV_GPU
 
 } // namespace videostab
 } // namespace cv

modules/videostab/src/precomp.hpp

@@ -49,6 +49,7 @@
 
 #include <stdexcept>
 #include <iostream>
+#include <ctime>
 #include "opencv2/core/core.hpp"
 #include "opencv2/highgui/highgui.hpp"
 #include "opencv2/imgproc/imgproc.hpp"

modules/videostab/src/stabilizer.cpp

@@ -321,6 +321,7 @@ void TwoPassStabilizer::runPrePassIfNecessary()
 {
     if (!isPrePassDone_)
     {
+        clock_t startTime = clock();
         log_->print("first pass: estimating motions");
 
         Mat prevFrame, frame;
@@ -346,6 +347,13 @@ void TwoPassStabilizer::runPrePassIfNecessary()
                     else
                         motions2_.push_back(motions_.back());
                 }
+
+                if (ok)
+                {
+                    if (ok2) log_->print(".");
+                    else log_->print("?");
+                }
+                else log_->print("x");
             }
             else
             {
@@ -356,13 +364,6 @@ void TwoPassStabilizer::runPrePassIfNecessary()
 
             prevFrame = frame;
             frameCount_++;
-
-            if (ok)
-            {
-                if (ok2) log_->print(".");
-                else log_->print("?");
-            }
-            else log_->print("x");
         }
 
         // add aux. motions
@@ -419,6 +420,9 @@ void TwoPassStabilizer::runPrePassIfNecessary()
 
         isPrePassDone_ = true;
         frameSource_->reset();
+
+        clock_t elapsedTime = clock() - startTime;
+        log_->print("first pass time: %.3f sec\n", static_cast<double>(elapsedTime) / CLOCKS_PER_SEC);
     }
 }