From 18dbed0e0dfab9477ed7514eb3e1ff4034743e7f Mon Sep 17 00:00:00 2001 From: atinfinity Date: Sun, 5 Feb 2017 22:10:37 +0900 Subject: [PATCH 1/5] added dense flow sample --- samples/gpu/pyrlk_optical_flow.cpp | 147 ++++++++++++++++++++++++++++- 1 file changed, 143 insertions(+), 4 deletions(-) diff --git a/samples/gpu/pyrlk_optical_flow.cpp b/samples/gpu/pyrlk_optical_flow.cpp index f13487b622..315301b6e3 100644 --- a/samples/gpu/pyrlk_optical_flow.cpp +++ b/samples/gpu/pyrlk_optical_flow.cpp @@ -2,12 +2,13 @@ #include #include "opencv2/core.hpp" -#include +#include "opencv2/core/utility.hpp" #include "opencv2/imgproc.hpp" #include "opencv2/highgui.hpp" #include "opencv2/video.hpp" #include "opencv2/cudaoptflow.hpp" #include "opencv2/cudaimgproc.hpp" +#include "opencv2/cudaarithm.hpp" using namespace std; using namespace cv; @@ -66,6 +67,132 @@ static void drawArrows(Mat& frame, const vector& prevPts, const vector< } } +inline bool isFlowCorrect(Point2f u) +{ + return !cvIsNaN(u.x) && !cvIsNaN(u.y) && fabs(u.x) < 1e9 && fabs(u.y) < 1e9; +} + +static Vec3b computeColor(float fx, float fy) +{ + static bool first = true; + + // relative lengths of color transitions: + // these are chosen based on perceptual similarity + // (e.g. one can distinguish more shades between red and yellow + // than between yellow and green) + const int RY = 15; + const int YG = 6; + const int GC = 4; + const int CB = 11; + const int BM = 13; + const int MR = 6; + const int NCOLS = RY + YG + GC + CB + BM + MR; + static Vec3i colorWheel[NCOLS]; + + if (first) + { + int k = 0; + + for (int i = 0; i < RY; ++i, ++k) + colorWheel[k] = Vec3i(255, 255 * i / RY, 0); + + for (int i = 0; i < YG; ++i, ++k) + colorWheel[k] = Vec3i(255 - 255 * i / YG, 255, 0); + + for (int i = 0; i < GC; ++i, ++k) + colorWheel[k] = Vec3i(0, 255, 255 * i / GC); + + for (int i = 0; i < CB; ++i, ++k) + colorWheel[k] = Vec3i(0, 255 - 255 * i / CB, 255); + + for (int i = 0; i < BM; ++i, ++k) + colorWheel[k] = Vec3i(255 * i / BM, 0, 255); + + for (int i = 0; i < MR; ++i, ++k) + colorWheel[k] = Vec3i(255, 0, 255 - 255 * i / MR); + + first = false; + } + + const float rad = sqrt(fx * fx + fy * fy); + const float a = atan2(-fy, -fx) / (float)CV_PI; + + const float fk = (a + 1.0f) / 2.0f * (NCOLS - 1); + const int k0 = static_cast(fk); + const int k1 = (k0 + 1) % NCOLS; + const float f = fk - k0; + + Vec3b pix; + + for (int b = 0; b < 3; b++) + { + const float col0 = colorWheel[k0][b] / 255.0f; + const float col1 = colorWheel[k1][b] / 255.0f; + + float col = (1 - f) * col0 + f * col1; + + if (rad <= 1) + col = 1 - rad * (1 - col); // increase saturation with radius + else + col *= .75; // out of range + + pix[2 - b] = static_cast(255.0 * col); + } + + return pix; +} + +static void drawOpticalFlow(const Mat_& flowx, const Mat_& flowy, Mat& dst, float maxmotion = -1) +{ + dst.create(flowx.size(), CV_8UC3); + dst.setTo(Scalar::all(0)); + + // determine motion range: + float maxrad = maxmotion; + + if (maxmotion <= 0) + { + maxrad = 1; + for (int y = 0; y < flowx.rows; ++y) + { + for (int x = 0; x < flowx.cols; ++x) + { + Point2f u(flowx(y, x), flowy(y, x)); + + if (!isFlowCorrect(u)) + continue; + + maxrad = max(maxrad, sqrt(u.x * u.x + u.y * u.y)); + } + } + } + + for (int y = 0; y < flowx.rows; ++y) + { + for (int x = 0; x < flowx.cols; ++x) + { + Point2f u(flowx(y, x), flowy(y, x)); + + if (isFlowCorrect(u)) + dst.at(y, x) = computeColor(u.x / maxrad, u.y / maxrad); + } + } +} + +static void showFlow(const char* name, const GpuMat& d_flow) +{ + GpuMat planes[2]; + cuda::split(d_flow, planes); + + Mat flowx(planes[0]); + Mat flowy(planes[1]); + + Mat out; + drawOpticalFlow(flowx, flowy, out, 10); + + imshow(name, out); +} + template inline T clamp (T x, T a, T b) { return ((x) > (a) ? ((x) < (b) ? (x) : (b)) : (a)); @@ -148,7 +275,7 @@ int main(int argc, const char* argv[]) // Sparse - Ptr d_pyrLK = cuda::SparsePyrLKOpticalFlow::create( + Ptr d_pyrLK_sparse = cuda::SparsePyrLKOpticalFlow::create( Size(winSize, winSize), maxLevel, iters); GpuMat d_frame0(frame0); @@ -157,7 +284,16 @@ int main(int argc, const char* argv[]) GpuMat d_nextPts; GpuMat d_status; - d_pyrLK->calc(useGray ? d_frame0Gray : d_frame0, useGray ? d_frame1Gray : d_frame1, d_prevPts, d_nextPts, d_status); + d_pyrLK_sparse->calc(useGray ? d_frame0Gray : d_frame0, useGray ? d_frame1Gray : d_frame1, d_prevPts, d_nextPts, d_status); + + // Dense + + Ptr d_pyrLK_dense = cuda::DensePyrLKOpticalFlow::create( + Size(winSize, winSize), maxLevel, iters); + + GpuMat d_flow(frame0.size(), CV_32FC2); + + d_pyrLK_dense->calc(d_frame0Gray, d_frame1Gray, d_flow); // Draw arrows @@ -171,9 +307,12 @@ int main(int argc, const char* argv[]) download(d_status, status); drawArrows(frame0, prevPts, nextPts, status, Scalar(255, 0, 0)); - imshow("PyrLK [Sparse]", frame0); + // Draw flows + + showFlow("PyrLK [Dense] Flow Field", d_flow); + waitKey(); return 0; From 9552206a4c49e6e6cd88b9ab2382b920759dcfe2 Mon Sep 17 00:00:00 2001 From: atinfinity Date: Sun, 12 Feb 2017 23:33:57 +0900 Subject: [PATCH 2/5] added option to specify flow type --- samples/gpu/pyrlk_optical_flow.cpp | 98 ++++++++++++++++++------------ 1 file changed, 58 insertions(+), 40 deletions(-) diff --git a/samples/gpu/pyrlk_optical_flow.cpp b/samples/gpu/pyrlk_optical_flow.cpp index 315301b6e3..11c8b731a0 100644 --- a/samples/gpu/pyrlk_optical_flow.cpp +++ b/samples/gpu/pyrlk_optical_flow.cpp @@ -207,15 +207,16 @@ template inline T mapValue(T x, T a, T b, T c, T d) int main(int argc, const char* argv[]) { const char* keys = - "{ h help | | print help message }" + "{ h help | | print help message }" "{ l left | ../data/pic1.png | specify left image }" "{ r right | ../data/pic2.png | specify right image }" - "{ gray | | use grayscale sources [PyrLK Sparse] }" - "{ win_size | 21 | specify windows size [PyrLK] }" - "{ max_level | 3 | specify max level [PyrLK] }" - "{ iters | 30 | specify iterations count [PyrLK] }" - "{ points | 4000 | specify points count [GoodFeatureToTrack] }" - "{ min_dist | 0 | specify minimal distance between points [GoodFeatureToTrack] }"; + "{ flow | sparse | specify flow type [PyrLK] }" + "{ gray | | use grayscale sources [PyrLK Sparse] }" + "{ win_size | 21 | specify windows size [PyrLK] }" + "{ max_level | 3 | specify max level [PyrLK] }" + "{ iters | 30 | specify iterations count [PyrLK] }" + "{ points | 4000 | specify points count [GoodFeatureToTrack] }" + "{ min_dist | 0 | specify minimal distance between points [GoodFeatureToTrack] }"; CommandLineParser cmd(argc, argv, keys); @@ -235,6 +236,22 @@ int main(int argc, const char* argv[]) return -1; } + string flow_type = cmd.get("flow"); + bool is_sparse = true; + if (flow_type == "sparse") + { + is_sparse = true; + } + else if (flow_type == "dense") + { + is_sparse = false; + } + else + { + cerr << "please specify 'sparse' or 'dense' as flow type" << endl; + return -1; + } + bool useGray = cmd.has("gray"); int winSize = cmd.get("win_size"); int maxLevel = cmd.get("max_level"); @@ -251,8 +268,14 @@ int main(int argc, const char* argv[]) return -1; } - namedWindow("PyrLK [Sparse]", WINDOW_NORMAL); - namedWindow("PyrLK [Dense] Flow Field", WINDOW_NORMAL); + if(is_sparse) + { + namedWindow("PyrLK [Sparse]", WINDOW_NORMAL); + } + else + { + namedWindow("PyrLK [Dense] Flow Field", WINDOW_NORMAL); + } cout << "Image size : " << frame0.cols << " x " << frame0.rows << endl; cout << "Points count : " << points << endl; @@ -265,55 +288,50 @@ int main(int argc, const char* argv[]) cv::cvtColor(frame1, frame1Gray, COLOR_BGR2GRAY); // goodFeaturesToTrack - GpuMat d_frame0Gray(frame0Gray); GpuMat d_prevPts; Ptr detector = cuda::createGoodFeaturesToTrackDetector(d_frame0Gray.type(), points, 0.01, minDist); - detector->detect(d_frame0Gray, d_prevPts); - // Sparse - - Ptr d_pyrLK_sparse = cuda::SparsePyrLKOpticalFlow::create( - Size(winSize, winSize), maxLevel, iters); - GpuMat d_frame0(frame0); GpuMat d_frame1(frame1); GpuMat d_frame1Gray(frame1Gray); GpuMat d_nextPts; GpuMat d_status; - - d_pyrLK_sparse->calc(useGray ? d_frame0Gray : d_frame0, useGray ? d_frame1Gray : d_frame1, d_prevPts, d_nextPts, d_status); - - // Dense - - Ptr d_pyrLK_dense = cuda::DensePyrLKOpticalFlow::create( - Size(winSize, winSize), maxLevel, iters); - GpuMat d_flow(frame0.size(), CV_32FC2); - d_pyrLK_dense->calc(d_frame0Gray, d_frame1Gray, d_flow); + if (is_sparse) + { + // Sparse + Ptr d_pyrLK_sparse = cuda::SparsePyrLKOpticalFlow::create( + Size(winSize, winSize), maxLevel, iters); + d_pyrLK_sparse->calc(useGray ? d_frame0Gray : d_frame0, useGray ? d_frame1Gray : d_frame1, d_prevPts, d_nextPts, d_status); - // Draw arrows + // Draw arrows + vector prevPts(d_prevPts.cols); + download(d_prevPts, prevPts); - vector prevPts(d_prevPts.cols); - download(d_prevPts, prevPts); + vector nextPts(d_nextPts.cols); + download(d_nextPts, nextPts); - vector nextPts(d_nextPts.cols); - download(d_nextPts, nextPts); + vector status(d_status.cols); + download(d_status, status); - vector status(d_status.cols); - download(d_status, status); + drawArrows(frame0, prevPts, nextPts, status, Scalar(255, 0, 0)); + imshow("PyrLK [Sparse]", frame0); + } + else { + // Dense + Ptr d_pyrLK_dense = cuda::DensePyrLKOpticalFlow::create( + Size(winSize, winSize), maxLevel, iters); + d_pyrLK_dense->calc(d_frame0Gray, d_frame1Gray, d_flow); - drawArrows(frame0, prevPts, nextPts, status, Scalar(255, 0, 0)); - imshow("PyrLK [Sparse]", frame0); + // Draw flows + showFlow("PyrLK [Dense] Flow Field", d_flow); + } - // Draw flows - - showFlow("PyrLK [Dense] Flow Field", d_flow); - - waitKey(); + waitKey(0); return 0; -} +} \ No newline at end of file From c2a880d7c9aa2170baa6a476f2632ead935a0403 Mon Sep 17 00:00:00 2001 From: atinfinity Date: Mon, 13 Feb 2017 00:26:48 +0900 Subject: [PATCH 3/5] fixed indent --- samples/gpu/pyrlk_optical_flow.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/samples/gpu/pyrlk_optical_flow.cpp b/samples/gpu/pyrlk_optical_flow.cpp index 11c8b731a0..096fe86bee 100644 --- a/samples/gpu/pyrlk_optical_flow.cpp +++ b/samples/gpu/pyrlk_optical_flow.cpp @@ -269,7 +269,7 @@ int main(int argc, const char* argv[]) } if(is_sparse) - { + { namedWindow("PyrLK [Sparse]", WINDOW_NORMAL); } else @@ -321,7 +321,8 @@ int main(int argc, const char* argv[]) drawArrows(frame0, prevPts, nextPts, status, Scalar(255, 0, 0)); imshow("PyrLK [Sparse]", frame0); } - else { + else + { // Dense Ptr d_pyrLK_dense = cuda::DensePyrLKOpticalFlow::create( Size(winSize, winSize), maxLevel, iters); From bbe0f2cef53ae460ffaf27d5d1dff22c1b5843a0 Mon Sep 17 00:00:00 2001 From: atinfinity Date: Mon, 13 Feb 2017 00:35:52 +0900 Subject: [PATCH 4/5] refactoring --- samples/gpu/pyrlk_optical_flow.cpp | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/samples/gpu/pyrlk_optical_flow.cpp b/samples/gpu/pyrlk_optical_flow.cpp index 096fe86bee..e540fa736d 100644 --- a/samples/gpu/pyrlk_optical_flow.cpp +++ b/samples/gpu/pyrlk_optical_flow.cpp @@ -268,15 +268,6 @@ int main(int argc, const char* argv[]) return -1; } - if(is_sparse) - { - namedWindow("PyrLK [Sparse]", WINDOW_NORMAL); - } - else - { - namedWindow("PyrLK [Dense] Flow Field", WINDOW_NORMAL); - } - cout << "Image size : " << frame0.cols << " x " << frame0.rows << endl; cout << "Points count : " << points << endl; @@ -318,6 +309,7 @@ int main(int argc, const char* argv[]) vector status(d_status.cols); download(d_status, status); + namedWindow("PyrLK [Sparse]", WINDOW_NORMAL); drawArrows(frame0, prevPts, nextPts, status, Scalar(255, 0, 0)); imshow("PyrLK [Sparse]", frame0); } @@ -329,6 +321,7 @@ int main(int argc, const char* argv[]) d_pyrLK_dense->calc(d_frame0Gray, d_frame1Gray, d_flow); // Draw flows + namedWindow("PyrLK [Dense] Flow Field", WINDOW_NORMAL); showFlow("PyrLK [Dense] Flow Field", d_flow); } From ca5e0f53b826a1ee5ff61a0755aaa6a7e375dac8 Mon Sep 17 00:00:00 2001 From: atinfinity Date: Mon, 13 Feb 2017 00:53:35 +0900 Subject: [PATCH 5/5] refactoring --- samples/gpu/pyrlk_optical_flow.cpp | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/samples/gpu/pyrlk_optical_flow.cpp b/samples/gpu/pyrlk_optical_flow.cpp index e540fa736d..fa6b376362 100644 --- a/samples/gpu/pyrlk_optical_flow.cpp +++ b/samples/gpu/pyrlk_optical_flow.cpp @@ -1,14 +1,14 @@ #include #include -#include "opencv2/core.hpp" -#include "opencv2/core/utility.hpp" -#include "opencv2/imgproc.hpp" -#include "opencv2/highgui.hpp" -#include "opencv2/video.hpp" -#include "opencv2/cudaoptflow.hpp" -#include "opencv2/cudaimgproc.hpp" -#include "opencv2/cudaarithm.hpp" +#include +#include +#include +#include +#include +#include +#include +#include using namespace std; using namespace cv;