From a33d7928a4313be7fd6c0de4943b6698c7db47f5 Mon Sep 17 00:00:00 2001
From: Vadim Pisarevsky <vadim.pisarevsky@gmail.com>
Date: Sat, 2 May 2015 22:12:12 +0300
Subject: [PATCH] Fixed getContinuousSize() to handle huge matrices properly.
 This should solve http://code.opencv.org/issues/3232

---
 modules/core/src/precomp.hpp | 27 +++++++++++++++++----------
 1 file changed, 17 insertions(+), 10 deletions(-)

diff --git a/modules/core/src/precomp.hpp b/modules/core/src/precomp.hpp
index 0b65c13de9..88b60e4713 100644
--- a/modules/core/src/precomp.hpp
+++ b/modules/core/src/precomp.hpp
@@ -151,39 +151,46 @@ template<typename T> struct OpMax
     T operator ()(const T a, const T b) const { return std::max(a, b); }
 };
 
+inline Size getContinuousSize_( int flags, int cols, int rows, int widthScale )
+{
+    int64 sz = (int64)cols * rows * widthScale;
+    return (flags & Mat::CONTINUOUS_FLAG) != 0 &&
+        (int)sz == sz ? Size((int)sz, 1) : Size(cols * widthScale, rows);
+}
+
 inline Size getContinuousSize( const Mat& m1, int widthScale=1 )
 {
-    return m1.isContinuous() ? Size(m1.cols*m1.rows*widthScale, 1) :
-        Size(m1.cols*widthScale, m1.rows);
+    return getContinuousSize_(m1.flags,
+                              m1.cols, m1.rows, widthScale);
 }
 
 inline Size getContinuousSize( const Mat& m1, const Mat& m2, int widthScale=1 )
 {
-    return (m1.flags & m2.flags & Mat::CONTINUOUS_FLAG) != 0 ?
-        Size(m1.cols*m1.rows*widthScale, 1) : Size(m1.cols*widthScale, m1.rows);
+    return getContinuousSize_(m1.flags & m2.flags,
+                              m1.cols, m1.rows, widthScale);
 }
 
 inline Size getContinuousSize( const Mat& m1, const Mat& m2,
                                const Mat& m3, int widthScale=1 )
 {
-    return (m1.flags & m2.flags & m3.flags & Mat::CONTINUOUS_FLAG) != 0 ?
-        Size(m1.cols*m1.rows*widthScale, 1) : Size(m1.cols*widthScale, m1.rows);
+    return getContinuousSize_(m1.flags & m2.flags & m3.flags,
+                              m1.cols, m1.rows, widthScale);
 }
 
 inline Size getContinuousSize( const Mat& m1, const Mat& m2,
                                const Mat& m3, const Mat& m4,
                                int widthScale=1 )
 {
-    return (m1.flags & m2.flags & m3.flags & m4.flags & Mat::CONTINUOUS_FLAG) != 0 ?
-        Size(m1.cols*m1.rows*widthScale, 1) : Size(m1.cols*widthScale, m1.rows);
+    return getContinuousSize_(m1.flags & m2.flags & m3.flags & m4.flags,
+                              m1.cols, m1.rows, widthScale);
 }
 
 inline Size getContinuousSize( const Mat& m1, const Mat& m2,
                                const Mat& m3, const Mat& m4,
                                const Mat& m5, int widthScale=1 )
 {
-    return (m1.flags & m2.flags & m3.flags & m4.flags & m5.flags & Mat::CONTINUOUS_FLAG) != 0 ?
-        Size(m1.cols*m1.rows*widthScale, 1) : Size(m1.cols*widthScale, m1.rows);
+    return getContinuousSize_(m1.flags & m2.flags & m3.flags & m4.flags & m5.flags,
+                              m1.cols, m1.rows, widthScale);
 }
 
 struct NoVec