From 70a27c7dd6a781c7d4d8729084283beffb4a0fc9 Mon Sep 17 00:00:00 2001
From: Alexander Alekhin <alexander.a.alekhin@gmail.com>
Date: Sat, 1 Sep 2018 14:31:15 +0000
Subject: [PATCH] core: add solveLP type checks for output

to forbid Mat1f

Checks are not reliable: empty uninitialized `cv::Mat` has `CV_8UC1` type
---
 modules/core/src/lpsolver.cpp       |  4 ++++
 modules/core/test/test_lpsolver.cpp | 10 ++++++++++
 2 files changed, 14 insertions(+)

diff --git a/modules/core/src/lpsolver.cpp b/modules/core/src/lpsolver.cpp
index e5f4d99739..1a1307d5b5 100644
--- a/modules/core/src/lpsolver.cpp
+++ b/modules/core/src/lpsolver.cpp
@@ -98,6 +98,10 @@ int solveLP(const Mat& Func, const Mat& Constr, Mat& z){
     CV_Assert(Constr.type()==CV_64FC1 || Constr.type()==CV_32FC1);
     CV_Assert((Func.rows==1 && (Constr.cols-Func.cols==1))||
             (Func.cols==1 && (Constr.cols-Func.rows==1)));
+    if (!z.empty())
+        CV_CheckTypeEQ(z.type(), CV_64FC1, "");
+    else
+        CV_CheckType(z.type(), z.type() == CV_64FC1 || z.type() == CV_8UC1/*empty cv::Mat*/, "");
 
     //copy arguments for we will shall modify them
     Mat_<double> bigC=Mat_<double>(1,(Func.rows==1?Func.cols:Func.rows)+1),
diff --git a/modules/core/test/test_lpsolver.cpp b/modules/core/test/test_lpsolver.cpp
index b47a6eb080..97f87f4c2f 100644
--- a/modules/core/test/test_lpsolver.cpp
+++ b/modules/core/test/test_lpsolver.cpp
@@ -141,4 +141,14 @@ TEST(Core_LPSolver, regression_cycling){
 #endif
 }
 
+TEST(Core_LPSolver, issue_12337)
+{
+    Mat A=(cv::Mat_<double>(3,1)<<3,1,2);
+    Mat B=(cv::Mat_<double>(3,4)<<1,1,3,30,2,2,5,24,4,1,2,36);
+    EXPECT_ANY_THROW(Mat1f z_float; cv::solveLP(A, B, z_float));
+    EXPECT_NO_THROW(Mat1d z_double; cv::solveLP(A, B, z_double));
+    EXPECT_ANY_THROW(Mat1i z_int; cv::solveLP(A, B, z_int));
+    //need to update interface: EXPECT_ANY_THROW(Mat1b z_8u; cv::solveLP(A, B, z_8u));
+}
+
 }} // namespace