Merge pull request #24527 from MaximSmolskiy:fix-out-of-image-corners-in-cv-cornersubpix

Fix out of image corners in cv::cornerSubPix #24527

### Pull Request Readiness Checklist

See details at https://github.com/opencv/opencv/wiki/How_to_contribute#making-a-good-pull-request

- [x] I agree to contribute to the project under Apache 2 License.
- [x] To the best of my knowledge, the proposed patch is not based on a code under GPL or another license that is incompatible with OpenCV
- [x] The PR is proposed to the proper branch
- [ ] There is a reference to the original bug report and related work
- [ ] There is accuracy test, performance test and test data in opencv_extra repository, if applicable
      Patch to opencv_extra has the same branch name.
- [ ] The feature is well documented and sample code can be built with the project CMake

modules/imgproc/src/cornersubpix.cpp

@@ -96,6 +96,7 @@ void cv::cornerSubPix( InputArray _image, InputOutputArray _corners,
     for( int pt_i = 0; pt_i < count; pt_i++ )
     {
         Point2f cT = corners[pt_i], cI = cT;
+        CV_Assert( Rect(0, 0, src.cols, src.rows).contains(cT) );
         int iter = 0;
         double err = 0;
 
@@ -140,9 +141,10 @@ void cv::cornerSubPix( InputArray _image, InputOutputArray _corners,
             cI2.x = (float)(cI.x + c*scale*bb1 - b*scale*bb2);
             cI2.y = (float)(cI.y - b*scale*bb1 + a*scale*bb2);
             err = (cI2.x - cI.x) * (cI2.x - cI.x) + (cI2.y - cI.y) * (cI2.y - cI.y);
-            cI = cI2;
-            if( cI.x < 0 || cI.x >= src.cols || cI.y < 0 || cI.y >= src.rows )
+            // if new point is out of image, leave previous point as the result
+            if( !Rect(0, 0, src.cols, src.rows).contains(cI2) )
                 break;
+            cI = cI2;
         }
         while( ++iter < max_iters && err > eps );
 

modules/imgproc/test/test_cornersubpix.cpp

@@ -0,0 +1,68 @@
+/*M///////////////////////////////////////////////////////////////////////////////////////
+//
+//  IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING.
+//
+//  By downloading, copying, installing or using the software you agree to this license.
+//  If you do not agree to this license, do not download, install,
+//  copy or use the software.
+//
+//
+//                           License Agreement
+//                For Open Source Computer Vision Library
+//
+// Copyright (C) 2015-2023, OpenCV Foundation, all rights reserved.
+// Third party copyrights are property of their respective owners.
+//
+// Redistribution and use in source and binary forms, with or without modification,
+// are permitted provided that the following conditions are met:
+//
+//   * Redistribution's of source code must retain the above copyright notice,
+//     this list of conditions and the following disclaimer.
+//
+//   * Redistribution's in binary form must reproduce the above copyright notice,
+//     this list of conditions and the following disclaimer in the documentation
+//     and/or other materials provided with the distribution.
+//
+//   * The name of the copyright holders may not be used to endorse or promote products
+//     derived from this software without specific prior written permission.
+//
+// This software is provided by the copyright holders and contributors "as is" and
+// any express or implied warranties, including, but not limited to, the implied
+// warranties of merchantability and fitness for a particular purpose are disclaimed.
+// In no event shall the Intel Corporation or contributors be liable for any direct,
+// indirect, incidental, special, exemplary, or consequential damages
+// (including, but not limited to, procurement of substitute goods or services;
+// loss of use, data, or profits; or business interruption) however caused
+// and on any theory of liability, whether in contract, strict liability,
+// or tort (including negligence or otherwise) arising in any way out of
+// the use of this software, even if advised of the possibility of such damage.
+//
+//M*/
+
+#include "test_precomp.hpp"
+
+namespace opencv_test { namespace {
+
+TEST(Imgproc_CornerSubPix, out_of_image_corners)
+{
+    const uint8_t image_pixels[] = {
+        0, 0, 0, 0, 0, 0, 0,
+        0, 0, 0, 0, 0, 0, 0,
+        0, 0, 0, 0, 0, 0, 0,
+        0, 0, 0, 0, 0, 0, 0,
+        0, 0, 0, 0, 0, 0, 1,
+        0, 0, 0, 0, 0, 0, 2,
+        0, 0, 0, 0, 0, 0, 3};
+
+    cv::Mat image(cv::Size(7, 7), CV_8UC1, (void*)image_pixels, cv::Mat::AUTO_STEP);
+    std::vector<cv::Point2f> corners = {cv::Point2f(5.25, 6.5)};
+    cv::Size win(1, 1);
+    cv::Size zeroZone(-1, -1);
+    cv::TermCriteria criteria;
+    cv::cornerSubPix(image, corners, win, zeroZone, criteria);
+
+    ASSERT_EQ(corners.size(), 1u);
+    ASSERT_TRUE(Rect(0, 0, image.cols, image.rows).contains(corners.front()));
+}
+
+}} // namespace