From 6623c62f56bb072d7d5ed90921991646ed4324c0 Mon Sep 17 00:00:00 2001 From: Alexander Smorkalov Date: Thu, 6 Jun 2024 11:37:31 +0300 Subject: [PATCH 01/30] Fixed result buffer overflow in intersectConvexConvex_ for non-convex input. --- modules/imgproc/include/opencv2/imgproc.hpp | 2 +- modules/imgproc/src/geometry.cpp | 35 +++++++++++++----- .../test/test_intersectconvexconvex.cpp | 36 +++++++++++++++++++ 3 files changed, 64 insertions(+), 9 deletions(-) diff --git a/modules/imgproc/include/opencv2/imgproc.hpp b/modules/imgproc/include/opencv2/imgproc.hpp index 471a857f63..d76301c7fd 100644 --- a/modules/imgproc/include/opencv2/imgproc.hpp +++ b/modules/imgproc/include/opencv2/imgproc.hpp @@ -4248,7 +4248,7 @@ Examples of how intersectConvexConvex works When false, no intersection is found. If the polygons share a side or the vertex of one polygon lies on an edge of the other, they are not considered nested and an intersection will be found regardless of the value of handleNested. -@returns Absolute value of area of intersecting polygon +@returns Area of intersecting polygon. May be negative, if algorithm has not converged, e.g. non-convex input. @note intersectConvexConvex doesn't confirm that both polygons are convex and will return invalid results if they aren't. */ diff --git a/modules/imgproc/src/geometry.cpp b/modules/imgproc/src/geometry.cpp index ae582fcafc..eed9ff3c5c 100644 --- a/modules/imgproc/src/geometry.cpp +++ b/modules/imgproc/src/geometry.cpp @@ -377,9 +377,12 @@ static void addSharedSeg( Point2f p, Point2f q, Point2f*& result ) *result++ = q; } - +// Note: The function and subroutings use direct pointer arithmetics instead of arrays indexing. +// Each loop iteration may push to result array up to 3 times. +// It means that we need +3 spare result elements against result_size +// to catch agorithmic overflow and prevent actual result array overflow. static int intersectConvexConvex_( const Point2f* P, int n, const Point2f* Q, int m, - Point2f* result, float* _area ) + Point2f* result, int result_size, float* _area ) { Point2f* result0 = result; // P has n vertices, Q has m vertices. @@ -457,7 +460,7 @@ static int intersectConvexConvex_( const Point2f* P, int n, const Point2f* Q, in } // Quit when both adv. indices have cycled, or one has cycled twice. } - while ( ((aa < n) || (ba < m)) && (aa < 2*n) && (ba < 2*m) ); + while ( ((aa < n) || (ba < m)) && (aa < 2*n) && (ba < 2*m) && ((int)(result - result0) <= result_size) ); // Deal with special cases: not implemented. if( inflag == Unknown ) @@ -466,10 +469,16 @@ static int intersectConvexConvex_( const Point2f* P, int n, const Point2f* Q, in // ... } - int i, nr = (int)(result - result0); + int nr = (int)(result - result0); + if (nr > result_size) + { + *_area = -1.f; + return -1; + } + double area = 0; Point2f prev = result0[nr-1]; - for( i = 1; i < nr; i++ ) + for(int i = 1; i < nr; i++ ) { result0[i-1] = result0[i]; area += (double)prev.x*result0[i].y - (double)prev.y*result0[i].x; @@ -504,9 +513,11 @@ float cv::intersectConvexConvex( InputArray _p1, InputArray _p2, OutputArray _p1 return 0.f; } - AutoBuffer _result(n*2 + m*2 + 1); - Point2f *fp1 = _result.data(), *fp2 = fp1 + n; + AutoBuffer _result(n + m + n+m+1+3); + Point2f* fp1 = _result.data(); + Point2f* fp2 = fp1 + n; Point2f* result = fp2 + m; + int orientation = 0; for( int k = 1; k <= 2; k++ ) @@ -535,7 +546,15 @@ float cv::intersectConvexConvex( InputArray _p1, InputArray _p2, OutputArray _p1 } float area = 0.f; - int nr = intersectConvexConvex_(fp1, n, fp2, m, result, &area); + int nr = intersectConvexConvex_(fp1, n, fp2, m, result, n+m+1, &area); + + if (nr < 0) + { + // The algorithm did not converge, e.g. some of inputs is not convex + _p12.release(); + return -1.f; + } + if( nr == 0 ) { if( !handleNested ) diff --git a/modules/imgproc/test/test_intersectconvexconvex.cpp b/modules/imgproc/test/test_intersectconvexconvex.cpp index fa25f3d531..00e3674f48 100644 --- a/modules/imgproc/test/test_intersectconvexconvex.cpp +++ b/modules/imgproc/test/test_intersectconvexconvex.cpp @@ -255,6 +255,42 @@ TEST(Imgproc_IntersectConvexConvex, intersection_4) EXPECT_NEAR(area, 391500, std::numeric_limits::epsilon()); } +// The inputs are not convex and cuased buffer overflow +// See https://github.com/opencv/opencv/issues/25259 +TEST(Imgproc_IntersectConvexConvex, not_convex) +{ + std::vector convex1 = { + { 46.077175f , 228.66121f }, { 5.428622f , 250.05899f }, {207.51741f , 109.645676f }, + {175.94789f , 32.6566f }, {217.4915f , 252.66176f }, {187.09386f , 6.3988557f}, + { 52.20488f , 69.266205f }, { 38.188286f , 134.48068f }, {246.4742f , 31.41043f }, + {178.97946f , 169.52287f }, {103.40764f , 153.30397f }, {160.67746f , 17.166115f }, + {152.44255f , 135.35f }, {197.03804f , 193.04782f }, {248.28397f , 56.821487f }, + { 10.907227f , 82.55291f }, {109.67949f , 70.7405f }, { 58.96842f , 150.132f }, + {150.7613f , 129.54753f }, {254.98463f , 228.21748f }, {139.02563f , 193.89336f }, + { 84.79946f , 162.25363f }, { 39.83567f , 44.626484f }, {107.034996f , 209.38887f }, + { 67.61073f , 17.119232f }, {208.8617f , 33.67367f }, {182.65207f , 8.291072f }, + { 72.89319f , 42.51845f }, {202.4902f , 123.97209f }, { 79.945076f , 140.99268f }, + {225.8952f , 66.226326f }, { 34.08404f , 219.2208f }, {243.1221f , 60.95162f } + }; + std::vector convex2 = { + {144.33624f , 247.15732f }, { 5.656847f , 17.461054f }, {230.54338f , 2.0446582f}, + {143.0578f , 215.27856f }, {250.44626f , 82.54287f }, { 0.3846766f, 11.101262f }, + { 70.81022f , 17.243904f }, { 77.18812f , 75.760666f }, {190.34933f , 234.30962f }, + {230.10204f , 133.67998f }, { 58.903755f , 252.96451f }, {213.57228f , 155.7058f }, + {190.80992f , 212.90802f }, {203.4356f , 36.55016f }, { 32.276424f , 2.5646307f}, + { 39.73823f , 87.23782f }, {112.46902f , 101.81753f }, { 58.154305f , 238.40395f }, + {187.01064f , 96.24343f }, { 44.42692f , 10.573529f }, {118.76949f , 233.35114f }, + { 86.26109f , 120.93148f }, {217.94751f , 130.5933f }, {148.2687f , 68.56015f }, + {187.44174f , 214.32857f }, {247.19875f , 180.8494f }, { 17.986013f , 61.451443f }, + {254.74344f , 204.71747f }, {211.92726f , 132.0139f }, { 51.36624f , 116.63085f }, + { 83.80044f , 124.20074f }, {122.125854f , 25.182402f }, { 39.08164f , 180.08517f } + }; + std::vector intersection; + + float area = cv::intersectConvexConvex(convex1, convex2, intersection, false); + EXPECT_TRUE(intersection.empty()); + EXPECT_LE(area, 0.f); +} } // namespace } // opencv_test From 88c6c7d9f71eedffdaaf3ac16ba55530ba652a25 Mon Sep 17 00:00:00 2001 From: Alexander Smorkalov Date: Fri, 17 May 2024 11:21:23 +0300 Subject: [PATCH 02/30] Handle BGRA sterams in GStreamer backend --- modules/videoio/src/cap_gstreamer.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/videoio/src/cap_gstreamer.cpp b/modules/videoio/src/cap_gstreamer.cpp index 41e98794b9..cda125334f 100644 --- a/modules/videoio/src/cap_gstreamer.cpp +++ b/modules/videoio/src/cap_gstreamer.cpp @@ -1618,7 +1618,7 @@ bool GStreamerCapture::open(const String &filename_, const cv::VideoCaptureParam { //do not emit signals: all calls will be synchronous and blocking gst_app_sink_set_emit_signals (GST_APP_SINK(sink.get()), FALSE); - caps.attach(gst_caps_from_string("video/x-raw, format=(string){BGR, GRAY8}; video/x-bayer,format=(string){rggb,bggr,grbg,gbrg}; image/jpeg")); + caps.attach(gst_caps_from_string("video/x-raw, format=(string){BGR}; video/x-raw, format=(string){BGRx, BGRA}; video/x-bayer,format=(string){rggb,bggr,grbg,gbrg}; video/x-raw, format=(string){GRAY8}; image/jpeg")); } if (audioStream >= 0) { From 7ff531b8ab57df6a63a2524f3a911663a3eb5457 Mon Sep 17 00:00:00 2001 From: Rostislav Vasilikhin Date: Sat, 15 Jun 2024 11:44:36 +0200 Subject: [PATCH 03/30] Merge pull request #25759 from savuor:rv/equalizeHist_tests Accuracy tests for equalizeHist() added #25759 ### Pull Request Readiness Checklist See details at https://github.com/opencv/opencv/wiki/How_to_contribute#making-a-good-pull-request - [x] I agree to contribute to the project under Apache 2 License. - [x] To the best of my knowledge, the proposed patch is not based on a code under GPL or another license that is incompatible with OpenCV - [x] The PR is proposed to the proper branch - [x] There is a reference to the original bug report and related work - [x] There is accuracy test, performance test and test data in opencv_extra repository, if applicable Patch to opencv_extra has the same branch name. - [x] The feature is well documented and sample code can be built with the project CMake --- modules/imgproc/test/test_histograms.cpp | 69 ++++++++++++++++++++++++ 1 file changed, 69 insertions(+) diff --git a/modules/imgproc/test/test_histograms.cpp b/modules/imgproc/test/test_histograms.cpp index efd045d31b..584b7087c3 100644 --- a/modules/imgproc/test/test_histograms.cpp +++ b/modules/imgproc/test/test_histograms.cpp @@ -2026,5 +2026,74 @@ TEST(Imgproc_Hist_Calc, IPP_ranges_with_nonequal_exponent_21595) ASSERT_EQ(histogram_u.at(2), 4.f) << "1 not counts correctly, res: " << histogram_u.at(2); } +////////////////////////////////////////// equalizeHist() ///////////////////////////////////////// + +void equalizeHistReference(const Mat& src, Mat& dst) +{ + std::vector hist(256, 0); + for (int y = 0; y < src.rows; y++) + { + const uchar* srow = src.ptr(y); + for (int x = 0; x < src.cols; x++) + { + hist[srow[x]]++; + } + } + + int first = 0; + while (!hist[first]) ++first; + + int total = (int)src.total(); + if (hist[first] == total) + { + dst.setTo(first); + return; + } + + std::vector lut(256); + lut[first] = 0; + float scale = (255.f)/(total - hist[first]); + + int sum = 0; + for (int i = first + 1; i < 256; ++i) + { + sum += hist[i]; + lut[i] = saturate_cast(sum * scale); + } + + cv::LUT(src, lut, dst); +} + +typedef ::testing::TestWithParam> Imgproc_Equalize_Hist; + +TEST_P(Imgproc_Equalize_Hist, accuracy) +{ + auto p = GetParam(); + cv::Size size = std::get<0>(p); + int idx = std::get<1>(p); + + RNG &rng = cvtest::TS::ptr()->get_rng(); + rng.state += idx; + + cv::Mat src(size, CV_8U); + cvtest::randUni(rng, src, Scalar::all(0), Scalar::all(255)); + + cv::Mat dst, gold; + + equalizeHistReference(src, gold); + + cv::equalizeHist(src, dst); + + ASSERT_EQ(CV_8UC1, dst.type()); + ASSERT_EQ(gold.size(), dst.size()); + + int nz = cv::countNonZero(dst != gold); + ASSERT_EQ(nz, 0); +} + +INSTANTIATE_TEST_CASE_P(Imgproc_Hist, Imgproc_Equalize_Hist, ::testing::Combine( + ::testing::Values(cv::Size(123, 321), cv::Size(256, 256), cv::Size(1024, 768)), + ::testing::Range(0, 10))); + }} // namespace /* End Of File */ From 24907f35a32b75d9c5b34a203b44327ba53f86f4 Mon Sep 17 00:00:00 2001 From: Dmitry Kurtaev Date: Mon, 17 Jun 2024 12:46:10 +0300 Subject: [PATCH 04/30] Merge pull request #25757 from dkurt:d.kurtaev/opencv_js_tests_old_emsdk Use onRuntimeInitialized with OpenCV.js Node tests #25757 ### Pull Request Readiness Checklist tests: https://github.com/opencv/ci-gha-workflow/pull/174 See details at https://github.com/opencv/opencv/wiki/How_to_contribute#making-a-good-pull-request - [x] I agree to contribute to the project under Apache 2 License. - [x] To the best of my knowledge, the proposed patch is not based on a code under GPL or another license that is incompatible with OpenCV - [x] The PR is proposed to the proper branch - [x] There is a reference to the original bug report and related work - [x] There is accuracy test, performance test and test data in opencv_extra repository, if applicable Patch to opencv_extra has the same branch name. - [x] The feature is well documented and sample code can be built with the project CMake --- modules/js/test/init_cv.js | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/modules/js/test/init_cv.js b/modules/js/test/init_cv.js index d053707407..e6c1d1c16a 100644 --- a/modules/js/test/init_cv.js +++ b/modules/js/test/init_cv.js @@ -2,13 +2,18 @@ // It is subject to the license terms in the LICENSE file found in the top-level directory // of this distribution and at http://opencv.org/license.html. -if (cv instanceof Promise) { - QUnit.test("init_cv", (assert) => { +QUnit.test("init_cv", (assert) => { + if (cv instanceof Promise) { const done = assert.async(); - assert.ok(true); cv.then((ready_cv) => { cv = ready_cv; done(); }); - }); -} + } else if (cv.getBuildInformation === undefined) { + const done = assert.async(); + cv['onRuntimeInitialized'] = () => { + done(); + } + } + assert.ok(true); +}); From 860b688cdd35bc089fea9f812b4c9fb0ecd09bbb Mon Sep 17 00:00:00 2001 From: eplankin Date: Mon, 17 Jun 2024 03:25:49 -0700 Subject: [PATCH 05/30] Enable build with both old and new layouts of IPP --- 3rdparty/ippicv/CMakeLists.txt | 2 +- cmake/OpenCVFindIPP.cmake | 19 ++++++++++++++++--- 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/3rdparty/ippicv/CMakeLists.txt b/3rdparty/ippicv/CMakeLists.txt index 4ef248f3c0..05cab2486f 100644 --- a/3rdparty/ippicv/CMakeLists.txt +++ b/3rdparty/ippicv/CMakeLists.txt @@ -10,7 +10,7 @@ if(HAVE_IPP_ICV) add_definitions(-DICV_BASE) endif() -file(GLOB lib_srcs ${IPP_IW_PATH}/src/*.c) +file(GLOB lib_srcs ${IPP_IW_PATH}/src/*.c ${IPP_IW_PATH}/src/*.cpp) file(GLOB lib_hdrs ${IPP_IW_PATH}/include/*.h ${IPP_IW_PATH}/include/iw/*.h ${IPP_IW_PATH}/include/iw++/*.hpp) # ---------------------------------------------------------------------------------- diff --git a/cmake/OpenCVFindIPP.cmake b/cmake/OpenCVFindIPP.cmake index 2328ef8b43..eb26bea5a1 100644 --- a/cmake/OpenCVFindIPP.cmake +++ b/cmake/OpenCVFindIPP.cmake @@ -87,9 +87,14 @@ macro(ipp_detect_version) get_filename_component(IPP_INCLUDE_DIRS ${IPP_VERSION_FILE} PATH) set(__msg) + set(IPP_NEW_LAYOUT 0) if(EXISTS ${IPP_ROOT_DIR}/include/ippicv_redefs.h) set(__msg " (ICV version)") set(HAVE_IPP_ICV 1) + elseif(EXISTS ${IPP_ROOT_DIR}/include/ipp/ipp.h) + set(IPP_NEW_LAYOUT 1) + # workaround to enable both layouts + add_definitions(-DIPP_PRESERVE_OLD_LAYOUT) elseif(EXISTS ${IPP_ROOT_DIR}/include/ipp.h) # nothing else() @@ -117,10 +122,18 @@ macro(ipp_detect_version) if(APPLE AND NOT HAVE_IPP_ICV) _ipp_set_library_dir(${IPP_ROOT_DIR}/lib) - elseif(IPP_X64) - _ipp_set_library_dir(${IPP_ROOT_DIR}/lib/intel64) + elseif (IPP_NEW_LAYOUT) + if(IPP_X64) + _ipp_set_library_dir(${IPP_ROOT_DIR}/lib) + else() + _ipp_set_library_dir(${IPP_ROOT_DIR}/lib32) + endif() else() - _ipp_set_library_dir(${IPP_ROOT_DIR}/lib/ia32) + if(IPP_X64) + _ipp_set_library_dir(${IPP_ROOT_DIR}/lib/intel64) + else() + _ipp_set_library_dir(${IPP_ROOT_DIR}/lib/ia32) + endif() endif() macro(_ipp_add_library name) From b9d2ecb72f4d79870b10d19631145928dbcf7e68 Mon Sep 17 00:00:00 2001 From: Letu Ren Date: Mon, 17 Jun 2024 18:50:53 +0800 Subject: [PATCH 06/30] Remove unused brow variable --- modules/imgproc/src/demosaicing.cpp | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/modules/imgproc/src/demosaicing.cpp b/modules/imgproc/src/demosaicing.cpp index 2bfa705b8a..dd285cab51 100644 --- a/modules/imgproc/src/demosaicing.cpp +++ b/modules/imgproc/src/demosaicing.cpp @@ -593,10 +593,10 @@ class Bayer2Gray_Invoker : public ParallelLoopBody { public: - Bayer2Gray_Invoker(const Mat& _srcmat, Mat& _dstmat, int _start_with_green, bool _brow, + Bayer2Gray_Invoker(const Mat& _srcmat, Mat& _dstmat, int _start_with_green, const Size& _size, int _bcoeff, int _rcoeff) : ParallelLoopBody(), srcmat(_srcmat), dstmat(_dstmat), Start_with_green(_start_with_green), - Brow(_brow), size(_size), Bcoeff(_bcoeff), Rcoeff(_rcoeff) + size(_size), Bcoeff(_bcoeff), Rcoeff(_rcoeff) { } @@ -612,13 +612,11 @@ public: int dst_step = (int)(dstmat.step/sizeof(T)); int bcoeff = Bcoeff, rcoeff = Rcoeff; int start_with_green = Start_with_green; - bool brow = Brow; dst0 += dst_step + 1; if (range.start % 2) { - brow = !brow; std::swap(bcoeff, rcoeff); start_with_green = !start_with_green; } @@ -680,7 +678,6 @@ public: dst0[-1] = dst0[0]; dst0[size.width] = dst0[size.width-1]; - brow = !brow; std::swap(bcoeff, rcoeff); start_with_green = !start_with_green; } @@ -690,7 +687,6 @@ private: Mat srcmat; Mat dstmat; int Start_with_green; - bool Brow; Size size; int Bcoeff, Rcoeff; }; @@ -704,11 +700,9 @@ static void Bayer2Gray_( const Mat& srcmat, Mat& dstmat, int code ) Size size = srcmat.size(); int bcoeff = B2Y, rcoeff = R2Y; int start_with_green = code == COLOR_BayerGB2GRAY || code == COLOR_BayerGR2GRAY; - bool brow = true; if( code != COLOR_BayerBG2GRAY && code != COLOR_BayerGB2GRAY ) { - brow = false; std::swap(bcoeff, rcoeff); } size.height -= 2; @@ -718,7 +712,7 @@ static void Bayer2Gray_( const Mat& srcmat, Mat& dstmat, int code ) { Range range(0, size.height); Bayer2Gray_Invoker invoker(srcmat, dstmat, - start_with_green, brow, size, bcoeff, rcoeff); + start_with_green, size, bcoeff, rcoeff); parallel_for_(range, invoker, dstmat.total()/static_cast(1<<16)); } From 60819979d4f69fb003d0915eede816582e2a1662 Mon Sep 17 00:00:00 2001 From: Alexander Smorkalov Date: Tue, 18 Jun 2024 10:38:57 +0300 Subject: [PATCH 07/30] Svgfig library update to version 1.0.1 as it uses BSD-3 license. --- doc/pattern_tools/svgfig.py | 75 ++++++++++++++++++++++++------------- 1 file changed, 49 insertions(+), 26 deletions(-) mode change 100755 => 100644 doc/pattern_tools/svgfig.py diff --git a/doc/pattern_tools/svgfig.py b/doc/pattern_tools/svgfig.py old mode 100755 new mode 100644 index 37eaf77c83..73a91ce134 --- a/doc/pattern_tools/svgfig.py +++ b/doc/pattern_tools/svgfig.py @@ -1,20 +1,32 @@ -# svgfig.py copyright (C) 2008 Jim Pivarski -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License -# as published by the Free Software Foundation; either version 2 -# of the License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA -# -# Full licence is in the file COPYING and at http://www.gnu.org/copyleft/gpl.html +# BSD 3-Clause License + +# Copyright (c) 2022, Jim Pivarski +# All rights reserved. + +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: + +# 1. Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. + +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. + +# 3. Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. + +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import re, codecs, os, platform, copy, itertools, math, cmath, random, sys, copy _epsilon = 1e-5 @@ -29,16 +41,24 @@ try: except NameError: unicode = lambda s: str(s) -try: - xrange # Python 2 -except NameError: - xrange = range # Python 3 +if re.search("windows", platform.system(), re.I): + try: + import _winreg + _default_directory = _winreg.QueryValueEx(_winreg.OpenKey(_winreg.HKEY_CURRENT_USER, + r"Software\Microsoft\Windows\Current Version\Explorer\Shell Folders"), "Desktop")[0] +# tmpdir = _winreg.QueryValueEx(_winreg.OpenKey(_winreg.HKEY_CURRENT_USER, "Environment"), "TEMP")[0] +# if tmpdir[0:13] != "%USERPROFILE%": +# tmpdir = os.path.expanduser("~") + tmpdir[13:] + except: + _default_directory = os.path.expanduser("~") + os.sep + "Desktop" _default_fileName = "tmp.svg" _hacks = {} _hacks["inkscape-text-vertical-shift"] = False +__version__ = "1.0.1" + def rgb(r, g, b, maximum=1.): """Create an SVG color string "#xxyyzz" from r, g, and b. @@ -437,9 +457,12 @@ class SVG: return output - @staticmethod - def interpret_fileName(fileName=None): - return fileName or _default_fileName + def interpret_fileName(self, fileName=None): + if fileName is None: + fileName = _default_fileName + if re.search("windows", platform.system(), re.I) and not os.path.isabs(fileName): + fileName = _default_directory + os.sep + fileName + return fileName def save(self, fileName=None, encoding="utf-8", compresslevel=None): """Save to a file for viewing. Note that svg.save() overwrites the file named _default_fileName. @@ -591,7 +614,7 @@ def template(fileName, svg, replaceme="REPLACEME"): def load(fileName): """Loads an SVG image from a file.""" - return load_stream(open(fileName)) + return load_stream(file(fileName)) def load_stream(stream): """Loads an SVG image from a stream (can be a string or a file object).""" @@ -1848,7 +1871,7 @@ class Poly: piecewise-linear segments joining the (x,y) points "bezier"/"B" d=[(x, y, c1x, c1y, c2x, c2y), ...] Bezier curve with two control points (control points - precede (x,y), as in SVG paths). If (c1x,c1y) and + preceed (x,y), as in SVG paths). If (c1x,c1y) and (c2x,c2y) both equal (x,y), you get a linear interpolation ("lines") "velocity"/"V" d=[(x, y, vx, vy), ...] From e3884a9ea88d32bb561dc8fc15017e49a3a4373e Mon Sep 17 00:00:00 2001 From: Yuantao Feng Date: Tue, 18 Jun 2024 17:48:28 +0800 Subject: [PATCH 08/30] Merge pull request #25771 from fengyuentau:vittrack_black_input video: fix vittrack in the case where crop size grows until out-of-memory when the input is black #25771 Fixes https://github.com/opencv/opencv/issues/25760 ### Pull Request Readiness Checklist See details at https://github.com/opencv/opencv/wiki/How_to_contribute#making-a-good-pull-request - [x] I agree to contribute to the project under Apache 2 License. - [x] To the best of my knowledge, the proposed patch is not based on a code under GPL or another license that is incompatible with OpenCV - [x] The PR is proposed to the proper branch - [x] There is a reference to the original bug report and related work - [x] There is accuracy test, performance test and test data in opencv_extra repository, if applicable Patch to opencv_extra has the same branch name. - [x] The feature is well documented and sample code can be built with the project CMake --- .../video/include/opencv2/video/tracking.hpp | 1 + modules/video/src/tracking/tracker_vit.cpp | 67 ++++++++++--------- modules/video/test/test_trackers.cpp | 4 +- samples/dnn/vit_tracker.cpp | 35 ++++++---- 4 files changed, 57 insertions(+), 50 deletions(-) diff --git a/modules/video/include/opencv2/video/tracking.hpp b/modules/video/include/opencv2/video/tracking.hpp index df34a9f97c..ac3788772a 100644 --- a/modules/video/include/opencv2/video/tracking.hpp +++ b/modules/video/include/opencv2/video/tracking.hpp @@ -920,6 +920,7 @@ public: CV_PROP_RW int target; CV_PROP_RW Scalar meanvalue; CV_PROP_RW Scalar stdvalue; + CV_PROP_RW float tracking_score_threshold; }; /** @brief Constructor diff --git a/modules/video/src/tracking/tracker_vit.cpp b/modules/video/src/tracking/tracker_vit.cpp index bef42dbb4d..1dfdde2a16 100644 --- a/modules/video/src/tracking/tracker_vit.cpp +++ b/modules/video/src/tracking/tracker_vit.cpp @@ -24,8 +24,8 @@ TrackerVit::~TrackerVit() TrackerVit::Params::Params() { net = "vitTracker.onnx"; - meanvalue = Scalar{0.485, 0.456, 0.406}; - stdvalue = Scalar{0.229, 0.224, 0.225}; + meanvalue = Scalar{0.485, 0.456, 0.406}; // normalized mean (already divided by 255) + stdvalue = Scalar{0.229, 0.224, 0.225}; // normalized std (already divided by 255) #ifdef HAVE_OPENCV_DNN backend = dnn::DNN_BACKEND_DEFAULT; target = dnn::DNN_TARGET_CPU; @@ -33,6 +33,7 @@ TrackerVit::Params::Params() backend = -1; // invalid value target = -1; // invalid value #endif + tracking_score_threshold = 0.20f; // safe threshold to filter out black frames } #ifdef HAVE_OPENCV_DNN @@ -48,6 +49,9 @@ public: net.setPreferableBackend(params.backend); net.setPreferableTarget(params.target); + + i2bp.mean = params.meanvalue * 255.0; + i2bp.scalefactor = (1.0 / params.stdvalue) * (1 / 255.0); } void init(InputArray image, const Rect& boundingBox) CV_OVERRIDE; @@ -58,6 +62,7 @@ public: float tracking_score; TrackerVit::Params params; + dnn::Image2BlobParams i2bp; protected: @@ -69,10 +74,9 @@ protected: Mat hanningWindow; dnn::Net net; - Mat image; }; -static void crop_image(const Mat& src, Mat& dst, Rect box, int factor) +static int crop_image(const Mat& src, Mat& dst, Rect box, int factor) { int x = box.x, y = box.y, w = box.width, h = box.height; int crop_sz = cvCeil(sqrt(w * h) * factor); @@ -90,21 +94,16 @@ static void crop_image(const Mat& src, Mat& dst, Rect box, int factor) Rect roi(x1 + x1_pad, y1 + y1_pad, x2 - x2_pad - x1 - x1_pad, y2 - y2_pad - y1 - y1_pad); Mat im_crop = src(roi); copyMakeBorder(im_crop, dst, y1_pad, y2_pad, x1_pad, x2_pad, BORDER_CONSTANT); + + return crop_sz; } void TrackerVitImpl::preprocess(const Mat& src, Mat& dst, Size size) { - Mat mean = Mat(size, CV_32FC3, params.meanvalue); - Mat std = Mat(size, CV_32FC3, params.stdvalue); - mean = dnn::blobFromImage(mean, 1.0, Size(), Scalar(), false); - std = dnn::blobFromImage(std, 1.0, Size(), Scalar(), false); - Mat img; resize(src, img, size); - dst = dnn::blobFromImage(img, 1.0, Size(), Scalar(), false); - dst /= 255; - dst = (dst - mean) / std; + dst = dnn::blobFromImageWithParams(img, i2bp); } static Mat hann1d(int sz, bool centered = true) { @@ -141,22 +140,21 @@ static Mat hann2d(Size size, bool centered = true) { return hanningWindow; } -static Rect returnfromcrop(float x, float y, float w, float h, Rect res_Last) +static void updateLastRect(float cx, float cy, float w, float h, int crop_size, Rect &rect_last) { - int cropwindowwh = 4 * cvFloor(sqrt(res_Last.width * res_Last.height)); - int x0 = res_Last.x + (res_Last.width - cropwindowwh) / 2; - int y0 = res_Last.y + (res_Last.height - cropwindowwh) / 2; - Rect finalres; - finalres.x = cvFloor(x * cropwindowwh + x0); - finalres.y = cvFloor(y * cropwindowwh + y0); - finalres.width = cvFloor(w * cropwindowwh); - finalres.height = cvFloor(h * cropwindowwh); - return finalres; + int x0 = rect_last.x + (rect_last.width - crop_size) / 2; + int y0 = rect_last.y + (rect_last.height - crop_size) / 2; + + float x1 = cx - w / 2, y1 = cy - h / 2; + rect_last.x = cvFloor(x1 * crop_size + x0); + rect_last.y = cvFloor(y1 * crop_size + y0); + rect_last.width = cvFloor(w * crop_size); + rect_last.height = cvFloor(h * crop_size); } void TrackerVitImpl::init(InputArray image_, const Rect &boundingBox_) { - image = image_.getMat().clone(); + Mat image = image_.getMat(); Mat crop; crop_image(image, crop, boundingBox_, 2); Mat blob; @@ -169,9 +167,9 @@ void TrackerVitImpl::init(InputArray image_, const Rect &boundingBox_) bool TrackerVitImpl::update(InputArray image_, Rect &boundingBoxRes) { - image = image_.getMat().clone(); + Mat image = image_.getMat(); Mat crop; - crop_image(image, crop, rect_last, 4); + int crop_size = crop_image(image, crop, rect_last, 4); // crop: [crop_size, crop_size] Mat blob; preprocess(crop, blob, searchSize); net.setInput(blob, "search"); @@ -191,15 +189,18 @@ bool TrackerVitImpl::update(InputArray image_, Rect &boundingBoxRes) minMaxLoc(conf_map, nullptr, &maxVal, nullptr, &maxLoc); tracking_score = static_cast(maxVal); - float cx = (maxLoc.x + offset_map.at(0, maxLoc.y, maxLoc.x)) / 16; - float cy = (maxLoc.y + offset_map.at(1, maxLoc.y, maxLoc.x)) / 16; - float w = size_map.at(0, maxLoc.y, maxLoc.x); - float h = size_map.at(1, maxLoc.y, maxLoc.x); + if (tracking_score >= params.tracking_score_threshold) { + float cx = (maxLoc.x + offset_map.at(0, maxLoc.y, maxLoc.x)) / 16; + float cy = (maxLoc.y + offset_map.at(1, maxLoc.y, maxLoc.x)) / 16; + float w = size_map.at(0, maxLoc.y, maxLoc.x); + float h = size_map.at(1, maxLoc.y, maxLoc.x); - Rect finalres = returnfromcrop(cx - w / 2, cy - h / 2, w, h, rect_last); - rect_last = finalres; - boundingBoxRes = finalres; - return true; + updateLastRect(cx, cy, w, h, crop_size, rect_last); + boundingBoxRes = rect_last; + return true; + } else { + return false; + } } float TrackerVitImpl::getTrackingScore() diff --git a/modules/video/test/test_trackers.cpp b/modules/video/test/test_trackers.cpp index aae4492bd7..1814764987 100644 --- a/modules/video/test/test_trackers.cpp +++ b/modules/video/test/test_trackers.cpp @@ -166,9 +166,7 @@ TEST(vittrack, accuracy_vittrack) cv::TrackerVit::Params params; params.net = model; cv::Ptr tracker = TrackerVit::create(params); - // NOTE: Test threshold was reduced from 0.67 (libjpeg-turbo) to 0.66 (libjpeg 9f), - // becase libjpeg and libjpeg-turbo produce slightly different images - checkTrackingAccuracy(tracker, 0.66); + checkTrackingAccuracy(tracker, 0.64); } }} // namespace opencv_test:: diff --git a/samples/dnn/vit_tracker.cpp b/samples/dnn/vit_tracker.cpp index 02e5cea83f..32974713a6 100644 --- a/samples/dnn/vit_tracker.cpp +++ b/samples/dnn/vit_tracker.cpp @@ -16,6 +16,7 @@ const char *keys = "{ help h | | Print help message }" "{ input i | | Full path to input video folder, the specific camera index. (empty for camera 0) }" "{ net | vitTracker.onnx | Path to onnx model of vitTracker.onnx}" + "{ tracking_score_threshold t | 0.3 | Tracking score threshold. If a bbox of score >= 0.3, it is considered as found }" "{ backend | 0 | Choose one of computation backends: " "0: automatically (by default), " "1: Halide language (http://halide-lang.org/), " @@ -49,6 +50,7 @@ int run(int argc, char** argv) std::string net = parser.get("net"); int backend = parser.get("backend"); int target = parser.get("target"); + float tracking_score_threshold = parser.get("tracking_score_threshold"); Ptr tracker; try @@ -57,6 +59,7 @@ int run(int argc, char** argv) params.net = samples::findFile(net); params.backend = backend; params.target = target; + params.tracking_score_threshold = tracking_score_threshold; tracker = TrackerVit::create(params); } catch (const cv::Exception& ee) @@ -108,6 +111,11 @@ int run(int argc, char** argv) Rect selectRect = selectROI(winName, image_select); std::cout << "ROI=" << selectRect << std::endl; + if (selectRect.empty()) + { + std::cerr << "Invalid ROI!" << std::endl; + return 2; + } tracker->init(image, selectRect); @@ -130,30 +138,29 @@ int run(int argc, char** argv) float score = tracker->getTrackingScore(); - std::cout << "frame " << count << - ": predicted score=" << score << - " rect=" << rect << - " time=" << tickMeter.getTimeMilli() << "ms" << - std::endl; + std::cout << "frame " << count; + if (ok) { + std::cout << ": predicted score=" << score << + "\trect=" << rect << + "\ttime=" << tickMeter.getTimeMilli() << "ms" << std::endl; - Mat render_image = image.clone(); - - if (ok) - { - rectangle(render_image, rect, Scalar(0, 255, 0), 2); + rectangle(image, rect, Scalar(0, 255, 0), 2); std::string timeLabel = format("Inference time: %.2f ms", tickMeter.getTimeMilli()); std::string scoreLabel = format("Score: %f", score); - putText(render_image, timeLabel, Point(0, 15), FONT_HERSHEY_SIMPLEX, 0.5, Scalar(0, 255, 0)); - putText(render_image, scoreLabel, Point(0, 35), FONT_HERSHEY_SIMPLEX, 0.5, Scalar(0, 255, 0)); + putText(image, timeLabel, Point(0, 15), FONT_HERSHEY_SIMPLEX, 0.5, Scalar(0, 255, 0)); + putText(image, scoreLabel, Point(0, 35), FONT_HERSHEY_SIMPLEX, 0.5, Scalar(0, 255, 0)); + } else { + std::cout << ": target lost" << std::endl; + putText(image, "Target lost", Point(0, 15), FONT_HERSHEY_SIMPLEX, 0.5, Scalar(0, 0, 255)); } - imshow(winName, render_image); + imshow(winName, image); tickMeter.reset(); int c = waitKey(1); - if (c == 27 /*ESC*/) + if (c == 27 /*ESC*/ || c == 'q' || c == 'Q') break; } From 553c111c5aa3ec9e706930d228b76159553b697f Mon Sep 17 00:00:00 2001 From: Alexander Smorkalov Date: Tue, 18 Jun 2024 15:44:26 +0300 Subject: [PATCH 09/30] Fixed input buffer read overflow in vectorized G-API convertTo implementation. --- modules/gapi/src/backends/fluid/gfluidcore_func.simd.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/gapi/src/backends/fluid/gfluidcore_func.simd.hpp b/modules/gapi/src/backends/fluid/gfluidcore_func.simd.hpp index 6191e9ab05..dead5e30c2 100644 --- a/modules/gapi/src/backends/fluid/gfluidcore_func.simd.hpp +++ b/modules/gapi/src/backends/fluid/gfluidcore_func.simd.hpp @@ -2915,7 +2915,7 @@ CV_ALWAYS_INLINE typename std::enable_if::type convertto_simd_nocoeff_impl(const uchar* inx, DST* outx) { - v_uint8 a = vx_load(inx); + v_uint8 a = vx_load_low(inx); v_uint16 res = v_expand_low(a); store_i16(outx, res); From 766f3f5fc41729ca8f050f042f940f27b93b4dfd Mon Sep 17 00:00:00 2001 From: Rostislav Vasilikhin Date: Tue, 18 Jun 2024 17:15:41 +0200 Subject: [PATCH 10/30] Merge pull request #25746 from savuor:rv/hwasan_flag_release Android SDK build script: HWAsan flags added for release mode #25746 A quick fix for #25718 ### Pull Request Readiness Checklist See details at https://github.com/opencv/opencv/wiki/How_to_contribute#making-a-good-pull-request - [x] I agree to contribute to the project under Apache 2 License. - [x] To the best of my knowledge, the proposed patch is not based on a code under GPL or another license that is incompatible with OpenCV - [x] The PR is proposed to the proper branch - [x] There is a reference to the original bug report and related work - [x] There is accuracy test, performance test and test data in opencv_extra repository, if applicable Patch to opencv_extra has the same branch name. - [x] The feature is well documented and sample code can be built with the project CMake --- platforms/android/build_sdk.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/platforms/android/build_sdk.py b/platforms/android/build_sdk.py index 7ba3fb8e0d..83f40904d5 100755 --- a/platforms/android/build_sdk.py +++ b/platforms/android/build_sdk.py @@ -267,10 +267,14 @@ class Builder: cmake_vars['WITH_ANDROID_MEDIANDK'] = "OFF" if self.hwasan and "arm64" in abi.name: + cmake_vars['OPENCV_ENABLE_MEMORY_SANITIZER'] = "ON" hwasan_flags = "-fno-omit-frame-pointer -fsanitize=hwaddress" - cmake_vars['CMAKE_CXX_FLAGS_DEBUG'] = hwasan_flags - cmake_vars['CMAKE_C_FLAGS_DEBUG'] = hwasan_flags - cmake_vars['CMAKE_LINKER_FLAGS_DEBUG'] = hwasan_flags + for s in ['OPENCV_EXTRA_C_FLAGS', 'OPENCV_EXTRA_CXX_FLAGS', 'OPENCV_EXTRA_EXE_LINKER_FLAGS', + 'OPENCV_EXTRA_SHARED_LINKER_FLAGS', 'OPENCV_EXTRA_MODULE_LINKER_FLAGS']: + if s in cmake_vars.keys(): + cmake_vars[s] = cmake_vars[s] + ' ' + hwasan_flags + else: + cmake_vars[s] = hwasan_flags cmake_vars.update(abi.cmake_vars) From e7108f48ab4c10f215006e57f2b101459da11bac Mon Sep 17 00:00:00 2001 From: Alexander Smorkalov Date: Wed, 19 Jun 2024 15:29:07 +0300 Subject: [PATCH 11/30] Extended bilateralFilter test to cover more branches. --- modules/imgproc/perf/perf_bilateral.cpp | 8 +++++--- modules/imgproc/test/test_bilateral_filter.cpp | 2 +- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/modules/imgproc/perf/perf_bilateral.cpp b/modules/imgproc/perf/perf_bilateral.cpp index ed6a741e08..38f2a503d3 100644 --- a/modules/imgproc/perf/perf_bilateral.cpp +++ b/modules/imgproc/perf/perf_bilateral.cpp @@ -7,23 +7,25 @@ namespace opencv_test { CV_ENUM(Mat_Type, CV_8UC1, CV_8UC3, CV_32FC1, CV_32FC3) -typedef TestBaseWithParam< tuple > TestBilateralFilter; +typedef TestBaseWithParam< tuple > TestBilateralFilter; PERF_TEST_P( TestBilateralFilter, BilateralFilter, Combine( Values( szVGA, sz1080p ), // image size Values( 3, 5 ), // d - Mat_Type::all() // image type + Mat_Type::all(), // image type + Values(1., 5.) ) ) { Size sz; int d, type; - const double sigmaColor = 1., sigmaSpace = 1.; + double sigmaColor, sigmaSpace; sz = get<0>(GetParam()); d = get<1>(GetParam()); type = get<2>(GetParam()); + sigmaColor = sigmaSpace = get<3>(GetParam()); Mat src(sz, type); Mat dst(sz, type); diff --git a/modules/imgproc/test/test_bilateral_filter.cpp b/modules/imgproc/test/test_bilateral_filter.cpp index badd879a8a..0f92836808 100644 --- a/modules/imgproc/test/test_bilateral_filter.cpp +++ b/modules/imgproc/test/test_bilateral_filter.cpp @@ -243,7 +243,7 @@ namespace opencv_test { namespace { rng.fill(_src, RNG::UNIFORM, 0, 256); - _sigma_color = _sigma_space = 1.; + _sigma_color = _sigma_space = rng.uniform(0., 10.); return 1; } From a102b24285de7abc982582a63ed72e027cea87cf Mon Sep 17 00:00:00 2001 From: Alexander Smorkalov Date: Wed, 19 Jun 2024 14:23:53 +0300 Subject: [PATCH 12/30] Added LUT for FP16 and accuracy test. --- modules/core/src/lut.cpp | 7 ++- modules/core/test/test_arithm.cpp | 98 +++++++++++++++++++++++++++++++ 2 files changed, 104 insertions(+), 1 deletion(-) diff --git a/modules/core/src/lut.cpp b/modules/core/src/lut.cpp index 031f113bec..62c02d2b84 100644 --- a/modules/core/src/lut.cpp +++ b/modules/core/src/lut.cpp @@ -56,6 +56,11 @@ static void LUT8u_32s( const uchar* src, const int* lut, int* dst, int len, int LUT8u_( src, lut, dst, len, cn, lutcn ); } +static void LUT8u_16f( const uchar* src, const hfloat* lut, hfloat* dst, int len, int cn, int lutcn ) +{ + LUT8u_( src, lut, dst, len, cn, lutcn ); +} + static void LUT8u_32f( const uchar* src, const float* lut, float* dst, int len, int cn, int lutcn ) { LUT8u_( src, lut, dst, len, cn, lutcn ); @@ -71,7 +76,7 @@ typedef void (*LUTFunc)( const uchar* src, const uchar* lut, uchar* dst, int len static LUTFunc lutTab[CV_DEPTH_MAX] = { (LUTFunc)LUT8u_8u, (LUTFunc)LUT8u_8s, (LUTFunc)LUT8u_16u, (LUTFunc)LUT8u_16s, - (LUTFunc)LUT8u_32s, (LUTFunc)LUT8u_32f, (LUTFunc)LUT8u_64f, 0 + (LUTFunc)LUT8u_32s, (LUTFunc)LUT8u_32f, (LUTFunc)LUT8u_64f, (LUTFunc)LUT8u_16f }; #ifdef HAVE_OPENCL diff --git a/modules/core/test/test_arithm.cpp b/modules/core/test/test_arithm.cpp index 6e4151c0cc..20c0cbce26 100644 --- a/modules/core/test/test_arithm.cpp +++ b/modules/core/test/test_arithm.cpp @@ -3196,5 +3196,103 @@ INSTANTIATE_TEST_CASE_P(Core_CartPolar, Core_PolarToCart_inplace, ) ); +CV_ENUM(LutMatType, CV_8U, CV_16U, CV_16F, CV_32S, CV_32F, CV_64F) + +struct Core_LUT: public testing::TestWithParam +{ + template + cv::Mat referenceWithType(cv::Mat input, cv::Mat table) + { + cv::Mat ref(input.size(), CV_MAKE_TYPE(table.type(), ch)); + for (int i = 0; i < input.rows; i++) + { + for (int j = 0; j < input.cols; j++) + { + if(ch == 1) + { + ref.at(i, j) = table.at(input.at(i, j)); + } + else + { + Vec val; + for (int k = 0; k < ch; k++) + { + val[k] = table.at(input.at>(i, j)[k]); + } + ref.at>(i, j) = val; + } + } + } + return ref; + } + + template + cv::Mat reference(cv::Mat input, cv::Mat table) + { + if (table.type() == CV_8U) + { + return referenceWithType(input, table); + } + else if (table.type() == CV_16U) + { + return referenceWithType(input, table); + } + else if (table.type() == CV_16F) + { + return referenceWithType(input, table); + } + else if (table.type() == CV_32S) + { + return referenceWithType(input, table); + } + else if (table.type() == CV_32F) + { + return referenceWithType(input, table); + } + else if (table.type() == CV_64F) + { + return referenceWithType(input, table); + } + + return cv::Mat(); + } +}; + +TEST_P(Core_LUT, accuracy) +{ + int type = GetParam(); + cv::Mat input(117, 113, CV_8UC1); + randu(input, 0, 256); + + cv::Mat table(1, 256, CV_MAKE_TYPE(type, 1)); + randu(table, 0, 127); + + cv::Mat output; + cv::LUT(input, table, output); + + cv::Mat gt = reference(input, table); + + ASSERT_EQ(0, cv::norm(output, gt, cv::NORM_INF)); +} + +TEST_P(Core_LUT, accuracy_multi) +{ + int type = (int)GetParam(); + cv::Mat input(117, 113, CV_8UC3); + randu(input, 0, 256); + + cv::Mat table(1, 256, CV_MAKE_TYPE(type, 1)); + randu(table, 0, 127); + + cv::Mat output; + cv::LUT(input, table, output); + + cv::Mat gt = reference<3>(input, table); + + ASSERT_EQ(0, cv::norm(output, gt, cv::NORM_INF)); +} + + +INSTANTIATE_TEST_CASE_P(/**/, Core_LUT, LutMatType::all()); }} // namespace From 7ef42d7706d627f55fff9879794ee9d68af2a264 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20K=C3=A4mpe?= Date: Thu, 20 Jun 2024 19:05:06 +0200 Subject: [PATCH 13/30] Merge pull request #25751 from simonkampe:fix-eigen-rowmajor Add missing cv2eigen overload #25751 Fixes #16606 Add overloads to cv2eigen to handle eigen matrices of type Eigen::Matrix ### Pull Request Readiness Checklist See details at https://github.com/opencv/opencv/wiki/How_to_contribute#making-a-good-pull-request - [x] I agree to contribute to the project under Apache 2 License. - [x] To the best of my knowledge, the proposed patch is not based on a code under GPL or another license that is incompatible with OpenCV - [x] The PR is proposed to the proper branch - [x] There is a reference to the original bug report and related work - [ ] There is accuracy test, performance test and test data in opencv_extra repository, if applicable Patch to opencv_extra has the same branch name. - [ ] The feature is well documented and sample code can be built with the project CMake --- modules/core/include/opencv2/core/eigen.hpp | 22 +++++++++++++++++++++ modules/core/test/test_mat.cpp | 21 ++++++++++++++++++++ 2 files changed, 43 insertions(+) diff --git a/modules/core/include/opencv2/core/eigen.hpp b/modules/core/include/opencv2/core/eigen.hpp index 231c6805c0..d4b1774cf9 100644 --- a/modules/core/include/opencv2/core/eigen.hpp +++ b/modules/core/include/opencv2/core/eigen.hpp @@ -287,6 +287,17 @@ void cv2eigen( const Mat& src, } } +template static inline +void cv2eigen( const Mat& src, + Eigen::Matrix<_Tp, Eigen::Dynamic, Eigen::Dynamic, Eigen::RowMajor>& dst ) +{ + CV_CheckEQ(src.dims, 2, ""); + dst.resize(src.rows, src.cols); + const Mat _dst(src.rows, src.cols, traits::Type<_Tp>::value, + dst.data(), (size_t)(dst.outerStride()*sizeof(_Tp))); + src.convertTo(_dst, _dst.type()); +} + // Matx case template static inline void cv2eigen( const Matx<_Tp, _rows, _cols>& src, @@ -307,6 +318,17 @@ void cv2eigen( const Matx<_Tp, _rows, _cols>& src, } } +template static inline +void cv2eigen( const Matx<_Tp, _rows, _cols>& src, + Eigen::Matrix<_Tp, Eigen::Dynamic, Eigen::Dynamic, Eigen::RowMajor>& dst ) +{ + CV_CheckEQ(src.dims, 2, ""); + dst.resize(_rows, _cols); + const Mat _dst(_rows, _cols, traits::Type<_Tp>::value, + dst.data(), (size_t)(dst.outerStride()*sizeof(_Tp))); + Mat(src).copyTo(_dst); +} + template static inline void cv2eigen( const Mat& src, Eigen::Matrix<_Tp, Eigen::Dynamic, 1>& dst ) diff --git a/modules/core/test/test_mat.cpp b/modules/core/test/test_mat.cpp index d13fd96f57..e0ec60955c 100644 --- a/modules/core/test/test_mat.cpp +++ b/modules/core/test/test_mat.cpp @@ -2259,6 +2259,27 @@ TEST(Core_Eigen, eigen2cv_check_Mat_type) EXPECT_ANY_THROW(eigen2cv(eigen_A, d_mat)); //EXPECT_EQ(CV_64FC1, d_mat.type()); } + +TEST(Core_Eigen, cv2eigen_check_RowMajor) +{ + Mat A(3, 2, CV_32FC1, Scalar::all(0)); + A.at(0,0) = 1.0; + A.at(0,1) = 2.0; + A.at(1,0) = 3.0; + A.at(1,1) = 4.0; + A.at(2,0) = 5.0; + A.at(2,1) = 6.0; + + Eigen::Matrix eigen_A; + EXPECT_NO_THROW(cv2eigen(A, eigen_A)); + + ASSERT_EQ(1.0, eigen_A(0, 0)); + ASSERT_EQ(2.0, eigen_A(0, 1)); + ASSERT_EQ(3.0, eigen_A(1, 0)); + ASSERT_EQ(4.0, eigen_A(1, 1)); + ASSERT_EQ(5.0, eigen_A(2, 0)); + ASSERT_EQ(6.0, eigen_A(2, 1)); +} #endif // HAVE_EIGEN #ifdef OPENCV_EIGEN_TENSOR_SUPPORT From 5dc1b39e4c9dfb3339e0b910f7d824a02474ceed Mon Sep 17 00:00:00 2001 From: Ujjayant Kadian <118752727+ujjayant-kadian@users.noreply.github.com> Date: Fri, 21 Jun 2024 12:34:26 +0100 Subject: [PATCH 14/30] Merge pull request #25791 from ujjayant-kadian:uk/extend-gapi-onnx-params-arbitrary-session-options Extending G-API onnx::Params to pass arbitrary session options #25791 ### Pull Request Readiness Checklist See details at https://github.com/opencv/opencv/wiki/How_to_contribute#making-a-good-pull-request - [ ] I agree to contribute to the project under Apache 2 License. - [ ] To the best of my knowledge, the proposed patch is not based on a code under GPL or another license that is incompatible with OpenCV - [ ] The PR is proposed to the proper branch - [ ] There is a reference to the original bug report and related work - [ ] There is accuracy test, performance test and test data in opencv_extra repository, if applicable Patch to opencv_extra has the same branch name. - [ ] The feature is well documented and sample code can be built with the project CMake --- .../opencv2/gapi/infer/bindings_onnx.hpp | 3 +++ .../gapi/include/opencv2/gapi/infer/onnx.hpp | 21 ++++++++++++++++++- .../gapi/src/backends/onnx/bindings_onnx.cpp | 6 ++++++ .../gapi/src/backends/onnx/gonnxbackend.cpp | 4 ++++ 4 files changed, 33 insertions(+), 1 deletion(-) diff --git a/modules/gapi/include/opencv2/gapi/infer/bindings_onnx.hpp b/modules/gapi/include/opencv2/gapi/infer/bindings_onnx.hpp index fb2376ece8..0b6dab6a9d 100644 --- a/modules/gapi/include/opencv2/gapi/infer/bindings_onnx.hpp +++ b/modules/gapi/include/opencv2/gapi/infer/bindings_onnx.hpp @@ -51,6 +51,9 @@ public: GAPI_WRAP PyParams& cfgDisableMemPattern(); + GAPI_WRAP + PyParams& cfgSessionOptions(const std::map& options); + GBackend backend() const; std::string tag() const; cv::util::any params() const; diff --git a/modules/gapi/include/opencv2/gapi/infer/onnx.hpp b/modules/gapi/include/opencv2/gapi/infer/onnx.hpp index f985b41d71..fd0f69a768 100644 --- a/modules/gapi/include/opencv2/gapi/infer/onnx.hpp +++ b/modules/gapi/include/opencv2/gapi/infer/onnx.hpp @@ -351,6 +351,7 @@ struct ParamDesc { std::unordered_map > generic_mstd; std::unordered_map generic_norm; + std::map session_options; std::vector execution_providers; bool disable_mem_pattern; }; @@ -634,6 +635,19 @@ public: return *this; } + /** @brief Configures session options for ONNX Runtime. + + This function is used to set various session options for the ONNX Runtime + session by accepting a map of key-value pairs. + + @param options A map of session option to be applied to the ONNX Runtime session. + @return the reference on modified object. + */ + Params& cfgSessionOptions(const std::map& options) { + desc.session_options.insert(options.begin(), options.end()); + return *this; + } + // BEGIN(G-API's network parametrization API) GBackend backend() const { return cv::gapi::onnx::backend(); } std::string tag() const { return Net::tag(); } @@ -661,7 +675,7 @@ public: @param model_path path to model file (.onnx file). */ Params(const std::string& tag, const std::string& model_path) - : desc{model_path, 0u, 0u, {}, {}, {}, {}, {}, {}, {}, {}, {}, true, {}, {}, {}, false }, m_tag(tag) {} + : desc{model_path, 0u, 0u, {}, {}, {}, {}, {}, {}, {}, {}, {}, true, {}, {}, {}, {}, false}, m_tag(tag) {} /** @see onnx::Params::cfgMeanStdDev. */ void cfgMeanStdDev(const std::string &layer, @@ -705,6 +719,11 @@ public: desc.disable_mem_pattern = true; } + /** @see onnx::Params::cfgSessionOptions. */ + void cfgSessionOptions(const std::map& options) { + desc.session_options.insert(options.begin(), options.end()); + } + // BEGIN(G-API's network parametrization API) GBackend backend() const { return cv::gapi::onnx::backend(); } std::string tag() const { return m_tag; } diff --git a/modules/gapi/src/backends/onnx/bindings_onnx.cpp b/modules/gapi/src/backends/onnx/bindings_onnx.cpp index 0703f1753d..294ad8a3cc 100644 --- a/modules/gapi/src/backends/onnx/bindings_onnx.cpp +++ b/modules/gapi/src/backends/onnx/bindings_onnx.cpp @@ -57,6 +57,12 @@ cv::gapi::onnx::PyParams::cfgDisableMemPattern() { return *this; } +cv::gapi::onnx::PyParams& +cv::gapi::onnx::PyParams::cfgSessionOptions(const std::map& options) { + m_priv->cfgSessionOptions(options); + return *this; +} + cv::gapi::GBackend cv::gapi::onnx::PyParams::backend() const { return m_priv->backend(); } diff --git a/modules/gapi/src/backends/onnx/gonnxbackend.cpp b/modules/gapi/src/backends/onnx/gonnxbackend.cpp index 92b908da70..e465a1ccc7 100644 --- a/modules/gapi/src/backends/onnx/gonnxbackend.cpp +++ b/modules/gapi/src/backends/onnx/gonnxbackend.cpp @@ -702,6 +702,10 @@ ONNXCompiled::ONNXCompiled(const gapi::onnx::detail::ParamDesc &pp) cv::gimpl::onnx::addExecutionProvider(&session_options, ep); } + for (const auto &option : pp.session_options) { + session_options.AddConfigEntry(option.first.c_str(), option.second.c_str()); + } + if (pp.disable_mem_pattern) { session_options.DisableMemPattern(); } From 3f13ce797b89b4bc83a3caddc682ea2158c3f32c Mon Sep 17 00:00:00 2001 From: Yuantao Feng Date: Sat, 22 Jun 2024 00:28:22 +0800 Subject: [PATCH 15/30] Merge pull request #25779 from fengyuentau:dnn/fix_onnx_depthtospace dnn: add DepthToSpace and SpaceToDepth #25779 We are working on updating WeChat QRCode module. One of the new models is a fully convolutional model and hence it should be able to run with different input shapes. However, it has an operator `DepthToSpace`, which is parsed as a subgraph of `Reshape -> Permute -> Reshape` with a fixed shape getting during parsing. The subgraph itself is not a problem, but the true problem is the subgraph with a fixed input and output shape regardless input changes. This does not allow the model to run with different input shapes. Solution is to add a dedicated layer for DepthtoSpace and SpaceToDepth. Backend support: - [x] CPU - [x] CUDA - [x] OpenCL - [x] OpenVINO - [x] CANN - [x] TIMVX - ~Vulkan~ (missing fundamental tools, like permutation and reshape) ### Pull Request Readiness Checklist See details at https://github.com/opencv/opencv/wiki/How_to_contribute#making-a-good-pull-request - [x] I agree to contribute to the project under Apache 2 License. - [x] To the best of my knowledge, the proposed patch is not based on a code under GPL or another license that is incompatible with OpenCV - [x] The PR is proposed to the proper branch - [x] There is a reference to the original bug report and related work - [x] There is accuracy test, performance test and test data in opencv_extra repository, if applicable Patch to opencv_extra has the same branch name. - [x] The feature is well documented and sample code can be built with the project CMake --- .../dnn/include/opencv2/dnn/all_layers.hpp | 10 + modules/dnn/src/cuda4dnn/csl/tensor.hpp | 29 +- .../cuda4dnn/primitives/depth_space_ops.hpp | 76 +++ modules/dnn/src/init.cpp | 4 + .../dnn/src/layers/depth_space_ops_layer.cpp | 492 ++++++++++++++++++ modules/dnn/src/onnx/onnx_importer.cpp | 82 +-- modules/dnn/test/test_int8_layers.cpp | 55 ++ ...conformance_layer_filter__openvino.inl.hpp | 10 + 8 files changed, 663 insertions(+), 95 deletions(-) create mode 100644 modules/dnn/src/cuda4dnn/primitives/depth_space_ops.hpp create mode 100644 modules/dnn/src/layers/depth_space_ops_layer.cpp diff --git a/modules/dnn/include/opencv2/dnn/all_layers.hpp b/modules/dnn/include/opencv2/dnn/all_layers.hpp index 3301f20fde..2abce0c87b 100644 --- a/modules/dnn/include/opencv2/dnn/all_layers.hpp +++ b/modules/dnn/include/opencv2/dnn/all_layers.hpp @@ -1188,6 +1188,16 @@ CV__DNN_INLINE_NS_BEGIN static Ptr create(const LayerParams ¶ms); }; + class CV_EXPORTS DepthToSpaceLayer : public Layer { + public: + static Ptr create(const LayerParams ¶ms); + }; + + class CV_EXPORTS SpaceToDepthLayer : public Layer { + public: + static Ptr create(const LayerParams ¶ms); + }; + //! @} //! @} CV__DNN_INLINE_NS_END diff --git a/modules/dnn/src/cuda4dnn/csl/tensor.hpp b/modules/dnn/src/cuda4dnn/csl/tensor.hpp index 8f495ac807..15cda2fff5 100644 --- a/modules/dnn/src/cuda4dnn/csl/tensor.hpp +++ b/modules/dnn/src/cuda4dnn/csl/tensor.hpp @@ -265,7 +265,6 @@ namespace cv { namespace dnn { namespace cuda4dnn { namespace csl { typename std::enable_if::value, void> ::type reshape(ForwardItr start, ForwardItr end) { CV_Assert(start != end); - CV_Assert(std::distance(start, end) <= rank()); using ItrValueType = typename std::iterator_traits::value_type; @@ -284,6 +283,7 @@ namespace cv { namespace dnn { namespace cuda4dnn { namespace csl { auto total = std::accumulate(start, end, 1, std::multiplies()); if (total < 0) { /* there is an unknown size */ + CV_CheckEQ(size() % std::abs(total), static_cast(0), "cannot be reshaped"); // must be divisible if (std::abs(total) <= size()) { unknown_size = size() / std::abs(total); total = size(); @@ -298,11 +298,9 @@ namespace cv { namespace dnn { namespace cuda4dnn { namespace csl { CV_Error(Error::StsBadArg, "new axes do not preserve the tensor element count"); } - /* we assume the size of the unspecified axes to be one */ - std::fill(std::begin(shape), std::end(shape), 1); - std::copy_backward(start, end, std::end(shape)); - - /* replace the unknown axis with the correct value */ + /* copy shape from given iterator and reshape -1 with deduced value */ + shape.resize(std::distance(start, end)); + std::copy(start, end, shape.begin()); std::replace(std::begin(shape), std::end(shape), size_type(-1), unknown_size); } @@ -600,6 +598,7 @@ namespace cv { namespace dnn { namespace cuda4dnn { namespace csl { auto total = std::accumulate(start, end, 1, std::multiplies()); if (total < 0) { /* there is an unknown size */ + CV_CheckEQ(size() % std::abs(total), static_cast(0), "cannot be reshaped"); // must be divisible if (std::abs(total) <= size()) { unknown_size = size() / std::abs(total); total = size(); @@ -614,11 +613,9 @@ namespace cv { namespace dnn { namespace cuda4dnn { namespace csl { CV_Error(Error::StsBadArg, "new axes do not preserve the tensor element count"); } - /* we assume the size of the unspecified axes to be one */ - std::fill(std::begin(shape), std::end(shape), 1); - std::copy_backward(start, end, std::end(shape)); - - /* replace the unknown axis with the correct value */ + /* copy shape from given iterator and reshape -1 with deduced value */ + shape.resize(std::distance(start, end)); + std::copy(start, end, shape.begin()); std::replace(std::begin(shape), std::end(shape), size_type(-1), unknown_size); } @@ -946,7 +943,6 @@ namespace cv { namespace dnn { namespace cuda4dnn { namespace csl { typename std::enable_if::value, void> ::type reshape(ForwardItr start, ForwardItr end) { CV_Assert(start != end); - CV_Assert(std::distance(start, end) <= rank()); using ItrValueType = typename std::iterator_traits::value_type; @@ -965,6 +961,7 @@ namespace cv { namespace dnn { namespace cuda4dnn { namespace csl { auto total = std::accumulate(start, end, 1, std::multiplies()); if (total < 0) { /* there is an unknown size */ + CV_CheckEQ(size() % std::abs(total), static_cast(0), "cannot be reshaped"); // must be divisible if (std::abs(total) <= size()) { unknown_size = size() / std::abs(total); total = size(); @@ -979,11 +976,9 @@ namespace cv { namespace dnn { namespace cuda4dnn { namespace csl { CV_Error(Error::StsBadArg, "new axes do not preserve the tensor element count"); } - /* we assume the size of the unspecified axes to be one */ - std::fill(std::begin(shape), std::end(shape), 1); - std::copy_backward(start, end, std::end(shape)); - - /* replace the unknown axis with the correct value */ + /* copy shape from given iterator and reshape -1 with deduced value */ + shape.resize(std::distance(start, end)); + std::copy(start, end, shape.begin()); std::replace(std::begin(shape), std::end(shape), size_type(-1), unknown_size); } diff --git a/modules/dnn/src/cuda4dnn/primitives/depth_space_ops.hpp b/modules/dnn/src/cuda4dnn/primitives/depth_space_ops.hpp new file mode 100644 index 0000000000..7846881e70 --- /dev/null +++ b/modules/dnn/src/cuda4dnn/primitives/depth_space_ops.hpp @@ -0,0 +1,76 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. + +#ifndef OPENCV_DNN_SRC_CUDA4DNN_PRIMITIVES_DEPTH_SPACE_OPS_HPP +#define OPENCV_DNN_SRC_CUDA4DNN_PRIMITIVES_DEPTH_SPACE_OPS_HPP + +#include "../../op_cuda.hpp" + +#include "../csl/stream.hpp" +#include "../csl/tensor.hpp" +#include "../csl/tensor_ops.hpp" +#include "../csl/memory.hpp" +#include "../kernels/permute.hpp" + +#include + +namespace cv { namespace dnn { namespace cuda4dnn { + + template + class DepthSpaceOps final : public CUDABackendNode { + public: + using wrapper_type = GetCUDABackendWrapperType; + + DepthSpaceOps(csl::Stream stream_, const std::vector &internal_shape_, + const std::vector &permutation_) + : stream(std::move(stream_)), internal_shape(internal_shape_), + permutation(permutation_) + { + transposed_internal_shape = std::vector(internal_shape.size()); + for (size_t i = 0; i < permutation.size(); i++) { + transposed_internal_shape[i] = internal_shape[permutation[i]]; + } + + size_t num_elements = std::accumulate(internal_shape.begin(), internal_shape.end(), 1, std::multiplies()); + csl::WorkspaceBuilder builder; + builder.require(num_elements); + scratch_mem_in_bytes = builder.required_workspace_size(); + } + + void forward(const std::vector> &inputs, + const std::vector> &outputs, + csl::Workspace &workspace) override { + CV_CheckEQ(inputs.size(), size_t(1), "DepthSpaceOps: only one input is accepted"); + CV_CheckEQ(outputs.size(), size_t(1), "DepthSpaceOps: only one output is accepted"); + + auto input_wrapper = inputs.front().dynamicCast(); + auto input = input_wrapper->getView(); + CV_CheckEQ(input.rank(), size_t(4), "DepthSpaceOps: input needs to be 4-dimensional [N, C, H, W]"); + auto output_wrapper = outputs.front().dynamicCast(); + auto output = output_wrapper->getSpan(); + auto ws_allocator = csl::WorkspaceAllocator(workspace); + auto transposed_internal = ws_allocator.get_tensor_span(transposed_internal_shape.begin(), transposed_internal_shape.end()); + + // Call reshape on input so that it has the correct shape for permutation + input.reshape(internal_shape.begin(), internal_shape.end()); + kernels::permute(stream, transposed_internal, input, permutation); + // Only copying is needed as output already has the expected shape + auto t = csl::TensorView(transposed_internal); + csl::memcpy(output.get(), t.get(), output.size(), stream); + } + + std::size_t get_workspace_memory_in_bytes() const noexcept override { return scratch_mem_in_bytes; } + + private: + csl::Stream stream; + std::vector internal_shape; + std::vector permutation; + std::vector transposed_internal_shape; + + std::size_t scratch_mem_in_bytes; + }; + +}}} // namespace cv::dnn::cuda4dnn + +#endif // OPENCV_DNN_SRC_CUDA4DNN_PRIMITIVES_DEPTH_SPACE_OPS_HPP diff --git a/modules/dnn/src/init.cpp b/modules/dnn/src/init.cpp index e8450c18f9..ce1eb77649 100644 --- a/modules/dnn/src/init.cpp +++ b/modules/dnn/src/init.cpp @@ -164,6 +164,10 @@ void initializeLayerFactory() CV_DNN_REGISTER_LAYER_CLASS(InstanceNormalization, InstanceNormLayer); CV_DNN_REGISTER_LAYER_CLASS(Attention, AttentionLayer); CV_DNN_REGISTER_LAYER_CLASS(GroupNormalization, GroupNormLayer); + CV_DNN_REGISTER_LAYER_CLASS(DepthToSpace, DepthToSpaceLayer) + CV_DNN_REGISTER_LAYER_CLASS(SpaceToDepth, SpaceToDepthLayer) + CV_DNN_REGISTER_LAYER_CLASS(DepthToSpaceInt8, DepthToSpaceLayer) + CV_DNN_REGISTER_LAYER_CLASS(SpaceToDepthInt8, SpaceToDepthLayer) CV_DNN_REGISTER_LAYER_CLASS(Crop, CropLayer); CV_DNN_REGISTER_LAYER_CLASS(Eltwise, EltwiseLayer); diff --git a/modules/dnn/src/layers/depth_space_ops_layer.cpp b/modules/dnn/src/layers/depth_space_ops_layer.cpp new file mode 100644 index 0000000000..3877758b20 --- /dev/null +++ b/modules/dnn/src/layers/depth_space_ops_layer.cpp @@ -0,0 +1,492 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. + +#include "../precomp.hpp" +#include + +// OpenCL backend +#ifdef HAVE_OPENCL +#include "opencl_kernels_dnn.hpp" +#endif + +// OpenVINO backend +#ifdef HAVE_DNN_NGRAPH +#include "../op_inf_engine.hpp" +#include "../ie_ngraph.hpp" +#endif + +// CUDA backend +#ifdef HAVE_CUDA +#include "../op_cuda.hpp" +#include "../cuda4dnn/primitives/depth_space_ops.hpp" +#endif + +// CANN backend +#ifdef HAVE_CANN +#include "../op_cann.hpp" +#endif + +// TIM-VX backend +#ifdef HAVE_TIMVX +#include "../op_timvx.hpp" +#endif + +namespace cv { namespace dnn { + +struct DepthSpaceOps { + MatShape internal_shape; + MatShape transposed_internal_shape; + std::vector permutation; + +#ifdef HAVE_OPENCL + UMat umat_permutation; + UMat umat_internal_strides; + UMat umat_transposed_internal_strides; +#endif + + void finalize(InputArrayOfArrays inputs_arr, OutputArrayOfArrays outputs_arr) { + transposed_internal_shape = MatShape(internal_shape.size()); + for (size_t i = 0; i < permutation.size(); i++) { + transposed_internal_shape[i] = internal_shape[permutation[i]]; + } + +#ifdef HAVE_OPENCL + umat_permutation.release(); + umat_internal_strides.release(); + umat_transposed_internal_strides.release(); +#endif + } + + void cpuCompute(const Mat &input, Mat &output) { + const auto output_shape = shape(output); + Mat tmp; + cv::transposeND(input.reshape(1, internal_shape), permutation, tmp); + tmp.reshape(1, output_shape).copyTo(output); + } + +#ifdef HAVE_OPENCL + bool oclCompute(InputArrayOfArrays inputs_arr, OutputArrayOfArrays outputs_arr, OutputArrayOfArrays internals_arr) { + std::vector inputs, outputs; + + inputs_arr.getUMatVector(inputs); + outputs_arr.getUMatVector(outputs); + + if (umat_permutation.empty() || umat_internal_strides.empty() || umat_transposed_internal_strides.empty()) { + Mat mat_permutation(1, permutation.size(), CV_32S, permutation.data()); + mat_permutation.copyTo(umat_permutation); + + std::vector internal_strides(permutation.size(), 1), transposed_internal_stides(permutation.size(), 1); + for (int i = static_cast(permutation.size()) - 2; i >= 0; i--) { + internal_strides[i] = internal_strides[i + 1] * internal_shape[i + 1]; + transposed_internal_stides[i] = transposed_internal_stides[i + 1] * transposed_internal_shape[i + 1]; + } + Mat mat_internal_strides(1, internal_strides.size(), CV_32S, internal_strides.data()); + mat_internal_strides.copyTo(umat_internal_strides); + Mat mat_transposed_internal_strides(1, transposed_internal_stides.size(), CV_32S, transposed_internal_stides.data()); + mat_transposed_internal_strides.copyTo(umat_transposed_internal_strides); + } + + const auto output_shape = shape(outputs.front()); + UMat tmp = inputs.front().reshape(1, static_cast(internal_shape.size()), internal_shape.data()); + + bool use_half = (inputs_arr.depth() == CV_16F); + std::string permute_options = cv::format("-DDtype=%s", use_half ? "half" : "float"); + ocl::Kernel permute_kernel("permute", ocl::dnn::permute_oclsrc, permute_options); + if (permute_kernel.empty()) { + return false; + } + UMat transposed_tmp(static_cast(transposed_internal_shape.size()), transposed_internal_shape.data(), inputs_arr.depth()); + size_t num_element = static_cast(std::accumulate(internal_shape.begin(), internal_shape.end(), 1, std::multiplies())); + permute_kernel.set(0, static_cast(num_element)); + permute_kernel.set(1, ocl::KernelArg::PtrReadOnly(tmp)); + permute_kernel.set(2, ocl::KernelArg::PtrReadOnly(umat_permutation)); + permute_kernel.set(3, ocl::KernelArg::PtrReadOnly(umat_internal_strides)); + permute_kernel.set(4, ocl::KernelArg::PtrReadOnly(umat_transposed_internal_strides)); + permute_kernel.set(5, static_cast(permutation.size())); + permute_kernel.set(6, ocl::KernelArg::PtrWriteOnly(transposed_tmp)); + if (!permute_kernel.run(1, &num_element, NULL, false)) { + return false; + } + + transposed_tmp.reshape(1, static_cast(output_shape.size()), output_shape.data()).copyTo(outputs.front()); + return true; + } +#endif // HAVE_OPENCL +}; + +class DepthToSpaceLayerImpl CV_FINAL : public DepthToSpaceLayer, public DepthSpaceOps { +public: + DepthToSpaceLayerImpl(const LayerParams ¶ms) { + setParamsFrom(params); + + CV_CheckTrue(params.has("blocksize"), "DepthSpaceLayer: blocksize is required"); + blocksize = params.get("blocksize"); + + auto mode = params.get("mode", "DCR"); + if (mode == "CRD") { + is_crd = true; + permutation = {0, 1, 4, 2, 5, 3}; + } else if (mode == "DCR") { + is_crd = false; + permutation = {0, 3, 4, 1, 5, 2}; + } else { + CV_Error(Error::StsBadArg, cv::format("DepthToSpace: unsupported mode %s\n", mode.c_str())); + } + } + + virtual bool supportBackend(int backendId) CV_OVERRIDE { + return backendId == DNN_BACKEND_OPENCV || + backendId == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH || + backendId == DNN_BACKEND_CUDA || + backendId == DNN_BACKEND_CANN || + (backendId == DNN_BACKEND_TIMVX && is_crd); + } + + virtual bool getMemoryShapes(const std::vector &inputs, + const int requiredOutputs, + std::vector &outputs, + std::vector &internals) const CV_OVERRIDE { + CV_CheckEQ(inputs.size(), static_cast(1), "DepthSpaceLayer: accepts only one input"); + const auto &input = inputs.front(); + CV_CheckEQ(input.size(), static_cast(4), "DepthSpaceLayer: input needs to be 4-dimensional [N, C, H, W]"); + int batch = input[0], input_depth = input[1], input_height = input[2], input_width = input[3]; + int output_depth = -1, output_height = -1, output_width = -1; + + CV_CheckEQ(input_depth % (blocksize * blocksize), 0, + "DepthSpaceLayer: requires input depth to be a multiple of (blocksize * blocksize)"); + output_depth = input_depth / blocksize / blocksize; + output_height = input_height * blocksize; + output_width = input_width * blocksize; + + outputs.assign(1, MatShape{batch, output_depth, output_height, output_width}); + return false; + } + + virtual void finalize(InputArrayOfArrays inputs_arr, OutputArrayOfArrays outputs_arr) CV_OVERRIDE { + std::vector inputs; + inputs_arr.getMatVector(inputs); + + auto input_shape = shape(inputs.front()); + int batch = input_shape[0], input_depth = input_shape[1], input_height = input_shape[2], input_width = input_shape[3]; + if (is_crd) { + internal_shape = MatShape{batch, input_depth / (blocksize * blocksize), blocksize, blocksize, input_height, input_width}; + } else { + internal_shape = MatShape{batch, blocksize, blocksize, input_depth / (blocksize * blocksize), input_height, input_width}; + } + + DepthSpaceOps::finalize(inputs_arr, outputs_arr); + } + + void forward(InputArrayOfArrays inputs_arr, OutputArrayOfArrays outputs_arr, OutputArrayOfArrays internals_arr) CV_OVERRIDE { + CV_TRACE_FUNCTION(); + CV_TRACE_ARG_VALUE(name, "name", name.c_str()); + + // TODO: support 8-bit int in permute kernel + CV_OCL_RUN(IS_DNN_OPENCL_TARGET(preferableTarget) && inputs_arr.depth() != CV_8S, + DepthSpaceOps::oclCompute(inputs_arr, outputs_arr, internals_arr)) + + if (inputs_arr.depth() == CV_16F) { + forward_fallback(inputs_arr, outputs_arr, internals_arr); + return; + } + + std::vector inputs, outputs; + inputs_arr.getMatVector(inputs); + outputs_arr.getMatVector(outputs); + + DepthSpaceOps::cpuCompute(inputs.front(), outputs.front()); + } + +#ifdef HAVE_DNN_NGRAPH + virtual Ptr initNgraph(const std::vector> &inputs, + const std::vector> &nodes) CV_OVERRIDE { + using namespace ov::op; + auto input_node = nodes[0].dynamicCast()->node; + std::shared_ptr output_node; + if (is_crd) { + output_node = std::make_shared(input_node, v0::DepthToSpace::DepthToSpaceMode::DEPTH_FIRST, static_cast(blocksize)); + } else { + output_node = std::make_shared(input_node, v0::DepthToSpace::DepthToSpaceMode::BLOCKS_FIRST, static_cast(blocksize)); + } + return Ptr(new InfEngineNgraphNode(output_node)); + } +#endif // HAVE_DNN_NGRAPH + +#ifdef HAVE_CUDA + Ptr initCUDA(void *context_, + const std::vector>& inputs, + const std::vector>& outputs) override { + using namespace cv::dnn::cuda4dnn; + auto context = reinterpret_cast(context_); + std::vector perm(permutation.begin(), permutation.end()); + return make_cuda_node(preferableTarget, std::move(context->stream), internal_shape, perm); + } +#endif // HAVE_CUDA + +#ifdef HAVE_CANN + virtual Ptr initCann(const std::vector> &inputs, + const std::vector> &outputs, + const std::vector> &nodes) CV_OVERRIDE { + CV_CheckEQ(inputs.size(), static_cast(1), "DepthToSpace/CANN: only accepts one input wrapper"); + CV_CheckEQ(nodes.size(), static_cast(1), "DepthToSpace/CANN: only accepts one input node"); + + auto input_tensor_wrapper = inputs.front().dynamicCast(); + auto input_tensor_desc = input_tensor_wrapper->getTensorDesc(); + auto input_node = nodes.front().dynamicCast()->getOp(); + + auto node = std::make_shared(name); + + node->set_attr_block_size(blocksize); + if (is_crd) { + node->set_attr_mode("CRD"); + } else { + node->set_attr_mode("DCR"); + } + node->set_attr_data_format("NCHW"); + + node->set_input_x_by_name(*input_node, input_tensor_wrapper->name.c_str()); + node->update_input_desc_x(*input_tensor_desc); + + auto output_tensor_desc = std::make_shared(ge::Shape(), ge::FORMAT_NCHW, ge::DT_FLOAT); + node->update_output_desc_y(*output_tensor_desc); + + return Ptr(new CannBackendNode(node)); + } +#endif + +#ifdef HAVE_TIMVX + virtual Ptr initTimVX(void* timvx_info_, + const std::vector> &inputs, + const std::vector> &outputs, + bool isLast) CV_OVERRIDE { + auto info = reinterpret_cast(timvx_info_); + CV_Assert(info); + auto timvx_graph = info->getGraph(); + CV_Assert(timvx_graph); + auto graph = timvx_graph->graph; + + auto input_wrapper = inputs.front().dynamicCast(); + int input_wrapper_index = -1; + if (input_wrapper->isTensor()) { + input_wrapper_index = timvx_graph->getTensorIndex(input_wrapper->getTensor()); + if (input_wrapper_index == -1) { + auto tmp = input_wrapper->getMat(); + input_wrapper = std::make_shared(tmp); + } + } + if (!input_wrapper->isTensor() || input_wrapper_index == 1) { + auto input_node_quant = Ptr(new tim::vx::Quantization(tim::vx::QuantType::ASYMMETRIC, 1.0f, 0)); + input_wrapper->createTensor(graph, tim::vx::TensorAttribute::INPUT, input_node_quant); + input_wrapper_index = timvx_graph->addWrapper(input_wrapper); + } + + auto output_wrapper = outputs.front().dynamicCast(); + auto output_node_quant = input_wrapper->getTensorQuantization(); + if (isLast) { + auto shape_type = getShapeTypeFromMat(output_wrapper->getMat()); + output_wrapper->setTensorShape(shape_type); + output_wrapper->createTensor(graph, tim::vx::TensorAttribute::OUTPUT, output_node_quant); + } else { + output_wrapper->createTensor(graph, tim::vx::TensorAttribute::TRANSIENT, output_node_quant); + } + int output_wrapper_index = timvx_graph->addWrapper(output_wrapper); + + std::shared_ptr timvx_node; + timvx_node = graph->CreateOperation(blocksize); + std::vector input_wrapper_indices{input_wrapper_index}, output_wrapper_indices{output_wrapper_index}; + return Ptr(new TimVXBackendNode(timvx_graph, timvx_node, input_wrapper_indices, output_wrapper_indices)); + } +#endif + + virtual bool tryQuantize(const std::vector> &scales, + const std::vector> &zeropoints, LayerParams ¶ms) CV_OVERRIDE { + return true; + } + +private: + int blocksize; + + bool is_crd; +}; + +Ptr DepthToSpaceLayer::create(const LayerParams ¶ms) { + return makePtr(params); +} + +class SpaceToDepthLayerImpl CV_FINAL : public SpaceToDepthLayer, public DepthSpaceOps { +public: + SpaceToDepthLayerImpl(const LayerParams ¶ms) { + setParamsFrom(params); + + CV_CheckTrue(params.has("blocksize"), "SpaceToDepthLayer: blocksize is required"); + blocksize = params.get("blocksize"); + + permutation = {0, 3, 5, 1, 2, 4}; + } + + virtual bool supportBackend(int backendId) CV_OVERRIDE { + return backendId == DNN_BACKEND_OPENCV || + backendId == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH || + backendId == DNN_BACKEND_CUDA || + backendId == DNN_BACKEND_CANN || + (backendId == DNN_BACKEND_TIMVX); + } + + virtual bool getMemoryShapes(const std::vector &inputs, + const int requiredOutputs, + std::vector &outputs, + std::vector &internals) const CV_OVERRIDE { + CV_CheckEQ(inputs.size(), static_cast(1), "SpaceToDepthLayer: accepts only one input"); + const auto &input = inputs.front(); + CV_CheckEQ(input.size(), static_cast(4), "SpaceToDepthLayer: input needs to be 4-dimensional [N, C, H, W]"); + int batch = input[0], input_depth = input[1], input_height = input[2], input_width = input[3]; + int output_depth = -1, output_height = -1, output_width = -1; + + CV_CheckEQ(input_height % blocksize, 0, "SpaceToDepthLayer: requires input height to be a multiple of blocksize"); + CV_CheckEQ(input_width % blocksize, 0, "SpaceToDepthLayer: requires input width to be a multiple of blocksize"); + output_depth = input_depth * blocksize * blocksize; + output_height = input_height / blocksize; + output_width = input_width / blocksize; + + outputs.assign(1, MatShape{batch, output_depth, output_height, output_width}); + return false; + } + + virtual void finalize(InputArrayOfArrays inputs_arr, OutputArrayOfArrays outputs_arr) CV_OVERRIDE { + std::vector inputs; + inputs_arr.getMatVector(inputs); + + auto input_shape = shape(inputs.front()); + int batch = input_shape[0], input_depth = input_shape[1], input_height = input_shape[2], input_width = input_shape[3]; + internal_shape = MatShape{batch, input_depth, input_height / blocksize, blocksize, input_width / blocksize, blocksize}; + + DepthSpaceOps::finalize(inputs_arr, outputs_arr); + } + + void forward(InputArrayOfArrays inputs_arr, OutputArrayOfArrays outputs_arr, OutputArrayOfArrays internals_arr) CV_OVERRIDE { + CV_TRACE_FUNCTION(); + CV_TRACE_ARG_VALUE(name, "name", name.c_str()); + + // TODO: support 8-bit int in permute kernel + CV_OCL_RUN(IS_DNN_OPENCL_TARGET(preferableTarget) && inputs_arr.depth() != CV_8S, + DepthSpaceOps::oclCompute(inputs_arr, outputs_arr, internals_arr)) + + if (inputs_arr.depth() == CV_16F) { + forward_fallback(inputs_arr, outputs_arr, internals_arr); + return; + } + + std::vector inputs, outputs; + inputs_arr.getMatVector(inputs); + outputs_arr.getMatVector(outputs); + + DepthSpaceOps::cpuCompute(inputs.front(), outputs.front()); + } + +#ifdef HAVE_DNN_NGRAPH + virtual Ptr initNgraph(const std::vector> &inputs, + const std::vector> &nodes) CV_OVERRIDE { + using namespace ov::op; + auto input_node = nodes[0].dynamicCast()->node; + std::shared_ptr output_node; + output_node = std::make_shared(input_node, v0::SpaceToDepth::SpaceToDepthMode::BLOCKS_FIRST, static_cast(blocksize)); + return Ptr(new InfEngineNgraphNode(output_node)); + } +#endif // HAVE_DNN_NGRAPH + +#ifdef HAVE_CUDA + Ptr initCUDA(void *context_, + const std::vector> &inputs, + const std::vector> &outputs) override { + using namespace cv::dnn::cuda4dnn; + auto context = reinterpret_cast(context_); + std::vector perm(permutation.begin(), permutation.end()); + return make_cuda_node(preferableTarget, std::move(context->stream), internal_shape, perm); + } +#endif // HAVE_CUDA + +#ifdef HAVE_CANN + virtual Ptr initCann(const std::vector> &inputs, + const std::vector> &outputs, + const std::vector> &nodes) CV_OVERRIDE { + CV_CheckEQ(inputs.size(), static_cast(1), "DepthToSpace/CANN: only accepts one input wrapper"); + CV_CheckEQ(nodes.size(), static_cast(1), "DepthToSpace/CANN: only accepts one input node"); + + auto input_tensor_wrapper = inputs.front().dynamicCast(); + auto input_tensor_desc = input_tensor_wrapper->getTensorDesc(); + auto input_node = nodes.front().dynamicCast()->getOp(); + + auto node = std::make_shared(name); + + node->set_attr_block_size(blocksize); + node->set_attr_data_format("NCHW"); + + node->set_input_x_by_name(*input_node, input_tensor_wrapper->name.c_str()); + node->update_input_desc_x(*input_tensor_desc); + + auto output_tensor_desc = std::make_shared(ge::Shape(), ge::FORMAT_NCHW, ge::DT_FLOAT); + node->update_output_desc_y(*output_tensor_desc); + + return Ptr(new CannBackendNode(node)); + } +#endif + +#ifdef HAVE_TIMVX + virtual Ptr initTimVX(void* timvx_info_, + const std::vector> &inputs, + const std::vector> &outputs, + bool isLast) CV_OVERRIDE { + auto info = reinterpret_cast(timvx_info_); + CV_Assert(info); + auto timvx_graph = info->getGraph(); + CV_Assert(timvx_graph); + auto graph = timvx_graph->graph; + + auto input_wrapper = inputs.front().dynamicCast(); + int input_wrapper_index = -1; + if (input_wrapper->isTensor()) { + input_wrapper_index = timvx_graph->getTensorIndex(input_wrapper->getTensor()); + if (input_wrapper_index == -1) { + auto tmp = input_wrapper->getMat(); + input_wrapper = std::make_shared(tmp); + } + } + if (!input_wrapper->isTensor() || input_wrapper_index == 1) { + auto input_node_quant = Ptr(new tim::vx::Quantization(tim::vx::QuantType::ASYMMETRIC, 1.0f, 0)); + input_wrapper->createTensor(graph, tim::vx::TensorAttribute::INPUT, input_node_quant); + input_wrapper_index = timvx_graph->addWrapper(input_wrapper); + } + + auto output_wrapper = outputs.front().dynamicCast(); + auto output_node_quant = input_wrapper->getTensorQuantization(); + if (isLast) { + auto shape_type = getShapeTypeFromMat(output_wrapper->getMat()); + output_wrapper->setTensorShape(shape_type); + output_wrapper->createTensor(graph, tim::vx::TensorAttribute::OUTPUT, output_node_quant); + } else { + output_wrapper->createTensor(graph, tim::vx::TensorAttribute::TRANSIENT, output_node_quant); + } + int output_wrapper_index = timvx_graph->addWrapper(output_wrapper); + + std::shared_ptr timvx_node; + timvx_node = graph->CreateOperation(std::vector{blocksize, blocksize}); + std::vector input_wrapper_indices{input_wrapper_index}, output_wrapper_indices{output_wrapper_index}; + return Ptr(new TimVXBackendNode(timvx_graph, timvx_node, input_wrapper_indices, output_wrapper_indices)); + } +#endif + + virtual bool tryQuantize(const std::vector> &scales, + const std::vector> &zeropoints, LayerParams ¶ms) CV_OVERRIDE { + return true; + } + +private: + int blocksize; +}; + +Ptr SpaceToDepthLayer::create(const LayerParams ¶ms) { + return makePtr(params); +} + +}} // namespace cv::dnn diff --git a/modules/dnn/src/onnx/onnx_importer.cpp b/modules/dnn/src/onnx/onnx_importer.cpp index 7b63e39a3a..565a88b760 100644 --- a/modules/dnn/src/onnx/onnx_importer.cpp +++ b/modules/dnn/src/onnx/onnx_importer.cpp @@ -189,7 +189,7 @@ private: void parseDetectionOutput (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); void parseCumSum (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); void parseElementWise (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); - void parseDepthToSpace (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseDepthSpaceOps (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); void parseRange (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); void parseScatter (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); void parseTile (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); @@ -2916,82 +2916,8 @@ void ONNXImporter::parseElementWise(LayerParams& layerParams, const opencv_onnx: addLayer(layerParams, node_proto); } -void ONNXImporter::parseDepthToSpace(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto_) -{ - // We parse "DepthToSpace" and "SpaceToDepth" in this function. - opencv_onnx::NodeProto node_proto = node_proto_; - const std::string& layer_type = node_proto.op_type(); - CV_Assert(layer_type == "DepthToSpace" || layer_type == "SpaceToDepth"); - - // Get blocksize - CV_Assert(layerParams.has("blocksize")); - int blocksize = layerParams.get("blocksize"); - CV_Assert(blocksize > 0); - - // Get mode, only for "DepthToSpace" - std::string modeType = layerParams.get("mode", "DCR"); - - MatShape inpShape = outShapes[node_proto.input(0)]; - CV_Assert(inpShape.size() == 4); - int N = inpShape[0], C = inpShape[1], H = inpShape[2], W = inpShape[3]; - - // Implement DepthToSpace and SpaceToDepth by the Reshape and Permute layer. - std::array shape0, perm; - std::array shape1; - - if (layer_type == "DepthToSpace") - { - if (modeType == "DCR") - { - shape0 = {N, blocksize, blocksize, C/(blocksize * blocksize), H, W}; - perm = {0, 3, 4, 1, 5, 2}; - shape1 = {N, C/(blocksize * blocksize), H * blocksize, W * blocksize}; - } - else if (modeType == "CRD") - { - shape0 = {N, C/(blocksize * blocksize), blocksize, blocksize, H, W}; - perm = {0, 1, 4, 2, 5, 3}; - shape1 = {N, C/(blocksize * blocksize), H * blocksize, W * blocksize}; - } - else - CV_Error(Error::StsNotImplemented, "The mode of " + modeType + " in " + layer_type + " Layer is not supported"); - } - else // SpaceToDepth - { - shape0 = {N, C, H/blocksize, blocksize, W/blocksize, blocksize}; - perm = {0, 3, 5, 1, 2, 4}; - shape1 = {N, C * blocksize * blocksize, H/blocksize, W/blocksize}; - } - - // Step1: Reshape - LayerParams reshapeLp; - reshapeLp.name = layerParams.name + "/reshape"; - reshapeLp.type = "Reshape"; - CV_Assert(layer_id.find(reshapeLp.name) == layer_id.end()); - reshapeLp.set("dim", DictValue::arrayInt(shape0.data(), shape0.size())); - - opencv_onnx::NodeProto protoReshape; - protoReshape.add_input(node_proto.input(0)); - protoReshape.add_output(reshapeLp.name); - addLayer(reshapeLp, protoReshape); - - // Step2: Transpose - LayerParams permuteLp; - permuteLp.name = layerParams.name + "/permute"; - permuteLp.type = "Permute"; - CV_Assert(layer_id.find(permuteLp.name) == layer_id.end()); - permuteLp.set("order", DictValue::arrayInt(perm.data(), perm.size())); - - opencv_onnx::NodeProto protoPermute; - protoPermute.add_input(reshapeLp.name); - protoPermute.add_output(permuteLp.name); - addLayer(permuteLp, protoPermute); - - // Step3: Reshape - layerParams.type = "Reshape"; - layerParams.set("dim", DictValue::arrayInt(shape1.data(), shape1.size())); - - node_proto.set_input(0, permuteLp.name); +void ONNXImporter::parseDepthSpaceOps(LayerParams &layerParams, const opencv_onnx::NodeProto& node_proto) { + CV_CheckTrue(layerParams.has("blocksize"), "blocksize is required but not found"); addLayer(layerParams, node_proto); } @@ -4002,7 +3928,7 @@ void ONNXImporter::buildDispatchMap_ONNX_AI(int opset_version) dispatch["SoftMax"] = dispatch["Softmax"] = dispatch["LogSoftmax"] = &ONNXImporter::parseSoftMax; dispatch["DetectionOutput"] = &ONNXImporter::parseDetectionOutput; dispatch["CumSum"] = &ONNXImporter::parseCumSum; - dispatch["SpaceToDepth"] = dispatch["DepthToSpace"] = &ONNXImporter::parseDepthToSpace; + dispatch["SpaceToDepth"] = dispatch["DepthToSpace"] = &ONNXImporter::parseDepthSpaceOps; dispatch["ScatterElements"] = dispatch["Scatter"] = dispatch["ScatterND"] = &ONNXImporter::parseScatter; dispatch["Tile"] = &ONNXImporter::parseTile; dispatch["LayerNormalization"] = &ONNXImporter::parseLayerNorm; diff --git a/modules/dnn/test/test_int8_layers.cpp b/modules/dnn/test/test_int8_layers.cpp index bc5d9388a9..9d102937bd 100644 --- a/modules/dnn/test/test_int8_layers.cpp +++ b/modules/dnn/test/test_int8_layers.cpp @@ -509,6 +509,61 @@ TEST_P(Test_Int8_layers, Eltwise) testLayer("split_max", "ONNX", 0.004, 0.012); } +TEST_P(Test_Int8_layers, DepthSpaceOps) { + auto test_layer_with_onnx_conformance_models = [&](const std::string &model_name, double l1, double lInf) { + std::string model_path = _tf("onnx/conformance/node/test_" + model_name + "/model.onnx"); + auto net = readNet(model_path); + + // load reference inputs and outputs + std::string data_base_path = _tf("onnx/conformance/node/test_" + model_name + "/test_data_set_0"); + Mat input = readTensorFromONNX(data_base_path + "/input_0.pb"); + Mat ref_output = readTensorFromONNX(data_base_path + "/output_0.pb"); + + std::vector input_scales, output_scales; + std::vector input_zeropoints, output_zeropoints; + auto qnet = net.quantize(std::vector{input}, CV_8S, CV_8S, false); + qnet.getInputDetails(input_scales, input_zeropoints); + qnet.getOutputDetails(output_scales, output_zeropoints); + qnet.setPreferableBackend(backend); + qnet.setPreferableTarget(target); + + Mat quantized_input, quantized_output; + input.convertTo(quantized_input, CV_8S, 1.f / input_scales.front(), input_zeropoints.front()); + qnet.setInput(quantized_input); + quantized_output = qnet.forward(); + + Mat output; + quantized_output.convertTo(output, CV_32F, output_scales.front(), -(output_scales.front() * output_zeropoints.front())); + normAssert(ref_output, output, model_name.c_str(), l1, lInf); + }; + + double l1 = default_l1, lInf = default_lInf; + { + l1 = 0.001; lInf = 0.002; + if (backend == DNN_BACKEND_TIMVX) { l1 = 0.001; lInf = 0.002; } + test_layer_with_onnx_conformance_models("spacetodepth", l1, lInf); + } + { + l1 = 0.022; lInf = 0.044; + if (backend == DNN_BACKEND_TIMVX) { l1 = 0.022; lInf = 0.044; } + test_layer_with_onnx_conformance_models("spacetodepth_example", l1, lInf); + } + { + l1 = 0.001; lInf = 0.002; + if (backend == DNN_BACKEND_TIMVX) { l1 = 0.24; lInf = 0.99; } + test_layer_with_onnx_conformance_models("depthtospace_crd_mode", l1, lInf); + } + test_layer_with_onnx_conformance_models("depthtospace_dcr_mode", 0.001, 0.002); + test_layer_with_onnx_conformance_models("depthtospace_example", 0.07, 0.14); + + { + l1 = 0.07; lInf = 0.14; + if (backend == DNN_BACKEND_TIMVX) // diff too huge, l1 = 13.6; lInf = 27.2 + applyTestTag(CV_TEST_TAG_DNN_SKIP_TIMVX); + test_layer_with_onnx_conformance_models("depthtospace_crd_mode_example", l1, lInf); + } +} + INSTANTIATE_TEST_CASE_P(/**/, Test_Int8_layers, dnnBackendsAndTargetsInt8()); class Test_Int8_nets : public DNNTestLayer diff --git a/modules/dnn/test/test_onnx_conformance_layer_filter__openvino.inl.hpp b/modules/dnn/test/test_onnx_conformance_layer_filter__openvino.inl.hpp index 509cf6007d..000e867217 100644 --- a/modules/dnn/test/test_onnx_conformance_layer_filter__openvino.inl.hpp +++ b/modules/dnn/test/test_onnx_conformance_layer_filter__openvino.inl.hpp @@ -544,10 +544,20 @@ CASE(test_cumsum_2d_negative_axis) // no filter CASE(test_depthtospace_crd_mode) // no filter + if (target == DNN_TARGET_OPENCL) + { + default_l1 = 1e-4; // Expected: (normL1) <= (l1), actual: 9.33057e-05 vs 1e-05 + default_lInf = 2.5e-4; // Expected: (normInf) <= (lInf), actual: 0.000243843 vs 0.0001 + } CASE(test_depthtospace_crd_mode_example) // no filter CASE(test_depthtospace_dcr_mode) // no filter + if (target == DNN_TARGET_OPENCL) + { + default_l1 = 1e-4; // Expected: (normL1) <= (l1), actual: 9.33057e-05 vs 1e-05 + default_lInf = 2.5e-4; // Expected: (normInf) <= (lInf), actual: 0.000243843 vs 0.0001 + } CASE(test_depthtospace_example) // no filter CASE(test_dequantizelinear) From d8c7debe5298887789cd513674cf601c6d3dffbc Mon Sep 17 00:00:00 2001 From: Kumataro Date: Sat, 22 Jun 2024 12:23:25 +0900 Subject: [PATCH 16/30] cmake: support 10+ minor version(10,11,...) --- cmake/OpenCVUtils.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/OpenCVUtils.cmake b/cmake/OpenCVUtils.cmake index 16babb4937..ae660f86b1 100644 --- a/cmake/OpenCVUtils.cmake +++ b/cmake/OpenCVUtils.cmake @@ -1992,7 +1992,7 @@ macro(ocv_git_describe var_name path) OUTPUT_STRIP_TRAILING_WHITESPACE ) if(NOT GIT_RESULT EQUAL 0) - execute_process(COMMAND "${GIT_EXECUTABLE}" describe --tags --always --dirty --match "[0-9].[0-9].[0-9]*" --exclude "[^-]*-cvsdk" + execute_process(COMMAND "${GIT_EXECUTABLE}" describe --tags --always --dirty --match "[0-9].[0-9]*.[0-9]*" --exclude "[^-]*-cvsdk" WORKING_DIRECTORY "${path}" OUTPUT_VARIABLE ${var_name} RESULT_VARIABLE GIT_RESULT From 8be2056cff7df2a953e9919912aef4e0eb6334bf Mon Sep 17 00:00:00 2001 From: TFiFiE Date: Sat, 22 Jun 2024 17:27:16 +0200 Subject: [PATCH 17/30] Replace use of "eval()" with "getattr()". --- .../py_template_matching/py_template_matching.markdown | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/py_tutorials/py_imgproc/py_template_matching/py_template_matching.markdown b/doc/py_tutorials/py_imgproc/py_template_matching/py_template_matching.markdown index 1557118ab6..3a59bf6b23 100644 --- a/doc/py_tutorials/py_imgproc/py_template_matching/py_template_matching.markdown +++ b/doc/py_tutorials/py_imgproc/py_template_matching/py_template_matching.markdown @@ -46,12 +46,12 @@ assert template is not None, "file could not be read, check with os.path.exists( w, h = template.shape[::-1] # All the 6 methods for comparison in a list -methods = ['cv.TM_CCOEFF', 'cv.TM_CCOEFF_NORMED', 'cv.TM_CCORR', - 'cv.TM_CCORR_NORMED', 'cv.TM_SQDIFF', 'cv.TM_SQDIFF_NORMED'] +methods = ['TM_CCOEFF', 'TM_CCOEFF_NORMED', 'TM_CCORR', + 'TM_CCORR_NORMED', 'TM_SQDIFF', 'TM_SQDIFF_NORMED'] for meth in methods: img = img2.copy() - method = eval(meth) + method = getattr(cv, meth) # Apply template Matching res = cv.matchTemplate(img,template,method) From 11c69bb1717e2e0f37b2c25c4741aa4b18538788 Mon Sep 17 00:00:00 2001 From: richard28039 <89371302+richard28039@users.noreply.github.com> Date: Mon, 24 Jun 2024 15:28:38 +0800 Subject: [PATCH 18/30] Merge pull request #25775 from richard28039:4.x Add yolov8l.onnx to samples #25775 ### Pull Request Readiness Checklist See details at https://github.com/opencv/opencv/wiki/How_to_contribute#making-a-good-pull-request - [x] I agree to contribute to the project under Apache 2 License. - [x] To the best of my knowledge, the proposed patch is not based on a code under GPL or another license that is incompatible with OpenCV - [x] The PR is proposed to the proper branch - [ ] There is a reference to the original bug report and related work - [ ] There is accuracy test, performance test and test data in opencv_extra repository, if applicable Patch to opencv_extra has the same branch name. - [ ] The feature is well documented and sample code can be built with the project CMake Hello, I noticed that the /samples/dnn/models.yml said it should be used for all yolov8 models, but the YOLOv8l is not included in the file, so I added it to the file, thanks. ![image](https://github.com/opencv/opencv/assets/89371302/7a7b0090-ef4c-478d-8f24-7d99260fe0c9) --- samples/dnn/models.yml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/samples/dnn/models.yml b/samples/dnn/models.yml index b14dae327c..15ab9d0f3a 100644 --- a/samples/dnn/models.yml +++ b/samples/dnn/models.yml @@ -76,6 +76,20 @@ yolov8m: background_label_id: 0 sample: "yolo_detector" +yolov8l: + load_info: + url: "https://github.com/CVHub520/X-AnyLabeling/releases/download/v0.1.0/yolov8l.onnx" + sha1: "462df53ca3a85d110bf6be7fc2e2bb1277124395" + model: "yolov8l.onnx" + mean: 0.0 + scale: 0.00392 + width: 640 + height: 640 + rgb: true + classes: "object_detection_classes_yolo.txt" + background_label_id: 0 + sample: "yolo_detector" + # YOLO4 object detection family from Darknet (https://github.com/AlexeyAB/darknet) # YOLO object detection family from Darknet (https://pjreddie.com/darknet/yolo/) # Might be used for all YOLOv2, TinyYolov2, YOLOv3, YOLOv4 and TinyYolov4 From 2179186a51efae902030f5e31c873b0ede96a60a Mon Sep 17 00:00:00 2001 From: Letu Ren Date: Wed, 26 Jun 2024 19:07:44 +0800 Subject: [PATCH 19/30] Remove unused variables in rgb2hsv_b simd --- modules/imgproc/src/color_hsv.simd.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/imgproc/src/color_hsv.simd.hpp b/modules/imgproc/src/color_hsv.simd.hpp index bef9497760..c450d609e5 100644 --- a/modules/imgproc/src/color_hsv.simd.hpp +++ b/modules/imgproc/src/color_hsv.simd.hpp @@ -130,7 +130,7 @@ struct RGB2HSV_b // sdiv = sdiv_table[v] v_int32 sdiv0, sdiv1, sdiv2, sdiv3;; - v_uint16 vd0, vd1, vd2; + v_uint16 vd0, vd1; v_expand(v, vd0, vd1); v_int32 vq0, vq1, vq2, vq3; v_expand(v_reinterpret_as_s16(vd0), vq0, vq1); @@ -150,7 +150,7 @@ struct RGB2HSV_b // hdiv = hdiv_table[diff] v_int32 hdiv0, hdiv1, hdiv2, hdiv3; - v_uint16 diffd0, diffd1, diffd2; + v_uint16 diffd0, diffd1; v_expand(diff, diffd0, diffd1); v_int32 diffq0, diffq1, diffq2, diffq3; v_expand(v_reinterpret_as_s16(diffd0), diffq0, diffq1); From efa4d9176a16ad30081a184124b25a8029ea788d Mon Sep 17 00:00:00 2001 From: kozinove Date: Wed, 26 Jun 2024 15:31:19 +0300 Subject: [PATCH 20/30] Merge pull request #25661 from itlab-vision:framebuffer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Highgui backend on top of Framebuffer #25661 ### Pull Request Readiness Checklist See details at https://github.com/opencv/opencv/wiki/How_to_contribute#making-a-good-pull-request - [ x] I agree to contribute to the project under Apache 2 License. - [ x] To the best of my knowledge, the proposed patch is not based on a code under GPL or another license that is incompatible with OpenCV - [ x] The PR is proposed to the proper branch - [ ] There is a reference to the original bug report and related work - [ x] There is accuracy test, performance test and test data in opencv_extra repository, if applicable Patch to opencv_extra has the same branch name. - [ x] The feature is well documented and sample code can be built with the project CMake Environment variables used: OPENCV_UI_BACKEND - you need to add the value “FB” OPENCV_UI_PRIORITY_FB - requires priority indication OPENCV_HIGHGUI_FB_MODE={FB|XVFB|EMU} - mode of using Framebuffer (default "FB") - FB - Linux Framebuffer - XVFB - virtual Framebuffer - EMU - emulation (images are not displayed) OPENCV_HIGHGUI_FB_DEVICE (FRAMEBUFFER) - path to the Framebuffer file (default "/dev/fb0"). Examples of using: sudo OPENCV_UI_BACKEND=FB ./opencv_test_highgui sudo OPENCV_UI_PRIORITY_FB=1111 ./opencv_test_highgui OPENCV_UI_BACKEND=FB OPENCV_HIGHGUI_FB_MODE=EMU ./opencv_test_highgui sudo OPENCV_UI_BACKEND=FB OPENCV_HIGHGUI_FB_MODE=FB ./opencv_test_highgui export DISPLAY=:99 Xvfb $DISPLAY -screen 0 1024x768x24 -fbdir /tmp/ -f /tmp/user.xvfb.auth& sudo -u sipeed XAUTHORITY=/tmp/user.xvfb.auth x11vnc -display $DISPLAY -listen localhost& DISPLAY=:0 gvncviewer localhost& FRAMEBUFFER=/tmp/Xvfb_screen0 OPENCV_UI_BACKEND=FB OPENCV_HIGHGUI_FB_MODE=XVFB ./opencv_test_highgui --- CMakeLists.txt | 11 + cmake/checks/framebuffer.cpp | 9 + modules/highgui/CMakeLists.txt | 10 + .../highgui/cmake/detect_framebuffer.cmake | 14 + modules/highgui/cmake/init.cmake | 2 + modules/highgui/src/backend.hpp | 4 + modules/highgui/src/registry.impl.hpp | 4 + modules/highgui/src/window_framebuffer.cpp | 798 ++++++++++++++++++ modules/highgui/src/window_framebuffer.hpp | 135 +++ modules/highgui/test/test_gui.cpp | 6 +- 10 files changed, 991 insertions(+), 2 deletions(-) create mode 100644 cmake/checks/framebuffer.cpp create mode 100644 modules/highgui/cmake/detect_framebuffer.cmake create mode 100644 modules/highgui/src/window_framebuffer.cpp create mode 100644 modules/highgui/src/window_framebuffer.hpp diff --git a/CMakeLists.txt b/CMakeLists.txt index a61b43abe4..58437a37b1 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -306,6 +306,10 @@ OCV_OPTION(WITH_GTK "Include GTK support" ON OCV_OPTION(WITH_GTK_2_X "Use GTK version 2" OFF VISIBLE_IF UNIX AND NOT APPLE AND NOT ANDROID VERIFY HAVE_GTK AND NOT HAVE_GTK3) +OCV_OPTION(WITH_FRAMEBUFFER "Include framebuffer support" OFF + VISIBLE_IF UNIX AND NOT APPLE AND NOT ANDROID) +OCV_OPTION(WITH_FRAMEBUFFER_XVFB "Include virtual framebuffer support" OFF + VISIBLE_IF UNIX AND NOT APPLE AND NOT ANDROID) OCV_OPTION(WITH_WAYLAND "Include Wayland support" OFF VISIBLE_IF UNIX AND NOT APPLE AND NOT ANDROID VERIFY HAVE_WAYLAND) @@ -1462,6 +1466,13 @@ if(WITH_GTK OR HAVE_GTK) endif() endif() +if(WITH_FRAMEBUFFER OR HAVE_FRAMEBUFFER) + status(" Framebuffer UI:" HAVE_FRAMEBUFFER THEN YES ELSE NO) + if(WITH_FRAMEBUFFER_XVFB OR HAVE_FRAMEBUFFER_XVFB) + status(" Virtual framebuffer UI:" HAVE_FRAMEBUFFER_XVFB THEN YES ELSE NO) + endif() +endif() + if(WITH_OPENGL OR HAVE_OPENGL) status(" OpenGL support:" HAVE_OPENGL THEN "YES (${OPENGL_LIBRARIES})" ELSE NO) endif() diff --git a/cmake/checks/framebuffer.cpp b/cmake/checks/framebuffer.cpp new file mode 100644 index 0000000000..77bb60275c --- /dev/null +++ b/cmake/checks/framebuffer.cpp @@ -0,0 +1,9 @@ +#include +#include + +int main(void) +{ + XWDFileHeader *xwd_header; + XWDColor *xwd_colors; + return 0; +} diff --git a/modules/highgui/CMakeLists.txt b/modules/highgui/CMakeLists.txt index d8a7bb050a..0108626dfd 100644 --- a/modules/highgui/CMakeLists.txt +++ b/modules/highgui/CMakeLists.txt @@ -49,6 +49,16 @@ list(REMOVE_ITEM highgui_ext_hdrs "${CMAKE_CURRENT_LIST_DIR}/include/opencv2/${n set(OPENCV_HIGHGUI_BUILTIN_BACKEND "") +if(WITH_FRAMEBUFFER AND HAVE_FRAMEBUFFER) + set(OPENCV_HIGHGUI_BUILTIN_BACKEND "FB") + add_definitions(-DHAVE_FRAMEBUFFER) + list(APPEND highgui_srcs ${CMAKE_CURRENT_LIST_DIR}/src/window_framebuffer.cpp) + list(APPEND highgui_hdrs ${CMAKE_CURRENT_LIST_DIR}/src/window_framebuffer.hpp) + if(HAVE_FRAMEBUFFER_XVFB) + add_definitions(-DHAVE_FRAMEBUFFER_XVFB) + endif() +endif() + if(WITH_WAYLAND AND HAVE_WAYLAND) set(OPENCV_HIGHGUI_BUILTIN_BACKEND "Wayland") add_definitions(-DHAVE_WAYLAND) diff --git a/modules/highgui/cmake/detect_framebuffer.cmake b/modules/highgui/cmake/detect_framebuffer.cmake new file mode 100644 index 0000000000..8e292f72ac --- /dev/null +++ b/modules/highgui/cmake/detect_framebuffer.cmake @@ -0,0 +1,14 @@ +# --- FB --- +set(HAVE_FRAMEBUFFER ON) +if(WITH_FRAMEBUFFER_XVFB) + try_compile(HAVE_FRAMEBUFFER_XVFB + "${CMAKE_CURRENT_BINARY_DIR}" + "${OpenCV_SOURCE_DIR}/cmake/checks/framebuffer.cpp") + if(HAVE_FRAMEBUFFER_XVFB) + message(STATUS "Check virtual framebuffer - done") + else() + message(STATUS + "Check virtual framebuffer - failed\n" + "Please install the xorg-x11-proto-devel or x11proto-dev package\n") + endif() +endif() diff --git a/modules/highgui/cmake/init.cmake b/modules/highgui/cmake/init.cmake index 49d4799b30..f52ce2a4a8 100644 --- a/modules/highgui/cmake/init.cmake +++ b/modules/highgui/cmake/init.cmake @@ -39,6 +39,8 @@ endmacro() add_backend("gtk" WITH_GTK) add_backend("win32ui" WITH_WIN32UI) add_backend("wayland" WITH_WAYLAND) +add_backend("framebuffer" WITH_FRAMEBUFFER) + # TODO cocoa # TODO qt # TODO opengl diff --git a/modules/highgui/src/backend.hpp b/modules/highgui/src/backend.hpp index 93d51da119..49878baefc 100644 --- a/modules/highgui/src/backend.hpp +++ b/modules/highgui/src/backend.hpp @@ -127,6 +127,10 @@ std::shared_ptr createUIBackendGTK(); std::shared_ptr createUIBackendQT(); #endif +#ifdef HAVE_FRAMEBUFFER +std::shared_ptr createUIBackendFramebuffer(); +#endif + #endif // BUILD_PLUGIN } // namespace highgui_backend diff --git a/modules/highgui/src/registry.impl.hpp b/modules/highgui/src/registry.impl.hpp index 23f4e9f4e1..782738a820 100644 --- a/modules/highgui/src/registry.impl.hpp +++ b/modules/highgui/src/registry.impl.hpp @@ -44,6 +44,10 @@ std::vector& getBuiltinBackendsInfo() DECLARE_DYNAMIC_BACKEND("GTK2") #endif +#ifdef HAVE_FRAMEBUFFER + DECLARE_STATIC_BACKEND("FB", createUIBackendFramebuffer) +#endif + #if 0 // TODO #ifdef HAVE_QT DECLARE_STATIC_BACKEND("QT", createUIBackendQT) diff --git a/modules/highgui/src/window_framebuffer.cpp b/modules/highgui/src/window_framebuffer.cpp new file mode 100644 index 0000000000..687bf91724 --- /dev/null +++ b/modules/highgui/src/window_framebuffer.cpp @@ -0,0 +1,798 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. + +#include "precomp.hpp" + +#include "window_framebuffer.hpp" + +#include +#include +#ifdef NDEBUG +#define CV_LOG_STRIP_LEVEL CV_LOG_LEVEL_DEBUG + 1 +#else +#define CV_LOG_STRIP_LEVEL CV_LOG_LEVEL_VERBOSE + 1 +#endif +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "opencv2/imgproc.hpp" + +#ifdef HAVE_FRAMEBUFFER_XVFB +#include +#include + +#define C32INT(ptr) ((((unsigned char*)ptr)[0] << 24) | (((unsigned char*)ptr)[1] << 16) | \ + (((unsigned char*)ptr)[2] << 8) | (((unsigned char*)ptr)[3] << 0)) +#endif + + +namespace cv { +namespace highgui_backend { + +std::shared_ptr createUIBackendFramebuffer() +{ + return std::make_shared(); +} + +static std::string& getFBMode() +{ + static std::string fbModeOpenCV = + cv::utils::getConfigurationParameterString("OPENCV_HIGHGUI_FB_MODE", "FB"); + return fbModeOpenCV; +} + +static std::string& getFBFileName() +{ + static std::string fbFileNameFB = + cv::utils::getConfigurationParameterString("FRAMEBUFFER", "/dev/fb0"); + static std::string fbFileNameOpenCV = + cv::utils::getConfigurationParameterString("OPENCV_HIGHGUI_FB_DEVICE", ""); + + if (!fbFileNameOpenCV.empty()) return fbFileNameOpenCV; + return fbFileNameFB; +} + +FramebufferWindow::FramebufferWindow(FramebufferBackend &_backend, int _flags): + backend(_backend), flags(_flags) +{ + CV_LOG_DEBUG(NULL, "UI: FramebufferWindow::FramebufferWindow()"); + FB_ID = "FramebufferWindow"; + windowRect = Rect(0,0, backend.getFBWidth(), backend.getFBHeight()); +} + +FramebufferWindow::~FramebufferWindow() +{ + CV_LOG_DEBUG(NULL, "UI: FramebufferWindow::~FramebufferWindow()"); +} + +void FramebufferWindow::imshow(InputArray image) +{ + CV_LOG_DEBUG(NULL, "UI: FramebufferWindow::imshow(InputArray image)"); + currentImg = image.getMat().clone(); + + CV_LOG_INFO(NULL, "UI: InputArray image: " + << cv::typeToString(image.type()) << " size " << image.size()); + + if (currentImg.empty()) + { + CV_LOG_WARNING(NULL, "UI: image is empty"); + return; + } + CV_CheckEQ(currentImg.dims, 2, "UI: dims != 2"); + + Mat img = image.getMat(); + switch (img.channels()) + { + case 1: + { + Mat tmp; + switch(img.type()) + { + case CV_8U: + tmp = img; + break; + case CV_8S: + cv::convertScaleAbs(img, tmp, 1, 127); + break; + case CV_16S: + cv::convertScaleAbs(img, tmp, 1/255., 127); + break; + case CV_16U: + cv::convertScaleAbs(img, tmp, 1/255.); + break; + case CV_32F: + case CV_64F: // assuming image has values in range [0, 1) + img.convertTo(tmp, CV_8U, 255., 0.); + break; + } + Mat rgb(img.rows, img.cols, CV_8UC3); + cvtColor(tmp, rgb, COLOR_GRAY2RGB); + img = rgb; + } + break; + case 3: + case 4: + { + Mat tmp(img.rows, img.cols, CV_8UC3); + convertToShow(img, tmp, true); + img = tmp; + } + break; + default: + CV_Error(cv::Error::StsBadArg, "Bad image: wrong number of channels"); + } + { + Mat bgra(img.rows, img.cols, CV_8UC4); + cvtColor(img, bgra, COLOR_RGB2BGRA, bgra.channels()); + img = bgra; + } + + int newWidth = windowRect.width; + int newHeight = windowRect.height; + int cntChannel = img.channels(); + cv::Size imgSize = currentImg.size(); + + if (flags & WINDOW_AUTOSIZE) + { + windowRect.width = imgSize.width; + windowRect.height = imgSize.height; + newWidth = windowRect.width; + newHeight = windowRect.height; + } + + if (flags & WINDOW_FREERATIO) + { + newWidth = windowRect.width; + newHeight = windowRect.height; + } + else //WINDOW_KEEPRATIO + { + double aspect_ratio = ((double)img.cols) / img.rows; + newWidth = windowRect.width; + newHeight = (int)(windowRect.width / aspect_ratio); + + if (newHeight > windowRect.height) + { + newWidth = (int)(windowRect.height * aspect_ratio); + newHeight = windowRect.height; + } + } + + if ((newWidth != img.cols) && (newHeight != img.rows)) + { + Mat imResize; + cv::resize(img, imResize, cv::Size(newWidth, newHeight), INTER_LINEAR); + img = imResize; + } + + CV_LOG_INFO(NULL, "UI: Formated image: " + << cv::typeToString(img.type()) << " size " << img.size()); + + if (backend.getMode() == FB_MODE_EMU) + { + CV_LOG_WARNING(NULL, "UI: FramebufferWindow::imshow is used in EMU mode"); + return; + } + + if (backend.getFBPointer() == MAP_FAILED) + { + CV_LOG_ERROR(NULL, "UI: Framebuffer is not mapped"); + return; + } + + int xOffset = backend.getFBXOffset(); + int yOffset = backend.getFBYOffset(); + int fbHeight = backend.getFBHeight(); + int fbWidth = backend.getFBWidth(); + int lineLength = backend.getFBLineLength(); + + int img_start_x; + int img_start_y; + int img_end_x; + int img_end_y; + int fb_start_x; + int fb_start_y; + + if (windowRect.y - yOffset < 0) + { + img_start_y = - (windowRect.y - yOffset); + } + else + { + img_start_y = 0; + } + if (windowRect.x - xOffset < 0) + { + img_start_x = - (windowRect.x - xOffset); + } + else + { + img_start_x = 0; + } + + if (windowRect.y + yOffset + img.rows > fbHeight) + { + img_end_y = fbHeight - windowRect.y - yOffset; + } + else + { + img_end_y = img.rows; + } + if (windowRect.x + xOffset + img.cols > fbWidth) + { + img_end_x = fbWidth - windowRect.x - xOffset; + } + else + { + img_end_x = img.cols; + } + + if (windowRect.y + yOffset >= 0) + { + fb_start_y = windowRect.y + yOffset; + } + else + { + fb_start_y = 0; + } + if (windowRect.x + xOffset >= 0) + { + fb_start_x = windowRect.x + xOffset; + } + else + { + fb_start_x = 0; + } + + for (int y = img_start_y; y < img_end_y; y++) + { + std::memcpy(backend.getFBPointer() + + (fb_start_y + y - img_start_y) * lineLength + fb_start_x * cntChannel, + img.ptr(y) + img_start_x * cntChannel, + (img_end_x - img_start_x) * cntChannel); + } +} + +double FramebufferWindow::getProperty(int /*prop*/) const +{ + CV_LOG_WARNING(NULL, "UI: getProperty (not supported)"); + return 0.0; +} + +bool FramebufferWindow::setProperty(int /*prop*/, double /*value*/) +{ + CV_LOG_WARNING(NULL, "UI: setProperty (not supported)"); + return false; +} + +void FramebufferWindow::resize(int width, int height) +{ + CV_LOG_DEBUG(NULL, "UI: FramebufferWindow::resize(int width " + << width <<", height " << height << ")"); + + CV_Assert(width > 0); + CV_Assert(height > 0); + + if (!(flags & WINDOW_AUTOSIZE)) + { + windowRect.width = width; + windowRect.height = height; + + if (!currentImg.empty()) + { + imshow(currentImg); + } + } +} + +void FramebufferWindow::move(int x, int y) +{ + CV_LOG_DEBUG(NULL, "UI: FramebufferWindow::move(int x " << x << ", y " << y <<")"); + + windowRect.x = x; + windowRect.y = y; + + if (!currentImg.empty()) + { + imshow(currentImg); + } +} + +Rect FramebufferWindow::getImageRect() const +{ + CV_LOG_DEBUG(NULL, "UI: FramebufferWindow::getImageRect()"); + return windowRect; +} + +void FramebufferWindow::setTitle(const std::string& /*title*/) +{ + CV_LOG_WARNING(NULL, "UI: setTitle (not supported)"); +} + +void FramebufferWindow::setMouseCallback(MouseCallback /*onMouse*/, void* /*userdata*/) +{ + CV_LOG_WARNING(NULL, "UI: setMouseCallback (not supported)"); +} + +std::shared_ptr FramebufferWindow::createTrackbar( + const std::string& /*name*/, + int /*count*/, + TrackbarCallback /*onChange*/, + void* /*userdata*/) +{ + CV_LOG_WARNING(NULL, "UI: createTrackbar (not supported)"); + return nullptr; +} + +std::shared_ptr FramebufferWindow::findTrackbar(const std::string& /*name*/) +{ + CV_LOG_WARNING(NULL, "UI: findTrackbar (not supported)"); + return nullptr; +} + +const std::string& FramebufferWindow::getID() const +{ + CV_LOG_DEBUG(NULL, "UI: FramebufferWindow::getID()"); + return FB_ID; +} + +bool FramebufferWindow::isActive() const +{ + CV_LOG_DEBUG(NULL, "UI: FramebufferWindow::isActive()"); + return true; +} + +void FramebufferWindow::destroy() +{ + CV_LOG_DEBUG(NULL, "UI: FramebufferWindow::destroy()"); +} + +int FramebufferBackend::fbOpenAndGetInfo() +{ + std::string fbFileName = getFBFileName(); + CV_LOG_INFO(NULL, "UI: FramebufferWindow::The following is used as a framebuffer file: \n" + << fbFileName); + + int fb_fd = open(fbFileName.c_str(), O_RDWR); + if (fb_fd == -1) + { + CV_LOG_ERROR(NULL, "UI: can't open framebuffer"); + return -1; + } + + if (ioctl(fb_fd, FBIOGET_FSCREENINFO, &fixInfo)) + { + CV_LOG_ERROR(NULL, "UI: can't read fix info for framebuffer"); + return -1; + } + + if (ioctl(fb_fd, FBIOGET_VSCREENINFO, &varInfo)) + { + CV_LOG_ERROR(NULL, "UI: can't read var info for framebuffer"); + return -1; + } + + CV_LOG_INFO(NULL, "UI: framebuffer info: \n" + << " red offset " << varInfo.red.offset << " length " << varInfo.red.length << "\n" + << " green offset " << varInfo.green.offset << " length " << varInfo.green.length << "\n" + << " blue offset " << varInfo.blue.offset << " length " << varInfo.blue.length << "\n" + << "transp offset " << varInfo.transp.offset << " length " <pixmap_format)) != ZPixmap) + { + CV_LOG_ERROR(NULL, "Unsupported pixmap format: " << xwd_header->pixmap_format); + return -1; + } + + if (xwd_header->xoffset != 0) + { + CV_LOG_ERROR(NULL, "UI: Unsupported xoffset value: " << xwd_header->xoffset ); + return -1; + } + + unsigned int r = C32INT(&(xwd_header->red_mask)); + unsigned int g = C32INT(&(xwd_header->green_mask)); + unsigned int b = C32INT(&(xwd_header->blue_mask)); + + fbWidth = C32INT(&(xwd_header->pixmap_width)); + fbHeight = C32INT(&(xwd_header->pixmap_height)); + fbXOffset = 0; + fbYOffset = 0; + fbLineLength = C32INT(&(xwd_header->bytes_per_line)); + fbBitsPerPixel = C32INT(&(xwd_header->bits_per_pixel)); + + CV_LOG_INFO(NULL, "UI: XVFB info: \n" + << " red_mask " << r << "\n" + << " green_mask " << g << "\n" + << " blue_mask " << b << "\n" + << "bits_per_pixel " << fbBitsPerPixel); + + if ((r != 16711680 ) && (g != 65280 ) && (b != 255 ) && + (fbBitsPerPixel != 32)) + { + CV_LOG_ERROR(NULL, "UI: Framebuffer format is not supported " + << "(use BGRA format with bits_per_pixel = 32)"); + return -1; + } + + xvfb_len_header = C32INT(&(xwd_header->header_size)); + xvfb_len_colors = sizeof(XWDColor) * C32INT(&(xwd_header->ncolors)); + xvfb_len_pixmap = C32INT(&(xwd_header->bytes_per_line)) * + C32INT(&(xwd_header->pixmap_height)); + + munmap(xwd_header, sizeof(XWDFileHeader)); + + fbScreenSize = xvfb_len_header + xvfb_len_colors + xvfb_len_pixmap; + xwd_header = (XWDFileHeader*) + mmap(NULL, fbScreenSize, PROT_READ | PROT_WRITE, MAP_SHARED, fb_fd, 0); + + fbPointer = (unsigned char*)xwd_header; + fbPointer_dist = xvfb_len_header + xvfb_len_colors; + +#else + CV_LOG_WARNING(NULL, "UI: To use virtual framebuffer, " + << "compile OpenCV with the WITH_FRAMEBUFFER_XVFB=ON"); +#endif + + return fb_fd; +} + +fb_var_screeninfo &FramebufferBackend::getVarInfo() +{ + return varInfo; +} + +fb_fix_screeninfo &FramebufferBackend::getFixInfo() +{ + return fixInfo; +} + +int FramebufferBackend::getFramebuffrerID() +{ + return fbID; +} + +int FramebufferBackend::getFBWidth() +{ + return fbWidth; +} + +int FramebufferBackend::getFBHeight() +{ + return fbHeight; +} + +int FramebufferBackend::getFBXOffset() +{ + return fbXOffset; +} + +int FramebufferBackend::getFBYOffset() +{ + return fbYOffset; +} + +int FramebufferBackend::getFBBitsPerPixel() +{ + return fbBitsPerPixel; +} + +int FramebufferBackend::getFBLineLength() +{ + return fbLineLength; +} + +unsigned char* FramebufferBackend::getFBPointer() +{ + return fbPointer + fbPointer_dist; +} + +Mat& FramebufferBackend::getBackgroundBuff() +{ + return backgroundBuff; +} + +OpenCVFBMode FramebufferBackend::getMode() +{ + return mode; +} + +FramebufferBackend::FramebufferBackend():mode(FB_MODE_FB), fbPointer_dist(0) +{ + CV_LOG_DEBUG(NULL, "UI: FramebufferWindow::FramebufferBackend()"); + + std::string fbModeStr = getFBMode(); + + if (fbModeStr == "EMU") + { + mode = FB_MODE_EMU; + CV_LOG_WARNING(NULL, "UI: FramebufferWindow is trying to use EMU mode"); + } + if (fbModeStr == "FB") + { + mode = FB_MODE_FB; + CV_LOG_WARNING(NULL, "UI: FramebufferWindow is trying to use FB mode"); + } + if (fbModeStr == "XVFB") + { + mode = FB_MODE_XVFB; + CV_LOG_WARNING(NULL, "UI: FramebufferWindow is trying to use XVFB mode"); + } + + fbID = -1; + if (mode == FB_MODE_FB) + { + fbID = fbOpenAndGetInfo(); + } + if (mode == FB_MODE_XVFB) + { + fbID = XvfbOpenAndGetInfo(); + } + + CV_LOG_INFO(NULL, "UI: FramebufferWindow::fbID " << fbID); + + if (fbID == -1) + { + mode = FB_MODE_EMU; + fbWidth = 640; + fbHeight = 480; + fbXOffset = 0; + fbYOffset = 0; + fbBitsPerPixel = 0; + fbLineLength = 0; + + CV_LOG_WARNING(NULL, "UI: FramebufferWindow is used in EMU mode"); + return; + } + + CV_LOG_INFO(NULL, "UI: Framebuffer's width, height, bits per pix: " + << fbWidth << " " << fbHeight << " " << fbBitsPerPixel); + + CV_LOG_INFO(NULL, "UI: Framebuffer's offsets (x, y), line length: " + << fbXOffset << " " << fbYOffset << " " << fbLineLength); + + backgroundBuff = Mat(fbHeight, fbWidth, CV_8UC4); + int cntChannel = 4; + for (int y = fbYOffset; y < backgroundBuff.rows + fbYOffset; y++) + { + std::memcpy(backgroundBuff.ptr(y - fbYOffset), + getFBPointer() + y * fbLineLength + fbXOffset * cntChannel, + backgroundBuff.cols * cntChannel); + } +} + +FramebufferBackend::~FramebufferBackend() +{ + CV_LOG_DEBUG(NULL, "UI: FramebufferBackend::~FramebufferBackend()"); + if(fbID == -1) return; + + if (fbPointer != MAP_FAILED) + { + int cntChannel = 4; + for (int y = fbYOffset; y < backgroundBuff.rows + fbYOffset; y++) + { + std::memcpy(getFBPointer() + y * fbLineLength + fbXOffset * cntChannel, + backgroundBuff.ptr(y - fbYOffset), + backgroundBuff.cols * cntChannel); + } + + munmap(fbPointer, fbScreenSize); + } + close(fbID); +} + +void FramebufferBackend::destroyAllWindows() { + CV_LOG_DEBUG(NULL, "UI: FramebufferBackend::destroyAllWindows()"); +} + +// namedWindow +std::shared_ptr FramebufferBackend::createWindow( + const std::string& winname, + int flags) +{ + CV_LOG_DEBUG(NULL, "UI: FramebufferBackend::createWindow(" + << winname << ", " << flags << ")"); + return std::make_shared(*this, flags); +} + +void FramebufferBackend::initTermios(int echo, int wait) +{ + tcgetattr(0, &old); + current = old; + current.c_lflag &= ~ICANON; + current.c_lflag &= ~ISIG; + current.c_cc[VMIN] = wait; + if (echo) + { + current.c_lflag |= ECHO; + } + else + { + current.c_lflag &= ~ECHO; + } + tcsetattr(0, TCSANOW, ¤t); +} + +void FramebufferBackend::resetTermios(void) +{ + tcsetattr(0, TCSANOW, &old); +} + +int FramebufferBackend::getch_(int echo, int wait) +{ + int ch; + initTermios(echo, wait); + ch = getchar(); + if (ch < 0) + { + rewind(stdin); + } + resetTermios(); + return ch; +} + +bool FramebufferBackend::kbhit() +{ + int byteswaiting = 0; + initTermios(0, 1); + if (ioctl(0, FIONREAD, &byteswaiting) < 0) + { + CV_LOG_ERROR(NULL, "UI: Framebuffer ERR byteswaiting" ); + } + resetTermios(); + + return byteswaiting > 0; +} + +int FramebufferBackend::waitKeyEx(int delay) +{ + CV_LOG_DEBUG(NULL, "UI: FramebufferBackend::waitKeyEx(int delay = " << delay << ")"); + + int code = -1; + + if (delay <= 0) + { + int ch = getch_(0, 1); + CV_LOG_INFO(NULL, "UI: FramebufferBackend::getch_() take value = " << (int)ch); + code = ch; + + while ((ch = getch_(0, 0)) >= 0) + { + CV_LOG_INFO(NULL, "UI: FramebufferBackend::getch_() take value = " + << (int)ch << " (additional code on )"); + code = ch; + } + } + else + { + bool f_kbhit = false; + while (!(f_kbhit = kbhit()) && (delay > 0)) + { + delay -= 1; + usleep(1000); + } + if (f_kbhit) + { + CV_LOG_INFO(NULL, "UI: FramebufferBackend kbhit is True "); + + int ch = getch_(0, 1); + CV_LOG_INFO(NULL, "UI: FramebufferBackend::getch_() take value = " << (int)ch); + code = ch; + + while ((ch = getch_(0, 0)) >= 0) + { + CV_LOG_INFO(NULL, "UI: FramebufferBackend::getch_() take value = " + << (int)ch << " (additional code on )"); + code = ch; + } + } + } + + CV_LOG_INFO(NULL, "UI: FramebufferBackend::waitKeyEx() result code = " << code); + return code; +} + +int FramebufferBackend::pollKey() +{ + CV_LOG_DEBUG(NULL, "UI: FramebufferBackend::pollKey()"); + int code = -1; + bool f_kbhit = false; + f_kbhit = kbhit(); + + if (f_kbhit) + { + CV_LOG_INFO(NULL, "UI: FramebufferBackend kbhit is True "); + + int ch = getch_(0, 1); + CV_LOG_INFO(NULL, "UI: FramebufferBackend::getch_() take value = " << (int)ch); + code = ch; + + while ((ch = getch_(0, 0)) >= 0) + { + CV_LOG_INFO(NULL, "UI: FramebufferBackend::getch_() take value = " + << (int)ch << " (additional code on )"); + code = ch; + } + } + + return code; +} + +const std::string FramebufferBackend::getName() const +{ + return "FB"; +} + +}} // cv::highgui_backend:: diff --git a/modules/highgui/src/window_framebuffer.hpp b/modules/highgui/src/window_framebuffer.hpp new file mode 100644 index 0000000000..e7bf50dff6 --- /dev/null +++ b/modules/highgui/src/window_framebuffer.hpp @@ -0,0 +1,135 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. + +#ifndef OPENCV_HIGHGUI_WINDOWS_FRAMEBUFFER_HPP +#define OPENCV_HIGHGUI_WINDOWS_FRAMEBUFFER_HPP + +#include "backend.hpp" + +#include +#include + +#include + +namespace cv { +namespace highgui_backend { + +enum OpenCVFBMode{ + FB_MODE_EMU, + FB_MODE_FB, + FB_MODE_XVFB +}; + +class FramebufferBackend; +class FramebufferWindow : public UIWindow +{ + FramebufferBackend &backend; + std::string FB_ID; + Rect windowRect; + + int flags; + Mat currentImg; + +public: + FramebufferWindow(FramebufferBackend &backend, int flags); + virtual ~FramebufferWindow(); + + virtual void imshow(InputArray image) override; + + virtual double getProperty(int prop) const override; + virtual bool setProperty(int prop, double value) override; + + virtual void resize(int width, int height) override; + virtual void move(int x, int y) override; + + virtual Rect getImageRect() const override; + + virtual void setTitle(const std::string& title) override; + + virtual void setMouseCallback(MouseCallback onMouse, void* userdata /*= 0*/) override; + + virtual std::shared_ptr createTrackbar( + const std::string& name, + int count, + TrackbarCallback onChange /*= 0*/, + void* userdata /*= 0*/ + ) override; + + virtual std::shared_ptr findTrackbar(const std::string& name) override; + + virtual const std::string& getID() const override; + + virtual bool isActive() const override; + + virtual void destroy() override; +}; // FramebufferWindow + +class FramebufferBackend: public UIBackend +{ + OpenCVFBMode mode; + + struct termios old, current; + + void initTermios(int echo, int wait); + void resetTermios(void); + int getch_(int echo, int wait); + bool kbhit(); + + fb_var_screeninfo varInfo; + fb_fix_screeninfo fixInfo; + int fbWidth; + int fbHeight; + int fbXOffset; + int fbYOffset; + int fbBitsPerPixel; + int fbLineLength; + long int fbScreenSize; + unsigned char* fbPointer; + unsigned int fbPointer_dist; + Mat backgroundBuff; + + int fbOpenAndGetInfo(); + int fbID; + + unsigned int xvfb_len_header; + unsigned int xvfb_len_colors; + unsigned int xvfb_len_pixmap; + int XvfbOpenAndGetInfo(); + +public: + + fb_var_screeninfo &getVarInfo(); + fb_fix_screeninfo &getFixInfo(); + int getFramebuffrerID(); + int getFBWidth(); + int getFBHeight(); + int getFBXOffset(); + int getFBYOffset(); + int getFBBitsPerPixel(); + int getFBLineLength(); + unsigned char* getFBPointer(); + Mat& getBackgroundBuff(); + OpenCVFBMode getMode(); + + FramebufferBackend(); + + virtual ~FramebufferBackend(); + + virtual void destroyAllWindows()override; + + // namedWindow + virtual std::shared_ptr createWindow( + const std::string& winname, + int flags + )override; + + virtual int waitKeyEx(int delay /*= 0*/)override; + virtual int pollKey() override; + + virtual const std::string getName() const override; +}; + +}} // cv::highgui_backend:: + +#endif diff --git a/modules/highgui/test/test_gui.cpp b/modules/highgui/test/test_gui.cpp index 5b72545faf..b8093e07ec 100644 --- a/modules/highgui/test/test_gui.cpp +++ b/modules/highgui/test/test_gui.cpp @@ -148,7 +148,8 @@ static void Foo(int, void* counter) && !defined HAVE_WIN32UI \ && !defined HAVE_WAYLAND \ ) \ - || defined(__APPLE__) // test fails on Mac (cocoa) + || defined(__APPLE__) /* test fails on Mac (cocoa) */ \ + || defined HAVE_FRAMEBUFFER /* trackbar is not supported */ TEST(Highgui_GUI, DISABLED_trackbar_unsafe) #else TEST(Highgui_GUI, trackbar_unsafe) @@ -188,7 +189,8 @@ void testTrackbarCallback(int pos, void* param) && !defined HAVE_WIN32UI \ && !defined HAVE_WAYLAND \ ) \ - || defined(__APPLE__) // test fails on Mac (cocoa) + || defined(__APPLE__) /* test fails on Mac (cocoa) */ \ + || defined HAVE_FRAMEBUFFER /* trackbar is not supported */ TEST(Highgui_GUI, DISABLED_trackbar) #else TEST(Highgui_GUI, trackbar) From 445022682e98e996dd2139557d6c62d14811b246 Mon Sep 17 00:00:00 2001 From: Alexander Smorkalov <2536374+asmorkalov@users.noreply.github.com> Date: Thu, 27 Jun 2024 19:11:05 +0300 Subject: [PATCH 21/30] Merge pull request #25789 from asmorkalov:as/HAL_meanStdDev_tails Fill mean and stdDev tails with zeros for HAL branch in meanStdDev #25789 as it's done for other branches. ### Pull Request Readiness Checklist See details at https://github.com/opencv/opencv/wiki/How_to_contribute#making-a-good-pull-request - [x] I agree to contribute to the project under Apache 2 License. - [x] To the best of my knowledge, the proposed patch is not based on a code under GPL or another license that is incompatible with OpenCV - [x] The PR is proposed to the proper branch - [ ] There is a reference to the original bug report and related work - [ ] There is accuracy test, performance test and test data in opencv_extra repository, if applicable Patch to opencv_extra has the same branch name. - [ ] The feature is well documented and sample code can be built with the project CMake --- modules/core/src/mean.dispatch.cpp | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/modules/core/src/mean.dispatch.cpp b/modules/core/src/mean.dispatch.cpp index b6ea926255..52c0954ff8 100644 --- a/modules/core/src/mean.dispatch.cpp +++ b/modules/core/src/mean.dispatch.cpp @@ -544,6 +544,10 @@ void meanStdDev(InputArray _src, OutputArray _mean, OutputArray _sdv, InputArray int dcn = (int)mean_mat.total(); CV_Assert( mean_mat.type() == CV_64F && mean_mat.isContinuous() && (mean_mat.cols == 1 || mean_mat.rows == 1) && dcn >= cn ); + + double* dptr = mean_mat.ptr(); + for(k = cn ; k < dcn; k++ ) + dptr[k] = 0; } if (_sdv.needed()) @@ -555,6 +559,11 @@ void meanStdDev(InputArray _src, OutputArray _mean, OutputArray _sdv, InputArray int dcn = (int)stddev_mat.total(); CV_Assert( stddev_mat.type() == CV_64F && stddev_mat.isContinuous() && (stddev_mat.cols == 1 || stddev_mat.rows == 1) && dcn >= cn ); + + double* dptr = stddev_mat.ptr(); + for(k = cn ; k < dcn; k++ ) + dptr[k] = 0; + } if (src.isContinuous() && mask.isContinuous()) @@ -646,23 +655,17 @@ void meanStdDev(InputArray _src, OutputArray _mean, OutputArray _sdv, InputArray if (_mean.needed()) { const double* sptr = s; - int dcn = (int)mean_mat.total(); double* dptr = mean_mat.ptr(); for( k = 0; k < cn; k++ ) dptr[k] = sptr[k]; - for( ; k < dcn; k++ ) - dptr[k] = 0; } if (_sdv.needed()) { const double* sptr = sq; - int dcn = (int)stddev_mat.total(); double* dptr = stddev_mat.ptr(); for( k = 0; k < cn; k++ ) dptr[k] = sptr[k]; - for( ; k < dcn; k++ ) - dptr[k] = 0; } } From ee2b0f9d635437954da9c08c05800adeb68fbe67 Mon Sep 17 00:00:00 2001 From: Alexander Smorkalov Date: Thu, 27 Jun 2024 08:21:41 +0300 Subject: [PATCH 22/30] Relax equalizeHist test for some HAL implementations. --- modules/imgproc/test/test_histograms.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/imgproc/test/test_histograms.cpp b/modules/imgproc/test/test_histograms.cpp index 584b7087c3..b6a97ef395 100644 --- a/modules/imgproc/test/test_histograms.cpp +++ b/modules/imgproc/test/test_histograms.cpp @@ -2087,8 +2087,8 @@ TEST_P(Imgproc_Equalize_Hist, accuracy) ASSERT_EQ(CV_8UC1, dst.type()); ASSERT_EQ(gold.size(), dst.size()); - int nz = cv::countNonZero(dst != gold); - ASSERT_EQ(nz, 0); + EXPECT_MAT_NEAR(dst, gold, 1); + EXPECT_MAT_N_DIFF(dst, gold, 0.05 * size.area()); // The 5% range could be accomodated to HAL } INSTANTIATE_TEST_CASE_P(Imgproc_Hist, Imgproc_Equalize_Hist, ::testing::Combine( From 786726719fbb82dce6d549c935ac3940b6f1e3fe Mon Sep 17 00:00:00 2001 From: Maxim Milashchenko <67949029+MaximMilashchenko@users.noreply.github.com> Date: Fri, 28 Jun 2024 09:00:16 +0300 Subject: [PATCH 23/30] Merge pull request #25793 from MaximMilashchenko:hal_rvv Fixed build error hal_rvv_071 #25793 Fixed bug with enabling vector header when vector extension is disabled (RVV=OFF) in hal_rvv_071 --- 3rdparty/hal_rvv/hal_rvv.hpp | 2 -- 3rdparty/hal_rvv/version/hal_rvv_071.hpp | 2 ++ CMakeLists.txt | 4 ++++ 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/3rdparty/hal_rvv/hal_rvv.hpp b/3rdparty/hal_rvv/hal_rvv.hpp index 8bb3545aab..e10a3258e9 100644 --- a/3rdparty/hal_rvv/hal_rvv.hpp +++ b/3rdparty/hal_rvv/hal_rvv.hpp @@ -5,8 +5,6 @@ #ifndef OPENCV_HAL_RVV_HPP_INCLUDED #define OPENCV_HAL_RVV_HPP_INCLUDED -#include - #include "opencv2/core/hal/interface.h" #ifndef CV_HAL_RVV_071_ENABLED diff --git a/3rdparty/hal_rvv/version/hal_rvv_071.hpp b/3rdparty/hal_rvv/version/hal_rvv_071.hpp index f86f8c363a..db235d6139 100644 --- a/3rdparty/hal_rvv/version/hal_rvv_071.hpp +++ b/3rdparty/hal_rvv/version/hal_rvv_071.hpp @@ -5,6 +5,8 @@ #ifndef OPENCV_HAL_RVV_071_HPP_INCLUDED #define OPENCV_HAL_RVV_071_HPP_INCLUDED +#include + #include namespace cv { namespace cv_hal_rvv { diff --git a/CMakeLists.txt b/CMakeLists.txt index 58437a37b1..b36482d9ce 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1037,9 +1037,13 @@ foreach(hal ${OpenCV_HAL}) message(STATUS "NDSRVP: Andes GNU Toolchain DSP extension is not open, disabling ndsrvp...") endif() elseif(hal STREQUAL "halrvv") + if(";${CPU_BASELINE_FINAL};" MATCHES ";RVV;") add_subdirectory(3rdparty/hal_rvv/) ocv_hal_register(RVV_HAL_LIBRARIES RVV_HAL_HEADERS RVV_HAL_INCLUDE_DIRS) list(APPEND OpenCV_USED_HAL "HAL RVV (ver ${RVV_HAL_VERSION})") + else() + message(STATUS "HAL RVV: RVV is not available, disabling halrvv...") + endif() elseif(hal STREQUAL "openvx") add_subdirectory(3rdparty/openvx) ocv_hal_register(OPENVX_HAL_LIBRARIES OPENVX_HAL_HEADERS OPENVX_HAL_INCLUDE_DIRS) From 4500eb937f662ec273d61dc90d83560d6e52a54d Mon Sep 17 00:00:00 2001 From: Alexander Smorkalov Date: Fri, 28 Jun 2024 10:45:52 +0300 Subject: [PATCH 24/30] Drop redundant dependency from download_models.py --- samples/dnn/download_models.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/samples/dnn/download_models.py b/samples/dnn/download_models.py index 4d94765182..f4dd45559a 100644 --- a/samples/dnn/download_models.py +++ b/samples/dnn/download_models.py @@ -3,7 +3,6 @@ Helper module to download extra data from Internet ''' from __future__ import print_function import os -import cv2 import sys import yaml import argparse @@ -361,4 +360,4 @@ if __name__ == '__main__': if selected_model_name and not m.name.startswith(selected_model_name): continue print('Model: ' + selected_model_name) - m.get() \ No newline at end of file + m.get() From 310169490af0861683f66687961cebdae929528a Mon Sep 17 00:00:00 2001 From: Alexander Smorkalov Date: Fri, 28 Jun 2024 13:02:59 +0300 Subject: [PATCH 25/30] Exclude cap_ios.h from installation where it's not needed. --- modules/videoio/CMakeLists.txt | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/modules/videoio/CMakeLists.txt b/modules/videoio/CMakeLists.txt index 776c1bb84d..c8a03e72ae 100644 --- a/modules/videoio/CMakeLists.txt +++ b/modules/videoio/CMakeLists.txt @@ -49,6 +49,9 @@ endif() # Removing WinRT API headers by default list(REMOVE_ITEM videoio_ext_hdrs "${CMAKE_CURRENT_LIST_DIR}/include/opencv2/${name}/cap_winrt.hpp") +# Remove iOS API header by default +list(REMOVE_ITEM videoio_ext_hdrs "${CMAKE_CURRENT_LIST_DIR}/include/opencv2/${name}/cap_ios.h") + if(DEFINED WINRT AND NOT DEFINED ENABLE_WINRT_MODE_NATIVE) set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /ZW") endif() @@ -221,6 +224,7 @@ if(TARGET ocv.3rdparty.cap_ios) ${CMAKE_CURRENT_LIST_DIR}/src/cap_ios_photo_camera.mm ${CMAKE_CURRENT_LIST_DIR}/src/cap_ios_video_camera.mm) list(APPEND tgts ocv.3rdparty.cap_ios) + list(APPEND videoio_ext_hdrs "${CMAKE_CURRENT_LIST_DIR}/include/opencv2/${name}/cap_ios.h") endif() if(TARGET ocv.3rdparty.android_mediandk) From b7c9221b805c948debde54d1339aca339310f61c Mon Sep 17 00:00:00 2001 From: Alexander Smorkalov Date: Fri, 28 Jun 2024 16:54:14 +0300 Subject: [PATCH 26/30] Force contributors to define Apache 2.0 license for the new PRs. --- .github/PULL_REQUEST_TEMPLATE.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 5e2e911cc8..026b11a2db 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -2,8 +2,8 @@ See details at https://github.com/opencv/opencv/wiki/How_to_contribute#making-a-good-pull-request -- [ ] I agree to contribute to the project under Apache 2 License. -- [ ] To the best of my knowledge, the proposed patch is not based on a code under GPL or another license that is incompatible with OpenCV +- [x] I agree to contribute to the project under Apache 2 License. +- [x] To the best of my knowledge, the proposed patch is not based on a code under GPL or another license that is incompatible with OpenCV - [ ] The PR is proposed to the proper branch - [ ] There is a reference to the original bug report and related work - [ ] There is accuracy test, performance test and test data in opencv_extra repository, if applicable From 1e5407a9ba571c8aedceaf0c2897f915dd98d925 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dan=20Ma=C5=A1ek?= Date: Sat, 29 Jun 2024 21:39:49 +0200 Subject: [PATCH 27/30] Fix #25833: The correct way to disable top-most state is with HWND_NOTOPMOST, not HWND_TOP. --- modules/highgui/src/window_w32.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/highgui/src/window_w32.cpp b/modules/highgui/src/window_w32.cpp index 32b2392904..381c9f52a4 100644 --- a/modules/highgui/src/window_w32.cpp +++ b/modules/highgui/src/window_w32.cpp @@ -681,7 +681,7 @@ void cvSetPropTopmost_W32(const char* name, const bool topmost) static bool setPropTopmost_(CvWindow& window, bool topmost) { - HWND flag = topmost ? HWND_TOPMOST : HWND_TOP; + HWND flag = topmost ? HWND_TOPMOST : HWND_NOTOPMOST; BOOL success = SetWindowPos(window.frame, flag, 0, 0, 0, 0, SWP_NOMOVE | SWP_NOSIZE); if (!success) From bbf65a166e027f079835920f117db5725d3ec030 Mon Sep 17 00:00:00 2001 From: Mikhail Khachayants Date: Sun, 30 Jun 2024 18:43:04 +0300 Subject: [PATCH 28/30] Fix file descriptor leak in HDR decoder --- modules/imgcodecs/src/grfmt_hdr.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/modules/imgcodecs/src/grfmt_hdr.cpp b/modules/imgcodecs/src/grfmt_hdr.cpp index 3a18b5bbeb..c9fec94aa3 100644 --- a/modules/imgcodecs/src/grfmt_hdr.cpp +++ b/modules/imgcodecs/src/grfmt_hdr.cpp @@ -59,6 +59,9 @@ HdrDecoder::HdrDecoder() HdrDecoder::~HdrDecoder() { + if(file) { + fclose(file); + } } size_t HdrDecoder::signatureLength() const From ce1d840adfe68504fd9b35f1568286e1911599c9 Mon Sep 17 00:00:00 2001 From: Thirumalai Nagalingam <81082960+thiru31@users.noreply.github.com> Date: Sat, 29 Jun 2024 11:58:48 +0530 Subject: [PATCH 29/30] Fix: compilation Issue on ARM64 (msys2 clangarm) Added a definition for M_PI in the code to resolve a compilation error encountered when building OpenCV on the MSYS2 environment. The M_PI constant was not defined, causing the compilation to fail. --- 3rdparty/carotene/CMakeLists.txt | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/3rdparty/carotene/CMakeLists.txt b/3rdparty/carotene/CMakeLists.txt index aa95956e7f..dc780fd645 100644 --- a/3rdparty/carotene/CMakeLists.txt +++ b/3rdparty/carotene/CMakeLists.txt @@ -52,5 +52,9 @@ if(WITH_NEON) endif() endif() + if(MINGW) + target_compile_definitions(carotene_objs PRIVATE "-D_USE_MATH_DEFINES=1") + endif() + # we add dummy file to fix XCode build add_library(carotene STATIC ${OPENCV_3RDPARTY_EXCLUDE_FROM_ALL} "$" "${CAROTENE_SOURCE_DIR}/dummy.cpp") From 39a7b3d1869d31df92029503b9a8670071eac596 Mon Sep 17 00:00:00 2001 From: WU Jia <35290826+kaingwade@users.noreply.github.com> Date: Mon, 1 Jul 2024 15:55:35 +0800 Subject: [PATCH 30/30] Merge pull request #25813 from kaingwade:orbbec-astra Update the tutorial of using Orbbec Astra cameras #25813 This PR is the backport of Orbbec OpenNI-based Astra camera related changes from #25410 to the 4.x branch, which includes updating the tutorial of Orbbec Astra cameras, renaming `orbbec_astra.cpp`. ### Pull Request Readiness Checklist See details at https://github.com/opencv/opencv/wiki/How_to_contribute#making-a-good-pull-request - [x] I agree to contribute to the project under Apache 2 License. - [x] To the best of my knowledge, the proposed patch is not based on a code under GPL or another license that is incompatible with OpenCV - [x] The PR is proposed to the proper branch - [ ] There is a reference to the original bug report and related work - [ ] There is accuracy test, performance test and test data in opencv_extra repository, if applicable Patch to opencv_extra has the same branch name. - [x] The feature is well documented and sample code can be built with the project CMake --- doc/tutorials/app/intelperc.markdown | 2 +- doc/tutorials/app/kinect_openni.markdown | 2 +- ....markdown => orbbec_astra_openni.markdown} | 24 ++++++++++++------- .../app/table_of_content_app.markdown | 2 +- .../openni_orbbec_astra.cpp} | 0 5 files changed, 18 insertions(+), 12 deletions(-) rename doc/tutorials/app/{orbbec_astra.markdown => orbbec_astra_openni.markdown} (86%) rename samples/cpp/tutorial_code/videoio/{orbbec_astra/orbbec_astra.cpp => openni_orbbec_astra/openni_orbbec_astra.cpp} (100%) diff --git a/doc/tutorials/app/intelperc.markdown b/doc/tutorials/app/intelperc.markdown index 574bfc9e6a..315bcf3e32 100644 --- a/doc/tutorials/app/intelperc.markdown +++ b/doc/tutorials/app/intelperc.markdown @@ -3,7 +3,7 @@ Using Creative Senz3D and other Intel RealSense SDK compatible depth sensors {#t @tableofcontents -@prev_tutorial{tutorial_orbbec_astra} +@prev_tutorial{tutorial_orbbec_astra_openni} @next_tutorial{tutorial_wayland_ubuntu} ![hardwares](images/realsense.jpg) diff --git a/doc/tutorials/app/kinect_openni.markdown b/doc/tutorials/app/kinect_openni.markdown index 737b935f5b..4bae1b0182 100644 --- a/doc/tutorials/app/kinect_openni.markdown +++ b/doc/tutorials/app/kinect_openni.markdown @@ -4,7 +4,7 @@ Using Kinect and other OpenNI compatible depth sensors {#tutorial_kinect_openni} @tableofcontents @prev_tutorial{tutorial_video_write} -@next_tutorial{tutorial_orbbec_astra} +@next_tutorial{tutorial_orbbec_astra_openni} Depth sensors compatible with OpenNI (Kinect, XtionPRO, ...) are supported through VideoCapture diff --git a/doc/tutorials/app/orbbec_astra.markdown b/doc/tutorials/app/orbbec_astra_openni.markdown similarity index 86% rename from doc/tutorials/app/orbbec_astra.markdown rename to doc/tutorials/app/orbbec_astra_openni.markdown index 01a685db57..a5d11108dd 100644 --- a/doc/tutorials/app/orbbec_astra.markdown +++ b/doc/tutorials/app/orbbec_astra_openni.markdown @@ -1,4 +1,4 @@ -Using Orbbec Astra 3D cameras {#tutorial_orbbec_astra} +Using Orbbec Astra 3D cameras {#tutorial_orbbec_astra_openni} ====================================================== @tableofcontents @@ -9,7 +9,7 @@ Using Orbbec Astra 3D cameras {#tutorial_orbbec_astra} ### Introduction -This tutorial is devoted to the Astra Series of Orbbec 3D cameras (https://orbbec3d.com/index/Product/info.html?cate=38&id=36). +This tutorial is devoted to the Astra Series of Orbbec 3D cameras (https://www.orbbec.com/products/structured-light-camera/astra-series/). That cameras have a depth sensor in addition to a common color sensor. The depth sensors can be read using the open source OpenNI API with @ref cv::VideoCapture class. The video stream is provided through the regular camera interface. @@ -18,7 +18,7 @@ camera interface. In order to use the Astra camera's depth sensor with OpenCV you should do the following steps: --# Download the latest version of Orbbec OpenNI SDK (from here ). +-# Download the latest version of Orbbec OpenNI SDK (from here ). Unzip the archive, choose the build according to your operating system and follow installation steps provided in the Readme file. @@ -70,6 +70,12 @@ In order to use the Astra camera's depth sensor with OpenCV you should do the fo echo "exit" @endcode + @note The last tried version `2.3.0.86_202210111154_4c8f5aa4_beta6` does not work correctly with + modern Linux, even after libusb rebuild as recommended by the instruction. The last know good + configuration is version 2.3.0.63 (tested with Ubuntu 18.04 amd64). It's not provided officialy + with the downloading page, but published by Orbbec technical suport on Orbbec community forum + [here](https://3dclub.orbbec3d.com/t/universal-download-thread-for-astra-series-cameras/622). + -# Now you can configure OpenCV with OpenNI support enabled by setting the `WITH_OPENNI2` flag in CMake. You may also like to enable the `BUILD_EXAMPLES` flag to get a code sample working with your Astra camera. Run the following commands in the directory containing OpenCV source code to enable OpenNI support: @@ -106,7 +112,7 @@ can be read using the OpenNI interface with @ref cv::VideoCapture class. The vid not available through OpenNI API and is only provided via the regular camera interface. So, to get both depth and color frames, two @ref cv::VideoCapture objects should be created: -@snippetlineno samples/cpp/tutorial_code/videoio/orbbec_astra/orbbec_astra.cpp Open streams +@snippetlineno samples/cpp/tutorial_code/videoio/openni_orbbec_astra/openni_orbbec_astra.cpp Open streams The first object will use the OpenNI2 API to retrieve depth data. The second one uses the Video4Linux2 interface to access the color sensor. Note that the example above assumes that @@ -119,12 +125,12 @@ For this example, we’ll configure width and height of both streams to VGA reso the maximum resolution available for both sensors, and we’d like both stream parameters to be the same for easier color-to-depth data registration: -@snippetlineno samples/cpp/tutorial_code/videoio/orbbec_astra/orbbec_astra.cpp Setup streams +@snippetlineno samples/cpp/tutorial_code/videoio/openni_orbbec_astra/openni_orbbec_astra.cpp Setup streams For setting and retrieving some property of sensor data generators use @ref cv::VideoCapture::set and @ref cv::VideoCapture::get methods respectively, e.g. : -@snippetlineno samples/cpp/tutorial_code/videoio/orbbec_astra/orbbec_astra.cpp Get properties +@snippetlineno samples/cpp/tutorial_code/videoio/openni_orbbec_astra/openni_orbbec_astra.cpp Get properties The following properties of cameras available through OpenNI interface are supported for the depth generator: @@ -156,7 +162,7 @@ As there are two video sources that should be read simultaneously, it’s necess threads to avoid blocking. Example implementation that gets frames from each sensor in a new thread and stores them in a list along with their timestamps: -@snippetlineno samples/cpp/tutorial_code/videoio/orbbec_astra/orbbec_astra.cpp Read streams +@snippetlineno samples/cpp/tutorial_code/videoio/openni_orbbec_astra/openni_orbbec_astra.cpp Read streams VideoCapture can retrieve the following data: @@ -177,7 +183,7 @@ two video streams may become out of sync even when both streams are set up for t A post-synchronization procedure can be applied to the streams to combine depth and color frames into pairs. The sample code below demonstrates this procedure: -@snippetlineno samples/cpp/tutorial_code/videoio/orbbec_astra/orbbec_astra.cpp Pair frames +@snippetlineno samples/cpp/tutorial_code/videoio/openni_orbbec_astra/openni_orbbec_astra.cpp Pair frames In the code snippet above the execution is blocked until there are some frames in both frame lists. When there are new frames, their timestamps are being checked -- if they differ more than a half of @@ -194,5 +200,5 @@ but the depth data makes it easy. ![Depth frame](images/astra_depth.png) The complete implementation can be found in -[orbbec_astra.cpp](https://github.com/opencv/opencv/tree/4.x/samples/cpp/tutorial_code/videoio/orbbec_astra/orbbec_astra.cpp) +[openni_orbbec_astra.cpp](https://github.com/opencv/opencv/tree/4.x/samples/cpp/tutorial_code/videoio/openni_orbbec_astra/openni_orbbec_astra.cpp) in `samples/cpp/tutorial_code/videoio` directory. diff --git a/doc/tutorials/app/table_of_content_app.markdown b/doc/tutorials/app/table_of_content_app.markdown index 6671f6b541..c4dc34b23d 100644 --- a/doc/tutorials/app/table_of_content_app.markdown +++ b/doc/tutorials/app/table_of_content_app.markdown @@ -6,6 +6,6 @@ Application utils (highgui, imgcodecs, videoio modules) {#tutorial_table_of_cont - @subpage tutorial_video_input_psnr_ssim - @subpage tutorial_video_write - @subpage tutorial_kinect_openni -- @subpage tutorial_orbbec_astra +- @subpage tutorial_orbbec_astra_openni - @subpage tutorial_intelperc - @subpage tutorial_wayland_ubuntu diff --git a/samples/cpp/tutorial_code/videoio/orbbec_astra/orbbec_astra.cpp b/samples/cpp/tutorial_code/videoio/openni_orbbec_astra/openni_orbbec_astra.cpp similarity index 100% rename from samples/cpp/tutorial_code/videoio/orbbec_astra/orbbec_astra.cpp rename to samples/cpp/tutorial_code/videoio/openni_orbbec_astra/openni_orbbec_astra.cpp