From bd565df379fdc0bee4f10a65b4a1c15cb47bc709 Mon Sep 17 00:00:00 2001 From: Alexander Smorkalov Date: Tue, 31 Oct 2023 11:23:47 +0300 Subject: [PATCH 001/156] Added Java bindings for BOWImgDescriptorExtractor constructor. --- .../features2d/include/opencv2/features2d.hpp | 4 +- .../test/BOWImgDescriptorExtractorTest.java | 48 +++++++++++++++++++ 2 files changed, 50 insertions(+), 2 deletions(-) create mode 100644 modules/features2d/misc/java/test/BOWImgDescriptorExtractorTest.java diff --git a/modules/features2d/include/opencv2/features2d.hpp b/modules/features2d/include/opencv2/features2d.hpp index a7d348d833..e0d0187bdc 100644 --- a/modules/features2d/include/opencv2/features2d.hpp +++ b/modules/features2d/include/opencv2/features2d.hpp @@ -1544,8 +1544,8 @@ public: @param dmatcher Descriptor matcher that is used to find the nearest word of the trained vocabulary for each keypoint descriptor of the image. */ - CV_WRAP BOWImgDescriptorExtractor( const Ptr& dextractor, - const Ptr& dmatcher ); + CV_WRAP BOWImgDescriptorExtractor( const Ptr& dextractor, + const Ptr& dmatcher ); /** @overload */ BOWImgDescriptorExtractor( const Ptr& dmatcher ); virtual ~BOWImgDescriptorExtractor(); diff --git a/modules/features2d/misc/java/test/BOWImgDescriptorExtractorTest.java b/modules/features2d/misc/java/test/BOWImgDescriptorExtractorTest.java new file mode 100644 index 0000000000..a8ff001e85 --- /dev/null +++ b/modules/features2d/misc/java/test/BOWImgDescriptorExtractorTest.java @@ -0,0 +1,48 @@ +package org.opencv.test.features2d; + +import org.opencv.core.Core; +import org.opencv.core.CvType; +import org.opencv.core.Mat; +import org.opencv.core.MatOfKeyPoint; +import org.opencv.core.Point; +import org.opencv.core.Scalar; +import org.opencv.core.KeyPoint; +import org.opencv.features2d.ORB; +import org.opencv.features2d.DescriptorMatcher; +import org.opencv.features2d.BOWImgDescriptorExtractor; +import org.opencv.test.OpenCVTestCase; +import org.opencv.test.OpenCVTestRunner; +import org.opencv.imgproc.Imgproc; + +public class BOWImgDescriptorExtractorTest extends OpenCVTestCase { + + ORB extractor; + DescriptorMatcher matcher; + int matSize; + + public static void assertDescriptorsClose(Mat expected, Mat actual, int allowedDistance) { + double distance = Core.norm(expected, actual, Core.NORM_HAMMING); + assertTrue("expected:<" + allowedDistance + "> but was:<" + distance + ">", distance <= allowedDistance); + } + + private Mat getTestImg() { + Mat cross = new Mat(matSize, matSize, CvType.CV_8U, new Scalar(255)); + Imgproc.line(cross, new Point(20, matSize / 2), new Point(matSize - 21, matSize / 2), new Scalar(100), 2); + Imgproc.line(cross, new Point(matSize / 2, 20), new Point(matSize / 2, matSize - 21), new Scalar(100), 2); + + return cross; + } + + @Override + protected void setUp() throws Exception { + super.setUp(); + extractor = ORB.create(); + matcher = DescriptorMatcher.create(DescriptorMatcher.BRUTEFORCE); + matSize = 100; + } + + public void testCreate() { + BOWImgDescriptorExtractor bow = new BOWImgDescriptorExtractor(extractor, matcher); + } + +} From abc4eeb9a704b525a6f46419dc85478d7b33a519 Mon Sep 17 00:00:00 2001 From: Alexander Smorkalov Date: Wed, 1 Nov 2023 17:45:04 +0300 Subject: [PATCH 002/156] Add JavaScript bindings for SimpleBlobDetector. --- modules/js/src/core_bindings.cpp | 4 ++++ modules/js/test/test_features2d.js | 9 +++++++++ platforms/js/opencv_js.config.py | 3 ++- 3 files changed, 15 insertions(+), 1 deletion(-) diff --git a/modules/js/src/core_bindings.cpp b/modules/js/src/core_bindings.cpp index addee2de20..dda6f9fe16 100644 --- a/modules/js/src/core_bindings.cpp +++ b/modules/js/src/core_bindings.cpp @@ -99,6 +99,10 @@ typedef QRCodeDetectorAruco::Params QRCodeDetectorAruco_Params; using namespace cv::dnn; #endif +#ifdef HAVE_OPENCV_FEATURES2D +typedef SimpleBlobDetector::Params SimpleBlobDetector_Params; +#endif + #ifdef HAVE_OPENCV_VIDEO typedef TrackerMIL::Params TrackerMIL_Params; #endif diff --git a/modules/js/test/test_features2d.js b/modules/js/test/test_features2d.js index 45e3d4d715..c5eb73a123 100644 --- a/modules/js/test/test_features2d.js +++ b/modules/js/test/test_features2d.js @@ -62,6 +62,15 @@ QUnit.test('Detectors', function(assert) { assert.equal(kp.size(), 53, 'AKAZE'); }); +QUnit.test('SimpleBlobDetector', function(assert) { + let image = generateTestFrame(); + + let kp = new cv.KeyPointVector(); + let sbd = new cv.SimpleBlobDetector(); + sbd.detect(image, kp); + assert.equal(kp.size(), 0); +}); + QUnit.test('BFMatcher', function(assert) { // Generate key points. let image = generateTestFrame(); diff --git a/platforms/js/opencv_js.config.py b/platforms/js/opencv_js.config.py index 5dca863bef..44f0813838 100644 --- a/platforms/js/opencv_js.config.py +++ b/platforms/js/opencv_js.config.py @@ -158,7 +158,8 @@ features2d = {'Feature2D': ['detect', 'compute', 'detectAndCompute', 'descriptor 'FastFeatureDetector': ['create', 'setThreshold', 'getThreshold', 'setNonmaxSuppression', 'getNonmaxSuppression', 'setType', 'getType', 'getDefaultName'], 'AgastFeatureDetector': ['create', 'setThreshold', 'getThreshold', 'setNonmaxSuppression', 'getNonmaxSuppression', 'setType', 'getType', 'getDefaultName'], 'GFTTDetector': ['create', 'setMaxFeatures', 'getMaxFeatures', 'setQualityLevel', 'getQualityLevel', 'setMinDistance', 'getMinDistance', 'setBlockSize', 'getBlockSize', 'setHarrisDetector', 'getHarrisDetector', 'setK', 'getK', 'getDefaultName'], - # 'SimpleBlobDetector': ['create'], + 'SimpleBlobDetector': ['create', 'setParams', 'getParams', 'getDefaultName'], + 'SimpleBlobDetector_Params': [], 'KAZE': ['create', 'setExtended', 'getExtended', 'setUpright', 'getUpright', 'setThreshold', 'getThreshold', 'setNOctaves', 'getNOctaves', 'setNOctaveLayers', 'getNOctaveLayers', 'setDiffusivity', 'getDiffusivity', 'getDefaultName'], 'AKAZE': ['create', 'setDescriptorType', 'getDescriptorType', 'setDescriptorSize', 'getDescriptorSize', 'setDescriptorChannels', 'getDescriptorChannels', 'setThreshold', 'getThreshold', 'setNOctaves', 'getNOctaves', 'setNOctaveLayers', 'getNOctaveLayers', 'setDiffusivity', 'getDiffusivity', 'getDefaultName'], 'DescriptorMatcher': ['add', 'clear', 'empty', 'isMaskSupported', 'train', 'match', 'knnMatch', 'radiusMatch', 'clone', 'create'], From 451ee3991ee1fe57309254cdf93b16cd29a7604e Mon Sep 17 00:00:00 2001 From: Liutong HAN Date: Fri, 3 Nov 2023 10:21:13 +0800 Subject: [PATCH 003/156] Use local variable. --- .../include/opencv2/core/hal/intrin_rvv_scalable.hpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/modules/core/include/opencv2/core/hal/intrin_rvv_scalable.hpp b/modules/core/include/opencv2/core/hal/intrin_rvv_scalable.hpp index 14988fc03c..004ef7130c 100644 --- a/modules/core/include/opencv2/core/hal/intrin_rvv_scalable.hpp +++ b/modules/core/include/opencv2/core/hal/intrin_rvv_scalable.hpp @@ -1390,23 +1390,23 @@ OPENCV_HAL_IMPL_RVV_REVERSE(v_float64, 64) #define OPENCV_HAL_IMPL_RVV_EXPAND(_Tp, _Tpwvec, _Tpwvec_m2, _Tpvec, width, suffix, suffix2, cvt) \ inline void v_expand(const _Tpvec& a, _Tpwvec& b0, _Tpwvec& b1) \ { \ - _Tpwvec_m2 temp = cvt(a, vsetvlmax_e##width##m1()); \ + _Tpwvec_m2 temp = cvt(a, VTraits<_Tpvec>::vlanes()); \ b0 = vget_##suffix##m1(temp, 0); \ b1 = vget_##suffix##m1(temp, 1); \ } \ inline _Tpwvec v_expand_low(const _Tpvec& a) \ { \ - _Tpwvec_m2 temp = cvt(a, vsetvlmax_e##width##m1()); \ + _Tpwvec_m2 temp = cvt(a, VTraits<_Tpvec>::vlanes()); \ return vget_##suffix##m1(temp, 0); \ } \ inline _Tpwvec v_expand_high(const _Tpvec& a) \ { \ - _Tpwvec_m2 temp = cvt(a, vsetvlmax_e##width##m1()); \ + _Tpwvec_m2 temp = cvt(a, VTraits<_Tpvec>::vlanes()); \ return vget_##suffix##m1(temp, 1); \ } \ inline _Tpwvec v_load_expand(const _Tp* ptr) \ { \ - return cvt(vle##width##_v_##suffix2##mf2(ptr, vsetvlmax_e##width##m1()), vsetvlmax_e##width##m1()); \ + return cvt(vle##width##_v_##suffix2##mf2(ptr, VTraits<_Tpvec>::vlanes()), VTraits<_Tpvec>::vlanes()); \ } OPENCV_HAL_IMPL_RVV_EXPAND(uchar, v_uint16, vuint16m2_t, v_uint8, 8, u16, u8, vwcvtu_x) @@ -1759,8 +1759,8 @@ inline int v_scan_forward(const v_float64& a) // mask: {0,0,0,1, ...} -> {T,T,T,F, ...} #define OPENCV_HAL_IMPL_RVV_PACK_TRIPLETS(_Tpvec, v_trunc) \ inline _Tpvec v_pack_triplets(const _Tpvec& vec) { \ - size_t vl = vsetvlmax_e8m1(); \ - vuint32m1_t one = vmv_v_x_u32m1(1, vl/4); \ + size_t vl = __cv_rvv_e8m1_nlanes; \ + vuint32m1_t one = vmv_v_x_u32m1(1, __cv_rvv_e32m1_nlanes); \ vuint8m1_t zero = vmv_v_x_u8m1(0, vl); \ vuint8m1_t mask = vreinterpret_u8m1(one); \ return vcompress(vmseq(v_trunc(vslideup(zero, mask, 3, vl)), 0, vl), vec, vec, VTraits<_Tpvec>::vlanes()); \ From ea47cb3ffeedae2da53acbaff8a108cc39a1a4b9 Mon Sep 17 00:00:00 2001 From: Rostislav Vasilikhin Date: Fri, 3 Nov 2023 06:58:07 +0100 Subject: [PATCH 004/156] Merge pull request #24480 from savuor:backport_patch_nans Backport to 4.x: patchNaNs() SIMD acceleration #24480 backport from #23098 connected PR in extra: [#1118@extra](https://github.com/opencv/opencv_extra/pull/1118) ### This PR contains: * new SIMD code for `patchNaNs()` * CPU perf test
Performance comparison Geometric mean (ms) |Name of Test|noopt|sse2|avx2|sse2 vs noopt (x-factor)|avx2 vs noopt (x-factor)| |---|:-:|:-:|:-:|:-:|:-:| |PatchNaNs::OCL_PatchNaNsFixture::(640x480, 32FC1)|0.019|0.017|0.018|1.11|1.07| |PatchNaNs::OCL_PatchNaNsFixture::(640x480, 32FC4)|0.037|0.037|0.033|1.00|1.10| |PatchNaNs::OCL_PatchNaNsFixture::(1280x720, 32FC1)|0.032|0.032|0.033|0.99|0.98| |PatchNaNs::OCL_PatchNaNsFixture::(1280x720, 32FC4)|0.072|0.072|0.070|1.00|1.03| |PatchNaNs::OCL_PatchNaNsFixture::(1920x1080, 32FC1)|0.051|0.051|0.050|1.00|1.01| |PatchNaNs::OCL_PatchNaNsFixture::(1920x1080, 32FC4)|0.137|0.138|0.128|0.99|1.06| |PatchNaNs::OCL_PatchNaNsFixture::(3840x2160, 32FC1)|0.137|0.128|0.129|1.07|1.06| |PatchNaNs::OCL_PatchNaNsFixture::(3840x2160, 32FC4)|0.450|0.450|0.448|1.00|1.01| |PatchNaNs::PatchNaNsFixture::(640x480, 32FC1)|0.149|0.029|0.020|5.13|7.44| |PatchNaNs::PatchNaNsFixture::(640x480, 32FC2)|0.304|0.058|0.040|5.25|7.65| |PatchNaNs::PatchNaNsFixture::(640x480, 32FC3)|0.448|0.086|0.059|5.22|7.55| |PatchNaNs::PatchNaNsFixture::(640x480, 32FC4)|0.601|0.133|0.083|4.51|7.23| |PatchNaNs::PatchNaNsFixture::(1280x720, 32FC1)|0.451|0.093|0.060|4.83|7.52| |PatchNaNs::PatchNaNsFixture::(1280x720, 32FC2)|0.892|0.184|0.126|4.85|7.06| |PatchNaNs::PatchNaNsFixture::(1280x720, 32FC3)|1.345|0.311|0.230|4.32|5.84| |PatchNaNs::PatchNaNsFixture::(1280x720, 32FC4)|1.831|0.546|0.436|3.35|4.20| |PatchNaNs::PatchNaNsFixture::(1920x1080, 32FC1)|1.017|0.250|0.160|4.06|6.35| |PatchNaNs::PatchNaNsFixture::(1920x1080, 32FC2)|2.077|0.646|0.605|3.21|3.43| |PatchNaNs::PatchNaNsFixture::(1920x1080, 32FC3)|3.134|1.053|0.961|2.97|3.26| |PatchNaNs::PatchNaNsFixture::(1920x1080, 32FC4)|4.222|1.436|1.288|2.94|3.28| |PatchNaNs::PatchNaNsFixture::(3840x2160, 32FC1)|4.225|1.401|1.277|3.01|3.31| |PatchNaNs::PatchNaNsFixture::(3840x2160, 32FC2)|8.310|2.953|2.635|2.81|3.15| |PatchNaNs::PatchNaNsFixture::(3840x2160, 32FC3)|12.396|4.455|4.252|2.78|2.92| |PatchNaNs::PatchNaNsFixture::(3840x2160, 32FC4)|17.174|5.831|5.824|2.95|2.95|
### Pull Request Readiness Checklist See details at https://github.com/opencv/opencv/wiki/How_to_contribute#making-a-good-pull-request - [x] I agree to contribute to the project under Apache 2 License. - [x] To the best of my knowledge, the proposed patch is not based on a code under GPL or another license that is incompatible with OpenCV - [x] The PR is proposed to the proper branch - [x] There is a reference to the original bug report and related work - [x] There is accuracy test, performance test and test data in opencv_extra repository, if applicable Patch to opencv_extra has the same branch name. - [x] The feature is well documented and sample code can be built with the project CMake --- modules/core/include/opencv2/core.hpp | 2 +- modules/core/perf/perf_arithm.cpp | 66 +++++++++++++++++++++++++++ modules/core/src/mathfuncs.cpp | 35 ++++++++------ 3 files changed, 88 insertions(+), 15 deletions(-) diff --git a/modules/core/include/opencv2/core.hpp b/modules/core/include/opencv2/core.hpp index 3cd5901af4..bd5de32d8d 100644 --- a/modules/core/include/opencv2/core.hpp +++ b/modules/core/include/opencv2/core.hpp @@ -1697,7 +1697,7 @@ elements. CV_EXPORTS_W bool checkRange(InputArray a, bool quiet = true, CV_OUT Point* pos = 0, double minVal = -DBL_MAX, double maxVal = DBL_MAX); -/** @brief converts NaNs to the given number +/** @brief Replaces NaNs by given number @param a input/output matrix (CV_32F type). @param val value to convert the NaNs */ diff --git a/modules/core/perf/perf_arithm.cpp b/modules/core/perf/perf_arithm.cpp index 872963fc65..c4cc7500a7 100644 --- a/modules/core/perf/perf_arithm.cpp +++ b/modules/core/perf/perf_arithm.cpp @@ -1,5 +1,6 @@ #include "perf_precomp.hpp" #include +#include "opencv2/core/softfloat.hpp" namespace opencv_test { @@ -451,4 +452,69 @@ INSTANTIATE_TEST_CASE_P(/*nothing*/ , BinaryOpTest, ) ); +///////////// PatchNaNs //////////////////////// + +template +_Tp randomNan(RNG& rng); + +template<> +float randomNan(RNG& rng) +{ + uint32_t r = rng.next(); + Cv32suf v; + v.u = r; + // exp & set a bit to avoid zero mantissa + v.u = v.u | 0x7f800001; + return v.f; +} + +template<> +double randomNan(RNG& rng) +{ + uint32_t r0 = rng.next(); + uint32_t r1 = rng.next(); + Cv64suf v; + v.u = (uint64_t(r0) << 32) | uint64_t(r1); + // exp &set a bit to avoid zero mantissa + v.u = v.u | 0x7ff0000000000001; + return v.f; +} + +typedef Size_MatType PatchNaNsFixture; + +PERF_TEST_P_(PatchNaNsFixture, PatchNaNs) +{ + const Size_MatType_t params = GetParam(); + Size srcSize = get<0>(params); + const int type = get<1>(params), cn = CV_MAT_CN(type); + + Mat src(srcSize, type); + declare.in(src, WARMUP_RNG).out(src); + + // generating NaNs + { + srcSize.width *= cn; + RNG& rng = theRNG(); + for (int y = 0; y < srcSize.height; ++y) + { + float *const ptrf = src.ptr(y); + for (int x = 0; x < srcSize.width; ++x) + { + ptrf[x] = (x + y) % 2 == 0 ? randomNan(rng) : ptrf[x]; + } + } + } + + TEST_CYCLE() cv::patchNaNs(src, 17.7); + + SANITY_CHECK(src); +} + +INSTANTIATE_TEST_CASE_P(/*nothing*/ , PatchNaNsFixture, + testing::Combine( + testing::Values(szVGA, sz720p, sz1080p, sz2160p), + testing::Values(CV_32FC1, CV_32FC2, CV_32FC3, CV_32FC4) + ) +); + } // namespace diff --git a/modules/core/src/mathfuncs.cpp b/modules/core/src/mathfuncs.cpp index 9e3a1dbad2..525d71ba09 100644 --- a/modules/core/src/mathfuncs.cpp +++ b/modules/core/src/mathfuncs.cpp @@ -1610,30 +1610,37 @@ void patchNaNs( InputOutputArray _a, double _val ) const Mat* arrays[] = {&a, 0}; int* ptrs[1] = {}; NAryMatIterator it(arrays, (uchar**)ptrs); - size_t len = it.size*a.channels(); + int len = (int)(it.size*a.channels()); Cv32suf val; val.f = (float)_val; -#if (CV_SIMD || CV_SIMD_SCALABLE) - v_int32 v_mask1 = vx_setall_s32(0x7fffffff), v_mask2 = vx_setall_s32(0x7f800000); - v_int32 v_val = vx_setall_s32(val.i); -#endif - for( size_t i = 0; i < it.nplanes; i++, ++it ) { int* tptr = ptrs[0]; - size_t j = 0; + int j = 0; #if (CV_SIMD || CV_SIMD_SCALABLE) - size_t cWidth = (size_t)VTraits::vlanes(); - for ( ; j + cWidth <= len; j += cWidth) + v_int32 v_pos_mask = vx_setall_s32(0x7fffffff), v_exp_mask = vx_setall_s32(0x7f800000); + v_int32 v_val = vx_setall_s32(val.i); + + int cWidth = VTraits::vlanes(); + for (; j < len - cWidth * 2 + 1; j += cWidth * 2) { - v_int32 v_src = vx_load(tptr + j); - v_int32 v_cmp_mask = v_lt(v_mask2, v_and(v_src, v_mask1)); - v_int32 v_dst = v_select(v_cmp_mask, v_val, v_src); - v_store(tptr + j, v_dst); + v_int32 v_src0 = vx_load(tptr + j); + v_int32 v_src1 = vx_load(tptr + j + cWidth); + + v_int32 v_cmp_mask0 = v_lt(v_exp_mask, v_and(v_src0, v_pos_mask)); + v_int32 v_cmp_mask1 = v_lt(v_exp_mask, v_and(v_src1, v_pos_mask)); + + if (v_check_any(v_or(v_cmp_mask0, v_cmp_mask1))) + { + v_int32 v_dst0 = v_select(v_cmp_mask0, v_val, v_src0); + v_int32 v_dst1 = v_select(v_cmp_mask1, v_val, v_src1); + + v_store(tptr + j, v_dst0); + v_store(tptr + j + cWidth, v_dst1); + } } - vx_cleanup(); #endif for( ; j < len; j++ ) From fa56623458e4e582f16b216448e21bb9e664a9ec Mon Sep 17 00:00:00 2001 From: Dmitry Kurtaev Date: Fri, 3 Nov 2023 12:34:09 +0300 Subject: [PATCH 005/156] Merge pull request #24463 from dkurt:dnn_shared_nodes_fusion DNN graph fusion with shared nodes #24463 ### Pull Request Readiness Checklist For now, nodes from matched pattern are removed during the matching process so if nodes are used in similar subgraph, they cannot be found. required for https://github.com/opencv/opencv/pull/24397 **Merge with extra**: https://github.com/opencv/opencv_extra/pull/1115 A part from [model_name ](https://github.com/onnx/models/blob/main/vision/object_detection_segmentation/fcn/model/fcn-resnet101-11.onnx) with two Resize subgraphs with shared nodes: ![image](https://github.com/opencv/opencv/assets/25801568/611d89d9-12fb-4add-9218-13b10d2c086a) See details at https://github.com/opencv/opencv/wiki/How_to_contribute#making-a-good-pull-request - [x] I agree to contribute to the project under Apache 2 License. - [x] To the best of my knowledge, the proposed patch is not based on a code under GPL or another license that is incompatible with OpenCV - [x] The PR is proposed to the proper branch - [x] There is a reference to the original bug report and related work - [x] There is accuracy test, performance test and test data in opencv_extra repository, if applicable Patch to opencv_extra has the same branch name. - [x] The feature is well documented and sample code can be built with the project CMake --- modules/dnn/src/graph_simplifier.cpp | 50 +++++++++++++++++-- .../dnn/src/onnx/onnx_graph_simplifier.cpp | 28 +++++++++++ modules/dnn/test/test_onnx_importer.cpp | 10 +++- 3 files changed, 82 insertions(+), 6 deletions(-) diff --git a/modules/dnn/src/graph_simplifier.cpp b/modules/dnn/src/graph_simplifier.cpp index e58e0e38e8..e1b6d6df40 100644 --- a/modules/dnn/src/graph_simplifier.cpp +++ b/modules/dnn/src/graph_simplifier.cpp @@ -165,10 +165,7 @@ void Subgraph::replace(const Ptr& net, const std::vector node = net->getNode(matchedNodesIds.back()); - for (int i = matchedNodesIds.size() - 2; i >= 0; --i) - net->removeNode(matchedNodesIds[i]); // Modify the last node to be a fused one. node->setType(fusedNodeOp); @@ -191,6 +188,7 @@ void simplifySubgraphs(const Ptr& net, { int numNodes = net->getNumNodes(); std::vector matchedNodesIds, targetNodesIds; + std::vector nodesToRemove; for (int j = 0; j < patterns.size(); ++j) { for (int i = 0; i < numNodes; ++i) @@ -198,10 +196,54 @@ void simplifySubgraphs(const Ptr& net, if (patterns[j]->match(net, i, matchedNodesIds, targetNodesIds)) { patterns[j]->replace(net, matchedNodesIds, targetNodesIds); - numNodes -= matchedNodesIds.size() - 1; // #matchedNodes removed and one added. + // Remove matched nodes except the last one. + nodesToRemove.insert(nodesToRemove.end(), matchedNodesIds.begin(), matchedNodesIds.end() - 1); } } } + + if (nodesToRemove.empty()) + return; + + // Collect reference counts for every node + std::vector refcounts(net->getNumNodes(), 0); + std::map nodeIds; + + // Register node outputs. + // Every usage of one of the node's outputs should be counted. + for (int nodeId = 0; nodeId < refcounts.size(); ++nodeId) { + for (int i = 0; i < net->getNumOutputs(nodeId); ++i) { + std::string name = net->getOutputName(nodeId, i); + nodeIds[name] = nodeId; + } + } + + for (int nodeId = 0; nodeId < refcounts.size(); ++nodeId) { + // Increase counters for node's inputs + auto node = net->getNode(nodeId); + for (int i = 0; i < node->getNumInputs(); ++i) { + std::string inpName = node->getInputName(i); + if (inpName.empty()) + continue; + CV_Assert(nodeIds.find(inpName) != nodeIds.end()); + refcounts[nodeIds[inpName]] += 1; + } + } + + // Remove all fused nodes. Indices expected to be in descending order. + std::sort(nodesToRemove.begin(), nodesToRemove.end(), [](int a, int b) { return a > b; }); + for (int nodeId : nodesToRemove) { + if (refcounts[nodeId] == 0) { + // Decrease references to node's inputs and remove node itself + auto node = net->getNode(nodeId); + for (int i = 0; i < node->getNumInputs(); ++i) { + std::string inpName = node->getInputName(i); + refcounts[nodeIds[inpName]] -= 1; + } + net->removeNode(nodeId); + refcounts[nodeId] = -1; // Same node cannot be removed twice + } + } } }} // namespace cv::dnn diff --git a/modules/dnn/src/onnx/onnx_graph_simplifier.cpp b/modules/dnn/src/onnx/onnx_graph_simplifier.cpp index a43815dbe4..15f79c8769 100644 --- a/modules/dnn/src/onnx/onnx_graph_simplifier.cpp +++ b/modules/dnn/src/onnx/onnx_graph_simplifier.cpp @@ -1136,6 +1136,33 @@ public: } }; +class ResizeSubgraph3 : public Subgraph +{ +public: + ResizeSubgraph3() : Subgraph() + { + int shapeSrc = addNodeToMatch(""); + int input = addNodeToMatch(""); + + int shape_h = addNodeToMatch("Shape", shapeSrc); + int shape_w = addNodeToMatch("Shape", shapeSrc); + int gather_h = addNodeToMatch("Gather", shape_h, addNodeToMatch("Constant")); + int gather_w = addNodeToMatch("Gather", shape_w, addNodeToMatch("Constant")); + int unsqueeze_h = addNodeToMatch("Unsqueeze", gather_h); + int unsqueeze_w = addNodeToMatch("Unsqueeze", gather_w); + int concat1 = addNodeToMatch("Concat", unsqueeze_h, unsqueeze_w); + int cast = addNodeToMatch("Cast", concat1); + + int shape2 = addNodeToMatch("Shape", input); + int slice = addNodeToMatch("Slice", shape2, addNodeToMatch("Constant"), addNodeToMatch("Constant"), addNodeToMatch("Constant")); + int concat2 = addNodeToMatch("Concat", slice, cast); + addNodeToMatch("Resize", input, addNodeToMatch("Constant"), addNodeToMatch("Constant"), concat2); + + setFusedNode("Upsample", input, shapeSrc); + } +}; + + class BatchNormalizationSubgraphBase : public Subgraph { public: @@ -1207,6 +1234,7 @@ void simplifySubgraphs(opencv_onnx::GraphProto& net) subgraphs.push_back(makePtr()); subgraphs.push_back(makePtr()); subgraphs.push_back(makePtr()); + subgraphs.push_back(makePtr()); subgraphs.push_back(makePtr()); subgraphs.push_back(makePtr()); subgraphs.push_back(makePtr()); diff --git a/modules/dnn/test/test_onnx_importer.cpp b/modules/dnn/test/test_onnx_importer.cpp index b7e4e73cbc..cea4ffd739 100644 --- a/modules/dnn/test/test_onnx_importer.cpp +++ b/modules/dnn/test/test_onnx_importer.cpp @@ -54,7 +54,8 @@ public: void testONNXModels(const String& basename, const Extension ext = npy, double l1 = 0, double lInf = 0, const bool useSoftmax = false, - bool checkNoFallbacks = true, int numInps = 1) + bool checkNoFallbacks = true, int numInps = 1, + bool testShapes = true) { String onnxmodel = _tf("models/" + basename + ".onnx", required); std::vector inps(numInps); @@ -76,7 +77,8 @@ public: Net net = readNetFromONNX(onnxmodel); ASSERT_FALSE(net.empty()); - testInputShapes(net, inps); + if (testShapes) + testInputShapes(net, inps); net.setPreferableBackend(backend); net.setPreferableTarget(target); @@ -248,6 +250,10 @@ TEST_P(Test_ONNX_layers, Gather_shared_indices) { testONNXModels("gather_shared_indices", npy, 0, 0, false, false, 1); } +TEST_P(Test_ONNX_layers, Two_resizes_with_shared_subgraphs) { + testONNXModels("two_resizes_with_shared_subgraphs", npy, 0, 0, false, false, 3, /*testShapes*/ false); +} + TEST_P(Test_ONNX_layers, Convolution3D) { if (backend == DNN_BACKEND_CUDA && target == DNN_TARGET_CUDA_FP16) From 6a656785929e50c352855391ef17177206896067 Mon Sep 17 00:00:00 2001 From: Alexander Smorkalov Date: Fri, 3 Nov 2023 13:59:31 +0300 Subject: [PATCH 006/156] Use video stream fps first in FFmpeg backend for VideoCapture. --- modules/videoio/src/cap_ffmpeg_impl.hpp | 13 +++++++------ modules/videoio/test/test_ffmpeg.cpp | 2 +- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/modules/videoio/src/cap_ffmpeg_impl.hpp b/modules/videoio/src/cap_ffmpeg_impl.hpp index bf259b4daf..17d02df95a 100644 --- a/modules/videoio/src/cap_ffmpeg_impl.hpp +++ b/modules/videoio/src/cap_ffmpeg_impl.hpp @@ -1877,15 +1877,16 @@ int64_t CvCapture_FFMPEG::get_bitrate() const double CvCapture_FFMPEG::get_fps() const { -#if LIBAVFORMAT_BUILD >= CALC_FFMPEG_VERSION(55, 1, 100) && LIBAVFORMAT_VERSION_MICRO >= 100 - double fps = r2d(av_guess_frame_rate(ic, ic->streams[video_stream], NULL)); -#else +#if LIBAVCODEC_BUILD >= CALC_FFMPEG_VERSION(54, 1, 0) || LIBAVFORMAT_BUILD >= CALC_FFMPEG_VERSION(52, 111, 0) double fps = r2d(ic->streams[video_stream]->avg_frame_rate); +#else + double fps = r2d(ic->streams[video_stream]->r_frame_rate); +#endif -#if LIBAVFORMAT_BUILD >= CALC_FFMPEG_VERSION(52, 111, 0) +#if LIBAVFORMAT_BUILD >= CALC_FFMPEG_VERSION(55, 1, 100) && LIBAVFORMAT_VERSION_MICRO >= 100 if (fps < eps_zero) { - fps = r2d(ic->streams[video_stream]->avg_frame_rate); + fps = r2d(av_guess_frame_rate(ic, ic->streams[video_stream], NULL)); } #endif @@ -1893,7 +1894,7 @@ double CvCapture_FFMPEG::get_fps() const { fps = 1.0 / r2d(ic->streams[video_stream]->time_base); } -#endif + return fps; } diff --git a/modules/videoio/test/test_ffmpeg.cpp b/modules/videoio/test/test_ffmpeg.cpp index 9f59480c9c..338138518b 100644 --- a/modules/videoio/test/test_ffmpeg.cpp +++ b/modules/videoio/test/test_ffmpeg.cpp @@ -296,7 +296,7 @@ TEST_P(videoio_encapsulate, write) ASSERT_TRUE(capActualRaw.isOpened()); const double fpsReference = capReference.get(CAP_PROP_FPS); const double fpsActual = capActual.get(CAP_PROP_FPS); - ASSERT_EQ(fpsReference, fpsActual); + ASSERT_NEAR(fpsReference, fpsActual, 1e-2); const int nFramesActual = static_cast(capActual.get(CAP_PROP_FRAME_COUNT)); ASSERT_EQ(nFrames, nFramesActual); From e95c0055af7458fcf76561e74c629912df9baaa9 Mon Sep 17 00:00:00 2001 From: richard28039 <89371302+richard28039@users.noreply.github.com> Date: Fri, 3 Nov 2023 20:42:43 +0800 Subject: [PATCH 007/156] Merge pull request #24397 from richard28039:add_fcnresnet101_to_dnn_sample Added PyTorch fcnresnet101 segmentation conversion cases #24397 We write a sample code about transforming Pytorch fcnresnet101 to ONNX running on OpenCV. The input source image was shooted by ourself. ### Pull Request Readiness Checklist See details at https://github.com/opencv/opencv/wiki/How_to_contribute#making-a-good-pull-request - [X] I agree to contribute to the project under Apache 2 License. - [X] To the best of my knowledge, the proposed patch is not based on a code under GPL or another license that is incompatible with OpenCV - [X] The PR is proposed to the proper branch - [ ] There is a reference to the original bug report and related work - [ ] There is accuracy test, performance test and test data in opencv_extra repository, if applicable Patch to opencv_extra has the same branch name. - [ ] The feature is well documented and sample code can be built with the project CMake --- samples/dnn/models.yml | 12 ++++++++++++ samples/dnn/segmentation.py | 2 +- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/samples/dnn/models.yml b/samples/dnn/models.yml index e645279605..c780bcb139 100644 --- a/samples/dnn/models.yml +++ b/samples/dnn/models.yml @@ -178,3 +178,15 @@ fcn8s: height: 500 rgb: false sample: "segmentation" + +fcnresnet101: + load_info: + url: "https://github.com/onnx/models/raw/fb8271d5d5d9b90dbb1eb5e8e40f8f580fb248b3/vision/object_detection_segmentation/fcn/model/fcn-resnet101-11.onnx" + sha1: "e7e76474bf6b73334ab32c4be1374c9e605f5aed" + model: "fcn-resnet101-11.onnx" + mean: [103.5, 116.2, 123.6] + scale: 0.019 + width: 500 + height: 500 + rgb: false + sample: "segmentation" diff --git a/samples/dnn/segmentation.py b/samples/dnn/segmentation.py index 09f3f8dd11..8e4e435225 100644 --- a/samples/dnn/segmentation.py +++ b/samples/dnn/segmentation.py @@ -14,7 +14,7 @@ parser = argparse.ArgumentParser(add_help=False) parser.add_argument('--zoo', default=os.path.join(os.path.dirname(os.path.abspath(__file__)), 'models.yml'), help='An optional path to file with preprocessing parameters.') parser.add_argument('--input', help='Path to input image or video file. Skip this argument to capture frames from a camera.') -parser.add_argument('--framework', choices=['caffe', 'tensorflow', 'torch', 'darknet'], +parser.add_argument('--framework', choices=['caffe', 'tensorflow', 'torch', 'darknet', 'onnx'], help='Optional name of an origin framework of the model. ' 'Detect it automatically if it does not set.') parser.add_argument('--colors', help='Optional path to a text file with colors for an every class. ' From ed52f7feeaae6f0e96ee4dc36ae5f9fb4a6d959c Mon Sep 17 00:00:00 2001 From: Wanli Date: Mon, 6 Nov 2023 09:48:32 +0800 Subject: [PATCH 008/156] Improve and refactor softmax layer (#24466) * improve and refactor softmax layer * fix building error * compatible region layer * fix axisStep when disable SIMD * fix dynamic array * try to fix error * use nlanes from VTraits * move axisBias to srcOffset * fix bug caused by axisBias * remove macro * replace #ifdef with #if for CV_SIMD --- modules/dnn/perf/perf_layer.cpp | 51 ++++++ .../dnn/src/layers/cpu_kernels/softmax.cpp | 157 ++++++++++++++++++ .../dnn/src/layers/cpu_kernels/softmax.hpp | 28 ++++ modules/dnn/src/layers/region_layer.cpp | 7 +- modules/dnn/src/layers/softmax_layer.cpp | 83 +-------- modules/dnn/src/onnx/onnx_importer.cpp | 7 + 6 files changed, 251 insertions(+), 82 deletions(-) create mode 100644 modules/dnn/src/layers/cpu_kernels/softmax.cpp create mode 100644 modules/dnn/src/layers/cpu_kernels/softmax.hpp diff --git a/modules/dnn/perf/perf_layer.cpp b/modules/dnn/perf/perf_layer.cpp index dbedc4319b..e2d7cf2ff4 100644 --- a/modules/dnn/perf/perf_layer.cpp +++ b/modules/dnn/perf/perf_layer.cpp @@ -758,4 +758,55 @@ INSTANTIATE_TEST_CASE_P(/**/, Layer_FullyConnected, Combine( dnnBackendsAndTargets() )); +typedef TestBaseWithParam, int, tuple > > Layer_Softmax; +PERF_TEST_P_(Layer_Softmax, softmax_3d) { + std::vector shape = get<0>(GetParam()); + int axis = get<1>(GetParam()); + int backendId = get<0>(get<2>(GetParam())); + int targetId = get<1>(get<2>(GetParam())); + + Mat data(shape, CV_32FC1); + Scalar mean = 0.f; + Scalar std = 1.f; + randn(data, mean, std); + + Net net; + LayerParams lp; + lp.type = "Softmax"; + lp.name = "testLayer"; + lp.set("axis", axis); + + net.addLayerToPrev(lp.name, lp.type, lp); + // warmup + { + net.setInput(data); + net.setPreferableBackend(backendId); + net.setPreferableTarget(targetId); + Mat out = net.forward(); + } + + TEST_CYCLE() { + Mat res = net.forward(); + } + + SANITY_CHECK_NOTHING(); +} + +INSTANTIATE_TEST_CASE_P(/**/, Layer_Softmax, Combine( + Values( // input size + std::vector({16, 50, 50}), + std::vector({16, 197, 197}), + std::vector({16, 1024, 1024}) + ), + Values(0, 1, 2), // axis + dnnBackendsAndTargets(/* withInferenceEngine= */ false, + /* withHalide= */ false, + /* withCpuOCV= */ true, + /* withVkCom= */ false, + /* withCUDA= */ false, + /* withNgraph= */ false, + /* withWebnn= */ false, + /* withCann= */ false) // only test on CPU +)); + } // namespace diff --git a/modules/dnn/src/layers/cpu_kernels/softmax.cpp b/modules/dnn/src/layers/cpu_kernels/softmax.cpp new file mode 100644 index 0000000000..15e50f17bd --- /dev/null +++ b/modules/dnn/src/layers/cpu_kernels/softmax.cpp @@ -0,0 +1,157 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. + +// This file is modified from the ficus (https://github.com/vpisarev/ficus/blob/master/lib/NN/OpNN.fx). +// Here is the original license: +/* + This file is a part of ficus language project. + See ficus/LICENSE for the licensing terms +*/ + +#include "../../precomp.hpp" +#include "softmax.hpp" + +namespace cv { namespace dnn { + +void softmax(Mat &dst, const Mat &src, int axis, int axisBias, int axisStep){ + CV_Assert(src.type() == CV_32F); + CV_Assert(src.isContinuous() && dst.isContinuous()); + CV_Assert(src.size == dst.size); + axis = normalize_axis(axis, src.dims); + + size_t outerSize = src.total(0, axis), + innerSize = src.total(axis + 1); + + const float *srcPtr = src.ptr(); + float *dstPtr = dst.ptr(); + + size_t outerStep = src.total(axis); + size_t cnStep = src.total(axis + 1); + + // multi-threads + size_t totalTasks = outerSize * innerSize; + double nstripes = (double) totalTasks / 1024.0; + // make the channel axis to be multiple of 8 + size_t channelAxis = (axisStep + 7) & -8; + +#if CV_SIMD + const int nlanes = VTraits::vlanes(); + // the number of redundant dimension + size_t redundantDim = nlanes - axisStep % nlanes; +#endif + + parallel_for_(Range(0, (int) totalTasks), [&](const Range &range) { + AutoBuffer axisBuf_(channelAxis); + float *axisBuf = axisBuf_.data(); + + for (size_t i = range.start; i < range.end; i++) { + size_t outerDim = i / innerSize; + size_t innerDim = i % innerSize; + size_t srcOffset = outerDim * outerStep + innerDim; + // copy data from src to buf along axis, since the data may not be continuous + for (size_t cnDim = 0; cnDim < axisStep; cnDim++) + axisBuf[cnDim] = srcPtr[srcOffset + (cnDim + axisBias) * cnStep]; + + float s = 0.f; +#if CV_SIMD + // make the value of the redundant dimension to be -FLT_MAX + if (redundantDim != nlanes) { + for (size_t j = axisStep; j < axisStep + redundantDim; j++) + axisBuf[j] = -FLT_MAX; + } + // calculate the max value along the axis + v_float32 vmax = vx_load(axisBuf); + for (size_t cnDim = nlanes; cnDim < axisStep; cnDim += nlanes) { + v_float32 val = vx_load(axisBuf + cnDim); + vmax = v_max(vmax, val); + } + float maxVal = v_reduce_max(vmax); + + // calculate the exp value along the axis + v_float32 vs = vx_setzero_f32(); + vmax = vx_setall_f32(maxVal); + // initialize vexp constant + v_float32 _vexp_lo = vx_setall_f32(-88.3762626647949f); + v_float32 _vexp_hi = vx_setall_f32(88.3762626647949f); + v_float32 _vexp_half = vx_setall_f32(0.5f); + v_float32 _vexp_one = vx_setall_f32(1.f); + v_float32 _vexp_LOG2EF = vx_setall_f32(1.44269504088896341f); + v_float32 _vexp_C1 = vx_setall_f32(-0.693359375f); + v_float32 _vexp_C2 = vx_setall_f32(2.12194440e-4f); + v_float32 _vexp_p0 = vx_setall_f32(1.9875691500E-4f); + v_float32 _vexp_p1 = vx_setall_f32(1.3981999507E-3f); + v_float32 _vexp_p2 = vx_setall_f32(8.3334519073E-3f); + v_float32 _vexp_p3 = vx_setall_f32(4.1665795894E-2f); + v_float32 _vexp_p4 = vx_setall_f32(1.6666665459E-1f); + v_float32 _vexp_p5 = vx_setall_f32(5.0000001201E-1f); + // initialize temp vectors for vexp + v_float32 val, _vexp_, _vexp_x, _vexp_y, _vexp_z; + v_int32 _vexp_mm; + + // calculate and sum all data along axis + for (size_t cnDim = 0; cnDim < axisStep; cnDim += nlanes) { + val = vx_load(axisBuf + cnDim); + val = v_sub(val, vmax); + + // compute vexp of val + _vexp_x = v_min(val, _vexp_hi); + _vexp_x = v_max(_vexp_x, _vexp_lo); + _vexp_ = v_fma(_vexp_x, _vexp_LOG2EF, _vexp_half); + _vexp_mm = v_floor(_vexp_); + _vexp_ = v_cvt_f32(_vexp_mm); + _vexp_mm = v_add(_vexp_mm, vx_setall_s32(0x7f)); + _vexp_mm = v_shl(_vexp_mm, 23); + _vexp_x = v_fma(_vexp_, _vexp_C1, _vexp_x); + _vexp_x = v_fma(_vexp_, _vexp_C2, _vexp_x); + _vexp_z = v_mul(_vexp_x, _vexp_x); + _vexp_y = v_fma(_vexp_x, _vexp_p0, _vexp_p1); + _vexp_y = v_fma(_vexp_y, _vexp_x, _vexp_p2); + _vexp_y = v_fma(_vexp_y, _vexp_x, _vexp_p3); + _vexp_y = v_fma(_vexp_y, _vexp_x, _vexp_p4); + _vexp_y = v_fma(_vexp_y, _vexp_x, _vexp_p5); + _vexp_y = v_fma(_vexp_y, _vexp_z, _vexp_x); + _vexp_y = v_add(_vexp_y, _vexp_one); + val = v_mul(_vexp_y, v_reinterpret_as_f32(_vexp_mm)); + + vs = v_add(vs, val); + v_store(axisBuf + cnDim, val); + } + + s = v_reduce_sum(vs); + // subtract the value of the redundant dimension + if (redundantDim != nlanes) { + float* _val = new float[nlanes]; + v_store(_val, val); + for (size_t j = nlanes - redundantDim; j < nlanes; j++) + s -= _val[j]; + } +#else + float maxVal = axisBuf[0]; + for (size_t cnDim = 1; cnDim < axisStep; cnDim++) { + maxVal = std::max(maxVal, axisBuf[cnDim]); + } + for (size_t j = 0; j < axisStep; j++) { + axisBuf[j] = expf(axisBuf[j] - maxVal); + s += axisBuf[j]; + } +#endif + s = 1.f / s; + + // copy back the result to src + for (size_t cnDim = 0; cnDim < axisStep; cnDim++) + dstPtr[srcOffset + (cnDim + axisBias) * cnStep] = axisBuf[cnDim] * s; + } + }, nstripes); +} + +void softmax(Mat &dst, const Mat &src, int axis) { + softmax(dst, src, axis, 0, src.size[axis]); +} + +void logSoftmax(Mat &dst, const Mat &src, int axis) { + softmax(dst, src, axis); + log(dst, dst); +} + +}} // cv::dnn diff --git a/modules/dnn/src/layers/cpu_kernels/softmax.hpp b/modules/dnn/src/layers/cpu_kernels/softmax.hpp new file mode 100644 index 0000000000..19a89fa878 --- /dev/null +++ b/modules/dnn/src/layers/cpu_kernels/softmax.hpp @@ -0,0 +1,28 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. + +// This file is modified from the ficus (https://github.com/vpisarev/ficus/blob/master/lib/NN/OpNN.fx). +// Here is the original license: +/* + This file is a part of ficus language project. + See ficus/LICENSE for the licensing terms +*/ + +#ifndef OPENCV_DNN_SOFTMAX_HPP +#define OPENCV_DNN_SOFTMAX_HPP + +#include "opencv2/core/hal/intrin.hpp" +#include + +namespace cv { namespace dnn { + +void softmax(Mat &dst, const Mat &src, int axis, int axisBias, int axisStep); + +void softmax(Mat &dst, const Mat &src, int axis); + +void logSoftmax(Mat &dst, const Mat &src, int axis); + +}} // cv::dnn + +#endif // OPENCV_DNN_SOFTMAX_HPP diff --git a/modules/dnn/src/layers/region_layer.cpp b/modules/dnn/src/layers/region_layer.cpp index 49952b4c83..38b809e0f9 100644 --- a/modules/dnn/src/layers/region_layer.cpp +++ b/modules/dnn/src/layers/region_layer.cpp @@ -45,6 +45,7 @@ #include #include #include "../nms.inl.hpp" +#include "cpu_kernels/softmax.hpp" #ifdef HAVE_OPENCL #include "opencl_kernels_dnn.hpp" @@ -280,10 +281,8 @@ public: } if (useSoftmax) { // Yolo v2 - for (int i = 0; i < batch_size*rows*cols*anchors; ++i) { - int index = cell_size*i; - softmax_activate(srcData + index + 5, classes, 1, dstData + index + 5); - } + Mat _inpBlob = inpBlob.reshape(0, outBlob.dims, outBlob.size); + softmax(outBlob, _inpBlob, -1, 5, classes); } else if (useLogistic) { // Yolo v3 for (int i = 0; i < batch_size*rows*cols*anchors; ++i){ diff --git a/modules/dnn/src/layers/softmax_layer.cpp b/modules/dnn/src/layers/softmax_layer.cpp index 7d83786fe1..5026b616f6 100644 --- a/modules/dnn/src/layers/softmax_layer.cpp +++ b/modules/dnn/src/layers/softmax_layer.cpp @@ -52,6 +52,7 @@ #include #include #include +#include "cpu_kernels/softmax.hpp" using std::max; #ifdef HAVE_OPENCL @@ -225,89 +226,15 @@ public: std::vector inputs, outputs, internals; inputs_arr.getMatVector(inputs); outputs_arr.getMatVector(outputs); - internals_arr.getMatVector(internals); const Mat &src = inputs[0]; Mat &dst = outputs[0]; - int axis = normalize_axis(axisRaw, src.dims); - size_t outerSize = src.total(0, axis), channels = src.size[axis], - innerSize = src.total(axis + 1); - CV_Assert(src.type() == CV_32F); - CV_Assert(src.isContinuous() && dst.isContinuous()); - - const float *srcPtr = src.ptr(); - float *dstPtr = dst.ptr(); - float *bufPtr = internals[0].ptr(); - - size_t outerStep = src.total(axis); - size_t cnStep = src.total(axis + 1); - - //compute max along axis - for (size_t outerDim = 0; outerDim < outerSize; outerDim++) - { - size_t srcOffset = outerDim * outerStep; - size_t bufOffset = outerDim * cnStep; - - memcpy(bufPtr + bufOffset, srcPtr + srcOffset, innerSize * sizeof(float)); - - for (size_t cnDim = 1; cnDim < channels; cnDim++) - { - for (size_t i = 0; i < innerSize; i++) - bufPtr[bufOffset + i] = std::max(bufPtr[bufOffset + i], srcPtr[srcOffset + cnDim * cnStep + i]); - } - } - - //subtract max - for (size_t outerDim = 0; outerDim < outerSize; outerDim++) - { - size_t srcOffset = outerDim * outerStep; - size_t bufOffset = outerDim * cnStep; - - for (size_t cnDim = 0; cnDim < channels; cnDim++) - { - const int offset = srcOffset + cnDim * cnStep; - for (size_t i = 0; i < innerSize; i++) - dstPtr[offset + i] = srcPtr[offset + i] - bufPtr[bufOffset + i]; - } - } - - cv::exp(dst, dst); - - for (size_t outerDim = 0; outerDim < outerSize; outerDim++) - { - size_t srcOffset = outerDim * outerStep; - size_t bufOffset = outerDim * cnStep; - - //sum exp along axis - for (size_t i = 0; i < innerSize; i++) - bufPtr[bufOffset + i] = 0.f; - - for (size_t cnDim = 0; cnDim < channels; cnDim++) - { - const int offset = srcOffset + cnDim * cnStep; - for (size_t i = 0; i < innerSize; i++) - bufPtr[bufOffset + i] += dstPtr[offset + i]; - } - - //divide by computed sum - for (size_t cnDim = 0; cnDim < channels; cnDim++) - { - const int offset = srcOffset + cnDim * cnStep; - for (size_t i = 0; i < innerSize; i++) - dstPtr[offset + i] /= bufPtr[bufOffset + i]; - } - if (logSoftMax) - { - for (size_t cnDim = 0; cnDim < channels; cnDim++) - { - const int offset = srcOffset + cnDim * cnStep; - for (size_t i = 0; i < innerSize; i++) - dstPtr[offset + i] = log(dstPtr[offset + i]); - } - } - } + if(logSoftMax) + logSoftmax(dst, src, axis); + else + softmax(dst, src, axis); } #ifdef HAVE_CUDA diff --git a/modules/dnn/src/onnx/onnx_importer.cpp b/modules/dnn/src/onnx/onnx_importer.cpp index 294bd259fe..4813d118c4 100644 --- a/modules/dnn/src/onnx/onnx_importer.cpp +++ b/modules/dnn/src/onnx/onnx_importer.cpp @@ -2788,6 +2788,13 @@ void ONNXImporter::parseUpsample(LayerParams& layerParams, const opencv_onnx::No void ONNXImporter::parseSoftMax(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) { const std::string& layer_type = node_proto.op_type(); + int axis; + if (layerParams.has("opset") && layerParams.get("opset") > 11) { + axis = layerParams.get("axis", -1); + } else { + axis = layerParams.get("axis", 1); + } + layerParams.set("axis", axis); layerParams.type = "Softmax"; layerParams.set("log_softmax", layer_type == "LogSoftmax"); addLayer(layerParams, node_proto); From 832f738db00cc6f1541c022692bd75e1e7853012 Mon Sep 17 00:00:00 2001 From: Vincent Rabaud Date: Tue, 7 Nov 2023 07:06:28 +0100 Subject: [PATCH 009/156] Merge pull request #24495 from vrabaud:fast_math_compile Get the SSE2 condition match the emmintrin.h inclusion condition. #24495 ### Pull Request Readiness Checklist See details at https://github.com/opencv/opencv/wiki/How_to_contribute#making-a-good-pull-request - [x] I agree to contribute to the project under Apache 2 License. - [x] To the best of my knowledge, the proposed patch is not based on a code under GPL or another license that is incompatible with OpenCV - [x] The PR is proposed to the proper branch - [x] There is a reference to the original bug report and related work - [x] There is accuracy test, performance test and test data in opencv_extra repository, if applicable Patch to opencv_extra has the same branch name. - [x] The feature is well documented and sample code can be built with the project CMake --- modules/core/include/opencv2/core/fast_math.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/core/include/opencv2/core/fast_math.hpp b/modules/core/include/opencv2/core/fast_math.hpp index 08401afbb8..ff9ee46af6 100644 --- a/modules/core/include/opencv2/core/fast_math.hpp +++ b/modules/core/include/opencv2/core/fast_math.hpp @@ -68,7 +68,7 @@ // nothing, intrinsics/asm code is not supported #else #if ((defined _MSC_VER && defined _M_X64) \ - || (defined __GNUC__ && defined __x86_64__ && defined __SSE2__)) \ + || (defined __GNUC__ && defined __SSE2__)) \ && !defined(OPENCV_SKIP_INCLUDE_EMMINTRIN_H) #include #endif From ee0822dc4d1794b57d61b2f148a7046bc511e75b Mon Sep 17 00:00:00 2001 From: Yuantao Feng Date: Tue, 7 Nov 2023 17:59:10 +0800 Subject: [PATCH 010/156] Merge pull request #24378 from fengyuentau:instance_norm dnn onnx: add instance norm layer #24378 Resolves https://github.com/opencv/opencv/issues/24377 Relates https://github.com/opencv/opencv/pull/24092#discussion_r1349841644 | Perf | multi-thread | single-thread | | - | - | - | | x: [2, 64, 180, 240] | 3.95ms | 11.12ms | Todo: - [x] speed up by multi-threading - [x] add perf - [x] add backend: OpenVINO - [x] add backend: CUDA - [x] add backend: OpenCL (no fp16) - [ ] add backend: CANN (will be done via https://github.com/opencv/opencv/pull/24462) ### Pull Request Readiness Checklist See details at https://github.com/opencv/opencv/wiki/How_to_contribute#making-a-good-pull-request - [x] I agree to contribute to the project under Apache 2 License. - [x] To the best of my knowledge, the proposed patch is not based on a code under GPL or another license that is incompatible with OpenCV - [x] The PR is proposed to the proper branch - [x] There is a reference to the original bug report and related work - [x] There is accuracy test, performance test and test data in opencv_extra repository, if applicable Patch to opencv_extra has the same branch name. - [x] The feature is well documented and sample code can be built with the project CMake ``` force_builders=Linux OpenCL,Win64 OpenCL,Custom buildworker:Custom=linux-4 build_image:Custom=ubuntu:18.04 modules_filter:Custom=none disable_ipp:Custom=ON ``` --- .../dnn/include/opencv2/dnn/all_layers.hpp | 7 + modules/dnn/perf/perf_layer.cpp | 57 +++++ modules/dnn/src/cuda/mvn.cu | 28 +++ modules/dnn/src/cuda4dnn/kernels/mvn.hpp | 3 + .../src/cuda4dnn/primitives/instance_norm.hpp | 86 +++++++ modules/dnn/src/init.cpp | 1 + .../dnn/src/layers/cpu_kernels/fast_norm.cpp | 7 +- .../dnn/src/layers/instance_norm_layer.cpp | 231 ++++++++++++++++++ modules/dnn/src/onnx/onnx_importer.cpp | 65 +++-- ..._conformance_layer_parser_denylist.inl.hpp | 2 - 10 files changed, 449 insertions(+), 38 deletions(-) create mode 100644 modules/dnn/src/cuda4dnn/primitives/instance_norm.hpp create mode 100644 modules/dnn/src/layers/instance_norm_layer.cpp diff --git a/modules/dnn/include/opencv2/dnn/all_layers.hpp b/modules/dnn/include/opencv2/dnn/all_layers.hpp index 7f8e919257..8154064e03 100644 --- a/modules/dnn/include/opencv2/dnn/all_layers.hpp +++ b/modules/dnn/include/opencv2/dnn/all_layers.hpp @@ -1166,6 +1166,13 @@ CV__DNN_INLINE_NS_BEGIN static Ptr create(const LayerParams ¶ms); }; + class CV_EXPORTS InstanceNormLayer : public Layer { + public: + float epsilon; + + static Ptr create(const LayerParams ¶ms); + }; + //! @} //! @} CV__DNN_INLINE_NS_END diff --git a/modules/dnn/perf/perf_layer.cpp b/modules/dnn/perf/perf_layer.cpp index e2d7cf2ff4..c26b7a1588 100644 --- a/modules/dnn/perf/perf_layer.cpp +++ b/modules/dnn/perf/perf_layer.cpp @@ -683,6 +683,62 @@ PERF_TEST_P_(Layer_GatherElements, GatherElements) test_layer({2700, 1, 2914}, {2700, 1, 81}, 2); } +struct Layer_InstanceNorm : public TestBaseWithParam > +{ + void test_layer(const std::vector& x_shape) + { + int backendId = get<0>(GetParam()); + int targetId = get<1>(GetParam()); + + Mat x(x_shape, CV_32FC1); + Mat scale(x_shape[1], 1, CV_32FC1); + Mat b(x_shape[1], 1, CV_32FC1); + + randu(x, 0.f, 1.f); + randu(scale, 0.f, 1.f); + randu(b, 0.f, 1.f); + + Net net; + LayerParams lp; + lp.type = "InstanceNormalization"; + lp.name = "testLayer"; + int id = net.addLayerToPrev(lp.name, lp.type, lp); + net.connect(0, 0, id, 0); + net.connect(0, 1, id, 1); + net.connect(0, 2, id, 2); + + // warmup + { + std::vector inpNames{"x", "scale", "b"}; + net.setInputsNames(inpNames); + net.setInput(x, inpNames[0]); + net.setInput(scale, inpNames[1]); + net.setInput(b, inpNames[2]); + + net.setPreferableBackend(backendId); + net.setPreferableTarget(targetId); + Mat out = net.forward(); + } + + TEST_CYCLE() + { + Mat res = net.forward(); + } + + SANITY_CHECK_NOTHING(); + } + + int N = 2; + int C = 64; + int H = 180; + int W = 240; +}; + +PERF_TEST_P_(Layer_InstanceNorm, InstanceNorm) +{ + test_layer({N, C, H, W}); +} + INSTANTIATE_TEST_CASE_P(/**/, Layer_Slice, dnnBackendsAndTargets(false, false)); INSTANTIATE_TEST_CASE_P(/**/, Layer_NaryEltwise, testing::Values(std::make_tuple(DNN_BACKEND_OPENCV, DNN_TARGET_CPU))); #ifdef HAVE_CUDA @@ -693,6 +749,7 @@ INSTANTIATE_TEST_CASE_P(/**/, Layer_ScatterND, testing::Values(std::make_tuple(D INSTANTIATE_TEST_CASE_P(/**/, Layer_LayerNorm, testing::Values(std::make_tuple(DNN_BACKEND_OPENCV, DNN_TARGET_CPU))); INSTANTIATE_TEST_CASE_P(/**/, Layer_LayerNormExpanded, testing::Values(std::make_tuple(DNN_BACKEND_OPENCV, DNN_TARGET_CPU))); INSTANTIATE_TEST_CASE_P(/**/, Layer_GatherElements, testing::Values(std::make_tuple(DNN_BACKEND_OPENCV, DNN_TARGET_CPU))); +INSTANTIATE_TEST_CASE_P(/**/, Layer_InstanceNorm, testing::Values(std::make_tuple(DNN_BACKEND_OPENCV, DNN_TARGET_CPU))); typedef TestBaseWithParam > > Layer_FullyConnected; diff --git a/modules/dnn/src/cuda/mvn.cu b/modules/dnn/src/cuda/mvn.cu index adf997c0b0..d4f0733676 100644 --- a/modules/dnn/src/cuda/mvn.cu +++ b/modules/dnn/src/cuda/mvn.cu @@ -66,6 +66,17 @@ namespace raw { output[idx] = (static_cast(input[idx]) - means[outer_idx]) * scale[outer_idx]; } } + + template + __global__ void normalize_mean_variance_channelwise(Span output, View input, View scale, View bias, View means, View stdev, size_type inner_size, size_type C) { + for (auto idx : grid_stride_range(output.size())) { + const index_type outer_idx = idx / inner_size; + const index_type c = outer_idx % C; + auto s = static_cast(scale[c]) * stdev[outer_idx]; + auto b = static_cast(bias[c]); + output[idx] = (static_cast(input[idx]) - means[outer_idx]) * s + b; + } + } } template @@ -142,4 +153,21 @@ template void normalize_mean_variance(const Stream&, Span<__half>, View<__half>, #endif template void normalize_mean_variance(const Stream&, Span, View, View, View, std::size_t); +template +void normalize_mean_variance_channelwise(const Stream& stream, Span output, View input, View scale, View bias, View means, View stdev, std::size_t inner_size, std::size_t C) +{ + CV_Assert(input.size() == output.size()); + CV_Assert(input.size() / inner_size == means.size()); + CV_Assert(means.size() == stdev.size()); + + auto kernel = raw::normalize_mean_variance_channelwise; + auto policy = make_policy(kernel, output.size(), 0, stream); + launch_kernel(kernel, policy, output, input, scale, bias, means, stdev, inner_size, C); +} + +#if !defined(__CUDA_ARCH__) || (__CUDA_ARCH__ >= 530) +template void normalize_mean_variance_channelwise(const Stream&, Span<__half> /*output*/, View<__half> /*input*/, View<__half> /*scale*/, View<__half> /*bias*/, View /*means*/, View /*stdev*/, std::size_t, std::size_t); +#endif +template void normalize_mean_variance_channelwise(const Stream&, Span /*output*/, View /*input*/, View /*scale*/, View /*bias*/, View /*means*/, View /*stdev*/, std::size_t, std::size_t); + }}}} /* namespace cv::dnn::cuda4dnn::kernels */ diff --git a/modules/dnn/src/cuda4dnn/kernels/mvn.hpp b/modules/dnn/src/cuda4dnn/kernels/mvn.hpp index b5a573e921..ebd7b9f659 100644 --- a/modules/dnn/src/cuda4dnn/kernels/mvn.hpp +++ b/modules/dnn/src/cuda4dnn/kernels/mvn.hpp @@ -26,6 +26,9 @@ void normalize_mean(const csl::Stream& stream, csl::Span output, csl::View template void normalize_mean_variance(const csl::Stream& stream, csl::Span output, csl::View input, csl::View means, csl::View scale, std::size_t inner_size); +template +void normalize_mean_variance_channelwise(const csl::Stream &stream, csl::Span output, csl::View input, csl::View scale, csl::View bias, csl::View means, csl::View stdev, std::size_t inner_size, std::size_t C); + }}}} /* namespace cv::dnn::cuda4dnn::kernels */ #endif /* OPENCV_DNN_SRC_CUDA4DNN_KERNELS_MVN_HPP */ diff --git a/modules/dnn/src/cuda4dnn/primitives/instance_norm.hpp b/modules/dnn/src/cuda4dnn/primitives/instance_norm.hpp new file mode 100644 index 0000000000..0a32e40fc0 --- /dev/null +++ b/modules/dnn/src/cuda4dnn/primitives/instance_norm.hpp @@ -0,0 +1,86 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. + +#ifndef OPENCV_DNN_SRC_CUDA4DNN_PRIMITIVES_INSTANCE_NORM_HPP +#define OPENCV_DNN_SRC_CUDA4DNN_PRIMITIVES_INSTANCE_NORM_HPP + +#include "../../op_cuda.hpp" + +#include "../csl/stream.hpp" +#include "../csl/span.hpp" +#include "../csl/tensor.hpp" +#include "../csl/workspace.hpp" + +#include "../kernels/fill_copy.hpp" +#include "../kernels/mvn.hpp" + +#include + +#include +#include +#include + +namespace cv { namespace dnn { namespace cuda4dnn { + + template + class InstanceNormOp final : public CUDABackendNode { + public: + using wrapper_type = GetCUDABackendWrapperType; + + InstanceNormOp(csl::Stream stream_, float epsilon_, size_t loops) + : stream(std::move(stream_)), epsilon(epsilon_) { + csl::WorkspaceBuilder builder; + builder.require(loops); + builder.require(loops); + scratch_mem_in_bytes = builder.required_workspace_size(); + } + + void forward(const std::vector>& inputs, + const std::vector>& outputs, + csl::Workspace& workspace) override { + auto input_wrapper = inputs[0].dynamicCast(); + auto scale_wrapper = inputs[1].dynamicCast(); + auto bias_wrapper = inputs[2].dynamicCast(); + + auto input = input_wrapper->getView(); + auto scale = scale_wrapper->getView(); + auto bias = bias_wrapper->getView(); + + auto output_wrapper = outputs[0].dynamicCast(); + auto output = output_wrapper->getSpan(); + + auto C = input.get_axis_size(1); + auto loops = input.size_range(0, 2); + auto norm_size = input.size_range(2, input.rank()); + if (norm_size == 1) { + kernels::fill(stream, output, 0.f); + return; + } else { + auto ws_allocator = csl::WorkspaceAllocator(workspace); + + auto mean = ws_allocator.get_span(loops); + kernels::fill(stream, mean, 0.f); + + auto stdev = ws_allocator.get_span(loops); + kernels::fill(stream, stdev, 0.f); + + kernels::reduce_mean_sqr_sum(stream, mean, stdev, input, norm_size); + kernels::compute_normalization_scale(stream, stdev, mean, stdev, norm_size, epsilon); + kernels::normalize_mean_variance_channelwise(stream, output, input, scale, bias, mean, stdev, norm_size, C); + } + } + + std::size_t get_workspace_memory_in_bytes() const noexcept override { return scratch_mem_in_bytes; } + + private: + csl::Stream stream; + + float epsilon; + + std::size_t scratch_mem_in_bytes; + }; + +}}} // cv::dnn::cuda4dnn + +#endif // OPENCV_DNN_SRC_CUDA4DNN_PRIMITIVES_INSTANCE_NORM_HPP diff --git a/modules/dnn/src/init.cpp b/modules/dnn/src/init.cpp index e70d5dad47..961e6e5c9a 100644 --- a/modules/dnn/src/init.cpp +++ b/modules/dnn/src/init.cpp @@ -160,6 +160,7 @@ void initializeLayerFactory() CV_DNN_REGISTER_LAYER_CLASS(GatherElements, GatherElementsLayer); CV_DNN_REGISTER_LAYER_CLASS(LayerNormalization, LayerNormLayer); CV_DNN_REGISTER_LAYER_CLASS(Expand, ExpandLayer); + CV_DNN_REGISTER_LAYER_CLASS(InstanceNormalization, InstanceNormLayer); CV_DNN_REGISTER_LAYER_CLASS(Crop, CropLayer); CV_DNN_REGISTER_LAYER_CLASS(Eltwise, EltwiseLayer); diff --git a/modules/dnn/src/layers/cpu_kernels/fast_norm.cpp b/modules/dnn/src/layers/cpu_kernels/fast_norm.cpp index 60b503513f..ab9d8ee0af 100644 --- a/modules/dnn/src/layers/cpu_kernels/fast_norm.cpp +++ b/modules/dnn/src/layers/cpu_kernels/fast_norm.cpp @@ -118,10 +118,11 @@ void fastNorm(const Mat &input, const Mat &scale, const Mat &bias, Mat &output, void fastNormChannel(const Mat &input, const Mat &scale, const Mat &bias, Mat &output, float epsilon) { const auto input_shape = shape(input); + size_t N = input_shape[0], C = input_shape[1]; CV_CheckEQ(scale.total(), bias.total(), "fastNormChannel: scale and bias should have the same shape"); + CV_CheckEQ(scale.total(), C, "fastNormChannel: scale should be a 1d tensor and match the channel of input"); CV_CheckGE(input.dims, 3, "fastNormChannel: input dimension >= 3"); - size_t N = input_shape[0], C = input_shape[1]; size_t loops = N * C, norm_size = static_cast(total(input_shape, 2)); float inv_norm_size = 1.0 / norm_size; @@ -147,9 +148,9 @@ void fastNormChannel(const Mat &input, const Mat &scale, const Mat &bias, Mat &o float inv_stdev = 1.f / mean_square; size_t c = i % C; - float s = scale_data[c], b = bias_data[c]; + float s = scale_data[c] * inv_stdev, b = bias_data[c]; for (size_t j = 0; j < norm_size; j++) { - y[j] = s * (x[j] - mean) * inv_stdev + b; + y[j] = s * (x[j] - mean) + b; } } }; diff --git a/modules/dnn/src/layers/instance_norm_layer.cpp b/modules/dnn/src/layers/instance_norm_layer.cpp new file mode 100644 index 0000000000..fda0efdb94 --- /dev/null +++ b/modules/dnn/src/layers/instance_norm_layer.cpp @@ -0,0 +1,231 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. + +#include "../precomp.hpp" +#include +#include "./cpu_kernels/fast_norm.hpp" + +// OpenVINO backend +#include "../op_inf_engine.hpp" +#include "../ie_ngraph.hpp" + +// CUDA backend +#include "../op_cuda.hpp" +#ifdef HAVE_CUDA +#include "../cuda4dnn/primitives/instance_norm.hpp" +using namespace cv::dnn::cuda4dnn; +#endif + +// OpenCL backend +#ifdef HAVE_OPENCL +#include "../ocl4dnn/include/math_functions.hpp" +#include "opencl_kernels_dnn.hpp" +#endif + +namespace cv { namespace dnn { + +// https://github.com/onnx/onnx/blob/main/docs/Operators.md#InstanceNormalization +class InstanceNormLayerImpl CV_FINAL : public InstanceNormLayer { +public: + InstanceNormLayerImpl(const LayerParams ¶ms) { + setParamsFrom(params); + + epsilon = params.get("epsilon", 1e-5); + } + + virtual bool supportBackend(int backendId) CV_OVERRIDE { +#ifdef HAVE_INF_ENGINE + if (backendId == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) + return true; +#endif + return backendId == DNN_BACKEND_OPENCV || + backendId == DNN_BACKEND_CUDA; + } + + bool getMemoryShapes(const std::vector &inputs, + const int requiredOutputs, + std::vector &outputs, + std::vector &internals) const CV_OVERRIDE { + const auto &input = inputs[0]; + const auto &scale = inputs[1]; + const auto &bias = inputs[2]; + CV_CheckGE(input.size(), static_cast(3), "DNN/InstanceNorm: input dimension >= 3 is required"); + + int C = input[1]; + int scale_dim = std::accumulate(scale.begin(), scale.end(), 1, std::multiplies()); + CV_CheckEQ(scale_dim, C, "DNN/InstanceNorm: scale must be a 1d tensor and match the channel of input"); + int bias_dim = std::accumulate(bias.begin(), bias.end(), 1, std::multiplies()); + CV_CheckEQ(bias_dim, C, "DNN/InstanceNorm: bias must be a 1d tensor and match the channel of input"); + + outputs.assign(1, inputs[0]); + return false; + } + + void forward(InputArrayOfArrays inputs_arr, OutputArrayOfArrays outputs_arr, OutputArrayOfArrays internals_arr) CV_OVERRIDE { + CV_TRACE_FUNCTION(); + CV_TRACE_ARG_VALUE(name, "name", name.c_str()); + + CV_OCL_RUN(IS_DNN_OPENCL_TARGET(preferableTarget), + forward_ocl(inputs_arr, outputs_arr, internals_arr)) + + if (inputs_arr.depth() == CV_16S) + { + forward_fallback(inputs_arr, outputs_arr, internals_arr); + return; + } + + std::vector inputs, outputs; + inputs_arr.getMatVector(inputs); + outputs_arr.getMatVector(outputs); + + const auto &input = inputs[0]; + const auto &scale = inputs[1]; + const auto &bias = inputs[2]; + + fastNormChannel(input, scale, bias, outputs[0], epsilon); + } + +#ifdef HAVE_OPENCL + bool forward_ocl(InputArrayOfArrays inputs_, OutputArrayOfArrays outputs_, OutputArrayOfArrays internals_) { + std::vector inputs; + std::vector outputs; + + inputs_.getUMatVector(inputs); + outputs_.getUMatVector(outputs); + + const auto &input = inputs[0], &scale = inputs[1], &bias = inputs[2]; + auto &output = outputs[0]; + + const auto input_shape = shape(input); + size_t N = input_shape[0], C = input_shape[1], + loops = N * C, norm_size = static_cast(total(input_shape, 2)); + float inv_norm_size = 1.f / norm_size; + + // no fp16 support + if (input.depth() == CV_16S) { + return false; + } + + String base_opts = format(" -DT=float -DT4=float4 -Dconvert_T=convert_float4"); + + // Calculate mean + UMat one = UMat::ones(norm_size, 1, CV_32F); + UMat mean = UMat(loops, 1, CV_32F); + UMat mean_square = UMat(loops, 1, CV_32F); + UMat tmp = UMat(loops, norm_size, CV_32F); + bool ret = ocl4dnn::ocl4dnnGEMV(ocl4dnn::CblasNoTrans, loops, norm_size, inv_norm_size, + input, 0, one, 0, 0.f, mean, 0); + if (!ret) { + return false; + } + // Calculate mean_square + int num_vector = (norm_size % 8 == 0) ? 8 : ((norm_size % 4 == 0) ? 4 : 1); + size_t global[] = {loops, static_cast(norm_size / num_vector)}; + String build_opt = format(" -DNUM=%d", num_vector) + base_opts; + String mean_square_kernel_name = format("calc_mean%d", num_vector); + ocl::Kernel mean_square_kernel(mean_square_kernel_name.c_str(), ocl::dnn::mvn_oclsrc, build_opt + " -DKERNEL_MEAN"); + if (mean_square_kernel.empty()) { + return false; + } + mean_square_kernel.set(0, ocl::KernelArg::PtrReadOnly(input)); + mean_square_kernel.set(1, (int)loops); + mean_square_kernel.set(2, (int)norm_size); + mean_square_kernel.set(3, ocl::KernelArg::PtrReadOnly(mean)); + mean_square_kernel.set(4, ocl::KernelArg::PtrWriteOnly(tmp)); + ret = mean_square_kernel.run(2, global, NULL, false); + if (!ret) { + return false; + } + ret = ocl4dnn::ocl4dnnGEMV(ocl4dnn::CblasNoTrans, loops, norm_size, inv_norm_size, + tmp, 0, one, 0, 0.f, mean_square, 0); + if (!ret) { + return false; + } + // Calculate instance norm: output = scale * (x - mean) / sqrt(var + eps) + bias + String mvn_kernel_name = format("mvn%d", num_vector); + build_opt += " -DNORM_VARIANCE -DFUSE_BATCH_NORM -DKERNEL_MVN"; + ocl::Kernel mvn_kernel(mvn_kernel_name.c_str(), ocl::dnn::mvn_oclsrc, build_opt); + if (mvn_kernel.empty()) { + return false; + } + mvn_kernel.set(0, ocl::KernelArg::PtrReadOnly(input)); + mvn_kernel.set(1, (int)loops); + mvn_kernel.set(2, (int)norm_size); + mvn_kernel.set(3, (float)epsilon); + mvn_kernel.set(4, ocl::KernelArg::PtrReadOnly(mean)); + mvn_kernel.set(5, ocl::KernelArg::PtrReadOnly(mean_square)); + mvn_kernel.set(6, ocl::KernelArg::PtrReadOnly(scale)); + mvn_kernel.set(7, ocl::KernelArg::PtrReadOnly(bias)); + mvn_kernel.set(8, (int)C); + mvn_kernel.set(9, (float)0.f); + mvn_kernel.set(10, ocl::KernelArg::PtrWriteOnly(output)); + ret = mvn_kernel.run(2, global, NULL, false); + if (!ret) { + return false; + } + + return true; + } +#endif + +#ifdef HAVE_DNN_NGRAPH + virtual Ptr initNgraph(const std::vector >& inputs, + const std::vector >& nodes) CV_OVERRIDE { + // onnx to openvino convertion: https://github.com/openvinotoolkit/openvino/blob/2023.1.0/src/frontends/onnx/frontend/src/op/instance_norm.cpp + + auto ieInpNode = nodes[0].dynamicCast()->node; + const auto &input_shape = ieInpNode.get_shape(); + std::shared_ptr mvn, result; + + // mvn +#if INF_ENGINE_VER_MAJOR_LE(INF_ENGINE_RELEASE_2021_2) + // https://docs.openvino.ai/2021.4/api/ngraph_python_api/_autosummary/ngraph.opset3.mvn.html?highlight=mvn#ngraph.opset3.mvn + bool across_channels = false; + bool normalize_variance = true; + mvn = std::make_shared(ieInpNode, across_channels, normalize_variance, epsilon); +#else + // https://docs.openvino.ai/2023.1/openvino_docs_ops_normalization_MVN_6.html + std::vector axes_v(input_shape.size() - 2); + std::iota(axes_v.begin(), axes_v.end(), 2); // {2, 3, ...} for nd input tensor, n>=3 + auto axes = std::make_shared(ngraph::element::i64, ngraph::Shape{axes_v.size()}, axes_v.data()); + bool normalize_variance = true; + mvn = std::make_shared(ieInpNode, axes, normalize_variance, epsilon, ngraph::op::MVNEpsMode::INSIDE_SQRT); +#endif + + // instance norm = scale * mvn + bias + auto scale = nodes[1].dynamicCast()->node; + std::vector shared_shape_v(input_shape.size(), 1); + shared_shape_v[1] = -1; + auto shared_shape = std::make_shared(ngraph::element::i64, ngraph::Shape{shared_shape_v.size()}, shared_shape_v.data()); + scale = std::make_shared(scale, shared_shape, true); + result = std::make_shared(mvn, scale); + auto bias = nodes[2].dynamicCast()->node; + bias = std::make_shared(bias, shared_shape, true); + result = std::make_shared(result, bias); + + return Ptr(new InfEngineNgraphNode(result)); + } +#endif // HAVE_DNN_NGRAPH + +#ifdef HAVE_CUDA + Ptr initCUDA(void *context_, + const std::vector>& inputs, + const std::vector>& outputs) override { + auto context = reinterpret_cast(context_); + + auto input_wrapper = inputs[0].dynamicCast(); + auto input_shape = input_wrapper->getShape(); + size_t loops = static_cast(total(input_shape, 0, 2)); + + return make_cuda_node(preferableTarget, std::move(context->stream), epsilon, loops); + } +#endif // HAVE_CUDA + +}; + +Ptr InstanceNormLayer::create(const LayerParams ¶ms) { + return Ptr(new InstanceNormLayerImpl(params)); +} + +}} // cv::dnn diff --git a/modules/dnn/src/onnx/onnx_importer.cpp b/modules/dnn/src/onnx/onnx_importer.cpp index 4813d118c4..28dd4d9b77 100644 --- a/modules/dnn/src/onnx/onnx_importer.cpp +++ b/modules/dnn/src/onnx/onnx_importer.cpp @@ -1844,44 +1844,43 @@ void ONNXImporter::parseLRN(LayerParams& layerParams, const opencv_onnx::NodePro addLayer(layerParams, node_proto); } -void ONNXImporter::parseInstanceNormalization(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto_) -{ - opencv_onnx::NodeProto node_proto = node_proto_; - if (node_proto.input_size() != 3) - CV_Error(Error::StsNotImplemented, - "Expected input, scale, bias"); +void ONNXImporter::parseInstanceNormalization(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) { + int num_inputs = node_proto.input_size(); + CV_CheckEQ(num_inputs, 3, "DNN/ONNXImporter - InstanceNorm: three inputs are required"); - layerParams.blobs.resize(4); - layerParams.blobs[2] = getBlob(node_proto, 1); // weightData - layerParams.blobs[3] = getBlob(node_proto, 2); // biasData - layerParams.set("has_bias", true); - layerParams.set("has_weight", true); + bool found_input = constBlobs.find(node_proto.input(0)) != constBlobs.end(); + bool found_scale = constBlobs.find(node_proto.input(1)) != constBlobs.end(); + bool found_bias = constBlobs.find(node_proto.input(2)) != constBlobs.end(); - // Get number of channels in input - int size = layerParams.blobs[2].total(); - layerParams.blobs[0] = Mat::zeros(size, 1, CV_32F); // mean - layerParams.blobs[1] = Mat::ones(size, 1, CV_32F); // std + if (found_input && found_scale && found_bias) { + std::vector inputs, output; - LayerParams mvnParams; - mvnParams.name = layerParams.name + "/MVN"; - mvnParams.type = "MVN"; - mvnParams.set("eps", layerParams.get("epsilon")); - layerParams.erase("epsilon"); + Mat input = getBlob(node_proto, 0); + Mat scale = getBlob(node_proto, 1); + Mat bias = getBlob(node_proto, 2); + inputs.push_back(input); + inputs.push_back(scale); + inputs.push_back(bias); - //Create MVN layer - int id = dstNet.addLayer(mvnParams.name, mvnParams.type, mvnParams); - //Connect to input - IterLayerId_t layerId = layer_id.find(node_proto.input(0)); - CV_Assert(layerId != layer_id.end()); - dstNet.connect(layerId->second.layerId, layerId->second.outputId, id, 0); - //Add shape - layer_id.insert(std::make_pair(mvnParams.name, LayerInfo(id, 0))); - outShapes[mvnParams.name] = outShapes[node_proto.input(0)]; + runLayer(layerParams, inputs, output); + addConstant(node_proto.output(0), output[0]); + } else { + auto add_const_node = [&] (int i) { + LayerParams const_params; + const_params.name = node_proto.input(i); + const_params.type = "Const"; + Mat blob = getBlob(node_proto, i); + const_params.blobs.push_back(blob); - //Replace Batch Norm's input to MVN - node_proto.set_input(0, mvnParams.name); - layerParams.type = "BatchNorm"; - addLayer(layerParams, node_proto); + opencv_onnx::NodeProto proto; + proto.add_output(const_params.name); + addLayer(const_params, proto); + }; + if (found_input && layer_id.find(node_proto.input(0)) == layer_id.end()) { add_const_node(0); } + if (found_scale && layer_id.find(node_proto.input(1)) == layer_id.end()) { add_const_node(1); } + if (found_bias && layer_id.find(node_proto.input(2)) == layer_id.end()) { add_const_node(2); } + addLayer(layerParams, node_proto); + } } void ONNXImporter::parseBatchNormalization(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) diff --git a/modules/dnn/test/test_onnx_conformance_layer_parser_denylist.inl.hpp b/modules/dnn/test/test_onnx_conformance_layer_parser_denylist.inl.hpp index 8c461b699f..be60c38b86 100644 --- a/modules/dnn/test/test_onnx_conformance_layer_parser_denylist.inl.hpp +++ b/modules/dnn/test/test_onnx_conformance_layer_parser_denylist.inl.hpp @@ -159,8 +159,6 @@ "test_if", "test_if_opt", "test_if_seq", -"test_instancenorm_epsilon", -"test_instancenorm_example", "test_isinf", "test_isinf_negative", "test_isinf_positive", From 30549d65c24e25c4041a01aa1658df63cb3bb265 Mon Sep 17 00:00:00 2001 From: alexlyulkov Date: Tue, 7 Nov 2023 18:23:33 +0700 Subject: [PATCH 011/156] Merge pull request #24456 from alexlyulkov:al/aar Added scripts for creating an AAR package and a local Maven repository with OpenCV library #24456 Added scripts for creating an AAR package and a local Maven repository with OpenCV library. The build_java_shared_aar.py script creates AAR with Java + C++ shared libraries. The build_static_aar.py script creates AAR with static C++ libraries. The scripts use an Android project template. The project is almost a default Android AAR library project with empty Java code and one empty C++ library. Only build.gradle.template and CMakeLists.txt.template files contain significant changes. See README.md for more information. --- .../aar-template/OpenCV/build.gradle.template | 88 +++++++ .../aar-template/OpenCV/proguard-rules.pro | 21 ++ .../OpenCV/src/main/AndroidManifest.xml | 4 + .../src/main/cpp/CMakeLists.txt.template | 5 + .../OpenCV/src/main/cpp/include/temp.h | 1 + .../OpenCV/src/main/cpp/native-lib.cpp | 1 + platforms/android/aar-template/README.md | 29 +++ platforms/android/aar-template/build.gradle | 4 + .../android/aar-template/gradle.properties | 21 ++ .../gradle/wrapper/gradle-wrapper.jar | Bin 0 -> 59203 bytes .../gradle/wrapper/gradle-wrapper.properties | 6 + platforms/android/aar-template/gradlew | 185 ++++++++++++++ platforms/android/aar-template/gradlew.bat | 89 +++++++ .../android/aar-template/settings.gradle | 16 ++ platforms/android/build_java_shared_aar.py | 156 ++++++++++++ platforms/android/build_static_aar.py | 229 ++++++++++++++++++ 16 files changed, 855 insertions(+) create mode 100644 platforms/android/aar-template/OpenCV/build.gradle.template create mode 100644 platforms/android/aar-template/OpenCV/proguard-rules.pro create mode 100644 platforms/android/aar-template/OpenCV/src/main/AndroidManifest.xml create mode 100644 platforms/android/aar-template/OpenCV/src/main/cpp/CMakeLists.txt.template create mode 100644 platforms/android/aar-template/OpenCV/src/main/cpp/include/temp.h create mode 100644 platforms/android/aar-template/OpenCV/src/main/cpp/native-lib.cpp create mode 100644 platforms/android/aar-template/README.md create mode 100644 platforms/android/aar-template/build.gradle create mode 100644 platforms/android/aar-template/gradle.properties create mode 100644 platforms/android/aar-template/gradle/wrapper/gradle-wrapper.jar create mode 100644 platforms/android/aar-template/gradle/wrapper/gradle-wrapper.properties create mode 100755 platforms/android/aar-template/gradlew create mode 100644 platforms/android/aar-template/gradlew.bat create mode 100644 platforms/android/aar-template/settings.gradle create mode 100755 platforms/android/build_java_shared_aar.py create mode 100755 platforms/android/build_static_aar.py diff --git a/platforms/android/aar-template/OpenCV/build.gradle.template b/platforms/android/aar-template/OpenCV/build.gradle.template new file mode 100644 index 0000000000..4f3a3846ec --- /dev/null +++ b/platforms/android/aar-template/OpenCV/build.gradle.template @@ -0,0 +1,88 @@ +plugins { + id 'com.android.library' + id 'maven-publish' +} + +android { + namespace 'org.opencv' + compileSdk ${COMPILE_SDK} + + defaultConfig { + minSdk ${MIN_SDK} + targetSdk ${TARGET_SDK} + + testInstrumentationRunner "androidx.test.runner.AndroidJUnitRunner" + externalNativeBuild { + cmake { + cppFlags "" + arguments "-DANDROID_STL=${LIB_TYPE}" + } + } + ndk { + abiFilters ${ABI_FILTERS} + } + } + + buildTypes { + release { + minifyEnabled false + proguardFiles getDefaultProguardFile('proguard-android-optimize.txt'), 'proguard-rules.pro' + } + } + compileOptions { + sourceCompatibility JavaVersion.VERSION_${JAVA_VERSION} + targetCompatibility JavaVersion.VERSION_${JAVA_VERSION} + } + externalNativeBuild { + cmake { + path file('src/main/cpp/CMakeLists.txt') + } + } + buildFeatures { + aidl true + prefabPublishing true + buildConfig true + } + prefab { + ${LIB_NAME} { + headers "src/main/cpp/include" + } + } + sourceSets { + main { + java.srcDirs = ['src/main/java'] + //jniLibs.srcDirs = ['libs'] + aidl.srcDirs = ['src/main/java'] + } + } + + publishing { + singleVariant('release') { + withSourcesJar() + } + } +} + +publishing { + publications { + release(MavenPublication) { + groupId = 'org.opencv' + artifactId = '${PACKAGE_NAME}' + version = '${OPENCV_VERSION}' + artifact("opencv-release.aar") + +// afterEvaluate { +// from components.release +// } + } + } + repositories { + maven { + name = 'myrepo' + url = "${project.buildDir}/repo" + } + } +} + +dependencies { +} \ No newline at end of file diff --git a/platforms/android/aar-template/OpenCV/proguard-rules.pro b/platforms/android/aar-template/OpenCV/proguard-rules.pro new file mode 100644 index 0000000000..481bb43481 --- /dev/null +++ b/platforms/android/aar-template/OpenCV/proguard-rules.pro @@ -0,0 +1,21 @@ +# Add project specific ProGuard rules here. +# You can control the set of applied configuration files using the +# proguardFiles setting in build.gradle. +# +# For more details, see +# http://developer.android.com/guide/developing/tools/proguard.html + +# If your project uses WebView with JS, uncomment the following +# and specify the fully qualified class name to the JavaScript interface +# class: +#-keepclassmembers class fqcn.of.javascript.interface.for.webview { +# public *; +#} + +# Uncomment this to preserve the line number information for +# debugging stack traces. +#-keepattributes SourceFile,LineNumberTable + +# If you keep the line number information, uncomment this to +# hide the original source file name. +#-renamesourcefileattribute SourceFile \ No newline at end of file diff --git a/platforms/android/aar-template/OpenCV/src/main/AndroidManifest.xml b/platforms/android/aar-template/OpenCV/src/main/AndroidManifest.xml new file mode 100644 index 0000000000..cce937ee78 --- /dev/null +++ b/platforms/android/aar-template/OpenCV/src/main/AndroidManifest.xml @@ -0,0 +1,4 @@ + + + + \ No newline at end of file diff --git a/platforms/android/aar-template/OpenCV/src/main/cpp/CMakeLists.txt.template b/platforms/android/aar-template/OpenCV/src/main/cpp/CMakeLists.txt.template new file mode 100644 index 0000000000..02ef035e0a --- /dev/null +++ b/platforms/android/aar-template/OpenCV/src/main/cpp/CMakeLists.txt.template @@ -0,0 +1,5 @@ +cmake_minimum_required(VERSION 3.6) + +project("opencv") + +add_library(${LIB_NAME} ${LIB_TYPE} native-lib.cpp) diff --git a/platforms/android/aar-template/OpenCV/src/main/cpp/include/temp.h b/platforms/android/aar-template/OpenCV/src/main/cpp/include/temp.h new file mode 100644 index 0000000000..4974d8ca96 --- /dev/null +++ b/platforms/android/aar-template/OpenCV/src/main/cpp/include/temp.h @@ -0,0 +1 @@ +// This empty .h file is used for creating an AAR with empty C++ lib that will be replaced with OpenCV C++ lib \ No newline at end of file diff --git a/platforms/android/aar-template/OpenCV/src/main/cpp/native-lib.cpp b/platforms/android/aar-template/OpenCV/src/main/cpp/native-lib.cpp new file mode 100644 index 0000000000..73ac04eb7a --- /dev/null +++ b/platforms/android/aar-template/OpenCV/src/main/cpp/native-lib.cpp @@ -0,0 +1 @@ +// This empty .cpp file is used for creating an AAR with empty C++ lib that will be replaced with OpenCV C++ lib \ No newline at end of file diff --git a/platforms/android/aar-template/README.md b/platforms/android/aar-template/README.md new file mode 100644 index 0000000000..a851364dd4 --- /dev/null +++ b/platforms/android/aar-template/README.md @@ -0,0 +1,29 @@ +## Scripts for creating an AAR package and a local Maven repository with OpenCV libraries for Android + +### How to run the scripts +1. Set JAVA_HOME and ANDROID_HOME enviroment variables. For example: +``` +export JAVA_HOME=~/Android Studio/jbr +export ANDROID_HOME=~/Android/SDK +``` +2. Download OpenCV SDK for Android +3. Run build script for version with Java and a shared C++ library: +``` +python build_java_shared_aar.py "~/opencv-4.7.0-android-sdk/OpenCV-android-sdk" +``` +4. Run build script for version with static C++ libraries: +``` +python build_static_aar.py "~/opencv-4.7.0-android-sdk/OpenCV-android-sdk" +``` +The AAR libraries and the local Maven repository will be created in the **outputs** directory +### Technical details +The scripts consist of 5 steps: +1. Preparing Android AAR library project template +2. Adding Java code to the project. Adding C++ public headers for shared version to the project. +3. Compiling the project to build an AAR package +4. Adding C++ binary libraries to the AAR package. Adding C++ public headers for static version to the AAR package. +5. Creating Maven repository with the AAR package + +There are a few minor limitations: +1. Due to the AAR design the Java + shared C++ AAR package contains duplicates of C++ binary libraries, but the final user's Android application contains only one library instance. +2. The compile definitions from cmake configs are skipped, but it shouldn't affect the library because the script uses precompiled C++ binaries from SDK. diff --git a/platforms/android/aar-template/build.gradle b/platforms/android/aar-template/build.gradle new file mode 100644 index 0000000000..b7a22c372c --- /dev/null +++ b/platforms/android/aar-template/build.gradle @@ -0,0 +1,4 @@ +// Top-level build file where you can add configuration options common to all sub-projects/modules. +plugins { + id 'com.android.library' version '8.0.2' apply false +} \ No newline at end of file diff --git a/platforms/android/aar-template/gradle.properties b/platforms/android/aar-template/gradle.properties new file mode 100644 index 0000000000..3e927b11ef --- /dev/null +++ b/platforms/android/aar-template/gradle.properties @@ -0,0 +1,21 @@ +# Project-wide Gradle settings. +# IDE (e.g. Android Studio) users: +# Gradle settings configured through the IDE *will override* +# any settings specified in this file. +# For more details on how to configure your build environment visit +# http://www.gradle.org/docs/current/userguide/build_environment.html +# Specifies the JVM arguments used for the daemon process. +# The setting is particularly useful for tweaking memory settings. +org.gradle.jvmargs=-Xmx2048m -Dfile.encoding=UTF-8 +# When configured, Gradle will run in incubating parallel mode. +# This option should only be used with decoupled projects. More details, visit +# http://www.gradle.org/docs/current/userguide/multi_project_builds.html#sec:decoupled_projects +# org.gradle.parallel=true +# AndroidX package structure to make it clearer which packages are bundled with the +# Android operating system, and which are packaged with your app's APK +# https://developer.android.com/topic/libraries/support-library/androidx-rn +android.useAndroidX=true +# Enables namespacing of each library's R class so that its R class includes only the +# resources declared in the library itself and none from the library's dependencies, +# thereby reducing the size of the R class for that library +android.nonTransitiveRClass=true \ No newline at end of file diff --git a/platforms/android/aar-template/gradle/wrapper/gradle-wrapper.jar b/platforms/android/aar-template/gradle/wrapper/gradle-wrapper.jar new file mode 100644 index 0000000000000000000000000000000000000000..e708b1c023ec8b20f512888fe07c5bd3ff77bb8f GIT binary patch literal 59203 zcma&O1CT9Y(k9%tZQHhO+qUh#ZQHhO+qmuS+qP|E@9xZO?0h@l{(r>DQ>P;GjjD{w zH}lENr;dU&FbEU?00aa80D$0M0RRB{U*7-#kbjS|qAG&4l5%47zyJ#WrfA#1$1Ctx zf&Z_d{GW=lf^w2#qRJ|CvSJUi(^E3iv~=^Z(zH}F)3Z%V3`@+rNB7gTVU{Bb~90p|f+0(v;nz01EG7yDMX9@S~__vVgv%rS$+?IH+oZ03D5zYrv|^ zC1J)SruYHmCki$jLBlTaE5&dFG9-kq3!^i>^UQL`%gn6)jz54$WDmeYdsBE9;PqZ_ zoGd=P4+|(-u4U1dbAVQrFWoNgNd;0nrghPFbQrJctO>nwDdI`Q^i0XJDUYm|T|RWc zZ3^Qgo_Qk$%Fvjj-G}1NB#ZJqIkh;kX%V{THPqOyiq)d)0+(r9o(qKlSp*hmK#iIY zA^)Vr$-Hz<#SF=0@tL@;dCQsm`V9s1vYNq}K1B)!XSK?=I1)tX+bUV52$YQu*0%fnWEukW>mxkz+%3-S!oguE8u#MGzST8_Dy^#U?fA@S#K$S@9msUiX!gd_ow>08w5)nX{-KxqMOo7d?k2&?Vf z&diGDtZr(0cwPe9z9FAUSD9KC)7(n^lMWuayCfxzy8EZsns%OEblHFSzP=cL6}?J| z0U$H!4S_TVjj<`6dy^2j`V`)mC;cB%* z8{>_%E1^FH!*{>4a7*C1v>~1*@TMcLK{7nEQ!_igZC}ikJ$*<$yHy>7)oy79A~#xE zWavoJOIOC$5b6*q*F_qN1>2#MY)AXVyr$6x4b=$x^*aqF*L?vmj>Mgv+|ITnw_BoW zO?jwHvNy^prH{9$rrik1#fhyU^MpFqF2fYEt(;4`Q&XWOGDH8k6M=%@fics4ajI;st# zCU^r1CK&|jzUhRMv;+W~6N;u<;#DI6cCw-otsc@IsN3MoSD^O`eNflIoR~l4*&-%RBYk@gb^|-JXs&~KuSEmMxB}xSb z@K76cXD=Y|=I&SNC2E+>Zg?R6E%DGCH5J1nU!A|@eX9oS(WPaMm==k2s_ueCqdZw| z&hqHp)47`c{BgwgvY2{xz%OIkY1xDwkw!<0veB#yF4ZKJyabhyyVS`gZepcFIk%e2 zTcrmt2@-8`7i-@5Nz>oQWFuMC_KlroCl(PLSodswHqJ3fn<;gxg9=}~3x_L3P`9Sn zChIf}8vCHvTriz~T2~FamRi?rh?>3bX1j}%bLH+uFX+p&+^aXbOK7clZxdU~6Uxgy z8R=obwO4dL%pmVo*Ktf=lH6hnlz_5k3cG;m8lgaPp~?eD!Yn2kf)tU6PF{kLyn|oI@eQ`F z3IF7~Blqg8-uwUuWZScRKn%c2_}dXB6Dx_&xR*n9M9LXasJhtZdr$vBY!rP{c@=)& z#!?L$2UrkvClwQO>U*fSMs67oSj2mxiJ$t;E|>q%Kh_GzzWWO&3;ufU%2z%ucBU8H z3WIwr$n)cfCXR&>tyB7BcSInK>=ByZA%;cVEJhcg<#6N{aZC4>K41XF>ZgjG`z_u& zGY?;Ad?-sgiOnI`oppF1o1Gurqbi*;#x2>+SSV6|1^G@ooVy@fg?wyf@0Y!UZ4!}nGuLeC^l)6pwkh|oRY`s1Pm$>zZ3u-83T|9 zGaKJIV3_x+u1>cRibsaJpJqhcm%?0-L;2 zitBrdRxNmb0OO2J%Y&Ym(6*`_P3&&5Bw157{o7LFguvxC$4&zTy#U=W*l&(Q2MNO} zfaUwYm{XtILD$3864IA_nn34oVa_g^FRuHL5wdUd)+W-p-iWCKe8m_cMHk+=? zeKX)M?Dt(|{r5t7IenkAXo%&EXIb-i^w+0CX0D=xApC=|Xy(`xy+QG^UyFe z+#J6h_&T5i#sV)hj3D4WN%z;2+jJcZxcI3*CHXGmOF3^)JD5j&wfX)e?-|V0GPuA+ zQFot%aEqGNJJHn$!_}#PaAvQ^{3-Ye7b}rWwrUmX53(|~i0v{}G_sI9uDch_brX&6 zWl5Ndj-AYg(W9CGfQf<6!YmY>Ey)+uYd_JNXH=>|`OH-CDCmcH(0%iD_aLlNHKH z7bcW-^5+QV$jK?R*)wZ>r9t}loM@XN&M-Pw=F#xn(;u3!(3SXXY^@=aoj70;_=QE9 zGghsG3ekq#N||u{4We_25U=y#T*S{4I{++Ku)> zQ!DZW;pVcn>b;&g2;YE#+V`v*Bl&Y-i@X6D*OpNA{G@JAXho&aOk(_j^weW{#3X5Y z%$q_wpb07EYPdmyH(1^09i$ca{O<}7) zRWncXdSPgBE%BM#by!E>tdnc$8RwUJg1*x($6$}ae$e9Knj8gvVZe#bLi!<+&BkFj zg@nOpDneyc+hU9P-;jmOSMN|*H#>^Ez#?;%C3hg_65leSUm;iz)UkW)jX#p)e&S&M z1|a?wDzV5NVnlhRBCd_;F87wp>6c<&nkgvC+!@KGiIqWY4l}=&1w7|r6{oBN8xyzh zG$b#2=RJp_iq6)#t5%yLkKx(0@D=C3w+oiXtSuaQ%I1WIb-eiE$d~!)b@|4XLy!CZ z9p=t=%3ad@Ep+<9003D2KZ5VyP~_n$=;~r&YUg5UZ0KVD&tR1DHy9x)qWtKJp#Kq# zP*8p#W(8JJ_*h_3W}FlvRam?<4Z+-H77^$Lvi+#vmhL9J zJ<1SV45xi;SrO2f=-OB(7#iNA5)x1uNC-yNxUw|!00vcW2PufRm>e~toH;M0Q85MQLWd?3O{i8H+5VkR@l9Dg-ma ze2fZ%>G(u5(k9EHj2L6!;(KZ8%8|*-1V|B#EagbF(rc+5iL_5;Eu)L4Z-V;0HfK4d z*{utLse_rvHZeQ>V5H=f78M3Ntg1BPxFCVD{HbNA6?9*^YIq;B-DJd{Ca2L#)qWP? zvX^NhFmX?CTWw&Ns}lgs;r3i+Bq@y}Ul+U%pzOS0Fcv9~aB(0!>GT0)NO?p=25LjN z2bh>6RhgqD7bQj#k-KOm@JLgMa6>%-ok1WpOe)FS^XOU{c?d5shG(lIn3GiVBxmg`u%-j=)^v&pX1JecJics3&jvPI)mDut52? z3jEA)DM%}BYbxxKrizVYwq?(P&19EXlwD9^-6J+4!}9{ywR9Gk42jjAURAF&EO|~N z)?s>$Da@ikI4|^z0e{r`J8zIs>SpM~Vn^{3fArRu;?+43>lD+^XtUcY1HidJwnR6+ z!;oG2=B6Z_=M%*{z-RaHc(n|1RTKQdNjjV!Pn9lFt^4w|AeN06*j}ZyhqZ^!-=cyGP_ShV1rGxkx8t zB;8`h!S{LD%ot``700d0@Grql(DTt4Awgmi+Yr0@#jbe=2#UkK%rv=OLqF)9D7D1j z!~McAwMYkeaL$~kI~90)5vBhBzWYc3Cj1WI0RS`z000R8-@ET0dA~*r(gSiCJmQMN&4%1D zyVNf0?}sBH8zNbBLn>~(W{d3%@kL_eQ6jEcR{l>C|JK z(R-fA!z|TTRG40|zv}7E@PqCAXP3n`;%|SCQ|ZS%ym$I{`}t3KPL&^l5`3>yah4*6 zifO#{VNz3)?ZL$be;NEaAk9b#{tV?V7 zP|wf5YA*1;s<)9A4~l3BHzG&HH`1xNr#%){4xZ!jq%o=7nN*wMuXlFV{HaiQLJ`5G zBhDi#D(m`Q1pLh@Tq+L;OwuC52RdW7b8}~60WCOK5iYMUad9}7aWBuILb({5=z~YF zt?*Jr5NG+WadM{mDL>GyiByCuR)hd zA=HM?J6l1Xv0Dl+LW@w$OTcEoOda^nFCw*Sy^I@$sSuneMl{4ys)|RY#9&NxW4S)9 zq|%83IpslTLoz~&vTo!Ga@?rj_kw{|k{nv+w&Ku?fyk4Ki4I?);M|5Axm)t+BaE)D zm(`AQ#k^DWrjbuXoJf2{Aj^KT zFb1zMSqxq|vceV+Mf-)$oPflsO$@*A0n0Z!R{&(xh8s}=;t(lIy zv$S8x>m;vQNHuRzoaOo?eiWFe{0;$s`Bc+Osz~}Van${u;g(su`3lJ^TEfo~nERfP z)?aFzpDgnLYiERsKPu|0tq4l2wT)Atr6Qb%m-AUn6HnCue*yWICp7TjW$@sO zm5rm4aTcPQ(rfi7a`xP7cKCFrJD}*&_~xgLyr^-bmsL}y;A5P|al8J3WUoBSjqu%v zxC;mK!g(7r6RRJ852Z~feoC&sD3(6}^5-uLK8o)9{8L_%%rItZK9C){UxB|;G>JbP zsRRtS4-3B*5c+K2kvmgZK8472%l>3cntWUOVHxB|{Ay~aOg5RN;{PJgeVD*H%ac+y!h#wi%o2bF2Ca8IyMyH{>4#{E_8u^@+l-+n=V}Sq?$O z{091@v%Bd*3pk0^2UtiF9Z+(a@wy6 zUdw8J*ze$K#=$48IBi1U%;hmhO>lu!uU;+RS}p&6@rQila7WftH->*A4=5W|Fmtze z)7E}jh@cbmr9iup^i%*(uF%LG&!+Fyl@LFA-}Ca#bxRfDJAiR2dt6644TaYw1Ma79 zt8&DYj31j^5WPNf5P&{)J?WlCe@<3u^78wnd(Ja4^a>{^Tw}W>|Cjt^If|7l^l)^Q zbz|7~CF(k_9~n|h;ysZ+jHzkXf(*O*@5m zLzUmbHp=x!Q|!9NVXyipZ3)^GuIG$k;D)EK!a5=8MFLI_lpf`HPKl=-Ww%z8H_0$j ztJ||IfFG1lE9nmQ0+jPQy zCBdKkjArH@K7jVcMNz);Q(Q^R{d5G?-kk;Uu_IXSyWB)~KGIizZL(^&qF;|1PI7!E zTP`%l)gpX|OFn&)M%txpQ2F!hdA~hX1Cm5)IrdljqzRg!f{mN%G~H1&oqe`5eJCIF zHdD7O;AX-{XEV(a`gBFJ9ews#CVS2y!&>Cm_dm3C8*n3MA*e67(WC?uP@8TXuMroq z{#w$%z@CBIkRM7?}Xib+>hRjy?%G!fiw8! z8(gB+8J~KOU}yO7UGm&1g_MDJ$IXS!`+*b*QW2x)9>K~Y*E&bYMnjl6h!{17_8d!%&9D`a7r&LKZjC<&XOvTRaKJ1 zUY@hl5^R&kZl3lU3njk`3dPzxj$2foOL26r(9zsVF3n_F#v)s5vv3@dgs|lP#eylq62{<-vczqP!RpVBTgI>@O6&sU>W|do17+#OzQ7o5A$ICH z?GqwqnK^n2%LR;$^oZM;)+>$X3s2n}2jZ7CdWIW0lnGK-b#EG01)P@aU`pg}th&J-TrU`tIpb5t((0eu|!u zQz+3ZiOQ^?RxxK4;zs=l8q!-n7X{@jSwK(iqNFiRColuEOg}!7cyZi`iBX4g1pNBj zAPzL?P^Ljhn;1$r8?bc=#n|Ed7wB&oHcw()&*k#SS#h}jO?ZB246EGItsz*;^&tzp zu^YJ0=lwsi`eP_pU8}6JA7MS;9pfD;DsSsLo~ogzMNP70@@;Fm8f0^;>$Z>~}GWRw!W5J3tNX*^2+1f3hz{~rIzJo z6W%J(H!g-eI_J1>0juX$X4Cl6i+3wbc~k146UIX&G22}WE>0ga#WLsn9tY(&29zBvH1$`iWtTe zG2jYl@P!P)eb<5DsR72BdI7-zP&cZNI{7q3e@?N8IKc4DE#UVr->|-ryuJXk^u^>4 z$3wE~=q390;XuOQP~TNoDR?#|NSPJ%sTMInA6*rJ%go|=YjGe!B>z6u$IhgQSwoV* zjy3F2#I>uK{42{&IqP59)Y(1*Z>>#W8rCf4_eVsH)`v!P#^;BgzKDR`ARGEZzkNX+ zJUQu=*-ol=Xqqt5=`=pA@BIn@6a9G8C{c&`i^(i+BxQO9?YZ3iu%$$da&Kb?2kCCo zo7t$UpSFWqmydXf@l3bVJ=%K?SSw)|?srhJ-1ZdFu*5QhL$~-IQS!K1s@XzAtv6*Y zl8@(5BlWYLt1yAWy?rMD&bwze8bC3-GfNH=p zynNFCdxyX?K&G(ZZ)afguQ2|r;XoV^=^(;Cku#qYn4Lus`UeKt6rAlFo_rU`|Rq z&G?~iWMBio<78of-2X(ZYHx~=U0Vz4btyXkctMKdc9UM!vYr~B-(>)(Hc|D zMzkN4!PBg%tZoh+=Gba!0++d193gbMk2&krfDgcbx0jI92cq?FFESVg0D$>F+bil} zY~$)|>1HZsX=5sAZ2WgPB5P=8X#TI+NQ(M~GqyVB53c6IdX=k>Wu@A0Svf5#?uHaF zsYn|koIi3$(%GZ2+G+7Fv^lHTb#5b8sAHSTnL^qWZLM<(1|9|QFw9pnRU{svj}_Al zL)b9>fN{QiA($8peNEJyy`(a{&uh-T4_kdZFIVsKKVM(?05}76EEz?#W za^fiZOAd14IJ4zLX-n7Lq0qlQ^lW8Cvz4UKkV9~P}>sq0?xD3vg+$4vLm~C(+ zM{-3Z#qnZ09bJ>}j?6ry^h+@PfaD7*jZxBEY4)UG&daWb??6)TP+|3#Z&?GL?1i+280CFsE|vIXQbm| zM}Pk!U`U5NsNbyKzkrul-DzwB{X?n3E6?TUHr{M&+R*2%yOiXdW-_2Yd6?38M9Vy^ z*lE%gA{wwoSR~vN0=no}tP2Ul5Gk5M(Xq`$nw#ndFk`tcpd5A=Idue`XZ!FS>Q zG^0w#>P4pPG+*NC9gLP4x2m=cKP}YuS!l^?sHSFftZy{4CoQrb_ z^20(NnG`wAhMI=eq)SsIE~&Gp9Ne0nD4%Xiu|0Fj1UFk?6avDqjdXz{O1nKao*46y zT8~iA%Exu=G#{x=KD;_C&M+Zx4+n`sHT>^>=-1YM;H<72k>$py1?F3#T1*ef9mLZw z5naLQr?n7K;2l+{_uIw*_1nsTn~I|kkCgrn;|G~##hM;9l7Jy$yJfmk+&}W@JeKcF zx@@Woiz8qdi|D%aH3XTx5*wDlbs?dC1_nrFpm^QbG@wM=i2?Zg;$VK!c^Dp8<}BTI zyRhAq@#%2pGV49*Y5_mV4+OICP|%I(dQ7x=6Ob}>EjnB_-_18*xrY?b%-yEDT(wrO z9RY2QT0`_OpGfMObKHV;QLVnrK%mc?$WAdIT`kJQT^n%GuzE7|9@k3ci5fYOh(287 zuIbg!GB3xLg$YN=n)^pHGB0jH+_iIiC=nUcD;G6LuJsjn2VI1cyZx=a?ShCsF==QK z;q~*m&}L<-cb+mDDXzvvrRsybcgQ;Vg21P(uLv5I+eGc7o7tc6`;OA9{soHFOz zT~2?>Ts}gprIX$wRBb4yE>ot<8+*Bv`qbSDv*VtRi|cyWS>)Fjs>fkNOH-+PX&4(~ z&)T8Zam2L6puQl?;5zg9h<}k4#|yH9czHw;1jw-pwBM*O2hUR6yvHATrI%^mvs9q_ z&ccT0>f#eDG<^WG^q@oVqlJrhxH)dcq2cty@l3~|5#UDdExyXUmLQ}f4#;6fI{f^t zDCsgIJ~0`af%YR%Ma5VQq-p21k`vaBu6WE?66+5=XUd%Ay%D$irN>5LhluRWt7 zov-=f>QbMk*G##&DTQyou$s7UqjjW@k6=!I@!k+S{pP8R(2=e@io;N8E`EOB;OGoI zw6Q+{X1_I{OO0HPpBz!X!@`5YQ2)t{+!?M_iH25X(d~-Zx~cXnS9z>u?+If|iNJbx zyFU2d1!ITX64D|lE0Z{dLRqL1Ajj=CCMfC4lD3&mYR_R_VZ>_7_~|<^o*%_&jevU+ zQ4|qzci=0}Jydw|LXLCrOl1_P6Xf@c0$ieK2^7@A9UbF{@V_0p%lqW|L?5k>bVM8|p5v&2g;~r>B8uo<4N+`B zH{J)h;SYiIVx@#jI&p-v3dwL5QNV1oxPr8J%ooezTnLW>i*3Isb49%5i!&ac_dEXv zvXmVUck^QHmyrF8>CGXijC_R-y(Qr{3Zt~EmW)-nC!tiH`wlw5D*W7Pip;T?&j%kX z6DkZX4&}iw>hE(boLyjOoupf6JpvBG8}jIh!!VhnD0>}KSMMo{1#uU6kiFcA04~|7 zVO8eI&x1`g4CZ<2cYUI(n#wz2MtVFHx47yE5eL~8bot~>EHbevSt}LLMQX?odD{Ux zJMnam{d)W4da{l7&y-JrgiU~qY3$~}_F#G7|MxT)e;G{U`In&?`j<5D->}cb{}{T(4DF0BOk-=1195KB-E*o@c?`>y#4=dMtYtSY=&L{!TAjFVcq0y@AH`vH! z$41+u!Ld&}F^COPgL(EE{0X7LY&%D7-(?!kjFF7=qw<;`V{nwWBq<)1QiGJgUc^Vz ztMUlq1bZqKn17|6x6iAHbWc~l1HcmAxr%$Puv!znW)!JiukwIrqQ00|H$Z)OmGG@= zv%A8*4cq}(?qn4rN6o`$Y))(MyXr8R<2S^J+v(wmFmtac!%VOfN?&(8Nr!T@kV`N; z*Q33V3t`^rN&aBiHet)18wy{*wi1=W!B%B-Q6}SCrUl$~Hl{@!95ydml@FK8P=u4s z4e*7gV2s=YxEvskw2Ju!2%{8h01rx-3`NCPc(O zH&J0VH5etNB2KY6k4R@2Wvl^Ck$MoR3=)|SEclT2ccJ!RI9Nuter7u9@;sWf-%um;GfI!=eEIQ2l2p_YWUd{|6EG ze{yO6;lMc>;2tPrsNdi@&1K6(1;|$xe8vLgiouj%QD%gYk`4p{Ktv9|j+!OF-P?@p z;}SV|oIK)iwlBs+`ROXkhd&NK zzo__r!B>tOXpBJMDcv!Mq54P+n4(@dijL^EpO1wdg~q+!DT3lB<>9AANSe!T1XgC=J^)IP0XEZ()_vpu!!3HQyJhwh?r`Ae%Yr~b% zO*NY9t9#qWa@GCPYOF9aron7thfWT`eujS4`t2uG6)~JRTI;f(ZuoRQwjZjp5Pg34 z)rp$)Kr?R+KdJ;IO;pM{$6|2y=k_siqvp%)2||cHTe|b5Ht8&A{wazGNca zX$Ol?H)E_R@SDi~4{d-|8nGFhZPW;Cts1;08TwUvLLv&_2$O6Vt=M)X;g%HUr$&06 zISZb(6)Q3%?;3r~*3~USIg=HcJhFtHhIV(siOwV&QkQe#J%H9&E21!C*d@ln3E@J* zVqRO^<)V^ky-R|%{(9`l-(JXq9J)1r$`uQ8a}$vr9E^nNiI*thK8=&UZ0dsFN_eSl z(q~lnD?EymWLsNa3|1{CRPW60>DSkY9YQ;$4o3W7Ms&@&lv9eH!tk~N&dhqX&>K@} zi1g~GqglxkZ5pEFkllJ)Ta1I^c&Bt6#r(QLQ02yHTaJB~- zCcE=5tmi`UA>@P=1LBfBiqk)HB4t8D?02;9eXj~kVPwv?m{5&!&TFYhu>3=_ zsGmYZ^mo*-j69-42y&Jj0cBLLEulNRZ9vXE)8~mt9C#;tZs;=#M=1*hebkS;7(aGf zcs7zH(I8Eui9UU4L--))yy`&d&$In&VA2?DAEss4LAPCLd>-$i?lpXvn!gu^JJ$(DoUlc6wE98VLZ*z`QGQov5l4Fm_h?V-;mHLYDVOwKz7>e4+%AzeO>P6v}ndPW| zM>m#6Tnp7K?0mbK=>gV}=@k*0Mr_PVAgGMu$j+pWxzq4MAa&jpCDU&-5eH27Iz>m^ zax1?*HhG%pJ((tkR(V(O(L%7v7L%!_X->IjS3H5kuXQT2!ow(;%FDE>16&3r){!ex zhf==oJ!}YU89C9@mfDq!P3S4yx$aGB?rbtVH?sHpg?J5C->!_FHM%Hl3#D4eplxzQ zRA+<@LD%LKSkTk2NyWCg7u=$%F#;SIL44~S_OGR}JqX}X+=bc@swpiClB`Zbz|f!4 z7Ysah7OkR8liXfI`}IIwtEoL}(URrGe;IM8%{>b1SsqXh)~w}P>yiFRaE>}rEnNkT z!HXZUtxUp1NmFm)Dm@-{FI^aRQqpSkz}ZSyKR%Y}YHNzBk)ZIp} zMtS=aMvkgWKm9&oTcU0?S|L~CDqA+sHpOxwnswF-fEG)cXCzUR?ps@tZa$=O)=L+5 zf%m58cq8g_o}3?Bhh+c!w4(7AjxwQ3>WnVi<{{38g7yFboo>q|+7qs<$8CPXUFAN< zG&}BHbbyQ5n|qqSr?U~GY{@GJ{(Jny{bMaOG{|IkUj7tj^9pa9|FB_<+KHLxSxR;@ zHpS$4V)PP+tx}22fWx(Ku9y+}Ap;VZqD0AZW4gCDTPCG=zgJmF{|x;(rvdM|2|9a}cex6xrMkERnkE;}jvU-kmzd%_J50$M`lIPCKf+^*zL=@LW`1SaEc%=m zQ+lT06Gw+wVwvQ9fZ~#qd430v2HndFsBa9WjD0P}K(rZYdAt^5WQIvb%D^Q|pkVE^ zte$&#~zmULFACGfS#g=2OLOnIf2Of-k!(BIHjs77nr!5Q1*I9 z1%?=~#Oss!rV~?-6Gm~BWJiA4mJ5TY&iPm_$)H1_rTltuU1F3I(qTQ^U$S>%$l z)Wx1}R?ij0idp@8w-p!Oz{&*W;v*IA;JFHA9%nUvVDy7Q8woheC#|8QuDZb-L_5@R zOqHwrh|mVL9b=+$nJxM`3eE{O$sCt$UK^2@L$R(r^-_+z?lOo+me-VW=Zw z-Bn>$4ovfWd%SPY`ab-u9{INc*k2h+yH%toDHIyqQ zO68=u`N}RIIs7lsn1D){)~%>ByF<>i@qFb<-axvu(Z+6t7v<^z&gm9McRB~BIaDn$ z#xSGT!rzgad8o>~kyj#h1?7g96tOcCJniQ+*#=b7wPio>|6a1Z?_(TS{)KrPe}(8j z!#&A=k(&Pj^F;r)CI=Z{LVu>uj!_W1q4b`N1}E(i%;BWjbEcnD=mv$FL$l?zS6bW!{$7j1GR5ocn94P2u{ z70tAAcpqtQo<@cXw~@i-@6B23;317|l~S>CB?hR5qJ%J3EFgyBdJd^fHZu7AzHF(BQ!tyAz^L0`X z23S4Fe{2X$W0$zu9gm%rg~A>ijaE#GlYlrF9$ds^QtaszE#4M(OLVP2O-;XdT(XIC zatwzF*)1c+t~c{L=fMG8Z=k5lv>U0;C{caN1NItnuSMp)6G3mbahu>E#sj&oy94KC zpH}8oEw{G@N3pvHhp{^-YaZeH;K+T_1AUv;IKD<=mv^&Ueegrb!yf`4VlRl$M?wsl zZyFol(2|_QM`e_2lYSABpKR{{NlxlDSYQNkS;J66aT#MSiTx~;tUmvs-b*CrR4w=f z8+0;*th6kfZ3|5!Icx3RV11sp=?`0Jy3Fs0N4GZQMN=8HmT6%x9@{Dza)k}UwL6JT zHRDh;%!XwXr6yuuy`4;Xsn0zlR$k%r%9abS1;_v?`HX_hI|+EibVnlyE@3aL5vhQq zlIG?tN^w@0(v9M*&L+{_+RQZw=o|&BRPGB>e5=ys7H`nc8nx)|-g;s7mRc7hg{GJC zAe^vCIJhajmm7C6g! zL&!WAQ~5d_5)00?w_*|*H>3$loHrvFbitw#WvLB!JASO?#5Ig5$Ys10n>e4|3d;tS zELJ0|R4n3Az(Fl3-r^QiV_C;)lQ1_CW{5bKS15U|E9?ZgLec@%kXr84>5jV2a5v=w z?pB1GPdxD$IQL4)G||B_lI+A=08MUFFR4MxfGOu07vfIm+j=z9tp~5i_6jb`tR>qV z$#`=BQ*jpCjm$F0+F)L%xRlnS%#&gro6PiRfu^l!EVan|r3y}AHJQOORGx4~ z&<)3=K-tx518DZyp%|!EqpU!+X3Et7n2AaC5(AtrkW>_57i}$eqs$rupubg0a1+WO zGHZKLN2L0D;ab%{_S1Plm|hx8R?O14*w*f&2&bB050n!R2by zw!@XOQx$SqZ5I<(Qu$V6g>o#A!JVwErWv#(Pjx=KeS0@hxr4?13zj#oWwPS(7Ro|v z>Mp@Kmxo79q|}!5qtX2-O@U&&@6s~!I&)1WQIl?lTnh6UdKT_1R640S4~f=_xoN3- zI+O)$R@RjV$F=>Ti7BlnG1-cFKCC(t|Qjm{SalS~V-tX#+2ekRhwmN zZr`8{QF6y~Z!D|{=1*2D-JUa<(1Z=;!Ei!KiRNH?o{p5o3crFF=_pX9O-YyJchr$~ zRC`+G+8kx~fD2k*ZIiiIGR<8r&M@3H?%JVOfE>)})7ScOd&?OjgAGT@WVNSCZ8N(p zuQG~76GE3%(%h1*vUXg$vH{ua0b`sQ4f0*y=u~lgyb^!#CcPJa2mkSEHGLsnO^kb$ zru5_l#nu=Y{rSMWiYx?nO{8I!gH+?wEj~UM?IrG}E|bRIBUM>UlY<`T1EHpRr36vv zBi&dG8oxS|J$!zoaq{+JpJy+O^W(nt*|#g32bd&K^w-t>!Vu9N!k9eA8r!Xc{utY> zg9aZ(D2E0gL#W0MdjwES-7~Wa8iubPrd?8-$C4BP?*wok&O8+ykOx{P=Izx+G~hM8 z*9?BYz!T8~dzcZr#ux8kS7u7r@A#DogBH8km8Ry4slyie^n|GrTbO|cLhpqgMdsjX zJ_LdmM#I&4LqqsOUIXK8gW;V0B(7^$y#h3h>J0k^WJfAMeYek%Y-Dcb_+0zPJez!GM zAmJ1u;*rK=FNM0Nf}Y!!P9c4)HIkMnq^b;JFd!S3?_Qi2G#LIQ)TF|iHl~WKK6JmK zbv7rPE6VkYr_%_BT}CK8h=?%pk@3cz(UrZ{@h40%XgThP*-Oeo`T0eq9 zA8BnWZKzCy5e&&_GEsU4*;_k}(8l_&al5K-V*BFM=O~;MgRkYsOs%9eOY6s6AtE*<7GQAR2ulC3RAJrG_P1iQK5Z~&B z&f8X<>yJV6)oDGIlS$Y*D^Rj(cszTy5c81a5IwBr`BtnC6_e`ArI8CaTX_%rx7;cn zR-0?J_LFg*?(#n~G8cXut(1nVF0Oka$A$1FGcERU<^ggx;p@CZc?3UB41RY+wLS`LWFNSs~YP zuw1@DNN3lTd|jDL7gjBsd9}wIw}4xT2+8dBQzI00m<@?c2L%>}QLfK5%r!a-iII`p zX@`VEUH)uj^$;7jVUYdADQ2k*!1O3WdfgF?OMtUXNpQ1}QINamBTKDuv19^{$`8A1 zeq%q*O0mi@(%sZU>Xdb0Ru96CFqk9-L3pzLVsMQ`Xpa~N6CR{9Rm2)A|CI21L(%GW zh&)Y$BNHa=FD+=mBw3{qTgw)j0b!Eahs!rZnpu)z!!E$*eXE~##yaXz`KE5(nQM`s zD!$vW9XH)iMxu9R>r$VlLk9oIR%HxpUiW=BK@4U)|1WNQ=mz9a z^!KkO=>GaJ!GBXm{KJj^;kh-MkUlEQ%lza`-G&}C5y1>La1sR6hT=d*NeCnuK%_LV zOXt$}iP6(YJKc9j-Fxq~*ItVUqljQ8?oaysB-EYtFQp9oxZ|5m0^Hq(qV!S+hq#g( z?|i*H2MIr^Kxgz+3vIljQ*Feejy6S4v~jKEPTF~Qhq!(ms5>NGtRgO5vfPPc4Z^AM zTj!`5xEreIN)vaNxa|q6qWdg>+T`Ol0Uz)ckXBXEGvPNEL3R8hB3=C5`@=SYgAju1 z!)UBr{2~=~xa{b8>x2@C7weRAEuatC)3pkRhT#pMPTpSbA|tan%U7NGMvzmF?c!V8 z=pEWxbdXbTAGtWTyI?Fml%lEr-^AE}w#l(<7OIw;ctw}imYax&vR4UYNJZK6P7ZOd zP87XfhnUHxCUHhM@b*NbTi#(-8|wcv%3BGNs#zRCVV(W?1Qj6^PPQa<{yaBwZ`+<`w|;rqUY_C z&AeyKwwf*q#OW-F()lir=T^<^wjK65Lif$puuU5+tk$;e_EJ;Lu+pH>=-8=PDhkBg z8cWt%@$Sc#C6F$Vd+0507;{OOyT7Hs%nKS88q-W!$f~9*WGBpHGgNp}=C*7!RiZ5s zn1L_DbKF@B8kwhDiLKRB@lsXVVLK|ph=w%_`#owlf@s@V(pa`GY$8h%;-#h@TsO|Y8V=n@*!Rog7<7Cid%apR|x zOjhHCyfbIt%+*PCveTEcuiDi%Wx;O;+K=W?OFUV%)%~6;gl?<0%)?snDDqIvkHF{ zyI02)+lI9ov42^hL>ZRrh*HhjF9B$A@=H94iaBESBF=eC_KT$8A@uB^6$~o?3Wm5t1OIaqF^~><2?4e3c&)@wKn9bD? zoeCs;H>b8DL^F&>Xw-xjZEUFFTv>JD^O#1E#)CMBaG4DX9bD(Wtc8Rzq}9soQ8`jf zeSnHOL}<+WVSKp4kkq&?SbETjq6yr@4%SAqOG=9E(3YeLG9dtV+8vmzq+6PFPk{L; z(&d++iu=^F%b+ea$i2UeTC{R*0Isk;vFK!no<;L+(`y`3&H-~VTdKROkdyowo1iqR zbVW(3`+(PQ2>TKY>N!jGmGo7oeoB8O|P_!Ic@ zZ^;3dnuXo;WJ?S+)%P>{Hcg!Jz#2SI(s&dY4QAy_vRlmOh)QHvs_7c&zkJCmJGVvV zX;Mtb>QE+xp`KyciG$Cn*0?AK%-a|=o!+7x&&yzHQOS>8=B*R=niSnta^Pxp1`=md z#;$pS$4WCT?mbiCYU?FcHGZ#)kHVJTTBt^%XE(Q};aaO=Zik0UgLcc0I(tUpt(>|& zcxB_|fxCF7>&~5eJ=Dpn&5Aj{A^cV^^}(7w#p;HG&Q)EaN~~EqrE1qKrMAc&WXIE;>@<&)5;gD2?={Xf@Mvn@OJKw=8Mgn z!JUFMwD+s==JpjhroT&d{$kQAy%+d`a*XxDEVxy3`NHzmITrE`o!;5ClXNPb4t*8P zzAivdr{j_v!=9!^?T3y?gzmqDWX6mkzhIzJ-3S{T5bcCFMr&RPDryMcdwbBuZbsgN zGrp@^i?rcfN7v0NKGzDPGE#4yszxu=I_`MI%Z|10nFjU-UjQXXA?k8Pk|OE<(?ae) zE%vG#eZAlj*E7_3dx#Zz4kMLj>H^;}33UAankJiDy5ZvEhrjr`!9eMD8COp}U*hP+ zF}KIYx@pkccIgyxFm#LNw~G&`;o&5)2`5aogs`1~7cMZQ7zj!%L4E`2yzlQN6REX20&O<9 zKV6fyr)TScJPPzNTC2gL+0x#=u>(({{D7j)c-%tvqls3#Y?Z1m zV5WUE)zdJ{$p>yX;^P!UcXP?UD~YM;IRa#Rs5~l+*$&nO(;Ers`G=0D!twR(0GF@c zHl9E5DQI}Oz74n zfKP>&$q0($T4y$6w(p=ERAFh+>n%iaeRA%!T%<^+pg?M)@ucY<&59$x9M#n+V&>}=nO9wCV{O~lg&v#+jcUj(tQ z`0u1YH)-`U$15a{pBkGyPL0THv1P|4e@pf@3IBZS4dVJPo#H>pWq%Lr0YS-SeWash z8R7=jb28KPMI|_lo#GEO|5B?N_e``H*23{~a!AmUJ+fb4HX-%QI@lSEUxKlGV7z7Q zSKw@-TR>@1RL%w{x}dW#k1NgW+q4yt2Xf1J62Bx*O^WG8OJ|FqI4&@d3_o8Id@*)4 zYrk=>@!wv~mh7YWv*bZhxqSmFh2Xq)o=m;%n$I?GSz49l1$xRpPu_^N(vZ>*>Z<04 z2+rP70oM=NDysd!@fQdM2OcyT?3T^Eb@lIC-UG=Bw{BjQ&P`KCv$AcJ;?`vdZ4){d z&gkoUK{$!$$K`3*O-jyM1~p-7T*qb)Ys>Myt^;#1&a%O@x8A+E>! zY8=eD`ZG)LVagDLBeHg>=atOG?Kr%h4B%E6m@J^C+U|y)XX@f z8oyJDW|9g=<#f<{JRr{y#~euMnv)`7j=%cHWLc}ngjq~7k**6%4u>Px&W%4D94(r* z+akunK}O0DC2A%Xo9jyF;DobX?!1I(7%}@7F>i%&nk*LMO)bMGg2N+1iqtg+r(70q zF5{Msgsm5GS7DT`kBsjMvOrkx&|EU!{{~gL4d2MWrAT=KBQ-^zQCUq{5PD1orxlIL zq;CvlWx#f1NWvh`hg011I%?T_s!e38l*lWVt|~z-PO4~~1g)SrJ|>*tXh=QfXT)%( z+ex+inPvD&O4Ur;JGz>$sUOnWdpSLcm1X%aQDw4{dB!cnj`^muI$CJ2%p&-kULVCE z>$eMR36kN$wCPR+OFDM3-U(VOrp9k3)lI&YVFqd;Kpz~K)@Fa&FRw}L(SoD z9B4a+hQzZT-BnVltst&=kq6Y(f^S4hIGNKYBgMxGJ^;2yrO}P3;r)(-I-CZ)26Y6? z&rzHI_1GCvGkgy-t1E;r^3Le30|%$ebDRu2+gdLG)r=A~Qz`}~&L@aGJ{}vVs_GE* zVUjFnzHiXfKQbpv&bR&}l2bzIjAooB)=-XNcYmrGmBh(&iu@o!^hn0^#}m2yZZUK8 zufVm7Gq0y`Mj;9b>`c?&PZkU0j4>IL=UL&-Lp3j&47B5pAW4JceG{!XCA)kT<%2nqCxj<)uy6XR_uws~>_MEKPOpAQ!H zkn>FKh)<9DwwS*|Y(q?$^N!6(51O0 z^JM~Ax{AI1Oj$fs-S5d4T7Z_i1?{%0SsIuQ&r8#(JA=2iLcTN+?>wOL532%&dMYkT z*T5xepC+V6zxhS@vNbMoi|i)=rpli@R9~P!39tWbSSb904ekv7D#quKbgFEMTb48P zuq(VJ+&L8aWU(_FCD$3^uD!YM%O^K(dvy~Wm2hUuh6bD|#(I39Xt>N1Y{ZqXL`Fg6 zKQ?T2htHN!(Bx;tV2bfTtIj7e)liN-29s1kew>v(D^@)#v;}C4-G=7x#;-dM4yRWm zyY`cS21ulzMK{PoaQ6xChEZ}o_#}X-o}<&0)$1#3we?+QeLt;aVCjeA)hn!}UaKt< zat1fHEx13y-rXNMvpUUmCVzocPmN~-Y4(YJvQ#db)4|%B!rBsgAe+*yor~}FrNH08 z3V!97S}D7d$zbSD{$z;@IYMxM6aHdypIuS*pr_U6;#Y!_?0i|&yU*@16l z*dcMqDQgfNBf}?quiu4e>H)yTVfsp#f+Du0@=Kc41QockXkCkvu>FBd6Q+@FL!(Yx z2`YuX#eMEiLEDhp+9uFqME_E^faV&~9qjBHJkIp~%$x^bN=N)K@kvSVEMdDuzA0sn z88CBG?`RX1@#hQNd`o^V{37)!w|nA)QfiYBE^m=yQKv-fQF+UCMcuEe1d4BH7$?>b zJl-r9@0^Ie=)guO1vOd=i$_4sz>y3x^R7n4ED!5oXL3@5**h(xr%Hv)_gILarO46q+MaDOF%ChaymKoI6JU5Pg;7#2n9-18|S1;AK+ zgsn6;k6-%!QD>D?cFy}8F;r@z8H9xN1jsOBw2vQONVqBVEbkiNUqgw~*!^##ht>w0 zUOykwH=$LwX2j&nLy=@{hr)2O&-wm-NyjW7n~Zs9UlH;P7iP3 zI}S(r0YFVYacnKH(+{*)Tbw)@;6>%=&Th=+Z6NHo_tR|JCI8TJiXv2N7ei7M^Q+RM z?9o`meH$5Yi;@9XaNR#jIK^&{N|DYNNbtdb)XW1Lv2k{E>;?F`#Pq|&_;gm~&~Zc9 zf+6ZE%{x4|{YdtE?a^gKyzr}dA>OxQv+pq|@IXL%WS0CiX!V zm$fCePA%lU{%pTKD7|5NJHeXg=I0jL@$tOF@K*MI$)f?om)D63K*M|r`gb9edD1~Y zc|w7N)Y%do7=0{RC|AziW7#am$)9jciRJ?IWl9PE{G3U+$%FcyKs_0Cgq`=K3@ttV z9g;M!3z~f_?P%y3-ph%vBMeS@p7P&Ea8M@97+%XEj*(1E6vHj==d zjsoviB>j^$_^OI_DEPvFkVo(BGRo%cJeD){6Uckei=~1}>sp299|IRjhXe)%?uP0I zF5+>?0#Ye}T^Y$u_rc4=lPcq4K^D(TZG-w30-YiEM=dcK+4#o*>lJ8&JLi+3UcpZk z!^?95S^C0ja^jwP`|{<+3cBVog$(mRdQmadS+Vh~z zS@|P}=|z3P6uS+&@QsMp0no9Od&27O&14zHXGAOEy zh~OKpymK5C%;LLb467@KgIiVwYbYd6wFxI{0-~MOGfTq$nBTB!{SrWmL9Hs}C&l&l#m?s*{tA?BHS4mVKHAVMqm63H<|c5n0~k)-kbg zXidai&9ZUy0~WFYYKT;oe~rytRk?)r8bptITsWj(@HLI;@=v5|XUnSls7$uaxFRL+ zRVMGuL3w}NbV1`^=Pw*0?>bm8+xfeY(1PikW*PB>>Tq(FR`91N0c2&>lL2sZo5=VD zQY{>7dh_TX98L2)n{2OV=T10~*YzX27i2Q7W86M4$?gZIXZaBq#sA*{PH8){|GUi;oM>e?ua7eF4WFuFYZSG| zze?srg|5Ti8Og{O zeFxuw9!U+zhyk?@w zjsA6(oKD=Ka;A>Ca)oPORxK+kxH#O@zhC!!XS4@=swnuMk>t+JmLmFiE^1aX3f<)D@`%K0FGK^gg1a1j>zi z2KhV>sjU7AX3F$SEqrXSC}fRx64GDoc%!u2Yag68Lw@w9v;xOONf@o)Lc|Uh3<21ctTYu-mFZuHk*+R{GjXHIGq3p)tFtQp%TYqD=j1&y)>@zxoxUJ!G@ zgI0XKmP6MNzw>nRxK$-Gbzs}dyfFzt>#5;f6oR27ql!%+{tr+(`(>%51|k`ML} zY4eE)Lxq|JMas(;JibNQds1bUB&r}ydMQXBY4x(^&fY_&LlQC)3hylc$~8&~|06-D z#T+%66rYbHX%^KuqJED_wuGB+=h`nWA!>1n0)3wZrBG3%`b^Ozv6__dNa@%V14|!D zQ?o$z5u0^8`giv%qE!BzZ!3j;BlDlJDk)h@9{nSQeEk!z9RGW) z${RSF3phEM*ce*>Xdp}585vj$|40=&S{S-GTiE?Op*vY&Lvr9}BO$XWy80IF+6@%n z5*2ueT_g@ofP#u5pxb7n*fv^Xtt7&?SRc{*2Ka-*!BuOpf}neHGCiHy$@Ka1^Dint z;DkmIL$-e)rj4o2WQV%Gy;Xg(_Bh#qeOsTM2f@KEe~4kJ8kNLQ+;(!j^bgJMcNhvklP5Z6I+9Fq@c&D~8Fb-4rmDT!MB5QC{Dsb;BharP*O;SF4& zc$wj-7Oep7#$WZN!1nznc@Vb<_Dn%ga-O#J(l=OGB`dy=Sy&$(5-n3zzu%d7E#^8`T@}V+5B;PP8J14#4cCPw-SQTdGa2gWL0*zKM z#DfSXs_iWOMt)0*+Y>Lkd=LlyoHjublNLefhKBv@JoC>P7N1_#> zv=mLWe96%EY;!ZGSQDbZWb#;tzqAGgx~uk+-$+2_8U`!ypbwXl z^2E-FkM1?lY@yt8=J3%QK+xaZ6ok=-y%=KXCD^0r!5vUneW>95PzCkOPO*t}p$;-> ze5j-BLT_;)cZQzR2CEsm@rU7GZfFtdp*a|g4wDr%8?2QkIGasRfDWT-Dvy*U{?IHT z*}wGnzdlSptl#ZF^sf)KT|BJs&kLG91^A6ls{CzFprZ6-Y!V0Xysh%9p%iMd7HLsS zN+^Un$tDV)T@i!v?3o0Fsx2qI(AX_$dDkBzQ@fRM%n zRXk6hb9Py#JXUs+7)w@eo;g%QQ95Yq!K_d=z{0dGS+pToEI6=Bo8+{k$7&Z zo4>PH(`ce8E-Ps&uv`NQ;U$%t;w~|@E3WVOCi~R4oj5wP?%<*1C%}Jq%a^q~T7u>K zML5AKfQDv6>PuT`{SrKHRAF+^&edg6+5R_#H?Lz3iGoWo#PCEd0DS;)2U({{X#zU^ zw_xv{4x7|t!S)>44J;KfA|DC?;uQ($l+5Vp7oeqf7{GBF9356nx|&B~gs+@N^gSdd zvb*>&W)|u#F{Z_b`f#GVtQ`pYv3#||N{xj1NgB<#=Odt6{eB%#9RLt5v zIi|0u70`#ai}9fJjKv7dE!9ZrOIX!3{$z_K5FBd-Kp-&e4(J$LD-)NMTp^_pB`RT; zftVVlK2g@+1Ahv2$D){@Y#cL#dUj9*&%#6 zd2m9{1NYp>)6=oAvqdCn5#cx{AJ%S8skUgMglu2*IAtd+z1>B&`MuEAS(D(<6X#Lj z?f4CFx$)M&$=7*>9v1ER4b6!SIz-m0e{o0BfkySREchp?WdVPpQCh!q$t>?rL!&Jg zd#heM;&~A}VEm8Dvy&P|J*eAV&w!&Nx6HFV&B8jJFVTmgLaswn!cx$&%JbTsloz!3 zMEz1d`k==`Ueub_JAy_&`!ogbwx27^ZXgFNAbx=g_I~5nO^r)}&myw~+yY*cJl4$I znNJ32M&K=0(2Dj_>@39`3=FX!v3nZHno_@q^!y}%(yw0PqOo=);6Y@&ylVe>nMOZ~ zd>j#QQSBn3oaWd;qy$&5(5H$Ayi)0haAYO6TH>FR?rhqHmNOO+(})NB zLI@B@v0)eq!ug`>G<@htRlp3n!EpU|n+G+AvXFrWSUsLMBfL*ZB`CRsIVHNTR&b?K zxBgsN0BjfB>UVcJ|x%=-zb%OV7lmZc& zxiupadZVF7)6QuhoY;;FK2b*qL0J-Rn-8!X4ZY$-ZSUXV5DFd7`T41c(#lAeLMoeT z4%g655v@7AqT!i@)Edt5JMbN(=Q-6{=L4iG8RA%}w;&pKmtWvI4?G9pVRp|RTw`g0 zD5c12B&A2&P6Ng~8WM2eIW=wxd?r7A*N+&!Be7PX3s|7~z=APxm=A?5 zt>xB4WG|*Td@VX{Rs)PV0|yK`oI3^xn(4c_j&vgxk_Y3o(-`_5o`V zRTghg6%l@(qodXN;dB#+OKJEEvhfcnc#BeO2|E(5df-!fKDZ!%9!^BJ_4)9P+9Dq5 zK1=(v?KmIp34r?z{NEWnLB3Px{XYwy-akun4F7xTRr2^zeYW{gcK9)>aJDdU5;w5@ zak=<+-PLH-|04pelTb%ULpuuuJC7DgyT@D|p{!V!0v3KpDnRjANN12q6SUR3mb9<- z>2r~IApQGhstZ!3*?5V z8#)hJ0TdZg0M-BK#nGFP>$i=qk82DO z7h;Ft!D5E15OgW)&%lej*?^1~2=*Z5$2VX>V{x8SC+{i10BbtUk9@I#Vi&hX)q
Q!LwySI{Bnv%Sm)yh{^sSVJ8&h_D-BJ_YZe5eCaAWU9b$O2c z$T|{vWVRtOL!xC0DTc(Qbe`ItNtt5hr<)VijD0{U;T#bUEp381_y`%ZIav?kuYG{iyYdEBPW=*xNSc;Rlt6~F4M`5G+VtOjc z*0qGzCb@gME5udTjJA-9O<&TWd~}ysBd(eVT1-H82-doyH9RST)|+Pb{o*;$j9Tjs zhU!IlsPsj8=(x3bAKJTopW3^6AKROHR^7wZ185wJGVhA~hEc|LP;k7NEz-@4p5o}F z`AD6naG3(n=NF9HTH81=F+Q|JOz$7wm9I<+#BSmB@o_cLt2GkW9|?7mM;r!JZp89l zbo!Hp8=n!XH1{GwaDU+k)pGp`C|cXkCU5%vcH)+v@0eK>%7gWxmuMu9YLlChA|_D@ zi#5zovN_!a-0?~pUV-Rj*1P)KwdU-LguR>YM&*Nen+ln8Q$?WFCJg%DY%K}2!!1FE zDv-A%Cbwo^p(lzac&_TZ-l#9kq`mhLcY3h9ZTUVCM(Ad&=EriQY5{jJv<5K&g|*Lk zgV%ILnf1%8V2B0E&;Sp4sYbYOvvMebLwYwzkRQ#F8GpTQq#uv=J`uaSJ34OWITeSGo6+-8Xw znCk*n{kdDEi)Hi&u^)~cs@iyCkFWB2SWZU|Uc%^43ZIZQ-vWNExCCtDWjqHs;;tWf$v{}0{p0Rvxkq``)*>+Akq%|Na zA`@~-Vfe|+(AIlqru+7Ceh4nsVmO9p9jc8}HX^W&ViBDXT+uXbT#R#idPn&L>+#b6 zflC-4C5-X;kUnR~L>PSLh*gvL68}RBsu#2l`s_9KjUWRhiqF`j)`y`2`YU(>3bdBj z?>iyjEhe-~$^I5!nn%B6Wh+I`FvLNvauve~eX<+Ipl&04 zT}};W&1a3%W?dJ2=N#0t?e+aK+%t}5q%jSLvp3jZ%?&F}nOOWr>+{GFIa%wO_2`et z=JzoRR~}iKuuR+azPI8;Gf9)z3kyA4EIOSl!sRR$DlW}0>&?GbgPojmjmnln;cTqCt=ADbE zZ8GAnoM+S1(5$i8^O4t`ue;vO4i}z0wz-QEIVe5_u03;}-!G1NyY8;h^}y;tzY}i5 zqQr#Ur3Fy8sSa$Q0ys+f`!`+>9WbvU_I`Sj;$4{S>O3?#inLHCrtLy~!s#WXV=oVP zeE93*Nc`PBi4q@%Ao$x4lw9vLHM!6mn3-b_cebF|n-2vt-zYVF_&sDE--J-P;2WHo z+@n2areE0o$LjvjlV2X7ZU@j+`{*8zq`JR3gKF#EW|#+{nMyo-a>nFFTg&vhyT=b} zDa8+v0(Dgx0yRL@ZXOYIlVSZ0|MFizy0VPW8;AfA5|pe!#j zX}Py^8fl5SyS4g1WSKKtnyP+_PoOwMMwu`(i@Z)diJp~U54*-miOchy7Z35eL>^M z4p<-aIxH4VUZgS783@H%M7P9hX>t{|RU7$n4T(brCG#h9e9p! z+o`i;EGGq3&pF;~5V~eBD}lC)>if$w%Vf}AFxGqO88|ApfHf&Bvu+xdG)@vuF}Yvk z)o;~k-%+0K0g+L`Wala!$=ZV|z$e%>f0%XoLib%)!R^RoS+{!#X?h-6uu zF&&KxORdZU&EwQFITIRLo(7TA3W}y6X{?Y%y2j0It!ekU#<)$qghZtpcS>L3uh`Uj z7GY;6f$9qKynP#oS3$$a{p^{D+0oJQ71`1?OAn_m8)UGZmj3l*ZI)`V-a>MKGGFG< z&^jg#Ok%(hhm>hSrZ5;Qga4u(?^i>GiW_j9%_7M>j(^|Om$#{k+^*ULnEgzW_1gCICtAD^WpC`A z{9&DXkG#01Xo)U$OC(L5Y$DQ|Q4C6CjUKk1UkPj$nXH##J{c8e#K|&{mA*;b$r0E4 zUNo0jthwA(c&N1l=PEe8Rw_8cEl|-eya9z&H3#n`B$t#+aJ03RFMzrV@gowbe8v(c zIFM60^0&lCFO10NU4w@|61xiZ4CVXeaKjd;d?sv52XM*lS8XiVjgWpRB;&U_C0g+`6B5V&w|O6B*_q zsATxL!M}+$He)1eOWECce#eS@2n^xhlB4<_Nn?yCVEQWDs(r`|@2GqLe<#(|&P0U? z$7V5IgpWf09uIf_RazRwC?qEqRaHyL?iiS05UiGesJy%^>-C{{ypTBI&B0-iUYhk> zIk<5xpsuV@g|z(AZD+C-;A!fTG=df1=<%nxy(a(IS+U{ME4ZbDEBtcD_3V=icT6*_ z)>|J?>&6%nvHhZERBtjK+s4xnut*@>GAmA5m*OTp$!^CHTr}vM4n(X1Q*;{e-Rd2BCF-u@1ZGm z!S8hJ6L=Gl4T_SDa7Xx|-{4mxveJg=ctf`BJ*fy!yF6Dz&?w(Q_6B}WQVtNI!BVBC zKfX<>7vd6C96}XAQmF-Jd?1Q4eTfRB3q7hCh0f!(JkdWT5<{iAE#dKy*Jxq&3a1@~ z8C||Dn2mFNyrUV|<-)C^_y7@8c2Fz+2jrae9deBDu;U}tJ{^xAdxCD248(k;dCJ%o z`y3sADe>U%suxwwv~8A1+R$VB=Q?%U?4joI$um;aH+eCrBqpn- z%79D_7rb;R-;-9RTrwi9dPlg8&@tfWhhZ(Vx&1PQ+6(huX`;M9x~LrW~~#3{j0Bh2kDU$}@!fFQej4VGkJv?M4rU^x!RU zEwhu$!CA_iDjFjrJa`aocySDX16?~;+wgav;}Zut6Mg%C4>}8FL?8)Kgwc(Qlj{@#2Pt0?G`$h7P#M+qoXtlV@d}%c&OzO+QYKK`kyXaK{U(O^2DyIXCZlNQjt0^8~8JzNGrIxhj}}M z&~QZlbx%t;MJ(Vux;2tgNKGlAqphLq%pd}JG9uoVHUo?|hN{pLQ6Em%r*+7t^<);X zm~6=qChlNAVXNN*Sow->*4;}T;l;D1I-5T{Bif@4_}=>l`tK;qqDdt5zvisCKhMAH z#r}`)7VW?LZqfdmXQ%zo5bJ00{Xb9^YKrk0Nf|oIW*K@(=`o2Vndz}ZDyk{!u}PVx zzd--+_WC*U{~DH3{?GI64IB+@On&@9X>EUAo&L+G{L^dozaI4C3G#2wr~hseW@K&g zKWs{uHu-9Je!3;4pE>eBltKUXb^*hG8I&413)$J&{D4N%7PcloU6bn%jPxJyQL?g* z9g+YFFEDiE`8rW^laCNzQmi7CTnPfwyg3VDHRAl>h=In6jeaVOP@!-CP60j3+#vpL zEYmh_oP0{-gTe7Or`L6x)6w?77QVi~jD8lWN@3RHcm80iV%M1A!+Y6iHM)05iC64tb$X2lV_%Txk@0l^hZqi^%Z?#- zE;LE0uFx)R08_S-#(wC=dS&}vj6P4>5ZWjhthP=*Hht&TdLtKDR;rXEX4*z0h74FA zMCINqrh3Vq;s%3MC1YL`{WjIAPkVL#3rj^9Pj9Ss7>7duy!9H0vYF%>1jh)EPqvlr6h%R%CxDsk| z!BACz7E%j?bm=pH6Eaw{+suniuY7C9Ut~1cWfOX9KW9=H><&kQlinPV3h9R>3nJvK z4L9(DRM=x;R&d#a@oFY7mB|m8h4692U5eYfcw|QKwqRsshN(q^v$4$)HgPpAJDJ`I zkqjq(8Cd!K!+wCd=d@w%~e$=gdUgD&wj$LQ1r>-E=O@c ze+Z$x{>6(JA-fNVr)X;*)40Eym1TtUZI1Pwwx1hUi+G1Jlk~vCYeXMNYtr)1?qwyg zsX_e*$h?380O00ou?0R@7-Fc59o$UvyVs4cUbujHUA>sH!}L54>`e` zHUx#Q+Hn&Og#YVOuo*niy*GU3rH;%f``nk#NN5-xrZ34NeH$l`4@t);4(+0|Z#I>Y z)~Kzs#exIAaf--65L0UHT_SvV8O2WYeD>Mq^Y6L!Xu8%vnpofG@w!}R7M28?i1*T&zp3X4^OMCY6(Dg<-! zXmcGQrRgHXGYre7GfTJ)rhl|rs%abKT_Nt24_Q``XH{88NVPW+`x4ZdrMuO0iZ0g` z%p}y};~T5gbb9SeL8BSc`SO#ixC$@QhXxZ=B}L`tP}&k?1oSPS=4%{UOHe0<_XWln zwbl5cn(j-qK`)vGHY5B5C|QZd5)W7c@{bNVXqJ!!n$^ufc?N9C-BF2QK1(kv++h!>$QbAjq)_b$$PcJdV+F7hz0Hu@ zqj+}m0qn{t^tD3DfBb~0B36|Q`bs*xs|$i^G4uNUEBl4g;op-;Wl~iThgga?+dL7s zUP(8lMO?g{GcYpDS{NM!UA8Hco?#}eNEioRBHy4`mq!Pd-9@-97|k$hpEX>xoX+dY zDr$wfm^P&}Wu{!%?)U_(%Mn79$(ywvu*kJ9r4u|MyYLI_67U7%6Gd_vb##Nerf@>& z8W11z$$~xEZt$dPG}+*IZky+os5Ju2eRi;1=rUEeIn>t-AzC_IGM-IXWK3^6QNU+2pe=MBn4I*R@A%-iLDCOHTE-O^wo$sL_h{dcPl=^muAQb`_BRm};=cy{qSkui;`WSsj9%c^+bIDQ z0`_?KX0<-=o!t{u(Ln)v>%VGL z0pC=GB7*AQ?N7N{ut*a%MH-tdtNmNC+Yf$|KS)BW(gQJ*z$d{+{j?(e&hgTy^2|AR9vx1Xre2fagGv0YXWqtNkg*v%40v?BJBt|f9wX5 z{QTlCM}b-0{mV?IG>TW_BdviUKhtosrBqdfq&Frdz>cF~yK{P@(w{Vr7z2qKFwLhc zQuogKO@~YwyS9%+d-zD7mJG~@?EFJLSn!a&mhE5$_4xBl&6QHMzL?CdzEnC~C3$X@ zvY!{_GR06ep5;<#cKCSJ%srxX=+pn?ywDwtJ2{TV;0DKBO2t++B(tIO4)Wh`rD13P z4fE$#%zkd=UzOB74gi=-*CuID&Z3zI^-`4U^S?dHxK8fP*;fE|a(KYMgMUo`THIS1f!*6dOI2 zFjC3O=-AL`6=9pp;`CYPTdVX z8(*?V&%QoipuH0>WKlL8A*zTKckD!paN@~hh zmXzm~qZhMGVdQGd=AG8&20HW0RGV8X{$9LldFZYm zE?}`Q3i?xJRz43S?VFMmqRyvWaS#(~Lempg9nTM$EFDP(Gzx#$r)W&lpFKqcAoJh-AxEw$-bjW>`_+gEi z2w`99#UbFZGiQjS8kj~@PGqpsPX`T{YOj`CaEqTFag;$jY z8_{Wzz>HXx&G*Dx<5skhpETxIdhKH?DtY@b9l8$l?UkM#J-Snmts7bd7xayKTFJ(u zyAT&@6cAYcs{PBfpqZa%sxhJ5nSZBPji?Zlf&}#L?t)vC4X5VLp%~fz2Sx<*oN<7` z?ge=k<=X7r<~F7Tvp9#HB{!mA!QWBOf%EiSJ6KIF8QZNjg&x~-%e*tflL(ji_S^sO ztmib1rp09uon}RcsFi#k)oLs@$?vs(i>5k3YN%$T(5Or(TZ5JW9mA6mIMD08=749$ z!d+l*iu{Il7^Yu}H;lgw=En1sJpCKPSqTCHy4(f&NPelr31^*l%KHq^QE>z>Ks_bH zjbD?({~8Din7IvZeJ>8Ey=e;I?thpzD=zE5UHeO|neioJwG;IyLk?xOz(yO&0DTU~ z^#)xcs|s>Flgmp;SmYJ4g(|HMu3v7#;c*Aa8iF#UZo7CvDq4>8#qLJ|YdZ!AsH%^_7N1IQjCro

K7UpUK$>l@ zw`1S}(D?mUXu_C{wupRS-jiX~w=Uqqhf|Vb3Cm9L=T+w91Cu^ z*&Ty%sN?x*h~mJc4g~k{xD4ZmF%FXZNC;oVDwLZ_WvrnzY|{v8hc1nmx4^}Z;yriXsAf+Lp+OFLbR!&Ox?xABwl zu8w&|5pCxmu#$?Cv2_-Vghl2LZ6m7}VLEfR5o2Ou$x02uA-%QB2$c(c1rH3R9hesc zfpn#oqpbKuVsdfV#cv@5pV4^f_!WS+F>SV6N0JQ9E!T90EX((_{bSSFv9ld%I0&}9 zH&Jd4MEX1e0iqDtq~h?DBrxQX1iI0lIs<|kB$Yrh&cpeK0-^K%=FBsCBT46@h#yi!AyDq1V(#V}^;{{V*@T4WJ&U-NTq43w=|K>z8%pr_nC>%C(Wa_l78Ufib$r8Od)IIN=u>417 z`Hl{9A$mI5A(;+-Q&$F&h-@;NR>Z<2U;Y21>>Z;s@0V@SbkMQQj%_;~+qTuQ?c|AV zcWm3XZQHhP&R%QWarS%mJ!9R^&!_)*s(v+VR@I#QrAT}`17Y+l<`b-nvmDNW`De%y zrwTZ9EJrj1AFA>B`1jYDow}~*dfPs}IZMO3=a{Fy#IOILc8F0;JS4x(k-NSpbN@qM z`@aE_e}5{!$v3+qVs7u?sOV(y@1Os*Fgu`fCW9=G@F_#VQ%xf$hj0~wnnP0$hFI+@ zkQj~v#V>xn)u??YutKsX>pxKCl^p!C-o?+9;!Nug^ z{rP!|+KsP5%uF;ZCa5F;O^9TGac=M|=V z_H(PfkV1rz4jl?gJ(ArXMyWT4y(86d3`$iI4^l9`vLdZkzpznSd5Ikfrs8qcSy&>z zTIZgWZGXw0n9ibQxYWE@gI0(3#KA-dAdPcsL_|hg2@~C!VZDM}5;v_Nykfq!*@*Zf zE_wVgx82GMDryKO{U{D>vSzSc%B~|cjDQrt5BN=Ugpsf8H8f1lR4SGo#hCuXPL;QQ z#~b?C4MoepT3X`qdW2dNn& zo8)K}%Lpu>0tQei+{>*VGErz|qjbK#9 zvtd8rcHplw%YyQCKR{kyo6fgg!)6tHUYT(L>B7er5)41iG`j$qe*kSh$fY!PehLcD zWeKZHn<492B34*JUQh=CY1R~jT9Jt=k=jCU2=SL&&y5QI2uAG2?L8qd2U(^AW#{(x zThSy=C#>k+QMo^7caQcpU?Qn}j-`s?1vXuzG#j8(A+RUAY})F@=r&F(8nI&HspAy4 z4>(M>hI9c7?DCW8rw6|23?qQMSq?*Vx?v30U%luBo)B-k2mkL)Ljk5xUha3pK>EEj z@(;tH|M@xkuN?gsz;*bygizwYR!6=(Xgcg^>WlGtRYCozY<rFX2E>kaZo)O<^J7a`MX8Pf`gBd4vrtD|qKn&B)C&wp0O-x*@-|m*0egT=-t@%dD zgP2D+#WPptnc;_ugD6%zN}Z+X4=c61XNLb7L1gWd8;NHrBXwJ7s0ce#lWnnFUMTR& z1_R9Fin4!d17d4jpKcfh?MKRxxQk$@)*hradH2$3)nyXep5Z;B z?yX+-Bd=TqO2!11?MDtG0n(*T^!CIiF@ZQymqq1wPM_X$Iu9-P=^}v7npvvPBu!d$ z7K?@CsA8H38+zjA@{;{kG)#AHME>Ix<711_iQ@WWMObXyVO)a&^qE1GqpP47Q|_AG zP`(AD&r!V^MXQ^e+*n5~Lp9!B+#y3#f8J^5!iC@3Y@P`;FoUH{G*pj*q7MVV)29+j z>BC`a|1@U_v%%o9VH_HsSnM`jZ-&CDvbiqDg)tQEnV>b%Ptm)T|1?TrpIl)Y$LnG_ zzKi5j2Fx^K^PG1=*?GhK;$(UCF-tM~^=Z*+Wp{FSuy7iHt9#4n(sUuHK??@v+6*|10Csdnyg9hAsC5_OrSL;jVkLlf zHXIPukLqbhs~-*oa^gqgvtpgTk_7GypwH><53riYYL*M=Q@F-yEPLqQ&1Sc zZB%w}T~RO|#jFjMWcKMZccxm-SL)s_ig?OC?y_~gLFj{n8D$J_Kw%{r0oB8?@dWzn zB528d-wUBQzrrSSLq?fR!K%59Zv9J4yCQhhDGwhptpA5O5U?Hjqt>8nOD zi{)0CI|&Gu%zunGI*XFZh(ix)q${jT8wnnzbBMPYVJc4HX*9d^mz|21$=R$J$(y7V zo0dxdbX3N#=F$zjstTf*t8vL)2*{XH!+<2IJ1VVFa67|{?LP&P41h$2i2;?N~RA30LV`BsUcj zfO9#Pg1$t}7zpv#&)8`mis3~o+P(DxOMgz-V*(?wWaxi?R=NhtW}<#^Z?(BhSwyar zG|A#Q7wh4OfK<|DAcl9THc-W4*>J4nTevsD%dkj`U~wSUCh15?_N@uMdF^Kw+{agk zJ`im^wDqj`Ev)W3k3stasP`88-M0ZBs7;B6{-tSm3>I@_e-QfT?7|n0D~0RRqDb^G zyHb=is;IwuQ&ITzL4KsP@Z`b$d%B0Wuhioo1CWttW8yhsER1ZUZzA{F*K=wmi-sb#Ju+j z-l@In^IKnb{bQG}Ps>+Vu_W#grNKNGto+yjA)?>0?~X`4I3T@5G1)RqGUZuP^NJCq&^HykuYtMDD8qq+l8RcZNJsvN(10{ zQ1$XcGt}QH-U^WU!-wRR1d--{B$%vY{JLWIV%P4-KQuxxDeJaF#{eu&&r!3Qu{w}0f--8^H|KwE>)ORrcR+2Qf zb})DRcH>k0zWK8@{RX}NYvTF;E~phK{+F;MkIP$)T$93Ba2R2TvKc>`D??#mv9wg$ zd~|-`Qx5LwwsZ2hb*Rt4S9dsF%Cny5<1fscy~)d;0m2r$f=83<->c~!GNyb!U)PA; zq^!`@@)UaG)Ew(9V?5ZBq#c%dCWZrplmuM`o~TyHjAIMh0*#1{B>K4po-dx$Tk-Cq z=WZDkP5x2W&Os`N8KiYHRH#UY*n|nvd(U>yO=MFI-2BEp?x@=N<~CbLJBf6P)}vLS?xJXYJ2^<3KJUdrwKnJnTp{ zjIi|R=L7rn9b*D#Xxr4*R<3T5AuOS+#U8hNlfo&^9JO{VbH!v9^JbK=TCGR-5EWR@ zN8T-_I|&@A}(hKeL4_*eb!1G8p~&_Im8|wc>Cdir+gg90n1dw?QaXcx6Op_W1r=axRw>4;rM*UOpT#Eb9xU1IiWo@h?|5uP zka>-XW0Ikp@dIe;MN8B01a7+5V@h3WN{J=HJ*pe0uwQ3S&MyWFni47X32Q7SyCTNQ z+sR!_9IZa5!>f&V$`q!%H8ci!a|RMx5}5MA_kr+bhtQy{-^)(hCVa@I!^TV4RBi zAFa!Nsi3y37I5EK;0cqu|9MRj<^r&h1lF}u0KpKQD^5Y+LvFEwM zLU@@v4_Na#Axy6tn3P%sD^5P#<7F;sd$f4a7LBMk zGU^RZHBcxSA%kCx*eH&wgA?Qwazm8>9SCSz_!;MqY-QX<1@p$*T8lc?@`ikEqJ>#w zcG``^CoFMAhdEXT9qt47g0IZkaU)4R7wkGs^Ax}usqJ5HfDYAV$!=6?>J6+Ha1I<5 z|6=9soU4>E))tW$<#>F ziZ$6>KJf0bPfbx_)7-}tMINlc=}|H+$uX)mhC6-Hz+XZxsKd^b?RFB6et}O#+>Wmw9Ec9) z{q}XFWp{3@qmyK*Jvzpyqv57LIR;hPXKsrh{G?&dRjF%Zt5&m20Ll?OyfUYC3WRn{cgQ?^V~UAv+5 z&_m#&nIwffgX1*Z2#5^Kl4DbE#NrD&Hi4|7SPqZ}(>_+JMz=s|k77aEL}<=0Zfb)a z%F(*L3zCA<=xO)2U3B|pcTqDbBoFp>QyAEU(jMu8(jLA61-H!ucI804+B!$E^cQQa z)_ERrW3g!B9iLb3nn3dlkvD7KsY?sRvls3QC0qPi>o<)GHx%4Xb$5a3GBTJ(k@`e@ z$RUa^%S15^1oLEmA=sayrP5;9qtf!Z1*?e$ORVPsXpL{jL<6E)0sj&swP3}NPmR%FM?O>SQgN5XfHE< zo(4#Cv11(%Nnw_{_Ro}r6=gKd{k?NebJ~<~Kv0r(r0qe4n3LFx$5%x(BKvrz$m?LG zjLIc;hbj0FMdb9aH9Lpsof#yG$(0sG2%RL;d(n>;#jb!R_+dad+K;Ccw!|RY?uS(a zj~?=&M!4C(5LnlH6k%aYvz@7?xRa^2gml%vn&eKl$R_lJ+e|xsNfXzr#xuh(>`}9g zLHSyiFwK^-p!;p$yt7$F|3*IfO3Mlu9e>Dpx8O`37?fA`cj`C0B-m9uRhJjs^mRp# zWB;Aj6|G^1V6`jg7#7V9UFvnB4((nIwG?k%c7h`?0tS8J3Bn0t#pb#SA}N-|45$-j z$R>%7cc2ebAClXc(&0UtHX<>pd)akR3Kx_cK+n<}FhzmTx!8e9^u2e4%x{>T6pQ`6 zO182bh$-W5A3^wos0SV_TgPmF4WUP-+D25KjbC{y_6W_9I2_vNKwU(^qSdn&>^=*t z&uvp*@c8#2*paD!ZMCi3;K{Na;I4Q35zw$YrW5U@Kk~)&rw;G?d7Q&c9|x<Hg|CNMsxovmfth*|E*GHezPTWa^Hd^F4!B3sF;)? z(NaPyAhocu1jUe(!5Cy|dh|W2=!@fNmuNOzxi^tE_jAtzNJ0JR-avc_H|ve#KO}#S z#a(8secu|^Tx553d4r@3#6^MHbH)vmiBpn0X^29xEv!Vuh1n(Sr5I0V&`jA2;WS|Y zbf0e}X|)wA-Pf5gBZ>r4YX3Mav1kKY(ulAJ0Q*jB)YhviHK)w!TJsi3^dMa$L@^{` z_De`fF4;M87vM3Ph9SzCoCi$#Fsd38u!^0#*sPful^p5oI(xGU?yeYjn;Hq1!wzFk zG&2w}W3`AX4bxoVm03y>ts{KaDf!}b&7$(P4KAMP=vK5?1In^-YYNtx1f#}+2QK@h zeSeAI@E6Z8a?)>sZ`fbq9_snl6LCu6g>o)rO;ijp3|$vig+4t} zylEo7$SEW<_U+qgVcaVhk+4k+C9THI5V10qV*dOV6pPtAI$)QN{!JRBKh-D zk2^{j@bZ}yqW?<#VVuI_27*cI-V~sJiqQv&m07+10XF+#ZnIJdr8t`9s_EE;T2V;B z4UnQUH9EdX%zwh-5&wflY#ve!IWt0UE-My3?L#^Bh%kcgP1q{&26eXLn zTkjJ*w+(|_>Pq0v8{%nX$QZbf)tbJaLY$03;MO=Ic-uqYUmUCuXD>J>o6BCRF=xa% z3R4SK9#t1!K4I_d>tZgE>&+kZ?Q}1qo4&h%U$GfY058s%*=!kac{0Z+4Hwm!)pFLR zJ+5*OpgWUrm0FPI2ib4NPJ+Sk07j(`diti^i#kh&f}i>P4~|d?RFb#!JN)~D@)beox}bw?4VCf^y*`2{4`-@%SFTry2h z>9VBc9#JxEs1+0i2^LR@B1J`B9Ac=#FW=(?2;5;#U$0E0UNag_!jY$&2diQk_n)bT zl5Me_SUvqUjwCqmVcyb`igygB_4YUB*m$h5oeKv3uIF0sk}~es!{D>4r%PC*F~FN3owq5e0|YeUTSG#Vq%&Gk7uwW z0lDo#_wvflqHeRm*}l?}o;EILszBt|EW*zNPmq#?4A+&i0xx^?9obLyY4xx=Y9&^G;xYXYPxG)DOpPg!i_Ccl#3L}6xAAZzNhPK1XaC_~ z!A|mlo?Be*8Nn=a+FhgpOj@G7yYs(Qk(8&|h@_>w8Y^r&5nCqe0V60rRz?b5%J;GYeBqSAjo|K692GxD4` zRZyM2FdI+-jK2}WAZTZ()w_)V{n5tEb@>+JYluDozCb$fA4H)$bzg(Ux{*hXurjO^ zwAxc+UXu=&JV*E59}h3kzQPG4M)X8E*}#_&}w*KEgtX)cU{vm9b$atHa;s>| z+L6&cn8xUL*OSjx4YGjf6{Eq+Q3{!ZyhrL&^6Vz@jGbI%cAM9GkmFlamTbcQGvOlL zmJ?(FI)c86=JEs|*;?h~o)88>12nXlpMR4@yh%qdwFNpct;vMlc=;{FSo*apJ;p}! zAX~t;3tb~VuP|ZW;z$=IHf->F@Ml)&-&Bnb{iQyE#;GZ@C$PzEf6~q}4D>9jic@mTO5x76ulDz@+XAcm35!VSu zT*Gs>;f0b2TNpjU_BjHZ&S6Sqk6V1370+!eppV2H+FY!q*n=GHQ!9Rn6MjY!Jc77A zG7Y!lFp8?TIHN!LXO?gCnsYM-gQxsm=Ek**VmZu7vnuufD7K~GIxfxbsQ@qv2T zPa`tvHB$fFCyZl>3oYg?_wW)C>^_iDOc^B7klnTOoytQH18WkOk)L2BSD0r%xgRSW zQS9elF^?O=_@|58zKLK;(f77l-Zzu}4{fXed2saq!5k#UZAoDBqYQS{sn@j@Vtp|$ zG%gnZ$U|9@u#w1@11Sjl8ze^Co=)7yS(}=;68a3~g;NDe_X^}yJj;~s8xq9ahQ5_r zxAlTMnep*)w1e(TG%tWsjo3RR;yVGPEO4V{Zp?=a_0R#=V^ioQu4YL=BO4r0$$XTX zZfnw#_$V}sDAIDrezGQ+h?q24St0QNug_?{s-pI(^jg`#JRxM1YBV;a@@JQvH8*>> zIJvku74E0NlXkYe_624>znU0J@L<-c=G#F3k4A_)*;ky!C(^uZfj%WB3-*{*B$?9+ zDm$WFp=0(xnt6`vDQV3Jl5f&R(Mp};;q8d3I%Kn>Kx=^;uSVCw0L=gw53%Bp==8Sw zxtx=cs!^-_+i{2OK`Q;913+AXc_&Z5$@z3<)So0CU3;JAv=H?@Zpi~riQ{z-zLtVL z!oF<}@IgJp)Iyz1zVJ42!SPHSkjYNS4%ulVVIXdRuiZ@5Mx8LJS}J#qD^Zi_xQ@>DKDr-_e#>5h3dtje*NcwH_h;i{Sx7}dkdpuW z(yUCjckQsagv*QGMSi9u1`Z|V^}Wjf7B@q%j2DQXyd0nOyqg%m{CK_lAoKlJ7#8M} z%IvR?Vh$6aDWK2W!=i?*<77q&B8O&3?zP(Cs@kapc)&p7En?J;t-TX9abGT#H?TW? ztO5(lPKRuC7fs}zwcUKbRh=7E8wzTsa#Z{a`WR}?UZ%!HohN}d&xJ=JQhpO1PI#>X zHkb>pW04pU%Bj_mf~U}1F1=wxdBZu1790>3Dm44bQ#F=T4V3&HlOLsGH)+AK$cHk6 zia$=$kog?)07HCL*PI6}DRhpM^*%I*kHM<#1Se+AQ!!xyhcy6j7`iDX7Z-2i73_n# zas*?7LkxS-XSqv;YBa zW_n*32D(HTYQ0$feV_Fru1ZxW0g&iwqixPX3=9t4o)o|kOo79V$?$uh?#8Q8e>4e)V6;_(x&ViUVxma+i25qea;d-oK7ouuDsB^ab{ zu1qjQ%`n56VtxBE#0qAzb7lph`Eb-}TYpXB!H-}3Ykqyp`otprp7{VEuW*^IR2n$Fb99*nAtqT&oOFIf z@w*6>YvOGw@Ja?Pp1=whZqydzx@9X4n^2!n83C5{C?G@|E?&$?p*g68)kNvUTJ)I6 z1Q|(#UuP6pj78GUxq11m-GSszc+)X{C2eo-?8ud9sB=3(D47v?`JAa{V(IF zPZQ_0AY*9M97>Jf<o%#O_%Wq}8>YM=q0|tGY+hlXcpE=Z4Od z`NT7Hu2hnvRoqOw@g1f=bv`+nba{GwA$Ak0INlqI1k<9!x_!sL()h?hEWoWrdU3w` zZ%%)VR+Bc@_v!C#koM1p-3v_^L6)_Ktj4HE>aUh%2XZE@JFMOn)J~c`_7VWNb9c-N z2b|SZMR4Z@E7j&q&9(6H3yjEu6HV7{2!1t0lgizD;mZ9$r(r7W5G$ky@w(T_dFnOD z*p#+z$@pKE+>o@%eT(2-p_C}wbQ5s(%Sn_{$HDN@MB+Ev?t@3dPy`%TZ!z}AThZSu zN<1i$siJhXFdjV zP*y|V<`V8t=h#XTRUR~5`c`Z9^-`*BZf?WAehGdg)E2Je)hqFa!k{V(u+(hTf^Yq& zoruUh2(^3pe)2{bvt4&4Y9CY3js)PUHtd4rVG57}uFJL)D(JfSIo^{P=7liFXG zq5yqgof0V8paQcP!gy+;^pp-DA5pj=gbMN0eW=-eY+N8~y+G>t+x}oa!5r>tW$xhI zPQSv=pi;~653Gvf6~*JcQ%t1xOrH2l3Zy@8AoJ+wz@daW@m7?%LXkr!bw9GY@ns3e zSfuWF_gkWnesv?s3I`@}NgE2xwgs&rj?kH-FEy82=O8`+szN ziHch`vvS`zNfap14!&#i9H@wF7}yIPm=UB%(o(}F{wsZ(wA0nJ2aD^@B41>>o-_U6 zUqD~vdo48S8~FTb^+%#zcbQiiYoDKYcj&$#^;Smmb+Ljp(L=1Kt_J!;0s%1|JK}Wi z;={~oL!foo5n8=}rs6MmUW~R&;SIJO3TL4Ky?kh+b2rT9B1Jl4>#Uh-Bec z`Hsp<==#UEW6pGPhNk8H!!DUQR~#F9jEMI6T*OWfN^Ze&X(4nV$wa8QUJ>oTkruH# zm~O<`J7Wxseo@FqaZMl#Y(mrFW9AHM9Kb|XBMqaZ2a)DvJgYipkDD_VUF_PKd~dT7 z#02}bBfPn9a!X!O#83=lbJSK#E}K&yx-HI#T6ua)6o0{|={*HFusCkHzs|Fn&|C3H zBck1cmfcWVUN&i>X$YU^Sn6k2H;r3zuXbJFz)r5~3$d$tUj(l1?o={MM){kjgqXRO zc5R*#{;V7AQh|G|)jLM@wGAK&rm2~@{Pewv#06pHbKn#wL0P6F1!^qw9g&cW3Z=9} zj)POhOlwsh@eF=>z?#sIs*C-Nl(yU!#DaiaxhEs#iJqQ8w%(?+6lU02MYSeDkr!B- zPjMv+on6OLXgGnAtl(ao>|X2Y8*Hb}GRW5}-IzXnoo-d0!m4Vy$GS!XOLy>3_+UGs z2D|YcQx@M#M|}TDOetGi{9lGo9m-=0-^+nKE^*?$^uHkxZh}I{#UTQd;X!L+W@jm( zDg@N4+lUqI92o_rNk{3P>1gxAL=&O;x)ZT=q1mk0kLlE$WeWuY_$0`0jY-Kkt zP*|m3AF}Ubd=`<>(Xg0har*_@x2YH}bn0Wk*OZz3*e5;Zc;2uBdnl8?&XjupbkOeNZsNh6pvsq_ydmJI+*z**{I{0K)-;p1~k8cpJXL$^t!-`E}=*4G^-E8>H!LjTPxSx zcF+cS`ommfKMhNSbas^@YbTpH1*RFrBuATUR zt{oFWSk^$xU&kbFQ;MCX22RAN5F6eq9UfR$ut`Jw--p2YX)A*J69m^!oYfj2y7NYcH6&r+0~_sH^c^nzeN1AU4Ga7=FlR{S|Mm~MpzY0$Z+p2W(a={b-pR9EO1Rs zB%KY|@wLcAA@)KXi!d2_BxrkhDn`DT1=Dec}V!okd{$+wK z4E{n8R*xKyci1(CnNdhf$Dp2(Jpof0-0%-38X=Dd9PQgT+w%Lshx9+loPS~MOm%ZT zt%2B2iL_KU_ita%N>xjB!#71_3=3c}o zgeW~^U_ZTJQ2!PqXulQd=3b=XOQhwATK$y(9$#1jOQ4}4?~l#&nek)H(04f(Sr=s| zWv7Lu1=%WGk4FSw^;;!8&YPM)pQDCY9DhU`hMty1@sq1=Tj7bFsOOBZOFlpR`W>-J$-(kezWJj;`?x-v>ev{*8V z8p|KXJPV$HyQr1A(9LVrM47u-XpcrIyO`yWvx1pVYc&?154aneRpLqgx)EMvRaa#|9?Wwqs2+W8n5~79G z(}iCiLk;?enn}ew`HzhG+tu+Ru@T+K5juvZN)wY;x6HjvqD!&!)$$;1VAh~7fg0K| zEha#aN=Yv|3^~YFH}cc38ovVb%L|g@9W6fo(JtT6$fa?zf@Ct88e}m?i)b*Jgc{fl zExfdvw-BYDmH6>(4QMt#p0;FUIQqkhD}aH?a7)_%JtA~soqj{ppP_82yi9kaxuK>~ ze_)Zt>1?q=ZH*kF{1iq9sr*tVuy=u>Zev}!gEZx@O6-fjyu9X00gpIl-fS_pzjpqJ z1yqBmf9NF!jaF<+YxgH6oXBdK)sH(>VZ)1siyA$P<#KDt;8NT*l_0{xit~5j1P)FN zI8hhYKhQ)i z37^aP13B~u65?sg+_@2Kr^iWHN=U;EDSZ@2W2!5ALhGNWXnFBY%7W?1 z=HI9JzQ-pLKZDYTv<0-lt|6c-RwhxZ)mU2Os{bsX_i^@*fKUj8*aDO5pks=qn3Dv6 zwggpKLuyRCTVPwmw1r}B#AS}?X7b837UlXwp~E2|PJw2SGVueL7){Y&z!jL!XN=0i zU^Eig`S2`{+gU$68aRdWx?BZ{sU_f=8sn~>s~M?GU~`fH5kCc; z8ICp+INM3(3{#k32RZdv6b9MQYdZXNuk7ed8;G?S2nT+NZBG=Tar^KFl2SvhW$bGW#kdWL-I)s_IqVnCDDM9fm8g;P;8 z7t4yZn3^*NQfx7SwmkzP$=fwdC}bafQSEF@pd&P8@H#`swGy_rz;Z?Ty5mkS%>m#% zp_!m9e<()sfKiY(nF<1zBz&&`ZlJf6QLvLhl`_``%RW&{+O>Xhp;lwSsyRqGf=RWd zpftiR`={2(siiPAS|p}@q=NhVc0ELprt%=fMXO3B)4ryC2LT(o=sLM7hJC!}T1@)E zA3^J$3&1*M6Xq>03FX`R&w*NkrZE?FwU+Muut;>qNhj@bX17ZJxnOlPSZ=Zeiz~T_ zOu#yc3t6ONHB;?|r4w+pI)~KGN;HOGC)txxiUN8#mexj+W(cz%9a4sx|IRG=}ia zuEBuba3AHsV2feqw-3MvuL`I+2|`Ud4~7ZkN=JZ;L20|Oxna5vx1qbIh#k2O4$RQF zo`tL()zxaqibg^GbB+BS5#U{@K;WWQj~GcB1zb}zJkPwH|5hZ9iH2308!>_;%msji zJHSL~s)YHBR=Koa1mLEOHos*`gp=s8KA-C zu0aE+W!#iJ*0xqKm3A`fUGy#O+X+5W36myS>Uh2!R*s$aCU^`K&KKLCCDkejX2p=5 z%o7-fl03x`gaSNyr?3_JLv?2RLS3F*8ub>Jd@^Cc17)v8vYEK4aqo?OS@W9mt%ITJ z9=S2%R8M){CugT@k~~0x`}Vl!svYqX=E)c_oU6o}#Hb^%G1l3BudxA{F*tbjG;W_>=xV73pKY53v%>I)@D36I_@&p$h|Aw zonQS`07z_F#@T-%@-Tb|)7;;anoD_WH>9ewFy(ZcEOM$#Y)8>qi7rCnsH9GO-_7zF zu*C87{Df1P4TEOsnzZ@H%&lvV(3V@;Q!%+OYRp`g05PjY^gL$^$-t0Y>H*CDDs?FZly*oZ&dxvsxaUWF!{em4{A>n@vpXg$dwvt@_rgmHF z-MER`ABa8R-t_H*kv>}CzOpz;!>p^^9ztHMsHL|SRnS<-y5Z*r(_}c4=fXF`l^-i}>e7v!qs_jv zqvWhX^F=2sDNWA9c@P0?lUlr6ecrTKM%pNQ^?*Lq?p-0~?_j50xV%^(+H>sMul#Tw zeciF*1=?a7cI(}352%>LO96pD+?9!fNyl^9v3^v&Y4L)mNGK0FN43&Xf8jUlxW1Bw zyiu2;qW-aGNhs=zbuoxnxiwZ3{PFZM#Kw)9H@(hgX23h(`Wm~m4&TvoZoYp{plb^> z_#?vXcxd>r7K+1HKJvhed>gtK`TAbJUazUWQY6T~t2af%#<+Veyr%7-#*A#@&*;@g58{i|E%6yC_InGXCOd{L0;$)z#?n7M`re zh!kO{6=>7I?*}czyF7_frt#)s1CFJ_XE&VrDA?Dp3XbvF{qsEJgb&OLSNz_5g?HpK z9)8rsr4JN!Af3G9!#Qn(6zaUDqLN(g2g8*M)Djap?WMK9NKlkC)E2|-g|#-rp%!Gz zAHd%`iq|81efi93m3yTBw3g0j#;Yb2X{mhRAI?&KDmbGqou(2xiRNb^sV}%%Wu0?< z?($L>(#BO*)^)rSgyNRni$i`R4v;GhlCZ8$@e^ROX(p=2_v6Y!%^As zu022)fHdv_-~Yu_H6WVPLpHQx!W%^6j)cBhS`O3QBW#x(eX54d&I22op(N59b*&$v zFiSRY6rOc^(dgSV1>a7-5C;(5S5MvKcM2Jm-LD9TGqDpP097%52V+0>Xqq!! zq4e3vj53SE6i8J`XcQB|MZPP8j;PAOnpGnllH6#Ku~vS42xP*Nz@~y%db7Xi8s09P z1)e%8ys6&M8D=Dt6&t`iKG_4X=!kgRQoh%Z`dc&mlOUqXk-k`jKv9@(a^2-Upw>?< zt5*^DV~6Zedbec4NVl($2T{&b)zA@b#dUyd>`2JC0=xa_fIm8{5um zr-!ApXZhC8@=vC2WyxO|!@0Km)h8ep*`^he92$@YwP>VcdoS5OC^s38e#7RPsg4j+ zbVGG}WRSET&ZfrcR(x~k8n1rTP%CnfUNKUonD$P?FtNFF#cn!wEIab-;jU=B1dHK@ z(;(yAQJ`O$sMn>h;pf^8{JISW%d+@v6@CnXh9n5TXGC}?FI9i-D0OMaIg&mAg=0Kn zNJ7oz5*ReJukD55fUsMuaP+H4tDN&V9zfqF@ zr=#ecUk9wu{0;!+gl;3Bw=Vn^)z$ahVhhw)io!na&9}LmWurLb0zubxK=UEnU*{5P z+SP}&*(iBKSO4{alBHaY^)5Q=mZ+2OwIooJ7*Q5XJ+2|q`9#f?6myq!&oz?klihLq z4C)$XP!BNS0G_Z1&TM>?Jk{S~{F3n83ioli=IO6f%wkvCl(RFFw~j0tb{GvXTx>*sB0McY0s&SNvj4+^h`9nJ_wM>F!Uc>X}9PifQekn0sKI2SAJP!a4h z5cyGTuCj3ZBM^&{dRelIlT^9zcfaAuL5Y~bl!ppSf`wZbK$z#6U~rdclk``e+!qhe z6Qspo*%<)eu6?C;Bp<^VuW6JI|Ncvyn+LlSl;Mp22Bl7ARQ0Xc24%29(ZrdsIPw&-=yHQ7_Vle|5h>AST0 zUGX2Zk34vp?U~IHT|;$U86T+UUHl_NE4m|}>E~6q``7hccCaT^#y+?wD##Q%HwPd8 zV3x4L4|qqu`B$4(LXqDJngNy-{&@aFBvVsywt@X^}iH7P%>bR?ciC$I^U-4Foa`YKI^qDyGK7k%E%c_P=yzAi`YnxGA%DeNd++j3*h^ z=rn>oBd0|~lZ<6YvmkKY*ZJlJ;Im0tqgWu&E92eqt;+NYdxx`eS(4Hw_Jb5|yVvBg z*tbdY^!AN;luEyN4VRhS@-_DC{({ziH{&Z}iGElSV~qvT>L-8G%+yEL zX#MFOhj{InyKG=mvW-<1B@c-}x$vA(nU?>S>0*eN#!SLzQ)Ex7fvQ)S4D<8|I#N$3 zT5Ei`Z?cxBODHX8(Xp73v`IsAYC@9b;t}z0wxVuQSY1J^GRwDPN@qbM-ZF48T$GZ< z8WU+;Pqo?{ghI-KZ-i*ydXu`Ep0Xw^McH_KE9J0S7G;x8Fe`DVG?j3Pv=0YzJ}yZR z%2=oqHiUjvuk0~Ca>Kol4CFi0_xQT~;_F?=u+!kIDl-9g`#ZNZ9HCy17Ga1v^Jv9# z{T4Kb1-AzUxq*MutfOWWZgD*HnFfyYg0&e9f(5tZ>krPF6{VikNeHoc{linPPt#Si z&*g>(c54V8rT_AX!J&bNm-!umPvOR}vDai#`CX___J#=zeB*{4<&2WpaDncZsOkp* zsg<%@@rbrMkR_ux9?LsQxzoBa1s%$BBn6vk#{&&zUwcfzeCBJUwFYSF$08qDsB;gWQN*g!p8pxjofWbqNSZOEKOaTx@+* zwdt5*Q47@EOZ~EZL9s?1o?A%9TJT=Ob_13yyugvPg*e&ZU(r6^k4=2+D-@n=Hv5vu zSXG|hM(>h9^zn=eQ=$6`JO&70&2|%V5Lsx>)(%#;pcOfu>*nk_3HB_BNaH$`jM<^S zcSftDU1?nL;jy)+sfonQN}(}gUW?d_ikr*3=^{G)=tjBtEPe>TO|0ddVB zTklrSHiW+!#26frPXQQ(YN8DG$PZo?(po(QUCCf_OJC`pw*uey00%gmH!`WJkrKXj2!#6?`T25mTu9OJp2L8z3! z=arrL$ZqxuE{%yV)14Kd>k}j7pxZ6#$Dz8$@WV5p8kTqN<-7W)Q7Gt2{KoOPK_tZ| zf2WG~O5@{qPI+W<4f_;reuFVdO^5`ADC1!JQE|N`s3cq@(0WB!n0uh@*c{=LAd;~} zyGK@hbF-Oo+!nN)@i*O(`@FA#u?o=~e{`4O#5}z&=UkU*50fOrzi11D^&FOqe>wii z?*k+2|EcUs;Gx{!@KBT~>PAwLrIDT7Th=Utu?~?np@t^gFs?zgX=D${RwOY^WGh-+ z+#4$066ISh8eYW#FXWp~S`<*%O^ZuItL1Tyqt8#tZ zY120E;^VG`!lZn&3sPd$RkdHpU#|w+bYV)pJC|SH9g%|5IkxVTQcBA4CL0}$&}ef@ zW^Vtj%M;;_1xxP9x#ex17&4N*{ksO*_4O}xYu(p*JkL#yr}@7b)t5X?%CY<+s5_MJ zuiqt+N_;A(_)%lumoyRFixWa-M7qK_9s6<1X?JDa9fP!+_6u~~M$5L=ipB=7(j#f< zZ34J%=bs549%~_mA(|={uZNs_0?o7;-LBP(ZRnkd{-^|2|=4vUTmtByHL8 zEph`(LSEzQj68a+`d$V<45J7cyv^#|^|%fD#si1Nx!4NW*`l*{->HEWNh6-|g>-=r zXmQ|-i}Ku$ndUeHQ^&ieT!Lf}vf6GaqW9$DJ2NWrqwPY%%4nip$@vK$nRp*_C-v<| zuKz~ZyN&<%!NS26&x?jhy+@awJipMQ-8(X4#Ae5??U<1QMt1l9R=w9fAnEF}NYu$2 z>6}Vkc zIb*A?G*z8^IvibmBKn_u^5&T_1oey0gZS2~obf(#xk=erZGTEdQnt3DMGM+0oPwss zj5zXD;(oWhB_T@~Ig#9@v)AKtXu3>Inmgf@A|-lD-1U>cNyl3h?ADD9)GG4}zUGPk zZzaXe!~Kf?<~@$G?Uql3t8jy9{2!doq4=J}j9ktTxss{p6!9UdjyDERlA*xZ!=Q)KDs5O)phz>Vq3BNGoM(H|=1*Q4$^2fTZw z(%nq1P|5Rt81}SYJpEEzMPl5VJsV5&4e)ZWKDyoZ>1EwpkHx-AQVQc8%JMz;{H~p{=FXV>jIxvm4X*qv52e?Y-f%DJ zxEA165GikEASQ^fH6K#d!Tpu2HP{sFs%E=e$gYd$aj$+xue6N+Wc(rAz~wUsk2`(b z8Kvmyz%bKQxpP}~baG-rwYcYCvkHOi zlkR<=>ZBTU*8RF_d#Bl@zZsRIhx<%~Z@Z=ik z>adw3!DK(8R|q$vy{FTxw%#xliD~6qXmY^7_9kthVPTF~Xy1CfBqbU~?1QmxmU=+k z(ggxvEuA;0e&+ci-zQR{-f7aO{O(Pz_OsEjLh_K>MbvoZ4nxtk5u{g@nPv)cgW_R} z9}EA4K4@z0?7ue}Z(o~R(X&FjejUI2g~08PH1E4w>9o{)S(?1>Z0XMvTb|;&EuyOE zGvWNpYX)Nv<8|a^;1>bh#&znEcl-r!T#pn= z4$?Yudha6F%4b>*8@=BdtXXY4N+`U4Dmx$}>HeVJk-QdTG@t!tVT#0(LeV0gvqyyw z2sEp^9eY0N`u10Tm4n8No&A=)IeEC|gnmEXoNSzu!1<4R<%-9kY_8~5Ej?zRegMn78wuMs#;i&eUA0Zk_RXQ3b&TT} z;SCI=7-FUB@*&;8|n>(_g^HGf3@QODE3LpmX~ELnymQm{Sx9xrKS zK29p~?v@R$0=v6Dr5aW>-!{+h@?Q58|Kz8{{W`%J+lDAdb&M5VHrX_mDY;1-JLnf)ezmPau$)1;=`-FU=-r-83tX=C`S#}GZufju zQ>sXNT0Ny=k@nc%cFnvA_i4SC)?_ORXHq8B4D%el1uPX`c~uG#S1M7C+*MMqLw78E zhY2dI8@+N^qrMI1+;TUda(vGqGSRyU{Fnm`aqrr7bz42c5xsOO-~oZpkzorD1g}Y<6rk&3>PsSGy}W?MtqFky@A(X# zIuNZK0cK?^=;PUAu>j0#HtjbHCV*6?jzA&OoE$*Jlga*}LF`SF?WLhv1O|zqC<>*> zYB;#lsYKx0&kH@BFpW8n*yDcc6?;_zaJs<-jPSkCsSX-!aV=P5kUgF@Nu<{a%#K*F z134Q{9|YX7X(v$62_cY3^G%t~rD>Q0z@)1|zs)vjJ6Jq9;7#Ki`w+eS**En?7;n&7 zu==V3T&eFboN3ZiMx3D8qYc;VjFUk_H-WWCau(VFXSQf~viH0L$gwD$UfFHqNcgN`x}M+YQ6RnN<+@t>JUp#)9YOkqst-Ga?{FsDpEeX0(5v{0J~SEbWiL zXC2}M4?UH@u&|;%0y`eb33ldo4~z-x8zY!oVmV=c+f$m?RfDC35mdQ2E>Pze7KWP- z>!Bh<&57I+O_^s}9Tg^k)h7{xx@0a0IA~GAOt2yy!X%Q$1rt~LbTB6@Du!_0%HV>N zlf)QI1&gvERKwso23mJ!Ou6ZS#zCS5W`gxE5T>C#E|{i<1D35C222I33?Njaz`On7 zi<+VWFP6D{e-{yiN#M|Jgk<44u1TiMI78S5W`Sdb5f+{zu34s{CfWN7a3Cf^@L%!& zN$?|!!9j2c)j$~+R6n#891w-z8(!oBpL2K=+%a$r2|~8-(vQj5_XT`<0Ksf;oP+tz z9CObS!0m)Tgg`K#xBM8B(|Z)Wb&DYL{WTYv`;A=q6~Nnx2+!lTIXtj8J7dZE!P_{z z#f8w6F}^!?^KE#+ZDv+xd5O&3EmomZzsv?>E-~ygGum45fk!SBN&|eo1rKw^?aZJ4 E2O(~oYXATM literal 0 HcmV?d00001 diff --git a/platforms/android/aar-template/gradle/wrapper/gradle-wrapper.properties b/platforms/android/aar-template/gradle/wrapper/gradle-wrapper.properties new file mode 100644 index 0000000000..9e6a7833f8 --- /dev/null +++ b/platforms/android/aar-template/gradle/wrapper/gradle-wrapper.properties @@ -0,0 +1,6 @@ +#Mon Jul 10 11:57:38 SGT 2023 +distributionBase=GRADLE_USER_HOME +distributionPath=wrapper/dists +distributionUrl=https\://services.gradle.org/distributions/gradle-8.0-bin.zip +zipStoreBase=GRADLE_USER_HOME +zipStorePath=wrapper/dists diff --git a/platforms/android/aar-template/gradlew b/platforms/android/aar-template/gradlew new file mode 100755 index 0000000000..4f906e0c81 --- /dev/null +++ b/platforms/android/aar-template/gradlew @@ -0,0 +1,185 @@ +#!/usr/bin/env sh + +# +# Copyright 2015 the original author or authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +############################################################################## +## +## Gradle start up script for UN*X +## +############################################################################## + +# Attempt to set APP_HOME +# Resolve links: $0 may be a link +PRG="$0" +# Need this for relative symlinks. +while [ -h "$PRG" ] ; do + ls=`ls -ld "$PRG"` + link=`expr "$ls" : '.*-> \(.*\)$'` + if expr "$link" : '/.*' > /dev/null; then + PRG="$link" + else + PRG=`dirname "$PRG"`"/$link" + fi +done +SAVED="`pwd`" +cd "`dirname \"$PRG\"`/" >/dev/null +APP_HOME="`pwd -P`" +cd "$SAVED" >/dev/null + +APP_NAME="Gradle" +APP_BASE_NAME=`basename "$0"` + +# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' + +# Use the maximum available, or set MAX_FD != -1 to use that value. +MAX_FD="maximum" + +warn () { + echo "$*" +} + +die () { + echo + echo "$*" + echo + exit 1 +} + +# OS specific support (must be 'true' or 'false'). +cygwin=false +msys=false +darwin=false +nonstop=false +case "`uname`" in + CYGWIN* ) + cygwin=true + ;; + Darwin* ) + darwin=true + ;; + MINGW* ) + msys=true + ;; + NONSTOP* ) + nonstop=true + ;; +esac + +CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar + + +# Determine the Java command to use to start the JVM. +if [ -n "$JAVA_HOME" ] ; then + if [ -x "$JAVA_HOME/jre/sh/java" ] ; then + # IBM's JDK on AIX uses strange locations for the executables + JAVACMD="$JAVA_HOME/jre/sh/java" + else + JAVACMD="$JAVA_HOME/bin/java" + fi + if [ ! -x "$JAVACMD" ] ; then + die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." + fi +else + JAVACMD="java" + which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." +fi + +# Increase the maximum file descriptors if we can. +if [ "$cygwin" = "false" -a "$darwin" = "false" -a "$nonstop" = "false" ] ; then + MAX_FD_LIMIT=`ulimit -H -n` + if [ $? -eq 0 ] ; then + if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then + MAX_FD="$MAX_FD_LIMIT" + fi + ulimit -n $MAX_FD + if [ $? -ne 0 ] ; then + warn "Could not set maximum file descriptor limit: $MAX_FD" + fi + else + warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT" + fi +fi + +# For Darwin, add options to specify how the application appears in the dock +if $darwin; then + GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\"" +fi + +# For Cygwin or MSYS, switch paths to Windows format before running java +if [ "$cygwin" = "true" -o "$msys" = "true" ] ; then + APP_HOME=`cygpath --path --mixed "$APP_HOME"` + CLASSPATH=`cygpath --path --mixed "$CLASSPATH"` + + JAVACMD=`cygpath --unix "$JAVACMD"` + + # We build the pattern for arguments to be converted via cygpath + ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null` + SEP="" + for dir in $ROOTDIRSRAW ; do + ROOTDIRS="$ROOTDIRS$SEP$dir" + SEP="|" + done + OURCYGPATTERN="(^($ROOTDIRS))" + # Add a user-defined pattern to the cygpath arguments + if [ "$GRADLE_CYGPATTERN" != "" ] ; then + OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)" + fi + # Now convert the arguments - kludge to limit ourselves to /bin/sh + i=0 + for arg in "$@" ; do + CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -` + CHECK2=`echo "$arg"|egrep -c "^-"` ### Determine if an option + + if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then ### Added a condition + eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"` + else + eval `echo args$i`="\"$arg\"" + fi + i=`expr $i + 1` + done + case $i in + 0) set -- ;; + 1) set -- "$args0" ;; + 2) set -- "$args0" "$args1" ;; + 3) set -- "$args0" "$args1" "$args2" ;; + 4) set -- "$args0" "$args1" "$args2" "$args3" ;; + 5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;; + 6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;; + 7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;; + 8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;; + 9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;; + esac +fi + +# Escape application args +save () { + for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done + echo " " +} +APP_ARGS=`save "$@"` + +# Collect all arguments for the java command, following the shell quoting and substitution rules +eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS" + +exec "$JAVACMD" "$@" diff --git a/platforms/android/aar-template/gradlew.bat b/platforms/android/aar-template/gradlew.bat new file mode 100644 index 0000000000..107acd32c4 --- /dev/null +++ b/platforms/android/aar-template/gradlew.bat @@ -0,0 +1,89 @@ +@rem +@rem Copyright 2015 the original author or authors. +@rem +@rem Licensed under the Apache License, Version 2.0 (the "License"); +@rem you may not use this file except in compliance with the License. +@rem You may obtain a copy of the License at +@rem +@rem https://www.apache.org/licenses/LICENSE-2.0 +@rem +@rem Unless required by applicable law or agreed to in writing, software +@rem distributed under the License is distributed on an "AS IS" BASIS, +@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +@rem See the License for the specific language governing permissions and +@rem limitations under the License. +@rem + +@if "%DEBUG%" == "" @echo off +@rem ########################################################################## +@rem +@rem Gradle startup script for Windows +@rem +@rem ########################################################################## + +@rem Set local scope for the variables with windows NT shell +if "%OS%"=="Windows_NT" setlocal + +set DIRNAME=%~dp0 +if "%DIRNAME%" == "" set DIRNAME=. +set APP_BASE_NAME=%~n0 +set APP_HOME=%DIRNAME% + +@rem Resolve any "." and ".." in APP_HOME to make it shorter. +for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi + +@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m" + +@rem Find java.exe +if defined JAVA_HOME goto findJavaFromJavaHome + +set JAVA_EXE=java.exe +%JAVA_EXE% -version >NUL 2>&1 +if "%ERRORLEVEL%" == "0" goto execute + +echo. +echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. +echo. +echo Please set the JAVA_HOME variable in your environment to match the +echo location of your Java installation. + +goto fail + +:findJavaFromJavaHome +set JAVA_HOME=%JAVA_HOME:"=% +set JAVA_EXE=%JAVA_HOME%/bin/java.exe + +if exist "%JAVA_EXE%" goto execute + +echo. +echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% +echo. +echo Please set the JAVA_HOME variable in your environment to match the +echo location of your Java installation. + +goto fail + +:execute +@rem Setup the command line + +set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar + + +@rem Execute Gradle +"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %* + +:end +@rem End local scope for the variables with windows NT shell +if "%ERRORLEVEL%"=="0" goto mainEnd + +:fail +rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of +rem the _cmd.exe /c_ return code! +if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 +exit /b 1 + +:mainEnd +if "%OS%"=="Windows_NT" endlocal + +:omega diff --git a/platforms/android/aar-template/settings.gradle b/platforms/android/aar-template/settings.gradle new file mode 100644 index 0000000000..fb1a50602c --- /dev/null +++ b/platforms/android/aar-template/settings.gradle @@ -0,0 +1,16 @@ +pluginManagement { + repositories { + google() + mavenCentral() + gradlePluginPortal() + } +} +dependencyResolutionManagement { + repositoriesMode.set(RepositoriesMode.FAIL_ON_PROJECT_REPOS) + repositories { + google() + mavenCentral() + } +} +rootProject.name = "OpenCV" +include ':OpenCV' diff --git a/platforms/android/build_java_shared_aar.py b/platforms/android/build_java_shared_aar.py new file mode 100755 index 0000000000..ea59bad839 --- /dev/null +++ b/platforms/android/build_java_shared_aar.py @@ -0,0 +1,156 @@ +#!/usr/bin/env python + +import argparse +from os import path +import os +import re +import shutil +import string +import subprocess + + +COPY_FROM_SDK_TO_ANDROID_PROJECT = [ + ["sdk/native/jni/include", "OpenCV/src/main/cpp/include"], + ["sdk/java/src/org", "OpenCV/src/main/java/org"], + ["sdk/java/res", "OpenCV/src/main/res"] +] + +COPY_FROM_SDK_TO_APK = [ + ["sdk/native/libs//lib.so", "jni//lib.so"], + ["sdk/native/libs//lib.so", "prefab/modules//libs/android./lib.so"], +] + +ANDROID_PROJECT_TEMPLATE_DIR = path.join(path.dirname(__file__), "aar-template") +TEMP_DIR = "build_java_shared" +ANDROID_PROJECT_DIR = path.join(TEMP_DIR, "AndroidProject") +COMPILED_AAR_PATH_1 = path.join(ANDROID_PROJECT_DIR, "OpenCV/build/outputs/aar/OpenCV-release.aar") # original package name +COMPILED_AAR_PATH_2 = path.join(ANDROID_PROJECT_DIR, "OpenCV/build/outputs/aar/opencv-release.aar") # lower case package name +AAR_UNZIPPED_DIR = path.join(TEMP_DIR, "aar_unzipped") +FINAL_AAR_PATH_TEMPLATE = "outputs/opencv_java_shared_.aar" +FINAL_REPO_PATH = "outputs/maven_repo" +MAVEN_PACKAGE_NAME = "opencv" + +def fill_template(src_path, dst_path, args_dict): + with open(src_path, "r") as f: + template_text = f.read() + template = string.Template(template_text) + text = template.safe_substitute(args_dict) + with open(dst_path, "w") as f: + f.write(text) + +def get_opencv_version(opencv_sdk_path): + version_hpp_path = path.join(opencv_sdk_path, "sdk/native/jni/include/opencv2/core/version.hpp") + with open(version_hpp_path, "rt") as f: + data = f.read() + major = re.search(r'^#define\W+CV_VERSION_MAJOR\W+(\d+)$', data, re.MULTILINE).group(1) + minor = re.search(r'^#define\W+CV_VERSION_MINOR\W+(\d+)$', data, re.MULTILINE).group(1) + revision = re.search(r'^#define\W+CV_VERSION_REVISION\W+(\d+)$', data, re.MULTILINE).group(1) + return "%(major)s.%(minor)s.%(revision)s" % locals() + +def get_compiled_aar_path(path1, path2): + if path.exists(path1): + return path1 + elif path.exists(path2): + return path2 + else: + raise Exception("Can't find compiled AAR path in [" + path1 + ", " + path2 + "]") + +def cleanup(paths_to_remove): + exists = False + for p in paths_to_remove: + if path.exists(p): + exists = True + if path.isdir(p): + shutil.rmtree(p) + else: + os.remove(p) + print("Removed", p) + if not exists: + print("Nothing to remove") + +def main(args): + opencv_version = get_opencv_version(args.opencv_sdk_path) + abis = os.listdir(path.join(args.opencv_sdk_path, "sdk/native/libs")) + lib_name = "opencv_java" + opencv_version.split(".")[0] + final_aar_path = FINAL_AAR_PATH_TEMPLATE.replace("", opencv_version) + + print("Removing data from previous runs...") + cleanup([TEMP_DIR, final_aar_path, path.join(FINAL_REPO_PATH, "org/opencv", MAVEN_PACKAGE_NAME)]) + + print("Preparing Android project...") + # ANDROID_PROJECT_TEMPLATE_DIR contains an Android project template that creates AAR + shutil.copytree(ANDROID_PROJECT_TEMPLATE_DIR, ANDROID_PROJECT_DIR) + + # Configuring the Android project to Java + shared C++ lib version + shutil.rmtree(path.join(ANDROID_PROJECT_DIR, "OpenCV/src/main/cpp/include")) + + fill_template(path.join(ANDROID_PROJECT_DIR, "OpenCV/build.gradle.template"), + path.join(ANDROID_PROJECT_DIR, "OpenCV/build.gradle"), + {"LIB_NAME": lib_name, + "LIB_TYPE": "c++_shared", + "PACKAGE_NAME": MAVEN_PACKAGE_NAME, + "OPENCV_VERSION": opencv_version, + "COMPILE_SDK": args.android_compile_sdk, + "MIN_SDK": args.android_min_sdk, + "TARGET_SDK": args.android_target_sdk, + "ABI_FILTERS": ", ".join(['"' + x + '"' for x in abis]), + "JAVA_VERSION": args.java_version, + }) + fill_template(path.join(ANDROID_PROJECT_DIR, "OpenCV/src/main/cpp/CMakeLists.txt.template"), + path.join(ANDROID_PROJECT_DIR, "OpenCV/src/main/cpp/CMakeLists.txt"), + {"LIB_NAME": lib_name, "LIB_TYPE": "SHARED"}) + + # Copying Java code and C++ public headers from SDK to the Android project + for src, dst in COPY_FROM_SDK_TO_ANDROID_PROJECT: + shutil.copytree(path.join(args.opencv_sdk_path, src), + path.join(ANDROID_PROJECT_DIR, dst)) + + print("Running gradle assembleRelease...") + # Running gradle to build the Android project + subprocess.run(["./gradlew", "assembleRelease"], + shell=False, + cwd=ANDROID_PROJECT_DIR, + check=True) + + print("Adding libs to AAR...") + # The created AAR package doesn't contain C++ shared libs. + # We need to add them manually. + # AAR package is just a zip archive. + complied_aar_path = get_compiled_aar_path(COMPILED_AAR_PATH_1, COMPILED_AAR_PATH_2) # two possible paths + shutil.unpack_archive(complied_aar_path, AAR_UNZIPPED_DIR, "zip") + + for abi in abis: + for src, dst in COPY_FROM_SDK_TO_APK: + src = src.replace("", abi).replace("", lib_name) + dst = dst.replace("", abi).replace("", lib_name) + shutil.copy(path.join(args.opencv_sdk_path, src), + path.join(AAR_UNZIPPED_DIR, dst)) + + # Creating final AAR zip archive + os.makedirs("outputs", exist_ok=True) + shutil.make_archive(final_aar_path, "zip", AAR_UNZIPPED_DIR, ".") + os.rename(final_aar_path + ".zip", final_aar_path) + + print("Creating local maven repo...") + + shutil.copy(final_aar_path, path.join(ANDROID_PROJECT_DIR, "OpenCV/opencv-release.aar")) + subprocess.run(["./gradlew", "publishReleasePublicationToMyrepoRepository"], + shell=False, + cwd=ANDROID_PROJECT_DIR, + check=True) + + os.makedirs(path.join(FINAL_REPO_PATH, "org/opencv"), exist_ok=True) + shutil.move(path.join(ANDROID_PROJECT_DIR, "OpenCV/build/repo/org/opencv", MAVEN_PACKAGE_NAME), + path.join(FINAL_REPO_PATH, "org/opencv", MAVEN_PACKAGE_NAME)) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Builds AAR with Java and shared C++ libs from OpenCV SDK") + parser.add_argument('opencv_sdk_path') + parser.add_argument('--android_compile_sdk', default="26") + parser.add_argument('--android_min_sdk', default="21") + parser.add_argument('--android_target_sdk', default="26") + parser.add_argument('--java_version', default="1_8") + args = parser.parse_args() + + main(args) diff --git a/platforms/android/build_static_aar.py b/platforms/android/build_static_aar.py new file mode 100755 index 0000000000..df6d55a6fd --- /dev/null +++ b/platforms/android/build_static_aar.py @@ -0,0 +1,229 @@ +#!/usr/bin/env python + +import argparse +import json +from os import path +import os +import shutil +import subprocess + +from build_java_shared_aar import cleanup, fill_template, get_compiled_aar_path, get_opencv_version + + +ANDROID_PROJECT_TEMPLATE_DIR = path.join(path.dirname(__file__), "aar-template") +TEMP_DIR = "build_static" +ANDROID_PROJECT_DIR = path.join(TEMP_DIR, "AndroidProject") +COMPILED_AAR_PATH_1 = path.join(ANDROID_PROJECT_DIR, "OpenCV/build/outputs/aar/OpenCV-release.aar") # original package name +COMPILED_AAR_PATH_2 = path.join(ANDROID_PROJECT_DIR, "OpenCV/build/outputs/aar/opencv-release.aar") # lower case package name +AAR_UNZIPPED_DIR = path.join(TEMP_DIR, "aar_unzipped") +FINAL_AAR_PATH_TEMPLATE = "outputs/opencv_static_.aar" +FINAL_REPO_PATH = "outputs/maven_repo" +MAVEN_PACKAGE_NAME = "opencv-static" + + +def get_list_of_opencv_libs(sdk_dir): + files = os.listdir(path.join(sdk_dir, "sdk/native/staticlibs/arm64-v8a")) + libs = [f[3:-2] for f in files if f[:3] == "lib" and f[-2:] == ".a"] + return libs + +def get_list_of_3rdparty_libs(sdk_dir, abis): + libs = [] + for abi in abis: + files = os.listdir(path.join(sdk_dir, "sdk/native/3rdparty/libs/" + abi)) + cur_libs = [f[3:-2] for f in files if f[:3] == "lib" and f[-2:] == ".a"] + for lib in cur_libs: + if lib not in libs: + libs.append(lib) + return libs + +def add_printing_linked_libs(sdk_dir, opencv_libs): + """ + Modifies CMakeLists.txt file in Android project, so it prints linked libraries for each OpenCV library" + """ + sdk_jni_dir = sdk_dir + "/sdk/native/jni" + with open(path.join(ANDROID_PROJECT_DIR, "OpenCV/src/main/cpp/CMakeLists.txt"), "a") as f: + f.write('\nset(OpenCV_DIR "' + sdk_jni_dir + '")\n') + f.write('find_package(OpenCV REQUIRED)\n') + for lib_name in opencv_libs: + output_filename_prefix = "linkedlibs." + lib_name + "." + f.write('get_target_property(OUT "' + lib_name + '" INTERFACE_LINK_LIBRARIES)\n') + f.write('file(WRITE "' + output_filename_prefix + '${ANDROID_ABI}.txt" "${OUT}")\n') + +def read_linked_libs(lib_name, abis): + """ + Reads linked libs for each OpenCV library from files, that was generated by gradle. See add_printing_linked_libs() + """ + deps_lists = [] + for abi in abis: + with open(path.join(ANDROID_PROJECT_DIR, "OpenCV/src/main/cpp", f"linkedlibs.{lib_name}.{abi}.txt")) as f: + text = f.read() + linked_libs = text.split(";") + linked_libs = [x.replace("$", "") for x in linked_libs] + deps_lists.append(linked_libs) + + return merge_dependencies_lists(deps_lists) + +def merge_dependencies_lists(deps_lists): + """ + One library may have different dependencies for different ABIS. + We need to merge them into one list with all the dependencies preserving the order. + """ + result = [] + for d_list in deps_lists: + for i in range(len(d_list)): + if d_list[i] not in result: + if i == 0: + result.append(d_list[i]) + else: + index = result.index(d_list[i-1]) + result = result[:index + 1] + [d_list[i]] + result[index + 1:] + + return result + +def convert_deps_list_to_prefab(linked_libs, opencv_libs, external_libs): + """ + Converting list of dependencies into prefab format. + """ + prefab_linked_libs = [] + for lib in linked_libs: + if (lib in opencv_libs) or (lib in external_libs): + prefab_linked_libs.append(":" + lib) + elif (lib[:3] == "lib" and lib[3:] in external_libs): + prefab_linked_libs.append(":" + lib[3:]) + elif lib == "ocv.3rdparty.android_mediandk": + prefab_linked_libs += ["-landroid", "-llog", "-lmediandk"] + print("Warning: manualy handled ocv.3rdparty.android_mediandk dependency") + elif lib == "ocv.3rdparty.flatbuffers": + print("Warning: manualy handled ocv.3rdparty.flatbuffers dependency") + elif lib.startswith("ocv.3rdparty"): + raise Exception("Unknown lib " + lib) + else: + prefab_linked_libs.append("-l" + lib) + return prefab_linked_libs + +def main(args): + opencv_version = get_opencv_version(args.opencv_sdk_path) + abis = os.listdir(path.join(args.opencv_sdk_path, "sdk/native/libs")) + final_aar_path = FINAL_AAR_PATH_TEMPLATE.replace("", opencv_version) + sdk_dir = args.opencv_sdk_path + + print("Removing data from previous runs...") + cleanup([TEMP_DIR, final_aar_path, path.join(FINAL_REPO_PATH, "org/opencv", MAVEN_PACKAGE_NAME)]) + + print("Preparing Android project...") + # ANDROID_PROJECT_TEMPLATE_DIR contains an Android project template that creates AAR + shutil.copytree(ANDROID_PROJECT_TEMPLATE_DIR, ANDROID_PROJECT_DIR) + + # Configuring the Android project to static C++ libs version + fill_template(path.join(ANDROID_PROJECT_DIR, "OpenCV/build.gradle.template"), + path.join(ANDROID_PROJECT_DIR, "OpenCV/build.gradle"), + {"LIB_NAME": "templib", + "LIB_TYPE": "c++_static", + "PACKAGE_NAME": MAVEN_PACKAGE_NAME, + "OPENCV_VERSION": opencv_version, + "COMPILE_SDK": args.android_compile_sdk, + "MIN_SDK": args.android_min_sdk, + "TARGET_SDK": args.android_target_sdk, + "ABI_FILTERS": ", ".join(['"' + x + '"' for x in abis]), + "JAVA_VERSION": args.java_version, + }) + fill_template(path.join(ANDROID_PROJECT_DIR, "OpenCV/src/main/cpp/CMakeLists.txt.template"), + path.join(ANDROID_PROJECT_DIR, "OpenCV/src/main/cpp/CMakeLists.txt"), + {"LIB_NAME": "templib", "LIB_TYPE": "STATIC"}) + + opencv_libs = get_list_of_opencv_libs(sdk_dir) + external_libs = get_list_of_3rdparty_libs(sdk_dir, abis) + + add_printing_linked_libs(sdk_dir, opencv_libs) + + print("Running gradle assembleRelease...") + # Running gradle to build the Android project + subprocess.run(["./gradlew", "assembleRelease"], + shell=False, + cwd=ANDROID_PROJECT_DIR, + check=True) + + # The created AAR package contains only one empty libtemplib.a library. + # We need to add OpenCV libraries manually. + # AAR package is just a zip archive + complied_aar_path = get_compiled_aar_path(COMPILED_AAR_PATH_1, COMPILED_AAR_PATH_2) # two possible paths + shutil.unpack_archive(complied_aar_path, AAR_UNZIPPED_DIR, "zip") + + print("Adding libs to AAR...") + + # Copying 3rdparty libs from SDK into the AAR + for lib in external_libs: + for abi in abis: + os.makedirs(path.join(AAR_UNZIPPED_DIR, "prefab/modules/" + lib + "/libs/android." + abi)) + if path.exists(path.join(sdk_dir, "sdk/native/3rdparty/libs/" + abi, "lib" + lib + ".a")): + shutil.copy(path.join(sdk_dir, "sdk/native/3rdparty/libs/" + abi, "lib" + lib + ".a"), + path.join(AAR_UNZIPPED_DIR, "prefab/modules/" + lib + "/libs/android." + abi, "lib" + lib + ".a")) + else: + # One OpenCV library may have different dependency lists for different ABIs, but we can write only one + # full dependency list for all ABIs. So we just add empty .a library if this ABI doesn't have this dependency. + shutil.copy(path.join(AAR_UNZIPPED_DIR, "prefab/modules/templib/libs/android." + abi, "libtemplib.a"), + path.join(AAR_UNZIPPED_DIR, "prefab/modules/" + lib + "/libs/android." + abi, "lib" + lib + ".a")) + shutil.copy(path.join(AAR_UNZIPPED_DIR, "prefab/modules/templib/libs/android." + abi + "/abi.json"), + path.join(AAR_UNZIPPED_DIR, "prefab/modules/" + lib + "/libs/android." + abi + "/abi.json")) + shutil.copy(path.join(AAR_UNZIPPED_DIR, "prefab/modules/templib/module.json"), + path.join(AAR_UNZIPPED_DIR, "prefab/modules/" + lib + "/module.json")) + + # Copying OpenV libs from SDK into the AAR + for lib in opencv_libs: + for abi in abis: + os.makedirs(path.join(AAR_UNZIPPED_DIR, "prefab/modules/" + lib + "/libs/android." + abi)) + shutil.copy(path.join(sdk_dir, "sdk/native/staticlibs/" + abi, "lib" + lib + ".a"), + path.join(AAR_UNZIPPED_DIR, "prefab/modules/" + lib + "/libs/android." + abi, "lib" + lib + ".a")) + shutil.copy(path.join(AAR_UNZIPPED_DIR, "prefab/modules/templib/libs/android." + abi + "/abi.json"), + path.join(AAR_UNZIPPED_DIR, "prefab/modules/" + lib + "/libs/android." + abi + "/abi.json")) + os.makedirs(path.join(AAR_UNZIPPED_DIR, "prefab/modules/" + lib + "/include/opencv2")) + shutil.copy(path.join(sdk_dir, "sdk/native/jni/include/opencv2/" + lib.replace("opencv_", "") + ".hpp"), + path.join(AAR_UNZIPPED_DIR, "prefab/modules/" + lib + "/include/opencv2/" + lib.replace("opencv_", "") + ".hpp")) + shutil.copytree(path.join(sdk_dir, "sdk/native/jni/include/opencv2/" + lib.replace("opencv_", "")), + path.join(AAR_UNZIPPED_DIR, "prefab/modules/" + lib + "/include/opencv2/" + lib.replace("opencv_", ""))) + + # Adding dependencies list + module_json_text = { + "export_libraries": convert_deps_list_to_prefab(read_linked_libs(lib, abis), opencv_libs, external_libs), + "android": {}, + } + with open(path.join(AAR_UNZIPPED_DIR, "prefab/modules/" + lib + "/module.json"), "w") as f: + json.dump(module_json_text, f) + + for h_file in ("cvconfig.h", "opencv.hpp", "opencv_modules.hpp"): + shutil.copy(path.join(sdk_dir, "sdk/native/jni/include/opencv2/" + h_file), + path.join(AAR_UNZIPPED_DIR, "prefab/modules/opencv_core/include/opencv2/" + h_file)) + + + shutil.rmtree(path.join(AAR_UNZIPPED_DIR, "prefab/modules/templib")) + + # Creating final AAR zip archive + os.makedirs("outputs", exist_ok=True) + shutil.make_archive(final_aar_path, "zip", AAR_UNZIPPED_DIR, ".") + os.rename(final_aar_path + ".zip", final_aar_path) + + print("Creating local maven repo...") + + shutil.copy(final_aar_path, path.join(ANDROID_PROJECT_DIR, "OpenCV/opencv-release.aar")) + + subprocess.run(["./gradlew", "publishReleasePublicationToMyrepoRepository"], + shell=False, + cwd=ANDROID_PROJECT_DIR, + check=True) + + os.makedirs(path.join(FINAL_REPO_PATH, "org/opencv"), exist_ok=True) + shutil.move(path.join(ANDROID_PROJECT_DIR, "OpenCV/build/repo/org/opencv", MAVEN_PACKAGE_NAME), + path.join(FINAL_REPO_PATH, "org/opencv", MAVEN_PACKAGE_NAME)) + print("Done") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Builds AAR with static C++ libs from OpenCV SDK") + parser.add_argument('opencv_sdk_path') + parser.add_argument('--android_compile_sdk', default="26") + parser.add_argument('--android_min_sdk', default="21") + parser.add_argument('--android_target_sdk', default="26") + parser.add_argument('--java_version', default="1_8") + args = parser.parse_args() + + main(args) From 6079e225237eeed344953bf375b75ac62902046d Mon Sep 17 00:00:00 2001 From: Yuantao Feng Date: Tue, 7 Nov 2023 22:40:31 +0800 Subject: [PATCH 012/156] Merge pull request #24500 from fengyuentau:test_layer_fusion dnn (onnx): add subgraph fusion tests #24500 ### Pull Request Readiness Checklist See details at https://github.com/opencv/opencv/wiki/How_to_contribute#making-a-good-pull-request - [x] I agree to contribute to the project under Apache 2 License. - [x] To the best of my knowledge, the proposed patch is not based on a code under GPL or another license that is incompatible with OpenCV - [x] The PR is proposed to the proper branch - [x] There is a reference to the original bug report and related work - [x] There is accuracy test, performance test and test data in opencv_extra repository, if applicable Patch to opencv_extra has the same branch name. - [x] The feature is well documented and sample code can be built with the project CMake --- modules/dnn/test/test_graph_simplifier.cpp | 131 +++++++++++++++++++++ 1 file changed, 131 insertions(+) create mode 100644 modules/dnn/test/test_graph_simplifier.cpp diff --git a/modules/dnn/test/test_graph_simplifier.cpp b/modules/dnn/test/test_graph_simplifier.cpp new file mode 100644 index 0000000000..f6334f929a --- /dev/null +++ b/modules/dnn/test/test_graph_simplifier.cpp @@ -0,0 +1,131 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. + +#include "test_precomp.hpp" + +namespace opencv_test { namespace { + +class Test_Graph_Simplifier : public ::testing::Test { + public: + bool required; + + Test_Graph_Simplifier() : required(true) {} + + void test_conformance(const std::string &basename, const std::string &expected_layer) { + test(basename + std::string("/model"), std::vector{expected_layer}, std::string("dnn/onnx/conformance/node/")); + } + + void test(const std::string &basename, const std::string &expected_layer) { + test(basename, std::vector{expected_layer}); + } + + void test(const std::string &basename, const std::vector &expected_layers, const std::string &model_path_prefix = std::string("dnn/onnx/models/")) { + std::string model_path = findDataFile(model_path_prefix + basename + std::string(".onnx"), required); + auto net = readNet(model_path); + std::vector layers; + net.getLayerTypes(layers); + + // remove Const, Identity (output layer), __NetInputLayer__ (input layer) + layers.erase(std::remove_if(layers.begin(), layers.end(), [] (const std::string l) { return l == "Const" || l == "Identity" || l == "__NetInputLayer__"; }), layers.end()); + + EXPECT_EQ(layers, expected_layers); + } +}; + +TEST_F(Test_Graph_Simplifier, GeluSubGraph) { + test("gelu", "Gelu"); +} + +TEST_F(Test_Graph_Simplifier, GeluApproximationSubGraph) { + test("gelu_approximation", "GeluApproximation"); +} + +TEST_F(Test_Graph_Simplifier, LayerNormSubGraph) { + test("layer_norm_expanded", "LayerNormalization"); +} + +TEST_F(Test_Graph_Simplifier, ResizeSubgraph) { + /* Test for 6 subgraphs: + - GatherCastSubgraph + - MulCastSubgraph + - UpsampleSubgraph + - ResizeSubgraph1 + - ResizeSubgraph2 + - ResizeSubgraph3 + */ + test("upsample_unfused_torch1.2", std::vector{"BatchNorm", "Resize"}); + test("resize_nearest_unfused_opset11_torch1.3", std::vector{"BatchNorm", "Convolution", "Resize"}); + test("resize_nearest_unfused_opset11_torch1.4", std::vector{"BatchNorm", "Convolution", "Resize"}); + test("upsample_unfused_opset9_torch1.4", std::vector{"BatchNorm", "Convolution", "Resize"}); + test("two_resizes_with_shared_subgraphs", std::vector{"NaryEltwise", "Resize"}); +} + +TEST_F(Test_Graph_Simplifier, SoftmaxSubgraph) { + /* Test for 3 subgraphs + - SoftMaxSubgraph + - SoftMaxSubgraph2 (conformance) + - LogSoftMaxSubgraph (conformance) + */ + test("softmax_unfused", "Softmax"); + test_conformance("test_softmax_example_expanded", "Softmax"); + test_conformance("test_softmax_axis_2_expanded", "Softmax"); + test_conformance("test_softmax_default_axis_expanded", "Softmax"); + test_conformance("test_softmax_axis_0_expanded", "Softmax"); + test_conformance("test_softmax_axis_1_expanded", "Softmax"); + test_conformance("test_softmax_large_number_expanded", "Softmax"); + test_conformance("test_softmax_negative_axis_expanded", "Softmax"); + test_conformance("test_logsoftmax_axis_2_expanded", "Softmax"); + test_conformance("test_logsoftmax_example_1_expanded", "Softmax"); + test_conformance("test_logsoftmax_negative_axis_expanded", "Softmax"); + test_conformance("test_logsoftmax_axis_0_expanded", "Softmax"); + test_conformance("test_logsoftmax_axis_1_expanded", "Softmax"); + test_conformance("test_logsoftmax_large_number_expanded", "Softmax"); + test_conformance("test_logsoftmax_default_axis_expanded", "Softmax"); +} + +TEST_F(Test_Graph_Simplifier, HardSwishSubgraph) { + test_conformance("test_hardswish_expanded", "HardSwish"); +} + +TEST_F(Test_Graph_Simplifier, CeluSubgraph) { + test_conformance("test_celu_expanded", "Celu"); +} + +TEST_F(Test_Graph_Simplifier, NormalizeSubgraph) { + /* Test for 6 subgraphs + - NormalizeSubgraph1 + - NormalizeSubgraph2 + - NormalizeSubgraph2_2 + - NormalizeSubgraph3 + - NormalizeSubgraph4 + - NormalizeSubgraph5 + */ + test("reduceL2_subgraph_2", "Normalize"); + test("reduceL2_subgraph", "Normalize"); + test("normalize_fusion", "Normalize"); +} + +TEST_F(Test_Graph_Simplifier, BatchNormalizationSubgraph) { + /* Test for 2 subgraphs + - BatchNormalizationSubgraph1 + - BatchNormalizationSubgraph2 + */ + test("frozenBatchNorm2d", "BatchNorm"); + test("batch_norm_subgraph", "BatchNorm"); +} + +TEST_F(Test_Graph_Simplifier, ExpandSubgraph) { + test("expand_neg_batch", "Expand"); +} + +TEST_F(Test_Graph_Simplifier, MishSubgraph) { + /* Test for 2 subgraphs + - SoftplusSubgraph + - MishSubgraph + */ + test("mish_no_softplus", "Mish"); + test("mish", "Mish"); +} + +}} From fb352e3098333af130aa01202cb90a5d9c849620 Mon Sep 17 00:00:00 2001 From: huafengchun Date: Fri, 3 Nov 2023 16:38:37 +0800 Subject: [PATCH 013/156] Link lib_acl_op_compiler when compile with CANN --- cmake/OpenCVFindCANN.cmake | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/cmake/OpenCVFindCANN.cmake b/cmake/OpenCVFindCANN.cmake index e1cd054a37..913c1887e7 100644 --- a/cmake/OpenCVFindCANN.cmake +++ b/cmake/OpenCVFindCANN.cmake @@ -46,6 +46,17 @@ if(CANN_INSTALL_DIR) set(HAVE_CANN OFF) return() endif() + # * libacl_op_compiler.so + set(lib_acl_op_compiler "${CANN_INSTALL_DIR}/lib64") + find_library(found_lib_acl_op_compiler NAMES acl_op_compiler PATHS ${lib_acl_op_compiler} NO_DEFAULT_PATH) + if(found_lib_acl_op_compiler) + set(lib_acl_op_compiler ${found_lib_acl_op_compiler}) + message(STATUS "CANN: libacl_op_compiler.so is found at ${lib_acl_op_compiler}") + else() + message(STATUS "CANN: Missing libacl_op_compiler.so. Turning off HAVE_CANN") + set(HAVE_CANN OFF) + return() + endif() # * libgraph.so set(lib_graph "${CANN_INSTALL_DIR}/compiler/lib64") find_library(found_lib_graph NAMES graph PATHS ${lib_graph} NO_DEFAULT_PATH) @@ -90,6 +101,7 @@ if(CANN_INSTALL_DIR) set(libs_cann "") list(APPEND libs_cann ${lib_ascendcl}) + list(APPEND libs_cann ${lib_acl_op_compiler}) list(APPEND libs_cann ${lib_opsproto}) list(APPEND libs_cann ${lib_graph}) list(APPEND libs_cann ${lib_ge_compiler}) From 9d0c8a9edbe4238af02085bfb7dd4c5e499d3b80 Mon Sep 17 00:00:00 2001 From: Abduragim Shtanchaev <44877829+Abdurrahheem@users.noreply.github.com> Date: Wed, 8 Nov 2023 12:56:21 +0400 Subject: [PATCH 014/156] Merge pull request #24445 from Abdurrahheem:ash/dev_einsum_pref MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Einsum Layer Performance Test #24445 ## This PR adds performance tests for Einsum Layer. See below results of performance test on different inputs **Notation:** - WX: windows10_x64 - MX: macos_x64 - MA: macos_arm64 - UX: ubuntu_x64 - UA: ubuntu_arm64 All data in ms (milliseconds). Gemm is backend for matrix multiplication --- Benchmarks: | Equation | Inputs Mat Dims | UX (ms) | UA (ms) | MX (ms) | MA (ms) | WX (ms) | |-------------------------|-----------------------------------|----------------|---------|---------|---------|---------| | "ij, jk -> ik" | [2, 3], [3,2] | 0.04 ± 0.00 | - | - | - | - | | "ij, jk -> ik" | [20, 30], [30,20] | 0.08 ± 0.00 | - | - | - | - | | "ij, jk -> ik" | [113, 127], [127,113] | 2.41 ± 0.05 | - | - | - | - | | "imkj, injs -> imnks" | [1, 4, 7, 9], [1, 5, 9, 8] | 0.11 ± 0.00 | - | - | - | - | | "imkj, injs -> imnks" | [1, 4, 70, 90], [1, 5, 90, 80] | 15.49 ± 0.46 | - | - | - | - | | "imkj, injs -> imnks" | [1, 4, 73, 91], [1, 5, 91, 57] | 11.53 ± 0.06 | - | - | - | - | | "ij -> i" | [30, 40] | 0.03 ± 0.00 | - | - | - | - | | "ij -> i" | [113, 374] | 0.13 ± 0.00 | - | - | - | - | | "...ij -> ...i" | [30, 40] | 0.03 ± 0.00 | - | - | - | - | | "...ij -> ...i" | [113, 374] | 0.13 ± 0.00 | - | - | - | - | | "...ij, ...jk -> ...ik" | [40, 50], [50,80] | 0.37 ± 0.01 | - | - | - | - | | "...ij, ...jk -> ...ik" | [47, 51], [51, 83] | 0.43 ± 0.01 | - | - | - | - | ----- ### Pull Request Readiness Checklist See details at https://github.com/opencv/opencv/wiki/How_to_contribute#making-a-good-pull-request - [x] I agree to contribute to the project under Apache 2 License. - [x] To the best of my knowledge, the proposed patch is not based on a code under GPL or another license that is incompatible with OpenCV - [x] The PR is proposed to the proper branch - [ ] There is a reference to the original bug report and related work - [x] There is accuracy test, performance test and test data in opencv_extra repository, if applicable Patch to opencv_extra has the same branch name. - [x] The feature is well documented and sample code can be built with the project CMake --- modules/dnn/perf/perf_einsum.cpp | 123 +++++++++++++++++++++++++++++++ 1 file changed, 123 insertions(+) create mode 100644 modules/dnn/perf/perf_einsum.cpp diff --git a/modules/dnn/perf/perf_einsum.cpp b/modules/dnn/perf/perf_einsum.cpp new file mode 100644 index 0000000000..c3706d3153 --- /dev/null +++ b/modules/dnn/perf/perf_einsum.cpp @@ -0,0 +1,123 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. + +#include "perf_precomp.hpp" + +namespace opencv_test { + +struct EinsumParams { + int inputSize; + int outputSize; + std::string equation; + std::vector einsumInpShapes; + EinsumParams(std::string equation_, int inputSize_, int outputSize_, std::vector einsumInpShapes_ = std::vector()) + { + inputSize = inputSize_; + outputSize = outputSize_; + equation = equation_; + einsumInpShapes = einsumInpShapes_; + } +}; + +static inline void PrintTo(const EinsumParams& params, ::std::ostream* os) { + (*os) << "Eqiation=" << params.equation << ", " + << "InputSize=" << params.inputSize << ", " + << "OutputSize=" << params.outputSize << ", "; + + (*os) << "InputShape={"; + for(int i = 0; i < params.einsumInpShapes.size(); i++) + { + (*os) << "{"; + for(int j = 0; j < params.einsumInpShapes[i].size(); j++) + { + (*os) << params.einsumInpShapes[i][j] << ((j < params.einsumInpShapes[i].size() - 1) ? ", " : ""); + } + (*os) << ((i < params.einsumInpShapes.size() - 1) ? "}, " : "}"); + } + (*os) << "}"; +} + +// test cases +static const EinsumParams testEinsumConfigs[] = { + // TODO: Add tests with one input after ellips merge + {"ij, jk -> ik", 2, 1, {{2, 3}, {3, 2}}}, + {"ij, jk -> ik", 2, 1, {{20, 30}, {30, 20}}}, + {"ij, jk -> ik", 2, 1, {{113, 127}, {127, 113}}}, + + {"imkj, injs -> imnks", 2, 1, {{1, 4, 7, 9}, {1, 5, 9, 8}}}, + {"imkj, injs -> imnks", 2, 1, {{1, 4, 70, 90}, {1, 5, 90, 80}}}, + {"imkj, injs -> imnks", 2, 1, {{1, 4, 73, 91}, {1, 5, 91, 57}}}, + + {"ij -> i", 1, 1, {{30, 40}}}, + {"ij -> i", 1, 1, {{113, 374}}}, + + {"...ij -> ...i", 1, 1, {{30, 40}}}, + {"...ij -> ...i", 1, 1, {{113, 374}}}, + + {"...ij, ...jk -> ...ik", 2, 1, {{40, 50}, {50, 80}}}, + {"...ij, ...jk -> ...ik", 2, 1, {{47, 51}, {51, 83}}}, +}; + +class Layer_Einsum: public TestBaseWithParam {}; + +PERF_TEST_P_(Layer_Einsum, einsum) { + const EinsumParams& params = GetParam(); + LayerParams lp; + lp.type = "Einsum"; + lp.name = "testEinsum"; + lp.set("equation", params.equation); + lp.set("inputSize", params.inputSize); + lp.set("outputSize", params.outputSize); + + CV_CheckFalse(params.einsumInpShapes.empty(), "ERROR no inputs shapes provided"); + + for (int i = 0; i < params.einsumInpShapes.size(); i++) { + lp.set("inputShapes" + cv::format("%d", i), DictValue::arrayInt(params.einsumInpShapes[i].begin(), params.einsumInpShapes[i].size())); + } + + Net net; + std::vector inputs; + std::vector input_names; + if (params.inputSize == 1){ + + // create inputs + inputs.emplace_back(Mat(params.einsumInpShapes[0].size(), params.einsumInpShapes[0].data(), CV_32FC1)); + + int id = net.addLayerToPrev(lp.name, lp.type, lp); + net.connect(0, 0, id, 0); + + input_names.emplace_back("input1"); + + } else { + + // create inputs + inputs.emplace_back(Mat(params.einsumInpShapes[0].size(), params.einsumInpShapes[0].data(), CV_32FC1)); + inputs.emplace_back(Mat(params.einsumInpShapes[1].size(), params.einsumInpShapes[1].data(), CV_32FC1)); + + int id = net.addLayerToPrev(lp.name, lp.type, lp); + net.connect(0, 0, id, 0); + net.connect(0, 1, id, 1); + + input_names.emplace_back("input1"); + input_names.emplace_back("input2"); + } + + //warm up + net.setInputsNames(input_names); + for (int i = 0; i < input_names.size(); i++){ + net.setInput(inputs[i], input_names[i]); + } + Mat out = net.forward(); + + std::vector outputs; + TEST_CYCLE() + { + net.forward(outputs, "testEinsum"); + } + SANITY_CHECK_NOTHING(); +} + +INSTANTIATE_TEST_CASE_P(/**/, Layer_Einsum, testing::ValuesIn(testEinsumConfigs)); + +}; //namespace From f85cf5d7f965f39f4d590fbc5cfcfd4f829a4bf8 Mon Sep 17 00:00:00 2001 From: Maksym Ivashechkin Date: Sun, 5 Nov 2023 21:21:59 +0000 Subject: [PATCH 015/156] Update usac tutorial. --- doc/opencv.bib | 82 +++++++++++++ .../interactive_calibration.markdown | 1 + .../calib3d/table_of_content_calib3d.markdown | 1 + doc/tutorials/calib3d/usac.markdown | 112 ++++++------------ 4 files changed, 121 insertions(+), 75 deletions(-) diff --git a/doc/opencv.bib b/doc/opencv.bib index 6071cc3e1b..6632271e4a 100644 --- a/doc/opencv.bib +++ b/doc/opencv.bib @@ -1385,3 +1385,85 @@ YEAR = {2016}, MONTH = {October}, } +@inproceedings{BarathGCRANSAC, + author = {Barath, Daniel and Matas, Jiri}, + title = {Graph-Cut RANSAC}, + booktitle = {Proceedings of the IEEE Conference on Computer Vision and Pattern Recognition (CVPR)}, + month = {June}, + year = {2018} +} +@misc{barath2019progressive, + title={Progressive NAPSAC: sampling from gradually growing neighborhoods}, + author={Barath, Daniel and Ivashechkin, Maksym and Matas, Jiri}, + year={2019}, + eprint={1906.02295}, + archivePrefix={arXiv}, + primaryClass={cs.CV} +} +@inproceedings{BarathMAGSAC, + author = {Barath, Daniel and Noskova, Jana and Ivashechkin, Maksym and Matas, Jiri}, + title = {MAGSAC++, a Fast, Reliable and Accurate Robust Estimator}, + booktitle = {Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR)}, + month = {June}, + year = {2020} +} +@inproceedings{ChumPROSAC, + title = {Matching with {PROSAC} - Progressive Sampling Consensus}, + author = {Chum, Ondrej and Matas, Jiri}, + booktitle = {Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR)}, + year = {2005} +} +@inproceedings{ChumLORANSAC, + title = {Locally Optimized {RANSAC}}, + author = {Chum, Ondrej and Matas, Jiri and Kittler, Josef}, + booktitle = {DAGM}, + year = {2003} +} +@inproceedings{ChumEpipolar, + author={Chum, Ondrej and Werner, Tomas and Matas, Jiri}, + booktitle={Proceedings of the 17th International Conference on Pattern Recognition. ICPR 2004}, + title={Epipolar geometry estimation via RANSAC benefits from the oriented epipolar constraint}, + year={2004}, + volume={1}, + pages={112-115 Vol.1} +} +@inproceedings{ChumDominant, + title = {Epipolar Geometry Estimation Unaffected by the Dominant Plane}, + author = {Chum, Ondrej and Werner, Tomas and Matas, Jiri.}, + booktitle = {Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR)}, + year = {2005} +} +@article{FischlerRANSAC, + author = {Fischler, Martin A. and Bolles, Robert C.}, + title = {Random Sample Consensus: A Paradigm for Model Fitting with Applications to Image Analysis and Automated Cartography}, + year = {1981}, + publisher = {Association for Computing Machinery}, + volume = {24}, + number = {6}, + month = {jun}, + pages = {381–395}, + numpages = {15} +} +@article{Matas2005RandomizedRW, + title={Randomized RANSAC with sequential probability ratio test}, + author={Matas, Jiri and Chum, Ondrej}, + journal={Tenth IEEE International Conference on Computer Vision (ICCV) Volume 1}, + year={2005}, + volume={2}, + pages={1727-1732 Vol. 2} +} +@inproceedings{MyattNAPSAC, + author = {Myatt, D. and Torr, Philip and Nasuto, Slawomir and Bishop, John and Craddock, R.}, + year = {2002}, + booktitle = {Proceedings of the British Machine Vision Conference (BMVC)}, + title = {NAPSAC: High Noise, High Dimensional Robust Estimation - it's in the Bag} +} +@article{SteweniusRecent, + author = {Stewenius, Henrik and Engels, Christopher and Nister, David}, + year = {2006}, + month = {06}, + pages = {284-294}, + title = {Recent developments on direct relative orientation}, + volume = {60}, + journal = {ISPRS Journal of Photogrammetry and Remote Sensing} +} diff --git a/doc/tutorials/calib3d/interactive_calibration/interactive_calibration.markdown b/doc/tutorials/calib3d/interactive_calibration/interactive_calibration.markdown index a50058ba84..b02e6ecfd2 100644 --- a/doc/tutorials/calib3d/interactive_calibration/interactive_calibration.markdown +++ b/doc/tutorials/calib3d/interactive_calibration/interactive_calibration.markdown @@ -4,6 +4,7 @@ Interactive camera calibration application {#tutorial_interactive_calibration} @tableofcontents @prev_tutorial{tutorial_real_time_pose} +@next_tutorial{tutorial_usac} | | | | -: | :- | diff --git a/doc/tutorials/calib3d/table_of_content_calib3d.markdown b/doc/tutorials/calib3d/table_of_content_calib3d.markdown index 5fc6e591e9..4be1058dd3 100644 --- a/doc/tutorials/calib3d/table_of_content_calib3d.markdown +++ b/doc/tutorials/calib3d/table_of_content_calib3d.markdown @@ -6,3 +6,4 @@ Camera calibration and 3D reconstruction (calib3d module) {#tutorial_table_of_co - @subpage tutorial_camera_calibration - @subpage tutorial_real_time_pose - @subpage tutorial_interactive_calibration +- @subpage tutorial_usac diff --git a/doc/tutorials/calib3d/usac.markdown b/doc/tutorials/calib3d/usac.markdown index df9e25f907..7fb18aa87c 100644 --- a/doc/tutorials/calib3d/usac.markdown +++ b/doc/tutorials/calib3d/usac.markdown @@ -1,14 +1,19 @@ ---- -author: -- Maksym Ivashechkin -bibliography: 'bibs.bib' -csl: 'acm-sigchi-proceedings.csl' -date: August 2020 -title: 'Google Summer of Code: Improvement of Random Sample Consensus in OpenCV' -... +USAC: Improvement of Random Sample Consensus in OpenCV {#tutorial_usac} +============================== + +@tableofcontents + +@prev_tutorial{tutorial_interactive_calibration} + +| | | +| -: | :- | +| Original author | Maksym Ivashechkin | +| Compatibility | OpenCV >= 4.0 | + +This work was integrated as part of the Google Summer of Code (August 2020). Contribution -============ +------ The integrated part to OpenCV `calib3d` module is RANSAC-based universal framework USAC (`namespace usac`) written in C++. The framework includes @@ -20,25 +25,25 @@ components: 1. Sampling method: - 1. Uniform – standard RANSAC sampling proposed in \[8\] which draw + 1. Uniform – standard RANSAC sampling proposed in @cite FischlerRANSAC which draw minimal subset independently uniformly at random. *The default option in proposed framework*. - 2. PROSAC – method \[4\] that assumes input data points sorted by + 2. PROSAC – method @cite ChumPROSAC that assumes input data points sorted by quality so sampling can start from the most promising points. Correspondences for this method can be sorted e.g., by ratio of descriptor distances of the best to second match obtained from SIFT detector. *This is method is recommended to use because it can find good model and terminate much earlier*. - 3. NAPSAC – sampling method \[10\] which takes initial point + 3. NAPSAC – sampling method @cite MyattNAPSAC which takes initial point uniformly at random and the rest of points for minimal sample in the neighborhood of initial point. This is method can be potentially useful when models are localized. For example, for plane fitting. However, in practise struggles from degenerate issues and defining optimal neighborhood size. - 4. Progressive-NAPSAC – sampler \[2\] which is similar to NAPSAC, + 4. Progressive-NAPSAC – sampler @cite barath2019progressive which is similar to NAPSAC, although it starts from local and gradually converges to global sampling. This method can be quite useful if local models are expected but distribution of data can be arbitrary. The @@ -56,7 +61,7 @@ components: default option in framework*. The model might not have as many inliers as using RANSAC score, however will be more accurate. - 3. MAGSAC – threshold-free method \[3\] to compute score. Using, + 3. MAGSAC – threshold-free method @cite BarathMAGSAC to compute score. Using, although, maximum sigma (standard deviation of noise) level to marginalize residual of point over sigma. Score of the point represents likelihood of point being inlier. *Recommended option @@ -86,7 +91,7 @@ components: 4. Degeneracy: - 1. DEGENSAC – method \[7\] which for Fundamental matrix estimation + 1. DEGENSAC – method @cite ChumDominant which for Fundamental matrix estimation efficiently verifies and recovers model which has at least 5 points in minimal sample lying on the dominant plane. @@ -96,11 +101,11 @@ components: in minimal sample lie on the same side w.r.t. to any line crossing any two points in sample (does not assume reflection). - 3. Oriented epipolar constraint – method \[6\] for epipolar + 3. Oriented epipolar constraint – method @cite ChumEpipolar for epipolar geometry which verifies model (fundamental and essential matrix) to have points visible in the front of the camera. -5. SPRT verification – method \[9\] which verifies model by its +5. SPRT verification – method @cite Matas2005RandomizedRW which verifies model by its evaluation on randomly shuffled points using statistical properties given by probability of inlier, relative time for estimation, average number of output models etc. Significantly speeding up @@ -109,17 +114,17 @@ components: 6. Local Optimization: - 1. Locally Optimized RANSAC – method \[5\] that iteratively + 1. Locally Optimized RANSAC – method @cite ChumLORANSAC that iteratively improves so-far-the-best model by non-minimal estimation. *The default option in framework. This procedure is the fastest and not worse than others local optimization methods.* - 2. Graph-Cut RANSAC – method \[1\] that refine so-far-the-best + 2. Graph-Cut RANSAC – method @cite BarathGCRANSAC that refine so-far-the-best model, however, it exploits spatial coherence of the data points. *This procedure is quite precise however computationally slower.* - 3. Sigma Consensus – method \[3\] which improves model by applying + 3. Sigma Consensus – method @cite BarathMAGSAC which improves model by applying non-minimal weighted estimation, where weights are computed with the same logic as in MAGSAC score. This method is better to use together with MAGSAC score. @@ -152,7 +157,7 @@ components: 4. Essential matrix – 4 null vectors are found using Gaussian elimination. Then the solver based on Gröbner basis - described in \[11\] is used. Essential matrix can be computed + described in @cite SteweniusRecent is used. Essential matrix can be computed only if LAPACK or Eigen are installed as it requires eigen decomposition with complex @@ -180,12 +185,12 @@ sequentially. However, using default options of framework parallel RANSAC is not deterministic since it depends on how often each thread is running. The easiest way to make it deterministic is using PROSAC sampler without SPRT and Local Optimization and not for Fundamental -matrix, because they internally use random generators.\ -\ +matrix, because they internally use random generators. + For NAPSAC, Progressive NAPSAC or Graph-Cut methods is required to build a neighborhood graph. In framework there are 3 options to do it: -1. `NEIGH_FLANN_KNN` – estimate neighborhood graph using OpenCV FLANN +1. NEIGH_FLANN_KNN – estimate neighborhood graph using OpenCV FLANN K nearest-neighbors. The default value for KNN is 7. KNN method may work good for sampling but not good for GC-RANSAC. @@ -193,14 +198,14 @@ a neighborhood graph. In framework there are 3 options to do it: points which distance is less than 20 pixels. 3. `NEIGH_GRID` – for finding points’ neighborhood tiles points in - cells using hash-table. The method is described in \[2\]. Less + cells using hash-table. The method is described in @cite barath2019progressive. Less accurate than `NEIGH_FLANN_RADIUS`, although significantly faster. Note, `NEIGH_FLANN_RADIUS` and `NEIGH_FLANN_RADIUS` are not able to PnP -solver, since there are 3D object points.\ -\ -New flags: +solver, since there are 3D object points. +New flags: +------ 1. `USAC_DEFAULT` – has standard LO-RANSAC. 2. `USAC_PARALLEL` – has LO-RANSAC and RANSACs run in parallel. @@ -220,9 +225,10 @@ New flags: Every flag uses SPRT verification. And in the end the final so-far-the-best model is polished by non minimal estimation of all found -inliers.\ -\ +inliers. + A few other important parameters: +------ 1. `randomGeneratorState` – since every USAC solver is deterministic in OpenCV (i.e., for the same points and parameters returns the @@ -240,6 +246,7 @@ A few other important parameters: estimation on low number of points is faster and more robust. Samples: +------ There are three new sample files in opencv/samples directory. @@ -260,48 +267,3 @@ There are three new sample files in opencv/samples directory. 3. `essential_mat_reconstr.py` – the same functionality as in .cpp file, however instead of clustering points to plane the 3D map of object points is plot. - -References: - -1\. Daniel Barath and Jiří Matas. 2018. Graph-Cut RANSAC. In *Proceedings -of the iEEE conference on computer vision and pattern recognition*, -6733–6741. - -2\. Daniel Barath, Maksym Ivashechkin, and Jiri Matas. 2019. Progressive -NAPSAC: Sampling from gradually growing neighborhoods. *arXiv preprint -arXiv:1906.02295*. - -3\. Daniel Barath, Jana Noskova, Maksym Ivashechkin, and Jiri Matas. -2020. MAGSAC++, a fast, reliable and accurate robust estimator. In -*Proceedings of the iEEE/CVF conference on computer vision and pattern -recognition (cVPR)*. - -4\. O. Chum and J. Matas. 2005. Matching with PROSAC-progressive sample -consensus. In *Computer vision and pattern recognition*. - -5\. O. Chum, J. Matas, and J. Kittler. 2003. Locally optimized RANSAC. In -*Joint pattern recognition symposium*. - -6\. O. Chum, T. Werner, and J. Matas. 2004. Epipolar geometry estimation -via RANSAC benefits from the oriented epipolar constraint. In -*International conference on pattern recognition*. - -7\. Ondrej Chum, Tomas Werner, and Jiri Matas. 2005. Two-view geometry -estimation unaffected by a dominant plane. In *2005 iEEE computer -society conference on computer vision and pattern recognition -(cVPR’05)*, 772–779. - -8\. M. A. Fischler and R. C. Bolles. 1981. Random sample consensus: A -paradigm for model fitting with applications to image analysis and -automated cartography. *Communications of the ACM*. - -9\. Jiri Matas and Ondrej Chum. 2005. Randomized RANSAC with sequential -probability ratio test. In *Tenth iEEE international conference on -computer vision (iCCV’05) volume 1*, 1727–1732. - -10\. D. R. Myatt, P. H. S. Torr, S. J. Nasuto, J. M. Bishop, and R. -Craddock. 2002. NAPSAC: High noise, high dimensional robust estimation. -In *In bMVC02*, 458–467. - -11\. Henrik Stewénius, Christopher Engels, and David Nistér. 2006. Recent -developments on direct relative orientation. From b1e0c4d119650b7065e794101857db9828ae6448 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Csaba=20Kert=C3=A9sz?= Date: Wed, 8 Nov 2023 11:41:50 +0100 Subject: [PATCH 016/156] Merge pull request #24243 from kecsap:4.x Fix gstreamer backend with manual pipelines #24243 - Fix broken seeking in audio/video playback - Fix broken audio playback - Fix unreliable seeking - Estimate frame count if it is not available directly - Return -1 for frame count and fps if it is not available. - Return 0 for fps if the video has variable frame rate - Enable and fix tests ### Pull Request Readiness Checklist See details at https://github.com/opencv/opencv/wiki/How_to_contribute#making-a-good-pull-request - [X] I agree to contribute to the project under Apache 2 License. - [X] To the best of my knowledge, the proposed patch is not based on a code under GPL or another license that is incompatible with OpenCV - [X] The PR is proposed to the proper branch - [-] There is a reference to the original bug report and related work => Reproducible test provided - [-] There is accuracy test, performance test and test data in opencv_extra repository, if applicable Patch to opencv_extra has the same branch name. - [X] The feature is well documented and sample code can be built with the project CMake 1. Download two test videos: ```bash wget https://github.com/ietf-wg-cellar/matroska-test-files/raw/master/test_files/test1.mkv wget https://test-videos.co.uk/vids/jellyfish/mkv/360/Jellyfish_360_10s_5MB.mkv ``` 2. I modified a OpenCV videoio sample to demonstrate the problem, here it is the patch: http://dpaste.com//C9MAT2K6W 3. Build the sample, on Ubuntu: ```bash g++ -g videocapture_audio_combination.cpp -I/usr/include/opencv4 `pkg-config --libs --cflags opencv4` -o videocapture_audio_combination ``` 4. Play an audio stream with seeking BEFORE the fix: ```bash $ ./videocapture_audio_combination --audio "filesrc location=test1.mkv ! queue ! matroskademux name=demux demux.audio_0 ! decodebin ! audioconvert ! appsink"[ERROR:0@0.009] global cap.cpp:164 open VIDEOIO(GSTREAMER): raised OpenCV exception: OpenCV(4.8.0-dev) ./modules/videoio/src/cap_gstreamer.cpp:153: error: (-215:Assertion failed) ptr in function 'get' [ WARN:0@0.009] global cap.cpp:204 open VIDEOIO(GSTREAMER): backend is generally available but can't be used to capture by name ERROR! Can't to open file: filesrc location=test1.mkv ! queue ! matroskademux name=demux demux.audio_0 ! decodebin ! audioconvert ! appsink ``` 5. Play a video stream with seeking BEFORE the fix: ```bash $ ./videocapture_audio_combination --audio "filesrc location=Jellyfish_360_10s_5MB.mkv ! queue ! matroskademux name=demux demux.video_0 ! decodebin ! videoconvert ! video/x-raw, format=BGR ! appsink drop=1" [ WARN:0@0.034] global cap_gstreamer.cpp:1728 open OpenCV | GStreamer warning: Cannot query video position: status=1, value=22, duration=300 CAP_PROP_AUDIO_DATA_DEPTH: CV_16S CAP_PROP_AUDIO_SAMPLES_PER_SECOND: 44100 CAP_PROP_AUDIO_TOTAL_CHANNELS: 0 CAP_PROP_AUDIO_TOTAL_STREAMS: [ WARN:0@0.034] global cap_gstreamer.cpp:1898 getProperty OpenCV | GStreamer: CAP_PROP_AUDIO_TOTAL_STREAMS property is not supported 0 [ WARN:0@0.034] global cap_gstreamer.cpp:1817 getProperty OpenCV | GStreamer: CAP_PROP_POS_MSEC property result may be unrealiable: https://github.com/opencv/opencv/issues/19025 Timestamp: 0.6218 Timestamp: 33.1085 Timestamp: 67.1274 Timestamp: 100.1182 Timestamp: 133.1204 Timestamp: 167.1195 Timestamp: 200.1161 Timestamp: 233.1147 Timestamp: 267.1194 Timestamp: 300.1202 [ WARN:0@0.338] global cap_gstreamer.cpp:1949 setProperty OpenCV | GStreamer warning: GStreamer: unable to seek 0:00:00.338215907 3892572 0x5592899c7580 WARN basesrc gstbasesrc.c:3127:gst_base_src_loop: error: Internal data stream error. 0:00:00.338235884 3892572 0x5592899c7580 WARN basesrc gstbasesrc.c:3127:gst_base_src_loop: error: streaming stopped, reason not-linked (-1) 0:00:00.338264287 3892572 0x5592899c7580 WARN queue gstqueue.c:992:gst_queue_handle_sink_event: error: Internal data stream error. 0:00:00.338270329 3892572 0x5592899c7580 WARN queue gstqueue.c:992:gst_queue_handle_sink_event: error: streaming stopped, reason not-linked (-1) [ WARN:0@0.339] global cap_gstreamer.cpp:2784 handleMessage OpenCV | GStreamer warning: Embedded video playback halted; module filesrc0 reported: Internal data stream error. [ WARN:0@0.339] global cap_gstreamer.cpp:1199 startPipeline OpenCV | GStreamer warning: unable to start pipeline Number of audio samples: 0 Number of video frames: 10 [ WARN:0@0.339] global cap_gstreamer.cpp:1164 isPipelinePlaying OpenCV | GStreamer warning: GStreamer: pipeline have not been created ``` 6. Play an audio stream with seeking AFTER the fix: ```bash $ ./videocapture_audio_combination --audio "filesrc location=test1.mkv ! queue ! matroskademux name=demux demux.audio_0 ! decodebin ! audioconvert ! appsink"CAP_PROP_AUDIO_DATA_DEPTH: CV_16S CAP_PROP_AUDIO_SAMPLES_PER_SECOND: 48000 CAP_PROP_AUDIO_TOTAL_CHANNELS: 2 CAP_PROP_AUDIO_TOTAL_STREAMS: [ WARN:0@0.025] global cap_gstreamer.cpp:1903 getProperty OpenCV | GStreamer: CAP_PROP_AUDIO_TOTAL_STREAMS property is not supported 0 Timestamp: 0.0000 Timestamp: 24.0000 Timestamp: 48.0000 Timestamp: 72.0000 Timestamp: 96.0000 Timestamp: 120.0000 Timestamp: 144.0000 Timestamp: 168.0000 Timestamp: 192.0000 Timestamp: 216.0000 Timestamp: 3500.0000 Timestamp: 3504.0000 Timestamp: 3528.0000 Timestamp: 3552.0000 Timestamp: 3576.0000 Timestamp: 3600.0000 Timestamp: 3624.0000 Timestamp: 3648.0000 Timestamp: 3672.0000 Timestamp: 3696.0000 Timestamp: 3720.0000 Timestamp: 3744.0000 Timestamp: 3768.0000 Timestamp: 3792.0000 Timestamp: 3816.0000 Timestamp: 3840.0000 Timestamp: 3864.0000 Timestamp: 3888.0000 Timestamp: 3912.0000 Timestamp: 3936.0000 ``` 7. Play a video stream with seeking AFTER the fix: ```bash $ ./videocapture_audio_combination --audio "filesrc location=Jellyfish_360_10s_5MB.mkv ! queue ! matroskademux name=demux demux.video_0 ! decodebin ! videoconvert ! video/x-raw, format=BGR ! appsink drop=1" [ WARN:0@0.033] global cap_gstreamer.cpp:1746 open OpenCV | GStreamer warning: Cannot query video position: status=1, value=22, duration=300 CAP_PROP_AUDIO_DATA_DEPTH: CV_16S CAP_PROP_AUDIO_SAMPLES_PER_SECOND: 44100 CAP_PROP_AUDIO_TOTAL_CHANNELS: 0 CAP_PROP_AUDIO_TOTAL_STREAMS: [ WARN:0@0.034] global cap_gstreamer.cpp:1903 getProperty OpenCV | GStreamer: CAP_PROP_AUDIO_TOTAL_STREAMS property is not supported 0 Timestamp: 0.0000 Timestamp: 33.0000 Timestamp: 67.0000 Timestamp: 100.0000 Timestamp: 133.0000 Timestamp: 167.0000 Timestamp: 200.0000 Timestamp: 233.0000 Timestamp: 267.0000 Timestamp: 300.0000 0:00:00.335931693 3893501 0x55bbe76ad920 WARN matroskareadcommon matroska-read-common.c:759:gst_matroska_read_common_parse_skip: Unknown CueTrackPositions subelement 0xf0 - ignoring 0:00:00.335952823 3893501 0x55bbe76ad920 WARN matroskareadcommon matroska-read-common.c:759:gst_matroska_read_common_parse_skip: Unknown CueTrackPositions subelement 0xf0 - ignoring 0:00:00.335988029 3893501 0x55bbe76ad920 WARN basesrc gstbasesrc.c:1742:gst_base_src_perform_seek: duplicate event found 184 Timestamp: 3467.0000 Timestamp: 3500.0000 Timestamp: 3533.0000 Timestamp: 3567.0000 Timestamp: 3600.0000 Timestamp: 3633.0000 Timestamp: 3667.0000 Timestamp: 3700.0000 Timestamp: 3733.0000 Timestamp: 3767.0000 Timestamp: 3800.0000 Timestamp: 3833.0000 Timestamp: 3867.0000 Timestamp: 3900.0000 Timestamp: 3933.0000 Timestamp: 3967.0000 Timestamp: 4000.0000 Timestamp: 4033.0000 Timestamp: 4067.0000 Timestamp: 4100.0000 ``` --- modules/videoio/src/cap_gstreamer.cpp | 144 ++++++++++++++++-------- modules/videoio/test/test_audio.cpp | 3 +- modules/videoio/test/test_gstreamer.cpp | 43 +++++++ modules/videoio/test/test_video_io.cpp | 21 ++-- 4 files changed, 158 insertions(+), 53 deletions(-) diff --git a/modules/videoio/src/cap_gstreamer.cpp b/modules/videoio/src/cap_gstreamer.cpp index 305d527ce9..cdaccabe45 100644 --- a/modules/videoio/src/cap_gstreamer.cpp +++ b/modules/videoio/src/cap_gstreamer.cpp @@ -114,6 +114,7 @@ template<> inline void GSafePtr_release(GstBuffer** pPtr) { if (pPtr) template<> inline void GSafePtr_release(GstSample** pPtr) { if (pPtr) { gst_sample_unref(*pPtr); *pPtr = NULL; } } template<> inline void GSafePtr_release(GstBus** pPtr) { if (pPtr) { gst_object_unref(G_OBJECT(*pPtr)); *pPtr = NULL; } } template<> inline void GSafePtr_release(GstMessage** pPtr) { if (pPtr) { gst_message_unref(*pPtr); *pPtr = NULL; } } +template<> inline void GSafePtr_release(GstQuery** pPtr) { if (pPtr) { gst_query_unref(*pPtr); *pPtr = NULL; } } template<> inline void GSafePtr_release(GMainLoop** pPtr) { if (pPtr) { g_main_loop_unref(*pPtr); *pPtr = NULL; } } template<> inline void GSafePtr_release(GstEncodingVideoProfile** pPtr) { if (pPtr) { gst_encoding_profile_unref(*pPtr); *pPtr = NULL; } } @@ -367,6 +368,7 @@ private: gint audioBitPerFrame; gint audioSampleSize; std::string audioFormat; + guint64 timestamp; Mat audioFrame; std::deque bufferAudioData; @@ -433,7 +435,8 @@ GStreamerCapture::GStreamerCapture() : audioSamplesPerSecond(44100), audioBitPerFrame(0), audioSampleSize(0), - audioFormat("S16LE") + audioFormat("S16LE"), + timestamp(0) , va_type(VIDEO_ACCELERATION_NONE) , hw_device(-1) {} @@ -680,6 +683,11 @@ bool GStreamerCapture::grabVideoFrame() stopFlag = true; emulatedFrameNumber++; } + if (usedVideoSample) + { + auto *buffer = gst_sample_get_buffer((GstSample*)usedVideoSample); + timestamp = GST_BUFFER_PTS(buffer); + } returnFlag = true; } } @@ -792,6 +800,7 @@ bool GStreamerCapture::grabAudioFrame() CV_LOG_ERROR(NULL, "GStreamer: Failed. Buffer is empty"); return false; } + timestamp = GST_BUFFER_PTS(buf); if (!gst_buffer_map(buf, &map_info, GST_MAP_READ)) { CV_LOG_ERROR(NULL, "GStreamer: Failed to map GStreamer buffer to system memory"); @@ -1389,6 +1398,7 @@ bool GStreamerCapture::open(const String &filename_, const cv::VideoCaptureParam GSafePtr uri; GSafePtr bus; + GSafePtr queue; GSafePtr uridecodebin; GSafePtr color; GSafePtr convert; @@ -1493,6 +1503,7 @@ bool GStreamerCapture::open(const String &filename_, const cv::VideoCaptureParam if (strstr(name, "opencvsink") != NULL || strstr(name, "appsink") != NULL) { sink.attach(GST_ELEMENT(gst_object_ref(element))); + audiosink.attach(GST_ELEMENT(gst_object_ref(element))); } else if (strstr(name, COLOR_ELEM_NAME) != NULL) { @@ -1534,6 +1545,8 @@ bool GStreamerCapture::open(const String &filename_, const cv::VideoCaptureParam if (videoStream >= 0) { + queue.reset(gst_element_factory_make("queue", NULL)); + CV_Assert(queue); sink.reset(gst_element_factory_make("appsink", NULL)); CV_Assert(sink); // videoconvert (in 0.10: ffmpegcolorspace, in 1.x autovideoconvert) @@ -1541,7 +1554,7 @@ bool GStreamerCapture::open(const String &filename_, const cv::VideoCaptureParam color.reset(gst_element_factory_make(COLOR_ELEM, NULL)); CV_Assert(color); - gst_bin_add_many(GST_BIN(pipeline.get()), uridecodebin.get(), color.get(), sink.get(), NULL); + gst_bin_add_many(GST_BIN(pipeline.get()), queue.get(), uridecodebin.get(), color.get(), sink.get(), NULL); if (element_from_uri) { @@ -1566,6 +1579,8 @@ bool GStreamerCapture::open(const String &filename_, const cv::VideoCaptureParam } if (audioStream >= 0) { + queue.reset(gst_element_factory_make("queue", NULL)); + CV_Assert(queue); convert.reset(gst_element_factory_make("audioconvert", NULL)); resample.reset(gst_element_factory_make("audioresample", NULL)); audiosink.reset(gst_element_factory_make("appsink", NULL)); @@ -1573,7 +1588,7 @@ bool GStreamerCapture::open(const String &filename_, const cv::VideoCaptureParam CV_Assert(resample); CV_Assert(audiosink); - gst_bin_add_many (GST_BIN (pipeline.get()), uridecodebin.get(), convert.get(), resample.get(), audiosink.get(), NULL); + gst_bin_add_many (GST_BIN (pipeline.get()), queue.get(), uridecodebin.get(), convert.get(), resample.get(), audiosink.get(), NULL); if (!gst_element_link_many (convert.get(), resample.get(), audiosink.get(), NULL)) { CV_WARN("GStreamer(audio): cannot link convert -> resample -> sink"); @@ -1646,14 +1661,17 @@ bool GStreamerCapture::open(const String &filename_, const cv::VideoCaptureParam } if (manualpipeline) { - GSafePtr peer_caps; - GSafePtr sink_pad; - sink_pad.attach(gst_element_get_static_pad(sink, "sink")); - peer_caps.attach(gst_pad_peer_query_caps(sink_pad, NULL)); - if (!gst_caps_can_intersect(caps, peer_caps)) + if (videoStream >= 0) { - caps.attach(gst_caps_from_string("video/x-raw, format=(string){UYVY,YUY2,YVYU,NV12,NV21,YV12,I420,BGRA,RGBA,BGRx,RGBx,GRAY16_LE,GRAY16_BE}")); - CV_Assert(caps); + GSafePtr peer_caps; + GSafePtr sink_pad; + sink_pad.attach(gst_element_get_static_pad(sink, "sink")); + peer_caps.attach(gst_pad_peer_query_caps(sink_pad, NULL)); + if (!gst_caps_can_intersect(caps, peer_caps)) + { + caps.attach(gst_caps_from_string("video/x-raw, format=(string){UYVY,YUY2,YVYU,NV12,NV21,YV12,I420,BGRA,RGBA,BGRx,RGBx,GRAY16_LE,GRAY16_BE}")); + CV_Assert(caps); + } } } if (videoStream >= 0) @@ -1661,6 +1679,7 @@ bool GStreamerCapture::open(const String &filename_, const cv::VideoCaptureParam gst_app_sink_set_caps(GST_APP_SINK(sink.get()), caps); caps.release(); } + { GST_DEBUG_BIN_TO_DOT_FILE(GST_BIN(pipeline.get()), GST_DEBUG_GRAPH_SHOW_ALL, "pipeline-init"); @@ -1688,18 +1707,6 @@ bool GStreamerCapture::open(const String &filename_, const cv::VideoCaptureParam GSafePtr buffer_caps; buffer_caps.attach(gst_pad_get_current_caps(pad)); - GstFormat format; - - format = GST_FORMAT_DEFAULT; - if(!gst_element_query_duration(sink, format, &duration)) - { - handleMessage(pipeline); - CV_WARN("unable to query duration of stream"); - duration = -1; - } - - handleMessage(pipeline); - const GstStructure *structure = gst_caps_get_structure(buffer_caps, 0); // no lifetime transfer if (!gst_structure_get_int (structure, "width", &width) || !gst_structure_get_int (structure, "height", &height)) @@ -1708,13 +1715,55 @@ bool GStreamerCapture::open(const String &filename_, const cv::VideoCaptureParam } gint num = 0, denom=1; + bool fps_query_success = true; + if (!gst_structure_get_fraction(structure, "framerate", &num, &denom)) { CV_WARN("cannot query video fps"); + fps_query_success = false; } fps = (double)num/(double)denom; + // If num == 0 and denom == 1 -> variable frame rate video. + if (fps_query_success && !(num == 0 && denom == 1)) + { + GSafePtr query; + query.attach(gst_query_new_duration(GST_FORMAT_DEFAULT)); + + gboolean res = gst_element_query(pipeline.get(), query); + + if (res) + { + gst_query_parse_duration(query, NULL, &duration); + } + else if (fps != 0) + { + GSafePtr query2; + query2.attach(gst_query_new_duration(GST_FORMAT_TIME)); + gboolean res2 = gst_element_query(pipeline.get(), query2); + + if (res2) + { + gst_query_parse_duration(query2, NULL, &duration); + duration = static_cast((float)duration / GST_SECOND * fps); + CV_WARN("frame count is estimated by duration and fps"); + } + else + { + CV_WARN("unable to query duration of stream"); + duration = -1; + } + } + else + { + CV_WARN("unable to query frame count of stream and fps are not available to estimate it"); + duration = -1; + } + } + + handleMessage(pipeline); + { GstFormat format_; gint64 value_ = -1; @@ -1814,20 +1863,7 @@ double GStreamerCapture::getProperty(int propId) const switch(propId) { case CV_CAP_PROP_POS_MSEC: - CV_LOG_ONCE_WARNING(NULL, "OpenCV | GStreamer: CAP_PROP_POS_MSEC property result may be unrealiable: " - "https://github.com/opencv/opencv/issues/19025"); - if (audioStream != -1) - { - return usedVideoSampleTimeNS * 1e-6; - } - format = GST_FORMAT_TIME; - status = gst_element_query_position(sink.get(), CV_GST_FORMAT(format), &value); - if(!status) { - handleMessage(pipeline); - CV_WARN("GStreamer: unable to query position of stream"); - return 0; - } - return value * 1e-6; // nano seconds to milli seconds + return double(timestamp) / GST_MSECOND; case CV_CAP_PROP_POS_FRAMES: if (!isPosFramesSupported) { @@ -1859,7 +1895,7 @@ double GStreamerCapture::getProperty(int propId) const case CV_CAP_PROP_FPS: return fps; case CV_CAP_PROP_FRAME_COUNT: - return duration; + return (double)duration; case CV_CAP_PROP_BRIGHTNESS: case CV_CAP_PROP_CONTRAST: case CV_CAP_PROP_SATURATION: @@ -1936,13 +1972,15 @@ bool GStreamerCapture::setProperty(int propId, double value) return false; } - bool wasPlaying = this->isPipelinePlaying(); - if (wasPlaying) + bool needRestart = this->isPipelinePlaying() && (propId == CV_CAP_PROP_FRAME_WIDTH || propId == CV_CAP_PROP_FRAME_HEIGHT || propId == CV_CAP_PROP_FPS); + if (needRestart) { this->stopPipeline(); + } switch(propId) { case CV_CAP_PROP_POS_MSEC: + { if(!gst_element_seek_simple(GST_ELEMENT(pipeline.get()), GST_FORMAT_TIME, flags, (gint64) (value * GST_MSECOND))) { handleMessage(pipeline); @@ -1950,6 +1988,9 @@ bool GStreamerCapture::setProperty(int propId, double value) } else { + // Optimistically caching the target timestamp before reading the first frame from the new position since + // the timestamp in GStreamer can be reliable extracted from the read frames. + timestamp = (gint64)value; if (isPosFramesEmulated) { if (value == 0) @@ -1963,7 +2004,8 @@ bool GStreamerCapture::setProperty(int propId, double value) } } } - break; + return true; + } case CV_CAP_PROP_POS_FRAMES: { if (!isPosFramesSupported) @@ -1977,24 +2019,34 @@ bool GStreamerCapture::setProperty(int propId, double value) return true; } } - return false; CV_WARN("unable to seek"); + return false; } + // Certain mov and mp4 files seek incorrectly if the pipeline is not stopped before. + if (this->isPipelinePlaying()) { + this->stopPipeline(); + } + if(!gst_element_seek_simple(GST_ELEMENT(pipeline.get()), GST_FORMAT_DEFAULT, flags, (gint64) value)) { handleMessage(pipeline); CV_WARN("GStreamer: unable to seek"); - break; + return false; } // wait for status update gst_element_get_state(pipeline, NULL, NULL, GST_CLOCK_TIME_NONE); return true; } case CV_CAP_PROP_POS_AVI_RATIO: + { + // https://stackoverflow.com/questions/31290315 + // GStreamer docs: GST_FORMAT_PERCENT (5) – percentage of stream (few, if any, elements implement this as of May 2009) + CV_WARN("GStreamer: seeking by file percent are not supported by most GStreamer elements"); if(!gst_element_seek_simple(GST_ELEMENT(pipeline.get()), GST_FORMAT_PERCENT, flags, (gint64) (value * GST_FORMAT_PERCENT_MAX))) { handleMessage(pipeline); CV_WARN("GStreamer: unable to seek"); + return false; } else { @@ -2011,7 +2063,8 @@ bool GStreamerCapture::setProperty(int propId, double value) } } } - break; + return true; + } case CV_CAP_PROP_FRAME_WIDTH: if(value > 0) setFilter("width", G_TYPE_INT, (int) value, 0); @@ -2099,8 +2152,9 @@ bool GStreamerCapture::setProperty(int propId, double value) CV_WARN("GStreamer: unhandled property"); } - if (wasPlaying) + if (needRestart) { this->startPipeline(); + } return false; } @@ -2572,7 +2626,7 @@ bool CvVideoWriter_GStreamer::open( const std::string &filename, int fourcc, if (stateret == GST_STATE_CHANGE_FAILURE) { handleMessage(pipeline); - CV_WARN("GStreamer: cannot put pipeline to play\n"); + CV_WARN("GStreamer: cannot put pipeline to play"); pipeline.release(); return false; } diff --git a/modules/videoio/test/test_audio.cpp b/modules/videoio/test/test_audio.cpp index ca46bfc4ca..bf2b4bad68 100644 --- a/modules/videoio/test/test_audio.cpp +++ b/modules/videoio/test/test_audio.cpp @@ -186,6 +186,7 @@ public: double audio_shift = cap.get(CAP_PROP_AUDIO_SHIFT_NSEC); double video0_timestamp = cap.get(CAP_PROP_POS_MSEC) * 1e-3; audio0_timestamp = video0_timestamp + audio_shift * 1e-9; + std::cout << "video0 timestamp: " << video0_timestamp << " audio0 timestamp: " << audio0_timestamp << " (audio shift nanoseconds: " << audio_shift << " , seconds: " << audio_shift * 1e-9 << ")" << std::endl; } ASSERT_TRUE(cap.retrieve(videoFrame)); @@ -228,7 +229,7 @@ public: EXPECT_NEAR( cap.get(CAP_PROP_AUDIO_POS) / samplePerSecond + audio0_timestamp, cap.get(CAP_PROP_POS_MSEC) * 1e-3, - (1.0 / fps) * 0.3) + (1.0 / fps) * 0.6) << "CAP_PROP_AUDIO_POS=" << cap.get(CAP_PROP_AUDIO_POS) << " CAP_PROP_POS_MSEC=" << cap.get(CAP_PROP_POS_MSEC); } if (frame != 0 && frame != numberOfFrames-1 && audioData[0].size() != (size_t)numberOfSamples) diff --git a/modules/videoio/test/test_gstreamer.cpp b/modules/videoio/test/test_gstreamer.cpp index a8c24be438..c9f94bf7ee 100644 --- a/modules/videoio/test/test_gstreamer.cpp +++ b/modules/videoio/test/test_gstreamer.cpp @@ -178,4 +178,47 @@ TEST(videoio_gstreamer, timeout_property) } } +//============================================================================== +// Seeking test with manual GStreamer pipeline +typedef testing::TestWithParam gstreamer_bunny; + +TEST_P(gstreamer_bunny, manual_seek) +{ + if (!videoio_registry::hasBackend(CAP_GSTREAMER)) + throw SkipTestException("GStreamer backend was not found"); + + const string video_file = BunnyParameters::getFilename("." + GetParam()); + const string pipeline = "filesrc location=" + video_file + " ! decodebin ! videoconvert ! video/x-raw, format=BGR ! appsink drop=1"; + const double target_pos = 3000.0; + const float ms_per_frame = 1000 / BunnyParameters::getFps(); + VideoCapture cap; + cap.open(pipeline, CAP_GSTREAMER); + ASSERT_TRUE(cap.isOpened()); + Mat img; + for (int i = 0; i < 10; i++) + { + cap >> img; + } + EXPECT_FALSE(img.empty()); + cap.set(CAP_PROP_POS_MSEC, target_pos); + cap >> img; + EXPECT_FALSE(img.empty()); + double actual_pos = cap.get(CAP_PROP_POS_MSEC); + EXPECT_NEAR(actual_pos, target_pos, ms_per_frame); +} + +static const string bunny_params[] = { + // string("wmv"), + string("mov"), + string("mp4"), + // string("mpg"), + string("avi"), + // string("h264"), + // string("h265"), + string("mjpg.avi") +}; + +INSTANTIATE_TEST_CASE_P(videoio, gstreamer_bunny, testing::ValuesIn(bunny_params)); + + }} // namespace diff --git a/modules/videoio/test/test_video_io.cpp b/modules/videoio/test/test_video_io.cpp index 7d7944f5eb..9c9cdddbff 100644 --- a/modules/videoio/test/test_video_io.cpp +++ b/modules/videoio/test/test_video_io.cpp @@ -67,6 +67,11 @@ public: std::cout << "CAP_PROP_FRAME_COUNT is not supported by backend. Assume 50 frames." << std::endl; n_frames = 50; } + // GStreamer can't read frame count of big_buck_bunny.wmv + if (apiPref == CAP_GSTREAMER && ext == "wmv") + { + n_frames = 125; + } { SCOPED_TRACE("consecutive read"); @@ -166,7 +171,7 @@ public: EXPECT_NO_THROW(count_prop = (int)cap.get(CAP_PROP_FRAME_COUNT)); // mpg file reports 5.08 sec * 24 fps => property returns 122 frames // but actual number of frames returned is 125 - if (ext != "mpg") + if (ext != "mpg" && !(apiPref == CAP_GSTREAMER && ext == "wmv")) { if (count_prop > 0) { @@ -200,12 +205,11 @@ public: if (!isBackendAvailable(apiPref, cv::videoio_registry::getStreamBackends())) throw SkipTestException(cv::String("Backend is not available/disabled: ") + cv::videoio_registry::getBackendName(apiPref)); - // GStreamer: https://github.com/opencv/opencv/issues/19025 - if (apiPref == CAP_GSTREAMER) + if (((apiPref == CAP_GSTREAMER) && (ext == "avi"))) throw SkipTestException(cv::String("Backend ") + cv::videoio_registry::getBackendName(apiPref) + - cv::String(" does not return reliable values for CAP_PROP_POS_MSEC property")); + cv::String(" does not support CAP_PROP_POS_MSEC option")); - if (((apiPref == CAP_FFMPEG) && ((ext == "h264") || (ext == "h265")))) + if (((apiPref == CAP_FFMPEG || apiPref == CAP_GSTREAMER) && ((ext == "h264") || (ext == "h265")))) throw SkipTestException(cv::String("Backend ") + cv::videoio_registry::getBackendName(apiPref) + cv::String(" does not support CAP_PROP_POS_MSEC option")); @@ -221,8 +225,8 @@ public: // mpg file reports 5.08 sec * 24 fps => property returns 122 frames,but actual number of frames returned is 125 // HACK: CAP_PROP_FRAME_COUNT is not supported for vmw + MSMF. Just force check for all 125 frames if (ext == "mpg") - EXPECT_GT(frame_count, 121); - else if ((ext == "wmv") && (apiPref == CAP_MSMF)) + EXPECT_GE(frame_count, 114); + else if ((ext == "wmv") && (apiPref == CAP_MSMF || apiPref == CAP_GSTREAMER)) frame_count = 125; else EXPECT_EQ(frame_count, 125); @@ -240,6 +244,9 @@ public: if (cvtest::debugLevel > 0) std::cout << "i = " << i << ": timestamp = " << timestamp << std::endl; const double frame_period = 1000.f/bunny_param.getFps(); + // big_buck_bunny.mpg starts at 0.500 msec + if ((ext == "mpg") && (apiPref == CAP_GSTREAMER)) + timestamp -= 500.0; // NOTE: eps == frame_period, because videoCapture returns frame beginning timestamp or frame end // timestamp depending on codec and back-end. So the first frame has timestamp 0 or frame_period. EXPECT_NEAR(timestamp, i*frame_period, frame_period) << "i=" << i; From 8df76fe0cb1fcfeff0876ada8e69e4522187165a Mon Sep 17 00:00:00 2001 From: Alexander Smorkalov Date: Wed, 8 Nov 2023 14:22:05 +0300 Subject: [PATCH 017/156] Exclude RVV UI internals from Doxygen documentation. --- modules/core/include/opencv2/core/hal/intrin_rvv.hpp | 5 ++++- modules/core/include/opencv2/core/hal/intrin_rvv071.hpp | 2 +- .../core/include/opencv2/core/hal/intrin_rvv_scalable.hpp | 5 +++++ 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/modules/core/include/opencv2/core/hal/intrin_rvv.hpp b/modules/core/include/opencv2/core/hal/intrin_rvv.hpp index 04e3c0e140..d70660249c 100644 --- a/modules/core/include/opencv2/core/hal/intrin_rvv.hpp +++ b/modules/core/include/opencv2/core/hal/intrin_rvv.hpp @@ -32,6 +32,8 @@ namespace cv { +//! @cond IGNORED + CV_CPU_OPTIMIZATION_HAL_NAMESPACE_BEGIN #define CV_SIMD128 1 @@ -3336,7 +3338,8 @@ inline void v_cleanup() {} CV_CPU_OPTIMIZATION_HAL_NAMESPACE_END +//! @endcond -} +} // namespace cv #endif diff --git a/modules/core/include/opencv2/core/hal/intrin_rvv071.hpp b/modules/core/include/opencv2/core/hal/intrin_rvv071.hpp index 9faefd97b7..26f478feda 100644 --- a/modules/core/include/opencv2/core/hal/intrin_rvv071.hpp +++ b/modules/core/include/opencv2/core/hal/intrin_rvv071.hpp @@ -2536,5 +2536,5 @@ CV_CPU_OPTIMIZATION_HAL_NAMESPACE_END //! @endcond -} +} // namespace cv #endif diff --git a/modules/core/include/opencv2/core/hal/intrin_rvv_scalable.hpp b/modules/core/include/opencv2/core/hal/intrin_rvv_scalable.hpp index 004ef7130c..c7770334ad 100644 --- a/modules/core/include/opencv2/core/hal/intrin_rvv_scalable.hpp +++ b/modules/core/include/opencv2/core/hal/intrin_rvv_scalable.hpp @@ -38,6 +38,9 @@ namespace cv { + +//! @cond IGNORED + CV_CPU_OPTIMIZATION_HAL_NAMESPACE_BEGIN #define CV_SIMD_SCALABLE 1 @@ -2126,6 +2129,8 @@ inline void v_cleanup() {} CV_CPU_OPTIMIZATION_HAL_NAMESPACE_END +//! @endcond + } //namespace cv #endif //OPENCV_HAL_INTRIN_RVV_SCALABLE_HPP From b203c7c3e24618beb631d4586f50450a2e4389a3 Mon Sep 17 00:00:00 2001 From: Alexander Smorkalov Date: Wed, 8 Nov 2023 14:46:44 +0300 Subject: [PATCH 018/156] Fixed build warning introduced in #24243. --- modules/videoio/test/test_gstreamer.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/videoio/test/test_gstreamer.cpp b/modules/videoio/test/test_gstreamer.cpp index c9f94bf7ee..ef9a6765a8 100644 --- a/modules/videoio/test/test_gstreamer.cpp +++ b/modules/videoio/test/test_gstreamer.cpp @@ -190,7 +190,7 @@ TEST_P(gstreamer_bunny, manual_seek) const string video_file = BunnyParameters::getFilename("." + GetParam()); const string pipeline = "filesrc location=" + video_file + " ! decodebin ! videoconvert ! video/x-raw, format=BGR ! appsink drop=1"; const double target_pos = 3000.0; - const float ms_per_frame = 1000 / BunnyParameters::getFps(); + const double ms_per_frame = 1000.0 / BunnyParameters::getFps(); VideoCapture cap; cap.open(pipeline, CAP_GSTREAMER); ASSERT_TRUE(cap.isOpened()); From b7ec2ebb55309a7b880edf5e5f483b537cbefe86 Mon Sep 17 00:00:00 2001 From: Dmitry Kurtaev Date: Wed, 8 Nov 2023 16:26:33 +0300 Subject: [PATCH 019/156] Merge pull request #24483 from dkurt:dnn_fusion_commutative_ops Commutative rules for DNN subgraphs fusion #24483 ### Pull Request Readiness Checklist related: https://github.com/opencv/opencv/pull/24463#issuecomment-1783033931 See details at https://github.com/opencv/opencv/wiki/How_to_contribute#making-a-good-pull-request - [x] I agree to contribute to the project under Apache 2 License. - [x] To the best of my knowledge, the proposed patch is not based on a code under GPL or another license that is incompatible with OpenCV - [x] The PR is proposed to the proper branch - [x] There is a reference to the original bug report and related work - [x] There is accuracy test, performance test and test data in opencv_extra repository, if applicable Patch to opencv_extra has the same branch name. - [x] The feature is well documented and sample code can be built with the project CMake --- modules/dnn/src/graph_simplifier.cpp | 68 ++-- modules/dnn/src/graph_simplifier.hpp | 8 +- .../dnn/src/onnx/onnx_graph_simplifier.cpp | 304 +++++++----------- .../src/tensorflow/tf_graph_simplifier.cpp | 27 +- 4 files changed, 179 insertions(+), 228 deletions(-) diff --git a/modules/dnn/src/graph_simplifier.cpp b/modules/dnn/src/graph_simplifier.cpp index e1b6d6df40..b9684afe69 100644 --- a/modules/dnn/src/graph_simplifier.cpp +++ b/modules/dnn/src/graph_simplifier.cpp @@ -77,14 +77,14 @@ int Subgraph::getInputNodeId(const Ptr& net, } bool Subgraph::match(const Ptr& net, int nodeId, - std::vector& matchedNodesIds, - std::vector& targetNodesIds) + std::vector& matchedNodesIds) { matchedNodesIds.clear(); - targetNodesIds.clear(); std::queue nodesToMatch; std::queue targetNodes; + std::vector > matchings; + matchings.reserve(nodes.size()); nodesToMatch.push(nodeId); targetNodes.push(nodes.size() - 1); while (!nodesToMatch.empty()) @@ -94,51 +94,63 @@ bool Subgraph::match(const Ptr& net, int nodeId, nodesToMatch.pop(); targetNodes.pop(); - if (std::find(matchedNodesIds.begin(), matchedNodesIds.end(), nodeToMatch) != - matchedNodesIds.end()) + if (std::find_if(matchings.begin(), matchings.end(), [&](const std::pair& match){ return match.first == targetNodeId; }) != + matchings.end()) continue; + // Empty placeholder matches with any input type + if (nodes[targetNodeId].empty()) { + matchings.push_back({targetNodeId, nodeToMatch}); + continue; + } + const Ptr node = net->getNode(nodeToMatch); if (node->getType() != nodes[targetNodeId]) - return false; + continue; std::vector& inputNodes = inputs[targetNodeId]; if (inputNodes.size() != node->getNumInputs()) - return false; + continue; + + bool isCommutative = net->isCommutativeOp(node->getType()); for (int j = 0; j < inputNodes.size(); ++j) { - if (nodes[inputNodes[j]].empty() || node->getInputName(j).empty()) // Unknown input node type. + // Sometimes, ONNX may have input but it's empty (see Clip layer from reduceL2_subgraph2_2 testcase) + if (node->getInputName(j).empty()) continue; nodeId = getInputNodeId(net, node, j); const Ptr inpNode = net->getNode(nodeId); - if (inpNode->getType() != "Const" && inpNode->getType() != "Constant") + if (isCommutative) + { + for (int i = 0; i < inputNodes.size(); ++i) + { + nodesToMatch.push(nodeId); + targetNodes.push(inputNodes[i]); + } + } + else { nodesToMatch.push(nodeId); targetNodes.push(inputNodes[j]); } - else if (nodes[inputNodes[j]] != "Const" && nodes[inputNodes[j]] != "Constant") - return false; } - matchedNodesIds.push_back(nodeToMatch); - targetNodesIds.push_back(targetNodeId); + matchings.push_back({targetNodeId, nodeToMatch}); } + if (matchings.size() != nodes.size()) + return false; - const int n = matchedNodesIds.size(); - std::vector > elements(n); - for (int i = 0; i < n; ++i) - elements[i] = std::make_pair(matchedNodesIds[i], targetNodesIds[i]); - std::sort(elements.begin(), elements.end()); - for (int i = 0; i < n; ++i) + // Sort matched by pattern nodes order. + std::sort(matchings.begin(), matchings.end()); + matchedNodesIds.resize(matchings.size()); + for (int i = 0; i < matchings.size(); ++i) { - matchedNodesIds[i] = elements[i].first; - targetNodesIds[i] = elements[i].second; + matchedNodesIds[i] = matchings[i].second; } return true; } -void Subgraph::replace(const Ptr& net, const std::vector& matchedNodesIds, - const std::vector& targetNodesIds) +void Subgraph::replace(const Ptr& net, const std::vector& matchedNodesIds) { // Extract names of input nodes. std::vector inputsNames(fusedNodeInputs.size()); @@ -149,9 +161,9 @@ void Subgraph::replace(const Ptr& net, const std::vector node = net->getNode(matchedNodesIds[j]); - std::vector& inpIndices = inputs[targetNodesIds[j]]; + std::vector& inpIndices = inputs[j]; - CV_Assert(node->getNumInputs() == inpIndices.size()); + CV_Assert(inpIndices.empty() || node->getNumInputs() == inpIndices.size()); for (int k = 0; k < inpIndices.size(); ++k) { if (inpIndices[k] == fusedNodeInputs[i]) @@ -187,15 +199,15 @@ void simplifySubgraphs(const Ptr& net, const std::vector >& patterns) { int numNodes = net->getNumNodes(); - std::vector matchedNodesIds, targetNodesIds; + std::vector matchedNodesIds; std::vector nodesToRemove; for (int j = 0; j < patterns.size(); ++j) { for (int i = 0; i < numNodes; ++i) { - if (patterns[j]->match(net, i, matchedNodesIds, targetNodesIds)) + if (patterns[j]->match(net, i, matchedNodesIds)) { - patterns[j]->replace(net, matchedNodesIds, targetNodesIds); + patterns[j]->replace(net, matchedNodesIds); // Remove matched nodes except the last one. nodesToRemove.insert(nodesToRemove.end(), matchedNodesIds.begin(), matchedNodesIds.end() - 1); } diff --git a/modules/dnn/src/graph_simplifier.hpp b/modules/dnn/src/graph_simplifier.hpp index 22bc50e3e5..aa9be32a91 100644 --- a/modules/dnn/src/graph_simplifier.hpp +++ b/modules/dnn/src/graph_simplifier.hpp @@ -44,6 +44,8 @@ public: virtual std::string getOutputName(int nodeId, int outId) const = 0; virtual void removeNode(int idx) = 0; + + virtual bool isCommutativeOp(const std::string& type) const = 0; }; class Subgraph // Interface to match and replace subgraphs. @@ -75,12 +77,10 @@ public: // Match TensorFlow subgraph starting from with a set of nodes to be fused. // Const nodes are skipped during matching. Returns true if nodes are matched and can be fused. virtual bool match(const Ptr& net, int nodeId, - std::vector& matchedNodesIds, - std::vector& targetNodesIds); + std::vector& matchedNodesIds); // Fuse matched subgraph. - void replace(const Ptr& net, const std::vector& matchedNodesIds, - const std::vector& targetNodesIds); + void replace(const Ptr& net, const std::vector& matchedNodesIds); virtual void finalize(const Ptr& net, const Ptr& fusedNode, diff --git a/modules/dnn/src/onnx/onnx_graph_simplifier.cpp b/modules/dnn/src/onnx/onnx_graph_simplifier.cpp index 15f79c8769..a8f4058d50 100644 --- a/modules/dnn/src/onnx/onnx_graph_simplifier.cpp +++ b/modules/dnn/src/onnx/onnx_graph_simplifier.cpp @@ -125,8 +125,13 @@ public: virtual void removeNode(int idx) CV_OVERRIDE { - CV_Assert(idx >= numInputs + numInitializers); - net.mutable_node()->DeleteSubrange(idx - numInputs - numInitializers, 1); + if (idx >= numInputs + numInitializers) + net.mutable_node()->DeleteSubrange(idx - numInputs - numInitializers, 1); + } + + virtual inline bool isCommutativeOp(const std::string& type) const CV_OVERRIDE + { + return type == "Add" || type == "Mul" || type == "Equal" || type == "Max"; } private: @@ -134,6 +139,25 @@ private: opencv_onnx::GraphProto& net; }; +static Mat extractConstant(const Ptr& net, int node_id, int input_id) +{ + auto onnx_net = net.dynamicCast(); + int initializer_id = onnx_net->getInputInitializerId(node_id, input_id); + if (initializer_id != -1) + { + return onnx_net->getMatFromInitializer(initializer_id); + } + else + { + const Ptr node = net->getNode(node_id); + int constant_id = Subgraph::getInputNodeId(net, node, input_id); + Ptr constant_ptr = net->getNode(constant_id); + opencv_onnx::NodeProto* constant_node = constant_ptr.dynamicCast()->node; + opencv_onnx::TensorProto constant_proto = constant_node->attribute(0).t(); + return getMatFromTensor(constant_proto); + } +} + /* Fusion for Gelu. Graph before fusion: @@ -151,54 +175,32 @@ public: GeluSubGraph() { int input = addNodeToMatch(""); - int div = addNodeToMatch("Div", input, addNodeToMatch("") /* B=sqrt(2) */ ); + div = addNodeToMatch("Div", input, addNodeToMatch("") /* B=sqrt(2) */ ); int erf = addNodeToMatch("Erf", div); - int add = addNodeToMatch("Add", erf, addNodeToMatch("") /* B=1 */ ); + add = addNodeToMatch("Add", erf, addNodeToMatch("") /* B=1 */ ); int mul = addNodeToMatch("Mul", input, add); - addNodeToMatch("Mul", mul, addNodeToMatch("") /* B=0.5 */) ; + mul2 = addNodeToMatch("Mul", mul, addNodeToMatch("") /* B=0.5 */) ; setFusedNode("Gelu", input); } - static float extractConstant(const Ptr& net, int node_id, int input_id) - { - auto onnx_net = net.dynamicCast(); - int initializer_id = onnx_net->getInputInitializerId(node_id, input_id); - if (initializer_id != -1) - { - Mat const_mat = onnx_net->getMatFromInitializer(initializer_id); - return *const_mat.ptr(); - } - else - { - const Ptr node = net->getNode(node_id); - int constant_id = getInputNodeId(net, node, input_id); - Ptr constant_ptr = net->getNode(constant_id); - opencv_onnx::NodeProto* constant_node = constant_ptr.dynamicCast()->node; - opencv_onnx::TensorProto constant_proto = constant_node->attribute(0).t(); - Mat constant_mat = getMatFromTensor(constant_proto); - return *constant_mat.ptr(); - } - } - virtual bool match(const Ptr& net, int nodeId, - std::vector& matchedNodesIds, - std::vector& targetNodesIds) CV_OVERRIDE + std::vector& matchedNodesIds) CV_OVERRIDE { - if (Subgraph::match(net, nodeId, matchedNodesIds, targetNodesIds)) + if (Subgraph::match(net, nodeId, matchedNodesIds)) { // Check Div[B=sqrt(2)] - float divisor = extractConstant(net, matchedNodesIds[0], 1); + float divisor = extractConstant(net, matchedNodesIds[div], 1).at(0); if (std::fabs(divisor - M_SQRT2) >= std::numeric_limits::epsilon()) return false; // Check Add[B=1] - float add_const = extractConstant(net, matchedNodesIds[2], 1); + float add_const = extractConstant(net, matchedNodesIds[add], 1).at(0); if (std::fabs(add_const - 1.f) >= std::numeric_limits::epsilon()) return false; // Check Mul[B=0.5] - float mul_const = extractConstant(net, matchedNodesIds[4], 1); + float mul_const = extractConstant(net, matchedNodesIds[mul2], 1).at(0); if (std::fabs(mul_const - 0.5f) >= std::numeric_limits::epsilon()) return false; @@ -206,6 +208,9 @@ public: } return false; } + +private: + int div, add, mul2; }; /* Fusion for GeluApproximation. @@ -229,61 +234,39 @@ public: int input = addNodeToMatch(""); int mul0 = addNodeToMatch("Mul", input, input); int mul1 = addNodeToMatch("Mul", input, mul0); - int mul2 = addNodeToMatch("Mul", addNodeToMatch("") /* A=0.044714998453855515 */, mul1); + mul2 = addNodeToMatch("Mul", addNodeToMatch("") /* A=0.044714998453855515 */, mul1); int add0 = addNodeToMatch("Add", input, mul2); - int mul3 = addNodeToMatch("Mul", addNodeToMatch("") /* A=sqrt(2/pie) */, add0); + mul3 = addNodeToMatch("Mul", addNodeToMatch("") /* A=sqrt(2/pie) */, add0); int tanh = addNodeToMatch("Tanh", mul3); - int add1 = addNodeToMatch("Add", addNodeToMatch("") /* A=1 */, tanh); + add1 = addNodeToMatch("Add", addNodeToMatch("") /* A=1 */, tanh); int mul4 = addNodeToMatch("Mul", input, add1); - addNodeToMatch("Mul", addNodeToMatch("") /* A=0.5 */, mul4); + mul5 = addNodeToMatch("Mul", addNodeToMatch("") /* A=0.5 */, mul4); setFusedNode("GeluApproximation", input); } - static float extractConstant(const Ptr& net, int node_id, int input_id) - { - auto onnx_net = net.dynamicCast(); - int initializer_id = onnx_net->getInputInitializerId(node_id, input_id); - if (initializer_id != -1) - { - Mat const_mat = onnx_net->getMatFromInitializer(initializer_id); - return *const_mat.ptr(); - } - else - { - const Ptr node = net->getNode(node_id); - int constant_id = getInputNodeId(net, node, input_id); - Ptr constant_ptr = net->getNode(constant_id); - opencv_onnx::NodeProto* constant_node = constant_ptr.dynamicCast()->node; - opencv_onnx::TensorProto constant_proto = constant_node->attribute(0).t(); - Mat constant_mat = getMatFromTensor(constant_proto); - return *constant_mat.ptr(); - } - } - virtual bool match(const Ptr& net, int nodeId, - std::vector& matchedNodesIds, - std::vector& targetNodesIds) CV_OVERRIDE + std::vector& matchedNodesIds) CV_OVERRIDE { - if (Subgraph::match(net, nodeId, matchedNodesIds, targetNodesIds)) + if (Subgraph::match(net, nodeId, matchedNodesIds)) { // Check Mul[A=0.044714998453855515] - float coef = extractConstant(net, matchedNodesIds[2], 0); + float coef = extractConstant(net, matchedNodesIds[mul2], 0).at(0); if (coef - 0.044714998453855515 >= 1e-6) return false; // Check Mul[A=sqrt(2/pie)] - float sqrt_2_pie = extractConstant(net, matchedNodesIds[4], 0); + float sqrt_2_pie = extractConstant(net, matchedNodesIds[mul3], 0).at(0); if (sqrt_2_pie - 0.7978845834732056 >= 1e-6) return false; // Check Add[A=1] - float add_const = extractConstant(net, matchedNodesIds[6], 0); + float add_const = extractConstant(net, matchedNodesIds[add1], 0).at(0); if (add_const - 1.f >= 1e-6) return false; // Check Mul[A=0.5] - float mul_const = extractConstant(net, matchedNodesIds[8], 0); + float mul_const = extractConstant(net, matchedNodesIds[mul5], 0).at(0); if (mul_const - 0.5f >= 1e-6) return false; @@ -291,6 +274,9 @@ public: } return false; } + +private: + int mul2, mul3, add1, mul5; }; /* Fusion for LayerNormalization. @@ -313,43 +299,22 @@ public: LayerNormSubGraph() : axis(-1), epsilon(1e-5) { int input = addNodeToMatch(""); - int mean = addNodeToMatch("ReduceMean", input); + mean = addNodeToMatch("ReduceMean", input); int sub = addNodeToMatch("Sub", input, mean); - int pow = addNodeToMatch("Pow", sub, addNodeToMatch("")); - int mean1 = addNodeToMatch("ReduceMean", pow); - int add = addNodeToMatch("Add", mean1, addNodeToMatch("")); + pow = addNodeToMatch("Pow", sub, addNodeToMatch("")); + mean1 = addNodeToMatch("ReduceMean", pow); + add = addNodeToMatch("Add", mean1, addNodeToMatch("")); int sqrt = addNodeToMatch("Sqrt", add); int div = addNodeToMatch("Div", sub, sqrt); - int mul = addNodeToMatch("Mul", div, addNodeToMatch("")); - addNodeToMatch("Add", mul, addNodeToMatch("")); + mul = addNodeToMatch("Mul", div, addNodeToMatch("")); + bias = addNodeToMatch("Add", mul, addNodeToMatch("")); setFusedNode("LayerNormalization", input); } - static float extractConstant(const Ptr& net, int node_id, int input_id) - { - auto onnx_net = net.dynamicCast(); - int initializer_id = onnx_net->getInputInitializerId(node_id, input_id); - if (initializer_id != -1) // initializer - { - Mat const_mat = onnx_net->getMatFromInitializer(initializer_id); - return *const_mat.ptr(); - } - else - { - const Ptr node = net->getNode(node_id); - int constant_id = getInputNodeId(net, node, input_id); - Ptr constant_ptr = net->getNode(constant_id); - opencv_onnx::NodeProto* constant_node = constant_ptr.dynamicCast()->node; - opencv_onnx::TensorProto constant_proto = constant_node->attribute(0).t(); - Mat constant_mat = getMatFromTensor(constant_proto); - return *constant_mat.ptr(); - } - } - static float extractAxis(const Ptr& net, int node_id) { Ptr mean_ptr = net->getNode(node_id); @@ -381,25 +346,24 @@ public: } virtual bool match(const Ptr& net, int nodeId, - std::vector& matchedNodesIds, - std::vector& targetNodesIds) CV_OVERRIDE + std::vector& matchedNodesIds) CV_OVERRIDE { - if (Subgraph::match(net, nodeId, matchedNodesIds, targetNodesIds)) + if (Subgraph::match(net, nodeId, matchedNodesIds)) { - float pow_exp = extractConstant(net, matchedNodesIds[2], 1); + float pow_exp = extractConstant(net, matchedNodesIds[pow], 1).at(0); if (pow_exp - 2 > 1e-5) // not pow(2) return false; - int axis_mean1 = extractAxis(net, matchedNodesIds[0]); - int axis_mean2 = extractAxis(net, matchedNodesIds[3]); + int axis_mean1 = extractAxis(net, matchedNodesIds[mean]); + int axis_mean2 = extractAxis(net, matchedNodesIds[mean1]); if (axis_mean1 != axis_mean2) return false; axis = axis_mean1; - epsilon = extractConstant(net, matchedNodesIds[4], 1); + epsilon = extractConstant(net, matchedNodesIds[add], 1).at(0); - weight_name = getInputName(net, matchedNodesIds[7], 1); - bias_name = getInputName(net, matchedNodesIds[8], 1); + weight_name = getInputName(net, matchedNodesIds[mul], 1); + bias_name = getInputName(net, matchedNodesIds[bias], 1); return true; } @@ -429,6 +393,7 @@ protected: float epsilon; std::string weight_name; std::string bias_name; + int pow, mean, mean1, add, mul, bias; }; class SoftMaxSubgraphBase : public Subgraph @@ -437,10 +402,9 @@ public: SoftMaxSubgraphBase() : axis(1), id(-1) {} virtual bool match(const Ptr& net, int nodeId, - std::vector& matchedNodesIds, - std::vector& targetNodesIds) CV_OVERRIDE + std::vector& matchedNodesIds) CV_OVERRIDE { - if (Subgraph::match(net, nodeId, matchedNodesIds, targetNodesIds)) + if (Subgraph::match(net, nodeId, matchedNodesIds)) { CV_Assert(id >= 0 && id < matchedNodesIds.size()); Ptr sum = net->getNode(matchedNodesIds[id]); @@ -485,7 +449,7 @@ public: int inpExp = addNodeToMatch("Exp", input); int sum = addNodeToMatch("ReduceSum", inpExp); - id = 1; + id = sum; addNodeToMatch("Div", inpExp, sum); setFusedNode("Softmax", input); @@ -498,7 +462,7 @@ public: int input = addNodeToMatch(""); int reducemax = addNodeToMatch("ReduceMax", input); - id = 0; + id = reducemax; int sub = addNodeToMatch("Sub", input, reducemax); int exp = addNodeToMatch("Exp", sub); @@ -516,7 +480,7 @@ public: int input = addNodeToMatch(""); int reducemax = addNodeToMatch("ReduceMax", input); - id = 0; + id = reducemax; int sub_1 = addNodeToMatch("Sub", input, reducemax); int exp = addNodeToMatch("Exp", sub_1); @@ -533,18 +497,17 @@ public: HardSwishSubgraph() { int input = addNodeToMatch(""); - int hardSigmoid = addNodeToMatch("HardSigmoid", input); - addNodeToMatch("Mul", input, hardSigmoid); + hardSigmoidId = addNodeToMatch("HardSigmoid", input); + addNodeToMatch("Mul", input, hardSigmoidId); setFusedNode("HardSwish", input); } virtual bool match(const Ptr& net, int nodeId, - std::vector& matchedNodesIds, - std::vector& targetNodesIds) CV_OVERRIDE + std::vector& matchedNodesIds) CV_OVERRIDE { - if (Subgraph::match(net, nodeId, matchedNodesIds, targetNodesIds)) + if (Subgraph::match(net, nodeId, matchedNodesIds)) { - Ptr hardSigmoid = net->getNode(matchedNodesIds[0]); + Ptr hardSigmoid = net->getNode(matchedNodesIds[hardSigmoidId]); opencv_onnx::NodeProto* node = hardSigmoid.dynamicCast()->node; uint8_t matched = 0; @@ -561,6 +524,9 @@ public: } return false; } + +private: + int hardSigmoidId; }; class CeluSubgraph : public Subgraph @@ -569,9 +535,9 @@ public: CeluSubgraph() : alpha(1.f) { int input = addNodeToMatch(""); - int div = addNodeToMatch("Div", input, addNodeToMatch("")); - int elu = addNodeToMatch("Elu", div); - addNodeToMatch("Mul", addNodeToMatch(""), elu); + div = addNodeToMatch("Div", input, addNodeToMatch("")); + elu = addNodeToMatch("Elu", div); + mul = addNodeToMatch("Mul", addNodeToMatch(""), elu); setFusedNode("Celu", input); } @@ -587,16 +553,15 @@ public: } virtual bool match(const Ptr& net, int nodeId, - std::vector& matchedNodesIds, - std::vector& targetNodesIds) CV_OVERRIDE + std::vector& matchedNodesIds) CV_OVERRIDE { - if (Subgraph::match(net, nodeId, matchedNodesIds, targetNodesIds)) + if (Subgraph::match(net, nodeId, matchedNodesIds)) { - float alpha_div = extractAlpha(net, matchedNodesIds[0], 1); - float alpha_mul = extractAlpha(net, matchedNodesIds[2], 0); + float alpha_div = extractAlpha(net, matchedNodesIds[div], 1); + float alpha_mul = extractAlpha(net, matchedNodesIds[mul], 0); float alpha_elu = 1.f; - Ptr elu_ptr = net->getNode(matchedNodesIds[1]); + Ptr elu_ptr = net->getNode(matchedNodesIds[elu]); opencv_onnx::NodeProto* elu_node = elu_ptr.dynamicCast()->node; for (int i = 0; i < elu_node->attribute_size(); i++) @@ -625,18 +590,18 @@ public: protected: float alpha; + int div, mul, elu; }; class NormalizeSubgraphBase : public Subgraph { public: - NormalizeSubgraphBase(int _normNodeOrder = 0) : axis(1), normNodeOrder(_normNodeOrder) {} + NormalizeSubgraphBase(int _normNodeOrder = 1) : axis(1), normNodeOrder(_normNodeOrder) {} virtual bool match(const Ptr& net, int nodeId, - std::vector& matchedNodesIds, - std::vector& targetNodesIds) CV_OVERRIDE + std::vector& matchedNodesIds) CV_OVERRIDE { - if (Subgraph::match(net, nodeId, matchedNodesIds, targetNodesIds)) + if (Subgraph::match(net, nodeId, matchedNodesIds)) { Ptr norm = net->getNode(matchedNodesIds[normNodeOrder]); opencv_onnx::NodeProto* node = norm.dynamicCast()->node; @@ -725,7 +690,7 @@ public: class NormalizeSubgraph3 : public NormalizeSubgraphBase { public: - NormalizeSubgraph3() : NormalizeSubgraphBase(1) + NormalizeSubgraph3() : NormalizeSubgraphBase(3) { int input = addNodeToMatch(""); int power = addNodeToMatch("Constant"); @@ -743,7 +708,7 @@ public: class NormalizeSubgraph4 : public NormalizeSubgraphBase { public: - NormalizeSubgraph4() : NormalizeSubgraphBase(1) + NormalizeSubgraph4() : NormalizeSubgraphBase(2) { int input = addNodeToMatch(""); int mul = addNodeToMatch("Mul", input, input); @@ -760,7 +725,7 @@ public: class NormalizeSubgraph5 : public NormalizeSubgraphBase { public: - NormalizeSubgraph5() : NormalizeSubgraphBase(1) + NormalizeSubgraph5() : NormalizeSubgraphBase(2) { int input = addNodeToMatch(""); int mul = addNodeToMatch("Mul", input, input); @@ -781,25 +746,24 @@ public: { int input = addNodeToMatch(""); int index = addNodeToMatch("Constant"); - int gather = addNodeToMatch("Gather", input, index); - addNodeToMatch("Cast", gather); + gather = addNodeToMatch("Gather", input, index); + cast = addNodeToMatch("Cast", gather); setFusedNode("Gather", input, index); } virtual bool match(const Ptr& net, int nodeId, - std::vector& matchedNodesIds, - std::vector& targetNodesIds) CV_OVERRIDE + std::vector& matchedNodesIds) CV_OVERRIDE { - bool retVal = Subgraph::match(net, nodeId, matchedNodesIds, targetNodesIds); + bool retVal = Subgraph::match(net, nodeId, matchedNodesIds); size_t matchedNodesNum = matchedNodesIds.size(); // Now we check if merging can be made for these Gather and Cast nodes if (!retVal || matchedNodesNum < 2) return retVal; else { - int nodeToMatch = matchedNodesIds[matchedNodesNum - 1]; + int nodeToMatch = matchedNodesIds[cast]; const Ptr node = net->getNode(nodeToMatch); if (node->getType() == "Cast") { - int inpNodeId = matchedNodesIds[matchedNodesNum - 2]; + int inpNodeId = matchedNodesIds[gather]; const Ptr inpNode = net->getNode(inpNodeId); if (inpNode->getType() == "Gather") { int numNodes = net->getNumNodes(); @@ -819,6 +783,9 @@ public: } return retVal; } + +private: + int cast, gather; }; /* Constant folding shape for Expand. @@ -838,12 +805,12 @@ public: { int input = addNodeToMatch(""); int values = addNodeToMatch(""); - int init = addNodeToMatch("ConstantOfShape", values); + init = addNodeToMatch("ConstantOfShape", values); int coeff = addNodeToMatch("Constant"); - int mul = addNodeToMatch("Mul", init, coeff); + mul = addNodeToMatch("Mul", init, coeff); int shape = addNodeToMatch("Constant"); - int condition = addNodeToMatch("Equal", shape, mul); - int where = addNodeToMatch("Where", condition, init, addNodeToMatch("Constant")); + condition = addNodeToMatch("Equal", shape, mul); + where = addNodeToMatch("Where", condition, init, addNodeToMatch("Constant")); addNodeToMatch("Expand", input, where); setFusedNode("Expand", input, shape); } @@ -872,53 +839,28 @@ public: return 0; } - static std::vector extractConstant(const Ptr& net, int node_id, int input_id) - { - auto onnx_net = net.dynamicCast(); - int initializer_id = onnx_net->getInputInitializerId(node_id, input_id); - Mat mat_constant; - if (initializer_id != -1) // initializer - { - mat_constant = onnx_net->getMatFromInitializer(initializer_id); - } - else - { - const Ptr node = net->getNode(node_id); - int constant_id = getInputNodeId(net, node, input_id); - Ptr constant_ptr = net->getNode(constant_id); - opencv_onnx::NodeProto* constant_node = constant_ptr.dynamicCast()->node; - opencv_onnx::TensorProto constant_proto = constant_node->attribute(0).t(); - mat_constant = getMatFromTensor(constant_proto); - } - - std::vector retvals{mat_constant.begin(), mat_constant.end()}; - return retvals; - } - virtual bool match(const Ptr& net, int nodeId, - std::vector& matchedNodesIds, - std::vector& targetNodesIds) CV_OVERRIDE { - if (Subgraph::match(net, nodeId, matchedNodesIds, targetNodesIds)) { + std::vector& matchedNodesIds) CV_OVERRIDE { + if (Subgraph::match(net, nodeId, matchedNodesIds)) { int64_t value_ConstantOfShape; - if (!extractValue(net, matchedNodesIds[0], value_ConstantOfShape)) { + if (!extractValue(net, matchedNodesIds[init], value_ConstantOfShape)) { return false; } - std::vector input_ConstantOfShape = extractConstant(net, matchedNodesIds[0], 0); + std::vector input_ConstantOfShape = extractConstant(net, matchedNodesIds[init], 0); if (input_ConstantOfShape.size() != static_cast(1)) { return false; } - - auto B_Mul = extractConstant(net, matchedNodesIds[1], 1); + std::vector B_Mul = extractConstant(net, matchedNodesIds[mul], 1); if (B_Mul.size() != static_cast(1)) { return false; } - auto A_Equal = extractConstant(net, matchedNodesIds[2], 0); + std::vector A_Equal = extractConstant(net, matchedNodesIds[condition], 0); if (A_Equal.size() != static_cast(input_ConstantOfShape[0])) { return false; } - auto Y_Where = extractConstant(net, matchedNodesIds[3], 2); + std::vector Y_Where = extractConstant(net, matchedNodesIds[where], 2); if (Y_Where.size() != A_Equal.size()) { return false; } @@ -969,6 +911,9 @@ public: protected: std::vector shape; + +private: + int init, mul, condition, where; }; class MishSubgraph : public Subgraph @@ -979,7 +924,7 @@ public: int input = addNodeToMatch(""); int softplus = addNodeToMatch("Softplus", input); int tanh = addNodeToMatch("Tanh", softplus); - addNodeToMatch("Mul", input, tanh); + addNodeToMatch("Mul", tanh, input); setFusedNode("Mish", input); } }; @@ -999,20 +944,6 @@ public: } }; -class SoftplusSubgraph2: public Subgraph -{ -public: - SoftplusSubgraph2() - { - int input = addNodeToMatch(""); - int exp = addNodeToMatch("Exp", input); - int addVal = addNodeToMatch(""); - int add = addNodeToMatch("Add", exp, addVal); - addNodeToMatch("Log", add); - setFusedNode("Softplus", input); - } -}; - class MulCastSubgraph : public Subgraph { public: @@ -1248,7 +1179,6 @@ void simplifySubgraphs(opencv_onnx::GraphProto& net) subgraphs.push_back(makePtr()); subgraphs.push_back(makePtr()); subgraphs.push_back(makePtr()); - subgraphs.push_back(makePtr()); subgraphs.push_back(makePtr()); subgraphs.push_back(makePtr()); subgraphs.push_back(makePtr()); diff --git a/modules/dnn/src/tensorflow/tf_graph_simplifier.cpp b/modules/dnn/src/tensorflow/tf_graph_simplifier.cpp index 5531b28111..8ba1963512 100644 --- a/modules/dnn/src/tensorflow/tf_graph_simplifier.cpp +++ b/modules/dnn/src/tensorflow/tf_graph_simplifier.cpp @@ -98,6 +98,14 @@ public: net.mutable_node()->DeleteSubrange(idx, 1); } + virtual inline bool isCommutativeOp(const std::string& type) const CV_OVERRIDE + { + return type == "Add" || type == "Sum" || + type == "Mul" || type == "Prod" || + type == "Max" || type == "Maximum" || type == "Minimum" || + type == "Mean" || type == "SquaredDifference"; + } + tensorflow::GraphDef& net; }; @@ -282,24 +290,26 @@ public: { int input = addNodeToMatch(""); int relu = addNodeToMatch("Relu", input); - int maxValue = addNodeToMatch("Const"); + maxValueId = addNodeToMatch("Const"); int clipValue = addNodeToMatch("Const"); - int minimum = addNodeToMatch("Minimum", relu, maxValue); + int minimum = addNodeToMatch("Minimum", relu, maxValueId); addNodeToMatch("Maximum", minimum, clipValue); setFusedNode("Relu6", input); } virtual bool match(const Ptr& net, int nodeId, - std::vector& matchedNodesIds, - std::vector& targetNodesIds) CV_OVERRIDE + std::vector& matchedNodesIds) CV_OVERRIDE { - if (!Subgraph::match(net, nodeId, matchedNodesIds, targetNodesIds)) + if (!Subgraph::match(net, nodeId, matchedNodesIds)) return false; - tensorflow::NodeDef* node = net->getNode(matchedNodesIds.front() + 1).dynamicCast()->node; + tensorflow::NodeDef* node = net->getNode(matchedNodesIds[maxValueId]).dynamicCast()->node; Mat maxValue = getTensorContent(node->attr().at("value").tensor()); return maxValue.type() == CV_32FC1 && maxValue.total() == 1 && maxValue.at(0) == 6; } + +private: + int maxValueId; }; // Keras' reshape stores output shape in separate Const nodes by one value. @@ -328,15 +338,14 @@ public: } virtual bool match(const Ptr& net, int nodeId, - std::vector& matchedNodesIds, - std::vector& targetNodesIds) CV_OVERRIDE + std::vector& matchedNodesIds) CV_OVERRIDE { Ptr node = net->getNode(nodeId); if (node->getNumInputs() == 0) return false; inpName = node->getInputName(0); - return Subgraph::match(net, nodeId, matchedNodesIds, targetNodesIds); + return Subgraph::match(net, nodeId, matchedNodesIds); } From 84590f96e5f2751b8a8966d578a456c6266dab8c Mon Sep 17 00:00:00 2001 From: Alex Date: Thu, 9 Nov 2023 11:22:04 +0300 Subject: [PATCH 020/156] add new contour filtering, test, refactoring --- doc/pattern_tools/test_charuco_board.py | 42 +- .../opencv2/objdetect/aruco_detector.hpp | 20 +- .../objdetect/src/aruco/aruco_detector.cpp | 521 +++++++++--------- .../objdetect/src/aruco/charuco_detector.cpp | 4 +- .../objdetect/test/test_arucodetection.cpp | 26 + .../objdetect/test/test_charucodetection.cpp | 2 +- 6 files changed, 336 insertions(+), 279 deletions(-) diff --git a/doc/pattern_tools/test_charuco_board.py b/doc/pattern_tools/test_charuco_board.py index 83d355b7a6..0258e4034f 100644 --- a/doc/pattern_tools/test_charuco_board.py +++ b/doc/pattern_tools/test_charuco_board.py @@ -34,7 +34,7 @@ class aruco_objdetect_test(NewOpenCVTests): aruco_dict = cv.aruco.getPredefinedDictionary(aruco_type[aruco_type_i]) board = cv.aruco.CharucoBoard((cols, rows), square_size, marker_size, aruco_dict) charuco_detector = cv.aruco.CharucoDetector(board) - from_cv_img = board.generateImage((cols*square_size*10, rows*square_size*10)) + from_cv_img = board.generateImage((cols*square_size, rows*square_size)) #draw desk using svg fd1, filesvg = tempfile.mkstemp(prefix="out", suffix=".svg") @@ -50,15 +50,20 @@ class aruco_objdetect_test(NewOpenCVTests): pm.make_charuco_board() pm.save() drawing = svg2rlg(filesvg) - renderPM.drawToFile(drawing, filepng, fmt='PNG', dpi=720) + renderPM.drawToFile(drawing, filepng, fmt='PNG', dpi=72) from_svg_img = cv.imread(filepng) + _charucoCorners, _charuco_ids_svg, marker_corners_svg, marker_ids_svg = charuco_detector.detectBoard(from_svg_img) + _charucoCorners, _charuco_ids_cv, marker_corners_cv, marker_ids_cv = charuco_detector.detectBoard(from_cv_img) + marker_corners_svg_map, marker_corners_cv_map = {}, {} + for i in range(len(marker_ids_svg)): + marker_corners_svg_map[int(marker_ids_svg[i][0])] = marker_corners_svg[i] + for i in range(len(marker_ids_cv)): + marker_corners_cv_map[int(marker_ids_cv[i][0])] = marker_corners_cv[i] - #test - _charucoCorners, _charucoIds, markerCorners_svg, markerIds_svg = charuco_detector.detectBoard(from_svg_img) - _charucoCorners, _charucoIds, markerCorners_cv, markerIds_cv = charuco_detector.detectBoard(from_cv_img) - - np.testing.assert_allclose(markerCorners_svg, markerCorners_cv, 0.1, 0.1) - np.testing.assert_allclose(markerIds_svg, markerIds_cv, 0.1, 0.1) + for key_svg in marker_corners_svg_map.keys(): + marker_svg = marker_corners_svg_map[key_svg] + marker_cv = marker_corners_cv_map[key_svg] + np.testing.assert_allclose(marker_svg, marker_cv, 0.1, 0.1) finally: if os.path.exists(filesvg): os.remove(filesvg) @@ -87,7 +92,7 @@ class aruco_objdetect_test(NewOpenCVTests): aruco_dict = cv.aruco.getPredefinedDictionary(aruco_type) board = cv.aruco.CharucoBoard((cols, rows), square_size, marker_size, aruco_dict) charuco_detector = cv.aruco.CharucoDetector(board) - from_cv_img = board.generateImage((cols*square_size*10, rows*square_size*10)) + from_cv_img = board.generateImage((cols*square_size, rows*square_size)) #draw desk using svg fd1, filesvg = tempfile.mkstemp(prefix="out", suffix=".svg") @@ -102,17 +107,24 @@ class aruco_objdetect_test(NewOpenCVTests): pm.make_charuco_board() pm.save() drawing = svg2rlg(filesvg) - renderPM.drawToFile(drawing, filepng, fmt='PNG', dpi=720) + renderPM.drawToFile(drawing, filepng, fmt='PNG', dpi=72) from_svg_img = cv.imread(filepng) #test - _charucoCorners, _charucoIds, markerCorners_svg, markerIds_svg = charuco_detector.detectBoard(from_svg_img) - _charucoCorners, _charucoIds, markerCorners_cv, markerIds_cv = charuco_detector.detectBoard(from_cv_img) + _charucoCorners, _charuco_ids_svg, marker_corners_svg, marker_ids_svg = charuco_detector.detectBoard(from_svg_img) + _charucoCorners, _charuco_ids_cv, marker_corners_cv, marker_ids_cv = charuco_detector.detectBoard(from_cv_img) + marker_corners_svg_map, marker_corners_cv_map = {}, {} + for i in range(len(marker_ids_svg)): + marker_corners_svg_map[int(marker_ids_svg[i][0])] = marker_corners_svg[i] + for i in range(len(marker_ids_cv)): + marker_corners_cv_map[int(marker_ids_cv[i][0])] = marker_corners_cv[i] - np.testing.assert_allclose(markerCorners_svg, markerCorners_cv, 0.1, 0.1) - np.testing.assert_allclose(markerIds_svg, markerIds_cv, 0.1, 0.1) + for key_svg in marker_corners_svg_map.keys(): + marker_svg = marker_corners_svg_map[key_svg] + marker_cv = marker_corners_cv_map[key_svg] + np.testing.assert_allclose(marker_svg, marker_cv, 0.1, 0.1) finally: if os.path.exists(filesvg): os.remove(filesvg) if os.path.exists(filepng): - os.remove(filepng) + os.remove(filepng) \ No newline at end of file diff --git a/modules/objdetect/include/opencv2/objdetect/aruco_detector.hpp b/modules/objdetect/include/opencv2/objdetect/aruco_detector.hpp index c51abb0c93..d537e43dc5 100644 --- a/modules/objdetect/include/opencv2/objdetect/aruco_detector.hpp +++ b/modules/objdetect/include/opencv2/objdetect/aruco_detector.hpp @@ -33,7 +33,7 @@ struct CV_EXPORTS_W_SIMPLE DetectorParameters { polygonalApproxAccuracyRate = 0.03; minCornerDistanceRate = 0.05; minDistanceToBorder = 3; - minMarkerDistanceRate = 0.05; + minMarkerDistanceRate = 0.125; cornerRefinementMethod = (int)CORNER_REFINE_NONE; cornerRefinementWinSize = 5; relativeCornerRefinmentWinSize = 0.3f; @@ -100,12 +100,26 @@ struct CV_EXPORTS_W_SIMPLE DetectorParameters { /// minimum distance of any corner to the image border for detected markers (in pixels) (default 3) CV_PROP_RW int minDistanceToBorder; - /** @brief minimum mean distance beetween two marker corners to be considered imilar, so that the smaller one is removed. + /** @brief minimum average distance between the corners of the two markers to be grouped (default 0.125). * - * The rate is relative to the smaller perimeter of the two markers (default 0.05). + * The rate is relative to the smaller perimeter of the two markers. + * Two markers are grouped if average distance between the corners of the two markers is less than + * min(MarkerPerimeter1, MarkerPerimeter2)*minMarkerDistanceRate. + * + * default value is 0.125 because 0.125*MarkerPerimeter = (MarkerPerimeter / 4) * 0.5 = half the side of the marker. + * + * @note default value was changed from 0.05 after 4.8.1 release, because the filtering algorithm has been changed. + * Now a few candidates from the same group can be added to the list of candidates if they are far from each other. + * @sa minGroupDistance. */ CV_PROP_RW double minMarkerDistanceRate; + /** @brief minimum average distance between the corners of the two markers in group to add them to the list of candidates + * + * The average distance between the corners of the two markers is calculated relative to its module size (default 0.21). + */ + CV_PROP_RW float minGroupDistance = 0.21f; + /** @brief default value CORNER_REFINE_NONE */ CV_PROP_RW int cornerRefinementMethod; diff --git a/modules/objdetect/src/aruco/aruco_detector.cpp b/modules/objdetect/src/aruco/aruco_detector.cpp index f5aba9b1be..97a40746a8 100644 --- a/modules/objdetect/src/aruco/aruco_detector.cpp +++ b/modules/objdetect/src/aruco/aruco_detector.cpp @@ -50,6 +50,7 @@ static inline bool readWrite(DetectorParameters ¶ms, const FileNode* readNod readNode, writeStorage); check |= readWriteParameter("minOtsuStdDev", params.minOtsuStdDev, readNode, writeStorage); check |= readWriteParameter("errorCorrectionRate", params.errorCorrectionRate, readNode, writeStorage); + check |= readWriteParameter("minGroupDistance", params.minGroupDistance, readNode, writeStorage); // new aruco 3 functionality check |= readWriteParameter("useAruco3Detection", params.useAruco3Detection, readNode, writeStorage); check |= readWriteParameter("minSideLengthCanonicalImg", params.minSideLengthCanonicalImg, readNode, writeStorage); @@ -212,149 +213,65 @@ static void _reorderCandidatesCorners(vector > &candidates) { } } -/** - * @brief to make sure that the corner's order of both candidates (default/white) is the same - */ -static vector alignContourOrder(Point2f corner, vector candidate) { - uint8_t r=0; - double min = norm( Vec2f( corner - candidate[0] ), NORM_L2SQR); - for(uint8_t pos=1; pos < 4; pos++) { - double nDiff = norm( Vec2f( corner - candidate[pos] ), NORM_L2SQR); - if(nDiff < min){ - r = pos; - min =nDiff; - } +static float getAverageModuleSize(const vector& markerCorners, int markerSize, int markerBorderBits) { + float averageArucoModuleSize = 0.f; + for (size_t i = 0ull; i < 4ull; i++) { + averageArucoModuleSize += sqrt(normL2Sqr(Point2f(markerCorners[i] - markerCorners[(i+1ull) % 4ull]))); } - std::rotate(candidate.begin(), candidate.begin() + r, candidate.end()); - return candidate; + int numModules = markerSize + markerBorderBits * 2; + averageArucoModuleSize /= ((float)markerCorners.size()*numModules); + return averageArucoModuleSize; } -/** - * @brief Check candidates that are too close to each other, save the potential candidates - * (i.e. biggest/smallest contour) and remove the rest - */ -static void _filterTooCloseCandidates(const vector > &candidatesIn, - vector > > &candidatesSetOut, - const vector > &contoursIn, - vector > > &contoursSetOut, - double minMarkerDistanceRate, bool detectInvertedMarker) { +static bool checkMarker1InMarker2(const vector& marker1, const vector& marker2) { + return pointPolygonTest(marker2, marker1[0], false) >= 0 && pointPolygonTest(marker2, marker1[1], false) >= 0 && + pointPolygonTest(marker2, marker1[2], false) >= 0 && pointPolygonTest(marker2, marker1[3], false) >= 0; +} - CV_Assert(minMarkerDistanceRate >= 0); - vector candGroup; - candGroup.resize(candidatesIn.size(), -1); - vector > groupedCandidates; - for(unsigned int i = 0; i < candidatesIn.size(); i++) { - bool isSingleContour = true; - for(unsigned int j = i + 1; j < candidatesIn.size(); j++) { +struct MarkerCandidate { + vector corners; + vector contour; + float perimeter = 0.f; +}; - int minimumPerimeter = min((int)contoursIn[i].size(), (int)contoursIn[j].size() ); +struct MarkerCandidateTree : MarkerCandidate{ + int parent = -1; + int depth = 0; + vector closeContours; - // fc is the first corner considered on one of the markers, 4 combinations are possible - for(int fc = 0; fc < 4; fc++) { - double distSq = 0; - for(int c = 0; c < 4; c++) { - // modC is the corner considering first corner is fc - int modC = (c + fc) % 4; - distSq += (candidatesIn[i][modC].x - candidatesIn[j][c].x) * - (candidatesIn[i][modC].x - candidatesIn[j][c].x) + - (candidatesIn[i][modC].y - candidatesIn[j][c].y) * - (candidatesIn[i][modC].y - candidatesIn[j][c].y); - } - distSq /= 4.; + MarkerCandidateTree() {} - // if mean square distance is too low, remove the smaller one of the two markers - double minMarkerDistancePixels = double(minimumPerimeter) * minMarkerDistanceRate; - if(distSq < minMarkerDistancePixels * minMarkerDistancePixels) { - isSingleContour = false; - // i and j are not related to a group - if(candGroup[i]<0 && candGroup[j]<0){ - // mark candidates with their corresponding group number - candGroup[i] = candGroup[j] = (int)groupedCandidates.size(); - - // create group - vector grouped; - grouped.push_back(i); - grouped.push_back(j); - groupedCandidates.push_back( grouped ); - } - // i is related to a group - else if(candGroup[i] > -1 && candGroup[j] == -1){ - int group = candGroup[i]; - candGroup[j] = group; - - // add to group - groupedCandidates[group].push_back( j ); - } - // j is related to a group - else if(candGroup[j] > -1 && candGroup[i] == -1){ - int group = candGroup[j]; - candGroup[i] = group; - - // add to group - groupedCandidates[group].push_back( i ); - } - } - } - } - if (isSingleContour && candGroup[i] < 0) - { - candGroup[i] = (int)groupedCandidates.size(); - vector grouped; - grouped.push_back(i); - grouped.push_back(i); // step "save possible candidates" require minimum 2 elements - groupedCandidates.push_back(grouped); + MarkerCandidateTree(vector&& corners_, vector&& contour_) { + corners = std::move(corners_); + contour = std::move(contour_); + perimeter = 0.f; + for (size_t i = 0ull; i < 4ull; i++) { + perimeter += sqrt(normL2Sqr(Point2f(corners[i] - corners[(i+1ull) % 4ull]))); } } - // save possible candidates - candidatesSetOut.clear(); - contoursSetOut.clear(); - - vector > biggerCandidates; - vector > biggerContours; - vector > smallerCandidates; - vector > smallerContours; - - // save possible candidates - for(unsigned int i = 0; i < groupedCandidates.size(); i++) { - unsigned int smallerIdx = groupedCandidates[i][0]; - unsigned int biggerIdx = smallerIdx; - double smallerArea = contourArea(candidatesIn[smallerIdx]); - double biggerArea = smallerArea; - - // evaluate group elements - for(unsigned int j = 1; j < groupedCandidates[i].size(); j++) { - unsigned int currIdx = groupedCandidates[i][j]; - double currArea = contourArea(candidatesIn[currIdx]); - - // check if current contour is bigger - if(currArea >= biggerArea) { - biggerIdx = currIdx; - biggerArea = currArea; - } - - // check if current contour is smaller - if(currArea < smallerArea && detectInvertedMarker) { - smallerIdx = currIdx; - smallerArea = currArea; - } - } - - // add contours and candidates - biggerCandidates.push_back(candidatesIn[biggerIdx]); - biggerContours.push_back(contoursIn[biggerIdx]); - if(detectInvertedMarker) { - smallerCandidates.push_back(alignContourOrder(candidatesIn[biggerIdx][0], candidatesIn[smallerIdx])); - smallerContours.push_back(contoursIn[smallerIdx]); - } + bool operator<(const MarkerCandidateTree& m) const { + // sorting the contors in descending order + return perimeter > m.perimeter; } - // to preserve the structure :: candidateSet< defaultCandidates, whiteCandidates > - // default candidates - candidatesSetOut.push_back(biggerCandidates); - contoursSetOut.push_back(biggerContours); - // white candidates - candidatesSetOut.push_back(smallerCandidates); - contoursSetOut.push_back(smallerContours); +}; + + +// returns the average distance between the marker points +float static inline getAverageDistance(const std::vector& marker1, const std::vector& marker2) { + float minDistSq = std::numeric_limits::max(); + // fc is the first corner considered on one of the markers, 4 combinations are possible + for(int fc = 0; fc < 4; fc++) { + float distSq = 0; + for(int c = 0; c < 4; c++) { + // modC is the corner considering first corner is fc + int modC = (c + fc) % 4; + distSq += normL2Sqr(marker1[modC] - marker2[c]); + } + distSq /= 4.f; + minDistSq = min(minDistSq, distSq); + } + return sqrt(minDistSq); } /** @@ -403,29 +320,6 @@ static void _detectInitialCandidates(const Mat &grey, vector > & } -/** - * @brief Detect square candidates in the input image - */ -static void _detectCandidates(InputArray _grayImage, vector > >& candidatesSetOut, - vector > >& contoursSetOut, const DetectorParameters &_params) { - Mat grey = _grayImage.getMat(); - CV_DbgAssert(grey.total() != 0); - CV_DbgAssert(grey.type() == CV_8UC1); - - /// 1. DETECT FIRST SET OF CANDIDATES - vector > candidates; - vector > contours; - _detectInitialCandidates(grey, candidates, contours, _params); - /// 2. SORT CORNERS - _reorderCandidatesCorners(candidates); - - /// 3. FILTER OUT NEAR CANDIDATE PAIRS - // save the outter/inner border (i.e. potential candidates) - _filterTooCloseCandidates(candidates, candidatesSetOut, contours, contoursSetOut, - _params.minMarkerDistanceRate, _params.detectInvertedMarker); -} - - /** * @brief Given an input image and candidate corners, extract the bits of the candidate, including * the border bits @@ -527,12 +421,10 @@ static int _getBorderErrors(const Mat &bits, int markerSize, int borderSize) { * 1 if the candidate is a black candidate (default candidate) * 2 if the candidate is a white candidate */ -static uint8_t _identifyOneCandidate(const Dictionary& dictionary, InputArray _image, +static uint8_t _identifyOneCandidate(const Dictionary& dictionary, const Mat& _image, const vector& _corners, int& idx, const DetectorParameters& params, int& rotation, const float scale = 1.f) { - CV_DbgAssert(_corners.size() == 4); - CV_DbgAssert(_image.getMat().total() != 0); CV_DbgAssert(params.markerBorderBits > 0); uint8_t typ=1; // get bits @@ -610,87 +502,6 @@ static size_t _findOptPyrImageForCanonicalImg( return optLevel; } -/** - * @brief Identify square candidates according to a marker dictionary - */ - -static void _identifyCandidates(InputArray grey, - const vector& image_pyr, - vector > >& _candidatesSet, - vector > >& _contoursSet, const Dictionary &_dictionary, - vector >& _accepted, vector >& _contours, vector& ids, - const DetectorParameters ¶ms, - OutputArrayOfArrays _rejected = noArray()) { - CV_DbgAssert(grey.getMat().total() != 0); - CV_DbgAssert(grey.getMat().type() == CV_8UC1); - int ncandidates = (int)_candidatesSet[0].size(); - vector > accepted; - vector > rejected; - vector > contours; - - vector idsTmp(ncandidates, -1); - vector rotated(ncandidates, 0); - vector validCandidates(ncandidates, 0); - - //// Analyze each of the candidates - parallel_for_(Range(0, ncandidates), [&](const Range &range) { - const int begin = range.start; - const int end = range.end; - - vector >& candidates = params.detectInvertedMarker ? _candidatesSet[1] : _candidatesSet[0]; - vector >& contourS = params.detectInvertedMarker ? _contoursSet[1] : _contoursSet[0]; - - for(int i = begin; i < end; i++) { - int currId = -1; - // implements equation (4) - if (params.useAruco3Detection) { - const int perimeterOfContour = static_cast(contourS[i].size()); - const int min_perimeter = params.minSideLengthCanonicalImg * 4; - const size_t nearestImgId = _findOptPyrImageForCanonicalImg(image_pyr, grey.cols(), perimeterOfContour, min_perimeter); - const float scale = image_pyr[nearestImgId].cols / static_cast(grey.cols()); - - validCandidates[i] = _identifyOneCandidate(_dictionary, image_pyr[nearestImgId], candidates[i], currId, params, rotated[i], scale); - } - else { - validCandidates[i] = _identifyOneCandidate(_dictionary, grey, candidates[i], currId, params, rotated[i]); - } - - if(validCandidates[i] > 0) - idsTmp[i] = currId; - } - }); - - for(int i = 0; i < ncandidates; i++) { - if(validCandidates[i] > 0) { - // to choose the right set of candidates :: 0 for default, 1 for white markers - uint8_t set = validCandidates[i]-1; - - // shift corner positions to the correct rotation - correctCornerPosition(_candidatesSet[set][i], rotated[i]); - - if( !params.detectInvertedMarker && validCandidates[i] == 2 ) - continue; - - // add valid candidate - accepted.push_back(_candidatesSet[set][i]); - ids.push_back(idsTmp[i]); - - contours.push_back(_contoursSet[set][i]); - - } else { - rejected.push_back(_candidatesSet[0][i]); - } - } - - // parse output - _accepted = accepted; - - _contours= contours; - - if(_rejected.needed()) { - _copyVector2Output(rejected, _rejected); - } -} /** * Line fitting A * B = C :: Called from function refineCandidateLines @@ -846,16 +657,210 @@ struct ArucoDetector::ArucoDetectorImpl { ArucoDetectorImpl(const Dictionary &_dictionary, const DetectorParameters &_detectorParams, const RefineParameters& _refineParams): dictionary(_dictionary), detectorParams(_detectorParams), refineParams(_refineParams) {} + /** + * @brief Detect square candidates in the input image + */ + void detectCandidates(const Mat& grey, vector >& candidates, vector >& contours) { + /// 1. DETECT FIRST SET OF CANDIDATES + _detectInitialCandidates(grey, candidates, contours, detectorParams); + /// 2. SORT CORNERS + _reorderCandidatesCorners(candidates); + } - float getAverageArucoPinSize(vector markerCorners) { - float averageArucoModuleSize = 0.f; - int numPins = dictionary.markerSize + detectorParams.markerBorderBits * 2; - for (size_t i = 0ull; i < markerCorners.size(); i++) { - averageArucoModuleSize += sqrt(normL2Sqr(Point2f(markerCorners[i] - markerCorners[(i+1ull)%markerCorners.size()]))); + /** + * @brief FILTER OUT NEAR CANDIDATE PAIRS + * + * save the outter/inner border (i.e. potential candidates) to vector, + * clear candidates and contours + */ + vector + filterTooCloseCandidates(vector > &candidates, vector > &contours) { + CV_Assert(detectorParams.minMarkerDistanceRate >= 0.); + vector candidateTree(candidates.size()); + for(size_t i = 0ull; i < candidates.size(); i++) { + candidateTree[i] = MarkerCandidateTree(std::move(candidates[i]), std::move(contours[i])); } - averageArucoModuleSize /= ((float)markerCorners.size()*numPins); - return averageArucoModuleSize; -} + candidates.clear(); + contours.clear(); + + // sort candidates from big to small + std::sort(candidateTree.begin(), candidateTree.end()); + // group index for each candidate + vector groupId(candidateTree.size(), -1); + vector > groupedCandidates; + vector isSelectedContours(candidateTree.size(), true); + + size_t countSelectedContours = 0ull; + for (size_t i = 0ull; i < candidateTree.size(); i++) { + for (size_t j = i + 1ull; j < candidateTree.size(); j++) { + float minDist = getAverageDistance(candidateTree[i].corners, candidateTree[j].corners); + // if mean distance is too low, group markers + // the distance between the points of two independent markers should be more than half the side of the marker + // half the side of the marker = (perimeter / 4) * 0.5 = perimeter * 0.125 + if(minDist < candidateTree[j].perimeter*(float)detectorParams.minMarkerDistanceRate) { + isSelectedContours[i] = false; + isSelectedContours[j] = false; + // i and j are not related to a group + if(groupId[i] < 0 && groupId[j] < 0){ + // mark candidates with their corresponding group number + groupId[i] = groupId[j] = (int)groupedCandidates.size(); + // create group + groupedCandidates.push_back({i, j}); + } + // i is related to a group + else if(groupId[i] > -1 && groupId[j] == -1) { + int group = groupId[i]; + groupId[j] = group; + // add to group + groupedCandidates[group].push_back(j); + } + // j is related to a group + else if(groupId[j] > -1 && groupId[i] == -1) { + int group = groupId[j]; + groupId[i] = group; + // add to group + groupedCandidates[group].push_back(i); + } + } + } + countSelectedContours += isSelectedContours[i]; + } + + for (vector& grouped : groupedCandidates) { + if (detectorParams.detectInvertedMarker) // if detectInvertedMarker choose smallest contours + std::sort(grouped.begin(), grouped.end(), [](const size_t &a, const size_t &b) { + return a > b; + }); + else // if detectInvertedMarker==false choose largest contours + std::sort(grouped.begin(), grouped.end()); + size_t currId = grouped[0]; + isSelectedContours[currId] = true; + for (size_t i = 1ull; i < grouped.size(); i++) { + size_t id = grouped[i]; + float dist = getAverageDistance(candidateTree[id].corners, candidateTree[currId].corners); + float moduleSize = getAverageModuleSize(candidateTree[id].corners, dictionary.markerSize, detectorParams.markerBorderBits); + if (dist > detectorParams.minGroupDistance*moduleSize) { + currId = id; + candidateTree[grouped[0]].closeContours.push_back(candidateTree[id]); + } + } + } + + vector selectedCandidates(countSelectedContours + groupedCandidates.size()); + countSelectedContours = 0ull; + for (size_t i = 0ull; i < candidateTree.size(); i++) { + if (isSelectedContours[i]) { + selectedCandidates[countSelectedContours] = std::move(candidateTree[i]); + countSelectedContours++; + } + } + + // find hierarchy in the candidate tree + for (int i = (int)selectedCandidates.size()-1; i >= 0; i--) { + for (int j = i - 1; j >= 0; j--) { + if (checkMarker1InMarker2(selectedCandidates[i].corners, selectedCandidates[j].corners)) { + selectedCandidates[i].parent = j; + selectedCandidates[j].depth = max(selectedCandidates[j].depth, selectedCandidates[i].depth + 1); + break; + } + } + } + return selectedCandidates; + } + + /** + * @brief Identify square candidates according to a marker dictionary + */ + void identifyCandidates(const Mat& grey, const vector& image_pyr, vector& selectedContours, + vector >& accepted, vector >& contours, + vector& ids, OutputArrayOfArrays _rejected = noArray()) { + size_t ncandidates = selectedContours.size(); + vector > rejected; + + vector idsTmp(ncandidates, -1); + vector rotated(ncandidates, 0); + vector validCandidates(ncandidates, 0); + vector was(ncandidates, false); + bool checkCloseContours = true; + + int maxDepth = 0; + for (size_t i = 0ull; i < selectedContours.size(); i++) + maxDepth = max(selectedContours[i].depth, maxDepth); + vector> depths(maxDepth+1); + for (size_t i = 0ull; i < selectedContours.size(); i++) { + depths[selectedContours[i].depth].push_back(i); + } + + //// Analyze each of the candidates + int depth = 0; + size_t counter = 0; + while (counter < ncandidates) { + parallel_for_(Range(0, (int)depths[depth].size()), [&](const Range& range) { + const int begin = range.start; + const int end = range.end; + for (int i = begin; i < end; i++) { + size_t v = depths[depth][i]; + was[v] = true; + Mat img = grey; + // implements equation (4) + if (detectorParams.useAruco3Detection) { + const int minPerimeter = detectorParams.minSideLengthCanonicalImg * 4; + const size_t nearestImgId = _findOptPyrImageForCanonicalImg(image_pyr, grey.cols, static_cast(selectedContours[v].contour.size()), minPerimeter); + img = image_pyr[nearestImgId]; + } + const float scale = detectorParams.useAruco3Detection ? img.cols / static_cast(grey.cols) : 1.f; + + validCandidates[v] = _identifyOneCandidate(dictionary, img, selectedContours[v].corners, idsTmp[v], detectorParams, rotated[v], scale); + + if (validCandidates[v] == 0 && checkCloseContours) { + for (const MarkerCandidate& closeMarkerCandidate: selectedContours[v].closeContours) { + validCandidates[v] = _identifyOneCandidate(dictionary, img, closeMarkerCandidate.corners, idsTmp[v], detectorParams, rotated[v], scale); + if (validCandidates[v] > 0) { + selectedContours[v].corners = closeMarkerCandidate.corners; + selectedContours[v].contour = closeMarkerCandidate.contour; + break; + } + } + } + } + }); + + // visit the parent vertices of the detected markers to skip identify parent contours + for(size_t v : depths[depth]) { + if(validCandidates[v] > 0) { + int parent = selectedContours[v].parent; + while (parent != -1) { + if (!was[parent]) { + was[parent] = true; + counter++; + } + parent = selectedContours[parent].parent; + } + } + counter++; + } + depth++; + } + + for (size_t i = 0ull; i < selectedContours.size(); i++) { + if (validCandidates[i] > 0) { + // shift corner positions to the correct rotation + correctCornerPosition(selectedContours[i].corners, rotated[i]); + + accepted.push_back(selectedContours[i].corners); + contours.push_back(selectedContours[i].contour); + ids.push_back(idsTmp[i]); + } + else { + rejected.push_back(selectedContours[i].corners); + } + } + + // parse output + if(_rejected.needed()) { + _copyVector2Output(rejected, _rejected); + } + } }; @@ -929,23 +934,21 @@ void ArucoDetector::detectMarkers(InputArray _image, OutputArrayOfArrays _corner vector > contours; vector ids; - vector > > candidatesSet; - vector > > contoursSet; - /// STEP 2.a Detect marker candidates :: using AprilTag if(detectorParams.cornerRefinementMethod == (int)CORNER_REFINE_APRILTAG){ _apriltag(grey, detectorParams, candidates, contours); - - candidatesSet.push_back(candidates); - contoursSet.push_back(contours); } /// STEP 2.b Detect marker candidates :: traditional way - else - _detectCandidates(grey, candidatesSet, contoursSet, detectorParams); + else { + arucoDetectorImpl->detectCandidates(grey, candidates, contours); + } + + /// STEP 2.c FILTER OUT NEAR CANDIDATE PAIRS + auto selectedCandidates = arucoDetectorImpl->filterTooCloseCandidates(candidates, contours); /// STEP 2: Check candidate codification (identify markers) - _identifyCandidates(grey, grey_pyramid, candidatesSet, contoursSet, dictionary, - candidates, contours, ids, detectorParams, _rejectedImgPoints); + arucoDetectorImpl->identifyCandidates(grey, grey_pyramid, selectedCandidates, candidates, contours, + ids, _rejectedImgPoints); /// STEP 3: Corner refinement :: use corner subpix if (detectorParams.cornerRefinementMethod == (int)CORNER_REFINE_SUBPIX) { @@ -963,7 +966,7 @@ void ArucoDetector::detectMarkers(InputArray _image, OutputArrayOfArrays _corner } else { int cornerRefinementWinSize = std::max(1, cvRound(detectorParams.relativeCornerRefinmentWinSize* - arucoDetectorImpl->getAverageArucoPinSize(candidates[i]))); + getAverageModuleSize(candidates[i], dictionary.markerSize, detectorParams.markerBorderBits))); cornerRefinementWinSize = min(cornerRefinementWinSize, detectorParams.cornerRefinementWinSize); cornerSubPix(grey, Mat(candidates[i]), Size(cornerRefinementWinSize, cornerRefinementWinSize), Size(-1, -1), TermCriteria(TermCriteria::MAX_ITER | TermCriteria::EPS, @@ -1238,7 +1241,7 @@ void ArucoDetector::refineDetectedMarkers(InputArray _image, const Board& _board std::vector marker(closestRotatedMarker.begin(), closestRotatedMarker.end()); int cornerRefinementWinSize = std::max(1, cvRound(detectorParams.relativeCornerRefinmentWinSize* - arucoDetectorImpl->getAverageArucoPinSize(marker))); + getAverageModuleSize(marker, dictionary.markerSize, detectorParams.markerBorderBits))); cornerRefinementWinSize = min(cornerRefinementWinSize, detectorParams.cornerRefinementWinSize); cornerSubPix(grey, closestRotatedMarker, Size(cornerRefinementWinSize, cornerRefinementWinSize), diff --git a/modules/objdetect/src/aruco/charuco_detector.cpp b/modules/objdetect/src/aruco/charuco_detector.cpp index 0ea7552a22..4f08747289 100644 --- a/modules/objdetect/src/aruco/charuco_detector.cpp +++ b/modules/objdetect/src/aruco/charuco_detector.cpp @@ -314,7 +314,9 @@ struct CharucoDetector::CharucoDetectorImpl { vector > rejectedMarkers; arucoDetector.detectMarkers(image, _markerCorners, _markerIds, rejectedMarkers); if (charucoParameters.tryRefineMarkers) - arucoDetector.refineDetectedMarkers(image, board, _markerCorners, _markerIds, rejectedMarkers); + arucoDetector.refineDetectedMarkers(image, board, _markerCorners, _markerIds, rejectedMarkers); + if (_markerCorners.empty() && _markerIds.empty()) + return; } // if camera parameters are avaible, use approximated calibration if(!charucoParameters.cameraMatrix.empty()) diff --git a/modules/objdetect/test/test_arucodetection.cpp b/modules/objdetect/test/test_arucodetection.cpp index 9cb6f07b33..7145b5d663 100644 --- a/modules/objdetect/test/test_arucodetection.cpp +++ b/modules/objdetect/test/test_arucodetection.cpp @@ -613,6 +613,32 @@ TEST(CV_ArucoDetectMarkers, regression_2492) } } + +TEST(CV_ArucoDetectMarkers, regression_contour_24220) +{ + aruco::ArucoDetector detector; + vector markerIds; + vector > markerCorners; + string imgPath = cvtest::findDataFile("aruco/failmask9.png"); + Mat image = imread(imgPath); + + const size_t N = 1ull; + const int goldCorners[8] = {392,175, 99,257, 117,109, 365,44}; + const int goldCornersId = 0; + + detector.detectMarkers(image, markerCorners, markerIds); + + ASSERT_EQ(N, markerIds.size()); + ASSERT_EQ(4ull, markerCorners[0].size()); + ASSERT_EQ(goldCornersId, markerIds[0]); + for (int j = 0; j < 4; j++) + { + EXPECT_NEAR(static_cast(goldCorners[j * 2]), markerCorners[0][j].x, 1.f); + EXPECT_NEAR(static_cast(goldCorners[j * 2 + 1]), markerCorners[0][j].y, 1.f); + } +} + + struct ArucoThreading: public testing::TestWithParam { struct NumThreadsSetter { diff --git a/modules/objdetect/test/test_charucodetection.cpp b/modules/objdetect/test/test_charucodetection.cpp index b60ad0ab51..87520c873e 100644 --- a/modules/objdetect/test/test_charucodetection.cpp +++ b/modules/objdetect/test/test_charucodetection.cpp @@ -650,7 +650,7 @@ TEST(Charuco, issue_14014) EXPECT_EQ(Size(4, 1), corners[0].size()); // check dimension of detected corners size_t numRejPoints = rejectedPoints.size(); - ASSERT_EQ(rejectedPoints.size(), 26ull); // optional check to track regressions + ASSERT_EQ(rejectedPoints.size(), 24ull); // optional check to track regressions EXPECT_EQ(Size(4, 1), rejectedPoints[0].size()); // check dimension of detected corners detector.refineDetectedMarkers(img, board, corners, ids, rejectedPoints); From 93ba5c9175eec7ad3195e1c171bc7bba21a2f605 Mon Sep 17 00:00:00 2001 From: Shubh Raheja Date: Fri, 10 Nov 2023 14:22:22 +0530 Subject: [PATCH 021/156] updated additional resources links --- .../py_knn/py_knn_understanding/py_knn_understanding.markdown | 2 +- .../py_ml/py_svm/py_svm_basics/py_svm_basics.markdown | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/doc/py_tutorials/py_ml/py_knn/py_knn_understanding/py_knn_understanding.markdown b/doc/py_tutorials/py_ml/py_knn/py_knn_understanding/py_knn_understanding.markdown index 9f76e0f808..5985cdd559 100644 --- a/doc/py_tutorials/py_ml/py_knn/py_knn_understanding/py_knn_understanding.markdown +++ b/doc/py_tutorials/py_ml/py_knn/py_knn_understanding/py_knn_understanding.markdown @@ -141,7 +141,7 @@ Additional Resources -------------------- 1. [NPTEL notes on Pattern Recognition, Chapter - 11](https://nptel.ac.in/courses/106/108/106108057/) + 11](https://nptel.ac.in/courses/106108057) 2. [Wikipedia article on Nearest neighbor search](https://en.wikipedia.org/wiki/Nearest_neighbor_search) 3. [Wikipedia article on k-d tree](https://en.wikipedia.org/wiki/K-d_tree) diff --git a/doc/py_tutorials/py_ml/py_svm/py_svm_basics/py_svm_basics.markdown b/doc/py_tutorials/py_ml/py_svm/py_svm_basics/py_svm_basics.markdown index c8dbe39920..55f74237e9 100644 --- a/doc/py_tutorials/py_ml/py_svm/py_svm_basics/py_svm_basics.markdown +++ b/doc/py_tutorials/py_ml/py_svm/py_svm_basics/py_svm_basics.markdown @@ -129,7 +129,6 @@ Additional Resources -------------------- -# [NPTEL notes on Statistical Pattern Recognition, Chapters - 25-29](http://www.nptel.ac.in/courses/106108057/26). - + 25-29](https://nptel.ac.in/courses/117108048) Exercises --------- From 53f2131681375757805bcdf0255afd60bab9f85f Mon Sep 17 00:00:00 2001 From: Dmitry Kurtaev Date: Sat, 11 Nov 2023 09:06:10 +0300 Subject: [PATCH 022/156] Merge pull request #24521 from dkurt:fix_broken_urls Fix some of the broken urls in docs #24521 ### Pull Request Readiness Checklist See details at https://github.com/opencv/opencv/wiki/How_to_contribute#making-a-good-pull-request - [x] I agree to contribute to the project under Apache 2 License. - [x] To the best of my knowledge, the proposed patch is not based on a code under GPL or another license that is incompatible with OpenCV - [x] The PR is proposed to the proper branch - [x] There is a reference to the original bug report and related work - [x] There is accuracy test, performance test and test data in opencv_extra repository, if applicable Patch to opencv_extra has the same branch name. - [x] The feature is well documented and sample code can be built with the project CMake --- doc/LICENSE_CHANGE_NOTICE.txt | 2 +- doc/js_tutorials/js_setup/js_setup/js_setup.markdown | 2 +- .../py_gui/py_video_display/py_video_display.markdown | 2 +- doc/tutorials/core/univ_intrin/univ_intrin.markdown | 2 +- .../out_of_focus_deblur_filter.markdown | 2 +- .../building_tegra_cuda/building_tegra_cuda.markdown | 2 +- doc/tutorials/introduction/java_eclipse/java_eclipse.markdown | 2 +- doc/tutorials/introduction/linux_install/linux_install.markdown | 2 +- doc/tutorials/introduction/macos_install/macos_install.markdown | 2 +- .../introduction/transition_guide/transition_guide.markdown | 2 +- modules/videoio/include/opencv2/videoio.hpp | 2 +- modules/videoio/include/opencv2/videoio/legacy/constants_c.h | 2 +- 12 files changed, 12 insertions(+), 12 deletions(-) diff --git a/doc/LICENSE_CHANGE_NOTICE.txt b/doc/LICENSE_CHANGE_NOTICE.txt index 0df3eca671..c2711d4671 100644 --- a/doc/LICENSE_CHANGE_NOTICE.txt +++ b/doc/LICENSE_CHANGE_NOTICE.txt @@ -1,4 +1,4 @@ -Starting from OpenCV 4.5-pre (2020 August) OpenCV has changed the license from BSD to Apache 2. See https://opencv.org/opencv-is-to-change-the-license-to-apache-2/ and https://github.com/opencv/opencv/wiki/OE-32.--Change-OpenCV-License-to-Apache-2 for details. +Starting from OpenCV 4.5-pre (2020 August) OpenCV has changed the license from BSD to Apache 2. See https://opencv.org/blog/opencv-is-to-change-the-license-to-apache-2/ and https://github.com/opencv/opencv/wiki/OE-32.--Change-OpenCV-License-to-Apache-2 for details. Here is the original OpenCV license: ------------------------------------------------------------------------------------ diff --git a/doc/js_tutorials/js_setup/js_setup/js_setup.markdown b/doc/js_tutorials/js_setup/js_setup/js_setup.markdown index 5b0e65b250..87a32a78cb 100644 --- a/doc/js_tutorials/js_setup/js_setup/js_setup.markdown +++ b/doc/js_tutorials/js_setup/js_setup/js_setup.markdown @@ -54,7 +54,7 @@ repository](https://github.com/opencv/opencv.git). ### Obtaining the Latest Stable OpenCV Version -- Go to our [releases page](http://opencv.org/releases.html). +- Go to our [releases page](https://opencv.org/releases). - Download the source archive and unpack it. ### Obtaining the Cutting-edge OpenCV from the Git Repository diff --git a/doc/py_tutorials/py_gui/py_video_display/py_video_display.markdown b/doc/py_tutorials/py_gui/py_video_display/py_video_display.markdown index d60b846245..5819653fa0 100644 --- a/doc/py_tutorials/py_gui/py_video_display/py_video_display.markdown +++ b/doc/py_tutorials/py_gui/py_video_display/py_video_display.markdown @@ -111,7 +111,7 @@ frames per second (fps) and frame size should be passed. And the last one is the `True`, the encoder expect color frame, otherwise it works with grayscale frame. [FourCC](http://en.wikipedia.org/wiki/FourCC) is a 4-byte code used to specify the video codec. The -list of available codes can be found in [fourcc.org](http://www.fourcc.org/codecs.php). It is +list of available codes can be found in [fourcc.org](https://fourcc.org/codecs.php). It is platform dependent. The following codecs work fine for me. - In Fedora: DIVX, XVID, MJPG, X264, WMV1, WMV2. (XVID is more preferable. MJPG results in high diff --git a/doc/tutorials/core/univ_intrin/univ_intrin.markdown b/doc/tutorials/core/univ_intrin/univ_intrin.markdown index 894a7440a0..a80b6d4bd3 100644 --- a/doc/tutorials/core/univ_intrin/univ_intrin.markdown +++ b/doc/tutorials/core/univ_intrin/univ_intrin.markdown @@ -245,7 +245,7 @@ In the following section, we will vectorize a simple convolution function for si You may learn more about convolution from the previous tutorial. We use the same naive implementation from the previous tutorial and compare it to the vectorized version. -The full tutorial code is [here](https://github.com/opencv/opencv/tree/4.x/samples/cpp/tutorial_code/univ_intrin/univ_intrin.cpp). +The full tutorial code is [here](https://github.com/opencv/opencv/tree/4.x/samples/cpp/tutorial_code/core/univ_intrin/univ_intrin.cpp). ### Vectorizing Convolution diff --git a/doc/tutorials/imgproc/out_of_focus_deblur_filter/out_of_focus_deblur_filter.markdown b/doc/tutorials/imgproc/out_of_focus_deblur_filter/out_of_focus_deblur_filter.markdown index 13db710b32..d2dc68bc90 100644 --- a/doc/tutorials/imgproc/out_of_focus_deblur_filter/out_of_focus_deblur_filter.markdown +++ b/doc/tutorials/imgproc/out_of_focus_deblur_filter/out_of_focus_deblur_filter.markdown @@ -117,6 +117,6 @@ References - [SmartDeblur] - SmartDeblur site -[Digital Image Processing]: http://web.ipac.caltech.edu/staff/fmasci/home/astro_refs/Digital_Image_Processing_2ndEd.pdf +[Digital Image Processing]: http://web.ipac.caltech.edu/staff/fmasci/home/RefMaterial/ImageProc/Book_DigitalImageProcessing.pdf [Image Deblurring in Matlab]: https://www.mathworks.com/help/images/image-deblurring.html [SmartDeblur]: http://yuzhikov.com/articles/BlurredImagesRestoration1.htm diff --git a/doc/tutorials/introduction/building_tegra_cuda/building_tegra_cuda.markdown b/doc/tutorials/introduction/building_tegra_cuda/building_tegra_cuda.markdown index a56c0fa17f..753b3efe58 100644 --- a/doc/tutorials/introduction/building_tegra_cuda/building_tegra_cuda.markdown +++ b/doc/tutorials/introduction/building_tegra_cuda/building_tegra_cuda.markdown @@ -40,7 +40,7 @@ Getting the Source Code {#tutorial_building_tegra_cuda_getting_the_code} There are two (2) ways to get the OpenCV source code: -* Direct download from the [OpenCV downloads](http://opencv.org/releases.html) page +* Direct download from the [OpenCV downloads](https://opencv.org/releases) page * Cloning the git repositories hosted on [GitHub](https://github.com/opencv) For this guide, the focus is on using the git repositories. This is because the 3.1.0 version of OpenCV will not build with CUDA 8.0 without applying a few small upstream changes from the git repository. diff --git a/doc/tutorials/introduction/java_eclipse/java_eclipse.markdown b/doc/tutorials/introduction/java_eclipse/java_eclipse.markdown index 9d9434adf7..46e2aac5d1 100644 --- a/doc/tutorials/introduction/java_eclipse/java_eclipse.markdown +++ b/doc/tutorials/introduction/java_eclipse/java_eclipse.markdown @@ -21,7 +21,7 @@ less mistakes. Here we go. Configuring Eclipse ------------------- -First, obtain a fresh release of OpenCV [from download page](http://opencv.org/releases.html) and +First, obtain a fresh release of OpenCV [from download page](https://opencv.org/releases) and extract it under a simple location like `C:\OpenCV-2.4.6\`. I am using version 2.4.6, but the steps are more or less the same for other versions. diff --git a/doc/tutorials/introduction/linux_install/linux_install.markdown b/doc/tutorials/introduction/linux_install/linux_install.markdown index e69f6ea707..77855e069e 100644 --- a/doc/tutorials/introduction/linux_install/linux_install.markdown +++ b/doc/tutorials/introduction/linux_install/linux_install.markdown @@ -65,7 +65,7 @@ There are two methods of getting OpenCV sources: @note -Snapshots of other branches, releases or commits can be found on the [GitHub](https://github.com/opencv/opencv) and the [official download page](https://opencv.org/releases.html). +Snapshots of other branches, releases or commits can be found on the [GitHub](https://github.com/opencv/opencv) and the [official download page](https://opencv.org/releases). ## Configure and build {#tutorial_linux_install_detailed_basic_build} diff --git a/doc/tutorials/introduction/macos_install/macos_install.markdown b/doc/tutorials/introduction/macos_install/macos_install.markdown index dadce9304c..f979f50b94 100644 --- a/doc/tutorials/introduction/macos_install/macos_install.markdown +++ b/doc/tutorials/introduction/macos_install/macos_install.markdown @@ -53,7 +53,7 @@ You can use the latest stable OpenCV version or you can grab the latest snapshot ### Getting the Latest Stable OpenCV Version -- Go to our [downloads page](http://opencv.org/releases.html). +- Go to our [downloads page](https://opencv.org/releases). - Download the source archive and unpack it. ### Getting the Cutting-edge OpenCV from the Git Repository diff --git a/doc/tutorials/introduction/transition_guide/transition_guide.markdown b/doc/tutorials/introduction/transition_guide/transition_guide.markdown index 454d3ca051..43bc5dd958 100644 --- a/doc/tutorials/introduction/transition_guide/transition_guide.markdown +++ b/doc/tutorials/introduction/transition_guide/transition_guide.markdown @@ -201,7 +201,7 @@ All specialized `ocl` implementations has been hidden behind general C++ algorit New class cv::UMat is intended to hide data exchange with OpenCL device in a convenient way. -Following example illustrate API modifications (from [OpenCV site](http://opencv.org/platforms/opencl.html)): +Following example illustrate API modifications (from [OpenCV site](https://opencv.org/opencl)): - OpenCL-aware code OpenCV-2.x @code{.cpp} diff --git a/modules/videoio/include/opencv2/videoio.hpp b/modules/videoio/include/opencv2/videoio.hpp index 0766bc4bdd..2ea95ac5b5 100644 --- a/modules/videoio/include/opencv2/videoio.hpp +++ b/modules/videoio/include/opencv2/videoio.hpp @@ -1023,7 +1023,7 @@ public: VideoWriter::fourcc('P','I','M','1') is a MPEG-1 codec, VideoWriter::fourcc('M','J','P','G') is a motion-jpeg codec etc. List of codes can be obtained at [MSDN](https://docs.microsoft.com/en-us/windows/win32/medfound/video-fourccs) page - or with this [archived page](https://web.archive.org/web/20220316062600/http://www.fourcc.org/codecs.php) + or with this [page](https://fourcc.org/codecs.php) of the fourcc site for a more complete list). FFMPEG backend with MP4 container natively uses other values as fourcc code: see [ObjectType](http://mp4ra.org/#/codecs), so you may receive a warning message from OpenCV about fourcc code conversion. diff --git a/modules/videoio/include/opencv2/videoio/legacy/constants_c.h b/modules/videoio/include/opencv2/videoio/legacy/constants_c.h index 91f85f87b8..f9831e358a 100644 --- a/modules/videoio/include/opencv2/videoio/legacy/constants_c.h +++ b/modules/videoio/include/opencv2/videoio/legacy/constants_c.h @@ -417,7 +417,7 @@ enum Simply call it with 4 chars fourcc code like `CV_FOURCC('I', 'Y', 'U', 'V')` -List of codes can be obtained at [Video Codecs by FOURCC](http://www.fourcc.org/codecs.php) page. +List of codes can be obtained at [Video Codecs by FOURCC](https://fourcc.org/codecs.php) page. FFMPEG backend with MP4 container natively uses other values as fourcc code: see [ObjectType](http://mp4ra.org/#/codecs). */ From 960a9260551314c42a401a4c61a17a4c4bfecd62 Mon Sep 17 00:00:00 2001 From: Alexander Smorkalov <2536374+asmorkalov@users.noreply.github.com> Date: Sat, 11 Nov 2023 09:09:14 +0300 Subject: [PATCH 023/156] Merge pull request #24510 from asmorkalov:as/softmax_rvv Enable softmax layer vectorization on RISC-V RVV #24510 Related: https://github.com/opencv/opencv/pull/24466 ### Pull Request Readiness Checklist See details at https://github.com/opencv/opencv/wiki/How_to_contribute#making-a-good-pull-request - [x] I agree to contribute to the project under Apache 2 License. - [x] To the best of my knowledge, the proposed patch is not based on a code under GPL or another license that is incompatible with OpenCV - [x] The PR is proposed to the proper branch - [x] There is a reference to the original bug report and related work - [x] There is accuracy test, performance test and test data in opencv_extra repository, if applicable Patch to opencv_extra has the same branch name. - [ ] The feature is well documented and sample code can be built with the project CMake --- modules/dnn/src/layers/cpu_kernels/softmax.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/modules/dnn/src/layers/cpu_kernels/softmax.cpp b/modules/dnn/src/layers/cpu_kernels/softmax.cpp index 15e50f17bd..eb258ecfa2 100644 --- a/modules/dnn/src/layers/cpu_kernels/softmax.cpp +++ b/modules/dnn/src/layers/cpu_kernels/softmax.cpp @@ -35,7 +35,7 @@ void softmax(Mat &dst, const Mat &src, int axis, int axisBias, int axisStep){ // make the channel axis to be multiple of 8 size_t channelAxis = (axisStep + 7) & -8; -#if CV_SIMD +#if (CV_SIMD || CV_SIMD_SCALABLE) const int nlanes = VTraits::vlanes(); // the number of redundant dimension size_t redundantDim = nlanes - axisStep % nlanes; @@ -54,7 +54,7 @@ void softmax(Mat &dst, const Mat &src, int axis, int axisBias, int axisStep){ axisBuf[cnDim] = srcPtr[srcOffset + (cnDim + axisBias) * cnStep]; float s = 0.f; -#if CV_SIMD +#if (CV_SIMD || CV_SIMD_SCALABLE) // make the value of the redundant dimension to be -FLT_MAX if (redundantDim != nlanes) { for (size_t j = axisStep; j < axisStep + redundantDim; j++) @@ -121,7 +121,7 @@ void softmax(Mat &dst, const Mat &src, int axis, int axisBias, int axisStep){ s = v_reduce_sum(vs); // subtract the value of the redundant dimension if (redundantDim != nlanes) { - float* _val = new float[nlanes]; + float _val[VTraits::max_nlanes]; v_store(_val, val); for (size_t j = nlanes - redundantDim; j < nlanes; j++) s -= _val[j]; From 031846f2e1427d0dc507d37cfebe349bb74ea65d Mon Sep 17 00:00:00 2001 From: fengyuentau Date: Mon, 13 Nov 2023 14:47:40 +0800 Subject: [PATCH 024/156] remove filter --- modules/dnn/test/test_onnx_importer.cpp | 3 --- 1 file changed, 3 deletions(-) diff --git a/modules/dnn/test/test_onnx_importer.cpp b/modules/dnn/test/test_onnx_importer.cpp index cea4ffd739..7be3e8a385 100644 --- a/modules/dnn/test/test_onnx_importer.cpp +++ b/modules/dnn/test/test_onnx_importer.cpp @@ -118,9 +118,6 @@ public: TEST_P(Test_ONNX_layers, InstanceNorm) { - if(backend == DNN_BACKEND_CUDA) - applyTestTag(CV_TEST_TAG_DNN_SKIP_CUDA); /* MVN is not supported */ - if (target == DNN_TARGET_MYRIAD) testONNXModels("instancenorm", npy, 0, 0, false, false); else From 9c86d68b4ce32051843f130b616ef26bfe96da5c Mon Sep 17 00:00:00 2001 From: Alex Date: Mon, 13 Nov 2023 10:49:19 +0300 Subject: [PATCH 025/156] remove filter fix --- modules/objdetect/src/qrcode.cpp | 17 +++-------------- 1 file changed, 3 insertions(+), 14 deletions(-) diff --git a/modules/objdetect/src/qrcode.cpp b/modules/objdetect/src/qrcode.cpp index 05f8dabeb6..811337b416 100644 --- a/modules/objdetect/src/qrcode.cpp +++ b/modules/objdetect/src/qrcode.cpp @@ -4476,25 +4476,14 @@ static vector analyzeFinderPatterns(const vector > &corners, const Mat& img, const QRCodeDetectorAruco::Params& qrDetectorParameters) { vector qrCodes; - vector patterns; + vector patterns(corners.size()); if (img.empty()) return qrCodes; float maxModuleSize = 0.f; for (size_t i = 0ull; i < corners.size(); i++) { FinderPatternInfo pattern = FinderPatternInfo(corners[i]); - // TODO: improve thinning Aruco markers - bool isUniq = true; - for (const FinderPatternInfo& tmp : patterns) { - Point2f dist = pattern.center - tmp.center; - if (max(abs(dist.x), abs(dist.y)) < 3.f * tmp.moduleSize) { - isUniq = false; - break; - } - } - if (isUniq) { - patterns.push_back(pattern); - maxModuleSize = max(maxModuleSize, patterns.back().moduleSize); - } + patterns[i] = pattern; + maxModuleSize = max(maxModuleSize, pattern.moduleSize); } const int threshold = cvRound(qrDetectorParameters.minModuleSizeInPyramid * 12.5f) + (cvRound(qrDetectorParameters.minModuleSizeInPyramid * 12.5f) % 2 ? 0 : 1); From 7cd4fc2fb8adbf3e03ca3a17f636db45f09940e7 Mon Sep 17 00:00:00 2001 From: thewoz Date: Sat, 11 Nov 2023 14:11:01 +0100 Subject: [PATCH 026/156] Update window_QT.cpp fix tab arrow issue --- modules/highgui/src/window_QT.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/modules/highgui/src/window_QT.cpp b/modules/highgui/src/window_QT.cpp index 56c51d2a02..e764b8544c 100644 --- a/modules/highgui/src/window_QT.cpp +++ b/modules/highgui/src/window_QT.cpp @@ -2544,6 +2544,10 @@ DefaultViewPort::DefaultViewPort(CvWindow* arg, int arg2) : QGraphicsView(arg), setInteractive(false); setMouseTracking(true); //receive mouse event everytime + + // #13657 Tab key disables arrow keys + // #20215 QT backend: cv::waitKey() and cv::waitKeyEx() do not capture arrow keys once you click on the image or press TAB + setFocusPolicy(Qt::NoFocus); } From adc55608f1af9ab544cfd325ce1918691c282d20 Mon Sep 17 00:00:00 2001 From: laolaolulu <33310261+laolaolulu@users.noreply.github.com> Date: Mon, 13 Nov 2023 19:51:20 +0800 Subject: [PATCH 027/156] Merge pull request #24458 from laolaolulu:4.x MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Added JS binding for `getUnconnectedOutLayersNames` * Fix error: no member named ‘vectorstd’ in namespace ‘std’ --- modules/js/generator/embindgen.py | 4 ++-- modules/js/src/core_bindings.cpp | 1 + platforms/js/opencv_js.config.py | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/modules/js/generator/embindgen.py b/modules/js/generator/embindgen.py index 8a16f92f5e..005ac9f175 100644 --- a/modules/js/generator/embindgen.py +++ b/modules/js/generator/embindgen.py @@ -482,7 +482,7 @@ class JSWrapperGenerator(object): ret_type = type_dict[ptr_type] for key in type_dict: if key in ret_type: - ret_type = re.sub('(^|[^\w])' + key + '($|[^\w])', type_dict[key], ret_type) + ret_type = re.sub(r"\b" + key + r"\b", type_dict[key], ret_type) arg_types = [] unwrapped_arg_types = [] for arg in variant.args: @@ -670,7 +670,7 @@ class JSWrapperGenerator(object): # Replace types. Instead of ret_type.replace we use regular # expression to exclude false matches. # See https://github.com/opencv/opencv/issues/15514 - ret_type = re.sub('(^|[^\w])' + key + '($|[^\w])', type_dict[key], ret_type) + ret_type = re.sub(r"\b" + key + r"\b", type_dict[key], ret_type) if variant.constret and ret_type.startswith('const') == False: ret_type = 'const ' + ret_type if variant.refret and ret_type.endswith('&') == False: diff --git a/modules/js/src/core_bindings.cpp b/modules/js/src/core_bindings.cpp index dda6f9fe16..d5bf9b076c 100644 --- a/modules/js/src/core_bindings.cpp +++ b/modules/js/src/core_bindings.cpp @@ -465,6 +465,7 @@ EMSCRIPTEN_BINDINGS(binding_utils) register_vector("CharVector"); register_vector("FloatVector"); register_vector("DoubleVector"); + register_vector("StringVector"); register_vector("PointVector"); register_vector("MatVector"); register_vector("RectVector"); diff --git a/platforms/js/opencv_js.config.py b/platforms/js/opencv_js.config.py index 44f0813838..f3ed847ef5 100644 --- a/platforms/js/opencv_js.config.py +++ b/platforms/js/opencv_js.config.py @@ -147,7 +147,7 @@ video = { 'TrackerMIL_Params': [], } -dnn = {'dnn_Net': ['setInput', 'forward', 'setPreferableBackend'], +dnn = {'dnn_Net': ['setInput', 'forward', 'setPreferableBackend','getUnconnectedOutLayersNames'], '': ['readNetFromCaffe', 'readNetFromTensorflow', 'readNetFromTorch', 'readNetFromDarknet', 'readNetFromONNX', 'readNetFromTFLite', 'readNet', 'blobFromImage']} From 4a69877eaaa84f5f20866e1a37933d191484041a Mon Sep 17 00:00:00 2001 From: "Alessandro de Oliveira Faria (A.K.A.CABELO)" Date: Tue, 14 Nov 2023 03:06:36 -0300 Subject: [PATCH 028/156] Merge pull request #24496 from cabelo:yolov3 Add weights yolov3 in models.yml #24496 ### Pull Request Readiness Checklist See details at https://github.com/opencv/opencv/wiki/How_to_contribute#making-a-good-pull-request - [X] I agree to contribute to the project under Apache 2 License. - [X] To the best of my knowledge, the proposed patch is not based on a code under GPL or another license that is incompatible with OpenCV - [X] The PR is proposed to the proper branch - [X] There is a reference to the original bug report and related work - [X] There is accuracy test, performance test and test data in opencv_extra repository, if applicable Patch to opencv_extra has the same branch name. - [X] The feature is well documented and sample code can be built with the project CMake I don't know if this action is necessary, or the previous PR scale for the brach master. Thanks. --- ....txt => object_detection_classes_yolo.txt} | 0 samples/dnn/models.yml | 20 ++++++++++++++++--- 2 files changed, 17 insertions(+), 3 deletions(-) rename samples/data/dnn/{object_detection_classes_yolov4.txt => object_detection_classes_yolo.txt} (100%) diff --git a/samples/data/dnn/object_detection_classes_yolov4.txt b/samples/data/dnn/object_detection_classes_yolo.txt similarity index 100% rename from samples/data/dnn/object_detection_classes_yolov4.txt rename to samples/data/dnn/object_detection_classes_yolo.txt diff --git a/samples/dnn/models.yml b/samples/dnn/models.yml index c780bcb139..53d8b8048f 100644 --- a/samples/dnn/models.yml +++ b/samples/dnn/models.yml @@ -21,7 +21,7 @@ opencv_fd: # YOLO4 object detection family from Darknet (https://github.com/AlexeyAB/darknet) # YOLO object detection family from Darknet (https://pjreddie.com/darknet/yolo/) # Might be used for all YOLOv2, TinyYolov2, YOLOv3, YOLOv4 and TinyYolov4 -yolo: +yolov4: load_info: url: "https://github.com/AlexeyAB/darknet/releases/download/darknet_yolo_v3_optimal/yolov4.weights" sha1: "0143deb6c46fcc7f74dd35bf3c14edc3784e99ee" @@ -32,7 +32,7 @@ yolo: width: 416 height: 416 rgb: true - classes: "object_detection_classes_yolov4.txt" + classes: "object_detection_classes_yolo.txt" sample: "object_detection" yolov4-tiny: @@ -46,7 +46,21 @@ yolov4-tiny: width: 416 height: 416 rgb: true - classes: "object_detection_classes_yolov4.txt" + classes: "object_detection_classes_yolo.txt" + sample: "object_detection" + +yolov3: + load_info: + url: "https://pjreddie.com/media/files/yolov3.weights" + sha1: "520878f12e97cf820529daea502acca380f1cb8e" + model: "yolov3.weights" + config: "yolov3.cfg" + mean: [0, 0, 0] + scale: 0.00392 + width: 416 + height: 416 + rgb: true + classes: "object_detection_classes_yolo.txt" sample: "object_detection" tiny-yolo-voc: From ec97c38ff90a18fec2934a34a8d9cd6d3a8901f5 Mon Sep 17 00:00:00 2001 From: Dmitry Kurtaev Date: Tue, 14 Nov 2023 11:44:17 +0300 Subject: [PATCH 029/156] Merge pull request #24535 from dkurt:ipp_distransform_update Handle huge images in IPP distanceTransform #24535 ### Pull Request Readiness Checklist * Do not use IPP for huge Mat (reproduced with https://github.com/opencv/opencv/issues/23895#issuecomment-1708132367 on `DIST_MASK_5`) I have observed two types of errors on the reproducer from the issue: 1. When `temp` is not allocated: ``` Thread 1 "app" received signal SIGSEGV, Segmentation fault. 0x00007ffff65dc755 in icv_l9_ownDistanceTransform_5x5_8u32f_C1R_21B_g9e9 () from /home/dkurtaev/opencv_install/bin/../lib/libopencv_imgproc.so.408 (gdb) bt #0 0x00007ffff65dc755 in icv_l9_ownDistanceTransform_5x5_8u32f_C1R_21B_g9e9 () from /home/dkurtaev/opencv_install/bin/../lib/libopencv_imgproc.so.408 #1 0x00007ffff659e8df in icv_l9_ippiDistanceTransform_5x5_8u32f_C1R () from /home/dkurtaev/opencv_install/bin/../lib/libopencv_imgproc.so.408 #2 0x00007ffff5c390f0 in cv::distanceTransform (_src=..., _dst=..., _labels=..., distType=2, maskSize=5, labelType=1) at /home/dkurtaev/opencv/modules/imgproc/src/distransform.cpp:854 #3 0x00007ffff5c396ef in cv::distanceTransform (_src=..., _dst=..., distanceType=2, maskSize=5, dstType=5) at /home/dkurtaev/opencv/modules/imgproc/src/distransform.cpp:903 #4 0x000055555555669e in main (argc=1, argv=0x7fffffffdef8) at /home/dkurtaev/main.cpp:18 ``` 2. When we keep `temp` allocated every time: ``` OpenCV(4.8.0-dev) Error: Assertion failed (udata < (uchar*)ptr && ((uchar*)ptr - udata) <= (ptrdiff_t)(sizeof(void*)+64)) in fastFree, file /home/dkurtaev/opencv/modules/core/src/alloc.cpp, line 191 terminate called after throwing an instance of 'cv::Exception' what(): OpenCV(4.8.0-dev) /home/dkurtaev/opencv/modules/core/src/alloc.cpp:191: error: (-215:Assertion failed) udata < (uchar*)ptr && ((uchar*)ptr - udata) <= (ptrdiff_t)(sizeof(void*)+64) in function 'fastFree' ``` * Try enable IPP for 3x3 (see https://github.com/opencv/opencv/issues/15904) * Reduce memory footprint with IPP See details at https://github.com/opencv/opencv/wiki/How_to_contribute#making-a-good-pull-request - [x] I agree to contribute to the project under Apache 2 License. - [x] To the best of my knowledge, the proposed patch is not based on a code under GPL or another license that is incompatible with OpenCV - [x] The PR is proposed to the proper branch - [x] There is a reference to the original bug report and related work - [x] There is accuracy test, performance test and test data in opencv_extra repository, if applicable Patch to opencv_extra has the same branch name. - [x] The feature is well documented and sample code can be built with the project CMake --- modules/imgproc/src/distransform.cpp | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/modules/imgproc/src/distransform.cpp b/modules/imgproc/src/distransform.cpp index 38223a0194..52167b2264 100644 --- a/modules/imgproc/src/distransform.cpp +++ b/modules/imgproc/src/distransform.cpp @@ -817,14 +817,15 @@ void cv::distanceTransform( InputArray _src, OutputArray _dst, OutputArray _labe Size size = src.size(); int border = maskSize == CV_DIST_MASK_3 ? 1 : 2; - Mat temp( size.height + border*2, size.width + border*2, CV_32SC1 ); + Mat temp; if( !need_labels ) { if( maskSize == CV_DIST_MASK_3 ) { -#if defined (HAVE_IPP) && (IPP_VERSION_X100 >= 700) && 0 // disabled: https://github.com/opencv/opencv/issues/15904 - CV_IPP_CHECK() +#if defined (HAVE_IPP) && (IPP_VERSION_X100 >= 700) + bool has_int_overflow = (int64)src.cols * src.rows >= INT_MAX; + if (!has_int_overflow && CV_IPP_CHECK_COND) { IppiSize roi = { src.cols, src.rows }; if (CV_INSTRUMENT_FUN_IPP(ippiDistanceTransform_3x3_8u32f_C1R, src.ptr(), (int)src.step, dst.ptr(), (int)dst.step, roi, _mask) >= 0) @@ -836,12 +837,14 @@ void cv::distanceTransform( InputArray _src, OutputArray _dst, OutputArray _labe } #endif + temp.create(size.height + border*2, size.width + border*2, CV_32SC1); distanceTransform_3x3(src, temp, dst, _mask); } else { #if defined (HAVE_IPP) && (IPP_VERSION_X100 >= 700) - CV_IPP_CHECK() + bool has_int_overflow = (int64)src.cols * src.rows >= INT_MAX; + if (!has_int_overflow && CV_IPP_CHECK_COND) { IppiSize roi = { src.cols, src.rows }; if (CV_INSTRUMENT_FUN_IPP(ippiDistanceTransform_5x5_8u32f_C1R, src.ptr(), (int)src.step, dst.ptr(), (int)dst.step, roi, _mask) >= 0) @@ -853,6 +856,7 @@ void cv::distanceTransform( InputArray _src, OutputArray _dst, OutputArray _labe } #endif + temp.create(size.height + border*2, size.width + border*2, CV_32SC1); distanceTransform_5x5(src, temp, dst, _mask); } } @@ -879,7 +883,8 @@ void cv::distanceTransform( InputArray _src, OutputArray _dst, OutputArray _labe } } - distanceTransformEx_5x5( src, temp, dst, labels, _mask ); + temp.create(size.height + border*2, size.width + border*2, CV_32SC1); + distanceTransformEx_5x5( src, temp, dst, labels, _mask ); } } From 024dfd54aff7459166e900962479d4643be746a8 Mon Sep 17 00:00:00 2001 From: Yuantao Feng Date: Wed, 15 Nov 2023 22:57:52 +0800 Subject: [PATCH 030/156] dnn cann backend: add hardswish, layernorm and instasnce norm for cann and bug fix (#24462) * add hardswish for cann * gemm cann bug fix * fix indentation * cann: add layer norm * cann: add instance norm * add supportBackend * cann: layer norm does not support axis=-1 due to 1d mat issue * disable instance norm for now * fix doc * remove tensor desc initialization for 1D tensor --- modules/dnn/src/layers/elementwise_layers.cpp | 25 +++++++- modules/dnn/src/layers/gemm_layer.cpp | 1 + .../dnn/src/layers/instance_norm_layer.cpp | 49 +++++++++++++++ modules/dnn/src/layers/layer_norm.cpp | 59 ++++++++++++++++++- 4 files changed, 132 insertions(+), 2 deletions(-) diff --git a/modules/dnn/src/layers/elementwise_layers.cpp b/modules/dnn/src/layers/elementwise_layers.cpp index 2a2245b909..746db69603 100644 --- a/modules/dnn/src/layers/elementwise_layers.cpp +++ b/modules/dnn/src/layers/elementwise_layers.cpp @@ -1890,7 +1890,9 @@ struct HardSwishFunctor : public BaseDefaultFunctor bool supportBackend(int backendId, int) { - return backendId == DNN_BACKEND_OPENCV || backendId == DNN_BACKEND_CUDA; + return backendId == DNN_BACKEND_OPENCV || + backendId == DNN_BACKEND_CUDA || + backendId == DNN_BACKEND_CANN; } inline float calculate(float x) const @@ -1905,6 +1907,27 @@ struct HardSwishFunctor : public BaseDefaultFunctor } #endif +#ifdef HAVE_CANN + Ptr initCannOp(const std::string& name, + const std::vector > &inputs, + const std::vector >& nodes) + { + auto x = inputs[0].dynamicCast(); + + auto op = std::make_shared(name); + + auto op_x = nodes[0].dynamicCast()->getOp(); + op->set_input_x_by_name(*op_x, x->name.c_str()); + auto x_desc = x->getTensorDesc(); + op->update_input_desc_x(*x_desc); + + auto output_desc = std::make_shared(ge::Shape(), ge::FORMAT_NCHW, ge::DT_FLOAT); + op->update_output_desc_y(*output_desc); + + return Ptr(new CannBackendNode(op)); + } +#endif + int64 getFLOPSPerElement() const { return 1; } }; diff --git a/modules/dnn/src/layers/gemm_layer.cpp b/modules/dnn/src/layers/gemm_layer.cpp index a553f97568..8bcec78343 100644 --- a/modules/dnn/src/layers/gemm_layer.cpp +++ b/modules/dnn/src/layers/gemm_layer.cpp @@ -274,6 +274,7 @@ public: op->update_input_desc_bias(*(op_const_C->getTensorDesc())); // set outputs + auto output_desc = std::make_shared(ge::Shape(), ge::FORMAT_NCHW, ge::DT_FLOAT); op->update_output_desc_y(*output_desc); return Ptr(new CannBackendNode(op)); } diff --git a/modules/dnn/src/layers/instance_norm_layer.cpp b/modules/dnn/src/layers/instance_norm_layer.cpp index fda0efdb94..b43e9bbb7a 100644 --- a/modules/dnn/src/layers/instance_norm_layer.cpp +++ b/modules/dnn/src/layers/instance_norm_layer.cpp @@ -6,6 +6,9 @@ #include #include "./cpu_kernels/fast_norm.hpp" +// CANN backend +#include "../op_cann.hpp" + // OpenVINO backend #include "../op_inf_engine.hpp" #include "../ie_ngraph.hpp" @@ -41,6 +44,7 @@ public: #endif return backendId == DNN_BACKEND_OPENCV || backendId == DNN_BACKEND_CUDA; + // backendId == DNN_BACKEND_CANN; // not supported due to 1d mat shape issue } bool getMemoryShapes(const std::vector &inputs, @@ -169,6 +173,51 @@ public: } #endif +#ifdef HAVE_CANN + virtual Ptr initCann(const std::vector > &inputs, + const std::vector > &outputs, + const std::vector >& nodes) CV_OVERRIDE { + auto input_tensor_wrapper = inputs[0].dynamicCast(); + auto input_tensor_desc = input_tensor_wrapper->getTensorDesc(); + + auto scale_tensor_wrapper = inputs[1].dynamicCast(); + auto scale_tensor_desc = scale_tensor_wrapper->getTensorDesc(); + + auto bias_tensor_wrapper = inputs[2].dynamicCast(); + auto bias_tensor_desc = bias_tensor_wrapper->getTensorDesc(); + + auto last_node = nodes[0].dynamicCast()->getOp(); + auto scale_node = nodes[1].dynamicCast()->getOp(); + auto bias_node = nodes[2].dynamicCast()->getOp(); + + auto op = std::make_shared(name); + + // set attrs + op->set_attr_epsilon(epsilon); + + // set inputs + // set inputs : x + op->set_input_x_by_name(*last_node, input_tensor_wrapper->name.c_str()); + op->update_input_desc_x(*input_tensor_desc); + // set inputs : gamma + op->set_input_gamma_by_name((*scale_node), scale_tensor_wrapper->name.c_str()); + op->update_input_desc_gamma(*scale_tensor_desc); + // set inputs : beta + op->set_input_beta_by_name(*bias_node, bias_tensor_wrapper->name.c_str()); + op->update_input_desc_beta(*bias_tensor_desc); + + // set outputs + auto output_desc_y = std::make_shared(ge::Shape(), ge::FORMAT_NCHW, ge::DT_FLOAT); + op->update_output_desc_y(*output_desc_y); + auto output_desc_mean = std::make_shared(ge::Shape(), ge::FORMAT_NCHW, ge::DT_FLOAT); + op->update_output_desc_mean(*output_desc_mean); + auto output_desc_var = std::make_shared(ge::Shape(), ge::FORMAT_NCHW, ge::DT_FLOAT); + op->update_output_desc_variance(*output_desc_var); + + return Ptr(new CannBackendNode(op)); + } +#endif // HAVE_CANN + #ifdef HAVE_DNN_NGRAPH virtual Ptr initNgraph(const std::vector >& inputs, const std::vector >& nodes) CV_OVERRIDE { diff --git a/modules/dnn/src/layers/layer_norm.cpp b/modules/dnn/src/layers/layer_norm.cpp index 9c16d19e41..0683bdb8c8 100644 --- a/modules/dnn/src/layers/layer_norm.cpp +++ b/modules/dnn/src/layers/layer_norm.cpp @@ -6,6 +6,9 @@ #include "layers_common.hpp" #include "cpu_kernels/fast_norm.hpp" +// CANN backend +#include "../op_cann.hpp" + namespace cv { namespace dnn { class LayerNormLayerImpl CV_FINAL : public LayerNormLayer @@ -22,7 +25,8 @@ public: virtual bool supportBackend(int backendId) CV_OVERRIDE { - return backendId == DNN_BACKEND_OPENCV; + return backendId == DNN_BACKEND_OPENCV || + (backendId == DNN_BACKEND_CANN && axis != -1); // axis=-1 not supported due to 1d mat shape problem } virtual bool getMemoryShapes(const std::vector &inputs, @@ -90,6 +94,59 @@ public: fastNorm(input, scale, output, epsilon, static_cast(axis)); } } + +#ifdef HAVE_CANN + virtual Ptr initCann(const std::vector > &inputs, + const std::vector > &outputs, + const std::vector >& nodes) CV_OVERRIDE { + CV_CheckEQ(inputs.size(), static_cast(3), "LayerNorm/CANN: requires three input wrappers"); + CV_CheckEQ(nodes.size(), static_cast(3), "LayerNorm/CANN: requires three input nodes"); + + auto input_tensor_wrapper = inputs[0].dynamicCast(); + auto input_tensor_desc = input_tensor_wrapper->getTensorDesc(); + + CV_CheckNE(axis, static_cast(input_tensor_desc->GetShape().GetDimNum() - 1), "LayerNorm: CANN does not support axis set as last axis due to 1D mat compatibility issue"); + + auto scale_tensor_wrapper = inputs[1].dynamicCast(); + auto scale_tensor_desc = scale_tensor_wrapper->getTensorDesc(); + + auto bias_tensor_wrapper = inputs[2].dynamicCast(); + auto bias_tensor_desc = bias_tensor_wrapper->getTensorDesc(); + + auto last_node = nodes[0].dynamicCast()->getOp(); + auto scale_node = nodes[1].dynamicCast()->getOp(); + auto bias_node = nodes[2].dynamicCast()->getOp(); + + auto op = std::make_shared(name); + + // set attrs + op->set_attr_begin_norm_axis(axis); + op->set_attr_begin_params_axis(axis); + op->set_attr_epsilon(epsilon); + + // set inputs + // set inputs : x + op->set_input_x_by_name(*last_node, input_tensor_wrapper->name.c_str()); + op->update_input_desc_x(*input_tensor_desc); + // set inputs : gamma + op->set_input_gamma_by_name(*scale_node, scale_tensor_wrapper->name.c_str()); + op->update_input_desc_gamma(*scale_tensor_desc); + // set inputs : beta + op->set_input_beta_by_name(*bias_node, bias_tensor_wrapper->name.c_str()); + op->update_input_desc_beta(*bias_tensor_desc); + + // set outputs + auto output_desc_y = std::make_shared(ge::Shape(), ge::FORMAT_NCHW, ge::DT_FLOAT); + op->update_output_desc_y(*output_desc_y); + auto output_desc_mean = std::make_shared(ge::Shape(), ge::FORMAT_NCHW, ge::DT_FLOAT); + op->update_output_desc_mean(*output_desc_mean); + auto output_desc_var = std::make_shared(ge::Shape(), ge::FORMAT_NCHW, ge::DT_FLOAT); + op->update_output_desc_variance(*output_desc_var); + + return Ptr(new CannBackendNode(op)); + } +#endif // HAVE_CANN + }; Ptr LayerNormLayer::create(const LayerParams& params) From 0e151e3c8837441c961d7f4788c733d634cd4964 Mon Sep 17 00:00:00 2001 From: Anatoliy Talamanov Date: Thu, 16 Nov 2023 05:49:53 +0000 Subject: [PATCH 031/156] Merge pull request #24060 from TolyaTalamanov:at/advanced-device-selection-onnxrt-directml G-API: Advanced device selection for ONNX DirectML Execution Provider #24060 ### Overview Extend `cv::gapi::onnx::ep::DirectML` to accept `adapter name` as `ctor` parameter in order to select execution device by `name`. E.g: ``` pp.cfgAddExecutionProvider(cv::gapi::onnx::ep::DirectML("Intel Graphics")); ``` ### Pull Request Readiness Checklist See details at https://github.com/opencv/opencv/wiki/How_to_contribute#making-a-good-pull-request - [ ] I agree to contribute to the project under Apache 2 License. - [ ] To the best of my knowledge, the proposed patch is not based on a code under GPL or another license that is incompatible with OpenCV - [ ] The PR is proposed to the proper branch - [ ] There is a reference to the original bug report and related work - [ ] There is accuracy test, performance test and test data in opencv_extra repository, if applicable Patch to opencv_extra has the same branch name. - [ ] The feature is well documented and sample code can be built with the project CMake --- CMakeLists.txt | 7 + cmake/OpenCVDetectDirectML.cmake | 13 + cmake/checks/directml.cpp | 38 +++ modules/gapi/CMakeLists.txt | 4 + .../gapi/include/opencv2/gapi/infer/onnx.hpp | 11 +- modules/gapi/src/backends/onnx/dml_ep.cpp | 243 +++++++++++++++++- 6 files changed, 312 insertions(+), 4 deletions(-) create mode 100644 cmake/OpenCVDetectDirectML.cmake create mode 100644 cmake/checks/directml.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 49554c15c6..49c93d2406 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -411,6 +411,9 @@ OCV_OPTION(WITH_OPENCLAMDBLAS "Include AMD OpenCL BLAS library support" ON OCV_OPTION(WITH_DIRECTX "Include DirectX support" ON VISIBLE_IF WIN32 AND NOT WINRT VERIFY HAVE_DIRECTX) +OCV_OPTION(WITH_DIRECTML "Include DirectML support" ON + VISIBLE_IF WIN32 AND NOT WINRT + VERIFY HAVE_DIRECTML) OCV_OPTION(WITH_OPENCL_D3D11_NV "Include NVIDIA OpenCL D3D11 support" WITH_DIRECTX VISIBLE_IF WIN32 AND NOT WINRT VERIFY HAVE_OPENCL_D3D11_NV) @@ -848,6 +851,10 @@ endif() if(WITH_DIRECTX) include(cmake/OpenCVDetectDirectX.cmake) endif() +# --- DirectML --- +if(WITH_DIRECTML) + include(cmake/OpenCVDetectDirectML.cmake) +endif() if(WITH_VTK) include(cmake/OpenCVDetectVTK.cmake) diff --git a/cmake/OpenCVDetectDirectML.cmake b/cmake/OpenCVDetectDirectML.cmake new file mode 100644 index 0000000000..0fc71eca03 --- /dev/null +++ b/cmake/OpenCVDetectDirectML.cmake @@ -0,0 +1,13 @@ +if(WIN32) + try_compile(__VALID_DIRECTML + "${OpenCV_BINARY_DIR}" + "${OpenCV_SOURCE_DIR}/cmake/checks/directml.cpp" + LINK_LIBRARIES d3d12 dxcore directml + OUTPUT_VARIABLE TRY_OUT + ) + if(NOT __VALID_DIRECTML) + message(STATUS "No support for DirectML (d3d12, dxcore, directml libs are required)") + return() + endif() + set(HAVE_DIRECTML ON) +endif() diff --git a/cmake/checks/directml.cpp b/cmake/checks/directml.cpp new file mode 100644 index 0000000000..1cf62b8fad --- /dev/null +++ b/cmake/checks/directml.cpp @@ -0,0 +1,38 @@ +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +int main(int /*argc*/, char** /*argv*/) +{ + IDXCoreAdapterFactory* factory; + DXCoreCreateAdapterFactory(__uuidof(IDXCoreAdapterFactory), (void**)&factory); + + IDXCoreAdapterList* adapterList; + const GUID dxGUIDs[] = { DXCORE_ADAPTER_ATTRIBUTE_D3D12_CORE_COMPUTE }; + factory->CreateAdapterList(ARRAYSIZE(dxGUIDs), dxGUIDs, __uuidof(IDXCoreAdapterList), (void**)&adapterList); + + IDXCoreAdapter* adapter; + adapterList->GetAdapter(0u, __uuidof(IDXCoreAdapter), (void**)&adapter); + + D3D_FEATURE_LEVEL d3dFeatureLevel = D3D_FEATURE_LEVEL_1_0_CORE; + ID3D12Device* d3d12Device = NULL; + D3D12CreateDevice((IUnknown*)adapter, d3dFeatureLevel, __uuidof(ID3D11Device), (void**)&d3d12Device); + + D3D12_COMMAND_LIST_TYPE commandQueueType = D3D12_COMMAND_LIST_TYPE_COMPUTE; + ID3D12CommandQueue* cmdQueue; + D3D12_COMMAND_QUEUE_DESC commandQueueDesc = {}; + commandQueueDesc.Type = commandQueueType; + + d3d12Device->CreateCommandQueue(&commandQueueDesc, __uuidof(ID3D12CommandQueue), (void**)&cmdQueue); + IDMLDevice* dmlDevice; + DMLCreateDevice(d3d12Device, DML_CREATE_DEVICE_FLAG_NONE, IID_PPV_ARGS(&dmlDevice)); + + return 0; +} \ No newline at end of file diff --git a/modules/gapi/CMakeLists.txt b/modules/gapi/CMakeLists.txt index 2caeb02ae2..a8714a99f5 100644 --- a/modules/gapi/CMakeLists.txt +++ b/modules/gapi/CMakeLists.txt @@ -367,6 +367,10 @@ if(WIN32) ocv_target_link_libraries(${the_module} PRIVATE wsock32 ws2_32) endif() +if(HAVE_DIRECTML) + ocv_target_compile_definitions(${the_module} PRIVATE HAVE_DIRECTML=1) +endif() + if(HAVE_ONNX) ocv_target_link_libraries(${the_module} PRIVATE ${ONNX_LIBRARY}) ocv_target_compile_definitions(${the_module} PRIVATE HAVE_ONNX=1) diff --git a/modules/gapi/include/opencv2/gapi/infer/onnx.hpp b/modules/gapi/include/opencv2/gapi/infer/onnx.hpp index 4efb750439..ae160ac3e5 100644 --- a/modules/gapi/include/opencv2/gapi/infer/onnx.hpp +++ b/modules/gapi/include/opencv2/gapi/infer/onnx.hpp @@ -189,7 +189,16 @@ public: GAPI_WRAP explicit DirectML(const int device_id) : ddesc(device_id) { }; - using DeviceDesc = cv::util::variant; + /** @brief Class constructor. + + Constructs DirectML parameters based on adapter name. + + @param adapter_name Target adapter_name to use. + */ + GAPI_WRAP + explicit DirectML(const std::string &adapter_name) : ddesc(adapter_name) { }; + + using DeviceDesc = cv::util::variant; DeviceDesc ddesc; }; diff --git a/modules/gapi/src/backends/onnx/dml_ep.cpp b/modules/gapi/src/backends/onnx/dml_ep.cpp index 7f59e1f3d6..671fa2dbcb 100644 --- a/modules/gapi/src/backends/onnx/dml_ep.cpp +++ b/modules/gapi/src/backends/onnx/dml_ep.cpp @@ -13,13 +13,240 @@ #ifdef HAVE_ONNX_DML #include "../providers/dml/dml_provider_factory.h" +#ifdef HAVE_DIRECTML + +#undef WINVER +#define WINVER 0x0A00 +#undef _WIN32_WINNT +#define _WIN32_WINNT 0x0A00 + +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#pragma comment (lib, "d3d11.lib") +#pragma comment (lib, "d3d12.lib") +#pragma comment (lib, "dxgi.lib") +#pragma comment (lib, "dxcore.lib") +#pragma comment (lib, "directml.lib") + +#endif // HAVE_DIRECTML + +static void addDMLExecutionProviderWithAdapterName(Ort::SessionOptions *session_options, + const std::string &adapter_name); + void cv::gimpl::onnx::addDMLExecutionProvider(Ort::SessionOptions *session_options, const cv::gapi::onnx::ep::DirectML &dml_ep) { namespace ep = cv::gapi::onnx::ep; - GAPI_Assert(cv::util::holds_alternative(dml_ep.ddesc)); - const int device_id = cv::util::get(dml_ep.ddesc); + switch (dml_ep.ddesc.index()) { + case ep::DirectML::DeviceDesc::index_of(): { + const int device_id = cv::util::get(dml_ep.ddesc); + try { + OrtSessionOptionsAppendExecutionProvider_DML(*session_options, device_id); + } catch (const std::exception &e) { + std::stringstream ss; + ss << "ONNX Backend: Failed to enable DirectML" + << " Execution Provider: " << e.what(); + cv::util::throw_error(std::runtime_error(ss.str())); + } + break; + } + case ep::DirectML::DeviceDesc::index_of(): { + const std::string adapter_name = cv::util::get(dml_ep.ddesc); + addDMLExecutionProviderWithAdapterName(session_options, adapter_name); + break; + } + default: + GAPI_Assert(false && "Invalid DirectML device description"); + } +} + +#ifdef HAVE_DIRECTML + +#define THROW_IF_FAILED(hr, error_msg) \ +{ \ + if ((hr) != S_OK) \ + throw std::runtime_error(error_msg); \ +} + +template +void release(T *ptr) { + if (ptr) { + ptr->Release(); + } +} + +template +using ComPtrGuard = std::unique_ptr)>; + +template +ComPtrGuard make_com_ptr(T *ptr) { + return ComPtrGuard{ptr, &release}; +} + +struct AdapterDesc { + ComPtrGuard ptr; + std::string description; +}; + +static std::vector getAvailableAdapters() { + std::vector all_adapters; + + IDXCoreAdapterFactory* factory_ptr; + GAPI_LOG_DEBUG(nullptr, "Create IDXCoreAdapterFactory"); + THROW_IF_FAILED( + DXCoreCreateAdapterFactory( + __uuidof(IDXCoreAdapterFactory), (void**)&factory_ptr), + "Failed to create IDXCoreAdapterFactory"); + auto factory = make_com_ptr(factory_ptr); + + IDXCoreAdapterList* adapter_list_ptr; + const GUID dxGUIDs[] = { DXCORE_ADAPTER_ATTRIBUTE_D3D12_CORE_COMPUTE }; + GAPI_LOG_DEBUG(nullptr, "CreateAdapterList"); + THROW_IF_FAILED( + factory->CreateAdapterList( + ARRAYSIZE(dxGUIDs), dxGUIDs, __uuidof(IDXCoreAdapterList), (void**)&adapter_list_ptr), + "Failed to create IDXCoreAdapterList"); + auto adapter_list = make_com_ptr(adapter_list_ptr); + + for (UINT i = 0; i < adapter_list->GetAdapterCount(); i++) + { + IDXCoreAdapter* curr_adapter_ptr; + GAPI_LOG_DEBUG(nullptr, "GetAdapter"); + THROW_IF_FAILED( + adapter_list->GetAdapter( + i, __uuidof(IDXCoreAdapter), (void**)&curr_adapter_ptr), + "Failed to obtain IDXCoreAdapter" + ); + auto curr_adapter = make_com_ptr(curr_adapter_ptr); + + bool is_hardware = false; + curr_adapter->GetProperty(DXCoreAdapterProperty::IsHardware, &is_hardware); + // NB: Filter out if not hardware adapter. + if (!is_hardware) { + continue; + } + + size_t desc_size = 0u; + char description[256]; + curr_adapter->GetPropertySize(DXCoreAdapterProperty::DriverDescription, &desc_size); + curr_adapter->GetProperty(DXCoreAdapterProperty::DriverDescription, desc_size, &description); + all_adapters.push_back(AdapterDesc{std::move(curr_adapter), description}); + } + return all_adapters; +}; + +struct DMLDeviceInfo { + ComPtrGuard device; + ComPtrGuard cmd_queue; +}; + +static DMLDeviceInfo createDMLInfo(IDXCoreAdapter* adapter) { + auto pAdapter = make_com_ptr(adapter); + D3D_FEATURE_LEVEL d3dFeatureLevel = D3D_FEATURE_LEVEL_1_0_CORE; + if (adapter->IsAttributeSupported(DXCORE_ADAPTER_ATTRIBUTE_D3D12_GRAPHICS)) + { + GAPI_LOG_INFO(nullptr, "DXCORE_ADAPTER_ATTRIBUTE_D3D12_GRAPHICS is supported"); + d3dFeatureLevel = D3D_FEATURE_LEVEL::D3D_FEATURE_LEVEL_11_0; + + IDXGIFactory4* dxgiFactory4; + GAPI_LOG_DEBUG(nullptr, "CreateDXGIFactory2"); + THROW_IF_FAILED( + CreateDXGIFactory2(0, __uuidof(IDXGIFactory4), (void**)&dxgiFactory4), + "Failed to create IDXGIFactory4" + ); + // If DXGI factory creation was successful then get the IDXGIAdapter from the LUID + // acquired from the selectedAdapter + LUID adapterLuid; + IDXGIAdapter* spDxgiAdapter; + + GAPI_LOG_DEBUG(nullptr, "Get DXCoreAdapterProperty::InstanceLuid property"); + THROW_IF_FAILED( + adapter->GetProperty(DXCoreAdapterProperty::InstanceLuid, &adapterLuid), + "Failed to get DXCoreAdapterProperty::InstanceLuid property"); + + GAPI_LOG_DEBUG(nullptr, "Get IDXGIAdapter by luid"); + THROW_IF_FAILED( + dxgiFactory4->EnumAdapterByLuid( + adapterLuid, __uuidof(IDXGIAdapter), (void**)&spDxgiAdapter), + "Failed to get IDXGIAdapter"); + pAdapter = make_com_ptr(spDxgiAdapter); + } else { + GAPI_LOG_INFO(nullptr, "DXCORE_ADAPTER_ATTRIBUTE_D3D12_GRAPHICS isn't supported"); + } + + ID3D12Device* d3d12_device_ptr; + GAPI_LOG_DEBUG(nullptr, "Create D3D12Device"); + THROW_IF_FAILED( + D3D12CreateDevice( + pAdapter.get(), d3dFeatureLevel, __uuidof(ID3D12Device), (void**)&d3d12_device_ptr), + "Failed to create ID3D12Device"); + auto d3d12_device = make_com_ptr(d3d12_device_ptr); + + D3D12_COMMAND_LIST_TYPE commandQueueType = D3D12_COMMAND_LIST_TYPE_COMPUTE; + ID3D12CommandQueue* cmd_queue_ptr; + D3D12_COMMAND_QUEUE_DESC commandQueueDesc = {}; + commandQueueDesc.Type = commandQueueType; + GAPI_LOG_DEBUG(nullptr, "Create D3D12CommandQueue"); + THROW_IF_FAILED( + d3d12_device->CreateCommandQueue( + &commandQueueDesc, __uuidof(ID3D12CommandQueue), (void**)&cmd_queue_ptr), + "Failed to create D3D12CommandQueue" + ); + GAPI_LOG_DEBUG(nullptr, "Create D3D12CommandQueue - successful"); + auto cmd_queue = make_com_ptr(cmd_queue_ptr); + + IDMLDevice* dml_device_ptr; + GAPI_LOG_DEBUG(nullptr, "Create DirectML device"); + THROW_IF_FAILED( + DMLCreateDevice( + d3d12_device.get(), DML_CREATE_DEVICE_FLAG_NONE, IID_PPV_ARGS(&dml_device_ptr)), + "Failed to create IDMLDevice"); + GAPI_LOG_DEBUG(nullptr, "Create DirectML device - successful"); + auto dml_device = make_com_ptr(dml_device_ptr); + + return {std::move(dml_device), std::move(cmd_queue)}; +}; + +static void addDMLExecutionProviderWithAdapterName(Ort::SessionOptions *session_options, + const std::string &adapter_name) { + auto all_adapters = getAvailableAdapters(); + + std::vector selected_adapters; + std::stringstream log_msg; + for (auto&& adapter : all_adapters) { + log_msg << adapter.description << std::endl; + if (std::strstr(adapter.description.c_str(), adapter_name.c_str())) { + selected_adapters.emplace_back(std::move(adapter)); + } + } + GAPI_LOG_INFO(NULL, "\nAvailable DirectML adapters:\n" << log_msg.str()); + + if (selected_adapters.empty()) { + std::stringstream error_msg; + error_msg << "ONNX Backend: No DirectML adapters found match to \"" << adapter_name << "\""; + cv::util::throw_error(std::runtime_error(error_msg.str())); + } else if (selected_adapters.size() > 1) { + std::stringstream error_msg; + error_msg << "ONNX Backend: More than one adapter matches to \"" << adapter_name << "\":\n"; + for (const auto &selected_adapter : selected_adapters) { + error_msg << selected_adapter.description << "\n"; + } + cv::util::throw_error(std::runtime_error(error_msg.str())); + } + + GAPI_LOG_INFO(NULL, "Selected device: " << selected_adapters.front().description); + auto dml = createDMLInfo(selected_adapters.front().ptr.get()); try { - OrtSessionOptionsAppendExecutionProvider_DML(*session_options, device_id); + OrtSessionOptionsAppendExecutionProviderEx_DML( + *session_options, dml.device.release(), dml.cmd_queue.release()); } catch (const std::exception &e) { std::stringstream ss; ss << "ONNX Backend: Failed to enable DirectML" @@ -28,6 +255,16 @@ void cv::gimpl::onnx::addDMLExecutionProvider(Ort::SessionOptions *session_optio } } +#else // HAVE_DIRECTML + +static void addDMLExecutionProviderWithAdapterName(Ort::SessionOptions*, const std::string&) { + std::stringstream ss; + ss << "ONNX Backend: Failed to add DirectML Execution Provider with adapter name." + << " DirectML support is required."; + cv::util::throw_error(std::runtime_error(ss.str())); +} + +#endif // HAVE_DIRECTML #else // HAVE_ONNX_DML void cv::gimpl::onnx::addDMLExecutionProvider(Ort::SessionOptions*, From 83d70b0f36904e8906dc3f446fc093dac9e6a590 Mon Sep 17 00:00:00 2001 From: Chia-Hsiang Tsai <84863554+Tsai-chia-hsiang@users.noreply.github.com> Date: Thu, 16 Nov 2023 18:40:00 +0800 Subject: [PATCH 032/156] Merge pull request #24396 from Tsai-chia-hsiang:yolov8cv Using cv2 dnn interface to run yolov8 model #24396 This is a sample code for using opencv dnn interface to run ultralytics yolov8 model for object detection. ### Pull Request Readiness Checklist See details at https://github.com/opencv/opencv/wiki/How_to_contribute#making-a-good-pull-request - [X] I agree to contribute to the project under Apache 2 License. - [X] To the best of my knowledge, the proposed patch is not based on a code under GPL or another license that is incompatible with OpenCV - [X] The PR is proposed to the proper branch - [] There is a reference to the original bug report and related work - [] There is accuracy test, performance test and test data in opencv_extra repository, if applicable Patch to opencv_extra has the same branch name. - [] The feature is well documented and sample code can be built with the project CMake --- samples/dnn/common.py | 4 ++++ samples/dnn/models.yml | 18 +++++++++++++++++ samples/dnn/object_detection.py | 36 ++++++++++++++++++++++----------- 3 files changed, 46 insertions(+), 12 deletions(-) diff --git a/samples/dnn/common.py b/samples/dnn/common.py index db9283b5d8..4765506eac 100644 --- a/samples/dnn/common.py +++ b/samples/dnn/common.py @@ -79,6 +79,10 @@ def add_preproc_args(zoo, parser, sample): help='Indicate that model works with RGB input images instead BGR ones.') add_argument(zoo, parser, 'classes', help='Optional path to a text file with names of classes to label detected objects.') + add_argument(zoo, parser, 'postprocessing', type=str, + help='Post-processing kind depends on model topology.') + add_argument(zoo, parser, 'background_label_id', type=int, default=-1, + help='An index of background class in predictions. If not negative, exclude such class from list of classes.') def findFile(filename): diff --git a/samples/dnn/models.yml b/samples/dnn/models.yml index 53d8b8048f..4d2774c71e 100644 --- a/samples/dnn/models.yml +++ b/samples/dnn/models.yml @@ -33,6 +33,7 @@ yolov4: height: 416 rgb: true classes: "object_detection_classes_yolo.txt" + background_label_id: 0 sample: "object_detection" yolov4-tiny: @@ -47,6 +48,7 @@ yolov4-tiny: height: 416 rgb: true classes: "object_detection_classes_yolo.txt" + background_label_id: 0 sample: "object_detection" yolov3: @@ -61,6 +63,7 @@ yolov3: height: 416 rgb: true classes: "object_detection_classes_yolo.txt" + background_label_id: 0 sample: "object_detection" tiny-yolo-voc: @@ -75,6 +78,21 @@ tiny-yolo-voc: height: 416 rgb: true classes: "object_detection_classes_pascal_voc.txt" + background_label_id: 0 + sample: "object_detection" + +yolov8: + load_info: + url: "https://github.com/CVHub520/X-AnyLabeling/releases/download/v0.1.0/yolov8n.onnx" + sha1: "68f864475d06e2ec4037181052739f268eeac38d" + model: "yolov8n.onnx" + mean: [0, 0, 0] + scale: 0.00392 + width: 640 + height: 640 + rgb: true + postprocessing: "yolov8" + classes: "object_detection_classes_yolo.txt" sample: "object_detection" # Caffe implementation of SSD model from https://github.com/chuanqi305/MobileNet-SSD diff --git a/samples/dnn/object_detection.py b/samples/dnn/object_detection.py index 0ca5586159..875ed3929f 100644 --- a/samples/dnn/object_detection.py +++ b/samples/dnn/object_detection.py @@ -2,6 +2,7 @@ import cv2 as cv import argparse import numpy as np import sys +import copy import time from threading import Thread if sys.version_info[0] == 2: @@ -27,7 +28,7 @@ parser.add_argument('--out_tf_graph', default='graph.pbtxt', help='For models from TensorFlow Object Detection API, you may ' 'pass a .config file which was used for training through --config ' 'argument. This way an additional .pbtxt file with TensorFlow graph will be created.') -parser.add_argument('--framework', choices=['caffe', 'tensorflow', 'torch', 'darknet', 'dldt'], +parser.add_argument('--framework', choices=['caffe', 'tensorflow', 'torch', 'darknet', 'dldt', 'onnx'], help='Optional name of an origin framework of the model. ' 'Detect it automatically if it does not set.') parser.add_argument('--thr', type=float, default=0.5, help='Confidence threshold') @@ -86,7 +87,7 @@ if args.classes: classes = f.read().rstrip('\n').split('\n') # Load a network -net = cv.dnn.readNet(cv.samples.findFile(args.model), cv.samples.findFile(args.config), args.framework) +net = cv.dnn.readNet(args.model, args.config, args.framework) net.setPreferableBackend(args.backend) net.setPreferableTarget(args.target) outNames = net.getUnconnectedOutLayersNames() @@ -145,20 +146,32 @@ def postprocess(frame, outs): classIds.append(int(detection[1]) - 1) # Skip background label confidences.append(float(confidence)) boxes.append([left, top, width, height]) - elif lastLayer.type == 'Region': + elif lastLayer.type == 'Region' or args.postprocessing == 'yolov8': # Network produces output blob with a shape NxC where N is a number of # detected objects and C is a number of classes + 4 where the first 4 # numbers are [center_x, center_y, width, height] + if args.postprocessing == 'yolov8': + box_scale_w = frameWidth / args.width + box_scale_h = frameHeight / args.height + else: + box_scale_w = frameWidth + box_scale_h = frameHeight + for out in outs: + if args.postprocessing == 'yolov8': + out = out[0].transpose(1, 0) + for detection in out: - scores = detection[5:] + scores = detection[4:] + if args.background_label_id >= 0: + scores = np.delete(scores, args.background_label_id) classId = np.argmax(scores) confidence = scores[classId] if confidence > confThreshold: - center_x = int(detection[0] * frameWidth) - center_y = int(detection[1] * frameHeight) - width = int(detection[2] * frameWidth) - height = int(detection[3] * frameHeight) + center_x = int(detection[0] * box_scale_w) + center_y = int(detection[1] * box_scale_h) + width = int(detection[2] * box_scale_w) + height = int(detection[3] * box_scale_h) left = int(center_x - width / 2) top = int(center_y - height / 2) classIds.append(classId) @@ -170,7 +183,7 @@ def postprocess(frame, outs): # NMS is used inside Region layer only on DNN_BACKEND_OPENCV for another backends we need NMS in sample # or NMS is required if number of outputs > 1 - if len(outNames) > 1 or lastLayer.type == 'Region' and args.backend != cv.dnn.DNN_BACKEND_OPENCV: + if len(outNames) > 1 or (lastLayer.type == 'Region' or args.postprocessing == 'yolov8') and args.backend != cv.dnn.DNN_BACKEND_OPENCV: indices = [] classIds = np.array(classIds) boxes = np.array(boxes) @@ -181,7 +194,6 @@ def postprocess(frame, outs): conf = confidences[class_indices] box = boxes[class_indices].tolist() nms_indices = cv.dnn.NMSBoxes(box, conf, confThreshold, nmsThreshold) - nms_indices = nms_indices[:, 0] if len(nms_indices) else [] indices.extend(class_indices[nms_indices]) else: indices = np.arange(0, len(classIds)) @@ -282,11 +294,11 @@ def processingThreadBody(): futureOutputs.append(net.forwardAsync()) else: outs = net.forward(outNames) - predictionsQueue.put(np.copy(outs)) + predictionsQueue.put(copy.deepcopy(outs)) while futureOutputs and futureOutputs[0].wait_for(0): out = futureOutputs[0].get() - predictionsQueue.put(np.copy([out])) + predictionsQueue.put(copy.deepcopy([out])) del futureOutputs[0] From 8c10545d3cea7e67f1788fdaa47dd92019e2bf81 Mon Sep 17 00:00:00 2001 From: Abduragim Shtanchaev <44877829+Abdurrahheem@users.noreply.github.com> Date: Thu, 16 Nov 2023 17:20:17 +0400 Subject: [PATCH 033/156] Merge pull request #24509 from Abdurrahheem:ash/dev_einsum_fast_gemm Fast gemm for einsum #24509 ## This PR adds performance tests for Einsum Layer with FastGemm. See below results of performance test on different inputs ### Pull Request Readiness Checklist See details at https://github.com/opencv/opencv/wiki/How_to_contribute#making-a-good-pull-request - [x] I agree to contribute to the project under Apache 2 License. - [x] To the best of my knowledge, the proposed patch is not based on a code under GPL or another license that is incompatible with OpenCV - [x] The PR is proposed to the proper branch - [ ] There is a reference to the original bug report and related work - [x] There is accuracy test, performance test and test data in opencv_extra repository, if applicable Patch to opencv_extra has the same branch name. - [x] The feature is well documented and sample code can be built with the project CMake --- modules/dnn/perf/perf_einsum.cpp | 64 +++---- modules/dnn/src/layers/einsum_layer.cpp | 222 +++++++++++++----------- 2 files changed, 141 insertions(+), 145 deletions(-) diff --git a/modules/dnn/perf/perf_einsum.cpp b/modules/dnn/perf/perf_einsum.cpp index c3706d3153..bad9d956be 100644 --- a/modules/dnn/perf/perf_einsum.cpp +++ b/modules/dnn/perf/perf_einsum.cpp @@ -11,19 +11,16 @@ struct EinsumParams { int outputSize; std::string equation; std::vector einsumInpShapes; - EinsumParams(std::string equation_, int inputSize_, int outputSize_, std::vector einsumInpShapes_ = std::vector()) + EinsumParams(std::string equation_, std::vector einsumInpShapes_ = std::vector()) { - inputSize = inputSize_; - outputSize = outputSize_; + inputSize = einsumInpShapes_.size(); equation = equation_; einsumInpShapes = einsumInpShapes_; } }; static inline void PrintTo(const EinsumParams& params, ::std::ostream* os) { - (*os) << "Eqiation=" << params.equation << ", " - << "InputSize=" << params.inputSize << ", " - << "OutputSize=" << params.outputSize << ", "; + (*os) << "Equation=" << params.equation << " "; (*os) << "InputShape={"; for(int i = 0; i < params.einsumInpShapes.size(); i++) @@ -41,22 +38,22 @@ static inline void PrintTo(const EinsumParams& params, ::std::ostream* os) { // test cases static const EinsumParams testEinsumConfigs[] = { // TODO: Add tests with one input after ellips merge - {"ij, jk -> ik", 2, 1, {{2, 3}, {3, 2}}}, - {"ij, jk -> ik", 2, 1, {{20, 30}, {30, 20}}}, - {"ij, jk -> ik", 2, 1, {{113, 127}, {127, 113}}}, + {"ij, jk -> ik", {{2, 3}, {3, 2}}}, + {"ij, jk -> ik", {{20, 30}, {30, 20}}}, + {"ij, jk -> ik", {{113, 127}, {127, 113}}}, - {"imkj, injs -> imnks", 2, 1, {{1, 4, 7, 9}, {1, 5, 9, 8}}}, - {"imkj, injs -> imnks", 2, 1, {{1, 4, 70, 90}, {1, 5, 90, 80}}}, - {"imkj, injs -> imnks", 2, 1, {{1, 4, 73, 91}, {1, 5, 91, 57}}}, + {"imkj, injs -> imnks", {{1, 4, 7, 9}, {1, 5, 9, 8}}}, + {"imkj, injs -> imnks", {{1, 4, 70, 90}, {1, 5, 90, 80}}}, + {"imkj, injs -> imnks", {{1, 4, 73, 91}, {1, 5, 91, 57}}}, - {"ij -> i", 1, 1, {{30, 40}}}, - {"ij -> i", 1, 1, {{113, 374}}}, + {"ij -> i", {{30, 40}}}, + {"ij -> i", {{113, 374}}}, - {"...ij -> ...i", 1, 1, {{30, 40}}}, - {"...ij -> ...i", 1, 1, {{113, 374}}}, + {"...ij -> ...i", {{30, 40}}}, + {"...ij -> ...i", {{113, 374}}}, - {"...ij, ...jk -> ...ik", 2, 1, {{40, 50}, {50, 80}}}, - {"...ij, ...jk -> ...ik", 2, 1, {{47, 51}, {51, 83}}}, + {"...ij, ...jk -> ...ik", {{40, 50}, {50, 80}}}, + {"...ij, ...jk -> ...ik", {{47, 51}, {51, 83}}}, }; class Layer_Einsum: public TestBaseWithParam {}; @@ -68,7 +65,7 @@ PERF_TEST_P_(Layer_Einsum, einsum) { lp.name = "testEinsum"; lp.set("equation", params.equation); lp.set("inputSize", params.inputSize); - lp.set("outputSize", params.outputSize); + lp.set("outputSize", 1); CV_CheckFalse(params.einsumInpShapes.empty(), "ERROR no inputs shapes provided"); @@ -79,38 +76,27 @@ PERF_TEST_P_(Layer_Einsum, einsum) { Net net; std::vector inputs; std::vector input_names; - if (params.inputSize == 1){ + int id = net.addLayer(lp.name, lp.type, lp); + for (int i = 0; i < params.inputSize; ++i) { // create inputs - inputs.emplace_back(Mat(params.einsumInpShapes[0].size(), params.einsumInpShapes[0].data(), CV_32FC1)); + inputs.emplace_back(Mat(params.einsumInpShapes[i].size(), params.einsumInpShapes[i].data(), CV_32FC1)); - int id = net.addLayerToPrev(lp.name, lp.type, lp); - net.connect(0, 0, id, 0); + // connect each input to the layer + net.connect(0, i, id, i); - input_names.emplace_back("input1"); - - } else { - - // create inputs - inputs.emplace_back(Mat(params.einsumInpShapes[0].size(), params.einsumInpShapes[0].data(), CV_32FC1)); - inputs.emplace_back(Mat(params.einsumInpShapes[1].size(), params.einsumInpShapes[1].data(), CV_32FC1)); - - int id = net.addLayerToPrev(lp.name, lp.type, lp); - net.connect(0, 0, id, 0); - net.connect(0, 1, id, 1); - - input_names.emplace_back("input1"); - input_names.emplace_back("input2"); + // create input names dynamically, assuming input naming follows a consistent pattern + input_names.emplace_back("input" + std::to_string(i + 1)); } //warm up + std::vector outputs; net.setInputsNames(input_names); for (int i = 0; i < input_names.size(); i++){ net.setInput(inputs[i], input_names[i]); } - Mat out = net.forward(); + net.forward(outputs, "testEinsum"); - std::vector outputs; TEST_CYCLE() { net.forward(outputs, "testEinsum"); diff --git a/modules/dnn/src/layers/einsum_layer.cpp b/modules/dnn/src/layers/einsum_layer.cpp index baf4297c0e..c7f9aaca06 100644 --- a/modules/dnn/src/layers/einsum_layer.cpp +++ b/modules/dnn/src/layers/einsum_layer.cpp @@ -6,6 +6,7 @@ #include #include "../precomp.hpp" #include "layers_common.hpp" +#include "cpu_kernels/fast_gemm.hpp" namespace cv { @@ -32,111 +33,6 @@ static bool IsTransposeReshapeForEinsum(const std::vector& perm, return true; } -static Mat batchwiseMatMul( - const Mat& input1, - const MatShape& input1ShapeOverride, - const Mat& input2, - const MatShape& input2ShapeOverride) -{ - // Sanity checks before the actual MatMul - CV_CheckType(input1.type(), input2.type(), "Data types of the inputs must match for MatMul"); - CV_CheckEQ(input1ShapeOverride.size(), (size_t) 3, "Only 1 batch dimension is allowed for MatMul"); - CV_CheckEQ(input2ShapeOverride.size(), (size_t) 3, "Only 1 batch dimension is allowed for MatMul"); - CV_CheckEQ((size_t) input1ShapeOverride[0], (size_t) input2ShapeOverride[0], "Batch dimension should match for MatMul;"); - CV_CheckEQ((size_t) input1ShapeOverride[2], (size_t) input2ShapeOverride[1], "Incompatible matrix dimensions for matMul"); - - size_t batches = input1ShapeOverride[0]; - size_t M = input1ShapeOverride[1]; - size_t K = input1ShapeOverride[2]; - size_t N = input2ShapeOverride[2]; - - std::vector output; - if (batches > 1) - { - Mat reshapedInput1 = input1; - Mat reshapedInput2 = input2; - - // input1 should of size MxK - // check if input1 needs reshape, if need reshape - if (input1.size[0] != M || input1.size[1] != K) - { - int shape[] = {static_cast(batches), static_cast(M), static_cast(K)}; - reshapedInput1 = input1.reshape(1, 3, shape); - } - - // input2 should be of size KxN - // check if input2 needs reshape, if needs reshape - if (input2.size[0] != K || input2.size[1] != N) - { - int shape[] = {static_cast(batches), static_cast(K), static_cast(N)}; - reshapedInput2 = input2.reshape(1, 3, shape); - } - - for (size_t i=0; i < batches; i++) - { - std::vector ranges1 = {cv::Range(i, i+1)}; - for (int j = 1; j < reshapedInput1.dims; j++) - ranges1.emplace_back(cv::Range::all()); - - Mat part1 = reshapedInput1(ranges1); - int shape[] = {static_cast(M), static_cast(K)}; - part1 = part1.reshape(1, sizeof(shape)/sizeof(shape[0]), shape); - - std::vector ranges2 = {cv::Range(i, i+1)}; - for (int j = 1; j < reshapedInput2.dims; j++) - ranges2.emplace_back(cv::Range::all()); - - Mat part2 = reshapedInput2(ranges2); - int shape2[] = {static_cast(K), static_cast(N)}; - part2 = part2.reshape(1, sizeof(shape2)/sizeof(shape2[0]), shape2); - - Mat tmp_output; - cv::gemm(part1, part2, 1.0, cv::Mat(), 1.0, tmp_output); - int newShape[] = {1, static_cast(M), static_cast(N)}; - tmp_output = tmp_output.reshape(1, sizeof(newShape)/sizeof(newShape[0]), newShape); - - output.emplace_back(tmp_output); - } - - } else { - - Mat reshapedInput1 = input1; - Mat reshapedInput2 = input2; - - // input1 should of size MxK - // check if input1 needs reshape, if need reshape - if (input1.dims > 2 || input1.size[0] != M || input1.size[1] != K) - { - int shape[] = {static_cast(M), static_cast(K)}; - reshapedInput1 = input1.reshape(1, 2, shape); - } - - // input2 should be of size KxN - // check if input2 needs reshape, if needs reshape - if (input2.dims > 2 || input2.size[0] != K || input2.size[1] != N) - { - int shape2[] = {static_cast(K), static_cast(N)}; - reshapedInput2 = input2.reshape(1, 2, shape2); - } - - Mat tmp_output; - cv::gemm(reshapedInput1, reshapedInput2, 1.0, cv::Mat(), 1.0, tmp_output); - - int newShape[] = {1, static_cast(M), static_cast(N)}; - tmp_output = tmp_output.reshape(1, sizeof(newShape)/sizeof(newShape[0]), newShape); - output.emplace_back(tmp_output); - - } - - int outputDim[] = {static_cast(output.size()), static_cast(M), static_cast(N)}; - Mat output_buffer = Mat::zeros(3, outputDim, CV_32F); - - for (size_t i = 0; i < output.size(); i++) { - Mat output_slice = output_buffer.row(i); - output[i].copyTo(output_slice); - } - return output_buffer; -}; static Mat Transpose( const Mat& input, @@ -452,6 +348,8 @@ public: // The number of dimensions that are encompassed by an "ellipsis" - "...". size_t numOfEllipsisDims = 0; + // Backend for fastgemm + FastGemmOpt opt; void parseEquation(String equation); void processEquation(const std::vector& inputs); @@ -469,7 +367,12 @@ public: const MatShape& reduceDims, bool isFinalPair ); - + Mat batchwiseMatMul( + const Mat& input1, + const MatShape& input1ShapeOverride, + const Mat& input2, + const MatShape& input2ShapeOverride + ); // constructor LayerEinsumImpl(const LayerParams& params) @@ -491,6 +394,7 @@ public: einsumInpShapes.emplace_back(shape); } + opt.init(); // Maintains a mapping between input indices and their corresponding subscript labels for each input inputSubscriptIndices.reserve(numInputs); @@ -1389,6 +1293,112 @@ Mat LayerEinsumImpl::pairwiseOperandProcess( return output; }; +Mat LayerEinsumImpl::batchwiseMatMul( + const Mat& input1, + const MatShape& input1ShapeOverride, + const Mat& input2, + const MatShape& input2ShapeOverride) +{ + + // Sanity checks before the actual MatMul + CV_CheckType(input1.type(), input2.type(), "Data types of the inputs must match for MatMul"); + CV_CheckEQ(input1ShapeOverride.size(), (size_t) 3, "Only 1 batch dimension is allowed for MatMul"); + CV_CheckEQ(input2ShapeOverride.size(), (size_t) 3, "Only 1 batch dimension is allowed for MatMul"); + CV_CheckEQ((size_t) input1ShapeOverride[0], (size_t) input2ShapeOverride[0], "Batch dimension should match for MatMul;"); + CV_CheckEQ((size_t) input1ShapeOverride[2], (size_t) input2ShapeOverride[1], "Incompatible matrix dimensions for matMul"); + + int batches = input1ShapeOverride[0]; + int M = input1ShapeOverride[1]; + int K = input1ShapeOverride[2]; + int N = input2ShapeOverride[2]; + + std::vector output; + if (batches > 1) + { + Mat reshapedInput1 = input1; + Mat reshapedInput2 = input2; + + // input1 should of size MxK + // check if input1 needs reshape, if need reshape + if (input1.size[0] != M || input1.size[1] != K) + { + int shape[] = {batches, M, K}; + reshapedInput1 = input1.reshape(1, 3, shape); + } + + // input2 should be of size KxN + // check if input2 needs reshape, if needs reshape + if (input2.size[0] != K || input2.size[1] != N) + { + int shape[] = {batches, K, N}; + reshapedInput2 = input2.reshape(1, 3, shape); + } + + for (size_t i=0; i < batches; i++) + { + std::vector ranges1 = {cv::Range(i, i+1)}; + for (int j = 1; j < reshapedInput1.dims; j++) + ranges1.emplace_back(cv::Range::all()); + + Mat part1 = reshapedInput1(ranges1); + int shape[] = {M, K}; + part1 = part1.reshape(1, sizeof(shape)/sizeof(shape[0]), shape); + + std::vector ranges2 = {cv::Range(i, i+1)}; + for (int j = 1; j < reshapedInput2.dims; j++) + ranges2.emplace_back(cv::Range::all()); + + Mat part2 = reshapedInput2(ranges2); + int shape2[] = {K, N}; + part2 = part2.reshape(1, sizeof(shape2)/sizeof(shape2[0]), shape2); + + Mat tmp_output(M, N, part1.type()); + fastGemm(false, false, 1.0, part1, part2, 0.0, tmp_output, opt); + int newShape[] = {1, M, N}; + tmp_output = tmp_output.reshape(1, sizeof(newShape)/sizeof(newShape[0]), newShape); + + output.emplace_back(tmp_output); + } + + } else { + + Mat reshapedInput1 = input1; + Mat reshapedInput2 = input2; + + // input1 should of size MxK + // check if input1 needs reshape, if need reshape + if (input1.dims > 2 || input1.size[0] != M || input1.size[1] != K) + { + int shape[] = {M, K}; + reshapedInput1 = input1.reshape(1, 2, shape); + } + + // input2 should be of size KxN + // check if input2 needs reshape, if needs reshape + if (input2.dims > 2 || input2.size[0] != K || input2.size[1] != N) + { + int shape2[] = {K, N}; + reshapedInput2 = input2.reshape(1, 2, shape2); + } + + Mat tmp_output(M, N, reshapedInput1.type()); + fastGemm(false, false, 1.0, reshapedInput1, reshapedInput2, 0.0, tmp_output, opt); + + int newShape[] = {1, M, N}; + tmp_output = tmp_output.reshape(1, sizeof(newShape)/sizeof(newShape[0]), newShape); + output.emplace_back(tmp_output); + + } + + int outputDim[] = {static_cast(output.size()), M, N}; + Mat output_buffer = Mat::zeros(3, outputDim, CV_32F); + + for (size_t i = 0; i < output.size(); i++) { + Mat output_slice = output_buffer.row(i); + output[i].copyTo(output_slice); + } + return output_buffer; +}; Ptr EinsumLayer::create(const LayerParams& params) { return makePtr(params); From fad0dbb9acc7d14b0a85bcadf305aa2a2ee21c81 Mon Sep 17 00:00:00 2001 From: Jeremy Lyda Date: Thu, 16 Nov 2023 08:28:28 -0500 Subject: [PATCH 034/156] Merge pull request #24364 from bagelbytes61:bugfix/qrcode-version-estimator Bugfix/qrcode version estimator #24364 Fixes https://github.com/opencv/opencv/issues/24366 ### Pull Request Readiness Checklist See details at https://github.com/opencv/opencv/wiki/How_to_contribute#making-a-good-pull-request - [x] I agree to contribute to the project under Apache 2 License. - [x] To the best of my knowledge, the proposed patch is not based on a code under GPL or another license that is incompatible with OpenCV - [x] The PR is proposed to the proper branch - [x] There is a reference to the original bug report and related work - [x] There is accuracy test, performance test and test data in opencv_extra repository, if applicable Patch to opencv_extra has the same branch name. - [x] The feature is well documented and sample code can be built with the project CMake --- modules/objdetect/src/qrcode_encoder.cpp | 175 ++++++++++++------ modules/objdetect/test/test_qrcode_encode.cpp | 18 ++ 2 files changed, 137 insertions(+), 56 deletions(-) diff --git a/modules/objdetect/src/qrcode_encoder.cpp b/modules/objdetect/src/qrcode_encoder.cpp index 4ab1e1ac40..95e3e9bc35 100644 --- a/modules/objdetect/src/qrcode_encoder.cpp +++ b/modules/objdetect/src/qrcode_encoder.cpp @@ -196,17 +196,18 @@ protected: uint8_t total_num; vector final_qrcodes; - Ptr version_info; - Ptr cur_ecc_params; + const VersionInfo* version_info; + const BlockParams* cur_ecc_params; - bool isNumeric(const std::string& input); - bool isAlphaNumeric(const std::string& input); + bool isNumeric(const std::string& input) const; + bool isAlphaNumeric(const std::string& input) const; + EncodeMode autoEncodeMode(const std::string &input) const ; bool encodeByte(const std::string& input, vector &output); bool encodeAlpha(const std::string& input, vector &output); bool encodeNumeric(const std::string& input, vector &output); bool encodeECI(const std::string& input, vector &output); bool encodeKanji(const std::string& input, vector &output); - bool encodeAuto(const std::string& input, vector &output); + bool encodeAuto(const std::string& input, vector &output, EncodeMode *mode = nullptr); bool encodeStructure(const std::string& input, vector &output); int eccLevelToCode(CorrectionLevel level); void padBitStream(); @@ -222,9 +223,9 @@ protected: void fillReserved(const vector &format_array, Mat &masked); void maskData(const int mask_type_num, Mat &masked); void findAutoMaskType(); - bool estimateVersion(const int input_length, vector &possible_version); + bool estimateVersion(const int input_length, EncodeMode mode, vector &possible_version); int versionAuto(const std::string &input_str); - int findVersionCapacity(const int input_length, const int ecc, const int version_begin, const int version_end); + int findVersionCapacity(const int input_length, const int ecc, const std::vector& possible_versions); void generatingProcess(const std::string& input, Mat &qrcode); void generateQR(const std::string& input); }; @@ -247,17 +248,17 @@ int QRCodeEncoderImpl::eccLevelToCode(CorrectionLevel level) "CORRECT_LEVEL_L, CORRECT_LEVEL_M, CORRECT_LEVEL_Q, CORRECT_LEVEL_H." ); } -int QRCodeEncoderImpl::findVersionCapacity(const int input_length, const int ecc, const int version_begin, const int version_end) +int QRCodeEncoderImpl::findVersionCapacity(const int input_length, const int ecc, const std::vector& possible_versions) { int data_codewords, version_index = -1; const int byte_len = 8; version_index = -1; - for (int i = version_begin; i < version_end; i++) + for (int i : possible_versions) { - Ptr tmp_ecc_params = makePtr(version_info_database[i].ecc[ecc]); - data_codewords = tmp_ecc_params->data_codewords_in_G1 * tmp_ecc_params->num_blocks_in_G1 + - tmp_ecc_params->data_codewords_in_G2 * tmp_ecc_params->num_blocks_in_G2; + auto& tmp_ecc_params = version_info_database[i].ecc[ecc]; + data_codewords = tmp_ecc_params.data_codewords_in_G1 * tmp_ecc_params.num_blocks_in_G1 + + tmp_ecc_params.data_codewords_in_G2 * tmp_ecc_params.num_blocks_in_G2; if (data_codewords * byte_len >= input_length) { @@ -268,53 +269,70 @@ int QRCodeEncoderImpl::findVersionCapacity(const int input_length, const int ecc return version_index; } -bool QRCodeEncoderImpl::estimateVersion(const int input_length, vector& possible_version) +static inline int getCapacity(int version, QRCodeEncoder::CorrectionLevel ecc_level, QRCodeEncoder::EncodeMode mode) { + const int* capacity = version_capacity_database[version].ec_level[ecc_level].encoding_modes; + switch (mode) { + case QRCodeEncoder::EncodeMode::MODE_NUMERIC: + return capacity[0]; + case QRCodeEncoder::EncodeMode::MODE_ALPHANUMERIC: + return capacity[1]; + case QRCodeEncoder::EncodeMode::MODE_BYTE: + return capacity[2]; + case QRCodeEncoder::EncodeMode::MODE_KANJI: + return capacity[3]; + default: + CV_Error(Error::StsNotImplemented, format("Unexpected mode %d", mode)); + } +} + +bool QRCodeEncoderImpl::estimateVersion(const int input_length, EncodeMode mode, vector& possible_version) { possible_version.clear(); - if (input_length > version_capacity_database[40].ec_level[ecc_level].encoding_modes[1]) + + CV_Assert(mode != EncodeMode::MODE_AUTO); + + if (input_length > getCapacity(MAX_VERSION, ecc_level, mode)) + { return false; - if (input_length <= version_capacity_database[9].ec_level[ecc_level].encoding_modes[3]) - { - possible_version.push_back(1); } - else if (input_length <= version_capacity_database[9].ec_level[ecc_level].encoding_modes[1]) + + int version = MAX_VERSION; + + for (; version > 0; --version) { - possible_version.push_back(1); - possible_version.push_back(2); + if (input_length > getCapacity(version, ecc_level, mode)) { + break; + } } - else if (input_length <= version_capacity_database[26].ec_level[ecc_level].encoding_modes[3]) + + if (version < MAX_VERSION) { - possible_version.push_back(2); + version += 1; } - else if (input_length <= version_capacity_database[26].ec_level[ecc_level].encoding_modes[1]) + + possible_version.push_back(version); + + if (version < MAX_VERSION) { - possible_version.push_back(2); - possible_version.push_back(3); - } - else - { - possible_version.push_back(3); + possible_version.push_back(version + 1); } + return true; } int QRCodeEncoderImpl::versionAuto(const std::string& input_str) { - vector possible_version; - estimateVersion((int)input_str.length(), possible_version); - int tmp_version = 0; vector payload_tmp; - int version_range[5] = {0, 1, 10, 27, 41}; - for(size_t i = 0; i < possible_version.size(); i++) - { - int version_range_index = possible_version[i]; + EncodeMode mode; + encodeAuto(input_str, payload_tmp, &mode); - encodeAuto(input_str, payload_tmp); - tmp_version = findVersionCapacity((int)payload_tmp.size(), ecc_level, - version_range[version_range_index], version_range[version_range_index + 1]); - if(tmp_version != -1) - break; + vector possible_version; + if (!estimateVersion((int)input_str.length(), mode, possible_version)) { + return -1; } + + const auto tmp_version = findVersionCapacity((int)payload_tmp.size(), ecc_level, possible_version); + return tmp_version; } @@ -342,20 +360,23 @@ void QRCodeEncoderImpl::generateQR(const std::string &input) std::string input_info = input.substr(segment_begin, segment_end); string_itr += segment_end; + int detected_version = versionAuto(input_info); - CV_Assert(detected_version != -1); - if (version_level == 0) - version_level = detected_version; - else if (version_level < detected_version) + int tmp_version_level = version_level; + if (detected_version == -1) + CV_Error(Error::StsBadArg, "The given input exceeds the maximum capacity of a QR code with the selected encoding mode and error correction level " ); + else if (tmp_version_level == 0) + tmp_version_level = detected_version; + else if (tmp_version_level < detected_version) CV_Error(Error::StsBadArg, "The given version is not suitable for the given input string length "); payload.clear(); payload.reserve(MAX_PAYLOAD_LEN); format = vector (15, 255); version_reserved = vector (18, 255); - version_size = (21 + (version_level - 1) * 4); - version_info = makePtr(version_info_database[version_level]); - cur_ecc_params = makePtr(version_info->ecc[ecc_level]); + version_size = (21 + (tmp_version_level - 1) * 4); + version_info = &version_info_database[tmp_version_level]; + cur_ecc_params = &version_info->ecc[ecc_level]; original = Mat(Size(version_size, version_size), CV_8UC1, Scalar(255)); masked_data = original.clone(); Mat qrcode = masked_data.clone(); @@ -613,7 +634,7 @@ bool QRCodeEncoderImpl::encodeStructure(const std::string& input, vector& output) +QRCodeEncoder::EncodeMode QRCodeEncoderImpl::autoEncodeMode(const std::string &input) const { if (isNumeric(input)) - encodeNumeric(input, output); - else if (isAlphaNumeric(input)) - encodeAlpha(input, output); - else - encodeByte(input, output); + { + return EncodeMode::MODE_NUMERIC; + } + + if (isAlphaNumeric(input)) + { + return EncodeMode::MODE_ALPHANUMERIC; + } + + return EncodeMode::MODE_BYTE; +} + +bool QRCodeEncoderImpl::encodeAuto(const std::string& input, vector& output, EncodeMode *mode) +{ + const auto selected_mode = autoEncodeMode(input); + + CV_Assert(selected_mode != EncodeMode::MODE_AUTO); + + switch (selected_mode) + { + case EncodeMode::MODE_NUMERIC: + encodeNumeric(input, output); + break; + case EncodeMode::MODE_ALPHANUMERIC: + encodeAlpha(input, output); + break; + case EncodeMode::MODE_STRUCTURED_APPEND: + encodeByte(input, output); + break; + case EncodeMode::MODE_BYTE: + encodeByte(input, output); + break; + case EncodeMode::MODE_KANJI: + encodeKanji(input, output); + break; + case EncodeMode::MODE_ECI: + encodeECI(input, output); + break; + default: + break; + } + + if (mode != nullptr) + { + *mode = selected_mode; + } + return true; } diff --git a/modules/objdetect/test/test_qrcode_encode.cpp b/modules/objdetect/test/test_qrcode_encode.cpp index 1005793269..5dc31a4c6e 100644 --- a/modules/objdetect/test/test_qrcode_encode.cpp +++ b/modules/objdetect/test/test_qrcode_encode.cpp @@ -548,4 +548,22 @@ TEST(Objdetect_QRCode_Encode_Decode, regression_issue22029) } } +// This test reproduces issue https://github.com/opencv/opencv/issues/24366 only in a loop +TEST(Objdetect_QRCode_Encode_Decode, auto_version_pick) +{ + cv::QRCodeEncoder::Params params; + params.correction_level = cv::QRCodeEncoder::CORRECT_LEVEL_L; + params.mode = cv::QRCodeEncoder::EncodeMode::MODE_AUTO; + + cv::Ptr encoder = cv::QRCodeEncoder::create(params); + + for (int len = 1; len < 19; len++) { + std::string input; + input.resize(len); + cv::randu(Mat(1, len, CV_8U, &input[0]), 'a', 'z' + 1); + cv::Mat qrcode; + encoder->encode(input, qrcode); + } +} + }} // namespace From 6c57ce9e09011763a973234f2e6d8acfe2984e1a Mon Sep 17 00:00:00 2001 From: alexlyulkov Date: Fri, 17 Nov 2023 18:17:17 +0700 Subject: [PATCH 035/156] Merge pull request #24473 from alexlyulkov:al/samples_with_maven Updated Android samples for modern Android studio. Added OpenCV from Maven support. #24473 Updated samples for recent Android studio: - added namespace field that is required in build.gradle files - replaced _switch_ by _if-else_ because it doesn't work with constants from resources - added missed log library dependency in face-detection/jni/CMakeLists.txt - use local.properties to define NDK location Added support for OpenCV from Maven. Now you can choose 3 possible sources of OpenCV lib in settings.gradle: SDK path, local Maven repository, public Maven repository. (Creating Maven repository from SDK is added here #24456 ) There are differences in project configs for SDK and Maven versions: - different dependencies in build.gradle - different OpenCV library names in CMakeLists.txt - SDK version requires OpenCV_DIR definition Requires: - https://github.com/opencv/ci-gha-workflow/pull/124 - https://github.com/opencv-infrastructure/opencv-gha-dockerfile/pull/26 --- cmake/android/android_gradle_projects.cmake | 40 ++++++++++++++----- platforms/android/build-tests/test_gradle.sh | 3 ++ samples/android/15-puzzle/build.gradle.in | 7 +++- samples/android/build.gradle.in | 5 +++ .../camera-calibration/build.gradle.in | 7 +++- .../CameraCalibrationActivity.java | 11 +++-- .../color-blob-detection/build.gradle.in | 7 +++- .../android/face-detection/build.gradle.in | 21 +++++++++- .../android/face-detection/jni/CMakeLists.txt | 10 ++++- .../image-manipulations/build.gradle.in | 7 +++- .../mobilenet-objdetect/build.gradle.in | 7 +++- .../tutorial-1-camerapreview/build.gradle.in | 7 +++- .../build.gradle.in | 21 +++++++++- .../jni/CMakeLists.txt | 7 +++- .../tutorial-3-cameracontrol/build.gradle.in | 7 +++- .../android/tutorial-4-opencl/build.gradle.in | 21 +++++++++- .../tutorial-4-opencl/jni/CMakeLists.txt | 9 ++++- 17 files changed, 163 insertions(+), 34 deletions(-) diff --git a/cmake/android/android_gradle_projects.cmake b/cmake/android/android_gradle_projects.cmake index 0a8b9f4b60..a3f4ba2c65 100644 --- a/cmake/android/android_gradle_projects.cmake +++ b/cmake/android/android_gradle_projects.cmake @@ -1,8 +1,8 @@ # https://developer.android.com/studio/releases/gradle-plugin -set(ANDROID_GRADLE_PLUGIN_VERSION "3.2.1" CACHE STRING "Android Gradle Plugin version") +set(ANDROID_GRADLE_PLUGIN_VERSION "7.3.1" CACHE STRING "Android Gradle Plugin version") message(STATUS "Android Gradle Plugin version: ${ANDROID_GRADLE_PLUGIN_VERSION}") -set(KOTLIN_PLUGIN_VERSION "1.4.10" CACHE STRING "Kotlin Plugin version") +set(KOTLIN_PLUGIN_VERSION "1.5.20" CACHE STRING "Kotlin Plugin version") message(STATUS "Kotlin Plugin version: ${KOTLIN_PLUGIN_VERSION}") if(BUILD_KOTLIN_EXTENSIONS) @@ -13,7 +13,7 @@ else() set(KOTLIN_STD_LIB "" CACHE STRING "Kotlin Standard Library dependency") endif() -set(GRADLE_VERSION "5.6.4" CACHE STRING "Gradle version") +set(GRADLE_VERSION "7.6.3" CACHE STRING "Gradle version") message(STATUS "Gradle version: ${GRADLE_VERSION}") set(ANDROID_COMPILE_SDK_VERSION "26" CACHE STRING "Android compileSdkVersion") @@ -22,7 +22,7 @@ if(ANDROID_NATIVE_API_LEVEL GREATER 21) else() set(ANDROID_MIN_SDK_VERSION "21" CACHE STRING "Android minSdkVersion") endif() -set(ANDROID_TARGET_SDK_VERSION "26" CACHE STRING "Android minSdkVersion") +set(ANDROID_TARGET_SDK_VERSION "31" CACHE STRING "Android minSdkVersion") set(ANDROID_BUILD_BASE_DIR "${OpenCV_BINARY_DIR}/opencv_android" CACHE INTERNAL "") set(ANDROID_TMP_INSTALL_BASE_DIR "${CMAKE_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/install/opencv_android") @@ -109,21 +109,41 @@ if(NOT OPENCV_SKIP_ANDROID_FORCE_CMAKE) get_filename_component(_CMAKE_INSTALL_DIR "${CMAKE_ROOT}" PATH) get_filename_component(_CMAKE_INSTALL_DIR "${_CMAKE_INSTALL_DIR}" PATH) endif() - ocv_update_file("${ANDROID_BUILD_BASE_DIR}/local.properties" "cmake.dir=${_CMAKE_INSTALL_DIR}") + ocv_update_file("${ANDROID_BUILD_BASE_DIR}/local.properties" "cmake.dir=${_CMAKE_INSTALL_DIR}\nndk.dir=${ANDROID_NDK}") endif() file(WRITE "${ANDROID_BUILD_BASE_DIR}/settings.gradle" " +gradle.ext { + //opencv_source = 'maven_central' + //opencv_source = 'maven_local' + opencv_source = 'sdk_path' +} + include ':opencv' ") file(WRITE "${ANDROID_TMP_INSTALL_BASE_DIR}/settings.gradle" " rootProject.name = 'opencv_samples' -def opencvsdk='../' -//def opencvsdk='/' -//println opencvsdk -include ':opencv' -project(':opencv').projectDir = new File(opencvsdk + '/sdk') +gradle.ext { + //opencv_source = 'maven_central' + //opencv_source = 'maven_local' + opencv_source = 'sdk_path' +} + +if (gradle.opencv_source == 'maven_local') { + gradle.ext { + opencv_maven_path = '/' + } +} + +if (gradle.opencv_source == 'sdk_path') { + def opencvsdk='../' + //def opencvsdk='/' + //println opencvsdk + include ':opencv' + project(':opencv').projectDir = new File(opencvsdk + '/sdk') +} ") ocv_check_environment_variables(OPENCV_GRADLE_VERBOSE_OPTIONS) diff --git a/platforms/android/build-tests/test_gradle.sh b/platforms/android/build-tests/test_gradle.sh index 9f1b233ff2..bb6bc91a8e 100755 --- a/platforms/android/build-tests/test_gradle.sh +++ b/platforms/android/build-tests/test_gradle.sh @@ -29,6 +29,9 @@ rm -rf "test-gradle" cp -rp "${SDK_DIR}" "test-gradle" echo "Cloning OpenCV Android SDK ... Done!" +echo "Force Current CMake for Gradle project" +# drop cmake bin name and "bin" folder from path +echo "cmake.dir=$(dirname $(dirname $(which cmake)))" > "test-gradle/samples/local.properties" echo "Run gradle ..." (cd "test-gradle/samples"; ./gradlew -i assemble) diff --git a/samples/android/15-puzzle/build.gradle.in b/samples/android/15-puzzle/build.gradle.in index ad5c9f93f9..b1808d3892 100644 --- a/samples/android/15-puzzle/build.gradle.in +++ b/samples/android/15-puzzle/build.gradle.in @@ -1,6 +1,7 @@ apply plugin: 'com.android.application' android { + namespace 'org.opencv.samples.puzzle15' compileSdkVersion @ANDROID_COMPILE_SDK_VERSION@ defaultConfig { applicationId "org.opencv.samples.puzzle15" @@ -27,5 +28,9 @@ android { dependencies { //implementation fileTree(dir: 'libs', include: ['*.jar']) - implementation project(':opencv') + if (gradle.opencv_source == "sdk_path") { + implementation project(':opencv') + } else if (gradle.opencv_source == "maven_local" || gradle.opencv_source == "maven_cenral") { + implementation 'org.opencv:opencv:@OPENCV_VERSION_PLAIN@' + } } diff --git a/samples/android/build.gradle.in b/samples/android/build.gradle.in index f36fe216f5..d159899132 100644 --- a/samples/android/build.gradle.in +++ b/samples/android/build.gradle.in @@ -19,6 +19,11 @@ allprojects { repositories { google() jcenter() + if (gradle.opencv_source == "maven_local") { + maven { + url gradle.opencv_maven_path + } + } } } diff --git a/samples/android/camera-calibration/build.gradle.in b/samples/android/camera-calibration/build.gradle.in index d62b151867..69c5b5bbee 100644 --- a/samples/android/camera-calibration/build.gradle.in +++ b/samples/android/camera-calibration/build.gradle.in @@ -1,6 +1,7 @@ apply plugin: 'com.android.application' android { + namespace 'org.opencv.samples.cameracalibration' compileSdkVersion @ANDROID_COMPILE_SDK_VERSION@ defaultConfig { applicationId "org.opencv.samples.cameracalibration" @@ -27,5 +28,9 @@ android { dependencies { //implementation fileTree(dir: 'libs', include: ['*.jar']) - implementation project(':opencv') + if (gradle.opencv_source == "sdk_path") { + implementation project(':opencv') + } else if (gradle.opencv_source == "maven_local" || gradle.opencv_source == "maven_cenral") { + implementation 'org.opencv:opencv:@OPENCV_VERSION_PLAIN@' + } } diff --git a/samples/android/camera-calibration/src/org/opencv/samples/cameracalibration/CameraCalibrationActivity.java b/samples/android/camera-calibration/src/org/opencv/samples/cameracalibration/CameraCalibrationActivity.java index 30a6ad9c1d..43132ab93b 100644 --- a/samples/android/camera-calibration/src/org/opencv/samples/cameracalibration/CameraCalibrationActivity.java +++ b/samples/android/camera-calibration/src/org/opencv/samples/cameracalibration/CameraCalibrationActivity.java @@ -136,23 +136,22 @@ public class CameraCalibrationActivity extends CameraActivity implements CvCamer @Override public boolean onOptionsItemSelected(MenuItem item) { - switch (item.getItemId()) { - case R.id.calibration: + if (item.getItemId() == R.id.calibration) { mOnCameraFrameRender = new OnCameraFrameRender(new CalibrationFrameRender(mCalibrator)); item.setChecked(true); return true; - case R.id.undistortion: + } else if (item.getItemId() == R.id.undistortion) { mOnCameraFrameRender = new OnCameraFrameRender(new UndistortionFrameRender(mCalibrator)); item.setChecked(true); return true; - case R.id.comparison: + } else if (item.getItemId() == R.id.comparison) { mOnCameraFrameRender = new OnCameraFrameRender(new ComparisonFrameRender(mCalibrator, mWidth, mHeight, getResources())); item.setChecked(true); return true; - case R.id.calibrate: + } else if (item.getItemId() == R.id.calibrate) { final Resources res = getResources(); if (mCalibrator.getCornersBufferSize() < 2) { (Toast.makeText(this, res.getString(R.string.more_samples), Toast.LENGTH_SHORT)).show(); @@ -196,7 +195,7 @@ public class CameraCalibrationActivity extends CameraActivity implements CvCamer } }.execute(); return true; - default: + } else { return super.onOptionsItemSelected(item); } } diff --git a/samples/android/color-blob-detection/build.gradle.in b/samples/android/color-blob-detection/build.gradle.in index 8900e25c16..82cbfe79f6 100644 --- a/samples/android/color-blob-detection/build.gradle.in +++ b/samples/android/color-blob-detection/build.gradle.in @@ -1,6 +1,7 @@ apply plugin: 'com.android.application' android { + namespace 'org.opencv.samples.colorblobdetect' compileSdkVersion @ANDROID_COMPILE_SDK_VERSION@ defaultConfig { applicationId "org.opencv.samples.colorblobdetect" @@ -27,5 +28,9 @@ android { dependencies { //implementation fileTree(dir: 'libs', include: ['*.jar']) - implementation project(':opencv') + if (gradle.opencv_source == "sdk_path") { + implementation project(':opencv') + } else if (gradle.opencv_source == "maven_local" || gradle.opencv_source == "maven_cenral") { + implementation 'org.opencv:opencv:@OPENCV_VERSION_PLAIN@' + } } diff --git a/samples/android/face-detection/build.gradle.in b/samples/android/face-detection/build.gradle.in index 9e3b187a84..87fdbbf533 100644 --- a/samples/android/face-detection/build.gradle.in +++ b/samples/android/face-detection/build.gradle.in @@ -1,6 +1,7 @@ apply plugin: 'com.android.application' android { + namespace 'org.opencv.samples.facedetect' compileSdkVersion @ANDROID_COMPILE_SDK_VERSION@ defaultConfig { applicationId "org.opencv.samples.facedetect" @@ -11,7 +12,14 @@ android { externalNativeBuild { cmake { - arguments "-DOpenCV_DIR=" + project(':opencv').projectDir + "/@ANDROID_PROJECT_JNI_PATH@"@OPENCV_ANDROID_CMAKE_EXTRA_ARGS@ + if (gradle.opencv_source == "sdk_path") { + arguments "-DOpenCV_DIR=" + project(':opencv').projectDir + "/@ANDROID_PROJECT_JNI_PATH@", + "-DOPENCV_FROM_SDK=TRUE"@OPENCV_ANDROID_CMAKE_EXTRA_ARGS@ + + } else { + arguments "-DOPENCV_VERSION_MAJOR=@OPENCV_VERSION_MAJOR@", + "-DOPENCV_FROM_SDK=FALSE"@OPENCV_ANDROID_CMAKE_EXTRA_ARGS@ + } targets "detection_based_tracker" } } @@ -35,9 +43,18 @@ android { path '@ANDROID_SAMPLE_JNI_PATH@/CMakeLists.txt' } } + buildFeatures { + if (gradle.opencv_source == "maven_local" || gradle.opencv_source == "maven_cenral") { + prefab true + } + } } dependencies { //implementation fileTree(dir: 'libs', include: ['*.jar']) - implementation project(':opencv') + if (gradle.opencv_source == "sdk_path") { + implementation project(':opencv') + } else if (gradle.opencv_source == "maven_local" || gradle.opencv_source == "maven_cenral") { + implementation 'org.opencv:opencv:@OPENCV_VERSION_PLAIN@' + } } diff --git a/samples/android/face-detection/jni/CMakeLists.txt b/samples/android/face-detection/jni/CMakeLists.txt index 15f8f4db38..4c18a0bca3 100644 --- a/samples/android/face-detection/jni/CMakeLists.txt +++ b/samples/android/face-detection/jni/CMakeLists.txt @@ -3,7 +3,12 @@ cmake_minimum_required(VERSION 3.6) set(target detection_based_tracker) project(${target} CXX) -set(ANDROID_OPENCV_COMPONENTS "opencv_java" CACHE STRING "") +if (OPENCV_FROM_SDK) + set(ANDROID_OPENCV_COMPONENTS "opencv_java" CACHE STRING "") +else() + set(ANDROID_OPENCV_COMPONENTS "OpenCV::opencv_java${OPENCV_VERSION_MAJOR}" CACHE STRING "") +endif() + message(STATUS "ANDROID_ABI=${ANDROID_ABI}") find_package(OpenCV REQUIRED COMPONENTS ${ANDROID_OPENCV_COMPONENTS}) @@ -12,4 +17,5 @@ file(GLOB hdrs *.hpp *.h) include_directories("${CMAKE_CURRENT_LIST_DIR}") add_library(${target} SHARED ${srcs} ${hdrs}) -target_link_libraries(${target} ${ANDROID_OPENCV_COMPONENTS}) +find_library(log_lib log) +target_link_libraries(${target} ${ANDROID_OPENCV_COMPONENTS} ${log_lib}) diff --git a/samples/android/image-manipulations/build.gradle.in b/samples/android/image-manipulations/build.gradle.in index 0d685ed410..d36d270a19 100644 --- a/samples/android/image-manipulations/build.gradle.in +++ b/samples/android/image-manipulations/build.gradle.in @@ -1,6 +1,7 @@ apply plugin: 'com.android.application' android { + namespace 'org.opencv.samples.imagemanipulations' compileSdkVersion @ANDROID_COMPILE_SDK_VERSION@ defaultConfig { applicationId "org.opencv.samples.imagemanipulations" @@ -27,5 +28,9 @@ android { dependencies { //implementation fileTree(dir: 'libs', include: ['*.jar']) - implementation project(':opencv') + if (gradle.opencv_source == "sdk_path") { + implementation project(':opencv') + } else if (gradle.opencv_source == "maven_local" || gradle.opencv_source == "maven_cenral") { + implementation 'org.opencv:opencv:@OPENCV_VERSION_PLAIN@' + } } diff --git a/samples/android/mobilenet-objdetect/build.gradle.in b/samples/android/mobilenet-objdetect/build.gradle.in index e8238f7324..e1ac8b503e 100644 --- a/samples/android/mobilenet-objdetect/build.gradle.in +++ b/samples/android/mobilenet-objdetect/build.gradle.in @@ -1,6 +1,7 @@ apply plugin: 'com.android.application' android { + namespace 'org.opencv.samples.opencv_mobilenet' compileSdkVersion @ANDROID_COMPILE_SDK_VERSION@ defaultConfig { applicationId "org.opencv.samples.opencv_mobilenet" @@ -27,5 +28,9 @@ android { dependencies { //implementation fileTree(dir: 'libs', include: ['*.jar']) - implementation project(':opencv') + if (gradle.opencv_source == "sdk_path") { + implementation project(':opencv') + } else if (gradle.opencv_source == "maven_local" || gradle.opencv_source == "maven_cenral") { + implementation 'org.opencv:opencv:@OPENCV_VERSION_PLAIN@' + } } diff --git a/samples/android/tutorial-1-camerapreview/build.gradle.in b/samples/android/tutorial-1-camerapreview/build.gradle.in index 5a649175dc..9de83844ab 100644 --- a/samples/android/tutorial-1-camerapreview/build.gradle.in +++ b/samples/android/tutorial-1-camerapreview/build.gradle.in @@ -1,6 +1,7 @@ apply plugin: 'com.android.application' android { + namespace 'org.opencv.samples.tutorial1' compileSdkVersion @ANDROID_COMPILE_SDK_VERSION@ defaultConfig { applicationId "org.opencv.samples.tutorial1" @@ -27,5 +28,9 @@ android { dependencies { //implementation fileTree(dir: 'libs', include: ['*.jar']) - implementation project(':opencv') + if (gradle.opencv_source == "sdk_path") { + implementation project(':opencv') + } else if (gradle.opencv_source == "maven_local" || gradle.opencv_source == "maven_cenral") { + implementation 'org.opencv:opencv:@OPENCV_VERSION_PLAIN@' + } } diff --git a/samples/android/tutorial-2-mixedprocessing/build.gradle.in b/samples/android/tutorial-2-mixedprocessing/build.gradle.in index 7eca49b1b1..601f946797 100644 --- a/samples/android/tutorial-2-mixedprocessing/build.gradle.in +++ b/samples/android/tutorial-2-mixedprocessing/build.gradle.in @@ -1,6 +1,7 @@ apply plugin: 'com.android.application' android { + namespace 'org.opencv.samples.tutorial2' compileSdkVersion @ANDROID_COMPILE_SDK_VERSION@ defaultConfig { applicationId "org.opencv.samples.tutorial2" @@ -11,7 +12,14 @@ android { externalNativeBuild { cmake { - arguments "-DOpenCV_DIR=" + project(':opencv').projectDir + "/@ANDROID_PROJECT_JNI_PATH@"@OPENCV_ANDROID_CMAKE_EXTRA_ARGS@ + if (gradle.opencv_source == "sdk_path") { + arguments "-DOpenCV_DIR=" + project(':opencv').projectDir + "/@ANDROID_PROJECT_JNI_PATH@", + "-DOPENCV_FROM_SDK=TRUE"@OPENCV_ANDROID_CMAKE_EXTRA_ARGS@ + + } else { + arguments "-DOPENCV_VERSION_MAJOR=@OPENCV_VERSION_MAJOR@", + "-DOPENCV_FROM_SDK=FALSE"@OPENCV_ANDROID_CMAKE_EXTRA_ARGS@ + } targets "mixed_sample" } } @@ -35,9 +43,18 @@ android { path '@ANDROID_SAMPLE_JNI_PATH@/CMakeLists.txt' } } + buildFeatures { + if (gradle.opencv_source == "maven_local" || gradle.opencv_source == "maven_cenral") { + prefab true + } + } } dependencies { //implementation fileTree(dir: 'libs', include: ['*.jar']) - implementation project(':opencv') + if (gradle.opencv_source == "sdk_path") { + implementation project(':opencv') + } else if (gradle.opencv_source == "maven_local" || gradle.opencv_source == "maven_cenral") { + implementation 'org.opencv:opencv:@OPENCV_VERSION_PLAIN@' + } } diff --git a/samples/android/tutorial-2-mixedprocessing/jni/CMakeLists.txt b/samples/android/tutorial-2-mixedprocessing/jni/CMakeLists.txt index 5b34f8b948..ff2a8a48f7 100644 --- a/samples/android/tutorial-2-mixedprocessing/jni/CMakeLists.txt +++ b/samples/android/tutorial-2-mixedprocessing/jni/CMakeLists.txt @@ -3,7 +3,12 @@ cmake_minimum_required(VERSION 3.6) set(target mixed_sample) project(${target} CXX) -set(ANDROID_OPENCV_COMPONENTS "opencv_java" CACHE STRING "") +if (OPENCV_FROM_SDK) + set(ANDROID_OPENCV_COMPONENTS "opencv_java" CACHE STRING "") +else() + set(ANDROID_OPENCV_COMPONENTS "OpenCV::opencv_java${OPENCV_VERSION_MAJOR}" CACHE STRING "") +endif() + message(STATUS "ANDROID_ABI=${ANDROID_ABI}") find_package(OpenCV REQUIRED COMPONENTS ${ANDROID_OPENCV_COMPONENTS}) diff --git a/samples/android/tutorial-3-cameracontrol/build.gradle.in b/samples/android/tutorial-3-cameracontrol/build.gradle.in index 0ba304f5e5..a577cb6984 100644 --- a/samples/android/tutorial-3-cameracontrol/build.gradle.in +++ b/samples/android/tutorial-3-cameracontrol/build.gradle.in @@ -1,6 +1,7 @@ apply plugin: 'com.android.application' android { + namespace 'org.opencv.samples.tutorial3' compileSdkVersion @ANDROID_COMPILE_SDK_VERSION@ defaultConfig { applicationId "org.opencv.samples.tutorial3" @@ -27,5 +28,9 @@ android { dependencies { //implementation fileTree(dir: 'libs', include: ['*.jar']) - implementation project(':opencv') + if (gradle.opencv_source == "sdk_path") { + implementation project(':opencv') + } else if (gradle.opencv_source == "maven_local" || gradle.opencv_source == "maven_cenral") { + implementation 'org.opencv:opencv:@OPENCV_VERSION_PLAIN@' + } } diff --git a/samples/android/tutorial-4-opencl/build.gradle.in b/samples/android/tutorial-4-opencl/build.gradle.in index 2cb9a7bcb9..17a5d37190 100644 --- a/samples/android/tutorial-4-opencl/build.gradle.in +++ b/samples/android/tutorial-4-opencl/build.gradle.in @@ -1,6 +1,7 @@ apply plugin: 'com.android.application' android { + namespace 'org.opencv.samples.tutorial4' compileSdkVersion @ANDROID_COMPILE_SDK_VERSION@ defaultConfig { applicationId "org.opencv.samples.tutorial4" @@ -11,7 +12,14 @@ android { externalNativeBuild { cmake { - arguments "-DOpenCV_DIR=" + project(':opencv').projectDir + "/@ANDROID_PROJECT_JNI_PATH@"@OPENCV_ANDROID_CMAKE_EXTRA_ARGS@ + if (gradle.opencv_source == "sdk_path") { + arguments "-DOpenCV_DIR=" + project(':opencv').projectDir + "/@ANDROID_PROJECT_JNI_PATH@", + "-DOPENCV_FROM_SDK=TRUE"@OPENCV_ANDROID_CMAKE_EXTRA_ARGS@ + + } else { + arguments "-DOPENCV_VERSION_MAJOR=@OPENCV_VERSION_MAJOR@", + "-DOPENCV_FROM_SDK=FALSE"@OPENCV_ANDROID_CMAKE_EXTRA_ARGS@ + } targets "JNIpart" } } @@ -35,9 +43,18 @@ android { path '@ANDROID_SAMPLE_JNI_PATH@/CMakeLists.txt' } } + buildFeatures { + if (gradle.opencv_source == "maven_local" || gradle.opencv_source == "maven_cenral") { + prefab true + } + } } dependencies { //implementation fileTree(dir: 'libs', include: ['*.jar']) - implementation project(':opencv') + if (gradle.opencv_source == "sdk_path") { + implementation project(':opencv') + } else if (gradle.opencv_source == "maven_local" || gradle.opencv_source == "maven_cenral") { + implementation 'org.opencv:opencv:@OPENCV_VERSION_PLAIN@' + } } diff --git a/samples/android/tutorial-4-opencl/jni/CMakeLists.txt b/samples/android/tutorial-4-opencl/jni/CMakeLists.txt index 4fdea1356e..923ccd318e 100644 --- a/samples/android/tutorial-4-opencl/jni/CMakeLists.txt +++ b/samples/android/tutorial-4-opencl/jni/CMakeLists.txt @@ -1,9 +1,14 @@ cmake_minimum_required(VERSION 3.6) -set(target mixed_sample) +set(target JNIpart) project(${target} CXX) -set(ANDROID_OPENCV_COMPONENTS "opencv_java" CACHE STRING "") +if (OPENCV_FROM_SDK) + set(ANDROID_OPENCV_COMPONENTS "opencv_java" CACHE STRING "") +else() + set(ANDROID_OPENCV_COMPONENTS "OpenCV::opencv_java${OPENCV_VERSION_MAJOR}" CACHE STRING "") +endif() + message(STATUS "ANDROID_ABI=${ANDROID_ABI}") find_package(OpenCV REQUIRED COMPONENTS ${ANDROID_OPENCV_COMPONENTS}) From a478757483c644666ad36219db0aa81550abefc8 Mon Sep 17 00:00:00 2001 From: Yuantao Feng Date: Mon, 20 Nov 2023 02:19:24 -0600 Subject: [PATCH 036/156] Merge pull request #24544 from fengyuentau:layernorm_conformance dnn test: move layer norm tests into conformance tests #24544 Merge with https://github.com/opencv/opencv_extra/pull/1122 ## Motivation Some ONNX operators, such as `LayerNormalization`, `BatchNormalization` and so on, produce outputs for training (mean, stdev). So they have reference outputs of conformance tests for those training outputs as well. However, when it comes to inference, we do not need and produce those outputs for training here in dnn. Hence, output size does not match if we use dnn to infer those conformance models. This has become the barrier if we want to test these operators using their conformance tests. **I checked all ONNX operators with optional outputs. Turns out there are only `BatchNormalization`, `Dropout`, `LayerNormalization` and `MaxPool` has optional outputs for training. All except `LayerNormalization` have models set for training mode and eval mode. Blame ONNX for that.** ## Solution In this pull request, we remove graph outputs if the graph looks like the following: ``` [X] [Scale] [Bias] [X] [Scale] [Bias] \ | / this patch \ | / LayerNormalization -----------> LayerNormalization / | \ | [Y] [Mean] [Stdev] [Y] ``` We can update conformance tests and turn on some cases as well if extending to more layers. Notes: 1. This workaround does not solve expanded function operators if they are fused into a single operator, such as `$onnx/onnx/backend/test/data/node/test_layer_normalization_2d_axis1_expanded`, but they can be run without fusion. Note that either dnn or onnxruntime does not fuse those expanded function operators. ### Pull Request Readiness Checklist See details at https://github.com/opencv/opencv/wiki/How_to_contribute#making-a-good-pull-request - [x] I agree to contribute to the project under Apache 2 License. - [x] To the best of my knowledge, the proposed patch is not based on a code under GPL or another license that is incompatible with OpenCV - [x] The PR is proposed to the proper branch - [x] There is a reference to the original bug report and related work - [x] There is accuracy test, performance test and test data in opencv_extra repository, if applicable Patch to opencv_extra has the same branch name. - [x] The feature is well documented and sample code can be built with the project CMake --- modules/dnn/src/onnx/onnx_importer.cpp | 13 +++- modules/dnn/test/test_graph_simplifier.cpp | 1 + modules/dnn/test/test_onnx_conformance.cpp | 19 +++++ ...conformance_layer_filter__openvino.inl.hpp | 76 +++++++++++++++++++ modules/dnn/test/test_onnx_importer.cpp | 30 -------- 5 files changed, 107 insertions(+), 32 deletions(-) diff --git a/modules/dnn/src/onnx/onnx_importer.cpp b/modules/dnn/src/onnx/onnx_importer.cpp index 28dd4d9b77..ba4d542731 100644 --- a/modules/dnn/src/onnx/onnx_importer.cpp +++ b/modules/dnn/src/onnx/onnx_importer.cpp @@ -3187,10 +3187,19 @@ void ONNXImporter::parseLayerNorm(LayerParams& layerParams, const opencv_onnx::N // Remove additional outputs (Mean, InvStdDev) if (node_proto.output_size() > 1) { + // remove from graph proto + for (size_t i = 1; i < node_proto.output_size(); i++) { + for (int j = graph_proto.output_size() - 1; j >= 0; j--) { + if (graph_proto.output(j).name() == node_proto.output(i)) { + graph_proto.mutable_output()->DeleteSubrange(j, 1); + break; + } + } + } + // remove from node proto auto outputName = node_proto.output(0); opencv_onnx::NodeProto node_proto_ = node_proto; - node_proto_.clear_output(); - node_proto_.add_output(outputName); + node_proto_.mutable_output()->DeleteSubrange(1, node_proto_.output_size() - 1); addLayer(layerParams, node_proto_); } else diff --git a/modules/dnn/test/test_graph_simplifier.cpp b/modules/dnn/test/test_graph_simplifier.cpp index f6334f929a..58fffd3713 100644 --- a/modules/dnn/test/test_graph_simplifier.cpp +++ b/modules/dnn/test/test_graph_simplifier.cpp @@ -43,6 +43,7 @@ TEST_F(Test_Graph_Simplifier, GeluApproximationSubGraph) { TEST_F(Test_Graph_Simplifier, LayerNormSubGraph) { test("layer_norm_expanded", "LayerNormalization"); + test("layer_norm_expanded_with_initializers", "LayerNormalization"); } TEST_F(Test_Graph_Simplifier, ResizeSubgraph) { diff --git a/modules/dnn/test/test_onnx_conformance.cpp b/modules/dnn/test/test_onnx_conformance.cpp index 13c5c5b232..5b783722c4 100644 --- a/modules/dnn/test/test_onnx_conformance.cpp +++ b/modules/dnn/test/test_onnx_conformance.cpp @@ -339,6 +339,25 @@ static const TestCase testConformanceConfig[] = { {"test_isinf_negative", 1, 1}, {"test_isinf_positive", 1, 1}, {"test_isnan", 1, 1}, + {"test_layer_normalization_2d_axis0", 3, 1}, + {"test_layer_normalization_2d_axis1", 3, 1}, + {"test_layer_normalization_2d_axis_negative_1", 3, 1}, + {"test_layer_normalization_2d_axis_negative_2", 3, 1}, + {"test_layer_normalization_3d_axis0_epsilon", 3, 1}, + {"test_layer_normalization_3d_axis1_epsilon", 3, 1}, + {"test_layer_normalization_3d_axis2_epsilon", 3, 1}, + {"test_layer_normalization_3d_axis_negative_1_epsilon", 3, 1}, + {"test_layer_normalization_3d_axis_negative_2_epsilon", 3, 1}, + {"test_layer_normalization_3d_axis_negative_3_epsilon", 3, 1}, + {"test_layer_normalization_4d_axis0", 3, 1}, + {"test_layer_normalization_4d_axis1", 3, 1}, + {"test_layer_normalization_4d_axis2", 3, 1}, + {"test_layer_normalization_4d_axis3", 3, 1}, + {"test_layer_normalization_4d_axis_negative_1", 3, 1}, + {"test_layer_normalization_4d_axis_negative_2", 3, 1}, + {"test_layer_normalization_4d_axis_negative_3", 3, 1}, + {"test_layer_normalization_4d_axis_negative_4", 3, 1}, + {"test_layer_normalization_default_axis", 3, 1}, {"test_leakyrelu", 1, 1}, {"test_leakyrelu_default", 1, 1}, {"test_leakyrelu_example", 1, 1}, diff --git a/modules/dnn/test/test_onnx_conformance_layer_filter__openvino.inl.hpp b/modules/dnn/test/test_onnx_conformance_layer_filter__openvino.inl.hpp index 339746f5f2..94687a1e26 100644 --- a/modules/dnn/test/test_onnx_conformance_layer_filter__openvino.inl.hpp +++ b/modules/dnn/test/test_onnx_conformance_layer_filter__openvino.inl.hpp @@ -792,6 +792,82 @@ CASE(test_isinf_positive) // no filter CASE(test_isnan) // no filter +CASE(test_layer_normalization_2d_axis0) +#if SKIP_SET_1 + SKIP_NON_CPU; +#endif +CASE(test_layer_normalization_2d_axis1) +#if SKIP_SET_1 + SKIP_NON_CPU; +#endif +CASE(test_layer_normalization_2d_axis_negative_1) +#if SKIP_SET_1 + SKIP_NON_CPU; +#endif +CASE(test_layer_normalization_2d_axis_negative_2) +#if SKIP_SET_1 + SKIP_NON_CPU; +#endif +CASE(test_layer_normalization_3d_axis0_epsilon) +#if SKIP_SET_1 + SKIP_NON_CPU; +#endif +CASE(test_layer_normalization_3d_axis1_epsilon) +#if SKIP_SET_1 + SKIP_NON_CPU; +#endif +CASE(test_layer_normalization_3d_axis2_epsilon) +#if SKIP_SET_1 + SKIP_NON_CPU; +#endif +CASE(test_layer_normalization_3d_axis_negative_1_epsilon) +#if SKIP_SET_1 + SKIP_NON_CPU; +#endif +CASE(test_layer_normalization_3d_axis_negative_2_epsilon) +#if SKIP_SET_1 + SKIP_NON_CPU; +#endif +CASE(test_layer_normalization_3d_axis_negative_3_epsilon) +#if SKIP_SET_1 + SKIP_NON_CPU; +#endif +CASE(test_layer_normalization_4d_axis0) +#if SKIP_SET_1 + SKIP_NON_CPU; +#endif +CASE(test_layer_normalization_4d_axis1) +#if SKIP_SET_1 + SKIP_NON_CPU; +#endif +CASE(test_layer_normalization_4d_axis2) +#if SKIP_SET_1 + SKIP_NON_CPU; +#endif +CASE(test_layer_normalization_4d_axis3) +#if SKIP_SET_1 + SKIP_NON_CPU; +#endif +CASE(test_layer_normalization_4d_axis_negative_1) +#if SKIP_SET_1 + SKIP_NON_CPU; +#endif +CASE(test_layer_normalization_4d_axis_negative_2) +#if SKIP_SET_1 + SKIP_NON_CPU; +#endif +CASE(test_layer_normalization_4d_axis_negative_3) +#if SKIP_SET_1 + SKIP_NON_CPU; +#endif +CASE(test_layer_normalization_4d_axis_negative_4) +#if SKIP_SET_1 + SKIP_NON_CPU; +#endif +CASE(test_layer_normalization_default_axis) +#if SKIP_SET_1 + SKIP_NON_CPU; +#endif CASE(test_leakyrelu) // no filter CASE(test_leakyrelu_default) diff --git a/modules/dnn/test/test_onnx_importer.cpp b/modules/dnn/test/test_onnx_importer.cpp index 7be3e8a385..aa60cea890 100644 --- a/modules/dnn/test/test_onnx_importer.cpp +++ b/modules/dnn/test/test_onnx_importer.cpp @@ -2674,36 +2674,6 @@ TEST_P(Test_ONNX_layers, Tile) testONNXModels("tile", pb); } -TEST_P(Test_ONNX_layers, LayerNorm) -{ - testONNXModels("test_layer_normalization_2d_axis0", pb, 0, 0, false, true, 3); - testONNXModels("test_layer_normalization_2d_axis1", pb, 0, 0, false, true, 3); - testONNXModels("test_layer_normalization_2d_axis_negative_1", pb, 0, 0, false, true, 3); - testONNXModels("test_layer_normalization_2d_axis_negative_2", pb, 0, 0, false, true, 3); - testONNXModels("test_layer_normalization_3d_axis0_epsilon", pb, 0, 0, false, true, 3); - testONNXModels("test_layer_normalization_3d_axis1_epsilon", pb, 0, 0, false, true, 3); - testONNXModels("test_layer_normalization_3d_axis2_epsilon", pb, 0, 0, false, true, 3); - testONNXModels("test_layer_normalization_3d_axis_negative_1_epsilon", pb, 0, 0, false, true, 3); - testONNXModels("test_layer_normalization_3d_axis_negative_2_epsilon", pb, 0, 0, false, true, 3); - testONNXModels("test_layer_normalization_3d_axis_negative_3_epsilon", pb, 0, 0, false, true, 3); - testONNXModels("test_layer_normalization_4d_axis0", pb, 0, 0, false, true, 3); - testONNXModels("test_layer_normalization_4d_axis1", pb, 0, 0, false, true, 3); - testONNXModels("test_layer_normalization_4d_axis2", pb, 0, 0, false, true, 3); - testONNXModels("test_layer_normalization_4d_axis3", pb, 0, 0, false, true, 3); - testONNXModels("test_layer_normalization_4d_axis_negative_1", pb, 0, 0, false, true, 3); - testONNXModels("test_layer_normalization_4d_axis_negative_2", pb, 0, 0, false, true, 3); - testONNXModels("test_layer_normalization_4d_axis_negative_3", pb, 0, 0, false, true, 3); - testONNXModels("test_layer_normalization_4d_axis_negative_4", pb, 0, 0, false, true, 3); - testONNXModels("test_layer_normalization_default_axis", pb, 0, 0, false, true, 3); -} - -// for testing graph simplification -TEST_P(Test_ONNX_layers, LayerNormExpanded) -{ - testONNXModels("layer_norm_expanded"); - testONNXModels("layer_norm_expanded_with_initializers"); -} - TEST_P(Test_ONNX_layers, Gelu) { testONNXModels("gelu"); From b913e73d04ffd490a9180cd3c1f1e93555e2f903 Mon Sep 17 00:00:00 2001 From: zihaomu Date: Mon, 20 Nov 2023 18:45:37 +0800 Subject: [PATCH 037/156] DNN: add the Winograd fp16 support (#23654) * add Winograd FP16 implementation * fixed dispatching of FP16 code paths in dnn; use dynamic dispatcher only when NEON_FP16 is enabled in the build and the feature is present in the host CPU at runtime * fixed some warnings * hopefully fixed winograd on x64 (and maybe other platforms) --------- Co-authored-by: Vadim Pisarevsky --- cmake/OpenCVCompilerOptimizations.cmake | 2 +- cmake/checks/cpu_fp16.cpp | 10 +- .../core/include/opencv2/core/cv_cpu_helper.h | 42 ++ modules/core/test/ocl/test_image2d.cpp | 2 +- modules/dnn/CMakeLists.txt | 4 +- .../dnn/include/opencv2/dnn/all_layers.hpp | 2 +- modules/dnn/include/opencv2/dnn/dnn.hpp | 3 + .../layers/cpu_kernels/conv_block.simd.hpp | 257 ++++----- .../src/layers/cpu_kernels/conv_depthwise.cpp | 2 +- .../layers/cpu_kernels/conv_winograd_f63.cpp | 155 ++++-- .../cpu_kernels/conv_winograd_f63.neon.cpp | 476 ++++++++++++++++ .../cpu_kernels/conv_winograd_f63.simd.hpp | 511 ++++++++++-------- .../src/layers/cpu_kernels/convolution.cpp | 206 +++---- .../src/layers/cpu_kernels/convolution.hpp | 39 +- modules/dnn/src/model.cpp | 9 + modules/dnn/test/test_backends.cpp | 7 +- modules/dnn/test/test_caffe_importer.cpp | 21 +- modules/dnn/test/test_darknet_importer.cpp | 17 +- modules/dnn/test/test_model.cpp | 2 + modules/dnn/test/test_onnx_importer.cpp | 32 +- modules/dnn/test/test_tf_importer.cpp | 12 +- modules/dnn/test/test_torch_importer.cpp | 7 +- 22 files changed, 1262 insertions(+), 556 deletions(-) create mode 100644 modules/dnn/src/layers/cpu_kernels/conv_winograd_f63.neon.cpp diff --git a/cmake/OpenCVCompilerOptimizations.cmake b/cmake/OpenCVCompilerOptimizations.cmake index d45f327beb..98de62324e 100644 --- a/cmake/OpenCVCompilerOptimizations.cmake +++ b/cmake/OpenCVCompilerOptimizations.cmake @@ -49,7 +49,7 @@ set(CPU_ALL_OPTIMIZATIONS "SSE;SSE2;SSE3;SSSE3;SSE4_1;SSE4_2;POPCNT;AVX;FP16;AVX2;FMA3;AVX_512F") list(APPEND CPU_ALL_OPTIMIZATIONS "AVX512_COMMON;AVX512_KNL;AVX512_KNM;AVX512_SKX;AVX512_CNL;AVX512_CLX;AVX512_ICL") -list(APPEND CPU_ALL_OPTIMIZATIONS NEON VFPV3 FP16 NEON_DOTPROD) +list(APPEND CPU_ALL_OPTIMIZATIONS NEON VFPV3 FP16 NEON_DOTPROD NEON_FP16 NEON_BF16) list(APPEND CPU_ALL_OPTIMIZATIONS MSA) list(APPEND CPU_ALL_OPTIMIZATIONS VSX VSX3) list(APPEND CPU_ALL_OPTIMIZATIONS RVV) diff --git a/cmake/checks/cpu_fp16.cpp b/cmake/checks/cpu_fp16.cpp index f12cb10f4d..51e4e7160f 100644 --- a/cmake/checks/cpu_fp16.cpp +++ b/cmake/checks/cpu_fp16.cpp @@ -15,12 +15,12 @@ int test() #include "arm_neon.h" int test() { - const float src[] = { 0.0f, 0.0f, 0.0f, 0.0f }; - short dst[8]; - float32x4_t v_src = *(float32x4_t*)src; + const float src[] = { 0.0f, 1.0f, 2.0f, 3.0f }; + short dst[4]; + float32x4_t v_src = vld1q_f32(src); float16x4_t v_dst = vcvt_f16_f32(v_src); - *(float16x4_t*)dst = v_dst; - return (int)dst[0]; + vst1_f16((__fp16*)dst, v_dst); + return dst[0] + dst[1] + dst[2] + dst[3]; } #else #error "FP16 is not supported" diff --git a/modules/core/include/opencv2/core/cv_cpu_helper.h b/modules/core/include/opencv2/core/cv_cpu_helper.h index daca592e65..04b00d2024 100644 --- a/modules/core/include/opencv2/core/cv_cpu_helper.h +++ b/modules/core/include/opencv2/core/cv_cpu_helper.h @@ -441,6 +441,48 @@ #endif #define __CV_CPU_DISPATCH_CHAIN_NEON_DOTPROD(fn, args, mode, ...) CV_CPU_CALL_NEON_DOTPROD(fn, args); __CV_EXPAND(__CV_CPU_DISPATCH_CHAIN_ ## mode(fn, args, __VA_ARGS__)) +#if !defined CV_DISABLE_OPTIMIZATION && defined CV_ENABLE_INTRINSICS && defined CV_CPU_COMPILE_NEON_FP16 +# define CV_TRY_NEON_FP16 1 +# define CV_CPU_FORCE_NEON_FP16 1 +# define CV_CPU_HAS_SUPPORT_NEON_FP16 1 +# define CV_CPU_CALL_NEON_FP16(fn, args) return (cpu_baseline::fn args) +# define CV_CPU_CALL_NEON_FP16_(fn, args) return (opt_NEON_FP16::fn args) +#elif !defined CV_DISABLE_OPTIMIZATION && defined CV_ENABLE_INTRINSICS && defined CV_CPU_DISPATCH_COMPILE_NEON_FP16 +# define CV_TRY_NEON_FP16 1 +# define CV_CPU_FORCE_NEON_FP16 0 +# define CV_CPU_HAS_SUPPORT_NEON_FP16 (cv::checkHardwareSupport(CV_CPU_NEON_FP16)) +# define CV_CPU_CALL_NEON_FP16(fn, args) if (CV_CPU_HAS_SUPPORT_NEON_FP16) return (opt_NEON_FP16::fn args) +# define CV_CPU_CALL_NEON_FP16_(fn, args) if (CV_CPU_HAS_SUPPORT_NEON_FP16) return (opt_NEON_FP16::fn args) +#else +# define CV_TRY_NEON_FP16 0 +# define CV_CPU_FORCE_NEON_FP16 0 +# define CV_CPU_HAS_SUPPORT_NEON_FP16 0 +# define CV_CPU_CALL_NEON_FP16(fn, args) +# define CV_CPU_CALL_NEON_FP16_(fn, args) +#endif +#define __CV_CPU_DISPATCH_CHAIN_NEON_FP16(fn, args, mode, ...) CV_CPU_CALL_NEON_FP16(fn, args); __CV_EXPAND(__CV_CPU_DISPATCH_CHAIN_ ## mode(fn, args, __VA_ARGS__)) + +#if !defined CV_DISABLE_OPTIMIZATION && defined CV_ENABLE_INTRINSICS && defined CV_CPU_COMPILE_NEON_BF16 +# define CV_TRY_NEON_BF16 1 +# define CV_CPU_FORCE_NEON_BF16 1 +# define CV_CPU_HAS_SUPPORT_NEON_BF16 1 +# define CV_CPU_CALL_NEON_BF16(fn, args) return (cpu_baseline::fn args) +# define CV_CPU_CALL_NEON_BF16_(fn, args) return (opt_NEON_BF16::fn args) +#elif !defined CV_DISABLE_OPTIMIZATION && defined CV_ENABLE_INTRINSICS && defined CV_CPU_DISPATCH_COMPILE_NEON_BF16 +# define CV_TRY_NEON_BF16 1 +# define CV_CPU_FORCE_NEON_BF16 0 +# define CV_CPU_HAS_SUPPORT_NEON_BF16 (cv::checkHardwareSupport(CV_CPU_NEON_BF16)) +# define CV_CPU_CALL_NEON_BF16(fn, args) if (CV_CPU_HAS_SUPPORT_NEON_BF16) return (opt_NEON_BF16::fn args) +# define CV_CPU_CALL_NEON_BF16_(fn, args) if (CV_CPU_HAS_SUPPORT_NEON_BF16) return (opt_NEON_BF16::fn args) +#else +# define CV_TRY_NEON_BF16 0 +# define CV_CPU_FORCE_NEON_BF16 0 +# define CV_CPU_HAS_SUPPORT_NEON_BF16 0 +# define CV_CPU_CALL_NEON_BF16(fn, args) +# define CV_CPU_CALL_NEON_BF16_(fn, args) +#endif +#define __CV_CPU_DISPATCH_CHAIN_NEON_BF16(fn, args, mode, ...) CV_CPU_CALL_NEON_BF16(fn, args); __CV_EXPAND(__CV_CPU_DISPATCH_CHAIN_ ## mode(fn, args, __VA_ARGS__)) + #if !defined CV_DISABLE_OPTIMIZATION && defined CV_ENABLE_INTRINSICS && defined CV_CPU_COMPILE_MSA # define CV_TRY_MSA 1 # define CV_CPU_FORCE_MSA 1 diff --git a/modules/core/test/ocl/test_image2d.cpp b/modules/core/test/ocl/test_image2d.cpp index 86561bddcf..07652e5c9d 100644 --- a/modules/core/test/ocl/test_image2d.cpp +++ b/modules/core/test/ocl/test_image2d.cpp @@ -83,7 +83,7 @@ TEST(Image2D, turnOffOpenCL) } else std::cout << "CV_8UC1 is not supported for OpenCL images. Test skipped." << std::endl; - + // reset state to the previous one cv::ocl::setUseOpenCL(useOCL); } diff --git a/modules/dnn/CMakeLists.txt b/modules/dnn/CMakeLists.txt index f45ac68a28..d6fd52f9ab 100644 --- a/modules/dnn/CMakeLists.txt +++ b/modules/dnn/CMakeLists.txt @@ -6,9 +6,9 @@ set(the_description "Deep neural network module. It allows to load models from d ocv_add_dispatched_file_force_all("layers/layers_common" AVX AVX2 AVX512_SKX RVV LASX) ocv_add_dispatched_file_force_all("int8layers/layers_common" AVX2 AVX512_SKX LASX) -ocv_add_dispatched_file_force_all("layers/cpu_kernels/conv_block" AVX AVX2) +ocv_add_dispatched_file_force_all("layers/cpu_kernels/conv_block" AVX AVX2 NEON NEON_FP16) ocv_add_dispatched_file_force_all("layers/cpu_kernels/conv_depthwise" AVX AVX2 RVV LASX) -ocv_add_dispatched_file_force_all("layers/cpu_kernels/conv_winograd_f63" AVX AVX2) +ocv_add_dispatched_file_force_all("layers/cpu_kernels/conv_winograd_f63" AVX AVX2 NEON_FP16) ocv_add_dispatched_file_force_all("layers/cpu_kernels/fast_gemm_kernels" AVX AVX2 NEON LASX) ocv_add_module(dnn opencv_core opencv_imgproc WRAP python java objc js) diff --git a/modules/dnn/include/opencv2/dnn/all_layers.hpp b/modules/dnn/include/opencv2/dnn/all_layers.hpp index 8154064e03..1d3dd3bc4a 100644 --- a/modules/dnn/include/opencv2/dnn/all_layers.hpp +++ b/modules/dnn/include/opencv2/dnn/all_layers.hpp @@ -303,7 +303,7 @@ CV__DNN_INLINE_NS_BEGIN // quantization type flag. The perChannel default is true, that means it contains the parameters // of per-Channel quantization. Otherwise, that means this layer contains per-Tensor quantized parameters. bool per_channel; - bool useWinograd = true; // Flag whether to use Winograd to speed up 3x3 convolution. + bool useWinograd = false; // Flag whether to use Winograd to speed up 3x3 convolution. static Ptr create(const LayerParams& params); }; diff --git a/modules/dnn/include/opencv2/dnn/dnn.hpp b/modules/dnn/include/opencv2/dnn/dnn.hpp index 02a76d403b..bd714c8e46 100644 --- a/modules/dnn/include/opencv2/dnn/dnn.hpp +++ b/modules/dnn/include/opencv2/dnn/dnn.hpp @@ -1458,6 +1458,9 @@ CV__DNN_INLINE_NS_BEGIN /// @sa Net::setPreferableTarget CV_WRAP Model& setPreferableTarget(dnn::Target targetId); + /// @sa Net::enableWinograd + CV_WRAP Model& enableWinograd(bool useWinograd); + CV_DEPRECATED_EXTERNAL operator Net&() const { return getNetwork_(); } diff --git a/modules/dnn/src/layers/cpu_kernels/conv_block.simd.hpp b/modules/dnn/src/layers/cpu_kernels/conv_block.simd.hpp index 3310a91b6b..22d7d5194a 100644 --- a/modules/dnn/src/layers/cpu_kernels/conv_block.simd.hpp +++ b/modules/dnn/src/layers/cpu_kernels/conv_block.simd.hpp @@ -8,16 +8,26 @@ namespace cv { namespace dnn { CV_CPU_OPTIMIZATION_NAMESPACE_BEGIN -void convBlock(int np, const float* a, const float* b, float* c, int ldc, bool init_c, int width, const int convMR, const int convNR); +void convBlock_F32(int np, const float* a, const float* b, float* c, int ldc, bool init_c, int width, const int convMR, const int convNR); -#if !defined(CV_CPU_OPTIMIZATION_DECLARATIONS_ONLY) && CV_AVX + +// FP 16 branch. +void convBlock_F16(int np, const char * _a, const char * _b, char * _c, int ldc, bool init_c, int width, + const int convMR_fp16, const int convNR_fp16); + +void convBlockMR1_F16(int np, const char* _a, const char* _b, float *c, const float _bias, bool init_c, + const float minval, const float maxval, bool ifMinMaxAct, const int width, const int convNR_FP16); + +#if !defined(CV_CPU_OPTIMIZATION_DECLARATIONS_ONLY) + +#if CV_AVX #if !CV_FMA3 // AVX workaround #undef _mm256_fmadd_ps #define _mm256_fmadd_ps(a, b, c) _mm256_add_ps(c, _mm256_mul_ps(a, b)) #endif -void convBlock(int np, const float* a, const float* b, float* c, int ldc, bool init_c, int width, const int convMR, const int convNR) +void convBlock_F32(int np, const float* a, const float* b, float* c, int ldc, bool init_c, int width, const int convMR, const int convNR) { CV_Assert(convMR == 4 && convNR == 24); __m256 c00 = _mm256_set1_ps(0.f), c01 = c00, c02 = c00; @@ -121,16 +131,11 @@ void convBlock(int np, const float* a, const float* b, float* c, int ldc, bool i _mm256_zeroupper(); } -#endif // CV_CPU_OPTIMIZATION_DECLARATIONS_ONLY +#endif -CV_CPU_OPTIMIZATION_NAMESPACE_END +#if CV_NEON -// NEON code work around. -namespace opt_NEON -{ -#if !defined(CV_CPU_OPTIMIZATION_DECLARATIONS_ONLY) && CV_NEON - -void convBlock(int np, const float* a, const float* b, float* c, int ldc, bool init_c, int width, const int convMR, const int convNR) +void convBlock_F32(int np, const float* a, const float* b, float* c, int ldc, bool init_c, int width, const int convMR, const int convNR) { #if CV_NEON_AARCH64 if (convMR == 4 && convNR == 28) // AARCH64 @@ -298,104 +303,104 @@ void convBlock(int np, const float* a, const float* b, float* c, int ldc, bool i } else #endif - if (convMR == 4 && convNR == 12) // ARMv7 - { - float32x4_t c0 = vdupq_n_f32(0.f), c1 = c0, c2 = c0; - float32x4_t c3 = vdupq_n_f32(0.f), c4 = c3, c5 = c3; - float32x4_t c6 = vdupq_n_f32(0.f), c7 = c6, c8 = c6; - float32x4_t c9 = vdupq_n_f32(0.f), c10 = c9, c11 = c9; - - float32x2_t a0 = vdup_n_f32(0.0f), a1 = a0; - float32x4_t b0 = vdupq_n_f32(0.0f), b1 = vdupq_n_f32(0.0f), b2 = vdupq_n_f32(0.0f); - - if (width > 8) + if (convMR == 4 && convNR == 12) // ARMv7 { - for (int p = 0; p < np; p++, a += convMR, b += convNR) + float32x4_t c0 = vdupq_n_f32(0.f), c1 = c0, c2 = c0; + float32x4_t c3 = vdupq_n_f32(0.f), c4 = c3, c5 = c3; + float32x4_t c6 = vdupq_n_f32(0.f), c7 = c6, c8 = c6; + float32x4_t c9 = vdupq_n_f32(0.f), c10 = c9, c11 = c9; + + float32x2_t a0 = vdup_n_f32(0.0f), a1 = a0; + float32x4_t b0 = vdupq_n_f32(0.0f), b1 = vdupq_n_f32(0.0f), b2 = vdupq_n_f32(0.0f); + + if (width > 8) { - a0 = vld1_f32(a), a1 = vld1_f32(a+2); - b0 = vld1q_f32(b), b1 = vld1q_f32(b + 4), b2 = vld1q_f32(b + 8); + for (int p = 0; p < np; p++, a += convMR, b += convNR) + { + a0 = vld1_f32(a), a1 = vld1_f32(a+2); + b0 = vld1q_f32(b), b1 = vld1q_f32(b + 4), b2 = vld1q_f32(b + 8); - c0 = vmlaq_lane_f32(c0, b0, a0, 0); - c1 = vmlaq_lane_f32(c1, b1, a0, 0); - c2 = vmlaq_lane_f32(c2, b2, a0, 0); + c0 = vmlaq_lane_f32(c0, b0, a0, 0); + c1 = vmlaq_lane_f32(c1, b1, a0, 0); + c2 = vmlaq_lane_f32(c2, b2, a0, 0); - c3 = vmlaq_lane_f32(c3, b0, a0, 1); - c4 = vmlaq_lane_f32(c4, b1, a0, 1); - c5 = vmlaq_lane_f32(c5, b2, a0, 1); + c3 = vmlaq_lane_f32(c3, b0, a0, 1); + c4 = vmlaq_lane_f32(c4, b1, a0, 1); + c5 = vmlaq_lane_f32(c5, b2, a0, 1); - c6 = vmlaq_lane_f32(c6, b0, a1, 0); - c7 = vmlaq_lane_f32(c7, b1, a1, 0); - c8 = vmlaq_lane_f32(c8, b2, a1, 0); + c6 = vmlaq_lane_f32(c6, b0, a1, 0); + c7 = vmlaq_lane_f32(c7, b1, a1, 0); + c8 = vmlaq_lane_f32(c8, b2, a1, 0); - c9 = vmlaq_lane_f32(c9 , b0, a1, 1); - c10 = vmlaq_lane_f32(c10, b1, a1, 1); - c11 = vmlaq_lane_f32(c11, b2, a1, 1); + c9 = vmlaq_lane_f32(c9 , b0, a1, 1); + c10 = vmlaq_lane_f32(c10, b1, a1, 1); + c11 = vmlaq_lane_f32(c11, b2, a1, 1); + } } - } - else if (width > 4) - { - for (int p = 0; p < np; p++, a += convMR, b += convNR) + else if (width > 4) { - a0 = vld1_f32(a), a1 = vld1_f32(a+2); - b0 = vld1q_f32(b), b1 = vld1q_f32(b + 4); + for (int p = 0; p < np; p++, a += convMR, b += convNR) + { + a0 = vld1_f32(a), a1 = vld1_f32(a+2); + b0 = vld1q_f32(b), b1 = vld1q_f32(b + 4); - c0 = vmlaq_lane_f32(c0, b0, a0, 0); - c1 = vmlaq_lane_f32(c1, b1, a0, 0); + c0 = vmlaq_lane_f32(c0, b0, a0, 0); + c1 = vmlaq_lane_f32(c1, b1, a0, 0); - c3 = vmlaq_lane_f32(c3, b0, a0, 1); - c4 = vmlaq_lane_f32(c4, b1, a0, 1); + c3 = vmlaq_lane_f32(c3, b0, a0, 1); + c4 = vmlaq_lane_f32(c4, b1, a0, 1); - c6 = vmlaq_lane_f32(c6, b0, a1, 0); - c7 = vmlaq_lane_f32(c7, b1, a1, 0); + c6 = vmlaq_lane_f32(c6, b0, a1, 0); + c7 = vmlaq_lane_f32(c7, b1, a1, 0); - c9 = vmlaq_lane_f32(c9 , b0, a1, 1); - c10 = vmlaq_lane_f32(c10, b1, a1, 1); + c9 = vmlaq_lane_f32(c9 , b0, a1, 1); + c10 = vmlaq_lane_f32(c10, b1, a1, 1); + } } + else + { + for (int p = 0; p < np; p++, a += convMR, b += convNR) + { + a0 = vld1_f32(a), a1 = vld1_f32(a+2); + b0 = vld1q_f32(b); + + c0 = vmlaq_lane_f32(c0, b0, a0, 0); + c3 = vmlaq_lane_f32(c3, b0, a0, 1); + c6 = vmlaq_lane_f32(c6, b0, a1, 0); + c9 = vmlaq_lane_f32(c9 , b0, a1, 1); + } + } + + if (!init_c) + { + c0 = vaddq_f32(c0, vld1q_f32(c)); + c1 = vaddq_f32(c1, vld1q_f32(c + 4)); + c2 = vaddq_f32(c2, vld1q_f32(c + 8)); + + c3 = vaddq_f32(c3, vld1q_f32(c + ldc)); + c4 = vaddq_f32(c4, vld1q_f32(c + ldc + 4)); + c5 = vaddq_f32(c5, vld1q_f32(c + ldc + 8)); + + c6 = vaddq_f32(c6, vld1q_f32(c + ldc * 2)); + c7 = vaddq_f32(c7, vld1q_f32(c + ldc * 2 + 4)); + c8 = vaddq_f32(c8, vld1q_f32(c + ldc * 2 + 8)); + + c9 = vaddq_f32(c9 , vld1q_f32(c + ldc * 3)); + c10 = vaddq_f32(c10, vld1q_f32(c + ldc * 3 + 4)); + c11 = vaddq_f32(c11, vld1q_f32(c + ldc * 3 + 8)); + } + + vst1q_f32(c, c0), vst1q_f32(c+4, c1), vst1q_f32(c+8, c2); + vst1q_f32(c + ldc, c3), vst1q_f32(c + ldc + 4, c4), vst1q_f32(c + ldc + 8, c5); + vst1q_f32(c + ldc*2, c6), vst1q_f32(c + ldc*2 + 4, c7), vst1q_f32(c + ldc*2 + 8, c8); + vst1q_f32(c + ldc*3, c9), vst1q_f32(c + ldc*3 + 4, c10), vst1q_f32(c + ldc*3 + 8, c11); } else - { - for (int p = 0; p < np; p++, a += convMR, b += convNR) - { - a0 = vld1_f32(a), a1 = vld1_f32(a+2); - b0 = vld1q_f32(b); - - c0 = vmlaq_lane_f32(c0, b0, a0, 0); - c3 = vmlaq_lane_f32(c3, b0, a0, 1); - c6 = vmlaq_lane_f32(c6, b0, a1, 0); - c9 = vmlaq_lane_f32(c9 , b0, a1, 1); - } - } - - if (!init_c) - { - c0 = vaddq_f32(c0, vld1q_f32(c)); - c1 = vaddq_f32(c1, vld1q_f32(c + 4)); - c2 = vaddq_f32(c2, vld1q_f32(c + 8)); - - c3 = vaddq_f32(c3, vld1q_f32(c + ldc)); - c4 = vaddq_f32(c4, vld1q_f32(c + ldc + 4)); - c5 = vaddq_f32(c5, vld1q_f32(c + ldc + 8)); - - c6 = vaddq_f32(c6, vld1q_f32(c + ldc * 2)); - c7 = vaddq_f32(c7, vld1q_f32(c + ldc * 2 + 4)); - c8 = vaddq_f32(c8, vld1q_f32(c + ldc * 2 + 8)); - - c9 = vaddq_f32(c9 , vld1q_f32(c + ldc * 3)); - c10 = vaddq_f32(c10, vld1q_f32(c + ldc * 3 + 4)); - c11 = vaddq_f32(c11, vld1q_f32(c + ldc * 3 + 8)); - } - - vst1q_f32(c, c0), vst1q_f32(c+4, c1), vst1q_f32(c+8, c2); - vst1q_f32(c + ldc, c3), vst1q_f32(c + ldc + 4, c4), vst1q_f32(c + ldc + 8, c5); - vst1q_f32(c + ldc*2, c6), vst1q_f32(c + ldc*2 + 4, c7), vst1q_f32(c + ldc*2 + 8, c8); - vst1q_f32(c + ldc*3, c9), vst1q_f32(c + ldc*3 + 4, c10), vst1q_f32(c + ldc*3 + 8, c11); - } - else - CV_Error(Error::StsNotImplemented, "Unsupported convMR and/or convNR in opt_NEON::convBlock"); + CV_Error(Error::StsNotImplemented, "Unsupported convMR and/or convNR in opt_NEON::convBlock"); } void convBlockMR1_F32(int np, const float * a, const float * b, float *c, const float bias, bool init_c, - const float minval, const float maxval, bool ifMinMaxAct, const int width, const int convNR) + const float minval, const float maxval, bool ifMinMaxAct, const int width, const int convNR) { CV_Assert(convNR == 28); float32x4_t c0 = vdupq_n_f32(bias), c1 = c0, c2 = c0; @@ -482,22 +487,17 @@ void convBlockMR1_F32(int np, const float * a, const float * b, float *c, const vst1q_f32(c + 20, c5); vst1q_f32(c + 24, c6); } - -#if CV_NEON_AARCH64 && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) -// Fix conflict between float16_t in arm_neon.h and float16_t in cvdef.h. -typedef __fp16 float16_t; - -#ifndef __ARM_FEATURE_FMA // Work around without FMA support. -#define vfmaq_f16(a, b, c) (a + b * c) #endif -void convBlock_FP16(int np, const char * _a, const char * _b, char * _c, int ldc, bool init_c, int width, + +#if defined(CV_NEON_AARCH64) && CV_NEON_AARCH64 && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) + +void convBlock_F16(int np, const char * _a, const char * _b, char * _c, int ldc, bool init_c, int width, const int convMR_fp16, const int convNR_fp16) { -#if 1 + typedef __fp16 float16_t; const float16_t* a = (const float16_t*)_a; const float16_t* b = (const float16_t*)_b; float16_t* c = (float16_t*)_c; - CV_Assert(convMR_fp16 == 8 && convNR_fp16 == 24); float16x8_t c00 = vdupq_n_f16(0), c01 = c00, c02 = c00; @@ -603,8 +603,8 @@ void convBlock_FP16(int np, const char * _a, const char * _b, char * _c, int ldc if (!init_c) { -#undef _FX_UPDATE_CBUF_ROW -#define _FX_UPDATE_CBUF_ROW(row) \ + #undef _FX_UPDATE_CBUF_ROW + #define _FX_UPDATE_CBUF_ROW(row) \ c##row##0 = c##row##0 + vld1q_f16(c + row*ldc); \ c##row##1 = c##row##1 + vld1q_f16(c + row*ldc + 8); \ c##row##2 = c##row##2 + vld1q_f16(c + row*ldc + 16) @@ -619,8 +619,8 @@ void convBlock_FP16(int np, const char * _a, const char * _b, char * _c, int ldc _FX_UPDATE_CBUF_ROW(7); } -#undef _FX_STORE_CBUF_ROW -#define _FX_STORE_CBUF_ROW(row) \ + #undef _FX_STORE_CBUF_ROW + #define _FX_STORE_CBUF_ROW(row) \ vst1q_f16(c + row*ldc, c##row##0); \ vst1q_f16(c + row*ldc + 8, c##row##1); \ vst1q_f16(c + row*ldc + 16, c##row##2) @@ -633,46 +633,12 @@ void convBlock_FP16(int np, const char * _a, const char * _b, char * _c, int ldc _FX_STORE_CBUF_ROW(5); _FX_STORE_CBUF_ROW(6); _FX_STORE_CBUF_ROW(7); -#else - // reference only. - const float16_t* a = (const float16_t*)_a; - const float16_t* b = (const float16_t*)_b; - float16_t* c = (float16_t*)_c; - float cbuf[convMR_fp16*convNR_fp16]; - memset(cbuf, 0, sizeof(cbuf)); - - for( int p = 0; p < np; p++ ) - { - for( int i = 0; i < convMR_fp16; i++ ) - { - float ai = float(a[convMR_fp16*p + i]); - for( int j = 0; j < convNR_fp16; j++ ) - cbuf[i*convNR_fp16+j] += float(b[convNR_fp16*p + j]) * ai; - } - } - - if (!init_c) - { - for(int i = 0; i < convMR_fp16; i++) - { - for(int j = 0; j < convNR_fp16; j++) - c[i*ldc + j] = float16_t(float(c[i*ldc + j]) + cbuf[i*convNR_fp16 + j]); - } - } - else - { - for(int i = 0; i < convMR_fp16; i++) - { - for(int j = 0; j < convNR_fp16; j++) - c[i*ldc + j] = (float16_t)(cbuf[i*convNR_fp16 + j]); - } - } -#endif } -void convBlockMR1_FP16(int np, const char* _a, const char* _b, float *c, const float _bias, bool init_c, - const float minval, const float maxval, bool ifMinMaxAct, const int width, const int convNR_FP16) +void convBlockMR1_F16(int np, const char* _a, const char* _b, float *c, const float _bias, bool init_c, + const float minval, const float maxval, bool ifMinMaxAct, const int width, const int convNR_FP16) { + typedef __fp16 float16_t; CV_Assert(convNR_FP16 == 24); // CONV_NR_FP16 = 24 const float16_t* a = (const float16_t*)_a; const float16_t* b = (const float16_t*)_b; @@ -685,7 +651,7 @@ void convBlockMR1_FP16(int np, const char* _a, const char* _b, float *c, const f { for (int p = 0; p < np; p++, a++, b += convNR_FP16) { - float16x8_t a0= vdupq_n_f16(a[0]); + float16x8_t a0 = vdupq_n_f16(a[0]); float16x8_t b0 = vld1q_f16(b), b1 = vld1q_f16(b + 8), b2 = vld1q_f16(b + 16); c0 = vfmaq_f16(c0, a0, b0); @@ -754,6 +720,7 @@ void convBlockMR1_FP16(int np, const char* _a, const char* _b, float *c, const f } #endif -#endif -} +#endif // CV_CPU_OPTIMIZATION_DECLARATIONS_ONLY + +CV_CPU_OPTIMIZATION_NAMESPACE_END }} // namespace cv::dnn diff --git a/modules/dnn/src/layers/cpu_kernels/conv_depthwise.cpp b/modules/dnn/src/layers/cpu_kernels/conv_depthwise.cpp index 59f069eeaa..8c1c643abe 100644 --- a/modules/dnn/src/layers/cpu_kernels/conv_depthwise.cpp +++ b/modules/dnn/src/layers/cpu_kernels/conv_depthwise.cpp @@ -92,7 +92,7 @@ void runDepthwise(InputArray _input, OutputArray _output, const Ptr& c ofstab[k] = dy * Wi + dx; } - const float *weights0 = conv->weightsBufPtr, *bias = conv->biasBuf.data(); + const float *weights0 = conv->getWeights(), *bias = conv->biasBuf.data(); const float* relu = reluslope.data(); CV_Assert(ksize > 1 || (pad_left == 0 && pad_right == 0 && pad_top == 0 && pad_bottom == 0)); diff --git a/modules/dnn/src/layers/cpu_kernels/conv_winograd_f63.cpp b/modules/dnn/src/layers/cpu_kernels/conv_winograd_f63.cpp index 605cf37949..6cf066576b 100644 --- a/modules/dnn/src/layers/cpu_kernels/conv_winograd_f63.cpp +++ b/modules/dnn/src/layers/cpu_kernels/conv_winograd_f63.cpp @@ -20,15 +20,15 @@ namespace cv { namespace dnn { #if CV_NEON || CV_SIMD128 || CV_TRY_AVX2 enum { VEC_ALIGN = 32, DFT_TYPE = CV_32F }; // Memory alignment. -void winofunc_accum_f32(const float* inwptr, const float* wptr, float* outbuf, int Cg, int iblock, +void winofunc_accum_F32(const float* inwptr, const float* wptr, float* outbuf, int Cg, int iblock, const int winoIblock, const int winoKblock, const int winoAtomF32, const int winoNatomF32); /*Input transform*/ -void winofunc_BtXB_8x8_f32(const float* inptr, int inpstep, +void winofunc_BtXB_8x8_F32(const float* inptr, int inpstep, float* outptr, int Cg, const int winoIblock, const int winoAtomF32); /*Output transform*/ -void winofunc_AtXA_8x8_f32(const float* inptr, int inpstep, float* bpptr, int bpstep, float* outptr, int outstep, +void winofunc_AtXA_8x8_F32(const float* inptr, int inpstep, float* bpptr, int bpstep, float* outptr, int outstep, float bias, float minval, float maxval, bool ifMinMaxAct); int runWinograd63(InputArray _input, InputArray _fusedAddMat, OutputArray _output, const Ptr& conv, @@ -67,6 +67,28 @@ int runWinograd63(InputArray _input, InputArray _fusedAddMat, OutputArray _outpu #endif const int CONV_WINO_NATOMS_F32 = CONV_WINO_AREA / CONV_WINO_ATOM_F32; // for AVX2, it is 8, otherwise, it's 16. + int CONV_WINO_ATOM = CONV_WINO_ATOM_F32; + int CONV_WINO_NATOMS = CONV_WINO_NATOMS_F32; + +#ifdef CONV_ARM_FP16 + // FP 16 + const int CONV_WINO_ATOM_F16 = CONV_WINO_ATOM_F32 * 2; + const int CONV_WINO_NATOMS_F16 = CONV_WINO_AREA / CONV_WINO_ATOM_F16; +#endif + + int esz = sizeof(float ); + +#ifdef CONV_ARM_FP16 + const bool useFP16 = conv->useFP16; + if (useFP16) + { + // works at FP 16. + CONV_WINO_ATOM = CONV_WINO_ATOM_F16; + CONV_WINO_NATOMS = CONV_WINO_NATOMS_F16; + esz = sizeof(float16_t); + } +#endif + int Kg_nblocks = (Kg + CONV_WINO_KBLOCK - 1)/CONV_WINO_KBLOCK; const size_t inp_planesize = (size_t)Hi*Wi; const size_t out_planesize = (size_t)H0*W0; @@ -78,9 +100,9 @@ int runWinograd63(InputArray _input, InputArray _fusedAddMat, OutputArray _outpu size_t totalbufsize = (size_t)N*C*blocks_per_plane_aligned*CONV_WINO_AREA; - AutoBuffer _buf; - _buf.allocate(totalbufsize + VEC_ALIGN); - float* wbuf_all = alignPtr(_buf.data(), VEC_ALIGN); + AutoBuffer _buf; + _buf.allocate((totalbufsize + VEC_ALIGN) * esz); + char* wbuf_all = alignPtr(_buf.data(), VEC_ALIGN * esz); float* inp = input.ptr(); float* out = output.ptr(); @@ -104,14 +126,15 @@ int runWinograd63(InputArray _input, InputArray _fusedAddMat, OutputArray _outpu int c = nc0 - n*C; int g = c / Cg; c -= g*Cg; + for (int block_id = 0; block_id < blocks_per_plane; block_id += CONV_WINO_IBLOCK) { for (int db = 0; db < CONV_WINO_IBLOCK; db++) { size_t inwofs = ((n*ngroups + g)*blocks_per_plane_aligned + block_id)*Cg*CONV_WINO_AREA + - (c*CONV_WINO_IBLOCK + db)*CONV_WINO_ATOM_F32; - float* inwptr = (float*)wbuf_all + inwofs; + (c*CONV_WINO_IBLOCK + db) * CONV_WINO_ATOM; + char* inwptr = wbuf_all + inwofs * esz; if (block_id + db < blocks_per_plane) { @@ -152,27 +175,40 @@ int runWinograd63(InputArray _input, InputArray _fusedAddMat, OutputArray _outpu inptr = inpbuf; inpstep = CONV_WINO_SIZE; } + #if CV_TRY_AVX2 if (conv->useAVX2) - opt_AVX2::winofunc_BtXB_8x8_f32(inptr, inpstep, inwptr, Cg, CONV_WINO_IBLOCK, CONV_WINO_ATOM_F32); + opt_AVX2::winofunc_BtXB_8x8_F32(inptr, inpstep, (float *)inwptr, Cg, CONV_WINO_IBLOCK, CONV_WINO_ATOM); else #endif #if CV_TRY_AVX if (conv->useAVX) - opt_AVX::winofunc_BtXB_8x8_f32(inptr, inpstep, inwptr, Cg, CONV_WINO_IBLOCK, CONV_WINO_ATOM_F32); + opt_AVX::winofunc_BtXB_8x8_F32(inptr, inpstep, (float *)inwptr, Cg, CONV_WINO_IBLOCK, CONV_WINO_ATOM); else #endif #if CV_NEON && CV_NEON_AARCH64 if (conv->useNEON) - opt_NEON::winofunc_BtXB_8x8_f32(inptr, inpstep, inwptr, Cg, CONV_WINO_IBLOCK, CONV_WINO_ATOM_F32); + { +#ifdef CONV_ARM_FP16 + if (useFP16) + { + opt_NEON_FP16::winofunc_BtXB_8x8_F16(inptr, inpstep, inwptr, Cg, CONV_WINO_IBLOCK, + CONV_WINO_ATOM); + } + else +#endif + opt_NEON::winofunc_BtXB_8x8_F32(inptr, inpstep, (float *)inwptr, Cg, CONV_WINO_IBLOCK, + CONV_WINO_ATOM); + } else #endif - winofunc_BtXB_8x8_f32(inptr, inpstep, inwptr, Cg, CONV_WINO_IBLOCK, CONV_WINO_ATOM_F32); + winofunc_BtXB_8x8_F32(inptr, inpstep, (float *)inwptr, Cg, CONV_WINO_IBLOCK, CONV_WINO_ATOM); + } else { - for (int i = 0; i < CONV_WINO_NATOMS_F32; i++, inwptr += CONV_WINO_IBLOCK*CONV_WINO_ATOM_F32) - memset(inwptr, 0, CONV_WINO_ATOM_F32*sizeof(inwptr[0])); + for (int i = 0; i < CONV_WINO_NATOMS; i++, inwptr += CONV_WINO_IBLOCK * CONV_WINO_ATOM * esz) + memset(inwptr, 0, CONV_WINO_ATOM * esz); } } } @@ -182,19 +218,37 @@ int runWinograd63(InputArray _input, InputArray _fusedAddMat, OutputArray _outpu // Phase 2. compute elemwise-weighted sums of transformed blocks, // apply inverse Winograd transforms to the sums, // add bias, apply activation function if any and store the results. + char* wptr0 = nullptr; +#ifdef CONV_ARM_FP16 + if (useFP16) + { + CV_Assert(!conv->weightsWinoBuf_FP16.empty()); + wptr0 = (char *)conv->getWeightsWinoFP16(); + } + else +#endif + { + CV_Assert(!conv->weightsWinoBuf.empty()); + wptr0 = (char *)conv->getWeightsWino(); + } + parallel_for_(Range(0, ntasks), [&](const Range& r0) { for (int task_id = r0.start; task_id < r0.end; task_id++) { - size_t out_wbuf_size = CONV_WINO_AREA*CONV_WINO_KBLOCK*CONV_WINO_IBLOCK; + size_t out_wbuf_size = CONV_WINO_AREA * CONV_WINO_KBLOCK * CONV_WINO_IBLOCK; size_t outbuf_size = CONV_WINO_AREA; - AutoBuffer out_wbuf_, outbuf_; - out_wbuf_.allocate(out_wbuf_size + VEC_ALIGN); - float* out_wbuf = alignPtr(out_wbuf_.data(), VEC_ALIGN); + + // For saving the accumulation output. + AutoBuffer out_wbuf_; + out_wbuf_.allocate((out_wbuf_size + VEC_ALIGN) * esz); + char* out_wbuf = alignPtr(out_wbuf_.data(), VEC_ALIGN * esz); + memset(out_wbuf, 0, out_wbuf_size * esz); + + // For saving the fuse_Add data. + AutoBuffer outbuf_; outbuf_.allocate(outbuf_size + VEC_ALIGN); float* outbuf = alignPtr(outbuf_.data(), VEC_ALIGN); - - memset(out_wbuf, 0, out_wbuf_size * sizeof(float)); - memset(outbuf, 0, outbuf_size * sizeof(float)); + memset(outbuf, 0, outbuf_size * sizeof(outbuf[0])); int ngk0 = (int)(((int64_t)N*Kg_nblocks*ngroups)*task_id/ntasks); int ngk1 = (int)(((int64_t)N*Kg_nblocks*ngroups)*(task_id+1)/ntasks); @@ -214,30 +268,40 @@ int runWinograd63(InputArray _input, InputArray _fusedAddMat, OutputArray _outpu size_t inwofs = ((n*ngroups + g)*blocks_per_plane_aligned + block_id0)*Cg*CONV_WINO_AREA; size_t wofs = (g*Kg_nblocks*CONV_WINO_KBLOCK + k0)*Cg*CONV_WINO_AREA; - float* inwptr = wbuf_all + inwofs; - const float* wptr = conv->weightsWinoBufPtr + wofs; + char* inwptr = wbuf_all + inwofs * esz; + char* wptr = wptr0 + wofs * esz; #if CV_TRY_AVX2 if (conv->useAVX2) - opt_AVX2::winofunc_accum_f32(inwptr, wptr, out_wbuf, Cg, block_id1 - block_id0, CONV_WINO_IBLOCK, - CONV_WINO_KBLOCK, CONV_WINO_ATOM_F32, CONV_WINO_NATOMS_F32); + opt_AVX2::winofunc_accum_F32((float *)inwptr, (float *)wptr, (float *)out_wbuf, Cg, block_id1 - block_id0, CONV_WINO_IBLOCK, + CONV_WINO_KBLOCK, CONV_WINO_ATOM, CONV_WINO_NATOMS); else #endif #if CV_TRY_AVX if (conv->useAVX) - opt_AVX::winofunc_accum_f32(inwptr, wptr, out_wbuf, Cg, block_id1 - block_id0, CONV_WINO_IBLOCK, - CONV_WINO_KBLOCK, CONV_WINO_ATOM_F32, CONV_WINO_NATOMS_F32); + opt_AVX::winofunc_accum_F32((float *)inwptr, (float *)wptr, (float *)out_wbuf, Cg, block_id1 - block_id0, CONV_WINO_IBLOCK, + CONV_WINO_KBLOCK, CONV_WINO_ATOM, CONV_WINO_NATOMS); else #endif #if CV_NEON && CV_NEON_AARCH64 if (conv->useNEON) - opt_NEON::winofunc_accum_f32(inwptr, wptr, out_wbuf, Cg, block_id1 - block_id0, CONV_WINO_IBLOCK, - CONV_WINO_KBLOCK, CONV_WINO_ATOM_F32, CONV_WINO_NATOMS_F32); + { +#ifdef CONV_ARM_FP16 + if (useFP16) + { + opt_NEON_FP16::winofunc_accum_F16(inwptr, wptr, out_wbuf, Cg, block_id1 - block_id0, CONV_WINO_IBLOCK, + CONV_WINO_KBLOCK, CONV_WINO_ATOM, CONV_WINO_NATOMS); + } + else +#endif + opt_NEON::winofunc_accum_F32((float *)inwptr, (float *)wptr, (float *)out_wbuf, Cg, block_id1 - block_id0, CONV_WINO_IBLOCK, + CONV_WINO_KBLOCK, CONV_WINO_ATOM, CONV_WINO_NATOMS); + } else #endif + winofunc_accum_F32((float *)inwptr, (float *)wptr, (float *)out_wbuf, Cg, block_id1 - block_id0, CONV_WINO_IBLOCK, + CONV_WINO_KBLOCK, CONV_WINO_ATOM, CONV_WINO_NATOMS); - winofunc_accum_f32(inwptr, wptr, out_wbuf, Cg, block_id1 - block_id0, CONV_WINO_IBLOCK, - CONV_WINO_KBLOCK, CONV_WINO_ATOM_F32, CONV_WINO_NATOMS_F32); for (int k = k0; k < k1; k++) { float biasv = conv->biasBuf[g*Kg + k]; @@ -274,31 +338,42 @@ int runWinograd63(InputArray _input, InputArray _fusedAddMat, OutputArray _outpu } #if CV_TRY_AVX2 if (conv->useAVX2) - opt_AVX::winofunc_AtXA_8x8_f32(out_wbuf + ((k - k0)*CONV_WINO_IBLOCK + (block_id - block_id0))*CONV_WINO_AREA, CONV_WINO_SIZE, + opt_AVX::winofunc_AtXA_8x8_F32((float *)out_wbuf + ((k - k0)*CONV_WINO_IBLOCK + (block_id - block_id0))*CONV_WINO_AREA, CONV_WINO_SIZE, bpptr, outstep, outptr, outstep, biasv, minval, maxval, ifMinMaxAct); else #endif #if CV_TRY_AVX if (conv->useAVX) - opt_AVX::winofunc_AtXA_8x8_f32(out_wbuf + ((k - k0)*CONV_WINO_IBLOCK + (block_id - block_id0))*CONV_WINO_AREA, CONV_WINO_SIZE, + opt_AVX::winofunc_AtXA_8x8_F32((float *)out_wbuf + ((k - k0)*CONV_WINO_IBLOCK + (block_id - block_id0))*CONV_WINO_AREA, CONV_WINO_SIZE, bpptr, outstep, outptr, outstep, biasv, minval, maxval, ifMinMaxAct); else #endif #if CV_NEON && CV_NEON_AARCH64 + // NEON optimization is only for ARMv8 device, and for ARMv7 device, we use the Universal intrinsics. if (conv->useNEON) - // NEON optimization is only for ARMv8 device, and for ARMv7 device, we use the Universal intrinsics. - opt_NEON::winofunc_AtXA_8x8_f32(out_wbuf + ((k - k0)*CONV_WINO_IBLOCK + (block_id - block_id0))*CONV_WINO_AREA, CONV_WINO_SIZE, + { +#ifdef CONV_ARM_FP16 + if (useFP16) + { + opt_NEON_FP16::winofunc_AtXA_8x8_F16(out_wbuf + ((k - k0)*CONV_WINO_IBLOCK + (block_id - block_id0))*CONV_WINO_AREA * esz, CONV_WINO_SIZE, bpptr, outstep, outptr, outstep, biasv, minval, maxval, ifMinMaxAct); + } + else +#endif + opt_NEON::winofunc_AtXA_8x8_F32((float *)out_wbuf + ((k - k0)*CONV_WINO_IBLOCK + (block_id - block_id0))*CONV_WINO_AREA, CONV_WINO_SIZE, + bpptr, outstep, outptr, outstep, biasv, minval, maxval, ifMinMaxAct); + } else #endif - winofunc_AtXA_8x8_f32(out_wbuf + ((k - k0)*CONV_WINO_IBLOCK + (block_id - block_id0))*CONV_WINO_AREA, CONV_WINO_SIZE, + winofunc_AtXA_8x8_F32((float *)out_wbuf + ((k - k0)*CONV_WINO_IBLOCK + (block_id - block_id0))*CONV_WINO_AREA, CONV_WINO_SIZE, bpptr, outstep, outptr, outstep, biasv, minval, maxval, ifMinMaxAct); + if (partial) { if (activ) activ->forwardSlice(outptr, outptr, CONV_WINO_SIZE*CONV_WINO_STEP, 0, g*Kg + k, g*Kg + k + 1); for (int y = 0; y < dy1; y++) - memcpy(outptr0 + y*W0, outptr + y*CONV_WINO_SIZE,dx1*sizeof(outptr0[0])); + memcpy(outptr0 + y*W0, outptr + y*CONV_WINO_SIZE, dx1*sizeof(outptr0[0])); } } } @@ -314,7 +389,7 @@ int runWinograd63(InputArray _input, InputArray _fusedAddMat, OutputArray _outpu #if CV_SIMD128 -void winofunc_accum_f32(const float* inwptr, const float* wptr, float* outbuf, int Cg, int iblock, +void winofunc_accum_F32(const float* inwptr, const float* wptr, float* outbuf, int Cg, int iblock, const int winoIblock, const int winoKblock, const int winoAtomF32, const int winoNatomF32) { #if 1 @@ -411,7 +486,7 @@ void winofunc_accum_f32(const float* inwptr, const float* wptr, float* outbuf, i } /*Input transform*/ -void winofunc_BtXB_8x8_f32(const float* inptr, int inpstep, +void winofunc_BtXB_8x8_F32(const float* inptr, int inpstep, float* outptr, int Cg, const int winoIblock, const int winoAtomF32) { CV_Assert(winoIblock == 3 && winoAtomF32 == 4); @@ -585,7 +660,7 @@ void winofunc_BtXB_8x8_f32(const float* inptr, int inpstep, the Winograd-transformed weights should also be transposed. init_conv() (see OpConv.fx) takes care of that. */ -void winofunc_AtXA_8x8_f32(const float* inptr, int inpstep, +void winofunc_AtXA_8x8_F32(const float* inptr, int inpstep, float* bpptr, int bpstep, float* outptr, int outstep, float bias, float minval, float maxval, bool ifMinMaxAct) { diff --git a/modules/dnn/src/layers/cpu_kernels/conv_winograd_f63.neon.cpp b/modules/dnn/src/layers/cpu_kernels/conv_winograd_f63.neon.cpp new file mode 100644 index 0000000000..70b716f9c7 --- /dev/null +++ b/modules/dnn/src/layers/cpu_kernels/conv_winograd_f63.neon.cpp @@ -0,0 +1,476 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. + +#include "../../precomp.hpp" +#include "convolution.hpp" +#include "opencv2/core/hal/intrin.hpp" + +namespace cv { +namespace dnn { + +// NEON code work around. +namespace opt_NEON +{ + +#if CV_NEON && CV_NEON_AARCH64 + +/* Accumulate */ +void winofunc_accum_F32(const float* inwptr, const float* wptr, float* outbuf, int Cg, int iblock, + const int winoIblock, const int winoKblock, const int winoAtomF32, const int winoNatomF32) +{ + CV_Assert(winoIblock == 6 && winoKblock == 4 && winoAtomF32 == 4); + if (iblock > 3) + { + for (int atom_id = 0; atom_id < winoNatomF32; atom_id++, + outbuf += winoAtomF32) + { + float32x4_t s00 = vdupq_n_f32(0.f), s01 = s00, s02 = s00, s03 = s00, s04 = s00, s05 = s00; + float32x4_t s10 = vdupq_n_f32(0.f), s11 = s00, s12 = s00, s13 = s00, s14 = s00, s15 = s00; + float32x4_t s20 = vdupq_n_f32(0.f), s21 = s00, s22 = s00, s23 = s00, s24 = s00, s25 = s00; + float32x4_t s30 = vdupq_n_f32(0.f), s31 = s00, s32 = s00, s33 = s00, s34 = s00, s35 = s00; + for (int c = 0; c < Cg; c++, inwptr += winoIblock*winoAtomF32, + wptr += winoKblock*winoAtomF32) { + float32x4_t w0 = vld1q_f32(wptr), w1 = vld1q_f32(wptr + 4); + float32x4_t w2 = vld1q_f32(wptr + 8), w3 = vld1q_f32(wptr + 12); + float32x4_t x0, x1; + x0 = vld1q_f32(inwptr); + x1 = vld1q_f32(inwptr + 4); + s00 = vfmaq_f32(s00, w0, x0); + s01 = vfmaq_f32(s01, w0, x1); + s10 = vfmaq_f32(s10, w1, x0); + s11 = vfmaq_f32(s11, w1, x1); + s20 = vfmaq_f32(s20, w2, x0); + s21 = vfmaq_f32(s21, w2, x1); + s30 = vfmaq_f32(s30, w3, x0); + s31 = vfmaq_f32(s31, w3, x1); + x0 = vld1q_f32(inwptr + 8); + x1 = vld1q_f32(inwptr + 12); + s02 = vfmaq_f32(s02, w0, x0); + s03 = vfmaq_f32(s03, w0, x1); + s12 = vfmaq_f32(s12, w1, x0); + s13 = vfmaq_f32(s13, w1, x1); + s22 = vfmaq_f32(s22, w2, x0); + s23 = vfmaq_f32(s23, w2, x1); + s32 = vfmaq_f32(s32, w3, x0); + s33 = vfmaq_f32(s33, w3, x1); + x0 = vld1q_f32(inwptr + 16); + x1 = vld1q_f32(inwptr + 20); + s04 = vfmaq_f32(s04, w0, x0); + s05 = vfmaq_f32(s05, w0, x1); + s14 = vfmaq_f32(s14, w1, x0); + s15 = vfmaq_f32(s15, w1, x1); + s24 = vfmaq_f32(s24, w2, x0); + s25 = vfmaq_f32(s25, w2, x1); + s34 = vfmaq_f32(s34, w3, x0); + s35 = vfmaq_f32(s35, w3, x1); + } + + vst1q_f32(outbuf, s00); + vst1q_f32(outbuf + 1*64, s01); + vst1q_f32(outbuf + 2*64, s02); + vst1q_f32(outbuf + 3*64, s03); + vst1q_f32(outbuf + 4*64, s04); + vst1q_f32(outbuf + 5*64, s05); + + vst1q_f32(outbuf + 6*64, s10); + vst1q_f32(outbuf + 7*64, s11); + vst1q_f32(outbuf + 8*64, s12); + vst1q_f32(outbuf + 9*64, s13); + vst1q_f32(outbuf + 10*64, s14); + vst1q_f32(outbuf + 11*64, s15); + + vst1q_f32(outbuf + 12*64, s20); + vst1q_f32(outbuf + 13*64, s21); + vst1q_f32(outbuf + 14*64, s22); + vst1q_f32(outbuf + 15*64, s23); + vst1q_f32(outbuf + 16*64, s24); + vst1q_f32(outbuf + 17*64, s25); + + vst1q_f32(outbuf + 18*64, s30); + vst1q_f32(outbuf + 19*64, s31); + vst1q_f32(outbuf + 20*64, s32); + vst1q_f32(outbuf + 21*64, s33); + vst1q_f32(outbuf + 22*64, s34); + vst1q_f32(outbuf + 23*64, s35); + } + } + else + { + for (int atom_id = 0; atom_id < winoNatomF32; atom_id++, + outbuf += winoAtomF32) + { + float32x4_t s00 = vdupq_n_f32(0.f), s01 = s00, s02 = s00; + float32x4_t s10 = vdupq_n_f32(0.f), s11 = s00, s12 = s00; + float32x4_t s20 = vdupq_n_f32(0.f), s21 = s00, s22 = s00; + float32x4_t s30 = vdupq_n_f32(0.f), s31 = s00, s32 = s00; + for (int c = 0; c < Cg; c++, inwptr += winoIblock*winoAtomF32, + wptr += winoKblock*winoAtomF32) { + float32x4_t w0 = vld1q_f32(wptr), w1 = vld1q_f32(wptr + 4); + float32x4_t w2 = vld1q_f32(wptr + 8), w3 = vld1q_f32(wptr + 12); + float32x4_t x0, x1, x2; + x0 = vld1q_f32(inwptr); + x1 = vld1q_f32(inwptr + 4); + x2 = vld1q_f32(inwptr + 8); + s00 = vfmaq_f32(s00, w0, x0); + s01 = vfmaq_f32(s01, w0, x1); + s02 = vfmaq_f32(s02, w0, x2); + s10 = vfmaq_f32(s10, w1, x0); + s11 = vfmaq_f32(s11, w1, x1); + s12 = vfmaq_f32(s12, w1, x2); + s20 = vfmaq_f32(s20, w2, x0); + s21 = vfmaq_f32(s21, w2, x1); + s22 = vfmaq_f32(s22, w2, x2); + s30 = vfmaq_f32(s30, w3, x0); + s31 = vfmaq_f32(s31, w3, x1); + s32 = vfmaq_f32(s32, w3, x2); + } + + vst1q_f32(outbuf, s00); + vst1q_f32(outbuf + 1*64, s01); + vst1q_f32(outbuf + 2*64, s02); + vst1q_f32(outbuf + 6*64, s10); + vst1q_f32(outbuf + 7*64, s11); + vst1q_f32(outbuf + 8*64, s12); + vst1q_f32(outbuf + 12*64, s20); + vst1q_f32(outbuf + 13*64, s21); + vst1q_f32(outbuf + 14*64, s22); + vst1q_f32(outbuf + 18*64, s30); + vst1q_f32(outbuf + 19*64, s31); + vst1q_f32(outbuf + 20*64, s32); + } + } +} + +#undef T4x4 +#define T4x4(a, b, c, d, tr0, tr1) \ + tr0 = vtrnq_f32(a, b); \ + tr1 = vtrnq_f32(c, d); \ + a = vcombine_f32(vget_low_f32(tr0.val[0]), vget_low_f32(tr1.val[0])); \ + b = vcombine_f32(vget_low_f32(tr0.val[1]), vget_low_f32(tr1.val[1])); \ + c = vcombine_f32(vget_high_f32(tr0.val[0]), vget_high_f32(tr1.val[0])); \ + d = vcombine_f32(vget_high_f32(tr0.val[1]), vget_high_f32(tr1.val[1])) + +/*Input transform*/ +void winofunc_BtXB_8x8_F32(const float* inptr, int inpstep, + float* outptr, int Cg, const int winoIblock, const int winoAtomF32) +{ + float32x4_t x00 = vld1q_f32(inptr), x01 = vld1q_f32(inptr + 4); + float32x4_t x10 = vld1q_f32(inptr + inpstep), x11 = vld1q_f32(inptr + inpstep + 4); + float32x4_t x20 = vld1q_f32(inptr + inpstep*2), x21 = vld1q_f32(inptr + inpstep*2 + 4); + float32x4_t x30 = vld1q_f32(inptr + inpstep*3), x31 = vld1q_f32(inptr + inpstep*3 + 4); + float32x4_t x40 = vld1q_f32(inptr + inpstep*4), x41 = vld1q_f32(inptr + inpstep*4 + 4); + float32x4_t x50 = vld1q_f32(inptr + inpstep*5), x51 = vld1q_f32(inptr + inpstep*5 + 4); + float32x4_t x60 = vld1q_f32(inptr + inpstep*6), x61 = vld1q_f32(inptr + inpstep*6 + 4); + float32x4_t x70 = vld1q_f32(inptr + inpstep*7), x71 = vld1q_f32(inptr + inpstep*7 + 4); + + float32x4_t z00, z01, z10, z11, z20, z21, z30, z31, z40, z41, z50, z51, z60, z61, z70, z71; + + { + /* Y[0] = [1.f, 0.f, -5.25f, 0.f, 5.25f, 0.f, -1.f, 0.f]*X */ + /* Y[7] = [0.f, -1.f, 0.f, 5.25f, 0.f, -5.25f, 0.f, 1.f]*X */ + float32x4_t q5_25 = vdupq_n_f32(5.25f), t00, t01, t10, t11; + t00 = vsubq_f32(x40, x20); + t01 = vsubq_f32(x41, x21); + t10 = vsubq_f32(x30, x50); + t11 = vsubq_f32(x31, x51); + float32x4_t y00 = vfmaq_f32(vsubq_f32(x00, x60), t00, q5_25); + float32x4_t y01 = vfmaq_f32(vsubq_f32(x01, x61), t01, q5_25); + float32x4_t y70 = vfmaq_f32(vsubq_f32(x70, x10), t10, q5_25); + float32x4_t y71 = vfmaq_f32(vsubq_f32(x71, x11), t11, q5_25); + + /* Y[1] = [0.f, 1.f, 1.f, -4.25f, -4.25f, 1.f, 1.f, 0.f]*X */ + /* Y[2] = [0.f, -1.f, 1.f, 4.25f, -4.25f, -1.f, 1.f, 0.f]*X */ + float32x4_t qm4_25 = vdupq_n_f32(-4.25f); + t00 = vfmaq_f32(vaddq_f32(x10, x50), x30, qm4_25); + t01 = vfmaq_f32(vaddq_f32(x11, x51), x31, qm4_25); + t10 = vfmaq_f32(vaddq_f32(x20, x60), x40, qm4_25); + t11 = vfmaq_f32(vaddq_f32(x21, x61), x41, qm4_25); + + float32x4_t y10 = vaddq_f32(t00, t10), y11 = vaddq_f32(t01, t11); + float32x4_t y20 = vsubq_f32(t10, t00), y21 = vsubq_f32(t11, t01); + + /* Y[3] = [0.f, 0.5f, 0.25f, -2.5f, -1.25f, 2.f, 1.f, 0.f]*X */ + /* Y[4] = [0.f, -0.5f, 0.25f, 2.5f, -1.25f, -2.f, 1.f, 0.f]*X */ + float32x4_t q0_5 = vdupq_n_f32(0.5f), q0_25 = vdupq_n_f32(0.25f); + float32x4_t qm2_5 = vdupq_n_f32(-2.5f), qm1_25 = vdupq_n_f32(-1.25f); + t00 = vfmaq_f32(vaddq_f32(x50, x50), x10, q0_5); + t01 = vfmaq_f32(vaddq_f32(x51, x51), x11, q0_5); + t10 = vfmaq_f32(x60, x20, q0_25); + t11 = vfmaq_f32(x61, x21, q0_25); + t00 = vfmaq_f32(t00, x30, qm2_5); + t01 = vfmaq_f32(t01, x31, qm2_5); + t10 = vfmaq_f32(t10, x40, qm1_25); + t11 = vfmaq_f32(t11, x41, qm1_25); + + float32x4_t y30 = vaddq_f32(t00, t10), y31 = vaddq_f32(t01, t11); + float32x4_t y40 = vsubq_f32(t10, t00), y41 = vsubq_f32(t11, t01); + + /* Y[5] = [0.f, 2.f, 4.f, -2.5f, -5.f, 0.5f, 1.f, 0.f]*X */ + /* Y[6] = [0.f, -2.f, 4.f, 2.5f, -5.f, -0.5f, 1.f, 0.f]*X */ + float32x4_t q4 = vdupq_n_f32(4.f), qm5 = vdupq_n_f32(-5.f); + t00 = vfmaq_f32(vaddq_f32(x10, x10), x50, q0_5); + t01 = vfmaq_f32(vaddq_f32(x11, x11), x51, q0_5); + t10 = vfmaq_f32(x60, x20, q4); + t11 = vfmaq_f32(x61, x21, q4); + t00 = vfmaq_f32(t00, x30, qm2_5); + t01 = vfmaq_f32(t01, x31, qm2_5); + t10 = vfmaq_f32(t10, x40, qm5); + t11 = vfmaq_f32(t11, x41, qm5); + + float32x4_t y50 = vaddq_f32(t00, t10), y51 = vaddq_f32(t01, t11); + float32x4_t y60 = vsubq_f32(t10, t00), y61 = vsubq_f32(t11, t01); + + /* transpose 8x8 matrix in-place with some renumeration of the elements: */ + /* Y: */ + /* y00 y01 */ + /* y10 y11 */ + /* ... */ + /* y70 y71 */ + /* Y': */ + /* y00 y40 */ + /* y10 y50 */ + /* y20 y60 */ + /* y30 y70 */ + /* y01 y41 */ + /* y11 y51 */ + /* y21 y61 */ + /* y31 y71 */ + /* in other words, y40 <-> y01, y50 <-> y11, y60 <-> y21, y70 <-> y31 */ + float32x4x2_t tr0, tr1; + + T4x4(y00, y10, y20, y30, tr0, tr1); + T4x4(y01, y11, y21, y31, tr0, tr1); + T4x4(y40, y50, y60, y70, tr0, tr1); + T4x4(y41, y51, y61, y71, tr0, tr1); + + /* Z[0] = [1.f, 0.f, -5.25f, 0.f, 5.25f, 0.f, -1.f, 0.f]*Y */ + /* Z[7] = [0.f, -1.f, 0.f, 5.25f, 0.f, -5.25f, 0.f, 1.f]*Y */ + t00 = vsubq_f32(y01, y20); + t01 = vsubq_f32(y41, y60); + t10 = vsubq_f32(y30, y11); + t11 = vsubq_f32(y70, y51); + z00 = vfmaq_f32(vsubq_f32(y00, y21), t00, q5_25); + z01 = vfmaq_f32(vsubq_f32(y40, y61), t01, q5_25); + z70 = vfmaq_f32(vsubq_f32(y31, y10), t10, q5_25); + z71 = vfmaq_f32(vsubq_f32(y71, y50), t11, q5_25); + + /* Z[1] = [0.f, 1.f, 1.f, -4.25f, -4.25f, 1.f, 1.f, 0.f]*Y */ + /* Z[2] = [0.f, -1.f, 1.f, 4.25f, -4.25f, -1.f, 1.f, 0.f]*Y */ + t00 = vfmaq_f32(vaddq_f32(y10, y11), y30, qm4_25); + t01 = vfmaq_f32(vaddq_f32(y50, y51), y70, qm4_25); + t10 = vfmaq_f32(vaddq_f32(y20, y21), y01, qm4_25); + t11 = vfmaq_f32(vaddq_f32(y60, y61), y41, qm4_25); + + z10 = vaddq_f32(t00, t10); z11 = vaddq_f32(t01, t11); + z20 = vsubq_f32(t10, t00); z21 = vsubq_f32(t11, t01); + + /* Z[3] = [0.f, 0.5f, 0.25f, -2.5f, -1.25f, 2.f, 1.f, 0.f]*Y */ + /* Z[4] = [0.f, -0.5f, 0.25f, 2.5f, -1.25f, -2.f, 1.f, 0.f]*Y */ + t00 = vfmaq_f32(vaddq_f32(y11, y11), y10, q0_5); + t01 = vfmaq_f32(vaddq_f32(y51, y51), y50, q0_5); + t10 = vfmaq_f32(y21, y20, q0_25); + t11 = vfmaq_f32(y61, y60, q0_25); + t00 = vfmaq_f32(t00, y30, qm2_5); + t01 = vfmaq_f32(t01, y70, qm2_5); + t10 = vfmaq_f32(t10, y01, qm1_25); + t11 = vfmaq_f32(t11, y41, qm1_25); + + z30 = vaddq_f32(t00, t10); z31 = vaddq_f32(t01, t11); + z40 = vsubq_f32(t10, t00); z41 = vsubq_f32(t11, t01); + + /* Z[5] = [0.f, 2.f, 4.f, -2.5f, -5.f, 0.5f, 1.f, 0.f]*Y */ + /* Z[6] = [0.f, -2.f, 4.f, 2.5f, -5.f, -0.5f, 1.f, 0.f]*Y */ + t00 = vfmaq_f32(vaddq_f32(y10, y10), y11, q0_5); + t01 = vfmaq_f32(vaddq_f32(y50, y50), y51, q0_5); + t10 = vfmaq_f32(y21, y20, q4); + t11 = vfmaq_f32(y61, y60, q4); + t00 = vfmaq_f32(t00, y30, qm2_5); + t01 = vfmaq_f32(t01, y70, qm2_5); + t10 = vfmaq_f32(t10, y01, qm5); + t11 = vfmaq_f32(t11, y41, qm5); + + z50 = vaddq_f32(t00, t10); z51 = vaddq_f32(t01, t11); + z60 = vsubq_f32(t10, t00); z61 = vsubq_f32(t11, t01); + } + + const int outstep = winoIblock*winoAtomF32*Cg; + + vst1q_f32(outptr, z00); + vst1q_f32(outptr + outstep, z01); + vst1q_f32(outptr + outstep*2, z10); + vst1q_f32(outptr + outstep*3, z11); + vst1q_f32(outptr + outstep*4, z20); + vst1q_f32(outptr + outstep*5, z21); + vst1q_f32(outptr + outstep*6, z30); + vst1q_f32(outptr + outstep*7, z31); + vst1q_f32(outptr + outstep*8, z40); + vst1q_f32(outptr + outstep*9, z41); + vst1q_f32(outptr + outstep*10, z50); + vst1q_f32(outptr + outstep*11, z51); + vst1q_f32(outptr + outstep*12, z60); + vst1q_f32(outptr + outstep*13, z61); + vst1q_f32(outptr + outstep*14, z70); + vst1q_f32(outptr + outstep*15, z71); +} + +/*Output transform*/ +void winofunc_AtXA_8x8_F32(const float* inptr, int inpstep, + float* bpptr, int bpstep, float* outptr, int outstep, + float bias, float minval, float maxval, bool ifMinMaxAct) +{ + float32x4_t x00 = vld1q_f32(inptr), x01 = vld1q_f32(inptr + 4); + float32x4_t x10 = vld1q_f32(inptr + inpstep), x11 = vld1q_f32(inptr + inpstep + 4); + float32x4_t x20 = vld1q_f32(inptr + inpstep*2), x21 = vld1q_f32(inptr + inpstep*2 + 4); + float32x4_t x30 = vld1q_f32(inptr + inpstep*3), x31 = vld1q_f32(inptr + inpstep*3 + 4); + float32x4_t x40 = vld1q_f32(inptr + inpstep*4), x41 = vld1q_f32(inptr + inpstep*4 + 4); + float32x4_t x50 = vld1q_f32(inptr + inpstep*5), x51 = vld1q_f32(inptr + inpstep*5 + 4); + float32x4_t x60 = vld1q_f32(inptr + inpstep*6), x61 = vld1q_f32(inptr + inpstep*6 + 4); + float32x4_t x70 = vld1q_f32(inptr + inpstep*7), x71 = vld1q_f32(inptr + inpstep*7 + 4); + float32x4_t z00, z01, z10, z11, z20, z21, z30, z31, z40, z41, z50, z51; + + { + float32x4_t s12_0, s12_1, s34_0, s34_1, s56_0, s56_1; + s12_0 = vaddq_f32(x10, x20); s12_1 = vaddq_f32(x11, x21); + s34_0 = vaddq_f32(x30, x40); s34_1 = vaddq_f32(x31, x41); + s56_0 = vaddq_f32(x50, x60); s56_1 = vaddq_f32(x51, x61); + + float32x4_t y00 = vaddq_f32(vaddq_f32(vaddq_f32(x00, s12_0), s34_0), s56_0); + float32x4_t y01 = vaddq_f32(vaddq_f32(vaddq_f32(x01, s12_1), s34_1), s56_1); + float32x4_t y20 = vfmaq_n_f32(vfmaq_n_f32(s12_0, s34_0, 4.0f), s56_0, 0.25f); + float32x4_t y21 = vfmaq_n_f32(vfmaq_n_f32(s12_1, s34_1, 4.0f), s56_1, 0.25f); + float32x4_t y40 = vfmaq_n_f32(vfmaq_n_f32(s12_0, s34_0, 16.0f), s56_0, 1.f/16); + float32x4_t y41 = vfmaq_n_f32(vfmaq_n_f32(s12_1, s34_1, 16.0f), s56_1, 1.f/16); + + s12_0 = vsubq_f32(x10, x20); s12_1 = vsubq_f32(x11, x21); + s34_0 = vsubq_f32(x30, x40); s34_1 = vsubq_f32(x31, x41); + s56_0 = vsubq_f32(x50, x60); s56_1 = vsubq_f32(x51, x61); + + float32x4_t y50 = vfmaq_n_f32(vfmaq_n_f32(vaddq_f32(x70, s12_0), + s34_0, 32.f), s56_0, 1.f/32); + float32x4_t y51 = vfmaq_n_f32(vfmaq_n_f32(vaddq_f32(x71, s12_1), + s34_1, 32.f), s56_1, 1.f/32); + float32x4_t y10 = vfmaq_n_f32(vfmaq_n_f32(s12_0, s34_0, 2.0f), s56_0, 0.5f); + float32x4_t y11 = vfmaq_n_f32(vfmaq_n_f32(s12_1, s34_1, 2.0f), s56_1, 0.5f); + float32x4_t y30 = vfmaq_n_f32(vfmaq_n_f32(s12_0, s34_0, 8.0f), s56_0, 0.125f); + float32x4_t y31 = vfmaq_n_f32(vfmaq_n_f32(s12_1, s34_1, 8.0f), s56_1, 0.125f); + float32x4_t y60 = vdupq_n_f32(0.f), y61 = y60, y70 = y60, y71 = y60; + + /* transpose 8x8 matrix in-place with some renumeration of the elements: */ + /* Y: */ + /* y00 y01 */ + /* y10 y11 */ + /* ... */ + /* y50 y51 */ + /* 0 0 */ + /* 0 0 */ + /* Y': */ + /* y00 y40 */ + /* y10 y50 */ + /* y20 y60 */ + /* y30 y70 */ + /* y01 y41 */ + /* y11 y51 */ + /* y21 y61 */ + /* y31 y71 */ + /* in other words, y40 <-> y01, y50 <-> y11, y60 <-> y21, y70 <-> y31 */ + float32x4x2_t tr0, tr1; + + T4x4(y00, y10, y20, y30, tr0, tr1); + T4x4(y01, y11, y21, y31, tr0, tr1); + T4x4(y40, y50, y60, y70, tr0, tr1); + T4x4(y41, y51, y61, y71, tr0, tr1); + + s12_0 = vaddq_f32(y10, y20); s12_1 = vaddq_f32(y50, y60); + s34_0 = vaddq_f32(y30, y01); s34_1 = vaddq_f32(y70, y41); + s56_0 = vaddq_f32(y11, y21); s56_1 = vaddq_f32(y51, y61); + + z00 = vaddq_f32(vaddq_f32(vaddq_f32(y00, s12_0), s34_0), s56_0); + z01 = vaddq_f32(vaddq_f32(vaddq_f32(y40, s12_1), s34_1), s56_1); + z20 = vfmaq_n_f32(vfmaq_n_f32(s12_0, s34_0, 4.0f), s56_0, 0.25f); + z21 = vfmaq_n_f32(vfmaq_n_f32(s12_1, s34_1, 4.0f), s56_1, 0.25f); + z40 = vfmaq_n_f32(vfmaq_n_f32(s12_0, s34_0, 16.0f), s56_0, 1.f/16); + z41 = vfmaq_n_f32(vfmaq_n_f32(s12_1, s34_1, 16.0f), s56_1, 1.f/16); + + s12_0 = vsubq_f32(y10, y20); s12_1 = vsubq_f32(y50, y60); + s34_0 = vsubq_f32(y30, y01); s34_1 = vsubq_f32(y70, y41); + s56_0 = vsubq_f32(y11, y21); s56_1 = vsubq_f32(y51, y61); + + z50 = vfmaq_n_f32(vfmaq_n_f32(vaddq_f32(y31, s12_0), + s34_0, 32.f), s56_0, 1.f/32); + z51 = vfmaq_n_f32(vfmaq_n_f32(vaddq_f32(y71, s12_1), + s34_1, 32.f), s56_1, 1.f/32); + z10 = vfmaq_n_f32(vfmaq_n_f32(s12_0, s34_0, 2.0f), s56_0, 0.5f); + z11 = vfmaq_n_f32(vfmaq_n_f32(s12_1, s34_1, 2.0f), s56_1, 0.5f); + z30 = vfmaq_n_f32(vfmaq_n_f32(s12_0, s34_0, 8.0f), s56_0, 0.125f); + z31 = vfmaq_n_f32(vfmaq_n_f32(s12_1, s34_1, 8.0f), s56_1, 0.125f); + float32x4_t vbias = vdupq_n_f32(bias); + + z00 = vaddq_f32(z00, vbias); + z01 = vaddq_f32(z01, vbias); + z10 = vaddq_f32(z10, vbias); + z11 = vaddq_f32(z11, vbias); + z20 = vaddq_f32(z20, vbias); + z21 = vaddq_f32(z21, vbias); + z30 = vaddq_f32(z30, vbias); + z31 = vaddq_f32(z31, vbias); + z40 = vaddq_f32(z40, vbias); + z41 = vaddq_f32(z41, vbias); + z50 = vaddq_f32(z50, vbias); + z51 = vaddq_f32(z51, vbias); + } + + if (bpptr) + { + float32x2_t zhalf = vdup_n_f32(0.f); + z00 = vaddq_f32(z00, vld1q_f32(bpptr)); + z01 = vaddq_f32(z01, vcombine_f32(vld1_f32(bpptr + 4), zhalf)); + z10 = vaddq_f32(z10, vld1q_f32(bpptr + bpstep)); + z11 = vaddq_f32(z11, vcombine_f32(vld1_f32(bpptr + bpstep + 4), zhalf)); + z20 = vaddq_f32(z20, vld1q_f32(bpptr + bpstep*2)); + z21 = vaddq_f32(z21, vcombine_f32(vld1_f32(bpptr + bpstep*2 + 4), zhalf)); + z30 = vaddq_f32(z30, vld1q_f32(bpptr + bpstep*3)); + z31 = vaddq_f32(z31, vcombine_f32(vld1_f32(bpptr + bpstep*3 + 4), zhalf)); + z40 = vaddq_f32(z40, vld1q_f32(bpptr + bpstep*4)); + z41 = vaddq_f32(z41, vcombine_f32(vld1_f32(bpptr + bpstep*4 + 4), zhalf)); + z50 = vaddq_f32(z50, vld1q_f32(bpptr + bpstep*5)); + z51 = vaddq_f32(z51, vcombine_f32(vld1_f32(bpptr + bpstep*5 + 4), zhalf)); + } + + if (ifMinMaxAct) + { + float32x4_t vmax = vdupq_n_f32(maxval); + float32x4_t vmin = vdupq_n_f32(minval); + + z00 = vminq_f32(vmaxq_f32(z00, vmin), vmax); + z01 = vminq_f32(vmaxq_f32(z01, vmin), vmax); + z10 = vminq_f32(vmaxq_f32(z10, vmin), vmax); + z11 = vminq_f32(vmaxq_f32(z11, vmin), vmax); + z20 = vminq_f32(vmaxq_f32(z20, vmin), vmax); + z21 = vminq_f32(vmaxq_f32(z21, vmin), vmax); + z30 = vminq_f32(vmaxq_f32(z30, vmin), vmax); + z31 = vminq_f32(vmaxq_f32(z31, vmin), vmax); + z40 = vminq_f32(vmaxq_f32(z40, vmin), vmax); + z41 = vminq_f32(vmaxq_f32(z41, vmin), vmax); + z50 = vminq_f32(vmaxq_f32(z50, vmin), vmax); + z51 = vminq_f32(vmaxq_f32(z51, vmin), vmax); + } + + vst1q_f32(outptr, z00); + vst1_f32(outptr + 4, vget_low_f32(z01)); + vst1q_f32(outptr + outstep, z10); + vst1_f32(outptr + outstep + 4, vget_low_f32(z11)); + vst1q_f32(outptr + outstep*2, z20); + vst1_f32(outptr + outstep*2 + 4, vget_low_f32(z21)); + vst1q_f32(outptr + outstep*3, z30); + vst1_f32(outptr + outstep*3 + 4, vget_low_f32(z31)); + vst1q_f32(outptr + outstep*4, z40); + vst1_f32(outptr + outstep*4 + 4, vget_low_f32(z41)); + vst1q_f32(outptr + outstep*5, z50); + vst1_f32(outptr + outstep*5 + 4, vget_low_f32(z51)); +} + +#endif +} + +}} // namespace diff --git a/modules/dnn/src/layers/cpu_kernels/conv_winograd_f63.simd.hpp b/modules/dnn/src/layers/cpu_kernels/conv_winograd_f63.simd.hpp index 2688c75785..d1f1610280 100644 --- a/modules/dnn/src/layers/cpu_kernels/conv_winograd_f63.simd.hpp +++ b/modules/dnn/src/layers/cpu_kernels/conv_winograd_f63.simd.hpp @@ -9,26 +9,37 @@ namespace dnn { CV_CPU_OPTIMIZATION_NAMESPACE_BEGIN /* Accumulate */ -void winofunc_accum_f32(const float* inwptr, const float* wptr, float* outbuf, int Cg, int iblock, +void winofunc_accum_F32(const float* inwptr, const float* wptr, float* outbuf, int Cg, int iblock, const int winoIblock, const int winoKblock, const int winoAtomF32, const int winoNatomF32); /*Input transform*/ -void winofunc_BtXB_8x8_f32(const float* inptr, int inpstep, +void winofunc_BtXB_8x8_F32(const float* inptr, int inpstep, float* outptr, int Cg, const int winoIblock, const int winoAtomF32); /*Output transform*/ -void winofunc_AtXA_8x8_f32(const float* inptr, int inpstep, +void winofunc_AtXA_8x8_F32(const float* inptr, int inpstep, float* bpptr, int bpstep, float* outptr, int outstep, float bias, float minval, float maxval, bool ifMinMaxAct); -#if !defined(CV_CPU_OPTIMIZATION_DECLARATIONS_ONLY) && CV_AVX +// FP 16 branch, only ARMv8 supports. +void winofunc_accum_F16(const char* _inwptr, const char* _wptr, char* _outbuf, int Cg, int iblock, + const int winoIblock, const int winoKblock, const int winoAtomF16, const int winoNatomF16); +void winofunc_BtXB_8x8_F16(const float * inptr, int inpstep, + char * _outptr, int Cg, const int winoIblock, const int winoAtomF16); +void winofunc_AtXA_8x8_F16(const char* inptr, int inpstep, + float * bpptr, int bpstep, float* outptr, int outstep, + float bias, float minval, float maxval, bool ifMinMaxAct); + +#if !defined(CV_CPU_OPTIMIZATION_DECLARATIONS_ONLY) + +#if CV_AVX #if !CV_FMA3 // AVX workaround #undef _mm256_fmadd_ps #define _mm256_fmadd_ps(a, b, c) _mm256_add_ps(c, _mm256_mul_ps(a, b)) #endif -void winofunc_accum_f32(const float* inwptr, const float* wptr, float* outbuf, int Cg, int iblock, +void winofunc_accum_F32(const float* inwptr, const float* wptr, float* outbuf, int Cg, int iblock, const int winoIblock, const int winoKblock, const int winoAtomF32, const int winoNatomF32) { CV_Assert(winoIblock == 6 && winoKblock == 4 && winoAtomF32 == 8); @@ -187,7 +198,7 @@ void transpose8_ps(__m256 &row0, __m256 &row1, __m256 &row2, __m256 &row3, __m25 } /*Input transform*/ -void winofunc_BtXB_8x8_f32(const float* inptr, int inpstep, +void winofunc_BtXB_8x8_F32(const float* inptr, int inpstep, float* outptr, int Cg, const int winoIblock, const int winoAtomF32) { __m256 x00 = _mm256_loadu_ps(inptr); @@ -311,7 +322,7 @@ void winofunc_BtXB_8x8_f32(const float* inptr, int inpstep, 0.f, 1.f, 1.f, 16.f, 16.f, 1.f/16, 1.f/16, 0.f, 0.f, 1.f, -1.f, 32.f, -32.f, 1.f/32, -1.f/32, 1.f] */ -void winofunc_AtXA_8x8_f32(const float* inptr, int inpstep, +void winofunc_AtXA_8x8_F32(const float* inptr, int inpstep, float* bpptr, int bpstep, float* outptr, int outstep, float bias, float minval, float maxval, bool ifMinMaxAct) { @@ -405,154 +416,13 @@ void winofunc_AtXA_8x8_f32(const float* inptr, int inpstep, STORE6_ELE_FROM_16(outptr + outstep * 5, z50, lowM, highM); _mm256_zeroupper(); } -#endif // CV_CPU_OPTIMIZATION_DECLARATIONS_ONLY -CV_CPU_OPTIMIZATION_NAMESPACE_END +#endif // CV_AVX -// NEON code work around. -namespace opt_NEON -{ - -#if !defined(CV_CPU_OPTIMIZATION_DECLARATIONS_ONLY) && CV_NEON && CV_NEON_AARCH64 -/* Accumulate */ -void winofunc_accum_f32(const float* inwptr, const float* wptr, float* outbuf, int Cg, int iblock, - const int winoIblock, const int winoKblock, const int winoAtomF32, const int winoNatomF32); - -/*Input transform*/ -void winofunc_BtXB_8x8_f32(const float* inptr, int inpstep, - float* outptr, int Cg, const int winoIblock, const int winoAtomF32); - -/*Output transform*/ -void winofunc_AtXA_8x8_f32(const float* inptr, int inpstep, - float* bpptr, int bpstep, float* outptr, int outstep, - float bias, float minval, float maxval, bool ifMinMaxAct); - -void winofunc_accum_f32(const float* inwptr, const float* wptr, float* outbuf, int Cg, int iblock, - const int winoIblock, const int winoKblock, const int winoAtomF32, const int winoNatomF32) -{ - CV_Assert(winoIblock == 6 && winoKblock == 4 && winoAtomF32 == 4); - if (iblock > 3) - { - for (int atom_id = 0; atom_id < winoNatomF32; atom_id++, - outbuf += winoAtomF32) - { - float32x4_t s00 = vdupq_n_f32(0.f), s01 = s00, s02 = s00, s03 = s00, s04 = s00, s05 = s00; - float32x4_t s10 = vdupq_n_f32(0.f), s11 = s00, s12 = s00, s13 = s00, s14 = s00, s15 = s00; - float32x4_t s20 = vdupq_n_f32(0.f), s21 = s00, s22 = s00, s23 = s00, s24 = s00, s25 = s00; - float32x4_t s30 = vdupq_n_f32(0.f), s31 = s00, s32 = s00, s33 = s00, s34 = s00, s35 = s00; - for (int c = 0; c < Cg; c++, inwptr += winoIblock*winoAtomF32, - wptr += winoKblock*winoAtomF32) { - float32x4_t w0 = vld1q_f32(wptr), w1 = vld1q_f32(wptr + 4); - float32x4_t w2 = vld1q_f32(wptr + 8), w3 = vld1q_f32(wptr + 12); - float32x4_t x0, x1; - x0 = vld1q_f32(inwptr); - x1 = vld1q_f32(inwptr + 4); - s00 = vfmaq_f32(s00, w0, x0); - s01 = vfmaq_f32(s01, w0, x1); - s10 = vfmaq_f32(s10, w1, x0); - s11 = vfmaq_f32(s11, w1, x1); - s20 = vfmaq_f32(s20, w2, x0); - s21 = vfmaq_f32(s21, w2, x1); - s30 = vfmaq_f32(s30, w3, x0); - s31 = vfmaq_f32(s31, w3, x1); - x0 = vld1q_f32(inwptr + 8); - x1 = vld1q_f32(inwptr + 12); - s02 = vfmaq_f32(s02, w0, x0); - s03 = vfmaq_f32(s03, w0, x1); - s12 = vfmaq_f32(s12, w1, x0); - s13 = vfmaq_f32(s13, w1, x1); - s22 = vfmaq_f32(s22, w2, x0); - s23 = vfmaq_f32(s23, w2, x1); - s32 = vfmaq_f32(s32, w3, x0); - s33 = vfmaq_f32(s33, w3, x1); - x0 = vld1q_f32(inwptr + 16); - x1 = vld1q_f32(inwptr + 20); - s04 = vfmaq_f32(s04, w0, x0); - s05 = vfmaq_f32(s05, w0, x1); - s14 = vfmaq_f32(s14, w1, x0); - s15 = vfmaq_f32(s15, w1, x1); - s24 = vfmaq_f32(s24, w2, x0); - s25 = vfmaq_f32(s25, w2, x1); - s34 = vfmaq_f32(s34, w3, x0); - s35 = vfmaq_f32(s35, w3, x1); - } - - vst1q_f32(outbuf, s00); - vst1q_f32(outbuf + 1*64, s01); - vst1q_f32(outbuf + 2*64, s02); - vst1q_f32(outbuf + 3*64, s03); - vst1q_f32(outbuf + 4*64, s04); - vst1q_f32(outbuf + 5*64, s05); - - vst1q_f32(outbuf + 6*64, s10); - vst1q_f32(outbuf + 7*64, s11); - vst1q_f32(outbuf + 8*64, s12); - vst1q_f32(outbuf + 9*64, s13); - vst1q_f32(outbuf + 10*64, s14); - vst1q_f32(outbuf + 11*64, s15); - - vst1q_f32(outbuf + 12*64, s20); - vst1q_f32(outbuf + 13*64, s21); - vst1q_f32(outbuf + 14*64, s22); - vst1q_f32(outbuf + 15*64, s23); - vst1q_f32(outbuf + 16*64, s24); - vst1q_f32(outbuf + 17*64, s25); - - vst1q_f32(outbuf + 18*64, s30); - vst1q_f32(outbuf + 19*64, s31); - vst1q_f32(outbuf + 20*64, s32); - vst1q_f32(outbuf + 21*64, s33); - vst1q_f32(outbuf + 22*64, s34); - vst1q_f32(outbuf + 23*64, s35); - } - } - else - { - for (int atom_id = 0; atom_id < winoNatomF32; atom_id++, - outbuf += winoAtomF32) - { - float32x4_t s00 = vdupq_n_f32(0.f), s01 = s00, s02 = s00; - float32x4_t s10 = vdupq_n_f32(0.f), s11 = s00, s12 = s00; - float32x4_t s20 = vdupq_n_f32(0.f), s21 = s00, s22 = s00; - float32x4_t s30 = vdupq_n_f32(0.f), s31 = s00, s32 = s00; - for (int c = 0; c < Cg; c++, inwptr += winoIblock*winoAtomF32, - wptr += winoKblock*winoAtomF32) { - float32x4_t w0 = vld1q_f32(wptr), w1 = vld1q_f32(wptr + 4); - float32x4_t w2 = vld1q_f32(wptr + 8), w3 = vld1q_f32(wptr + 12); - float32x4_t x0, x1, x2; - x0 = vld1q_f32(inwptr); - x1 = vld1q_f32(inwptr + 4); - x2 = vld1q_f32(inwptr + 8); - s00 = vfmaq_f32(s00, w0, x0); - s01 = vfmaq_f32(s01, w0, x1); - s02 = vfmaq_f32(s02, w0, x2); - s10 = vfmaq_f32(s10, w1, x0); - s11 = vfmaq_f32(s11, w1, x1); - s12 = vfmaq_f32(s12, w1, x2); - s20 = vfmaq_f32(s20, w2, x0); - s21 = vfmaq_f32(s21, w2, x1); - s22 = vfmaq_f32(s22, w2, x2); - s30 = vfmaq_f32(s30, w3, x0); - s31 = vfmaq_f32(s31, w3, x1); - s32 = vfmaq_f32(s32, w3, x2); - } - - vst1q_f32(outbuf, s00); - vst1q_f32(outbuf + 1*64, s01); - vst1q_f32(outbuf + 2*64, s02); - vst1q_f32(outbuf + 6*64, s10); - vst1q_f32(outbuf + 7*64, s11); - vst1q_f32(outbuf + 8*64, s12); - vst1q_f32(outbuf + 12*64, s20); - vst1q_f32(outbuf + 13*64, s21); - vst1q_f32(outbuf + 14*64, s22); - vst1q_f32(outbuf + 18*64, s30); - vst1q_f32(outbuf + 19*64, s31); - vst1q_f32(outbuf + 20*64, s32); - } - } -} +// FP16, currently, only ARMv8 may support it +#if defined(CV_NEON_AARCH64) && CV_NEON_AARCH64 && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) +#undef T4x4 #define T4x4(a, b, c, d, tr0, tr1) \ tr0 = vtrnq_f32(a, b); \ tr1 = vtrnq_f32(c, d); \ @@ -561,10 +431,168 @@ void winofunc_accum_f32(const float* inwptr, const float* wptr, float* outbuf, i c = vcombine_f32(vget_high_f32(tr0.val[0]), vget_high_f32(tr1.val[0])); \ d = vcombine_f32(vget_high_f32(tr0.val[1]), vget_high_f32(tr1.val[1])) -/*Input transform*/ -void winofunc_BtXB_8x8_f32(const float* inptr, int inpstep, - float* outptr, int Cg, const int winoIblock, const int winoAtomF32) +/* Accumulate */ +void winofunc_accum_F16(const char* _inwptr, const char* _wptr, char* _outbuf, int Cg, int iblock, + const int winoIblock, const int winoKblock, const int winoAtomF16, const int winoNatomF16) { + typedef __fp16 float16_t; + const float16_t* inwptr = (const float16_t*)_inwptr; + const float16_t* wptr = (const float16_t*)_wptr; + float16_t* outbuf = (float16_t*)_outbuf; + + CV_Assert(winoIblock == 6 && winoKblock == 4 && winoAtomF16 == 8); + + if (iblock > 3) + { + for (int atom_id = 0; atom_id < winoNatomF16; atom_id++, outbuf += winoAtomF16) + { + float16x8_t s00 = vdupq_n_f16(0.f), s01 = s00, s02 = s00, s03 = s00, s04 = s00, s05 = s00; + float16x8_t s10 = vdupq_n_f16(0.f), s11 = s00, s12 = s00, s13 = s00, s14 = s00, s15 = s00; + float16x8_t s20 = vdupq_n_f16(0.f), s21 = s00, s22 = s00, s23 = s00, s24 = s00, s25 = s00; + float16x8_t s30 = vdupq_n_f16(0.f), s31 = s00, s32 = s00, s33 = s00, s34 = s00, s35 = s00; + + for (int c = 0; c < Cg; c++, inwptr += winoIblock*winoAtomF16, + wptr += winoKblock*winoAtomF16) + { + float16x8_t w0 = vld1q_f16(wptr), w1 = vld1q_f16(wptr + 8); + float16x8_t w2 = vld1q_f16(wptr + 16), w3 = vld1q_f16(wptr + 24); + + float16x8_t x0, x1, x2; + x0 = vld1q_f16(inwptr); + x1 = vld1q_f16(inwptr + 8); + x2 = vld1q_f16(inwptr + 16); + + s00 = vfmaq_f16(s00, w0, x0); + s01 = vfmaq_f16(s01, w0, x1); + s02 = vfmaq_f16(s02, w0, x2); + + s10 = vfmaq_f16(s10, w1, x0); + s11 = vfmaq_f16(s11, w1, x1); + s12 = vfmaq_f16(s12, w1, x2); + + s20 = vfmaq_f16(s20, w2, x0); + s21 = vfmaq_f16(s21, w2, x1); + s22 = vfmaq_f16(s22, w2, x2); + + s30 = vfmaq_f16(s30, w3, x0); + s31 = vfmaq_f16(s31, w3, x1); + s32 = vfmaq_f16(s32, w3, x2); + + x0 = vld1q_f16(inwptr + 24); + x1 = vld1q_f16(inwptr + 32); + x2 = vld1q_f16(inwptr + 40); + + s03 = vfmaq_f16(s03, w0, x0); + s04 = vfmaq_f16(s04, w0, x1); + s05 = vfmaq_f16(s05, w0, x2); + + s13 = vfmaq_f16(s13, w1, x0); + s14 = vfmaq_f16(s14, w1, x1); + s15 = vfmaq_f16(s15, w1, x2); + + s23 = vfmaq_f16(s23, w2, x0); + s24 = vfmaq_f16(s24, w2, x1); + s25 = vfmaq_f16(s25, w2, x2); + + s33 = vfmaq_f16(s33, w3, x0); + s34 = vfmaq_f16(s34, w3, x1); + s35 = vfmaq_f16(s35, w3, x2); + } + + vst1q_f16(outbuf, s00); + vst1q_f16(outbuf + 1*64, s01); + vst1q_f16(outbuf + 2*64, s02); + vst1q_f16(outbuf + 3*64, s03); + vst1q_f16(outbuf + 4*64, s04); + vst1q_f16(outbuf + 5*64, s05); + + vst1q_f16(outbuf + 6*64, s10); + vst1q_f16(outbuf + 7*64, s11); + vst1q_f16(outbuf + 8*64, s12); + vst1q_f16(outbuf + 9*64, s13); + vst1q_f16(outbuf + 10*64, s14); + vst1q_f16(outbuf + 11*64, s15); + + vst1q_f16(outbuf + 12*64, s20); + vst1q_f16(outbuf + 13*64, s21); + vst1q_f16(outbuf + 14*64, s22); + vst1q_f16(outbuf + 15*64, s23); + vst1q_f16(outbuf + 16*64, s24); + vst1q_f16(outbuf + 17*64, s25); + + vst1q_f16(outbuf + 18*64, s30); + vst1q_f16(outbuf + 19*64, s31); + vst1q_f16(outbuf + 20*64, s32); + vst1q_f16(outbuf + 21*64, s33); + vst1q_f16(outbuf + 22*64, s34); + vst1q_f16(outbuf + 23*64, s35); + } + } + else + { + for (int atom_id = 0; atom_id < winoNatomF16; atom_id++, + outbuf += winoAtomF16) + { + float16x8_t s00 = vdupq_n_f16(0.f), s01 = s00, s02 = s00; + float16x8_t s10 = vdupq_n_f16(0.f), s11 = s00, s12 = s00; + float16x8_t s20 = vdupq_n_f16(0.f), s21 = s00, s22 = s00; + float16x8_t s30 = vdupq_n_f16(0.f), s31 = s00, s32 = s00; + + for (int c = 0; c < Cg; c++, inwptr += winoIblock*winoAtomF16, + wptr += winoKblock*winoAtomF16) + { + float16x8_t w0 = vld1q_f16(wptr), w1 = vld1q_f16(wptr + 8); + float16x8_t w2 = vld1q_f16(wptr + 16), w3 = vld1q_f16(wptr + 24); + float16x8_t x0, x1, x2; + + x0 = vld1q_f16(inwptr); + x1 = vld1q_f16(inwptr + 8); + x2 = vld1q_f16(inwptr + 16); + + s00 = vfmaq_f16(s00, w0, x0); + s01 = vfmaq_f16(s01, w0, x1); + s02 = vfmaq_f16(s02, w0, x2); + + s10 = vfmaq_f16(s10, w1, x0); + s11 = vfmaq_f16(s11, w1, x1); + s12 = vfmaq_f16(s12, w1, x2); + + s20 = vfmaq_f16(s20, w2, x0); + s21 = vfmaq_f16(s21, w2, x1); + s22 = vfmaq_f16(s22, w2, x2); + + s30 = vfmaq_f16(s30, w3, x0); + s31 = vfmaq_f16(s31, w3, x1); + s32 = vfmaq_f16(s32, w3, x2); + } + + vst1q_f16(outbuf, s00); + vst1q_f16(outbuf + 1*64, s01); + vst1q_f16(outbuf + 2*64, s02); + + vst1q_f16(outbuf + 6*64, s10); + vst1q_f16(outbuf + 7*64, s11); + vst1q_f16(outbuf + 8*64, s12); + + vst1q_f16(outbuf + 12*64, s20); + vst1q_f16(outbuf + 13*64, s21); + vst1q_f16(outbuf + 14*64, s22); + + vst1q_f16(outbuf + 18*64, s30); + vst1q_f16(outbuf + 19*64, s31); + vst1q_f16(outbuf + 20*64, s32); + } + } +} + +/*Input transform*/ +//NOTE: Since we don't have the fully fp16 support. Current work around is that we need packing the data and +// convert it to FP16 in input transform stage. And at output transform stage we will convert it back to FP32. +void winofunc_BtXB_8x8_F16(const float * inptr, int inpstep, + char * _outptr, int Cg, const int winoIblock, const int winoAtomF16) +{ + typedef __fp16 float16_t; + float16_t* outptr = (float16_t*)_outptr; float32x4_t x00 = vld1q_f32(inptr), x01 = vld1q_f32(inptr + 4); float32x4_t x10 = vld1q_f32(inptr + inpstep), x11 = vld1q_f32(inptr + inpstep + 4); float32x4_t x20 = vld1q_f32(inptr + inpstep*2), x21 = vld1q_f32(inptr + inpstep*2 + 4); @@ -577,8 +605,8 @@ void winofunc_BtXB_8x8_f32(const float* inptr, int inpstep, float32x4_t z00, z01, z10, z11, z20, z21, z30, z31, z40, z41, z50, z51, z60, z61, z70, z71; { - /* Y[0] = [1.f, 0.f, -5.25f, 0.f, 5.25f, 0.f, -1.f, 0.f]*X */ - /* Y[7] = [0.f, -1.f, 0.f, 5.25f, 0.f, -5.25f, 0.f, 1.f]*X */ + // Y[0] = [1.f, 0.f, -5.25f, 0.f, 5.25f, 0.f, -1.f, 0.f]*X + // Y[7] = [0.f, -1.f, 0.f, 5.25f, 0.f, -5.25f, 0.f, 1.f]*X float32x4_t q5_25 = vdupq_n_f32(5.25f), t00, t01, t10, t11; t00 = vsubq_f32(x40, x20); t01 = vsubq_f32(x41, x21); @@ -589,8 +617,8 @@ void winofunc_BtXB_8x8_f32(const float* inptr, int inpstep, float32x4_t y70 = vfmaq_f32(vsubq_f32(x70, x10), t10, q5_25); float32x4_t y71 = vfmaq_f32(vsubq_f32(x71, x11), t11, q5_25); - /* Y[1] = [0.f, 1.f, 1.f, -4.25f, -4.25f, 1.f, 1.f, 0.f]*X */ - /* Y[2] = [0.f, -1.f, 1.f, 4.25f, -4.25f, -1.f, 1.f, 0.f]*X */ + // Y[1] = [0.f, 1.f, 1.f, -4.25f, -4.25f, 1.f, 1.f, 0.f]*X + // Y[2] = [0.f, -1.f, 1.f, 4.25f, -4.25f, -1.f, 1.f, 0.f]*X float32x4_t qm4_25 = vdupq_n_f32(-4.25f); t00 = vfmaq_f32(vaddq_f32(x10, x50), x30, qm4_25); t01 = vfmaq_f32(vaddq_f32(x11, x51), x31, qm4_25); @@ -600,8 +628,8 @@ void winofunc_BtXB_8x8_f32(const float* inptr, int inpstep, float32x4_t y10 = vaddq_f32(t00, t10), y11 = vaddq_f32(t01, t11); float32x4_t y20 = vsubq_f32(t10, t00), y21 = vsubq_f32(t11, t01); - /* Y[3] = [0.f, 0.5f, 0.25f, -2.5f, -1.25f, 2.f, 1.f, 0.f]*X */ - /* Y[4] = [0.f, -0.5f, 0.25f, 2.5f, -1.25f, -2.f, 1.f, 0.f]*X */ + // Y[3] = [0.f, 0.5f, 0.25f, -2.5f, -1.25f, 2.f, 1.f, 0.f]*X + // Y[4] = [0.f, -0.5f, 0.25f, 2.5f, -1.25f, -2.f, 1.f, 0.f]*X float32x4_t q0_5 = vdupq_n_f32(0.5f), q0_25 = vdupq_n_f32(0.25f); float32x4_t qm2_5 = vdupq_n_f32(-2.5f), qm1_25 = vdupq_n_f32(-1.25f); t00 = vfmaq_f32(vaddq_f32(x50, x50), x10, q0_5); @@ -616,8 +644,8 @@ void winofunc_BtXB_8x8_f32(const float* inptr, int inpstep, float32x4_t y30 = vaddq_f32(t00, t10), y31 = vaddq_f32(t01, t11); float32x4_t y40 = vsubq_f32(t10, t00), y41 = vsubq_f32(t11, t01); - /* Y[5] = [0.f, 2.f, 4.f, -2.5f, -5.f, 0.5f, 1.f, 0.f]*X */ - /* Y[6] = [0.f, -2.f, 4.f, 2.5f, -5.f, -0.5f, 1.f, 0.f]*X */ + // Y[5] = [0.f, 2.f, 4.f, -2.5f, -5.f, 0.5f, 1.f, 0.f]*X + // Y[6] = [0.f, -2.f, 4.f, 2.5f, -5.f, -0.5f, 1.f, 0.f]*X float32x4_t q4 = vdupq_n_f32(4.f), qm5 = vdupq_n_f32(-5.f); t00 = vfmaq_f32(vaddq_f32(x10, x10), x50, q0_5); t01 = vfmaq_f32(vaddq_f32(x11, x11), x51, q0_5); @@ -631,22 +659,22 @@ void winofunc_BtXB_8x8_f32(const float* inptr, int inpstep, float32x4_t y50 = vaddq_f32(t00, t10), y51 = vaddq_f32(t01, t11); float32x4_t y60 = vsubq_f32(t10, t00), y61 = vsubq_f32(t11, t01); - /* transpose 8x8 matrix in-place with some renumeration of the elements: */ - /* Y: */ - /* y00 y01 */ - /* y10 y11 */ - /* ... */ - /* y70 y71 */ - /* Y': */ - /* y00 y40 */ - /* y10 y50 */ - /* y20 y60 */ - /* y30 y70 */ - /* y01 y41 */ - /* y11 y51 */ - /* y21 y61 */ - /* y31 y71 */ - /* in other words, y40 <-> y01, y50 <-> y11, y60 <-> y21, y70 <-> y31 */ + // transpose 8x8 matrix in-place with some renumeration of the elements: + // Y: + // y00 y01 + // y10 y11 + // ... + // y70 y71 + // Y': + // y00 y40 + // y10 y50 + // y20 y60 + // y30 y70 + // y01 y41 + // y11 y51 + // y21 y61 + // y31 y71 + // in other words, y40 <-> y01, y50 <-> y11, y60 <-> y21, y70 <-> y31 float32x4x2_t tr0, tr1; T4x4(y00, y10, y20, y30, tr0, tr1); @@ -654,8 +682,8 @@ void winofunc_BtXB_8x8_f32(const float* inptr, int inpstep, T4x4(y40, y50, y60, y70, tr0, tr1); T4x4(y41, y51, y61, y71, tr0, tr1); - /* Z[0] = [1.f, 0.f, -5.25f, 0.f, 5.25f, 0.f, -1.f, 0.f]*Y */ - /* Z[7] = [0.f, -1.f, 0.f, 5.25f, 0.f, -5.25f, 0.f, 1.f]*Y */ + // Z[0] = [1.f, 0.f, -5.25f, 0.f, 5.25f, 0.f, -1.f, 0.f]*Y + // Z[7] = [0.f, -1.f, 0.f, 5.25f, 0.f, -5.25f, 0.f, 1.f]*Y t00 = vsubq_f32(y01, y20); t01 = vsubq_f32(y41, y60); t10 = vsubq_f32(y30, y11); @@ -665,8 +693,8 @@ void winofunc_BtXB_8x8_f32(const float* inptr, int inpstep, z70 = vfmaq_f32(vsubq_f32(y31, y10), t10, q5_25); z71 = vfmaq_f32(vsubq_f32(y71, y50), t11, q5_25); - /* Z[1] = [0.f, 1.f, 1.f, -4.25f, -4.25f, 1.f, 1.f, 0.f]*Y */ - /* Z[2] = [0.f, -1.f, 1.f, 4.25f, -4.25f, -1.f, 1.f, 0.f]*Y */ + // Z[1] = [0.f, 1.f, 1.f, -4.25f, -4.25f, 1.f, 1.f, 0.f]*Y + // Z[2] = [0.f, -1.f, 1.f, 4.25f, -4.25f, -1.f, 1.f, 0.f]*Y t00 = vfmaq_f32(vaddq_f32(y10, y11), y30, qm4_25); t01 = vfmaq_f32(vaddq_f32(y50, y51), y70, qm4_25); t10 = vfmaq_f32(vaddq_f32(y20, y21), y01, qm4_25); @@ -675,8 +703,8 @@ void winofunc_BtXB_8x8_f32(const float* inptr, int inpstep, z10 = vaddq_f32(t00, t10); z11 = vaddq_f32(t01, t11); z20 = vsubq_f32(t10, t00); z21 = vsubq_f32(t11, t01); - /* Z[3] = [0.f, 0.5f, 0.25f, -2.5f, -1.25f, 2.f, 1.f, 0.f]*Y */ - /* Z[4] = [0.f, -0.5f, 0.25f, 2.5f, -1.25f, -2.f, 1.f, 0.f]*Y */ + // Z[3] = [0.f, 0.5f, 0.25f, -2.5f, -1.25f, 2.f, 1.f, 0.f]*Y + // Z[4] = [0.f, -0.5f, 0.25f, 2.5f, -1.25f, -2.f, 1.f, 0.f]*Y t00 = vfmaq_f32(vaddq_f32(y11, y11), y10, q0_5); t01 = vfmaq_f32(vaddq_f32(y51, y51), y50, q0_5); t10 = vfmaq_f32(y21, y20, q0_25); @@ -689,8 +717,8 @@ void winofunc_BtXB_8x8_f32(const float* inptr, int inpstep, z30 = vaddq_f32(t00, t10); z31 = vaddq_f32(t01, t11); z40 = vsubq_f32(t10, t00); z41 = vsubq_f32(t11, t01); - /* Z[5] = [0.f, 2.f, 4.f, -2.5f, -5.f, 0.5f, 1.f, 0.f]*Y */ - /* Z[6] = [0.f, -2.f, 4.f, 2.5f, -5.f, -0.5f, 1.f, 0.f]*Y */ + // Z[5] = [0.f, 2.f, 4.f, -2.5f, -5.f, 0.5f, 1.f, 0.f]*Y + // Z[6] = [0.f, -2.f, 4.f, 2.5f, -5.f, -0.5f, 1.f, 0.f]*Y t00 = vfmaq_f32(vaddq_f32(y10, y10), y11, q0_5); t01 = vfmaq_f32(vaddq_f32(y50, y50), y51, q0_5); t10 = vfmaq_f32(y21, y20, q4); @@ -704,39 +732,42 @@ void winofunc_BtXB_8x8_f32(const float* inptr, int inpstep, z60 = vsubq_f32(t10, t00); z61 = vsubq_f32(t11, t01); } - const int outstep = winoIblock*winoAtomF32*Cg; + const int outstep = winoIblock*winoAtomF16*Cg; - vst1q_f32(outptr, z00); - vst1q_f32(outptr + outstep, z01); - vst1q_f32(outptr + outstep*2, z10); - vst1q_f32(outptr + outstep*3, z11); - vst1q_f32(outptr + outstep*4, z20); - vst1q_f32(outptr + outstep*5, z21); - vst1q_f32(outptr + outstep*6, z30); - vst1q_f32(outptr + outstep*7, z31); - vst1q_f32(outptr + outstep*8, z40); - vst1q_f32(outptr + outstep*9, z41); - vst1q_f32(outptr + outstep*10, z50); - vst1q_f32(outptr + outstep*11, z51); - vst1q_f32(outptr + outstep*12, z60); - vst1q_f32(outptr + outstep*13, z61); - vst1q_f32(outptr + outstep*14, z70); - vst1q_f32(outptr + outstep*15, z71); + vst1_f16(outptr, vcvt_f16_f32(z00)); + vst1_f16(outptr + 4, vcvt_f16_f32(z01)); + vst1_f16(outptr + outstep, vcvt_f16_f32(z10)); + vst1_f16(outptr + outstep + 4, vcvt_f16_f32(z11)); + vst1_f16(outptr + outstep*2, vcvt_f16_f32(z20)); + vst1_f16(outptr + outstep*2 + 4, vcvt_f16_f32(z21)); + vst1_f16(outptr + outstep*3, vcvt_f16_f32(z30)); + vst1_f16(outptr + outstep*3 + 4, vcvt_f16_f32(z31)); + vst1_f16(outptr + outstep*4, vcvt_f16_f32(z40)); + vst1_f16(outptr + outstep*4 + 4, vcvt_f16_f32(z41)); + vst1_f16(outptr + outstep*5, vcvt_f16_f32(z50)); + vst1_f16(outptr + outstep*5 + 4, vcvt_f16_f32(z51)); + vst1_f16(outptr + outstep*6, vcvt_f16_f32(z60)); + vst1_f16(outptr + outstep*6 + 4, vcvt_f16_f32(z61)); + vst1_f16(outptr + outstep*7, vcvt_f16_f32(z70)); + vst1_f16(outptr + outstep*7 + 4, vcvt_f16_f32(z71)); } -/*Output transform*/ -void winofunc_AtXA_8x8_f32(const float* inptr, int inpstep, - float* bpptr, int bpstep, float* outptr, int outstep, - float bias, float minval, float maxval, bool ifMinMaxAct) +// Output transform +void winofunc_AtXA_8x8_F16(const char* _inptr, int inpstep, + float * bpptr, int bpstep, float* outptr, int outstep, + float bias, float minval, float maxval, bool ifMinMaxAct) { - float32x4_t x00 = vld1q_f32(inptr), x01 = vld1q_f32(inptr + 4); - float32x4_t x10 = vld1q_f32(inptr + inpstep), x11 = vld1q_f32(inptr + inpstep + 4); - float32x4_t x20 = vld1q_f32(inptr + inpstep*2), x21 = vld1q_f32(inptr + inpstep*2 + 4); - float32x4_t x30 = vld1q_f32(inptr + inpstep*3), x31 = vld1q_f32(inptr + inpstep*3 + 4); - float32x4_t x40 = vld1q_f32(inptr + inpstep*4), x41 = vld1q_f32(inptr + inpstep*4 + 4); - float32x4_t x50 = vld1q_f32(inptr + inpstep*5), x51 = vld1q_f32(inptr + inpstep*5 + 4); - float32x4_t x60 = vld1q_f32(inptr + inpstep*6), x61 = vld1q_f32(inptr + inpstep*6 + 4); - float32x4_t x70 = vld1q_f32(inptr + inpstep*7), x71 = vld1q_f32(inptr + inpstep*7 + 4); + typedef __fp16 float16_t; + const float16_t* inptr = (const float16_t*)_inptr; + + float32x4_t x00 = vcvt_f32_f16(vld1_f16(inptr)), x01 = vcvt_f32_f16(vld1_f16(inptr + 4)); + float32x4_t x10 = vcvt_f32_f16(vld1_f16(inptr + inpstep)), x11 = vcvt_f32_f16(vld1_f16(inptr + inpstep + 4)); + float32x4_t x20 = vcvt_f32_f16(vld1_f16(inptr + inpstep*2)), x21 = vcvt_f32_f16(vld1_f16(inptr + inpstep*2 + 4)); + float32x4_t x30 = vcvt_f32_f16(vld1_f16(inptr + inpstep*3)), x31 = vcvt_f32_f16(vld1_f16(inptr + inpstep*3 + 4)); + float32x4_t x40 = vcvt_f32_f16(vld1_f16(inptr + inpstep*4)), x41 = vcvt_f32_f16(vld1_f16(inptr + inpstep*4 + 4)); + float32x4_t x50 = vcvt_f32_f16(vld1_f16(inptr + inpstep*5)), x51 = vcvt_f32_f16(vld1_f16(inptr + inpstep*5 + 4)); + float32x4_t x60 = vcvt_f32_f16(vld1_f16(inptr + inpstep*6)), x61 = vcvt_f32_f16(vld1_f16(inptr + inpstep*6 + 4)); + float32x4_t x70 = vcvt_f32_f16(vld1_f16(inptr + inpstep*7)), x71 = vcvt_f32_f16(vld1_f16(inptr + inpstep*7 + 4)); float32x4_t z00, z01, z10, z11, z20, z21, z30, z31, z40, z41, z50, z51; { @@ -757,33 +788,33 @@ void winofunc_AtXA_8x8_f32(const float* inptr, int inpstep, s56_0 = vsubq_f32(x50, x60); s56_1 = vsubq_f32(x51, x61); float32x4_t y50 = vfmaq_n_f32(vfmaq_n_f32(vaddq_f32(x70, s12_0), - s34_0, 32.f), s56_0, 1.f/32); + s34_0, 32.f), s56_0, 1.f/32); float32x4_t y51 = vfmaq_n_f32(vfmaq_n_f32(vaddq_f32(x71, s12_1), - s34_1, 32.f), s56_1, 1.f/32); + s34_1, 32.f), s56_1, 1.f/32); float32x4_t y10 = vfmaq_n_f32(vfmaq_n_f32(s12_0, s34_0, 2.0f), s56_0, 0.5f); float32x4_t y11 = vfmaq_n_f32(vfmaq_n_f32(s12_1, s34_1, 2.0f), s56_1, 0.5f); float32x4_t y30 = vfmaq_n_f32(vfmaq_n_f32(s12_0, s34_0, 8.0f), s56_0, 0.125f); float32x4_t y31 = vfmaq_n_f32(vfmaq_n_f32(s12_1, s34_1, 8.0f), s56_1, 0.125f); float32x4_t y60 = vdupq_n_f32(0.f), y61 = y60, y70 = y60, y71 = y60; - /* transpose 8x8 matrix in-place with some renumeration of the elements: */ - /* Y: */ - /* y00 y01 */ - /* y10 y11 */ - /* ... */ - /* y50 y51 */ - /* 0 0 */ - /* 0 0 */ - /* Y': */ - /* y00 y40 */ - /* y10 y50 */ - /* y20 y60 */ - /* y30 y70 */ - /* y01 y41 */ - /* y11 y51 */ - /* y21 y61 */ - /* y31 y71 */ - /* in other words, y40 <-> y01, y50 <-> y11, y60 <-> y21, y70 <-> y31 */ + // transpose 8x8 matrix in-place with some renumeration of the elements: + // Y: + // y00 y01 + // y10 y11 + // ... + // y50 y51 + // 0 0 + // 0 0 + // Y': + // y00 y40 + // y10 y50 + // y20 y60 + // y30 y70 + // y01 y41 + // y11 y51 + // y21 y61 + // y31 y71 + // in other words, y40 <-> y01, y50 <-> y11, y60 <-> y21, y70 <-> y31 float32x4x2_t tr0, tr1; T4x4(y00, y10, y20, y30, tr0, tr1); @@ -807,9 +838,9 @@ void winofunc_AtXA_8x8_f32(const float* inptr, int inpstep, s56_0 = vsubq_f32(y11, y21); s56_1 = vsubq_f32(y51, y61); z50 = vfmaq_n_f32(vfmaq_n_f32(vaddq_f32(y31, s12_0), - s34_0, 32.f), s56_0, 1.f/32); + s34_0, 32.f), s56_0, 1.f/32); z51 = vfmaq_n_f32(vfmaq_n_f32(vaddq_f32(y71, s12_1), - s34_1, 32.f), s56_1, 1.f/32); + s34_1, 32.f), s56_1, 1.f/32); z10 = vfmaq_n_f32(vfmaq_n_f32(s12_0, s34_0, 2.0f), s56_0, 0.5f); z11 = vfmaq_n_f32(vfmaq_n_f32(s12_1, s34_1, 2.0f), s56_1, 0.5f); z30 = vfmaq_n_f32(vfmaq_n_f32(s12_0, s34_0, 8.0f), s56_0, 0.125f); @@ -879,8 +910,8 @@ void winofunc_AtXA_8x8_f32(const float* inptr, int inpstep, vst1q_f32(outptr + outstep*5, z50); vst1_f32(outptr + outstep*5 + 4, vget_low_f32(z51)); } - #endif -} +#endif +CV_CPU_OPTIMIZATION_NAMESPACE_END }} // namespace diff --git a/modules/dnn/src/layers/cpu_kernels/convolution.cpp b/modules/dnn/src/layers/cpu_kernels/convolution.cpp index 68777ff65e..649fb88211 100644 --- a/modules/dnn/src/layers/cpu_kernels/convolution.cpp +++ b/modules/dnn/src/layers/cpu_kernels/convolution.cpp @@ -14,15 +14,76 @@ #include "conv_block.simd.hpp" #include "layers/cpu_kernels/conv_block.simd_declarations.hpp" // defines CV_CPU_DISPATCH_MODES_ALL=AVX2,...,BASELINE based on CMakeLists.txt content +#include namespace cv { namespace dnn { -enum { VEC_ALIGN = 32, DFT_TYPE = CV_32F }; // Memory alignment. +enum { VEC_ALIGN = 32}; // Memory alignment. -void convBlock(int np, const float* a, const float* b, float* c, int ldc, bool init_c, const int outLen, +void convBlock_F32(int np, const float* a, const float* b, float* c, int ldc, bool init_c, const int outLen, const int convMR, const int convNR); -void convBlockMR1(int np, const float* a, const float* b, float *c, const float bias, bool init_c, +void convBlockMR1_F32(int np, const float* a, const float* b, float *c, const float bias, bool init_c, const float minval, const float maxval, bool ifMinMaxAct, const int outLen, const int convNR); +#ifdef CONV_ARM_FP16 +// Fast convert float 32 to float16 +static inline void _cvt32f16f(const float* src, float16_t* dst, int len) +{ + int j = 0; + const int VECSZ = 4; + __fp16* dst_FP16 = (__fp16 *)dst; + if (len > VECSZ * 4) + { + const int VECSZ4 = 4 * VECSZ; + for( ; j + VECSZ4 < len; j += VECSZ4) + { + + float32x4_t v0 = vld1q_f32(src + j); + float32x4_t v1 = vld1q_f32(src + j + 4); + float32x4_t v2 = vld1q_f32(src + j + 8); + float32x4_t v3 = vld1q_f32(src + j + 12); + + vst1q_f16(dst_FP16 + j, vcombine_f16(vcvt_f16_f32(v0), vcvt_f16_f32(v1))); + vst1q_f16(dst_FP16 + j + 8, vcombine_f16(vcvt_f16_f32(v2), vcvt_f16_f32(v3))); + } + } + + for( ; j < len; j += VECSZ ) + { + if( j > len - VECSZ ) + { + if( j == 0 ) + break; + j = len - VECSZ; + } + + float16x4_t hv = vcvt_f16_f32(vld1q_f32(src + j)); + vst1_f16(dst_FP16 + j, hv); + } + for( ; j < len; j++ ) + dst[j] = float16_t(src[j]); +} +#endif + +float* FastConv::getWeights() +{ + return alignPtr(weightsBuf.data(), VEC_ALIGN); +} + +float* FastConv::getWeightsWino() +{ + return alignPtr(weightsWinoBuf.data(), VEC_ALIGN); +} + +float16_t* FastConv::getWeightsFP16() +{ + return alignPtr(weightsBuf_FP16.data(), VEC_ALIGN); +} + +float16_t* FastConv::getWeightsWinoFP16() +{ + return alignPtr(weightsWinoBuf_FP16.data(), VEC_ALIGN); +} + Ptr initFastConv( InputArray _weightsMat, float* srcBias, @@ -119,9 +180,16 @@ Ptr initFastConv( conv->useFP16 = false; #ifdef CONV_ARM_FP16 - // TODO: add FP16 support for Winograd. - if (_useFP16 && (conv->conv_type == CONV_TYPE_GENERIC || conv->conv_type == CONV_TYPE_DEPTHWISE_REMAIN)) + if (_useFP16 && (conv->conv_type == CONV_TYPE_GENERIC || conv->conv_type == CONV_TYPE_DEPTHWISE_REMAIN + || conv->conv_type == CONV_TYPE_WINOGRAD3X3)) conv->useFP16 = true; + + // Runtime FP16 check. + if (conv->useFP16 && !checkHardwareSupport(CPU_NEON_FP16)) + { + conv->useFP16 = false; + CV_LOG_ONCE_WARNING(NULL, "DNN: the CPU does not support the instruction set required by FP16, fallback to FP32."); + } #endif float *srcWeights = (float *)weightsMat.data; @@ -141,31 +209,25 @@ Ptr initFastConv( if (conv->useFP16) { conv->weightsBuf_FP16.resize(nweights + VEC_ALIGN); - conv->weightsBufPtr_FP16 = alignPtr(conv->weightsBuf_FP16.data(), VEC_ALIGN * sizeof(float16_t )); - memset(conv->weightsBufPtr_FP16, 0, nweights * sizeof(float16_t )); - auto weightsBufPtr_FP16 = conv->weightsBufPtr_FP16; + auto weightsPtr_FP16 = conv->getWeightsFP16(); + memset(reinterpret_cast(weightsPtr_FP16), 0, nweights * sizeof(weightsPtr_FP16[0])); parallel_for_(Range(0, C), [&](const Range& r0){ - for(int c = r0.start; c < r0.end; c++) - { - for (int k = 0; k < ksize; k++) - weightsBufPtr_FP16[c*padded_ksize + k] = (float16_t)srcWeights[c*wstep + k]; - }}); + for(int c = r0.start; c < r0.end; c++) + _cvt32f16f(srcWeights + c*wstep, weightsPtr_FP16 + c*padded_ksize, ksize); + }); } else #endif { conv->weightsBuf.resize(nweights + VEC_ALIGN); - conv->weightsBufPtr = alignPtr(conv->weightsBuf.data(), VEC_ALIGN * sizeof(float )); - memset(conv->weightsBufPtr, 0, nweights*sizeof(float )); - auto weightsBufPtr = conv->weightsBufPtr; + auto weightsPtr = conv->getWeights(); + memset(weightsPtr, 0, nweights*sizeof(weightsPtr[0])); - parallel_for_(Range(0, C), [&](const Range& r0){ - for(int c = r0.start; c < r0.end; c++) - { - for (int k = 0; k < ksize; k++) - weightsBufPtr[c*padded_ksize + k] = srcWeights[c*wstep + k]; - }}); + parallel_for_(Range(0, C), [&](const Range& r0) { + for(int c = r0.start; c < r0.end; c++) + memcpy(weightsPtr + c*padded_ksize, srcWeights + c*wstep, ksize*sizeof(weightsPtr[0])); + }); } } else if(conv->conv_type == CONV_TYPE_WINOGRAD3X3) // winograd @@ -213,16 +275,14 @@ Ptr initFastConv( if (conv->useFP16) { conv->weightsWinoBuf_FP16.resize(nweights + VEC_ALIGN); - conv->weightsWinoBufPtr_FP16 = alignPtr(conv->weightsWinoBuf_FP16.data(), VEC_ALIGN); - wptrWino_FP16 = conv->weightsWinoBufPtr_FP16; - memset(wptrWino_FP16, 0, nweights * sizeof(wptrWino_FP16[0])); + wptrWino_FP16 = conv->getWeightsWinoFP16(); + memset(reinterpret_cast(wptrWino_FP16), 0, nweights * sizeof(wptrWino_FP16[0])); } else #endif { conv->weightsWinoBuf.resize(nweights + VEC_ALIGN); - conv->weightsWinoBufPtr = alignPtr(conv->weightsWinoBuf.data(), VEC_ALIGN); - wptrWino = conv->weightsWinoBufPtr; + wptrWino = conv->getWeightsWino(); memset(wptrWino, 0, nweights * sizeof(wptrWino[0])); } @@ -272,7 +332,7 @@ Ptr initFastConv( for (int i = 0; i < CONV_WINO_NATOMS_F16; i++, wptr += Cg * CONV_WINO_KBLOCK * CONV_WINO_ATOM_F16) { - CV_Assert(conv->weightsWinoBufPtr_FP16 <= wptr && wptr + CONV_WINO_ATOM_F16 <= conv->weightsWinoBufPtr_FP16 + nweights); + CV_Assert(wptrWino_FP16 <= wptr && wptr + CONV_WINO_ATOM_F16 <= wptrWino_FP16 + nweights); for (int j = 0; j < CONV_WINO_ATOM_F16; j++) { wptr[j] = (float16_t)kernelTm[i * CONV_WINO_ATOM_F16 + j]; @@ -287,7 +347,7 @@ Ptr initFastConv( for (int i = 0; i < CONV_WINO_NATOMS_F32; i++, wptr += Cg * CONV_WINO_KBLOCK * CONV_WINO_ATOM_F32) { - CV_Assert(conv->weightsWinoBufPtr <= wptr && wptr + CONV_WINO_ATOM_F32 <= conv->weightsWinoBufPtr + nweights); + CV_Assert(wptrWino <= wptr && wptr + CONV_WINO_ATOM_F32 <= wptrWino + nweights); memcpy(wptr, kernelTm + i * CONV_WINO_ATOM_F32, CONV_WINO_ATOM_F32*sizeof (wptr[0])); } } @@ -305,29 +365,26 @@ Ptr initFastConv( int numStripsMR = (Kg + CONV_MR_FP32 - 1) / CONV_MR_FP32; int Kg_aligned = numStripsMR * CONV_MR_FP32; size_t nweights = ngroups*Kg_aligned*DkHkWkCg; - - float* weightsBufPtr = nullptr; + float* weightsPtr = nullptr; #ifdef CONV_ARM_FP16 int numStripsMR_FP16 = (Kg + CONV_MR_FP16 - 1) / CONV_MR_FP16; int Kg_aligned_FP16 = numStripsMR_FP16 * CONV_MR_FP16; size_t nweights_FP16 = ngroups * Kg_aligned_FP16 * DkHkWkCg; + float16_t* weightsPtr_FP16 = nullptr; - float16_t* weightsBufPtr_FP16 = nullptr; if (conv->useFP16) { conv->weightsBuf_FP16.resize(nweights_FP16 + VEC_ALIGN); - conv->weightsBufPtr_FP16 = alignPtr(conv->weightsBuf_FP16.data(), VEC_ALIGN); - weightsBufPtr_FP16 = conv->weightsBufPtr_FP16; - memset(weightsBufPtr_FP16, 0, nweights_FP16*sizeof(weightsBufPtr_FP16[0])); + weightsPtr_FP16 = conv->getWeightsFP16(); + memset(reinterpret_cast(weightsPtr_FP16), 0, nweights_FP16*sizeof(weightsPtr_FP16[0])); } else #endif { conv->weightsBuf.resize(nweights + VEC_ALIGN); - conv->weightsBufPtr = alignPtr(conv->weightsBuf.data(), VEC_ALIGN); - weightsBufPtr = conv->weightsBufPtr; - memset(weightsBufPtr, 0, nweights*sizeof(weightsBufPtr[0])); + weightsPtr = conv->getWeights(); + memset(weightsPtr, 0, nweights*sizeof(weightsPtr[0])); } // Pack the weight. @@ -343,7 +400,7 @@ Ptr initFastConv( int startK = si * CONV_MR_FP16; CV_Assert(startK < Kg_aligned_FP16); - float16_t* packed_wptr = weightsBufPtr_FP16 + DkHkWkCg * (startK + g * Kg_aligned_FP16); + float16_t* packed_wptr = weightsPtr_FP16 + DkHkWkCg * (startK + g * Kg_aligned_FP16); int dk = Kg - startK < CONV_MR_FP16 ? Kg - startK : CONV_MR_FP16; // check if we need zero padding. int k_idx = g*Kg + startK; @@ -373,7 +430,7 @@ Ptr initFastConv( int startK = si * CONV_MR_FP32; CV_Assert(startK < Kg_aligned); - float* packed_wptr = weightsBufPtr + DkHkWkCg * (startK + g * Kg_aligned); + float* packed_wptr = weightsPtr + DkHkWkCg * (startK + g * Kg_aligned); int dk = Kg - startK < CONV_MR_FP32 ? Kg - startK : CONV_MR_FP32; // check if we need zero padding. int k_idx = g*Kg + startK; @@ -410,7 +467,7 @@ Ptr initFastConv( } static inline void packData8(char*& inpbuf, float*& inptrIn, int& in_w, int& x0, int& s0, const int* ofstab, - const int stride_w, const int ksize, const int esz) + const int stride_w, const int ksize, const int esz) { char * inpbufC = inpbuf + s0 * esz; float* inptrInC = (float* )inptrIn; @@ -516,7 +573,7 @@ static inline void packData8(char*& inpbuf, float*& inptrIn, int& in_w, int& x0, } static inline void packData2(char *& inpbuf, float*& inptrIn, int& in_w, int& x0, int& s0, const int* ofstab, - const int stride_w, const int ksize, const int esz) + const int stride_w, const int ksize, const int esz) { char* inpbufC = inpbuf + s0 * esz; float* inptrInC = inptrIn; @@ -553,46 +610,6 @@ static inline void packData2(char *& inpbuf, float*& inptrIn, int& in_w, int& x0 in_w += stride_w; } -#ifdef CONV_ARM_FP16 -// Fast convert float 32 to float16 -static inline void _cvt32f16f( const float* src, float16_t* dst, int len) -{ - int j = 0; - const int VECSZ = 4; - __fp16* dst_FP16 = (__fp16 *)dst; - if (len > VECSZ * 4) - { - const int VECSZ4 = 4 * VECSZ; - for( ; j + VECSZ4 < len; j += VECSZ4) - { - - float32x4_t v0 = vld1q_f32(src + j); - float32x4_t v1 = vld1q_f32(src + j + 4); - float32x4_t v2 = vld1q_f32(src + j + 8); - float32x4_t v3 = vld1q_f32(src + j + 12); - - vst1q_f16(dst_FP16 + j, vcombine_f16(vcvt_f16_f32(v0), vcvt_f16_f32(v1))); - vst1q_f16(dst_FP16 + j + 8, vcombine_f16(vcvt_f16_f32(v2), vcvt_f16_f32(v3))); - } - } - - for( ; j < len; j += VECSZ ) - { - if( j > len - VECSZ ) - { - if( j == 0 ) - break; - j = len - VECSZ; - } - - float16x4_t hv = vcvt_f16_f32(vld1q_f32(src + j)); - vst1_f16(dst_FP16 + j, hv); - } - for( ; j < len; j++ ) - dst[j] = float16_t(src[j]); -} -#endif - static inline void packInputData(char* inpbuf_task, float* inp, const int* ofstab, const int* dhwTab, int zyx0, int zyx_limit, int ksize, int stride_d, int stride_h, int stride_w, int pad_front, int pad_top, int pad_left, int Dk, int Hk, int Wk, int dilation_d, int dilation_h, int dilation_w, int Di, int Hi, int Wi, @@ -1174,10 +1191,9 @@ void runFastConv(InputArray _input, OutputArray _output, const Ptr& co else activ = nullptr; - // TODO: support FP16 for winograd. if (conv->conv_type == CONV_TYPE_WINOGRAD3X3) // winograd { - CV_Assert(conv->weightsWinoBufPtr && input.dims == 4 && conv_dim == CONV_2D && !useFP16); + CV_Assert((!conv->weightsWinoBuf.empty() || !conv->weightsWinoBuf_FP16.empty()) && input.dims == 4 && conv_dim == CONV_2D); if (runWinograd63(input, fusedAddMat, output, conv, ntasks, minval, maxval, activ, ifMinMaxAct)) return; } @@ -1437,13 +1453,13 @@ void runFastConv(InputArray _input, OutputArray _output, const Ptr& co if (useFP16) { CV_Assert(!conv->weightsBuf_FP16.empty()); - weights = (char *)conv->weightsBufPtr_FP16; + weights = (char *)conv->getWeightsFP16(); } else #endif { CV_Assert(!conv->weightsBuf.empty()); - weights = (char *)conv->weightsBufPtr; + weights = (char *)conv->getWeights(); } // optional branch, only for depth-wise convolution which was implemented by generic convolution. // In this case, CONV_MR is 1, and CONV_NR remains the same. @@ -1477,7 +1493,7 @@ void runFastConv(InputArray _input, OutputArray _output, const Ptr& co #ifdef CONV_ARM_FP16 if (useFP16) { - opt_NEON::convBlockMR1_FP16(DkHkWkCg, weights, inptr, cptr, biasVal, fusedAdd, minval, maxval, ifMinMaxAct, outLen, CONV_NR); + opt_NEON_FP16::convBlockMR1_F16(DkHkWkCg, weights, inptr, cptr, biasVal, fusedAdd, minval, maxval, ifMinMaxAct, outLen, CONV_NR); } else #endif @@ -1485,7 +1501,7 @@ void runFastConv(InputArray _input, OutputArray _output, const Ptr& co } else #endif - convBlockMR1(DkHkWkCg, (const float *)weights, (const float *)inptr, cptr, biasVal, fusedAdd, minval, maxval, ifMinMaxAct, outLen, CONV_NR); + convBlockMR1_F32(DkHkWkCg, (const float *)weights, (const float *)inptr, cptr, biasVal, fusedAdd, minval, maxval, ifMinMaxAct, outLen, CONV_NR); if (ifBuffer) { @@ -1526,12 +1542,12 @@ void runFastConv(InputArray _input, OutputArray _output, const Ptr& co { #if CV_TRY_AVX2 if (conv->useAVX2) - opt_AVX2::convBlock(c1 - c0, (const float *)wptr, (const float *)inptr, cptr, ldc, c0 == 0, outLen, CONV_MR, CONV_NR); + opt_AVX2::convBlock_F32(c1 - c0, (const float *)wptr, (const float *)inptr, cptr, ldc, c0 == 0, outLen, CONV_MR, CONV_NR); else #endif #if CV_TRY_AVX if (conv->useAVX) - opt_AVX::convBlock(c1 - c0, (const float *)wptr, (const float *)inptr, cptr, ldc, c0 == 0, outLen, CONV_MR, CONV_NR); + opt_AVX::convBlock_F32(c1 - c0, (const float *)wptr, (const float *)inptr, cptr, ldc, c0 == 0, outLen, CONV_MR, CONV_NR); else #endif #if CV_NEON @@ -1540,16 +1556,16 @@ void runFastConv(InputArray _input, OutputArray _output, const Ptr& co #ifdef CONV_ARM_FP16 if (useFP16) { - opt_NEON::convBlock_FP16(c1 - c0, wptr, inptr, (char *)cptr_f16, ldc, c0 == 0, outLen, CONV_MR, CONV_NR); + opt_NEON_FP16::convBlock_F16(c1 - c0, wptr, inptr, (char *)cptr_f16, ldc, c0 == 0, outLen, CONV_MR, CONV_NR); } else #endif - opt_NEON::convBlock(c1 - c0, (const float *)wptr, (const float *)inptr, cptr, ldc, c0 == 0, outLen, CONV_MR, CONV_NR); + opt_NEON::convBlock_F32(c1 - c0, (const float *)wptr, (const float *)inptr, cptr, ldc, c0 == 0, outLen, CONV_MR, CONV_NR); } else #endif // The possible outLen range is 24 or 8~1. - convBlock(c1 - c0, (const float *)wptr, (const float *)inptr, cptr, ldc, c0 == 0, outLen, CONV_MR, CONV_NR); + convBlock_F32(c1 - c0, (const float *)wptr, (const float *)inptr, cptr, ldc, c0 == 0, outLen, CONV_MR, CONV_NR); } } } @@ -1838,7 +1854,7 @@ static inline void convBlockMR1x12(int np, const float* a, const float* b, float } #endif -void convBlockMR1(int np, const float* a, const float* b, float *c, const float bias, bool init_c, +void convBlockMR1_F32(int np, const float* a, const float* b, float *c, const float bias, bool init_c, const float minval, const float maxval, bool ifMinMaxAct, const int outLen, const int convNR) { #if CV_SIMD128 @@ -2088,7 +2104,7 @@ static inline void convBlockNoSIMD(int np, const float* a, const float* b, float } } -void convBlock(int np, const float* a, const float* b, float* c, int ldc, bool init_c, const int outLen, +void convBlock_F32(int np, const float* a, const float* b, float* c, int ldc, bool init_c, const int outLen, const int convMR, const int convNR) { // The possible outLen range is [24, 8~1]. diff --git a/modules/dnn/src/layers/cpu_kernels/convolution.hpp b/modules/dnn/src/layers/cpu_kernels/convolution.hpp index 5effdc2d0c..e3ba4a83a4 100644 --- a/modules/dnn/src/layers/cpu_kernels/convolution.hpp +++ b/modules/dnn/src/layers/cpu_kernels/convolution.hpp @@ -14,7 +14,7 @@ #define CONV_NR_FP32 28 // The FP16 can only be supported by ARM64 and with FP16 FMA supported. -#if defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) && CV_FP16 // check FP16 FMA. +#if CV_FP16 // check FP16 FMA. #define CONV_ARM_FP16 1 #endif @@ -22,7 +22,6 @@ // Currently, only ARM 64 support FP16. #define CONV_MR_FP16 8 #define CONV_NR_FP16 24 -typedef __fp16 float16_t; // Fix conflict between float16_t in arm_neon.h and float16_t in cvdef.h. #endif #elif CV_NEON // 16 registers. @@ -58,17 +57,15 @@ struct FastConv int pad_top, pad_bottom, pad_left, pad_right, pad_front, pad_behind; std::vector weightsBuf; // For generic Conv 2D - float* weightsBufPtr; std::vector weightsWinoBuf; // For Winograd F(6x6, 3x3). - float* weightsWinoBufPtr; std::vector biasBuf; + float* getWeights(); + float* getWeightsWino(); -#if CV_NEON && CV_NEON_AARCH64 && CV_FP16 std::vector weightsBuf_FP16; - float16_t* weightsBufPtr_FP16; std::vector weightsWinoBuf_FP16; - float16_t* weightsWinoBufPtr_FP16; -#endif + float16_t* getWeightsFP16(); + float16_t* getWeightsWinoFP16(); int conv_type; int conv_dim; // Flag for conv1d, conv2d, or conv3d. @@ -115,6 +112,32 @@ void runDepthwise(InputArray _input, OutputArray _output, const Ptr& c int runWinograd63(InputArray _input, InputArray _fusedAddMat, OutputArray _output, const Ptr& conv, int ntasks, float minval, float maxval, ActivationLayer* activ, bool ifMinMaxAct); +// Work around of NEON, the following functions are only used internally. +namespace opt_NEON { +#if CV_NEON +void convBlock_F32(int np, const float* a, const float* b, float* c, int ldc, bool init_c, int width, const int convMR, const int convNR); + +void convBlockMR1_F32(int np, const float* a, const float* b, float* c, const float bias, bool init_c, + const float minval, const float maxval, bool ifMinMaxAct, const int width, const int convNR); + +#if CV_NEON_AARCH64 +/* Accumulate */ +void winofunc_accum_F32(const float* inwptr, const float* wptr, float* outbuf, int Cg, int iblock, + const int winoIblock, const int winoKblock, const int winoAtom, const int winoNatom); + +/*Input transform*/ +void winofunc_BtXB_8x8_F32(const float* inptr, int inpstep, + float* outptr, int Cg, const int winoIblock, const int winoAtom); + +/*Output transform*/ +void winofunc_AtXA_8x8_F32(const float* inptr, int inpstep, + float* bpptr, int bpstep, float* outptr, int outstep, + float bias, float minval, float maxval, bool ifMinMaxAct); +#endif // CV_NEON_AARCH64 +#endif // CV_NEON +} // namespace opt_NEON. + + } // namespace dnn } // namespace cv diff --git a/modules/dnn/src/model.cpp b/modules/dnn/src/model.cpp index 64b2706d38..bc8e2ebe33 100644 --- a/modules/dnn/src/model.cpp +++ b/modules/dnn/src/model.cpp @@ -37,6 +37,7 @@ public: virtual void setPreferableBackend(Backend backendId) { net.setPreferableBackend(backendId); } virtual void setPreferableTarget(Target targetId) { net.setPreferableTarget(targetId); } + virtual void enableWinograd(bool useWinograd) { net.enableWinograd(useWinograd); } virtual void initNet(const Net& network) @@ -151,6 +152,7 @@ Model& Model::setPreferableBackend(Backend backendId) impl->setPreferableBackend(backendId); return *this; } + Model& Model::setPreferableTarget(Target targetId) { CV_DbgAssert(impl); @@ -158,6 +160,13 @@ Model& Model::setPreferableTarget(Target targetId) return *this; } +Model& Model::enableWinograd(bool useWinograd) +{ + CV_DbgAssert(impl); + impl->enableWinograd(useWinograd); + return *this; +} + Model& Model::setInputSize(const Size& size) { CV_DbgAssert(impl); diff --git a/modules/dnn/test/test_backends.cpp b/modules/dnn/test/test_backends.cpp index 368f1ca27c..289d3e683d 100644 --- a/modules/dnn/test/test_backends.cpp +++ b/modules/dnn/test/test_backends.cpp @@ -49,7 +49,10 @@ public: net.setInput(inp); net.setPreferableBackend(backend); net.setPreferableTarget(target); - net.enableWinograd(useWinograd); + + if (target == DNN_TARGET_CPU_FP16) + net.enableWinograd(false); + if (backend == DNN_BACKEND_HALIDE && !halideScheduler.empty()) { halideScheduler = findDataFile(halideScheduler); @@ -545,7 +548,7 @@ TEST_P(DNNTestNetwork, FastNeuralStyle_eccv16) else if (target == DNN_TARGET_CPU_FP16) { l1 = 0.4; - lInf = 19.; + lInf = 22.; } else if (target == DNN_TARGET_VULKAN) { diff --git a/modules/dnn/test/test_caffe_importer.cpp b/modules/dnn/test/test_caffe_importer.cpp index 14c3ffc6d4..34b84622fa 100644 --- a/modules/dnn/test/test_caffe_importer.cpp +++ b/modules/dnn/test/test_caffe_importer.cpp @@ -62,6 +62,10 @@ public: findDataFile("dnn/" + model, false)); net.setPreferableBackend(backend); net.setPreferableTarget(target); + + if (target == DNN_TARGET_CPU_FP16) + net.enableWinograd(false); + Mat img = imread(findDataFile("dnn/dog416.png")); resize(img, img, Size(800, 600)); Mat blob = blobFromImage(img, 1.0, Size(), Scalar(102.9801, 115.9465, 122.7717), false, false); @@ -219,6 +223,9 @@ TEST_P(Reproducibility_AlexNet, Accuracy) net.setPreferableBackend(DNN_BACKEND_OPENCV); net.setPreferableTarget(targetId); + if (targetId == DNN_TARGET_CPU_FP16) + net.enableWinograd(false); + Mat sample = imread(_tf("grace_hopper_227.png")); ASSERT_TRUE(!sample.empty()); @@ -383,6 +390,9 @@ TEST_P(Reproducibility_ResNet50, Accuracy) net.setPreferableBackend(DNN_BACKEND_OPENCV); net.setPreferableTarget(targetId); + if (targetId == DNN_TARGET_CPU_FP16) + net.enableWinograd(false); + float l1 = (targetId == DNN_TARGET_OPENCL_FP16 || targetId == DNN_TARGET_CPU_FP16) ? 3e-5 : 1e-5; float lInf = (targetId == DNN_TARGET_OPENCL_FP16 || targetId == DNN_TARGET_CPU_FP16) ? 6e-3 : 1e-4; @@ -503,6 +513,10 @@ TEST_P(Test_Caffe_nets, Colorization) net.setPreferableBackend(backend); net.setPreferableTarget(target); + // This model has bad accuracy when the FP16 and Winograd are enable at same time. + if (target == DNN_TARGET_CPU_FP16) + net.enableWinograd(false); + net.getLayer(net.getLayerId("class8_ab"))->blobs.push_back(kernel); net.getLayer(net.getLayerId("conv8_313_rh"))->blobs.push_back(Mat(1, 313, CV_32F, 2.606)); @@ -568,10 +582,15 @@ TEST_P(Test_Caffe_nets, DenseNet_121) { l1 = 0.11; lInf = 0.5; } - else if (target == DNN_TARGET_CUDA_FP16 || target == DNN_TARGET_CPU_FP16) + else if (target == DNN_TARGET_CUDA_FP16) { l1 = 0.04; lInf = 0.2; } + else if (target == DNN_TARGET_CPU_FP16) + { + l1 = 0.06; lInf = 0.3; + } + normAssert(outs[0], ref, "", l1, lInf); if (target != DNN_TARGET_MYRIAD || getInferenceEngineVPUType() != CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_X) expectNoFallbacksFromIE(model.getNetwork_()); diff --git a/modules/dnn/test/test_darknet_importer.cpp b/modules/dnn/test/test_darknet_importer.cpp index 2c734b4492..24f1056bb8 100644 --- a/modules/dnn/test/test_darknet_importer.cpp +++ b/modules/dnn/test/test_darknet_importer.cpp @@ -473,7 +473,8 @@ TEST_P(Test_Darknet_nets, TinyYoloVoc) 1, 6, 0.928758f, 0.651024f, 0.463539f, 0.823784f, 0.654998f); // a car double scoreDiff = 8e-5, iouDiff = 3e-4; - if (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD || target == DNN_TARGET_CPU_FP16) + bool useWinograd = true; + if (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) { scoreDiff = 8e-3; iouDiff = 0.018; @@ -483,18 +484,24 @@ TEST_P(Test_Darknet_nets, TinyYoloVoc) scoreDiff = 0.008; iouDiff = 0.02; } + else if (target == DNN_TARGET_CPU_FP16) + { + useWinograd = false; + scoreDiff = 8e-3; + iouDiff = 0.018; + } std::string config_file = "tiny-yolo-voc.cfg"; std::string weights_file = "tiny-yolo-voc.weights"; { SCOPED_TRACE("batch size 1"); - testDarknetModel(config_file, weights_file, ref.rowRange(0, 2), scoreDiff, iouDiff); + testDarknetModel(config_file, weights_file, ref.rowRange(0, 2), scoreDiff, iouDiff, 0.24, 0.4, useWinograd); } { SCOPED_TRACE("batch size 2"); - testDarknetModel(config_file, weights_file, ref, scoreDiff, iouDiff); + testDarknetModel(config_file, weights_file, ref, scoreDiff, iouDiff, 0.24, 0.4, useWinograd); } } @@ -890,12 +897,12 @@ TEST_P(Test_Darknet_nets, YOLOv4_tiny) { SCOPED_TRACE("batch size 1"); - testDarknetModel(config_file, weights_file, ref.rowRange(0, N0), scoreDiff, iouDiff, confThreshold); + testDarknetModel(config_file, weights_file, ref.rowRange(0, N0), scoreDiff, iouDiff, confThreshold, 0.4, false); } { SCOPED_TRACE("batch size 2"); - testDarknetModel(config_file, weights_file, ref, scoreDiff, iouDiff, confThreshold); + testDarknetModel(config_file, weights_file, ref, scoreDiff, iouDiff, confThreshold, 0.4, false); } #if defined(INF_ENGINE_RELEASE) diff --git a/modules/dnn/test/test_model.cpp b/modules/dnn/test/test_model.cpp index 59b51c4bc0..f891d55ab5 100644 --- a/modules/dnn/test/test_model.cpp +++ b/modules/dnn/test/test_model.cpp @@ -40,6 +40,8 @@ public: model.setPreferableTarget(target); model.setNmsAcrossClasses(nmsAcrossClasses); + if (target == DNN_TARGET_CPU_FP16) + model.enableWinograd(false); std::vector classIds; std::vector confidences; diff --git a/modules/dnn/test/test_onnx_importer.cpp b/modules/dnn/test/test_onnx_importer.cpp index aa60cea890..8246ff7dec 100644 --- a/modules/dnn/test/test_onnx_importer.cpp +++ b/modules/dnn/test/test_onnx_importer.cpp @@ -55,7 +55,7 @@ public: void testONNXModels(const String& basename, const Extension ext = npy, double l1 = 0, double lInf = 0, const bool useSoftmax = false, bool checkNoFallbacks = true, int numInps = 1, - bool testShapes = true) + bool testShapes = true, bool useWinograd = true) { String onnxmodel = _tf("models/" + basename + ".onnx", required); std::vector inps(numInps); @@ -82,6 +82,7 @@ public: net.setPreferableBackend(backend); net.setPreferableTarget(target); + net.enableWinograd(useWinograd); std::vector inputNames; for (int i = 0; i < numInps; ++i) @@ -1929,7 +1930,9 @@ TEST_P(Test_ONNX_layers, ConvResizePool1d) #endif } #endif - testONNXModels("conv_resize_pool_1d"); + + const double lInf = (target == DNN_TARGET_CPU_FP16) ? 0.024 : default_lInf; + testONNXModels("conv_resize_pool_1d", npy, default_l1, lInf); } TEST_P(Test_ONNX_layers, DepthWiseAdd) @@ -2130,6 +2133,7 @@ TEST_P(Test_ONNX_nets, Alexnet) net.setPreferableBackend(backend); net.setPreferableTarget(target); + net.enableWinograd(false); Mat inp = imread(_tf("../grace_hopper_227.png")); Mat ref = blobFromNPY(_tf("../caffe_alexnet_prob.npy")); @@ -2202,6 +2206,9 @@ TEST_P(Test_ONNX_nets, Googlenet) net.setPreferableBackend(backend); net.setPreferableTarget(target); + if (target == DNN_TARGET_CPU_FP16) + net.enableWinograd(false); + std::vector images; images.push_back( imread(_tf("../googlenet_0.png")) ); images.push_back( imread(_tf("../googlenet_1.png")) ); @@ -2346,7 +2353,7 @@ TEST_P(Test_ONNX_nets, TinyYolov2) } #endif - testONNXModels("tiny_yolo2", pb, l1, lInf); + testONNXModels("tiny_yolo2", pb, l1, lInf, false, true, 1, true, false); } TEST_P(Test_ONNX_nets, CNN_MNIST) @@ -2391,6 +2398,7 @@ TEST_P(Test_ONNX_nets, LResNet100E_IR) double l1 = default_l1, lInf = default_lInf; // output range: [-3; 3] + bool useWinograd = true; if (backend == DNN_BACKEND_OPENCV && target == DNN_TARGET_OPENCL_FP16) { l1 = 0.009; @@ -2406,7 +2414,14 @@ TEST_P(Test_ONNX_nets, LResNet100E_IR) l1 = 0.009; lInf = 0.04; } - testONNXModels("LResNet100E_IR", pb, l1, lInf); + else if (target == DNN_TARGET_CPU_FP16) + { + useWinograd = false; + l1 = 0.009; + lInf = 0.035; + } + + testONNXModels("LResNet100E_IR", pb, l1, lInf, false, true, 1, true, useWinograd); } TEST_P(Test_ONNX_nets, Emotion_ferplus) @@ -2421,7 +2436,7 @@ TEST_P(Test_ONNX_nets, Emotion_ferplus) double l1 = default_l1; double lInf = default_lInf; - + bool useWinograd = true; // Output values are in range [-2.011, 2.111] if ((backend == DNN_BACKEND_OPENCV && target == DNN_TARGET_OPENCL_FP16) || (target == DNN_TARGET_CUDA_FP16)) l1 = 0.007; @@ -2434,6 +2449,11 @@ TEST_P(Test_ONNX_nets, Emotion_ferplus) l1 = 2.4e-4; lInf = 6e-4; } + else if (backend == DNN_BACKEND_OPENCV && target == DNN_TARGET_CPU_FP16) + { + useWinograd = false; + l1 = 0.007; + } #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_GE(2020040000) if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target == DNN_TARGET_OPENCL_FP16) { @@ -2441,7 +2461,7 @@ TEST_P(Test_ONNX_nets, Emotion_ferplus) } #endif - testONNXModels("emotion_ferplus", pb, l1, lInf); + testONNXModels("emotion_ferplus", pb, l1, lInf, false, true, 1, true, useWinograd); } TEST_P(Test_ONNX_nets, Inception_v2) diff --git a/modules/dnn/test/test_tf_importer.cpp b/modules/dnn/test/test_tf_importer.cpp index 1d36cddc23..c2301c726a 100644 --- a/modules/dnn/test/test_tf_importer.cpp +++ b/modules/dnn/test/test_tf_importer.cpp @@ -974,6 +974,9 @@ TEST_P(Test_TensorFlow_nets, Inception_v2_SSD) net.setPreferableBackend(backend); net.setPreferableTarget(target); + if (target == DNN_TARGET_CPU_FP16) + net.enableWinograd(false); + net.setInput(blob); // Output has shape 1x1xNx7 where N - number of detections. // An every detection is a vector of values [id, classId, confidence, left, top, right, bottom] @@ -1307,6 +1310,8 @@ TEST_P(Test_TensorFlow_nets, EAST_text_detection) net.setPreferableBackend(backend); net.setPreferableTarget(target); + if (target == DNN_TARGET_CPU_FP16) + net.enableWinograd(false); Mat img = imread(imgPath); Mat inp = blobFromImage(img, 1.0, Size(), Scalar(123.68, 116.78, 103.94), true, false); @@ -1341,8 +1346,9 @@ TEST_P(Test_TensorFlow_nets, EAST_text_detection) } else if (target == DNN_TARGET_CPU_FP16) { - lInf_scores = 0.1; - l1_geometry = 0.28; lInf_geometry = 5.94; + lInf_scores = 0.17; + l1_geometry = 0.28; + lInf_geometry = 5.94; } else { @@ -1810,6 +1816,8 @@ TEST_P(Test_TensorFlow_nets, Mask_RCNN) net.setPreferableBackend(backend); net.setPreferableTarget(target); + if (target == DNN_TARGET_CPU_FP16) + net.enableWinograd(false); net.setInput(blob); diff --git a/modules/dnn/test/test_torch_importer.cpp b/modules/dnn/test/test_torch_importer.cpp index de1cf2cf04..f1d7521e7b 100644 --- a/modules/dnn/test/test_torch_importer.cpp +++ b/modules/dnn/test/test_torch_importer.cpp @@ -358,6 +358,8 @@ TEST_P(Test_Torch_nets, OpenFace_accuracy) net.setPreferableBackend(backend); net.setPreferableTarget(target); + if (target == DNN_TARGET_CPU_FP16) + net.enableWinograd(false); Mat sample = imread(findDataFile("cv/shared/lena.png")); Mat sampleF32(sample.size(), CV_32FC3); @@ -542,6 +544,9 @@ TEST_P(Test_Torch_nets, FastNeuralStyle_accuracy) Mat img = imread(findDataFile("dnn/googlenet_1.png")); Mat inputBlob = blobFromImage(img, 1.0, Size(), Scalar(103.939, 116.779, 123.68), false); + if (target == DNN_TARGET_CPU_FP16) + net.enableWinograd(false); + net.setInput(inputBlob); Mat out = net.forward(); @@ -570,7 +575,7 @@ TEST_P(Test_Torch_nets, FastNeuralStyle_accuracy) } else if (target == DNN_TARGET_CPU_FP16) { - normAssert(out, refBlob, "", 0.64, 25); + normAssert(out, refBlob, "", 0.7, 25); } else normAssert(out, refBlob, "", 0.5, 1.16); From 23481b716bbab9c33e55cc3dd0db1b094d69f72f Mon Sep 17 00:00:00 2001 From: Alexander Smorkalov Date: Mon, 20 Nov 2023 13:55:32 +0300 Subject: [PATCH 038/156] Print warning, but not throw exceptions in cv::VideoCapture for OpenNI2. --- modules/videoio/src/cap_openni2.cpp | 88 ++++++++++++++++++----------- 1 file changed, 54 insertions(+), 34 deletions(-) diff --git a/modules/videoio/src/cap_openni2.cpp b/modules/videoio/src/cap_openni2.cpp index d4b3ae4153..034509cdf7 100644 --- a/modules/videoio/src/cap_openni2.cpp +++ b/modules/videoio/src/cap_openni2.cpp @@ -150,7 +150,7 @@ protected: IplImage* retrieveIrImage(); void toggleStream(int stream, bool toggle); - void readCamerasParams(); + bool readCamerasParams(); double getDepthGeneratorProperty(int propIdx) const; bool setDepthGeneratorProperty(int propIdx, double propVal); @@ -396,13 +396,14 @@ void CvCapture_OpenNI2::toggleStream(int stream, bool toggle) } -void CvCapture_OpenNI2::readCamerasParams() +bool CvCapture_OpenNI2::readCamerasParams() { double pixelSize = 0; if (streams[CV_DEPTH_STREAM].getProperty(XN_STREAM_PROPERTY_ZERO_PLANE_PIXEL_SIZE, &pixelSize) != openni::STATUS_OK) { - CV_Error(CV_StsError, "CvCapture_OpenNI2::readCamerasParams : Could not read pixel size!" + - std::string(openni::OpenNI::getExtendedError())); + CV_LOG_ERROR(NULL, "CvCapture_OpenNI2::readCamerasParams : Could not read pixel size!" + + std::string(openni::OpenNI::getExtendedError())); + return false; } // pixel size @ VGA = pixel size @ SXGA x 2 @@ -412,14 +413,16 @@ void CvCapture_OpenNI2::readCamerasParams() unsigned long long zeroPlaneDistance; // in mm if (streams[CV_DEPTH_STREAM].getProperty(XN_STREAM_PROPERTY_ZERO_PLANE_DISTANCE, &zeroPlaneDistance) != openni::STATUS_OK) { - CV_Error(CV_StsError, "CvCapture_OpenNI2::readCamerasParams : Could not read virtual plane distance!" + - std::string(openni::OpenNI::getExtendedError())); + CV_LOG_ERROR(NULL, "CvCapture_OpenNI2::readCamerasParams : Could not read virtual plane distance!" + + std::string(openni::OpenNI::getExtendedError())); + return false; } if (streams[CV_DEPTH_STREAM].getProperty(XN_STREAM_PROPERTY_EMITTER_DCMOS_DISTANCE, &baseline) != openni::STATUS_OK) { - CV_Error(CV_StsError, "CvCapture_OpenNI2::readCamerasParams : Could not read base line!" + - std::string(openni::OpenNI::getExtendedError())); + CV_LOG_ERROR(NULL, "CvCapture_OpenNI2::readCamerasParams : Could not read base line!" + + std::string(openni::OpenNI::getExtendedError())); + return false; } // baseline from cm -> mm @@ -427,6 +430,8 @@ void CvCapture_OpenNI2::readCamerasParams() // focal length from mm -> pixels (valid for 640x480) depthFocalLength_VGA = (int)((double)zeroPlaneDistance / (double)pixelSize); + + return true; } double CvCapture_OpenNI2::getProperty( int propIdx ) const @@ -513,7 +518,7 @@ double CvCapture_OpenNI2::getCommonProperty( int propIdx ) const break; } default : - CV_Error( CV_StsBadArg, cv::format("Such parameter (propIdx=%d) isn't supported for getting.", propIdx) ); + CV_LOG_WARNING( NULL, cv::format("Such parameter (propIdx=%d) isn't supported for getting.", propIdx) ); } return propValue; @@ -551,7 +556,7 @@ bool CvCapture_OpenNI2::setCommonProperty( int propIdx, double propValue ) break; default: - CV_Error(CV_StsBadArg, cv::format("Such parameter (propIdx=%d) isn't supported for setting.", propIdx)); + CV_LOG_WARNING(NULL, cv::format("Such parameter (propIdx=%d) isn't supported for setting.", propIdx)); } return isSet; @@ -585,12 +590,14 @@ double CvCapture_OpenNI2::getDepthGeneratorProperty( int propIdx ) const break; case CV_CAP_PROP_OPENNI_BASELINE : if(baseline <= 0) - const_cast(this)->readCamerasParams(); + if (!const_cast(this)->readCamerasParams()) + return 0; propValue = baseline; break; case CV_CAP_PROP_OPENNI_FOCAL_LENGTH : if(depthFocalLength_VGA <= 0) - const_cast(this)->readCamerasParams(); + if (!const_cast(this)->readCamerasParams()) + return 0; propValue = (double)depthFocalLength_VGA; break; case CV_CAP_PROP_OPENNI_REGISTRATION : @@ -603,7 +610,7 @@ double CvCapture_OpenNI2::getDepthGeneratorProperty( int propIdx ) const propValue = streamFrames[CV_DEPTH_STREAM].getFrameIndex(); break; default : - CV_Error( CV_StsBadArg, cv::format("Depth generator does not support such parameter (propIdx=%d) for getting.", propIdx) ); + CV_LOG_WARNING( NULL, cv::format("Depth generator does not support such parameter (propIdx=%d) for getting.", propIdx) ); } return propValue; @@ -638,13 +645,17 @@ bool CvCapture_OpenNI2::setDepthGeneratorProperty( int propIdx, double propValue { openni::Status status = device.setImageRegistrationMode(mode); if( status != openni::STATUS_OK ) - CV_Error(CV_StsError, std::string("CvCapture_OpenNI2::setDepthGeneratorProperty: ") + - std::string(openni::OpenNI::getExtendedError())); + { + CV_LOG_ERROR(NULL, std::string("CvCapture_OpenNI2::setDepthGeneratorProperty: ") + + std::string(openni::OpenNI::getExtendedError())); + } else isSet = true; } else - CV_Error(CV_StsError, "CvCapture_OpenNI2::setDepthGeneratorProperty: Unsupported viewpoint."); + { + CV_LOG_ERROR(NULL, "CvCapture_OpenNI2::setDepthGeneratorProperty: Unsupported viewpoint."); + } } else isSet = true; @@ -654,15 +665,17 @@ bool CvCapture_OpenNI2::setDepthGeneratorProperty( int propIdx, double propValue { openni::Status status = device.setImageRegistrationMode(openni::IMAGE_REGISTRATION_OFF); if( status != openni::STATUS_OK ) - CV_Error(CV_StsError, std::string("CvCapture_OpenNI2::setDepthGeneratorProperty: ") + - std::string(openni::OpenNI::getExtendedError())); + { + CV_LOG_ERROR(NULL, std::string("CvCapture_OpenNI2::setDepthGeneratorProperty: ") + + std::string(openni::OpenNI::getExtendedError())); + } else isSet = true; } } break; default: - CV_Error( CV_StsBadArg, cv::format("Depth generator does not support such parameter (propIdx=%d) for setting.", propIdx) ); + CV_LOG_WARNING( NULL, cv::format("OpenNI2: Depth generator does not support such parameter (propIdx=%d) for setting.", propIdx) ); } return isSet; @@ -696,7 +709,7 @@ double CvCapture_OpenNI2::getImageGeneratorProperty( int propIdx ) const propValue = (double)streamFrames[CV_COLOR_STREAM].getFrameIndex(); break; default : - CV_Error( CV_StsBadArg, cv::format("Image generator does not support such parameter (propIdx=%d) for getting.", propIdx) ); + CV_LOG_WARNING( NULL, cv::format("OpenNI2: Image generator does not support such parameter (propIdx=%d) for getting.", propIdx) ); } return propValue; @@ -744,19 +757,22 @@ bool CvCapture_OpenNI2::setImageGeneratorProperty(int propIdx, double propValue) mode.setFps(60); break; default : - CV_Error( CV_StsBadArg, "Unsupported image generator output mode."); + CV_LOG_WARNING( NULL, "Unsupported image generator output mode."); + return false; } openni::Status status = streams[CV_COLOR_STREAM].setVideoMode( mode ); if( status != openni::STATUS_OK ) - CV_Error(CV_StsError, std::string("CvCapture_OpenNI2::setImageGeneratorProperty: ") + - std::string(openni::OpenNI::getExtendedError())); + { + CV_LOG_ERROR(NULL, std::string("CvCapture_OpenNI2::setImageGeneratorProperty: ") + + std::string(openni::OpenNI::getExtendedError())); + } else isSet = true; break; } default: - CV_Error( CV_StsBadArg, cv::format("Image generator does not support such parameter (propIdx=%d) for setting.", propIdx) ); + CV_LOG_WARNING( NULL, cv::format("Image generator does not support such parameter (propIdx=%d) for setting.", propIdx) ); } return isSet; @@ -790,7 +806,7 @@ double CvCapture_OpenNI2::getIrGeneratorProperty(int propIdx) const propValue = (double)streamFrames[CV_IR_STREAM].getFrameIndex(); break; default: - CV_Error(CV_StsBadArg, cv::format("Image generator does not support such parameter (propIdx=%d) for getting.", propIdx)); + CV_LOG_WARNING(NULL, cv::format("Image generator does not support such parameter (propIdx=%d) for getting.", propIdx)); } return propValue; @@ -838,19 +854,21 @@ bool CvCapture_OpenNI2::setIrGeneratorProperty(int propIdx, double propValue) mode.setFps(60); break; default: - CV_Error(CV_StsBadArg, "Unsupported image generator output mode."); + CV_LOG_WARNING(NULL, "Unsupported image generator output mode."); } openni::Status status = streams[CV_IR_STREAM].setVideoMode(mode); if (status != openni::STATUS_OK) - CV_Error(CV_StsError, std::string("CvCapture_OpenNI2::setImageGeneratorProperty: ") + - std::string(openni::OpenNI::getExtendedError())); + { + CV_LOG_ERROR(NULL, std::string("CvCapture_OpenNI2::setImageGeneratorProperty: ") + + std::string(openni::OpenNI::getExtendedError())); + } else isSet = true; break; } default: - CV_Error(CV_StsBadArg, cv::format("Image generator does not support such parameter (propIdx=%d) for setting.", propIdx)); + CV_LOG_WARNING(NULL, cv::format("Image generator does not support such parameter (propIdx=%d) for setting.", propIdx)); } return isSet; @@ -965,9 +983,10 @@ static void computeDisparity_32F( const openni::VideoFrameRef& depthMetaData, cv IplImage* CvCapture_OpenNI2::retrieveDisparityMap() { if (!streamFrames[CV_DEPTH_STREAM].isValid()) - return 0; + return nullptr; - readCamerasParams(); + if (!readCamerasParams()) + return nullptr; cv::Mat disp32; computeDisparity_32F(streamFrames[CV_DEPTH_STREAM], disp32, baseline, depthFocalLength_VGA, noSampleValue, shadowValue); @@ -980,9 +999,10 @@ IplImage* CvCapture_OpenNI2::retrieveDisparityMap() IplImage* CvCapture_OpenNI2::retrieveDisparityMap_32F() { if (!streamFrames[CV_DEPTH_STREAM].isValid()) - return 0; + return nullptr; - readCamerasParams(); + if (!readCamerasParams()) + return nullptr; computeDisparity_32F(streamFrames[CV_DEPTH_STREAM], outputMaps[CV_CAP_OPENNI_DISPARITY_MAP_32F].mat, baseline, depthFocalLength_VGA, noSampleValue, shadowValue); @@ -992,7 +1012,7 @@ IplImage* CvCapture_OpenNI2::retrieveDisparityMap_32F() IplImage* CvCapture_OpenNI2::retrieveValidDepthMask() { if (!streamFrames[CV_DEPTH_STREAM].isValid()) - return 0; + return nullptr; cv::Mat d; getDepthMapFromMetaData(streamFrames[CV_DEPTH_STREAM], d, noSampleValue, shadowValue); From 16928806f998a3112d52a086090833f54b65a3f1 Mon Sep 17 00:00:00 2001 From: Maksym Ivashechkin Date: Mon, 20 Nov 2023 12:47:35 +0000 Subject: [PATCH 039/156] Merge pull request #24499 from ivashmak:usac_bug_fix Replace double atomic in USAC #24499 ### Pull Request Readiness Checklist See details at https://github.com/opencv/opencv/wiki/How_to_contribute#making-a-good-pull-request - [x] I agree to contribute to the project under Apache 2 License. - [x] To the best of my knowledge, the proposed patch is not based on a code under GPL or another license that is incompatible with OpenCV - [x] The PR is proposed to the proper branch - [x] There is a reference to the original bug report and related work - [x] There is accuracy test, performance test and test data in opencv_extra repository, if applicable Patch to opencv_extra has the same branch name. - [x] The feature is well documented and sample code can be built with the project CMake Reference to issue with atomic variable: #24281 Reference to bug with essential matrix: #24482 --- modules/calib3d/src/usac.hpp | 8 ++--- modules/calib3d/src/usac/essential_solver.cpp | 2 ++ modules/calib3d/src/usac/quality.cpp | 35 +++++++++---------- modules/calib3d/src/usac/ransac_solvers.cpp | 9 ++--- 4 files changed, 28 insertions(+), 26 deletions(-) diff --git a/modules/calib3d/src/usac.hpp b/modules/calib3d/src/usac.hpp index 57d91ab7f3..9b66a4576f 100644 --- a/modules/calib3d/src/usac.hpp +++ b/modules/calib3d/src/usac.hpp @@ -210,12 +210,12 @@ public: class Score { public: int inlier_number; - double score; + float score; Score () { // set worst case inlier_number = 0; - score = std::numeric_limits::max(); + score = std::numeric_limits::max(); } - Score (int inlier_number_, double score_) { // copy constructor + Score (int inlier_number_, float score_) { // copy constructor inlier_number = inlier_number_; score = score_; } @@ -254,7 +254,7 @@ public: // get @inliers of the @model for given threshold virtual int getInliers (const Mat &model, std::vector &inliers, double thr) const = 0; // Set the best score, so evaluation of the model can terminate earlier - virtual void setBestScore (double best_score_) = 0; + virtual void setBestScore (float best_score_) = 0; // set @inliers_mask: true if point i is inlier, false - otherwise. virtual int getInliers (const Mat &model, std::vector &inliers_mask) const = 0; virtual int getPointsSize() const = 0; diff --git a/modules/calib3d/src/usac/essential_solver.cpp b/modules/calib3d/src/usac/essential_solver.cpp index 434db6d373..6f0e6b1cfc 100644 --- a/modules/calib3d/src/usac/essential_solver.cpp +++ b/modules/calib3d/src/usac/essential_solver.cpp @@ -175,6 +175,8 @@ public: Matx33d Bz(bz); // Bz is rank 2, matrix, so epipole is its null-vector Vec3d xy1 = Utils::getRightEpipole(Mat(Bz * (1/sqrt(norm_bz)))); + const double one_over_xy1_norm = 1 / sqrt(xy1[0] * xy1[0] + xy1[1] * xy1[1] + xy1[2] * xy1[2]); + xy1 *= one_over_xy1_norm; if (fabs(xy1(2)) < 1e-10) continue; Mat_ E(3,3); diff --git a/modules/calib3d/src/usac/quality.cpp b/modules/calib3d/src/usac/quality.cpp index 89c5760c1d..9a72f754f5 100644 --- a/modules/calib3d/src/usac/quality.cpp +++ b/modules/calib3d/src/usac/quality.cpp @@ -69,7 +69,7 @@ public: else if (inlier_number - point < preemptive_thr) break; // score is negative inlier number! If less then better - return {inlier_number, -static_cast(inlier_number)}; + return {inlier_number, -static_cast(inlier_number)}; } Score getScore (const std::vector &errors) const override { @@ -78,10 +78,10 @@ public: if (errors[point] < threshold) inlier_number++; // score is negative inlier number! If less then better - return {inlier_number, -static_cast(inlier_number)}; + return {inlier_number, -static_cast(inlier_number)}; } - void setBestScore(double best_score_) override { + void setBestScore(float best_score_) override { if (best_score > best_score_) best_score = best_score_; } @@ -106,18 +106,17 @@ protected: const Ptr error; const int points_size; const double threshold, k_msac; - double best_score, norm_thr, one_over_thr; + const float norm_thr, one_over_thr; + float best_score; public: MsacQualityImpl (int points_size_, double threshold_, const Ptr &error_, double k_msac_) - : error (error_), points_size (points_size_), threshold (threshold_), k_msac(k_msac_) { - best_score = std::numeric_limits::max(); - norm_thr = threshold*k_msac; - one_over_thr = 1 / norm_thr; - } + : error (error_), points_size (points_size_), threshold (threshold_), k_msac(k_msac_), + norm_thr(static_cast(threshold*k_msac)), one_over_thr(1.f/norm_thr), + best_score(std::numeric_limits::max()) {} inline Score getScore (const Mat &model) const override { error->setModelParameters(model); - double err, sum_errors = 0; + float err, sum_errors = 0; int inlier_number = 0; const auto preemptive_thr = points_size + best_score; for (int point = 0; point < points_size; point++) { @@ -133,7 +132,7 @@ public: } Score getScore (const std::vector &errors) const override { - double sum_errors = 0; + float sum_errors = 0; int inlier_number = 0; for (int point = 0; point < points_size; point++) { const auto err = errors[point]; @@ -146,7 +145,7 @@ public: return {inlier_number, sum_errors}; } - void setBestScore(double best_score_) override { + void setBestScore(float best_score_) override { if (best_score > best_score_) best_score = best_score_; } @@ -244,7 +243,7 @@ public: } else if (total_loss + point_idx > preemptive_thr) break; } - return {num_tentative_inliers, total_loss}; + return {num_tentative_inliers, (float)total_loss}; } Score getScore (const std::vector &errors) const override { @@ -263,10 +262,10 @@ public: (stored_complete_gamma_values[x] - gamma_value_of_k)) * norm_loss); } } - return {num_tentative_inliers, total_loss}; + return {num_tentative_inliers, (float)total_loss}; } - void setBestScore (double best_loss) override { + void setBestScore (float best_loss) override { if (previous_best_loss > best_loss) previous_best_loss = best_loss; } @@ -317,7 +316,7 @@ public: return {inlier_number, Utils::findMedian (errors)}; } - void setBestScore (double /*best_score*/) override {} + void setBestScore (float /*best_score*/) override {} int getPointsSize () const override { return points_size; } int getInliers (const Mat &model, std::vector &inliers) const override @@ -487,9 +486,9 @@ public: if (last_model_is_good && do_sprt) { out_score.inlier_number = tested_inliers; if (score_type == ScoreMethod::SCORE_METHOD_MSAC) - out_score.score = sum_errors; + out_score.score = static_cast(sum_errors); else if (score_type == ScoreMethod::SCORE_METHOD_RANSAC) - out_score.score = -static_cast(tested_inliers); + out_score.score = -static_cast(tested_inliers); else out_score = quality->getScore(errors); } return last_model_is_good; diff --git a/modules/calib3d/src/usac/ransac_solvers.cpp b/modules/calib3d/src/usac/ransac_solvers.cpp index 28620c0b3f..494bbc1517 100644 --- a/modules/calib3d/src/usac/ransac_solvers.cpp +++ b/modules/calib3d/src/usac/ransac_solvers.cpp @@ -733,7 +733,7 @@ public: const bool is_prosac = params->getSampler() == SamplingMethod::SAMPLING_PROSAC; std::atomic_bool success(false); std::atomic_int num_hypothesis_tested(0), thread_cnt(0), max_number_inliers(0), subset_size, termination_length; - std::atomic best_score_all(std::numeric_limits::max()); + std::atomic best_score_all(std::numeric_limits::max()); std::vector best_scores(MAX_THREADS), best_scores_not_LO; std::vector best_models(MAX_THREADS), best_models_not_LO, K1_apx, K2_apx; std::vector num_tested_models_threads(MAX_THREADS), growth_function, non_random_inliers; @@ -782,7 +782,7 @@ public: model_verifier, local_optimization, termination, sampler, lo_sampler, weight_fnc, true); bool is_last_from_LO_thread = false; Mat best_model_thread, non_degenerate_model, lo_model, best_not_LO_thread; - Score best_score_thread, current_score, non_denegenerate_model_score, lo_score,best_score_all_threads, best_not_LO_score_thread; + Score best_score_thread, current_score, non_denegenerate_model_score, lo_score, best_score_all_threads, best_not_LO_score_thread; std::vector sample(estimator->getMinimalSampleSize()), best_sample_thread, supports; supports.reserve(3*MAX_MODELS_ADAPT); // store model supports std::vector best_inliers_mask_local(points_size, false), model_inliers_mask(points_size, false); @@ -790,7 +790,8 @@ public: auto update_best = [&] (const Score &new_score, const Mat &new_model, bool from_LO=false) { // update best score of all threads if (max_number_inliers < new_score.inlier_number) max_number_inliers = new_score.inlier_number; - if (best_score_all > new_score.score) best_score_all = new_score.score; + if (best_score_all > new_score.score) + best_score_all = new_score.score; best_score_all_threads = Score(max_number_inliers, best_score_all); // quality->getInliers(new_model, model_inliers_mask); @@ -839,7 +840,7 @@ public: success = num_hypothesis_tested++ > max_iters; if (iters % 10 && !adapt) { // Synchronize threads. just to speed verification of model. - quality->setBestScore(std::min(best_score_thread.score, (double)best_score_all)); + quality->setBestScore(std::min(best_score_thread.score, (float)best_score_all)); model_verifier->update(best_score_thread.inlier_number > max_number_inliers ? best_score_thread : best_score_all_threads, iters); } From c19adb4953418acfcde0dc29522054e789b7d46c Mon Sep 17 00:00:00 2001 From: Hao Chen Date: Wed, 8 Nov 2023 09:07:19 +0800 Subject: [PATCH 040/156] Change the lsx to baseline features. This patch change lsx to baseline feature, and lasx to dispatch feature. Additionally, the runtime detection methods for lasx and lsx have been modified. --- cmake/OpenCVCompilerOptimizations.cmake | 7 +-- modules/core/CMakeLists.txt | 26 +++++----- .../core/include/opencv2/core/hal/intrin.hpp | 6 --- .../include/opencv2/core/hal/intrin_lasx.hpp | 47 ++++++++++--------- modules/core/src/system.cpp | 15 ++++-- 5 files changed, 49 insertions(+), 52 deletions(-) diff --git a/cmake/OpenCVCompilerOptimizations.cmake b/cmake/OpenCVCompilerOptimizations.cmake index d45f327beb..32f176e930 100644 --- a/cmake/OpenCVCompilerOptimizations.cmake +++ b/cmake/OpenCVCompilerOptimizations.cmake @@ -403,11 +403,8 @@ elseif(LOONGARCH64) ocv_update(CPU_KNOWN_OPTIMIZATIONS "LSX;LASX") ocv_update(CPU_LSX_FLAGS_ON "-mlsx") ocv_update(CPU_LASX_FLAGS_ON "-mlasx") - if("${CPU_BASELINE_DISABLE}" STREQUAL "LASX") - set(CPU_BASELINE "LSX" CACHE STRING "${HELP_CPU_BASELINE}") - else() - set(CPU_BASELINE "LASX" CACHE STRING "${HELP_CPU_BASELINE}") - endif() + set(CPU_BASELINE "LSX" CACHE STRING "${HELP_CPU_BASELINE}") + set(CPU_DISPATCH "LASX" CACHE STRING "${HELP_CPU_DISPATCH}") endif() diff --git a/modules/core/CMakeLists.txt b/modules/core/CMakeLists.txt index ba5b61ef5f..4d5ebf3483 100644 --- a/modules/core/CMakeLists.txt +++ b/modules/core/CMakeLists.txt @@ -1,21 +1,21 @@ set(the_description "The Core Functionality") -ocv_add_dispatched_file(mathfuncs_core SSE2 AVX AVX2) -ocv_add_dispatched_file(stat SSE4_2 AVX2) -ocv_add_dispatched_file(arithm SSE2 SSE4_1 AVX2 VSX3) -ocv_add_dispatched_file(convert SSE2 AVX2 VSX3) -ocv_add_dispatched_file(convert_scale SSE2 AVX2) -ocv_add_dispatched_file(count_non_zero SSE2 AVX2) -ocv_add_dispatched_file(has_non_zero SSE2 AVX2) -ocv_add_dispatched_file(matmul SSE2 SSE4_1 AVX2 AVX512_SKX NEON_DOTPROD) -ocv_add_dispatched_file(mean SSE2 AVX2) -ocv_add_dispatched_file(merge SSE2 AVX2) -ocv_add_dispatched_file(split SSE2 AVX2) -ocv_add_dispatched_file(sum SSE2 AVX2) +ocv_add_dispatched_file(mathfuncs_core SSE2 AVX AVX2 LASX) +ocv_add_dispatched_file(stat SSE4_2 AVX2 LASX) +ocv_add_dispatched_file(arithm SSE2 SSE4_1 AVX2 VSX3 LASX) +ocv_add_dispatched_file(convert SSE2 AVX2 VSX3 LASX) +ocv_add_dispatched_file(convert_scale SSE2 AVX2 LASX) +ocv_add_dispatched_file(count_non_zero SSE2 AVX2 LASX) +ocv_add_dispatched_file(has_non_zero SSE2 AVX2 LASX ) +ocv_add_dispatched_file(matmul SSE2 SSE4_1 AVX2 AVX512_SKX NEON_DOTPROD LASX) +ocv_add_dispatched_file(mean SSE2 AVX2 LASX) +ocv_add_dispatched_file(merge SSE2 AVX2 LASX) +ocv_add_dispatched_file(split SSE2 AVX2 LASX) +ocv_add_dispatched_file(sum SSE2 AVX2 LASX) # dispatching for accuracy tests ocv_add_dispatched_file_force_all(test_intrin128 TEST SSE2 SSE3 SSSE3 SSE4_1 SSE4_2 AVX FP16 AVX2 AVX512_SKX) -ocv_add_dispatched_file_force_all(test_intrin256 TEST AVX2 AVX512_SKX) +ocv_add_dispatched_file_force_all(test_intrin256 TEST AVX2 AVX512_SKX LASX) ocv_add_dispatched_file_force_all(test_intrin512 TEST AVX512_SKX) diff --git a/modules/core/include/opencv2/core/hal/intrin.hpp b/modules/core/include/opencv2/core/hal/intrin.hpp index 3968cba8f0..7897fb503f 100644 --- a/modules/core/include/opencv2/core/hal/intrin.hpp +++ b/modules/core/include/opencv2/core/hal/intrin.hpp @@ -246,12 +246,6 @@ using namespace CV_CPU_OPTIMIZATION_HAL_NAMESPACE; #include "opencv2/core/hal/intrin_lsx.hpp" -#elif CV_LASX - #if !defined(CV_FORCE_SIMD128_CPP) - #define CV_FORCE_SIMD128_CPP 1 - #endif -#include "opencv2/core/hal/intrin_cpp.hpp" - #else #include "opencv2/core/hal/intrin_cpp.hpp" diff --git a/modules/core/include/opencv2/core/hal/intrin_lasx.hpp b/modules/core/include/opencv2/core/hal/intrin_lasx.hpp index 5214e80743..6546d6db7d 100644 --- a/modules/core/include/opencv2/core/hal/intrin_lasx.hpp +++ b/modules/core/include/opencv2/core/hal/intrin_lasx.hpp @@ -1419,20 +1419,6 @@ inline v_uint32x8 v_popcount(const v_int32x8& a) inline v_uint64x4 v_popcount(const v_int64x4& a) { return v_popcount(v_reinterpret_as_u64(a)); } -/** Mask **/ -#define OPENCV_HAL_IMPL_REINTERPRET_INT(ft, tt) \ -inline tt reinterpret_int(ft x) { union { ft l; tt i; } v; v.l = x; return v.i; } -OPENCV_HAL_IMPL_REINTERPRET_INT(uchar, schar) -OPENCV_HAL_IMPL_REINTERPRET_INT(schar, schar) -OPENCV_HAL_IMPL_REINTERPRET_INT(ushort, short) -OPENCV_HAL_IMPL_REINTERPRET_INT(short, short) -OPENCV_HAL_IMPL_REINTERPRET_INT(unsigned, int) -OPENCV_HAL_IMPL_REINTERPRET_INT(int, int) -OPENCV_HAL_IMPL_REINTERPRET_INT(float, int) -OPENCV_HAL_IMPL_REINTERPRET_INT(uint64, int64) -OPENCV_HAL_IMPL_REINTERPRET_INT(int64, int64) -OPENCV_HAL_IMPL_REINTERPRET_INT(double, int64) - inline int v_signmask(const v_int8x32& a) { __m256i result = __lasx_xvmskltz_b(a.val); @@ -2151,7 +2137,8 @@ template inline void v_rshr_pack_store(uchar* ptr, const v_uint16x16& a) { __m256i res = __lasx_xvssrlrni_bu_h(a.val, a.val, n); - __lsx_vst(_v256_extract_low(_v256_shuffle_odd_64(res)), ptr, 0); + __lasx_xvstelm_d(res, ptr, 0, 0); + __lasx_xvstelm_d(res, ptr, 8, 2); } template inline @@ -2165,7 +2152,8 @@ template inline void v_rshr_pack_u_store(uchar* ptr, const v_int16x16& a) { __m256i res = __lasx_xvssrarni_bu_h(a.val, a.val, n); - __lsx_vst(_v256_extract_low(_v256_shuffle_odd_64(res)), ptr, 0); + __lasx_xvstelm_d(res, ptr, 0, 0); + __lasx_xvstelm_d(res, ptr, 8, 2); } template inline @@ -2179,7 +2167,8 @@ template inline void v_rshr_pack_store(schar* ptr, const v_int16x16& a) { __m256i res = __lasx_xvssrarni_b_h(a.val, a.val, n); - __lsx_vst(_v256_extract_low(_v256_shuffle_odd_64(res)), ptr, 0); + __lasx_xvstelm_d(res, ptr, 0, 0); + __lasx_xvstelm_d(res, ptr, 8, 2); } // 32 @@ -2198,7 +2187,8 @@ inline void v_pack_store(short* ptr, const v_int32x8& a) inline void v_pack_store(ushort* ptr, const v_uint32x8& a) { __m256i res = __lasx_xvssrlrni_hu_w(a.val, a.val, 0); - __lsx_vst(_v256_extract_low(_v256_shuffle_odd_64(res)), ptr, 0); + __lasx_xvstelm_d(res, ptr, 0, 0); + __lasx_xvstelm_d(res, ptr, 8, 2); } inline void v_pack_u_store(ushort* ptr, const v_int32x8& a) @@ -2212,7 +2202,8 @@ template inline void v_rshr_pack_store(ushort* ptr, const v_uint32x8& a) { __m256i res = __lasx_xvssrlrni_hu_w(a.val, a.val, n); - __lsx_vst(_v256_extract_low(_v256_shuffle_odd_64(res)), ptr, 0); + __lasx_xvstelm_d(res, ptr, 0, 0); + __lasx_xvstelm_d(res, ptr, 8, 2); } template inline @@ -2223,7 +2214,8 @@ template inline void v_rshr_pack_u_store(ushort* ptr, const v_int32x8& a) { __m256i res = __lasx_xvssrarni_hu_w(a.val, a.val, n); - __lsx_vst(_v256_extract_low(_v256_shuffle_odd_64(res)), ptr, 0); + __lasx_xvstelm_d(res, ptr, 0, 0); + __lasx_xvstelm_d(res, ptr, 8, 2); } template inline @@ -2234,7 +2226,8 @@ template inline void v_rshr_pack_store(short* ptr, const v_int32x8& a) { __m256i res = __lasx_xvssrarni_h_w(a.val, a.val, n); - __lsx_vst(_v256_extract_low(_v256_shuffle_odd_64(res)), ptr, 0); + __lasx_xvstelm_d(res, ptr, 0, 0); + __lasx_xvstelm_d(res, ptr, 8, 2); } // 64 @@ -2263,7 +2256,11 @@ v_uint32x8 v_rshr_pack(const v_uint64x4& a, const v_uint64x4& b) template inline void v_rshr_pack_store(unsigned* ptr, const v_uint64x4& a) -{ __lsx_vst(_v256_shuffle_odd_64(__lasx_xvsrlrni_w_d(a.val, a.val, n)), ptr, 0); } +{ + __m256i res = __lasx_xvsrlrni_w_d(a.val, a.val, n); + __lasx_xvstelm_d(res, ptr, 0, 0); + __lasx_xvstelm_d(res, ptr, 8, 2); +} template inline v_int32x8 v_rshr_pack(const v_int64x4& a, const v_int64x4& b) @@ -2271,7 +2268,11 @@ v_int32x8 v_rshr_pack(const v_int64x4& a, const v_int64x4& b) template inline void v_rshr_pack_store(int* ptr, const v_int64x4& a) -{ __lsx_vst(_v256_shuffle_odd_64(__lasx_xvsrarni_w_d(a.val, a.val, n)), ptr, 0); } +{ + __m256i res = __lasx_xvsrarni_w_d(a.val, a.val, n); + __lasx_xvstelm_d(res, ptr, 0, 0); + __lasx_xvstelm_d(res, ptr, 8, 2); +} // pack boolean inline v_uint8x32 v_pack_b(const v_uint16x16& a, const v_uint16x16& b) diff --git a/modules/core/src/system.cpp b/modules/core/src/system.cpp index 8d651b8c8d..9f67d92a43 100644 --- a/modules/core/src/system.cpp +++ b/modules/core/src/system.cpp @@ -154,6 +154,12 @@ void* allocSingletonNewBuffer(size_t size) { return malloc(size); } # endif #endif +#if defined __loongarch64 +#include "sys/auxv.h" +#define LA_HWCAP_LSX (1<<4) +#define LA_HWCAP_LASX (1<<5) +#endif + #if defined _WIN32 || defined WINCE #ifndef _WIN32_WINNT // This is needed for the declaration of TryEnterCriticalSection in winbase.h with Visual Studio 2005 (and older?) #define _WIN32_WINNT 0x0400 // http://msdn.microsoft.com/en-us/library/ms686857(VS.85).aspx @@ -704,12 +710,11 @@ struct HWFeatures have[CV_CPU_RVV] = true; #endif - #if defined __loongarch_sx - have[CV_CPU_LSX] = true; - #endif + #if defined __loongarch64 && defined __linux__ + int flag = (int)getauxval(AT_HWCAP); - #if defined __loongarch_asx - have[CV_CPU_LASX] = true; + have[CV_CPU_LSX] = (flag & LA_HWCAP_LSX) != 0; + have[CV_CPU_LASX] = (flag & LA_HWCAP_LASX) != 0; #endif bool skip_baseline_check = false; From a57ea2b775896e554d8ae065dff1b8fb45d582e7 Mon Sep 17 00:00:00 2001 From: MaximSmolskiy Date: Tue, 21 Nov 2023 09:34:54 +0300 Subject: [PATCH 041/156] Fix typos in calibinit.cpp --- modules/calib3d/src/calibinit.cpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/modules/calib3d/src/calibinit.cpp b/modules/calib3d/src/calibinit.cpp index 135519e776..a4aa622bbe 100644 --- a/modules/calib3d/src/calibinit.cpp +++ b/modules/calib3d/src/calibinit.cpp @@ -46,7 +46,7 @@ Here is the copyright notice from the original Vladimir's code: =============================================================== - The algorithms developed and implemented by Vezhnevets Vldimir + The algorithms developed and implemented by Vezhnevets Vladimir aka Dead Moroz (vvp@graphics.cs.msu.ru) See http://graphics.cs.msu.su/en/research/calibration/opencv.html for detailed information. @@ -54,7 +54,7 @@ Reliability additions and modifications made by Philip Gruebele. pgruebele@cox.net - Some further improvements for detection of partially ocluded boards at non-ideal + Some further improvements for detection of partially occluded boards at non-ideal lighting conditions have been made by Alex Bovyrin and Kurt Kolonige \************************************************************************************/ @@ -325,7 +325,7 @@ static void icvGradientOfHistogram256(const ArrayContainer& piHist, ArrayContain piHistGrad[255] = 0; } /***************************************************************************************************/ -//PERFORM SMART IMAGE THRESHOLDING BASED ON ANALYSIS OF INTENSTY HISTOGRAM +//PERFORM SMART IMAGE THRESHOLDING BASED ON ANALYSIS OF INTENSITY HISTOGRAM static void icvBinarizationHistogramBased(Mat & img) { CV_Assert(img.channels() == 1 && img.depth() == CV_8U); @@ -671,7 +671,7 @@ bool findChessboardCorners(InputArray image_, Size pattern_size, // of the neighbor corners in the same row/column. // // This function has been created as temporary workaround for the bug in current implementation -// of cvFindChessboardCornes that produces absolutely unordered sets of corners. +// of cvFindChessboardCorners that produces absolutely unordered sets of corners. // bool ChessBoardDetector::checkBoardMonotony(const std::vector& corners) { @@ -1213,7 +1213,7 @@ int ChessBoardDetector::cleanFoundConnectedQuads(std::vector& q // We iteratively remove the point which reduces the size of // the bounding box of the blobs the most // (since we want the rectangle to be as small as possible) - // remove the quadrange that causes the biggest reduction + // remove the quadrangle that causes the biggest reduction // in pattern size until we have the correct number for (; quad_count > count; quad_count--) { From d05fb709f959070b2ed67b3912c3586ed10a52b5 Mon Sep 17 00:00:00 2001 From: Yuantao Feng Date: Tue, 21 Nov 2023 06:33:01 -0600 Subject: [PATCH 042/156] Merge pull request #24552 from fengyuentau:layernorm_backends dnn: add openvino, opencl and cuda backends for layer normalization layer #24552 Merge after https://github.com/opencv/opencv/pull/24544. Todo: - [x] openvino - [x] opencl - [x] cuda ### Pull Request Readiness Checklist See details at https://github.com/opencv/opencv/wiki/How_to_contribute#making-a-good-pull-request - [x] I agree to contribute to the project under Apache 2 License. - [x] To the best of my knowledge, the proposed patch is not based on a code under GPL or another license that is incompatible with OpenCV - [x] The PR is proposed to the proper branch - [x] There is a reference to the original bug report and related work - [x] There is accuracy test, performance test and test data in opencv_extra repository, if applicable Patch to opencv_extra has the same branch name. - [x] The feature is well documented and sample code can be built with the project CMake --- modules/dnn/src/cuda/mvn.cu | 69 ++++++- modules/dnn/src/cuda4dnn/kernels/mvn.hpp | 8 +- .../src/cuda4dnn/primitives/layer_norm.hpp | 93 ++++++++++ modules/dnn/src/layers/layer_norm.cpp | 172 ++++++++++++++++++ modules/dnn/src/opencl/mvn.cl | 6 + ...conformance_layer_filter__openvino.inl.hpp | 76 ++------ 6 files changed, 359 insertions(+), 65 deletions(-) create mode 100644 modules/dnn/src/cuda4dnn/primitives/layer_norm.hpp diff --git a/modules/dnn/src/cuda/mvn.cu b/modules/dnn/src/cuda/mvn.cu index d4f0733676..0accc499a2 100644 --- a/modules/dnn/src/cuda/mvn.cu +++ b/modules/dnn/src/cuda/mvn.cu @@ -68,15 +68,36 @@ namespace raw { } template - __global__ void normalize_mean_variance_channelwise(Span output, View input, View scale, View bias, View means, View stdev, size_type inner_size, size_type C) { + __global__ void normalize_mean_variance_channelwise(Span output, View input, View scale, View bias, View means, View inv_stddev, size_type inner_size, size_type C) { for (auto idx : grid_stride_range(output.size())) { const index_type outer_idx = idx / inner_size; const index_type c = outer_idx % C; - auto s = static_cast(scale[c]) * stdev[outer_idx]; + auto s = static_cast(scale[c]) * inv_stddev[outer_idx]; auto b = static_cast(bias[c]); output[idx] = (static_cast(input[idx]) - means[outer_idx]) * s + b; } } + + template + __global__ void normalize_mean_variance_layernorm(Span output, View input, View scale, View means, View inv_stddev, size_type inner_size) { + for (auto idx : grid_stride_range(output.size())) { + const index_type outer_idx = idx / inner_size; + const index_type inner_idx = idx % inner_size; + auto s = static_cast(scale[inner_idx]) * inv_stddev[outer_idx]; + output[idx] = (static_cast(input[idx]) - means[outer_idx]) * s; + } + } + + template + __global__ void normalize_mean_variance_layernorm_with_bias(Span output, View input, View scale, View bias, View means, View inv_stddev, size_type inner_size) { + for (auto idx : grid_stride_range(output.size())) { + const index_type outer_idx = idx / inner_size; + const index_type inner_idx = idx % inner_size; + auto s = static_cast(scale[inner_idx]) * inv_stddev[outer_idx]; + auto b = static_cast(bias[inner_idx]); + output[idx] = (static_cast(input[idx]) - means[outer_idx]) * s + b; + } + } } template @@ -154,20 +175,54 @@ template void normalize_mean_variance(const Stream&, Span<__half>, View<__half>, template void normalize_mean_variance(const Stream&, Span, View, View, View, std::size_t); template -void normalize_mean_variance_channelwise(const Stream& stream, Span output, View input, View scale, View bias, View means, View stdev, std::size_t inner_size, std::size_t C) +void normalize_mean_variance_channelwise(const Stream& stream, Span output, View input, View scale, View bias, View means, View inv_stddev, std::size_t inner_size, std::size_t C) { CV_Assert(input.size() == output.size()); CV_Assert(input.size() / inner_size == means.size()); - CV_Assert(means.size() == stdev.size()); + CV_Assert(means.size() == inv_stddev.size()); auto kernel = raw::normalize_mean_variance_channelwise; auto policy = make_policy(kernel, output.size(), 0, stream); - launch_kernel(kernel, policy, output, input, scale, bias, means, stdev, inner_size, C); + launch_kernel(kernel, policy, output, input, scale, bias, means, inv_stddev, inner_size, C); } #if !defined(__CUDA_ARCH__) || (__CUDA_ARCH__ >= 530) -template void normalize_mean_variance_channelwise(const Stream&, Span<__half> /*output*/, View<__half> /*input*/, View<__half> /*scale*/, View<__half> /*bias*/, View /*means*/, View /*stdev*/, std::size_t, std::size_t); +template void normalize_mean_variance_channelwise(const Stream&, Span<__half> /*output*/, View<__half> /*input*/, View<__half> /*scale*/, View<__half> /*bias*/, View /*means*/, View /*inv_stddev*/, std::size_t, std::size_t); #endif -template void normalize_mean_variance_channelwise(const Stream&, Span /*output*/, View /*input*/, View /*scale*/, View /*bias*/, View /*means*/, View /*stdev*/, std::size_t, std::size_t); +template void normalize_mean_variance_channelwise(const Stream&, Span /*output*/, View /*input*/, View /*scale*/, View /*bias*/, View /*means*/, View /*inv_stddev*/, std::size_t, std::size_t); + +template +void normalize_mean_variance_layernorm(const Stream& stream, Span output, View input, View scale, View means, View inv_stddev, std::size_t inner_size) +{ + CV_Assert(input.size() == output.size()); + CV_Assert(input.size() / inner_size == means.size()); + CV_Assert(means.size() == inv_stddev.size()); + + auto kernel = raw::normalize_mean_variance_layernorm; + auto policy = make_policy(kernel, output.size(), 0, stream); + launch_kernel(kernel, policy, output, input, scale, means, inv_stddev, inner_size); +} + +#if !defined(__CUDA_ARCH__) || (__CUDA_ARCH__ >= 530) +template void normalize_mean_variance_layernorm(const Stream&, Span<__half> /*output*/, View<__half> /*input*/, View<__half> /*scale*/, View /*means*/, View /*inv_stddev*/, std::size_t); +#endif +template void normalize_mean_variance_layernorm(const Stream&, Span /*output*/, View /*input*/, View /*scale*/, View /*means*/, View /*inv_stddev*/, std::size_t); + +template +void normalize_mean_variance_layernorm(const Stream& stream, Span output, View input, View scale, View bias, View means, View inv_stddev, std::size_t inner_size) +{ + CV_Assert(input.size() == output.size()); + CV_Assert(input.size() / inner_size == means.size()); + CV_Assert(means.size() == inv_stddev.size()); + + auto kernel = raw::normalize_mean_variance_layernorm_with_bias; + auto policy = make_policy(kernel, output.size(), 0, stream); + launch_kernel(kernel, policy, output, input, scale, bias, means, inv_stddev, inner_size); +} + +#if !defined(__CUDA_ARCH__) || (__CUDA_ARCH__ >= 530) +template void normalize_mean_variance_layernorm(const Stream&, Span<__half> /*output*/, View<__half> /*input*/, View<__half> /*scale*/, View<__half> /*bias*/, View /*means*/, View /*inv_stddev*/, std::size_t); +#endif +template void normalize_mean_variance_layernorm(const Stream&, Span /*output*/, View /*input*/, View /*scale*/, View /*bias*/, View /*means*/, View /*inv_stddev*/, std::size_t); }}}} /* namespace cv::dnn::cuda4dnn::kernels */ diff --git a/modules/dnn/src/cuda4dnn/kernels/mvn.hpp b/modules/dnn/src/cuda4dnn/kernels/mvn.hpp index ebd7b9f659..6cddeb22bb 100644 --- a/modules/dnn/src/cuda4dnn/kernels/mvn.hpp +++ b/modules/dnn/src/cuda4dnn/kernels/mvn.hpp @@ -27,7 +27,13 @@ template void normalize_mean_variance(const csl::Stream& stream, csl::Span output, csl::View input, csl::View means, csl::View scale, std::size_t inner_size); template -void normalize_mean_variance_channelwise(const csl::Stream &stream, csl::Span output, csl::View input, csl::View scale, csl::View bias, csl::View means, csl::View stdev, std::size_t inner_size, std::size_t C); +void normalize_mean_variance_channelwise(const csl::Stream &stream, csl::Span output, csl::View input, csl::View scale, csl::View bias, csl::View means, csl::View inv_stddev, std::size_t inner_size, std::size_t C); + +template +void normalize_mean_variance_layernorm(const csl::Stream &stream, csl::Span output, csl::View input, csl::View scale, csl::View means, csl::View inv_stddev, std::size_t inner_size); + +template +void normalize_mean_variance_layernorm(const csl::Stream &stream, csl::Span output, csl::View input, csl::View scale, csl::View bias, csl::View means, csl::View inv_stddev, std::size_t inner_size); }}}} /* namespace cv::dnn::cuda4dnn::kernels */ diff --git a/modules/dnn/src/cuda4dnn/primitives/layer_norm.hpp b/modules/dnn/src/cuda4dnn/primitives/layer_norm.hpp new file mode 100644 index 0000000000..7f4658a50a --- /dev/null +++ b/modules/dnn/src/cuda4dnn/primitives/layer_norm.hpp @@ -0,0 +1,93 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. + +#ifndef OPENCV_DNN_SRC_CUDA4DNN_PRIMITIVES_LAYER_NORM_HPP +#define OPENCV_DNN_SRC_CUDA4DNN_PRIMITIVES_LAYER_NORM_HPP + +#include "../../op_cuda.hpp" + +#include "../csl/stream.hpp" +#include "../csl/span.hpp" +#include "../csl/tensor.hpp" +#include "../csl/workspace.hpp" + +#include "../kernels/fill_copy.hpp" +#include "../kernels/mvn.hpp" + +#include + +#include +#include +#include + +namespace cv { namespace dnn { namespace cuda4dnn { + + template + class LayerNormOp final : public CUDABackendNode { + public: + using wrapper_type = GetCUDABackendWrapperType; + + LayerNormOp(csl::Stream stream_, int normalized_axis, float epsilon_, size_t loops) + : stream(std::move(stream_)), epsilon(epsilon_) { + CV_CheckGE(normalized_axis, 0, "LayerNorm/CUDA: axis needs to be normalized"); + axis = static_cast(normalized_axis); + + csl::WorkspaceBuilder builder; + builder.require(loops); + builder.require(loops); + scratch_mem_in_bytes = builder.required_workspace_size(); + } + + void forward(const std::vector>& inputs, + const std::vector>& outputs, + csl::Workspace& workspace) override { + auto input_wrapper = inputs[0].dynamicCast(); + auto scale_wrapper = inputs[1].dynamicCast(); + + auto input = input_wrapper->getView(); + auto scale = scale_wrapper->getView(); + + auto output_wrapper = outputs[0].dynamicCast(); + auto output = output_wrapper->getSpan(); + + auto loops = input.size_range(0, axis); + auto norm_size = input.size_range(axis, input.rank()); + if (norm_size == 1) { + kernels::fill(stream, output, 0.f); + return; + } else { + auto ws_allocator = csl::WorkspaceAllocator(workspace); + + auto mean = ws_allocator.get_span(loops); + kernels::fill(stream, mean, 0.f); + + auto inv_stddev = ws_allocator.get_span(loops); + kernels::fill(stream, inv_stddev, 0.f); + + kernels::reduce_mean_sqr_sum(stream, mean, inv_stddev, input, norm_size); + kernels::compute_normalization_scale(stream, inv_stddev, mean, inv_stddev, norm_size, epsilon); + if (inputs.size() == 3) { + auto bias_wrapper = inputs[2].dynamicCast(); + auto bias = bias_wrapper->getView(); + kernels::normalize_mean_variance_layernorm(stream, output, input, scale, bias, mean, inv_stddev, norm_size); + } else { + kernels::normalize_mean_variance_layernorm(stream, output, input, scale, mean, inv_stddev, norm_size); + } + } + } + + std::size_t get_workspace_memory_in_bytes() const noexcept override { return scratch_mem_in_bytes; } + + private: + csl::Stream stream; + + float epsilon; + size_t axis; + + std::size_t scratch_mem_in_bytes; + }; + +}}} // cv::dnn::cuda4dnn + +#endif // OPENCV_DNN_SRC_CUDA4DNN_PRIMITIVES_LAYER_NORM_HPP diff --git a/modules/dnn/src/layers/layer_norm.cpp b/modules/dnn/src/layers/layer_norm.cpp index 0683bdb8c8..f3d2667a0a 100644 --- a/modules/dnn/src/layers/layer_norm.cpp +++ b/modules/dnn/src/layers/layer_norm.cpp @@ -9,8 +9,26 @@ // CANN backend #include "../op_cann.hpp" +// OpenVINO backend +#include "../op_inf_engine.hpp" +#include "../ie_ngraph.hpp" + +// CUDA backend +#include "../op_cuda.hpp" +#ifdef HAVE_CUDA +#include "../cuda4dnn/primitives/layer_norm.hpp" +using namespace cv::dnn::cuda4dnn; +#endif + +// OpenCL backend +#ifdef HAVE_OPENCL +#include "../ocl4dnn/include/math_functions.hpp" +#include "opencl_kernels_dnn.hpp" +#endif + namespace cv { namespace dnn { +// https://github.com/onnx/onnx/blob/main/docs/Operators.md#LayerNormalization class LayerNormLayerImpl CV_FINAL : public LayerNormLayer { public: @@ -25,7 +43,12 @@ public: virtual bool supportBackend(int backendId) CV_OVERRIDE { +#ifdef HAVE_INF_ENGINE + if (backendId == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) + return true; +#endif return backendId == DNN_BACKEND_OPENCV || + backendId == DNN_BACKEND_CUDA || (backendId == DNN_BACKEND_CANN && axis != -1); // axis=-1 not supported due to 1d mat shape problem } @@ -73,6 +96,9 @@ public: CV_TRACE_FUNCTION(); CV_TRACE_ARG_VALUE(name, "name", name.c_str()); + CV_OCL_RUN(IS_DNN_OPENCL_TARGET(preferableTarget), + forward_ocl(inputs_arr, outputs_arr, internals_arr)) + if (inputs_arr.depth() == CV_16S) { forward_fallback(inputs_arr, outputs_arr, internals_arr); @@ -95,6 +121,91 @@ public: } } +#ifdef HAVE_OPENCL + bool forward_ocl(InputArrayOfArrays inputs_, OutputArrayOfArrays outputs_, OutputArrayOfArrays internals_) { + std::vector inputs; + std::vector outputs; + + inputs_.getUMatVector(inputs); + outputs_.getUMatVector(outputs); + + const auto &input = inputs[0], &scale = inputs[1]; // &bias = inputs[2]; // bias is optional + auto &output = outputs[0]; + + const auto input_shape = shape(input); + size_t loops = static_cast(total(input_shape, 0, axis)), + norm_size = static_cast(total(input_shape, axis)); + float inv_norm_size = 1.f / norm_size; + + const auto &bias = inputs.size() == 3 ? inputs[2] : UMat::zeros(norm_size, 1, CV_32F); + + // no fp16 support + if (input.depth() == CV_16S) { + return false; + } + + String base_opts = format(" -DT=float -DT4=float4 -Dconvert_T=convert_float4"); + + // Calculate mean + UMat one = UMat::ones(norm_size, 1, CV_32F); + UMat mean = UMat(loops, 1, CV_32F); + UMat mean_square = UMat(loops, 1, CV_32F); + UMat tmp = UMat(loops, norm_size, CV_32F); + bool ret = ocl4dnn::ocl4dnnGEMV(ocl4dnn::CblasNoTrans, loops, norm_size, inv_norm_size, + input, 0, one, 0, 0.f, mean, 0); + if (!ret) { + return false; + } + // Calculate mean_square + int num_vector = (norm_size % 8 == 0) ? 8 : ((norm_size % 4 == 0) ? 4 : 1); + size_t global[] = {loops, static_cast(norm_size / num_vector)}; + String build_opt = format(" -DNUM=%d", num_vector) + base_opts; + String mean_square_kernel_name = format("calc_mean%d", num_vector); + ocl::Kernel mean_square_kernel(mean_square_kernel_name.c_str(), ocl::dnn::mvn_oclsrc, build_opt + " -DKERNEL_MEAN"); + if (mean_square_kernel.empty()) { + return false; + } + mean_square_kernel.set(0, ocl::KernelArg::PtrReadOnly(input)); + mean_square_kernel.set(1, (int)loops); + mean_square_kernel.set(2, (int)norm_size); + mean_square_kernel.set(3, ocl::KernelArg::PtrReadOnly(mean)); + mean_square_kernel.set(4, ocl::KernelArg::PtrWriteOnly(tmp)); + ret = mean_square_kernel.run(2, global, NULL, false); + if (!ret) { + return false; + } + ret = ocl4dnn::ocl4dnnGEMV(ocl4dnn::CblasNoTrans, loops, norm_size, inv_norm_size, + tmp, 0, one, 0, 0.f, mean_square, 0); + if (!ret) { + return false; + } + // Calculate instance norm: output = scale * (x - mean) / sqrt(var + eps) + bias + String mvn_kernel_name = format("mvn%d", num_vector); + build_opt += " -DNORM_VARIANCE -DLAYER_NORM -DKERNEL_MVN"; + ocl::Kernel mvn_kernel(mvn_kernel_name.c_str(), ocl::dnn::mvn_oclsrc, build_opt); + if (mvn_kernel.empty()) { + return false; + } + mvn_kernel.set(0, ocl::KernelArg::PtrReadOnly(input)); + mvn_kernel.set(1, (int)loops); + mvn_kernel.set(2, (int)norm_size); + mvn_kernel.set(3, (float)epsilon); + mvn_kernel.set(4, ocl::KernelArg::PtrReadOnly(mean)); + mvn_kernel.set(5, ocl::KernelArg::PtrReadOnly(mean_square)); + mvn_kernel.set(6, ocl::KernelArg::PtrReadOnly(scale)); + mvn_kernel.set(7, ocl::KernelArg::PtrReadOnly(bias)); + mvn_kernel.set(8, (int)1); + mvn_kernel.set(9, (float)0.f); + mvn_kernel.set(10, ocl::KernelArg::PtrWriteOnly(output)); + ret = mvn_kernel.run(2, global, NULL, false); + if (!ret) { + return false; + } + + return true; + } +#endif + #ifdef HAVE_CANN virtual Ptr initCann(const std::vector > &inputs, const std::vector > &outputs, @@ -147,6 +258,67 @@ public: } #endif // HAVE_CANN +#ifdef HAVE_DNN_NGRAPH + virtual Ptr initNgraph(const std::vector >& inputs, + const std::vector >& nodes) CV_OVERRIDE { + auto ieInpNode = nodes[0].dynamicCast()->node; + const auto &input_shape = ieInpNode.get_shape(); + std::shared_ptr mvn, result; + + // mvn +#if INF_ENGINE_VER_MAJOR_LE(INF_ENGINE_RELEASE_2021_2) + // https://docs.openvino.ai/2021.4/api/ngraph_python_api/_autosummary/ngraph.opset3.mvn.html?highlight=mvn#ngraph.opset3.mvn + bool across_channels = false; + bool normalize_variance = true; + mvn = std::make_shared(ieInpNode, across_channels, normalize_variance, epsilon); +#else + // https://docs.openvino.ai/2023.1/openvino_docs_ops_normalization_MVN_6.html + std::vector axes_v(input_shape.size() - axis); + std::iota(axes_v.begin(), axes_v.end(), axis); + auto axes = std::make_shared(ngraph::element::i64, ngraph::Shape{axes_v.size()}, axes_v.data()); + bool normalize_variance = true; + mvn = std::make_shared(ieInpNode, axes, normalize_variance, epsilon, ngraph::op::MVNEpsMode::INSIDE_SQRT); +#endif + + // layer norm = scale * mvn + bias + auto scale = nodes[1].dynamicCast()->node; + ngraph::Output bias; + if (nodes.size() == 3) { + bias = nodes[2].dynamicCast()->node; + } + if (axis == -1 || axis == input_shape.size() - 1) { // special case for 1D tensor (2D mat) + std::vector shared_shape_v(input_shape.size(), 1); + shared_shape_v.back() = -1; + auto shared_shape = std::make_shared(ngraph::element::i64, ngraph::Shape{shared_shape_v.size()}, shared_shape_v.data()); + scale = std::make_shared(scale, shared_shape, true); + if (nodes.size() == 3) { + bias = std::make_shared(bias, shared_shape, true); + } + } + + result = std::make_shared(mvn, scale); + if (nodes.size() == 3) { + result = std::make_shared(result, bias); + } + + return Ptr(new InfEngineNgraphNode(result)); + } +#endif // HAVE_DNN_NGRAPH + +#ifdef HAVE_CUDA + Ptr initCUDA(void *context_, + const std::vector>& inputs, + const std::vector>& outputs) override { + auto context = reinterpret_cast(context_); + + auto input_wrapper = inputs[0].dynamicCast(); + auto input_shape = input_wrapper->getShape(); + size_t loops = static_cast(total(input_shape, 0, axis)); + + return make_cuda_node(preferableTarget, std::move(context->stream), axis, epsilon, loops); + } +#endif // HAVE_CUDA + }; Ptr LayerNormLayer::create(const LayerParams& params) diff --git a/modules/dnn/src/opencl/mvn.cl b/modules/dnn/src/opencl/mvn.cl index f84d04502c..7353ed8b82 100644 --- a/modules/dnn/src/opencl/mvn.cl +++ b/modules/dnn/src/opencl/mvn.cl @@ -126,12 +126,18 @@ __kernel void MVN(__global const Dtype* src, alpha = 1; #endif +#ifdef LAYER_NORM + vec_type w = load(bnorm_weight, y), b = load(bnorm_bias, y); +#else + Dtype w = 1.f, b = 0.f; #ifdef FUSE_BATCH_NORM w = bnorm_weight[x % channels]; b = bnorm_bias[x % channels]; #endif +#endif // LAYER_NORM + vec_type src_vec = load(src, index) - (vec_type)mean_val; vec_type dst_vec = src_vec * alpha; dst_vec = dst_vec * w + (vec_type)b; diff --git a/modules/dnn/test/test_onnx_conformance_layer_filter__openvino.inl.hpp b/modules/dnn/test/test_onnx_conformance_layer_filter__openvino.inl.hpp index 94687a1e26..17d561d64b 100644 --- a/modules/dnn/test/test_onnx_conformance_layer_filter__openvino.inl.hpp +++ b/modules/dnn/test/test_onnx_conformance_layer_filter__openvino.inl.hpp @@ -793,81 +793,43 @@ CASE(test_isinf_positive) CASE(test_isnan) // no filter CASE(test_layer_normalization_2d_axis0) -#if SKIP_SET_1 - SKIP_NON_CPU; -#endif + // no filter CASE(test_layer_normalization_2d_axis1) -#if SKIP_SET_1 - SKIP_NON_CPU; -#endif + // no filter CASE(test_layer_normalization_2d_axis_negative_1) -#if SKIP_SET_1 - SKIP_NON_CPU; -#endif + // no filter CASE(test_layer_normalization_2d_axis_negative_2) -#if SKIP_SET_1 - SKIP_NON_CPU; -#endif + // no filter CASE(test_layer_normalization_3d_axis0_epsilon) -#if SKIP_SET_1 - SKIP_NON_CPU; -#endif + // no filter CASE(test_layer_normalization_3d_axis1_epsilon) -#if SKIP_SET_1 - SKIP_NON_CPU; -#endif + // no filter CASE(test_layer_normalization_3d_axis2_epsilon) -#if SKIP_SET_1 - SKIP_NON_CPU; -#endif + // no filter CASE(test_layer_normalization_3d_axis_negative_1_epsilon) -#if SKIP_SET_1 - SKIP_NON_CPU; -#endif + // no filter CASE(test_layer_normalization_3d_axis_negative_2_epsilon) -#if SKIP_SET_1 - SKIP_NON_CPU; -#endif + // no filter CASE(test_layer_normalization_3d_axis_negative_3_epsilon) -#if SKIP_SET_1 - SKIP_NON_CPU; -#endif + // no filter CASE(test_layer_normalization_4d_axis0) -#if SKIP_SET_1 - SKIP_NON_CPU; -#endif + // no filter CASE(test_layer_normalization_4d_axis1) -#if SKIP_SET_1 - SKIP_NON_CPU; -#endif + // no filter CASE(test_layer_normalization_4d_axis2) -#if SKIP_SET_1 - SKIP_NON_CPU; -#endif + // no filter CASE(test_layer_normalization_4d_axis3) -#if SKIP_SET_1 - SKIP_NON_CPU; -#endif + // no filter CASE(test_layer_normalization_4d_axis_negative_1) -#if SKIP_SET_1 - SKIP_NON_CPU; -#endif + // no filter CASE(test_layer_normalization_4d_axis_negative_2) -#if SKIP_SET_1 - SKIP_NON_CPU; -#endif + // no filter CASE(test_layer_normalization_4d_axis_negative_3) -#if SKIP_SET_1 - SKIP_NON_CPU; -#endif + // no filter CASE(test_layer_normalization_4d_axis_negative_4) -#if SKIP_SET_1 - SKIP_NON_CPU; -#endif + // no filter CASE(test_layer_normalization_default_axis) -#if SKIP_SET_1 - SKIP_NON_CPU; -#endif + // no filter CASE(test_leakyrelu) // no filter CASE(test_leakyrelu_default) From 2c1ec4245dc793153c50fcbd255863dd1382c961 Mon Sep 17 00:00:00 2001 From: Maxim Smolskiy Date: Tue, 21 Nov 2023 15:36:43 +0300 Subject: [PATCH 043/156] Merge pull request #24527 from MaximSmolskiy:fix-out-of-image-corners-in-cv-cornersubpix Fix out of image corners in cv::cornerSubPix #24527 ### Pull Request Readiness Checklist See details at https://github.com/opencv/opencv/wiki/How_to_contribute#making-a-good-pull-request - [x] I agree to contribute to the project under Apache 2 License. - [x] To the best of my knowledge, the proposed patch is not based on a code under GPL or another license that is incompatible with OpenCV - [x] The PR is proposed to the proper branch - [ ] There is a reference to the original bug report and related work - [ ] There is accuracy test, performance test and test data in opencv_extra repository, if applicable Patch to opencv_extra has the same branch name. - [ ] The feature is well documented and sample code can be built with the project CMake --- modules/imgproc/src/cornersubpix.cpp | 6 +- modules/imgproc/test/test_cornersubpix.cpp | 68 ++++++++++++++++++++++ 2 files changed, 72 insertions(+), 2 deletions(-) create mode 100644 modules/imgproc/test/test_cornersubpix.cpp diff --git a/modules/imgproc/src/cornersubpix.cpp b/modules/imgproc/src/cornersubpix.cpp index 1e0841271f..24fd59910b 100644 --- a/modules/imgproc/src/cornersubpix.cpp +++ b/modules/imgproc/src/cornersubpix.cpp @@ -96,6 +96,7 @@ void cv::cornerSubPix( InputArray _image, InputOutputArray _corners, for( int pt_i = 0; pt_i < count; pt_i++ ) { Point2f cT = corners[pt_i], cI = cT; + CV_Assert( Rect(0, 0, src.cols, src.rows).contains(cT) ); int iter = 0; double err = 0; @@ -140,9 +141,10 @@ void cv::cornerSubPix( InputArray _image, InputOutputArray _corners, cI2.x = (float)(cI.x + c*scale*bb1 - b*scale*bb2); cI2.y = (float)(cI.y - b*scale*bb1 + a*scale*bb2); err = (cI2.x - cI.x) * (cI2.x - cI.x) + (cI2.y - cI.y) * (cI2.y - cI.y); - cI = cI2; - if( cI.x < 0 || cI.x >= src.cols || cI.y < 0 || cI.y >= src.rows ) + // if new point is out of image, leave previous point as the result + if( !Rect(0, 0, src.cols, src.rows).contains(cI2) ) break; + cI = cI2; } while( ++iter < max_iters && err > eps ); diff --git a/modules/imgproc/test/test_cornersubpix.cpp b/modules/imgproc/test/test_cornersubpix.cpp new file mode 100644 index 0000000000..86484d2482 --- /dev/null +++ b/modules/imgproc/test/test_cornersubpix.cpp @@ -0,0 +1,68 @@ +/*M/////////////////////////////////////////////////////////////////////////////////////// +// +// IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. +// +// By downloading, copying, installing or using the software you agree to this license. +// If you do not agree to this license, do not download, install, +// copy or use the software. +// +// +// License Agreement +// For Open Source Computer Vision Library +// +// Copyright (C) 2015-2023, OpenCV Foundation, all rights reserved. +// Third party copyrights are property of their respective owners. +// +// Redistribution and use in source and binary forms, with or without modification, +// are permitted provided that the following conditions are met: +// +// * Redistribution's of source code must retain the above copyright notice, +// this list of conditions and the following disclaimer. +// +// * Redistribution's in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// +// * The name of the copyright holders may not be used to endorse or promote products +// derived from this software without specific prior written permission. +// +// This software is provided by the copyright holders and contributors "as is" and +// any express or implied warranties, including, but not limited to, the implied +// warranties of merchantability and fitness for a particular purpose are disclaimed. +// In no event shall the Intel Corporation or contributors be liable for any direct, +// indirect, incidental, special, exemplary, or consequential damages +// (including, but not limited to, procurement of substitute goods or services; +// loss of use, data, or profits; or business interruption) however caused +// and on any theory of liability, whether in contract, strict liability, +// or tort (including negligence or otherwise) arising in any way out of +// the use of this software, even if advised of the possibility of such damage. +// +//M*/ + +#include "test_precomp.hpp" + +namespace opencv_test { namespace { + +TEST(Imgproc_CornerSubPix, out_of_image_corners) +{ + const uint8_t image_pixels[] = { + 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 1, + 0, 0, 0, 0, 0, 0, 2, + 0, 0, 0, 0, 0, 0, 3}; + + cv::Mat image(cv::Size(7, 7), CV_8UC1, (void*)image_pixels, cv::Mat::AUTO_STEP); + std::vector corners = {cv::Point2f(5.25, 6.5)}; + cv::Size win(1, 1); + cv::Size zeroZone(-1, -1); + cv::TermCriteria criteria; + cv::cornerSubPix(image, corners, win, zeroZone, criteria); + + ASSERT_EQ(corners.size(), 1u); + ASSERT_TRUE(Rect(0, 0, image.cols, image.rows).contains(corners.front())); +} + +}} // namespace From ce0516282a53947933387bb8e4262a0594ce0913 Mon Sep 17 00:00:00 2001 From: Liutong HAN Date: Thu, 23 Nov 2023 15:06:04 +0800 Subject: [PATCH 044/156] Optimize the v_lut for RVV. --- .../opencv2/core/hal/intrin_rvv_scalable.hpp | 74 +++++++++++++------ 1 file changed, 50 insertions(+), 24 deletions(-) diff --git a/modules/core/include/opencv2/core/hal/intrin_rvv_scalable.hpp b/modules/core/include/opencv2/core/hal/intrin_rvv_scalable.hpp index c7770334ad..e2475e0e7d 100644 --- a/modules/core/include/opencv2/core/hal/intrin_rvv_scalable.hpp +++ b/modules/core/include/opencv2/core/hal/intrin_rvv_scalable.hpp @@ -448,29 +448,7 @@ OPENCV_HAL_IMPL_RVV_LOADSTORE_OP(v_float64, vfloat64m1_t, double, VTraits::vlanes())), sizeof(_Tp), VTraits<_Tpvec>::vlanes()); \ - return vloxei32(tab, vidx, VTraits<_Tpvec>::vlanes()); \ -} \ -inline _Tpvec v_lut_pairs(const _Tp* tab, const int* idx) \ -{ \ - std::vector idx_; \ - for (int i = 0; i < VTraits::vlanes(); ++i) { \ - idx_.push_back(idx[i]); \ - idx_.push_back(idx[i]+1); \ - } \ - vuint32##suffix##_t vidx = vmul(vle32_v_u32##suffix(idx_.data(), VTraits<_Tpvec>::vlanes()), sizeof(_Tp), VTraits<_Tpvec>::vlanes()); \ - return vloxei32(tab, vidx, VTraits<_Tpvec>::vlanes()); \ -} \ -inline _Tpvec v_lut_quads(const _Tp* tab, const int* idx) \ -{ \ - std::vector idx_; \ - for (int i = 0; i < VTraits::vlanes(); ++i) { \ - idx_.push_back(idx[i]); \ - idx_.push_back(idx[i]+1); \ - idx_.push_back(idx[i]+2); \ - idx_.push_back(idx[i]+3); \ - } \ - vuint32##suffix##_t vidx = vmul(vle32_v_u32##suffix(idx_.data(), VTraits<_Tpvec>::vlanes()), sizeof(_Tp), VTraits<_Tpvec>::vlanes()); \ + auto vidx = vmul(vreinterpret_u32##suffix(vle32_v_i32##suffix(idx, VTraits<_Tpvec>::vlanes())), sizeof(_Tp), VTraits<_Tpvec>::vlanes()); \ return vloxei32(tab, vidx, VTraits<_Tpvec>::vlanes()); \ } OPENCV_HAL_IMPL_RVV_LUT(v_int8, schar, m4) @@ -482,6 +460,55 @@ OPENCV_HAL_IMPL_RVV_LUT(v_float32, float, m1) OPENCV_HAL_IMPL_RVV_LUT(v_float64, double, mf2) #endif +#define OPENCV_HAL_IMPL_RVV_LUT_PAIRS(_Tpvec, _Tp, suffix1, suffix2, v_trunc) \ +inline _Tpvec v_lut_pairs(const _Tp* tab, const int* idx) \ +{ \ + auto v0 = vle32_v_u32##suffix1((unsigned*)idx, VTraits<_Tpvec>::vlanes()/2); \ + auto v1 = vadd(v0, 1, VTraits<_Tpvec>::vlanes()/2); \ + auto w0 = vwcvtu_x(v0, VTraits<_Tpvec>::vlanes()/2); \ + auto w1 = vwcvtu_x(v1, VTraits<_Tpvec>::vlanes()/2); \ + auto sh1 = vslide1up(v_trunc(vreinterpret_u32##suffix2(w1)),0, VTraits<_Tpvec>::vlanes()); \ + auto vid = vor(sh1, v_trunc(vreinterpret_u32##suffix2(w0)), VTraits<_Tpvec>::vlanes()); \ + auto vidx = vmul(vid, sizeof(_Tp), VTraits<_Tpvec>::vlanes()); \ + return vloxei32(tab, vidx, VTraits<_Tpvec>::vlanes()); \ +} +OPENCV_HAL_IMPL_RVV_LUT_PAIRS(v_int8, schar, m2, m4, OPENCV_HAL_NOP) +OPENCV_HAL_IMPL_RVV_LUT_PAIRS(v_int16, short, m1, m2, OPENCV_HAL_NOP) +OPENCV_HAL_IMPL_RVV_LUT_PAIRS(v_int32, int, mf2, m1, OPENCV_HAL_NOP) +OPENCV_HAL_IMPL_RVV_LUT_PAIRS(v_float32, float, mf2, m1, OPENCV_HAL_NOP) +OPENCV_HAL_IMPL_RVV_LUT_PAIRS(v_int64, int64_t, mf2, m1, vlmul_trunc_u32mf2) +#if CV_SIMD_SCALABLE_64F +OPENCV_HAL_IMPL_RVV_LUT_PAIRS(v_float64, double, mf2, m1, vlmul_trunc_u32mf2) +#endif + + +#define OPENCV_HAL_IMPL_RVV_LUT_QUADS(_Tpvec, _Tp, suffix0, suffix1, suffix2, v_trunc) \ +inline _Tpvec v_lut_quads(const _Tp* tab, const int* idx) \ +{ \ + auto v0 = vle32_v_u32##suffix0((unsigned*)idx, VTraits<_Tpvec>::vlanes()/4); \ + auto v1 = vadd(v0, 1, VTraits<_Tpvec>::vlanes()/4); \ + auto v2 = vadd(v0, 2, VTraits<_Tpvec>::vlanes()/4); \ + auto v3 = vadd(v0, 3, VTraits<_Tpvec>::vlanes()/4); \ + auto w0 = vwcvtu_x(v0, VTraits<_Tpvec>::vlanes()/4); \ + auto w1 = vwcvtu_x(v1, VTraits<_Tpvec>::vlanes()/4); \ + auto w2 = vwcvtu_x(v2, VTraits<_Tpvec>::vlanes()/4); \ + auto w3 = vwcvtu_x(v3, VTraits<_Tpvec>::vlanes()/4); \ + auto sh2 = vslide1up(vreinterpret_u32##suffix1(w2),0, VTraits<_Tpvec>::vlanes()/2); \ + auto sh3 = vslide1up(vreinterpret_u32##suffix1(w3),0, VTraits<_Tpvec>::vlanes()/2); \ + auto vid0 = vor(sh2, vreinterpret_u32##suffix1(w0), VTraits<_Tpvec>::vlanes()/2); \ + auto vid1 = vor(sh3, vreinterpret_u32##suffix1(w1), VTraits<_Tpvec>::vlanes()/2); \ + auto wid0 = vwcvtu_x(v_trunc(vid0), VTraits<_Tpvec>::vlanes()/2); \ + auto wid1 = vwcvtu_x(v_trunc(vid1), VTraits<_Tpvec>::vlanes()/2); \ + auto shwid1 = vslide1up(vreinterpret_u32##suffix2(wid1),0, VTraits<_Tpvec>::vlanes()); \ + auto vid = vor(shwid1, vreinterpret_u32##suffix2(wid0), VTraits<_Tpvec>::vlanes()); \ + auto vidx = vmul(vid, sizeof(_Tp), VTraits<_Tpvec>::vlanes()); \ + return vloxei32(tab, vidx, VTraits<_Tpvec>::vlanes()); \ +} +OPENCV_HAL_IMPL_RVV_LUT_QUADS(v_int8, schar, m1, m2, m4, OPENCV_HAL_NOP) +OPENCV_HAL_IMPL_RVV_LUT_QUADS(v_int16, short, mf2 , m1, m2, OPENCV_HAL_NOP) +OPENCV_HAL_IMPL_RVV_LUT_QUADS(v_int32, int, mf2, m1, m1, vlmul_trunc_u32mf2) +OPENCV_HAL_IMPL_RVV_LUT_QUADS(v_float32, float, mf2, m1, m1, vlmul_trunc_u32mf2) + #define OPENCV_HAL_IMPL_RVV_LUT_VEC(_Tpvec, _Tp) \ inline _Tpvec v_lut(const _Tp* tab, const v_int32& vidx) \ { \ @@ -512,7 +539,6 @@ inline v_uint32 v_lut_pairs(const unsigned* tab, const int* idx) { return v_rein inline v_uint32 v_lut_quads(const unsigned* tab, const int* idx) { return v_reinterpret_as_u32(v_lut_quads((int*)tab, idx)); } inline v_uint64 v_lut(const uint64* tab, const int* idx) { return v_reinterpret_as_u64(v_lut((const int64_t *)tab, idx)); } inline v_uint64 v_lut_pairs(const uint64* tab, const int* idx) { return v_reinterpret_as_u64(v_lut_pairs((const int64_t *)tab, idx)); } -inline v_uint64 v_lut_quads(const uint64* tab, const int* idx) { return v_reinterpret_as_u64(v_lut_quads((const int64_t*)tab, idx)); } ////////////// Pack boolean //////////////////// inline v_uint8 v_pack_b(const v_uint16& a, const v_uint16& b) From 2830551e89ce0c0aac8d99908e8261f3da175735 Mon Sep 17 00:00:00 2001 From: Alexander Smorkalov Date: Wed, 22 Nov 2023 13:52:41 +0300 Subject: [PATCH 045/156] Drop OpenCV Manager from samples initialization. --- CMakeLists.txt | 4 - .../opencv/android/AsyncServiceHelper.java | 391 ------------------ .../opencv/android/BaseLoaderCallback.java | 141 ------- .../android/InstallCallbackInterface.java | 34 -- .../android/LoaderCallbackInterface.java | 40 -- .../org/opencv/android/OpenCVLoader.java.in | 110 +---- .../opencv/engine/OpenCVEngineInterface.aidl | 33 -- platforms/android/service/CMakeLists.txt | 10 - .../service/engine/AndroidManifest.xml | 30 -- .../android/service/engine/CMakeLists.txt | 3 - .../service/engine/res/drawable/icon.png | Bin 1997 -> 0 bytes .../service/engine/res/layout/main.xml | 49 --- .../service/engine/res/values/strings.xml | 7 - .../org/opencv/engine/HardwareDetector.java | 118 ------ .../org/opencv/engine/MarketConnector.java | 31 -- .../opencv/engine/OpenCVEngineInterface.aidl | 33 -- .../opencv/engine/OpenCVEngineService.java | 165 -------- .../engine/manager/ManagerActivity.java | 113 ----- platforms/android/service/readme.txt | 25 -- .../samples/puzzle15/Puzzle15Activity.java | 42 +- .../CameraCalibrationActivity.java | 38 +- .../ColorBlobDetectionActivity.java | 39 +- .../opencv/samples/facedetect/FdActivity.java | 107 +++-- .../ImageManipulationsActivity.java | 38 +- .../opencv_mobilenet/MainActivity.java | 33 +- .../samples/tutorial1/Tutorial1Activity.java | 38 +- .../samples/tutorial2/Tutorial2Activity.java | 42 +- .../samples/tutorial3/Tutorial3Activity.java | 38 +- 28 files changed, 154 insertions(+), 1598 deletions(-) delete mode 100644 modules/java/generator/android/java/org/opencv/android/AsyncServiceHelper.java delete mode 100644 modules/java/generator/android/java/org/opencv/android/BaseLoaderCallback.java delete mode 100644 modules/java/generator/android/java/org/opencv/android/InstallCallbackInterface.java delete mode 100644 modules/java/generator/android/java/org/opencv/android/LoaderCallbackInterface.java delete mode 100644 modules/java/generator/android/java/org/opencv/engine/OpenCVEngineInterface.aidl delete mode 100644 platforms/android/service/CMakeLists.txt delete mode 100644 platforms/android/service/engine/AndroidManifest.xml delete mode 100644 platforms/android/service/engine/CMakeLists.txt delete mode 100644 platforms/android/service/engine/res/drawable/icon.png delete mode 100644 platforms/android/service/engine/res/layout/main.xml delete mode 100644 platforms/android/service/engine/res/values/strings.xml delete mode 100644 platforms/android/service/engine/src/org/opencv/engine/HardwareDetector.java delete mode 100644 platforms/android/service/engine/src/org/opencv/engine/MarketConnector.java delete mode 100644 platforms/android/service/engine/src/org/opencv/engine/OpenCVEngineInterface.aidl delete mode 100644 platforms/android/service/engine/src/org/opencv/engine/OpenCVEngineService.java delete mode 100644 platforms/android/service/engine/src/org/opencv/engine/manager/ManagerActivity.java delete mode 100644 platforms/android/service/readme.txt diff --git a/CMakeLists.txt b/CMakeLists.txt index 49c93d2406..f614bae777 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -987,10 +987,6 @@ if(BUILD_EXAMPLES OR BUILD_ANDROID_EXAMPLES OR INSTALL_ANDROID_EXAMPLES OR INSTA add_subdirectory(samples) endif() -if(ANDROID) - add_subdirectory(platforms/android/service) -endif() - # ---------------------------------------------------------------------------- # Finalization: generate configuration-based files # ---------------------------------------------------------------------------- diff --git a/modules/java/generator/android/java/org/opencv/android/AsyncServiceHelper.java b/modules/java/generator/android/java/org/opencv/android/AsyncServiceHelper.java deleted file mode 100644 index cb3c6428d1..0000000000 --- a/modules/java/generator/android/java/org/opencv/android/AsyncServiceHelper.java +++ /dev/null @@ -1,391 +0,0 @@ -package org.opencv.android; - -import java.io.File; -import java.util.StringTokenizer; - -import org.opencv.core.Core; -import org.opencv.engine.OpenCVEngineInterface; - -import android.content.ComponentName; -import android.content.Context; -import android.content.Intent; -import android.content.ServiceConnection; -import android.net.Uri; -import android.os.IBinder; -import android.os.RemoteException; -import android.util.Log; - -class AsyncServiceHelper -{ - public static boolean initOpenCV(String Version, final Context AppContext, - final LoaderCallbackInterface Callback) - { - AsyncServiceHelper helper = new AsyncServiceHelper(Version, AppContext, Callback); - Intent intent = new Intent("org.opencv.engine.BIND"); - intent.setPackage("org.opencv.engine"); - if (AppContext.bindService(intent, helper.mServiceConnection, Context.BIND_AUTO_CREATE)) - { - return true; - } - else - { - AppContext.unbindService(helper.mServiceConnection); - InstallService(AppContext, Callback); - return false; - } - } - - protected AsyncServiceHelper(String Version, Context AppContext, LoaderCallbackInterface Callback) - { - mOpenCVersion = Version; - mUserAppCallback = Callback; - mAppContext = AppContext; - } - - protected static final String TAG = "OpenCVManager/Helper"; - protected static final int MINIMUM_ENGINE_VERSION = 2; - protected OpenCVEngineInterface mEngineService; - protected LoaderCallbackInterface mUserAppCallback; - protected String mOpenCVersion; - protected Context mAppContext; - protected static boolean mServiceInstallationProgress = false; - protected static boolean mLibraryInstallationProgress = false; - - protected static boolean InstallServiceQuiet(Context context) - { - boolean result = true; - try - { - Intent intent = new Intent(Intent.ACTION_VIEW, Uri.parse(OPEN_CV_SERVICE_URL)); - intent.addFlags(Intent.FLAG_ACTIVITY_NEW_TASK); - context.startActivity(intent); - } - catch(Exception e) - { - result = false; - } - - return result; - } - - protected static void InstallService(final Context AppContext, final LoaderCallbackInterface Callback) - { - if (!mServiceInstallationProgress) - { - Log.d(TAG, "Request new service installation"); - InstallCallbackInterface InstallQuery = new InstallCallbackInterface() { - private LoaderCallbackInterface mUserAppCallback = Callback; - public String getPackageName() - { - return "OpenCV Manager"; - } - public void install() { - Log.d(TAG, "Trying to install OpenCV Manager via Google Play"); - - boolean result = InstallServiceQuiet(AppContext); - if (result) - { - mServiceInstallationProgress = true; - Log.d(TAG, "Package installation started"); - } - else - { - Log.d(TAG, "OpenCV package was not installed!"); - int Status = LoaderCallbackInterface.MARKET_ERROR; - Log.d(TAG, "Init finished with status " + Status); - Log.d(TAG, "Unbind from service"); - Log.d(TAG, "Calling using callback"); - mUserAppCallback.onManagerConnected(Status); - } - } - - public void cancel() - { - Log.d(TAG, "OpenCV library installation was canceled"); - int Status = LoaderCallbackInterface.INSTALL_CANCELED; - Log.d(TAG, "Init finished with status " + Status); - Log.d(TAG, "Calling using callback"); - mUserAppCallback.onManagerConnected(Status); - } - - public void wait_install() - { - Log.e(TAG, "Installation was not started! Nothing to wait!"); - } - }; - - Callback.onPackageInstall(InstallCallbackInterface.NEW_INSTALLATION, InstallQuery); - } - else - { - Log.d(TAG, "Waiting current installation process"); - InstallCallbackInterface WaitQuery = new InstallCallbackInterface() { - private LoaderCallbackInterface mUserAppCallback = Callback; - public String getPackageName() - { - return "OpenCV Manager"; - } - public void install() - { - Log.e(TAG, "Nothing to install we just wait current installation"); - } - public void cancel() - { - Log.d(TAG, "Waiting for OpenCV canceled by user"); - mServiceInstallationProgress = false; - int Status = LoaderCallbackInterface.INSTALL_CANCELED; - Log.d(TAG, "Init finished with status " + Status); - Log.d(TAG, "Calling using callback"); - mUserAppCallback.onManagerConnected(Status); - } - public void wait_install() - { - InstallServiceQuiet(AppContext); - } - }; - - Callback.onPackageInstall(InstallCallbackInterface.INSTALLATION_PROGRESS, WaitQuery); - } - } - - /** - * URL of OpenCV Manager page on Google Play Market. - */ - protected static final String OPEN_CV_SERVICE_URL = "market://details?id=org.opencv.engine"; - - protected ServiceConnection mServiceConnection = new ServiceConnection() - { - public void onServiceConnected(ComponentName className, IBinder service) - { - Log.d(TAG, "Service connection created"); - mEngineService = OpenCVEngineInterface.Stub.asInterface(service); - if (null == mEngineService) - { - Log.d(TAG, "OpenCV Manager Service connection fails. May be service was not installed?"); - InstallService(mAppContext, mUserAppCallback); - } - else - { - mServiceInstallationProgress = false; - try - { - if (mEngineService.getEngineVersion() < MINIMUM_ENGINE_VERSION) - { - Log.d(TAG, "Init finished with status " + LoaderCallbackInterface.INCOMPATIBLE_MANAGER_VERSION); - Log.d(TAG, "Unbind from service"); - mAppContext.unbindService(mServiceConnection); - Log.d(TAG, "Calling using callback"); - mUserAppCallback.onManagerConnected(LoaderCallbackInterface.INCOMPATIBLE_MANAGER_VERSION); - return; - } - - Log.d(TAG, "Trying to get library path"); - String path = mEngineService.getLibPathByVersion(mOpenCVersion); - if ((null == path) || (path.length() == 0)) - { - if (!mLibraryInstallationProgress) - { - InstallCallbackInterface InstallQuery = new InstallCallbackInterface() { - public String getPackageName() - { - return "OpenCV library"; - } - public void install() { - Log.d(TAG, "Trying to install OpenCV lib via Google Play"); - try - { - if (mEngineService.installVersion(mOpenCVersion)) - { - mLibraryInstallationProgress = true; - Log.d(TAG, "Package installation started"); - Log.d(TAG, "Unbind from service"); - mAppContext.unbindService(mServiceConnection); - } - else - { - Log.d(TAG, "OpenCV package was not installed!"); - Log.d(TAG, "Init finished with status " + LoaderCallbackInterface.MARKET_ERROR); - Log.d(TAG, "Unbind from service"); - mAppContext.unbindService(mServiceConnection); - Log.d(TAG, "Calling using callback"); - mUserAppCallback.onManagerConnected(LoaderCallbackInterface.MARKET_ERROR); - } - } catch (RemoteException e) { - e.printStackTrace();; - Log.d(TAG, "Init finished with status " + LoaderCallbackInterface.INIT_FAILED); - Log.d(TAG, "Unbind from service"); - mAppContext.unbindService(mServiceConnection); - Log.d(TAG, "Calling using callback"); - mUserAppCallback.onManagerConnected(LoaderCallbackInterface.INIT_FAILED); - } - } - public void cancel() { - Log.d(TAG, "OpenCV library installation was canceled"); - Log.d(TAG, "Init finished with status " + LoaderCallbackInterface.INSTALL_CANCELED); - Log.d(TAG, "Unbind from service"); - mAppContext.unbindService(mServiceConnection); - Log.d(TAG, "Calling using callback"); - mUserAppCallback.onManagerConnected(LoaderCallbackInterface.INSTALL_CANCELED); - } - public void wait_install() { - Log.e(TAG, "Installation was not started! Nothing to wait!"); - } - }; - - mUserAppCallback.onPackageInstall(InstallCallbackInterface.NEW_INSTALLATION, InstallQuery); - } - else - { - InstallCallbackInterface WaitQuery = new InstallCallbackInterface() { - public String getPackageName() - { - return "OpenCV library"; - } - - public void install() { - Log.e(TAG, "Nothing to install we just wait current installation"); - } - public void cancel() - { - Log.d(TAG, "OpenCV library installation was canceled"); - mLibraryInstallationProgress = false; - Log.d(TAG, "Init finished with status " + LoaderCallbackInterface.INSTALL_CANCELED); - Log.d(TAG, "Unbind from service"); - mAppContext.unbindService(mServiceConnection); - Log.d(TAG, "Calling using callback"); - mUserAppCallback.onManagerConnected(LoaderCallbackInterface.INSTALL_CANCELED); - } - public void wait_install() { - Log.d(TAG, "Waiting for current installation"); - try - { - if (!mEngineService.installVersion(mOpenCVersion)) - { - Log.d(TAG, "OpenCV package was not installed!"); - Log.d(TAG, "Init finished with status " + LoaderCallbackInterface.MARKET_ERROR); - Log.d(TAG, "Calling using callback"); - mUserAppCallback.onManagerConnected(LoaderCallbackInterface.MARKET_ERROR); - } - else - { - Log.d(TAG, "Waiting for package installation"); - } - - Log.d(TAG, "Unbind from service"); - mAppContext.unbindService(mServiceConnection); - - } catch (RemoteException e) { - e.printStackTrace(); - Log.d(TAG, "Init finished with status " + LoaderCallbackInterface.INIT_FAILED); - Log.d(TAG, "Unbind from service"); - mAppContext.unbindService(mServiceConnection); - Log.d(TAG, "Calling using callback"); - mUserAppCallback.onManagerConnected(LoaderCallbackInterface.INIT_FAILED); - } - } - }; - - mUserAppCallback.onPackageInstall(InstallCallbackInterface.INSTALLATION_PROGRESS, WaitQuery); - } - return; - } - else - { - Log.d(TAG, "Trying to get library list"); - mLibraryInstallationProgress = false; - String libs = mEngineService.getLibraryList(mOpenCVersion); - Log.d(TAG, "Library list: \"" + libs + "\""); - Log.d(TAG, "First attempt to load libs"); - int status; - if (initOpenCVLibs(path, libs)) - { - Log.d(TAG, "First attempt to load libs is OK"); - String eol = System.getProperty("line.separator"); - for (String str : Core.getBuildInformation().split(eol)) - Log.i(TAG, str); - - status = LoaderCallbackInterface.SUCCESS; - } - else - { - Log.d(TAG, "First attempt to load libs fails"); - status = LoaderCallbackInterface.INIT_FAILED; - } - - Log.d(TAG, "Init finished with status " + status); - Log.d(TAG, "Unbind from service"); - mAppContext.unbindService(mServiceConnection); - Log.d(TAG, "Calling using callback"); - mUserAppCallback.onManagerConnected(status); - } - } - catch (RemoteException e) - { - e.printStackTrace(); - Log.d(TAG, "Init finished with status " + LoaderCallbackInterface.INIT_FAILED); - Log.d(TAG, "Unbind from service"); - mAppContext.unbindService(mServiceConnection); - Log.d(TAG, "Calling using callback"); - mUserAppCallback.onManagerConnected(LoaderCallbackInterface.INIT_FAILED); - } - } - } - - public void onServiceDisconnected(ComponentName className) - { - mEngineService = null; - } - }; - - private boolean loadLibrary(String AbsPath) - { - boolean result = true; - - Log.d(TAG, "Trying to load library " + AbsPath); - try - { - System.load(AbsPath); - Log.d(TAG, "OpenCV libs init was ok!"); - } - catch(UnsatisfiedLinkError e) - { - Log.d(TAG, "Cannot load library \"" + AbsPath + "\""); - e.printStackTrace(); - result = false; - } - - return result; - } - - private boolean initOpenCVLibs(String Path, String Libs) - { - Log.d(TAG, "Trying to init OpenCV libs"); - if ((null != Path) && (Path.length() != 0)) - { - boolean result = true; - if ((null != Libs) && (Libs.length() != 0)) - { - Log.d(TAG, "Trying to load libs by dependency list"); - StringTokenizer splitter = new StringTokenizer(Libs, ";"); - while(splitter.hasMoreTokens()) - { - String AbsLibraryPath = Path + File.separator + splitter.nextToken(); - result &= loadLibrary(AbsLibraryPath); - } - } - else - { - // If the dependencies list is not defined or empty. - String AbsLibraryPath = Path + File.separator + "libopencv_java4.so"; - result = loadLibrary(AbsLibraryPath); - } - - return result; - } - else - { - Log.d(TAG, "Library path \"" + Path + "\" is empty"); - return false; - } - } -} diff --git a/modules/java/generator/android/java/org/opencv/android/BaseLoaderCallback.java b/modules/java/generator/android/java/org/opencv/android/BaseLoaderCallback.java deleted file mode 100644 index 8ece662514..0000000000 --- a/modules/java/generator/android/java/org/opencv/android/BaseLoaderCallback.java +++ /dev/null @@ -1,141 +0,0 @@ -package org.opencv.android; - -import android.app.Activity; -import android.app.AlertDialog; -import android.content.Context; -import android.content.DialogInterface; -import android.content.DialogInterface.OnClickListener; -import android.util.Log; - -/** - * Basic implementation of LoaderCallbackInterface. - */ -public abstract class BaseLoaderCallback implements LoaderCallbackInterface { - - public BaseLoaderCallback(Context AppContext) { - mAppContext = AppContext; - } - - public void onManagerConnected(int status) - { - switch (status) - { - /** OpenCV initialization was successful. **/ - case LoaderCallbackInterface.SUCCESS: - { - /** Application must override this method to handle successful library initialization. **/ - } break; - /** OpenCV loader can not start Google Play Market. **/ - case LoaderCallbackInterface.MARKET_ERROR: - { - Log.e(TAG, "Package installation failed!"); - AlertDialog MarketErrorMessage = new AlertDialog.Builder(mAppContext).create(); - MarketErrorMessage.setTitle("OpenCV Manager"); - MarketErrorMessage.setMessage("Package installation failed!"); - MarketErrorMessage.setCancelable(false); // This blocks the 'BACK' button - MarketErrorMessage.setButton(AlertDialog.BUTTON_POSITIVE, "OK", new OnClickListener() { - public void onClick(DialogInterface dialog, int which) { - finish(); - } - }); - MarketErrorMessage.show(); - } break; - /** Package installation has been canceled. **/ - case LoaderCallbackInterface.INSTALL_CANCELED: - { - Log.d(TAG, "OpenCV library installation was canceled by user"); - finish(); - } break; - /** Application is incompatible with this version of OpenCV Manager. Possibly, a service update is required. **/ - case LoaderCallbackInterface.INCOMPATIBLE_MANAGER_VERSION: - { - Log.d(TAG, "OpenCV Manager Service is uncompatible with this app!"); - AlertDialog IncomatibilityMessage = new AlertDialog.Builder(mAppContext).create(); - IncomatibilityMessage.setTitle("OpenCV Manager"); - IncomatibilityMessage.setMessage("OpenCV Manager service is incompatible with this app. Try to update it via Google Play."); - IncomatibilityMessage.setCancelable(false); // This blocks the 'BACK' button - IncomatibilityMessage.setButton(AlertDialog.BUTTON_POSITIVE, "OK", new OnClickListener() { - public void onClick(DialogInterface dialog, int which) { - finish(); - } - }); - IncomatibilityMessage.show(); - } break; - /** Other status, i.e. INIT_FAILED. **/ - default: - { - Log.e(TAG, "OpenCV loading failed!"); - AlertDialog InitFailedDialog = new AlertDialog.Builder(mAppContext).create(); - InitFailedDialog.setTitle("OpenCV error"); - InitFailedDialog.setMessage("OpenCV was not initialised correctly. Application will be shut down"); - InitFailedDialog.setCancelable(false); // This blocks the 'BACK' button - InitFailedDialog.setButton(AlertDialog.BUTTON_POSITIVE, "OK", new OnClickListener() { - - public void onClick(DialogInterface dialog, int which) { - finish(); - } - }); - - InitFailedDialog.show(); - } break; - } - } - - public void onPackageInstall(final int operation, final InstallCallbackInterface callback) - { - switch (operation) - { - case InstallCallbackInterface.NEW_INSTALLATION: - { - AlertDialog InstallMessage = new AlertDialog.Builder(mAppContext).create(); - InstallMessage.setTitle("Package not found"); - InstallMessage.setMessage(callback.getPackageName() + " package was not found! Try to install it?"); - InstallMessage.setCancelable(false); // This blocks the 'BACK' button - InstallMessage.setButton(AlertDialog.BUTTON_POSITIVE, "Yes", new OnClickListener() - { - public void onClick(DialogInterface dialog, int which) - { - callback.install(); - } - }); - - InstallMessage.setButton(AlertDialog.BUTTON_NEGATIVE, "No", new OnClickListener() { - - public void onClick(DialogInterface dialog, int which) - { - callback.cancel(); - } - }); - - InstallMessage.show(); - } break; - case InstallCallbackInterface.INSTALLATION_PROGRESS: - { - AlertDialog WaitMessage = new AlertDialog.Builder(mAppContext).create(); - WaitMessage.setTitle("OpenCV is not ready"); - WaitMessage.setMessage("Installation is in progress. Wait or exit?"); - WaitMessage.setCancelable(false); // This blocks the 'BACK' button - WaitMessage.setButton(AlertDialog.BUTTON_POSITIVE, "Wait", new OnClickListener() { - public void onClick(DialogInterface dialog, int which) { - callback.wait_install(); - } - }); - WaitMessage.setButton(AlertDialog.BUTTON_NEGATIVE, "Exit", new OnClickListener() { - public void onClick(DialogInterface dialog, int which) { - callback.cancel(); - } - }); - - WaitMessage.show(); - } break; - } - } - - void finish() - { - ((Activity) mAppContext).finish(); - } - - protected Context mAppContext; - private final static String TAG = "OCV/BaseLoaderCallback"; -} diff --git a/modules/java/generator/android/java/org/opencv/android/InstallCallbackInterface.java b/modules/java/generator/android/java/org/opencv/android/InstallCallbackInterface.java deleted file mode 100644 index f68027a7ba..0000000000 --- a/modules/java/generator/android/java/org/opencv/android/InstallCallbackInterface.java +++ /dev/null @@ -1,34 +0,0 @@ -package org.opencv.android; - -/** - * Installation callback interface. - */ -public interface InstallCallbackInterface -{ - /** - * New package installation is required. - */ - static final int NEW_INSTALLATION = 0; - /** - * Current package installation is in progress. - */ - static final int INSTALLATION_PROGRESS = 1; - - /** - * Target package name. - * @return Return target package name. - */ - public String getPackageName(); - /** - * Installation is approved. - */ - public void install(); - /** - * Installation is canceled. - */ - public void cancel(); - /** - * Wait for package installation. - */ - public void wait_install(); -}; diff --git a/modules/java/generator/android/java/org/opencv/android/LoaderCallbackInterface.java b/modules/java/generator/android/java/org/opencv/android/LoaderCallbackInterface.java deleted file mode 100644 index a941e8377b..0000000000 --- a/modules/java/generator/android/java/org/opencv/android/LoaderCallbackInterface.java +++ /dev/null @@ -1,40 +0,0 @@ -package org.opencv.android; - -/** - * Interface for callback object in case of asynchronous initialization of OpenCV. - */ -public interface LoaderCallbackInterface -{ - /** - * OpenCV initialization finished successfully. - */ - static final int SUCCESS = 0; - /** - * Google Play Market cannot be invoked. - */ - static final int MARKET_ERROR = 2; - /** - * OpenCV library installation has been canceled by the user. - */ - static final int INSTALL_CANCELED = 3; - /** - * This version of OpenCV Manager Service is incompatible with the app. Possibly, a service update is required. - */ - static final int INCOMPATIBLE_MANAGER_VERSION = 4; - /** - * OpenCV library initialization has failed. - */ - static final int INIT_FAILED = 0xff; - - /** - * Callback method, called after OpenCV library initialization. - * @param status status of initialization (see initialization status constants). - */ - public void onManagerConnected(int status); - - /** - * Callback method, called in case the package installation is needed. - * @param callback answer object with approve and cancel methods and the package description. - */ - public void onPackageInstall(final int operation, InstallCallbackInterface callback); -}; diff --git a/modules/java/generator/android/java/org/opencv/android/OpenCVLoader.java.in b/modules/java/generator/android/java/org/opencv/android/OpenCVLoader.java.in index 625c3daf27..91cc534e84 100644 --- a/modules/java/generator/android/java/org/opencv/android/OpenCVLoader.java.in +++ b/modules/java/generator/android/java/org/opencv/android/OpenCVLoader.java.in @@ -7,102 +7,26 @@ import android.content.Context; */ public class OpenCVLoader { - /** - * OpenCV Library version 2.4.2. - */ - public static final String OPENCV_VERSION_2_4_2 = "2.4.2"; - - /** - * OpenCV Library version 2.4.3. - */ - public static final String OPENCV_VERSION_2_4_3 = "2.4.3"; - - /** - * OpenCV Library version 2.4.4. - */ - public static final String OPENCV_VERSION_2_4_4 = "2.4.4"; - - /** - * OpenCV Library version 2.4.5. - */ - public static final String OPENCV_VERSION_2_4_5 = "2.4.5"; - - /** - * OpenCV Library version 2.4.6. - */ - public static final String OPENCV_VERSION_2_4_6 = "2.4.6"; - - /** - * OpenCV Library version 2.4.7. - */ - public static final String OPENCV_VERSION_2_4_7 = "2.4.7"; - - /** - * OpenCV Library version 2.4.8. - */ - public static final String OPENCV_VERSION_2_4_8 = "2.4.8"; - - /** - * OpenCV Library version 2.4.9. - */ - public static final String OPENCV_VERSION_2_4_9 = "2.4.9"; - - /** - * OpenCV Library version 2.4.10. - */ - public static final String OPENCV_VERSION_2_4_10 = "2.4.10"; - - /** - * OpenCV Library version 2.4.11. - */ - public static final String OPENCV_VERSION_2_4_11 = "2.4.11"; - - /** - * OpenCV Library version 2.4.12. - */ - public static final String OPENCV_VERSION_2_4_12 = "2.4.12"; - - /** - * OpenCV Library version 2.4.13. - */ - public static final String OPENCV_VERSION_2_4_13 = "2.4.13"; - - /** - * OpenCV Library version 3.0.0. - */ - public static final String OPENCV_VERSION_3_0_0 = "3.0.0"; - - /** - * OpenCV Library version 3.1.0. - */ - public static final String OPENCV_VERSION_3_1_0 = "3.1.0"; - - /** - * OpenCV Library version 3.2.0. - */ - public static final String OPENCV_VERSION_3_2_0 = "3.2.0"; - - /** - * OpenCV Library version 3.3.0. - */ - public static final String OPENCV_VERSION_3_3_0 = "3.3.0"; - - /** - * OpenCV Library version 3.4.0. - */ - public static final String OPENCV_VERSION_3_4_0 = "3.4.0"; - /** * Current OpenCV Library version */ public static final String OPENCV_VERSION = "@OPENCV_VERSION_MAJOR@.@OPENCV_VERSION_MINOR@.@OPENCV_VERSION_PATCH@"; + /** + * Synonym for initLocal. Deprecated. + */ + @Deprecated + public static boolean initDebug() + { + return StaticHelper.initOpenCV(false); + } + /** * Loads and initializes OpenCV library from current application package. Roughly, it's an analog of system.loadLibrary("opencv_java"). * @return Returns true is initialization of OpenCV was successful. */ - public static boolean initDebug() + public static boolean initLocal() { return StaticHelper.initOpenCV(false); } @@ -112,21 +36,9 @@ public class OpenCVLoader * @param InitCuda load and initialize CUDA runtime libraries. * @return Returns true is initialization of OpenCV was successful. */ + @Deprecated public static boolean initDebug(boolean InitCuda) { return StaticHelper.initOpenCV(InitCuda); } - - /** - * Loads and initializes OpenCV library using OpenCV Engine service. - * @param Version OpenCV library version. - * @param AppContext application context for connecting to the service. - * @param Callback object, that implements LoaderCallbackInterface for handling the connection status. - * @return Returns true if initialization of OpenCV is successful. - */ - public static boolean initAsync(String Version, Context AppContext, - LoaderCallbackInterface Callback) - { - return AsyncServiceHelper.initOpenCV(Version, AppContext, Callback); - } } diff --git a/modules/java/generator/android/java/org/opencv/engine/OpenCVEngineInterface.aidl b/modules/java/generator/android/java/org/opencv/engine/OpenCVEngineInterface.aidl deleted file mode 100644 index 21fe5f716b..0000000000 --- a/modules/java/generator/android/java/org/opencv/engine/OpenCVEngineInterface.aidl +++ /dev/null @@ -1,33 +0,0 @@ -package org.opencv.engine; - -/** -* Class provides a Java interface for OpenCV Engine Service. It's synchronous with native OpenCVEngine class. -*/ -interface OpenCVEngineInterface -{ - /** - * @return Returns service version. - */ - int getEngineVersion(); - - /** - * Finds an installed OpenCV library. - * @param OpenCV version. - * @return Returns path to OpenCV native libs or an empty string if OpenCV can not be found. - */ - String getLibPathByVersion(String version); - - /** - * Tries to install defined version of OpenCV from Google Play Market. - * @param OpenCV version. - * @return Returns true if installation was successful or OpenCV package has been already installed. - */ - boolean installVersion(String version); - - /** - * Returns list of libraries in loading order, separated by semicolon. - * @param OpenCV version. - * @return Returns names of OpenCV libraries, separated by semicolon. - */ - String getLibraryList(String version); -} diff --git a/platforms/android/service/CMakeLists.txt b/platforms/android/service/CMakeLists.txt deleted file mode 100644 index 66e0c468a9..0000000000 --- a/platforms/android/service/CMakeLists.txt +++ /dev/null @@ -1,10 +0,0 @@ -if(NOT ANDROID_PROJECTS_BUILD_TYPE STREQUAL "ANT") - message(STATUS "Android OpenCV Manager is ignored") - return() -endif() - -if(BUILD_ANDROID_SERVICE) - add_subdirectory(engine) -endif() - -install(FILES "readme.txt" DESTINATION "apk/" COMPONENT libs) diff --git a/platforms/android/service/engine/AndroidManifest.xml b/platforms/android/service/engine/AndroidManifest.xml deleted file mode 100644 index 660152ed29..0000000000 --- a/platforms/android/service/engine/AndroidManifest.xml +++ /dev/null @@ -1,30 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - diff --git a/platforms/android/service/engine/CMakeLists.txt b/platforms/android/service/engine/CMakeLists.txt deleted file mode 100644 index a31790a903..0000000000 --- a/platforms/android/service/engine/CMakeLists.txt +++ /dev/null @@ -1,3 +0,0 @@ -configure_file("${CMAKE_CURRENT_SOURCE_DIR}/${ANDROID_MANIFEST_FILE}" "${OpenCV_BINARY_DIR}/platforms/android/service/engine/.build/${ANDROID_MANIFEST_FILE}" @ONLY) -unset(__android_project_chain CACHE) -add_android_project(opencv_engine "${CMAKE_CURRENT_SOURCE_DIR}" SDK_TARGET 9 ${ANDROID_SDK_TARGET} IGNORE_JAVA ON IGNORE_MANIFEST ON COPY_LIBS ON) diff --git a/platforms/android/service/engine/res/drawable/icon.png b/platforms/android/service/engine/res/drawable/icon.png deleted file mode 100644 index 630454927b592eb585c21527c430fc739c7970a6..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1997 zcmV;;2Qv7HP)Px#24YJ`L;(K){{a7>y{D4^000SaNLh0L04^f{04^f|c%?sf00007bV*G`2iyk& z2s4Z(uWcvB1k+?B|HcZ5+p(*;Q^&-8AtzSnVCK7?)^Xiwf0*7z0ZN;t#z9i`sgtpqdV3hDNF*c2bKc!qcQaX zUny)Tz}^_l!dPca%!U9i64N>!i~1_h=?F58~*Mqs?aUag-wNp3eJ zaArHlobMV1PMO^yg*noOAUz|E0i{}8`nIiHOW|~F4mjoR_GH_vZVKN?bHNdXe;To> zxdyn_TnD>62p5lGlR~e9qrf7C+ui6sX@;J1@Mx>Yo=qMBRs~*)bDAoXUZYSHTab?f z_PACCXF@bcF}lW`X>mi~9D%@SGZ2{F%CTpbhLf>?v)%*vE3D~)z{*wz=t^t3pfEGA zffL(4AU5DX%yUkKoB_Jb(8oFW!NI-`j{z#&Z&^_sT-34vIXFYp`=GEPgYAu)4n9D4 z%K`*+8v7m2z|O!Kz|Xto?PB|!;VL`0G=ur`jDlQ$D=+i6dSt)j#Si?i#3rh3ZDkkx z+9TWVDcFEP;8fsZmts4LZyQ^=NS&&oMq^DBPd9*r!p~~YrdwMdQuxcK)CgcvlC7iQ z6m}XDL=_ke!d;S<2IvOy5aI>4B-sk!6i-qAqn6i#qR%}BH;aqaE~#zv6|u2ViZHn? z1hW8E8rkz`nyn$2WU0dhK4^p<)W{{jH|2^S^#FZ&jV&V)7;HC58VOgls*{S?bCuB! zgJG~P)&TrE+Oa9jPq^~G`MQI^ISDar1?}7f7D?&qi-Zc{Oivqep0%xlm22B7?$lt? zRDoZ$&ZSsjO2nE0ft%CR$aVrKp5WRX7^zue>Cw3+QGx=M@MFBYs{CEWmLZ zN*9hpz)s*B96QRjjj`Qi_;Vb>Z73h0DK9}$-axr}l%1BFSp9)x9Ky;`(^n%*$`F!V zkY*oPaAK+6&unXUeSit`5c;F3iU&9)kOVILi%-C3#za9jc>jUr;Wvs~#jZ#FaZ!2fi34Sb1N@BsFCj&@J zs;eb(o@Ffe=D?d6qPKGdXKQjn9~5j94Pu>ge`)`61V~3frX{l7>dTEi%6<8;N9K*(u}%^bWomltk*Vp@k|^lo)yA?qH}(jBlCHBo51XA|gNf(* zoeh<>k{r9Y4)pw$!k7FX&+PyB(?mq~(*^;K7{UQF&X#s!ILS zHX&7zhmyZ|_z=x$C7sRWW=r7+&daHU&gPWWQt)uC*X>tEFZ6J;H6WaAcCCQRo2VjP z>v8^kmJJ*U_eqdHY-p9+xixW&Umgf^mpLS>_nMj zvaJlD2pv8o7#@|SuT=D$XZ)5=GJyZFvEPhNoO#NE^Nc>q2{?8F#Z+({^MQk9zw9zH z=)VjA4HzfT(Fq(eDSwVGl!7_^3!$70OgHG7MYIxpx7Q{~Cgnag|7WoceAizs@IKwlGBBq*Ioi0qby4ylKkgqvR5BcLR&Vy39?8 - - - - - - - - - - - - -