diff --git a/modules/core/include/opencv2/core/hal/intrin_rvv_scalable.hpp b/modules/core/include/opencv2/core/hal/intrin_rvv_scalable.hpp index 05fdbc4bdc..f4142f6af3 100644 --- a/modules/core/include/opencv2/core/hal/intrin_rvv_scalable.hpp +++ b/modules/core/include/opencv2/core/hal/intrin_rvv_scalable.hpp @@ -939,7 +939,7 @@ inline scalartype v_reduce_sum(const _Tpvec& a) \ } OPENCV_HAL_IMPL_RVV_REDUCE_SUM_FP(v_float32, v_float32, vfloat32m1_t, float, f32, VTraits::vlanes()) #if CV_SIMD_SCALABLE_64F -OPENCV_HAL_IMPL_RVV_REDUCE_SUM_FP(v_float64, v_float64, vfloat64m1_t, float, f64, VTraits::vlanes()) +OPENCV_HAL_IMPL_RVV_REDUCE_SUM_FP(v_float64, v_float64, vfloat64m1_t, double, f64, VTraits::vlanes()) #endif #define OPENCV_HAL_IMPL_RVV_REDUCE(_Tpvec, _nTpvec, func, scalartype, suffix, vl, red) \ diff --git a/modules/core/perf/opencl/perf_arithm.cpp b/modules/core/perf/opencl/perf_arithm.cpp index 2ee7862aa8..c7f8736b58 100644 --- a/modules/core/perf/opencl/perf_arithm.cpp +++ b/modules/core/perf/opencl/perf_arithm.cpp @@ -1202,6 +1202,9 @@ typedef TestBaseWithParam ReduceMinMaxFixture; OCL_PERF_TEST_P(ReduceMinMaxFixture, Reduce, ::testing::Combine(OCL_TEST_SIZES, OCL_PERF_ENUM(std::make_pair(CV_8UC1, CV_8UC1), + std::make_pair(CV_8UC4, CV_8UC4), + std::make_pair(CV_32FC1, CV_32FC1), + std::make_pair(CV_32FC3, CV_32FC3), std::make_pair(CV_32FC4, CV_32FC4)), OCL_PERF_ENUM(0, 1), ReduceMinMaxOp::all())) diff --git a/modules/core/src/matrix_operations.cpp b/modules/core/src/matrix_operations.cpp index f1ec202f1e..56ef76d896 100644 --- a/modules/core/src/matrix_operations.cpp +++ b/modules/core/src/matrix_operations.cpp @@ -338,9 +338,13 @@ cv::Mat cv::Mat::cross(InputArray _m) const namespace cv { -typedef void (*ReduceSumFunc)(const Mat& src, Mat& dst); -ReduceSumFunc getReduceCSumFunc(int sdepth, int ddepth); -ReduceSumFunc getReduceRSumFunc(int sdepth, int ddepth); +typedef void (*ReduceFunc)( const Mat& src, Mat& dst ); +ReduceFunc getReduceCSumFunc(int sdepth, int ddepth); +ReduceFunc getReduceCAvgFunc(int sdepth, int ddepth); +ReduceFunc getReduceCMaxFunc(int sdepth, int ddepth); +ReduceFunc getReduceCMinFunc(int sdepth, int ddepth); +ReduceFunc getReduceCSum2Func(int sdepth, int ddepth); +ReduceFunc getReduceRSumFunc(int sdepth, int ddepth); template struct ReduceR_SIMD @@ -351,7 +355,6 @@ struct ReduceR_SIMD } }; - template class ReduceR_Invoker : public ParallelLoopBody { @@ -471,8 +474,6 @@ reduceC_( const Mat& srcmat, Mat& dstmat) parallel_for_(Range(0, srcmat.size().height), body); } -typedef void (*ReduceFunc)( const Mat& src, Mat& dst ); - } #define reduceSumR8u32s reduceR_, OpNop > @@ -818,9 +819,9 @@ void cv::reduce(InputArray _src, OutputArray _dst, int dim, int op, int dtype) { if( op == REDUCE_SUM ) { - ReduceSumFunc simd_func = getReduceRSumFunc(sdepth, ddepth); + ReduceFunc simd_func = getReduceRSumFunc(sdepth, ddepth); if(simd_func) - func = (ReduceFunc)simd_func; + func = simd_func; else if(sdepth == CV_8U && ddepth == CV_32S) func = reduceSumR8u32s; else if(sdepth == CV_8U && ddepth == CV_32F) @@ -896,9 +897,11 @@ void cv::reduce(InputArray _src, OutputArray _dst, int dim, int op, int dtype) { if(op == REDUCE_SUM) { - ReduceSumFunc simd_func = getReduceCSumFunc(sdepth, ddepth); + ReduceFunc simd_func = op0 == REDUCE_AVG + ? getReduceCAvgFunc(sdepth, ddepth) + : getReduceCSumFunc(sdepth, ddepth); if(simd_func) - func = (ReduceFunc)simd_func; + func = simd_func; else if(sdepth == CV_8U && ddepth == CV_32S) func = reduceSumC8u32s; else if(sdepth == CV_8U && ddepth == CV_32F) @@ -922,7 +925,10 @@ void cv::reduce(InputArray _src, OutputArray _dst, int dim, int op, int dtype) } else if(op == REDUCE_MAX) { - if(sdepth == CV_8U && ddepth == CV_8U) + ReduceFunc simd_func = getReduceCMaxFunc(sdepth, ddepth); + if(simd_func) + func = simd_func; + else if(sdepth == CV_8U && ddepth == CV_8U) func = reduceMaxC8u; else if(sdepth == CV_16U && ddepth == CV_16U) func = reduceMaxC16u; @@ -935,7 +941,10 @@ void cv::reduce(InputArray _src, OutputArray _dst, int dim, int op, int dtype) } else if(op == REDUCE_MIN) { - if(sdepth == CV_8U && ddepth == CV_8U) + ReduceFunc simd_func = getReduceCMinFunc(sdepth, ddepth); + if(simd_func) + func = simd_func; + else if(sdepth == CV_8U && ddepth == CV_8U) func = reduceMinC8u; else if(sdepth == CV_16U && ddepth == CV_16U) func = reduceMinC16u; @@ -948,7 +957,10 @@ void cv::reduce(InputArray _src, OutputArray _dst, int dim, int op, int dtype) } else if(op == REDUCE_SUM2) { - if(sdepth == CV_8U && ddepth == CV_32S) + ReduceFunc simd_func = getReduceCSum2Func(sdepth, ddepth); + if(simd_func) + func = simd_func; + else if(sdepth == CV_8U && ddepth == CV_32S) func = reduceSum2C8u32s; else if(sdepth == CV_8U && ddepth == CV_32F) func = reduceSum2C8u32f; @@ -1038,7 +1050,6 @@ template static void sort_( const Mat& src, Mat& dst, int flags ) }); } - #if defined(HAVE_IPP) && !IPP_DISABLE_SORT typedef IppStatus (CV_STDCALL *IppSortFunc)(void *pSrcDst, int len, Ipp8u *pBuffer); diff --git a/modules/core/src/reduce.dispatch.cpp b/modules/core/src/reduce.dispatch.cpp index 0b63560780..0c2d316145 100644 --- a/modules/core/src/reduce.dispatch.cpp +++ b/modules/core/src/reduce.dispatch.cpp @@ -9,18 +9,50 @@ namespace cv { -typedef void (*ReduceSumFunc)(const Mat& src, Mat& dst); -ReduceSumFunc getReduceCSumFunc(int sdepth, int ddepth); -ReduceSumFunc getReduceRSumFunc(int sdepth, int ddepth); +typedef void (*ReduceFunc)( const Mat& src, Mat& dst ); +ReduceFunc getReduceCSumFunc(int sdepth, int ddepth); +ReduceFunc getReduceCAvgFunc(int sdepth, int ddepth); +ReduceFunc getReduceCMaxFunc(int sdepth, int ddepth); +ReduceFunc getReduceCMinFunc(int sdepth, int ddepth); +ReduceFunc getReduceCSum2Func(int sdepth, int ddepth); +ReduceFunc getReduceRSumFunc(int sdepth, int ddepth); -ReduceSumFunc getReduceCSumFunc(int sdepth, int ddepth) +ReduceFunc getReduceCSumFunc(int sdepth, int ddepth) { CV_INSTRUMENT_REGION(); CV_CPU_DISPATCH(getReduceCSumFunc, (sdepth, ddepth), CV_CPU_DISPATCH_MODES_ALL); } -ReduceSumFunc getReduceRSumFunc(int sdepth, int ddepth) +ReduceFunc getReduceCAvgFunc(int sdepth, int ddepth) +{ + CV_INSTRUMENT_REGION(); + CV_CPU_DISPATCH(getReduceCAvgFunc, (sdepth, ddepth), + CV_CPU_DISPATCH_MODES_ALL); +} + +ReduceFunc getReduceCMaxFunc(int sdepth, int ddepth) +{ + CV_INSTRUMENT_REGION(); + CV_CPU_DISPATCH(getReduceCMaxFunc, (sdepth, ddepth), + CV_CPU_DISPATCH_MODES_ALL); +} + +ReduceFunc getReduceCMinFunc(int sdepth, int ddepth) +{ + CV_INSTRUMENT_REGION(); + CV_CPU_DISPATCH(getReduceCMinFunc, (sdepth, ddepth), + CV_CPU_DISPATCH_MODES_ALL); +} + +ReduceFunc getReduceCSum2Func(int sdepth, int ddepth) +{ + CV_INSTRUMENT_REGION(); + CV_CPU_DISPATCH(getReduceCSum2Func, (sdepth, ddepth), + CV_CPU_DISPATCH_MODES_ALL); +} + +ReduceFunc getReduceRSumFunc(int sdepth, int ddepth) { CV_INSTRUMENT_REGION(); CV_CPU_DISPATCH(getReduceRSumFunc, (sdepth, ddepth), diff --git a/modules/core/src/reduce.simd.hpp b/modules/core/src/reduce.simd.hpp index 374853bb29..b1a40b5dc1 100644 --- a/modules/core/src/reduce.simd.hpp +++ b/modules/core/src/reduce.simd.hpp @@ -7,12 +7,20 @@ namespace cv { CV_CPU_OPTIMIZATION_NAMESPACE_BEGIN -typedef void (*ReduceSumFunc)(const Mat& src, Mat& dst); -ReduceSumFunc getReduceCSumFunc(int sdepth, int ddepth); -ReduceSumFunc getReduceRSumFunc(int sdepth, int ddepth); +typedef void (*ReduceFunc)( const Mat& src, Mat& dst ); +ReduceFunc getReduceCSumFunc(int sdepth, int ddepth); +ReduceFunc getReduceCAvgFunc(int sdepth, int ddepth); +ReduceFunc getReduceCMaxFunc(int sdepth, int ddepth); +ReduceFunc getReduceCMinFunc(int sdepth, int ddepth); +ReduceFunc getReduceCSum2Func(int sdepth, int ddepth); +ReduceFunc getReduceRSumFunc(int sdepth, int ddepth); #ifndef CV_CPU_OPTIMIZATION_DECLARATIONS_ONLY +#if (CV_SIMD || CV_SIMD_SCALABLE) +#include "reduce_c.simd.hpp" +#endif + // ===================================================================== // Col reduce SUM (dim=1): sum each row into cn output values // ===================================================================== @@ -1089,7 +1097,7 @@ static void reduceRowSum_64f64f(const Mat& srcmat, Mat& dstmat) // Dispatchers // ===================================================================== -ReduceSumFunc getReduceCSumFunc(int sdepth, int ddepth) +ReduceFunc getReduceCSumFunc(int sdepth, int ddepth) { #if (CV_SIMD || CV_SIMD_SCALABLE) if (sdepth == CV_8U && ddepth == CV_32S) return reduceColSum_8u32s; @@ -1108,7 +1116,68 @@ ReduceSumFunc getReduceCSumFunc(int sdepth, int ddepth) return nullptr; } -ReduceSumFunc getReduceRSumFunc(int sdepth, int ddepth) +ReduceFunc getReduceCAvgFunc(int sdepth, int ddepth) +{ + return getReduceCSumFunc(sdepth, ddepth); +} + +ReduceFunc getReduceCMaxFunc(int sdepth, int ddepth) +{ +#if (CV_SIMD || CV_SIMD_SCALABLE) + if (sdepth == CV_8U && ddepth == CV_8U) return reduceColMax_8u; + if (sdepth == CV_16U && ddepth == CV_16U) return reduceColMax_16u; + if (sdepth == CV_16S && ddepth == CV_16S) return reduceColMax_16s; + if (sdepth == CV_32F && ddepth == CV_32F) return reduceColMax_32f; +#if (CV_SIMD_64F || CV_SIMD_SCALABLE_64F) + if (sdepth == CV_64F && ddepth == CV_64F) return reduceColMax_64f; +#endif +#else + CV_UNUSED(sdepth); + CV_UNUSED(ddepth); +#endif + return nullptr; +} + +ReduceFunc getReduceCMinFunc(int sdepth, int ddepth) +{ +#if (CV_SIMD || CV_SIMD_SCALABLE) + if (sdepth == CV_8U && ddepth == CV_8U) return reduceColMin_8u; + if (sdepth == CV_16U && ddepth == CV_16U) return reduceColMin_16u; + if (sdepth == CV_16S && ddepth == CV_16S) return reduceColMin_16s; + if (sdepth == CV_32F && ddepth == CV_32F) return reduceColMin_32f; +#if (CV_SIMD_64F || CV_SIMD_SCALABLE_64F) + if (sdepth == CV_64F && ddepth == CV_64F) return reduceColMin_64f; +#endif +#else + CV_UNUSED(sdepth); + CV_UNUSED(ddepth); +#endif + return nullptr; +} + +ReduceFunc getReduceCSum2Func(int sdepth, int ddepth) +{ +#if (CV_SIMD || CV_SIMD_SCALABLE) + if (sdepth == CV_8U && ddepth == CV_32S) return reduceColSum2_8u32s; + if (sdepth == CV_8U && ddepth == CV_32F) return reduceColSum2_8u32f; + if (sdepth == CV_8U && ddepth == CV_64F) return reduceColSum2_8u64f; + if (sdepth == CV_16U && ddepth == CV_32F) return reduceColSum2_16u32f; + if (sdepth == CV_16S && ddepth == CV_32F) return reduceColSum2_16s32f; + if (sdepth == CV_32F && ddepth == CV_32F) return reduceColSum2_32f32f; +#if (CV_SIMD_64F || CV_SIMD_SCALABLE_64F) + if (sdepth == CV_16U && ddepth == CV_64F) return reduceColSum2_16u64f; + if (sdepth == CV_16S && ddepth == CV_64F) return reduceColSum2_16s64f; + if (sdepth == CV_32F && ddepth == CV_64F) return reduceColSum2_32f64f; + if (sdepth == CV_64F && ddepth == CV_64F) return reduceColSum2_64f64f; +#endif +#else + CV_UNUSED(sdepth); + CV_UNUSED(ddepth); +#endif + return nullptr; +} + +ReduceFunc getReduceRSumFunc(int sdepth, int ddepth) { #if (CV_SIMD || CV_SIMD_SCALABLE) if (sdepth == CV_8U && ddepth == CV_32S) return reduceRowSum_8u32s; diff --git a/modules/core/src/reduce_c.simd.hpp b/modules/core/src/reduce_c.simd.hpp new file mode 100644 index 0000000000..6a6e428488 --- /dev/null +++ b/modules/core/src/reduce_c.simd.hpp @@ -0,0 +1,539 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html + +#include "reduce_c_generic.hpp" + +#if CV_RVV +#include "reduce_c_rvv.hpp" +#endif + +#if CV_NEON +#include "reduce_c_neon.hpp" +#endif + +#if CV_AVX2 +#include "reduce_c_avx2.hpp" +#endif + +template +static void reduceColMinMax_8uC1(const Mat& srcmat, Mat& dstmat) +{ +#if CV_RVV + reduce_c_rvv::minMax8uC1(srcmat, dstmat); +#elif CV_NEON + reduce_c_neon::minMax8uC1(srcmat, dstmat); +#elif CV_AVX2 + reduce_c_avx2::minMax8uC1(srcmat, dstmat); +#else + reduceColMinMax_8uFallback(srcmat, dstmat); +#endif +} + +template +static void reduceColMinMax_8uC3(const Mat& srcmat, Mat& dstmat) +{ +#if CV_RVV + reduce_c_rvv::minMax8uC3(srcmat, dstmat); +#elif CV_NEON + reduce_c_neon::minMax8uC3(srcmat, dstmat); +#elif CV_AVX2 + reduce_c_avx2::minMax8uC3(srcmat, dstmat); +#else + reduceColMinMax_8uFallback(srcmat, dstmat); +#endif +} + +template +static void reduceColMinMax_8uC4(const Mat& srcmat, Mat& dstmat) +{ +#if CV_RVV + reduce_c_rvv::minMax8uC4(srcmat, dstmat); +#elif CV_NEON + reduce_c_neon::minMax8uC4(srcmat, dstmat); +#elif CV_AVX2 + reduce_c_avx2::minMax8uC4(srcmat, dstmat); +#else + reduceColMinMax_8uFallback(srcmat, dstmat); +#endif +} + +template +static void reduceColMinMax_8u(const Mat& srcmat, Mat& dstmat) +{ + const int cn = srcmat.channels(); + if (cn == 1) + reduceColMinMax_8uC1(srcmat, dstmat); + else if (cn == 3) + reduceColMinMax_8uC3(srcmat, dstmat); + else if (cn == 4) + reduceColMinMax_8uC4(srcmat, dstmat); + else + reduceColMinMax_8uFallback(srcmat, dstmat); +} + +static void reduceColMax_8u(const Mat& srcmat, Mat& dstmat) +{ + reduceColMinMax_8u(srcmat, dstmat); +} + +static void reduceColMin_8u(const Mat& srcmat, Mat& dstmat) +{ + reduceColMinMax_8u(srcmat, dstmat); +} + +template +static void reduceColMinMax_16uC1(const Mat& srcmat, Mat& dstmat) +{ +#if CV_RVV + reduce_c_rvv::minMax16uC1(srcmat, dstmat); +#elif CV_NEON + reduce_c_neon::minMax16uC1(srcmat, dstmat); +#elif CV_AVX2 + reduce_c_avx2::minMax16uC1(srcmat, dstmat); +#else + reduceColMinMax_16uFallback(srcmat, dstmat); +#endif +} + +template +static void reduceColMinMax_16uC4(const Mat& srcmat, Mat& dstmat) +{ +#if CV_RVV + reduce_c_rvv::minMax16uC4(srcmat, dstmat); +#elif CV_NEON + reduce_c_neon::minMax16uC4(srcmat, dstmat); +#elif CV_AVX2 + reduce_c_avx2::minMax16uC4(srcmat, dstmat); +#else + reduceColMinMax_16uFallback(srcmat, dstmat); +#endif +} + +template +static void reduceColMinMax_16uC3(const Mat& srcmat, Mat& dstmat) +{ +#if CV_RVV + reduce_c_rvv::minMax16uC3(srcmat, dstmat); +#elif CV_NEON + reduce_c_neon::minMax16uC3(srcmat, dstmat); +#elif CV_AVX2 + reduce_c_avx2::minMax16uC3(srcmat, dstmat); +#else + reduceColMinMax_16uFallback(srcmat, dstmat); +#endif +} + +template +static void reduceColMinMax_16u(const Mat& srcmat, Mat& dstmat) +{ + if (srcmat.channels() == 1) + reduceColMinMax_16uC1(srcmat, dstmat); + else if (srcmat.channels() == 3) + reduceColMinMax_16uC3(srcmat, dstmat); + else if (srcmat.channels() == 4) + reduceColMinMax_16uC4(srcmat, dstmat); + else + reduceColMinMax_16uFallback(srcmat, dstmat); +} + +static void reduceColMax_16u(const Mat& srcmat, Mat& dstmat) +{ + reduceColMinMax_16u(srcmat, dstmat); +} + +static void reduceColMin_16u(const Mat& srcmat, Mat& dstmat) +{ + reduceColMinMax_16u(srcmat, dstmat); +} + +template +static void reduceColMinMax_16sC1(const Mat& srcmat, Mat& dstmat) +{ +#if CV_RVV + reduce_c_rvv::minMax16sC1(srcmat, dstmat); +#elif CV_NEON + reduce_c_neon::minMax16sC1(srcmat, dstmat); +#elif CV_AVX2 + reduce_c_avx2::minMax16sC1(srcmat, dstmat); +#else + reduceColMinMax_16sFallback(srcmat, dstmat); +#endif +} + +template +static void reduceColMinMax_16sC4(const Mat& srcmat, Mat& dstmat) +{ +#if CV_RVV + reduce_c_rvv::minMax16sC4(srcmat, dstmat); +#elif CV_NEON + reduce_c_neon::minMax16sC4(srcmat, dstmat); +#elif CV_AVX2 + reduce_c_avx2::minMax16sC4(srcmat, dstmat); +#else + reduceColMinMax_16sFallback(srcmat, dstmat); +#endif +} + +template +static void reduceColMinMax_16sC3(const Mat& srcmat, Mat& dstmat) +{ +#if CV_RVV + reduce_c_rvv::minMax16sC3(srcmat, dstmat); +#elif CV_NEON + reduce_c_neon::minMax16sC3(srcmat, dstmat); +#elif CV_AVX2 + reduce_c_avx2::minMax16sC3(srcmat, dstmat); +#else + reduceColMinMax_16sFallback(srcmat, dstmat); +#endif +} + +template +static void reduceColMinMax_16s(const Mat& srcmat, Mat& dstmat) +{ + if (srcmat.channels() == 1) + reduceColMinMax_16sC1(srcmat, dstmat); + else if (srcmat.channels() == 3) + reduceColMinMax_16sC3(srcmat, dstmat); + else if (srcmat.channels() == 4) + reduceColMinMax_16sC4(srcmat, dstmat); + else + reduceColMinMax_16sFallback(srcmat, dstmat); +} + +static void reduceColMax_16s(const Mat& srcmat, Mat& dstmat) +{ + reduceColMinMax_16s(srcmat, dstmat); +} + +static void reduceColMin_16s(const Mat& srcmat, Mat& dstmat) +{ + reduceColMinMax_16s(srcmat, dstmat); +} + +template +static void reduceColMinMax_32fC1(const Mat& srcmat, Mat& dstmat) +{ +#if CV_RVV + reduce_c_rvv::minMax32fC1(srcmat, dstmat); +#elif CV_NEON + reduce_c_neon::minMax32fC1(srcmat, dstmat); +#elif CV_AVX2 + reduce_c_avx2::minMax32fC1(srcmat, dstmat); +#else + reduceColMinMax_32fFallback(srcmat, dstmat); +#endif +} + +template +static void reduceColMinMax_32fC3(const Mat& srcmat, Mat& dstmat) +{ +#if CV_RVV + reduce_c_rvv::minMax32fC3(srcmat, dstmat); +#else + reduceColMinMax_32fFallback(srcmat, dstmat); +#endif +} + +template +static void reduceColMinMax_32fC4(const Mat& srcmat, Mat& dstmat) +{ +#if CV_RVV + reduce_c_rvv::minMax32fC4(srcmat, dstmat); +#elif CV_NEON + reduce_c_neon::minMax32fC4(srcmat, dstmat); +#elif CV_AVX2 + reduce_c_avx2::minMax32fC4(srcmat, dstmat); +#else + reduceColMinMax_32fFallback(srcmat, dstmat); +#endif +} + +template +static void reduceColMinMax_32f(const Mat& srcmat, Mat& dstmat) +{ + const int cn = srcmat.channels(); + if (cn == 1) + reduceColMinMax_32fC1(srcmat, dstmat); + else if (cn == 3) + reduceColMinMax_32fC3(srcmat, dstmat); + else if (cn == 4) + reduceColMinMax_32fC4(srcmat, dstmat); + else + reduceColMinMax_32fFallback(srcmat, dstmat); +} + +static void reduceColMax_32f(const Mat& srcmat, Mat& dstmat) +{ + reduceColMinMax_32f(srcmat, dstmat); +} + +static void reduceColMin_32f(const Mat& srcmat, Mat& dstmat) +{ + reduceColMinMax_32f(srcmat, dstmat); +} + +#if (CV_SIMD_64F || CV_SIMD_SCALABLE_64F) +template +static void reduceColMinMax_64fC1(const Mat& srcmat, Mat& dstmat) +{ +#if CV_RVV && CV_SIMD_SCALABLE_64F + reduce_c_rvv::minMax64fC1(srcmat, dstmat); +#elif CV_NEON && (defined(__aarch64__) || defined(_M_ARM64)) + reduce_c_neon::minMax64fC1(srcmat, dstmat); +#elif CV_AVX2 + reduce_c_avx2::minMax64fC1(srcmat, dstmat); +#else + reduceColMinMax_64fFallback(srcmat, dstmat); +#endif +} + +template +static void reduceColMinMax_64f(const Mat& srcmat, Mat& dstmat) +{ + if (srcmat.channels() == 1) + reduceColMinMax_64fC1(srcmat, dstmat); + else + reduceColMinMax_64fFallback(srcmat, dstmat); +} + +static void reduceColMax_64f(const Mat& srcmat, Mat& dstmat) +{ + reduceColMinMax_64f(srcmat, dstmat); +} + +static void reduceColMin_64f(const Mat& srcmat, Mat& dstmat) +{ + reduceColMinMax_64f(srcmat, dstmat); +} +#endif + +template +static void reduceColSum2_8uC1(const Mat& srcmat, Mat& dstmat) +{ +#if CV_RVV + reduce_c_rvv::sum2_8uC1
(srcmat, dstmat); +#elif CV_NEON && (defined(__aarch64__) || defined(_M_ARM64)) + reduce_c_neon::sum2_8uC1
(srcmat, dstmat); +#elif CV_AVX2 + reduce_c_avx2::sum2_8uC1
(srcmat, dstmat); +#else + reduceColSum2_8uFallback
(srcmat, dstmat); +#endif +} + +template +static void reduceColSum2_8uC3(const Mat& srcmat, Mat& dstmat) +{ +#if CV_RVV + reduce_c_rvv::sum2_8uC3
(srcmat, dstmat); +#elif CV_NEON && (defined(__aarch64__) || defined(_M_ARM64)) + reduce_c_neon::sum2_8uC3
(srcmat, dstmat); +#elif CV_AVX2 + reduce_c_avx2::sum2_8uC3
(srcmat, dstmat); +#else + reduceColSum2_8uFallback
(srcmat, dstmat); +#endif +} + +template +static void reduceColSum2_8uC4(const Mat& srcmat, Mat& dstmat) +{ +#if CV_RVV + reduce_c_rvv::sum2_8uC4
(srcmat, dstmat); +#elif CV_NEON && (defined(__aarch64__) || defined(_M_ARM64)) + reduce_c_neon::sum2_8uC4
(srcmat, dstmat); +#elif CV_AVX2 + reduce_c_avx2::sum2_8uC4
(srcmat, dstmat); +#else + reduceColSum2_8uFallback
(srcmat, dstmat); +#endif +} + +template +static void reduceColSum2_8u(const Mat& srcmat, Mat& dstmat) +{ + const int cn = srcmat.channels(); + if (cn == 1) + reduceColSum2_8uC1
(srcmat, dstmat); + else if (cn == 3) + reduceColSum2_8uC3
(srcmat, dstmat); + else if (cn == 4) + reduceColSum2_8uC4
(srcmat, dstmat); + else + reduceColSum2_8uFallback
(srcmat, dstmat); +} + +static void reduceColSum2_8u32s(const Mat& srcmat, Mat& dstmat) +{ + reduceColSum2_8u(srcmat, dstmat); +} + +static void reduceColSum2_8u32f(const Mat& srcmat, Mat& dstmat) +{ + reduceColSum2_8u(srcmat, dstmat); +} + +static void reduceColSum2_8u64f(const Mat& srcmat, Mat& dstmat) +{ + reduceColSum2_8u(srcmat, dstmat); +} + +static void reduceColSum2_16u32f(const Mat& srcmat, Mat& dstmat) +{ + if (srcmat.channels() != 1) + { + reduceColSum2_16u32fFallback(srcmat, dstmat); + return; + } +#if CV_RVV + reduce_c_rvv::sum2_16u32fC1(srcmat, dstmat); +#elif CV_NEON && (defined(__aarch64__) || defined(_M_ARM64)) + reduce_c_neon::sum2_16u32fC1(srcmat, dstmat); +#elif CV_AVX2 + reduce_c_avx2::sum2_16u32fC1(srcmat, dstmat); +#else + reduceColSum2_16u32fFallback(srcmat, dstmat); +#endif +} + +static void reduceColSum2_16s32f(const Mat& srcmat, Mat& dstmat) +{ + if (srcmat.channels() != 1) + { + reduceColSum2_16s32fFallback(srcmat, dstmat); + return; + } +#if CV_RVV + reduce_c_rvv::sum2_16s32fC1(srcmat, dstmat); +#elif CV_NEON && (defined(__aarch64__) || defined(_M_ARM64)) + reduce_c_neon::sum2_16s32fC1(srcmat, dstmat); +#elif CV_AVX2 + reduce_c_avx2::sum2_16s32fC1(srcmat, dstmat); +#else + reduceColSum2_16s32fFallback(srcmat, dstmat); +#endif +} + +static void reduceColSum2_32f32fC1(const Mat& srcmat, Mat& dstmat) +{ +#if CV_RVV + reduce_c_rvv::sum2_32fC1(srcmat, dstmat); +#elif CV_NEON + reduce_c_neon::sum2_32fC1(srcmat, dstmat); +#elif CV_AVX2 + reduce_c_avx2::sum2_32fC1(srcmat, dstmat); +#else + reduceColSum2_32f32fFallback(srcmat, dstmat); +#endif +} + +static void reduceColSum2_32f32fC3(const Mat& srcmat, Mat& dstmat) +{ +#if CV_RVV + reduce_c_rvv::sum2_32fC3(srcmat, dstmat); +#else + reduceColSum2_32f32fFallback(srcmat, dstmat); +#endif +} + +static void reduceColSum2_32f32fC4(const Mat& srcmat, Mat& dstmat) +{ +#if CV_RVV + reduce_c_rvv::sum2_32fC4(srcmat, dstmat); +#elif CV_NEON + reduce_c_neon::sum2_32fC4(srcmat, dstmat); +#elif CV_AVX2 + reduce_c_avx2::sum2_32fC4(srcmat, dstmat); +#else + reduceColSum2_32f32fFallback(srcmat, dstmat); +#endif +} + +static void reduceColSum2_32f32f(const Mat& srcmat, Mat& dstmat) +{ + const int cn = srcmat.channels(); + if (cn == 1) + reduceColSum2_32f32fC1(srcmat, dstmat); + else if (cn == 3) + reduceColSum2_32f32fC3(srcmat, dstmat); + else if (cn == 4) + reduceColSum2_32f32fC4(srcmat, dstmat); + else + reduceColSum2_32f32fFallback(srcmat, dstmat); +} + +#if (CV_SIMD_64F || CV_SIMD_SCALABLE_64F) +static void reduceColSum2_16u64f(const Mat& srcmat, Mat& dstmat) +{ + if (srcmat.channels() != 1) + { + reduceColSum2_16u64fFallback(srcmat, dstmat); + return; + } +#if CV_RVV && CV_SIMD_SCALABLE_64F + reduce_c_rvv::sum2_16u64fC1(srcmat, dstmat); +#elif CV_NEON && (defined(__aarch64__) || defined(_M_ARM64)) + reduce_c_neon::sum2_16u64fC1(srcmat, dstmat); +#elif CV_AVX2 + reduce_c_avx2::sum2_16u64fC1(srcmat, dstmat); +#else + reduceColSum2_16u64fFallback(srcmat, dstmat); +#endif +} + +static void reduceColSum2_16s64f(const Mat& srcmat, Mat& dstmat) +{ + if (srcmat.channels() != 1) + { + reduceColSum2_16s64fFallback(srcmat, dstmat); + return; + } +#if CV_RVV && CV_SIMD_SCALABLE_64F + reduce_c_rvv::sum2_16s64fC1(srcmat, dstmat); +#elif CV_NEON && (defined(__aarch64__) || defined(_M_ARM64)) + reduce_c_neon::sum2_16s64fC1(srcmat, dstmat); +#elif CV_AVX2 + reduce_c_avx2::sum2_16s64fC1(srcmat, dstmat); +#else + reduceColSum2_16s64fFallback(srcmat, dstmat); +#endif +} + +static void reduceColSum2_32f64f(const Mat& srcmat, Mat& dstmat) +{ + if (srcmat.channels() != 1) + { + reduceColSum2_32f64fFallback(srcmat, dstmat); + return; + } +#if CV_RVV && CV_SIMD_SCALABLE_64F + reduce_c_rvv::sum2_32f64fC1(srcmat, dstmat); +#elif CV_NEON && (defined(__aarch64__) || defined(_M_ARM64)) + reduce_c_neon::sum2_32f64fC1(srcmat, dstmat); +#elif CV_AVX2 + reduce_c_avx2::sum2_32f64fC1(srcmat, dstmat); +#else + reduceColSum2_32f64fFallback(srcmat, dstmat); +#endif +} + +static void reduceColSum2_64f64f(const Mat& srcmat, Mat& dstmat) +{ + if (srcmat.channels() != 1) + { + reduceColSum2_64f64fFallback(srcmat, dstmat); + return; + } +#if CV_RVV && CV_SIMD_SCALABLE_64F + reduce_c_rvv::sum2_64f64fC1(srcmat, dstmat); +#elif CV_NEON && (defined(__aarch64__) || defined(_M_ARM64)) + reduce_c_neon::sum2_64f64fC1(srcmat, dstmat); +#elif CV_AVX2 + reduce_c_avx2::sum2_64f64fC1(srcmat, dstmat); +#else + reduceColSum2_64f64fFallback(srcmat, dstmat); +#endif +} +#endif diff --git a/modules/core/src/reduce_c_avx2.hpp b/modules/core/src/reduce_c_avx2.hpp new file mode 100644 index 0000000000..1cfac5ce10 --- /dev/null +++ b/modules/core/src/reduce_c_avx2.hpp @@ -0,0 +1,989 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html + +namespace reduce_c_avx2 +{ + +// Optimized ReduceC support in this backend: +// +// | Input -> output type/channel | SUM | AVG | MIN | MAX | SUM2 | +// |------------------------------------|:---:|:---:|:---:|:---:|:----:| +// | 8UC1/C3/C4 -> 8UC1/C3/C4 | - | - | x | x | - | +// | 8UC1/C3/C4 -> 32SC1/C3/C4 | x | x | - | - | x | +// | 8UC1/C3/C4 -> 32FC1/C3/C4 | x | x | - | - | x | +// | 8UC1/C3/C4 -> 64FC1/C3/C4 | - | - | - | - | x | +// | 16UC1/C3/C4 -> 16UC1/C3/C4 | - | - | x | x | - | +// | 16UC1 -> 32FC1 | x | x | - | - | x | +// | 16UC3/C4 -> 32FC3/C4 | x | x | - | - | - | +// | 16UC1 -> 64FC1 | - | - | - | - | x | +// | 16UC3/C4 -> 64FC3/C4 | - | - | - | - | - | +// | 16SC1/C3/C4 -> 16SC1/C3/C4 | - | - | x | x | - | +// | 16SC1 -> 32FC1 | x | x | - | - | x | +// | 16SC3/C4 -> 32FC3/C4 | x | x | - | - | - | +// | 16SC1 -> 64FC1 | - | - | - | - | x | +// | 16SC3/C4 -> 64FC3/C4 | - | - | - | - | - | +// | 32FC1 -> 32FC1 | x | x | x | x | x | +// | 32FC3 -> 32FC3 | x | x | - | - | - | +// | 32FC4 -> 32FC4 | x | x | x | x | x | +// | 32FC1 -> 64FC1 | x | x | - | - | x | +// | 32FC3/C4 -> 64FC3/C4 | x | x | - | - | - | +// | 64FC1 -> 64FC1 | x | x | x | x | x | +// | 64FC3/C4 -> 64FC3/C4 | x | x | - | - | - | +// +// 'x' in SUM/AVG denotes the existing shared universal-intrinsics kernel; 'x' +// in MIN/MAX/SUM2 denotes a native AVX2 kernel. For legal MIN/MAX/SUM2 +// combinations marked '-', and for other channel counts, dispatch uses the +// shared generic fallback. + +template +static void minMax8uC1(const Mat& srcmat, Mat& dstmat) +{ + const int cols = srcmat.cols; + parallel_for_(Range(0, srcmat.rows), [&](const Range& range) { + for (int y = range.start; y < range.end; y++) + { + const uchar* src = srcmat.ptr(y); + uchar* dst = dstmat.ptr(y); + uchar result = isMax ? 0 : UCHAR_MAX; + int x = 0; + __m256i acc = _mm256_set1_epi8((char)result); + for (; x <= cols - 32; x += 32) + { + __m256i v = _mm256_loadu_si256((const __m256i*)(src + x)); + acc = isMax ? _mm256_max_epu8(acc, v) : _mm256_min_epu8(acc, v); + } + uchar lanes[32]; + _mm256_storeu_si256((__m256i*)lanes, acc); + for (int i = 0; i < 32; i++) + result = reduceScalarMinMax(result, lanes[i]); + for (; x < cols; x++) + result = reduceScalarMinMax(result, src[x]); + dst[0] = result; + } + }); + v_cleanup(); +} + +template +static void minMax16uC1(const Mat& srcmat, Mat& dstmat) +{ + const int cols = srcmat.cols; + parallel_for_(Range(0, srcmat.rows), [&](const Range& range) { + for (int y = range.start; y < range.end; y++) + { + const ushort* src = srcmat.ptr(y); + ushort result = isMax ? 0 : USHRT_MAX; + int x = 0; + __m256i acc = _mm256_set1_epi16((short)result); + for (; x <= cols - 16; x += 16) + { + __m256i v = _mm256_loadu_si256((const __m256i*)(src + x)); + acc = isMax ? _mm256_max_epu16(acc, v) : _mm256_min_epu16(acc, v); + } + ushort lanes[16]; + _mm256_storeu_si256((__m256i*)lanes, acc); + for (int i = 0; i < 16; i++) + result = isMax ? std::max(result, lanes[i]) : std::min(result, lanes[i]); + for (; x < cols; x++) + result = isMax ? std::max(result, src[x]) : std::min(result, src[x]); + dstmat.ptr(y)[0] = result; + } + }); + v_cleanup(); +} + +template +static void minMax16sC1(const Mat& srcmat, Mat& dstmat) +{ + const int cols = srcmat.cols; + parallel_for_(Range(0, srcmat.rows), [&](const Range& range) { + for (int y = range.start; y < range.end; y++) + { + const short* src = srcmat.ptr(y); + short result = isMax ? SHRT_MIN : SHRT_MAX; + int x = 0; + __m256i acc = _mm256_set1_epi16(result); + for (; x <= cols - 16; x += 16) + { + __m256i v = _mm256_loadu_si256((const __m256i*)(src + x)); + acc = isMax ? _mm256_max_epi16(acc, v) : _mm256_min_epi16(acc, v); + } + short lanes[16]; + _mm256_storeu_si256((__m256i*)lanes, acc); + for (int i = 0; i < 16; i++) + result = isMax ? std::max(result, lanes[i]) : std::min(result, lanes[i]); + for (; x < cols; x++) + result = isMax ? std::max(result, src[x]) : std::min(result, src[x]); + dstmat.ptr(y)[0] = result; + } + }); + v_cleanup(); +} + +template +static void minMax16uC4(const Mat& srcmat, Mat& dstmat) +{ + const int cols = srcmat.cols; + const ushort initial = isMax ? 0 : USHRT_MAX; + const __m256i accInit = _mm256_set1_epi16((short)initial); + const __m256i validMask = _mm256_broadcastsi128_si256( + _mm_setr_epi8(-1, -1, -1, -1, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0)); + const __m256i masks[4] = { + _mm256_broadcastsi128_si256(_mm_setr_epi8(0, 1, 8, 9, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1)), + _mm256_broadcastsi128_si256(_mm_setr_epi8(2, 3, 10, 11, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1)), + _mm256_broadcastsi128_si256(_mm_setr_epi8(4, 5, 12, 13, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1)), + _mm256_broadcastsi128_si256(_mm_setr_epi8(6, 7, 14, 15, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1)) + }; + parallel_for_(Range(0, srcmat.rows), [&](const Range& range) { + for (int y = range.start; y < range.end; y++) + { + const ushort* src = srcmat.ptr(y); + ushort* dst = dstmat.ptr(y); + __m256i acc[4] = {accInit, accInit, accInit, accInit}; + int x = 0; + for (; x <= cols - 4; x += 4) + { + __m256i v = _mm256_loadu_si256((const __m256i*)(src + x * 4)); + for (int c = 0; c < 4; c++) + { + __m256i vc = _mm256_shuffle_epi8(v, masks[c]); + vc = _mm256_blendv_epi8(accInit, vc, validMask); + acc[c] = isMax ? _mm256_max_epu16(acc[c], vc) + : _mm256_min_epu16(acc[c], vc); + } + } + for (int c = 0; c < 4; c++) + { + ushort lanes[16]; + _mm256_storeu_si256((__m256i*)lanes, acc[c]); + ushort result = initial; + for (int i = 0; i < 16; i++) + result = isMax ? std::max(result, lanes[i]) : std::min(result, lanes[i]); + for (int i = x; i < cols; i++) + result = isMax ? std::max(result, src[i * 4 + c]) + : std::min(result, src[i * 4 + c]); + dst[c] = result; + } + } + }); + v_cleanup(); +} + +template +static void minMax16sC4(const Mat& srcmat, Mat& dstmat) +{ + const int cols = srcmat.cols; + const short initial = isMax ? SHRT_MIN : SHRT_MAX; + const __m256i accInit = _mm256_set1_epi16(initial); + const __m256i validMask = _mm256_broadcastsi128_si256( + _mm_setr_epi8(-1, -1, -1, -1, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0)); + const __m256i masks[4] = { + _mm256_broadcastsi128_si256(_mm_setr_epi8(0, 1, 8, 9, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1)), + _mm256_broadcastsi128_si256(_mm_setr_epi8(2, 3, 10, 11, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1)), + _mm256_broadcastsi128_si256(_mm_setr_epi8(4, 5, 12, 13, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1)), + _mm256_broadcastsi128_si256(_mm_setr_epi8(6, 7, 14, 15, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1)) + }; + parallel_for_(Range(0, srcmat.rows), [&](const Range& range) { + for (int y = range.start; y < range.end; y++) + { + const short* src = srcmat.ptr(y); + short* dst = dstmat.ptr(y); + __m256i acc[4] = {accInit, accInit, accInit, accInit}; + int x = 0; + for (; x <= cols - 4; x += 4) + { + __m256i v = _mm256_loadu_si256((const __m256i*)(src + x * 4)); + for (int c = 0; c < 4; c++) + { + __m256i vc = _mm256_shuffle_epi8(v, masks[c]); + vc = _mm256_blendv_epi8(accInit, vc, validMask); + acc[c] = isMax ? _mm256_max_epi16(acc[c], vc) + : _mm256_min_epi16(acc[c], vc); + } + } + for (int c = 0; c < 4; c++) + { + short lanes[16]; + _mm256_storeu_si256((__m256i*)lanes, acc[c]); + short result = initial; + for (int i = 0; i < 16; i++) + result = isMax ? std::max(result, lanes[i]) : std::min(result, lanes[i]); + for (int i = x; i < cols; i++) + result = isMax ? std::max(result, src[i * 4 + c]) + : std::min(result, src[i * 4 + c]); + dst[c] = result; + } + } + }); + v_cleanup(); +} + +template +static void minMax16C3(const Mat& srcmat, Mat& dstmat) +{ + const int cols = srcmat.cols; + const T initial = isMax ? std::numeric_limits::lowest() : std::numeric_limits::max(); + const __m256i accInit = _mm256_set1_epi16((short)initial); + const __m256i masks[3] = { + _mm256_setr_epi8(0, 1, 6, 7, 12, 13, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + 2, 3, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1), + _mm256_setr_epi8(2, 3, 8, 9, 14, 15, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + 4, 5, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1), + _mm256_setr_epi8(4, 5, 10, 11, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + 0, 1, 6, 7, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1) + }; + const __m256i validMasks[3] = { + _mm256_setr_epi8(-1, -1, -1, -1, -1, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + -1, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0), + _mm256_setr_epi8(-1, -1, -1, -1, -1, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + -1, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0), + _mm256_setr_epi8(-1, -1, -1, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + -1, -1, -1, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0) + }; + parallel_for_(Range(0, srcmat.rows), [&](const Range& range) { + for (int y = range.start; y < range.end; y++) + { + const T* src = srcmat.ptr(y); + T* dst = dstmat.ptr(y); + __m256i acc[3] = {accInit, accInit, accInit}; + int x = 0; + for (; x <= cols - 6; x += 4) + { + __m256i v = _mm256_loadu_si256((const __m256i*)(src + x * 3)); + for (int c = 0; c < 3; c++) + { + __m256i vc = _mm256_shuffle_epi8(v, masks[c]); + vc = _mm256_blendv_epi8(accInit, vc, validMasks[c]); + if (isSigned) + acc[c] = isMax ? _mm256_max_epi16(acc[c], vc) : _mm256_min_epi16(acc[c], vc); + else + acc[c] = isMax ? _mm256_max_epu16(acc[c], vc) : _mm256_min_epu16(acc[c], vc); + } + } + for (int c = 0; c < 3; c++) + { + T lanes[16]; + _mm256_storeu_si256((__m256i*)lanes, acc[c]); + T result = initial; + for (int i = 0; i < 16; i++) + result = isMax ? std::max(result, lanes[i]) : std::min(result, lanes[i]); + for (int i = x; i < cols; i++) + result = isMax ? std::max(result, src[i * 3 + c]) + : std::min(result, src[i * 3 + c]); + dst[c] = result; + } + } + }); + v_cleanup(); +} + +template +static void minMax16uC3(const Mat& srcmat, Mat& dstmat) +{ + minMax16C3(srcmat, dstmat); +} + +template +static void minMax16sC3(const Mat& srcmat, Mat& dstmat) +{ + minMax16C3(srcmat, dstmat); +} + +template +static void minMax8uC3(const Mat& srcmat, Mat& dstmat) +{ + const int cols = srcmat.cols; + const uchar initial = isMax ? 0 : UCHAR_MAX; + const __m256i accInit = _mm256_set1_epi8((char)initial); + const __m256i mask0 = _mm256_setr_epi8( + 0, 3, 6, 9, 12, 15, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + 2, 5, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1); + const __m256i mask1 = _mm256_setr_epi8( + 1, 4, 7, 10, 13, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + 0, 3, 6, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1); + const __m256i mask2 = _mm256_setr_epi8( + 2, 5, 8, 11, 14, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + 1, 4, 7, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1); + const __m256i invalid0 = _mm256_setr_epi8( + 0, 0, 0, 0, 0, 0, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + 0, 0, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1); + const __m256i invalid12 = _mm256_setr_epi8( + 0, 0, 0, 0, 0, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + 0, 0, 0, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1); + + parallel_for_(Range(0, srcmat.rows), [&](const Range& range) { + for (int y = range.start; y < range.end; y++) + { + const uchar* src = srcmat.ptr(y); + uchar* dst = dstmat.ptr(y); + __m256i acc0 = accInit, acc1 = accInit, acc2 = accInit; + int x = 0; + for (; x <= cols - 11; x += 8) + { + __m256i v = _mm256_loadu_si256((const __m256i*)(src + x * 3)); + __m256i v0 = _mm256_shuffle_epi8(v, mask0); + __m256i v1 = _mm256_shuffle_epi8(v, mask1); + __m256i v2 = _mm256_shuffle_epi8(v, mask2); + if (!isMax) + { + v0 = _mm256_or_si256(v0, invalid0); + v1 = _mm256_or_si256(v1, invalid12); + v2 = _mm256_or_si256(v2, invalid12); + } + acc0 = isMax ? _mm256_max_epu8(acc0, v0) : _mm256_min_epu8(acc0, v0); + acc1 = isMax ? _mm256_max_epu8(acc1, v1) : _mm256_min_epu8(acc1, v1); + acc2 = isMax ? _mm256_max_epu8(acc2, v2) : _mm256_min_epu8(acc2, v2); + } + + __m256i accs[3] = {acc0, acc1, acc2}; + for (int c = 0; c < 3; c++) + { + uchar lanes[32]; + _mm256_storeu_si256((__m256i*)lanes, accs[c]); + uchar result = initial; + for (int i = 0; i < 32; i++) + result = reduceScalarMinMax(result, lanes[i]); + for (int i = x; i < cols; i++) + result = reduceScalarMinMax(result, src[i * 3 + c]); + dst[c] = result; + } + } + }); + v_cleanup(); +} + +template +static void minMax8uC4(const Mat& srcmat, Mat& dstmat) +{ + const int cols = srcmat.cols; + const uchar initial = isMax ? 0 : UCHAR_MAX; + const __m256i accInit = _mm256_set1_epi8((char)initial); + const __m256i maskInvalid = _mm256_broadcastsi128_si256( + _mm_setr_epi8(0, 0, 0, 0, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1)); + const __m256i mask0 = _mm256_broadcastsi128_si256( + _mm_setr_epi8(0, 4, 8, 12, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1)); + const __m256i mask1 = _mm256_broadcastsi128_si256( + _mm_setr_epi8(1, 5, 9, 13, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1)); + const __m256i mask2 = _mm256_broadcastsi128_si256( + _mm_setr_epi8(2, 6, 10, 14, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1)); + const __m256i mask3 = _mm256_broadcastsi128_si256( + _mm_setr_epi8(3, 7, 11, 15, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1)); + + parallel_for_(Range(0, srcmat.rows), [&](const Range& range) { + for (int y = range.start; y < range.end; y++) + { + const uchar* src = srcmat.ptr(y); + uchar* dst = dstmat.ptr(y); + __m256i acc0 = accInit, acc1 = accInit, acc2 = accInit, acc3 = accInit; + int x = 0; + for (; x <= cols - 8; x += 8) + { + __m256i v = _mm256_loadu_si256((const __m256i*)(src + x * 4)); + __m256i v0 = _mm256_shuffle_epi8(v, mask0); + __m256i v1 = _mm256_shuffle_epi8(v, mask1); + __m256i v2 = _mm256_shuffle_epi8(v, mask2); + __m256i v3 = _mm256_shuffle_epi8(v, mask3); + if (!isMax) + { + v0 = _mm256_or_si256(v0, maskInvalid); + v1 = _mm256_or_si256(v1, maskInvalid); + v2 = _mm256_or_si256(v2, maskInvalid); + v3 = _mm256_or_si256(v3, maskInvalid); + } + acc0 = isMax ? _mm256_max_epu8(acc0, v0) : _mm256_min_epu8(acc0, v0); + acc1 = isMax ? _mm256_max_epu8(acc1, v1) : _mm256_min_epu8(acc1, v1); + acc2 = isMax ? _mm256_max_epu8(acc2, v2) : _mm256_min_epu8(acc2, v2); + acc3 = isMax ? _mm256_max_epu8(acc3, v3) : _mm256_min_epu8(acc3, v3); + } + + __m256i accs[4] = {acc0, acc1, acc2, acc3}; + for (int c = 0; c < 4; c++) + { + uchar lanes[32]; + _mm256_storeu_si256((__m256i*)lanes, accs[c]); + uchar result = initial; + for (int i = 0; i < 32; i++) + result = reduceScalarMinMax(result, lanes[i]); + for (int i = x; i < cols; i++) + result = reduceScalarMinMax(result, src[i * 4 + c]); + dst[c] = result; + } + } + }); + v_cleanup(); +} + +template +static void minMax32fC1(const Mat& srcmat, Mat& dstmat) +{ + const int cols = srcmat.cols; + parallel_for_(Range(0, srcmat.rows), [&](const Range& range) { + for (int y = range.start; y < range.end; y++) + { + const float* src = srcmat.ptr(y); + float result = src[0]; + int x = 0; + __m256 acc = _mm256_set1_ps(result); + for (; x <= cols - 8; x += 8) + { + __m256 v = _mm256_loadu_ps(src + x); + acc = isMax ? _mm256_max_ps(acc, v) : _mm256_min_ps(acc, v); + } + float lanes[8]; + _mm256_storeu_ps(lanes, acc); + for (int i = 0; i < 8; i++) + result = reduceScalarMinMax(result, lanes[i]); + for (; x < cols; x++) + result = reduceScalarMinMax(result, src[x]); + dstmat.ptr(y)[0] = result; + } + }); + v_cleanup(); +} + +template +static void minMax32fC4(const Mat& srcmat, Mat& dstmat) +{ + const int cols = srcmat.cols; + const float initial = isMax ? std::numeric_limits::lowest() : std::numeric_limits::max(); + const __m256 accInit = _mm256_set1_ps(initial); + const __m256 validMask = _mm256_castsi256_ps(_mm256_setr_epi32(-1, -1, 0, 0, 0, 0, 0, 0)); + const __m256i idx0 = _mm256_setr_epi32(0, 4, 0, 0, 0, 0, 0, 0); + const __m256i idx1 = _mm256_setr_epi32(1, 5, 0, 0, 0, 0, 0, 0); + const __m256i idx2 = _mm256_setr_epi32(2, 6, 0, 0, 0, 0, 0, 0); + const __m256i idx3 = _mm256_setr_epi32(3, 7, 0, 0, 0, 0, 0, 0); + + parallel_for_(Range(0, srcmat.rows), [&](const Range& range) { + for (int y = range.start; y < range.end; y++) + { + const float* src = srcmat.ptr(y); + float* dst = dstmat.ptr(y); + __m256 acc0 = accInit, acc1 = accInit, acc2 = accInit, acc3 = accInit; + int x = 0; + for (; x <= cols - 2; x += 2) + { + __m256 v = _mm256_loadu_ps(src + x * 4); + __m256 v0 = _mm256_blendv_ps(accInit, _mm256_permutevar8x32_ps(v, idx0), validMask); + __m256 v1 = _mm256_blendv_ps(accInit, _mm256_permutevar8x32_ps(v, idx1), validMask); + __m256 v2 = _mm256_blendv_ps(accInit, _mm256_permutevar8x32_ps(v, idx2), validMask); + __m256 v3 = _mm256_blendv_ps(accInit, _mm256_permutevar8x32_ps(v, idx3), validMask); + acc0 = isMax ? _mm256_max_ps(acc0, v0) : _mm256_min_ps(acc0, v0); + acc1 = isMax ? _mm256_max_ps(acc1, v1) : _mm256_min_ps(acc1, v1); + acc2 = isMax ? _mm256_max_ps(acc2, v2) : _mm256_min_ps(acc2, v2); + acc3 = isMax ? _mm256_max_ps(acc3, v3) : _mm256_min_ps(acc3, v3); + } + + __m256 accs[4] = {acc0, acc1, acc2, acc3}; + for (int c = 0; c < 4; c++) + { + float lanes[8]; + _mm256_storeu_ps(lanes, accs[c]); + float result = initial; + for (int i = 0; i < 8; i++) + result = reduceScalarMinMax(result, lanes[i]); + for (int i = x; i < cols; i++) + result = reduceScalarMinMax(result, src[i * 4 + c]); + dst[c] = result; + } + } + }); + v_cleanup(); +} + +template +static void minMax64fC1(const Mat& srcmat, Mat& dstmat) +{ + const int cols = srcmat.cols; + const double initial = isMax ? std::numeric_limits::lowest() + : std::numeric_limits::max(); + parallel_for_(Range(0, srcmat.rows), [&](const Range& range) { + for (int y = range.start; y < range.end; y++) + { + const double* src = srcmat.ptr(y); + double result = initial; + int x = 0; + __m256d acc0 = _mm256_set1_pd(initial); + __m256d acc1 = acc0, acc2 = acc0, acc3 = acc0; + for (; x <= cols - 16; x += 16) + { + __m256d v0 = _mm256_loadu_pd(src + x); + __m256d v1 = _mm256_loadu_pd(src + x + 4); + __m256d v2 = _mm256_loadu_pd(src + x + 8); + __m256d v3 = _mm256_loadu_pd(src + x + 12); + acc0 = isMax ? _mm256_max_pd(acc0, v0) : _mm256_min_pd(acc0, v0); + acc1 = isMax ? _mm256_max_pd(acc1, v1) : _mm256_min_pd(acc1, v1); + acc2 = isMax ? _mm256_max_pd(acc2, v2) : _mm256_min_pd(acc2, v2); + acc3 = isMax ? _mm256_max_pd(acc3, v3) : _mm256_min_pd(acc3, v3); + } + acc0 = isMax ? _mm256_max_pd(acc0, acc1) : _mm256_min_pd(acc0, acc1); + acc2 = isMax ? _mm256_max_pd(acc2, acc3) : _mm256_min_pd(acc2, acc3); + acc0 = isMax ? _mm256_max_pd(acc0, acc2) : _mm256_min_pd(acc0, acc2); + double lanes[4]; + _mm256_storeu_pd(lanes, acc0); + for (int i = 0; i < 4; i++) + result = isMax ? std::max(result, lanes[i]) : std::min(result, lanes[i]); + for (; x < cols; x++) + result = isMax ? std::max(result, src[x]) : std::min(result, src[x]); + dstmat.ptr(y)[0] = result; + } + }); + v_cleanup(); +} + +template +static void sum2_8uC1(const Mat& srcmat, Mat& dstmat) +{ + const int cols = srcmat.cols; + + parallel_for_(Range(0, srcmat.rows), [&](const Range& range) { + for (int y = range.start; y < range.end; y++) + { + const uchar* src = srcmat.ptr(y); + DT* dst = dstmat.ptr
(y); + uint32_t result = 0; + int x = 0; + __m256i acc = _mm256_setzero_si256(); + for (; x <= cols - 32; x += 32) + { + __m256i bytes = _mm256_loadu_si256((const __m256i*)(src + x)); + __m128i lo = _mm256_castsi256_si128(bytes); + __m128i hi = _mm256_extracti128_si256(bytes, 1); + __m256i lo16 = _mm256_cvtepu8_epi16(lo); + __m256i hi16 = _mm256_cvtepu8_epi16(hi); + acc = _mm256_add_epi32(acc, _mm256_madd_epi16(lo16, lo16)); + acc = _mm256_add_epi32(acc, _mm256_madd_epi16(hi16, hi16)); + } + uint32_t lanes[8]; + _mm256_storeu_si256((__m256i*)lanes, acc); + for (int i = 0; i < 8; i++) + result += lanes[i]; + for (; x < cols; x++) + result += (uint32_t)src[x] * src[x]; + dst[0] = (DT)(int32_t)result; + } + }); + v_cleanup(); +} + +static void sum2_16u32fC1(const Mat& srcmat, Mat& dstmat) +{ + const int cols = srcmat.cols; + parallel_for_(Range(0, srcmat.rows), [&](const Range& range) { + for (int y = range.start; y < range.end; y++) + { + const ushort* src = srcmat.ptr(y); + __m256 acc0 = _mm256_setzero_ps(), acc1 = acc0; + int x = 0; + for (; x <= cols - 16; x += 16) + { + __m256i v = _mm256_loadu_si256((const __m256i*)(src + x)); + __m256 v0 = _mm256_cvtepi32_ps(_mm256_cvtepu16_epi32(_mm256_castsi256_si128(v))); + __m256 v1 = _mm256_cvtepi32_ps(_mm256_cvtepu16_epi32(_mm256_extracti128_si256(v, 1))); + acc0 = _mm256_add_ps(acc0, _mm256_mul_ps(v0, v0)); + acc1 = _mm256_add_ps(acc1, _mm256_mul_ps(v1, v1)); + } + acc0 = _mm256_add_ps(acc0, acc1); + float lanes[8]; + _mm256_storeu_ps(lanes, acc0); + float result = 0; + for (int i = 0; i < 8; i++) + result += lanes[i]; + for (; x < cols; x++) + { + float value = (float)src[x]; + result += value * value; + } + dstmat.ptr(y)[0] = result; + } + }); + v_cleanup(); +} + +static void sum2_16s32fC1(const Mat& srcmat, Mat& dstmat) +{ + const int cols = srcmat.cols; + parallel_for_(Range(0, srcmat.rows), [&](const Range& range) { + for (int y = range.start; y < range.end; y++) + { + const short* src = srcmat.ptr(y); + __m256 acc0 = _mm256_setzero_ps(), acc1 = acc0; + int x = 0; + for (; x <= cols - 16; x += 16) + { + __m256i v = _mm256_loadu_si256((const __m256i*)(src + x)); + __m256 v0 = _mm256_cvtepi32_ps(_mm256_cvtepi16_epi32(_mm256_castsi256_si128(v))); + __m256 v1 = _mm256_cvtepi32_ps(_mm256_cvtepi16_epi32(_mm256_extracti128_si256(v, 1))); + acc0 = _mm256_add_ps(acc0, _mm256_mul_ps(v0, v0)); + acc1 = _mm256_add_ps(acc1, _mm256_mul_ps(v1, v1)); + } + acc0 = _mm256_add_ps(acc0, acc1); + float lanes[8]; + _mm256_storeu_ps(lanes, acc0); + float result = 0; + for (int i = 0; i < 8; i++) + result += lanes[i]; + for (; x < cols; x++) + { + float value = (float)src[x]; + result += value * value; + } + dstmat.ptr(y)[0] = result; + } + }); + v_cleanup(); +} + +static void sum2_16u64fC1(const Mat& srcmat, Mat& dstmat) +{ + const int cols = srcmat.cols; + parallel_for_(Range(0, srcmat.rows), [&](const Range& range) { + for (int y = range.start; y < range.end; y++) + { + const ushort* src = srcmat.ptr(y); + __m256d acc0 = _mm256_setzero_pd(), acc1 = acc0, acc2 = acc0, acc3 = acc0; + int x = 0; + for (; x <= cols - 16; x += 16) + { + __m256i v = _mm256_loadu_si256((const __m256i*)(src + x)); + __m256i v0 = _mm256_cvtepu16_epi32(_mm256_castsi256_si128(v)); + __m256i v1 = _mm256_cvtepu16_epi32(_mm256_extracti128_si256(v, 1)); + __m256d d0 = _mm256_cvtepi32_pd(_mm256_castsi256_si128(v0)); + __m256d d1 = _mm256_cvtepi32_pd(_mm256_extracti128_si256(v0, 1)); + __m256d d2 = _mm256_cvtepi32_pd(_mm256_castsi256_si128(v1)); + __m256d d3 = _mm256_cvtepi32_pd(_mm256_extracti128_si256(v1, 1)); + acc0 = _mm256_add_pd(acc0, _mm256_mul_pd(d0, d0)); + acc1 = _mm256_add_pd(acc1, _mm256_mul_pd(d1, d1)); + acc2 = _mm256_add_pd(acc2, _mm256_mul_pd(d2, d2)); + acc3 = _mm256_add_pd(acc3, _mm256_mul_pd(d3, d3)); + } + acc0 = _mm256_add_pd(_mm256_add_pd(acc0, acc1), _mm256_add_pd(acc2, acc3)); + double lanes[4]; + _mm256_storeu_pd(lanes, acc0); + double result = lanes[0] + lanes[1] + lanes[2] + lanes[3]; + for (; x < cols; x++) + { + double value = (double)src[x]; + result += value * value; + } + dstmat.ptr(y)[0] = result; + } + }); + v_cleanup(); +} + +static void sum2_16s64fC1(const Mat& srcmat, Mat& dstmat) +{ + const int cols = srcmat.cols; + parallel_for_(Range(0, srcmat.rows), [&](const Range& range) { + for (int y = range.start; y < range.end; y++) + { + const short* src = srcmat.ptr(y); + __m256d acc0 = _mm256_setzero_pd(), acc1 = acc0, acc2 = acc0, acc3 = acc0; + int x = 0; + for (; x <= cols - 16; x += 16) + { + __m256i v = _mm256_loadu_si256((const __m256i*)(src + x)); + __m256i v0 = _mm256_cvtepi16_epi32(_mm256_castsi256_si128(v)); + __m256i v1 = _mm256_cvtepi16_epi32(_mm256_extracti128_si256(v, 1)); + __m256d d0 = _mm256_cvtepi32_pd(_mm256_castsi256_si128(v0)); + __m256d d1 = _mm256_cvtepi32_pd(_mm256_extracti128_si256(v0, 1)); + __m256d d2 = _mm256_cvtepi32_pd(_mm256_castsi256_si128(v1)); + __m256d d3 = _mm256_cvtepi32_pd(_mm256_extracti128_si256(v1, 1)); + acc0 = _mm256_add_pd(acc0, _mm256_mul_pd(d0, d0)); + acc1 = _mm256_add_pd(acc1, _mm256_mul_pd(d1, d1)); + acc2 = _mm256_add_pd(acc2, _mm256_mul_pd(d2, d2)); + acc3 = _mm256_add_pd(acc3, _mm256_mul_pd(d3, d3)); + } + acc0 = _mm256_add_pd(_mm256_add_pd(acc0, acc1), _mm256_add_pd(acc2, acc3)); + double lanes[4]; + _mm256_storeu_pd(lanes, acc0); + double result = lanes[0] + lanes[1] + lanes[2] + lanes[3]; + for (; x < cols; x++) + { + double value = (double)src[x]; + result += value * value; + } + dstmat.ptr(y)[0] = result; + } + }); + v_cleanup(); +} + +static void sum2_32f64fC1(const Mat& srcmat, Mat& dstmat) +{ + const int cols = srcmat.cols; + parallel_for_(Range(0, srcmat.rows), [&](const Range& range) { + for (int y = range.start; y < range.end; y++) + { + const float* src = srcmat.ptr(y); + __m256d acc0 = _mm256_setzero_pd(), acc1 = acc0; + int x = 0; + for (; x <= cols - 8; x += 8) + { + __m256 v = _mm256_loadu_ps(src + x); + __m256d v0 = _mm256_cvtps_pd(_mm256_castps256_ps128(v)); + __m256d v1 = _mm256_cvtps_pd(_mm256_extractf128_ps(v, 1)); + acc0 = _mm256_add_pd(acc0, _mm256_mul_pd(v0, v0)); + acc1 = _mm256_add_pd(acc1, _mm256_mul_pd(v1, v1)); + } + acc0 = _mm256_add_pd(acc0, acc1); + double lanes[4]; + _mm256_storeu_pd(lanes, acc0); + double result = lanes[0] + lanes[1] + lanes[2] + lanes[3]; + for (; x < cols; x++) + { + double value = (double)src[x]; + result += value * value; + } + dstmat.ptr(y)[0] = result; + } + }); + v_cleanup(); +} + +static void sum2_64f64fC1(const Mat& srcmat, Mat& dstmat) +{ + const int cols = srcmat.cols; + parallel_for_(Range(0, srcmat.rows), [&](const Range& range) { + for (int y = range.start; y < range.end; y++) + { + const double* src = srcmat.ptr(y); + __m256d acc0 = _mm256_setzero_pd(), acc1 = acc0, acc2 = acc0, acc3 = acc0; + int x = 0; + for (; x <= cols - 16; x += 16) + { + __m256d v0 = _mm256_loadu_pd(src + x); + __m256d v1 = _mm256_loadu_pd(src + x + 4); + __m256d v2 = _mm256_loadu_pd(src + x + 8); + __m256d v3 = _mm256_loadu_pd(src + x + 12); + acc0 = _mm256_add_pd(acc0, _mm256_mul_pd(v0, v0)); + acc1 = _mm256_add_pd(acc1, _mm256_mul_pd(v1, v1)); + acc2 = _mm256_add_pd(acc2, _mm256_mul_pd(v2, v2)); + acc3 = _mm256_add_pd(acc3, _mm256_mul_pd(v3, v3)); + } + acc0 = _mm256_add_pd(_mm256_add_pd(acc0, acc1), _mm256_add_pd(acc2, acc3)); + double lanes[4]; + _mm256_storeu_pd(lanes, acc0); + double result = lanes[0] + lanes[1] + lanes[2] + lanes[3]; + for (; x < cols; x++) + result += src[x] * src[x]; + dstmat.ptr(y)[0] = result; + } + }); + v_cleanup(); +} + +template +static void sum2_8uC3(const Mat& srcmat, Mat& dstmat) +{ + const int cols = srcmat.cols; + const __m256i mask0 = _mm256_setr_epi8( + 0, 3, 6, 9, 12, 15, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + 2, 5, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1); + const __m256i mask1 = _mm256_setr_epi8( + 1, 4, 7, 10, 13, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + 0, 3, 6, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1); + const __m256i mask2 = _mm256_setr_epi8( + 2, 5, 8, 11, 14, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + 1, 4, 7, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1); + + parallel_for_(Range(0, srcmat.rows), [&](const Range& range) { + for (int y = range.start; y < range.end; y++) + { + const uchar* src = srcmat.ptr(y); + DT* dst = dstmat.ptr
(y); + __m256i acc0 = _mm256_setzero_si256(); + __m256i acc1 = _mm256_setzero_si256(); + __m256i acc2 = _mm256_setzero_si256(); + int x = 0; + for (; x <= cols - 11; x += 8) + { + __m256i v = _mm256_loadu_si256((const __m256i*)(src + x * 3)); + __m256i channels[3] = { + _mm256_shuffle_epi8(v, mask0), + _mm256_shuffle_epi8(v, mask1), + _mm256_shuffle_epi8(v, mask2) + }; + __m256i* accs[3] = {&acc0, &acc1, &acc2}; + for (int c = 0; c < 3; c++) + { + __m128i lo = _mm256_castsi256_si128(channels[c]); + __m128i hi = _mm256_extracti128_si256(channels[c], 1); + __m256i lo16 = _mm256_cvtepu8_epi16(lo); + __m256i hi16 = _mm256_cvtepu8_epi16(hi); + *accs[c] = _mm256_add_epi32(*accs[c], _mm256_madd_epi16(lo16, lo16)); + *accs[c] = _mm256_add_epi32(*accs[c], _mm256_madd_epi16(hi16, hi16)); + } + } + + __m256i accs[3] = {acc0, acc1, acc2}; + for (int c = 0; c < 3; c++) + { + uint32_t lanes[8]; + uint32_t result = 0; + _mm256_storeu_si256((__m256i*)lanes, accs[c]); + for (int i = 0; i < 8; i++) + result += lanes[i]; + for (int i = x; i < cols; i++) + { + uint32_t value = src[i * 3 + c]; + result += value * value; + } + dst[c] = (DT)(int32_t)result; + } + } + }); + v_cleanup(); +} + +template +static void sum2_8uC4(const Mat& srcmat, Mat& dstmat) +{ + const int cols = srcmat.cols; + const __m256i mask0 = _mm256_broadcastsi128_si256( + _mm_setr_epi8(0, 4, 8, 12, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1)); + const __m256i mask1 = _mm256_broadcastsi128_si256( + _mm_setr_epi8(1, 5, 9, 13, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1)); + const __m256i mask2 = _mm256_broadcastsi128_si256( + _mm_setr_epi8(2, 6, 10, 14, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1)); + const __m256i mask3 = _mm256_broadcastsi128_si256( + _mm_setr_epi8(3, 7, 11, 15, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1)); + + parallel_for_(Range(0, srcmat.rows), [&](const Range& range) { + for (int y = range.start; y < range.end; y++) + { + const uchar* src = srcmat.ptr(y); + DT* dst = dstmat.ptr
(y); + __m256i acc0 = _mm256_setzero_si256(); + __m256i acc1 = _mm256_setzero_si256(); + __m256i acc2 = _mm256_setzero_si256(); + __m256i acc3 = _mm256_setzero_si256(); + int x = 0; + for (; x <= cols - 8; x += 8) + { + __m256i v = _mm256_loadu_si256((const __m256i*)(src + x * 4)); + __m256i channels[4] = { + _mm256_shuffle_epi8(v, mask0), + _mm256_shuffle_epi8(v, mask1), + _mm256_shuffle_epi8(v, mask2), + _mm256_shuffle_epi8(v, mask3) + }; + __m256i* accs[4] = {&acc0, &acc1, &acc2, &acc3}; + for (int c = 0; c < 4; c++) + { + __m128i lo = _mm256_castsi256_si128(channels[c]); + __m128i hi = _mm256_extracti128_si256(channels[c], 1); + __m256i lo16 = _mm256_cvtepu8_epi16(lo); + __m256i hi16 = _mm256_cvtepu8_epi16(hi); + *accs[c] = _mm256_add_epi32(*accs[c], _mm256_madd_epi16(lo16, lo16)); + *accs[c] = _mm256_add_epi32(*accs[c], _mm256_madd_epi16(hi16, hi16)); + } + } + + __m256i accs[4] = {acc0, acc1, acc2, acc3}; + for (int c = 0; c < 4; c++) + { + uint32_t lanes[8]; + uint32_t result = 0; + _mm256_storeu_si256((__m256i*)lanes, accs[c]); + for (int i = 0; i < 8; i++) + result += lanes[i]; + for (int i = x; i < cols; i++) + { + uint32_t value = src[i * 4 + c]; + result += value * value; + } + dst[c] = (DT)(int32_t)result; + } + } + }); + v_cleanup(); +} + +static void sum2_32fC1(const Mat& srcmat, Mat& dstmat) +{ + const int cols = srcmat.cols; + parallel_for_(Range(0, srcmat.rows), [&](const Range& range) { + for (int y = range.start; y < range.end; y++) + { + const float* src = srcmat.ptr(y); + float result = 0; + int x = 0; + __m256 acc = _mm256_setzero_ps(); + for (; x <= cols - 8; x += 8) + { + __m256 v = _mm256_loadu_ps(src + x); + acc = _mm256_add_ps(acc, _mm256_mul_ps(v, v)); + } + float lanes[8]; + _mm256_storeu_ps(lanes, acc); + for (int i = 0; i < 8; i++) + result += lanes[i]; + for (; x < cols; x++) + result += src[x] * src[x]; + dstmat.ptr(y)[0] = result; + } + }); + v_cleanup(); +} + +static void sum2_32fC4(const Mat& srcmat, Mat& dstmat) +{ + const int cols = srcmat.cols; + const __m256 zero = _mm256_setzero_ps(); + const __m256 validMask = _mm256_castsi256_ps(_mm256_setr_epi32(-1, -1, 0, 0, 0, 0, 0, 0)); + const __m256i idx0 = _mm256_setr_epi32(0, 4, 0, 0, 0, 0, 0, 0); + const __m256i idx1 = _mm256_setr_epi32(1, 5, 0, 0, 0, 0, 0, 0); + const __m256i idx2 = _mm256_setr_epi32(2, 6, 0, 0, 0, 0, 0, 0); + const __m256i idx3 = _mm256_setr_epi32(3, 7, 0, 0, 0, 0, 0, 0); + + parallel_for_(Range(0, srcmat.rows), [&](const Range& range) { + for (int y = range.start; y < range.end; y++) + { + const float* src = srcmat.ptr(y); + float* dst = dstmat.ptr(y); + __m256 acc0 = zero, acc1 = zero, acc2 = zero, acc3 = zero; + int x = 0; + for (; x <= cols - 2; x += 2) + { + __m256 v = _mm256_loadu_ps(src + x * 4); + __m256 v0 = _mm256_blendv_ps(zero, _mm256_permutevar8x32_ps(v, idx0), validMask); + __m256 v1 = _mm256_blendv_ps(zero, _mm256_permutevar8x32_ps(v, idx1), validMask); + __m256 v2 = _mm256_blendv_ps(zero, _mm256_permutevar8x32_ps(v, idx2), validMask); + __m256 v3 = _mm256_blendv_ps(zero, _mm256_permutevar8x32_ps(v, idx3), validMask); + acc0 = _mm256_add_ps(acc0, _mm256_mul_ps(v0, v0)); + acc1 = _mm256_add_ps(acc1, _mm256_mul_ps(v1, v1)); + acc2 = _mm256_add_ps(acc2, _mm256_mul_ps(v2, v2)); + acc3 = _mm256_add_ps(acc3, _mm256_mul_ps(v3, v3)); + } + + __m256 accs[4] = {acc0, acc1, acc2, acc3}; + for (int c = 0; c < 4; c++) + { + float lanes[8]; + float result = 0; + _mm256_storeu_ps(lanes, accs[c]); + for (int i = 0; i < 8; i++) + result += lanes[i]; + for (int i = x; i < cols; i++) + result += src[i * 4 + c] * src[i * 4 + c]; + dst[c] = result; + } + } + }); + v_cleanup(); +} + +} // namespace reduce_c_avx2 diff --git a/modules/core/src/reduce_c_generic.hpp b/modules/core/src/reduce_c_generic.hpp new file mode 100644 index 0000000000..217009674b --- /dev/null +++ b/modules/core/src/reduce_c_generic.hpp @@ -0,0 +1,701 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html + +#if (CV_SIMD || CV_SIMD_SCALABLE) + +template +static inline VT vx_load_strided(const T *ptr, size_t step) +{ + constexpr int nlanes = VTraits::max_nlanes; + T buf[nlanes]; + for (int i = 0; i < VTraits::vlanes(); i++) + { + buf[i] = *ptr; + ptr += step; + } + return vx_load(buf); +} + +#if CV_RVV +template<> +inline v_uint8 vx_load_strided(const uchar *ptr, size_t step) +{ + return __riscv_vlse8_v_u8m2(ptr, step * sizeof(uchar), __riscv_vsetvlmax_e8m2()); +} +template<> +inline v_uint16 vx_load_strided(const ushort *ptr, size_t step) +{ + return __riscv_vlse16_v_u16m2(ptr, step * sizeof(ushort), __riscv_vsetvlmax_e16m2()); +} +template<> +inline v_int16 vx_load_strided(const short *ptr, size_t step) +{ + return __riscv_vlse16_v_i16m2(ptr, step * sizeof(short), __riscv_vsetvlmax_e16m2()); +} +template<> +inline v_int32 vx_load_strided(const int *ptr, size_t step) +{ + return __riscv_vlse32_v_i32m2(ptr, step * sizeof(int), __riscv_vsetvlmax_e32m2()); +} +template<> +inline v_float32 vx_load_strided(const float *ptr, size_t step) +{ + return __riscv_vlse32_v_f32m2(ptr, step * sizeof(float), __riscv_vsetvlmax_e32m2()); +} +template<> +inline v_float64 vx_load_strided(const double *ptr, size_t step) +{ + return __riscv_vlse64_v_f64m2(ptr, step * sizeof(double), __riscv_vsetvlmax_e64m2()); +} +#endif +#endif +template +struct ReduceOpAddSqr +{ + using v_stype = stype; + using v_itype = itype; + static const int vlanes; + static inline stype load(const stype *ptr, size_t step) { (void)step; return *ptr; } + static inline itype init() { return (itype)0; } + static inline itype reduce(const itype &val) { return val; } + inline itype operator()(const itype &a, const stype &b) const { return a + (itype)b * (itype)b; } +}; +template +const int ReduceOpAddSqr::vlanes = 1; + +#if (CV_SIMD || CV_SIMD_SCALABLE) + +template +struct ReduceVecOpAddSqr; + +template<> +struct ReduceVecOpAddSqr +{ + using stype = uchar; + using itype = int; + using v_stype = v_uint8; + using v_itype = v_int32; + static const int vlanes; + static inline v_stype load(const stype *ptr, size_t step) { return vx_load_strided(ptr, step); } + static inline v_itype init() { return vx_setzero_s32(); } + static inline itype reduce(const v_itype &val) { return v_reduce_sum(val); } + inline v_itype operator()(const v_itype &a, const v_stype &b) const + { + v_uint16 b0, b1; + v_mul_expand(b, b, b0, b1); + + v_uint32 s00, s01; + v_expand(b0, s00, s01); + s00 = v_add(s00, s01); + v_uint32 s10, s11; + v_expand(b1, s10, s11); + s10 = v_add(s10, s11); + return v_add(a, v_reinterpret_as_s32(v_add(s00, s10))); + } +}; +const int ReduceVecOpAddSqr::vlanes = VTraits::vlanes(); + +template<> +struct ReduceVecOpAddSqr +{ + using stype = ushort; + using itype = float; + using v_stype = v_uint16; + using v_itype = v_float32; + static const int vlanes; + static inline v_stype load(const stype *ptr, size_t step) { return vx_load_strided(ptr, step); } + static inline v_itype init() { return vx_setzero_f32(); } + static inline itype reduce(const v_itype &val) { return v_reduce_sum(val); } + inline v_itype operator()(const v_itype &a, const v_stype &b) const + { + v_uint32 b0, b1; + v_mul_expand(b, b, b0, b1); + v_int32 sb = v_reinterpret_as_s32(v_add(b0, b1)); + return v_add(a, v_cvt_f32(sb)); + } +}; +const int ReduceVecOpAddSqr::vlanes = VTraits::vlanes(); + +template<> +struct ReduceVecOpAddSqr +{ + using stype = short; + using itype = float; + using v_stype = v_int16; + using v_itype = v_float32; + static const int vlanes; + static inline v_stype load(const stype *ptr, size_t step) { return vx_load_strided(ptr, step); } + static inline v_itype init() { return vx_setzero_f32(); } + static inline itype reduce(const v_itype &val) { return v_reduce_sum(val); } + inline v_itype operator()(const v_itype &a, const v_stype &b) const + { + v_int32 b0, b1; + v_mul_expand(b, b, b0, b1); + v_int32 sb = v_add(b0, b1); + return v_add(a, v_cvt_f32(sb)); + } +}; +const int ReduceVecOpAddSqr::vlanes = VTraits::vlanes(); + +template<> +struct ReduceVecOpAddSqr +{ + using stype = float; + using itype = float; + using v_stype = v_float32; + using v_itype = v_float32; + static const int vlanes; + static inline v_stype load(const stype *ptr, size_t step) { return vx_load_strided(ptr, step); } + static inline v_itype init() { return vx_setzero_f32(); } + static inline itype reduce(const v_itype &val) { return v_reduce_sum(val); } + inline v_itype operator()(const v_itype &a, const v_stype &b) const { return v_add(a, v_mul(b, b)); } +}; +const int ReduceVecOpAddSqr::vlanes = VTraits::vlanes(); + +#endif + +#if (CV_SIMD_64F || CV_SIMD_SCALABLE_64F) + +template<> +struct ReduceVecOpAddSqr +{ + using stype = short; + using itype = double; + using v_stype = v_int16; + using v_itype = v_float64; + static const int vlanes; + static inline v_stype load(const stype *ptr, size_t step) { return vx_load_strided(ptr, step); } + static inline v_itype init() { return vx_setzero_f64(); } + static inline itype reduce(const v_itype &val) { return v_reduce_sum(val); } + inline v_itype operator()(const v_itype &a, const v_stype &b) const + { + v_int32 b0, b1; + v_mul_expand(b, b, b0, b1); + v_int32 sb = v_add(b0, b1); + return v_add(a, v_add(v_cvt_f64(sb), v_cvt_f64_high(sb))); + } +}; +const int ReduceVecOpAddSqr::vlanes = VTraits::vlanes(); + +template<> +struct ReduceVecOpAddSqr +{ + using stype = ushort; + using itype = double; + using v_stype = v_uint16; + using v_itype = v_float64; + static const int vlanes; + static inline v_stype load(const stype *ptr, size_t step) { return vx_load_strided(ptr, step); } + static inline v_itype init() { return vx_setzero_f64(); } + static inline itype reduce(const v_itype &val) { return v_reduce_sum(val); } + inline v_itype operator()(const v_itype &a, const v_stype &b) const + { + v_uint32 b0, b1; + v_mul_expand(b, b, b0, b1); + v_int32 sb = v_reinterpret_as_s32(v_add(b0, b1)); + return v_add(a, v_add(v_cvt_f64(sb), v_cvt_f64_high(sb))); + } +}; +const int ReduceVecOpAddSqr::vlanes = VTraits::vlanes(); + +template<> +struct ReduceVecOpAddSqr +{ + using stype = float; + using itype = double; + using v_stype = v_float32; + using v_itype = v_float64; + static const int vlanes; + static inline v_stype load(const stype *ptr, size_t step) { return vx_load_strided(ptr, step); } + static inline v_itype init() { return vx_setzero_f64(); } + static inline itype reduce(const v_itype &val) { return v_reduce_sum(val); } + inline v_itype operator()(const v_itype &a, const v_stype &b) const + { + v_itype b0 = v_cvt_f64(b), b1 = v_cvt_f64_high(b); + return v_add(a, v_add(v_mul(b0, b0), v_mul(b1, b1))); + } +}; +const int ReduceVecOpAddSqr::vlanes = VTraits::vlanes(); + +template<> +struct ReduceVecOpAddSqr +{ + using stype = double; + using itype = double; + using v_stype = v_float64; + using v_itype = v_float64; + static const int vlanes; + static inline v_stype load(const stype *ptr, size_t step) { return vx_load_strided(ptr, step); } + static inline v_itype init() { return vx_setzero_f64(); } + static inline itype reduce(const v_itype &val) { return v_reduce_sum(val); } + inline v_itype operator()(const v_itype &a, const v_stype &b) const { return v_add(a, v_mul(b, b)); } +}; +const int ReduceVecOpAddSqr::vlanes = VTraits::vlanes(); + +#endif + +template +struct ReduceOpMax +{ + using v_stype = stype; + using v_itype = stype; + static const int vlanes; + static inline stype load(const stype *ptr, size_t step) { (void)step; return *ptr; } + static inline stype init() { return std::numeric_limits::lowest(); } + static inline stype reduce(const stype &val) { return val; } + inline stype operator()(const stype &a, const stype &b) const { return std::max(a, b); } +}; +template +const int ReduceOpMax::vlanes = 1; + +#if (CV_SIMD || CV_SIMD_SCALABLE) + +template +struct ReduceVecOpMax; + +template<> +struct ReduceVecOpMax +{ + using stype = uchar; + using itype = uchar; + using v_stype = v_uint8; + using v_itype = v_uint8; + static const int vlanes; + static inline v_stype load(const stype *ptr, size_t step) { return vx_load_strided(ptr, step); } + static inline v_itype init() { return vx_setall_u8(std::numeric_limits::lowest()); } + static inline itype reduce(const v_itype &val) { return v_reduce_max(val); } + inline v_itype operator()(const v_itype &a, const v_stype &b) const { return v_max(a, b); } +}; +const int ReduceVecOpMax::vlanes = VTraits::vlanes(); + +template<> +struct ReduceVecOpMax +{ + using stype = ushort; + using itype = ushort; + using v_stype = v_uint16; + using v_itype = v_uint16; + static const int vlanes; + static inline v_stype load(const stype *ptr, size_t step) { return vx_load_strided(ptr, step); } + static inline v_itype init() { return vx_setall_u16(std::numeric_limits::lowest()); } + static inline itype reduce(const v_itype &val) { return v_reduce_max(val); } + inline v_itype operator()(const v_itype &a, const v_stype &b) const { return v_max(a, b); } +}; +const int ReduceVecOpMax::vlanes = VTraits::vlanes(); + +template<> +struct ReduceVecOpMax +{ + using stype = short; + using itype = short; + using v_stype = v_int16; + using v_itype = v_int16; + static const int vlanes; + static inline v_stype load(const stype *ptr, size_t step) { return vx_load_strided(ptr, step); } + static inline v_itype init() { return vx_setall_s16(std::numeric_limits::lowest()); } + static inline itype reduce(const v_itype &val) { return v_reduce_max(val); } + inline v_itype operator()(const v_itype &a, const v_stype &b) const { return v_max(a, b); } +}; +const int ReduceVecOpMax::vlanes = VTraits::vlanes(); + +template<> +struct ReduceVecOpMax +{ + using stype = float; + using itype = float; + using v_stype = v_float32; + using v_itype = v_float32; + static const int vlanes; + static inline v_stype load(const stype *ptr, size_t step) { return vx_load_strided(ptr, step); } + static inline v_itype init() { return vx_setall_f32(std::numeric_limits::lowest()); } + static inline itype reduce(const v_itype &val) { return v_reduce_max(val); } + inline v_itype operator()(const v_itype &a, const v_stype &b) const { return v_max(a, b); } +}; +const int ReduceVecOpMax::vlanes = VTraits::vlanes(); + +#endif + +#if (CV_SIMD_64F || CV_SIMD_SCALABLE_64F) + +template<> +struct ReduceVecOpMax +{ + using stype = double; + using itype = double; + using v_stype = v_float64; + using v_itype = v_float64; + static const int vlanes; + static inline v_stype load(const stype *ptr, size_t step) { return vx_load_strided(ptr, step); } + static inline v_itype init() { return vx_setall_f64(std::numeric_limits::lowest()); } + static inline itype reduce(const v_itype &val) + { + constexpr int nlanes = VTraits::max_nlanes; + itype buf[nlanes]; + vx_store(buf, val); + itype m = buf[0]; + for (int i = 1; i < VTraits::vlanes(); i++) + { + if (m < buf[i]) m = buf[i]; + } + return m; + } + inline v_itype operator()(const v_itype &a, const v_stype &b) const { return v_max(a, b); } +}; +const int ReduceVecOpMax::vlanes = VTraits::vlanes(); + +#endif + +template +struct ReduceOpMin +{ + using v_stype = stype; + using v_itype = stype; + static const int vlanes; + static inline stype load(const stype *ptr, size_t step) { (void)step; return *ptr; } + static inline stype init() { return std::numeric_limits::max(); } + static inline stype reduce(const stype &val) { return (stype)val; } + inline stype operator()(const stype &a, const stype &b) const { return std::min(a, b); } +}; +template +const int ReduceOpMin::vlanes = 1; + +#if (CV_SIMD || CV_SIMD_SCALABLE) + +template +struct ReduceVecOpMin; + +template<> +struct ReduceVecOpMin +{ + using stype = uchar; + using itype = uchar; + using v_stype = v_uint8; + using v_itype = v_uint8; + static const int vlanes; + static inline v_stype load(const stype *ptr, size_t step) { return vx_load_strided(ptr, step); } + static inline v_itype init() { return vx_setall_u8(std::numeric_limits::max()); } + static inline itype reduce(const v_itype &val) { return v_reduce_min(val); } + inline v_itype operator()(const v_itype &a, const v_stype &b) const { return v_min(a, b); } +}; +const int ReduceVecOpMin::vlanes = VTraits::vlanes(); + +template<> +struct ReduceVecOpMin +{ + using stype = ushort; + using itype = ushort; + using v_stype = v_uint16; + using v_itype = v_uint16; + static const int vlanes; + static inline v_stype load(const stype *ptr, size_t step) { return vx_load_strided(ptr, step); } + static inline v_itype init() { return vx_setall_u16(std::numeric_limits::max()); } + static inline itype reduce(const v_itype &val) { return v_reduce_min(val); } + inline v_itype operator()(const v_itype &a, const v_stype &b) const { return v_min(a, b); } +}; +const int ReduceVecOpMin::vlanes = VTraits::vlanes(); + +template<> +struct ReduceVecOpMin +{ + using stype = short; + using itype = short; + using v_stype = v_int16; + using v_itype = v_int16; + static const int vlanes; + static inline v_stype load(const stype *ptr, size_t step) { return vx_load_strided(ptr, step); } + static inline v_itype init() { return vx_setall_s16(std::numeric_limits::max()); } + static inline itype reduce(const v_itype &val) { return v_reduce_min(val); } + inline v_itype operator()(const v_itype &a, const v_stype &b) const { return v_min(a, b); } +}; +const int ReduceVecOpMin::vlanes = VTraits::vlanes(); + +template<> +struct ReduceVecOpMin +{ + using stype = float; + using itype = float; + using v_stype = v_float32; + using v_itype = v_float32; + static const int vlanes; + static inline v_stype load(const stype *ptr, size_t step) { return vx_load_strided(ptr, step); } + static inline v_itype init() { return vx_setall_f32(std::numeric_limits::max()); } + static inline itype reduce(const v_itype &val) { return v_reduce_min(val); } + inline v_itype operator()(const v_itype &a, const v_stype &b) const { return v_min(a, b); } +}; +const int ReduceVecOpMin::vlanes = VTraits::vlanes(); + +#endif + +#if (CV_SIMD_64F || CV_SIMD_SCALABLE_64F) + +template<> +struct ReduceVecOpMin +{ + using stype = double; + using itype = double; + using v_stype = v_float64; + using v_itype = v_float64; + static const int vlanes; + static inline v_stype load(const stype *ptr, size_t step) { return vx_load_strided(ptr, step); } + static inline v_itype init() { return vx_setall_f64(std::numeric_limits::max()); } + static inline itype reduce(const v_itype &val) + { + constexpr int nlanes = VTraits::max_nlanes; + itype buf[nlanes]; + vx_store(buf, val); + itype m = buf[0]; + for (int i = 1; i < VTraits::vlanes(); i++) + { + if (m > buf[i]) m = buf[i]; + } + return m; + } + inline v_itype operator()(const v_itype &a, const v_stype &b) const { return v_min(a, b); } +}; +const int ReduceVecOpMin::vlanes = VTraits::vlanes(); + +#endif + + +using ReduceOpAddSqr_8U32S = ReduceOpAddSqr; +using ReduceOpAddSqr_8U32F = ReduceOpAddSqr; +using ReduceOpAddSqr_8U64F = ReduceOpAddSqr; +using ReduceOpAddSqr_16U32F = ReduceOpAddSqr; +using ReduceOpAddSqr_16U64F = ReduceOpAddSqr; +using ReduceOpAddSqr_16S32F = ReduceOpAddSqr; +using ReduceOpAddSqr_16S64F = ReduceOpAddSqr; +using ReduceOpAddSqr_32F32F = ReduceOpAddSqr; +using ReduceOpAddSqr_32F64F = ReduceOpAddSqr; +using ReduceOpAddSqr_64F64F = ReduceOpAddSqr; + +using ReduceOpMax_8U = ReduceOpMax; +using ReduceOpMax_16U = ReduceOpMax; +using ReduceOpMax_16S = ReduceOpMax; +using ReduceOpMax_32F = ReduceOpMax; +using ReduceOpMax_64F = ReduceOpMax; + +using ReduceOpMin_8U = ReduceOpMin; +using ReduceOpMin_16U = ReduceOpMin; +using ReduceOpMin_16S = ReduceOpMin; +using ReduceOpMin_32F = ReduceOpMin; +using ReduceOpMin_64F = ReduceOpMin; + +#if (CV_SIMD || CV_SIMD_SCALABLE) + +using ReduceVecOpAddSqr_8U32S = ReduceVecOpAddSqr; +using ReduceVecOpAddSqr_8U32F = ReduceVecOpAddSqr; +using ReduceVecOpAddSqr_8U64F = ReduceVecOpAddSqr; +using ReduceVecOpAddSqr_16U32F = ReduceVecOpAddSqr; +using ReduceVecOpAddSqr_16S32F = ReduceVecOpAddSqr; +using ReduceVecOpAddSqr_32F32F = ReduceVecOpAddSqr; +using ReduceVecOpMax_8U = ReduceVecOpMax; +using ReduceVecOpMax_16U = ReduceVecOpMax; +using ReduceVecOpMax_16S = ReduceVecOpMax; +using ReduceVecOpMax_32F = ReduceVecOpMax; +using ReduceVecOpMin_8U = ReduceVecOpMin; +using ReduceVecOpMin_16U = ReduceVecOpMin; +using ReduceVecOpMin_16S = ReduceVecOpMin; +using ReduceVecOpMin_32F = ReduceVecOpMin; + +#else + +using ReduceVecOpAddSqr_8U32S = ReduceOpAddSqr; +using ReduceVecOpAddSqr_8U32F = ReduceOpAddSqr; +using ReduceVecOpAddSqr_8U64F = ReduceOpAddSqr; +using ReduceVecOpAddSqr_16U32F = ReduceOpAddSqr; +using ReduceVecOpAddSqr_16S32F = ReduceOpAddSqr; +using ReduceVecOpAddSqr_32F32F = ReduceOpAddSqr; +using ReduceVecOpMax_8U = ReduceOpMax; +using ReduceVecOpMax_16U = ReduceOpMax; +using ReduceVecOpMax_16S = ReduceOpMax; +using ReduceVecOpMax_32F = ReduceOpMax; +using ReduceVecOpMin_8U = ReduceOpMin; +using ReduceVecOpMin_16U = ReduceOpMin; +using ReduceVecOpMin_16S = ReduceOpMin; +using ReduceVecOpMin_32F = ReduceOpMin; + +#endif + +#if (CV_SIMD_64F || CV_SIMD_SCALABLE_64F) + +using ReduceVecOpAddSqr_16U64F = ReduceVecOpAddSqr; +using ReduceVecOpAddSqr_16S64F = ReduceVecOpAddSqr; +using ReduceVecOpAddSqr_32F64F = ReduceVecOpAddSqr; +using ReduceVecOpAddSqr_64F64F = ReduceVecOpAddSqr; +using ReduceVecOpMax_64F = ReduceVecOpMax; +using ReduceVecOpMin_64F = ReduceVecOpMin; + +#else + +using ReduceVecOpAddSqr_16U64F = ReduceOpAddSqr; +using ReduceVecOpAddSqr_16S64F = ReduceOpAddSqr; +using ReduceVecOpAddSqr_32F64F = ReduceOpAddSqr; +using ReduceVecOpAddSqr_64F64F = ReduceOpAddSqr; +using ReduceVecOpMax_64F = ReduceOpMax; +using ReduceVecOpMin_64F = ReduceOpMin; + +#endif + +template +class ReduceC_Invoker : public ParallelLoopBody +{ + using WT = typename Op::v_itype; + using VT = typename VecOp::v_itype; +public: + ReduceC_Invoker(const Mat& aSrcmat, Mat& aDstmat, Op& aOp, VecOp& aVop) + :srcmat(aSrcmat),dstmat(aDstmat),op(aOp),vop(aVop) + { + } + void operator()(const Range& range) const CV_OVERRIDE + { + int channels = srcmat.channels(); + int width = srcmat.cols; + + const int nlanes = VecOp::vlanes; + + for (int h = range.start; h < range.end; h++) + { + const T *srcrow = srcmat.ptr(h); + ST *dst = dstmat.ptr(h); + for (int cn = 0; cn < channels; cn++) + { + const T *src = srcrow + cn; + VT vbuf = vop.init(); + int w = 0; + for (; w <= width - nlanes; w += nlanes) + { + vbuf = vop(vbuf, vop.load(src+w*channels, channels)); + } + WT wbuf = vop.reduce(vbuf); + for (; w < width; w++) + { + wbuf = op(wbuf, op.load(src+w*channels, channels)); + } + dst[cn] = (ST)op.reduce(wbuf); + } + } + } +private: + const Mat& srcmat; + Mat& dstmat; + Op& op; + VecOp& vop; +}; + +template static void +reduceColGeneric(const Mat& srcmat, Mat& dstmat) +{ + Op op; + VecOp vop; + + ReduceC_Invoker body(srcmat, dstmat, op, vop); + parallel_for_(Range(0, srcmat.size().height), body); +} + +template +static inline uchar reduceScalarMinMax(uchar a, uchar b) +{ + return isMax ? std::max(a, b) : std::min(a, b); +} + +template +static void reduceColMinMax_8uFallback(const Mat& srcmat, Mat& dstmat) +{ + if (isMax) + reduceColGeneric(srcmat, dstmat); + else + reduceColGeneric(srcmat, dstmat); +} + +template +static void reduceColMinMax_16uFallback(const Mat& srcmat, Mat& dstmat) +{ + if (isMax) + reduceColGeneric(srcmat, dstmat); + else + reduceColGeneric(srcmat, dstmat); +} + +template +static void reduceColMinMax_16sFallback(const Mat& srcmat, Mat& dstmat) +{ + if (isMax) + reduceColGeneric(srcmat, dstmat); + else + reduceColGeneric(srcmat, dstmat); +} + +template +static inline float reduceScalarMinMax(float a, float b) +{ + return isMax ? std::max(a, b) : std::min(a, b); +} + +template +static void reduceColMinMax_32fFallback(const Mat& srcmat, Mat& dstmat) +{ + if (isMax) + reduceColGeneric(srcmat, dstmat); + else + reduceColGeneric(srcmat, dstmat); +} + +#if (CV_SIMD_64F || CV_SIMD_SCALABLE_64F) +template +static void reduceColMinMax_64fFallback(const Mat& srcmat, Mat& dstmat) +{ + if (isMax) + reduceColGeneric(srcmat, dstmat); + else + reduceColGeneric(srcmat, dstmat); +} +#endif + +template +static void reduceColSum2_8uFallback(const Mat& srcmat, Mat& dstmat) +{ + if (std::is_same::value) + reduceColGeneric(srcmat, dstmat); + else if (std::is_same::value) + reduceColGeneric(srcmat, dstmat); + else + reduceColGeneric(srcmat, dstmat); +} + +static void reduceColSum2_16u32fFallback(const Mat& srcmat, Mat& dstmat) +{ + reduceColGeneric(srcmat, dstmat); +} + +static void reduceColSum2_16s32fFallback(const Mat& srcmat, Mat& dstmat) +{ + reduceColGeneric(srcmat, dstmat); +} + +static void reduceColSum2_32f32fFallback(const Mat& srcmat, Mat& dstmat) +{ + reduceColGeneric(srcmat, dstmat); +} + +#if (CV_SIMD_64F || CV_SIMD_SCALABLE_64F) +static void reduceColSum2_16u64fFallback(const Mat& srcmat, Mat& dstmat) +{ + reduceColGeneric(srcmat, dstmat); +} + +static void reduceColSum2_16s64fFallback(const Mat& srcmat, Mat& dstmat) +{ + reduceColGeneric(srcmat, dstmat); +} + +static void reduceColSum2_32f64fFallback(const Mat& srcmat, Mat& dstmat) +{ + reduceColGeneric(srcmat, dstmat); +} + +static void reduceColSum2_64f64fFallback(const Mat& srcmat, Mat& dstmat) +{ + reduceColGeneric(srcmat, dstmat); +} +#endif diff --git a/modules/core/src/reduce_c_neon.hpp b/modules/core/src/reduce_c_neon.hpp new file mode 100644 index 0000000000..de6a99ea7c --- /dev/null +++ b/modules/core/src/reduce_c_neon.hpp @@ -0,0 +1,827 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html + +namespace reduce_c_neon +{ + +// Optimized ReduceC support in this backend: +// +// | Input -> output type/channel | SUM | AVG | MIN | MAX | SUM2 | +// |------------------------------------|:---:|:---:|:---:|:---:|:----:| +// | 8UC1/C3/C4 -> 8UC1/C3/C4 | - | - | x | x | - | +// | 8UC1/C3/C4 -> 32SC1/C3/C4 | x | x | - | - | x* | +// | 8UC1/C3/C4 -> 32FC1/C3/C4 | x | x | - | - | x* | +// | 8UC1/C3/C4 -> 64FC1/C3/C4 | - | - | - | - | x* | +// | 16UC1/C3/C4 -> 16UC1/C3/C4 | - | - | x | x | - | +// | 16UC1 -> 32FC1 | x | x | - | - | x* | +// | 16UC3/C4 -> 32FC3/C4 | x | x | - | - | - | +// | 16UC1 -> 64FC1 | - | - | - | - | x* | +// | 16UC3/C4 -> 64FC3/C4 | - | - | - | - | - | +// | 16SC1/C3/C4 -> 16SC1/C3/C4 | - | - | x | x | - | +// | 16SC1 -> 32FC1 | x | x | - | - | x* | +// | 16SC3/C4 -> 32FC3/C4 | x | x | - | - | - | +// | 16SC1 -> 64FC1 | - | - | - | - | x* | +// | 16SC3/C4 -> 64FC3/C4 | - | - | - | - | - | +// | 32FC1 -> 32FC1 | x | x | x | x | x | +// | 32FC3 -> 32FC3 | x | x | - | - | - | +// | 32FC4 -> 32FC4 | x | x | x | x | x | +// | 32FC1 -> 64FC1 | x* | x* | - | - | x* | +// | 32FC3/C4 -> 64FC3/C4 | x* | x* | - | - | - | +// | 64FC1 -> 64FC1 | x* | x* | x* | x* | x* | +// | 64FC3/C4 -> 64FC3/C4 | x* | x* | - | - | - | +// +// 'x' in SUM/AVG denotes the existing shared universal-intrinsics kernel; 'x' +// in MIN/MAX/SUM2 denotes a native NEON kernel. '*' requires AArch64. For legal +// MIN/MAX/SUM2 combinations marked '-', and for other channel counts, dispatch +// uses the shared generic fallback. + +template +static void minMax8uC1(const Mat& srcmat, Mat& dstmat) +{ + const int cols = srcmat.cols; + parallel_for_(Range(0, srcmat.rows), [&](const Range& range) { + for (int y = range.start; y < range.end; y++) + { + const uchar* src = srcmat.ptr(y); + uchar* dst = dstmat.ptr(y); + uchar result = isMax ? 0 : UCHAR_MAX; + int x = 0; + uint8x16_t acc = vdupq_n_u8(result); + for (; x <= cols - 16; x += 16) + { + uint8x16_t v = vld1q_u8(src + x); + acc = isMax ? vmaxq_u8(acc, v) : vminq_u8(acc, v); + } + uchar lanes[16]; + vst1q_u8(lanes, acc); + for (int i = 0; i < 16; i++) + result = reduceScalarMinMax(result, lanes[i]); + for (; x < cols; x++) + result = reduceScalarMinMax(result, src[x]); + dst[0] = result; + } + }); + v_cleanup(); +} + +template +static void minMax16uC1(const Mat& srcmat, Mat& dstmat) +{ + const int cols = srcmat.cols; + parallel_for_(Range(0, srcmat.rows), [&](const Range& range) { + for (int y = range.start; y < range.end; y++) + { + const ushort* src = srcmat.ptr(y); + ushort result = isMax ? 0 : USHRT_MAX; + int x = 0; + uint16x8_t acc = vdupq_n_u16(result); + for (; x <= cols - 8; x += 8) + { + uint16x8_t v = vld1q_u16(src + x); + acc = isMax ? vmaxq_u16(acc, v) : vminq_u16(acc, v); + } + ushort lanes[8]; + vst1q_u16(lanes, acc); + for (int i = 0; i < 8; i++) + result = isMax ? std::max(result, lanes[i]) : std::min(result, lanes[i]); + for (; x < cols; x++) + result = isMax ? std::max(result, src[x]) : std::min(result, src[x]); + dstmat.ptr(y)[0] = result; + } + }); + v_cleanup(); +} + +template +static void minMax16sC1(const Mat& srcmat, Mat& dstmat) +{ + const int cols = srcmat.cols; + parallel_for_(Range(0, srcmat.rows), [&](const Range& range) { + for (int y = range.start; y < range.end; y++) + { + const short* src = srcmat.ptr(y); + short result = isMax ? SHRT_MIN : SHRT_MAX; + int x = 0; + int16x8_t acc = vdupq_n_s16(result); + for (; x <= cols - 8; x += 8) + { + int16x8_t v = vld1q_s16(src + x); + acc = isMax ? vmaxq_s16(acc, v) : vminq_s16(acc, v); + } + short lanes[8]; + vst1q_s16(lanes, acc); + for (int i = 0; i < 8; i++) + result = isMax ? std::max(result, lanes[i]) : std::min(result, lanes[i]); + for (; x < cols; x++) + result = isMax ? std::max(result, src[x]) : std::min(result, src[x]); + dstmat.ptr(y)[0] = result; + } + }); + v_cleanup(); +} + +template +static void minMax16uC4(const Mat& srcmat, Mat& dstmat) +{ + const int cols = srcmat.cols; + const ushort initial = isMax ? 0 : USHRT_MAX; + parallel_for_(Range(0, srcmat.rows), [&](const Range& range) { + for (int y = range.start; y < range.end; y++) + { + const ushort* src = srcmat.ptr(y); + ushort* dst = dstmat.ptr(y); + uint16x8x4_t acc = {{ + vdupq_n_u16(initial), vdupq_n_u16(initial), + vdupq_n_u16(initial), vdupq_n_u16(initial) + }}; + int x = 0; + for (; x <= cols - 8; x += 8) + { + uint16x8x4_t v = vld4q_u16(src + x * 4); + for (int c = 0; c < 4; c++) + acc.val[c] = isMax ? vmaxq_u16(acc.val[c], v.val[c]) + : vminq_u16(acc.val[c], v.val[c]); + } + for (int c = 0; c < 4; c++) + { + ushort lanes[8]; + vst1q_u16(lanes, acc.val[c]); + ushort result = initial; + for (int i = 0; i < 8; i++) + result = isMax ? std::max(result, lanes[i]) : std::min(result, lanes[i]); + for (int i = x; i < cols; i++) + result = isMax ? std::max(result, src[i * 4 + c]) + : std::min(result, src[i * 4 + c]); + dst[c] = result; + } + } + }); + v_cleanup(); +} + +template +static void minMax16sC4(const Mat& srcmat, Mat& dstmat) +{ + const int cols = srcmat.cols; + const short initial = isMax ? SHRT_MIN : SHRT_MAX; + parallel_for_(Range(0, srcmat.rows), [&](const Range& range) { + for (int y = range.start; y < range.end; y++) + { + const short* src = srcmat.ptr(y); + short* dst = dstmat.ptr(y); + int16x8x4_t acc = {{ + vdupq_n_s16(initial), vdupq_n_s16(initial), + vdupq_n_s16(initial), vdupq_n_s16(initial) + }}; + int x = 0; + for (; x <= cols - 8; x += 8) + { + int16x8x4_t v = vld4q_s16(src + x * 4); + for (int c = 0; c < 4; c++) + acc.val[c] = isMax ? vmaxq_s16(acc.val[c], v.val[c]) + : vminq_s16(acc.val[c], v.val[c]); + } + for (int c = 0; c < 4; c++) + { + short lanes[8]; + vst1q_s16(lanes, acc.val[c]); + short result = initial; + for (int i = 0; i < 8; i++) + result = isMax ? std::max(result, lanes[i]) : std::min(result, lanes[i]); + for (int i = x; i < cols; i++) + result = isMax ? std::max(result, src[i * 4 + c]) + : std::min(result, src[i * 4 + c]); + dst[c] = result; + } + } + }); + v_cleanup(); +} + +template +static void minMax16uC3(const Mat& srcmat, Mat& dstmat) +{ + const int cols = srcmat.cols; + const ushort initial = isMax ? 0 : USHRT_MAX; + parallel_for_(Range(0, srcmat.rows), [&](const Range& range) { + for (int y = range.start; y < range.end; y++) + { + const ushort* src = srcmat.ptr(y); + ushort* dst = dstmat.ptr(y); + uint16x8x3_t acc = {{vdupq_n_u16(initial), vdupq_n_u16(initial), vdupq_n_u16(initial)}}; + int x = 0; + for (; x <= cols - 8; x += 8) + { + uint16x8x3_t v = vld3q_u16(src + x * 3); + for (int c = 0; c < 3; c++) + acc.val[c] = isMax ? vmaxq_u16(acc.val[c], v.val[c]) + : vminq_u16(acc.val[c], v.val[c]); + } + for (int c = 0; c < 3; c++) + { + ushort lanes[8]; + vst1q_u16(lanes, acc.val[c]); + ushort result = initial; + for (int i = 0; i < 8; i++) + result = isMax ? std::max(result, lanes[i]) : std::min(result, lanes[i]); + for (int i = x; i < cols; i++) + result = isMax ? std::max(result, src[i * 3 + c]) + : std::min(result, src[i * 3 + c]); + dst[c] = result; + } + } + }); + v_cleanup(); +} + +template +static void minMax16sC3(const Mat& srcmat, Mat& dstmat) +{ + const int cols = srcmat.cols; + const short initial = isMax ? SHRT_MIN : SHRT_MAX; + parallel_for_(Range(0, srcmat.rows), [&](const Range& range) { + for (int y = range.start; y < range.end; y++) + { + const short* src = srcmat.ptr(y); + short* dst = dstmat.ptr(y); + int16x8x3_t acc = {{vdupq_n_s16(initial), vdupq_n_s16(initial), vdupq_n_s16(initial)}}; + int x = 0; + for (; x <= cols - 8; x += 8) + { + int16x8x3_t v = vld3q_s16(src + x * 3); + for (int c = 0; c < 3; c++) + acc.val[c] = isMax ? vmaxq_s16(acc.val[c], v.val[c]) + : vminq_s16(acc.val[c], v.val[c]); + } + for (int c = 0; c < 3; c++) + { + short lanes[8]; + vst1q_s16(lanes, acc.val[c]); + short result = initial; + for (int i = 0; i < 8; i++) + result = isMax ? std::max(result, lanes[i]) : std::min(result, lanes[i]); + for (int i = x; i < cols; i++) + result = isMax ? std::max(result, src[i * 3 + c]) + : std::min(result, src[i * 3 + c]); + dst[c] = result; + } + } + }); + v_cleanup(); +} + +template +static void minMax8uC3(const Mat& srcmat, Mat& dstmat) +{ + const int cols = srcmat.cols; + parallel_for_(Range(0, srcmat.rows), [&](const Range& range) { + for (int y = range.start; y < range.end; y++) + { + const uchar* src = srcmat.ptr(y); + uchar* dst = dstmat.ptr(y); + const uchar initial = isMax ? 0 : UCHAR_MAX; + uint8x16x3_t acc = {{ + vdupq_n_u8(initial), vdupq_n_u8(initial), vdupq_n_u8(initial) + }}; + int x = 0; + for (; x <= cols - 16; x += 16) + { + uint8x16x3_t v = vld3q_u8(src + x * 3); + for (int c = 0; c < 3; c++) + acc.val[c] = isMax ? vmaxq_u8(acc.val[c], v.val[c]) + : vminq_u8(acc.val[c], v.val[c]); + } + for (int c = 0; c < 3; c++) + { + uchar lanes[16]; + vst1q_u8(lanes, acc.val[c]); + uchar result = initial; + for (int i = 0; i < 16; i++) + result = reduceScalarMinMax(result, lanes[i]); + for (int i = x; i < cols; i++) + result = reduceScalarMinMax(result, src[i * 3 + c]); + dst[c] = result; + } + } + }); + v_cleanup(); +} + +template +static void minMax8uC4(const Mat& srcmat, Mat& dstmat) +{ + const int cols = srcmat.cols; + parallel_for_(Range(0, srcmat.rows), [&](const Range& range) { + for (int y = range.start; y < range.end; y++) + { + const uchar* src = srcmat.ptr(y); + uchar* dst = dstmat.ptr(y); + const uchar initial = isMax ? 0 : UCHAR_MAX; + uint8x16x4_t acc = {{ + vdupq_n_u8(initial), vdupq_n_u8(initial), + vdupq_n_u8(initial), vdupq_n_u8(initial) + }}; + int x = 0; + for (; x <= cols - 16; x += 16) + { + uint8x16x4_t v = vld4q_u8(src + x * 4); + for (int c = 0; c < 4; c++) + acc.val[c] = isMax ? vmaxq_u8(acc.val[c], v.val[c]) + : vminq_u8(acc.val[c], v.val[c]); + } + for (int c = 0; c < 4; c++) + { + uchar lanes[16]; + vst1q_u8(lanes, acc.val[c]); + uchar result = initial; + for (int i = 0; i < 16; i++) + result = reduceScalarMinMax(result, lanes[i]); + for (int i = x; i < cols; i++) + result = reduceScalarMinMax(result, src[i * 4 + c]); + dst[c] = result; + } + } + }); + v_cleanup(); +} + +template +static void minMax32fC1(const Mat& srcmat, Mat& dstmat) +{ + const int cols = srcmat.cols; + parallel_for_(Range(0, srcmat.rows), [&](const Range& range) { + for (int y = range.start; y < range.end; y++) + { + const float* src = srcmat.ptr(y); + float result = src[0]; + int x = 0; + float32x4_t acc = vdupq_n_f32(result); + for (; x <= cols - 4; x += 4) + { + float32x4_t v = vld1q_f32(src + x); + acc = isMax ? vmaxq_f32(acc, v) : vminq_f32(acc, v); + } + float lanes[4]; + vst1q_f32(lanes, acc); + for (int i = 0; i < 4; i++) + result = reduceScalarMinMax(result, lanes[i]); + for (; x < cols; x++) + result = reduceScalarMinMax(result, src[x]); + dstmat.ptr(y)[0] = result; + } + }); + v_cleanup(); +} + +template +static void minMax32fC4(const Mat& srcmat, Mat& dstmat) +{ + const int cols = srcmat.cols; + const float initial = isMax ? std::numeric_limits::lowest() : std::numeric_limits::max(); + parallel_for_(Range(0, srcmat.rows), [&](const Range& range) { + for (int y = range.start; y < range.end; y++) + { + const float* src = srcmat.ptr(y); + float* dst = dstmat.ptr(y); + float32x4x4_t acc = {{ + vdupq_n_f32(initial), vdupq_n_f32(initial), + vdupq_n_f32(initial), vdupq_n_f32(initial) + }}; + int x = 0; + for (; x <= cols - 4; x += 4) + { + float32x4x4_t v = vld4q_f32(src + x * 4); + for (int c = 0; c < 4; c++) + acc.val[c] = isMax ? vmaxq_f32(acc.val[c], v.val[c]) + : vminq_f32(acc.val[c], v.val[c]); + } + for (int c = 0; c < 4; c++) + { + float lanes[4]; + vst1q_f32(lanes, acc.val[c]); + float result = initial; + for (int i = 0; i < 4; i++) + result = reduceScalarMinMax(result, lanes[i]); + for (int i = x; i < cols; i++) + result = reduceScalarMinMax(result, src[i * 4 + c]); + dst[c] = result; + } + } + }); + v_cleanup(); +} + +#if defined(__aarch64__) || defined(_M_ARM64) +template +static void minMax64fC1(const Mat& srcmat, Mat& dstmat) +{ + const int cols = srcmat.cols; + const double initial = isMax ? std::numeric_limits::lowest() + : std::numeric_limits::max(); + parallel_for_(Range(0, srcmat.rows), [&](const Range& range) { + for (int y = range.start; y < range.end; y++) + { + const double* src = srcmat.ptr(y); + double result = initial; + int x = 0; + float64x2_t acc0 = vdupq_n_f64(initial); + float64x2_t acc1 = acc0, acc2 = acc0, acc3 = acc0; + for (; x <= cols - 8; x += 8) + { + float64x2_t v0 = vld1q_f64(src + x); + float64x2_t v1 = vld1q_f64(src + x + 2); + float64x2_t v2 = vld1q_f64(src + x + 4); + float64x2_t v3 = vld1q_f64(src + x + 6); + acc0 = isMax ? vmaxq_f64(acc0, v0) : vminq_f64(acc0, v0); + acc1 = isMax ? vmaxq_f64(acc1, v1) : vminq_f64(acc1, v1); + acc2 = isMax ? vmaxq_f64(acc2, v2) : vminq_f64(acc2, v2); + acc3 = isMax ? vmaxq_f64(acc3, v3) : vminq_f64(acc3, v3); + } + acc0 = isMax ? vmaxq_f64(acc0, acc1) : vminq_f64(acc0, acc1); + acc2 = isMax ? vmaxq_f64(acc2, acc3) : vminq_f64(acc2, acc3); + acc0 = isMax ? vmaxq_f64(acc0, acc2) : vminq_f64(acc0, acc2); + double lanes[2]; + vst1q_f64(lanes, acc0); + for (int i = 0; i < 2; i++) + result = isMax ? std::max(result, lanes[i]) : std::min(result, lanes[i]); + for (; x < cols; x++) + result = isMax ? std::max(result, src[x]) : std::min(result, src[x]); + dstmat.ptr(y)[0] = result; + } + }); + v_cleanup(); +} +#endif + +#if defined(__aarch64__) || defined(_M_ARM64) +static inline uint32_t reduceSum2_8u_NEON(uint8x16_t v) +{ + uint16x8_t lo = vmull_u8(vget_low_u8(v), vget_low_u8(v)); + uint16x8_t hi = vmull_u8(vget_high_u8(v), vget_high_u8(v)); + return vaddvq_u32(vpaddlq_u16(lo)) + vaddvq_u32(vpaddlq_u16(hi)); +} + +template +static void sum2_8uC1(const Mat& srcmat, Mat& dstmat) +{ + const int cols = srcmat.cols; + + parallel_for_(Range(0, srcmat.rows), [&](const Range& range) { + for (int y = range.start; y < range.end; y++) + { + const uchar* src = srcmat.ptr(y); + DT* dst = dstmat.ptr
(y); + uint32_t result = 0; + int x = 0; + for (; x <= cols - 16; x += 16) + result += reduceSum2_8u_NEON(vld1q_u8(src + x)); + for (; x < cols; x++) + result += (uint32_t)src[x] * src[x]; + dst[0] = (DT)(int32_t)result; + } + }); + v_cleanup(); +} + +static void sum2_16u32fC1(const Mat& srcmat, Mat& dstmat) +{ + const int cols = srcmat.cols; + parallel_for_(Range(0, srcmat.rows), [&](const Range& range) { + for (int y = range.start; y < range.end; y++) + { + const ushort* src = srcmat.ptr(y); + float result = 0; + float32x4_t acc0 = vdupq_n_f32(0), acc1 = acc0, acc2 = acc0, acc3 = acc0; + int x = 0; + for (; x <= cols - 16; x += 16) + { + uint16x8_t v01 = vld1q_u16(src + x); + uint16x8_t v23 = vld1q_u16(src + x + 8); + float32x4_t v0 = vcvtq_f32_u32(vmovl_u16(vget_low_u16(v01))); + float32x4_t v1 = vcvtq_f32_u32(vmovl_u16(vget_high_u16(v01))); + float32x4_t v2 = vcvtq_f32_u32(vmovl_u16(vget_low_u16(v23))); + float32x4_t v3 = vcvtq_f32_u32(vmovl_u16(vget_high_u16(v23))); + acc0 = vmlaq_f32(acc0, v0, v0); + acc1 = vmlaq_f32(acc1, v1, v1); + acc2 = vmlaq_f32(acc2, v2, v2); + acc3 = vmlaq_f32(acc3, v3, v3); + } + acc0 = vaddq_f32(vaddq_f32(acc0, acc1), vaddq_f32(acc2, acc3)); + float lanes[4]; + vst1q_f32(lanes, acc0); + for (int i = 0; i < 4; i++) + result += lanes[i]; + for (; x < cols; x++) + { + float value = (float)src[x]; + result += value * value; + } + dstmat.ptr(y)[0] = result; + } + }); + v_cleanup(); +} + +static void sum2_16s32fC1(const Mat& srcmat, Mat& dstmat) +{ + const int cols = srcmat.cols; + parallel_for_(Range(0, srcmat.rows), [&](const Range& range) { + for (int y = range.start; y < range.end; y++) + { + const short* src = srcmat.ptr(y); + float result = 0; + float32x4_t acc0 = vdupq_n_f32(0), acc1 = acc0, acc2 = acc0, acc3 = acc0; + int x = 0; + for (; x <= cols - 16; x += 16) + { + int16x8_t v01 = vld1q_s16(src + x); + int16x8_t v23 = vld1q_s16(src + x + 8); + float32x4_t v0 = vcvtq_f32_s32(vmovl_s16(vget_low_s16(v01))); + float32x4_t v1 = vcvtq_f32_s32(vmovl_s16(vget_high_s16(v01))); + float32x4_t v2 = vcvtq_f32_s32(vmovl_s16(vget_low_s16(v23))); + float32x4_t v3 = vcvtq_f32_s32(vmovl_s16(vget_high_s16(v23))); + acc0 = vmlaq_f32(acc0, v0, v0); + acc1 = vmlaq_f32(acc1, v1, v1); + acc2 = vmlaq_f32(acc2, v2, v2); + acc3 = vmlaq_f32(acc3, v3, v3); + } + acc0 = vaddq_f32(vaddq_f32(acc0, acc1), vaddq_f32(acc2, acc3)); + float lanes[4]; + vst1q_f32(lanes, acc0); + for (int i = 0; i < 4; i++) + result += lanes[i]; + for (; x < cols; x++) + { + float value = (float)src[x]; + result += value * value; + } + dstmat.ptr(y)[0] = result; + } + }); + v_cleanup(); +} + +static void sum2_16u64fC1(const Mat& srcmat, Mat& dstmat) +{ + const int cols = srcmat.cols; + parallel_for_(Range(0, srcmat.rows), [&](const Range& range) { + for (int y = range.start; y < range.end; y++) + { + const ushort* src = srcmat.ptr(y); + uint64x2_t acc0 = vdupq_n_u64(0), acc1 = acc0, acc2 = acc0, acc3 = acc0; + int x = 0; + for (; x <= cols - 8; x += 8) + { + uint16x8_t v = vld1q_u16(src + x); + uint32x4_t sq0 = vmull_u16(vget_low_u16(v), vget_low_u16(v)); + uint32x4_t sq1 = vmull_u16(vget_high_u16(v), vget_high_u16(v)); + acc0 = vaddq_u64(acc0, vmovl_u32(vget_low_u32(sq0))); + acc1 = vaddq_u64(acc1, vmovl_high_u32(sq0)); + acc2 = vaddq_u64(acc2, vmovl_u32(vget_low_u32(sq1))); + acc3 = vaddq_u64(acc3, vmovl_high_u32(sq1)); + } + acc0 = vaddq_u64(vaddq_u64(acc0, acc1), vaddq_u64(acc2, acc3)); + uint64_t lanes[2]; + vst1q_u64(lanes, acc0); + uint64_t result = lanes[0] + lanes[1]; + for (; x < cols; x++) + result += (uint64_t)src[x] * src[x]; + dstmat.ptr(y)[0] = (double)result; + } + }); + v_cleanup(); +} + +static void sum2_16s64fC1(const Mat& srcmat, Mat& dstmat) +{ + const int cols = srcmat.cols; + parallel_for_(Range(0, srcmat.rows), [&](const Range& range) { + for (int y = range.start; y < range.end; y++) + { + const short* src = srcmat.ptr(y); + uint64x2_t acc0 = vdupq_n_u64(0), acc1 = acc0, acc2 = acc0, acc3 = acc0; + int x = 0; + for (; x <= cols - 8; x += 8) + { + int16x8_t v = vld1q_s16(src + x); + int32x4_t sq0 = vmull_s16(vget_low_s16(v), vget_low_s16(v)); + int32x4_t sq1 = vmull_s16(vget_high_s16(v), vget_high_s16(v)); + uint32x4_t usq0 = vreinterpretq_u32_s32(sq0); + uint32x4_t usq1 = vreinterpretq_u32_s32(sq1); + acc0 = vaddq_u64(acc0, vmovl_u32(vget_low_u32(usq0))); + acc1 = vaddq_u64(acc1, vmovl_high_u32(usq0)); + acc2 = vaddq_u64(acc2, vmovl_u32(vget_low_u32(usq1))); + acc3 = vaddq_u64(acc3, vmovl_high_u32(usq1)); + } + acc0 = vaddq_u64(vaddq_u64(acc0, acc1), vaddq_u64(acc2, acc3)); + uint64_t lanes[2]; + vst1q_u64(lanes, acc0); + uint64_t result = lanes[0] + lanes[1]; + for (; x < cols; x++) + { + int64_t value = src[x]; + result += (uint64_t)(value * value); + } + dstmat.ptr(y)[0] = (double)result; + } + }); + v_cleanup(); +} + +static void sum2_32f64fC1(const Mat& srcmat, Mat& dstmat) +{ + const int cols = srcmat.cols; + parallel_for_(Range(0, srcmat.rows), [&](const Range& range) { + for (int y = range.start; y < range.end; y++) + { + const float* src = srcmat.ptr(y); + float64x2_t acc0 = vdupq_n_f64(0), acc1 = acc0, acc2 = acc0, acc3 = acc0; + int x = 0; + for (; x <= cols - 8; x += 8) + { + float32x4_t v01 = vld1q_f32(src + x); + float32x4_t v23 = vld1q_f32(src + x + 4); + float64x2_t v0 = vcvt_f64_f32(vget_low_f32(v01)); + float64x2_t v1 = vcvt_high_f64_f32(v01); + float64x2_t v2 = vcvt_f64_f32(vget_low_f32(v23)); + float64x2_t v3 = vcvt_high_f64_f32(v23); + acc0 = vmlaq_f64(acc0, v0, v0); + acc1 = vmlaq_f64(acc1, v1, v1); + acc2 = vmlaq_f64(acc2, v2, v2); + acc3 = vmlaq_f64(acc3, v3, v3); + } + acc0 = vaddq_f64(vaddq_f64(acc0, acc1), vaddq_f64(acc2, acc3)); + double lanes[2]; + vst1q_f64(lanes, acc0); + double result = lanes[0] + lanes[1]; + for (; x < cols; x++) + { + double value = (double)src[x]; + result += value * value; + } + dstmat.ptr(y)[0] = result; + } + }); + v_cleanup(); +} + +static void sum2_64f64fC1(const Mat& srcmat, Mat& dstmat) +{ + const int cols = srcmat.cols; + parallel_for_(Range(0, srcmat.rows), [&](const Range& range) { + for (int y = range.start; y < range.end; y++) + { + const double* src = srcmat.ptr(y); + float64x2_t acc0 = vdupq_n_f64(0), acc1 = acc0, acc2 = acc0, acc3 = acc0; + int x = 0; + for (; x <= cols - 8; x += 8) + { + float64x2_t v0 = vld1q_f64(src + x); + float64x2_t v1 = vld1q_f64(src + x + 2); + float64x2_t v2 = vld1q_f64(src + x + 4); + float64x2_t v3 = vld1q_f64(src + x + 6); + acc0 = vmlaq_f64(acc0, v0, v0); + acc1 = vmlaq_f64(acc1, v1, v1); + acc2 = vmlaq_f64(acc2, v2, v2); + acc3 = vmlaq_f64(acc3, v3, v3); + } + acc0 = vaddq_f64(vaddq_f64(acc0, acc1), vaddq_f64(acc2, acc3)); + double lanes[2]; + vst1q_f64(lanes, acc0); + double result = lanes[0] + lanes[1]; + for (; x < cols; x++) + result += src[x] * src[x]; + dstmat.ptr(y)[0] = result; + } + }); + v_cleanup(); +} + +template +static void sum2_8uC3(const Mat& srcmat, Mat& dstmat) +{ + const int cols = srcmat.cols; + parallel_for_(Range(0, srcmat.rows), [&](const Range& range) { + for (int y = range.start; y < range.end; y++) + { + const uchar* src = srcmat.ptr(y); + DT* dst = dstmat.ptr
(y); + uint32_t results[3] = {0, 0, 0}; + int x = 0; + for (; x <= cols - 16; x += 16) + { + uint8x16x3_t v = vld3q_u8(src + x * 3); + for (int c = 0; c < 3; c++) + results[c] += reduceSum2_8u_NEON(v.val[c]); + } + for (int c = 0; c < 3; c++) + { + for (int i = x; i < cols; i++) + { + uint32_t value = src[i * 3 + c]; + results[c] += value * value; + } + dst[c] = (DT)(int32_t)results[c]; + } + } + }); + v_cleanup(); +} + +template +static void sum2_8uC4(const Mat& srcmat, Mat& dstmat) +{ + const int cols = srcmat.cols; + parallel_for_(Range(0, srcmat.rows), [&](const Range& range) { + for (int y = range.start; y < range.end; y++) + { + const uchar* src = srcmat.ptr(y); + DT* dst = dstmat.ptr
(y); + uint32_t results[4] = {0, 0, 0, 0}; + int x = 0; + for (; x <= cols - 16; x += 16) + { + uint8x16x4_t v = vld4q_u8(src + x * 4); + for (int c = 0; c < 4; c++) + results[c] += reduceSum2_8u_NEON(v.val[c]); + } + for (int c = 0; c < 4; c++) + { + for (int i = x; i < cols; i++) + { + uint32_t value = src[i * 4 + c]; + results[c] += value * value; + } + dst[c] = (DT)(int32_t)results[c]; + } + } + }); + v_cleanup(); +} + +#endif + +static void sum2_32fC1(const Mat& srcmat, Mat& dstmat) +{ + const int cols = srcmat.cols; + parallel_for_(Range(0, srcmat.rows), [&](const Range& range) { + for (int y = range.start; y < range.end; y++) + { + const float* src = srcmat.ptr(y); + float result = 0; + int x = 0; + float32x4_t acc = vdupq_n_f32(0); + for (; x <= cols - 4; x += 4) + { + float32x4_t v = vld1q_f32(src + x); + acc = vaddq_f32(acc, vmulq_f32(v, v)); + } + float lanes[4]; + vst1q_f32(lanes, acc); + for (int i = 0; i < 4; i++) + result += lanes[i]; + for (; x < cols; x++) + result += src[x] * src[x]; + dstmat.ptr(y)[0] = result; + } + }); + v_cleanup(); +} + +static void sum2_32fC4(const Mat& srcmat, Mat& dstmat) +{ + const int cols = srcmat.cols; + parallel_for_(Range(0, srcmat.rows), [&](const Range& range) { + for (int y = range.start; y < range.end; y++) + { + const float* src = srcmat.ptr(y); + float* dst = dstmat.ptr(y); + float32x4x4_t acc = {{ + vdupq_n_f32(0), vdupq_n_f32(0), + vdupq_n_f32(0), vdupq_n_f32(0) + }}; + int x = 0; + for (; x <= cols - 4; x += 4) + { + float32x4x4_t v = vld4q_f32(src + x * 4); + for (int c = 0; c < 4; c++) + acc.val[c] = vmlaq_f32(acc.val[c], v.val[c], v.val[c]); + } + for (int c = 0; c < 4; c++) + { + float lanes[4]; + float result = 0; + vst1q_f32(lanes, acc.val[c]); + for (int i = 0; i < 4; i++) + result += lanes[i]; + for (int i = x; i < cols; i++) + result += src[i * 4 + c] * src[i * 4 + c]; + dst[c] = result; + } + } + }); + v_cleanup(); +} + +} // namespace reduce_c_neon diff --git a/modules/core/src/reduce_c_rvv.hpp b/modules/core/src/reduce_c_rvv.hpp new file mode 100644 index 0000000000..58bbaa9a2c --- /dev/null +++ b/modules/core/src/reduce_c_rvv.hpp @@ -0,0 +1,866 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html + +namespace reduce_c_rvv +{ + +// Optimized ReduceC support in this backend: +// +// | Input -> output type/channel | SUM | AVG | MIN | MAX | SUM2 | +// |------------------------------------|:---:|:---:|:---:|:---:|:----:| +// | 8UC1/C3/C4 -> 8UC1/C3/C4 | - | - | x | x | - | +// | 8UC1/C3/C4 -> 32SC1/C3/C4 | x | x | - | - | x | +// | 8UC1/C3/C4 -> 32FC1/C3/C4 | x | x | - | - | x | +// | 8UC1/C3/C4 -> 64FC1/C3/C4 | - | - | - | - | x | +// | 16UC1/C3/C4 -> 16UC1/C3/C4 | - | - | x | x | - | +// | 16UC1 -> 32FC1 | x | x | - | - | x | +// | 16UC3/C4 -> 32FC3/C4 | x | x | - | - | - | +// | 16UC1 -> 64FC1 | - | - | - | - | x* | +// | 16UC3/C4 -> 64FC3/C4 | - | - | - | - | - | +// | 16SC1/C3/C4 -> 16SC1/C3/C4 | - | - | x | x | - | +// | 16SC1 -> 32FC1 | x | x | - | - | x | +// | 16SC3/C4 -> 32FC3/C4 | x | x | - | - | - | +// | 16SC1 -> 64FC1 | - | - | - | - | x* | +// | 16SC3/C4 -> 64FC3/C4 | - | - | - | - | - | +// | 32FC1/C3/C4 -> 32FC1/C3/C4 | x | x | x | x | x | +// | 32FC1 -> 64FC1 | x* | x* | - | - | x* | +// | 32FC3/C4 -> 64FC3/C4 | x* | x* | - | - | - | +// | 64FC1 -> 64FC1 | x* | x* | x* | x* | x* | +// | 64FC3/C4 -> 64FC3/C4 | x* | x* | - | - | - | +// +// 'x' in SUM/AVG denotes the existing shared universal-intrinsics kernel; 'x' +// in MIN/MAX/SUM2 denotes a native RVV kernel. '*' requires +// CV_SIMD_SCALABLE_64F. For legal MIN/MAX/SUM2 combinations marked '-', and for +// other channel counts, dispatch uses the shared generic fallback. + +template +static void minMax8uC1(const Mat& srcmat, Mat& dstmat) +{ + const int cols = srcmat.cols; + parallel_for_(Range(0, srcmat.rows), [&](const Range& range) { + for (int y = range.start; y < range.end; y++) + { + const uchar* src = srcmat.ptr(y); + uchar* dst = dstmat.ptr(y); + uchar result = isMax ? 0 : UCHAR_MAX; + int x = 0; + const int vlmax = __riscv_vsetvlmax_e8m8(); + vuint8m8_t acc = __riscv_vmv_v_x_u8m8(result, vlmax); + for (; x < cols; ) + { + const int vl = __riscv_vsetvl_e8m8(cols - x); + vuint8m8_t v = __riscv_vle8_v_u8m8(src + x, vl); + acc = isMax ? __riscv_vmaxu_tu(acc, acc, v, vl) + : __riscv_vminu_tu(acc, acc, v, vl); + x += vl; + } + vuint8m1_t seed = __riscv_vmv_s_x_u8m1(result, __riscv_vsetvlmax_e8m1()); + vuint8m1_t reduced = isMax ? __riscv_vredmaxu(acc, seed, vlmax) + : __riscv_vredminu(acc, seed, vlmax); + result = (uchar)__riscv_vmv_x(reduced); + for (; x < cols; x++) + result = reduceScalarMinMax(result, src[x]); + dst[0] = result; + } + }); + v_cleanup(); +} + +template +static void minMax16uC1(const Mat& srcmat, Mat& dstmat) +{ + const int cols = srcmat.cols; + parallel_for_(Range(0, srcmat.rows), [&](const Range& range) { + for (int y = range.start; y < range.end; y++) + { + const ushort* src = srcmat.ptr(y); + const ushort initial = isMax ? 0 : USHRT_MAX; + const int vlmax = __riscv_vsetvlmax_e16m8(); + vuint16m8_t acc = __riscv_vmv_v_x_u16m8(initial, vlmax); + int x = 0; + for (; x < cols; ) + { + const int vl = __riscv_vsetvl_e16m8(cols - x); + vuint16m8_t v = __riscv_vle16_v_u16m8(src + x, vl); + acc = isMax ? __riscv_vmaxu_tu(acc, acc, v, vl) + : __riscv_vminu_tu(acc, acc, v, vl); + x += vl; + } + vuint16m1_t seed = __riscv_vmv_s_x_u16m1(initial, __riscv_vsetvlmax_e16m1()); + vuint16m1_t reduced = isMax ? __riscv_vredmaxu(acc, seed, vlmax) + : __riscv_vredminu(acc, seed, vlmax); + dstmat.ptr(y)[0] = (ushort)__riscv_vmv_x(reduced); + } + }); + v_cleanup(); +} + +template +static void minMax16sC1(const Mat& srcmat, Mat& dstmat) +{ + const int cols = srcmat.cols; + parallel_for_(Range(0, srcmat.rows), [&](const Range& range) { + for (int y = range.start; y < range.end; y++) + { + const short* src = srcmat.ptr(y); + const short initial = isMax ? SHRT_MIN : SHRT_MAX; + const int vlmax = __riscv_vsetvlmax_e16m8(); + vint16m8_t acc = __riscv_vmv_v_x_i16m8(initial, vlmax); + int x = 0; + for (; x < cols; ) + { + const int vl = __riscv_vsetvl_e16m8(cols - x); + vint16m8_t v = __riscv_vle16_v_i16m8(src + x, vl); + acc = isMax ? __riscv_vmax_tu(acc, acc, v, vl) + : __riscv_vmin_tu(acc, acc, v, vl); + x += vl; + } + vint16m1_t seed = __riscv_vmv_s_x_i16m1(initial, __riscv_vsetvlmax_e16m1()); + vint16m1_t reduced = isMax ? __riscv_vredmax(acc, seed, vlmax) + : __riscv_vredmin(acc, seed, vlmax); + dstmat.ptr(y)[0] = (short)__riscv_vmv_x(reduced); + } + }); + v_cleanup(); +} + +template +static void minMax16uC4(const Mat& srcmat, Mat& dstmat) +{ + const int cols = srcmat.cols; + const ushort initial = isMax ? 0 : USHRT_MAX; + const int vlmax = __riscv_vsetvlmax_e16m2(); + parallel_for_(Range(0, srcmat.rows), [&](const Range& range) { + for (int y = range.start; y < range.end; y++) + { + const ushort* src = srcmat.ptr(y); + ushort* dst = dstmat.ptr(y); + vuint16m2_t acc0 = __riscv_vmv_v_x_u16m2(initial, vlmax); + vuint16m2_t acc1 = acc0, acc2 = acc0, acc3 = acc0; + int x = 0; + for (; x < cols; ) + { + const int vl = __riscv_vsetvl_e16m2(cols - x); + vuint16m2x4_t v = __riscv_vlseg4e16_v_u16m2x4(src + x * 4, vl); + vuint16m2_t v0 = __riscv_vget_v_u16m2x4_u16m2(v, 0); + vuint16m2_t v1 = __riscv_vget_v_u16m2x4_u16m2(v, 1); + vuint16m2_t v2 = __riscv_vget_v_u16m2x4_u16m2(v, 2); + vuint16m2_t v3 = __riscv_vget_v_u16m2x4_u16m2(v, 3); + acc0 = isMax ? __riscv_vmaxu_tu(acc0, acc0, v0, vl) + : __riscv_vminu_tu(acc0, acc0, v0, vl); + acc1 = isMax ? __riscv_vmaxu_tu(acc1, acc1, v1, vl) + : __riscv_vminu_tu(acc1, acc1, v1, vl); + acc2 = isMax ? __riscv_vmaxu_tu(acc2, acc2, v2, vl) + : __riscv_vminu_tu(acc2, acc2, v2, vl); + acc3 = isMax ? __riscv_vmaxu_tu(acc3, acc3, v3, vl) + : __riscv_vminu_tu(acc3, acc3, v3, vl); + x += vl; + } + vuint16m1_t seed = __riscv_vmv_s_x_u16m1(initial, __riscv_vsetvlmax_e16m1()); + dst[0] = (ushort)__riscv_vmv_x(isMax ? __riscv_vredmaxu(acc0, seed, vlmax) : __riscv_vredminu(acc0, seed, vlmax)); + dst[1] = (ushort)__riscv_vmv_x(isMax ? __riscv_vredmaxu(acc1, seed, vlmax) : __riscv_vredminu(acc1, seed, vlmax)); + dst[2] = (ushort)__riscv_vmv_x(isMax ? __riscv_vredmaxu(acc2, seed, vlmax) : __riscv_vredminu(acc2, seed, vlmax)); + dst[3] = (ushort)__riscv_vmv_x(isMax ? __riscv_vredmaxu(acc3, seed, vlmax) : __riscv_vredminu(acc3, seed, vlmax)); + } + }); + v_cleanup(); +} + +template +static void minMax16sC4(const Mat& srcmat, Mat& dstmat) +{ + const int cols = srcmat.cols; + const short initial = isMax ? SHRT_MIN : SHRT_MAX; + const int vlmax = __riscv_vsetvlmax_e16m2(); + parallel_for_(Range(0, srcmat.rows), [&](const Range& range) { + for (int y = range.start; y < range.end; y++) + { + const short* src = srcmat.ptr(y); + short* dst = dstmat.ptr(y); + vint16m2_t acc0 = __riscv_vmv_v_x_i16m2(initial, vlmax); + vint16m2_t acc1 = acc0, acc2 = acc0, acc3 = acc0; + int x = 0; + for (; x < cols; ) + { + const int vl = __riscv_vsetvl_e16m2(cols - x); + vint16m2x4_t v = __riscv_vlseg4e16_v_i16m2x4(src + x * 4, vl); + vint16m2_t v0 = __riscv_vget_v_i16m2x4_i16m2(v, 0); + vint16m2_t v1 = __riscv_vget_v_i16m2x4_i16m2(v, 1); + vint16m2_t v2 = __riscv_vget_v_i16m2x4_i16m2(v, 2); + vint16m2_t v3 = __riscv_vget_v_i16m2x4_i16m2(v, 3); + acc0 = isMax ? __riscv_vmax_tu(acc0, acc0, v0, vl) + : __riscv_vmin_tu(acc0, acc0, v0, vl); + acc1 = isMax ? __riscv_vmax_tu(acc1, acc1, v1, vl) + : __riscv_vmin_tu(acc1, acc1, v1, vl); + acc2 = isMax ? __riscv_vmax_tu(acc2, acc2, v2, vl) + : __riscv_vmin_tu(acc2, acc2, v2, vl); + acc3 = isMax ? __riscv_vmax_tu(acc3, acc3, v3, vl) + : __riscv_vmin_tu(acc3, acc3, v3, vl); + x += vl; + } + vint16m1_t seed = __riscv_vmv_s_x_i16m1(initial, __riscv_vsetvlmax_e16m1()); + dst[0] = (short)__riscv_vmv_x(isMax ? __riscv_vredmax(acc0, seed, vlmax) : __riscv_vredmin(acc0, seed, vlmax)); + dst[1] = (short)__riscv_vmv_x(isMax ? __riscv_vredmax(acc1, seed, vlmax) : __riscv_vredmin(acc1, seed, vlmax)); + dst[2] = (short)__riscv_vmv_x(isMax ? __riscv_vredmax(acc2, seed, vlmax) : __riscv_vredmin(acc2, seed, vlmax)); + dst[3] = (short)__riscv_vmv_x(isMax ? __riscv_vredmax(acc3, seed, vlmax) : __riscv_vredmin(acc3, seed, vlmax)); + } + }); + v_cleanup(); +} + +template +static void minMax16uC3(const Mat& srcmat, Mat& dstmat) +{ + const int cols = srcmat.cols; + const ushort initial = isMax ? 0 : USHRT_MAX; + const int vlmax = __riscv_vsetvlmax_e16m2(); + parallel_for_(Range(0, srcmat.rows), [&](const Range& range) { + for (int y = range.start; y < range.end; y++) + { + const ushort* src = srcmat.ptr(y); + ushort* dst = dstmat.ptr(y); + vuint16m2_t acc0 = __riscv_vmv_v_x_u16m2(initial, vlmax); + vuint16m2_t acc1 = acc0, acc2 = acc0; + int x = 0; + for (; x < cols; ) + { + const int vl = __riscv_vsetvl_e16m2(cols - x); + vuint16m2x3_t v = __riscv_vlseg3e16_v_u16m2x3(src + x * 3, vl); + vuint16m2_t v0 = __riscv_vget_v_u16m2x3_u16m2(v, 0); + vuint16m2_t v1 = __riscv_vget_v_u16m2x3_u16m2(v, 1); + vuint16m2_t v2 = __riscv_vget_v_u16m2x3_u16m2(v, 2); + acc0 = isMax ? __riscv_vmaxu_tu(acc0, acc0, v0, vl) : __riscv_vminu_tu(acc0, acc0, v0, vl); + acc1 = isMax ? __riscv_vmaxu_tu(acc1, acc1, v1, vl) : __riscv_vminu_tu(acc1, acc1, v1, vl); + acc2 = isMax ? __riscv_vmaxu_tu(acc2, acc2, v2, vl) : __riscv_vminu_tu(acc2, acc2, v2, vl); + x += vl; + } + vuint16m1_t seed = __riscv_vmv_s_x_u16m1(initial, __riscv_vsetvlmax_e16m1()); + dst[0] = (ushort)__riscv_vmv_x(isMax ? __riscv_vredmaxu(acc0, seed, vlmax) : __riscv_vredminu(acc0, seed, vlmax)); + dst[1] = (ushort)__riscv_vmv_x(isMax ? __riscv_vredmaxu(acc1, seed, vlmax) : __riscv_vredminu(acc1, seed, vlmax)); + dst[2] = (ushort)__riscv_vmv_x(isMax ? __riscv_vredmaxu(acc2, seed, vlmax) : __riscv_vredminu(acc2, seed, vlmax)); + } + }); + v_cleanup(); +} + +template +static void minMax16sC3(const Mat& srcmat, Mat& dstmat) +{ + const int cols = srcmat.cols; + const short initial = isMax ? SHRT_MIN : SHRT_MAX; + const int vlmax = __riscv_vsetvlmax_e16m2(); + parallel_for_(Range(0, srcmat.rows), [&](const Range& range) { + for (int y = range.start; y < range.end; y++) + { + const short* src = srcmat.ptr(y); + short* dst = dstmat.ptr(y); + vint16m2_t acc0 = __riscv_vmv_v_x_i16m2(initial, vlmax); + vint16m2_t acc1 = acc0, acc2 = acc0; + int x = 0; + for (; x < cols; ) + { + const int vl = __riscv_vsetvl_e16m2(cols - x); + vint16m2x3_t v = __riscv_vlseg3e16_v_i16m2x3(src + x * 3, vl); + vint16m2_t v0 = __riscv_vget_v_i16m2x3_i16m2(v, 0); + vint16m2_t v1 = __riscv_vget_v_i16m2x3_i16m2(v, 1); + vint16m2_t v2 = __riscv_vget_v_i16m2x3_i16m2(v, 2); + acc0 = isMax ? __riscv_vmax_tu(acc0, acc0, v0, vl) : __riscv_vmin_tu(acc0, acc0, v0, vl); + acc1 = isMax ? __riscv_vmax_tu(acc1, acc1, v1, vl) : __riscv_vmin_tu(acc1, acc1, v1, vl); + acc2 = isMax ? __riscv_vmax_tu(acc2, acc2, v2, vl) : __riscv_vmin_tu(acc2, acc2, v2, vl); + x += vl; + } + vint16m1_t seed = __riscv_vmv_s_x_i16m1(initial, __riscv_vsetvlmax_e16m1()); + dst[0] = (short)__riscv_vmv_x(isMax ? __riscv_vredmax(acc0, seed, vlmax) : __riscv_vredmin(acc0, seed, vlmax)); + dst[1] = (short)__riscv_vmv_x(isMax ? __riscv_vredmax(acc1, seed, vlmax) : __riscv_vredmin(acc1, seed, vlmax)); + dst[2] = (short)__riscv_vmv_x(isMax ? __riscv_vredmax(acc2, seed, vlmax) : __riscv_vredmin(acc2, seed, vlmax)); + } + }); + v_cleanup(); +} + +template +static void minMax8uC3(const Mat& srcmat, Mat& dstmat) +{ + const int cols = srcmat.cols; + const uchar initial = isMax ? 0 : UCHAR_MAX; + const int vlmax = __riscv_vsetvlmax_e8m1(); + parallel_for_(Range(0, srcmat.rows), [&](const Range& range) { + for (int y = range.start; y < range.end; y++) + { + const uchar* src = srcmat.ptr(y); + uchar* dst = dstmat.ptr(y); + vuint8m1_t acc0 = __riscv_vmv_v_x_u8m1(initial, vlmax); + vuint8m1_t acc1 = acc0, acc2 = acc0; + int x = 0; + for (; x < cols; ) + { + const int vl = __riscv_vsetvl_e8m1(cols - x); + vuint8m1x3_t v = __riscv_vlseg3e8_v_u8m1x3(src + x * 3, vl); + vuint8m1_t v0 = __riscv_vget_v_u8m1x3_u8m1(v, 0); + vuint8m1_t v1 = __riscv_vget_v_u8m1x3_u8m1(v, 1); + vuint8m1_t v2 = __riscv_vget_v_u8m1x3_u8m1(v, 2); + acc0 = isMax ? __riscv_vmaxu_tu(acc0, acc0, v0, vl) + : __riscv_vminu_tu(acc0, acc0, v0, vl); + acc1 = isMax ? __riscv_vmaxu_tu(acc1, acc1, v1, vl) + : __riscv_vminu_tu(acc1, acc1, v1, vl); + acc2 = isMax ? __riscv_vmaxu_tu(acc2, acc2, v2, vl) + : __riscv_vminu_tu(acc2, acc2, v2, vl); + x += vl; + } + vuint8m1_t seed = __riscv_vmv_s_x_u8m1(initial, vlmax); + dst[0] = (uchar)__riscv_vmv_x(isMax ? __riscv_vredmaxu(acc0, seed, vlmax) + : __riscv_vredminu(acc0, seed, vlmax)); + dst[1] = (uchar)__riscv_vmv_x(isMax ? __riscv_vredmaxu(acc1, seed, vlmax) + : __riscv_vredminu(acc1, seed, vlmax)); + dst[2] = (uchar)__riscv_vmv_x(isMax ? __riscv_vredmaxu(acc2, seed, vlmax) + : __riscv_vredminu(acc2, seed, vlmax)); + } + }); + v_cleanup(); +} + +template +static void minMax8uC4(const Mat& srcmat, Mat& dstmat) +{ + const int cols = srcmat.cols; + const uchar initial = isMax ? 0 : UCHAR_MAX; + const int vlmax = __riscv_vsetvlmax_e8m1(); + parallel_for_(Range(0, srcmat.rows), [&](const Range& range) { + for (int y = range.start; y < range.end; y++) + { + const uchar* src = srcmat.ptr(y); + uchar* dst = dstmat.ptr(y); + vuint8m1_t acc0 = __riscv_vmv_v_x_u8m1(initial, vlmax); + vuint8m1_t acc1 = acc0, acc2 = acc0, acc3 = acc0; + int x = 0; + for (; x < cols; ) + { + const int vl = __riscv_vsetvl_e8m1(cols - x); + vuint8m1x4_t v = __riscv_vlseg4e8_v_u8m1x4(src + x * 4, vl); + vuint8m1_t v0 = __riscv_vget_v_u8m1x4_u8m1(v, 0); + vuint8m1_t v1 = __riscv_vget_v_u8m1x4_u8m1(v, 1); + vuint8m1_t v2 = __riscv_vget_v_u8m1x4_u8m1(v, 2); + vuint8m1_t v3 = __riscv_vget_v_u8m1x4_u8m1(v, 3); + acc0 = isMax ? __riscv_vmaxu_tu(acc0, acc0, v0, vl) + : __riscv_vminu_tu(acc0, acc0, v0, vl); + acc1 = isMax ? __riscv_vmaxu_tu(acc1, acc1, v1, vl) + : __riscv_vminu_tu(acc1, acc1, v1, vl); + acc2 = isMax ? __riscv_vmaxu_tu(acc2, acc2, v2, vl) + : __riscv_vminu_tu(acc2, acc2, v2, vl); + acc3 = isMax ? __riscv_vmaxu_tu(acc3, acc3, v3, vl) + : __riscv_vminu_tu(acc3, acc3, v3, vl); + x += vl; + } + vuint8m1_t seed = __riscv_vmv_s_x_u8m1(initial, vlmax); + dst[0] = (uchar)__riscv_vmv_x(isMax ? __riscv_vredmaxu(acc0, seed, vlmax) + : __riscv_vredminu(acc0, seed, vlmax)); + dst[1] = (uchar)__riscv_vmv_x(isMax ? __riscv_vredmaxu(acc1, seed, vlmax) + : __riscv_vredminu(acc1, seed, vlmax)); + dst[2] = (uchar)__riscv_vmv_x(isMax ? __riscv_vredmaxu(acc2, seed, vlmax) + : __riscv_vredminu(acc2, seed, vlmax)); + dst[3] = (uchar)__riscv_vmv_x(isMax ? __riscv_vredmaxu(acc3, seed, vlmax) + : __riscv_vredminu(acc3, seed, vlmax)); + } + }); + v_cleanup(); +} + +template +static void minMax32fC1(const Mat& srcmat, Mat& dstmat) +{ + const int cols = srcmat.cols; + parallel_for_(Range(0, srcmat.rows), [&](const Range& range) { + for (int y = range.start; y < range.end; y++) + { + const float* src = srcmat.ptr(y); + float result = src[0]; + int x = 0; + const int vlmax = __riscv_vsetvlmax_e32m8(); + vfloat32m8_t acc = __riscv_vfmv_v_f_f32m8(result, vlmax); + for (; x < cols; ) + { + const int vl = __riscv_vsetvl_e32m8(cols - x); + vfloat32m8_t v = __riscv_vle32_v_f32m8(src + x, vl); + acc = isMax ? __riscv_vfmax_tu(acc, acc, v, vl) + : __riscv_vfmin_tu(acc, acc, v, vl); + x += vl; + } + vfloat32m1_t seed = __riscv_vfmv_s_f_f32m1(result, __riscv_vsetvlmax_e32m1()); + vfloat32m1_t reduced = isMax ? __riscv_vfredmax(acc, seed, vlmax) + : __riscv_vfredmin(acc, seed, vlmax); + result = __riscv_vfmv_f(reduced); + for (; x < cols; x++) + result = reduceScalarMinMax(result, src[x]); + dstmat.ptr(y)[0] = result; + } + }); + v_cleanup(); +} + +template +static void minMax32fC3(const Mat& srcmat, Mat& dstmat) +{ + const int cols = srcmat.cols; + const float initial = isMax ? std::numeric_limits::lowest() : std::numeric_limits::max(); + const int vlmax = __riscv_vsetvlmax_e32m2(); + parallel_for_(Range(0, srcmat.rows), [&](const Range& range) { + for (int y = range.start; y < range.end; y++) + { + const float* src = srcmat.ptr(y); + float* dst = dstmat.ptr(y); + vfloat32m2_t acc0 = __riscv_vfmv_v_f_f32m2(initial, vlmax); + vfloat32m2_t acc1 = acc0, acc2 = acc0; + int x = 0; + for (; x < cols; ) + { + const int vl = __riscv_vsetvl_e32m2(cols - x); + vfloat32m2x3_t v = __riscv_vlseg3e32_v_f32m2x3(src + x * 3, vl); + vfloat32m2_t v0 = __riscv_vget_v_f32m2x3_f32m2(v, 0); + vfloat32m2_t v1 = __riscv_vget_v_f32m2x3_f32m2(v, 1); + vfloat32m2_t v2 = __riscv_vget_v_f32m2x3_f32m2(v, 2); + acc0 = isMax ? __riscv_vfmax_tu(acc0, acc0, v0, vl) + : __riscv_vfmin_tu(acc0, acc0, v0, vl); + acc1 = isMax ? __riscv_vfmax_tu(acc1, acc1, v1, vl) + : __riscv_vfmin_tu(acc1, acc1, v1, vl); + acc2 = isMax ? __riscv_vfmax_tu(acc2, acc2, v2, vl) + : __riscv_vfmin_tu(acc2, acc2, v2, vl); + x += vl; + } + vfloat32m1_t seed = __riscv_vfmv_s_f_f32m1(initial, __riscv_vsetvlmax_e32m1()); + dst[0] = __riscv_vfmv_f(isMax ? __riscv_vfredmax(acc0, seed, vlmax) + : __riscv_vfredmin(acc0, seed, vlmax)); + dst[1] = __riscv_vfmv_f(isMax ? __riscv_vfredmax(acc1, seed, vlmax) + : __riscv_vfredmin(acc1, seed, vlmax)); + dst[2] = __riscv_vfmv_f(isMax ? __riscv_vfredmax(acc2, seed, vlmax) + : __riscv_vfredmin(acc2, seed, vlmax)); + } + }); + v_cleanup(); +} + +template +static void minMax32fC4(const Mat& srcmat, Mat& dstmat) +{ + const int cols = srcmat.cols; + const float initial = isMax ? std::numeric_limits::lowest() : std::numeric_limits::max(); + const int vlmax = __riscv_vsetvlmax_e32m2(); + parallel_for_(Range(0, srcmat.rows), [&](const Range& range) { + for (int y = range.start; y < range.end; y++) + { + const float* src = srcmat.ptr(y); + float* dst = dstmat.ptr(y); + vfloat32m2_t acc0 = __riscv_vfmv_v_f_f32m2(initial, vlmax); + vfloat32m2_t acc1 = acc0, acc2 = acc0, acc3 = acc0; + int x = 0; + for (; x < cols; ) + { + const int vl = __riscv_vsetvl_e32m2(cols - x); + vfloat32m2x4_t v = __riscv_vlseg4e32_v_f32m2x4(src + x * 4, vl); + vfloat32m2_t v0 = __riscv_vget_v_f32m2x4_f32m2(v, 0); + vfloat32m2_t v1 = __riscv_vget_v_f32m2x4_f32m2(v, 1); + vfloat32m2_t v2 = __riscv_vget_v_f32m2x4_f32m2(v, 2); + vfloat32m2_t v3 = __riscv_vget_v_f32m2x4_f32m2(v, 3); + acc0 = isMax ? __riscv_vfmax_tu(acc0, acc0, v0, vl) + : __riscv_vfmin_tu(acc0, acc0, v0, vl); + acc1 = isMax ? __riscv_vfmax_tu(acc1, acc1, v1, vl) + : __riscv_vfmin_tu(acc1, acc1, v1, vl); + acc2 = isMax ? __riscv_vfmax_tu(acc2, acc2, v2, vl) + : __riscv_vfmin_tu(acc2, acc2, v2, vl); + acc3 = isMax ? __riscv_vfmax_tu(acc3, acc3, v3, vl) + : __riscv_vfmin_tu(acc3, acc3, v3, vl); + x += vl; + } + vfloat32m1_t seed = __riscv_vfmv_s_f_f32m1(initial, __riscv_vsetvlmax_e32m1()); + dst[0] = __riscv_vfmv_f(isMax ? __riscv_vfredmax(acc0, seed, vlmax) + : __riscv_vfredmin(acc0, seed, vlmax)); + dst[1] = __riscv_vfmv_f(isMax ? __riscv_vfredmax(acc1, seed, vlmax) + : __riscv_vfredmin(acc1, seed, vlmax)); + dst[2] = __riscv_vfmv_f(isMax ? __riscv_vfredmax(acc2, seed, vlmax) + : __riscv_vfredmin(acc2, seed, vlmax)); + dst[3] = __riscv_vfmv_f(isMax ? __riscv_vfredmax(acc3, seed, vlmax) + : __riscv_vfredmin(acc3, seed, vlmax)); + } + }); + v_cleanup(); +} + +#if CV_SIMD_SCALABLE_64F +template +static void minMax64fC1(const Mat& srcmat, Mat& dstmat) +{ + const int cols = srcmat.cols; + const double initial = isMax ? std::numeric_limits::lowest() + : std::numeric_limits::max(); + parallel_for_(Range(0, srcmat.rows), [&](const Range& range) { + for (int y = range.start; y < range.end; y++) + { + const double* src = srcmat.ptr(y); + const int vlmax = __riscv_vsetvlmax_e64m8(); + vfloat64m8_t acc = __riscv_vfmv_v_f_f64m8(initial, vlmax); + int x = 0; + for (; x < cols; ) + { + const int vl = __riscv_vsetvl_e64m8(cols - x); + vfloat64m8_t v = __riscv_vle64_v_f64m8(src + x, vl); + acc = isMax ? __riscv_vfmax_tu(acc, acc, v, vl) + : __riscv_vfmin_tu(acc, acc, v, vl); + x += vl; + } + vfloat64m1_t seed = __riscv_vfmv_s_f_f64m1(initial, __riscv_vsetvlmax_e64m1()); + vfloat64m1_t reduced = isMax ? __riscv_vfredmax(acc, seed, vlmax) + : __riscv_vfredmin(acc, seed, vlmax); + dstmat.ptr(y)[0] = __riscv_vfmv_f(reduced); + } + }); + v_cleanup(); +} +#endif + +template +static void sum2_8uC1(const Mat& srcmat, Mat& dstmat) +{ + const int cols = srcmat.cols; + + parallel_for_(Range(0, srcmat.rows), [&](const Range& range) { + for (int y = range.start; y < range.end; y++) + { + const uchar* src = srcmat.ptr(y); + DT* dst = dstmat.ptr
(y); + uint32_t result = 0; + int x = 0; + vuint32m1_t acc = __riscv_vmv_v_x_u32m1(0, __riscv_vsetvlmax_e32m1()); + for (; x < cols; ) + { + const int vl = __riscv_vsetvl_e8m4(cols - x); + vuint8m4_t v = __riscv_vle8_v_u8m4(src + x, vl); + acc = __riscv_vwredsumu(__riscv_vwmulu(v, v, vl), acc, vl); + x += vl; + } + result = (uint32_t)__riscv_vmv_x(acc); + for (; x < cols; x++) + result += (uint32_t)src[x] * src[x]; + dst[0] = (DT)(int32_t)result; + } + }); + v_cleanup(); +} + +static void sum2_16u32fC1(const Mat& srcmat, Mat& dstmat) +{ + const int cols = srcmat.cols; + parallel_for_(Range(0, srcmat.rows), [&](const Range& range) { + for (int y = range.start; y < range.end; y++) + { + const ushort* src = srcmat.ptr(y); + const int vlmax = __riscv_vsetvlmax_e32m8(); + vfloat32m8_t acc = __riscv_vfmv_v_f_f32m8(0, vlmax); + int x = 0; + for (; x < cols; ) + { + const int vl = __riscv_vsetvl_e16m4(cols - x); + vuint16m4_t v = __riscv_vle16_v_u16m4(src + x, vl); + vfloat32m8_t vf = __riscv_vfwcvt_f_xu_v_f32m8(v, vl); + acc = __riscv_vfmacc_tu(acc, vf, vf, vl); + x += vl; + } + vfloat32m1_t zero = __riscv_vfmv_s_f_f32m1(0, __riscv_vsetvlmax_e32m1()); + dstmat.ptr(y)[0] = __riscv_vfmv_f(__riscv_vfredusum(acc, zero, vlmax)); + } + }); + v_cleanup(); +} + +static void sum2_16s32fC1(const Mat& srcmat, Mat& dstmat) +{ + const int cols = srcmat.cols; + parallel_for_(Range(0, srcmat.rows), [&](const Range& range) { + for (int y = range.start; y < range.end; y++) + { + const short* src = srcmat.ptr(y); + const int vlmax = __riscv_vsetvlmax_e32m8(); + vfloat32m8_t acc = __riscv_vfmv_v_f_f32m8(0, vlmax); + int x = 0; + for (; x < cols; ) + { + const int vl = __riscv_vsetvl_e16m4(cols - x); + vint16m4_t v = __riscv_vle16_v_i16m4(src + x, vl); + vfloat32m8_t vf = __riscv_vfwcvt_f_x_v_f32m8(v, vl); + acc = __riscv_vfmacc_tu(acc, vf, vf, vl); + x += vl; + } + vfloat32m1_t zero = __riscv_vfmv_s_f_f32m1(0, __riscv_vsetvlmax_e32m1()); + dstmat.ptr(y)[0] = __riscv_vfmv_f(__riscv_vfredusum(acc, zero, vlmax)); + } + }); + v_cleanup(); +} + +#if CV_SIMD_SCALABLE_64F +static void sum2_16u64fC1(const Mat& srcmat, Mat& dstmat) +{ + const int cols = srcmat.cols; + parallel_for_(Range(0, srcmat.rows), [&](const Range& range) { + for (int y = range.start; y < range.end; y++) + { + const ushort* src = srcmat.ptr(y); + const int vlmax = __riscv_vsetvlmax_e64m8(); + vfloat64m8_t acc = __riscv_vfmv_v_f_f64m8(0, vlmax); + int x = 0; + for (; x < cols; ) + { + const int vl = __riscv_vsetvl_e16m2(cols - x); + vuint16m2_t v = __riscv_vle16_v_u16m2(src + x, vl); + vfloat32m4_t vf = __riscv_vfwcvt_f_xu_v_f32m4(v, vl); + vfloat64m8_t vd = __riscv_vfwcvt_f_f_v_f64m8(vf, vl); + acc = __riscv_vfmacc_tu(acc, vd, vd, vl); + x += vl; + } + vfloat64m1_t zero = __riscv_vfmv_s_f_f64m1(0, __riscv_vsetvlmax_e64m1()); + dstmat.ptr(y)[0] = __riscv_vfmv_f(__riscv_vfredusum(acc, zero, vlmax)); + } + }); + v_cleanup(); +} + +static void sum2_16s64fC1(const Mat& srcmat, Mat& dstmat) +{ + const int cols = srcmat.cols; + parallel_for_(Range(0, srcmat.rows), [&](const Range& range) { + for (int y = range.start; y < range.end; y++) + { + const short* src = srcmat.ptr(y); + const int vlmax = __riscv_vsetvlmax_e64m8(); + vfloat64m8_t acc = __riscv_vfmv_v_f_f64m8(0, vlmax); + int x = 0; + for (; x < cols; ) + { + const int vl = __riscv_vsetvl_e16m2(cols - x); + vint16m2_t v = __riscv_vle16_v_i16m2(src + x, vl); + vfloat32m4_t vf = __riscv_vfwcvt_f_x_v_f32m4(v, vl); + vfloat64m8_t vd = __riscv_vfwcvt_f_f_v_f64m8(vf, vl); + acc = __riscv_vfmacc_tu(acc, vd, vd, vl); + x += vl; + } + vfloat64m1_t zero = __riscv_vfmv_s_f_f64m1(0, __riscv_vsetvlmax_e64m1()); + dstmat.ptr(y)[0] = __riscv_vfmv_f(__riscv_vfredusum(acc, zero, vlmax)); + } + }); + v_cleanup(); +} + +static void sum2_32f64fC1(const Mat& srcmat, Mat& dstmat) +{ + const int cols = srcmat.cols; + parallel_for_(Range(0, srcmat.rows), [&](const Range& range) { + for (int y = range.start; y < range.end; y++) + { + const float* src = srcmat.ptr(y); + const int vlmax = __riscv_vsetvlmax_e64m8(); + vfloat64m8_t acc = __riscv_vfmv_v_f_f64m8(0, vlmax); + int x = 0; + for (; x < cols; ) + { + const int vl = __riscv_vsetvl_e32m4(cols - x); + vfloat32m4_t v = __riscv_vle32_v_f32m4(src + x, vl); + vfloat64m8_t vd = __riscv_vfwcvt_f_f_v_f64m8(v, vl); + acc = __riscv_vfmacc_tu(acc, vd, vd, vl); + x += vl; + } + vfloat64m1_t zero = __riscv_vfmv_s_f_f64m1(0, __riscv_vsetvlmax_e64m1()); + dstmat.ptr(y)[0] = __riscv_vfmv_f(__riscv_vfredusum(acc, zero, vlmax)); + } + }); + v_cleanup(); +} + +static void sum2_64f64fC1(const Mat& srcmat, Mat& dstmat) +{ + const int cols = srcmat.cols; + parallel_for_(Range(0, srcmat.rows), [&](const Range& range) { + for (int y = range.start; y < range.end; y++) + { + const double* src = srcmat.ptr(y); + const int vlmax = __riscv_vsetvlmax_e64m8(); + vfloat64m8_t acc = __riscv_vfmv_v_f_f64m8(0, vlmax); + int x = 0; + for (; x < cols; ) + { + const int vl = __riscv_vsetvl_e64m8(cols - x); + vfloat64m8_t v = __riscv_vle64_v_f64m8(src + x, vl); + acc = __riscv_vfmacc_tu(acc, v, v, vl); + x += vl; + } + vfloat64m1_t zero = __riscv_vfmv_s_f_f64m1(0, __riscv_vsetvlmax_e64m1()); + dstmat.ptr(y)[0] = __riscv_vfmv_f(__riscv_vfredusum(acc, zero, vlmax)); + } + }); + v_cleanup(); +} +#endif + +template +static void sum2_8uC3(const Mat& srcmat, Mat& dstmat) +{ + const int cols = srcmat.cols; + parallel_for_(Range(0, srcmat.rows), [&](const Range& range) { + for (int y = range.start; y < range.end; y++) + { + const uchar* src = srcmat.ptr(y); + DT* dst = dstmat.ptr
(y); + vuint32m1_t acc0 = __riscv_vmv_v_x_u32m1(0, __riscv_vsetvlmax_e32m1()); + vuint32m1_t acc1 = acc0, acc2 = acc0; + int x = 0; + for (; x < cols; ) + { + const int vl = __riscv_vsetvl_e8m1(cols - x); + vuint8m1x3_t v = __riscv_vlseg3e8_v_u8m1x3(src + x * 3, vl); + vuint8m1_t v0 = __riscv_vget_v_u8m1x3_u8m1(v, 0); + vuint8m1_t v1 = __riscv_vget_v_u8m1x3_u8m1(v, 1); + vuint8m1_t v2 = __riscv_vget_v_u8m1x3_u8m1(v, 2); + acc0 = __riscv_vwredsumu(__riscv_vwmulu(v0, v0, vl), acc0, vl); + acc1 = __riscv_vwredsumu(__riscv_vwmulu(v1, v1, vl), acc1, vl); + acc2 = __riscv_vwredsumu(__riscv_vwmulu(v2, v2, vl), acc2, vl); + x += vl; + } + dst[0] = (DT)(int32_t)(uint32_t)__riscv_vmv_x(acc0); + dst[1] = (DT)(int32_t)(uint32_t)__riscv_vmv_x(acc1); + dst[2] = (DT)(int32_t)(uint32_t)__riscv_vmv_x(acc2); + } + }); + v_cleanup(); +} + +template +static void sum2_8uC4(const Mat& srcmat, Mat& dstmat) +{ + const int cols = srcmat.cols; + parallel_for_(Range(0, srcmat.rows), [&](const Range& range) { + for (int y = range.start; y < range.end; y++) + { + const uchar* src = srcmat.ptr(y); + DT* dst = dstmat.ptr
(y); + vuint32m1_t acc0 = __riscv_vmv_v_x_u32m1(0, __riscv_vsetvlmax_e32m1()); + vuint32m1_t acc1 = acc0, acc2 = acc0, acc3 = acc0; + int x = 0; + for (; x < cols; ) + { + const int vl = __riscv_vsetvl_e8m1(cols - x); + vuint8m1x4_t v = __riscv_vlseg4e8_v_u8m1x4(src + x * 4, vl); + vuint8m1_t v0 = __riscv_vget_v_u8m1x4_u8m1(v, 0); + vuint8m1_t v1 = __riscv_vget_v_u8m1x4_u8m1(v, 1); + vuint8m1_t v2 = __riscv_vget_v_u8m1x4_u8m1(v, 2); + vuint8m1_t v3 = __riscv_vget_v_u8m1x4_u8m1(v, 3); + acc0 = __riscv_vwredsumu(__riscv_vwmulu(v0, v0, vl), acc0, vl); + acc1 = __riscv_vwredsumu(__riscv_vwmulu(v1, v1, vl), acc1, vl); + acc2 = __riscv_vwredsumu(__riscv_vwmulu(v2, v2, vl), acc2, vl); + acc3 = __riscv_vwredsumu(__riscv_vwmulu(v3, v3, vl), acc3, vl); + x += vl; + } + dst[0] = (DT)(int32_t)(uint32_t)__riscv_vmv_x(acc0); + dst[1] = (DT)(int32_t)(uint32_t)__riscv_vmv_x(acc1); + dst[2] = (DT)(int32_t)(uint32_t)__riscv_vmv_x(acc2); + dst[3] = (DT)(int32_t)(uint32_t)__riscv_vmv_x(acc3); + } + }); + v_cleanup(); +} + +static void sum2_32fC1(const Mat& srcmat, Mat& dstmat) +{ + const int cols = srcmat.cols; + parallel_for_(Range(0, srcmat.rows), [&](const Range& range) { + for (int y = range.start; y < range.end; y++) + { + const float* src = srcmat.ptr(y); + float result = 0; + int x = 0; + const int vlmax = __riscv_vsetvlmax_e32m8(); + vfloat32m8_t acc = __riscv_vfmv_v_f_f32m8(0, vlmax); + for (; x < cols; ) + { + const int vl = __riscv_vsetvl_e32m8(cols - x); + vfloat32m8_t v = __riscv_vle32_v_f32m8(src + x, vl); + acc = __riscv_vfmacc_tu(acc, v, v, vl); + x += vl; + } + vfloat32m1_t zero = __riscv_vfmv_s_f_f32m1(0, __riscv_vsetvlmax_e32m1()); + result = __riscv_vfmv_f(__riscv_vfredusum(acc, zero, vlmax)); + for (; x < cols; x++) + result += src[x] * src[x]; + dstmat.ptr(y)[0] = result; + } + }); + v_cleanup(); +} + +static void sum2_32fC3(const Mat& srcmat, Mat& dstmat) +{ + const int cols = srcmat.cols; + const int vlmax = __riscv_vsetvlmax_e32m2(); + parallel_for_(Range(0, srcmat.rows), [&](const Range& range) { + for (int y = range.start; y < range.end; y++) + { + const float* src = srcmat.ptr(y); + float* dst = dstmat.ptr(y); + vfloat32m2_t acc0 = __riscv_vfmv_v_f_f32m2(0, vlmax); + vfloat32m2_t acc1 = acc0, acc2 = acc0; + int x = 0; + for (; x < cols; ) + { + const int vl = __riscv_vsetvl_e32m2(cols - x); + vfloat32m2x3_t v = __riscv_vlseg3e32_v_f32m2x3(src + x * 3, vl); + vfloat32m2_t v0 = __riscv_vget_v_f32m2x3_f32m2(v, 0); + vfloat32m2_t v1 = __riscv_vget_v_f32m2x3_f32m2(v, 1); + vfloat32m2_t v2 = __riscv_vget_v_f32m2x3_f32m2(v, 2); + acc0 = __riscv_vfmacc_tu(acc0, v0, v0, vl); + acc1 = __riscv_vfmacc_tu(acc1, v1, v1, vl); + acc2 = __riscv_vfmacc_tu(acc2, v2, v2, vl); + x += vl; + } + vfloat32m1_t zero = __riscv_vfmv_s_f_f32m1(0, __riscv_vsetvlmax_e32m1()); + dst[0] = __riscv_vfmv_f(__riscv_vfredusum(acc0, zero, vlmax)); + dst[1] = __riscv_vfmv_f(__riscv_vfredusum(acc1, zero, vlmax)); + dst[2] = __riscv_vfmv_f(__riscv_vfredusum(acc2, zero, vlmax)); + } + }); + v_cleanup(); +} + +static void sum2_32fC4(const Mat& srcmat, Mat& dstmat) +{ + const int cols = srcmat.cols; + const int vlmax = __riscv_vsetvlmax_e32m2(); + parallel_for_(Range(0, srcmat.rows), [&](const Range& range) { + for (int y = range.start; y < range.end; y++) + { + const float* src = srcmat.ptr(y); + float* dst = dstmat.ptr(y); + vfloat32m2_t acc0 = __riscv_vfmv_v_f_f32m2(0, vlmax); + vfloat32m2_t acc1 = acc0, acc2 = acc0, acc3 = acc0; + int x = 0; + for (; x < cols; ) + { + const int vl = __riscv_vsetvl_e32m2(cols - x); + vfloat32m2x4_t v = __riscv_vlseg4e32_v_f32m2x4(src + x * 4, vl); + vfloat32m2_t v0 = __riscv_vget_v_f32m2x4_f32m2(v, 0); + vfloat32m2_t v1 = __riscv_vget_v_f32m2x4_f32m2(v, 1); + vfloat32m2_t v2 = __riscv_vget_v_f32m2x4_f32m2(v, 2); + vfloat32m2_t v3 = __riscv_vget_v_f32m2x4_f32m2(v, 3); + acc0 = __riscv_vfmacc_tu(acc0, v0, v0, vl); + acc1 = __riscv_vfmacc_tu(acc1, v1, v1, vl); + acc2 = __riscv_vfmacc_tu(acc2, v2, v2, vl); + acc3 = __riscv_vfmacc_tu(acc3, v3, v3, vl); + x += vl; + } + vfloat32m1_t zero = __riscv_vfmv_s_f_f32m1(0, __riscv_vsetvlmax_e32m1()); + dst[0] = __riscv_vfmv_f(__riscv_vfredusum(acc0, zero, vlmax)); + dst[1] = __riscv_vfmv_f(__riscv_vfredusum(acc1, zero, vlmax)); + dst[2] = __riscv_vfmv_f(__riscv_vfredusum(acc2, zero, vlmax)); + dst[3] = __riscv_vfmv_f(__riscv_vfredusum(acc3, zero, vlmax)); + } + }); + v_cleanup(); +} + +} // namespace reduce_c_rvv diff --git a/modules/core/test/test_mat.cpp b/modules/core/test/test_mat.cpp index dfd17db8d1..cc1ccde46b 100644 --- a/modules/core/test/test_mat.cpp +++ b/modules/core/test/test_mat.cpp @@ -170,7 +170,7 @@ int Core_ReduceTest::checkOp( const Mat& src, int dstType, int opType, const Mat getMatTypeStr( dstType, dstTypeStr ); const char* dimStr = dim == 0 ? "ROWS" : "COLS"; - snprintf( msg, sizeof(msg), "bad accuracy with srcType = %s, dstType = %s, opType = %s, dim = %s", + snprintf( msg, sizeof(msg), "bad accuracy with srcType = %s, dstType = %s, opType = %s, dim = %s\n", srcTypeStr.c_str(), dstTypeStr.c_str(), opTypeStr, dimStr ); ts->printf( cvtest::TS::LOG, msg ); return cvtest::TS::FAIL_BAD_ACCURACY;