mirror of
https://github.com/opencv/opencv.git
synced 2026-07-21 19:33:03 +04:00
Merge pull request #27510 from fengyuentau:4x/core/reduce_simd
core: vectorize cv::reduce #27510 - [ ] ~reduceR_~ Dropped due to performance - [x] reduceC_ ### Pull Request Readiness Checklist See details at https://github.com/opencv/opencv/wiki/How_to_contribute#making-a-good-pull-request - [x] I agree to contribute to the project under Apache 2 License. - [x] To the best of my knowledge, the proposed patch is not based on a code under GPL or another license that is incompatible with OpenCV - [x] The PR is proposed to the proper branch - [ ] There is a reference to the original bug report and related work - [ ] There is accuracy test, performance test and test data in opencv_extra repository, if applicable Patch to opencv_extra has the same branch name. - [ ] The feature is well documented and sample code can be built with the project CMake
This commit is contained in:
Yuantao Feng
GitHub
@@ -939,7 +939,7 @@ inline scalartype v_reduce_sum(const _Tpvec& a) \
|
||||
}
|
||||
OPENCV_HAL_IMPL_RVV_REDUCE_SUM_FP(v_float32, v_float32, vfloat32m1_t, float, f32, VTraits<v_float32>::vlanes())
|
||||
#if CV_SIMD_SCALABLE_64F
|
||||
OPENCV_HAL_IMPL_RVV_REDUCE_SUM_FP(v_float64, v_float64, vfloat64m1_t, float, f64, VTraits<v_float64>::vlanes())
|
||||
OPENCV_HAL_IMPL_RVV_REDUCE_SUM_FP(v_float64, v_float64, vfloat64m1_t, double, f64, VTraits<v_float64>::vlanes())
|
||||
#endif
|
||||
|
||||
#define OPENCV_HAL_IMPL_RVV_REDUCE(_Tpvec, _nTpvec, func, scalartype, suffix, vl, red) \
|
||||
|
||||
@@ -1202,6 +1202,9 @@ typedef TestBaseWithParam<ReduceMinMaxParams> ReduceMinMaxFixture;
|
||||
OCL_PERF_TEST_P(ReduceMinMaxFixture, Reduce,
|
||||
::testing::Combine(OCL_TEST_SIZES,
|
||||
OCL_PERF_ENUM(std::make_pair<MatType, MatType>(CV_8UC1, CV_8UC1),
|
||||
std::make_pair<MatType, MatType>(CV_8UC4, CV_8UC4),
|
||||
std::make_pair<MatType, MatType>(CV_32FC1, CV_32FC1),
|
||||
std::make_pair<MatType, MatType>(CV_32FC3, CV_32FC3),
|
||||
std::make_pair<MatType, MatType>(CV_32FC4, CV_32FC4)),
|
||||
OCL_PERF_ENUM(0, 1),
|
||||
ReduceMinMaxOp::all()))
|
||||
|
||||
@@ -338,9 +338,13 @@ cv::Mat cv::Mat::cross(InputArray _m) const
|
||||
namespace cv
|
||||
{
|
||||
|
||||
typedef void (*ReduceSumFunc)(const Mat& src, Mat& dst);
|
||||
ReduceSumFunc getReduceCSumFunc(int sdepth, int ddepth);
|
||||
ReduceSumFunc getReduceRSumFunc(int sdepth, int ddepth);
|
||||
typedef void (*ReduceFunc)( const Mat& src, Mat& dst );
|
||||
ReduceFunc getReduceCSumFunc(int sdepth, int ddepth);
|
||||
ReduceFunc getReduceCAvgFunc(int sdepth, int ddepth);
|
||||
ReduceFunc getReduceCMaxFunc(int sdepth, int ddepth);
|
||||
ReduceFunc getReduceCMinFunc(int sdepth, int ddepth);
|
||||
ReduceFunc getReduceCSum2Func(int sdepth, int ddepth);
|
||||
ReduceFunc getReduceRSumFunc(int sdepth, int ddepth);
|
||||
|
||||
template <typename T, typename WT, typename Op>
|
||||
struct ReduceR_SIMD
|
||||
@@ -351,7 +355,6 @@ struct ReduceR_SIMD
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
template<typename T, typename ST, typename WT, class Op, class OpInit>
|
||||
class ReduceR_Invoker : public ParallelLoopBody
|
||||
{
|
||||
@@ -471,8 +474,6 @@ reduceC_( const Mat& srcmat, Mat& dstmat)
|
||||
parallel_for_(Range(0, srcmat.size().height), body);
|
||||
}
|
||||
|
||||
typedef void (*ReduceFunc)( const Mat& src, Mat& dst );
|
||||
|
||||
}
|
||||
|
||||
#define reduceSumR8u32s reduceR_<uchar, int, OpAdd<int>, OpNop<int> >
|
||||
@@ -818,9 +819,9 @@ void cv::reduce(InputArray _src, OutputArray _dst, int dim, int op, int dtype)
|
||||
{
|
||||
if( op == REDUCE_SUM )
|
||||
{
|
||||
ReduceSumFunc simd_func = getReduceRSumFunc(sdepth, ddepth);
|
||||
ReduceFunc simd_func = getReduceRSumFunc(sdepth, ddepth);
|
||||
if(simd_func)
|
||||
func = (ReduceFunc)simd_func;
|
||||
func = simd_func;
|
||||
else if(sdepth == CV_8U && ddepth == CV_32S)
|
||||
func = reduceSumR8u32s;
|
||||
else if(sdepth == CV_8U && ddepth == CV_32F)
|
||||
@@ -896,9 +897,11 @@ void cv::reduce(InputArray _src, OutputArray _dst, int dim, int op, int dtype)
|
||||
{
|
||||
if(op == REDUCE_SUM)
|
||||
{
|
||||
ReduceSumFunc simd_func = getReduceCSumFunc(sdepth, ddepth);
|
||||
ReduceFunc simd_func = op0 == REDUCE_AVG
|
||||
? getReduceCAvgFunc(sdepth, ddepth)
|
||||
: getReduceCSumFunc(sdepth, ddepth);
|
||||
if(simd_func)
|
||||
func = (ReduceFunc)simd_func;
|
||||
func = simd_func;
|
||||
else if(sdepth == CV_8U && ddepth == CV_32S)
|
||||
func = reduceSumC8u32s;
|
||||
else if(sdepth == CV_8U && ddepth == CV_32F)
|
||||
@@ -922,7 +925,10 @@ void cv::reduce(InputArray _src, OutputArray _dst, int dim, int op, int dtype)
|
||||
}
|
||||
else if(op == REDUCE_MAX)
|
||||
{
|
||||
if(sdepth == CV_8U && ddepth == CV_8U)
|
||||
ReduceFunc simd_func = getReduceCMaxFunc(sdepth, ddepth);
|
||||
if(simd_func)
|
||||
func = simd_func;
|
||||
else if(sdepth == CV_8U && ddepth == CV_8U)
|
||||
func = reduceMaxC8u;
|
||||
else if(sdepth == CV_16U && ddepth == CV_16U)
|
||||
func = reduceMaxC16u;
|
||||
@@ -935,7 +941,10 @@ void cv::reduce(InputArray _src, OutputArray _dst, int dim, int op, int dtype)
|
||||
}
|
||||
else if(op == REDUCE_MIN)
|
||||
{
|
||||
if(sdepth == CV_8U && ddepth == CV_8U)
|
||||
ReduceFunc simd_func = getReduceCMinFunc(sdepth, ddepth);
|
||||
if(simd_func)
|
||||
func = simd_func;
|
||||
else if(sdepth == CV_8U && ddepth == CV_8U)
|
||||
func = reduceMinC8u;
|
||||
else if(sdepth == CV_16U && ddepth == CV_16U)
|
||||
func = reduceMinC16u;
|
||||
@@ -948,7 +957,10 @@ void cv::reduce(InputArray _src, OutputArray _dst, int dim, int op, int dtype)
|
||||
}
|
||||
else if(op == REDUCE_SUM2)
|
||||
{
|
||||
if(sdepth == CV_8U && ddepth == CV_32S)
|
||||
ReduceFunc simd_func = getReduceCSum2Func(sdepth, ddepth);
|
||||
if(simd_func)
|
||||
func = simd_func;
|
||||
else if(sdepth == CV_8U && ddepth == CV_32S)
|
||||
func = reduceSum2C8u32s;
|
||||
else if(sdepth == CV_8U && ddepth == CV_32F)
|
||||
func = reduceSum2C8u32f;
|
||||
@@ -1038,7 +1050,6 @@ template<typename T> static void sort_( const Mat& src, Mat& dst, int flags )
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
#if defined(HAVE_IPP) && !IPP_DISABLE_SORT
|
||||
typedef IppStatus (CV_STDCALL *IppSortFunc)(void *pSrcDst, int len, Ipp8u *pBuffer);
|
||||
|
||||
|
||||
@@ -9,18 +9,50 @@
|
||||
|
||||
namespace cv {
|
||||
|
||||
typedef void (*ReduceSumFunc)(const Mat& src, Mat& dst);
|
||||
ReduceSumFunc getReduceCSumFunc(int sdepth, int ddepth);
|
||||
ReduceSumFunc getReduceRSumFunc(int sdepth, int ddepth);
|
||||
typedef void (*ReduceFunc)( const Mat& src, Mat& dst );
|
||||
ReduceFunc getReduceCSumFunc(int sdepth, int ddepth);
|
||||
ReduceFunc getReduceCAvgFunc(int sdepth, int ddepth);
|
||||
ReduceFunc getReduceCMaxFunc(int sdepth, int ddepth);
|
||||
ReduceFunc getReduceCMinFunc(int sdepth, int ddepth);
|
||||
ReduceFunc getReduceCSum2Func(int sdepth, int ddepth);
|
||||
ReduceFunc getReduceRSumFunc(int sdepth, int ddepth);
|
||||
|
||||
ReduceSumFunc getReduceCSumFunc(int sdepth, int ddepth)
|
||||
ReduceFunc getReduceCSumFunc(int sdepth, int ddepth)
|
||||
{
|
||||
CV_INSTRUMENT_REGION();
|
||||
CV_CPU_DISPATCH(getReduceCSumFunc, (sdepth, ddepth),
|
||||
CV_CPU_DISPATCH_MODES_ALL);
|
||||
}
|
||||
|
||||
ReduceSumFunc getReduceRSumFunc(int sdepth, int ddepth)
|
||||
ReduceFunc getReduceCAvgFunc(int sdepth, int ddepth)
|
||||
{
|
||||
CV_INSTRUMENT_REGION();
|
||||
CV_CPU_DISPATCH(getReduceCAvgFunc, (sdepth, ddepth),
|
||||
CV_CPU_DISPATCH_MODES_ALL);
|
||||
}
|
||||
|
||||
ReduceFunc getReduceCMaxFunc(int sdepth, int ddepth)
|
||||
{
|
||||
CV_INSTRUMENT_REGION();
|
||||
CV_CPU_DISPATCH(getReduceCMaxFunc, (sdepth, ddepth),
|
||||
CV_CPU_DISPATCH_MODES_ALL);
|
||||
}
|
||||
|
||||
ReduceFunc getReduceCMinFunc(int sdepth, int ddepth)
|
||||
{
|
||||
CV_INSTRUMENT_REGION();
|
||||
CV_CPU_DISPATCH(getReduceCMinFunc, (sdepth, ddepth),
|
||||
CV_CPU_DISPATCH_MODES_ALL);
|
||||
}
|
||||
|
||||
ReduceFunc getReduceCSum2Func(int sdepth, int ddepth)
|
||||
{
|
||||
CV_INSTRUMENT_REGION();
|
||||
CV_CPU_DISPATCH(getReduceCSum2Func, (sdepth, ddepth),
|
||||
CV_CPU_DISPATCH_MODES_ALL);
|
||||
}
|
||||
|
||||
ReduceFunc getReduceRSumFunc(int sdepth, int ddepth)
|
||||
{
|
||||
CV_INSTRUMENT_REGION();
|
||||
CV_CPU_DISPATCH(getReduceRSumFunc, (sdepth, ddepth),
|
||||
|
||||
@@ -7,12 +7,20 @@
|
||||
namespace cv {
|
||||
CV_CPU_OPTIMIZATION_NAMESPACE_BEGIN
|
||||
|
||||
typedef void (*ReduceSumFunc)(const Mat& src, Mat& dst);
|
||||
ReduceSumFunc getReduceCSumFunc(int sdepth, int ddepth);
|
||||
ReduceSumFunc getReduceRSumFunc(int sdepth, int ddepth);
|
||||
typedef void (*ReduceFunc)( const Mat& src, Mat& dst );
|
||||
ReduceFunc getReduceCSumFunc(int sdepth, int ddepth);
|
||||
ReduceFunc getReduceCAvgFunc(int sdepth, int ddepth);
|
||||
ReduceFunc getReduceCMaxFunc(int sdepth, int ddepth);
|
||||
ReduceFunc getReduceCMinFunc(int sdepth, int ddepth);
|
||||
ReduceFunc getReduceCSum2Func(int sdepth, int ddepth);
|
||||
ReduceFunc getReduceRSumFunc(int sdepth, int ddepth);
|
||||
|
||||
#ifndef CV_CPU_OPTIMIZATION_DECLARATIONS_ONLY
|
||||
|
||||
#if (CV_SIMD || CV_SIMD_SCALABLE)
|
||||
#include "reduce_c.simd.hpp"
|
||||
#endif
|
||||
|
||||
// =====================================================================
|
||||
// Col reduce SUM (dim=1): sum each row into cn output values
|
||||
// =====================================================================
|
||||
@@ -1089,7 +1097,7 @@ static void reduceRowSum_64f64f(const Mat& srcmat, Mat& dstmat)
|
||||
// Dispatchers
|
||||
// =====================================================================
|
||||
|
||||
ReduceSumFunc getReduceCSumFunc(int sdepth, int ddepth)
|
||||
ReduceFunc getReduceCSumFunc(int sdepth, int ddepth)
|
||||
{
|
||||
#if (CV_SIMD || CV_SIMD_SCALABLE)
|
||||
if (sdepth == CV_8U && ddepth == CV_32S) return reduceColSum_8u32s;
|
||||
@@ -1108,7 +1116,68 @@ ReduceSumFunc getReduceCSumFunc(int sdepth, int ddepth)
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
ReduceSumFunc getReduceRSumFunc(int sdepth, int ddepth)
|
||||
ReduceFunc getReduceCAvgFunc(int sdepth, int ddepth)
|
||||
{
|
||||
return getReduceCSumFunc(sdepth, ddepth);
|
||||
}
|
||||
|
||||
ReduceFunc getReduceCMaxFunc(int sdepth, int ddepth)
|
||||
{
|
||||
#if (CV_SIMD || CV_SIMD_SCALABLE)
|
||||
if (sdepth == CV_8U && ddepth == CV_8U) return reduceColMax_8u;
|
||||
if (sdepth == CV_16U && ddepth == CV_16U) return reduceColMax_16u;
|
||||
if (sdepth == CV_16S && ddepth == CV_16S) return reduceColMax_16s;
|
||||
if (sdepth == CV_32F && ddepth == CV_32F) return reduceColMax_32f;
|
||||
#if (CV_SIMD_64F || CV_SIMD_SCALABLE_64F)
|
||||
if (sdepth == CV_64F && ddepth == CV_64F) return reduceColMax_64f;
|
||||
#endif
|
||||
#else
|
||||
CV_UNUSED(sdepth);
|
||||
CV_UNUSED(ddepth);
|
||||
#endif
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
ReduceFunc getReduceCMinFunc(int sdepth, int ddepth)
|
||||
{
|
||||
#if (CV_SIMD || CV_SIMD_SCALABLE)
|
||||
if (sdepth == CV_8U && ddepth == CV_8U) return reduceColMin_8u;
|
||||
if (sdepth == CV_16U && ddepth == CV_16U) return reduceColMin_16u;
|
||||
if (sdepth == CV_16S && ddepth == CV_16S) return reduceColMin_16s;
|
||||
if (sdepth == CV_32F && ddepth == CV_32F) return reduceColMin_32f;
|
||||
#if (CV_SIMD_64F || CV_SIMD_SCALABLE_64F)
|
||||
if (sdepth == CV_64F && ddepth == CV_64F) return reduceColMin_64f;
|
||||
#endif
|
||||
#else
|
||||
CV_UNUSED(sdepth);
|
||||
CV_UNUSED(ddepth);
|
||||
#endif
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
ReduceFunc getReduceCSum2Func(int sdepth, int ddepth)
|
||||
{
|
||||
#if (CV_SIMD || CV_SIMD_SCALABLE)
|
||||
if (sdepth == CV_8U && ddepth == CV_32S) return reduceColSum2_8u32s;
|
||||
if (sdepth == CV_8U && ddepth == CV_32F) return reduceColSum2_8u32f;
|
||||
if (sdepth == CV_8U && ddepth == CV_64F) return reduceColSum2_8u64f;
|
||||
if (sdepth == CV_16U && ddepth == CV_32F) return reduceColSum2_16u32f;
|
||||
if (sdepth == CV_16S && ddepth == CV_32F) return reduceColSum2_16s32f;
|
||||
if (sdepth == CV_32F && ddepth == CV_32F) return reduceColSum2_32f32f;
|
||||
#if (CV_SIMD_64F || CV_SIMD_SCALABLE_64F)
|
||||
if (sdepth == CV_16U && ddepth == CV_64F) return reduceColSum2_16u64f;
|
||||
if (sdepth == CV_16S && ddepth == CV_64F) return reduceColSum2_16s64f;
|
||||
if (sdepth == CV_32F && ddepth == CV_64F) return reduceColSum2_32f64f;
|
||||
if (sdepth == CV_64F && ddepth == CV_64F) return reduceColSum2_64f64f;
|
||||
#endif
|
||||
#else
|
||||
CV_UNUSED(sdepth);
|
||||
CV_UNUSED(ddepth);
|
||||
#endif
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
ReduceFunc getReduceRSumFunc(int sdepth, int ddepth)
|
||||
{
|
||||
#if (CV_SIMD || CV_SIMD_SCALABLE)
|
||||
if (sdepth == CV_8U && ddepth == CV_32S) return reduceRowSum_8u32s;
|
||||
|
||||
@@ -0,0 +1,539 @@
|
||||
// This file is part of OpenCV project.
|
||||
// It is subject to the license terms in the LICENSE file found in the top-level directory
|
||||
// of this distribution and at http://opencv.org/license.html
|
||||
|
||||
#include "reduce_c_generic.hpp"
|
||||
|
||||
#if CV_RVV
|
||||
#include "reduce_c_rvv.hpp"
|
||||
#endif
|
||||
|
||||
#if CV_NEON
|
||||
#include "reduce_c_neon.hpp"
|
||||
#endif
|
||||
|
||||
#if CV_AVX2
|
||||
#include "reduce_c_avx2.hpp"
|
||||
#endif
|
||||
|
||||
template<bool isMax>
|
||||
static void reduceColMinMax_8uC1(const Mat& srcmat, Mat& dstmat)
|
||||
{
|
||||
#if CV_RVV
|
||||
reduce_c_rvv::minMax8uC1<isMax>(srcmat, dstmat);
|
||||
#elif CV_NEON
|
||||
reduce_c_neon::minMax8uC1<isMax>(srcmat, dstmat);
|
||||
#elif CV_AVX2
|
||||
reduce_c_avx2::minMax8uC1<isMax>(srcmat, dstmat);
|
||||
#else
|
||||
reduceColMinMax_8uFallback<isMax>(srcmat, dstmat);
|
||||
#endif
|
||||
}
|
||||
|
||||
template<bool isMax>
|
||||
static void reduceColMinMax_8uC3(const Mat& srcmat, Mat& dstmat)
|
||||
{
|
||||
#if CV_RVV
|
||||
reduce_c_rvv::minMax8uC3<isMax>(srcmat, dstmat);
|
||||
#elif CV_NEON
|
||||
reduce_c_neon::minMax8uC3<isMax>(srcmat, dstmat);
|
||||
#elif CV_AVX2
|
||||
reduce_c_avx2::minMax8uC3<isMax>(srcmat, dstmat);
|
||||
#else
|
||||
reduceColMinMax_8uFallback<isMax>(srcmat, dstmat);
|
||||
#endif
|
||||
}
|
||||
|
||||
template<bool isMax>
|
||||
static void reduceColMinMax_8uC4(const Mat& srcmat, Mat& dstmat)
|
||||
{
|
||||
#if CV_RVV
|
||||
reduce_c_rvv::minMax8uC4<isMax>(srcmat, dstmat);
|
||||
#elif CV_NEON
|
||||
reduce_c_neon::minMax8uC4<isMax>(srcmat, dstmat);
|
||||
#elif CV_AVX2
|
||||
reduce_c_avx2::minMax8uC4<isMax>(srcmat, dstmat);
|
||||
#else
|
||||
reduceColMinMax_8uFallback<isMax>(srcmat, dstmat);
|
||||
#endif
|
||||
}
|
||||
|
||||
template<bool isMax>
|
||||
static void reduceColMinMax_8u(const Mat& srcmat, Mat& dstmat)
|
||||
{
|
||||
const int cn = srcmat.channels();
|
||||
if (cn == 1)
|
||||
reduceColMinMax_8uC1<isMax>(srcmat, dstmat);
|
||||
else if (cn == 3)
|
||||
reduceColMinMax_8uC3<isMax>(srcmat, dstmat);
|
||||
else if (cn == 4)
|
||||
reduceColMinMax_8uC4<isMax>(srcmat, dstmat);
|
||||
else
|
||||
reduceColMinMax_8uFallback<isMax>(srcmat, dstmat);
|
||||
}
|
||||
|
||||
static void reduceColMax_8u(const Mat& srcmat, Mat& dstmat)
|
||||
{
|
||||
reduceColMinMax_8u<true>(srcmat, dstmat);
|
||||
}
|
||||
|
||||
static void reduceColMin_8u(const Mat& srcmat, Mat& dstmat)
|
||||
{
|
||||
reduceColMinMax_8u<false>(srcmat, dstmat);
|
||||
}
|
||||
|
||||
template<bool isMax>
|
||||
static void reduceColMinMax_16uC1(const Mat& srcmat, Mat& dstmat)
|
||||
{
|
||||
#if CV_RVV
|
||||
reduce_c_rvv::minMax16uC1<isMax>(srcmat, dstmat);
|
||||
#elif CV_NEON
|
||||
reduce_c_neon::minMax16uC1<isMax>(srcmat, dstmat);
|
||||
#elif CV_AVX2
|
||||
reduce_c_avx2::minMax16uC1<isMax>(srcmat, dstmat);
|
||||
#else
|
||||
reduceColMinMax_16uFallback<isMax>(srcmat, dstmat);
|
||||
#endif
|
||||
}
|
||||
|
||||
template<bool isMax>
|
||||
static void reduceColMinMax_16uC4(const Mat& srcmat, Mat& dstmat)
|
||||
{
|
||||
#if CV_RVV
|
||||
reduce_c_rvv::minMax16uC4<isMax>(srcmat, dstmat);
|
||||
#elif CV_NEON
|
||||
reduce_c_neon::minMax16uC4<isMax>(srcmat, dstmat);
|
||||
#elif CV_AVX2
|
||||
reduce_c_avx2::minMax16uC4<isMax>(srcmat, dstmat);
|
||||
#else
|
||||
reduceColMinMax_16uFallback<isMax>(srcmat, dstmat);
|
||||
#endif
|
||||
}
|
||||
|
||||
template<bool isMax>
|
||||
static void reduceColMinMax_16uC3(const Mat& srcmat, Mat& dstmat)
|
||||
{
|
||||
#if CV_RVV
|
||||
reduce_c_rvv::minMax16uC3<isMax>(srcmat, dstmat);
|
||||
#elif CV_NEON
|
||||
reduce_c_neon::minMax16uC3<isMax>(srcmat, dstmat);
|
||||
#elif CV_AVX2
|
||||
reduce_c_avx2::minMax16uC3<isMax>(srcmat, dstmat);
|
||||
#else
|
||||
reduceColMinMax_16uFallback<isMax>(srcmat, dstmat);
|
||||
#endif
|
||||
}
|
||||
|
||||
template<bool isMax>
|
||||
static void reduceColMinMax_16u(const Mat& srcmat, Mat& dstmat)
|
||||
{
|
||||
if (srcmat.channels() == 1)
|
||||
reduceColMinMax_16uC1<isMax>(srcmat, dstmat);
|
||||
else if (srcmat.channels() == 3)
|
||||
reduceColMinMax_16uC3<isMax>(srcmat, dstmat);
|
||||
else if (srcmat.channels() == 4)
|
||||
reduceColMinMax_16uC4<isMax>(srcmat, dstmat);
|
||||
else
|
||||
reduceColMinMax_16uFallback<isMax>(srcmat, dstmat);
|
||||
}
|
||||
|
||||
static void reduceColMax_16u(const Mat& srcmat, Mat& dstmat)
|
||||
{
|
||||
reduceColMinMax_16u<true>(srcmat, dstmat);
|
||||
}
|
||||
|
||||
static void reduceColMin_16u(const Mat& srcmat, Mat& dstmat)
|
||||
{
|
||||
reduceColMinMax_16u<false>(srcmat, dstmat);
|
||||
}
|
||||
|
||||
template<bool isMax>
|
||||
static void reduceColMinMax_16sC1(const Mat& srcmat, Mat& dstmat)
|
||||
{
|
||||
#if CV_RVV
|
||||
reduce_c_rvv::minMax16sC1<isMax>(srcmat, dstmat);
|
||||
#elif CV_NEON
|
||||
reduce_c_neon::minMax16sC1<isMax>(srcmat, dstmat);
|
||||
#elif CV_AVX2
|
||||
reduce_c_avx2::minMax16sC1<isMax>(srcmat, dstmat);
|
||||
#else
|
||||
reduceColMinMax_16sFallback<isMax>(srcmat, dstmat);
|
||||
#endif
|
||||
}
|
||||
|
||||
template<bool isMax>
|
||||
static void reduceColMinMax_16sC4(const Mat& srcmat, Mat& dstmat)
|
||||
{
|
||||
#if CV_RVV
|
||||
reduce_c_rvv::minMax16sC4<isMax>(srcmat, dstmat);
|
||||
#elif CV_NEON
|
||||
reduce_c_neon::minMax16sC4<isMax>(srcmat, dstmat);
|
||||
#elif CV_AVX2
|
||||
reduce_c_avx2::minMax16sC4<isMax>(srcmat, dstmat);
|
||||
#else
|
||||
reduceColMinMax_16sFallback<isMax>(srcmat, dstmat);
|
||||
#endif
|
||||
}
|
||||
|
||||
template<bool isMax>
|
||||
static void reduceColMinMax_16sC3(const Mat& srcmat, Mat& dstmat)
|
||||
{
|
||||
#if CV_RVV
|
||||
reduce_c_rvv::minMax16sC3<isMax>(srcmat, dstmat);
|
||||
#elif CV_NEON
|
||||
reduce_c_neon::minMax16sC3<isMax>(srcmat, dstmat);
|
||||
#elif CV_AVX2
|
||||
reduce_c_avx2::minMax16sC3<isMax>(srcmat, dstmat);
|
||||
#else
|
||||
reduceColMinMax_16sFallback<isMax>(srcmat, dstmat);
|
||||
#endif
|
||||
}
|
||||
|
||||
template<bool isMax>
|
||||
static void reduceColMinMax_16s(const Mat& srcmat, Mat& dstmat)
|
||||
{
|
||||
if (srcmat.channels() == 1)
|
||||
reduceColMinMax_16sC1<isMax>(srcmat, dstmat);
|
||||
else if (srcmat.channels() == 3)
|
||||
reduceColMinMax_16sC3<isMax>(srcmat, dstmat);
|
||||
else if (srcmat.channels() == 4)
|
||||
reduceColMinMax_16sC4<isMax>(srcmat, dstmat);
|
||||
else
|
||||
reduceColMinMax_16sFallback<isMax>(srcmat, dstmat);
|
||||
}
|
||||
|
||||
static void reduceColMax_16s(const Mat& srcmat, Mat& dstmat)
|
||||
{
|
||||
reduceColMinMax_16s<true>(srcmat, dstmat);
|
||||
}
|
||||
|
||||
static void reduceColMin_16s(const Mat& srcmat, Mat& dstmat)
|
||||
{
|
||||
reduceColMinMax_16s<false>(srcmat, dstmat);
|
||||
}
|
||||
|
||||
template<bool isMax>
|
||||
static void reduceColMinMax_32fC1(const Mat& srcmat, Mat& dstmat)
|
||||
{
|
||||
#if CV_RVV
|
||||
reduce_c_rvv::minMax32fC1<isMax>(srcmat, dstmat);
|
||||
#elif CV_NEON
|
||||
reduce_c_neon::minMax32fC1<isMax>(srcmat, dstmat);
|
||||
#elif CV_AVX2
|
||||
reduce_c_avx2::minMax32fC1<isMax>(srcmat, dstmat);
|
||||
#else
|
||||
reduceColMinMax_32fFallback<isMax>(srcmat, dstmat);
|
||||
#endif
|
||||
}
|
||||
|
||||
template<bool isMax>
|
||||
static void reduceColMinMax_32fC3(const Mat& srcmat, Mat& dstmat)
|
||||
{
|
||||
#if CV_RVV
|
||||
reduce_c_rvv::minMax32fC3<isMax>(srcmat, dstmat);
|
||||
#else
|
||||
reduceColMinMax_32fFallback<isMax>(srcmat, dstmat);
|
||||
#endif
|
||||
}
|
||||
|
||||
template<bool isMax>
|
||||
static void reduceColMinMax_32fC4(const Mat& srcmat, Mat& dstmat)
|
||||
{
|
||||
#if CV_RVV
|
||||
reduce_c_rvv::minMax32fC4<isMax>(srcmat, dstmat);
|
||||
#elif CV_NEON
|
||||
reduce_c_neon::minMax32fC4<isMax>(srcmat, dstmat);
|
||||
#elif CV_AVX2
|
||||
reduce_c_avx2::minMax32fC4<isMax>(srcmat, dstmat);
|
||||
#else
|
||||
reduceColMinMax_32fFallback<isMax>(srcmat, dstmat);
|
||||
#endif
|
||||
}
|
||||
|
||||
template<bool isMax>
|
||||
static void reduceColMinMax_32f(const Mat& srcmat, Mat& dstmat)
|
||||
{
|
||||
const int cn = srcmat.channels();
|
||||
if (cn == 1)
|
||||
reduceColMinMax_32fC1<isMax>(srcmat, dstmat);
|
||||
else if (cn == 3)
|
||||
reduceColMinMax_32fC3<isMax>(srcmat, dstmat);
|
||||
else if (cn == 4)
|
||||
reduceColMinMax_32fC4<isMax>(srcmat, dstmat);
|
||||
else
|
||||
reduceColMinMax_32fFallback<isMax>(srcmat, dstmat);
|
||||
}
|
||||
|
||||
static void reduceColMax_32f(const Mat& srcmat, Mat& dstmat)
|
||||
{
|
||||
reduceColMinMax_32f<true>(srcmat, dstmat);
|
||||
}
|
||||
|
||||
static void reduceColMin_32f(const Mat& srcmat, Mat& dstmat)
|
||||
{
|
||||
reduceColMinMax_32f<false>(srcmat, dstmat);
|
||||
}
|
||||
|
||||
#if (CV_SIMD_64F || CV_SIMD_SCALABLE_64F)
|
||||
template<bool isMax>
|
||||
static void reduceColMinMax_64fC1(const Mat& srcmat, Mat& dstmat)
|
||||
{
|
||||
#if CV_RVV && CV_SIMD_SCALABLE_64F
|
||||
reduce_c_rvv::minMax64fC1<isMax>(srcmat, dstmat);
|
||||
#elif CV_NEON && (defined(__aarch64__) || defined(_M_ARM64))
|
||||
reduce_c_neon::minMax64fC1<isMax>(srcmat, dstmat);
|
||||
#elif CV_AVX2
|
||||
reduce_c_avx2::minMax64fC1<isMax>(srcmat, dstmat);
|
||||
#else
|
||||
reduceColMinMax_64fFallback<isMax>(srcmat, dstmat);
|
||||
#endif
|
||||
}
|
||||
|
||||
template<bool isMax>
|
||||
static void reduceColMinMax_64f(const Mat& srcmat, Mat& dstmat)
|
||||
{
|
||||
if (srcmat.channels() == 1)
|
||||
reduceColMinMax_64fC1<isMax>(srcmat, dstmat);
|
||||
else
|
||||
reduceColMinMax_64fFallback<isMax>(srcmat, dstmat);
|
||||
}
|
||||
|
||||
static void reduceColMax_64f(const Mat& srcmat, Mat& dstmat)
|
||||
{
|
||||
reduceColMinMax_64f<true>(srcmat, dstmat);
|
||||
}
|
||||
|
||||
static void reduceColMin_64f(const Mat& srcmat, Mat& dstmat)
|
||||
{
|
||||
reduceColMinMax_64f<false>(srcmat, dstmat);
|
||||
}
|
||||
#endif
|
||||
|
||||
template<typename DT>
|
||||
static void reduceColSum2_8uC1(const Mat& srcmat, Mat& dstmat)
|
||||
{
|
||||
#if CV_RVV
|
||||
reduce_c_rvv::sum2_8uC1<DT>(srcmat, dstmat);
|
||||
#elif CV_NEON && (defined(__aarch64__) || defined(_M_ARM64))
|
||||
reduce_c_neon::sum2_8uC1<DT>(srcmat, dstmat);
|
||||
#elif CV_AVX2
|
||||
reduce_c_avx2::sum2_8uC1<DT>(srcmat, dstmat);
|
||||
#else
|
||||
reduceColSum2_8uFallback<DT>(srcmat, dstmat);
|
||||
#endif
|
||||
}
|
||||
|
||||
template<typename DT>
|
||||
static void reduceColSum2_8uC3(const Mat& srcmat, Mat& dstmat)
|
||||
{
|
||||
#if CV_RVV
|
||||
reduce_c_rvv::sum2_8uC3<DT>(srcmat, dstmat);
|
||||
#elif CV_NEON && (defined(__aarch64__) || defined(_M_ARM64))
|
||||
reduce_c_neon::sum2_8uC3<DT>(srcmat, dstmat);
|
||||
#elif CV_AVX2
|
||||
reduce_c_avx2::sum2_8uC3<DT>(srcmat, dstmat);
|
||||
#else
|
||||
reduceColSum2_8uFallback<DT>(srcmat, dstmat);
|
||||
#endif
|
||||
}
|
||||
|
||||
template<typename DT>
|
||||
static void reduceColSum2_8uC4(const Mat& srcmat, Mat& dstmat)
|
||||
{
|
||||
#if CV_RVV
|
||||
reduce_c_rvv::sum2_8uC4<DT>(srcmat, dstmat);
|
||||
#elif CV_NEON && (defined(__aarch64__) || defined(_M_ARM64))
|
||||
reduce_c_neon::sum2_8uC4<DT>(srcmat, dstmat);
|
||||
#elif CV_AVX2
|
||||
reduce_c_avx2::sum2_8uC4<DT>(srcmat, dstmat);
|
||||
#else
|
||||
reduceColSum2_8uFallback<DT>(srcmat, dstmat);
|
||||
#endif
|
||||
}
|
||||
|
||||
template<typename DT>
|
||||
static void reduceColSum2_8u(const Mat& srcmat, Mat& dstmat)
|
||||
{
|
||||
const int cn = srcmat.channels();
|
||||
if (cn == 1)
|
||||
reduceColSum2_8uC1<DT>(srcmat, dstmat);
|
||||
else if (cn == 3)
|
||||
reduceColSum2_8uC3<DT>(srcmat, dstmat);
|
||||
else if (cn == 4)
|
||||
reduceColSum2_8uC4<DT>(srcmat, dstmat);
|
||||
else
|
||||
reduceColSum2_8uFallback<DT>(srcmat, dstmat);
|
||||
}
|
||||
|
||||
static void reduceColSum2_8u32s(const Mat& srcmat, Mat& dstmat)
|
||||
{
|
||||
reduceColSum2_8u<int>(srcmat, dstmat);
|
||||
}
|
||||
|
||||
static void reduceColSum2_8u32f(const Mat& srcmat, Mat& dstmat)
|
||||
{
|
||||
reduceColSum2_8u<float>(srcmat, dstmat);
|
||||
}
|
||||
|
||||
static void reduceColSum2_8u64f(const Mat& srcmat, Mat& dstmat)
|
||||
{
|
||||
reduceColSum2_8u<double>(srcmat, dstmat);
|
||||
}
|
||||
|
||||
static void reduceColSum2_16u32f(const Mat& srcmat, Mat& dstmat)
|
||||
{
|
||||
if (srcmat.channels() != 1)
|
||||
{
|
||||
reduceColSum2_16u32fFallback(srcmat, dstmat);
|
||||
return;
|
||||
}
|
||||
#if CV_RVV
|
||||
reduce_c_rvv::sum2_16u32fC1(srcmat, dstmat);
|
||||
#elif CV_NEON && (defined(__aarch64__) || defined(_M_ARM64))
|
||||
reduce_c_neon::sum2_16u32fC1(srcmat, dstmat);
|
||||
#elif CV_AVX2
|
||||
reduce_c_avx2::sum2_16u32fC1(srcmat, dstmat);
|
||||
#else
|
||||
reduceColSum2_16u32fFallback(srcmat, dstmat);
|
||||
#endif
|
||||
}
|
||||
|
||||
static void reduceColSum2_16s32f(const Mat& srcmat, Mat& dstmat)
|
||||
{
|
||||
if (srcmat.channels() != 1)
|
||||
{
|
||||
reduceColSum2_16s32fFallback(srcmat, dstmat);
|
||||
return;
|
||||
}
|
||||
#if CV_RVV
|
||||
reduce_c_rvv::sum2_16s32fC1(srcmat, dstmat);
|
||||
#elif CV_NEON && (defined(__aarch64__) || defined(_M_ARM64))
|
||||
reduce_c_neon::sum2_16s32fC1(srcmat, dstmat);
|
||||
#elif CV_AVX2
|
||||
reduce_c_avx2::sum2_16s32fC1(srcmat, dstmat);
|
||||
#else
|
||||
reduceColSum2_16s32fFallback(srcmat, dstmat);
|
||||
#endif
|
||||
}
|
||||
|
||||
static void reduceColSum2_32f32fC1(const Mat& srcmat, Mat& dstmat)
|
||||
{
|
||||
#if CV_RVV
|
||||
reduce_c_rvv::sum2_32fC1(srcmat, dstmat);
|
||||
#elif CV_NEON
|
||||
reduce_c_neon::sum2_32fC1(srcmat, dstmat);
|
||||
#elif CV_AVX2
|
||||
reduce_c_avx2::sum2_32fC1(srcmat, dstmat);
|
||||
#else
|
||||
reduceColSum2_32f32fFallback(srcmat, dstmat);
|
||||
#endif
|
||||
}
|
||||
|
||||
static void reduceColSum2_32f32fC3(const Mat& srcmat, Mat& dstmat)
|
||||
{
|
||||
#if CV_RVV
|
||||
reduce_c_rvv::sum2_32fC3(srcmat, dstmat);
|
||||
#else
|
||||
reduceColSum2_32f32fFallback(srcmat, dstmat);
|
||||
#endif
|
||||
}
|
||||
|
||||
static void reduceColSum2_32f32fC4(const Mat& srcmat, Mat& dstmat)
|
||||
{
|
||||
#if CV_RVV
|
||||
reduce_c_rvv::sum2_32fC4(srcmat, dstmat);
|
||||
#elif CV_NEON
|
||||
reduce_c_neon::sum2_32fC4(srcmat, dstmat);
|
||||
#elif CV_AVX2
|
||||
reduce_c_avx2::sum2_32fC4(srcmat, dstmat);
|
||||
#else
|
||||
reduceColSum2_32f32fFallback(srcmat, dstmat);
|
||||
#endif
|
||||
}
|
||||
|
||||
static void reduceColSum2_32f32f(const Mat& srcmat, Mat& dstmat)
|
||||
{
|
||||
const int cn = srcmat.channels();
|
||||
if (cn == 1)
|
||||
reduceColSum2_32f32fC1(srcmat, dstmat);
|
||||
else if (cn == 3)
|
||||
reduceColSum2_32f32fC3(srcmat, dstmat);
|
||||
else if (cn == 4)
|
||||
reduceColSum2_32f32fC4(srcmat, dstmat);
|
||||
else
|
||||
reduceColSum2_32f32fFallback(srcmat, dstmat);
|
||||
}
|
||||
|
||||
#if (CV_SIMD_64F || CV_SIMD_SCALABLE_64F)
|
||||
static void reduceColSum2_16u64f(const Mat& srcmat, Mat& dstmat)
|
||||
{
|
||||
if (srcmat.channels() != 1)
|
||||
{
|
||||
reduceColSum2_16u64fFallback(srcmat, dstmat);
|
||||
return;
|
||||
}
|
||||
#if CV_RVV && CV_SIMD_SCALABLE_64F
|
||||
reduce_c_rvv::sum2_16u64fC1(srcmat, dstmat);
|
||||
#elif CV_NEON && (defined(__aarch64__) || defined(_M_ARM64))
|
||||
reduce_c_neon::sum2_16u64fC1(srcmat, dstmat);
|
||||
#elif CV_AVX2
|
||||
reduce_c_avx2::sum2_16u64fC1(srcmat, dstmat);
|
||||
#else
|
||||
reduceColSum2_16u64fFallback(srcmat, dstmat);
|
||||
#endif
|
||||
}
|
||||
|
||||
static void reduceColSum2_16s64f(const Mat& srcmat, Mat& dstmat)
|
||||
{
|
||||
if (srcmat.channels() != 1)
|
||||
{
|
||||
reduceColSum2_16s64fFallback(srcmat, dstmat);
|
||||
return;
|
||||
}
|
||||
#if CV_RVV && CV_SIMD_SCALABLE_64F
|
||||
reduce_c_rvv::sum2_16s64fC1(srcmat, dstmat);
|
||||
#elif CV_NEON && (defined(__aarch64__) || defined(_M_ARM64))
|
||||
reduce_c_neon::sum2_16s64fC1(srcmat, dstmat);
|
||||
#elif CV_AVX2
|
||||
reduce_c_avx2::sum2_16s64fC1(srcmat, dstmat);
|
||||
#else
|
||||
reduceColSum2_16s64fFallback(srcmat, dstmat);
|
||||
#endif
|
||||
}
|
||||
|
||||
static void reduceColSum2_32f64f(const Mat& srcmat, Mat& dstmat)
|
||||
{
|
||||
if (srcmat.channels() != 1)
|
||||
{
|
||||
reduceColSum2_32f64fFallback(srcmat, dstmat);
|
||||
return;
|
||||
}
|
||||
#if CV_RVV && CV_SIMD_SCALABLE_64F
|
||||
reduce_c_rvv::sum2_32f64fC1(srcmat, dstmat);
|
||||
#elif CV_NEON && (defined(__aarch64__) || defined(_M_ARM64))
|
||||
reduce_c_neon::sum2_32f64fC1(srcmat, dstmat);
|
||||
#elif CV_AVX2
|
||||
reduce_c_avx2::sum2_32f64fC1(srcmat, dstmat);
|
||||
#else
|
||||
reduceColSum2_32f64fFallback(srcmat, dstmat);
|
||||
#endif
|
||||
}
|
||||
|
||||
static void reduceColSum2_64f64f(const Mat& srcmat, Mat& dstmat)
|
||||
{
|
||||
if (srcmat.channels() != 1)
|
||||
{
|
||||
reduceColSum2_64f64fFallback(srcmat, dstmat);
|
||||
return;
|
||||
}
|
||||
#if CV_RVV && CV_SIMD_SCALABLE_64F
|
||||
reduce_c_rvv::sum2_64f64fC1(srcmat, dstmat);
|
||||
#elif CV_NEON && (defined(__aarch64__) || defined(_M_ARM64))
|
||||
reduce_c_neon::sum2_64f64fC1(srcmat, dstmat);
|
||||
#elif CV_AVX2
|
||||
reduce_c_avx2::sum2_64f64fC1(srcmat, dstmat);
|
||||
#else
|
||||
reduceColSum2_64f64fFallback(srcmat, dstmat);
|
||||
#endif
|
||||
}
|
||||
#endif
|
||||
@@ -0,0 +1,989 @@
|
||||
// This file is part of OpenCV project.
|
||||
// It is subject to the license terms in the LICENSE file found in the top-level directory
|
||||
// of this distribution and at http://opencv.org/license.html
|
||||
|
||||
namespace reduce_c_avx2
|
||||
{
|
||||
|
||||
// Optimized ReduceC support in this backend:
|
||||
//
|
||||
// | Input -> output type/channel | SUM | AVG | MIN | MAX | SUM2 |
|
||||
// |------------------------------------|:---:|:---:|:---:|:---:|:----:|
|
||||
// | 8UC1/C3/C4 -> 8UC1/C3/C4 | - | - | x | x | - |
|
||||
// | 8UC1/C3/C4 -> 32SC1/C3/C4 | x | x | - | - | x |
|
||||
// | 8UC1/C3/C4 -> 32FC1/C3/C4 | x | x | - | - | x |
|
||||
// | 8UC1/C3/C4 -> 64FC1/C3/C4 | - | - | - | - | x |
|
||||
// | 16UC1/C3/C4 -> 16UC1/C3/C4 | - | - | x | x | - |
|
||||
// | 16UC1 -> 32FC1 | x | x | - | - | x |
|
||||
// | 16UC3/C4 -> 32FC3/C4 | x | x | - | - | - |
|
||||
// | 16UC1 -> 64FC1 | - | - | - | - | x |
|
||||
// | 16UC3/C4 -> 64FC3/C4 | - | - | - | - | - |
|
||||
// | 16SC1/C3/C4 -> 16SC1/C3/C4 | - | - | x | x | - |
|
||||
// | 16SC1 -> 32FC1 | x | x | - | - | x |
|
||||
// | 16SC3/C4 -> 32FC3/C4 | x | x | - | - | - |
|
||||
// | 16SC1 -> 64FC1 | - | - | - | - | x |
|
||||
// | 16SC3/C4 -> 64FC3/C4 | - | - | - | - | - |
|
||||
// | 32FC1 -> 32FC1 | x | x | x | x | x |
|
||||
// | 32FC3 -> 32FC3 | x | x | - | - | - |
|
||||
// | 32FC4 -> 32FC4 | x | x | x | x | x |
|
||||
// | 32FC1 -> 64FC1 | x | x | - | - | x |
|
||||
// | 32FC3/C4 -> 64FC3/C4 | x | x | - | - | - |
|
||||
// | 64FC1 -> 64FC1 | x | x | x | x | x |
|
||||
// | 64FC3/C4 -> 64FC3/C4 | x | x | - | - | - |
|
||||
//
|
||||
// 'x' in SUM/AVG denotes the existing shared universal-intrinsics kernel; 'x'
|
||||
// in MIN/MAX/SUM2 denotes a native AVX2 kernel. For legal MIN/MAX/SUM2
|
||||
// combinations marked '-', and for other channel counts, dispatch uses the
|
||||
// shared generic fallback.
|
||||
|
||||
template<bool isMax>
|
||||
static void minMax8uC1(const Mat& srcmat, Mat& dstmat)
|
||||
{
|
||||
const int cols = srcmat.cols;
|
||||
parallel_for_(Range(0, srcmat.rows), [&](const Range& range) {
|
||||
for (int y = range.start; y < range.end; y++)
|
||||
{
|
||||
const uchar* src = srcmat.ptr<uchar>(y);
|
||||
uchar* dst = dstmat.ptr<uchar>(y);
|
||||
uchar result = isMax ? 0 : UCHAR_MAX;
|
||||
int x = 0;
|
||||
__m256i acc = _mm256_set1_epi8((char)result);
|
||||
for (; x <= cols - 32; x += 32)
|
||||
{
|
||||
__m256i v = _mm256_loadu_si256((const __m256i*)(src + x));
|
||||
acc = isMax ? _mm256_max_epu8(acc, v) : _mm256_min_epu8(acc, v);
|
||||
}
|
||||
uchar lanes[32];
|
||||
_mm256_storeu_si256((__m256i*)lanes, acc);
|
||||
for (int i = 0; i < 32; i++)
|
||||
result = reduceScalarMinMax<isMax>(result, lanes[i]);
|
||||
for (; x < cols; x++)
|
||||
result = reduceScalarMinMax<isMax>(result, src[x]);
|
||||
dst[0] = result;
|
||||
}
|
||||
});
|
||||
v_cleanup();
|
||||
}
|
||||
|
||||
template<bool isMax>
|
||||
static void minMax16uC1(const Mat& srcmat, Mat& dstmat)
|
||||
{
|
||||
const int cols = srcmat.cols;
|
||||
parallel_for_(Range(0, srcmat.rows), [&](const Range& range) {
|
||||
for (int y = range.start; y < range.end; y++)
|
||||
{
|
||||
const ushort* src = srcmat.ptr<ushort>(y);
|
||||
ushort result = isMax ? 0 : USHRT_MAX;
|
||||
int x = 0;
|
||||
__m256i acc = _mm256_set1_epi16((short)result);
|
||||
for (; x <= cols - 16; x += 16)
|
||||
{
|
||||
__m256i v = _mm256_loadu_si256((const __m256i*)(src + x));
|
||||
acc = isMax ? _mm256_max_epu16(acc, v) : _mm256_min_epu16(acc, v);
|
||||
}
|
||||
ushort lanes[16];
|
||||
_mm256_storeu_si256((__m256i*)lanes, acc);
|
||||
for (int i = 0; i < 16; i++)
|
||||
result = isMax ? std::max(result, lanes[i]) : std::min(result, lanes[i]);
|
||||
for (; x < cols; x++)
|
||||
result = isMax ? std::max(result, src[x]) : std::min(result, src[x]);
|
||||
dstmat.ptr<ushort>(y)[0] = result;
|
||||
}
|
||||
});
|
||||
v_cleanup();
|
||||
}
|
||||
|
||||
template<bool isMax>
|
||||
static void minMax16sC1(const Mat& srcmat, Mat& dstmat)
|
||||
{
|
||||
const int cols = srcmat.cols;
|
||||
parallel_for_(Range(0, srcmat.rows), [&](const Range& range) {
|
||||
for (int y = range.start; y < range.end; y++)
|
||||
{
|
||||
const short* src = srcmat.ptr<short>(y);
|
||||
short result = isMax ? SHRT_MIN : SHRT_MAX;
|
||||
int x = 0;
|
||||
__m256i acc = _mm256_set1_epi16(result);
|
||||
for (; x <= cols - 16; x += 16)
|
||||
{
|
||||
__m256i v = _mm256_loadu_si256((const __m256i*)(src + x));
|
||||
acc = isMax ? _mm256_max_epi16(acc, v) : _mm256_min_epi16(acc, v);
|
||||
}
|
||||
short lanes[16];
|
||||
_mm256_storeu_si256((__m256i*)lanes, acc);
|
||||
for (int i = 0; i < 16; i++)
|
||||
result = isMax ? std::max(result, lanes[i]) : std::min(result, lanes[i]);
|
||||
for (; x < cols; x++)
|
||||
result = isMax ? std::max(result, src[x]) : std::min(result, src[x]);
|
||||
dstmat.ptr<short>(y)[0] = result;
|
||||
}
|
||||
});
|
||||
v_cleanup();
|
||||
}
|
||||
|
||||
template<bool isMax>
|
||||
static void minMax16uC4(const Mat& srcmat, Mat& dstmat)
|
||||
{
|
||||
const int cols = srcmat.cols;
|
||||
const ushort initial = isMax ? 0 : USHRT_MAX;
|
||||
const __m256i accInit = _mm256_set1_epi16((short)initial);
|
||||
const __m256i validMask = _mm256_broadcastsi128_si256(
|
||||
_mm_setr_epi8(-1, -1, -1, -1, 0, 0, 0, 0,
|
||||
0, 0, 0, 0, 0, 0, 0, 0));
|
||||
const __m256i masks[4] = {
|
||||
_mm256_broadcastsi128_si256(_mm_setr_epi8(0, 1, 8, 9, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1)),
|
||||
_mm256_broadcastsi128_si256(_mm_setr_epi8(2, 3, 10, 11, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1)),
|
||||
_mm256_broadcastsi128_si256(_mm_setr_epi8(4, 5, 12, 13, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1)),
|
||||
_mm256_broadcastsi128_si256(_mm_setr_epi8(6, 7, 14, 15, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1))
|
||||
};
|
||||
parallel_for_(Range(0, srcmat.rows), [&](const Range& range) {
|
||||
for (int y = range.start; y < range.end; y++)
|
||||
{
|
||||
const ushort* src = srcmat.ptr<ushort>(y);
|
||||
ushort* dst = dstmat.ptr<ushort>(y);
|
||||
__m256i acc[4] = {accInit, accInit, accInit, accInit};
|
||||
int x = 0;
|
||||
for (; x <= cols - 4; x += 4)
|
||||
{
|
||||
__m256i v = _mm256_loadu_si256((const __m256i*)(src + x * 4));
|
||||
for (int c = 0; c < 4; c++)
|
||||
{
|
||||
__m256i vc = _mm256_shuffle_epi8(v, masks[c]);
|
||||
vc = _mm256_blendv_epi8(accInit, vc, validMask);
|
||||
acc[c] = isMax ? _mm256_max_epu16(acc[c], vc)
|
||||
: _mm256_min_epu16(acc[c], vc);
|
||||
}
|
||||
}
|
||||
for (int c = 0; c < 4; c++)
|
||||
{
|
||||
ushort lanes[16];
|
||||
_mm256_storeu_si256((__m256i*)lanes, acc[c]);
|
||||
ushort result = initial;
|
||||
for (int i = 0; i < 16; i++)
|
||||
result = isMax ? std::max(result, lanes[i]) : std::min(result, lanes[i]);
|
||||
for (int i = x; i < cols; i++)
|
||||
result = isMax ? std::max(result, src[i * 4 + c])
|
||||
: std::min(result, src[i * 4 + c]);
|
||||
dst[c] = result;
|
||||
}
|
||||
}
|
||||
});
|
||||
v_cleanup();
|
||||
}
|
||||
|
||||
template<bool isMax>
|
||||
static void minMax16sC4(const Mat& srcmat, Mat& dstmat)
|
||||
{
|
||||
const int cols = srcmat.cols;
|
||||
const short initial = isMax ? SHRT_MIN : SHRT_MAX;
|
||||
const __m256i accInit = _mm256_set1_epi16(initial);
|
||||
const __m256i validMask = _mm256_broadcastsi128_si256(
|
||||
_mm_setr_epi8(-1, -1, -1, -1, 0, 0, 0, 0,
|
||||
0, 0, 0, 0, 0, 0, 0, 0));
|
||||
const __m256i masks[4] = {
|
||||
_mm256_broadcastsi128_si256(_mm_setr_epi8(0, 1, 8, 9, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1)),
|
||||
_mm256_broadcastsi128_si256(_mm_setr_epi8(2, 3, 10, 11, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1)),
|
||||
_mm256_broadcastsi128_si256(_mm_setr_epi8(4, 5, 12, 13, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1)),
|
||||
_mm256_broadcastsi128_si256(_mm_setr_epi8(6, 7, 14, 15, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1))
|
||||
};
|
||||
parallel_for_(Range(0, srcmat.rows), [&](const Range& range) {
|
||||
for (int y = range.start; y < range.end; y++)
|
||||
{
|
||||
const short* src = srcmat.ptr<short>(y);
|
||||
short* dst = dstmat.ptr<short>(y);
|
||||
__m256i acc[4] = {accInit, accInit, accInit, accInit};
|
||||
int x = 0;
|
||||
for (; x <= cols - 4; x += 4)
|
||||
{
|
||||
__m256i v = _mm256_loadu_si256((const __m256i*)(src + x * 4));
|
||||
for (int c = 0; c < 4; c++)
|
||||
{
|
||||
__m256i vc = _mm256_shuffle_epi8(v, masks[c]);
|
||||
vc = _mm256_blendv_epi8(accInit, vc, validMask);
|
||||
acc[c] = isMax ? _mm256_max_epi16(acc[c], vc)
|
||||
: _mm256_min_epi16(acc[c], vc);
|
||||
}
|
||||
}
|
||||
for (int c = 0; c < 4; c++)
|
||||
{
|
||||
short lanes[16];
|
||||
_mm256_storeu_si256((__m256i*)lanes, acc[c]);
|
||||
short result = initial;
|
||||
for (int i = 0; i < 16; i++)
|
||||
result = isMax ? std::max(result, lanes[i]) : std::min(result, lanes[i]);
|
||||
for (int i = x; i < cols; i++)
|
||||
result = isMax ? std::max(result, src[i * 4 + c])
|
||||
: std::min(result, src[i * 4 + c]);
|
||||
dst[c] = result;
|
||||
}
|
||||
}
|
||||
});
|
||||
v_cleanup();
|
||||
}
|
||||
|
||||
template<typename T, bool isMax, bool isSigned>
|
||||
static void minMax16C3(const Mat& srcmat, Mat& dstmat)
|
||||
{
|
||||
const int cols = srcmat.cols;
|
||||
const T initial = isMax ? std::numeric_limits<T>::lowest() : std::numeric_limits<T>::max();
|
||||
const __m256i accInit = _mm256_set1_epi16((short)initial);
|
||||
const __m256i masks[3] = {
|
||||
_mm256_setr_epi8(0, 1, 6, 7, 12, 13, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
|
||||
2, 3, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1),
|
||||
_mm256_setr_epi8(2, 3, 8, 9, 14, 15, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
|
||||
4, 5, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1),
|
||||
_mm256_setr_epi8(4, 5, 10, 11, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
|
||||
0, 1, 6, 7, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1)
|
||||
};
|
||||
const __m256i validMasks[3] = {
|
||||
_mm256_setr_epi8(-1, -1, -1, -1, -1, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
||||
-1, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0),
|
||||
_mm256_setr_epi8(-1, -1, -1, -1, -1, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
||||
-1, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0),
|
||||
_mm256_setr_epi8(-1, -1, -1, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
||||
-1, -1, -1, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0)
|
||||
};
|
||||
parallel_for_(Range(0, srcmat.rows), [&](const Range& range) {
|
||||
for (int y = range.start; y < range.end; y++)
|
||||
{
|
||||
const T* src = srcmat.ptr<T>(y);
|
||||
T* dst = dstmat.ptr<T>(y);
|
||||
__m256i acc[3] = {accInit, accInit, accInit};
|
||||
int x = 0;
|
||||
for (; x <= cols - 6; x += 4)
|
||||
{
|
||||
__m256i v = _mm256_loadu_si256((const __m256i*)(src + x * 3));
|
||||
for (int c = 0; c < 3; c++)
|
||||
{
|
||||
__m256i vc = _mm256_shuffle_epi8(v, masks[c]);
|
||||
vc = _mm256_blendv_epi8(accInit, vc, validMasks[c]);
|
||||
if (isSigned)
|
||||
acc[c] = isMax ? _mm256_max_epi16(acc[c], vc) : _mm256_min_epi16(acc[c], vc);
|
||||
else
|
||||
acc[c] = isMax ? _mm256_max_epu16(acc[c], vc) : _mm256_min_epu16(acc[c], vc);
|
||||
}
|
||||
}
|
||||
for (int c = 0; c < 3; c++)
|
||||
{
|
||||
T lanes[16];
|
||||
_mm256_storeu_si256((__m256i*)lanes, acc[c]);
|
||||
T result = initial;
|
||||
for (int i = 0; i < 16; i++)
|
||||
result = isMax ? std::max(result, lanes[i]) : std::min(result, lanes[i]);
|
||||
for (int i = x; i < cols; i++)
|
||||
result = isMax ? std::max(result, src[i * 3 + c])
|
||||
: std::min(result, src[i * 3 + c]);
|
||||
dst[c] = result;
|
||||
}
|
||||
}
|
||||
});
|
||||
v_cleanup();
|
||||
}
|
||||
|
||||
template<bool isMax>
|
||||
static void minMax16uC3(const Mat& srcmat, Mat& dstmat)
|
||||
{
|
||||
minMax16C3<ushort, isMax, false>(srcmat, dstmat);
|
||||
}
|
||||
|
||||
template<bool isMax>
|
||||
static void minMax16sC3(const Mat& srcmat, Mat& dstmat)
|
||||
{
|
||||
minMax16C3<short, isMax, true>(srcmat, dstmat);
|
||||
}
|
||||
|
||||
template<bool isMax>
|
||||
static void minMax8uC3(const Mat& srcmat, Mat& dstmat)
|
||||
{
|
||||
const int cols = srcmat.cols;
|
||||
const uchar initial = isMax ? 0 : UCHAR_MAX;
|
||||
const __m256i accInit = _mm256_set1_epi8((char)initial);
|
||||
const __m256i mask0 = _mm256_setr_epi8(
|
||||
0, 3, 6, 9, 12, 15, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
|
||||
2, 5, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1);
|
||||
const __m256i mask1 = _mm256_setr_epi8(
|
||||
1, 4, 7, 10, 13, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
|
||||
0, 3, 6, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1);
|
||||
const __m256i mask2 = _mm256_setr_epi8(
|
||||
2, 5, 8, 11, 14, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
|
||||
1, 4, 7, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1);
|
||||
const __m256i invalid0 = _mm256_setr_epi8(
|
||||
0, 0, 0, 0, 0, 0, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
|
||||
0, 0, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1);
|
||||
const __m256i invalid12 = _mm256_setr_epi8(
|
||||
0, 0, 0, 0, 0, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
|
||||
0, 0, 0, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1);
|
||||
|
||||
parallel_for_(Range(0, srcmat.rows), [&](const Range& range) {
|
||||
for (int y = range.start; y < range.end; y++)
|
||||
{
|
||||
const uchar* src = srcmat.ptr<uchar>(y);
|
||||
uchar* dst = dstmat.ptr<uchar>(y);
|
||||
__m256i acc0 = accInit, acc1 = accInit, acc2 = accInit;
|
||||
int x = 0;
|
||||
for (; x <= cols - 11; x += 8)
|
||||
{
|
||||
__m256i v = _mm256_loadu_si256((const __m256i*)(src + x * 3));
|
||||
__m256i v0 = _mm256_shuffle_epi8(v, mask0);
|
||||
__m256i v1 = _mm256_shuffle_epi8(v, mask1);
|
||||
__m256i v2 = _mm256_shuffle_epi8(v, mask2);
|
||||
if (!isMax)
|
||||
{
|
||||
v0 = _mm256_or_si256(v0, invalid0);
|
||||
v1 = _mm256_or_si256(v1, invalid12);
|
||||
v2 = _mm256_or_si256(v2, invalid12);
|
||||
}
|
||||
acc0 = isMax ? _mm256_max_epu8(acc0, v0) : _mm256_min_epu8(acc0, v0);
|
||||
acc1 = isMax ? _mm256_max_epu8(acc1, v1) : _mm256_min_epu8(acc1, v1);
|
||||
acc2 = isMax ? _mm256_max_epu8(acc2, v2) : _mm256_min_epu8(acc2, v2);
|
||||
}
|
||||
|
||||
__m256i accs[3] = {acc0, acc1, acc2};
|
||||
for (int c = 0; c < 3; c++)
|
||||
{
|
||||
uchar lanes[32];
|
||||
_mm256_storeu_si256((__m256i*)lanes, accs[c]);
|
||||
uchar result = initial;
|
||||
for (int i = 0; i < 32; i++)
|
||||
result = reduceScalarMinMax<isMax>(result, lanes[i]);
|
||||
for (int i = x; i < cols; i++)
|
||||
result = reduceScalarMinMax<isMax>(result, src[i * 3 + c]);
|
||||
dst[c] = result;
|
||||
}
|
||||
}
|
||||
});
|
||||
v_cleanup();
|
||||
}
|
||||
|
||||
template<bool isMax>
|
||||
static void minMax8uC4(const Mat& srcmat, Mat& dstmat)
|
||||
{
|
||||
const int cols = srcmat.cols;
|
||||
const uchar initial = isMax ? 0 : UCHAR_MAX;
|
||||
const __m256i accInit = _mm256_set1_epi8((char)initial);
|
||||
const __m256i maskInvalid = _mm256_broadcastsi128_si256(
|
||||
_mm_setr_epi8(0, 0, 0, 0, -1, -1, -1, -1,
|
||||
-1, -1, -1, -1, -1, -1, -1, -1));
|
||||
const __m256i mask0 = _mm256_broadcastsi128_si256(
|
||||
_mm_setr_epi8(0, 4, 8, 12, -1, -1, -1, -1,
|
||||
-1, -1, -1, -1, -1, -1, -1, -1));
|
||||
const __m256i mask1 = _mm256_broadcastsi128_si256(
|
||||
_mm_setr_epi8(1, 5, 9, 13, -1, -1, -1, -1,
|
||||
-1, -1, -1, -1, -1, -1, -1, -1));
|
||||
const __m256i mask2 = _mm256_broadcastsi128_si256(
|
||||
_mm_setr_epi8(2, 6, 10, 14, -1, -1, -1, -1,
|
||||
-1, -1, -1, -1, -1, -1, -1, -1));
|
||||
const __m256i mask3 = _mm256_broadcastsi128_si256(
|
||||
_mm_setr_epi8(3, 7, 11, 15, -1, -1, -1, -1,
|
||||
-1, -1, -1, -1, -1, -1, -1, -1));
|
||||
|
||||
parallel_for_(Range(0, srcmat.rows), [&](const Range& range) {
|
||||
for (int y = range.start; y < range.end; y++)
|
||||
{
|
||||
const uchar* src = srcmat.ptr<uchar>(y);
|
||||
uchar* dst = dstmat.ptr<uchar>(y);
|
||||
__m256i acc0 = accInit, acc1 = accInit, acc2 = accInit, acc3 = accInit;
|
||||
int x = 0;
|
||||
for (; x <= cols - 8; x += 8)
|
||||
{
|
||||
__m256i v = _mm256_loadu_si256((const __m256i*)(src + x * 4));
|
||||
__m256i v0 = _mm256_shuffle_epi8(v, mask0);
|
||||
__m256i v1 = _mm256_shuffle_epi8(v, mask1);
|
||||
__m256i v2 = _mm256_shuffle_epi8(v, mask2);
|
||||
__m256i v3 = _mm256_shuffle_epi8(v, mask3);
|
||||
if (!isMax)
|
||||
{
|
||||
v0 = _mm256_or_si256(v0, maskInvalid);
|
||||
v1 = _mm256_or_si256(v1, maskInvalid);
|
||||
v2 = _mm256_or_si256(v2, maskInvalid);
|
||||
v3 = _mm256_or_si256(v3, maskInvalid);
|
||||
}
|
||||
acc0 = isMax ? _mm256_max_epu8(acc0, v0) : _mm256_min_epu8(acc0, v0);
|
||||
acc1 = isMax ? _mm256_max_epu8(acc1, v1) : _mm256_min_epu8(acc1, v1);
|
||||
acc2 = isMax ? _mm256_max_epu8(acc2, v2) : _mm256_min_epu8(acc2, v2);
|
||||
acc3 = isMax ? _mm256_max_epu8(acc3, v3) : _mm256_min_epu8(acc3, v3);
|
||||
}
|
||||
|
||||
__m256i accs[4] = {acc0, acc1, acc2, acc3};
|
||||
for (int c = 0; c < 4; c++)
|
||||
{
|
||||
uchar lanes[32];
|
||||
_mm256_storeu_si256((__m256i*)lanes, accs[c]);
|
||||
uchar result = initial;
|
||||
for (int i = 0; i < 32; i++)
|
||||
result = reduceScalarMinMax<isMax>(result, lanes[i]);
|
||||
for (int i = x; i < cols; i++)
|
||||
result = reduceScalarMinMax<isMax>(result, src[i * 4 + c]);
|
||||
dst[c] = result;
|
||||
}
|
||||
}
|
||||
});
|
||||
v_cleanup();
|
||||
}
|
||||
|
||||
template<bool isMax>
|
||||
static void minMax32fC1(const Mat& srcmat, Mat& dstmat)
|
||||
{
|
||||
const int cols = srcmat.cols;
|
||||
parallel_for_(Range(0, srcmat.rows), [&](const Range& range) {
|
||||
for (int y = range.start; y < range.end; y++)
|
||||
{
|
||||
const float* src = srcmat.ptr<float>(y);
|
||||
float result = src[0];
|
||||
int x = 0;
|
||||
__m256 acc = _mm256_set1_ps(result);
|
||||
for (; x <= cols - 8; x += 8)
|
||||
{
|
||||
__m256 v = _mm256_loadu_ps(src + x);
|
||||
acc = isMax ? _mm256_max_ps(acc, v) : _mm256_min_ps(acc, v);
|
||||
}
|
||||
float lanes[8];
|
||||
_mm256_storeu_ps(lanes, acc);
|
||||
for (int i = 0; i < 8; i++)
|
||||
result = reduceScalarMinMax<isMax>(result, lanes[i]);
|
||||
for (; x < cols; x++)
|
||||
result = reduceScalarMinMax<isMax>(result, src[x]);
|
||||
dstmat.ptr<float>(y)[0] = result;
|
||||
}
|
||||
});
|
||||
v_cleanup();
|
||||
}
|
||||
|
||||
template<bool isMax>
|
||||
static void minMax32fC4(const Mat& srcmat, Mat& dstmat)
|
||||
{
|
||||
const int cols = srcmat.cols;
|
||||
const float initial = isMax ? std::numeric_limits<float>::lowest() : std::numeric_limits<float>::max();
|
||||
const __m256 accInit = _mm256_set1_ps(initial);
|
||||
const __m256 validMask = _mm256_castsi256_ps(_mm256_setr_epi32(-1, -1, 0, 0, 0, 0, 0, 0));
|
||||
const __m256i idx0 = _mm256_setr_epi32(0, 4, 0, 0, 0, 0, 0, 0);
|
||||
const __m256i idx1 = _mm256_setr_epi32(1, 5, 0, 0, 0, 0, 0, 0);
|
||||
const __m256i idx2 = _mm256_setr_epi32(2, 6, 0, 0, 0, 0, 0, 0);
|
||||
const __m256i idx3 = _mm256_setr_epi32(3, 7, 0, 0, 0, 0, 0, 0);
|
||||
|
||||
parallel_for_(Range(0, srcmat.rows), [&](const Range& range) {
|
||||
for (int y = range.start; y < range.end; y++)
|
||||
{
|
||||
const float* src = srcmat.ptr<float>(y);
|
||||
float* dst = dstmat.ptr<float>(y);
|
||||
__m256 acc0 = accInit, acc1 = accInit, acc2 = accInit, acc3 = accInit;
|
||||
int x = 0;
|
||||
for (; x <= cols - 2; x += 2)
|
||||
{
|
||||
__m256 v = _mm256_loadu_ps(src + x * 4);
|
||||
__m256 v0 = _mm256_blendv_ps(accInit, _mm256_permutevar8x32_ps(v, idx0), validMask);
|
||||
__m256 v1 = _mm256_blendv_ps(accInit, _mm256_permutevar8x32_ps(v, idx1), validMask);
|
||||
__m256 v2 = _mm256_blendv_ps(accInit, _mm256_permutevar8x32_ps(v, idx2), validMask);
|
||||
__m256 v3 = _mm256_blendv_ps(accInit, _mm256_permutevar8x32_ps(v, idx3), validMask);
|
||||
acc0 = isMax ? _mm256_max_ps(acc0, v0) : _mm256_min_ps(acc0, v0);
|
||||
acc1 = isMax ? _mm256_max_ps(acc1, v1) : _mm256_min_ps(acc1, v1);
|
||||
acc2 = isMax ? _mm256_max_ps(acc2, v2) : _mm256_min_ps(acc2, v2);
|
||||
acc3 = isMax ? _mm256_max_ps(acc3, v3) : _mm256_min_ps(acc3, v3);
|
||||
}
|
||||
|
||||
__m256 accs[4] = {acc0, acc1, acc2, acc3};
|
||||
for (int c = 0; c < 4; c++)
|
||||
{
|
||||
float lanes[8];
|
||||
_mm256_storeu_ps(lanes, accs[c]);
|
||||
float result = initial;
|
||||
for (int i = 0; i < 8; i++)
|
||||
result = reduceScalarMinMax<isMax>(result, lanes[i]);
|
||||
for (int i = x; i < cols; i++)
|
||||
result = reduceScalarMinMax<isMax>(result, src[i * 4 + c]);
|
||||
dst[c] = result;
|
||||
}
|
||||
}
|
||||
});
|
||||
v_cleanup();
|
||||
}
|
||||
|
||||
template<bool isMax>
|
||||
static void minMax64fC1(const Mat& srcmat, Mat& dstmat)
|
||||
{
|
||||
const int cols = srcmat.cols;
|
||||
const double initial = isMax ? std::numeric_limits<double>::lowest()
|
||||
: std::numeric_limits<double>::max();
|
||||
parallel_for_(Range(0, srcmat.rows), [&](const Range& range) {
|
||||
for (int y = range.start; y < range.end; y++)
|
||||
{
|
||||
const double* src = srcmat.ptr<double>(y);
|
||||
double result = initial;
|
||||
int x = 0;
|
||||
__m256d acc0 = _mm256_set1_pd(initial);
|
||||
__m256d acc1 = acc0, acc2 = acc0, acc3 = acc0;
|
||||
for (; x <= cols - 16; x += 16)
|
||||
{
|
||||
__m256d v0 = _mm256_loadu_pd(src + x);
|
||||
__m256d v1 = _mm256_loadu_pd(src + x + 4);
|
||||
__m256d v2 = _mm256_loadu_pd(src + x + 8);
|
||||
__m256d v3 = _mm256_loadu_pd(src + x + 12);
|
||||
acc0 = isMax ? _mm256_max_pd(acc0, v0) : _mm256_min_pd(acc0, v0);
|
||||
acc1 = isMax ? _mm256_max_pd(acc1, v1) : _mm256_min_pd(acc1, v1);
|
||||
acc2 = isMax ? _mm256_max_pd(acc2, v2) : _mm256_min_pd(acc2, v2);
|
||||
acc3 = isMax ? _mm256_max_pd(acc3, v3) : _mm256_min_pd(acc3, v3);
|
||||
}
|
||||
acc0 = isMax ? _mm256_max_pd(acc0, acc1) : _mm256_min_pd(acc0, acc1);
|
||||
acc2 = isMax ? _mm256_max_pd(acc2, acc3) : _mm256_min_pd(acc2, acc3);
|
||||
acc0 = isMax ? _mm256_max_pd(acc0, acc2) : _mm256_min_pd(acc0, acc2);
|
||||
double lanes[4];
|
||||
_mm256_storeu_pd(lanes, acc0);
|
||||
for (int i = 0; i < 4; i++)
|
||||
result = isMax ? std::max(result, lanes[i]) : std::min(result, lanes[i]);
|
||||
for (; x < cols; x++)
|
||||
result = isMax ? std::max(result, src[x]) : std::min(result, src[x]);
|
||||
dstmat.ptr<double>(y)[0] = result;
|
||||
}
|
||||
});
|
||||
v_cleanup();
|
||||
}
|
||||
|
||||
template<typename DT>
|
||||
static void sum2_8uC1(const Mat& srcmat, Mat& dstmat)
|
||||
{
|
||||
const int cols = srcmat.cols;
|
||||
|
||||
parallel_for_(Range(0, srcmat.rows), [&](const Range& range) {
|
||||
for (int y = range.start; y < range.end; y++)
|
||||
{
|
||||
const uchar* src = srcmat.ptr<uchar>(y);
|
||||
DT* dst = dstmat.ptr<DT>(y);
|
||||
uint32_t result = 0;
|
||||
int x = 0;
|
||||
__m256i acc = _mm256_setzero_si256();
|
||||
for (; x <= cols - 32; x += 32)
|
||||
{
|
||||
__m256i bytes = _mm256_loadu_si256((const __m256i*)(src + x));
|
||||
__m128i lo = _mm256_castsi256_si128(bytes);
|
||||
__m128i hi = _mm256_extracti128_si256(bytes, 1);
|
||||
__m256i lo16 = _mm256_cvtepu8_epi16(lo);
|
||||
__m256i hi16 = _mm256_cvtepu8_epi16(hi);
|
||||
acc = _mm256_add_epi32(acc, _mm256_madd_epi16(lo16, lo16));
|
||||
acc = _mm256_add_epi32(acc, _mm256_madd_epi16(hi16, hi16));
|
||||
}
|
||||
uint32_t lanes[8];
|
||||
_mm256_storeu_si256((__m256i*)lanes, acc);
|
||||
for (int i = 0; i < 8; i++)
|
||||
result += lanes[i];
|
||||
for (; x < cols; x++)
|
||||
result += (uint32_t)src[x] * src[x];
|
||||
dst[0] = (DT)(int32_t)result;
|
||||
}
|
||||
});
|
||||
v_cleanup();
|
||||
}
|
||||
|
||||
static void sum2_16u32fC1(const Mat& srcmat, Mat& dstmat)
|
||||
{
|
||||
const int cols = srcmat.cols;
|
||||
parallel_for_(Range(0, srcmat.rows), [&](const Range& range) {
|
||||
for (int y = range.start; y < range.end; y++)
|
||||
{
|
||||
const ushort* src = srcmat.ptr<ushort>(y);
|
||||
__m256 acc0 = _mm256_setzero_ps(), acc1 = acc0;
|
||||
int x = 0;
|
||||
for (; x <= cols - 16; x += 16)
|
||||
{
|
||||
__m256i v = _mm256_loadu_si256((const __m256i*)(src + x));
|
||||
__m256 v0 = _mm256_cvtepi32_ps(_mm256_cvtepu16_epi32(_mm256_castsi256_si128(v)));
|
||||
__m256 v1 = _mm256_cvtepi32_ps(_mm256_cvtepu16_epi32(_mm256_extracti128_si256(v, 1)));
|
||||
acc0 = _mm256_add_ps(acc0, _mm256_mul_ps(v0, v0));
|
||||
acc1 = _mm256_add_ps(acc1, _mm256_mul_ps(v1, v1));
|
||||
}
|
||||
acc0 = _mm256_add_ps(acc0, acc1);
|
||||
float lanes[8];
|
||||
_mm256_storeu_ps(lanes, acc0);
|
||||
float result = 0;
|
||||
for (int i = 0; i < 8; i++)
|
||||
result += lanes[i];
|
||||
for (; x < cols; x++)
|
||||
{
|
||||
float value = (float)src[x];
|
||||
result += value * value;
|
||||
}
|
||||
dstmat.ptr<float>(y)[0] = result;
|
||||
}
|
||||
});
|
||||
v_cleanup();
|
||||
}
|
||||
|
||||
static void sum2_16s32fC1(const Mat& srcmat, Mat& dstmat)
|
||||
{
|
||||
const int cols = srcmat.cols;
|
||||
parallel_for_(Range(0, srcmat.rows), [&](const Range& range) {
|
||||
for (int y = range.start; y < range.end; y++)
|
||||
{
|
||||
const short* src = srcmat.ptr<short>(y);
|
||||
__m256 acc0 = _mm256_setzero_ps(), acc1 = acc0;
|
||||
int x = 0;
|
||||
for (; x <= cols - 16; x += 16)
|
||||
{
|
||||
__m256i v = _mm256_loadu_si256((const __m256i*)(src + x));
|
||||
__m256 v0 = _mm256_cvtepi32_ps(_mm256_cvtepi16_epi32(_mm256_castsi256_si128(v)));
|
||||
__m256 v1 = _mm256_cvtepi32_ps(_mm256_cvtepi16_epi32(_mm256_extracti128_si256(v, 1)));
|
||||
acc0 = _mm256_add_ps(acc0, _mm256_mul_ps(v0, v0));
|
||||
acc1 = _mm256_add_ps(acc1, _mm256_mul_ps(v1, v1));
|
||||
}
|
||||
acc0 = _mm256_add_ps(acc0, acc1);
|
||||
float lanes[8];
|
||||
_mm256_storeu_ps(lanes, acc0);
|
||||
float result = 0;
|
||||
for (int i = 0; i < 8; i++)
|
||||
result += lanes[i];
|
||||
for (; x < cols; x++)
|
||||
{
|
||||
float value = (float)src[x];
|
||||
result += value * value;
|
||||
}
|
||||
dstmat.ptr<float>(y)[0] = result;
|
||||
}
|
||||
});
|
||||
v_cleanup();
|
||||
}
|
||||
|
||||
static void sum2_16u64fC1(const Mat& srcmat, Mat& dstmat)
|
||||
{
|
||||
const int cols = srcmat.cols;
|
||||
parallel_for_(Range(0, srcmat.rows), [&](const Range& range) {
|
||||
for (int y = range.start; y < range.end; y++)
|
||||
{
|
||||
const ushort* src = srcmat.ptr<ushort>(y);
|
||||
__m256d acc0 = _mm256_setzero_pd(), acc1 = acc0, acc2 = acc0, acc3 = acc0;
|
||||
int x = 0;
|
||||
for (; x <= cols - 16; x += 16)
|
||||
{
|
||||
__m256i v = _mm256_loadu_si256((const __m256i*)(src + x));
|
||||
__m256i v0 = _mm256_cvtepu16_epi32(_mm256_castsi256_si128(v));
|
||||
__m256i v1 = _mm256_cvtepu16_epi32(_mm256_extracti128_si256(v, 1));
|
||||
__m256d d0 = _mm256_cvtepi32_pd(_mm256_castsi256_si128(v0));
|
||||
__m256d d1 = _mm256_cvtepi32_pd(_mm256_extracti128_si256(v0, 1));
|
||||
__m256d d2 = _mm256_cvtepi32_pd(_mm256_castsi256_si128(v1));
|
||||
__m256d d3 = _mm256_cvtepi32_pd(_mm256_extracti128_si256(v1, 1));
|
||||
acc0 = _mm256_add_pd(acc0, _mm256_mul_pd(d0, d0));
|
||||
acc1 = _mm256_add_pd(acc1, _mm256_mul_pd(d1, d1));
|
||||
acc2 = _mm256_add_pd(acc2, _mm256_mul_pd(d2, d2));
|
||||
acc3 = _mm256_add_pd(acc3, _mm256_mul_pd(d3, d3));
|
||||
}
|
||||
acc0 = _mm256_add_pd(_mm256_add_pd(acc0, acc1), _mm256_add_pd(acc2, acc3));
|
||||
double lanes[4];
|
||||
_mm256_storeu_pd(lanes, acc0);
|
||||
double result = lanes[0] + lanes[1] + lanes[2] + lanes[3];
|
||||
for (; x < cols; x++)
|
||||
{
|
||||
double value = (double)src[x];
|
||||
result += value * value;
|
||||
}
|
||||
dstmat.ptr<double>(y)[0] = result;
|
||||
}
|
||||
});
|
||||
v_cleanup();
|
||||
}
|
||||
|
||||
static void sum2_16s64fC1(const Mat& srcmat, Mat& dstmat)
|
||||
{
|
||||
const int cols = srcmat.cols;
|
||||
parallel_for_(Range(0, srcmat.rows), [&](const Range& range) {
|
||||
for (int y = range.start; y < range.end; y++)
|
||||
{
|
||||
const short* src = srcmat.ptr<short>(y);
|
||||
__m256d acc0 = _mm256_setzero_pd(), acc1 = acc0, acc2 = acc0, acc3 = acc0;
|
||||
int x = 0;
|
||||
for (; x <= cols - 16; x += 16)
|
||||
{
|
||||
__m256i v = _mm256_loadu_si256((const __m256i*)(src + x));
|
||||
__m256i v0 = _mm256_cvtepi16_epi32(_mm256_castsi256_si128(v));
|
||||
__m256i v1 = _mm256_cvtepi16_epi32(_mm256_extracti128_si256(v, 1));
|
||||
__m256d d0 = _mm256_cvtepi32_pd(_mm256_castsi256_si128(v0));
|
||||
__m256d d1 = _mm256_cvtepi32_pd(_mm256_extracti128_si256(v0, 1));
|
||||
__m256d d2 = _mm256_cvtepi32_pd(_mm256_castsi256_si128(v1));
|
||||
__m256d d3 = _mm256_cvtepi32_pd(_mm256_extracti128_si256(v1, 1));
|
||||
acc0 = _mm256_add_pd(acc0, _mm256_mul_pd(d0, d0));
|
||||
acc1 = _mm256_add_pd(acc1, _mm256_mul_pd(d1, d1));
|
||||
acc2 = _mm256_add_pd(acc2, _mm256_mul_pd(d2, d2));
|
||||
acc3 = _mm256_add_pd(acc3, _mm256_mul_pd(d3, d3));
|
||||
}
|
||||
acc0 = _mm256_add_pd(_mm256_add_pd(acc0, acc1), _mm256_add_pd(acc2, acc3));
|
||||
double lanes[4];
|
||||
_mm256_storeu_pd(lanes, acc0);
|
||||
double result = lanes[0] + lanes[1] + lanes[2] + lanes[3];
|
||||
for (; x < cols; x++)
|
||||
{
|
||||
double value = (double)src[x];
|
||||
result += value * value;
|
||||
}
|
||||
dstmat.ptr<double>(y)[0] = result;
|
||||
}
|
||||
});
|
||||
v_cleanup();
|
||||
}
|
||||
|
||||
static void sum2_32f64fC1(const Mat& srcmat, Mat& dstmat)
|
||||
{
|
||||
const int cols = srcmat.cols;
|
||||
parallel_for_(Range(0, srcmat.rows), [&](const Range& range) {
|
||||
for (int y = range.start; y < range.end; y++)
|
||||
{
|
||||
const float* src = srcmat.ptr<float>(y);
|
||||
__m256d acc0 = _mm256_setzero_pd(), acc1 = acc0;
|
||||
int x = 0;
|
||||
for (; x <= cols - 8; x += 8)
|
||||
{
|
||||
__m256 v = _mm256_loadu_ps(src + x);
|
||||
__m256d v0 = _mm256_cvtps_pd(_mm256_castps256_ps128(v));
|
||||
__m256d v1 = _mm256_cvtps_pd(_mm256_extractf128_ps(v, 1));
|
||||
acc0 = _mm256_add_pd(acc0, _mm256_mul_pd(v0, v0));
|
||||
acc1 = _mm256_add_pd(acc1, _mm256_mul_pd(v1, v1));
|
||||
}
|
||||
acc0 = _mm256_add_pd(acc0, acc1);
|
||||
double lanes[4];
|
||||
_mm256_storeu_pd(lanes, acc0);
|
||||
double result = lanes[0] + lanes[1] + lanes[2] + lanes[3];
|
||||
for (; x < cols; x++)
|
||||
{
|
||||
double value = (double)src[x];
|
||||
result += value * value;
|
||||
}
|
||||
dstmat.ptr<double>(y)[0] = result;
|
||||
}
|
||||
});
|
||||
v_cleanup();
|
||||
}
|
||||
|
||||
static void sum2_64f64fC1(const Mat& srcmat, Mat& dstmat)
|
||||
{
|
||||
const int cols = srcmat.cols;
|
||||
parallel_for_(Range(0, srcmat.rows), [&](const Range& range) {
|
||||
for (int y = range.start; y < range.end; y++)
|
||||
{
|
||||
const double* src = srcmat.ptr<double>(y);
|
||||
__m256d acc0 = _mm256_setzero_pd(), acc1 = acc0, acc2 = acc0, acc3 = acc0;
|
||||
int x = 0;
|
||||
for (; x <= cols - 16; x += 16)
|
||||
{
|
||||
__m256d v0 = _mm256_loadu_pd(src + x);
|
||||
__m256d v1 = _mm256_loadu_pd(src + x + 4);
|
||||
__m256d v2 = _mm256_loadu_pd(src + x + 8);
|
||||
__m256d v3 = _mm256_loadu_pd(src + x + 12);
|
||||
acc0 = _mm256_add_pd(acc0, _mm256_mul_pd(v0, v0));
|
||||
acc1 = _mm256_add_pd(acc1, _mm256_mul_pd(v1, v1));
|
||||
acc2 = _mm256_add_pd(acc2, _mm256_mul_pd(v2, v2));
|
||||
acc3 = _mm256_add_pd(acc3, _mm256_mul_pd(v3, v3));
|
||||
}
|
||||
acc0 = _mm256_add_pd(_mm256_add_pd(acc0, acc1), _mm256_add_pd(acc2, acc3));
|
||||
double lanes[4];
|
||||
_mm256_storeu_pd(lanes, acc0);
|
||||
double result = lanes[0] + lanes[1] + lanes[2] + lanes[3];
|
||||
for (; x < cols; x++)
|
||||
result += src[x] * src[x];
|
||||
dstmat.ptr<double>(y)[0] = result;
|
||||
}
|
||||
});
|
||||
v_cleanup();
|
||||
}
|
||||
|
||||
template<typename DT>
|
||||
static void sum2_8uC3(const Mat& srcmat, Mat& dstmat)
|
||||
{
|
||||
const int cols = srcmat.cols;
|
||||
const __m256i mask0 = _mm256_setr_epi8(
|
||||
0, 3, 6, 9, 12, 15, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
|
||||
2, 5, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1);
|
||||
const __m256i mask1 = _mm256_setr_epi8(
|
||||
1, 4, 7, 10, 13, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
|
||||
0, 3, 6, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1);
|
||||
const __m256i mask2 = _mm256_setr_epi8(
|
||||
2, 5, 8, 11, 14, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
|
||||
1, 4, 7, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1);
|
||||
|
||||
parallel_for_(Range(0, srcmat.rows), [&](const Range& range) {
|
||||
for (int y = range.start; y < range.end; y++)
|
||||
{
|
||||
const uchar* src = srcmat.ptr<uchar>(y);
|
||||
DT* dst = dstmat.ptr<DT>(y);
|
||||
__m256i acc0 = _mm256_setzero_si256();
|
||||
__m256i acc1 = _mm256_setzero_si256();
|
||||
__m256i acc2 = _mm256_setzero_si256();
|
||||
int x = 0;
|
||||
for (; x <= cols - 11; x += 8)
|
||||
{
|
||||
__m256i v = _mm256_loadu_si256((const __m256i*)(src + x * 3));
|
||||
__m256i channels[3] = {
|
||||
_mm256_shuffle_epi8(v, mask0),
|
||||
_mm256_shuffle_epi8(v, mask1),
|
||||
_mm256_shuffle_epi8(v, mask2)
|
||||
};
|
||||
__m256i* accs[3] = {&acc0, &acc1, &acc2};
|
||||
for (int c = 0; c < 3; c++)
|
||||
{
|
||||
__m128i lo = _mm256_castsi256_si128(channels[c]);
|
||||
__m128i hi = _mm256_extracti128_si256(channels[c], 1);
|
||||
__m256i lo16 = _mm256_cvtepu8_epi16(lo);
|
||||
__m256i hi16 = _mm256_cvtepu8_epi16(hi);
|
||||
*accs[c] = _mm256_add_epi32(*accs[c], _mm256_madd_epi16(lo16, lo16));
|
||||
*accs[c] = _mm256_add_epi32(*accs[c], _mm256_madd_epi16(hi16, hi16));
|
||||
}
|
||||
}
|
||||
|
||||
__m256i accs[3] = {acc0, acc1, acc2};
|
||||
for (int c = 0; c < 3; c++)
|
||||
{
|
||||
uint32_t lanes[8];
|
||||
uint32_t result = 0;
|
||||
_mm256_storeu_si256((__m256i*)lanes, accs[c]);
|
||||
for (int i = 0; i < 8; i++)
|
||||
result += lanes[i];
|
||||
for (int i = x; i < cols; i++)
|
||||
{
|
||||
uint32_t value = src[i * 3 + c];
|
||||
result += value * value;
|
||||
}
|
||||
dst[c] = (DT)(int32_t)result;
|
||||
}
|
||||
}
|
||||
});
|
||||
v_cleanup();
|
||||
}
|
||||
|
||||
template<typename DT>
|
||||
static void sum2_8uC4(const Mat& srcmat, Mat& dstmat)
|
||||
{
|
||||
const int cols = srcmat.cols;
|
||||
const __m256i mask0 = _mm256_broadcastsi128_si256(
|
||||
_mm_setr_epi8(0, 4, 8, 12, -1, -1, -1, -1,
|
||||
-1, -1, -1, -1, -1, -1, -1, -1));
|
||||
const __m256i mask1 = _mm256_broadcastsi128_si256(
|
||||
_mm_setr_epi8(1, 5, 9, 13, -1, -1, -1, -1,
|
||||
-1, -1, -1, -1, -1, -1, -1, -1));
|
||||
const __m256i mask2 = _mm256_broadcastsi128_si256(
|
||||
_mm_setr_epi8(2, 6, 10, 14, -1, -1, -1, -1,
|
||||
-1, -1, -1, -1, -1, -1, -1, -1));
|
||||
const __m256i mask3 = _mm256_broadcastsi128_si256(
|
||||
_mm_setr_epi8(3, 7, 11, 15, -1, -1, -1, -1,
|
||||
-1, -1, -1, -1, -1, -1, -1, -1));
|
||||
|
||||
parallel_for_(Range(0, srcmat.rows), [&](const Range& range) {
|
||||
for (int y = range.start; y < range.end; y++)
|
||||
{
|
||||
const uchar* src = srcmat.ptr<uchar>(y);
|
||||
DT* dst = dstmat.ptr<DT>(y);
|
||||
__m256i acc0 = _mm256_setzero_si256();
|
||||
__m256i acc1 = _mm256_setzero_si256();
|
||||
__m256i acc2 = _mm256_setzero_si256();
|
||||
__m256i acc3 = _mm256_setzero_si256();
|
||||
int x = 0;
|
||||
for (; x <= cols - 8; x += 8)
|
||||
{
|
||||
__m256i v = _mm256_loadu_si256((const __m256i*)(src + x * 4));
|
||||
__m256i channels[4] = {
|
||||
_mm256_shuffle_epi8(v, mask0),
|
||||
_mm256_shuffle_epi8(v, mask1),
|
||||
_mm256_shuffle_epi8(v, mask2),
|
||||
_mm256_shuffle_epi8(v, mask3)
|
||||
};
|
||||
__m256i* accs[4] = {&acc0, &acc1, &acc2, &acc3};
|
||||
for (int c = 0; c < 4; c++)
|
||||
{
|
||||
__m128i lo = _mm256_castsi256_si128(channels[c]);
|
||||
__m128i hi = _mm256_extracti128_si256(channels[c], 1);
|
||||
__m256i lo16 = _mm256_cvtepu8_epi16(lo);
|
||||
__m256i hi16 = _mm256_cvtepu8_epi16(hi);
|
||||
*accs[c] = _mm256_add_epi32(*accs[c], _mm256_madd_epi16(lo16, lo16));
|
||||
*accs[c] = _mm256_add_epi32(*accs[c], _mm256_madd_epi16(hi16, hi16));
|
||||
}
|
||||
}
|
||||
|
||||
__m256i accs[4] = {acc0, acc1, acc2, acc3};
|
||||
for (int c = 0; c < 4; c++)
|
||||
{
|
||||
uint32_t lanes[8];
|
||||
uint32_t result = 0;
|
||||
_mm256_storeu_si256((__m256i*)lanes, accs[c]);
|
||||
for (int i = 0; i < 8; i++)
|
||||
result += lanes[i];
|
||||
for (int i = x; i < cols; i++)
|
||||
{
|
||||
uint32_t value = src[i * 4 + c];
|
||||
result += value * value;
|
||||
}
|
||||
dst[c] = (DT)(int32_t)result;
|
||||
}
|
||||
}
|
||||
});
|
||||
v_cleanup();
|
||||
}
|
||||
|
||||
static void sum2_32fC1(const Mat& srcmat, Mat& dstmat)
|
||||
{
|
||||
const int cols = srcmat.cols;
|
||||
parallel_for_(Range(0, srcmat.rows), [&](const Range& range) {
|
||||
for (int y = range.start; y < range.end; y++)
|
||||
{
|
||||
const float* src = srcmat.ptr<float>(y);
|
||||
float result = 0;
|
||||
int x = 0;
|
||||
__m256 acc = _mm256_setzero_ps();
|
||||
for (; x <= cols - 8; x += 8)
|
||||
{
|
||||
__m256 v = _mm256_loadu_ps(src + x);
|
||||
acc = _mm256_add_ps(acc, _mm256_mul_ps(v, v));
|
||||
}
|
||||
float lanes[8];
|
||||
_mm256_storeu_ps(lanes, acc);
|
||||
for (int i = 0; i < 8; i++)
|
||||
result += lanes[i];
|
||||
for (; x < cols; x++)
|
||||
result += src[x] * src[x];
|
||||
dstmat.ptr<float>(y)[0] = result;
|
||||
}
|
||||
});
|
||||
v_cleanup();
|
||||
}
|
||||
|
||||
static void sum2_32fC4(const Mat& srcmat, Mat& dstmat)
|
||||
{
|
||||
const int cols = srcmat.cols;
|
||||
const __m256 zero = _mm256_setzero_ps();
|
||||
const __m256 validMask = _mm256_castsi256_ps(_mm256_setr_epi32(-1, -1, 0, 0, 0, 0, 0, 0));
|
||||
const __m256i idx0 = _mm256_setr_epi32(0, 4, 0, 0, 0, 0, 0, 0);
|
||||
const __m256i idx1 = _mm256_setr_epi32(1, 5, 0, 0, 0, 0, 0, 0);
|
||||
const __m256i idx2 = _mm256_setr_epi32(2, 6, 0, 0, 0, 0, 0, 0);
|
||||
const __m256i idx3 = _mm256_setr_epi32(3, 7, 0, 0, 0, 0, 0, 0);
|
||||
|
||||
parallel_for_(Range(0, srcmat.rows), [&](const Range& range) {
|
||||
for (int y = range.start; y < range.end; y++)
|
||||
{
|
||||
const float* src = srcmat.ptr<float>(y);
|
||||
float* dst = dstmat.ptr<float>(y);
|
||||
__m256 acc0 = zero, acc1 = zero, acc2 = zero, acc3 = zero;
|
||||
int x = 0;
|
||||
for (; x <= cols - 2; x += 2)
|
||||
{
|
||||
__m256 v = _mm256_loadu_ps(src + x * 4);
|
||||
__m256 v0 = _mm256_blendv_ps(zero, _mm256_permutevar8x32_ps(v, idx0), validMask);
|
||||
__m256 v1 = _mm256_blendv_ps(zero, _mm256_permutevar8x32_ps(v, idx1), validMask);
|
||||
__m256 v2 = _mm256_blendv_ps(zero, _mm256_permutevar8x32_ps(v, idx2), validMask);
|
||||
__m256 v3 = _mm256_blendv_ps(zero, _mm256_permutevar8x32_ps(v, idx3), validMask);
|
||||
acc0 = _mm256_add_ps(acc0, _mm256_mul_ps(v0, v0));
|
||||
acc1 = _mm256_add_ps(acc1, _mm256_mul_ps(v1, v1));
|
||||
acc2 = _mm256_add_ps(acc2, _mm256_mul_ps(v2, v2));
|
||||
acc3 = _mm256_add_ps(acc3, _mm256_mul_ps(v3, v3));
|
||||
}
|
||||
|
||||
__m256 accs[4] = {acc0, acc1, acc2, acc3};
|
||||
for (int c = 0; c < 4; c++)
|
||||
{
|
||||
float lanes[8];
|
||||
float result = 0;
|
||||
_mm256_storeu_ps(lanes, accs[c]);
|
||||
for (int i = 0; i < 8; i++)
|
||||
result += lanes[i];
|
||||
for (int i = x; i < cols; i++)
|
||||
result += src[i * 4 + c] * src[i * 4 + c];
|
||||
dst[c] = result;
|
||||
}
|
||||
}
|
||||
});
|
||||
v_cleanup();
|
||||
}
|
||||
|
||||
} // namespace reduce_c_avx2
|
||||
@@ -0,0 +1,701 @@
|
||||
// This file is part of OpenCV project.
|
||||
// It is subject to the license terms in the LICENSE file found in the top-level directory
|
||||
// of this distribution and at http://opencv.org/license.html
|
||||
|
||||
#if (CV_SIMD || CV_SIMD_SCALABLE)
|
||||
|
||||
template<typename T, typename VT>
|
||||
static inline VT vx_load_strided(const T *ptr, size_t step)
|
||||
{
|
||||
constexpr int nlanes = VTraits<VT>::max_nlanes;
|
||||
T buf[nlanes];
|
||||
for (int i = 0; i < VTraits<VT>::vlanes(); i++)
|
||||
{
|
||||
buf[i] = *ptr;
|
||||
ptr += step;
|
||||
}
|
||||
return vx_load(buf);
|
||||
}
|
||||
|
||||
#if CV_RVV
|
||||
template<>
|
||||
inline v_uint8 vx_load_strided(const uchar *ptr, size_t step)
|
||||
{
|
||||
return __riscv_vlse8_v_u8m2(ptr, step * sizeof(uchar), __riscv_vsetvlmax_e8m2());
|
||||
}
|
||||
template<>
|
||||
inline v_uint16 vx_load_strided(const ushort *ptr, size_t step)
|
||||
{
|
||||
return __riscv_vlse16_v_u16m2(ptr, step * sizeof(ushort), __riscv_vsetvlmax_e16m2());
|
||||
}
|
||||
template<>
|
||||
inline v_int16 vx_load_strided(const short *ptr, size_t step)
|
||||
{
|
||||
return __riscv_vlse16_v_i16m2(ptr, step * sizeof(short), __riscv_vsetvlmax_e16m2());
|
||||
}
|
||||
template<>
|
||||
inline v_int32 vx_load_strided(const int *ptr, size_t step)
|
||||
{
|
||||
return __riscv_vlse32_v_i32m2(ptr, step * sizeof(int), __riscv_vsetvlmax_e32m2());
|
||||
}
|
||||
template<>
|
||||
inline v_float32 vx_load_strided(const float *ptr, size_t step)
|
||||
{
|
||||
return __riscv_vlse32_v_f32m2(ptr, step * sizeof(float), __riscv_vsetvlmax_e32m2());
|
||||
}
|
||||
template<>
|
||||
inline v_float64 vx_load_strided(const double *ptr, size_t step)
|
||||
{
|
||||
return __riscv_vlse64_v_f64m2(ptr, step * sizeof(double), __riscv_vsetvlmax_e64m2());
|
||||
}
|
||||
#endif
|
||||
#endif
|
||||
template<typename stype, typename itype>
|
||||
struct ReduceOpAddSqr
|
||||
{
|
||||
using v_stype = stype;
|
||||
using v_itype = itype;
|
||||
static const int vlanes;
|
||||
static inline stype load(const stype *ptr, size_t step) { (void)step; return *ptr; }
|
||||
static inline itype init() { return (itype)0; }
|
||||
static inline itype reduce(const itype &val) { return val; }
|
||||
inline itype operator()(const itype &a, const stype &b) const { return a + (itype)b * (itype)b; }
|
||||
};
|
||||
template<typename stype, typename itype>
|
||||
const int ReduceOpAddSqr<stype, itype>::vlanes = 1;
|
||||
|
||||
#if (CV_SIMD || CV_SIMD_SCALABLE)
|
||||
|
||||
template<typename stype, typename itype>
|
||||
struct ReduceVecOpAddSqr;
|
||||
|
||||
template<>
|
||||
struct ReduceVecOpAddSqr<uchar, int>
|
||||
{
|
||||
using stype = uchar;
|
||||
using itype = int;
|
||||
using v_stype = v_uint8;
|
||||
using v_itype = v_int32;
|
||||
static const int vlanes;
|
||||
static inline v_stype load(const stype *ptr, size_t step) { return vx_load_strided<stype, v_stype>(ptr, step); }
|
||||
static inline v_itype init() { return vx_setzero_s32(); }
|
||||
static inline itype reduce(const v_itype &val) { return v_reduce_sum(val); }
|
||||
inline v_itype operator()(const v_itype &a, const v_stype &b) const
|
||||
{
|
||||
v_uint16 b0, b1;
|
||||
v_mul_expand(b, b, b0, b1);
|
||||
|
||||
v_uint32 s00, s01;
|
||||
v_expand(b0, s00, s01);
|
||||
s00 = v_add(s00, s01);
|
||||
v_uint32 s10, s11;
|
||||
v_expand(b1, s10, s11);
|
||||
s10 = v_add(s10, s11);
|
||||
return v_add(a, v_reinterpret_as_s32(v_add(s00, s10)));
|
||||
}
|
||||
};
|
||||
const int ReduceVecOpAddSqr<uchar, int>::vlanes = VTraits<v_stype>::vlanes();
|
||||
|
||||
template<>
|
||||
struct ReduceVecOpAddSqr<ushort, float>
|
||||
{
|
||||
using stype = ushort;
|
||||
using itype = float;
|
||||
using v_stype = v_uint16;
|
||||
using v_itype = v_float32;
|
||||
static const int vlanes;
|
||||
static inline v_stype load(const stype *ptr, size_t step) { return vx_load_strided<stype, v_stype>(ptr, step); }
|
||||
static inline v_itype init() { return vx_setzero_f32(); }
|
||||
static inline itype reduce(const v_itype &val) { return v_reduce_sum(val); }
|
||||
inline v_itype operator()(const v_itype &a, const v_stype &b) const
|
||||
{
|
||||
v_uint32 b0, b1;
|
||||
v_mul_expand(b, b, b0, b1);
|
||||
v_int32 sb = v_reinterpret_as_s32(v_add(b0, b1));
|
||||
return v_add(a, v_cvt_f32(sb));
|
||||
}
|
||||
};
|
||||
const int ReduceVecOpAddSqr<ushort, float>::vlanes = VTraits<v_stype>::vlanes();
|
||||
|
||||
template<>
|
||||
struct ReduceVecOpAddSqr<short, float>
|
||||
{
|
||||
using stype = short;
|
||||
using itype = float;
|
||||
using v_stype = v_int16;
|
||||
using v_itype = v_float32;
|
||||
static const int vlanes;
|
||||
static inline v_stype load(const stype *ptr, size_t step) { return vx_load_strided<stype, v_stype>(ptr, step); }
|
||||
static inline v_itype init() { return vx_setzero_f32(); }
|
||||
static inline itype reduce(const v_itype &val) { return v_reduce_sum(val); }
|
||||
inline v_itype operator()(const v_itype &a, const v_stype &b) const
|
||||
{
|
||||
v_int32 b0, b1;
|
||||
v_mul_expand(b, b, b0, b1);
|
||||
v_int32 sb = v_add(b0, b1);
|
||||
return v_add(a, v_cvt_f32(sb));
|
||||
}
|
||||
};
|
||||
const int ReduceVecOpAddSqr<short, float>::vlanes = VTraits<v_stype>::vlanes();
|
||||
|
||||
template<>
|
||||
struct ReduceVecOpAddSqr<float, float>
|
||||
{
|
||||
using stype = float;
|
||||
using itype = float;
|
||||
using v_stype = v_float32;
|
||||
using v_itype = v_float32;
|
||||
static const int vlanes;
|
||||
static inline v_stype load(const stype *ptr, size_t step) { return vx_load_strided<stype, v_stype>(ptr, step); }
|
||||
static inline v_itype init() { return vx_setzero_f32(); }
|
||||
static inline itype reduce(const v_itype &val) { return v_reduce_sum(val); }
|
||||
inline v_itype operator()(const v_itype &a, const v_stype &b) const { return v_add(a, v_mul(b, b)); }
|
||||
};
|
||||
const int ReduceVecOpAddSqr<float, float>::vlanes = VTraits<v_stype>::vlanes();
|
||||
|
||||
#endif
|
||||
|
||||
#if (CV_SIMD_64F || CV_SIMD_SCALABLE_64F)
|
||||
|
||||
template<>
|
||||
struct ReduceVecOpAddSqr<short, double>
|
||||
{
|
||||
using stype = short;
|
||||
using itype = double;
|
||||
using v_stype = v_int16;
|
||||
using v_itype = v_float64;
|
||||
static const int vlanes;
|
||||
static inline v_stype load(const stype *ptr, size_t step) { return vx_load_strided<stype, v_stype>(ptr, step); }
|
||||
static inline v_itype init() { return vx_setzero_f64(); }
|
||||
static inline itype reduce(const v_itype &val) { return v_reduce_sum(val); }
|
||||
inline v_itype operator()(const v_itype &a, const v_stype &b) const
|
||||
{
|
||||
v_int32 b0, b1;
|
||||
v_mul_expand(b, b, b0, b1);
|
||||
v_int32 sb = v_add(b0, b1);
|
||||
return v_add(a, v_add(v_cvt_f64(sb), v_cvt_f64_high(sb)));
|
||||
}
|
||||
};
|
||||
const int ReduceVecOpAddSqr<short, double>::vlanes = VTraits<v_stype>::vlanes();
|
||||
|
||||
template<>
|
||||
struct ReduceVecOpAddSqr<ushort, double>
|
||||
{
|
||||
using stype = ushort;
|
||||
using itype = double;
|
||||
using v_stype = v_uint16;
|
||||
using v_itype = v_float64;
|
||||
static const int vlanes;
|
||||
static inline v_stype load(const stype *ptr, size_t step) { return vx_load_strided<stype, v_stype>(ptr, step); }
|
||||
static inline v_itype init() { return vx_setzero_f64(); }
|
||||
static inline itype reduce(const v_itype &val) { return v_reduce_sum(val); }
|
||||
inline v_itype operator()(const v_itype &a, const v_stype &b) const
|
||||
{
|
||||
v_uint32 b0, b1;
|
||||
v_mul_expand(b, b, b0, b1);
|
||||
v_int32 sb = v_reinterpret_as_s32(v_add(b0, b1));
|
||||
return v_add(a, v_add(v_cvt_f64(sb), v_cvt_f64_high(sb)));
|
||||
}
|
||||
};
|
||||
const int ReduceVecOpAddSqr<ushort, double>::vlanes = VTraits<v_stype>::vlanes();
|
||||
|
||||
template<>
|
||||
struct ReduceVecOpAddSqr<float, double>
|
||||
{
|
||||
using stype = float;
|
||||
using itype = double;
|
||||
using v_stype = v_float32;
|
||||
using v_itype = v_float64;
|
||||
static const int vlanes;
|
||||
static inline v_stype load(const stype *ptr, size_t step) { return vx_load_strided<stype, v_stype>(ptr, step); }
|
||||
static inline v_itype init() { return vx_setzero_f64(); }
|
||||
static inline itype reduce(const v_itype &val) { return v_reduce_sum(val); }
|
||||
inline v_itype operator()(const v_itype &a, const v_stype &b) const
|
||||
{
|
||||
v_itype b0 = v_cvt_f64(b), b1 = v_cvt_f64_high(b);
|
||||
return v_add(a, v_add(v_mul(b0, b0), v_mul(b1, b1)));
|
||||
}
|
||||
};
|
||||
const int ReduceVecOpAddSqr<float, double>::vlanes = VTraits<v_stype>::vlanes();
|
||||
|
||||
template<>
|
||||
struct ReduceVecOpAddSqr<double, double>
|
||||
{
|
||||
using stype = double;
|
||||
using itype = double;
|
||||
using v_stype = v_float64;
|
||||
using v_itype = v_float64;
|
||||
static const int vlanes;
|
||||
static inline v_stype load(const stype *ptr, size_t step) { return vx_load_strided<stype, v_stype>(ptr, step); }
|
||||
static inline v_itype init() { return vx_setzero_f64(); }
|
||||
static inline itype reduce(const v_itype &val) { return v_reduce_sum(val); }
|
||||
inline v_itype operator()(const v_itype &a, const v_stype &b) const { return v_add(a, v_mul(b, b)); }
|
||||
};
|
||||
const int ReduceVecOpAddSqr<double, double>::vlanes = VTraits<v_stype>::vlanes();
|
||||
|
||||
#endif
|
||||
|
||||
template<typename stype>
|
||||
struct ReduceOpMax
|
||||
{
|
||||
using v_stype = stype;
|
||||
using v_itype = stype;
|
||||
static const int vlanes;
|
||||
static inline stype load(const stype *ptr, size_t step) { (void)step; return *ptr; }
|
||||
static inline stype init() { return std::numeric_limits<stype>::lowest(); }
|
||||
static inline stype reduce(const stype &val) { return val; }
|
||||
inline stype operator()(const stype &a, const stype &b) const { return std::max(a, b); }
|
||||
};
|
||||
template<typename stype>
|
||||
const int ReduceOpMax<stype>::vlanes = 1;
|
||||
|
||||
#if (CV_SIMD || CV_SIMD_SCALABLE)
|
||||
|
||||
template<typename stype>
|
||||
struct ReduceVecOpMax;
|
||||
|
||||
template<>
|
||||
struct ReduceVecOpMax<uchar>
|
||||
{
|
||||
using stype = uchar;
|
||||
using itype = uchar;
|
||||
using v_stype = v_uint8;
|
||||
using v_itype = v_uint8;
|
||||
static const int vlanes;
|
||||
static inline v_stype load(const stype *ptr, size_t step) { return vx_load_strided<stype, v_stype>(ptr, step); }
|
||||
static inline v_itype init() { return vx_setall_u8(std::numeric_limits<itype>::lowest()); }
|
||||
static inline itype reduce(const v_itype &val) { return v_reduce_max(val); }
|
||||
inline v_itype operator()(const v_itype &a, const v_stype &b) const { return v_max(a, b); }
|
||||
};
|
||||
const int ReduceVecOpMax<uchar>::vlanes = VTraits<v_stype>::vlanes();
|
||||
|
||||
template<>
|
||||
struct ReduceVecOpMax<ushort>
|
||||
{
|
||||
using stype = ushort;
|
||||
using itype = ushort;
|
||||
using v_stype = v_uint16;
|
||||
using v_itype = v_uint16;
|
||||
static const int vlanes;
|
||||
static inline v_stype load(const stype *ptr, size_t step) { return vx_load_strided<stype, v_stype>(ptr, step); }
|
||||
static inline v_itype init() { return vx_setall_u16(std::numeric_limits<itype>::lowest()); }
|
||||
static inline itype reduce(const v_itype &val) { return v_reduce_max(val); }
|
||||
inline v_itype operator()(const v_itype &a, const v_stype &b) const { return v_max(a, b); }
|
||||
};
|
||||
const int ReduceVecOpMax<ushort>::vlanes = VTraits<v_stype>::vlanes();
|
||||
|
||||
template<>
|
||||
struct ReduceVecOpMax<short>
|
||||
{
|
||||
using stype = short;
|
||||
using itype = short;
|
||||
using v_stype = v_int16;
|
||||
using v_itype = v_int16;
|
||||
static const int vlanes;
|
||||
static inline v_stype load(const stype *ptr, size_t step) { return vx_load_strided<stype, v_stype>(ptr, step); }
|
||||
static inline v_itype init() { return vx_setall_s16(std::numeric_limits<itype>::lowest()); }
|
||||
static inline itype reduce(const v_itype &val) { return v_reduce_max(val); }
|
||||
inline v_itype operator()(const v_itype &a, const v_stype &b) const { return v_max(a, b); }
|
||||
};
|
||||
const int ReduceVecOpMax<short>::vlanes = VTraits<v_stype>::vlanes();
|
||||
|
||||
template<>
|
||||
struct ReduceVecOpMax<float>
|
||||
{
|
||||
using stype = float;
|
||||
using itype = float;
|
||||
using v_stype = v_float32;
|
||||
using v_itype = v_float32;
|
||||
static const int vlanes;
|
||||
static inline v_stype load(const stype *ptr, size_t step) { return vx_load_strided<stype, v_stype>(ptr, step); }
|
||||
static inline v_itype init() { return vx_setall_f32(std::numeric_limits<itype>::lowest()); }
|
||||
static inline itype reduce(const v_itype &val) { return v_reduce_max(val); }
|
||||
inline v_itype operator()(const v_itype &a, const v_stype &b) const { return v_max(a, b); }
|
||||
};
|
||||
const int ReduceVecOpMax<float>::vlanes = VTraits<v_stype>::vlanes();
|
||||
|
||||
#endif
|
||||
|
||||
#if (CV_SIMD_64F || CV_SIMD_SCALABLE_64F)
|
||||
|
||||
template<>
|
||||
struct ReduceVecOpMax<double>
|
||||
{
|
||||
using stype = double;
|
||||
using itype = double;
|
||||
using v_stype = v_float64;
|
||||
using v_itype = v_float64;
|
||||
static const int vlanes;
|
||||
static inline v_stype load(const stype *ptr, size_t step) { return vx_load_strided<stype, v_stype>(ptr, step); }
|
||||
static inline v_itype init() { return vx_setall_f64(std::numeric_limits<itype>::lowest()); }
|
||||
static inline itype reduce(const v_itype &val)
|
||||
{
|
||||
constexpr int nlanes = VTraits<v_itype>::max_nlanes;
|
||||
itype buf[nlanes];
|
||||
vx_store(buf, val);
|
||||
itype m = buf[0];
|
||||
for (int i = 1; i < VTraits<v_itype>::vlanes(); i++)
|
||||
{
|
||||
if (m < buf[i]) m = buf[i];
|
||||
}
|
||||
return m;
|
||||
}
|
||||
inline v_itype operator()(const v_itype &a, const v_stype &b) const { return v_max(a, b); }
|
||||
};
|
||||
const int ReduceVecOpMax<double>::vlanes = VTraits<v_stype>::vlanes();
|
||||
|
||||
#endif
|
||||
|
||||
template<typename stype>
|
||||
struct ReduceOpMin
|
||||
{
|
||||
using v_stype = stype;
|
||||
using v_itype = stype;
|
||||
static const int vlanes;
|
||||
static inline stype load(const stype *ptr, size_t step) { (void)step; return *ptr; }
|
||||
static inline stype init() { return std::numeric_limits<stype>::max(); }
|
||||
static inline stype reduce(const stype &val) { return (stype)val; }
|
||||
inline stype operator()(const stype &a, const stype &b) const { return std::min(a, b); }
|
||||
};
|
||||
template<typename stype>
|
||||
const int ReduceOpMin<stype>::vlanes = 1;
|
||||
|
||||
#if (CV_SIMD || CV_SIMD_SCALABLE)
|
||||
|
||||
template<typename stype>
|
||||
struct ReduceVecOpMin;
|
||||
|
||||
template<>
|
||||
struct ReduceVecOpMin<uchar>
|
||||
{
|
||||
using stype = uchar;
|
||||
using itype = uchar;
|
||||
using v_stype = v_uint8;
|
||||
using v_itype = v_uint8;
|
||||
static const int vlanes;
|
||||
static inline v_stype load(const stype *ptr, size_t step) { return vx_load_strided<stype, v_stype>(ptr, step); }
|
||||
static inline v_itype init() { return vx_setall_u8(std::numeric_limits<itype>::max()); }
|
||||
static inline itype reduce(const v_itype &val) { return v_reduce_min(val); }
|
||||
inline v_itype operator()(const v_itype &a, const v_stype &b) const { return v_min(a, b); }
|
||||
};
|
||||
const int ReduceVecOpMin<uchar>::vlanes = VTraits<v_stype>::vlanes();
|
||||
|
||||
template<>
|
||||
struct ReduceVecOpMin<ushort>
|
||||
{
|
||||
using stype = ushort;
|
||||
using itype = ushort;
|
||||
using v_stype = v_uint16;
|
||||
using v_itype = v_uint16;
|
||||
static const int vlanes;
|
||||
static inline v_stype load(const stype *ptr, size_t step) { return vx_load_strided<stype, v_stype>(ptr, step); }
|
||||
static inline v_itype init() { return vx_setall_u16(std::numeric_limits<itype>::max()); }
|
||||
static inline itype reduce(const v_itype &val) { return v_reduce_min(val); }
|
||||
inline v_itype operator()(const v_itype &a, const v_stype &b) const { return v_min(a, b); }
|
||||
};
|
||||
const int ReduceVecOpMin<ushort>::vlanes = VTraits<v_stype>::vlanes();
|
||||
|
||||
template<>
|
||||
struct ReduceVecOpMin<short>
|
||||
{
|
||||
using stype = short;
|
||||
using itype = short;
|
||||
using v_stype = v_int16;
|
||||
using v_itype = v_int16;
|
||||
static const int vlanes;
|
||||
static inline v_stype load(const stype *ptr, size_t step) { return vx_load_strided<stype, v_stype>(ptr, step); }
|
||||
static inline v_itype init() { return vx_setall_s16(std::numeric_limits<itype>::max()); }
|
||||
static inline itype reduce(const v_itype &val) { return v_reduce_min(val); }
|
||||
inline v_itype operator()(const v_itype &a, const v_stype &b) const { return v_min(a, b); }
|
||||
};
|
||||
const int ReduceVecOpMin<short>::vlanes = VTraits<v_stype>::vlanes();
|
||||
|
||||
template<>
|
||||
struct ReduceVecOpMin<float>
|
||||
{
|
||||
using stype = float;
|
||||
using itype = float;
|
||||
using v_stype = v_float32;
|
||||
using v_itype = v_float32;
|
||||
static const int vlanes;
|
||||
static inline v_stype load(const stype *ptr, size_t step) { return vx_load_strided<stype, v_stype>(ptr, step); }
|
||||
static inline v_itype init() { return vx_setall_f32(std::numeric_limits<itype>::max()); }
|
||||
static inline itype reduce(const v_itype &val) { return v_reduce_min(val); }
|
||||
inline v_itype operator()(const v_itype &a, const v_stype &b) const { return v_min(a, b); }
|
||||
};
|
||||
const int ReduceVecOpMin<float>::vlanes = VTraits<v_stype>::vlanes();
|
||||
|
||||
#endif
|
||||
|
||||
#if (CV_SIMD_64F || CV_SIMD_SCALABLE_64F)
|
||||
|
||||
template<>
|
||||
struct ReduceVecOpMin<double>
|
||||
{
|
||||
using stype = double;
|
||||
using itype = double;
|
||||
using v_stype = v_float64;
|
||||
using v_itype = v_float64;
|
||||
static const int vlanes;
|
||||
static inline v_stype load(const stype *ptr, size_t step) { return vx_load_strided<stype, v_stype>(ptr, step); }
|
||||
static inline v_itype init() { return vx_setall_f64(std::numeric_limits<itype>::max()); }
|
||||
static inline itype reduce(const v_itype &val)
|
||||
{
|
||||
constexpr int nlanes = VTraits<v_itype>::max_nlanes;
|
||||
itype buf[nlanes];
|
||||
vx_store(buf, val);
|
||||
itype m = buf[0];
|
||||
for (int i = 1; i < VTraits<v_itype>::vlanes(); i++)
|
||||
{
|
||||
if (m > buf[i]) m = buf[i];
|
||||
}
|
||||
return m;
|
||||
}
|
||||
inline v_itype operator()(const v_itype &a, const v_stype &b) const { return v_min(a, b); }
|
||||
};
|
||||
const int ReduceVecOpMin<double>::vlanes = VTraits<v_stype>::vlanes();
|
||||
|
||||
#endif
|
||||
|
||||
|
||||
using ReduceOpAddSqr_8U32S = ReduceOpAddSqr<uchar, int>;
|
||||
using ReduceOpAddSqr_8U32F = ReduceOpAddSqr<uchar, int>;
|
||||
using ReduceOpAddSqr_8U64F = ReduceOpAddSqr<uchar, int>;
|
||||
using ReduceOpAddSqr_16U32F = ReduceOpAddSqr<ushort, float>;
|
||||
using ReduceOpAddSqr_16U64F = ReduceOpAddSqr<ushort, double>;
|
||||
using ReduceOpAddSqr_16S32F = ReduceOpAddSqr<short, float>;
|
||||
using ReduceOpAddSqr_16S64F = ReduceOpAddSqr<short, double>;
|
||||
using ReduceOpAddSqr_32F32F = ReduceOpAddSqr<float, float>;
|
||||
using ReduceOpAddSqr_32F64F = ReduceOpAddSqr<float, double>;
|
||||
using ReduceOpAddSqr_64F64F = ReduceOpAddSqr<double, double>;
|
||||
|
||||
using ReduceOpMax_8U = ReduceOpMax<uchar>;
|
||||
using ReduceOpMax_16U = ReduceOpMax<ushort>;
|
||||
using ReduceOpMax_16S = ReduceOpMax<short>;
|
||||
using ReduceOpMax_32F = ReduceOpMax<float>;
|
||||
using ReduceOpMax_64F = ReduceOpMax<double>;
|
||||
|
||||
using ReduceOpMin_8U = ReduceOpMin<uchar>;
|
||||
using ReduceOpMin_16U = ReduceOpMin<ushort>;
|
||||
using ReduceOpMin_16S = ReduceOpMin<short>;
|
||||
using ReduceOpMin_32F = ReduceOpMin<float>;
|
||||
using ReduceOpMin_64F = ReduceOpMin<double>;
|
||||
|
||||
#if (CV_SIMD || CV_SIMD_SCALABLE)
|
||||
|
||||
using ReduceVecOpAddSqr_8U32S = ReduceVecOpAddSqr<uchar, int>;
|
||||
using ReduceVecOpAddSqr_8U32F = ReduceVecOpAddSqr<uchar, int>;
|
||||
using ReduceVecOpAddSqr_8U64F = ReduceVecOpAddSqr<uchar, int>;
|
||||
using ReduceVecOpAddSqr_16U32F = ReduceVecOpAddSqr<ushort, float>;
|
||||
using ReduceVecOpAddSqr_16S32F = ReduceVecOpAddSqr<short, float>;
|
||||
using ReduceVecOpAddSqr_32F32F = ReduceVecOpAddSqr<float, float>;
|
||||
using ReduceVecOpMax_8U = ReduceVecOpMax<uchar>;
|
||||
using ReduceVecOpMax_16U = ReduceVecOpMax<ushort>;
|
||||
using ReduceVecOpMax_16S = ReduceVecOpMax<short>;
|
||||
using ReduceVecOpMax_32F = ReduceVecOpMax<float>;
|
||||
using ReduceVecOpMin_8U = ReduceVecOpMin<uchar>;
|
||||
using ReduceVecOpMin_16U = ReduceVecOpMin<ushort>;
|
||||
using ReduceVecOpMin_16S = ReduceVecOpMin<short>;
|
||||
using ReduceVecOpMin_32F = ReduceVecOpMin<float>;
|
||||
|
||||
#else
|
||||
|
||||
using ReduceVecOpAddSqr_8U32S = ReduceOpAddSqr<uchar, int>;
|
||||
using ReduceVecOpAddSqr_8U32F = ReduceOpAddSqr<uchar, int>;
|
||||
using ReduceVecOpAddSqr_8U64F = ReduceOpAddSqr<uchar, int>;
|
||||
using ReduceVecOpAddSqr_16U32F = ReduceOpAddSqr<ushort, float>;
|
||||
using ReduceVecOpAddSqr_16S32F = ReduceOpAddSqr<short, float>;
|
||||
using ReduceVecOpAddSqr_32F32F = ReduceOpAddSqr<float, float>;
|
||||
using ReduceVecOpMax_8U = ReduceOpMax<uchar>;
|
||||
using ReduceVecOpMax_16U = ReduceOpMax<ushort>;
|
||||
using ReduceVecOpMax_16S = ReduceOpMax<short>;
|
||||
using ReduceVecOpMax_32F = ReduceOpMax<float>;
|
||||
using ReduceVecOpMin_8U = ReduceOpMin<uchar>;
|
||||
using ReduceVecOpMin_16U = ReduceOpMin<ushort>;
|
||||
using ReduceVecOpMin_16S = ReduceOpMin<short>;
|
||||
using ReduceVecOpMin_32F = ReduceOpMin<float>;
|
||||
|
||||
#endif
|
||||
|
||||
#if (CV_SIMD_64F || CV_SIMD_SCALABLE_64F)
|
||||
|
||||
using ReduceVecOpAddSqr_16U64F = ReduceVecOpAddSqr<ushort, double>;
|
||||
using ReduceVecOpAddSqr_16S64F = ReduceVecOpAddSqr<short, double>;
|
||||
using ReduceVecOpAddSqr_32F64F = ReduceVecOpAddSqr<float, double>;
|
||||
using ReduceVecOpAddSqr_64F64F = ReduceVecOpAddSqr<double, double>;
|
||||
using ReduceVecOpMax_64F = ReduceVecOpMax<double>;
|
||||
using ReduceVecOpMin_64F = ReduceVecOpMin<double>;
|
||||
|
||||
#else
|
||||
|
||||
using ReduceVecOpAddSqr_16U64F = ReduceOpAddSqr<ushort, double>;
|
||||
using ReduceVecOpAddSqr_16S64F = ReduceOpAddSqr<short, double>;
|
||||
using ReduceVecOpAddSqr_32F64F = ReduceOpAddSqr<float, double>;
|
||||
using ReduceVecOpAddSqr_64F64F = ReduceOpAddSqr<double, double>;
|
||||
using ReduceVecOpMax_64F = ReduceOpMax<double>;
|
||||
using ReduceVecOpMin_64F = ReduceOpMin<double>;
|
||||
|
||||
#endif
|
||||
|
||||
template<typename T, typename ST, class Op, class VecOp>
|
||||
class ReduceC_Invoker : public ParallelLoopBody
|
||||
{
|
||||
using WT = typename Op::v_itype;
|
||||
using VT = typename VecOp::v_itype;
|
||||
public:
|
||||
ReduceC_Invoker(const Mat& aSrcmat, Mat& aDstmat, Op& aOp, VecOp& aVop)
|
||||
:srcmat(aSrcmat),dstmat(aDstmat),op(aOp),vop(aVop)
|
||||
{
|
||||
}
|
||||
void operator()(const Range& range) const CV_OVERRIDE
|
||||
{
|
||||
int channels = srcmat.channels();
|
||||
int width = srcmat.cols;
|
||||
|
||||
const int nlanes = VecOp::vlanes;
|
||||
|
||||
for (int h = range.start; h < range.end; h++)
|
||||
{
|
||||
const T *srcrow = srcmat.ptr<T>(h);
|
||||
ST *dst = dstmat.ptr<ST>(h);
|
||||
for (int cn = 0; cn < channels; cn++)
|
||||
{
|
||||
const T *src = srcrow + cn;
|
||||
VT vbuf = vop.init();
|
||||
int w = 0;
|
||||
for (; w <= width - nlanes; w += nlanes)
|
||||
{
|
||||
vbuf = vop(vbuf, vop.load(src+w*channels, channels));
|
||||
}
|
||||
WT wbuf = vop.reduce(vbuf);
|
||||
for (; w < width; w++)
|
||||
{
|
||||
wbuf = op(wbuf, op.load(src+w*channels, channels));
|
||||
}
|
||||
dst[cn] = (ST)op.reduce(wbuf);
|
||||
}
|
||||
}
|
||||
}
|
||||
private:
|
||||
const Mat& srcmat;
|
||||
Mat& dstmat;
|
||||
Op& op;
|
||||
VecOp& vop;
|
||||
};
|
||||
|
||||
template<typename T, typename ST, class Op, class VecOp> static void
|
||||
reduceColGeneric(const Mat& srcmat, Mat& dstmat)
|
||||
{
|
||||
Op op;
|
||||
VecOp vop;
|
||||
|
||||
ReduceC_Invoker<T, ST, Op, VecOp> body(srcmat, dstmat, op, vop);
|
||||
parallel_for_(Range(0, srcmat.size().height), body);
|
||||
}
|
||||
|
||||
template<bool isMax>
|
||||
static inline uchar reduceScalarMinMax(uchar a, uchar b)
|
||||
{
|
||||
return isMax ? std::max(a, b) : std::min(a, b);
|
||||
}
|
||||
|
||||
template<bool isMax>
|
||||
static void reduceColMinMax_8uFallback(const Mat& srcmat, Mat& dstmat)
|
||||
{
|
||||
if (isMax)
|
||||
reduceColGeneric<uchar, uchar, ReduceOpMax_8U, ReduceVecOpMax_8U>(srcmat, dstmat);
|
||||
else
|
||||
reduceColGeneric<uchar, uchar, ReduceOpMin_8U, ReduceVecOpMin_8U>(srcmat, dstmat);
|
||||
}
|
||||
|
||||
template<bool isMax>
|
||||
static void reduceColMinMax_16uFallback(const Mat& srcmat, Mat& dstmat)
|
||||
{
|
||||
if (isMax)
|
||||
reduceColGeneric<ushort, ushort, ReduceOpMax_16U, ReduceVecOpMax_16U>(srcmat, dstmat);
|
||||
else
|
||||
reduceColGeneric<ushort, ushort, ReduceOpMin_16U, ReduceVecOpMin_16U>(srcmat, dstmat);
|
||||
}
|
||||
|
||||
template<bool isMax>
|
||||
static void reduceColMinMax_16sFallback(const Mat& srcmat, Mat& dstmat)
|
||||
{
|
||||
if (isMax)
|
||||
reduceColGeneric<short, short, ReduceOpMax_16S, ReduceVecOpMax_16S>(srcmat, dstmat);
|
||||
else
|
||||
reduceColGeneric<short, short, ReduceOpMin_16S, ReduceVecOpMin_16S>(srcmat, dstmat);
|
||||
}
|
||||
|
||||
template<bool isMax>
|
||||
static inline float reduceScalarMinMax(float a, float b)
|
||||
{
|
||||
return isMax ? std::max(a, b) : std::min(a, b);
|
||||
}
|
||||
|
||||
template<bool isMax>
|
||||
static void reduceColMinMax_32fFallback(const Mat& srcmat, Mat& dstmat)
|
||||
{
|
||||
if (isMax)
|
||||
reduceColGeneric<float, float, ReduceOpMax_32F, ReduceVecOpMax_32F>(srcmat, dstmat);
|
||||
else
|
||||
reduceColGeneric<float, float, ReduceOpMin_32F, ReduceVecOpMin_32F>(srcmat, dstmat);
|
||||
}
|
||||
|
||||
#if (CV_SIMD_64F || CV_SIMD_SCALABLE_64F)
|
||||
template<bool isMax>
|
||||
static void reduceColMinMax_64fFallback(const Mat& srcmat, Mat& dstmat)
|
||||
{
|
||||
if (isMax)
|
||||
reduceColGeneric<double, double, ReduceOpMax_64F, ReduceVecOpMax_64F>(srcmat, dstmat);
|
||||
else
|
||||
reduceColGeneric<double, double, ReduceOpMin_64F, ReduceVecOpMin_64F>(srcmat, dstmat);
|
||||
}
|
||||
#endif
|
||||
|
||||
template<typename DT>
|
||||
static void reduceColSum2_8uFallback(const Mat& srcmat, Mat& dstmat)
|
||||
{
|
||||
if (std::is_same<DT, int>::value)
|
||||
reduceColGeneric<uchar, int, ReduceOpAddSqr_8U32S, ReduceVecOpAddSqr_8U32S>(srcmat, dstmat);
|
||||
else if (std::is_same<DT, float>::value)
|
||||
reduceColGeneric<uchar, float, ReduceOpAddSqr_8U32F, ReduceVecOpAddSqr_8U32F>(srcmat, dstmat);
|
||||
else
|
||||
reduceColGeneric<uchar, double, ReduceOpAddSqr_8U64F, ReduceVecOpAddSqr_8U64F>(srcmat, dstmat);
|
||||
}
|
||||
|
||||
static void reduceColSum2_16u32fFallback(const Mat& srcmat, Mat& dstmat)
|
||||
{
|
||||
reduceColGeneric<ushort, float, ReduceOpAddSqr_16U32F, ReduceVecOpAddSqr_16U32F>(srcmat, dstmat);
|
||||
}
|
||||
|
||||
static void reduceColSum2_16s32fFallback(const Mat& srcmat, Mat& dstmat)
|
||||
{
|
||||
reduceColGeneric<short, float, ReduceOpAddSqr_16S32F, ReduceVecOpAddSqr_16S32F>(srcmat, dstmat);
|
||||
}
|
||||
|
||||
static void reduceColSum2_32f32fFallback(const Mat& srcmat, Mat& dstmat)
|
||||
{
|
||||
reduceColGeneric<float, float, ReduceOpAddSqr_32F32F, ReduceVecOpAddSqr_32F32F>(srcmat, dstmat);
|
||||
}
|
||||
|
||||
#if (CV_SIMD_64F || CV_SIMD_SCALABLE_64F)
|
||||
static void reduceColSum2_16u64fFallback(const Mat& srcmat, Mat& dstmat)
|
||||
{
|
||||
reduceColGeneric<ushort, double, ReduceOpAddSqr_16U64F, ReduceVecOpAddSqr_16U64F>(srcmat, dstmat);
|
||||
}
|
||||
|
||||
static void reduceColSum2_16s64fFallback(const Mat& srcmat, Mat& dstmat)
|
||||
{
|
||||
reduceColGeneric<short, double, ReduceOpAddSqr_16S64F, ReduceVecOpAddSqr_16S64F>(srcmat, dstmat);
|
||||
}
|
||||
|
||||
static void reduceColSum2_32f64fFallback(const Mat& srcmat, Mat& dstmat)
|
||||
{
|
||||
reduceColGeneric<float, double, ReduceOpAddSqr_32F64F, ReduceVecOpAddSqr_32F64F>(srcmat, dstmat);
|
||||
}
|
||||
|
||||
static void reduceColSum2_64f64fFallback(const Mat& srcmat, Mat& dstmat)
|
||||
{
|
||||
reduceColGeneric<double, double, ReduceOpAddSqr_64F64F, ReduceVecOpAddSqr_64F64F>(srcmat, dstmat);
|
||||
}
|
||||
#endif
|
||||
@@ -0,0 +1,827 @@
|
||||
// This file is part of OpenCV project.
|
||||
// It is subject to the license terms in the LICENSE file found in the top-level directory
|
||||
// of this distribution and at http://opencv.org/license.html
|
||||
|
||||
namespace reduce_c_neon
|
||||
{
|
||||
|
||||
// Optimized ReduceC support in this backend:
|
||||
//
|
||||
// | Input -> output type/channel | SUM | AVG | MIN | MAX | SUM2 |
|
||||
// |------------------------------------|:---:|:---:|:---:|:---:|:----:|
|
||||
// | 8UC1/C3/C4 -> 8UC1/C3/C4 | - | - | x | x | - |
|
||||
// | 8UC1/C3/C4 -> 32SC1/C3/C4 | x | x | - | - | x* |
|
||||
// | 8UC1/C3/C4 -> 32FC1/C3/C4 | x | x | - | - | x* |
|
||||
// | 8UC1/C3/C4 -> 64FC1/C3/C4 | - | - | - | - | x* |
|
||||
// | 16UC1/C3/C4 -> 16UC1/C3/C4 | - | - | x | x | - |
|
||||
// | 16UC1 -> 32FC1 | x | x | - | - | x* |
|
||||
// | 16UC3/C4 -> 32FC3/C4 | x | x | - | - | - |
|
||||
// | 16UC1 -> 64FC1 | - | - | - | - | x* |
|
||||
// | 16UC3/C4 -> 64FC3/C4 | - | - | - | - | - |
|
||||
// | 16SC1/C3/C4 -> 16SC1/C3/C4 | - | - | x | x | - |
|
||||
// | 16SC1 -> 32FC1 | x | x | - | - | x* |
|
||||
// | 16SC3/C4 -> 32FC3/C4 | x | x | - | - | - |
|
||||
// | 16SC1 -> 64FC1 | - | - | - | - | x* |
|
||||
// | 16SC3/C4 -> 64FC3/C4 | - | - | - | - | - |
|
||||
// | 32FC1 -> 32FC1 | x | x | x | x | x |
|
||||
// | 32FC3 -> 32FC3 | x | x | - | - | - |
|
||||
// | 32FC4 -> 32FC4 | x | x | x | x | x |
|
||||
// | 32FC1 -> 64FC1 | x* | x* | - | - | x* |
|
||||
// | 32FC3/C4 -> 64FC3/C4 | x* | x* | - | - | - |
|
||||
// | 64FC1 -> 64FC1 | x* | x* | x* | x* | x* |
|
||||
// | 64FC3/C4 -> 64FC3/C4 | x* | x* | - | - | - |
|
||||
//
|
||||
// 'x' in SUM/AVG denotes the existing shared universal-intrinsics kernel; 'x'
|
||||
// in MIN/MAX/SUM2 denotes a native NEON kernel. '*' requires AArch64. For legal
|
||||
// MIN/MAX/SUM2 combinations marked '-', and for other channel counts, dispatch
|
||||
// uses the shared generic fallback.
|
||||
|
||||
template<bool isMax>
|
||||
static void minMax8uC1(const Mat& srcmat, Mat& dstmat)
|
||||
{
|
||||
const int cols = srcmat.cols;
|
||||
parallel_for_(Range(0, srcmat.rows), [&](const Range& range) {
|
||||
for (int y = range.start; y < range.end; y++)
|
||||
{
|
||||
const uchar* src = srcmat.ptr<uchar>(y);
|
||||
uchar* dst = dstmat.ptr<uchar>(y);
|
||||
uchar result = isMax ? 0 : UCHAR_MAX;
|
||||
int x = 0;
|
||||
uint8x16_t acc = vdupq_n_u8(result);
|
||||
for (; x <= cols - 16; x += 16)
|
||||
{
|
||||
uint8x16_t v = vld1q_u8(src + x);
|
||||
acc = isMax ? vmaxq_u8(acc, v) : vminq_u8(acc, v);
|
||||
}
|
||||
uchar lanes[16];
|
||||
vst1q_u8(lanes, acc);
|
||||
for (int i = 0; i < 16; i++)
|
||||
result = reduceScalarMinMax<isMax>(result, lanes[i]);
|
||||
for (; x < cols; x++)
|
||||
result = reduceScalarMinMax<isMax>(result, src[x]);
|
||||
dst[0] = result;
|
||||
}
|
||||
});
|
||||
v_cleanup();
|
||||
}
|
||||
|
||||
template<bool isMax>
|
||||
static void minMax16uC1(const Mat& srcmat, Mat& dstmat)
|
||||
{
|
||||
const int cols = srcmat.cols;
|
||||
parallel_for_(Range(0, srcmat.rows), [&](const Range& range) {
|
||||
for (int y = range.start; y < range.end; y++)
|
||||
{
|
||||
const ushort* src = srcmat.ptr<ushort>(y);
|
||||
ushort result = isMax ? 0 : USHRT_MAX;
|
||||
int x = 0;
|
||||
uint16x8_t acc = vdupq_n_u16(result);
|
||||
for (; x <= cols - 8; x += 8)
|
||||
{
|
||||
uint16x8_t v = vld1q_u16(src + x);
|
||||
acc = isMax ? vmaxq_u16(acc, v) : vminq_u16(acc, v);
|
||||
}
|
||||
ushort lanes[8];
|
||||
vst1q_u16(lanes, acc);
|
||||
for (int i = 0; i < 8; i++)
|
||||
result = isMax ? std::max(result, lanes[i]) : std::min(result, lanes[i]);
|
||||
for (; x < cols; x++)
|
||||
result = isMax ? std::max(result, src[x]) : std::min(result, src[x]);
|
||||
dstmat.ptr<ushort>(y)[0] = result;
|
||||
}
|
||||
});
|
||||
v_cleanup();
|
||||
}
|
||||
|
||||
template<bool isMax>
|
||||
static void minMax16sC1(const Mat& srcmat, Mat& dstmat)
|
||||
{
|
||||
const int cols = srcmat.cols;
|
||||
parallel_for_(Range(0, srcmat.rows), [&](const Range& range) {
|
||||
for (int y = range.start; y < range.end; y++)
|
||||
{
|
||||
const short* src = srcmat.ptr<short>(y);
|
||||
short result = isMax ? SHRT_MIN : SHRT_MAX;
|
||||
int x = 0;
|
||||
int16x8_t acc = vdupq_n_s16(result);
|
||||
for (; x <= cols - 8; x += 8)
|
||||
{
|
||||
int16x8_t v = vld1q_s16(src + x);
|
||||
acc = isMax ? vmaxq_s16(acc, v) : vminq_s16(acc, v);
|
||||
}
|
||||
short lanes[8];
|
||||
vst1q_s16(lanes, acc);
|
||||
for (int i = 0; i < 8; i++)
|
||||
result = isMax ? std::max(result, lanes[i]) : std::min(result, lanes[i]);
|
||||
for (; x < cols; x++)
|
||||
result = isMax ? std::max(result, src[x]) : std::min(result, src[x]);
|
||||
dstmat.ptr<short>(y)[0] = result;
|
||||
}
|
||||
});
|
||||
v_cleanup();
|
||||
}
|
||||
|
||||
template<bool isMax>
|
||||
static void minMax16uC4(const Mat& srcmat, Mat& dstmat)
|
||||
{
|
||||
const int cols = srcmat.cols;
|
||||
const ushort initial = isMax ? 0 : USHRT_MAX;
|
||||
parallel_for_(Range(0, srcmat.rows), [&](const Range& range) {
|
||||
for (int y = range.start; y < range.end; y++)
|
||||
{
|
||||
const ushort* src = srcmat.ptr<ushort>(y);
|
||||
ushort* dst = dstmat.ptr<ushort>(y);
|
||||
uint16x8x4_t acc = {{
|
||||
vdupq_n_u16(initial), vdupq_n_u16(initial),
|
||||
vdupq_n_u16(initial), vdupq_n_u16(initial)
|
||||
}};
|
||||
int x = 0;
|
||||
for (; x <= cols - 8; x += 8)
|
||||
{
|
||||
uint16x8x4_t v = vld4q_u16(src + x * 4);
|
||||
for (int c = 0; c < 4; c++)
|
||||
acc.val[c] = isMax ? vmaxq_u16(acc.val[c], v.val[c])
|
||||
: vminq_u16(acc.val[c], v.val[c]);
|
||||
}
|
||||
for (int c = 0; c < 4; c++)
|
||||
{
|
||||
ushort lanes[8];
|
||||
vst1q_u16(lanes, acc.val[c]);
|
||||
ushort result = initial;
|
||||
for (int i = 0; i < 8; i++)
|
||||
result = isMax ? std::max(result, lanes[i]) : std::min(result, lanes[i]);
|
||||
for (int i = x; i < cols; i++)
|
||||
result = isMax ? std::max(result, src[i * 4 + c])
|
||||
: std::min(result, src[i * 4 + c]);
|
||||
dst[c] = result;
|
||||
}
|
||||
}
|
||||
});
|
||||
v_cleanup();
|
||||
}
|
||||
|
||||
template<bool isMax>
|
||||
static void minMax16sC4(const Mat& srcmat, Mat& dstmat)
|
||||
{
|
||||
const int cols = srcmat.cols;
|
||||
const short initial = isMax ? SHRT_MIN : SHRT_MAX;
|
||||
parallel_for_(Range(0, srcmat.rows), [&](const Range& range) {
|
||||
for (int y = range.start; y < range.end; y++)
|
||||
{
|
||||
const short* src = srcmat.ptr<short>(y);
|
||||
short* dst = dstmat.ptr<short>(y);
|
||||
int16x8x4_t acc = {{
|
||||
vdupq_n_s16(initial), vdupq_n_s16(initial),
|
||||
vdupq_n_s16(initial), vdupq_n_s16(initial)
|
||||
}};
|
||||
int x = 0;
|
||||
for (; x <= cols - 8; x += 8)
|
||||
{
|
||||
int16x8x4_t v = vld4q_s16(src + x * 4);
|
||||
for (int c = 0; c < 4; c++)
|
||||
acc.val[c] = isMax ? vmaxq_s16(acc.val[c], v.val[c])
|
||||
: vminq_s16(acc.val[c], v.val[c]);
|
||||
}
|
||||
for (int c = 0; c < 4; c++)
|
||||
{
|
||||
short lanes[8];
|
||||
vst1q_s16(lanes, acc.val[c]);
|
||||
short result = initial;
|
||||
for (int i = 0; i < 8; i++)
|
||||
result = isMax ? std::max(result, lanes[i]) : std::min(result, lanes[i]);
|
||||
for (int i = x; i < cols; i++)
|
||||
result = isMax ? std::max(result, src[i * 4 + c])
|
||||
: std::min(result, src[i * 4 + c]);
|
||||
dst[c] = result;
|
||||
}
|
||||
}
|
||||
});
|
||||
v_cleanup();
|
||||
}
|
||||
|
||||
template<bool isMax>
|
||||
static void minMax16uC3(const Mat& srcmat, Mat& dstmat)
|
||||
{
|
||||
const int cols = srcmat.cols;
|
||||
const ushort initial = isMax ? 0 : USHRT_MAX;
|
||||
parallel_for_(Range(0, srcmat.rows), [&](const Range& range) {
|
||||
for (int y = range.start; y < range.end; y++)
|
||||
{
|
||||
const ushort* src = srcmat.ptr<ushort>(y);
|
||||
ushort* dst = dstmat.ptr<ushort>(y);
|
||||
uint16x8x3_t acc = {{vdupq_n_u16(initial), vdupq_n_u16(initial), vdupq_n_u16(initial)}};
|
||||
int x = 0;
|
||||
for (; x <= cols - 8; x += 8)
|
||||
{
|
||||
uint16x8x3_t v = vld3q_u16(src + x * 3);
|
||||
for (int c = 0; c < 3; c++)
|
||||
acc.val[c] = isMax ? vmaxq_u16(acc.val[c], v.val[c])
|
||||
: vminq_u16(acc.val[c], v.val[c]);
|
||||
}
|
||||
for (int c = 0; c < 3; c++)
|
||||
{
|
||||
ushort lanes[8];
|
||||
vst1q_u16(lanes, acc.val[c]);
|
||||
ushort result = initial;
|
||||
for (int i = 0; i < 8; i++)
|
||||
result = isMax ? std::max(result, lanes[i]) : std::min(result, lanes[i]);
|
||||
for (int i = x; i < cols; i++)
|
||||
result = isMax ? std::max(result, src[i * 3 + c])
|
||||
: std::min(result, src[i * 3 + c]);
|
||||
dst[c] = result;
|
||||
}
|
||||
}
|
||||
});
|
||||
v_cleanup();
|
||||
}
|
||||
|
||||
template<bool isMax>
|
||||
static void minMax16sC3(const Mat& srcmat, Mat& dstmat)
|
||||
{
|
||||
const int cols = srcmat.cols;
|
||||
const short initial = isMax ? SHRT_MIN : SHRT_MAX;
|
||||
parallel_for_(Range(0, srcmat.rows), [&](const Range& range) {
|
||||
for (int y = range.start; y < range.end; y++)
|
||||
{
|
||||
const short* src = srcmat.ptr<short>(y);
|
||||
short* dst = dstmat.ptr<short>(y);
|
||||
int16x8x3_t acc = {{vdupq_n_s16(initial), vdupq_n_s16(initial), vdupq_n_s16(initial)}};
|
||||
int x = 0;
|
||||
for (; x <= cols - 8; x += 8)
|
||||
{
|
||||
int16x8x3_t v = vld3q_s16(src + x * 3);
|
||||
for (int c = 0; c < 3; c++)
|
||||
acc.val[c] = isMax ? vmaxq_s16(acc.val[c], v.val[c])
|
||||
: vminq_s16(acc.val[c], v.val[c]);
|
||||
}
|
||||
for (int c = 0; c < 3; c++)
|
||||
{
|
||||
short lanes[8];
|
||||
vst1q_s16(lanes, acc.val[c]);
|
||||
short result = initial;
|
||||
for (int i = 0; i < 8; i++)
|
||||
result = isMax ? std::max(result, lanes[i]) : std::min(result, lanes[i]);
|
||||
for (int i = x; i < cols; i++)
|
||||
result = isMax ? std::max(result, src[i * 3 + c])
|
||||
: std::min(result, src[i * 3 + c]);
|
||||
dst[c] = result;
|
||||
}
|
||||
}
|
||||
});
|
||||
v_cleanup();
|
||||
}
|
||||
|
||||
template<bool isMax>
|
||||
static void minMax8uC3(const Mat& srcmat, Mat& dstmat)
|
||||
{
|
||||
const int cols = srcmat.cols;
|
||||
parallel_for_(Range(0, srcmat.rows), [&](const Range& range) {
|
||||
for (int y = range.start; y < range.end; y++)
|
||||
{
|
||||
const uchar* src = srcmat.ptr<uchar>(y);
|
||||
uchar* dst = dstmat.ptr<uchar>(y);
|
||||
const uchar initial = isMax ? 0 : UCHAR_MAX;
|
||||
uint8x16x3_t acc = {{
|
||||
vdupq_n_u8(initial), vdupq_n_u8(initial), vdupq_n_u8(initial)
|
||||
}};
|
||||
int x = 0;
|
||||
for (; x <= cols - 16; x += 16)
|
||||
{
|
||||
uint8x16x3_t v = vld3q_u8(src + x * 3);
|
||||
for (int c = 0; c < 3; c++)
|
||||
acc.val[c] = isMax ? vmaxq_u8(acc.val[c], v.val[c])
|
||||
: vminq_u8(acc.val[c], v.val[c]);
|
||||
}
|
||||
for (int c = 0; c < 3; c++)
|
||||
{
|
||||
uchar lanes[16];
|
||||
vst1q_u8(lanes, acc.val[c]);
|
||||
uchar result = initial;
|
||||
for (int i = 0; i < 16; i++)
|
||||
result = reduceScalarMinMax<isMax>(result, lanes[i]);
|
||||
for (int i = x; i < cols; i++)
|
||||
result = reduceScalarMinMax<isMax>(result, src[i * 3 + c]);
|
||||
dst[c] = result;
|
||||
}
|
||||
}
|
||||
});
|
||||
v_cleanup();
|
||||
}
|
||||
|
||||
template<bool isMax>
|
||||
static void minMax8uC4(const Mat& srcmat, Mat& dstmat)
|
||||
{
|
||||
const int cols = srcmat.cols;
|
||||
parallel_for_(Range(0, srcmat.rows), [&](const Range& range) {
|
||||
for (int y = range.start; y < range.end; y++)
|
||||
{
|
||||
const uchar* src = srcmat.ptr<uchar>(y);
|
||||
uchar* dst = dstmat.ptr<uchar>(y);
|
||||
const uchar initial = isMax ? 0 : UCHAR_MAX;
|
||||
uint8x16x4_t acc = {{
|
||||
vdupq_n_u8(initial), vdupq_n_u8(initial),
|
||||
vdupq_n_u8(initial), vdupq_n_u8(initial)
|
||||
}};
|
||||
int x = 0;
|
||||
for (; x <= cols - 16; x += 16)
|
||||
{
|
||||
uint8x16x4_t v = vld4q_u8(src + x * 4);
|
||||
for (int c = 0; c < 4; c++)
|
||||
acc.val[c] = isMax ? vmaxq_u8(acc.val[c], v.val[c])
|
||||
: vminq_u8(acc.val[c], v.val[c]);
|
||||
}
|
||||
for (int c = 0; c < 4; c++)
|
||||
{
|
||||
uchar lanes[16];
|
||||
vst1q_u8(lanes, acc.val[c]);
|
||||
uchar result = initial;
|
||||
for (int i = 0; i < 16; i++)
|
||||
result = reduceScalarMinMax<isMax>(result, lanes[i]);
|
||||
for (int i = x; i < cols; i++)
|
||||
result = reduceScalarMinMax<isMax>(result, src[i * 4 + c]);
|
||||
dst[c] = result;
|
||||
}
|
||||
}
|
||||
});
|
||||
v_cleanup();
|
||||
}
|
||||
|
||||
template<bool isMax>
|
||||
static void minMax32fC1(const Mat& srcmat, Mat& dstmat)
|
||||
{
|
||||
const int cols = srcmat.cols;
|
||||
parallel_for_(Range(0, srcmat.rows), [&](const Range& range) {
|
||||
for (int y = range.start; y < range.end; y++)
|
||||
{
|
||||
const float* src = srcmat.ptr<float>(y);
|
||||
float result = src[0];
|
||||
int x = 0;
|
||||
float32x4_t acc = vdupq_n_f32(result);
|
||||
for (; x <= cols - 4; x += 4)
|
||||
{
|
||||
float32x4_t v = vld1q_f32(src + x);
|
||||
acc = isMax ? vmaxq_f32(acc, v) : vminq_f32(acc, v);
|
||||
}
|
||||
float lanes[4];
|
||||
vst1q_f32(lanes, acc);
|
||||
for (int i = 0; i < 4; i++)
|
||||
result = reduceScalarMinMax<isMax>(result, lanes[i]);
|
||||
for (; x < cols; x++)
|
||||
result = reduceScalarMinMax<isMax>(result, src[x]);
|
||||
dstmat.ptr<float>(y)[0] = result;
|
||||
}
|
||||
});
|
||||
v_cleanup();
|
||||
}
|
||||
|
||||
template<bool isMax>
|
||||
static void minMax32fC4(const Mat& srcmat, Mat& dstmat)
|
||||
{
|
||||
const int cols = srcmat.cols;
|
||||
const float initial = isMax ? std::numeric_limits<float>::lowest() : std::numeric_limits<float>::max();
|
||||
parallel_for_(Range(0, srcmat.rows), [&](const Range& range) {
|
||||
for (int y = range.start; y < range.end; y++)
|
||||
{
|
||||
const float* src = srcmat.ptr<float>(y);
|
||||
float* dst = dstmat.ptr<float>(y);
|
||||
float32x4x4_t acc = {{
|
||||
vdupq_n_f32(initial), vdupq_n_f32(initial),
|
||||
vdupq_n_f32(initial), vdupq_n_f32(initial)
|
||||
}};
|
||||
int x = 0;
|
||||
for (; x <= cols - 4; x += 4)
|
||||
{
|
||||
float32x4x4_t v = vld4q_f32(src + x * 4);
|
||||
for (int c = 0; c < 4; c++)
|
||||
acc.val[c] = isMax ? vmaxq_f32(acc.val[c], v.val[c])
|
||||
: vminq_f32(acc.val[c], v.val[c]);
|
||||
}
|
||||
for (int c = 0; c < 4; c++)
|
||||
{
|
||||
float lanes[4];
|
||||
vst1q_f32(lanes, acc.val[c]);
|
||||
float result = initial;
|
||||
for (int i = 0; i < 4; i++)
|
||||
result = reduceScalarMinMax<isMax>(result, lanes[i]);
|
||||
for (int i = x; i < cols; i++)
|
||||
result = reduceScalarMinMax<isMax>(result, src[i * 4 + c]);
|
||||
dst[c] = result;
|
||||
}
|
||||
}
|
||||
});
|
||||
v_cleanup();
|
||||
}
|
||||
|
||||
#if defined(__aarch64__) || defined(_M_ARM64)
|
||||
template<bool isMax>
|
||||
static void minMax64fC1(const Mat& srcmat, Mat& dstmat)
|
||||
{
|
||||
const int cols = srcmat.cols;
|
||||
const double initial = isMax ? std::numeric_limits<double>::lowest()
|
||||
: std::numeric_limits<double>::max();
|
||||
parallel_for_(Range(0, srcmat.rows), [&](const Range& range) {
|
||||
for (int y = range.start; y < range.end; y++)
|
||||
{
|
||||
const double* src = srcmat.ptr<double>(y);
|
||||
double result = initial;
|
||||
int x = 0;
|
||||
float64x2_t acc0 = vdupq_n_f64(initial);
|
||||
float64x2_t acc1 = acc0, acc2 = acc0, acc3 = acc0;
|
||||
for (; x <= cols - 8; x += 8)
|
||||
{
|
||||
float64x2_t v0 = vld1q_f64(src + x);
|
||||
float64x2_t v1 = vld1q_f64(src + x + 2);
|
||||
float64x2_t v2 = vld1q_f64(src + x + 4);
|
||||
float64x2_t v3 = vld1q_f64(src + x + 6);
|
||||
acc0 = isMax ? vmaxq_f64(acc0, v0) : vminq_f64(acc0, v0);
|
||||
acc1 = isMax ? vmaxq_f64(acc1, v1) : vminq_f64(acc1, v1);
|
||||
acc2 = isMax ? vmaxq_f64(acc2, v2) : vminq_f64(acc2, v2);
|
||||
acc3 = isMax ? vmaxq_f64(acc3, v3) : vminq_f64(acc3, v3);
|
||||
}
|
||||
acc0 = isMax ? vmaxq_f64(acc0, acc1) : vminq_f64(acc0, acc1);
|
||||
acc2 = isMax ? vmaxq_f64(acc2, acc3) : vminq_f64(acc2, acc3);
|
||||
acc0 = isMax ? vmaxq_f64(acc0, acc2) : vminq_f64(acc0, acc2);
|
||||
double lanes[2];
|
||||
vst1q_f64(lanes, acc0);
|
||||
for (int i = 0; i < 2; i++)
|
||||
result = isMax ? std::max(result, lanes[i]) : std::min(result, lanes[i]);
|
||||
for (; x < cols; x++)
|
||||
result = isMax ? std::max(result, src[x]) : std::min(result, src[x]);
|
||||
dstmat.ptr<double>(y)[0] = result;
|
||||
}
|
||||
});
|
||||
v_cleanup();
|
||||
}
|
||||
#endif
|
||||
|
||||
#if defined(__aarch64__) || defined(_M_ARM64)
|
||||
static inline uint32_t reduceSum2_8u_NEON(uint8x16_t v)
|
||||
{
|
||||
uint16x8_t lo = vmull_u8(vget_low_u8(v), vget_low_u8(v));
|
||||
uint16x8_t hi = vmull_u8(vget_high_u8(v), vget_high_u8(v));
|
||||
return vaddvq_u32(vpaddlq_u16(lo)) + vaddvq_u32(vpaddlq_u16(hi));
|
||||
}
|
||||
|
||||
template<typename DT>
|
||||
static void sum2_8uC1(const Mat& srcmat, Mat& dstmat)
|
||||
{
|
||||
const int cols = srcmat.cols;
|
||||
|
||||
parallel_for_(Range(0, srcmat.rows), [&](const Range& range) {
|
||||
for (int y = range.start; y < range.end; y++)
|
||||
{
|
||||
const uchar* src = srcmat.ptr<uchar>(y);
|
||||
DT* dst = dstmat.ptr<DT>(y);
|
||||
uint32_t result = 0;
|
||||
int x = 0;
|
||||
for (; x <= cols - 16; x += 16)
|
||||
result += reduceSum2_8u_NEON(vld1q_u8(src + x));
|
||||
for (; x < cols; x++)
|
||||
result += (uint32_t)src[x] * src[x];
|
||||
dst[0] = (DT)(int32_t)result;
|
||||
}
|
||||
});
|
||||
v_cleanup();
|
||||
}
|
||||
|
||||
static void sum2_16u32fC1(const Mat& srcmat, Mat& dstmat)
|
||||
{
|
||||
const int cols = srcmat.cols;
|
||||
parallel_for_(Range(0, srcmat.rows), [&](const Range& range) {
|
||||
for (int y = range.start; y < range.end; y++)
|
||||
{
|
||||
const ushort* src = srcmat.ptr<ushort>(y);
|
||||
float result = 0;
|
||||
float32x4_t acc0 = vdupq_n_f32(0), acc1 = acc0, acc2 = acc0, acc3 = acc0;
|
||||
int x = 0;
|
||||
for (; x <= cols - 16; x += 16)
|
||||
{
|
||||
uint16x8_t v01 = vld1q_u16(src + x);
|
||||
uint16x8_t v23 = vld1q_u16(src + x + 8);
|
||||
float32x4_t v0 = vcvtq_f32_u32(vmovl_u16(vget_low_u16(v01)));
|
||||
float32x4_t v1 = vcvtq_f32_u32(vmovl_u16(vget_high_u16(v01)));
|
||||
float32x4_t v2 = vcvtq_f32_u32(vmovl_u16(vget_low_u16(v23)));
|
||||
float32x4_t v3 = vcvtq_f32_u32(vmovl_u16(vget_high_u16(v23)));
|
||||
acc0 = vmlaq_f32(acc0, v0, v0);
|
||||
acc1 = vmlaq_f32(acc1, v1, v1);
|
||||
acc2 = vmlaq_f32(acc2, v2, v2);
|
||||
acc3 = vmlaq_f32(acc3, v3, v3);
|
||||
}
|
||||
acc0 = vaddq_f32(vaddq_f32(acc0, acc1), vaddq_f32(acc2, acc3));
|
||||
float lanes[4];
|
||||
vst1q_f32(lanes, acc0);
|
||||
for (int i = 0; i < 4; i++)
|
||||
result += lanes[i];
|
||||
for (; x < cols; x++)
|
||||
{
|
||||
float value = (float)src[x];
|
||||
result += value * value;
|
||||
}
|
||||
dstmat.ptr<float>(y)[0] = result;
|
||||
}
|
||||
});
|
||||
v_cleanup();
|
||||
}
|
||||
|
||||
static void sum2_16s32fC1(const Mat& srcmat, Mat& dstmat)
|
||||
{
|
||||
const int cols = srcmat.cols;
|
||||
parallel_for_(Range(0, srcmat.rows), [&](const Range& range) {
|
||||
for (int y = range.start; y < range.end; y++)
|
||||
{
|
||||
const short* src = srcmat.ptr<short>(y);
|
||||
float result = 0;
|
||||
float32x4_t acc0 = vdupq_n_f32(0), acc1 = acc0, acc2 = acc0, acc3 = acc0;
|
||||
int x = 0;
|
||||
for (; x <= cols - 16; x += 16)
|
||||
{
|
||||
int16x8_t v01 = vld1q_s16(src + x);
|
||||
int16x8_t v23 = vld1q_s16(src + x + 8);
|
||||
float32x4_t v0 = vcvtq_f32_s32(vmovl_s16(vget_low_s16(v01)));
|
||||
float32x4_t v1 = vcvtq_f32_s32(vmovl_s16(vget_high_s16(v01)));
|
||||
float32x4_t v2 = vcvtq_f32_s32(vmovl_s16(vget_low_s16(v23)));
|
||||
float32x4_t v3 = vcvtq_f32_s32(vmovl_s16(vget_high_s16(v23)));
|
||||
acc0 = vmlaq_f32(acc0, v0, v0);
|
||||
acc1 = vmlaq_f32(acc1, v1, v1);
|
||||
acc2 = vmlaq_f32(acc2, v2, v2);
|
||||
acc3 = vmlaq_f32(acc3, v3, v3);
|
||||
}
|
||||
acc0 = vaddq_f32(vaddq_f32(acc0, acc1), vaddq_f32(acc2, acc3));
|
||||
float lanes[4];
|
||||
vst1q_f32(lanes, acc0);
|
||||
for (int i = 0; i < 4; i++)
|
||||
result += lanes[i];
|
||||
for (; x < cols; x++)
|
||||
{
|
||||
float value = (float)src[x];
|
||||
result += value * value;
|
||||
}
|
||||
dstmat.ptr<float>(y)[0] = result;
|
||||
}
|
||||
});
|
||||
v_cleanup();
|
||||
}
|
||||
|
||||
static void sum2_16u64fC1(const Mat& srcmat, Mat& dstmat)
|
||||
{
|
||||
const int cols = srcmat.cols;
|
||||
parallel_for_(Range(0, srcmat.rows), [&](const Range& range) {
|
||||
for (int y = range.start; y < range.end; y++)
|
||||
{
|
||||
const ushort* src = srcmat.ptr<ushort>(y);
|
||||
uint64x2_t acc0 = vdupq_n_u64(0), acc1 = acc0, acc2 = acc0, acc3 = acc0;
|
||||
int x = 0;
|
||||
for (; x <= cols - 8; x += 8)
|
||||
{
|
||||
uint16x8_t v = vld1q_u16(src + x);
|
||||
uint32x4_t sq0 = vmull_u16(vget_low_u16(v), vget_low_u16(v));
|
||||
uint32x4_t sq1 = vmull_u16(vget_high_u16(v), vget_high_u16(v));
|
||||
acc0 = vaddq_u64(acc0, vmovl_u32(vget_low_u32(sq0)));
|
||||
acc1 = vaddq_u64(acc1, vmovl_high_u32(sq0));
|
||||
acc2 = vaddq_u64(acc2, vmovl_u32(vget_low_u32(sq1)));
|
||||
acc3 = vaddq_u64(acc3, vmovl_high_u32(sq1));
|
||||
}
|
||||
acc0 = vaddq_u64(vaddq_u64(acc0, acc1), vaddq_u64(acc2, acc3));
|
||||
uint64_t lanes[2];
|
||||
vst1q_u64(lanes, acc0);
|
||||
uint64_t result = lanes[0] + lanes[1];
|
||||
for (; x < cols; x++)
|
||||
result += (uint64_t)src[x] * src[x];
|
||||
dstmat.ptr<double>(y)[0] = (double)result;
|
||||
}
|
||||
});
|
||||
v_cleanup();
|
||||
}
|
||||
|
||||
static void sum2_16s64fC1(const Mat& srcmat, Mat& dstmat)
|
||||
{
|
||||
const int cols = srcmat.cols;
|
||||
parallel_for_(Range(0, srcmat.rows), [&](const Range& range) {
|
||||
for (int y = range.start; y < range.end; y++)
|
||||
{
|
||||
const short* src = srcmat.ptr<short>(y);
|
||||
uint64x2_t acc0 = vdupq_n_u64(0), acc1 = acc0, acc2 = acc0, acc3 = acc0;
|
||||
int x = 0;
|
||||
for (; x <= cols - 8; x += 8)
|
||||
{
|
||||
int16x8_t v = vld1q_s16(src + x);
|
||||
int32x4_t sq0 = vmull_s16(vget_low_s16(v), vget_low_s16(v));
|
||||
int32x4_t sq1 = vmull_s16(vget_high_s16(v), vget_high_s16(v));
|
||||
uint32x4_t usq0 = vreinterpretq_u32_s32(sq0);
|
||||
uint32x4_t usq1 = vreinterpretq_u32_s32(sq1);
|
||||
acc0 = vaddq_u64(acc0, vmovl_u32(vget_low_u32(usq0)));
|
||||
acc1 = vaddq_u64(acc1, vmovl_high_u32(usq0));
|
||||
acc2 = vaddq_u64(acc2, vmovl_u32(vget_low_u32(usq1)));
|
||||
acc3 = vaddq_u64(acc3, vmovl_high_u32(usq1));
|
||||
}
|
||||
acc0 = vaddq_u64(vaddq_u64(acc0, acc1), vaddq_u64(acc2, acc3));
|
||||
uint64_t lanes[2];
|
||||
vst1q_u64(lanes, acc0);
|
||||
uint64_t result = lanes[0] + lanes[1];
|
||||
for (; x < cols; x++)
|
||||
{
|
||||
int64_t value = src[x];
|
||||
result += (uint64_t)(value * value);
|
||||
}
|
||||
dstmat.ptr<double>(y)[0] = (double)result;
|
||||
}
|
||||
});
|
||||
v_cleanup();
|
||||
}
|
||||
|
||||
static void sum2_32f64fC1(const Mat& srcmat, Mat& dstmat)
|
||||
{
|
||||
const int cols = srcmat.cols;
|
||||
parallel_for_(Range(0, srcmat.rows), [&](const Range& range) {
|
||||
for (int y = range.start; y < range.end; y++)
|
||||
{
|
||||
const float* src = srcmat.ptr<float>(y);
|
||||
float64x2_t acc0 = vdupq_n_f64(0), acc1 = acc0, acc2 = acc0, acc3 = acc0;
|
||||
int x = 0;
|
||||
for (; x <= cols - 8; x += 8)
|
||||
{
|
||||
float32x4_t v01 = vld1q_f32(src + x);
|
||||
float32x4_t v23 = vld1q_f32(src + x + 4);
|
||||
float64x2_t v0 = vcvt_f64_f32(vget_low_f32(v01));
|
||||
float64x2_t v1 = vcvt_high_f64_f32(v01);
|
||||
float64x2_t v2 = vcvt_f64_f32(vget_low_f32(v23));
|
||||
float64x2_t v3 = vcvt_high_f64_f32(v23);
|
||||
acc0 = vmlaq_f64(acc0, v0, v0);
|
||||
acc1 = vmlaq_f64(acc1, v1, v1);
|
||||
acc2 = vmlaq_f64(acc2, v2, v2);
|
||||
acc3 = vmlaq_f64(acc3, v3, v3);
|
||||
}
|
||||
acc0 = vaddq_f64(vaddq_f64(acc0, acc1), vaddq_f64(acc2, acc3));
|
||||
double lanes[2];
|
||||
vst1q_f64(lanes, acc0);
|
||||
double result = lanes[0] + lanes[1];
|
||||
for (; x < cols; x++)
|
||||
{
|
||||
double value = (double)src[x];
|
||||
result += value * value;
|
||||
}
|
||||
dstmat.ptr<double>(y)[0] = result;
|
||||
}
|
||||
});
|
||||
v_cleanup();
|
||||
}
|
||||
|
||||
static void sum2_64f64fC1(const Mat& srcmat, Mat& dstmat)
|
||||
{
|
||||
const int cols = srcmat.cols;
|
||||
parallel_for_(Range(0, srcmat.rows), [&](const Range& range) {
|
||||
for (int y = range.start; y < range.end; y++)
|
||||
{
|
||||
const double* src = srcmat.ptr<double>(y);
|
||||
float64x2_t acc0 = vdupq_n_f64(0), acc1 = acc0, acc2 = acc0, acc3 = acc0;
|
||||
int x = 0;
|
||||
for (; x <= cols - 8; x += 8)
|
||||
{
|
||||
float64x2_t v0 = vld1q_f64(src + x);
|
||||
float64x2_t v1 = vld1q_f64(src + x + 2);
|
||||
float64x2_t v2 = vld1q_f64(src + x + 4);
|
||||
float64x2_t v3 = vld1q_f64(src + x + 6);
|
||||
acc0 = vmlaq_f64(acc0, v0, v0);
|
||||
acc1 = vmlaq_f64(acc1, v1, v1);
|
||||
acc2 = vmlaq_f64(acc2, v2, v2);
|
||||
acc3 = vmlaq_f64(acc3, v3, v3);
|
||||
}
|
||||
acc0 = vaddq_f64(vaddq_f64(acc0, acc1), vaddq_f64(acc2, acc3));
|
||||
double lanes[2];
|
||||
vst1q_f64(lanes, acc0);
|
||||
double result = lanes[0] + lanes[1];
|
||||
for (; x < cols; x++)
|
||||
result += src[x] * src[x];
|
||||
dstmat.ptr<double>(y)[0] = result;
|
||||
}
|
||||
});
|
||||
v_cleanup();
|
||||
}
|
||||
|
||||
template<typename DT>
|
||||
static void sum2_8uC3(const Mat& srcmat, Mat& dstmat)
|
||||
{
|
||||
const int cols = srcmat.cols;
|
||||
parallel_for_(Range(0, srcmat.rows), [&](const Range& range) {
|
||||
for (int y = range.start; y < range.end; y++)
|
||||
{
|
||||
const uchar* src = srcmat.ptr<uchar>(y);
|
||||
DT* dst = dstmat.ptr<DT>(y);
|
||||
uint32_t results[3] = {0, 0, 0};
|
||||
int x = 0;
|
||||
for (; x <= cols - 16; x += 16)
|
||||
{
|
||||
uint8x16x3_t v = vld3q_u8(src + x * 3);
|
||||
for (int c = 0; c < 3; c++)
|
||||
results[c] += reduceSum2_8u_NEON(v.val[c]);
|
||||
}
|
||||
for (int c = 0; c < 3; c++)
|
||||
{
|
||||
for (int i = x; i < cols; i++)
|
||||
{
|
||||
uint32_t value = src[i * 3 + c];
|
||||
results[c] += value * value;
|
||||
}
|
||||
dst[c] = (DT)(int32_t)results[c];
|
||||
}
|
||||
}
|
||||
});
|
||||
v_cleanup();
|
||||
}
|
||||
|
||||
template<typename DT>
|
||||
static void sum2_8uC4(const Mat& srcmat, Mat& dstmat)
|
||||
{
|
||||
const int cols = srcmat.cols;
|
||||
parallel_for_(Range(0, srcmat.rows), [&](const Range& range) {
|
||||
for (int y = range.start; y < range.end; y++)
|
||||
{
|
||||
const uchar* src = srcmat.ptr<uchar>(y);
|
||||
DT* dst = dstmat.ptr<DT>(y);
|
||||
uint32_t results[4] = {0, 0, 0, 0};
|
||||
int x = 0;
|
||||
for (; x <= cols - 16; x += 16)
|
||||
{
|
||||
uint8x16x4_t v = vld4q_u8(src + x * 4);
|
||||
for (int c = 0; c < 4; c++)
|
||||
results[c] += reduceSum2_8u_NEON(v.val[c]);
|
||||
}
|
||||
for (int c = 0; c < 4; c++)
|
||||
{
|
||||
for (int i = x; i < cols; i++)
|
||||
{
|
||||
uint32_t value = src[i * 4 + c];
|
||||
results[c] += value * value;
|
||||
}
|
||||
dst[c] = (DT)(int32_t)results[c];
|
||||
}
|
||||
}
|
||||
});
|
||||
v_cleanup();
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
static void sum2_32fC1(const Mat& srcmat, Mat& dstmat)
|
||||
{
|
||||
const int cols = srcmat.cols;
|
||||
parallel_for_(Range(0, srcmat.rows), [&](const Range& range) {
|
||||
for (int y = range.start; y < range.end; y++)
|
||||
{
|
||||
const float* src = srcmat.ptr<float>(y);
|
||||
float result = 0;
|
||||
int x = 0;
|
||||
float32x4_t acc = vdupq_n_f32(0);
|
||||
for (; x <= cols - 4; x += 4)
|
||||
{
|
||||
float32x4_t v = vld1q_f32(src + x);
|
||||
acc = vaddq_f32(acc, vmulq_f32(v, v));
|
||||
}
|
||||
float lanes[4];
|
||||
vst1q_f32(lanes, acc);
|
||||
for (int i = 0; i < 4; i++)
|
||||
result += lanes[i];
|
||||
for (; x < cols; x++)
|
||||
result += src[x] * src[x];
|
||||
dstmat.ptr<float>(y)[0] = result;
|
||||
}
|
||||
});
|
||||
v_cleanup();
|
||||
}
|
||||
|
||||
static void sum2_32fC4(const Mat& srcmat, Mat& dstmat)
|
||||
{
|
||||
const int cols = srcmat.cols;
|
||||
parallel_for_(Range(0, srcmat.rows), [&](const Range& range) {
|
||||
for (int y = range.start; y < range.end; y++)
|
||||
{
|
||||
const float* src = srcmat.ptr<float>(y);
|
||||
float* dst = dstmat.ptr<float>(y);
|
||||
float32x4x4_t acc = {{
|
||||
vdupq_n_f32(0), vdupq_n_f32(0),
|
||||
vdupq_n_f32(0), vdupq_n_f32(0)
|
||||
}};
|
||||
int x = 0;
|
||||
for (; x <= cols - 4; x += 4)
|
||||
{
|
||||
float32x4x4_t v = vld4q_f32(src + x * 4);
|
||||
for (int c = 0; c < 4; c++)
|
||||
acc.val[c] = vmlaq_f32(acc.val[c], v.val[c], v.val[c]);
|
||||
}
|
||||
for (int c = 0; c < 4; c++)
|
||||
{
|
||||
float lanes[4];
|
||||
float result = 0;
|
||||
vst1q_f32(lanes, acc.val[c]);
|
||||
for (int i = 0; i < 4; i++)
|
||||
result += lanes[i];
|
||||
for (int i = x; i < cols; i++)
|
||||
result += src[i * 4 + c] * src[i * 4 + c];
|
||||
dst[c] = result;
|
||||
}
|
||||
}
|
||||
});
|
||||
v_cleanup();
|
||||
}
|
||||
|
||||
} // namespace reduce_c_neon
|
||||
@@ -0,0 +1,866 @@
|
||||
// This file is part of OpenCV project.
|
||||
// It is subject to the license terms in the LICENSE file found in the top-level directory
|
||||
// of this distribution and at http://opencv.org/license.html
|
||||
|
||||
namespace reduce_c_rvv
|
||||
{
|
||||
|
||||
// Optimized ReduceC support in this backend:
|
||||
//
|
||||
// | Input -> output type/channel | SUM | AVG | MIN | MAX | SUM2 |
|
||||
// |------------------------------------|:---:|:---:|:---:|:---:|:----:|
|
||||
// | 8UC1/C3/C4 -> 8UC1/C3/C4 | - | - | x | x | - |
|
||||
// | 8UC1/C3/C4 -> 32SC1/C3/C4 | x | x | - | - | x |
|
||||
// | 8UC1/C3/C4 -> 32FC1/C3/C4 | x | x | - | - | x |
|
||||
// | 8UC1/C3/C4 -> 64FC1/C3/C4 | - | - | - | - | x |
|
||||
// | 16UC1/C3/C4 -> 16UC1/C3/C4 | - | - | x | x | - |
|
||||
// | 16UC1 -> 32FC1 | x | x | - | - | x |
|
||||
// | 16UC3/C4 -> 32FC3/C4 | x | x | - | - | - |
|
||||
// | 16UC1 -> 64FC1 | - | - | - | - | x* |
|
||||
// | 16UC3/C4 -> 64FC3/C4 | - | - | - | - | - |
|
||||
// | 16SC1/C3/C4 -> 16SC1/C3/C4 | - | - | x | x | - |
|
||||
// | 16SC1 -> 32FC1 | x | x | - | - | x |
|
||||
// | 16SC3/C4 -> 32FC3/C4 | x | x | - | - | - |
|
||||
// | 16SC1 -> 64FC1 | - | - | - | - | x* |
|
||||
// | 16SC3/C4 -> 64FC3/C4 | - | - | - | - | - |
|
||||
// | 32FC1/C3/C4 -> 32FC1/C3/C4 | x | x | x | x | x |
|
||||
// | 32FC1 -> 64FC1 | x* | x* | - | - | x* |
|
||||
// | 32FC3/C4 -> 64FC3/C4 | x* | x* | - | - | - |
|
||||
// | 64FC1 -> 64FC1 | x* | x* | x* | x* | x* |
|
||||
// | 64FC3/C4 -> 64FC3/C4 | x* | x* | - | - | - |
|
||||
//
|
||||
// 'x' in SUM/AVG denotes the existing shared universal-intrinsics kernel; 'x'
|
||||
// in MIN/MAX/SUM2 denotes a native RVV kernel. '*' requires
|
||||
// CV_SIMD_SCALABLE_64F. For legal MIN/MAX/SUM2 combinations marked '-', and for
|
||||
// other channel counts, dispatch uses the shared generic fallback.
|
||||
|
||||
template<bool isMax>
|
||||
static void minMax8uC1(const Mat& srcmat, Mat& dstmat)
|
||||
{
|
||||
const int cols = srcmat.cols;
|
||||
parallel_for_(Range(0, srcmat.rows), [&](const Range& range) {
|
||||
for (int y = range.start; y < range.end; y++)
|
||||
{
|
||||
const uchar* src = srcmat.ptr<uchar>(y);
|
||||
uchar* dst = dstmat.ptr<uchar>(y);
|
||||
uchar result = isMax ? 0 : UCHAR_MAX;
|
||||
int x = 0;
|
||||
const int vlmax = __riscv_vsetvlmax_e8m8();
|
||||
vuint8m8_t acc = __riscv_vmv_v_x_u8m8(result, vlmax);
|
||||
for (; x < cols; )
|
||||
{
|
||||
const int vl = __riscv_vsetvl_e8m8(cols - x);
|
||||
vuint8m8_t v = __riscv_vle8_v_u8m8(src + x, vl);
|
||||
acc = isMax ? __riscv_vmaxu_tu(acc, acc, v, vl)
|
||||
: __riscv_vminu_tu(acc, acc, v, vl);
|
||||
x += vl;
|
||||
}
|
||||
vuint8m1_t seed = __riscv_vmv_s_x_u8m1(result, __riscv_vsetvlmax_e8m1());
|
||||
vuint8m1_t reduced = isMax ? __riscv_vredmaxu(acc, seed, vlmax)
|
||||
: __riscv_vredminu(acc, seed, vlmax);
|
||||
result = (uchar)__riscv_vmv_x(reduced);
|
||||
for (; x < cols; x++)
|
||||
result = reduceScalarMinMax<isMax>(result, src[x]);
|
||||
dst[0] = result;
|
||||
}
|
||||
});
|
||||
v_cleanup();
|
||||
}
|
||||
|
||||
template<bool isMax>
|
||||
static void minMax16uC1(const Mat& srcmat, Mat& dstmat)
|
||||
{
|
||||
const int cols = srcmat.cols;
|
||||
parallel_for_(Range(0, srcmat.rows), [&](const Range& range) {
|
||||
for (int y = range.start; y < range.end; y++)
|
||||
{
|
||||
const ushort* src = srcmat.ptr<ushort>(y);
|
||||
const ushort initial = isMax ? 0 : USHRT_MAX;
|
||||
const int vlmax = __riscv_vsetvlmax_e16m8();
|
||||
vuint16m8_t acc = __riscv_vmv_v_x_u16m8(initial, vlmax);
|
||||
int x = 0;
|
||||
for (; x < cols; )
|
||||
{
|
||||
const int vl = __riscv_vsetvl_e16m8(cols - x);
|
||||
vuint16m8_t v = __riscv_vle16_v_u16m8(src + x, vl);
|
||||
acc = isMax ? __riscv_vmaxu_tu(acc, acc, v, vl)
|
||||
: __riscv_vminu_tu(acc, acc, v, vl);
|
||||
x += vl;
|
||||
}
|
||||
vuint16m1_t seed = __riscv_vmv_s_x_u16m1(initial, __riscv_vsetvlmax_e16m1());
|
||||
vuint16m1_t reduced = isMax ? __riscv_vredmaxu(acc, seed, vlmax)
|
||||
: __riscv_vredminu(acc, seed, vlmax);
|
||||
dstmat.ptr<ushort>(y)[0] = (ushort)__riscv_vmv_x(reduced);
|
||||
}
|
||||
});
|
||||
v_cleanup();
|
||||
}
|
||||
|
||||
template<bool isMax>
|
||||
static void minMax16sC1(const Mat& srcmat, Mat& dstmat)
|
||||
{
|
||||
const int cols = srcmat.cols;
|
||||
parallel_for_(Range(0, srcmat.rows), [&](const Range& range) {
|
||||
for (int y = range.start; y < range.end; y++)
|
||||
{
|
||||
const short* src = srcmat.ptr<short>(y);
|
||||
const short initial = isMax ? SHRT_MIN : SHRT_MAX;
|
||||
const int vlmax = __riscv_vsetvlmax_e16m8();
|
||||
vint16m8_t acc = __riscv_vmv_v_x_i16m8(initial, vlmax);
|
||||
int x = 0;
|
||||
for (; x < cols; )
|
||||
{
|
||||
const int vl = __riscv_vsetvl_e16m8(cols - x);
|
||||
vint16m8_t v = __riscv_vle16_v_i16m8(src + x, vl);
|
||||
acc = isMax ? __riscv_vmax_tu(acc, acc, v, vl)
|
||||
: __riscv_vmin_tu(acc, acc, v, vl);
|
||||
x += vl;
|
||||
}
|
||||
vint16m1_t seed = __riscv_vmv_s_x_i16m1(initial, __riscv_vsetvlmax_e16m1());
|
||||
vint16m1_t reduced = isMax ? __riscv_vredmax(acc, seed, vlmax)
|
||||
: __riscv_vredmin(acc, seed, vlmax);
|
||||
dstmat.ptr<short>(y)[0] = (short)__riscv_vmv_x(reduced);
|
||||
}
|
||||
});
|
||||
v_cleanup();
|
||||
}
|
||||
|
||||
template<bool isMax>
|
||||
static void minMax16uC4(const Mat& srcmat, Mat& dstmat)
|
||||
{
|
||||
const int cols = srcmat.cols;
|
||||
const ushort initial = isMax ? 0 : USHRT_MAX;
|
||||
const int vlmax = __riscv_vsetvlmax_e16m2();
|
||||
parallel_for_(Range(0, srcmat.rows), [&](const Range& range) {
|
||||
for (int y = range.start; y < range.end; y++)
|
||||
{
|
||||
const ushort* src = srcmat.ptr<ushort>(y);
|
||||
ushort* dst = dstmat.ptr<ushort>(y);
|
||||
vuint16m2_t acc0 = __riscv_vmv_v_x_u16m2(initial, vlmax);
|
||||
vuint16m2_t acc1 = acc0, acc2 = acc0, acc3 = acc0;
|
||||
int x = 0;
|
||||
for (; x < cols; )
|
||||
{
|
||||
const int vl = __riscv_vsetvl_e16m2(cols - x);
|
||||
vuint16m2x4_t v = __riscv_vlseg4e16_v_u16m2x4(src + x * 4, vl);
|
||||
vuint16m2_t v0 = __riscv_vget_v_u16m2x4_u16m2(v, 0);
|
||||
vuint16m2_t v1 = __riscv_vget_v_u16m2x4_u16m2(v, 1);
|
||||
vuint16m2_t v2 = __riscv_vget_v_u16m2x4_u16m2(v, 2);
|
||||
vuint16m2_t v3 = __riscv_vget_v_u16m2x4_u16m2(v, 3);
|
||||
acc0 = isMax ? __riscv_vmaxu_tu(acc0, acc0, v0, vl)
|
||||
: __riscv_vminu_tu(acc0, acc0, v0, vl);
|
||||
acc1 = isMax ? __riscv_vmaxu_tu(acc1, acc1, v1, vl)
|
||||
: __riscv_vminu_tu(acc1, acc1, v1, vl);
|
||||
acc2 = isMax ? __riscv_vmaxu_tu(acc2, acc2, v2, vl)
|
||||
: __riscv_vminu_tu(acc2, acc2, v2, vl);
|
||||
acc3 = isMax ? __riscv_vmaxu_tu(acc3, acc3, v3, vl)
|
||||
: __riscv_vminu_tu(acc3, acc3, v3, vl);
|
||||
x += vl;
|
||||
}
|
||||
vuint16m1_t seed = __riscv_vmv_s_x_u16m1(initial, __riscv_vsetvlmax_e16m1());
|
||||
dst[0] = (ushort)__riscv_vmv_x(isMax ? __riscv_vredmaxu(acc0, seed, vlmax) : __riscv_vredminu(acc0, seed, vlmax));
|
||||
dst[1] = (ushort)__riscv_vmv_x(isMax ? __riscv_vredmaxu(acc1, seed, vlmax) : __riscv_vredminu(acc1, seed, vlmax));
|
||||
dst[2] = (ushort)__riscv_vmv_x(isMax ? __riscv_vredmaxu(acc2, seed, vlmax) : __riscv_vredminu(acc2, seed, vlmax));
|
||||
dst[3] = (ushort)__riscv_vmv_x(isMax ? __riscv_vredmaxu(acc3, seed, vlmax) : __riscv_vredminu(acc3, seed, vlmax));
|
||||
}
|
||||
});
|
||||
v_cleanup();
|
||||
}
|
||||
|
||||
template<bool isMax>
|
||||
static void minMax16sC4(const Mat& srcmat, Mat& dstmat)
|
||||
{
|
||||
const int cols = srcmat.cols;
|
||||
const short initial = isMax ? SHRT_MIN : SHRT_MAX;
|
||||
const int vlmax = __riscv_vsetvlmax_e16m2();
|
||||
parallel_for_(Range(0, srcmat.rows), [&](const Range& range) {
|
||||
for (int y = range.start; y < range.end; y++)
|
||||
{
|
||||
const short* src = srcmat.ptr<short>(y);
|
||||
short* dst = dstmat.ptr<short>(y);
|
||||
vint16m2_t acc0 = __riscv_vmv_v_x_i16m2(initial, vlmax);
|
||||
vint16m2_t acc1 = acc0, acc2 = acc0, acc3 = acc0;
|
||||
int x = 0;
|
||||
for (; x < cols; )
|
||||
{
|
||||
const int vl = __riscv_vsetvl_e16m2(cols - x);
|
||||
vint16m2x4_t v = __riscv_vlseg4e16_v_i16m2x4(src + x * 4, vl);
|
||||
vint16m2_t v0 = __riscv_vget_v_i16m2x4_i16m2(v, 0);
|
||||
vint16m2_t v1 = __riscv_vget_v_i16m2x4_i16m2(v, 1);
|
||||
vint16m2_t v2 = __riscv_vget_v_i16m2x4_i16m2(v, 2);
|
||||
vint16m2_t v3 = __riscv_vget_v_i16m2x4_i16m2(v, 3);
|
||||
acc0 = isMax ? __riscv_vmax_tu(acc0, acc0, v0, vl)
|
||||
: __riscv_vmin_tu(acc0, acc0, v0, vl);
|
||||
acc1 = isMax ? __riscv_vmax_tu(acc1, acc1, v1, vl)
|
||||
: __riscv_vmin_tu(acc1, acc1, v1, vl);
|
||||
acc2 = isMax ? __riscv_vmax_tu(acc2, acc2, v2, vl)
|
||||
: __riscv_vmin_tu(acc2, acc2, v2, vl);
|
||||
acc3 = isMax ? __riscv_vmax_tu(acc3, acc3, v3, vl)
|
||||
: __riscv_vmin_tu(acc3, acc3, v3, vl);
|
||||
x += vl;
|
||||
}
|
||||
vint16m1_t seed = __riscv_vmv_s_x_i16m1(initial, __riscv_vsetvlmax_e16m1());
|
||||
dst[0] = (short)__riscv_vmv_x(isMax ? __riscv_vredmax(acc0, seed, vlmax) : __riscv_vredmin(acc0, seed, vlmax));
|
||||
dst[1] = (short)__riscv_vmv_x(isMax ? __riscv_vredmax(acc1, seed, vlmax) : __riscv_vredmin(acc1, seed, vlmax));
|
||||
dst[2] = (short)__riscv_vmv_x(isMax ? __riscv_vredmax(acc2, seed, vlmax) : __riscv_vredmin(acc2, seed, vlmax));
|
||||
dst[3] = (short)__riscv_vmv_x(isMax ? __riscv_vredmax(acc3, seed, vlmax) : __riscv_vredmin(acc3, seed, vlmax));
|
||||
}
|
||||
});
|
||||
v_cleanup();
|
||||
}
|
||||
|
||||
template<bool isMax>
|
||||
static void minMax16uC3(const Mat& srcmat, Mat& dstmat)
|
||||
{
|
||||
const int cols = srcmat.cols;
|
||||
const ushort initial = isMax ? 0 : USHRT_MAX;
|
||||
const int vlmax = __riscv_vsetvlmax_e16m2();
|
||||
parallel_for_(Range(0, srcmat.rows), [&](const Range& range) {
|
||||
for (int y = range.start; y < range.end; y++)
|
||||
{
|
||||
const ushort* src = srcmat.ptr<ushort>(y);
|
||||
ushort* dst = dstmat.ptr<ushort>(y);
|
||||
vuint16m2_t acc0 = __riscv_vmv_v_x_u16m2(initial, vlmax);
|
||||
vuint16m2_t acc1 = acc0, acc2 = acc0;
|
||||
int x = 0;
|
||||
for (; x < cols; )
|
||||
{
|
||||
const int vl = __riscv_vsetvl_e16m2(cols - x);
|
||||
vuint16m2x3_t v = __riscv_vlseg3e16_v_u16m2x3(src + x * 3, vl);
|
||||
vuint16m2_t v0 = __riscv_vget_v_u16m2x3_u16m2(v, 0);
|
||||
vuint16m2_t v1 = __riscv_vget_v_u16m2x3_u16m2(v, 1);
|
||||
vuint16m2_t v2 = __riscv_vget_v_u16m2x3_u16m2(v, 2);
|
||||
acc0 = isMax ? __riscv_vmaxu_tu(acc0, acc0, v0, vl) : __riscv_vminu_tu(acc0, acc0, v0, vl);
|
||||
acc1 = isMax ? __riscv_vmaxu_tu(acc1, acc1, v1, vl) : __riscv_vminu_tu(acc1, acc1, v1, vl);
|
||||
acc2 = isMax ? __riscv_vmaxu_tu(acc2, acc2, v2, vl) : __riscv_vminu_tu(acc2, acc2, v2, vl);
|
||||
x += vl;
|
||||
}
|
||||
vuint16m1_t seed = __riscv_vmv_s_x_u16m1(initial, __riscv_vsetvlmax_e16m1());
|
||||
dst[0] = (ushort)__riscv_vmv_x(isMax ? __riscv_vredmaxu(acc0, seed, vlmax) : __riscv_vredminu(acc0, seed, vlmax));
|
||||
dst[1] = (ushort)__riscv_vmv_x(isMax ? __riscv_vredmaxu(acc1, seed, vlmax) : __riscv_vredminu(acc1, seed, vlmax));
|
||||
dst[2] = (ushort)__riscv_vmv_x(isMax ? __riscv_vredmaxu(acc2, seed, vlmax) : __riscv_vredminu(acc2, seed, vlmax));
|
||||
}
|
||||
});
|
||||
v_cleanup();
|
||||
}
|
||||
|
||||
template<bool isMax>
|
||||
static void minMax16sC3(const Mat& srcmat, Mat& dstmat)
|
||||
{
|
||||
const int cols = srcmat.cols;
|
||||
const short initial = isMax ? SHRT_MIN : SHRT_MAX;
|
||||
const int vlmax = __riscv_vsetvlmax_e16m2();
|
||||
parallel_for_(Range(0, srcmat.rows), [&](const Range& range) {
|
||||
for (int y = range.start; y < range.end; y++)
|
||||
{
|
||||
const short* src = srcmat.ptr<short>(y);
|
||||
short* dst = dstmat.ptr<short>(y);
|
||||
vint16m2_t acc0 = __riscv_vmv_v_x_i16m2(initial, vlmax);
|
||||
vint16m2_t acc1 = acc0, acc2 = acc0;
|
||||
int x = 0;
|
||||
for (; x < cols; )
|
||||
{
|
||||
const int vl = __riscv_vsetvl_e16m2(cols - x);
|
||||
vint16m2x3_t v = __riscv_vlseg3e16_v_i16m2x3(src + x * 3, vl);
|
||||
vint16m2_t v0 = __riscv_vget_v_i16m2x3_i16m2(v, 0);
|
||||
vint16m2_t v1 = __riscv_vget_v_i16m2x3_i16m2(v, 1);
|
||||
vint16m2_t v2 = __riscv_vget_v_i16m2x3_i16m2(v, 2);
|
||||
acc0 = isMax ? __riscv_vmax_tu(acc0, acc0, v0, vl) : __riscv_vmin_tu(acc0, acc0, v0, vl);
|
||||
acc1 = isMax ? __riscv_vmax_tu(acc1, acc1, v1, vl) : __riscv_vmin_tu(acc1, acc1, v1, vl);
|
||||
acc2 = isMax ? __riscv_vmax_tu(acc2, acc2, v2, vl) : __riscv_vmin_tu(acc2, acc2, v2, vl);
|
||||
x += vl;
|
||||
}
|
||||
vint16m1_t seed = __riscv_vmv_s_x_i16m1(initial, __riscv_vsetvlmax_e16m1());
|
||||
dst[0] = (short)__riscv_vmv_x(isMax ? __riscv_vredmax(acc0, seed, vlmax) : __riscv_vredmin(acc0, seed, vlmax));
|
||||
dst[1] = (short)__riscv_vmv_x(isMax ? __riscv_vredmax(acc1, seed, vlmax) : __riscv_vredmin(acc1, seed, vlmax));
|
||||
dst[2] = (short)__riscv_vmv_x(isMax ? __riscv_vredmax(acc2, seed, vlmax) : __riscv_vredmin(acc2, seed, vlmax));
|
||||
}
|
||||
});
|
||||
v_cleanup();
|
||||
}
|
||||
|
||||
template<bool isMax>
|
||||
static void minMax8uC3(const Mat& srcmat, Mat& dstmat)
|
||||
{
|
||||
const int cols = srcmat.cols;
|
||||
const uchar initial = isMax ? 0 : UCHAR_MAX;
|
||||
const int vlmax = __riscv_vsetvlmax_e8m1();
|
||||
parallel_for_(Range(0, srcmat.rows), [&](const Range& range) {
|
||||
for (int y = range.start; y < range.end; y++)
|
||||
{
|
||||
const uchar* src = srcmat.ptr<uchar>(y);
|
||||
uchar* dst = dstmat.ptr<uchar>(y);
|
||||
vuint8m1_t acc0 = __riscv_vmv_v_x_u8m1(initial, vlmax);
|
||||
vuint8m1_t acc1 = acc0, acc2 = acc0;
|
||||
int x = 0;
|
||||
for (; x < cols; )
|
||||
{
|
||||
const int vl = __riscv_vsetvl_e8m1(cols - x);
|
||||
vuint8m1x3_t v = __riscv_vlseg3e8_v_u8m1x3(src + x * 3, vl);
|
||||
vuint8m1_t v0 = __riscv_vget_v_u8m1x3_u8m1(v, 0);
|
||||
vuint8m1_t v1 = __riscv_vget_v_u8m1x3_u8m1(v, 1);
|
||||
vuint8m1_t v2 = __riscv_vget_v_u8m1x3_u8m1(v, 2);
|
||||
acc0 = isMax ? __riscv_vmaxu_tu(acc0, acc0, v0, vl)
|
||||
: __riscv_vminu_tu(acc0, acc0, v0, vl);
|
||||
acc1 = isMax ? __riscv_vmaxu_tu(acc1, acc1, v1, vl)
|
||||
: __riscv_vminu_tu(acc1, acc1, v1, vl);
|
||||
acc2 = isMax ? __riscv_vmaxu_tu(acc2, acc2, v2, vl)
|
||||
: __riscv_vminu_tu(acc2, acc2, v2, vl);
|
||||
x += vl;
|
||||
}
|
||||
vuint8m1_t seed = __riscv_vmv_s_x_u8m1(initial, vlmax);
|
||||
dst[0] = (uchar)__riscv_vmv_x(isMax ? __riscv_vredmaxu(acc0, seed, vlmax)
|
||||
: __riscv_vredminu(acc0, seed, vlmax));
|
||||
dst[1] = (uchar)__riscv_vmv_x(isMax ? __riscv_vredmaxu(acc1, seed, vlmax)
|
||||
: __riscv_vredminu(acc1, seed, vlmax));
|
||||
dst[2] = (uchar)__riscv_vmv_x(isMax ? __riscv_vredmaxu(acc2, seed, vlmax)
|
||||
: __riscv_vredminu(acc2, seed, vlmax));
|
||||
}
|
||||
});
|
||||
v_cleanup();
|
||||
}
|
||||
|
||||
template<bool isMax>
|
||||
static void minMax8uC4(const Mat& srcmat, Mat& dstmat)
|
||||
{
|
||||
const int cols = srcmat.cols;
|
||||
const uchar initial = isMax ? 0 : UCHAR_MAX;
|
||||
const int vlmax = __riscv_vsetvlmax_e8m1();
|
||||
parallel_for_(Range(0, srcmat.rows), [&](const Range& range) {
|
||||
for (int y = range.start; y < range.end; y++)
|
||||
{
|
||||
const uchar* src = srcmat.ptr<uchar>(y);
|
||||
uchar* dst = dstmat.ptr<uchar>(y);
|
||||
vuint8m1_t acc0 = __riscv_vmv_v_x_u8m1(initial, vlmax);
|
||||
vuint8m1_t acc1 = acc0, acc2 = acc0, acc3 = acc0;
|
||||
int x = 0;
|
||||
for (; x < cols; )
|
||||
{
|
||||
const int vl = __riscv_vsetvl_e8m1(cols - x);
|
||||
vuint8m1x4_t v = __riscv_vlseg4e8_v_u8m1x4(src + x * 4, vl);
|
||||
vuint8m1_t v0 = __riscv_vget_v_u8m1x4_u8m1(v, 0);
|
||||
vuint8m1_t v1 = __riscv_vget_v_u8m1x4_u8m1(v, 1);
|
||||
vuint8m1_t v2 = __riscv_vget_v_u8m1x4_u8m1(v, 2);
|
||||
vuint8m1_t v3 = __riscv_vget_v_u8m1x4_u8m1(v, 3);
|
||||
acc0 = isMax ? __riscv_vmaxu_tu(acc0, acc0, v0, vl)
|
||||
: __riscv_vminu_tu(acc0, acc0, v0, vl);
|
||||
acc1 = isMax ? __riscv_vmaxu_tu(acc1, acc1, v1, vl)
|
||||
: __riscv_vminu_tu(acc1, acc1, v1, vl);
|
||||
acc2 = isMax ? __riscv_vmaxu_tu(acc2, acc2, v2, vl)
|
||||
: __riscv_vminu_tu(acc2, acc2, v2, vl);
|
||||
acc3 = isMax ? __riscv_vmaxu_tu(acc3, acc3, v3, vl)
|
||||
: __riscv_vminu_tu(acc3, acc3, v3, vl);
|
||||
x += vl;
|
||||
}
|
||||
vuint8m1_t seed = __riscv_vmv_s_x_u8m1(initial, vlmax);
|
||||
dst[0] = (uchar)__riscv_vmv_x(isMax ? __riscv_vredmaxu(acc0, seed, vlmax)
|
||||
: __riscv_vredminu(acc0, seed, vlmax));
|
||||
dst[1] = (uchar)__riscv_vmv_x(isMax ? __riscv_vredmaxu(acc1, seed, vlmax)
|
||||
: __riscv_vredminu(acc1, seed, vlmax));
|
||||
dst[2] = (uchar)__riscv_vmv_x(isMax ? __riscv_vredmaxu(acc2, seed, vlmax)
|
||||
: __riscv_vredminu(acc2, seed, vlmax));
|
||||
dst[3] = (uchar)__riscv_vmv_x(isMax ? __riscv_vredmaxu(acc3, seed, vlmax)
|
||||
: __riscv_vredminu(acc3, seed, vlmax));
|
||||
}
|
||||
});
|
||||
v_cleanup();
|
||||
}
|
||||
|
||||
template<bool isMax>
|
||||
static void minMax32fC1(const Mat& srcmat, Mat& dstmat)
|
||||
{
|
||||
const int cols = srcmat.cols;
|
||||
parallel_for_(Range(0, srcmat.rows), [&](const Range& range) {
|
||||
for (int y = range.start; y < range.end; y++)
|
||||
{
|
||||
const float* src = srcmat.ptr<float>(y);
|
||||
float result = src[0];
|
||||
int x = 0;
|
||||
const int vlmax = __riscv_vsetvlmax_e32m8();
|
||||
vfloat32m8_t acc = __riscv_vfmv_v_f_f32m8(result, vlmax);
|
||||
for (; x < cols; )
|
||||
{
|
||||
const int vl = __riscv_vsetvl_e32m8(cols - x);
|
||||
vfloat32m8_t v = __riscv_vle32_v_f32m8(src + x, vl);
|
||||
acc = isMax ? __riscv_vfmax_tu(acc, acc, v, vl)
|
||||
: __riscv_vfmin_tu(acc, acc, v, vl);
|
||||
x += vl;
|
||||
}
|
||||
vfloat32m1_t seed = __riscv_vfmv_s_f_f32m1(result, __riscv_vsetvlmax_e32m1());
|
||||
vfloat32m1_t reduced = isMax ? __riscv_vfredmax(acc, seed, vlmax)
|
||||
: __riscv_vfredmin(acc, seed, vlmax);
|
||||
result = __riscv_vfmv_f(reduced);
|
||||
for (; x < cols; x++)
|
||||
result = reduceScalarMinMax<isMax>(result, src[x]);
|
||||
dstmat.ptr<float>(y)[0] = result;
|
||||
}
|
||||
});
|
||||
v_cleanup();
|
||||
}
|
||||
|
||||
template<bool isMax>
|
||||
static void minMax32fC3(const Mat& srcmat, Mat& dstmat)
|
||||
{
|
||||
const int cols = srcmat.cols;
|
||||
const float initial = isMax ? std::numeric_limits<float>::lowest() : std::numeric_limits<float>::max();
|
||||
const int vlmax = __riscv_vsetvlmax_e32m2();
|
||||
parallel_for_(Range(0, srcmat.rows), [&](const Range& range) {
|
||||
for (int y = range.start; y < range.end; y++)
|
||||
{
|
||||
const float* src = srcmat.ptr<float>(y);
|
||||
float* dst = dstmat.ptr<float>(y);
|
||||
vfloat32m2_t acc0 = __riscv_vfmv_v_f_f32m2(initial, vlmax);
|
||||
vfloat32m2_t acc1 = acc0, acc2 = acc0;
|
||||
int x = 0;
|
||||
for (; x < cols; )
|
||||
{
|
||||
const int vl = __riscv_vsetvl_e32m2(cols - x);
|
||||
vfloat32m2x3_t v = __riscv_vlseg3e32_v_f32m2x3(src + x * 3, vl);
|
||||
vfloat32m2_t v0 = __riscv_vget_v_f32m2x3_f32m2(v, 0);
|
||||
vfloat32m2_t v1 = __riscv_vget_v_f32m2x3_f32m2(v, 1);
|
||||
vfloat32m2_t v2 = __riscv_vget_v_f32m2x3_f32m2(v, 2);
|
||||
acc0 = isMax ? __riscv_vfmax_tu(acc0, acc0, v0, vl)
|
||||
: __riscv_vfmin_tu(acc0, acc0, v0, vl);
|
||||
acc1 = isMax ? __riscv_vfmax_tu(acc1, acc1, v1, vl)
|
||||
: __riscv_vfmin_tu(acc1, acc1, v1, vl);
|
||||
acc2 = isMax ? __riscv_vfmax_tu(acc2, acc2, v2, vl)
|
||||
: __riscv_vfmin_tu(acc2, acc2, v2, vl);
|
||||
x += vl;
|
||||
}
|
||||
vfloat32m1_t seed = __riscv_vfmv_s_f_f32m1(initial, __riscv_vsetvlmax_e32m1());
|
||||
dst[0] = __riscv_vfmv_f(isMax ? __riscv_vfredmax(acc0, seed, vlmax)
|
||||
: __riscv_vfredmin(acc0, seed, vlmax));
|
||||
dst[1] = __riscv_vfmv_f(isMax ? __riscv_vfredmax(acc1, seed, vlmax)
|
||||
: __riscv_vfredmin(acc1, seed, vlmax));
|
||||
dst[2] = __riscv_vfmv_f(isMax ? __riscv_vfredmax(acc2, seed, vlmax)
|
||||
: __riscv_vfredmin(acc2, seed, vlmax));
|
||||
}
|
||||
});
|
||||
v_cleanup();
|
||||
}
|
||||
|
||||
template<bool isMax>
|
||||
static void minMax32fC4(const Mat& srcmat, Mat& dstmat)
|
||||
{
|
||||
const int cols = srcmat.cols;
|
||||
const float initial = isMax ? std::numeric_limits<float>::lowest() : std::numeric_limits<float>::max();
|
||||
const int vlmax = __riscv_vsetvlmax_e32m2();
|
||||
parallel_for_(Range(0, srcmat.rows), [&](const Range& range) {
|
||||
for (int y = range.start; y < range.end; y++)
|
||||
{
|
||||
const float* src = srcmat.ptr<float>(y);
|
||||
float* dst = dstmat.ptr<float>(y);
|
||||
vfloat32m2_t acc0 = __riscv_vfmv_v_f_f32m2(initial, vlmax);
|
||||
vfloat32m2_t acc1 = acc0, acc2 = acc0, acc3 = acc0;
|
||||
int x = 0;
|
||||
for (; x < cols; )
|
||||
{
|
||||
const int vl = __riscv_vsetvl_e32m2(cols - x);
|
||||
vfloat32m2x4_t v = __riscv_vlseg4e32_v_f32m2x4(src + x * 4, vl);
|
||||
vfloat32m2_t v0 = __riscv_vget_v_f32m2x4_f32m2(v, 0);
|
||||
vfloat32m2_t v1 = __riscv_vget_v_f32m2x4_f32m2(v, 1);
|
||||
vfloat32m2_t v2 = __riscv_vget_v_f32m2x4_f32m2(v, 2);
|
||||
vfloat32m2_t v3 = __riscv_vget_v_f32m2x4_f32m2(v, 3);
|
||||
acc0 = isMax ? __riscv_vfmax_tu(acc0, acc0, v0, vl)
|
||||
: __riscv_vfmin_tu(acc0, acc0, v0, vl);
|
||||
acc1 = isMax ? __riscv_vfmax_tu(acc1, acc1, v1, vl)
|
||||
: __riscv_vfmin_tu(acc1, acc1, v1, vl);
|
||||
acc2 = isMax ? __riscv_vfmax_tu(acc2, acc2, v2, vl)
|
||||
: __riscv_vfmin_tu(acc2, acc2, v2, vl);
|
||||
acc3 = isMax ? __riscv_vfmax_tu(acc3, acc3, v3, vl)
|
||||
: __riscv_vfmin_tu(acc3, acc3, v3, vl);
|
||||
x += vl;
|
||||
}
|
||||
vfloat32m1_t seed = __riscv_vfmv_s_f_f32m1(initial, __riscv_vsetvlmax_e32m1());
|
||||
dst[0] = __riscv_vfmv_f(isMax ? __riscv_vfredmax(acc0, seed, vlmax)
|
||||
: __riscv_vfredmin(acc0, seed, vlmax));
|
||||
dst[1] = __riscv_vfmv_f(isMax ? __riscv_vfredmax(acc1, seed, vlmax)
|
||||
: __riscv_vfredmin(acc1, seed, vlmax));
|
||||
dst[2] = __riscv_vfmv_f(isMax ? __riscv_vfredmax(acc2, seed, vlmax)
|
||||
: __riscv_vfredmin(acc2, seed, vlmax));
|
||||
dst[3] = __riscv_vfmv_f(isMax ? __riscv_vfredmax(acc3, seed, vlmax)
|
||||
: __riscv_vfredmin(acc3, seed, vlmax));
|
||||
}
|
||||
});
|
||||
v_cleanup();
|
||||
}
|
||||
|
||||
#if CV_SIMD_SCALABLE_64F
|
||||
template<bool isMax>
|
||||
static void minMax64fC1(const Mat& srcmat, Mat& dstmat)
|
||||
{
|
||||
const int cols = srcmat.cols;
|
||||
const double initial = isMax ? std::numeric_limits<double>::lowest()
|
||||
: std::numeric_limits<double>::max();
|
||||
parallel_for_(Range(0, srcmat.rows), [&](const Range& range) {
|
||||
for (int y = range.start; y < range.end; y++)
|
||||
{
|
||||
const double* src = srcmat.ptr<double>(y);
|
||||
const int vlmax = __riscv_vsetvlmax_e64m8();
|
||||
vfloat64m8_t acc = __riscv_vfmv_v_f_f64m8(initial, vlmax);
|
||||
int x = 0;
|
||||
for (; x < cols; )
|
||||
{
|
||||
const int vl = __riscv_vsetvl_e64m8(cols - x);
|
||||
vfloat64m8_t v = __riscv_vle64_v_f64m8(src + x, vl);
|
||||
acc = isMax ? __riscv_vfmax_tu(acc, acc, v, vl)
|
||||
: __riscv_vfmin_tu(acc, acc, v, vl);
|
||||
x += vl;
|
||||
}
|
||||
vfloat64m1_t seed = __riscv_vfmv_s_f_f64m1(initial, __riscv_vsetvlmax_e64m1());
|
||||
vfloat64m1_t reduced = isMax ? __riscv_vfredmax(acc, seed, vlmax)
|
||||
: __riscv_vfredmin(acc, seed, vlmax);
|
||||
dstmat.ptr<double>(y)[0] = __riscv_vfmv_f(reduced);
|
||||
}
|
||||
});
|
||||
v_cleanup();
|
||||
}
|
||||
#endif
|
||||
|
||||
template<typename DT>
|
||||
static void sum2_8uC1(const Mat& srcmat, Mat& dstmat)
|
||||
{
|
||||
const int cols = srcmat.cols;
|
||||
|
||||
parallel_for_(Range(0, srcmat.rows), [&](const Range& range) {
|
||||
for (int y = range.start; y < range.end; y++)
|
||||
{
|
||||
const uchar* src = srcmat.ptr<uchar>(y);
|
||||
DT* dst = dstmat.ptr<DT>(y);
|
||||
uint32_t result = 0;
|
||||
int x = 0;
|
||||
vuint32m1_t acc = __riscv_vmv_v_x_u32m1(0, __riscv_vsetvlmax_e32m1());
|
||||
for (; x < cols; )
|
||||
{
|
||||
const int vl = __riscv_vsetvl_e8m4(cols - x);
|
||||
vuint8m4_t v = __riscv_vle8_v_u8m4(src + x, vl);
|
||||
acc = __riscv_vwredsumu(__riscv_vwmulu(v, v, vl), acc, vl);
|
||||
x += vl;
|
||||
}
|
||||
result = (uint32_t)__riscv_vmv_x(acc);
|
||||
for (; x < cols; x++)
|
||||
result += (uint32_t)src[x] * src[x];
|
||||
dst[0] = (DT)(int32_t)result;
|
||||
}
|
||||
});
|
||||
v_cleanup();
|
||||
}
|
||||
|
||||
static void sum2_16u32fC1(const Mat& srcmat, Mat& dstmat)
|
||||
{
|
||||
const int cols = srcmat.cols;
|
||||
parallel_for_(Range(0, srcmat.rows), [&](const Range& range) {
|
||||
for (int y = range.start; y < range.end; y++)
|
||||
{
|
||||
const ushort* src = srcmat.ptr<ushort>(y);
|
||||
const int vlmax = __riscv_vsetvlmax_e32m8();
|
||||
vfloat32m8_t acc = __riscv_vfmv_v_f_f32m8(0, vlmax);
|
||||
int x = 0;
|
||||
for (; x < cols; )
|
||||
{
|
||||
const int vl = __riscv_vsetvl_e16m4(cols - x);
|
||||
vuint16m4_t v = __riscv_vle16_v_u16m4(src + x, vl);
|
||||
vfloat32m8_t vf = __riscv_vfwcvt_f_xu_v_f32m8(v, vl);
|
||||
acc = __riscv_vfmacc_tu(acc, vf, vf, vl);
|
||||
x += vl;
|
||||
}
|
||||
vfloat32m1_t zero = __riscv_vfmv_s_f_f32m1(0, __riscv_vsetvlmax_e32m1());
|
||||
dstmat.ptr<float>(y)[0] = __riscv_vfmv_f(__riscv_vfredusum(acc, zero, vlmax));
|
||||
}
|
||||
});
|
||||
v_cleanup();
|
||||
}
|
||||
|
||||
static void sum2_16s32fC1(const Mat& srcmat, Mat& dstmat)
|
||||
{
|
||||
const int cols = srcmat.cols;
|
||||
parallel_for_(Range(0, srcmat.rows), [&](const Range& range) {
|
||||
for (int y = range.start; y < range.end; y++)
|
||||
{
|
||||
const short* src = srcmat.ptr<short>(y);
|
||||
const int vlmax = __riscv_vsetvlmax_e32m8();
|
||||
vfloat32m8_t acc = __riscv_vfmv_v_f_f32m8(0, vlmax);
|
||||
int x = 0;
|
||||
for (; x < cols; )
|
||||
{
|
||||
const int vl = __riscv_vsetvl_e16m4(cols - x);
|
||||
vint16m4_t v = __riscv_vle16_v_i16m4(src + x, vl);
|
||||
vfloat32m8_t vf = __riscv_vfwcvt_f_x_v_f32m8(v, vl);
|
||||
acc = __riscv_vfmacc_tu(acc, vf, vf, vl);
|
||||
x += vl;
|
||||
}
|
||||
vfloat32m1_t zero = __riscv_vfmv_s_f_f32m1(0, __riscv_vsetvlmax_e32m1());
|
||||
dstmat.ptr<float>(y)[0] = __riscv_vfmv_f(__riscv_vfredusum(acc, zero, vlmax));
|
||||
}
|
||||
});
|
||||
v_cleanup();
|
||||
}
|
||||
|
||||
#if CV_SIMD_SCALABLE_64F
|
||||
static void sum2_16u64fC1(const Mat& srcmat, Mat& dstmat)
|
||||
{
|
||||
const int cols = srcmat.cols;
|
||||
parallel_for_(Range(0, srcmat.rows), [&](const Range& range) {
|
||||
for (int y = range.start; y < range.end; y++)
|
||||
{
|
||||
const ushort* src = srcmat.ptr<ushort>(y);
|
||||
const int vlmax = __riscv_vsetvlmax_e64m8();
|
||||
vfloat64m8_t acc = __riscv_vfmv_v_f_f64m8(0, vlmax);
|
||||
int x = 0;
|
||||
for (; x < cols; )
|
||||
{
|
||||
const int vl = __riscv_vsetvl_e16m2(cols - x);
|
||||
vuint16m2_t v = __riscv_vle16_v_u16m2(src + x, vl);
|
||||
vfloat32m4_t vf = __riscv_vfwcvt_f_xu_v_f32m4(v, vl);
|
||||
vfloat64m8_t vd = __riscv_vfwcvt_f_f_v_f64m8(vf, vl);
|
||||
acc = __riscv_vfmacc_tu(acc, vd, vd, vl);
|
||||
x += vl;
|
||||
}
|
||||
vfloat64m1_t zero = __riscv_vfmv_s_f_f64m1(0, __riscv_vsetvlmax_e64m1());
|
||||
dstmat.ptr<double>(y)[0] = __riscv_vfmv_f(__riscv_vfredusum(acc, zero, vlmax));
|
||||
}
|
||||
});
|
||||
v_cleanup();
|
||||
}
|
||||
|
||||
static void sum2_16s64fC1(const Mat& srcmat, Mat& dstmat)
|
||||
{
|
||||
const int cols = srcmat.cols;
|
||||
parallel_for_(Range(0, srcmat.rows), [&](const Range& range) {
|
||||
for (int y = range.start; y < range.end; y++)
|
||||
{
|
||||
const short* src = srcmat.ptr<short>(y);
|
||||
const int vlmax = __riscv_vsetvlmax_e64m8();
|
||||
vfloat64m8_t acc = __riscv_vfmv_v_f_f64m8(0, vlmax);
|
||||
int x = 0;
|
||||
for (; x < cols; )
|
||||
{
|
||||
const int vl = __riscv_vsetvl_e16m2(cols - x);
|
||||
vint16m2_t v = __riscv_vle16_v_i16m2(src + x, vl);
|
||||
vfloat32m4_t vf = __riscv_vfwcvt_f_x_v_f32m4(v, vl);
|
||||
vfloat64m8_t vd = __riscv_vfwcvt_f_f_v_f64m8(vf, vl);
|
||||
acc = __riscv_vfmacc_tu(acc, vd, vd, vl);
|
||||
x += vl;
|
||||
}
|
||||
vfloat64m1_t zero = __riscv_vfmv_s_f_f64m1(0, __riscv_vsetvlmax_e64m1());
|
||||
dstmat.ptr<double>(y)[0] = __riscv_vfmv_f(__riscv_vfredusum(acc, zero, vlmax));
|
||||
}
|
||||
});
|
||||
v_cleanup();
|
||||
}
|
||||
|
||||
static void sum2_32f64fC1(const Mat& srcmat, Mat& dstmat)
|
||||
{
|
||||
const int cols = srcmat.cols;
|
||||
parallel_for_(Range(0, srcmat.rows), [&](const Range& range) {
|
||||
for (int y = range.start; y < range.end; y++)
|
||||
{
|
||||
const float* src = srcmat.ptr<float>(y);
|
||||
const int vlmax = __riscv_vsetvlmax_e64m8();
|
||||
vfloat64m8_t acc = __riscv_vfmv_v_f_f64m8(0, vlmax);
|
||||
int x = 0;
|
||||
for (; x < cols; )
|
||||
{
|
||||
const int vl = __riscv_vsetvl_e32m4(cols - x);
|
||||
vfloat32m4_t v = __riscv_vle32_v_f32m4(src + x, vl);
|
||||
vfloat64m8_t vd = __riscv_vfwcvt_f_f_v_f64m8(v, vl);
|
||||
acc = __riscv_vfmacc_tu(acc, vd, vd, vl);
|
||||
x += vl;
|
||||
}
|
||||
vfloat64m1_t zero = __riscv_vfmv_s_f_f64m1(0, __riscv_vsetvlmax_e64m1());
|
||||
dstmat.ptr<double>(y)[0] = __riscv_vfmv_f(__riscv_vfredusum(acc, zero, vlmax));
|
||||
}
|
||||
});
|
||||
v_cleanup();
|
||||
}
|
||||
|
||||
static void sum2_64f64fC1(const Mat& srcmat, Mat& dstmat)
|
||||
{
|
||||
const int cols = srcmat.cols;
|
||||
parallel_for_(Range(0, srcmat.rows), [&](const Range& range) {
|
||||
for (int y = range.start; y < range.end; y++)
|
||||
{
|
||||
const double* src = srcmat.ptr<double>(y);
|
||||
const int vlmax = __riscv_vsetvlmax_e64m8();
|
||||
vfloat64m8_t acc = __riscv_vfmv_v_f_f64m8(0, vlmax);
|
||||
int x = 0;
|
||||
for (; x < cols; )
|
||||
{
|
||||
const int vl = __riscv_vsetvl_e64m8(cols - x);
|
||||
vfloat64m8_t v = __riscv_vle64_v_f64m8(src + x, vl);
|
||||
acc = __riscv_vfmacc_tu(acc, v, v, vl);
|
||||
x += vl;
|
||||
}
|
||||
vfloat64m1_t zero = __riscv_vfmv_s_f_f64m1(0, __riscv_vsetvlmax_e64m1());
|
||||
dstmat.ptr<double>(y)[0] = __riscv_vfmv_f(__riscv_vfredusum(acc, zero, vlmax));
|
||||
}
|
||||
});
|
||||
v_cleanup();
|
||||
}
|
||||
#endif
|
||||
|
||||
template<typename DT>
|
||||
static void sum2_8uC3(const Mat& srcmat, Mat& dstmat)
|
||||
{
|
||||
const int cols = srcmat.cols;
|
||||
parallel_for_(Range(0, srcmat.rows), [&](const Range& range) {
|
||||
for (int y = range.start; y < range.end; y++)
|
||||
{
|
||||
const uchar* src = srcmat.ptr<uchar>(y);
|
||||
DT* dst = dstmat.ptr<DT>(y);
|
||||
vuint32m1_t acc0 = __riscv_vmv_v_x_u32m1(0, __riscv_vsetvlmax_e32m1());
|
||||
vuint32m1_t acc1 = acc0, acc2 = acc0;
|
||||
int x = 0;
|
||||
for (; x < cols; )
|
||||
{
|
||||
const int vl = __riscv_vsetvl_e8m1(cols - x);
|
||||
vuint8m1x3_t v = __riscv_vlseg3e8_v_u8m1x3(src + x * 3, vl);
|
||||
vuint8m1_t v0 = __riscv_vget_v_u8m1x3_u8m1(v, 0);
|
||||
vuint8m1_t v1 = __riscv_vget_v_u8m1x3_u8m1(v, 1);
|
||||
vuint8m1_t v2 = __riscv_vget_v_u8m1x3_u8m1(v, 2);
|
||||
acc0 = __riscv_vwredsumu(__riscv_vwmulu(v0, v0, vl), acc0, vl);
|
||||
acc1 = __riscv_vwredsumu(__riscv_vwmulu(v1, v1, vl), acc1, vl);
|
||||
acc2 = __riscv_vwredsumu(__riscv_vwmulu(v2, v2, vl), acc2, vl);
|
||||
x += vl;
|
||||
}
|
||||
dst[0] = (DT)(int32_t)(uint32_t)__riscv_vmv_x(acc0);
|
||||
dst[1] = (DT)(int32_t)(uint32_t)__riscv_vmv_x(acc1);
|
||||
dst[2] = (DT)(int32_t)(uint32_t)__riscv_vmv_x(acc2);
|
||||
}
|
||||
});
|
||||
v_cleanup();
|
||||
}
|
||||
|
||||
template<typename DT>
|
||||
static void sum2_8uC4(const Mat& srcmat, Mat& dstmat)
|
||||
{
|
||||
const int cols = srcmat.cols;
|
||||
parallel_for_(Range(0, srcmat.rows), [&](const Range& range) {
|
||||
for (int y = range.start; y < range.end; y++)
|
||||
{
|
||||
const uchar* src = srcmat.ptr<uchar>(y);
|
||||
DT* dst = dstmat.ptr<DT>(y);
|
||||
vuint32m1_t acc0 = __riscv_vmv_v_x_u32m1(0, __riscv_vsetvlmax_e32m1());
|
||||
vuint32m1_t acc1 = acc0, acc2 = acc0, acc3 = acc0;
|
||||
int x = 0;
|
||||
for (; x < cols; )
|
||||
{
|
||||
const int vl = __riscv_vsetvl_e8m1(cols - x);
|
||||
vuint8m1x4_t v = __riscv_vlseg4e8_v_u8m1x4(src + x * 4, vl);
|
||||
vuint8m1_t v0 = __riscv_vget_v_u8m1x4_u8m1(v, 0);
|
||||
vuint8m1_t v1 = __riscv_vget_v_u8m1x4_u8m1(v, 1);
|
||||
vuint8m1_t v2 = __riscv_vget_v_u8m1x4_u8m1(v, 2);
|
||||
vuint8m1_t v3 = __riscv_vget_v_u8m1x4_u8m1(v, 3);
|
||||
acc0 = __riscv_vwredsumu(__riscv_vwmulu(v0, v0, vl), acc0, vl);
|
||||
acc1 = __riscv_vwredsumu(__riscv_vwmulu(v1, v1, vl), acc1, vl);
|
||||
acc2 = __riscv_vwredsumu(__riscv_vwmulu(v2, v2, vl), acc2, vl);
|
||||
acc3 = __riscv_vwredsumu(__riscv_vwmulu(v3, v3, vl), acc3, vl);
|
||||
x += vl;
|
||||
}
|
||||
dst[0] = (DT)(int32_t)(uint32_t)__riscv_vmv_x(acc0);
|
||||
dst[1] = (DT)(int32_t)(uint32_t)__riscv_vmv_x(acc1);
|
||||
dst[2] = (DT)(int32_t)(uint32_t)__riscv_vmv_x(acc2);
|
||||
dst[3] = (DT)(int32_t)(uint32_t)__riscv_vmv_x(acc3);
|
||||
}
|
||||
});
|
||||
v_cleanup();
|
||||
}
|
||||
|
||||
static void sum2_32fC1(const Mat& srcmat, Mat& dstmat)
|
||||
{
|
||||
const int cols = srcmat.cols;
|
||||
parallel_for_(Range(0, srcmat.rows), [&](const Range& range) {
|
||||
for (int y = range.start; y < range.end; y++)
|
||||
{
|
||||
const float* src = srcmat.ptr<float>(y);
|
||||
float result = 0;
|
||||
int x = 0;
|
||||
const int vlmax = __riscv_vsetvlmax_e32m8();
|
||||
vfloat32m8_t acc = __riscv_vfmv_v_f_f32m8(0, vlmax);
|
||||
for (; x < cols; )
|
||||
{
|
||||
const int vl = __riscv_vsetvl_e32m8(cols - x);
|
||||
vfloat32m8_t v = __riscv_vle32_v_f32m8(src + x, vl);
|
||||
acc = __riscv_vfmacc_tu(acc, v, v, vl);
|
||||
x += vl;
|
||||
}
|
||||
vfloat32m1_t zero = __riscv_vfmv_s_f_f32m1(0, __riscv_vsetvlmax_e32m1());
|
||||
result = __riscv_vfmv_f(__riscv_vfredusum(acc, zero, vlmax));
|
||||
for (; x < cols; x++)
|
||||
result += src[x] * src[x];
|
||||
dstmat.ptr<float>(y)[0] = result;
|
||||
}
|
||||
});
|
||||
v_cleanup();
|
||||
}
|
||||
|
||||
static void sum2_32fC3(const Mat& srcmat, Mat& dstmat)
|
||||
{
|
||||
const int cols = srcmat.cols;
|
||||
const int vlmax = __riscv_vsetvlmax_e32m2();
|
||||
parallel_for_(Range(0, srcmat.rows), [&](const Range& range) {
|
||||
for (int y = range.start; y < range.end; y++)
|
||||
{
|
||||
const float* src = srcmat.ptr<float>(y);
|
||||
float* dst = dstmat.ptr<float>(y);
|
||||
vfloat32m2_t acc0 = __riscv_vfmv_v_f_f32m2(0, vlmax);
|
||||
vfloat32m2_t acc1 = acc0, acc2 = acc0;
|
||||
int x = 0;
|
||||
for (; x < cols; )
|
||||
{
|
||||
const int vl = __riscv_vsetvl_e32m2(cols - x);
|
||||
vfloat32m2x3_t v = __riscv_vlseg3e32_v_f32m2x3(src + x * 3, vl);
|
||||
vfloat32m2_t v0 = __riscv_vget_v_f32m2x3_f32m2(v, 0);
|
||||
vfloat32m2_t v1 = __riscv_vget_v_f32m2x3_f32m2(v, 1);
|
||||
vfloat32m2_t v2 = __riscv_vget_v_f32m2x3_f32m2(v, 2);
|
||||
acc0 = __riscv_vfmacc_tu(acc0, v0, v0, vl);
|
||||
acc1 = __riscv_vfmacc_tu(acc1, v1, v1, vl);
|
||||
acc2 = __riscv_vfmacc_tu(acc2, v2, v2, vl);
|
||||
x += vl;
|
||||
}
|
||||
vfloat32m1_t zero = __riscv_vfmv_s_f_f32m1(0, __riscv_vsetvlmax_e32m1());
|
||||
dst[0] = __riscv_vfmv_f(__riscv_vfredusum(acc0, zero, vlmax));
|
||||
dst[1] = __riscv_vfmv_f(__riscv_vfredusum(acc1, zero, vlmax));
|
||||
dst[2] = __riscv_vfmv_f(__riscv_vfredusum(acc2, zero, vlmax));
|
||||
}
|
||||
});
|
||||
v_cleanup();
|
||||
}
|
||||
|
||||
static void sum2_32fC4(const Mat& srcmat, Mat& dstmat)
|
||||
{
|
||||
const int cols = srcmat.cols;
|
||||
const int vlmax = __riscv_vsetvlmax_e32m2();
|
||||
parallel_for_(Range(0, srcmat.rows), [&](const Range& range) {
|
||||
for (int y = range.start; y < range.end; y++)
|
||||
{
|
||||
const float* src = srcmat.ptr<float>(y);
|
||||
float* dst = dstmat.ptr<float>(y);
|
||||
vfloat32m2_t acc0 = __riscv_vfmv_v_f_f32m2(0, vlmax);
|
||||
vfloat32m2_t acc1 = acc0, acc2 = acc0, acc3 = acc0;
|
||||
int x = 0;
|
||||
for (; x < cols; )
|
||||
{
|
||||
const int vl = __riscv_vsetvl_e32m2(cols - x);
|
||||
vfloat32m2x4_t v = __riscv_vlseg4e32_v_f32m2x4(src + x * 4, vl);
|
||||
vfloat32m2_t v0 = __riscv_vget_v_f32m2x4_f32m2(v, 0);
|
||||
vfloat32m2_t v1 = __riscv_vget_v_f32m2x4_f32m2(v, 1);
|
||||
vfloat32m2_t v2 = __riscv_vget_v_f32m2x4_f32m2(v, 2);
|
||||
vfloat32m2_t v3 = __riscv_vget_v_f32m2x4_f32m2(v, 3);
|
||||
acc0 = __riscv_vfmacc_tu(acc0, v0, v0, vl);
|
||||
acc1 = __riscv_vfmacc_tu(acc1, v1, v1, vl);
|
||||
acc2 = __riscv_vfmacc_tu(acc2, v2, v2, vl);
|
||||
acc3 = __riscv_vfmacc_tu(acc3, v3, v3, vl);
|
||||
x += vl;
|
||||
}
|
||||
vfloat32m1_t zero = __riscv_vfmv_s_f_f32m1(0, __riscv_vsetvlmax_e32m1());
|
||||
dst[0] = __riscv_vfmv_f(__riscv_vfredusum(acc0, zero, vlmax));
|
||||
dst[1] = __riscv_vfmv_f(__riscv_vfredusum(acc1, zero, vlmax));
|
||||
dst[2] = __riscv_vfmv_f(__riscv_vfredusum(acc2, zero, vlmax));
|
||||
dst[3] = __riscv_vfmv_f(__riscv_vfredusum(acc3, zero, vlmax));
|
||||
}
|
||||
});
|
||||
v_cleanup();
|
||||
}
|
||||
|
||||
} // namespace reduce_c_rvv
|
||||
@@ -170,7 +170,7 @@ int Core_ReduceTest::checkOp( const Mat& src, int dstType, int opType, const Mat
|
||||
getMatTypeStr( dstType, dstTypeStr );
|
||||
const char* dimStr = dim == 0 ? "ROWS" : "COLS";
|
||||
|
||||
snprintf( msg, sizeof(msg), "bad accuracy with srcType = %s, dstType = %s, opType = %s, dim = %s",
|
||||
snprintf( msg, sizeof(msg), "bad accuracy with srcType = %s, dstType = %s, opType = %s, dim = %s\n",
|
||||
srcTypeStr.c_str(), dstTypeStr.c_str(), opTypeStr, dimStr );
|
||||
ts->printf( cvtest::TS::LOG, msg );
|
||||
return cvtest::TS::FAIL_BAD_ACCURACY;
|
||||
|
||||
Reference in New Issue
Block a user