potok/opencv
1
0
Fork 0
mirror of https://github.com/opencv/opencv.git synced 2026-07-29 23:33:05 +04:00
Code Packages Releases Wiki Activity

Extended several core functions to support new types (#24962)

Browse Source 
* started adding support for new types (16f, 16bf, 32u, 64u, 64s) to arithmetic functions

* fixed several tests; refactored and extended sum(), extended inRange().

* extended countNonZero(), mean(), meanStdDev(), minMaxIdx(), norm() and sum() to support new types (F16, BF16, U32, U64, S64)

* put missing CV_DEPTH_MAX to some function dispatcher tables
* extended findnonzero, hasnonzero with the new types support

* extended mixChannels() to support new types

* minor fix

* fixed a few compile errors on Linux and a few failures in core tests

* fixed a few more warnings and test failures

* trying to fix the remaining warnings and test failures. The test `MulTestGPU.MathOpTest` was disabled - not clear whether to set tolerance - it's not bit-exact operation, as possibly assumed by the test, due to the use of scale and possibly limited accuracy of the intermediate floating-point calculations.

* found that in the current snapshot G-API produces incorrect results in Mul, Div and AddWeighted (at least when using OpenCL on Windows x64 or MacOS x64). Disabled the respective tests.
This commit is contained in:
Vadim Pisarevsky
2024-02-11 10:42:41 +03:00
committed by GitHub
parent f05ef64df8
commit 1d18aba587
45 changed files with 3286 additions and 4706 deletions
Download Patch File Download Diff File Expand all files Collapse all files
modules/core/src/arithm.cpp
+216 -43
View File
@@ -331,10 +331,19 @@ static BinaryFuncC* getMaxTab()
{
static BinaryFuncC maxTab[CV_DEPTH_MAX] =
{
(BinaryFuncC)GET_OPTIMIZED(cv::hal::max8u), (BinaryFuncC)GET_OPTIMIZED(cv::hal::max8s),
(BinaryFuncC)GET_OPTIMIZED(cv::hal::max16u), (BinaryFuncC)GET_OPTIMIZED(cv::hal::max16s),
(BinaryFuncC)GET_OPTIMIZED(cv::hal::max8u),
(BinaryFuncC)GET_OPTIMIZED(cv::hal::max8s),
(BinaryFuncC)GET_OPTIMIZED(cv::hal::max16u),
(BinaryFuncC)GET_OPTIMIZED(cv::hal::max16s),
(BinaryFuncC)GET_OPTIMIZED(cv::hal::max32s),
(BinaryFuncC)GET_OPTIMIZED(cv::hal::max32f), (BinaryFuncC)cv::hal::max64f,
(BinaryFuncC)GET_OPTIMIZED(cv::hal::max32f),
(BinaryFuncC)cv::hal::max64f,
(BinaryFuncC)cv::hal::max16f,
(BinaryFuncC)cv::hal::max16bf,
(BinaryFuncC)GET_OPTIMIZED(cv::hal::max8u), // bool
(BinaryFuncC)cv::hal::max64u,
(BinaryFuncC)cv::hal::max64s,
(BinaryFuncC)cv::hal::max32u,
0
};
@@ -345,10 +354,19 @@ static BinaryFuncC* getMinTab()
{
static BinaryFuncC minTab[CV_DEPTH_MAX] =
{
(BinaryFuncC)GET_OPTIMIZED(cv::hal::min8u), (BinaryFuncC)GET_OPTIMIZED(cv::hal::min8s),
(BinaryFuncC)GET_OPTIMIZED(cv::hal::min16u), (BinaryFuncC)GET_OPTIMIZED(cv::hal::min16s),
(BinaryFuncC)GET_OPTIMIZED(cv::hal::min8u),
(BinaryFuncC)GET_OPTIMIZED(cv::hal::min8s),
(BinaryFuncC)GET_OPTIMIZED(cv::hal::min16u),
(BinaryFuncC)GET_OPTIMIZED(cv::hal::min16s),
(BinaryFuncC)GET_OPTIMIZED(cv::hal::min32s),
(BinaryFuncC)GET_OPTIMIZED(cv::hal::min32f), (BinaryFuncC)cv::hal::min64f,
(BinaryFuncC)GET_OPTIMIZED(cv::hal::min32f),
(BinaryFuncC)cv::hal::min64f,
(BinaryFuncC)cv::hal::min16f,
(BinaryFuncC)cv::hal::min16bf,
(BinaryFuncC)GET_OPTIMIZED(cv::hal::min8u), // bool
(BinaryFuncC)cv::hal::min64u,
(BinaryFuncC)cv::hal::min64s,
(BinaryFuncC)cv::hal::min32u,
0
};
@@ -462,6 +480,14 @@ static int actualScalarDepth(const double* data, int len)
CV_32S;
}
static int coerceTypes(int depth1, int depth2, bool muldiv)
{
return depth1 == depth2 ? depth1 :
((depth1 <= CV_32S) & (depth2 <= CV_32S)) != 0 ?
(((int)!muldiv & (depth1 <= CV_8S) & (depth2 <= CV_8S)) != 0 ? CV_16S : CV_32S) :
((CV_ELEM_SIZE1(depth1) > 4) | (CV_ELEM_SIZE1(depth2) > 4)) != 0 ? CV_64F : CV_32F;
}
#ifdef HAVE_OPENCL
static bool ocl_arithm_op(InputArray _src1, InputArray _src2, OutputArray _dst,
@@ -658,7 +684,7 @@ static void arithm_op(InputArray _src1, InputArray _src2, OutputArray _dst,
{
Mat sc = psrc2->getMat();
depth2 = actualScalarDepth(sc.ptr<double>(), sz2 == Size(1, 1) ? cn2 : cn);
if( depth2 == CV_64F && (depth1 < CV_32S || depth1 == CV_32F) )
if( depth2 == CV_64F && CV_ELEM_SIZE1(depth1) < 8 )
depth2 = CV_32F;
}
else
@@ -684,9 +710,8 @@ static void arithm_op(InputArray _src1, InputArray _src2, OutputArray _dst,
wtype = dtype;
else if( !muldiv )
{
wtype = depth1 <= CV_8S && depth2 <= CV_8S ? CV_16S :
depth1 <= CV_32S && depth2 <= CV_32S ? CV_32S : std::max(depth1, depth2);
wtype = std::max(wtype, dtype);
wtype = coerceTypes(depth1, depth2, false);
wtype = coerceTypes(wtype, dtype, false);
// when the result of addition should be converted to an integer type,
// and just one of the input arrays is floating-point, it makes sense to convert that input to integer type before the operation,
@@ -696,8 +721,8 @@ static void arithm_op(InputArray _src1, InputArray _src2, OutputArray _dst,
}
else
{
wtype = std::max(depth1, std::max(depth2, CV_32F));
wtype = std::max(wtype, dtype);
wtype = coerceTypes(depth1, depth2, true);
wtype = coerceTypes(wtype, dtype, true);
}
dtype = CV_MAKETYPE(dtype, cn);
@@ -873,10 +898,19 @@ static BinaryFuncC* getAddTab()
{
static BinaryFuncC addTab[CV_DEPTH_MAX] =
{
(BinaryFuncC)GET_OPTIMIZED(cv::hal::add8u), (BinaryFuncC)GET_OPTIMIZED(cv::hal::add8s),
(BinaryFuncC)GET_OPTIMIZED(cv::hal::add16u), (BinaryFuncC)GET_OPTIMIZED(cv::hal::add16s),
(BinaryFuncC)GET_OPTIMIZED(cv::hal::add8u),
(BinaryFuncC)GET_OPTIMIZED(cv::hal::add8s),
(BinaryFuncC)GET_OPTIMIZED(cv::hal::add16u),
(BinaryFuncC)GET_OPTIMIZED(cv::hal::add16s),
(BinaryFuncC)GET_OPTIMIZED(cv::hal::add32s),
(BinaryFuncC)GET_OPTIMIZED(cv::hal::add32f), (BinaryFuncC)cv::hal::add64f,
(BinaryFuncC)GET_OPTIMIZED(cv::hal::add32f),
(BinaryFuncC)cv::hal::add64f,
(BinaryFuncC)cv::hal::add16f,
(BinaryFuncC)cv::hal::add16bf,
0,
(BinaryFuncC)cv::hal::add64u,
(BinaryFuncC)cv::hal::add64s,
(BinaryFuncC)cv::hal::add32u,
0
};
@@ -887,10 +921,19 @@ static BinaryFuncC* getSubTab()
{
static BinaryFuncC subTab[CV_DEPTH_MAX] =
{
(BinaryFuncC)GET_OPTIMIZED(cv::hal::sub8u), (BinaryFuncC)GET_OPTIMIZED(cv::hal::sub8s),
(BinaryFuncC)GET_OPTIMIZED(cv::hal::sub16u), (BinaryFuncC)GET_OPTIMIZED(cv::hal::sub16s),
(BinaryFuncC)GET_OPTIMIZED(cv::hal::sub8u),
(BinaryFuncC)GET_OPTIMIZED(cv::hal::sub8s),
(BinaryFuncC)GET_OPTIMIZED(cv::hal::sub16u),
(BinaryFuncC)GET_OPTIMIZED(cv::hal::sub16s),
(BinaryFuncC)GET_OPTIMIZED(cv::hal::sub32s),
(BinaryFuncC)GET_OPTIMIZED(cv::hal::sub32f), (BinaryFuncC)cv::hal::sub64f,
(BinaryFuncC)GET_OPTIMIZED(cv::hal::sub32f),
(BinaryFuncC)cv::hal::sub64f,
(BinaryFuncC)cv::hal::sub16f,
(BinaryFuncC)cv::hal::sub16bf,
0,
(BinaryFuncC)cv::hal::sub64u,
(BinaryFuncC)cv::hal::sub64s,
(BinaryFuncC)cv::hal::sub32u,
0
};
@@ -901,10 +944,19 @@ static BinaryFuncC* getAbsDiffTab()
{
static BinaryFuncC absDiffTab[CV_DEPTH_MAX] =
{
(BinaryFuncC)GET_OPTIMIZED(cv::hal::absdiff8u), (BinaryFuncC)GET_OPTIMIZED(cv::hal::absdiff8s),
(BinaryFuncC)GET_OPTIMIZED(cv::hal::absdiff16u), (BinaryFuncC)GET_OPTIMIZED(cv::hal::absdiff16s),
(BinaryFuncC)GET_OPTIMIZED(cv::hal::absdiff8u),
(BinaryFuncC)GET_OPTIMIZED(cv::hal::absdiff8s),
(BinaryFuncC)GET_OPTIMIZED(cv::hal::absdiff16u),
(BinaryFuncC)GET_OPTIMIZED(cv::hal::absdiff16s),
(BinaryFuncC)GET_OPTIMIZED(cv::hal::absdiff32s),
(BinaryFuncC)GET_OPTIMIZED(cv::hal::absdiff32f), (BinaryFuncC)cv::hal::absdiff64f,
(BinaryFuncC)GET_OPTIMIZED(cv::hal::absdiff32f),
(BinaryFuncC)cv::hal::absdiff64f,
(BinaryFuncC)cv::hal::absdiff16f,
(BinaryFuncC)cv::hal::absdiff16bf,
0,
(BinaryFuncC)cv::hal::absdiff64u,
(BinaryFuncC)cv::hal::absdiff64s,
(BinaryFuncC)cv::hal::absdiff32u,
0
};
@@ -956,7 +1008,8 @@ static BinaryFuncC* getMulTab()
{
(BinaryFuncC)cv::hal::mul8u, (BinaryFuncC)cv::hal::mul8s, (BinaryFuncC)cv::hal::mul16u,
(BinaryFuncC)cv::hal::mul16s, (BinaryFuncC)cv::hal::mul32s, (BinaryFuncC)cv::hal::mul32f,
(BinaryFuncC)cv::hal::mul64f, 0
(BinaryFuncC)cv::hal::mul64f, (BinaryFuncC)cv::hal::mul16f, (BinaryFuncC)cv::hal::mul16bf, 0,
(BinaryFuncC)cv::hal::mul64u, (BinaryFuncC)cv::hal::mul64s, (BinaryFuncC)cv::hal::mul32u, 0
};
return mulTab;
@@ -968,7 +1021,8 @@ static BinaryFuncC* getDivTab()
{
(BinaryFuncC)cv::hal::div8u, (BinaryFuncC)cv::hal::div8s, (BinaryFuncC)cv::hal::div16u,
(BinaryFuncC)cv::hal::div16s, (BinaryFuncC)cv::hal::div32s, (BinaryFuncC)cv::hal::div32f,
(BinaryFuncC)cv::hal::div64f, 0
(BinaryFuncC)cv::hal::div64f, (BinaryFuncC)cv::hal::div16f, (BinaryFuncC)cv::hal::div16bf, 0,
(BinaryFuncC)cv::hal::div64u, (BinaryFuncC)cv::hal::div64s, (BinaryFuncC)cv::hal::div32u, 0
};
return divTab;
@@ -980,7 +1034,8 @@ static BinaryFuncC* getRecipTab()
{
(BinaryFuncC)cv::hal::recip8u, (BinaryFuncC)cv::hal::recip8s, (BinaryFuncC)cv::hal::recip16u,
(BinaryFuncC)cv::hal::recip16s, (BinaryFuncC)cv::hal::recip32s, (BinaryFuncC)cv::hal::recip32f,
(BinaryFuncC)cv::hal::recip64f, 0
(BinaryFuncC)cv::hal::recip64f, (BinaryFuncC)cv::hal::recip16f, (BinaryFuncC)cv::hal::recip16bf, 0,
(BinaryFuncC)cv::hal::recip64u, (BinaryFuncC)cv::hal::recip64s, (BinaryFuncC)cv::hal::recip32u, 0
};
return recipTab;
@@ -1026,9 +1081,18 @@ static BinaryFuncC* getAddWeightedTab()
{
static BinaryFuncC addWeightedTab[CV_DEPTH_MAX] =
{
(BinaryFuncC)GET_OPTIMIZED(cv::hal::addWeighted8u), (BinaryFuncC)GET_OPTIMIZED(cv::hal::addWeighted8s), (BinaryFuncC)GET_OPTIMIZED(cv::hal::addWeighted16u),
(BinaryFuncC)GET_OPTIMIZED(cv::hal::addWeighted16s), (BinaryFuncC)GET_OPTIMIZED(cv::hal::addWeighted32s), (BinaryFuncC)cv::hal::addWeighted32f,
(BinaryFuncC)cv::hal::addWeighted64f, 0
(BinaryFuncC)GET_OPTIMIZED(cv::hal::addWeighted8u),
(BinaryFuncC)GET_OPTIMIZED(cv::hal::addWeighted8s),
(BinaryFuncC)GET_OPTIMIZED(cv::hal::addWeighted16u),
(BinaryFuncC)GET_OPTIMIZED(cv::hal::addWeighted16s),
(BinaryFuncC)GET_OPTIMIZED(cv::hal::addWeighted32s),
(BinaryFuncC)cv::hal::addWeighted32f,
(BinaryFuncC)cv::hal::addWeighted64f,
(BinaryFuncC)cv::hal::addWeighted16f,
(BinaryFuncC)cv::hal::addWeighted16bf, 0,
(BinaryFuncC)cv::hal::addWeighted64u,
(BinaryFuncC)cv::hal::addWeighted64s,
(BinaryFuncC)cv::hal::addWeighted32u, 0
};
return addWeightedTab;
@@ -1057,10 +1121,19 @@ static BinaryFuncC getCmpFunc(int depth)
{
static BinaryFuncC cmpTab[CV_DEPTH_MAX] =
{
(BinaryFuncC)GET_OPTIMIZED(cv::hal::cmp8u), (BinaryFuncC)GET_OPTIMIZED(cv::hal::cmp8s),
(BinaryFuncC)GET_OPTIMIZED(cv::hal::cmp16u), (BinaryFuncC)GET_OPTIMIZED(cv::hal::cmp16s),
(BinaryFuncC)GET_OPTIMIZED(cv::hal::cmp8u),
(BinaryFuncC)GET_OPTIMIZED(cv::hal::cmp8s),
(BinaryFuncC)GET_OPTIMIZED(cv::hal::cmp16u),
(BinaryFuncC)GET_OPTIMIZED(cv::hal::cmp16s),
(BinaryFuncC)GET_OPTIMIZED(cv::hal::cmp32s),
(BinaryFuncC)GET_OPTIMIZED(cv::hal::cmp32f), (BinaryFuncC)cv::hal::cmp64f,
(BinaryFuncC)GET_OPTIMIZED(cv::hal::cmp32f),
(BinaryFuncC)cv::hal::cmp64f,
(BinaryFuncC)cv::hal::cmp16f,
(BinaryFuncC)cv::hal::cmp16bf,
0,
(BinaryFuncC)cv::hal::cmp64u,
(BinaryFuncC)cv::hal::cmp64s,
(BinaryFuncC)cv::hal::cmp32u,
0
};
@@ -1069,13 +1142,20 @@ static BinaryFuncC getCmpFunc(int depth)
static double getMinVal(int depth)
{
static const double tab[] = {0, -128, 0, -32768, INT_MIN, -FLT_MAX, -DBL_MAX, 0};
static const double tab[CV_DEPTH_MAX] =
{
0, -128, 0, -32768, INT_MIN, -FLT_MAX, -DBL_MAX,
-65504, -FLT_MAX, 0, 0, (double)INT64_MIN, 0
};
return tab[depth];
}
static double getMaxVal(int depth)
{
static const double tab[] = {255, 127, 65535, 32767, INT_MAX, FLT_MAX, DBL_MAX, 0};
static const double tab[CV_DEPTH_MAX] = {
255, 127, 65535, 32767, INT_MAX, FLT_MAX, DBL_MAX,
65504, FLT_MAX, 255, (double)UINT64_MAX, (double)INT64_MAX, (double)UINT32_MAX, 0
};
return tab[depth];
}
@@ -1220,10 +1300,7 @@ void cv::compare(InputArray _src1, InputArray _src2, OutputArray _dst, int op)
_InputArray::KindFlag kind1 = _src1.kind(), kind2 = _src2.kind();
Mat src1 = _src1.getMat(), src2 = _src2.getMat();
int depth1 = src1.depth(), depth2 = src2.depth();
if (depth1 == CV_16F || depth2 == CV_16F)
CV_Error(Error::StsNotImplemented, "Unsupported depth value CV_16F");
if( kind1 == kind2 && src1.dims <= 2 && src2.dims <= 2 && src1.size() == src2.size() && src1.type() == src2.type() )
{
@@ -1270,7 +1347,8 @@ void cv::compare(InputArray _src1, InputArray _src2, OutputArray _dst, int op)
AutoBuffer<uchar> _buf(blocksize*esz);
uchar *buf = _buf.data();
if( depth1 > CV_32S )
if( ((depth1 == CV_16F) | (depth1 == CV_16BF) |
(depth1 == CV_32F) | (depth1 == CV_64F)) != 0 )
convertAndUnrollScalar( src2, depth1, buf, blocksize );
else
{
@@ -1290,20 +1368,20 @@ void cv::compare(InputArray _src1, InputArray _src2, OutputArray _dst, int op)
return;
}
int ival = cvRound(fval);
double ival = round(fval);
if( fval != ival )
{
if( op == CMP_LT || op == CMP_GE )
ival = cvCeil(fval);
ival = ceil(fval);
else if( op == CMP_LE || op == CMP_GT )
ival = cvFloor(fval);
ival = floor(fval);
else
{
dst = Scalar::all(op == CMP_NE ? 255 : 0);
return;
}
}
convertAndUnrollScalar(Mat(1, 1, CV_32S, &ival), depth1, buf, blocksize);
convertAndUnrollScalar(Mat(1, 1, CV_64F, &ival), depth1, buf, blocksize);
}
for( size_t i = 0; i < it.nplanes; i++, ++it )
@@ -1486,6 +1564,60 @@ struct InRange_SIMD<float>
}
};
template <>
struct InRange_SIMD<float16_t>
{
int operator () (const float16_t * src1, const float16_t * src2, const float16_t * src3,
uchar * dst, int len) const
{
int x = 0;
const int width = (int)VTraits<v_float32>::vlanes()*2;
for (; x <= len - width; x += width)
{
v_float32 values1 = vx_load_expand(src1 + x);
v_float32 low1 = vx_load_expand(src2 + x);
v_float32 high1 = vx_load_expand(src3 + x);
v_float32 values2 = vx_load_expand(src1 + x + VTraits<v_float32>::vlanes());
v_float32 low2 = vx_load_expand(src2 + x + VTraits<v_float32>::vlanes());
v_float32 high2 = vx_load_expand(src3 + x + VTraits<v_float32>::vlanes());
v_pack_store(dst + x, v_pack(v_and(v_reinterpret_as_u32(v_ge(values1, low1)), v_reinterpret_as_u32(v_ge(high1, values1))),
v_and(v_reinterpret_as_u32(v_ge(values2, low2)), v_reinterpret_as_u32(v_ge(high2, values2)))));
}
vx_cleanup();
return x;
}
};
template <>
struct InRange_SIMD<bfloat16_t>
{
int operator () (const bfloat16_t * src1, const bfloat16_t * src2, const bfloat16_t * src3,
uchar * dst, int len) const
{
int x = 0;
const int width = (int)VTraits<v_float32>::vlanes()*2;
for (; x <= len - width; x += width)
{
v_float32 values1 = vx_load_expand(src1 + x);
v_float32 low1 = vx_load_expand(src2 + x);
v_float32 high1 = vx_load_expand(src3 + x);
v_float32 values2 = vx_load_expand(src1 + x + VTraits<v_float32>::vlanes());
v_float32 low2 = vx_load_expand(src2 + x + VTraits<v_float32>::vlanes());
v_float32 high2 = vx_load_expand(src3 + x + VTraits<v_float32>::vlanes());
v_pack_store(dst + x, v_pack(v_and(v_reinterpret_as_u32(v_ge(values1, low1)), v_reinterpret_as_u32(v_ge(high1, values1))),
v_and(v_reinterpret_as_u32(v_ge(values2, low2)), v_reinterpret_as_u32(v_ge(high2, values2)))));
}
vx_cleanup();
return x;
}
};
#endif
template <typename T>
@@ -1544,12 +1676,30 @@ static void inRange16s(const short* src1, size_t step1, const short* src2, size_
inRange_(src1, step1, src2, step2, src3, step3, dst, step, size);
}
static void inRange32u(const unsigned* src1, size_t step1, const unsigned* src2, size_t step2,
const unsigned* src3, size_t step3, uchar* dst, size_t step, Size size)
{
inRange_(src1, step1, src2, step2, src3, step3, dst, step, size);
}
static void inRange32s(const int* src1, size_t step1, const int* src2, size_t step2,
const int* src3, size_t step3, uchar* dst, size_t step, Size size)
{
inRange_(src1, step1, src2, step2, src3, step3, dst, step, size);
}
static void inRange64u(const uint64* src1, size_t step1, const uint64* src2, size_t step2,
const uint64* src3, size_t step3, uchar* dst, size_t step, Size size)
{
inRange_(src1, step1, src2, step2, src3, step3, dst, step, size);
}
static void inRange64s(const int64* src1, size_t step1, const int64* src2, size_t step2,
const int64* src3, size_t step3, uchar* dst, size_t step, Size size)
{
inRange_(src1, step1, src2, step2, src3, step3, dst, step, size);
}
static void inRange32f(const float* src1, size_t step1, const float* src2, size_t step2,
const float* src3, size_t step3, uchar* dst, size_t step, Size size)
{
@@ -1562,6 +1712,18 @@ static void inRange64f(const double* src1, size_t step1, const double* src2, siz
inRange_(src1, step1, src2, step2, src3, step3, dst, step, size);
}
static void inRange16f(const float16_t* src1, size_t step1, const float16_t* src2, size_t step2,
const float16_t* src3, size_t step3, uchar* dst, size_t step, Size size)
{
inRange_(src1, step1, src2, step2, src3, step3, dst, step, size);
}
static void inRange16bf(const bfloat16_t* src1, size_t step1, const bfloat16_t* src2, size_t step2,
const bfloat16_t* src3, size_t step3, uchar* dst, size_t step, Size size)
{
inRange_(src1, step1, src2, step2, src3, step3, dst, step, size);
}
static void inRangeReduce(const uchar* src, uchar* dst, size_t len, int cn)
{
int k = cn % 4 ? cn % 4 : 4;
@@ -1593,9 +1755,20 @@ static InRangeFunc getInRangeFunc(int depth)
{
static InRangeFunc inRangeTab[CV_DEPTH_MAX] =
{
(InRangeFunc)GET_OPTIMIZED(inRange8u), (InRangeFunc)GET_OPTIMIZED(inRange8s), (InRangeFunc)GET_OPTIMIZED(inRange16u),
(InRangeFunc)GET_OPTIMIZED(inRange16s), (InRangeFunc)GET_OPTIMIZED(inRange32s), (InRangeFunc)GET_OPTIMIZED(inRange32f),
(InRangeFunc)inRange64f, 0
(InRangeFunc)GET_OPTIMIZED(inRange8u),
(InRangeFunc)GET_OPTIMIZED(inRange8s),
(InRangeFunc)GET_OPTIMIZED(inRange16u),
(InRangeFunc)GET_OPTIMIZED(inRange16s),
(InRangeFunc)GET_OPTIMIZED(inRange32s),
(InRangeFunc)GET_OPTIMIZED(inRange32f),
(InRangeFunc)inRange64f,
(InRangeFunc)inRange16f,
(InRangeFunc)inRange16bf,
0,
(InRangeFunc)inRange64u,
(InRangeFunc)inRange64s,
(InRangeFunc)inRange32u,
0,
};
return inRangeTab[depth];
modules/core/src/arithm.simd.hpp
+844 -1823
View File
File diff suppressed because it is too large Load Diff
modules/core/src/channels.cpp
+3 -1
View File
@@ -83,7 +83,9 @@ static MixChannelsFunc getMixchFunc(int depth)
{
mixChannels8u, mixChannels8u, mixChannels16u,
mixChannels16u, mixChannels32s, mixChannels32s,
mixChannels64s, 0
mixChannels64s, mixChannels16u, mixChannels16u,
mixChannels8u, mixChannels64s, mixChannels64s,
mixChannels32s, 0
};
return mixchTab[depth];
modules/core/src/count_non_zero.dispatch.cpp
+18 -8
View File
@@ -161,13 +161,11 @@ void findNonZero(InputArray _src, OutputArray _idx)
AutoBuffer<int> buf_(cols + 1);
int* buf = buf_.data();
CV_Assert( depth < CV_16F );
for( int i = 0; i < rows; i++ )
{
int j, k = 0;
const uchar* ptr8 = src.ptr(i);
if( depth == CV_8U || depth == CV_8S )
if( depth == CV_8U || depth == CV_8S || depth == CV_Bool )
{
for( j = 0; j < cols; j++ )
if( ptr8[j] != 0 ) buf[k++] = j;
@@ -178,23 +176,35 @@ void findNonZero(InputArray _src, OutputArray _idx)
for( j = 0; j < cols; j++ )
if( ptr16[j] != 0 ) buf[k++] = j;
}
else if( depth == CV_32S )
else if( depth == CV_32S || depth == CV_32U )
{
const int* ptr32s = (const int*)ptr8;
for( j = 0; j < cols; j++ )
if( ptr32s[j] != 0 ) buf[k++] = j;
}
else if( depth == CV_64S || depth == CV_64U )
{
const int64* ptr64s = (const int64*)ptr8;
for( j = 0; j < cols; j++ )
if( ptr64s[j] != 0 ) buf[k++] = j;
}
else if( depth == CV_32F )
{
const float* ptr32f = (const float*)ptr8;
const int* ptr32s = (const int*)ptr8;
for( j = 0; j < cols; j++ )
if( ptr32f[j] != 0 ) buf[k++] = j;
if( (ptr32s[j]<<1) != 0 ) buf[k++] = j;
}
else if( depth == CV_16F || depth == CV_16BF )
{
const ushort* ptr16 = (const ushort*)ptr8;
for( j = 0; j < cols; j++ )
if( (ptr16[j]<<1) != 0 ) buf[k++] = j;
}
else
{
const double* ptr64f = (const double*)ptr8;
const int64* ptr64s = (const int64*)ptr8;
for( j = 0; j < cols; j++ )
if( ptr64f[j] != 0 ) buf[k++] = j;
if( (ptr64s[j]<<1) != 0 ) buf[k++] = j;
}
if( k > 0 )
modules/core/src/count_non_zero.simd.hpp
+112 -169
View File
@@ -8,200 +8,143 @@ namespace cv {
typedef int (*CountNonZeroFunc)(const uchar*, int);
CV_CPU_OPTIMIZATION_NAMESPACE_BEGIN
CountNonZeroFunc getCountNonZeroTab(int depth);
#ifndef CV_CPU_OPTIMIZATION_DECLARATIONS_ONLY
template<typename T>
static int countNonZero_(const T* src, int len )
{
int i=0, nz = 0;
#if CV_ENABLE_UNROLLED
for(; i <= len - 4; i += 4 )
nz += (src[i] != 0) + (src[i+1] != 0) + (src[i+2] != 0) + (src[i+3] != 0);
#endif
for( ; i < len; i++ )
int nz = 0;
for( int i = 0; i < len; i++ )
nz += src[i] != 0;
return nz;
}
static int countNonZero8u( const uchar* src, int len )
{
int i=0, nz = 0;
#undef SIMD_ONLY
#if (CV_SIMD || CV_SIMD_SCALABLE)
int len0 = len & -VTraits<v_uint8>::vlanes();
v_uint8 v_zero = vx_setzero_u8();
v_uint8 v_one = vx_setall_u8(1);
v_uint32 v_sum32 = vx_setzero_u32();
while (i < len0)
{
v_uint16 v_sum16 = vx_setzero_u16();
int j = i;
while (j < std::min(len0, i + 65280 * VTraits<v_uint16>::vlanes()))
{
v_uint8 v_sum8 = vx_setzero_u8();
int k = j;
for (; k < std::min(len0, j + 255 * VTraits<v_uint8>::vlanes()); k += VTraits<v_uint8>::vlanes())
v_sum8 = v_add(v_sum8, v_and(v_one, v_eq(vx_load(src + k), v_zero)));
v_uint16 part1, part2;
v_expand(v_sum8, part1, part2);
v_sum16 = v_add(v_sum16, v_add(part1, part2));
j = k;
}
v_uint32 part1, part2;
v_expand(v_sum16, part1, part2);
v_sum32 = v_add(v_sum32, v_add(part1, part2));
i = j;
}
nz = i - v_reduce_sum(v_sum32);
v_cleanup();
#define SIMD_ONLY(expr) expr
#else
#define SIMD_ONLY(expr)
#endif
for( ; i < len; i++ )
nz += src[i] != 0;
return nz;
#undef DEFINE_NONZERO_FUNC
#define DEFINE_NONZERO_FUNC(funcname, suffix, ssuffix, T, VT, ST, cmp_op, add_op, update_sum, scalar_cmp_op) \
static int funcname( const T* src, int len ) \
{ \
int i = 0, nz = 0; \
SIMD_ONLY( \
const int vlanes = VTraits<VT>::vlanes(); \
VT v_zero = vx_setzero_##suffix(); \
VT v_1 = vx_setall_##suffix(1); \
VT v_8 = vx_setall_##suffix(8); \
ST v_sum0 = vx_setzero_##ssuffix(); \
ST v_sum1 = v_sum0; \
for (i = 0; i <= len - vlanes*8; i += vlanes*8) \
{ \
VT x0 = vx_load(src + i); \
VT x1 = vx_load(src + i + vlanes); \
VT x2 = vx_load(src + i + vlanes*2); \
VT x3 = vx_load(src + i + vlanes*3); \
VT x4 = vx_load(src + i + vlanes*4); \
VT x5 = vx_load(src + i + vlanes*5); \
VT x6 = vx_load(src + i + vlanes*6); \
VT x7 = vx_load(src + i + vlanes*7); \
x0 = cmp_op(x0, v_zero); \
x1 = cmp_op(x1, v_zero); \
x2 = cmp_op(x2, v_zero); \
x3 = cmp_op(x3, v_zero); \
x4 = cmp_op(x4, v_zero); \
x5 = cmp_op(x5, v_zero); \
x6 = cmp_op(x6, v_zero); \
x7 = cmp_op(x7, v_zero); \
x0 = add_op(x0, x1); \
x2 = add_op(x2, x3); \
x4 = add_op(x4, x5); \
x6 = add_op(x6, x7); \
x0 = add_op(x0, x2); \
x4 = add_op(x4, x6); \
x0 = add_op(add_op(x0, x4), v_8); \
update_sum(v_sum0, v_sum1, x0); \
} \
for (; i <= len - vlanes; i += vlanes) \
{ \
VT x0 = vx_load(src + i); \
x0 = add_op(cmp_op(x0, v_zero), v_1); \
update_sum(v_sum0, v_sum1, x0); \
} \
nz += (int)v_reduce_sum(v_add(v_sum0, v_sum1)); \
v_cleanup();) \
for( ; i < len; i++ ) \
{ \
nz += scalar_cmp_op(src[i]); \
} \
return nz; \
}
static int countNonZero16u( const ushort* src, int len )
{
int i = 0, nz = 0;
#if (CV_SIMD || CV_SIMD_SCALABLE)
int len0 = len & -VTraits<v_int8>::vlanes();
v_uint16 v_zero = vx_setzero_u16();
v_int8 v_one = vx_setall_s8(1);
#undef CHECK_NZ_INT
#define CHECK_NZ_INT(x) ((x) != 0)
#undef CHECK_NZ_FP
#define CHECK_NZ_FP(x) ((x)*2 != 0)
#undef VEC_CMP_EQ_Z_FP16
#define VEC_CMP_EQ_Z_FP16(x, z) v_eq(v_add_wrap(x, x), z)
#undef VEC_CMP_EQ_Z_FP
#define VEC_CMP_EQ_Z_FP(x, z) v_eq(v_add(x, x), z)
v_int32 v_sum32 = vx_setzero_s32();
while (i < len0)
{
v_int16 v_sum16 = vx_setzero_s16();
int j = i;
while (j < std::min(len0, i + 32766 * VTraits<v_int16>::vlanes()))
{
v_int8 v_sum8 = vx_setzero_s8();
int k = j;
for (; k < std::min(len0, j + 127 * VTraits<v_int8>::vlanes()); k += VTraits<v_int8>::vlanes())
v_sum8 = v_add(v_sum8, v_and(v_one, v_pack(v_reinterpret_as_s16(v_eq(vx_load(src + k), v_zero)), v_reinterpret_as_s16(v_eq(vx_load(src + k + VTraits<v_uint16>::vlanes()), v_zero)))));
v_int16 part1, part2;
v_expand(v_sum8, part1, part2);
v_sum16 = v_add(v_sum16, v_add(part1, part2));
j = k;
}
v_int32 part1, part2;
v_expand(v_sum16, part1, part2);
v_sum32 = v_add(v_sum32, v_add(part1, part2));
i = j;
}
nz = i - v_reduce_sum(v_sum32);
v_cleanup();
#endif
return nz + countNonZero_(src + i, len - i);
#undef UPDATE_SUM_U8
#define UPDATE_SUM_U8(v_sum0, v_sum1, x0) \
v_uint16 w0 = v_expand_low(x0); \
v_uint16 w1 = v_expand_high(x0); \
v_sum0 = v_add(v_sum0, v_expand_low(w0)); \
v_sum1 = v_add(v_sum1, v_expand_high(w0)); \
v_sum0 = v_add(v_sum0, v_expand_low(w1)); \
v_sum1 = v_add(v_sum1, v_expand_high(w1))
#undef UPDATE_SUM_U16
#define UPDATE_SUM_U16(v_sum0, v_sum1, x0) \
v_sum0 = v_add(v_sum0, v_expand_low(x0)); \
v_sum1 = v_add(v_sum1, v_expand_high(x0))
#undef UPDATE_SUM_S32
#define UPDATE_SUM_S32(v_sum0, v_sum1, x0) \
v_sum0 = v_add(v_sum0, x0)
DEFINE_NONZERO_FUNC(countNonZero8u, u8, u32, uchar, v_uint8, v_uint32, v_eq, v_add_wrap, UPDATE_SUM_U8, CHECK_NZ_INT)
DEFINE_NONZERO_FUNC(countNonZero16u, u16, u32, ushort, v_uint16, v_uint32, v_eq, v_add_wrap, UPDATE_SUM_U16, CHECK_NZ_INT)
DEFINE_NONZERO_FUNC(countNonZero32s, s32, s32, int, v_int32, v_int32, v_eq, v_add, UPDATE_SUM_S32, CHECK_NZ_INT)
DEFINE_NONZERO_FUNC(countNonZero32f, s32, s32, int, v_int32, v_int32, VEC_CMP_EQ_Z_FP, v_add, UPDATE_SUM_S32, CHECK_NZ_FP)
DEFINE_NONZERO_FUNC(countNonZero16f, u16, u32, ushort, v_uint16, v_uint32, VEC_CMP_EQ_Z_FP16, v_add_wrap, UPDATE_SUM_U16, CHECK_NZ_FP)
#undef DEFINE_NONZERO_FUNC_NOSIMD
#define DEFINE_NONZERO_FUNC_NOSIMD(funcname, T) \
static int funcname(const T* src, int len) \
{ \
return countNonZero_(src, len); \
}
static int countNonZero32s( const int* src, int len )
{
int i = 0, nz = 0;
#if (CV_SIMD || CV_SIMD_SCALABLE)
int len0 = len & -VTraits<v_int8>::vlanes();
v_int32 v_zero = vx_setzero_s32();
v_int8 v_one = vx_setall_s8(1);
v_int32 v_sum32 = vx_setzero_s32();
while (i < len0)
{
v_int16 v_sum16 = vx_setzero_s16();
int j = i;
while (j < std::min(len0, i + 32766 * VTraits<v_int16>::vlanes()))
{
v_int8 v_sum8 = vx_setzero_s8();
int k = j;
for (; k < std::min(len0, j + 127 * VTraits<v_int8>::vlanes()); k += VTraits<v_int8>::vlanes())
v_sum8 = v_add(v_sum8, v_and(v_one, v_pack(v_pack(v_eq(vx_load(src + k), v_zero), v_eq(vx_load(src + k + VTraits<v_int32>::vlanes()), v_zero)), v_pack(v_eq(vx_load(src + k + 2 * VTraits<v_int32>::vlanes()), v_zero), v_eq(vx_load(src + k + 3 * VTraits<v_int32>::vlanes()), v_zero)))));
v_int16 part1, part2;
v_expand(v_sum8, part1, part2);
v_sum16 = v_add(v_sum16, v_add(part1, part2));
j = k;
}
v_int32 part1, part2;
v_expand(v_sum16, part1, part2);
v_sum32 = v_add(v_sum32, v_add(part1, part2));
i = j;
}
nz = i - v_reduce_sum(v_sum32);
v_cleanup();
#endif
return nz + countNonZero_(src + i, len - i);
}
static int countNonZero32f( const float* src, int len )
{
int i = 0, nz = 0;
#if (CV_SIMD || CV_SIMD_SCALABLE)
int len0 = len & -VTraits<v_int8>::vlanes();
v_float32 v_zero = vx_setzero_f32();
v_int8 v_one = vx_setall_s8(1);
v_int32 v_sum32 = vx_setzero_s32();
while (i < len0)
{
v_int16 v_sum16 = vx_setzero_s16();
int j = i;
while (j < std::min(len0, i + 32766 * VTraits<v_int16>::vlanes()))
{
v_int8 v_sum8 = vx_setzero_s8();
int k = j;
for (; k < std::min(len0, j + 127 * VTraits<v_int8>::vlanes()); k += VTraits<v_int8>::vlanes())
v_sum8 = v_add(v_sum8, v_and(v_one, v_pack(v_pack(v_reinterpret_as_s32(v_eq(vx_load(src + k), v_zero)), v_reinterpret_as_s32(v_eq(vx_load(src + k + VTraits<v_float32>::vlanes()), v_zero))), v_pack(v_reinterpret_as_s32(v_eq(vx_load(src + k + 2 * VTraits<v_float32>::vlanes()), v_zero)), v_reinterpret_as_s32(v_eq(vx_load(src + k + 3 * VTraits<v_float32>::vlanes()), v_zero))))));
v_int16 part1, part2;
v_expand(v_sum8, part1, part2);
v_sum16 = v_add(v_sum16, v_add(part1, part2));
j = k;
}
v_int32 part1, part2;
v_expand(v_sum16, part1, part2);
v_sum32 = v_add(v_sum32, v_add(part1, part2));
i = j;
}
nz = i - v_reduce_sum(v_sum32);
v_cleanup();
#endif
return nz + countNonZero_(src + i, len - i);
}
static int countNonZero64f( const double* src, int len )
{
int nz = 0, i = 0;
#if (CV_SIMD_64F || CV_SIMD_SCALABLE_64F)
v_int64 sum1 = vx_setzero_s64();
v_int64 sum2 = vx_setzero_s64();
v_float64 zero = vx_setzero_f64();
int step = VTraits<v_float64>::vlanes() * 2;
int len0 = len & -step;
for(i = 0; i < len0; i += step )
{
sum1 = v_add(sum1, v_reinterpret_as_s64(v_eq(vx_load(&src[i]), zero)));
sum2 = v_add(sum2, v_reinterpret_as_s64(v_eq(vx_load(&src[i + step / 2]), zero)));
}
// N.B the value is incremented by -1 (0xF...F) for each value
nz = i + (int)v_reduce_sum(v_add(sum1, sum2));
v_cleanup();
#endif
return nz + countNonZero_(src + i, len - i);
}
DEFINE_NONZERO_FUNC_NOSIMD(countNonZero64s, int64)
DEFINE_NONZERO_FUNC_NOSIMD(countNonZero64f, double)
CountNonZeroFunc getCountNonZeroTab(int depth)
{
static CountNonZeroFunc countNonZeroTab[CV_DEPTH_MAX] =
{
(CountNonZeroFunc)GET_OPTIMIZED(countNonZero8u), (CountNonZeroFunc)GET_OPTIMIZED(countNonZero8u),
(CountNonZeroFunc)GET_OPTIMIZED(countNonZero16u), (CountNonZeroFunc)GET_OPTIMIZED(countNonZero16u),
(CountNonZeroFunc)GET_OPTIMIZED(countNonZero32s), (CountNonZeroFunc)GET_OPTIMIZED(countNonZero32f),
(CountNonZeroFunc)GET_OPTIMIZED(countNonZero64f), 0
(CountNonZeroFunc)GET_OPTIMIZED(countNonZero8u),
(CountNonZeroFunc)GET_OPTIMIZED(countNonZero8u),
(CountNonZeroFunc)GET_OPTIMIZED(countNonZero16u),
(CountNonZeroFunc)GET_OPTIMIZED(countNonZero16u),
(CountNonZeroFunc)GET_OPTIMIZED(countNonZero32s),
(CountNonZeroFunc)GET_OPTIMIZED(countNonZero32f),
(CountNonZeroFunc)GET_OPTIMIZED(countNonZero64f),
(CountNonZeroFunc)GET_OPTIMIZED(countNonZero16f),
(CountNonZeroFunc)GET_OPTIMIZED(countNonZero16f), // for bf16 it's the same code as for f16
(CountNonZeroFunc)GET_OPTIMIZED(countNonZero8u),
(CountNonZeroFunc)GET_OPTIMIZED(countNonZero64s),
(CountNonZeroFunc)GET_OPTIMIZED(countNonZero64s),
(CountNonZeroFunc)GET_OPTIMIZED(countNonZero32s),
0
};
return countNonZeroTab[depth];
modules/core/src/hal_replacement.hpp
+101
View File
@@ -84,17 +84,28 @@ inline int hal_ni_add8u(const uchar *src1_data, size_t src1_step, const uchar *s
inline int hal_ni_add8s(const schar *src1_data, size_t src1_step, const schar *src2_data, size_t src2_step, schar *dst_data, size_t dst_step, int width, int height) { return CV_HAL_ERROR_NOT_IMPLEMENTED; }
inline int hal_ni_add16u(const ushort *src1_data, size_t src1_step, const ushort *src2_data, size_t src2_step, ushort *dst_data, size_t dst_step, int width, int height) { return CV_HAL_ERROR_NOT_IMPLEMENTED; }
inline int hal_ni_add16s(const short *src1_data, size_t src1_step, const short *src2_data, size_t src2_step, short *dst_data, size_t dst_step, int width, int height) { return CV_HAL_ERROR_NOT_IMPLEMENTED; }
inline int hal_ni_add32u(const unsigned *src1_data, size_t src1_step, const unsigned *src2_data, size_t src2_step, unsigned *dst_data, size_t dst_step, int width, int height) { return CV_HAL_ERROR_NOT_IMPLEMENTED; }
inline int hal_ni_add32s(const int *src1_data, size_t src1_step, const int *src2_data, size_t src2_step, int *dst_data, size_t dst_step, int width, int height) { return CV_HAL_ERROR_NOT_IMPLEMENTED; }
inline int hal_ni_add32f(const float *src1_data, size_t src1_step, const float *src2_data, size_t src2_step, float *dst_data, size_t dst_step, int width, int height) { return CV_HAL_ERROR_NOT_IMPLEMENTED; }
inline int hal_ni_add64f(const double *src1_data, size_t src1_step, const double *src2_data, size_t src2_step, double *dst_data, size_t dst_step, int width, int height) { return CV_HAL_ERROR_NOT_IMPLEMENTED; }
inline int hal_ni_add64u(const uint64 *src1_data, size_t src1_step, const uint64 *src2_data, size_t src2_step, uint64 *dst_data, size_t dst_step, int width, int height) { return CV_HAL_ERROR_NOT_IMPLEMENTED; }
inline int hal_ni_add64s(const int64 *src1_data, size_t src1_step, const int64 *src2_data, size_t src2_step, int64 *dst_data, size_t dst_step, int width, int height) { return CV_HAL_ERROR_NOT_IMPLEMENTED; }
inline int hal_ni_add16f(const cv_hal_f16 *src1_data, size_t src1_step, const cv_hal_f16 *src2_data, size_t src2_step, cv_hal_f16 *dst_data, size_t dst_step, int width, int height) { return CV_HAL_ERROR_NOT_IMPLEMENTED; }
inline int hal_ni_add16bf(const cv_hal_bf16 *src1_data, size_t src1_step, const cv_hal_bf16 *src2_data, size_t src2_step, cv_hal_bf16 *dst_data, size_t dst_step, int width, int height) { return CV_HAL_ERROR_NOT_IMPLEMENTED; }
inline int hal_ni_sub8u(const uchar *src1_data, size_t src1_step, const uchar *src2_data, size_t src2_step, uchar *dst_data, size_t dst_step, int width, int height) { return CV_HAL_ERROR_NOT_IMPLEMENTED; }
inline int hal_ni_sub8s(const schar *src1_data, size_t src1_step, const schar *src2_data, size_t src2_step, schar *dst_data, size_t dst_step, int width, int height) { return CV_HAL_ERROR_NOT_IMPLEMENTED; }
inline int hal_ni_sub16u(const ushort *src1_data, size_t src1_step, const ushort *src2_data, size_t src2_step, ushort *dst_data, size_t dst_step, int width, int height) { return CV_HAL_ERROR_NOT_IMPLEMENTED; }
inline int hal_ni_sub16s(const short *src1_data, size_t src1_step, const short *src2_data, size_t src2_step, short *dst_data, size_t dst_step, int width, int height) { return CV_HAL_ERROR_NOT_IMPLEMENTED; }
inline int hal_ni_sub32u(const unsigned *src1_data, size_t src1_step, const unsigned *src2_data, size_t src2_step, unsigned *dst_data, size_t dst_step, int width, int height) { return CV_HAL_ERROR_NOT_IMPLEMENTED; }
inline int hal_ni_sub32s(const int *src1_data, size_t src1_step, const int *src2_data, size_t src2_step, int *dst_data, size_t dst_step, int width, int height) { return CV_HAL_ERROR_NOT_IMPLEMENTED; }
inline int hal_ni_sub32f(const float *src1_data, size_t src1_step, const float *src2_data, size_t src2_step, float *dst_data, size_t dst_step, int width, int height) { return CV_HAL_ERROR_NOT_IMPLEMENTED; }
inline int hal_ni_sub64f(const double *src1_data, size_t src1_step, const double *src2_data, size_t src2_step, double *dst_data, size_t dst_step, int width, int height) { return CV_HAL_ERROR_NOT_IMPLEMENTED; }
inline int hal_ni_sub64u(const uint64 *src1_data, size_t src1_step, const uint64 *src2_data, size_t src2_step, uint64 *dst_data, size_t dst_step, int width, int height) { return CV_HAL_ERROR_NOT_IMPLEMENTED; }
inline int hal_ni_sub64s(const int64 *src1_data, size_t src1_step, const int64 *src2_data, size_t src2_step, int64 *dst_data, size_t dst_step, int width, int height) { return CV_HAL_ERROR_NOT_IMPLEMENTED; }
inline int hal_ni_sub16f(const cv_hal_f16 *src1_data, size_t src1_step, const cv_hal_f16 *src2_data, size_t src2_step, cv_hal_f16 *dst_data, size_t dst_step, int width, int height) { return CV_HAL_ERROR_NOT_IMPLEMENTED; }
inline int hal_ni_sub16bf(const cv_hal_bf16 *src1_data, size_t src1_step, const cv_hal_bf16 *src2_data, size_t src2_step, cv_hal_bf16 *dst_data, size_t dst_step, int width, int height) { return CV_HAL_ERROR_NOT_IMPLEMENTED; }
//! @}
/**
@@ -115,17 +126,27 @@ inline int hal_ni_max8u(const uchar *src1_data, size_t src1_step, const uchar *s
inline int hal_ni_max8s(const schar *src1_data, size_t src1_step, const schar *src2_data, size_t src2_step, schar *dst_data, size_t dst_step, int width, int height) { return CV_HAL_ERROR_NOT_IMPLEMENTED; }
inline int hal_ni_max16u(const ushort *src1_data, size_t src1_step, const ushort *src2_data, size_t src2_step, ushort *dst_data, size_t dst_step, int width, int height) { return CV_HAL_ERROR_NOT_IMPLEMENTED; }
inline int hal_ni_max16s(const short *src1_data, size_t src1_step, const short *src2_data, size_t src2_step, short *dst_data, size_t dst_step, int width, int height) { return CV_HAL_ERROR_NOT_IMPLEMENTED; }
inline int hal_ni_max32u(const unsigned *src1_data, size_t src1_step, const unsigned *src2_data, size_t src2_step, unsigned *dst_data, size_t dst_step, int width, int height) { return CV_HAL_ERROR_NOT_IMPLEMENTED; }
inline int hal_ni_max32s(const int *src1_data, size_t src1_step, const int *src2_data, size_t src2_step, int *dst_data, size_t dst_step, int width, int height) { return CV_HAL_ERROR_NOT_IMPLEMENTED; }
inline int hal_ni_max32f(const float *src1_data, size_t src1_step, const float *src2_data, size_t src2_step, float *dst_data, size_t dst_step, int width, int height) { return CV_HAL_ERROR_NOT_IMPLEMENTED; }
inline int hal_ni_max64f(const double *src1_data, size_t src1_step, const double *src2_data, size_t src2_step, double *dst_data, size_t dst_step, int width, int height) { return CV_HAL_ERROR_NOT_IMPLEMENTED; }
inline int hal_ni_max64u(const uint64 *src1_data, size_t src1_step, const uint64 *src2_data, size_t src2_step, uint64 *dst_data, size_t dst_step, int width, int height) { return CV_HAL_ERROR_NOT_IMPLEMENTED; }
inline int hal_ni_max64s(const int64 *src1_data, size_t src1_step, const int64 *src2_data, size_t src2_step, int64 *dst_data, size_t dst_step, int width, int height) { return CV_HAL_ERROR_NOT_IMPLEMENTED; }
inline int hal_ni_max16f(const cv_hal_f16 *src1_data, size_t src1_step, const cv_hal_f16 *src2_data, size_t src2_step, cv_hal_f16 *dst_data, size_t dst_step, int width, int height) { return CV_HAL_ERROR_NOT_IMPLEMENTED; }
inline int hal_ni_max16bf(const cv_hal_bf16 *src1_data, size_t src1_step, const cv_hal_bf16 *src2_data, size_t src2_step, cv_hal_bf16 *dst_data, size_t dst_step, int width, int height) { return CV_HAL_ERROR_NOT_IMPLEMENTED; }
inline int hal_ni_min8u(const uchar *src1_data, size_t src1_step, const uchar *src2_data, size_t src2_step, uchar *dst_data, size_t dst_step, int width, int height) { return CV_HAL_ERROR_NOT_IMPLEMENTED; }
inline int hal_ni_min8s(const schar *src1_data, size_t src1_step, const schar *src2_data, size_t src2_step, schar *dst_data, size_t dst_step, int width, int height) { return CV_HAL_ERROR_NOT_IMPLEMENTED; }
inline int hal_ni_min16u(const ushort *src1_data, size_t src1_step, const ushort *src2_data, size_t src2_step, ushort *dst_data, size_t dst_step, int width, int height) { return CV_HAL_ERROR_NOT_IMPLEMENTED; }
inline int hal_ni_min16s(const short *src1_data, size_t src1_step, const short *src2_data, size_t src2_step, short *dst_data, size_t dst_step, int width, int height) { return CV_HAL_ERROR_NOT_IMPLEMENTED; }
inline int hal_ni_min32u(const unsigned *src1_data, size_t src1_step, const unsigned *src2_data, size_t src2_step, unsigned *dst_data, size_t dst_step, int width, int height) { return CV_HAL_ERROR_NOT_IMPLEMENTED; }
inline int hal_ni_min32s(const int *src1_data, size_t src1_step, const int *src2_data, size_t src2_step, int *dst_data, size_t dst_step, int width, int height) { return CV_HAL_ERROR_NOT_IMPLEMENTED; }
inline int hal_ni_min32f(const float *src1_data, size_t src1_step, const float *src2_data, size_t src2_step, float *dst_data, size_t dst_step, int width, int height) { return CV_HAL_ERROR_NOT_IMPLEMENTED; }
inline int hal_ni_min64f(const double *src1_data, size_t src1_step, const double *src2_data, size_t src2_step, double *dst_data, size_t dst_step, int width, int height) { return CV_HAL_ERROR_NOT_IMPLEMENTED; }
inline int hal_ni_min64u(const uint64 *src1_data, size_t src1_step, const uint64 *src2_data, size_t src2_step, uint64 *dst_data, size_t dst_step, int width, int height) { return CV_HAL_ERROR_NOT_IMPLEMENTED; }
inline int hal_ni_min64s(const int64 *src1_data, size_t src1_step, const int64 *src2_data, size_t src2_step, int64 *dst_data, size_t dst_step, int width, int height) { return CV_HAL_ERROR_NOT_IMPLEMENTED; }
inline int hal_ni_min16f(const cv_hal_f16 *src1_data, size_t src1_step, const cv_hal_f16 *src2_data, size_t src2_step, cv_hal_f16 *dst_data, size_t dst_step, int width, int height) { return CV_HAL_ERROR_NOT_IMPLEMENTED; }
inline int hal_ni_min16bf(const cv_hal_bf16 *src1_data, size_t src1_step, const cv_hal_bf16 *src2_data, size_t src2_step, cv_hal_bf16 *dst_data, size_t dst_step, int width, int height) { return CV_HAL_ERROR_NOT_IMPLEMENTED; }
//! @}
/**
@@ -145,9 +166,14 @@ inline int hal_ni_absdiff8u(const uchar *src1_data, size_t src1_step, const ucha
inline int hal_ni_absdiff8s(const schar *src1_data, size_t src1_step, const schar *src2_data, size_t src2_step, schar *dst_data, size_t dst_step, int width, int height) { return CV_HAL_ERROR_NOT_IMPLEMENTED; }
inline int hal_ni_absdiff16u(const ushort *src1_data, size_t src1_step, const ushort *src2_data, size_t src2_step, ushort *dst_data, size_t dst_step, int width, int height) { return CV_HAL_ERROR_NOT_IMPLEMENTED; }
inline int hal_ni_absdiff16s(const short *src1_data, size_t src1_step, const short *src2_data, size_t src2_step, short *dst_data, size_t dst_step, int width, int height) { return CV_HAL_ERROR_NOT_IMPLEMENTED; }
inline int hal_ni_absdiff32u(const unsigned *src1_data, size_t src1_step, const unsigned *src2_data, size_t src2_step, unsigned *dst_data, size_t dst_step, int width, int height) { return CV_HAL_ERROR_NOT_IMPLEMENTED; }
inline int hal_ni_absdiff32s(const int *src1_data, size_t src1_step, const int *src2_data, size_t src2_step, int *dst_data, size_t dst_step, int width, int height) { return CV_HAL_ERROR_NOT_IMPLEMENTED; }
inline int hal_ni_absdiff32f(const float *src1_data, size_t src1_step, const float *src2_data, size_t src2_step, float *dst_data, size_t dst_step, int width, int height) { return CV_HAL_ERROR_NOT_IMPLEMENTED; }
inline int hal_ni_absdiff64f(const double *src1_data, size_t src1_step, const double *src2_data, size_t src2_step, double *dst_data, size_t dst_step, int width, int height) { return CV_HAL_ERROR_NOT_IMPLEMENTED; }
inline int hal_ni_absdiff64u(const uint64 *src1_data, size_t src1_step, const uint64 *src2_data, size_t src2_step, uint64 *dst_data, size_t dst_step, int width, int height) { return CV_HAL_ERROR_NOT_IMPLEMENTED; }
inline int hal_ni_absdiff64s(const int64 *src1_data, size_t src1_step, const int64 *src2_data, size_t src2_step, int64 *dst_data, size_t dst_step, int width, int height) { return CV_HAL_ERROR_NOT_IMPLEMENTED; }
inline int hal_ni_absdiff16f(const cv_hal_f16 *src1_data, size_t src1_step, const cv_hal_f16 *src2_data, size_t src2_step, cv_hal_f16 *dst_data, size_t dst_step, int width, int height) { return CV_HAL_ERROR_NOT_IMPLEMENTED; }
inline int hal_ni_absdiff16bf(const cv_hal_bf16 *src1_data, size_t src1_step, const cv_hal_bf16 *src2_data, size_t src2_step, cv_hal_bf16 *dst_data, size_t dst_step, int width, int height) { return CV_HAL_ERROR_NOT_IMPLEMENTED; }
//! @}
/**
@@ -177,37 +203,62 @@ inline int hal_ni_not8u(const uchar *src_data, size_t src_step, uchar *dst_data,
#define cv_hal_add8s hal_ni_add8s
#define cv_hal_add16u hal_ni_add16u
#define cv_hal_add16s hal_ni_add16s
#define cv_hal_add32u hal_ni_add32u
#define cv_hal_add32s hal_ni_add32s
#define cv_hal_add64u hal_ni_add64u
#define cv_hal_add64s hal_ni_add64s
#define cv_hal_add32f hal_ni_add32f
#define cv_hal_add64f hal_ni_add64f
#define cv_hal_add16f hal_ni_add16f
#define cv_hal_add16bf hal_ni_add16bf
#define cv_hal_sub8u hal_ni_sub8u
#define cv_hal_sub8s hal_ni_sub8s
#define cv_hal_sub16u hal_ni_sub16u
#define cv_hal_sub16s hal_ni_sub16s
#define cv_hal_sub32u hal_ni_sub32u
#define cv_hal_sub32s hal_ni_sub32s
#define cv_hal_sub64u hal_ni_sub64u
#define cv_hal_sub64s hal_ni_sub64s
#define cv_hal_sub32f hal_ni_sub32f
#define cv_hal_sub64f hal_ni_sub64f
#define cv_hal_sub16f hal_ni_sub16f
#define cv_hal_sub16bf hal_ni_sub16bf
#define cv_hal_max8u hal_ni_max8u
#define cv_hal_max8s hal_ni_max8s
#define cv_hal_max16u hal_ni_max16u
#define cv_hal_max16s hal_ni_max16s
#define cv_hal_max32u hal_ni_max32u
#define cv_hal_max32s hal_ni_max32s
#define cv_hal_max64u hal_ni_max64u
#define cv_hal_max64s hal_ni_max64s
#define cv_hal_max32f hal_ni_max32f
#define cv_hal_max64f hal_ni_max64f
#define cv_hal_max16f hal_ni_max16f
#define cv_hal_max16bf hal_ni_max16bf
#define cv_hal_min8u hal_ni_min8u
#define cv_hal_min8s hal_ni_min8s
#define cv_hal_min16u hal_ni_min16u
#define cv_hal_min16s hal_ni_min16s
#define cv_hal_min32u hal_ni_min32u
#define cv_hal_min32s hal_ni_min32s
#define cv_hal_min64u hal_ni_min64u
#define cv_hal_min64s hal_ni_min64s
#define cv_hal_min32f hal_ni_min32f
#define cv_hal_min64f hal_ni_min64f
#define cv_hal_min16f hal_ni_min16f
#define cv_hal_min16bf hal_ni_min16bf
#define cv_hal_absdiff8u hal_ni_absdiff8u
#define cv_hal_absdiff8s hal_ni_absdiff8s
#define cv_hal_absdiff16u hal_ni_absdiff16u
#define cv_hal_absdiff16s hal_ni_absdiff16s
#define cv_hal_absdiff32u hal_ni_absdiff32u
#define cv_hal_absdiff32s hal_ni_absdiff32s
#define cv_hal_absdiff64u hal_ni_absdiff64u
#define cv_hal_absdiff64s hal_ni_absdiff64s
#define cv_hal_absdiff32f hal_ni_absdiff32f
#define cv_hal_absdiff64f hal_ni_absdiff64f
#define cv_hal_absdiff16f hal_ni_absdiff16f
#define cv_hal_absdiff16bf hal_ni_absdiff16bf
#define cv_hal_and8u hal_ni_and8u
#define cv_hal_or8u hal_ni_or8u
#define cv_hal_xor8u hal_ni_xor8u
@@ -232,9 +283,14 @@ inline int hal_ni_cmp8u(const uchar *src1_data, size_t src1_step, const uchar *s
inline int hal_ni_cmp8s(const schar *src1_data, size_t src1_step, const schar *src2_data, size_t src2_step, uchar *dst_data, size_t dst_step, int width, int height, int operation) { return CV_HAL_ERROR_NOT_IMPLEMENTED; }
inline int hal_ni_cmp16u(const ushort *src1_data, size_t src1_step, const ushort *src2_data, size_t src2_step, uchar *dst_data, size_t dst_step, int width, int height, int operation) { return CV_HAL_ERROR_NOT_IMPLEMENTED; }
inline int hal_ni_cmp16s(const short *src1_data, size_t src1_step, const short *src2_data, size_t src2_step, uchar *dst_data, size_t dst_step, int width, int height, int operation) { return CV_HAL_ERROR_NOT_IMPLEMENTED; }
inline int hal_ni_cmp32u(const unsigned *src1_data, size_t src1_step, const unsigned *src2_data, size_t src2_step, uchar *dst_data, size_t dst_step, int width, int height, int operation) { return CV_HAL_ERROR_NOT_IMPLEMENTED; }
inline int hal_ni_cmp32s(const int *src1_data, size_t src1_step, const int *src2_data, size_t src2_step, uchar *dst_data, size_t dst_step, int width, int height, int operation) { return CV_HAL_ERROR_NOT_IMPLEMENTED; }
inline int hal_ni_cmp32f(const float *src1_data, size_t src1_step, const float *src2_data, size_t src2_step, uchar *dst_data, size_t dst_step, int width, int height, int operation) { return CV_HAL_ERROR_NOT_IMPLEMENTED; }
inline int hal_ni_cmp64f(const double *src1_data, size_t src1_step, const double *src2_data, size_t src2_step, uchar *dst_data, size_t dst_step, int width, int height, int operation) { return CV_HAL_ERROR_NOT_IMPLEMENTED; }
inline int hal_ni_cmp64u(const uint64 *src1_data, size_t src1_step, const uint64 *src2_data, size_t src2_step, uchar *dst_data, size_t dst_step, int width, int height, int operation) { return CV_HAL_ERROR_NOT_IMPLEMENTED; }
inline int hal_ni_cmp64s(const int64 *src1_data, size_t src1_step, const int64 *src2_data, size_t src2_step, uchar *dst_data, size_t dst_step, int width, int height, int operation) { return CV_HAL_ERROR_NOT_IMPLEMENTED; }
inline int hal_ni_cmp16f(const cv_hal_f16 *src1_data, size_t src1_step, const cv_hal_f16 *src2_data, size_t src2_step, uchar *dst_data, size_t dst_step, int width, int height, int operation) { return CV_HAL_ERROR_NOT_IMPLEMENTED; }
inline int hal_ni_cmp16bf(const cv_hal_bf16 *src1_data, size_t src1_step, const cv_hal_bf16 *src2_data, size_t src2_step, uchar *dst_data, size_t dst_step, int width, int height, int operation) { return CV_HAL_ERROR_NOT_IMPLEMENTED; }
//! @}
//! @cond IGNORED
@@ -242,9 +298,14 @@ inline int hal_ni_cmp64f(const double *src1_data, size_t src1_step, const double
#define cv_hal_cmp8s hal_ni_cmp8s
#define cv_hal_cmp16u hal_ni_cmp16u
#define cv_hal_cmp16s hal_ni_cmp16s
#define cv_hal_cmp32u hal_ni_cmp32u
#define cv_hal_cmp32s hal_ni_cmp32s
#define cv_hal_cmp64u hal_ni_cmp64u
#define cv_hal_cmp64s hal_ni_cmp64s
#define cv_hal_cmp32f hal_ni_cmp32f
#define cv_hal_cmp64f hal_ni_cmp64f
#define cv_hal_cmp16f hal_ni_cmp16f
#define cv_hal_cmp16bf hal_ni_cmp16bf
//! @endcond
/**
@@ -265,9 +326,14 @@ inline int hal_ni_mul8u(const uchar *src1_data, size_t src1_step, const uchar *s
inline int hal_ni_mul8s(const schar *src1_data, size_t src1_step, const schar *src2_data, size_t src2_step, schar *dst_data, size_t dst_step, int width, int height, double scale) { return CV_HAL_ERROR_NOT_IMPLEMENTED; }
inline int hal_ni_mul16u(const ushort *src1_data, size_t src1_step, const ushort *src2_data, size_t src2_step, ushort *dst_data, size_t dst_step, int width, int height, double scale) { return CV_HAL_ERROR_NOT_IMPLEMENTED; }
inline int hal_ni_mul16s(const short *src1_data, size_t src1_step, const short *src2_data, size_t src2_step, short *dst_data, size_t dst_step, int width, int height, double scale) { return CV_HAL_ERROR_NOT_IMPLEMENTED; }
inline int hal_ni_mul32u(const unsigned *src1_data, size_t src1_step, const unsigned *src2_data, size_t src2_step, unsigned *dst_data, size_t dst_step, int width, int height, double scale) { return CV_HAL_ERROR_NOT_IMPLEMENTED; }
inline int hal_ni_mul32s(const int *src1_data, size_t src1_step, const int *src2_data, size_t src2_step, int *dst_data, size_t dst_step, int width, int height, double scale) { return CV_HAL_ERROR_NOT_IMPLEMENTED; }
inline int hal_ni_mul32f(const float *src1_data, size_t src1_step, const float *src2_data, size_t src2_step, float *dst_data, size_t dst_step, int width, int height, double scale) { return CV_HAL_ERROR_NOT_IMPLEMENTED; }
inline int hal_ni_mul64f(const double *src1_data, size_t src1_step, const double *src2_data, size_t src2_step, double *dst_data, size_t dst_step, int width, int height, double scale) { return CV_HAL_ERROR_NOT_IMPLEMENTED; }
inline int hal_ni_mul64u(const uint64 *src1_data, size_t src1_step, const uint64 *src2_data, size_t src2_step, uint64 *dst_data, size_t dst_step, int width, int height, double scale) { return CV_HAL_ERROR_NOT_IMPLEMENTED; }
inline int hal_ni_mul64s(const int64 *src1_data, size_t src1_step, const int64 *src2_data, size_t src2_step, int64 *dst_data, size_t dst_step, int width, int height, double scale) { return CV_HAL_ERROR_NOT_IMPLEMENTED; }
inline int hal_ni_mul16f(const cv_hal_f16 *src1_data, size_t src1_step, const cv_hal_f16 *src2_data, size_t src2_step, cv_hal_f16 *dst_data, size_t dst_step, int width, int height, double scale) { return CV_HAL_ERROR_NOT_IMPLEMENTED; }
inline int hal_ni_mul16bf(const cv_hal_bf16 *src1_data, size_t src1_step, const cv_hal_bf16 *src2_data, size_t src2_step, cv_hal_bf16 *dst_data, size_t dst_step, int width, int height, double scale) { return CV_HAL_ERROR_NOT_IMPLEMENTED; }
//! @}
/**
@@ -288,9 +354,14 @@ inline int hal_ni_div8u(const uchar *src1_data, size_t src1_step, const uchar *s
inline int hal_ni_div8s(const schar *src1_data, size_t src1_step, const schar *src2_data, size_t src2_step, schar *dst_data, size_t dst_step, int width, int height, double scale) { return CV_HAL_ERROR_NOT_IMPLEMENTED; }
inline int hal_ni_div16u(const ushort *src1_data, size_t src1_step, const ushort *src2_data, size_t src2_step, ushort *dst_data, size_t dst_step, int width, int height, double scale) { return CV_HAL_ERROR_NOT_IMPLEMENTED; }
inline int hal_ni_div16s(const short *src1_data, size_t src1_step, const short *src2_data, size_t src2_step, short *dst_data, size_t dst_step, int width, int height, double scale) { return CV_HAL_ERROR_NOT_IMPLEMENTED; }
inline int hal_ni_div32u(const unsigned *src1_data, size_t src1_step, const unsigned *src2_data, size_t src2_step, unsigned *dst_data, size_t dst_step, int width, int height, double scale) { return CV_HAL_ERROR_NOT_IMPLEMENTED; }
inline int hal_ni_div32s(const int *src1_data, size_t src1_step, const int *src2_data, size_t src2_step, int *dst_data, size_t dst_step, int width, int height, double scale) { return CV_HAL_ERROR_NOT_IMPLEMENTED; }
inline int hal_ni_div32f(const float *src1_data, size_t src1_step, const float *src2_data, size_t src2_step, float *dst_data, size_t dst_step, int width, int height, double scale) { return CV_HAL_ERROR_NOT_IMPLEMENTED; }
inline int hal_ni_div64f(const double *src1_data, size_t src1_step, const double *src2_data, size_t src2_step, double *dst_data, size_t dst_step, int width, int height, double scale) { return CV_HAL_ERROR_NOT_IMPLEMENTED; }
inline int hal_ni_div64u(const uint64 *src1_data, size_t src1_step, const uint64 *src2_data, size_t src2_step, uint64 *dst_data, size_t dst_step, int width, int height, double scale) { return CV_HAL_ERROR_NOT_IMPLEMENTED; }
inline int hal_ni_div64s(const int64 *src1_data, size_t src1_step, const int64 *src2_data, size_t src2_step, int64 *dst_data, size_t dst_step, int width, int height, double scale) { return CV_HAL_ERROR_NOT_IMPLEMENTED; }
inline int hal_ni_div16f(const cv_hal_f16 *src1_data, size_t src1_step, const cv_hal_f16 *src2_data, size_t src2_step, cv_hal_f16 *dst_data, size_t dst_step, int width, int height, double scale) { return CV_HAL_ERROR_NOT_IMPLEMENTED; }
inline int hal_ni_div16bf(const cv_hal_bf16 *src1_data, size_t src1_step, const cv_hal_bf16 *src2_data, size_t src2_step, cv_hal_bf16 *dst_data, size_t dst_step, int width, int height, double scale) { return CV_HAL_ERROR_NOT_IMPLEMENTED; }
//! @}
/**
@@ -309,9 +380,14 @@ inline int hal_ni_recip8u(const uchar *src_data, size_t src_step, uchar *dst_dat
inline int hal_ni_recip8s(const schar *src_data, size_t src_step, schar *dst_data, size_t dst_step, int width, int height, double scale) { return CV_HAL_ERROR_NOT_IMPLEMENTED; }
inline int hal_ni_recip16u(const ushort *src_data, size_t src_step, ushort *dst_data, size_t dst_step, int width, int height, double scale) { return CV_HAL_ERROR_NOT_IMPLEMENTED; }
inline int hal_ni_recip16s(const short *src_data, size_t src_step, short *dst_data, size_t dst_step, int width, int height, double scale) { return CV_HAL_ERROR_NOT_IMPLEMENTED; }
inline int hal_ni_recip32u(const unsigned *src_data, size_t src_step, unsigned *dst_data, size_t dst_step, int width, int height, double scale) { return CV_HAL_ERROR_NOT_IMPLEMENTED; }
inline int hal_ni_recip32s(const int *src_data, size_t src_step, int *dst_data, size_t dst_step, int width, int height, double scale) { return CV_HAL_ERROR_NOT_IMPLEMENTED; }
inline int hal_ni_recip32f(const float *src_data, size_t src_step, float *dst_data, size_t dst_step, int width, int height, double scale) { return CV_HAL_ERROR_NOT_IMPLEMENTED; }
inline int hal_ni_recip64f(const double *src_data, size_t src_step, double *dst_data, size_t dst_step, int width, int height, double scale) { return CV_HAL_ERROR_NOT_IMPLEMENTED; }
inline int hal_ni_recip64u(const uint64 *src_data, size_t src_step, uint64 *dst_data, size_t dst_step, int width, int height, double scale) { return CV_HAL_ERROR_NOT_IMPLEMENTED; }
inline int hal_ni_recip64s(const int64 *src_data, size_t src_step, int64 *dst_data, size_t dst_step, int width, int height, double scale) { return CV_HAL_ERROR_NOT_IMPLEMENTED; }
inline int hal_ni_recip16f(const cv_hal_f16 *src_data, size_t src_step, cv_hal_f16 *dst_data, size_t dst_step, int width, int height, double scale) { return CV_HAL_ERROR_NOT_IMPLEMENTED; }
inline int hal_ni_recip16bf(const cv_hal_bf16 *src_data, size_t src_step, cv_hal_bf16 *dst_data, size_t dst_step, int width, int height, double scale) { return CV_HAL_ERROR_NOT_IMPLEMENTED; }
//! @}
//! @cond IGNORED
@@ -319,23 +395,38 @@ inline int hal_ni_recip64f(const double *src_data, size_t src_step, double *dst_
#define cv_hal_mul8s hal_ni_mul8s
#define cv_hal_mul16u hal_ni_mul16u
#define cv_hal_mul16s hal_ni_mul16s
#define cv_hal_mul32u hal_ni_mul32u
#define cv_hal_mul32s hal_ni_mul32s
#define cv_hal_mul64u hal_ni_mul64u
#define cv_hal_mul64s hal_ni_mul64s
#define cv_hal_mul32f hal_ni_mul32f
#define cv_hal_mul64f hal_ni_mul64f
#define cv_hal_mul16f hal_ni_mul16f
#define cv_hal_mul16bf hal_ni_mul16bf
#define cv_hal_div8u hal_ni_div8u
#define cv_hal_div8s hal_ni_div8s
#define cv_hal_div16u hal_ni_div16u
#define cv_hal_div16s hal_ni_div16s
#define cv_hal_div32u hal_ni_div32u
#define cv_hal_div32s hal_ni_div32s
#define cv_hal_div64u hal_ni_div64u
#define cv_hal_div64s hal_ni_div64s
#define cv_hal_div32f hal_ni_div32f
#define cv_hal_div64f hal_ni_div64f
#define cv_hal_div16f hal_ni_div16f
#define cv_hal_div16bf hal_ni_div16bf
#define cv_hal_recip8u hal_ni_recip8u
#define cv_hal_recip8s hal_ni_recip8s
#define cv_hal_recip16u hal_ni_recip16u
#define cv_hal_recip16s hal_ni_recip16s
#define cv_hal_recip32u hal_ni_recip32u
#define cv_hal_recip32s hal_ni_recip32s
#define cv_hal_recip64u hal_ni_recip64u
#define cv_hal_recip64s hal_ni_recip64s
#define cv_hal_recip32f hal_ni_recip32f
#define cv_hal_recip64f hal_ni_recip64f
#define cv_hal_recip16f hal_ni_recip16f
#define cv_hal_recip16bf hal_ni_recip16bf
//! @endcond
/**
@@ -356,9 +447,14 @@ inline int hal_ni_addWeighted8u(const uchar *src1_data, size_t src1_step, const
inline int hal_ni_addWeighted8s(const schar *src1_data, size_t src1_step, const schar *src2_data, size_t src2_step, schar *dst_data, size_t dst_step, int width, int height, const double scalars[3]) { return CV_HAL_ERROR_NOT_IMPLEMENTED; }
inline int hal_ni_addWeighted16u(const ushort *src1_data, size_t src1_step, const ushort *src2_data, size_t src2_step, ushort *dst_data, size_t dst_step, int width, int height, const double scalars[3]) { return CV_HAL_ERROR_NOT_IMPLEMENTED; }
inline int hal_ni_addWeighted16s(const short *src1_data, size_t src1_step, const short *src2_data, size_t src2_step, short *dst_data, size_t dst_step, int width, int height, const double scalars[3]) { return CV_HAL_ERROR_NOT_IMPLEMENTED; }
inline int hal_ni_addWeighted32u(const unsigned *src1_data, size_t src1_step, const unsigned *src2_data, size_t src2_step, unsigned *dst_data, size_t dst_step, int width, int height, const double scalars[3]) { return CV_HAL_ERROR_NOT_IMPLEMENTED; }
inline int hal_ni_addWeighted32s(const int *src1_data, size_t src1_step, const int *src2_data, size_t src2_step, int *dst_data, size_t dst_step, int width, int height, const double scalars[3]) { return CV_HAL_ERROR_NOT_IMPLEMENTED; }
inline int hal_ni_addWeighted32f(const float *src1_data, size_t src1_step, const float *src2_data, size_t src2_step, float *dst_data, size_t dst_step, int width, int height, const double scalars[3]) { return CV_HAL_ERROR_NOT_IMPLEMENTED; }
inline int hal_ni_addWeighted64f(const double *src1_data, size_t src1_step, const double *src2_data, size_t src2_step, double *dst_data, size_t dst_step, int width, int height, const double scalars[3]) { return CV_HAL_ERROR_NOT_IMPLEMENTED; }
inline int hal_ni_addWeighted64u(const uint64 *src1_data, size_t src1_step, const uint64 *src2_data, size_t src2_step, uint64 *dst_data, size_t dst_step, int width, int height, const double scalars[3]) { return CV_HAL_ERROR_NOT_IMPLEMENTED; }
inline int hal_ni_addWeighted64s(const int64 *src1_data, size_t src1_step, const int64 *src2_data, size_t src2_step, int64 *dst_data, size_t dst_step, int width, int height, const double scalars[3]) { return CV_HAL_ERROR_NOT_IMPLEMENTED; }
inline int hal_ni_addWeighted16f(const cv_hal_f16 *src1_data, size_t src1_step, const cv_hal_f16 *src2_data, size_t src2_step, cv_hal_f16 *dst_data, size_t dst_step, int width, int height, const double scalars[3]) { return CV_HAL_ERROR_NOT_IMPLEMENTED; }
inline int hal_ni_addWeighted16bf(const cv_hal_bf16 *src1_data, size_t src1_step, const cv_hal_bf16 *src2_data, size_t src2_step, cv_hal_bf16 *dst_data, size_t dst_step, int width, int height, const double scalars[3]) { return CV_HAL_ERROR_NOT_IMPLEMENTED; }
//! @}
//! @cond IGNORED
@@ -366,9 +462,14 @@ inline int hal_ni_addWeighted64f(const double *src1_data, size_t src1_step, cons
#define cv_hal_addWeighted8s hal_ni_addWeighted8s
#define cv_hal_addWeighted16u hal_ni_addWeighted16u
#define cv_hal_addWeighted16s hal_ni_addWeighted16s
#define cv_hal_addWeighted32u hal_ni_addWeighted32u
#define cv_hal_addWeighted32s hal_ni_addWeighted32s
#define cv_hal_addWeighted64u hal_ni_addWeighted64u
#define cv_hal_addWeighted64s hal_ni_addWeighted64s
#define cv_hal_addWeighted32f hal_ni_addWeighted32f
#define cv_hal_addWeighted64f hal_ni_addWeighted64f
#define cv_hal_addWeighted16f hal_ni_addWeighted16f
#define cv_hal_addWeighted16bf hal_ni_addWeighted16bf
//! @endcond
/**
modules/core/src/has_non_zero.dispatch.cpp
+3 -3
View File
@@ -12,10 +12,10 @@
namespace cv {
static HasNonZeroFunc getHasNonZeroTab(int depth)
static HasNonZeroFunc getHasNonZeroFunc(int depth)
{
CV_INSTRUMENT_REGION();
CV_CPU_DISPATCH(getHasNonZeroTab, (depth),
CV_CPU_DISPATCH(getHasNonZeroFunc, (depth),
CV_CPU_DISPATCH_MODES_ALL);
}
@@ -74,7 +74,7 @@ bool hasNonZero(InputArray _src)
Mat src = _src.getMat();
HasNonZeroFunc func = getHasNonZeroTab(src.depth());
HasNonZeroFunc func = getHasNonZeroFunc(src.depth());
CV_Assert( func != 0 );
if (src.dims == 2)//fast path to avoid creating planes of single rows
modules/core/src/has_non_zero.simd.hpp
+87 -293
View File
@@ -8,314 +8,108 @@ namespace cv {
typedef bool (*HasNonZeroFunc)(const uchar*, size_t);
CV_CPU_OPTIMIZATION_NAMESPACE_BEGIN
HasNonZeroFunc getHasNonZeroTab(int depth);
HasNonZeroFunc getHasNonZeroFunc(int depth);
#ifndef CV_CPU_OPTIMIZATION_DECLARATIONS_ONLY
template<typename T>
inline bool hasNonZero_(const T* src, size_t len )
{
bool res = false;
if (len > 0)
{
size_t i=0;
#if CV_ENABLE_UNROLLED
for(; !res && (i+4 <= len); i += 4 )
res |= ((src[i] | src[i+1] | src[i+2] | src[i+3]) != 0);
#endif
for( ; !res && (i < len); i++ )
res |= (src[i] != 0);
}
return res;
}
template<>
inline bool hasNonZero_(const float* src, size_t len )
{
bool res = false;
if (len > 0)
{
size_t i=0;
if (sizeof(float) == sizeof(unsigned int))
{
#if CV_ENABLE_UNROLLED
typedef unsigned int float_as_uint_t;
const float_as_uint_t* src_as_ui = reinterpret_cast<const float_as_uint_t*>(src);
for(; !res && (i+4 <= len); i += 4 )
{
const float_as_uint_t gathered = (src_as_ui[i] | src_as_ui[i+1] | src_as_ui[i+2] | src_as_ui[i+3]);
res |= ((gathered<<1) != 0);//remove what would be the sign bit
}
#endif
}
for( ; !res && (i < len); i++ )
res |= (src[i] != 0);
}
return res;
}
template<>
inline bool hasNonZero_(const double* src, size_t len )
{
bool res = false;
if (len > 0)
{
size_t i=0;
if (sizeof(double) == sizeof(uint64_t))
{
#if CV_ENABLE_UNROLLED
typedef uint64_t double_as_uint_t;
const double_as_uint_t* src_as_ui = reinterpret_cast<const double_as_uint_t*>(src);
for(; !res && (i+4 <= len); i += 4 )
{
const double_as_uint_t gathered = (src_as_ui[i] | src_as_ui[i+1] | src_as_ui[i+2] | src_as_ui[i+3]);
res |= ((gathered<<1) != 0);//remove what would be the sign bit
}
#endif
}
for( ; !res && (i < len); i++ )
res |= (src[i] != 0);
}
return res;
}
static bool hasNonZero8u( const uchar* src, size_t len )
{
bool res = false;
const uchar* srcEnd = src+len;
#undef SIMD_ONLY
#if (CV_SIMD || CV_SIMD_SCALABLE)
typedef v_uint8 v_type;
const v_type v_zero = vx_setzero_u8();
constexpr const int unrollCount = 2;
int step = VTraits<v_type>::vlanes() * unrollCount;
int len0 = len & -step;
const uchar* srcSimdEnd = src+len0;
int countSIMD = static_cast<int>((srcSimdEnd-src)/step);
while(!res && countSIMD--)
{
v_type v0 = vx_load(src);
src += VTraits<v_type>::vlanes();
v_type v1 = vx_load(src);
src += VTraits<v_type>::vlanes();
res = v_check_any((v_ne(v_or(v0, v1), v_zero)));
}
v_cleanup();
#define SIMD_ONLY(expr) expr
#else
#define SIMD_ONLY(expr)
#endif
return res || hasNonZero_(src, srcEnd-src);
#undef DEFINE_HASNONZERO_FUNC
#define DEFINE_HASNONZERO_FUNC(funcname, suffix, T, VT, cmp_op, scalar_nz_op) \
static bool funcname( const T* src, size_t len ) \
{ \
size_t i = 0; \
SIMD_ONLY( \
const int vlanes = VTraits<VT>::vlanes(); \
VT v_zero = vx_setzero_##suffix(); \
for (i = 0; i + vlanes*8 <= len; i += vlanes*8) \
{ \
VT x0 = vx_load(src + i); \
VT x1 = vx_load(src + i + vlanes); \
VT x2 = vx_load(src + i + vlanes*2); \
VT x3 = vx_load(src + i + vlanes*3); \
VT x4 = vx_load(src + i + vlanes*4); \
VT x5 = vx_load(src + i + vlanes*5); \
VT x6 = vx_load(src + i + vlanes*6); \
VT x7 = vx_load(src + i + vlanes*7); \
x0 = v_or(x0, x1); \
x2 = v_or(x2, x3); \
x4 = v_or(x4, x5); \
x6 = v_or(x6, x7); \
x0 = v_or(x0, x2); \
x4 = v_or(x4, x6); \
x0 = v_or(x0, x4); \
x0 = cmp_op(x0, v_zero); \
if (v_check_any(x0)) \
return true; \
} \
for (; i < len; i += vlanes) \
{ \
if (i + vlanes > len) { \
if (i == 0) \
break; \
i = len - vlanes; \
} \
VT x0 = vx_load(src + i); \
x0 = cmp_op(x0, v_zero); \
if (v_check_any(x0)) \
return true; \
} \
v_cleanup();) \
for( ; i < len; i++ ) \
{ \
T x = src[i]; \
if (scalar_nz_op(x) != 0) \
return true; \
} \
return false; \
}
static bool hasNonZero16u( const ushort* src, size_t len )
{
bool res = false;
const ushort* srcEnd = src+len;
#if (CV_SIMD || CV_SIMD_SCALABLE)
typedef v_uint16 v_type;
const v_type v_zero = vx_setzero_u16();
constexpr const int unrollCount = 4;
int step = VTraits<v_type>::vlanes() * unrollCount;
int len0 = len & -step;
const ushort* srcSimdEnd = src+len0;
#undef CHECK_NZ_INT
#define CHECK_NZ_INT(x) ((x) != 0)
#undef CHECK_NZ_FP
#define CHECK_NZ_FP(x) (((x)<<1) != 0)
#undef CHECK_NZ_FP16
#define CHECK_NZ_FP16(x) (((x)&0x7fff) != 0)
#undef VEC_CMP_EQ_Z_FP16
#define VEC_CMP_EQ_Z_FP16(x, z) v_ne(v_add_wrap(x, x), z)
#undef VEC_CMP_EQ_Z_FP
#define VEC_CMP_EQ_Z_FP(x, z) v_ne(v_add(x, x), z)
int countSIMD = static_cast<int>((srcSimdEnd-src)/step);
while(!res && countSIMD--)
{
v_type v0 = vx_load(src);
src += VTraits<v_type>::vlanes();
v_type v1 = vx_load(src);
src += VTraits<v_type>::vlanes();
v_type v2 = vx_load(src);
src += VTraits<v_type>::vlanes();
v_type v3 = vx_load(src);
src += VTraits<v_type>::vlanes();
v0 = v_or(v0, v1);
v2 = v_or(v2, v3);
res = v_check_any((v_ne(v_or(v0, v2), v_zero)));
}
DEFINE_HASNONZERO_FUNC(hasNonZero8u, u8, uchar, v_uint8, v_ne, CHECK_NZ_INT)
DEFINE_HASNONZERO_FUNC(hasNonZero16u, u16, ushort, v_uint16, v_ne, CHECK_NZ_INT)
DEFINE_HASNONZERO_FUNC(hasNonZero32s, s32, int, v_int32, v_ne, CHECK_NZ_INT)
DEFINE_HASNONZERO_FUNC(hasNonZero64s, s64, int64, v_int64, v_ne, CHECK_NZ_INT)
v_cleanup();
#endif
return res || hasNonZero_(src, srcEnd-src);
}
DEFINE_HASNONZERO_FUNC(hasNonZero32f, s32, int, v_int32, VEC_CMP_EQ_Z_FP, CHECK_NZ_FP)
DEFINE_HASNONZERO_FUNC(hasNonZero64f, s64, int64, v_int64, VEC_CMP_EQ_Z_FP, CHECK_NZ_FP)
DEFINE_HASNONZERO_FUNC(hasNonZero16f, u16, ushort, v_uint16, VEC_CMP_EQ_Z_FP16, CHECK_NZ_FP16)
static bool hasNonZero32s( const int* src, size_t len )
{
bool res = false;
const int* srcEnd = src+len;
#if (CV_SIMD || CV_SIMD_SCALABLE)
typedef v_int32 v_type;
const v_type v_zero = vx_setzero_s32();
constexpr const int unrollCount = 8;
int step = VTraits<v_type>::vlanes() * unrollCount;
int len0 = len & -step;
const int* srcSimdEnd = src+len0;
int countSIMD = static_cast<int>((srcSimdEnd-src)/step);
while(!res && countSIMD--)
{
v_type v0 = vx_load(src);
src += VTraits<v_type>::vlanes();
v_type v1 = vx_load(src);
src += VTraits<v_type>::vlanes();
v_type v2 = vx_load(src);
src += VTraits<v_type>::vlanes();
v_type v3 = vx_load(src);
src += VTraits<v_type>::vlanes();
v_type v4 = vx_load(src);
src += VTraits<v_type>::vlanes();
v_type v5 = vx_load(src);
src += VTraits<v_type>::vlanes();
v_type v6 = vx_load(src);
src += VTraits<v_type>::vlanes();
v_type v7 = vx_load(src);
src += VTraits<v_type>::vlanes();
v0 = v_or(v0, v1);
v2 = v_or(v2, v3);
v4 = v_or(v4, v5);
v6 = v_or(v6, v7);
v0 = v_or(v0, v2);
v4 = v_or(v4, v6);
res = v_check_any((v_ne(v_or(v0, v4), v_zero)));
}
v_cleanup();
#endif
return res || hasNonZero_(src, srcEnd-src);
}
static bool hasNonZero32f( const float* src, size_t len )
{
bool res = false;
const float* srcEnd = src+len;
#if (CV_SIMD || CV_SIMD_SCALABLE)
typedef v_float32 v_type;
const v_type v_zero = vx_setzero_f32();
constexpr const int unrollCount = 8;
int step = VTraits<v_type>::vlanes() * unrollCount;
int len0 = len & -step;
const float* srcSimdEnd = src+len0;
int countSIMD = static_cast<int>((srcSimdEnd-src)/step);
while(!res && countSIMD--)
{
v_type v0 = vx_load(src);
src += VTraits<v_type>::vlanes();
v_type v1 = vx_load(src);
src += VTraits<v_type>::vlanes();
v_type v2 = vx_load(src);
src += VTraits<v_type>::vlanes();
v_type v3 = vx_load(src);
src += VTraits<v_type>::vlanes();
v_type v4 = vx_load(src);
src += VTraits<v_type>::vlanes();
v_type v5 = vx_load(src);
src += VTraits<v_type>::vlanes();
v_type v6 = vx_load(src);
src += VTraits<v_type>::vlanes();
v_type v7 = vx_load(src);
src += VTraits<v_type>::vlanes();
v0 = v_or(v0, v1);
v2 = v_or(v2, v3);
v4 = v_or(v4, v5);
v6 = v_or(v6, v7);
v0 = v_or(v0, v2);
v4 = v_or(v4, v6);
//res = v_check_any(((v0 | v4) != v_zero));//beware : (NaN != 0) returns "false" since != is mapped to _CMP_NEQ_OQ and not _CMP_NEQ_UQ
res = !v_check_all((v_eq(v_or(v0, v4), v_zero)));
}
v_cleanup();
#endif
return res || hasNonZero_(src, srcEnd-src);
}
static bool hasNonZero64f( const double* src, size_t len )
{
bool res = false;
const double* srcEnd = src+len;
#if (CV_SIMD_64F || CV_SIMD_SCALABLE_64F)
typedef v_float64 v_type;
const v_type v_zero = vx_setzero_f64();
constexpr const int unrollCount = 16;
int step = VTraits<v_type>::vlanes() * unrollCount;
int len0 = len & -step;
const double* srcSimdEnd = src+len0;
int countSIMD = static_cast<int>((srcSimdEnd-src)/step);
while(!res && countSIMD--)
{
v_type v0 = vx_load(src);
src += VTraits<v_type>::vlanes();
v_type v1 = vx_load(src);
src += VTraits<v_type>::vlanes();
v_type v2 = vx_load(src);
src += VTraits<v_type>::vlanes();
v_type v3 = vx_load(src);
src += VTraits<v_type>::vlanes();
v_type v4 = vx_load(src);
src += VTraits<v_type>::vlanes();
v_type v5 = vx_load(src);
src += VTraits<v_type>::vlanes();
v_type v6 = vx_load(src);
src += VTraits<v_type>::vlanes();
v_type v7 = vx_load(src);
src += VTraits<v_type>::vlanes();
v_type v8 = vx_load(src);
src += VTraits<v_type>::vlanes();
v_type v9 = vx_load(src);
src += VTraits<v_type>::vlanes();
v_type v10 = vx_load(src);
src += VTraits<v_type>::vlanes();
v_type v11 = vx_load(src);
src += VTraits<v_type>::vlanes();
v_type v12 = vx_load(src);
src += VTraits<v_type>::vlanes();
v_type v13 = vx_load(src);
src += VTraits<v_type>::vlanes();
v_type v14 = vx_load(src);
src += VTraits<v_type>::vlanes();
v_type v15 = vx_load(src);
src += VTraits<v_type>::vlanes();
v0 = v_or(v0, v1);
v2 = v_or(v2, v3);
v4 = v_or(v4, v5);
v6 = v_or(v6, v7);
v8 = v_or(v8, v9);
v10 = v_or(v10, v11);
v12 = v_or(v12, v13);
v14 = v_or(v14, v15);
v0 = v_or(v0, v2);
v4 = v_or(v4, v6);
v8 = v_or(v8, v10);
v12 = v_or(v12, v14);
v0 = v_or(v0, v4);
v8 = v_or(v8, v12);
//res = v_check_any(((v0 | v8) != v_zero));//beware : (NaN != 0) returns "false" since != is mapped to _CMP_NEQ_OQ and not _CMP_NEQ_UQ
res = !v_check_all((v_eq(v_or(v0, v8), v_zero)));
}
v_cleanup();
#endif
return res || hasNonZero_(src, srcEnd-src);
}
HasNonZeroFunc getHasNonZeroTab(int depth)
HasNonZeroFunc getHasNonZeroFunc(int depth)
{
static HasNonZeroFunc hasNonZeroTab[CV_DEPTH_MAX] =
{
(HasNonZeroFunc)GET_OPTIMIZED(hasNonZero8u), (HasNonZeroFunc)GET_OPTIMIZED(hasNonZero8u),
(HasNonZeroFunc)GET_OPTIMIZED(hasNonZero16u), (HasNonZeroFunc)GET_OPTIMIZED(hasNonZero16u),
(HasNonZeroFunc)GET_OPTIMIZED(hasNonZero32s), (HasNonZeroFunc)GET_OPTIMIZED(hasNonZero32f),
(HasNonZeroFunc)GET_OPTIMIZED(hasNonZero64f), 0
(HasNonZeroFunc)GET_OPTIMIZED(hasNonZero8u),
(HasNonZeroFunc)GET_OPTIMIZED(hasNonZero8u),
(HasNonZeroFunc)GET_OPTIMIZED(hasNonZero16u),
(HasNonZeroFunc)GET_OPTIMIZED(hasNonZero16u),
(HasNonZeroFunc)GET_OPTIMIZED(hasNonZero32s),
(HasNonZeroFunc)GET_OPTIMIZED(hasNonZero32f),
(HasNonZeroFunc)GET_OPTIMIZED(hasNonZero64f),
(HasNonZeroFunc)GET_OPTIMIZED(hasNonZero16f),
(HasNonZeroFunc)GET_OPTIMIZED(hasNonZero16f),
(HasNonZeroFunc)GET_OPTIMIZED(hasNonZero8u),
(HasNonZeroFunc)GET_OPTIMIZED(hasNonZero64s),
(HasNonZeroFunc)GET_OPTIMIZED(hasNonZero64s),
(HasNonZeroFunc)GET_OPTIMIZED(hasNonZero32s),
0
};
return hasNonZeroTab[depth];
modules/core/src/mathfuncs.cpp
+1 -1
View File
@@ -1137,7 +1137,7 @@ static void iPow64f(const double* src, double* dst, int len, int power)
typedef void (*IPowFunc)( const uchar* src, uchar* dst, int len, int power );
static IPowFunc ipowTab[] =
static IPowFunc ipowTab[CV_DEPTH_MAX] =
{
(IPowFunc)iPow8u, (IPowFunc)iPow8s, (IPowFunc)iPow16u, (IPowFunc)iPow16s,
(IPowFunc)iPow32s, (IPowFunc)iPow32f, (IPowFunc)iPow64f, 0
modules/core/src/matrix_operations.cpp
+2 -2
View File
@@ -1270,7 +1270,7 @@ void cv::sort( InputArray _src, OutputArray _dst, int flags )
Mat dst = _dst.getMat();
CV_IPP_RUN_FAST(ipp_sort(src, dst, flags));
static SortFunc tab[] =
static SortFunc tab[CV_DEPTH_MAX] =
{
sort_<uchar>, sort_<schar>, sort_<ushort>, sort_<short>,
sort_<int>, sort_<float>, sort_<double>, 0
@@ -1295,7 +1295,7 @@ void cv::sortIdx( InputArray _src, OutputArray _dst, int flags )
CV_IPP_RUN_FAST(ipp_sortIdx(src, dst, flags));
static SortFunc tab[] =
static SortFunc tab[CV_DEPTH_MAX] =
{
sortIdx_<uchar>, sortIdx_<schar>, sortIdx_<ushort>, sortIdx_<short>,
sortIdx_<int>, sortIdx_<float>, sortIdx_<double>, 0
modules/core/src/mean.dispatch.cpp
+41 -24
View File
@@ -141,20 +141,19 @@ Scalar mean(InputArray _src, InputArray _mask)
const Mat* arrays[] = {&src, &mask, 0};
uchar* ptrs[2] = {};
NAryMatIterator it(arrays, ptrs);
int total = (int)it.size, blockSize = total, intSumBlockSize = 0;
int total = (int)it.size, blockSize = total, partialBlockSize = 0;
int j, count = 0;
AutoBuffer<int> _buf;
int _buf[CV_CN_MAX];
int* buf = (int*)&s[0];
bool blockSum = depth <= CV_16S;
bool partialSumIsInt = depth < CV_32S;
bool blockSum = partialSumIsInt || depth == CV_16F || depth == CV_16BF;
size_t esz = 0, nz0 = 0;
if( blockSum )
{
intSumBlockSize = depth <= CV_8S ? (1 << 23) : (1 << 15);
blockSize = std::min(blockSize, intSumBlockSize);
_buf.allocate(cn);
buf = _buf.data();
partialBlockSize = depth <= CV_8S ? (1 << 23) : (1 << 15);
blockSize = std::min(blockSize, partialBlockSize);
buf = _buf;
for( k = 0; k < cn; k++ )
buf[k] = 0;
esz = src.elemSize();
@@ -168,12 +167,20 @@ Scalar mean(InputArray _src, InputArray _mask)
int nz = func( ptrs[0], ptrs[1], (uchar*)buf, bsz, cn );
count += nz;
nz0 += nz;
if( blockSum && (count + blockSize >= intSumBlockSize || (i+1 >= it.nplanes && j+bsz >= total)) )
if( blockSum && (count + blockSize >= partialBlockSize || (i+1 >= it.nplanes && j+bsz >= total)) )
{
for( k = 0; k < cn; k++ )
{
s[k] += buf[k];
buf[k] = 0;
if (partialSumIsInt) {
for( k = 0; k < cn; k++ )
{
s[k] += buf[k];
buf[k] = 0;
}
} else {
for( k = 0; k < cn; k++ )
{
s[k] += ((float*)buf)[k];
buf[k] = 0;
}
}
count = 0;
}
@@ -539,12 +546,14 @@ void meanStdDev(InputArray _src, OutputArray _mean, OutputArray _sdv, InputArray
const Mat* arrays[] = {&src, &mask, 0};
uchar* ptrs[2] = {};
NAryMatIterator it(arrays, ptrs);
int total = (int)it.size, blockSize = total, intSumBlockSize = 0;
int total = (int)it.size, blockSize = total, partialBlockSize = 0;
int j, count = 0, nz0 = 0;
AutoBuffer<double> _buf(cn*4);
double *s = (double*)_buf.data(), *sq = s + cn;
double _buf[CV_CN_MAX*4];
double *s = _buf, *sq = s + cn;
int *sbuf = (int*)s, *sqbuf = (int*)sq;
bool blockSum = depth <= CV_16S, blockSqSum = depth <= CV_8S;
bool partialSumIsInt = depth < CV_32S;
bool blockSum = partialSumIsInt || depth == CV_16F || depth == CV_16BF;
bool blockSqSum = depth <= CV_8S;
size_t esz = 0;
for( k = 0; k < cn; k++ )
@@ -552,8 +561,8 @@ void meanStdDev(InputArray _src, OutputArray _mean, OutputArray _sdv, InputArray
if( blockSum )
{
intSumBlockSize = 1 << 15;
blockSize = std::min(blockSize, intSumBlockSize);
partialBlockSize = 1 << 15;
blockSize = std::min(blockSize, partialBlockSize);
sbuf = (int*)(sq + cn);
if( blockSqSum )
sqbuf = sbuf + cn;
@@ -570,12 +579,20 @@ void meanStdDev(InputArray _src, OutputArray _mean, OutputArray _sdv, InputArray
int nz = func( ptrs[0], ptrs[1], (uchar*)sbuf, (uchar*)sqbuf, bsz, cn );
count += nz;
nz0 += nz;
if( blockSum && (count + blockSize >= intSumBlockSize || (i+1 >= it.nplanes && j+bsz >= total)) )
if( blockSum && (count + blockSize >= partialBlockSize || (i+1 >= it.nplanes && j+bsz >= total)) )
{
for( k = 0; k < cn; k++ )
{
s[k] += sbuf[k];
sbuf[k] = 0;
if (partialSumIsInt) {
for( k = 0; k < cn; k++ )
{
s[k] += sbuf[k];
sbuf[k] = 0;
}
} else {
for( k = 0; k < cn; k++ )
{
s[k] += ((float*)sbuf)[k];
sbuf[k] = 0;
}
}
if( blockSqSum )
{
modules/core/src/mean.simd.hpp
+27 -10
View File
@@ -179,7 +179,7 @@ static int sumsqr_(const T* src0, const uchar* mask, ST* sum, SQT* sqsum, int le
SQT sq0 = sqsum[0];
for(int i = x; i < len; i++, src += cn )
{
T v = src[0];
ST v = (ST)src[0];
s0 += v; sq0 += (SQT)v*v;
}
sum[0] = s0;
@@ -191,7 +191,7 @@ static int sumsqr_(const T* src0, const uchar* mask, ST* sum, SQT* sqsum, int le
SQT sq0 = sqsum[0], sq1 = sqsum[1];
for(int i = x; i < len; i++, src += cn )
{
T v0 = src[0], v1 = src[1];
ST v0 = (ST)src[0], v1 = (ST)src[1];
s0 += v0; sq0 += (SQT)v0*v0;
s1 += v1; sq1 += (SQT)v1*v1;
}
@@ -204,7 +204,7 @@ static int sumsqr_(const T* src0, const uchar* mask, ST* sum, SQT* sqsum, int le
SQT sq0 = sqsum[0], sq1 = sqsum[1], sq2 = sqsum[2];
for(int i = x; i < len; i++, src += cn )
{
T v0 = src[0], v1 = src[1], v2 = src[2];
ST v0 = (ST)src[0], v1 = (ST)src[1], v2 = (ST)src[2];
s0 += v0; sq0 += (SQT)v0*v0;
s1 += v1; sq1 += (SQT)v1*v1;
s2 += v2; sq2 += (SQT)v2*v2;
@@ -220,11 +220,11 @@ static int sumsqr_(const T* src0, const uchar* mask, ST* sum, SQT* sqsum, int le
SQT sq0 = sqsum[k], sq1 = sqsum[k+1], sq2 = sqsum[k+2], sq3 = sqsum[k+3];
for(int i = x; i < len; i++, src += cn )
{
T v0, v1;
v0 = src[0], v1 = src[1];
ST v0, v1;
v0 = (ST)src[0], v1 = (ST)src[1];
s0 += v0; sq0 += (SQT)v0*v0;
s1 += v1; sq1 += (SQT)v1*v1;
v0 = src[2], v1 = src[3];
v0 = (ST)src[2], v1 = (ST)src[3];
s2 += v0; sq2 += (SQT)v0*v0;
s3 += v1; sq3 += (SQT)v1*v1;
}
@@ -245,7 +245,7 @@ static int sumsqr_(const T* src0, const uchar* mask, ST* sum, SQT* sqsum, int le
for( i = 0; i < len; i++ )
if( mask[i] )
{
T v = src[i];
ST v = (ST)src[i];
s0 += v; sq0 += (SQT)v*v;
nzm++;
}
@@ -259,7 +259,7 @@ static int sumsqr_(const T* src0, const uchar* mask, ST* sum, SQT* sqsum, int le
for( i = 0; i < len; i++, src += 3 )
if( mask[i] )
{
T v0 = src[0], v1 = src[1], v2 = src[2];
ST v0 = (ST)src[0], v1 = (ST)src[1], v2 = (ST)src[2];
s0 += v0; sq0 += (SQT)v0*v0;
s1 += v1; sq1 += (SQT)v1*v1;
s2 += v2; sq2 += (SQT)v2*v2;
@@ -275,7 +275,7 @@ static int sumsqr_(const T* src0, const uchar* mask, ST* sum, SQT* sqsum, int le
{
for( int k = 0; k < cn; k++ )
{
T v = src[k];
ST v = (ST)src[k];
ST s = sum[k] + v;
SQT sq = sqsum[k] + (SQT)v*v;
sum[k] = s; sqsum[k] = sq;
@@ -308,13 +308,30 @@ static int sqsum32f( const float* src, const uchar* mask, double* sum, double* s
static int sqsum64f( const double* src, const uchar* mask, double* sum, double* sqsum, int len, int cn )
{ CV_INSTRUMENT_REGION(); return sumsqr_(src, mask, sum, sqsum, len, cn); }
static int sqsum16f( const float16_t* src, const uchar* mask, float* sum, double* sqsum, int len, int cn )
{ CV_INSTRUMENT_REGION(); return sumsqr_(src, mask, sum, sqsum, len, cn); }
static int sqsum16bf( const bfloat16_t* src, const uchar* mask, float* sum, double* sqsum, int len, int cn )
{ CV_INSTRUMENT_REGION(); return sumsqr_(src, mask, sum, sqsum, len, cn); }
static int sqsum64u( const uint64* src, const uchar* mask, double* sum, double* sqsum, int len, int cn )
{ CV_INSTRUMENT_REGION(); return sumsqr_(src, mask, sum, sqsum, len, cn); }
static int sqsum64s( const int64* src, const uchar* mask, double* sum, double* sqsum, int len, int cn )
{ CV_INSTRUMENT_REGION(); return sumsqr_(src, mask, sum, sqsum, len, cn); }
static int sqsum32u( const unsigned* src, const uchar* mask, double* sum, double* sqsum, int len, int cn )
{ CV_INSTRUMENT_REGION(); return sumsqr_(src, mask, sum, sqsum, len, cn); }
SumSqrFunc getSumSqrFunc(int depth)
{
CV_INSTRUMENT_REGION();
static SumSqrFunc sumSqrTab[CV_DEPTH_MAX] =
{
(SumSqrFunc)GET_OPTIMIZED(sqsum8u), (SumSqrFunc)sqsum8s, (SumSqrFunc)sqsum16u, (SumSqrFunc)sqsum16s,
(SumSqrFunc)sqsum32s, (SumSqrFunc)GET_OPTIMIZED(sqsum32f), (SumSqrFunc)sqsum64f, 0
(SumSqrFunc)sqsum32s, (SumSqrFunc)GET_OPTIMIZED(sqsum32f), (SumSqrFunc)sqsum64f,
(SumSqrFunc)sqsum16f, (SumSqrFunc)sqsum16bf, 0,
(SumSqrFunc)sqsum64u, (SumSqrFunc)sqsum64s, (SumSqrFunc)sqsum32u, 0
};
return sumSqrTab[depth];
modules/core/src/minmax.cpp
-1710
View File
File diff suppressed because it is too large Load Diff
modules/core/src/minmax.dispatch.cpp
+498
View File
@@ -0,0 +1,498 @@
// This file is part of OpenCV project.
// It is subject to the license terms in the LICENSE file found in the top-level directory
// of this distribution and at http://opencv.org/license.html
#include "precomp.hpp"
#include "opencl_kernels_core.hpp"
#include "opencv2/core/openvx/ovx_defs.hpp"
#include "stat.hpp"
#include "opencv2/core/detail/dispatch_helper.impl.hpp"
#include <algorithm>
#include "minmax.simd.hpp"
#include "minmax.simd_declarations.hpp" // defines CV_CPU_DISPATCH_MODES_ALL=AVX2,...,BASELINE based on CMakeLists.txt content
namespace cv {
static MinMaxIdxFunc getMinMaxIdxFunc(int depth)
{
CV_INSTRUMENT_REGION();
CV_CPU_DISPATCH(getMinMaxIdxFunc, (depth),
CV_CPU_DISPATCH_MODES_ALL);
}
static void ofs2idx(const Mat& a, size_t ofs, int* idx)
{
int i, d = a.dims;
if( ofs > 0 )
{
ofs--;
for( i = d-1; i >= 0; i-- )
{
int sz = a.size[i];
idx[i] = (int)(ofs % sz);
ofs /= sz;
}
}
else
{
for( i = d-1; i >= 0; i-- )
idx[i] = -1;
}
}
#ifdef HAVE_OPENCL
#define MINMAX_STRUCT_ALIGNMENT 8 // sizeof double
template <typename T>
void getMinMaxRes(const Mat & db, double * minVal, double * maxVal,
int* minLoc, int* maxLoc,
int groupnum, int cols, double * maxVal2)
{
uint index_max = std::numeric_limits<uint>::max();
T minval = std::numeric_limits<T>::max();
T maxval = std::numeric_limits<T>::min() > 0 ? -std::numeric_limits<T>::max() : std::numeric_limits<T>::min(), maxval2 = maxval;
uint minloc = index_max, maxloc = index_max;
size_t index = 0;
const T * minptr = NULL, * maxptr = NULL, * maxptr2 = NULL;
const uint * minlocptr = NULL, * maxlocptr = NULL;
if (minVal || minLoc)
{
minptr = db.ptr<T>();
index += sizeof(T) * groupnum;
index = alignSize(index, MINMAX_STRUCT_ALIGNMENT);
}
if (maxVal || maxLoc)
{
maxptr = (const T *)(db.ptr() + index);
index += sizeof(T) * groupnum;
index = alignSize(index, MINMAX_STRUCT_ALIGNMENT);
}
if (minLoc)
{
minlocptr = (const uint *)(db.ptr() + index);
index += sizeof(uint) * groupnum;
index = alignSize(index, MINMAX_STRUCT_ALIGNMENT);
}
if (maxLoc)
{
maxlocptr = (const uint *)(db.ptr() + index);
index += sizeof(uint) * groupnum;
index = alignSize(index, MINMAX_STRUCT_ALIGNMENT);
}
if (maxVal2)
maxptr2 = (const T *)(db.ptr() + index);
for (int i = 0; i < groupnum; i++)
{
if (minptr && minptr[i] <= minval)
{
if (minptr[i] == minval)
{
if (minlocptr)
minloc = std::min(minlocptr[i], minloc);
}
else
{
if (minlocptr)
minloc = minlocptr[i];
minval = minptr[i];
}
}
if (maxptr && maxptr[i] >= maxval)
{
if (maxptr[i] == maxval)
{
if (maxlocptr)
maxloc = std::min(maxlocptr[i], maxloc);
}
else
{
if (maxlocptr)
maxloc = maxlocptr[i];
maxval = maxptr[i];
}
}
if (maxptr2 && maxptr2[i] > maxval2)
maxval2 = maxptr2[i];
}
bool zero_mask = (minLoc && minloc == index_max) ||
(maxLoc && maxloc == index_max);
if (minVal)
*minVal = zero_mask ? 0 : (double)minval;
if (maxVal)
*maxVal = zero_mask ? 0 : (double)maxval;
if (maxVal2)
*maxVal2 = zero_mask ? 0 : (double)maxval2;
if (minLoc)
{
minLoc[0] = zero_mask ? -1 : minloc / cols;
minLoc[1] = zero_mask ? -1 : minloc % cols;
}
if (maxLoc)
{
maxLoc[0] = zero_mask ? -1 : maxloc / cols;
maxLoc[1] = zero_mask ? -1 : maxloc % cols;
}
}
typedef void (*getMinMaxResFunc)(const Mat & db, double * minVal, double * maxVal,
int * minLoc, int *maxLoc, int gropunum, int cols, double * maxVal2);
bool ocl_minMaxIdx( InputArray _src, double* minVal, double* maxVal, int* minLoc, int* maxLoc, InputArray _mask,
int ddepth, bool absValues, InputArray _src2, double * maxVal2)
{
const ocl::Device & dev = ocl::Device::getDefault();
#ifdef __ANDROID__
if (dev.isNVidia())
return false;
#endif
if (dev.deviceVersionMajor() == 1 && dev.deviceVersionMinor() < 2)
{
// 'static' storage class specifier used by "minmaxloc" is available from OpenCL 1.2+ only
return false;
}
bool doubleSupport = dev.doubleFPConfig() > 0, haveMask = !_mask.empty(),
haveSrc2 = _src2.kind() != _InputArray::NONE;
int type = _src.type(), depth = CV_MAT_DEPTH(type), cn = CV_MAT_CN(type),
kercn = haveMask ? cn : std::min(4, ocl::predictOptimalVectorWidth(_src, _src2));
if (depth >= CV_16F)
return false;
// disabled following modes since it occasionally fails on AMD devices (e.g. A10-6800K, sep. 2014)
if ((haveMask || type == CV_32FC1) && dev.isAMD())
return false;
CV_Assert( (cn == 1 && (!haveMask || _mask.type() == CV_8U)) ||
(cn >= 1 && !minLoc && !maxLoc) );
if (ddepth < 0)
ddepth = depth;
CV_Assert(!haveSrc2 || _src2.type() == type);
if (depth == CV_32S || depth == CV_8S || depth == CV_32U || depth == CV_64U ||
depth == CV_64S || depth == CV_16F || depth == CV_16BF)
return false;
if ((depth == CV_64F || ddepth == CV_64F) && !doubleSupport)
return false;
int groupnum = dev.maxComputeUnits();
size_t wgs = dev.maxWorkGroupSize();
int wgs2_aligned = 1;
while (wgs2_aligned < (int)wgs)
wgs2_aligned <<= 1;
wgs2_aligned >>= 1;
bool needMinVal = minVal || minLoc, needMinLoc = minLoc != NULL,
needMaxVal = maxVal || maxLoc, needMaxLoc = maxLoc != NULL;
// in case of mask we must know whether mask is filled with zeros or not
// so let's calculate min or max location, if it's undefined, so mask is zeros
if (!(needMaxLoc || needMinLoc) && haveMask)
{
if (needMinVal)
needMinLoc = true;
else
needMaxLoc = true;
}
char cvt[2][50];
String opts = format("-D DEPTH_%d -D srcT1=%s%s -D WGS=%d -D srcT=%s"
" -D WGS2_ALIGNED=%d%s%s%s -D kercn=%d%s%s%s%s"
" -D dstT1=%s -D dstT=%s -D convertToDT=%s%s%s%s%s -D wdepth=%d -D convertFromU=%s"
" -D MINMAX_STRUCT_ALIGNMENT=%d",
depth, ocl::typeToStr(depth), haveMask ? " -D HAVE_MASK" : "", (int)wgs,
ocl::typeToStr(CV_MAKE_TYPE(depth, kercn)), wgs2_aligned,
doubleSupport ? " -D DOUBLE_SUPPORT" : "",
_src.isContinuous() ? " -D HAVE_SRC_CONT" : "",
_mask.isContinuous() ? " -D HAVE_MASK_CONT" : "", kercn,
needMinVal ? " -D NEED_MINVAL" : "", needMaxVal ? " -D NEED_MAXVAL" : "",
needMinLoc ? " -D NEED_MINLOC" : "", needMaxLoc ? " -D NEED_MAXLOC" : "",
ocl::typeToStr(ddepth), ocl::typeToStr(CV_MAKE_TYPE(ddepth, kercn)),
ocl::convertTypeStr(depth, ddepth, kercn, cvt[0], sizeof(cvt[0])),
absValues ? " -D OP_ABS" : "",
haveSrc2 ? " -D HAVE_SRC2" : "", maxVal2 ? " -D OP_CALC2" : "",
haveSrc2 && _src2.isContinuous() ? " -D HAVE_SRC2_CONT" : "", ddepth,
depth <= CV_32S && ddepth == CV_32S ? ocl::convertTypeStr(CV_8U, ddepth, kercn, cvt[1], sizeof(cvt[1])) : "noconvert",
MINMAX_STRUCT_ALIGNMENT);
ocl::Kernel k("minmaxloc", ocl::core::minmaxloc_oclsrc, opts);
if (k.empty())
return false;
int esz = CV_ELEM_SIZE(ddepth), esz32s = CV_ELEM_SIZE1(CV_32S),
dbsize = groupnum * ((needMinVal ? esz : 0) + (needMaxVal ? esz : 0) +
(needMinLoc ? esz32s : 0) + (needMaxLoc ? esz32s : 0) +
(maxVal2 ? esz : 0))
+ 5 * MINMAX_STRUCT_ALIGNMENT;
UMat src = _src.getUMat(), src2 = _src2.getUMat(), db(1, dbsize, CV_8UC1), mask = _mask.getUMat();
if (cn > 1 && !haveMask)
{
src = src.reshape(1);
src2 = src2.reshape(1);
}
if (haveSrc2)
{
if (!haveMask)
k.args(ocl::KernelArg::ReadOnlyNoSize(src), src.cols, (int)src.total(),
groupnum, ocl::KernelArg::PtrWriteOnly(db), ocl::KernelArg::ReadOnlyNoSize(src2));
else
k.args(ocl::KernelArg::ReadOnlyNoSize(src), src.cols, (int)src.total(),
groupnum, ocl::KernelArg::PtrWriteOnly(db), ocl::KernelArg::ReadOnlyNoSize(mask),
ocl::KernelArg::ReadOnlyNoSize(src2));
}
else
{
if (!haveMask)
k.args(ocl::KernelArg::ReadOnlyNoSize(src), src.cols, (int)src.total(),
groupnum, ocl::KernelArg::PtrWriteOnly(db));
else
k.args(ocl::KernelArg::ReadOnlyNoSize(src), src.cols, (int)src.total(),
groupnum, ocl::KernelArg::PtrWriteOnly(db), ocl::KernelArg::ReadOnlyNoSize(mask));
}
size_t globalsize = groupnum * wgs;
if (!k.run(1, &globalsize, &wgs, true))
return false;
static const getMinMaxResFunc functab[7] =
{
getMinMaxRes<uchar>,
getMinMaxRes<char>,
getMinMaxRes<ushort>,
getMinMaxRes<short>,
getMinMaxRes<int>,
getMinMaxRes<float>,
getMinMaxRes<double>
};
CV_Assert(ddepth <= CV_64F);
getMinMaxResFunc func = functab[ddepth];
int locTemp[2];
func(db.getMat(ACCESS_READ), minVal, maxVal,
needMinLoc ? minLoc ? minLoc : locTemp : minLoc,
needMaxLoc ? maxLoc ? maxLoc : locTemp : maxLoc,
groupnum, src.cols, maxVal2);
return true;
}
#endif
}
void cv::minMaxIdx(InputArray _src, double* minVal,
double* maxVal, int* minIdx, int* maxIdx,
InputArray _mask)
{
CV_INSTRUMENT_REGION();
int type = _src.type(), depth = CV_MAT_DEPTH(type), cn = CV_MAT_CN(type);
CV_Assert( (cn == 1 && (_mask.empty() || _mask.type() == CV_8U)) ||
(cn > 1 && _mask.empty() && !minIdx && !maxIdx) );
CV_OCL_RUN(OCL_PERFORMANCE_CHECK(_src.isUMat()) && _src.dims() <= 2 && (_mask.empty() || _src.size() == _mask.size()),
ocl_minMaxIdx(_src, minVal, maxVal, minIdx, maxIdx, _mask))
Mat src = _src.getMat(), mask = _mask.getMat();
MinMaxIdxFunc func = getMinMaxIdxFunc(depth);
CV_Assert( func != 0 );
const Mat* arrays[] = {&src, &mask, 0};
uchar* ptrs[2] = {};
NAryMatIterator it(arrays, ptrs);
size_t minidx = 0, maxidx = 0;
size_t startidx = 1;
union {
int i;
float f;
double d;
int64 L;
uint64 UL;
} minval, maxval;
int planeSize = (int)it.size*cn;
minval.L = maxval.L = 0;
for( size_t i = 0; i < it.nplanes; i++, ++it, startidx += planeSize )
func( ptrs[0], ptrs[1], &minval.L, &maxval.L, &minidx, &maxidx, planeSize, startidx );
double dminval, dmaxval;
if( depth <= CV_32S || depth == CV_Bool )
dminval = minval.i, dmaxval = maxval.i;
else if( depth == CV_32F || depth == CV_16F || depth == CV_16BF )
dminval = minval.f, dmaxval = maxval.f;
else if( depth == CV_64F )
dminval = minval.d, dmaxval = maxval.d;
else if( depth == CV_64S || depth == CV_32U )
dminval = (double)minval.L, dmaxval = (double)maxval.L;
else {
CV_Assert(depth == CV_64U);
dminval = (double)minval.UL, dmaxval = (double)maxval.UL;
}
if( minVal )
*minVal = dminval;
if( maxVal )
*maxVal = dmaxval;
if( minIdx )
ofs2idx(src, minidx, minIdx);
if( maxIdx )
ofs2idx(src, maxidx, maxIdx);
}
void cv::minMaxLoc( InputArray _img, double* minVal, double* maxVal,
Point* minLoc, Point* maxLoc, InputArray mask )
{
CV_INSTRUMENT_REGION();
int dims = _img.dims();
CV_CheckLE(dims, 2, "");
minMaxIdx(_img, minVal, maxVal, (int*)minLoc, (int*)maxLoc, mask);
if( minLoc) {
if (dims == 2)
std::swap(minLoc->x, minLoc->y);
else {
minLoc->y = 0;
}
}
if( maxLoc) {
if (dims == 2)
std::swap(maxLoc->x, maxLoc->y);
else {
maxLoc->y = 0;
}
}
}
enum class ReduceMode
{
FIRST_MIN = 0, //!< get index of first min occurrence
LAST_MIN = 1, //!< get index of last min occurrence
FIRST_MAX = 2, //!< get index of first max occurrence
LAST_MAX = 3, //!< get index of last max occurrence
};
template <typename T>
struct reduceMinMaxImpl
{
void operator()(const cv::Mat& src, cv::Mat& dst, ReduceMode mode, const int axis) const
{
switch(mode)
{
case ReduceMode::FIRST_MIN:
reduceMinMaxApply<std::less>(src, dst, axis);
break;
case ReduceMode::LAST_MIN:
reduceMinMaxApply<std::less_equal>(src, dst, axis);
break;
case ReduceMode::FIRST_MAX:
reduceMinMaxApply<std::greater>(src, dst, axis);
break;
case ReduceMode::LAST_MAX:
reduceMinMaxApply<std::greater_equal>(src, dst, axis);
break;
}
}
template <template<class> class Cmp>
static void reduceMinMaxApply(const cv::Mat& src, cv::Mat& dst, const int axis)
{
Cmp<T> cmp;
const auto *src_ptr = src.ptr<T>();
auto *dst_ptr = dst.ptr<int32_t>();
const size_t outer_size = src.total(0, axis);
const auto mid_size = static_cast<size_t>(src.size[axis]);
const size_t outer_step = src.total(axis);
const size_t dst_step = dst.total(axis);
const size_t mid_step = src.total(axis + 1);
for (size_t outer = 0; outer < outer_size; ++outer)
{
const size_t outer_offset = outer * outer_step;
const size_t dst_offset = outer * dst_step;
for (size_t mid = 0; mid != mid_size; ++mid)
{
const size_t src_offset = outer_offset + mid * mid_step;
for (size_t inner = 0; inner < mid_step; inner++)
{
int32_t& index = dst_ptr[dst_offset + inner];
const size_t prev = outer_offset + index * mid_step + inner;
const size_t curr = src_offset + inner;
if (cmp(src_ptr[curr], src_ptr[prev]))
{
index = static_cast<int32_t>(mid);
}
}
}
}
}
};
static void reduceMinMax(cv::InputArray src, cv::OutputArray dst, ReduceMode mode, int axis)
{
CV_INSTRUMENT_REGION();
cv::Mat srcMat = src.getMat();
axis = (axis + srcMat.dims) % srcMat.dims;
CV_Assert(srcMat.channels() == 1 && axis >= 0 && axis < srcMat.dims);
std::vector<int> sizes(srcMat.dims);
std::copy(srcMat.size.p, srcMat.size.p + srcMat.dims, sizes.begin());
sizes[axis] = 1;
dst.create(srcMat.dims, sizes.data(), CV_32SC1); // indices
cv::Mat dstMat = dst.getMat();
dstMat.setTo(cv::Scalar::all(0));
if (!srcMat.isContinuous())
{
srcMat = srcMat.clone();
}
bool needs_copy = !dstMat.isContinuous();
if (needs_copy)
{
dstMat = dstMat.clone();
}
cv::detail::depthDispatch<reduceMinMaxImpl>(srcMat.depth(), srcMat, dstMat, mode, axis);
if (needs_copy)
{
dstMat.copyTo(dst);
}
}
void cv::reduceArgMin(InputArray src, OutputArray dst, int axis, bool lastIndex)
{
reduceMinMax(src, dst, lastIndex ? ReduceMode::LAST_MIN : ReduceMode::FIRST_MIN, axis);
}
void cv::reduceArgMax(InputArray src, OutputArray dst, int axis, bool lastIndex)
{
reduceMinMax(src, dst, lastIndex ? ReduceMode::LAST_MAX : ReduceMode::FIRST_MAX, axis);
}
modules/core/src/minmax.simd.hpp
+394
View File
@@ -0,0 +1,394 @@
// This file is part of OpenCV project.
// It is subject to the license terms in the LICENSE file found in the top-level directory
// of this distribution and at http://opencv.org/license.html
#include "precomp.hpp"
namespace cv {
typedef void (*MinMaxIdxFunc)(const uchar* data, const uchar* mask,
void* minval, void* maxval,
size_t* minidx, size_t* maxidx,
int len, size_t startidx);
CV_CPU_OPTIMIZATION_NAMESPACE_BEGIN
MinMaxIdxFunc getMinMaxIdxFunc(int depth);
#ifndef CV_CPU_OPTIMIZATION_DECLARATIONS_ONLY
template<typename T, typename WT> static void
minMaxIdx_( const T* src, const uchar* mask, WT* _minVal, WT* _maxVal,
size_t* _minIdx, size_t* _maxIdx, int len, size_t startIdx )
{
WT minVal = *_minVal, maxVal = *_maxVal;
size_t minIdx = *_minIdx, maxIdx = *_maxIdx;
int i = 0;
if (minIdx == 0 || maxIdx == 0) {
if (mask) {
for (; i < len; i++) {
if (mask[i]) {
minVal = maxVal = (WT)src[i];
minIdx = maxIdx = startIdx + i;
i++;
break;
}
}
}
else if (len > 0) {
minVal = maxVal = (WT)src[0];
minIdx = maxIdx = startIdx;
i++;
}
}
if( !mask )
{
for( ; i < len; i++ )
{
WT val = (WT)src[i];
if( val < minVal )
{
minVal = val;
minIdx = startIdx + i;
}
if( val > maxVal )
{
maxVal = val;
maxIdx = startIdx + i;
}
}
}
else
{
for( ; i < len; i++ )
{
WT val = (WT)src[i];
uchar m = mask[i];
if( m && val < minVal )
{
minVal = val;
minIdx = startIdx + i;
}
if( m && val > maxVal )
{
maxVal = val;
maxIdx = startIdx + i;
}
}
}
*_minIdx = minIdx;
*_maxIdx = maxIdx;
*_minVal = minVal;
*_maxVal = maxVal;
}
#undef SIMD_ONLY
#if (CV_SIMD || CV_SIMD_SCALABLE)
#define SIMD_ONLY(expr) expr
#else
#define SIMD_ONLY(expr)
#endif
static int minMaxInit(const uchar* mask, int len)
{
int i = 0;
SIMD_ONLY(
int vlanes = VTraits<v_uint8>::vlanes();
v_uint8 v_zero = vx_setzero_u8();
for (; i < len; i += vlanes) {
if (i + vlanes > len) {
if (i == 0)
break;
i = len - vlanes;
}
v_uint8 mask_i = v_ne(vx_load(mask + i), v_zero);
if (v_check_any(mask_i))
return i + v_scan_forward(mask_i);
})
for (; i < len; i++) {
if (mask[i] != 0)
return i;
}
return -1;
}
// vectorized implementation for u8, s8, u16 and s16
// uses blocks to decrease the lane size necessary to store indices
#undef DEFINE_MINMAXIDX_SMALLINT_FUNC
#define DEFINE_MINMAXIDX_SMALLINT_FUNC(funcname, suffix, usuffix, T, UT, VT, UVT, WT, BLOCK_SIZE, load_mask) \
static void funcname(const T* src, const uchar* mask, WT* _minVal, WT* _maxVal, \
size_t* _minIdx, size_t* _maxIdx, int len, size_t startIdx) \
{ \
T minVal = T(*_minVal), maxVal = T(*_maxVal); \
size_t minIdx = *_minIdx, maxIdx = *_maxIdx; \
int i = 0; \
/* initialize minVal/maxVal/minIdx/maxIdx to the proper values in the beginning */ \
if (minIdx == 0) { \
if (mask) { \
i = minMaxInit(mask, len); \
if (i < 0) \
return; \
} \
minVal = maxVal = src[i]; \
minIdx = maxIdx = startIdx + i; \
i++; \
} \
SIMD_ONLY( \
const int vlanes = VTraits<VT>::vlanes(); \
const int block_size0 = BLOCK_SIZE - vlanes; \
if (len-i >= vlanes && block_size0 > 0 && block_size0 % vlanes == 0) { \
UT idxbuf[VTraits<UVT>::max_nlanes]; \
for (int j = 0; j < vlanes; j++) \
idxbuf[j] = (UT)j; \
UVT v_idx0 = vx_load(idxbuf); \
UVT v_idx_delta = vx_setall_##usuffix((UT)vlanes); \
UVT v_invalid_idx = vx_setall_##usuffix((UT)-1); \
VT v_minval = vx_setall_##suffix(minVal); \
VT v_maxval = vx_setall_##suffix(maxVal); \
int block_size = block_size0; \
/* process data by blocks: */ \
/* - for u8/s8 data each block contains up to 256-vlanes elements */ \
/* - for u16/s16 data each block contains up to 65536-vlanes elements */ \
/* inside each block we can store the relative (local) index (v_locidx) */ \
/* in a compact way: 8 bits per lane for u8/s8 data, */ \
/* 16 bits per lane for u16/s16 data */ \
/* 0b111...111 is "invalid index", meaning that this */ \
/* particular lane has not been updated. */ \
/* after each block we update minVal, maxVal, minIdx and maxIdx */ \
for (; i <= len - vlanes; i += block_size) { \
block_size = std::min(block_size, (len - i) & -vlanes); \
UVT v_locidx = v_idx0; \
UVT v_minidx = v_invalid_idx; \
UVT v_maxidx = v_invalid_idx; \
if (!mask) { \
for (int j = 0; j < block_size; j += vlanes) { \
VT data = vx_load(src + i + j); \
UVT lt_min = v_reinterpret_as_##usuffix(v_lt(data, v_minval)); \
UVT gt_max = v_reinterpret_as_##usuffix(v_gt(data, v_maxval)); \
v_minidx = v_select(lt_min, v_locidx, v_minidx); \
v_maxidx = v_select(gt_max, v_locidx, v_maxidx); \
v_minval = v_min(v_minval, data); \
v_maxval = v_max(v_maxval, data); \
v_locidx = v_add(v_locidx, v_idx_delta); \
} \
} else { \
UVT v_zero = vx_setzero_##usuffix(); \
for (int j = 0; j < block_size; j += vlanes) { \
VT data = vx_load(src + i + j); \
UVT msk = v_ne(load_mask(mask + i + j), v_zero); \
UVT lt_min = v_reinterpret_as_##usuffix(v_lt(data, v_minval)); \
UVT gt_max = v_reinterpret_as_##usuffix(v_gt(data, v_maxval)); \
lt_min = v_and(lt_min, msk); \
gt_max = v_and(gt_max, msk); \
v_minidx = v_select(lt_min, v_locidx, v_minidx); \
v_maxidx = v_select(gt_max, v_locidx, v_maxidx); \
VT lt_min_data = v_reinterpret_as_##suffix(lt_min); \
VT gt_max_data = v_reinterpret_as_##suffix(gt_max); \
v_minval = v_select(lt_min_data, data, v_minval); \
v_maxval = v_select(gt_max_data, data, v_maxval); \
v_locidx = v_add(v_locidx, v_idx_delta); \
} \
} \
/* for both minimum and maximum we check whether global extremum */ \
/* and its index need to be updated. If yes, we compute */ \
/* the smallest index within the block where the new global \
/* extremum value occurs */ \
UVT idxmask = v_ne(v_minidx, v_invalid_idx); \
if (v_check_any(idxmask)) { \
minVal = (T)v_reduce_min(v_minval); \
VT invmask = v_ne(v_minval, vx_setall_##suffix(minVal)); \
v_minidx = v_or(v_minidx, v_reinterpret_as_##usuffix(invmask)); \
minIdx = startIdx + i + v_reduce_min(v_minidx); \
v_minval = vx_setall_##suffix(minVal); \
} \
idxmask = v_ne(v_maxidx, v_invalid_idx); \
if (v_check_any(idxmask)) { \
maxVal = (T)v_reduce_max(v_maxval); \
VT invmask = v_ne(v_maxval, vx_setall_##suffix(maxVal)); \
v_maxidx = v_or(v_maxidx, v_reinterpret_as_##usuffix(invmask)); \
maxIdx = startIdx + i + v_reduce_min(v_maxidx); \
v_maxval = vx_setall_##suffix(maxVal); \
} \
} \
}) \
*_minVal = (WT)minVal; \
*_maxVal = (WT)maxVal; \
*_minIdx = minIdx; \
*_maxIdx = maxIdx; \
/* [TODO]: unlike sum, countNonZero and other reduce operations, */ \
/* in the case of minMaxIdx we can process the tail using */ \
/* vector overlapping technique (as in arithmetic operations) */ \
if (i < len) { \
src += i; \
if (mask) mask += i; \
startIdx += i; \
len -= i; \
minMaxIdx_(src, mask, _minVal, _maxVal, _minIdx, _maxIdx, len, startIdx); \
} \
}
// vectorized implementation for s32, f32, f16 and bf16
// (potentially can be extended for u32)
// no need to use blocks here
#undef DEFINE_MINMAXIDX_FUNC
#define DEFINE_MINMAXIDX_FUNC(funcname, suffix, usuffix, T, UT, VT, UVT, WT, load_op) \
static void funcname(const T* src, const uchar* mask, WT* _minVal, WT* _maxVal, \
size_t* _minIdx, size_t* _maxIdx, int len, size_t startIdx) \
{ \
WT minVal = *_minVal, maxVal = *_maxVal; \
size_t minIdx = *_minIdx, maxIdx = *_maxIdx; \
int i = 0; \
/* initialize minVal/maxVal/minIdx/maxIdx to the proper values in the beginning */ \
if (minIdx == 0) { \
if (mask) { \
i = minMaxInit(mask, len); \
if (i < 0) \
return; \
} \
minVal = maxVal = src[i]; \
minIdx = maxIdx = startIdx + i; \
i++; \
} \
SIMD_ONLY( \
const int vlanes = VTraits<VT>::vlanes(); \
UT idxbuf[VTraits<UVT>::max_nlanes]; \
for (int j = 0; j < vlanes; j++) \
idxbuf[j] = (UT)(i+j); \
UVT v_locidx = vx_load(idxbuf); \
UVT v_idx_delta = vx_setall_##usuffix((UT)vlanes); \
UVT v_invalid_idx = vx_setall_##usuffix((UT)-1); \
VT v_minval = vx_setall_##suffix(minVal); \
VT v_maxval = vx_setall_##suffix(maxVal); \
UVT v_minidx = v_invalid_idx; \
UVT v_maxidx = v_invalid_idx; \
/* process data by blocks: */ \
/* - for u8/s8 data each block contains up to 256-vlanes elements */ \
/* - for u16/s16 data each block contains up to 65536-vlanes elements */ \
/* inside each block we can store the relative (local) index (v_locidx) */ \
/* in a compact way: 8 bits per lane for u8/s8 data, */ \
/* 16 bits per lane for u16/s16 data */ \
/* 0b111...111 is "invalid index", meaning that this */ \
/* particular lane has not been updated. */ \
/* after each block we update minVal, maxVal, minIdx and maxIdx */ \
if (!mask) { \
for (; i <= len - vlanes; i += vlanes) { \
VT data = load_op(src + i); \
UVT lt_min = v_reinterpret_as_##usuffix(v_lt(data, v_minval)); \
UVT gt_max = v_reinterpret_as_##usuffix(v_gt(data, v_maxval)); \
v_minidx = v_select(lt_min, v_locidx, v_minidx); \
v_maxidx = v_select(gt_max, v_locidx, v_maxidx); \
v_minval = v_min(v_minval, data); \
v_maxval = v_max(v_maxval, data); \
v_locidx = v_add(v_locidx, v_idx_delta); \
} \
} else { \
UVT v_zero = vx_setzero_##usuffix(); \
for (; i <= len - vlanes; i += vlanes) { \
VT data = load_op(src + i); \
UVT msk = v_ne(vx_load_expand_q(mask + i), v_zero); \
UVT lt_min = v_reinterpret_as_##usuffix(v_lt(data, v_minval)); \
UVT gt_max = v_reinterpret_as_##usuffix(v_gt(data, v_maxval)); \
lt_min = v_and(lt_min, msk); \
gt_max = v_and(gt_max, msk); \
v_minidx = v_select(lt_min, v_locidx, v_minidx); \
v_maxidx = v_select(gt_max, v_locidx, v_maxidx); \
VT lt_min_data = v_reinterpret_as_##suffix(lt_min); \
VT gt_max_data = v_reinterpret_as_##suffix(gt_max); \
v_minval = v_select(lt_min_data, data, v_minval); \
v_maxval = v_select(gt_max_data, data, v_maxval); \
v_locidx = v_add(v_locidx, v_idx_delta); \
} \
} \
/* for both minimum and maximum we check whether global extremum */ \
/* and its index need to be updated. If yes, we compute */ \
/* the smallest index within the block where the new global \
/* extremum value occurs */ \
UVT idxmask = v_ne(v_minidx, v_invalid_idx); \
if (v_check_any(idxmask)) { \
minVal = v_reduce_min(v_minval); \
VT invmask = v_ne(v_minval, vx_setall_##suffix(minVal)); \
v_minidx = v_or(v_minidx, v_reinterpret_as_##usuffix(invmask)); \
minIdx = startIdx + v_reduce_min(v_minidx); \
v_minval = vx_setall_##suffix(minVal); \
} \
idxmask = v_ne(v_maxidx, v_invalid_idx); \
if (v_check_any(idxmask)) { \
maxVal = v_reduce_max(v_maxval); \
VT invmask = v_ne(v_maxval, vx_setall_##suffix(maxVal)); \
v_maxidx = v_or(v_maxidx, v_reinterpret_as_##usuffix(invmask)); \
maxIdx = startIdx + v_reduce_min(v_maxidx); \
v_maxval = vx_setall_##suffix(maxVal); \
}) \
*_minVal = minVal; \
*_maxVal = maxVal; \
*_minIdx = minIdx; \
*_maxIdx = maxIdx; \
/* [TODO]: unlike sum, countNonZero and other reduce operations, */ \
/* in the case of minMaxIdx we can process the tail using */ \
/* vector overlapping technique (as in arithmetic operations) */ \
if (i < len) { \
src += i; \
if (mask) mask += i; \
startIdx += i; \
len -= i; \
minMaxIdx_(src, mask, _minVal, _maxVal, _minIdx, _maxIdx, len, startIdx); \
} \
}
#undef DEFINE_MINMAXIDX_FUNC_NOSIMD
#define DEFINE_MINMAXIDX_FUNC_NOSIMD(funcname, T, WT) \
static void funcname(const T* src, const uchar* mask, WT* _minVal, WT* _maxVal, \
size_t* _minIdx, size_t* _maxIdx, int len, size_t startIdx) \
{ \
minMaxIdx_(src, mask, _minVal, _maxVal, _minIdx, _maxIdx, len, startIdx); \
}
DEFINE_MINMAXIDX_SMALLINT_FUNC(minMaxIdx8u, u8, u8, uchar, uchar, v_uint8, v_uint8, int, 256, vx_load)
DEFINE_MINMAXIDX_SMALLINT_FUNC(minMaxIdx8s, s8, u8, schar, uchar, v_int8, v_uint8, int, 256, vx_load)
DEFINE_MINMAXIDX_SMALLINT_FUNC(minMaxIdx16u, u16, u16, ushort, ushort, v_uint16, v_uint16, int, 65536, vx_load_expand)
DEFINE_MINMAXIDX_SMALLINT_FUNC(minMaxIdx16s, s16, u16, short, ushort, v_int16, v_uint16, int, 65536, vx_load_expand)
DEFINE_MINMAXIDX_FUNC(minMaxIdx32s, s32, u32, int, unsigned, v_int32, v_uint32, int, vx_load)
DEFINE_MINMAXIDX_FUNC(minMaxIdx32f, f32, u32, float, unsigned, v_float32, v_uint32, float, vx_load)
DEFINE_MINMAXIDX_FUNC(minMaxIdx16f, f32, u32, float16_t, unsigned, v_float32, v_uint32, float, vx_load_expand)
DEFINE_MINMAXIDX_FUNC(minMaxIdx16bf, f32, u32, bfloat16_t, unsigned, v_float32, v_uint32, float, vx_load_expand)
//DEFINE_MINMAXIDX_FUNC_NOSIMD(minMaxIdx32s, int, int)
//DEFINE_MINMAXIDX_FUNC_NOSIMD(minMaxIdx32f, float, float)
DEFINE_MINMAXIDX_FUNC_NOSIMD(minMaxIdx64f, double, double)
//DEFINE_MINMAXIDX_FUNC_NOSIMD(minMaxIdx16f, float16_t, float)
//DEFINE_MINMAXIDX_FUNC_NOSIMD(minMaxIdx16bf, bfloat16_t, float)
DEFINE_MINMAXIDX_FUNC_NOSIMD(minMaxIdx64u, uint64, uint64)
DEFINE_MINMAXIDX_FUNC_NOSIMD(minMaxIdx64s, int64, int64)
DEFINE_MINMAXIDX_FUNC_NOSIMD(minMaxIdx32u, unsigned, int64)
MinMaxIdxFunc getMinMaxIdxFunc(int depth)
{
static MinMaxIdxFunc minMaxIdxTab[CV_DEPTH_MAX] =
{
(MinMaxIdxFunc)GET_OPTIMIZED(minMaxIdx8u),
(MinMaxIdxFunc)GET_OPTIMIZED(minMaxIdx8s),
(MinMaxIdxFunc)GET_OPTIMIZED(minMaxIdx16u),
(MinMaxIdxFunc)GET_OPTIMIZED(minMaxIdx16s),
(MinMaxIdxFunc)GET_OPTIMIZED(minMaxIdx32s),
(MinMaxIdxFunc)GET_OPTIMIZED(minMaxIdx32f),
(MinMaxIdxFunc)GET_OPTIMIZED(minMaxIdx64f),
(MinMaxIdxFunc)GET_OPTIMIZED(minMaxIdx16f),
(MinMaxIdxFunc)GET_OPTIMIZED(minMaxIdx16bf),
(MinMaxIdxFunc)GET_OPTIMIZED(minMaxIdx8u),
(MinMaxIdxFunc)GET_OPTIMIZED(minMaxIdx64u),
(MinMaxIdxFunc)GET_OPTIMIZED(minMaxIdx64s),
(MinMaxIdxFunc)GET_OPTIMIZED(minMaxIdx32u),
0
};
return minMaxIdxTab[depth];
}
#endif
CV_CPU_OPTIMIZATION_NAMESPACE_END
} // namespace
modules/core/src/nan_mask.simd.hpp
+1 -1
View File
@@ -419,7 +419,7 @@ void finiteMask_(const uchar *src, uchar *dst, size_t total)
FiniteMaskFunc getFiniteMaskFunc(bool isDouble, int cn)
{
static FiniteMaskFunc tab[] =
static FiniteMaskFunc tab[CV_DEPTH_MAX] =
{
(FiniteMaskFunc)GET_OPTIMIZED((finiteMask_<float, 1>)),
(FiniteMaskFunc)GET_OPTIMIZED((finiteMask_<float, 2>)),
modules/core/src/norm.cpp
+151 -110
View File
@@ -223,7 +223,7 @@ normInf_(const T* src, const uchar* mask, ST* _result, int len, int cn)
if( mask[i] )
{
for( int k = 0; k < cn; k++ )
result = std::max(result, ST(cv_abs(src[k])));
result = std::max(result, (ST)cv_abs(src[k]));
}
}
*_result = result;
@@ -266,8 +266,8 @@ normL2_(const T* src, const uchar* mask, ST* _result, int len, int cn)
{
for( int k = 0; k < cn; k++ )
{
T v = src[k];
result += (ST)v*v;
ST v = (ST)src[k];
result += v*v;
}
}
}
@@ -289,14 +289,14 @@ normDiffInf_(const T* src1, const T* src2, const uchar* mask, ST* _result, int l
if( mask[i] )
{
for( int k = 0; k < cn; k++ )
result = std::max(result, (ST)std::abs(src1[k] - src2[k]));
result = std::max(result, (ST)cv_absdiff(src1[k], src2[k]));
}
}
*_result = result;
return 0;
}
template<typename T, typename ST> int
template<typename T, typename ST, typename WT=T> int
normDiffL1_(const T* src1, const T* src2, const uchar* mask, ST* _result, int len, int cn)
{
ST result = *_result;
@@ -310,7 +310,7 @@ normDiffL1_(const T* src1, const T* src2, const uchar* mask, ST* _result, int le
if( mask[i] )
{
for( int k = 0; k < cn; k++ )
result += std::abs(src1[k] - src2[k]);
result += cv_absdiff(src1[k], src2[k]);
}
}
*_result = result;
@@ -332,7 +332,7 @@ normDiffL2_(const T* src1, const T* src2, const uchar* mask, ST* _result, int le
{
for( int k = 0; k < cn; k++ )
{
ST v = src1[k] - src2[k];
ST v = (ST)src1[k] - (ST)src2[k];
result += v*v;
}
}
@@ -343,10 +343,10 @@ normDiffL2_(const T* src1, const T* src2, const uchar* mask, ST* _result, int le
#define CV_DEF_NORM_FUNC(L, suffix, type, ntype) \
static int norm##L##_##suffix(const type* src, const uchar* mask, ntype* r, int len, int cn) \
{ return norm##L##_(src, mask, r, len, cn); } \
{ return norm##L##_<type, ntype>(src, mask, r, len, cn); } \
static int normDiff##L##_##suffix(const type* src1, const type* src2, \
const uchar* mask, ntype* r, int len, int cn) \
{ return normDiff##L##_(src1, src2, mask, r, (int)len, cn); }
{ return normDiff##L##_<type, ntype>(src1, src2, mask, r, (int)len, cn); }
#define CV_DEF_NORM_ALL(suffix, type, inftype, l1type, l2type) \
CV_DEF_NORM_FUNC(Inf, suffix, type, inftype) \
@@ -357,29 +357,69 @@ CV_DEF_NORM_ALL(8u, uchar, int, int, int)
CV_DEF_NORM_ALL(8s, schar, int, int, int)
CV_DEF_NORM_ALL(16u, ushort, int, int, double)
CV_DEF_NORM_ALL(16s, short, int, int, double)
CV_DEF_NORM_ALL(32s, int, int, double, double)
CV_DEF_NORM_ALL(32u, unsigned, unsigned, double, double)
CV_DEF_NORM_ALL(32s, int, unsigned, double, double)
CV_DEF_NORM_ALL(32f, float, float, double, double)
CV_DEF_NORM_ALL(64f, double, double, double, double)
CV_DEF_NORM_ALL(64u, uint64, uint64, double, double)
CV_DEF_NORM_ALL(64s, int64, uint64, double, double)
CV_DEF_NORM_ALL(16f, float16_t, float, float, float)
CV_DEF_NORM_ALL(16bf, bfloat16_t, float, float, float)
typedef int (*NormFunc)(const uchar*, const uchar*, uchar*, int, int);
typedef int (*NormDiffFunc)(const uchar*, const uchar*, const uchar*, uchar*, int, int);
typedef int (*NormFunc)(const uchar*, const uchar*, void*, int, int);
typedef int (*NormDiffFunc)(const uchar*, const uchar*, const uchar*, void*, int, int);
static NormFunc getNormFunc(int normType, int depth)
{
static NormFunc normTab[3][CV_DEPTH_MAX] =
{
{
(NormFunc)GET_OPTIMIZED(normInf_8u), (NormFunc)GET_OPTIMIZED(normInf_8s), (NormFunc)GET_OPTIMIZED(normInf_16u), (NormFunc)GET_OPTIMIZED(normInf_16s),
(NormFunc)GET_OPTIMIZED(normInf_32s), (NormFunc)GET_OPTIMIZED(normInf_32f), (NormFunc)normInf_64f, 0
(NormFunc)GET_OPTIMIZED(normInf_8u),
(NormFunc)GET_OPTIMIZED(normInf_8s),
(NormFunc)GET_OPTIMIZED(normInf_16u),
(NormFunc)GET_OPTIMIZED(normInf_16s),
(NormFunc)GET_OPTIMIZED(normInf_32s),
(NormFunc)GET_OPTIMIZED(normInf_32f),
(NormFunc)normInf_64f,
(NormFunc)GET_OPTIMIZED(normInf_16f),
(NormFunc)GET_OPTIMIZED(normInf_16bf),
0,
(NormFunc)GET_OPTIMIZED(normInf_64u),
(NormFunc)GET_OPTIMIZED(normInf_64s),
(NormFunc)GET_OPTIMIZED(normInf_32u),
0
},
{
(NormFunc)GET_OPTIMIZED(normL1_8u), (NormFunc)GET_OPTIMIZED(normL1_8s), (NormFunc)GET_OPTIMIZED(normL1_16u), (NormFunc)GET_OPTIMIZED(normL1_16s),
(NormFunc)GET_OPTIMIZED(normL1_32s), (NormFunc)GET_OPTIMIZED(normL1_32f), (NormFunc)normL1_64f, 0
(NormFunc)GET_OPTIMIZED(normL1_8u),
(NormFunc)GET_OPTIMIZED(normL1_8s),
(NormFunc)GET_OPTIMIZED(normL1_16u),
(NormFunc)GET_OPTIMIZED(normL1_16s),
(NormFunc)GET_OPTIMIZED(normL1_32s),
(NormFunc)GET_OPTIMIZED(normL1_32f),
(NormFunc)normL1_64f,
(NormFunc)GET_OPTIMIZED(normL1_16f),
(NormFunc)GET_OPTIMIZED(normL1_16bf),
0,
(NormFunc)GET_OPTIMIZED(normL1_64u),
(NormFunc)GET_OPTIMIZED(normL1_64s),
(NormFunc)GET_OPTIMIZED(normL1_32u),
0
},
{
(NormFunc)GET_OPTIMIZED(normL2_8u), (NormFunc)GET_OPTIMIZED(normL2_8s), (NormFunc)GET_OPTIMIZED(normL2_16u), (NormFunc)GET_OPTIMIZED(normL2_16s),
(NormFunc)GET_OPTIMIZED(normL2_32s), (NormFunc)GET_OPTIMIZED(normL2_32f), (NormFunc)normL2_64f, 0
(NormFunc)GET_OPTIMIZED(normL2_8u),
(NormFunc)GET_OPTIMIZED(normL2_8s),
(NormFunc)GET_OPTIMIZED(normL2_16u),
(NormFunc)GET_OPTIMIZED(normL2_16s),
(NormFunc)GET_OPTIMIZED(normL2_32s),
(NormFunc)GET_OPTIMIZED(normL2_32f),
(NormFunc)normL2_64f,
(NormFunc)GET_OPTIMIZED(normL2_16f),
(NormFunc)GET_OPTIMIZED(normL2_16bf),
0,
(NormFunc)GET_OPTIMIZED(normL2_64u),
(NormFunc)GET_OPTIMIZED(normL2_64s),
(NormFunc)GET_OPTIMIZED(normL2_32u),
0
}
};
@@ -391,22 +431,52 @@ static NormDiffFunc getNormDiffFunc(int normType, int depth)
static NormDiffFunc normDiffTab[3][CV_DEPTH_MAX] =
{
{
(NormDiffFunc)GET_OPTIMIZED(normDiffInf_8u), (NormDiffFunc)normDiffInf_8s,
(NormDiffFunc)normDiffInf_16u, (NormDiffFunc)normDiffInf_16s,
(NormDiffFunc)normDiffInf_32s, (NormDiffFunc)GET_OPTIMIZED(normDiffInf_32f),
(NormDiffFunc)normDiffInf_64f, 0
(NormDiffFunc)GET_OPTIMIZED(normDiffInf_8u),
(NormDiffFunc)GET_OPTIMIZED(normDiffInf_8s),
(NormDiffFunc)GET_OPTIMIZED(normDiffInf_16u),
(NormDiffFunc)GET_OPTIMIZED(normDiffInf_16s),
(NormDiffFunc)GET_OPTIMIZED(normDiffInf_32s),
(NormDiffFunc)GET_OPTIMIZED(normDiffInf_32f),
(NormDiffFunc)normDiffInf_64f,
(NormDiffFunc)GET_OPTIMIZED(normDiffInf_16f),
(NormDiffFunc)GET_OPTIMIZED(normDiffInf_16bf),
0,
(NormDiffFunc)GET_OPTIMIZED(normDiffInf_64u),
(NormDiffFunc)GET_OPTIMIZED(normDiffInf_64s),
(NormDiffFunc)GET_OPTIMIZED(normDiffInf_32u),
0
},
{
(NormDiffFunc)GET_OPTIMIZED(normDiffL1_8u), (NormDiffFunc)normDiffL1_8s,
(NormDiffFunc)normDiffL1_16u, (NormDiffFunc)normDiffL1_16s,
(NormDiffFunc)normDiffL1_32s, (NormDiffFunc)GET_OPTIMIZED(normDiffL1_32f),
(NormDiffFunc)normDiffL1_64f, 0
(NormDiffFunc)GET_OPTIMIZED(normDiffL1_8u),
(NormDiffFunc)GET_OPTIMIZED(normDiffL1_8s),
(NormDiffFunc)GET_OPTIMIZED(normDiffL1_16u),
(NormDiffFunc)GET_OPTIMIZED(normDiffL1_16s),
(NormDiffFunc)GET_OPTIMIZED(normDiffL1_32s),
(NormDiffFunc)GET_OPTIMIZED(normDiffL1_32f),
(NormDiffFunc)normDiffL1_64f,
(NormDiffFunc)GET_OPTIMIZED(normDiffL1_16f),
(NormDiffFunc)GET_OPTIMIZED(normDiffL1_16bf),
0,
(NormDiffFunc)GET_OPTIMIZED(normDiffL1_64u),
(NormDiffFunc)GET_OPTIMIZED(normDiffL1_64s),
(NormDiffFunc)GET_OPTIMIZED(normDiffL1_32u),
0
},
{
(NormDiffFunc)GET_OPTIMIZED(normDiffL2_8u), (NormDiffFunc)normDiffL2_8s,
(NormDiffFunc)normDiffL2_16u, (NormDiffFunc)normDiffL2_16s,
(NormDiffFunc)normDiffL2_32s, (NormDiffFunc)GET_OPTIMIZED(normDiffL2_32f),
(NormDiffFunc)normDiffL2_64f, 0
(NormDiffFunc)GET_OPTIMIZED(normDiffL2_8u),
(NormDiffFunc)GET_OPTIMIZED(normDiffL2_8s),
(NormDiffFunc)GET_OPTIMIZED(normDiffL2_16u),
(NormDiffFunc)GET_OPTIMIZED(normDiffL2_16s),
(NormDiffFunc)GET_OPTIMIZED(normDiffL2_32s),
(NormDiffFunc)GET_OPTIMIZED(normDiffL2_32f),
(NormDiffFunc)normDiffL2_64f,
(NormDiffFunc)GET_OPTIMIZED(normDiffL2_16f),
(NormDiffFunc)GET_OPTIMIZED(normDiffL2_16bf),
0,
(NormDiffFunc)GET_OPTIMIZED(normDiffL2_64u),
(NormDiffFunc)GET_OPTIMIZED(normDiffL2_64s),
(NormDiffFunc)GET_OPTIMIZED(normDiffL2_32u),
0
}
};
@@ -694,7 +764,7 @@ double norm( InputArray _src, int normType, InputArray _mask )
return result;
}
NormFunc func = getNormFunc(normType >> 1, depth == CV_16F ? CV_32F : depth);
NormFunc func = getNormFunc(normType >> 1, depth);
CV_Assert( func != 0 );
const Mat* arrays[] = {&src, &mask, 0};
@@ -702,23 +772,30 @@ double norm( InputArray _src, int normType, InputArray _mask )
union
{
double d;
int i;
unsigned u;
uint64 UL;
float f;
}
result;
result.d = 0;
NAryMatIterator it(arrays, ptrs);
CV_CheckLT((size_t)it.size, (size_t)INT_MAX, "");
bool is_fp16 = depth == CV_16F || depth == CV_16BF;
if ((normType == NORM_L1 && depth <= CV_16S) ||
((normType == NORM_L2 || normType == NORM_L2SQR) && depth <= CV_8S))
if ((normType == NORM_L1 && (depth <= CV_16S || is_fp16)) ||
((normType == NORM_L2 || normType == NORM_L2SQR) && (depth <= CV_8S || is_fp16)))
{
// special case to handle "integer" overflow in accumulator
const size_t esz = src.elemSize();
const int total = (int)it.size;
const int intSumBlockSize = (normType == NORM_L1 && depth <= CV_8S ? (1 << 23) : (1 << 15))/cn;
const int blockSize = std::min(total, intSumBlockSize);
int isum = 0;
const int blockSize0 = (is_fp16 ? (1 << 10) :
normType == NORM_L1 && depth <= CV_8S ? (1 << 23) : (1 << 15))/cn;
const int blockSize = std::min(total, blockSize0);
union {
int i;
float f;
} blocksum;
blocksum.i = 0;
int count = 0;
for (size_t i = 0; i < it.nplanes; i++, ++it)
@@ -726,12 +803,12 @@ double norm( InputArray _src, int normType, InputArray _mask )
for (int j = 0; j < total; j += blockSize)
{
int bsz = std::min(total - j, blockSize);
func(ptrs[0], ptrs[1], (uchar*)&isum, bsz, cn);
func(ptrs[0], ptrs[1], &blocksum.i, bsz, cn);
count += bsz;
if (count + blockSize >= intSumBlockSize || (i+1 >= it.nplanes && j+bsz >= total))
if (count + blockSize >= blockSize0 || (i+1 >= it.nplanes && j+bsz >= total))
{
result.d += isum;
isum = 0;
result.d += is_fp16 ? (double)blocksum.f : (double)blocksum.i;
blocksum.i = 0;
count = 0;
}
ptrs[0] += bsz*esz;
@@ -740,45 +817,25 @@ double norm( InputArray _src, int normType, InputArray _mask )
}
}
}
else if (depth == CV_16F)
{
const size_t esz = src.elemSize();
const int total = (int)it.size;
const int blockSize = std::min(total, divUp(1024, cn));
AutoBuffer<float, 1026/*divUp(1024,3)*3*/> fltbuf(blockSize * cn);
float* data0 = fltbuf.data();
for (size_t i = 0; i < it.nplanes; i++, ++it)
{
for (int j = 0; j < total; j += blockSize)
{
int bsz = std::min(total - j, blockSize);
hal::cvt16f32f((const float16_t*)ptrs[0], data0, bsz * cn);
func((uchar*)data0, ptrs[1], (uchar*)&result.f, bsz, cn);
ptrs[0] += bsz*esz;
if (ptrs[1])
ptrs[1] += bsz;
}
}
}
else
{
// generic implementation
for (size_t i = 0; i < it.nplanes; i++, ++it)
{
func(ptrs[0], ptrs[1], (uchar*)&result, (int)it.size, cn);
func(ptrs[0], ptrs[1], &result, (int)it.size, cn);
}
}
if( normType == NORM_INF )
{
if(depth == CV_64F)
return result.d;
else if (depth == CV_32F || depth == CV_16F)
if(depth <= CV_32S || depth == CV_32U)
return result.u;
if (depth == CV_32F || is_fp16)
return result.f;
else
return result.i;
if (depth == CV_64U || depth == CV_64S)
return (double)result.UL;
}
else if( normType == NORM_L2 )
if( normType == NORM_L2 )
return std::sqrt(result.d);
return result.d;
@@ -1161,7 +1218,7 @@ double norm( InputArray _src1, InputArray _src2, int normType, InputArray _mask
return result;
}
NormDiffFunc func = getNormDiffFunc(normType >> 1, depth == CV_16F ? CV_32F : depth);
NormDiffFunc func = getNormDiffFunc(normType >> 1, depth);
CV_Assert( func != 0 );
const Mat* arrays[] = {&src1, &src2, &mask, 0};
@@ -1170,23 +1227,30 @@ double norm( InputArray _src1, InputArray _src2, int normType, InputArray _mask
{
double d;
float f;
int i;
unsigned u;
uint64 UL;
}
result;
result.d = 0;
NAryMatIterator it(arrays, ptrs);
CV_CheckLT((size_t)it.size, (size_t)INT_MAX, "");
if ((normType == NORM_L1 && depth <= CV_16S) ||
((normType == NORM_L2 || normType == NORM_L2SQR) && depth <= CV_8S))
bool is_fp16 = depth == CV_16F || depth == CV_16BF;
if ((normType == NORM_L1 && (depth <= CV_16S || is_fp16)) ||
((normType == NORM_L2 || normType == NORM_L2SQR) && (depth <= CV_8S || is_fp16)))
{
// special case to handle "integer" overflow in accumulator
const size_t esz = src1.elemSize();
const int total = (int)it.size;
const int intSumBlockSize = (normType == NORM_L1 && depth <= CV_8S ? (1 << 23) : (1 << 15))/cn;
const int blockSize = std::min(total, intSumBlockSize);
int isum = 0;
const int blockSize0 = (is_fp16 ? (1 << 10) :
normType == NORM_L1 && depth <= CV_8S ? (1 << 23) : (1 << 15))/cn;
const int blockSize = std::min(total, blockSize0);
union {
int i;
float f;
} blocksum;
blocksum.i = 0;
int count = 0;
for (size_t i = 0; i < it.nplanes; i++, ++it)
@@ -1194,12 +1258,12 @@ double norm( InputArray _src1, InputArray _src2, int normType, InputArray _mask
for (int j = 0; j < total; j += blockSize)
{
int bsz = std::min(total - j, blockSize);
func(ptrs[0], ptrs[1], ptrs[2], (uchar*)&isum, bsz, cn);
func(ptrs[0], ptrs[1], ptrs[2], &blocksum.i, bsz, cn);
count += bsz;
if (count + blockSize >= intSumBlockSize || (i+1 >= it.nplanes && j+bsz >= total))
if (count + blockSize >= blockSize0 || (i+1 >= it.nplanes && j+bsz >= total))
{
result.d += isum;
isum = 0;
result.d += is_fp16 ? (double)blocksum.f : (double)blocksum.i;
blocksum.i = 0;
count = 0;
}
ptrs[0] += bsz*esz;
@@ -1209,48 +1273,25 @@ double norm( InputArray _src1, InputArray _src2, int normType, InputArray _mask
}
}
}
else if (depth == CV_16F)
{
const size_t esz = src1.elemSize();
const int total = (int)it.size;
const int blockSize = std::min(total, divUp(512, cn));
AutoBuffer<float, 1026/*divUp(512,3)*3*2*/> fltbuf(blockSize * cn * 2);
float* data0 = fltbuf.data();
float* data1 = fltbuf.data() + blockSize * cn;
for (size_t i = 0; i < it.nplanes; i++, ++it)
{
for (int j = 0; j < total; j += blockSize)
{
int bsz = std::min(total - j, blockSize);
hal::cvt16f32f((const float16_t*)ptrs[0], data0, bsz * cn);
hal::cvt16f32f((const float16_t*)ptrs[1], data1, bsz * cn);
func((uchar*)data0, (uchar*)data1, ptrs[2], (uchar*)&result.f, bsz, cn);
ptrs[0] += bsz*esz;
ptrs[1] += bsz*esz;
if (ptrs[2])
ptrs[2] += bsz;
}
}
}
else
{
// generic implementation
for (size_t i = 0; i < it.nplanes; i++, ++it)
{
func(ptrs[0], ptrs[1], ptrs[2], (uchar*)&result, (int)it.size, cn);
func(ptrs[0], ptrs[1], ptrs[2], &result, (int)it.size, cn);
}
}
if( normType == NORM_INF )
{
if (depth == CV_64F)
return result.d;
else if (depth == CV_32F || depth == CV_16F)
return result.f;
else
if (depth <= CV_32S || depth == CV_32U)
return result.u;
if (depth == CV_32F || is_fp16)
return result.f;
if (depth == CV_64U || depth == CV_64S)
return (double)result.UL;
}
else if( normType == NORM_L2 )
if( normType == NORM_L2 )
return std::sqrt(result.d);
return result.d;
modules/core/src/rand.cpp
+2 -2
View File
@@ -271,7 +271,7 @@ randf_64f( double* arr, int len_, int cn, uint64* state, const Vec2d* p, void*,
typedef void (*RandFunc)(uchar* arr, int len, int cn, uint64* state,
const void* p, void* tempbuf, int flags);
static RandFunc randTab[][16] =
static RandFunc randTab[][CV_DEPTH_MAX] =
{
{
(RandFunc)randi_8u, (RandFunc)randi_8s, (RandFunc)randi_16u,
@@ -502,7 +502,7 @@ DEF_RANDNSCALE_FUNC(64f, double, double)
typedef void (*RandnScaleFunc)(float* src, void* dst, int len, int cn,
const void* mean, const void* stddev, int flags);
static RandnScaleFunc randnScaleTab[] =
static RandnScaleFunc randnScaleTab[CV_DEPTH_MAX] =
{
(RandnScaleFunc)randnScale_8u, (RandnScaleFunc)randnScale_8s, (RandnScaleFunc)randnScale_16u,
(RandnScaleFunc)randnScale_16s, (RandnScaleFunc)randnScale_32s, (RandnScaleFunc)randnScale_16_or_32f,
modules/core/src/sum.dispatch.cpp
+25 -13
View File
@@ -200,26 +200,30 @@ Scalar sum(InputArray _src)
int k, cn = src.channels(), depth = src.depth();
SumFunc func = getSumFunc(depth);
if (func == nullptr) {
if (depth == CV_Bool && cn == 1)
return Scalar((double)countNonZero(src));
CV_Error(Error::StsNotImplemented, "");
}
CV_Assert( cn <= 4 && func != 0 );
const Mat* arrays[] = {&src, 0};
uchar* ptrs[1] = {};
NAryMatIterator it(arrays, ptrs);
Scalar s;
int total = (int)it.size, blockSize = total, intSumBlockSize = 0;
int total = (int)it.size, blockSize = total, partialBlockSize = 0;
int j, count = 0;
AutoBuffer<int> _buf;
int _buf[CV_CN_MAX];
int* buf = (int*)&s[0];
size_t esz = 0;
bool blockSum = depth < CV_32S;
bool partialSumIsInt = depth < CV_32S;
bool blockSum = partialSumIsInt || depth == CV_16F || depth == CV_16BF;
if( blockSum )
{
intSumBlockSize = depth <= CV_8S ? (1 << 23) : (1 << 15);
blockSize = std::min(blockSize, intSumBlockSize);
_buf.allocate(cn);
buf = _buf.data();
partialBlockSize = depth <= CV_8S ? (1 << 23) : (1 << 15);
blockSize = std::min(blockSize, partialBlockSize);
buf = _buf;
for( k = 0; k < cn; k++ )
buf[k] = 0;
esz = src.elemSize();
@@ -232,12 +236,20 @@ Scalar sum(InputArray _src)
int bsz = std::min(total - j, blockSize);
func( ptrs[0], 0, (uchar*)buf, bsz, cn );
count += bsz;
if( blockSum && (count + blockSize >= intSumBlockSize || (i+1 >= it.nplanes && j+bsz >= total)) )
if( blockSum && (count + blockSize >= partialBlockSize || (i+1 >= it.nplanes && j+bsz >= total)) )
{
for( k = 0; k < cn; k++ )
{
s[k] += buf[k];
buf[k] = 0;
if (partialSumIsInt) {
for( k = 0; k < cn; k++ )
{
s[k] += buf[k];
buf[k] = 0;
}
} else {
for( k = 0; k < cn; k++ )
{
s[k] += ((float*)buf)[k];
buf[k] = 0;
}
}
count = 0;
}
modules/core/src/sum.simd.hpp
+240 -283
View File
@@ -16,7 +16,8 @@ SumFunc getSumFunc(int depth);
template <typename T, typename ST>
struct Sum_SIMD
{
int operator () (const T *, const uchar *, ST *, int, int) const
Sum_SIMD(int) {}
int operator () (const T*, const uchar*, ST*, int, int) const
{
return 0;
}
@@ -24,284 +25,216 @@ struct Sum_SIMD
#if (CV_SIMD || CV_SIMD_SCALABLE)
template <>
struct Sum_SIMD<uchar, int>
{
int operator () (const uchar * src0, const uchar * mask, int * dst, int len, int cn) const
{
if (mask || (cn != 1 && cn != 2 && cn != 4))
return 0;
len *= cn;
int x = 0;
v_uint32 v_sum = vx_setzero_u32();
int len0 = len & -VTraits<v_uint8>::vlanes();
while (x < len0)
{
const int len_tmp = min(x + 256*VTraits<v_uint16>::vlanes(), len0);
v_uint16 v_sum16 = vx_setzero_u16();
for (; x < len_tmp; x += VTraits<v_uint8>::vlanes())
{
v_uint16 v_src0, v_src1;
v_expand(vx_load(src0 + x), v_src0, v_src1);
v_sum16 = v_add(v_sum16, v_add(v_src0, v_src1));
}
v_uint32 v_half0, v_half1;
v_expand(v_sum16, v_half0, v_half1);
v_sum = v_add(v_sum, v_add(v_half0, v_half1));
}
if (x <= len - VTraits<v_uint16>::vlanes())
{
v_uint32 v_half0, v_half1;
v_expand(vx_load_expand(src0 + x), v_half0, v_half1);
v_sum = v_add(v_sum, v_add(v_half0, v_half1));
x += VTraits<v_uint16>::vlanes();
}
if (x <= len - VTraits<v_uint32>::vlanes())
{
v_sum = v_add(v_sum, vx_load_expand_q(src0 + x));
x += VTraits<v_uint32>::vlanes();
}
if (cn == 1)
*dst += v_reduce_sum(v_sum);
else
{
uint32_t CV_DECL_ALIGNED(CV_SIMD_WIDTH) ar[VTraits<v_uint32>::max_nlanes];
v_store_aligned(ar, v_sum);
for (int i = 0; i < VTraits<v_uint32>::vlanes(); ++i)
dst[i % cn] += ar[i];
}
v_cleanup();
return x / cn;
#undef REDUCE_PARTIAL_SUMS
#define REDUCE_PARTIAL_SUMS() \
if (cn == 1) \
dst[0] += v_reduce_sum(v_add(v_add(s0, s1), s2)); \
else if (cn == 2) { \
s0 = v_add(v_add(s0, s1), s2); \
dst[0] += v_reduce_sum(v_and(s0, m0)); \
dst[1] += v_reduce_sum(v_and(s0, m1)); \
} else if (cn == 3) { \
dst[0] += v_reduce_sum(v_add(v_add(v_and(s0, m0), v_and(s1, m1)), v_and(s2, m2))); \
dst[1] += v_reduce_sum(v_add(v_add(v_and(s0, m3), v_and(s1, m4)), v_and(s2, m5))); \
dst[2] += v_reduce_sum(v_add(v_add(v_and(s0, m6), v_and(s1, m7)), v_and(s2, m8))); \
} else if (cn == 4) { \
s0 = v_add(v_add(s0, s1), s2); \
dst[0] += v_reduce_sum(v_and(s0, m0)); \
dst[1] += v_reduce_sum(v_and(s0, m1)); \
dst[2] += v_reduce_sum(v_and(s0, m2)); \
dst[3] += v_reduce_sum(v_and(s0, m3)); \
}
template<typename ST>
static void init_maskbuf(ST* maskbuf, int cn, int simd_width)
{
memset(maskbuf, 0, simd_width*9*sizeof(maskbuf[0]));
if (cn == 1)
;
else if (cn == 2)
for (int i = 0; i < simd_width; i += 2) {
maskbuf[i] = (ST)-1;
maskbuf[i+1+simd_width] = (ST)-1;
}
else if (cn == 3)
for (int i = 0; i < simd_width*3; i += 3) {
maskbuf[i] = (ST)-1;
maskbuf[i+1+simd_width*3] = (ST)-1;
maskbuf[i+2+simd_width*6] = (ST)-1;
}
else if (cn == 4 && simd_width >= 4) {
for (int i = 0; i < simd_width; i += 4) {
maskbuf[i] = (ST)-1;
maskbuf[i+1+simd_width] = (ST)-1;
maskbuf[i+2+simd_width*2] = (ST)-1;
maskbuf[i+3+simd_width*3] = (ST)-1;
}
}
}
#undef DEFINE_SUM_SIMD_8
#define DEFINE_SUM_SIMD_8(T, ST, iST, VecT, load_op) \
template<> struct Sum_SIMD<T, ST> \
{ \
Sum_SIMD(int cn) \
{ \
init_maskbuf((iST*)maskbuf, cn, VTraits<VecT>::vlanes()); \
} \
int operator ()(const T* src, const uchar* mask, ST* dst, int len, int cn) const \
{ \
if (mask || (cn < 1 || cn > 4)) \
return 0; \
len *= cn; \
int x = 0, simd_width = VTraits<VecT>::vlanes(); \
VecT m0 = vx_load(maskbuf), m1, m2, m3, m4, m5, m6, m7, m8; \
if (cn == 1) { \
m1 = m2 = m3 = m4 = m5 = m6 = m7 = m8 = m0; \
} else { \
m1 = vx_load(maskbuf + simd_width); \
m2 = vx_load(maskbuf + simd_width*2); \
m3 = vx_load(maskbuf + simd_width*3); \
m4 = vx_load(maskbuf + simd_width*4); \
m5 = vx_load(maskbuf + simd_width*5); \
m6 = vx_load(maskbuf + simd_width*6); \
m7 = vx_load(maskbuf + simd_width*7); \
m8 = vx_load(maskbuf + simd_width*8); \
} \
VecT s0 = v_xor(m0, m0), s1 = s0, s2 = s0; \
for (; x <= len - simd_width*6; x += simd_width*6) { \
auto v0 = load_op(src + x); \
auto v1 = load_op(src + x + simd_width*2); \
auto v2 = load_op(src + x + simd_width*4); \
s0 = v_add(s0, v_expand_low(v0)); \
s1 = v_add(s1, v_expand_high(v0)); \
s2 = v_add(s2, v_expand_low(v1)); \
s0 = v_add(s0, v_expand_high(v1)); \
s1 = v_add(s1, v_expand_low(v2)); \
s2 = v_add(s2, v_expand_high(v2)); \
} \
REDUCE_PARTIAL_SUMS(); \
vx_cleanup(); \
return x / cn; \
} \
ST maskbuf[VTraits<VecT>::max_nlanes*9]; \
};
template <>
struct Sum_SIMD<schar, int>
{
int operator () (const schar * src0, const uchar * mask, int * dst, int len, int cn) const
{
if (mask || (cn != 1 && cn != 2 && cn != 4))
return 0;
len *= cn;
int x = 0;
v_int32 v_sum = vx_setzero_s32();
int len0 = len & -VTraits<v_int8>::vlanes();
while (x < len0)
{
const int len_tmp = min(x + 256*VTraits<v_int16>::vlanes(), len0);
v_int16 v_sum16 = vx_setzero_s16();
for (; x < len_tmp; x += VTraits<v_int8>::vlanes())
{
v_int16 v_src0, v_src1;
v_expand(vx_load(src0 + x), v_src0, v_src1);
v_sum16 = v_add(v_sum16, v_add(v_src0, v_src1));
}
v_int32 v_half0, v_half1;
v_expand(v_sum16, v_half0, v_half1);
v_sum = v_add(v_sum, v_add(v_half0, v_half1));
}
if (x <= len - VTraits<v_int16>::vlanes())
{
v_int32 v_half0, v_half1;
v_expand(vx_load_expand(src0 + x), v_half0, v_half1);
v_sum = v_add(v_sum, v_add(v_half0, v_half1));
x += VTraits<v_int16>::vlanes();
}
if (x <= len - VTraits<v_int32>::vlanes())
{
v_sum = v_add(v_sum, vx_load_expand_q(src0 + x));
x += VTraits<v_int32>::vlanes();
}
if (cn == 1)
*dst += v_reduce_sum(v_sum);
else
{
int32_t CV_DECL_ALIGNED(CV_SIMD_WIDTH) ar[VTraits<v_int32>::max_nlanes];
v_store_aligned(ar, v_sum);
for (int i = 0; i < VTraits<v_int32>::vlanes(); ++i)
dst[i % cn] += ar[i];
}
v_cleanup();
return x / cn;
}
#undef DEFINE_SUM_SIMD_16
#define DEFINE_SUM_SIMD_16(T, ST, iST, VecT, load_op) \
template<> struct Sum_SIMD<T, ST> \
{ \
Sum_SIMD(int cn) \
{ \
init_maskbuf((iST*)maskbuf, cn, VTraits<VecT>::vlanes()); \
} \
int operator ()(const T* src, const uchar* mask, ST* dst, int len, int cn) const \
{ \
if (mask || (cn < 1 || cn > 4)) \
return 0; \
len *= cn; \
int x = 0, simd_width = VTraits<VecT>::vlanes(); \
VecT m0 = vx_load(maskbuf), m1, m2, m3, m4, m5, m6, m7, m8; \
if (cn == 1) { \
m1 = m2 = m3 = m4 = m5 = m6 = m7 = m8 = m0; \
} else { \
m1 = vx_load(maskbuf + simd_width); \
m2 = vx_load(maskbuf + simd_width*2); \
m3 = vx_load(maskbuf + simd_width*3); \
m4 = vx_load(maskbuf + simd_width*4); \
m5 = vx_load(maskbuf + simd_width*5); \
m6 = vx_load(maskbuf + simd_width*6); \
m7 = vx_load(maskbuf + simd_width*7); \
m8 = vx_load(maskbuf + simd_width*8); \
} \
VecT s0 = v_xor(m0, m0), s1 = s0, s2 = s0; \
for (; x <= len - simd_width*3; x += simd_width*3) { \
auto v0 = load_op(src + x); \
auto v1 = load_op(src + x + simd_width); \
auto v2 = load_op(src + x + simd_width*2); \
s0 = v_add(s0, v0); \
s1 = v_add(s1, v1); \
s2 = v_add(s2, v2); \
} \
REDUCE_PARTIAL_SUMS(); \
vx_cleanup(); \
return x / cn; \
} \
ST maskbuf[VTraits<VecT>::max_nlanes*9]; \
};
template <>
struct Sum_SIMD<ushort, int>
{
int operator () (const ushort * src0, const uchar * mask, int * dst, int len, int cn) const
{
if (mask || (cn != 1 && cn != 2 && cn != 4))
return 0;
len *= cn;
#undef load_u8_as_s16
#undef load_u16_as_s32
#define load_u8_as_s16(addr) v_reinterpret_as_s16(vx_load_expand(addr))
#define load_u16_as_s32(addr) v_reinterpret_as_s32(vx_load_expand(addr))
int x = 0;
v_uint32 v_sum = vx_setzero_u32();
for (; x <= len - VTraits<v_uint16>::vlanes(); x += VTraits<v_uint16>::vlanes())
{
v_uint32 v_src0, v_src1;
v_expand(vx_load(src0 + x), v_src0, v_src1);
v_sum = v_add(v_sum, v_add(v_src0, v_src1));
}
if (x <= len - VTraits<v_uint32>::vlanes())
{
v_sum = v_add(v_sum, vx_load_expand(src0 + x));
x += VTraits<v_uint32>::vlanes();
}
if (cn == 1)
*dst += v_reduce_sum(v_sum);
else
{
uint32_t CV_DECL_ALIGNED(CV_SIMD_WIDTH) ar[VTraits<v_uint32>::max_nlanes];
v_store_aligned(ar, v_sum);
for (int i = 0; i < VTraits<v_uint32>::vlanes(); ++i)
dst[i % cn] += ar[i];
}
v_cleanup();
return x / cn;
}
};
template <>
struct Sum_SIMD<short, int>
{
int operator () (const short * src0, const uchar * mask, int * dst, int len, int cn) const
{
if (mask || (cn != 1 && cn != 2 && cn != 4))
return 0;
len *= cn;
int x = 0;
v_int32 v_sum = vx_setzero_s32();
for (; x <= len - VTraits<v_int16>::vlanes(); x += VTraits<v_int16>::vlanes())
{
v_int32 v_src0, v_src1;
v_expand(vx_load(src0 + x), v_src0, v_src1);
v_sum = v_add(v_sum, v_add(v_src0, v_src1));
}
if (x <= len - VTraits<v_int32>::vlanes())
{
v_sum = v_add(v_sum, vx_load_expand(src0 + x));
x += VTraits<v_int32>::vlanes();
}
if (cn == 1)
*dst += v_reduce_sum(v_sum);
else
{
int32_t CV_DECL_ALIGNED(CV_SIMD_WIDTH) ar[VTraits<v_int32>::max_nlanes];
v_store_aligned(ar, v_sum);
for (int i = 0; i < VTraits<v_int32>::vlanes(); ++i)
dst[i % cn] += ar[i];
}
v_cleanup();
return x / cn;
}
};
DEFINE_SUM_SIMD_8(uchar, int, int, v_int32, load_u8_as_s16)
DEFINE_SUM_SIMD_8(schar, int, int, v_int32, vx_load_expand)
DEFINE_SUM_SIMD_16(ushort, int, int, v_int32, load_u16_as_s32)
DEFINE_SUM_SIMD_16(short, int, int, v_int32, vx_load_expand)
DEFINE_SUM_SIMD_16(float16_t, float, int, v_float32, vx_load_expand)
DEFINE_SUM_SIMD_16(bfloat16_t, float, int, v_float32, vx_load_expand)
#if (CV_SIMD_64F || CV_SIMD_SCALABLE_64F)
template <>
struct Sum_SIMD<int, double>
{
int operator () (const int * src0, const uchar * mask, double * dst, int len, int cn) const
{
if (mask || (cn != 1 && cn != 2 && cn != 4))
return 0;
len *= cn;
int x = 0;
v_float64 v_sum0 = vx_setzero_f64();
v_float64 v_sum1 = vx_setzero_f64();
for (; x <= len - 2 * VTraits<v_int32>::vlanes(); x += 2 * VTraits<v_int32>::vlanes())
{
v_int32 v_src0 = vx_load(src0 + x);
v_int32 v_src1 = vx_load(src0 + x + VTraits<v_int32>::vlanes());
v_sum0 = v_add(v_sum0, v_add(v_cvt_f64(v_src0), v_cvt_f64(v_src1)));
v_sum1 = v_add(v_sum1, v_add(v_cvt_f64_high(v_src0), v_cvt_f64_high(v_src1)));
}
#if CV_SIMD256 || CV_SIMD512
double CV_DECL_ALIGNED(CV_SIMD_WIDTH) ar[VTraits<v_float64>::max_nlanes];
v_store_aligned(ar, v_add(v_sum0, v_sum1));
for (int i = 0; i < VTraits<v_float64>::vlanes(); ++i)
dst[i % cn] += ar[i];
#else
double CV_DECL_ALIGNED(CV_SIMD_WIDTH) ar[2 * VTraits<v_float64>::max_nlanes];
v_store_aligned(ar, v_sum0);
v_store_aligned(ar + VTraits<v_float64>::vlanes(), v_sum1);
for (int i = 0; i < 2 * VTraits<v_float64>::vlanes(); ++i)
dst[i % cn] += ar[i];
#endif
v_cleanup();
return x / cn;
}
#undef DEFINE_SUM_SIMD_32
#define DEFINE_SUM_SIMD_32(T, ST, iST, VecT) \
template<> struct Sum_SIMD<T, ST> \
{ \
Sum_SIMD(int cn) \
{ \
init_maskbuf((iST*)maskbuf, cn, VTraits<VecT>::vlanes()); \
} \
int operator ()(const T* src, const uchar* mask, ST* dst, int len, int cn) const \
{ \
int x = 0, simd_width = VTraits<VecT>::vlanes(); \
if (mask || (cn < 1 || cn > 3+(simd_width>=4))) \
return 0; \
len *= cn; \
VecT m0 = vx_load(maskbuf), m1, m2, m3, m4, m5, m6, m7, m8; \
if (cn == 1) { \
m1 = m2 = m3 = m4 = m5 = m6 = m7 = m8 = m0; \
} else { \
m1 = vx_load(maskbuf + simd_width); \
m2 = vx_load(maskbuf + simd_width*2); \
m3 = vx_load(maskbuf + simd_width*3); \
m4 = vx_load(maskbuf + simd_width*4); \
m5 = vx_load(maskbuf + simd_width*5); \
m6 = vx_load(maskbuf + simd_width*6); \
m7 = vx_load(maskbuf + simd_width*7); \
m8 = vx_load(maskbuf + simd_width*8); \
} \
VecT s0 = v_xor(m0, m0), s1 = s0, s2 = s0; \
for (; x <= len - simd_width*6; x += simd_width*6) { \
auto v0 = vx_load(src + x); \
auto v1 = vx_load(src + x + simd_width*2); \
auto v2 = vx_load(src + x + simd_width*4); \
s0 = v_add(s0, v_cvt_f64(v0)); \
s1 = v_add(s1, v_cvt_f64_high(v0)); \
s2 = v_add(s2, v_cvt_f64(v1)); \
s0 = v_add(s0, v_cvt_f64_high(v1)); \
s1 = v_add(s1, v_cvt_f64(v2)); \
s2 = v_add(s2, v_cvt_f64_high(v2)); \
} \
REDUCE_PARTIAL_SUMS(); \
vx_cleanup(); \
return x / cn; \
} \
ST maskbuf[VTraits<VecT>::max_nlanes*9]; \
};
template <>
struct Sum_SIMD<float, double>
{
int operator () (const float * src0, const uchar * mask, double * dst, int len, int cn) const
{
if (mask || (cn != 1 && cn != 2 && cn != 4))
return 0;
len *= cn;
int x = 0;
v_float64 v_sum0 = vx_setzero_f64();
v_float64 v_sum1 = vx_setzero_f64();
for (; x <= len - 2 * VTraits<v_float32>::vlanes(); x += 2 * VTraits<v_float32>::vlanes())
{
v_float32 v_src0 = vx_load(src0 + x);
v_float32 v_src1 = vx_load(src0 + x + VTraits<v_float32>::vlanes());
v_sum0 = v_add(v_sum0, v_add(v_cvt_f64(v_src0), v_cvt_f64(v_src1)));
v_sum1 = v_add(v_sum1, v_add(v_cvt_f64_high(v_src0), v_cvt_f64_high(v_src1)));
}
#if CV_SIMD256 || CV_SIMD512
double CV_DECL_ALIGNED(CV_SIMD_WIDTH) ar[VTraits<v_float64>::max_nlanes];
v_store_aligned(ar, v_add(v_sum0, v_sum1));
for (int i = 0; i < VTraits<v_float64>::vlanes(); ++i)
dst[i % cn] += ar[i];
#else
double CV_DECL_ALIGNED(CV_SIMD_WIDTH) ar[2 * VTraits<v_float64>::max_nlanes];
v_store_aligned(ar, v_sum0);
v_store_aligned(ar + VTraits<v_float64>::vlanes(), v_sum1);
for (int i = 0; i < 2 * VTraits<v_float64>::vlanes(); ++i)
dst[i % cn] += ar[i];
#endif
v_cleanup();
return x / cn;
}
};
DEFINE_SUM_SIMD_32(int, double, int64, v_float64)
DEFINE_SUM_SIMD_32(float, double, int64, v_float64)
#endif
#endif
template<typename T, typename ST>
template<typename T, typename ST, typename WT=T>
static int sum_(const T* src0, const uchar* mask, ST* dst, int len, int cn )
{
const T* src = src0;
if( !mask )
{
Sum_SIMD<T, ST> vop;
int i = vop(src0, mask, dst, len, cn), k = cn % 4;
src += i * cn;
Sum_SIMD<T, ST> vop(cn);
int i0 = vop(src0, mask, dst, len, cn), i = i0, k = cn % 4;
src += i0 * cn;
if( k == 1 )
{
@@ -309,10 +242,10 @@ static int sum_(const T* src0, const uchar* mask, ST* dst, int len, int cn )
#if CV_ENABLE_UNROLLED
for(; i <= len - 4; i += 4, src += cn*4 )
s0 += src[0] + src[cn] + src[cn*2] + src[cn*3];
s0 += (WT)src[0] + (WT)src[cn] + (WT)src[cn*2] + (WT)src[cn*3];
#endif
for( ; i < len; i++, src += cn )
s0 += src[0];
s0 += (WT)src[0];
dst[0] = s0;
}
else if( k == 2 )
@@ -320,8 +253,8 @@ static int sum_(const T* src0, const uchar* mask, ST* dst, int len, int cn )
ST s0 = dst[0], s1 = dst[1];
for( ; i < len; i++, src += cn )
{
s0 += src[0];
s1 += src[1];
s0 += (WT)src[0];
s1 += (WT)src[1];
}
dst[0] = s0;
dst[1] = s1;
@@ -331,9 +264,9 @@ static int sum_(const T* src0, const uchar* mask, ST* dst, int len, int cn )
ST s0 = dst[0], s1 = dst[1], s2 = dst[2];
for( ; i < len; i++, src += cn )
{
s0 += src[0];
s1 += src[1];
s2 += src[2];
s0 += (WT)src[0];
s1 += (WT)src[1];
s2 += (WT)src[2];
}
dst[0] = s0;
dst[1] = s1;
@@ -342,12 +275,12 @@ static int sum_(const T* src0, const uchar* mask, ST* dst, int len, int cn )
for( ; k < cn; k += 4 )
{
src = src0 + i*cn + k;
src = src0 + i0*cn + k;
ST s0 = dst[k], s1 = dst[k+1], s2 = dst[k+2], s3 = dst[k+3];
for( ; i < len; i++, src += cn )
for( i = i0; i < len; i++, src += cn )
{
s0 += src[0]; s1 += src[1];
s2 += src[2]; s3 += src[3];
s0 += (WT)src[0]; s1 += (WT)src[1];
s2 += (WT)src[2]; s3 += (WT)src[3];
}
dst[k] = s0;
dst[k+1] = s1;
@@ -364,7 +297,7 @@ static int sum_(const T* src0, const uchar* mask, ST* dst, int len, int cn )
for( i = 0; i < len; i++ )
if( mask[i] )
{
s += src[i];
s += (WT)src[i];
nzm++;
}
dst[0] = s;
@@ -375,9 +308,9 @@ static int sum_(const T* src0, const uchar* mask, ST* dst, int len, int cn )
for( i = 0; i < len; i++, src += 3 )
if( mask[i] )
{
s0 += src[0];
s1 += src[1];
s2 += src[2];
s0 += (WT)src[0];
s1 += (WT)src[1];
s2 += (WT)src[2];
nzm++;
}
dst[0] = s0;
@@ -394,16 +327,16 @@ static int sum_(const T* src0, const uchar* mask, ST* dst, int len, int cn )
for( ; k <= cn - 4; k += 4 )
{
ST s0, s1;
s0 = dst[k] + src[k];
s1 = dst[k+1] + src[k+1];
s0 = dst[k] + (WT)src[k];
s1 = dst[k+1] + (WT)src[k+1];
dst[k] = s0; dst[k+1] = s1;
s0 = dst[k+2] + src[k+2];
s1 = dst[k+3] + src[k+3];
s0 = dst[k+2] + (WT)src[k+2];
s1 = dst[k+3] + (WT)src[k+3];
dst[k+2] = s0; dst[k+3] = s1;
}
#endif
for( ; k < cn; k++ )
dst[k] += src[k];
dst[k] += (WT)src[k];
nzm++;
}
}
@@ -423,23 +356,47 @@ static int sum16u( const ushort* src, const uchar* mask, int* dst, int len, int
static int sum16s( const short* src, const uchar* mask, int* dst, int len, int cn )
{ CV_INSTRUMENT_REGION(); return sum_(src, mask, dst, len, cn); }
static int sum32u( const unsigned* src, const uchar* mask, double* dst, int len, int cn )
{ CV_INSTRUMENT_REGION(); return sum_(src, mask, dst, len, cn); }
static int sum32s( const int* src, const uchar* mask, double* dst, int len, int cn )
{ CV_INSTRUMENT_REGION(); return sum_(src, mask, dst, len, cn); }
static int sum64u( const uint64* src, const uchar* mask, double* dst, int len, int cn )
{ CV_INSTRUMENT_REGION(); return sum_(src, mask, dst, len, cn); }
static int sum64s( const int64* src, const uchar* mask, double* dst, int len, int cn )
{ CV_INSTRUMENT_REGION(); return sum_(src, mask, dst, len, cn); }
static int sum32f( const float* src, const uchar* mask, double* dst, int len, int cn )
{ CV_INSTRUMENT_REGION(); return sum_(src, mask, dst, len, cn); }
static int sum64f( const double* src, const uchar* mask, double* dst, int len, int cn )
{ CV_INSTRUMENT_REGION(); return sum_(src, mask, dst, len, cn); }
static int sum16f( const float16_t* src, const uchar* mask, float* dst, int len, int cn )
{ CV_INSTRUMENT_REGION(); return sum_<float16_t, float, float>(src, mask, dst, len, cn); }
static int sum16bf( const bfloat16_t* src, const uchar* mask, float* dst, int len, int cn )
{ CV_INSTRUMENT_REGION(); return sum_<bfloat16_t, float, float>(src, mask, dst, len, cn); }
SumFunc getSumFunc(int depth)
{
static SumFunc sumTab[CV_DEPTH_MAX] =
{
(SumFunc)GET_OPTIMIZED(sum8u), (SumFunc)sum8s,
(SumFunc)sum16u, (SumFunc)sum16s,
(SumFunc)GET_OPTIMIZED(sum8u),
(SumFunc)sum8s,
(SumFunc)sum16u,
(SumFunc)sum16s,
(SumFunc)sum32s,
(SumFunc)GET_OPTIMIZED(sum32f), (SumFunc)sum64f,
(SumFunc)GET_OPTIMIZED(sum32f),
(SumFunc)sum64f,
(SumFunc)sum16f,
(SumFunc)sum16bf,
0,
(SumFunc)sum64u,
(SumFunc)sum64s,
(SumFunc)sum32u,
0
};