potok/opencv
1
0
Fork 0
mirror of https://github.com/opencv/opencv.git synced 2026-07-31 08:13:04 +04:00
Code Packages Releases Wiki Activity

Merge branch 4.x

Browse Source
This commit is contained in:
Alexander Smorkalov
2023-09-11 13:22:22 +03:00
parent a6748df587 ee867ead1d
commit fdab565711
148 changed files with 2384 additions and 1876 deletions
Download Patch File Download Diff File Expand all files Collapse all files
modules/core/src/arithm.cpp
+26 -26
View File
@@ -1335,7 +1335,7 @@ struct InRange_SIMD
}
};
#if CV_SIMD
#if (CV_SIMD || CV_SIMD_SCALABLE)
template <>
struct InRange_SIMD<uchar>
@@ -1344,7 +1344,7 @@ struct InRange_SIMD<uchar>
uchar * dst, int len) const
{
int x = 0;
const int width = v_uint8::nlanes;
const int width = VTraits<v_uint8>::vlanes();
for (; x <= len - width; x += width)
{
@@ -1352,7 +1352,7 @@ struct InRange_SIMD<uchar>
v_uint8 low = vx_load(src2 + x);
v_uint8 high = vx_load(src3 + x);
v_store(dst + x, (values >= low) & (high >= values));
v_store(dst + x, v_and(v_ge(values, low), v_ge(high, values)));
}
vx_cleanup();
return x;
@@ -1366,7 +1366,7 @@ struct InRange_SIMD<schar>
uchar * dst, int len) const
{
int x = 0;
const int width = v_int8::nlanes;
const int width = VTraits<v_int8>::vlanes();
for (; x <= len - width; x += width)
{
@@ -1374,7 +1374,7 @@ struct InRange_SIMD<schar>
v_int8 low = vx_load(src2 + x);
v_int8 high = vx_load(src3 + x);
v_store((schar*)(dst + x), (values >= low) & (high >= values));
v_store((schar*)(dst + x), v_and(v_ge(values, low), v_ge(high, values)));
}
vx_cleanup();
return x;
@@ -1388,7 +1388,7 @@ struct InRange_SIMD<ushort>
uchar * dst, int len) const
{
int x = 0;
const int width = v_uint16::nlanes * 2;
const int width = VTraits<v_uint16>::vlanes() * 2;
for (; x <= len - width; x += width)
{
@@ -1396,11 +1396,11 @@ struct InRange_SIMD<ushort>
v_uint16 low1 = vx_load(src2 + x);
v_uint16 high1 = vx_load(src3 + x);
v_uint16 values2 = vx_load(src1 + x + v_uint16::nlanes);
v_uint16 low2 = vx_load(src2 + x + v_uint16::nlanes);
v_uint16 high2 = vx_load(src3 + x + v_uint16::nlanes);
v_uint16 values2 = vx_load(src1 + x + VTraits<v_uint16>::vlanes());
v_uint16 low2 = vx_load(src2 + x + VTraits<v_uint16>::vlanes());
v_uint16 high2 = vx_load(src3 + x + VTraits<v_uint16>::vlanes());
v_store(dst + x, v_pack((values1 >= low1) & (high1 >= values1), (values2 >= low2) & (high2 >= values2)));
v_store(dst + x, v_pack(v_and(v_ge(values1, low1), v_ge(high1, values1)), v_and(v_ge(values2, low2), v_ge(high2, values2))));
}
vx_cleanup();
return x;
@@ -1414,7 +1414,7 @@ struct InRange_SIMD<short>
uchar * dst, int len) const
{
int x = 0;
const int width = (int)v_int16::nlanes * 2;
const int width = (int)VTraits<v_int16>::vlanes() * 2;
for (; x <= len - width; x += width)
{
@@ -1422,11 +1422,11 @@ struct InRange_SIMD<short>
v_int16 low1 = vx_load(src2 + x);
v_int16 high1 = vx_load(src3 + x);
v_int16 values2 = vx_load(src1 + x + v_int16::nlanes);
v_int16 low2 = vx_load(src2 + x + v_int16::nlanes);
v_int16 high2 = vx_load(src3 + x + v_int16::nlanes);
v_int16 values2 = vx_load(src1 + x + VTraits<v_int16>::vlanes());
v_int16 low2 = vx_load(src2 + x + VTraits<v_int16>::vlanes());
v_int16 high2 = vx_load(src3 + x + VTraits<v_int16>::vlanes());
v_store((schar*)(dst + x), v_pack((values1 >= low1) & (high1 >= values1), (values2 >= low2) & (high2 >= values2)));
v_store((schar*)(dst + x), v_pack(v_and(v_ge(values1, low1), v_ge(high1, values1)), v_and(v_ge(values2, low2), v_ge(high2, values2))));
}
vx_cleanup();
return x;
@@ -1440,7 +1440,7 @@ struct InRange_SIMD<int>
uchar * dst, int len) const
{
int x = 0;
const int width = (int)v_int32::nlanes * 2;
const int width = (int)VTraits<v_int32>::vlanes() * 2;
for (; x <= len - width; x += width)
{
@@ -1448,11 +1448,11 @@ struct InRange_SIMD<int>
v_int32 low1 = vx_load(src2 + x);
v_int32 high1 = vx_load(src3 + x);
v_int32 values2 = vx_load(src1 + x + v_int32::nlanes);
v_int32 low2 = vx_load(src2 + x + v_int32::nlanes);
v_int32 high2 = vx_load(src3 + x + v_int32::nlanes);
v_int32 values2 = vx_load(src1 + x + VTraits<v_int32>::vlanes());
v_int32 low2 = vx_load(src2 + x + VTraits<v_int32>::vlanes());
v_int32 high2 = vx_load(src3 + x + VTraits<v_int32>::vlanes());
v_pack_store(dst + x, v_reinterpret_as_u16(v_pack((values1 >= low1) & (high1 >= values1), (values2 >= low2) & (high2 >= values2))));
v_pack_store(dst + x, v_reinterpret_as_u16(v_pack(v_and(v_ge(values1, low1), v_ge(high1, values1)), v_and(v_ge(values2, low2), v_ge(high2, values2)))));
}
vx_cleanup();
return x;
@@ -1466,7 +1466,7 @@ struct InRange_SIMD<float>
uchar * dst, int len) const
{
int x = 0;
const int width = (int)v_float32::nlanes * 2;
const int width = (int)VTraits<v_float32>::vlanes() * 2;
for (; x <= len - width; x += width)
{
@@ -1474,12 +1474,12 @@ struct InRange_SIMD<float>
v_float32 low1 = vx_load(src2 + x);
v_float32 high1 = vx_load(src3 + x);
v_float32 values2 = vx_load(src1 + x + v_float32::nlanes);
v_float32 low2 = vx_load(src2 + x + v_float32::nlanes);
v_float32 high2 = vx_load(src3 + x + v_float32::nlanes);
v_float32 values2 = vx_load(src1 + x + VTraits<v_float32>::vlanes());
v_float32 low2 = vx_load(src2 + x + VTraits<v_float32>::vlanes());
v_float32 high2 = vx_load(src3 + x + VTraits<v_float32>::vlanes());
v_pack_store(dst + x, v_pack(v_reinterpret_as_u32(values1 >= low1) & v_reinterpret_as_u32(high1 >= values1),
v_reinterpret_as_u32(values2 >= low2) & v_reinterpret_as_u32(high2 >= values2)));
v_pack_store(dst + x, v_pack(v_and(v_reinterpret_as_u32(v_ge(values1, low1)), v_reinterpret_as_u32(v_ge(high1, values1))),
v_and(v_reinterpret_as_u32(v_ge(values2, low2)), v_reinterpret_as_u32(v_ge(high2, values2)))));
}
vx_cleanup();
return x;
modules/core/src/arithm.simd.hpp
+60 -59
View File
@@ -215,7 +215,7 @@ template<typename T1, typename Tvec>
struct op_add
{
static inline Tvec r(const Tvec& a, const Tvec& b)
{ return a + b; }
{ return v_add(a, b); }
static inline T1 r(T1 a, T1 b)
{ return c_add(a, b); }
};
@@ -225,7 +225,7 @@ template<typename T1, typename Tvec>
struct op_sub
{
static inline Tvec r(const Tvec& a, const Tvec& b)
{ return a - b; }
{ return v_sub(a, b); }
static inline T1 r(T1 a, T1 b)
{ return c_sub(a, b); }
};
@@ -262,7 +262,7 @@ struct op_absdiff
template<>
struct op_absdiff<schar, v_int8>
{
#if CV_SIMD
#if CV_SIMD || CV_SIMD_SCALABLE
static inline v_int8 r(const v_int8& a, const v_int8& b)
{ return v_absdiffs(a, b); }
#endif
@@ -272,7 +272,7 @@ struct op_absdiff<schar, v_int8>
template<>
struct op_absdiff<short, v_int16>
{
#if CV_SIMD
#if CV_SIMD || CV_SIMD_SCALABLE
static inline v_int16 r(const v_int16& a, const v_int16& b)
{ return v_absdiffs(a, b); }
#endif
@@ -282,7 +282,7 @@ struct op_absdiff<short, v_int16>
template<>
struct op_absdiff<int, v_int32>
{
#if CV_SIMD
#if CV_SIMD || CV_SIMD_SCALABLE
static inline v_int32 r(const v_int32& a, const v_int32& b)
{ return v_reinterpret_as_s32(v_absdiff(a, b)); }
#endif
@@ -295,7 +295,7 @@ template<typename T1, typename Tvec>
struct op_or
{
static inline Tvec r(const Tvec& a, const Tvec& b)
{ return a | b; }
{ return v_or(a, b); }
static inline T1 r(T1 a, T1 b)
{ return a | b; }
};
@@ -303,7 +303,7 @@ template<typename T1, typename Tvec>
struct op_xor
{
static inline Tvec r(const Tvec& a, const Tvec& b)
{ return a ^ b; }
{ return v_xor(a, b); }
static inline T1 r(T1 a, T1 b)
{ return a ^ b; }
};
@@ -311,7 +311,7 @@ template<typename T1, typename Tvec>
struct op_and
{
static inline Tvec r(const Tvec& a, const Tvec& b)
{ return a & b; }
{ return v_and(a, b); }
static inline T1 r(T1 a, T1 b)
{ return a & b; }
};
@@ -320,14 +320,14 @@ struct op_not
{
// ignored b from loader level
static inline Tvec r(const Tvec& a)
{ return ~a; }
{ return v_not(a); }
static inline T1 r(T1 a, T1)
{ return ~a; }
};
//////////////////////////// Loaders /////////////////////////////////
#if CV_SIMD
#if CV_SIMD || CV_SIMD_SCALABLE
template< template<typename T1, typename Tvec> class OP, typename T1, typename Tvec>
struct bin_loader
@@ -392,13 +392,13 @@ template<template<typename T1, typename Tvec> class OP, typename T1, typename Tv
static void bin_loop(const T1* src1, size_t step1, const T1* src2, size_t step2, T1* dst, size_t step, int width, int height)
{
typedef OP<T1, Tvec> op;
#if CV_SIMD
#if CV_SIMD || CV_SIMD_SCALABLE
typedef bin_loader<OP, T1, Tvec> ldr;
enum {wide_step = Tvec::nlanes};
const int wide_step = VTraits<Tvec>::vlanes();
#if !CV_NEON && CV_SIMD_WIDTH == 16
enum {wide_step_l = wide_step * 2};
const int wide_step_l = wide_step * 2;
#else
enum {wide_step_l = wide_step};
const int wide_step_l = wide_step;
#endif
#endif // CV_SIMD
@@ -410,7 +410,7 @@ static void bin_loop(const T1* src1, size_t step1, const T1* src2, size_t step2,
{
int x = 0;
#if CV_SIMD
#if CV_SIMD || CV_SIMD_SCALABLE
#if !CV_NEON && !CV_MSA
if (is_aligned(src1, src2, dst))
{
@@ -583,7 +583,7 @@ template<typename T1, typename Tvec>
struct op_cmplt
{
static inline Tvec r(const Tvec& a, const Tvec& b)
{ return a < b; }
{ return v_lt(a, b); }
static inline uchar r(T1 a, T1 b)
{ return (uchar)-(int)(a < b); }
};
@@ -592,7 +592,7 @@ template<typename T1, typename Tvec>
struct op_cmple
{
static inline Tvec r(const Tvec& a, const Tvec& b)
{ return a <= b; }
{ return v_le(a, b); }
static inline uchar r(T1 a, T1 b)
{ return (uchar)-(int)(a <= b); }
};
@@ -601,7 +601,7 @@ template<typename T1, typename Tvec>
struct op_cmpeq
{
static inline Tvec r(const Tvec& a, const Tvec& b)
{ return a == b; }
{ return v_eq(a, b); }
static inline uchar r(T1 a, T1 b)
{ return (uchar)-(int)(a == b); }
};
@@ -610,14 +610,14 @@ template<typename T1, typename Tvec>
struct op_cmpne
{
static inline Tvec r(const Tvec& a, const Tvec& b)
{ return a != b; }
{ return v_ne(a, b); }
static inline uchar r(T1 a, T1 b)
{ return (uchar)-(int)(a != b); }
};
//////////////////////////// Loaders /////////////////////////////////
#if CV_SIMD
#if CV_SIMD || CV_SIMD_SCALABLE
// todo: add support for RW alignment & stream
template<int nload, template<typename T1, typename Tvec> class OP, typename T1, typename Tvec>
struct cmp_loader_n
@@ -642,10 +642,10 @@ template<template<typename T1, typename Tvec> class OP, typename T1, typename Tv
struct cmp_loader_n<sizeof(ushort), OP, T1, Tvec>
{
typedef OP<T1, Tvec> op;
enum {step = Tvec::nlanes};
static inline void l(const T1* src1, const T1* src2, uchar* dst)
{
const int step = VTraits<Tvec>::vlanes();
Tvec c0 = op::r(vx_load(src1), vx_load(src2));
Tvec c1 = op::r(vx_load(src1 + step), vx_load(src2 + step));
v_store(dst, v_pack_b(v_reinterpret_as_u16(c0), v_reinterpret_as_u16(c1)));
@@ -656,10 +656,10 @@ template<template<typename T1, typename Tvec> class OP, typename T1, typename Tv
struct cmp_loader_n<sizeof(unsigned), OP, T1, Tvec>
{
typedef OP<T1, Tvec> op;
enum {step = Tvec::nlanes};
static inline void l(const T1* src1, const T1* src2, uchar* dst)
{
const int step = VTraits<Tvec>::vlanes();
v_uint32 c0 = v_reinterpret_as_u32(op::r(vx_load(src1), vx_load(src2)));
v_uint32 c1 = v_reinterpret_as_u32(op::r(vx_load(src1 + step), vx_load(src2 + step)));
v_uint32 c2 = v_reinterpret_as_u32(op::r(vx_load(src1 + step * 2), vx_load(src2 + step * 2)));
@@ -672,10 +672,10 @@ template<template<typename T1, typename Tvec> class OP, typename T1, typename Tv
struct cmp_loader_n<sizeof(double), OP, T1, Tvec>
{
typedef OP<T1, Tvec> op;
enum {step = Tvec::nlanes};
static inline void l(const T1* src1, const T1* src2, uchar* dst)
{
const int step = VTraits<Tvec>::vlanes();
v_uint64 c0 = v_reinterpret_as_u64(op::r(vx_load(src1), vx_load(src2)));
v_uint64 c1 = v_reinterpret_as_u64(op::r(vx_load(src1 + step), vx_load(src2 + step)));
v_uint64 c2 = v_reinterpret_as_u64(op::r(vx_load(src1 + step * 2), vx_load(src2 + step * 2)));
@@ -697,9 +697,9 @@ template<template<typename T1, typename Tvec> class OP, typename T1, typename Tv
static void cmp_loop(const T1* src1, size_t step1, const T1* src2, size_t step2, uchar* dst, size_t step, int width, int height)
{
typedef OP<T1, Tvec> op;
#if CV_SIMD
#if CV_SIMD || CV_SIMD_SCALABLE
typedef cmp_loader_n<sizeof(T1), OP, T1, Tvec> ldr;
enum {wide_step = Tvec::nlanes * sizeof(T1)};
const int wide_step = VTraits<Tvec>::vlanes() * sizeof(T1);
#endif // CV_SIMD
step1 /= sizeof(T1);
@@ -709,7 +709,7 @@ static void cmp_loop(const T1* src1, size_t step1, const T1* src2, size_t step2,
{
int x = 0;
#if CV_SIMD
#if CV_SIMD || CV_SIMD_SCALABLE
for (; x <= width - wide_step; x += wide_step)
{
ldr::l(src1 + x, src2 + x, dst + x);
@@ -876,7 +876,7 @@ DEFINE_SIMD_ALL(cmp)
//////////////////////////// Loaders ///////////////////////////////
#if CV_SIMD
#if CV_SIMD || CV_SIMD_SCALABLE
// todo: add support for RW alignment & stream
template<int nload, template<typename T1, typename T2, typename Tvec> class OP, typename T1, typename T2, typename Tvec>
struct scalar_loader_n
@@ -1009,10 +1009,10 @@ template<template<typename T1, typename T2, typename Tvec> class OP, typename T2
struct scalar_loader_n<sizeof(int), OP, int, T2, v_int32>
{
typedef OP<int, T2, v_int32> op;
enum {step = v_int32::nlanes};
static inline void l(const int* src1, const int* src2, const T2* scalar, int* dst)
{
const int step = VTraits<v_int32>::vlanes();
v_int32 v_src1 = vx_load(src1);
v_int32 v_src2 = vx_load(src2);
v_int32 v_src1s = vx_load(src1 + step);
@@ -1039,6 +1039,7 @@ struct scalar_loader_n<sizeof(int), OP, int, T2, v_int32>
static inline void l(const int* src1, const T2* scalar, int* dst)
{
const int step = VTraits<v_int32>::vlanes();
v_int32 v_src1 = vx_load(src1);
v_int32 v_src1s = vx_load(src1 + step);
@@ -1064,10 +1065,9 @@ template<template<typename T1, typename T2, typename Tvec> class OP, typename T2
struct scalar_loader_n<sizeof(float), OP, float, T2, v_float32>
{
typedef OP<float, T2, v_float32> op;
enum {step = v_float32::nlanes};
static inline void l(const float* src1, const float* src2, const T2* scalar, float* dst)
{
const int step = VTraits<v_float32>::vlanes();
v_float32 v_src1 = vx_load(src1);
v_float32 v_src2 = vx_load(src2);
v_float32 v_src1s = vx_load(src1 + step);
@@ -1082,6 +1082,7 @@ struct scalar_loader_n<sizeof(float), OP, float, T2, v_float32>
static inline void l(const float* src1, const T2* scalar, float* dst)
{
const int step = VTraits<v_float32>::vlanes();
v_float32 v_src1 = vx_load(src1);
v_float32 v_src1s = vx_load(src1 + step);
@@ -1258,10 +1259,10 @@ static void scalar_loop(const T1* src1, size_t step1, const T1* src2, size_t ste
T1* dst, size_t step, int width, int height, const T2* scalar)
{
typedef OP<T1, T2, Tvec> op;
#if CV_SIMD
#if CV_SIMD || CV_SIMD_SCALABLE
typedef scalar_loader_n<sizeof(T1), OP, T1, T2, Tvec> ldr;
const int wide_step = sizeof(T1) > sizeof(ushort) ? Tvec::nlanes * 2 :
sizeof(T1) == sizeof(uchar) ? Tvec::nlanes / 2 : Tvec::nlanes;
const int wide_step = sizeof(T1) > sizeof(ushort) ? VTraits<Tvec>::vlanes() * 2 :
sizeof(T1) == sizeof(uchar) ? VTraits<Tvec>::vlanes() / 2 : VTraits<Tvec>::vlanes();
#endif // CV_SIMD
step1 /= sizeof(T1);
@@ -1272,7 +1273,7 @@ static void scalar_loop(const T1* src1, size_t step1, const T1* src2, size_t ste
{
int x = 0;
#if CV_SIMD
#if CV_SIMD || CV_SIMD_SCALABLE
for (; x <= width - wide_step; x += wide_step)
{
ldr::l(src1 + x, src2 + x, scalar, dst + x);
@@ -1304,10 +1305,10 @@ template<template<typename T1, typename T2, typename Tvec> class OP, typename T1
static void scalar_loop(const T1* src1, size_t step1, T1* dst, size_t step, int width, int height, const T2* scalar)
{
typedef OP<T1, T2, Tvec> op;
#if CV_SIMD
#if CV_SIMD || CV_SIMD_SCALABLE
typedef scalar_loader_n<sizeof(T1), OP, T1, T2, Tvec> ldr;
const int wide_step = sizeof(T1) > sizeof(ushort) ? Tvec::nlanes * 2 :
sizeof(T1) == sizeof(uchar) ? Tvec::nlanes / 2 : Tvec::nlanes;
const int wide_step = sizeof(T1) > sizeof(ushort) ? VTraits<Tvec>::vlanes() * 2 :
sizeof(T1) == sizeof(uchar) ? VTraits<Tvec>::vlanes() / 2 : VTraits<Tvec>::vlanes();
#endif // CV_SIMD
step1 /= sizeof(T1);
@@ -1317,7 +1318,7 @@ static void scalar_loop(const T1* src1, size_t step1, T1* dst, size_t step, int
{
int x = 0;
#if CV_SIMD
#if CV_SIMD || CV_SIMD_SCALABLE
for (; x <= width - wide_step; x += wide_step)
{
ldr::l(src1 + x, scalar, dst + x);
@@ -1424,7 +1425,7 @@ template<typename T1, typename Tvec>
struct op_mul
{
static inline Tvec r(const Tvec& a, const Tvec& b)
{ return a * b; }
{ return v_mul(a, b); }
static inline T1 r(T1 a, T1 b)
{ return saturate_cast<T1>(a * b); }
};
@@ -1432,11 +1433,11 @@ struct op_mul
template<typename T1, typename T2, typename Tvec>
struct op_mul_scale
{
#if CV_SIMD
#if CV_SIMD || CV_SIMD_SCALABLE
static inline v_float32 r(const v_float32& a, const v_float32& b, const T2* scalar)
{
const v_float32 v_scalar = vx_setall_f32(*scalar);
return v_scalar * a * b;
return v_mul(v_scalar , a , b);
}
#endif
static inline T1 r(T1 a, T1 b, const T2* scalar)
@@ -1452,7 +1453,7 @@ struct op_mul_scale<double, double, v_float64>
static inline v_float64 r(const v_float64& a, const v_float64& b, const double* scalar)
{
const v_float64 v_scalar = vx_setall_f64(*scalar);
return v_scalar * a * b;
return v_mul(v_mul(v_scalar, a), b);
}
#endif
static inline double r(double a, double b, const double* scalar)
@@ -1565,7 +1566,7 @@ template<typename T1, typename Tvec>
struct op_div_f
{
static inline Tvec r(const Tvec& a, const Tvec& b)
{ return a / b; }
{ return v_div(a, b); }
static inline T1 r(T1 a, T1 b)
{ return a / b; }
};
@@ -1573,16 +1574,16 @@ struct op_div_f
template<typename T1, typename T2, typename Tvec>
struct op_div_scale
{
#if CV_SIMD
#if CV_SIMD || CV_SIMD_SCALABLE
static inline v_float32 r(const v_float32& a, const v_float32& b, const T2* scalar)
{
const v_float32 v_scalar = vx_setall_f32(*scalar);
return a * v_scalar / b;
return v_div(v_mul(a, v_scalar), b);
}
static inline Tvec pre(const Tvec& denom, const Tvec& res)
{
const Tvec v_zero = vx_setall<typename Tvec::lane_type>(0);
return v_select(denom == v_zero, v_zero, res);
const Tvec v_zero = vx_setall<typename VTraits<Tvec>::lane_type>(0);
return v_select(v_eq(denom, v_zero), v_zero, res);
}
#endif
static inline T1 r(T1 a, T1 denom, const T2* scalar)
@@ -1595,11 +1596,11 @@ struct op_div_scale
template<>
struct op_div_scale<float, float, v_float32>
{
#if CV_SIMD
#if CV_SIMD || CV_SIMD_SCALABLE
static inline v_float32 r(const v_float32& a, const v_float32& b, const float* scalar)
{
const v_float32 v_scalar = vx_setall_f32(*scalar);
return a * v_scalar / b;
return v_div(v_mul(a, v_scalar), b);
}
#endif
static inline float r(float a, float denom, const float* scalar)
@@ -1613,7 +1614,7 @@ struct op_div_scale<double, double, v_float64>
static inline v_float64 r(const v_float64& a, const v_float64& b, const double* scalar)
{
const v_float64 v_scalar = vx_setall_f64(*scalar);
return a * v_scalar / b;
return v_div(v_mul(a, v_scalar), b);
}
#endif
static inline double r(double a, double denom, const double* scalar)
@@ -1681,7 +1682,7 @@ DEFINE_SIMD_ALL(div, div_loop)
template<typename T1, typename T2, typename Tvec>
struct op_add_scale
{
#if CV_SIMD
#if CV_SIMD || CV_SIMD_SCALABLE
static inline v_float32 r(const v_float32& a, const v_float32& b, const T2* scalar)
{
const v_float32 v_alpha = vx_setall_f32(*scalar);
@@ -1714,7 +1715,7 @@ struct op_add_scale<double, double, v_float64>
template<typename T1, typename T2, typename Tvec>
struct op_add_weighted
{
#if CV_SIMD
#if CV_SIMD || CV_SIMD_SCALABLE
static inline v_float32 r(const v_float32& a, const v_float32& b, const T2* scalars)
{
const v_float32 v_alpha = vx_setall_f32(scalars[0]);
@@ -1831,16 +1832,16 @@ DEFINE_SIMD_F64(addWeighted, add_weighted_loop_d)
template<typename T1, typename T2, typename Tvec>
struct op_recip
{
#if CV_SIMD
#if CV_SIMD || CV_SIMD_SCALABLE
static inline v_float32 r(const v_float32& a, const T2* scalar)
{
const v_float32 v_scalar = vx_setall_f32(*scalar);
return v_scalar / a;
return v_div(v_scalar, a);
}
static inline Tvec pre(const Tvec& denom, const Tvec& res)
{
const Tvec v_zero = vx_setall<typename Tvec::lane_type>(0);
return v_select(denom == v_zero, v_zero, res);
const Tvec v_zero = vx_setall<typename VTraits<Tvec>::lane_type>(0);
return v_select(v_eq(denom, v_zero), v_zero, res);
}
#endif
static inline T1 r(T1 denom, const T2* scalar)
@@ -1853,11 +1854,11 @@ struct op_recip
template<>
struct op_recip<float, float, v_float32>
{
#if CV_SIMD
#if CV_SIMD || CV_SIMD_SCALABLE
static inline v_float32 r(const v_float32& a, const float* scalar)
{
const v_float32 v_scalar = vx_setall_f32(*scalar);
return v_scalar / a;
return v_div(v_scalar, a);
}
#endif
static inline float r(float denom, const float* scalar)
@@ -1871,7 +1872,7 @@ struct op_recip<double, double, v_float64>
static inline v_float64 r(const v_float64& a, const double* scalar)
{
const v_float64 v_scalar = vx_setall_f64(*scalar);
return v_scalar / a;
return v_div(v_scalar, a);
}
#endif
static inline double r(double denom, const double* scalar)
modules/core/src/check.cpp
+2
View File
@@ -4,6 +4,8 @@
#include "precomp.hpp"
#include <sstream>
#include "opencv2/core/check.hpp"
namespace cv {
modules/core/src/convert.hpp
+58 -58
View File
@@ -11,7 +11,7 @@
namespace cv
{
#if CV_SIMD
#if (CV_SIMD || CV_SIMD_SCALABLE)
static inline void vx_load_as(const uchar* ptr, v_float32& a)
{ a = v_cvt_f32(v_reinterpret_as_s32(vx_load_expand_q(ptr))); }
@@ -78,7 +78,7 @@ static inline void v_store_as(int64_t* ptr, const v_float32& a)
v_int64 ia_0, ia_1;
v_expand(ia, ia_0, ia_1);
v_store(ptr, ia_0);
v_store(ptr + v_int64::nlanes, ia_1);
v_store(ptr + VTraits<v_uint64>::vlanes(), ia_1);
}
static inline void v_store_as(uint64_t* ptr, const v_float32& a)
@@ -88,7 +88,7 @@ static inline void v_store_as(uint64_t* ptr, const v_float32& a)
ia = v_max(ia, vx_setzero_s32());
v_expand(v_reinterpret_as_u32(ia), ia_0, ia_1);
v_store(ptr, ia_0);
v_store(ptr + v_int64::nlanes, ia_1);
v_store(ptr + VTraits<v_uint64>::vlanes(), ia_1);
}
static inline void vx_load_pair_as(const uchar* ptr, v_uint16& a, v_uint16& b)
@@ -104,7 +104,7 @@ static inline void vx_load_pair_as(const schar* ptr, v_uint16& a, v_uint16& b)
}
static inline void vx_load_pair_as(const ushort* ptr, v_uint16& a, v_uint16& b)
{ a = vx_load(ptr); b = vx_load(ptr + v_uint16::nlanes); }
{ a = vx_load(ptr); b = vx_load(ptr + VTraits<v_uint16>::vlanes()); }
static inline void vx_load_pair_as(const uchar* ptr, v_int16& a, v_int16& b)
{
@@ -118,7 +118,7 @@ static inline void vx_load_pair_as(const schar* ptr, v_int16& a, v_int16& b)
{ v_expand(vx_load(ptr), a, b); }
static inline void vx_load_pair_as(const short* ptr, v_int16& a, v_int16& b)
{ a = vx_load(ptr); b = vx_load(ptr + v_uint16::nlanes); }
{ a = vx_load(ptr); b = vx_load(ptr + VTraits<v_uint16>::vlanes()); }
static inline void vx_load_pair_as(const uchar* ptr, v_int32& a, v_int32& b)
{
@@ -147,7 +147,7 @@ static inline void vx_load_pair_as(const short* ptr, v_int32& a, v_int32& b)
static inline void vx_load_pair_as(const int* ptr, v_int32& a, v_int32& b)
{
a = vx_load(ptr);
b = vx_load(ptr + v_int32::nlanes);
b = vx_load(ptr + VTraits<v_int32>::vlanes());
}
static inline void vx_load_pair_as(const uchar* ptr, v_float32& a, v_float32& b)
@@ -184,14 +184,14 @@ static inline void vx_load_pair_as(const short* ptr, v_float32& a, v_float32& b)
static inline void vx_load_pair_as(const int* ptr, v_float32& a, v_float32& b)
{
v_int32 ia = vx_load(ptr), ib = vx_load(ptr + v_int32::nlanes);
v_int32 ia = vx_load(ptr), ib = vx_load(ptr + VTraits<v_int32>::vlanes());
a = v_cvt_f32(ia);
b = v_cvt_f32(ib);
}
static inline void vx_load_pair_as(const int64_t* ptr, v_int32& a, v_int32& b)
{
const int int64_nlanes = v_int64::nlanes;
const int int64_nlanes = VTraits<v_uint64>::vlanes();
a = v_pack(vx_load(ptr), vx_load(ptr + int64_nlanes));
b = v_pack(vx_load(ptr + int64_nlanes*2), vx_load(ptr + int64_nlanes*3));
}
@@ -199,7 +199,7 @@ static inline void vx_load_pair_as(const int64_t* ptr, v_int32& a, v_int32& b)
static inline void vx_load_pair_as(const int64_t* ptr, v_uint64& a, v_uint64& b)
{
v_int64 z = vx_setzero_s64();
v_int64 ia = vx_load(ptr), ib = vx_load(ptr + v_int64::nlanes);
v_int64 ia = vx_load(ptr), ib = vx_load(ptr + VTraits<v_uint64>::vlanes());
ia &= (ia > z);
ib &= (ib > z);
a = v_reinterpret_as_u64(ia);
@@ -208,7 +208,7 @@ static inline void vx_load_pair_as(const int64_t* ptr, v_uint64& a, v_uint64& b)
static inline void vx_load_pair_as(const int64_t* ptr, v_uint32& a, v_uint32& b)
{
const int nlanes = v_int64::nlanes;
const int nlanes = VTraits<v_uint64>::vlanes();
v_int64 z = vx_setzero_s64();
v_int64 ia0 = vx_load(ptr), ia1 = vx_load(ptr + nlanes);
v_int64 ib0 = vx_load(ptr + nlanes*2), ib1 = vx_load(ptr + nlanes*3);
@@ -222,8 +222,8 @@ static inline void vx_load_pair_as(const int64_t* ptr, v_uint32& a, v_uint32& b)
static inline void vx_load_pair_as(const uint64_t* ptr, v_float32& a, v_float32& b)
{
const int nlanes = v_uint64::nlanes;
float buf[v_uint64::nlanes*4];
const int nlanes = VTraits<v_uint64>::vlanes();
float buf[VTraits<v_uint64>::max_nlanes*4];
for (int i = 0; i < nlanes*4; i++) {
buf[i] = (float)ptr[i];
}
@@ -233,8 +233,8 @@ static inline void vx_load_pair_as(const uint64_t* ptr, v_float32& a, v_float32&
static inline void vx_load_pair_as(const int64_t* ptr, v_float32& a, v_float32& b)
{
const int nlanes = v_int64::nlanes;
float buf[v_int64::nlanes*4];
const int nlanes = VTraits<v_uint64>::vlanes();
float buf[VTraits<v_uint64>::max_nlanes*4];
for (int i = 0; i < nlanes*4; i++) {
buf[i] = (float)ptr[i];
}
@@ -277,21 +277,21 @@ static inline void vx_load_pair_as(const int* ptr, v_uint32& a, v_uint32& b)
{
v_int32 z = vx_setzero_s32();
v_int32 ia = v_max(vx_load(ptr), z);
v_int32 ib = v_max(vx_load(ptr + v_int32::nlanes), z);
v_int32 ib = v_max(vx_load(ptr + VTraits<v_int32>::vlanes()), z);
a = v_reinterpret_as_u32(ia);
b = v_reinterpret_as_u32(ib);
}
static inline void vx_load_pair_as(const uint64_t* ptr, v_uint32& a, v_uint32& b)
{
const int int64_nlanes = v_int64::nlanes;
const int int64_nlanes = VTraits<v_uint64>::vlanes();
a = v_pack(vx_load(ptr), vx_load(ptr + int64_nlanes));
b = v_pack(vx_load(ptr + int64_nlanes*2), vx_load(ptr + int64_nlanes*3));
}
static inline void vx_load_pair_as(const uint64_t* ptr, v_int32& a, v_int32& b)
{
const int int64_nlanes = v_int64::nlanes;
const int int64_nlanes = VTraits<v_uint64>::vlanes();
v_uint32 ua = v_pack(vx_load(ptr), vx_load(ptr + int64_nlanes));
v_uint32 ub = v_pack(vx_load(ptr + int64_nlanes*2), vx_load(ptr + int64_nlanes*3));
a = v_reinterpret_as_s32(ua);
@@ -299,37 +299,37 @@ static inline void vx_load_pair_as(const uint64_t* ptr, v_int32& a, v_int32& b)
}
static inline void vx_load_pair_as(const float* ptr, v_float32& a, v_float32& b)
{ a = vx_load(ptr); b = vx_load(ptr + v_float32::nlanes); }
{ a = vx_load(ptr); b = vx_load(ptr + VTraits<v_float32>::vlanes()); }
static inline void vx_load_pair_as(const float16_t* ptr, v_float32& a, v_float32& b)
{
a = vx_load_expand(ptr);
b = vx_load_expand(ptr + v_float32::nlanes);
b = vx_load_expand(ptr + VTraits<v_float32>::vlanes());
}
static inline void vx_load_pair_as(const bfloat16_t* ptr, v_float32& a, v_float32& b)
{
a = vx_load_expand(ptr);
b = vx_load_expand(ptr + v_float32::nlanes);
b = vx_load_expand(ptr + VTraits<v_float32>::vlanes());
}
static inline void vx_load_pair_as(const unsigned* ptr, v_uint32& a, v_uint32& b)
{
a = vx_load(ptr);
b = vx_load(ptr + v_uint32::nlanes);
b = vx_load(ptr + VTraits<v_uint32>::vlanes());
}
static inline void vx_load_pair_as(const unsigned* ptr, v_int32& a, v_int32& b)
{
a = v_reinterpret_as_s32(vx_load(ptr));
b = v_reinterpret_as_s32(vx_load(ptr + v_uint32::nlanes));
b = v_reinterpret_as_s32(vx_load(ptr + VTraits<v_uint32>::vlanes()));
}
static inline void vx_load_pair_as(const unsigned* ptr, v_float32& a, v_float32& b)
{
v_uint32 delta = vx_setall_u32(0x80000000U);
v_uint32 ua = vx_load(ptr);
v_uint32 ub = vx_load(ptr + v_uint32::nlanes);
v_uint32 ub = vx_load(ptr + VTraits<v_uint32>::vlanes());
v_uint32 mask_a = (ua >= delta) & delta, mask_b = (ub >= delta) & delta;
v_float32 fmask_a = v_cvt_f32(v_reinterpret_as_s32(mask_a)); // 0.f or (float)(-(1 << 31))
v_float32 fmask_b = v_cvt_f32(v_reinterpret_as_s32(mask_b)); // 0.f or (float)(-(1 << 31))
@@ -353,7 +353,7 @@ static inline void v_store_pair_as(schar* ptr, const v_uint16& a, const v_uint16
}
static inline void v_store_pair_as(ushort* ptr, const v_uint16& a, const v_uint16& b)
{ v_store(ptr, a); v_store(ptr + v_uint16::nlanes, b); }
{ v_store(ptr, a); v_store(ptr + VTraits<v_uint16>::vlanes(), b); }
static inline void v_store_pair_as(uchar* ptr, const v_int16& a, const v_int16& b)
{ v_store(ptr, v_pack_u(a, b)); }
@@ -362,7 +362,7 @@ static inline void v_store_pair_as(schar* ptr, const v_int16& a, const v_int16&
{ v_store(ptr, v_pack(a, b)); }
static inline void v_store_pair_as(short* ptr, const v_int16& a, const v_int16& b)
{ v_store(ptr, a); v_store(ptr + v_int16::nlanes, b); }
{ v_store(ptr, a); v_store(ptr + VTraits<v_int16>::vlanes(), b); }
static inline void v_store_pair_as(uchar* ptr, const v_int32& a, const v_int32& b)
{ v_pack_u_store(ptr, v_pack(a, b)); }
@@ -379,7 +379,7 @@ static inline void v_store_pair_as(short* ptr, const v_int32& a, const v_int32&
static inline void v_store_pair_as(int* ptr, const v_int32& a, const v_int32& b)
{
v_store(ptr, a);
v_store(ptr + v_int32::nlanes, b);
v_store(ptr + VTraits<v_int32>::vlanes(), b);
}
static inline void v_store_pair_as(int64_t* ptr, const v_int32& a, const v_int32& b)
@@ -387,7 +387,7 @@ static inline void v_store_pair_as(int64_t* ptr, const v_int32& a, const v_int32
v_int64 q0, q1, q2, q3;
v_expand(a, q0, q1);
v_expand(b, q2, q3);
const int nlanes = v_int64::nlanes;
const int nlanes = VTraits<v_uint64>::vlanes();
v_store(ptr, q0);
v_store(ptr + nlanes, q1);
v_store(ptr + nlanes*2, q2);
@@ -419,11 +419,11 @@ static inline void v_store_pair_as(int* ptr, const v_float32& a, const v_float32
{
v_int32 ia = v_round(a), ib = v_round(b);
v_store(ptr, ia);
v_store(ptr + v_int32::nlanes, ib);
v_store(ptr + VTraits<v_int32>::vlanes(), ib);
}
static inline void v_store_pair_as(float* ptr, const v_float32& a, const v_float32& b)
{ v_store(ptr, a); v_store(ptr + v_float32::nlanes, b); }
{ v_store(ptr, a); v_store(ptr + VTraits<v_float32>::vlanes(), b); }
static inline void v_store_pair_as(unsigned* ptr, const v_float32& a, const v_float32& b)
{
@@ -431,7 +431,7 @@ static inline void v_store_pair_as(unsigned* ptr, const v_float32& a, const v_fl
v_int32 ia = v_max(v_round(a), z);
v_int32 ib = v_max(v_round(b), z);
v_store(ptr, v_reinterpret_as_u32(ia));
v_store(ptr + v_int32::nlanes, v_reinterpret_as_u32(ib));
v_store(ptr + VTraits<v_int32>::vlanes(), v_reinterpret_as_u32(ib));
}
static inline void v_store_pair_as(uchar* ptr, const v_uint32& a, const v_uint32& b)
@@ -447,7 +447,7 @@ static inline void v_store_pair_as(ushort* ptr, const v_uint32& a, const v_uint3
static inline void v_store_pair_as(unsigned* ptr, const v_uint32& a, const v_uint32& b)
{
v_store(ptr, a);
v_store(ptr + v_uint32::nlanes, b);
v_store(ptr + VTraits<v_uint32>::vlanes(), b);
}
static inline void v_store_pair_as(uint64_t* ptr, const v_uint32& a, const v_uint32& b)
@@ -455,7 +455,7 @@ static inline void v_store_pair_as(uint64_t* ptr, const v_uint32& a, const v_uin
v_uint64 q0, q1, q2, q3;
v_expand(a, q0, q1);
v_expand(b, q2, q3);
const int nlanes = v_uint64::nlanes;
const int nlanes = VTraits<v_uint64>::vlanes();
v_store(ptr, q0);
v_store(ptr + nlanes, q1);
v_store(ptr + nlanes*2, q2);
@@ -465,28 +465,28 @@ static inline void v_store_pair_as(uint64_t* ptr, const v_uint32& a, const v_uin
static inline void v_store_pair_as(uint64_t* ptr, const v_uint64& a, const v_uint64& b)
{
v_store(ptr, a);
v_store(ptr + v_uint64::nlanes, b);
v_store(ptr + VTraits<v_uint64>::vlanes(), b);
}
#if CV_SIMD_64F
#if (CV_SIMD_64F || CV_SIMD_SCALABLE_64F)
static inline void vx_load_as(const uint64_t* ptr, v_float32& a)
{
v_float64 a_0 = v_cvt_f64(v_reinterpret_as_s64(vx_load(ptr)));
v_float64 a_1 = v_cvt_f64(v_reinterpret_as_s64(vx_load(ptr + v_uint64::nlanes)));
v_float64 a_1 = v_cvt_f64(v_reinterpret_as_s64(vx_load(ptr + VTraits<v_uint64>::vlanes())));
a = v_cvt_f32(a_0, a_1);
}
static inline void vx_load_as(const int64_t* ptr, v_float32& a)
{
v_float64 a_0 = v_cvt_f64(vx_load(ptr));
v_float64 a_1 = v_cvt_f64(vx_load(ptr + v_uint64::nlanes));
v_float64 a_1 = v_cvt_f64(vx_load(ptr + VTraits<v_uint64>::vlanes()));
a = v_cvt_f32(a_0, a_1);
}
static inline void vx_load_as(const double* ptr, v_float32& a)
{
v_float64 v0 = vx_load(ptr), v1 = vx_load(ptr + v_float64::nlanes);
v_float64 v0 = vx_load(ptr), v1 = vx_load(ptr + VTraits<v_float64>::vlanes());
a = v_cvt_f32(v0, v1);
}
@@ -516,8 +516,8 @@ static inline void vx_load_pair_as(const bfloat16_t* ptr, v_float64& a, v_float6
static inline void vx_load_pair_as(const double* ptr, v_int32& a, v_int32& b)
{
v_float64 v0 = vx_load(ptr), v1 = vx_load(ptr + v_float64::nlanes);
v_float64 v2 = vx_load(ptr + v_float64::nlanes*2), v3 = vx_load(ptr + v_float64::nlanes*3);
v_float64 v0 = vx_load(ptr), v1 = vx_load(ptr + VTraits<v_float64>::vlanes());
v_float64 v2 = vx_load(ptr + VTraits<v_float64>::vlanes()*2), v3 = vx_load(ptr + VTraits<v_float64>::vlanes()*3);
v_int32 iv0 = v_round(v0), iv1 = v_round(v1);
v_int32 iv2 = v_round(v2), iv3 = v_round(v3);
a = v_combine_low(iv0, iv1);
@@ -526,15 +526,15 @@ static inline void vx_load_pair_as(const double* ptr, v_int32& a, v_int32& b)
static inline void vx_load_pair_as(const uint64_t* ptr, v_float64& a, v_float64& b)
{
const int int64_nlanes = v_int64::nlanes;
const int int64_nlanes = VTraits<v_uint64>::vlanes();
a = v_cvt_f64(v_reinterpret_as_s64(vx_load(ptr)));
b = v_cvt_f64(v_reinterpret_as_s64(vx_load(ptr + int64_nlanes)));
}
static inline void vx_load_pair_as(const double* ptr, v_float32& a, v_float32& b)
{
v_float64 v0 = vx_load(ptr), v1 = vx_load(ptr + v_float64::nlanes);
v_float64 v2 = vx_load(ptr + v_float64::nlanes*2), v3 = vx_load(ptr + v_float64::nlanes*3);
v_float64 v0 = vx_load(ptr), v1 = vx_load(ptr + VTraits<v_float64>::vlanes());
v_float64 v2 = vx_load(ptr + VTraits<v_float64>::vlanes()*2), v3 = vx_load(ptr + VTraits<v_float64>::vlanes()*3);
a = v_cvt_f32(v0, v1);
b = v_cvt_f32(v2, v3);
}
@@ -584,19 +584,19 @@ static inline void vx_load_pair_as(const float* ptr, v_float64& a, v_float64& b)
static inline void vx_load_pair_as(const double* ptr, v_float64& a, v_float64& b)
{
a = vx_load(ptr);
b = vx_load(ptr + v_float64::nlanes);
b = vx_load(ptr + VTraits<v_float64>::vlanes());
}
static inline void vx_load_pair_as(const int64_t* ptr, v_float64& a, v_float64& b)
{
a = v_cvt_f64(vx_load(ptr));
b = v_cvt_f64(vx_load(ptr + v_float64::nlanes));
b = v_cvt_f64(vx_load(ptr + VTraits<v_float64>::vlanes()));
}
static inline void vx_load_pair_as(const unsigned* ptr, v_float64& a, v_float64& b)
{
const int nlanes = v_uint64::nlanes;
double buf[v_uint64::nlanes*2];
const int nlanes = VTraits<v_uint64>::vlanes();
double buf[VTraits<v_uint64>::max_nlanes*2];
for (int i = 0; i < nlanes*2; i++)
buf[i] = (double)ptr[i];
a = vx_load(buf);
@@ -607,7 +607,7 @@ static inline void v_store_as(double* ptr, const v_float32& a)
{
v_float64 fa0 = v_cvt_f64(a), fa1 = v_cvt_f64_high(a);
v_store(ptr, fa0);
v_store(ptr + v_float64::nlanes, fa1);
v_store(ptr + VTraits<v_float64>::vlanes(), fa1);
}
static inline void v_store_pair_as(double* ptr, const v_int32& a, const v_int32& b)
@@ -616,9 +616,9 @@ static inline void v_store_pair_as(double* ptr, const v_int32& a, const v_int32&
v_float64 fb0 = v_cvt_f64(b), fb1 = v_cvt_f64_high(b);
v_store(ptr, fa0);
v_store(ptr + v_float64::nlanes, fa1);
v_store(ptr + v_float64::nlanes*2, fb0);
v_store(ptr + v_float64::nlanes*3, fb1);
v_store(ptr + VTraits<v_float64>::vlanes(), fa1);
v_store(ptr + VTraits<v_float64>::vlanes()*2, fb0);
v_store(ptr + VTraits<v_float64>::vlanes()*3, fb1);
}
static inline void v_store_pair_as(double* ptr, const v_float32& a, const v_float32& b)
@@ -627,15 +627,15 @@ static inline void v_store_pair_as(double* ptr, const v_float32& a, const v_floa
v_float64 fb0 = v_cvt_f64(b), fb1 = v_cvt_f64_high(b);
v_store(ptr, fa0);
v_store(ptr + v_float64::nlanes, fa1);
v_store(ptr + v_float64::nlanes*2, fb0);
v_store(ptr + v_float64::nlanes*3, fb1);
v_store(ptr + VTraits<v_float64>::vlanes(), fa1);
v_store(ptr + VTraits<v_float64>::vlanes()*2, fb0);
v_store(ptr + VTraits<v_float64>::vlanes()*3, fb1);
}
static inline void v_store_pair_as(double* ptr, const v_float64& a, const v_float64& b)
{
v_store(ptr, a);
v_store(ptr + v_float64::nlanes, b);
v_store(ptr + VTraits<v_float64>::vlanes(), b);
}
static inline void v_store_pair_as(int* ptr, const v_float64& a, const v_float64& b)
@@ -662,7 +662,7 @@ static inline void v_store_pair_as(uint64_t* ptr, const v_float64& a, const v_fl
v_int64 ia, ib;
v_expand(v_round(v_max(a, z), v_max(b, z)), ia, ib);
v_store(ptr, v_reinterpret_as_u64(ia));
v_store(ptr + v_int64::nlanes, v_reinterpret_as_u64(ib));
v_store(ptr + VTraits<v_uint64>::vlanes(), v_reinterpret_as_u64(ib));
}
static inline void v_store_pair_as(int64_t* ptr, const v_float64& a, const v_float64& b)
@@ -670,7 +670,7 @@ static inline void v_store_pair_as(int64_t* ptr, const v_float64& a, const v_flo
v_int64 ia, ib;
v_expand(v_round(a, b), ia, ib);
v_store(ptr, ia);
v_store(ptr + v_int64::nlanes, ib);
v_store(ptr + VTraits<v_uint64>::vlanes(), ib);
}
static inline void v_store_pair_as(unsigned* ptr, const v_float64& a, const v_float64& b)
@@ -744,9 +744,9 @@ static inline void v_store_pair_as(double* ptr, const _Tsvec& a, const _Tsvec& b
ptr[i] = (double)buf[i];
}
#endif /////////// CV_SIMD_64F
#endif /////////// CV_SIMD_64F || CV_SIMD_SCALABLE_64F
#endif /////////// CV_SIMD
#endif /////////// CV_SIMD || CV_SIMD_SCALABLE
}
modules/core/src/convert.simd.hpp
+12 -12
View File
@@ -41,8 +41,8 @@ void cvt16f32f( const float16_t* src, float* dst, int len )
{
CV_INSTRUMENT_REGION();
int j = 0;
#if CV_SIMD
const int VECSZ = v_float32::nlanes;
#if (CV_SIMD || CV_SIMD_SCALABLE)
const int VECSZ = VTraits<v_float32>::vlanes();
for( ; j < len; j += VECSZ )
{
if( j > len - VECSZ )
@@ -62,8 +62,8 @@ void cvt32f16f( const float* src, float16_t* dst, int len )
{
CV_INSTRUMENT_REGION();
int j = 0;
#if CV_SIMD
const int VECSZ = v_float32::nlanes;
#if (CV_SIMD || CV_SIMD_SCALABLE)
const int VECSZ = VTraits<v_float32>::vlanes();
for( ; j < len; j += VECSZ )
{
if( j > len - VECSZ )
@@ -83,8 +83,8 @@ void cvt32f16bf( const float* src, bfloat16_t* dst, int len )
{
CV_INSTRUMENT_REGION();
int j = 0;
#if CV_SIMD
const int VECSZ = v_float32::nlanes;
#if (CV_SIMD || CV_SIMD_SCALABLE)
const int VECSZ = VTraits<v_float32>::vlanes();
for( ; j < len; j += VECSZ )
{
if( j > len - VECSZ )
@@ -153,8 +153,8 @@ cvt_( const _Ts* src, size_t sstep, _Td* dst, size_t dstep, Size size )
for( int i = 0; i < size.height; i++, src += sstep, dst += dstep )
{
int j = 0;
#if CV_SIMD
const int VECSZ = _Twvec::nlanes*2;
#if (CV_SIMD || CV_SIMD_SCALABLE)
const int VECSZ = VTraits<_Twvec>::vlanes()*2;
for( ; j < size.width; j += VECSZ )
{
if( j > size.width - VECSZ )
@@ -182,8 +182,8 @@ cvt_64f( const _Ts* src, size_t sstep, _Td* dst, size_t dstep, Size size )
for( int i = 0; i < size.height; i++, src += sstep, dst += dstep )
{
int j = 0;
#if CV_SIMD_64F
const int VECSZ = v_float64::nlanes*2;
#if (CV_SIMD_64F || CV_SIMD_SCALABLE_64F)
const int VECSZ = VTraits<v_float64>::vlanes()*2;
for( ; j < size.width; j += VECSZ )
{
if( j > size.width - VECSZ )
@@ -213,8 +213,8 @@ cvt1_( const _Ts* src, size_t sstep, _Td* dst, size_t dstep, Size size )
for( int i = 0; i < size.height; i++, src += sstep, dst += dstep )
{
int j = 0;
#if CV_SIMD
const int VECSZ = _Twvec::nlanes;
#if (CV_SIMD || CV_SIMD_SCALABLE)
const int VECSZ = VTraits<_Twvec>::vlanes();
for( ; j < size.width; j += VECSZ )
{
if( j > size.width - VECSZ )
modules/core/src/convert_scale.simd.hpp
+12 -12
View File
@@ -22,9 +22,9 @@ template<typename _Ts, typename _Td> inline void
cvtabs_32f( const _Ts* src, size_t sstep, _Td* dst, size_t dstep,
Size size, float a, float b )
{
#if CV_SIMD
#if (CV_SIMD || CV_SIMD_SCALABLE)
v_float32 va = vx_setall_f32(a), vb = vx_setall_f32(b);
const int VECSZ = v_float32::nlanes*2;
const int VECSZ = VTraits<v_float32>::vlanes()*2;
#endif
sstep /= sizeof(src[0]);
dstep /= sizeof(dst[0]);
@@ -32,7 +32,7 @@ cvtabs_32f( const _Ts* src, size_t sstep, _Td* dst, size_t dstep,
for( int i = 0; i < size.height; i++, src += sstep, dst += dstep )
{
int j = 0;
#if CV_SIMD
#if (CV_SIMD || CV_SIMD_SCALABLE)
for( ; j < size.width; j += VECSZ )
{
if( j > size.width - VECSZ )
@@ -72,9 +72,9 @@ template<typename _Ts, typename _Td> inline void
cvt_32f( const _Ts* src, size_t sstep, _Td* dst, size_t dstep,
Size size, float a, float b )
{
#if CV_SIMD
#if (CV_SIMD || CV_SIMD_SCALABLE)
v_float32 va = vx_setall_f32(a), vb = vx_setall_f32(b);
const int VECSZ = v_float32::nlanes*2;
const int VECSZ = VTraits<v_float32>::vlanes()*2;
#endif
sstep /= sizeof(src[0]);
dstep /= sizeof(dst[0]);
@@ -82,7 +82,7 @@ cvt_32f( const _Ts* src, size_t sstep, _Td* dst, size_t dstep,
for( int i = 0; i < size.height; i++, src += sstep, dst += dstep )
{
int j = 0;
#if CV_SIMD
#if (CV_SIMD || CV_SIMD_SCALABLE)
for( ; j < size.width; j += VECSZ )
{
if( j > size.width - VECSZ )
@@ -108,9 +108,9 @@ template<typename _Ts, typename _Td> inline void
cvt1_32f( const _Ts* src, size_t sstep, _Td* dst, size_t dstep,
Size size, float a, float b )
{
#if CV_SIMD
#if (CV_SIMD || CV_SIMD_SCALABLE)
v_float32 va = vx_setall_f32(a), vb = vx_setall_f32(b);
const int VECSZ = v_float32::nlanes;
const int VECSZ = VTraits<v_float32>::vlanes();
#endif
sstep /= sizeof(src[0]);
dstep /= sizeof(dst[0]);
@@ -118,7 +118,7 @@ cvt1_32f( const _Ts* src, size_t sstep, _Td* dst, size_t dstep,
for( int i = 0; i < size.height; i++, src += sstep, dst += dstep )
{
int j = 0;
#if CV_SIMD
#if (CV_SIMD || CV_SIMD_SCALABLE)
for( ; j < size.width; j += VECSZ )
{
if( j > size.width - VECSZ )
@@ -143,9 +143,9 @@ template<typename _Ts, typename _Td> inline void
cvt_64f( const _Ts* src, size_t sstep, _Td* dst, size_t dstep,
Size size, double a, double b )
{
#if CV_SIMD_64F
#if (CV_SIMD_64F || CV_SIMD_SCALABLE_64F)
v_float64 va = vx_setall_f64(a), vb = vx_setall_f64(b);
const int VECSZ = v_float64::nlanes*2;
const int VECSZ = VTraits<v_float64>::vlanes()*2;
#endif
sstep /= sizeof(src[0]);
dstep /= sizeof(dst[0]);
@@ -153,7 +153,7 @@ cvt_64f( const _Ts* src, size_t sstep, _Td* dst, size_t dstep,
for( int i = 0; i < size.height; i++, src += sstep, dst += dstep )
{
int j = 0;
#if CV_SIMD_64F
#if (CV_SIMD_64F || CV_SIMD_SCALABLE_64F)
for( ; j < size.width; j += VECSZ )
{
if( j > size.width - VECSZ )
modules/core/src/copy.cpp
+9 -9
View File
@@ -171,15 +171,15 @@ copyMask_<uchar>(const uchar* _src, size_t sstep, const uchar* mask, size_t mste
const uchar* src = (const uchar*)_src;
uchar* dst = (uchar*)_dst;
int x = 0;
#if CV_SIMD
#if (CV_SIMD || CV_SIMD_SCALABLE)
{
v_uint8 v_zero = vx_setzero_u8();
for( ; x <= size.width - v_uint8::nlanes; x += v_uint8::nlanes )
for( ; x <= size.width - VTraits<v_uint8>::vlanes(); x += VTraits<v_uint8>::vlanes() )
{
v_uint8 v_src = vx_load(src + x),
v_dst = vx_load(dst + x),
v_nmask = vx_load(mask + x) == v_zero;
v_nmask = v_eq(vx_load(mask + x), v_zero);
v_dst = v_select(v_nmask, v_dst, v_src);
v_store(dst + x, v_dst);
@@ -203,23 +203,23 @@ copyMask_<ushort>(const uchar* _src, size_t sstep, const uchar* mask, size_t mst
const ushort* src = (const ushort*)_src;
ushort* dst = (ushort*)_dst;
int x = 0;
#if CV_SIMD
#if (CV_SIMD || CV_SIMD_SCALABLE)
{
v_uint8 v_zero = vx_setzero_u8();
for( ; x <= size.width - v_uint8::nlanes; x += v_uint8::nlanes )
for( ; x <= size.width - VTraits<v_uint8>::vlanes(); x += VTraits<v_uint8>::vlanes() )
{
v_uint16 v_src1 = vx_load(src + x), v_src2 = vx_load(src + x + v_uint16::nlanes),
v_dst1 = vx_load(dst + x), v_dst2 = vx_load(dst + x + v_uint16::nlanes);
v_uint16 v_src1 = vx_load(src + x), v_src2 = vx_load(src + x + VTraits<v_uint16>::vlanes()),
v_dst1 = vx_load(dst + x), v_dst2 = vx_load(dst + x + VTraits<v_uint16>::vlanes());
v_uint8 v_nmask1, v_nmask2;
v_uint8 v_nmask = vx_load(mask + x) == v_zero;
v_uint8 v_nmask = v_eq(vx_load(mask + x), v_zero);
v_zip(v_nmask, v_nmask, v_nmask1, v_nmask2);
v_dst1 = v_select(v_reinterpret_as_u16(v_nmask1), v_dst1, v_src1);
v_dst2 = v_select(v_reinterpret_as_u16(v_nmask2), v_dst2, v_src2);
v_store(dst + x, v_dst1);
v_store(dst + x + v_uint16::nlanes, v_dst2);
v_store(dst + x + VTraits<v_uint16>::vlanes(), v_dst2);
}
}
vx_cleanup();
modules/core/src/count_non_zero.simd.hpp
+33 -39
View File
@@ -32,8 +32,8 @@ static int countNonZero_(const T* src, int len )
static int countNonZero8u( const uchar* src, int len )
{
int i=0, nz = 0;
#if CV_SIMD
int len0 = len & -v_uint8::nlanes;
#if (CV_SIMD || CV_SIMD_SCALABLE)
int len0 = len & -VTraits<v_uint8>::vlanes();
v_uint8 v_zero = vx_setzero_u8();
v_uint8 v_one = vx_setall_u8(1);
@@ -42,20 +42,20 @@ static int countNonZero8u( const uchar* src, int len )
{
v_uint16 v_sum16 = vx_setzero_u16();
int j = i;
while (j < std::min(len0, i + 65280 * v_uint16::nlanes))
while (j < std::min(len0, i + 65280 * VTraits<v_uint16>::vlanes()))
{
v_uint8 v_sum8 = vx_setzero_u8();
int k = j;
for (; k < std::min(len0, j + 255 * v_uint8::nlanes); k += v_uint8::nlanes)
v_sum8 += v_one & (vx_load(src + k) == v_zero);
for (; k < std::min(len0, j + 255 * VTraits<v_uint8>::vlanes()); k += VTraits<v_uint8>::vlanes())
v_sum8 = v_add(v_sum8, v_and(v_one, v_eq(vx_load(src + k), v_zero)));
v_uint16 part1, part2;
v_expand(v_sum8, part1, part2);
v_sum16 += part1 + part2;
v_sum16 = v_add(v_sum16, v_add(part1, part2));
j = k;
}
v_uint32 part1, part2;
v_expand(v_sum16, part1, part2);
v_sum32 += part1 + part2;
v_sum32 = v_add(v_sum32, v_add(part1, part2));
i = j;
}
nz = i - v_reduce_sum(v_sum32);
@@ -69,8 +69,8 @@ static int countNonZero8u( const uchar* src, int len )
static int countNonZero16u( const ushort* src, int len )
{
int i = 0, nz = 0;
#if CV_SIMD
int len0 = len & -v_int8::nlanes;
#if (CV_SIMD || CV_SIMD_SCALABLE)
int len0 = len & -VTraits<v_int8>::vlanes();
v_uint16 v_zero = vx_setzero_u16();
v_int8 v_one = vx_setall_s8(1);
@@ -79,20 +79,20 @@ static int countNonZero16u( const ushort* src, int len )
{
v_int16 v_sum16 = vx_setzero_s16();
int j = i;
while (j < std::min(len0, i + 32766 * v_int16::nlanes))
while (j < std::min(len0, i + 32766 * VTraits<v_int16>::vlanes()))
{
v_int8 v_sum8 = vx_setzero_s8();
int k = j;
for (; k < std::min(len0, j + 127 * v_int8::nlanes); k += v_int8::nlanes)
v_sum8 += v_one & v_pack(v_reinterpret_as_s16(vx_load(src + k) == v_zero), v_reinterpret_as_s16(vx_load(src + k + v_uint16::nlanes) == v_zero));
for (; k < std::min(len0, j + 127 * VTraits<v_int8>::vlanes()); k += VTraits<v_int8>::vlanes())
v_sum8 = v_add(v_sum8, v_and(v_one, v_pack(v_reinterpret_as_s16(v_eq(vx_load(src + k), v_zero)), v_reinterpret_as_s16(v_eq(vx_load(src + k + VTraits<v_uint16>::vlanes()), v_zero)))));
v_int16 part1, part2;
v_expand(v_sum8, part1, part2);
v_sum16 += part1 + part2;
v_sum16 = v_add(v_sum16, v_add(part1, part2));
j = k;
}
v_int32 part1, part2;
v_expand(v_sum16, part1, part2);
v_sum32 += part1 + part2;
v_sum32 = v_add(v_sum32, v_add(part1, part2));
i = j;
}
nz = i - v_reduce_sum(v_sum32);
@@ -104,8 +104,8 @@ static int countNonZero16u( const ushort* src, int len )
static int countNonZero32s( const int* src, int len )
{
int i = 0, nz = 0;
#if CV_SIMD
int len0 = len & -v_int8::nlanes;
#if (CV_SIMD || CV_SIMD_SCALABLE)
int len0 = len & -VTraits<v_int8>::vlanes();
v_int32 v_zero = vx_setzero_s32();
v_int8 v_one = vx_setall_s8(1);
@@ -114,23 +114,20 @@ static int countNonZero32s( const int* src, int len )
{
v_int16 v_sum16 = vx_setzero_s16();
int j = i;
while (j < std::min(len0, i + 32766 * v_int16::nlanes))
while (j < std::min(len0, i + 32766 * VTraits<v_int16>::vlanes()))
{
v_int8 v_sum8 = vx_setzero_s8();
int k = j;
for (; k < std::min(len0, j + 127 * v_int8::nlanes); k += v_int8::nlanes)
v_sum8 += v_one & v_pack(
v_pack(vx_load(src + k ) == v_zero, vx_load(src + k + v_int32::nlanes) == v_zero),
v_pack(vx_load(src + k + 2*v_int32::nlanes) == v_zero, vx_load(src + k + 3*v_int32::nlanes) == v_zero)
);
for (; k < std::min(len0, j + 127 * VTraits<v_int8>::vlanes()); k += VTraits<v_int8>::vlanes())
v_sum8 = v_add(v_sum8, v_and(v_one, v_pack(v_pack(v_eq(vx_load(src + k), v_zero), v_eq(vx_load(src + k + VTraits<v_int32>::vlanes()), v_zero)), v_pack(v_eq(vx_load(src + k + 2 * VTraits<v_int32>::vlanes()), v_zero), v_eq(vx_load(src + k + 3 * VTraits<v_int32>::vlanes()), v_zero)))));
v_int16 part1, part2;
v_expand(v_sum8, part1, part2);
v_sum16 += part1 + part2;
v_sum16 = v_add(v_sum16, v_add(part1, part2));
j = k;
}
v_int32 part1, part2;
v_expand(v_sum16, part1, part2);
v_sum32 += part1 + part2;
v_sum32 = v_add(v_sum32, v_add(part1, part2));
i = j;
}
nz = i - v_reduce_sum(v_sum32);
@@ -142,8 +139,8 @@ static int countNonZero32s( const int* src, int len )
static int countNonZero32f( const float* src, int len )
{
int i = 0, nz = 0;
#if CV_SIMD
int len0 = len & -v_int8::nlanes;
#if (CV_SIMD || CV_SIMD_SCALABLE)
int len0 = len & -VTraits<v_int8>::vlanes();
v_float32 v_zero = vx_setzero_f32();
v_int8 v_one = vx_setall_s8(1);
@@ -152,23 +149,20 @@ static int countNonZero32f( const float* src, int len )
{
v_int16 v_sum16 = vx_setzero_s16();
int j = i;
while (j < std::min(len0, i + 32766 * v_int16::nlanes))
while (j < std::min(len0, i + 32766 * VTraits<v_int16>::vlanes()))
{
v_int8 v_sum8 = vx_setzero_s8();
int k = j;
for (; k < std::min(len0, j + 127 * v_int8::nlanes); k += v_int8::nlanes)
v_sum8 += v_one & v_pack(
v_pack(v_reinterpret_as_s32(vx_load(src + k ) == v_zero), v_reinterpret_as_s32(vx_load(src + k + v_float32::nlanes) == v_zero)),
v_pack(v_reinterpret_as_s32(vx_load(src + k + 2*v_float32::nlanes) == v_zero), v_reinterpret_as_s32(vx_load(src + k + 3*v_float32::nlanes) == v_zero))
);
for (; k < std::min(len0, j + 127 * VTraits<v_int8>::vlanes()); k += VTraits<v_int8>::vlanes())
v_sum8 = v_add(v_sum8, v_and(v_one, v_pack(v_pack(v_reinterpret_as_s32(v_eq(vx_load(src + k), v_zero)), v_reinterpret_as_s32(v_eq(vx_load(src + k + VTraits<v_float32>::vlanes()), v_zero))), v_pack(v_reinterpret_as_s32(v_eq(vx_load(src + k + 2 * VTraits<v_float32>::vlanes()), v_zero)), v_reinterpret_as_s32(v_eq(vx_load(src + k + 3 * VTraits<v_float32>::vlanes()), v_zero))))));
v_int16 part1, part2;
v_expand(v_sum8, part1, part2);
v_sum16 += part1 + part2;
v_sum16 = v_add(v_sum16, v_add(part1, part2));
j = k;
}
v_int32 part1, part2;
v_expand(v_sum16, part1, part2);
v_sum32 += part1 + part2;
v_sum32 = v_add(v_sum32, v_add(part1, part2));
i = j;
}
nz = i - v_reduce_sum(v_sum32);
@@ -180,21 +174,21 @@ static int countNonZero32f( const float* src, int len )
static int countNonZero64f( const double* src, int len )
{
int nz = 0, i = 0;
#if CV_SIMD_64F
#if (CV_SIMD_64F || CV_SIMD_SCALABLE_64F)
v_int64 sum1 = vx_setzero_s64();
v_int64 sum2 = vx_setzero_s64();
v_float64 zero = vx_setzero_f64();
int step = v_float64::nlanes * 2;
int step = VTraits<v_float64>::vlanes() * 2;
int len0 = len & -step;
for(i = 0; i < len0; i += step )
{
sum1 += v_reinterpret_as_s64(vx_load(&src[i]) == zero);
sum2 += v_reinterpret_as_s64(vx_load(&src[i + step / 2]) == zero);
sum1 = v_add(sum1, v_reinterpret_as_s64(v_eq(vx_load(&src[i]), zero)));
sum2 = v_add(sum2, v_reinterpret_as_s64(v_eq(vx_load(&src[i + step / 2]), zero)));
}
// N.B the value is incremented by -1 (0xF...F) for each value
nz = i + (int)v_reduce_sum(sum1 + sum2);
nz = i + (int)v_reduce_sum(v_add(sum1, sum2));
v_cleanup();
#endif
return nz + countNonZero_(src + i, len - i);
modules/core/src/lapack.cpp
+19 -69
View File
@@ -274,22 +274,21 @@ template<typename T> struct VBLAS
{
int dot(const T*, const T*, int, T*) const { return 0; }
int givens(T*, T*, int, T, T) const { return 0; }
int givensx(T*, T*, int, T, T, T*, T*) const { return 0; }
};
#if CV_SIMD
#if CV_SIMD // TODO: enable for CV_SIMD_SCALABLE_64F
template<> inline int VBLAS<float>::dot(const float* a, const float* b, int n, float* result) const
{
if( n < 2*v_float32::nlanes )
if( n < 2*VTraits<v_float32>::vlanes() )
return 0;
int k = 0;
v_float32 s0 = vx_setzero_f32();
for( ; k <= n - v_float32::nlanes; k += v_float32::nlanes )
for( ; k <= n - VTraits<v_float32>::vlanes(); k += VTraits<v_float32>::vlanes() )
{
v_float32 a0 = vx_load(a + k);
v_float32 b0 = vx_load(b + k);
s0 += a0 * b0;
s0 = v_add(s0, v_mul(a0, b0));
}
*result = v_reduce_sum(s0);
vx_cleanup();
@@ -299,16 +298,16 @@ template<> inline int VBLAS<float>::dot(const float* a, const float* b, int n, f
template<> inline int VBLAS<float>::givens(float* a, float* b, int n, float c, float s) const
{
if( n < v_float32::nlanes)
if( n < VTraits<v_float32>::vlanes())
return 0;
int k = 0;
v_float32 c4 = vx_setall_f32(c), s4 = vx_setall_f32(s);
for( ; k <= n - v_float32::nlanes; k += v_float32::nlanes )
for( ; k <= n - VTraits<v_float32>::vlanes(); k += VTraits<v_float32>::vlanes() )
{
v_float32 a0 = vx_load(a + k);
v_float32 b0 = vx_load(b + k);
v_float32 t0 = (a0 * c4) + (b0 * s4);
v_float32 t1 = (b0 * c4) - (a0 * s4);
v_float32 t0 = v_add(v_mul(a0, c4), v_mul(b0, s4));
v_float32 t1 = v_sub(v_mul(b0, c4), v_mul(a0, s4));
v_store(a + k, t0);
v_store(b + k, t1);
}
@@ -317,44 +316,19 @@ template<> inline int VBLAS<float>::givens(float* a, float* b, int n, float c, f
}
template<> inline int VBLAS<float>::givensx(float* a, float* b, int n, float c, float s,
float* anorm, float* bnorm) const
{
if( n < v_float32::nlanes)
return 0;
int k = 0;
v_float32 c4 = vx_setall_f32(c), s4 = vx_setall_f32(s);
v_float32 sa = vx_setzero_f32(), sb = vx_setzero_f32();
for( ; k <= n - v_float32::nlanes; k += v_float32::nlanes )
{
v_float32 a0 = vx_load(a + k);
v_float32 b0 = vx_load(b + k);
v_float32 t0 = (a0 * c4) + (b0 * s4);
v_float32 t1 = (b0 * c4) - (a0 * s4);
v_store(a + k, t0);
v_store(b + k, t1);
sa += t0 + t0;
sb += t1 + t1;
}
*anorm = v_reduce_sum(sa);
*bnorm = v_reduce_sum(sb);
vx_cleanup();
return k;
}
#if CV_SIMD_64F
#if (CV_SIMD_64F || CV_SIMD_SCALABLE_64F)
template<> inline int VBLAS<double>::dot(const double* a, const double* b, int n, double* result) const
{
if( n < 2*v_float64::nlanes )
if( n < 2*VTraits<v_float64>::vlanes() )
return 0;
int k = 0;
v_float64 s0 = vx_setzero_f64();
for( ; k <= n - v_float64::nlanes; k += v_float64::nlanes )
for( ; k <= n - VTraits<v_float64>::vlanes(); k += VTraits<v_float64>::vlanes() )
{
v_float64 a0 = vx_load(a + k);
v_float64 b0 = vx_load(b + k);
s0 += a0 * b0;
s0 = v_add(s0, v_mul(a0, b0));
}
double sbuf[2];
v_store(sbuf, s0);
@@ -368,12 +342,12 @@ template<> inline int VBLAS<double>::givens(double* a, double* b, int n, double
{
int k = 0;
v_float64 c2 = vx_setall_f64(c), s2 = vx_setall_f64(s);
for( ; k <= n - v_float64::nlanes; k += v_float64::nlanes )
for( ; k <= n - VTraits<v_float64>::vlanes(); k += VTraits<v_float64>::vlanes() )
{
v_float64 a0 = vx_load(a + k);
v_float64 b0 = vx_load(b + k);
v_float64 t0 = (a0 * c2) + (b0 * s2);
v_float64 t1 = (b0 * c2) - (a0 * s2);
v_float64 t0 = v_add(v_mul(a0, c2), v_mul(b0, s2));
v_float64 t1 = v_sub(v_mul(b0, c2), v_mul(a0, s2));
v_store(a + k, t0);
v_store(b + k, t1);
}
@@ -382,30 +356,6 @@ template<> inline int VBLAS<double>::givens(double* a, double* b, int n, double
}
template<> inline int VBLAS<double>::givensx(double* a, double* b, int n, double c, double s,
double* anorm, double* bnorm) const
{
int k = 0;
v_float64 c2 = vx_setall_f64(c), s2 = vx_setall_f64(s);
v_float64 sa = vx_setzero_f64(), sb = vx_setzero_f64();
for( ; k <= n - v_float64::nlanes; k += v_float64::nlanes )
{
v_float64 a0 = vx_load(a + k);
v_float64 b0 = vx_load(b + k);
v_float64 t0 = (a0 * c2) + (b0 * s2);
v_float64 t1 = (b0 * c2) - (a0 * s2);
v_store(a + k, t0);
v_store(b + k, t1);
sa += t0 * t0;
sb += t1 * t1;
}
double abuf[2], bbuf[2];
v_store(abuf, sa);
v_store(bbuf, sb);
*anorm = abuf[0] + abuf[1];
*bnorm = bbuf[0] + bbuf[1];
return k;
}
#endif //CV_SIMD_64F
#endif //CV_SIMD
@@ -916,7 +866,7 @@ double invert( InputArray _src, OutputArray _dst, int method )
#if CV_SIMD128
const float d_32f = (float)d;
const v_float32x4 d_vec(d_32f, -d_32f, -d_32f, d_32f);
v_float32x4 s0 = v_load_halves((const float*)srcdata, (const float*)(srcdata + srcstep)) * d_vec;//0123//3120
v_float32x4 s0 = v_mul(v_load_halves((const float *)srcdata, (const float *)(srcdata + srcstep)), d_vec);//0123//3120
s0 = v_extract<3>(s0, v_combine_low(v_rotate_right<1>(s0), s0));
v_store_low((float*)dstdata, s0);
v_store_high((float*)(dstdata + dststep), s0);
@@ -942,10 +892,10 @@ double invert( InputArray _src, OutputArray _dst, int method )
d = 1./d;
#if CV_SIMD128_64F
v_float64x2 det = v_setall_f64(d);
v_float64x2 s0 = v_load((const double*)srcdata) * det;
v_float64x2 s1 = v_load((const double*)(srcdata+srcstep)) * det;
v_float64x2 s0 = v_mul(v_load((const double *)srcdata), det);
v_float64x2 s1 = v_mul(v_load((const double *)(srcdata + srcstep)), det);
v_float64x2 sm = v_extract<1>(s1, s0);//30
v_float64x2 ss = v_setall<double>(0) - v_extract<1>(s0, s1);//12
v_float64x2 ss = v_sub(v_setall<double>(0), v_extract<1>(s0, s1));//12
v_store((double*)dstdata, v_combine_low(sm, ss));//31
v_store((double*)(dstdata + dststep), v_combine_high(ss, sm));//20
#else
modules/core/src/mathfuncs.cpp
+69 -69
View File
@@ -614,13 +614,13 @@ void polarToCart( InputArray src1, InputArray src2,
{
k = 0;
#if CV_SIMD
int cWidth = v_float32::nlanes;
#if (CV_SIMD || CV_SIMD_SCALABLE)
int cWidth = VTraits<v_float32>::vlanes();
for( ; k <= len - cWidth; k += cWidth )
{
v_float32 v_m = vx_load(mag + k);
v_store(x + k, vx_load(x + k) * v_m);
v_store(y + k, vx_load(y + k) * v_m);
v_store(x + k, v_mul(vx_load(x + k), v_m));
v_store(y + k, v_mul(vx_load(y + k), v_m));
}
vx_cleanup();
#endif
@@ -741,7 +741,7 @@ struct iPow_SIMD
}
};
#if CV_SIMD
#if (CV_SIMD || CV_SIMD_SCALABLE)
template <>
struct iPow_SIMD<uchar, int>
@@ -751,7 +751,7 @@ struct iPow_SIMD<uchar, int>
int i = 0;
v_uint32 v_1 = vx_setall_u32(1u);
for ( ; i <= len - v_uint16::nlanes; i += v_uint16::nlanes)
for ( ; i <= len - VTraits<v_uint16>::vlanes(); i += VTraits<v_uint16>::vlanes())
{
v_uint32 v_a1 = v_1, v_a2 = v_1;
v_uint16 v = vx_load_expand(src + i);
@@ -763,16 +763,16 @@ struct iPow_SIMD<uchar, int>
{
if (p & 1)
{
v_a1 *= v_b1;
v_a2 *= v_b2;
v_a1 = v_mul(v_a1, v_b1);
v_a2 = v_mul(v_a2, v_b2);
}
v_b1 *= v_b1;
v_b2 *= v_b2;
v_b1 = v_mul(v_b1, v_b1);
v_b2 = v_mul(v_b2, v_b2);
p >>= 1;
}
v_a1 *= v_b1;
v_a2 *= v_b2;
v_a1 = v_mul(v_a1, v_b1);
v_a2 = v_mul(v_a2, v_b2);
v = v_pack(v_a1, v_a2);
v_pack_store(dst + i, v);
@@ -791,7 +791,7 @@ struct iPow_SIMD<schar, int>
int i = 0;
v_int32 v_1 = vx_setall_s32(1);
for ( ; i <= len - v_int16::nlanes; i += v_int16::nlanes)
for ( ; i <= len - VTraits<v_int16>::vlanes(); i += VTraits<v_int16>::vlanes())
{
v_int32 v_a1 = v_1, v_a2 = v_1;
v_int16 v = vx_load_expand(src + i);
@@ -803,16 +803,16 @@ struct iPow_SIMD<schar, int>
{
if (p & 1)
{
v_a1 *= v_b1;
v_a2 *= v_b2;
v_a1 = v_mul(v_a1, v_b1);
v_a2 = v_mul(v_a2, v_b2);
}
v_b1 *= v_b1;
v_b2 *= v_b2;
v_b1 = v_mul(v_b1, v_b1);
v_b2 = v_mul(v_b2, v_b2);
p >>= 1;
}
v_a1 *= v_b1;
v_a2 *= v_b2;
v_a1 = v_mul(v_a1, v_b1);
v_a2 = v_mul(v_a2, v_b2);
v = v_pack(v_a1, v_a2);
v_pack_store(dst + i, v);
@@ -831,7 +831,7 @@ struct iPow_SIMD<ushort, int>
int i = 0;
v_uint32 v_1 = vx_setall_u32(1u);
for ( ; i <= len - v_uint16::nlanes; i += v_uint16::nlanes)
for ( ; i <= len - VTraits<v_uint16>::vlanes(); i += VTraits<v_uint16>::vlanes())
{
v_uint32 v_a1 = v_1, v_a2 = v_1;
v_uint16 v = vx_load(src + i);
@@ -843,16 +843,16 @@ struct iPow_SIMD<ushort, int>
{
if (p & 1)
{
v_a1 *= v_b1;
v_a2 *= v_b2;
v_a1 = v_mul(v_a1, v_b1);
v_a2 = v_mul(v_a2, v_b2);
}
v_b1 *= v_b1;
v_b2 *= v_b2;
v_b1 = v_mul(v_b1, v_b1);
v_b2 = v_mul(v_b2, v_b2);
p >>= 1;
}
v_a1 *= v_b1;
v_a2 *= v_b2;
v_a1 = v_mul(v_a1, v_b1);
v_a2 = v_mul(v_a2, v_b2);
v = v_pack(v_a1, v_a2);
v_store(dst + i, v);
@@ -871,7 +871,7 @@ struct iPow_SIMD<short, int>
int i = 0;
v_int32 v_1 = vx_setall_s32(1);
for ( ; i <= len - v_int16::nlanes; i += v_int16::nlanes)
for ( ; i <= len - VTraits<v_int16>::vlanes(); i += VTraits<v_int16>::vlanes())
{
v_int32 v_a1 = v_1, v_a2 = v_1;
v_int16 v = vx_load(src + i);
@@ -883,16 +883,16 @@ struct iPow_SIMD<short, int>
{
if (p & 1)
{
v_a1 *= v_b1;
v_a2 *= v_b2;
v_a1 = v_mul(v_a1, v_b1);
v_a2 = v_mul(v_a2, v_b2);
}
v_b1 *= v_b1;
v_b2 *= v_b2;
v_b1 = v_mul(v_b1, v_b1);
v_b2 = v_mul(v_b2, v_b2);
p >>= 1;
}
v_a1 *= v_b1;
v_a2 *= v_b2;
v_a1 = v_mul(v_a1, v_b1);
v_a2 = v_mul(v_a2, v_b2);
v = v_pack(v_a1, v_a2);
v_store(dst + i, v);
@@ -911,29 +911,29 @@ struct iPow_SIMD<int, int>
int i = 0;
v_int32 v_1 = vx_setall_s32(1);
for ( ; i <= len - v_int32::nlanes*2; i += v_int32::nlanes*2)
for ( ; i <= len - VTraits<v_int32>::vlanes()*2; i += VTraits<v_int32>::vlanes()*2)
{
v_int32 v_a1 = v_1, v_a2 = v_1;
v_int32 v_b1 = vx_load(src + i), v_b2 = vx_load(src + i + v_int32::nlanes);
v_int32 v_b1 = vx_load(src + i), v_b2 = vx_load(src + i + VTraits<v_int32>::vlanes());
int p = power;
while( p > 1 )
{
if (p & 1)
{
v_a1 *= v_b1;
v_a2 *= v_b2;
v_a1 = v_mul(v_a1, v_b1);
v_a2 = v_mul(v_a2, v_b2);
}
v_b1 *= v_b1;
v_b2 *= v_b2;
v_b1 = v_mul(v_b1, v_b1);
v_b2 = v_mul(v_b2, v_b2);
p >>= 1;
}
v_a1 *= v_b1;
v_a2 *= v_b2;
v_a1 = v_mul(v_a1, v_b1);
v_a2 = v_mul(v_a2, v_b2);
v_store(dst + i, v_a1);
v_store(dst + i + v_int32::nlanes, v_a2);
v_store(dst + i + VTraits<v_int32>::vlanes(), v_a2);
}
vx_cleanup();
@@ -949,34 +949,34 @@ struct iPow_SIMD<float, float>
int i = 0;
v_float32 v_1 = vx_setall_f32(1.f);
for ( ; i <= len - v_float32::nlanes*2; i += v_float32::nlanes*2)
for ( ; i <= len - VTraits<v_float32>::vlanes()*2; i += VTraits<v_float32>::vlanes()*2)
{
v_float32 v_a1 = v_1, v_a2 = v_1;
v_float32 v_b1 = vx_load(src + i), v_b2 = vx_load(src + i + v_float32::nlanes);
v_float32 v_b1 = vx_load(src + i), v_b2 = vx_load(src + i + VTraits<v_float32>::vlanes());
int p = std::abs(power);
if( power < 0 )
{
v_b1 = v_1 / v_b1;
v_b2 = v_1 / v_b2;
v_b1 = v_div(v_1, v_b1);
v_b2 = v_div(v_1, v_b2);
}
while( p > 1 )
{
if (p & 1)
{
v_a1 *= v_b1;
v_a2 *= v_b2;
v_a1 = v_mul(v_a1, v_b1);
v_a2 = v_mul(v_a2, v_b2);
}
v_b1 *= v_b1;
v_b2 *= v_b2;
v_b1 = v_mul(v_b1, v_b1);
v_b2 = v_mul(v_b2, v_b2);
p >>= 1;
}
v_a1 *= v_b1;
v_a2 *= v_b2;
v_a1 = v_mul(v_a1, v_b1);
v_a2 = v_mul(v_a2, v_b2);
v_store(dst + i, v_a1);
v_store(dst + i + v_float32::nlanes, v_a2);
v_store(dst + i + VTraits<v_float32>::vlanes(), v_a2);
}
vx_cleanup();
@@ -984,7 +984,7 @@ struct iPow_SIMD<float, float>
}
};
#if CV_SIMD_64F
#if (CV_SIMD_64F || CV_SIMD_SCALABLE_64F)
template <>
struct iPow_SIMD<double, double>
{
@@ -993,34 +993,34 @@ struct iPow_SIMD<double, double>
int i = 0;
v_float64 v_1 = vx_setall_f64(1.);
for ( ; i <= len - v_float64::nlanes*2; i += v_float64::nlanes*2)
for ( ; i <= len - VTraits<v_float64>::vlanes()*2; i += VTraits<v_float64>::vlanes()*2)
{
v_float64 v_a1 = v_1, v_a2 = v_1;
v_float64 v_b1 = vx_load(src + i), v_b2 = vx_load(src + i + v_float64::nlanes);
v_float64 v_b1 = vx_load(src + i), v_b2 = vx_load(src + i + VTraits<v_float64>::vlanes());
int p = std::abs(power);
if( power < 0 )
{
v_b1 = v_1 / v_b1;
v_b2 = v_1 / v_b2;
v_b1 = v_div(v_1, v_b1);
v_b2 = v_div(v_1, v_b2);
}
while( p > 1 )
{
if (p & 1)
{
v_a1 *= v_b1;
v_a2 *= v_b2;
v_a1 = v_mul(v_a1, v_b1);
v_a2 = v_mul(v_a2, v_b2);
}
v_b1 *= v_b1;
v_b2 *= v_b2;
v_b1 = v_mul(v_b1, v_b1);
v_b2 = v_mul(v_b2, v_b2);
p >>= 1;
}
v_a1 *= v_b1;
v_a2 *= v_b2;
v_a1 = v_mul(v_a1, v_b1);
v_a2 = v_mul(v_a2, v_b2);
v_store(dst + i, v_a1);
v_store(dst + i + v_float64::nlanes, v_a2);
v_store(dst + i + VTraits<v_float64>::vlanes(), v_a2);
}
vx_cleanup();
@@ -1614,7 +1614,7 @@ void patchNaNs( InputOutputArray _a, double _val )
Cv32suf val;
val.f = (float)_val;
#if CV_SIMD
#if (CV_SIMD || CV_SIMD_SCALABLE)
v_int32 v_mask1 = vx_setall_s32(0x7fffffff), v_mask2 = vx_setall_s32(0x7f800000);
v_int32 v_val = vx_setall_s32(val.i);
#endif
@@ -1624,12 +1624,12 @@ void patchNaNs( InputOutputArray _a, double _val )
int* tptr = ptrs[0];
size_t j = 0;
#if CV_SIMD
size_t cWidth = (size_t)v_int32::nlanes;
#if (CV_SIMD || CV_SIMD_SCALABLE)
size_t cWidth = (size_t)VTraits<v_int32>::vlanes();
for ( ; j + cWidth <= len; j += cWidth)
{
v_int32 v_src = vx_load(tptr + j);
v_int32 v_cmp_mask = v_mask2 < (v_src & v_mask1);
v_int32 v_cmp_mask = v_lt(v_mask2, v_and(v_src, v_mask1));
v_int32 v_dst = v_select(v_cmp_mask, v_val, v_src);
v_store(tptr + j, v_dst);
}
modules/core/src/matmul.simd.hpp
+71 -57
View File
@@ -1454,7 +1454,7 @@ transform_( const T* src, T* dst, const WT* m, int len, int scn, int dcn )
static void
transform_8u( const uchar* src, uchar* dst, const float* m, int len, int scn, int dcn )
{
#if CV_SIMD
#if (CV_SIMD || CV_SIMD_SCALABLE)
const int BITS = 10, SCALE = 1 << BITS;
const float MAX_M = (float)(1 << (15 - BITS));
@@ -1485,7 +1485,7 @@ transform_8u( const uchar* src, uchar* dst, const float* m, int len, int scn, in
v_int32 m10 = vx_setall_s32(m32[4]);
v_int32 m11 = vx_setall_s32(m32[5]);
int x = 0;
for (; x <= (len - v_uint8::nlanes) * nChannels; x += v_uint8::nlanes * nChannels)
for (; x <= (len - VTraits<v_uint8>::vlanes()) * nChannels; x += VTraits<v_uint8>::vlanes() * nChannels)
{
v_uint8 b, g, r;
v_load_deinterleave(src + x, b, g, r);
@@ -1499,20 +1499,20 @@ transform_8u( const uchar* src, uchar* dst, const float* m, int len, int scn, in
v_int32 p1, p3;
v_expand(bgl, p0, p2);
v_expand(v_reinterpret_as_s16(rl), p1, p3);
dbl = v_rshr_pack<BITS>(v_dotprod(v_reinterpret_as_s16(p0), m01) + p1 * m2 + m3,
v_dotprod(v_reinterpret_as_s16(p2), m01) + p3 * m2 + m3);
dgl = v_rshr_pack<BITS>(v_dotprod(v_reinterpret_as_s16(p0), m45) + p1 * m6 + m7,
v_dotprod(v_reinterpret_as_s16(p2), m45) + p3 * m6 + m7);
drl = v_rshr_pack<BITS>(v_dotprod(v_reinterpret_as_s16(p0), m89) + p1 * m10 + m11,
v_dotprod(v_reinterpret_as_s16(p2), m89) + p3 * m10 + m11);
dbl = v_rshr_pack<BITS>(v_add(v_add(v_dotprod(v_reinterpret_as_s16(p0), m01), v_mul(p1, m2)), m3),
v_add(v_add(v_dotprod(v_reinterpret_as_s16(p2), m01), v_mul(p3, m2)), m3));
dgl = v_rshr_pack<BITS>(v_add(v_add(v_dotprod(v_reinterpret_as_s16(p0), m45), v_mul(p1, m6)), m7),
v_add(v_add(v_dotprod(v_reinterpret_as_s16(p2), m45), v_mul(p3, m6)), m7));
drl = v_rshr_pack<BITS>(v_add(v_add(v_dotprod(v_reinterpret_as_s16(p0), m89), v_mul(p1, m10)), m11),
v_add(v_add(v_dotprod(v_reinterpret_as_s16(p2), m89), v_mul(p3, m10)), m11));
v_expand(bgh, p0, p2);
v_expand(v_reinterpret_as_s16(rh), p1, p3);
dbh = v_rshr_pack<BITS>(v_dotprod(v_reinterpret_as_s16(p0), m01) + p1 * m2 + m3,
v_dotprod(v_reinterpret_as_s16(p2), m01) + p3 * m2 + m3);
dgh = v_rshr_pack<BITS>(v_dotprod(v_reinterpret_as_s16(p0), m45) + p1 * m6 + m7,
v_dotprod(v_reinterpret_as_s16(p2), m45) + p3 * m6 + m7);
drh = v_rshr_pack<BITS>(v_dotprod(v_reinterpret_as_s16(p0), m89) + p1 * m10 + m11,
v_dotprod(v_reinterpret_as_s16(p2), m89) + p3 * m10 + m11);
dbh = v_rshr_pack<BITS>(v_add(v_add(v_dotprod(v_reinterpret_as_s16(p0), m01), v_mul(p1, m2)), m3),
v_add(v_add(v_dotprod(v_reinterpret_as_s16(p2), m01), v_mul(p3, m2)), m3));
dgh = v_rshr_pack<BITS>(v_add(v_add(v_dotprod(v_reinterpret_as_s16(p0), m45), v_mul(p1, m6)), m7),
v_add(v_add(v_dotprod(v_reinterpret_as_s16(p2), m45), v_mul(p3, m6)), m7));
drh = v_rshr_pack<BITS>(v_add(v_add(v_dotprod(v_reinterpret_as_s16(p0), m89), v_mul(p1, m10)), m11),
v_add(v_add(v_dotprod(v_reinterpret_as_s16(p2), m89), v_mul(p3, m10)), m11));
v_store_interleave(dst + x, v_pack_u(dbl, dbh), v_pack_u(dgl, dgh), v_pack_u(drl, drh));
}
m32[1] = saturate_cast<int>((m[3] + 0.5f)*SCALE);
@@ -1537,7 +1537,7 @@ transform_8u( const uchar* src, uchar* dst, const float* m, int len, int scn, in
static void
transform_16u( const ushort* src, ushort* dst, const float* m, int len, int scn, int dcn )
{
#if CV_SIMD
#if (CV_SIMD || CV_SIMD_SCALABLE)
if( scn == 3 && dcn == 3 )
{
int x = 0;
@@ -1555,7 +1555,7 @@ transform_16u( const ushort* src, ushort* dst, const float* m, int len, int scn,
v_float32 m10 = vx_setall_f32(m[10]);
v_float32 m11 = vx_setall_f32(m[11] - 32768.f);
v_int16 delta = vx_setall_s16(-32768);
for (; x <= (len - v_uint16::nlanes)*3; x += v_uint16::nlanes*3)
for (; x <= (len - VTraits<v_uint16>::vlanes())*3; x += VTraits<v_uint16>::vlanes()*3)
{
v_uint16 b, g, r;
v_load_deinterleave(src + x, b, g, r);
@@ -1574,6 +1574,7 @@ transform_16u( const ushort* src, ushort* dst, const float* m, int len, int scn,
v_store_interleave(dst + x, v_reinterpret_as_u16(db), v_reinterpret_as_u16(dg), v_reinterpret_as_u16(dr));
}
#endif
#if CV_SIMD128
v_float32x4 _m0l(m[0], m[4], m[ 8], 0.f);
v_float32x4 _m1l(m[1], m[5], m[ 9], 0.f);
v_float32x4 _m2l(m[2], m[6], m[10], 0.f);
@@ -1587,6 +1588,7 @@ transform_16u( const ushort* src, ushort* dst, const float* m, int len, int scn,
v_store(dst + x, v_rotate_right<1>(v_reinterpret_as_u16(v_add_wrap(v_pack(
v_round(v_matmuladd(v_cvt_f32(v_reinterpret_as_s32(v_load_expand(src + x ))), _m0h, _m1h, _m2h, _m3h)),
v_round(v_matmuladd(v_cvt_f32(v_reinterpret_as_s32(v_load_expand(src + x + 3))), _m0l, _m1l, _m2l, _m3l))), _delta))));
#endif //CV_SIMD128
for( ; x < len * 3; x += 3 )
{
float v0 = src[x], v1 = src[x + 1], v2 = src[x + 2];
@@ -1606,25 +1608,25 @@ transform_16u( const ushort* src, ushort* dst, const float* m, int len, int scn,
static void
transform_32f( const float* src, float* dst, const float* m, int len, int scn, int dcn )
{
#if CV_SIMD && !defined(__aarch64__) && !defined(_M_ARM64)
#if (CV_SIMD || CV_SIMD_SCALABLE) && !defined(__aarch64__) && !defined(_M_ARM64)
int x = 0;
if( scn == 3 && dcn == 3 )
{
int idx[v_float32::nlanes/2];
for( int i = 0; i < v_float32::nlanes/4; i++ )
int idx[VTraits<v_float32>::max_nlanes/2];
for( int i = 0; i < VTraits<v_float32>::vlanes()/4; i++ )
{
idx[i] = 3*i;
idx[i + v_float32::nlanes/4] = 0;
idx[i + VTraits<v_float32>::vlanes()/4] = 0;
}
float _m[] = { m[0], m[4], m[ 8], 0.f,
m[1], m[5], m[ 9], 0.f,
m[2], m[6], m[10], 0.f,
m[3], m[7], m[11], 0.f };
v_float32 m0 = vx_lut_quads(_m , idx + v_float32::nlanes/4);
v_float32 m1 = vx_lut_quads(_m + 4, idx + v_float32::nlanes/4);
v_float32 m2 = vx_lut_quads(_m + 8, idx + v_float32::nlanes/4);
v_float32 m3 = vx_lut_quads(_m + 12, idx + v_float32::nlanes/4);
for( ; x <= len*3 - v_float32::nlanes; x += 3*v_float32::nlanes/4 )
v_float32 m0 = vx_lut_quads(_m , idx + VTraits<v_float32>::vlanes()/4);
v_float32 m1 = vx_lut_quads(_m + 4, idx + VTraits<v_float32>::vlanes()/4);
v_float32 m2 = vx_lut_quads(_m + 8, idx + VTraits<v_float32>::vlanes()/4);
v_float32 m3 = vx_lut_quads(_m + 12, idx + VTraits<v_float32>::vlanes()/4);
for( ; x <= len*3 - VTraits<v_float32>::vlanes(); x += 3*VTraits<v_float32>::vlanes()/4 )
v_store(dst + x, v_pack_triplets(v_matmuladd(vx_lut_quads(src + x, idx), m0, m1, m2, m3)));
for( ; x < len*3; x += 3 )
{
@@ -1641,8 +1643,8 @@ transform_32f( const float* src, float* dst, const float* m, int len, int scn, i
if( scn == 4 && dcn == 4 )
{
#if CV_SIMD_WIDTH > 16
int idx[v_float32::nlanes/4];
for( int i = 0; i < v_float32::nlanes/4; i++ )
int idx[VTraits<v_float32>::max_nlanes/4];
for( int i = 0; i < VTraits<v_float32>::vlanes()/4; i++ )
idx[i] = 0;
float _m[] = { m[4], m[9], m[14], m[19] };
v_float32 m0 = vx_lut_quads(m , idx);
@@ -1650,12 +1652,13 @@ transform_32f( const float* src, float* dst, const float* m, int len, int scn, i
v_float32 m2 = vx_lut_quads(m+10, idx);
v_float32 m3 = vx_lut_quads(m+15, idx);
v_float32 m4 = vx_lut_quads(_m, idx);
for( ; x <= len*4 - v_float32::nlanes; x += v_float32::nlanes )
for( ; x <= len*4 - VTraits<v_float32>::vlanes(); x += VTraits<v_float32>::vlanes() )
{
v_float32 v_src = vx_load(src + x);
v_store(dst + x, v_reduce_sum4(v_src * m0, v_src * m1, v_src * m2, v_src * m3) + m4);
v_store(dst + x, v_add(v_reduce_sum4(v_mul(v_src, m0), v_mul(v_src, m1), v_mul(v_src, m2), v_mul(v_src, m3)), m4));
}
#endif
#if CV_SIMD128
v_float32x4 _m0 = v_load(m );
v_float32x4 _m1 = v_load(m + 5);
v_float32x4 _m2 = v_load(m + 10);
@@ -1666,6 +1669,17 @@ transform_32f( const float* src, float* dst, const float* m, int len, int scn, i
v_float32x4 v_src = v_load(src + x);
v_store(dst + x, v_reduce_sum4(v_src * _m0, v_src * _m1, v_src * _m2, v_src * _m3) + _m4);
}
#else // CV_SIMD_WIDTH >= 16 && !CV_SIMD128
for( ; x < len*4; x += 4 )
{
float v0 = src[x], v1 = src[x+1], v2 = src[x+2], v3 = src[x+3];
float t0 = saturate_cast<float>(m[0]*v0 + m[1]*v1 + m[ 2]*v2 + m[ 3]*v3 + m[ 4]);
float t1 = saturate_cast<float>(m[5]*v0 + m[6]*v1 + m[ 7]*v2 + m[ 8]*v3 + m[ 9]);
float t2 = saturate_cast<float>(m[10]*v0 + m[11]*v1 + m[12]*v2 + m[13]*v3 + m[14]);
float t3 = saturate_cast<float>(m[15]*v0 + m[16]*v1 + m[17]*v2 + m[18]*v3 + m[19]);
dst[x] = t0; dst[x+1] = t1; dst[x+2] = t2; dst[x+3] = t3;
}
#endif
vx_cleanup();
return;
}
@@ -1936,9 +1950,9 @@ static void scaleAdd_32f(const float* src1, const float* src2, float* dst,
{
float alpha = *_alpha;
int i = 0;
#if CV_SIMD
#if (CV_SIMD || CV_SIMD_SCALABLE)
v_float32 v_alpha = vx_setall_f32(alpha);
const int cWidth = v_float32::nlanes;
const int cWidth = VTraits<v_float32>::vlanes();
for (; i <= len - cWidth; i += cWidth)
v_store(dst + i, v_muladd(vx_load(src1 + i), v_alpha, vx_load(src2 + i)));
vx_cleanup();
@@ -1953,9 +1967,9 @@ static void scaleAdd_64f(const double* src1, const double* src2, double* dst,
{
double alpha = *_alpha;
int i = 0;
#if CV_SIMD_64F
#if (CV_SIMD_64F || CV_SIMD_SCALABLE_64F)
v_float64 a2 = vx_setall_f64(alpha);
const int cWidth = v_float64::nlanes;
const int cWidth = VTraits<v_float64>::vlanes();
for (; i <= len - cWidth; i += cWidth)
v_store(dst + i, v_muladd(vx_load(src1 + i), a2, vx_load(src2 + i)));
vx_cleanup();
@@ -2078,7 +2092,7 @@ MulTransposedR(const Mat& srcmat, const Mat& dstmat, const Mat& deltamat, double
deltastep = deltastep ? 4 : 0;
}
#if CV_SIMD_64F
#if CV_SIMD128_64F
v_float64x2 v_scale = v_setall_f64(scale);
#endif
@@ -2090,7 +2104,7 @@ MulTransposedR(const Mat& srcmat, const Mat& dstmat, const Mat& deltamat, double
for( j = i; j <= size.width - 4; j += 4 )
{
#if CV_SIMD_64F
#if CV_SIMD128_64F
if (DataType<sT>::depth == CV_64F && DataType<dT>::depth == CV_64F)
{
v_float64x2 s0 = v_setzero_f64(), s1 = v_setzero_f64();
@@ -2150,7 +2164,7 @@ MulTransposedR(const Mat& srcmat, const Mat& dstmat, const Mat& deltamat, double
for( j = i; j <= size.width - 4; j += 4 )
{
#if CV_SIMD_64F
#if CV_SIMD128_64F
if (DataType<sT>::depth == CV_64F && DataType<dT>::depth == CV_64F)
{
v_float64x2 s0 = v_setzero_f64(), s1 = v_setzero_f64();
@@ -2227,7 +2241,7 @@ MulTransposedL(const Mat& srcmat, const Mat& dstmat, const Mat& deltamat, double
double s = 0;
const sT *tsrc1 = src + i*srcstep;
const sT *tsrc2 = src + j*srcstep;
#if CV_SIMD_64F
#if CV_SIMD128_64F
if (DataType<sT>::depth == CV_64F && DataType<dT>::depth == CV_64F)
{
const double *v_tsrc1 = (double *)(tsrc1);
@@ -2280,7 +2294,7 @@ MulTransposedL(const Mat& srcmat, const Mat& dstmat, const Mat& deltamat, double
delta_buf[2] = delta_buf[3] = tdelta2[0];
tdelta2 = delta_buf;
}
#if CV_SIMD_64F
#if CV_SIMD128_64F
if (DataType<sT>::depth == CV_64F && DataType<dT>::depth == CV_64F)
{
const double *v_tsrc2 = (double *)(tsrc2);
@@ -2393,14 +2407,14 @@ double dotProd_8u(const uchar* src1, const uchar* src2, int len)
double r = 0;
int i = 0;
#if CV_SIMD
int len0 = len & -v_uint16::nlanes, blockSize0 = (1 << 15), blockSize;
#if (CV_SIMD || CV_SIMD_SCALABLE)
int len0 = len & -VTraits<v_uint16>::vlanes(), blockSize0 = (1 << 15), blockSize;
while (i < len0)
{
blockSize = std::min(len0 - i, blockSize0);
v_uint32 v_sum = vx_setzero_u32();
const int cWidth = v_uint16::nlanes;
const int cWidth = VTraits<v_uint16>::vlanes();
int j = 0;
for (; j <= blockSize - cWidth * 2; j += cWidth * 2)
@@ -2414,7 +2428,7 @@ double dotProd_8u(const uchar* src1, const uchar* src2, int len)
{
v_int16 v_src10 = v_reinterpret_as_s16(vx_load_expand(src1 + j));
v_int16 v_src20 = v_reinterpret_as_s16(vx_load_expand(src2 + j));
v_sum += v_reinterpret_as_u32(v_dotprod_fast(v_src10, v_src20));
v_sum = v_add(v_sum, v_reinterpret_as_u32(v_dotprod_fast(v_src10, v_src20)));
}
r += (double)v_reduce_sum(v_sum);
@@ -2433,14 +2447,14 @@ double dotProd_8s(const schar* src1, const schar* src2, int len)
double r = 0.0;
int i = 0;
#if CV_SIMD
int len0 = len & -v_int16::nlanes, blockSize0 = (1 << 14), blockSize;
#if (CV_SIMD || CV_SIMD_SCALABLE)
int len0 = len & -VTraits<v_int16>::vlanes(), blockSize0 = (1 << 14), blockSize;
while (i < len0)
{
blockSize = std::min(len0 - i, blockSize0);
v_int32 v_sum = vx_setzero_s32();
const int cWidth = v_int16::nlanes;
const int cWidth = VTraits<v_int16>::vlanes();
int j = 0;
for (; j <= blockSize - cWidth * 2; j += cWidth * 2)
@@ -2473,14 +2487,14 @@ double dotProd_16u(const ushort* src1, const ushort* src2, int len)
double r = 0.0;
int i = 0;
#if CV_SIMD
int len0 = len & -v_uint16::nlanes, blockSize0 = (1 << 24), blockSize;
#if (CV_SIMD || CV_SIMD_SCALABLE)
int len0 = len & -VTraits<v_uint16>::vlanes(), blockSize0 = (1 << 24), blockSize;
while (i < len0)
{
blockSize = std::min(len0 - i, blockSize0);
v_uint64 v_sum = vx_setzero_u64();
const int cWidth = v_uint16::nlanes;
const int cWidth = VTraits<v_uint16>::vlanes();
int j = 0;
for (; j <= blockSize - cWidth; j += cWidth)
@@ -2505,14 +2519,14 @@ double dotProd_16s(const short* src1, const short* src2, int len)
double r = 0.0;
int i = 0;
#if CV_SIMD
int len0 = len & -v_int16::nlanes, blockSize0 = (1 << 24), blockSize;
#if (CV_SIMD || CV_SIMD_SCALABLE)
int len0 = len & -VTraits<v_int16>::vlanes(), blockSize0 = (1 << 24), blockSize;
while (i < len0)
{
blockSize = std::min(len0 - i, blockSize0);
v_int64 v_sum = vx_setzero_s64();
const int cWidth = v_int16::nlanes;
const int cWidth = VTraits<v_int16>::vlanes();
int j = 0;
for (; j <= blockSize - cWidth; j += cWidth)
@@ -2534,10 +2548,10 @@ double dotProd_16s(const short* src1, const short* src2, int len)
double dotProd_32s(const int* src1, const int* src2, int len)
{
#if CV_SIMD_64F
#if CV_SIMD_64F // TODO: enable for CV_SIMD_SCALABLE_64F
double r = .0;
int i = 0;
const int step = v_int32::nlanes;
const int step = VTraits<v_int32>::vlanes();
v_float64 v_sum0 = vx_setzero_f64();
#if CV_SIMD_WIDTH == 16
const int wstep = step * 2;
@@ -2572,8 +2586,8 @@ double dotProd_32f(const float* src1, const float* src2, int len)
double r = 0.0;
int i = 0;
#if CV_SIMD
int len0 = len & -v_float32::nlanes, blockSize0 = (1 << 13), blockSize;
#if (CV_SIMD || CV_SIMD_SCALABLE)
int len0 = len & -VTraits<v_float32>::vlanes(), blockSize0 = (1 << 13), blockSize;
while (i < len0)
{
@@ -2581,7 +2595,7 @@ double dotProd_32f(const float* src1, const float* src2, int len)
v_float32 v_sum = vx_setzero_f32();
int j = 0;
int cWidth = v_float32::nlanes;
int cWidth = VTraits<v_float32>::vlanes();
#if CV_ENABLE_UNROLLED
v_float32 v_sum1 = vx_setzero_f32();
@@ -2600,7 +2614,7 @@ double dotProd_32f(const float* src1, const float* src2, int len)
vx_load(src2 + j + (cWidth * 3)), v_sum3);
}
v_sum += v_sum1 + v_sum2 + v_sum3;
v_sum = v_add(v_sum, v_add(v_add(v_sum1, v_sum2), v_sum3));
#endif
for (; j <= blockSize - cWidth; j += cWidth)
modules/core/src/matrix_transform.cpp
+223 -5
View File
@@ -7,6 +7,7 @@
#include "opencv2/core/detail/dispatch_helper.impl.hpp"
#include <algorithm> // std::swap_ranges
#include <numeric> // std::accumulate
namespace cv {
@@ -440,7 +441,7 @@ template<typename T1, typename T2> CV_ALWAYS_INLINE void flipHoriz_double( const
static void
flipHoriz( const uchar* src, size_t sstep, uchar* dst, size_t dstep, Size size, size_t esz )
{
#if CV_SIMD
#if CV_SIMD128
#if CV_STRONG_ALIGNMENT
size_t alignmentMark = ((size_t)src)|((size_t)dst)|sstep|dstep;
#endif
@@ -563,7 +564,7 @@ flipHoriz( const uchar* src, size_t sstep, uchar* dst, size_t dstep, Size size,
}
#endif
else
#endif // CV_SIMD
#endif // CV_SIMD128
{
int i, j, limit = (int)(((size.width + 1)/2)*esz);
AutoBuffer<int> _tab(size.width*esz);
@@ -596,12 +597,12 @@ flipVert( const uchar* src0, size_t sstep, uchar* dst0, size_t dstep, Size size,
dst0 += dstep, dst1 -= dstep )
{
int i = 0;
#if CV_SIMD
#if (CV_SIMD || CV_SIMD_SCALABLE)
#if CV_STRONG_ALIGNMENT
if (isAligned<sizeof(int)>(src0, src1, dst0, dst1))
#endif
{
for (; i <= size.width - CV_SIMD_WIDTH; i += CV_SIMD_WIDTH)
for (; i <= size.width - VTraits<v_uint8>::vlanes(); i += VTraits<v_uint8>::vlanes())
{
v_int32 t0 = v_reinterpret_as_s32(vx_load(src0 + i));
v_int32 t1 = v_reinterpret_as_s32(vx_load(src1 + i));
@@ -612,7 +613,7 @@ flipVert( const uchar* src0, size_t sstep, uchar* dst0, size_t dstep, Size size,
#if CV_STRONG_ALIGNMENT
else
{
for (; i <= size.width - CV_SIMD_WIDTH; i += CV_SIMD_WIDTH)
for (; i <= size.width - VTraits<v_uint8>::vlanes(); i += VTraits<v_uint8>::vlanes())
{
v_uint8 t0 = vx_load(src0 + i);
v_uint8 t1 = vx_load(src1 + i);
@@ -857,6 +858,223 @@ void flipND(InputArray _src, OutputArray _dst, int _axis)
flipNDImpl(dst.ptr(), dst.size.p, dst.step.p, axis);
}
/*
This function first prepends 1 to each tensor shape to have a common max_ndims dimension, then flatten non-broadcast dimensions.
*/
static bool _flatten_for_broadcast(int narrays, int max_ndims, const int* ndims, const int** orig_shape,
int** flatten_shape, size_t** flatten_step) {
int i, j, k;
// step 1.
// * make all inputs and the output max_ndims-dimensional.
// * compute proper step's
for (i = max_ndims - 1; i >= 0; i-- ) {
for (k = 0; k < narrays; k++) {
j = ndims[k] - (max_ndims - i);
int sz_i = j >= 0 ? orig_shape[k][j] : 1;
size_t st_i = i == max_ndims - 1 ? 1 : flatten_step[k][i+1] * flatten_shape[k][i+1];
flatten_shape[k][i] = sz_i;
flatten_step[k][i] = st_i;
if (flatten_shape[k][i] == 0)
return false;
}
}
// step 2. Let's do the flattening first,
// since we'd need proper values of steps to check continuity.
// this loop is probably the most tricky part
// in the whole implementation of broadcasting.
j = max_ndims-1;
for (i = j - 1; i >= 0; i--) {
bool all_contiguous = true, all_scalars = true, all_consistent = true;
for(k = 0; k < narrays; k++) {
size_t st = flatten_step[k][j] * flatten_shape[k][j];
bool prev_scalar = flatten_shape[k][j] == 1;
bool scalar = flatten_shape[k][i] == 1;
all_contiguous = all_contiguous && (st == flatten_step[k][i]);
all_scalars = all_scalars && scalar;
all_consistent = all_consistent && (scalar == prev_scalar);
}
if (all_contiguous && (all_consistent || all_scalars)) {
for(k = 0; k < narrays; k++)
flatten_shape[k][j] *= flatten_shape[k][i];
} else {
j--;
if (i < j) {
for(k = 0; k < narrays; k++) {
flatten_shape[k][j] = flatten_shape[k][i];
flatten_step[k][j] = flatten_step[k][i];
}
}
}
}
// step 3. Set some step's to 0's.
for (i = max_ndims-1; i >= j; i--) {
for (k = 0; k < narrays; k++)
flatten_step[k][i] = flatten_shape[k][i] == 1 ? 0 : flatten_step[k][i];
}
for (; i >= 0; i--) {
for (k = 0; k < narrays; k++) {
flatten_step[k][i] = 0;
flatten_shape[k][i] = 1;
}
}
return true;
}
void broadcast(InputArray _src, InputArray _shape, OutputArray _dst) {
CV_INSTRUMENT_REGION();
Mat src = _src.getMat();
CV_CheckTrue(src.isContinuous(), "broadcast: input array must be contiguous");
CV_CheckChannelsEQ(src.channels(), 1, "broadcast: input array must be single channel");
Mat shape = _shape.getMat();
CV_CheckTypeEQ(shape.type(), CV_32S, "broadcast: target shape must be of type int32");
const auto dims_shape = static_cast<int>(shape.total());
const auto *ptr_shape = shape.ptr<int>();
// check valid shape, 1D/0D Mat would fail in the following checks
const auto dims_src = src.dims;
CV_CheckLE(dims_src, dims_shape,
"broadcast: dimension of input array must be less than or equal to dimension of target shape");
std::vector<int> shape_src{src.size.p, src.size.p + dims_src};
if (shape_src.size() < static_cast<size_t>(dims_shape)) {
shape_src.insert(shape_src.begin(), dims_shape - shape_src.size(), 1);
}
for (int i = 0; i < static_cast<int>(shape_src.size()); ++i) {
const auto *shape_target = ptr_shape;
if (shape_src[i] != 1) {
CV_CheckEQ(shape_src[i], shape_target[i], "target shape must be equal to input shape or 1");
}
}
// impl
_dst.create(dims_shape, shape.ptr<int>(), src.type());
Mat dst = _dst.getMat();
std::vector<int> is_same_shape(dims_shape, 0);
for (int i = 0; i < static_cast<int>(shape_src.size()); ++i) {
if (shape_src[i] == ptr_shape[i]) {
is_same_shape[i] = 1;
}
}
// copy if same shape
if (std::accumulate(is_same_shape.begin(), is_same_shape.end(), 1, std::multiplies<int>()) != 0) {
const auto *p_src = src.ptr<const char>();
auto *p_dst = dst.ptr<char>();
std::memcpy(p_dst, p_src, dst.total() * dst.elemSize());
return;
}
// other cases
int max_ndims = std::max(dims_src, dims_shape);
const int all_ndims[2] = {src.dims, dst.dims};
const int* orig_shapes[2] = {src.size.p, dst.size.p};
cv::AutoBuffer<size_t> buff(max_ndims * 4);
int* flatten_shapes[2] = {(int*)buff.data(), (int*)(buff.data() + max_ndims)};
size_t* flatten_steps[2] = {(size_t*)(buff.data() + 2 * max_ndims), (size_t*)(buff.data() + 3 * max_ndims)};
if (_flatten_for_broadcast(2, max_ndims, all_ndims, orig_shapes, flatten_shapes, flatten_steps)) {
size_t src_dp = flatten_steps[0][max_ndims - 1];
size_t dst_dp = flatten_steps[1][max_ndims - 1];
CV_Assert(dst_dp == 1);
CV_Assert(max_ndims >= 2); // >= 3?
size_t rowstep_src = flatten_steps[0][max_ndims - 2];
size_t rowstep_dst = flatten_steps[1][max_ndims - 2];
const char* ptr_src = src.ptr<const char>();
char* ptr_dst = dst.ptr<char>();
size_t esz = src.elemSize();
int nrows = flatten_shapes[1][max_ndims - 2];
int ncols = flatten_shapes[1][max_ndims - 1];
int nplanes = 1;
CV_Check(esz, esz == 1 || esz == 2 || esz == 4 || esz == 8, "broadcast: not supported data type");
for (int k = 0; k < max_ndims - 2; k++) {
nplanes *= flatten_shapes[1][k];
}
for (int plane_idx = 0; plane_idx < nplanes; plane_idx++) {
size_t offset_src = 0, offset_dst = 0;
size_t idx = (size_t)plane_idx;
for (int k = max_ndims - 3; k >= 0; k--) {
size_t prev_idx = idx / flatten_shapes[1][k];
size_t i_k = (int)(idx - prev_idx * flatten_shapes[1][k]);
offset_src += i_k * flatten_steps[0][k];
offset_dst += i_k * flatten_steps[1][k];
idx = prev_idx;
}
#define OPENCV_CORE_BROADCAST_LOOP(_Tp) \
for (int i = 0; i < nrows; i++) { \
const _Tp *ptr_src_ = (const _Tp*)ptr_src + offset_src + rowstep_src * i; \
_Tp *ptr_dst_ = (_Tp*)ptr_dst + offset_dst + rowstep_dst * i; \
if (src_dp == 1) { \
for (int j = 0; j < ncols; j++) { \
ptr_dst_[j] = ptr_src_[j]; \
} \
} else { \
_Tp x = *ptr_src_; \
for (int j = 0; j < ncols; j++) { \
ptr_dst_[j] = x; \
} \
} \
}
if (esz == 1) {
OPENCV_CORE_BROADCAST_LOOP(int8_t);
} else if (esz == 2) {
OPENCV_CORE_BROADCAST_LOOP(int16_t);
} else if (esz == 4) {
OPENCV_CORE_BROADCAST_LOOP(int32_t);
} else if (esz == 8) {
OPENCV_CORE_BROADCAST_LOOP(int64_t);
} else {
CV_Error(cv::Error::StsNotImplemented, "");
}
#undef OPENCV_CORE_BROADCAST_LOOP
}
} else {
// initial copy (src to dst)
std::vector<size_t> step_src{src.step.p, src.step.p + dims_src};
if (step_src.size() < static_cast<size_t>(dims_shape)) {
step_src.insert(step_src.begin(), dims_shape - step_src.size(), step_src[0]);
}
for (size_t i = 0; i < src.total(); ++i) {
size_t t = i;
size_t src_offset = 0, dst_offset = 0;
for (int j = static_cast<int>(shape_src.size() - 1); j >= 0; --j) {
size_t idx = t / shape_src[j];
size_t offset = static_cast<size_t>(t - idx * shape_src[j]);
src_offset += offset * step_src[j];
dst_offset += offset * dst.step[j];
t = idx;
}
const auto *p_src = src.ptr<const char>();
auto *p_dst = dst.ptr<char>();
std::memcpy(p_dst + dst_offset, p_src + src_offset, dst.elemSize());
}
// broadcast copy (dst inplace)
std::vector<int> cumulative_shape(dims_shape, 1);
int total = static_cast<int>(dst.total());
for (int i = dims_shape - 1; i >= 0; --i) {
cumulative_shape[i] = static_cast<int>(total / ptr_shape[i]);
total = cumulative_shape[i];
}
for (int i = dims_shape - 1; i >= 0; --i) {
if (is_same_shape[i] == 1) {
continue;
}
auto step = dst.step[i];
auto *p_dst = dst.ptr<char>();
for (int j = 0; j < cumulative_shape[i]; j++) {
for (int k = 0; k < ptr_shape[i] - 1; k++) {
std::memcpy(p_dst + step, p_dst, step);
p_dst += step;
}
p_dst += step;
}
}
}
}
void rotate(InputArray _src, OutputArray _dst, int rotateMode)
{
CV_Assert(_src.dims() <= 2);
modules/core/src/mean.dispatch.cpp
+4
View File
@@ -8,20 +8,24 @@
#include "opencv2/core/openvx/ovx_defs.hpp"
#include "stat.hpp"
#ifndef OPENCV_IPP_MEAN
#undef HAVE_IPP
#undef CV_IPP_RUN_FAST
#define CV_IPP_RUN_FAST(f, ...)
#undef CV_IPP_RUN
#define CV_IPP_RUN(c, f, ...)
#endif // OPENCV_IPP_MEAN
#include "mean.simd.hpp"
#include "mean.simd_declarations.hpp" // defines CV_CPU_DISPATCH_MODES_ALL=AVX2,...,BASELINE based on CMakeLists.txt content
#ifndef OPENCV_IPP_MEAN
#undef HAVE_IPP
#undef CV_IPP_RUN_FAST
#define CV_IPP_RUN_FAST(f, ...)
#undef CV_IPP_RUN
#define CV_IPP_RUN(c, f, ...)
#endif // OPENCV_IPP_MEAN
namespace cv {
modules/core/src/merge.dispatch.cpp
+1
View File
@@ -121,6 +121,7 @@ void merge(const Mat* mv, size_t n, OutputArray _dst)
CV_INSTRUMENT_REGION();
CV_Assert( mv && n > 0 );
CV_Assert(!mv[0].empty());
int depth = mv[0].depth();
bool allch1 = true;
modules/core/src/merge.simd.hpp
+10 -10
View File
@@ -15,7 +15,7 @@ void merge64s(const int64** src, int64* dst, int len, int cn);
#ifndef CV_CPU_OPTIMIZATION_DECLARATIONS_ONLY
#if CV_SIMD
#if (CV_SIMD || CV_SIMD_SCALABLE)
/*
The trick with STORE_UNALIGNED/STORE_ALIGNED_NOCACHE is the following:
on IA there are instructions movntps and such to which
@@ -38,7 +38,7 @@ void merge64s(const int64** src, int64* dst, int len, int cn);
template<typename T, typename VecT> static void
vecmerge_( const T** src, T* dst, int len, int cn )
{
const int VECSZ = VecT::nlanes;
const int VECSZ = VTraits<VecT>::vlanes();
int i, i0 = 0;
const T* src0 = src[0];
const T* src1 = src[1];
@@ -173,8 +173,8 @@ merge_( const T** src, T* dst, int len, int cn )
void merge8u(const uchar** src, uchar* dst, int len, int cn )
{
CV_INSTRUMENT_REGION();
#if CV_SIMD
if( len >= v_uint8::nlanes && 2 <= cn && cn <= 4 )
#if (CV_SIMD || CV_SIMD_SCALABLE)
if( len >= VTraits<v_uint8>::vlanes() && 2 <= cn && cn <= 4 )
vecmerge_<uchar, v_uint8>(src, dst, len, cn);
else
#endif
@@ -184,8 +184,8 @@ void merge8u(const uchar** src, uchar* dst, int len, int cn )
void merge16u(const ushort** src, ushort* dst, int len, int cn )
{
CV_INSTRUMENT_REGION();
#if CV_SIMD
if( len >= v_uint16::nlanes && 2 <= cn && cn <= 4 )
#if (CV_SIMD || CV_SIMD_SCALABLE)
if( len >= VTraits<v_uint16>::vlanes() && 2 <= cn && cn <= 4 )
vecmerge_<ushort, v_uint16>(src, dst, len, cn);
else
#endif
@@ -195,8 +195,8 @@ void merge16u(const ushort** src, ushort* dst, int len, int cn )
void merge32s(const int** src, int* dst, int len, int cn )
{
CV_INSTRUMENT_REGION();
#if CV_SIMD
if( len >= v_int32::nlanes && 2 <= cn && cn <= 4 )
#if (CV_SIMD || CV_SIMD_SCALABLE)
if( len >= VTraits<v_int32>::vlanes() && 2 <= cn && cn <= 4 )
vecmerge_<int, v_int32>(src, dst, len, cn);
else
#endif
@@ -206,8 +206,8 @@ void merge32s(const int** src, int* dst, int len, int cn )
void merge64s(const int64** src, int64* dst, int len, int cn )
{
CV_INSTRUMENT_REGION();
#if CV_SIMD
if( len >= v_int64::nlanes && 2 <= cn && cn <= 4 )
#if (CV_SIMD || CV_SIMD_SCALABLE)
if( len >= VTraits<v_int64>::vlanes() && 2 <= cn && cn <= 4 )
vecmerge_<int64, v_int64>(src, dst, len, cn);
else
#endif
modules/core/src/minmax.cpp
+2
View File
@@ -11,11 +11,13 @@
#include <algorithm>
#ifndef OPENCV_IPP_MINMAX
#undef HAVE_IPP
#undef CV_IPP_RUN_FAST
#define CV_IPP_RUN_FAST(f, ...)
#undef CV_IPP_RUN
#define CV_IPP_RUN(c, f, ...)
#endif // OPENCV_IPP_MINMAX
#define IPP_DISABLE_MINMAXIDX_MANY_ROWS 1 // see Core_MinMaxIdx.rows_overflow test
modules/core/src/norm.cpp
+33 -33
View File
@@ -63,25 +63,25 @@ int normHamming(const uchar* a, int n, int cellSize)
return -1;
int i = 0;
int result = 0;
#if CV_SIMD
#if (CV_SIMD || CV_SIMD_SCALABLE)
v_uint64 t = vx_setzero_u64();
if ( cellSize == 2)
{
v_uint16 mask = v_reinterpret_as_u16(vx_setall_u8(0x55));
for(; i <= n - v_uint8::nlanes; i += v_uint8::nlanes)
for(; i <= n - VTraits<v_uint8>::vlanes(); i += VTraits<v_uint8>::vlanes())
{
v_uint16 a0 = v_reinterpret_as_u16(vx_load(a + i));
t += v_popcount(v_reinterpret_as_u64((a0 | (a0 >> 1)) & mask));
t = v_add(t, v_popcount(v_reinterpret_as_u64(v_and(v_or(a0, v_shr<1>(a0)), mask))));
}
}
else // cellSize == 4
{
v_uint16 mask = v_reinterpret_as_u16(vx_setall_u8(0x11));
for(; i <= n - v_uint8::nlanes; i += v_uint8::nlanes)
for(; i <= n - VTraits<v_uint8>::vlanes(); i += VTraits<v_uint8>::vlanes())
{
v_uint16 a0 = v_reinterpret_as_u16(vx_load(a + i));
v_uint16 a1 = a0 | (a0 >> 2);
t += v_popcount(v_reinterpret_as_u64((a1 | (a1 >> 1)) & mask));
v_uint16 a1 = v_or(a0, v_shr<2>(a0));
t = v_add(t, v_popcount(v_reinterpret_as_u64(v_and(v_or(a1, v_shr<1>(a1)), mask))));
}
}
@@ -109,25 +109,25 @@ int normHamming(const uchar* a, const uchar* b, int n, int cellSize)
return -1;
int i = 0;
int result = 0;
#if CV_SIMD
#if (CV_SIMD || CV_SIMD_SCALABLE)
v_uint64 t = vx_setzero_u64();
if ( cellSize == 2)
{
v_uint16 mask = v_reinterpret_as_u16(vx_setall_u8(0x55));
for(; i <= n - v_uint8::nlanes; i += v_uint8::nlanes)
for(; i <= n - VTraits<v_uint8>::vlanes(); i += VTraits<v_uint8>::vlanes())
{
v_uint16 ab0 = v_reinterpret_as_u16(vx_load(a + i) ^ vx_load(b + i));
t += v_popcount(v_reinterpret_as_u64((ab0 | (ab0 >> 1)) & mask));
v_uint16 ab0 = v_reinterpret_as_u16(v_xor(vx_load(a + i), vx_load(b + i)));
t = v_add(t, v_popcount(v_reinterpret_as_u64(v_and(v_or(ab0, v_shr<1>(ab0)), mask))));
}
}
else // cellSize == 4
{
v_uint16 mask = v_reinterpret_as_u16(vx_setall_u8(0x11));
for(; i <= n - v_uint8::nlanes; i += v_uint8::nlanes)
for(; i <= n - VTraits<v_uint8>::vlanes(); i += VTraits<v_uint8>::vlanes())
{
v_uint16 ab0 = v_reinterpret_as_u16(vx_load(a + i) ^ vx_load(b + i));
v_uint16 ab1 = ab0 | (ab0 >> 2);
t += v_popcount(v_reinterpret_as_u64((ab1 | (ab1 >> 1)) & mask));
v_uint16 ab0 = v_reinterpret_as_u16(v_xor(vx_load(a + i), vx_load(b + i)));
v_uint16 ab1 = v_or(ab0, v_shr<2>(ab0));
t = v_add(t, v_popcount(v_reinterpret_as_u64(v_and(v_or(ab1, v_shr<1>(ab1)), mask))));
}
}
result += (int)v_reduce_sum(t);
@@ -145,21 +145,21 @@ int normHamming(const uchar* a, const uchar* b, int n, int cellSize)
float normL2Sqr_(const float* a, const float* b, int n)
{
int j = 0; float d = 0.f;
#if CV_SIMD
#if (CV_SIMD || CV_SIMD_SCALABLE)
v_float32 v_d0 = vx_setzero_f32(), v_d1 = vx_setzero_f32();
v_float32 v_d2 = vx_setzero_f32(), v_d3 = vx_setzero_f32();
for (; j <= n - 4 * v_float32::nlanes; j += 4 * v_float32::nlanes)
for (; j <= n - 4 * VTraits<v_float32>::vlanes(); j += 4 * VTraits<v_float32>::vlanes())
{
v_float32 t0 = vx_load(a + j) - vx_load(b + j);
v_float32 t1 = vx_load(a + j + v_float32::nlanes) - vx_load(b + j + v_float32::nlanes);
v_float32 t0 = v_sub(vx_load(a + j), vx_load(b + j));
v_float32 t1 = v_sub(vx_load(a + j + VTraits<v_float32>::vlanes()), vx_load(b + j + VTraits<v_float32>::vlanes()));
v_d0 = v_muladd(t0, t0, v_d0);
v_float32 t2 = vx_load(a + j + 2 * v_float32::nlanes) - vx_load(b + j + 2 * v_float32::nlanes);
v_float32 t2 = v_sub(vx_load(a + j + 2 * VTraits<v_float32>::vlanes()), vx_load(b + j + 2 * VTraits<v_float32>::vlanes()));
v_d1 = v_muladd(t1, t1, v_d1);
v_float32 t3 = vx_load(a + j + 3 * v_float32::nlanes) - vx_load(b + j + 3 * v_float32::nlanes);
v_float32 t3 = v_sub(vx_load(a + j + 3 * VTraits<v_float32>::vlanes()), vx_load(b + j + 3 * VTraits<v_float32>::vlanes()));
v_d2 = v_muladd(t2, t2, v_d2);
v_d3 = v_muladd(t3, t3, v_d3);
}
d = v_reduce_sum(v_d0 + v_d1 + v_d2 + v_d3);
d = v_reduce_sum(v_add(v_add(v_add(v_d0, v_d1), v_d2), v_d3));
#endif
for( ; j < n; j++ )
{
@@ -173,17 +173,17 @@ float normL2Sqr_(const float* a, const float* b, int n)
float normL1_(const float* a, const float* b, int n)
{
int j = 0; float d = 0.f;
#if CV_SIMD
#if (CV_SIMD || CV_SIMD_SCALABLE)
v_float32 v_d0 = vx_setzero_f32(), v_d1 = vx_setzero_f32();
v_float32 v_d2 = vx_setzero_f32(), v_d3 = vx_setzero_f32();
for (; j <= n - 4 * v_float32::nlanes; j += 4 * v_float32::nlanes)
for (; j <= n - 4 * VTraits<v_float32>::vlanes(); j += 4 * VTraits<v_float32>::vlanes())
{
v_d0 += v_absdiff(vx_load(a + j), vx_load(b + j));
v_d1 += v_absdiff(vx_load(a + j + v_float32::nlanes), vx_load(b + j + v_float32::nlanes));
v_d2 += v_absdiff(vx_load(a + j + 2 * v_float32::nlanes), vx_load(b + j + 2 * v_float32::nlanes));
v_d3 += v_absdiff(vx_load(a + j + 3 * v_float32::nlanes), vx_load(b + j + 3 * v_float32::nlanes));
v_d0 = v_add(v_d0, v_absdiff(vx_load(a + j), vx_load(b + j)));
v_d1 = v_add(v_d1, v_absdiff(vx_load(a + j + VTraits<v_float32>::vlanes()), vx_load(b + j + VTraits<v_float32>::vlanes())));
v_d2 = v_add(v_d2, v_absdiff(vx_load(a + j + 2 * VTraits<v_float32>::vlanes()), vx_load(b + j + 2 * VTraits<v_float32>::vlanes())));
v_d3 = v_add(v_d3, v_absdiff(vx_load(a + j + 3 * VTraits<v_float32>::vlanes()), vx_load(b + j + 3 * VTraits<v_float32>::vlanes())));
}
d = v_reduce_sum(v_d0 + v_d1 + v_d2 + v_d3);
d = v_reduce_sum(v_add(v_add(v_add(v_d0, v_d1), v_d2), v_d3));
#endif
for( ; j < n; j++ )
d += std::abs(a[j] - b[j]);
@@ -193,12 +193,12 @@ float normL1_(const float* a, const float* b, int n)
int normL1_(const uchar* a, const uchar* b, int n)
{
int j = 0, d = 0;
#if CV_SIMD
for (; j <= n - 4 * v_uint8::nlanes; j += 4 * v_uint8::nlanes)
#if (CV_SIMD || CV_SIMD_SCALABLE)
for (; j <= n - 4 * VTraits<v_uint8>::vlanes(); j += 4 * VTraits<v_uint8>::vlanes())
d += v_reduce_sad(vx_load(a + j), vx_load(b + j)) +
v_reduce_sad(vx_load(a + j + v_uint8::nlanes), vx_load(b + j + v_uint8::nlanes)) +
v_reduce_sad(vx_load(a + j + 2 * v_uint8::nlanes), vx_load(b + j + 2 * v_uint8::nlanes)) +
v_reduce_sad(vx_load(a + j + 3 * v_uint8::nlanes), vx_load(b + j + 3 * v_uint8::nlanes));
v_reduce_sad(vx_load(a + j + VTraits<v_uint8>::vlanes()), vx_load(b + j + VTraits<v_uint8>::vlanes())) +
v_reduce_sad(vx_load(a + j + 2 * VTraits<v_uint8>::vlanes()), vx_load(b + j + 2 * VTraits<v_uint8>::vlanes())) +
v_reduce_sad(vx_load(a + j + 3 * VTraits<v_uint8>::vlanes()), vx_load(b + j + 3 * VTraits<v_uint8>::vlanes()));
#endif
for( ; j < n; j++ )
d += std::abs(a[j] - b[j]);
modules/core/src/persistence.cpp
+20 -13
View File
@@ -322,16 +322,20 @@ int decodeSimpleFormat( const char* dt )
}
#if defined __i386__ || defined(_M_IX86) || defined __x86_64__ || defined(_M_X64)
#define CV_UNALIGNED_LITTLE_ENDIAN_MEM_ACCESS 1
#if defined __i386__ || defined(_M_IX86) || defined __x86_64__ || defined(_M_X64) || \
(defined (__LITTLE_ENDIAN__) && __LITTLE_ENDIAN__)
#define CV_LITTLE_ENDIAN_MEM_ACCESS 1
#else
#define CV_UNALIGNED_LITTLE_ENDIAN_MEM_ACCESS 0
#define CV_LITTLE_ENDIAN_MEM_ACCESS 0
#endif
static inline int readInt(const uchar* p)
{
#if CV_UNALIGNED_LITTLE_ENDIAN_MEM_ACCESS
return *(const int*)p;
// On little endian CPUs, both branches produce the same result. On big endian, only the else branch does.
#if CV_LITTLE_ENDIAN_MEM_ACCESS
int val;
memcpy(&val, p, sizeof(val));
return val;
#else
int val = (int)(p[0] | (p[1] << 8) | (p[2] << 16) | (p[3] << 24));
return val;
@@ -340,8 +344,11 @@ static inline int readInt(const uchar* p)
static inline double readReal(const uchar* p)
{
#if CV_UNALIGNED_LITTLE_ENDIAN_MEM_ACCESS
return *(const double*)p;
// On little endian CPUs, both branches produce the same result. On big endian, only the else branch does.
#if CV_LITTLE_ENDIAN_MEM_ACCESS
double val;
memcpy(&val, p, sizeof(val));
return val;
#else
unsigned val0 = (unsigned)(p[0] | (p[1] << 8) | (p[2] << 16) | (p[3] << 24));
unsigned val1 = (unsigned)(p[4] | (p[5] << 8) | (p[6] << 16) | (p[7] << 24));
@@ -353,9 +360,9 @@ static inline double readReal(const uchar* p)
static inline void writeInt(uchar* p, int ival)
{
#if CV_UNALIGNED_LITTLE_ENDIAN_MEM_ACCESS
int* ip = (int*)p;
*ip = ival;
// On little endian CPUs, both branches produce the same result. On big endian, only the else branch does.
#if CV_LITTLE_ENDIAN_MEM_ACCESS
memcpy(p, &ival, sizeof(ival));
#else
p[0] = (uchar)ival;
p[1] = (uchar)(ival >> 8);
@@ -366,9 +373,9 @@ static inline void writeInt(uchar* p, int ival)
static inline void writeReal(uchar* p, double fval)
{
#if CV_UNALIGNED_LITTLE_ENDIAN_MEM_ACCESS
double* fp = (double*)p;
*fp = fval;
// On little endian CPUs, both branches produce the same result. On big endian, only the else branch does.
#if CV_LITTLE_ENDIAN_MEM_ACCESS
memcpy(p, &fval, sizeof(fval));
#else
Cv64suf v;
v.f = fval;
modules/core/src/persistence_xml.cpp
+3 -3
View File
@@ -308,8 +308,8 @@ public:
if( !multiline )
{
ptr = fs->resizeWriteBuffer( ptr, len + 9 );
sprintf( ptr, "<!-- %s -->", comment );
ptr = fs->resizeWriteBuffer( ptr, len + 5+4+1 );
snprintf( ptr, len + 5+4+1, "<!-- %s -->", comment );
len = (int)strlen(ptr);
}
else
@@ -344,7 +344,7 @@ public:
fs->setBufferPtr(ptr);
ptr = fs->flush();
}
sprintf( ptr, "-->" );
strcpy( ptr, "-->" );
fs->setBufferPtr(ptr + 3);
fs->flush();
}
modules/core/src/split.simd.hpp
+10 -10
View File
@@ -15,12 +15,12 @@ void split64s(const int64* src, int64** dst, int len, int cn);
#ifndef CV_CPU_OPTIMIZATION_DECLARATIONS_ONLY
#if CV_SIMD
#if (CV_SIMD || CV_SIMD_SCALABLE)
// see the comments for vecmerge_ in merge.cpp
template<typename T, typename VecT> static void
vecsplit_( const T* src, T** dst, int len, int cn )
{
const int VECSZ = VecT::nlanes;
const int VECSZ = VTraits<VecT>::vlanes();
int i, i0 = 0;
T* dst0 = dst[0];
T* dst1 = dst[1];
@@ -177,8 +177,8 @@ split_( const T* src, T** dst, int len, int cn )
void split8u(const uchar* src, uchar** dst, int len, int cn )
{
CV_INSTRUMENT_REGION();
#if CV_SIMD
if( len >= v_uint8::nlanes && 2 <= cn && cn <= 4 )
#if (CV_SIMD || CV_SIMD_SCALABLE)
if( len >= VTraits<v_uint8>::vlanes() && 2 <= cn && cn <= 4 )
vecsplit_<uchar, v_uint8>(src, dst, len, cn);
else
#endif
@@ -188,8 +188,8 @@ void split8u(const uchar* src, uchar** dst, int len, int cn )
void split16u(const ushort* src, ushort** dst, int len, int cn )
{
CV_INSTRUMENT_REGION();
#if CV_SIMD
if( len >= v_uint16::nlanes && 2 <= cn && cn <= 4 )
#if (CV_SIMD || CV_SIMD_SCALABLE)
if( len >= VTraits<v_uint16>::vlanes() && 2 <= cn && cn <= 4 )
vecsplit_<ushort, v_uint16>(src, dst, len, cn);
else
#endif
@@ -199,8 +199,8 @@ void split16u(const ushort* src, ushort** dst, int len, int cn )
void split32s(const int* src, int** dst, int len, int cn )
{
CV_INSTRUMENT_REGION();
#if CV_SIMD
if( len >= v_uint32::nlanes && 2 <= cn && cn <= 4 )
#if (CV_SIMD || CV_SIMD_SCALABLE)
if( len >= VTraits<v_uint32>::vlanes() && 2 <= cn && cn <= 4 )
vecsplit_<int, v_int32>(src, dst, len, cn);
else
#endif
@@ -210,8 +210,8 @@ void split32s(const int* src, int** dst, int len, int cn )
void split64s(const int64* src, int64** dst, int len, int cn )
{
CV_INSTRUMENT_REGION();
#if CV_SIMD
if( len >= v_int64::nlanes && 2 <= cn && cn <= 4 )
#if (CV_SIMD || CV_SIMD_SCALABLE)
if( len >= VTraits<v_int64>::vlanes() && 2 <= cn && cn <= 4 )
vecsplit_<int64, v_int64>(src, dst, len, cn);
else
#endif
modules/core/src/stat.simd.hpp
+6 -20
View File
@@ -33,11 +33,11 @@ int normHamming(const uchar* a, int n)
int i = 0;
int result = 0;
#if CV_SIMD && CV_SIMD_WIDTH > 16
#if (CV_SIMD || CV_SIMD_SCALABLE)
{
v_uint64 t = vx_setzero_u64();
for (; i <= n - v_uint8::nlanes; i += v_uint8::nlanes)
t += v_popcount(v_reinterpret_as_u64(vx_load(a + i)));
for (; i <= n - VTraits<v_uint8>::vlanes(); i += VTraits<v_uint8>::vlanes())
t = v_add(t, v_popcount(v_reinterpret_as_u64(vx_load(a + i))));
result = (int)v_reduce_sum(t);
vx_cleanup();
}
@@ -56,13 +56,6 @@ int normHamming(const uchar* a, int n)
result += CV_POPCNT_U32(*(uint*)(a + i));
}
}
#elif CV_SIMD
{
v_uint64x2 t = v_setzero_u64();
for(; i <= n - v_uint8x16::nlanes; i += v_uint8x16::nlanes)
t += v_popcount(v_reinterpret_as_u64(v_load(a + i)));
result += (int)v_reduce_sum(t);
}
#endif
#if CV_ENABLE_UNROLLED
for(; i <= n - 4; i += 4)
@@ -85,11 +78,11 @@ int normHamming(const uchar* a, const uchar* b, int n)
int i = 0;
int result = 0;
#if CV_SIMD && CV_SIMD_WIDTH > 16
#if (CV_SIMD || CV_SIMD_SCALABLE)
{
v_uint64 t = vx_setzero_u64();
for (; i <= n - v_uint8::nlanes; i += v_uint8::nlanes)
t += v_popcount(v_reinterpret_as_u64(vx_load(a + i) ^ vx_load(b + i)));
for (; i <= n - VTraits<v_uint8>::vlanes(); i += VTraits<v_uint8>::vlanes())
t = v_add(t, v_popcount(v_reinterpret_as_u64(v_xor(vx_load(a + i), vx_load(b + i)))));
result += (int)v_reduce_sum(t);
}
#endif
@@ -107,13 +100,6 @@ int normHamming(const uchar* a, const uchar* b, int n)
result += CV_POPCNT_U32(*(uint*)(a + i) ^ *(uint*)(b + i));
}
}
#elif CV_SIMD
{
v_uint64x2 t = v_setzero_u64();
for(; i <= n - v_uint8x16::nlanes; i += v_uint8x16::nlanes)
t += v_popcount(v_reinterpret_as_u64(v_load(a + i) ^ v_load(b + i)));
result += (int)v_reduce_sum(t);
}
#endif
#if CV_ENABLE_UNROLLED
for(; i <= n - 4; i += 4)
modules/core/src/sum.dispatch.cpp
+2
View File
@@ -10,11 +10,13 @@
#include "sum.simd.hpp"
#include "sum.simd_declarations.hpp" // defines CV_CPU_DISPATCH_MODES_ALL=AVX2,...,BASELINE based on CMakeLists.txt content
#ifndef OPENCV_IPP_SUM
#undef HAVE_IPP
#undef CV_IPP_RUN_FAST
#define CV_IPP_RUN_FAST(f, ...)
#undef CV_IPP_RUN
#define CV_IPP_RUN(c, f, ...)
#endif // OPENCV_IPP_SUM
namespace cv
{
modules/core/src/sum.simd.hpp
+62 -62
View File
@@ -22,7 +22,7 @@ struct Sum_SIMD
}
};
#if CV_SIMD
#if (CV_SIMD || CV_SIMD_SCALABLE)
template <>
struct Sum_SIMD<uchar, int>
@@ -36,41 +36,41 @@ struct Sum_SIMD<uchar, int>
int x = 0;
v_uint32 v_sum = vx_setzero_u32();
int len0 = len & -v_uint8::nlanes;
int len0 = len & -VTraits<v_uint8>::vlanes();
while (x < len0)
{
const int len_tmp = min(x + 256*v_uint16::nlanes, len0);
const int len_tmp = min(x + 256*VTraits<v_uint16>::vlanes(), len0);
v_uint16 v_sum16 = vx_setzero_u16();
for (; x < len_tmp; x += v_uint8::nlanes)
for (; x < len_tmp; x += VTraits<v_uint8>::vlanes())
{
v_uint16 v_src0, v_src1;
v_expand(vx_load(src0 + x), v_src0, v_src1);
v_sum16 += v_src0 + v_src1;
v_sum16 = v_add(v_sum16, v_add(v_src0, v_src1));
}
v_uint32 v_half0, v_half1;
v_expand(v_sum16, v_half0, v_half1);
v_sum += v_half0 + v_half1;
v_sum = v_add(v_sum, v_add(v_half0, v_half1));
}
if (x <= len - v_uint16::nlanes)
if (x <= len - VTraits<v_uint16>::vlanes())
{
v_uint32 v_half0, v_half1;
v_expand(vx_load_expand(src0 + x), v_half0, v_half1);
v_sum += v_half0 + v_half1;
x += v_uint16::nlanes;
v_sum = v_add(v_sum, v_add(v_half0, v_half1));
x += VTraits<v_uint16>::vlanes();
}
if (x <= len - v_uint32::nlanes)
if (x <= len - VTraits<v_uint32>::vlanes())
{
v_sum += vx_load_expand_q(src0 + x);
x += v_uint32::nlanes;
v_sum = v_add(v_sum, vx_load_expand_q(src0 + x));
x += VTraits<v_uint32>::vlanes();
}
if (cn == 1)
*dst += v_reduce_sum(v_sum);
else
{
uint32_t CV_DECL_ALIGNED(CV_SIMD_WIDTH) ar[v_uint32::nlanes];
uint32_t CV_DECL_ALIGNED(CV_SIMD_WIDTH) ar[VTraits<v_uint32>::max_nlanes];
v_store_aligned(ar, v_sum);
for (int i = 0; i < v_uint32::nlanes; ++i)
for (int i = 0; i < VTraits<v_uint32>::vlanes(); ++i)
dst[i % cn] += ar[i];
}
v_cleanup();
@@ -91,41 +91,41 @@ struct Sum_SIMD<schar, int>
int x = 0;
v_int32 v_sum = vx_setzero_s32();
int len0 = len & -v_int8::nlanes;
int len0 = len & -VTraits<v_int8>::vlanes();
while (x < len0)
{
const int len_tmp = min(x + 256*v_int16::nlanes, len0);
const int len_tmp = min(x + 256*VTraits<v_int16>::vlanes(), len0);
v_int16 v_sum16 = vx_setzero_s16();
for (; x < len_tmp; x += v_int8::nlanes)
for (; x < len_tmp; x += VTraits<v_int8>::vlanes())
{
v_int16 v_src0, v_src1;
v_expand(vx_load(src0 + x), v_src0, v_src1);
v_sum16 += v_src0 + v_src1;
v_sum16 = v_add(v_sum16, v_add(v_src0, v_src1));
}
v_int32 v_half0, v_half1;
v_expand(v_sum16, v_half0, v_half1);
v_sum += v_half0 + v_half1;
v_sum = v_add(v_sum, v_add(v_half0, v_half1));
}
if (x <= len - v_int16::nlanes)
if (x <= len - VTraits<v_int16>::vlanes())
{
v_int32 v_half0, v_half1;
v_expand(vx_load_expand(src0 + x), v_half0, v_half1);
v_sum += v_half0 + v_half1;
x += v_int16::nlanes;
v_sum = v_add(v_sum, v_add(v_half0, v_half1));
x += VTraits<v_int16>::vlanes();
}
if (x <= len - v_int32::nlanes)
if (x <= len - VTraits<v_int32>::vlanes())
{
v_sum += vx_load_expand_q(src0 + x);
x += v_int32::nlanes;
v_sum = v_add(v_sum, vx_load_expand_q(src0 + x));
x += VTraits<v_int32>::vlanes();
}
if (cn == 1)
*dst += v_reduce_sum(v_sum);
else
{
int32_t CV_DECL_ALIGNED(CV_SIMD_WIDTH) ar[v_int32::nlanes];
int32_t CV_DECL_ALIGNED(CV_SIMD_WIDTH) ar[VTraits<v_int32>::max_nlanes];
v_store_aligned(ar, v_sum);
for (int i = 0; i < v_int32::nlanes; ++i)
for (int i = 0; i < VTraits<v_int32>::vlanes(); ++i)
dst[i % cn] += ar[i];
}
v_cleanup();
@@ -146,25 +146,25 @@ struct Sum_SIMD<ushort, int>
int x = 0;
v_uint32 v_sum = vx_setzero_u32();
for (; x <= len - v_uint16::nlanes; x += v_uint16::nlanes)
for (; x <= len - VTraits<v_uint16>::vlanes(); x += VTraits<v_uint16>::vlanes())
{
v_uint32 v_src0, v_src1;
v_expand(vx_load(src0 + x), v_src0, v_src1);
v_sum += v_src0 + v_src1;
v_sum = v_add(v_sum, v_add(v_src0, v_src1));
}
if (x <= len - v_uint32::nlanes)
if (x <= len - VTraits<v_uint32>::vlanes())
{
v_sum += vx_load_expand(src0 + x);
x += v_uint32::nlanes;
v_sum = v_add(v_sum, vx_load_expand(src0 + x));
x += VTraits<v_uint32>::vlanes();
}
if (cn == 1)
*dst += v_reduce_sum(v_sum);
else
{
uint32_t CV_DECL_ALIGNED(CV_SIMD_WIDTH) ar[v_uint32::nlanes];
uint32_t CV_DECL_ALIGNED(CV_SIMD_WIDTH) ar[VTraits<v_uint32>::max_nlanes];
v_store_aligned(ar, v_sum);
for (int i = 0; i < v_uint32::nlanes; ++i)
for (int i = 0; i < VTraits<v_uint32>::vlanes(); ++i)
dst[i % cn] += ar[i];
}
v_cleanup();
@@ -185,25 +185,25 @@ struct Sum_SIMD<short, int>
int x = 0;
v_int32 v_sum = vx_setzero_s32();
for (; x <= len - v_int16::nlanes; x += v_int16::nlanes)
for (; x <= len - VTraits<v_int16>::vlanes(); x += VTraits<v_int16>::vlanes())
{
v_int32 v_src0, v_src1;
v_expand(vx_load(src0 + x), v_src0, v_src1);
v_sum += v_src0 + v_src1;
v_sum = v_add(v_sum, v_add(v_src0, v_src1));
}
if (x <= len - v_int32::nlanes)
if (x <= len - VTraits<v_int32>::vlanes())
{
v_sum += vx_load_expand(src0 + x);
x += v_int32::nlanes;
v_sum = v_add(v_sum, vx_load_expand(src0 + x));
x += VTraits<v_int32>::vlanes();
}
if (cn == 1)
*dst += v_reduce_sum(v_sum);
else
{
int32_t CV_DECL_ALIGNED(CV_SIMD_WIDTH) ar[v_int32::nlanes];
int32_t CV_DECL_ALIGNED(CV_SIMD_WIDTH) ar[VTraits<v_int32>::max_nlanes];
v_store_aligned(ar, v_sum);
for (int i = 0; i < v_int32::nlanes; ++i)
for (int i = 0; i < VTraits<v_int32>::vlanes(); ++i)
dst[i % cn] += ar[i];
}
v_cleanup();
@@ -212,7 +212,7 @@ struct Sum_SIMD<short, int>
}
};
#if CV_SIMD_64F
#if (CV_SIMD_64F || CV_SIMD_SCALABLE_64F)
template <>
struct Sum_SIMD<int, double>
{
@@ -226,24 +226,24 @@ struct Sum_SIMD<int, double>
v_float64 v_sum0 = vx_setzero_f64();
v_float64 v_sum1 = vx_setzero_f64();
for (; x <= len - 2 * v_int32::nlanes; x += 2 * v_int32::nlanes)
for (; x <= len - 2 * VTraits<v_int32>::vlanes(); x += 2 * VTraits<v_int32>::vlanes())
{
v_int32 v_src0 = vx_load(src0 + x);
v_int32 v_src1 = vx_load(src0 + x + v_int32::nlanes);
v_sum0 += v_cvt_f64(v_src0) + v_cvt_f64(v_src1);
v_sum1 += v_cvt_f64_high(v_src0) + v_cvt_f64_high(v_src1);
v_int32 v_src1 = vx_load(src0 + x + VTraits<v_int32>::vlanes());
v_sum0 = v_add(v_sum0, v_add(v_cvt_f64(v_src0), v_cvt_f64(v_src1)));
v_sum1 = v_add(v_sum1, v_add(v_cvt_f64_high(v_src0), v_cvt_f64_high(v_src1)));
}
#if CV_SIMD256 || CV_SIMD512
double CV_DECL_ALIGNED(CV_SIMD_WIDTH) ar[v_float64::nlanes];
v_store_aligned(ar, v_sum0 + v_sum1);
for (int i = 0; i < v_float64::nlanes; ++i)
double CV_DECL_ALIGNED(CV_SIMD_WIDTH) ar[VTraits<v_float64>::max_nlanes];
v_store_aligned(ar, v_add(v_sum0, v_sum1));
for (int i = 0; i < VTraits<v_float64>::vlanes(); ++i)
dst[i % cn] += ar[i];
#else
double CV_DECL_ALIGNED(CV_SIMD_WIDTH) ar[2 * v_float64::nlanes];
double CV_DECL_ALIGNED(CV_SIMD_WIDTH) ar[2 * VTraits<v_float64>::max_nlanes];
v_store_aligned(ar, v_sum0);
v_store_aligned(ar + v_float64::nlanes, v_sum1);
for (int i = 0; i < 2 * v_float64::nlanes; ++i)
v_store_aligned(ar + VTraits<v_float64>::vlanes(), v_sum1);
for (int i = 0; i < 2 * VTraits<v_float64>::vlanes(); ++i)
dst[i % cn] += ar[i];
#endif
v_cleanup();
@@ -265,24 +265,24 @@ struct Sum_SIMD<float, double>
v_float64 v_sum0 = vx_setzero_f64();
v_float64 v_sum1 = vx_setzero_f64();
for (; x <= len - 2 * v_float32::nlanes; x += 2 * v_float32::nlanes)
for (; x <= len - 2 * VTraits<v_float32>::vlanes(); x += 2 * VTraits<v_float32>::vlanes())
{
v_float32 v_src0 = vx_load(src0 + x);
v_float32 v_src1 = vx_load(src0 + x + v_float32::nlanes);
v_sum0 += v_cvt_f64(v_src0) + v_cvt_f64(v_src1);
v_sum1 += v_cvt_f64_high(v_src0) + v_cvt_f64_high(v_src1);
v_float32 v_src1 = vx_load(src0 + x + VTraits<v_float32>::vlanes());
v_sum0 = v_add(v_sum0, v_add(v_cvt_f64(v_src0), v_cvt_f64(v_src1)));
v_sum1 = v_add(v_sum1, v_add(v_cvt_f64_high(v_src0), v_cvt_f64_high(v_src1)));
}
#if CV_SIMD256 || CV_SIMD512
double CV_DECL_ALIGNED(CV_SIMD_WIDTH) ar[v_float64::nlanes];
v_store_aligned(ar, v_sum0 + v_sum1);
for (int i = 0; i < v_float64::nlanes; ++i)
double CV_DECL_ALIGNED(CV_SIMD_WIDTH) ar[VTraits<v_float64>::max_nlanes];
v_store_aligned(ar, v_add(v_sum0, v_sum1));
for (int i = 0; i < VTraits<v_float64>::vlanes(); ++i)
dst[i % cn] += ar[i];
#else
double CV_DECL_ALIGNED(CV_SIMD_WIDTH) ar[2 * v_float64::nlanes];
double CV_DECL_ALIGNED(CV_SIMD_WIDTH) ar[2 * VTraits<v_float64>::max_nlanes];
v_store_aligned(ar, v_sum0);
v_store_aligned(ar + v_float64::nlanes, v_sum1);
for (int i = 0; i < 2 * v_float64::nlanes; ++i)
v_store_aligned(ar + VTraits<v_float64>::vlanes(), v_sum1);
for (int i = 0; i < 2 * VTraits<v_float64>::vlanes(); ++i)
dst[i % cn] += ar[i];
#endif
v_cleanup();
modules/core/src/utils/filesystem.cpp
+3 -3
View File
@@ -34,7 +34,7 @@
#include <errno.h>
#include <io.h>
#include <stdio.h>
#elif defined __linux__ || defined __APPLE__ || defined __HAIKU__ || defined __FreeBSD__
#elif defined __linux__ || defined __APPLE__ || defined __HAIKU__ || defined __FreeBSD__ || defined __GNU__
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
@@ -343,7 +343,7 @@ private:
Impl& operator=(const Impl&); // disabled
};
#elif defined __linux__ || defined __APPLE__ || defined __HAIKU__ || defined __FreeBSD__
#elif defined __linux__ || defined __APPLE__ || defined __HAIKU__ || defined __FreeBSD__ || defined __GNU__
struct FileLock::Impl
{
@@ -457,7 +457,7 @@ cv::String getCacheDirectory(const char* sub_directory_name, const char* configu
default_cache_path = "/tmp/";
CV_LOG_WARNING(NULL, "Using world accessible cache directory. This may be not secure: " << default_cache_path);
}
#elif defined __linux__ || defined __HAIKU__ || defined __FreeBSD__
#elif defined __linux__ || defined __HAIKU__ || defined __FreeBSD__ || defined __GNU__
// https://specifications.freedesktop.org/basedir-spec/basedir-spec-latest.html
if (default_cache_path.empty())
{