diff --git a/modules/core/CMakeLists.txt b/modules/core/CMakeLists.txt index 0d310864f5..aea0b6de66 100644 --- a/modules/core/CMakeLists.txt +++ b/modules/core/CMakeLists.txt @@ -5,7 +5,7 @@ ocv_add_dispatched_file(stat SSE4_2 AVX2 AVX512_SKX AVX512_ICL LASX) ocv_add_dispatched_file(arithm SSE2 SSE4_1 AVX2 VSX3 LASX) ocv_add_dispatched_file(convert SSE2 AVX2 VSX3 LASX) ocv_add_dispatched_file(convert_scale SSE2 AVX2 LASX) -ocv_add_dispatched_file(count_non_zero SSE2 AVX2 LASX) +ocv_add_dispatched_file(count_non_zero SSE2 AVX2 AVX512_SKX AVX512_ICL LASX) ocv_add_dispatched_file(has_non_zero SSE2 AVX2 LASX ) ocv_add_dispatched_file(matmul SSE2 SSE4_1 AVX2 AVX512_SKX NEON_DOTPROD LASX) ocv_add_dispatched_file(mean SSE2 AVX2 AVX512_SKX AVX512_ICL LASX) diff --git a/modules/core/src/count_non_zero.simd.hpp b/modules/core/src/count_non_zero.simd.hpp index c93f31cce6..f40c356c09 100644 --- a/modules/core/src/count_non_zero.simd.hpp +++ b/modules/core/src/count_non_zero.simd.hpp @@ -29,16 +29,138 @@ static int countNonZero_(const T* src, int len ) return nz; } -static int countNonZero8u( const void* src_ptr, int len ) -{ - const uchar* src = static_cast(src_ptr); - int i=0, nz = 0; #if (CV_SIMD || CV_SIMD_SCALABLE) - int len0 = len & -VTraits::vlanes(); + +#if defined(CV_CPU_COMPILE_AVX512_SKX) || defined(CV_CPU_COMPILE_AVX512_ICL) +#define CV_COUNTNONZERO_AVX512 1 +#else +#define CV_COUNTNONZERO_AVX512 0 +#endif + +#if CV_COUNTNONZERO_AVX512 + +static inline int cnz_popcount32(unsigned x) +{ +#if defined(__GNUC__) || defined(__clang__) + return __builtin_popcount(x); +#else + int c = 0; + for (; x; x &= x - 1) + ++c; + return c; +#endif +} + +static inline int cnz_popcount64(uint64 x) +{ +#if defined(__GNUC__) || defined(__clang__) + return (int)__builtin_popcountll((unsigned long long)x); +#else + int c = 0; + for (; x; x &= x - 1) + ++c; + return c; +#endif +} + +static inline int cnz_lane_pop(int m) { return cnz_popcount32((unsigned)m); } +static inline int cnz_lane_pop(int64 m) { return cnz_popcount64((uint64)m); } + +template static inline int cnz_pop_ne(const VT& v, const VT& z); + +template<> inline int cnz_pop_ne(const v_uint8& v, const v_uint8& z) +{ return cnz_lane_pop(v_signmask(v_reinterpret_as_s8(v_ne(v, z)))); } + +template<> inline int cnz_pop_ne(const v_uint16& v, const v_uint16& z) +{ return cnz_lane_pop(v_signmask(v_reinterpret_as_s16(v_ne(v, z)))); } + +template<> inline int cnz_pop_ne(const v_int32& v, const v_int32& z) +{ return cnz_lane_pop(v_signmask(v_ne(v, z))); } + +template<> inline int cnz_pop_ne(const v_float32& v, const v_float32& z) +{ return cnz_lane_pop(v_signmask(v_ne(v, z))); } + +#if (CV_SIMD_64F || CV_SIMD_SCALABLE_64F) +template<> inline int cnz_pop_ne(const v_float64& v, const v_float64& z) +{ return cnz_lane_pop(v_signmask(v_ne(v, z))); } +#endif + +#endif // CV_COUNTNONZERO_AVX512 + +struct cnz_pack16u +{ + static inline v_int8 eq(const ushort* src, int k, v_uint16 z) + { + const int n = VTraits::vlanes(); + return v_pack(v_reinterpret_as_s16(v_eq(vx_load(src + k), z)), + v_reinterpret_as_s16(v_eq(vx_load(src + k + n), z))); + } +}; + +struct cnz_pack32s +{ + static inline v_int8 eq(const int* src, int k, v_int32 z) + { + const int n = VTraits::vlanes(); + return v_pack(v_pack(v_eq(vx_load(src + k), z), v_eq(vx_load(src + k + n), z)), + v_pack(v_eq(vx_load(src + k + 2 * n), z), v_eq(vx_load(src + k + 3 * n), z))); + } +}; + +struct cnz_pack32f +{ + static inline v_int8 eq(const float* src, int k, v_float32 z) + { + const int n = VTraits::vlanes(); + return v_pack(v_pack(v_reinterpret_as_s32(v_eq(vx_load(src + k), z)), + v_reinterpret_as_s32(v_eq(vx_load(src + k + n), z))), + v_pack(v_reinterpret_as_s32(v_eq(vx_load(src + k + 2 * n), z)), + v_reinterpret_as_s32(v_eq(vx_load(src + k + 3 * n), z)))); + } +}; + +template +static int countNonZeroVT_batched(const ST* src, int len, const ZVT& z) +{ + int i = 0, nz = 0; + const int vl8 = VTraits::vlanes(); + const int len0 = len & -vl8; + v_int8 v_one = vx_setall_s8(1); + v_int32 v_sum32 = vx_setzero_s32(); + + while (i < len0) + { + v_int16 v_sum16 = vx_setzero_s16(); + int j = i; + while (j < std::min(len0, i + 32766 * VTraits::vlanes())) + { + v_int8 v_sum8 = vx_setzero_s8(); + int k = j; + for (; k < std::min(len0, j + 127 * vl8); k += vl8) + v_sum8 = v_add(v_sum8, v_and(v_one, PackOp::eq(src, k, z))); + v_int16 part1, part2; + v_expand(v_sum8, part1, part2); + v_sum16 = v_add(v_sum16, v_add(part1, part2)); + j = k; + } + v_int32 part1, part2; + v_expand(v_sum16, part1, part2); + v_sum32 = v_add(v_sum32, v_add(part1, part2)); + i = j; + } + nz = i - v_reduce_sum(v_sum32); + v_cleanup(); + return nz + countNonZero_(src + i, len - i); +} + +static int countNonZeroVT_u8(const uchar* src, int len) +{ + int i = 0, nz = 0; + const int len0 = len & -VTraits::vlanes(); v_uint8 v_zero = vx_setzero_u8(); v_uint8 v_one = vx_setall_u8(1); - v_uint32 v_sum32 = vx_setzero_u32(); + while (i < len0) { v_uint16 v_sum16 = vx_setzero_u16(); @@ -61,143 +183,145 @@ static int countNonZero8u( const void* src_ptr, int len ) } nz = i - v_reduce_sum(v_sum32); v_cleanup(); -#endif - for( ; i < len; i++ ) + for (; i < len; i++) nz += src[i] != 0; return nz; } -static int countNonZero16u( const void* src_ptr, int len ) -{ - const ushort* src = static_cast(src_ptr); - int i = 0, nz = 0; -#if (CV_SIMD || CV_SIMD_SCALABLE) - int len0 = len & -VTraits::vlanes(); - v_uint16 v_zero = vx_setzero_u16(); - v_int8 v_one = vx_setall_s8(1); - - v_int32 v_sum32 = vx_setzero_s32(); - while (i < len0) - { - v_int16 v_sum16 = vx_setzero_s16(); - int j = i; - while (j < std::min(len0, i + 32766 * VTraits::vlanes())) - { - v_int8 v_sum8 = vx_setzero_s8(); - int k = j; - for (; k < std::min(len0, j + 127 * VTraits::vlanes()); k += VTraits::vlanes()) - v_sum8 = v_add(v_sum8, v_and(v_one, v_pack(v_reinterpret_as_s16(v_eq(vx_load(src + k), v_zero)), v_reinterpret_as_s16(v_eq(vx_load(src + k + VTraits::vlanes()), v_zero))))); - v_int16 part1, part2; - v_expand(v_sum8, part1, part2); - v_sum16 = v_add(v_sum16, v_add(part1, part2)); - j = k; - } - v_int32 part1, part2; - v_expand(v_sum16, part1, part2); - v_sum32 = v_add(v_sum32, v_add(part1, part2)); - i = j; - } - nz = i - v_reduce_sum(v_sum32); - v_cleanup(); -#endif - return nz + countNonZero_(src + i, len - i); -} - -static int countNonZero32s( const void* src_ptr, int len ) -{ - const int* src = static_cast(src_ptr); - int i = 0, nz = 0; -#if (CV_SIMD || CV_SIMD_SCALABLE) - int len0 = len & -VTraits::vlanes(); - v_int32 v_zero = vx_setzero_s32(); - v_int8 v_one = vx_setall_s8(1); - - v_int32 v_sum32 = vx_setzero_s32(); - while (i < len0) - { - v_int16 v_sum16 = vx_setzero_s16(); - int j = i; - while (j < std::min(len0, i + 32766 * VTraits::vlanes())) - { - v_int8 v_sum8 = vx_setzero_s8(); - int k = j; - for (; k < std::min(len0, j + 127 * VTraits::vlanes()); k += VTraits::vlanes()) - v_sum8 = v_add(v_sum8, v_and(v_one, v_pack(v_pack(v_eq(vx_load(src + k), v_zero), v_eq(vx_load(src + k + VTraits::vlanes()), v_zero)), v_pack(v_eq(vx_load(src + k + 2 * VTraits::vlanes()), v_zero), v_eq(vx_load(src + k + 3 * VTraits::vlanes()), v_zero))))); - v_int16 part1, part2; - v_expand(v_sum8, part1, part2); - v_sum16 = v_add(v_sum16, v_add(part1, part2)); - j = k; - } - v_int32 part1, part2; - v_expand(v_sum16, part1, part2); - v_sum32 = v_add(v_sum32, v_add(part1, part2)); - i = j; - } - nz = i - v_reduce_sum(v_sum32); - v_cleanup(); -#endif - return nz + countNonZero_(src + i, len - i); -} - -static int countNonZero32f( const void* src_ptr, int len ) -{ - const float* src = static_cast(src_ptr); - int i = 0, nz = 0; -#if (CV_SIMD || CV_SIMD_SCALABLE) - int len0 = len & -VTraits::vlanes(); - v_float32 v_zero = vx_setzero_f32(); - v_int8 v_one = vx_setall_s8(1); - - v_int32 v_sum32 = vx_setzero_s32(); - while (i < len0) - { - v_int16 v_sum16 = vx_setzero_s16(); - int j = i; - while (j < std::min(len0, i + 32766 * VTraits::vlanes())) - { - v_int8 v_sum8 = vx_setzero_s8(); - int k = j; - for (; k < std::min(len0, j + 127 * VTraits::vlanes()); k += VTraits::vlanes()) - v_sum8 = v_add(v_sum8, v_and(v_one, v_pack(v_pack(v_reinterpret_as_s32(v_eq(vx_load(src + k), v_zero)), v_reinterpret_as_s32(v_eq(vx_load(src + k + VTraits::vlanes()), v_zero))), v_pack(v_reinterpret_as_s32(v_eq(vx_load(src + k + 2 * VTraits::vlanes()), v_zero)), v_reinterpret_as_s32(v_eq(vx_load(src + k + 3 * VTraits::vlanes()), v_zero)))))); - v_int16 part1, part2; - v_expand(v_sum8, part1, part2); - v_sum16 = v_add(v_sum16, v_add(part1, part2)); - j = k; - } - v_int32 part1, part2; - v_expand(v_sum16, part1, part2); - v_sum32 = v_add(v_sum32, v_add(part1, part2)); - i = j; - } - nz = i - v_reduce_sum(v_sum32); - v_cleanup(); -#endif - return nz + countNonZero_(src + i, len - i); -} - -static int countNonZero64f( const void* src_ptr, int len ) -{ - const double* src = static_cast(src_ptr); - int nz = 0, i = 0; #if (CV_SIMD_64F || CV_SIMD_SCALABLE_64F) +static int countNonZeroVT_f64(const double* src, int len) +{ + int nz = 0, i = 0; v_int64 sum1 = vx_setzero_s64(); v_int64 sum2 = vx_setzero_s64(); v_float64 zero = vx_setzero_f64(); - int step = VTraits::vlanes() * 2; - int len0 = len & -step; + const int step = VTraits::vlanes() * 2; + const int len0 = len & -step; - for(i = 0; i < len0; i += step ) - { + for (i = 0; i < len0; i += step) + { sum1 = v_add(sum1, v_reinterpret_as_s64(v_eq(vx_load(&src[i]), zero))); sum2 = v_add(sum2, v_reinterpret_as_s64(v_eq(vx_load(&src[i + step / 2]), zero))); - } + } - // N.B the value is incremented by -1 (0xF...F) for each value nz = i + (int)v_reduce_sum(v_add(sum1, sum2)); v_cleanup(); -#endif return nz + countNonZero_(src + i, len - i); } +#endif + +template struct cnz_vt_traits; + +template<> +struct cnz_vt_traits +{ + typedef uchar ST; + static inline v_uint8 zero() { return vx_setzero_u8(); } + static int legacy(const uchar* src, int len, const v_uint8&) { return countNonZeroVT_u8(src, len); } +}; + +template<> +struct cnz_vt_traits +{ + typedef ushort ST; + static inline v_uint16 zero() { return vx_setzero_u16(); } + static int legacy(const ushort* src, int len, const v_uint16& z) + { return countNonZeroVT_batched(src, len, z); } +}; + +template<> +struct cnz_vt_traits +{ + typedef int ST; + static inline v_int32 zero() { return vx_setzero_s32(); } + static int legacy(const int* src, int len, const v_int32& z) + { return countNonZeroVT_batched(src, len, z); } +}; + +template<> +struct cnz_vt_traits +{ + typedef float ST; + static inline v_float32 zero() { return vx_setzero_f32(); } + static int legacy(const float* src, int len, const v_float32& z) + { return countNonZeroVT_batched(src, len, z); } +}; + +#if (CV_SIMD_64F || CV_SIMD_SCALABLE_64F) +template<> +struct cnz_vt_traits +{ + typedef double ST; + static inline v_float64 zero() { return vx_setzero_f64(); } + static int legacy(const double* src, int len, const v_float64&) + { return countNonZeroVT_f64(src, len); } +}; +#endif + +template +static int countNonZeroVT(const ST* src, int len, const VT& z) +{ +#if CV_COUNTNONZERO_AVX512 + const int nlanes = VTraits::vlanes(); + const int step = nlanes * 2; + const int len0 = len & -step; + int i = 0, nz = 0; + + for (; i < len0; i += step) + { + nz += cnz_pop_ne(vx_load(src + i), z); + nz += cnz_pop_ne(vx_load(src + i + nlanes), z); + } + for (; i <= len - nlanes; i += nlanes) + nz += cnz_pop_ne(vx_load(src + i), z); + + v_cleanup(); + return nz + countNonZero_(src + i, len - i); +#else + return cnz_vt_traits::legacy(src, len, z); +#endif +} + +template +static int countNonZeroSimd(const void* src_ptr, int len) +{ + typedef cnz_vt_traits traits; + const typename traits::ST* src = static_cast(src_ptr); + return countNonZeroVT(src, len, traits::zero()); +} + +#endif // CV_SIMD + +template +static int countNonZeroDepth(const void* src_ptr, int len) +{ +#if (CV_SIMD || CV_SIMD_SCALABLE) + return countNonZeroSimd(src_ptr, len); +#else + return countNonZero_(static_cast(src_ptr), len); +#endif +} + +static int countNonZero8u( const void* src_ptr, int len ) +{ return countNonZeroDepth(src_ptr, len); } + +static int countNonZero16u( const void* src_ptr, int len ) +{ return countNonZeroDepth(src_ptr, len); } + +static int countNonZero32s( const void* src_ptr, int len ) +{ return countNonZeroDepth(src_ptr, len); } + +static int countNonZero32f( const void* src_ptr, int len ) +{ return countNonZeroDepth(src_ptr, len); } + +static int countNonZero64f( const void* src_ptr, int len ) +{ +#if (CV_SIMD || CV_SIMD_SCALABLE) && (CV_SIMD_64F || CV_SIMD_SCALABLE_64F) + return countNonZeroSimd(src_ptr, len); +#else + return countNonZero_(static_cast(src_ptr), len); +#endif +} CountNonZeroFunc getCountNonZeroTab(int depth) {