diff --git a/modules/core/CMakeLists.txt b/modules/core/CMakeLists.txt index ecc27202d5..27885f4b7d 100644 --- a/modules/core/CMakeLists.txt +++ b/modules/core/CMakeLists.txt @@ -5,7 +5,7 @@ ocv_add_dispatched_file(stat SSE4_2 AVX2 LASX) ocv_add_dispatched_file(arithm SSE2 SSE4_1 AVX2 VSX3 LASX) ocv_add_dispatched_file(convert SSE2 AVX2 VSX3 LASX) ocv_add_dispatched_file(convert_scale SSE2 AVX2 LASX) -ocv_add_dispatched_file(count_non_zero SSE2 AVX2 LASX) +ocv_add_dispatched_file(count_non_zero SSE2 AVX2 AVX512_SKX AVX512_ICL LASX) ocv_add_dispatched_file(has_non_zero SSE2 AVX2 LASX ) ocv_add_dispatched_file(matmul SSE2 SSE4_1 AVX2 AVX512_SKX NEON_DOTPROD LASX) ocv_add_dispatched_file(mean SSE2 AVX2 AVX512_SKX AVX512_ICL LASX) diff --git a/modules/core/src/count_non_zero.simd.hpp b/modules/core/src/count_non_zero.simd.hpp index 983ade0308..56ad79475e 100644 --- a/modules/core/src/count_non_zero.simd.hpp +++ b/modules/core/src/count_non_zero.simd.hpp @@ -30,13 +30,67 @@ static int countNonZero_(const T* src, int len ) #define SIMD_ONLY(expr) #endif -#undef DEFINE_NONZERO_FUNC -#define DEFINE_NONZERO_FUNC(funcname, suffix, ssuffix, T, VT, ST, cmp_op, add_op, update_sum, scalar_cmp_op) \ -static int funcname( const void* src_ptr, int len ) \ -{ \ - const T* src = static_cast(src_ptr); \ - int i = 0, nz = 0; \ - SIMD_ONLY( \ +// AVX-512 exposes wide per-lane mask registers, so counting zero lanes with +// v_signmask()+popcount is cheaper than the batched add-based reduction. +// Narrower SIMD (AVX2, NEON, LASX, ...) keeps the legacy batched kernel to +// avoid regressions on smaller vector widths. +#undef CV_COUNTNONZERO_AVX512 +#if defined(CV_CPU_COMPILE_AVX512_SKX) || defined(CV_CPU_COMPILE_AVX512_ICL) +#define CV_COUNTNONZERO_AVX512 1 +#else +#define CV_COUNTNONZERO_AVX512 0 +#endif + +#if CV_COUNTNONZERO_AVX512 +static inline int cnz_popcount32(unsigned x) +{ +#if defined(__GNUC__) || defined(__clang__) + return __builtin_popcount(x); +#else + int c = 0; + for (; x; x &= x - 1) + ++c; + return c; +#endif +} + +static inline int cnz_popcount64(uint64 x) +{ +#if defined(__GNUC__) || defined(__clang__) + return (int)__builtin_popcountll((unsigned long long)x); +#else + int c = 0; + for (; x; x &= x - 1) + ++c; + return c; +#endif +} + +static inline int cnz_lane_pop(int m) { return cnz_popcount32((unsigned)m); } +static inline int cnz_lane_pop(int64 m) { return cnz_popcount64((uint64)m); } +#endif // CV_COUNTNONZERO_AVX512 + +// SIMD body of the countNonZero kernels. The AVX-512 variant counts zero lanes +// via signmask+popcount; every other target keeps the legacy batched add kernel. +#undef CNZ_SIMD_BODY +#if CV_COUNTNONZERO_AVX512 +#define CNZ_SIMD_BODY(suffix, ssuffix, rsuffix, VT, ST, cmp_op, add_op, update_sum) \ + const int vlanes = VTraits::vlanes(); \ + VT v_zero = vx_setzero_##suffix(); \ + int nzeros = 0; \ + for (; i <= len - vlanes*2; i += vlanes*2) \ + { \ + nzeros += cnz_lane_pop(v_signmask(v_reinterpret_as_##rsuffix(cmp_op(vx_load(src + i), v_zero)))); \ + nzeros += cnz_lane_pop(v_signmask(v_reinterpret_as_##rsuffix(cmp_op(vx_load(src + i + vlanes), v_zero)))); \ + } \ + for (; i <= len - vlanes; i += vlanes) \ + { \ + nzeros += cnz_lane_pop(v_signmask(v_reinterpret_as_##rsuffix(cmp_op(vx_load(src + i), v_zero)))); \ + } \ + nz += i - nzeros; \ + v_cleanup(); +#else +#define CNZ_SIMD_BODY(suffix, ssuffix, rsuffix, VT, ST, cmp_op, add_op, update_sum) \ const int vlanes = VTraits::vlanes(); \ VT v_zero = vx_setzero_##suffix(); \ VT v_1 = vx_setall_##suffix(1); \ @@ -77,7 +131,16 @@ static int funcname( const void* src_ptr, int len ) \ update_sum(v_sum0, v_sum1, x0); \ } \ nz += (int)v_reduce_sum(v_add(v_sum0, v_sum1)); \ - v_cleanup();) \ + v_cleanup(); +#endif + +#undef DEFINE_NONZERO_FUNC +#define DEFINE_NONZERO_FUNC(funcname, suffix, ssuffix, rsuffix, T, VT, ST, cmp_op, add_op, update_sum, scalar_cmp_op) \ +static int funcname( const void* src_ptr, int len ) \ +{ \ + const T* src = static_cast(src_ptr); \ + int i = 0, nz = 0; \ + SIMD_ONLY( CNZ_SIMD_BODY(suffix, ssuffix, rsuffix, VT, ST, cmp_op, add_op, update_sum) ) \ for( ; i < len; i++ ) \ { \ nz += scalar_cmp_op(src[i]); \ @@ -115,11 +178,11 @@ static int funcname( const void* src_ptr, int len ) \ #define UPDATE_SUM_S32(v_sum0, v_sum1, x0) \ v_sum0 = v_add(v_sum0, x0) -DEFINE_NONZERO_FUNC(countNonZero8u, u8, u32, uchar, v_uint8, v_uint32, v_eq, v_add_wrap, UPDATE_SUM_U8, CHECK_NZ_INT) -DEFINE_NONZERO_FUNC(countNonZero16u, u16, u32, ushort, v_uint16, v_uint32, v_eq, v_add_wrap, UPDATE_SUM_U16, CHECK_NZ_INT) -DEFINE_NONZERO_FUNC(countNonZero32s, s32, s32, int, v_int32, v_int32, v_eq, v_add, UPDATE_SUM_S32, CHECK_NZ_INT) -DEFINE_NONZERO_FUNC(countNonZero32f, u32, u32, uint, v_uint32, v_uint32, VEC_CMP_EQ_Z_FP, v_add, UPDATE_SUM_S32, CHECK_NZ_FP) -DEFINE_NONZERO_FUNC(countNonZero16f, u16, u32, ushort, v_uint16, v_uint32, VEC_CMP_EQ_Z_FP16, v_add_wrap, UPDATE_SUM_U16, CHECK_NZ_FP16) +DEFINE_NONZERO_FUNC(countNonZero8u, u8, u32, s8, uchar, v_uint8, v_uint32, v_eq, v_add_wrap, UPDATE_SUM_U8, CHECK_NZ_INT) +DEFINE_NONZERO_FUNC(countNonZero16u, u16, u32, s16, ushort, v_uint16, v_uint32, v_eq, v_add_wrap, UPDATE_SUM_U16, CHECK_NZ_INT) +DEFINE_NONZERO_FUNC(countNonZero32s, s32, s32, s32, int, v_int32, v_int32, v_eq, v_add, UPDATE_SUM_S32, CHECK_NZ_INT) +DEFINE_NONZERO_FUNC(countNonZero32f, u32, u32, s32, uint, v_uint32, v_uint32, VEC_CMP_EQ_Z_FP, v_add, UPDATE_SUM_S32, CHECK_NZ_FP) +DEFINE_NONZERO_FUNC(countNonZero16f, u16, u32, s16, ushort, v_uint16, v_uint32, VEC_CMP_EQ_Z_FP16, v_add_wrap, UPDATE_SUM_U16, CHECK_NZ_FP16) #undef DEFINE_NONZERO_FUNC_NOSIMD #define DEFINE_NONZERO_FUNC_NOSIMD(funcname, T) \ @@ -129,7 +192,33 @@ static int funcname(const void* src, int len) \ } DEFINE_NONZERO_FUNC_NOSIMD(countNonZero64s, int64) + +#if CV_COUNTNONZERO_AVX512 && (CV_SIMD_64F || CV_SIMD_SCALABLE_64F) +// 64-bit floats: FP compare against zero treats -0.0 as zero, matching the +// scalar `src[i] != 0` path. +static int countNonZero64f( const void* src_ptr, int len ) +{ + const double* src = static_cast(src_ptr); + int i = 0, nz = 0; + const int vlanes = VTraits::vlanes(); + v_float64 v_zero = vx_setzero_f64(); + int nzeros = 0; + for (; i <= len - vlanes*2; i += vlanes*2) + { + nzeros += cnz_lane_pop(v_signmask(v_eq(vx_load(src + i), v_zero))); + nzeros += cnz_lane_pop(v_signmask(v_eq(vx_load(src + i + vlanes), v_zero))); + } + for (; i <= len - vlanes; i += vlanes) + nzeros += cnz_lane_pop(v_signmask(v_eq(vx_load(src + i), v_zero))); + nz += i - nzeros; + v_cleanup(); + for (; i < len; i++) + nz += src[i] != 0; + return nz; +} +#else DEFINE_NONZERO_FUNC_NOSIMD(countNonZero64f, double) +#endif CountNonZeroFunc getCountNonZeroTab(int depth) {