potok/opencv
1
0
Fork 0
mirror of https://github.com/opencv/opencv.git synced 2026-07-21 19:33:03 +04:00
Code Packages Releases Wiki Activity

Merge pull request #29480 from amd:5_fast_countNonZero

Browse Source 
core: speed up countNonZero with AVX-512 signmask path (5.x)
This commit is contained in:
Alexander Smorkalov
2026-07-09 18:40:40 +03:00
committed by GitHub
parent 4b38c5562b 9ce06a5675
commit 7559555599
2 changed files with 103 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files
modules/core/CMakeLists.txt
+1 -1
View File
@@ -5,7 +5,7 @@ ocv_add_dispatched_file(stat SSE4_2 AVX2 LASX)
ocv_add_dispatched_file(arithm SSE2 SSE4_1 AVX2 VSX3 LASX)
ocv_add_dispatched_file(convert SSE2 AVX2 VSX3 LASX)
ocv_add_dispatched_file(convert_scale SSE2 AVX2 LASX)
ocv_add_dispatched_file(count_non_zero SSE2 AVX2 LASX)
ocv_add_dispatched_file(count_non_zero SSE2 AVX2 AVX512_SKX AVX512_ICL LASX)
ocv_add_dispatched_file(has_non_zero SSE2 AVX2 LASX )
ocv_add_dispatched_file(matmul SSE2 SSE4_1 AVX2 AVX512_SKX NEON_DOTPROD LASX)
ocv_add_dispatched_file(mean SSE2 AVX2 AVX512_SKX AVX512_ICL LASX)
modules/core/src/count_non_zero.simd.hpp
+102 -13
View File
@@ -30,13 +30,67 @@ static int countNonZero_(const T* src, int len )
#define SIMD_ONLY(expr)
#endif
#undef DEFINE_NONZERO_FUNC
#define DEFINE_NONZERO_FUNC(funcname, suffix, ssuffix, T, VT, ST, cmp_op, add_op, update_sum, scalar_cmp_op) \
static int funcname( const void* src_ptr, int len ) \
{ \
const T* src = static_cast<const T*>(src_ptr); \
int i = 0, nz = 0; \
SIMD_ONLY( \
// AVX-512 exposes wide per-lane mask registers, so counting zero lanes with
// v_signmask()+popcount is cheaper than the batched add-based reduction.
// Narrower SIMD (AVX2, NEON, LASX, ...) keeps the legacy batched kernel to
// avoid regressions on smaller vector widths.
#undef CV_COUNTNONZERO_AVX512
#if defined(CV_CPU_COMPILE_AVX512_SKX) || defined(CV_CPU_COMPILE_AVX512_ICL)
#define CV_COUNTNONZERO_AVX512 1
#else
#define CV_COUNTNONZERO_AVX512 0
#endif
#if CV_COUNTNONZERO_AVX512
static inline int cnz_popcount32(unsigned x)
{
#if defined(__GNUC__) || defined(__clang__)
return __builtin_popcount(x);
#else
int c = 0;
for (; x; x &= x - 1)
++c;
return c;
#endif
}
static inline int cnz_popcount64(uint64 x)
{
#if defined(__GNUC__) || defined(__clang__)
return (int)__builtin_popcountll((unsigned long long)x);
#else
int c = 0;
for (; x; x &= x - 1)
++c;
return c;
#endif
}
static inline int cnz_lane_pop(int m) { return cnz_popcount32((unsigned)m); }
static inline int cnz_lane_pop(int64 m) { return cnz_popcount64((uint64)m); }
#endif // CV_COUNTNONZERO_AVX512
// SIMD body of the countNonZero kernels. The AVX-512 variant counts zero lanes
// via signmask+popcount; every other target keeps the legacy batched add kernel.
#undef CNZ_SIMD_BODY
#if CV_COUNTNONZERO_AVX512
#define CNZ_SIMD_BODY(suffix, ssuffix, rsuffix, VT, ST, cmp_op, add_op, update_sum) \
const int vlanes = VTraits<VT>::vlanes(); \
VT v_zero = vx_setzero_##suffix(); \
int nzeros = 0; \
for (; i <= len - vlanes*2; i += vlanes*2) \
{ \
nzeros += cnz_lane_pop(v_signmask(v_reinterpret_as_##rsuffix(cmp_op(vx_load(src + i), v_zero)))); \
nzeros += cnz_lane_pop(v_signmask(v_reinterpret_as_##rsuffix(cmp_op(vx_load(src + i + vlanes), v_zero)))); \
} \
for (; i <= len - vlanes; i += vlanes) \
{ \
nzeros += cnz_lane_pop(v_signmask(v_reinterpret_as_##rsuffix(cmp_op(vx_load(src + i), v_zero)))); \
} \
nz += i - nzeros; \
v_cleanup();
#else
#define CNZ_SIMD_BODY(suffix, ssuffix, rsuffix, VT, ST, cmp_op, add_op, update_sum) \
const int vlanes = VTraits<VT>::vlanes(); \
VT v_zero = vx_setzero_##suffix(); \
VT v_1 = vx_setall_##suffix(1); \
@@ -77,7 +131,16 @@ static int funcname( const void* src_ptr, int len ) \
update_sum(v_sum0, v_sum1, x0); \
} \
nz += (int)v_reduce_sum(v_add(v_sum0, v_sum1)); \
v_cleanup();) \
v_cleanup();
#endif
#undef DEFINE_NONZERO_FUNC
#define DEFINE_NONZERO_FUNC(funcname, suffix, ssuffix, rsuffix, T, VT, ST, cmp_op, add_op, update_sum, scalar_cmp_op) \
static int funcname( const void* src_ptr, int len ) \
{ \
const T* src = static_cast<const T*>(src_ptr); \
int i = 0, nz = 0; \
SIMD_ONLY( CNZ_SIMD_BODY(suffix, ssuffix, rsuffix, VT, ST, cmp_op, add_op, update_sum) ) \
for( ; i < len; i++ ) \
{ \
nz += scalar_cmp_op(src[i]); \
@@ -115,11 +178,11 @@ static int funcname( const void* src_ptr, int len ) \
#define UPDATE_SUM_S32(v_sum0, v_sum1, x0) \
v_sum0 = v_add(v_sum0, x0)
DEFINE_NONZERO_FUNC(countNonZero8u, u8, u32, uchar, v_uint8, v_uint32, v_eq, v_add_wrap, UPDATE_SUM_U8, CHECK_NZ_INT)
DEFINE_NONZERO_FUNC(countNonZero16u, u16, u32, ushort, v_uint16, v_uint32, v_eq, v_add_wrap, UPDATE_SUM_U16, CHECK_NZ_INT)
DEFINE_NONZERO_FUNC(countNonZero32s, s32, s32, int, v_int32, v_int32, v_eq, v_add, UPDATE_SUM_S32, CHECK_NZ_INT)
DEFINE_NONZERO_FUNC(countNonZero32f, u32, u32, uint, v_uint32, v_uint32, VEC_CMP_EQ_Z_FP, v_add, UPDATE_SUM_S32, CHECK_NZ_FP)
DEFINE_NONZERO_FUNC(countNonZero16f, u16, u32, ushort, v_uint16, v_uint32, VEC_CMP_EQ_Z_FP16, v_add_wrap, UPDATE_SUM_U16, CHECK_NZ_FP16)
DEFINE_NONZERO_FUNC(countNonZero8u, u8, u32, s8, uchar, v_uint8, v_uint32, v_eq, v_add_wrap, UPDATE_SUM_U8, CHECK_NZ_INT)
DEFINE_NONZERO_FUNC(countNonZero16u, u16, u32, s16, ushort, v_uint16, v_uint32, v_eq, v_add_wrap, UPDATE_SUM_U16, CHECK_NZ_INT)
DEFINE_NONZERO_FUNC(countNonZero32s, s32, s32, s32, int, v_int32, v_int32, v_eq, v_add, UPDATE_SUM_S32, CHECK_NZ_INT)
DEFINE_NONZERO_FUNC(countNonZero32f, u32, u32, s32, uint, v_uint32, v_uint32, VEC_CMP_EQ_Z_FP, v_add, UPDATE_SUM_S32, CHECK_NZ_FP)
DEFINE_NONZERO_FUNC(countNonZero16f, u16, u32, s16, ushort, v_uint16, v_uint32, VEC_CMP_EQ_Z_FP16, v_add_wrap, UPDATE_SUM_U16, CHECK_NZ_FP16)
#undef DEFINE_NONZERO_FUNC_NOSIMD
#define DEFINE_NONZERO_FUNC_NOSIMD(funcname, T) \
@@ -129,7 +192,33 @@ static int funcname(const void* src, int len) \
}
DEFINE_NONZERO_FUNC_NOSIMD(countNonZero64s, int64)
#if CV_COUNTNONZERO_AVX512 && (CV_SIMD_64F || CV_SIMD_SCALABLE_64F)
// 64-bit floats: FP compare against zero treats -0.0 as zero, matching the
// scalar `src[i] != 0` path.
static int countNonZero64f( const void* src_ptr, int len )
{
const double* src = static_cast<const double*>(src_ptr);
int i = 0, nz = 0;
const int vlanes = VTraits<v_float64>::vlanes();
v_float64 v_zero = vx_setzero_f64();
int nzeros = 0;
for (; i <= len - vlanes*2; i += vlanes*2)
{
nzeros += cnz_lane_pop(v_signmask(v_eq(vx_load(src + i), v_zero)));
nzeros += cnz_lane_pop(v_signmask(v_eq(vx_load(src + i + vlanes), v_zero)));
}
for (; i <= len - vlanes; i += vlanes)
nzeros += cnz_lane_pop(v_signmask(v_eq(vx_load(src + i), v_zero)));
nz += i - nzeros;
v_cleanup();
for (; i < len; i++)
nz += src[i] != 0;
return nz;
}
#else
DEFINE_NONZERO_FUNC_NOSIMD(countNonZero64f, double)
#endif
CountNonZeroFunc getCountNonZeroTab(int depth)
{