From 498e4692d6a164f61b999ecb64aa1e228fcc028d Mon Sep 17 00:00:00 2001 From: Madan mohan Manokar Date: Fri, 10 Jul 2026 21:01:01 +0530 Subject: [PATCH] Merge pull request #29479 from amd:5_fast_norm_simd core: SIMD optimizations for norm, distance and Hamming APIs (5.x) #29479 - Cache normHamming SIMD impl via a function pointer to cut per-call dispatch cost (improves ORB & BRISK) - Add masked norm and norm-diff SIMD kernels (incl. cn==4 paths) and AVX-512 dispatch for norm/stat - Add brute-force matcher perf test - Alternative port of PR #29335 adapted to the 5.x norm.simd.hpp structure ### Pull Request Readiness Checklist See details at https://github.com/opencv/opencv/wiki/How_to_contribute#making-a-good-pull-request - [x] I agree to contribute to the project under Apache 2 License. - [x] To the best of my knowledge, the proposed patch is not based on a code under GPL or another license that is incompatible with OpenCV - [x] The PR is proposed to the proper branch - [ ] There is a reference to the original bug report and related work - [x] There is accuracy test, performance test and test data in opencv_extra repository, if applicable Patch to opencv_extra has the same branch name. - [x] The feature is well documented and sample code can be built with the project CMake --- modules/core/CMakeLists.txt | 4 +- modules/core/src/norm.simd.hpp | 475 ++++++++++++++++++++++++++--- modules/core/src/stat.dispatch.cpp | 24 +- 3 files changed, 445 insertions(+), 58 deletions(-) diff --git a/modules/core/CMakeLists.txt b/modules/core/CMakeLists.txt index 27885f4b7d..80d0e395ad 100644 --- a/modules/core/CMakeLists.txt +++ b/modules/core/CMakeLists.txt @@ -1,7 +1,7 @@ set(the_description "The Core Functionality") ocv_add_dispatched_file(mathfuncs_core SSE2 AVX AVX2 LASX) -ocv_add_dispatched_file(stat SSE4_2 AVX2 LASX) +ocv_add_dispatched_file(stat SSE4_2 AVX2 AVX512_SKX AVX512_ICL LASX) ocv_add_dispatched_file(arithm SSE2 SSE4_1 AVX2 VSX3 LASX) ocv_add_dispatched_file(convert SSE2 AVX2 VSX3 LASX) ocv_add_dispatched_file(convert_scale SSE2 AVX2 LASX) @@ -15,7 +15,7 @@ ocv_add_dispatched_file(nan_mask SSE2 AVX2 LASX) ocv_add_dispatched_file(split SSE2 AVX2 LASX) ocv_add_dispatched_file(sum SSE2 AVX2 AVX512_SKX AVX512_ICL LASX) ocv_add_dispatched_file(reduce SSE2 SSSE3 AVX2 NEON_DOTPROD) -ocv_add_dispatched_file(norm SSE2 SSE4_1 AVX AVX2 NEON_DOTPROD LASX) +ocv_add_dispatched_file(norm SSE2 SSE4_1 AVX AVX2 AVX512_SKX AVX512_ICL NEON_DOTPROD LASX) ocv_add_dispatched_file(lut AVX512_ICL) ocv_add_dispatched_file(transpose AVX AVX2 NEON RVV LASX) diff --git a/modules/core/src/norm.simd.hpp b/modules/core/src/norm.simd.hpp index 79da073180..eaa07eee32 100644 --- a/modules/core/src/norm.simd.hpp +++ b/modules/core/src/norm.simd.hpp @@ -3,6 +3,7 @@ // of this distribution and at http://opencv.org/license.html // // Copyright (C) 2025, SpaceMIT Inc., all rights reserved. +// Copyright (C) 2026, Advanced Micro Devices, Inc., all rights reserved. // Third party copyrights are property of their respective owners. #include "precomp.hpp" @@ -1456,21 +1457,38 @@ struct MaskedNormInf_SIMD { result = std::max(result, (int)src[i]); } } + else if (cn == 4 && len >= VTraits::vlanes()) { + const int vstep = VTraits::vlanes(); + v_uint8 acc = vx_setzero_u8(); + int i = 0; + for (;;) { + if (i > len - vstep) + i = len - vstep; // back-step (max is idempotent) + v_uint8 c0, c1, c2, c3; + v_load_deinterleave(src + i * 4, c0, c1, c2, c3); + v_uint8 pmax = v_max(v_max(c0, c1), v_max(c2, c3)); + v_uint8 m = v_gt(vx_load(mask + i), vx_setzero_u8()); + acc = v_max(acc, v_and(pmax, m)); + if (i >= len - vstep) + break; + i += vstep; + } + result = (int)v_reduce_max(acc); + } + else if (cn == 4) { + for (int i = 0; i < len; i++) { + if (mask[i]) { + const uchar* elem = src + i * 4; + result = std::max(result, (int)std::max(std::max(elem[0], elem[1]), + std::max(elem[2], elem[3]))); + } + } + } else { for (int i = 0; i < len; i++) { if (mask[i]) { const uchar* elem = src + i * cn; - int k = 0; - const int vstep = VTraits::vlanes(); - v_uint8 acc = vx_setzero_u8(); - - for (; k <= cn - vstep; k += vstep) { - acc = v_max(acc, vx_load(elem + k)); - } - - result = std::max(result, (int)v_reduce_max(acc)); - - for (; k < cn; k++) + for (int k = 0; k < cn; k++) result = std::max(result, (int)elem[k]); } } @@ -1502,22 +1520,30 @@ struct MaskedNormL1_SIMD { } } else { - for (int i = 0; i < len; i++) { - if (mask[i]) { - const uchar* elem = src + i * cn; - int k = 0; - const int vstep = VTraits::vlanes() / 4; - v_uint32 acc = vx_setzero_u32(); - - for (; k <= cn - vstep; k += vstep) { - v_uint32 s = vx_load_expand_q(elem + k); - acc = v_add(acc, s); + const int vstep = VTraits::vlanes() / 4; + if (cn >= vstep) { + for (int i = 0; i < len; i++) { + if (mask[i]) { + const uchar* elem = src + i * cn; + int k = 0; + v_uint32 acc = vx_setzero_u32(); + for (; k <= cn - vstep; k += vstep) { + v_uint32 s = vx_load_expand_q(elem + k); + acc = v_add(acc, s); + } + result += (int)v_reduce_sum(acc); + for (; k < cn; k++) + result += elem[k]; + } + } + } + else { + for (int i = 0; i < len; i++) { + if (mask[i]) { + const uchar* elem = src + i * cn; + for (int k = 0; k < cn; k++) + result += elem[k]; } - - result += (int)v_reduce_sum(acc); - - for (; k < cn; k++) - result += elem[k]; } } } @@ -1795,6 +1821,375 @@ normL2_Bool(const uchar* src, const uchar* mask, int* _result, int len, int cn) return normL1_Bool(src, mask, _result, len, cn); } +// ==================================================================== +// Masked norm-diff kernels. +// Scalar base templates (safe for every depth) plus SIMD cn==1 +// specializations for the concrete (T, ST) pairs dispatched by +// CV_DEF_NORM_ALL. The masked normDiff*_ functions below select them. +// ==================================================================== +template +struct MaskedNormDiffInf_SIMD { + inline ST operator()(const T* s1, const T* s2, const uchar* mask, int len, int cn) const { + ST result = 0; + for (int i = 0; i < len; i++) if (mask[i]) { + const T* e1 = s1 + i*cn; const T* e2 = s2 + i*cn; + for (int k = 0; k < cn; k++) + result = std::max(result, (ST)cv_absdiff(e1[k], e2[k])); + } + return result; + } +}; +template +struct MaskedNormDiffL1_SIMD { + inline ST operator()(const T* s1, const T* s2, const uchar* mask, int len, int cn) const { + ST result = 0; + for (int i = 0; i < len; i++) if (mask[i]) { + const T* e1 = s1 + i*cn; const T* e2 = s2 + i*cn; + for (int k = 0; k < cn; k++) + result += (ST)cv_absdiff(e1[k], e2[k]); + } + return result; + } +}; +template +struct MaskedNormDiffL2_SIMD { + inline ST operator()(const T* s1, const T* s2, const uchar* mask, int len, int cn) const { + ST result = 0; + for (int i = 0; i < len; i++) if (mask[i]) { + const T* e1 = s1 + i*cn; const T* e2 = s2 + i*cn; + for (int k = 0; k < cn; k++) { ST v = (ST)e1[k] - (ST)e2[k]; result += v*v; } + } + return result; + } +}; + +#if (CV_SIMD || CV_SIMD_SCALABLE) +// AND a per-lane diff vector with the (expanded) mask>0 predicate. +inline v_uint8 v_diffmask(const v_uint8& d, const uchar* m) { return v_and(d, v_gt(vx_load(m), vx_setzero_u8())); } +inline v_uint16 v_diffmask(const v_uint16& d, const uchar* m) { return v_and(d, v_gt(vx_load_expand(m), vx_setzero_u16())); } +inline v_uint32 v_diffmask(const v_uint32& d, const uchar* m) { return v_and(d, v_gt(vx_load_expand_q(m), vx_setzero_u32())); } +inline v_float32 v_diffmask(const v_float32& d, const uchar* m) { + v_uint32 cm = v_gt(vx_load_expand_q(m), vx_setzero_u32()); + return v_reinterpret_as_f32(v_and(v_reinterpret_as_u32(d), cm)); +} + +// Shared scalar cn>1 fallbacks (match the base templates' cv_absdiff semantics). +template +static inline RT maskedNormDiffInfTail(const T* s1, const T* s2, const uchar* mask, int len, int cn, RT result) { + for (int i = 0; i < len; i++) if (mask[i]) { + const T* e1 = s1 + i*cn; const T* e2 = s2 + i*cn; + for (int k = 0; k < cn; k++) result = std::max(result, (RT)cv_absdiff(e1[k], e2[k])); + } + return result; +} +template +static inline RT maskedNormDiffL1Tail(const T* s1, const T* s2, const uchar* mask, int len, int cn, RT result) { + for (int i = 0; i < len; i++) if (mask[i]) { + const T* e1 = s1 + i*cn; const T* e2 = s2 + i*cn; + for (int k = 0; k < cn; k++) result += (RT)cv_absdiff(e1[k], e2[k]); + } + return result; +} +template +static inline RT maskedNormDiffL2Tail(const T* s1, const T* s2, const uchar* mask, int len, int cn, RT result) { + for (int i = 0; i < len; i++) if (mask[i]) { + const T* e1 = s1 + i*cn; const T* e2 = s2 + i*cn; + for (int k = 0; k < cn; k++) { RT v = (RT)e1[k] - (RT)e2[k]; result += v*v; } + } + return result; +} + +// Inf (cn==1): reduce_max of masked |a-b|. Shared across SIMD-capable depths. +template +static inline ST maskedNormDiffInfCn1(const T* s1, const T* s2, const uchar* mask, int len) { + const int vstep = VTraits::vlanes(); + ST result = 0; + int i = 0; + if (len >= vstep) { + auto acc = v_diffmask(v_absdiff(vx_load(s1), vx_load(s2)), mask); + for (i = vstep; i <= len - vstep; i += vstep) + acc = v_max(acc, v_diffmask(v_absdiff(vx_load(s1 + i), vx_load(s2 + i)), mask + i)); + result = (ST)v_reduce_max(acc); + } + for (; i < len; i++) if (mask[i]) result = std::max(result, (ST)cv_absdiff(s1[i], s2[i])); + return result; +} + +// Shared 8-bit cn==1 masked L1/L2 kernels (uchar/schar): v_absdiff yields v_uint8 for both. +template +static inline int maskedNormDiffL1_8(const T* s1, const T* s2, const uchar* mask, int len) { + int i = 0; const int vstep = VTraits::vlanes(); + const v_uint8 one = vx_setall_u8(1); + v_uint32 acc = vx_setzero_u32(); + for (; i <= len - vstep; i += vstep) { + v_uint8 m = v_gt(vx_load(mask + i), vx_setzero_u8()); + v_uint8 ad = v_and(v_absdiff(vx_load(s1 + i), vx_load(s2 + i)), m); + acc = v_dotprod_expand_fast(ad, one, acc); + } + int result = (int)v_reduce_sum(acc); + for (; i < len; i++) if (mask[i]) result += (int)cv_absdiff(s1[i], s2[i]); + return result; +} +template +static inline int maskedNormDiffL2_8(const T* s1, const T* s2, const uchar* mask, int len) { + int i = 0; const int vstep = VTraits::vlanes(); + v_uint32 acc = vx_setzero_u32(); + for (; i <= len - vstep; i += vstep) { + v_uint8 m = v_gt(vx_load(mask + i), vx_setzero_u8()); + v_uint8 ad = v_and(v_absdiff(vx_load(s1 + i), vx_load(s2 + i)), m); + acc = v_dotprod_expand_fast(ad, ad, acc); + } + int result = (int)v_reduce_sum(acc); + for (; i < len; i++) if (mask[i]) { int v = (int)s1[i] - (int)s2[i]; result += v*v; } + return result; +} + +// Shared 16-bit cn==1 masked L1 kernel (ushort/short): v_absdiff yields v_uint16 for both. +template +static inline int maskedNormDiffL1_16(const T* s1, const T* s2, const uchar* mask, int len) { + int i = 0; const int vstep = VTraits::vlanes(); + v_uint32 acc = vx_setzero_u32(); + for (; i <= len - vstep; i += vstep) { + v_uint16 m = v_gt(vx_load_expand(mask + i), vx_setzero_u16()); + v_uint16 ad = v_and(v_absdiff(vx_load(s1 + i), vx_load(s2 + i)), m); + v_uint32 lo, hi; v_expand(ad, lo, hi); + acc = v_add(acc, v_add(lo, hi)); + } + int result = (int)v_reduce_sum(acc); + for (; i < len; i++) if (mask[i]) result += (int)cv_absdiff(s1[i], s2[i]); + return result; +} + +template<> struct MaskedNormDiffInf_SIMD { + int operator()(const uchar* s1, const uchar* s2, const uchar* mask, int len, int cn) const { + if (cn == 1) return maskedNormDiffInfCn1(s1, s2, mask, len); + if (cn == 4 && len >= VTraits::vlanes()) { + const int vstep = VTraits::vlanes(); + v_uint8 acc = vx_setzero_u8(); + int i = 0; + for (;;) { + if (i > len - vstep) i = len - vstep; // back-step (max is idempotent) + v_uint8 a0,a1,a2,a3,b0,b1,b2,b3; + v_load_deinterleave(s1 + i*4, a0,a1,a2,a3); + v_load_deinterleave(s2 + i*4, b0,b1,b2,b3); + v_uint8 ad = v_max(v_max(v_absdiff(a0,b0), v_absdiff(a1,b1)), + v_max(v_absdiff(a2,b2), v_absdiff(a3,b3))); + v_uint8 m = v_gt(vx_load(mask + i), vx_setzero_u8()); + acc = v_max(acc, v_and(ad, m)); + if (i >= len - vstep) break; + i += vstep; + } + return (int)v_reduce_max(acc); + } + return maskedNormDiffInfTail(s1, s2, mask, len, cn, 0); + } +}; +template<> struct MaskedNormDiffInf_SIMD { + int operator()(const schar* s1, const schar* s2, const uchar* mask, int len, int cn) const { + if (cn == 1) return maskedNormDiffInfCn1(s1, s2, mask, len); + return maskedNormDiffInfTail(s1, s2, mask, len, cn, 0); + } +}; +template<> struct MaskedNormDiffInf_SIMD { + int operator()(const ushort* s1, const ushort* s2, const uchar* mask, int len, int cn) const { + if (cn == 1) return maskedNormDiffInfCn1(s1, s2, mask, len); + return maskedNormDiffInfTail(s1, s2, mask, len, cn, 0); + } +}; +template<> struct MaskedNormDiffInf_SIMD { + int operator()(const short* s1, const short* s2, const uchar* mask, int len, int cn) const { + if (cn == 1) return maskedNormDiffInfCn1(s1, s2, mask, len); + return maskedNormDiffInfTail(s1, s2, mask, len, cn, 0); + } +}; +template<> struct MaskedNormDiffInf_SIMD { + unsigned operator()(const int* s1, const int* s2, const uchar* mask, int len, int cn) const { + if (cn == 1) return maskedNormDiffInfCn1(s1, s2, mask, len); + return maskedNormDiffInfTail(s1, s2, mask, len, cn, 0u); + } +}; +template<> struct MaskedNormDiffInf_SIMD { + float operator()(const float* s1, const float* s2, const uchar* mask, int len, int cn) const { + if (cn == 1) return maskedNormDiffInfCn1(s1, s2, mask, len); + return maskedNormDiffInfTail(s1, s2, mask, len, cn, 0.0f); + } +}; + +template<> struct MaskedNormDiffL1_SIMD { + int operator()(const uchar* s1, const uchar* s2, const uchar* mask, int len, int cn) const { + if (cn == 1) return maskedNormDiffL1_8(s1, s2, mask, len); + if (cn == 4) { + int i = 0; const int vstep = VTraits::vlanes(); + const v_uint8 one = vx_setall_u8(1); + v_uint32 acc = vx_setzero_u32(); + for (; i <= len - vstep; i += vstep) { + v_uint8 a0,a1,a2,a3,b0,b1,b2,b3; + v_load_deinterleave(s1 + i*4, a0,a1,a2,a3); + v_load_deinterleave(s2 + i*4, b0,b1,b2,b3); + v_uint8 m = v_gt(vx_load(mask + i), vx_setzero_u8()); + acc = v_dotprod_expand_fast(v_and(v_absdiff(a0,b0), m), one, acc); + acc = v_dotprod_expand_fast(v_and(v_absdiff(a1,b1), m), one, acc); + acc = v_dotprod_expand_fast(v_and(v_absdiff(a2,b2), m), one, acc); + acc = v_dotprod_expand_fast(v_and(v_absdiff(a3,b3), m), one, acc); + } + int result = (int)v_reduce_sum(acc); + for (; i < len; i++) if (mask[i]) { + const uchar* e1 = s1 + i*4; const uchar* e2 = s2 + i*4; + for (int k = 0; k < 4; k++) result += (int)cv_absdiff(e1[k], e2[k]); + } + return result; + } + return maskedNormDiffL1Tail(s1, s2, mask, len, cn, 0); + } +}; +template<> struct MaskedNormDiffL1_SIMD { + int operator()(const schar* s1, const schar* s2, const uchar* mask, int len, int cn) const { + if (cn == 1) return maskedNormDiffL1_8(s1, s2, mask, len); + return maskedNormDiffL1Tail(s1, s2, mask, len, cn, 0); + } +}; +template<> struct MaskedNormDiffL1_SIMD { + int operator()(const ushort* s1, const ushort* s2, const uchar* mask, int len, int cn) const { + if (cn == 1) return maskedNormDiffL1_16(s1, s2, mask, len); + return maskedNormDiffL1Tail(s1, s2, mask, len, cn, 0); + } +}; +template<> struct MaskedNormDiffL1_SIMD { + int operator()(const short* s1, const short* s2, const uchar* mask, int len, int cn) const { + if (cn == 1) return maskedNormDiffL1_16(s1, s2, mask, len); + return maskedNormDiffL1Tail(s1, s2, mask, len, cn, 0); + } +}; + +template<> struct MaskedNormDiffL2_SIMD { + int operator()(const uchar* s1, const uchar* s2, const uchar* mask, int len, int cn) const { + if (cn == 1) return maskedNormDiffL2_8(s1, s2, mask, len); + if (cn == 4) { + int i = 0; const int vstep = VTraits::vlanes(); + v_uint32 acc = vx_setzero_u32(); + for (; i <= len - vstep; i += vstep) { + v_uint8 a0,a1,a2,a3,b0,b1,b2,b3; + v_load_deinterleave(s1 + i*4, a0,a1,a2,a3); + v_load_deinterleave(s2 + i*4, b0,b1,b2,b3); + v_uint8 m = v_gt(vx_load(mask + i), vx_setzero_u8()); + v_uint8 d0 = v_and(v_absdiff(a0,b0), m), d1 = v_and(v_absdiff(a1,b1), m); + v_uint8 d2 = v_and(v_absdiff(a2,b2), m), d3 = v_and(v_absdiff(a3,b3), m); + acc = v_dotprod_expand_fast(d0, d0, acc); + acc = v_dotprod_expand_fast(d1, d1, acc); + acc = v_dotprod_expand_fast(d2, d2, acc); + acc = v_dotprod_expand_fast(d3, d3, acc); + } + int result = (int)v_reduce_sum(acc); + for (; i < len; i++) if (mask[i]) { + const uchar* e1 = s1 + i*4; const uchar* e2 = s2 + i*4; + for (int k = 0; k < 4; k++) { int v = (int)e1[k] - (int)e2[k]; result += v*v; } + } + return result; + } + return maskedNormDiffL2Tail(s1, s2, mask, len, cn, 0); + } +}; +template<> struct MaskedNormDiffL2_SIMD { + int operator()(const schar* s1, const schar* s2, const uchar* mask, int len, int cn) const { + if (cn == 1) return maskedNormDiffL2_8(s1, s2, mask, len); + return maskedNormDiffL2Tail(s1, s2, mask, len, cn, 0); + } +}; + +#if CV_SIMD_64F +// Depths whose L1/L2 accumulate in double. +template<> struct MaskedNormDiffL1_SIMD { + double operator()(const int* s1, const int* s2, const uchar* mask, int len, int cn) const { + if (cn != 1) return maskedNormDiffL1Tail(s1, s2, mask, len, cn, 0.0); + int i = 0; const int vstep = VTraits::vlanes(); + v_float64 acc = vx_setzero_f64(); + for (; i <= len - vstep; i += vstep) { + v_uint32 m = v_gt(vx_load_expand_q(mask + i), vx_setzero_u32()); + v_uint32 ad = v_and(v_absdiff(vx_load(s1 + i), vx_load(s2 + i)), m); + v_uint64 e0, e1; v_expand(ad, e0, e1); + acc = v_add(acc, v_cvt_f64(v_reinterpret_as_s64(e0))); + acc = v_add(acc, v_cvt_f64(v_reinterpret_as_s64(e1))); + } + double result = v_reduce_sum(acc); + for (; i < len; i++) if (mask[i]) result += (double)cv_absdiff(s1[i], s2[i]); + return result; + } +}; +template<> struct MaskedNormDiffL1_SIMD { + double operator()(const float* s1, const float* s2, const uchar* mask, int len, int cn) const { + if (cn != 1) return maskedNormDiffL1Tail(s1, s2, mask, len, cn, 0.0); + int i = 0; const int vstep = VTraits::vlanes(); + v_float64 acc0 = vx_setzero_f64(), acc1 = vx_setzero_f64(); + for (; i <= len - vstep; i += vstep) { + v_float32 m = v_reinterpret_as_f32(v_gt(vx_load_expand_q(mask + i), vx_setzero_u32())); + v_float32 ad = v_and(v_absdiff(vx_load(s1 + i), vx_load(s2 + i)), m); + acc0 = v_add(acc0, v_cvt_f64(ad)); + acc1 = v_add(acc1, v_cvt_f64_high(ad)); + } + double result = v_reduce_sum(v_add(acc0, acc1)); + for (; i < len; i++) if (mask[i]) result += (double)cv_absdiff(s1[i], s2[i]); + return result; + } +}; + +template +static inline double maskedNormDiffL2_16(const T* s1, const T* s2, const uchar* mask, int len, int cn) { + if (cn != 1) return maskedNormDiffL2Tail(s1, s2, mask, len, cn, 0.0); + int i = 0; const int vstep = VTraits::vlanes(); + v_float64 acc = vx_setzero_f64(); + for (; i <= len - vstep; i += vstep) { + v_uint16 m = v_gt(vx_load_expand(mask + i), vx_setzero_u16()); + v_uint16 ad = v_and(v_absdiff(vx_load(s1 + i), vx_load(s2 + i)), m); + v_uint64 u = v_dotprod_expand_fast(ad, ad); + acc = v_add(acc, v_cvt_f64(v_reinterpret_as_s64(u))); + } + double result = v_reduce_sum(acc); + for (; i < len; i++) if (mask[i]) { double v = (double)s1[i] - (double)s2[i]; result += v*v; } + return result; +} +template<> struct MaskedNormDiffL2_SIMD { + double operator()(const ushort* s1, const ushort* s2, const uchar* mask, int len, int cn) const + { return maskedNormDiffL2_16(s1, s2, mask, len, cn); } }; +template<> struct MaskedNormDiffL2_SIMD { + double operator()(const short* s1, const short* s2, const uchar* mask, int len, int cn) const + { return maskedNormDiffL2_16(s1, s2, mask, len, cn); } }; + +template<> struct MaskedNormDiffL2_SIMD { + double operator()(const int* s1, const int* s2, const uchar* mask, int len, int cn) const { + if (cn != 1) return maskedNormDiffL2Tail(s1, s2, mask, len, cn, 0.0); + int i = 0; const int vstep = VTraits::vlanes(); + v_float64 r0 = vx_setzero_f64(), r1 = vx_setzero_f64(); + for (; i <= len - vstep; i += vstep) { + v_uint32 m = v_gt(vx_load_expand_q(mask + i), vx_setzero_u32()); + v_uint32 ad = v_and(v_absdiff(vx_load(s1 + i), vx_load(s2 + i)), m); + v_uint64 e0, e1; v_expand(ad, e0, e1); + v_float64 f0 = v_cvt_f64(v_reinterpret_as_s64(e0)), f1 = v_cvt_f64(v_reinterpret_as_s64(e1)); + r0 = v_fma(f0, f0, r0); r1 = v_fma(f1, f1, r1); + } + double result = v_reduce_sum(v_add(r0, r1)); + for (; i < len; i++) if (mask[i]) { double v = (double)s1[i] - (double)s2[i]; result += v*v; } + return result; + } +}; +template<> struct MaskedNormDiffL2_SIMD { + double operator()(const float* s1, const float* s2, const uchar* mask, int len, int cn) const { + if (cn != 1) return maskedNormDiffL2Tail(s1, s2, mask, len, cn, 0.0); + int i = 0; const int vstep = VTraits::vlanes(); + v_float64 r0 = vx_setzero_f64(), r1 = vx_setzero_f64(); + for (; i <= len - vstep; i += vstep) { + v_float32 m = v_reinterpret_as_f32(v_gt(vx_load_expand_q(mask + i), vx_setzero_u32())); + v_float32 ad = v_and(v_absdiff(vx_load(s1 + i), vx_load(s2 + i)), m); + v_float64 f0 = v_cvt_f64(ad), f1 = v_cvt_f64_high(ad); + r0 = v_fma(f0, f0, r0); r1 = v_fma(f1, f1, r1); + } + double result = v_reduce_sum(v_add(r0, r1)); + for (; i < len; i++) if (mask[i]) { double v = (double)s1[i] - (double)s2[i]; result += v*v; } + return result; + } +}; +#endif // CV_SIMD_64F +#endif // CV_SIMD + template int normDiffInf_(const T* src1, const T* src2, const uchar* mask, ST* _result, int len, int cn) { ST result = *_result; @@ -1802,13 +2197,8 @@ normDiffInf_(const T* src1, const T* src2, const uchar* mask, ST* _result, int l NormDiffInf_SIMD op; result = std::max(result, op(src1, src2, len*cn)); } else { - for( int i = 0; i < len; i++, src1 += cn, src2 += cn ) { - if( mask[i] ) { - for( int k = 0; k < cn; k++ ) { - result = std::max(result, (ST)cv_absdiff(src1[k], src2[k])); - } - } - } + MaskedNormDiffInf_SIMD op; + result = std::max(result, op(src1, src2, mask, len, cn)); } *_result = result; return 0; @@ -1821,13 +2211,8 @@ normDiffL1_(const T* src1, const T* src2, const uchar* mask, ST* _result, int le NormDiffL1_SIMD op; result += op(src1, src2, len*cn); } else { - for( int i = 0; i < len; i++, src1 += cn, src2 += cn ) { - if( mask[i] ) { - for( int k = 0; k < cn; k++ ) { - result += (ST)cv_absdiff(src1[k], src2[k]); - } - } - } + MaskedNormDiffL1_SIMD op; + result += op(src1, src2, mask, len, cn); } *_result = result; return 0; @@ -1840,14 +2225,8 @@ normDiffL2_(const T* src1, const T* src2, const uchar* mask, ST* _result, int le NormDiffL2_SIMD op; result += op(src1, src2, len*cn); } else { - for( int i = 0; i < len; i++, src1 += cn, src2 += cn ) { - if( mask[i] ) { - for( int k = 0; k < cn; k++ ) { - ST v = (ST)src1[k] - (ST)src2[k]; - result += v*v; - } - } - } + MaskedNormDiffL2_SIMD op; + result += op(src1, src2, mask, len, cn); } *_result = result; return 0; diff --git a/modules/core/src/stat.dispatch.cpp b/modules/core/src/stat.dispatch.cpp index 08275fac50..9df94a4efb 100644 --- a/modules/core/src/stat.dispatch.cpp +++ b/modules/core/src/stat.dispatch.cpp @@ -1,6 +1,8 @@ // This file is part of OpenCV project. // It is subject to the license terms in the LICENSE file found in the top-level directory // of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2026, Advanced Micro Devices, Inc., all rights reserved. #include "precomp.hpp" @@ -9,20 +11,26 @@ namespace cv { namespace hal { +// Resolve the SIMD implementation ONCE via a cached function pointer. normHamming +// is called per-descriptor in hot matcher loops on short (32/64-byte) descriptors, +// where a per-call dispatch chain + CV_INSTRUMENT_REGION dominate the cost. +static NormHammingFunc getNormHammingFunc() { + CV_CPU_DISPATCH(getNormHammingFunc, (), CV_CPU_DISPATCH_MODES_ALL); +} +static NormHammingDiffFunc getNormHammingDiffFunc() { + CV_CPU_DISPATCH(getNormHammingDiffFunc, (), CV_CPU_DISPATCH_MODES_ALL); +} + int normHamming(const uchar* a, int n) { - CV_INSTRUMENT_REGION(); - - CV_CPU_DISPATCH(normHamming, (a, n), - CV_CPU_DISPATCH_MODES_ALL); + static const NormHammingFunc fn = getNormHammingFunc(); + return fn(a, n); } int normHamming(const uchar* a, const uchar* b, int n) { - CV_INSTRUMENT_REGION(); - - CV_CPU_DISPATCH(normHamming, (a, b, n), - CV_CPU_DISPATCH_MODES_ALL); + static const NormHammingDiffFunc fn = getNormHammingDiffFunc(); + return fn(a, b, n); } }} //cv::hal