From 550251b3c2c4ea73a5824afedb0df25895085d24 Mon Sep 17 00:00:00 2001 From: Madan mohan Manokar Date: Thu, 18 Jun 2026 13:23:53 +0000 Subject: [PATCH] core: Optimize minMaxIdx/minMaxLoc - Replace the fixed-128-bit per-depth minMaxIdx kernels with a single templated universal-intrinsic core - added simd dispatch AVX-512 dispatch - A dual-accumulator inner loop added Uses universal intrinsics, so all SIMD backends benefit. --- modules/core/CMakeLists.txt | 1 + modules/core/src/minmax.cpp | 1368 -------------------------- modules/core/src/minmax.dispatch.cpp | 540 ++++++++++ modules/core/src/minmax.simd.hpp | 471 +++++++++ 4 files changed, 1012 insertions(+), 1368 deletions(-) delete mode 100644 modules/core/src/minmax.cpp create mode 100644 modules/core/src/minmax.dispatch.cpp create mode 100644 modules/core/src/minmax.simd.hpp diff --git a/modules/core/CMakeLists.txt b/modules/core/CMakeLists.txt index 414dc9907b..6d969f3c8d 100644 --- a/modules/core/CMakeLists.txt +++ b/modules/core/CMakeLists.txt @@ -10,6 +10,7 @@ ocv_add_dispatched_file(has_non_zero SSE2 AVX2 LASX ) ocv_add_dispatched_file(matmul SSE2 SSE4_1 AVX2 AVX512_SKX NEON_DOTPROD LASX) ocv_add_dispatched_file(mean SSE2 AVX2 LASX) ocv_add_dispatched_file(merge SSE2 AVX2 LASX) +ocv_add_dispatched_file(minmax SSE2 SSE4_1 AVX2 AVX512_SKX AVX512_ICL LASX) ocv_add_dispatched_file(split SSE2 AVX2 LASX) ocv_add_dispatched_file(sum SSE2 AVX2 LASX) ocv_add_dispatched_file(reduce SSE2 SSSE3 AVX2 NEON_DOTPROD) diff --git a/modules/core/src/minmax.cpp b/modules/core/src/minmax.cpp deleted file mode 100644 index ae5823679f..0000000000 --- a/modules/core/src/minmax.cpp +++ /dev/null @@ -1,1368 +0,0 @@ -// This file is part of OpenCV project. -// It is subject to the license terms in the LICENSE file found in the top-level directory -// of this distribution and at http://opencv.org/license.html - - -#include "precomp.hpp" -#include "opencl_kernels_core.hpp" -#include "stat.hpp" -#include "opencv2/core/detail/dispatch_helper.impl.hpp" - -#include - -/****************************************************************************************\ -* minMaxLoc * -\****************************************************************************************/ - -namespace cv -{ - -template static void -minMaxIdx_( const T* src, const uchar* mask, WT* _minVal, WT* _maxVal, - size_t* _minIdx, size_t* _maxIdx, int len, size_t startIdx ) -{ - WT minVal = *_minVal, maxVal = *_maxVal; - size_t minIdx = *_minIdx, maxIdx = *_maxIdx; - - if( !mask ) - { - for( int i = 0; i < len; i++ ) - { - T val = src[i]; - if( val < minVal ) - { - minVal = val; - minIdx = startIdx + i; - } - if( val > maxVal ) - { - maxVal = val; - maxIdx = startIdx + i; - } - } - } - else - { - for( int i = 0; i < len; i++ ) - { - T val = src[i]; - if( mask[i] && val < minVal ) - { - minVal = val; - minIdx = startIdx + i; - } - if( mask[i] && val > maxVal ) - { - maxVal = val; - maxIdx = startIdx + i; - } - } - } - - *_minIdx = minIdx; - *_maxIdx = maxIdx; - *_minVal = minVal; - *_maxVal = maxVal; -} - -#if CV_SIMD128 -template CV_ALWAYS_INLINE void -minMaxIdx_init( const T* src, const uchar* mask, WT* minval, WT* maxval, - size_t* minidx, size_t* maxidx, WT &minVal, WT &maxVal, - size_t &minIdx, size_t &maxIdx, const WT minInit, const WT maxInit, - const int nlanes, int len, size_t startidx, int &j, int &len0 ) -{ - len0 = len & -nlanes; - j = 0; - - minVal = *minval, maxVal = *maxval; - minIdx = *minidx, maxIdx = *maxidx; - - // To handle start values out of range - if ( minVal < minInit || maxVal < minInit || minVal > maxInit || maxVal > maxInit ) - { - uchar done = 0x00; - - for ( ; (j < len) && (done != 0x03); j++ ) - { - if ( !mask || mask[j] ) { - T val = src[j]; - if ( val < minVal ) - { - minVal = val; - minIdx = startidx + j; - done |= 0x01; - } - if ( val > maxVal ) - { - maxVal = val; - maxIdx = startidx + j; - done |= 0x02; - } - } - } - - len0 = j + ((len - j) & -nlanes); - } -} - -#if CV_SIMD128_64F -CV_ALWAYS_INLINE double v_reduce_min(const v_float64x2& a) -{ - double CV_DECL_ALIGNED(32) idx[2]; - v_store_aligned(idx, a); - return std::min(idx[0], idx[1]); -} - -CV_ALWAYS_INLINE double v_reduce_max(const v_float64x2& a) -{ - double CV_DECL_ALIGNED(32) idx[2]; - v_store_aligned(idx, a); - return std::max(idx[0], idx[1]); -} - -CV_ALWAYS_INLINE uint64_t v_reduce_min(const v_uint64x2& a) -{ - uint64_t CV_DECL_ALIGNED(32) idx[2]; - v_store_aligned(idx, a); - return std::min(idx[0], idx[1]); -} - -CV_ALWAYS_INLINE v_uint64x2 v_select(const v_uint64x2& mask, const v_uint64x2& a, const v_uint64x2& b) -{ - return v_xor(b, v_and(v_xor(a, b), mask)); -} -#endif - -#define MINMAXIDX_REDUCE(suffix, suffix2, maxLimit, IR) \ -template CV_ALWAYS_INLINE void \ -minMaxIdx_reduce_##suffix( VT &valMin, VT &valMax, IT &idxMin, IT &idxMax, IT &none, \ - T &minVal, T &maxVal, size_t &minIdx, size_t &maxIdx, \ - size_t delta ) \ -{ \ - if ( v_check_any(v_ne(idxMin, none)) ) \ - { \ - minVal = v_reduce_min(valMin); \ - minIdx = (size_t)v_reduce_min(v_select(v_reinterpret_as_##suffix2(v_eq(v_setall_##suffix((IR)minVal), valMin)), \ - idxMin, v_setall_##suffix2(maxLimit))) + delta; \ - } \ - if ( v_check_any(v_ne(idxMax, none)) ) \ - { \ - maxVal = v_reduce_max(valMax); \ - maxIdx = (size_t)v_reduce_min(v_select(v_reinterpret_as_##suffix2(v_eq(v_setall_##suffix((IR)maxVal), valMax)), \ - idxMax, v_setall_##suffix2(maxLimit))) + delta; \ - } \ -} - -MINMAXIDX_REDUCE(u8, u8, UCHAR_MAX, uchar) -MINMAXIDX_REDUCE(s8, u8, UCHAR_MAX, uchar) -MINMAXIDX_REDUCE(u16, u16, USHRT_MAX, ushort) -MINMAXIDX_REDUCE(s16, u16, USHRT_MAX, ushort) -MINMAXIDX_REDUCE(s32, u32, UINT_MAX, uint) -MINMAXIDX_REDUCE(f32, u32, (1 << 23) - 1, float) -#if CV_SIMD128_64F -MINMAXIDX_REDUCE(f64, u64, UINT_MAX, double) -#endif - -template CV_ALWAYS_INLINE void -minMaxIdx_finish( const T* src, const uchar* mask, WT* minval, WT* maxval, - size_t* minidx, size_t* maxidx, WT minVal, WT maxVal, - size_t minIdx, size_t maxIdx, int len, size_t startidx, - int j ) -{ - for ( ; j < len ; j++ ) - { - if ( !mask || mask[j] ) - { - T val = src[j]; - if ( val < minVal ) - { - minVal = val; - minIdx = startidx + j; - } - if ( val > maxVal ) - { - maxVal = val; - maxIdx = startidx + j; - } - } - } - - *minidx = minIdx; - *maxidx = maxIdx; - *minval = minVal; - *maxval = maxVal; -} -#endif - -static void minMaxIdx_8u(const void* src_, const uchar* mask, void* minval_, void* maxval_, - size_t* minidx, size_t* maxidx, int len, size_t startidx ) -{ - const uchar* src = static_cast(src_); - int* minval = static_cast(minval_); - int* maxval = static_cast(maxval_); -#if CV_SIMD128 - if ( len >= VTraits::vlanes() ) - { - int j, len0; - int minVal, maxVal; - size_t minIdx, maxIdx; - - minMaxIdx_init( src, mask, minval, maxval, minidx, maxidx, minVal, maxVal, minIdx, maxIdx, - (int)0, (int)UCHAR_MAX, VTraits::vlanes(), len, startidx, j, len0 ); - - if ( j <= len0 - VTraits::vlanes() ) - { - v_uint8x16 inc = v_setall_u8((uchar)VTraits::vlanes()); - v_uint8x16 none = v_reinterpret_as_u8(v_setall_s8(-1)); - v_uint8x16 idxStart(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); - - do - { - v_uint8x16 valMin = v_setall_u8((uchar)minVal), valMax = v_setall_u8((uchar)maxVal); - v_uint8x16 idx = idxStart, idxMin = none, idxMax = none; - - int k = j; - size_t delta = startidx + j; - - if ( !mask ) - { - for( ; k < std::min(len0, j + 15 * VTraits::vlanes()); k += VTraits::vlanes() ) - { - v_uint8x16 data = v_load(src + k); - v_uint8x16 cmpMin = (v_lt(data, valMin)); - v_uint8x16 cmpMax = (v_gt(data, valMax)); - idxMin = v_select(cmpMin, idx, idxMin); - idxMax = v_select(cmpMax, idx, idxMax); - valMin = v_min(data, valMin); - valMax = v_max(data, valMax); - idx = v_add(idx, inc); - } - } - else - { - for( ; k < std::min(len0, j + 15 * VTraits::vlanes()); k += VTraits::vlanes() ) - { - v_uint8x16 data = v_load(src + k); - v_uint8x16 maskVal = v_ne(v_load(mask + k), v_setzero_u8()); - v_uint8x16 cmpMin = v_and(v_lt(data, valMin), maskVal); - v_uint8x16 cmpMax = v_and(v_gt(data, valMax), maskVal); - idxMin = v_select(cmpMin, idx, idxMin); - idxMax = v_select(cmpMax, idx, idxMax); - valMin = v_select(cmpMin, data, valMin); - valMax = v_select(cmpMax, data, valMax); - idx = v_add(idx, inc); - } - } - - j = k; - - minMaxIdx_reduce_u8( valMin, valMax, idxMin, idxMax, none, minVal, maxVal, - minIdx, maxIdx, delta ); - } - while ( j < len0 ); - } - - minMaxIdx_finish( src, mask, minval, maxval, minidx, maxidx, minVal, maxVal, - minIdx, maxIdx, len, startidx, j ); - } - else - { - minMaxIdx_(src, mask, minval, maxval, minidx, maxidx, len, startidx); - } -#else - minMaxIdx_(src, mask, minval, maxval, minidx, maxidx, len, startidx); -#endif -} - -static void minMaxIdx_8s(const void* src_, const uchar* mask, void* minval_, void* maxval_, - size_t* minidx, size_t* maxidx, int len, size_t startidx ) -{ - const schar* src = static_cast(src_); - int* minval = static_cast(minval_); - int* maxval = static_cast(maxval_); -#if CV_SIMD128 - if ( len >= VTraits::vlanes() ) - { - int j, len0; - int minVal, maxVal; - size_t minIdx, maxIdx; - - minMaxIdx_init( src, mask, minval, maxval, minidx, maxidx, minVal, maxVal, minIdx, maxIdx, - (int)SCHAR_MIN, (int)SCHAR_MAX, VTraits::vlanes(), len, startidx, j, len0 ); - - if ( j <= len0 - VTraits::vlanes() ) - { - v_uint8x16 inc = v_setall_u8((uchar)VTraits::vlanes()); - v_uint8x16 none = v_reinterpret_as_u8(v_setall_s8(-1)); - v_uint8x16 idxStart(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); - - do - { - v_int8x16 valMin = v_setall_s8((schar)minVal), valMax = v_setall_s8((schar)maxVal); - v_uint8x16 idx = idxStart, idxMin = none, idxMax = none; - - int k = j; - size_t delta = startidx + j; - - if ( !mask ) - { - for( ; k < std::min(len0, j + 15 * VTraits::vlanes()); k += VTraits::vlanes() ) - { - v_int8x16 data = v_load(src + k); - v_uint8x16 cmpMin = v_reinterpret_as_u8(v_lt(data, valMin)); - v_uint8x16 cmpMax = v_reinterpret_as_u8(v_gt(data, valMax)); - idxMin = v_select(cmpMin, idx, idxMin); - idxMax = v_select(cmpMax, idx, idxMax); - valMin = v_min(data, valMin); - valMax = v_max(data, valMax); - idx = v_add(idx, inc); - } - } - else - { - for( ; k < std::min(len0, j + 15 * VTraits::vlanes()); k += VTraits::vlanes() ) - { - v_int8x16 data = v_load(src + k); - v_uint8x16 maskVal = v_ne(v_load(mask + k), v_setzero_u8()); - v_uint8x16 cmpMin = v_and(v_reinterpret_as_u8(v_lt(data, valMin)), maskVal); - v_uint8x16 cmpMax = v_and(v_reinterpret_as_u8(v_gt(data, valMax)), maskVal); - idxMin = v_select(cmpMin, idx, idxMin); - idxMax = v_select(cmpMax, idx, idxMax); - valMin = v_select(v_reinterpret_as_s8(cmpMin), data, valMin); - valMax = v_select(v_reinterpret_as_s8(cmpMax), data, valMax); - idx = v_add(idx, inc); - } - } - - j = k; - - minMaxIdx_reduce_s8( valMin, valMax, idxMin, idxMax, none, minVal, maxVal, - minIdx, maxIdx, delta ); - } - while ( j < len0 ); - } - - minMaxIdx_finish( src, mask, minval, maxval, minidx, maxidx, minVal, maxVal, - minIdx, maxIdx, len, startidx, j ); - } - else - { - minMaxIdx_(src, mask, minval, maxval, minidx, maxidx, len, startidx); - } -#else - minMaxIdx_(src, mask, minval, maxval, minidx, maxidx, len, startidx ); -#endif -} - -static void minMaxIdx_16u(const void* src_, const uchar* mask, void* minval_, void* maxval_, - size_t* minidx, size_t* maxidx, int len, size_t startidx ) -{ - const ushort* src = static_cast(src_); - int* minval = static_cast(minval_); - int* maxval = static_cast(maxval_); -#if CV_SIMD128 - if ( len >= VTraits::vlanes() ) - { - int j, len0; - int minVal, maxVal; - size_t minIdx, maxIdx; - - minMaxIdx_init( src, mask, minval, maxval, minidx, maxidx, minVal, maxVal, minIdx, maxIdx, - (int)0, (int)USHRT_MAX, VTraits::vlanes(), len, startidx, j, len0 ); - - if ( j <= len0 - VTraits::vlanes() ) - { - v_uint16x8 inc = v_setall_u16((uchar)VTraits::vlanes()); - v_uint16x8 none = v_reinterpret_as_u16(v_setall_s16(-1)); - v_uint16x8 idxStart(0, 1, 2, 3, 4, 5, 6, 7); - - do - { - v_uint16x8 valMin = v_setall_u16((ushort)minVal), valMax = v_setall_u16((ushort)maxVal); - v_uint16x8 idx = idxStart, idxMin = none, idxMax = none; - - int k = j; - size_t delta = startidx + j; - - if ( !mask ) - { - for( ; k < std::min(len0, j + 8191 * VTraits::vlanes()); k += VTraits::vlanes() ) - { - v_uint16x8 data = v_load(src + k); - v_uint16x8 cmpMin = (v_lt(data, valMin)); - v_uint16x8 cmpMax = (v_gt(data, valMax)); - idxMin = v_select(cmpMin, idx, idxMin); - idxMax = v_select(cmpMax, idx, idxMax); - valMin = v_min(data, valMin); - valMax = v_max(data, valMax); - idx = v_add(idx, inc); - } - } - else - { - for( ; k < std::min(len0, j + 8191 * VTraits::vlanes()); k += VTraits::vlanes() ) - { - v_uint16x8 data = v_load(src + k); - v_uint16x8 maskVal = v_ne(v_load_expand(mask + k), v_setzero_u16()); - v_uint16x8 cmpMin = v_and(v_lt(data, valMin), maskVal); - v_uint16x8 cmpMax = v_and(v_gt(data, valMax), maskVal); - idxMin = v_select(cmpMin, idx, idxMin); - idxMax = v_select(cmpMax, idx, idxMax); - valMin = v_select(cmpMin, data, valMin); - valMax = v_select(cmpMax, data, valMax); - idx = v_add(idx, inc); - } - } - - j = k; - - minMaxIdx_reduce_u16( valMin, valMax, idxMin, idxMax, none, minVal, maxVal, - minIdx, maxIdx, delta ); - } - while ( j < len0 ); - } - - minMaxIdx_finish( src, mask, minval, maxval, minidx, maxidx, minVal, maxVal, - minIdx, maxIdx, len, startidx, j ); - } - else - { - minMaxIdx_(src, mask, minval, maxval, minidx, maxidx, len, startidx); - } -#else - minMaxIdx_(src, mask, minval, maxval, minidx, maxidx, len, startidx ); -#endif -} - -static void minMaxIdx_16s(const void* src_, const uchar* mask, void* minval_, void* maxval_, - size_t* minidx, size_t* maxidx, int len, size_t startidx ) -{ - const short* src = static_cast(src_); - int* minval = static_cast(minval_); - int* maxval = static_cast(maxval_); -#if CV_SIMD128 - if ( len >= VTraits::vlanes() ) - { - int j, len0; - int minVal, maxVal; - size_t minIdx, maxIdx; - - minMaxIdx_init( src, mask, minval, maxval, minidx, maxidx, minVal, maxVal, minIdx, maxIdx, - (int)SHRT_MIN, (int)SHRT_MAX, VTraits::vlanes(), len, startidx, j, len0 ); - - if ( j <= len0 - VTraits::vlanes() ) - { - v_uint16x8 inc = v_setall_u16((uchar)VTraits::vlanes()); - v_uint16x8 none = v_reinterpret_as_u16(v_setall_s16(-1)); - v_uint16x8 idxStart(0, 1, 2, 3, 4, 5, 6, 7); - - do - { - v_int16x8 valMin = v_setall_s16((short)minVal), valMax = v_setall_s16((short)maxVal); - v_uint16x8 idx = idxStart, idxMin = none, idxMax = none; - - int k = j; - size_t delta = startidx + j; - - if ( !mask ) - { - for( ; k < std::min(len0, j + 8191 * VTraits::vlanes()); k += VTraits::vlanes() ) - { - v_int16x8 data = v_load(src + k); - v_uint16x8 cmpMin = v_reinterpret_as_u16(v_lt(data, valMin)); - v_uint16x8 cmpMax = v_reinterpret_as_u16(v_gt(data, valMax)); - idxMin = v_select(cmpMin, idx, idxMin); - idxMax = v_select(cmpMax, idx, idxMax); - valMin = v_min(data, valMin); - valMax = v_max(data, valMax); - idx = v_add(idx, inc); - } - } - else - { - for( ; k < std::min(len0, j + 8191 * VTraits::vlanes()); k += VTraits::vlanes() ) - { - v_int16x8 data = v_load(src + k); - v_uint16x8 maskVal = v_ne(v_load_expand(mask + k), v_setzero_u16()); - v_uint16x8 cmpMin = v_and(v_reinterpret_as_u16(v_lt(data, valMin)), maskVal); - v_uint16x8 cmpMax = v_and(v_reinterpret_as_u16(v_gt(data, valMax)), maskVal); - idxMin = v_select(cmpMin, idx, idxMin); - idxMax = v_select(cmpMax, idx, idxMax); - valMin = v_select(v_reinterpret_as_s16(cmpMin), data, valMin); - valMax = v_select(v_reinterpret_as_s16(cmpMax), data, valMax); - idx = v_add(idx, inc); - } - } - - j = k; - - minMaxIdx_reduce_s16( valMin, valMax, idxMin, idxMax, none, minVal, maxVal, - minIdx, maxIdx, delta ); - } - while ( j < len0 ); - } - - minMaxIdx_finish( src, mask, minval, maxval, minidx, maxidx, minVal, maxVal, - minIdx, maxIdx, len, startidx, j ); - } - else - { - minMaxIdx_(src, mask, minval, maxval, minidx, maxidx, len, startidx); - } -#else - minMaxIdx_(src, mask, minval, maxval, minidx, maxidx, len, startidx ); -#endif -} - -static void minMaxIdx_32s(const void* src_, const uchar* mask, void* minval_, void* maxval_, - size_t* minidx, size_t* maxidx, int len, size_t startidx ) -{ - const int* src = static_cast(src_); - int* minval = static_cast(minval_); - int* maxval = static_cast(maxval_); -#if CV_SIMD128 - if ( len >= 2 * VTraits::vlanes() ) - { - int j = 0, len0 = len & -(2 * VTraits::vlanes()); - int minVal = *minval, maxVal = *maxval; - size_t minIdx = *minidx, maxIdx = *maxidx; - - { - v_uint32x4 inc = v_setall_u32(VTraits::vlanes()); - v_uint32x4 none = v_reinterpret_as_u32(v_setall_s32(-1)); - v_uint32x4 idxStart(0, 1, 2, 3); - - do - { - v_int32x4 valMin = v_setall_s32(minVal), valMax = v_setall_s32(maxVal); - v_uint32x4 idx = idxStart, idxMin = none, idxMax = none; - - int k = j; - size_t delta = startidx + j; - - if ( !mask ) - { - for( ; k < std::min(len0, j + 32766 * 2 * VTraits::vlanes()); k += 2 * VTraits::vlanes() ) - { - v_int32x4 data = v_load(src + k); - v_uint32x4 cmpMin = v_reinterpret_as_u32(v_lt(data, valMin)); - v_uint32x4 cmpMax = v_reinterpret_as_u32(v_gt(data, valMax)); - idxMin = v_select(cmpMin, idx, idxMin); - idxMax = v_select(cmpMax, idx, idxMax); - valMin = v_min(data, valMin); - valMax = v_max(data, valMax); - idx = v_add(idx, inc); - data = v_load(src + k + VTraits::vlanes()); - cmpMin = v_reinterpret_as_u32(v_lt(data, valMin)); - cmpMax = v_reinterpret_as_u32(v_gt(data, valMax)); - idxMin = v_select(cmpMin, idx, idxMin); - idxMax = v_select(cmpMax, idx, idxMax); - valMin = v_min(data, valMin); - valMax = v_max(data, valMax); - idx = v_add(idx, inc); - } - } - else - { - for( ; k < std::min(len0, j + 32766 * 2 * VTraits::vlanes()); k += 2 * VTraits::vlanes() ) - { - v_int32x4 data = v_load(src + k); - v_uint16x8 maskVal = v_ne(v_load_expand(mask + k), v_setzero_u16()); - v_int32x4 maskVal1, maskVal2; - v_expand(v_reinterpret_as_s16(maskVal), maskVal1, maskVal2); - v_uint32x4 cmpMin = v_reinterpret_as_u32(v_and(v_lt(data, valMin), maskVal1)); - v_uint32x4 cmpMax = v_reinterpret_as_u32(v_and(v_gt(data, valMax), maskVal1)); - idxMin = v_select(cmpMin, idx, idxMin); - idxMax = v_select(cmpMax, idx, idxMax); - valMin = v_select(v_reinterpret_as_s32(cmpMin), data, valMin); - valMax = v_select(v_reinterpret_as_s32(cmpMax), data, valMax); - idx = v_add(idx, inc); - data = v_load(src + k + VTraits::vlanes()); - cmpMin = v_reinterpret_as_u32(v_and(v_lt(data, valMin), maskVal2)); - cmpMax = v_reinterpret_as_u32(v_and(v_gt(data, valMax), maskVal2)); - idxMin = v_select(cmpMin, idx, idxMin); - idxMax = v_select(cmpMax, idx, idxMax); - valMin = v_select(v_reinterpret_as_s32(cmpMin), data, valMin); - valMax = v_select(v_reinterpret_as_s32(cmpMax), data, valMax); - idx = v_add(idx, inc); - } - } - - j = k; - - minMaxIdx_reduce_s32( valMin, valMax, idxMin, idxMax, none, minVal, maxVal, - minIdx, maxIdx, delta ); - } - while ( j < len0 ); - } - - minMaxIdx_finish( src, mask, minval, maxval, minidx, maxidx, minVal, maxVal, - minIdx, maxIdx, len, startidx, j ); - } - else - { - minMaxIdx_(src, mask, minval, maxval, minidx, maxidx, len, startidx); - } -#else - minMaxIdx_(src, mask, minval, maxval, minidx, maxidx, len, startidx ); -#endif -} - -static void minMaxIdx_32f(const void* src_, const uchar* mask, void* minval_, void* maxval_, - size_t* minidx, size_t* maxidx, int len, size_t startidx ) -{ - const float* src = static_cast(src_); - float* minval = static_cast(minval_); - float* maxval = static_cast(maxval_); -#if CV_SIMD128 - if ( len >= 2 * VTraits::vlanes() ) - { - int j, len0; - float minVal, maxVal; - size_t minIdx, maxIdx; - - minMaxIdx_init( src, mask, minval, maxval, minidx, maxidx, minVal, maxVal, minIdx, maxIdx, - FLT_MIN, FLT_MAX, 2 * VTraits::vlanes(), len, startidx, j, len0 ); - - if ( j <= len0 - 2 * VTraits::vlanes() ) - { - v_uint32x4 inc = v_setall_u32(VTraits::vlanes()); - v_uint32x4 none = v_reinterpret_as_u32(v_setall_s32(-1)); - v_uint32x4 idxStart(0, 1, 2, 3); - - do - { - v_float32x4 valMin = v_setall_f32(minVal), valMax = v_setall_f32(maxVal); - v_uint32x4 idx = idxStart, idxMin = none, idxMax = none; - - int k = j; - size_t delta = startidx + j; - - if ( !mask ) - { - for( ; k < std::min(len0, j + 32766 * 2 * VTraits::vlanes()); k += 2 * VTraits::vlanes() ) - { - v_float32x4 data = v_load(src + k); - v_uint32x4 cmpMin = v_reinterpret_as_u32(v_lt(data, valMin)); - v_uint32x4 cmpMax = v_reinterpret_as_u32(v_gt(data, valMax)); - idxMin = v_select(cmpMin, idx, idxMin); - idxMax = v_select(cmpMax, idx, idxMax); - valMin = v_min(data, valMin); - valMax = v_max(data, valMax); - idx = v_add(idx, inc); - data = v_load(src + k + VTraits::vlanes()); - cmpMin = v_reinterpret_as_u32(v_lt(data, valMin)); - cmpMax = v_reinterpret_as_u32(v_gt(data, valMax)); - idxMin = v_select(cmpMin, idx, idxMin); - idxMax = v_select(cmpMax, idx, idxMax); - valMin = v_min(data, valMin); - valMax = v_max(data, valMax); - idx = v_add(idx, inc); - } - } - else - { - for( ; k < std::min(len0, j + 32766 * 2 * VTraits::vlanes()); k += 2 * VTraits::vlanes() ) - { - v_float32x4 data = v_load(src + k); - v_uint16x8 maskVal = v_ne(v_load_expand(mask + k), v_setzero_u16()); - v_int32x4 maskVal1, maskVal2; - v_expand(v_reinterpret_as_s16(maskVal), maskVal1, maskVal2); - v_uint32x4 cmpMin = v_reinterpret_as_u32(v_and(v_reinterpret_as_s32(v_lt(data, valMin)), maskVal1)); - v_uint32x4 cmpMax = v_reinterpret_as_u32(v_and(v_reinterpret_as_s32(v_gt(data, valMax)), maskVal1)); - idxMin = v_select(cmpMin, idx, idxMin); - idxMax = v_select(cmpMax, idx, idxMax); - valMin = v_select(v_reinterpret_as_f32(cmpMin), data, valMin); - valMax = v_select(v_reinterpret_as_f32(cmpMax), data, valMax); - idx = v_add(idx, inc); - data = v_load(src + k + VTraits::vlanes()); - cmpMin = v_reinterpret_as_u32(v_and(v_reinterpret_as_s32(v_lt(data, valMin)), maskVal2)); - cmpMax = v_reinterpret_as_u32(v_and(v_reinterpret_as_s32(v_gt(data, valMax)), maskVal2)); - idxMin = v_select(cmpMin, idx, idxMin); - idxMax = v_select(cmpMax, idx, idxMax); - valMin = v_select(v_reinterpret_as_f32(cmpMin), data, valMin); - valMax = v_select(v_reinterpret_as_f32(cmpMax), data, valMax); - idx = v_add(idx, inc); - } - } - - j = k; - - minMaxIdx_reduce_f32( valMin, valMax, idxMin, idxMax, none, minVal, maxVal, - minIdx, maxIdx, delta ); - } - while ( j < len0 ); - } - - minMaxIdx_finish( src, mask, minval, maxval, minidx, maxidx, minVal, maxVal, - minIdx, maxIdx, len, startidx, j ); - } - else - { - minMaxIdx_(src, mask, minval, maxval, minidx, maxidx, len, startidx); - } -#else - minMaxIdx_(src, mask, minval, maxval, minidx, maxidx, len, startidx ); -#endif -} - -static void minMaxIdx_64f(const void* src_, const uchar* mask, void* minval_, void* maxval_, - size_t* minidx, size_t* maxidx, int len, size_t startidx ) -{ - const double* src = static_cast(src_); - double* minval = static_cast(minval_); - double* maxval = static_cast(maxval_); -#if CV_SIMD128_64F - if ( len >= 4 * VTraits::vlanes() ) - { - int j, len0; - double minVal, maxVal; - size_t minIdx, maxIdx; - - minMaxIdx_init( src, mask, minval, maxval, minidx, maxidx, minVal, maxVal, minIdx, maxIdx, - DBL_MIN, DBL_MAX, 4 * VTraits::vlanes(), len, startidx, j, len0 ); - - if ( j <= len0 - 4 * VTraits::vlanes() ) - { - v_uint64x2 inc = v_setall_u64(VTraits::vlanes()); - v_uint64x2 none = v_reinterpret_as_u64(v_setall_s64(-1)); - v_uint64x2 idxStart(0, 1); - - do - { - v_float64x2 valMin = v_setall_f64(minVal), valMax = v_setall_f64(maxVal); - v_uint64x2 idx = idxStart, idxMin = none, idxMax = none; - - int k = j; - size_t delta = startidx + j; - - if ( !mask ) - { - for( ; k < std::min(len0, j + 32764 * 4 * VTraits::vlanes()); k += 4 * VTraits::vlanes() ) - { - v_float64x2 data = v_load(src + k); - v_uint64x2 cmpMin = v_reinterpret_as_u64(v_lt(data, valMin)); - v_uint64x2 cmpMax = v_reinterpret_as_u64(v_gt(data, valMax)); - idxMin = v_select(cmpMin, idx, idxMin); - idxMax = v_select(cmpMax, idx, idxMax); - valMin = v_min(data, valMin); - valMax = v_max(data, valMax); - idx = v_add(idx, inc); - data = v_load(src + k + VTraits::vlanes()); - cmpMin = v_reinterpret_as_u64(v_lt(data, valMin)); - cmpMax = v_reinterpret_as_u64(v_gt(data, valMax)); - idxMin = v_select(cmpMin, idx, idxMin); - idxMax = v_select(cmpMax, idx, idxMax); - valMin = v_min(data, valMin); - valMax = v_max(data, valMax); - idx = v_add(idx, inc); - data = v_load(src + k + 2 * VTraits::vlanes()); - cmpMin = v_reinterpret_as_u64(v_lt(data, valMin)); - cmpMax = v_reinterpret_as_u64(v_gt(data, valMax)); - idxMin = v_select(cmpMin, idx, idxMin); - idxMax = v_select(cmpMax, idx, idxMax); - valMin = v_min(data, valMin); - valMax = v_max(data, valMax); - idx = v_add(idx, inc); - data = v_load(src + k + 3 * VTraits::vlanes()); - cmpMin = v_reinterpret_as_u64(v_lt(data, valMin)); - cmpMax = v_reinterpret_as_u64(v_gt(data, valMax)); - idxMin = v_select(cmpMin, idx, idxMin); - idxMax = v_select(cmpMax, idx, idxMax); - valMin = v_min(data, valMin); - valMax = v_max(data, valMax); - idx = v_add(idx, inc); - } - } - else - { - for( ; k < std::min(len0, j + 32764 * 4 * VTraits::vlanes()); k += 4 * VTraits::vlanes() ) - { - v_float64x2 data = v_load(src + k); - v_uint16x8 maskVal = v_ne(v_load_expand(mask + k), v_setzero_u16()); - v_int32x4 maskVal1, maskVal2; - v_expand(v_reinterpret_as_s16(maskVal), maskVal1, maskVal2); - v_int64x2 maskVal3, maskVal4; - v_expand(maskVal1, maskVal3, maskVal4); - v_uint64x2 cmpMin = v_reinterpret_as_u64(v_and(v_reinterpret_as_s64(v_lt(data, valMin)), maskVal3)); - v_uint64x2 cmpMax = v_reinterpret_as_u64(v_and(v_reinterpret_as_s64(v_gt(data, valMax)), maskVal3)); - idxMin = v_select(cmpMin, idx, idxMin); - idxMax = v_select(cmpMax, idx, idxMax); - valMin = v_select(v_reinterpret_as_f64(cmpMin), data, valMin); - valMax = v_select(v_reinterpret_as_f64(cmpMax), data, valMax); - idx = v_add(idx, inc); - data = v_load(src + k + VTraits::vlanes()); - cmpMin = v_reinterpret_as_u64(v_and(v_reinterpret_as_s64(v_lt(data, valMin)), maskVal4)); - cmpMax = v_reinterpret_as_u64(v_and(v_reinterpret_as_s64(v_gt(data, valMax)), maskVal4)); - idxMin = v_select(cmpMin, idx, idxMin); - idxMax = v_select(cmpMax, idx, idxMax); - valMin = v_select(v_reinterpret_as_f64(cmpMin), data, valMin); - valMax = v_select(v_reinterpret_as_f64(cmpMax), data, valMax); - idx = v_add(idx, inc); - data = v_load(src + k + 2 * VTraits::vlanes()); - v_expand(maskVal2, maskVal3, maskVal4); - cmpMin = v_reinterpret_as_u64(v_and(v_reinterpret_as_s64(v_lt(data, valMin)), maskVal3)); - cmpMax = v_reinterpret_as_u64(v_and(v_reinterpret_as_s64(v_gt(data, valMax)), maskVal3)); - idxMin = v_select(cmpMin, idx, idxMin); - idxMax = v_select(cmpMax, idx, idxMax); - valMin = v_select(v_reinterpret_as_f64(cmpMin), data, valMin); - valMax = v_select(v_reinterpret_as_f64(cmpMax), data, valMax); - idx = v_add(idx, inc); - data = v_load(src + k + 3 * VTraits::vlanes()); - cmpMin = v_reinterpret_as_u64(v_and(v_reinterpret_as_s64(v_lt(data, valMin)), maskVal4)); - cmpMax = v_reinterpret_as_u64(v_and(v_reinterpret_as_s64(v_gt(data, valMax)), maskVal4)); - idxMin = v_select(cmpMin, idx, idxMin); - idxMax = v_select(cmpMax, idx, idxMax); - valMin = v_select(v_reinterpret_as_f64(cmpMin), data, valMin); - valMax = v_select(v_reinterpret_as_f64(cmpMax), data, valMax); - idx = v_add(idx, inc); - } - } - - j = k; - - minMaxIdx_reduce_f64( valMin, valMax, idxMin, idxMax, none, minVal, maxVal, - minIdx, maxIdx, delta ); - } - while ( j < len0 ); - } - - minMaxIdx_finish( src, mask, minval, maxval, minidx, maxidx, minVal, maxVal, - minIdx, maxIdx, len, startidx, j ); - } - else - { - minMaxIdx_(src, mask, minval, maxval, minidx, maxidx, len, startidx); - } -#else - minMaxIdx_(src, mask, minval, maxval, minidx, maxidx, len, startidx ); -#endif -} - -typedef void (*MinMaxIdxFunc)(const void*, const uchar*, void*, void*, size_t*, size_t*, int, size_t); - -static MinMaxIdxFunc getMinmaxTab(int depth) -{ - static MinMaxIdxFunc minmaxTab[CV_DEPTH_MAX] = - { - GET_OPTIMIZED(minMaxIdx_8u), GET_OPTIMIZED(minMaxIdx_8s), - GET_OPTIMIZED(minMaxIdx_16u), GET_OPTIMIZED(minMaxIdx_16s), - GET_OPTIMIZED(minMaxIdx_32s), - GET_OPTIMIZED(minMaxIdx_32f), GET_OPTIMIZED(minMaxIdx_64f), - 0 - }; - - return minmaxTab[depth]; -} - -// The function expects 1-based indexing for ofs -// Zero is treated as invalid offset (not found) -static void ofs2idx(const Mat& a, size_t ofs, int* idx) -{ - int i, d = a.dims; - if( ofs > 0 ) - { - ofs--; - for( i = d-1; i >= 0; i-- ) - { - int sz = a.size[i]; - idx[i] = (int)(ofs % sz); - ofs /= sz; - } - } - else - { - for( i = d-1; i >= 0; i-- ) - idx[i] = -1; - } -} - -#ifdef HAVE_OPENCL - -#define MINMAX_STRUCT_ALIGNMENT 8 // sizeof double - -template -void getMinMaxRes(const Mat & db, double * minVal, double * maxVal, - int* minLoc, int* maxLoc, - int groupnum, int cols, double * maxVal2) -{ - uint index_max = std::numeric_limits::max(); - T minval = std::numeric_limits::max(); - T maxval = std::numeric_limits::min() > 0 ? -std::numeric_limits::max() : std::numeric_limits::min(), maxval2 = maxval; - uint minloc = index_max, maxloc = index_max; - - size_t index = 0; - const T * minptr = NULL, * maxptr = NULL, * maxptr2 = NULL; - const uint * minlocptr = NULL, * maxlocptr = NULL; - if (minVal || minLoc) - { - minptr = db.ptr(); - index += sizeof(T) * groupnum; - index = alignSize(index, MINMAX_STRUCT_ALIGNMENT); - } - if (maxVal || maxLoc) - { - maxptr = (const T *)(db.ptr() + index); - index += sizeof(T) * groupnum; - index = alignSize(index, MINMAX_STRUCT_ALIGNMENT); - } - if (minLoc) - { - minlocptr = (const uint *)(db.ptr() + index); - index += sizeof(uint) * groupnum; - index = alignSize(index, MINMAX_STRUCT_ALIGNMENT); - } - if (maxLoc) - { - maxlocptr = (const uint *)(db.ptr() + index); - index += sizeof(uint) * groupnum; - index = alignSize(index, MINMAX_STRUCT_ALIGNMENT); - } - if (maxVal2) - maxptr2 = (const T *)(db.ptr() + index); - - for (int i = 0; i < groupnum; i++) - { - if (minptr && minptr[i] <= minval) - { - if (minptr[i] == minval) - { - if (minlocptr) - minloc = std::min(minlocptr[i], minloc); - } - else - { - if (minlocptr) - minloc = minlocptr[i]; - minval = minptr[i]; - } - } - if (maxptr && maxptr[i] >= maxval) - { - if (maxptr[i] == maxval) - { - if (maxlocptr) - maxloc = std::min(maxlocptr[i], maxloc); - } - else - { - if (maxlocptr) - maxloc = maxlocptr[i]; - maxval = maxptr[i]; - } - } - if (maxptr2 && maxptr2[i] > maxval2) - maxval2 = maxptr2[i]; - } - bool zero_mask = (minLoc && minloc == index_max) || - (maxLoc && maxloc == index_max); - - if (minVal) - *minVal = zero_mask ? 0 : (double)minval; - if (maxVal) - *maxVal = zero_mask ? 0 : (double)maxval; - if (maxVal2) - *maxVal2 = zero_mask ? 0 : (double)maxval2; - - if (minLoc) - { - minLoc[0] = zero_mask ? -1 : minloc / cols; - minLoc[1] = zero_mask ? -1 : minloc % cols; - } - if (maxLoc) - { - maxLoc[0] = zero_mask ? -1 : maxloc / cols; - maxLoc[1] = zero_mask ? -1 : maxloc % cols; - } -} - -typedef void (*getMinMaxResFunc)(const Mat & db, double * minVal, double * maxVal, - int * minLoc, int *maxLoc, int gropunum, int cols, double * maxVal2); - -bool ocl_minMaxIdx( InputArray _src, double* minVal, double* maxVal, int* minLoc, int* maxLoc, InputArray _mask, - int ddepth, bool absValues, InputArray _src2, double * maxVal2) -{ - const ocl::Device & dev = ocl::Device::getDefault(); - -#ifdef __ANDROID__ - if (dev.isNVidia()) - return false; -#endif - - if (dev.deviceVersionMajor() == 1 && dev.deviceVersionMinor() < 2) - { - // 'static' storage class specifier used by "minmaxloc" is available from OpenCL 1.2+ only - return false; - } - - bool doubleSupport = dev.doubleFPConfig() > 0, haveMask = !_mask.empty(), - haveSrc2 = _src2.kind() != _InputArray::NONE; - int type = _src.type(), depth = CV_MAT_DEPTH(type), cn = CV_MAT_CN(type), - kercn = haveMask ? cn : std::min(4, ocl::predictOptimalVectorWidth(_src, _src2)); - - if (depth >= CV_16F) - return false; - - // disabled following modes since it occasionally fails on AMD devices (e.g. A10-6800K, sep. 2014) - if ((haveMask || type == CV_32FC1) && dev.isAMD()) - return false; - - CV_Assert( (cn == 1 && (!haveMask || _mask.type() == CV_8U)) || - (cn >= 1 && !minLoc && !maxLoc) ); - - if (ddepth < 0) - ddepth = depth; - - CV_Assert(!haveSrc2 || _src2.type() == type); - - if (depth == CV_32S) - return false; - - if ((depth == CV_64F || ddepth == CV_64F) && !doubleSupport) - return false; - - int groupnum = dev.maxComputeUnits(); - size_t wgs = dev.maxWorkGroupSize(); - - int wgs2_aligned = 1; - while (wgs2_aligned < (int)wgs) - wgs2_aligned <<= 1; - wgs2_aligned >>= 1; - - bool needMinVal = minVal || minLoc, needMinLoc = minLoc != NULL, - needMaxVal = maxVal || maxLoc, needMaxLoc = maxLoc != NULL; - - // in case of mask we must know whether mask is filled with zeros or not - // so let's calculate min or max location, if it's undefined, so mask is zeros - if (!(needMaxLoc || needMinLoc) && haveMask) - { - if (needMinVal) - needMinLoc = true; - else - needMaxLoc = true; - } - - char cvt[2][50]; - String opts = format("-D DEPTH_%d -D srcT1=%s%s -D WGS=%d -D srcT=%s" - " -D WGS2_ALIGNED=%d%s%s%s -D kercn=%d%s%s%s%s" - " -D dstT1=%s -D dstT=%s -D convertToDT=%s%s%s%s%s -D wdepth=%d -D convertFromU=%s" - " -D MINMAX_STRUCT_ALIGNMENT=%d", - depth, ocl::typeToStr(depth), haveMask ? " -D HAVE_MASK" : "", (int)wgs, - ocl::typeToStr(CV_MAKE_TYPE(depth, kercn)), wgs2_aligned, - doubleSupport ? " -D DOUBLE_SUPPORT" : "", - _src.isContinuous() ? " -D HAVE_SRC_CONT" : "", - _mask.isContinuous() ? " -D HAVE_MASK_CONT" : "", kercn, - needMinVal ? " -D NEED_MINVAL" : "", needMaxVal ? " -D NEED_MAXVAL" : "", - needMinLoc ? " -D NEED_MINLOC" : "", needMaxLoc ? " -D NEED_MAXLOC" : "", - ocl::typeToStr(ddepth), ocl::typeToStr(CV_MAKE_TYPE(ddepth, kercn)), - ocl::convertTypeStr(depth, ddepth, kercn, cvt[0], sizeof(cvt[0])), - absValues ? " -D OP_ABS" : "", - haveSrc2 ? " -D HAVE_SRC2" : "", maxVal2 ? " -D OP_CALC2" : "", - haveSrc2 && _src2.isContinuous() ? " -D HAVE_SRC2_CONT" : "", ddepth, - depth <= CV_32S && ddepth == CV_32S ? ocl::convertTypeStr(CV_8U, ddepth, kercn, cvt[1], sizeof(cvt[1])) : "noconvert", - MINMAX_STRUCT_ALIGNMENT); - - ocl::Kernel k("minmaxloc", ocl::core::minmaxloc_oclsrc, opts); - if (k.empty()) - return false; - - int esz = CV_ELEM_SIZE(ddepth), esz32s = CV_ELEM_SIZE1(CV_32S), - dbsize = groupnum * ((needMinVal ? esz : 0) + (needMaxVal ? esz : 0) + - (needMinLoc ? esz32s : 0) + (needMaxLoc ? esz32s : 0) + - (maxVal2 ? esz : 0)) - + 5 * MINMAX_STRUCT_ALIGNMENT; - UMat src = _src.getUMat(), src2 = _src2.getUMat(), db(1, dbsize, CV_8UC1), mask = _mask.getUMat(); - - if (cn > 1 && !haveMask) - { - src = src.reshape(1); - src2 = src2.reshape(1); - } - - if (haveSrc2) - { - if (!haveMask) - k.args(ocl::KernelArg::ReadOnlyNoSize(src), src.cols, (int)src.total(), - groupnum, ocl::KernelArg::PtrWriteOnly(db), ocl::KernelArg::ReadOnlyNoSize(src2)); - else - k.args(ocl::KernelArg::ReadOnlyNoSize(src), src.cols, (int)src.total(), - groupnum, ocl::KernelArg::PtrWriteOnly(db), ocl::KernelArg::ReadOnlyNoSize(mask), - ocl::KernelArg::ReadOnlyNoSize(src2)); - } - else - { - if (!haveMask) - k.args(ocl::KernelArg::ReadOnlyNoSize(src), src.cols, (int)src.total(), - groupnum, ocl::KernelArg::PtrWriteOnly(db)); - else - k.args(ocl::KernelArg::ReadOnlyNoSize(src), src.cols, (int)src.total(), - groupnum, ocl::KernelArg::PtrWriteOnly(db), ocl::KernelArg::ReadOnlyNoSize(mask)); - } - - size_t globalsize = groupnum * wgs; - if (!k.run(1, &globalsize, &wgs, true)) - return false; - - static const getMinMaxResFunc functab[7] = - { - getMinMaxRes, - getMinMaxRes, - getMinMaxRes, - getMinMaxRes, - getMinMaxRes, - getMinMaxRes, - getMinMaxRes - }; - - CV_Assert(ddepth <= CV_64F); - getMinMaxResFunc func = functab[ddepth]; - - int locTemp[2]; - func(db.getMat(ACCESS_READ), minVal, maxVal, - needMinLoc ? minLoc ? minLoc : locTemp : minLoc, - needMaxLoc ? maxLoc ? maxLoc : locTemp : maxLoc, - groupnum, src.cols, maxVal2); - - return true; -} - -#endif - -} - -void cv::minMaxIdx(InputArray _src, double* minVal, - double* maxVal, int* minIdx, int* maxIdx, - InputArray _mask) -{ - CV_INSTRUMENT_REGION(); - - int type = _src.type(), depth = CV_MAT_DEPTH(type), cn = CV_MAT_CN(type); - CV_Assert( (cn == 1 && (_mask.empty() || _mask.type() == CV_8U)) || - (cn > 1 && _mask.empty() && !minIdx && !maxIdx) ); - - CV_OCL_RUN(OCL_PERFORMANCE_CHECK(_src.isUMat()) && _src.dims() <= 2 && (_mask.empty() || _src.size() == _mask.size()), - ocl_minMaxIdx(_src, minVal, maxVal, minIdx, maxIdx, _mask)) - - Mat src = _src.getMat(), mask = _mask.getMat(); - - if (src.dims <= 2) - { - if ((size_t)src.step == (size_t)mask.step || mask.empty()) - { - CALL_HAL(minMaxIdx, cv_hal_minMaxIdx, src.data, src.step, src.cols*cn, src.rows, - src.depth(), minVal, maxVal, minIdx, maxIdx, mask.data); - } - - CALL_HAL(minMaxIdxMaskStep, cv_hal_minMaxIdxMaskStep, src.data, src.step, src.cols*cn, src.rows, - src.depth(), minVal, maxVal, minIdx, maxIdx, mask.data, mask.step); - } - else if (src.isContinuous() && (mask.isContinuous() || mask.empty())) - { - int res = cv_hal_minMaxIdx(src.data, 0, (int)src.total()*cn, 1, src.depth(), - minVal, maxVal, minIdx, maxIdx, mask.data); - - if (res == CV_HAL_ERROR_OK) - { - // minIdx[0] and minIdx[0] are always 0 for "flatten" version - if (minIdx) - ofs2idx(src, minIdx[1]+1, minIdx); - if (maxIdx) - ofs2idx(src, maxIdx[1]+1, maxIdx); - return; - } - else if (res != CV_HAL_ERROR_NOT_IMPLEMENTED) - { - CV_Error_(cv::Error::StsInternal, - ("HAL implementation minMaxIdx ==> " CVAUX_STR(cv_hal_minMaxIdx) " returned %d (0x%08x)", res, res)); - } - } - - MinMaxIdxFunc func = getMinmaxTab(depth); - CV_Assert( func != 0 ); - - const Mat* arrays[] = {&src, &mask, 0}; - uchar* ptrs[2] = {}; - NAryMatIterator it(arrays, ptrs); - - size_t minidx = 0, maxidx = 0; - int iminval = INT_MAX, imaxval = INT_MIN; - float fminval = std::numeric_limits::infinity(), fmaxval = -fminval; - double dminval = std::numeric_limits::infinity(), dmaxval = -dminval; - size_t startidx = 1; - void *minval = &iminval, *maxval = &imaxval; - int planeSize = (int)it.size*cn; - - if( depth == CV_32F ) - minval = &fminval, maxval = &fmaxval; - else if( depth == CV_64F ) - minval = &dminval, maxval = &dmaxval; - - for( size_t i = 0; i < it.nplanes; i++, ++it, startidx += planeSize ) - func( ptrs[0], ptrs[1], minval, maxval, &minidx, &maxidx, planeSize, startidx ); - - if (!src.empty() && mask.empty()) - { - if( minidx == 0 ) - minidx = 1; - if( maxidx == 0 ) - maxidx = 1; - } - - if( minidx == 0 ) - dminval = dmaxval = 0; - else if( depth == CV_32F ) - dminval = fminval, dmaxval = fmaxval; - else if( depth <= CV_32S ) - dminval = iminval, dmaxval = imaxval; - - if( minVal ) - *minVal = dminval; - if( maxVal ) - *maxVal = dmaxval; - - if( minIdx ) - ofs2idx(src, minidx, minIdx); - if( maxIdx ) - ofs2idx(src, maxidx, maxIdx); -} - -void cv::minMaxLoc( InputArray _img, double* minVal, double* maxVal, - Point* minLoc, Point* maxLoc, InputArray mask ) -{ - CV_INSTRUMENT_REGION(); - - int dims = _img.dims(); - CV_CheckLE(dims, 2, ""); - - minMaxIdx(_img, minVal, maxVal, (int*)minLoc, (int*)maxLoc, mask); - if( minLoc ) - { - if (dims == 2) - std::swap(minLoc->x, minLoc->y); - else - minLoc->y = 0; - } - if( maxLoc ) - { - if (dims == 2) - std::swap(maxLoc->x, maxLoc->y); - else - maxLoc->y = 0; - } -} - -enum class ReduceMode -{ - FIRST_MIN = 0, //!< get index of first min occurrence - LAST_MIN = 1, //!< get index of last min occurrence - FIRST_MAX = 2, //!< get index of first max occurrence - LAST_MAX = 3, //!< get index of last max occurrence -}; - -template -struct reduceMinMaxImpl -{ - void operator()(const cv::Mat& src, cv::Mat& dst, ReduceMode mode, const int axis) const - { - switch(mode) - { - case ReduceMode::FIRST_MIN: - reduceMinMaxApply(src, dst, axis); - break; - case ReduceMode::LAST_MIN: - reduceMinMaxApply(src, dst, axis); - break; - case ReduceMode::FIRST_MAX: - reduceMinMaxApply(src, dst, axis); - break; - case ReduceMode::LAST_MAX: - reduceMinMaxApply(src, dst, axis); - break; - } - } - - template class Cmp> - static void reduceMinMaxApply(const cv::Mat& src, cv::Mat& dst, const int axis) - { - Cmp cmp; - - const auto *src_ptr = src.ptr(); - auto *dst_ptr = dst.ptr(); - - const size_t outer_size = src.total(0, axis); - const auto mid_size = static_cast(src.size[axis]); - - const size_t outer_step = src.total(axis); - const size_t dst_step = dst.total(axis); - - const size_t mid_step = src.total(axis + 1); - - for (size_t outer = 0; outer < outer_size; ++outer) - { - const size_t outer_offset = outer * outer_step; - const size_t dst_offset = outer * dst_step; - for (size_t mid = 0; mid != mid_size; ++mid) - { - const size_t src_offset = outer_offset + mid * mid_step; - for (size_t inner = 0; inner < mid_step; inner++) - { - int32_t& index = dst_ptr[dst_offset + inner]; - - const size_t prev = outer_offset + index * mid_step + inner; - const size_t curr = src_offset + inner; - - if (cmp(src_ptr[curr], src_ptr[prev])) - { - index = static_cast(mid); - } - } - } - } - } -}; - -static void reduceMinMax(cv::InputArray src, cv::OutputArray dst, ReduceMode mode, int axis) -{ - CV_INSTRUMENT_REGION(); - - cv::Mat srcMat = src.getMat(); - axis = (axis + srcMat.dims) % srcMat.dims; - CV_Assert(srcMat.channels() == 1 && axis >= 0 && axis < srcMat.dims); - - cv::AutoBuffer sizes(srcMat.dims); - std::copy(srcMat.size.p, srcMat.size.p + srcMat.dims, sizes.begin()); - sizes[axis] = 1; - - dst.create(srcMat.dims, sizes.data(), CV_32SC1); // indices - cv::Mat dstMat = dst.getMat(); - dstMat.setTo(cv::Scalar::all(0)); - - if (!srcMat.isContinuous()) - { - srcMat = srcMat.clone(); - } - - bool needs_copy = !dstMat.isContinuous(); - if (needs_copy) - { - dstMat = dstMat.clone(); - } - - cv::detail::depthDispatch(srcMat.depth(), srcMat, dstMat, mode, axis); - - if (needs_copy) - { - dstMat.copyTo(dst); - } -} - -void cv::reduceArgMin(InputArray src, OutputArray dst, int axis, bool lastIndex) -{ - reduceMinMax(src, dst, lastIndex ? ReduceMode::LAST_MIN : ReduceMode::FIRST_MIN, axis); -} - -void cv::reduceArgMax(InputArray src, OutputArray dst, int axis, bool lastIndex) -{ - reduceMinMax(src, dst, lastIndex ? ReduceMode::LAST_MAX : ReduceMode::FIRST_MAX, axis); -} diff --git a/modules/core/src/minmax.dispatch.cpp b/modules/core/src/minmax.dispatch.cpp new file mode 100644 index 0000000000..4352444dbb --- /dev/null +++ b/modules/core/src/minmax.dispatch.cpp @@ -0,0 +1,540 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html + +// Copyright (C) 2026, Advanced Micro Devices, Inc., all rights reserved. + +#include "precomp.hpp" +#include "opencl_kernels_core.hpp" +#include "stat.hpp" +#include "opencv2/core/detail/dispatch_helper.impl.hpp" + +#include + +#include "minmax.simd.hpp" +#include "minmax.simd_declarations.hpp" // defines CV_CPU_DISPATCH_MODES_ALL=AVX2,...,BASELINE based on CMakeLists.txt content + +/****************************************************************************************\ +* minMaxLoc * +\****************************************************************************************/ + +namespace cv +{ + +static MinMaxIdxFunc getMinmaxTab(int depth) +{ + CV_INSTRUMENT_REGION(); + CV_CPU_DISPATCH(getMinmaxTab, (depth), + CV_CPU_DISPATCH_MODES_ALL); +} + +// The function expects 1-based indexing for ofs +// Zero is treated as invalid offset (not found) +static void ofs2idx(const Mat& a, size_t ofs, int* idx) +{ + int i, d = a.dims; + if( ofs > 0 ) + { + ofs--; + for( i = d-1; i >= 0; i-- ) + { + int sz = a.size[i]; + idx[i] = (int)(ofs % sz); + ofs /= sz; + } + } + else + { + for( i = d-1; i >= 0; i-- ) + idx[i] = -1; + } +} + +#ifdef HAVE_OPENCL + +#define MINMAX_STRUCT_ALIGNMENT 8 // sizeof double + +template +void getMinMaxRes(const Mat & db, double * minVal, double * maxVal, + int* minLoc, int* maxLoc, + int groupnum, int cols, double * maxVal2) +{ + uint index_max = std::numeric_limits::max(); + T minval = std::numeric_limits::max(); + T maxval = std::numeric_limits::min() > 0 ? -std::numeric_limits::max() : std::numeric_limits::min(), maxval2 = maxval; + uint minloc = index_max, maxloc = index_max; + + size_t index = 0; + const T * minptr = NULL, * maxptr = NULL, * maxptr2 = NULL; + const uint * minlocptr = NULL, * maxlocptr = NULL; + if (minVal || minLoc) + { + minptr = db.ptr(); + index += sizeof(T) * groupnum; + index = alignSize(index, MINMAX_STRUCT_ALIGNMENT); + } + if (maxVal || maxLoc) + { + maxptr = (const T *)(db.ptr() + index); + index += sizeof(T) * groupnum; + index = alignSize(index, MINMAX_STRUCT_ALIGNMENT); + } + if (minLoc) + { + minlocptr = (const uint *)(db.ptr() + index); + index += sizeof(uint) * groupnum; + index = alignSize(index, MINMAX_STRUCT_ALIGNMENT); + } + if (maxLoc) + { + maxlocptr = (const uint *)(db.ptr() + index); + index += sizeof(uint) * groupnum; + index = alignSize(index, MINMAX_STRUCT_ALIGNMENT); + } + if (maxVal2) + maxptr2 = (const T *)(db.ptr() + index); + + for (int i = 0; i < groupnum; i++) + { + if (minptr && minptr[i] <= minval) + { + if (minptr[i] == minval) + { + if (minlocptr) + minloc = std::min(minlocptr[i], minloc); + } + else + { + if (minlocptr) + minloc = minlocptr[i]; + minval = minptr[i]; + } + } + if (maxptr && maxptr[i] >= maxval) + { + if (maxptr[i] == maxval) + { + if (maxlocptr) + maxloc = std::min(maxlocptr[i], maxloc); + } + else + { + if (maxlocptr) + maxloc = maxlocptr[i]; + maxval = maxptr[i]; + } + } + if (maxptr2 && maxptr2[i] > maxval2) + maxval2 = maxptr2[i]; + } + bool zero_mask = (minLoc && minloc == index_max) || + (maxLoc && maxloc == index_max); + + if (minVal) + *minVal = zero_mask ? 0 : (double)minval; + if (maxVal) + *maxVal = zero_mask ? 0 : (double)maxval; + if (maxVal2) + *maxVal2 = zero_mask ? 0 : (double)maxval2; + + if (minLoc) + { + minLoc[0] = zero_mask ? -1 : minloc / cols; + minLoc[1] = zero_mask ? -1 : minloc % cols; + } + if (maxLoc) + { + maxLoc[0] = zero_mask ? -1 : maxloc / cols; + maxLoc[1] = zero_mask ? -1 : maxloc % cols; + } +} + +typedef void (*getMinMaxResFunc)(const Mat & db, double * minVal, double * maxVal, + int * minLoc, int *maxLoc, int gropunum, int cols, double * maxVal2); + +bool ocl_minMaxIdx( InputArray _src, double* minVal, double* maxVal, int* minLoc, int* maxLoc, InputArray _mask, + int ddepth, bool absValues, InputArray _src2, double * maxVal2) +{ + const ocl::Device & dev = ocl::Device::getDefault(); + +#ifdef __ANDROID__ + if (dev.isNVidia()) + return false; +#endif + + if (dev.deviceVersionMajor() == 1 && dev.deviceVersionMinor() < 2) + { + // 'static' storage class specifier used by "minmaxloc" is available from OpenCL 1.2+ only + return false; + } + + bool doubleSupport = dev.doubleFPConfig() > 0, haveMask = !_mask.empty(), + haveSrc2 = _src2.kind() != _InputArray::NONE; + int type = _src.type(), depth = CV_MAT_DEPTH(type), cn = CV_MAT_CN(type), + kercn = haveMask ? cn : std::min(4, ocl::predictOptimalVectorWidth(_src, _src2)); + + if (depth >= CV_16F) + return false; + + // disabled following modes since it occasionally fails on AMD devices (e.g. A10-6800K, sep. 2014) + if ((haveMask || type == CV_32FC1) && dev.isAMD()) + return false; + + CV_Assert( (cn == 1 && (!haveMask || _mask.type() == CV_8U)) || + (cn >= 1 && !minLoc && !maxLoc) ); + + if (ddepth < 0) + ddepth = depth; + + CV_Assert(!haveSrc2 || _src2.type() == type); + + if (depth == CV_32S) + return false; + + if ((depth == CV_64F || ddepth == CV_64F) && !doubleSupport) + return false; + + int groupnum = dev.maxComputeUnits(); + size_t wgs = dev.maxWorkGroupSize(); + + int wgs2_aligned = 1; + while (wgs2_aligned < (int)wgs) + wgs2_aligned <<= 1; + wgs2_aligned >>= 1; + + bool needMinVal = minVal || minLoc, needMinLoc = minLoc != NULL, + needMaxVal = maxVal || maxLoc, needMaxLoc = maxLoc != NULL; + + // in case of mask we must know whether mask is filled with zeros or not + // so let's calculate min or max location, if it's undefined, so mask is zeros + if (!(needMaxLoc || needMinLoc) && haveMask) + { + if (needMinVal) + needMinLoc = true; + else + needMaxLoc = true; + } + + char cvt[2][50]; + String opts = format("-D DEPTH_%d -D srcT1=%s%s -D WGS=%d -D srcT=%s" + " -D WGS2_ALIGNED=%d%s%s%s -D kercn=%d%s%s%s%s" + " -D dstT1=%s -D dstT=%s -D convertToDT=%s%s%s%s%s -D wdepth=%d -D convertFromU=%s" + " -D MINMAX_STRUCT_ALIGNMENT=%d", + depth, ocl::typeToStr(depth), haveMask ? " -D HAVE_MASK" : "", (int)wgs, + ocl::typeToStr(CV_MAKE_TYPE(depth, kercn)), wgs2_aligned, + doubleSupport ? " -D DOUBLE_SUPPORT" : "", + _src.isContinuous() ? " -D HAVE_SRC_CONT" : "", + _mask.isContinuous() ? " -D HAVE_MASK_CONT" : "", kercn, + needMinVal ? " -D NEED_MINVAL" : "", needMaxVal ? " -D NEED_MAXVAL" : "", + needMinLoc ? " -D NEED_MINLOC" : "", needMaxLoc ? " -D NEED_MAXLOC" : "", + ocl::typeToStr(ddepth), ocl::typeToStr(CV_MAKE_TYPE(ddepth, kercn)), + ocl::convertTypeStr(depth, ddepth, kercn, cvt[0], sizeof(cvt[0])), + absValues ? " -D OP_ABS" : "", + haveSrc2 ? " -D HAVE_SRC2" : "", maxVal2 ? " -D OP_CALC2" : "", + haveSrc2 && _src2.isContinuous() ? " -D HAVE_SRC2_CONT" : "", ddepth, + depth <= CV_32S && ddepth == CV_32S ? ocl::convertTypeStr(CV_8U, ddepth, kercn, cvt[1], sizeof(cvt[1])) : "noconvert", + MINMAX_STRUCT_ALIGNMENT); + + ocl::Kernel k("minmaxloc", ocl::core::minmaxloc_oclsrc, opts); + if (k.empty()) + return false; + + int esz = CV_ELEM_SIZE(ddepth), esz32s = CV_ELEM_SIZE1(CV_32S), + dbsize = groupnum * ((needMinVal ? esz : 0) + (needMaxVal ? esz : 0) + + (needMinLoc ? esz32s : 0) + (needMaxLoc ? esz32s : 0) + + (maxVal2 ? esz : 0)) + + 5 * MINMAX_STRUCT_ALIGNMENT; + UMat src = _src.getUMat(), src2 = _src2.getUMat(), db(1, dbsize, CV_8UC1), mask = _mask.getUMat(); + + if (cn > 1 && !haveMask) + { + src = src.reshape(1); + src2 = src2.reshape(1); + } + + if (haveSrc2) + { + if (!haveMask) + k.args(ocl::KernelArg::ReadOnlyNoSize(src), src.cols, (int)src.total(), + groupnum, ocl::KernelArg::PtrWriteOnly(db), ocl::KernelArg::ReadOnlyNoSize(src2)); + else + k.args(ocl::KernelArg::ReadOnlyNoSize(src), src.cols, (int)src.total(), + groupnum, ocl::KernelArg::PtrWriteOnly(db), ocl::KernelArg::ReadOnlyNoSize(mask), + ocl::KernelArg::ReadOnlyNoSize(src2)); + } + else + { + if (!haveMask) + k.args(ocl::KernelArg::ReadOnlyNoSize(src), src.cols, (int)src.total(), + groupnum, ocl::KernelArg::PtrWriteOnly(db)); + else + k.args(ocl::KernelArg::ReadOnlyNoSize(src), src.cols, (int)src.total(), + groupnum, ocl::KernelArg::PtrWriteOnly(db), ocl::KernelArg::ReadOnlyNoSize(mask)); + } + + size_t globalsize = groupnum * wgs; + if (!k.run(1, &globalsize, &wgs, true)) + return false; + + static const getMinMaxResFunc functab[7] = + { + getMinMaxRes, + getMinMaxRes, + getMinMaxRes, + getMinMaxRes, + getMinMaxRes, + getMinMaxRes, + getMinMaxRes + }; + + CV_Assert(ddepth <= CV_64F); + getMinMaxResFunc func = functab[ddepth]; + + int locTemp[2]; + func(db.getMat(ACCESS_READ), minVal, maxVal, + needMinLoc ? minLoc ? minLoc : locTemp : minLoc, + needMaxLoc ? maxLoc ? maxLoc : locTemp : maxLoc, + groupnum, src.cols, maxVal2); + + return true; +} + +#endif + +} + +void cv::minMaxIdx(InputArray _src, double* minVal, + double* maxVal, int* minIdx, int* maxIdx, + InputArray _mask) +{ + CV_INSTRUMENT_REGION(); + + int type = _src.type(), depth = CV_MAT_DEPTH(type), cn = CV_MAT_CN(type); + CV_Assert( (cn == 1 && (_mask.empty() || _mask.type() == CV_8U)) || + (cn > 1 && _mask.empty() && !minIdx && !maxIdx) ); + + CV_OCL_RUN(OCL_PERFORMANCE_CHECK(_src.isUMat()) && _src.dims() <= 2 && (_mask.empty() || _src.size() == _mask.size()), + ocl_minMaxIdx(_src, minVal, maxVal, minIdx, maxIdx, _mask)) + + Mat src = _src.getMat(), mask = _mask.getMat(); + + if (src.dims <= 2) + { + if ((size_t)src.step == (size_t)mask.step || mask.empty()) + { + CALL_HAL(minMaxIdx, cv_hal_minMaxIdx, src.data, src.step, src.cols*cn, src.rows, + src.depth(), minVal, maxVal, minIdx, maxIdx, mask.data); + } + + CALL_HAL(minMaxIdxMaskStep, cv_hal_minMaxIdxMaskStep, src.data, src.step, src.cols*cn, src.rows, + src.depth(), minVal, maxVal, minIdx, maxIdx, mask.data, mask.step); + } + else if (src.isContinuous() && (mask.isContinuous() || mask.empty())) + { + int res = cv_hal_minMaxIdx(src.data, 0, (int)src.total()*cn, 1, src.depth(), + minVal, maxVal, minIdx, maxIdx, mask.data); + + if (res == CV_HAL_ERROR_OK) + { + // minIdx[0] and minIdx[0] are always 0 for "flatten" version + if (minIdx) + ofs2idx(src, minIdx[1]+1, minIdx); + if (maxIdx) + ofs2idx(src, maxIdx[1]+1, maxIdx); + return; + } + else if (res != CV_HAL_ERROR_NOT_IMPLEMENTED) + { + CV_Error_(cv::Error::StsInternal, + ("HAL implementation minMaxIdx ==> " CVAUX_STR(cv_hal_minMaxIdx) " returned %d (0x%08x)", res, res)); + } + } + + MinMaxIdxFunc func = getMinmaxTab(depth); + CV_Assert( func != 0 ); + + const Mat* arrays[] = {&src, &mask, 0}; + uchar* ptrs[2] = {}; + NAryMatIterator it(arrays, ptrs); + + size_t minidx = 0, maxidx = 0; + int iminval = INT_MAX, imaxval = INT_MIN; + float fminval = std::numeric_limits::infinity(), fmaxval = -fminval; + double dminval = std::numeric_limits::infinity(), dmaxval = -dminval; + size_t startidx = 1; + void *minval = &iminval, *maxval = &imaxval; + int planeSize = (int)it.size*cn; + + if( depth == CV_32F ) + minval = &fminval, maxval = &fmaxval; + else if( depth == CV_64F ) + minval = &dminval, maxval = &dmaxval; + + for( size_t i = 0; i < it.nplanes; i++, ++it, startidx += planeSize ) + func( ptrs[0], ptrs[1], minval, maxval, &minidx, &maxidx, planeSize, startidx ); + + if (!src.empty() && mask.empty()) + { + if( minidx == 0 ) + minidx = 1; + if( maxidx == 0 ) + maxidx = 1; + } + + if( minidx == 0 ) + dminval = dmaxval = 0; + else if( depth == CV_32F ) + dminval = fminval, dmaxval = fmaxval; + else if( depth <= CV_32S ) + dminval = iminval, dmaxval = imaxval; + + if( minVal ) + *minVal = dminval; + if( maxVal ) + *maxVal = dmaxval; + + if( minIdx ) + ofs2idx(src, minidx, minIdx); + if( maxIdx ) + ofs2idx(src, maxidx, maxIdx); +} + +void cv::minMaxLoc( InputArray _img, double* minVal, double* maxVal, + Point* minLoc, Point* maxLoc, InputArray mask ) +{ + CV_INSTRUMENT_REGION(); + + int dims = _img.dims(); + CV_CheckLE(dims, 2, ""); + + minMaxIdx(_img, minVal, maxVal, (int*)minLoc, (int*)maxLoc, mask); + if( minLoc ) + { + if (dims == 2) + std::swap(minLoc->x, minLoc->y); + else + minLoc->y = 0; + } + if( maxLoc ) + { + if (dims == 2) + std::swap(maxLoc->x, maxLoc->y); + else + maxLoc->y = 0; + } +} + +enum class ReduceMode +{ + FIRST_MIN = 0, //!< get index of first min occurrence + LAST_MIN = 1, //!< get index of last min occurrence + FIRST_MAX = 2, //!< get index of first max occurrence + LAST_MAX = 3, //!< get index of last max occurrence +}; + +template +struct reduceMinMaxImpl +{ + void operator()(const cv::Mat& src, cv::Mat& dst, ReduceMode mode, const int axis) const + { + switch(mode) + { + case ReduceMode::FIRST_MIN: + reduceMinMaxApply(src, dst, axis); + break; + case ReduceMode::LAST_MIN: + reduceMinMaxApply(src, dst, axis); + break; + case ReduceMode::FIRST_MAX: + reduceMinMaxApply(src, dst, axis); + break; + case ReduceMode::LAST_MAX: + reduceMinMaxApply(src, dst, axis); + break; + } + } + + template class Cmp> + static void reduceMinMaxApply(const cv::Mat& src, cv::Mat& dst, const int axis) + { + Cmp cmp; + + const auto *src_ptr = src.ptr(); + auto *dst_ptr = dst.ptr(); + + const size_t outer_size = src.total(0, axis); + const auto mid_size = static_cast(src.size[axis]); + + const size_t outer_step = src.total(axis); + const size_t dst_step = dst.total(axis); + + const size_t mid_step = src.total(axis + 1); + + for (size_t outer = 0; outer < outer_size; ++outer) + { + const size_t outer_offset = outer * outer_step; + const size_t dst_offset = outer * dst_step; + for (size_t mid = 0; mid != mid_size; ++mid) + { + const size_t src_offset = outer_offset + mid * mid_step; + for (size_t inner = 0; inner < mid_step; inner++) + { + int32_t& index = dst_ptr[dst_offset + inner]; + + const size_t prev = outer_offset + index * mid_step + inner; + const size_t curr = src_offset + inner; + + if (cmp(src_ptr[curr], src_ptr[prev])) + { + index = static_cast(mid); + } + } + } + } + } +}; + +static void reduceMinMax(cv::InputArray src, cv::OutputArray dst, ReduceMode mode, int axis) +{ + CV_INSTRUMENT_REGION(); + + cv::Mat srcMat = src.getMat(); + axis = (axis + srcMat.dims) % srcMat.dims; + CV_Assert(srcMat.channels() == 1 && axis >= 0 && axis < srcMat.dims); + + cv::AutoBuffer sizes(srcMat.dims); + std::copy(srcMat.size.p, srcMat.size.p + srcMat.dims, sizes.begin()); + sizes[axis] = 1; + + dst.create(srcMat.dims, sizes.data(), CV_32SC1); // indices + cv::Mat dstMat = dst.getMat(); + dstMat.setTo(cv::Scalar::all(0)); + + if (!srcMat.isContinuous()) + { + srcMat = srcMat.clone(); + } + + bool needs_copy = !dstMat.isContinuous(); + if (needs_copy) + { + dstMat = dstMat.clone(); + } + + cv::detail::depthDispatch(srcMat.depth(), srcMat, dstMat, mode, axis); + + if (needs_copy) + { + dstMat.copyTo(dst); + } +} + +void cv::reduceArgMin(InputArray src, OutputArray dst, int axis, bool lastIndex) +{ + reduceMinMax(src, dst, lastIndex ? ReduceMode::LAST_MIN : ReduceMode::FIRST_MIN, axis); +} + +void cv::reduceArgMax(InputArray src, OutputArray dst, int axis, bool lastIndex) +{ + reduceMinMax(src, dst, lastIndex ? ReduceMode::LAST_MAX : ReduceMode::FIRST_MAX, axis); +} diff --git a/modules/core/src/minmax.simd.hpp b/modules/core/src/minmax.simd.hpp new file mode 100644 index 0000000000..f7c82e3753 --- /dev/null +++ b/modules/core/src/minmax.simd.hpp @@ -0,0 +1,471 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html + +// Copyright (C) 2026, Advanced Micro Devices, Inc., all rights reserved. + +#include "precomp.hpp" +#include "stat.hpp" + +#include + +namespace cv { + +typedef void (*MinMaxIdxFunc)(const void*, const uchar*, void*, void*, size_t*, size_t*, int, size_t); + +CV_CPU_OPTIMIZATION_NAMESPACE_BEGIN + +MinMaxIdxFunc getMinmaxTab(int depth); + +#ifndef CV_CPU_OPTIMIZATION_DECLARATIONS_ONLY + +template static void +minMaxIdx_( const T* src, const uchar* mask, WT* _minVal, WT* _maxVal, + size_t* _minIdx, size_t* _maxIdx, int len, size_t startIdx ) +{ + WT minVal = *_minVal, maxVal = *_maxVal; + size_t minIdx = *_minIdx, maxIdx = *_maxIdx; + + if( !mask ) + { + for( int i = 0; i < len; i++ ) + { + T val = src[i]; + if( val < minVal ) + { + minVal = val; + minIdx = startIdx + i; + } + if( val > maxVal ) + { + maxVal = val; + maxIdx = startIdx + i; + } + } + } + else + { + for( int i = 0; i < len; i++ ) + { + T val = src[i]; + if( mask[i] && val < minVal ) + { + minVal = val; + minIdx = startIdx + i; + } + if( mask[i] && val > maxVal ) + { + maxVal = val; + maxIdx = startIdx + i; + } + } + } + + *_minIdx = minIdx; + *_maxIdx = maxIdx; + *_minVal = minVal; + *_maxVal = maxVal; +} + +#if (CV_SIMD || CV_SIMD_SCALABLE) +template CV_ALWAYS_INLINE void +minMaxIdx_init( const T* src, const uchar* mask, WT* minval, WT* maxval, + size_t* minidx, size_t* maxidx, WT &minVal, WT &maxVal, + size_t &minIdx, size_t &maxIdx, const WT minInit, const WT maxInit, + const int nlanes, int len, size_t startidx, int &j, int &len0 ) +{ + len0 = len & -nlanes; + j = 0; + + minVal = *minval, maxVal = *maxval; + minIdx = *minidx, maxIdx = *maxidx; + + // To handle start values out of range + if ( minVal < minInit || maxVal < minInit || minVal > maxInit || maxVal > maxInit ) + { + uchar done = 0x00; + + for ( ; (j < len) && (done != 0x03); j++ ) + { + if ( !mask || mask[j] ) { + T val = src[j]; + if ( val < minVal ) + { + minVal = val; + minIdx = startidx + j; + done |= 0x01; + } + if ( val > maxVal ) + { + maxVal = val; + maxIdx = startidx + j; + done |= 0x02; + } + } + } + + len0 = j + ((len - j) & -nlanes); + } +} + +template CV_ALWAYS_INLINE void +minMaxIdx_finish( const T* src, const uchar* mask, WT* minval, WT* maxval, + size_t* minidx, size_t* maxidx, WT minVal, WT maxVal, + size_t minIdx, size_t maxIdx, int len, size_t startidx, + int j ) +{ + for ( ; j < len ; j++ ) + { + if ( !mask || mask[j] ) + { + T val = src[j]; + if ( val < minVal ) + { + minVal = val; + minIdx = startidx + j; + } + if ( val > maxVal ) + { + maxVal = val; + maxIdx = startidx + j; + } + } + } + + *minidx = minIdx; + *maxidx = maxIdx; + *minval = minVal; + *maxval = maxVal; +} + +//============================================================================ +// Templated SIMD core (universal intrinsics) shared by every depth. The +// per-depth wrappers below only select the value/index vector + result types. +//============================================================================ + +// Fast single-instruction broadcast into any universal vector (value or index). +template static inline V mm_set(typename VTraits::lane_type v); +template<> inline v_uint8 mm_set(uchar v) { return vx_setall_u8(v); } +template<> inline v_int8 mm_set(schar v) { return vx_setall_s8(v); } +template<> inline v_uint16 mm_set(ushort v) { return vx_setall_u16(v); } +template<> inline v_int16 mm_set(short v) { return vx_setall_s16(v); } +template<> inline v_uint32 mm_set(uint v) { return vx_setall_u32(v); } +template<> inline v_int32 mm_set(int v) { return vx_setall_s32(v); } +template<> inline v_float32 mm_set(float v) { return vx_setall_f32(v); } +#if (CV_SIMD_64F || CV_SIMD_SCALABLE_64F) +template<> inline v_uint64 mm_set(uint64_t v) { return vx_setall_u64(v); } +template<> inline v_float64 mm_set(double v) { return vx_setall_f64(v); } +#endif + +// Reinterpret a value-domain comparison mask to the matching unsigned index vector. +static inline v_uint8 mm_idxmask(const v_uint8& m) { return m; } +static inline v_uint8 mm_idxmask(const v_int8& m) { return v_reinterpret_as_u8(m); } +static inline v_uint16 mm_idxmask(const v_uint16& m) { return m; } +static inline v_uint16 mm_idxmask(const v_int16& m) { return v_reinterpret_as_u16(m); } +static inline v_uint32 mm_idxmask(const v_int32& m) { return v_reinterpret_as_u32(m); } +static inline v_uint32 mm_idxmask(const v_float32& m) { return v_reinterpret_as_u32(m); } +#if (CV_SIMD_64F || CV_SIMD_SCALABLE_64F) +static inline v_uint64 mm_idxmask(const v_float64& m) { return v_reinterpret_as_u64(m); } +#endif + +// Blend for index vectors (universal API has no v_select for u64). +template static inline IT mm_sel(const IT& mask, const IT& a, const IT& b) +{ return v_xor(b, v_and(v_xor(a, b), mask)); } + +// Value reduce-min / reduce-max (f64 has no universal v_reduce_*). +static inline int mm_vmin(const v_uint8& v){ return v_reduce_min(v); } +static inline int mm_vmin(const v_int8& v){ return v_reduce_min(v); } +static inline int mm_vmin(const v_uint16& v){ return v_reduce_min(v); } +static inline int mm_vmin(const v_int16& v){ return v_reduce_min(v); } +static inline int mm_vmin(const v_int32& v){ return v_reduce_min(v); } +static inline float mm_vmin(const v_float32&v){ return v_reduce_min(v); } +static inline int mm_vmax(const v_uint8& v){ return v_reduce_max(v); } +static inline int mm_vmax(const v_int8& v){ return v_reduce_max(v); } +static inline int mm_vmax(const v_uint16& v){ return v_reduce_max(v); } +static inline int mm_vmax(const v_int16& v){ return v_reduce_max(v); } +static inline int mm_vmax(const v_int32& v){ return v_reduce_max(v); } +static inline float mm_vmax(const v_float32&v){ return v_reduce_max(v); } +#if (CV_SIMD_64F || CV_SIMD_SCALABLE_64F) +static inline double mm_vmin(const v_float64& v) +{ double b[VTraits::max_nlanes]; v_store(b, v); double r=b[0]; const int n=VTraits::vlanes(); for(int i=1;i::max_nlanes]; v_store(b, v); double r=b[0]; const int n=VTraits::vlanes(); for(int i=1;ir) r=b[i]; return r; } +#endif + +// Index reduce-min (u64 has no universal v_reduce_min). +static inline unsigned mm_imin(const v_uint8& v){ return v_reduce_min(v); } +static inline unsigned mm_imin(const v_uint16& v){ return v_reduce_min(v); } +static inline unsigned mm_imin(const v_uint32& v){ return v_reduce_min(v); } +#if (CV_SIMD_64F || CV_SIMD_SCALABLE_64F) +static inline uint64_t mm_imin(const v_uint64& v) +{ uint64_t b[VTraits::max_nlanes]; v_store(b, v); uint64_t r=b[0]; const int n=VTraits::vlanes(); for(int i=1;i static inline VT mm_active(const uchar* mask, int k, int nlanes); +template<> inline v_uint8 mm_active(const uchar* mask, int k, int) +{ return v_ne(vx_load(mask + k), vx_setzero_u8()); } +template<> inline v_int8 mm_active(const uchar* mask, int k, int) +{ return v_reinterpret_as_s8(v_ne(vx_load(mask + k), vx_setzero_u8())); } +template<> inline v_uint16 mm_active(const uchar* mask, int k, int) +{ return v_ne(vx_load_expand(mask + k), vx_setzero_u16()); } +template<> inline v_int16 mm_active(const uchar* mask, int k, int) +{ return v_reinterpret_as_s16(v_ne(vx_load_expand(mask + k), vx_setzero_u16())); } +template<> inline v_int32 mm_active(const uchar* mask, int k, int nlanes) +{ uint32_t b[VTraits::max_nlanes]; for(int t=0;t inline v_float32 mm_active(const uchar* mask, int k, int nlanes) +{ uint32_t b[VTraits::max_nlanes]; for(int t=0;t inline v_float64 mm_active(const uchar* mask, int k, int nlanes) +{ uint64_t b[VTraits::max_nlanes]; for(int t=0;t +static inline void mm_fold_min(const VT& valMin, const IT& idxMin, const IT& none, + size_t delta, WT& minVal, size_t& minIdx) +{ + if ( v_check_any(v_ne(idxMin, none)) ) + { + WT cv = (WT)mm_vmin(valMin); + IT sel = mm_sel(mm_idxmask(v_eq(mm_set((typename VTraits::lane_type)cv), valMin)), idxMin, none); + size_t ci = (size_t)mm_imin(sel) + delta; + if ( cv < minVal || (cv == minVal && ci < minIdx) ) { minVal = cv; minIdx = ci; } + } +} +template +static inline void mm_fold_max(const VT& valMax, const IT& idxMax, const IT& none, + size_t delta, WT& maxVal, size_t& maxIdx) +{ + if ( v_check_any(v_ne(idxMax, none)) ) + { + WT cv = (WT)mm_vmax(valMax); + IT sel = mm_sel(mm_idxmask(v_eq(mm_set((typename VTraits::lane_type)cv), valMax)), idxMax, none); + size_t ci = (size_t)mm_imin(sel) + delta; + if ( cv > maxVal || (cv == maxVal && ci < maxIdx) ) { maxVal = cv; maxIdx = ci; } + } +} + +// IST = unsigned index lane type (uchar / ushort / uint / uint64_t). +template +static void minMaxIdx_simd_(const T* src, const uchar* mask, WT* minval, WT* maxval, + size_t* minidx, size_t* maxidx, int len, size_t startidx, + WT minInit, WT maxInit) +{ + const int nlanes = VTraits::vlanes(); + if ( len >= nlanes ) + { + int j, len0; + WT minVal, maxVal; + size_t minIdx, maxIdx; + + minMaxIdx_init( src, mask, minval, maxval, minidx, maxidx, minVal, maxVal, minIdx, maxIdx, + minInit, maxInit, nlanes, len, startidx, j, len0 ); + + if ( j <= len0 - nlanes ) + { + IST idxbuf[VTraits::max_nlanes]; + for ( int t = 0; t < nlanes; t++ ) idxbuf[t] = (IST)t; + const IT idxStart = vx_load(idxbuf); + const IT inc = mm_set((IST)nlanes); + const IT none = mm_set((IST)~(IST)0); + // Reduce before the per-lane index could reach the 'none' sentinel. + // For >= 32-bit indices (len <= INT_MAX) one block covers everything. + const int64_t idxcap = (sizeof(IST) <= 2) ? (((int64_t)1 << (8 * (int)sizeof(IST))) - 1) : (int64_t)INT_MAX; + const int blockStep = (int)((idxcap / nlanes) * nlanes); + + const IT inc2 = mm_set((IST)(nlanes * 2)); + + do + { + // Two independent accumulator streams break the serial + // min/max + index-select dependency chain so the CPU can + // overlap iterations (latency-bound loop). Stream 0 covers the + // even vector slots, stream 1 the odd ones; results are merged + // by mm_fold_*. (2 streams is the sweet spot: 4 spills the 16 + // YMM regs on AVX2 and gives no gain on AVX-512.) The masked + // path keeps a single stream (stream 0). + VT vMin0 = mm_set((T)minVal), vMin1 = vMin0; + VT vMax0 = mm_set((T)maxVal), vMax1 = vMax0; + IT idx0 = idxStart, idx1 = v_add(idxStart, inc); + IT iMin0 = none, iMin1 = none, iMax0 = none, iMax1 = none; + + int k = j; + size_t delta = startidx + j; + // 64-bit math: blockStep can be ~INT_MAX (32/64-bit indices), so + // j + blockStep would overflow int once j advances past the start. + const int limit = (int)std::min((int64_t)len0, (int64_t)j + (int64_t)blockStep); + + if ( !mask ) + { + // Dual stream only for >= 16-bit indices: 8-bit forces tiny + // reduce-blocks (u8 index < 256), where extra per-block folds + // outweigh the dependency-break benefit. + for ( ; (sizeof(IST) > 1) && k <= limit - 2 * nlanes; k += 2 * nlanes ) + { + VT d0 = vx_load(src + k), d1 = vx_load(src + k + nlanes); + iMin0 = mm_sel(mm_idxmask(v_lt(d0, vMin0)), idx0, iMin0); + iMin1 = mm_sel(mm_idxmask(v_lt(d1, vMin1)), idx1, iMin1); + iMax0 = mm_sel(mm_idxmask(v_gt(d0, vMax0)), idx0, iMax0); + iMax1 = mm_sel(mm_idxmask(v_gt(d1, vMax1)), idx1, iMax1); + vMin0 = v_min(d0, vMin0); vMin1 = v_min(d1, vMin1); + vMax0 = v_max(d0, vMax0); vMax1 = v_max(d1, vMax1); + idx0 = v_add(idx0, inc2); idx1 = v_add(idx1, inc2); + } + for ( ; k < limit; k += nlanes ) // odd trailing vector + { + VT d0 = vx_load(src + k); + iMin0 = mm_sel(mm_idxmask(v_lt(d0, vMin0)), idx0, iMin0); + iMax0 = mm_sel(mm_idxmask(v_gt(d0, vMax0)), idx0, iMax0); + vMin0 = v_min(d0, vMin0); vMax0 = v_max(d0, vMax0); + idx0 = v_add(idx0, inc); + } + } + else + { + for ( ; k < limit; k += nlanes ) + { + VT data = vx_load(src + k); + VT active = mm_active(mask, k, nlanes); + VT cmpMin = v_and(v_lt(data, vMin0), active); + VT cmpMax = v_and(v_gt(data, vMax0), active); + iMin0 = mm_sel(mm_idxmask(cmpMin), idx0, iMin0); + iMax0 = mm_sel(mm_idxmask(cmpMax), idx0, iMax0); + vMin0 = v_select(cmpMin, data, vMin0); + vMax0 = v_select(cmpMax, data, vMax0); + idx0 = v_add(idx0, inc); + } + } + + j = k; + + mm_fold_min(vMin0, iMin0, none, delta, minVal, minIdx); + mm_fold_min(vMin1, iMin1, none, delta, minVal, minIdx); + mm_fold_max(vMax0, iMax0, none, delta, maxVal, maxIdx); + mm_fold_max(vMax1, iMax1, none, delta, maxVal, maxIdx); + } + while ( j < len0 ); + } + + minMaxIdx_finish( src, mask, minval, maxval, minidx, maxidx, minVal, maxVal, + minIdx, maxIdx, len, startidx, j ); + vx_cleanup(); + } + else + { + minMaxIdx_(src, mask, minval, maxval, minidx, maxidx, len, startidx); + } +} +#endif + +static void minMaxIdx_8u(const void* src_, const uchar* mask, void* minval_, void* maxval_, + size_t* minidx, size_t* maxidx, int len, size_t startidx ) +{ +#if (CV_SIMD || CV_SIMD_SCALABLE) + minMaxIdx_simd_( + static_cast(src_), mask, static_cast(minval_), static_cast(maxval_), + minidx, maxidx, len, startidx, (int)0, (int)UCHAR_MAX); +#else + minMaxIdx_(static_cast(src_), mask, static_cast(minval_), + static_cast(maxval_), minidx, maxidx, len, startidx); +#endif +} + +static void minMaxIdx_8s(const void* src_, const uchar* mask, void* minval_, void* maxval_, + size_t* minidx, size_t* maxidx, int len, size_t startidx ) +{ +#if (CV_SIMD || CV_SIMD_SCALABLE) + minMaxIdx_simd_( + static_cast(src_), mask, static_cast(minval_), static_cast(maxval_), + minidx, maxidx, len, startidx, (int)SCHAR_MIN, (int)SCHAR_MAX); +#else + minMaxIdx_(static_cast(src_), mask, static_cast(minval_), + static_cast(maxval_), minidx, maxidx, len, startidx); +#endif +} + +static void minMaxIdx_16u(const void* src_, const uchar* mask, void* minval_, void* maxval_, + size_t* minidx, size_t* maxidx, int len, size_t startidx ) +{ +#if (CV_SIMD || CV_SIMD_SCALABLE) + minMaxIdx_simd_( + static_cast(src_), mask, static_cast(minval_), static_cast(maxval_), + minidx, maxidx, len, startidx, (int)0, (int)USHRT_MAX); +#else + minMaxIdx_(static_cast(src_), mask, static_cast(minval_), + static_cast(maxval_), minidx, maxidx, len, startidx); +#endif +} + +static void minMaxIdx_16s(const void* src_, const uchar* mask, void* minval_, void* maxval_, + size_t* minidx, size_t* maxidx, int len, size_t startidx ) +{ +#if (CV_SIMD || CV_SIMD_SCALABLE) + minMaxIdx_simd_( + static_cast(src_), mask, static_cast(minval_), static_cast(maxval_), + minidx, maxidx, len, startidx, (int)SHRT_MIN, (int)SHRT_MAX); +#else + minMaxIdx_(static_cast(src_), mask, static_cast(minval_), + static_cast(maxval_), minidx, maxidx, len, startidx); +#endif +} + +static void minMaxIdx_32s(const void* src_, const uchar* mask, void* minval_, void* maxval_, + size_t* minidx, size_t* maxidx, int len, size_t startidx ) +{ +#if (CV_SIMD || CV_SIMD_SCALABLE) + minMaxIdx_simd_( + static_cast(src_), mask, static_cast(minval_), static_cast(maxval_), + minidx, maxidx, len, startidx, INT_MIN, INT_MAX); +#else + minMaxIdx_(static_cast(src_), mask, static_cast(minval_), + static_cast(maxval_), minidx, maxidx, len, startidx); +#endif +} + +static void minMaxIdx_32f(const void* src_, const uchar* mask, void* minval_, void* maxval_, + size_t* minidx, size_t* maxidx, int len, size_t startidx ) +{ +#if (CV_SIMD || CV_SIMD_SCALABLE) + minMaxIdx_simd_( + static_cast(src_), mask, static_cast(minval_), static_cast(maxval_), + minidx, maxidx, len, startidx, FLT_MIN, FLT_MAX); +#else + minMaxIdx_(static_cast(src_), mask, static_cast(minval_), + static_cast(maxval_), minidx, maxidx, len, startidx); +#endif +} + +static void minMaxIdx_64f(const void* src_, const uchar* mask, void* minval_, void* maxval_, + size_t* minidx, size_t* maxidx, int len, size_t startidx ) +{ +#if (CV_SIMD_64F || CV_SIMD_SCALABLE_64F) + minMaxIdx_simd_( + static_cast(src_), mask, static_cast(minval_), static_cast(maxval_), + minidx, maxidx, len, startidx, DBL_MIN, DBL_MAX); +#else + minMaxIdx_(static_cast(src_), mask, static_cast(minval_), + static_cast(maxval_), minidx, maxidx, len, startidx); +#endif +} + +MinMaxIdxFunc getMinmaxTab(int depth) +{ + static MinMaxIdxFunc minmaxTab[CV_DEPTH_MAX] = + { + GET_OPTIMIZED(minMaxIdx_8u), GET_OPTIMIZED(minMaxIdx_8s), + GET_OPTIMIZED(minMaxIdx_16u), GET_OPTIMIZED(minMaxIdx_16s), + GET_OPTIMIZED(minMaxIdx_32s), + GET_OPTIMIZED(minMaxIdx_32f), GET_OPTIMIZED(minMaxIdx_64f), + 0 + }; + + return minmaxTab[depth]; +} + +#endif // CV_CPU_OPTIMIZATION_DECLARATIONS_ONLY + +CV_CPU_OPTIMIZATION_NAMESPACE_END +} // namespace cv