diff --git a/modules/core/src/matrix_operations.cpp b/modules/core/src/matrix_operations.cpp index 6ca7168bf5..f1ec202f1e 100644 --- a/modules/core/src/matrix_operations.cpp +++ b/modules/core/src/matrix_operations.cpp @@ -6,11 +6,8 @@ #include "opencv2/core/mat.hpp" #include "opencl_kernels_core.hpp" -#undef HAVE_IPP -#undef CV_IPP_RUN_FAST -#define CV_IPP_RUN_FAST(f, ...) -#undef CV_IPP_RUN -#define CV_IPP_RUN(c, f, ...) +#define IPP_DISABLE_REDUCE 1 +#define IPP_DISABLE_SORT 0 /*************************************************************************************************\ Matrix Operations @@ -512,7 +509,7 @@ typedef void (*ReduceFunc)( const Mat& src, Mat& dst ); #define reduceMinR32f reduceR_ > #define reduceMinR64f reduceR_ > -#ifdef HAVE_IPP +#if defined(HAVE_IPP) && !IPP_DISABLE_REDUCE static inline bool ipp_reduceSumC_8u16u16s32f_64f(const cv::Mat& srcmat, cv::Mat& dstmat) { int sstep = (int)srcmat.step, stype = srcmat.type(), @@ -601,7 +598,7 @@ static inline void reduceSumC_8u16u16s32f_64f(const cv::Mat& srcmat, cv::Mat& ds #define reduceSum2C32f32f reduceC_, OpSqr > #define reduceSum2C64f64f reduceC_,OpSqr > -#ifdef HAVE_IPP +#if defined(HAVE_IPP) && !IPP_DISABLE_REDUCE #define reduceSumC8u64f reduceSumC_8u16u16s32f_64f #define reduceSumC16u64f reduceSumC_8u16u16s32f_64f #define reduceSumC16s64f reduceSumC_8u16u16s32f_64f @@ -611,14 +608,14 @@ static inline void reduceSumC_8u16u16s32f_64f(const cv::Mat& srcmat, cv::Mat& ds #define reduceSumC16u64f reduceC_ > #define reduceSumC16s64f reduceC_ > #define reduceSumC32f64f reduceC_ > +#endif #define reduceSum2C8u64f reduceC_, OpSqr > #define reduceSum2C16u64f reduceC_,OpSqr > #define reduceSum2C16s64f reduceC_,OpSqr > #define reduceSum2C32f64f reduceC_,OpSqr > -#endif -#ifdef HAVE_IPP +#if defined(HAVE_IPP) && !IPP_DISABLE_REDUCE #define REDUCE_OP(favor, optype, type1, type2) \ static inline bool ipp_reduce##optype##C##favor(const cv::Mat& srcmat, cv::Mat& dstmat) \ { \ @@ -643,7 +640,7 @@ static inline void reduce##optype##C##favor(const cv::Mat& srcmat, cv::Mat& dstm } #endif -#ifdef HAVE_IPP +#if defined(HAVE_IPP) && !IPP_DISABLE_REDUCE REDUCE_OP(8u, Max, uchar, uchar) REDUCE_OP(16u, Max, ushort, ushort) REDUCE_OP(16s, Max, short, short) @@ -656,7 +653,7 @@ REDUCE_OP(32f, Max, float, float) #endif #define reduceMaxC64f reduceC_ > -#ifdef HAVE_IPP +#if defined(HAVE_IPP) && !IPP_DISABLE_REDUCE REDUCE_OP(8u, Min, uchar, uchar) REDUCE_OP(16u, Min, ushort, ushort) REDUCE_OP(16s, Min, short, short) @@ -1042,7 +1039,7 @@ template static void sort_( const Mat& src, Mat& dst, int flags ) } -#ifdef HAVE_IPP +#if defined(HAVE_IPP) && !IPP_DISABLE_SORT typedef IppStatus (CV_STDCALL *IppSortFunc)(void *pSrcDst, int len, Ipp8u *pBuffer); static IppSortFunc getSortFunc(int depth, bool sortDescending) @@ -1079,54 +1076,64 @@ static bool ipp_sort(const Mat& src, Mat& dst, int flags) if(!ippsSortRadix_I) return false; - if(sortRows) - { - AutoBuffer buffer; - int bufferSize; - if(ippsSortRadixGetBufferSize(src.cols, type, &bufferSize) < 0) - return false; - - buffer.allocate(bufferSize); - - if(!inplace) - src.copyTo(dst); - - for(int i = 0; i < dst.rows; i++) - { - if(CV_INSTRUMENT_FUN_IPP(ippsSortRadix_I, (void*)dst.ptr(i), dst.cols, buffer.data()) < 0) - return false; - } - } + int n, len; + if( sortRows ) + n = src.rows, len = src.cols; else + n = src.cols, len = src.rows; + + int bufferSize; + if(ippsSortRadixGetBufferSize(len, type, &bufferSize) < 0) + return false; + + // Each row/column sorts independently, so parallelize over n like sort_ does. + // IPP scratch (buffer) and per-column temporaries are thread-local; a shared flag + // records IPP failures since the lambda cannot return from ipp_sort. + volatile bool ok = true; + parallel_for_(Range(0, n), [&](const Range& range) { - AutoBuffer buffer; - int bufferSize; - if(ippsSortRadixGetBufferSize(src.rows, type, &bufferSize) < 0) - return false; - - buffer.allocate(bufferSize); - - Mat row(1, src.rows, src.type()); - Mat srcSub; - Mat dstSub; + AutoBuffer buffer(bufferSize); + Mat row; Rect subRect(0,0,1,src.rows); + if( !sortRows ) + row.create(1, src.rows, src.type()); - for(int i = 0; i < src.cols; i++) + for( int i = range.start; i < range.end; i++ ) { - subRect.x = i; - srcSub = Mat(src, subRect); - dstSub = Mat(dst, subRect); - srcSub.copyTo(row); + if( !ok ) + break; + void* ptr; + if( sortRows ) + { + uchar* dptr = dst.ptr(i); + if( !inplace ) + memcpy(dptr, src.ptr(i), src.cols * src.elemSize()); + ptr = dptr; + } + else + { + subRect.x = i; + Mat(src, subRect).copyTo(row); + ptr = row.ptr(); + } - if(CV_INSTRUMENT_FUN_IPP(ippsSortRadix_I, (void*)row.ptr(), dst.rows, buffer.data()) < 0) - return false; + if( CV_INSTRUMENT_FUN_IPP(ippsSortRadix_I, ptr, len, buffer.data()) < 0 ) + { + ok = false; + break; + } - row = row.reshape(1, dstSub.rows); - row.copyTo(dstSub); + if( !sortRows ) + { + Mat dstSub(dst, subRect); + row = row.reshape(1, dstSub.rows); + row.copyTo(dstSub); + row = row.reshape(1, 1); + } } - } + }); - return true; + return ok; } #endif @@ -1190,7 +1197,7 @@ template static void sortIdx_( const Mat& src, Mat& dst, int flags ) } } -#ifdef HAVE_IPP +#if defined(HAVE_IPP) && !IPP_DISABLE_SORT typedef IppStatus (CV_STDCALL *IppSortIndexFunc)(const void* pSrc, Ipp32s srcStrideBytes, Ipp32s *pDstIndx, int len, Ipp8u *pBuffer); static IppSortIndexFunc getSortIndexFunc(int depth, bool sortDescending) @@ -1281,7 +1288,9 @@ void cv::sort( InputArray _src, OutputArray _dst, int flags ) CV_Assert( src.dims <= 2 && src.channels() == 1 ); _dst.create( src.size(), src.type() ); Mat dst = _dst.getMat(); +#if !IPP_DISABLE_SORT CV_IPP_RUN_FAST(ipp_sort(src, dst, flags)); +#endif static SortFunc tab[CV_DEPTH_MAX] = { @@ -1306,7 +1315,9 @@ void cv::sortIdx( InputArray _src, OutputArray _dst, int flags ) _dst.create( src.size(), CV_32S ); dst = _dst.getMat(); +#if !IPP_DISABLE_SORT CV_IPP_RUN_FAST(ipp_sortIdx(src, dst, flags)); +#endif static SortFunc tab[CV_DEPTH_MAX] = {