From 9636b5e82126705df4efb8f68992ba8dda86b75e Mon Sep 17 00:00:00 2001 From: Ishank gulati Date: Tue, 22 Dec 2015 15:47:11 +0530 Subject: [PATCH 001/128] reduce k_fold parameter --- modules/ml/src/svm.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/ml/src/svm.cpp b/modules/ml/src/svm.cpp index 757bb7a171..e6f2dc9c87 100644 --- a/modules/ml/src/svm.cpp +++ b/modules/ml/src/svm.cpp @@ -1442,7 +1442,7 @@ public: //check that while cross-validation there were the samples from all the classes if( class_ranges[class_count] <= 0 ) CV_Error( CV_StsBadArg, "While cross-validation one or more of the classes have " - "been fell out of the sample. Try to enlarge " ); + "been fell out of the sample. Try to reduce " ); if( svmType == NU_SVC ) { From b3bbe7704d160ca9678b57dc17c4a9074122fdfd Mon Sep 17 00:00:00 2001 From: Pavel Rojtberg Date: Wed, 23 Dec 2015 13:06:56 +0100 Subject: [PATCH 002/128] add jaccardDistance measure for rectangle overlap computes the complement of the Jaccard Index as described in https://en.wikipedia.org/wiki/Jaccard_index. For rectangles this reduces to computing the intersection over the union. --- modules/core/include/opencv2/core/types.hpp | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/modules/core/include/opencv2/core/types.hpp b/modules/core/include/opencv2/core/types.hpp index e166556af7..6909676dc3 100644 --- a/modules/core/include/opencv2/core/types.hpp +++ b/modules/core/include/opencv2/core/types.hpp @@ -51,6 +51,7 @@ #include #include #include +#include #include "opencv2/core/cvdef.h" #include "opencv2/core/cvstd.hpp" @@ -1832,7 +1833,26 @@ Rect_<_Tp> operator | (const Rect_<_Tp>& a, const Rect_<_Tp>& b) return c |= b; } +/** + * @brief measure dissimilarity between two sample sets + * + * computes the complement of the Jaccard Index as described in . + * For rectangles this reduces to computing the intersection over the union. + */ +template static inline +double jaccardDistance(const Rect_<_Tp>& a, const Rect_<_Tp>& b) { + _Tp Aa = a.area(); + _Tp Ab = b.area(); + if ((Aa + Ab) <= std::numeric_limits<_Tp>::epsilon()) { + // jaccard_index = 1 -> distance = 0 + return 0.0; + } + + double Aab = (a & b).area(); + // distance = 1 - jaccard_index + return 1.0 - Aab / (Aa + Ab - Aab); +} ////////////////////////////// RotatedRect ////////////////////////////// From 25b044e6c3b0ac950497cc5838c684d797aefb01 Mon Sep 17 00:00:00 2001 From: Teng Cao Date: Thu, 24 Dec 2015 10:26:42 +0800 Subject: [PATCH 003/128] fix a bug in updateTrainingSet In case fillPassedSamples find zero passed negatives and leafFA is achieved, function updateTrainingSet will return false and print misleading information, this PR fix the case. --- apps/traincascade/cascadeclassifier.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/apps/traincascade/cascadeclassifier.cpp b/apps/traincascade/cascadeclassifier.cpp index 2924fe341f..176cefe7da 100644 --- a/apps/traincascade/cascadeclassifier.cpp +++ b/apps/traincascade/cascadeclassifier.cpp @@ -309,7 +309,8 @@ bool CvCascadeClassifier::updateTrainingSet( double minimumAcceptanceRatio, doub int proNumNeg = cvRound( ( ((double)numNeg) * ((double)posCount) ) / numPos ); // apply only a fraction of negative samples. double is required since overflow is possible int negCount = fillPassedSamples( posCount, proNumNeg, false, minimumAcceptanceRatio, negConsumed ); if ( !negCount ) - return false; + if ( !(negConsumed > 0 && ((double)negCount+1)/(double)negConsumed <= minimumAcceptanceRatio) ) + return false; curNumSamples = posCount + negCount; acceptanceRatio = negConsumed == 0 ? 0 : ( (double)negCount/(double)(int64)negConsumed ); From 551b5d3e1af5891959d797f947c850dbe9055ccd Mon Sep 17 00:00:00 2001 From: Kai Hugo Hustoft Endresen Date: Fri, 8 Jan 2016 00:32:52 +0100 Subject: [PATCH 004/128] StereoSGBM.cpp - use SSE2 for pass 2 using MODE_HH With a test image set of 2800x1400 bytes on a Intel Core i7 5960X this improves runtime of MODE_HH with about 10%. (this particular replaced code segment is approx 3 times faster than the non-SSE2 variant). I was able to reduce runtime by 130 ms by this simple fix. The second part of the SSE2 optimized part could probably be optimized further by using shift SSE2 operations, but I imagine this would improve performance 10-20 ms at best. --- modules/calib3d/src/stereosgbm.cpp | 50 +++++++++++++++++++++++++----- 1 file changed, 43 insertions(+), 7 deletions(-) diff --git a/modules/calib3d/src/stereosgbm.cpp b/modules/calib3d/src/stereosgbm.cpp index 1c085f9b3a..9b783e9d65 100644 --- a/modules/calib3d/src/stereosgbm.cpp +++ b/modules/calib3d/src/stereosgbm.cpp @@ -759,14 +759,50 @@ static void computeDisparitySGBM( const Mat& img1, const Mat& img2, } else { - for( d = 0; d < D; d++ ) + #if CV_SSE2 + if( useSIMD ) { - int Sval = Sp[d]; - if( Sval < minS ) - { - minS = Sval; - bestDisp = d; - } + __m128i _minS = _mm_set1_epi16(MAX_COST), _bestDisp = _mm_set1_epi16(-1); + __m128i _d8 = _mm_setr_epi16(0, 1, 2, 3, 4, 5, 6, 7), _8 = _mm_set1_epi16(8); + + for( d = 0; d < D; d+= 8 ) + { + __m128i L0 = _mm_load_si128((const __m128i*)( Sp + d )); + __m128i mask = _mm_cmplt_epi16( L0, _minS ); + _minS = _mm_min_epi16( L0, _minS ); + _bestDisp = _mm_xor_si128(_bestDisp, _mm_and_si128(_mm_xor_si128( _bestDisp, _d8), mask)); + _d8 = _mm_adds_epi16(_d8, _8 ); + } + short CV_DECL_ALIGNED(16) bestDispBuf[8]; + _mm_store_si128((__m128i*)bestDispBuf, _bestDisp); + short CV_DECL_ALIGNED(16) minSBuf[8]; + _mm_store_si128((__m128i*)minSBuf, _minS ); + + for( int i = 0; i < 8; i++ ) + { + int Sval = minSBuf[ i ]; + if( Sval <= minS ) + { + if( ( Sval < minS ) || ( bestDispBuf[i] < bestDisp ) ) + { + bestDisp = bestDispBuf[i]; + } + minS = Sval; + } + } + } + else + #endif + { + for( d = 0; d < D; d++ ) + { + int Sval = Sp[d]; + if( Sval < minS ) + { + minS = Sval; + bestDisp = d; + } + } } } From 1b5352688f3646b0721e5973ce727238c0ae3d3e Mon Sep 17 00:00:00 2001 From: thierry Date: Fri, 15 Jan 2016 17:35:10 +0100 Subject: [PATCH 005/128] better png transparency handling --- modules/imgcodecs/src/grfmt_png.cpp | 27 ++++++++++++++++----------- modules/imgcodecs/src/loadsave.cpp | 5 ----- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/modules/imgcodecs/src/grfmt_png.cpp b/modules/imgcodecs/src/grfmt_png.cpp index 2f4a62baee..e75f0bf45f 100644 --- a/modules/imgcodecs/src/grfmt_png.cpp +++ b/modules/imgcodecs/src/grfmt_png.cpp @@ -187,15 +187,12 @@ bool PngDecoder::readHeader() if( bit_depth <= 8 || bit_depth == 16 ) { + png_get_tRNS(png_ptr, info_ptr, &trans, &num_trans, &trans_values); switch(color_type) { case PNG_COLOR_TYPE_RGB: - m_type = CV_8UC3; - break; case PNG_COLOR_TYPE_PALETTE: - png_get_tRNS( png_ptr, info_ptr, &trans, &num_trans, &trans_values); - //Check if there is a transparency value in the palette - if ( num_trans > 0 ) + if( num_trans > 0 ) m_type = CV_8UC4; else m_type = CV_8UC3; @@ -203,8 +200,14 @@ bool PngDecoder::readHeader() case PNG_COLOR_TYPE_RGB_ALPHA: m_type = CV_8UC4; break; + case PNG_COLOR_TYPE_GRAY_ALPHA: + m_type = CV_8UC2; + break; default: - m_type = CV_8UC1; + if( num_trans > 0 ) + m_type = CV_8UC2; + else + m_type = CV_8UC1; } if( bit_depth == 16 ) m_type = CV_MAKETYPE(CV_16U, CV_MAT_CN(m_type)); @@ -227,7 +230,7 @@ bool PngDecoder::readData( Mat& img ) volatile bool result = false; AutoBuffer _buffer(m_height); uchar** buffer = _buffer; - int color = img.channels() > 1; + int color = img.channels() > 2; uchar* data = img.ptr(); int step = (int)img.step; @@ -246,7 +249,7 @@ bool PngDecoder::readData( Mat& img ) else if( !isBigEndian() ) png_set_swap( png_ptr ); - if(img.channels() < 4) + if(img.channels() != 4 && img.channels() != 2) { /* observation: png_read_image() writes 400 bytes beyond * end of data when reading a 400x118 color png @@ -257,12 +260,13 @@ bool PngDecoder::readData( Mat& img ) * stripping alpha.. 18.11.2004 Axel Walthelm */ png_set_strip_alpha( png_ptr ); - } + } else + png_set_tRNS_to_alpha( png_ptr ); if( m_color_type == PNG_COLOR_TYPE_PALETTE ) png_set_palette_to_rgb( png_ptr ); - if( m_color_type == PNG_COLOR_TYPE_GRAY && m_bit_depth < 8 ) + if( (m_color_type & PNG_COLOR_MASK_COLOR) != 0 && m_bit_depth < 8 ) #if (PNG_LIBPNG_VER_MAJOR*10000 + PNG_LIBPNG_VER_MINOR*100 + PNG_LIBPNG_VER_RELEASE >= 10209) || \ (PNG_LIBPNG_VER_MAJOR == 1 && PNG_LIBPNG_VER_MINOR == 0 && PNG_LIBPNG_VER_RELEASE >= 18) png_set_expand_gray_1_2_4_to_8( png_ptr ); @@ -270,7 +274,7 @@ bool PngDecoder::readData( Mat& img ) png_set_gray_1_2_4_to_8( png_ptr ); #endif - if( CV_MAT_CN(m_type) > 1 && color ) + if( CV_MAT_CN(m_type) > 2 && color ) png_set_bgr( png_ptr ); // convert RGB to BGR else if( color ) png_set_gray_to_rgb( png_ptr ); // Gray->RGB @@ -410,6 +414,7 @@ bool PngEncoder::write( const Mat& img, const std::vector& params ) png_set_IHDR( png_ptr, info_ptr, width, height, depth == CV_8U ? isBilevel?1:8 : 16, channels == 1 ? PNG_COLOR_TYPE_GRAY : + channels == 2 ? PNG_COLOR_TYPE_GRAY_ALPHA : channels == 3 ? PNG_COLOR_TYPE_RGB : PNG_COLOR_TYPE_RGBA, PNG_INTERLACE_NONE, PNG_COMPRESSION_TYPE_DEFAULT, PNG_FILTER_TYPE_DEFAULT ); diff --git a/modules/imgcodecs/src/loadsave.cpp b/modules/imgcodecs/src/loadsave.cpp index 70a31c37a2..e0080bd3b9 100644 --- a/modules/imgcodecs/src/loadsave.cpp +++ b/modules/imgcodecs/src/loadsave.cpp @@ -452,8 +452,6 @@ static bool imwrite_( const String& filename, const Mat& image, Mat temp; const Mat* pimage = ℑ - CV_Assert( image.channels() == 1 || image.channels() == 3 || image.channels() == 4 ); - ImageEncoder encoder = findEncoder( filename ); if( !encoder ) CV_Error( CV_StsError, "could not find a writer for the specified extension" ); @@ -590,9 +588,6 @@ bool imencode( const String& ext, InputArray _image, { Mat image = _image.getMat(); - int channels = image.channels(); - CV_Assert( channels == 1 || channels == 3 || channels == 4 ); - ImageEncoder encoder = findEncoder( ext ); if( !encoder ) CV_Error( CV_StsError, "could not find encoder for the specified extension" ); From bdb9cf4d476a3d11956dc058ee114f0dd0718406 Mon Sep 17 00:00:00 2001 From: thierry Date: Mon, 18 Jan 2016 17:56:26 +0100 Subject: [PATCH 006/128] fix inversed grayscale conditional --- modules/imgcodecs/src/grfmt_png.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/imgcodecs/src/grfmt_png.cpp b/modules/imgcodecs/src/grfmt_png.cpp index e75f0bf45f..5b33c66484 100644 --- a/modules/imgcodecs/src/grfmt_png.cpp +++ b/modules/imgcodecs/src/grfmt_png.cpp @@ -266,7 +266,7 @@ bool PngDecoder::readData( Mat& img ) if( m_color_type == PNG_COLOR_TYPE_PALETTE ) png_set_palette_to_rgb( png_ptr ); - if( (m_color_type & PNG_COLOR_MASK_COLOR) != 0 && m_bit_depth < 8 ) + if( (m_color_type & PNG_COLOR_MASK_COLOR) == 0 && m_bit_depth < 8 ) #if (PNG_LIBPNG_VER_MAJOR*10000 + PNG_LIBPNG_VER_MINOR*100 + PNG_LIBPNG_VER_RELEASE >= 10209) || \ (PNG_LIBPNG_VER_MAJOR == 1 && PNG_LIBPNG_VER_MINOR == 0 && PNG_LIBPNG_VER_RELEASE >= 18) png_set_expand_gray_1_2_4_to_8( png_ptr ); From be4312ec3dc7c36f4799a6955f7f6e5b9786a439 Mon Sep 17 00:00:00 2001 From: alcinos Date: Wed, 27 Jan 2016 15:36:23 +0100 Subject: [PATCH 007/128] Wrap DenseOptFlow class around Farneback optical flow computation --- modules/cudaoptflow/src/farneback.cpp | 4 +- .../video/include/opencv2/video/tracking.hpp | 40 +++ modules/video/src/optflowgf.cpp | 233 ++++++++++-------- 3 files changed, 173 insertions(+), 104 deletions(-) diff --git a/modules/cudaoptflow/src/farneback.cpp b/modules/cudaoptflow/src/farneback.cpp index fb62df6864..43032b4f8f 100644 --- a/modules/cudaoptflow/src/farneback.cpp +++ b/modules/cudaoptflow/src/farneback.cpp @@ -93,7 +93,7 @@ namespace cv { namespace cuda { namespace device { namespace optflow_farneback namespace { - class FarnebackOpticalFlowImpl : public FarnebackOpticalFlow + class FarnebackOpticalFlowImpl : public cv::cuda::FarnebackOpticalFlow { public: FarnebackOpticalFlowImpl(int numLevels, double pyrScale, bool fastPyramids, int winSize, @@ -459,7 +459,7 @@ namespace } } -Ptr cv::cuda::FarnebackOpticalFlow::create(int numLevels, double pyrScale, bool fastPyramids, int winSize, +Ptr cv::cuda::FarnebackOpticalFlow::create(int numLevels, double pyrScale, bool fastPyramids, int winSize, int numIters, int polyN, double polySigma, int flags) { return makePtr(numLevels, pyrScale, fastPyramids, winSize, diff --git a/modules/video/include/opencv2/video/tracking.hpp b/modules/video/include/opencv2/video/tracking.hpp index d6954fecca..5afe209ce6 100644 --- a/modules/video/include/opencv2/video/tracking.hpp +++ b/modules/video/include/opencv2/video/tracking.hpp @@ -508,6 +508,46 @@ public: */ CV_EXPORTS_W Ptr createOptFlow_DualTVL1(); +/** @brief Class computing a dense optical flow using the Gunnar Farneback’s algorithm. + */ +class CV_EXPORTS_W FarnebackOpticalFlow : public DenseOpticalFlow +{ +public: + virtual int getNumLevels() const = 0; + virtual void setNumLevels(int numLevels) = 0; + + virtual double getPyrScale() const = 0; + virtual void setPyrScale(double pyrScale) = 0; + + virtual bool getFastPyramids() const = 0; + virtual void setFastPyramids(bool fastPyramids) = 0; + + virtual int getWinSize() const = 0; + virtual void setWinSize(int winSize) = 0; + + virtual int getNumIters() const = 0; + virtual void setNumIters(int numIters) = 0; + + virtual int getPolyN() const = 0; + virtual void setPolyN(int polyN) = 0; + + virtual double getPolySigma() const = 0; + virtual void setPolySigma(double polySigma) = 0; + + virtual int getFlags() const = 0; + virtual void setFlags(int flags) = 0; + + static Ptr create( + int numLevels = 5, + double pyrScale = 0.5, + bool fastPyramids = false, + int winSize = 13, + int numIters = 10, + int polyN = 5, + double polySigma = 1.1, + int flags = 0); +}; + //! @} video_track } // cv diff --git a/modules/video/src/optflowgf.cpp b/modules/video/src/optflowgf.cpp index 3b61bb24bf..b486edaf84 100644 --- a/modules/video/src/optflowgf.cpp +++ b/modules/video/src/optflowgf.cpp @@ -583,39 +583,63 @@ FarnebackUpdateFlow_GaussianBlur( const Mat& _R0, const Mat& _R1, } -#ifdef HAVE_OPENCL namespace cv { -class FarnebackOpticalFlow +namespace +{ +class FarnebackOpticalFlowImpl : public FarnebackOpticalFlow { public: - FarnebackOpticalFlow() + FarnebackOpticalFlowImpl(int numLevels=5, double pyrScale=0.5, bool fastPyramids=false, int winSize=13, + int numIters=10, int polyN=5, double polySigma=1.1, int flags=0) : + numLevels_(numLevels), pyrScale_(pyrScale), fastPyramids_(fastPyramids), winSize_(winSize), + numIters_(numIters), polyN_(polyN), polySigma_(polySigma), flags_(flags) { - numLevels = 5; - pyrScale = 0.5; - fastPyramids = false; - winSize = 13; - numIters = 10; - polyN = 5; - polySigma = 1.1; - flags = 0; } - int numLevels; - double pyrScale; - bool fastPyramids; - int winSize; - int numIters; - int polyN; - double polySigma; - int flags; + virtual int getNumLevels() const { return numLevels_; } + virtual void setNumLevels(int numLevels) { numLevels_ = numLevels; } + virtual double getPyrScale() const { return pyrScale_; } + virtual void setPyrScale(double pyrScale) { pyrScale_ = pyrScale; } + + virtual bool getFastPyramids() const { return fastPyramids_; } + virtual void setFastPyramids(bool fastPyramids) { fastPyramids_ = fastPyramids; } + + virtual int getWinSize() const { return winSize_; } + virtual void setWinSize(int winSize) { winSize_ = winSize; } + + virtual int getNumIters() const { return numIters_; } + virtual void setNumIters(int numIters) { numIters_ = numIters; } + + virtual int getPolyN() const { return polyN_; } + virtual void setPolyN(int polyN) { polyN_ = polyN; } + + virtual double getPolySigma() const { return polySigma_; } + virtual void setPolySigma(double polySigma) { polySigma_ = polySigma; } + + virtual int getFlags() const { return flags_; } + virtual void setFlags(int flags) { flags_ = flags; } + + virtual void calc(InputArray I0, InputArray I1, InputOutputArray flow); + +private: + int numLevels_; + double pyrScale_; + bool fastPyramids_; + int winSize_; + int numIters_; + int polyN_; + double polySigma_; + int flags_; + +#ifdef HAVE_OPENCL bool operator ()(const UMat &frame0, const UMat &frame1, UMat &flowx, UMat &flowy) { CV_Assert(frame0.channels() == 1 && frame1.channels() == 1); CV_Assert(frame0.size() == frame1.size()); - CV_Assert(polyN == 5 || polyN == 7); - CV_Assert(!fastPyramids || std::abs(pyrScale - 0.5) < 1e-6); + CV_Assert(polyN_ == 5 || polyN_ == 7); + CV_Assert(!fastPyramids_ || std::abs(pyrScale_ - 0.5) < 1e-6); const int min_size = 32; @@ -630,9 +654,9 @@ public: // Crop unnecessary levels double scale = 1; int numLevelsCropped = 0; - for (; numLevelsCropped < numLevels; numLevelsCropped++) + for (; numLevelsCropped < numLevels_; numLevelsCropped++) { - scale *= pyrScale; + scale *= pyrScale_; if (size.width*scale < min_size || size.height*scale < min_size) break; } @@ -640,7 +664,7 @@ public: frame0.convertTo(frames_[0], CV_32F); frame1.convertTo(frames_[1], CV_32F); - if (fastPyramids) + if (fastPyramids_) { // Build Gaussian pyramids using pyrDown() pyramid0_.resize(numLevelsCropped + 1); @@ -654,13 +678,13 @@ public: } } - setPolynomialExpansionConsts(polyN, polySigma); + setPolynomialExpansionConsts(polyN_, polySigma_); for (int k = numLevelsCropped; k >= 0; k--) { scale = 1; for (int i = 0; i < k; i++) - scale *= pyrScale; + scale *= pyrScale_; double sigma = (1./scale - 1) * 0.5; int smoothSize = cvRound(sigma*5) | 1; @@ -669,7 +693,7 @@ public: int width = cvRound(size.width*scale); int height = cvRound(size.height*scale); - if (fastPyramids) + if (fastPyramids_) { width = pyramid0_[k].cols; height = pyramid0_[k].rows; @@ -688,7 +712,7 @@ public: if (prevFlowX.empty()) { - if (flags & cv::OPTFLOW_USE_INITIAL_FLOW) + if (flags_ & cv::OPTFLOW_USE_INITIAL_FLOW) { resize(flowx0, curFlowX, Size(width, height), 0, 0, INTER_LINEAR); resize(flowy0, curFlowY, Size(width, height), 0, 0, INTER_LINEAR); @@ -705,8 +729,8 @@ public: { resize(prevFlowX, curFlowX, Size(width, height), 0, 0, INTER_LINEAR); resize(prevFlowY, curFlowY, Size(width, height), 0, 0, INTER_LINEAR); - multiply(1./pyrScale, curFlowX, curFlowX); - multiply(1./pyrScale, curFlowY, curFlowY); + multiply(1./pyrScale_, curFlowX, curFlowX); + multiply(1./pyrScale_, curFlowY, curFlowY); } UMat M = allocMatFromBuf(5*height, width, CV_32F, M_); @@ -717,7 +741,7 @@ public: allocMatFromBuf(5*height, width, CV_32F, R_[1]) }; - if (fastPyramids) + if (fastPyramids_) { if (!polynomialExpansionOcl(pyramid0_[k], R[0])) return false; @@ -752,18 +776,18 @@ public: if (!updateMatricesOcl(curFlowX, curFlowY, R[0], R[1], M)) return false; - if (flags & OPTFLOW_FARNEBACK_GAUSSIAN) - setGaussianBlurKernel(winSize, winSize/2*0.3f); - for (int i = 0; i < numIters; i++) + if (flags_ & OPTFLOW_FARNEBACK_GAUSSIAN) + setGaussianBlurKernel(winSize_, winSize_/2*0.3f); + for (int i = 0; i < numIters_; i++) { - if (flags & OPTFLOW_FARNEBACK_GAUSSIAN) + if (flags_ & OPTFLOW_FARNEBACK_GAUSSIAN) { - if (!updateFlow_gaussianBlur(R[0], R[1], curFlowX, curFlowY, M, bufM, winSize, i < numIters-1)) + if (!updateFlow_gaussianBlur(R[0], R[1], curFlowX, curFlowY, M, bufM, winSize_, i < numIters_-1)) return false; } else { - if (!updateFlow_boxFilter(R[0], R[1], curFlowX, curFlowY, M, bufM, winSize, i < numIters-1)) + if (!updateFlow_boxFilter(R[0], R[1], curFlowX, curFlowY, M, bufM, winSize_, i < numIters_-1)) return false; } } @@ -776,7 +800,9 @@ public: flowy = curFlowY; return true; } - + virtual void collectGarbage(){ + releaseMemory(); + } void releaseMemory() { frames_[0].release(); @@ -898,15 +924,15 @@ private: #else size_t localsize[2] = { 256, 1}; #endif - size_t globalsize[2] = { DIVUP((size_t)src.cols, localsize[0] - 2*polyN) * localsize[0], (size_t)src.rows}; + size_t globalsize[2] = { DIVUP((size_t)src.cols, localsize[0] - 2*polyN_) * localsize[0], (size_t)src.rows}; #if 0 const cv::ocl::Device &device = cv::ocl::Device::getDefault(); bool useDouble = (0 != device.doubleFPConfig()); - cv::String build_options = cv::format("-D polyN=%d -D USE_DOUBLE=%d", polyN, useDouble ? 1 : 0); + cv::String build_options = cv::format("-D polyN=%d -D USE_DOUBLE=%d", polyN_, useDouble ? 1 : 0); #else - cv::String build_options = cv::format("-D polyN=%d", polyN); + cv::String build_options = cv::format("-D polyN=%d", polyN_); #endif ocl::Kernel kernel; if (!kernel.create("polynomialExpansion", cv::ocl::video::optical_flow_farneback_oclsrc, build_options)) @@ -1036,60 +1062,43 @@ private: return false; return true; } + bool calc_ocl( InputArray _prev0, InputArray _next0, + InputOutputArray _flow0) + { + if ((5 != polyN_) && (7 != polyN_)) + return false; + if (_next0.size() != _prev0.size()) + return false; + int typePrev = _prev0.type(); + int typeNext = _next0.type(); + if ((1 != CV_MAT_CN(typePrev)) || (1 != CV_MAT_CN(typeNext))) + return false; + + std::vector flowar; + if (!_flow0.empty()) + split(_flow0, flowar); + else + { + flowar.push_back(UMat()); + flowar.push_back(UMat()); + } + if(!this->operator()(_prev0.getUMat(), _next0.getUMat(), flowar[0], flowar[1])){ + return false; + } + merge(flowar, _flow0); + return true; + } +#else // HAVE_OPENCL + virtual void collectGarbage(){} +#endif }; -static bool ocl_calcOpticalFlowFarneback( InputArray _prev0, InputArray _next0, - InputOutputArray _flow0, double pyr_scale, int levels, int winsize, - int iterations, int poly_n, double poly_sigma, int flags ) +void FarnebackOpticalFlowImpl::calc(InputArray _prev0, InputArray _next0, + InputOutputArray _flow0) { - if ((5 != poly_n) && (7 != poly_n)) - return false; - if (_next0.size() != _prev0.size()) - return false; - int typePrev = _prev0.type(); - int typeNext = _next0.type(); - if ((1 != CV_MAT_CN(typePrev)) || (1 != CV_MAT_CN(typeNext))) - return false; - - FarnebackOpticalFlow opticalFlow; - opticalFlow.numLevels = levels; - opticalFlow.pyrScale = pyr_scale; - opticalFlow.fastPyramids= false; - opticalFlow.winSize = winsize; - opticalFlow.numIters = iterations; - opticalFlow.polyN = poly_n; - opticalFlow.polySigma = poly_sigma; - opticalFlow.flags = flags; - - std::vector flowar; - if (!_flow0.empty()) - split(_flow0, flowar); - else - { - flowar.push_back(UMat()); - flowar.push_back(UMat()); - } - if (!opticalFlow(_prev0.getUMat(), _next0.getUMat(), flowar[0], flowar[1])) - return false; - merge(flowar, _flow0); - return true; -} -} -#endif // HAVE_OPENCL - -void cv::calcOpticalFlowFarneback( InputArray _prev0, InputArray _next0, - InputOutputArray _flow0, double pyr_scale, int levels, int winsize, - int iterations, int poly_n, double poly_sigma, int flags ) -{ -#ifdef HAVE_OPENCL - bool use_opencl = ocl::useOpenCL() && _flow0.isUMat(); - if( use_opencl && ocl_calcOpticalFlowFarneback(_prev0, _next0, _flow0, pyr_scale, levels, winsize, iterations, poly_n, poly_sigma, flags)) - { - CV_IMPL_ADD(CV_IMPL_OCL); - return; - } -#endif - + CV_OCL_RUN(_flow0.isUMat() && + ocl::Image2D::isFormatSupported(CV_32F, 1, false), + calc_ocl(_prev0,_next0,_flow0)) Mat prev0 = _prev0.getMat(), next0 = _next0.getMat(); const int min_size = 32; const Mat* img[2] = { &prev0, &next0 }; @@ -1097,15 +1106,16 @@ void cv::calcOpticalFlowFarneback( InputArray _prev0, InputArray _next0, int i, k; double scale; Mat prevFlow, flow, fimg; + int levels = numLevels_; CV_Assert( prev0.size() == next0.size() && prev0.channels() == next0.channels() && - prev0.channels() == 1 && pyr_scale < 1 ); + prev0.channels() == 1 && pyrScale_ < 1 ); _flow0.create( prev0.size(), CV_32FC2 ); Mat flow0 = _flow0.getMat(); for( k = 0, scale = 1; k < levels; k++ ) { - scale *= pyr_scale; + scale *= pyrScale_; if( prev0.cols*scale < min_size || prev0.rows*scale < min_size ) break; } @@ -1115,7 +1125,7 @@ void cv::calcOpticalFlowFarneback( InputArray _prev0, InputArray _next0, for( k = levels; k >= 0; k-- ) { for( i = 0, scale = 1; i < k; i++ ) - scale *= pyr_scale; + scale *= pyrScale_; double sigma = (1./scale-1)*0.5; int smooth_sz = cvRound(sigma*5)|1; @@ -1131,7 +1141,7 @@ void cv::calcOpticalFlowFarneback( InputArray _prev0, InputArray _next0, if( prevFlow.empty() ) { - if( flags & OPTFLOW_USE_INITIAL_FLOW ) + if( flags_ & OPTFLOW_USE_INITIAL_FLOW ) { resize( flow0, flow, Size(width, height), 0, 0, INTER_AREA ); flow *= scale; @@ -1142,7 +1152,7 @@ void cv::calcOpticalFlowFarneback( InputArray _prev0, InputArray _next0, else { resize( prevFlow, flow, Size(width, height), 0, 0, INTER_LINEAR ); - flow *= 1./pyr_scale; + flow *= 1./pyrScale_; } Mat R[2], I, M; @@ -1151,19 +1161,38 @@ void cv::calcOpticalFlowFarneback( InputArray _prev0, InputArray _next0, img[i]->convertTo(fimg, CV_32F); GaussianBlur(fimg, fimg, Size(smooth_sz, smooth_sz), sigma, sigma); resize( fimg, I, Size(width, height), INTER_LINEAR ); - FarnebackPolyExp( I, R[i], poly_n, poly_sigma ); + FarnebackPolyExp( I, R[i], polyN_, polySigma_ ); } FarnebackUpdateMatrices( R[0], R[1], flow, M, 0, flow.rows ); - for( i = 0; i < iterations; i++ ) + for( i = 0; i < numIters_; i++ ) { - if( flags & OPTFLOW_FARNEBACK_GAUSSIAN ) - FarnebackUpdateFlow_GaussianBlur( R[0], R[1], flow, M, winsize, i < iterations - 1 ); + if( flags_ & OPTFLOW_FARNEBACK_GAUSSIAN ) + FarnebackUpdateFlow_GaussianBlur( R[0], R[1], flow, M, winSize_, i < numIters_ - 1 ); else - FarnebackUpdateFlow_Blur( R[0], R[1], flow, M, winsize, i < iterations - 1 ); + FarnebackUpdateFlow_Blur( R[0], R[1], flow, M, winSize_, i < numIters_ - 1 ); } prevFlow = flow; } } +} // namespace +} // namespace cv + +void cv::calcOpticalFlowFarneback( InputArray _prev0, InputArray _next0, + InputOutputArray _flow0, double pyr_scale, int levels, int winsize, + int iterations, int poly_n, double poly_sigma, int flags ) +{ + Ptr optflow; + optflow = makePtr(levels,pyr_scale,false,winsize,iterations,poly_n,poly_sigma,flags); + optflow->calc(_prev0,_next0,_flow0); +} + + +cv::Ptr cv::FarnebackOpticalFlow::create(int numLevels, double pyrScale, bool fastPyramids, int winSize, + int numIters, int polyN, double polySigma, int flags) +{ + return makePtr(numLevels, pyrScale, fastPyramids, winSize, + numIters, polyN, polySigma, flags); +} From 6e3b90de9b12c02e6be93f0f3cde01769cd5078a Mon Sep 17 00:00:00 2001 From: alcinos Date: Thu, 28 Jan 2016 16:26:11 +0100 Subject: [PATCH 008/128] Add static creator for TVL1 optical flow class --- modules/core/include/opencv2/core/ptr.inl.hpp | 11 ++++++++++ .../video/include/opencv2/video/tracking.hpp | 15 +++++++++++++ modules/video/src/tvl1flow.cpp | 21 +++++++++++++++++++ modules/video/test/test_tvl1optflow.cpp | 2 +- samples/cpp/tvl1_optical_flow.cpp | 2 +- 5 files changed, 49 insertions(+), 2 deletions(-) diff --git a/modules/core/include/opencv2/core/ptr.inl.hpp b/modules/core/include/opencv2/core/ptr.inl.hpp index 3f6f214a8f..8c09d93cd8 100644 --- a/modules/core/include/opencv2/core/ptr.inl.hpp +++ b/modules/core/include/opencv2/core/ptr.inl.hpp @@ -358,6 +358,17 @@ Ptr makePtr(const A1& a1, const A2& a2, const A3& a3, const A4& a4, const A5& return Ptr(new T(a1, a2, a3, a4, a5, a6, a7, a8, a9, a10)); } +template +Ptr makePtr(const A1& a1, const A2& a2, const A3& a3, const A4& a4, const A5& a5, const A6& a6, const A7& a7, const A8& a8, const A9& a9, const A10& a10, const A11& a11) +{ + return Ptr(new T(a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11)); +} + +template +Ptr makePtr(const A1& a1, const A2& a2, const A3& a3, const A4& a4, const A5& a5, const A6& a6, const A7& a7, const A8& a8, const A9& a9, const A10& a10, const A11& a11, const A12& a12) +{ + return Ptr(new T(a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11, a12)); +} } // namespace cv //! @endcond diff --git a/modules/video/include/opencv2/video/tracking.hpp b/modules/video/include/opencv2/video/tracking.hpp index 5afe209ce6..f0993e5f81 100644 --- a/modules/video/include/opencv2/video/tracking.hpp +++ b/modules/video/include/opencv2/video/tracking.hpp @@ -502,6 +502,21 @@ public: virtual int getMedianFiltering() const = 0; /** @copybrief getMedianFiltering @see getMedianFiltering */ virtual void setMedianFiltering(int val) = 0; + + /** @brief Creates instance of cv::DualTVL1OpticalFlow*/ + static Ptr create( + double tau = 0.25, + double lambda = 0.15, + double theta = 0.3, + int nscales = 5, + int warps = 5, + double epsilon = 0.01, + int innnerIterations = 30, + int outerIterations = 10, + double scaleStep = 0.8, + double gamma = 0.0, + int medianFiltering = 5, + bool useInitialFlow = false); }; /** @brief Creates instance of cv::DenseOpticalFlow diff --git a/modules/video/src/tvl1flow.cpp b/modules/video/src/tvl1flow.cpp index b10dc3f344..c0299e5c95 100644 --- a/modules/video/src/tvl1flow.cpp +++ b/modules/video/src/tvl1flow.cpp @@ -89,6 +89,17 @@ namespace { class OpticalFlowDual_TVL1 : public DualTVL1OpticalFlow { public: + + OpticalFlowDual_TVL1(double tau_, double lambda_, double theta_, int nscales_, int warps_, + double epsilon_, int innerIterations_, int outerIterations_, + double scaleStep_, double gamma_, int medianFiltering_, + bool useInitialFlow_) : + tau(tau_), lambda(lambda_), theta(theta_), gamma(gamma_), nscales(nscales_), + warps(warps_), epsilon(epsilon_), innerIterations(innerIterations_), + outerIterations(outerIterations_), useInitialFlow(useInitialFlow_), + scaleStep(scaleStep_), medianFiltering(medianFiltering_) + { + } OpticalFlowDual_TVL1(); void calc(InputArray I0, InputArray I1, InputOutputArray flow); @@ -1450,3 +1461,13 @@ Ptr cv::createOptFlow_DualTVL1() { return makePtr(); } + +Ptr cv::DualTVL1OpticalFlow::create( + double tau, double lambda, double theta, int nscales, int warps, + double epsilon, int innerIterations, int outerIterations, double scaleStep, + double gamma, int medianFilter, bool useInitialFlow) +{ + return makePtr(tau, lambda, theta, nscales, warps, + epsilon, innerIterations, outerIterations, + scaleStep, gamma, medianFilter, useInitialFlow); +} diff --git a/modules/video/test/test_tvl1optflow.cpp b/modules/video/test/test_tvl1optflow.cpp index e4b80637ec..3976f25599 100644 --- a/modules/video/test/test_tvl1optflow.cpp +++ b/modules/video/test/test_tvl1optflow.cpp @@ -154,7 +154,7 @@ TEST(Video_calcOpticalFlowDual_TVL1, Regression) ASSERT_FALSE(frame2.empty()); Mat_ flow; - Ptr tvl1 = createOptFlow_DualTVL1(); + Ptr tvl1 = cv::DualTVL1OpticalFlow::create(); tvl1->calc(frame1, frame2, flow); diff --git a/samples/cpp/tvl1_optical_flow.cpp b/samples/cpp/tvl1_optical_flow.cpp index 55b5558ebe..95871c7298 100644 --- a/samples/cpp/tvl1_optical_flow.cpp +++ b/samples/cpp/tvl1_optical_flow.cpp @@ -185,7 +185,7 @@ int main(int argc, const char* argv[]) } Mat_ flow; - Ptr tvl1 = createOptFlow_DualTVL1(); + Ptr tvl1 = cv::DualTVL1OpticalFlow::create(); const double start = (double)getTickCount(); tvl1->calc(frame0, frame1, flow); From 9b70c44f009bd06ca599647a96158e08c20db1b5 Mon Sep 17 00:00:00 2001 From: alcinos Date: Thu, 28 Jan 2016 16:55:45 +0100 Subject: [PATCH 009/128] Adding interface for Sparse flow computation --- .../video/include/opencv2/video/tracking.hpp | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/modules/video/include/opencv2/video/tracking.hpp b/modules/video/include/opencv2/video/tracking.hpp index f0993e5f81..983a0c390d 100644 --- a/modules/video/include/opencv2/video/tracking.hpp +++ b/modules/video/include/opencv2/video/tracking.hpp @@ -397,6 +397,27 @@ public: CV_WRAP virtual void collectGarbage() = 0; }; +/** @brief Base interface for sparse optical flow algorithms. + */ +class CV_EXPORTS_W SparseOpticalFlow : public Algorithm +{ +public: + /** @brief Calculates a sparse optical flow. + + @param prevImg First input image. + @param nextImg Second input image of the same size and the same type as prevImg. + @param prevPts Vector of 2D points for which the flow needs to be found. + @param nextPts Output vector of 2D points containing the calculated new positions of input features in the second image. + @param status Output status vector. Each element of the vector is set to 1 if the + flow for the corresponding features has been found. Otherwise, it is set to 0. + @param err Optional output vector that contains error response for each point (inverse confidence). + */ + CV_WRAP virtual void calc(InputArray prevImg, InputArray nextImg, + InputArray prevPts, InputOutputArray nextPts, + OutputArray status, + OutputArray err = cv::noArray()) = 0; +}; + /** @brief "Dual TV L1" Optical Flow Algorithm. The class implements the "Dual TV L1" optical flow algorithm described in @cite Zach2007 and @@ -563,6 +584,7 @@ public: int flags = 0); }; + //! @} video_track } // cv From e22b838af8a0078ca472583d76d0244dc9fad455 Mon Sep 17 00:00:00 2001 From: alcinos Date: Thu, 28 Jan 2016 18:45:52 +0100 Subject: [PATCH 010/128] Wrap SparseOptFlow class around PyrLK optical flow computation --- modules/cudaoptflow/src/pyrlk.cpp | 12 +- .../video/include/opencv2/video/tracking.hpp | 34 ++++++ modules/video/src/lkpyramid.cpp | 112 +++++++++++------- 3 files changed, 107 insertions(+), 51 deletions(-) diff --git a/modules/cudaoptflow/src/pyrlk.cpp b/modules/cudaoptflow/src/pyrlk.cpp index dcfd1f66de..c7f706087b 100644 --- a/modules/cudaoptflow/src/pyrlk.cpp +++ b/modules/cudaoptflow/src/pyrlk.cpp @@ -47,9 +47,9 @@ using namespace cv::cuda; #if !defined (HAVE_CUDA) || defined (CUDA_DISABLER) -Ptr cv::cuda::SparsePyrLKOpticalFlow::create(Size, int, int, bool) { throw_no_cuda(); return Ptr(); } +Ptr cv::cuda::SparsePyrLKOpticalFlow::create(Size, int, int, bool) { throw_no_cuda(); return Ptr(); } -Ptr cv::cuda::DensePyrLKOpticalFlow::create(Size, int, int, bool) { throw_no_cuda(); return Ptr(); } +Ptr cv::cuda::DensePyrLKOpticalFlow::create(Size, int, int, bool) { throw_no_cuda(); return Ptr(); } #else /* !defined (HAVE_CUDA) */ @@ -283,7 +283,7 @@ namespace vPyr[idx].copyTo(v, stream); } - class SparsePyrLKOpticalFlowImpl : public SparsePyrLKOpticalFlow, private PyrLKOpticalFlowBase + class SparsePyrLKOpticalFlowImpl : public cv::cuda::SparsePyrLKOpticalFlow, private PyrLKOpticalFlowBase { public: SparsePyrLKOpticalFlowImpl(Size winSize, int maxLevel, int iters, bool useInitialFlow) : @@ -366,14 +366,14 @@ namespace }; } -Ptr cv::cuda::SparsePyrLKOpticalFlow::create(Size winSize, int maxLevel, int iters, bool useInitialFlow) +Ptr cv::cuda::SparsePyrLKOpticalFlow::create(Size winSize, int maxLevel, int iters, bool useInitialFlow) { return makePtr(winSize, maxLevel, iters, useInitialFlow); } -Ptr cv::cuda::DensePyrLKOpticalFlow::create(Size winSize, int maxLevel, int iters, bool useInitialFlow) +Ptr cv::cuda::DensePyrLKOpticalFlow::create(Size winSize, int maxLevel, int iters, bool useInitialFlow) { return makePtr(winSize, maxLevel, iters, useInitialFlow); } -#endif /* !defined (HAVE_CUDA) */ \ No newline at end of file +#endif /* !defined (HAVE_CUDA) */ diff --git a/modules/video/include/opencv2/video/tracking.hpp b/modules/video/include/opencv2/video/tracking.hpp index 983a0c390d..996e60b2dd 100644 --- a/modules/video/include/opencv2/video/tracking.hpp +++ b/modules/video/include/opencv2/video/tracking.hpp @@ -585,6 +585,40 @@ public: }; +/** @brief Class used for calculating a sparse optical flow. + +The class can calculate an optical flow for a sparse feature set using the +iterative Lucas-Kanade method with pyramids. + +@sa calcOpticalFlowPyrLK + +*/ +class CV_EXPORTS SparsePyrLKOpticalFlow : public SparseOpticalFlow +{ +public: + virtual Size getWinSize() const = 0; + virtual void setWinSize(Size winSize) = 0; + + virtual int getMaxLevel() const = 0; + virtual void setMaxLevel(int maxLevel) = 0; + + virtual TermCriteria getTermCriteria() const = 0; + virtual void setTermCriteria(TermCriteria& crit) = 0; + + virtual int getFlags() const = 0; + virtual void setFlags(int flags) = 0; + + virtual double getMinEigThreshold() const = 0; + virtual void setMinEigThreshold(double minEigThreshold) = 0; + + static Ptr create( + Size winSize = Size(21, 21), + int maxLevel = 3, TermCriteria crit = + TermCriteria(TermCriteria::COUNT+TermCriteria::EPS, 30, 0.01), + int flags = 0, + double minEigThreshold = 1e-4); +}; + //! @} video_track } // cv diff --git a/modules/video/src/lkpyramid.cpp b/modules/video/src/lkpyramid.cpp index 891ae78609..25614d9ced 100644 --- a/modules/video/src/lkpyramid.cpp +++ b/modules/video/src/lkpyramid.cpp @@ -837,10 +837,11 @@ int cv::buildOpticalFlowPyramid(InputArray _img, OutputArrayOfArrays pyramid, Si return maxLevel; } -#ifdef HAVE_OPENCL namespace cv { - class PyrLKOpticalFlow +namespace +{ + class SparsePyrLKOpticalFlowImpl : public SparsePyrLKOpticalFlow { struct dim3 { @@ -848,17 +849,40 @@ namespace cv dim3() : x(0), y(0), z(0) { } }; public: - PyrLKOpticalFlow() + SparsePyrLKOpticalFlowImpl(Size winSize_ = Size(21,21), + int maxLevel_ = 3, + TermCriteria criteria_ = TermCriteria(TermCriteria::COUNT+TermCriteria::EPS, 30, 0.01), + int flags_ = 0, + double minEigThreshold_ = 1e-4) : + winSize(winSize_), maxLevel(maxLevel_), criteria(criteria_), flags(flags_), minEigThreshold(minEigThreshold_) +#ifdef HAVE_OPENCL + , iters(criteria_.maxCount), derivLambda(criteria_.epsilon), useInitialFlow(0 != (flags_ & OPTFLOW_LK_GET_MIN_EIGENVALS)), waveSize(0) +#endif { - winSize = Size(21, 21); - maxLevel = 3; - iters = 30; - derivLambda = 0.5; - useInitialFlow = false; - - waveSize = 0; } + virtual Size getWinSize() const {return winSize;} + virtual void setWinSize(Size winSize_){winSize = winSize_;} + + virtual int getMaxLevel() const {return maxLevel;} + virtual void setMaxLevel(int maxLevel_){maxLevel = maxLevel_;} + + virtual TermCriteria getTermCriteria() const {return criteria;} + virtual void setTermCriteria(TermCriteria& crit_){criteria=crit_;} + + virtual int getFlags() const {return flags; } + virtual void setFlags(int flags_){flags=flags_;} + + virtual double getMinEigThreshold() const {return minEigThreshold;} + virtual void setMinEigThreshold(double minEigThreshold_){minEigThreshold=minEigThreshold_;} + + virtual void calc(InputArray prevImg, InputArray nextImg, + InputArray prevPts, InputOutputArray nextPts, + OutputArray status, + OutputArray err = cv::noArray()); + + private: +#ifdef HAVE_OPENCL bool checkParam() { iters = std::min(std::max(iters, 0), 100); @@ -930,14 +954,17 @@ namespace cv } return true; } +#endif Size winSize; int maxLevel; + TermCriteria criteria; + int flags; + double minEigThreshold; +#ifdef HAVE_OPENCL int iters; double derivLambda; bool useInitialFlow; - - private: int waveSize; bool initWaveSize() { @@ -1017,15 +1044,11 @@ namespace cv { return (cv::ocl::Device::TYPE_CPU == cv::ocl::Device::getDefault().type()); } - }; - static bool ocl_calcOpticalFlowPyrLK(InputArray _prevImg, InputArray _nextImg, - InputArray _prevPts, InputOutputArray _nextPts, - OutputArray _status, OutputArray _err, - Size winSize, int maxLevel, - TermCriteria criteria, - int flags/*, double minEigThreshold*/ ) + bool ocl_calcOpticalFlowPyrLK(InputArray _prevImg, InputArray _nextImg, + InputArray _prevPts, InputOutputArray _nextPts, + OutputArray _status, OutputArray _err) { if (0 != (OPTFLOW_LK_GET_MIN_EIGENVALS & flags)) return false; @@ -1045,7 +1068,6 @@ namespace cv if ((1 != _prevPts.size().height) && (1 != _prevPts.size().width)) return false; size_t npoints = _prevPts.total(); - bool useInitialFlow = (0 != (flags & OPTFLOW_USE_INITIAL_FLOW)); if (useInitialFlow) { if (_nextPts.empty() || _nextPts.type() != CV_32FC2 || (!_prevPts.isContinuous())) @@ -1060,14 +1082,7 @@ namespace cv _nextPts.create(_prevPts.size(), _prevPts.type()); } - PyrLKOpticalFlow opticalFlow; - opticalFlow.winSize = winSize; - opticalFlow.maxLevel = maxLevel; - opticalFlow.iters = criteria.maxCount; - opticalFlow.derivLambda = criteria.epsilon; - opticalFlow.useInitialFlow = useInitialFlow; - - if (!opticalFlow.checkParam()) + if (!checkParam()) return false; UMat umatErr; @@ -1082,28 +1097,19 @@ namespace cv _status.create((int)npoints, 1, CV_8UC1); UMat umatNextPts = _nextPts.getUMat(); UMat umatStatus = _status.getUMat(); - return opticalFlow.sparse(_prevImg.getUMat(), _nextImg.getUMat(), _prevPts.getUMat(), umatNextPts, umatStatus, umatErr); + return sparse(_prevImg.getUMat(), _nextImg.getUMat(), _prevPts.getUMat(), umatNextPts, umatStatus, umatErr); } +#endif }; -#endif -void cv::calcOpticalFlowPyrLK( InputArray _prevImg, InputArray _nextImg, +void SparsePyrLKOpticalFlowImpl::calc( InputArray _prevImg, InputArray _nextImg, InputArray _prevPts, InputOutputArray _nextPts, - OutputArray _status, OutputArray _err, - Size winSize, int maxLevel, - TermCriteria criteria, - int flags, double minEigThreshold ) + OutputArray _status, OutputArray _err) { -#ifdef HAVE_OPENCL - bool use_opencl = ocl::useOpenCL() && - (_prevImg.isUMat() || _nextImg.isUMat()) && - ocl::Image2D::isFormatSupported(CV_32F, 1, false); - if ( use_opencl && ocl_calcOpticalFlowPyrLK(_prevImg, _nextImg, _prevPts, _nextPts, _status, _err, winSize, maxLevel, criteria, flags/*, minEigThreshold*/)) - { - CV_IMPL_ADD(CV_IMPL_OCL); - return; - } -#endif + CV_OCL_RUN(ocl::useOpenCL() && + (_prevImg.isUMat() || _nextImg.isUMat()) && + ocl::Image2D::isFormatSupported(CV_32F, 1, false), + ocl_calcOpticalFlowPyrLK(_prevImg, _nextImg, _prevPts, _nextPts, _status, _err)) Mat prevPtsMat = _prevPts.getMat(); const int derivDepth = DataType::depth; @@ -1262,6 +1268,22 @@ void cv::calcOpticalFlowPyrLK( InputArray _prevImg, InputArray _nextImg, } } +} // namespace +} // namespace cv +cv::Ptr cv::SparsePyrLKOpticalFlow::create(Size winSize, int maxLevel, TermCriteria crit, int flags, double minEigThreshold){ + return makePtr(winSize,maxLevel,crit,flags,minEigThreshold); +} +void cv::calcOpticalFlowPyrLK( InputArray _prevImg, InputArray _nextImg, + InputArray _prevPts, InputOutputArray _nextPts, + OutputArray _status, OutputArray _err, + Size winSize, int maxLevel, + TermCriteria criteria, + int flags, double minEigThreshold ) +{ + Ptr optflow = cv::SparsePyrLKOpticalFlow::create(winSize,maxLevel,criteria,flags,minEigThreshold); + optflow->calc(_prevImg,_nextImg,_prevPts,_nextPts,_status,_err); +} + namespace cv { From b030ac0433768b2e9eef1e53849a7a719023db6f Mon Sep 17 00:00:00 2001 From: Mathieu Barnachon Date: Mon, 8 Feb 2016 13:17:08 +0100 Subject: [PATCH 011/128] Ensure the Cuda context is initialized correctly as long as the setDevice is not called in a multi-thread environment. --- modules/core/src/cuda_info.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/modules/core/src/cuda_info.cpp b/modules/core/src/cuda_info.cpp index 5ad33ce8a1..08013ca988 100644 --- a/modules/core/src/cuda_info.cpp +++ b/modules/core/src/cuda_info.cpp @@ -70,6 +70,7 @@ void cv::cuda::setDevice(int device) (void) device; throw_no_cuda(); #else + cudaFree(0); cudaSafeCall( cudaSetDevice(device) ); #endif } From 6a0d3b3e42a1afd8f63b7e4b290c8227b93d7735 Mon Sep 17 00:00:00 2001 From: Mathieu Barnachon Date: Tue, 9 Feb 2016 14:40:09 +0100 Subject: [PATCH 012/128] Called after setDevice. Wrap in a cudaSafeCall. --- modules/core/src/cuda_info.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/core/src/cuda_info.cpp b/modules/core/src/cuda_info.cpp index 08013ca988..b412438581 100644 --- a/modules/core/src/cuda_info.cpp +++ b/modules/core/src/cuda_info.cpp @@ -70,8 +70,8 @@ void cv::cuda::setDevice(int device) (void) device; throw_no_cuda(); #else - cudaFree(0); cudaSafeCall( cudaSetDevice(device) ); + cudaSafeCall( cudaFree(0) ); #endif } From 93ff1fb2f21a552c258fc2e9f7973f7d5b159cfc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=9F=D1=80=D1=83=D0=BD=20=D0=92=D0=B8=D0=BA=D1=82=D0=BE?= =?UTF-8?q?=D1=80?= Date: Mon, 28 Mar 2016 15:59:18 +0300 Subject: [PATCH 013/128] Correct image borders and principal point computation in cv::stereoRectify --- modules/calib3d/src/calibration.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/modules/calib3d/src/calibration.cpp b/modules/calib3d/src/calibration.cpp index b8f569aeb8..8ff0c2cc8c 100644 --- a/modules/calib3d/src/calibration.cpp +++ b/modules/calib3d/src/calibration.cpp @@ -2289,8 +2289,8 @@ void cvStereoRectify( const CvMat* _cameraMatrix1, const CvMat* _cameraMatrix2, for( i = 0; i < 4; i++ ) { int j = (i<2) ? 0 : 1; - _pts[i].x = (float)((i % 2)*(nx-1)); - _pts[i].y = (float)(j*(ny-1)); + _pts[i].x = (float)((i % 2)*(nx)); + _pts[i].y = (float)(j*(ny)); } cvUndistortPoints( &pts, &pts, A, Dk, 0, 0 ); cvConvertPointsHomogeneous( &pts, &pts_3 ); @@ -2304,8 +2304,8 @@ void cvStereoRectify( const CvMat* _cameraMatrix1, const CvMat* _cameraMatrix2, _a_tmp[1][2]=0.0; cvProjectPoints2( &pts_3, k == 0 ? _R1 : _R2, &Z, &A_tmp, 0, &pts ); CvScalar avg = cvAvg(&pts); - cc_new[k].x = (nx-1)/2 - avg.val[0]; - cc_new[k].y = (ny-1)/2 - avg.val[1]; + cc_new[k].x = (nx)/2 - avg.val[0]; + cc_new[k].y = (ny)/2 - avg.val[1]; } // vertical focal length must be the same for both images to keep the epipolar constraint From b973b73ae749e376879cf72d815cb2e9604744e1 Mon Sep 17 00:00:00 2001 From: Bob Paulin Date: Wed, 20 Apr 2016 21:53:52 -0500 Subject: [PATCH 014/128] Add -maxscale parameter to limit the amount sample images can scale in background images --- apps/createsamples/createsamples.cpp | 13 ++++++++++--- apps/createsamples/utility.cpp | 12 +++++++----- apps/createsamples/utility.hpp | 2 +- 3 files changed, 18 insertions(+), 9 deletions(-) diff --git a/apps/createsamples/createsamples.cpp b/apps/createsamples/createsamples.cpp index 9f05e416bd..1660adee44 100644 --- a/apps/createsamples/createsamples.cpp +++ b/apps/createsamples/createsamples.cpp @@ -76,6 +76,7 @@ int main( int argc, char* argv[] ) double scale = 4.0; int width = 24; int height = 24; + double maxscale = -1.0; srand((unsigned int)time(0)); @@ -92,9 +93,10 @@ int main( int argc, char* argv[] ) " [-maxyangle ]\n" " [-maxzangle ]\n" " [-show []]\n" - " [-w ]\n [-h ]\n", + " [-w ]\n [-h ]\n" + " [-maxscale ]\n", argv[0], num, bgcolor, bgthreshold, maxintensitydev, - maxxangle, maxyangle, maxzangle, scale, width, height ); + maxxangle, maxyangle, maxzangle, scale, width, height, maxscale ); return 0; } @@ -172,6 +174,10 @@ int main( int argc, char* argv[] ) { height = atoi( argv[++i] ); } + else if( !strcmp( argv[i], "-maxscale" ) ) + { + maxscale = atof( argv[++i] ); + } } printf( "Info file name: %s\n", ((infoname == NULL) ? nullname : infoname ) ); @@ -194,6 +200,7 @@ int main( int argc, char* argv[] ) } printf( "Width: %d\n", width ); printf( "Height: %d\n", height ); + printf( "Max Scale: %g\n", maxscale); /* determine action */ if( imagename && vecname ) @@ -213,7 +220,7 @@ int main( int argc, char* argv[] ) cvCreateTestSamples( infoname, imagename, bgcolor, bgthreshold, bgfilename, num, invert, maxintensitydev, - maxxangle, maxyangle, maxzangle, showsamples, width, height ); + maxxangle, maxyangle, maxzangle, showsamples, width, height, maxscale); printf( "Done\n" ); } diff --git a/apps/createsamples/utility.cpp b/apps/createsamples/utility.cpp index b5834f3e02..ddcc1ebe5c 100644 --- a/apps/createsamples/utility.cpp +++ b/apps/createsamples/utility.cpp @@ -38,7 +38,6 @@ // the use of this software, even if advised of the possibility of such damage. // //M*/ - #include #include @@ -1308,7 +1307,7 @@ void cvCreateTestSamples( const char* infoname, int invert, int maxintensitydev, double maxxangle, double maxyangle, double maxzangle, int showsamples, - int winwidth, int winheight ) + int winwidth, int winheight, double maxscale ) { CvSampleDistortionData data; @@ -1337,7 +1336,6 @@ void cvCreateTestSamples( const char* infoname, int i; int x, y, width, height; float scale; - float maxscale; int inverse; if( showsamples ) @@ -1366,12 +1364,16 @@ void cvCreateTestSamples( const char* infoname, for( i = 0; i < count; i++ ) { icvGetNextFromBackgroundData( cvbgdata, cvbgreader ); - - maxscale = MIN( 0.7F * cvbgreader->src.cols / winwidth, + if( maxscale < 0.0 ) + { + maxscale = MIN( 0.7F * cvbgreader->src.cols / winwidth, 0.7F * cvbgreader->src.rows / winheight ); + } + if( maxscale < 1.0F ) continue; scale = (maxscale - 1.0F) * rand() / RAND_MAX + 1.0F; + width = (int) (scale * winwidth); height = (int) (scale * winheight); x = (int) ((0.1+0.8 * rand()/RAND_MAX) * (cvbgreader->src.cols - width)); diff --git a/apps/createsamples/utility.hpp b/apps/createsamples/utility.hpp index 9367778daf..d04947c9be 100644 --- a/apps/createsamples/utility.hpp +++ b/apps/createsamples/utility.hpp @@ -86,7 +86,7 @@ void cvCreateTestSamples( const char* infoname, int invert, int maxintensitydev, double maxxangle, double maxyangle, double maxzangle, int showsamples, - int winwidth, int winheight ); + int winwidth, int winheight, double maxscale ); /* * cvCreateTrainingSamplesFromInfo From bb8faec885a87836b94d5d7c3ea52c11fd380c9e Mon Sep 17 00:00:00 2001 From: Bob Paulin Date: Thu, 21 Apr 2016 07:47:13 -0500 Subject: [PATCH 015/128] #6443 Cast maxscale from double to float for scale calculation. --- apps/createsamples/utility.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/createsamples/utility.cpp b/apps/createsamples/utility.cpp index ddcc1ebe5c..cf2bdebbf9 100644 --- a/apps/createsamples/utility.cpp +++ b/apps/createsamples/utility.cpp @@ -1372,7 +1372,7 @@ void cvCreateTestSamples( const char* infoname, if( maxscale < 1.0F ) continue; - scale = (maxscale - 1.0F) * rand() / RAND_MAX + 1.0F; + scale = ((float)maxscale - 1.0F) * rand() / RAND_MAX + 1.0F; width = (int) (scale * winwidth); height = (int) (scale * winheight); From 86959310f900b19a0510d60990bd45e59baf6c3a Mon Sep 17 00:00:00 2001 From: Pavel Rojtberg Date: Tue, 26 Apr 2016 18:02:47 +0200 Subject: [PATCH 016/128] calibrationMatrixValues: bind C++ function in C instead of vice versa --- modules/calib3d/src/calibration.cpp | 85 +++++++++++++---------------- 1 file changed, 38 insertions(+), 47 deletions(-) diff --git a/modules/calib3d/src/calibration.cpp b/modules/calib3d/src/calibration.cpp index b8f569aeb8..bd759d294b 100644 --- a/modules/calib3d/src/calibration.cpp +++ b/modules/calib3d/src/calibration.cpp @@ -1626,58 +1626,24 @@ void cvCalibrationMatrixValues( const CvMat *calibMatr, CvSize imgSize, double apertureWidth, double apertureHeight, double *fovx, double *fovy, double *focalLength, CvPoint2D64f *principalPoint, double *pasp ) { - double alphax, alphay, mx, my; - int imgWidth = imgSize.width, imgHeight = imgSize.height; - /* Validate parameters. */ - if(calibMatr == 0) CV_Error(CV_StsNullPtr, "Some of parameters is a NULL pointer!"); if(!CV_IS_MAT(calibMatr)) CV_Error(CV_StsUnsupportedFormat, "Input parameters must be a matrices!"); - if(calibMatr->cols != 3 || calibMatr->rows != 3) - CV_Error(CV_StsUnmatchedSizes, "Size of matrices must be 3x3!"); - - alphax = cvmGet(calibMatr, 0, 0); - alphay = cvmGet(calibMatr, 1, 1); - assert(imgWidth != 0 && imgHeight != 0 && alphax != 0.0 && alphay != 0.0); - - /* Calculate pixel aspect ratio. */ - if(pasp) - *pasp = alphay / alphax; - - /* Calculate number of pixel per realworld unit. */ - - if(apertureWidth != 0.0 && apertureHeight != 0.0) { - mx = imgWidth / apertureWidth; - my = imgHeight / apertureHeight; - } else { - mx = 1.0; - if(pasp) - my = *pasp; - else - my = 1.0; - } - - /* Calculate fovx and fovy. */ - - if(fovx) - *fovx = 2 * atan(imgWidth / (2 * alphax)) * 180.0 / CV_PI; - - if(fovy) - *fovy = 2 * atan(imgHeight / (2 * alphay)) * 180.0 / CV_PI; - - /* Calculate focal length. */ - - if(focalLength) - *focalLength = alphax / mx; - - /* Calculate principle point. */ + double dummy; + Point2d pp; + cv::calibrationMatrixValues(cvarrToMat(calibMatr), imgSize, apertureWidth, apertureHeight, + fovx ? *fovx : dummy, + fovy ? *fovy : dummy, + focalLength ? *focalLength : dummy, + pp, + pasp ? *pasp : dummy); if(principalPoint) - *principalPoint = cvPoint2D64f(cvmGet(calibMatr, 0, 2) / mx, cvmGet(calibMatr, 1, 2) / my); + *principalPoint = cvPoint2D64f(pp.x, pp.y); } @@ -3369,10 +3335,35 @@ void cv::calibrationMatrixValues( InputArray _cameraMatrix, Size imageSize, double& fovx, double& fovy, double& focalLength, Point2d& principalPoint, double& aspectRatio ) { - Mat cameraMatrix = _cameraMatrix.getMat(); - CvMat c_cameraMatrix = cameraMatrix; - cvCalibrationMatrixValues( &c_cameraMatrix, imageSize, apertureWidth, apertureHeight, - &fovx, &fovy, &focalLength, (CvPoint2D64f*)&principalPoint, &aspectRatio ); + if(_cameraMatrix.size() != Size(3, 3)) + CV_Error(CV_StsUnmatchedSizes, "Size of cameraMatrix must be 3x3!"); + + Matx33d K = _cameraMatrix.getMat(); + + CV_DbgAssert(imageSize.width != 0 && imageSize.height != 0 && K(0, 0) != 0.0 && K(1, 1) != 0.0); + + /* Calculate pixel aspect ratio. */ + aspectRatio = K(1, 1) / K(0, 0); + + /* Calculate number of pixel per realworld unit. */ + double mx, my; + if(apertureWidth != 0.0 && apertureHeight != 0.0) { + mx = imageSize.width / apertureWidth; + my = imageSize.height / apertureHeight; + } else { + mx = 1.0; + my = aspectRatio; + } + + /* Calculate fovx and fovy. */ + fovx = 2 * atan(imageSize.width / (2 * K(0, 0))) * 180.0 / CV_PI; + fovy = 2 * atan(imageSize.height / (2 * K(1, 1))) * 180.0 / CV_PI; + + /* Calculate focal length. */ + focalLength = K(0, 0) / mx; + + /* Calculate principle point. */ + principalPoint = Point2d(K(0, 2) / mx, K(1, 2) / my); } double cv::stereoCalibrate( InputArrayOfArrays _objectPoints, From 8ed1945ccd52501f5ab22bdec6aa1f91f1e2cfd4 Mon Sep 17 00:00:00 2001 From: Pavel Rojtberg Date: Wed, 27 Apr 2016 14:47:52 +0200 Subject: [PATCH 017/128] calibrationMatrixValues: consider principalPoint in FOV computation The FOV depends on the principal point location. Use formula of viz::Camera. --- modules/calib3d/src/calibration.cpp | 6 ++++-- modules/calib3d/test/test_cameracalibration.cpp | 4 ++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/modules/calib3d/src/calibration.cpp b/modules/calib3d/src/calibration.cpp index bd759d294b..8ae9bb2c12 100644 --- a/modules/calib3d/src/calibration.cpp +++ b/modules/calib3d/src/calibration.cpp @@ -3356,8 +3356,10 @@ void cv::calibrationMatrixValues( InputArray _cameraMatrix, Size imageSize, } /* Calculate fovx and fovy. */ - fovx = 2 * atan(imageSize.width / (2 * K(0, 0))) * 180.0 / CV_PI; - fovy = 2 * atan(imageSize.height / (2 * K(1, 1))) * 180.0 / CV_PI; + fovx = atan2(K(0, 2), K(0, 0)) + atan2(imageSize.width - K(0, 2), K(0, 0)); + fovy = atan2(K(1, 2), K(1, 1)) + atan2(imageSize.height - K(1, 2), K(1, 1)); + fovx *= 180.0 / CV_PI; + fovy *= 180.0 / CV_PI; /* Calculate focal length. */ focalLength = K(0, 0) / mx; diff --git a/modules/calib3d/test/test_cameracalibration.cpp b/modules/calib3d/test/test_cameracalibration.cpp index 49d73fc616..329a825245 100644 --- a/modules/calib3d/test/test_cameracalibration.cpp +++ b/modules/calib3d/test/test_cameracalibration.cpp @@ -875,8 +875,8 @@ void CV_CalibrationMatrixValuesTest::run(int) ny = goodAspectRatio; } - goodFovx = 2 * atan( imageSize.width / (2 * fx)) * 180.0 / CV_PI; - goodFovy = 2 * atan( imageSize.height / (2 * fy)) * 180.0 / CV_PI; + goodFovx = (atan2(cx, fx) + atan2(imageSize.width - cx, fx)) * 180.0 / CV_PI; + goodFovy = (atan2(cy, fy) + atan2(imageSize.height - cy, fy)) * 180.0 / CV_PI; goodFocalLength = fx / nx; From 3db19c046b376aff6274aeeb69ea751d9f7988d4 Mon Sep 17 00:00:00 2001 From: DozyC Date: Tue, 10 May 2016 17:20:59 -0700 Subject: [PATCH 018/128] Fix command line argument handling, fixes #6525 --- samples/tapi/hog.cpp | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/samples/tapi/hog.cpp b/samples/tapi/hog.cpp index e361c2b735..0529f24a1c 100644 --- a/samples/tapi/hog.cpp +++ b/samples/tapi/hog.cpp @@ -76,10 +76,8 @@ int main(int argc, char** argv) "{ s scale | 1.0 | resize the image before detect}" "{ o output | | specify output path when input is images}"; CommandLineParser cmd(argc, argv, keys); - if (cmd.has("help")) + if (cmd.get("help")) { - cout << "Usage : hog [options]" << endl; - cout << "Available options:" << endl; cmd.printMessage(); return EXIT_SUCCESS; } @@ -117,7 +115,7 @@ App::App(CommandLineParser& cmd) << "\t4/r - increase/decrease hit threshold\n" << endl; - make_gray = cmd.has("gray"); + make_gray = cmd.get("gray"); resize_scale = cmd.get("s"); vdo_source = cmd.get("v"); img_source = cmd.get("i"); From e646f9d2f1b276991a59edf01bc87dcdf28e2b8f Mon Sep 17 00:00:00 2001 From: Han Hu Date: Fri, 20 May 2016 11:45:46 +0800 Subject: [PATCH 019/128] Fix subpixel problem of akaze. This is found in the original akaze repo. Previous sub pixel localization method assumes the coordinate (0, 0) is the up-left corner of the up-left pixel. But as far as I know, opencv uses the center of the up-left corner, in this case it should be done in this way. https://github.com/pablofdezalc/akaze/commit/35aeb83a7142e9ec42419603a094ffc2a1401c05 https://github.com/pablofdezalc/akaze/commit/db3dc22981e856ca8111f2f7fe57d9c2e0286efc --- modules/features2d/src/kaze/AKAZEFeatures.cpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/modules/features2d/src/kaze/AKAZEFeatures.cpp b/modules/features2d/src/kaze/AKAZEFeatures.cpp index 6f1b610ccf..cc317249a7 100644 --- a/modules/features2d/src/kaze/AKAZEFeatures.cpp +++ b/modules/features2d/src/kaze/AKAZEFeatures.cpp @@ -342,14 +342,14 @@ void AKAZEFeatures::Find_Scale_Space_Extrema(std::vector& kpts) if (is_out == false) { if (is_repeated == false) { - point.pt.x *= ratio; - point.pt.y *= ratio; + point.pt.x = (float)(point.pt.x*ratio + .5*(ratio-1.0)); + point.pt.y = (float)(point.pt.y*ratio + .5*(ratio-1.0)); kpts_aux.push_back(point); npoints++; } else { - point.pt.x *= ratio; - point.pt.y *= ratio; + point.pt.x = (float)(point.pt.x*ratio + .5*(ratio-1.0)); + point.pt.y = (float)(point.pt.y*ratio + .5*(ratio-1.0)); kpts_aux[id_repeated] = point; } } // if is_out @@ -439,8 +439,8 @@ void AKAZEFeatures::Do_Subpixel_Refinement(std::vector& kpts) kpts[i].pt.x = x + dst(0); kpts[i].pt.y = y + dst(1); int power = fastpow(2, evolution_[kpts[i].class_id].octave); - kpts[i].pt.x *= power; - kpts[i].pt.y *= power; + kpts[i].pt.x = (float)(kpts[i].pt.x*power + .5*(power-1)); + kpts[i].pt.y = (float)(kpts[i].pt.y*power + .5*(power-1)); kpts[i].angle = 0.0; // In OpenCV the size of a keypoint its the diameter From 6c4aae98f7b7d3f83014204ff7a276b01c99e294 Mon Sep 17 00:00:00 2001 From: DozyC Date: Thu, 19 May 2016 23:20:55 -0700 Subject: [PATCH 020/128] tapi examples - Removing defaults from all command line switches accessed with has() --- samples/tapi/bgfg_segm.cpp | 4 ++-- samples/tapi/clahe.cpp | 2 +- samples/tapi/hog.cpp | 8 ++++---- samples/tapi/pyrlk_optical_flow.cpp | 2 +- samples/tapi/squares.cpp | 4 ++-- samples/tapi/tvl1_optical_flow.cpp | 4 ++-- 6 files changed, 12 insertions(+), 12 deletions(-) diff --git a/samples/tapi/bgfg_segm.cpp b/samples/tapi/bgfg_segm.cpp index 40587713b7..8b944d184a 100644 --- a/samples/tapi/bgfg_segm.cpp +++ b/samples/tapi/bgfg_segm.cpp @@ -17,10 +17,10 @@ using namespace cv; int main(int argc, const char** argv) { CommandLineParser cmd(argc, argv, - "{ c camera | false | use camera }" + "{ c camera | | use camera }" "{ f file | ../data/768x576.avi | input video file }" "{ t type | mog2 | method's type (knn, mog2) }" - "{ h help | false | print help message }" + "{ h help | | print help message }" "{ m cpu_mode | false | press 'm' to switch OpenCL<->CPU}"); if (cmd.has("help")) diff --git a/samples/tapi/clahe.cpp b/samples/tapi/clahe.cpp index 905ea1f1ae..1b6c53b9b3 100644 --- a/samples/tapi/clahe.cpp +++ b/samples/tapi/clahe.cpp @@ -33,7 +33,7 @@ int main(int argc, char** argv) "{ i input | | specify input image }" "{ c camera | 0 | specify camera id }" "{ o output | clahe_output.jpg | specify output save path}" - "{ h help | false | print help message }"; + "{ h help | | print help message }"; cv::CommandLineParser cmd(argc, argv, keys); if (cmd.has("help")) diff --git a/samples/tapi/hog.cpp b/samples/tapi/hog.cpp index 0529f24a1c..db31396106 100644 --- a/samples/tapi/hog.cpp +++ b/samples/tapi/hog.cpp @@ -68,15 +68,15 @@ private: int main(int argc, char** argv) { const char* keys = - "{ h help | false | print help message }" + "{ h help | | print help message }" "{ i input | | specify input image}" "{ c camera | -1 | enable camera capturing }" "{ v video | ../data/768x576.avi | use video as input }" - "{ g gray | false | convert image to gray one or not}" + "{ g gray | | convert image to gray one or not}" "{ s scale | 1.0 | resize the image before detect}" "{ o output | | specify output path when input is images}"; CommandLineParser cmd(argc, argv, keys); - if (cmd.get("help")) + if (cmd.has("help")) { cmd.printMessage(); return EXIT_SUCCESS; @@ -115,7 +115,7 @@ App::App(CommandLineParser& cmd) << "\t4/r - increase/decrease hit threshold\n" << endl; - make_gray = cmd.get("gray"); + make_gray = cmd.has("gray"); resize_scale = cmd.get("s"); vdo_source = cmd.get("v"); img_source = cmd.get("i"); diff --git a/samples/tapi/pyrlk_optical_flow.cpp b/samples/tapi/pyrlk_optical_flow.cpp index 9cdbd7c5bf..bb426cbf76 100644 --- a/samples/tapi/pyrlk_optical_flow.cpp +++ b/samples/tapi/pyrlk_optical_flow.cpp @@ -75,7 +75,7 @@ static void drawArrows(UMat& _frame, const vector& prevPts, const vecto int main(int argc, const char* argv[]) { const char* keys = - "{ h help | false | print help message }" + "{ h help | | print help message }" "{ l left | | specify left image }" "{ r right | | specify right image }" "{ c camera | 0 | enable camera capturing }" diff --git a/samples/tapi/squares.cpp b/samples/tapi/squares.cpp index e18e533d66..7df4a42672 100644 --- a/samples/tapi/squares.cpp +++ b/samples/tapi/squares.cpp @@ -143,8 +143,8 @@ int main(int argc, char** argv) const char* keys = "{ i input | ../data/pic1.png | specify input image }" "{ o output | squares_output.jpg | specify output save path}" - "{ h help | false | print help message }" - "{ m cpu_mode | false | run without OpenCL }"; + "{ h help | | print help message }" + "{ m cpu_mode | | run without OpenCL }"; CommandLineParser cmd(argc, argv, keys); diff --git a/samples/tapi/tvl1_optical_flow.cpp b/samples/tapi/tvl1_optical_flow.cpp index f7bebacbeb..18c7a7f47b 100644 --- a/samples/tapi/tvl1_optical_flow.cpp +++ b/samples/tapi/tvl1_optical_flow.cpp @@ -83,12 +83,12 @@ static void getFlowField(const Mat& u, const Mat& v, Mat& flowField) int main(int argc, const char* argv[]) { const char* keys = - "{ h help | false | print help message }" + "{ h help | | print help message }" "{ l left | | specify left image }" "{ r right | | specify right image }" "{ o output | tvl1_output.jpg | specify output save path }" "{ c camera | 0 | enable camera capturing }" - "{ m cpu_mode | false | run without OpenCL }" + "{ m cpu_mode | | run without OpenCL }" "{ v video | | use video as input }"; CommandLineParser cmd(argc, argv, keys); From b2ad7cd9c018d9acf5e319e3f6a99651c0dca244 Mon Sep 17 00:00:00 2001 From: Tomoaki Teshima Date: Sat, 21 May 2016 21:31:33 +0900 Subject: [PATCH 021/128] add feature to convert FP32(float) to FP16(half) * check compiler support * check HW support before executing * add test doing round trip conversion from / to FP32 * treat array correctly if size is not multiple of 4 * add declaration to prevent warning * make it possible to enable fp16 on 32bit ARM * let the conversion possible on non-supported HW, too. * add test using both HW and SW implementation --- cmake/OpenCVCompilerOptions.cmake | 8 +- modules/core/include/opencv2/core.hpp | 11 + modules/core/include/opencv2/core/cvdef.h | 15 +- modules/core/src/convert.cpp | 359 ++++++++++++++++++++++ modules/core/src/precomp.hpp | 1 + modules/core/src/system.cpp | 12 +- modules/core/test/test_arithm.cpp | 55 ++++ modules/ts/src/ts_func.cpp | 3 + 8 files changed, 459 insertions(+), 5 deletions(-) diff --git a/cmake/OpenCVCompilerOptions.cmake b/cmake/OpenCVCompilerOptions.cmake index 0dcf7ed263..33dd575243 100644 --- a/cmake/OpenCVCompilerOptions.cmake +++ b/cmake/OpenCVCompilerOptions.cmake @@ -146,8 +146,11 @@ if(CMAKE_COMPILER_IS_GNUCXX) elseif(X86 OR X86_64) add_extra_compiler_option(-mno-sse2) endif() + if(ARM) + add_extra_compiler_option("-mfp16-format=ieee") + endif(ARM) if(ENABLE_NEON) - add_extra_compiler_option("-mfpu=neon") + add_extra_compiler_option("-mfpu=neon-fp16") endif() if(ENABLE_VFPV3 AND NOT ENABLE_NEON) add_extra_compiler_option("-mfpu=vfpv3") @@ -167,6 +170,9 @@ if(CMAKE_COMPILER_IS_GNUCXX) add_extra_compiler_option(-mfma) endif() endif() + if((X86 OR X86_64) AND NOT MSVC) + add_extra_compiler_option(-mf16c) + endif((X86 OR X86_64) AND NOT MSVC) # GCC depresses SSEx instructions when -mavx is used. Instead, it generates new AVX instructions or AVX equivalence for all SSEx instructions when needed. if(NOT OPENCV_EXTRA_CXX_FLAGS MATCHES "-mavx") diff --git a/modules/core/include/opencv2/core.hpp b/modules/core/include/opencv2/core.hpp index 0d180f51ad..fa7ab469b2 100644 --- a/modules/core/include/opencv2/core.hpp +++ b/modules/core/include/opencv2/core.hpp @@ -524,6 +524,17 @@ For example: CV_EXPORTS_W void convertScaleAbs(InputArray src, OutputArray dst, double alpha = 1, double beta = 0); +/** @brief Converts an array to half precision floating number. + +convertFp16 converts FP32 to FP16 or FP16 to FP32. The input array has to have type of CV_32F or +CV_16S to represent the bit depth. If the input array is neither of them, it'll do nothing. + +@param src input array. +@param dst output array. +@param useHW if possible use HW SIMD instruction to convert +*/ +CV_EXPORTS_W void convertFp16(InputArray src, OutputArray dst, bool useHW = true); + /** @brief Performs a look-up table transform of an array. The function LUT fills the output array with values from the look-up table. Indices of the entries diff --git a/modules/core/include/opencv2/core/cvdef.h b/modules/core/include/opencv2/core/cvdef.h index c005914190..42e93118b7 100644 --- a/modules/core/include/opencv2/core/cvdef.h +++ b/modules/core/include/opencv2/core/cvdef.h @@ -112,7 +112,7 @@ #define CV_CPU_SSE4_1 6 #define CV_CPU_SSE4_2 7 #define CV_CPU_POPCNT 8 - +#define CV_CPU_FP16 9 #define CV_CPU_AVX 10 #define CV_CPU_AVX2 11 #define CV_CPU_FMA3 12 @@ -143,7 +143,7 @@ enum CpuFeatures { CPU_SSE4_1 = 6, CPU_SSE4_2 = 7, CPU_POPCNT = 8, - + CPU_FP16 = 9, CPU_AVX = 10, CPU_AVX2 = 11, CPU_FMA3 = 12, @@ -193,6 +193,10 @@ enum CpuFeatures { # endif # define CV_POPCNT 1 # endif +# if defined __F16C__ || (defined _MSC_VER && _MSC_VER >= 1700) +# include +# define CV_FP16 1 +# endif # if defined __AVX__ || (defined _MSC_VER && _MSC_VER >= 1600 && 0) // MS Visual Studio 2010 (2012?) has no macro pre-defined to identify the use of /arch:AVX // See: http://connect.microsoft.com/VisualStudio/feedback/details/605858/arch-avx-should-define-a-predefined-macro-in-x64-and-set-a-unique-value-for-m-ix86-fp-in-win32 @@ -223,6 +227,10 @@ enum CpuFeatures { # define CV_NEON 1 #endif +#if defined __GNUC__ && ((defined (__arm__) && (__ARM_FP & 0x2)) || defined(__aarch64__)) +# define CV_FP16 1 +#endif + #if defined __GNUC__ && defined __arm__ && (defined __ARM_PCS_VFP || defined __ARM_VFPV3__ || defined __ARM_NEON__) && !defined __SOFTFP__ # define CV_VFP 1 #endif @@ -253,6 +261,9 @@ enum CpuFeatures { #ifndef CV_SSE4_2 # define CV_SSE4_2 0 #endif +#ifndef CV_FP16 +# define CV_FP16 0 +#endif #ifndef CV_AVX # define CV_AVX 0 #endif diff --git a/modules/core/src/convert.cpp b/modules/core/src/convert.cpp index f41bfa105f..4ff9830db8 100644 --- a/modules/core/src/convert.cpp +++ b/modules/core/src/convert.cpp @@ -4356,6 +4356,283 @@ struct Cvt_SIMD #endif +#if !(defined (__arm__) || defined (__aarch64__)) +// const numbers for floating points format +const unsigned int kShiftSignificand = 13; +const unsigned int kMaskFp16Significand = 0x3ff; +const unsigned int kBiasFp16Exponent = 15; +const unsigned int kBiasFp32Exponent = 127; + +union fp32Int32 +{ + int i; + float f; + struct _fp32Format + { + unsigned int significand : 23; + unsigned int exponent : 8; + unsigned int sign : 1; + } fmt; +}; +#endif + +union fp16Int16 +{ + short i; +#if defined (__arm__) || defined (__aarch64__) + __fp16 h; +#endif + struct _fp16Format + { + unsigned int significand : 10; + unsigned int exponent : 5; + unsigned int sign : 1; + } fmt; +}; + +#if defined (__arm__) || defined (__aarch64__) +static float convertFp16SW(short fp16) +{ + // Fp16 -> Fp32 + fp16Int16 a; + a.i = fp16; + return (float)a.h; +} +#else +static float convertFp16SW(short fp16) +{ + // Fp16 -> Fp32 + fp16Int16 b; + b.i = fp16; + int exponent = b.fmt.exponent - kBiasFp16Exponent; + int significand = b.fmt.significand; + + fp32Int32 a; + a.i = 0; + a.fmt.sign = b.fmt.sign; // sign bit + if( exponent == 16 ) + { + // Inf or NaN + a.i = a.i | 0x7F800000; + if( significand != 0 ) + { + // NaN +#if defined(__x86_64__) || defined(_M_X64) + // 64bit + a.i = a.i | 0x7FC00000; +#endif + a.fmt.significand = a.fmt.significand | (significand << kShiftSignificand); + } + return a.f; + } + else if ( exponent == -15 ) + { + // subnormal in Fp16 + if( significand == 0 ) + { + // zero + return a.f; + } + else + { + int shift = -1; + while( ( significand & 0x400 ) == 0 ) + { + significand = significand << 1; + shift++; + } + significand = significand & kMaskFp16Significand; + exponent -= shift; + } + } + + a.fmt.exponent = (exponent+kBiasFp32Exponent); + a.fmt.significand = significand << kShiftSignificand; + return a.f; +} +#endif + +#if defined (__arm__) || defined (__aarch64__) +static short convertFp16SW(float fp32) +{ + // Fp32 -> Fp16 + fp16Int16 a; + a.h = (__fp16)fp32; + return a.i; +} +#else +static short convertFp16SW(float fp32) +{ + // Fp32 -> Fp16 + fp32Int32 a; + a.f = fp32; + int exponent = a.fmt.exponent - kBiasFp32Exponent; + int significand = a.fmt.significand; + + fp16Int16 result; + result.i = 0; + if( 0x477ff000 <= ( a.i & 0x7fffffff ) ) + { + // Inf in Fp16 + result.i = result.i | 0x7C00; + if( exponent == 128 && significand != 0 ) + { + // NaN + result.i = (short)(result.i | 0x200 | (significand >> kShiftSignificand)); + } + } + else if ( ( a.i & 0x7fffffff ) <= 0x387fe000 ) + { + // subnormal in Fp16 + int fp16Significand = significand | 0x800000; + int bitShift = (-exponent) - 1; + fp16Significand = fp16Significand >> bitShift; + + // special cases to round up + int threshold = 0x8000 + ( ( fp16Significand & 1 ) ? 0 : 1 ); + if( threshold <= ( significand & 0xffff ) ) + { + fp16Significand++; + } + result.i = (short)fp16Significand; + } + else + { + // usual situation + // exponent + result.fmt.exponent = (exponent + kBiasFp16Exponent); + + // significand; + short fp16Significand = (short)(significand >> kShiftSignificand); + result.fmt.significand = fp16Significand; + + // special cases to round up + short lsb10bitsFp32 = (significand & 0x1fff); + short threshold = 0x1000 + ( ( fp16Significand & 0x1 ) ? 0 : 1 ); + if( threshold <= lsb10bitsFp32 ) + { + result.i++; + } + else if ( fp16Significand == 0x3ff && exponent == -15) + { + result.i++; + } + } + + // sign bit + result.fmt.sign = a.fmt.sign; + return result.i; +} +#endif + +template static void +cvtScaleHalfSW_( const T* src, size_t sstep, DT* dst, size_t dstep, Size size) +{ + sstep /= sizeof(src[0]); + dstep /= sizeof(dst[0]); + + for( ; size.height--; src += sstep, dst += dstep ) + { + for ( int x = 0 ; x < size.width; x ++ ) + { + dst[x] = convertFp16SW(src[x]); + } + } +} + +// template for FP16 HW conversion function +template static void +cvtScaleHalfHW_( const T* src, size_t sstep, DT* dst, size_t dstep, Size size) +{ + sstep /= sizeof(src[0]); + dstep /= sizeof(dst[0]); + + for( ; size.height--; src += sstep, dst += dstep ) + { + int x = 0; + + for ( ; x < size.width; x++ ) + { + } + } +} + +template<> void +cvtScaleHalfHW_( const float* src, size_t sstep, short* dst, size_t dstep, Size size) +{ + sstep /= sizeof(src[0]); + dstep /= sizeof(dst[0]); + + for( ; size.height--; src += sstep, dst += dstep ) + { + int x = 0; + + if ( ( (intptr_t)dst & 0xf ) == 0 && ( (intptr_t)src & 0xf ) == 0 ) + { +#if CV_FP16 + for ( ; x <= size.width - 4; x += 4) + { +#if defined(__x86_64__) || defined(_M_X64) || defined(_M_IX86) || defined(i386) + __m128 v_src = _mm_load_ps(src + x); + + __m128i v_dst = _mm_cvtps_ph(v_src, 0); + + _mm_storel_epi64((__m128i *)(dst + x), v_dst); +#elif defined __GNUC__ && (defined __arm__ || defined __aarch64__) + float32x4_t v_src = *(float32x4_t*)(src + x); + + float16x4_t v_dst = vcvt_f16_f32(v_src); + + *(float16x4_t*)(dst + x) = v_dst; +#endif + } +#endif + } + for ( ; x < size.width; x++ ) + { + dst[x] = convertFp16SW(src[x]); + } + } +} + +template<> void +cvtScaleHalfHW_( const short* src, size_t sstep, float* dst, size_t dstep, Size size) +{ + sstep /= sizeof(src[0]); + dstep /= sizeof(dst[0]); + + for( ; size.height--; src += sstep, dst += dstep ) + { + int x = 0; + + if ( ( (intptr_t)dst & 0xf ) == 0 && ( (intptr_t)src & 0xf ) == 0 ) + { +#if CV_FP16 + for ( ; x <= size.width - 4; x += 4) + { +#if defined(__x86_64__) || defined(_M_X64) || defined(_M_IX86) || defined(i386) + __m128i v_src = _mm_loadl_epi64((__m128i*)(src+x)); + + __m128 v_dst = _mm_cvtph_ps(v_src); + + _mm_store_ps((dst + x), v_dst); +#elif defined __GNUC__ && (defined __arm__ || defined __aarch64__) + float16x4_t v_src = *(float16x4_t*)(src + x); + + float32x4_t v_dst = vcvt_f32_f16(v_src); + + *(float32x4_t*)(dst + x) = v_dst; +#endif + } +#endif + } + for ( ; x < size.width; x++ ) + { + dst[x] = convertFp16SW(src[x]); + } + } +} + template static void cvt_( const T* src, size_t sstep, DT* dst, size_t dstep, Size size ) @@ -4443,6 +4720,13 @@ static void cvtScaleAbs##suffix( const stype* src, size_t sstep, const uchar*, s tfunc(src, sstep, dst, dstep, size, (wtype)scale[0], (wtype)scale[1]); \ } +#define DEF_CVT_SCALE_FP16_FUNC(suffix, stype, dtype, resource) \ +static void cvtScaleHalf##suffix##resource( const stype* src, size_t sstep, const uchar*, size_t, \ +dtype* dst, size_t dstep, Size size, double*) \ +{ \ + cvtScaleHalf##resource##_(src, sstep, dst, dstep, size); \ +} + #define DEF_CVT_SCALE_FUNC(suffix, stype, dtype, wtype) \ static void cvtScale##suffix( const stype* src, size_t sstep, const uchar*, size_t, \ dtype* dst, size_t dstep, Size size, double* scale) \ @@ -4499,6 +4783,11 @@ DEF_CVT_SCALE_ABS_FUNC(32s8u, cvtScaleAbs_, int, uchar, float) DEF_CVT_SCALE_ABS_FUNC(32f8u, cvtScaleAbs_, float, uchar, float) DEF_CVT_SCALE_ABS_FUNC(64f8u, cvtScaleAbs_, double, uchar, float) +DEF_CVT_SCALE_FP16_FUNC(32f16f, float, short, SW) +DEF_CVT_SCALE_FP16_FUNC(16f32f, short, float, SW) +DEF_CVT_SCALE_FP16_FUNC(32f16f, float, short, HW) +DEF_CVT_SCALE_FP16_FUNC(16f32f, short, float, HW) + DEF_CVT_SCALE_FUNC(8u, uchar, uchar, float) DEF_CVT_SCALE_FUNC(8s8u, schar, uchar, float) DEF_CVT_SCALE_FUNC(16u8u, ushort, uchar, float) @@ -4620,6 +4909,30 @@ static BinaryFunc getCvtScaleAbsFunc(int depth) return cvtScaleAbsTab[depth]; } +BinaryFunc getConvertFuncFp16(int ddepth, bool useHW) +{ + static BinaryFunc cvtTabHW[] = + { + 0, 0, 0, + (BinaryFunc)(cvtScaleHalf32f16fHW), 0, (BinaryFunc)(cvtScaleHalf16f32fHW), + 0, 0, + }; + static BinaryFunc cvtTabSW[] = + { + 0, 0, 0, + (BinaryFunc)(cvtScaleHalf32f16fSW), 0, (BinaryFunc)(cvtScaleHalf16f32fSW), + 0, 0, + }; + if( useHW == true) + { + return cvtTabHW[CV_MAT_DEPTH(ddepth)]; + } + else + { + return cvtTabSW[CV_MAT_DEPTH(ddepth)]; + } +} + BinaryFunc getConvertFunc(int sdepth, int ddepth) { static BinaryFunc cvtTab[][8] = @@ -4804,6 +5117,52 @@ void cv::convertScaleAbs( InputArray _src, OutputArray _dst, double alpha, doubl } } +void cv::convertFp16( InputArray _src, OutputArray _dst, bool useHW ) +{ + if ( checkHardwareSupport(CV_CPU_FP16) == false) + { + useHW = false; + } + + Mat src = _src.getMat(); + int ddepth = 0; + + switch( src.depth() ) + { + case CV_32F: + ddepth = CV_16S; + break; + case CV_16S: + ddepth = CV_32F; + break; + default: + return; + } + + int type = CV_MAKETYPE(ddepth, src.channels()); + _dst.create( src.dims, src.size, type ); + Mat dst = _dst.getMat(); + BinaryFunc func = getConvertFuncFp16(ddepth, useHW); + int cn = src.channels(); + CV_Assert( func != 0 ); + + if( src.dims <= 2 ) + { + Size sz = getContinuousSize(src, dst, cn); + func( src.data, src.step, 0, 0, dst.data, dst.step, sz, 0); + } + else + { + const Mat* arrays[] = {&src, &dst, 0}; + uchar* ptrs[2]; + NAryMatIterator it(arrays, ptrs); + Size sz((int)(it.size*cn), 1); + + for( size_t i = 0; i < it.nplanes; i++, ++it ) + func(ptrs[0], 1, 0, 0, ptrs[1], 1, sz, 0); + } +} + void cv::Mat::convertTo(OutputArray _dst, int _type, double alpha, double beta) const { bool noScale = fabs(alpha-1) < DBL_EPSILON && fabs(beta) < DBL_EPSILON; diff --git a/modules/core/src/precomp.hpp b/modules/core/src/precomp.hpp index f699ede392..cece96cb08 100644 --- a/modules/core/src/precomp.hpp +++ b/modules/core/src/precomp.hpp @@ -135,6 +135,7 @@ typedef void (*BinaryFuncC)(const uchar* src1, size_t step1, uchar* dst, size_t step, int width, int height, void*); +BinaryFunc getConvertFuncFp16(int ddepth, bool useHW); BinaryFunc getConvertFunc(int sdepth, int ddepth); BinaryFunc getCopyMaskFunc(size_t esz); diff --git a/modules/core/src/system.cpp b/modules/core/src/system.cpp index 768280e1a6..a3858c1d19 100644 --- a/modules/core/src/system.cpp +++ b/modules/core/src/system.cpp @@ -291,6 +291,7 @@ struct HWFeatures f.have[CV_CPU_SSE4_2] = (cpuid_data[2] & (1<<20)) != 0; f.have[CV_CPU_POPCNT] = (cpuid_data[2] & (1<<23)) != 0; f.have[CV_CPU_AVX] = (((cpuid_data[2] & (1<<28)) != 0)&&((cpuid_data[2] & (1<<27)) != 0));//OS uses XSAVE_XRSTORE and CPU support AVX + f.have[CV_CPU_FP16] = (cpuid_data[2] & (1<<29)) != 0; // make the second call to the cpuid command in order to get // information about extended features like AVX2 @@ -338,7 +339,8 @@ struct HWFeatures #if defined ANDROID || defined __linux__ #ifdef __aarch64__ f.have[CV_CPU_NEON] = true; - #else + f.have[CV_CPU_FP16] = true; + #elif defined __arm__ int cpufile = open("/proc/self/auxv", O_RDONLY); if (cpufile >= 0) @@ -351,6 +353,7 @@ struct HWFeatures if (auxv.a_type == AT_HWCAP) { f.have[CV_CPU_NEON] = (auxv.a_un.a_val & 4096) != 0; + f.have[CV_CPU_FP16] = (auxv.a_un.a_val & 2) != 0; break; } } @@ -358,8 +361,13 @@ struct HWFeatures close(cpufile); } #endif - #elif (defined __clang__ || defined __APPLE__) && (defined __ARM_NEON__ || (defined __ARM_NEON && defined __aarch64__)) + #elif (defined __clang__ || defined __APPLE__) + #if (defined __ARM_NEON__ || (defined __ARM_NEON && defined __aarch64__)) f.have[CV_CPU_NEON] = true; + #endif + #if (defined __ARM_FP && (((__ARM_FP & 0x2) != 0) && defined __ARM_NEON__)) + f.have[CV_CPU_FP16] = true; + #endif #endif return f; diff --git a/modules/core/test/test_arithm.cpp b/modules/core/test/test_arithm.cpp index ace7950a64..58974a8f5c 100644 --- a/modules/core/test/test_arithm.cpp +++ b/modules/core/test/test_arithm.cpp @@ -737,6 +737,60 @@ struct ConvertScaleOp : public BaseElemWiseOp int ddepth; }; +struct ConvertScaleFp16Op : public BaseElemWiseOp +{ + ConvertScaleFp16Op() : BaseElemWiseOp(1, FIX_BETA+REAL_GAMMA, 1, 1, Scalar::all(0)), nextRange(0) { } + void op(const vector& src, Mat& dst, const Mat&) + { + convertFp16(src[0], dst, true); + } + void refop(const vector& src, Mat& dst, const Mat&) + { + convertFp16(src[0], dst, false); + } + int getRandomType(RNG&) + { + // 0: FP32 -> FP16 + // 1: FP16 -> FP32 + int srctype = (nextRange & 1) == 0 ? CV_32F : CV_16S; + return srctype; + } + void getValueRange(int, double& minval, double& maxval) + { + // 0: FP32 -> FP16 + // 1: FP16 -> FP32 + if( (nextRange & 1) == 0 ) + { + // largest integer number that fp16 can express + maxval = 65504.f; + minval = -maxval; + } + else + { + // 0: positive number range + // 1: negative number range + if( (nextRange & 2) == 0 ) + { + minval = 0; // 0x0000 +0 + maxval = 31744; // 0x7C00 +Inf + } + else + { + minval = -32768; // 0x8000 -0 + maxval = -1024; // 0xFC00 -Inf + } + } + } + double getMaxErr(int) + { + return 0.5f; + } + void generateScalars(int, RNG& rng) + { + nextRange = rng.next(); + } + int nextRange; +}; struct ConvertScaleAbsOp : public BaseElemWiseOp { @@ -1371,6 +1425,7 @@ INSTANTIATE_TEST_CASE_P(Core_Copy, ElemWiseTest, ::testing::Values(ElemWiseOpPtr INSTANTIATE_TEST_CASE_P(Core_Set, ElemWiseTest, ::testing::Values(ElemWiseOpPtr(new cvtest::SetOp))); INSTANTIATE_TEST_CASE_P(Core_SetZero, ElemWiseTest, ::testing::Values(ElemWiseOpPtr(new cvtest::SetZeroOp))); INSTANTIATE_TEST_CASE_P(Core_ConvertScale, ElemWiseTest, ::testing::Values(ElemWiseOpPtr(new cvtest::ConvertScaleOp))); +INSTANTIATE_TEST_CASE_P(Core_ConvertScaleFp16, ElemWiseTest, ::testing::Values(ElemWiseOpPtr(new cvtest::ConvertScaleFp16Op))); INSTANTIATE_TEST_CASE_P(Core_ConvertScaleAbs, ElemWiseTest, ::testing::Values(ElemWiseOpPtr(new cvtest::ConvertScaleAbsOp))); INSTANTIATE_TEST_CASE_P(Core_Add, ElemWiseTest, ::testing::Values(ElemWiseOpPtr(new cvtest::AddOp))); diff --git a/modules/ts/src/ts_func.cpp b/modules/ts/src/ts_func.cpp index ca7664cce9..a8f146031e 100644 --- a/modules/ts/src/ts_func.cpp +++ b/modules/ts/src/ts_func.cpp @@ -3064,6 +3064,9 @@ void printVersionInfo(bool useStdOut) #if CV_NEON if (checkHardwareSupport(CV_CPU_NEON)) cpu_features += " neon"; #endif +#if CV_FP16 + if (checkHardwareSupport(CV_CPU_FP16)) cpu_features += " fp16"; +#endif cpu_features.erase(0, 1); // erase initial space From ef450050561b9107f3cb7eff57e27917dc4165dc Mon Sep 17 00:00:00 2001 From: Marek Smigielski Date: Tue, 31 May 2016 08:35:50 +0200 Subject: [PATCH 022/128] Adding support for pointer generation. Fixes #6605 --- modules/java/generator/gen_java.py | 14 +++++++------- modules/ml/include/opencv2/ml.hpp | 10 +++++----- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/modules/java/generator/gen_java.py b/modules/java/generator/gen_java.py index d4560a972e..f8d3f7d0a5 100755 --- a/modules/java/generator/gen_java.py +++ b/modules/java/generator/gen_java.py @@ -991,12 +991,12 @@ class JavaWrapperGenerator(object): if classinfo.base: classinfo.addImports(classinfo.base) - type_dict["Ptr_"+name] = \ - { "j_type" : name, - "jn_type" : "long", "jn_args" : (("__int64", ".nativeObj"),), - "jni_name" : "Ptr<"+name+">(("+name+"*)%(n)s_nativeObj)", "jni_type" : "jlong", - "suffix" : "J" } - logging.info('ok: %s', classinfo) + type_dict["Ptr_"+name] = \ + { "j_type" : name, + "jn_type" : "long", "jn_args" : (("__int64", ".nativeObj"),), + "jni_name" : "Ptr<"+name+">(("+classinfo.fullName(isCPP=True)+"*)%(n)s_nativeObj)", "jni_type" : "jlong", + "suffix" : "J" } + logging.info('ok: class %s, name: %s, base: %s', classinfo, name, classinfo.base) def add_const(self, decl): # [ "const cname", val, [], [] ] constinfo = ConstInfo(decl, namespaces=self.namespaces) @@ -1347,7 +1347,7 @@ class JavaWrapperGenerator(object): ret = "return (jlong) new %s(_retval_);" % self.fullTypeName(fi.ctype) elif fi.ctype.startswith('Ptr_'): c_prologue.append("typedef Ptr<%s> %s;" % (self.fullTypeName(fi.ctype[4:]), fi.ctype)) - ret = "return (jlong)(new %(ctype)s(_retval_));" % { 'ctype':fi.ctype } + ret = "%(ctype)s* curval = new %(ctype)s(_retval_);return (jlong)curval->get();" % { 'ctype':fi.ctype } elif self.isWrapped(ret_type): # pointer to wrapped class: ret = "return (jlong) _retval_;" elif type_dict[fi.ctype]["jni_type"] == "jdoubleArray": diff --git a/modules/ml/include/opencv2/ml.hpp b/modules/ml/include/opencv2/ml.hpp index cea8aec48c..d016810874 100644 --- a/modules/ml/include/opencv2/ml.hpp +++ b/modules/ml/include/opencv2/ml.hpp @@ -285,7 +285,7 @@ public: `, containing types of each input and output variable. See ml::VariableTypes. */ - CV_WRAP static Ptr create(InputArray samples, int layout, InputArray responses, + CV_WRAP static Ptr create(InputArray samples, int layout, InputArray responses, InputArray varIdx=noArray(), InputArray sampleIdx=noArray(), InputArray sampleWeights=noArray(), InputArray varType=noArray()); }; @@ -320,7 +320,7 @@ public: @param flags optional flags, depending on the model. Some of the models can be updated with the new training samples, not completely overwritten (such as NormalBayesClassifier or ANN_MLP). */ - CV_WRAP virtual bool train( const Ptr& trainData, int flags=0 ); + CV_WRAP virtual bool train( const Ptr& trainData, int flags=0 ); /** @brief Trains the statistical model @@ -343,7 +343,7 @@ public: The method uses StatModel::predict to compute the error. For regression models the error is computed as RMS, for classifiers - as a percent of missclassified samples (0%-100%). */ - CV_WRAP virtual float calcError( const Ptr& data, bool test, OutputArray resp ) const; + CV_WRAP virtual float calcError( const Ptr& data, bool test, OutputArray resp ) const; /** @brief Predicts response(s) for the provided sample(s) @@ -357,7 +357,7 @@ public: The class must implement static `create()` method with no parameters or with all default parameter values */ - template static Ptr<_Tp> train(const Ptr& data, int flags=0) + template static Ptr<_Tp> train(const Ptr& data, int flags=0) { Ptr<_Tp> model = _Tp::create(); return !model.empty() && model->train(data, flags) ? model : Ptr<_Tp>(); @@ -667,7 +667,7 @@ public: regression (SVM::EPS_SVR or SVM::NU_SVR). If it is SVM::ONE_CLASS, no optimization is made and the usual %SVM with parameters specified in params is executed. */ - virtual bool trainAuto( const Ptr& data, int kFold = 10, + virtual bool trainAuto( const Ptr& data, int kFold = 10, ParamGrid Cgrid = SVM::getDefaultGrid(SVM::C), ParamGrid gammaGrid = SVM::getDefaultGrid(SVM::GAMMA), ParamGrid pGrid = SVM::getDefaultGrid(SVM::P), From 98dce911ca46b192423d595e63c272a6488b6810 Mon Sep 17 00:00:00 2001 From: Susmit Date: Thu, 2 Jun 2016 01:06:17 +0530 Subject: [PATCH 023/128] Update py_calibration.markdown In the camera calibration code { cv2.cornerSubPix() } will be of no use.In the updated code it is assigned to the (corners2) variable which is passed down to { cv2.drawChessboardCorners() } --- .../py_calib3d/py_calibration/py_calibration.markdown | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/py_tutorials/py_calib3d/py_calibration/py_calibration.markdown b/doc/py_tutorials/py_calib3d/py_calibration/py_calibration.markdown index 9c6c1fb643..1e22cedfb0 100644 --- a/doc/py_tutorials/py_calib3d/py_calibration/py_calibration.markdown +++ b/doc/py_tutorials/py_calib3d/py_calibration/py_calibration.markdown @@ -130,11 +130,11 @@ for fname in images: if ret == True: objpoints.append(objp) - cv2.cornerSubPix(gray,corners, (11,11), (-1,-1), criteria) + corners2=cv2.cornerSubPix(gray,corners, (11,11), (-1,-1), criteria) imgpoints.append(corners) # Draw and display the corners - cv2.drawChessboardCorners(img, (7,6), corners, ret) + cv2.drawChessboardCorners(img, (7,6), corners2, ret) cv2.imshow('img', img) cv2.waitKey(500) From cf0df733dad004df3505866a67c1b62474d8cbf0 Mon Sep 17 00:00:00 2001 From: Aleksandar Atanasov Date: Thu, 2 Jun 2016 10:58:46 +0200 Subject: [PATCH 024/128] Fix houghcircles.py when no circles found In the C++ equivalent of this example a check is made whether the vector (here in Python we have a list) actually has any circles in it that is whether the Hough circles function has managed to find any in the given image. This check is missing for the Python example and if no circles are found the application breaks. --- samples/python/houghcircles.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/samples/python/houghcircles.py b/samples/python/houghcircles.py index 5386fc2102..41d42eb1e1 100755 --- a/samples/python/houghcircles.py +++ b/samples/python/houghcircles.py @@ -29,10 +29,12 @@ if __name__ == '__main__': cimg = src.copy() # numpy function circles = cv2.HoughCircles(img, cv2.HOUGH_GRADIENT, 1, 10, np.array([]), 100, 30, 1, 30) - a, b, c = circles.shape - for i in range(b): - cv2.circle(cimg, (circles[0][i][0], circles[0][i][1]), circles[0][i][2], (0, 0, 255), 3, cv2.LINE_AA) - cv2.circle(cimg, (circles[0][i][0], circles[0][i][1]), 2, (0, 255, 0), 3, cv2.LINE_AA) # draw center of circle + + if circles != None: # Check if circles have been found and only then iterate over these and add them to the image + a, b, c = circles.shape + for i in range(b): + cv2.circle(cimg, (circles[0][i][0], circles[0][i][1]), circles[0][i][2], (0, 0, 255), 3, cv2.LINE_AA) + cv2.circle(cimg, (circles[0][i][0], circles[0][i][1]), 2, (0, 255, 0), 3, cv2.LINE_AA) # draw center of circle cv2.imshow("source", src) cv2.imshow("detected circles", cimg) From 14deab252bc50f2f99d0944bce3d80b04450ea1b Mon Sep 17 00:00:00 2001 From: Aleksandar Atanasov Date: Thu, 2 Jun 2016 11:00:23 +0200 Subject: [PATCH 025/128] Fix houghlines.py when no lines found In the C++ equivalent of this example a check is made whether the vector (here in Python we have a list) actually has any lines in it that is whether the Hough lines function has managed to find any in the given image. This check is missing for the Python example and if no lines are found the application breaks. --- samples/python/houghlines.py | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/samples/python/houghlines.py b/samples/python/houghlines.py index 120d8bbaae..98c3640147 100755 --- a/samples/python/houghlines.py +++ b/samples/python/houghlines.py @@ -36,16 +36,17 @@ if __name__ == '__main__': else: # HoughLines lines = cv2.HoughLines(dst, 1, math.pi/180.0, 50, np.array([]), 0, 0) - a,b,c = lines.shape - for i in range(a): - rho = lines[i][0][0] - theta = lines[i][0][1] - a = math.cos(theta) - b = math.sin(theta) - x0, y0 = a*rho, b*rho - pt1 = ( int(x0+1000*(-b)), int(y0+1000*(a)) ) - pt2 = ( int(x0-1000*(-b)), int(y0-1000*(a)) ) - cv2.line(cdst, pt1, pt2, (0, 0, 255), 3, cv2.LINE_AA) + if lines != None: + a,b,c = lines.shape + for i in range(a): + rho = lines[i][0][0] + theta = lines[i][0][1] + a = math.cos(theta) + b = math.sin(theta) + x0, y0 = a*rho, b*rho + pt1 = ( int(x0+1000*(-b)), int(y0+1000*(a)) ) + pt2 = ( int(x0-1000*(-b)), int(y0-1000*(a)) ) + cv2.line(cdst, pt1, pt2, (0, 0, 255), 3, cv2.LINE_AA) cv2.imshow("source", src) cv2.imshow("detected lines", cdst) From d913463932440b7ba56b4bf527887b509b5ca71a Mon Sep 17 00:00:00 2001 From: Susmit Date: Thu, 2 Jun 2016 17:47:45 +0530 Subject: [PATCH 026/128] Terrible bugs in the tutorial code in py_pose.markdown There were two bugs that were solved here.Changes were done after extreme testing. 1.replaced cv2.solvePnPRansac() with cv2.solvePnP() previous fc was giving terrible errors. 2.The code was incapable of saving edited pics;Now fixed with little code mods. --- doc/py_tutorials/py_calib3d/py_pose/py_pose.markdown | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/py_tutorials/py_calib3d/py_pose/py_pose.markdown b/doc/py_tutorials/py_calib3d/py_pose/py_pose.markdown index f0d48265c9..0ec22c6297 100644 --- a/doc/py_tutorials/py_calib3d/py_pose/py_pose.markdown +++ b/doc/py_tutorials/py_calib3d/py_pose/py_pose.markdown @@ -70,15 +70,15 @@ for fname in glob.glob('left*.jpg'): corners2 = cv2.cornerSubPix(gray,corners,(11,11),(-1,-1),criteria) # Find the rotation and translation vectors. - rvecs, tvecs, inliers = cv2.solvePnPRansac(objp, corners2, mtx, dist) + ret,rvecs, tvecs, inliers = cv2.solvePnP(objp, corners2, mtx, dist) # project 3D points to image plane imgpts, jac = cv2.projectPoints(axis, rvecs, tvecs, mtx, dist) img = draw(img,corners2,imgpts) cv2.imshow('img',img) - k = cv2.waitKey(0) & 0xff - if k == 's': + k = cv2.waitKey(0) & 0xFF + if k == ord('s'): cv2.imwrite(fname[:6]+'.png', img) cv2.destroyAllWindows() From 7c5b981c17d0308e84e8a5fe86d87c1ac43b1e2b Mon Sep 17 00:00:00 2001 From: Suleyman TURKMEN Date: Sun, 5 Jun 2016 01:06:55 +0300 Subject: [PATCH 027/128] Update drawing.cpp --- modules/imgproc/src/drawing.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/imgproc/src/drawing.cpp b/modules/imgproc/src/drawing.cpp index 4396e0a5ab..a73a30f9dc 100644 --- a/modules/imgproc/src/drawing.cpp +++ b/modules/imgproc/src/drawing.cpp @@ -1813,7 +1813,7 @@ void circle( InputOutputArray _img, Point center, int radius, double buf[4]; scalarToRawData(color, buf, img.type(), 0); - if( thickness > 1 || line_type >= CV_AA || shift > 0 ) + if( thickness > 1 || line_type != LINE_8 || shift > 0 ) { center.x <<= XY_SHIFT - shift; center.y <<= XY_SHIFT - shift; From 43d5988df6731f7753ecc58b64fa6ce498c3b05e Mon Sep 17 00:00:00 2001 From: k-shinotsuka Date: Sat, 4 Jun 2016 15:20:22 +0900 Subject: [PATCH 028/128] improve to calculate norm --- modules/imgproc/src/canny.cpp | 28 ++++++++++++---------------- 1 file changed, 12 insertions(+), 16 deletions(-) diff --git a/modules/imgproc/src/canny.cpp b/modules/imgproc/src/canny.cpp index 18cdbd73ee..53fb66bbbe 100644 --- a/modules/imgproc/src/canny.cpp +++ b/modules/imgproc/src/canny.cpp @@ -408,14 +408,12 @@ public: __m128i v_dx = _mm_loadu_si128((const __m128i *)(_dx + j)); __m128i v_dy = _mm_loadu_si128((const __m128i *)(_dy + j)); - __m128i v_dx_ml = _mm_mullo_epi16(v_dx, v_dx), v_dx_mh = _mm_mulhi_epi16(v_dx, v_dx); - __m128i v_dy_ml = _mm_mullo_epi16(v_dy, v_dy), v_dy_mh = _mm_mulhi_epi16(v_dy, v_dy); - - __m128i v_norm = _mm_add_epi32(_mm_unpacklo_epi16(v_dx_ml, v_dx_mh), _mm_unpacklo_epi16(v_dy_ml, v_dy_mh)); - _mm_storeu_si128((__m128i *)(_norm + j), v_norm); - - v_norm = _mm_add_epi32(_mm_unpackhi_epi16(v_dx_ml, v_dx_mh), _mm_unpackhi_epi16(v_dy_ml, v_dy_mh)); - _mm_storeu_si128((__m128i *)(_norm + j + 4), v_norm); + __m128i v_dx_dy_ml = _mm_unpacklo_epi16(v_dx, v_dy); + __m128i v_dx_dy_mh = _mm_unpackhi_epi16(v_dx, v_dy); + __m128i v_norm_ml = _mm_madd_epi16(v_dx_dy_ml, v_dx_dy_ml); + __m128i v_norm_mh = _mm_madd_epi16(v_dx_dy_mh, v_dx_dy_mh); + _mm_storeu_si128((__m128i *)(_norm + j), v_norm_ml); + _mm_storeu_si128((__m128i *)(_norm + j + 4), v_norm_mh); } } #elif CV_NEON @@ -799,14 +797,12 @@ while (borderPeaks.try_pop(m)) __m128i v_dx = _mm_loadu_si128((const __m128i *)(_dx + j)); __m128i v_dy = _mm_loadu_si128((const __m128i *)(_dy + j)); - __m128i v_dx_ml = _mm_mullo_epi16(v_dx, v_dx), v_dx_mh = _mm_mulhi_epi16(v_dx, v_dx); - __m128i v_dy_ml = _mm_mullo_epi16(v_dy, v_dy), v_dy_mh = _mm_mulhi_epi16(v_dy, v_dy); - - __m128i v_norm = _mm_add_epi32(_mm_unpacklo_epi16(v_dx_ml, v_dx_mh), _mm_unpacklo_epi16(v_dy_ml, v_dy_mh)); - _mm_storeu_si128((__m128i *)(_norm + j), v_norm); - - v_norm = _mm_add_epi32(_mm_unpackhi_epi16(v_dx_ml, v_dx_mh), _mm_unpackhi_epi16(v_dy_ml, v_dy_mh)); - _mm_storeu_si128((__m128i *)(_norm + j + 4), v_norm); + __m128i v_dx_dy_ml = _mm_unpacklo_epi16(v_dx, v_dy); + __m128i v_dx_dy_mh = _mm_unpackhi_epi16(v_dx, v_dy); + __m128i v_norm_ml = _mm_madd_epi16(v_dx_dy_ml, v_dx_dy_ml); + __m128i v_norm_mh = _mm_madd_epi16(v_dx_dy_mh, v_dx_dy_mh); + _mm_storeu_si128((__m128i *)(_norm + j), v_norm_ml); + _mm_storeu_si128((__m128i *)(_norm + j + 4), v_norm_mh); } } #elif CV_NEON From eccf2fa4c3a48832c4b3855fea1674100579e13a Mon Sep 17 00:00:00 2001 From: Tomoaki Teshima Date: Mon, 6 Jun 2016 08:56:37 +0900 Subject: [PATCH 029/128] follow other interface * remove useHW option * update test --- modules/core/include/opencv2/core.hpp | 2 +- modules/core/src/convert.cpp | 5 +++-- modules/core/test/test_arithm.cpp | 18 ++++++++++-------- 3 files changed, 14 insertions(+), 11 deletions(-) diff --git a/modules/core/include/opencv2/core.hpp b/modules/core/include/opencv2/core.hpp index fa7ab469b2..fef0395fef 100644 --- a/modules/core/include/opencv2/core.hpp +++ b/modules/core/include/opencv2/core.hpp @@ -533,7 +533,7 @@ CV_16S to represent the bit depth. If the input array is neither of them, it'll @param dst output array. @param useHW if possible use HW SIMD instruction to convert */ -CV_EXPORTS_W void convertFp16(InputArray src, OutputArray dst, bool useHW = true); +CV_EXPORTS_W void convertFp16(InputArray src, OutputArray dst); /** @brief Performs a look-up table transform of an array. diff --git a/modules/core/src/convert.cpp b/modules/core/src/convert.cpp index 4ff9830db8..431942fbea 100644 --- a/modules/core/src/convert.cpp +++ b/modules/core/src/convert.cpp @@ -5117,9 +5117,10 @@ void cv::convertScaleAbs( InputArray _src, OutputArray _dst, double alpha, doubl } } -void cv::convertFp16( InputArray _src, OutputArray _dst, bool useHW ) +void cv::convertFp16( InputArray _src, OutputArray _dst) { - if ( checkHardwareSupport(CV_CPU_FP16) == false) + bool useHW = true; + if ( checkHardwareSupport(CV_CPU_FP16) == false ) { useHW = false; } diff --git a/modules/core/test/test_arithm.cpp b/modules/core/test/test_arithm.cpp index 58974a8f5c..3548765295 100644 --- a/modules/core/test/test_arithm.cpp +++ b/modules/core/test/test_arithm.cpp @@ -742,27 +742,29 @@ struct ConvertScaleFp16Op : public BaseElemWiseOp ConvertScaleFp16Op() : BaseElemWiseOp(1, FIX_BETA+REAL_GAMMA, 1, 1, Scalar::all(0)), nextRange(0) { } void op(const vector& src, Mat& dst, const Mat&) { - convertFp16(src[0], dst, true); + Mat m; + convertFp16(src[0], m); + convertFp16(m, dst); } void refop(const vector& src, Mat& dst, const Mat&) { - convertFp16(src[0], dst, false); + cvtest::copy(src[0], dst); } int getRandomType(RNG&) { - // 0: FP32 -> FP16 - // 1: FP16 -> FP32 + // 0: FP32 -> FP16 -> FP32 + // 1: FP16 -> FP32 -> FP16 int srctype = (nextRange & 1) == 0 ? CV_32F : CV_16S; return srctype; } void getValueRange(int, double& minval, double& maxval) { - // 0: FP32 -> FP16 - // 1: FP16 -> FP32 + // 0: FP32 -> FP16 -> FP32 + // 1: FP16 -> FP32 -> FP16 if( (nextRange & 1) == 0 ) { - // largest integer number that fp16 can express - maxval = 65504.f; + // largest integer number that fp16 can express exactly + maxval = 2048.f; minval = -maxval; } else From 4239bac4edd646e60ee5114c4993bdf5a327a73f Mon Sep 17 00:00:00 2001 From: Tomoaki Teshima Date: Mon, 6 Jun 2016 18:06:23 +0900 Subject: [PATCH 030/128] fix warning of doc * update the comment to real header --- modules/core/include/opencv2/core.hpp | 1 - 1 file changed, 1 deletion(-) diff --git a/modules/core/include/opencv2/core.hpp b/modules/core/include/opencv2/core.hpp index fef0395fef..22121cef1c 100644 --- a/modules/core/include/opencv2/core.hpp +++ b/modules/core/include/opencv2/core.hpp @@ -531,7 +531,6 @@ CV_16S to represent the bit depth. If the input array is neither of them, it'll @param src input array. @param dst output array. -@param useHW if possible use HW SIMD instruction to convert */ CV_EXPORTS_W void convertFp16(InputArray src, OutputArray dst); From 15f8bc6f37d4e4f882b3a090da0179742b3e6dd0 Mon Sep 17 00:00:00 2001 From: Tomoaki Teshima Date: Mon, 6 Jun 2016 18:49:12 +0900 Subject: [PATCH 031/128] fix cmake * enable fp16 feature correctly with gcc on x86/x86_64 --- cmake/OpenCVCompilerOptions.cmake | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cmake/OpenCVCompilerOptions.cmake b/cmake/OpenCVCompilerOptions.cmake index 33dd575243..8b2b54f81e 100644 --- a/cmake/OpenCVCompilerOptions.cmake +++ b/cmake/OpenCVCompilerOptions.cmake @@ -170,9 +170,6 @@ if(CMAKE_COMPILER_IS_GNUCXX) add_extra_compiler_option(-mfma) endif() endif() - if((X86 OR X86_64) AND NOT MSVC) - add_extra_compiler_option(-mf16c) - endif((X86 OR X86_64) AND NOT MSVC) # GCC depresses SSEx instructions when -mavx is used. Instead, it generates new AVX instructions or AVX equivalence for all SSEx instructions when needed. if(NOT OPENCV_EXTRA_CXX_FLAGS MATCHES "-mavx") @@ -204,6 +201,9 @@ if(CMAKE_COMPILER_IS_GNUCXX) add_extra_compiler_option(-mpopcnt) endif() endif() + if((X86 OR X86_64) AND NOT MSVC) + add_extra_compiler_option(-mf16c) + endif((X86 OR X86_64) AND NOT MSVC) endif(NOT MINGW) if(X86 OR X86_64) From fbfd3158a76d161fd4148a61553218ee1cdb98d4 Mon Sep 17 00:00:00 2001 From: Tomoaki Teshima Date: Tue, 7 Jun 2016 08:59:28 +0900 Subject: [PATCH 032/128] fix corner case when number is small --- modules/core/src/convert.cpp | 27 +++++++++++++++++++++------ 1 file changed, 21 insertions(+), 6 deletions(-) diff --git a/modules/core/src/convert.cpp b/modules/core/src/convert.cpp index 431942fbea..432494dd31 100644 --- a/modules/core/src/convert.cpp +++ b/modules/core/src/convert.cpp @@ -4471,17 +4471,31 @@ static short convertFp16SW(float fp32) fp16Int16 result; result.i = 0; - if( 0x477ff000 <= ( a.i & 0x7fffffff ) ) + unsigned int absolute = a.i & 0x7fffffff; + if( 0x477ff000 <= absolute ) { // Inf in Fp16 result.i = result.i | 0x7C00; if( exponent == 128 && significand != 0 ) { // NaN - result.i = (short)(result.i | 0x200 | (significand >> kShiftSignificand)); + result.i = (short)( result.i | 0x200 | ( significand >> kShiftSignificand ) ); } } - else if ( ( a.i & 0x7fffffff ) <= 0x387fe000 ) + else if ( absolute < 0x33000001 ) + { + // too small for fp16 + result.i = 0; + } + else if ( absolute < 0x33c00000 ) + { + result.i = 1; + } + else if ( absolute < 0x34200001 ) + { + result.i = 2; + } + else if ( absolute < 0x387fe000 ) { // subnormal in Fp16 int fp16Significand = significand | 0x800000; @@ -4489,8 +4503,9 @@ static short convertFp16SW(float fp32) fp16Significand = fp16Significand >> bitShift; // special cases to round up - int threshold = 0x8000 + ( ( fp16Significand & 1 ) ? 0 : 1 ); - if( threshold <= ( significand & 0xffff ) ) + bitShift = exponent + 24; + unsigned int threshold = ( ( 0x400000 >> bitShift ) | ( ( ( significand & ( 0x800000 >> bitShift ) ) >> ( 126 - a.fmt.exponent ) ) ^ 1 ) ); + if( threshold <= ( significand & ( 0xffffff >> ( exponent + 25 ) ) ) ) { fp16Significand++; } @@ -4500,7 +4515,7 @@ static short convertFp16SW(float fp32) { // usual situation // exponent - result.fmt.exponent = (exponent + kBiasFp16Exponent); + result.fmt.exponent = ( exponent + kBiasFp16Exponent ); // significand; short fp16Significand = (short)(significand >> kShiftSignificand); From 6f6eebbcb9471106a7777329a970ac0aed25e24f Mon Sep 17 00:00:00 2001 From: Tomoaki Teshima Date: Tue, 7 Jun 2016 18:31:18 +0900 Subject: [PATCH 033/128] fix warning --- modules/core/src/convert.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/core/src/convert.cpp b/modules/core/src/convert.cpp index 432494dd31..d44e5eb8cd 100644 --- a/modules/core/src/convert.cpp +++ b/modules/core/src/convert.cpp @@ -4504,7 +4504,7 @@ static short convertFp16SW(float fp32) // special cases to round up bitShift = exponent + 24; - unsigned int threshold = ( ( 0x400000 >> bitShift ) | ( ( ( significand & ( 0x800000 >> bitShift ) ) >> ( 126 - a.fmt.exponent ) ) ^ 1 ) ); + int threshold = ( ( 0x400000 >> bitShift ) | ( ( ( significand & ( 0x800000 >> bitShift ) ) >> ( 126 - a.fmt.exponent ) ) ^ 1 ) ); if( threshold <= ( significand & ( 0xffffff >> ( exponent + 25 ) ) ) ) { fp16Significand++; From fd76ed5c0f83fec927b5a98bc192af0776a1432d Mon Sep 17 00:00:00 2001 From: Tomoaki Teshima Date: Tue, 7 Jun 2016 18:32:47 +0900 Subject: [PATCH 034/128] fix to support wider compiler * check compiler more strictly * use gcc version of fp16 conversion if it's possible (gcc 4.7 and later) * use current SW implementation in other cases --- modules/core/src/convert.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/modules/core/src/convert.cpp b/modules/core/src/convert.cpp index d44e5eb8cd..5e983da3a0 100644 --- a/modules/core/src/convert.cpp +++ b/modules/core/src/convert.cpp @@ -4356,7 +4356,7 @@ struct Cvt_SIMD #endif -#if !(defined (__arm__) || defined (__aarch64__)) +#if !( ( defined (__arm__) || defined (__aarch64__) ) && ( defined (__GNUC__) && ( ( ( 4 <= __GNUC__ ) && ( 7 <= __GNUC__ ) ) || ( 5 <= __GNUC__ ) ) ) ) // const numbers for floating points format const unsigned int kShiftSignificand = 13; const unsigned int kMaskFp16Significand = 0x3ff; @@ -4379,7 +4379,7 @@ union fp32Int32 union fp16Int16 { short i; -#if defined (__arm__) || defined (__aarch64__) +#if ( defined (__arm__) || defined (__aarch64__) ) && ( defined (__GNUC__) && ( ( ( 4 <= __GNUC__ ) && ( 7 <= __GNUC__ ) ) || ( 5 <= __GNUC__ ) ) ) __fp16 h; #endif struct _fp16Format @@ -4390,7 +4390,7 @@ union fp16Int16 } fmt; }; -#if defined (__arm__) || defined (__aarch64__) +#if ( defined (__arm__) || defined (__aarch64__) ) && ( defined (__GNUC__) && ( ( ( 4 <= __GNUC__ ) && ( 7 <= __GNUC__ ) ) || ( 5 <= __GNUC__ ) ) ) static float convertFp16SW(short fp16) { // Fp16 -> Fp32 @@ -4452,7 +4452,7 @@ static float convertFp16SW(short fp16) } #endif -#if defined (__arm__) || defined (__aarch64__) +#if ( defined (__arm__) || defined (__aarch64__) ) && ( defined (__GNUC__) && ( ( ( 4 <= __GNUC__ ) && ( 7 <= __GNUC__ ) ) || ( 5 <= __GNUC__ ) ) ) static short convertFp16SW(float fp32) { // Fp32 -> Fp16 From d0a83909635bc78343e7fcbfa495e1f07325fa17 Mon Sep 17 00:00:00 2001 From: Tomoaki Teshima Date: Thu, 9 Jun 2016 08:41:37 +0900 Subject: [PATCH 035/128] fix run time error on Mac * integrate HW version and SW version to same function --- modules/core/src/convert.cpp | 150 +++++++++++++++++------------------ 1 file changed, 71 insertions(+), 79 deletions(-) diff --git a/modules/core/src/convert.cpp b/modules/core/src/convert.cpp index 5e983da3a0..dc41eff37a 100644 --- a/modules/core/src/convert.cpp +++ b/modules/core/src/convert.cpp @@ -4540,24 +4540,9 @@ static short convertFp16SW(float fp32) } #endif -template static void -cvtScaleHalfSW_( const T* src, size_t sstep, DT* dst, size_t dstep, Size size) -{ - sstep /= sizeof(src[0]); - dstep /= sizeof(dst[0]); - - for( ; size.height--; src += sstep, dst += dstep ) - { - for ( int x = 0 ; x < size.width; x ++ ) - { - dst[x] = convertFp16SW(src[x]); - } - } -} - // template for FP16 HW conversion function template static void -cvtScaleHalfHW_( const T* src, size_t sstep, DT* dst, size_t dstep, Size size) +cvtScaleHalf_( const T* src, size_t sstep, DT* dst, size_t dstep, Size size) { sstep /= sizeof(src[0]); dstep /= sizeof(dst[0]); @@ -4573,77 +4558,105 @@ cvtScaleHalfHW_( const T* src, size_t sstep, DT* dst, size_t dstep, Size size) } template<> void -cvtScaleHalfHW_( const float* src, size_t sstep, short* dst, size_t dstep, Size size) +cvtScaleHalf_( const float* src, size_t sstep, short* dst, size_t dstep, Size size) { sstep /= sizeof(src[0]); dstep /= sizeof(dst[0]); - for( ; size.height--; src += sstep, dst += dstep ) + if( checkHardwareSupport(CV_FP16) ) { - int x = 0; - - if ( ( (intptr_t)dst & 0xf ) == 0 && ( (intptr_t)src & 0xf ) == 0 ) + for( ; size.height--; src += sstep, dst += dstep ) { -#if CV_FP16 - for ( ; x <= size.width - 4; x += 4) + int x = 0; + + if ( ( (intptr_t)dst & 0xf ) == 0 && ( (intptr_t)src & 0xf ) == 0 ) { +#if CV_FP16 + for ( ; x <= size.width - 4; x += 4) + { #if defined(__x86_64__) || defined(_M_X64) || defined(_M_IX86) || defined(i386) - __m128 v_src = _mm_load_ps(src + x); + __m128 v_src = _mm_load_ps(src + x); - __m128i v_dst = _mm_cvtps_ph(v_src, 0); + __m128i v_dst = _mm_cvtps_ph(v_src, 0); - _mm_storel_epi64((__m128i *)(dst + x), v_dst); + _mm_storel_epi64((__m128i *)(dst + x), v_dst); #elif defined __GNUC__ && (defined __arm__ || defined __aarch64__) - float32x4_t v_src = *(float32x4_t*)(src + x); + float32x4_t v_src = *(float32x4_t*)(src + x); - float16x4_t v_dst = vcvt_f16_f32(v_src); + float16x4_t v_dst = vcvt_f16_f32(v_src); - *(float16x4_t*)(dst + x) = v_dst; + *(float16x4_t*)(dst + x) = v_dst; +#endif + } #endif } -#endif + for ( ; x < size.width; x++ ) + { + dst[x] = convertFp16SW(src[x]); + } } - for ( ; x < size.width; x++ ) + } + else + { + for( ; size.height--; src += sstep, dst += dstep ) { - dst[x] = convertFp16SW(src[x]); + int x = 0; + for ( ; x < size.width; x++ ) + { + dst[x] = convertFp16SW(src[x]); + } } } } template<> void -cvtScaleHalfHW_( const short* src, size_t sstep, float* dst, size_t dstep, Size size) +cvtScaleHalf_( const short* src, size_t sstep, float* dst, size_t dstep, Size size) { sstep /= sizeof(src[0]); dstep /= sizeof(dst[0]); - for( ; size.height--; src += sstep, dst += dstep ) + if( checkHardwareSupport(CV_FP16) ) { - int x = 0; - - if ( ( (intptr_t)dst & 0xf ) == 0 && ( (intptr_t)src & 0xf ) == 0 ) + for( ; size.height--; src += sstep, dst += dstep ) { -#if CV_FP16 - for ( ; x <= size.width - 4; x += 4) + int x = 0; + + if ( ( (intptr_t)dst & 0xf ) == 0 && ( (intptr_t)src & 0xf ) == 0 && checkHardwareSupport(CV_CPU_FP16) ) { +#if CV_FP16 + for ( ; x <= size.width - 4; x += 4) + { #if defined(__x86_64__) || defined(_M_X64) || defined(_M_IX86) || defined(i386) - __m128i v_src = _mm_loadl_epi64((__m128i*)(src+x)); + __m128i v_src = _mm_loadl_epi64((__m128i*)(src+x)); - __m128 v_dst = _mm_cvtph_ps(v_src); + __m128 v_dst = _mm_cvtph_ps(v_src); - _mm_store_ps((dst + x), v_dst); + _mm_store_ps((dst + x), v_dst); #elif defined __GNUC__ && (defined __arm__ || defined __aarch64__) - float16x4_t v_src = *(float16x4_t*)(src + x); + float16x4_t v_src = *(float16x4_t*)(src + x); - float32x4_t v_dst = vcvt_f32_f16(v_src); + float32x4_t v_dst = vcvt_f32_f16(v_src); - *(float32x4_t*)(dst + x) = v_dst; + *(float32x4_t*)(dst + x) = v_dst; +#endif + } #endif } -#endif + for ( ; x < size.width; x++ ) + { + dst[x] = convertFp16SW(src[x]); + } } - for ( ; x < size.width; x++ ) + } + else + { + for( ; size.height--; src += sstep, dst += dstep ) { - dst[x] = convertFp16SW(src[x]); + int x = 0; + for ( ; x < size.width; x++ ) + { + dst[x] = convertFp16SW(src[x]); + } } } } @@ -4735,11 +4748,11 @@ static void cvtScaleAbs##suffix( const stype* src, size_t sstep, const uchar*, s tfunc(src, sstep, dst, dstep, size, (wtype)scale[0], (wtype)scale[1]); \ } -#define DEF_CVT_SCALE_FP16_FUNC(suffix, stype, dtype, resource) \ -static void cvtScaleHalf##suffix##resource( const stype* src, size_t sstep, const uchar*, size_t, \ +#define DEF_CVT_SCALE_FP16_FUNC(suffix, stype, dtype) \ +static void cvtScaleHalf##suffix( const stype* src, size_t sstep, const uchar*, size_t, \ dtype* dst, size_t dstep, Size size, double*) \ { \ - cvtScaleHalf##resource##_(src, sstep, dst, dstep, size); \ + cvtScaleHalf##_(src, sstep, dst, dstep, size); \ } #define DEF_CVT_SCALE_FUNC(suffix, stype, dtype, wtype) \ @@ -4798,10 +4811,8 @@ DEF_CVT_SCALE_ABS_FUNC(32s8u, cvtScaleAbs_, int, uchar, float) DEF_CVT_SCALE_ABS_FUNC(32f8u, cvtScaleAbs_, float, uchar, float) DEF_CVT_SCALE_ABS_FUNC(64f8u, cvtScaleAbs_, double, uchar, float) -DEF_CVT_SCALE_FP16_FUNC(32f16f, float, short, SW) -DEF_CVT_SCALE_FP16_FUNC(16f32f, short, float, SW) -DEF_CVT_SCALE_FP16_FUNC(32f16f, float, short, HW) -DEF_CVT_SCALE_FP16_FUNC(16f32f, short, float, HW) +DEF_CVT_SCALE_FP16_FUNC(32f16f, float, short) +DEF_CVT_SCALE_FP16_FUNC(16f32f, short, float) DEF_CVT_SCALE_FUNC(8u, uchar, uchar, float) DEF_CVT_SCALE_FUNC(8s8u, schar, uchar, float) @@ -4924,28 +4935,15 @@ static BinaryFunc getCvtScaleAbsFunc(int depth) return cvtScaleAbsTab[depth]; } -BinaryFunc getConvertFuncFp16(int ddepth, bool useHW) +BinaryFunc getConvertFuncFp16(int ddepth) { - static BinaryFunc cvtTabHW[] = + static BinaryFunc cvtTab[] = { 0, 0, 0, - (BinaryFunc)(cvtScaleHalf32f16fHW), 0, (BinaryFunc)(cvtScaleHalf16f32fHW), + (BinaryFunc)(cvtScaleHalf32f16f), 0, (BinaryFunc)(cvtScaleHalf16f32f), 0, 0, }; - static BinaryFunc cvtTabSW[] = - { - 0, 0, 0, - (BinaryFunc)(cvtScaleHalf32f16fSW), 0, (BinaryFunc)(cvtScaleHalf16f32fSW), - 0, 0, - }; - if( useHW == true) - { - return cvtTabHW[CV_MAT_DEPTH(ddepth)]; - } - else - { - return cvtTabSW[CV_MAT_DEPTH(ddepth)]; - } + return cvtTab[CV_MAT_DEPTH(ddepth)]; } BinaryFunc getConvertFunc(int sdepth, int ddepth) @@ -5134,12 +5132,6 @@ void cv::convertScaleAbs( InputArray _src, OutputArray _dst, double alpha, doubl void cv::convertFp16( InputArray _src, OutputArray _dst) { - bool useHW = true; - if ( checkHardwareSupport(CV_CPU_FP16) == false ) - { - useHW = false; - } - Mat src = _src.getMat(); int ddepth = 0; @@ -5158,7 +5150,7 @@ void cv::convertFp16( InputArray _src, OutputArray _dst) int type = CV_MAKETYPE(ddepth, src.channels()); _dst.create( src.dims, src.size, type ); Mat dst = _dst.getMat(); - BinaryFunc func = getConvertFuncFp16(ddepth, useHW); + BinaryFunc func = getConvertFuncFp16(ddepth); int cn = src.channels(); CV_Assert( func != 0 ); From 25e2e8aa3c22ddbad5a2006ffb654ec28827cb30 Mon Sep 17 00:00:00 2001 From: Aleksandar Atanasov Date: Thu, 9 Jun 2016 07:18:47 +0200 Subject: [PATCH 036/128] Removed trailing spaces at line 32 --- samples/python/houghcircles.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/python/houghcircles.py b/samples/python/houghcircles.py index 41d42eb1e1..477dac00d4 100755 --- a/samples/python/houghcircles.py +++ b/samples/python/houghcircles.py @@ -29,7 +29,7 @@ if __name__ == '__main__': cimg = src.copy() # numpy function circles = cv2.HoughCircles(img, cv2.HOUGH_GRADIENT, 1, 10, np.array([]), 100, 30, 1, 30) - + if circles != None: # Check if circles have been found and only then iterate over these and add them to the image a, b, c = circles.shape for i in range(b): From 87d0c91dcfe7c5e323e7c98f23a38f81f09358ea Mon Sep 17 00:00:00 2001 From: Tomoaki Teshima Date: Thu, 9 Jun 2016 18:24:00 +0900 Subject: [PATCH 037/128] fix warning of build --- modules/core/src/precomp.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/core/src/precomp.hpp b/modules/core/src/precomp.hpp index cece96cb08..df6c5b93ec 100644 --- a/modules/core/src/precomp.hpp +++ b/modules/core/src/precomp.hpp @@ -135,7 +135,7 @@ typedef void (*BinaryFuncC)(const uchar* src1, size_t step1, uchar* dst, size_t step, int width, int height, void*); -BinaryFunc getConvertFuncFp16(int ddepth, bool useHW); +BinaryFunc getConvertFuncFp16(int ddepth); BinaryFunc getConvertFunc(int sdepth, int ddepth); BinaryFunc getCopyMaskFunc(size_t esz); From c5bbc0353c1daadae8d953baf2940b146cd87185 Mon Sep 17 00:00:00 2001 From: Aleksandar Atanasov Date: Sun, 12 Jun 2016 12:54:16 +0200 Subject: [PATCH 038/128] Added small fix when circles are not detected I noticed that I missed the fact that `cimg` is used in the second `imshow()` call. Changed the scope of the second function call to be within the if-statement. Otherwise in cases where have not been detected the second `imshow()` will attempt to use `cimg` which will be empty leading to an error. --- samples/python/houghcircles.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/samples/python/houghcircles.py b/samples/python/houghcircles.py index 477dac00d4..0bfee42933 100755 --- a/samples/python/houghcircles.py +++ b/samples/python/houghcircles.py @@ -35,7 +35,8 @@ if __name__ == '__main__': for i in range(b): cv2.circle(cimg, (circles[0][i][0], circles[0][i][1]), circles[0][i][2], (0, 0, 255), 3, cv2.LINE_AA) cv2.circle(cimg, (circles[0][i][0], circles[0][i][1]), 2, (0, 255, 0), 3, cv2.LINE_AA) # draw center of circle + + cv2.imshow("detected circles", cimg) cv2.imshow("source", src) - cv2.imshow("detected circles", cimg) cv2.waitKey(0) From 445349dd7d2ab81f2fd027f6f0575277d35882f4 Mon Sep 17 00:00:00 2001 From: Aleksandar Atanasov Date: Sun, 12 Jun 2016 12:55:29 +0200 Subject: [PATCH 039/128] Same fix as with the houghcircles Moved second `imshow()` inside the if-statement to prevent error when no lines have been found and the function is called with an empty `cdst`. --- samples/python/houghlines.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/samples/python/houghlines.py b/samples/python/houghlines.py index 98c3640147..4a98828824 100755 --- a/samples/python/houghlines.py +++ b/samples/python/houghlines.py @@ -47,7 +47,8 @@ if __name__ == '__main__': pt1 = ( int(x0+1000*(-b)), int(y0+1000*(a)) ) pt2 = ( int(x0-1000*(-b)), int(y0-1000*(a)) ) cv2.line(cdst, pt1, pt2, (0, 0, 255), 3, cv2.LINE_AA) + + cv2.imshow("detected lines", cdst) cv2.imshow("source", src) - cv2.imshow("detected lines", cdst) cv2.waitKey(0) From 4fa86dad263bd8c31f40d7b1fcb151ca29c441cd Mon Sep 17 00:00:00 2001 From: Aleksandar Atanasov Date: Mon, 13 Jun 2016 09:00:29 +0200 Subject: [PATCH 040/128] Update houghcircles.py --- samples/python/houghcircles.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/python/houghcircles.py b/samples/python/houghcircles.py index 0bfee42933..41cce29be7 100755 --- a/samples/python/houghcircles.py +++ b/samples/python/houghcircles.py @@ -35,7 +35,7 @@ if __name__ == '__main__': for i in range(b): cv2.circle(cimg, (circles[0][i][0], circles[0][i][1]), circles[0][i][2], (0, 0, 255), 3, cv2.LINE_AA) cv2.circle(cimg, (circles[0][i][0], circles[0][i][1]), 2, (0, 255, 0), 3, cv2.LINE_AA) # draw center of circle - + cv2.imshow("detected circles", cimg) cv2.imshow("source", src) From 0637ca21dcc91bb2ebb6f26530107dc2b49693bf Mon Sep 17 00:00:00 2001 From: Aleksandar Atanasov Date: Mon, 13 Jun 2016 09:00:42 +0200 Subject: [PATCH 041/128] Update houghlines.py --- samples/python/houghlines.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/python/houghlines.py b/samples/python/houghlines.py index 4a98828824..445068aef4 100755 --- a/samples/python/houghlines.py +++ b/samples/python/houghlines.py @@ -47,7 +47,7 @@ if __name__ == '__main__': pt1 = ( int(x0+1000*(-b)), int(y0+1000*(a)) ) pt2 = ( int(x0-1000*(-b)), int(y0-1000*(a)) ) cv2.line(cdst, pt1, pt2, (0, 0, 255), 3, cv2.LINE_AA) - + cv2.imshow("detected lines", cdst) cv2.imshow("source", src) From addb15383a2780f177b80fed51bdf7724fe0a24a Mon Sep 17 00:00:00 2001 From: Tomoaki Teshima Date: Wed, 15 Jun 2016 18:02:41 +0900 Subject: [PATCH 042/128] fix run time error --- cmake/OpenCVCompilerOptions.cmake | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmake/OpenCVCompilerOptions.cmake b/cmake/OpenCVCompilerOptions.cmake index 8b2b54f81e..db4b239f0b 100644 --- a/cmake/OpenCVCompilerOptions.cmake +++ b/cmake/OpenCVCompilerOptions.cmake @@ -201,9 +201,9 @@ if(CMAKE_COMPILER_IS_GNUCXX) add_extra_compiler_option(-mpopcnt) endif() endif() - if((X86 OR X86_64) AND NOT MSVC) + if((X86 OR X86_64) AND NOT MSVC AND NOT APPLE) add_extra_compiler_option(-mf16c) - endif((X86 OR X86_64) AND NOT MSVC) + endif((X86 OR X86_64) AND NOT MSVC AND NOT APPLE) endif(NOT MINGW) if(X86 OR X86_64) From d2bad6febb9e8f76ca67e485905c0575d1a311ad Mon Sep 17 00:00:00 2001 From: Suleyman TURKMEN Date: Wed, 25 May 2016 17:16:09 +0300 Subject: [PATCH 043/128] cv::TickMeter class addition --- modules/core/include/opencv2/core/utility.hpp | 120 +++++++++++++++++- samples/gpu/cascadeclassifier.cpp | 2 - samples/gpu/generalized_hough.cpp | 2 - samples/gpu/stereo_multi.cpp | 2 - samples/gpu/super_resolution.cpp | 2 - samples/gpu/tick_meter.hpp | 48 ------- samples/gpu/video_reader.cpp | 2 - samples/gpu/video_writer.cpp | 2 - 8 files changed, 119 insertions(+), 61 deletions(-) delete mode 100644 samples/gpu/tick_meter.hpp diff --git a/modules/core/include/opencv2/core/utility.hpp b/modules/core/include/opencv2/core/utility.hpp index 60b2d3a064..c74c1184f2 100644 --- a/modules/core/include/opencv2/core/utility.hpp +++ b/modules/core/include/opencv2/core/utility.hpp @@ -251,7 +251,8 @@ CV_EXPORTS_W const String& getBuildInformation(); The function returns the number of ticks after the certain event (for example, when the machine was turned on). It can be used to initialize RNG or to measure a function execution time by reading the -tick count before and after the function call. See also the tick frequency. +tick count before and after the function call. +@sa getTickFrequency, TickMeter */ CV_EXPORTS_W int64 getTickCount(); @@ -264,9 +265,126 @@ execution time in seconds: // do something ... t = ((double)getTickCount() - t)/getTickFrequency(); @endcode +@sa getTickCount, TickMeter */ CV_EXPORTS_W double getTickFrequency(); +/** @brief a Class to measure passing time. + +The class computes passing time by counting the number of ticks per second. That is, the following code computes the +execution time in seconds: +@code +TickMeter tm; +tm.start(); +// do something ... +tm.stop(); +std::cout << tm.getTimeSec(); +@endcode +@sa getTickCount, getTickFrequency +*/ + +class CV_EXPORTS_W TickMeter +{ +public: + //! the default constructor + CV_WRAP TickMeter() + { + reset(); + } + + /** + starts counting ticks. + */ + CV_WRAP void start() + { + startTime = cv::getTickCount(); + } + + /** + stops counting ticks. + */ + CV_WRAP void stop() + { + int64 time = cv::getTickCount(); + if (startTime == 0) + return; + ++counter; + sumTime += (time - startTime); + startTime = 0; + } + + /** + returns counted ticks. + */ + CV_WRAP int64 getTimeTicks() const + { + return sumTime; + } + + /** + returns passed time in microseconds. + */ + CV_WRAP double getTimeMicro() const + { + return getTimeMilli()*1e3; + } + + /** + returns passed time in milliseconds. + */ + CV_WRAP double getTimeMilli() const + { + return getTimeSec()*1e3; + } + + /** + returns passed time in seconds. + */ + CV_WRAP double getTimeSec() const + { + return (double)getTimeTicks() / getTickFrequency(); + } + + /** + returns internal counter value. + */ + CV_WRAP int64 getCounter() const + { + return counter; + } + + /** + resets internal values. + */ + CV_WRAP void reset() + { + startTime = 0; + sumTime = 0; + counter = 0; + } + +private: + int64 counter; + int64 sumTime; + int64 startTime; +}; + +/** @brief output operator +@code +TickMeter tm; +tm.start(); +// do something ... +tm.stop(); +std::cout << tm; +@endcode +*/ + +static inline +std::ostream& operator << (std::ostream& out, const TickMeter& tm) +{ + return out << tm.getTimeSec() << "sec"; +} + /** @brief Returns the number of CPU ticks. The function returns the current number of CPU ticks on some architectures (such as x86, x64, diff --git a/samples/gpu/cascadeclassifier.cpp b/samples/gpu/cascadeclassifier.cpp index f6209f9fa3..156efbf035 100644 --- a/samples/gpu/cascadeclassifier.cpp +++ b/samples/gpu/cascadeclassifier.cpp @@ -13,8 +13,6 @@ #include "opencv2/cudaimgproc.hpp" #include "opencv2/cudawarping.hpp" -#include "tick_meter.hpp" - using namespace std; using namespace cv; using namespace cv::cuda; diff --git a/samples/gpu/generalized_hough.cpp b/samples/gpu/generalized_hough.cpp index fb1cb8979c..7b7e80ab9d 100644 --- a/samples/gpu/generalized_hough.cpp +++ b/samples/gpu/generalized_hough.cpp @@ -8,8 +8,6 @@ #include "opencv2/cudaimgproc.hpp" #include "opencv2/highgui.hpp" -#include "tick_meter.hpp" - using namespace std; using namespace cv; diff --git a/samples/gpu/stereo_multi.cpp b/samples/gpu/stereo_multi.cpp index bfb3e8a48b..7ef656719d 100644 --- a/samples/gpu/stereo_multi.cpp +++ b/samples/gpu/stereo_multi.cpp @@ -17,8 +17,6 @@ #include "opencv2/imgproc.hpp" #include "opencv2/cudastereo.hpp" -#include "tick_meter.hpp" - using namespace std; using namespace cv; using namespace cv::cuda; diff --git a/samples/gpu/super_resolution.cpp b/samples/gpu/super_resolution.cpp index 026afd9710..94a922cdba 100644 --- a/samples/gpu/super_resolution.cpp +++ b/samples/gpu/super_resolution.cpp @@ -11,8 +11,6 @@ #include "opencv2/superres/optical_flow.hpp" #include "opencv2/opencv_modules.hpp" -#include "tick_meter.hpp" - using namespace std; using namespace cv; using namespace cv::superres; diff --git a/samples/gpu/tick_meter.hpp b/samples/gpu/tick_meter.hpp deleted file mode 100644 index c11a22d140..0000000000 --- a/samples/gpu/tick_meter.hpp +++ /dev/null @@ -1,48 +0,0 @@ -#ifndef OPENCV_CUDA_SAMPLES_TICKMETER_ -#define OPENCV_CUDA_SAMPLES_TICKMETER_ - -class CV_EXPORTS TickMeter -{ -public: - TickMeter(); - void start(); - void stop(); - - int64 getTimeTicks() const; - double getTimeMicro() const; - double getTimeMilli() const; - double getTimeSec() const; - int64 getCounter() const; - - void reset(); -private: - int64 counter; - int64 sumTime; - int64 startTime; -}; - -std::ostream& operator << (std::ostream& out, const TickMeter& tm); - - -TickMeter::TickMeter() { reset(); } -int64 TickMeter::getTimeTicks() const { return sumTime; } -double TickMeter::getTimeMicro() const { return getTimeMilli()*1e3;} -double TickMeter::getTimeMilli() const { return getTimeSec()*1e3; } -double TickMeter::getTimeSec() const { return (double)getTimeTicks()/cv::getTickFrequency();} -int64 TickMeter::getCounter() const { return counter; } -void TickMeter::reset() {startTime = 0; sumTime = 0; counter = 0; } - -void TickMeter::start(){ startTime = cv::getTickCount(); } -void TickMeter::stop() -{ - int64 time = cv::getTickCount(); - if ( startTime == 0 ) - return; - ++counter; - sumTime += ( time - startTime ); - startTime = 0; -} - -std::ostream& operator << (std::ostream& out, const TickMeter& tm) { return out << tm.getTimeSec() << "sec"; } - -#endif diff --git a/samples/gpu/video_reader.cpp b/samples/gpu/video_reader.cpp index d8d6e136f8..a40a6800b4 100644 --- a/samples/gpu/video_reader.cpp +++ b/samples/gpu/video_reader.cpp @@ -14,8 +14,6 @@ #include #include -#include "tick_meter.hpp" - int main(int argc, const char* argv[]) { if (argc != 2) diff --git a/samples/gpu/video_writer.cpp b/samples/gpu/video_writer.cpp index 6c5d1412d6..80d2cfc47b 100644 --- a/samples/gpu/video_writer.cpp +++ b/samples/gpu/video_writer.cpp @@ -11,8 +11,6 @@ #include "opencv2/cudacodec.hpp" #include "opencv2/highgui.hpp" -#include "tick_meter.hpp" - int main(int argc, const char* argv[]) { if (argc != 2) From 73e16dbc3f791213667c8c511585504f747ff8b0 Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Tue, 28 Jun 2016 17:21:54 +0300 Subject: [PATCH 044/128] cmake: update arm toolchain --- platforms/linux/aarch64-gnu.toolchain.cmake | 4 + platforms/linux/arm-gnueabi.toolchain.cmake | 89 +--------------- platforms/linux/arm.toolchain.cmake | 87 ++++++++++++++++ platforms/linux/gnu.toolchain.cmake | 106 ++++++++++++++++++++ 4 files changed, 199 insertions(+), 87 deletions(-) create mode 100644 platforms/linux/aarch64-gnu.toolchain.cmake create mode 100644 platforms/linux/arm.toolchain.cmake create mode 100644 platforms/linux/gnu.toolchain.cmake diff --git a/platforms/linux/aarch64-gnu.toolchain.cmake b/platforms/linux/aarch64-gnu.toolchain.cmake new file mode 100644 index 0000000000..4e1c80b39d --- /dev/null +++ b/platforms/linux/aarch64-gnu.toolchain.cmake @@ -0,0 +1,4 @@ +set(CMAKE_SYSTEM_PROCESSOR aarch64) +set(GCC_COMPILER_VERSION "4.8" CACHE STRING "GCC Compiler version") +set(GNU_MACHINE "aarch64-linux-gnu" CACHE STRING "GNU compiler triple") +include("${CMAKE_CURRENT_LIST_DIR}/arm.toolchain.cmake") diff --git a/platforms/linux/arm-gnueabi.toolchain.cmake b/platforms/linux/arm-gnueabi.toolchain.cmake index 448dfa6b1c..d31da377fe 100644 --- a/platforms/linux/arm-gnueabi.toolchain.cmake +++ b/platforms/linux/arm-gnueabi.toolchain.cmake @@ -1,88 +1,3 @@ -set(CMAKE_SYSTEM_NAME Linux) -set(CMAKE_SYSTEM_VERSION 1) -set(CMAKE_SYSTEM_PROCESSOR arm) - set(GCC_COMPILER_VERSION "4.6" CACHE STRING "GCC Compiler version") - -set(FLOAT_ABI_SUFFIX "") -if (NOT SOFTFP) - set(FLOAT_ABI_SUFFIX "hf") -endif() - -find_program(CMAKE_C_COMPILER NAMES arm-linux-gnueabi${FLOAT_ABI_SUFFIX}-gcc-${GCC_COMPILER_VERSION}) -find_program(CMAKE_CXX_COMPILER NAMES arm-linux-gnueabi${FLOAT_ABI_SUFFIX}-g++-${GCC_COMPILER_VERSION}) -set(ARM_LINUX_SYSROOT /usr/arm-linux-gnueabi${FLOAT_ABI_SUFFIX} CACHE PATH "ARM cross compilation system root") - -set(CMAKE_CXX_FLAGS "" CACHE STRING "c++ flags") -set(CMAKE_C_FLAGS "" CACHE STRING "c flags") -set(CMAKE_SHARED_LINKER_FLAGS "" CACHE STRING "shared linker flags") -set(CMAKE_MODULE_LINKER_FLAGS "" CACHE STRING "module linker flags") -set(CMAKE_EXE_LINKER_FLAGS "-Wl,-z,nocopyreloc" CACHE STRING "executable linker flags") - -set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mthumb -fdata-sections -Wa,--noexecstack -fsigned-char -Wno-psabi") -set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mthumb -fdata-sections -Wa,--noexecstack -fsigned-char -Wno-psabi") - -set(CMAKE_SHARED_LINKER_FLAGS "-Wl,--fix-cortex-a8 -Wl,--no-undefined -Wl,--gc-sections -Wl,-z,noexecstack -Wl,-z,relro -Wl,-z,now ${CMAKE_SHARED_LINKER_FLAGS}") -set(CMAKE_MODULE_LINKER_FLAGS "-Wl,--fix-cortex-a8 -Wl,--no-undefined -Wl,--gc-sections -Wl,-z,noexecstack -Wl,-z,relro -Wl,-z,now ${CMAKE_MODULE_LINKER_FLAGS}") -set(CMAKE_EXE_LINKER_FLAGS "-Wl,--fix-cortex-a8 -Wl,--no-undefined -Wl,--gc-sections -Wl,-z,noexecstack -Wl,-z,relro -Wl,-z,now ${CMAKE_EXE_LINKER_FLAGS}") - -if(USE_NEON) - message(WARNING "You use obsolete variable USE_NEON to enable NEON instruction set. Use -DENABLE_NEON=ON instead." ) - set(ENABLE_NEON TRUE) -elseif(USE_VFPV3) - message(WARNING "You use obsolete variable USE_VFPV3 to enable VFPV3 instruction set. Use -DENABLE_VFPV3=ON instead." ) - set(ENABLE_VFPV3 TRUE) -endif() - -set(CMAKE_FIND_ROOT_PATH ${CMAKE_FIND_ROOT_PATH} ${ARM_LINUX_SYSROOT}) - -if(EXISTS ${CUDA_TOOLKIT_ROOT_DIR}) - set(CMAKE_FIND_ROOT_PATH ${CMAKE_FIND_ROOT_PATH} ${CUDA_TOOLKIT_ROOT_DIR}) -endif() - -set( CMAKE_SKIP_RPATH TRUE CACHE BOOL "If set, runtime paths are not added when using shared libraries." ) -set(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY) -set(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY) -set(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM ONLY) - -# macro to find programs on the host OS -macro( find_host_program ) - set( CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER ) - set( CMAKE_FIND_ROOT_PATH_MODE_LIBRARY NEVER ) - set( CMAKE_FIND_ROOT_PATH_MODE_INCLUDE NEVER ) - if( CMAKE_HOST_WIN32 ) - SET( WIN32 1 ) - SET( UNIX ) - elseif( CMAKE_HOST_APPLE ) - SET( APPLE 1 ) - SET( UNIX ) - endif() - find_program( ${ARGN} ) - SET( WIN32 ) - SET( APPLE ) - SET( UNIX 1 ) - set( CMAKE_FIND_ROOT_PATH_MODE_PROGRAM ONLY ) - set( CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY ) - set( CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY ) -endmacro() - -# macro to find packages on the host OS -macro( find_host_package ) - set( CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER ) - set( CMAKE_FIND_ROOT_PATH_MODE_LIBRARY NEVER ) - set( CMAKE_FIND_ROOT_PATH_MODE_INCLUDE NEVER ) - if( CMAKE_HOST_WIN32 ) - SET( WIN32 1 ) - SET( UNIX ) - elseif( CMAKE_HOST_APPLE ) - SET( APPLE 1 ) - SET( UNIX ) - endif() - find_package( ${ARGN} ) - SET( WIN32 ) - SET( APPLE ) - SET( UNIX 1 ) - set( CMAKE_FIND_ROOT_PATH_MODE_PROGRAM ONLY ) - set( CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY ) - set( CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY ) -endmacro() +set(GNU_MACHINE "arm-linux-gnueabi" CACHE STRING "GNU compiler triple") +include("${CMAKE_CURRENT_LIST_DIR}/arm.toolchain.cmake") diff --git a/platforms/linux/arm.toolchain.cmake b/platforms/linux/arm.toolchain.cmake new file mode 100644 index 0000000000..23c03fb6d7 --- /dev/null +++ b/platforms/linux/arm.toolchain.cmake @@ -0,0 +1,87 @@ +if(COMMAND toolchain_save_config) + return() # prevent recursive call +endif() + +set(CMAKE_SYSTEM_NAME Linux) +set(CMAKE_SYSTEM_VERSION 1) +if(NOT DEFINED CMAKE_SYSTEM_PROCESSOR) + set(CMAKE_SYSTEM_PROCESSOR arm) +else() + #message("CMAKE_SYSTEM_PROCESSOR=${CMAKE_SYSTEM_PROCESSOR}") +endif() + +include("${CMAKE_CURRENT_LIST_DIR}/gnu.toolchain.cmake") + +if(CMAKE_SYSTEM_PROCESSOR STREQUAL arm AND NOT ARM_IGNORE_FP) + set(FLOAT_ABI_SUFFIX "") + if(NOT SOFTFP) + set(FLOAT_ABI_SUFFIX "hf") + endif() +endif() + +if(NOT "x${GCC_COMPILER_VERSION}" STREQUAL "x") + set(__GCC_VER_SUFFIX "-${GCC_COMPILER_VERSION}") +endif() + +if(NOT DEFINED CMAKE_C_COMPILER) + find_program(CMAKE_C_COMPILER NAMES ${GNU_MACHINE}${FLOAT_ABI_SUFFIX}-gcc${__GCC_VER_SUFFIX}) +else() + #message(WARNING "CMAKE_C_COMPILER=${CMAKE_C_COMPILER} is defined") +endif() +if(NOT DEFINED CMAKE_CXX_COMPILER) + find_program(CMAKE_CXX_COMPILER NAMES ${GNU_MACHINE}${FLOAT_ABI_SUFFIX}-g++${__GCC_VER_SUFFIX}) +else() + #message(WARNING "CMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER} is defined") +endif() + +if(NOT DEFINED ARM_LINUX_SYSROOT AND DEFINED GNU_MACHINE) + set(ARM_LINUX_SYSROOT /usr/${GNU_MACHINE}${FLOAT_ABI_SUFFIX}) +endif() + +if(NOT DEFINED CMAKE_CXX_FLAGS) + set(CMAKE_CXX_FLAGS "" CACHE INTERAL "") + set(CMAKE_C_FLAGS "" CACHE INTERAL "") + set(CMAKE_SHARED_LINKER_FLAGS "" CACHE INTERAL "") + set(CMAKE_MODULE_LINKER_FLAGS "" CACHE INTERAL "") + set(CMAKE_EXE_LINKER_FLAGS "" CACHE INTERAL "") + + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fdata-sections -Wa,--noexecstack -fsigned-char -Wno-psabi") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fdata-sections -Wa,--noexecstack -fsigned-char -Wno-psabi") + if(CMAKE_SYSTEM_PROCESSOR STREQUAL arm) + set(CMAKE_CXX_FLAGS "-mthumb ${CMAKE_CXX_FLAGS}") + set(CMAKE_C_FLAGS "-mthumb ${CMAKE_C_FLAGS}") + set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,-z,nocopyreloc") + endif() + if(CMAKE_SYSTEM_PROCESSOR STREQUAL arm) + set(ARM_LINKER_FLAGS "-Wl,--fix-cortex-a8 -Wl,--no-undefined -Wl,--gc-sections -Wl,-z,noexecstack -Wl,-z,relro -Wl,-z,now") + elseif(CMAKE_SYSTEM_PROCESSOR STREQUAL aarch64) + set(ARM_LINKER_FLAGS "-Wl,--no-undefined -Wl,--gc-sections -Wl,-z,noexecstack -Wl,-z,relro -Wl,-z,now") + endif() + set(CMAKE_SHARED_LINKER_FLAGS "${ARM_LINKER_FLAGS} ${CMAKE_SHARED_LINKER_FLAGS}") + set(CMAKE_MODULE_LINKER_FLAGS "${ARM_LINKER_FLAGS} ${CMAKE_MODULE_LINKER_FLAGS}") + set(CMAKE_EXE_LINKER_FLAGS "${ARM_LINKER_FLAGS} ${CMAKE_EXE_LINKER_FLAGS}") +else() + #message(WARNING "CMAKE_CXX_FLAGS='${CMAKE_CXX_FLAGS}' is defined") +endif() + +if(USE_NEON) + message(WARNING "You use obsolete variable USE_NEON to enable NEON instruction set. Use -DENABLE_NEON=ON instead." ) + set(ENABLE_NEON TRUE) +elseif(USE_VFPV3) + message(WARNING "You use obsolete variable USE_VFPV3 to enable VFPV3 instruction set. Use -DENABLE_VFPV3=ON instead." ) + set(ENABLE_VFPV3 TRUE) +endif() + +set(CMAKE_FIND_ROOT_PATH ${CMAKE_FIND_ROOT_PATH} ${ARM_LINUX_SYSROOT}) + +if(EXISTS ${CUDA_TOOLKIT_ROOT_DIR}) + set(CMAKE_FIND_ROOT_PATH ${CMAKE_FIND_ROOT_PATH} ${CUDA_TOOLKIT_ROOT_DIR}) +endif() + +set(TOOLCHAIN_CONFIG_VARS ${TOOLCHAIN_CONFIG_VARS} + ARM_LINUX_SYSROOT + ENABLE_NEON + ENABLE_VFPV3 + CUDA_TOOLKIT_ROOT_DIR +) +toolchain_save_config() diff --git a/platforms/linux/gnu.toolchain.cmake b/platforms/linux/gnu.toolchain.cmake new file mode 100644 index 0000000000..4050d83f61 --- /dev/null +++ b/platforms/linux/gnu.toolchain.cmake @@ -0,0 +1,106 @@ +cmake_minimum_required(VERSION 2.8) + +# load settings in case of "try compile" +set(TOOLCHAIN_CONFIG_FILE "${CMAKE_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/toolchain.config.cmake") +get_property(__IN_TRY_COMPILE GLOBAL PROPERTY IN_TRY_COMPILE) +if(__IN_TRY_COMPILE) + include("${CMAKE_CURRENT_SOURCE_DIR}/../toolchain.config.cmake" OPTIONAL) # CMAKE_BINARY_DIR is different + macro(toolchain_save_config) + # nothing + endmacro() +else() + macro(toolchain_save_config) + set(__config "#message(\"Load TOOLCHAIN config...\")\n") + get_cmake_property(__variableNames VARIABLES) + set(__vars_list ${ARGN}) + list(APPEND __vars_list + ${TOOLCHAIN_CONFIG_VARS} + CMAKE_SYSTEM_NAME + CMAKE_SYSTEM_VERSION + CMAKE_SYSTEM_PROCESSOR + CMAKE_C_COMPILER + CMAKE_CXX_COMPILER + CMAKE_C_FLAGS + CMAKE_CXX_FLAGS + CMAKE_SHARED_LINKER_FLAGS + CMAKE_MODULE_LINKER_FLAGS + CMAKE_EXE_LINKER_FLAGS + CMAKE_SKIP_RPATH + CMAKE_FIND_ROOT_PATH + GCC_COMPILER_VERSION + ) + foreach(__var ${__variableNames}) + foreach(_v ${__vars_list}) + if("x${__var}" STREQUAL "x${_v}") + if(${__var} MATCHES " ") + set(__config "${__config}set(${__var} \"${${__var}}\")\n") + else() + set(__config "${__config}set(${__var} ${${__var}})\n") + endif() + endif() + endforeach() + endforeach() + if(EXISTS "${TOOLCHAIN_CONFIG_FILE}") + file(READ "${TOOLCHAIN_CONFIG_FILE}" __config_old) + endif() + if("${__config_old}" STREQUAL "${__config}") + # nothing + else() + #message("Update TOOLCHAIN config: ${__config}") + file(WRITE "${TOOLCHAIN_CONFIG_FILE}" "${__config}") + endif() + unset(__config) + unset(__config_old) + unset(__vars_list) + unset(__variableNames) + endmacro() +endif() # IN_TRY_COMPILE + +set(CMAKE_SKIP_RPATH TRUE) +set(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY) +set(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY) +set(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM ONLY) + +# macro to find programs on the host OS +macro(find_host_program) + set(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER) + set(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY NEVER) + set(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE NEVER) + if(CMAKE_HOST_WIN32) + SET(WIN32 1) + SET(UNIX) + elseif(CMAKE_HOST_APPLE) + SET(APPLE 1) + SET(UNIX) + endif() + find_program(${ARGN}) + SET(WIN32) + SET(APPLE) + SET(UNIX 1) + set(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM ONLY) + set(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY) + set(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY) +endmacro() + +# macro to find packages on the host OS +macro(find_host_package) + set(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER) + set(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY NEVER) + set(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE NEVER) + if(CMAKE_HOST_WIN32) + SET(WIN32 1) + SET(UNIX) + elseif(CMAKE_HOST_APPLE) + SET(APPLE 1) + SET(UNIX) + endif() + find_package(${ARGN}) + SET(WIN32) + SET(APPLE) + SET(UNIX 1) + set(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM ONLY) + set(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY) + set(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY) +endmacro() + +set(CMAKE_SKIP_RPATH TRUE CACHE BOOL "If set, runtime paths are not added when using shared libraries.") From b23527bf02d193cef9c6bc753851c5ab62c21d89 Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Thu, 30 Jun 2016 17:07:38 +0300 Subject: [PATCH 045/128] core: fix arguments types for cblas calls --- modules/core/src/hal_internal.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/core/src/hal_internal.cpp b/modules/core/src/hal_internal.cpp index 096ac0b400..07054f1d6c 100644 --- a/modules/core/src/hal_internal.cpp +++ b/modules/core/src/hal_internal.cpp @@ -399,9 +399,9 @@ lapack_gemm_c(const fptype *src1, size_t src1_step, const fptype *src2, size_t s set_value((std::complex*)dst, lddst, std::complex(0.0, 0.0), d_m, d_n); if(typeid(fptype) == typeid(float)) - cblas_cgemm(CblasRowMajor, transA, transB, a_m, d_n, a_n, &cAlpha, (void*)src1, ldsrc1, (void*)src2, ldsrc2, &cBeta, (void*)dst, lddst); + cblas_cgemm(CblasRowMajor, transA, transB, a_m, d_n, a_n, (float*)reinterpret_cast(cAlpha), (float*)src1, ldsrc1, (float*)src2, ldsrc2, (float*)reinterpret_cast(cBeta), (float*)dst, lddst); else if(typeid(fptype) == typeid(double)) - cblas_zgemm(CblasRowMajor, transA, transB, a_m, d_n, a_n, &cAlpha, (void*)src1, ldsrc1, (void*)src2, ldsrc2, &cBeta, (void*)dst, lddst); + cblas_zgemm(CblasRowMajor, transA, transB, a_m, d_n, a_n, (double*)reinterpret_cast(cAlpha), (double*)src1, ldsrc1, (double*)src2, ldsrc2, (double*)reinterpret_cast(cBeta), (double*)dst, lddst); return CV_HAL_ERROR_OK; } From ee4f409584cdc3910229d389cd09f9c7f3d63fe7 Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Thu, 30 Jun 2016 17:10:27 +0300 Subject: [PATCH 046/128] cmake: lapack optimized code requires cblas.h --- cmake/OpenCVFindLibsPerf.cmake | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/cmake/OpenCVFindLibsPerf.cmake b/cmake/OpenCVFindLibsPerf.cmake index 59ee42d32c..c679102d37 100644 --- a/cmake/OpenCVFindLibsPerf.cmake +++ b/cmake/OpenCVFindLibsPerf.cmake @@ -6,14 +6,17 @@ if(WITH_LAPACK) find_package(LAPACK) if(LAPACK_FOUND) - find_path(LAPACK_INCLUDE_DIR "lapacke.h") - if(LAPACK_INCLUDE_DIR) + find_path(LAPACKE_INCLUDE_DIR "lapacke.h") + if(LAPACKE_INCLUDE_DIR) + find_path(CBLAS_INCLUDE_DIR "cblas.h") + if(CBLAS_INCLUDE_DIR) set(HAVE_LAPACK 1) - ocv_include_directories(${LAPACK_INCLUDE_DIR}) + ocv_include_directories(${LAPACKE_INCLUDE_DIR} ${CBLAS_INCLUDE_DIR}) list(APPEND OPENCV_LINKER_LIBS ${LAPACK_LIBRARIES}) endif() - endif(LAPACK_FOUND) -endif(WITH_LAPACK) + endif() + endif() +endif() # --- TBB --- if(WITH_TBB) From 2b08f29543c5115402d6d963e232493ce0d6d8ca Mon Sep 17 00:00:00 2001 From: Arthur Cinader Date: Thu, 30 Jun 2016 17:04:25 -0700 Subject: [PATCH 047/128] Allow for an optional mask for MatchTemplate_Demo --- .../MatchTemplate_Demo.cpp | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/samples/cpp/tutorial_code/Histograms_Matching/MatchTemplate_Demo.cpp b/samples/cpp/tutorial_code/Histograms_Matching/MatchTemplate_Demo.cpp index 0bf447d2f9..7cd07a5f03 100644 --- a/samples/cpp/tutorial_code/Histograms_Matching/MatchTemplate_Demo.cpp +++ b/samples/cpp/tutorial_code/Histograms_Matching/MatchTemplate_Demo.cpp @@ -13,7 +13,8 @@ using namespace std; using namespace cv; /// Global Variables -Mat img; Mat templ; Mat result; +bool use_mask; +Mat img; Mat templ; Mat mask; Mat result; const char* image_window = "Source Image"; const char* result_window = "Result window"; @@ -31,7 +32,7 @@ int main( int argc, char** argv ) if (argc < 3) { cout << "Not enough parameters" << endl; - cout << "Usage:\n./MatchTemplate_Demo " << endl; + cout << "Usage:\n./MatchTemplate_Demo []" << endl; return -1; } @@ -39,7 +40,12 @@ int main( int argc, char** argv ) img = imread( argv[1], IMREAD_COLOR ); templ = imread( argv[2], IMREAD_COLOR ); - if(img.empty() || templ.empty()) + if(argc > 3) { + use_mask = true; + mask = imread(argv[3], IMREAD_COLOR); + } + + if(img.empty() || templ.empty() || (use_mask && mask.empty())) { cout << "Can't read one of the images" << endl; return -1; @@ -76,7 +82,12 @@ void MatchingMethod( int, void* ) result.create( result_rows, result_cols, CV_32FC1 ); /// Do the Matching and Normalize - matchTemplate( img, templ, result, match_method ); + bool method_accepts_mask = CV_TM_SQDIFF == match_method || match_method == CV_TM_CCORR_NORMED; + if (use_mask && method_accepts_mask) + { matchTemplate( img, templ, result, match_method, mask); } + else + { matchTemplate( img, templ, result, match_method); } + normalize( result, result, 0, 1, NORM_MINMAX, -1, Mat() ); /// Localizing the best match with minMaxLoc From 3f2ab5d3b5dec3f45331267995766241516a8d4f Mon Sep 17 00:00:00 2001 From: Michael Shtutman Date: Fri, 1 Jul 2016 22:00:15 +0300 Subject: [PATCH 048/128] Fix bug #6682 (Can't change resolution for YUY2 camera with DShow) --- modules/videoio/src/cap_dshow.cpp | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/modules/videoio/src/cap_dshow.cpp b/modules/videoio/src/cap_dshow.cpp index 4380b9b2c1..8b793ac202 100644 --- a/modules/videoio/src/cap_dshow.cpp +++ b/modules/videoio/src/cap_dshow.cpp @@ -2447,13 +2447,15 @@ static bool setSizeAndSubtype(videoDevice * VD, int attemptWidth, int attemptHei VD->pAmMediaType->subtype = mediatype; //buffer size - if (mediatype == MEDIASUBTYPE_RGB24) - { - VD->pAmMediaType->lSampleSize = attemptWidth*attemptHeight*3; + if (mediatype == MEDIASUBTYPE_RGB24){ + VD->pAmMediaType->lSampleSize = attemptWidth*attemptHeight * 3; } - else - { - // For compressed data, the value can be zero. + else if ((mediatype == MEDIASUBTYPE_YUY2) || (mediatype == MEDIASUBTYPE_YVYU) || + (mediatype == MEDIASUBTYPE_UYVY)){ + + VD->pAmMediaType->lSampleSize = attemptWidth*attemptHeight * 2; + } + else{ VD->pAmMediaType->lSampleSize = 0; } From 61390e407d8fa7ad13b070f7ce0274cf9bab67fc Mon Sep 17 00:00:00 2001 From: Dikay900 Date: Tue, 31 May 2016 19:27:59 +0200 Subject: [PATCH 049/128] Rephrase parts of the templates and use comments --- .github/ISSUE_TEMPLATE.md | 51 ++++++++++++++------------------ .github/PULL_REQUEST_TEMPLATE.md | 11 +++++-- 2 files changed, 30 insertions(+), 32 deletions(-) diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md index babefd3182..4f1453a2ff 100644 --- a/.github/ISSUE_TEMPLATE.md +++ b/.github/ISSUE_TEMPLATE.md @@ -1,37 +1,30 @@ -This is a template helping you to create an issue which can be processes as quickly as possible. Feel free to add additional information or remove not relevant points if you do not need them. - + -### In which part of the OpenCV library you got the issue? -Examples: -- objdetect, highgui, imgproc, cuda, tests -- face recognition, resizing an image, reading an jpg image +##### System information (version) + -### Expected behaviour +- OpenCV => :grey_question: +- Operating System / Platform => :grey_question: +- Compiler => :grey_question: -### Actual behaviour +##### Detailed description -### Additional description + -### Code example to reproduce the issue / Steps to reproduce the issue -Please try to give a full example which will compile as is. -``` -#include "opencv2/core.hpp" -#include -using namespace std; -using namespace cv; +##### Steps to reproduce -int main() -{ - double d[] = { 546,2435,7,4534,23423,3 }; - cout << "d = 0x" << reinterpret_cast(d) << endl; - - return 0; -} -``` + \ No newline at end of file diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 496d748731..210a253113 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -1,4 +1,9 @@ -resolves #XXXX + -### What does this PR change? -Please add your changes here. +### This pullrequest changes + + From 8d43e2b945d051bc1e042922307add79bd637499 Mon Sep 17 00:00:00 2001 From: Andreas Franek Date: Mon, 4 Jul 2016 16:13:31 +0200 Subject: [PATCH 050/128] fixed the cuda optical flow error normalization factor texture channels were not considered correctly, nor was the cuda texture normalization --- modules/cudaoptflow/src/cuda/pyrlk.cu | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/modules/cudaoptflow/src/cuda/pyrlk.cu b/modules/cudaoptflow/src/cuda/pyrlk.cu index 5d40a47eae..5c81edface 100644 --- a/modules/cudaoptflow/src/cuda/pyrlk.cu +++ b/modules/cudaoptflow/src/cuda/pyrlk.cu @@ -344,6 +344,18 @@ namespace pyrlk return ret; } + template + struct DenormalizationFactor + { + static const float factor = 1.0; + }; + + template <> + struct DenormalizationFactor + { + static const float factor = 255.0; + }; + template __global__ void sparseKernel(const float2* prevPts, float2* nextPts, uchar* status, float* err, const int level, const int rows, const int cols) { @@ -532,7 +544,7 @@ namespace pyrlk nextPts[blockIdx.x] = nextPt; if (calcErr) - err[blockIdx.x] = static_cast(errval) / (cn * c_winSize_x * c_winSize_y); + err[blockIdx.x] = static_cast(errval) / (::min(cn, 3) * c_winSize_x * c_winSize_y) * DenormalizationFactor::factor; } } @@ -725,7 +737,7 @@ namespace pyrlk nextPts[blockIdx.x] = nextPt; if (calcErr) - err[blockIdx.x] = static_cast(errval) / (3 * c_winSize_x * c_winSize_y); + err[blockIdx.x] = static_cast(errval) / (::min(cn, 3)*c_winSize_x * c_winSize_y); } } // __global__ void sparseKernel_ @@ -1109,4 +1121,4 @@ namespace pyrlk template class pyrLK_caller; } -#endif /* CUDA_DISABLER */ \ No newline at end of file +#endif /* CUDA_DISABLER */ From c65d2a0d86c47b3b5515c53175bcdc1d4e621fd2 Mon Sep 17 00:00:00 2001 From: Elif Albuz Date: Mon, 4 Jul 2016 23:45:07 -0700 Subject: [PATCH 051/128] Add Carotene - NVIDIA Hardware-Abstraction-Layer for ARM platforms --- CMakeLists.txt | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index bc84df4a34..7ea42680b9 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -169,6 +169,7 @@ endif() OCV_OPTION(WITH_1394 "Include IEEE1394 support" ON IF (NOT ANDROID AND NOT IOS AND NOT WINRT) ) OCV_OPTION(WITH_AVFOUNDATION "Use AVFoundation for Video I/O" ON IF IOS) OCV_OPTION(WITH_CARBON "Use Carbon for UI instead of Cocoa" OFF IF APPLE ) +OCV_OPTION(WITH_CAROTENE "Use NVidia carotene acceleration library for ARM platform" ON IF (ARM OR AARCH64) AND NOT IOS AND NOT (CMAKE_VERSION VERSION_LESS "2.8.11")) OCV_OPTION(WITH_VTK "Include VTK library support (and build opencv_viz module eiher)" ON IF (NOT ANDROID AND NOT IOS AND NOT WINRT AND NOT CMAKE_CROSSCOMPILING) ) OCV_OPTION(WITH_CUDA "Include NVidia Cuda Runtime support" ON IF (NOT IOS AND NOT WINRT) ) OCV_OPTION(WITH_CUFFT "Include NVidia Cuda Fast Fourier Transform (FFT) library support" ON IF (NOT IOS AND NOT WINRT) ) @@ -634,7 +635,20 @@ endmacro() if(NOT DEFINED OpenCV_HAL) set(OpenCV_HAL "OpenCV_HAL") endif() + +if(WITH_CAROTENE) + ocv_debug_message(STATUS "Enable carotene acceleration") + if(NOT ";${OpenCV_HAL};" MATCHES ";carotene;") + set(OpenCV_HAL "carotene;${OpenCV_HAL}") + endif() +endif() + foreach(hal ${OpenCV_HAL}) + if(hal STREQUAL "carotene") + add_subdirectory(3rdparty/carotene/hal) + ocv_hal_register(CAROTENE_HAL_LIBRARIES CAROTENE_HAL_HEADERS CAROTENE_HAL_INCLUDE_DIRS) + list(APPEND OpenCV_USED_HAL "carotene (ver ${CAROTENE_HAL_VERSION})") + else() ocv_debug_message(STATUS "OpenCV HAL: ${hal} ...") ocv_clear_vars(OpenCV_HAL_LIBRARIES OpenCV_HAL_HEADERS OpenCV_HAL_INCLUDE_DIRS) find_package(${hal} NO_MODULE QUIET) @@ -642,6 +656,7 @@ foreach(hal ${OpenCV_HAL}) ocv_hal_register(OpenCV_HAL_LIBRARIES OpenCV_HAL_HEADERS OpenCV_HAL_INCLUDE_DIRS) list(APPEND OpenCV_USED_HAL "${hal} (ver ${${hal}_VERSION})") endif() + endif() endforeach() configure_file("${OpenCV_SOURCE_DIR}/cmake/templates/custom_hal.hpp.in" "${CMAKE_BINARY_DIR}/custom_hal.hpp" @ONLY) unset(_hal_includes) From 8f91529edfd3a74383a47120f574bb7de09bfe9f Mon Sep 17 00:00:00 2001 From: Elif Albuz Date: Mon, 4 Jul 2016 23:56:15 -0700 Subject: [PATCH 052/128] Add Carotene - NVIDIA Hardware-Abstraction-Layer for ARM platforms --- 3rdparty/carotene/.gitignore | 8 + 3rdparty/carotene/CMakeLists.txt | 42 + 3rdparty/carotene/README.md | 2 + 3rdparty/carotene/hal/CMakeLists.txt | 137 + 3rdparty/carotene/hal/tegra_hal.hpp | 1851 +++++++++++ .../carotene/include/carotene/definitions.hpp | 47 + .../carotene/include/carotene/functions.hpp | 2492 +++++++++++++++ 3rdparty/carotene/include/carotene/types.hpp | 125 + 3rdparty/carotene/src/absdiff.cpp | 241 ++ 3rdparty/carotene/src/accumulate.cpp | 408 +++ 3rdparty/carotene/src/add.cpp | 475 +++ 3rdparty/carotene/src/add_weighted.cpp | 265 ++ 3rdparty/carotene/src/bitwise.cpp | 225 ++ 3rdparty/carotene/src/blur.cpp | 1337 ++++++++ 3rdparty/carotene/src/canny.cpp | 773 +++++ 3rdparty/carotene/src/channel_extract.cpp | 486 +++ 3rdparty/carotene/src/channels_combine.cpp | 389 +++ 3rdparty/carotene/src/cmp.cpp | 340 ++ 3rdparty/carotene/src/colorconvert.cpp | 2846 +++++++++++++++++ 3rdparty/carotene/src/common.cpp | 108 + 3rdparty/carotene/src/common.hpp | 96 + 3rdparty/carotene/src/convert.cpp | 1331 ++++++++ 3rdparty/carotene/src/convert_depth.cpp | 399 +++ 3rdparty/carotene/src/convert_scale.cpp | 2498 +++++++++++++++ 3rdparty/carotene/src/convolution.cpp | 340 ++ 3rdparty/carotene/src/count_nonzero.cpp | 430 +++ 3rdparty/carotene/src/div.cpp | 694 ++++ 3rdparty/carotene/src/dot_product.cpp | 260 ++ 3rdparty/carotene/src/fast.cpp | 428 +++ 3rdparty/carotene/src/fill_minmaxloc.cpp | 442 +++ 3rdparty/carotene/src/flip.cpp | 222 ++ 3rdparty/carotene/src/gaussian_blur.cpp | 1059 ++++++ 3rdparty/carotene/src/in_range.cpp | 195 ++ 3rdparty/carotene/src/integral.cpp | 238 ++ 3rdparty/carotene/src/intrinsics.hpp | 112 + 3rdparty/carotene/src/laplacian.cpp | 713 +++++ 3rdparty/carotene/src/magnitude.cpp | 160 + 3rdparty/carotene/src/meanstddev.cpp | 163 + 3rdparty/carotene/src/median_filter.cpp | 227 ++ 3rdparty/carotene/src/min_max.cpp | 139 + 3rdparty/carotene/src/minmaxloc.cpp | 1340 ++++++++ 3rdparty/carotene/src/morph.cpp | 728 +++++ 3rdparty/carotene/src/mul.cpp | 1572 +++++++++ 3rdparty/carotene/src/norm.cpp | 1310 ++++++++ 3rdparty/carotene/src/opticalflow.cpp | 539 ++++ 3rdparty/carotene/src/phase.cpp | 274 ++ 3rdparty/carotene/src/pyramid.cpp | 1414 ++++++++ 3rdparty/carotene/src/reduce.cpp | 460 +++ 3rdparty/carotene/src/remap.cpp | 694 ++++ 3rdparty/carotene/src/remap.hpp | 85 + 3rdparty/carotene/src/resize.cpp | 2191 +++++++++++++ 3rdparty/carotene/src/saturate_cast.hpp | 199 ++ 3rdparty/carotene/src/scharr.cpp | 219 ++ 3rdparty/carotene/src/separable_filter.cpp | 109 + 3rdparty/carotene/src/separable_filter.hpp | 1161 +++++++ 3rdparty/carotene/src/sobel.cpp | 317 ++ 3rdparty/carotene/src/sub.cpp | 621 ++++ 3rdparty/carotene/src/sum.cpp | 385 +++ 3rdparty/carotene/src/template_matching.cpp | 241 ++ 3rdparty/carotene/src/threshold.cpp | 1627 ++++++++++ 3rdparty/carotene/src/vtransform.hpp | 689 ++++ 3rdparty/carotene/src/warp_affine.cpp | 434 +++ 3rdparty/carotene/src/warp_perspective.cpp | 464 +++ 63 files changed, 39816 insertions(+) create mode 100644 3rdparty/carotene/.gitignore create mode 100644 3rdparty/carotene/CMakeLists.txt create mode 100644 3rdparty/carotene/README.md create mode 100644 3rdparty/carotene/hal/CMakeLists.txt create mode 100644 3rdparty/carotene/hal/tegra_hal.hpp create mode 100644 3rdparty/carotene/include/carotene/definitions.hpp create mode 100644 3rdparty/carotene/include/carotene/functions.hpp create mode 100644 3rdparty/carotene/include/carotene/types.hpp create mode 100644 3rdparty/carotene/src/absdiff.cpp create mode 100644 3rdparty/carotene/src/accumulate.cpp create mode 100644 3rdparty/carotene/src/add.cpp create mode 100644 3rdparty/carotene/src/add_weighted.cpp create mode 100644 3rdparty/carotene/src/bitwise.cpp create mode 100644 3rdparty/carotene/src/blur.cpp create mode 100644 3rdparty/carotene/src/canny.cpp create mode 100644 3rdparty/carotene/src/channel_extract.cpp create mode 100644 3rdparty/carotene/src/channels_combine.cpp create mode 100644 3rdparty/carotene/src/cmp.cpp create mode 100644 3rdparty/carotene/src/colorconvert.cpp create mode 100644 3rdparty/carotene/src/common.cpp create mode 100644 3rdparty/carotene/src/common.hpp create mode 100644 3rdparty/carotene/src/convert.cpp create mode 100644 3rdparty/carotene/src/convert_depth.cpp create mode 100644 3rdparty/carotene/src/convert_scale.cpp create mode 100644 3rdparty/carotene/src/convolution.cpp create mode 100644 3rdparty/carotene/src/count_nonzero.cpp create mode 100644 3rdparty/carotene/src/div.cpp create mode 100644 3rdparty/carotene/src/dot_product.cpp create mode 100644 3rdparty/carotene/src/fast.cpp create mode 100644 3rdparty/carotene/src/fill_minmaxloc.cpp create mode 100644 3rdparty/carotene/src/flip.cpp create mode 100644 3rdparty/carotene/src/gaussian_blur.cpp create mode 100644 3rdparty/carotene/src/in_range.cpp create mode 100644 3rdparty/carotene/src/integral.cpp create mode 100644 3rdparty/carotene/src/intrinsics.hpp create mode 100644 3rdparty/carotene/src/laplacian.cpp create mode 100644 3rdparty/carotene/src/magnitude.cpp create mode 100644 3rdparty/carotene/src/meanstddev.cpp create mode 100644 3rdparty/carotene/src/median_filter.cpp create mode 100644 3rdparty/carotene/src/min_max.cpp create mode 100644 3rdparty/carotene/src/minmaxloc.cpp create mode 100644 3rdparty/carotene/src/morph.cpp create mode 100644 3rdparty/carotene/src/mul.cpp create mode 100644 3rdparty/carotene/src/norm.cpp create mode 100644 3rdparty/carotene/src/opticalflow.cpp create mode 100644 3rdparty/carotene/src/phase.cpp create mode 100644 3rdparty/carotene/src/pyramid.cpp create mode 100644 3rdparty/carotene/src/reduce.cpp create mode 100644 3rdparty/carotene/src/remap.cpp create mode 100644 3rdparty/carotene/src/remap.hpp create mode 100644 3rdparty/carotene/src/resize.cpp create mode 100644 3rdparty/carotene/src/saturate_cast.hpp create mode 100644 3rdparty/carotene/src/scharr.cpp create mode 100644 3rdparty/carotene/src/separable_filter.cpp create mode 100644 3rdparty/carotene/src/separable_filter.hpp create mode 100644 3rdparty/carotene/src/sobel.cpp create mode 100644 3rdparty/carotene/src/sub.cpp create mode 100644 3rdparty/carotene/src/sum.cpp create mode 100644 3rdparty/carotene/src/template_matching.cpp create mode 100644 3rdparty/carotene/src/threshold.cpp create mode 100644 3rdparty/carotene/src/vtransform.hpp create mode 100644 3rdparty/carotene/src/warp_affine.cpp create mode 100644 3rdparty/carotene/src/warp_perspective.cpp diff --git a/3rdparty/carotene/.gitignore b/3rdparty/carotene/.gitignore new file mode 100644 index 0000000000..062445879b --- /dev/null +++ b/3rdparty/carotene/.gitignore @@ -0,0 +1,8 @@ +# Gedit temp files +*~ + +# Qt Creator file +*.user + +# MacOS-specific (Desktop Services Store) +.DS_Store diff --git a/3rdparty/carotene/CMakeLists.txt b/3rdparty/carotene/CMakeLists.txt new file mode 100644 index 0000000000..4dd7807c61 --- /dev/null +++ b/3rdparty/carotene/CMakeLists.txt @@ -0,0 +1,42 @@ +cmake_minimum_required(VERSION 2.8.11 FATAL_ERROR) + +project(Carotene) + +set(CAROTENE_NS "carotene" CACHE STRING "Namespace for Carotene definitions") + +set(CAROTENE_INCLUDE_DIR include) +set(CAROTENE_SOURCE_DIR src) + +file(GLOB_RECURSE carotene_headers RELATIVE "${CMAKE_CURRENT_LIST_DIR}" "${CAROTENE_INCLUDE_DIR}/*.hpp") +file(GLOB_RECURSE carotene_sources RELATIVE "${CMAKE_CURRENT_LIST_DIR}" "${CAROTENE_SOURCE_DIR}/*.cpp" + "${CAROTENE_SOURCE_DIR}/*.hpp") + +include_directories(${CAROTENE_INCLUDE_DIR}) + +if(CMAKE_COMPILER_IS_GNUCC) + set(CMAKE_CXX_FLAGS "-fvisibility=hidden ${CMAKE_CXX_FLAGS}") + + # allow more inlines - these parameters improve performance for: + # - matchTemplate about 5-10% + # - goodFeaturesToTrack 10-20% + # - cornerHarris 30% for some cases + + set_source_files_properties(${carotene_sources} COMPILE_FLAGS "--param ipcp-unit-growth=100000 --param inline-unit-growth=100000 --param large-stack-frame-growth=5000") +endif() + +add_library(carotene_objs OBJECT + ${carotene_headers} + ${carotene_sources} +) + +if(NOT CAROTENE_NS STREQUAL "carotene") + target_compile_definitions(carotene_objs PUBLIC "-DCAROTENE_NS=${CAROTENE_NS}") +endif() + +if(WITH_NEON) + target_compile_definitions(carotene_objs PRIVATE "-DWITH_NEON") +endif() + +set_target_properties(carotene_objs PROPERTIES POSITION_INDEPENDENT_CODE TRUE) + +add_library(carotene STATIC EXCLUDE_FROM_ALL "$") diff --git a/3rdparty/carotene/README.md b/3rdparty/carotene/README.md new file mode 100644 index 0000000000..fbaae5e970 --- /dev/null +++ b/3rdparty/carotene/README.md @@ -0,0 +1,2 @@ +This is Carotene, a low-level library containing optimized CPU routines +that are useful for computer vision algorithms. diff --git a/3rdparty/carotene/hal/CMakeLists.txt b/3rdparty/carotene/hal/CMakeLists.txt new file mode 100644 index 0000000000..9eaa94a9f8 --- /dev/null +++ b/3rdparty/carotene/hal/CMakeLists.txt @@ -0,0 +1,137 @@ +cmake_minimum_required(VERSION 2.8.8 FATAL_ERROR) + +include(CheckCCompilerFlag) +include(CheckCXXCompilerFlag) + +set(CMAKE_POSITION_INDEPENDENT_CODE ON) + +set(TEGRA_HAL_DIR "${CMAKE_CURRENT_SOURCE_DIR}") +set(CAROTENE_DIR "${TEGRA_HAL_DIR}/../") + +if(CMAKE_SYSTEM_PROCESSOR MATCHES "^(arm.*|ARM.*)") + set(ARM TRUE) +elseif (CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64.*|AARCH64.*") + set(AARCH64 TRUE) +endif() + +if(ANDROID AND ARM) + set(WITH_TGPU ON CACHE BOOL "Enable Tegra GPGPU optimization") +endif() + +set(TEGRA_COMPILER_FLAGS "") + +if(CMAKE_COMPILER_IS_GNUCXX) + # Generate unwind information even for functions that can't throw/propagate exceptions. + # This lets debuggers and such get non-broken backtraces for such functions, even without debugging symbols. + list(APPEND TEGRA_COMPILER_FLAGS -funwind-tables) +endif() + +if(CMAKE_COMPILER_IS_GNUCXX) + if(X86 OR ARMEABI_V6 OR (MIPS AND ANDROID_COMPILER_VERSION VERSION_LESS "4.6")) + list(APPEND TEGRA_COMPILER_FLAGS -fweb -fwrapv -frename-registers -fsched-stalled-insns-dep=100 -fsched-stalled-insns=2) + else() + list(APPEND TEGRA_COMPILER_FLAGS -fweb -fwrapv -frename-registers -fsched2-use-superblocks -fsched2-use-traces + -fsched-stalled-insns-dep=100 -fsched-stalled-insns=2) + endif() + if((ANDROID_COMPILER_IS_CLANG OR NOT ANDROID_COMPILER_VERSION VERSION_LESS "4.7") AND ANDROID_NDK_RELEASE STRGREATER "r8d" ) + list(APPEND TEGRA_COMPILER_FLAGS -fgraphite -fgraphite-identity -floop-block -floop-flatten -floop-interchange + -floop-strip-mine -floop-parallelize-all -ftree-loop-linear) + endif() +endif() + +if(ARM OR AARCH64) + set(CHECK_TEGRA_HARDWARE_DEFAULT ON) +else() + set(CHECK_TEGRA_HARDWARE_DEFAULT OFF) +endif() +set(CHECK_TEGRA_HARDWARE ${CHECK_TEGRA_HARDWARE_DEFAULT} CACHE BOOL + "Verify Tegra platform before running optimized code") + +string(REPLACE ";" " " TEGRA_COMPILER_FLAGS "${TEGRA_COMPILER_FLAGS}") +set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${TEGRA_COMPILER_FLAGS}") +set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${TEGRA_COMPILER_FLAGS}") + +if(ANDROID_NATIVE_API_LEVEL LESS 9 AND (WITH_TGPU OR CHECK_TEGRA_HARDWARE)) + message(FATAL_ERROR "GPU support and Hardware detector is not available for API levels below 9. +Please disable Tegra GPU support and hardware detection or configure project for API level 9 or above.") +endif() + +if(ARMEABI_V7A) + if (CMAKE_COMPILER_IS_GNUCXX) + set( CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fno-tree-vectorize" ) + set( CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fno-tree-vectorize" ) + endif() +endif() + +if (CHECK_TEGRA_HARDWARE) + add_definitions(-DCHECK_TEGRA_HARDWARE) +endif() + +if(WITH_TGPU) + add_definitions(-DHAVE_TGPU) +endif() + +if(WITH_LOGS) + add_definitions(-DHAVE_LOGS) +endif() + +set(CAROTENE_NS "carotene_o4t" CACHE STRING "" FORCE) + +function(compile_carotene) + if(ENABLE_NEON) + set(WITH_NEON ON) + endif() + + add_subdirectory("${CAROTENE_DIR}" "${CMAKE_CURRENT_BINARY_DIR}/carotene") + + if(ARM OR AARCH64) + if(CMAKE_BUILD_TYPE) + set(CMAKE_TRY_COMPILE_CONFIGURATION ${CMAKE_BUILD_TYPE}) + endif() + check_cxx_compiler_flag("-mfpu=neon" CXX_HAS_MFPU_NEON) + check_c_compiler_flag("-mfpu=neon" C_HAS_MFPU_NEON) + if(${CXX_HAS_MFPU_NEON} AND ${C_HAS_MFPU_NEON}) + get_target_property(old_flags "carotene_objs" COMPILE_FLAGS) + if(old_flags) + set_target_properties("carotene_objs" PROPERTIES COMPILE_FLAGS "${old_flags} -mfpu=neon") + else() + set_target_properties("carotene_objs" PROPERTIES COMPILE_FLAGS "-mfpu=neon") + endif() + endif() + endif() +endfunction() + +compile_carotene() + +include_directories("${CAROTENE_DIR}/include") + +get_target_property(carotene_defs carotene_objs INTERFACE_COMPILE_DEFINITIONS) +set_property(DIRECTORY APPEND PROPERTY COMPILE_DEFINITIONS ${carotene_defs}) + + if (CMAKE_COMPILER_IS_GNUCXX) + # allow more inlines - these parameters improve performance for: + # matchTemplate about 5-10% + # goodFeaturesToTrack 10-20% + # cornerHarris 30% for some cases + set_source_files_properties(impl.cpp $ COMPILE_FLAGS "--param ipcp-unit-growth=100000 --param inline-unit-growth=100000 --param large-stack-frame-growth=5000") +# set_source_files_properties(impl.cpp $ COMPILE_FLAGS "--param ipcp-unit-growth=100000 --param inline-unit-growth=100000 --param large-stack-frame-growth=5000") + endif() + +add_library(tegra_hal STATIC $) +set_target_properties(tegra_hal PROPERTIES POSITION_INDEPENDENT_CODE TRUE) +set_target_properties(tegra_hal PROPERTIES ARCHIVE_OUTPUT_DIRECTORY ${3P_LIBRARY_OUTPUT_PATH}) +set(OPENCV_SRC_DIR "${CMAKE_SOURCE_DIR}") +if(NOT BUILD_SHARED_LIBS) + ocv_install_target(tegra_hal EXPORT OpenCVModules ARCHIVE DESTINATION ${OPENCV_3P_LIB_INSTALL_PATH} COMPONENT dev) +endif() +target_include_directories(tegra_hal PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} ${OPENCV_SRC_DIR}/modules/core/include) + +set(CAROTENE_HAL_VERSION "0.0.1" PARENT_SCOPE) +set(CAROTENE_HAL_LIBRARIES "tegra_hal" PARENT_SCOPE) +set(CAROTENE_HAL_HEADERS "carotene/tegra_hal.hpp" PARENT_SCOPE) +set(CAROTENE_HAL_INCLUDE_DIRS "${CMAKE_BINARY_DIR}" PARENT_SCOPE) + +configure_file("tegra_hal.hpp" "${CMAKE_BINARY_DIR}/carotene/tegra_hal.hpp" COPYONLY) +configure_file("${CAROTENE_DIR}/include/carotene/definitions.hpp" "${CMAKE_BINARY_DIR}/carotene/definitions.hpp" COPYONLY) +configure_file("${CAROTENE_DIR}/include/carotene/functions.hpp" "${CMAKE_BINARY_DIR}/carotene/functions.hpp" COPYONLY) +configure_file("${CAROTENE_DIR}/include/carotene/types.hpp" "${CMAKE_BINARY_DIR}/carotene/types.hpp" COPYONLY) diff --git a/3rdparty/carotene/hal/tegra_hal.hpp b/3rdparty/carotene/hal/tegra_hal.hpp new file mode 100644 index 0000000000..f1bf5c67a7 --- /dev/null +++ b/3rdparty/carotene/hal/tegra_hal.hpp @@ -0,0 +1,1851 @@ +/* + * By downloading, copying, installing or using the software you agree to this license. + * If you do not agree to this license, do not download, install, + * copy or use the software. + * + * + * License Agreement + * For Open Source Computer Vision Library + * (3-clause BSD License) + * + * Copyright (C) 2016, NVIDIA Corporation, all rights reserved. + * Third party copyrights are property of their respective owners. + * + * Redistribution and use in source and binary forms, with or without modification, + * are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * * Neither the names of the copyright holders nor the names of the contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * This software is provided by the copyright holders and contributors "as is" and + * any express or implied warranties, including, but not limited to, the implied + * warranties of merchantability and fitness for a particular purpose are disclaimed. + * In no event shall copyright holders or contributors be liable for any direct, + * indirect, incidental, special, exemplary, or consequential damages + * (including, but not limited to, procurement of substitute goods or services; + * loss of use, data, or profits; or business interruption) however caused + * and on any theory of liability, whether in contract, strict liability, + * or tort (including negligence or otherwise) arising in any way out of + * the use of this software, even if advised of the possibility of such damage. + */ + +#ifndef _tegra_hal_H_INCLUDED_ +#define _tegra_hal_H_INCLUDED_ + +#define CAROTENE_NS carotene_o4t + +#include "carotene/functions.hpp" +#include +#include +#include +#include + +#define RANGE_DATA(type, base, step) reinterpret_cast(const_cast(reinterpret_cast(base)) + static_cast(range.start) * step) + +#define PARALLEL_CORE 0 +#if PARALLEL_CORE + +#define SRC_ARG1 ST * src1_data_, size_t src1_step_, +#define SRC_STORE1 src1_data(src1_data_), src1_step(src1_step_), +#define SRC_VAR1 ST * src1_data; \ + size_t src1_step; +#define SRC_ARG2 ST * src1_data_, size_t src1_step_, \ + ST * src2_data_, size_t src2_step_, +#define SRC_STORE2 src1_data(src1_data_), src1_step(src1_step_), \ + src2_data(src2_data_), src2_step(src2_step_), +#define SRC_VAR2 ST * src1_data; \ + size_t src1_step; \ + ST * src2_data; \ + size_t src2_step; + +#define DST_ARG1 DT * dst1_data_, size_t dst1_step_, +#define DST_STORE1 dst1_data(dst1_data_), dst1_step(dst1_step_), +#define DST_VAR1 DT * dst1_data; \ + size_t dst1_step; + +#define SCALE_ARG0 +#define SCALE_STORE0 +#define SCALE_VAR0 +#define SCALE_ARG1 , double scale_ +#define SCALE_STORE1 , scale(scale_) +#define SCALE_VAR1 double scale; +#define SCALE_ARG3 , const double *scales_ +#define SCALE_STORE3 , scales(scales_, scales_ + 3) +#define SCALE_VAR3 std::vector scales; + +#define TegraGenOp_Invoker(name, func, src_cnt, dst_cnt, scale_cnt, ...) \ +template \ +class TegraGenOp_##name##_Invoker : public cv::ParallelLoopBody \ +{ \ +public: \ + TegraGenOp_##name##_Invoker(SRC_ARG##src_cnt \ + DST_ARG##dst_cnt \ + int width_, int height_ \ + SCALE_ARG##scale_cnt) : \ + cv::ParallelLoopBody(), SRC_STORE##src_cnt \ + DST_STORE##dst_cnt \ + width(width_), height(height_) \ + SCALE_STORE##scale_cnt {} \ + virtual void operator()(const cv::Range& range) const \ + { \ + CAROTENE_NS::func(CAROTENE_NS::Size2D(width, range.end-range.start), __VA_ARGS__); \ + } \ +private: \ + SRC_VAR##src_cnt \ + DST_VAR##dst_cnt \ + int width, height; \ + SCALE_VAR##scale_cnt \ + const TegraGenOp_##name##_Invoker& operator= (const TegraGenOp_##name##_Invoker&); \ +}; + +#define TegraBinaryOp_Invoker(name, func) TegraGenOp_Invoker(name, func, 2, 1, 0, \ + RANGE_DATA(ST, src1_data, src1_step), src1_step, \ + RANGE_DATA(ST, src2_data, src2_step), src2_step, \ + RANGE_DATA(DT, dst1_data, dst1_step), dst1_step ) + +#define TegraBinaryOp_InvokerVAArg(name, func, ...) TegraGenOp_Invoker(name, func, 2, 1, 0, \ + RANGE_DATA(ST, src1_data, src1_step), src1_step, \ + RANGE_DATA(ST, src2_data, src2_step), src2_step, \ + RANGE_DATA(DT, dst1_data, dst1_step), dst1_step, __VA_ARGS__) + +#define TEGRA_BINARYOP(type, op, src1, sz1, src2, sz2, dst, sz, w, h) \ +( \ + CAROTENE_NS::isSupportedConfiguration() ? \ + parallel_for_(Range(0, h), \ + TegraGenOp_##op##_Invoker(src1, sz1, src2, sz2, dst, sz, w, h), \ + (w * h) / static_cast(1<<16)), \ + CV_HAL_ERROR_OK \ + : CV_HAL_ERROR_NOT_IMPLEMENTED \ +) + +TegraBinaryOp_InvokerVAArg(add, add, CAROTENE_NS::CONVERT_POLICY_SATURATE) /*Original addition use saturated operator, so use the same from CAROTENE*/ + +TegraBinaryOp_Invoker(addf, add) + +TegraBinaryOp_InvokerVAArg(sub, sub, CAROTENE_NS::CONVERT_POLICY_SATURATE) /*Original addition use saturated operator, so use the same from CAROTENE*/ + +TegraBinaryOp_Invoker(subf, sub) + +TegraBinaryOp_Invoker(max, max) + +TegraBinaryOp_Invoker(min, min) + +TegraBinaryOp_Invoker(absDiff, absDiff) + +TegraBinaryOp_Invoker(bitwiseAnd, bitwiseAnd) + +TegraBinaryOp_Invoker(bitwiseOr, bitwiseOr) + +TegraBinaryOp_Invoker(bitwiseXor, bitwiseXor) + +#define TegraUnaryOp_Invoker(name, func) TegraGenOp_Invoker(name, func, 1, 1, 0, \ + RANGE_DATA(ST, src1_data, src1_step), src1_step, \ + RANGE_DATA(DT, dst1_data, dst1_step), dst1_step ) + +TegraUnaryOp_Invoker(bitwiseNot, bitwiseNot) +#define TEGRA_UNARYOP(type, op, src1, sz1, dst, sz, w, h) \ +( \ + CAROTENE_NS::isSupportedConfiguration() ? \ + parallel_for_(Range(0, h), \ + TegraGenOp_##op##_Invoker(src1, sz1, dst, sz, w, h), \ + (w * h) / static_cast(1<<16)), \ + CV_HAL_ERROR_OK \ + : CV_HAL_ERROR_NOT_IMPLEMENTED \ +) + +#undef cv_hal_add8u +#define cv_hal_add8u(src1, sz1, src2, sz2, dst, sz, w, h) TEGRA_BINARYOP(CAROTENE_NS::u8, add, src1, sz1, src2, sz2, dst, sz, w, h) +#undef cv_hal_add8s +#define cv_hal_add8s(src1, sz1, src2, sz2, dst, sz, w, h) TEGRA_BINARYOP(CAROTENE_NS::s8, add, src1, sz1, src2, sz2, dst, sz, w, h) +#undef cv_hal_add16u +#define cv_hal_add16u(src1, sz1, src2, sz2, dst, sz, w, h) TEGRA_BINARYOP(CAROTENE_NS::u16, add, src1, sz1, src2, sz2, dst, sz, w, h) +#undef cv_hal_add16s +#define cv_hal_add16s(src1, sz1, src2, sz2, dst, sz, w, h) TEGRA_BINARYOP(CAROTENE_NS::s16, add, src1, sz1, src2, sz2, dst, sz, w, h) +#undef cv_hal_add32s +#define cv_hal_add32s(src1, sz1, src2, sz2, dst, sz, w, h) TEGRA_BINARYOP(CAROTENE_NS::s32, add, src1, sz1, src2, sz2, dst, sz, w, h) +#undef cv_hal_add32f +#define cv_hal_add32f(src1, sz1, src2, sz2, dst, sz, w, h) TEGRA_BINARYOP(CAROTENE_NS::f32, addf, src1, sz1, src2, sz2, dst, sz, w, h) +//#undef cv_hal_add64f +//#define cv_hal_add64f(src1, sz1, src2, sz2, dst, sz, w, h) TEGRA_BINARYOP(CAROTENE_NS::f64, addf, src1, sz1, src2, sz2, dst, sz, w, h) +#undef cv_hal_sub8u +#define cv_hal_sub8u(src1, sz1, src2, sz2, dst, sz, w, h) TEGRA_BINARYOP(CAROTENE_NS::u8, sub, src1, sz1, src2, sz2, dst, sz, w, h) +#undef cv_hal_sub8s +#define cv_hal_sub8s(src1, sz1, src2, sz2, dst, sz, w, h) TEGRA_BINARYOP(CAROTENE_NS::s8, sub, src1, sz1, src2, sz2, dst, sz, w, h) +#undef cv_hal_sub16u +#define cv_hal_sub16u(src1, sz1, src2, sz2, dst, sz, w, h) TEGRA_BINARYOP(CAROTENE_NS::u16, sub, src1, sz1, src2, sz2, dst, sz, w, h) +#undef cv_hal_sub16s +#define cv_hal_sub16s(src1, sz1, src2, sz2, dst, sz, w, h) TEGRA_BINARYOP(CAROTENE_NS::s16, sub, src1, sz1, src2, sz2, dst, sz, w, h) +#undef cv_hal_sub32s +#define cv_hal_sub32s(src1, sz1, src2, sz2, dst, sz, w, h) TEGRA_BINARYOP(CAROTENE_NS::s32, sub, src1, sz1, src2, sz2, dst, sz, w, h) +#undef cv_hal_sub32f +#define cv_hal_sub32f(src1, sz1, src2, sz2, dst, sz, w, h) TEGRA_BINARYOP(CAROTENE_NS::f32, subf, src1, sz1, src2, sz2, dst, sz, w, h) +//#undef cv_hal_sub64f +//#define cv_hal_sub64f(src1, sz1, src2, sz2, dst, sz, w, h) TEGRA_BINARYOP(CAROTENE_NS::f64, subf, src1, sz1, src2, sz2, dst, sz, w, h) +#undef cv_hal_max8u +#define cv_hal_max8u(src1, sz1, src2, sz2, dst, sz, w, h) TEGRA_BINARYOP(CAROTENE_NS::u8, max, src1, sz1, src2, sz2, dst, sz, w, h) +#undef cv_hal_max8s +#define cv_hal_max8s(src1, sz1, src2, sz2, dst, sz, w, h) TEGRA_BINARYOP(CAROTENE_NS::s8, max, src1, sz1, src2, sz2, dst, sz, w, h) +#undef cv_hal_max16u +#define cv_hal_max16u(src1, sz1, src2, sz2, dst, sz, w, h) TEGRA_BINARYOP(CAROTENE_NS::u16, max, src1, sz1, src2, sz2, dst, sz, w, h) +#undef cv_hal_max16s +#define cv_hal_max16s(src1, sz1, src2, sz2, dst, sz, w, h) TEGRA_BINARYOP(CAROTENE_NS::s16, max, src1, sz1, src2, sz2, dst, sz, w, h) +#undef cv_hal_max32s +#define cv_hal_max32s(src1, sz1, src2, sz2, dst, sz, w, h) TEGRA_BINARYOP(CAROTENE_NS::s32, max, src1, sz1, src2, sz2, dst, sz, w, h) +#undef cv_hal_max32f +#define cv_hal_max32f(src1, sz1, src2, sz2, dst, sz, w, h) TEGRA_BINARYOP(CAROTENE_NS::f32, max, src1, sz1, src2, sz2, dst, sz, w, h) +//#undef cv_hal_max64f +//#define cv_hal_max64f(src1, sz1, src2, sz2, dst, sz, w, h) TEGRA_BINARYOP(CAROTENE_NS::f64, max, src1, sz1, src2, sz2, dst, sz, w, h) +#undef cv_hal_min8u +#define cv_hal_min8u(src1, sz1, src2, sz2, dst, sz, w, h) TEGRA_BINARYOP(CAROTENE_NS::u8, min, src1, sz1, src2, sz2, dst, sz, w, h) +#undef cv_hal_min8s +#define cv_hal_min8s(src1, sz1, src2, sz2, dst, sz, w, h) TEGRA_BINARYOP(CAROTENE_NS::s8, min, src1, sz1, src2, sz2, dst, sz, w, h) +#undef cv_hal_min16u +#define cv_hal_min16u(src1, sz1, src2, sz2, dst, sz, w, h) TEGRA_BINARYOP(CAROTENE_NS::u16, min, src1, sz1, src2, sz2, dst, sz, w, h) +#undef cv_hal_min16s +#define cv_hal_min16s(src1, sz1, src2, sz2, dst, sz, w, h) TEGRA_BINARYOP(CAROTENE_NS::s16, min, src1, sz1, src2, sz2, dst, sz, w, h) +#undef cv_hal_min32s +#define cv_hal_min32s(src1, sz1, src2, sz2, dst, sz, w, h) TEGRA_BINARYOP(CAROTENE_NS::s32, min, src1, sz1, src2, sz2, dst, sz, w, h) +#undef cv_hal_min32f +#define cv_hal_min32f(src1, sz1, src2, sz2, dst, sz, w, h) TEGRA_BINARYOP(CAROTENE_NS::f32, min, src1, sz1, src2, sz2, dst, sz, w, h) +//#undef cv_hal_min64f +//#define cv_hal_min64f(src1, sz1, src2, sz2, dst, sz, w, h) TEGRA_BINARYOP(CAROTENE_NS::f64, min, src1, sz1, src2, sz2, dst, sz, w, h) +#undef cv_hal_absdiff8u +#define cv_hal_absdiff8u(src1, sz1, src2, sz2, dst, sz, w, h) TEGRA_BINARYOP(CAROTENE_NS::u8, absDiff, src1, sz1, src2, sz2, dst, sz, w, h) +#undef cv_hal_absdiff8s +#define cv_hal_absdiff8s(src1, sz1, src2, sz2, dst, sz, w, h) TEGRA_BINARYOP(CAROTENE_NS::s8, absDiff, src1, sz1, src2, sz2, dst, sz, w, h) +#undef cv_hal_absdiff16u +#define cv_hal_absdiff16u(src1, sz1, src2, sz2, dst, sz, w, h) TEGRA_BINARYOP(CAROTENE_NS::u16, absDiff, src1, sz1, src2, sz2, dst, sz, w, h) +#undef cv_hal_absdiff16s +#define cv_hal_absdiff16s(src1, sz1, src2, sz2, dst, sz, w, h) TEGRA_BINARYOP(CAROTENE_NS::s16, absDiff, src1, sz1, src2, sz2, dst, sz, w, h) +#undef cv_hal_absdiff32s +#define cv_hal_absdiff32s(src1, sz1, src2, sz2, dst, sz, w, h) TEGRA_BINARYOP(CAROTENE_NS::s32, absDiff, src1, sz1, src2, sz2, dst, sz, w, h) +#undef cv_hal_absdiff32f +#define cv_hal_absdiff32f(src1, sz1, src2, sz2, dst, sz, w, h) TEGRA_BINARYOP(CAROTENE_NS::f32, absDiff, src1, sz1, src2, sz2, dst, sz, w, h) +//#undef cv_hal_absdiff64f +//#define cv_hal_absdiff64f(src1, sz1, src2, sz2, dst, sz, w, h) TEGRA_BINARYOP(CAROTENE_NS::f64, absDiff, src1, sz1, src2, sz2, dst, sz, w, h) +#undef cv_hal_and8u +#define cv_hal_and8u(src1, sz1, src2, sz2, dst, sz, w, h) TEGRA_BINARYOP(CAROTENE_NS::u8, bitwiseAnd, src1, sz1, src2, sz2, dst, sz, w, h) +#undef cv_hal_or8u +#define cv_hal_or8u(src1, sz1, src2, sz2, dst, sz, w, h) TEGRA_BINARYOP(CAROTENE_NS::u8, bitwiseOr, src1, sz1, src2, sz2, dst, sz, w, h) +#undef cv_hal_xor8u +#define cv_hal_xor8u(src1, sz1, src2, sz2, dst, sz, w, h) TEGRA_BINARYOP(CAROTENE_NS::u8, bitwiseXor, src1, sz1, src2, sz2, dst, sz, w, h) +#undef cv_hal_not8u +#define cv_hal_not8u(src1, sz1, dst, sz, w, h) TEGRA_UNARYOP(CAROTENE_NS::u8, bitwiseNot, src1, sz1, dst, sz, w, h) + +TegraBinaryOp_Invoker(cmpEQ, cmpEQ) +TegraBinaryOp_Invoker(cmpNE, cmpNE) +TegraBinaryOp_Invoker(cmpGT, cmpGT) +TegraBinaryOp_Invoker(cmpGE, cmpGE) +TegraGenOp_Invoker(cmpLT, cmpGT, 2, 1, 0, RANGE_DATA(ST, src2_data, src2_step), src2_step, \ + RANGE_DATA(ST, src1_data, src1_step), src1_step, \ + RANGE_DATA(DT, dst1_data, dst1_step), dst1_step) +TegraGenOp_Invoker(cmpLE, cmpGE, 2, 1, 0, RANGE_DATA(ST, src2_data, src2_step), src2_step, \ + RANGE_DATA(ST, src1_data, src1_step), src1_step, \ + RANGE_DATA(DT, dst1_data, dst1_step), dst1_step) +#define TEGRA_CMP(type, src1, sz1, src2, sz2, dst, sz, w, h, op) \ +( \ + CAROTENE_NS::isSupportedConfiguration() ? \ + ((op) == cv::CMP_EQ) ? \ + parallel_for_(Range(0, h), \ + TegraGenOp_cmpEQ_Invoker(src1, sz1, src2, sz2, dst, sz, w, h), \ + (w * h) / static_cast(1<<16)), \ + CV_HAL_ERROR_OK : \ + ((op) == cv::CMP_NE) ? \ + parallel_for_(Range(0, h), \ + TegraGenOp_cmpNE_Invoker(src1, sz1, src2, sz2, dst, sz, w, h), \ + (w * h) / static_cast(1<<16)), \ + CV_HAL_ERROR_OK : \ + ((op) == cv::CMP_GT) ? \ + parallel_for_(Range(0, h), \ + TegraGenOp_cmpGT_Invoker(src1, sz1, src2, sz2, dst, sz, w, h), \ + (w * h) / static_cast(1<<16)), \ + CV_HAL_ERROR_OK : \ + ((op) == cv::CMP_GE) ? \ + parallel_for_(Range(0, h), \ + TegraGenOp_cmpGE_Invoker(src1, sz1, src2, sz2, dst, sz, w, h), \ + (w * h) / static_cast(1<<16)), \ + CV_HAL_ERROR_OK : \ + ((op) == cv::CMP_LT) ? \ + parallel_for_(Range(0, h), \ + TegraGenOp_cmpLT_Invoker(src1, sz1, src2, sz2, dst, sz, w, h), \ + (w * h) / static_cast(1<<16)), \ + CV_HAL_ERROR_OK : \ + ((op) == cv::CMP_LE) ? \ + parallel_for_(Range(0, h), \ + TegraGenOp_cmpLE_Invoker(src1, sz1, src2, sz2, dst, sz, w, h), \ + (w * h) / static_cast(1<<16)), \ + CV_HAL_ERROR_OK : \ + CV_HAL_ERROR_NOT_IMPLEMENTED \ + : CV_HAL_ERROR_NOT_IMPLEMENTED \ +) + +#undef cv_hal_cmp8u +#define cv_hal_cmp8u(src1, sz1, src2, sz2, dst, sz, w, h, op) TEGRA_CMP(CAROTENE_NS::u8, src1, sz1, src2, sz2, dst, sz, w, h, op) +#undef cv_hal_cmp8s +#define cv_hal_cmp8s(src1, sz1, src2, sz2, dst, sz, w, h, op) TEGRA_CMP(CAROTENE_NS::s8, src1, sz1, src2, sz2, dst, sz, w, h, op) +#undef cv_hal_cmp16u +#define cv_hal_cmp16u(src1, sz1, src2, sz2, dst, sz, w, h, op) TEGRA_CMP(CAROTENE_NS::u16, src1, sz1, src2, sz2, dst, sz, w, h, op) +#undef cv_hal_cmp16s +#define cv_hal_cmp16s(src1, sz1, src2, sz2, dst, sz, w, h, op) TEGRA_CMP(CAROTENE_NS::s16, src1, sz1, src2, sz2, dst, sz, w, h, op) +#undef cv_hal_cmp32s +#define cv_hal_cmp32s(src1, sz1, src2, sz2, dst, sz, w, h, op) TEGRA_CMP(CAROTENE_NS::s32, src1, sz1, src2, sz2, dst, sz, w, h, op) +#undef cv_hal_cmp32f +#define cv_hal_cmp32f(src1, sz1, src2, sz2, dst, sz, w, h, op) TEGRA_CMP(CAROTENE_NS::f32, src1, sz1, src2, sz2, dst, sz, w, h, op) +//#undef cv_hal_cmp64f +//#define cv_hal_cmp64f(src1, sz1, src2, sz2, dst, sz, w, h, op) TEGRA_CMP(CAROTENE_NS::f64, src1, sz1, src2, sz2, dst, sz, w, h, op) + +#define TegraBinaryOpScale_Invoker(name, func, scale_cnt, ...) TegraGenOp_Invoker(name, func, 2, 1, scale_cnt, \ + RANGE_DATA(ST, src1_data, src1_step), src1_step, \ + RANGE_DATA(ST, src2_data, src2_step), src2_step, \ + RANGE_DATA(DT, dst1_data, dst1_step), dst1_step, __VA_ARGS__) + +#define TEGRA_BINARYOPSCALE(type, op, src1, sz1, src2, sz2, dst, sz, w, h, scales) \ +( \ + CAROTENE_NS::isSupportedConfiguration() ? \ + parallel_for_(Range(0, h), \ + TegraGenOp_##op##_Invoker(src1, sz1, src2, sz2, dst, sz, w, h, scales), \ + (w * h) / static_cast(1<<16)), \ + CV_HAL_ERROR_OK \ + : CV_HAL_ERROR_NOT_IMPLEMENTED \ +) + +TegraBinaryOpScale_Invoker(mul, mul, 1, scale, CAROTENE_NS::CONVERT_POLICY_SATURATE) + +TegraBinaryOpScale_Invoker(mulf, mul, 1, scale) + +TegraBinaryOpScale_Invoker(div, div, 1, scale, CAROTENE_NS::CONVERT_POLICY_SATURATE) + +TegraBinaryOpScale_Invoker(divf, div, 1, scale) + +#define TegraUnaryOpScale_Invoker(name, func, scale_cnt, ...) TegraGenOp_Invoker(name, func, 1, 1, scale_cnt, \ + RANGE_DATA(ST, src1_data, src1_step), src1_step, \ + RANGE_DATA(DT, dst1_data, dst1_step), dst1_step, __VA_ARGS__) + +#define TEGRA_UNARYOPSCALE(type, op, src1, sz1, dst, sz, w, h, scales) \ +( \ + CAROTENE_NS::isSupportedConfiguration() ? \ + parallel_for_(Range(0, h), \ + TegraGenOp_##op##_Invoker(src1, sz1, dst, sz, w, h, scales), \ + (w * h) / static_cast(1<<16)), \ + CV_HAL_ERROR_OK \ + : CV_HAL_ERROR_NOT_IMPLEMENTED \ +) + +TegraUnaryOpScale_Invoker(recip, reciprocal, 1, scale, CAROTENE_NS::CONVERT_POLICY_SATURATE) + +TegraUnaryOpScale_Invoker(recipf, reciprocal, 1, scale) + +#undef cv_hal_mul8u +#define cv_hal_mul8u(src1, sz1, src2, sz2, dst, sz, w, h, scales) TEGRA_BINARYOPSCALE(CAROTENE_NS::u8, mul, src1, sz1, src2, sz2, dst, sz, w, h, scales) +#undef cv_hal_mul8s +#define cv_hal_mul8s(src1, sz1, src2, sz2, dst, sz, w, h, scales) TEGRA_BINARYOPSCALE(CAROTENE_NS::s8, mul, src1, sz1, src2, sz2, dst, sz, w, h, scales) +#undef cv_hal_mul16u +#define cv_hal_mul16u(src1, sz1, src2, sz2, dst, sz, w, h, scales) TEGRA_BINARYOPSCALE(CAROTENE_NS::u16, mul, src1, sz1, src2, sz2, dst, sz, w, h, scales) +#undef cv_hal_mul16s +#define cv_hal_mul16s(src1, sz1, src2, sz2, dst, sz, w, h, scales) TEGRA_BINARYOPSCALE(CAROTENE_NS::s16, mul, src1, sz1, src2, sz2, dst, sz, w, h, scales) +#undef cv_hal_mul32s +#define cv_hal_mul32s(src1, sz1, src2, sz2, dst, sz, w, h, scales) TEGRA_BINARYOPSCALE(CAROTENE_NS::s32, mul, src1, sz1, src2, sz2, dst, sz, w, h, scales) +#undef cv_hal_mul32f +#define cv_hal_mul32f(src1, sz1, src2, sz2, dst, sz, w, h, scales) TEGRA_BINARYOPSCALE(CAROTENE_NS::f32, mulf, src1, sz1, src2, sz2, dst, sz, w, h, scales) +//#undef cv_hal_mul64f +//#define cv_hal_mul64f(src1, sz1, src2, sz2, dst, sz, w, h, scales) TEGRA_BINARYOPSCALE(CAROTENE_NS::f64, mulf, src1, sz1, src2, sz2, dst, sz, w, h, scales) +#undef cv_hal_div8u +#define cv_hal_div8u(src1, sz1, src2, sz2, dst, sz, w, h, scales) TEGRA_BINARYOPSCALE(CAROTENE_NS::u8, div, src1, sz1, src2, sz2, dst, sz, w, h, scales) +#undef cv_hal_div8s +#define cv_hal_div8s(src1, sz1, src2, sz2, dst, sz, w, h, scales) TEGRA_BINARYOPSCALE(CAROTENE_NS::s8, div, src1, sz1, src2, sz2, dst, sz, w, h, scales) +#undef cv_hal_div16u +#define cv_hal_div16u(src1, sz1, src2, sz2, dst, sz, w, h, scales) TEGRA_BINARYOPSCALE(CAROTENE_NS::u16, div, src1, sz1, src2, sz2, dst, sz, w, h, scales) +#undef cv_hal_div16s +#define cv_hal_div16s(src1, sz1, src2, sz2, dst, sz, w, h, scales) TEGRA_BINARYOPSCALE(CAROTENE_NS::s16, div, src1, sz1, src2, sz2, dst, sz, w, h, scales) +#undef cv_hal_div32s +#define cv_hal_div32s(src1, sz1, src2, sz2, dst, sz, w, h, scales) TEGRA_BINARYOPSCALE(CAROTENE_NS::s32, div, src1, sz1, src2, sz2, dst, sz, w, h, scales) +#undef cv_hal_div32f +#define cv_hal_div32f(src1, sz1, src2, sz2, dst, sz, w, h, scales) TEGRA_BINARYOPSCALE(CAROTENE_NS::f32, divf, src1, sz1, src2, sz2, dst, sz, w, h, scales) +//#undef cv_hal_div64f +//#define cv_hal_div64f(src1, sz1, src2, sz2, dst, sz, w, h, scales) TEGRA_BINARYOPSCALE(CAROTENE_NS::f64, divf, src1, sz1, src2, sz2, dst, sz, w, h, scales) +#undef cv_hal_recip8u +#define cv_hal_recip8u(src1, sz1, dst, sz, w, h, scales) TEGRA_UNARYOPSCALE(CAROTENE_NS::u8, recip, src1, sz1, dst, sz, w, h, scales) +#undef cv_hal_recip8s +#define cv_hal_recip8s(src1, sz1, dst, sz, w, h, scales) TEGRA_UNARYOPSCALE(CAROTENE_NS::s8, recip, src1, sz1, dst, sz, w, h, scales) +#undef cv_hal_recip16u +#define cv_hal_recip16u(src1, sz1, dst, sz, w, h, scales) TEGRA_UNARYOPSCALE(CAROTENE_NS::u16, recip, src1, sz1, dst, sz, w, h, scales) +#undef cv_hal_recip16s +#define cv_hal_recip16s(src1, sz1, dst, sz, w, h, scales) TEGRA_UNARYOPSCALE(CAROTENE_NS::s16, recip, src1, sz1, dst, sz, w, h, scales) +#undef cv_hal_recip32s +#define cv_hal_recip32s(src1, sz1, dst, sz, w, h, scales) TEGRA_UNARYOPSCALE(CAROTENE_NS::s32, recip, src1, sz1, dst, sz, w, h, scales) +#undef cv_hal_recip32f +#define cv_hal_recip32f(src1, sz1, dst, sz, w, h, scales) TEGRA_UNARYOPSCALE(CAROTENE_NS::f32, recipf, src1, sz1, dst, sz, w, h, scales) +//#undef cv_hal_recip64f +//#define cv_hal_recip64f(src1, sz1, dst, sz, w, h, scales) TEGRA_UNARYOPSCALE(CAROTENE_NS::f64, recipf, src1, sz1, dst, sz, w, h, scales) + +TegraBinaryOpScale_Invoker(addWeighted, addWeighted, 3, scales[0], scales[1], scales[2]) + +#undef cv_hal_addWeighted8u +#define cv_hal_addWeighted8u(src1, sz1, src2, sz2, dst, sz, w, h, scales) TEGRA_BINARYOPSCALE(CAROTENE_NS::u8, addWeighted, src1, sz1, src2, sz2, dst, sz, w, h, scales) +#undef cv_hal_addWeighted8s +#define cv_hal_addWeighted8s(src1, sz1, src2, sz2, dst, sz, w, h, scales) TEGRA_BINARYOPSCALE(CAROTENE_NS::s8, addWeighted, src1, sz1, src2, sz2, dst, sz, w, h, scales) +#undef cv_hal_addWeighted16u +#define cv_hal_addWeighted16u(src1, sz1, src2, sz2, dst, sz, w, h, scales) TEGRA_BINARYOPSCALE(CAROTENE_NS::u16, addWeighted, src1, sz1, src2, sz2, dst, sz, w, h, scales) +#undef cv_hal_addWeighted16s +#define cv_hal_addWeighted16s(src1, sz1, src2, sz2, dst, sz, w, h, scales) TEGRA_BINARYOPSCALE(CAROTENE_NS::s16, addWeighted, src1, sz1, src2, sz2, dst, sz, w, h, scales) +#undef cv_hal_addWeighted32s +#define cv_hal_addWeighted32s(src1, sz1, src2, sz2, dst, sz, w, h, scales) TEGRA_BINARYOPSCALE(CAROTENE_NS::s32, addWeighted, src1, sz1, src2, sz2, dst, sz, w, h, scales) +//#undef cv_hal_addWeighted32f +//#define cv_hal_addWeighted32f(src1, sz1, src2, sz2, dst, sz, w, h, scales) TEGRA_BINARYOPSCALE(CAROTENE_NS::f32, addWeighted, src1, sz1, src2, sz2, dst, sz, w, h, scales) +//#undef cv_hal_addWeighted64f +//#define cv_hal_addWeighted64f(src1, sz1, src2, sz2, dst, sz, w, h, scales) TEGRA_BINARYOPSCALE(CAROTENE_NS::f64, addWeighted, src1, sz1, src2, sz2, dst, sz, w, h, scales) + +#else + +#define TEGRA_ADD(src1, sz1, src2, sz2, dst, sz, w, h) \ +( \ + CAROTENE_NS::isSupportedConfiguration() ? \ + CAROTENE_NS::add(CAROTENE_NS::Size2D(w, h), \ + src1, sz1, \ + src2, sz2, \ + dst, sz, \ + CAROTENE_NS::CONVERT_POLICY_SATURATE), /*Original addition use saturated operator*/ \ + /*so use the same from CAROTENE*/ \ + CV_HAL_ERROR_OK \ + : CV_HAL_ERROR_NOT_IMPLEMENTED \ +) + +#define TEGRA_ADDF(src1, sz1, src2, sz2, dst, sz, w, h) \ +( \ + CAROTENE_NS::isSupportedConfiguration() ? \ + CAROTENE_NS::add(CAROTENE_NS::Size2D(w, h), \ + src1, sz1, \ + src2, sz2, \ + dst, sz), \ + CV_HAL_ERROR_OK \ + : CV_HAL_ERROR_NOT_IMPLEMENTED \ +) + +#define TEGRA_SUB(src1, sz1, src2, sz2, dst, sz, w, h) \ +( \ + CAROTENE_NS::isSupportedConfiguration() ? \ + CAROTENE_NS::sub(CAROTENE_NS::Size2D(w, h), \ + src1, sz1, \ + src2, sz2, \ + dst, sz, \ + CAROTENE_NS::CONVERT_POLICY_SATURATE), /*Original addition use saturated operator*/ \ + /*so use the same from CAROTENE*/ \ + CV_HAL_ERROR_OK \ + : CV_HAL_ERROR_NOT_IMPLEMENTED \ +) + +#define TEGRA_SUBF(src1, sz1, src2, sz2, dst, sz, w, h) \ +( \ + CAROTENE_NS::isSupportedConfiguration() ? \ + CAROTENE_NS::sub(CAROTENE_NS::Size2D(w, h), \ + src1, sz1, \ + src2, sz2, \ + dst, sz), \ + CV_HAL_ERROR_OK \ + : CV_HAL_ERROR_NOT_IMPLEMENTED \ +) + +#define TEGRA_MAX(src1, sz1, src2, sz2, dst, sz, w, h) \ +( \ + CAROTENE_NS::isSupportedConfiguration() ? \ + CAROTENE_NS::max(CAROTENE_NS::Size2D(w, h), \ + src1, sz1, \ + src2, sz2, \ + dst, sz), \ + CV_HAL_ERROR_OK \ + : CV_HAL_ERROR_NOT_IMPLEMENTED \ +) + +#define TEGRA_MIN(src1, sz1, src2, sz2, dst, sz, w, h) \ +( \ + CAROTENE_NS::isSupportedConfiguration() ? \ + CAROTENE_NS::min(CAROTENE_NS::Size2D(w, h), \ + src1, sz1, \ + src2, sz2, \ + dst, sz), \ + CV_HAL_ERROR_OK \ + : CV_HAL_ERROR_NOT_IMPLEMENTED \ +) + +#define TEGRA_ABSDIFF(src1, sz1, src2, sz2, dst, sz, w, h) \ +( \ + CAROTENE_NS::isSupportedConfiguration() ? \ + CAROTENE_NS::absDiff(CAROTENE_NS::Size2D(w, h), \ + src1, sz1, \ + src2, sz2, \ + dst, sz), \ + CV_HAL_ERROR_OK \ + : CV_HAL_ERROR_NOT_IMPLEMENTED \ +) + +#define TEGRA_AND(src1, sz1, src2, sz2, dst, sz, w, h) \ +( \ + CAROTENE_NS::isSupportedConfiguration() ? \ + CAROTENE_NS::bitwiseAnd(CAROTENE_NS::Size2D(w, h), \ + src1, sz1, \ + src2, sz2, \ + dst, sz), \ + CV_HAL_ERROR_OK \ + : CV_HAL_ERROR_NOT_IMPLEMENTED \ +) +#define TEGRA_OR(src1, sz1, src2, sz2, dst, sz, w, h) \ +( \ + CAROTENE_NS::isSupportedConfiguration() ? \ + CAROTENE_NS::bitwiseOr(CAROTENE_NS::Size2D(w, h), \ + src1, sz1, \ + src2, sz2, \ + dst, sz), \ + CV_HAL_ERROR_OK \ + : CV_HAL_ERROR_NOT_IMPLEMENTED \ +) + +#define TEGRA_XOR(src1, sz1, src2, sz2, dst, sz, w, h) \ +( \ + CAROTENE_NS::isSupportedConfiguration() ? \ + CAROTENE_NS::bitwiseXor(CAROTENE_NS::Size2D(w, h), \ + src1, sz1, \ + src2, sz2, \ + dst, sz), \ + CV_HAL_ERROR_OK \ + : CV_HAL_ERROR_NOT_IMPLEMENTED \ +) + +#define TEGRA_NOT(src1, sz1, dst, sz, w, h) \ +( \ + CAROTENE_NS::isSupportedConfiguration() ? \ + CAROTENE_NS::bitwiseNot(CAROTENE_NS::Size2D(w, h), \ + src1, sz1, \ + dst, sz), \ + CV_HAL_ERROR_OK \ + : CV_HAL_ERROR_NOT_IMPLEMENTED \ +) + +#undef cv_hal_add8u +#define cv_hal_add8u TEGRA_ADD +#undef cv_hal_add8s +#define cv_hal_add8s TEGRA_ADD +#undef cv_hal_add16u +#define cv_hal_add16u TEGRA_ADD +#undef cv_hal_add16s +#define cv_hal_add16s TEGRA_ADD +#undef cv_hal_add32s +#define cv_hal_add32s TEGRA_ADD +#undef cv_hal_add32f +#define cv_hal_add32f TEGRA_ADDF +//#undef cv_hal_add64f +//#define cv_hal_add64f TEGRA_ADDF +#undef cv_hal_sub8u +#define cv_hal_sub8u TEGRA_SUB +#undef cv_hal_sub8s +#define cv_hal_sub8s TEGRA_SUB +#undef cv_hal_sub16u +#define cv_hal_sub16u TEGRA_SUB +#undef cv_hal_sub16s +#define cv_hal_sub16s TEGRA_SUB +#undef cv_hal_sub32s +#define cv_hal_sub32s TEGRA_SUB +#undef cv_hal_sub32f +#define cv_hal_sub32f TEGRA_SUBF +//#undef cv_hal_sub64f +//#define cv_hal_sub64f TEGRA_SUBF +#undef cv_hal_max8u +#define cv_hal_max8u TEGRA_MAX +#undef cv_hal_max8s +#define cv_hal_max8s TEGRA_MAX +#undef cv_hal_max16u +#define cv_hal_max16u TEGRA_MAX +#undef cv_hal_max16s +#define cv_hal_max16s TEGRA_MAX +#undef cv_hal_max32s +#define cv_hal_max32s TEGRA_MAX +#undef cv_hal_max32f +#define cv_hal_max32f TEGRA_MAX +//#undef cv_hal_max64f +//#define cv_hal_max64f TEGRA_MAX +#undef cv_hal_min8u +#define cv_hal_min8u TEGRA_MIN +#undef cv_hal_min8s +#define cv_hal_min8s TEGRA_MIN +#undef cv_hal_min16u +#define cv_hal_min16u TEGRA_MIN +#undef cv_hal_min16s +#define cv_hal_min16s TEGRA_MIN +#undef cv_hal_min32s +#define cv_hal_min32s TEGRA_MIN +#undef cv_hal_min32f +#define cv_hal_min32f TEGRA_MIN +//#undef cv_hal_min64f +//#define cv_hal_min64f TEGRA_MIN +#undef cv_hal_absdiff8u +#define cv_hal_absdiff8u TEGRA_ABSDIFF +#undef cv_hal_absdiff8s +#define cv_hal_absdiff8s TEGRA_ABSDIFF +#undef cv_hal_absdiff16u +#define cv_hal_absdiff16u TEGRA_ABSDIFF +#undef cv_hal_absdiff16s +#define cv_hal_absdiff16s TEGRA_ABSDIFF +#undef cv_hal_absdiff32s +#define cv_hal_absdiff32s TEGRA_ABSDIFF +#undef cv_hal_absdiff32f +#define cv_hal_absdiff32f TEGRA_ABSDIFF +//#undef cv_hal_absdiff64f +//#define cv_hal_absdiff64f TEGRA_ABSDIFF +#undef cv_hal_and8u +#define cv_hal_and8u TEGRA_AND +#undef cv_hal_or8u +#define cv_hal_or8u TEGRA_OR +#undef cv_hal_xor8u +#define cv_hal_xor8u TEGRA_XOR +#undef cv_hal_not8u +#define cv_hal_not8u TEGRA_NOT + +#define TEGRA_CMP(src1, sz1, src2, sz2, dst, sz, w, h, op) \ +( \ + CAROTENE_NS::isSupportedConfiguration() ? \ + ((op) == cv::CMP_EQ) ? \ + CAROTENE_NS::cmpEQ(CAROTENE_NS::Size2D(w, h), \ + src1, sz1, \ + src2, sz2, \ + dst, sz), \ + CV_HAL_ERROR_OK : \ + ((op) == cv::CMP_NE) ? \ + CAROTENE_NS::cmpNE(CAROTENE_NS::Size2D(w, h), \ + src1, sz1, \ + src2, sz2, \ + dst, sz), \ + CV_HAL_ERROR_OK : \ + ((op) == cv::CMP_GT) ? \ + CAROTENE_NS::cmpGT(CAROTENE_NS::Size2D(w, h), \ + src1, sz1, \ + src2, sz2, \ + dst, sz), \ + CV_HAL_ERROR_OK : \ + ((op) == cv::CMP_GE) ? \ + CAROTENE_NS::cmpGE(CAROTENE_NS::Size2D(w, h), \ + src1, sz1, \ + src2, sz2, \ + dst, sz), \ + CV_HAL_ERROR_OK : \ + ((op) == cv::CMP_LT) ? \ + CAROTENE_NS::cmpGT(CAROTENE_NS::Size2D(w, h), \ + src2, sz2, \ + src1, sz1, \ + dst, sz), \ + CV_HAL_ERROR_OK : \ + ((op) == cv::CMP_LE) ? \ + CAROTENE_NS::cmpGE(CAROTENE_NS::Size2D(w, h), \ + src2, sz2, \ + src1, sz1, \ + dst, sz), \ + CV_HAL_ERROR_OK : \ + CV_HAL_ERROR_NOT_IMPLEMENTED \ + : CV_HAL_ERROR_NOT_IMPLEMENTED \ +) + +#undef cv_hal_cmp8u +#define cv_hal_cmp8u TEGRA_CMP +#undef cv_hal_cmp8s +#define cv_hal_cmp8s TEGRA_CMP +#undef cv_hal_cmp16u +#define cv_hal_cmp16u TEGRA_CMP +#undef cv_hal_cmp16s +#define cv_hal_cmp16s TEGRA_CMP +#undef cv_hal_cmp32s +#define cv_hal_cmp32s TEGRA_CMP +#undef cv_hal_cmp32f +#define cv_hal_cmp32f TEGRA_CMP +//#undef cv_hal_cmp64f +//#define cv_hal_cmp64f TEGRA_CMP + +#define TEGRA_MUL(src1, sz1, src2, sz2, dst, sz, w, h, scale) \ +( \ + CAROTENE_NS::isSupportedConfiguration() ? \ + CAROTENE_NS::mul(CAROTENE_NS::Size2D(w, h), \ + src1, sz1, \ + src2, sz2, \ + dst, sz, \ + scale, \ + CAROTENE_NS::CONVERT_POLICY_SATURATE), \ + CV_HAL_ERROR_OK \ + : CV_HAL_ERROR_NOT_IMPLEMENTED \ +) + +#define TEGRA_MULF(src1, sz1, src2, sz2, dst, sz, w, h, scale) \ +( \ + CAROTENE_NS::isSupportedConfiguration() ? \ + CAROTENE_NS::mul(CAROTENE_NS::Size2D(w, h), \ + src1, sz1, \ + src2, sz2, \ + dst, sz, \ + (float)scale), \ + CV_HAL_ERROR_OK \ + : CV_HAL_ERROR_NOT_IMPLEMENTED \ +) + +#define TEGRA_DIV(src1, sz1, src2, sz2, dst, sz, w, h, scale) \ +( \ + CAROTENE_NS::isSupportedConfiguration() ? \ + CAROTENE_NS::div(CAROTENE_NS::Size2D(w, h), \ + src1, sz1, \ + src2, sz2, \ + dst, sz, \ + scale, \ + CAROTENE_NS::CONVERT_POLICY_SATURATE), \ + CV_HAL_ERROR_OK \ + : CV_HAL_ERROR_NOT_IMPLEMENTED \ +) + +#define TEGRA_DIVF(src1, sz1, src2, sz2, dst, sz, w, h, scale) \ +( \ + CAROTENE_NS::isSupportedConfiguration() ? \ + CAROTENE_NS::div(CAROTENE_NS::Size2D(w, h), \ + src1, sz1, \ + src2, sz2, \ + dst, sz, \ + (float)scale), \ + CV_HAL_ERROR_OK \ + : CV_HAL_ERROR_NOT_IMPLEMENTED \ +) + +#define TEGRA_RECIP(src2, sz2, dst, sz, w, h, scale) \ +( \ + CAROTENE_NS::isSupportedConfiguration() ? \ + CAROTENE_NS::reciprocal(CAROTENE_NS::Size2D(w, h), \ + src2, sz2, \ + dst, sz, \ + scale, \ + CAROTENE_NS::CONVERT_POLICY_SATURATE), \ + CV_HAL_ERROR_OK \ + : CV_HAL_ERROR_NOT_IMPLEMENTED \ +) + +#define TEGRA_RECIPF(src2, sz2, dst, sz, w, h, scale) \ +( \ + CAROTENE_NS::isSupportedConfiguration() ? \ + CAROTENE_NS::reciprocal(CAROTENE_NS::Size2D(w, h), \ + src2, sz2, \ + dst, sz, \ + (float)scale), \ + CV_HAL_ERROR_OK \ + : CV_HAL_ERROR_NOT_IMPLEMENTED \ +) + +#undef cv_hal_mul8u +#define cv_hal_mul8u TEGRA_MUL +#undef cv_hal_mul8s +#define cv_hal_mul8s TEGRA_MUL +#undef cv_hal_mul16u +#define cv_hal_mul16u TEGRA_MUL +#undef cv_hal_mul16s +#define cv_hal_mul16s TEGRA_MUL +#undef cv_hal_mul32s +#define cv_hal_mul32s TEGRA_MUL +#undef cv_hal_mul32f +#define cv_hal_mul32f TEGRA_MULF +//#undef cv_hal_mul64f +//#define cv_hal_mul64f TEGRA_MULF +#undef cv_hal_div8u +#define cv_hal_div8u TEGRA_DIV +#undef cv_hal_div8s +#define cv_hal_div8s TEGRA_DIV +#undef cv_hal_div16u +#define cv_hal_div16u TEGRA_DIV +#undef cv_hal_div16s +#define cv_hal_div16s TEGRA_DIV +#undef cv_hal_div32s +#define cv_hal_div32s TEGRA_DIV +#undef cv_hal_div32f +#define cv_hal_div32f TEGRA_DIVF +//#undef cv_hal_div64f +//#define cv_hal_div64f TEGRA_DIVF +#undef cv_hal_recip8u +#define cv_hal_recip8u TEGRA_RECIP +#undef cv_hal_recip8s +#define cv_hal_recip8s TEGRA_RECIP +#undef cv_hal_recip16u +#define cv_hal_recip16u TEGRA_RECIP +#undef cv_hal_recip16s +#define cv_hal_recip16s TEGRA_RECIP +#undef cv_hal_recip32s +#define cv_hal_recip32s TEGRA_RECIP +#undef cv_hal_recip32f +#define cv_hal_recip32f TEGRA_RECIPF +//#undef cv_hal_recip64f +//#define cv_hal_recip64f TEGRA_RECIPF + +#define TEGRA_ADDWEIGHTED(src1, sz1, src2, sz2, dst, sz, w, h, scales) \ +( \ + CAROTENE_NS::isSupportedConfiguration() ? \ + CAROTENE_NS::addWeighted(CAROTENE_NS::Size2D(w, h), \ + src1, sz1, \ + src2, sz2, \ + dst, sz, \ + ((double *)scales)[0], ((double *)scales)[1], ((double *)scales)[2]), \ + CV_HAL_ERROR_OK \ + : CV_HAL_ERROR_NOT_IMPLEMENTED \ +) + +#undef cv_hal_addWeighted8u +#define cv_hal_addWeighted8u TEGRA_ADDWEIGHTED +#undef cv_hal_addWeighted8s +#define cv_hal_addWeighted8s TEGRA_ADDWEIGHTED +#undef cv_hal_addWeighted16u +#define cv_hal_addWeighted16u TEGRA_ADDWEIGHTED +#undef cv_hal_addWeighted16s +#define cv_hal_addWeighted16s TEGRA_ADDWEIGHTED +#undef cv_hal_addWeighted32s +#define cv_hal_addWeighted32s TEGRA_ADDWEIGHTED +//#undef cv_hal_addWeighted32f +//#define cv_hal_addWeighted32f TEGRA_ADDWEIGHTED +//#undef cv_hal_addWeighted64f +//#define cv_hal_addWeighted64f TEGRA_ADDWEIGHTED + +#endif //PARALLEL_CORE + +#define ROW_SRC_ARG1 const ST * src1_data_ +#define ROW_SRC_STORE1 , src1_data(src1_data_) +#define ROW_SRC_VAR1 const ST * src1_data; +#define ROW_SRC_ARG2 ROW_SRC_ARG1 \ + , const ST * src2_data_ +#define ROW_SRC_STORE2 ROW_SRC_STORE1 \ + , src2_data(src2_data_) +#define ROW_SRC_VAR2 ROW_SRC_VAR1 \ + const ST * src2_data; +#define ROW_SRC_ARG3 ROW_SRC_ARG2 \ + , const ST * src3_data_ +#define ROW_SRC_STORE3 ROW_SRC_STORE2 \ + , src3_data(src3_data_) +#define ROW_SRC_VAR3 ROW_SRC_VAR2 \ + const ST * src3_data; +#define ROW_SRC_ARG4 ROW_SRC_ARG3 \ + , const ST * src4_data_ +#define ROW_SRC_STORE4 ROW_SRC_STORE3 \ + , src4_data(src4_data_) +#define ROW_SRC_VAR4 ROW_SRC_VAR3 \ + const ST * src4_data; + +#define ROW_DST_ARG1 , DT * dst1_data_ +#define ROW_DST_STORE1 , dst1_data(dst1_data_) +#define ROW_DST_VAR1 DT * dst1_data; +#define ROW_DST_ARG2 ROW_DST_ARG1 \ + , DT * dst2_data_ +#define ROW_DST_STORE2 ROW_DST_STORE1 \ + , dst2_data(dst2_data_) +#define ROW_DST_VAR2 ROW_DST_VAR1 \ + DT * dst2_data; +#define ROW_DST_ARG3 ROW_DST_ARG2 \ + , DT * dst3_data_ +#define ROW_DST_STORE3 ROW_DST_STORE2 \ + , dst3_data(dst3_data_) +#define ROW_DST_VAR3 ROW_DST_VAR2 \ + DT * dst3_data; +#define ROW_DST_ARG4 ROW_DST_ARG3 \ + , DT * dst4_data_ +#define ROW_DST_STORE4 ROW_DST_STORE3 \ + , dst4_data(dst4_data_) +#define ROW_DST_VAR4 ROW_DST_VAR3 \ + DT * dst4_data; + +#define ROW_VAL_ARG0 +#define ROW_VAL_STORE0 +#define ROW_VAL_VAR0 +#define ROW_VAL_ARG1 , double val_ +#define ROW_VAL_STORE1 , val(val_) +#define ROW_VAL_VAR1 double val; + +#define TegraRowOp_Invoker(name, func, src_cnt, dst_cnt, val_cnt, ...) \ +template \ +class TegraRowOp_##name##_Invoker : public cv::ParallelLoopBody \ +{ \ +public: \ + TegraRowOp_##name##_Invoker(ROW_SRC_ARG##src_cnt \ + ROW_DST_ARG##dst_cnt \ + ROW_VAL_ARG##val_cnt) : \ + cv::ParallelLoopBody() ROW_SRC_STORE##src_cnt \ + ROW_DST_STORE##dst_cnt \ + ROW_VAL_STORE##val_cnt {} \ + virtual void operator()(const cv::Range& range) const \ + { \ + CAROTENE_NS::func(CAROTENE_NS::Size2D(range.end-range.start, 1), __VA_ARGS__); \ + } \ +private: \ + ROW_SRC_VAR##src_cnt \ + ROW_DST_VAR##dst_cnt \ + ROW_VAL_VAR##val_cnt \ + const TegraRowOp_##name##_Invoker& operator= (const TegraRowOp_##name##_Invoker&); \ +}; + + +#define TEGRA_SPLIT(src, dst, len, cn) \ +( \ + CAROTENE_NS::isSupportedConfiguration() ? \ + cn == 2 ? \ + CAROTENE_NS::split2(CAROTENE_NS::Size2D(len, 1), \ + src, len, \ + dst[0], len, \ + dst[1], len), \ + CV_HAL_ERROR_OK : \ + cn == 3 ? \ + CAROTENE_NS::split3(CAROTENE_NS::Size2D(len, 1), \ + src, len, \ + dst[0], len, \ + dst[1], len, \ + dst[2], len), \ + CV_HAL_ERROR_OK : \ + cn == 4 ? \ + CAROTENE_NS::split4(CAROTENE_NS::Size2D(len, 1), \ + src, len, \ + dst[0], len, \ + dst[1], len, \ + dst[2], len, \ + dst[3], len), \ + CV_HAL_ERROR_OK : \ + CV_HAL_ERROR_NOT_IMPLEMENTED \ + : CV_HAL_ERROR_NOT_IMPLEMENTED \ +) + +TegraRowOp_Invoker(split2, split2, 1, 2, 0, RANGE_DATA(ST, src1_data, 2*sizeof(ST)), range.end-range.start, + RANGE_DATA(DT, dst1_data, sizeof(DT)), range.end-range.start, + RANGE_DATA(DT, dst2_data, sizeof(DT)), range.end-range.start) +TegraRowOp_Invoker(split3, split3, 1, 3, 0, RANGE_DATA(ST, src1_data, 3*sizeof(ST)), range.end-range.start, + RANGE_DATA(DT, dst1_data, sizeof(DT)), range.end-range.start, + RANGE_DATA(DT, dst2_data, sizeof(DT)), range.end-range.start, + RANGE_DATA(DT, dst3_data, sizeof(DT)), range.end-range.start) +TegraRowOp_Invoker(split4, split4, 1, 4, 0, RANGE_DATA(ST, src1_data, 4*sizeof(ST)), range.end-range.start, + RANGE_DATA(DT, dst1_data, sizeof(DT)), range.end-range.start, + RANGE_DATA(DT, dst2_data, sizeof(DT)), range.end-range.start, + RANGE_DATA(DT, dst3_data, sizeof(DT)), range.end-range.start, + RANGE_DATA(DT, dst4_data, sizeof(DT)), range.end-range.start) +#define TEGRA_SPLIT64S(type, src, dst, len, cn) \ +( \ + CAROTENE_NS::isSupportedConfiguration() ? \ + cn == 2 ? \ + parallel_for_(Range(0, len), \ + TegraRowOp_split2_Invoker(src, dst[0], dst[1]), \ + (len) / static_cast(1<<16)), \ + CV_HAL_ERROR_OK : \ + cn == 3 ? \ + parallel_for_(Range(0, len), \ + TegraRowOp_split3_Invoker(src, dst[0], dst[1], dst[2]), \ + (len) / static_cast(1<<16)), \ + CV_HAL_ERROR_OK : \ + cn == 4 ? \ + parallel_for_(Range(0, len), \ + TegraRowOp_split4_Invoker(src, dst[0], dst[1], dst[2], dst[3]), \ + (len) / static_cast(1<<16)), \ + CV_HAL_ERROR_OK : \ + CV_HAL_ERROR_NOT_IMPLEMENTED \ + : CV_HAL_ERROR_NOT_IMPLEMENTED \ +) + +#define TEGRA_MERGE(src, dst, len, cn) \ +( \ + CAROTENE_NS::isSupportedConfiguration() ? \ + cn == 2 ? \ + CAROTENE_NS::combine2(CAROTENE_NS::Size2D(len, 1), \ + src[0], len, \ + src[1], len, \ + dst, len), \ + CV_HAL_ERROR_OK : \ + cn == 3 ? \ + CAROTENE_NS::combine3(CAROTENE_NS::Size2D(len, 1), \ + src[0], len, \ + src[1], len, \ + src[2], len, \ + dst, len), \ + CV_HAL_ERROR_OK : \ + cn == 4 ? \ + CAROTENE_NS::combine4(CAROTENE_NS::Size2D(len, 1), \ + src[0], len, \ + src[1], len, \ + src[2], len, \ + src[3], len, \ + dst, len), \ + CV_HAL_ERROR_OK : \ + CV_HAL_ERROR_NOT_IMPLEMENTED \ + : CV_HAL_ERROR_NOT_IMPLEMENTED \ +) + +TegraRowOp_Invoker(combine2, combine2, 2, 1, 0, RANGE_DATA(ST, src1_data, sizeof(ST)), range.end-range.start, + RANGE_DATA(ST, src2_data, sizeof(ST)), range.end-range.start, + RANGE_DATA(DT, dst1_data, 2*sizeof(DT)), range.end-range.start) +TegraRowOp_Invoker(combine3, combine3, 3, 1, 0, RANGE_DATA(ST, src1_data, sizeof(ST)), range.end-range.start, + RANGE_DATA(ST, src2_data, sizeof(ST)), range.end-range.start, + RANGE_DATA(ST, src3_data, sizeof(ST)), range.end-range.start, + RANGE_DATA(DT, dst1_data, 3*sizeof(DT)), range.end-range.start) +TegraRowOp_Invoker(combine4, combine4, 4, 1, 0, RANGE_DATA(ST, src1_data, sizeof(ST)), range.end-range.start, + RANGE_DATA(ST, src2_data, sizeof(ST)), range.end-range.start, + RANGE_DATA(ST, src3_data, sizeof(ST)), range.end-range.start, + RANGE_DATA(ST, src4_data, sizeof(ST)), range.end-range.start, + RANGE_DATA(DT, dst1_data, 4*sizeof(DT)), range.end-range.start) +#define TEGRA_MERGE64S(type, src, dst, len, cn) \ +( \ + CAROTENE_NS::isSupportedConfiguration() ? \ + cn == 2 ? \ + parallel_for_(Range(0, len), \ + TegraRowOp_combine2_Invoker(src[0], src[1], dst), \ + (len) / static_cast(1<<16)), \ + CV_HAL_ERROR_OK : \ + cn == 3 ? \ + parallel_for_(Range(0, len), \ + TegraRowOp_combine3_Invoker(src[0], src[1], src[2], dst), \ + (len) / static_cast(1<<16)), \ + CV_HAL_ERROR_OK : \ + cn == 4 ? \ + parallel_for_(Range(0, len), \ + TegraRowOp_combine4_Invoker(src[0], src[1], src[2], src[3], dst), \ + (len) / static_cast(1<<16)), \ + CV_HAL_ERROR_OK : \ + CV_HAL_ERROR_NOT_IMPLEMENTED \ + : CV_HAL_ERROR_NOT_IMPLEMENTED \ +) + +#undef cv_hal_split8u +#define cv_hal_split8u TEGRA_SPLIT +#undef cv_hal_split16u +#define cv_hal_split16u TEGRA_SPLIT +#undef cv_hal_split32s +#define cv_hal_split32s TEGRA_SPLIT +#undef cv_hal_split64s +#define cv_hal_split64s(src, dst, len, cn) TEGRA_SPLIT64S(CAROTENE_NS::s64, src, dst, len, cn) + +#undef cv_hal_merge8u +#define cv_hal_merge8u TEGRA_MERGE +#undef cv_hal_merge16u +#define cv_hal_merge16u TEGRA_MERGE +#undef cv_hal_merge32s +#define cv_hal_merge32s TEGRA_MERGE +#undef cv_hal_merge64s +#define cv_hal_merge64s(src, dst, len, cn) TEGRA_MERGE64S(CAROTENE_NS::s64, src, dst, len, cn) + + +TegraRowOp_Invoker(phase, phase, 2, 1, 1, RANGE_DATA(ST, src1_data, sizeof(CAROTENE_NS::f32)), range.end-range.start, + RANGE_DATA(ST, src2_data, sizeof(CAROTENE_NS::f32)), range.end-range.start, + RANGE_DATA(DT, dst1_data, sizeof(CAROTENE_NS::f32)), range.end-range.start, val) +#define TEGRA_FASTATAN(y, x, dst, len, angleInDegrees) \ +( \ + CAROTENE_NS::isSupportedConfiguration() ? \ + parallel_for_(Range(0, len), \ + TegraRowOp_phase_Invoker(x, y, dst, angleInDegrees ? 1.0f : M_PI/180), \ + (len) / static_cast(1<<16)), \ + CV_HAL_ERROR_OK \ + : CV_HAL_ERROR_NOT_IMPLEMENTED \ +) + +#undef cv_hal_fastAtan32f +#define cv_hal_fastAtan32f TEGRA_FASTATAN + +TegraRowOp_Invoker(magnitude, magnitude, 2, 1, 0, RANGE_DATA(ST, src1_data, sizeof(CAROTENE_NS::f32)), range.end-range.start, + RANGE_DATA(ST, src2_data, sizeof(CAROTENE_NS::f32)), range.end-range.start, + RANGE_DATA(DT, dst1_data, sizeof(CAROTENE_NS::f32)), range.end-range.start) +#define TEGRA_MAGNITUDE(x, y, dst, len) \ +( \ + CAROTENE_NS::isSupportedConfiguration() ? \ + parallel_for_(Range(0, len), \ + TegraRowOp_magnitude_Invoker(x, y, dst), \ + (len) / static_cast(1<<16)), \ + CV_HAL_ERROR_OK \ + : CV_HAL_ERROR_NOT_IMPLEMENTED \ +) + +#undef cv_hal_magnitude32f +#define cv_hal_magnitude32f TEGRA_MAGNITUDE + + +#if defined OPENCV_IMGPROC_HAL_INTERFACE_H + +struct cvhalFilter2D; + +struct FilterCtx +{ + CAROTENE_NS::Size2D ksize; + CAROTENE_NS::s16* kernel_data; + CAROTENE_NS::BORDER_MODE border; +}; +inline int TEGRA_FILTERINIT(cvhalFilter2D **context, uchar *kernel_data, size_t kernel_step, int kernel_type, int kernel_width, int kernel_height, + int max_width, int max_height, int src_type, int dst_type, int borderType, double delta, int anchor_x, int anchor_y, bool allowSubmatrix, bool allowInplace) +{ + if(!context || !kernel_data || allowSubmatrix || allowInplace || + src_type != CV_8UC1 || dst_type != CV_8UC1 || + delta != 0 || anchor_x != kernel_width / 2 || anchor_y != kernel_height / 2 ) + return CV_HAL_ERROR_NOT_IMPLEMENTED; + + FilterCtx* ctx = new FilterCtx; + if(!ctx) + return CV_HAL_ERROR_UNKNOWN; + ctx->ksize.width = kernel_width; + ctx->ksize.height = kernel_height; + switch(borderType) + { + case CV_HAL_BORDER_CONSTANT: + ctx->border = CAROTENE_NS::BORDER_MODE_CONSTANT; + break; + case CV_HAL_BORDER_REPLICATE: + ctx->border = CAROTENE_NS::BORDER_MODE_REPLICATE; + break; + case CV_HAL_BORDER_REFLECT: + ctx->border = CAROTENE_NS::BORDER_MODE_REFLECT; + break; + case CV_HAL_BORDER_WRAP: + ctx->border = CAROTENE_NS::BORDER_MODE_WRAP; + break; + case CV_HAL_BORDER_REFLECT_101: + ctx->border = CAROTENE_NS::BORDER_MODE_REFLECT101; + break; + default: + delete ctx; + return CV_HAL_ERROR_NOT_IMPLEMENTED; + } + + if(!CAROTENE_NS::isConvolutionSupported(CAROTENE_NS::Size2D(max_width, max_height), ctx->ksize, ctx->border)) + { + delete ctx; + return CV_HAL_ERROR_NOT_IMPLEMENTED; + } + + ctx->kernel_data = new CAROTENE_NS::s16[kernel_width*kernel_height]; + if(!ctx->kernel_data) + return CV_HAL_ERROR_UNKNOWN; + switch(kernel_type) + { + case CV_8UC1: + convert(ctx->ksize, (CAROTENE_NS::u8*)kernel_data, kernel_step, ctx->kernel_data, kernel_width); + break; + case CV_8SC1: + convert(ctx->ksize, (CAROTENE_NS::s8*)kernel_data, kernel_step, ctx->kernel_data, kernel_width); + break; + case CV_16UC1: + for(int j = 0; j < kernel_height; ++j) + { + std::memcpy(ctx->kernel_data + kernel_width * j, kernel_data + kernel_step * j, kernel_width * sizeof(int16_t)); + } + default: + delete[] ctx->kernel_data; + delete ctx; + return CV_HAL_ERROR_NOT_IMPLEMENTED; + } + + *context = (cvhalFilter2D*)(ctx); + return CV_HAL_ERROR_OK; +} +inline int TEGRA_FILTERFREE(cvhalFilter2D *context) +{ + if(context) + { + if(((FilterCtx*)context)->kernel_data) + delete[] ((FilterCtx*)context)->kernel_data; + delete (FilterCtx*)context; + return CV_HAL_ERROR_OK; + } + else + { + return CV_HAL_ERROR_UNKNOWN; + } +} +#define TEGRA_FILTERIMPL(context, src_data, src_step, dst_data, dst_step, width, height, full_width, full_height, offset_x, offset_y) \ +( \ + (void)full_width, (void)full_height, (void)offset_x, (void)offset_y, \ + context && CAROTENE_NS::isConvolutionSupported(CAROTENE_NS::Size2D(width, height), ((FilterCtx*)context)->ksize, ((FilterCtx*)context)->border) ? \ + CAROTENE_NS::convolution(CAROTENE_NS::Size2D(width, height), \ + src_data, src_step, \ + dst_data, dst_step, \ + ((FilterCtx*)context)->border, 0, \ + ((FilterCtx*)context)->ksize, ((FilterCtx*)context)->kernel_data, 1), \ + CV_HAL_ERROR_OK \ + : CV_HAL_ERROR_NOT_IMPLEMENTED \ +) + +#undef cv_hal_filterInit +#define cv_hal_filterInit TEGRA_FILTERINIT +#undef cv_hal_filter +#define cv_hal_filter TEGRA_FILTERIMPL +#undef cv_hal_filterFree +#define cv_hal_filterFree TEGRA_FILTERFREE + + +struct SepFilterCtx +{ + int16_t kernelx_data[3]; + int16_t kernely_data[3]; + CAROTENE_NS::BORDER_MODE border; +}; +inline int TEGRA_SEPFILTERINIT(cvhalFilter2D **context, int src_type, int dst_type, int kernel_type, + uchar *kernelx_data, size_t , int kernelx_width, int kernelx_height, + uchar *kernely_data, size_t kernely_step, int kernely_width, int kernely_height, + int anchor_x, int anchor_y, double delta, int borderType) +{ + if(!context || !kernelx_data || !kernely_data || src_type != CV_8UC1 || dst_type != CV_16SC1 || + !(kernelx_width == 3 && kernelx_height == 1) || !(kernely_width == 1 && kernely_height == 3) || + delta != 0 || anchor_x != 1 || anchor_y != 1) + return CV_HAL_ERROR_NOT_IMPLEMENTED; + + SepFilterCtx* ctx = new SepFilterCtx; + if(!ctx) + return CV_HAL_ERROR_UNKNOWN; + switch(borderType) + { + case CV_HAL_BORDER_CONSTANT: + ctx->border = CAROTENE_NS::BORDER_MODE_CONSTANT; + break; + case CV_HAL_BORDER_REPLICATE: + ctx->border = CAROTENE_NS::BORDER_MODE_REPLICATE; + break; + case CV_HAL_BORDER_REFLECT: + ctx->border = CAROTENE_NS::BORDER_MODE_REFLECT; + break; + case CV_HAL_BORDER_WRAP: + ctx->border = CAROTENE_NS::BORDER_MODE_WRAP; + break; + case CV_HAL_BORDER_REFLECT_101: + ctx->border = CAROTENE_NS::BORDER_MODE_REFLECT101; + break; + default: + delete ctx; + return CV_HAL_ERROR_NOT_IMPLEMENTED; + } + + if(!CAROTENE_NS::isSeparableFilter3x3Supported(CAROTENE_NS::Size2D(16, 16), ctx->border, 3, 3)) + { + delete ctx; + return CV_HAL_ERROR_NOT_IMPLEMENTED; + } + + switch(kernel_type) + { + case CV_8UC1: + ctx->kernelx_data[0]=kernelx_data[0]; + ctx->kernelx_data[1]=kernelx_data[1]; + ctx->kernelx_data[2]=kernelx_data[2]; + ctx->kernely_data[0]=kernely_data[0]; + ctx->kernely_data[1]=kernely_data[kernely_step]; + ctx->kernely_data[2]=kernely_data[2*kernely_step]; + break; + case CV_8SC1: + ctx->kernelx_data[0]=((char*)kernelx_data)[0]; + ctx->kernelx_data[1]=((char*)kernelx_data)[1]; + ctx->kernelx_data[2]=((char*)kernelx_data)[2]; + ctx->kernely_data[0]=((char*)kernely_data)[0]; + ctx->kernely_data[1]=((char*)(kernely_data+kernely_step))[0]; + ctx->kernely_data[2]=((char*)(kernely_data+2*kernely_step))[0]; + break; + case CV_16UC1: + ctx->kernelx_data[0]=((int16_t*)kernelx_data)[0]; + ctx->kernelx_data[1]=((int16_t*)kernelx_data)[1]; + ctx->kernelx_data[2]=((int16_t*)kernelx_data)[2]; + ctx->kernely_data[0]=((int16_t*)kernely_data)[0]; + ctx->kernely_data[1]=((int16_t*)(kernely_data+kernely_step))[0]; + ctx->kernely_data[2]=((int16_t*)(kernely_data+2*kernely_step))[0]; + default: + delete ctx; + return CV_HAL_ERROR_NOT_IMPLEMENTED; + } + + *context = (cvhalFilter2D*)(ctx); + return CV_HAL_ERROR_OK; +} +inline int TEGRA_SEPFILTERFREE(cvhalFilter2D *context) +{ + if(context) + { + delete (SepFilterCtx*)context; + return CV_HAL_ERROR_OK; + } + else + { + return CV_HAL_ERROR_UNKNOWN; + } +} +#define TEGRA_SEPFILTERIMPL(context, src_data, src_step, dst_data, dst_step, width, height, full_width, full_height, offset_x, offset_y) \ +( \ + context && CAROTENE_NS::isSeparableFilter3x3Supported(CAROTENE_NS::Size2D(width, height), ((SepFilterCtx*)context)->border, 3, 3, \ + CAROTENE_NS::Margin(offset_x, full_width - width - offset_x, offset_y, full_height - height - offset_y)) ? \ + CAROTENE_NS::SeparableFilter3x3(CAROTENE_NS::Size2D(width, height), \ + src_data, src_step, \ + (CAROTENE_NS::s16*)dst_data, dst_step, \ + 3, 3, ((SepFilterCtx*)context)->kernelx_data, ((SepFilterCtx*)context)->kernely_data, \ + ((SepFilterCtx*)context)->border, 0, \ + CAROTENE_NS::Margin(offset_x, full_width - width - offset_x, offset_y, full_height - height - offset_y)), \ + CV_HAL_ERROR_OK \ + : CV_HAL_ERROR_NOT_IMPLEMENTED \ +) + +#undef cv_hal_sepFilterInit +#define cv_hal_sepFilterInit TEGRA_SEPFILTERINIT +#undef cv_hal_sepFilter +#define cv_hal_sepFilter TEGRA_SEPFILTERIMPL +#undef cv_hal_sepFilterFree +#define cv_hal_sepFilterFree TEGRA_SEPFILTERFREE + + +struct MorphCtx +{ + int operation; + int channels; + CAROTENE_NS::Size2D ksize; + int anchor_x, anchor_y; + CAROTENE_NS::BORDER_MODE border; + uchar borderValues[4]; +}; +inline int TEGRA_MORPHINIT(cvhalFilter2D **context, int operation, int src_type, int dst_type, int, int, + int kernel_type, uchar *kernel_data, size_t kernel_step, int kernel_width, int kernel_height, int anchor_x, int anchor_y, + int borderType, const double borderValue[4], int iterations, bool allowSubmatrix, bool allowInplace) +{ + if(!context || !kernel_data || src_type != dst_type || + CV_MAT_DEPTH(src_type) != CV_8U || src_type < 0 || (src_type >> CV_CN_SHIFT) > 3 || + + allowSubmatrix || allowInplace || iterations != 1 || + !CAROTENE_NS::isSupportedConfiguration()) + return CV_HAL_ERROR_NOT_IMPLEMENTED; + + switch(CV_MAT_DEPTH(kernel_type)) + { + case CV_8U: + if(CAROTENE_NS::countNonZero(CAROTENE_NS::Size2D(kernel_width, kernel_height), kernel_data, kernel_step) != kernel_width * kernel_height) + return CV_HAL_ERROR_NOT_IMPLEMENTED; + break; + case CV_16U: + if(CAROTENE_NS::countNonZero(CAROTENE_NS::Size2D(kernel_width, kernel_height), (uint16_t*)kernel_data, kernel_step) != kernel_width * kernel_height) + return CV_HAL_ERROR_NOT_IMPLEMENTED; + break; + case CV_32S: + if(CAROTENE_NS::countNonZero(CAROTENE_NS::Size2D(kernel_width, kernel_height), (int32_t*)kernel_data, kernel_step) != kernel_width * kernel_height) + return CV_HAL_ERROR_NOT_IMPLEMENTED; + break; + case CV_32F: + if(CAROTENE_NS::countNonZero(CAROTENE_NS::Size2D(kernel_width, kernel_height), (float*)kernel_data, kernel_step) != kernel_width * kernel_height) + return CV_HAL_ERROR_NOT_IMPLEMENTED; + break; + case CV_64F: + if(CAROTENE_NS::countNonZero(CAROTENE_NS::Size2D(kernel_width, kernel_height), (double*)kernel_data, kernel_step) != kernel_width * kernel_height) + return CV_HAL_ERROR_NOT_IMPLEMENTED; + break; + default: + return CV_HAL_ERROR_NOT_IMPLEMENTED; + } + + MorphCtx* ctx = new MorphCtx; + if(!ctx) + return CV_HAL_ERROR_UNKNOWN; + ctx->channels = (src_type >> CV_CN_SHIFT) + 1; + ctx->ksize.width = kernel_width; + ctx->ksize.height = kernel_height; + ctx->anchor_x = anchor_x; + ctx->anchor_y = anchor_y; + switch(operation) + { + case MORPH_ERODE: + case MORPH_DILATE: + ctx->operation = operation; + break; + default: + delete ctx; + return CV_HAL_ERROR_NOT_IMPLEMENTED; + } + switch(borderType) + { + case CV_HAL_BORDER_CONSTANT: + ctx->border = CAROTENE_NS::BORDER_MODE_CONSTANT; + if( borderValue[0] == DBL_MAX && borderValue[1] == DBL_MAX && borderValue[2] == DBL_MAX && borderValue[3] == DBL_MAX ) + { + if( operation == MORPH_ERODE ) + for(int i = 0; i < ctx->channels; ++i) + ctx->borderValues[i] = (CAROTENE_NS::u8)UCHAR_MAX; + else + for(int i = 0; i < ctx->channels; ++i) + ctx->borderValues[i] = 0; + } + else + { + for(int i = 0; i < ctx->channels; ++i) + ctx->borderValues[i] = (CAROTENE_NS::u8)cv::saturate_cast(borderValue[i]); + } + break; + case CV_HAL_BORDER_REPLICATE: + ctx->border = CAROTENE_NS::BORDER_MODE_REPLICATE; + break; + case CV_HAL_BORDER_REFLECT: + ctx->border = CAROTENE_NS::BORDER_MODE_REFLECT; + break; + case CV_HAL_BORDER_WRAP: + ctx->border = CAROTENE_NS::BORDER_MODE_WRAP; + break; + case CV_HAL_BORDER_REFLECT_101: + ctx->border = CAROTENE_NS::BORDER_MODE_REFLECT101; + break; + default: + delete ctx; + return CV_HAL_ERROR_NOT_IMPLEMENTED; + } + + *context = (cvhalFilter2D*)(ctx); + return CV_HAL_ERROR_OK; +} +inline int TEGRA_MORPHFREE(cvhalFilter2D *context) +{ + if(context) + { + delete (MorphCtx*)context; + return CV_HAL_ERROR_OK; + } + else + { + return CV_HAL_ERROR_UNKNOWN; + } +} +#define TEGRA_MORPHIMPL(context, src_data, src_step, dst_data, dst_step, width, height, src_full_width, src_full_height, src_roi_x, src_roi_y, dst_full_width, dst_full_height, dst_roi_x, dst_roi_y) \ +( \ + (void)dst_full_width, (void)dst_full_height, (void)dst_roi_x, (void)dst_roi_y, \ + context && CAROTENE_NS::isSupportedConfiguration() ? \ + ((MorphCtx*)context)->operation == MORPH_ERODE ? \ + CAROTENE_NS::erode(CAROTENE_NS::Size2D(width, height), ((MorphCtx*)context)->channels, \ + src_data, src_step, dst_data, dst_step, \ + ((MorphCtx*)context)->ksize, ((MorphCtx*)context)->anchor_x, ((MorphCtx*)context)->anchor_y, \ + ((MorphCtx*)context)->border, ((MorphCtx*)context)->border, ((MorphCtx*)context)->borderValues, \ + CAROTENE_NS::Margin(src_roi_x, src_full_width - width - src_roi_x, src_roi_y, src_full_height - height - src_roi_y)), \ + CV_HAL_ERROR_OK : \ + ((MorphCtx*)context)->operation == MORPH_DILATE ? \ + CAROTENE_NS::dilate(CAROTENE_NS::Size2D(width, height), ((MorphCtx*)context)->channels, \ + src_data, src_step, dst_data, dst_step, \ + ((MorphCtx*)context)->ksize, ((MorphCtx*)context)->anchor_x, ((MorphCtx*)context)->anchor_y, \ + ((MorphCtx*)context)->border, ((MorphCtx*)context)->border, ((MorphCtx*)context)->borderValues, \ + CAROTENE_NS::Margin(src_roi_x, src_full_width - width - src_roi_x, src_roi_y, src_full_height - height - src_roi_y)), \ + CV_HAL_ERROR_OK : \ + CV_HAL_ERROR_NOT_IMPLEMENTED \ + : CV_HAL_ERROR_NOT_IMPLEMENTED \ +) + +#undef cv_hal_morphInit +#define cv_hal_morphInit TEGRA_MORPHINIT +#undef cv_hal_morph +#define cv_hal_morph TEGRA_MORPHIMPL +#undef cv_hal_morphFree +#define cv_hal_morphFree TEGRA_MORPHFREE + + + +#define TEGRA_RESIZE(src_type, src_data, src_step, src_width, src_height, dst_data, dst_step, dst_width, dst_height, inv_scale_x, inv_scale_y, interpolation) \ +( \ + /*interpolation == CV_HAL_INTER_LINEAR ? \ + CV_MAT_DEPTH(src_type) == CV_8U && CAROTENE_NS::isResizeLinearOpenCVSupported(CAROTENE_NS::Size2D(src_width, src_height), CAROTENE_NS::Size2D(dst_width, dst_height), ((src_type >> CV_CN_SHIFT) + 1)) && \ + inv_scale_x > 0 && inv_scale_y > 0 && \ + (dst_width - 0.5)/inv_scale_x - 0.5 < src_width && (dst_height - 0.5)/inv_scale_y - 0.5 < src_height && \ + (dst_width + 0.5)/inv_scale_x + 0.5 >= src_width && (dst_height + 0.5)/inv_scale_y + 0.5 >= src_height && \ + std::abs(dst_width / inv_scale_x - src_width) < 0.1 && std::abs(dst_height / inv_scale_y - src_height) < 0.1 ? \ + CAROTENE_NS::resizeLinearOpenCV(CAROTENE_NS::Size2D(src_width, src_height), CAROTENE_NS::Size2D(dst_width, dst_height), \ + src_data, src_step, dst_data, dst_step, 1.0/inv_scale_x, 1.0/inv_scale_y, ((src_type >> CV_CN_SHIFT) + 1)), \ + CV_HAL_ERROR_OK : CV_HAL_ERROR_NOT_IMPLEMENTED :*/ \ + interpolation == CV_HAL_INTER_AREA ? \ + CV_MAT_DEPTH(src_type) == CV_8U && CAROTENE_NS::isResizeAreaSupported(1.0/inv_scale_x, 1.0/inv_scale_y, ((src_type >> CV_CN_SHIFT) + 1)) && \ + std::abs(dst_width / inv_scale_x - src_width) < 0.1 && std::abs(dst_height / inv_scale_y - src_height) < 0.1 ? \ + CAROTENE_NS::resizeAreaOpenCV(CAROTENE_NS::Size2D(src_width, src_height), CAROTENE_NS::Size2D(dst_width, dst_height), \ + src_data, src_step, dst_data, dst_step, 1.0/inv_scale_x, 1.0/inv_scale_y, ((src_type >> CV_CN_SHIFT) + 1)), \ + CV_HAL_ERROR_OK : CV_HAL_ERROR_NOT_IMPLEMENTED : \ + /*nearest neighbour interpolation disabled due to rounding accuracy issues*/ \ + /*interpolation == CV_HAL_INTER_NEAREST ? \ + (src_type == CV_8UC1 || src_type == CV_8SC1) && CAROTENE_NS::isResizeNearestNeighborSupported(CAROTENE_NS::Size2D(src_width, src_height), 1) ? \ + CAROTENE_NS::resizeNearestNeighbor(CAROTENE_NS::Size2D(src_width, src_height), CAROTENE_NS::Size2D(dst_width, dst_height), \ + src_data, src_step, dst_data, dst_step, 1.0/inv_scale_x, 1.0/inv_scale_y, 1), \ + CV_HAL_ERROR_OK : \ + (src_type == CV_8UC3 || src_type == CV_8SC3) && CAROTENE_NS::isResizeNearestNeighborSupported(CAROTENE_NS::Size2D(src_width, src_height), 3) ? \ + CAROTENE_NS::resizeNearestNeighbor(CAROTENE_NS::Size2D(src_width, src_height), CAROTENE_NS::Size2D(dst_width, dst_height), \ + src_data, src_step, dst_data, dst_step, 1.0/inv_scale_x, 1.0/inv_scale_y, 3), \ + CV_HAL_ERROR_OK : \ + (src_type == CV_8UC4 || src_type == CV_8SC4 || src_type == CV_16UC2 || src_type == CV_16SC2 || src_type == CV_32SC1) && \ + CAROTENE_NS::isResizeNearestNeighborSupported(CAROTENE_NS::Size2D(src_width, src_height), 4) ? \ + CAROTENE_NS::resizeNearestNeighbor(CAROTENE_NS::Size2D(src_width, src_height), CAROTENE_NS::Size2D(dst_width, dst_height), \ + src_data, src_step, dst_data, dst_step, 1.0/inv_scale_x, 1.0/inv_scale_y, 4), \ + CV_HAL_ERROR_OK : CV_HAL_ERROR_NOT_IMPLEMENTED :*/ \ + CV_HAL_ERROR_NOT_IMPLEMENTED \ +) + +#define TEGRA_WARPAFFINE(src_type, src_data, src_step, src_width, src_height, dst_data, dst_step, dst_width, dst_height, M, interpolation, borderType, borderValue) \ +( \ + interpolation == CV_HAL_INTER_NEAREST ? \ + (src_type == CV_8UC1 || src_type == CV_8SC1) && (borderType == CV_HAL_BORDER_REPLICATE || borderType == CV_HAL_BORDER_CONSTANT) && \ + CAROTENE_NS::isWarpAffineNearestNeighborSupported(CAROTENE_NS::Size2D(src_width, src_height)) ? \ + CAROTENE_NS::warpAffineNearestNeighbor(CAROTENE_NS::Size2D(src_width, src_height), CAROTENE_NS::Size2D(dst_width, dst_height), \ + src_data, src_step, \ + std::vector(M+0,M+6).data(), \ + dst_data, dst_step, \ + borderType == CV_HAL_BORDER_REPLICATE ? CAROTENE_NS::BORDER_MODE_REPLICATE : CAROTENE_NS::BORDER_MODE_CONSTANT, \ + (CAROTENE_NS::u8)borderValue[0]), \ + CV_HAL_ERROR_OK : CV_HAL_ERROR_NOT_IMPLEMENTED : \ + interpolation == CV_HAL_INTER_LINEAR ? \ + (src_type == CV_8UC1 || src_type == CV_8SC1) && (borderType == CV_HAL_BORDER_REPLICATE || borderType == CV_HAL_BORDER_CONSTANT) && \ + CAROTENE_NS::isWarpAffineLinearSupported(CAROTENE_NS::Size2D(src_width, src_height)) ? \ + CAROTENE_NS::warpAffineLinear(CAROTENE_NS::Size2D(src_width, src_height), CAROTENE_NS::Size2D(dst_width, dst_height), \ + src_data, src_step, \ + std::vector(M+0,M+6).data(), \ + dst_data, dst_step, \ + borderType == CV_HAL_BORDER_REPLICATE ? CAROTENE_NS::BORDER_MODE_REPLICATE : CAROTENE_NS::BORDER_MODE_CONSTANT, \ + (CAROTENE_NS::u8)borderValue[0]), \ + CV_HAL_ERROR_OK : CV_HAL_ERROR_NOT_IMPLEMENTED : \ + CV_HAL_ERROR_NOT_IMPLEMENTED \ +) + +#define TEGRA_WARPPERSPECTIVE(src_type, src_data, src_step, src_width, src_height, dst_data, dst_step, dst_width, dst_height, M, interpolation, borderType, borderValue) \ +( \ + interpolation == CV_HAL_INTER_NEAREST ? \ + (src_type == CV_8UC1 || src_type == CV_8SC1) && (borderType == CV_HAL_BORDER_REPLICATE || borderType == CV_HAL_BORDER_CONSTANT) && \ + CAROTENE_NS::isWarpPerspectiveNearestNeighborSupported(CAROTENE_NS::Size2D(src_width, src_height)) ? \ + CAROTENE_NS::warpPerspectiveNearestNeighbor(CAROTENE_NS::Size2D(src_width, src_height), CAROTENE_NS::Size2D(dst_width, dst_height), \ + src_data, src_step, \ + std::vector(M+0,M+9).data(), \ + dst_data, dst_step, \ + borderType == CV_HAL_BORDER_REPLICATE ? CAROTENE_NS::BORDER_MODE_REPLICATE : CAROTENE_NS::BORDER_MODE_CONSTANT, \ + (CAROTENE_NS::u8)borderValue[0]), \ + CV_HAL_ERROR_OK : CV_HAL_ERROR_NOT_IMPLEMENTED : \ + interpolation == CV_HAL_INTER_LINEAR ? \ + (src_type == CV_8UC1 || src_type == CV_8SC1) && (borderType == CV_HAL_BORDER_REPLICATE || borderType == CV_HAL_BORDER_CONSTANT) && \ + CAROTENE_NS::isWarpPerspectiveLinearSupported(CAROTENE_NS::Size2D(src_width, src_height)) ? \ + CAROTENE_NS::warpPerspectiveLinear(CAROTENE_NS::Size2D(src_width, src_height), CAROTENE_NS::Size2D(dst_width, dst_height), \ + src_data, src_step, \ + std::vector(M+0,M+9).data(), \ + dst_data, dst_step, \ + borderType == CV_HAL_BORDER_REPLICATE ? CAROTENE_NS::BORDER_MODE_REPLICATE : CAROTENE_NS::BORDER_MODE_CONSTANT, \ + (CAROTENE_NS::u8)borderValue[0]), \ + CV_HAL_ERROR_OK : CV_HAL_ERROR_NOT_IMPLEMENTED : \ + CV_HAL_ERROR_NOT_IMPLEMENTED \ +) + +#undef cv_hal_resize +#define cv_hal_resize TEGRA_RESIZE +//warpAffine/warpPerspective disabled due to rounding accuracy issue +//#undef cv_hal_warpAffine +//#define cv_hal_warpAffine TEGRA_WARPAFFINE +//#undef cv_hal_warpPerspective +//#define cv_hal_warpPerspective TEGRA_WARPPERSPECTIVE + + +#define TegraCvtColor_Invoker(name, func, ...) \ +class TegraCvtColor_##name##_Invoker : public cv::ParallelLoopBody \ +{ \ +public: \ + TegraCvtColor_##name##_Invoker(const uchar * src_data_, size_t src_step_, uchar * dst_data_, size_t dst_step_, int width_, int height_) : \ + cv::ParallelLoopBody(), src_data(src_data_), src_step(src_step_), dst_data(dst_data_), dst_step(dst_step_), width(width_), height(height_) {} \ + virtual void operator()(const cv::Range& range) const \ + { \ + CAROTENE_NS::func(CAROTENE_NS::Size2D(width, range.end-range.start), __VA_ARGS__); \ + } \ +private: \ + const uchar * src_data; \ + size_t src_step; \ + uchar * dst_data; \ + size_t dst_step; \ + int width, height; \ + const TegraCvtColor_##name##_Invoker& operator= (const TegraCvtColor_##name##_Invoker&); \ +}; + +TegraCvtColor_Invoker(rgb2bgr, rgb2bgr, src_data + static_cast(range.start) * src_step, src_step, \ + dst_data + static_cast(range.start) * dst_step, dst_step) +TegraCvtColor_Invoker(rgb2bgrx, rgb2bgrx, src_data + static_cast(range.start) * src_step, src_step, \ + dst_data + static_cast(range.start) * dst_step, dst_step) +TegraCvtColor_Invoker(rgb2rgbx, rgb2rgbx, src_data + static_cast(range.start) * src_step, src_step, \ + dst_data + static_cast(range.start) * dst_step, dst_step) +TegraCvtColor_Invoker(rgbx2bgr, rgbx2bgr, src_data + static_cast(range.start) * src_step, src_step, \ + dst_data + static_cast(range.start) * dst_step, dst_step) +TegraCvtColor_Invoker(rgbx2rgb, rgbx2rgb, src_data + static_cast(range.start) * src_step, src_step, \ + dst_data + static_cast(range.start) * dst_step, dst_step) +TegraCvtColor_Invoker(rgbx2bgrx, rgbx2bgrx, src_data + static_cast(range.start) * src_step, src_step, \ + dst_data + static_cast(range.start) * dst_step, dst_step) +#define TEGRA_CVTBGRTOBGR(src_data, src_step, dst_data, dst_step, width, height, depth, scn, dcn, swapBlue) \ +( \ + depth == CV_8U && CAROTENE_NS::isSupportedConfiguration() ? \ + scn == 3 ? \ + dcn == 3 ? \ + swapBlue ? \ + parallel_for_(Range(0, height), \ + TegraCvtColor_rgb2bgr_Invoker(src_data, src_step, dst_data, dst_step, width, height), \ + (width * height) / static_cast(1<<16)), \ + CV_HAL_ERROR_OK : \ + CV_HAL_ERROR_NOT_IMPLEMENTED : \ + dcn == 4 ? \ + (swapBlue ? \ + parallel_for_(Range(0, height), \ + TegraCvtColor_rgb2bgrx_Invoker(src_data, src_step, dst_data, dst_step, width, height), \ + (width * height) / static_cast(1<<16)) : \ + parallel_for_(Range(0, height), \ + TegraCvtColor_rgb2rgbx_Invoker(src_data, src_step, dst_data, dst_step, width, height), \ + (width * height) / static_cast(1<<16)) ), \ + CV_HAL_ERROR_OK : \ + CV_HAL_ERROR_NOT_IMPLEMENTED : \ + scn == 4 ? \ + dcn == 3 ? \ + (swapBlue ? \ + parallel_for_(Range(0, height), \ + TegraCvtColor_rgbx2bgr_Invoker(src_data, src_step, dst_data, dst_step, width, height), \ + (width * height) / static_cast(1<<16)) : \ + parallel_for_(Range(0, height), \ + TegraCvtColor_rgbx2rgb_Invoker(src_data, src_step, dst_data, dst_step, width, height), \ + (width * height) / static_cast(1<<16)) ), \ + CV_HAL_ERROR_OK : \ + dcn == 4 ? \ + swapBlue ? \ + parallel_for_(Range(0, height), \ + TegraCvtColor_rgbx2bgrx_Invoker(src_data, src_step, dst_data, dst_step, width, height), \ + (width * height) / static_cast(1<<16)), \ + CV_HAL_ERROR_OK : \ + CV_HAL_ERROR_NOT_IMPLEMENTED : \ + CV_HAL_ERROR_NOT_IMPLEMENTED : \ + CV_HAL_ERROR_NOT_IMPLEMENTED \ + : CV_HAL_ERROR_NOT_IMPLEMENTED \ +) + +TegraCvtColor_Invoker(rgb2bgr565, rgb2bgr565, src_data + static_cast(range.start) * src_step, src_step, \ + dst_data + static_cast(range.start) * dst_step, dst_step) +TegraCvtColor_Invoker(rgb2rgb565, rgb2rgb565, src_data + static_cast(range.start) * src_step, src_step, \ + dst_data + static_cast(range.start) * dst_step, dst_step) +TegraCvtColor_Invoker(rgbx2bgr565, rgbx2bgr565, src_data + static_cast(range.start) * src_step, src_step, \ + dst_data + static_cast(range.start) * dst_step, dst_step) +TegraCvtColor_Invoker(rgbx2rgb565, rgbx2rgb565, src_data + static_cast(range.start) * src_step, src_step, \ + dst_data + static_cast(range.start) * dst_step, dst_step) +#define TEGRA_CVTBGRTOBGR565(src_data, src_step, dst_data, dst_step, width, height, scn, swapBlue, greenBits) \ +( \ + greenBits == 6 && CAROTENE_NS::isSupportedConfiguration() ? \ + scn == 3 ? \ + (swapBlue ? \ + parallel_for_(Range(0, height), \ + TegraCvtColor_rgb2bgr565_Invoker(src_data, src_step, dst_data, dst_step, width, height), \ + (width * height) / static_cast(1<<16)) : \ + parallel_for_(Range(0, height), \ + TegraCvtColor_rgb2rgb565_Invoker(src_data, src_step, dst_data, dst_step, width, height), \ + (width * height) / static_cast(1<<16)) ), \ + CV_HAL_ERROR_OK : \ + scn == 4 ? \ + (swapBlue ? \ + parallel_for_(Range(0, height), \ + TegraCvtColor_rgbx2bgr565_Invoker(src_data, src_step, dst_data, dst_step, width, height), \ + (width * height) / static_cast(1<<16)) : \ + parallel_for_(Range(0, height), \ + TegraCvtColor_rgbx2rgb565_Invoker(src_data, src_step, dst_data, dst_step, width, height), \ + (width * height) / static_cast(1<<16)) ), \ + CV_HAL_ERROR_OK : \ + CV_HAL_ERROR_NOT_IMPLEMENTED \ + : CV_HAL_ERROR_NOT_IMPLEMENTED \ +) + +TegraCvtColor_Invoker(rgb2gray, rgb2gray, CAROTENE_NS::COLOR_SPACE_BT601, src_data + static_cast(range.start) * src_step, src_step, \ + dst_data + static_cast(range.start) * dst_step, dst_step) +TegraCvtColor_Invoker(bgr2gray, bgr2gray, CAROTENE_NS::COLOR_SPACE_BT601, src_data + static_cast(range.start) * src_step, src_step, \ + dst_data + static_cast(range.start) * dst_step, dst_step) +TegraCvtColor_Invoker(rgbx2gray, rgbx2gray, CAROTENE_NS::COLOR_SPACE_BT601, src_data + static_cast(range.start) * src_step, src_step, \ + dst_data + static_cast(range.start) * dst_step, dst_step) +TegraCvtColor_Invoker(bgrx2gray, bgrx2gray, CAROTENE_NS::COLOR_SPACE_BT601, src_data + static_cast(range.start) * src_step, src_step, \ + dst_data + static_cast(range.start) * dst_step, dst_step) +#define TEGRA_CVTBGRTOGRAY(src_data, src_step, dst_data, dst_step, width, height, depth, scn, swapBlue) \ +( \ + depth == CV_8U && CAROTENE_NS::isSupportedConfiguration() ? \ + scn == 3 ? \ + (swapBlue ? \ + parallel_for_(Range(0, height), \ + TegraCvtColor_rgb2gray_Invoker(src_data, src_step, dst_data, dst_step, width, height), \ + (width * height) / static_cast(1<<16)) : \ + parallel_for_(Range(0, height), \ + TegraCvtColor_bgr2gray_Invoker(src_data, src_step, dst_data, dst_step, width, height), \ + (width * height) / static_cast(1<<16)) ), \ + CV_HAL_ERROR_OK : \ + scn == 4 ? \ + (swapBlue ? \ + parallel_for_(Range(0, height), \ + TegraCvtColor_rgbx2gray_Invoker(src_data, src_step, dst_data, dst_step, width, height), \ + (width * height) / static_cast(1<<16)) : \ + parallel_for_(Range(0, height), \ + TegraCvtColor_bgrx2gray_Invoker(src_data, src_step, dst_data, dst_step, width, height), \ + (width * height) / static_cast(1<<16)) ), \ + CV_HAL_ERROR_OK : \ + CV_HAL_ERROR_NOT_IMPLEMENTED \ + : CV_HAL_ERROR_NOT_IMPLEMENTED \ +) + +TegraCvtColor_Invoker(gray2rgb, gray2rgb, src_data + static_cast(range.start) * src_step, src_step, \ + dst_data + static_cast(range.start) * dst_step, dst_step) +TegraCvtColor_Invoker(gray2rgbx, gray2rgbx, src_data + static_cast(range.start) * src_step, src_step, \ + dst_data + static_cast(range.start) * dst_step, dst_step) +#define TEGRA_CVTGRAYTOBGR(src_data, src_step, dst_data, dst_step, width, height, depth, dcn) \ +( \ + depth == CV_8U && CAROTENE_NS::isSupportedConfiguration() ? \ + dcn == 3 ? \ + parallel_for_(Range(0, height), \ + TegraCvtColor_gray2rgb_Invoker(src_data, src_step, dst_data, dst_step, width, height), \ + (width * height) / static_cast(1<<16)), \ + CV_HAL_ERROR_OK : \ + dcn == 4 ? \ + parallel_for_(Range(0, height), \ + TegraCvtColor_gray2rgbx_Invoker(src_data, src_step, dst_data, dst_step, width, height), \ + (width * height) / static_cast(1<<16)), \ + CV_HAL_ERROR_OK : \ + CV_HAL_ERROR_NOT_IMPLEMENTED \ + : CV_HAL_ERROR_NOT_IMPLEMENTED \ +) + +TegraCvtColor_Invoker(rgb2ycrcb, rgb2ycrcb, src_data + static_cast(range.start) * src_step, src_step, \ + dst_data + static_cast(range.start) * dst_step, dst_step) +TegraCvtColor_Invoker(bgr2ycrcb, bgr2ycrcb, src_data + static_cast(range.start) * src_step, src_step, \ + dst_data + static_cast(range.start) * dst_step, dst_step) +TegraCvtColor_Invoker(rgbx2ycrcb, rgbx2ycrcb, src_data + static_cast(range.start) * src_step, src_step, \ + dst_data + static_cast(range.start) * dst_step, dst_step) +TegraCvtColor_Invoker(bgrx2ycrcb, bgrx2ycrcb, src_data + static_cast(range.start) * src_step, src_step, \ + dst_data + static_cast(range.start) * dst_step, dst_step) +#define TEGRA_CVTBGRTOYUV(src_data, src_step, dst_data, dst_step, width, height, depth, scn, swapBlue, isCbCr) \ +( \ + isCbCr && depth == CV_8U && CAROTENE_NS::isSupportedConfiguration() ? \ + scn == 3 ? \ + (swapBlue ? \ + parallel_for_(Range(0, height), \ + TegraCvtColor_rgb2ycrcb_Invoker(src_data, src_step, dst_data, dst_step, width, height), \ + (width * height) / static_cast(1<<16)) : \ + parallel_for_(Range(0, height), \ + TegraCvtColor_bgr2ycrcb_Invoker(src_data, src_step, dst_data, dst_step, width, height), \ + (width * height) / static_cast(1<<16)) ), \ + CV_HAL_ERROR_OK : \ + scn == 4 ? \ + (swapBlue ? \ + parallel_for_(Range(0, height), \ + TegraCvtColor_rgbx2ycrcb_Invoker(src_data, src_step, dst_data, dst_step, width, height), \ + (width * height) / static_cast(1<<16)) : \ + parallel_for_(Range(0, height), \ + TegraCvtColor_bgrx2ycrcb_Invoker(src_data, src_step, dst_data, dst_step, width, height), \ + (width * height) / static_cast(1<<16)) ), \ + CV_HAL_ERROR_OK : \ + CV_HAL_ERROR_NOT_IMPLEMENTED \ + : CV_HAL_ERROR_NOT_IMPLEMENTED \ +) + +TegraCvtColor_Invoker(rgb2hsv, rgb2hsv, src_data + static_cast(range.start) * src_step, src_step, \ + dst_data + static_cast(range.start) * dst_step, dst_step, 180) +TegraCvtColor_Invoker(bgr2hsv, bgr2hsv, src_data + static_cast(range.start) * src_step, src_step, \ + dst_data + static_cast(range.start) * dst_step, dst_step, 180) +TegraCvtColor_Invoker(rgbx2hsv, rgbx2hsv, src_data + static_cast(range.start) * src_step, src_step, \ + dst_data + static_cast(range.start) * dst_step, dst_step, 180) +TegraCvtColor_Invoker(bgrx2hsv, bgrx2hsv, src_data + static_cast(range.start) * src_step, src_step, \ + dst_data + static_cast(range.start) * dst_step, dst_step, 180) +TegraCvtColor_Invoker(rgb2hsvf, rgb2hsv, src_data + static_cast(range.start) * src_step, src_step, \ + dst_data + static_cast(range.start) * dst_step, dst_step, 256) +TegraCvtColor_Invoker(bgr2hsvf, bgr2hsv, src_data + static_cast(range.start) * src_step, src_step, \ + dst_data + static_cast(range.start) * dst_step, dst_step, 256) +TegraCvtColor_Invoker(rgbx2hsvf, rgbx2hsv, src_data + static_cast(range.start) * src_step, src_step, \ + dst_data + static_cast(range.start) * dst_step, dst_step, 256) +TegraCvtColor_Invoker(bgrx2hsvf, bgrx2hsv, src_data + static_cast(range.start) * src_step, src_step, \ + dst_data + static_cast(range.start) * dst_step, dst_step, 256) +#define TEGRA_CVTBGRTOHSV(src_data, src_step, dst_data, dst_step, width, height, depth, scn, swapBlue, isFullRange, isHSV) \ +( \ + isHSV && depth == CV_8U && CAROTENE_NS::isSupportedConfiguration() ? \ + scn == 3 ? \ + (swapBlue ? \ + isFullRange ? \ + parallel_for_(Range(0, height), \ + TegraCvtColor_rgb2hsvf_Invoker(src_data, src_step, dst_data, dst_step, width, height), \ + (width * height) / static_cast(1<<16)) : \ + parallel_for_(Range(0, height), \ + TegraCvtColor_rgb2hsv_Invoker(src_data, src_step, dst_data, dst_step, width, height), \ + (width * height) / static_cast(1<<16)) : \ + isFullRange ? \ + parallel_for_(Range(0, height), \ + TegraCvtColor_bgr2hsvf_Invoker(src_data, src_step, dst_data, dst_step, width, height), \ + (width * height) / static_cast(1<<16)) : \ + parallel_for_(Range(0, height), \ + TegraCvtColor_bgr2hsv_Invoker(src_data, src_step, dst_data, dst_step, width, height), \ + (width * height) / static_cast(1<<16)) ), \ + CV_HAL_ERROR_OK : \ + scn == 4 ? \ + (swapBlue ? \ + isFullRange ? \ + parallel_for_(Range(0, height), \ + TegraCvtColor_rgbx2hsvf_Invoker(src_data, src_step, dst_data, dst_step, width, height), \ + (width * height) / static_cast(1<<16)) : \ + parallel_for_(Range(0, height), \ + TegraCvtColor_rgbx2hsv_Invoker(src_data, src_step, dst_data, dst_step, width, height), \ + (width * height) / static_cast(1<<16)) : \ + isFullRange ? \ + parallel_for_(Range(0, height), \ + TegraCvtColor_bgrx2hsvf_Invoker(src_data, src_step, dst_data, dst_step, width, height), \ + (width * height) / static_cast(1<<16)) : \ + parallel_for_(Range(0, height), \ + TegraCvtColor_bgrx2hsv_Invoker(src_data, src_step, dst_data, dst_step, width, height), \ + (width * height) / static_cast(1<<16)) ), \ + CV_HAL_ERROR_OK : \ + CV_HAL_ERROR_NOT_IMPLEMENTED \ + : CV_HAL_ERROR_NOT_IMPLEMENTED \ +) + +#define TEGRA_CVT2PYUVTOBGR(src_data, src_step, dst_data, dst_step, dst_width, dst_height, dcn, swapBlue, uIdx) \ +( \ + CAROTENE_NS::isSupportedConfiguration() ? \ + dcn == 3 ? \ + uIdx == 0 ? \ + (swapBlue ? \ + CAROTENE_NS::yuv420i2rgb(CAROTENE_NS::Size2D(dst_width, dst_height), \ + src_data, src_step, \ + src_data + src_step * dst_height, src_step, \ + dst_data, dst_step) : \ + CAROTENE_NS::yuv420i2bgr(CAROTENE_NS::Size2D(dst_width, dst_height), \ + src_data, src_step, \ + src_data + src_step * dst_height, src_step, \ + dst_data, dst_step)), \ + CV_HAL_ERROR_OK : \ + uIdx == 1 ? \ + (swapBlue ? \ + CAROTENE_NS::yuv420sp2rgb(CAROTENE_NS::Size2D(dst_width, dst_height), \ + src_data, src_step, \ + src_data + src_step * dst_height, src_step, \ + dst_data, dst_step) : \ + CAROTENE_NS::yuv420sp2bgr(CAROTENE_NS::Size2D(dst_width, dst_height), \ + src_data, src_step, \ + src_data + src_step * dst_height, src_step, \ + dst_data, dst_step)), \ + CV_HAL_ERROR_OK : \ + CV_HAL_ERROR_NOT_IMPLEMENTED : \ + dcn == 4 ? \ + uIdx == 0 ? \ + (swapBlue ? \ + CAROTENE_NS::yuv420i2rgbx(CAROTENE_NS::Size2D(dst_width, dst_height), \ + src_data, src_step, \ + src_data + src_step * dst_height, src_step, \ + dst_data, dst_step) : \ + CAROTENE_NS::yuv420i2bgrx(CAROTENE_NS::Size2D(dst_width, dst_height), \ + src_data, src_step, \ + src_data + src_step * dst_height, src_step, \ + dst_data, dst_step)), \ + CV_HAL_ERROR_OK : \ + uIdx == 1 ? \ + (swapBlue ? \ + CAROTENE_NS::yuv420sp2rgbx(CAROTENE_NS::Size2D(dst_width, dst_height), \ + src_data, src_step, \ + src_data + src_step * dst_height, src_step, \ + dst_data, dst_step) : \ + CAROTENE_NS::yuv420sp2bgrx(CAROTENE_NS::Size2D(dst_width, dst_height), \ + src_data, src_step, \ + src_data + src_step * dst_height, src_step, \ + dst_data, dst_step)), \ + CV_HAL_ERROR_OK : \ + CV_HAL_ERROR_NOT_IMPLEMENTED : \ + CV_HAL_ERROR_NOT_IMPLEMENTED \ + : CV_HAL_ERROR_NOT_IMPLEMENTED \ +) + +#undef cv_hal_cvtBGRtoBGR +#define cv_hal_cvtBGRtoBGR TEGRA_CVTBGRTOBGR +#undef cv_hal_cvtBGRtoBGR5x5 +#define cv_hal_cvtBGRtoBGR5x5 TEGRA_CVTBGRTOBGR565 +#undef cv_hal_cvtBGRtoGray +#define cv_hal_cvtBGRtoGray TEGRA_CVTBGRTOGRAY +#undef cv_hal_cvtGraytoBGR +#define cv_hal_cvtGraytoBGR TEGRA_CVTGRAYTOBGR +#undef cv_hal_cvtBGRtoYUV +#define cv_hal_cvtBGRtoYUV TEGRA_CVTBGRTOYUV +#undef cv_hal_cvtBGRtoHSV +#define cv_hal_cvtBGRtoHSV TEGRA_CVTBGRTOHSV +#undef cv_hal_cvtTwoPlaneYUVtoBGR +#define cv_hal_cvtTwoPlaneYUVtoBGR TEGRA_CVT2PYUVTOBGR + +#endif // OPENCV_IMGPROC_HAL_INTERFACE_H + +#endif diff --git a/3rdparty/carotene/include/carotene/definitions.hpp b/3rdparty/carotene/include/carotene/definitions.hpp new file mode 100644 index 0000000000..124a674d61 --- /dev/null +++ b/3rdparty/carotene/include/carotene/definitions.hpp @@ -0,0 +1,47 @@ +/* + * By downloading, copying, installing or using the software you agree to this license. + * If you do not agree to this license, do not download, install, + * copy or use the software. + * + * + * License Agreement + * For Open Source Computer Vision Library + * (3-clause BSD License) + * + * Copyright (C) 2015, NVIDIA Corporation, all rights reserved. + * Third party copyrights are property of their respective owners. + * + * Redistribution and use in source and binary forms, with or without modification, + * are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * * Neither the names of the copyright holders nor the names of the contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * This software is provided by the copyright holders and contributors "as is" and + * any express or implied warranties, including, but not limited to, the implied + * warranties of merchantability and fitness for a particular purpose are disclaimed. + * In no event shall copyright holders or contributors be liable for any direct, + * indirect, incidental, special, exemplary, or consequential damages + * (including, but not limited to, procurement of substitute goods or services; + * loss of use, data, or profits; or business interruption) however caused + * and on any theory of liability, whether in contract, strict liability, + * or tort (including negligence or otherwise) arising in any way out of + * the use of this software, even if advised of the possibility of such damage. + */ + +#ifndef CAROTENE_DEFINITIONS_HPP +#define CAROTENE_DEFINITIONS_HPP + +#ifndef CAROTENE_NS +#define CAROTENE_NS carotene +#endif + +#endif diff --git a/3rdparty/carotene/include/carotene/functions.hpp b/3rdparty/carotene/include/carotene/functions.hpp new file mode 100644 index 0000000000..76d1328194 --- /dev/null +++ b/3rdparty/carotene/include/carotene/functions.hpp @@ -0,0 +1,2492 @@ +/* + * By downloading, copying, installing or using the software you agree to this license. + * If you do not agree to this license, do not download, install, + * copy or use the software. + * + * + * License Agreement + * For Open Source Computer Vision Library + * (3-clause BSD License) + * + * Copyright (C) 2014-2015, NVIDIA Corporation, all rights reserved. + * Third party copyrights are property of their respective owners. + * + * Redistribution and use in source and binary forms, with or without modification, + * are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * * Neither the names of the copyright holders nor the names of the contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * This software is provided by the copyright holders and contributors "as is" and + * any express or implied warranties, including, but not limited to, the implied + * warranties of merchantability and fitness for a particular purpose are disclaimed. + * In no event shall copyright holders or contributors be liable for any direct, + * indirect, incidental, special, exemplary, or consequential damages + * (including, but not limited to, procurement of substitute goods or services; + * loss of use, data, or profits; or business interruption) however caused + * and on any theory of liability, whether in contract, strict liability, + * or tort (including negligence or otherwise) arising in any way out of + * the use of this software, even if advised of the possibility of such damage. + */ + +#ifndef CAROTENE_FUNCTIONS_HPP +#define CAROTENE_FUNCTIONS_HPP + +#include +#include + +namespace CAROTENE_NS { + /* If this returns false, none of the functions will work. */ + bool isSupportedConfiguration(); + + /* + For each point `p` within `size`, do: + dst[p] = src0[p] + src1[p] + */ + void add(const Size2D &size, + const u8 *src0Base, ptrdiff_t src0Stride, + const u8 *src1Base, ptrdiff_t src1Stride, + u8 *dstBase, ptrdiff_t dstStride, + CONVERT_POLICY policy); + + void add(const Size2D &size, + const u8 *src0Base, ptrdiff_t src0Stride, + const u8 *src1Base, ptrdiff_t src1Stride, + s16 *dstBase, ptrdiff_t dstStride, + CONVERT_POLICY policy); + + void add(const Size2D &size, + const u8 *src0Base, ptrdiff_t src0Stride, + const s16 *src1Base, ptrdiff_t src1Stride, + s16 *dstBase, ptrdiff_t dstStride, + CONVERT_POLICY policy); + + void add(const Size2D &size, + const s8 *src0Base, ptrdiff_t src0Stride, + const s8 *src1Base, ptrdiff_t src1Stride, + s8 *dstBase, ptrdiff_t dstStride, + CONVERT_POLICY policy); + + void add(const Size2D &size, + const s16 *src0Base, ptrdiff_t src0Stride, + const s16 *src1Base, ptrdiff_t src1Stride, + s16 *dstBase, ptrdiff_t dstStride, + CONVERT_POLICY policy); + + void add(const Size2D &size, + const u16 * src0Base, ptrdiff_t src0Stride, + const u16 * src1Base, ptrdiff_t src1Stride, + u16 * dstBase, ptrdiff_t dstStride, + CONVERT_POLICY policy); + + void add(const Size2D &size, + const s32 * src0Base, ptrdiff_t src0Stride, + const s32 * src1Base, ptrdiff_t src1Stride, + s32 * dstBase, ptrdiff_t dstStride, + CONVERT_POLICY policy); + + void add(const Size2D &size, + const u32 * src0Base, ptrdiff_t src0Stride, + const u32 * src1Base, ptrdiff_t src1Stride, + u32 * dstBase, ptrdiff_t dstStride, + CONVERT_POLICY policy); + + void add(const Size2D &size, + const f32 * src0Base, ptrdiff_t src0Stride, + const f32 * src1Base, ptrdiff_t src1Stride, + f32 * dstBase, ptrdiff_t dstStride); + + /* + For each point `p` within `size`, do: + dst[p] = src0[p] - src1[p] + */ + void sub(const Size2D &size, + const u8 *src0Base, ptrdiff_t src0Stride, + const u8 *src1Base, ptrdiff_t src1Stride, + u8 *dstBase, ptrdiff_t dstStride, + CONVERT_POLICY policy); + + void sub(const Size2D &size, + const u8 *src0Base, ptrdiff_t src0Stride, + const u8 *src1Base, ptrdiff_t src1Stride, + s16 *dstBase, ptrdiff_t dstStride, + CONVERT_POLICY policy); + + void sub(const Size2D &size, + const u8 *src0Base, ptrdiff_t src0Stride, + const u8 *src1Base, ptrdiff_t src1Stride, + f32 *dstBase, ptrdiff_t dstStride); + + void sub(const Size2D &size, + const u8 *src0Base, ptrdiff_t src0Stride, + const s16 *src1Base, ptrdiff_t src1Stride, + s16 *dstBase, ptrdiff_t dstStride, + CONVERT_POLICY policy); + + void sub(const Size2D &size, + const s16 *src0Base, ptrdiff_t src0Stride, + const u8 *src1Base, ptrdiff_t src1Stride, + s16 *dstBase, ptrdiff_t dstStride, + CONVERT_POLICY policy); + + void sub(const Size2D &size, + const s16 *src0Base, ptrdiff_t src0Stride, + const s16 *src1Base, ptrdiff_t src1Stride, + s16 *dstBase, ptrdiff_t dstStride, + CONVERT_POLICY policy); + + void sub(const Size2D &size, + const s8 *src0Base, ptrdiff_t src0Stride, + const s8 *src1Base, ptrdiff_t src1Stride, + s8 *dstBase, ptrdiff_t dstStride, + CONVERT_POLICY policy); + + void sub(const Size2D &size, + const u16 * src0Base, ptrdiff_t src0Stride, + const u16 * src1Base, ptrdiff_t src1Stride, + u16 * dstBase, ptrdiff_t dstStride, + CONVERT_POLICY policy); + + void sub(const Size2D &size, + const s32 * src0Base, ptrdiff_t src0Stride, + const s32 * src1Base, ptrdiff_t src1Stride, + s32 * dstBase, ptrdiff_t dstStride, + CONVERT_POLICY policy); + + void sub(const Size2D &size, + const u32 * src0Base, ptrdiff_t src0Stride, + const u32 * src1Base, ptrdiff_t src1Stride, + u32 * dstBase, ptrdiff_t dstStride, + CONVERT_POLICY policy); + + void sub(const Size2D &size, + const f32 * src0Base, ptrdiff_t src0Stride, + const f32 * src1Base, ptrdiff_t src1Stride, + f32 * dstBase, ptrdiff_t dstStride); + + /* + For each point `p` within `size`, do: + dst[p] = src0[p] * alpha + src1[p] * beta + gamma + */ + void addWeighted(const Size2D &size, + const u8 * src0Base, ptrdiff_t src0Stride, + const u8 * src1Base, ptrdiff_t src1Stride, + u8 * dstBase, ptrdiff_t dstStride, + f32 alpha, f32 beta, f32 gamma); + + void addWeighted(const Size2D &size, + const s8 * src0Base, ptrdiff_t src0Stride, + const s8 * src1Base, ptrdiff_t src1Stride, + s8 * dstBase, ptrdiff_t dstStride, + f32 alpha, f32 beta, f32 gamma); + + void addWeighted(const Size2D &size, + const u16 * src0Base, ptrdiff_t src0Stride, + const u16 * src1Base, ptrdiff_t src1Stride, + u16 * dstBase, ptrdiff_t dstStride, + f32 alpha, f32 beta, f32 gamma); + + void addWeighted(const Size2D &size, + const s16 * src0Base, ptrdiff_t src0Stride, + const s16 * src1Base, ptrdiff_t src1Stride, + s16 * dstBase, ptrdiff_t dstStride, + f32 alpha, f32 beta, f32 gamma); + + void addWeighted(const Size2D &size, + const u32 * src0Base, ptrdiff_t src0Stride, + const u32 * src1Base, ptrdiff_t src1Stride, + u32 * dstBase, ptrdiff_t dstStride, + f32 alpha, f32 beta, f32 gamma); + + void addWeighted(const Size2D &size, + const s32 * src0Base, ptrdiff_t src0Stride, + const s32 * src1Base, ptrdiff_t src1Stride, + s32 * dstBase, ptrdiff_t dstStride, + f32 alpha, f32 beta, f32 gamma); + + void addWeighted(const Size2D &size, + const f32 * src0Base, ptrdiff_t src0Stride, + const f32 * src1Base, ptrdiff_t src1Stride, + f32 * dstBase, ptrdiff_t dstStride, + f32 alpha, f32 beta, f32 gamma); + + /* + For each point `p` within `size`, do: + dst[p] = min(src0[p], src1[p]) + */ + void min(const Size2D &size, + const u8 *src0Base, ptrdiff_t src0Stride, + const u8 *src1Base, ptrdiff_t src1Stride, + u8 *dstBase, ptrdiff_t dstStride); + + void min(const Size2D &size, + const s8 *src0Base, ptrdiff_t src0Stride, + const s8 *src1Base, ptrdiff_t src1Stride, + s8 *dstBase, ptrdiff_t dstStride); + + void min(const Size2D &size, + const u16 * src0Base, ptrdiff_t src0Stride, + const u16 * src1Base, ptrdiff_t src1Stride, + u16 * dstBase, ptrdiff_t dstStride); + + void min(const Size2D &size, + const s16 *src0Base, ptrdiff_t src0Stride, + const s16 *src1Base, ptrdiff_t src1Stride, + s16 *dstBase, ptrdiff_t dstStride); + + void min(const Size2D &size, + const s32 * src0Base, ptrdiff_t src0Stride, + const s32 * src1Base, ptrdiff_t src1Stride, + s32 * dstBase, ptrdiff_t dstStride); + + void min(const Size2D &size, + const u32 * src0Base, ptrdiff_t src0Stride, + const u32 * src1Base, ptrdiff_t src1Stride, + u32 * dstBase, ptrdiff_t dstStride); + + void min(const Size2D &size, + const f32 * src0Base, ptrdiff_t src0Stride, + const f32 * src1Base, ptrdiff_t src1Stride, + f32 * dstBase, ptrdiff_t dstStride); + + /* + For each point `p` within `size`, do: + dst[p] = max(src0[p], src1[p]) + */ + void max(const Size2D &size, + const u8 *src0Base, ptrdiff_t src0Stride, + const u8 *src1Base, ptrdiff_t src1Stride, + u8 *dstBase, ptrdiff_t dstStride); + + void max(const Size2D &size, + const s8 *src0Base, ptrdiff_t src0Stride, + const s8 *src1Base, ptrdiff_t src1Stride, + s8 *dstBase, ptrdiff_t dstStride); + + void max(const Size2D &size, + const u16 * src0Base, ptrdiff_t src0Stride, + const u16 * src1Base, ptrdiff_t src1Stride, + u16 * dstBase, ptrdiff_t dstStride); + + void max(const Size2D &size, + const s16 *src0Base, ptrdiff_t src0Stride, + const s16 *src1Base, ptrdiff_t src1Stride, + s16 *dstBase, ptrdiff_t dstStride); + + void max(const Size2D &size, + const s32 * src0Base, ptrdiff_t src0Stride, + const s32 * src1Base, ptrdiff_t src1Stride, + s32 * dstBase, ptrdiff_t dstStride); + + void max(const Size2D &size, + const u32 * src0Base, ptrdiff_t src0Stride, + const u32 * src1Base, ptrdiff_t src1Stride, + u32 * dstBase, ptrdiff_t dstStride); + + void max(const Size2D &size, + const f32 * src0Base, ptrdiff_t src0Stride, + const f32 * src1Base, ptrdiff_t src1Stride, + f32 * dstBase, ptrdiff_t dstStride); + + /* + For each point `p` within `size`, do: + dst[p] = src0[p] * src1[p] * scale + + NOTE: ROUND_TO_ZERO convert policy is used + */ + void mul(const Size2D &size, + const u8 * src0Base, ptrdiff_t src0Stride, + const u8 * src1Base, ptrdiff_t src1Stride, + u8 * dstBase, ptrdiff_t dstStride, + f32 scale, + CONVERT_POLICY cpolicy); + + void mul(const Size2D &size, + const u8 * src0Base, ptrdiff_t src0Stride, + const u8 * src1Base, ptrdiff_t src1Stride, + s16 * dstBase, ptrdiff_t dstStride, + f32 scale, + CONVERT_POLICY cpolicy); + + void mul(const Size2D &size, + const u8 * src0Base, ptrdiff_t src0Stride, + const s16 * src1Base, ptrdiff_t src1Stride, + s16 * dstBase, ptrdiff_t dstStride, + f32 scale, + CONVERT_POLICY cpolicy); + + void mul(const Size2D &size, + const s8 * src0Base, ptrdiff_t src0Stride, + const s8 * src1Base, ptrdiff_t src1Stride, + s8 * dstBase, ptrdiff_t dstStride, + f32 scale, + CONVERT_POLICY cpolicy); + + void mul(const Size2D &size, + const u16 * src0Base, ptrdiff_t src0Stride, + const u16 * src1Base, ptrdiff_t src1Stride, + u16 * dstBase, ptrdiff_t dstStride, + f32 scale, + CONVERT_POLICY cpolicy); + + void mul(const Size2D &size, + const s16 * src0Base, ptrdiff_t src0Stride, + const s16 * src1Base, ptrdiff_t src1Stride, + s16 * dstBase, ptrdiff_t dstStride, + f32 scale, + CONVERT_POLICY cpolicy); + + void mul(const Size2D &size, + const s32 * src0Base, ptrdiff_t src0Stride, + const s32 * src1Base, ptrdiff_t src1Stride, + s32 * dstBase, ptrdiff_t dstStride, + f64 scale, + CONVERT_POLICY cpolicy); + + void mul(const Size2D &size, + const f32 * src0Base, ptrdiff_t src0Stride, + const f32 * src1Base, ptrdiff_t src1Stride, + f32 * dstBase, ptrdiff_t dstStride, + f32 scale); + + /* + For each point `p` within `size`, do: + dst[p] = src0[p] * scale / src1[p] + + NOTE: ROUND_TO_ZERO convert policy is used + */ + void div(const Size2D &size, + const u8 * src0Base, ptrdiff_t src0Stride, + const u8 * src1Base, ptrdiff_t src1Stride, + u8 * dstBase, ptrdiff_t dstStride, + f32 scale, + CONVERT_POLICY cpolicy); + + void div(const Size2D &size, + const u8 * src0Base, ptrdiff_t src0Stride, + const u8 * src1Base, ptrdiff_t src1Stride, + s16 * dstBase, ptrdiff_t dstStride, + f32 scale, + CONVERT_POLICY cpolicy); + + void div(const Size2D &size, + const u8 * src0Base, ptrdiff_t src0Stride, + const s16 * src1Base, ptrdiff_t src1Stride, + s16 * dstBase, ptrdiff_t dstStride, + f32 scale, + CONVERT_POLICY cpolicy); + + void div(const Size2D &size, + const s8 * src0Base, ptrdiff_t src0Stride, + const s8 * src1Base, ptrdiff_t src1Stride, + s8 * dstBase, ptrdiff_t dstStride, + f32 scale, + CONVERT_POLICY cpolicy); + + void div(const Size2D &size, + const u16 * src0Base, ptrdiff_t src0Stride, + const u16 * src1Base, ptrdiff_t src1Stride, + u16 * dstBase, ptrdiff_t dstStride, + f32 scale, + CONVERT_POLICY cpolicy); + + void div(const Size2D &size, + const s16 * src0Base, ptrdiff_t src0Stride, + const s16 * src1Base, ptrdiff_t src1Stride, + s16 * dstBase, ptrdiff_t dstStride, + f32 scale, + CONVERT_POLICY cpolicy); + + void div(const Size2D &size, + const s32 * src0Base, ptrdiff_t src0Stride, + const s32 * src1Base, ptrdiff_t src1Stride, + s32 * dstBase, ptrdiff_t dstStride, + f32 scale, + CONVERT_POLICY cpolicy); + + void div(const Size2D &size, + const f32 * src0Base, ptrdiff_t src0Stride, + const f32 * src1Base, ptrdiff_t src1Stride, + f32 * dstBase, ptrdiff_t dstStride, + f32 scale); + + /* + For each point `p` within `size`, do: + dst[p] = scale / src[p] + + NOTE: ROUND_TO_ZERO convert policy is used + */ + void reciprocal(const Size2D &size, + const u8 * srcBase, ptrdiff_t srcStride, + u8 * dstBase, ptrdiff_t dstStride, + f32 scale, + CONVERT_POLICY cpolicy); + + void reciprocal(const Size2D &size, + const s8 * srcBase, ptrdiff_t srcStride, + s8 * dstBase, ptrdiff_t dstStride, + f32 scale, + CONVERT_POLICY cpolicy); + + void reciprocal(const Size2D &size, + const u16 * srcBase, ptrdiff_t srcStride, + u16 * dstBase, ptrdiff_t dstStride, + f32 scale, + CONVERT_POLICY cpolicy); + + void reciprocal(const Size2D &size, + const s16 * srcBase, ptrdiff_t srcStride, + s16 * dstBase, ptrdiff_t dstStride, + f32 scale, + CONVERT_POLICY cpolicy); + + void reciprocal(const Size2D &size, + const s32 * srcBase, ptrdiff_t srcStride, + s32 * dstBase, ptrdiff_t dstStride, + f32 scale, + CONVERT_POLICY cpolicy); + + void reciprocal(const Size2D &size, + const f32 * srcBase, ptrdiff_t srcStride, + f32 * dstBase, ptrdiff_t dstStride, + f32 scale); + + /* + For each point `p` within `size`, set `dst[p]` to the median + of `src[p]` and the 8 points around it. If `srcMargin` is + zero on any side, get the neighbors on that side by replicating + the edge. + */ + bool isMedianFilter3x3Supported(const Size2D &size, u32 numChannels); + void medianFilter3x3(const Size2D &size, u32 numChannels, + const u8 *srcBase, ptrdiff_t srcStride, + const Margin &srcMargin, + u8 *dstBase, ptrdiff_t dstStride); + + /* + Apply a half Gaussian filter + half Scale, as one level of a Gaussian + pyramid. For all `p` within `dstSize`, set `dst[p]` to `f[2 * p]`, where + `f` is an image of size srcSize obtained by filtering src with the 5x5 + Gaussian kernel ([1 4 6 4 1]'*[1 4 6 4 1]/256) using the border mode + passed in, and round-to-zero rounding. + dstSize must be (srcSize.width / 2, srcSize.height / 2), rounded by any method. + */ + bool isGaussianPyramidDownRTZSupported(const Size2D &srcSize, const Size2D &dstSize, BORDER_MODE border); + void gaussianPyramidDownRTZ(const Size2D &srcSize, + const u8 *srcBase, ptrdiff_t srcStride, + const Size2D &dstSize, + u8 *dstBase, ptrdiff_t dstStride, + BORDER_MODE border, u8 borderValue); + + /* Same as above, but uses round-half-up rounding. */ + + bool isGaussianPyramidDownU8Supported(const Size2D &srcSize, const Size2D &dstSize, u8 cn); + void gaussianPyramidDown(const Size2D &srcSize, + const u8 *srcBase, ptrdiff_t srcStride, + const Size2D &dstSize, + u8 *dstBase, ptrdiff_t dstStride, u8 cn); + + + bool isGaussianPyramidDownS16Supported(const Size2D &srcSize, const Size2D &dstSize, u8 cn); + void gaussianPyramidDown(const Size2D &srcSize, + const s16 *srcBase, ptrdiff_t srcStride, + const Size2D &dstSize, + s16 *dstBase, ptrdiff_t dstStride, u8 cn); + + bool isGaussianPyramidDownF32Supported(const Size2D &srcSize, const Size2D &dstSize, u8 cn); + void gaussianPyramidDown(const Size2D &srcSize, + const f32 *srcBase, ptrdiff_t srcStride, + const Size2D &dstSize, + f32 *dstBase, ptrdiff_t dstStride, u8 cn); + + bool isGaussianPyramidUpU8Supported(const Size2D &srcSize, const Size2D &dstSize, u8 cn); + void gaussianPyramidUp(const Size2D &srcSize, + const u8 *srcBase, ptrdiff_t srcStride, + const Size2D &dstSize, + u8 *dstBase, ptrdiff_t dstStride, u8 cn); + + bool isGaussianPyramidUpS16Supported(const Size2D &srcSize, const Size2D &dstSize, u8 cn); + void gaussianPyramidUp(const Size2D &srcSize, + const s16 *srcBase, ptrdiff_t srcStride, + const Size2D &dstSize, + s16 *dstBase, ptrdiff_t dstStride, u8 cn); + + /* + For each point `p` within `size`, do: + dst[p] = src[p] > threshold ? trueValue : falseValue + */ + void thresholdBinary(const Size2D &size, + const u8 *srcBase, ptrdiff_t srcStride, + u8 *dstBase, ptrdiff_t dstStride, + u8 threshold, u8 trueValue = 255, u8 falseValue = 0); + + /* + For each point `p` within `size`, do: + dst[p] = lower <= src[p] && src[p] <= upper ? trueValue : falseValue + */ + void thresholdRange(const Size2D &size, + const u8 *srcBase, ptrdiff_t srcStride, + u8 *dstBase, ptrdiff_t dstStride, + u8 lowerThreshold, u8 upperThreshold, + u8 trueValue = 255, u8 falseValue = 0); + + /* + For each point `p` within `size`, do: + dst[p] = src[p] > threshold ? value : 0 + */ + void thresholdBinary(const Size2D &size, + const u8 *srcBase, ptrdiff_t srcStride, + u8 *dstBase, ptrdiff_t dstStride, + u8 threshold, u8 value); + + void thresholdBinary(const Size2D &size, + const s8 *srcBase, ptrdiff_t srcStride, + s8 *dstBase, ptrdiff_t dstStride, + s8 threshold, s8 value); + + void thresholdBinary(const Size2D &size, + const u16 *srcBase, ptrdiff_t srcStride, + u16 *dstBase, ptrdiff_t dstStride, + u16 threshold, u16 value); + + void thresholdBinary(const Size2D &size, + const s16 *srcBase, ptrdiff_t srcStride, + s16 *dstBase, ptrdiff_t dstStride, + s16 threshold, s16 value); + + void thresholdBinary(const Size2D &size, + const s32 *srcBase, ptrdiff_t srcStride, + s32 *dstBase, ptrdiff_t dstStride, + s32 threshold, s32 value); + + void thresholdBinary(const Size2D &size, + const f32 *srcBase, ptrdiff_t srcStride, + f32 *dstBase, ptrdiff_t dstStride, + f32 threshold, f32 value); + + /* + For each point `p` within `size`, do: + dst[p] = src[p] > threshold ? 0 : value + */ + void thresholdBinaryInv(const Size2D &size, + const u8 *srcBase, ptrdiff_t srcStride, + u8 *dstBase, ptrdiff_t dstStride, + u8 threshold, u8 value); + + void thresholdBinaryInv(const Size2D &size, + const s8 *srcBase, ptrdiff_t srcStride, + s8 *dstBase, ptrdiff_t dstStride, + s8 threshold, s8 value); + + void thresholdBinaryInv(const Size2D &size, + const u16 *srcBase, ptrdiff_t srcStride, + u16 *dstBase, ptrdiff_t dstStride, + u16 threshold, u16 value); + + void thresholdBinaryInv(const Size2D &size, + const s16 *srcBase, ptrdiff_t srcStride, + s16 *dstBase, ptrdiff_t dstStride, + s16 threshold, s16 value); + + void thresholdBinaryInv(const Size2D &size, + const s32 *srcBase, ptrdiff_t srcStride, + s32 *dstBase, ptrdiff_t dstStride, + s32 threshold, s32 value); + + void thresholdBinaryInv(const Size2D &size, + const f32 *srcBase, ptrdiff_t srcStride, + f32 *dstBase, ptrdiff_t dstStride, + f32 threshold, f32 value); + + /* + For each point `p` within `size`, do: + dst[p] = src[p] > threshold ? threshold : src[p] + */ + void thresholdTruncate(const Size2D &size, + const u8 *srcBase, ptrdiff_t srcStride, + u8 *dstBase, ptrdiff_t dstStride, + u8 threshold); + + void thresholdTruncate(const Size2D &size, + const s8 *srcBase, ptrdiff_t srcStride, + s8 *dstBase, ptrdiff_t dstStride, + s8 threshold); + + void thresholdTruncate(const Size2D &size, + const u16 *srcBase, ptrdiff_t srcStride, + u16 *dstBase, ptrdiff_t dstStride, + u16 threshold); + + void thresholdTruncate(const Size2D &size, + const s16 *srcBase, ptrdiff_t srcStride, + s16 *dstBase, ptrdiff_t dstStride, + s16 threshold); + + void thresholdTruncate(const Size2D &size, + const s32 *srcBase, ptrdiff_t srcStride, + s32 *dstBase, ptrdiff_t dstStride, + s32 threshold); + + void thresholdTruncate(const Size2D &size, + const f32 *srcBase, ptrdiff_t srcStride, + f32 *dstBase, ptrdiff_t dstStride, + f32 threshold); + + /* + For each point `p` within `size`, do: + dst[p] = src[p] > threshold ? src[p] : 0 + */ + void thresholdToZero(const Size2D &size, + const u8 *srcBase, ptrdiff_t srcStride, + u8 *dstBase, ptrdiff_t dstStride, + u8 threshold); + + void thresholdToZero(const Size2D &size, + const s8 *srcBase, ptrdiff_t srcStride, + s8 *dstBase, ptrdiff_t dstStride, + s8 threshold); + + void thresholdToZero(const Size2D &size, + const u16 *srcBase, ptrdiff_t srcStride, + u16 *dstBase, ptrdiff_t dstStride, + u16 threshold); + + void thresholdToZero(const Size2D &size, + const s16 *srcBase, ptrdiff_t srcStride, + s16 *dstBase, ptrdiff_t dstStride, + s16 threshold); + + void thresholdToZero(const Size2D &size, + const s32 *srcBase, ptrdiff_t srcStride, + s32 *dstBase, ptrdiff_t dstStride, + s32 threshold); + + void thresholdToZero(const Size2D &size, + const f32 *srcBase, ptrdiff_t srcStride, + f32 *dstBase, ptrdiff_t dstStride, + f32 threshold); + + /* + For each point `p` within `size`, do: + dst[p] = src[p] > threshold ? 0 : src[p] + */ + void thresholdToZeroInv(const Size2D &size, + const u8 *srcBase, ptrdiff_t srcStride, + u8 *dstBase, ptrdiff_t dstStride, + u8 threshold); + + void thresholdToZeroInv(const Size2D &size, + const s8 *srcBase, ptrdiff_t srcStride, + s8 *dstBase, ptrdiff_t dstStride, + s8 threshold); + + void thresholdToZeroInv(const Size2D &size, + const u16 *srcBase, ptrdiff_t srcStride, + u16 *dstBase, ptrdiff_t dstStride, + u16 threshold); + + void thresholdToZeroInv(const Size2D &size, + const s16 *srcBase, ptrdiff_t srcStride, + s16 *dstBase, ptrdiff_t dstStride, + s16 threshold); + + void thresholdToZeroInv(const Size2D &size, + const s32 *srcBase, ptrdiff_t srcStride, + s32 *dstBase, ptrdiff_t dstStride, + s32 threshold); + + void thresholdToZeroInv(const Size2D &size, + const f32 *srcBase, ptrdiff_t srcStride, + f32 *dstBase, ptrdiff_t dstStride, + f32 threshold); + + /* + For each point `p` within `size`, do: + dst[p] = abs(src0[p] - src1[p]) + */ + void absDiff(const Size2D &size, + const u8 *src0Base, ptrdiff_t src0Stride, + const u8 *src1Base, ptrdiff_t src1Stride, + u8 *dstBase, ptrdiff_t dstStride); + + void absDiff(const Size2D &size, + const u16 *src0Base, ptrdiff_t src0Stride, + const u16 *src1Base, ptrdiff_t src1Stride, + u16 *dstBase, ptrdiff_t dstStride); + + void absDiff(const Size2D &size, + const s8 *src0Base, ptrdiff_t src0Stride, + const s8 *src1Base, ptrdiff_t src1Stride, + s8 *dstBase, ptrdiff_t dstStride); + + void absDiff(const Size2D &size, + const s16 *src0Base, ptrdiff_t src0Stride, + const s16 *src1Base, ptrdiff_t src1Stride, + s16 *dstBase, ptrdiff_t dstStride); + + void absDiff(const Size2D &size, + const s32 * src0Base, ptrdiff_t src0Stride, + const s32 * src1Base, ptrdiff_t src1Stride, + s32 * dstBase, ptrdiff_t dstStride); + + void absDiff(const Size2D &size, + const f32 * src0Base, ptrdiff_t src0Stride, + const f32 * src1Base, ptrdiff_t src1Stride, + f32 * dstBase, ptrdiff_t dstStride); + + /* + For each point `p` within `size`, do: + dst[p] = ~src[p] + */ + void bitwiseNot(const Size2D &size, + const u8 *srcBase, ptrdiff_t srcStride, + u8 *dstBase, ptrdiff_t dstStride); + + /* + For each point `p` within `size`, do: + dst[p] = src0[p] & src1[p] + */ + void bitwiseAnd(const Size2D &size, + const u8 *src0Base, ptrdiff_t src0Stride, + const u8 *src1Base, ptrdiff_t src1Stride, + u8 *dstBase, ptrdiff_t dstStride); + + /* + For each point `p` within `size`, do: + dst[p] = src0[p] | src1[p] + */ + void bitwiseOr(const Size2D &size, + const u8 *src0Base, ptrdiff_t src0Stride, + const u8 *src1Base, ptrdiff_t src1Stride, + u8 *dstBase, ptrdiff_t dstStride); + + /* + For each point `p` within `size`, do: + dst[p] = src0[p] ^ src1[p] + */ + void bitwiseXor(const Size2D &size, + const u8 *src0Base, ptrdiff_t src0Stride, + const u8 *src1Base, ptrdiff_t src1Stride, + u8 *dstBase, ptrdiff_t dstStride); + + /* + For each point `p` within `size`, do: + dst[p] = src0[p] == src1[p] ? 255 : 0 + */ + void cmpEQ(const Size2D &size, + const u8 *src0Base, ptrdiff_t src0Stride, + const u8 *src1Base, ptrdiff_t src1Stride, + u8 *dstBase, ptrdiff_t dstStride); + + void cmpEQ(const Size2D &size, + const s8 *src0Base, ptrdiff_t src0Stride, + const s8 *src1Base, ptrdiff_t src1Stride, + u8 *dstBase, ptrdiff_t dstStride); + + void cmpEQ(const Size2D &size, + const u16 *src0Base, ptrdiff_t src0Stride, + const u16 *src1Base, ptrdiff_t src1Stride, + u8 *dstBase, ptrdiff_t dstStride); + + void cmpEQ(const Size2D &size, + const s16 *src0Base, ptrdiff_t src0Stride, + const s16 *src1Base, ptrdiff_t src1Stride, + u8 *dstBase, ptrdiff_t dstStride); + + void cmpEQ(const Size2D &size, + const u32 *src0Base, ptrdiff_t src0Stride, + const u32 *src1Base, ptrdiff_t src1Stride, + u8 *dstBase, ptrdiff_t dstStride); + + void cmpEQ(const Size2D &size, + const s32 *src0Base, ptrdiff_t src0Stride, + const s32 *src1Base, ptrdiff_t src1Stride, + u8 *dstBase, ptrdiff_t dstStride); + + void cmpEQ(const Size2D &size, + const f32 *src0Base, ptrdiff_t src0Stride, + const f32 *src1Base, ptrdiff_t src1Stride, + u8 *dstBase, ptrdiff_t dstStride); + + /* + For each point `p` within `size`, do: + dst[p] = src0[p] != src1[p] ? 255 : 0 + */ + void cmpNE(const Size2D &size, + const u8 *src0Base, ptrdiff_t src0Stride, + const u8 *src1Base, ptrdiff_t src1Stride, + u8 *dstBase, ptrdiff_t dstStride); + + void cmpNE(const Size2D &size, + const s8 *src0Base, ptrdiff_t src0Stride, + const s8 *src1Base, ptrdiff_t src1Stride, + u8 *dstBase, ptrdiff_t dstStride); + + void cmpNE(const Size2D &size, + const u16 *src0Base, ptrdiff_t src0Stride, + const u16 *src1Base, ptrdiff_t src1Stride, + u8 *dstBase, ptrdiff_t dstStride); + + void cmpNE(const Size2D &size, + const s16 *src0Base, ptrdiff_t src0Stride, + const s16 *src1Base, ptrdiff_t src1Stride, + u8 *dstBase, ptrdiff_t dstStride); + + void cmpNE(const Size2D &size, + const u32 *src0Base, ptrdiff_t src0Stride, + const u32 *src1Base, ptrdiff_t src1Stride, + u8 *dstBase, ptrdiff_t dstStride); + + void cmpNE(const Size2D &size, + const s32 *src0Base, ptrdiff_t src0Stride, + const s32 *src1Base, ptrdiff_t src1Stride, + u8 *dstBase, ptrdiff_t dstStride); + + void cmpNE(const Size2D &size, + const f32 *src0Base, ptrdiff_t src0Stride, + const f32 *src1Base, ptrdiff_t src1Stride, + u8 *dstBase, ptrdiff_t dstStride); + + /* + For each point `p` within `size`, do: + dst[p] = src0[p] > src1[p] ? 255 : 0 + */ + void cmpGT(const Size2D &size, + const u8 *src0Base, ptrdiff_t src0Stride, + const u8 *src1Base, ptrdiff_t src1Stride, + u8 *dstBase, ptrdiff_t dstStride); + + void cmpGT(const Size2D &size, + const s8 *src0Base, ptrdiff_t src0Stride, + const s8 *src1Base, ptrdiff_t src1Stride, + u8 *dstBase, ptrdiff_t dstStride); + + void cmpGT(const Size2D &size, + const u16 *src0Base, ptrdiff_t src0Stride, + const u16 *src1Base, ptrdiff_t src1Stride, + u8 *dstBase, ptrdiff_t dstStride); + + void cmpGT(const Size2D &size, + const s16 *src0Base, ptrdiff_t src0Stride, + const s16 *src1Base, ptrdiff_t src1Stride, + u8 *dstBase, ptrdiff_t dstStride); + + void cmpGT(const Size2D &size, + const u32 *src0Base, ptrdiff_t src0Stride, + const u32 *src1Base, ptrdiff_t src1Stride, + u8 *dstBase, ptrdiff_t dstStride); + + void cmpGT(const Size2D &size, + const s32 *src0Base, ptrdiff_t src0Stride, + const s32 *src1Base, ptrdiff_t src1Stride, + u8 *dstBase, ptrdiff_t dstStride); + + void cmpGT(const Size2D &size, + const f32 *src0Base, ptrdiff_t src0Stride, + const f32 *src1Base, ptrdiff_t src1Stride, + u8 *dstBase, ptrdiff_t dstStride); + + /* + For each point `p` within `size`, do: + dst[p] = src0[p] >= src1[p] ? 255 : 0 + */ + void cmpGE(const Size2D &size, + const u8 *src0Base, ptrdiff_t src0Stride, + const u8 *src1Base, ptrdiff_t src1Stride, + u8 *dstBase, ptrdiff_t dstStride); + + void cmpGE(const Size2D &size, + const s8 *src0Base, ptrdiff_t src0Stride, + const s8 *src1Base, ptrdiff_t src1Stride, + u8 *dstBase, ptrdiff_t dstStride); + + void cmpGE(const Size2D &size, + const u16 *src0Base, ptrdiff_t src0Stride, + const u16 *src1Base, ptrdiff_t src1Stride, + u8 *dstBase, ptrdiff_t dstStride); + + void cmpGE(const Size2D &size, + const s16 *src0Base, ptrdiff_t src0Stride, + const s16 *src1Base, ptrdiff_t src1Stride, + u8 *dstBase, ptrdiff_t dstStride); + + void cmpGE(const Size2D &size, + const u32 *src0Base, ptrdiff_t src0Stride, + const u32 *src1Base, ptrdiff_t src1Stride, + u8 *dstBase, ptrdiff_t dstStride); + + void cmpGE(const Size2D &size, + const s32 *src0Base, ptrdiff_t src0Stride, + const s32 *src1Base, ptrdiff_t src1Stride, + u8 *dstBase, ptrdiff_t dstStride); + + void cmpGE(const Size2D &size, + const f32 *src0Base, ptrdiff_t src0Stride, + const f32 *src1Base, ptrdiff_t src1Stride, + u8 *dstBase, ptrdiff_t dstStride); + + /* + Calculates dot product + */ + f64 dotProduct(const Size2D &size, + const u8 * src0Base, ptrdiff_t src0Stride, + const u8 * src1Base, ptrdiff_t src1Stride); + + f64 dotProduct(const Size2D &size, + const s8 * src0Base, ptrdiff_t src0Stride, + const s8 * src1Base, ptrdiff_t src1Stride); + + f64 dotProduct(const Size2D &size, + const f32 * src0Base, ptrdiff_t src0Stride, + const f32 * src1Base, ptrdiff_t src1Stride); + + /* + Calculates mean and stddev + */ + void meanStdDev(const Size2D &size, + const u8 * srcBase, ptrdiff_t srcStride, + f32 * pMean, f32 * pStdDev); + + void meanStdDev(const Size2D &size, + const u16 * srcBase, ptrdiff_t srcStride, + f32 * pMean, f32 * pStdDev); + + /* + For each point `p` within `size`, do: + dst[p] = sqrt(src0[p] ^ 2 + src1[p] ^ 2) + */ + void magnitude(const Size2D &size, + const s16 *src0Base, ptrdiff_t src0Stride, + const s16 *src1Base, ptrdiff_t src1Stride, + s16 *dstBase, ptrdiff_t dstStride); + + void magnitude(const Size2D &size, + const f32 *src0Base, ptrdiff_t src0Stride, + const f32 *src1Base, ptrdiff_t src1Stride, + f32 *dstBase, ptrdiff_t dstStride); + + /* + Compute an integral image + */ + void integral(const Size2D &size, + const u8 * srcBase, ptrdiff_t srcStride, + u32 * sumBase, ptrdiff_t sumStride); + + /* + Compute an integral of squared image values + */ + void sqrIntegral(const Size2D &size, + const u8 * srcBase, ptrdiff_t srcStride, + f64 * sqsumBase, ptrdiff_t sqsumStride); + + /* + Among each pixel `p` within `src` find min and max values + */ + void minMaxVals(const Size2D &size, + const u8 *srcBase, ptrdiff_t srcStride, + u8 * minVal, u8 * maxVal); + + void minMaxVals(const Size2D &size, + const s16 *srcBase, ptrdiff_t srcStride, + s16 * minVal, s16 * maxVal); + + void minMaxVals(const Size2D &size, + const u16 *srcBase, ptrdiff_t srcStride, + u16 * minVal, u16 * maxVal); + + void minMaxVals(const Size2D &size, + const s32 *srcBase, ptrdiff_t srcStride, + s32 * minVal, s32 * maxVal); + + void minMaxVals(const Size2D &size, + const u32 *srcBase, ptrdiff_t srcStride, + u32 * minVal, u32 * maxVal); + + /* + Fill the arrays `minLocPtr`, `maxLocPtr` with locations of + given values `minVal`, `maxVal` + */ + void fillMinMaxLocs(const Size2D & size, + const u8 *srcBase, ptrdiff_t srcStride, + u8 minVal, size_t * minLocPtr, s32 & minLocCount, s32 minLocCapacity, + u8 maxVal, size_t * maxLocPtr, s32 & maxLocCount, s32 maxLocCapacity); + + void fillMinMaxLocs(const Size2D & size, + const u16 *srcBase, ptrdiff_t srcStride, + u16 minVal, size_t * minLocPtr, s32 & minLocCount, s32 minLocCapacity, + u16 maxVal, size_t * maxLocPtr, s32 & maxLocCount, s32 maxLocCapacity); + + void fillMinMaxLocs(const Size2D & size, + const s16 *srcBase, ptrdiff_t srcStride, + s16 minVal, size_t * minLocPtr, s32 & minLocCount, s32 minLocCapacity, + s16 maxVal, size_t * maxLocPtr, s32 & maxLocCount, s32 maxLocCapacity); + + void fillMinMaxLocs(const Size2D & size, + const u32 *srcBase, ptrdiff_t srcStride, + u32 minVal, size_t * minLocPtr, s32 & minLocCount, s32 minLocCapacity, + u32 maxVal, size_t * maxLocPtr, s32 & maxLocCount, s32 maxLocCapacity); + + void fillMinMaxLocs(const Size2D & size, + const s32 *srcBase, ptrdiff_t srcStride, + s32 minVal, size_t * minLocPtr, s32 & minLocCount, s32 minLocCapacity, + s32 maxVal, size_t * maxLocPtr, s32 & maxLocCount, s32 maxLocCapacity); + + /* + Among each pixel `p` within `src` find min and max values and its first occurences + */ + void minMaxLoc(const Size2D &size, + const s8 * srcBase, ptrdiff_t srcStride, + s8 &minVal, size_t &minCol, size_t &minRow, + s8 &maxVal, size_t &maxCol, size_t &maxRow); + + void minMaxLoc(const Size2D &size, + const u8 * srcBase, ptrdiff_t srcStride, + u8 &minVal, size_t &minCol, size_t &minRow, + u8 &maxVal, size_t &maxCol, size_t &maxRow); + + void minMaxLoc(const Size2D &size, + const s16 * srcBase, ptrdiff_t srcStride, + s16 &minVal, size_t &minCol, size_t &minRow, + s16 &maxVal, size_t &maxCol, size_t &maxRow); + + void minMaxLoc(const Size2D &size, + const u16 * srcBase, ptrdiff_t srcStride, + u16 &minVal, size_t &minCol, size_t &minRow, + u16 &maxVal, size_t &maxCol, size_t &maxRow); + + void minMaxLoc(const Size2D &size, + const s32 * srcBase, ptrdiff_t srcStride, + s32 &minVal, size_t &minCol, size_t &minRow, + s32 &maxVal, size_t &maxCol, size_t &maxRow); + + void minMaxLoc(const Size2D &size, + const f32 * srcBase, ptrdiff_t srcStride, + f32 &minVal, size_t &minCol, size_t &minRow, + f32 &maxVal, size_t &maxCol, size_t &maxRow); + + void minMaxLoc(const Size2D &size, + const f32 * srcBase, ptrdiff_t srcStride, + const u8 * maskBase, ptrdiff_t maskStride, + f32 &minVal, size_t &minCol, size_t &minRow, + f32 &maxVal, size_t &maxCol, size_t &maxRow); + + /* + For each point `p` within `size`, do: + dst[p] += src[p] + */ + void accumulate(const Size2D &size, + const u8 *srcBase, ptrdiff_t srcStride, + s16 *dstBase, ptrdiff_t dstStride); + + /* + For each point `p` within `size`, do: + dst[p] = (dst[p] + ((src[p] ^ 2) >> shift)) + */ + void accumulateSquare(const Size2D &size, + const u8 *srcBase, ptrdiff_t srcStride, + s16 *dstBase, ptrdiff_t dstStride, + u32 shift); + + /* + For each point `p` within `size`, do: + dst[p] = (1 - alpha) * dst[p] + alpha * src[p] + */ + void accumulateWeighted(const Size2D &size, + const u8 *srcBase, ptrdiff_t srcStride, + u8 *dstBase, ptrdiff_t dstStride, + f32 alpha); + + /* + orient[p] = atan2(src0[p], src1[p]) + */ + void phase(const Size2D &size, + const s16 * src0Base, ptrdiff_t src0Stride, + const s16 * src1Base, ptrdiff_t src1Stride, + u8 * orientBase, ptrdiff_t orientStride); + + void phase(const Size2D &size, + const f32 * src0Base, ptrdiff_t src0Stride, + const f32 * src1Base, ptrdiff_t src1Stride, + f32 * orientBase, ptrdiff_t orientStride, + f32 scale); + + /* + Combine 2 planes to a single one + */ + void combine2(const Size2D &size, + const u8 * src0Base, ptrdiff_t src0Stride, + const u8 * src1Base, ptrdiff_t src1Stride, + u8 * dstBase, ptrdiff_t dstStride); + + void combine2(const Size2D &size, + const u16 * src0Base, ptrdiff_t src0Stride, + const u16 * src1Base, ptrdiff_t src1Stride, + u16 * dstBase, ptrdiff_t dstStride); + + void combine2(const Size2D &size, + const s32 * src0Base, ptrdiff_t src0Stride, + const s32 * src1Base, ptrdiff_t src1Stride, + s32 * dstBase, ptrdiff_t dstStride); + + void combine2(const Size2D &size, + const s64 * src0Base, ptrdiff_t src0Stride, + const s64 * src1Base, ptrdiff_t src1Stride, + s64 * dstBase, ptrdiff_t dstStride); + + /* + Combine 3 planes to a single one + */ + void combine3(const Size2D &size, + const u8 * src0Base, ptrdiff_t src0Stride, + const u8 * src1Base, ptrdiff_t src1Stride, + const u8 * src2Base, ptrdiff_t src2Stride, + u8 * dstBase, ptrdiff_t dstStride); + + void combine3(const Size2D &size, + const u16 * src0Base, ptrdiff_t src0Stride, + const u16 * src1Base, ptrdiff_t src1Stride, + const u16 * src2Base, ptrdiff_t src2Stride, + u16 * dstBase, ptrdiff_t dstStride); + + void combine3(const Size2D &size, + const s32 * src0Base, ptrdiff_t src0Stride, + const s32 * src1Base, ptrdiff_t src1Stride, + const s32 * src2Base, ptrdiff_t src2Stride, + s32 * dstBase, ptrdiff_t dstStride); + + void combine3(const Size2D &size, + const s64 * src0Base, ptrdiff_t src0Stride, + const s64 * src1Base, ptrdiff_t src1Stride, + const s64 * src2Base, ptrdiff_t src2Stride, + s64 * dstBase, ptrdiff_t dstStride); + + /* + Combine 4 planes to a single one + */ + void combine4(const Size2D &size, + const u8 * src0Base, ptrdiff_t src0Stride, + const u8 * src1Base, ptrdiff_t src1Stride, + const u8 * src2Base, ptrdiff_t src2Stride, + const u8 * src3Base, ptrdiff_t src3Stride, + u8 * dstBase, ptrdiff_t dstStride); + + void combine4(const Size2D &size, + const u16 * src0Base, ptrdiff_t src0Stride, + const u16 * src1Base, ptrdiff_t src1Stride, + const u16 * src2Base, ptrdiff_t src2Stride, + const u16 * src3Base, ptrdiff_t src3Stride, + u16 * dstBase, ptrdiff_t dstStride); + + void combine4(const Size2D &size, + const s32 * src0Base, ptrdiff_t src0Stride, + const s32 * src1Base, ptrdiff_t src1Stride, + const s32 * src2Base, ptrdiff_t src2Stride, + const s32 * src3Base, ptrdiff_t src3Stride, + s32 * dstBase, ptrdiff_t dstStride); + + void combine4(const Size2D &size, + const s64 * src0Base, ptrdiff_t src0Stride, + const s64 * src1Base, ptrdiff_t src1Stride, + const s64 * src2Base, ptrdiff_t src2Stride, + const s64 * src3Base, ptrdiff_t src3Stride, + s64 * dstBase, ptrdiff_t dstStride); + + /* + Combine 3 planes to YUYV one + */ + void combineYUYV(const Size2D &size, + const u8 * srcyBase, ptrdiff_t srcyStride, + const u8 * srcuBase, ptrdiff_t srcuStride, + const u8 * srcvBase, ptrdiff_t srcvStride, + u8 * dstBase, ptrdiff_t dstStride); + + /* + Combine 3 planes to UYVY one + */ + void combineUYVY(const Size2D &size, + const u8 * srcyBase, ptrdiff_t srcyStride, + const u8 * srcuBase, ptrdiff_t srcuStride, + const u8 * srcvBase, ptrdiff_t srcvStride, + u8 * dstBase, ptrdiff_t dstStride); + + /* + Convert RGB image to grayscale one + */ + void rgb2gray(const Size2D &size, COLOR_SPACE color_space, + const u8 * srcBase, ptrdiff_t srcStride, + u8 * dstBase, ptrdiff_t dstStride); + + /* + Convert RGBX image to grayscale one + */ + void rgbx2gray(const Size2D &size, COLOR_SPACE color_space, + const u8 * srcBase, ptrdiff_t srcStride, + u8 * dstBase, ptrdiff_t dstStride); + + /* + Convert BGR image to grayscale one + */ + void bgr2gray(const Size2D &size, COLOR_SPACE color_space, + const u8 * srcBase, ptrdiff_t srcStride, + u8 * dstBase, ptrdiff_t dstStride); + + /* + Convert BGRX image to grayscale one + */ + void bgrx2gray(const Size2D &size, COLOR_SPACE color_space, + const u8 * srcBase, ptrdiff_t srcStride, + u8 * dstBase, ptrdiff_t dstStride); + + /* + Convert grayscale image to RGB one + */ + void gray2rgb(const Size2D &size, + const u8 * srcBase, ptrdiff_t srcStride, + u8 * dstBase, ptrdiff_t dstStride); + + /* + Convert grayscale image to RGBX one + */ + void gray2rgbx(const Size2D &size, + const u8 * srcBase, ptrdiff_t srcStride, + u8 * dstBase, ptrdiff_t dstStride); + + /* + Convert RGB image to RGBX + */ + void rgb2rgbx(const Size2D &size, + const u8 * srcBase, ptrdiff_t srcStride, + u8 * dstBase, ptrdiff_t dstStride); + + /* + Convert RGBX image to RGB + */ + void rgbx2rgb(const Size2D &size, + const u8 * srcBase, ptrdiff_t srcStride, + u8 * dstBase, ptrdiff_t dstStride); + + /* + Convert RGB image to BGR + */ + void rgb2bgr(const Size2D &size, + const u8 * srcBase, ptrdiff_t srcStride, + u8 * dstBase, ptrdiff_t dstStride); + + /* + Convert RGBX image to BGRX + */ + void rgbx2bgrx(const Size2D &size, + const u8 * srcBase, ptrdiff_t srcStride, + u8 * dstBase, ptrdiff_t dstStride); + + /* + Convert RGBX image to BGR + */ + void rgbx2bgr(const Size2D &size, + const u8 * srcBase, ptrdiff_t srcStride, + u8 * dstBase, ptrdiff_t dstStride); + + /* + Convert RGB image to BGRX + */ + void rgb2bgrx(const Size2D &size, + const u8 * srcBase, ptrdiff_t srcStride, + u8 * dstBase, ptrdiff_t dstStride); + + /* + Convert RGB image to HSV + */ + void rgb2hsv(const Size2D &size, + const u8 * srcBase, ptrdiff_t srcStride, + u8 * dstBase, ptrdiff_t dstStride, + s32 hrange); + + /* + Convert RGBX image to HSV + */ + void rgbx2hsv(const Size2D &size, + const u8 * srcBase, ptrdiff_t srcStride, + u8 * dstBase, ptrdiff_t dstStride, + s32 hrange); + + /* + Convert BGR image to HSV + */ + void bgr2hsv(const Size2D &size, + const u8 * srcBase, ptrdiff_t srcStride, + u8 * dstBase, ptrdiff_t dstStride, + s32 hrange); + + /* + Convert BGRX image to HSV + */ + void bgrx2hsv(const Size2D &size, + const u8 * srcBase, ptrdiff_t srcStride, + u8 * dstBase, ptrdiff_t dstStride, + s32 hrange); + + /* + Convert RGBX image to BGR565 + RRRRrrrr GGGGgggg BBBBbbbb XXXXxxxx -> GggBBBBb RRRRrGGG + */ + void rgbx2bgr565(const Size2D &size, + const u8 * srcBase, ptrdiff_t srcStride, + u8 * dstBase, ptrdiff_t dstStride); + + /* + Convert RGB image to BGR565 + RRRRrrrr GGGGgggg BBBBbbbb -> GggBBBBb RRRRrGGG + */ + void rgb2bgr565(const Size2D &size, + const u8 * srcBase, ptrdiff_t srcStride, + u8 * dstBase, ptrdiff_t dstStride); + + /* + Convert RGBX image to RGB565 + RRRRrrrr GGGGgggg BBBBbbbb XXXXxxxx -> GggRRRRr BBBBbGGG + */ + void rgbx2rgb565(const Size2D &size, + const u8 * srcBase, ptrdiff_t srcStride, + u8 * dstBase, ptrdiff_t dstStride); + + /* + Convert RGB image to RGB565 + RRRRrrrr GGGGgggg BBBBbbbb -> GggRRRRr BBBBbGGG + */ + void rgb2rgb565(const Size2D &size, + const u8 * srcBase, ptrdiff_t srcStride, + u8 * dstBase, ptrdiff_t dstStride); + + /* + Convert RGB image to YCrCb + */ + void rgb2ycrcb(const Size2D &size, + const u8 * srcBase, ptrdiff_t srcStride, + u8 * dstBase, ptrdiff_t dstStride); + + /* + Convert RGBX image to YCrCb + */ + void rgbx2ycrcb(const Size2D &size, + const u8 * srcBase, ptrdiff_t srcStride, + u8 * dstBase, ptrdiff_t dstStride); + + /* + Convert BGR image to YCrCb + */ + void bgr2ycrcb(const Size2D &size, + const u8 * srcBase, ptrdiff_t srcStride, + u8 * dstBase, ptrdiff_t dstStride); + + /* + Convert BGRX image to YCrCb + */ + void bgrx2ycrcb(const Size2D &size, + const u8 * srcBase, ptrdiff_t srcStride, + u8 * dstBase, ptrdiff_t dstStride); + + /* + Convert YUV420sp image to RGB + */ + void yuv420sp2rgb(const Size2D &size, + const u8 * yBase, ptrdiff_t yStride, + const u8 * uvBase, ptrdiff_t uvStride, + u8 * dstBase, ptrdiff_t dstStride); + + /* + Convert YUV420sp image to RGBX + */ + void yuv420sp2rgbx(const Size2D &size, + const u8 * yBase, ptrdiff_t yStride, + const u8 * uvBase, ptrdiff_t uvStride, + u8 * dstBase, ptrdiff_t dstStride); + + /* + Convert YUV420i image to RGB + */ + void yuv420i2rgb(const Size2D &size, + const u8 * yBase, ptrdiff_t yStride, + const u8 * uvBase, ptrdiff_t uvStride, + u8 * dstBase, ptrdiff_t dstStride); + + /* + Convert YUV420i image to RGBX + */ + void yuv420i2rgbx(const Size2D &size, + const u8 * yBase, ptrdiff_t yStride, + const u8 * uvBase, ptrdiff_t uvStride, + u8 * dstBase, ptrdiff_t dstStride); + + /* + Convert YUV420sp image to BGR + */ + void yuv420sp2bgr(const Size2D &size, + const u8 * yBase, ptrdiff_t yStride, + const u8 * uvBase, ptrdiff_t uvStride, + u8 * dstBase, ptrdiff_t dstStride); + + /* + Convert YUV420sp image to BGRX + */ + void yuv420sp2bgrx(const Size2D &size, + const u8 * yBase, ptrdiff_t yStride, + const u8 * uvBase, ptrdiff_t uvStride, + u8 * dstBase, ptrdiff_t dstStride); + + /* + Convert YUV420i image to BGR + */ + void yuv420i2bgr(const Size2D &size, + const u8 * yBase, ptrdiff_t yStride, + const u8 * uvBase, ptrdiff_t uvStride, + u8 * dstBase, ptrdiff_t dstStride); + + /* + Convert YUV420i image to BGRX + */ + void yuv420i2bgrx(const Size2D &size, + const u8 * yBase, ptrdiff_t yStride, + const u8 * uvBase, ptrdiff_t uvStride, + u8 * dstBase, ptrdiff_t dstStride); + + /* + For each point `p` within `size`, do: + dst[p] = src[p] << shift + */ + void lshift(const Size2D &size, + const u8 * srcBase, ptrdiff_t srcStride, + s16 * dstBase, ptrdiff_t dstStride, + u32 shift); + + /* + For each point `p` within `size`, do sign-extending shift: + dst[p] = src[p] >> shift + */ + void rshift(const Size2D &size, + const s16 * srcBase, ptrdiff_t srcStride, + u8 * dstBase, ptrdiff_t dstStride, + u32 shift, CONVERT_POLICY cpolicy); + + /* + For each point `p` within `size`, set `dst[p]` to the average + of `src[p]` and the 8 (or 24 for blur5x5) points around it + NOTE: the function cannot operate inplace + */ + bool isBlur3x3Supported(const Size2D &size, BORDER_MODE border); + void blur3x3(const Size2D &size, + const u8 * srcBase, ptrdiff_t srcStride, + u8 * dstBase, ptrdiff_t dstStride, + BORDER_MODE border, u8 borderValue); + + bool isBlurU8Supported(const Size2D &size, s32 cn, BORDER_MODE border); + void blur3x3(const Size2D &size, s32 cn, + const u8 * srcBase, ptrdiff_t srcStride, + u8 * dstBase, ptrdiff_t dstStride, + BORDER_MODE borderType, u8 borderValue); + + void blur5x5(const Size2D &size, s32 cn, + const u8 * srcBase, ptrdiff_t srcStride, + u8 * dstBase, ptrdiff_t dstStride, + BORDER_MODE borderType, u8 borderValue); + + /* + For each point `p` within `size`, set `dst[p]` to the average + of `src[p]` and the 8 points around it + NOTE: the function can operate inplace + */ + bool isBlurF32Supported(const Size2D &size, s32 cn, BORDER_MODE border); + void blur3x3(const Size2D &size, s32 cn, + const f32 * srcBase, ptrdiff_t srcStride, + f32 * dstBase, ptrdiff_t dstStride, + BORDER_MODE borderType, f32 borderValue, Margin borderMargin); + + bool isBlurS32Supported(const Size2D &size, s32 cn, BORDER_MODE border); + void blur3x3(const Size2D &size, s32 cn, + const s32 * srcBase, ptrdiff_t srcStride, + s32 * dstBase, ptrdiff_t dstStride, + BORDER_MODE borderType, s32 borderValue, Margin borderMargin); + + /* + For each point `p` within `size`, set `dst[p]` to gaussian smooth + of `src[p]` and the 8(24 for 5x5 version) points around it + NOTE: the function cannot operate inplace + */ + bool isGaussianBlur3x3Supported(const Size2D &size, BORDER_MODE border); + void gaussianBlur3x3(const Size2D &size, + const u8 * srcBase, ptrdiff_t srcStride, + u8 * dstBase, ptrdiff_t dstStride, + BORDER_MODE border, u8 borderValue); + bool isGaussianBlur3x3MarginSupported(const Size2D &size, BORDER_MODE border, Margin borderMargin = Margin()); + void gaussianBlur3x3Margin(const Size2D &size, + const u8 * srcBase, ptrdiff_t srcStride, + u8 * dstBase, ptrdiff_t dstStride, + BORDER_MODE border, u8 borderValue, Margin borderMargin = Margin()); + + bool isGaussianBlur5x5Supported(const Size2D &size, s32 cn, BORDER_MODE border); + void gaussianBlur5x5(const Size2D &size, s32 cn, + const u8 * srcBase, ptrdiff_t srcStride, + u8 * dstBase, ptrdiff_t dstStride, + BORDER_MODE borderType, u8 borderValue, Margin borderMargin); + + void gaussianBlur5x5(const Size2D &size, s32 cn, + const u16 * srcBase, ptrdiff_t srcStride, + u16 * dstBase, ptrdiff_t dstStride, + BORDER_MODE borderType, u16 borderValue, Margin borderMargin); + + void gaussianBlur5x5(const Size2D &size, s32 cn, + const s16 * srcBase, ptrdiff_t srcStride, + s16 * dstBase, ptrdiff_t dstStride, + BORDER_MODE borderType, s16 borderValue, Margin borderMargin); + + void gaussianBlur5x5(const Size2D &size, s32 cn, + const s32 * srcBase, ptrdiff_t srcStride, + s32 * dstBase, ptrdiff_t dstStride, + BORDER_MODE borderType, s32 borderValue, Margin borderMargin); + + /* + Calculation of Sobel operator + NOTE: the function cannot operate inplace + */ + bool isSobel3x3Supported(const Size2D &size, BORDER_MODE border, s32 dx, s32 dy, Margin borderMargin = Margin()); + void Sobel3x3(const Size2D &size, + const u8 * srcBase, ptrdiff_t srcStride, + s16 * dstBase, ptrdiff_t dstStride, + s32 dx, s32 dy, + BORDER_MODE border, u8 borderValue, Margin borderMargin = Margin()); + + /* + Calculation of Sobel operator for f32 data + NOTE: the function can operate inplace + */ + bool isSobel3x3f32Supported(const Size2D &size, BORDER_MODE border, s32 dx, s32 dy); + void Sobel3x3(const Size2D &size, + const f32 * srcBase, ptrdiff_t srcStride, + f32 * dstBase, ptrdiff_t dstStride, + s32 dx, s32 dy, + BORDER_MODE borderType, f32 borderValue); + + /* + Calculation of Scharr operator + NOTE: the function cannot operate inplace + */ + bool isScharr3x3Supported(const Size2D &size, BORDER_MODE border, s32 dx, s32 dy, Margin borderMargin = Margin()); + void Scharr3x3(const Size2D &size, + const u8 * srcBase, ptrdiff_t srcStride, + s16 * dstBase, ptrdiff_t dstStride, + s32 dx, s32 dy, + BORDER_MODE borderType, u8 borderValue, Margin borderMargin = Margin()); + + void ScharrDeriv(const Size2D &size, s32 cn, + const u8 * srcBase, ptrdiff_t srcStride, + s16 * dstBase, ptrdiff_t dstStride); + + /* + Calculation of generic separable filtering operator + rowFilter/colFilter define filter weights + 0 - predefined 1 2 1 + 1 - predefined -1 0 1 + 2 - predefined 1 -2 1 + 3 - weights provided as xw/yw + */ + bool isSeparableFilter3x3Supported(const Size2D &size, BORDER_MODE border, s32 dx, s32 dy, Margin borderMargin = Margin()); + void SeparableFilter3x3(const Size2D &size, + const u8 * srcBase, ptrdiff_t srcStride, + s16 * dstBase, ptrdiff_t dstStride, + const u8 rowFilter, const u8 colFilter, const s16 *xw, const s16 *yw, + BORDER_MODE border, u8 borderValue, Margin borderMargin = Margin()); + + /* + Extract a single plane from 2 channel image + */ + void extract2(const Size2D &size, + const u8 * srcBase, ptrdiff_t srcStride, + u8 * dstBase, ptrdiff_t dstStride, + u32 coi); + + /* + Extract a single plane from 3 channel image + */ + void extract3(const Size2D &size, + const u8 * srcBase, ptrdiff_t srcStride, + u8 * dstBase, ptrdiff_t dstStride, + u32 coi); + + /* + Extract a single plane from 4 channel image + */ + void extract4(const Size2D &size, + const u8 * srcBase, ptrdiff_t srcStride, + u8 * dstBase, ptrdiff_t dstStride, + u32 coi); + + /* + Split 2 channel image to separate planes + */ + void split2(const Size2D &size, + const u8 * srcBase, ptrdiff_t srcStride, + u8 * dst0Base, ptrdiff_t dst0Stride, + u8 * dst1Base, ptrdiff_t dst1Stride); + + void split2(const Size2D &size, + const u16* srcBase, ptrdiff_t srcStride, + u16 * dst0Base, ptrdiff_t dst0Stride, + u16 * dst1Base, ptrdiff_t dst1Stride); + + void split2(const Size2D &size, + const s32 * srcBase, ptrdiff_t srcStride, + s32 * dst0Base, ptrdiff_t dst0Stride, + s32 * dst1Base, ptrdiff_t dst1Stride); + + void split2(const Size2D &size, + const s64 * srcBase, ptrdiff_t srcStride, + s64 * dst0Base, ptrdiff_t dst0Stride, + s64 * dst1Base, ptrdiff_t dst1Stride); + + /* + Split 3 channel image to separate planes + */ + void split3(const Size2D &size, + const u8 * srcBase, ptrdiff_t srcStride, + u8 * dst0Base, ptrdiff_t dst0Stride, + u8 * dst1Base, ptrdiff_t dst1Stride, + u8 * dst2Base, ptrdiff_t dst2Stride); + + void split3(const Size2D &size, + const u16* srcBase, ptrdiff_t srcStride, + u16 * dst0Base, ptrdiff_t dst0Stride, + u16 * dst1Base, ptrdiff_t dst1Stride, + u16 * dst2Base, ptrdiff_t dst2Stride); + + void split3(const Size2D &size, + const s32 * srcBase, ptrdiff_t srcStride, + s32 * dst0Base, ptrdiff_t dst0Stride, + s32 * dst1Base, ptrdiff_t dst1Stride, + s32 * dst2Base, ptrdiff_t dst2Stride); + + void split3(const Size2D &size, + const s64 * srcBase, ptrdiff_t srcStride, + s64 * dst0Base, ptrdiff_t dst0Stride, + s64 * dst1Base, ptrdiff_t dst1Stride, + s64 * dst2Base, ptrdiff_t dst2Stride); + + /* + Split 4 channel image to separate planes + */ + void split4(const Size2D &size, + const u8 * srcBase, ptrdiff_t srcStride, + u8 * dst0Base, ptrdiff_t dst0Stride, + u8 * dst1Base, ptrdiff_t dst1Stride, + u8 * dst2Base, ptrdiff_t dst2Stride, + u8 * dst3Base, ptrdiff_t dst3Stride); + + void split4(const Size2D &size, + const u16* srcBase, ptrdiff_t srcStride, + u16 * dst0Base, ptrdiff_t dst0Stride, + u16 * dst1Base, ptrdiff_t dst1Stride, + u16 * dst2Base, ptrdiff_t dst2Stride, + u16 * dst3Base, ptrdiff_t dst3Stride); + + void split4(const Size2D &size, + const s32 * srcBase, ptrdiff_t srcStride, + s32 * dst0Base, ptrdiff_t dst0Stride, + s32 * dst1Base, ptrdiff_t dst1Stride, + s32 * dst2Base, ptrdiff_t dst2Stride, + s32 * dst3Base, ptrdiff_t dst3Stride); + + void split4(const Size2D &size, + const s64 * srcBase, ptrdiff_t srcStride, + s64 * dst0Base, ptrdiff_t dst0Stride, + s64 * dst1Base, ptrdiff_t dst1Stride, + s64 * dst2Base, ptrdiff_t dst2Stride, + s64 * dst3Base, ptrdiff_t dst3Stride); + + /* + Split 4 channel image to 3 channel image and 1 channel image + */ + void split4(const Size2D &size, + const u8 * srcBase, ptrdiff_t srcStride, + u8 * dst3Base, ptrdiff_t dst3Stride, + u8 * dst1Base, ptrdiff_t dst1Stride); + + /* + Flip image using specified flip mode + */ + bool isFlipSupported(FLIP_MODE flipMode, u32 elemSize); + void flip(const Size2D &size, + const u8 * srcBase, ptrdiff_t srcStride, + u8 * dstBase, ptrdiff_t dstStride, + FLIP_MODE flipMode, u32 elemSize); + + /* + For each point `p` within `size`, set `dst[p]` to the maximum + of `src[p]` and the 8 points around it + NOTE: the function cannot operate inplace + */ + bool isMorph3x3Supported(const Size2D &size, BORDER_MODE border); + + void erode3x3(const Size2D &size, + const u8 * srcBase, ptrdiff_t srcStride, + u8 * dstBase, ptrdiff_t dstStride, + BORDER_MODE border, u8 borderValue); + + void dilate3x3(const Size2D &size, + const u8 * srcBase, ptrdiff_t srcStride, + u8 * dstBase, ptrdiff_t dstStride, + BORDER_MODE border, u8 borderValue); + + void erode(const Size2D &ssize, u32 cn, + const u8 * srcBase, ptrdiff_t srcStride, + u8 * dstBase, ptrdiff_t dstStride, + const Size2D &ksize, + size_t anchorX, size_t anchorY, + BORDER_MODE rowBorderType, BORDER_MODE columnBorderType, + const u8 * borderValues, Margin borderMargin); + + void dilate(const Size2D &ssize, u32 cn, + const u8 * srcBase, ptrdiff_t srcStride, + u8 * dstBase, ptrdiff_t dstStride, + const Size2D &ksize, + size_t anchorX, size_t anchorY, + BORDER_MODE rowBorderType, BORDER_MODE columnBorderType, + const u8 * borderValues, Margin borderMargin); + + /* + Resize a source image using "nearest neighbor" interpolation type + + wr = src_width / dst_width + hr = src_height / dst_height + */ + bool isResizeNearestNeighborSupported(const Size2D &ssize, u32 elemSize); + void resizeNearestNeighbor(const Size2D &ssize, const Size2D &dsize, + const void * srcBase, ptrdiff_t srcStride, + void * dstBase, ptrdiff_t dstStride, + f32 wr, f32 hr, u32 elemSize); + + /* + Resize a source image using "area" interpolation type + + wr = src_width / dst_width + hr = src_height / dst_height + */ + bool isResizeAreaSupported(f32 wr, f32 hr, u32 channels); + void resizeAreaOpenCV(const Size2D &ssize, const Size2D &dsize, + const u8 * srcBase, ptrdiff_t srcStride, + u8 * dstBase, ptrdiff_t dstStride, + f32 wr, f32 hr, u32 channels); + void resizeArea(const Size2D &ssize, const Size2D &dsize, + const u8 * srcBase, ptrdiff_t srcStride, + u8 * dstBase, ptrdiff_t dstStride, + f32 wr, f32 hr, u32 channels); + + /* + Resize a source image using "linear" interpolation type + + wr = src_width / dst_width + hr = src_height / dst_height + */ + bool isResizeLinearOpenCVSupported(const Size2D &ssize, const Size2D &dsize, u32 channels); + bool isResizeLinearSupported(const Size2D &ssize, const Size2D &dsize, + f32 wr, f32 hr, u32 channels); + void resizeLinearOpenCV(const Size2D &ssize, const Size2D &dsize, + const u8 * srcBase, ptrdiff_t srcStride, + u8 * dstBase, ptrdiff_t dstStride, + f32 wr, f32 hr, u32 channels); + void resizeLinear(const Size2D &ssize, const Size2D &dsize, + const u8 * srcBase, ptrdiff_t srcStride, + u8 * dstBase, ptrdiff_t dstStride, + f32 wr, f32 hr, u32 channels); + + /* + For each point `p` within `size`, set `dst[p]` to convolution + of `src[p]` and the (ksize * ksize - 1) points around it + The function uses OpenVX semantic (so, in order to use this function + in OpenCV you should flip kernel in both directions) + NOTE: the function cannot operate inplace + */ + bool isConvolutionSupported(const Size2D &size, const Size2D &ksize, BORDER_MODE border); + void convolution(const Size2D &size, + const u8 * srcBase, ptrdiff_t srcStride, + u8 * dstBase, ptrdiff_t dstStride, + BORDER_MODE border, u8 borderValue, + const Size2D & ksize, s16 * kernelBase, u32 scale); + + /* + For each point `p` within `dstSize`, does convolution + of tmpl points and size*size square of src points starting with `src[p]`. + Src should be of size (dstSize+size-1)*(dstSize+size-1) + NOTE: the function cannot operate inplace + */ + bool isMatchTemplateSupported(const Size2D &tmplSize); + void matchTemplate(const Size2D &srcSize, + const u8 * srcBase, ptrdiff_t srcStride, + const Size2D &tmplSize, + const u8 * tmplBase, ptrdiff_t tmplStride, + f32 * dstBase, ptrdiff_t dstStride, + bool normalize); + + /* + Calculation of Laplacian operator + + 1 1 1 + 1 -8 1 + 1 1 1 + + NOTE: the function cannot operate inplace + */ + bool isLaplacian3x3Supported(const Size2D &size, BORDER_MODE border); + void Laplacian3x3(const Size2D &size, + const u8 * srcBase, ptrdiff_t srcStride, + u8 * dstBase, ptrdiff_t dstStride, + BORDER_MODE border, u8 borderValue); + + /* + OpenCV like calculation of Laplacian operator + + kernel 1 kernel 3 kernel 5 + 0 1 0 2 0 2 1 2 2 2 1 + 1 -4 1 0 -8 0 2 0 -4 0 2 + 0 1 0 2 0 2 2 -4 -12 -4 2 + 2 0 -4 0 2 + 1 2 2 2 1 + + NOTE: the function cannot operate inplace + */ + bool isLaplacianOpenCVSupported(const Size2D &size, BORDER_MODE border); + void Laplacian1OpenCV(const Size2D &size, + const u8 * srcBase, ptrdiff_t srcStride, + s16 * dstBase, ptrdiff_t dstStride, + BORDER_MODE border, u8 borderValue); + void Laplacian3OpenCV(const Size2D &size, + const u8 * srcBase, ptrdiff_t srcStride, + s16 * dstBase, ptrdiff_t dstStride, + BORDER_MODE border, u8 borderValue); + void Laplacian5OpenCV(const Size2D &size, + const u8 * srcBase, ptrdiff_t srcStride, + s16 * dstBase, ptrdiff_t dstStride, + BORDER_MODE border, u8 borderValue); + + /* + Detect image edges using Canny algorithm + These functions perform derivatives estimation using sobel algorithm + */ + bool isCanny3x3Supported(const Size2D &size); + void Canny3x3L1(const Size2D &size, + const u8 * srcBase, ptrdiff_t srcStride, + u8 * dstBase, ptrdiff_t dstStride, + f64 low_thresh, f64 high_thresh, + Margin borderMargin); + + void Canny3x3L2(const Size2D &size, + const u8 * srcBase, ptrdiff_t srcStride, + u8 * dstBase, ptrdiff_t dstStride, + f64 low_thresh, f64 high_thresh, + Margin borderMargin); + + /* + Detect image edges using Canny algorithm + These functions don't estimate derivatives and thus require + precomputed derivatives estimation instead of source image + */ + void Canny3x3L1(const Size2D &size, s32 cn, + s16 * dxBase, ptrdiff_t dxStride, + s16 * dyBase, ptrdiff_t dyStride, + u8 * dstBase, ptrdiff_t dstStride, + f64 low_thresh, f64 high_thresh); + + void Canny3x3L2(const Size2D &size, s32 cn, + s16 * dxBase, ptrdiff_t dxStride, + s16 * dyBase, ptrdiff_t dyStride, + u8 * dstBase, ptrdiff_t dstStride, + f64 low_thresh, f64 high_thresh); + + /* + Performs detection of FAST features + */ + void FAST(const Size2D &size, + u8 *srcBase, ptrdiff_t srcStride, + KeypointStore *keypoints, + u8 threshold, bool nonmax_suppression); + + /* + Remap a source image using table and specified + extrapolation method + */ + bool isRemapNearestNeighborSupported(const Size2D &ssize); + void remapNearestNeighbor(const Size2D &ssize, const Size2D &dsize, + const u8 * srcBase, ptrdiff_t srcStride, + const f32 * tableBase, ptrdiff_t tableStride, + u8 * dstBase, ptrdiff_t dstStride, + BORDER_MODE borderMode, u8 borderValue); + + bool isRemapLinearSupported(const Size2D &ssize); + void remapLinear(const Size2D &ssize, const Size2D &dsize, + const u8 * srcBase, ptrdiff_t srcStride, + const f32 * tableBase, ptrdiff_t tableStride, + u8 * dstBase, ptrdiff_t dstStride, + BORDER_MODE borderMode, u8 borderValue); + + /* + Perform an affine transform on an input image + + src_x = dst_x * m[0] + dst_y * m[2] + m[4] + src_y = dst_x * m[1] + dst_y * m[3] + m[5] + */ + bool isWarpAffineNearestNeighborSupported(const Size2D &ssize); + void warpAffineNearestNeighbor(const Size2D &ssize, const Size2D &dsize, + const u8 * srcBase, ptrdiff_t srcStride, + const f32 * m, + u8 * dstBase, ptrdiff_t dstStride, + BORDER_MODE borderMode, u8 borderValue); + + bool isWarpAffineLinearSupported(const Size2D &ssize); + void warpAffineLinear(const Size2D &ssize, const Size2D &dsize, + const u8 * srcBase, ptrdiff_t srcStride, + const f32 * m, + u8 * dstBase, ptrdiff_t dstStride, + BORDER_MODE borderMode, u8 borderValue); + + /* + Perform a perspective transform on an input image + + src_x = dst_x * m[0] + dst_y * m[3] + m[6] + src_y = dst_x * m[1] + dst_y * m[4] + m[7] + w = dst_x * m[2] + dst_y * m[5] + m[8] + + src_x = w == 0 ? 0 : src_x / w + src_y = w == 0 ? 0 : src_y / w + */ + bool isWarpPerspectiveNearestNeighborSupported(const Size2D &ssize); + void warpPerspectiveNearestNeighbor(const Size2D &ssize, const Size2D &dsize, + const u8 * srcBase, ptrdiff_t srcStride, + const f32 * m, + u8 * dstBase, ptrdiff_t dstStride, + BORDER_MODE borderMode, u8 borderValue); + + bool isWarpPerspectiveLinearSupported(const Size2D &ssize); + void warpPerspectiveLinear(const Size2D &ssize, const Size2D &dsize, + const u8 * srcBase, ptrdiff_t srcStride, + const f32 * m, + u8 * dstBase, ptrdiff_t dstStride, + BORDER_MODE borderMode, u8 borderValue); + + /* + Convert data from source to destination type + */ + void convert(const Size2D &_size, + const u8 * srcBase, ptrdiff_t srcStride, + s8 * dstBase, ptrdiff_t dstStride); + + void convert(const Size2D &_size, + const u8 * srcBase, ptrdiff_t srcStride, + u16 * dstBase, ptrdiff_t dstStride); + + void convert(const Size2D &_size, + const u8 * srcBase, ptrdiff_t srcStride, + s16 * dstBase, ptrdiff_t dstStride); + + void convert(const Size2D &_size, + const u8 * srcBase, ptrdiff_t srcStride, + s32 * dstBase, ptrdiff_t dstStride); + + void convert(const Size2D &_size, + const u8 * srcBase, ptrdiff_t srcStride, + f32 * dstBase, ptrdiff_t dstStride); + + void convert(const Size2D &_size, + const s8 * srcBase, ptrdiff_t srcStride, + u8 * dstBase, ptrdiff_t dstStride); + + void convert(const Size2D &_size, + const s8 * srcBase, ptrdiff_t srcStride, + u16 * dstBase, ptrdiff_t dstStride); + + void convert(const Size2D &_size, + const s8 * srcBase, ptrdiff_t srcStride, + s16 * dstBase, ptrdiff_t dstStride); + + void convert(const Size2D &_size, + const s8 * srcBase, ptrdiff_t srcStride, + s32 * dstBase, ptrdiff_t dstStride); + + void convert(const Size2D &_size, + const s8 * srcBase, ptrdiff_t srcStride, + f32 * dstBase, ptrdiff_t dstStride); + + void convert(const Size2D &_size, + const u16 * srcBase, ptrdiff_t srcStride, + u8 * dstBase, ptrdiff_t dstStride); + + void convert(const Size2D &_size, + const u16 * srcBase, ptrdiff_t srcStride, + s8 * dstBase, ptrdiff_t dstStride); + + void convert(const Size2D &_size, + const u16 * srcBase, ptrdiff_t srcStride, + s16 * dstBase, ptrdiff_t dstStride); + + void convert(const Size2D &_size, + const u16 * srcBase, ptrdiff_t srcStride, + s32 * dstBase, ptrdiff_t dstStride); + + void convert(const Size2D &_size, + const u16 * srcBase, ptrdiff_t srcStride, + f32 * dstBase, ptrdiff_t dstStride); + + void convert(const Size2D &_size, + const s16 * srcBase, ptrdiff_t srcStride, + u8 * dstBase, ptrdiff_t dstStride); + + void convert(const Size2D &_size, + const s16 * srcBase, ptrdiff_t srcStride, + s8 * dstBase, ptrdiff_t dstStride); + + void convert(const Size2D &_size, + const s16 * srcBase, ptrdiff_t srcStride, + u16 * dstBase, ptrdiff_t dstStride); + + void convert(const Size2D &_size, + const s16 * srcBase, ptrdiff_t srcStride, + s32 * dstBase, ptrdiff_t dstStride); + + void convert(const Size2D &_size, + const s16 * srcBase, ptrdiff_t srcStride, + f32 * dstBase, ptrdiff_t dstStride); + + void convert(const Size2D &_size, + const s32 * srcBase, ptrdiff_t srcStride, + u8 * dstBase, ptrdiff_t dstStride); + + void convert(const Size2D &_size, + const s32 * srcBase, ptrdiff_t srcStride, + s8 * dstBase, ptrdiff_t dstStride); + + void convert(const Size2D &_size, + const s32 * srcBase, ptrdiff_t srcStride, + u16 * dstBase, ptrdiff_t dstStride); + + void convert(const Size2D &_size, + const s32 * srcBase, ptrdiff_t srcStride, + s16 * dstBase, ptrdiff_t dstStride); + + void convert(const Size2D &_size, + const s32 * srcBase, ptrdiff_t srcStride, + f32 * dstBase, ptrdiff_t dstStride); + + void convert(const Size2D &_size, + const f32 * srcBase, ptrdiff_t srcStride, + u8 * dstBase, ptrdiff_t dstStride); + + void convert(const Size2D &_size, + const f32 * srcBase, ptrdiff_t srcStride, + s8 * dstBase, ptrdiff_t dstStride); + + void convert(const Size2D &_size, + const f32 * srcBase, ptrdiff_t srcStride, + u16 * dstBase, ptrdiff_t dstStride); + + void convert(const Size2D &_size, + const f32 * srcBase, ptrdiff_t srcStride, + s16 * dstBase, ptrdiff_t dstStride); + + void convert(const Size2D &_size, + const f32 * srcBase, ptrdiff_t srcStride, + s32 * dstBase, ptrdiff_t dstStride); + + /* + Convert data from source to destination type with scaling + dst = saturate_cast(src * alpha + beta) + */ + void convertScale(const Size2D &_size, + const u8 * srcBase, ptrdiff_t srcStride, + u8 * dstBase, ptrdiff_t dstStride, + f64 alpha, f64 beta); + + void convertScale(const Size2D &_size, + const u8 * srcBase, ptrdiff_t srcStride, + s8 * dstBase, ptrdiff_t dstStride, + f64 alpha, f64 beta); + + void convertScale(const Size2D &_size, + const u8 * srcBase, ptrdiff_t srcStride, + u16 * dstBase, ptrdiff_t dstStride, + f64 alpha, f64 beta); + + void convertScale(const Size2D &_size, + const u8 * srcBase, ptrdiff_t srcStride, + s16 * dstBase, ptrdiff_t dstStride, + f64 alpha, f64 beta); + + void convertScale(const Size2D &_size, + const u8 * srcBase, ptrdiff_t srcStride, + s32 * dstBase, ptrdiff_t dstStride, + f64 alpha, f64 beta); + + void convertScale(const Size2D &_size, + const u8 * srcBase, ptrdiff_t srcStride, + f32 * dstBase, ptrdiff_t dstStride, + f64 alpha, f64 beta); + + void convertScale(const Size2D &_size, + const s8 * srcBase, ptrdiff_t srcStride, + u8 * dstBase, ptrdiff_t dstStride, + f64 alpha, f64 beta); + + void convertScale(const Size2D &_size, + const s8 * srcBase, ptrdiff_t srcStride, + s8 * dstBase, ptrdiff_t dstStride, + f64 alpha, f64 beta); + + void convertScale(const Size2D &_size, + const s8 * srcBase, ptrdiff_t srcStride, + u16 * dstBase, ptrdiff_t dstStride, + f64 alpha, f64 beta); + + void convertScale(const Size2D &_size, + const s8 * srcBase, ptrdiff_t srcStride, + s16 * dstBase, ptrdiff_t dstStride, + f64 alpha, f64 beta); + + void convertScale(const Size2D &_size, + const s8 * srcBase, ptrdiff_t srcStride, + s32 * dstBase, ptrdiff_t dstStride, + f64 alpha, f64 beta); + + void convertScale(const Size2D &_size, + const s8 * srcBase, ptrdiff_t srcStride, + f32 * dstBase, ptrdiff_t dstStride, + f64 alpha, f64 beta); + + void convertScale(const Size2D &_size, + const u16 * srcBase, ptrdiff_t srcStride, + u8 * dstBase, ptrdiff_t dstStride, + f64 alpha, f64 beta); + + void convertScale(const Size2D &_size, + const u16 * srcBase, ptrdiff_t srcStride, + s8 * dstBase, ptrdiff_t dstStride, + f64 alpha, f64 beta); + + void convertScale(const Size2D &_size, + const u16 * srcBase, ptrdiff_t srcStride, + u16 * dstBase, ptrdiff_t dstStride, + f64 alpha, f64 beta); + + void convertScale(const Size2D &_size, + const u16 * srcBase, ptrdiff_t srcStride, + s16 * dstBase, ptrdiff_t dstStride, + f64 alpha, f64 beta); + + void convertScale(const Size2D &_size, + const u16 * srcBase, ptrdiff_t srcStride, + s32 * dstBase, ptrdiff_t dstStride, + f64 alpha, f64 beta); + + void convertScale(const Size2D &_size, + const u16 * srcBase, ptrdiff_t srcStride, + f32 * dstBase, ptrdiff_t dstStride, + f64 alpha, f64 beta); + + void convertScale(const Size2D &_size, + const s16 * srcBase, ptrdiff_t srcStride, + u8 * dstBase, ptrdiff_t dstStride, + f64 alpha, f64 beta); + + void convertScale(const Size2D &_size, + const s16 * srcBase, ptrdiff_t srcStride, + s8 * dstBase, ptrdiff_t dstStride, + f64 alpha, f64 beta); + + void convertScale(const Size2D &_size, + const s16 * srcBase, ptrdiff_t srcStride, + u16 * dstBase, ptrdiff_t dstStride, + f64 alpha, f64 beta); + + void convertScale(const Size2D &_size, + const s16 * srcBase, ptrdiff_t srcStride, + s16 * dstBase, ptrdiff_t dstStride, + f64 alpha, f64 beta); + + void convertScale(const Size2D &_size, + const s16 * srcBase, ptrdiff_t srcStride, + s32 * dstBase, ptrdiff_t dstStride, + f64 alpha, f64 beta); + + void convertScale(const Size2D &_size, + const s16 * srcBase, ptrdiff_t srcStride, + f32 * dstBase, ptrdiff_t dstStride, + f64 alpha, f64 beta); + + void convertScale(const Size2D &_size, + const s32 * srcBase, ptrdiff_t srcStride, + u8 * dstBase, ptrdiff_t dstStride, + f64 alpha, f64 beta); + + void convertScale(const Size2D &_size, + const s32 * srcBase, ptrdiff_t srcStride, + s8 * dstBase, ptrdiff_t dstStride, + f64 alpha, f64 beta); + + void convertScale(const Size2D &_size, + const s32 * srcBase, ptrdiff_t srcStride, + u16 * dstBase, ptrdiff_t dstStride, + f64 alpha, f64 beta); + + void convertScale(const Size2D &_size, + const s32 * srcBase, ptrdiff_t srcStride, + s16 * dstBase, ptrdiff_t dstStride, + f64 alpha, f64 beta); + + void convertScale(const Size2D &_size, + const s32 * srcBase, ptrdiff_t srcStride, + s32 * dstBase, ptrdiff_t dstStride, + f64 alpha, f64 beta); + + void convertScale(const Size2D &_size, + const s32 * srcBase, ptrdiff_t srcStride, + f32 * dstBase, ptrdiff_t dstStride, + f64 alpha, f64 beta); + + void convertScale(const Size2D &_size, + const f32 * srcBase, ptrdiff_t srcStride, + u8 * dstBase, ptrdiff_t dstStride, + f64 alpha, f64 beta); + + void convertScale(const Size2D &_size, + const f32 * srcBase, ptrdiff_t srcStride, + s8 * dstBase, ptrdiff_t dstStride, + f64 alpha, f64 beta); + + void convertScale(const Size2D &_size, + const f32 * srcBase, ptrdiff_t srcStride, + u16 * dstBase, ptrdiff_t dstStride, + f64 alpha, f64 beta); + + void convertScale(const Size2D &_size, + const f32 * srcBase, ptrdiff_t srcStride, + s16 * dstBase, ptrdiff_t dstStride, + f64 alpha, f64 beta); + + void convertScale(const Size2D &_size, + const f32 * srcBase, ptrdiff_t srcStride, + s32 * dstBase, ptrdiff_t dstStride, + f64 alpha, f64 beta); + + void convertScale(const Size2D &_size, + const f32 * srcBase, ptrdiff_t srcStride, + f32 * dstBase, ptrdiff_t dstStride, + f64 alpha, f64 beta); + + /* + Reduce matrix to a vector by calculatin given operation for each column + */ + void reduceColSum(const Size2D &size, + const u8 * srcBase, ptrdiff_t srcStride, + s32 * dstBase); + + void reduceColMax(const Size2D &size, + const u8 * srcBase, ptrdiff_t srcStride, + u8 * dstBase); + + void reduceColMin(const Size2D &size, + const u8 * srcBase, ptrdiff_t srcStride, + u8 * dstBase); + + void reduceColSum(const Size2D &size, + const f32 * srcBase, ptrdiff_t srcStride, + f32 * dstBase); + + void reduceColMax(const Size2D &size, + const f32 * srcBase, ptrdiff_t srcStride, + f32 * dstBase); + + void reduceColMin(const Size2D &size, + const f32 * srcBase, ptrdiff_t srcStride, + f32 * dstBase); + + /* + For each point `p` within `size`, do: + dst[p] = (rng1[p] <= src[p] && src[p] <= rng2[p]) ? 255 : 0 + */ + + void inRange(const Size2D &_size, + const u8 * srcBase, ptrdiff_t srcStride, + const u8 * rng1Base, ptrdiff_t rng1Stride, + const u8 * rng2Base, ptrdiff_t rng2Stride, + u8 * dstBase, ptrdiff_t dstStride); + + void inRange(const Size2D &_size, + const s8 * srcBase, ptrdiff_t srcStride, + const s8 * rng1Base, ptrdiff_t rng1Stride, + const s8 * rng2Base, ptrdiff_t rng2Stride, + u8 * dstBase, ptrdiff_t dstStride); + + void inRange(const Size2D &_size, + const u16 * srcBase, ptrdiff_t srcStride, + const u16 * rng1Base, ptrdiff_t rng1Stride, + const u16 * rng2Base, ptrdiff_t rng2Stride, + u8 * dstBase, ptrdiff_t dstStride); + + void inRange(const Size2D &_size, + const s16 * srcBase, ptrdiff_t srcStride, + const s16 * rng1Base, ptrdiff_t rng1Stride, + const s16 * rng2Base, ptrdiff_t rng2Stride, + u8 * dstBase, ptrdiff_t dstStride); + + void inRange(const Size2D &_size, + const s32 * srcBase, ptrdiff_t srcStride, + const s32 * rng1Base, ptrdiff_t rng1Stride, + const s32 * rng2Base, ptrdiff_t rng2Stride, + u8 * dstBase, ptrdiff_t dstStride); + + void inRange(const Size2D &_size, + const f32 * srcBase, ptrdiff_t srcStride, + const f32 * rng1Base, ptrdiff_t rng1Stride, + const f32 * rng2Base, ptrdiff_t rng2Stride, + u8 * dstBase, ptrdiff_t dstStride); + + /* + Estimate amount of non zero elements + */ + s32 countNonZero(const Size2D &_size, + const u8 * srcBase, ptrdiff_t srcStride); + + s32 countNonZero(const Size2D &_size, + const u16 * srcBase, ptrdiff_t srcStride); + + s32 countNonZero(const Size2D &_size, + const s32 * srcBase, ptrdiff_t srcStride); + + s32 countNonZero(const Size2D &_size, + const f32 * srcBase, ptrdiff_t srcStride); + + s32 countNonZero(const Size2D &_size, + const f64 * srcBase, ptrdiff_t srcStride); + + /* + Calculates sum of all image pixel values and squared values + */ + bool isSumSupported(u32 channels); + + void sum(const Size2D &_size, + const u8 * srcBase, ptrdiff_t srcStride, + u32 * sumdst, u32 channels); + + void sum(const Size2D &_size, + const f32 * srcBase, ptrdiff_t srcStride, + f64 * sumdst, u32 channels); + + bool isSqsumSupported(u32 channels); + + void sqsum(const Size2D &_size, + const u8 * srcBase, ptrdiff_t srcStride, + f64 * sumdst, f64 * sqsumdst, u32 channels); + + /* + Calculates norm + */ + s32 normInf(const Size2D &_size, + const u8 * srcBase, ptrdiff_t srcStride); + + s32 normInf(const Size2D &_size, + const s8 * srcBase, ptrdiff_t srcStride); + + s32 normInf(const Size2D &_size, + const u16 * srcBase, ptrdiff_t srcStride); + + s32 normInf(const Size2D &_size, + const s16 * srcBase, ptrdiff_t srcStride); + + s32 normInf(const Size2D &_size, + const s32 * srcBase, ptrdiff_t srcStride); + + f32 normInf(const Size2D &_size, + const f32 * srcBase, ptrdiff_t srcStride); + + s32 normL1(const Size2D &_size, + const u8 * srcBase, ptrdiff_t srcStride); + + s32 normL1(const Size2D &_size, + const s8 * srcBase, ptrdiff_t srcStride); + + s32 normL1(const Size2D &_size, + const u16 * srcBase, ptrdiff_t srcStride); + + s32 normL1(const Size2D &_size, + const s16 * srcBase, ptrdiff_t srcStride); + + f64 normL1(const Size2D &_size, + const s32 * srcBase, ptrdiff_t srcStride); + + f64 normL1(const Size2D &_size, + const f32 * srcBase, ptrdiff_t srcStride); + + s32 normL2(const Size2D &_size, + const u8 * srcBase, ptrdiff_t srcStride); + + s32 normL2(const Size2D &_size, + const s8 * srcBase, ptrdiff_t srcStride); + + f64 normL2(const Size2D &_size, + const u16 * srcBase, ptrdiff_t srcStride); + + f64 normL2(const Size2D &_size, + const s16 * srcBase, ptrdiff_t srcStride); + + f64 normL2(const Size2D &_size, + const s32 * srcBase, ptrdiff_t srcStride); + + f64 normL2(const Size2D &_size, + const f32 * srcBase, ptrdiff_t srcStride); + + /* + Calculates norm of per element difference + */ + s32 diffNormInf(const Size2D &_size, + const u8 * src0Base, ptrdiff_t src0Stride, + const u8 * src1Base, ptrdiff_t src1Stride); + + f32 diffNormInf(const Size2D &_size, + const f32 * src0Base, ptrdiff_t src0Stride, + const f32 * src1Base, ptrdiff_t src1Stride); + + s32 diffNormL1(const Size2D &_size, + const u8 * src0Base, ptrdiff_t src0Stride, + const u8 * src1Base, ptrdiff_t src1Stride); + + f64 diffNormL1(const Size2D &_size, + const f32 * src0Base, ptrdiff_t src0Stride, + const f32 * src1Base, ptrdiff_t src1Stride); + + s32 diffNormL2(const Size2D &_size, + const u8 * src0Base, ptrdiff_t src0Stride, + const u8 * src1Base, ptrdiff_t src1Stride); + + f64 diffNormL2(const Size2D &_size, + const f32 * src0Base, ptrdiff_t src0Stride, + const f32 * src1Base, ptrdiff_t src1Stride); + + /* + * Pyramidal Lucas-Kanade Optical Flow level processing + */ + void pyrLKOptFlowLevel(const Size2D &size, s32 cn, + const u8 *prevData, ptrdiff_t prevStride, + const s16 *prevDerivData, ptrdiff_t prevDerivStride, + const u8 *nextData, ptrdiff_t nextStride, + u32 ptCount, + const f32 *prevPts, f32 *nextPts, + u8 *status, f32 *err, + const Size2D &winSize, + u32 terminationCount, f64 terminationEpsilon, + u32 level, u32 maxLevel, bool useInitialFlow, bool getMinEigenVals, + f32 minEigThreshold); +} + +#endif diff --git a/3rdparty/carotene/include/carotene/types.hpp b/3rdparty/carotene/include/carotene/types.hpp new file mode 100644 index 0000000000..81b03d649a --- /dev/null +++ b/3rdparty/carotene/include/carotene/types.hpp @@ -0,0 +1,125 @@ +/* + * By downloading, copying, installing or using the software you agree to this license. + * If you do not agree to this license, do not download, install, + * copy or use the software. + * + * + * License Agreement + * For Open Source Computer Vision Library + * (3-clause BSD License) + * + * Copyright (C) 2014-2015, NVIDIA Corporation, all rights reserved. + * Third party copyrights are property of their respective owners. + * + * Redistribution and use in source and binary forms, with or without modification, + * are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * * Neither the names of the copyright holders nor the names of the contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * This software is provided by the copyright holders and contributors "as is" and + * any express or implied warranties, including, but not limited to, the implied + * warranties of merchantability and fitness for a particular purpose are disclaimed. + * In no event shall copyright holders or contributors be liable for any direct, + * indirect, incidental, special, exemplary, or consequential damages + * (including, but not limited to, procurement of substitute goods or services; + * loss of use, data, or profits; or business interruption) however caused + * and on any theory of liability, whether in contract, strict liability, + * or tort (including negligence or otherwise) arising in any way out of + * the use of this software, even if advised of the possibility of such damage. + */ + +#ifndef CAROTENE_TYPES_HPP +#define CAROTENE_TYPES_HPP + +#include +#include +#include + +#ifndef UINT32_MAX + #define UINT32_MAX (4294967295U) +#endif + +namespace CAROTENE_NS { + using std::size_t; + using std::ptrdiff_t; + + typedef int8_t s8; + typedef uint8_t u8; + typedef int16_t s16; + typedef uint16_t u16; + typedef int32_t s32; + typedef uint32_t u32; + typedef float f32; + typedef int64_t s64; + typedef uint64_t u64; + typedef double f64; + + typedef ptrdiff_t stride_t; + + enum CONVERT_POLICY + { + CONVERT_POLICY_WRAP, + CONVERT_POLICY_SATURATE + }; + + enum BORDER_MODE + { + BORDER_MODE_UNDEFINED, + BORDER_MODE_CONSTANT, + BORDER_MODE_REPLICATE, + BORDER_MODE_REFLECT, + BORDER_MODE_REFLECT101, + BORDER_MODE_WRAP + }; + + enum FLIP_MODE + { + FLIP_HORIZONTAL_MODE = 1, + FLIP_VERTICAL_MODE = 2, + FLIP_BOTH_MODE = FLIP_HORIZONTAL_MODE | FLIP_VERTICAL_MODE + }; + + enum COLOR_SPACE + { + COLOR_SPACE_BT601, + COLOR_SPACE_BT709 + }; + + struct Size2D { + Size2D() : width(0), height(0) {} + Size2D(size_t width_, size_t height_) : width(width_), height(height_) {} + + size_t width; + size_t height; + + inline size_t total() const + { + return width * height; + } + }; + + struct Margin { + Margin() : left(0), right(0), top(0), bottom(0) {} + Margin(size_t left_, size_t right_, size_t top_, size_t bottom_) + : left(left_), right(right_), top(top_), bottom(bottom_) {} + + // these are measured in elements + size_t left, right, top, bottom; + }; + + struct KeypointStore { + virtual void push(f32 kpX, f32 kpY, f32 kpSize, f32 kpAngle=-1, f32 kpResponse=0, s32 kpOctave=0, s32 kpClass_id=-1) = 0; + virtual ~KeypointStore() {}; + }; +} + +#endif diff --git a/3rdparty/carotene/src/absdiff.cpp b/3rdparty/carotene/src/absdiff.cpp new file mode 100644 index 0000000000..02008ceb3e --- /dev/null +++ b/3rdparty/carotene/src/absdiff.cpp @@ -0,0 +1,241 @@ +/* + * By downloading, copying, installing or using the software you agree to this license. + * If you do not agree to this license, do not download, install, + * copy or use the software. + * + * + * License Agreement + * For Open Source Computer Vision Library + * (3-clause BSD License) + * + * Copyright (C) 2014-2015, NVIDIA Corporation, all rights reserved. + * Third party copyrights are property of their respective owners. + * + * Redistribution and use in source and binary forms, with or without modification, + * are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * * Neither the names of the copyright holders nor the names of the contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * This software is provided by the copyright holders and contributors "as is" and + * any express or implied warranties, including, but not limited to, the implied + * warranties of merchantability and fitness for a particular purpose are disclaimed. + * In no event shall copyright holders or contributors be liable for any direct, + * indirect, incidental, special, exemplary, or consequential damages + * (including, but not limited to, procurement of substitute goods or services; + * loss of use, data, or profits; or business interruption) however caused + * and on any theory of liability, whether in contract, strict liability, + * or tort (including negligence or otherwise) arising in any way out of + * the use of this software, even if advised of the possibility of such damage. + */ + +#include + +#include "common.hpp" +#include "vtransform.hpp" + +namespace CAROTENE_NS { + +#ifdef CAROTENE_NEON + +namespace { + +template +struct AbsDiff +{ + typedef T type; + + void operator() (const typename internal::VecTraits::vec128 & v_src0, + const typename internal::VecTraits::vec128 & v_src1, + typename internal::VecTraits::vec128 & v_dst) const + { + v_dst = internal::vabdq(v_src0, v_src1); + } + + void operator() (const typename internal::VecTraits::vec64 & v_src0, + const typename internal::VecTraits::vec64 & v_src1, + typename internal::VecTraits::vec64 & v_dst) const + { + v_dst = internal::vabd(v_src0, v_src1); + } + + void operator() (const T * src0, const T * src1, T * dst) const + { + dst[0] = src0[0] >= src1[0] ? src0[0] - src1[0] : src1[0] - src0[0]; + } +}; + +template +struct AbsDiffSigned +{ + typedef T type; + + void operator() (const typename internal::VecTraits::vec128 & v_src0, + const typename internal::VecTraits::vec128 & v_src1, + typename internal::VecTraits::vec128 & v_dst) const + { + typename internal::VecTraits::vec128 v_min = internal::vminq(v_src0, v_src1); + typename internal::VecTraits::vec128 v_max = internal::vmaxq(v_src0, v_src1); + v_dst = internal::vqsubq(v_max, v_min); + } + + void operator() (const typename internal::VecTraits::vec64 & v_src0, + const typename internal::VecTraits::vec64 & v_src1, + typename internal::VecTraits::vec64 & v_dst) const + { + typename internal::VecTraits::vec64 v_min = internal::vmin(v_src0, v_src1); + typename internal::VecTraits::vec64 v_max = internal::vmax(v_src0, v_src1); + v_dst = internal::vqsub(v_max, v_min); + } + + void operator() (const T * src0, const T * src1, T * dst) const + { + dst[0] = internal::saturate_cast(src0[0] >= src1[0] ? (s64)src0[0] - src1[0] : (s64)src1[0] - src0[0]); + } +}; + +} // namespace + +#endif + +void absDiff(const Size2D &size, + const u8 *src0Base, ptrdiff_t src0Stride, + const u8 *src1Base, ptrdiff_t src1Stride, + u8 *dstBase, ptrdiff_t dstStride) +{ + internal::assertSupportedConfiguration(); +#ifdef CAROTENE_NEON + internal::vtransform(size, + src0Base, src0Stride, + src1Base, src1Stride, + dstBase, dstStride, AbsDiff()); +#else + (void)size; + (void)src0Base; + (void)src0Stride; + (void)src1Base; + (void)src1Stride; + (void)dstBase; + (void)dstStride; +#endif +} + +void absDiff(const Size2D &size, + const u16 *src0Base, ptrdiff_t src0Stride, + const u16 *src1Base, ptrdiff_t src1Stride, + u16 *dstBase, ptrdiff_t dstStride) +{ + internal::assertSupportedConfiguration(); +#ifdef CAROTENE_NEON + internal::vtransform(size, + src0Base, src0Stride, + src1Base, src1Stride, + dstBase, dstStride, AbsDiff()); +#else + (void)size; + (void)src0Base; + (void)src0Stride; + (void)src1Base; + (void)src1Stride; + (void)dstBase; + (void)dstStride; +#endif +} + +void absDiff(const Size2D &size, + const s8 *src0Base, ptrdiff_t src0Stride, + const s8 *src1Base, ptrdiff_t src1Stride, + s8 *dstBase, ptrdiff_t dstStride) +{ + internal::assertSupportedConfiguration(); +#ifdef CAROTENE_NEON + internal::vtransform(size, + src0Base, src0Stride, + src1Base, src1Stride, + dstBase, dstStride, AbsDiffSigned()); +#else + (void)size; + (void)src0Base; + (void)src0Stride; + (void)src1Base; + (void)src1Stride; + (void)dstBase; + (void)dstStride; +#endif +} + +void absDiff(const Size2D &size, + const s16 *src0Base, ptrdiff_t src0Stride, + const s16 *src1Base, ptrdiff_t src1Stride, + s16 *dstBase, ptrdiff_t dstStride) +{ + internal::assertSupportedConfiguration(); +#ifdef CAROTENE_NEON + internal::vtransform(size, + src0Base, src0Stride, + src1Base, src1Stride, + dstBase, dstStride, AbsDiffSigned()); +#else + (void)size; + (void)src0Base; + (void)src0Stride; + (void)src1Base; + (void)src1Stride; + (void)dstBase; + (void)dstStride; +#endif +} + +void absDiff(const Size2D &size, + const s32 *src0Base, ptrdiff_t src0Stride, + const s32 *src1Base, ptrdiff_t src1Stride, + s32 *dstBase, ptrdiff_t dstStride) +{ + internal::assertSupportedConfiguration(); +#ifdef CAROTENE_NEON + internal::vtransform(size, + src0Base, src0Stride, + src1Base, src1Stride, + dstBase, dstStride, AbsDiffSigned()); +#else + (void)size; + (void)src0Base; + (void)src0Stride; + (void)src1Base; + (void)src1Stride; + (void)dstBase; + (void)dstStride; +#endif +} + +void absDiff(const Size2D &size, + const f32 * src0Base, ptrdiff_t src0Stride, + const f32 * src1Base, ptrdiff_t src1Stride, + f32 * dstBase, ptrdiff_t dstStride) +{ + internal::assertSupportedConfiguration(); +#ifdef CAROTENE_NEON + internal::vtransform(size, + src0Base, src0Stride, + src1Base, src1Stride, + dstBase, dstStride, AbsDiff()); +#else + (void)size; + (void)src0Base; + (void)src0Stride; + (void)src1Base; + (void)src1Stride; + (void)dstBase; + (void)dstStride; +#endif +} + +} // namespace CAROTENE_NS diff --git a/3rdparty/carotene/src/accumulate.cpp b/3rdparty/carotene/src/accumulate.cpp new file mode 100644 index 0000000000..ee9ce22d35 --- /dev/null +++ b/3rdparty/carotene/src/accumulate.cpp @@ -0,0 +1,408 @@ +/* + * By downloading, copying, installing or using the software you agree to this license. + * If you do not agree to this license, do not download, install, + * copy or use the software. + * + * + * License Agreement + * For Open Source Computer Vision Library + * (3-clause BSD License) + * + * Copyright (C) 2014, NVIDIA Corporation, all rights reserved. + * Third party copyrights are property of their respective owners. + * + * Redistribution and use in source and binary forms, with or without modification, + * are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * * Neither the names of the copyright holders nor the names of the contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * This software is provided by the copyright holders and contributors "as is" and + * any express or implied warranties, including, but not limited to, the implied + * warranties of merchantability and fitness for a particular purpose are disclaimed. + * In no event shall copyright holders or contributors be liable for any direct, + * indirect, incidental, special, exemplary, or consequential damages + * (including, but not limited to, procurement of substitute goods or services; + * loss of use, data, or profits; or business interruption) however caused + * and on any theory of liability, whether in contract, strict liability, + * or tort (including negligence or otherwise) arising in any way out of + * the use of this software, even if advised of the possibility of such damage. + */ + + +#include "common.hpp" +#include "vtransform.hpp" + +#include + +namespace CAROTENE_NS { + +void accumulate(const Size2D &size, + const u8 *srcBase, ptrdiff_t srcStride, + s16 *dstBase, ptrdiff_t dstStride) +{ + internal::assertSupportedConfiguration(); +#ifdef CAROTENE_NEON + size_t roiw16 = size.width >= 15 ? size.width - 15 : 0; + size_t roiw8 = size.width >= 7 ? size.width - 7 : 0; + + for (size_t i = 0; i < size.height; ++i) + { + const u8* src = internal::getRowPtr(srcBase, srcStride, i); + s16* dst = internal::getRowPtr(dstBase, dstStride, i); + size_t j = 0; + + for (; j < roiw16; j += 16) + { + internal::prefetch(src + j); + internal::prefetch(dst + j); + uint8x16_t v_src = vld1q_u8(src + j); + int16x8_t v_dst0 = vld1q_s16(dst + j); + int16x8_t v_dst1 = vld1q_s16(dst + j + 8); + int16x8_t v_src0 = vreinterpretq_s16_u16(vmovl_u8(vget_low_u8(v_src))); + int16x8_t v_src1 = vreinterpretq_s16_u16(vmovl_u8(vget_high_u8(v_src))); + v_dst0 = vqaddq_s16(v_dst0, v_src0); + v_dst1 = vqaddq_s16(v_dst1, v_src1); + vst1q_s16(dst + j, v_dst0); + vst1q_s16(dst + j + 8, v_dst1); + } + for (; j < roiw8; j += 8) + { + uint8x8_t v_src = vld1_u8(src + j); + int16x8_t v_src16 = vreinterpretq_s16_u16(vmovl_u8(v_src)); + int16x8_t v_dst = vld1q_s16(dst + j); + v_dst = vqaddq_s16(v_dst, v_src16); + vst1q_s16(dst + j, v_dst); + } + + for (; j < size.width; j++) + dst[j] = internal::saturate_cast(src[j] + dst[j]); + } +#else + (void)size; + (void)srcBase; + (void)srcStride; + (void)dstBase; + (void)dstStride; +#endif +} + +#ifdef CAROTENE_NEON + +namespace { + +template +void accumulateSquareConst(const Size2D &size, + const u8 *srcBase, ptrdiff_t srcStride, + s16 *dstBase, ptrdiff_t dstStride) +{ + size_t roiw16 = size.width >= 15 ? size.width - 15 : 0; + size_t roiw8 = size.width >= 7 ? size.width - 7 : 0; + + for (size_t i = 0; i < size.height; ++i) + { + const u8* src = internal::getRowPtr(srcBase, srcStride, i); + s16* dst = internal::getRowPtr(dstBase, dstStride, i); + size_t j = 0; + + for (; j < roiw16; j += 16) + { + internal::prefetch(src + j); + internal::prefetch(dst + j); + uint8x16_t v_src = vld1q_u8(src + j); + int16x8_t v_dst0 = vld1q_s16(dst + j), v_dst1 = vld1q_s16(dst + j + 8); + int16x8_t v_src0 = vreinterpretq_s16_u16(vmovl_u8(vget_low_u8(v_src))); + int16x8_t v_src1 = vreinterpretq_s16_u16(vmovl_u8(vget_high_u8(v_src))); + + int16x4_t v_srclo = vget_low_s16(v_src0), v_srchi = vget_high_s16(v_src0); + v_dst0 = vcombine_s16(vqmovn_s32(vaddw_s16(vshrq_n_s32(vmull_s16(v_srclo, v_srclo), shift), vget_low_s16(v_dst0))), + vqmovn_s32(vaddw_s16(vshrq_n_s32(vmull_s16(v_srchi, v_srchi), shift), vget_high_s16(v_dst0)))); + + v_srclo = vget_low_s16(v_src1); + v_srchi = vget_high_s16(v_src1); + v_dst1 = vcombine_s16(vqmovn_s32(vaddw_s16(vshrq_n_s32(vmull_s16(v_srclo, v_srclo), shift), vget_low_s16(v_dst1))), + vqmovn_s32(vaddw_s16(vshrq_n_s32(vmull_s16(v_srchi, v_srchi), shift), vget_high_s16(v_dst1)))); + + vst1q_s16(dst + j, v_dst0); + vst1q_s16(dst + j + 8, v_dst1); + } + for (; j < roiw8; j += 8) + { + int16x8_t v_src = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(src + j))); + int16x8_t v_dst = vld1q_s16(dst + j); + int16x4_t v_srclo = vget_low_s16(v_src), v_srchi = vget_high_s16(v_src); + v_dst = vcombine_s16(vqmovn_s32(vaddw_s16(vshrq_n_s32(vmull_s16(v_srclo, v_srclo), shift), vget_low_s16(v_dst))), + vqmovn_s32(vaddw_s16(vshrq_n_s32(vmull_s16(v_srchi, v_srchi), shift), vget_high_s16(v_dst)))); + vst1q_s16(dst + j, v_dst); + } + + for (; j < size.width; j++) + { + s32 srcVal = src[j]; + dst[j] = internal::saturate_cast(dst[j] + ((srcVal * srcVal) >> shift)); + } + } +} + +template <> +void accumulateSquareConst<0>(const Size2D &size, + const u8 *srcBase, ptrdiff_t srcStride, + s16 *dstBase, ptrdiff_t dstStride) +{ + size_t roiw16 = size.width >= 15 ? size.width - 15 : 0; + size_t roiw8 = size.width >= 7 ? size.width - 7 : 0; + + for (size_t i = 0; i < size.height; ++i) + { + const u8* src = internal::getRowPtr(srcBase, srcStride, i); + s16* dst = internal::getRowPtr(dstBase, dstStride, i); + size_t j = 0; + + for (; j < roiw16; j += 16) + { + internal::prefetch(src + j); + internal::prefetch(dst + j); + uint8x16_t v_src = vld1q_u8(src + j); + int16x8_t v_dst0 = vld1q_s16(dst + j), v_dst1 = vld1q_s16(dst + j + 8); + int16x8_t v_src0 = vreinterpretq_s16_u16(vmovl_u8(vget_low_u8(v_src))); + int16x8_t v_src1 = vreinterpretq_s16_u16(vmovl_u8(vget_high_u8(v_src))); + + int16x4_t v_srclo = vget_low_s16(v_src0), v_srchi = vget_high_s16(v_src0); + v_dst0 = vcombine_s16(vqmovn_s32(vaddw_s16(vmull_s16(v_srclo, v_srclo), vget_low_s16(v_dst0))), + vqmovn_s32(vaddw_s16(vmull_s16(v_srchi, v_srchi), vget_high_s16(v_dst0)))); + + v_srclo = vget_low_s16(v_src1); + v_srchi = vget_high_s16(v_src1); + v_dst1 = vcombine_s16(vqmovn_s32(vaddw_s16(vmull_s16(v_srclo, v_srclo), vget_low_s16(v_dst1))), + vqmovn_s32(vaddw_s16(vmull_s16(v_srchi, v_srchi), vget_high_s16(v_dst1)))); + + vst1q_s16(dst + j, v_dst0); + vst1q_s16(dst + j + 8, v_dst1); + } + for (; j < roiw8; j += 8) + { + int16x8_t v_src = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(src + j))); + int16x8_t v_dst = vld1q_s16(dst + j); + int16x4_t v_srclo = vget_low_s16(v_src), v_srchi = vget_high_s16(v_src); + v_dst = vcombine_s16(vqmovn_s32(vaddw_s16(vmull_s16(v_srclo, v_srclo), vget_low_s16(v_dst))), + vqmovn_s32(vaddw_s16(vmull_s16(v_srchi, v_srchi), vget_high_s16(v_dst)))); + vst1q_s16(dst + j, v_dst); + } + + for (; j < size.width; j++) + { + s32 srcVal = src[j]; + dst[j] = internal::saturate_cast(dst[j] + srcVal * srcVal); + } + } +} + +typedef void (* accumulateSquareConstFunc)(const Size2D &size, + const u8 *srcBase, ptrdiff_t srcStride, + s16 *dstBase, ptrdiff_t dstStride); + +} // namespace + +#endif + +void accumulateSquare(const Size2D &size, + const u8 *srcBase, ptrdiff_t srcStride, + s16 *dstBase, ptrdiff_t dstStride, + u32 shift) +{ + if (shift >= 16) + { + for (size_t i = 0; i < size.height; ++i) + { + s16 * dst = internal::getRowPtr(dstBase, dstStride, i); + std::memset(dst, 0, sizeof(s16) * size.width); + } + return; + } + + internal::assertSupportedConfiguration(); + +#ifdef CAROTENE_NEON + // this ugly contruction is needed to avoid: + // /usr/lib/gcc/arm-linux-gnueabihf/4.8/include/arm_neon.h:3581:59: error: argument must be a constant + // return (int16x8_t)__builtin_neon_vshr_nv8hi (__a, __b, 1); + + accumulateSquareConstFunc funcs[16] = + { + accumulateSquareConst<0>, + accumulateSquareConst<1>, + accumulateSquareConst<2>, + accumulateSquareConst<3>, + accumulateSquareConst<4>, + accumulateSquareConst<5>, + accumulateSquareConst<6>, + accumulateSquareConst<7>, + accumulateSquareConst<8>, + accumulateSquareConst<9>, + accumulateSquareConst<10>, + accumulateSquareConst<11>, + accumulateSquareConst<12>, + accumulateSquareConst<13>, + accumulateSquareConst<14>, + accumulateSquareConst<15> + }, func = funcs[shift]; + + func(size, srcBase, srcStride, dstBase, dstStride); +#else + (void)size; + (void)srcBase; + (void)srcStride; + (void)dstBase; + (void)dstStride; + (void)shift; +#endif +} + +#ifdef CAROTENE_NEON + +namespace { + +struct AccumulateWeightedHalf +{ + typedef u8 type; + + void operator() (const uint8x16_t & v_src0, const uint8x16_t & v_src1, + uint8x16_t & v_dst) const + { + v_dst = vhaddq_u8(v_src0, v_src1); + } + + void operator() (const uint8x8_t & v_src0, const uint8x8_t & v_src1, + uint8x8_t & v_dst) const + { + v_dst = vhadd_u8(v_src0, v_src1); + } + + void operator() (const u8 * src0, const u8 * src1, u8 * dst) const + { + dst[0] = ((u16)(src0[0]) + src1[0]) >> 1; + } +}; + +struct AccumulateWeighted +{ + typedef u8 type; + + float alpha, beta; + float32x4_t v_alpha, v_beta; + + explicit AccumulateWeighted(float _alpha) : + alpha(_alpha), beta(1 - _alpha) + { + v_alpha = vdupq_n_f32(alpha); + v_beta = vdupq_n_f32(beta); + } + + void operator() (const uint8x16_t & v_src0, const uint8x16_t & v_src1, + uint8x16_t & v_dst) const + { + uint16x8_t v_src0_p = vmovl_u8(vget_low_u8(v_src0)); + uint16x8_t v_src1_p = vmovl_u8(vget_low_u8(v_src1)); + float32x4_t v_dst0f = vmlaq_f32(vmulq_f32(vcvtq_f32_u32(vmovl_u16(vget_low_u16(v_src1_p))), v_beta), + v_alpha, vcvtq_f32_u32(vmovl_u16(vget_low_u16(v_src0_p)))); + float32x4_t v_dst1f = vmlaq_f32(vmulq_f32(vcvtq_f32_u32(vmovl_u16(vget_high_u16(v_src1_p))), v_beta), + v_alpha, vcvtq_f32_u32(vmovl_u16(vget_high_u16(v_src0_p)))); + uint16x8_t v_dst0 = vcombine_u16(vmovn_u32(vcvtq_u32_f32(v_dst0f)), + vmovn_u32(vcvtq_u32_f32(v_dst1f))); + + v_src0_p = vmovl_u8(vget_high_u8(v_src0)); + v_src1_p = vmovl_u8(vget_high_u8(v_src1)); + v_dst0f = vmlaq_f32(vmulq_f32(vcvtq_f32_u32(vmovl_u16(vget_low_u16(v_src1_p))), v_beta), + v_alpha, vcvtq_f32_u32(vmovl_u16(vget_low_u16(v_src0_p)))); + v_dst1f = vmlaq_f32(vmulq_f32(vcvtq_f32_u32(vmovl_u16(vget_high_u16(v_src1_p))), v_beta), + v_alpha, vcvtq_f32_u32(vmovl_u16(vget_high_u16(v_src0_p)))); + uint16x8_t v_dst1 = vcombine_u16(vmovn_u32(vcvtq_u32_f32(v_dst0f)), + vmovn_u32(vcvtq_u32_f32(v_dst1f))); + + v_dst = vcombine_u8(vmovn_u16(v_dst0), vmovn_u16(v_dst1)); + } + + void operator() (const uint8x8_t & _v_src0, const uint8x8_t & _v_src1, + uint8x8_t & v_dst) const + { + uint16x8_t v_src0 = vmovl_u8(_v_src0), v_src1 = vmovl_u8(_v_src1); + + float32x4_t v_dst0f = vmlaq_f32(vmulq_f32(vcvtq_f32_u32(vmovl_u16(vget_low_u16(v_src1))), v_beta), + v_alpha, vcvtq_f32_u32(vmovl_u16(vget_low_u16(v_src0)))); + float32x4_t v_dst1f = vmlaq_f32(vmulq_f32(vcvtq_f32_u32(vmovl_u16(vget_high_u16(v_src1))), v_beta), + v_alpha, vcvtq_f32_u32(vmovl_u16(vget_high_u16(v_src0)))); + uint16x8_t _v_dst = vcombine_u16(vmovn_u32(vcvtq_u32_f32(v_dst0f)), + vmovn_u32(vcvtq_u32_f32(v_dst1f))); + + v_dst = vmovn_u16(_v_dst); + } + + void operator() (const u8 * src0, const u8 * src1, u8 * dst) const + { + dst[0] = beta * src1[0] + alpha * src0[0]; + } +}; + +} // namespace + +#endif + +void accumulateWeighted(const Size2D &size, + const u8 *srcBase, ptrdiff_t srcStride, + u8 *dstBase, ptrdiff_t dstStride, + f32 alpha) +{ + if (alpha == 0.0f) + return; + if (alpha == 1.0f) + { + for (size_t i = 0; i < size.height; ++i) + { + const u8 * src = internal::getRowPtr(srcBase, srcStride, i); + u8 * dst = internal::getRowPtr(dstBase, dstStride, i); + std::memcpy(dst, src, sizeof(u8) * size.width); + } + return; + } + + internal::assertSupportedConfiguration(); + +#ifdef CAROTENE_NEON + // in this case we can use the following scheme: + // dst[p] = (src[p] + dst[p]) >> 1 + // which is faster + if (alpha == 0.5f) + { + internal::vtransform(size, + srcBase, srcStride, + dstBase, dstStride, + dstBase, dstStride, + AccumulateWeightedHalf()); + + return; + } + + internal::vtransform(size, + srcBase, srcStride, + dstBase, dstStride, + dstBase, dstStride, + AccumulateWeighted(alpha)); +#else + (void)size; + (void)srcBase; + (void)srcStride; + (void)dstBase; + (void)dstStride; + (void)alpha; +#endif +} + +} //namespace CAROTENE_NS diff --git a/3rdparty/carotene/src/add.cpp b/3rdparty/carotene/src/add.cpp new file mode 100644 index 0000000000..e8ace53122 --- /dev/null +++ b/3rdparty/carotene/src/add.cpp @@ -0,0 +1,475 @@ +/* + * By downloading, copying, installing or using the software you agree to this license. + * If you do not agree to this license, do not download, install, + * copy or use the software. + * + * + * License Agreement + * For Open Source Computer Vision Library + * (3-clause BSD License) + * + * Copyright (C) 2014, NVIDIA Corporation, all rights reserved. + * Third party copyrights are property of their respective owners. + * + * Redistribution and use in source and binary forms, with or without modification, + * are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * * Neither the names of the copyright holders nor the names of the contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * This software is provided by the copyright holders and contributors "as is" and + * any express or implied warranties, including, but not limited to, the implied + * warranties of merchantability and fitness for a particular purpose are disclaimed. + * In no event shall copyright holders or contributors be liable for any direct, + * indirect, incidental, special, exemplary, or consequential damages + * (including, but not limited to, procurement of substitute goods or services; + * loss of use, data, or profits; or business interruption) however caused + * and on any theory of liability, whether in contract, strict liability, + * or tort (including negligence or otherwise) arising in any way out of + * the use of this software, even if advised of the possibility of such damage. + */ + +#include "common.hpp" +#include "vtransform.hpp" + +namespace CAROTENE_NS { + +#ifdef CAROTENE_NEON + +namespace { + +template +struct AddWrap +{ + typedef T type; + + void operator() (const typename internal::VecTraits::vec128 & v_src0, + const typename internal::VecTraits::vec128 & v_src1, + typename internal::VecTraits::vec128 & v_dst) const + { + v_dst = internal::vaddq(v_src0, v_src1); + } + + void operator() (const typename internal::VecTraits::vec64 & v_src0, + const typename internal::VecTraits::vec64 & v_src1, + typename internal::VecTraits::vec64 & v_dst) const + { + v_dst = internal::vadd(v_src0, v_src1); + } + + void operator() (const T * src0, const T * src1, T * dst) const + { + dst[0] = (T)((WT)src0[0] + (WT)src1[0]); + } +}; + +template +struct AddSaturate +{ + typedef T type; + + void operator() (const typename internal::VecTraits::vec128 & v_src0, + const typename internal::VecTraits::vec128 & v_src1, + typename internal::VecTraits::vec128 & v_dst) const + { + v_dst = internal::vqaddq(v_src0, v_src1); + } + + void operator() (const typename internal::VecTraits::vec64 & v_src0, + const typename internal::VecTraits::vec64 & v_src1, + typename internal::VecTraits::vec64 & v_dst) const + { + v_dst = internal::vqadd(v_src0, v_src1); + } + + void operator() (const T * src0, const T * src1, T * dst) const + { + dst[0] = internal::saturate_cast((WT)src0[0] + (WT)src1[0]); + } +}; + +} // namespace + +#endif + +void add(const Size2D &size, + const u8 * src0Base, ptrdiff_t src0Stride, + const u8 * src1Base, ptrdiff_t src1Stride, + u8 *dstBase, ptrdiff_t dstStride, + CONVERT_POLICY policy) +{ + internal::assertSupportedConfiguration(); +#ifdef CAROTENE_NEON + if (policy == CONVERT_POLICY_SATURATE) + { + internal::vtransform(size, + src0Base, src0Stride, + src1Base, src1Stride, + dstBase, dstStride, + AddSaturate()); + } + else + { + internal::vtransform(size, + src0Base, src0Stride, + src1Base, src1Stride, + dstBase, dstStride, + AddWrap()); + } +#else + (void)size; + (void)src0Base; + (void)src0Stride; + (void)src1Base; + (void)src1Stride; + (void)dstBase; + (void)dstStride; + (void)policy; +#endif +} + +void add(const Size2D &size, + const s8 * src0Base, ptrdiff_t src0Stride, + const s8 * src1Base, ptrdiff_t src1Stride, + s8 *dstBase, ptrdiff_t dstStride, + CONVERT_POLICY policy) +{ + internal::assertSupportedConfiguration(); +#ifdef CAROTENE_NEON + if (policy == CONVERT_POLICY_SATURATE) + { + internal::vtransform(size, + src0Base, src0Stride, + src1Base, src1Stride, + dstBase, dstStride, + AddSaturate()); + } + else + { + internal::vtransform(size, + src0Base, src0Stride, + src1Base, src1Stride, + dstBase, dstStride, + AddWrap()); + } +#else + (void)size; + (void)src0Base; + (void)src0Stride; + (void)src1Base; + (void)src1Stride; + (void)dstBase; + (void)dstStride; + (void)policy; +#endif +} + +void add(const Size2D &size, + const u8 * src0Base, ptrdiff_t src0Stride, + const u8 * src1Base, ptrdiff_t src1Stride, + s16 *dstBase, ptrdiff_t dstStride, + CONVERT_POLICY) +{ + internal::assertSupportedConfiguration(); +#ifdef CAROTENE_NEON + size_t roiw32 = size.width >= 31 ? size.width - 31 : 0; + size_t roiw8 = size.width >= 7 ? size.width - 7 : 0; + + for (size_t i = 0; i < size.height; ++i) + { + const u8 * src0 = internal::getRowPtr(src0Base, src0Stride, i); + const u8 * src1 = internal::getRowPtr(src1Base, src1Stride, i); + u16 * dst = internal::getRowPtr((u16 *)dstBase, dstStride, i); + size_t j = 0; + + for (; j < roiw32; j += 32) + { + internal::prefetch(src0 + j); + internal::prefetch(src1 + j); + uint8x16_t v_src00 = vld1q_u8(src0 + j), v_src01 = vld1q_u8(src0 + j + 16); + uint8x16_t v_src10 = vld1q_u8(src1 + j), v_src11 = vld1q_u8(src1 + j + 16); + vst1q_u16(dst + j, vaddl_u8(vget_low_u8(v_src00), vget_low_u8(v_src10))); + vst1q_u16(dst + j + 8, vaddl_u8(vget_high_u8(v_src00), vget_high_u8(v_src10))); + vst1q_u16(dst + j + 16, vaddl_u8(vget_low_u8(v_src01), vget_low_u8(v_src11))); + vst1q_u16(dst + j + 24, vaddl_u8(vget_high_u8(v_src01), vget_high_u8(v_src11))); + } + for (; j < roiw8; j += 8) + { + uint8x8_t v_src0 = vld1_u8(src0 + j); + uint8x8_t v_src1 = vld1_u8(src1 + j); + vst1q_u16(dst + j, vaddl_u8(v_src0, v_src1)); + } + + for (; j < size.width; j++) + dst[j] = (u16)src0[j] + (u16)src1[j]; + } +#else + (void)size; + (void)src0Base; + (void)src0Stride; + (void)src1Base; + (void)src1Stride; + (void)dstBase; + (void)dstStride; +#endif +} + +void add(const Size2D &size, + const u8 * src0Base, ptrdiff_t src0Stride, + const s16 * src1Base, ptrdiff_t src1Stride, + s16 *dstBase, ptrdiff_t dstStride, + CONVERT_POLICY policy) +{ + internal::assertSupportedConfiguration(); +#ifdef CAROTENE_NEON + size_t roiw16 = size.width >= 15 ? size.width - 15 : 0; + size_t roiw8 = size.width >= 7 ? size.width - 7 : 0; + + for (size_t i = 0; i < size.height; ++i) + { + const u8 * src0 = internal::getRowPtr(src0Base, src0Stride, i); + const s16 * src1 = internal::getRowPtr(src1Base, src1Stride, i); + s16 * dst = internal::getRowPtr(dstBase, dstStride, i); + size_t j = 0; + + if (policy == CONVERT_POLICY_SATURATE) + { + for (; j < roiw16; j += 16) + { + internal::prefetch(src0 + j); + internal::prefetch(src1 + j); + uint8x16_t v_src0 = vld1q_u8(src0 + j); + int16x8_t v_src00 = vreinterpretq_s16_u16(vmovl_u8(vget_low_u8(v_src0))); + int16x8_t v_src01 = vreinterpretq_s16_u16(vmovl_u8(vget_high_u8(v_src0))); + int16x8_t v_src10 = vld1q_s16(src1 + j), v_src11 = vld1q_s16(src1 + j + 8); + int16x8_t v_dst0 = vqaddq_s16(v_src00, v_src10); + int16x8_t v_dst1 = vqaddq_s16(v_src01, v_src11); + vst1q_s16(dst + j, v_dst0); + vst1q_s16(dst + j + 8, v_dst1); + } + for (; j < roiw8; j += 8) + { + int16x8_t v_src0 = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(src0 + j))); + int16x8_t v_src1 = vld1q_s16(src1 + j); + int16x8_t v_dst = vqaddq_s16(v_src0, v_src1); + vst1q_s16(dst + j, v_dst); + } + + for (; j < size.width; j++) + dst[j] = internal::saturate_cast((s32)src0[j] + (s32)src1[j]); + } + else + { + for (; j < roiw16; j += 16) + { + internal::prefetch(src0 + j); + internal::prefetch(src1 + j); + uint8x16_t v_src0 = vld1q_u8(src0 + j); + int16x8_t v_src00 = vreinterpretq_s16_u16(vmovl_u8(vget_low_u8(v_src0))); + int16x8_t v_src01 = vreinterpretq_s16_u16(vmovl_u8(vget_high_u8(v_src0))); + int16x8_t v_src10 = vld1q_s16(src1 + j), v_src11 = vld1q_s16(src1 + j + 8); + int16x8_t v_dst0 = vaddq_s16(v_src00, v_src10); + int16x8_t v_dst1 = vaddq_s16(v_src01, v_src11); + vst1q_s16(dst + j, v_dst0); + vst1q_s16(dst + j + 8, v_dst1); + } + for (; j < roiw8; j += 8) + { + int16x8_t v_src0 = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(src0 + j))); + int16x8_t v_src1 = vld1q_s16(src1 + j); + int16x8_t v_dst = vaddq_s16(v_src0, v_src1); + vst1q_s16(dst + j, v_dst); + } + + for (; j < size.width; j++) + dst[j] = (s16)((s32)src0[j] + (s32)src1[j]); + } + } +#else + (void)size; + (void)src0Base; + (void)src0Stride; + (void)src1Base; + (void)src1Stride; + (void)dstBase; + (void)dstStride; + (void)policy; +#endif +} + +void add(const Size2D &size, + const s16 * src0Base, ptrdiff_t src0Stride, + const s16 * src1Base, ptrdiff_t src1Stride, + s16 *dstBase, ptrdiff_t dstStride, + CONVERT_POLICY policy) +{ + internal::assertSupportedConfiguration(); +#ifdef CAROTENE_NEON + if (policy == CONVERT_POLICY_SATURATE) + { + internal::vtransform(size, + src0Base, src0Stride, + src1Base, src1Stride, + dstBase, dstStride, + AddSaturate()); + } + else + { + internal::vtransform(size, + src0Base, src0Stride, + src1Base, src1Stride, + dstBase, dstStride, + AddWrap()); + } +#else + (void)size; + (void)src0Base; + (void)src0Stride; + (void)src1Base; + (void)src1Stride; + (void)dstBase; + (void)dstStride; + (void)policy; +#endif +} + +void add(const Size2D &size, + const u16 * src0Base, ptrdiff_t src0Stride, + const u16 * src1Base, ptrdiff_t src1Stride, + u16 * dstBase, ptrdiff_t dstStride, + CONVERT_POLICY policy) +{ + internal::assertSupportedConfiguration(); +#ifdef CAROTENE_NEON + if (policy == CONVERT_POLICY_SATURATE) + { + internal::vtransform(size, + src0Base, src0Stride, + src1Base, src1Stride, + dstBase, dstStride, + AddSaturate()); + } + else + { + internal::vtransform(size, + src0Base, src0Stride, + src1Base, src1Stride, + dstBase, dstStride, + AddWrap()); + } +#else + (void)size; + (void)src0Base; + (void)src0Stride; + (void)src1Base; + (void)src1Stride; + (void)dstBase; + (void)dstStride; + (void)policy; +#endif +} + +void add(const Size2D &size, + const s32 * src0Base, ptrdiff_t src0Stride, + const s32 * src1Base, ptrdiff_t src1Stride, + s32 *dstBase, ptrdiff_t dstStride, + CONVERT_POLICY policy) +{ + internal::assertSupportedConfiguration(); +#ifdef CAROTENE_NEON + if (policy == CONVERT_POLICY_SATURATE) + { + internal::vtransform(size, + src0Base, src0Stride, + src1Base, src1Stride, + dstBase, dstStride, + AddSaturate()); + } + else + { + internal::vtransform(size, + src0Base, src0Stride, + src1Base, src1Stride, + dstBase, dstStride, + AddWrap()); + } +#else + (void)size; + (void)src0Base; + (void)src0Stride; + (void)src1Base; + (void)src1Stride; + (void)dstBase; + (void)dstStride; + (void)policy; +#endif +} + +void add(const Size2D &size, + const u32 * src0Base, ptrdiff_t src0Stride, + const u32 * src1Base, ptrdiff_t src1Stride, + u32 * dstBase, ptrdiff_t dstStride, + CONVERT_POLICY policy) +{ + internal::assertSupportedConfiguration(); +#ifdef CAROTENE_NEON + if (policy == CONVERT_POLICY_SATURATE) + { + internal::vtransform(size, + src0Base, src0Stride, + src1Base, src1Stride, + dstBase, dstStride, + AddSaturate()); + } + else + { + internal::vtransform(size, + src0Base, src0Stride, + src1Base, src1Stride, + dstBase, dstStride, + AddWrap()); + } +#else + (void)size; + (void)src0Base; + (void)src0Stride; + (void)src1Base; + (void)src1Stride; + (void)dstBase; + (void)dstStride; + (void)policy; +#endif +} + +void add(const Size2D &size, + const f32 * src0Base, ptrdiff_t src0Stride, + const f32 * src1Base, ptrdiff_t src1Stride, + f32 * dstBase, ptrdiff_t dstStride) +{ + internal::assertSupportedConfiguration(); +#ifdef CAROTENE_NEON + internal::vtransform(size, + src0Base, src0Stride, + src1Base, src1Stride, + dstBase, dstStride, + AddWrap()); +#else + (void)size; + (void)src0Base; + (void)src0Stride; + (void)src1Base; + (void)src1Stride; + (void)dstBase; + (void)dstStride; +#endif +} + +} // namespace CAROTENE_NS diff --git a/3rdparty/carotene/src/add_weighted.cpp b/3rdparty/carotene/src/add_weighted.cpp new file mode 100644 index 0000000000..1f89fb5372 --- /dev/null +++ b/3rdparty/carotene/src/add_weighted.cpp @@ -0,0 +1,265 @@ +/* + * By downloading, copying, installing or using the software you agree to this license. + * If you do not agree to this license, do not download, install, + * copy or use the software. + * + * + * License Agreement + * For Open Source Computer Vision Library + * (3-clause BSD License) + * + * Copyright (C) 2012-2015, NVIDIA Corporation, all rights reserved. + * Third party copyrights are property of their respective owners. + * + * Redistribution and use in source and binary forms, with or without modification, + * are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * * Neither the names of the copyright holders nor the names of the contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * This software is provided by the copyright holders and contributors "as is" and + * any express or implied warranties, including, but not limited to, the implied + * warranties of merchantability and fitness for a particular purpose are disclaimed. + * In no event shall copyright holders or contributors be liable for any direct, + * indirect, incidental, special, exemplary, or consequential damages + * (including, but not limited to, procurement of substitute goods or services; + * loss of use, data, or profits; or business interruption) however caused + * and on any theory of liability, whether in contract, strict liability, + * or tort (including negligence or otherwise) arising in any way out of + * the use of this software, even if advised of the possibility of such damage. + */ + +#include "common.hpp" +#include "vtransform.hpp" + +namespace CAROTENE_NS { + +#ifdef CAROTENE_NEON + +namespace { + +using namespace internal; + +template struct TypeTraits; +template <> struct TypeTraits< u8> { typedef u16 wide; typedef u8 unsign; typedef uint8x16_t vec128; }; +template <> struct TypeTraits< s8> { typedef s16 wide; typedef u8 unsign; typedef int8x16_t vec128; }; +template <> struct TypeTraits { typedef u32 wide; typedef u8 narrow; typedef u16 unsign; typedef uint16x8_t vec128; }; +template <> struct TypeTraits { typedef s32 wide; typedef s8 narrow; typedef u16 unsign; typedef int16x8_t vec128; }; +template <> struct TypeTraits { typedef u64 wide; typedef u16 narrow; typedef u32 unsign; typedef uint32x4_t vec128; }; +template <> struct TypeTraits { typedef s64 wide; typedef s16 narrow; typedef u32 unsign; typedef int32x4_t vec128; }; +template <> struct TypeTraits { typedef f64 wide; typedef float32x4_t vec128; }; + +template struct wAdd +{ + typedef T type; + + f32 alpha, beta, gamma; + typedef typename TypeTraits::wide wtype; + wAdd wideAdd; + wAdd(f32 _alpha, f32 _beta, f32 _gamma): + alpha(_alpha), beta(_beta), gamma(_gamma), + wideAdd(_alpha, _beta, _gamma) {} + + void operator() (const typename VecTraits::vec128 & v_src0, + const typename VecTraits::vec128 & v_src1, + typename VecTraits::vec128 & v_dst) const + { + typename VecTraits::vec128 vrl, vrh; + wideAdd(vmovl( vget_low(v_src0)), vmovl( vget_low(v_src1)), vrl); + wideAdd(vmovl(vget_high(v_src0)), vmovl(vget_high(v_src1)), vrh); + + v_dst = vcombine(vqmovn(vrl), vqmovn(vrh)); + } + + void operator() (const typename VecTraits::vec64 & v_src0, + const typename VecTraits::vec64 & v_src1, + typename VecTraits::vec64 & v_dst) const + { + typename VecTraits::vec128 vr; + wideAdd(vmovl(v_src0), vmovl(v_src1), vr); + + v_dst = vqmovn(vr); + } + + void operator() (const T * src0, const T * src1, T * dst) const + { + dst[0] = saturate_cast(alpha*src0[0] + beta*src1[0] + gamma); + } +}; + +template <> struct wAdd +{ + typedef s32 type; + + f32 alpha, beta, gamma; + float32x4_t valpha, vbeta, vgamma; + wAdd(f32 _alpha, f32 _beta, f32 _gamma): + alpha(_alpha), beta(_beta), gamma(_gamma) + { + valpha = vdupq_n_f32(_alpha); + vbeta = vdupq_n_f32(_beta); + vgamma = vdupq_n_f32(_gamma + 0.5); + } + + void operator() (const typename VecTraits::vec128 & v_src0, + const typename VecTraits::vec128 & v_src1, + typename VecTraits::vec128 & v_dst) const + { + float32x4_t vs1 = vcvtq_f32_s32(v_src0); + float32x4_t vs2 = vcvtq_f32_s32(v_src1); + + vs1 = vmlaq_f32(vgamma, vs1, valpha); + vs1 = vmlaq_f32(vs1, vs2, vbeta); + v_dst = vcvtq_s32_f32(vs1); + } + + void operator() (const typename VecTraits::vec64 & v_src0, + const typename VecTraits::vec64 & v_src1, + typename VecTraits::vec64 & v_dst) const + { + float32x2_t vs1 = vcvt_f32_s32(v_src0); + float32x2_t vs2 = vcvt_f32_s32(v_src1); + + vs1 = vmla_f32(vget_low(vgamma), vs1, vget_low(valpha)); + vs1 = vmla_f32(vs1, vs2, vget_low(vbeta)); + v_dst = vcvt_s32_f32(vs1); + } + + void operator() (const s32 * src0, const s32 * src1, s32 * dst) const + { + dst[0] = saturate_cast(alpha*src0[0] + beta*src1[0] + gamma); + } +}; + +template <> struct wAdd +{ + typedef u32 type; + + f32 alpha, beta, gamma; + float32x4_t valpha, vbeta, vgamma; + wAdd(f32 _alpha, f32 _beta, f32 _gamma): + alpha(_alpha), beta(_beta), gamma(_gamma) + { + valpha = vdupq_n_f32(_alpha); + vbeta = vdupq_n_f32(_beta); + vgamma = vdupq_n_f32(_gamma + 0.5); + } + + void operator() (const typename VecTraits::vec128 & v_src0, + const typename VecTraits::vec128 & v_src1, + typename VecTraits::vec128 & v_dst) const + { + float32x4_t vs1 = vcvtq_f32_u32(v_src0); + float32x4_t vs2 = vcvtq_f32_u32(v_src1); + + vs1 = vmlaq_f32(vgamma, vs1, valpha); + vs1 = vmlaq_f32(vs1, vs2, vbeta); + v_dst = vcvtq_u32_f32(vs1); + } + + void operator() (const typename VecTraits::vec64 & v_src0, + const typename VecTraits::vec64 & v_src1, + typename VecTraits::vec64 & v_dst) const + { + float32x2_t vs1 = vcvt_f32_u32(v_src0); + float32x2_t vs2 = vcvt_f32_u32(v_src1); + + vs1 = vmla_f32(vget_low(vgamma), vs1, vget_low(valpha)); + vs1 = vmla_f32(vs1, vs2, vget_low(vbeta)); + v_dst = vcvt_u32_f32(vs1); + } + + void operator() (const u32 * src0, const u32 * src1, u32 * dst) const + { + dst[0] = saturate_cast(alpha*src0[0] + beta*src1[0] + gamma); + } +}; + +template <> struct wAdd +{ + typedef f32 type; + + f32 alpha, beta, gamma; + float32x4_t valpha, vbeta, vgamma; + wAdd(f32 _alpha, f32 _beta, f32 _gamma): + alpha(_alpha), beta(_beta), gamma(_gamma) + { + valpha = vdupq_n_f32(_alpha); + vbeta = vdupq_n_f32(_beta); + vgamma = vdupq_n_f32(_gamma + 0.5); + } + + void operator() (const typename VecTraits::vec128 & v_src0, + const typename VecTraits::vec128 & v_src1, + typename VecTraits::vec128 & v_dst) const + { + float32x4_t vs1 = vmlaq_f32(vgamma, v_src0, valpha); + v_dst = vmlaq_f32(vs1, v_src1, vbeta); + } + + void operator() (const typename VecTraits::vec64 & v_src0, + const typename VecTraits::vec64 & v_src1, + typename VecTraits::vec64 & v_dst) const + { + float32x2_t vs1 = vmla_f32(vget_low(vgamma), v_src0, vget_low(valpha)); + v_dst = vmla_f32(vs1, v_src1, vget_low(vbeta)); + + } + + void operator() (const f32 * src0, const f32 * src1, f32 * dst) const + { + dst[0] = alpha*src0[0] + beta*src1[0] + gamma; + } +}; + +} // namespace + +#define IMPL_ADDWEIGHTED(type) \ +void addWeighted(const Size2D &size, \ + const type * src0Base, ptrdiff_t src0Stride, \ + const type * src1Base, ptrdiff_t src1Stride, \ + type * dstBase, ptrdiff_t dstStride, \ + f32 alpha, f32 beta, f32 gamma) \ +{ \ + internal::assertSupportedConfiguration(); \ + wAdd wgtAdd(alpha, \ + beta, \ + gamma); \ + internal::vtransform(size, \ + src0Base, src0Stride, \ + src1Base, src1Stride, \ + dstBase, dstStride, \ + wgtAdd); \ +} + +#else + +#define IMPL_ADDWEIGHTED(type) \ +void addWeighted(const Size2D &, \ + const type *, ptrdiff_t, \ + const type *, ptrdiff_t, \ + type *, ptrdiff_t, \ + f32, f32, f32) \ +{ \ + internal::assertSupportedConfiguration(); \ +} + +#endif + +IMPL_ADDWEIGHTED(u8) +IMPL_ADDWEIGHTED(s8) +IMPL_ADDWEIGHTED(u16) +IMPL_ADDWEIGHTED(s16) +IMPL_ADDWEIGHTED(u32) +IMPL_ADDWEIGHTED(s32) +IMPL_ADDWEIGHTED(f32) + +} // namespace CAROTENE_NS diff --git a/3rdparty/carotene/src/bitwise.cpp b/3rdparty/carotene/src/bitwise.cpp new file mode 100644 index 0000000000..ee00775111 --- /dev/null +++ b/3rdparty/carotene/src/bitwise.cpp @@ -0,0 +1,225 @@ +/* + * By downloading, copying, installing or using the software you agree to this license. + * If you do not agree to this license, do not download, install, + * copy or use the software. + * + * + * License Agreement + * For Open Source Computer Vision Library + * (3-clause BSD License) + * + * Copyright (C) 2014, NVIDIA Corporation, all rights reserved. + * Third party copyrights are property of their respective owners. + * + * Redistribution and use in source and binary forms, with or without modification, + * are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * * Neither the names of the copyright holders nor the names of the contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * This software is provided by the copyright holders and contributors "as is" and + * any express or implied warranties, including, but not limited to, the implied + * warranties of merchantability and fitness for a particular purpose are disclaimed. + * In no event shall copyright holders or contributors be liable for any direct, + * indirect, incidental, special, exemplary, or consequential damages + * (including, but not limited to, procurement of substitute goods or services; + * loss of use, data, or profits; or business interruption) however caused + * and on any theory of liability, whether in contract, strict liability, + * or tort (including negligence or otherwise) arising in any way out of + * the use of this software, even if advised of the possibility of such damage. + */ + +#include "common.hpp" +#include "vtransform.hpp" + +namespace CAROTENE_NS { + +#ifdef CAROTENE_NEON + +struct BitwiseAnd +{ + typedef u8 type; + + void operator() (const uint8x16_t & v_src0, const uint8x16_t & v_src1, + uint8x16_t & v_dst) const + { + v_dst = vandq_u8(v_src0, v_src1); + } + + void operator() (const uint8x8_t & v_src0, const uint8x8_t & v_src1, + uint8x8_t & v_dst) const + { + v_dst = vand_u8(v_src0, v_src1); + } + + void operator() (const u8 * src0, const u8 * src1, u8 * dst) const + { + dst[0] = src0[0] & src1[0]; + } +}; + +struct BitwiseOr +{ + typedef u8 type; + + void operator() (const uint8x16_t & v_src0, const uint8x16_t & v_src1, + uint8x16_t & v_dst) const + { + v_dst = vorrq_u8(v_src0, v_src1); + } + + void operator() (const uint8x8_t & v_src0, const uint8x8_t & v_src1, + uint8x8_t & v_dst) const + { + v_dst = vorr_u8(v_src0, v_src1); + } + + void operator() (const u8 * src0, const u8 * src1, u8 * dst) const + { + dst[0] = src0[0] | src1[0]; + } +}; + +struct BitwiseXor +{ + typedef u8 type; + + void operator() (const uint8x16_t & v_src0, const uint8x16_t & v_src1, + uint8x16_t & v_dst) const + { + v_dst = veorq_u8(v_src0, v_src1); + } + + void operator() (const uint8x8_t & v_src0, const uint8x8_t & v_src1, + uint8x8_t & v_dst) const + { + v_dst = veor_u8(v_src0, v_src1); + } + + void operator() (const u8 * src0, const u8 * src1, u8 * dst) const + { + dst[0] = src0[0] ^ src1[0]; + } +}; + +#endif + +void bitwiseNot(const Size2D &size, + const u8 *srcBase, ptrdiff_t srcStride, + u8 *dstBase, ptrdiff_t dstStride) +{ + internal::assertSupportedConfiguration(); +#ifdef CAROTENE_NEON + size_t roiw32 = size.width >= 31 ? size.width - 31 : 0; + size_t roiw8 = size.width >= 7 ? size.width - 7 : 0; + + for (size_t i = 0; i < size.height; ++i) + { + const u8* src = internal::getRowPtr(srcBase, srcStride, i); + u8* dst = internal::getRowPtr(dstBase, dstStride, i); + size_t j = 0; + + for (; j < roiw32; j += 32) + { + internal::prefetch(src + j); + uint8x16_t v_src0 = vld1q_u8(src + j), v_src1 = vld1q_u8(src + j + 16); + uint8x16_t v_dst0 = vmvnq_u8(v_src0), v_dst1 = vmvnq_u8(v_src1); + vst1q_u8(dst + j, v_dst0); + vst1q_u8(dst + j + 16, v_dst1); + } + for (; j < roiw8; j += 8) + { + uint8x8_t v_src = vld1_u8(src + j); + uint8x8_t v_dst = vmvn_u8(v_src); + vst1_u8(dst + j, v_dst); + } + + for (; j < size.width; j++) + { + dst[j] = ~src[j]; + } + } +#else + (void)size; + (void)srcBase; + (void)srcStride; + (void)dstBase; + (void)dstStride; +#endif +} + +void bitwiseAnd(const Size2D &size, + const u8 *src0Base, ptrdiff_t src0Stride, + const u8 *src1Base, ptrdiff_t src1Stride, + u8 *dstBase, ptrdiff_t dstStride) +{ + internal::assertSupportedConfiguration(); +#ifdef CAROTENE_NEON + internal::vtransform(size, + src0Base, src0Stride, + src1Base, src1Stride, + dstBase, dstStride, BitwiseAnd()); +#else + (void)size; + (void)src0Base; + (void)src0Stride; + (void)src1Base; + (void)src1Stride; + (void)dstBase; + (void)dstStride; +#endif +} + +void bitwiseOr(const Size2D &size, + const u8 *src0Base, ptrdiff_t src0Stride, + const u8 *src1Base, ptrdiff_t src1Stride, + u8 *dstBase, ptrdiff_t dstStride) +{ + internal::assertSupportedConfiguration(); +#ifdef CAROTENE_NEON + internal::vtransform(size, + src0Base, src0Stride, + src1Base, src1Stride, + dstBase, dstStride, BitwiseOr()); +#else + (void)size; + (void)src0Base; + (void)src0Stride; + (void)src1Base; + (void)src1Stride; + (void)dstBase; + (void)dstStride; +#endif +} + +void bitwiseXor(const Size2D &size, + const u8 *src0Base, ptrdiff_t src0Stride, + const u8 *src1Base, ptrdiff_t src1Stride, + u8 *dstBase, ptrdiff_t dstStride) +{ + internal::assertSupportedConfiguration(); +#ifdef CAROTENE_NEON + internal::vtransform(size, + src0Base, src0Stride, + src1Base, src1Stride, + dstBase, dstStride, BitwiseXor()); +#else + (void)size; + (void)src0Base; + (void)src0Stride; + (void)src1Base; + (void)src1Stride; + (void)dstBase; + (void)dstStride; +#endif +} + +} // namespace CAROTENE_NS diff --git a/3rdparty/carotene/src/blur.cpp b/3rdparty/carotene/src/blur.cpp new file mode 100644 index 0000000000..798cce5a71 --- /dev/null +++ b/3rdparty/carotene/src/blur.cpp @@ -0,0 +1,1337 @@ +/* + * By downloading, copying, installing or using the software you agree to this license. + * If you do not agree to this license, do not download, install, + * copy or use the software. + * + * + * License Agreement + * For Open Source Computer Vision Library + * (3-clause BSD License) + * + * Copyright (C) 2012-2015, NVIDIA Corporation, all rights reserved. + * Third party copyrights are property of their respective owners. + * + * Redistribution and use in source and binary forms, with or without modification, + * are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * * Neither the names of the copyright holders nor the names of the contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * This software is provided by the copyright holders and contributors "as is" and + * any express or implied warranties, including, but not limited to, the implied + * warranties of merchantability and fitness for a particular purpose are disclaimed. + * In no event shall copyright holders or contributors be liable for any direct, + * indirect, incidental, special, exemplary, or consequential damages + * (including, but not limited to, procurement of substitute goods or services; + * loss of use, data, or profits; or business interruption) however caused + * and on any theory of liability, whether in contract, strict liability, + * or tort (including negligence or otherwise) arising in any way out of + * the use of this software, even if advised of the possibility of such damage. + */ + +#include + +#include "common.hpp" +#include "saturate_cast.hpp" + +namespace CAROTENE_NS { + +bool isBlur3x3Supported(const Size2D &size, BORDER_MODE border) +{ + return isSupportedConfiguration() && size.width >= 8 && + (border == BORDER_MODE_CONSTANT || + border == BORDER_MODE_REPLICATE); +} + +void blur3x3(const Size2D &size, + const u8 * srcBase, ptrdiff_t srcStride, + u8 * dstBase, ptrdiff_t dstStride, + BORDER_MODE border, u8 borderValue) +{ + internal::assertSupportedConfiguration(isBlur3x3Supported(size, border)); +#ifdef CAROTENE_NEON + const int16x8_t v_scale = vmovq_n_s16(3640); + const uint16x8_t v_border_x3 = vdupq_n_u16(borderValue * 3); + const uint16x8_t v_zero = vdupq_n_u16(0); + const uint8x8_t v_border = vdup_n_u8(borderValue); + + uint16x8_t tprev = v_zero, tcurr = v_zero, tnext = v_zero; + uint16x8_t t0 = v_zero, t1 = v_zero, t2 = v_zero; + + ptrdiff_t width = (ptrdiff_t)size.width, height = (ptrdiff_t)size.height; + + for (ptrdiff_t y = 0; y < height; ++y) + { + const u8 * srow0 = y == 0 && border == BORDER_MODE_CONSTANT ? NULL : internal::getRowPtr(srcBase, srcStride, std::max(y - 1, 0)); + const u8 * srow1 = internal::getRowPtr(srcBase, srcStride, y); + const u8 * srow2 = y + 1 == height && border == BORDER_MODE_CONSTANT ? NULL : internal::getRowPtr(srcBase, srcStride, std::min(y + 1, height - 1)); + u8 * drow = internal::getRowPtr(dstBase, dstStride, y); + + s16 prevx = 0, currx = 0, nextx = 0; + ptrdiff_t x = 0; + const ptrdiff_t bwidth = y + 2 < height ? width : (width - 8); + + // perform vertical convolution + for ( ; x <= bwidth; x += 8) + { + internal::prefetch(srow0 + x); + internal::prefetch(srow1 + x); + internal::prefetch(srow2 + x); + + uint8x8_t x0 = !srow0 ? v_border : vld1_u8(srow0 + x); + uint8x8_t x1 = vld1_u8(srow1 + x); + uint8x8_t x2 = !srow2 ? v_border : vld1_u8(srow2 + x); + + // calculate values for plain CPU part below if needed + if (x + 8 >= bwidth) + { + ptrdiff_t x3 = x == width ? width - 1 : x; + ptrdiff_t x4 = border == BORDER_MODE_CONSTANT ? x3 - 1 : std::max(x3 - 1, 0); + + if (border == BORDER_MODE_CONSTANT && x4 < 0) + prevx = borderValue; + else + prevx = (srow2 ? srow2[x4] : borderValue) + srow1[x4] + (srow0 ? srow0[x4] : borderValue); + + currx = (srow2 ? srow2[x3] : borderValue) + srow1[x3] + (srow0 ? srow0[x3] : borderValue); + } + + // make shift + if (x) + { + tprev = tcurr; + tcurr = tnext; + } + + // and calculate next value + tnext = vaddw_u8(vaddl_u8(x0, x1), x2); + + // make extrapolation for the first elements + if (!x) + { + // make border + if (border == BORDER_MODE_CONSTANT) + tcurr = v_border_x3; + else if (border == BORDER_MODE_REPLICATE) + tcurr = vdupq_n_u16(vgetq_lane_u16(tnext, 0)); + + continue; + } + + // combine 3 "shifted" vectors + t0 = vextq_u16(tprev, tcurr, 7); + t1 = tcurr; + t2 = vextq_u16(tcurr, tnext, 1); + + // and add them + t0 = vqaddq_u16(t0, vqaddq_u16(t1, t2)); + + int16x8_t tt0 = vqrdmulhq_s16(vreinterpretq_s16_u16(t0), v_scale); + uint8x8_t it0 = vmovn_u16(vreinterpretq_u16_s16(tt0)); + vst1_u8(drow + x - 8, it0); + } + + x -= 8; + if (x == width) + --x; + + for ( ; x < width; ++x) + { + // make extrapolation for the last elements + if (x + 1 >= width) + { + if (border == BORDER_MODE_CONSTANT) + nextx = borderValue * 3; + else if (border == BORDER_MODE_REPLICATE) + nextx = srow2[x] + srow1[x] + srow0[x]; + } + else + nextx = (srow2 ? srow2[x + 1] : borderValue) + + srow1[x + 1] + + (srow0 ? srow0[x + 1] : borderValue); + + f32 val = (prevx + currx + nextx) * (1 / 9.f) + 0.5f; + drow[x] = internal::saturate_cast((s32)val); + + // make shift + prevx = currx; + currx = nextx; + } + } +#else + (void)size; + (void)srcBase; + (void)srcStride; + (void)dstBase; + (void)dstStride; + (void)border; + (void)borderValue; +#endif +} + +bool isBlurU8Supported(const Size2D &size, s32 cn, BORDER_MODE border) +{ + return isSupportedConfiguration() && + cn > 0 && cn <= 4 && + size.width*cn >= 8 && size.height >= 2 && + (border == BORDER_MODE_CONSTANT || + border == BORDER_MODE_REFLECT101 || + border == BORDER_MODE_REFLECT || + border == BORDER_MODE_REPLICATE); +} + +void blur3x3(const Size2D &size, s32 cn, + const u8 * srcBase, ptrdiff_t srcStride, + u8 * dstBase, ptrdiff_t dstStride, + BORDER_MODE borderType, u8 borderValue) +{ + internal::assertSupportedConfiguration(isBlurU8Supported(size, cn, borderType)); +#ifdef CAROTENE_NEON +//#define FLOAT_VARIANT_1_9 +#ifdef FLOAT_VARIANT_1_9 + float32x4_t v1_9 = vdupq_n_f32 (1.0/9.0); + float32x4_t v0_5 = vdupq_n_f32 (.5); +#else + const int16x8_t vScale = vmovq_n_s16(3640); +#endif + + size_t colsn = size.width*cn; + + std::vector _tmp; + u8 *tmp = 0; + if (borderType == BORDER_MODE_CONSTANT) + { + _tmp.assign(colsn + 2*cn, borderValue); + tmp = &_tmp[cn]; + } + + uint16x8_t tprev = vdupq_n_u16(0x0); + uint16x8_t tcurr = tprev; + uint16x8_t tnext = tprev; + uint16x8_t t0, t1, t2; + if(cn == 1) + { + for( size_t y = 0; y < size.height; y++ ) + { + const u8* srow0; + const u8* srow1 = internal::getRowPtr(srcBase, srcStride, y); + const u8* srow2; + u8* drow = internal::getRowPtr(dstBase, dstStride, y); + if (borderType == BORDER_MODE_REFLECT101) { + srow0 = internal::getRowPtr(srcBase, srcStride, y > 0 ? y-1 : 1); + srow2 = internal::getRowPtr(srcBase, srcStride, y < size.height-1 ? y+1 : size.height-2); + } else if (borderType == BORDER_MODE_CONSTANT) { + srow0 = y > 0 ? internal::getRowPtr(srcBase, srcStride, y-1) : tmp; + srow2 = y < size.height-1 ? internal::getRowPtr(srcBase, srcStride, y+1) : tmp; + } else { // BORDER_MODE_REFLECT || BORDER_MODE_REPLICATE + srow0 = internal::getRowPtr(srcBase, srcStride, y > 0 ? y-1 : 0); + srow2 = internal::getRowPtr(srcBase, srcStride, y < size.height-1 ? y+1 : size.height-1); + } + + // do vertical convolution + size_t x = 0; + const size_t bcols = y + 2 < size.height ? colsn : (colsn - 8); + for( ; x <= bcols; x += 8 ) + { + internal::prefetch(srow0 + x); + internal::prefetch(srow1 + x); + internal::prefetch(srow2 + x); + + uint8x8_t x0 = vld1_u8(srow0 + x); + uint8x8_t x1 = vld1_u8(srow1 + x); + uint8x8_t x2 = vld1_u8(srow2 + x); + + tprev = tcurr; + tcurr = tnext; + tnext = vaddw_u8(vaddl_u8(x0, x1), x2); + + if(!x) { + tcurr = tnext; + + // make border + if (borderType == BORDER_MODE_CONSTANT) + { + tcurr = vsetq_lane_u16(borderValue, tcurr, 7); + } + else if (borderType == BORDER_MODE_REFLECT101) + { + tcurr = vsetq_lane_u16(vgetq_lane_u16(tcurr, 1),tcurr, 7); + } + else // borderType == BORDER_MODE_REFLECT || borderType == BORDER_MODE_REPLICATE + { + tcurr = vsetq_lane_u16(vgetq_lane_u16(tcurr, 0),tcurr, 7); + } + continue; + } + + t0 = vextq_u16(tprev, tcurr, 7); + t1 = tcurr; + t2 = vextq_u16(tcurr, tnext, 1); + + t0 = vqaddq_u16(t0, vqaddq_u16(t1, t2)); + +#ifdef FLOAT_VARIANT_1_9 + uint32x4_t tres1 = vmovl_u16(vget_low_u16(t0)); + uint32x4_t tres2 = vmovl_u16(vget_high_u16(t0)); + float32x4_t vf1 = vmulq_f32(v1_9, vcvtq_f32_u32(tres1)); + float32x4_t vf2 = vmulq_f32(v1_9, vcvtq_f32_u32(tres2)); + tres1 = vcvtq_u32_f32(vaddq_f32(vf1, v0_5)); + tres2 = vcvtq_u32_f32(vaddq_f32(vf2, v0_5)); + t0 = vcombine_u16(vmovn_u32(tres1),vmovn_u32(tres2)); + vst1_u8(drow + x - 8, vmovn_u16(t0)); +#else + int16x8_t tt0 = vqrdmulhq_s16(vreinterpretq_s16_u16(t0), vScale); + uint8x8_t it0 = vmovn_u16(vreinterpretq_u16_s16(tt0)); + vst1_u8(drow + x - 8, it0); +#endif + } + + x -= 8; + if(x == colsn){ + x--; + } + s16 prevx, rowx, nextx; + prevx = srow2[x-1] + srow1[x-1] + srow0[x-1]; + rowx = srow2[x] + srow1[x] + srow0[x]; + for( ; x < colsn; x++ ) + { + if(x+1 >= colsn) { + // make border + if (borderType == BORDER_MODE_CONSTANT) + { + nextx = borderValue; + } else if (borderType == BORDER_MODE_REFLECT101) + { + nextx = srow2[x-1] + srow1[x-1] + srow0[x-1]; + } else { + nextx = srow2[x] + srow1[x] + srow0[x]; + } + } else { + nextx = srow2[x+1] + srow1[x+1] + srow0[x+1]; + } + *(drow+x) = internal::saturate_cast((prevx + rowx + nextx)*(1/9.)); + prevx = rowx; + rowx = nextx; + } + } + } + else + { + for( size_t y = 0; y < size.height; y++ ) + { + const u8* srow0; + const u8* srow1 = internal::getRowPtr(srcBase, srcStride, y); + const u8* srow2; + u8* drow = internal::getRowPtr(dstBase, dstStride, y); + if (borderType == BORDER_MODE_REFLECT101) { + srow0 = internal::getRowPtr(srcBase, srcStride, y > 0 ? y-1 : 1); + srow2 = internal::getRowPtr(srcBase, srcStride, y < size.height-1 ? y+1 : size.height-2); + } else if (borderType == BORDER_MODE_CONSTANT) { + srow0 = y > 0 ? internal::getRowPtr(srcBase, srcStride, y-1) : tmp; + srow2 = y < size.height-1 ? internal::getRowPtr(srcBase, srcStride, y+1) : tmp; + } else { // BORDER_MODE_REFLECT || BORDER_MODE_REPLICATE + srow0 = internal::getRowPtr(srcBase, srcStride, y > 0 ? y-1 : 0); + srow2 = internal::getRowPtr(srcBase, srcStride, y < size.height-1 ? y+1 : size.height-1); + } + + // do vertical convolution + size_t x = 0; + const size_t bcols = y + 2 < size.height ? colsn : (colsn - 8); + for( ; x <= bcols; x += 8 ) + { + internal::prefetch(srow0 + x); + internal::prefetch(srow1 + x); + internal::prefetch(srow2 + x); + + uint8x8_t x0 = vld1_u8(srow0 + x); + uint8x8_t x1 = vld1_u8(srow1 + x); + uint8x8_t x2 = vld1_u8(srow2 + x); + + tprev = tcurr; + tcurr = tnext; + tnext = vaddw_u8(vaddl_u8(x0, x1), x2); + + if(!x) { + tcurr = tnext; + + // make border + switch(cn) + { + case 2: + if (borderType == BORDER_MODE_CONSTANT) + { + tcurr = vsetq_lane_u16(borderValue, tcurr, 6); + tcurr = vsetq_lane_u16(borderValue, tcurr, 7); + } + else if (borderType == BORDER_MODE_REFLECT101) + { + tcurr = vsetq_lane_u16(vgetq_lane_u16(tcurr, 2),tcurr, 6); + tcurr = vsetq_lane_u16(vgetq_lane_u16(tcurr, 3),tcurr, 6); + } + else + { + tcurr = vsetq_lane_u16(vgetq_lane_u16(tcurr, 0),tcurr, 6); + tcurr = vsetq_lane_u16(vgetq_lane_u16(tcurr, 1),tcurr, 7); + } + break; + case 3: + if (borderType == BORDER_MODE_CONSTANT) + { + tcurr = vsetq_lane_u16(borderValue, tcurr, 5); + tcurr = vsetq_lane_u16(borderValue, tcurr, 6); + tcurr = vsetq_lane_u16(borderValue, tcurr, 7); + } + else if (borderType == BORDER_MODE_REFLECT101) + { + tcurr = vsetq_lane_u16(vgetq_lane_u16(tcurr, 3),tcurr, 5); + tcurr = vsetq_lane_u16(vgetq_lane_u16(tcurr, 4),tcurr, 6); + tcurr = vsetq_lane_u16(vgetq_lane_u16(tcurr, 5),tcurr, 7); + } + else + { + tcurr = vsetq_lane_u16(vgetq_lane_u16(tcurr, 0),tcurr, 5); + tcurr = vsetq_lane_u16(vgetq_lane_u16(tcurr, 1),tcurr, 6); + tcurr = vsetq_lane_u16(vgetq_lane_u16(tcurr, 2),tcurr, 7); + } + break; + case 4: + if (borderType == BORDER_MODE_CONSTANT) + { + tcurr = vsetq_lane_u16(borderValue, tcurr, 4); + tcurr = vsetq_lane_u16(borderValue, tcurr, 5); + tcurr = vsetq_lane_u16(borderValue, tcurr, 6); + tcurr = vsetq_lane_u16(borderValue, tcurr, 7); + } + else if (borderType != BORDER_MODE_REFLECT101) + { + tcurr = vsetq_lane_u16(vgetq_lane_u16(tcurr, 0),tcurr, 4); + tcurr = vsetq_lane_u16(vgetq_lane_u16(tcurr, 1),tcurr, 5); + tcurr = vsetq_lane_u16(vgetq_lane_u16(tcurr, 2),tcurr, 6); + tcurr = vsetq_lane_u16(vgetq_lane_u16(tcurr, 3),tcurr, 7); + } + break; + } + continue; + } + + if(cn==2) + t0 = vextq_u16(tprev, tcurr, 6); + else if(cn==3) + t0 = vextq_u16(tprev, tcurr, 5); + else if(cn==4) + t0 = vextq_u16(tprev, tcurr, 4); + + t1 = tcurr; + + if(cn==2) + t2 = vextq_u16(tcurr, tnext, 2); + else if(cn==3) + t2 = vextq_u16(tcurr, tnext, 3); + else if(cn==4) + t2 = vextq_u16(tcurr, tnext, 4); + + t0 = vqaddq_u16(t0, vqaddq_u16(t1, t2)); + +#ifdef FLOAT_VARIANT_1_9 + uint32x4_t tres1 = vmovl_u16(vget_low_u16(t0)); + uint32x4_t tres2 = vmovl_u16(vget_high_u16(t0)); + float32x4_t vf1 = vmulq_f32(v1_9, vcvtq_f32_u32(tres1)); + float32x4_t vf2 = vmulq_f32(v1_9, vcvtq_f32_u32(tres2)); + tres1 = vcvtq_u32_f32(vaddq_f32(vf1, v0_5)); + tres2 = vcvtq_u32_f32(vaddq_f32(vf2, v0_5)); + t0 = vcombine_u16(vmovn_u32(tres1),vmovn_u32(tres2)); + vst1_u8(drow + x - 8, vmovn_u16(t0)); +#else + int16x8_t tt0 = vqrdmulhq_s16(vreinterpretq_s16_u16(t0), vScale); + uint8x8_t it0 = vmovn_u16(vreinterpretq_u16_s16(tt0)); + vst1_u8(drow + x - 8, it0); +#endif + } + + x -= 8; + if(x == colsn){ + x -= cn; + } + s16 prevx[4], rowx[4], nextx[4]; + for( s32 k = 0; k < cn; k++ ) + { + prevx[(k + x%cn)%cn] = srow2[x+k-cn] + srow1[x+k-cn] + srow0[x+k-cn]; + rowx[(k + x%cn)%cn] = srow2[x+k] + srow1[x+k] + srow0[x+k]; + } + for( ; x < colsn; x++ ) + { + size_t xx = x%cn; + if(x+cn >= colsn) { + // make border + if (borderType == BORDER_MODE_CONSTANT) + { + nextx[xx] = borderValue; + } else if (borderType == BORDER_MODE_REFLECT101) + { + nextx[xx] = srow2[x-cn] + srow1[x-cn] + srow0[x-cn]; + } else { + nextx[xx] = srow2[x] + srow1[x] + srow0[x]; + } + } else { + nextx[xx] = srow2[x+cn] + srow1[x+cn] + srow0[x+cn]; + } + *(drow+x) = internal::saturate_cast((prevx[xx] + rowx[xx] + nextx[xx])*(1/9.)); + prevx[xx] = rowx[xx]; + rowx[xx] = nextx[xx]; + } + } + } +#else + (void)srcBase; + (void)srcStride; + (void)dstBase; + (void)dstStride; + (void)borderValue; +#endif +} + +void blur5x5(const Size2D &size, s32 cn, + const u8 * srcBase, ptrdiff_t srcStride, + u8 * dstBase, ptrdiff_t dstStride, + BORDER_MODE borderType, u8 borderValue) +{ + internal::assertSupportedConfiguration(isBlurU8Supported(size, cn, borderType)); +#ifdef CAROTENE_NEON +#define FLOAT_VARIANT_1_25 +#ifdef FLOAT_VARIANT_1_25 + float32x4_t v1_25 = vdupq_n_f32 (1.0f/25.0f); + float32x4_t v0_5 = vdupq_n_f32 (.5f); +#else + const int16x8_t vScale = vmovq_n_s16(1310); +#endif + size_t colsn = size.width*cn; + + std::vector _tmp; + u8 *tmp = 0; + if (borderType == BORDER_MODE_CONSTANT) + { + _tmp.assign(colsn + 2*cn, borderValue); + tmp = &_tmp[cn]; + } + + uint16x8_t tprev = vdupq_n_u16(0x0); + uint16x8_t tcurr = tprev; + uint16x8_t tnext = tprev; + uint16x8_t t0, t1, t2, t3, t4; + for( size_t y = 0; y < size.height; y++ ) + { + const u8 *srow0, *srow1; + const u8 *srow2 = internal::getRowPtr(srcBase, srcStride, y); + const u8 *srow3, *srow4; + u8 *drow = internal::getRowPtr(dstBase, dstStride, y); + if (borderType == BORDER_MODE_REFLECT101) { + srow0 = internal::getRowPtr(srcBase, srcStride, y > 1 ? y-2 : 2-y); + srow1 = internal::getRowPtr(srcBase, srcStride, y > 0 ? y-1 : 1); + srow3 = internal::getRowPtr(srcBase, srcStride, y < size.height-1 ? y+1 : size.height-2); + srow4 = internal::getRowPtr(srcBase, srcStride, y < size.height-2 ? y+2 : (size.height<<1)-4-y); + } else if (borderType == BORDER_MODE_CONSTANT) { + srow0 = y > 1 ? internal::getRowPtr(srcBase, srcStride, y-2) : tmp; + srow1 = y > 0 ? internal::getRowPtr(srcBase, srcStride, y-1) : tmp; + srow3 = y < size.height-1 ? internal::getRowPtr(srcBase, srcStride, y+1) : tmp; + srow4 = y < size.height-2 ? internal::getRowPtr(srcBase, srcStride, y+2) : tmp; + } else if (borderType == BORDER_MODE_REFLECT) { + srow0 = internal::getRowPtr(srcBase, srcStride, y > 1 ? y-2 : 1-y); + srow1 = internal::getRowPtr(srcBase, srcStride, y > 0 ? y-1 : 0); + srow3 = internal::getRowPtr(srcBase, srcStride, y < size.height-1 ? y+1 : size.height-1); + srow4 = internal::getRowPtr(srcBase, srcStride, y < size.height-2 ? y+2 : (size.height<<1)-3-y); + } else { // BORDER_MODE_REPLICATE + srow0 = internal::getRowPtr(srcBase, srcStride, y > 1 ? y-2 : 0); + srow1 = internal::getRowPtr(srcBase, srcStride, y > 0 ? y-1 : 0); + srow3 = internal::getRowPtr(srcBase, srcStride, y < size.height-1 ? y+1 : size.height-1); + srow4 = internal::getRowPtr(srcBase, srcStride, y < size.height-2 ? y+2 : size.height-1); + } + + // do vertical convolution + size_t x = 0; + const size_t bcols = y + 3 < size.height ? colsn : (colsn - 8); + for( ; x <= bcols; x += 8 ) + { + internal::prefetch(srow0 + x); + internal::prefetch(srow1 + x); + internal::prefetch(srow2 + x); + internal::prefetch(srow3 + x); + internal::prefetch(srow4 + x); + + uint8x8_t x0 = vld1_u8(srow0 + x); + uint8x8_t x1 = vld1_u8(srow1 + x); + uint8x8_t x2 = vld1_u8(srow2 + x); + uint8x8_t x3 = vld1_u8(srow3 + x); + uint8x8_t x4 = vld1_u8(srow4 + x); + + tprev = tcurr; + tcurr = tnext; + tnext = vaddw_u8(vaddq_u16(vaddl_u8(x0, x1), vaddl_u8(x2, x3)), x4); + + if(!x) { + tcurr = tnext; + + if(borderType == BORDER_MODE_REFLECT101 && size.width < 3) + { + x = 8; + break; + } + + // make border + switch(cn) + { + case 1: + if (borderType == BORDER_MODE_CONSTANT) + { + tcurr = vsetq_lane_u16(borderValue, tcurr, 6); + tcurr = vsetq_lane_u16(borderValue, tcurr, 7); + } + else if (borderType == BORDER_MODE_REFLECT101) + { + tcurr = vsetq_lane_u16(vgetq_lane_u16(tcurr, 2),tcurr, 6); + tcurr = vsetq_lane_u16(vgetq_lane_u16(tcurr, 1),tcurr, 7); + } + else if (borderType == BORDER_MODE_REFLECT) + { + tcurr = vsetq_lane_u16(vgetq_lane_u16(tcurr, 1),tcurr, 6); + tcurr = vsetq_lane_u16(vgetq_lane_u16(tcurr, 0),tcurr, 7); + } + else + { + tcurr = vsetq_lane_u16(vgetq_lane_u16(tcurr, 0),tcurr, 6); + tcurr = vsetq_lane_u16(vgetq_lane_u16(tcurr, 0),tcurr, 7); + } + break; + case 2: + if (borderType == BORDER_MODE_CONSTANT) + { + tcurr = vsetq_lane_u16(borderValue, tcurr, 4); + tcurr = vsetq_lane_u16(borderValue, tcurr, 5); + tcurr = vsetq_lane_u16(borderValue, tcurr, 6); + tcurr = vsetq_lane_u16(borderValue, tcurr, 7); + } + else if (borderType == BORDER_MODE_REFLECT101) + { + tcurr = vsetq_lane_u16(vgetq_lane_u16(tcurr, 2),tcurr, 6); + tcurr = vsetq_lane_u16(vgetq_lane_u16(tcurr, 3),tcurr, 7); + } + else if (borderType == BORDER_MODE_REFLECT) + { + tcurr = vsetq_lane_u16(vgetq_lane_u16(tcurr, 2),tcurr, 4); + tcurr = vsetq_lane_u16(vgetq_lane_u16(tcurr, 3),tcurr, 5); + tcurr = vsetq_lane_u16(vgetq_lane_u16(tcurr, 0),tcurr, 6); + tcurr = vsetq_lane_u16(vgetq_lane_u16(tcurr, 1),tcurr, 7); + } + else + { + tcurr = vsetq_lane_u16(vgetq_lane_u16(tcurr, 0),tcurr, 4); + tcurr = vsetq_lane_u16(vgetq_lane_u16(tcurr, 1),tcurr, 5); + tcurr = vsetq_lane_u16(vgetq_lane_u16(tcurr, 0),tcurr, 6); + tcurr = vsetq_lane_u16(vgetq_lane_u16(tcurr, 1),tcurr, 7); + } + break; + case 3: + if (borderType == BORDER_MODE_CONSTANT) + { + tcurr = vsetq_lane_u16(borderValue, tcurr, 2); + tcurr = vsetq_lane_u16(borderValue, tcurr, 3); + tcurr = vsetq_lane_u16(borderValue, tcurr, 4); + tcurr = vsetq_lane_u16(borderValue, tcurr, 5); + tcurr = vsetq_lane_u16(borderValue, tcurr, 6); + tcurr = vsetq_lane_u16(borderValue, tcurr, 7); + } + else if (borderType == BORDER_MODE_REFLECT101) + { + tprev = vsetq_lane_u16(vgetq_lane_u16(tcurr, 6),tcurr, 2); + tprev = vsetq_lane_u16(vgetq_lane_u16(tcurr, 7),tprev, 3); + tprev = vsetq_lane_u16(vgetq_lane_u16(tcurr, 3),tprev, 5); + tprev = vsetq_lane_u16(vgetq_lane_u16(tcurr, 4),tprev, 6); + tprev = vsetq_lane_u16(vgetq_lane_u16(tcurr, 5),tprev, 7); + s16 lane8 = srow4[8] + srow3[8] + srow2[8] + srow1[8] + srow0[8]; + tcurr = vsetq_lane_u16(lane8,tprev, 4); + } + else if (borderType == BORDER_MODE_REFLECT) + { + tprev = vsetq_lane_u16(vgetq_lane_u16(tcurr, 3),tcurr, 2); + tprev = vsetq_lane_u16(vgetq_lane_u16(tcurr, 4),tprev, 3); + tprev = vsetq_lane_u16(vgetq_lane_u16(tcurr, 5),tprev, 4); + tprev = vsetq_lane_u16(vgetq_lane_u16(tcurr, 0),tprev, 5); + tprev = vsetq_lane_u16(vgetq_lane_u16(tcurr, 1),tprev, 6); + tcurr = vsetq_lane_u16(vgetq_lane_u16(tcurr, 2),tprev, 7); + } + else + { + tprev = vsetq_lane_u16(vgetq_lane_u16(tcurr, 0),tcurr, 2); + tprev = vsetq_lane_u16(vgetq_lane_u16(tcurr, 1),tprev, 3); + tprev = vsetq_lane_u16(vgetq_lane_u16(tcurr, 2),tprev, 4); + tprev = vsetq_lane_u16(vgetq_lane_u16(tcurr, 0),tprev, 5); + tprev = vsetq_lane_u16(vgetq_lane_u16(tcurr, 1),tprev, 6); + tcurr = vsetq_lane_u16(vgetq_lane_u16(tcurr, 2),tprev, 7); + } + break; + case 4: + if (borderType == BORDER_MODE_CONSTANT) + { + tcurr = vsetq_lane_u16(borderValue, tcurr, 0); + tcurr = vsetq_lane_u16(borderValue, tcurr, 1); + tcurr = vsetq_lane_u16(borderValue, tcurr, 2); + tcurr = vsetq_lane_u16(borderValue, tcurr, 3); + tcurr = vsetq_lane_u16(borderValue, tcurr, 4); + tcurr = vsetq_lane_u16(borderValue, tcurr, 5); + tcurr = vsetq_lane_u16(borderValue, tcurr, 6); + tcurr = vsetq_lane_u16(borderValue, tcurr, 7); + } + else if (borderType == BORDER_MODE_REFLECT101) + { + s16 lane8 = srow4[ 8] + srow3[ 8] + srow2[ 8] + srow1[ 8] + srow0[ 8]; + s16 lane9 = srow4[ 9] + srow3[ 9] + srow2[ 9] + srow1[ 9] + srow0[ 9]; + s16 lane10 = srow4[10] + srow3[10] + srow2[10] + srow1[10] + srow0[10]; + s16 lane11 = srow4[11] + srow3[11] + srow2[11] + srow1[11] + srow0[11]; + tprev = vsetq_lane_u16( lane8,tcurr, 0); + tprev = vsetq_lane_u16( lane9,tprev, 1); + tprev = vsetq_lane_u16(lane10,tprev, 2); + tcurr = vsetq_lane_u16(lane11,tprev, 3); + } + else if (borderType == BORDER_MODE_REFLECT) + { + tcurr = vcombine_u16(vget_high_u16(tcurr),vget_low_u16(tcurr));//swap 64-bit parts + } + else + { + tcurr = vcombine_u16(vget_low_u16(tcurr),vget_low_u16(tcurr));//double 64-bit part + } + break; + } + continue; + } + switch(cn) + { + case 1: + t0 = vextq_u16(tprev, tcurr, 6); + t1 = vextq_u16(tprev, tcurr, 7); + t2 = tcurr; + t3 = vextq_u16(tcurr, tnext, 1); + t4 = vextq_u16(tcurr, tnext, 2); + break; + case 2: + t0 = vextq_u16(tprev, tcurr, 4); + t1 = vextq_u16(tprev, tcurr, 6); + t2 = tcurr; + t3 = vextq_u16(tcurr, tnext, 2); + t4 = vextq_u16(tcurr, tnext, 4); + break; + case 3: + t0 = vextq_u16(tprev, tcurr, 2); + t1 = vextq_u16(tprev, tcurr, 5); + t2 = tcurr; + t3 = vextq_u16(tcurr, tnext, 3); + t4 = vextq_u16(tcurr, tnext, 6); + break; + case 4: + t0 = tprev; + t1 = vextq_u16(tprev, tcurr, 4); + t2 = tcurr; + t3 = vextq_u16(tcurr, tnext, 4); + t4 = tnext; + break; + default: + internal::assertSupportedConfiguration(false);//Unsupported channels number + return; + } + t0 = vqaddq_u16(vqaddq_u16(vqaddq_u16(t0, t1), vqaddq_u16(t2, t3)), t4); + +#ifdef FLOAT_VARIANT_1_25 + uint32x4_t tres1 = vmovl_u16(vget_low_u16(t0)); + uint32x4_t tres2 = vmovl_u16(vget_high_u16(t0)); + float32x4_t vf1 = vmulq_f32(v1_25, vcvtq_f32_u32(tres1)); + float32x4_t vf2 = vmulq_f32(v1_25, vcvtq_f32_u32(tres2)); + tres1 = vcvtq_u32_f32(vaddq_f32(vf1, v0_5)); + tres2 = vcvtq_u32_f32(vaddq_f32(vf2, v0_5)); + t0 = vcombine_u16(vmovn_u32(tres1),vmovn_u32(tres2)); + vst1_u8(drow + x - 8, vmovn_u16(t0)); +#else + int16x8_t tt0 = vqrdmulhq_s16(vreinterpretq_s16_u16(t0), vScale); + uint8x8_t it0 = vmovn_u16(vreinterpretq_u16_s16(tt0)); + vst1_u8(drow + x - 8, it0); +#endif + } + + x -= 8; + if(x == colsn){ + x -= cn; + } + s16 pprevx[4], prevx[4], rowx[4], nextx[4], nnextx[4]; + ptrdiff_t px = x / cn; + for( s32 k = 0; k < cn; k++ ) + { + ptrdiff_t ploc; + ploc = internal::borderInterpolate(px-2, size.width, borderType); + pprevx[k] = ploc < 0 ? 5*borderValue : + srow4[ploc*cn+k] + srow3[ploc*cn+k] + srow2[ploc*cn+k] + srow1[ploc*cn+k] + srow0[ploc*cn+k]; + + ploc = internal::borderInterpolate(px-1, size.width, borderType); + prevx[k] = ploc < 0 ? 5*borderValue : + srow4[ploc*cn+k] + srow3[ploc*cn+k] + srow2[ploc*cn+k] + srow1[ploc*cn+k] + srow0[ploc*cn+k]; + + rowx[k] = srow4[px*cn+k] + srow3[px*cn+k] + srow2[px*cn+k] + srow1[px*cn+k] + srow0[px*cn+k]; + + ploc = internal::borderInterpolate(px+1, size.width, borderType); + nextx[k] = ploc < 0 ? 5*borderValue : + srow4[ploc*cn+k] + srow3[ploc*cn+k] + srow2[ploc*cn+k] + srow1[ploc*cn+k] + srow0[ploc*cn+k]; + } + x = px*cn; + for( ; x < colsn; x+=cn, px++ ) + { + for( s32 k = 0; k < cn; k++ ) + { + ptrdiff_t ploc = internal::borderInterpolate(px+2, size.width, borderType); + nnextx[k] = ploc < 0 ? 5*borderValue : + srow4[ploc*cn+k] + srow3[ploc*cn+k] + srow2[ploc*cn+k] + srow1[ploc*cn+k] + srow0[ploc*cn+k]; + *(drow+x+k) = internal::saturate_cast((pprevx[k] + prevx[k] + rowx[k] + nextx[k] +nnextx[k])*(1/25.)); + pprevx[k] = prevx[k]; + prevx[k] = rowx[k]; + rowx[k] = nextx[k]; + nextx[k] = nnextx[k]; + } + } + } +#else + (void)srcBase; + (void)srcStride; + (void)dstBase; + (void)dstStride; + (void)borderValue; +#endif +} + +bool isBlurF32Supported(const Size2D &size, s32 cn, BORDER_MODE border) +{ + return isSupportedConfiguration() && + cn > 0 && cn <= 4 && + size.width*cn >= 4 && size.height >= 2 && + (border == BORDER_MODE_CONSTANT || + border == BORDER_MODE_REFLECT101 || + border == BORDER_MODE_REFLECT || + border == BORDER_MODE_REPLICATE || + border == BORDER_MODE_WRAP); +} + +void blur3x3(const Size2D &size, s32 cn, + const f32 * srcBase, ptrdiff_t srcStride, + f32 * dstBase, ptrdiff_t dstStride, + BORDER_MODE borderType, f32 borderValue, Margin borderMargin) +{ + internal::assertSupportedConfiguration(isBlurF32Supported(size, cn, borderType)); +#ifdef CAROTENE_NEON + size_t colsn = size.width * cn; + + std::vector _tmp; + f32 *tmp = 0; + if (borderType == BORDER_MODE_CONSTANT) + { + _tmp.assign(colsn + 2*cn, borderValue); + tmp = &_tmp[cn]; + } + + ptrdiff_t idx_l = internal::borderInterpolate(-1, size.width, borderType, borderMargin.left, borderMargin.right) * cn; + ptrdiff_t idx_r = internal::borderInterpolate(size.width, size.width, borderType, borderMargin.left, borderMargin.right) * cn; + + //2-line buffer + std::vector _buf(4*(cn * (size.width + 2) + 32 / sizeof(f32))); + f32* lanea = internal::alignPtr(&_buf[cn], 32); + f32* laneA = internal::alignPtr(lanea + cn * (size.width + 2), 32); + + f32* laneb = internal::alignPtr(laneA + cn * (size.width + 2), 32); + f32* laneB = internal::alignPtr(laneb + cn * (size.width + 2), 32); + + if (borderType == BORDER_MODE_CONSTANT) + for (s32 k = 0; k < cn; ++k) + { + lanea[-cn+k] = borderValue; + lanea[colsn+k] = borderValue; + laneA[-cn+k] = borderValue; + laneA[colsn+k] = borderValue; + laneb[-cn+k] = borderValue; + laneb[colsn+k] = borderValue; + laneB[-cn+k] = borderValue; + laneB[colsn+k] = borderValue; + } + + size_t i = 0; + f32* dsta = internal::getRowPtr(dstBase, dstStride, 0); + for (; i < size.height-1; i+=2) + { + //vertical convolution + ptrdiff_t idx_rm1 = internal::borderInterpolate(i - 1, size.height, borderType, borderMargin.top, borderMargin.bottom); + ptrdiff_t idx_rp2 = internal::borderInterpolate(i + 2, size.height, borderType, borderMargin.top, borderMargin.bottom); + + const f32* ln0 = idx_rm1 >= -(ptrdiff_t)borderMargin.top ? internal::getRowPtr(srcBase, srcStride, idx_rm1) : tmp; + const f32* ln1 = internal::getRowPtr(srcBase, srcStride, i); + const f32* ln2 = internal::getRowPtr(srcBase, srcStride, i + 1); + const f32* ln3 = idx_rp2 >= -(ptrdiff_t)borderMargin.top ? internal::getRowPtr(srcBase, srcStride, idx_rp2) : tmp; + + size_t x = 0; + for (; x <= colsn - 4; x += 4) + { + internal::prefetch(ln1 + x); + internal::prefetch(ln2 + x); + internal::prefetch(ln0 + x); + internal::prefetch(ln3 + x); +box3x3f32_vert: + float32x4_t v1 = vld1q_f32(ln1 + x); + float32x4_t v2 = vld1q_f32(ln2 + x); + float32x4_t v0 = vld1q_f32(ln0 + x); + float32x4_t v3 = vld1q_f32(ln3 + x); + + float32x4_t v = vaddq_f32(v1, v2); + float32x4_t w0 = vaddq_f32(v, v0); + float32x4_t w1 = vaddq_f32(v, v3); + + vst1q_f32(lanea + x, w0); + vst1q_f32(laneb + x, w1); + } + if(x < colsn) + { + x = colsn-4; + goto box3x3f32_vert; + } + + //left&right borders + if (borderType != BORDER_MODE_CONSTANT) + for (s32 k = 0; k < cn; ++k) + { + lanea[-cn+k] = lanea[idx_l + k]; + lanea[colsn+k] = lanea[idx_r + k]; + laneb[-cn+k] = laneb[idx_l + k]; + laneb[colsn+k] = laneb[idx_r + k]; + } + + //horizontal convolution (2 lines from previous iteration) + if (i > 0) + { + f32* dstb = internal::getRowPtr(dstBase, dstStride, i-1); + x = 0; + for (; x <= colsn - 4; x += 4) + { + internal::prefetch(laneA + x + cn); + internal::prefetch(laneB + x + cn); +box3x3f32_horiz: + float32x4_t lane0a = vld1q_f32(laneA + x - cn); + float32x4_t lane2a = vld1q_f32(laneA + x + cn); + float32x4_t lane1a = vld1q_f32(laneA + x); + + float32x4_t lane0b = vld1q_f32(laneB + x - cn); + float32x4_t lane2b = vld1q_f32(laneB + x + cn); + float32x4_t lane1b = vld1q_f32(laneB + x); + + float32x4_t va = vaddq_f32(lane0a, lane2a); + float32x4_t vb = vaddq_f32(lane0b, lane2b); + float32x4_t wa = vaddq_f32(va, lane1a); + float32x4_t wb = vaddq_f32(vb, lane1b); + + vst1q_f32(dsta + x, wa); + vst1q_f32(dstb + x, wb); + } + if(x < colsn) + { + x = colsn-4; + goto box3x3f32_horiz; + } + dsta = internal::getRowPtr(dstBase, dstStride, i); + } + + std::swap(lanea, laneA); + std::swap(laneb, laneB); + } + + //last line + if(i < size.height) + { + //vertical convolution + ptrdiff_t idx_rm1 = internal::borderInterpolate(i - 1, size.height, borderType, borderMargin.top, borderMargin.bottom); + ptrdiff_t idx_rp1 = internal::borderInterpolate(i + 1, size.height, borderType, borderMargin.top, borderMargin.bottom); + + const f32* ln0 = idx_rm1 >= -(ptrdiff_t)borderMargin.top ? internal::getRowPtr(srcBase, srcStride, idx_rm1) : tmp; + const f32* ln1 = internal::getRowPtr(srcBase, srcStride, i); + const f32* ln2 = idx_rp1 >= -(ptrdiff_t)borderMargin.top ? internal::getRowPtr(srcBase, srcStride, idx_rp1) : tmp; + + size_t x = 0; + for (; x <= colsn - 4; x += 4) + { + internal::prefetch(ln0 + x); + internal::prefetch(ln1 + x); + internal::prefetch(ln2 + x); +box3x3f32_vert_ll: + float32x4_t v0 = vld1q_f32(ln0+x); + float32x4_t v1 = vld1q_f32(ln1+x); + float32x4_t v2 = vld1q_f32(ln2+x); + + float32x4_t v = vaddq_f32(v0, v1); + float32x4_t w = vaddq_f32(v, v2); + + vst1q_f32(lanea + x, w); + } + if(x < colsn) + { + x = colsn-4; + goto box3x3f32_vert_ll; + } + + //left&right borders + if (borderType != BORDER_MODE_CONSTANT) + for (s32 k = 0; k < cn; ++k) + { + lanea[-cn+k] = lanea[idx_l + k]; + lanea[colsn+k] = lanea[idx_r + k]; + } + + //horizontal convolution (last 3 lines) + x = 0; + f32* dstb = internal::getRowPtr(dstBase, dstStride, i-1); + f32* dstc = internal::getRowPtr(dstBase, dstStride, i); + for (; x <= colsn - 4; x += 4) + { + internal::prefetch(laneA + x + cn); + internal::prefetch(laneB + x + cn); + internal::prefetch(lanea + x + cn); +box3x3f32_horiz_ll: + float32x4_t lane0a = vld1q_f32(laneA + x - cn); + float32x4_t lane2a = vld1q_f32(laneA + x + cn); + float32x4_t lane1a = vld1q_f32(laneA + x); + + float32x4_t lane0b = vld1q_f32(laneB + x - cn); + float32x4_t lane2b = vld1q_f32(laneB + x + cn); + float32x4_t lane1b = vld1q_f32(laneB + x); + + float32x4_t lane0c = vld1q_f32(lanea + x - cn); + float32x4_t lane2c = vld1q_f32(lanea + x + cn); + float32x4_t lane1c = vld1q_f32(lanea + x); + + float32x4_t va = vaddq_f32(lane0a, lane2a); + float32x4_t vb = vaddq_f32(lane0b, lane2b); + float32x4_t vc = vaddq_f32(lane0c, lane2c); + float32x4_t wa = vaddq_f32(va, lane1a); + float32x4_t wb = vaddq_f32(vb, lane1b); + float32x4_t wc = vaddq_f32(vc, lane1c); + + vst1q_f32(dsta + x, wa); + vst1q_f32(dstb + x, wb); + vst1q_f32(dstc + x, wc); + } + if(x < colsn) + { + x = colsn-4; + goto box3x3f32_horiz_ll; + } + } + else + { + //horizontal convolution (last 2 lines) + f32* dstb = internal::getRowPtr(dstBase, dstStride, i-1); + size_t x = 0; + for (; x <= colsn - 4; x += 4) + { + internal::prefetch(laneA + x + cn); + internal::prefetch(laneB + x + cn); +box3x3f32_horiz_last2: + float32x4_t lane0a = vld1q_f32(laneA + x - cn); + float32x4_t lane2a = vld1q_f32(laneA + x + cn); + float32x4_t lane1a = vld1q_f32(laneA + x); + + float32x4_t lane0b = vld1q_f32(laneB + x - cn); + float32x4_t lane2b = vld1q_f32(laneB + x + cn); + float32x4_t lane1b = vld1q_f32(laneB + x); + + float32x4_t va = vaddq_f32(lane0a, lane2a); + float32x4_t vb = vaddq_f32(lane0b, lane2b); + float32x4_t wa = vaddq_f32(va, lane1a); + float32x4_t wb = vaddq_f32(vb, lane1b); + + vst1q_f32(dsta + x, wa); + vst1q_f32(dstb + x, wb); + } + if(x < colsn) + { + x = colsn-4; + goto box3x3f32_horiz_last2; + } + } +#else + (void)srcBase; + (void)srcStride; + (void)dstBase; + (void)dstStride; + (void)borderValue; + (void)borderMargin; +#endif +} + +bool isBlurS32Supported(const Size2D &size, s32 cn, BORDER_MODE border) +{ + return isSupportedConfiguration() && + cn > 0 && cn <= 4 && + size.width*cn >= 4 && size.height >= 2 && + (border == BORDER_MODE_CONSTANT || + border == BORDER_MODE_REFLECT101 || + border == BORDER_MODE_REFLECT || + border == BORDER_MODE_REPLICATE || + border == BORDER_MODE_WRAP); +} + +void blur3x3(const Size2D &size, s32 cn, + const s32 * srcBase, ptrdiff_t srcStride, + s32 * dstBase, ptrdiff_t dstStride, + BORDER_MODE borderType, s32 borderValue, Margin borderMargin) +{ + internal::assertSupportedConfiguration(isBlurS32Supported(size, cn, borderType)); +#ifdef CAROTENE_NEON + size_t colsn = size.width * cn; + + std::vector _tmp; + s32 *tmp = 0; + if (borderType == BORDER_MODE_CONSTANT) + { + _tmp.assign(colsn + 2*cn, borderValue); + tmp = &_tmp[cn]; + } + + ptrdiff_t idx_l = internal::borderInterpolate(-1, size.width, borderType, borderMargin.left, borderMargin.right) * cn; + ptrdiff_t idx_r = internal::borderInterpolate(size.width, size.width, borderType, borderMargin.left, borderMargin.right) * cn; + + //2-line buffer + std::vector _buf(4*(cn * (size.width + 2) + 32 / sizeof(s32))); + s32* lanea = internal::alignPtr(&_buf[cn], 32); + s32* laneA = internal::alignPtr(lanea + cn * (size.width + 2), 32); + + s32* laneb = internal::alignPtr(laneA + cn * (size.width + 2), 32); + s32* laneB = internal::alignPtr(laneb + cn * (size.width + 2), 32); + + if (borderType == BORDER_MODE_CONSTANT) + for (s32 k = 0; k < cn; ++k) + { + lanea[-cn+k] = borderValue; + lanea[colsn+k] = borderValue; + laneA[-cn+k] = borderValue; + laneA[colsn+k] = borderValue; + laneb[-cn+k] = borderValue; + laneb[colsn+k] = borderValue; + laneB[-cn+k] = borderValue; + laneB[colsn+k] = borderValue; + } + + size_t i = 0; + s32* dsta = internal::getRowPtr(dstBase, dstStride, 0); + for (; i < size.height-1; i+=2) + { + //vertical convolution + ptrdiff_t idx_rm1 = internal::borderInterpolate(i - 1, size.height, borderType, borderMargin.top, borderMargin.bottom); + ptrdiff_t idx_rp2 = internal::borderInterpolate(i + 2, size.height, borderType, borderMargin.top, borderMargin.bottom); + + const s32* ln0 = idx_rm1 >= -(ptrdiff_t)borderMargin.top ? internal::getRowPtr(srcBase, srcStride, idx_rm1) : tmp; + const s32* ln1 = internal::getRowPtr(srcBase, srcStride, i); + const s32* ln2 = internal::getRowPtr(srcBase, srcStride, i + 1); + const s32* ln3 = idx_rp2 >= -(ptrdiff_t)borderMargin.top ? internal::getRowPtr(srcBase, srcStride, idx_rp2) : tmp; + + size_t x = 0; + for (; x <= colsn - 4; x += 4) + { + internal::prefetch(ln1 + x); + internal::prefetch(ln2 + x); + internal::prefetch(ln0 + x); + internal::prefetch(ln3 + x); +box3x3s32_vert: + int32x4_t v1 = vld1q_s32(ln1 + x); + int32x4_t v2 = vld1q_s32(ln2 + x); + int32x4_t v0 = vld1q_s32(ln0 + x); + int32x4_t v3 = vld1q_s32(ln3 + x); + + int32x4_t v = vaddq_s32(v1, v2); + int32x4_t w0 = vaddq_s32(v, v0); + int32x4_t w1 = vaddq_s32(v, v3); + + vst1q_s32(lanea + x, w0); + vst1q_s32(laneb + x, w1); + } + if(x < colsn) + { + x = colsn-4; + goto box3x3s32_vert; + } + + //left&right borders + if (borderType != BORDER_MODE_CONSTANT) + for (s32 k = 0; k < cn; ++k) + { + lanea[-cn+k] = lanea[idx_l + k]; + lanea[colsn+k] = lanea[idx_r + k]; + laneb[-cn+k] = laneb[idx_l + k]; + laneb[colsn+k] = laneb[idx_r + k]; + } + + //horizontal convolution (2 lines from previous iteration) + if (i > 0) + { + s32* dstb = internal::getRowPtr(dstBase, dstStride, i-1); + x = 0; + for (; x <= colsn - 4; x += 4) + { + internal::prefetch(laneA + x + cn); + internal::prefetch(laneB + x + cn); +box3x3s32_horiz: + int32x4_t lane0a = vld1q_s32(laneA + x - cn); + int32x4_t lane2a = vld1q_s32(laneA + x + cn); + int32x4_t lane1a = vld1q_s32(laneA + x); + + int32x4_t lane0b = vld1q_s32(laneB + x - cn); + int32x4_t lane2b = vld1q_s32(laneB + x + cn); + int32x4_t lane1b = vld1q_s32(laneB + x); + + int32x4_t va = vaddq_s32(lane0a, lane2a); + int32x4_t vb = vaddq_s32(lane0b, lane2b); + int32x4_t wa = vaddq_s32(va, lane1a); + int32x4_t wb = vaddq_s32(vb, lane1b); + + vst1q_s32(dsta + x, wa); + vst1q_s32(dstb + x, wb); + } + if(x < colsn) + { + x = colsn-4; + goto box3x3s32_horiz; + } + dsta = internal::getRowPtr(dstBase, dstStride, i); + } + + std::swap(lanea, laneA); + std::swap(laneb, laneB); + } + //last line + if(i < size.height) + { + //vertical convolution + ptrdiff_t idx_rm1 = internal::borderInterpolate(i - 1, size.height, borderType, borderMargin.top, borderMargin.bottom); + ptrdiff_t idx_rp1 = internal::borderInterpolate(i + 1, size.height, borderType, borderMargin.top, borderMargin.bottom); + + const s32* ln0 = idx_rm1 >= -(ptrdiff_t)borderMargin.top ? internal::getRowPtr(srcBase, srcStride, idx_rm1) : tmp; + const s32* ln1 = internal::getRowPtr(srcBase, srcStride, i); + const s32* ln2 = idx_rp1 >= -(ptrdiff_t)borderMargin.top ? internal::getRowPtr(srcBase, srcStride, idx_rp1) : tmp; + + size_t x = 0; + for (; x <= colsn - 4; x += 4) + { + internal::prefetch(ln0 + x); + internal::prefetch(ln1 + x); + internal::prefetch(ln2 + x); +box3x3s32_vert_ll: + int32x4_t v0 = vld1q_s32(ln0+x); + int32x4_t v1 = vld1q_s32(ln1+x); + int32x4_t v2 = vld1q_s32(ln2+x); + + int32x4_t v = vaddq_s32(v0, v1); + int32x4_t w = vaddq_s32(v, v2); + + vst1q_s32(lanea + x, w); + } + if(x < colsn) + { + x = colsn-4; + goto box3x3s32_vert_ll; + } + + //left&right borders + if (borderType != BORDER_MODE_CONSTANT) + for (s32 k = 0; k < cn; ++k) + { + lanea[-cn+k] = lanea[idx_l + k]; + lanea[colsn+k] = lanea[idx_r + k]; + } + + //horizontal convolution (last 3 lines) + x = 0; + s32* dstb = internal::getRowPtr(dstBase, dstStride, i-1); + s32* dstc = internal::getRowPtr(dstBase, dstStride, i); + for (; x <= colsn - 4; x += 4) + { + internal::prefetch(laneA + x + cn); + internal::prefetch(laneB + x + cn); + internal::prefetch(lanea + x + cn); +box3x3s32_horiz_ll: + int32x4_t lane0a = vld1q_s32(laneA + x - cn); + int32x4_t lane2a = vld1q_s32(laneA + x + cn); + int32x4_t lane1a = vld1q_s32(laneA + x); + + int32x4_t lane0b = vld1q_s32(laneB + x - cn); + int32x4_t lane2b = vld1q_s32(laneB + x + cn); + int32x4_t lane1b = vld1q_s32(laneB + x); + + int32x4_t lane0c = vld1q_s32(lanea + x - cn); + int32x4_t lane2c = vld1q_s32(lanea + x + cn); + int32x4_t lane1c = vld1q_s32(lanea + x); + + int32x4_t va = vaddq_s32(lane0a, lane2a); + int32x4_t vb = vaddq_s32(lane0b, lane2b); + int32x4_t vc = vaddq_s32(lane0c, lane2c); + int32x4_t wa = vaddq_s32(va, lane1a); + int32x4_t wb = vaddq_s32(vb, lane1b); + int32x4_t wc = vaddq_s32(vc, lane1c); + + vst1q_s32(dsta + x, wa); + vst1q_s32(dstb + x, wb); + vst1q_s32(dstc + x, wc); + } + if(x < colsn) + { + x = colsn-4; + goto box3x3s32_horiz_ll; + } + } + else + { + //horizontal convolution (last 2 lines) + s32* dstb = internal::getRowPtr(dstBase, dstStride, i-1); + size_t x = 0; + for (; x <= colsn - 4; x += 4) + { + internal::prefetch(laneA + x + cn); + internal::prefetch(laneB + x + cn); +box3x3s32_horiz_last2: + int32x4_t lane0a = vld1q_s32(laneA + x - cn); + int32x4_t lane2a = vld1q_s32(laneA + x + cn); + int32x4_t lane1a = vld1q_s32(laneA + x); + + int32x4_t lane0b = vld1q_s32(laneB + x - cn); + int32x4_t lane2b = vld1q_s32(laneB + x + cn); + int32x4_t lane1b = vld1q_s32(laneB + x); + + int32x4_t va = vaddq_s32(lane0a, lane2a); + int32x4_t vb = vaddq_s32(lane0b, lane2b); + int32x4_t wa = vaddq_s32(va, lane1a); + int32x4_t wb = vaddq_s32(vb, lane1b); + + vst1q_s32(dsta + x, wa); + vst1q_s32(dstb + x, wb); + } + if(x < colsn) + { + x = colsn-4; + goto box3x3s32_horiz_last2; + } + } +#else + (void)srcBase; + (void)srcStride; + (void)dstBase; + (void)dstStride; + (void)borderValue; + (void)borderMargin; +#endif +} + +} //namespace CAROTENE_NS diff --git a/3rdparty/carotene/src/canny.cpp b/3rdparty/carotene/src/canny.cpp new file mode 100644 index 0000000000..f61bc23e9b --- /dev/null +++ b/3rdparty/carotene/src/canny.cpp @@ -0,0 +1,773 @@ +/* + * By downloading, copying, installing or using the software you agree to this license. + * If you do not agree to this license, do not download, install, + * copy or use the software. + * + * + * License Agreement + * For Open Source Computer Vision Library + * (3-clause BSD License) + * + * Copyright (C) 2012-2015, NVIDIA Corporation, all rights reserved. + * Third party copyrights are property of their respective owners. + * + * Redistribution and use in source and binary forms, with or without modification, + * are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * * Neither the names of the copyright holders nor the names of the contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * This software is provided by the copyright holders and contributors "as is" and + * any express or implied warranties, including, but not limited to, the implied + * warranties of merchantability and fitness for a particular purpose are disclaimed. + * In no event shall copyright holders or contributors be liable for any direct, + * indirect, incidental, special, exemplary, or consequential damages + * (including, but not limited to, procurement of substitute goods or services; + * loss of use, data, or profits; or business interruption) however caused + * and on any theory of liability, whether in contract, strict liability, + * or tort (including negligence or otherwise) arising in any way out of + * the use of this software, even if advised of the possibility of such damage. + */ + +#include "common.hpp" + +#include "saturate_cast.hpp" +#include +#include + +namespace CAROTENE_NS { + +#ifdef CAROTENE_NEON +namespace { +struct RowFilter3x3Canny +{ + inline RowFilter3x3Canny(const ptrdiff_t borderxl, const ptrdiff_t borderxr) + { + vfmask = vreinterpret_u8_u64(vmov_n_u64(borderxl ? 0x0000FFffFFffFFffULL : 0x0100FFffFFffFFffULL)); + vtmask = vreinterpret_u8_u64(vmov_n_u64(borderxr ? 0x0707060504030201ULL : 0x0706050403020100ULL)); + lookLeft = offsetk - borderxl; + lookRight = offsetk - borderxr; + } + + inline void operator()(const u8* src, s16* dstx, s16* dsty, ptrdiff_t width) + { + uint8x8_t l = vtbl1_u8(vld1_u8(src - lookLeft), vfmask); + ptrdiff_t i = 0; + for (; i < width - 8 + lookRight; i += 8) + { + internal::prefetch(src + i); + uint8x8_t l18u = vld1_u8(src + i + 1); + + uint8x8_t l2 = l18u; + uint8x8_t l0 = vext_u8(l, l18u, 6); + int16x8_t l1x2 = vreinterpretq_s16_u16(vshll_n_u8(vext_u8(l, l18u, 7), 1)); + + l = l18u; + + int16x8_t l02 = vreinterpretq_s16_u16(vaddl_u8(l2, l0)); + int16x8_t ldx = vreinterpretq_s16_u16(vsubl_u8(l2, l0)); + int16x8_t ldy = vaddq_s16(l02, l1x2); + + vst1q_s16(dstx + i, ldx); + vst1q_s16(dsty + i, ldy); + } + + //tail + if (lookRight == 0 || i != width) + { + uint8x8_t tail0 = vld1_u8(src + (width - 9));//can't get left 1 pixel another way if width==8*k+1 + uint8x8_t tail2 = vtbl1_u8(vld1_u8(src + (width - 8 + lookRight)), vtmask); + uint8x8_t tail1 = vext_u8(vreinterpret_u8_u64(vshl_n_u64(vreinterpret_u64_u8(tail0), 8*6)), tail2, 7); + + int16x8_t tail02 = vreinterpretq_s16_u16(vaddl_u8(tail2, tail0)); + int16x8_t tail1x2 = vreinterpretq_s16_u16(vshll_n_u8(tail1, 1)); + int16x8_t taildx = vreinterpretq_s16_u16(vsubl_u8(tail2, tail0)); + int16x8_t taildy = vqaddq_s16(tail02, tail1x2); + + vst1q_s16(dstx + (width - 8), taildx); + vst1q_s16(dsty + (width - 8), taildy); + } + } + + uint8x8_t vfmask; + uint8x8_t vtmask; + enum { offsetk = 1}; + ptrdiff_t lookLeft; + ptrdiff_t lookRight; +}; + +template +inline void ColFilter3x3Canny(const s16* src0, const s16* src1, const s16* src2, s16* dstx, s16* dsty, s32* mag, ptrdiff_t width) +{ + ptrdiff_t j = 0; + for (; j <= width - 8; j += 8) + { + ColFilter3x3CannyL1Loop: + int16x8_t line0x = vld1q_s16(src0 + j); + int16x8_t line1x = vld1q_s16(src1 + j); + int16x8_t line2x = vld1q_s16(src2 + j); + int16x8_t line0y = vld1q_s16(src0 + j + width); + int16x8_t line2y = vld1q_s16(src2 + j + width); + + int16x8_t l02 = vaddq_s16(line0x, line2x); + int16x8_t l1x2 = vshlq_n_s16(line1x, 1); + int16x8_t dy = vsubq_s16(line2y, line0y); + int16x8_t dx = vaddq_s16(l1x2, l02); + + int16x8_t dya = vabsq_s16(dy); + int16x8_t dxa = vabsq_s16(dx); + int16x8_t norm = vaddq_s16(dya, dxa); + + int32x4_t normh = vmovl_s16(vget_high_s16(norm)); + int32x4_t norml = vmovl_s16(vget_low_s16(norm)); + + vst1q_s16(dsty + j, dy); + vst1q_s16(dstx + j, dx); + vst1q_s32(mag + j + 4, normh); + vst1q_s32(mag + j, norml); + } + if (j != width) + { + j = width - 8; + goto ColFilter3x3CannyL1Loop; + } +} +template <> +inline void ColFilter3x3Canny(const s16* src0, const s16* src1, const s16* src2, s16* dstx, s16* dsty, s32* mag, ptrdiff_t width) +{ + ptrdiff_t j = 0; + for (; j <= width - 8; j += 8) + { + ColFilter3x3CannyL2Loop: + int16x8_t line0x = vld1q_s16(src0 + j); + int16x8_t line1x = vld1q_s16(src1 + j); + int16x8_t line2x = vld1q_s16(src2 + j); + int16x8_t line0y = vld1q_s16(src0 + j + width); + int16x8_t line2y = vld1q_s16(src2 + j + width); + + int16x8_t l02 = vaddq_s16(line0x, line2x); + int16x8_t l1x2 = vshlq_n_s16(line1x, 1); + int16x8_t dy = vsubq_s16(line2y, line0y); + int16x8_t dx = vaddq_s16(l1x2, l02); + + int32x4_t norml = vmull_s16(vget_low_s16(dx), vget_low_s16(dx)); + int32x4_t normh = vmull_s16(vget_high_s16(dy), vget_high_s16(dy)); + + norml = vmlal_s16(norml, vget_low_s16(dy), vget_low_s16(dy)); + normh = vmlal_s16(normh, vget_high_s16(dx), vget_high_s16(dx)); + + vst1q_s16(dsty + j, dy); + vst1q_s16(dstx + j, dx); + vst1q_s32(mag + j, norml); + vst1q_s32(mag + j + 4, normh); + } + if (j != width) + { + j = width - 8; + goto ColFilter3x3CannyL2Loop; + } +} + +template +inline void NormCanny(const ptrdiff_t colscn, s16* _dx, s16* _dy, s32* _norm) +{ + ptrdiff_t j = 0; + if (colscn >= 8) + { + int16x8_t vx = vld1q_s16(_dx); + int16x8_t vy = vld1q_s16(_dy); + for (; j <= colscn - 16; j+=8) + { + internal::prefetch(_dx); + internal::prefetch(_dy); + + int16x8_t vx2 = vld1q_s16(_dx + j + 8); + int16x8_t vy2 = vld1q_s16(_dy + j + 8); + + int16x8_t vabsx = vabsq_s16(vx); + int16x8_t vabsy = vabsq_s16(vy); + + int16x8_t norm = vaddq_s16(vabsx, vabsy); + + int32x4_t normh = vmovl_s16(vget_high_s16(norm)); + int32x4_t norml = vmovl_s16(vget_low_s16(norm)); + + vst1q_s32(_norm + j + 4, normh); + vst1q_s32(_norm + j + 0, norml); + + vx = vx2; + vy = vy2; + } + int16x8_t vabsx = vabsq_s16(vx); + int16x8_t vabsy = vabsq_s16(vy); + + int16x8_t norm = vaddq_s16(vabsx, vabsy); + + int32x4_t normh = vmovl_s16(vget_high_s16(norm)); + int32x4_t norml = vmovl_s16(vget_low_s16(norm)); + + vst1q_s32(_norm + j + 4, normh); + vst1q_s32(_norm + j + 0, norml); + } + for (; j < colscn; j++) + _norm[j] = std::abs(s32(_dx[j])) + std::abs(s32(_dy[j])); +} + +template <> +inline void NormCanny(const ptrdiff_t colscn, s16* _dx, s16* _dy, s32* _norm) +{ + ptrdiff_t j = 0; + if (colscn >= 8) + { + int16x8_t vx = vld1q_s16(_dx); + int16x8_t vy = vld1q_s16(_dy); + + for (; j <= colscn - 16; j+=8) + { + internal::prefetch(_dx); + internal::prefetch(_dy); + + int16x8_t vxnext = vld1q_s16(_dx + j + 8); + int16x8_t vynext = vld1q_s16(_dy + j + 8); + + int32x4_t norml = vmull_s16(vget_low_s16(vx), vget_low_s16(vx)); + int32x4_t normh = vmull_s16(vget_high_s16(vy), vget_high_s16(vy)); + + norml = vmlal_s16(norml, vget_low_s16(vy), vget_low_s16(vy)); + normh = vmlal_s16(normh, vget_high_s16(vx), vget_high_s16(vx)); + + vst1q_s32(_norm + j + 0, norml); + vst1q_s32(_norm + j + 4, normh); + + vx = vxnext; + vy = vynext; + } + int32x4_t norml = vmull_s16(vget_low_s16(vx), vget_low_s16(vx)); + int32x4_t normh = vmull_s16(vget_high_s16(vy), vget_high_s16(vy)); + + norml = vmlal_s16(norml, vget_low_s16(vy), vget_low_s16(vy)); + normh = vmlal_s16(normh, vget_high_s16(vx), vget_high_s16(vx)); + + vst1q_s32(_norm + j + 0, norml); + vst1q_s32(_norm + j + 4, normh); + } + for (; j < colscn; j++) + _norm[j] = s32(_dx[j])*_dx[j] + s32(_dy[j])*_dy[j]; +} + +template +inline void prepareThresh(f64 low_thresh, f64 high_thresh, + s32 &low, s32 &high) +{ + if (low_thresh > high_thresh) + std::swap(low_thresh, high_thresh); +#if defined __GNUC__ + low = (s32)low_thresh; + high = (s32)high_thresh; + low -= (low > low_thresh); + high -= (high > high_thresh); +#else + low = internal::round(low_thresh); + high = internal::round(high_thresh); + f32 ldiff = (f32)(low_thresh - low); + f32 hdiff = (f32)(high_thresh - high); + low -= (ldiff < 0); + high -= (hdiff < 0); +#endif +} +template <> +inline void prepareThresh(f64 low_thresh, f64 high_thresh, + s32 &low, s32 &high) +{ + if (low_thresh > high_thresh) + std::swap(low_thresh, high_thresh); + if (low_thresh > 0) low_thresh *= low_thresh; + if (high_thresh > 0) high_thresh *= high_thresh; +#if defined __GNUC__ + low = (s32)low_thresh; + high = (s32)high_thresh; + low -= (low > low_thresh); + high -= (high > high_thresh); +#else + low = internal::round(low_thresh); + high = internal::round(high_thresh); + f32 ldiff = (f32)(low_thresh - low); + f32 hdiff = (f32)(high_thresh - high); + low -= (ldiff < 0); + high -= (hdiff < 0); +#endif +} + +template +struct _normEstimator +{ + ptrdiff_t magstep; + ptrdiff_t dxOffset; + ptrdiff_t dyOffset; + ptrdiff_t shxOffset; + ptrdiff_t shyOffset; + std::vector buffer; + const ptrdiff_t offsetk; + ptrdiff_t borderyt, borderyb; + RowFilter3x3Canny sobelRow; + + inline _normEstimator(const Size2D &size, s32, Margin borderMargin, + ptrdiff_t &mapstep, s32** mag_buf, u8* &map): + offsetk(1), + sobelRow(std::max(0, offsetk - (ptrdiff_t)borderMargin.left), + std::max(0, offsetk - (ptrdiff_t)borderMargin.right)) + { + mapstep = size.width + 2; + magstep = size.width + 2 + size.width * (4 * sizeof(s16)/sizeof(s32)); + dxOffset = mapstep * sizeof(s32)/sizeof(s16); + dyOffset = dxOffset + size.width * 1; + shxOffset = dxOffset + size.width * 2; + shyOffset = dxOffset + size.width * 3; + buffer.resize( (size.width+2)*(size.height+2) + magstep*3*sizeof(s32) ); + mag_buf[0] = (s32*)&buffer[0]; + mag_buf[1] = mag_buf[0] + magstep; + mag_buf[2] = mag_buf[1] + magstep; + memset(mag_buf[0], 0, mapstep * sizeof(s32)); + + map = (u8*)(mag_buf[2] + magstep); + memset(map, 1, mapstep); + memset(map + mapstep*(size.height + 1), 1, mapstep); + borderyt = std::max(0, offsetk - (ptrdiff_t)borderMargin.top); + borderyb = std::max(0, offsetk - (ptrdiff_t)borderMargin.bottom); + } + inline void firstRow(const Size2D &size, s32, + const u8 *srcBase, ptrdiff_t srcStride, + s16*, ptrdiff_t, + s16*, ptrdiff_t, + s32** mag_buf) + { + //sobelH row #0 + const u8* _src = internal::getRowPtr(srcBase, srcStride, 0); + sobelRow(_src, ((s16*)mag_buf[0]) + shxOffset, ((s16*)mag_buf[0]) + shyOffset, size.width); + //sobelH row #1 + _src = internal::getRowPtr(srcBase, srcStride, 1); + sobelRow(_src, ((s16*)mag_buf[1]) + shxOffset, ((s16*)mag_buf[1]) + shyOffset, size.width); + + mag_buf[1][0] = mag_buf[1][size.width+1] = 0; + if (borderyt == 0) + { + //sobelH row #-1 + _src = internal::getRowPtr(srcBase, srcStride, -1); + sobelRow(_src, ((s16*)mag_buf[2]) + shxOffset, ((s16*)mag_buf[2]) + shyOffset, size.width); + + ColFilter3x3Canny( ((s16*)mag_buf[2]) + shxOffset, ((s16*)mag_buf[0]) + shxOffset, ((s16*)mag_buf[1]) + shxOffset, + ((s16*)mag_buf[1]) + dxOffset, ((s16*)mag_buf[1]) + dyOffset, mag_buf[1] + 1, size.width); + } + else + { + ColFilter3x3Canny( ((s16*)mag_buf[0]) + shxOffset, ((s16*)mag_buf[0]) + shxOffset, ((s16*)mag_buf[1]) + shxOffset, + ((s16*)mag_buf[1]) + dxOffset, ((s16*)mag_buf[1]) + dyOffset, mag_buf[1] + 1, size.width); + } + } + inline void nextRow(const Size2D &size, s32, + const u8 *srcBase, ptrdiff_t srcStride, + s16*, ptrdiff_t, + s16*, ptrdiff_t, + const ptrdiff_t &mapstep, s32** mag_buf, + size_t i, const s16* &_x, const s16* &_y) + { + mag_buf[2][0] = mag_buf[2][size.width+1] = 0; + if (i < size.height - borderyb) + { + const u8* _src = internal::getRowPtr(srcBase, srcStride, i+1); + //sobelH row #i+1 + sobelRow(_src, ((s16*)mag_buf[2]) + shxOffset, ((s16*)mag_buf[2]) + shyOffset, size.width); + + ColFilter3x3Canny( ((s16*)mag_buf[0]) + shxOffset, ((s16*)mag_buf[1]) + shxOffset, ((s16*)mag_buf[2]) + shxOffset, + ((s16*)mag_buf[2]) + dxOffset, ((s16*)mag_buf[2]) + dyOffset, mag_buf[2] + 1, size.width); + } + else if (i < size.height) + { + ColFilter3x3Canny( ((s16*)mag_buf[0]) + shxOffset, ((s16*)mag_buf[1]) + shxOffset, ((s16*)mag_buf[1]) + shxOffset, + ((s16*)mag_buf[2]) + dxOffset, ((s16*)mag_buf[2]) + dyOffset, mag_buf[2] + 1, size.width); + } + else + memset(mag_buf[2], 0, mapstep*sizeof(s32)); + _x = ((s16*)mag_buf[1]) + dxOffset; + _y = ((s16*)mag_buf[1]) + dyOffset; + } +}; +template +struct _normEstimator +{ + std::vector buffer; + + inline _normEstimator(const Size2D &size, s32 cn, Margin, + ptrdiff_t &mapstep, s32** mag_buf, u8* &map) + { + mapstep = size.width + 2; + buffer.resize( (size.width+2)*(size.height+2) + cn*mapstep*3*sizeof(s32) ); + mag_buf[0] = (s32*)&buffer[0]; + mag_buf[1] = mag_buf[0] + mapstep*cn; + mag_buf[2] = mag_buf[1] + mapstep*cn; + memset(mag_buf[0], 0, /* cn* */mapstep * sizeof(s32)); + + map = (u8*)(mag_buf[2] + mapstep*cn); + memset(map, 1, mapstep); + memset(map + mapstep*(size.height + 1), 1, mapstep); + } + inline void firstRow(const Size2D &size, s32 cn, + const u8 *, ptrdiff_t, + s16* dxBase, ptrdiff_t dxStride, + s16* dyBase, ptrdiff_t dyStride, + s32** mag_buf) + { + s32* _norm = mag_buf[1] + 1; + + s16* _dx = internal::getRowPtr(dxBase, dxStride, 0); + s16* _dy = internal::getRowPtr(dyBase, dyStride, 0); + + NormCanny(size.width*cn, _dx, _dy, _norm); + + if(cn > 1) + { + for(size_t j = 0, jn = 0; j < size.width; ++j, jn += cn) + { + size_t maxIdx = jn; + for(s32 k = 1; k < cn; ++k) + if(_norm[jn + k] > _norm[maxIdx]) maxIdx = jn + k; + _norm[j] = _norm[maxIdx]; + _dx[j] = _dx[maxIdx]; + _dy[j] = _dy[maxIdx]; + } + } + + _norm[-1] = _norm[size.width] = 0; + } + inline void nextRow(const Size2D &size, s32 cn, + const u8 *, ptrdiff_t, + s16* dxBase, ptrdiff_t dxStride, + s16* dyBase, ptrdiff_t dyStride, + const ptrdiff_t &mapstep, s32** mag_buf, + size_t i, const s16* &_x, const s16* &_y) + { + s32* _norm = mag_buf[(i > 0) + 1] + 1; + if (i < size.height) + { + s16* _dx = internal::getRowPtr(dxBase, dxStride, i); + s16* _dy = internal::getRowPtr(dyBase, dyStride, i); + + NormCanny(size.width*cn, _dx, _dy, _norm); + + if(cn > 1) + { + for(size_t j = 0, jn = 0; j < size.width; ++j, jn += cn) + { + size_t maxIdx = jn; + for(s32 k = 1; k < cn; ++k) + if(_norm[jn + k] > _norm[maxIdx]) maxIdx = jn + k; + _norm[j] = _norm[maxIdx]; + _dx[j] = _dx[maxIdx]; + _dy[j] = _dy[maxIdx]; + } + } + + _norm[-1] = _norm[size.width] = 0; + } + else + memset(_norm-1, 0, /* cn* */mapstep*sizeof(s32)); + + _x = internal::getRowPtr(dxBase, dxStride, i-1); + _y = internal::getRowPtr(dyBase, dyStride, i-1); + } +}; + +template +inline void Canny3x3(const Size2D &size, s32 cn, + const u8 * srcBase, ptrdiff_t srcStride, + u8 * dstBase, ptrdiff_t dstStride, + s16 * dxBase, ptrdiff_t dxStride, + s16 * dyBase, ptrdiff_t dyStride, + f64 low_thresh, f64 high_thresh, + Margin borderMargin) +{ + s32 low, high; + prepareThresh(low_thresh, high_thresh, low, high); + + ptrdiff_t mapstep; + s32* mag_buf[3]; + u8* map; + _normEstimator normEstimator(size, cn, borderMargin, mapstep, mag_buf, map); + + size_t maxsize = std::max( 1u << 10, size.width * size.height / 10 ); + std::vector stack( maxsize ); + u8 **stack_top = &stack[0]; + u8 **stack_bottom = &stack[0]; + + /* sector numbers + (Top-Left Origin) + + 1 2 3 + * * * + * * * + 0*******0 + * * * + * * * + 3 2 1 + */ + + #define CANNY_PUSH(d) *(d) = u8(2), *stack_top++ = (d) + #define CANNY_POP(d) (d) = *--stack_top + + //i == 0 + normEstimator.firstRow(size, cn, srcBase, srcStride, dxBase, dxStride, dyBase, dyStride, mag_buf); + // calculate magnitude and angle of gradient, perform non-maxima supression. + // fill the map with one of the following values: + // 0 - the pixel might belong to an edge + // 1 - the pixel can not belong to an edge + // 2 - the pixel does belong to an edge + for (size_t i = 1; i <= size.height; i++) + { + const s16 *_x, *_y; + normEstimator.nextRow(size, cn, srcBase, srcStride, dxBase, dxStride, dyBase, dyStride, mapstep, mag_buf, i, _x, _y); + + u8* _map = map + mapstep*i + 1; + _map[-1] = _map[size.width] = 1; + + s32* _mag = mag_buf[1] + 1; // take the central row + ptrdiff_t magstep1 = mag_buf[2] - mag_buf[1]; + ptrdiff_t magstep2 = mag_buf[0] - mag_buf[1]; + + if ((stack_top - stack_bottom) + size.width > maxsize) + { + ptrdiff_t sz = (ptrdiff_t)(stack_top - stack_bottom); + maxsize = maxsize * 3/2; + stack.resize(maxsize); + stack_bottom = &stack[0]; + stack_top = stack_bottom + sz; + } + + s32 prev_flag = 0; + for (ptrdiff_t j = 0; j < (ptrdiff_t)size.width; j++) + { + #define CANNY_SHIFT 15 + const s32 TG22 = (s32)(0.4142135623730950488016887242097*(1< low) + { + s32 xs = _x[j]; + s32 ys = _y[j]; + s32 x = abs(xs); + s32 y = abs(ys) << CANNY_SHIFT; + + s32 tg22x = x * TG22; + + if (y < tg22x) + { + if (m > _mag[j-1] && m >= _mag[j+1]) goto __push; + } + else + { + s32 tg67x = tg22x + (x << (CANNY_SHIFT+1)); + if (y > tg67x) + { + if (m > _mag[j+magstep2] && m >= _mag[j+magstep1]) goto __push; + } + else + { + s32 s = (xs ^ ys) < 0 ? -1 : 1; + if(m > _mag[j+magstep2-s] && m > _mag[j+magstep1+s]) goto __push; + } + } + } + prev_flag = 0; + _map[j] = u8(1); + continue; + __push: + if (!prev_flag && m > high && _map[j-mapstep] != 2) + { + CANNY_PUSH(_map + j); + prev_flag = 1; + } + else + _map[j] = 0; + } + + // scroll the ring buffer + _mag = mag_buf[0]; + mag_buf[0] = mag_buf[1]; + mag_buf[1] = mag_buf[2]; + mag_buf[2] = _mag; + } + + // now track the edges (hysteresis thresholding) + while (stack_top > stack_bottom) + { + u8* m; + if ((size_t)(stack_top - stack_bottom) + 8u > maxsize) + { + ptrdiff_t sz = (ptrdiff_t)(stack_top - stack_bottom); + maxsize = maxsize * 3/2; + stack.resize(maxsize); + stack_bottom = &stack[0]; + stack_top = stack_bottom + sz; + } + + CANNY_POP(m); + + if (!m[-1]) CANNY_PUSH(m - 1); + if (!m[1]) CANNY_PUSH(m + 1); + if (!m[-mapstep-1]) CANNY_PUSH(m - mapstep - 1); + if (!m[-mapstep]) CANNY_PUSH(m - mapstep); + if (!m[-mapstep+1]) CANNY_PUSH(m - mapstep + 1); + if (!m[mapstep-1]) CANNY_PUSH(m + mapstep - 1); + if (!m[mapstep]) CANNY_PUSH(m + mapstep); + if (!m[mapstep+1]) CANNY_PUSH(m + mapstep + 1); + } + + // the final pass, form the final image + uint8x16_t v2 = vmovq_n_u8(2); + const u8* ptrmap = map + mapstep + 1; + for (size_t i = 0; i < size.height; i++, ptrmap += mapstep) + { + u8* _dst = internal::getRowPtr(dstBase, dstStride, i); + ptrdiff_t j = 0; + for (; j < (ptrdiff_t)size.width - 16; j += 16) + { + internal::prefetch(ptrmap); + uint8x16_t vmap = vld1q_u8(ptrmap + j); + uint8x16_t vdst = vceqq_u8(vmap, v2); + vst1q_u8(_dst+j, vdst); + } + for (; j < (ptrdiff_t)size.width; j++) + _dst[j] = (u8)-(ptrmap[j] >> 1); + } +} + +} // namespace +#endif + +bool isCanny3x3Supported(const Size2D &size) +{ + return isSupportedConfiguration() && + size.height >= 2 && size.width >= 9; +} + +void Canny3x3L1(const Size2D &size, + const u8 * srcBase, ptrdiff_t srcStride, + u8 * dstBase, ptrdiff_t dstStride, + f64 low_thresh, f64 high_thresh, + Margin borderMargin) +{ + internal::assertSupportedConfiguration(isCanny3x3Supported(size)); +#ifdef CAROTENE_NEON + Canny3x3(size, 1, + srcBase, srcStride, + dstBase, dstStride, + NULL, 0, + NULL, 0, + low_thresh, high_thresh, + borderMargin); +#else + (void)size; + (void)srcBase; + (void)srcStride; + (void)dstBase; + (void)dstStride; + (void)low_thresh; + (void)high_thresh; + (void)borderMargin; +#endif +} + +void Canny3x3L2(const Size2D &size, + const u8 * srcBase, ptrdiff_t srcStride, + u8 * dstBase, ptrdiff_t dstStride, + f64 low_thresh, f64 high_thresh, + Margin borderMargin) +{ + internal::assertSupportedConfiguration(isCanny3x3Supported(size)); +#ifdef CAROTENE_NEON + Canny3x3(size, 1, + srcBase, srcStride, + dstBase, dstStride, + NULL, 0, + NULL, 0, + low_thresh, high_thresh, + borderMargin); +#else + (void)size; + (void)srcBase; + (void)srcStride; + (void)dstBase; + (void)dstStride; + (void)low_thresh; + (void)high_thresh; + (void)borderMargin; +#endif +} + +void Canny3x3L1(const Size2D &size, s32 cn, + s16 * dxBase, ptrdiff_t dxStride, + s16 * dyBase, ptrdiff_t dyStride, + u8 * dstBase, ptrdiff_t dstStride, + f64 low_thresh, f64 high_thresh) +{ + internal::assertSupportedConfiguration(); +#ifdef CAROTENE_NEON + Canny3x3(size, cn, + NULL, 0, + dstBase, dstStride, + dxBase, dxStride, + dyBase, dyStride, + low_thresh, high_thresh, + Margin()); +#else + (void)size; + (void)cn; + (void)dstBase; + (void)dstStride; + (void)dxBase; + (void)dxStride; + (void)dyBase; + (void)dyStride; + (void)low_thresh; + (void)high_thresh; +#endif +} + +void Canny3x3L2(const Size2D &size, s32 cn, + s16 * dxBase, ptrdiff_t dxStride, + s16 * dyBase, ptrdiff_t dyStride, + u8 * dstBase, ptrdiff_t dstStride, + f64 low_thresh, f64 high_thresh) +{ + internal::assertSupportedConfiguration(); +#ifdef CAROTENE_NEON + Canny3x3(size, cn, + NULL, 0, + dstBase, dstStride, + dxBase, dxStride, + dyBase, dyStride, + low_thresh, high_thresh, + Margin()); +#else + (void)size; + (void)cn; + (void)dstBase; + (void)dstStride; + (void)dxBase; + (void)dxStride; + (void)dyBase; + (void)dyStride; + (void)low_thresh; + (void)high_thresh; +#endif +} + +} // namespace CAROTENE_NS diff --git a/3rdparty/carotene/src/channel_extract.cpp b/3rdparty/carotene/src/channel_extract.cpp new file mode 100644 index 0000000000..fda8f6e153 --- /dev/null +++ b/3rdparty/carotene/src/channel_extract.cpp @@ -0,0 +1,486 @@ +/* + * By downloading, copying, installing or using the software you agree to this license. + * If you do not agree to this license, do not download, install, + * copy or use the software. + * + * + * License Agreement + * For Open Source Computer Vision Library + * (3-clause BSD License) + * + * Copyright (C) 2014-2015, NVIDIA Corporation, all rights reserved. + * Third party copyrights are property of their respective owners. + * + * Redistribution and use in source and binary forms, with or without modification, + * are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * * Neither the names of the copyright holders nor the names of the contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * This software is provided by the copyright holders and contributors "as is" and + * any express or implied warranties, including, but not limited to, the implied + * warranties of merchantability and fitness for a particular purpose are disclaimed. + * In no event shall copyright holders or contributors be liable for any direct, + * indirect, incidental, special, exemplary, or consequential damages + * (including, but not limited to, procurement of substitute goods or services; + * loss of use, data, or profits; or business interruption) however caused + * and on any theory of liability, whether in contract, strict liability, + * or tort (including negligence or otherwise) arising in any way out of + * the use of this software, even if advised of the possibility of such damage. + */ + +#include "common.hpp" +#include "vtransform.hpp" + +namespace CAROTENE_NS { + +void extract2(const Size2D &size, + const u8 * srcBase, ptrdiff_t srcStride, + u8 * dstBase, ptrdiff_t dstStride, + u32 coi) +{ + internal::assertSupportedConfiguration(); +#ifdef CAROTENE_NEON +#ifndef ANDROID + size_t roiw32 = size.width >= 31 ? size.width - 31 : 0; +#endif + size_t roiw8 = size.width >= 7 ? size.width - 7 : 0; + + for (size_t i = 0u; i < size.height; ++i) + { + const u8 * src = internal::getRowPtr(srcBase, srcStride, i); + u8 * dst = internal::getRowPtr(dstBase, dstStride, i); + size_t sj = 0u, dj = 0u; + +#ifndef ANDROID + for (; dj < roiw32; sj += 64, dj += 32) + { + internal::prefetch(src + sj); + + uint8x16x2_t v_src = vld2q_u8(src + sj); + vst1q_u8(dst + dj, v_src.val[coi]); + + v_src = vld2q_u8(src + sj + 32); + vst1q_u8(dst + dj + 16, v_src.val[coi]); + } +#endif + + for (; dj < roiw8; sj += 16, dj += 8) + { + uint8x8x2_t v_src = vld2_u8(src + sj); + vst1_u8(dst + dj, v_src.val[coi]); + } + + for (; dj < size.width; sj += 2, ++dj) + { + dst[dj] = src[sj + coi]; + } + } +#else + (void)size; + (void)srcBase; + (void)srcStride; + (void)dstBase; + (void)dstStride; + (void)coi; +#endif +} + +void extract3(const Size2D &size, + const u8 * srcBase, ptrdiff_t srcStride, + u8 * dstBase, ptrdiff_t dstStride, + u32 coi) +{ + internal::assertSupportedConfiguration(); +#ifdef CAROTENE_NEON +#ifndef ANDROID + size_t roiw32 = size.width >= 31 ? size.width - 31 : 0; +#endif + size_t roiw8 = size.width >= 7 ? size.width - 7 : 0; + + for (size_t i = 0u; i < size.height; ++i) + { + const u8 * src = internal::getRowPtr(srcBase, srcStride, i); + u8 * dst = internal::getRowPtr(dstBase, dstStride, i); + size_t sj = 0u, dj = 0u; + +#ifndef ANDROID + for (; dj < roiw32; sj += 96, dj += 32) + { + internal::prefetch(src + sj); + + uint8x16x3_t v_src = vld3q_u8(src + sj); + vst1q_u8(dst + dj, v_src.val[coi]); + + v_src = vld3q_u8(src + sj + 48); + vst1q_u8(dst + dj + 16, v_src.val[coi]); + } +#endif + + for (; dj < roiw8; sj += 24, dj += 8) + { + uint8x8x3_t v_src = vld3_u8(src + sj); + vst1_u8(dst + dj, v_src.val[coi]); + } + + for (; dj < size.width; sj += 3, ++dj) + { + dst[dj] = src[sj + coi]; + } + } +#else + (void)size; + (void)srcBase; + (void)srcStride; + (void)dstBase; + (void)dstStride; + (void)coi; +#endif +} + +void extract4(const Size2D &size, + const u8 * srcBase, ptrdiff_t srcStride, + u8 * dstBase, ptrdiff_t dstStride, + u32 coi) +{ + internal::assertSupportedConfiguration(); +#ifdef CAROTENE_NEON +#ifndef ANDROID + size_t roiw32 = size.width >= 31 ? size.width - 31 : 0; +#endif + size_t roiw8 = size.width >= 7 ? size.width - 7 : 0; + + for (size_t i = 0u; i < size.height; ++i) + { + const u8 * src = internal::getRowPtr(srcBase, srcStride, i); + u8 * dst = internal::getRowPtr(dstBase, dstStride, i); + size_t sj = 0u, dj = 0u; + +#ifndef ANDROID + for (; dj < roiw32; sj += 128, dj += 32) + { + internal::prefetch(src + sj); + + uint8x16x4_t v_src = vld4q_u8(src + sj); + vst1q_u8(dst + dj, v_src.val[coi]); + + v_src = vld4q_u8(src + sj + 64); + vst1q_u8(dst + dj + 16, v_src.val[coi]); + } +#endif + + for (; dj < roiw8; sj += 32, dj += 8) + { + uint8x8x4_t v_src = vld4_u8(src + sj); + vst1_u8(dst + dj, v_src.val[coi]); + } + + for (; dj < size.width; sj += 4, ++dj) + { + dst[dj] = src[sj + coi]; + } + } +#else + (void)size; + (void)srcBase; + (void)srcStride; + (void)dstBase; + (void)dstStride; + (void)coi; +#endif +} + +#define FILL_LINES2(macro,type) \ + macro##_LINE(type,0) \ + macro##_LINE(type,1) +#define FILL_LINES3(macro,type) \ + FILL_LINES2(macro,type) \ + macro##_LINE(type,2) +#define FILL_LINES4(macro,type) \ + FILL_LINES3(macro,type) \ + macro##_LINE(type,3) + +#define FARG_LINE(type, n) , type * dst##n##Base, ptrdiff_t dst##n##Stride + +#ifdef CAROTENE_NEON + +#define VROW_LINE(type, n) type * dst##n = internal::getRowPtr(dst##n##Base, dst##n##Stride, i); +#define VST1Q_LINE(type, n) vst1q_##type(dst##n + dj, v_src.val[n]); +#define VST1_LINE(type, n) vst1_##type(dst##n + dj, v_src.val[n]); +#define SST_LINE(type, n) dst##n[dj] = src[sj + n]; + +#define MUL2(val) (val << 1) +#define MUL3(val) (MUL2(val) + val) +#define MUL4(val) (val << 2) + +#define CONTDST2 srcStride == dst0Stride && \ + srcStride == dst1Stride && +#define CONTDST3 srcStride == dst0Stride && \ + srcStride == dst1Stride && \ + srcStride == dst2Stride && +#define CONTDST4 srcStride == dst0Stride && \ + srcStride == dst1Stride && \ + srcStride == dst2Stride && \ + srcStride == dst3Stride && + +#if __GNUC__ == 4 && __GNUC_MINOR__ < 7 + +#define SPLIT_ASM2(sgn, bits) __asm__ ( \ + "vld2." #bits " {d0, d2}, [%[in0]] \n\t" \ + "vld2." #bits " {d1, d3}, [%[in1]] \n\t" \ + "vst1." #bits " {d0-d1}, [%[out0]] \n\t" \ + "vst1." #bits " {d2-d3}, [%[out1]] \n\t" \ + : \ + : [out0] "r" (dst0 + dj), [out1] "r" (dst1 + dj), \ + [in0] "r" (src + sj), [in1] "r" (src + sj + MUL2(8)/sizeof(sgn##bits)) \ + : "d0","d1","d2","d3" \ + ); +#define SPLIT_ASM3(sgn, bits) __asm__ ( \ + "vld3." #bits " {d0, d2, d4}, [%[in0]] \n\t" \ + "vld3." #bits " {d1, d3, d5}, [%[in1]] \n\t" \ + "vst1." #bits " {d0-d1}, [%[out0]] \n\t" \ + "vst1." #bits " {d2-d3}, [%[out1]] \n\t" \ + "vst1." #bits " {d4-d5}, [%[out2]] \n\t" \ + : \ + : [out0] "r" (dst0 + dj), [out1] "r" (dst1 + dj), [out2] "r" (dst2 + dj), \ + [in0] "r" (src + sj), [in1] "r" (src + sj + MUL3(8)/sizeof(sgn##bits)) \ + : "d0","d1","d2","d3","d4","d5" \ + ); +#define SPLIT_ASM4(sgn, bits) __asm__ ( \ + "vld4." #bits " {d0, d2, d4, d6}, [%[in0]] \n\t" \ + "vld4." #bits " {d1, d3, d5, d7}, [%[in1]] \n\t" \ + "vst1." #bits " {d0-d1}, [%[out0]] \n\t" \ + "vst1." #bits " {d2-d3}, [%[out1]] \n\t" \ + "vst1." #bits " {d4-d5}, [%[out2]] \n\t" \ + "vst1." #bits " {d6-d7}, [%[out3]] \n\t" \ + : \ + : [out0] "r" (dst0 + dj), [out1] "r" (dst1 + dj), [out2] "r" (dst2 + dj), [out3] "r" (dst3 + dj), \ + [in0] "r" (src + sj), [in1] "r" (src + sj + MUL4(8)/sizeof(sgn##bits)) \ + : "d0","d1","d2","d3","d4","d5","d6","d7" \ + ); + +#define SPLIT_QUAD(sgn, bits, n) { \ + internal::prefetch(src + sj); \ + SPLIT_ASM##n(sgn, bits) \ + } + +#else + +#define SPLIT_QUAD(sgn, bits, n) { \ + internal::prefetch(src + sj); \ + vec128 v_src = vld##n##q_##sgn##bits(src + sj); \ + FILL_LINES##n(VST1Q, sgn##bits) \ + } + +#endif // __GNUC__ == 4 && __GNUC_MINOR__ < 7 + +#define SPLIT(sgn,bits,n) void split##n(const Size2D &_size, \ + const sgn##bits * srcBase, ptrdiff_t srcStride \ + FILL_LINES##n(FARG, sgn##bits) ) \ +{ \ + internal::assertSupportedConfiguration(); \ + Size2D size(_size); \ + if (CONTDST##n \ + dst0Stride == (ptrdiff_t)(size.width)) \ + { \ + size.width *= size.height; \ + size.height = 1; \ + } \ + typedef internal::VecTraits::vec128 vec128; \ + size_t roiw16 = size.width >= (16/sizeof(sgn##bits)-1) ? size.width - (16/sizeof(sgn##bits)-1) : 0; \ + typedef internal::VecTraits::vec64 vec64; \ + size_t roiw8 = size.width >= (8/sizeof(sgn##bits)-1) ? size.width - (8/sizeof(sgn##bits)-1) : 0; \ + \ + for (size_t i = 0u; i < size.height; ++i) \ + { \ + const sgn##bits * src = internal::getRowPtr(srcBase, srcStride, i); \ + FILL_LINES##n(VROW, sgn##bits) \ + size_t sj = 0u, dj = 0u; \ + \ + for (; dj < roiw16; sj += MUL##n(16)/sizeof(sgn##bits), dj += 16/sizeof(sgn##bits)) \ + SPLIT_QUAD(sgn, bits, n) \ + \ + if (dj < roiw8) \ + { \ + vec64 v_src = vld##n##_##sgn##bits(src + sj); \ + FILL_LINES##n(VST1, sgn##bits) \ + sj += MUL##n(8)/sizeof(sgn##bits); \ + dj += 8/sizeof(sgn##bits); \ + } \ + \ + for (; dj < size.width; sj += n, ++dj) \ + { \ + FILL_LINES##n(SST, sgn##bits) \ + } \ + } \ +} + +#define SPLIT64(sgn,n) void split##n(const Size2D &_size, \ + const sgn##64 * srcBase, ptrdiff_t srcStride \ + FILL_LINES##n(FARG, sgn##64) ) \ +{ \ + internal::assertSupportedConfiguration(); \ + Size2D size(_size); \ + if (CONTDST##n \ + dst0Stride == (ptrdiff_t)(size.width)) \ + { \ + size.width *= size.height; \ + size.height = 1; \ + } \ + typedef internal::VecTraits::vec64 vec64; \ + \ + for (size_t i = 0u; i < size.height; ++i) \ + { \ + const sgn##64 * src = internal::getRowPtr(srcBase, srcStride, i); \ + FILL_LINES##n(VROW, sgn##64) \ + size_t sj = 0u, dj = 0u; \ + \ + for (; dj < size.width; sj += n, ++dj) \ + { \ + vec64 v_src = vld##n##_##sgn##64(src + sj); \ + FILL_LINES##n(VST1, sgn##64) \ + } \ + } \ +} + +#if __GNUC__ == 4 && __GNUC_MINOR__ < 7 + +#define ALPHA_QUAD(sgn, bits) { \ + internal::prefetch(src + sj); \ + __asm__ ( \ + "vld4." #bits " {d0, d2, d4, d6}, [%[in0]] \n\t" \ + "vld4." #bits " {d1, d3, d5, d7}, [%[in1]] \n\t" \ + "vst3." #bits " {d0, d2, d4}, [%[out3_1]] \n\t" \ + "vst3." #bits " {d1, d3, d5}, [%[out3_2]] \n\t" \ + "vst1." #bits " {d6-d7}, [%[out1]] \n\t" \ + : \ + : [out3_1] "r" (dst3 + d3j), [out3_2] "r" (dst3 + d3j + 24/sizeof(sgn##bits)), [out1] "r" (dst1 + d1j), \ + [in0] "r" (src + sj), [in1] "r" (src + sj + 32/sizeof(sgn##bits)) \ + : "d0","d1","d2","d3","d4","d5","d6","d7" \ + ); \ + } + +#else + +#define ALPHA_QUAD(sgn, bits) { \ + internal::prefetch(src + sj); \ + union { vec128_4 v4; vec128_3 v3; } vals; \ + vals.v4 = vld4q_##sgn##bits(src + sj); \ + vst3q_##sgn##bits(dst3 + d3j, vals.v3); \ + vst1q_##sgn##bits(dst1 + d1j, vals.v4.val[3]); \ + } + +#endif // __GNUC__ == 4 && __GNUC_MINOR__ < 7 + +#define SPLIT4ALPHA(sgn,bits) void split4(const Size2D &_size, \ + const sgn##bits * srcBase, ptrdiff_t srcStride, \ + sgn##bits * dst3Base, ptrdiff_t dst3Stride, \ + sgn##bits * dst1Base, ptrdiff_t dst1Stride) \ +{ \ + internal::assertSupportedConfiguration(); \ + Size2D size(_size); \ + if (srcStride == dst3Stride && \ + srcStride == dst1Stride && \ + srcStride == (ptrdiff_t)(size.width)) \ + { \ + size.width *= size.height; \ + size.height = 1; \ + } \ + typedef internal::VecTraits::vec128 vec128_4; \ + typedef internal::VecTraits::vec128 vec128_3; \ + size_t roiw16 = size.width >= (16/sizeof(sgn##bits)-1) ? size.width - (16/sizeof(sgn##bits)-1) : 0; \ + typedef internal::VecTraits::vec64 vec64_4; \ + typedef internal::VecTraits::vec64 vec64_3; \ + size_t roiw8 = size.width >= (8/sizeof(sgn##bits)-1) ? size.width - (8/sizeof(sgn##bits)-1) : 0; \ + \ + for (size_t i = 0u; i < size.height; ++i) \ + { \ + const sgn##bits * src = internal::getRowPtr(srcBase, srcStride, i); \ + sgn##bits * dst3 = internal::getRowPtr(dst3Base, dst3Stride, i); \ + sgn##bits * dst1 = internal::getRowPtr(dst1Base, dst1Stride, i); \ + size_t sj = 0u, d3j = 0u, d1j = 0u; \ + \ + for (; d1j < roiw16; sj += MUL4(16)/sizeof(sgn##bits), d3j += MUL3(16)/sizeof(sgn##bits), \ + d1j += 16/sizeof(sgn##bits)) \ + ALPHA_QUAD(sgn, bits) \ + \ + if (d1j < roiw8) \ + { \ + union { vec64_4 v4; vec64_3 v3; } vals; \ + vals.v4 = vld4_##sgn##bits(src + sj); \ + vst3_u8(dst3 + d3j, vals.v3); \ + vst1_u8(dst1 + d1j, vals.v4.val[3]); \ + sj += MUL4(8)/sizeof(sgn##bits); \ + d3j += MUL3(8)/sizeof(sgn##bits); \ + d1j += 8/sizeof(sgn##bits); \ + } \ + \ + for (; d1j < size.width; sj += 4, d3j += 3, ++d1j) \ + { \ + dst3[d3j+0] = src[sj + 0]; \ + dst3[d3j+1] = src[sj + 1]; \ + dst3[d3j+2] = src[sj + 2]; \ + dst1[d1j] = src[sj + 3]; \ + } \ + } \ +} + +#else + +#define VOID_LINE(type, n) (void)dst##n##Base; (void)dst##n##Stride; + +#define SPLIT(sgn,bits,n) void split##n(const Size2D &size, \ + const sgn##bits * srcBase, ptrdiff_t srcStride \ + FILL_LINES##n(FARG, sgn##bits) ) \ +{ \ + internal::assertSupportedConfiguration(); \ + (void)size; \ + (void)srcBase; \ + (void)srcStride; \ + FILL_LINES##n(VOID, sgn##bits) \ +} + +#define SPLIT64(sgn,n) SPLIT(sgn,64,n) + +#define SPLIT4ALPHA(sgn,bits) void split4(const Size2D &size, \ + const sgn##bits * srcBase, ptrdiff_t srcStride, \ + sgn##bits * dst3Base, ptrdiff_t dst3Stride, \ + sgn##bits * dst1Base, ptrdiff_t dst1Stride) \ +{ \ + internal::assertSupportedConfiguration(); \ + (void)size; \ + (void)srcBase; \ + (void)srcStride; \ + (void)dst3Base; \ + (void)dst3Stride; \ + (void)dst1Base; \ + (void)dst1Stride; \ +} + +#endif //CAROTENE_NEON + +SPLIT(u, 8,2) +SPLIT(u, 8,3) +SPLIT(u, 8,4) +SPLIT(u,16,2) +SPLIT(u,16,3) +SPLIT(u,16,4) +SPLIT(s,32,2) +SPLIT(s,32,3) +SPLIT(s,32,4) + +SPLIT64(s, 2) +SPLIT64(s, 3) +SPLIT64(s, 4) + +SPLIT4ALPHA(u,8) + +} // namespace CAROTENE_NS diff --git a/3rdparty/carotene/src/channels_combine.cpp b/3rdparty/carotene/src/channels_combine.cpp new file mode 100644 index 0000000000..32b71470e2 --- /dev/null +++ b/3rdparty/carotene/src/channels_combine.cpp @@ -0,0 +1,389 @@ +/* + * By downloading, copying, installing or using the software you agree to this license. + * If you do not agree to this license, do not download, install, + * copy or use the software. + * + * + * License Agreement + * For Open Source Computer Vision Library + * (3-clause BSD License) + * + * Copyright (C) 2014-2015, NVIDIA Corporation, all rights reserved. + * Third party copyrights are property of their respective owners. + * + * Redistribution and use in source and binary forms, with or without modification, + * are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * * Neither the names of the copyright holders nor the names of the contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * This software is provided by the copyright holders and contributors "as is" and + * any express or implied warranties, including, but not limited to, the implied + * warranties of merchantability and fitness for a particular purpose are disclaimed. + * In no event shall copyright holders or contributors be liable for any direct, + * indirect, incidental, special, exemplary, or consequential damages + * (including, but not limited to, procurement of substitute goods or services; + * loss of use, data, or profits; or business interruption) however caused + * and on any theory of liability, whether in contract, strict liability, + * or tort (including negligence or otherwise) arising in any way out of + * the use of this software, even if advised of the possibility of such damage. + */ + +#include "common.hpp" +#include "vtransform.hpp" + +namespace CAROTENE_NS { + +#define FILL_LINES2(macro,type) \ + macro##_LINE(type,0) \ + macro##_LINE(type,1) +#define FILL_LINES3(macro,type) \ + FILL_LINES2(macro,type) \ + macro##_LINE(type,2) +#define FILL_LINES4(macro,type) \ + FILL_LINES3(macro,type) \ + macro##_LINE(type,3) + +#define FARG_LINE(type, n) , const type * src##n##Base, ptrdiff_t src##n##Stride + +#ifdef CAROTENE_NEON + +#define VROW_LINE(type, n) const type * src##n = internal::getRowPtr(src##n##Base, src##n##Stride, i); +#define PREF_LINE(type, n) internal::prefetch(src##n + sj); +#define VLD1Q_LINE(type, n) v_dst.val[n] = vld1q_##type(src##n + sj); +#define PRLD_LINE(type, n) internal::prefetch(src##n + sj); v_dst.val[n] = vld1q_##type(src##n + sj); +#define VLD1_LINE(type, n) v_dst.val[n] = vld1_##type(src##n + sj); +#define SLD_LINE(type, n) dst[dj + n] = src##n[sj]; + +#define MUL2(val) (val << 1) +#define MUL3(val) (MUL2(val) + val) +#define MUL4(val) (val << 2) + +#define CONTSRC2 dstStride == src0Stride && \ + dstStride == src1Stride && +#define CONTSRC3 dstStride == src0Stride && \ + dstStride == src1Stride && \ + dstStride == src2Stride && +#define CONTSRC4 dstStride == src0Stride && \ + dstStride == src1Stride && \ + dstStride == src2Stride && \ + dstStride == src3Stride && + +#if __GNUC__ == 4 && __GNUC_MINOR__ < 7 + +#define MERGE_ASM2(sgn, bits) __asm__ ( \ + "vld1." #bits " {d0-d1}, [%[in0]] \n\t" \ + "vld1." #bits " {d2-d3}, [%[in1]] \n\t" \ + "vst2." #bits " {d0, d2}, [%[out0]] \n\t" \ + "vst2." #bits " {d1, d3}, [%[out1]] \n\t" \ + : \ + : [in0] "r" (src0 + sj), [in1] "r" (src1 + sj), \ + [out0] "r" (dst + dj), [out1] "r" (dst + dj + MUL2(8)/sizeof(sgn##bits)) \ + : "d0","d1","d2","d3" \ + ); +#define MERGE_ASM3(sgn, bits) __asm__ ( \ + "vld1." #bits " {d0-d1}, [%[in0]] \n\t" \ + "vld1." #bits " {d2-d3}, [%[in1]] \n\t" \ + "vld1." #bits " {d4-d5}, [%[in2]] \n\t" \ + "vst3." #bits " {d0, d2, d4}, [%[out0]] \n\t" \ + "vst3." #bits " {d1, d3, d5}, [%[out1]] \n\t" \ + : \ + : [in0] "r" (src0 + sj), [in1] "r" (src1 + sj), [in2] "r" (src2 + sj), \ + [out0] "r" (dst + dj), [out1] "r" (dst + dj + MUL3(8)/sizeof(sgn##bits)) \ + : "d0","d1","d2","d3","d4","d5" \ + ); +#define MERGE_ASM4(sgn, bits) __asm__ ( \ + "vld1." #bits " {d0-d1}, [%[in0]] \n\t" \ + "vld1." #bits " {d2-d3}, [%[in1]] \n\t" \ + "vld1." #bits " {d4-d5}, [%[in2]] \n\t" \ + "vld1." #bits " {d6-d7}, [%[in3]] \n\t" \ + "vst4." #bits " {d0, d2, d4, d6}, [%[out0]] \n\t" \ + "vst4." #bits " {d1, d3, d5, d7}, [%[out1]] \n\t" \ + : \ + : [in0] "r" (src0 + sj), [in1] "r" (src1 + sj), [in2] "r" (src2 + sj), [in3] "r" (src3 + sj), \ + [out0] "r" (dst + dj), [out1] "r" (dst + dj + MUL4(8)/sizeof(sgn##bits)) \ + : "d0","d1","d2","d3","d4","d5","d6","d7" \ + ); + +#define MERGE_QUAD(sgn, bits, n) { \ + FILL_LINES##n(PREF, sgn##bits) \ + MERGE_ASM##n(sgn, bits) \ + } + +#else + +#define MERGE_QUAD(sgn, bits, n) { \ + vec128 v_dst; \ + /*FILL_LINES##n(PREF, sgn##bits) \ + FILL_LINES##n(VLD1Q, sgn##bits)*/ \ + FILL_LINES##n(PRLD, sgn##bits) \ + vst##n##q_##sgn##bits(dst + dj, v_dst); \ + } + +#endif // __GNUC__ == 4 && __GNUC_MINOR__ < 7 + +#define COMBINE(sgn,bits,n) void combine##n(const Size2D &_size \ + FILL_LINES##n(FARG, sgn##bits), \ + sgn##bits * dstBase, ptrdiff_t dstStride) \ +{ \ + internal::assertSupportedConfiguration(); \ + Size2D size(_size); \ + if (CONTSRC##n \ + dstStride == (ptrdiff_t)(size.width)) \ + { \ + size.width *= size.height; \ + size.height = 1; \ + } \ + typedef internal::VecTraits::vec128 vec128; \ + size_t roiw16 = size.width >= (16/sizeof(sgn##bits) - 1) ? size.width - (16/sizeof(sgn##bits) - 1) : 0; \ + typedef internal::VecTraits::vec64 vec64; \ + size_t roiw8 = size.width >= (8/sizeof(sgn##bits) - 1) ? size.width - (8/sizeof(sgn##bits) - 1) : 0; \ + \ + for (size_t i = 0u; i < size.height; ++i) \ + { \ + FILL_LINES##n(VROW, sgn##bits) \ + sgn##bits * dst = internal::getRowPtr(dstBase, dstStride, i); \ + size_t sj = 0u, dj = 0u; \ + \ + for (; sj < roiw16; sj += 16/sizeof(sgn##bits), dj += MUL##n(16)/sizeof(sgn##bits)) \ + MERGE_QUAD(sgn, bits, n) \ + \ + if ( sj < roiw8 ) \ + { \ + vec64 v_dst; \ + FILL_LINES##n(VLD1, sgn##bits) \ + vst##n##_##sgn##bits(dst + dj, v_dst); \ + sj += 8/sizeof(sgn##bits); dj += MUL##n(8)/sizeof(sgn##bits); \ + } \ + \ + for (; sj < size.width; ++sj, dj += n) \ + { \ + FILL_LINES##n(SLD, sgn##bits) \ + } \ + } \ +} + +#define COMBINE64(sgn,n) void combine##n(const Size2D &_size \ + FILL_LINES##n(FARG, sgn##64), \ + sgn##64 * dstBase, ptrdiff_t dstStride) \ +{ \ + internal::assertSupportedConfiguration(); \ + Size2D size(_size); \ + if (CONTSRC##n \ + dstStride == (ptrdiff_t)(size.width)) \ + { \ + size.width *= size.height; \ + size.height = 1; \ + } \ + typedef internal::VecTraits::vec64 vec64; \ + \ + for (size_t i = 0u; i < size.height; ++i) \ + { \ + FILL_LINES##n(VROW, sgn##64) \ + sgn##64 * dst = internal::getRowPtr(dstBase, dstStride, i); \ + size_t sj = 0u, dj = 0u; \ + \ + for (; sj < size.width; ++sj, dj += n) \ + { \ + vec64 v_dst; \ + FILL_LINES##n(VLD1, sgn##64) \ + vst##n##_##sgn##64(dst + dj, v_dst); \ + /*FILL_LINES##n(SLD, sgn##64)*/ \ + } \ + } \ +} + +#else + +#define VOID_LINE(type, n) (void)src##n##Base; (void)src##n##Stride; + +#define COMBINE(sgn,bits,n) void combine##n(const Size2D &size \ + FILL_LINES##n(FARG, sgn##bits), \ + sgn##bits * dstBase, ptrdiff_t dstStride) \ +{ \ + internal::assertSupportedConfiguration(); \ + (void)size; \ + FILL_LINES##n(VOID, sgn##bits) \ + (void)dstBase; \ + (void)dstStride; \ +} +#define COMBINE64(sgn,n) COMBINE(sgn,64,n) + +#endif //CAROTENE_NEON + +COMBINE(u, 8,2) +COMBINE(u, 8,3) +COMBINE(u, 8,4) +COMBINE(u,16,2) +COMBINE(u,16,3) +COMBINE(u,16,4) +COMBINE(s,32,2) +COMBINE(s,32,3) +COMBINE(s,32,4) +COMBINE64(s, 2) +COMBINE64(s, 3) +COMBINE64(s, 4) + +void combineYUYV(const Size2D &size, + const u8 * srcyBase, ptrdiff_t srcyStride, + const u8 * srcuBase, ptrdiff_t srcuStride, + const u8 * srcvBase, ptrdiff_t srcvStride, + u8 * dstBase, ptrdiff_t dstStride) +{ + internal::assertSupportedConfiguration(); +#ifdef CAROTENE_NEON +#ifndef ANDROID + size_t roiw32 = size.width >= 31 ? size.width - 31 : 0; +#endif + size_t roiw8 = size.width >= 7 ? size.width - 7 : 0; + + for (size_t i = 0u; i < size.height; i += 1) + { + const u8 * srcy = internal::getRowPtr(srcyBase, srcyStride, i); + const u8 * srcu = internal::getRowPtr(srcuBase, srcuStride, i); + const u8 * srcv = internal::getRowPtr(srcvBase, srcvStride, i); + u8 * dst = internal::getRowPtr(dstBase, dstStride, i); + size_t syj = 0u, sj = 0u, dj = 0u; + +#ifndef ANDROID + for (; sj < roiw32; sj += 32, syj += 64, dj += 128) + { + internal::prefetch(srcy + syj); + internal::prefetch(srcu + sj); + internal::prefetch(srcv + sj); + + uint8x16x2_t v_y = vld2q_u8(srcy + syj); + uint8x16x4_t v_dst; + v_dst.val[0] = v_y.val[0]; + v_dst.val[1] = vld1q_u8(srcu + sj); + v_dst.val[2] = v_y.val[1]; + v_dst.val[3] = vld1q_u8(srcv + sj); + vst4q_u8(dst + dj, v_dst); + + v_y = vld2q_u8(srcy + syj + 32); + v_dst.val[0] = v_y.val[0]; + v_dst.val[1] = vld1q_u8(srcu + sj + 16); + v_dst.val[2] = v_y.val[1]; + v_dst.val[3] = vld1q_u8(srcv + sj + 16); + vst4q_u8(dst + dj + 64, v_dst); + } +#endif + + for (; sj < roiw8; sj += 8, syj += 16, dj += 32) + { + uint8x8x2_t v_y = vld2_u8(srcy + syj); + uint8x8x4_t v_dst; + v_dst.val[0] = v_y.val[0]; + v_dst.val[1] = vld1_u8(srcu + sj); + v_dst.val[2] = v_y.val[1]; + v_dst.val[3] = vld1_u8(srcv + sj); + vst4_u8(dst + dj, v_dst); + } + + for (; sj < size.width; ++sj, syj += 2, dj += 4) + { + dst[dj] = srcy[syj]; + dst[dj + 1] = srcu[sj]; + dst[dj + 2] = srcy[syj + 1]; + dst[dj + 3] = srcv[sj]; + } + } +#else + (void)size; + (void)srcyBase; + (void)srcyStride; + (void)srcuBase; + (void)srcuStride; + (void)srcvBase; + (void)srcvStride; + (void)dstBase; + (void)dstStride; +#endif +} + +void combineUYVY(const Size2D &size, + const u8 * srcyBase, ptrdiff_t srcyStride, + const u8 * srcuBase, ptrdiff_t srcuStride, + const u8 * srcvBase, ptrdiff_t srcvStride, + u8 * dstBase, ptrdiff_t dstStride) +{ + internal::assertSupportedConfiguration(); +#ifdef CAROTENE_NEON +#ifndef ANDROID + size_t roiw32 = size.width >= 31 ? size.width - 31 : 0; +#endif + size_t roiw8 = size.width >= 7 ? size.width - 7 : 0; + + for (size_t i = 0u; i < size.height; ++i) + { + const u8 * srcy = internal::getRowPtr(srcyBase, srcyStride, i); + const u8 * srcu = internal::getRowPtr(srcuBase, srcuStride, i); + const u8 * srcv = internal::getRowPtr(srcvBase, srcvStride, i); + u8 * dst = internal::getRowPtr(dstBase, dstStride, i); + size_t syj = 0u, sj = 0u, dj = 0u; + +#ifndef ANDROID + for (; sj < roiw32; sj += 32, syj += 64, dj += 128) + { + internal::prefetch(srcy + syj); + internal::prefetch(srcu + sj); + internal::prefetch(srcv + sj); + + uint8x16x2_t v_y = vld2q_u8(srcy + syj); + uint8x16x4_t v_dst; + v_dst.val[0] = vld1q_u8(srcu + sj); + v_dst.val[1] = v_y.val[0]; + v_dst.val[2] = vld1q_u8(srcv + sj); + v_dst.val[3] = v_y.val[1]; + vst4q_u8(dst + dj, v_dst); + + v_y = vld2q_u8(srcy + syj + 32); + v_dst.val[0] = vld1q_u8(srcu + sj + 16); + v_dst.val[1] = v_y.val[0]; + v_dst.val[2] = vld1q_u8(srcv + sj + 16); + v_dst.val[3] = v_y.val[1]; + vst4q_u8(dst + dj + 64, v_dst); + } +#endif + + for (; sj < roiw8; sj += 8, syj += 16, dj += 32) + { + uint8x8x2_t v_y = vld2_u8(srcy + syj); + uint8x8x4_t v_dst; + v_dst.val[0] = vld1_u8(srcu + sj); + v_dst.val[1] = v_y.val[0]; + v_dst.val[2] = vld1_u8(srcv + sj); + v_dst.val[3] = v_y.val[1]; + vst4_u8(dst + dj, v_dst); + } + + for (; sj < size.width; ++sj, syj += 2, dj += 4) + { + dst[dj] = srcu[sj]; + dst[dj + 1] = srcy[syj]; + dst[dj + 2] = srcv[sj]; + dst[dj + 3] = srcy[syj + 1]; + } + } +#else + (void)size; + (void)srcyBase; + (void)srcyStride; + (void)srcuBase; + (void)srcuStride; + (void)srcvBase; + (void)srcvStride; + (void)dstBase; + (void)dstStride; +#endif +} + +} // namespace CAROTENE_NS diff --git a/3rdparty/carotene/src/cmp.cpp b/3rdparty/carotene/src/cmp.cpp new file mode 100644 index 0000000000..eda121985e --- /dev/null +++ b/3rdparty/carotene/src/cmp.cpp @@ -0,0 +1,340 @@ +/* + * By downloading, copying, installing or using the software you agree to this license. + * If you do not agree to this license, do not download, install, + * copy or use the software. + * + * + * License Agreement + * For Open Source Computer Vision Library + * (3-clause BSD License) + * + * Copyright (C) 2014-2015, NVIDIA Corporation, all rights reserved. + * Third party copyrights are property of their respective owners. + * + * Redistribution and use in source and binary forms, with or without modification, + * are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * * Neither the names of the copyright holders nor the names of the contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * This software is provided by the copyright holders and contributors "as is" and + * any express or implied warranties, including, but not limited to, the implied + * warranties of merchantability and fitness for a particular purpose are disclaimed. + * In no event shall copyright holders or contributors be liable for any direct, + * indirect, incidental, special, exemplary, or consequential damages + * (including, but not limited to, procurement of substitute goods or services; + * loss of use, data, or profits; or business interruption) however caused + * and on any theory of liability, whether in contract, strict liability, + * or tort (including negligence or otherwise) arising in any way out of + * the use of this software, even if advised of the possibility of such damage. + */ + +#include "common.hpp" +#include "vtransform.hpp" + +namespace CAROTENE_NS { + +#ifdef CAROTENE_NEON + +namespace { + +inline void vnst(u8* dst, uint8x16_t v1, uint8x16_t v2) { vst1q_u8(dst, v1); vst1q_u8(dst+16, v2); } +inline void vnst(u8* dst, uint16x8_t v1, uint16x8_t v2) { vst1q_u8(dst, vcombine_u8(vmovn_u16(v1), vmovn_u16(v2))); } +inline void vnst(u8* dst, uint32x4_t v1, uint32x4_t v2) { vst1_u8(dst, vmovn_u16(vcombine_u16(vmovn_u32(v1), vmovn_u32(v2)))); } + +template struct vtail +{ + static inline void compare(const typename Op::type * src0, const typename Op::type * src1, + u8 * dst, const Op & op, + size_t &x, size_t width) + { + //do nothing since there couldn't be enough data + (void)src0; + (void)src1; + (void)dst; + (void)op; + (void)x; + (void)width; + } +}; +template struct vtail +{ + static inline void compare(const typename Op::type * src0, const typename Op::type * src1, + u8 * dst, const Op & op, + size_t &x, size_t width) + { + typedef typename Op::type type; + typedef typename internal::VecTraits::vec128 vec128; + typedef typename internal::VecTraits::unsign::vec128 uvec128; + //There no more than 15 elements in the tail, so we could handle 8 element vector only once + if( x + 8 < width) + { + vec128 v_src0, v_src1; + uvec128 v_dst; + + v_src0 = internal::vld1q(src0 + x); + v_src1 = internal::vld1q(src1 + x); + op(v_src0, v_src1, v_dst); + internal::vst1(dst + x, internal::vmovn(v_dst)); + x+=8; + } + } +}; +template struct vtail +{ + static inline void compare(const typename Op::type * src0, const typename Op::type * src1, + u8 * dst, const Op & op, + size_t &x, size_t width) + { + typedef typename Op::type type; + typedef typename internal::VecTraits::vec128 vec128; + typedef typename internal::VecTraits::unsign::vec128 uvec128; + typedef typename internal::VecTraits::vec64 vec64; + typedef typename internal::VecTraits::unsign::vec64 uvec64; + //There no more than 31 elements in the tail, so we could handle once 16+8 or 16 or 8 elements + if( x + 16 < width) + { + vec128 v_src0, v_src1; + uvec128 v_dst; + + v_src0 = internal::vld1q(src0 + x); + v_src1 = internal::vld1q(src1 + x); + op(v_src0, v_src1, v_dst); + internal::vst1q(dst + x, v_dst); + x+=16; + } + if( x + 8 < width) + { + vec64 v_src0, v_src1; + uvec64 v_dst; + + v_src0 = internal::vld1(src0 + x); + v_src1 = internal::vld1(src1 + x); + op(v_src0, v_src1, v_dst); + internal::vst1(dst + x, v_dst); + x+=8; + } + } +}; + +template +void vcompare(Size2D size, + const typename Op::type * src0Base, ptrdiff_t src0Stride, + const typename Op::type * src1Base, ptrdiff_t src1Stride, + u8 * dstBase, ptrdiff_t dstStride, const Op & op) +{ + typedef typename Op::type type; + typedef typename internal::VecTraits::vec128 vec128; + typedef typename internal::VecTraits::unsign::vec128 uvec128; + + if (src0Stride == src1Stride && src0Stride == dstStride && + src0Stride == (ptrdiff_t)(size.width * sizeof(type))) + { + size.width *= size.height; + size.height = 1; + } + + const u32 step_base = 32 / sizeof(type); + size_t roiw_base = size.width >= (step_base - 1) ? size.width - step_base + 1 : 0; + + for (size_t y = 0; y < size.height; ++y) + { + const type * src0 = internal::getRowPtr(src0Base, src0Stride, y); + const type * src1 = internal::getRowPtr(src1Base, src1Stride, y); + u8 * dst = internal::getRowPtr(dstBase, dstStride, y); + size_t x = 0; + + for( ; x < roiw_base; x += step_base ) + { + internal::prefetch(src0 + x); + internal::prefetch(src1 + x); + + vec128 v_src00 = internal::vld1q(src0 + x), v_src01 = internal::vld1q(src0 + x + 16 / sizeof(type)); + vec128 v_src10 = internal::vld1q(src1 + x), v_src11 = internal::vld1q(src1 + x + 16 / sizeof(type)); + uvec128 v_dst0; + uvec128 v_dst1; + + op(v_src00, v_src10, v_dst0); + op(v_src01, v_src11, v_dst1); + + vnst(dst + x, v_dst0, v_dst1); + } + + vtail::compare(src0, src1, dst, op, x, size.width); + + for (; x < size.width; ++x) + { + op(src0 + x, src1 + x, dst + x); + } + } +} + +template +struct OpCmpEQ +{ + typedef T type; + + void operator() (const typename internal::VecTraits::vec128 & v_src0, const typename internal::VecTraits::vec128 & v_src1, + typename internal::VecTraits::unsign::vec128 & v_dst) const + { + v_dst = internal::vceqq(v_src0, v_src1); + } + + void operator() (const typename internal::VecTraits::vec64 & v_src0, const typename internal::VecTraits::vec64 & v_src1, + typename internal::VecTraits::unsign::vec64 & v_dst) const + { + v_dst = internal::vceq(v_src0, v_src1); + } + + void operator() (const T * src0, const T * src1, u8 * dst) const + { + dst[0] = src0[0] == src1[0] ? 255 : 0; + } +}; + +template +struct OpCmpNE +{ + typedef T type; + + void operator() (const typename internal::VecTraits::vec128 & v_src0, const typename internal::VecTraits::vec128 & v_src1, + typename internal::VecTraits::unsign::vec128 & v_dst) const + { + v_dst = internal::vmvnq(internal::vceqq(v_src0, v_src1)); + } + + void operator() (const typename internal::VecTraits::vec64 & v_src0, const typename internal::VecTraits::vec64 & v_src1, + typename internal::VecTraits::unsign::vec64 & v_dst) const + { + v_dst = internal::vmvn(internal::vceq(v_src0, v_src1)); + } + + void operator() (const T * src0, const T * src1, u8 * dst) const + { + dst[0] = src0[0] == src1[0] ? 0 : 255; + } +}; + +template +struct OpCmpGT +{ + typedef T type; + + void operator() (const typename internal::VecTraits::vec128 & v_src0, const typename internal::VecTraits::vec128 & v_src1, + typename internal::VecTraits::unsign::vec128 & v_dst) const + { + v_dst = internal::vcgtq(v_src0, v_src1); + } + + void operator() (const typename internal::VecTraits::vec64 & v_src0, const typename internal::VecTraits::vec64 & v_src1, + typename internal::VecTraits::unsign::vec64 & v_dst) const + { + v_dst = internal::vcgt(v_src0, v_src1); + } + + void operator() (const T * src0, const T * src1, u8 * dst) const + { + dst[0] = src0[0] > src1[0] ? 255 : 0; + } +}; + +template +struct OpCmpGE +{ + typedef T type; + + void operator() (const typename internal::VecTraits::vec128 & v_src0, const typename internal::VecTraits::vec128 & v_src1, + typename internal::VecTraits::unsign::vec128 & v_dst) const + { + v_dst = internal::vcgeq(v_src0, v_src1); + } + + void operator() (const typename internal::VecTraits::vec64 & v_src0, const typename internal::VecTraits::vec64 & v_src1, + typename internal::VecTraits::unsign::vec64 & v_dst) const + { + v_dst = internal::vcge(v_src0, v_src1); + } + + void operator() (const T * src0, const T * src1, u8 * dst) const + { + dst[0] = src0[0] >= src1[0] ? 255 : 0; + } +}; + +} + +#define IMPL_CMPOP(op, type) \ +void cmp##op(const Size2D &size, \ + const type * src0Base, ptrdiff_t src0Stride, \ + const type * src1Base, ptrdiff_t src1Stride, \ + u8 *dstBase, ptrdiff_t dstStride) \ +{ \ + internal::assertSupportedConfiguration(); \ + vcompare(size, \ + src0Base, src0Stride, \ + src1Base, src1Stride, \ + dstBase, dstStride, \ + OpCmp##op()); \ +} + +#else + +#define IMPL_CMPOP(op, type) \ +void cmp##op(const Size2D &size, \ + const type * src0Base, ptrdiff_t src0Stride, \ + const type * src1Base, ptrdiff_t src1Stride, \ + u8 *dstBase, ptrdiff_t dstStride) \ +{ \ + internal::assertSupportedConfiguration(); \ + (void)size; \ + (void)src0Base; \ + (void)src0Stride; \ + (void)src1Base; \ + (void)src1Stride; \ + (void)dstBase; \ + (void)dstStride; \ +} + +#endif + +IMPL_CMPOP(EQ, u8) +IMPL_CMPOP(EQ, s8) +IMPL_CMPOP(EQ, u16) +IMPL_CMPOP(EQ, s16) +IMPL_CMPOP(EQ, u32) +IMPL_CMPOP(EQ, s32) +IMPL_CMPOP(EQ, f32) + +IMPL_CMPOP(NE, u8) +IMPL_CMPOP(NE, s8) +IMPL_CMPOP(NE, u16) +IMPL_CMPOP(NE, s16) +IMPL_CMPOP(NE, u32) +IMPL_CMPOP(NE, s32) +IMPL_CMPOP(NE, f32) + +IMPL_CMPOP(GT, u8) +IMPL_CMPOP(GT, s8) +IMPL_CMPOP(GT, u16) +IMPL_CMPOP(GT, s16) +IMPL_CMPOP(GT, u32) +IMPL_CMPOP(GT, s32) +IMPL_CMPOP(GT, f32) + +IMPL_CMPOP(GE, u8) +IMPL_CMPOP(GE, s8) +IMPL_CMPOP(GE, u16) +IMPL_CMPOP(GE, s16) +IMPL_CMPOP(GE, u32) +IMPL_CMPOP(GE, s32) +IMPL_CMPOP(GE, f32) + +} // namespace CAROTENE_NS diff --git a/3rdparty/carotene/src/colorconvert.cpp b/3rdparty/carotene/src/colorconvert.cpp new file mode 100644 index 0000000000..ea2db6043a --- /dev/null +++ b/3rdparty/carotene/src/colorconvert.cpp @@ -0,0 +1,2846 @@ +/* + * By downloading, copying, installing or using the software you agree to this license. + * If you do not agree to this license, do not download, install, + * copy or use the software. + * + * + * License Agreement + * For Open Source Computer Vision Library + * (3-clause BSD License) + * + * Copyright (C) 2012-2015, NVIDIA Corporation, all rights reserved. + * Third party copyrights are property of their respective owners. + * + * Redistribution and use in source and binary forms, with or without modification, + * are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * * Neither the names of the copyright holders nor the names of the contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * This software is provided by the copyright holders and contributors "as is" and + * any express or implied warranties, including, but not limited to, the implied + * warranties of merchantability and fitness for a particular purpose are disclaimed. + * In no event shall copyright holders or contributors be liable for any direct, + * indirect, incidental, special, exemplary, or consequential damages + * (including, but not limited to, procurement of substitute goods or services; + * loss of use, data, or profits; or business interruption) however caused + * and on any theory of liability, whether in contract, strict liability, + * or tort (including negligence or otherwise) arising in any way out of + * the use of this software, even if advised of the possibility of such damage. + */ + +#include "common.hpp" + +#include "saturate_cast.hpp" + +namespace CAROTENE_NS { + +#ifdef CAROTENE_NEON + +namespace { + +enum +{ + SHIFT = 14, + SHIFT_DELTA = 1 << (SHIFT - 1), + + R2Y_BT601 = 4899, + G2Y_BT601 = 9617, + B2Y_BT601 = 1868, + + R2Y_BT709 = 3483, + G2Y_BT709 = 11718, + B2Y_BT709 = 1183, +}; + +inline uint8x8_t convertToGray(const uint16x8_t & v_r, + const uint16x8_t & v_g, + const uint16x8_t & v_b, + const uint16x4_t & v_r2y, + const uint16x4_t & v_g2y, + const uint16x4_t & v_b2y) +{ + uint32x4_t v_dst0 = vmull_u16(vget_low_u16(v_g), v_g2y); + uint32x4_t v_dst1 = vmull_u16(vget_high_u16(v_g), v_g2y); + + v_dst0 = vmlal_u16(v_dst0, vget_low_u16(v_r), v_r2y); + v_dst1 = vmlal_u16(v_dst1, vget_high_u16(v_r), v_r2y); + + v_dst0 = vmlal_u16(v_dst0, vget_low_u16(v_b), v_b2y); + v_dst1 = vmlal_u16(v_dst1, vget_high_u16(v_b), v_b2y); + + uint8x8_t v_gray = vqmovn_u16(vcombine_u16(vrshrn_n_u32(v_dst0, SHIFT), + vrshrn_n_u32(v_dst1, SHIFT))); + + return v_gray; +} + +} // namespace + +#endif + +void rgb2gray(const Size2D &size, COLOR_SPACE color_space, + const u8 * srcBase, ptrdiff_t srcStride, + u8 * dstBase, ptrdiff_t dstStride) +{ + internal::assertSupportedConfiguration(); +#ifdef CAROTENE_NEON + const u32 R2Y = color_space == COLOR_SPACE_BT601 ? R2Y_BT601 : R2Y_BT709; + const u32 G2Y = color_space == COLOR_SPACE_BT601 ? G2Y_BT601 : G2Y_BT709; + const u32 B2Y = color_space == COLOR_SPACE_BT601 ? B2Y_BT601 : B2Y_BT709; + +#if defined(__GNUC__) && __GNUC_MINOR__ < 7 + register int16x4_t v_r2y asm ("d31") = vmov_n_s16(R2Y); + register int16x4_t v_g2y asm ("d30") = vmov_n_s16(G2Y); + register int16x4_t v_b2y asm ("d29") = vmov_n_s16(B2Y); +#else + uint16x4_t v_r2y = vdup_n_u16(R2Y), + v_g2y = vdup_n_u16(G2Y), + v_b2y = vdup_n_u16(B2Y); + + size_t roiw16 = size.width >= 15 ? size.width - 15 : 0; +#endif + size_t roiw8 = size.width >= 7 ? size.width - 7 : 0; + + for (size_t i = 0u; i < size.height; ++i) + { + const u8 * src = internal::getRowPtr(srcBase, srcStride, i); + u8 * dst = internal::getRowPtr(dstBase, dstStride, i); + size_t sj = 0u, dj = 0u; + +#if defined(__GNUC__) && __GNUC_MINOR__ < 7 + for (; dj < roiw8; sj += 24, dj += 8) + { + internal::prefetch(src + sj); + __asm__ ( + "vld3.8 {d0-d2}, [%[in]] @RGB \n\t" + "vmovl.u8 q2, d0 @R (d4,d5) \n\t" + "vmovl.u8 q3, d1 @G (d6,d7) \n\t" + "vmovl.u8 q4, d2 @B (d8,d9) \n\t" + "vmull.u16 q5, d6, d30 @Y (q5,q6): G \n\t" + "vmull.u16 q6, d7, d30 @Y (q5,q6): G \n\t" + "vmlal.s16 q5, d8, d29 @Y (q5,q6): GB \n\t" + "vmlal.s16 q6, d9, d29 @Y (q5,q6): GB \n\t" + "vmlal.s16 q5, d4, d31 @Y (q5,q6): GBR \n\t" + "vmlal.s16 q6, d5, d31 @Y (q5,q6): GBR \n\t" + "vrshrn.s32 d8, q5, #14 @Y -> q4 \n\t" + "vrshrn.s32 d9, q6, #14 @Y -> q4 \n\t" + "vqmovn.u16 d4, q4 \n\t" + "vst1.8 {d4}, [%[out]] \n\t" + : /*no output*/ + : [out] "r" (dst + dj), [in] "r" (src + sj), "w" (v_r2y), "w" (v_g2y), "w" (v_b2y) + : "d0","d1","d2","d3","d4","d5","d6","d7","d8","d9","d10","d11","d12","d13" + ); + } +#else + for (; dj < roiw16; sj += 48, dj += 16) + { + internal::prefetch(src + sj); + uint8x16x3_t v_src0 = vld3q_u8(src + sj); + // 0 + uint16x8_t v_r = vmovl_u8(vget_low_u8(v_src0.val[0])), + v_g = vmovl_u8(vget_low_u8(v_src0.val[1])), + v_b = vmovl_u8(vget_low_u8(v_src0.val[2])); + uint8x8_t v_gray0 = convertToGray(v_r, v_g, v_b, v_r2y, v_g2y, v_b2y); + + v_r = vmovl_u8(vget_high_u8(v_src0.val[0])), + v_g = vmovl_u8(vget_high_u8(v_src0.val[1])), + v_b = vmovl_u8(vget_high_u8(v_src0.val[2])); + uint8x8_t v_gray1 = convertToGray(v_r, v_g, v_b, v_r2y, v_g2y, v_b2y); + + vst1q_u8(dst + dj, vcombine_u8(v_gray0, v_gray1)); + } + + if (dj < roiw8) + { + uint8x8x3_t v_src = vld3_u8(src + sj); + uint16x8_t v_r = vmovl_u8(v_src.val[0]), + v_g = vmovl_u8(v_src.val[1]), + v_b = vmovl_u8(v_src.val[2]); + uint8x8_t v_gray = convertToGray(v_r, v_g, v_b, v_r2y, v_g2y, v_b2y); + + vst1_u8(dst + dj, v_gray); + sj += 24; dj += 8; + } +#endif + + for (; dj < size.width; sj += 3, dj++) + { + u32 val = src[sj] * R2Y + src[sj + 1] * G2Y + src[sj + 2] * B2Y; + dst[dj] = internal::saturate_cast((val + SHIFT_DELTA) >> SHIFT); + } + } +#else + (void)size; + (void)color_space; + (void)srcBase; + (void)srcStride; + (void)dstBase; + (void)dstStride; +#endif +} + +void rgbx2gray(const Size2D &size, COLOR_SPACE color_space, + const u8 * srcBase, ptrdiff_t srcStride, + u8 * dstBase, ptrdiff_t dstStride) +{ + internal::assertSupportedConfiguration(); +#ifdef CAROTENE_NEON + const u32 R2Y = color_space == COLOR_SPACE_BT601 ? R2Y_BT601 : R2Y_BT709; + const u32 G2Y = color_space == COLOR_SPACE_BT601 ? G2Y_BT601 : G2Y_BT709; + const u32 B2Y = color_space == COLOR_SPACE_BT601 ? B2Y_BT601 : B2Y_BT709; + +#if defined(__GNUC__) && __GNUC_MINOR__ < 7 + register int16x4_t v_r2y asm ("d31") = vmov_n_s16(R2Y); + register int16x4_t v_g2y asm ("d30") = vmov_n_s16(G2Y); + register int16x4_t v_b2y asm ("d29") = vmov_n_s16(B2Y); +#else + uint16x4_t v_r2y = vdup_n_u16(R2Y), + v_g2y = vdup_n_u16(G2Y), + v_b2y = vdup_n_u16(B2Y); + + size_t roiw16 = size.width >= 15 ? size.width - 15 : 0; +#endif + size_t roiw8 = size.width >= 7 ? size.width - 7 : 0; + + for (size_t i = 0u; i < size.height; ++i) + { + const u8 * src = internal::getRowPtr(srcBase, srcStride, i); + u8 * dst = internal::getRowPtr(dstBase, dstStride, i); + size_t sj = 0u, dj = 0u; + +#if defined(__GNUC__) && __GNUC_MINOR__ < 7 + for (; dj < roiw8; sj += 32, dj += 8) + { + internal::prefetch(src + sj); + __asm__ ( + "vld4.8 {d0-d3}, [%[in]] @RGBA \n\t" + "vmovl.u8 q2, d0 @R (d4,d5) \n\t" + "vmovl.u8 q3, d1 @G (d6,d7) \n\t" + "vmovl.u8 q4, d2 @B (d8,d9) \n\t" + "vmull.u16 q5, d6, d30 @Y (q5,q6): G \n\t" + "vmull.u16 q6, d7, d30 @Y (q5,q6): G \n\t" + "vmlal.s16 q5, d8, d29 @Y (q5,q6): GB \n\t" + "vmlal.s16 q6, d9, d29 @Y (q5,q6): GB \n\t" + "vmlal.s16 q5, d4, d31 @Y (q5,q6): GBR \n\t" + "vmlal.s16 q6, d5, d31 @Y (q5,q6): GBR \n\t" + "vrshrn.s32 d8, q5, #14 @Y -> q4 \n\t" + "vrshrn.s32 d9, q6, #14 @Y -> q4 \n\t" + "vqmovn.u16 d4, q4 \n\t" + "vst1.8 {d4}, [%[out]] \n\t" + : /*no output*/ + : [out] "r" (dst + dj), [in] "r" (src + sj), "w" (v_r2y), "w" (v_g2y), "w" (v_b2y) + : "d0","d1","d2","d3","d4","d5","d6","d7","d8","d9","d10","d11","d12","d13" + ); + } +#else + for (; dj < roiw16; sj += 64, dj += 16) + { + internal::prefetch(src + sj); + uint8x16x4_t v_src0 = vld4q_u8(src + sj); + + // 0 + uint16x8_t v_r = vmovl_u8(vget_low_u8(v_src0.val[0])), + v_g = vmovl_u8(vget_low_u8(v_src0.val[1])), + v_b = vmovl_u8(vget_low_u8(v_src0.val[2])); + uint8x8_t v_gray0 = convertToGray(v_r, v_g, v_b, v_r2y, v_g2y, v_b2y); + + v_r = vmovl_u8(vget_high_u8(v_src0.val[0])), + v_g = vmovl_u8(vget_high_u8(v_src0.val[1])), + v_b = vmovl_u8(vget_high_u8(v_src0.val[2])); + uint8x8_t v_gray1 = convertToGray(v_r, v_g, v_b, v_r2y, v_g2y, v_b2y); + + vst1q_u8(dst + dj, vcombine_u8(v_gray0, v_gray1)); + } + + if (dj < roiw8) + { + uint8x8x4_t v_src = vld4_u8(src + sj); + uint16x8_t v_r = vmovl_u8(v_src.val[0]), + v_g = vmovl_u8(v_src.val[1]), + v_b = vmovl_u8(v_src.val[2]); + uint8x8_t v_gray = convertToGray(v_r, v_g, v_b, v_r2y, v_g2y, v_b2y); + + vst1_u8(dst + dj, v_gray); + sj += 32; dj += 8; + } +#endif + + for (; dj < size.width; sj += 4, dj++) + { + u32 val = src[sj] * R2Y + src[sj + 1] * G2Y + src[sj + 2] * B2Y; + dst[dj] = internal::saturate_cast((val + SHIFT_DELTA) >> SHIFT); + } + } +#else + (void)size; + (void)color_space; + (void)srcBase; + (void)srcStride; + (void)dstBase; + (void)dstStride; +#endif +} + +void bgr2gray(const Size2D &size, COLOR_SPACE color_space, + const u8 * srcBase, ptrdiff_t srcStride, + u8 * dstBase, ptrdiff_t dstStride) +{ + internal::assertSupportedConfiguration(); +#ifdef CAROTENE_NEON + const u32 R2Y = color_space == COLOR_SPACE_BT601 ? R2Y_BT601 : R2Y_BT709; + const u32 G2Y = color_space == COLOR_SPACE_BT601 ? G2Y_BT601 : G2Y_BT709; + const u32 B2Y = color_space == COLOR_SPACE_BT601 ? B2Y_BT601 : B2Y_BT709; + +#if defined(__GNUC__) && __GNUC_MINOR__ < 7 + register int16x4_t v_r2y asm ("d31") = vmov_n_s16(R2Y); + register int16x4_t v_g2y asm ("d30") = vmov_n_s16(G2Y); + register int16x4_t v_b2y asm ("d29") = vmov_n_s16(B2Y); +#else + uint16x4_t v_r2y = vdup_n_u16(R2Y), + v_g2y = vdup_n_u16(G2Y), + v_b2y = vdup_n_u16(B2Y); + + size_t roiw16 = size.width >= 15 ? size.width - 15 : 0; +#endif + size_t roiw8 = size.width >= 7 ? size.width - 7 : 0; + + for (size_t i = 0u; i < size.height; ++i) + { + const u8 * src = internal::getRowPtr(srcBase, srcStride, i); + u8 * dst = internal::getRowPtr(dstBase, dstStride, i); + size_t sj = 0u, dj = 0u; + +#if defined(__GNUC__) && __GNUC_MINOR__ < 7 + for (; dj < roiw8; sj += 24, dj += 8) + { + internal::prefetch(src + sj); + __asm__ ( + "vld3.8 {d0-d2}, [%[in]] @BGR \n\t" + "vmovl.u8 q2, d2 @R (d4,d5) \n\t" + "vmovl.u8 q3, d1 @G (d6,d7) \n\t" + "vmovl.u8 q4, d0 @B (d8,d9) \n\t" + "vmull.u16 q5, d6, d30 @Y (q5,q6): G \n\t" + "vmull.u16 q6, d7, d30 @Y (q5,q6): G \n\t" + "vmlal.s16 q5, d8, d29 @Y (q5,q6): GB \n\t" + "vmlal.s16 q6, d9, d29 @Y (q5,q6): GB \n\t" + "vmlal.s16 q5, d4, d31 @Y (q5,q6): GBR \n\t" + "vmlal.s16 q6, d5, d31 @Y (q5,q6): GBR \n\t" + "vrshrn.s32 d8, q5, #14 @Y -> q4 \n\t" + "vrshrn.s32 d9, q6, #14 @Y -> q4 \n\t" + "vqmovn.u16 d4, q4 \n\t" + "vst1.8 {d4}, [%[out]] \n\t" + : /*no output*/ + : [out] "r" (dst + dj), [in] "r" (src + sj), "w" (v_r2y), "w" (v_g2y), "w" (v_b2y) + : "d0","d1","d2","d3","d4","d5","d6","d7","d8","d9","d10","d11","d12","d13" + ); + } +#else + for (; dj < roiw16; sj += 48, dj += 16) + { + internal::prefetch(src + sj); + uint8x16x3_t v_src0 = vld3q_u8(src + sj); + + // 0 + uint16x8_t v_b = vmovl_u8(vget_low_u8(v_src0.val[0])), + v_g = vmovl_u8(vget_low_u8(v_src0.val[1])), + v_r = vmovl_u8(vget_low_u8(v_src0.val[2])); + uint8x8_t v_gray0 = convertToGray(v_r, v_g, v_b, v_r2y, v_g2y, v_b2y); + + v_b = vmovl_u8(vget_high_u8(v_src0.val[0])), + v_g = vmovl_u8(vget_high_u8(v_src0.val[1])), + v_r = vmovl_u8(vget_high_u8(v_src0.val[2])); + uint8x8_t v_gray1 = convertToGray(v_r, v_g, v_b, v_r2y, v_g2y, v_b2y); + + vst1q_u8(dst + dj, vcombine_u8(v_gray0, v_gray1)); + } + + if (dj < roiw8) + { + uint8x8x3_t v_src = vld3_u8(src + sj); + uint16x8_t v_b = vmovl_u8(v_src.val[0]), + v_g = vmovl_u8(v_src.val[1]), + v_r = vmovl_u8(v_src.val[2]); + uint8x8_t v_gray = convertToGray(v_r, v_g, v_b, v_r2y, v_g2y, v_b2y); + + vst1_u8(dst + dj, v_gray); + sj += 24; dj += 8; + } +#endif + + for (; dj < size.width; sj += 3, dj++) + { + u32 val = src[sj] * B2Y + src[sj + 1] * G2Y + src[sj + 2] * R2Y; + dst[dj] = internal::saturate_cast((val + SHIFT_DELTA) >> SHIFT); + } + } +#else + (void)size; + (void)color_space; + (void)srcBase; + (void)srcStride; + (void)dstBase; + (void)dstStride; +#endif +} + +void bgrx2gray(const Size2D &size, COLOR_SPACE color_space, + const u8 * srcBase, ptrdiff_t srcStride, + u8 * dstBase, ptrdiff_t dstStride) +{ + internal::assertSupportedConfiguration(); +#ifdef CAROTENE_NEON + const u32 R2Y = color_space == COLOR_SPACE_BT601 ? R2Y_BT601 : R2Y_BT709; + const u32 G2Y = color_space == COLOR_SPACE_BT601 ? G2Y_BT601 : G2Y_BT709; + const u32 B2Y = color_space == COLOR_SPACE_BT601 ? B2Y_BT601 : B2Y_BT709; + +#if defined(__GNUC__) && __GNUC_MINOR__ < 7 + register int16x4_t v_r2y asm ("d31") = vmov_n_s16(R2Y); + register int16x4_t v_g2y asm ("d30") = vmov_n_s16(G2Y); + register int16x4_t v_b2y asm ("d29") = vmov_n_s16(B2Y); +#else + uint16x4_t v_r2y = vdup_n_u16(R2Y), + v_g2y = vdup_n_u16(G2Y), + v_b2y = vdup_n_u16(B2Y); + + size_t roiw16 = size.width >= 15 ? size.width - 15 : 0; +#endif + size_t roiw8 = size.width >= 7 ? size.width - 7 : 0; + + for (size_t i = 0u; i < size.height; ++i) + { + const u8 * src = internal::getRowPtr(srcBase, srcStride, i); + u8 * dst = internal::getRowPtr(dstBase, dstStride, i); + size_t sj = 0u, dj = 0u; + +#if defined(__GNUC__) && __GNUC_MINOR__ < 7 + for (; dj < roiw8; sj += 32, dj += 8) + { + internal::prefetch(src + sj); + __asm__ ( + "vld4.8 {d0-d3}, [%[in]] @BGRA \n\t" + "vmovl.u8 q2, d2 @R (d4,d5) \n\t" + "vmovl.u8 q3, d1 @G (d6,d7) \n\t" + "vmovl.u8 q4, d0 @B (d8,d9) \n\t" + "vmull.u16 q5, d6, d30 @Y (q5,q6): G \n\t" + "vmull.u16 q6, d7, d30 @Y (q5,q6): G \n\t" + "vmlal.s16 q5, d8, d29 @Y (q5,q6): GB \n\t" + "vmlal.s16 q6, d9, d29 @Y (q5,q6): GB \n\t" + "vmlal.s16 q5, d4, d31 @Y (q5,q6): GBR \n\t" + "vmlal.s16 q6, d5, d31 @Y (q5,q6): GBR \n\t" + "vrshrn.s32 d8, q5, #14 @Y -> q4 \n\t" + "vrshrn.s32 d9, q6, #14 @Y -> q4 \n\t" + "vqmovn.u16 d4, q4 \n\t" + "vst1.8 {d4}, [%[out]] \n\t" + : /*no output*/ + : [out] "r" (dst + dj), [in] "r" (src + sj), "w" (v_r2y), "w" (v_g2y), "w" (v_b2y) + : "d0","d1","d2","d3","d4","d5","d6","d7","d8","d9","d10","d11","d12","d13" + ); + } +#else + for (; dj < roiw16; sj += 64, dj += 16) + { + internal::prefetch(src + sj); + uint8x16x4_t v_src0 = vld4q_u8(src + sj); + + // 0 + uint16x8_t v_b = vmovl_u8(vget_low_u8(v_src0.val[0])), + v_g = vmovl_u8(vget_low_u8(v_src0.val[1])), + v_r = vmovl_u8(vget_low_u8(v_src0.val[2])); + uint8x8_t v_gray0 = convertToGray(v_r, v_g, v_b, v_r2y, v_g2y, v_b2y); + + v_b = vmovl_u8(vget_high_u8(v_src0.val[0])), + v_g = vmovl_u8(vget_high_u8(v_src0.val[1])), + v_r = vmovl_u8(vget_high_u8(v_src0.val[2])); + uint8x8_t v_gray1 = convertToGray(v_r, v_g, v_b, v_r2y, v_g2y, v_b2y); + + vst1q_u8(dst + dj, vcombine_u8(v_gray0, v_gray1)); + } + + if (dj < roiw8) + { + uint8x8x4_t v_src = vld4_u8(src + sj); + uint16x8_t v_b = vmovl_u8(v_src.val[0]), + v_g = vmovl_u8(v_src.val[1]), + v_r = vmovl_u8(v_src.val[2]); + uint8x8_t v_gray = convertToGray(v_r, v_g, v_b, v_r2y, v_g2y, v_b2y); + + vst1_u8(dst + dj, v_gray); + sj += 32; dj += 8; + } +#endif + + for (; dj < size.width; sj += 4, dj++) + { + u32 val = src[sj] * B2Y + src[sj + 1] * G2Y + src[sj + 2] * R2Y; + dst[dj] = internal::saturate_cast((val + SHIFT_DELTA) >> SHIFT); + } + } +#else + (void)size; + (void)color_space; + (void)srcBase; + (void)srcStride; + (void)dstBase; + (void)dstStride; +#endif +} + +void gray2rgb(const Size2D &size, + const u8 * srcBase, ptrdiff_t srcStride, + u8 * dstBase, ptrdiff_t dstStride) +{ + internal::assertSupportedConfiguration(); +#ifdef CAROTENE_NEON + size_t roiw16 = size.width >= 15 ? size.width - 15 : 0; + size_t roiw8 = size.width >= 7 ? size.width - 7 : 0; + + for (size_t i = 0u; i < size.height; ++i) + { + const u8 * src = internal::getRowPtr(srcBase, srcStride, i); + u8 * dst = internal::getRowPtr(dstBase, dstStride, i); + size_t sj = 0u, dj = 0u; + + for (; sj < roiw16; sj += 16, dj += 48) + { + internal::prefetch(src + sj); +#if defined(__GNUC__) && __GNUC_MINOR__ < 7 + __asm__ ( + "vld1.8 {d0-d1}, [%[in0]] \n\t" + "vmov.8 q1, q0 \n\t" + "vmov.8 q2, q0 \n\t" + "vmov.8 q3, q1 \n\t" + "vst3.8 {d2, d4, d6}, [%[out0]] \n\t" + "vst3.8 {d3, d5, d7}, [%[out1]] \n\t" + : /*no output*/ + : [out0] "r" (dst + dj), [out1] "r" (dst + dj + 24), + [in0] "r" (src + sj) + : "d0","d1","d2","d3","d4","d5","d6","d7" + ); +#else + uint8x16x3_t vRgb1; + vRgb1.val[0] = vld1q_u8(src + sj); + + vRgb1.val[1] = vRgb1.val[0]; + vRgb1.val[2] = vRgb1.val[0]; + + vst3q_u8(dst + dj, vRgb1); +#endif + } + + if (sj < roiw8) + { +#if defined(__GNUC__) && __GNUC_MINOR__ < 7 + __asm__ ( + "vld1.8 {d0}, [%[in]] \n\t" + "vmov.8 d1, d0 \n\t" + "vmov.8 d2, d0 \n\t" + "vst3.8 {d0-d2}, [%[out]] \n\t" + : /*no output*/ + : [out] "r" (dst + dj), [in] "r" (src + sj) + : "d0","d1","d2" + ); +#else + uint8x8x3_t vRgb2; + vRgb2.val[0] = vld1_u8(src + sj); + vRgb2.val[1] = vRgb2.val[0]; + vRgb2.val[2] = vRgb2.val[0]; + + vst3_u8(dst + dj, vRgb2); +#endif + sj += 8; dj += 24; + } + + for (; sj < size.width; sj++, dj += 3) + { + dst[dj+0] = src[sj]; + dst[dj+1] = src[sj]; + dst[dj+2] = src[sj]; + } + } +#else + (void)size; + (void)srcBase; + (void)srcStride; + (void)dstBase; + (void)dstStride; +#endif +} + +void gray2rgbx(const Size2D &size, + const u8 * srcBase, ptrdiff_t srcStride, + u8 * dstBase, ptrdiff_t dstStride) +{ + internal::assertSupportedConfiguration(); +#ifdef CAROTENE_NEON + size_t roiw16 = size.width >= 15 ? size.width - 15 : 0; + size_t roiw8 = size.width >= 7 ? size.width - 7 : 0; + +#if defined(__GNUC__) && __GNUC_MINOR__ < 7 + register uint8x16_t vc255 asm ("q4") = vmovq_n_u8(255); +#else + uint8x16x4_t vRgba; + uint8x8x4_t vRgba2; + vRgba.val[3] = vmovq_n_u8(255); + vRgba2.val[3] = vget_low_u8(vRgba.val[3]); +#endif + + for (size_t i = 0u; i < size.height; ++i) + { + const u8 * src = internal::getRowPtr(srcBase, srcStride, i); + u8 * dst = internal::getRowPtr(dstBase, dstStride, i); + size_t sj = 0u, dj = 0u; + + for (; sj < roiw16; sj += 16, dj += 64) + { + internal::prefetch(src + sj); +#if defined(__GNUC__) && __GNUC_MINOR__ < 7 + __asm__ ( + "vld1.8 {d0-d1}, [%[in0]] \n\t" + "vmov.8 q1, q0 \n\t" + "vmov.8 q2, q0 \n\t" + "vmov.8 q3, q1 \n\t" + "vst4.8 {d2, d4, d6, d8}, [%[out0]] \n\t" + "vst4.8 {d3, d5, d7, d9}, [%[out1]] \n\t" + : /*no output*/ + : [out0] "r" (dst + dj), [out1] "r" (dst + dj + 32), + [in0] "r" (src + sj), + "w" (vc255) + : "d0","d1","d2","d3","d4","d5","d6","d7" + ); +#else + vRgba.val[0] = vld1q_u8(src + sj); + + vRgba.val[1] = vRgba.val[0]; + vRgba.val[2] = vRgba.val[0]; + + vst4q_u8(dst + dj, vRgba); +#endif + } + + if (sj < roiw8) + { +#if defined(__GNUC__) && __GNUC_MINOR__ < 7 + __asm__ ( + "vld1.8 {d5}, [%[in]] \n\t" + "vmov.8 d6, d5 \n\t" + "vmov.8 d7, d5 \n\t" + "vst4.8 {d5-d8}, [%[out]] \n\t" + : /*no output*/ + : [out] "r" (dst + dj), [in] "r" (src + sj), "w" (vc255) + : "d5","d6","d7" + ); +#else + vRgba2.val[0] = vld1_u8(src + sj); + vRgba2.val[1] = vRgba2.val[0]; + vRgba2.val[2] = vRgba2.val[0]; + + vst4_u8(dst + dj, vRgba2); +#endif + sj += 8; dj += 32; + } + + for (; sj < size.width; sj++, dj += 4) + { + dst[dj+0] = src[sj]; + dst[dj+1] = src[sj]; + dst[dj+2] = src[sj]; + dst[dj+3] = 255; + } + } +#else + (void)size; + (void)srcBase; + (void)srcStride; + (void)dstBase; + (void)dstStride; +#endif +} + +void rgb2rgbx(const Size2D &size, + const u8 * srcBase, ptrdiff_t srcStride, + u8 * dstBase, ptrdiff_t dstStride) +{ + internal::assertSupportedConfiguration(); +#ifdef CAROTENE_NEON + size_t roiw8 = size.width >= 7 ? size.width - 7 : 0; +#if defined(__GNUC__) && defined(__arm__) + register uint8x8_t vc255_0 asm ("d3") = vmov_n_u8(255); +#else + size_t roiw16 = size.width >= 15 ? size.width - 15 : 0; + union { uint8x16x4_t v4; uint8x16x3_t v3; } v_dst0; + v_dst0.v4.val[3] = vdupq_n_u8(255); + union { uint8x8x4_t v4; uint8x8x3_t v3; } v_dst; + v_dst.v4.val[3] = vdup_n_u8(255); +#endif + + for (size_t i = 0u; i < size.height; ++i) + { + const u8 * src = internal::getRowPtr(srcBase, srcStride, i); + u8 * dst = internal::getRowPtr(dstBase, dstStride, i); + size_t sj = 0u, dj = 0u, j = 0u; + +#if defined(__GNUC__) && defined(__arm__) + for (; j < roiw8; sj += 24, dj += 32, j += 8) + { + internal::prefetch(src + sj); + __asm__ ( + "vld3.8 {d0, d1, d2}, [%[in0]] \n\t" + "vst4.8 {d0, d1, d2, d3}, [%[out0]] \n\t" + : /*no output*/ + : [out0] "r" (dst + dj), + [in0] "r" (src + sj), + "w" (vc255_0) + : "d0","d1","d2" + ); + } +#else + for (; j < roiw16; sj += 48, dj += 64, j += 16) + { + internal::prefetch(src + sj); + v_dst0.v3 = vld3q_u8(src + sj); + vst4q_u8(dst + dj, v_dst0.v4); + } + + if (j < roiw8) + { + v_dst.v3 = vld3_u8(src + sj); + vst4_u8(dst + dj, v_dst.v4); + sj += 24; dj += 32; j += 8; + } +#endif + + for (; j < size.width; ++j, sj += 3, dj += 4) + { + dst[dj] = src[sj]; + dst[dj + 1] = src[sj + 1]; + dst[dj + 2] = src[sj + 2]; + dst[dj + 3] = 255; + } + } +#else + (void)size; + (void)srcBase; + (void)srcStride; + (void)dstBase; + (void)dstStride; +#endif +} + +void rgbx2rgb(const Size2D &size, + const u8 * srcBase, ptrdiff_t srcStride, + u8 * dstBase, ptrdiff_t dstStride) +{ + internal::assertSupportedConfiguration(); +#ifdef CAROTENE_NEON + size_t roiw8 = size.width >= 7 ? size.width - 7 : 0; +#if !defined(__GNUC__) || !defined(__arm__) + size_t roiw16 = size.width >= 15 ? size.width - 15 : 0; + union { uint8x16x4_t v4; uint8x16x3_t v3; } v_dst0; + union { uint8x8x4_t v4; uint8x8x3_t v3; } v_dst; +#endif + + for (size_t i = 0u; i < size.height; ++i) + { + const u8 * src = internal::getRowPtr(srcBase, srcStride, i); + u8 * dst = internal::getRowPtr(dstBase, dstStride, i); + size_t sj = 0u, dj = 0u, j = 0u; + +#if defined(__GNUC__) && defined(__arm__) + for (; j < roiw8; sj += 32, dj += 24, j += 8) + { + internal::prefetch(src + sj); + __asm__ ( + "vld4.8 {d0, d1, d2, d3}, [%[in0]] \n\t" + "vst3.8 {d0, d1, d2}, [%[out0]] \n\t" + : /*no output*/ + : [out0] "r" (dst + dj), + [in0] "r" (src + sj) + : "d0","d1","d2","d3" + ); + } +#else + for (; j < roiw16; sj += 64, dj += 48, j += 16) + { + internal::prefetch(src + sj); + v_dst0.v4 = vld4q_u8(src + sj); + vst3q_u8(dst + dj, v_dst0.v3); + } + + if (j < roiw8) + { + v_dst.v4 = vld4_u8(src + sj); + vst3_u8(dst + dj, v_dst.v3); + sj += 32; dj += 24; j += 8; + } +#endif + + for (; j < size.width; ++j, sj += 4, dj += 3) + { + dst[dj] = src[sj]; + dst[dj + 1] = src[sj + 1]; + dst[dj + 2] = src[sj + 2]; + } + } +#else + (void)size; + (void)srcBase; + (void)srcStride; + (void)dstBase; + (void)dstStride; +#endif +} + +void rgb2bgr(const Size2D &size, + const u8 * srcBase, ptrdiff_t srcStride, + u8 * dstBase, ptrdiff_t dstStride) +{ + internal::assertSupportedConfiguration(); +#ifdef CAROTENE_NEON +#if !defined(__GNUC__) || !defined(__arm__) + size_t roiw16 = size.width >= 15 ? size.width - 15 : 0; +#endif + size_t roiw8 = size.width >= 7 ? size.width - 7 : 0; + + for (size_t i = 0u; i < size.height; ++i) + { + const u8 * src = internal::getRowPtr(srcBase, srcStride, i); + u8 * dst = internal::getRowPtr(dstBase, dstStride, i); + size_t sj = 0u, dj = 0u, j = 0u; + + +#if defined(__GNUC__) && defined(__arm__) + for (; j < roiw8; sj += 24, dj += 24, j += 8) + { + internal::prefetch(src + sj); + __asm__ ( + "vld3.8 {d0, d1, d2}, [%[in0]] \n\t" + "vswp d0, d2 \n\t" + "vst3.8 {d0, d1, d2}, [%[out0]] \n\t" + : /*no output*/ + : [out0] "r" (dst + dj), + [in0] "r" (src + sj) + : "d0","d1","d2" + ); + } +#else + for (; j < roiw16; sj += 48, dj += 48, j += 16) + { + internal::prefetch(src + sj); + uint8x16x3_t vals0 = vld3q_u8(src + sj); + + std::swap(vals0.val[0], vals0.val[2]); + + vst3q_u8(dst + dj, vals0); + } + + if (j < roiw8) + { + uint8x8x3_t vals = vld3_u8(src + sj); + std::swap(vals.val[0], vals.val[2]); + vst3_u8(dst + dj, vals); + sj += 24; dj += 24; j += 8; + } +#endif + + for (; j < size.width; ++j, sj += 3, dj += 3) + { + u8 b = src[sj + 2];//Handle src == dst case + dst[dj + 2] = src[sj ]; + dst[dj + 1] = src[sj + 1]; + dst[dj ] = b; + } + } +#else + (void)size; + (void)srcBase; + (void)srcStride; + (void)dstBase; + (void)dstStride; +#endif +} + +void rgbx2bgrx(const Size2D &size, + const u8 * srcBase, ptrdiff_t srcStride, + u8 * dstBase, ptrdiff_t dstStride) +{ + internal::assertSupportedConfiguration(); +#ifdef CAROTENE_NEON +#if !defined(__GNUC__) || !defined(__arm__) + size_t roiw16 = size.width >= 15 ? size.width - 15 : 0; +#endif + size_t roiw8 = size.width >= 7 ? size.width - 7 : 0; + + for (size_t i = 0u; i < size.height; ++i) + { + const u8 * src = internal::getRowPtr(srcBase, srcStride, i); + u8 * dst = internal::getRowPtr(dstBase, dstStride, i); + size_t sj = 0u, dj = 0u, j = 0u; + +#if defined(__GNUC__) && defined(__arm__) + for (; j < roiw8; sj += 32, dj += 32, j += 8) + { + internal::prefetch(src + sj); + __asm__ ( + "vld4.8 {d0, d1, d2, d3}, [%[in0]] \n\t" + "vswp d0, d2 \n\t" + "vst4.8 {d0, d1, d2, d3}, [%[out0]] \n\t" + : /*no output*/ + : [out0] "r" (dst + dj), + [in0] "r" (src + sj) + : "d0","d1","d2","d3" + ); + } +#else + for (; j < roiw16; sj += 64, dj += 64, j += 16) + { + internal::prefetch(src + sj); + uint8x16x4_t vals0 = vld4q_u8(src + sj); + + std::swap(vals0.val[0], vals0.val[2]); + + vst4q_u8(dst + dj, vals0); + } + + if (j < roiw8) + { + uint8x8x4_t vals = vld4_u8(src + sj); + std::swap(vals.val[0], vals.val[2]); + vst4_u8(dst + dj, vals); + sj += 32; dj += 32; j += 8; + } +#endif + + for (; j < size.width; ++j, sj += 4, dj += 4) + { + u8 b = src[sj + 2];//Handle src == dst case + dst[dj + 2] = src[sj ]; + dst[dj + 1] = src[sj + 1]; + dst[dj ] = b; + dst[dj + 3] = src[sj + 3]; + } + } +#else + (void)size; + (void)srcBase; + (void)srcStride; + (void)dstBase; + (void)dstStride; +#endif +} + +void rgbx2bgr(const Size2D &size, + const u8 * srcBase, ptrdiff_t srcStride, + u8 * dstBase, ptrdiff_t dstStride) +{ + internal::assertSupportedConfiguration(); +#ifdef CAROTENE_NEON +#if !defined(__GNUC__) || !defined(__arm__) + size_t roiw16 = size.width >= 15 ? size.width - 15 : 0; +#endif + size_t roiw8 = size.width >= 7 ? size.width - 7 : 0; + + for (size_t i = 0u; i < size.height; ++i) + { + const u8 * src = internal::getRowPtr(srcBase, srcStride, i); + u8 * dst = internal::getRowPtr(dstBase, dstStride, i); + size_t sj = 0u, dj = 0u, j = 0u; + +#if defined(__GNUC__) && defined(__arm__) + for (; j < roiw8; sj += 32, dj += 24, j += 8) + { + internal::prefetch(src + sj); + __asm__ ( + "vld4.8 {d0, d1, d2, d3}, [%[in0]] \n\t" + "vswp d0, d2 \n\t" + "vst3.8 {d0, d1, d2}, [%[out0]] \n\t" + : /*no output*/ + : [out0] "r" (dst + dj), + [in0] "r" (src + sj) + : "d0","d1","d2","d3" + ); + } +#else + for (; j < roiw16; sj += 64, dj += 48, j += 16) + { + internal::prefetch(src + sj); + union { uint8x16x4_t v4; uint8x16x3_t v3; } vals0; + vals0.v4 = vld4q_u8(src + sj); + std::swap(vals0.v3.val[0], vals0.v3.val[2]); + vst3q_u8(dst + dj, vals0.v3); + } + + if (j < roiw8) + { + union { uint8x8x4_t v4; uint8x8x3_t v3; } vals; + vals.v4 = vld4_u8(src + sj); + std::swap(vals.v3.val[0], vals.v3.val[2]); + vst3_u8(dst + dj, vals.v3); + sj += 32; dj += 24; j += 8; + } +#endif + + for (; j < size.width; ++j, sj += 4, dj += 3) + { + dst[dj + 2] = src[sj ]; + dst[dj + 1] = src[sj + 1]; + dst[dj ] = src[sj + 2]; + } + } +#else + (void)size; + (void)srcBase; + (void)srcStride; + (void)dstBase; + (void)dstStride; +#endif +} + +void rgb2bgrx(const Size2D &size, + const u8 * srcBase, ptrdiff_t srcStride, + u8 * dstBase, ptrdiff_t dstStride) +{ + internal::assertSupportedConfiguration(); +#ifdef CAROTENE_NEON +#if defined(__GNUC__) && defined(__arm__) + register uint8x8_t vc255 asm ("d3") = vmov_n_u8(255); +#else + union { uint8x16x4_t v4; uint8x16x3_t v3; } vals0; + vals0.v4.val[3] = vmovq_n_u8(255); + union { uint8x8x4_t v4; uint8x8x3_t v3; } vals8; + vals8.v4.val[3] = vmov_n_u8(255); +#endif + +#if !defined(__GNUC__) || !defined(__arm__) + size_t roiw16 = size.width >= 15 ? size.width - 15 : 0; +#endif + size_t roiw8 = size.width >= 7 ? size.width - 7 : 0; + + for (size_t i = 0u; i < size.height; ++i) + { + const u8 * src = internal::getRowPtr(srcBase, srcStride, i); + u8 * dst = internal::getRowPtr(dstBase, dstStride, i); + size_t sj = 0u, dj = 0u, j = 0u; + +#if defined(__GNUC__) && defined(__arm__) + for (; j < roiw8; sj += 24, dj += 32, j += 8) + { + internal::prefetch(src + sj); + __asm__ ( + "vld3.8 {d0, d1, d2}, [%[in0]] \n\t" + "vswp d0, d2 \n\t" + "vst4.8 {d0, d1, d2, d3}, [%[out0]] \n\t" + : /*no output*/ + : [out0] "r" (dst + dj), + [in0] "r" (src + sj), + "w" (vc255) + : "d0","d1","d2" + ); + } +#else + for (; j < roiw16; sj += 48, dj += 64, j += 16) + { + internal::prefetch(src + sj); + vals0.v3 = vld3q_u8(src + sj); + std::swap(vals0.v4.val[0], vals0.v4.val[2]); + vst4q_u8(dst + dj, vals0.v4); + } + + if (j < roiw8) + { + vals8.v3 = vld3_u8(src + sj); + std::swap(vals8.v4.val[0], vals8.v4.val[2]); + vst4_u8(dst + dj, vals8.v4); + sj += 24; dj += 32; j += 8; + } +#endif + + for (; j < size.width; ++j, sj += 3, dj += 4) + { + dst[dj + 3] = 255; + dst[dj + 2] = src[sj ]; + dst[dj + 1] = src[sj + 1]; + dst[dj ] = src[sj + 2]; + } + } +#else + (void)size; + (void)srcBase; + (void)srcStride; + (void)dstBase; + (void)dstStride; +#endif +} + +namespace { + +#ifdef CAROTENE_NEON +inline uint8x8x3_t convertToHSV(const uint8x8_t vR, const uint8x8_t vG, const uint8x8_t vB, + const s32 hrange ) +{ + const s32 hsv_shift = 12; + register const f32 vsdiv_table = f32(255 << hsv_shift); + register f32 vhdiv_table = f32(hrange << hsv_shift); + register const s32 vhrange = hrange; + register const s32 v0 = s32(0); + register const s32 vshift = s32(1 << (hsv_shift-1)); + register const s32 v6 = s32(6); + + uint8x8_t vMin = vmin_u8(vR, vG); + uint8x8_t vMax = vmax_u8(vR, vG); + + uint16x8_t vR_u16 = vmovl_u8(vR); + uint16x8_t vG_u16 = vmovl_u8(vG); + + vMax = vmax_u8(vMax, vB); + vMin = vmin_u8(vMin, vB); + uint16x8_t vB_u16 = vmovl_u8(vB); + + uint16x8_t vDiff = vsubl_u8(vMax, vMin); + + uint16x8_t vV = vmovl_u8(vMax); + uint16x8_t vDiffx2 = vaddq_u16(vDiff, vDiff); + uint32x4_t vDiffL = vmovl_u16(vget_low_u16(vDiff)); + uint32x4_t vDiffH = vmovl_u16(vget_high_u16(vDiff)); + + uint16x8_t vVEqR = vceqq_u16(vR_u16, vV); + uint16x8_t vVEqG = vceqq_u16(vG_u16, vV); + + int16x8_t vG_B = vsubq_s16(vreinterpretq_s16_u16(vG_u16), vreinterpretq_s16_u16(vB_u16)); + uint16x8_t vInvR = vmvnq_u16(vVEqR); + int16x8_t vB_R = vsubq_s16(vreinterpretq_s16_u16(vB_u16), vreinterpretq_s16_u16(vR_u16)); + int16x8_t vR_G = vsubq_s16(vreinterpretq_s16_u16(vR_u16), vreinterpretq_s16_u16(vG_u16)); + + uint16x8_t vMask2 = vandq_u16(vVEqG, vInvR); + vR_u16 = vandq_u16(vreinterpretq_u16_s16(vG_B), vVEqR); + int16x8_t vH2 = vaddq_s16(vB_R, vreinterpretq_s16_u16(vDiffx2)); + + vVEqR = vmvnq_u16(vVEqG); + vB_R = vaddq_s16(vreinterpretq_s16_u16(vDiffx2), vreinterpretq_s16_u16(vDiffx2)); + vG_B = vandq_s16(vreinterpretq_s16_u16(vInvR), vreinterpretq_s16_u16(vVEqR)); + vInvR = vandq_u16(vreinterpretq_u16_s16(vH2), vMask2); + vR_G = vaddq_s16(vR_G, vB_R); + int16x8_t vH = vaddq_s16(vreinterpretq_s16_u16(vR_u16), vreinterpretq_s16_u16(vInvR)); + + uint32x4_t vV_L = vmovl_u16(vget_low_u16(vV)); + vR_G = vandq_s16(vR_G, vG_B); + uint32x4_t vV_H = vmovl_u16(vget_high_u16(vV)); + int16x8_t vDiff4 = vaddq_s16(vH, vR_G); + + int32x4_t vc6 = vdupq_n_s32(v6); + uint32x4_t vLine1 = vmulq_u32(vDiffL, vreinterpretq_u32_s32(vc6)); + uint32x4_t vLine2 = vmulq_u32(vDiffH, vreinterpretq_u32_s32(vc6)); + + float32x4_t vF1 = vcvtq_f32_u32(vV_L); + float32x4_t vF2 = vcvtq_f32_u32(vV_H); + float32x4_t vHF1 = vcvtq_f32_u32(vLine1); + float32x4_t vHF2 = vcvtq_f32_u32(vLine2); + + float32x4_t vXInv1 = vrecpeq_f32(vF1); + float32x4_t vXInv2 = vrecpeq_f32(vF2); + float32x4_t vXInv3 = vrecpeq_f32(vHF1); + float32x4_t vXInv4 = vrecpeq_f32(vHF2); + + float32x4_t vSt1 = vrecpsq_f32(vXInv1, vF1); + float32x4_t vSt2 = vrecpsq_f32(vXInv2, vF2); + float32x4_t vSt3 = vrecpsq_f32(vXInv3, vHF1); + float32x4_t vSt4 = vrecpsq_f32(vXInv4, vHF2); + + vF1 = vmulq_f32(vXInv1, vSt1); + vF2 = vmulq_f32(vXInv2, vSt2); + vHF1 = vmulq_f32(vXInv3, vSt3); + vHF2 = vmulq_f32(vXInv4, vSt4); + + float32x4_t vDivTab = vdupq_n_f32(vsdiv_table); + vSt1 = vmulq_f32(vF1, vDivTab); + vSt2 = vmulq_f32(vF2, vDivTab); + vDivTab = vdupq_n_f32(vhdiv_table); + vSt3 = vmulq_f32(vHF1, vDivTab); + vSt4 = vmulq_f32(vHF2, vDivTab); + + float32x4_t bias = vdupq_n_f32(0.5f); + + vSt1 = vaddq_f32(vSt1, bias); + vSt2 = vaddq_f32(vSt2, bias); + vSt3 = vaddq_f32(vSt3, bias); + vSt4 = vaddq_f32(vSt4, bias); + + uint32x4_t vRes1 = vcvtq_u32_f32(vSt1); + uint32x4_t vRes2 = vcvtq_u32_f32(vSt2); + uint32x4_t vRes3 = vcvtq_u32_f32(vSt3); + uint32x4_t vRes4 = vcvtq_u32_f32(vSt4); + + int32x4_t vH_L = vmovl_s16(vget_low_s16(vDiff4)); + int32x4_t vH_H = vmovl_s16(vget_high_s16(vDiff4)); + + uint32x4_t vDiff_Res1 = vmulq_u32(vDiffL, vRes1); + uint32x4_t vDiff_Res2 = vmulq_u32(vDiffH, vRes2); + uint32x4_t vDiff_Res3 = vmulq_u32(vreinterpretq_u32_s32(vH_L), vRes3); + uint32x4_t vDiff_Res4 = vmulq_u32(vreinterpretq_u32_s32(vH_H), vRes4); + + int32x4_t vShift = vdupq_n_s32(vshift); + uint32x4_t vAddRes1 = vaddq_u32(vDiff_Res1, vreinterpretq_u32_s32(vShift)); + uint32x4_t vAddRes2 = vaddq_u32(vDiff_Res2, vreinterpretq_u32_s32(vShift)); + uint32x4_t vAddRes3 = vaddq_u32(vDiff_Res3, vreinterpretq_u32_s32(vShift)); + uint32x4_t vAddRes4 = vaddq_u32(vDiff_Res4, vreinterpretq_u32_s32(vShift)); + int16x4_t vShrRes1 = vshrn_n_s32(vreinterpretq_s32_u32(vAddRes1), 8); + int16x4_t vShrRes2 = vshrn_n_s32(vreinterpretq_s32_u32(vAddRes2), 8); + int16x4_t vShrRes3 = vshrn_n_s32(vreinterpretq_s32_u32(vAddRes3), 8); + int16x4_t vShrRes4 = vshrn_n_s32(vreinterpretq_s32_u32(vAddRes4), 8); + + int16x8_t vc0 = vdupq_n_s16((s16)v0); + int8x8_t vShrRes1_s8 = vshrn_n_s16(vcombine_s16(vShrRes1, vShrRes2), 4); + uint16x8_t vCltRes_u16 = vcltq_s16(vcombine_s16(vShrRes3, vShrRes4), vc0); + int8x8_t vShrRes2_s8 = vshrn_n_s16(vcombine_s16(vShrRes3, vShrRes4), 4); + + int8x8_t vCltRes_s8 = vmovn_s16(vreinterpretq_s16_u16(vCltRes_u16)); + int8x8_t vcHRange = vdup_n_s8((s8)vhrange); + uint8x8_t vHResAdd = vand_u8(vreinterpret_u8_s8(vCltRes_s8), vreinterpret_u8_s8(vcHRange)); + int8x8_t vHRes = vadd_s8(vShrRes2_s8, vreinterpret_s8_u8(vHResAdd)); + + uint8x8x3_t vHsv; + vHsv.val[0] = vreinterpret_u8_s8(vHRes); + vHsv.val[1] = vreinterpret_u8_s8(vShrRes1_s8); + vHsv.val[2] = vMax; + + return vHsv; +} + +const u8 fastSaturate8u[] = +{ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, + 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, + 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, + 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, + 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, + 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, + 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, + 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, + 144, 145, 146, 147, 148, 149, 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, + 160, 161, 162, 163, 164, 165, 166, 167, 168, 169, 170, 171, 172, 173, 174, 175, + 176, 177, 178, 179, 180, 181, 182, 183, 184, 185, 186, 187, 188, 189, 190, 191, + 192, 193, 194, 195, 196, 197, 198, 199, 200, 201, 202, 203, 204, 205, 206, 207, + 208, 209, 210, 211, 212, 213, 214, 215, 216, 217, 218, 219, 220, 221, 222, 223, + 224, 225, 226, 227, 228, 229, 230, 231, 232, 233, 234, 235, 236, 237, 238, 239, + 240, 241, 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, 254, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255 +}; + +inline void convertToHSV(const s32 r, const s32 g, const s32 b, + const s32 &hrange, const s32 &hsv_shift, + u8* dst) +{ + s32 h, s, v = b; + s32 vmin = b, diff; + s32 vr, vg; + + v += fastSaturate8u[g-v+256]; + v += fastSaturate8u[r-v+256]; + vmin -= fastSaturate8u[vmin-g+256]; + vmin -= fastSaturate8u[vmin-r+256]; + + diff = v - vmin; + vr = v == r ? -1 : 0; + vg = v == g ? -1 : 0; + + s = (s32(diff * (255 << hsv_shift) * (1.0f/(f32)v)) + (1 << (hsv_shift-1))) >> hsv_shift; + h = (vr & (g - b)) + (~vr & ((vg & (b - r + 2 * diff)) + ((~vg) & (r - g + 4 * diff)))); + h = ((h * s32((hrange << hsv_shift)/(6.f*diff) + 0.5)) + (1 << (hsv_shift-1))) >> hsv_shift; + h += h < 0 ? hrange : 0; + + dst[0] = internal::saturate_cast(h); + dst[1] = (u8)s; + dst[2] = (u8)v; +} + +#define CONVERT_TO_HSV_ASM(loadop, rreg, breg) \ + __asm__ ( \ + #loadop ", [%[in]] @RGB \n\t" \ + "vmin.u8 d3, d0, d1 @VMin (d3) \n\t" \ + "vmax.u8 d6, d0, d1 @V (d6) \n\t" \ + "vmovl.u8 q2, " #rreg " @V16_R (d4,d5) \n\t" \ + "vmovl.u8 q4, d1 @V16_G (d8,d9) \n\t" \ + "vmax.u8 d6, d6, d2 \n\t" \ + "vmin.u8 d3, d3, d2 \n\t" \ + "vmovl.u8 q0, " #breg " @V16_B (d0,d1) \n\t" \ + "vsubl.u8 q8, d6, d3 @V16_Diff (d16,d17) \n\t" \ + \ + "vmovl.u8 q5, d6 @V16_V (d10,d11) \n\t" \ + "vadd.s16 q10, q8, q8 @V16_Diff_2 (d20,d21) \n\t" \ + "vmovl.u16 q9, d16 @V32_Diff_L (d18,d19) \n\t" \ + "vmovl.u16 q11, d17 @V32_Diff_H (d22,d23) \n\t" \ + "vceq.u16 q12, q2, q5 @V==R(d24,d25) \n\t" \ + "vceq.u16 q13, q4, q5 @V==G(d26,d27) \n\t" \ + \ + "vsub.s16 q8, q4, q0 @V16_G-B (d16,d17) \n\t" \ + "vmvn.u16 q15, q12 @V16~R \n\t" \ + "vsub.s16 q6, q0, q2 @V16_B-R (d12,d13) \n\t" \ + "vsub.s16 q7, q2, q4 @V16_R-G (d14,d15) \n\t" \ + "vand.u16 q1, q13, q15 @VMask2 \n\t" \ + "vand.u16 q2, q8, q12 @V16_H(d4,d5) \n\t" \ + "vadd.s16 q4, q6, q10 @V16_H2 \n\t" \ + "vmvn.u16 q12, q13 @V16~G \n\t" \ + "vadd.s16 q6, q10, q10 @VDiff16_4 (d12,d13) \n\t" \ + "vand.u16 q8, q15, q12 @VMask3 \n\t" \ + "vand.u16 q15, q4, q1 @vH2(d30,d31) \n\t" \ + "vadd.s16 q7, q7, q6 @V16_H3 (d14,d15) \n\t" \ + "vadd.s16 q14, q2, q15 @vH16 \n\t" \ + "vmovl.u16 q12, d10 @V32_V_L \n\t" \ + "vand.s16 q7, q7, q8 @vH16 \n\t" \ + "vmovl.u16 q13, d11 @V32_V_H \n\t" \ + "vadd.s16 q2, q14, q7 @V16_Diff_4 \n\t" \ + \ + "vdup.32 q4, %[v6] \n\t" \ + "vmul.u32 q14, q9, q4 \n\t" \ + "vmul.u32 q15, q11, q4 \n\t" \ + "vcvt.f32.u32 q4, q12 @VF1 (d8,d9) \n\t" \ + "vcvt.f32.u32 q8, q13 @VF2 \n\t" \ + "vcvt.f32.u32 q0, q14 @HF1 \n\t" \ + "vcvt.f32.u32 q1, q15 @HF2 \n\t" \ + "vrecpe.f32 q12, q4 @Vxinv \n\t" \ + "vrecpe.f32 q13, q8 @Vxinv \n\t" \ + "vrecpe.f32 q5, q0 @Vxinv \n\t" \ + "vrecpe.f32 q7, q1 @Vxinv \n\t" \ + "vrecps.f32 q14, q12, q4 @Vst1 \n\t" \ + "vrecps.f32 q15, q13, q8 @Vst1 \n\t" \ + "vrecps.f32 q10, q5, q0 @Vst1 \n\t" \ + "vrecps.f32 q6, q7, q1 @Vst1 \n\t" \ + "vmul.f32 q4, q12, q14 \n\t" \ + "vmul.f32 q8, q13, q15 \n\t" \ + "vmul.f32 q0, q5, q10 \n\t" \ + "vmul.f32 q1, q7, q6 \n\t" \ + "vdup.32 q12, %[vsdiv_table] \n\t" \ + "vmul.f32 q14, q4, q12 \n\t" \ + "vmul.f32 q15, q8, q12 \n\t" \ + "vdup.32 q12, %[vhdiv_table] \n\t" \ + "vmul.f32 q10, q0, q12 \n\t" \ + "vmul.f32 q6, q1, q12 \n\t" \ + \ + "vdup.32 q12, %[bias] \n\t" \ + \ + "vadd.f32 q7, q14, q12 \n\t" \ + "vadd.f32 q13, q15, q12 \n\t" \ + "vcvt.u32.f32 q4, q7 \n\t" \ + "vcvt.u32.f32 q8, q13 \n\t" \ + \ + "vadd.f32 q14, q10, q12 \n\t" \ + "vadd.f32 q7, q6, q12 \n\t" \ + "vcvt.u32.f32 q0, q14 \n\t" \ + "vcvt.u32.f32 q1, q7 @Vres \n\t" \ + \ + "vmovl.s16 q7, d4 @V32_H_L (d14,d15) \n\t" \ + "vmovl.s16 q5, d5 @V32_H_H (d10,d11) \n\t" \ + "vmul.u32 q14, q9, q4 \n\t" \ + "vmul.u32 q15, q11, q8 \n\t" \ + "vmul.u32 q10, q7, q0 \n\t" \ + "vmul.u32 q6, q5, q1 \n\t" \ + \ + "vdup.32 q12, %[vshift] \n\t" \ + "vadd.u32 q13, q14, q12 \n\t" \ + "vadd.u32 q8, q15, q12 \n\t" \ + "vadd.u32 q0, q10, q12 \n\t" \ + "vadd.u32 q1, q6, q12 \n\t" \ + "vshrn.s32 d8, q13, #8 \n\t" \ + "vshrn.s32 d9, q8, #8 \n\t" \ + "vshrn.s32 d10, q0, #8 \n\t" \ + "vshrn.s32 d11, q1, #8 \n\t" \ + \ + "vdup.16 q8, %[v0] \n\t" \ + "vshrn.s16 d5, q4, #4 \n\t" \ + "vclt.s16 q9, q5, q8 \n\t" \ + "vshrn.s16 d4, q5, #4 \n\t" \ + \ + "vmovn.s16 d9, q9 \n\t" \ + "vdup.8 d7, %[vhrange] \n\t" \ + "vand.u8 d10, d9, d7 \n\t" \ + "vadd.s8 d4, d4, d10 \n\t" \ + "vst3.8 {d4-d6}, [%[out]] @HSV \n\t" \ + : /*no output*/ \ + : [out] "r" (dst + dj), [in] "r" (src + sj), \ + [vsdiv_table] "r" (vsdiv_table), \ + [vshift] "r" (vshift), \ + [vhdiv_table] "r" (vhdiv_table), \ + [v6] "r" (v6), [vhrange] "r" (vhrange), \ + [v0] "r" (v0), [bias] "r" (bias) \ + : "d0","d1","d2","d3","d4","d5","d6","d7", \ + "d8","d9","d10","d11","d12","d13","d14","d15", \ + "d16","d17","d18","d19","d20","d21","d22","d23", \ + "d24","d25","d26","d27","d28","d29","d30","d31" \ + ); + +#if __GNUC_MINOR__ < 7 + +#define YCRCB_CONSTS \ + register int16x4_t vcYR asm ("d31") = vmov_n_s16(4899); \ + register int16x4_t vcYG asm ("d30") = vmov_n_s16(9617); \ + register int16x4_t vcYB asm ("d29") = vmov_n_s16(1868); \ + register int16x4_t vcCrG asm ("d28") = vmov_n_s16(6860); \ + register int16x4_t vcCrB asm ("d27") = vmov_n_s16(1332); \ + register int16x4_t vcCbR asm ("d26") = vmov_n_s16(2765); \ + register int16x4_t vcCbG asm ("d25") = vmov_n_s16(5427); + +#else + +#define YCRCB_CONSTS \ + const s16 convertCoeffs[] = { 4899, 4899, 4899, 4899, \ + 9617, 9617, 9617, 9617, \ + 1868, 1868, 1868, 1868, \ + 6860, 6860, 6860, 6860, \ + 1332, 1332, 1332, 1332, \ + 2765, 2765, 2765, 2765, \ + 5427, 5427, 5427, 5427 }; \ + const int16x8_t vcYRG = vld1q_s16(convertCoeffs); /*YR and YG*/ \ + const int16x4_t vcYB = vld1_s16(convertCoeffs + 8); /*YB*/ \ + const int16x8_t vcCrGB = vld1q_s16(convertCoeffs + 12); /*CrG and CrB*/ \ + const int16x8_t vcCbRG = vld1q_s16(convertCoeffs + 20); /*CbR and CbG*/ + +#endif + +#define CONVERTTOYCRCB(loadcmd, rreg, greg, breg) \ + __asm__ ( \ + #loadcmd ", [%[in]] @RGB \n\t" \ + "vmovl.u8 q2, " #rreg " @R (d4,d5) \n\t" \ + "vmovl.u8 q3, " #greg " @G (d6,d7) \n\t" \ + "vmovl.u8 q4, " #breg " @B (d8,d9) \n\t" \ + \ + "vshll.u16 q7, d4, #13 @Cr(q7,q8): R \n\t" \ + "vmull.u16 q5, d6, d30 @Y (q5,q6): G \n\t" \ + "vshll.u16 q9, d8, #13 @Cb(q9,q10): B \n\t" \ + "vshll.u16 q8, d5, #13 @Cr(q7,q8): R \n\t" \ + "vmull.u16 q6, d7, d30 @Y (q5,q6): G \n\t" \ + "vshll.u16 q10, d9, #13 @Cb(q9,q10): B \n\t" \ + \ + "vmlsl.s16 q7, d6, d28 @Cr(q7,q8): RG \n\t" \ + "vmlal.s16 q5, d8, d29 @Y (q5,q6): GB \n\t" \ + "vmlsl.s16 q9, d4, d26 @Cb(q9,q10): BR \n\t" \ + "vmlsl.s16 q8, d7, d28 @Cr(q7,q8): RG \n\t" \ + "vmlal.s16 q6, d9, d29 @Y (q5,q6): GB \n\t" \ + "vmlsl.s16 q10, d5, d26 @Cb(q9,q10): BR \n\t" \ + \ + "vmlsl.s16 q7, d8, d27 @Cr(q7,q8): RGB \n\t" \ + "vmlal.s16 q5, d4, d31 @Y (q5,q6): GBR \n\t" \ + "vmlsl.s16 q9, d6, d25 @Cb(q9,q10): BRG \n\t" \ + "vmlsl.s16 q8, d9, d27 @Cr(q7,q8): RGB \n\t" \ + "vmlal.s16 q6, d5, d31 @Y (q5,q6): GBR \n\t" \ + "vmlsl.s16 q10, d7, d25 @Cb(q9,q10): BRG \n\t" \ + \ + "vrshrn.s32 d4, q7, #14 @Cr -> q2 \n\t" \ + "vrshrn.s32 d8, q5, #14 @Y -> q4 \n\t" \ + "vrshrn.s32 d6, q9, #14 @Cb -> q3 \n\t" \ + "vrshrn.s32 d5, q8, #14 @Cr -> q2 \n\t" \ + "vrshrn.s32 d9, q6, #14 @Y -> q4 \n\t" \ + "vrshrn.s32 d7, q10, #14 @Cb -> q3 \n\t" \ + \ + "vmov.s16 q5, #128 \n\t" \ + "vmov.s16 q6, #128 \n\t" \ + "vadd.i16 q5, q2 @Cr -> q5 \n\t" \ + "vadd.i16 q6, q3 @Cb -> q6 \n\t" \ + \ + "vqmovn.u16 d4, q4 \n\t" \ + "vqmovun.s16 d5, q5 \n\t" \ + "vqmovun.s16 d6, q6 \n\t" \ + \ + "vst3.8 {d4-d6}, [%[out]] \n\t" \ + : /*no output*/ \ + : [out] "r" (dst + dj), [in] "r" (src + sj), \ + "w" (vcYR), "w" (vcYG), "w" (vcYB), \ + "w" (vcCrB), "w" (vcCrG), "w" (vcCbG), "w" (vcCbR) \ + : "d0","d1","d2","d3","d4","d5","d6","d7", \ + "d8","d9","d10","d11","d12","d13","d14","d15", \ + "d16","d17","d18","d19","d20","d21" \ + ); + + +inline uint8x8x3_t convertToYCrCb( const int16x8_t& vR, const int16x8_t& vG, const int16x8_t& vB, + const int16x8_t& vcYRG, const int16x4_t& vcYB, + const int16x8_t& vcCrGB, const int16x8_t& vcCbRG ) +{ + int32x4_t vCrL = vshll_n_s16(vget_low_s16(vR), 13); // R + int32x4_t vCrH = vshll_n_s16(vget_high_s16(vR), 13); // R + int32x4_t vYL = vmull_s16(vget_low_s16(vG), vget_high_s16(vcYRG)); // G + int32x4_t vYH = vmull_s16(vget_high_s16(vG), vget_high_s16(vcYRG)); // G + int32x4_t vCbL = vshll_n_s16(vget_low_s16(vB), 13); // B + int32x4_t vCbH = vshll_n_s16(vget_high_s16(vB), 13); // B + + vCrL = vmlsl_s16(vCrL, vget_low_s16(vG), vget_low_s16(vcCrGB)); // RG + vCrH = vmlsl_s16(vCrH, vget_high_s16(vG), vget_low_s16(vcCrGB)); // RG + vYL = vmlal_s16(vYL, vget_low_s16(vB), vcYB); // GB + vYH = vmlal_s16(vYH, vget_high_s16(vB), vcYB); // GB + vCbL = vmlsl_s16(vCbL, vget_low_s16(vR), vget_low_s16(vcCbRG)); // BR + vCbH = vmlsl_s16(vCbH, vget_high_s16(vR), vget_low_s16(vcCbRG)); // BR + + vCrL = vmlsl_s16(vCrL, vget_low_s16(vB), vget_high_s16(vcCrGB)); // RGB + vCrH = vmlsl_s16(vCrH, vget_high_s16(vB), vget_high_s16(vcCrGB)); // RGB + vYL = vmlal_s16(vYL, vget_low_s16(vR), vget_low_s16(vcYRG)); // GBR + vYH = vmlal_s16(vYH, vget_high_s16(vR), vget_low_s16(vcYRG)); // GBR + vCbL = vmlsl_s16(vCbL, vget_low_s16(vG), vget_high_s16(vcCbRG)); // BRG + vCbH = vmlsl_s16(vCbH, vget_high_s16(vG), vget_high_s16(vcCbRG)); // BRG + + int16x4_t vCrL_ = vrshrn_n_s32(vCrL, 14); + int16x4_t vCrH_ = vrshrn_n_s32(vCrH, 14); + int16x4_t vYL_ = vrshrn_n_s32(vYL, 14); + int16x4_t vYH_ = vrshrn_n_s32(vYH, 14); + int16x4_t vCbL_ = vrshrn_n_s32(vCbL, 14); + int16x4_t vCbH_ = vrshrn_n_s32(vCbH, 14); + + int16x8_t vCr = vmovq_n_s16(128); + int16x8_t vCb = vmovq_n_s16(128); + + vCr = vaddq_s16(vCr, vcombine_s16(vCrL_, vCrH_)); + vCb = vaddq_s16(vCb, vcombine_s16(vCbL_, vCbH_)); + + uint8x8x3_t vYCrCb; + vYCrCb.val[0] = vqmovn_u16(vreinterpretq_u16_s16(vcombine_s16(vYL_, vYH_))); + vYCrCb.val[1] = vqmovun_s16(vCr); + vYCrCb.val[2] = vqmovun_s16(vCb); + + return vYCrCb; +} + +#define S_CONVERTTOYCRCB(R, G, B) \ + s32 Y = (R * 4899 + G * 9617 + B * 1868 + (1 << 13)) >> 14; \ + s32 Cr = 128 + ((R * 8192 - G * 6860 - B * 1332 + (1 << 13)) >> 14); \ + s32 Cb = 128 + ((R * (-2765) - G * 5427 + B * 8192 + (1 << 13)) >> 14); \ + dst[dj + 0] = internal::saturate_cast(Y); \ + dst[dj + 1] = internal::saturate_cast(Cr); \ + dst[dj + 2] = internal::saturate_cast(Cb); + +#define COEFF_Y ( 149) +#define COEFF_BU ( 129) +#define COEFF_RV ( 102) +#define COEFF_GU ( 25) +#define COEFF_GV ( 52) +#define COEFF_R (-14248) +#define COEFF_G ( 8663) +#define COEFF_B (-17705) + +#if defined(__GNUC__) && __GNUC_MINOR__ < 7 +#define YUV420ALPHA3_CONST +#define YUV420ALPHA4_CONST register uint8x16_t c255 asm ("q13") = vmovq_n_u8(255); +#define YUV420ALPHA3_CONVERT +#define YUV420ALPHA4_CONVERT , "w" (c255) +#define YUV420STORE1CMD3 "vst3.8 {d20, d22, d24}" +#define YUV420STORE2CMD3 "vst3.8 {d21, d23, d25}" +#define YUV420STORE1CMD4 "vst4.8 {d20, d22, d24, d26}" +#define YUV420STORE2CMD4 "vst4.8 {d21, d23, d25, d27}" + +#define YUV420_CONSTS(cn, bIdx, vIdx) \ + register const s32 cR = s16(COEFF_R); \ + register const s32 cG = s16(COEFF_G); \ + register const s32 cB = s16(COEFF_B); \ + \ + register uint8x16_t vc16 asm ("q15") = vmovq_n_u8(16); \ + register uint8x8_t cGU asm ("d14") = vmov_n_u8(COEFF_GU); \ + register uint8x8_t cGV asm ("d15") = vmov_n_u8(COEFF_GV); \ + register uint8x8_t cRV asm ("d16") = vmov_n_u8(COEFF_RV); \ + register uint8x8_t cBU asm ("d17") = vmov_n_u8(COEFF_BU); \ + register uint8x16_t cRGBY asm ("q3") = vmovq_n_u8(COEFF_Y); \ + YUV420ALPHA##cn##_CONST + +#define CONVERTYUV420TORGB(cn, ureg, vreg, rreg, breg) \ + __asm__ ( \ + "vld2.8 {d0-d1}, [%[inUV]] @UV \n\t" \ + "vdup.16 q4, %[cG] @cG \n\t" \ + "vld2.8 {d2-d3}, [%[inY1]] @YY \n\t" \ + "vdup.16 "#rreg", %[cR] @cR \n\t" \ + "vld2.8 {d4-d5}, [%[inY2]] @YY \n\t" \ + "vdup.16 "#breg", %[cB] @cB \n\t" \ + "vmlsl.u8 q4, "#ureg", d14 @cG-25u \n\t" \ + "vmax.u8 q1, q15 @max(Y,16) \n\t" \ + "vmlal.u8 "#rreg", "#vreg", d16 @cR+102*v \n\t" \ + "vmlal.u8 "#breg", "#ureg", d17 @cB+129*u \n\t" \ + "vmax.u8 q2, q15 @max(Y,16) \n\t" \ + "vmlsl.u8 q4, "#vreg", d15 @cG-25u-52v \n\t" \ + /*q10,q11,q12,q13 - for output*/ \ + "vmull.u8 q9, d3, d6 @h 149*y \n\t" \ + "vmull.u8 q10, d2, d7 @l 149*y \n\t" \ + "vshr.u16 q9, #1 @h (149*y)/2 \n\t" \ + "vshr.u16 q10, #1 @l (149*y)/2 \n\t" \ + \ + "vhadd.s16 q0, q9, q4 @hG ((149*y)/2 + cG - 25*u - 52*v)/2 \n\t" \ + "vhadd.s16 q12, q10, q6 @lB ((149*y)/2 + cB + 129*u)/2 \n\t" \ + "vhadd.s16 q1, q9, q5 @hR ((149*y)/2 + cR + 102*v)/2 \n\t" \ + "vhadd.s16 q11, q10, q4 @lG ((149*y)/2 + cG - 25*u - 52*v)/2 \n\t" \ + "vhadd.s16 q9, q6 @hB ((149*y)/2 + cB + 129*u)/2 \n\t" \ + "vhadd.s16 q10, q5 @lR ((149*y)/2 + cR + 102*v)/2 \n\t" \ + \ + "vqrshrun.s16 d24, q12, #5 @lB ((149*y)/2 + cB + 129*u)/2/32 \n\t" \ + "vqrshrun.s16 d22, q11, #5 @lG ((149*y)/2 + cG - 25*u - 52*v)/2/32 \n\t" \ + "vqrshrun.s16 d20, q10, #5 @lR ((149*y)/2 + cR + 102*v)/2/32 \n\t" \ + "vqrshrun.s16 d23, q0, #5 @hG ((149*y)/2 + cG - 25*u - 52*v)/2/32 \n\t" \ + "vqrshrun.s16 d21, q1, #5 @hR ((149*y)/2 + cR + 102*v)/2/32 \n\t" \ + "vqrshrun.s16 d25, q9, #5 @hB ((149*y)/2 + cB + 129*u)/2/32 \n\t" \ + \ + "vzip.8 d22, d23 @G \n\t" \ + "vzip.8 d20, d21 @R \n\t" \ + "vzip.8 d24, d25 @B \n\t" \ + \ + YUV420STORE1CMD##cn", [%[out1]] \n\t" \ + YUV420STORE2CMD##cn", [%[out1x]] \n\t" \ + \ + "vmull.u8 q9, d5, d6 @h 149*y \n\t" \ + "vmull.u8 q10, d4, d7 @l 149*y \n\t" \ + "vshr.u16 q9, #1 @h (149*y)/2 \n\t" \ + "vshr.u16 q10, #1 @l (149*y)/2 \n\t" \ + \ + "vhadd.s16 q0, q9, q4 @hG ((149*y)/2 + cG - 25*u - 52*v)/2 \n\t" \ + "vhadd.s16 q12, q10, q6 @lB ((149*y)/2 + cB + 129*u)/2 \n\t" \ + "vhadd.s16 q1, q9, q5 @hR ((149*y)/2 + cR + 102*v)/2 \n\t" \ + "vhadd.s16 q11, q10, q4 @lG ((149*y)/2 + cG - 25*u - 52*v)/2 \n\t" \ + "vhadd.s16 q9, q6 @hB ((149*y)/2 + cB + 129*u)/2 \n\t" \ + "vhadd.s16 q10, q5 @lR ((149*y)/2 + cR + 102*v)/2 \n\t" \ + \ + "vqrshrun.s16 d24, q12, #5 @lB ((149*y)/2 + cB + 129*u)/2/32 \n\t" \ + "vqrshrun.s16 d22, q11, #5 @lG ((149*y)/2 + cG - 25*u - 52*v)/2/32 \n\t" \ + "vqrshrun.s16 d20, q10, #5 @lR ((149*y)/2 + cR + 102*v)/2/32 \n\t" \ + "vqrshrun.s16 d23, q0, #5 @hG ((149*y)/2 + cG - 25*u - 52*v)/2/32 \n\t" \ + "vqrshrun.s16 d21, q1, #5 @hR ((149*y)/2 + cR + 102*v)/2/32 \n\t" \ + "vqrshrun.s16 d25, q9, #5 @hB ((149*y)/2 + cB + 129*u)/2/32 \n\t" \ + \ + "vzip.8 d22, d23 @G \n\t" \ + "vzip.8 d20, d21 @R \n\t" \ + "vzip.8 d24, d25 @B \n\t" \ + \ + YUV420STORE1CMD##cn", [%[out2]] \n\t" \ + YUV420STORE2CMD##cn", [%[out2x]] \n\t" \ + \ + : /*no output*/ \ + : [out1] "r" (dst1 + dj), [out2] "r" (dst2 + dj), \ + [out1x] "r" (dst1 + dj+cn*8), [out2x] "r" (dst2 + dj+cn*8), \ + [inUV] "r" (uv+j), [inY1] "r" (y1+j), [inY2] "r" (y2+j), \ + [cR] "r" (cR), [cG] "r" (cG), [cB] "r" (cB), \ + "w" (vc16), "w" (cGU), "w" (cGV), "w" (cBU), "w" (cRV), "w" (cRGBY) YUV420ALPHA##cn##_CONVERT \ + : "d0","d1","d2","d3","d4","d5","d8","d9","d10","d11","d12", \ + "d13","d18","d19","d20","d21","d22","d23","d24","d25" \ + ); + +#else + +template +struct _convertYUV420Internals +{ + uint16x8_t vc14216; + uint16x8_t vc17672; + uint16x8_t vc8696; + uint8x8_t vc102; + uint8x8_t vc25; + uint8x8_t vc129; + uint8x8_t vc52; + uint16x8_t vc_1; + uint8x8_t vc149; + uint8x8_t vc16; + _convertYUV420Internals() + { + vc14216 = vdupq_n_u16(-COEFF_R); + vc17672 = vdupq_n_u16(-COEFF_B); + vc8696 = vdupq_n_u16(COEFF_G); + vc102 = vdup_n_u8(COEFF_RV); + vc25 = vdup_n_u8(COEFF_GU); + vc129 = vdup_n_u8(COEFF_BU); + vc52 = vdup_n_u8(COEFF_GV); + vc_1 = vdupq_n_u16((uint16_t)-1); + vc149 = vdup_n_u8(COEFF_Y); + vc16 = vdup_n_u8(16); + } + + inline void UVrgbToRGB( const int16x8_t &ruv, const int16x8_t &guv, const int16x8_t &buv, + const u8 *y, uint8x16x3_t &rgbl ) + { + //y get line + uint8x8x2_t yl = vld2_u8(y); + yl.val[0] = vmax_u8(yl.val[0], vc16); + yl.val[1] = vmax_u8(yl.val[1], vc16); + + //y part line + uint16x8_t yodd1 = vmlal_u8(vc_1, yl.val[0], vc149); //(-1+149*y) + uint16x8_t yevn1 = vmlal_u8(vc_1, yl.val[1], vc149); //(-1+149*y) + int16x8_t yodd1h = (int16x8_t)vshrq_n_u16(yodd1, 1); //(-1+149*y)/2 + int16x8_t yevn1h = (int16x8_t)vshrq_n_u16(yevn1, 1); //(-1+149*y)/2 + + //y line calc rgb + int16x8_t rodd1w = vhsubq_s16(yodd1h, ruv); //((-1+149*y)/2 - (14216-102*v))/2 + int16x8_t gevn1w = vhaddq_s16(yevn1h, guv); //((-1+149*y)/2 + ((8696-25*u)-52*v))/2 + int16x8_t bodd1w = vhsubq_s16(yodd1h, buv); //((-1+149*y)/2 - (17672-129*u))/2 + int16x8_t revn1w = vhsubq_s16(yevn1h, ruv); //((-1+149*y)/2 - (14216-102*v))/2 + int16x8_t godd1w = vhaddq_s16(yodd1h, guv); //((-1+149*y)/2 + ((8696-25*u)-52*v))/2 + int16x8_t bevn1w = vhsubq_s16(yevn1h, buv); //((-1+149*y)/2 - (17672-129*u))/2 + + //y line clamp + narrow + uint8x8_t rodd1n = vqshrun_n_s16(rodd1w, 5); + uint8x8_t revn1n = vqshrun_n_s16(revn1w, 5); + uint8x8_t godd1n = vqshrun_n_s16(godd1w, 5); + uint8x8x2_t r1 = vzip_u8 (rodd1n, revn1n); + uint8x8_t gevn1n = vqshrun_n_s16(gevn1w, 5); + uint8x8_t bodd1n = vqshrun_n_s16(bodd1w, 5); + uint8x8x2_t g1 = vzip_u8 (godd1n, gevn1n); + uint8x8_t bevn1n = vqshrun_n_s16(bevn1w, 5); + uint8x8x2_t b1 = vzip_u8 (bodd1n, bevn1n); + rgbl.val[2 - bIdx] = vcombine_u8(r1.val[0], r1.val[1]); + rgbl.val[1] = vcombine_u8(g1.val[0], g1.val[1]); + rgbl.val[0 + bIdx] = vcombine_u8(b1.val[0], b1.val[1]); + } +}; + +template +struct _convertYUV420 +{ + _convertYUV420Internals convertYUV420Internals; + + inline void ToRGB( const u8 *y1, const u8 *y2, const u8 *uv, + u8 *dst1, u8 *dst2 ) + { + uint8x8x2_t raw_uv = vld2_u8(uv); + uint16x8_t gu = vmlsl_u8(convertYUV420Internals.vc8696, raw_uv.val[1-vIdx], convertYUV420Internals.vc25); //(8696-25*u) + int16x8_t ruv = (int16x8_t)vmlsl_u8(convertYUV420Internals.vc14216, raw_uv.val[vIdx], convertYUV420Internals.vc102); //(14216-102*v) + + int16x8_t buv = (int16x8_t)vmlsl_u8(convertYUV420Internals.vc17672, raw_uv.val[1-vIdx], convertYUV420Internals.vc129); //(17672-129*u) + int16x8_t guv = (int16x8_t)vmlsl_u8(gu, raw_uv.val[vIdx], convertYUV420Internals.vc52); //((8696-25*u)-52*v)) + + uint8x16x3_t rgbl; + //y line1 + convertYUV420Internals.UVrgbToRGB(ruv, guv, buv, y1, rgbl); + vst3q_u8(dst1, rgbl); + //y line2 + convertYUV420Internals.UVrgbToRGB(ruv, guv, buv, y2, rgbl); + vst3q_u8(dst2, rgbl); + } +}; + +template +struct _convertYUV420<4, bIdx, vIdx> +{ + _convertYUV420Internals convertYUV420Internals; + + inline void ToRGB( const u8 *y1, const u8 *y2, const u8 *uv, + u8 *dst1, u8 *dst2 ) + { + uint8x8x2_t raw_uv = vld2_u8(uv); + uint16x8_t gu = vmlsl_u8(convertYUV420Internals.vc8696, raw_uv.val[1-vIdx], convertYUV420Internals.vc25); //(8696-25*u) + int16x8_t ruv = (int16x8_t)vmlsl_u8(convertYUV420Internals.vc14216, raw_uv.val[vIdx], convertYUV420Internals.vc102); //(14216-102*v) + + int16x8_t buv = (int16x8_t)vmlsl_u8(convertYUV420Internals.vc17672, raw_uv.val[1-vIdx], convertYUV420Internals.vc129); //(17672-129*u) + int16x8_t guv = (int16x8_t)vmlsl_u8(gu, raw_uv.val[vIdx], convertYUV420Internals.vc52); //((8696-25*u)-52*v)) + + union { uint8x16x4_t v4; uint8x16x3_t v3; } rgbl; + rgbl.v4.val[3] = vdupq_n_u8(0xff); + //y line1 + convertYUV420Internals.UVrgbToRGB(ruv, guv, buv, y1, rgbl.v3); + vst4q_u8(dst1, rgbl.v4); + //y line2 + convertYUV420Internals.UVrgbToRGB(ruv, guv, buv, y2, rgbl.v3); + vst4q_u8(dst2, rgbl.v4); + } +}; + +#define YUV420_CONSTS(cn, bIdx, vIdx) _convertYUV420 convertYUV420; + +#endif + +template inline void fillAlpha(u8 *, u8 *){} +template <> inline void fillAlpha<4>(u8 *dst1, u8 *dst2) +{ + dst1[3] = 255; + dst1[7] = 255; + dst2[3] = 255; + dst2[7] = 255; +} +template +inline void convertYUV420ToRGB(const u8 *y1, const u8 *y2, const u8 *uv, u8* dst1, u8 *dst2) +{ + int Y11 = y1[0]; + int Y12 = y1[1]; + int Y21 = y2[0]; + int Y22 = y2[1]; + + int U = uv[1 - vIdx]; + int V = uv[vIdx]; + + int y11 = (COEFF_Y * std::max(16, Y11)) >> 1; + int y12 = (COEFF_Y * std::max(16, Y12)) >> 1; + int y21 = (COEFF_Y * std::max(16, Y21)) >> 1; + int y22 = (COEFF_Y * std::max(16, Y22)) >> 1; + + int uvR = COEFF_R + COEFF_RV * V; + int uvG = COEFF_G - COEFF_GU * U - COEFF_GV * V; + int uvB = COEFF_B + COEFF_BU * U; + + dst1[2-bIdx] = internal::saturate_cast((((y11 + uvR) >> 1) + (1 << 4)) >> 5); + dst1[1] = internal::saturate_cast((((y11 + uvG) >> 1) + (1 << 4)) >> 5); + dst1[bIdx] = internal::saturate_cast((((y11 + uvB) >> 1) + (1 << 4)) >> 5); + + dst1[cn+2-bIdx] = internal::saturate_cast((((y12 + uvR) >> 1) + (1 << 4)) >> 5); + dst1[cn+1] = internal::saturate_cast((((y12 + uvG) >> 1) + (1 << 4)) >> 5); + dst1[cn+bIdx] = internal::saturate_cast((((y12 + uvB) >> 1) + (1 << 4)) >> 5); + + dst2[2-bIdx] = internal::saturate_cast((((y21 + uvR) >> 1) + (1 << 4)) >> 5); + dst2[1] = internal::saturate_cast((((y21 + uvG) >> 1) + (1 << 4)) >> 5); + dst2[bIdx] = internal::saturate_cast((((y21 + uvB) >> 1) + (1 << 4)) >> 5); + + dst2[cn+2-bIdx] = internal::saturate_cast((((y22 + uvR) >> 1) + (1 << 4)) >> 5); + dst2[cn+1] = internal::saturate_cast((((y22 + uvG) >> 1) + (1 << 4)) >> 5); + dst2[cn+bIdx] = internal::saturate_cast((((y22 + uvB) >> 1) + (1 << 4)) >> 5); + + fillAlpha(dst1, dst2); +} + +// converts R, G, B (B, G, R) pixels to RGB(BGR)565 format respectively +inline uint8x16x2_t convertTo565( const uint8x16_t& vR, const uint8x16_t& vG, const uint8x16_t& vB ) +{ + uint8x16x2_t vRgb565; // rrrrRRRR ggggGGGG bbbbBBBB + + vRgb565.val[1] = vsriq_n_u8(vB, vG, 5); // xxxxxxxx bbbbBggg + vRgb565.val[0] = vshlq_n_u8(vG, 3); // gGGGG000 bbbbBggg + vRgb565.val[0] = vsriq_n_u8(vRgb565.val[0], vR, 3); // gGGrrrrR bbbbBggg + + return vRgb565; +} +inline void convertTo565( const u16 R, const u16 G, const u16 B, u8 * dst ) +{ + *((u16*)dst) = (R >> 3)|((G&~3) << 3)|((B&~7) << 8); +} +#endif + +} //namespace + +void rgb2hsv(const Size2D &size, + const u8 * srcBase, ptrdiff_t srcStride, + u8 * dstBase, ptrdiff_t dstStride, + s32 hrange) +{ + internal::assertSupportedConfiguration(); +#ifdef CAROTENE_NEON + size_t roiw8 = size.width >= 7 ? size.width - 7 : 0; + const s32 hsv_shift = 12; +#if defined(__GNUC__) && __GNUC_MINOR__ < 7 + register const f32 vsdiv_table = f32(255 << hsv_shift); + register f32 vhdiv_table = f32(hrange << hsv_shift); + register const s32 vhrange = hrange; + register const s32 v0 = s32(0); + register const s32 vshift = s32(1 << (hsv_shift-1)); + register const s32 v6 = s32(6); + register const f32 bias = 0.5f; +#endif + + for (size_t i = 0u; i < size.height; ++i) + { + const u8 * src = internal::getRowPtr(srcBase, srcStride, i); + u8 * dst = internal::getRowPtr(dstBase, dstStride, i); + size_t sj = 0u, dj = 0u, j = 0u; + + for (; j < roiw8; sj += 24, dj += 24, j += 8) + { + internal::prefetch(src + sj); +#if defined(__GNUC__) && __GNUC_MINOR__ < 7 + CONVERT_TO_HSV_ASM(vld3.8 {d0-d2}, d0, d2) +#else + uint8x8x3_t vRgb = vld3_u8(src + sj); + uint8x8x3_t vHsv = convertToHSV(vRgb.val[0], vRgb.val[1], vRgb.val[2], hrange); + vst3_u8(dst + dj, vHsv); +#endif + } + + for (; j < size.width; ++j, sj += 3, dj += 3) + { + convertToHSV(src[sj], src[sj+1], src[sj+2], hrange, hsv_shift, dst+dj); + } + } +#else + (void)size; + (void)srcBase; + (void)srcStride; + (void)dstBase; + (void)dstStride; + (void)hrange; +#endif +} + +void rgbx2hsv(const Size2D &size, + const u8 * srcBase, ptrdiff_t srcStride, + u8 * dstBase, ptrdiff_t dstStride, + s32 hrange) +{ + internal::assertSupportedConfiguration(); +#ifdef CAROTENE_NEON + size_t roiw8 = size.width >= 7 ? size.width - 7 : 0; + const s32 hsv_shift = 12; +#if defined(__GNUC__) && __GNUC_MINOR__ < 7 + register const f32 vsdiv_table = f32(255 << hsv_shift); + register f32 vhdiv_table = f32(hrange << hsv_shift); + register const s32 vhrange = hrange; + register const s32 v0 = s32(0); + register const s32 vshift = s32(1 << (hsv_shift-1)); + register const s32 v6 = s32(6); + register const f32 bias = 0.5f; +#endif + + for (size_t i = 0u; i < size.height; ++i) + { + const u8 * src = internal::getRowPtr(srcBase, srcStride, i); + u8 * dst = internal::getRowPtr(dstBase, dstStride, i); + size_t sj = 0u, dj = 0u, j = 0u; + + for (; j < roiw8; sj += 32, dj += 24, j += 8) + { + internal::prefetch(src + sj); +#if defined(__GNUC__) && __GNUC_MINOR__ < 7 + CONVERT_TO_HSV_ASM(vld4.8 {d0-d3}, d0, d2) +#else + uint8x8x4_t vRgb = vld4_u8(src + sj); + uint8x8x3_t vHsv = convertToHSV(vRgb.val[0], vRgb.val[1], vRgb.val[2], hrange); + vst3_u8(dst + dj, vHsv); +#endif + } + + for (; j < size.width; ++j, sj += 4, dj += 3) + { + convertToHSV(src[sj], src[sj+1], src[sj+2], hrange, hsv_shift, dst+dj); + } + } +#else + (void)size; + (void)srcBase; + (void)srcStride; + (void)dstBase; + (void)dstStride; + (void)hrange; +#endif +} + +void bgr2hsv(const Size2D &size, + const u8 * srcBase, ptrdiff_t srcStride, + u8 * dstBase, ptrdiff_t dstStride, + s32 hrange) +{ + internal::assertSupportedConfiguration(); +#ifdef CAROTENE_NEON + size_t roiw8 = size.width >= 7 ? size.width - 7 : 0; + const s32 hsv_shift = 12; +#if defined(__GNUC__) && __GNUC_MINOR__ < 7 + register const f32 vsdiv_table = f32(255 << hsv_shift); + register f32 vhdiv_table = f32(hrange << hsv_shift); + register const s32 vhrange = hrange; + register const s32 v0 = s32(0); + register const s32 vshift = s32(1 << (hsv_shift-1)); + register const s32 v6 = s32(6); + register const f32 bias = 0.5f; +#endif + + for (size_t i = 0u; i < size.height; ++i) + { + const u8 * src = internal::getRowPtr(srcBase, srcStride, i); + u8 * dst = internal::getRowPtr(dstBase, dstStride, i); + size_t sj = 0u, dj = 0u, j = 0u; + + for (; j < roiw8; sj += 24, dj += 24, j += 8) + { + internal::prefetch(src + sj); +#if defined(__GNUC__) && __GNUC_MINOR__ < 7 + CONVERT_TO_HSV_ASM(vld3.8 {d0-d2}, d2, d0) +#else + uint8x8x3_t vRgb = vld3_u8(src + sj); + uint8x8x3_t vHsv = convertToHSV(vRgb.val[2], vRgb.val[1], vRgb.val[0], hrange); + vst3_u8(dst + dj, vHsv); +#endif + } + + for (; j < size.width; ++j, sj += 3, dj += 3) + { + convertToHSV(src[sj+2], src[sj+1], src[sj], hrange, hsv_shift, dst+dj); + } + } +#else + (void)size; + (void)srcBase; + (void)srcStride; + (void)dstBase; + (void)dstStride; + (void)hrange; +#endif +} + +void bgrx2hsv(const Size2D &size, + const u8 * srcBase, ptrdiff_t srcStride, + u8 * dstBase, ptrdiff_t dstStride, + s32 hrange) +{ + internal::assertSupportedConfiguration(); +#ifdef CAROTENE_NEON + size_t roiw8 = size.width >= 7 ? size.width - 7 : 0; + const s32 hsv_shift = 12; +#if defined(__GNUC__) && __GNUC_MINOR__ < 7 + register const f32 vsdiv_table = f32(255 << hsv_shift); + register f32 vhdiv_table = f32(hrange << hsv_shift); + register const s32 vhrange = hrange; + register const s32 v0 = s32(0); + register const s32 vshift = s32(1 << (hsv_shift-1)); + register const s32 v6 = s32(6); + register const f32 bias = 0.5f; +#endif + + for (size_t i = 0u; i < size.height; ++i) + { + const u8 * src = internal::getRowPtr(srcBase, srcStride, i); + u8 * dst = internal::getRowPtr(dstBase, dstStride, i); + size_t sj = 0u, dj = 0u, j = 0u; + + for (; j < roiw8; sj += 32, dj += 24, j += 8) + { + internal::prefetch(src + sj); +#if defined(__GNUC__) && __GNUC_MINOR__ < 7 + CONVERT_TO_HSV_ASM(vld4.8 {d0-d3}, d2, d0) +#else + uint8x8x4_t vRgb = vld4_u8(src + sj); + uint8x8x3_t vHsv = convertToHSV(vRgb.val[2], vRgb.val[1], vRgb.val[0], hrange); + vst3_u8(dst + dj, vHsv); +#endif + } + + for (; j < size.width; ++j, sj += 4, dj += 3) + { + convertToHSV(src[sj+2], src[sj+1], src[sj], hrange, hsv_shift, dst+dj); + } + } +#else + (void)size; + (void)srcBase; + (void)srcStride; + (void)dstBase; + (void)dstStride; + (void)hrange; +#endif +} + +void rgbx2bgr565(const Size2D &size, + const u8 * srcBase, ptrdiff_t srcStride, + u8 * dstBase, ptrdiff_t dstStride) +{ + internal::assertSupportedConfiguration(); +#ifdef CAROTENE_NEON + size_t roiw16 = size.width >= 15 ? size.width - 15 : 0; + + for (size_t i = 0u; i < size.height; ++i) + { + const u8 * src = internal::getRowPtr(srcBase, srcStride, i); + u8 * dst = internal::getRowPtr(dstBase, dstStride, i); + size_t sj = 0u, dj = 0u, j = 0u; + + for (; j < roiw16; sj += 64, dj += 32, j += 16) + { + internal::prefetch(src + sj); +#if defined(__GNUC__) && __GNUC_MINOR__ < 7 + __asm__ ( + "vld4.8 {d2, d4, d6, d8}, [%[in0]] @ q0 q1 q2 q3 q4 \n\t" + "vld4.8 {d3, d5, d7, d9}, [%[in1]] @ xxxxxxxx rrrrRRRR ggggGGGG bbbbBBBB xxxxxxxx \n\t" + "vsri.8 q1, q2, #5 @ xxxxxxxx rrrrRggg ggggGGGG bbbbBBBB xxxxxxxx \n\t" + "vshl.u8 q0, q2, #3 @ gGGGG000 rrrrRggg ggggGGGG bbbbBBBB xxxxxxxx \n\t" + "vsri.8 q0, q3, #3 @ gGGbbbbB rrrrRggg ggggGGGG bbbbBBBB xxxxxxxx \n\t" + "vst2.8 {d0, d2}, [%[out0]] \n\t" + "vst2.8 {d1, d3}, [%[out1]] \n\t" + : /*no output*/ + : [out0] "r" (dst + dj), + [out1] "r" (dst + dj + 16), + [in0] "r" (src + sj), + [in1] "r" (src + sj + 32) + : "d0","d1","d2","d3","d4","d5","d6","d7","d8","d9" + ); +#else + uint8x16x4_t vRgba = vld4q_u8(src + sj); + uint8x16x2_t vVal565 = convertTo565(vRgba.val[2], vRgba.val[1], vRgba.val[0]); + vst2q_u8(dst + dj, vVal565); +#endif + } + + for (; j < size.width; ++j, sj += 4, dj += 2) + { + convertTo565(src[sj + 2], src[sj + 1], src[sj], dst + dj); + } + } +#else + (void)size; + (void)srcBase; + (void)srcStride; + (void)dstBase; + (void)dstStride; +#endif +} + +void rgb2bgr565(const Size2D &size, + const u8 * srcBase, ptrdiff_t srcStride, + u8 * dstBase, ptrdiff_t dstStride) +{ + internal::assertSupportedConfiguration(); +#ifdef CAROTENE_NEON + size_t roiw16 = size.width >= 15 ? size.width - 15 : 0; + + for (size_t i = 0u; i < size.height; ++i) + { + const u8 * src = internal::getRowPtr(srcBase, srcStride, i); + u8 * dst = internal::getRowPtr(dstBase, dstStride, i); + size_t sj = 0u, dj = 0u, j = 0u; + + for (; j < roiw16; sj += 48, dj += 32, j += 16) + { + internal::prefetch(src + sj); +#if defined(__GNUC__) && __GNUC_MINOR__ < 7 + __asm__ ( + "vld3.8 {d2, d4, d6}, [%[in0]] @ q0 q1 q2 q3 q4 \n\t" + "vld3.8 {d3, d5, d7}, [%[in1]] @ xxxxxxxx rrrrRRRR ggggGGGG bbbbBBBB xxxxxxxx \n\t" + "vsri.8 q1, q2, #5 @ xxxxxxxx rrrrRggg ggggGGGG bbbbBBBB xxxxxxxx \n\t" + "vshl.u8 q0, q2, #3 @ gGGGG000 rrrrRggg ggggGGGG bbbbBBBB xxxxxxxx \n\t" + "vsri.8 q0, q3, #3 @ gGGbbbbB rrrrRggg ggggGGGG bbbbBBBB xxxxxxxx \n\t" + "vst2.8 {d0, d2}, [%[out0]] \n\t" + "vst2.8 {d1, d3}, [%[out1]] \n\t" + : /*no output*/ + : [out0] "r" (dst + dj), + [out1] "r" (dst + dj + 16), + [in0] "r" (src + sj), + [in1] "r" (src + sj + 24) + : "d0","d1","d2","d3","d4","d5","d6","d7" + ); +#else + uint8x16x3_t vRgba = vld3q_u8(src + sj); + uint8x16x2_t vVal565 = convertTo565(vRgba.val[2], vRgba.val[1], vRgba.val[0]); + vst2q_u8(dst + dj, vVal565); +#endif + } + + for (; j < size.width; ++j, sj += 3, dj += 2) + { + convertTo565(src[sj + 2], src[sj + 1], src[sj], dst + dj); + } + } +#else + (void)size; + (void)srcBase; + (void)srcStride; + (void)dstBase; + (void)dstStride; +#endif +} + +void rgbx2rgb565(const Size2D &size, + const u8 * srcBase, ptrdiff_t srcStride, + u8 * dstBase, ptrdiff_t dstStride) +{ + internal::assertSupportedConfiguration(); +#ifdef CAROTENE_NEON + size_t roiw16 = size.width >= 15 ? size.width - 15 : 0; + + for (size_t i = 0u; i < size.height; ++i) + { + const u8 * src = internal::getRowPtr(srcBase, srcStride, i); + u8 * dst = internal::getRowPtr(dstBase, dstStride, i); + size_t sj = 0u, dj = 0u, j = 0u; + + for (; j < roiw16; sj += 64, dj += 32, j += 16) + { + internal::prefetch(src + sj); +#if defined(__GNUC__) && __GNUC_MINOR__ < 7 + __asm__ ( + "vld4.8 {d0, d2, d4, d6}, [%[in0]] @ q0 q1 q2 q3 \n\t" + "vld4.8 {d1, d3, d5, d7}, [%[in1]] @ rrrrRRRR ggggGGGG bbbbBBBB aaaaAAAA \n\t" + "vsri.8 q2, q1, #5 @ rrrrRRRR ggggGGGG bbbbBggg aaaaAAAA \n\t" + "vshl.u8 q1, #3 @ rrrrRRRR gGGGG000 bbbbBggg aaaaAAAA \n\t" + "vsri.8 q1, q0, #3 @ rrrrRRRR gGGrrrrR bbbbBggg aaaaAAAA \n\t" + "vst2.8 {d2, d4}, [%[out0]] \n\t" + "vst2.8 {d3, d5}, [%[out1]] \n\t" + : /*no output*/ + : [out0] "r" (dst + dj), + [out1] "r" (dst + dj + 16), + [in0] "r" (src + sj), + [in1] "r" (src + sj + 32) + : "d0","d1","d2","d3","d4","d5","d6","d7" + ); +#else + uint8x16x4_t vRgba = vld4q_u8(src + sj); + uint8x16x2_t vVal565 = convertTo565(vRgba.val[0], vRgba.val[1], vRgba.val[2]); + vst2q_u8(dst + dj, vVal565); +#endif + } + + for (; j < size.width; ++j, sj += 4, dj += 2) + { + convertTo565(src[sj], src[sj + 1], src[sj + 2], dst + dj); + } + } +#else + (void)size; + (void)srcBase; + (void)srcStride; + (void)dstBase; + (void)dstStride; +#endif +} + +void rgb2rgb565(const Size2D &size, + const u8 * srcBase, ptrdiff_t srcStride, + u8 * dstBase, ptrdiff_t dstStride) +{ + internal::assertSupportedConfiguration(); +#ifdef CAROTENE_NEON + size_t roiw16 = size.width >= 15 ? size.width - 15 : 0; + + for (size_t i = 0u; i < size.height; ++i) + { + const u8 * src = internal::getRowPtr(srcBase, srcStride, i); + u8 * dst = internal::getRowPtr(dstBase, dstStride, i); + size_t sj = 0u, dj = 0u, j = 0u; + + for (; j < roiw16; sj += 48, dj += 32, j += 16) + { + internal::prefetch(src + sj); +#if defined(__GNUC__) && __GNUC_MINOR__ < 7 + __asm__ ( + "vld3.8 {d0, d2, d4}, [%[in0]] @ q0 q1 q2 q3 \n\t" + "vld3.8 {d1, d3, d5}, [%[in1]] @ rrrrRRRR ggggGGGG bbbbBBBB xxxxxxxx \n\t" + "vsri.8 q2, q1, #5 @ rrrrRRRR ggggGGGG bbbbBggg xxxxxxxx \n\t" + "vshl.u8 q1, #3 @ rrrrRRRR gGGGG000 bbbbBggg xxxxxxxx \n\t" + "vsri.8 q1, q0, #3 @ rrrrRRRR gGGrrrrR bbbbBggg xxxxxxxx \n\t" + "vst2.8 {d2, d4}, [%[out0]] \n\t" + "vst2.8 {d3, d5}, [%[out1]] \n\t" + : /*no output*/ + : [out0] "r" (dst + dj), + [out1] "r" (dst + dj + 16), + [in0] "r" (src + sj), + [in1] "r" (src + sj + 24) + : "d0","d1","d2","d3","d4","d5" + ); +#else + uint8x16x3_t vRgba = vld3q_u8(src + sj); + uint8x16x2_t vVal565 = convertTo565(vRgba.val[0], vRgba.val[1], vRgba.val[2]); + vst2q_u8(dst + dj, vVal565); +#endif + } + + for (; j < size.width; ++j, sj += 3, dj += 2) + { + convertTo565(src[sj], src[sj + 1], src[sj + 2], dst + dj); + } + } +#else + (void)size; + (void)srcBase; + (void)srcStride; + (void)dstBase; + (void)dstStride; +#endif +} + +void rgb2ycrcb(const Size2D &size, + const u8 * srcBase, ptrdiff_t srcStride, + u8 * dstBase, ptrdiff_t dstStride) +{ + internal::assertSupportedConfiguration(); +#ifdef CAROTENE_NEON + YCRCB_CONSTS + size_t roiw8 = size.width >= 7 ? size.width - 7 : 0; + + for (size_t i = 0u; i < size.height; ++i) + { + const u8 * src = internal::getRowPtr(srcBase, srcStride, i); + u8 * dst = internal::getRowPtr(dstBase, dstStride, i); + size_t sj = 0u, dj = 0u, j = 0u; + + for (; j < roiw8; sj += 24, dj += 24, j += 8) + { + internal::prefetch(src + sj); +#if defined(__GNUC__) && __GNUC_MINOR__ < 7 + CONVERTTOYCRCB(vld3.8 {d0-d2}, d0, d1, d2) +#else + uint8x8x3_t vRgb = vld3_u8(src + sj); + int16x8_t vR = vreinterpretq_s16_u16(vmovl_u8(vRgb.val[0])); + int16x8_t vG = vreinterpretq_s16_u16(vmovl_u8(vRgb.val[1])); + int16x8_t vB = vreinterpretq_s16_u16(vmovl_u8(vRgb.val[2])); + uint8x8x3_t vYCrCb = convertToYCrCb(vR, vG, vB, vcYRG, vcYB, vcCrGB, vcCbRG); + vst3_u8(dst + dj, vYCrCb); +#endif + } + + for (; j < size.width; ++j, sj += 3, dj += 3) + { + S_CONVERTTOYCRCB(src[sj], src[sj + 1], src[sj + 2]); + } + } +#else + (void)size; + (void)srcBase; + (void)srcStride; + (void)dstBase; + (void)dstStride; +#endif +} + +void rgbx2ycrcb(const Size2D &size, + const u8 * srcBase, ptrdiff_t srcStride, + u8 * dstBase, ptrdiff_t dstStride) +{ + internal::assertSupportedConfiguration(); +#ifdef CAROTENE_NEON + YCRCB_CONSTS + size_t roiw8 = size.width >= 7 ? size.width - 7 : 0; + + for (size_t i = 0u; i < size.height; ++i) + { + const u8 * src = internal::getRowPtr(srcBase, srcStride, i); + u8 * dst = internal::getRowPtr(dstBase, dstStride, i); + size_t sj = 0u, dj = 0u, j = 0u; + + for (; j < roiw8; sj += 32, dj += 24, j += 8) + { + internal::prefetch(src + sj); +#if defined(__GNUC__) && __GNUC_MINOR__ < 7 + CONVERTTOYCRCB(vld4.8 {d0-d3}, d0, d1, d2) +#else + uint8x8x4_t vRgba = vld4_u8(src + sj); + int16x8_t vR = vreinterpretq_s16_u16(vmovl_u8(vRgba.val[0])); + int16x8_t vG = vreinterpretq_s16_u16(vmovl_u8(vRgba.val[1])); + int16x8_t vB = vreinterpretq_s16_u16(vmovl_u8(vRgba.val[2])); + uint8x8x3_t vYCrCb = convertToYCrCb(vR, vG, vB, vcYRG, vcYB, vcCrGB, vcCbRG); + vst3_u8(dst + dj, vYCrCb); +#endif + } + + for (; j < size.width; ++j, sj += 4, dj += 3) + { + S_CONVERTTOYCRCB(src[sj], src[sj + 1], src[sj + 2]); + } + } +#else + (void)size; + (void)srcBase; + (void)srcStride; + (void)dstBase; + (void)dstStride; +#endif +} + +void bgr2ycrcb(const Size2D &size, + const u8 * srcBase, ptrdiff_t srcStride, + u8 * dstBase, ptrdiff_t dstStride) +{ + internal::assertSupportedConfiguration(); +#ifdef CAROTENE_NEON + YCRCB_CONSTS + size_t roiw8 = size.width >= 7 ? size.width - 7 : 0; + + for (size_t i = 0u; i < size.height; ++i) + { + const u8 * src = internal::getRowPtr(srcBase, srcStride, i); + u8 * dst = internal::getRowPtr(dstBase, dstStride, i); + size_t sj = 0u, dj = 0u, j = 0u; + + for (; j < roiw8; sj += 24, dj += 24, j += 8) + { + internal::prefetch(src + sj); +#if defined(__GNUC__) && __GNUC_MINOR__ < 7 + CONVERTTOYCRCB(vld3.8 {d0-d2}, d2, d1, d0) +#else + uint8x8x3_t vBgr = vld3_u8(src + sj); + int16x8_t vB = vreinterpretq_s16_u16(vmovl_u8(vBgr.val[0])); + int16x8_t vG = vreinterpretq_s16_u16(vmovl_u8(vBgr.val[1])); + int16x8_t vR = vreinterpretq_s16_u16(vmovl_u8(vBgr.val[2])); + uint8x8x3_t vYCrCb = convertToYCrCb(vR, vG, vB, vcYRG, vcYB, vcCrGB, vcCbRG); + vst3_u8(dst + dj, vYCrCb); +#endif + } + + for (; j < size.width; ++j, sj += 3, dj += 3) + { + S_CONVERTTOYCRCB(src[sj + 2], src[sj + 1], src[sj]); + } + } +#else + (void)size; + (void)srcBase; + (void)srcStride; + (void)dstBase; + (void)dstStride; +#endif +} + +void bgrx2ycrcb(const Size2D &size, + const u8 * srcBase, ptrdiff_t srcStride, + u8 * dstBase, ptrdiff_t dstStride) +{ + internal::assertSupportedConfiguration(); +#ifdef CAROTENE_NEON + YCRCB_CONSTS + size_t roiw8 = size.width >= 7 ? size.width - 7 : 0; + + for (size_t i = 0u; i < size.height; ++i) + { + const u8 * src = internal::getRowPtr(srcBase, srcStride, i); + u8 * dst = internal::getRowPtr(dstBase, dstStride, i); + size_t sj = 0u, dj = 0u, j = 0u; + + for (; j < roiw8; sj += 32, dj += 24, j += 8) + { + internal::prefetch(src + sj); +#if defined(__GNUC__) && __GNUC_MINOR__ < 7 + CONVERTTOYCRCB(vld4.8 {d0-d3}, d2, d1, d0) +#else + uint8x8x4_t vBgra = vld4_u8(src + sj); + int16x8_t vB = vreinterpretq_s16_u16(vmovl_u8(vBgra.val[0])); + int16x8_t vG = vreinterpretq_s16_u16(vmovl_u8(vBgra.val[1])); + int16x8_t vR = vreinterpretq_s16_u16(vmovl_u8(vBgra.val[2])); + uint8x8x3_t vYCrCb = convertToYCrCb(vR, vG, vB, vcYRG, vcYB, vcCrGB, vcCbRG); + vst3_u8(dst + dj, vYCrCb); +#endif + } + + for (; j < size.width; ++j, sj += 4, dj += 3) + { + S_CONVERTTOYCRCB(src[sj + 2], src[sj + 1], src[sj]); + } + } +#else + (void)size; + (void)srcBase; + (void)srcStride; + (void)dstBase; + (void)dstStride; +#endif +} + +void yuv420sp2rgb(const Size2D &size, + const u8 * yBase, ptrdiff_t yStride, + const u8 * uvBase, ptrdiff_t uvStride, + u8 * dstBase, ptrdiff_t dstStride) +{ + // input data: + ////////////// Y matrix: + // {y1, y2, y3, y4, y5, y6, y7, y8, y9, y10, y11, y12, y13, y14, y15, y16} + // {Y1, Y2, Y3, Y4, Y5, Y6, Y7, Y8, Y9, Y10, Y11, Y12, Y13, Y14, Y15, Y16} + ////////////// UV matrix: + // {v12, u12, v34, u34, v56, u56, v78, u78, v90 u90, V12, U12, V34, U34, V56, U56} + + // fp version + // R = 1.164(Y - 16) + 1.596(V - 128) + // G = 1.164(Y - 16) - 0.813(V - 128) - 0.391(U - 128) + // B = 1.164(Y - 16) + 2.018(U - 128) + + // integer version + // R = [((149*y)/2 + (-14248+102*v) )/2]/32 + // G = [((149*y)/2 + ((8663- 25*u)-52*v))/2]/32 + // B = [((149*y)/2 + (-17705+129*u) )/2]/32 + + // error estimation: + //Rerr = 0.0000625 * y − 0.00225 * v − 0.287 + //Gerr = 0.0000625 * y + 0.0005 * v + 0.000375 * u + 0.128625 + //Berr = 0.0000625 * y − 0.002375 * u - 0.287375 + + //real error test: + //================= + //R: 1 less: 520960 == 3.11% of full space + //G: 1 less: 251425 == 1.50% of full space + //B: 1 less: 455424 == 2.71% of full space + //================= + //R: 1 more: 642048 == 3.83% of full space + //G: 1 more: 192458 == 1.15% of full space + //B: 1 more: 445184 == 2.65% of full space + + internal::assertSupportedConfiguration(); +#ifdef CAROTENE_NEON + YUV420_CONSTS(3, 2, 0) + size_t roiw16 = size.width >= 15 ? size.width - 15 : 0; + + for (size_t i = 0u; i < size.height; i+=2) + { + const u8 * uv = internal::getRowPtr(uvBase, uvStride, i>>1); + const u8 * y1 = internal::getRowPtr(yBase, yStride, i); + const u8 * y2 = internal::getRowPtr(yBase, yStride, i+1); + u8 * dst1 = internal::getRowPtr(dstBase, dstStride, i); + u8 * dst2 = internal::getRowPtr(dstBase, dstStride, i+1); + + size_t dj = 0u, j = 0u; + for (; j < roiw16; dj += 48, j += 16) + { + internal::prefetch(uv + j); + internal::prefetch(y1 + j); + internal::prefetch(y2 + j); +#if defined(__GNUC__) && __GNUC_MINOR__ < 7 + CONVERTYUV420TORGB(3, d1, d0, q5, q6) +#else + convertYUV420.ToRGB(y1 + j, y2 + j, uv + j, dst1 + dj, dst2 + dj); +#endif + } + for (; j + 2 <= size.width; j+=2, dj += 6) + { + convertYUV420ToRGB<3, 2, 0>(y1+j, y2+j, uv+j, dst1 + dj, dst2 + dj); + } + } +#else + (void)size; + (void)yBase; + (void)yStride; + (void)uvBase; + (void)uvStride; + (void)dstBase; + (void)dstStride; +#endif +} + +void yuv420sp2rgbx(const Size2D &size, + const u8 * yBase, ptrdiff_t yStride, + const u8 * uvBase, ptrdiff_t uvStride, + u8 * dstBase, ptrdiff_t dstStride) +{ + internal::assertSupportedConfiguration(); +#ifdef CAROTENE_NEON + YUV420_CONSTS(4, 2, 0) + size_t roiw16 = size.width >= 15 ? size.width - 15 : 0; + + for (size_t i = 0u; i < size.height; i+=2) + { + const u8 * uv = internal::getRowPtr(uvBase, uvStride, i>>1); + const u8 * y1 = internal::getRowPtr(yBase, yStride, i); + const u8 * y2 = internal::getRowPtr(yBase, yStride, i+1); + u8 * dst1 = internal::getRowPtr(dstBase, dstStride, i); + u8 * dst2 = internal::getRowPtr(dstBase, dstStride, i+1); + + size_t dj = 0u, j = 0u; + for (; j < roiw16; dj += 64, j += 16) + { + internal::prefetch(uv + j); + internal::prefetch(y1 + j); + internal::prefetch(y2 + j); +#if defined(__GNUC__) && __GNUC_MINOR__ < 7 + CONVERTYUV420TORGB(4, d1, d0, q5, q6) +#else + convertYUV420.ToRGB(y1 + j, y2 + j, uv + j, dst1 + dj, dst2 + dj); +#endif + } + for (; j + 2 <= size.width; j+=2, dj += 8) + { + convertYUV420ToRGB<4, 2, 0>(y1+j, y2+j, uv+j, dst1 + dj, dst2 + dj); + } + } +#else + (void)size; + (void)yBase; + (void)yStride; + (void)uvBase; + (void)uvStride; + (void)dstBase; + (void)dstStride; +#endif +} + +void yuv420i2rgb(const Size2D &size, + const u8 * yBase, ptrdiff_t yStride, + const u8 * uvBase, ptrdiff_t uvStride, + u8 * dstBase, ptrdiff_t dstStride) +{ + internal::assertSupportedConfiguration(); +#ifdef CAROTENE_NEON + YUV420_CONSTS(3, 2, 1) + size_t roiw16 = size.width >= 15 ? size.width - 15 : 0; + + for (size_t i = 0u; i < size.height; i+=2) + { + const u8 * uv = internal::getRowPtr(uvBase, uvStride, i>>1); + const u8 * y1 = internal::getRowPtr(yBase, yStride, i); + const u8 * y2 = internal::getRowPtr(yBase, yStride, i+1); + u8 * dst1 = internal::getRowPtr(dstBase, dstStride, i); + u8 * dst2 = internal::getRowPtr(dstBase, dstStride, i+1); + + size_t dj = 0u, j = 0u; + for (; j < roiw16; dj += 48, j += 16) + { + internal::prefetch(uv + j); + internal::prefetch(y1 + j); + internal::prefetch(y2 + j); +#if defined(__GNUC__) && __GNUC_MINOR__ < 7 + CONVERTYUV420TORGB(3, d0, d1, q5, q6) +#else + convertYUV420.ToRGB(y1 + j, y2 + j, uv + j, dst1 + dj, dst2 + dj); +#endif + } + for (; j + 2 <= size.width; j+=2, dj += 6) + { + convertYUV420ToRGB<3, 2, 1>(y1+j, y2+j, uv+j, dst1 + dj, dst2 + dj); + } + } +#else + (void)size; + (void)yBase; + (void)yStride; + (void)uvBase; + (void)uvStride; + (void)dstBase; + (void)dstStride; +#endif +} + +void yuv420i2rgbx(const Size2D &size, + const u8 * yBase, ptrdiff_t yStride, + const u8 * uvBase, ptrdiff_t uvStride, + u8 * dstBase, ptrdiff_t dstStride) +{ + internal::assertSupportedConfiguration(); +#ifdef CAROTENE_NEON + YUV420_CONSTS(4, 2, 1) + size_t roiw16 = size.width >= 15 ? size.width - 15 : 0; + + for (size_t i = 0u; i < size.height; i+=2) + { + const u8 * uv = internal::getRowPtr(uvBase, uvStride, i>>1); + const u8 * y1 = internal::getRowPtr(yBase, yStride, i); + const u8 * y2 = internal::getRowPtr(yBase, yStride, i+1); + u8 * dst1 = internal::getRowPtr(dstBase, dstStride, i); + u8 * dst2 = internal::getRowPtr(dstBase, dstStride, i+1); + + size_t dj = 0u, j = 0u; + for (; j < roiw16; dj += 64, j += 16) + { + internal::prefetch(uv + j); + internal::prefetch(y1 + j); + internal::prefetch(y2 + j); +#if defined(__GNUC__) && __GNUC_MINOR__ < 7 + CONVERTYUV420TORGB(4, d0, d1, q5, q6) +#else + convertYUV420.ToRGB(y1 + j, y2 + j, uv + j, dst1 + dj, dst2 + dj); +#endif + } + for (; j + 2 <= size.width; j+=2, dj += 8) + { + convertYUV420ToRGB<4, 2, 1>(y1+j, y2+j, uv+j, dst1 + dj, dst2 + dj); + } + } +#else + (void)size; + (void)yBase; + (void)yStride; + (void)uvBase; + (void)uvStride; + (void)dstBase; + (void)dstStride; +#endif +} + +void yuv420sp2bgr(const Size2D &size, + const u8 * yBase, ptrdiff_t yStride, + const u8 * uvBase, ptrdiff_t uvStride, + u8 * dstBase, ptrdiff_t dstStride) +{ + internal::assertSupportedConfiguration(); +#ifdef CAROTENE_NEON + YUV420_CONSTS(3, 0, 0) + size_t roiw16 = size.width >= 15 ? size.width - 15 : 0; + + for (size_t i = 0u; i < size.height; i+=2) + { + const u8 * uv = internal::getRowPtr(uvBase, uvStride, i>>1); + const u8 * y1 = internal::getRowPtr(yBase, yStride, i); + const u8 * y2 = internal::getRowPtr(yBase, yStride, i+1); + u8 * dst1 = internal::getRowPtr(dstBase, dstStride, i); + u8 * dst2 = internal::getRowPtr(dstBase, dstStride, i+1); + + size_t dj = 0u, j = 0u; + for (; j < roiw16; dj += 48, j += 16) + { + internal::prefetch(uv + j); + internal::prefetch(y1 + j); + internal::prefetch(y2 + j); +#if defined(__GNUC__) && __GNUC_MINOR__ < 7 + CONVERTYUV420TORGB(3, d1, d0, q6, q5) +#else + convertYUV420.ToRGB(y1 + j, y2 + j, uv + j, dst1 + dj, dst2 + dj); +#endif + } + for (; j + 2 <= size.width; j+=2, dj += 6) + { + convertYUV420ToRGB<3, 0, 0>(y1+j, y2+j, uv+j, dst1 + dj, dst2 + dj); + } + } +#else + (void)size; + (void)yBase; + (void)yStride; + (void)uvBase; + (void)uvStride; + (void)dstBase; + (void)dstStride; +#endif +} + +void yuv420sp2bgrx(const Size2D &size, + const u8 * yBase, ptrdiff_t yStride, + const u8 * uvBase, ptrdiff_t uvStride, + u8 * dstBase, ptrdiff_t dstStride) +{ + internal::assertSupportedConfiguration(); +#ifdef CAROTENE_NEON + YUV420_CONSTS(4, 0, 0) + size_t roiw16 = size.width >= 15 ? size.width - 15 : 0; + + for (size_t i = 0u; i < size.height; i+=2) + { + const u8 * uv = internal::getRowPtr(uvBase, uvStride, i>>1); + const u8 * y1 = internal::getRowPtr(yBase, yStride, i); + const u8 * y2 = internal::getRowPtr(yBase, yStride, i+1); + u8 * dst1 = internal::getRowPtr(dstBase, dstStride, i); + u8 * dst2 = internal::getRowPtr(dstBase, dstStride, i+1); + + size_t dj = 0u, j = 0u; + for (; j < roiw16; dj += 64, j += 16) + { + internal::prefetch(uv + j); + internal::prefetch(y1 + j); + internal::prefetch(y2 + j); +#if defined(__GNUC__) && __GNUC_MINOR__ < 7 + CONVERTYUV420TORGB(4, d1, d0, q6, q5) +#else + convertYUV420.ToRGB(y1 + j, y2 + j, uv + j, dst1 + dj, dst2 + dj); +#endif + } + for (; j + 2 <= size.width; j+=2, dj += 8) + { + convertYUV420ToRGB<4, 0, 0>(y1+j, y2+j, uv+j, dst1 + dj, dst2 + dj); + } + } +#else + (void)size; + (void)yBase; + (void)yStride; + (void)uvBase; + (void)uvStride; + (void)dstBase; + (void)dstStride; +#endif +} + +void yuv420i2bgr(const Size2D &size, + const u8 * yBase, ptrdiff_t yStride, + const u8 * uvBase, ptrdiff_t uvStride, + u8 * dstBase, ptrdiff_t dstStride) +{ + internal::assertSupportedConfiguration(); +#ifdef CAROTENE_NEON + YUV420_CONSTS(3, 0, 1) + size_t roiw16 = size.width >= 15 ? size.width - 15 : 0; + + for (size_t i = 0u; i < size.height; i+=2) + { + const u8 * uv = internal::getRowPtr(uvBase, uvStride, i>>1); + const u8 * y1 = internal::getRowPtr(yBase, yStride, i); + const u8 * y2 = internal::getRowPtr(yBase, yStride, i+1); + u8 * dst1 = internal::getRowPtr(dstBase, dstStride, i); + u8 * dst2 = internal::getRowPtr(dstBase, dstStride, i+1); + + size_t dj = 0u, j = 0u; + for (; j < roiw16; dj += 48, j += 16) + { + internal::prefetch(uv + j); + internal::prefetch(y1 + j); + internal::prefetch(y2 + j); +#if defined(__GNUC__) && __GNUC_MINOR__ < 7 + CONVERTYUV420TORGB(3, d0, d1, q6, q5) +#else + convertYUV420.ToRGB(y1 + j, y2 + j, uv + j, dst1 + dj, dst2 + dj); +#endif + } + for (; j + 2 <= size.width; j+=2, dj += 6) + { + convertYUV420ToRGB<3, 0, 1>(y1+j, y2+j, uv+j, dst1 + dj, dst2 + dj); + } + } +#else + (void)size; + (void)yBase; + (void)yStride; + (void)uvBase; + (void)uvStride; + (void)dstBase; + (void)dstStride; +#endif +} + +void yuv420i2bgrx(const Size2D &size, + const u8 * yBase, ptrdiff_t yStride, + const u8 * uvBase, ptrdiff_t uvStride, + u8 * dstBase, ptrdiff_t dstStride) +{ + internal::assertSupportedConfiguration(); +#ifdef CAROTENE_NEON + YUV420_CONSTS(4, 0, 1) + size_t roiw16 = size.width >= 15 ? size.width - 15 : 0; + + for (size_t i = 0u; i < size.height; i+=2) + { + const u8 * uv = internal::getRowPtr(uvBase, uvStride, i>>1); + const u8 * y1 = internal::getRowPtr(yBase, yStride, i); + const u8 * y2 = internal::getRowPtr(yBase, yStride, i+1); + u8 * dst1 = internal::getRowPtr(dstBase, dstStride, i); + u8 * dst2 = internal::getRowPtr(dstBase, dstStride, i+1); + + size_t dj = 0u, j = 0u; + for (; j < roiw16; dj += 64, j += 16) + { + internal::prefetch(uv + j); + internal::prefetch(y1 + j); + internal::prefetch(y2 + j); +#if defined(__GNUC__) && __GNUC_MINOR__ < 7 + CONVERTYUV420TORGB(4, d0, d1, q6, q5) +#else + convertYUV420.ToRGB(y1 + j, y2 + j, uv + j, dst1 + dj, dst2 + dj); +#endif + } + for (; j + 2 <= size.width; j+=2, dj += 8) + { + convertYUV420ToRGB<4, 0, 1>(y1+j, y2+j, uv+j, dst1 + dj, dst2 + dj); + } + } +#else + (void)size; + (void)yBase; + (void)yStride; + (void)uvBase; + (void)uvStride; + (void)dstBase; + (void)dstStride; +#endif +} + +} // namespace CAROTENE_NS diff --git a/3rdparty/carotene/src/common.cpp b/3rdparty/carotene/src/common.cpp new file mode 100644 index 0000000000..c85b0123b6 --- /dev/null +++ b/3rdparty/carotene/src/common.cpp @@ -0,0 +1,108 @@ +/* + * By downloading, copying, installing or using the software you agree to this license. + * If you do not agree to this license, do not download, install, + * copy or use the software. + * + * + * License Agreement + * For Open Source Computer Vision Library + * (3-clause BSD License) + * + * Copyright (C) 2014, NVIDIA Corporation, all rights reserved. + * Third party copyrights are property of their respective owners. + * + * Redistribution and use in source and binary forms, with or without modification, + * are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * * Neither the names of the copyright holders nor the names of the contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * This software is provided by the copyright holders and contributors "as is" and + * any express or implied warranties, including, but not limited to, the implied + * warranties of merchantability and fitness for a particular purpose are disclaimed. + * In no event shall copyright holders or contributors be liable for any direct, + * indirect, incidental, special, exemplary, or consequential damages + * (including, but not limited to, procurement of substitute goods or services; + * loss of use, data, or profits; or business interruption) however caused + * and on any theory of liability, whether in contract, strict liability, + * or tort (including negligence or otherwise) arising in any way out of + * the use of this software, even if advised of the possibility of such damage. + */ + +#include +#include + +#include "common.hpp" + +namespace CAROTENE_NS { + +bool isSupportedConfiguration() +{ +#ifdef CAROTENE_NEON + return true; +#else + return false; +#endif +} + +namespace internal { + +void assertSupportedConfiguration(bool parametersSupported) +{ + if (!isSupportedConfiguration()) { + std::cerr << "internal error: attempted to use an unavailable function" << std::endl; + std::abort(); + } + + if (!parametersSupported) { + std::cerr << "internal error: attempted to use a function with unsupported parameters" << std::endl; + std::abort(); + } +} + +ptrdiff_t borderInterpolate(ptrdiff_t _p, size_t _len, BORDER_MODE borderType, size_t startMargin, size_t endMargin) +{ + ptrdiff_t p = _p + (ptrdiff_t)startMargin; + size_t len = _len + startMargin + endMargin; + if( (size_t)p < len ) + return _p; + else if( borderType == BORDER_MODE_REPLICATE ) + p = p < 0 ? 0 : (ptrdiff_t)len - 1; + else if( borderType == BORDER_MODE_REFLECT || borderType == BORDER_MODE_REFLECT101 ) + { + s32 delta = borderType == BORDER_MODE_REFLECT101; + if( len == 1 ) + return 0; + do + { + if( p < 0 ) + p = -p - 1 + delta; + else + p = (ptrdiff_t)len - 1 - (p - (ptrdiff_t)len) - delta; + } + while( (size_t)p >= len ); + } + else if( borderType == BORDER_MODE_WRAP ) + { + if( p < 0 ) + p -= ((p-(ptrdiff_t)len+1)/(ptrdiff_t)len)*(ptrdiff_t)len; + if( p >= (ptrdiff_t)len ) + p %= (ptrdiff_t)len; + } + else if( borderType == BORDER_MODE_CONSTANT ) + p = -1; + else + internal::assertSupportedConfiguration(false); + return p - (ptrdiff_t)startMargin; +} + +} // namespace internal +} // namespace CAROTENE_NS diff --git a/3rdparty/carotene/src/common.hpp b/3rdparty/carotene/src/common.hpp new file mode 100644 index 0000000000..e46231a58a --- /dev/null +++ b/3rdparty/carotene/src/common.hpp @@ -0,0 +1,96 @@ +/* + * By downloading, copying, installing or using the software you agree to this license. + * If you do not agree to this license, do not download, install, + * copy or use the software. + * + * + * License Agreement + * For Open Source Computer Vision Library + * (3-clause BSD License) + * + * Copyright (C) 2014-2015, NVIDIA Corporation, all rights reserved. + * Third party copyrights are property of their respective owners. + * + * Redistribution and use in source and binary forms, with or without modification, + * are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * * Neither the names of the copyright holders nor the names of the contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * This software is provided by the copyright holders and contributors "as is" and + * any express or implied warranties, including, but not limited to, the implied + * warranties of merchantability and fitness for a particular purpose are disclaimed. + * In no event shall copyright holders or contributors be liable for any direct, + * indirect, incidental, special, exemplary, or consequential damages + * (including, but not limited to, procurement of substitute goods or services; + * loss of use, data, or profits; or business interruption) however caused + * and on any theory of liability, whether in contract, strict liability, + * or tort (including negligence or otherwise) arising in any way out of + * the use of this software, even if advised of the possibility of such damage. + */ + +#ifndef CAROTENE_SRC_COMMON_HPP +#define CAROTENE_SRC_COMMON_HPP + +#include +#include + +#if defined WITH_NEON && (defined __ARM_NEON__ || defined __ARM_NEON) +#define CAROTENE_NEON +#endif + +#ifdef CAROTENE_NEON +#include +#include "intrinsics.hpp" +#endif + +#include +#include "saturate_cast.hpp" + +namespace CAROTENE_NS { namespace internal { + +inline void prefetch(const void *ptr, size_t offset = 32*10) +{ +#if defined __GNUC__ + __builtin_prefetch(reinterpret_cast(ptr) + offset); +#elif defined _MSC_VER && defined CAROTENE_NEON + __prefetch(reinterpret_cast(ptr) + offset); +#else + (void)ptr; + (void)offset; +#endif +} + +template +inline T *getRowPtr(T *base, ptrdiff_t stride, size_t row) +{ + char *baseRaw = const_cast(reinterpret_cast(base)); + return reinterpret_cast(baseRaw + ptrdiff_t(row) * stride); +} + +void assertSupportedConfiguration(bool parametersSupported = true); + +ptrdiff_t borderInterpolate(ptrdiff_t _p, size_t _len, BORDER_MODE borderType, size_t startMargin = 0, size_t endMargin = 0); + +/*! + * Aligns pointer by the certain number of bytes + * + * This small inline function aligns the pointer by the certain number of bytes by shifting + * it forward by 0 or a positive offset. + */ +template inline T* alignPtr(T* ptr, size_t n=sizeof(T)) +{ + return (T*)(((size_t)ptr + n-1) & -n); +} + +}} + +#endif diff --git a/3rdparty/carotene/src/convert.cpp b/3rdparty/carotene/src/convert.cpp new file mode 100644 index 0000000000..2f95e29cb3 --- /dev/null +++ b/3rdparty/carotene/src/convert.cpp @@ -0,0 +1,1331 @@ +/* + * By downloading, copying, installing or using the software you agree to this license. + * If you do not agree to this license, do not download, install, + * copy or use the software. + * + * + * License Agreement + * For Open Source Computer Vision Library + * (3-clause BSD License) + * + * Copyright (C) 2012-2015, NVIDIA Corporation, all rights reserved. + * Third party copyrights are property of their respective owners. + * + * Redistribution and use in source and binary forms, with or without modification, + * are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * * Neither the names of the copyright holders nor the names of the contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * This software is provided by the copyright holders and contributors "as is" and + * any express or implied warranties, including, but not limited to, the implied + * warranties of merchantability and fitness for a particular purpose are disclaimed. + * In no event shall copyright holders or contributors be liable for any direct, + * indirect, incidental, special, exemplary, or consequential damages + * (including, but not limited to, procurement of substitute goods or services; + * loss of use, data, or profits; or business interruption) however caused + * and on any theory of liability, whether in contract, strict liability, + * or tort (including negligence or otherwise) arising in any way out of + * the use of this software, even if advised of the possibility of such damage. + */ + +#include "common.hpp" + +namespace CAROTENE_NS { + +#ifdef CAROTENE_NEON + +#define CVT_FUNC(T1, T2, SIMD_SIZE, CVTINIT, CVTROW) \ + void convert(const Size2D &_size, \ + const T1 * srcBase, ptrdiff_t srcStride, \ + T2 * dstBase, ptrdiff_t dstStride) \ + { \ + internal::assertSupportedConfiguration(); \ + Size2D size(_size); \ + if (srcStride == dstStride && \ + srcStride == (ptrdiff_t)(size.width)) \ + { \ + size.width *= size.height; \ + size.height = 1; \ + } \ + const ptrdiff_t sstep = srcStride / sizeof(T1); \ + const ptrdiff_t dstep = dstStride / sizeof(T2); \ + const size_t w = size.width & ~(SIMD_SIZE-1); \ + if (size.width >= SIMD_SIZE) \ + { \ + const T1* _src = srcBase; \ + T2* _dst = dstBase; \ + CVTINIT \ + for (ptrdiff_t h = size.height; h--; _src += sstep, _dst += dstep ) \ + CVTROW \ + } \ + if(w < size.width) \ + { \ + const T1* _src = srcBase; \ + T2* _dst = dstBase; \ + for (ptrdiff_t h = size.height; h--; _src += sstep, _dst += dstep ) \ + for(size_t i = w; i < size.width; i++ ) \ + _dst[i] = internal::saturate_cast(_src[i]); \ + } \ + } + +#else + +#define CVT_FUNC(T1, T2, SIMD_SIZE, CVTINIT, CVTROW) \ + void convert(const Size2D &, \ + const T1 *, ptrdiff_t, \ + T2 *, ptrdiff_t) \ + { \ + internal::assertSupportedConfiguration(); \ + } + +#endif + +CVT_FUNC(u8, s8, 16, + uint8x16_t v127 = vdupq_n_u8(127);, +{ + for (size_t i = 0; i < w; i += 16) + { + internal::prefetch(_src + i); + uint8x16_t vu8 = vld1q_u8(_src + i); + int8x16_t vu1 = vreinterpretq_s8_u8(vminq_u8(vu8, v127)); + vst1q_s8(_dst + i, vu1); + } +}) + +#if __GNUC_MINOR__ < 7 +CVT_FUNC(u8, u16, 16, + register uint8x16_t zero0 asm ("q1") = vmovq_n_u8(0);, +{ + for (size_t i = 0; i < w; i += 16) + { + internal::prefetch(_src + i); + __asm__ ( + "vld1.8 {d0-d1}, [%[src]] \n\t" + "vst2.8 {d0,d2}, [%[dst1]] \n\t" + "vst2.8 {d1,d3}, [%[dst2]] \n\t" + : /*no output*/ + : [src] "r" (_src + i), + [dst1] "r" (_dst + i + 0), + [dst2] "r" (_dst + i + 8), + "w" (zero0) + : "d0","d1" + ); + } +}) +#else +CVT_FUNC(u8, u16, 16, + uint8x16x2_t vline; + vline.val[1] = vmovq_n_u8(0);, +{ + for (size_t i = 0; i < w; i += 16) + { + internal::prefetch(_src + i); + vline.val[0] = vld1q_u8(_src + i); + vst2q_u8((uint8_t*)(_dst + i), vline); + } +}) +#endif + +#if __GNUC_MINOR__ < 7 +CVT_FUNC(u8, s32, 16, + register uint8x16_t zero0 asm ("q1") = vmovq_n_u8(0); + register uint8x16_t zero1 asm ("q2") = vmovq_n_u8(0); + register uint8x16_t zero2 asm ("q3") = vmovq_n_u8(0);, +{ + for (size_t i = 0; i < w; i += 16) + { + internal::prefetch(_src + i); + __asm__ ( + "vld1.8 {d0-d1}, [%[src]] \n\t" + "vst4.8 {d0,d2,d4,d6}, [%[dst1]] \n\t" + "vst4.8 {d1,d3,d5,d7}, [%[dst2]] \n\t" + : /*no output*/ + : [src] "r" (_src + i), + [dst1] "r" (_dst + i + 0), + [dst2] "r" (_dst + i + 8), + "w" (zero0), "w" (zero1), "w" (zero2) + : "d0","d1" + ); + } +}) +#else +CVT_FUNC(u8, s32, 16, + uint8x16x4_t vline; + vline.val[1] = vmovq_n_u8(0); + vline.val[2] = vmovq_n_u8(0); + vline.val[3] = vmovq_n_u8(0);, +{ + for (size_t i = 0; i < w; i += 16) + { + internal::prefetch(_src + i); + vline.val[0] = vld1q_u8(_src + i); + vst4q_u8((uint8_t*)(_dst + i), vline); + } +}) +#endif + +#if __GNUC_MINOR__ < 6 +CVT_FUNC(u8, f32, 16, +, +{ + for (size_t i = 0; i < w; i += 16) + { + internal::prefetch(_src + i); + __asm__ ( + "vld1.8 {d0-d1}, [%[src]] \n\t" + "vmovl.u8 q1, d0 \n\t" + "vmovl.u8 q2, d1 \n\t" + "vmovl.u16 q3, d2 \n\t" + "vmovl.u16 q4, d3 \n\t" + "vmovl.u16 q5, d4 \n\t" + "vmovl.u16 q6, d5 \n\t" + "vcvt.f32.u32 q7, q3 \n\t" + "vcvt.f32.u32 q8, q4 \n\t" + "vcvt.f32.u32 q9, q5 \n\t" + "vcvt.f32.u32 q10, q6 \n\t" + "vst1.32 {d14-d15}, [%[dst1]] \n\t" + "vst1.32 {d16-d17}, [%[dst2]] \n\t" + "vst1.32 {d18-d19}, [%[dst3]] \n\t" + "vst1.32 {d20-d21}, [%[dst4]] \n\t" + : /*no output*/ + : [src] "r" (_src + i), + [dst1] "r" (_dst + i + 0), + [dst2] "r" (_dst + i + 4), + [dst3] "r" (_dst + i + 8), + [dst4] "r" (_dst + i + 12) + : "d0","d1","d2","d3","d4","d5","d6","d7","d8","d9","d10","d11","d12","d13","d14","d15","d16","d17","d18","d19","d20","d21" + ); + } +}) +#else +CVT_FUNC(u8, f32, 16, +, +{ + for (size_t i = 0; i < w; i += 16) + { + internal::prefetch(_src + i); + uint8x16_t vline_u8 = vld1q_u8(_src + i); + + uint16x8_t vline1_u16 = vmovl_u8(vget_low_u8(vline_u8)); + uint16x8_t vline2_u16 = vmovl_u8(vget_high_u8(vline_u8)); + + uint32x4_t vline1_u32 = vmovl_u16(vget_low_u16(vline1_u16)); + uint32x4_t vline2_u32 = vmovl_u16(vget_high_u16(vline1_u16)); + uint32x4_t vline3_u32 = vmovl_u16(vget_low_u16(vline2_u16)); + uint32x4_t vline4_u32 = vmovl_u16(vget_high_u16(vline2_u16)); + + float32x4_t vline1_f32 = vcvtq_f32_u32(vline1_u32); + float32x4_t vline2_f32 = vcvtq_f32_u32(vline2_u32); + float32x4_t vline3_f32 = vcvtq_f32_u32(vline3_u32); + float32x4_t vline4_f32 = vcvtq_f32_u32(vline4_u32); + + vst1q_f32(_dst + i, vline1_f32); + vst1q_f32(_dst + i + 4, vline2_f32); + vst1q_f32(_dst + i + 8, vline3_f32); + vst1q_f32(_dst + i + 12, vline4_f32); + } +}) +#endif + +CVT_FUNC(s8, u8, 16, + int8x16_t vZero = vdupq_n_s8(0);, +{ + for (size_t i = 0; i < w; i += 16) + { + internal::prefetch(_src + i); + int8x16_t vu8 = vld1q_s8(_src + i); + uint8x16_t vu1 = vreinterpretq_u8_s8(vmaxq_s8(vu8, vZero)); + vst1q_u8(_dst + i, vu1); + } +}) + +#if __GNUC_MINOR__ < 7 +CVT_FUNC(s8, u16, 16, + register uint8x16_t zero0 asm ("q1") = vmovq_n_u8(0);, +{ + for (size_t i = 0; i < w; i += 16) + { + internal::prefetch(_src + i); + __asm__ ( + "vld1.8 {d0-d1}, [%[src]] \n\t" + "vmax.s8 q0, q1 \n\t" + "vst2.8 {d0,d2}, [%[dst1]] \n\t" + "vst2.8 {d1,d3}, [%[dst2]] \n\t" + : /*no output*/ + : [src] "r" (_src + i), + [dst1] "r" (_dst + i + 0), + [dst2] "r" (_dst + i + 8), + "w" (zero0) + : "d0","d1" + ); + } +}) +#else +CVT_FUNC(s8, u16, 16, + int8x16x2_t vline_s8; + vline_s8.val[1] = vmovq_n_s8(0);, +{ + for (size_t i = 0; i < w; i += 16) + { + internal::prefetch(_src + i); + vline_s8.val[0] = vld1q_s8(_src + i); + vline_s8.val[0] = vmaxq_s8(vline_s8.val[0], vline_s8.val[1]); + vst2q_s8((int8_t*)(_dst + i), vline_s8); + } +}) +#endif + +#if __GNUC_MINOR__ < 6 +CVT_FUNC(s8, s16, 16, +, +{ + for (size_t i = 0; i < w; i += 16) + { + internal::prefetch(_src + i); + __asm__ ( + "vld1.8 {d0-d1}, [%[src]] \n\t" + "vmovl.s8 q1, d0 \n\t" + "vmovl.s8 q2, d1 \n\t" + "vst1.16 {d2-d3}, [%[dst1]] \n\t" + "vst1.16 {d4-d5}, [%[dst2]] \n\t" + : /*no output*/ + : [src] "r" (_src + i), + [dst1] "r" (_dst + i + 0), + [dst2] "r" (_dst + i + 8) + : "d0","d1","d2","d3","d4","d5" + ); + } +}) +#else +CVT_FUNC(s8, s16, 16, +, +{ + for (size_t i = 0; i < w; i += 16) + { + internal::prefetch(_src + i); + int8x16_t vline_s8 = vld1q_s8(_src + i); + + int16x8_t vline1_s16 = vmovl_s8(vget_low_s8(vline_s8)); + int16x8_t vline2_s16 = vmovl_s8(vget_high_s8(vline_s8)); + + vst1q_s16(_dst + i, vline1_s16); + vst1q_s16(_dst + i + 8, vline2_s16); + } +}) +#endif + +#if __GNUC_MINOR__ < 7 +CVT_FUNC(s8, s32, 16, +, +{ + for (size_t i = 0; i < w; i += 16) + { + internal::prefetch(_src + i); + __asm__ ( + "vld1.8 {d0-d1}, [%[src]] \n\t" + "vmovl.s8 q1, d0 \n\t" + "vmovl.s8 q2, d1 \n\t" + "vmovl.s16 q3, d2 \n\t" + "vmovl.s16 q4, d3 \n\t" + "vmovl.s16 q5, d4 \n\t" + "vmovl.s16 q6, d5 \n\t" + "vst1.32 {d6-d7}, [%[dst1]] \n\t" + "vst1.32 {d8-d9}, [%[dst2]] \n\t" + "vst1.32 {d10-d11}, [%[dst3]] \n\t" + "vst1.32 {d12-d13}, [%[dst4]] \n\t" + : /*no output*/ + : [src] "r" (_src + i), + [dst1] "r" (_dst + i + 0), + [dst2] "r" (_dst + i + 4), + [dst3] "r" (_dst + i + 8), + [dst4] "r" (_dst + i + 12) + : "d0","d1","d2","d3","d4","d5","d6","d7","d8","d9","d10","d11","d12","d13" + ); + } +}) +#else +CVT_FUNC(s8, s32, 16, +, +{ + for (size_t i = 0; i < w; i += 16) + { + internal::prefetch(_src + i); + int8x16_t vline_s8 = vld1q_s8(_src + i); + + int16x8_t vline1_s16 = vmovl_s8(vget_low_s8(vline_s8)); + int16x8_t vline2_s16 = vmovl_s8(vget_high_s8(vline_s8)); + + int32x4_t vline1_s32 = vmovl_s16(vget_low_s16(vline1_s16)); + int32x4_t vline2_s32 = vmovl_s16(vget_high_s16(vline1_s16)); + int32x4_t vline3_s32 = vmovl_s16(vget_low_s16(vline2_s16)); + int32x4_t vline4_s32 = vmovl_s16(vget_high_s16(vline2_s16)); + + vst1q_s32(_dst + i, vline1_s32); + vst1q_s32(_dst + i + 4, vline2_s32); + vst1q_s32(_dst + i + 8, vline3_s32); + vst1q_s32(_dst + i + 12, vline4_s32); + } +}) +#endif + +#if __GNUC_MINOR__ < 6 +CVT_FUNC(s8, f32, 16, +, +{ + for (size_t i = 0; i < w; i += 16) + { + internal::prefetch(_src + i); + __asm__ ( + "vld1.8 {d0-d1}, [%[src]] \n\t" + "vmovl.s8 q1, d0 \n\t" + "vmovl.s8 q2, d1 \n\t" + "vmovl.s16 q3, d2 \n\t" + "vmovl.s16 q4, d3 \n\t" + "vmovl.s16 q5, d4 \n\t" + "vmovl.s16 q6, d5 \n\t" + "vcvt.f32.s32 q7, q3 \n\t" + "vcvt.f32.s32 q8, q4 \n\t" + "vcvt.f32.s32 q9, q5 \n\t" + "vcvt.f32.s32 q10, q6 \n\t" + "vst1.32 {d14-d15}, [%[dst1]] \n\t" + "vst1.32 {d16-d17}, [%[dst2]] \n\t" + "vst1.32 {d18-d19}, [%[dst3]] \n\t" + "vst1.32 {d20-d21}, [%[dst4]] \n\t" + : /*no output*/ + : [src] "r" (_src + i), + [dst1] "r" (_dst + i + 0), + [dst2] "r" (_dst + i + 4), + [dst3] "r" (_dst + i + 8), + [dst4] "r" (_dst + i + 12) + : "d0","d1","d2","d3","d4","d5","d6","d7","d8","d9","d10","d11","d12","d13","d14","d15","d16","d17","d18","d19","d20","d21" + ); + } +}) +#else +CVT_FUNC(s8, f32, 16, +, +{ + for (size_t i = 0; i < w; i += 16) + { + internal::prefetch(_src + i); + int8x16_t vline_s8 = vld1q_s8(_src + i); + + int16x8_t vline1_s16 = vmovl_s8(vget_low_s8(vline_s8)); + int16x8_t vline2_s16 = vmovl_s8(vget_high_s8(vline_s8)); + + int32x4_t vline1_s32 = vmovl_s16(vget_low_s16(vline1_s16)); + int32x4_t vline2_s32 = vmovl_s16(vget_high_s16(vline1_s16)); + int32x4_t vline3_s32 = vmovl_s16(vget_low_s16(vline2_s16)); + int32x4_t vline4_s32 = vmovl_s16(vget_high_s16(vline2_s16)); + + float32x4_t vline1_f32 = vcvtq_f32_s32(vline1_s32); + float32x4_t vline2_f32 = vcvtq_f32_s32(vline2_s32); + float32x4_t vline3_f32 = vcvtq_f32_s32(vline3_s32); + float32x4_t vline4_f32 = vcvtq_f32_s32(vline4_s32); + + vst1q_f32(_dst + i, vline1_f32); + vst1q_f32(_dst + i + 4, vline2_f32); + vst1q_f32(_dst + i + 8, vline3_f32); + vst1q_f32(_dst + i + 12, vline4_f32); + } +}) +#endif + +#if __GNUC_MINOR__ < 6 +CVT_FUNC(u16, u8, 16, +, +{ + for (size_t i = 0; i < w; i += 16) + { + internal::prefetch(_src + i); + __asm__ ( + "vld1.8 {d0-d1}, [%[src1]] \n\t" + "vqmovn.u16 d4, q0 \n\t" + "vld1.8 {d2-d3}, [%[src2]] \n\t" + "vqmovn.u16 d5, q1 \n\t" + "vst1.8 {d4-d5}, [%[dst]] \n\t" + : /*no output*/ + : [src1] "r" (_src + i), + [src2] "r" (_src + i + 8), + [dst] "r" (_dst + i + 0) + : "d0","d1","d2","d3","d4","d5" + ); + } +}) +#else +CVT_FUNC(u16, u8, 16, +, +{ + for (size_t i = 0; i < w; i += 16) + { + internal::prefetch(_src + i); + uint16x8_t vline1_u16 = vld1q_u16(_src + i); + uint16x8_t vline2_u16 = vld1q_u16(_src + i + 8); + + uint8x8_t vline1_u8 = vqmovn_u16(vline1_u16); + uint8x8_t vline2_u8 = vqmovn_u16(vline2_u16); + + vst1q_u8(_dst + i, vcombine_u8(vline1_u8, vline2_u8)); + } +}) +#endif + +#if __GNUC_MINOR__ < 6 +CVT_FUNC(u16, s8, 16, + register uint8x16_t v127 asm ("q4") = vmovq_n_u8(127);, +{ + for (size_t i = 0; i < w; i += 16) + { + internal::prefetch(_src + i); + __asm__ ( + "vld1.8 {d0-d1}, [%[src1]] \n\t" + "vqmovn.u16 d4, q0 \n\t" + "vld1.8 {d2-d3}, [%[src2]] \n\t" + "vqmovn.u16 d5, q1 \n\t" + "vmin.u8 q3, q2, q4 \n\t" + "vst1.8 {d6-d7}, [%[dst]] \n\t" + : /*no output*/ + : [src1] "r" (_src + i), + [src2] "r" (_src + i + 8), + [dst] "r" (_dst + i + 0), + "w" (v127) + : "d0","d1","d2","d3","d4","d5","d6","d7" + ); + } +}) +#else +CVT_FUNC(u16, s8, 16, + uint8x8_t v127 = vmov_n_u8(127);, +{ + for (size_t i = 0; i < w; i += 16) + { + internal::prefetch(_src + i); + uint16x8_t vline1_u16 = vld1q_u16(_src + i); + uint16x8_t vline2_u16 = vld1q_u16(_src + i + 8); + + uint8x8_t vline1_u8 = vqmovn_u16(vline1_u16); + uint8x8_t vline2_u8 = vqmovn_u16(vline2_u16); + vline1_u8 = vmin_u8(vline1_u8, v127); + vline2_u8 = vmin_u8(vline2_u8, v127); + + vst1q_s8(_dst + i, vcombine_s8(vreinterpret_s8_u8(vline1_u8), vreinterpret_s8_u8(vline2_u8))); + } +}) +#endif + +#if __GNUC_MINOR__ < 7 +CVT_FUNC(u16, s16, 8, + register uint16x8_t v32767 asm ("q4") = vmovq_n_u16(0x7FFF);, +{ + for (size_t i = 0; i < w; i += 8) + { + internal::prefetch(_src + i); + __asm__ ( + "vld1.16 {d0-d1}, [%[src]] \n\t" + "vmin.u16 q1, q0, q4 \n\t" + "vst1.16 {d2-d3}, [%[dst]] \n\t" + : /*no output*/ + : [src] "r" (_src + i), + [dst] "r" (_dst + i + 0), + "w" (v32767) + : "d0","d1","d2","d3" + ); + } +}) +#else +CVT_FUNC(u16, s16, 8, + uint16x8_t v32767 = vmovq_n_u16(0x7FFF);, +{ + for (size_t i = 0; i < w; i += 8) + { + internal::prefetch(_src + i); + uint16x8_t vline_u16 = vld1q_u16(_src + i); + vline_u16 = vminq_u16(vline_u16, v32767); + vst1q_s16((_dst + i), vreinterpretq_s16_u16(vline_u16)); + } +}) +#endif + +#if __GNUC_MINOR__ < 7 +CVT_FUNC(u16, s32, 8, + register uint16x8_t zero0 asm ("q1") = vmovq_n_u16(0);, +{ + for (size_t i = 0; i < w; i += 8) + { + internal::prefetch(_src + i); + __asm__ ( + "vld1.16 {d0-d1}, [%[src]] \n\t" + "vst2.16 {d0,d2}, [%[dst1]] \n\t" + "vst2.16 {d1,d3}, [%[dst2]] \n\t" + : /*no output*/ + : [src] "r" (_src + i), + [dst1] "r" (_dst + i), + [dst2] "r" (_dst + i + 4), + "w" (zero0) + : "d0","d1"//,"d2","d3"//,"d4","d5","d6","d7" + ); + } +}) +#else +CVT_FUNC(u16, s32, 8, + uint16x8x2_t vline; + vline.val[1] = vmovq_n_u16(0);, +{ + for (size_t i = 0; i < w; i += 8) + { + internal::prefetch(_src + i); + vline.val[0] = vld1q_u16(_src + i); + vst2q_u16((uint16_t*)(_dst + i), vline); + } +}) +#endif + +#if __GNUC_MINOR__ < 6 +CVT_FUNC(u16, f32, 8, +, +{ + for (size_t i = 0; i < w; i += 8) + { + internal::prefetch(_src + i); + __asm__ ( + "vld1.16 {d0-d1}, [%[src]] \n\t" + "vmovl.u16 q1, d0 \n\t" + "vmovl.u16 q2, d1 \n\t" + "vcvt.f32.u32 q3, q1 \n\t" + "vcvt.f32.u32 q4, q2 \n\t" + "vst1.32 {d6-d7}, [%[dst1]] \n\t" + "vst1.32 {d8-d9}, [%[dst2]] \n\t" + : /*no output*/ + : [src] "r" (_src + i), + [dst1] "r" (_dst + i + 0), + [dst2] "r" (_dst + i + 4) + : "d0","d1","d2","d3","d4","d5","d6","d7","d8","d9" + ); + } +}) +#else +CVT_FUNC(u16, f32, 8, +, +{ + for (size_t i = 0; i < w; i += 8) + { + internal::prefetch(_src + i); + uint16x8_t vline_u16 = vld1q_u16(_src + i); + + uint32x4_t vline_u32_lo = vmovl_u16(vget_low_u16(vline_u16)); + uint32x4_t vline_u32_hi = vmovl_u16(vget_high_u16(vline_u16)); + + float32x4_t vline_f32_lo = vcvtq_f32_u32(vline_u32_lo); + float32x4_t vline_f32_hi = vcvtq_f32_u32(vline_u32_hi); + + vst1q_f32(_dst + i, vline_f32_lo); + vst1q_f32(_dst + i + 4, vline_f32_hi); + } +}) +#endif + +#if __GNUC_MINOR__ < 6 +CVT_FUNC(s16, u8, 16, +, +{ + for (size_t i = 0; i < w; i += 16) + { + internal::prefetch(_src + i); + __asm__ ( + "vld1.8 {d0-d1}, [%[src1]] \n\t" + "vld1.8 {d2-d3}, [%[src2]] \n\t" + "vqmovun.s16 d4, q0 \n\t" + "vqmovun.s16 d5, q1 \n\t" + "vst1.8 {d4-d5}, [%[dst]] \n\t" + : /*no output*/ + : [src1] "r" (_src + i), + [src2] "r" (_src + i + 8), + [dst] "r" (_dst + i + 0) + : "d0","d1","d2","d3","d4","d5" + ); + } +}) +#else +CVT_FUNC(s16, u8, 16, +, +{ + for (size_t i = 0; i < w; i += 16) + { + internal::prefetch(_src + i); + int16x8_t vline1_s16 = vld1q_s16(_src + i); + int16x8_t vline2_s16 = vld1q_s16(_src + i + 8); + + uint8x8_t vline1_u8 = vqmovun_s16(vline1_s16); + uint8x8_t vline2_u8 = vqmovun_s16(vline2_s16); + + vst1q_u8(_dst + i, vcombine_u8(vline1_u8, vline2_u8)); + } +}) +#endif + +#if __GNUC_MINOR__ < 6 +CVT_FUNC(s16, s8, 16, +, +{ + for (size_t i = 0; i < w; i += 16) + { + internal::prefetch(_src + i); + __asm__ ( + "vld1.8 {d0-d1}, [%[src1]] \n\t" + "vld1.8 {d2-d3}, [%[src2]] \n\t" + "vqmovn.s16 d4, q0 \n\t" + "vqmovn.s16 d5, q1 \n\t" + "vst1.8 {d4-d5}, [%[dst]] \n\t" + : /*no output*/ + : [src1] "r" (_src + i), + [src2] "r" (_src + i + 8), + [dst] "r" (_dst + i + 0) + : "d0","d1","d2","d3","d4","d5" + ); + } +}) +#else +CVT_FUNC(s16, s8, 16, +, +{ + for (size_t i = 0; i < w; i += 16) + { + internal::prefetch(_src + i); + int16x8_t vline1_s16 = vld1q_s16(_src + i); + int16x8_t vline2_s16 = vld1q_s16(_src + i + 8); + + int8x8_t vline1_s8 = vqmovn_s16(vline1_s16); + int8x8_t vline2_s8 = vqmovn_s16(vline2_s16); + + vst1q_s8(_dst + i, vcombine_s8(vline1_s8, vline2_s8)); + } +}) +#endif + +#if __GNUC_MINOR__ < 7 +CVT_FUNC(s16, u16, 8, + register int16x8_t vZero asm ("q4") = vmovq_n_s16(0);, +{ + for (size_t i = 0; i < w; i += 8) + { + internal::prefetch(_src + i); + __asm__ ( + "vld1.16 {d0-d1}, [%[src]] \n\t" + "vmax.s16 q1, q0, q4 \n\t" + "vst1.16 {d2-d3}, [%[dst]] \n\t" + : /*no output*/ + : [src] "r" (_src + i), + [dst] "r" (_dst + i + 0), + "w" (vZero) + : "d0","d1","d2","d3" + ); + } +}) +#else +CVT_FUNC(s16, u16, 8, + int16x4_t vZero = vmov_n_s16(0);, +{ + for (size_t i = 0; i < w; i += 8) + { + internal::prefetch(_src + i); + int16x8_t vline_s16 = vld1q_s16(_src + i); + + int16x4_t vline_s16_lo = vmax_s16(vget_low_s16(vline_s16), vZero); + int16x4_t vline_s16_hi = vmax_s16(vget_high_s16(vline_s16), vZero); + + vst1q_u16(_dst + i, vcombine_u16(vreinterpret_u16_s16(vline_s16_lo), vreinterpret_u16_s16(vline_s16_hi))); + } +}) +#endif + +#if __GNUC_MINOR__ < 6 +CVT_FUNC(s16, s32, 8, +, +{ + for (size_t i = 0; i < w; i += 8) + { + internal::prefetch(_src + i); + __asm__ ( + "vld1.16 {d0-d1}, [%[src]] \n\t" + "vmovl.s16 q1, d0 \n\t" + "vmovl.s16 q2, d1 \n\t" + "vst1.32 {d2-d3}, [%[dst1]] \n\t" + "vst1.32 {d4-d5}, [%[dst2]] \n\t" + : /*no output*/ + : [src] "r" (_src + i), + [dst1] "r" (_dst + i + 0), + [dst2] "r" (_dst + i + 4) + : "d0","d1","d2","d3","d4","d5" + ); + } +}) +#else +CVT_FUNC(s16, s32, 8, +, +{ + for (size_t i = 0; i < w; i += 8) + { + internal::prefetch(_src + i); + int16x8_t vline_s16 = vld1q_s16(_src + i); + + int32x4_t vline_s32_lo = vmovl_s16(vget_low_s16(vline_s16)); + int32x4_t vline_s32_hi = vmovl_s16(vget_high_s16(vline_s16)); + + vst1q_s32(_dst + i, vline_s32_lo); + vst1q_s32(_dst + i + 4, vline_s32_hi); + } +}) +#endif + +#if __GNUC_MINOR__ < 6 +CVT_FUNC(s16, f32, 8, +, +{ + for (size_t i = 0; i < w; i += 8) + { + internal::prefetch(_src + i); + __asm__ ( + "vld1.16 {d0-d1}, [%[src]] \n\t" + "vmovl.s16 q1, d0 \n\t" + "vmovl.s16 q2, d1 \n\t" + "vcvt.f32.s32 q3, q1 \n\t" + "vcvt.f32.s32 q4, q2 \n\t" + "vst1.32 {d6-d7}, [%[dst1]] \n\t" + "vst1.32 {d8-d9}, [%[dst2]] \n\t" + : /*no output*/ + : [src] "r" (_src + i), + [dst1] "r" (_dst + i + 0), + [dst2] "r" (_dst + i + 4) + : "d0","d1","d2","d3","d4","d5","d6","d7","d8","d9" + ); + } +}) +#else +CVT_FUNC(s16, f32, 8, +, +{ + for (size_t i = 0; i < w; i += 8) + { + internal::prefetch(_src + i); + int16x8_t vline_s16 = vld1q_s16(_src + i); + + int32x4_t vline_s32_lo = vmovl_s16(vget_low_s16(vline_s16)); + int32x4_t vline_s32_hi = vmovl_s16(vget_high_s16(vline_s16)); + float32x4_t vline_f32_lo = vcvtq_f32_s32(vline_s32_lo); + float32x4_t vline_f32_hi = vcvtq_f32_s32(vline_s32_hi); + + vst1q_f32(_dst + i, vline_f32_lo); + vst1q_f32(_dst + i + 4, vline_f32_hi); + } +}) +#endif + +#if __GNUC_MINOR__ < 6 +CVT_FUNC(s32, u8, 8, +, +{ + for (size_t i = 0; i < w; i += 8) + { + internal::prefetch(_src + i); + __asm__ ( + "vld1.32 {d0-d1}, [%[src1]] \n\t" + "vld1.32 {d2-d3}, [%[src2]] \n\t" + "vqmovun.s32 d4, q0 \n\t" + "vqmovun.s32 d5, q1 \n\t" + "vqmovn.u16 d6, q2 \n\t" + "vst1.8 {d6}, [%[dst]] \n\t" + : /*no output*/ + : [src1] "r" (_src + i + 0), + [src2] "r" (_src + i + 4), + [dst] "r" (_dst + i) + : "d0","d1","d2","d3","d4","d5","d6" + ); + } +}) +#else +CVT_FUNC(s32, u8, 8, +, +{ + for (size_t i = 0; i < w; i += 8) + { + internal::prefetch(_src + i); + int32x4_t vline1_s32 = vld1q_s32(_src + i); + int32x4_t vline2_s32 = vld1q_s32(_src + i + 4); + + uint16x4_t vline1_u16 = vqmovun_s32(vline1_s32); + uint16x4_t vline2_u16 = vqmovun_s32(vline2_s32); + uint8x8_t vline_u8 = vqmovn_u16(vcombine_u16(vline1_u16, vline2_u16)); + + vst1_u8(_dst + i, vline_u8); + } +}) +#endif + +#if __GNUC_MINOR__ < 6 +CVT_FUNC(s32, s8, 8, +, +{ + for (size_t i = 0; i < w; i += 8) + { + internal::prefetch(_src + i); + __asm__ ( + "vld1.32 {d0-d1}, [%[src1]] \n\t" + "vld1.32 {d2-d3}, [%[src2]] \n\t" + "vqmovn.s32 d4, q0 \n\t" + "vqmovn.s32 d5, q1 \n\t" + "vqmovn.s16 d6, q2 \n\t" + "vst1.8 {d6}, [%[dst]] \n\t" + : /*no output*/ + : [src1] "r" (_src + i + 0), + [src2] "r" (_src + i + 4), + [dst] "r" (_dst + i) + : "d0","d1","d2","d3","d4","d5","d6" + ); + } +}) +#else +CVT_FUNC(s32, s8, 8, +, +{ + for (size_t i = 0; i < w; i += 8) + { + internal::prefetch(_src + i); + int32x4_t vline1_s32 = vld1q_s32(_src + i); + int32x4_t vline2_s32 = vld1q_s32(_src + i + 4); + + int16x4_t vline1_s16 = vqmovn_s32(vline1_s32); + int16x4_t vline2_s16 = vqmovn_s32(vline2_s32); + int8x8_t vline_s8 = vqmovn_s16(vcombine_s16(vline1_s16, vline2_s16)); + + vst1_s8(_dst + i, vline_s8); + } +}) +#endif + +#if __GNUC_MINOR__ < 6 +CVT_FUNC(s32, u16, 8, +, +{ + for (size_t i = 0; i < w; i += 8) + { + internal::prefetch(_src + i); + __asm__ ( + "vld1.32 {d0-d1}, [%[src1]] \n\t" + "vld1.32 {d2-d3}, [%[src2]] \n\t" + "vqmovun.s32 d4, q0 \n\t" + "vqmovun.s32 d5, q1 \n\t" + "vst1.16 {d4-d5}, [%[dst]] \n\t" + : /*no output*/ + : [src1] "r" (_src + i + 0), + [src2] "r" (_src + i + 4), + [dst] "r" (_dst + i) + : "d0","d1","d2","d3","d4","d5" + ); + } +}) +#else +CVT_FUNC(s32, u16, 8, +, +{ + for (size_t i = 0; i < w; i += 8) + { + internal::prefetch(_src + i); + int32x4_t vline1_s32 = vld1q_s32(_src + i); + int32x4_t vline2_s32 = vld1q_s32(_src + i + 4); + + uint16x4_t vline1_u16 = vqmovun_s32(vline1_s32); + uint16x4_t vline2_u16 = vqmovun_s32(vline2_s32); + + vst1q_u16(_dst + i, vcombine_u16(vline1_u16, vline2_u16)); + } +}) +#endif + +#if __GNUC_MINOR__ < 6 +CVT_FUNC(s32, s16, 8, +, +{ + for (size_t i = 0; i < w; i += 8) + { + internal::prefetch(_src + i); + __asm__ ( + "vld1.32 {d0-d1}, [%[src1]] \n\t" + "vld1.32 {d2-d3}, [%[src2]] \n\t" + "vqmovn.s32 d4, q0 \n\t" + "vqmovn.s32 d5, q1 \n\t" + "vst1.8 {d4-d5}, [%[dst]] \n\t" + : /*no output*/ + : [src1] "r" (_src + i + 0), + [src2] "r" (_src + i + 4), + [dst] "r" (_dst + i) + : "d0","d1","d2","d3","d4","d5" + ); + } +}) +#else +CVT_FUNC(s32, s16, 8, +, +{ + for (size_t i = 0; i < w; i += 8) + { + internal::prefetch(_src + i); + int32x4_t vline1_s32 = vld1q_s32(_src + i); + int32x4_t vline2_s32 = vld1q_s32(_src + i + 4); + + int16x4_t vline1_s16 = vqmovn_s32(vline1_s32); + int16x4_t vline2_s16 = vqmovn_s32(vline2_s32); + + vst1q_s16(_dst + i, vcombine_s16(vline1_s16, vline2_s16)); + } +}) +#endif + +#if __GNUC_MINOR__ < 6 +CVT_FUNC(s32, f32, 8, +, +{ + for (size_t i = 0; i < w; i += 8) + { + internal::prefetch(_src + i); + __asm__ ( + "vld1.32 {d0-d1}, [%[src]] \n\t" + "vcvt.f32.s32 q1, q0 \n\t" + "vst1.32 {d2-d3}, [%[dst]] \n\t" + : /*no output*/ + : [src] "r" (_src + i), + [dst] "r" (_dst + i) + : "d0","d1","d2","d3"//,"d4","d5" + ); + __asm__ ( + "vld1.32 {d0-d1}, [%[src]] \n\t" + "vcvt.f32.s32 q1, q0 \n\t" + "vst1.32 {d2-d3}, [%[dst]] \n\t" + : /*no output*/ + : [src] "r" (_src + i + 4), + [dst] "r" (_dst + i + 4) + : "d0","d1","d2","d3"//,"d4","d5" + ); + } +}) +#else +CVT_FUNC(s32, f32, 8, +, +{ + for (size_t i = 0; i < w; i += 8) + { + internal::prefetch(_src + i); + int32x4_t vline_s32 = vld1q_s32(_src + i); + float32x4_t vline_f32 = vcvtq_f32_s32(vline_s32); + vst1q_f32(_dst + i, vline_f32); + + vline_s32 = vld1q_s32(_src + i + 4); + vline_f32 = vcvtq_f32_s32(vline_s32); + vst1q_f32(_dst + i + 4, vline_f32); + } +}) +#endif + +#if __GNUC_MINOR__ < 6 +CVT_FUNC(f32, u8, 8, + register float32x4_t vmult asm ("q0") = vdupq_n_f32((float)(1 << 16)); + register uint32x4_t vmask asm ("q1") = vdupq_n_u32(1<<16);, +{ + for (size_t i = 0; i < w; i += 8) + { + internal::prefetch(_src + i); + __asm__ ( + "vld1.32 {d4-d5}, [%[src1]] \n\t" + "vld1.32 {d6-d7}, [%[src2]] \n\t" + "vmul.f32 q4, q2, q0 \n\t" + "vmul.f32 q5, q3, q0 \n\t" + "vcvt.u32.f32 q6, q4 \n\t" + "vcvt.u32.f32 q7, q5 \n\t" + "vbic q8, q1, q6 \n\t" + "vbic q9, q1, q7 \n\t" + "vshr.u32 q10, q8, #16 \n\t" + "vshr.u32 q11, q9, #16 \n\t" + "vqsub.u32 q12, q6, q10 \n\t" + "vqsub.u32 q13, q7, q11 \n\t" + "vqrshrn.u32 d28, q12, #16 \n\t" + "vqrshrn.u32 d29, q13, #16 \n\t" + "vqmovn.u16 d30, q14 \n\t" + "vst1.8 {d30}, [%[dst]] \n\t" + : /*no output*/ + : [src1] "r" (_src + i + 0), + [src2] "r" (_src + i + 4), + [dst] "r" (_dst + i), + "w" (vmult), "w" (vmask) + : "d4","d5","d6","d7","d8","d9","d10","d11","d12","d13","d14","d15","d16","d17","d18","d19","d20","d21","d22","d23","d24","d25","d26","d27","d28","d29","d30" + ); + } +}) +#else +CVT_FUNC(f32, u8, 8, + float32x4_t vmult = vdupq_n_f32((float)(1 << 16)); + uint32x4_t vmask = vdupq_n_u32(1<<16);, +{ + for (size_t i = 0; i < w; i += 8) + { + internal::prefetch(_src + i); + float32x4_t vline1_f32 = vld1q_f32(_src + i); + float32x4_t vline2_f32 = vld1q_f32(_src + i + 4); + + float32x4_t vline1w_f32 = vmulq_f32(vline1_f32, vmult); + float32x4_t vline2w_f32 = vmulq_f32(vline2_f32, vmult); + + uint32x4_t vline1_u32 = vcvtq_u32_f32(vline1w_f32); + uint32x4_t vline2_u32 = vcvtq_u32_f32(vline2w_f32); + + uint32x4_t vl1_masked = vbicq_u32(vmask, vline1_u32); + uint32x4_t vl2_masked = vbicq_u32(vmask, vline2_u32); + uint32x4_t vl1_masked2 = vshrq_n_u32(vl1_masked, 16); + uint32x4_t vl2_masked2 = vshrq_n_u32(vl2_masked, 16); + uint32x4_t vline1r_u32 = vqsubq_u32(vline1_u32, vl1_masked2); + uint32x4_t vline2r_u32 = vqsubq_u32(vline2_u32, vl2_masked2); + + uint16x4_t vline1_u16 = vqrshrn_n_u32(vline1r_u32, 16); + uint16x4_t vline2_u16 = vqrshrn_n_u32(vline2r_u32, 16); + + uint8x8_t vline_u8 = vqmovn_u16(vcombine_u16(vline1_u16, vline2_u16)); + vst1_u8(_dst + i, vline_u8); + } +}) +#endif + +#if __GNUC_MINOR__ < 6 +CVT_FUNC(f32, s8, 8, + register float32x4_t vhalf asm ("q0") = vdupq_n_f32(0.5f);, +{ + for (size_t i = 0; i < w; i += 8) + { + internal::prefetch(_src + i); + __asm__ ( + "vld1.32 {d2-d3}, [%[src1]] \n\t" + "vld1.32 {d4-d5}, [%[src2]] \n\t" + "vadd.f32 q3, q1, q0 \n\t" + "vadd.f32 q4, q2, q0 \n\t" + "vcvt.s32.f32 q5, q3 \n\t" + "vcvt.s32.f32 q6, q4 \n\t" + "vqmovn.s32 d14, q5 \n\t" + "vqmovn.s32 d15, q6 \n\t" + "vqmovn.s16 d16, q7 \n\t" + "vst1.8 {d16}, [%[dst]] \n\t" + : /*no output*/ + : [src1] "r" (_src + i + 0), + [src2] "r" (_src + i + 4), + [dst] "r" (_dst + i), + "w" (vhalf) + : "d2","d3","d4","d5","d6","d7","d8","d9","d10","d11","d12","d13","d14","d15","d16","d17" + ); + } +}) +#else +CVT_FUNC(f32, s8, 8, + float32x4_t vhalf = vdupq_n_f32(0.5f);, +{ + for (size_t i = 0; i < w; i += 8) + { + internal::prefetch(_src + i); + float32x4_t vline1_f32 = vld1q_f32(_src + i); + float32x4_t vline2_f32 = vld1q_f32(_src + i + 4); + + vline1_f32 = vaddq_f32(vline1_f32, vhalf); + vline2_f32 = vaddq_f32(vline2_f32, vhalf); + + int32x4_t vline1_s32 = vcvtq_s32_f32(vline1_f32); + int32x4_t vline2_s32 = vcvtq_s32_f32(vline2_f32); + int16x4_t vline1_s16 = vqmovn_s32(vline1_s32); + int16x4_t vline2_s16 = vqmovn_s32(vline2_s32); + + int8x8_t vline_s8 = vqmovn_s16(vcombine_s16(vline1_s16, vline2_s16)); + + vst1_s8(_dst + i, vline_s8); + } +}) +#endif + +#if __GNUC_MINOR__ < 6 +CVT_FUNC(f32, u16, 8, + register float32x4_t vhalf asm ("q0") = vdupq_n_f32(0.5f);, +{ + for (size_t i = 0; i < w; i += 8) + { + internal::prefetch(_src + i); + __asm__ ( + "vld1.32 {d2-d3}, [%[src]] \n\t" + "vadd.f32 q2, q1, q0 \n\t" + "vcvt.u32.f32 q3, q2 \n\t" + "vqmovn.u32 d8, q3 \n\t" + "vst1.16 {d8}, [%[dst]] \n\t" + : /*no output*/ + : [src] "r" (_src + i), + [dst] "r" (_dst + i), + "w" (vhalf) + : "d2","d3","d4","d5","d6","d7","d8" + ); + __asm__ ( + "vld1.32 {d2-d3}, [%[src]] \n\t" + "vadd.f32 q2, q1, q0 \n\t" + "vcvt.u32.f32 q3, q2 \n\t" + "vqmovn.u32 d8, q3 \n\t" + "vst1.16 {d8}, [%[dst]] \n\t" + : /*no output*/ + : [src] "r" (_src + i + 4), + [dst] "r" (_dst + i + 4), + "w" (vhalf) + : "d2","d3","d4","d5","d6","d7","d8" + ); + } +}) +#else +CVT_FUNC(f32, u16, 8, + float32x4_t vhalf = vdupq_n_f32(0.5f);, +{ + for (size_t i = 0; i < w; i += 8) + { + internal::prefetch(_src + i); + float32x4_t vline_f32 = vld1q_f32(_src + i); + + vline_f32 = vaddq_f32(vline_f32, vhalf); + uint32x4_t vline_u32 = vcvtq_u32_f32(vline_f32); + uint16x4_t vline_u16 = vqmovn_u32(vline_u32); + + vst1_u16(_dst + i, vline_u16); + + vline_f32 = vld1q_f32(_src + i + 4); + + vline_f32 = vaddq_f32(vline_f32, vhalf); + vline_u32 = vcvtq_u32_f32(vline_f32); + vline_u16 = vqmovn_u32(vline_u32); + + vst1_u16(_dst + i + 4, vline_u16); + } +}) +#endif + +#if __GNUC_MINOR__ < 6 +CVT_FUNC(f32, s16, 8, + register float32x4_t vhalf asm ("q0") = vdupq_n_f32(0.5f);, +{ + for (size_t i = 0; i < w; i += 8) + { + internal::prefetch(_src + i); + __asm__ ( + "vld1.32 {d2-d3}, [%[src]] \n\t" + "vadd.f32 q2, q1, q0 \n\t" + "vcvt.s32.f32 q3, q2 \n\t" + "vqmovn.s32 d8, q3 \n\t" + "vst1.16 {d8}, [%[dst]] \n\t" + : /*no output*/ + : [src] "r" (_src + i), + [dst] "r" (_dst + i), + "w" (vhalf) + : "d2","d3","d4","d5","d6","d7","d8" + ); + __asm__ ( + "vld1.32 {d2-d3}, [%[src]] \n\t" + "vadd.f32 q2, q1, q0 \n\t" + "vcvt.s32.f32 q3, q2 \n\t" + "vqmovn.s32 d8, q3 \n\t" + "vst1.16 {d8}, [%[dst]] \n\t" + : /*no output*/ + : [src] "r" (_src + i + 4), + [dst] "r" (_dst + i + 4), + "w" (vhalf) + : "d2","d3","d4","d5","d6","d7","d8" + ); + } +}) +#else +CVT_FUNC(f32, s16, 8, + float32x4_t vhalf = vdupq_n_f32(0.5f);, +{ + for (size_t i = 0; i < w; i += 8) + { + internal::prefetch(_src + i); + float32x4_t vline_f32 = vld1q_f32(_src + i); + + vline_f32 = vaddq_f32(vline_f32, vhalf); + int32x4_t vline_s32 = vcvtq_s32_f32(vline_f32); + int16x4_t vline_s16 = vqmovn_s32(vline_s32); + + vst1_s16(_dst + i, vline_s16); + + vline_f32 = vld1q_f32(_src + i + 4); + + vline_f32 = vaddq_f32(vline_f32, vhalf); + vline_s32 = vcvtq_s32_f32(vline_f32); + vline_s16 = vqmovn_s32(vline_s32); + + vst1_s16(_dst + i + 4, vline_s16); + } +}) +#endif + +#if __GNUC_MINOR__ < 6 +CVT_FUNC(f32, s32, 8, + register float32x4_t vhalf asm ("q0") = vdupq_n_f32(0.5f);, +{ + for (size_t i = 0; i < w; i += 8) + { + internal::prefetch(_src + i); + __asm__ ( + "vld1.32 {d2-d3}, [%[src1]] \n\t" + "vld1.32 {d4-d5}, [%[src2]] \n\t" + "vadd.f32 q3, q1, q0 \n\t" + "vadd.f32 q4, q2, q0 \n\t" + "vcvt.s32.f32 q5, q3 \n\t" + "vcvt.s32.f32 q6, q4 \n\t" + "vst1.32 {q5}, [%[dst1]] \n\t" + "vst1.32 {q6}, [%[dst2]] \n\t" + : /*no output*/ + : [src1] "r" (_src + i), + [src2] "r" (_src + i + 4), + [dst1] "r" (_dst + i), + [dst2] "r" (_dst + i + 4), + "w" (vhalf) + : "d2","d3","d4","d5","d6","d7","d8","d9","d10","d11","d12","d13" + ); + } +}) +#else +CVT_FUNC(f32, s32, 8, + float32x4_t vhalf = vdupq_n_f32(0.5f);, +{ + for (size_t i = 0; i < w; i += 8) + { + internal::prefetch(_src + i); + float32x4_t vline_f32 = vld1q_f32(_src + i); + + vline_f32 = vaddq_f32(vline_f32, vhalf); + int32x4_t vline_s32 = vcvtq_s32_f32(vline_f32); + + vst1q_s32(_dst + i, vline_s32); + + vline_f32 = vld1q_f32(_src + i + 4); + + vline_f32 = vaddq_f32(vline_f32, vhalf); + vline_s32 = vcvtq_s32_f32(vline_f32); + + vst1q_s32(_dst + i + 4, vline_s32); + } +}) +#endif + +void convert(const Size2D &_size, + const u8 * srcBase, ptrdiff_t srcStride, + s16 * dstBase, ptrdiff_t dstStride) +{ + convert(_size, srcBase, srcStride, (u16*)dstBase, dstStride); +} + +} // namespace CAROTENE_NS diff --git a/3rdparty/carotene/src/convert_depth.cpp b/3rdparty/carotene/src/convert_depth.cpp new file mode 100644 index 0000000000..21b0c18a69 --- /dev/null +++ b/3rdparty/carotene/src/convert_depth.cpp @@ -0,0 +1,399 @@ +/* + * By downloading, copying, installing or using the software you agree to this license. + * If you do not agree to this license, do not download, install, + * copy or use the software. + * + * + * License Agreement + * For Open Source Computer Vision Library + * (3-clause BSD License) + * + * Copyright (C) 2014, NVIDIA Corporation, all rights reserved. + * Third party copyrights are property of their respective owners. + * + * Redistribution and use in source and binary forms, with or without modification, + * are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * * Neither the names of the copyright holders nor the names of the contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * This software is provided by the copyright holders and contributors "as is" and + * any express or implied warranties, including, but not limited to, the implied + * warranties of merchantability and fitness for a particular purpose are disclaimed. + * In no event shall copyright holders or contributors be liable for any direct, + * indirect, incidental, special, exemplary, or consequential damages + * (including, but not limited to, procurement of substitute goods or services; + * loss of use, data, or profits; or business interruption) however caused + * and on any theory of liability, whether in contract, strict liability, + * or tort (including negligence or otherwise) arising in any way out of + * the use of this software, even if advised of the possibility of such damage. + */ + +#include "common.hpp" + +#include + +namespace CAROTENE_NS { + +#ifdef CAROTENE_NEON + +namespace { + +template +void lshiftConst(const Size2D &size, + const u8 * srcBase, ptrdiff_t srcStride, + s16 * dstBase, ptrdiff_t dstStride) +{ + size_t roiw16 = size.width >= 15 ? size.width - 15 : 0; + size_t roiw8 = size.width >= 7 ? size.width - 7 : 0; + + for (size_t i = 0; i < size.height; ++i) + { + const u8 * src = internal::getRowPtr(srcBase, srcStride, i); + s16 * dst = internal::getRowPtr(dstBase, dstStride, i); + size_t j = 0; + + for (; j < roiw16; j += 16) + { + internal::prefetch(src + j); + uint8x16_t v_src = vld1q_u8(src + j); + int16x8_t v_dst0 = vreinterpretq_s16_u16(vmovl_u8(vget_low_u8(v_src))); + int16x8_t v_dst1 = vreinterpretq_s16_u16(vmovl_u8(vget_high_u8(v_src))); + + vst1q_s16(dst + j, vshlq_n_s16(v_dst0, shift)); + vst1q_s16(dst + j + 8, vshlq_n_s16(v_dst1, shift)); + } + for (; j < roiw8; j += 8) + { + int16x8_t v_dst = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(src + j))); + vst1q_s16(dst + j, vshlq_n_s16(v_dst, shift)); + } + + for (; j < size.width; j++) + { + dst[j] = ((s16)src[j] << shift); + } + } +} + +template <> +void lshiftConst<0>(const Size2D &size, + const u8 * srcBase, ptrdiff_t srcStride, + s16 * dstBase, ptrdiff_t dstStride) +{ + size_t roiw16 = size.width >= 15 ? size.width - 15 : 0; + size_t roiw8 = size.width >= 7 ? size.width - 7 : 0; + + for (size_t i = 0; i < size.height; ++i) + { + const u8 * src = internal::getRowPtr(srcBase, srcStride, i); + s16 * dst = internal::getRowPtr(dstBase, dstStride, i); + size_t j = 0; + + for (; j < roiw16; j += 16) + { + internal::prefetch(src + j); + uint8x16_t v_src = vld1q_u8(src + j); + int16x8_t v_dst0 = vreinterpretq_s16_u16(vmovl_u8(vget_low_u8(v_src))); + int16x8_t v_dst1 = vreinterpretq_s16_u16(vmovl_u8(vget_high_u8(v_src))); + + vst1q_s16(dst + j, v_dst0); + vst1q_s16(dst + j + 8, v_dst1); + } + for (; j < roiw8; j += 8) + { + int16x8_t v_dst = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(src + j))); + vst1q_s16(dst + j, v_dst); + } + + for (; j < size.width; j++) + { + dst[j] = (s16)src[j]; + } + } +} + +template +void rshiftConst(const Size2D &size, + const s16 * srcBase, ptrdiff_t srcStride, + u8 * dstBase, ptrdiff_t dstStride, + CONVERT_POLICY cpolicy) +{ + size_t roiw16 = size.width >= 15 ? size.width - 15 : 0; + size_t roiw8 = size.width >= 7 ? size.width - 7 : 0; + + for (size_t i = 0; i < size.height; ++i) + { + const s16 * src = internal::getRowPtr(srcBase, srcStride, i); + u8 * dst = internal::getRowPtr(dstBase, dstStride, i); + size_t j = 0; + + if (cpolicy == CONVERT_POLICY_SATURATE) + { + for (; j < roiw16; j += 16) + { + internal::prefetch(src + j); + int16x8_t v_src0 = vshrq_n_s16(vld1q_s16(src + j), shift), + v_src1 = vshrq_n_s16(vld1q_s16(src + j + 8), shift); + uint8x16_t v_dst = vcombine_u8(vqmovun_s16(v_src0), + vqmovun_s16(v_src1)); + vst1q_u8(dst + j, v_dst); + } + for (; j < roiw8; j += 8) + { + int16x8_t v_src = vshrq_n_s16(vld1q_s16(src + j), shift); + vst1_u8(dst + j, vqmovun_s16(v_src)); + } + + for (; j < size.width; j++) + { + dst[j] = internal::saturate_cast((src[j] >> shift)); + } + } + else // CONVERT_POLICY_WRAP + { + for (; j < roiw16; j += 16) + { + internal::prefetch(src + j); + int16x8_t v_src0 = vshrq_n_s16(vld1q_s16(src + j), shift), + v_src1 = vshrq_n_s16(vld1q_s16(src + j + 8), shift); + int8x16_t v_dst = vcombine_s8(vmovn_s16(v_src0), + vmovn_s16(v_src1)); + vst1q_u8(dst + j, vreinterpretq_u8_s8(v_dst)); + } + for (; j < roiw8; j += 8) + { + int16x8_t v_src = vshrq_n_s16(vld1q_s16(src + j), shift); + vst1_u8(dst + j, vreinterpret_u8_s8(vmovn_s16(v_src))); + } + + for (; j < size.width; j++) + { + dst[j] = (u8)((src[j] >> shift)); + } + } + } +} + +template <> +void rshiftConst<0>(const Size2D &size, + const s16 * srcBase, ptrdiff_t srcStride, + u8 * dstBase, ptrdiff_t dstStride, + CONVERT_POLICY cpolicy) +{ + size_t roiw16 = size.width >= 15 ? size.width - 15 : 0; + size_t roiw8 = size.width >= 7 ? size.width - 7 : 0; + + for (size_t i = 0; i < size.height; ++i) + { + const s16 * src = internal::getRowPtr(srcBase, srcStride, i); + u8 * dst = internal::getRowPtr(dstBase, dstStride, i); + size_t j = 0; + + if (cpolicy == CONVERT_POLICY_SATURATE) + { + for (; j < roiw16; j += 16) + { + internal::prefetch(src + j); + int16x8_t v_src0 = vld1q_s16(src + j), v_src1 = vld1q_s16(src + j + 8); + uint8x16_t v_dst = vcombine_u8(vqmovun_s16(v_src0), vqmovun_s16(v_src1)); + vst1q_u8(dst + j, v_dst); + } + for (; j < roiw8; j += 8) + { + int16x8_t v_src = vld1q_s16(src + j); + vst1_u8(dst + j, vqmovun_s16(v_src)); + } + + for (; j < size.width; j++) + { + dst[j] = internal::saturate_cast(src[j]); + } + } + else // CONVERT_POLICY_WRAP + { + for (; j < roiw16; j += 16) + { + internal::prefetch(src + j); + int16x8_t v_src0 = vld1q_s16(src + j), v_src1 = vld1q_s16(src + j + 8); + int8x16_t v_dst = vcombine_s8(vmovn_s16(v_src0), vmovn_s16(v_src1)); + vst1q_u8(dst + j, vreinterpretq_u8_s8(v_dst)); + } + for (; j < roiw8; j += 8) + { + int16x8_t v_src = vld1q_s16(src + j); + vst1_u8(dst + j, vreinterpret_u8_s8(vmovn_s16(v_src))); + } + + for (; j < size.width; j++) + { + dst[j] = (u8)src[j]; + } + } + } +} + +typedef void (* lshiftConstFunc)(const Size2D &size, + const u8 * srcBase, ptrdiff_t srcStride, + s16 * dstBase, ptrdiff_t dstStride); + +typedef void (* rshiftConstFunc)(const Size2D &size, + const s16 * srcBase, ptrdiff_t srcStride, + u8 * dstBase, ptrdiff_t dstStride, + CONVERT_POLICY cpolicy); + +} // namespace + +#endif + +void lshift(const Size2D &size, + const u8 * srcBase, ptrdiff_t srcStride, + s16 * dstBase, ptrdiff_t dstStride, + u32 shift) +{ + internal::assertSupportedConfiguration(); + +#ifdef CAROTENE_NEON + if (shift >= 16u) + { + for (size_t i = 0; i < size.height; ++i) + { + s16 * dst = internal::getRowPtr(dstBase, dstStride, i); + std::memset(dst, 0, sizeof(s16) * size.width); + } + return; + } + + // this ugly contruction is needed to avoid: + // /usr/lib/gcc/arm-linux-gnueabihf/4.8/include/arm_neon.h:3581:59: error: argument must be a constant + // return (int16x8_t)__builtin_neon_vshl_nv8hi (__a, __b, 1); + + lshiftConstFunc funcs[16] = + { + lshiftConst<0>, + lshiftConst<1>, + lshiftConst<2>, + lshiftConst<3>, + lshiftConst<4>, + lshiftConst<5>, + lshiftConst<6>, + lshiftConst<7>, + lshiftConst<8>, + lshiftConst<9>, + lshiftConst<10>, + lshiftConst<11>, + lshiftConst<12>, + lshiftConst<13>, + lshiftConst<14>, + lshiftConst<15> + }, func = funcs[shift]; + + func(size, srcBase, srcStride, dstBase, dstStride); +#else + (void)size; + (void)srcBase; + (void)srcStride; + (void)dstBase; + (void)dstStride; + (void)shift; +#endif +} + +void rshift(const Size2D &size, + const s16 * srcBase, ptrdiff_t srcStride, + u8 * dstBase, ptrdiff_t dstStride, + u32 shift, CONVERT_POLICY cpolicy) +{ + internal::assertSupportedConfiguration(); + +#ifdef CAROTENE_NEON + if (shift >= 16) + { + if (cpolicy == CONVERT_POLICY_WRAP) + { + size_t roiw16 = size.width >= 15 ? size.width - 15 : 0; + size_t roiw8 = size.width >= 7 ? size.width - 7 : 0; + int16x8_t v_zero = vdupq_n_s16(0); + + for (size_t i = 0; i < size.height; ++i) + { + const s16 * src = internal::getRowPtr(srcBase, srcStride, i); + u8 * dst = internal::getRowPtr(dstBase, dstStride, i); + size_t j = 0; + + for (; j < roiw16; j += 16) + { + internal::prefetch(src + j); + int16x8_t v_src0 = vld1q_s16(src + j), v_src1 = vld1q_s16(src + j + 8); + uint8x16_t v_dst = vcombine_u8(vmovn_u16(vcltq_s16(v_src0, v_zero)), + vmovn_u16(vcltq_s16(v_src1, v_zero))); + vst1q_u8(dst + j, v_dst); + } + for (; j < roiw8; j += 8) + { + int16x8_t v_src = vld1q_s16(src + j); + vst1_u8(dst + j, vmovn_u16(vcltq_s16(v_src, v_zero))); + } + + for (; j < size.width; j++) + { + dst[j] = src[j] >= 0 ? 0 : 255; + } + } + } + else + { + for (size_t i = 0; i < size.height; ++i) + { + u8 * dst = internal::getRowPtr(dstBase, dstStride, i); + std::memset(dst, 0, sizeof(u8) * size.width); + } + } + return; + } + + // this ugly contruction is needed to avoid: + // /usr/lib/gcc/arm-linux-gnueabihf/4.8/include/arm_neon.h:3581:59: error: argument must be a constant + // return (int16x8_t)__builtin_neon_vshr_nv8hi (__a, __b, 1); + + rshiftConstFunc funcs[16] = + { + rshiftConst<0>, + rshiftConst<1>, + rshiftConst<2>, + rshiftConst<3>, + rshiftConst<4>, + rshiftConst<5>, + rshiftConst<6>, + rshiftConst<7>, + rshiftConst<8>, + rshiftConst<9>, + rshiftConst<10>, + rshiftConst<11>, + rshiftConst<12>, + rshiftConst<13>, + rshiftConst<14>, + rshiftConst<15> + }, func = funcs[shift]; + + func(size, srcBase, srcStride, dstBase, dstStride, cpolicy); +#else + (void)size; + (void)srcBase; + (void)srcStride; + (void)dstBase; + (void)dstStride; + (void)shift; + (void)cpolicy; +#endif +} + +} // namespace CAROTENE_NS diff --git a/3rdparty/carotene/src/convert_scale.cpp b/3rdparty/carotene/src/convert_scale.cpp new file mode 100644 index 0000000000..50c110b3ee --- /dev/null +++ b/3rdparty/carotene/src/convert_scale.cpp @@ -0,0 +1,2498 @@ +/* + * By downloading, copying, installing or using the software you agree to this license. + * If you do not agree to this license, do not download, install, + * copy or use the software. + * + * + * License Agreement + * For Open Source Computer Vision Library + * (3-clause BSD License) + * + * Copyright (C) 2012-2015, NVIDIA Corporation, all rights reserved. + * Third party copyrights are property of their respective owners. + * + * Redistribution and use in source and binary forms, with or without modification, + * are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * * Neither the names of the copyright holders nor the names of the contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * This software is provided by the copyright holders and contributors "as is" and + * any express or implied warranties, including, but not limited to, the implied + * warranties of merchantability and fitness for a particular purpose are disclaimed. + * In no event shall copyright holders or contributors be liable for any direct, + * indirect, incidental, special, exemplary, or consequential damages + * (including, but not limited to, procurement of substitute goods or services; + * loss of use, data, or profits; or business interruption) however caused + * and on any theory of liability, whether in contract, strict liability, + * or tort (including negligence or otherwise) arising in any way out of + * the use of this software, even if advised of the possibility of such damage. + */ + +#include "common.hpp" + +namespace CAROTENE_NS { + +#ifdef CAROTENE_NEON + +#define CVTS_FUNC(T1, T2, SIMD_SIZE, CVTINIT, CVTROW) \ + void convertScale(const Size2D &_size, \ + const T1 * srcBase, ptrdiff_t srcStride, \ + T2 * dstBase, ptrdiff_t dstStride, \ + f64 alpha, f64 beta) \ + { \ + internal::assertSupportedConfiguration(); \ + Size2D size(_size); \ + if (srcStride == dstStride && \ + srcStride == (ptrdiff_t)(size.width)) \ + { \ + size.width *= size.height; \ + size.height = 1; \ + } \ + const ptrdiff_t sstep = srcStride / sizeof(T1); \ + const ptrdiff_t dstep = dstStride / sizeof(T2); \ + const size_t w = size.width & ~(SIMD_SIZE-1); \ + if (size.width >= SIMD_SIZE) \ + { \ + const T1* _src = srcBase; \ + T2* _dst = dstBase; \ + CVTINIT \ + for (ptrdiff_t h = size.height; h--; _src += sstep, _dst += dstep ) \ + CVTROW \ + } \ + if(w < size.width) \ + { \ + const T1* _src = srcBase; \ + T2* _dst = dstBase; \ + for (ptrdiff_t h = size.height; h--; _src += sstep, _dst += dstep ) \ + for(size_t i = w; i < size.width; i++ ) \ + _dst[i] = internal::saturate_cast(_src[i]*alpha + beta); \ + } \ + } + +#define CVTS_FUNC1(T1, SIMD_SIZE, CVTSINIT, CVTSROW) \ + void convertScale(const Size2D &_size, \ + const T1 * srcBase, ptrdiff_t srcStride, \ + T1 * dstBase, ptrdiff_t dstStride, \ + f64 alpha, f64 beta) \ + { \ + internal::assertSupportedConfiguration(); \ + Size2D size(_size); \ + if (srcStride == dstStride && \ + srcStride == (ptrdiff_t)(size.width)) \ + { \ + size.width *= size.height; \ + size.height = 1; \ + } \ + const ptrdiff_t sstep = srcStride / sizeof(T1); \ + const ptrdiff_t dstep = dstStride / sizeof(T1); \ + const size_t w = size.width & ~(SIMD_SIZE-1); \ + if (size.width >= SIMD_SIZE) \ + { \ + const T1* _src = srcBase; \ + T1* _dst = dstBase; \ + CVTSINIT \ + for (ptrdiff_t h = size.height; h--; _src += sstep, _dst += dstep ) \ + CVTSROW \ + } \ + if(w < size.width) \ + { \ + const T1* _src = srcBase; \ + T1* _dst = dstBase; \ + for (ptrdiff_t h = size.height; h--; _src += sstep, _dst += dstep ) \ + for(size_t i = w; i < size.width; i++ ) \ + _dst[i] = internal::saturate_cast(_src[i]*alpha + beta); \ + } \ + } + +#else + +#define CVTS_FUNC(T1, T2, SIMD_SIZE, CVTINIT, CVTROW) \ + void convertScale(const Size2D &, \ + const T1 *, ptrdiff_t, \ + T2 *, ptrdiff_t, \ + f64, f64) \ + { \ + internal::assertSupportedConfiguration(); \ + } + +#define CVTS_FUNC1(T1, SIMD_SIZE, CVTSINIT, CVTSROW) \ + void convertScale(const Size2D &, \ + const T1 *, ptrdiff_t, \ + T1 *, ptrdiff_t, \ + f64, f64) \ + { \ + internal::assertSupportedConfiguration(); \ + } + +#endif + +#if defined(__GNUC__) && defined(__arm__) +CVTS_FUNC1(u8, 16, + register float32x4_t vscale asm ("q0") = vdupq_n_f32((f32)alpha); + register float32x4_t vshift asm ("q1") = vdupq_n_f32((f32)beta + 0.5f);, +{ + for (size_t i = 0; i < w; i += 16) + { + internal::prefetch(_src + i); + __asm__ ( + "vld1.8 {d4-d5}, [%[src]] \n\t" + "vmovl.u8 q3, d4 \n\t" + "vmovl.u8 q4, d5 \n\t" + "vmovl.u16 q5, d6 \n\t" + "vmovl.u16 q6, d7 \n\t" + "vmovl.u16 q7, d8 \n\t" + "vmovl.u16 q8, d9 \n\t" + "vcvt.f32.u32 q9, q5 \n\t" + "vcvt.f32.u32 q10, q6 \n\t" + "vcvt.f32.u32 q11, q7 \n\t" + "vcvt.f32.u32 q12, q8 \n\t" + "vmul.f32 q13, q9, q0 \n\t" + "vmul.f32 q14, q10, q0 \n\t" + "vmul.f32 q15, q11, q0 \n\t" + "vmul.f32 q2, q12, q0 \n\t" + "vadd.f32 q3, q13, q1 \n\t" + "vadd.f32 q4, q14, q1 \n\t" + "vadd.f32 q5, q15, q1 \n\t" + "vadd.f32 q6, q2, q1 \n\t" + "vcvt.s32.f32 q7, q3 \n\t" + "vcvt.s32.f32 q8, q4 \n\t" + "vcvt.s32.f32 q9, q5 \n\t" + "vcvt.s32.f32 q10, q6 \n\t" + "vqmovun.s32 d22, q7 \n\t" + "vqmovun.s32 d23, q8 \n\t" + "vqmovun.s32 d24, q9 \n\t" + "vqmovun.s32 d25, q10 \n\t" + "vqmovn.u16 d26, q11 \n\t" + "vqmovn.u16 d27, q12 \n\t" + "vst1.8 {d26-d27}, [%[dst1]] \n\t" + : /*no output*/ + : [src] "r" (_src + i), + [dst1] "r" (_dst + i + 0), + "w" (vscale), "w" (vshift) + : "d4","d5","d6","d7","d8","d9","d10","d11","d12","d13","d14","d15","d16","d17","d18","d19","d20","d21","d22","d23","d24","d25","d26","d27","d28","d29","d30","d31" + ); + } +}) +#else +CVTS_FUNC1(u8, 16, + float32x4_t vscale = vdupq_n_f32((f32)alpha); + float32x4_t vshift = vdupq_n_f32((f32)beta + 0.5f);, +{ + for (size_t i = 0; i < w; i += 16) + { + internal::prefetch(_src + i); + uint8x16_t vline = vld1q_u8(_src + i); + uint16x8_t vline1_u16 = vmovl_u8(vget_low_u8 (vline)); + uint16x8_t vline2_u16 = vmovl_u8(vget_high_u8(vline)); + uint32x4_t vline1_u32 = vmovl_u16(vget_low_u16 (vline1_u16)); + uint32x4_t vline2_u32 = vmovl_u16(vget_high_u16(vline1_u16)); + uint32x4_t vline3_u32 = vmovl_u16(vget_low_u16 (vline2_u16)); + uint32x4_t vline4_u32 = vmovl_u16(vget_high_u16(vline2_u16)); + float32x4_t vline1_f32 = vcvtq_f32_u32(vline1_u32); + float32x4_t vline2_f32 = vcvtq_f32_u32(vline2_u32); + float32x4_t vline3_f32 = vcvtq_f32_u32(vline3_u32); + float32x4_t vline4_f32 = vcvtq_f32_u32(vline4_u32); + vline1_f32 = vmulq_f32(vline1_f32, vscale); + vline2_f32 = vmulq_f32(vline2_f32, vscale); + vline3_f32 = vmulq_f32(vline3_f32, vscale); + vline4_f32 = vmulq_f32(vline4_f32, vscale); + vline1_f32 = vaddq_f32(vline1_f32, vshift); + vline2_f32 = vaddq_f32(vline2_f32, vshift); + vline3_f32 = vaddq_f32(vline3_f32, vshift); + vline4_f32 = vaddq_f32(vline4_f32, vshift); + int32x4_t vline1_s32 = vcvtq_s32_f32(vline1_f32); + int32x4_t vline2_s32 = vcvtq_s32_f32(vline2_f32); + int32x4_t vline3_s32 = vcvtq_s32_f32(vline3_f32); + int32x4_t vline4_s32 = vcvtq_s32_f32(vline4_f32); + uint16x8_t vRes1_u16 = vcombine_u16(vqmovun_s32(vline1_s32), vqmovun_s32(vline2_s32)); + uint16x8_t vRes2_u16 = vcombine_u16(vqmovun_s32(vline3_s32), vqmovun_s32(vline4_s32)); + vst1q_u8(_dst + i, vcombine_u8(vqmovn_u16(vRes1_u16), vqmovn_u16(vRes2_u16))); + } +}) +#endif + +#if defined(__GNUC__) && defined(__arm__) +CVTS_FUNC(u8, s8, 16, + register float32x4_t vscale asm ("q0") = vdupq_n_f32((f32)alpha); + register float32x4_t vshift asm ("q1") = vdupq_n_f32((f32)beta + 0.5f);, +{ + for (size_t i = 0; i < w; i += 16) + { + internal::prefetch(_src + i); + __asm__ ( + "vld1.8 {d4-d5}, [%[src]] \n\t" + "vmovl.u8 q3, d4 \n\t" + "vmovl.u8 q4, d5 \n\t" + "vmovl.u16 q5, d6 \n\t" + "vmovl.u16 q6, d7 \n\t" + "vmovl.u16 q7, d8 \n\t" + "vmovl.u16 q8, d9 \n\t" + "vcvt.f32.u32 q9, q5 \n\t" + "vcvt.f32.u32 q10, q6 \n\t" + "vcvt.f32.u32 q11, q7 \n\t" + "vcvt.f32.u32 q12, q8 \n\t" + "vmul.f32 q13, q9, q0 \n\t" + "vmul.f32 q14, q10, q0 \n\t" + "vmul.f32 q15, q11, q0 \n\t" + "vmul.f32 q2, q12, q0 \n\t" + "vadd.f32 q3, q13, q1 \n\t" + "vadd.f32 q4, q14, q1 \n\t" + "vadd.f32 q5, q15, q1 \n\t" + "vadd.f32 q6, q2, q1 \n\t" + "vcvt.s32.f32 q7, q3 \n\t" + "vcvt.s32.f32 q8, q4 \n\t" + "vcvt.s32.f32 q9, q5 \n\t" + "vcvt.s32.f32 q10, q6 \n\t" + "vqmovn.s32 d22, q7 \n\t" + "vqmovn.s32 d23, q8 \n\t" + "vqmovn.s32 d24, q9 \n\t" + "vqmovn.s32 d25, q10 \n\t" + "vqmovn.s16 d26, q11 \n\t" + "vqmovn.s16 d27, q12 \n\t" + "vst1.8 {d26-d27}, [%[dst1]] \n\t" + : //no output + : [src] "r" (_src + i), + [dst1] "r" (_dst + i + 0), + "w" (vscale), "w" (vshift) + : "d4","d5","d6","d7","d8","d9","d10","d11","d12","d13","d14","d15","d16","d17","d18","d19","d20","d21","d22","d23","d24","d25","d26","d27","d28","d29","d30","d31" + ); + } +}) +#else +CVTS_FUNC(u8, s8, 16, + float32x4_t vscale = vdupq_n_f32((f32)alpha); + float32x4_t vshift = vdupq_n_f32((f32)beta + 0.5f);, +{ + for (size_t i = 0; i < w; i += 16) + { + internal::prefetch(_src + i); + uint8x16_t vline = vld1q_u8(_src + i); + uint16x8_t vline1_u16 = vmovl_u8(vget_low_u8 (vline)); + uint16x8_t vline2_u16 = vmovl_u8(vget_high_u8(vline)); + uint32x4_t vline1_u32 = vmovl_u16(vget_low_u16 (vline1_u16)); + uint32x4_t vline2_u32 = vmovl_u16(vget_high_u16(vline1_u16)); + uint32x4_t vline3_u32 = vmovl_u16(vget_low_u16 (vline2_u16)); + uint32x4_t vline4_u32 = vmovl_u16(vget_high_u16(vline2_u16)); + float32x4_t vline1_f32 = vcvtq_f32_u32(vline1_u32); + float32x4_t vline2_f32 = vcvtq_f32_u32(vline2_u32); + float32x4_t vline3_f32 = vcvtq_f32_u32(vline3_u32); + float32x4_t vline4_f32 = vcvtq_f32_u32(vline4_u32); + vline1_f32 = vmulq_f32(vline1_f32, vscale); + vline2_f32 = vmulq_f32(vline2_f32, vscale); + vline3_f32 = vmulq_f32(vline3_f32, vscale); + vline4_f32 = vmulq_f32(vline4_f32, vscale); + vline1_f32 = vaddq_f32(vline1_f32, vshift); + vline2_f32 = vaddq_f32(vline2_f32, vshift); + vline3_f32 = vaddq_f32(vline3_f32, vshift); + vline4_f32 = vaddq_f32(vline4_f32, vshift); + int32x4_t vline1_s32 = vcvtq_s32_f32(vline1_f32); + int32x4_t vline2_s32 = vcvtq_s32_f32(vline2_f32); + int32x4_t vline3_s32 = vcvtq_s32_f32(vline3_f32); + int32x4_t vline4_s32 = vcvtq_s32_f32(vline4_f32); + int16x8_t vRes1_u16 = vcombine_s16(vqmovn_s32(vline1_s32), vqmovn_s32(vline2_s32)); + int16x8_t vRes2_u16 = vcombine_s16(vqmovn_s32(vline3_s32), vqmovn_s32(vline4_s32)); + vst1q_s8(_dst + i, vcombine_s8(vqmovn_s16(vRes1_u16), vqmovn_s16(vRes2_u16))); + } +}) +#endif + +#if defined(__GNUC__) && defined(__arm__) +CVTS_FUNC(u8, u16, 16, + register float32x4_t vscale asm ("q0") = vdupq_n_f32((f32)alpha); + register float32x4_t vshift asm ("q1") = vdupq_n_f32((f32)beta + 0.5f);, +{ + for (size_t i = 0; i < w; i += 16) + { + internal::prefetch(_src + i); + __asm__ ( + "vld1.8 {d4-d5}, [%[src]] \n\t" + "vmovl.u8 q3, d4 \n\t" + "vmovl.u8 q4, d5 \n\t" + "vmovl.u16 q5, d6 \n\t" + "vmovl.u16 q6, d7 \n\t" + "vmovl.u16 q7, d8 \n\t" + "vmovl.u16 q8, d9 \n\t" + "vcvt.f32.u32 q9, q5 \n\t" + "vcvt.f32.u32 q10, q6 \n\t" + "vcvt.f32.u32 q11, q7 \n\t" + "vcvt.f32.u32 q12, q8 \n\t" + "vmul.f32 q13, q9, q0 \n\t" + "vmul.f32 q14, q10, q0 \n\t" + "vmul.f32 q15, q11, q0 \n\t" + "vmul.f32 q2, q12, q0 \n\t" + "vadd.f32 q3, q13, q1 \n\t" + "vadd.f32 q4, q14, q1 \n\t" + "vadd.f32 q5, q15, q1 \n\t" + "vadd.f32 q6, q2, q1 \n\t" + "vcvt.s32.f32 q7, q3 \n\t" + "vcvt.s32.f32 q8, q4 \n\t" + "vcvt.s32.f32 q9, q5 \n\t" + "vcvt.s32.f32 q10, q6 \n\t" + "vqmovun.s32 d22, q7 \n\t" + "vqmovun.s32 d23, q8 \n\t" + "vqmovun.s32 d24, q9 \n\t" + "vqmovun.s32 d25, q10 \n\t" + "vst1.16 {d22-d23}, [%[dst1]] \n\t" + "vst1.16 {d24-d25}, [%[dst2]] \n\t" + : /*no output*/ + : [src] "r" (_src + i), + [dst1] "r" (_dst + i + 0), + [dst2] "r" (_dst + i + 8), + "w" (vscale), "w" (vshift) + : "d4","d5","d6","d7","d8","d9","d10","d11","d12","d13","d14","d15","d16","d17","d18","d19","d20","d21","d22","d23","d24","d25","d26","d27","d28","d29","d30","d31" + ); + } +}) +#else +CVTS_FUNC(u8, u16, 16, + float32x4_t vscale = vdupq_n_f32((f32)alpha); + float32x4_t vshift = vdupq_n_f32((f32)beta + 0.5f);, +{ + for (size_t i = 0; i < w; i += 16) + { + internal::prefetch(_src + i); + uint8x16_t vline = vld1q_u8(_src + i); + uint16x8_t vline1_u16 = vmovl_u8(vget_low_u8 (vline)); + uint16x8_t vline2_u16 = vmovl_u8(vget_high_u8(vline)); + uint32x4_t vline1_u32 = vmovl_u16(vget_low_u16 (vline1_u16)); + uint32x4_t vline2_u32 = vmovl_u16(vget_high_u16(vline1_u16)); + uint32x4_t vline3_u32 = vmovl_u16(vget_low_u16 (vline2_u16)); + uint32x4_t vline4_u32 = vmovl_u16(vget_high_u16(vline2_u16)); + float32x4_t vline1_f32 = vcvtq_f32_u32(vline1_u32); + float32x4_t vline2_f32 = vcvtq_f32_u32(vline2_u32); + float32x4_t vline3_f32 = vcvtq_f32_u32(vline3_u32); + float32x4_t vline4_f32 = vcvtq_f32_u32(vline4_u32); + vline1_f32 = vmulq_f32(vline1_f32, vscale); + vline2_f32 = vmulq_f32(vline2_f32, vscale); + vline3_f32 = vmulq_f32(vline3_f32, vscale); + vline4_f32 = vmulq_f32(vline4_f32, vscale); + vline1_f32 = vaddq_f32(vline1_f32, vshift); + vline2_f32 = vaddq_f32(vline2_f32, vshift); + vline3_f32 = vaddq_f32(vline3_f32, vshift); + vline4_f32 = vaddq_f32(vline4_f32, vshift); + int32x4_t vline1_s32 = vcvtq_s32_f32(vline1_f32); + int32x4_t vline2_s32 = vcvtq_s32_f32(vline2_f32); + int32x4_t vline3_s32 = vcvtq_s32_f32(vline3_f32); + int32x4_t vline4_s32 = vcvtq_s32_f32(vline4_f32); + vst1q_u16(_dst + i + 0, vcombine_u16(vqmovun_s32(vline1_s32), vqmovun_s32(vline2_s32))); + vst1q_u16(_dst + i + 8, vcombine_u16(vqmovun_s32(vline3_s32), vqmovun_s32(vline4_s32))); + } +}) +#endif + +#if defined(__GNUC__) && defined(__arm__) +CVTS_FUNC(u8, s16, 16, + register float32x4_t vscale asm ("q0") = vdupq_n_f32((f32)alpha); + register float32x4_t vshift asm ("q1") = vdupq_n_f32((f32)beta + 0.5f);, +{ + for (size_t i = 0; i < w; i += 16) + { + internal::prefetch(_src + i); + __asm__ ( + "vld1.8 {d4-d5}, [%[src]] \n\t" + "vmovl.u8 q3, d4 \n\t" + "vmovl.u8 q4, d5 \n\t" + "vmovl.u16 q5, d6 \n\t" + "vmovl.u16 q6, d7 \n\t" + "vmovl.u16 q7, d8 \n\t" + "vmovl.u16 q8, d9 \n\t" + "vcvt.f32.u32 q9, q5 \n\t" + "vcvt.f32.u32 q10, q6 \n\t" + "vcvt.f32.u32 q11, q7 \n\t" + "vcvt.f32.u32 q12, q8 \n\t" + "vmul.f32 q13, q9, q0 \n\t" + "vmul.f32 q14, q10, q0 \n\t" + "vmul.f32 q15, q11, q0 \n\t" + "vmul.f32 q2, q12, q0 \n\t" + "vadd.f32 q3, q13, q1 \n\t" + "vadd.f32 q4, q14, q1 \n\t" + "vadd.f32 q5, q15, q1 \n\t" + "vadd.f32 q6, q2, q1 \n\t" + "vcvt.s32.f32 q7, q3 \n\t" + "vcvt.s32.f32 q8, q4 \n\t" + "vcvt.s32.f32 q9, q5 \n\t" + "vcvt.s32.f32 q10, q6 \n\t" + "vqmovn.s32 d22, q7 \n\t" + "vqmovn.s32 d23, q8 \n\t" + "vqmovn.s32 d24, q9 \n\t" + "vqmovn.s32 d25, q10 \n\t" + "vst1.16 {d22-d23}, [%[dst1]] \n\t" + "vst1.16 {d24-d25}, [%[dst2]] \n\t" + : //no output + : [src] "r" (_src + i), + [dst1] "r" (_dst + i + 0), + [dst2] "r" (_dst + i + 8), + "w" (vscale), "w" (vshift) + : "d4","d5","d6","d7","d8","d9","d10","d11","d12","d13","d14","d15","d16","d17","d18","d19","d20","d21","d22","d23","d24","d25","d26","d27","d28","d29","d30","d31" + ); + } +}) +#else +CVTS_FUNC(u8, s16, 16, + float32x4_t vscale = vdupq_n_f32((f32)alpha); + float32x4_t vshift = vdupq_n_f32((f32)beta + 0.5f);, +{ + for (size_t i = 0; i < w; i += 16) + { + internal::prefetch(_src + i); + uint8x16_t vline = vld1q_u8(_src + i); + uint16x8_t vline1_u16 = vmovl_u8(vget_low_u8 (vline)); + uint16x8_t vline2_u16 = vmovl_u8(vget_high_u8(vline)); + uint32x4_t vline1_u32 = vmovl_u16(vget_low_u16 (vline1_u16)); + uint32x4_t vline2_u32 = vmovl_u16(vget_high_u16(vline1_u16)); + uint32x4_t vline3_u32 = vmovl_u16(vget_low_u16 (vline2_u16)); + uint32x4_t vline4_u32 = vmovl_u16(vget_high_u16(vline2_u16)); + float32x4_t vline1_f32 = vcvtq_f32_u32(vline1_u32); + float32x4_t vline2_f32 = vcvtq_f32_u32(vline2_u32); + float32x4_t vline3_f32 = vcvtq_f32_u32(vline3_u32); + float32x4_t vline4_f32 = vcvtq_f32_u32(vline4_u32); + vline1_f32 = vmulq_f32(vline1_f32, vscale); + vline2_f32 = vmulq_f32(vline2_f32, vscale); + vline3_f32 = vmulq_f32(vline3_f32, vscale); + vline4_f32 = vmulq_f32(vline4_f32, vscale); + vline1_f32 = vaddq_f32(vline1_f32, vshift); + vline2_f32 = vaddq_f32(vline2_f32, vshift); + vline3_f32 = vaddq_f32(vline3_f32, vshift); + vline4_f32 = vaddq_f32(vline4_f32, vshift); + int32x4_t vline1_s32 = vcvtq_s32_f32(vline1_f32); + int32x4_t vline2_s32 = vcvtq_s32_f32(vline2_f32); + int32x4_t vline3_s32 = vcvtq_s32_f32(vline3_f32); + int32x4_t vline4_s32 = vcvtq_s32_f32(vline4_f32); + vst1q_s16(_dst + i + 0, vcombine_s16(vqmovn_s32(vline1_s32), vqmovn_s32(vline2_s32))); + vst1q_s16(_dst + i + 8, vcombine_s16(vqmovn_s32(vline3_s32), vqmovn_s32(vline4_s32))); + } +}) +#endif + +#if __GNUC_MINOR__ < 7 +CVTS_FUNC(u8, s32, 16, + register float32x4_t vscale asm ("q0") = vdupq_n_f32((f32)alpha); + register float32x4_t vshift asm ("q1") = vdupq_n_f32((f32)beta + 0.5f);, +{ + for (size_t i = 0; i < w; i += 16) + { + internal::prefetch(_src + i); + __asm__ ( + "vld1.8 {d4-d5}, [%[src]] \n\t" + "vmovl.u8 q3, d4 \n\t" + "vmovl.u8 q4, d5 \n\t" + "vmovl.u16 q5, d6 \n\t" + "vmovl.u16 q6, d7 \n\t" + "vmovl.u16 q7, d8 \n\t" + "vmovl.u16 q8, d9 \n\t" + "vcvt.f32.u32 q9, q5 \n\t" + "vcvt.f32.u32 q10, q6 \n\t" + "vcvt.f32.u32 q11, q7 \n\t" + "vcvt.f32.u32 q12, q8 \n\t" + "vmul.f32 q13, q9, q0 \n\t" + "vmul.f32 q14, q10, q0 \n\t" + "vmul.f32 q15, q11, q0 \n\t" + "vmul.f32 q2, q12, q0 \n\t" + "vadd.f32 q3, q13, q1 \n\t" + "vadd.f32 q4, q14, q1 \n\t" + "vadd.f32 q5, q15, q1 \n\t" + "vadd.f32 q6, q2, q1 \n\t" + "vcvt.s32.f32 q7, q3 \n\t" + "vcvt.s32.f32 q8, q4 \n\t" + "vcvt.s32.f32 q9, q5 \n\t" + "vcvt.s32.f32 q10, q6 \n\t" + "vst1.32 {d14-d15}, [%[dst1]] \n\t" + "vst1.32 {d16-d17}, [%[dst2]] \n\t" + "vst1.32 {d18-d19}, [%[dst3]] \n\t" + "vst1.32 {d20-d21}, [%[dst4]] \n\t" + : /*no output*/ + : [src] "r" (_src + i), + [dst1] "r" (_dst + i + 0), + [dst2] "r" (_dst + i + 4), + [dst3] "r" (_dst + i + 8), + [dst4] "r" (_dst + i + 12), + "w" (vscale), "w" (vshift) + : "d4","d5","d6","d7","d8","d9","d10", + "d11","d12","d13","d14","d15","d16","d17", + "d18","d19","d20","d21","d22","d23","d24", + "d25","d26","d27","d28","d29","d30","d31" + ); + } +}) +#else +CVTS_FUNC(u8, s32, 16, + float32x4_t vscale = vdupq_n_f32((f32)alpha); + float32x4_t vshift = vdupq_n_f32((f32)beta + 0.5f);, +{ + for (size_t i = 0; i < w; i += 16) + { + internal::prefetch(_src + i); + uint8x16_t vline = vld1q_u8(_src + i); + uint16x8_t vline1_u16 = vmovl_u8(vget_low_u8 (vline)); + uint16x8_t vline2_u16 = vmovl_u8(vget_high_u8(vline)); + uint32x4_t vline1_u32 = vmovl_u16(vget_low_u16 (vline1_u16)); + uint32x4_t vline2_u32 = vmovl_u16(vget_high_u16(vline1_u16)); + uint32x4_t vline3_u32 = vmovl_u16(vget_low_u16 (vline2_u16)); + uint32x4_t vline4_u32 = vmovl_u16(vget_high_u16(vline2_u16)); + float32x4_t vline1_f32 = vcvtq_f32_u32(vline1_u32); + float32x4_t vline2_f32 = vcvtq_f32_u32(vline2_u32); + float32x4_t vline3_f32 = vcvtq_f32_u32(vline3_u32); + float32x4_t vline4_f32 = vcvtq_f32_u32(vline4_u32); + vline1_f32 = vmulq_f32(vline1_f32, vscale); + vline2_f32 = vmulq_f32(vline2_f32, vscale); + vline3_f32 = vmulq_f32(vline3_f32, vscale); + vline4_f32 = vmulq_f32(vline4_f32, vscale); + vline1_f32 = vaddq_f32(vline1_f32, vshift); + vline2_f32 = vaddq_f32(vline2_f32, vshift); + vline3_f32 = vaddq_f32(vline3_f32, vshift); + vline4_f32 = vaddq_f32(vline4_f32, vshift); + int32x4_t vline1_s32 = vcvtq_s32_f32(vline1_f32); + int32x4_t vline2_s32 = vcvtq_s32_f32(vline2_f32); + int32x4_t vline3_s32 = vcvtq_s32_f32(vline3_f32); + int32x4_t vline4_s32 = vcvtq_s32_f32(vline4_f32); + vst1q_s32(_dst + i + 0, vline1_s32); + vst1q_s32(_dst + i + 4, vline2_s32); + vst1q_s32(_dst + i + 8, vline3_s32); + vst1q_s32(_dst + i + 12, vline4_s32); + } +}) +#endif + +#if __GNUC_MINOR__ < 7 +CVTS_FUNC(u8, f32, 16, + register float32x4_t vscale asm ("q0") = vdupq_n_f32((f32)alpha); + register float32x4_t vshift asm ("q1") = vdupq_n_f32((f32)beta);, +{ + for (size_t i = 0; i < w; i += 16) + { + internal::prefetch(_src + i); + __asm__ ( + "vld1.8 {d4-d5}, [%[src]] \n\t" + "vmovl.u8 q3, d4 \n\t" + "vmovl.u8 q4, d5 \n\t" + "vmovl.u16 q5, d6 \n\t" + "vmovl.u16 q6, d7 \n\t" + "vmovl.u16 q7, d8 \n\t" + "vmovl.u16 q8, d9 \n\t" + "vcvt.f32.u32 q9, q5 \n\t" + "vcvt.f32.u32 q10, q6 \n\t" + "vcvt.f32.u32 q11, q7 \n\t" + "vcvt.f32.u32 q12, q8 \n\t" + "vmul.f32 q13, q9, q0 \n\t" + "vmul.f32 q14, q10, q0 \n\t" + "vmul.f32 q15, q11, q0 \n\t" + "vmul.f32 q2, q12, q0 \n\t" + "vadd.f32 q3, q13, q1 \n\t" + "vadd.f32 q4, q14, q1 \n\t" + "vadd.f32 q5, q15, q1 \n\t" + "vadd.f32 q6, q2, q1 \n\t" + "vst1.32 {d6-d7}, [%[dst1]] \n\t" + "vst1.32 {d8-d9}, [%[dst2]] \n\t" + "vst1.32 {d10-d11}, [%[dst3]] \n\t" + "vst1.32 {d12-d13}, [%[dst4]] \n\t" + : /*no output*/ + : [src] "r" (_src + i), + [dst1] "r" (_dst + i + 0), + [dst2] "r" (_dst + i + 4), + [dst3] "r" (_dst + i + 8), + [dst4] "r" (_dst + i + 12), + "w" (vscale), "w" (vshift) + : "d4","d5","d6","d7","d8","d9","d10", + "d11","d12","d13","d14","d15","d16","d17", + "d18","d19","d20","d21","d22","d23","d24", + "d25","d26","d27","d28","d29","d30","d31" + ); + } +}) +#else +CVTS_FUNC(u8, f32, 16, + float32x4_t vscale = vdupq_n_f32((f32)alpha); + float32x4_t vshift = vdupq_n_f32((f32)beta);, +{ + for (size_t i = 0; i < w; i += 16) + { + internal::prefetch(_src + i); + uint8x16_t vline = vld1q_u8(_src + i); + uint16x8_t vline1_u16 = vmovl_u8(vget_low_u8 (vline)); + uint16x8_t vline2_u16 = vmovl_u8(vget_high_u8(vline)); + uint32x4_t vline1_u32 = vmovl_u16(vget_low_u16 (vline1_u16)); + uint32x4_t vline2_u32 = vmovl_u16(vget_high_u16(vline1_u16)); + uint32x4_t vline3_u32 = vmovl_u16(vget_low_u16 (vline2_u16)); + uint32x4_t vline4_u32 = vmovl_u16(vget_high_u16(vline2_u16)); + float32x4_t vline1_f32 = vcvtq_f32_u32(vline1_u32); + float32x4_t vline2_f32 = vcvtq_f32_u32(vline2_u32); + float32x4_t vline3_f32 = vcvtq_f32_u32(vline3_u32); + float32x4_t vline4_f32 = vcvtq_f32_u32(vline4_u32); + vline1_f32 = vmulq_f32(vline1_f32, vscale); + vline2_f32 = vmulq_f32(vline2_f32, vscale); + vline3_f32 = vmulq_f32(vline3_f32, vscale); + vline4_f32 = vmulq_f32(vline4_f32, vscale); + vline1_f32 = vaddq_f32(vline1_f32, vshift); + vline2_f32 = vaddq_f32(vline2_f32, vshift); + vline3_f32 = vaddq_f32(vline3_f32, vshift); + vline4_f32 = vaddq_f32(vline4_f32, vshift); + vst1q_f32(_dst + i + 0, vline1_f32); + vst1q_f32(_dst + i + 4, vline2_f32); + vst1q_f32(_dst + i + 8, vline3_f32); + vst1q_f32(_dst + i + 12, vline4_f32); + } +}) +#endif + +#if defined(__GNUC__) && defined(__arm__) +CVTS_FUNC(s8, u8, 16, + register float32x4_t vscale asm ("q0") = vdupq_n_f32((f32)alpha); + register float32x4_t vshift asm ("q1") = vdupq_n_f32((f32)beta + 0.5f);, +{ + for (size_t i = 0; i < w; i += 16) + { + internal::prefetch(_src + i); + __asm__ ( + "vld1.8 {d4-d5}, [%[src]] \n\t" + "vmovl.s8 q3, d4 \n\t" + "vmovl.s8 q4, d5 \n\t" + "vmovl.s16 q5, d6 \n\t" + "vmovl.s16 q6, d7 \n\t" + "vmovl.s16 q7, d8 \n\t" + "vmovl.s16 q8, d9 \n\t" + "vcvt.f32.s32 q9, q5 \n\t" + "vcvt.f32.s32 q10, q6 \n\t" + "vcvt.f32.s32 q11, q7 \n\t" + "vcvt.f32.s32 q12, q8 \n\t" + "vmul.f32 q13, q9, q0 \n\t" + "vmul.f32 q14, q10, q0 \n\t" + "vmul.f32 q15, q11, q0 \n\t" + "vmul.f32 q2, q12, q0 \n\t" + "vadd.f32 q3, q13, q1 \n\t" + "vadd.f32 q4, q14, q1 \n\t" + "vadd.f32 q5, q15, q1 \n\t" + "vadd.f32 q6, q2, q1 \n\t" + "vcvt.s32.f32 q7, q3 \n\t" + "vcvt.s32.f32 q8, q4 \n\t" + "vcvt.s32.f32 q9, q5 \n\t" + "vcvt.s32.f32 q10, q6 \n\t" + "vqmovun.s32 d22, q7 \n\t" + "vqmovun.s32 d23, q8 \n\t" + "vqmovun.s32 d24, q9 \n\t" + "vqmovun.s32 d25, q10 \n\t" + "vqmovn.u16 d26, q11 \n\t" + "vqmovn.u16 d27, q12 \n\t" + "vst1.8 {d26-d27}, [%[dst1]] \n\t" + : /*no output*/ + : [src] "r" (_src + i), + [dst1] "r" (_dst + i + 0), + "w" (vscale), "w" (vshift) + : "d4","d5","d6","d7","d8","d9","d10","d11","d12","d13","d14","d15","d16","d17","d18","d19","d20","d21","d22","d23","d24","d25","d26","d27","d28","d29","d30","d31" + ); + } +}) +#else +CVTS_FUNC(s8, u8, 16, + float32x4_t vscale = vdupq_n_f32((f32)alpha); + float32x4_t vshift = vdupq_n_f32((f32)beta + 0.5f);, +{ + for (size_t i = 0; i < w; i += 16) + { + internal::prefetch(_src + i); + int8x16_t vline = vld1q_s8(_src + i); + int16x8_t vline1_s16 = vmovl_s8(vget_low_s8 (vline)); + int16x8_t vline2_s16 = vmovl_s8(vget_high_s8(vline)); + int32x4_t vline1_s32 = vmovl_s16(vget_low_s16 (vline1_s16)); + int32x4_t vline2_s32 = vmovl_s16(vget_high_s16(vline1_s16)); + int32x4_t vline3_s32 = vmovl_s16(vget_low_s16 (vline2_s16)); + int32x4_t vline4_s32 = vmovl_s16(vget_high_s16(vline2_s16)); + float32x4_t vline1_f32 = vcvtq_f32_s32(vline1_s32); + float32x4_t vline2_f32 = vcvtq_f32_s32(vline2_s32); + float32x4_t vline3_f32 = vcvtq_f32_s32(vline3_s32); + float32x4_t vline4_f32 = vcvtq_f32_s32(vline4_s32); + vline1_f32 = vmulq_f32(vline1_f32, vscale); + vline2_f32 = vmulq_f32(vline2_f32, vscale); + vline3_f32 = vmulq_f32(vline3_f32, vscale); + vline4_f32 = vmulq_f32(vline4_f32, vscale); + vline1_f32 = vaddq_f32(vline1_f32, vshift); + vline2_f32 = vaddq_f32(vline2_f32, vshift); + vline3_f32 = vaddq_f32(vline3_f32, vshift); + vline4_f32 = vaddq_f32(vline4_f32, vshift); + vline1_s32 = vcvtq_s32_f32(vline1_f32); + vline2_s32 = vcvtq_s32_f32(vline2_f32); + vline3_s32 = vcvtq_s32_f32(vline3_f32); + vline4_s32 = vcvtq_s32_f32(vline4_f32); + uint16x8_t vRes1_u16 = vcombine_u16(vqmovun_s32(vline1_s32), vqmovun_s32(vline2_s32)); + uint16x8_t vRes2_u16 = vcombine_u16(vqmovun_s32(vline3_s32), vqmovun_s32(vline4_s32)); + vst1q_u8(_dst + i, vcombine_u8(vqmovn_u16(vRes1_u16), vqmovn_u16(vRes2_u16))); + } +}) +#endif + +#if defined(__GNUC__) && defined(__arm__) +CVTS_FUNC1(s8, 16, + register float32x4_t vscale asm ("q0") = vdupq_n_f32((f32)alpha); + register float32x4_t vshift asm ("q1") = vdupq_n_f32((f32)beta + 0.5f);, +{ + for (size_t i = 0; i < w; i += 16) + { + internal::prefetch(_src + i); + __asm__ ( + "vld1.8 {d4-d5}, [%[src]] \n\t" + "vmovl.s8 q3, d4 \n\t" + "vmovl.s8 q4, d5 \n\t" + "vmovl.s16 q5, d6 \n\t" + "vmovl.s16 q6, d7 \n\t" + "vmovl.s16 q7, d8 \n\t" + "vmovl.s16 q8, d9 \n\t" + "vcvt.f32.s32 q9, q5 \n\t" + "vcvt.f32.s32 q10, q6 \n\t" + "vcvt.f32.s32 q11, q7 \n\t" + "vcvt.f32.s32 q12, q8 \n\t" + "vmul.f32 q13, q9, q0 \n\t" + "vmul.f32 q14, q10, q0 \n\t" + "vmul.f32 q15, q11, q0 \n\t" + "vmul.f32 q2, q12, q0 \n\t" + "vadd.f32 q3, q13, q1 \n\t" + "vadd.f32 q4, q14, q1 \n\t" + "vadd.f32 q5, q15, q1 \n\t" + "vadd.f32 q6, q2, q1 \n\t" + "vcvt.s32.f32 q7, q3 \n\t" + "vcvt.s32.f32 q8, q4 \n\t" + "vcvt.s32.f32 q9, q5 \n\t" + "vcvt.s32.f32 q10, q6 \n\t" + "vqmovn.s32 d22, q7 \n\t" + "vqmovn.s32 d23, q8 \n\t" + "vqmovn.s32 d24, q9 \n\t" + "vqmovn.s32 d25, q10 \n\t" + "vqmovn.s16 d26, q11 \n\t" + "vqmovn.s16 d27, q12 \n\t" + "vst1.8 {d26-d27}, [%[dst1]] \n\t" + : /*no output*/ + : [src] "r" (_src + i), + [dst1] "r" (_dst + i + 0), + "w" (vscale), "w" (vshift) + : "d4","d5","d6","d7","d8","d9","d10","d11","d12","d13","d14","d15","d16","d17","d18","d19","d20","d21","d22","d23","d24","d25","d26","d27","d28","d29","d30","d31" + ); + } +}) +#else +CVTS_FUNC1(s8, 16, + float32x4_t vscale = vdupq_n_f32((f32)alpha); + float32x4_t vshift = vdupq_n_f32((f32)beta + 0.5f);, +{ + for (size_t i = 0; i < w; i += 16) + { + internal::prefetch(_src + i); + int8x16_t vline = vld1q_s8(_src + i); + int16x8_t vline1_s16 = vmovl_s8(vget_low_s8 (vline)); + int16x8_t vline2_s16 = vmovl_s8(vget_high_s8(vline)); + int32x4_t vline1_s32 = vmovl_s16(vget_low_s16 (vline1_s16)); + int32x4_t vline2_s32 = vmovl_s16(vget_high_s16(vline1_s16)); + int32x4_t vline3_s32 = vmovl_s16(vget_low_s16 (vline2_s16)); + int32x4_t vline4_s32 = vmovl_s16(vget_high_s16(vline2_s16)); + float32x4_t vline1_f32 = vcvtq_f32_s32(vline1_s32); + float32x4_t vline2_f32 = vcvtq_f32_s32(vline2_s32); + float32x4_t vline3_f32 = vcvtq_f32_s32(vline3_s32); + float32x4_t vline4_f32 = vcvtq_f32_s32(vline4_s32); + vline1_f32 = vmulq_f32(vline1_f32, vscale); + vline2_f32 = vmulq_f32(vline2_f32, vscale); + vline3_f32 = vmulq_f32(vline3_f32, vscale); + vline4_f32 = vmulq_f32(vline4_f32, vscale); + vline1_f32 = vaddq_f32(vline1_f32, vshift); + vline2_f32 = vaddq_f32(vline2_f32, vshift); + vline3_f32 = vaddq_f32(vline3_f32, vshift); + vline4_f32 = vaddq_f32(vline4_f32, vshift); + vline1_s32 = vcvtq_s32_f32(vline1_f32); + vline2_s32 = vcvtq_s32_f32(vline2_f32); + vline3_s32 = vcvtq_s32_f32(vline3_f32); + vline4_s32 = vcvtq_s32_f32(vline4_f32); + int16x8_t vRes1_s16 = vcombine_s16(vqmovn_s32(vline1_s32), vqmovn_s32(vline2_s32)); + int16x8_t vRes2_s16 = vcombine_s16(vqmovn_s32(vline3_s32), vqmovn_s32(vline4_s32)); + vst1q_s8(_dst + i, vcombine_s8(vqmovn_s16(vRes1_s16), vqmovn_s16(vRes2_s16))); + } +}) +#endif + +#if defined(__GNUC__) && defined(__arm__) +CVTS_FUNC(s8, u16, 16, + register float32x4_t vscale asm ("q0") = vdupq_n_f32((f32)alpha); + register float32x4_t vshift asm ("q1") = vdupq_n_f32((f32)beta + 0.5f);, +{ + for (size_t i = 0; i < w; i += 16) + { + internal::prefetch(_src + i); + __asm__ ( + "vld1.8 {d4-d5}, [%[src]] \n\t" + "vmovl.s8 q3, d4 \n\t" + "vmovl.s8 q4, d5 \n\t" + "vmovl.s16 q5, d6 \n\t" + "vmovl.s16 q6, d7 \n\t" + "vmovl.s16 q7, d8 \n\t" + "vmovl.s16 q8, d9 \n\t" + "vcvt.f32.s32 q9, q5 \n\t" + "vcvt.f32.s32 q10, q6 \n\t" + "vcvt.f32.s32 q11, q7 \n\t" + "vcvt.f32.s32 q12, q8 \n\t" + "vmul.f32 q13, q9, q0 \n\t" + "vmul.f32 q14, q10, q0 \n\t" + "vmul.f32 q15, q11, q0 \n\t" + "vmul.f32 q2, q12, q0 \n\t" + "vadd.f32 q3, q13, q1 \n\t" + "vadd.f32 q4, q14, q1 \n\t" + "vadd.f32 q5, q15, q1 \n\t" + "vadd.f32 q6, q2, q1 \n\t" + "vcvt.s32.f32 q7, q3 \n\t" + "vcvt.s32.f32 q8, q4 \n\t" + "vcvt.s32.f32 q9, q5 \n\t" + "vcvt.s32.f32 q10, q6 \n\t" + "vqmovun.s32 d22, q7 \n\t" + "vqmovun.s32 d23, q8 \n\t" + "vqmovun.s32 d24, q9 \n\t" + "vqmovun.s32 d25, q10 \n\t" + "vst1.16 {d22-d23}, [%[dst1]] \n\t" + "vst1.16 {d24-d25}, [%[dst2]] \n\t" + : /*no output*/ + : [src] "r" (_src + i), + [dst1] "r" (_dst + i + 0), + [dst2] "r" (_dst + i + 8), + "w" (vscale), "w" (vshift) + : "d4","d5","d6","d7","d8","d9","d10","d11","d12","d13","d14","d15","d16","d17","d18","d19","d20","d21","d22","d23","d24","d25","d26","d27","d28","d29","d30","d31" + ); + } +}) +#else +CVTS_FUNC(s8, u16, 16, + float32x4_t vscale = vdupq_n_f32((f32)alpha); + float32x4_t vshift = vdupq_n_f32((f32)beta + 0.5f);, +{ + for (size_t i = 0; i < w; i += 16) + { + internal::prefetch(_src + i); + int8x16_t vline = vld1q_s8(_src + i); + int16x8_t vline1_s16 = vmovl_s8(vget_low_s8 (vline)); + int16x8_t vline2_s16 = vmovl_s8(vget_high_s8(vline)); + int32x4_t vline1_s32 = vmovl_s16(vget_low_s16 (vline1_s16)); + int32x4_t vline2_s32 = vmovl_s16(vget_high_s16(vline1_s16)); + int32x4_t vline3_s32 = vmovl_s16(vget_low_s16 (vline2_s16)); + int32x4_t vline4_s32 = vmovl_s16(vget_high_s16(vline2_s16)); + float32x4_t vline1_f32 = vcvtq_f32_s32(vline1_s32); + float32x4_t vline2_f32 = vcvtq_f32_s32(vline2_s32); + float32x4_t vline3_f32 = vcvtq_f32_s32(vline3_s32); + float32x4_t vline4_f32 = vcvtq_f32_s32(vline4_s32); + vline1_f32 = vmulq_f32(vline1_f32, vscale); + vline2_f32 = vmulq_f32(vline2_f32, vscale); + vline3_f32 = vmulq_f32(vline3_f32, vscale); + vline4_f32 = vmulq_f32(vline4_f32, vscale); + vline1_f32 = vaddq_f32(vline1_f32, vshift); + vline2_f32 = vaddq_f32(vline2_f32, vshift); + vline3_f32 = vaddq_f32(vline3_f32, vshift); + vline4_f32 = vaddq_f32(vline4_f32, vshift); + vline1_s32 = vcvtq_s32_f32(vline1_f32); + vline2_s32 = vcvtq_s32_f32(vline2_f32); + vline3_s32 = vcvtq_s32_f32(vline3_f32); + vline4_s32 = vcvtq_s32_f32(vline4_f32); + uint16x8_t vRes1_u16 = vcombine_u16(vqmovun_s32(vline1_s32), vqmovun_s32(vline2_s32)); + uint16x8_t vRes2_u16 = vcombine_u16(vqmovun_s32(vline3_s32), vqmovun_s32(vline4_s32)); + vst1q_u16(_dst + i + 0, vRes1_u16); + vst1q_u16(_dst + i + 8, vRes2_u16); + } +}) +#endif + +#if defined(__GNUC__) && defined(__arm__) +CVTS_FUNC(s8, s16, 16, + register float32x4_t vscale asm ("q0") = vdupq_n_f32((f32)alpha); + register float32x4_t vshift asm ("q1") = vdupq_n_f32((f32)beta + 0.5f);, +{ + for (size_t i = 0; i < w; i += 16) + { + internal::prefetch(_src + i); + __asm__ ( + "vld1.8 {d4-d5}, [%[src]] \n\t" + "vmovl.s8 q3, d4 \n\t" + "vmovl.s8 q4, d5 \n\t" + "vmovl.s16 q5, d6 \n\t" + "vmovl.s16 q6, d7 \n\t" + "vmovl.s16 q7, d8 \n\t" + "vmovl.s16 q8, d9 \n\t" + "vcvt.f32.s32 q9, q5 \n\t" + "vcvt.f32.s32 q10, q6 \n\t" + "vcvt.f32.s32 q11, q7 \n\t" + "vcvt.f32.s32 q12, q8 \n\t" + "vmul.f32 q13, q9, q0 \n\t" + "vmul.f32 q14, q10, q0 \n\t" + "vmul.f32 q15, q11, q0 \n\t" + "vmul.f32 q2, q12, q0 \n\t" + "vadd.f32 q3, q13, q1 \n\t" + "vadd.f32 q4, q14, q1 \n\t" + "vadd.f32 q5, q15, q1 \n\t" + "vadd.f32 q6, q2, q1 \n\t" + "vcvt.s32.f32 q7, q3 \n\t" + "vcvt.s32.f32 q8, q4 \n\t" + "vcvt.s32.f32 q9, q5 \n\t" + "vcvt.s32.f32 q10, q6 \n\t" + "vqmovn.s32 d22, q7 \n\t" + "vqmovn.s32 d23, q8 \n\t" + "vqmovn.s32 d24, q9 \n\t" + "vqmovn.s32 d25, q10 \n\t" + "vst1.16 {d22-d23}, [%[dst1]] \n\t" + "vst1.16 {d24-d25}, [%[dst2]] \n\t" + : /*no output*/ + : [src] "r" (_src + i), + [dst1] "r" (_dst + i + 0), + [dst2] "r" (_dst + i + 8), + "w" (vscale), "w" (vshift) + : "d4","d5","d6","d7","d8","d9","d10","d11","d12","d13","d14","d15","d16","d17","d18","d19","d20","d21","d22","d23","d24","d25","d26","d27","d28","d29","d30","d31" + ); + } +}) +#else +CVTS_FUNC(s8, s16, 16, + float32x4_t vscale = vdupq_n_f32((f32)alpha); + float32x4_t vshift = vdupq_n_f32((f32)beta + 0.5f);, +{ + for (size_t i = 0; i < w; i += 16) + { + internal::prefetch(_src + i); + int8x16_t vline = vld1q_s8(_src + i); + int16x8_t vline1_s16 = vmovl_s8(vget_low_s8 (vline)); + int16x8_t vline2_s16 = vmovl_s8(vget_high_s8(vline)); + int32x4_t vline1_s32 = vmovl_s16(vget_low_s16 (vline1_s16)); + int32x4_t vline2_s32 = vmovl_s16(vget_high_s16(vline1_s16)); + int32x4_t vline3_s32 = vmovl_s16(vget_low_s16 (vline2_s16)); + int32x4_t vline4_s32 = vmovl_s16(vget_high_s16(vline2_s16)); + float32x4_t vline1_f32 = vcvtq_f32_s32(vline1_s32); + float32x4_t vline2_f32 = vcvtq_f32_s32(vline2_s32); + float32x4_t vline3_f32 = vcvtq_f32_s32(vline3_s32); + float32x4_t vline4_f32 = vcvtq_f32_s32(vline4_s32); + vline1_f32 = vmulq_f32(vline1_f32, vscale); + vline2_f32 = vmulq_f32(vline2_f32, vscale); + vline3_f32 = vmulq_f32(vline3_f32, vscale); + vline4_f32 = vmulq_f32(vline4_f32, vscale); + vline1_f32 = vaddq_f32(vline1_f32, vshift); + vline2_f32 = vaddq_f32(vline2_f32, vshift); + vline3_f32 = vaddq_f32(vline3_f32, vshift); + vline4_f32 = vaddq_f32(vline4_f32, vshift); + vline1_s32 = vcvtq_s32_f32(vline1_f32); + vline2_s32 = vcvtq_s32_f32(vline2_f32); + vline3_s32 = vcvtq_s32_f32(vline3_f32); + vline4_s32 = vcvtq_s32_f32(vline4_f32); + int16x8_t vRes1_s16 = vcombine_s16(vqmovn_s32(vline1_s32), vqmovn_s32(vline2_s32)); + int16x8_t vRes2_s16 = vcombine_s16(vqmovn_s32(vline3_s32), vqmovn_s32(vline4_s32)); + vst1q_s16(_dst + i + 0, vRes1_s16); + vst1q_s16(_dst + i + 8, vRes2_s16); + } +}) +#endif + +#if __GNUC_MINOR__ < 7 +CVTS_FUNC(s8, s32, 16, + register float32x4_t vscale asm ("q0") = vdupq_n_f32((f32)alpha); + register float32x4_t vshift asm ("q1") = vdupq_n_f32((f32)beta + 0.5f);, +{ + for (size_t i = 0; i < w; i += 16) + { + internal::prefetch(_src + i); + __asm__ ( + "vld1.8 {d4-d5}, [%[src]] \n\t" + "vmovl.s8 q3, d4 \n\t" + "vmovl.s8 q4, d5 \n\t" + "vmovl.s16 q5, d6 \n\t" + "vmovl.s16 q6, d7 \n\t" + "vmovl.s16 q7, d8 \n\t" + "vmovl.s16 q8, d9 \n\t" + "vcvt.f32.s32 q9, q5 \n\t" + "vcvt.f32.s32 q10, q6 \n\t" + "vcvt.f32.s32 q11, q7 \n\t" + "vcvt.f32.s32 q12, q8 \n\t" + "vmul.f32 q13, q9, q0 \n\t" + "vmul.f32 q14, q10, q0 \n\t" + "vmul.f32 q15, q11, q0 \n\t" + "vmul.f32 q2, q12, q0 \n\t" + "vadd.f32 q3, q13, q1 \n\t" + "vadd.f32 q4, q14, q1 \n\t" + "vadd.f32 q5, q15, q1 \n\t" + "vadd.f32 q6, q2, q1 \n\t" + "vcvt.s32.f32 q7, q3 \n\t" + "vcvt.s32.f32 q8, q4 \n\t" + "vcvt.s32.f32 q9, q5 \n\t" + "vcvt.s32.f32 q10, q6 \n\t" + "vst1.32 {d14-d15}, [%[dst1]] \n\t" + "vst1.32 {d16-d17}, [%[dst2]] \n\t" + "vst1.32 {d18-d19}, [%[dst3]] \n\t" + "vst1.32 {d20-d21}, [%[dst4]] \n\t" + : /*no output*/ + : [src] "r" (_src + i), + [dst1] "r" (_dst + i + 0), + [dst2] "r" (_dst + i + 4), + [dst3] "r" (_dst + i + 8), + [dst4] "r" (_dst + i + 12), + "w" (vscale), "w" (vshift) + : "d4","d5","d6","d7","d8","d9","d10", + "d11","d12","d13","d14","d15","d16","d17", + "d18","d19","d20","d21","d22","d23","d24", + "d25","d26","d27","d28","d29","d30","d31" + ); + } +}) +#else +CVTS_FUNC(s8, s32, 16, + float32x4_t vscale = vdupq_n_f32((f32)alpha); + float32x4_t vshift = vdupq_n_f32((f32)beta + 0.5f);, +{ + for (size_t i = 0; i < w; i += 16) + { + internal::prefetch(_src + i); + int8x16_t vline = vld1q_s8(_src + i); + int16x8_t vline1_s16 = vmovl_s8(vget_low_s8 (vline)); + int16x8_t vline2_s16 = vmovl_s8(vget_high_s8(vline)); + int32x4_t vline1_s32 = vmovl_s16(vget_low_s16 (vline1_s16)); + int32x4_t vline2_s32 = vmovl_s16(vget_high_s16(vline1_s16)); + int32x4_t vline3_s32 = vmovl_s16(vget_low_s16 (vline2_s16)); + int32x4_t vline4_s32 = vmovl_s16(vget_high_s16(vline2_s16)); + float32x4_t vline1_f32 = vcvtq_f32_s32(vline1_s32); + float32x4_t vline2_f32 = vcvtq_f32_s32(vline2_s32); + float32x4_t vline3_f32 = vcvtq_f32_s32(vline3_s32); + float32x4_t vline4_f32 = vcvtq_f32_s32(vline4_s32); + vline1_f32 = vmulq_f32(vline1_f32, vscale); + vline2_f32 = vmulq_f32(vline2_f32, vscale); + vline3_f32 = vmulq_f32(vline3_f32, vscale); + vline4_f32 = vmulq_f32(vline4_f32, vscale); + vline1_f32 = vaddq_f32(vline1_f32, vshift); + vline2_f32 = vaddq_f32(vline2_f32, vshift); + vline3_f32 = vaddq_f32(vline3_f32, vshift); + vline4_f32 = vaddq_f32(vline4_f32, vshift); + vline1_s32 = vcvtq_s32_f32(vline1_f32); + vline2_s32 = vcvtq_s32_f32(vline2_f32); + vline3_s32 = vcvtq_s32_f32(vline3_f32); + vline4_s32 = vcvtq_s32_f32(vline4_f32); + vst1q_s32(_dst + i + 0, vline1_s32); + vst1q_s32(_dst + i + 4, vline2_s32); + vst1q_s32(_dst + i + 8, vline3_s32); + vst1q_s32(_dst + i + 12, vline4_s32); + } +}) +#endif + +#if __GNUC_MINOR__ < 7 +CVTS_FUNC(s8, f32, 16, + register float32x4_t vscale asm ("q0") = vdupq_n_f32((f32)alpha); + register float32x4_t vshift asm ("q1") = vdupq_n_f32((f32)beta);, +{ + for (size_t i = 0; i < w; i += 16) + { + internal::prefetch(_src + i); + __asm__ ( + "vld1.8 {d4-d5}, [%[src]] \n\t" + "vmovl.s8 q3, d4 \n\t" + "vmovl.s8 q4, d5 \n\t" + "vmovl.s16 q5, d6 \n\t" + "vmovl.s16 q6, d7 \n\t" + "vmovl.s16 q7, d8 \n\t" + "vmovl.s16 q8, d9 \n\t" + "vcvt.f32.s32 q9, q5 \n\t" + "vcvt.f32.s32 q10, q6 \n\t" + "vcvt.f32.s32 q11, q7 \n\t" + "vcvt.f32.s32 q12, q8 \n\t" + "vmul.f32 q13, q9, q0 \n\t" + "vmul.f32 q14, q10, q0 \n\t" + "vmul.f32 q15, q11, q0 \n\t" + "vmul.f32 q2, q12, q0 \n\t" + "vadd.f32 q3, q13, q1 \n\t" + "vadd.f32 q4, q14, q1 \n\t" + "vadd.f32 q5, q15, q1 \n\t" + "vadd.f32 q6, q2, q1 \n\t" + "vst1.32 {d6-d7}, [%[dst1]] \n\t" + "vst1.32 {d8-d9}, [%[dst2]] \n\t" + "vst1.32 {d10-d11}, [%[dst3]] \n\t" + "vst1.32 {d12-d13}, [%[dst4]] \n\t" + : /*no output*/ + : [src] "r" (_src + i), + [dst1] "r" (_dst + i + 0), + [dst2] "r" (_dst + i + 4), + [dst3] "r" (_dst + i + 8), + [dst4] "r" (_dst + i + 12), + "w" (vscale), "w" (vshift) + : "d4","d5","d6","d7","d8","d9","d10", + "d11","d12","d13","d14","d15","d16","d17", + "d18","d19","d20","d21","d22","d23","d24", + "d25","d26","d27","d28","d29","d30","d31" + ); + } +}) +#else +CVTS_FUNC(s8, f32, 16, + float32x4_t vscale = vdupq_n_f32((f32)alpha); + float32x4_t vshift = vdupq_n_f32((f32)beta);, +{ + for (size_t i = 0; i < w; i += 16) + { + internal::prefetch(_src + i); + int8x16_t vline = vld1q_s8(_src + i); + int16x8_t vline1_s16 = vmovl_s8(vget_low_s8 (vline)); + int16x8_t vline2_s16 = vmovl_s8(vget_high_s8(vline)); + int32x4_t vline1_s32 = vmovl_s16(vget_low_s16 (vline1_s16)); + int32x4_t vline2_s32 = vmovl_s16(vget_high_s16(vline1_s16)); + int32x4_t vline3_s32 = vmovl_s16(vget_low_s16 (vline2_s16)); + int32x4_t vline4_s32 = vmovl_s16(vget_high_s16(vline2_s16)); + float32x4_t vline1_f32 = vcvtq_f32_s32(vline1_s32); + float32x4_t vline2_f32 = vcvtq_f32_s32(vline2_s32); + float32x4_t vline3_f32 = vcvtq_f32_s32(vline3_s32); + float32x4_t vline4_f32 = vcvtq_f32_s32(vline4_s32); + vline1_f32 = vmulq_f32(vline1_f32, vscale); + vline2_f32 = vmulq_f32(vline2_f32, vscale); + vline3_f32 = vmulq_f32(vline3_f32, vscale); + vline4_f32 = vmulq_f32(vline4_f32, vscale); + vline1_f32 = vaddq_f32(vline1_f32, vshift); + vline2_f32 = vaddq_f32(vline2_f32, vshift); + vline3_f32 = vaddq_f32(vline3_f32, vshift); + vline4_f32 = vaddq_f32(vline4_f32, vshift); + vst1q_f32(_dst + i + 0, vline1_f32); + vst1q_f32(_dst + i + 4, vline2_f32); + vst1q_f32(_dst + i + 8, vline3_f32); + vst1q_f32(_dst + i + 12, vline4_f32); + } +}) +#endif + +#if __GNUC_MINOR__ < 7 +CVTS_FUNC(u16, u8, 16, + register float32x4_t vscale asm ("q0") = vdupq_n_f32((f32)alpha); + register float32x4_t vshift asm ("q1") = vdupq_n_f32((f32)beta + 0.5f);, +{ + for (size_t i = 0; i < w; i += 8) + { + internal::prefetch(_src + i); + __asm__ ( + "vld1.8 {d4-d5}, [%[src1]] \n\t" + "vmovl.u16 q3, d4 \n\t" + "vmovl.u16 q4, d5 \n\t" + "vcvt.f32.u32 q5, q3 \n\t" + "vcvt.f32.u32 q6, q4 \n\t" + "vmul.f32 q7, q5, q0 \n\t" + "vmul.f32 q8, q6, q0 \n\t" + "vadd.f32 q9, q7, q1 \n\t" + "vadd.f32 q10, q8, q1 \n\t" + "vcvt.s32.f32 q11, q9 \n\t" + "vcvt.s32.f32 q12, q10 \n\t" + "vqmovn.s32 d26, q11 \n\t" + "vqmovn.s32 d27, q12 \n\t" + "vqmovun.s16 d28, q13 \n\t" + "vst1.8 {d28}, [%[dst]] \n\t" + : /*no output*/ + : [src1] "r" (_src + i), + [dst] "r" (_dst + i + 0), + "w" (vscale), "w" (vshift) + : "d4","d5","d6","d7","d8","d9","d10","d11","d12","d13","d14","d15","d16","d17","d18","d19","d20","d21","d22","d23","d24","d25","d26","d27","d28" + ); + } +}) +#else +CVTS_FUNC(u16, u8, 16, + float32x4_t vscale = vdupq_n_f32((f32)alpha); + float32x4_t vshift = vdupq_n_f32((f32)beta + 0.5f);, +{ + for (size_t i = 0; i < w; i += 8) + { + internal::prefetch(_src + i); + uint16x8_t vline = vld1q_u16(_src + i); + uint32x4_t vline1_u32 = vmovl_u16(vget_low_u16 (vline)); + uint32x4_t vline2_u32 = vmovl_u16(vget_high_u16(vline)); + float32x4_t vline1_f32 = vcvtq_f32_u32(vline1_u32); + float32x4_t vline2_f32 = vcvtq_f32_u32(vline2_u32); + vline1_f32 = vmulq_f32(vline1_f32, vscale); + vline2_f32 = vmulq_f32(vline2_f32, vscale); + vline1_f32 = vaddq_f32(vline1_f32, vshift); + vline2_f32 = vaddq_f32(vline2_f32, vshift); + int32x4_t vline1_s32 = vcvtq_s32_f32(vline1_f32); + int32x4_t vline2_s32 = vcvtq_s32_f32(vline2_f32); + int16x4_t vRes1 = vqmovn_s32(vline1_s32); + int16x4_t vRes2 = vqmovn_s32(vline2_s32); + uint8x8_t vRes = vqmovun_s16(vcombine_s16(vRes1, vRes2)); + vst1_u8(_dst + i, vRes); + } +}) +#endif + +#if __GNUC_MINOR__ < 7 +CVTS_FUNC(u16, s8, 16, + register float32x4_t vscale asm ("q0") = vdupq_n_f32((f32)alpha); + register float32x4_t vshift asm ("q1") = vdupq_n_f32((f32)beta + 0.5f);, +{ + for (size_t i = 0; i < w; i += 8) + { + internal::prefetch(_src + i); + __asm__ ( + "vld1.8 {d4-d5}, [%[src1]] \n\t" + "vmovl.u16 q3, d4 \n\t" + "vmovl.u16 q4, d5 \n\t" + "vcvt.f32.u32 q5, q3 \n\t" + "vcvt.f32.u32 q6, q4 \n\t" + "vmul.f32 q7, q5, q0 \n\t" + "vmul.f32 q8, q6, q0 \n\t" + "vadd.f32 q9, q7, q1 \n\t" + "vadd.f32 q10, q8, q1 \n\t" + "vcvt.s32.f32 q11, q9 \n\t" + "vcvt.s32.f32 q12, q10 \n\t" + "vqmovn.s32 d26, q11 \n\t" + "vqmovn.s32 d27, q12 \n\t" + "vqmovn.s16 d28, q13 \n\t" + "vst1.8 {d28}, [%[dst]] \n\t" + : /*no output*/ + : [src1] "r" (_src + i), + [dst] "r" (_dst + i + 0), + "w" (vscale), "w" (vshift) + : "d4","d5","d6","d7","d8","d9","d10","d11","d12","d13","d14","d15","d16","d17","d18","d19","d20","d21","d22","d23","d24","d25","d26","d27","d28" + ); + } +}) +#else +CVTS_FUNC(u16, s8, 16, + float32x4_t vscale = vdupq_n_f32((f32)alpha); + float32x4_t vshift = vdupq_n_f32((f32)beta + 0.5f);, +{ + for (size_t i = 0; i < w; i += 8) + { + internal::prefetch(_src + i); + uint16x8_t vline = vld1q_u16(_src + i); + uint32x4_t vline1_u32 = vmovl_u16(vget_low_u16 (vline)); + uint32x4_t vline2_u32 = vmovl_u16(vget_high_u16(vline)); + float32x4_t vline1_f32 = vcvtq_f32_u32(vline1_u32); + float32x4_t vline2_f32 = vcvtq_f32_u32(vline2_u32); + vline1_f32 = vmulq_f32(vline1_f32, vscale); + vline2_f32 = vmulq_f32(vline2_f32, vscale); + vline1_f32 = vaddq_f32(vline1_f32, vshift); + vline2_f32 = vaddq_f32(vline2_f32, vshift); + int32x4_t vline1_s32 = vcvtq_s32_f32(vline1_f32); + int32x4_t vline2_s32 = vcvtq_s32_f32(vline2_f32); + int16x4_t vRes1 = vqmovn_s32(vline1_s32); + int16x4_t vRes2 = vqmovn_s32(vline2_s32); + int8x8_t vRes = vqmovn_s16(vcombine_s16(vRes1, vRes2)); + vst1_s8(_dst + i, vRes); + } +}) +#endif + +#if __GNUC_MINOR__ < 7 +CVTS_FUNC1(u16, 16, + register float32x4_t vscale asm ("q0") = vdupq_n_f32((f32)alpha); + register float32x4_t vshift asm ("q1") = vdupq_n_f32((f32)beta + 0.5f);, +{ + for (size_t i = 0; i < w; i += 8) + { + internal::prefetch(_src + i); + __asm__ ( + "vld1.16 {d4-d5}, [%[src]] \n\t" + "vmovl.u16 q3, d4 \n\t" + "vmovl.u16 q4, d5 \n\t" + "vcvt.f32.u32 q5, q3 \n\t" + "vcvt.f32.u32 q6, q4 \n\t" + "vmul.f32 q7, q5, q0 \n\t" + "vmul.f32 q8, q6, q0 \n\t" + "vadd.f32 q9, q7, q1 \n\t" + "vadd.f32 q10, q8, q1 \n\t" + "vcvt.s32.f32 q11, q9 \n\t" + "vcvt.s32.f32 q12, q10 \n\t" + "vqmovun.s32 d26, q11 \n\t" + "vqmovun.s32 d27, q12 \n\t" + "vst1.16 {d26-d27}, [%[dst]] \n\t" + : /*no output*/ + : [src] "r" (_src + i), + [dst] "r" (_dst + i + 0), + "w" (vshift), "w" (vscale) + : "d6","d7","d8","d9","d10","d11","d12","d13","d14","d15","d16","d17","d18","d19","d20","d21","d22","d23","d24","d25","d26","d27" + ); + } +}) +#else +CVTS_FUNC1(u16, 16, + float32x4_t vscale = vdupq_n_f32((f32)alpha); + float32x4_t vshift = vdupq_n_f32((f32)beta + 0.5f);, +{ + for (size_t i = 0; i < w; i += 8) + { + internal::prefetch(_src + i); + uint16x8_t vline = vld1q_u16(_src + i); + uint32x4_t vline1_u32 = vmovl_u16(vget_low_u16 (vline)); + uint32x4_t vline2_u32 = vmovl_u16(vget_high_u16(vline)); + float32x4_t vline1_f32 = vcvtq_f32_u32(vline1_u32); + float32x4_t vline2_f32 = vcvtq_f32_u32(vline2_u32); + vline1_f32 = vmulq_f32(vline1_f32, vscale); + vline2_f32 = vmulq_f32(vline2_f32, vscale); + vline1_f32 = vaddq_f32(vline1_f32, vshift); + vline2_f32 = vaddq_f32(vline2_f32, vshift); + int32x4_t vline1_s32 = vcvtq_s32_f32(vline1_f32); + int32x4_t vline2_s32 = vcvtq_s32_f32(vline2_f32); + uint16x4_t vRes1 = vqmovun_s32(vline1_s32); + uint16x4_t vRes2 = vqmovun_s32(vline2_s32); + vst1q_u16(_dst + i, vcombine_u16(vRes1, vRes2)); + } +}) +#endif + +#if __GNUC_MINOR__ < 7 +CVTS_FUNC(u16, s16, 8, + register float32x4_t vscale asm ("q0") = vdupq_n_f32((f32)alpha); + register float32x4_t vshift asm ("q1") = vdupq_n_f32((f32)beta + 0.5f);, +{ + for (size_t i = 0; i < w; i += 8) + { + internal::prefetch(_src + i); + __asm__ ( + "vld1.16 {d4-d5}, [%[src]] \n\t" + "vmovl.u16 q3, d4 \n\t" + "vmovl.u16 q4, d5 \n\t" + "vcvt.f32.u32 q5, q3 \n\t" + "vcvt.f32.u32 q6, q4 \n\t" + "vmul.f32 q7, q5, q0 \n\t" + "vmul.f32 q8, q6, q0 \n\t" + "vadd.f32 q9, q7, q1 \n\t" + "vadd.f32 q10, q8, q1 \n\t" + "vcvt.s32.f32 q11, q9 \n\t" + "vcvt.s32.f32 q12, q10 \n\t" + "vqmovn.s32 d26, q11 \n\t" + "vqmovn.s32 d27, q12 \n\t" + "vst1.16 {d26-d27}, [%[dst]] \n\t" + : /*no output*/ + : [src] "r" (_src + i), + [dst] "r" (_dst + i + 0), + "w" (vshift), "w" (vscale) + : "d6","d7","d8","d9","d10","d11","d12","d13","d14","d15","d16","d17","d18","d19","d20","d21","d22","d23","d24","d25","d26","d27" + ); + } +}) +#else +CVTS_FUNC(u16, s16, 8, + float32x4_t vscale = vdupq_n_f32((f32)alpha); + float32x4_t vshift = vdupq_n_f32((f32)beta + 0.5f);, +{ + for (size_t i = 0; i < w; i += 8) + { + internal::prefetch(_src + i); + uint16x8_t vline = vld1q_u16(_src + i); + uint32x4_t vline1_u32 = vmovl_u16(vget_low_u16 (vline)); + uint32x4_t vline2_u32 = vmovl_u16(vget_high_u16(vline)); + float32x4_t vline1_f32 = vcvtq_f32_u32(vline1_u32); + float32x4_t vline2_f32 = vcvtq_f32_u32(vline2_u32); + vline1_f32 = vmulq_f32(vline1_f32, vscale); + vline2_f32 = vmulq_f32(vline2_f32, vscale); + vline1_f32 = vaddq_f32(vline1_f32, vshift); + vline2_f32 = vaddq_f32(vline2_f32, vshift); + int32x4_t vline1_s32 = vcvtq_s32_f32(vline1_f32); + int32x4_t vline2_s32 = vcvtq_s32_f32(vline2_f32); + int16x4_t vRes1 = vqmovn_s32(vline1_s32); + int16x4_t vRes2 = vqmovn_s32(vline2_s32); + vst1q_s16(_dst + i, vcombine_s16(vRes1, vRes2)); + } +}) +#endif + +#if __GNUC_MINOR__ < 7 +CVTS_FUNC(u16, s32, 8, + register float32x4_t vscale asm ("q0") = vdupq_n_f32((f32)alpha); + register float32x4_t vshift asm ("q1") = vdupq_n_f32((f32)beta + 0.5f);, +{ + for (size_t i = 0; i < w; i += 8) + { + internal::prefetch(_src + i); + __asm__ ( + "vld1.16 {d4-d5}, [%[src]] \n\t" + "vmovl.u16 q3, d4 \n\t" + "vmovl.u16 q4, d5 \n\t" + "vcvt.f32.u32 q5, q3 \n\t" + "vcvt.f32.u32 q6, q4 \n\t" + "vmul.f32 q7, q5, q0 \n\t" + "vmul.f32 q8, q6, q0 \n\t" + "vadd.f32 q9, q7, q1 \n\t" + "vadd.f32 q10, q8, q1 \n\t" + "vcvt.s32.f32 q11, q9 \n\t" + "vcvt.s32.f32 q12, q10 \n\t" + "vst1.32 {d22-d23}, [%[dst1]] \n\t" + "vst1.32 {d24-d25}, [%[dst2]] \n\t" + : /*no output*/ + : [src] "r" (_src + i), + [dst1] "r" (_dst + i), + [dst2] "r" (_dst + i + 4), + "w" (vshift), "w" (vscale) + : "d4","d5","d6","d7","d8","d9","d10","d11","d12","d13","d14","d15","d16","d17","d18","d19","d20","d21","d22","d23","d24","d25" + ); + } +}) +#else +CVTS_FUNC(u16, s32, 8, + float32x4_t vscale = vdupq_n_f32((f32)alpha); + float32x4_t vshift = vdupq_n_f32((f32)beta + 0.5f);, +{ + for (size_t i = 0; i < w; i += 8) + { + internal::prefetch(_src + i); + uint16x8_t vline = vld1q_u16(_src + i); + uint32x4_t vline1_u32 = vmovl_u16(vget_low_u16 (vline)); + uint32x4_t vline2_u32 = vmovl_u16(vget_high_u16(vline)); + float32x4_t vline1_f32 = vcvtq_f32_u32(vline1_u32); + float32x4_t vline2_f32 = vcvtq_f32_u32(vline2_u32); + vline1_f32 = vmulq_f32(vline1_f32, vscale); + vline2_f32 = vmulq_f32(vline2_f32, vscale); + vline1_f32 = vaddq_f32(vline1_f32, vshift); + vline2_f32 = vaddq_f32(vline2_f32, vshift); + int32x4_t vline1_s32 = vcvtq_s32_f32(vline1_f32); + int32x4_t vline2_s32 = vcvtq_s32_f32(vline2_f32); + vst1q_s32(_dst + i + 0, vline1_s32); + vst1q_s32(_dst + i + 4, vline2_s32); + } +}) +#endif + +#if __GNUC_MINOR__ < 7 +CVTS_FUNC(u16, f32, 8, + register float32x4_t vscale asm ("q0") = vdupq_n_f32((f32)alpha); + register float32x4_t vshift asm ("q1") = vdupq_n_f32((f32)beta);, +{ + for (size_t i = 0; i < w; i += 8) + { + internal::prefetch(_src + i); + __asm__ ( + "vld1.16 {d4-d5}, [%[src]] \n\t" + "vmovl.u16 q3, d4 \n\t" + "vmovl.u16 q4, d5 \n\t" + "vcvt.f32.u32 q5, q3 \n\t" + "vcvt.f32.u32 q6, q4 \n\t" + "vmul.f32 q7, q5, q0 \n\t" + "vmul.f32 q8, q6, q0 \n\t" + "vadd.f32 q9, q7, q1 \n\t" + "vadd.f32 q10, q8, q1 \n\t" + "vst1.32 {d18-d19}, [%[dst1]] \n\t" + "vst1.32 {d20-d21}, [%[dst2]] \n\t" + : /*no output*/ + : [src] "r" (_src + i), + [dst1] "r" (_dst + i + 0), + [dst2] "r" (_dst + i + 4), + "w" (vscale), "w" (vshift) + : "d4","d5","d6","d7","d8","d9","d10","d11","d12","d13","d14","d15","d16","d17","d18","d19","d20","d21" + ); + } +}) +#else +CVTS_FUNC(u16, f32, 8, + float32x4_t vscale = vdupq_n_f32((f32)alpha); + float32x4_t vshift = vdupq_n_f32((f32)beta);, +{ + for (size_t i = 0; i < w; i += 8) + { + internal::prefetch(_src + i); + uint16x8_t vline = vld1q_u16(_src + i); + uint32x4_t vline1_u32 = vmovl_u16(vget_low_u16 (vline)); + uint32x4_t vline2_u32 = vmovl_u16(vget_high_u16(vline)); + float32x4_t vline1_f32 = vcvtq_f32_u32(vline1_u32); + float32x4_t vline2_f32 = vcvtq_f32_u32(vline2_u32); + vline1_f32 = vmulq_f32(vline1_f32, vscale); + vline2_f32 = vmulq_f32(vline2_f32, vscale); + vline1_f32 = vaddq_f32(vline1_f32, vshift); + vline2_f32 = vaddq_f32(vline2_f32, vshift); + vst1q_f32(_dst + i + 0, vline1_f32); + vst1q_f32(_dst + i + 4, vline2_f32); + } +}) +#endif + +#if __GNUC_MINOR__ < 7 +CVTS_FUNC(s16, u8, 16, + register float32x4_t vscale asm ("q0") = vdupq_n_f32((f32)alpha); + register float32x4_t vshift asm ("q1") = vdupq_n_f32((f32)beta + 0.5f);, +{ + for (size_t i = 0; i < w; i += 8) + { + internal::prefetch(_src + i); + __asm__ ( + "vld1.8 {d4-d5}, [%[src1]] \n\t" + "vmovl.s16 q3, d4 \n\t" + "vmovl.s16 q4, d5 \n\t" + "vcvt.f32.s32 q5, q3 \n\t" + "vcvt.f32.s32 q6, q4 \n\t" + "vmul.f32 q7, q5, q0 \n\t" + "vmul.f32 q8, q6, q0 \n\t" + "vadd.f32 q9, q7, q1 \n\t" + "vadd.f32 q10, q8, q1 \n\t" + "vcvt.s32.f32 q11, q9 \n\t" + "vcvt.s32.f32 q12, q10 \n\t" + "vqmovn.s32 d26, q11 \n\t" + "vqmovn.s32 d27, q12 \n\t" + "vqmovun.s16 d28, q13 \n\t" + "vst1.8 {d28}, [%[dst]] \n\t" + : /*no output*/ + : [src1] "r" (_src + i), + [dst] "r" (_dst + i + 0), + "w" (vscale), "w" (vshift) + : "d4","d5","d6","d7","d8","d9","d10","d11","d12","d13","d14","d15","d16","d17","d18","d19","d20","d21","d22","d23","d24","d25","d26","d27","d28" + ); + } +}) +#else +CVTS_FUNC(s16, u8, 16, + float32x4_t vscale = vdupq_n_f32((f32)alpha); + float32x4_t vshift = vdupq_n_f32((f32)beta + 0.5f);, +{ + for (size_t i = 0; i < w; i += 8) + { + internal::prefetch(_src + i); + int16x8_t vline = vld1q_s16(_src + i); + int32x4_t vline1_s32 = vmovl_s16(vget_low_s16 (vline)); + int32x4_t vline2_s32 = vmovl_s16(vget_high_s16(vline)); + float32x4_t vline1_f32 = vcvtq_f32_s32(vline1_s32); + float32x4_t vline2_f32 = vcvtq_f32_s32(vline2_s32); + vline1_f32 = vmulq_f32(vline1_f32, vscale); + vline2_f32 = vmulq_f32(vline2_f32, vscale); + vline1_f32 = vaddq_f32(vline1_f32, vshift); + vline2_f32 = vaddq_f32(vline2_f32, vshift); + vline1_s32 = vcvtq_s32_f32(vline1_f32); + vline2_s32 = vcvtq_s32_f32(vline2_f32); + int16x4_t vRes1 = vqmovn_s32(vline1_s32); + int16x4_t vRes2 = vqmovn_s32(vline2_s32); + uint8x8_t vRes = vqmovun_s16(vcombine_s16(vRes1, vRes2)); + vst1_u8(_dst + i, vRes); + } +}) +#endif + +#if __GNUC_MINOR__ < 7 +CVTS_FUNC(s16, s8, 16, + register float32x4_t vscale asm ("q0") = vdupq_n_f32((f32)alpha); + register float32x4_t vshift asm ("q1") = vdupq_n_f32((f32)beta + 0.5f);, +{ + for (size_t i = 0; i < w; i += 8) + { + internal::prefetch(_src + i); + __asm__ ( + "vld1.8 {d4-d5}, [%[src1]] \n\t" + "vmovl.s16 q3, d4 \n\t" + "vmovl.s16 q4, d5 \n\t" + "vcvt.f32.s32 q5, q3 \n\t" + "vcvt.f32.s32 q6, q4 \n\t" + "vmul.f32 q7, q5, q0 \n\t" + "vmul.f32 q8, q6, q0 \n\t" + "vadd.f32 q9, q7, q1 \n\t" + "vadd.f32 q10, q8, q1 \n\t" + "vcvt.s32.f32 q11, q9 \n\t" + "vcvt.s32.f32 q12, q10 \n\t" + "vqmovn.s32 d26, q11 \n\t" + "vqmovn.s32 d27, q12 \n\t" + "vqmovn.s16 d28, q13 \n\t" + "vst1.8 {d28}, [%[dst]] \n\t" + : /*no output*/ + : [src1] "r" (_src + i), + [dst] "r" (_dst + i + 0), + "w" (vscale), "w" (vshift) + : "d4","d5","d6","d7","d8","d9","d10","d11","d12","d13","d14","d15","d16","d17","d18","d19","d20","d21","d22","d23","d24","d25","d26","d27","d28" + ); + } +}) +#else +CVTS_FUNC(s16, s8, 16, + float32x4_t vscale = vdupq_n_f32((f32)alpha); + float32x4_t vshift = vdupq_n_f32((f32)beta + 0.5f);, +{ + for (size_t i = 0; i < w; i += 8) + { + internal::prefetch(_src + i); + int16x8_t vline = vld1q_s16(_src + i); + int32x4_t vline1_s32 = vmovl_s16(vget_low_s16 (vline)); + int32x4_t vline2_s32 = vmovl_s16(vget_high_s16(vline)); + float32x4_t vline1_f32 = vcvtq_f32_s32(vline1_s32); + float32x4_t vline2_f32 = vcvtq_f32_s32(vline2_s32); + vline1_f32 = vmulq_f32(vline1_f32, vscale); + vline2_f32 = vmulq_f32(vline2_f32, vscale); + vline1_f32 = vaddq_f32(vline1_f32, vshift); + vline2_f32 = vaddq_f32(vline2_f32, vshift); + vline1_s32 = vcvtq_s32_f32(vline1_f32); + vline2_s32 = vcvtq_s32_f32(vline2_f32); + int16x4_t vRes1 = vqmovn_s32(vline1_s32); + int16x4_t vRes2 = vqmovn_s32(vline2_s32); + int8x8_t vRes = vqmovn_s16(vcombine_s16(vRes1, vRes2)); + vst1_s8(_dst + i, vRes); + } +}) +#endif + +#if __GNUC_MINOR__ < 7 +CVTS_FUNC(s16, u16, 8, + register float32x4_t vscale asm ("q0") = vdupq_n_f32((f32)alpha); + register float32x4_t vshift asm ("q1") = vdupq_n_f32((f32)beta + 0.5f);, +{ + for (size_t i = 0; i < w; i += 8) + { + internal::prefetch(_src + i); + __asm__ ( + "vld1.16 {d4-d5}, [%[src]] \n\t" + "vmovl.s16 q3, d4 \n\t" + "vmovl.s16 q4, d5 \n\t" + "vcvt.f32.s32 q5, q3 \n\t" + "vcvt.f32.s32 q6, q4 \n\t" + "vmul.f32 q7, q5, q0 \n\t" + "vmul.f32 q8, q6, q0 \n\t" + "vadd.f32 q9, q7, q1 \n\t" + "vadd.f32 q10, q8, q1 \n\t" + "vcvt.s32.f32 q11, q9 \n\t" + "vcvt.s32.f32 q12, q10 \n\t" + "vqmovun.s32 d26, q11 \n\t" + "vqmovun.s32 d27, q12 \n\t" + "vst1.16 {d26-d27}, [%[dst]] \n\t" + : /*no output*/ + : [src] "r" (_src + i), + [dst] "r" (_dst + i + 0), + "w" (vscale), "w" (vshift) + : "d4","d5","d6","d7","d8","d9","d10","d11","d12","d13","d14","d15","d16","d17","d18","d19","d20","d21","d22","d23","d24","d25","d26","d27" + ); + } +}) +#else +CVTS_FUNC(s16, u16, 8, + float32x4_t vscale = vdupq_n_f32((f32)alpha); + float32x4_t vshift = vdupq_n_f32((f32)beta + 0.5f);, +{ + for (size_t i = 0; i < w; i += 8) + { + internal::prefetch(_src + i); + int16x8_t vline = vld1q_s16(_src + i); + int32x4_t vline1_s32 = vmovl_s16(vget_low_s16 (vline)); + int32x4_t vline2_s32 = vmovl_s16(vget_high_s16(vline)); + float32x4_t vline1_f32 = vcvtq_f32_s32(vline1_s32); + float32x4_t vline2_f32 = vcvtq_f32_s32(vline2_s32); + vline1_f32 = vmulq_f32(vline1_f32, vscale); + vline2_f32 = vmulq_f32(vline2_f32, vscale); + vline1_f32 = vaddq_f32(vline1_f32, vshift); + vline2_f32 = vaddq_f32(vline2_f32, vshift); + vline1_s32 = vcvtq_s32_f32(vline1_f32); + vline2_s32 = vcvtq_s32_f32(vline2_f32); + uint16x4_t vRes1 = vqmovun_s32(vline1_s32); + uint16x4_t vRes2 = vqmovun_s32(vline2_s32); + vst1q_u16(_dst + i, vcombine_u16(vRes1, vRes2)); + } +}) +#endif + +#if __GNUC_MINOR__ < 7 +CVTS_FUNC1(s16, 16, + register float32x4_t vscale asm ("q0") = vdupq_n_f32((f32)alpha); + register float32x4_t vshift asm ("q1") = vdupq_n_f32((f32)beta + 0.5f);, +{ + for (size_t i = 0; i < w; i += 8) + { + internal::prefetch(_src + i); + __asm__ ( + "vld1.16 {d4-d5}, [%[src]] \n\t" + "vmovl.s16 q3, d4 \n\t" + "vmovl.s16 q4, d5 \n\t" + "vcvt.f32.s32 q5, q3 \n\t" + "vcvt.f32.s32 q6, q4 \n\t" + "vmul.f32 q7, q5, q0 \n\t" + "vmul.f32 q8, q6, q0 \n\t" + "vadd.f32 q9, q7, q1 \n\t" + "vadd.f32 q10, q8, q1 \n\t" + "vcvt.s32.f32 q11, q9 \n\t" + "vcvt.s32.f32 q12, q10 \n\t" + "vqmovn.s32 d26, q11 \n\t" + "vqmovn.s32 d27, q12 \n\t" + "vst1.16 {d26-d27}, [%[dst]] \n\t" + : /*no output*/ + : [src] "r" (_src + i), + [dst] "r" (_dst + i + 0), + "w" (vshift), "w" (vscale) + : "d6","d7","d8","d9","d10","d11","d12","d13","d14","d15","d16","d17","d18","d19","d20","d21","d22","d23","d24","d25","d26","d27" + ); + } +}) +#else +CVTS_FUNC1(s16, 16, + float32x4_t vscale = vdupq_n_f32((f32)alpha); + float32x4_t vshift = vdupq_n_f32((f32)beta + 0.5f);, +{ + for (size_t i = 0; i < w; i += 8) + { + internal::prefetch(_src + i); + int16x8_t vline = vld1q_s16(_src + i); + int32x4_t vline1_s32 = vmovl_s16(vget_low_s16 (vline)); + int32x4_t vline2_s32 = vmovl_s16(vget_high_s16(vline)); + float32x4_t vline1_f32 = vcvtq_f32_s32(vline1_s32); + float32x4_t vline2_f32 = vcvtq_f32_s32(vline2_s32); + vline1_f32 = vmulq_f32(vline1_f32, vscale); + vline2_f32 = vmulq_f32(vline2_f32, vscale); + vline1_f32 = vaddq_f32(vline1_f32, vshift); + vline2_f32 = vaddq_f32(vline2_f32, vshift); + vline1_s32 = vcvtq_s32_f32(vline1_f32); + vline2_s32 = vcvtq_s32_f32(vline2_f32); + int16x4_t vRes1 = vqmovn_s32(vline1_s32); + int16x4_t vRes2 = vqmovn_s32(vline2_s32); + vst1q_s16(_dst + i, vcombine_s16(vRes1, vRes2)); + } +}) +#endif + +#if __GNUC_MINOR__ < 7 +CVTS_FUNC(s16, s32, 8, + register float32x4_t vscale asm ("q0") = vdupq_n_f32((f32)alpha); + register float32x4_t vshift asm ("q1") = vdupq_n_f32((f32)beta + 0.5f);, +{ + for (size_t i = 0; i < w; i += 8) + { + internal::prefetch(_src + i); + __asm__ ( + "vld1.16 {d4-d5}, [%[src]] \n\t" + "vmovl.s16 q3, d4 \n\t" + "vmovl.s16 q4, d5 \n\t" + "vcvt.f32.s32 q5, q3 \n\t" + "vcvt.f32.s32 q6, q4 \n\t" + "vmul.f32 q7, q5, q0 \n\t" + "vmul.f32 q8, q6, q0 \n\t" + "vadd.f32 q9, q7, q1 \n\t" + "vadd.f32 q10, q8, q1 \n\t" + "vcvt.s32.f32 q11, q9 \n\t" + "vcvt.s32.f32 q12, q10 \n\t" + "vst1.32 {d22-d23}, [%[dst1]] \n\t" + "vst1.32 {d24-d25}, [%[dst2]] \n\t" + : /*no output*/ + : [src] "r" (_src + i), + [dst1] "r" (_dst + i + 0), + [dst2] "r" (_dst + i + 4), + "w" (vscale), "w" (vshift) + : "d4","d5","d6","d7","d8","d9","d10","d11","d12","d13","d14","d15","d16","d17","d18","d19","d20","d21","d22","d23","d24","d25" + ); + } +}) +#else +CVTS_FUNC(s16, s32, 8, + float32x4_t vscale = vdupq_n_f32((f32)alpha); + float32x4_t vshift = vdupq_n_f32((f32)beta + 0.5f);, +{ + for (size_t i = 0; i < w; i += 8) + { + internal::prefetch(_src + i); + int16x8_t vline = vld1q_s16(_src + i); + int32x4_t vline1_s32 = vmovl_s16(vget_low_s16 (vline)); + int32x4_t vline2_s32 = vmovl_s16(vget_high_s16(vline)); + float32x4_t vline1_f32 = vcvtq_f32_s32(vline1_s32); + float32x4_t vline2_f32 = vcvtq_f32_s32(vline2_s32); + vline1_f32 = vmulq_f32(vline1_f32, vscale); + vline2_f32 = vmulq_f32(vline2_f32, vscale); + vline1_f32 = vaddq_f32(vline1_f32, vshift); + vline2_f32 = vaddq_f32(vline2_f32, vshift); + vline1_s32 = vcvtq_s32_f32(vline1_f32); + vline2_s32 = vcvtq_s32_f32(vline2_f32); + vst1q_s32(_dst + i + 0, vline1_s32); + vst1q_s32(_dst + i + 4, vline2_s32); + } +}) +#endif + +#if __GNUC_MINOR__ < 7 +CVTS_FUNC(s16, f32, 8, + register float32x4_t vscale asm ("q0") = vdupq_n_f32((f32)alpha); + register float32x4_t vshift asm ("q1") = vdupq_n_f32((f32)beta);, +{ + for (size_t i = 0; i < w; i += 8) + { + internal::prefetch(_src + i); + __asm__ ( + "vld1.16 {d4-d5}, [%[src]] \n\t" + "vmovl.s16 q3, d4 \n\t" + "vmovl.s16 q4, d5 \n\t" + "vcvt.f32.s32 q5, q3 \n\t" + "vcvt.f32.s32 q6, q4 \n\t" + "vmul.f32 q7, q5, q0 \n\t" + "vmul.f32 q8, q6, q0 \n\t" + "vadd.f32 q9, q7, q1 \n\t" + "vadd.f32 q10, q8, q1 \n\t" + "vst1.32 {d18-d19}, [%[dst1]] \n\t" + "vst1.32 {d20-d21}, [%[dst2]] \n\t" + : /*no output*/ + : [src] "r" (_src + i), + [dst1] "r" (_dst + i + 0), + [dst2] "r" (_dst + i + 4), + "w" (vscale), "w" (vshift) + : "d4","d5","d6","d7","d8","d9","d10","d11","d12","d13","d14","d15","d16","d17","d18","d19","d20","d21" + ); + } +}) +#else +CVTS_FUNC(s16, f32, 8, + float32x4_t vscale = vdupq_n_f32((f32)alpha); + float32x4_t vshift = vdupq_n_f32((f32)beta);, +{ + for (size_t i = 0; i < w; i += 8) + { + internal::prefetch(_src + i); + int16x8_t vline = vld1q_s16(_src + i); + int32x4_t vline1_s32 = vmovl_s16(vget_low_s16 (vline)); + int32x4_t vline2_s32 = vmovl_s16(vget_high_s16(vline)); + float32x4_t vline1_f32 = vcvtq_f32_s32(vline1_s32); + float32x4_t vline2_f32 = vcvtq_f32_s32(vline2_s32); + vline1_f32 = vmulq_f32(vline1_f32, vscale); + vline2_f32 = vmulq_f32(vline2_f32, vscale); + vline1_f32 = vaddq_f32(vline1_f32, vshift); + vline2_f32 = vaddq_f32(vline2_f32, vshift); + vst1q_f32(_dst + i + 0, vline1_f32); + vst1q_f32(_dst + i + 4, vline2_f32); + } +}) +#endif + +#if __GNUC_MINOR__ < 7 +CVTS_FUNC(s32, u8, 8, + register float32x4_t vscale asm ("q0") = vdupq_n_f32((f32)alpha); + register float32x4_t vshift asm ("q1") = vdupq_n_f32((f32)beta + 0.5f);, +{ + for (size_t i = 0; i < w; i += 8) + { + internal::prefetch(_src + i); + __asm__ ( + "vld1.32 {d4-d5}, [%[src1]] \n\t" + "vld1.32 {d6-d7}, [%[src2]] \n\t" + "vcvt.f32.s32 q4, q2 \n\t" + "vcvt.f32.s32 q5, q3 \n\t" + "vmul.f32 q6, q4, q0 \n\t" + "vmul.f32 q7, q5, q0 \n\t" + "vadd.f32 q8, q6, q1 \n\t" + "vadd.f32 q9, q7, q1 \n\t" + "vcvt.s32.f32 q10, q8 \n\t" + "vcvt.s32.f32 q11, q9 \n\t" + "vqmovun.s32 d24, q10 \n\t" + "vqmovun.s32 d25, q11 \n\t" + "vqmovn.u16 d26, q12 \n\t" + "vst1.8 {d26}, [%[dst]] \n\t" + : /*no output*/ + : [src1] "r" (_src + i + 0), + [src2] "r" (_src + i + 4), + [dst] "r" (_dst + i), + "w" (vscale), "w" (vshift) + : "d4","d5","d6","d7","d8","d9","d10","d11","d12","d13","d14","d15","d16","d17","d18","d19","d20","d21","d22","d23","d24","d25","d26" + ); + } +}) +#else +CVTS_FUNC(s32, u8, 8, + float32x4_t vscale = vdupq_n_f32((f32)alpha); + float32x4_t vshift = vdupq_n_f32((f32)beta + 0.5f);, +{ + for (size_t i = 0; i < w; i += 8) + { + internal::prefetch(_src + i); + int32x4_t vline1_s32 = vld1q_s32(_src + i + 0); + int32x4_t vline2_s32 = vld1q_s32(_src + i + 4); + float32x4_t vline1_f32 = vcvtq_f32_s32(vline1_s32); + float32x4_t vline2_f32 = vcvtq_f32_s32(vline2_s32); + vline1_f32 = vmulq_f32(vline1_f32, vscale); + vline2_f32 = vmulq_f32(vline2_f32, vscale); + vline1_f32 = vaddq_f32(vline1_f32, vshift); + vline2_f32 = vaddq_f32(vline2_f32, vshift); + vline1_s32 = vcvtq_s32_f32(vline1_f32); + vline2_s32 = vcvtq_s32_f32(vline2_f32); + uint16x4_t vRes1 = vqmovun_s32(vline1_s32); + uint16x4_t vRes2 = vqmovun_s32(vline2_s32); + uint8x8_t vRes = vqmovn_u16(vcombine_u16(vRes1, vRes2)); + vst1_u8(_dst + i, vRes); + } +}) +#endif + +#if __GNUC_MINOR__ < 7 +CVTS_FUNC(s32, s8, 8, + register float32x4_t vscale asm ("q0") = vdupq_n_f32((f32)alpha); + register float32x4_t vshift asm ("q1") = vdupq_n_f32((f32)beta + 0.5f);, +{ + for (size_t i = 0; i < w; i += 8) + { + internal::prefetch(_src + i); + __asm__ ( + "vld1.32 {d4-d5}, [%[src1]] \n\t" + "vld1.32 {d6-d7}, [%[src2]] \n\t" + "vcvt.f32.s32 q4, q2 \n\t" + "vcvt.f32.s32 q5, q3 \n\t" + "vmul.f32 q6, q4, q0 \n\t" + "vmul.f32 q7, q5, q0 \n\t" + "vadd.f32 q8, q6, q1 \n\t" + "vadd.f32 q9, q7, q1 \n\t" + "vcvt.s32.f32 q10, q8 \n\t" + "vcvt.s32.f32 q11, q9 \n\t" + "vqmovn.s32 d24, q10 \n\t" + "vqmovn.s32 d25, q11 \n\t" + "vqmovn.s16 d26, q12 \n\t" + "vst1.8 {d26}, [%[dst]] \n\t" + : /*no output*/ + : [src1] "r" (_src + i + 0), + [src2] "r" (_src + i + 4), + [dst] "r" (_dst + i), + "w" (vscale), "w" (vshift) + : "d4","d5","d6","d7","d8","d9","d10","d11","d12","d13","d14","d15","d16","d17","d18","d19","d20","d21","d22","d23","d24","d25","d26" + ); + } +}) +#else +CVTS_FUNC(s32, s8, 8, + float32x4_t vscale = vdupq_n_f32((f32)alpha); + float32x4_t vshift = vdupq_n_f32((f32)beta + 0.5f);, +{ + for (size_t i = 0; i < w; i += 8) + { + internal::prefetch(_src + i); + int32x4_t vline1_s32 = vld1q_s32(_src + i + 0); + int32x4_t vline2_s32 = vld1q_s32(_src + i + 4); + float32x4_t vline1_f32 = vcvtq_f32_s32(vline1_s32); + float32x4_t vline2_f32 = vcvtq_f32_s32(vline2_s32); + vline1_f32 = vmulq_f32(vline1_f32, vscale); + vline2_f32 = vmulq_f32(vline2_f32, vscale); + vline1_f32 = vaddq_f32(vline1_f32, vshift); + vline2_f32 = vaddq_f32(vline2_f32, vshift); + vline1_s32 = vcvtq_s32_f32(vline1_f32); + vline2_s32 = vcvtq_s32_f32(vline2_f32); + int16x4_t vRes1 = vqmovn_s32(vline1_s32); + int16x4_t vRes2 = vqmovn_s32(vline2_s32); + int8x8_t vRes = vqmovn_s16(vcombine_s16(vRes1, vRes2)); + vst1_s8(_dst + i, vRes); + } +}) +#endif + +#if __GNUC_MINOR__ < 7 +CVTS_FUNC(s32, u16, 8, + register float32x4_t vscale asm ("q0") = vdupq_n_f32((f32)alpha); + register float32x4_t vshift asm ("q1") = vdupq_n_f32((f32)beta + 0.5f);, +{ + for (size_t i = 0; i < w; i += 8) + { + internal::prefetch(_src + i); + __asm__ ( + "vld1.32 {d4-d5}, [%[src1]] \n\t" + "vld1.32 {d6-d7}, [%[src2]] \n\t" + "vcvt.f32.s32 q4, q2 \n\t" + "vcvt.f32.s32 q5, q3 \n\t" + "vmul.f32 q6, q4, q0 \n\t" + "vmul.f32 q7, q5, q0 \n\t" + "vadd.f32 q8, q6, q1 \n\t" + "vadd.f32 q9, q7, q1 \n\t" + "vcvt.s32.f32 q10, q8 \n\t" + "vcvt.s32.f32 q11, q9 \n\t" + "vqmovun.s32 d24, q10 \n\t" + "vqmovun.s32 d25, q11 \n\t" + "vst1.16 {d24-d25}, [%[dst]] \n\t" + : /*no output*/ + : [src1] "r" (_src + i + 0), + [src2] "r" (_src + i + 4), + [dst] "r" (_dst + i), + "w" (vscale), "w" (vshift) + : "d4","d5","d6","d7","d8","d9","d10","d11","d12","d13","d14","d15","d16","d17","d18","d19","d20","d21","d22","d23","d24","d25" + ); + } +}) +#else +CVTS_FUNC(s32, u16, 8, + float32x4_t vscale = vdupq_n_f32((f32)alpha); + float32x4_t vshift = vdupq_n_f32((f32)beta + 0.5f);, +{ + for (size_t i = 0; i < w; i += 8) + { + internal::prefetch(_src + i); + int32x4_t vline1_s32 = vld1q_s32(_src + i + 0); + int32x4_t vline2_s32 = vld1q_s32(_src + i + 4); + float32x4_t vline1_f32 = vcvtq_f32_s32(vline1_s32); + float32x4_t vline2_f32 = vcvtq_f32_s32(vline2_s32); + vline1_f32 = vmulq_f32(vline1_f32, vscale); + vline2_f32 = vmulq_f32(vline2_f32, vscale); + vline1_f32 = vaddq_f32(vline1_f32, vshift); + vline2_f32 = vaddq_f32(vline2_f32, vshift); + vline1_s32 = vcvtq_s32_f32(vline1_f32); + vline2_s32 = vcvtq_s32_f32(vline2_f32); + uint16x4_t vRes1 = vqmovun_s32(vline1_s32); + uint16x4_t vRes2 = vqmovun_s32(vline2_s32); + vst1q_u16(_dst + i, vcombine_u16(vRes1, vRes2)); + } +}) +#endif + +#if __GNUC_MINOR__ < 7 +CVTS_FUNC(s32, s16, 8, + register float32x4_t vscale asm ("q0") = vdupq_n_f32((f32)alpha); + register float32x4_t vshift asm ("q1") = vdupq_n_f32((f32)beta + 0.5f);, +{ + for (size_t i = 0; i < w; i += 8) + { + internal::prefetch(_src + i); + __asm__ ( + "vld1.32 {d4-d5}, [%[src1]] \n\t" + "vld1.32 {d6-d7}, [%[src2]] \n\t" + "vcvt.f32.s32 q4, q2 \n\t" + "vcvt.f32.s32 q5, q3 \n\t" + "vmul.f32 q6, q4, q0 \n\t" + "vmul.f32 q7, q5, q0 \n\t" + "vadd.f32 q8, q6, q1 \n\t" + "vadd.f32 q9, q7, q1 \n\t" + "vcvt.s32.f32 q10, q8 \n\t" + "vcvt.s32.f32 q11, q9 \n\t" + "vqmovn.s32 d24, q10 \n\t" + "vqmovn.s32 d25, q11 \n\t" + "vst1.8 {d24-d25}, [%[dst]] \n\t" + : /*no output*/ + : [src1] "r" (_src + i + 0), + [src2] "r" (_src + i + 4), + [dst] "r" (_dst + i), + "w" (vscale), "w" (vshift) + : "d4","d5","d6","d7","d8","d9","d10","d11","d12","d13","d14","d15","d16","d17","d18","d19","d20","d21","d22","d23","d24","d25" + ); + } +}) +#else +CVTS_FUNC(s32, s16, 8, + float32x4_t vscale = vdupq_n_f32((f32)alpha); + float32x4_t vshift = vdupq_n_f32((f32)beta + 0.5f);, +{ + for (size_t i = 0; i < w; i += 8) + { + internal::prefetch(_src + i); + int32x4_t vline1_s32 = vld1q_s32(_src + i + 0); + int32x4_t vline2_s32 = vld1q_s32(_src + i + 4); + float32x4_t vline1_f32 = vcvtq_f32_s32(vline1_s32); + float32x4_t vline2_f32 = vcvtq_f32_s32(vline2_s32); + vline1_f32 = vmulq_f32(vline1_f32, vscale); + vline2_f32 = vmulq_f32(vline2_f32, vscale); + vline1_f32 = vaddq_f32(vline1_f32, vshift); + vline2_f32 = vaddq_f32(vline2_f32, vshift); + vline1_s32 = vcvtq_s32_f32(vline1_f32); + vline2_s32 = vcvtq_s32_f32(vline2_f32); + int16x4_t vRes1 = vqmovn_s32(vline1_s32); + int16x4_t vRes2 = vqmovn_s32(vline2_s32); + vst1q_s16(_dst + i, vcombine_s16(vRes1, vRes2)); + } +}) +#endif + +#if __GNUC_MINOR__ < 7 +CVTS_FUNC1(s32, 8, + register float32x4_t vscale asm ("q0") = vdupq_n_f32((f32)alpha); + register float32x4_t vshift asm ("q1") = vdupq_n_f32((f32)beta + 0.5f);, +{ + for (size_t i = 0; i < w; i += 8) + { + internal::prefetch(_src + i); + __asm__ ( + "vld1.32 {d4-d5}, [%[src1]] \n\t" + "vld1.32 {d6-d7}, [%[src2]] \n\t" + "vcvt.f32.s32 q4, q2 \n\t" + "vcvt.f32.s32 q5, q3 \n\t" + "vmul.f32 q6, q4, q0 \n\t" + "vmul.f32 q7, q5, q0 \n\t" + "vadd.f32 q8, q6, q1 \n\t" + "vadd.f32 q9, q7, q1 \n\t" + "vcvt.s32.f32 q10, q8 \n\t" + "vcvt.s32.f32 q11, q9 \n\t" + "vst1.32 {d20-d21}, [%[dst1]] \n\t" + "vst1.32 {d22-d23}, [%[dst2]] \n\t" + : /*no output*/ + : [src1] "r" (_src + i + 0), + [src2] "r" (_src + i + 4), + [dst1] "r" (_dst + i + 0), + [dst2] "r" (_dst + i + 4), + "w" (vscale), "w" (vshift) + : "d4","d5","d6","d7","d8","d9","d10","d11","d12","d13","d14","d15","d16","d17","d18","d19","d20","d21","d22","d23" + ); + } +}) +#else +CVTS_FUNC1(s32, 8, + float32x4_t vscale = vdupq_n_f32((f32)alpha); + float32x4_t vshift = vdupq_n_f32((f32)beta + 0.5f);, +{ + for (size_t i = 0; i < w; i += 8) + { + internal::prefetch(_src + i); + int32x4_t vline1_s32 = vld1q_s32(_src + i + 0); + int32x4_t vline2_s32 = vld1q_s32(_src + i + 4); + float32x4_t vline1_f32 = vcvtq_f32_s32(vline1_s32); + float32x4_t vline2_f32 = vcvtq_f32_s32(vline2_s32); + vline1_f32 = vmulq_f32(vline1_f32, vscale); + vline2_f32 = vmulq_f32(vline2_f32, vscale); + vline1_f32 = vaddq_f32(vline1_f32, vshift); + vline2_f32 = vaddq_f32(vline2_f32, vshift); + vline1_s32 = vcvtq_s32_f32(vline1_f32); + vline2_s32 = vcvtq_s32_f32(vline2_f32); + vst1q_s32(_dst + i + 0, vline1_s32); + vst1q_s32(_dst + i + 4, vline2_s32); + } +}) +#endif + +#if __GNUC_MINOR__ < 7 +CVTS_FUNC(s32, f32, 8, + register float32x4_t vscale asm ("q0") = vdupq_n_f32((f32)alpha); + register float32x4_t vshift asm ("q1") = vdupq_n_f32((f32)beta);, +{ + for (size_t i = 0; i < w; i += 8) + { + internal::prefetch(_src + i); + __asm__ ( + "vld1.32 {d4-d5}, [%[src1]] \n\t" + "vld1.32 {d6-d7}, [%[src2]] \n\t" + "vcvt.f32.s32 q4, q2 \n\t" + "vcvt.f32.s32 q5, q3 \n\t" + "vmul.f32 q6, q4, q0 \n\t" + "vmul.f32 q7, q5, q0 \n\t" + "vadd.f32 q8, q6, q1 \n\t" + "vadd.f32 q9, q7, q1 \n\t" + "vst1.32 {d16-d17}, [%[dst1]] \n\t" + "vst1.32 {d18-d19}, [%[dst2]] \n\t" + : /*no output*/ + : [src1] "r" (_src + i), + [src2] "r" (_src + i + 4), + [dst1] "r" (_dst + i), + [dst2] "r" (_dst + i + 4), + "w" (vscale), "w" (vshift) + : "d4","d5","d6","d7","d8","d9","d10","d11","d12","d13","d14","d15","d16","d17","d18","d19" + ); + } +}) +#else +CVTS_FUNC(s32, f32, 8, + float32x4_t vscale = vdupq_n_f32((f32)alpha); + float32x4_t vshift = vdupq_n_f32((f32)beta);, +{ + for (size_t i = 0; i < w; i += 8) + { + internal::prefetch(_src + i); + int32x4_t vline1_s32 = vld1q_s32(_src + i + 0); + int32x4_t vline2_s32 = vld1q_s32(_src + i + 4); + float32x4_t vline1_f32 = vcvtq_f32_s32(vline1_s32); + float32x4_t vline2_f32 = vcvtq_f32_s32(vline2_s32); + vline1_f32 = vmulq_f32(vline1_f32, vscale); + vline2_f32 = vmulq_f32(vline2_f32, vscale); + vline1_f32 = vaddq_f32(vline1_f32, vshift); + vline2_f32 = vaddq_f32(vline2_f32, vshift); + vst1q_f32(_dst + i + 0, vline1_f32); + vst1q_f32(_dst + i + 4, vline2_f32); + } +}) +#endif + +#if __GNUC_MINOR__ < 7 +CVTS_FUNC(f32, u8, 8, + register float32x4_t vscale asm ("q0") = vdupq_n_f32((f32)((1 << 16)*alpha)); + register float32x4_t vshift asm ("q1") = vdupq_n_f32((f32)((1 << 16)*beta)); + register uint32x4_t vmask asm ("q2") = vdupq_n_u32(1<<16);, +{ + for (size_t i = 0; i < w; i += 8) + { + internal::prefetch(_src + i); + __asm__ ( + "vld1.32 {d6-d7}, [%[src1]] \n\t" + "vld1.32 {d8-d9}, [%[src2]] \n\t" + "vmul.f32 q5, q3, q0 \n\t" + "vmul.f32 q6, q4, q0 \n\t" + "vadd.f32 q7, q5, q1 \n\t" + "vadd.f32 q8, q6, q1 \n\t" + "vcvt.u32.f32 q9, q7 \n\t" + "vcvt.u32.f32 q10, q8 \n\t" + "vbic q11, q2, q6 \n\t" + "vbic q12, q2, q7 \n\t" + "vshr.u32 q13, q11, #16 \n\t" + "vshr.u32 q14, q12, #16 \n\t" + "vqsub.u32 q7, q9, q13 \n\t" + "vqsub.u32 q8, q10, q14 \n\t" + "vqrshrn.u32 d22, q7, #16 \n\t" + "vqrshrn.u32 d23, q8, #16 \n\t" + "vqmovn.u16 d30, q11 \n\t" + "vst1.8 {d30}, [%[dst]] \n\t" + : /*no output*/ + : [src1] "r" (_src + i + 0), + [src2] "r" (_src + i + 4), + [dst] "r" (_dst + i), + "w" (vscale), "w" (vshift), "w" (vmask) + : "d6","d7","d8","d9","d10","d11","d12","d13","d14","d15","d16","d17","d18","d19","d20","d21","d22","d23","d24","d25","d26","d27","d28","d29","d30" + ); + } +}) +#else +CVTS_FUNC(f32, u8, 8, + float32x4_t vscale = vdupq_n_f32((f32)((1 << 16)*alpha)); + float32x4_t vshift = vdupq_n_f32((f32)((1 << 16)*beta)); + uint32x4_t vmask = vdupq_n_u32(1<<16);, +{ + for (size_t i = 0; i < w; i += 8) + { + internal::prefetch(_src + i); + float32x4_t vline1_f32 = vld1q_f32(_src + i + 0); + float32x4_t vline2_f32 = vld1q_f32(_src + i + 4); + + vline1_f32 = vmulq_f32(vline1_f32, vscale); + vline2_f32 = vmulq_f32(vline2_f32, vscale); + float32x4_t vline1Shifted_f32 = vaddq_f32(vline1_f32, vshift); + float32x4_t vline2Shifted_f32 = vaddq_f32(vline2_f32, vshift); + uint32x4_t vline1_u32 = vcvtq_u32_f32(vline1Shifted_f32); + uint32x4_t vline2_u32 = vcvtq_u32_f32(vline2Shifted_f32); + uint32x4_t vline1Mask = vbicq_u32(vmask, vreinterpretq_u32_f32(vline2_f32)); + uint32x4_t vline2Mask = vbicq_u32(vmask, vreinterpretq_u32_f32(vline1Shifted_f32)); + vline1Mask = vshrq_n_u32(vline1Mask, 16); + vline2Mask = vshrq_n_u32(vline2Mask, 16); + vline1_u32 = vqsubq_u32(vline1_u32, vline1Mask); + vline2_u32 = vqsubq_u32(vline2_u32, vline2Mask); + uint16x4_t vRes1 = vqrshrn_n_u32(vline1_u32, 16); + uint16x4_t vRes2 = vqrshrn_n_u32(vline2_u32, 16); + uint8x8_t vRes = vqmovn_u16(vcombine_u16(vRes1, vRes2)); + + vst1_u8(_dst + i, vRes); + } +}) +#endif + +#if __GNUC_MINOR__ < 7 +CVTS_FUNC(f32, s8, 8, + register float32x4_t vscale asm ("q0") = vdupq_n_f32((f32)alpha); + register float32x4_t vshift asm ("q1") = vdupq_n_f32((f32)beta + 0.5f);, +{ + for (size_t i = 0; i < w; i += 8) + { + internal::prefetch(_src + i); + __asm__ ( + "vld1.32 {d4-d5}, [%[src1]] \n\t" + "vld1.32 {d6-d7}, [%[src2]] \n\t" + "vmul.f32 q4, q2, q0 \n\t" + "vmul.f32 q5, q3, q0 \n\t" + "vadd.f32 q6, q4, q1 \n\t" + "vadd.f32 q7, q5, q1 \n\t" + "vcvt.s32.f32 q8, q6 \n\t" + "vcvt.s32.f32 q9, q7 \n\t" + "vqmovn.s32 d14, q8 \n\t" + "vqmovn.s32 d15, q9 \n\t" + "vqmovn.s16 d16, q7 \n\t" + "vst1.8 {d16}, [%[dst]] \n\t" + : /*no output*/ + : [src1] "r" (_src + i + 0), + [src2] "r" (_src + i + 4), + [dst] "r" (_dst + i), + "w" (vscale), "w" (vshift) + : "d4","d5","d6","d7","d8","d9","d10","d11","d12","d13","d14","d15","d16","d17","d18","d19" + ); + } +}) +#else +CVTS_FUNC(f32, s8, 8, + float32x4_t vscale = vdupq_n_f32((f32)alpha); + float32x4_t vshift = vdupq_n_f32((f32)beta + 0.5f);, +{ + for (size_t i = 0; i < w; i += 8) + { + internal::prefetch(_src + i); + float32x4_t vline1_f32 = vld1q_f32(_src + i + 0); + float32x4_t vline2_f32 = vld1q_f32(_src + i + 4); + vline1_f32 = vmulq_f32(vline1_f32, vscale); + vline2_f32 = vmulq_f32(vline2_f32, vscale); + vline1_f32 = vaddq_f32(vline1_f32, vshift); + vline2_f32 = vaddq_f32(vline2_f32, vshift); + int32x4_t vline1_s32 = vcvtq_s32_f32(vline1_f32); + int32x4_t vline2_s32 = vcvtq_s32_f32(vline2_f32); + int16x4_t vRes1 = vqmovn_s32(vline1_s32); + int16x4_t vRes2 = vqmovn_s32(vline2_s32); + int8x8_t vRes = vqmovn_s16(vcombine_s16(vRes1, vRes2)); + vst1_s8(_dst + i, vRes); + } +}) +#endif + +#if __GNUC_MINOR__ < 7 +CVTS_FUNC(f32, u16, 8, + register float32x4_t vscale asm ("q0") = vdupq_n_f32((f32)alpha); + register float32x4_t vshift asm ("q1") = vdupq_n_f32((f32)beta + 0.5f);, +{ + for (size_t i = 0; i < w; i += 8) + { + internal::prefetch(_src + i); + __asm__ ( + "vld1.32 {d4-d5}, [%[src1]] \n\t" + "vld1.32 {d6-d7}, [%[src2]] \n\t" + "vmul.f32 q4, q2, q0 \n\t" + "vmul.f32 q5, q3, q0 \n\t" + "vadd.f32 q6, q4, q1 \n\t" + "vadd.f32 q7, q5, q1 \n\t" + "vcvt.u32.f32 q8, q6 \n\t" + "vcvt.u32.f32 q9, q7 \n\t" + "vqmovn.u32 d8, q8 \n\t" + "vqmovn.u32 d9, q9 \n\t" + "vst1.16 {d8-d9}, [%[dst]] \n\t" + : /*no output*/ + : [src1] "r" (_src + i + 0), + [src2] "r" (_src + i + 4), + [dst] "r" (_dst + i), + "w" (vscale), "w" (vshift) + : "d4","d5","d6","d7","d8","d9","d10","d11","d12","d13","d14","d15","d16","d17","d18","d19" + ); + } +}) +#else +CVTS_FUNC(f32, u16, 8, + float32x4_t vscale = vdupq_n_f32((f32)alpha); + float32x4_t vshift = vdupq_n_f32((f32)beta + 0.5f);, +{ + for (size_t i = 0; i < w; i += 8) + { + internal::prefetch(_src + i); + float32x4_t vline1_f32 = vld1q_f32(_src + i + 0); + float32x4_t vline2_f32 = vld1q_f32(_src + i + 4); + vline1_f32 = vmulq_f32(vline1_f32, vscale); + vline2_f32 = vmulq_f32(vline2_f32, vscale); + vline1_f32 = vaddq_f32(vline1_f32, vshift); + vline2_f32 = vaddq_f32(vline2_f32, vshift); + uint32x4_t vline1_u32 = vcvtq_u32_f32(vline1_f32); + uint32x4_t vline2_u32 = vcvtq_u32_f32(vline2_f32); + uint16x4_t vRes1 = vqmovn_u32(vline1_u32); + uint16x4_t vRes2 = vqmovn_u32(vline2_u32); + vst1q_u16(_dst + i, vcombine_u16(vRes1, vRes2)); + } +}) +#endif + +#if __GNUC_MINOR__ < 7 +CVTS_FUNC(f32, s16, 8, + register float32x4_t vscale asm ("q0") = vdupq_n_f32((f32)alpha); + register float32x4_t vshift asm ("q1") = vdupq_n_f32((f32)beta + 0.5f);, +{ + for (size_t i = 0; i < w; i += 8) + { + internal::prefetch(_src + i); + __asm__ ( + "vld1.32 {d4-d5}, [%[src1]] \n\t" + "vld1.32 {d6-d7}, [%[src2]] \n\t" + "vmul.f32 q4, q2, q0 \n\t" + "vmul.f32 q5, q3, q0 \n\t" + "vadd.f32 q6, q4, q1 \n\t" + "vadd.f32 q7, q5, q1 \n\t" + "vcvt.s32.f32 q8, q6 \n\t" + "vcvt.s32.f32 q9, q7 \n\t" + "vqmovn.s32 d8, q8 \n\t" + "vqmovn.s32 d9, q9 \n\t" + "vst1.16 {d8-d9}, [%[dst]] \n\t" + : /*no output*/ + : [src1] "r" (_src + i + 0), + [src2] "r" (_src + i + 4), + [dst] "r" (_dst + i), + "w" (vscale), "w" (vshift) + : "d4","d5","d6","d7","d8","d9","d10","d11","d12","d13","d14","d15","d16","d17","d18","d19" + ); + } +}) +#else +CVTS_FUNC(f32, s16, 8, + float32x4_t vscale = vdupq_n_f32((f32)alpha); + float32x4_t vshift = vdupq_n_f32((f32)beta + 0.5f);, +{ + for (size_t i = 0; i < w; i += 8) + { + internal::prefetch(_src + i); + float32x4_t vline1_f32 = vld1q_f32(_src + i + 0); + float32x4_t vline2_f32 = vld1q_f32(_src + i + 4); + vline1_f32 = vmulq_f32(vline1_f32, vscale); + vline2_f32 = vmulq_f32(vline2_f32, vscale); + vline1_f32 = vaddq_f32(vline1_f32, vshift); + vline2_f32 = vaddq_f32(vline2_f32, vshift); + int32x4_t vline1_s32 = vcvtq_s32_f32(vline1_f32); + int32x4_t vline2_s32 = vcvtq_s32_f32(vline2_f32); + int16x4_t vRes1 = vqmovn_s32(vline1_s32); + int16x4_t vRes2 = vqmovn_s32(vline2_s32); + vst1q_s16(_dst + i, vcombine_s16(vRes1, vRes2)); + } +}) +#endif + +#if __GNUC_MINOR__ < 7 +CVTS_FUNC(f32, s32, 8, + register float32x4_t vscale asm ("q0") = vdupq_n_f32((f32)alpha); + register float32x4_t vshift asm ("q1") = vdupq_n_f32((f32)beta + 0.5f);, +{ + for (size_t i = 0; i < w; i += 8) + { + internal::prefetch(_src + i); + __asm__ ( + "vld1.32 {d4-d5}, [%[src1]] \n\t" + "vld1.32 {d6-d7}, [%[src2]] \n\t" + "vmul.f32 q4, q2, q0 \n\t" + "vmul.f32 q5, q3, q0 \n\t" + "vadd.f32 q6, q4, q1 \n\t" + "vadd.f32 q7, q5, q1 \n\t" + "vcvt.s32.f32 q4, q6 \n\t" + "vcvt.s32.f32 q5, q7 \n\t" + "vst1.32 {d8-d9}, [%[dst1]] \n\t" + "vst1.32 {d10-d11}, [%[dst2]] \n\t" + : //no output + : [src1] "r" (_src + i), + [src2] "r" (_src + i + 4), + [dst1] "r" (_dst + i), + [dst2] "r" (_dst + i + 4), + "w" (vscale), "w" (vshift) + : "d4","d5","d6","d7","d8","d9","d10","d11","d12","d13","d14","d15" + ); + } +}) +#else +CVTS_FUNC(f32, s32, 8, + float32x4_t vscale = vdupq_n_f32((f32)alpha); + float32x4_t vshift = vdupq_n_f32((f32)beta + 0.5f);, +{ + for (size_t i = 0; i < w; i += 8) + { + internal::prefetch(_src + i); + float32x4_t vline1_f32 = vld1q_f32(_src + i + 0); + float32x4_t vline2_f32 = vld1q_f32(_src + i + 4); + vline1_f32 = vmulq_f32(vline1_f32, vscale); + vline2_f32 = vmulq_f32(vline2_f32, vscale); + vline1_f32 = vaddq_f32(vline1_f32, vshift); + vline2_f32 = vaddq_f32(vline2_f32, vshift); + int32x4_t vline1_s32 = vcvtq_s32_f32(vline1_f32); + int32x4_t vline2_s32 = vcvtq_s32_f32(vline2_f32); + vst1q_s32(_dst + i + 0, vline1_s32); + vst1q_s32(_dst + i + 4, vline2_s32); + } +}) +#endif + +#if __GNUC_MINOR__ < 7 +CVTS_FUNC1(f32, 8, + register float32x4_t vscale asm ("q0") = vdupq_n_f32((f32)alpha); + register float32x4_t vshift asm ("q1") = vdupq_n_f32((f32)beta);, +{ + for (size_t i = 0; i < w; i += 8) + { + internal::prefetch(_src + i); + __asm__ ( + "vld1.32 {d4-d5}, [%[src1]] \n\t" + "vld1.32 {d6-d7}, [%[src2]] \n\t" + "vmul.f32 q4, q2, q0 \n\t" + "vmul.f32 q5, q3, q0 \n\t" + "vadd.f32 q6, q4, q1 \n\t" + "vadd.f32 q7, q5, q1 \n\t" + "vst1.32 {d12-d13}, [%[dst1]] \n\t" + "vst1.32 {d14-d15}, [%[dst2]] \n\t" + : /*no output*/ + : [src1] "r" (_src + i + 0), + [src2] "r" (_src + i + 4), + [dst1] "r" (_dst + i + 0), + [dst2] "r" (_dst + i + 4), + "w" (vscale), "w" (vshift) + : "d4","d5","d6","d7","d8","d9","d10","d11","d12","d13","d14","d15","d16","d17","d18","d19" + ); + } +}) +#else +CVTS_FUNC1(f32, 8, + float32x4_t vscale = vdupq_n_f32((f32)alpha); + float32x4_t vshift = vdupq_n_f32((f32)beta);, +{ + for (size_t i = 0; i < w; i += 8) + { + internal::prefetch(_src + i); + float32x4_t vline1_f32 = vld1q_f32(_src + i + 0); + float32x4_t vline2_f32 = vld1q_f32(_src + i + 4); + vline1_f32 = vmulq_f32(vline1_f32, vscale); + vline2_f32 = vmulq_f32(vline2_f32, vscale); + vline1_f32 = vaddq_f32(vline1_f32, vshift); + vline2_f32 = vaddq_f32(vline2_f32, vshift); + vst1q_f32(_dst + i + 0, vline1_f32); + vst1q_f32(_dst + i + 4, vline2_f32); + } +}) +#endif + +} // namespace CAROTENE_NS diff --git a/3rdparty/carotene/src/convolution.cpp b/3rdparty/carotene/src/convolution.cpp new file mode 100644 index 0000000000..498d7ad883 --- /dev/null +++ b/3rdparty/carotene/src/convolution.cpp @@ -0,0 +1,340 @@ +/* + * By downloading, copying, installing or using the software you agree to this license. + * If you do not agree to this license, do not download, install, + * copy or use the software. + * + * + * License Agreement + * For Open Source Computer Vision Library + * (3-clause BSD License) + * + * Copyright (C) 2014, NVIDIA Corporation, all rights reserved. + * Third party copyrights are property of their respective owners. + * + * Redistribution and use in source and binary forms, with or without modification, + * are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * * Neither the names of the copyright holders nor the names of the contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * This software is provided by the copyright holders and contributors "as is" and + * any express or implied warranties, including, but not limited to, the implied + * warranties of merchantability and fitness for a particular purpose are disclaimed. + * In no event shall copyright holders or contributors be liable for any direct, + * indirect, incidental, special, exemplary, or consequential damages + * (including, but not limited to, procurement of substitute goods or services; + * loss of use, data, or profits; or business interruption) however caused + * and on any theory of liability, whether in contract, strict liability, + * or tort (including negligence or otherwise) arising in any way out of + * the use of this software, even if advised of the possibility of such damage. + */ + +#include "common.hpp" +#include "saturate_cast.hpp" + +namespace CAROTENE_NS { + +bool isConvolutionSupported(const Size2D &size, const Size2D &ksize, + BORDER_MODE border) +{ + return isSupportedConfiguration() && size.width >= 8 && + (border == BORDER_MODE_CONSTANT || + border == BORDER_MODE_REPLICATE) && + (ksize.width == 3) && (ksize.height == 3); +} + +#ifdef CAROTENE_NEON + +namespace { + +template +int32x4_t vshrq_s32(int32x4_t value) +{ + return vshrq_n_s32(value, shift); +} + +template <> +int32x4_t vshrq_s32<0>(int32x4_t value) +{ + return value; +} + +} // namespace + +typedef int32x4_t (* vshrq_s32_func)(int32x4_t value); + +#endif + +void convolution(const Size2D &size, + const u8 * srcBase, ptrdiff_t srcStride, + u8 * dstBase, ptrdiff_t dstStride, + BORDER_MODE border, u8 borderValue, + const Size2D & ksize, s16 * kernelBase, u32 scale) +{ + internal::assertSupportedConfiguration(isConvolutionSupported(size, ksize, border)); +#ifdef CAROTENE_NEON + const uint8x8_t v_zero_u8 = vdup_n_u8(0); + const uint8x8_t v_border = vdup_n_u8(borderValue); + const int32x4_t v_zero_s32 = vdupq_n_s32(0); + + uint8x8_t tprev[3] = { v_zero_u8, v_zero_u8, v_zero_u8 }, + tcurr[3] = { v_zero_u8, v_zero_u8, v_zero_u8 }, + tnext[3] = { v_zero_u8, v_zero_u8, v_zero_u8 }; + uint8x8_t t0 = v_zero_u8, t1 = v_zero_u8, t2 = v_zero_u8; + + ptrdiff_t width = (ptrdiff_t)size.width, height = (ptrdiff_t)size.height; + static const vshrq_s32_func vshrq_s32_a[33] = + { + vshrq_s32<0>, + vshrq_s32<1>, + vshrq_s32<2>, + vshrq_s32<3>, + vshrq_s32<4>, + vshrq_s32<5>, + vshrq_s32<6>, + vshrq_s32<7>, + vshrq_s32<8>, + vshrq_s32<9>, + vshrq_s32<10>, + vshrq_s32<11>, + vshrq_s32<12>, + vshrq_s32<13>, + vshrq_s32<14>, + vshrq_s32<15>, + vshrq_s32<16>, + vshrq_s32<17>, + vshrq_s32<18>, + vshrq_s32<19>, + vshrq_s32<20>, + vshrq_s32<21>, + vshrq_s32<22>, + vshrq_s32<23>, + vshrq_s32<24>, + vshrq_s32<25>, + vshrq_s32<26>, + vshrq_s32<27>, + vshrq_s32<28>, + vshrq_s32<29>, + vshrq_s32<30>, + vshrq_s32<31>, + vshrq_s32<32> + }; + vshrq_s32_func vshrq_s32_p = vshrq_s32_a[scale]; + + for (ptrdiff_t y = 0; y < height; ++y) + { + const u8 * srow0 = y == 0 && border == BORDER_MODE_CONSTANT ? NULL : internal::getRowPtr(srcBase, srcStride, std::max(y - 1, 0)); + const u8 * srow1 = internal::getRowPtr(srcBase, srcStride, y); + const u8 * srow2 = y + 1 == height && border == BORDER_MODE_CONSTANT ? NULL : internal::getRowPtr(srcBase, srcStride, std::min(y + 1, height - 1)); + u8 * drow = internal::getRowPtr(dstBase, dstStride, y); + + u8 prevx[3] = { 0, 0, 0 }, + currx[3] = { 0, 0, 0 }, + nextx[3] = { 0, 0, 0 }; + ptrdiff_t x = 0; + const ptrdiff_t bwidth = y + 2 < height ? width : (width - 8); + + // perform vertical convolution + for ( ; x <= bwidth; x += 8) + { + internal::prefetch(srow0 + x); + internal::prefetch(srow1 + x); + internal::prefetch(srow2 + x); + + uint8x8_t x0 = !srow0 ? v_border : vld1_u8(srow0 + x); + uint8x8_t x1 = vld1_u8(srow1 + x); + uint8x8_t x2 = !srow2 ? v_border : vld1_u8(srow2 + x); + + // calculate values for plain CPU part below if needed + if (x + 8 >= bwidth) + { + ptrdiff_t x3 = x == width ? width - 1 : x; + ptrdiff_t x4 = border == BORDER_MODE_CONSTANT ? x3 - 1 : std::max(x3 - 1, 0); + + if (border == BORDER_MODE_CONSTANT && x4 < 0) + prevx[0] = prevx[1] = prevx[2] = borderValue; + else + { + prevx[0] = srow0 ? srow0[x4] : borderValue; + prevx[1] = srow1[x4] ; + prevx[2] = srow2 ? srow2[x4] : borderValue; + } + + currx[0] = srow0 ? srow0[x3] : borderValue; + currx[1] = srow1[x3] ; + currx[2] = srow2 ? srow2[x3] : borderValue; + } + + // make shift + if (x) + { + tprev[0] = tcurr[0]; + tcurr[0] = tnext[0]; + + tprev[1] = tcurr[1]; + tcurr[1] = tnext[1]; + + tprev[2] = tcurr[2]; + tcurr[2] = tnext[2]; + } + + tnext[0] = x0; + tnext[1] = x1; + tnext[2] = x2; + + // make extrapolation for the first elements + if (!x) + { + // make border + if (border == BORDER_MODE_CONSTANT) + tcurr[0] = tcurr[1] = tcurr[2] = v_border; + else if (border == BORDER_MODE_REPLICATE) + { + tcurr[0] = vdup_n_u8(vget_lane_u8(tnext[0], 0)); + tcurr[1] = vdup_n_u8(vget_lane_u8(tnext[1], 0)); + tcurr[2] = vdup_n_u8(vget_lane_u8(tnext[2], 0)); + } + + continue; + } + + int32x4_t v_dst0 = v_zero_s32, v_dst1 = v_zero_s32; + + { + // combine 3 "shifted" vectors + t0 = vext_u8(tprev[0], tcurr[0], 7); + t1 = tcurr[0]; + t2 = vext_u8(tcurr[0], tnext[0], 1); + + int16x8_t t0_16s = vreinterpretq_s16_u16(vmovl_u8(t0)); + int16x8_t t1_16s = vreinterpretq_s16_u16(vmovl_u8(t1)); + int16x8_t t2_16s = vreinterpretq_s16_u16(vmovl_u8(t2)); + + v_dst0 = vmlal_n_s16(v_dst0, vget_low_s16(t0_16s), kernelBase[8]); + v_dst0 = vmlal_n_s16(v_dst0, vget_low_s16(t1_16s), kernelBase[7]); + v_dst0 = vmlal_n_s16(v_dst0, vget_low_s16(t2_16s), kernelBase[6]); + + v_dst1 = vmlal_n_s16(v_dst1, vget_high_s16(t0_16s), kernelBase[8]); + v_dst1 = vmlal_n_s16(v_dst1, vget_high_s16(t1_16s), kernelBase[7]); + v_dst1 = vmlal_n_s16(v_dst1, vget_high_s16(t2_16s), kernelBase[6]); + } + + { + // combine 3 "shifted" vectors + t0 = vext_u8(tprev[1], tcurr[1], 7); + t1 = tcurr[1]; + t2 = vext_u8(tcurr[1], tnext[1], 1); + + int16x8_t t0_16s = vreinterpretq_s16_u16(vmovl_u8(t0)); + int16x8_t t1_16s = vreinterpretq_s16_u16(vmovl_u8(t1)); + int16x8_t t2_16s = vreinterpretq_s16_u16(vmovl_u8(t2)); + + v_dst0 = vmlal_n_s16(v_dst0, vget_low_s16(t0_16s), kernelBase[5]); + v_dst0 = vmlal_n_s16(v_dst0, vget_low_s16(t1_16s), kernelBase[4]); + v_dst0 = vmlal_n_s16(v_dst0, vget_low_s16(t2_16s), kernelBase[3]); + + v_dst1 = vmlal_n_s16(v_dst1, vget_high_s16(t0_16s), kernelBase[5]); + v_dst1 = vmlal_n_s16(v_dst1, vget_high_s16(t1_16s), kernelBase[4]); + v_dst1 = vmlal_n_s16(v_dst1, vget_high_s16(t2_16s), kernelBase[3]); + } + + { + // combine 3 "shifted" vectors + t0 = vext_u8(tprev[2], tcurr[2], 7); + t1 = tcurr[2]; + t2 = vext_u8(tcurr[2], tnext[2], 1); + + int16x8_t t0_16s = vreinterpretq_s16_u16(vmovl_u8(t0)); + int16x8_t t1_16s = vreinterpretq_s16_u16(vmovl_u8(t1)); + int16x8_t t2_16s = vreinterpretq_s16_u16(vmovl_u8(t2)); + + v_dst0 = vmlal_n_s16(v_dst0, vget_low_s16(t0_16s), kernelBase[2]); + v_dst0 = vmlal_n_s16(v_dst0, vget_low_s16(t1_16s), kernelBase[1]); + v_dst0 = vmlal_n_s16(v_dst0, vget_low_s16(t2_16s), kernelBase[0]); + + v_dst1 = vmlal_n_s16(v_dst1, vget_high_s16(t0_16s), kernelBase[2]); + v_dst1 = vmlal_n_s16(v_dst1, vget_high_s16(t1_16s), kernelBase[1]); + v_dst1 = vmlal_n_s16(v_dst1, vget_high_s16(t2_16s), kernelBase[0]); + } + + + // make scale + v_dst0 = vshrq_s32_p(v_dst0); + v_dst1 = vshrq_s32_p(v_dst1); + + // and add them + vst1_u8(drow + x - 8, vqmovn_u16(vcombine_u16(vqmovun_s32(v_dst0), + vqmovun_s32(v_dst1)))); + } + + x -= 8; + if (x == width) + --x; + + for ( ; x < width; ++x) + { + // make extrapolation for the last elements + if (x + 1 >= width) + { + if (border == BORDER_MODE_CONSTANT) + { + nextx[0] = borderValue; + nextx[1] = borderValue; + nextx[2] = borderValue; + } + else if (border == BORDER_MODE_REPLICATE) + { + nextx[0] = srow0[x]; + nextx[1] = srow1[x]; + nextx[2] = srow2[x]; + } + } + else + { + nextx[0] = srow0 ? srow0[x + 1] : borderValue; + nextx[1] = srow1[x + 1] ; + nextx[2] = srow2 ? srow2[x + 1] : borderValue; + } + + s32 val = 0; + for (s32 _y = 0; _y < 3; ++_y) + val += prevx[_y] * kernelBase[(2 - _y) * 3 + 2] + + currx[_y] * kernelBase[(2 - _y) * 3 + 1] + + nextx[_y] * kernelBase[(2 - _y) * 3 + 0]; + + drow[x] = internal::saturate_cast(val >> scale); + + // make shift + prevx[0] = currx[0]; + currx[0] = nextx[0]; + + prevx[1] = currx[1]; + currx[1] = nextx[1]; + + prevx[2] = currx[2]; + currx[2] = nextx[2]; + } + } +#else + (void)size; + (void)srcBase; + (void)srcStride; + (void)dstBase; + (void)dstStride; + (void)border; + (void)borderValue; + (void)ksize; + (void)kernelBase; + (void)scale; +#endif +} + +} // namespace CAROTENE_NS diff --git a/3rdparty/carotene/src/count_nonzero.cpp b/3rdparty/carotene/src/count_nonzero.cpp new file mode 100644 index 0000000000..be87767cbd --- /dev/null +++ b/3rdparty/carotene/src/count_nonzero.cpp @@ -0,0 +1,430 @@ +/* + * By downloading, copying, installing or using the software you agree to this license. + * If you do not agree to this license, do not download, install, + * copy or use the software. + * + * + * License Agreement + * For Open Source Computer Vision Library + * (3-clause BSD License) + * + * Copyright (C) 2012-2015, NVIDIA Corporation, all rights reserved. + * Third party copyrights are property of their respective owners. + * + * Redistribution and use in source and binary forms, with or without modification, + * are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * * Neither the names of the copyright holders nor the names of the contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * This software is provided by the copyright holders and contributors "as is" and + * any express or implied warranties, including, but not limited to, the implied + * warranties of merchantability and fitness for a particular purpose are disclaimed. + * In no event shall copyright holders or contributors be liable for any direct, + * indirect, incidental, special, exemplary, or consequential damages + * (including, but not limited to, procurement of substitute goods or services; + * loss of use, data, or profits; or business interruption) however caused + * and on any theory of liability, whether in contract, strict liability, + * or tort (including negligence or otherwise) arising in any way out of + * the use of this software, even if advised of the possibility of such damage. + */ + +#include "common.hpp" + +#include + +namespace CAROTENE_NS { + +s32 countNonZero(const Size2D &_size, + const u8 * srcBase, ptrdiff_t srcStride) +{ + internal::assertSupportedConfiguration(); +#ifdef CAROTENE_NEON + Size2D size(_size); + if (srcStride == (ptrdiff_t)(size.width)) + { + size.width *= size.height; + size.height = 1; + } + size_t roiw16 = size.width & ~15u; + s32 result = 0; + for(size_t k = 0; k < size.height; ++k) + { + const u8* src = internal::getRowPtr( srcBase, srcStride, k); + size_t i = 0; + + #define COUNTNONZERO8U_BLOCK_SIZE (16*255) + uint8x16_t vc1 = vmovq_n_u8(1); + for (; i < roiw16;) + { + size_t lim = std::min(i + COUNTNONZERO8U_BLOCK_SIZE, size.width) - 16; + uint8x16_t vs = vmovq_n_u8(0); + + for (; i <= lim; i+= 16) + { + internal::prefetch(src + i); + uint8x16_t vln = vld1q_u8(src + i); + uint8x16_t vnz = vminq_u8(vln, vc1); + vs = vaddq_u8(vs, vnz); + } + + uint32x4_t vs4 = vpaddlq_u16(vpaddlq_u8(vs)); + uint32x2_t vs2 = vadd_u32(vget_low_u32(vs4), vget_high_u32(vs4)); + + s32 s[2]; + vst1_u32((u32*)s, vs2); + + if (s[0] < 0 || s[1] < 0)//saturate in case of overflow ~ 2GB of non-zeros... + { + return 0x7fFFffFF; + } + result += (s[0] += s[1]); + if (s[0] < 0 || result < 0) + { + return 0x7fFFffFF; + } + } + for (; i < size.width; i++) + result += (src[i] != 0)?1:0; + if (result < 0)//saturate in case of overflow ~ 2GB of non-zeros... + { + return 0x7fFFffFF; + } + } + return result; +#else + (void)_size; + (void)srcBase; + (void)srcStride; + + return 0; +#endif +} + +s32 countNonZero(const Size2D &_size, + const u16 * srcBase, ptrdiff_t srcStride) +{ + internal::assertSupportedConfiguration(); +#ifdef CAROTENE_NEON + Size2D size(_size); + if (srcStride == (ptrdiff_t)(size.width)) + { + size.width *= size.height; + size.height = 1; + } + size_t roiw8 = size.width & ~7u; + s32 result = 0; + for(size_t k = 0; k < size.height; ++k) + { + const u16* src = internal::getRowPtr( srcBase, srcStride, k); + size_t i = 0; + + #define COUNTNONZERO16U_BLOCK_SIZE (8*(256*256-1)) + uint16x8_t vc1 = vmovq_n_u16(1); + for (; i < roiw8;) + { + size_t lim = std::min(i + COUNTNONZERO16U_BLOCK_SIZE, size.width) - 8; + uint16x8_t vs = vmovq_n_u16(0); + + for (; i <= lim; i+= 8) + { + internal::prefetch(src + i); + uint16x8_t vln = vld1q_u16(src + i); + uint16x8_t vnz = vminq_u16(vln, vc1); + vs = vaddq_u16(vs, vnz); + } + + uint32x4_t vs4 = vpaddlq_u16(vs); + uint32x2_t vs2 = vadd_u32(vget_low_u32(vs4), vget_high_u32(vs4)); + + s32 s[2]; + vst1_u32((u32*)s, vs2); + + if (s[0] < 0 || s[1] < 0)//saturate in case of overflow ~ 4GB of non-zeros... + { + return 0x7fFFffFF; + } + result += (s[0] += s[1]); + if (s[0] < 0 || result < 0) + { + return 0x7fFFffFF; + } + } + for (; i < size.width; i++) + result += (src[i] != 0)?1:0; + if (result < 0)//saturate in case of overflow ~ 4GB of non-zeros... + { + return 0x7fFFffFF; + } + } + return result; +#else + (void)_size; + (void)srcBase; + (void)srcStride; + + return 0; +#endif +} + +s32 countNonZero(const Size2D &_size, + const s32 * srcBase, ptrdiff_t srcStride) +{ + internal::assertSupportedConfiguration(); +#ifdef CAROTENE_NEON + Size2D size(_size); + if (srcStride == (ptrdiff_t)(size.width)) + { + size.width *= size.height; + size.height = 1; + } + size_t roiw4 = size.width & ~3u; + s32 result = 0; + for(size_t k = 0; k < size.height; ++k) + { + const u32* src = (const u32*)internal::getRowPtr( srcBase, srcStride, k); + u32 i = 0; + + uint32x4_t vc1 = vmovq_n_u32(1); + uint32x4_t vs = vmovq_n_u32(0); + + for (; i < roiw4; i += 4 ) + { + internal::prefetch(src + i); + uint32x4_t vln = vld1q_u32(src + i); + uint32x4_t vnz = vminq_u32(vln, vc1); + vs = vqaddq_u32(vs, vnz); + } + + uint32x2_t vs2 = vqadd_u32(vget_low_u32(vs), vget_high_u32(vs)); + + s32 s[2]; + vst1_u32((u32*)s, vs2); + + if (s[0] < 0 || s[1] < 0)//saturate in case of overflow ~ 8GB of non-zeros... + { + return 0x7fFFffFF; + } + result += (s[0] += s[1]); + if (s[0] < 0 || result < 0) + { + return 0x7fFFffFF; + } + + for (; i < size.width; i++) + result += (src[i] != 0)?1:0; + if (result < 0)//saturate in case of overflow ~ 8GB of non-zeros... + { + return 0x7fFFffFF; + } + } + return result; +#else + (void)_size; + (void)srcBase; + (void)srcStride; + + return 0; +#endif +} + +s32 countNonZero(const Size2D &_size, + const f32 * srcBase, ptrdiff_t srcStride) +{ + internal::assertSupportedConfiguration(); +#ifdef CAROTENE_NEON + Size2D size(_size); + if (srcStride == (ptrdiff_t)(size.width)) + { + size.width *= size.height; + size.height = 1; + } + size_t roiw4 = size.width & ~3u; + s32 result = 0; + for(size_t k = 0; k < size.height; ++k) + { + const f32* src = internal::getRowPtr( srcBase, srcStride, k); + size_t i = 0; + + float32x4_t vc0 = vmovq_n_f32(0); + int32x4_t vs = vmovq_n_s32(0); + + for (; i < roiw4; i += 4 ) + { + internal::prefetch(src + i); + float32x4_t vln = vld1q_f32(src + i); + int32x4_t vnz = vreinterpretq_s32_u32(vmvnq_u32(vceqq_f32(vln, vc0))); + vs = vqaddq_s32(vs, vnz); + } + + int32x2_t vs2 = vqneg_s32(vqadd_s32(vget_low_s32(vs), vget_high_s32(vs))); + + int s[2]; + vst1_s32(s, vs2); + + result += (s[0] += s[1]); + if (s[0] < 0 || result < 0)//case of overflow ~ 8GB of non-zeros... + { + return 0x7fFFffFF; + } + + for (; i < size.width; i++) + result += (src[i] < std::numeric_limits::min() && src[i] > -std::numeric_limits::min())?0:1; + + if (result < 0) + { + return 0x7fFFffFF; + } + } + return result; +#else + (void)_size; + (void)srcBase; + (void)srcStride; + + return 0; +#endif +} + +s32 countNonZero(const Size2D &_size, + const f64 * srcBase, ptrdiff_t srcStride) +{ + internal::assertSupportedConfiguration(); +#ifdef CAROTENE_NEON + Size2D size(_size); + if (srcStride == (ptrdiff_t)(size.width)) + { + size.width *= size.height; + size.height = 1; + } + size_t roiw8 = size.width & ~7u; + size_t roiw4 = size.width & ~3u; + size_t roiw2 = size.width & ~1u; + uint64x2_t vmask1 = vdupq_n_u64(0x7fFFffFFffFFffFFULL); //will treat denormals as non-zero + uint32x4_t vc0 = vmovq_n_u32(0); + + s32 result = 0; + for(size_t k = 0; k < size.height; ++k) + { + const f64* src = internal::getRowPtr( srcBase, srcStride, k); + size_t i = 0; + + int32x2_t vs1 = vmov_n_s32(0); + int32x2_t vs2 = vmov_n_s32(0); + int32x2_t vs3 = vmov_n_s32(0); + int32x2_t vs4 = vmov_n_s32(0); + + for (; i < roiw8; i += 8 ) + { + internal::prefetch(src + i + 6); + uint64x2_t vln1 = vld1q_u64((const u64*)(src + i)); + uint64x2_t vln2 = vld1q_u64((const u64*)(src + i + 2)); + uint64x2_t vln3 = vld1q_u64((const u64*)(src + i + 4)); + uint64x2_t vln4 = vld1q_u64((const u64*)(src + i + 6)); + + uint64x2_t vm1 = vandq_u64(vln1, vmask1); + uint64x2_t vm2 = vandq_u64(vln2, vmask1); + uint64x2_t vm3 = vandq_u64(vln3, vmask1); + uint64x2_t vm4 = vandq_u64(vln4, vmask1); + + uint32x4_t vequ1 = vceqq_u32(vreinterpretq_u32_u64(vm1), vc0); + uint32x4_t vequ2 = vceqq_u32(vreinterpretq_u32_u64(vm2), vc0); + uint32x4_t vequ3 = vceqq_u32(vreinterpretq_u32_u64(vm3), vc0); + uint32x4_t vequ4 = vceqq_u32(vreinterpretq_u32_u64(vm4), vc0); + + uint32x4_t vlx1 = vmvnq_u32(vequ1); + uint32x4_t vlx2 = vmvnq_u32(vequ2); + uint32x4_t vlx3 = vmvnq_u32(vequ3); + uint32x4_t vlx4 = vmvnq_u32(vequ4); + + int32x2_t vnz1 = vreinterpret_s32_u32(vpmax_u32(vget_low_u32(vlx1), vget_high_u32(vlx1))); + int32x2_t vnz2 = vreinterpret_s32_u32(vpmax_u32(vget_low_u32(vlx2), vget_high_u32(vlx2))); + int32x2_t vnz3 = vreinterpret_s32_u32(vpmax_u32(vget_low_u32(vlx3), vget_high_u32(vlx3))); + int32x2_t vnz4 = vreinterpret_s32_u32(vpmax_u32(vget_low_u32(vlx4), vget_high_u32(vlx4))); + + vs1 = vqadd_s32(vs1, vnz1); + vs2 = vqadd_s32(vs2, vnz2); + vs3 = vqadd_s32(vs3, vnz3); + vs4 = vqadd_s32(vs4, vnz4); + } + + if (i < roiw4) + { + internal::prefetch(src + i + 2); + uint64x2_t vln1 = vld1q_u64((const u64*)(src + i)); + uint64x2_t vln2 = vld1q_u64((const u64*)(src + i + 2)); + + uint64x2_t vm1 = vandq_u64(vln1, vmask1); + uint64x2_t vm2 = vandq_u64(vln2, vmask1); + + uint32x4_t vequ1 = vceqq_u32(vreinterpretq_u32_u64(vm1), vc0); + uint32x4_t vequ2 = vceqq_u32(vreinterpretq_u32_u64(vm2), vc0); + + uint32x4_t vlx1 = vmvnq_u32(vequ1); + uint32x4_t vlx2 = vmvnq_u32(vequ2); + + int32x2_t vnz1 = vreinterpret_s32_u32(vpmax_u32(vget_low_u32(vlx1), vget_high_u32(vlx1))); + int32x2_t vnz2 = vreinterpret_s32_u32(vpmax_u32(vget_low_u32(vlx2), vget_high_u32(vlx2))); + + vs1 = vqadd_s32(vs1, vnz1); + vs2 = vqadd_s32(vs2, vnz2); + i += 4; + } + + if (i < roiw2) + { + internal::prefetch(src + i); + uint64x2_t vln1 = vld1q_u64((const u64*)(src + i)); + + uint64x2_t vm1 = vandq_u64(vln1, vmask1); + + uint32x4_t vequ1 = vceqq_u32(vreinterpretq_u32_u64(vm1), vc0); + + uint32x4_t vlx1 = vmvnq_u32(vequ1); + + int32x2_t vnz1 = vreinterpret_s32_u32(vpmax_u32(vget_low_u32(vlx1), vget_high_u32(vlx1))); + + vs1 = vqadd_s32(vs1, vnz1); + i += 2; + } + + vs1 = vqadd_s32(vs1, vs2); + vs3 = vqadd_s32(vs3, vs4); + vs1 = vqadd_s32(vs1, vs3); + int32x2_t vsneg = vqneg_s32(vs1); + + s32 s[2]; + vst1_s32(s, vsneg); + + result += (s[0] += s[1]); + if (s[0] < 0 || result < 0)//case of overflow ~ 16GB of non-zeros... + { + return 0x7fFFffFF; + } + + for (; i < size.width; i++) + result += (src[i] < std::numeric_limits::min() && src[i] > -std::numeric_limits::min())?0:1; + if (result < 0) + { + return 0x7fFFffFF; + } + } + return result; +#else + (void)_size; + (void)srcBase; + (void)srcStride; + + return 0; +#endif +} + +} // namespace CAROTENE_NS diff --git a/3rdparty/carotene/src/div.cpp b/3rdparty/carotene/src/div.cpp new file mode 100644 index 0000000000..9c03202a83 --- /dev/null +++ b/3rdparty/carotene/src/div.cpp @@ -0,0 +1,694 @@ +/* + * By downloading, copying, installing or using the software you agree to this license. + * If you do not agree to this license, do not download, install, + * copy or use the software. + * + * + * License Agreement + * For Open Source Computer Vision Library + * (3-clause BSD License) + * + * Copyright (C) 2016, NVIDIA Corporation, all rights reserved. + * Third party copyrights are property of their respective owners. + * + * Redistribution and use in source and binary forms, with or without modification, + * are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * * Neither the names of the copyright holders nor the names of the contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * This software is provided by the copyright holders and contributors "as is" and + * any express or implied warranties, including, but not limited to, the implied + * warranties of merchantability and fitness for a particular purpose are disclaimed. + * In no event shall copyright holders or contributors be liable for any direct, + * indirect, incidental, special, exemplary, or consequential damages + * (including, but not limited to, procurement of substitute goods or services; + * loss of use, data, or profits; or business interruption) however caused + * and on any theory of liability, whether in contract, strict liability, + * or tort (including negligence or otherwise) arising in any way out of + * the use of this software, even if advised of the possibility of such damage. + */ + +#include "common.hpp" +#include "vtransform.hpp" + +#include +#include +#include +#include + +namespace CAROTENE_NS { + +namespace { + +#ifdef CAROTENE_NEON + +template +inline T divSaturateQ(const T &v1, const T &v2, const float scale) +{ + return internal::vcombine(internal::vqmovn(divSaturateQ(internal::vmovl(internal::vget_low(v1)), + internal::vmovl(internal::vget_low(v2)), scale)), + internal::vqmovn(divSaturateQ(internal::vmovl(internal::vget_high(v1)), + internal::vmovl(internal::vget_high(v2)), scale)) + ); +} +template <> +inline int32x4_t divSaturateQ(const int32x4_t &v1, const int32x4_t &v2, const float scale) +{ return vcvtq_s32_f32(vmulq_f32(vmulq_n_f32(vcvtq_f32_s32(v1), scale), internal::vrecpq_f32(vcvtq_f32_s32(v2)))); } +template <> +inline uint32x4_t divSaturateQ(const uint32x4_t &v1, const uint32x4_t &v2, const float scale) +{ return vcvtq_u32_f32(vmulq_f32(vmulq_n_f32(vcvtq_f32_u32(v1), scale), internal::vrecpq_f32(vcvtq_f32_u32(v2)))); } + +template +inline T divSaturate(const T &v1, const T &v2, const float scale) +{ + return internal::vqmovn(divSaturateQ(internal::vmovl(v1), internal::vmovl(v2), scale)); +} +template <> +inline int32x2_t divSaturate(const int32x2_t &v1, const int32x2_t &v2, const float scale) +{ return vcvt_s32_f32(vmul_f32(vmul_n_f32(vcvt_f32_s32(v1), scale), internal::vrecp_f32(vcvt_f32_s32(v2)))); } +template <> +inline uint32x2_t divSaturate(const uint32x2_t &v1, const uint32x2_t &v2, const float scale) +{ return vcvt_u32_f32(vmul_f32(vmul_n_f32(vcvt_f32_u32(v1), scale), internal::vrecp_f32(vcvt_f32_u32(v2)))); } + + +template +inline T divWrapQ(const T &v1, const T &v2, const float scale) +{ + return internal::vcombine(internal::vmovn(divWrapQ(internal::vmovl(internal::vget_low(v1)), + internal::vmovl(internal::vget_low(v2)), scale)), + internal::vmovn(divWrapQ(internal::vmovl(internal::vget_high(v1)), + internal::vmovl(internal::vget_high(v2)), scale)) + ); +} +template <> +inline int32x4_t divWrapQ(const int32x4_t &v1, const int32x4_t &v2, const float scale) +{ return vcvtq_s32_f32(vmulq_f32(vmulq_n_f32(vcvtq_f32_s32(v1), scale), internal::vrecpq_f32(vcvtq_f32_s32(v2)))); } +template <> +inline uint32x4_t divWrapQ(const uint32x4_t &v1, const uint32x4_t &v2, const float scale) +{ return vcvtq_u32_f32(vmulq_f32(vmulq_n_f32(vcvtq_f32_u32(v1), scale), internal::vrecpq_f32(vcvtq_f32_u32(v2)))); } + +template +inline T divWrap(const T &v1, const T &v2, const float scale) +{ + return internal::vmovn(divWrapQ(internal::vmovl(v1), internal::vmovl(v2), scale)); +} +template <> +inline int32x2_t divWrap(const int32x2_t &v1, const int32x2_t &v2, const float scale) +{ return vcvt_s32_f32(vmul_f32(vmul_n_f32(vcvt_f32_s32(v1), scale), internal::vrecp_f32(vcvt_f32_s32(v2)))); } +template <> +inline uint32x2_t divWrap(const uint32x2_t &v1, const uint32x2_t &v2, const float scale) +{ return vcvt_u32_f32(vmul_f32(vmul_n_f32(vcvt_f32_u32(v1), scale), internal::vrecp_f32(vcvt_f32_u32(v2)))); } + +inline uint8x16_t vtstq(const uint8x16_t & v0, const uint8x16_t & v1) { return vtstq_u8 (v0, v1); } +inline uint16x8_t vtstq(const uint16x8_t & v0, const uint16x8_t & v1) { return vtstq_u16(v0, v1); } +inline uint32x4_t vtstq(const uint32x4_t & v0, const uint32x4_t & v1) { return vtstq_u32(v0, v1); } +inline int8x16_t vtstq(const int8x16_t & v0, const int8x16_t & v1) { return vreinterpretq_s8_u8 (vtstq_s8 (v0, v1)); } +inline int16x8_t vtstq(const int16x8_t & v0, const int16x8_t & v1) { return vreinterpretq_s16_u16(vtstq_s16(v0, v1)); } +inline int32x4_t vtstq(const int32x4_t & v0, const int32x4_t & v1) { return vreinterpretq_s32_u32(vtstq_s32(v0, v1)); } + +inline uint8x8_t vtst(const uint8x8_t & v0, const uint8x8_t & v1) { return vtst_u8 (v0, v1); } +inline uint16x4_t vtst(const uint16x4_t & v0, const uint16x4_t & v1) { return vtst_u16(v0, v1); } +inline uint32x2_t vtst(const uint32x2_t & v0, const uint32x2_t & v1) { return vtst_u32(v0, v1); } +inline int8x8_t vtst(const int8x8_t & v0, const int8x8_t & v1) { return vreinterpret_s8_u8 (vtst_s8 (v0, v1)); } +inline int16x4_t vtst(const int16x4_t & v0, const int16x4_t & v1) { return vreinterpret_s16_u16(vtst_s16(v0, v1)); } +inline int32x2_t vtst(const int32x2_t & v0, const int32x2_t & v1) { return vreinterpret_s32_u32(vtst_s32(v0, v1)); } +#endif + +template +void div(const Size2D &size, + const T * src0Base, ptrdiff_t src0Stride, + const T * src1Base, ptrdiff_t src1Stride, + T * dstBase, ptrdiff_t dstStride, + f32 scale, + CONVERT_POLICY cpolicy) +{ + internal::assertSupportedConfiguration(); + +#ifdef CAROTENE_NEON + typedef typename internal::VecTraits::vec128 vec128; + typedef typename internal::VecTraits::vec64 vec64; + + if (scale == 0.0f || + (std::numeric_limits::is_integer && + (scale * std::numeric_limits::max()) < 1.0f && + (scale * std::numeric_limits::max()) > -1.0f)) + { + for (size_t y = 0; y < size.height; ++y) + { + T * dst = internal::getRowPtr(dstBase, dstStride, y); + std::memset(dst, 0, sizeof(T) * size.width); + } + return; + } + + const size_t step128 = 16 / sizeof(T); + size_t roiw128 = size.width >= (step128 - 1) ? size.width - step128 + 1 : 0; + const size_t step64 = 8 / sizeof(T); + size_t roiw64 = size.width >= (step64 - 1) ? size.width - step64 + 1 : 0; + + for (size_t i = 0; i < size.height; ++i) + { + const T * src0 = internal::getRowPtr(src0Base, src0Stride, i); + const T * src1 = internal::getRowPtr(src1Base, src1Stride, i); + T * dst = internal::getRowPtr(dstBase, dstStride, i); + size_t j = 0; + + if (cpolicy == CONVERT_POLICY_SATURATE) + { + for (; j < roiw128; j += step128) + { + internal::prefetch(src0 + j); + internal::prefetch(src1 + j); + + vec128 v_src0 = internal::vld1q(src0 + j); + vec128 v_src1 = internal::vld1q(src1 + j); + + vec128 v_mask = vtstq(v_src1,v_src1); + internal::vst1q(dst + j, internal::vandq(v_mask, divSaturateQ(v_src0, v_src1, scale))); + } + for (; j < roiw64; j += step64) + { + vec64 v_src0 = internal::vld1(src0 + j); + vec64 v_src1 = internal::vld1(src1 + j); + + vec64 v_mask = vtst(v_src1,v_src1); + internal::vst1(dst + j, internal::vand(v_mask,divSaturate(v_src0, v_src1, scale))); + } + for (; j < size.width; j++) + { + dst[j] = src1[j] ? internal::saturate_cast(scale * src0[j] / src1[j]) : 0; + } + } + else // CONVERT_POLICY_WRAP + { + for (; j < roiw128; j += step128) + { + internal::prefetch(src0 + j); + internal::prefetch(src1 + j); + + vec128 v_src0 = internal::vld1q(src0 + j); + vec128 v_src1 = internal::vld1q(src1 + j); + + vec128 v_mask = vtstq(v_src1,v_src1); + internal::vst1q(dst + j, internal::vandq(v_mask, divWrapQ(v_src0, v_src1, scale))); + } + for (; j < roiw64; j += step64) + { + vec64 v_src0 = internal::vld1(src0 + j); + vec64 v_src1 = internal::vld1(src1 + j); + + vec64 v_mask = vtst(v_src1,v_src1); + internal::vst1(dst + j, internal::vand(v_mask,divWrap(v_src0, v_src1, scale))); + } + for (; j < size.width; j++) + { + dst[j] = src1[j] ? (T)((s32)trunc(scale * src0[j] / src1[j])) : 0; + } + } + } +#else + (void)size; + (void)src0Base; + (void)src0Stride; + (void)src1Base; + (void)src1Stride; + (void)dstBase; + (void)dstStride; + (void)cpolicy; + (void)scale; +#endif +} + +#ifdef CAROTENE_NEON + +template +inline T recipSaturateQ(const T &v2, const float scale) +{ + return internal::vcombine(internal::vqmovn(recipSaturateQ(internal::vmovl(internal::vget_low(v2)), scale)), + internal::vqmovn(recipSaturateQ(internal::vmovl(internal::vget_high(v2)), scale)) + ); +} +template <> +inline int32x4_t recipSaturateQ(const int32x4_t &v2, const float scale) +{ return vcvtq_s32_f32(vmulq_n_f32(internal::vrecpq_f32(vcvtq_f32_s32(v2)), scale)); } +template <> +inline uint32x4_t recipSaturateQ(const uint32x4_t &v2, const float scale) +{ return vcvtq_u32_f32(vmulq_n_f32(internal::vrecpq_f32(vcvtq_f32_u32(v2)), scale)); } + +template +inline T recipSaturate(const T &v2, const float scale) +{ + return internal::vqmovn(recipSaturateQ(internal::vmovl(v2), scale)); +} +template <> +inline int32x2_t recipSaturate(const int32x2_t &v2, const float scale) +{ return vcvt_s32_f32(vmul_n_f32(internal::vrecp_f32(vcvt_f32_s32(v2)), scale)); } +template <> +inline uint32x2_t recipSaturate(const uint32x2_t &v2, const float scale) +{ return vcvt_u32_f32(vmul_n_f32(internal::vrecp_f32(vcvt_f32_u32(v2)), scale)); } + + +template +inline T recipWrapQ(const T &v2, const float scale) +{ + return internal::vcombine(internal::vmovn(recipWrapQ(internal::vmovl(internal::vget_low(v2)), scale)), + internal::vmovn(recipWrapQ(internal::vmovl(internal::vget_high(v2)), scale)) + ); +} +template <> +inline int32x4_t recipWrapQ(const int32x4_t &v2, const float scale) +{ return vcvtq_s32_f32(vmulq_n_f32(internal::vrecpq_f32(vcvtq_f32_s32(v2)), scale)); } +template <> +inline uint32x4_t recipWrapQ(const uint32x4_t &v2, const float scale) +{ return vcvtq_u32_f32(vmulq_n_f32(internal::vrecpq_f32(vcvtq_f32_u32(v2)), scale)); } + +template +inline T recipWrap(const T &v2, const float scale) +{ + return internal::vmovn(recipWrapQ(internal::vmovl(v2), scale)); +} +template <> +inline int32x2_t recipWrap(const int32x2_t &v2, const float scale) +{ return vcvt_s32_f32(vmul_n_f32(internal::vrecp_f32(vcvt_f32_s32(v2)), scale)); } +template <> +inline uint32x2_t recipWrap(const uint32x2_t &v2, const float scale) +{ return vcvt_u32_f32(vmul_n_f32(internal::vrecp_f32(vcvt_f32_u32(v2)), scale)); } +#endif + +template +void recip(const Size2D &size, + const T * src1Base, ptrdiff_t src1Stride, + T * dstBase, ptrdiff_t dstStride, + f32 scale, + CONVERT_POLICY cpolicy) +{ + internal::assertSupportedConfiguration(); + +#ifdef CAROTENE_NEON + typedef typename internal::VecTraits::vec128 vec128; + typedef typename internal::VecTraits::vec64 vec64; + + if (scale == 0.0f || + (std::numeric_limits::is_integer && + scale < 1.0f && + scale > -1.0f)) + { + for (size_t y = 0; y < size.height; ++y) + { + T * dst = internal::getRowPtr(dstBase, dstStride, y); + std::memset(dst, 0, sizeof(T) * size.width); + } + return; + } + + const size_t step128 = 16 / sizeof(T); + size_t roiw128 = size.width >= (step128 - 1) ? size.width - step128 + 1 : 0; + const size_t step64 = 8 / sizeof(T); + size_t roiw64 = size.width >= (step64 - 1) ? size.width - step64 + 1 : 0; + + for (size_t i = 0; i < size.height; ++i) + { + const T * src1 = internal::getRowPtr(src1Base, src1Stride, i); + T * dst = internal::getRowPtr(dstBase, dstStride, i); + size_t j = 0; + + if (cpolicy == CONVERT_POLICY_SATURATE) + { + for (; j < roiw128; j += step128) + { + internal::prefetch(src1 + j); + + vec128 v_src1 = internal::vld1q(src1 + j); + + vec128 v_mask = vtstq(v_src1,v_src1); + internal::vst1q(dst + j, internal::vandq(v_mask, recipSaturateQ(v_src1, scale))); + } + for (; j < roiw64; j += step64) + { + vec64 v_src1 = internal::vld1(src1 + j); + + vec64 v_mask = vtst(v_src1,v_src1); + internal::vst1(dst + j, internal::vand(v_mask, recipSaturate(v_src1, scale))); + } + for (; j < size.width; j++) + { + dst[j] = src1[j] ? internal::saturate_cast(scale / src1[j]) : 0; + } + } + else // CONVERT_POLICY_WRAP + { + for (; j < roiw128; j += step128) + { + internal::prefetch(src1 + j); + + vec128 v_src1 = internal::vld1q(src1 + j); + + vec128 v_mask = vtstq(v_src1,v_src1); + internal::vst1q(dst + j, internal::vandq(v_mask, recipWrapQ(v_src1, scale))); + } + for (; j < roiw64; j += step64) + { + vec64 v_src1 = internal::vld1(src1 + j); + + vec64 v_mask = vtst(v_src1,v_src1); + internal::vst1(dst + j, internal::vand(v_mask, recipWrap(v_src1, scale))); + } + for (; j < size.width; j++) + { + dst[j] = src1[j] ? (T)((s32)trunc(scale / src1[j])) : 0; + } + } + } +#else + (void)size; + (void)src1Base; + (void)src1Stride; + (void)dstBase; + (void)dstStride; + (void)cpolicy; + (void)scale; +#endif +} + +} + +void div(const Size2D &size, + const u8 * src0Base, ptrdiff_t src0Stride, + const u8 * src1Base, ptrdiff_t src1Stride, + u8 * dstBase, ptrdiff_t dstStride, + f32 scale, + CONVERT_POLICY cpolicy) +{ + div(size, src0Base, src0Stride, src1Base, src1Stride, dstBase, dstStride, scale, cpolicy); +} + +void div(const Size2D &size, + const s8 * src0Base, ptrdiff_t src0Stride, + const s8 * src1Base, ptrdiff_t src1Stride, + s8 * dstBase, ptrdiff_t dstStride, + f32 scale, + CONVERT_POLICY cpolicy) +{ + div(size, src0Base, src0Stride, src1Base, src1Stride, dstBase, dstStride, scale, cpolicy); +} + +void div(const Size2D &size, + const u16 * src0Base, ptrdiff_t src0Stride, + const u16 * src1Base, ptrdiff_t src1Stride, + u16 * dstBase, ptrdiff_t dstStride, + f32 scale, + CONVERT_POLICY cpolicy) +{ + div(size, src0Base, src0Stride, src1Base, src1Stride, dstBase, dstStride, scale, cpolicy); +} + +void div(const Size2D &size, + const s16 * src0Base, ptrdiff_t src0Stride, + const s16 * src1Base, ptrdiff_t src1Stride, + s16 * dstBase, ptrdiff_t dstStride, + f32 scale, + CONVERT_POLICY cpolicy) +{ + div(size, src0Base, src0Stride, src1Base, src1Stride, dstBase, dstStride, scale, cpolicy); +} + +void div(const Size2D &size, + const s32 * src0Base, ptrdiff_t src0Stride, + const s32 * src1Base, ptrdiff_t src1Stride, + s32 * dstBase, ptrdiff_t dstStride, + f32 scale, + CONVERT_POLICY cpolicy) +{ + div(size, src0Base, src0Stride, src1Base, src1Stride, dstBase, dstStride, scale, cpolicy); +} + +void div(const Size2D &size, + const f32 * src0Base, ptrdiff_t src0Stride, + const f32 * src1Base, ptrdiff_t src1Stride, + f32 * dstBase, ptrdiff_t dstStride, + f32 scale) +{ + internal::assertSupportedConfiguration(); +#ifdef CAROTENE_NEON + if (scale == 0.0f) + { + for (size_t y = 0; y < size.height; ++y) + { + f32 * dst = internal::getRowPtr(dstBase, dstStride, y); + std::memset(dst, 0, sizeof(f32) * size.width); + } + return; + } + + float32x4_t v_zero = vdupq_n_f32(0.0f); + + size_t roiw128 = size.width >= 3 ? size.width - 3 : 0; + size_t roiw64 = size.width >= 1 ? size.width - 1 : 0; + + if (std::fabs(scale - 1.0f) < FLT_EPSILON) + { + for (size_t i = 0; i < size.height; ++i) + { + const f32 * src0 = internal::getRowPtr(src0Base, src0Stride, i); + const f32 * src1 = internal::getRowPtr(src1Base, src1Stride, i); + f32 * dst = internal::getRowPtr(dstBase, dstStride, i); + size_t j = 0; + + for (; j < roiw128; j += 4) + { + internal::prefetch(src0 + j); + internal::prefetch(src1 + j); + + float32x4_t v_src0 = vld1q_f32(src0 + j); + float32x4_t v_src1 = vld1q_f32(src1 + j); + + uint32x4_t v_mask = vceqq_f32(v_src1,v_zero); + vst1q_f32(dst + j, vreinterpretq_f32_u32(vbicq_u32( + vreinterpretq_u32_f32(vmulq_f32(v_src0, internal::vrecpq_f32(v_src1))), v_mask))); + } + + for (; j < roiw64; j += 2) + { + float32x2_t v_src0 = vld1_f32(src0 + j); + float32x2_t v_src1 = vld1_f32(src1 + j); + + uint32x2_t v_mask = vceq_f32(v_src1,vget_low_f32(v_zero)); + vst1_f32(dst + j, vreinterpret_f32_u32(vbic_u32( + vreinterpret_u32_f32(vmul_f32(v_src0, internal::vrecp_f32(v_src1))), v_mask))); + } + + for (; j < size.width; j++) + { + dst[j] = src1[j] ? src0[j] / src1[j] : 0.0f; + } + } + } + else + { + for (size_t i = 0; i < size.height; ++i) + { + const f32 * src0 = internal::getRowPtr(src0Base, src0Stride, i); + const f32 * src1 = internal::getRowPtr(src1Base, src1Stride, i); + f32 * dst = internal::getRowPtr(dstBase, dstStride, i); + size_t j = 0; + + for (; j < roiw128; j += 4) + { + internal::prefetch(src0 + j); + internal::prefetch(src1 + j); + + float32x4_t v_src0 = vld1q_f32(src0 + j); + float32x4_t v_src1 = vld1q_f32(src1 + j); + + uint32x4_t v_mask = vceqq_f32(v_src1,v_zero); + vst1q_f32(dst + j, vreinterpretq_f32_u32(vbicq_u32( + vreinterpretq_u32_f32(vmulq_f32(vmulq_n_f32(v_src0, scale), + internal::vrecpq_f32(v_src1))), v_mask))); + } + + for (; j < roiw64; j += 2) + { + float32x2_t v_src0 = vld1_f32(src0 + j); + float32x2_t v_src1 = vld1_f32(src1 + j); + + uint32x2_t v_mask = vceq_f32(v_src1,vget_low_f32(v_zero)); + vst1_f32(dst + j, vreinterpret_f32_u32(vbic_u32( + vreinterpret_u32_f32(vmul_f32(vmul_n_f32(v_src0, scale), + internal::vrecp_f32(v_src1))), v_mask))); + } + + for (; j < size.width; j++) + { + dst[j] = src1[j] ? src0[j] * scale / src1[j] : 0.0f; + } + } + } +#else + (void)size; + (void)src0Base; + (void)src0Stride; + (void)src1Base; + (void)src1Stride; + (void)dstBase; + (void)dstStride; + (void)scale; +#endif +} + +void reciprocal(const Size2D &size, + const u8 * srcBase, ptrdiff_t srcStride, + u8 * dstBase, ptrdiff_t dstStride, + f32 scale, + CONVERT_POLICY cpolicy) +{ + recip(size, srcBase, srcStride, dstBase, dstStride, scale, cpolicy); +} + +void reciprocal(const Size2D &size, + const s8 * srcBase, ptrdiff_t srcStride, + s8 * dstBase, ptrdiff_t dstStride, + f32 scale, + CONVERT_POLICY cpolicy) +{ + recip(size, srcBase, srcStride, dstBase, dstStride, scale, cpolicy); +} + +void reciprocal(const Size2D &size, + const u16 * srcBase, ptrdiff_t srcStride, + u16 * dstBase, ptrdiff_t dstStride, + f32 scale, + CONVERT_POLICY cpolicy) +{ + recip(size, srcBase, srcStride, dstBase, dstStride, scale, cpolicy); +} + +void reciprocal(const Size2D &size, + const s16 * srcBase, ptrdiff_t srcStride, + s16 * dstBase, ptrdiff_t dstStride, + f32 scale, + CONVERT_POLICY cpolicy) +{ + recip(size, srcBase, srcStride, dstBase, dstStride, scale, cpolicy); +} + +void reciprocal(const Size2D &size, + const s32 * srcBase, ptrdiff_t srcStride, + s32 * dstBase, ptrdiff_t dstStride, + f32 scale, + CONVERT_POLICY cpolicy) +{ + recip(size, srcBase, srcStride, dstBase, dstStride, scale, cpolicy); +} + +void reciprocal(const Size2D &size, + const f32 * srcBase, ptrdiff_t srcStride, + f32 * dstBase, ptrdiff_t dstStride, + f32 scale) +{ + internal::assertSupportedConfiguration(); +#ifdef CAROTENE_NEON + if (scale == 0.0f) + { + for (size_t y = 0; y < size.height; ++y) + { + f32 * dst = internal::getRowPtr(dstBase, dstStride, y); + std::memset(dst, 0, sizeof(f32) * size.width); + } + return; + } + + float32x4_t v_zero = vdupq_n_f32(0.0f); + + size_t roiw128 = size.width >= 3 ? size.width - 3 : 0; + size_t roiw64 = size.width >= 1 ? size.width - 1 : 0; + + if (std::fabs(scale - 1.0f) < FLT_EPSILON) + { + for (size_t i = 0; i < size.height; ++i) + { + const f32 * src1 = internal::getRowPtr(srcBase, srcStride, i); + f32 * dst = internal::getRowPtr(dstBase, dstStride, i); + size_t j = 0; + + for (; j < roiw128; j += 4) + { + internal::prefetch(src1 + j); + + float32x4_t v_src1 = vld1q_f32(src1 + j); + + uint32x4_t v_mask = vceqq_f32(v_src1,v_zero); + vst1q_f32(dst + j, vreinterpretq_f32_u32(vbicq_u32( + vreinterpretq_u32_f32(internal::vrecpq_f32(v_src1)), v_mask))); + } + + for (; j < roiw64; j += 2) + { + float32x2_t v_src1 = vld1_f32(src1 + j); + + uint32x2_t v_mask = vceq_f32(v_src1,vget_low_f32(v_zero)); + vst1_f32(dst + j, vreinterpret_f32_u32(vbic_u32( + vreinterpret_u32_f32(internal::vrecp_f32(v_src1)), v_mask))); + } + + for (; j < size.width; j++) + { + dst[j] = src1[j] ? 1.0f / src1[j] : 0; + } + } + } + else + { + for (size_t i = 0; i < size.height; ++i) + { + const f32 * src1 = internal::getRowPtr(srcBase, srcStride, i); + f32 * dst = internal::getRowPtr(dstBase, dstStride, i); + size_t j = 0; + + for (; j < roiw128; j += 4) + { + internal::prefetch(src1 + j); + + float32x4_t v_src1 = vld1q_f32(src1 + j); + + uint32x4_t v_mask = vceqq_f32(v_src1,v_zero); + vst1q_f32(dst + j, vreinterpretq_f32_u32(vbicq_u32( + vreinterpretq_u32_f32(vmulq_n_f32(internal::vrecpq_f32(v_src1), + scale)),v_mask))); + } + + for (; j < roiw64; j += 2) + { + float32x2_t v_src1 = vld1_f32(src1 + j); + + uint32x2_t v_mask = vceq_f32(v_src1,vget_low_f32(v_zero)); + vst1_f32(dst + j, vreinterpret_f32_u32(vbic_u32( + vreinterpret_u32_f32(vmul_n_f32(internal::vrecp_f32(v_src1), + scale)), v_mask))); + } + + for (; j < size.width; j++) + { + dst[j] = src1[j] ? scale / src1[j] : 0; + } + } + } +#else + (void)size; + (void)srcBase; + (void)srcStride; + (void)dstBase; + (void)dstStride; + (void)scale; +#endif +} + +} // namespace CAROTENE_NS diff --git a/3rdparty/carotene/src/dot_product.cpp b/3rdparty/carotene/src/dot_product.cpp new file mode 100644 index 0000000000..1759ea7cd5 --- /dev/null +++ b/3rdparty/carotene/src/dot_product.cpp @@ -0,0 +1,260 @@ +/* + * By downloading, copying, installing or using the software you agree to this license. + * If you do not agree to this license, do not download, install, + * copy or use the software. + * + * + * License Agreement + * For Open Source Computer Vision Library + * (3-clause BSD License) + * + * Copyright (C) 2012-2015, NVIDIA Corporation, all rights reserved. + * Third party copyrights are property of their respective owners. + * + * Redistribution and use in source and binary forms, with or without modification, + * are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * * Neither the names of the copyright holders nor the names of the contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * This software is provided by the copyright holders and contributors "as is" and + * any express or implied warranties, including, but not limited to, the implied + * warranties of merchantability and fitness for a particular purpose are disclaimed. + * In no event shall copyright holders or contributors be liable for any direct, + * indirect, incidental, special, exemplary, or consequential damages + * (including, but not limited to, procurement of substitute goods or services; + * loss of use, data, or profits; or business interruption) however caused + * and on any theory of liability, whether in contract, strict liability, + * or tort (including negligence or otherwise) arising in any way out of + * the use of this software, even if advised of the possibility of such damage. + */ + +#include "common.hpp" + +namespace CAROTENE_NS { + +f64 dotProduct(const Size2D &_size, + const u8 * src0Base, ptrdiff_t src0Stride, + const u8 * src1Base, ptrdiff_t src1Stride) +{ + internal::assertSupportedConfiguration(); +#ifdef CAROTENE_NEON + Size2D size(_size); + if (src0Stride == src1Stride && + src0Stride == (ptrdiff_t)(size.width)) + { + size.width *= size.height; + size.height = 1; + } + +// It is possible to accumulate up to 66051 uchar multiplication results in uint32 without overflow +// We process 16 elements and accumulate two new elements per step. So we could handle 66051/2*16 elements +#define DOT_UINT_BLOCKSIZE 66050*8 + f64 result = 0.0; + for (size_t row = 0; row < size.height; ++row) + { + const u8 * src0 = internal::getRowPtr(src0Base, src0Stride, row); + const u8 * src1 = internal::getRowPtr(src1Base, src1Stride, row); + + size_t i = 0; + uint64x2_t ws = vmovq_n_u64(0); + + while(i + 16 <= size.width) + { + size_t lim = std::min(i + DOT_UINT_BLOCKSIZE, size.width) - 16; + + uint32x4_t s1 = vmovq_n_u32(0); + uint32x4_t s2 = vmovq_n_u32(0); + + for (; i <= lim; i += 16) + { + internal::prefetch(src0 + i); + internal::prefetch(src1 + i); + + uint8x16_t vs1 = vld1q_u8(src0 + i); + uint8x16_t vs2 = vld1q_u8(src1 + i); + + uint16x8_t vdot1 = vmull_u8(vget_low_u8(vs1), vget_low_u8(vs2)); + uint16x8_t vdot2 = vmull_u8(vget_high_u8(vs1), vget_high_u8(vs2)); + + s1 = vpadalq_u16(s1, vdot1); + s2 = vpadalq_u16(s2, vdot2); + } + + ws = vpadalq_u32(ws, s1); + ws = vpadalq_u32(ws, s2); + } + + if(i + 8 <= size.width) + { + uint8x8_t vs1 = vld1_u8(src0 + i); + uint8x8_t vs2 = vld1_u8(src1 + i); + + ws = vpadalq_u32(ws, vpaddlq_u16(vmull_u8(vs1, vs2))); + i += 8; + } + + result += (double)vget_lane_u64(vadd_u64(vget_low_u64(ws), vget_high_u64(ws)), 0); + + for (; i < size.width; ++i) + result += s32(src0[i]) * s32(src1[i]); + } + return result; +#else + (void)_size; + (void)src0Base; + (void)src0Stride; + (void)src1Base; + (void)src1Stride; + + return 0; +#endif +} + +f64 dotProduct(const Size2D &_size, + const s8 * src0Base, ptrdiff_t src0Stride, + const s8 * src1Base, ptrdiff_t src1Stride) +{ + internal::assertSupportedConfiguration(); +#ifdef CAROTENE_NEON + Size2D size(_size); + if (src0Stride == src1Stride && + src0Stride == (ptrdiff_t)(size.width)) + { + size.width *= size.height; + size.height = 1; + } + +// It is possible to accumulate up to 131071 schar multiplication results in sint32 without overflow +// We process 16 elements and accumulate two new elements per step. So we could handle 131071/2*16 elements +#define DOT_INT_BLOCKSIZE 131070*8 + f64 result = 0.0; + for (size_t row = 0; row < size.height; ++row) + { + const s8 * src0 = internal::getRowPtr(src0Base, src0Stride, row); + const s8 * src1 = internal::getRowPtr(src1Base, src1Stride, row); + + size_t i = 0; + int64x2_t ws = vmovq_n_s64(0); + + while(i + 16 <= size.width) + { + size_t lim = std::min(i + DOT_UINT_BLOCKSIZE, size.width) - 16; + + int32x4_t s1 = vmovq_n_s32(0); + int32x4_t s2 = vmovq_n_s32(0); + + for (; i <= lim; i += 16) + { + internal::prefetch(src0 + i); + internal::prefetch(src1 + i); + + int8x16_t vs1 = vld1q_s8(src0 + i); + int8x16_t vs2 = vld1q_s8(src1 + i); + + int16x8_t vdot1 = vmull_s8(vget_low_s8(vs1), vget_low_s8(vs2)); + int16x8_t vdot2 = vmull_s8(vget_high_s8(vs1), vget_high_s8(vs2)); + + s1 = vpadalq_s16(s1, vdot1); + s2 = vpadalq_s16(s2, vdot2); + } + + ws = vpadalq_s32(ws, s1); + ws = vpadalq_s32(ws, s2); + } + + if(i + 8 <= size.width) + { + int8x8_t vs1 = vld1_s8(src0 + i); + int8x8_t vs2 = vld1_s8(src1 + i); + + ws = vpadalq_s32(ws, vpaddlq_s16(vmull_s8(vs1, vs2))); + i += 8; + } + + result += (double)vget_lane_s64(vadd_s64(vget_low_s64(ws), vget_high_s64(ws)), 0); + + for (; i < size.width; ++i) + result += s32(src0[i]) * s32(src1[i]); + } + return result; +#else + (void)_size; + (void)src0Base; + (void)src0Stride; + (void)src1Base; + (void)src1Stride; + + return 0; +#endif +} + +f64 dotProduct(const Size2D &_size, + const f32 * src0Base, ptrdiff_t src0Stride, + const f32 * src1Base, ptrdiff_t src1Stride) +{ + internal::assertSupportedConfiguration(); +#ifdef CAROTENE_NEON + Size2D size(_size); + if (src0Stride == src1Stride && + src0Stride == (ptrdiff_t)(size.width * sizeof(f32))) + { + size.width *= size.height; + size.height = 1; + } + +#define DOT_FLOAT_BLOCKSIZE (1 << 13) + f64 result = 0.0; + for (size_t row = 0; row < size.height; ++row) + { + const f32 * src0 = internal::getRowPtr(src0Base, src0Stride, row); + const f32 * src1 = internal::getRowPtr(src1Base, src1Stride, row); + + size_t i = 0; + while(i + 4 <= size.width) + { + size_t lim = std::min(i + DOT_FLOAT_BLOCKSIZE, size.width) - 4; + float32x4_t v_sum = vdupq_n_f32(0.0f); + + for( ; i <= lim; i += 4 ) + { + internal::prefetch(src0 + i); + internal::prefetch(src1 + i); + v_sum = vmlaq_f32(v_sum, vld1q_f32(src0 + i), vld1q_f32(src1 + i)); + } + + float32x2_t vres = vpadd_f32(vget_low_f32(v_sum),vget_high_f32(v_sum)); + result += vget_lane_f32(vres, 0) + vget_lane_f32(vres, 1); + } + + if(i + 2 <= size.width) + { + float32x2_t vres = vmul_f32(vld1_f32(src0 + i), vld1_f32(src1 + i)); + result += vget_lane_f32(vres, 0) + vget_lane_f32(vres, 1); + i += 2; + } + + for (; i < size.width; ++i) + result += src0[i] * src1[i]; + } + return result; +#else + (void)_size; + (void)src0Base; + (void)src0Stride; + (void)src1Base; + (void)src1Stride; + + return 0; +#endif +} + +} // namespace CAROTENE_NS diff --git a/3rdparty/carotene/src/fast.cpp b/3rdparty/carotene/src/fast.cpp new file mode 100644 index 0000000000..9506c1b6be --- /dev/null +++ b/3rdparty/carotene/src/fast.cpp @@ -0,0 +1,428 @@ +/* + * By downloading, copying, installing or using the software you agree to this license. + * If you do not agree to this license, do not download, install, + * copy or use the software. + * + * + * License Agreement + * For Open Source Computer Vision Library + * (3-clause BSD License) + * + * Copyright (C) 2012-2015, NVIDIA Corporation, all rights reserved. + * Third party copyrights are property of their respective owners. + * + * Redistribution and use in source and binary forms, with or without modification, + * are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * * Neither the names of the copyright holders nor the names of the contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * This software is provided by the copyright holders and contributors "as is" and + * any express or implied warranties, including, but not limited to, the implied + * warranties of merchantability and fitness for a particular purpose are disclaimed. + * In no event shall copyright holders or contributors be liable for any direct, + * indirect, incidental, special, exemplary, or consequential damages + * (including, but not limited to, procurement of substitute goods or services; + * loss of use, data, or profits; or business interruption) however caused + * and on any theory of liability, whether in contract, strict liability, + * or tort (including negligence or otherwise) arising in any way out of + * the use of this software, even if advised of the possibility of such damage. + */ + + +/* This is FAST corner detector, contributed to OpenCV by the author, Edward Rosten. + Below is the original copyright and the references */ + +/* +Copyright (c) 2006, 2008 Edward Rosten +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: + + *Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + *Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + *Neither the name of the University of Cambridge nor the names of + its contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +/* +The references are: + * Machine learning for high-speed corner detection, + E. Rosten and T. Drummond, ECCV 2006 + * Faster and better: A machine learning approach to corner detection + E. Rosten, R. Porter and T. Drummond, PAMI, 2009 +*/ + +#include "common.hpp" + +#include +#include + +namespace CAROTENE_NS { + +#ifdef CAROTENE_NEON +namespace +{ + +void makeOffsets(ptrdiff_t pixel[], ptrdiff_t row_stride) +{ + pixel[0] = 0 + row_stride * 3; + pixel[1] = 1 + row_stride * 3; + pixel[2] = 2 + row_stride * 2; + pixel[3] = 3 + row_stride * 1; + pixel[4] = 3 + row_stride * 0; + pixel[5] = 3 + row_stride * -1; + pixel[6] = 2 + row_stride * -2; + pixel[7] = 1 + row_stride * -3; + pixel[8] = 0 + row_stride * -3; + pixel[9] = -1 + row_stride * -3; + pixel[10] = -2 + row_stride * -2; + pixel[11] = -3 + row_stride * -1; + pixel[12] = -3 + row_stride * 0; + pixel[13] = -3 + row_stride * 1; + pixel[14] = -2 + row_stride * 2; + pixel[15] = -1 + row_stride * 3; +} + +u8 cornerScore(const u8* ptr, const ptrdiff_t pixel[]) +{ + const s32 K = 8, N = 16 + K + 1; + s32 k, v = ptr[0]; + s16 d[(N + 7) & ~7]; + for( k = 0; k < N; k++ ) + d[k] = (s16)(v - ptr[pixel[k]]); + + int16x8_t q0 = vdupq_n_s16((s16)(-1000)); + int16x8_t q1 = vdupq_n_s16((s16)(1000)); + + int16x8_t d0_7 = vld1q_s16(d + 0); + int16x8_t d8_15 = vld1q_s16(d + 8); + int16x8_t d16_23 = vld1q_s16(d + 16); + int16x8_t d24 = vld1q_s16(d + 24); + + //k == 0 + int16x8_t v0k0 = vextq_s16(d0_7, d8_15, 1); + int16x8_t v1k0 = vextq_s16(d0_7, d8_15, 2); + int16x8_t ak0 = vminq_s16(v0k0, v1k0); + int16x8_t bk0 = vmaxq_s16(v0k0, v1k0); + + v0k0 = vextq_s16(d0_7, d8_15, 3); + ak0 = vminq_s16(ak0, v0k0); + bk0 = vmaxq_s16(bk0, v0k0); + + v1k0 = vextq_s16(d0_7, d8_15, 4); + ak0 = vminq_s16(ak0, v1k0); + bk0 = vmaxq_s16(bk0, v1k0); + + v0k0 = vextq_s16(d0_7, d8_15, 5); + ak0 = vminq_s16(ak0, v0k0); + bk0 = vmaxq_s16(bk0, v0k0); + + v1k0 = vextq_s16(d0_7, d8_15, 6); + ak0 = vminq_s16(ak0, v1k0); + bk0 = vmaxq_s16(bk0, v1k0); + + v0k0 = vextq_s16(d0_7, d8_15, 7); + ak0 = vminq_s16(ak0, v0k0); + bk0 = vmaxq_s16(bk0, v0k0); + + ak0 = vminq_s16(ak0, d8_15); + bk0 = vmaxq_s16(bk0, d8_15); + + q0 = vmaxq_s16(q0, vminq_s16(ak0, d0_7)); + q1 = vminq_s16(q1, vmaxq_s16(bk0, d0_7)); + + v1k0 = vextq_s16(d8_15, d16_23, 1); + q0 = vmaxq_s16(q0, vminq_s16(ak0, v1k0)); + q1 = vminq_s16(q1, vmaxq_s16(bk0, v1k0)); + + //k == 8 + int16x8_t v0k8 = v1k0; + int16x8_t v1k8 = vextq_s16(d8_15, d16_23, 2); + int16x8_t ak8 = vminq_s16(v0k8, v1k8); + int16x8_t bk8 = vmaxq_s16(v0k8, v1k8); + + v0k8 = vextq_s16(d8_15, d16_23, 3); + ak8 = vminq_s16(ak8, v0k8); + bk8 = vmaxq_s16(bk8, v0k8); + + v1k8 = vextq_s16(d8_15, d16_23, 4); + ak8 = vminq_s16(ak8, v1k8); + bk8 = vmaxq_s16(bk8, v1k8); + + v0k8 = vextq_s16(d8_15, d16_23, 5); + ak8 = vminq_s16(ak8, v0k8); + bk8 = vmaxq_s16(bk8, v0k8); + + v1k8 = vextq_s16(d8_15, d16_23, 6); + ak8 = vminq_s16(ak8, v1k8); + bk8 = vmaxq_s16(bk8, v1k8); + + v0k8 = vextq_s16(d8_15, d16_23, 7); + ak8 = vminq_s16(ak8, v0k8); + bk8 = vmaxq_s16(bk8, v0k8); + + ak8 = vminq_s16(ak8, d16_23); + bk8 = vmaxq_s16(bk8, d16_23); + + q0 = vmaxq_s16(q0, vminq_s16(ak8, d8_15)); + q1 = vminq_s16(q1, vmaxq_s16(bk8, d8_15)); + + v1k8 = vextq_s16(d16_23, d24, 1); + q0 = vmaxq_s16(q0, vminq_s16(ak8, v1k8)); + q1 = vminq_s16(q1, vmaxq_s16(bk8, v1k8)); + + //fin + int16x8_t q = vmaxq_s16(q0, vsubq_s16(vmovq_n_s16(0), q1)); + int16x4_t q2 = vmax_s16(vget_low_s16(q), vget_high_s16(q)); + int32x4_t q2w = vmovl_s16(q2); + int32x2_t q4 = vmax_s32(vget_low_s32(q2w), vget_high_s32(q2w)); + int32x2_t q8 = vmax_s32(q4, vreinterpret_s32_s64(vshr_n_s64(vreinterpret_s64_s32(q4), 32))); + + return (u8)(vget_lane_s32(q8, 0) - 1); +} + +} //namespace +#endif + +void FAST(const Size2D &size, + u8 *srcBase, ptrdiff_t srcStride, + KeypointStore *keypoints, + u8 threshold, bool nonmax_suppression) +{ + internal::assertSupportedConfiguration(); +#ifdef CAROTENE_NEON + //keypoints.clear(); + + const s32 K = 8, N = 16 + K + 1; + ptrdiff_t i, j, k, pixel[N]; + makeOffsets(pixel, srcStride); + for(k = 16; k < N; k++) + pixel[k] = pixel[k - 16]; + + uint8x16_t delta = vdupq_n_u8(128); + uint8x16_t t = vdupq_n_u8(threshold); + uint8x16_t K16 = vdupq_n_u8((u8)K); + + u8 threshold_tab[512]; + for( i = -255; i <= 255; i++ ) + threshold_tab[i+255] = (u8)(i < -threshold ? 1 : i > threshold ? 2 : 0); + + std::vector _buf((size.width+16)*3*(sizeof(ptrdiff_t) + sizeof(u8)) + 128); + u8* buf[3]; + buf[0] = &_buf[0]; buf[1] = buf[0] + size.width; buf[2] = buf[1] + size.width; + ptrdiff_t* cpbuf[3]; + cpbuf[0] = (ptrdiff_t*)internal::alignPtr(buf[2] + size.width, sizeof(ptrdiff_t)) + 1; + cpbuf[1] = cpbuf[0] + size.width + 1; + cpbuf[2] = cpbuf[1] + size.width + 1; + memset(buf[0], 0, size.width*3); + + for(i = 3; i < (ptrdiff_t)size.height-2; i++) + { + const u8* ptr = internal::getRowPtr(srcBase, srcStride, i) + 3; + u8* curr = buf[(i - 3)%3]; + ptrdiff_t* cornerpos = cpbuf[(i - 3)%3]; + memset(curr, 0, size.width); + ptrdiff_t ncorners = 0; + + if( i < (ptrdiff_t)size.height - 3 ) + { + j = 3; + + for(; j < (ptrdiff_t)size.width - 16 - 3; j += 16, ptr += 16) + { + internal::prefetch(ptr); + internal::prefetch(ptr + pixel[0]); + internal::prefetch(ptr + pixel[2]); + + uint8x16_t v0 = vld1q_u8(ptr); + int8x16_t v1 = vreinterpretq_s8_u8(veorq_u8(vqsubq_u8(v0, t), delta)); + int8x16_t v2 = vreinterpretq_s8_u8(veorq_u8(vqaddq_u8(v0, t), delta)); + + int8x16_t x0 = vreinterpretq_s8_u8(vsubq_u8(vld1q_u8(ptr + pixel[0]), delta)); + int8x16_t x1 = vreinterpretq_s8_u8(vsubq_u8(vld1q_u8(ptr + pixel[4]), delta)); + int8x16_t x2 = vreinterpretq_s8_u8(vsubq_u8(vld1q_u8(ptr + pixel[8]), delta)); + int8x16_t x3 = vreinterpretq_s8_u8(vsubq_u8(vld1q_u8(ptr + pixel[12]), delta)); + + uint8x16_t m0 = vandq_u8(vcgtq_s8(x0, v2), vcgtq_s8(x1, v2)); + uint8x16_t m1 = vandq_u8(vcgtq_s8(v1, x0), vcgtq_s8(v1, x1)); + m0 = vorrq_u8(m0, vandq_u8(vcgtq_s8(x1, v2), vcgtq_s8(x2, v2))); + m1 = vorrq_u8(m1, vandq_u8(vcgtq_s8(v1, x1), vcgtq_s8(v1, x2))); + m0 = vorrq_u8(m0, vandq_u8(vcgtq_s8(x2, v2), vcgtq_s8(x3, v2))); + m1 = vorrq_u8(m1, vandq_u8(vcgtq_s8(v1, x2), vcgtq_s8(v1, x3))); + m0 = vorrq_u8(m0, vandq_u8(vcgtq_s8(x3, v2), vcgtq_s8(x0, v2))); + m1 = vorrq_u8(m1, vandq_u8(vcgtq_s8(v1, x3), vcgtq_s8(v1, x0))); + m0 = vorrq_u8(m0, m1); + + u64 mask[2]; + vst1q_u64(mask, vreinterpretq_u64_u8(m0)); + + if( mask[0] == 0 ) + { + if (mask[1] != 0) + { + j -= 8; + ptr -= 8; + } + continue; + } + + uint8x16_t c0 = vmovq_n_u8(0); + uint8x16_t c1 = vmovq_n_u8(0); + uint8x16_t max0 = vmovq_n_u8(0); + uint8x16_t max1 = vmovq_n_u8(0); + for( k = 0; k < N; k++ ) + { + int8x16_t x = vreinterpretq_s8_u8(veorq_u8(vld1q_u8(ptr + pixel[k]), delta)); + m0 = vcgtq_s8(x, v2); + m1 = vcgtq_s8(v1, x); + + c0 = vandq_u8(vsubq_u8(c0, m0), m0); + c1 = vandq_u8(vsubq_u8(c1, m1), m1); + + max0 = vmaxq_u8(max0, c0); + max1 = vmaxq_u8(max1, c1); + } + + max0 = vmaxq_u8(max0, max1); + u8 m[16]; + vst1q_u8(m, vcgtq_u8(max0, K16)); + + for( k = 0; k < 16; ++k ) + if(m[k]) + { + cornerpos[ncorners++] = j+k; + if(nonmax_suppression) + curr[j+k] = cornerScore(ptr+k, pixel); + } + } + + for( ; j < (s32)size.width - 3; j++, ptr++ ) + { + s32 v = ptr[0]; + const u8* tab = &threshold_tab[0] - v + 255; + s32 d = tab[ptr[pixel[0]]] | tab[ptr[pixel[8]]]; + + if( d == 0 ) + continue; + + d &= tab[ptr[pixel[2]]] | tab[ptr[pixel[10]]]; + d &= tab[ptr[pixel[4]]] | tab[ptr[pixel[12]]]; + d &= tab[ptr[pixel[6]]] | tab[ptr[pixel[14]]]; + + if( d == 0 ) + continue; + + d &= tab[ptr[pixel[1]]] | tab[ptr[pixel[9]]]; + d &= tab[ptr[pixel[3]]] | tab[ptr[pixel[11]]]; + d &= tab[ptr[pixel[5]]] | tab[ptr[pixel[13]]]; + d &= tab[ptr[pixel[7]]] | tab[ptr[pixel[15]]]; + + if( d & 1 ) + { + s32 vt = v - threshold, count = 0; + + for( k = 0; k < N; k++ ) + { + s32 x = ptr[pixel[k]]; + if(x < vt) + { + if( ++count > K ) + { + cornerpos[ncorners++] = j; + if(nonmax_suppression) + curr[j] = cornerScore(ptr, pixel); + break; + } + } + else + count = 0; + } + } + + if( d & 2 ) + { + s32 vt = v + threshold, count = 0; + + for( k = 0; k < N; k++ ) + { + s32 x = ptr[pixel[k]]; + if(x > vt) + { + if( ++count > K ) + { + cornerpos[ncorners++] = j; + if(nonmax_suppression) + curr[j] = cornerScore(ptr, pixel); + break; + } + } + else + count = 0; + } + } + } + } + + cornerpos[-1] = ncorners; + + if( i == 3 ) + continue; + + const u8* prev = buf[(i - 4 + 3)%3]; + const u8* pprev = buf[(i - 5 + 3)%3]; + cornerpos = cpbuf[(i - 4 + 3)%3]; + ncorners = cornerpos[-1]; + + for( k = 0; k < ncorners; k++ ) + { + j = cornerpos[k]; + s32 score = prev[j]; + if( !nonmax_suppression || + (score > prev[j+1] && score > prev[j-1] && + score > pprev[j-1] && score > pprev[j] && score > pprev[j+1] && + score > curr[j-1] && score > curr[j] && score > curr[j+1]) ) + { + keypoints->push((f32)j, (f32)(i-1), 7.f, -1, (f32)score); + } + } + } +#else + (void)size; + (void)srcBase; + (void)srcStride; + (void)keypoints; + (void)threshold; + (void)nonmax_suppression; +#endif +} + +} // namespace CAROTENE_NS diff --git a/3rdparty/carotene/src/fill_minmaxloc.cpp b/3rdparty/carotene/src/fill_minmaxloc.cpp new file mode 100644 index 0000000000..fdf0e35d03 --- /dev/null +++ b/3rdparty/carotene/src/fill_minmaxloc.cpp @@ -0,0 +1,442 @@ +/* + * By downloading, copying, installing or using the software you agree to this license. + * If you do not agree to this license, do not download, install, + * copy or use the software. + * + * + * License Agreement + * For Open Source Computer Vision Library + * (3-clause BSD License) + * + * Copyright (C) 2014, NVIDIA Corporation, all rights reserved. + * Third party copyrights are property of their respective owners. + * + * Redistribution and use in source and binary forms, with or without modification, + * are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * * Neither the names of the copyright holders nor the names of the contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * This software is provided by the copyright holders and contributors "as is" and + * any express or implied warranties, including, but not limited to, the implied + * warranties of merchantability and fitness for a particular purpose are disclaimed. + * In no event shall copyright holders or contributors be liable for any direct, + * indirect, incidental, special, exemplary, or consequential damages + * (including, but not limited to, procurement of substitute goods or services; + * loss of use, data, or profits; or business interruption) however caused + * and on any theory of liability, whether in contract, strict liability, + * or tort (including negligence or otherwise) arising in any way out of + * the use of this software, even if advised of the possibility of such damage. + */ + +#include "common.hpp" + +namespace CAROTENE_NS { + +#ifdef CAROTENE_NEON + +namespace { + +template +void process(const T * src, size_t j0, size_t j1, size_t i, + T minVal, size_t * minLocPtr, s32 & minLocCount, s32 minLocCapacity, + T maxVal, size_t * maxLocPtr, s32 & maxLocCount, s32 maxLocCapacity) +{ + for (size_t j = j0; j < j1; ++j) + { + T val = src[j]; + + if (val == maxVal) + { + if (maxLocCount < maxLocCapacity) + { + maxLocPtr[maxLocCount] = j; + maxLocPtr[maxLocCount + 1] = i; + } + maxLocCount += 2; + } + + if (val == minVal) + { + if (minLocCount < minLocCapacity) + { + minLocPtr[minLocCount] = j; + minLocPtr[minLocCount + 1] = i; + } + minLocCount += 2; + } + } +} + +} // namespace + +#endif + +void fillMinMaxLocs(const Size2D & size, + const u8 * srcBase, ptrdiff_t srcStride, + u8 minVal, size_t * minLocPtr, s32 & minLocCount, s32 minLocCapacity, + u8 maxVal, size_t * maxLocPtr, s32 & maxLocCount, s32 maxLocCapacity) +{ + internal::assertSupportedConfiguration(); +#ifdef CAROTENE_NEON + size_t roiw16 = size.width >= 15 ? size.width - 15 : 0; + size_t roiw8 = size.width >= 7 ? size.width - 7 : 0; + + uint8x16_t v_maxval16 = vdupq_n_u8(maxVal), v_minval16 = vdupq_n_u8(minVal); + uint8x8_t v_maxval8 = vdup_n_u8(maxVal), v_minval8 = vdup_n_u8(minVal); + + u64 mask[2] = { 0ul }; + + minLocCapacity <<= 1; + maxLocCapacity <<= 1; + + for (size_t i = 0; i < size.height; ++i) + { + const u8 * src = internal::getRowPtr(srcBase, srcStride, i); + size_t j = 0; + + for ( ; j < roiw16; j += 16) + { + internal::prefetch(src + j); + uint8x16_t v_src = vld1q_u8(src + j); + + uint8x16_t v_maxmask = vceqq_u8(v_src, v_maxval16); + uint8x16_t v_minmask = vceqq_u8(v_src, v_minval16); + uint8x16_t v_mask = vorrq_u8(v_maxmask, v_minmask); + + vst1q_u8((u8 *)&mask[0], v_mask); + + if (mask[0]) + process(src, j, j + 8, i, + minVal, minLocPtr, minLocCount, minLocCapacity, + maxVal, maxLocPtr, maxLocCount, maxLocCapacity); + if (mask[1]) + process(src, j + 8, j + 16, i, + minVal, minLocPtr, minLocCount, minLocCapacity, + maxVal, maxLocPtr, maxLocCount, maxLocCapacity); + } + for ( ; j < roiw8; j += 8) + { + uint8x8_t v_src = vld1_u8(src + j); + + uint8x8_t v_maxmask = vceq_u8(v_src, v_maxval8); + uint8x8_t v_minmask = vceq_u8(v_src, v_minval8); + uint8x8_t v_mask = vorr_u8(v_maxmask, v_minmask); + + vst1_u8((u8 *)&mask[0], v_mask); + + if (mask[0]) + process(src, j, j + 8, i, + minVal, minLocPtr, minLocCount, minLocCapacity, + maxVal, maxLocPtr, maxLocCount, maxLocCapacity); + } + + process(src, j, size.width, i, + minVal, minLocPtr, minLocCount, minLocCapacity, + maxVal, maxLocPtr, maxLocCount, maxLocCapacity); + } + + minLocCount >>= 1; + maxLocCount >>= 1; +#else + (void)size; + (void)srcBase; + (void)srcStride; + (void)minVal; + (void)minLocPtr; + (void)minLocCount; + (void)minLocCapacity; + (void)maxVal; + (void)maxLocPtr; + (void)maxLocCount; + (void)maxLocCapacity; +#endif +} + +void fillMinMaxLocs(const Size2D & size, + const u16 * srcBase, ptrdiff_t srcStride, + u16 minVal, size_t * minLocPtr, s32 & minLocCount, s32 minLocCapacity, + u16 maxVal, size_t * maxLocPtr, s32 & maxLocCount, s32 maxLocCapacity) +{ + internal::assertSupportedConfiguration(); +#ifdef CAROTENE_NEON + size_t roiw16 = size.width >= 15 ? size.width - 15 : 0; + size_t roiw8 = size.width >= 7 ? size.width - 7 : 0; + + uint16x8_t v_maxval8 = vdupq_n_u16(maxVal), + v_minval8 = vdupq_n_u16(minVal); + u64 mask[2] = { 0ul }; + + minLocCapacity <<= 1; + maxLocCapacity <<= 1; + + for (size_t i = 0; i < size.height; ++i) + { + const u16 * src = internal::getRowPtr(srcBase, srcStride, i); + size_t j = 0; + + for ( ; j < roiw16; j += 16) + { + internal::prefetch(src + j); + uint16x8_t v_src0 = vld1q_u16(src + j), v_src1 = vld1q_u16(src + j + 8); + + uint16x8_t v_mask0 = vorrq_u16(vceqq_u16(v_src0, v_maxval8), vceqq_u16(v_src0, v_minval8)); + uint16x8_t v_mask1 = vorrq_u16(vceqq_u16(v_src1, v_maxval8), vceqq_u16(v_src1, v_minval8)); + + vst1q_u8((u8 *)&mask[0], vcombine_u8(vmovn_u16(v_mask0), vmovn_u16(v_mask1))); + + if (mask[0]) + process(src, j, j + 8, i, + minVal, minLocPtr, minLocCount, minLocCapacity, + maxVal, maxLocPtr, maxLocCount, maxLocCapacity); + if (mask[1]) + process(src, j + 8, j + 16, i, + minVal, minLocPtr, minLocCount, minLocCapacity, + maxVal, maxLocPtr, maxLocCount, maxLocCapacity); + } + for ( ; j < roiw8; j += 8) + { + internal::prefetch(src + j); + uint16x8_t v_src = vld1q_u16(src + j); + + uint16x8_t v_maxmask = vceqq_u16(v_src, v_maxval8); + uint16x8_t v_minmask = vceqq_u16(v_src, v_minval8); + uint16x8_t v_mask = vorrq_u16(v_maxmask, v_minmask); + + vst1_u8((u8 *)&mask[0], vmovn_u16(v_mask)); + + if (mask[0]) + process(src, j, j + 8, i, + minVal, minLocPtr, minLocCount, minLocCapacity, + maxVal, maxLocPtr, maxLocCount, maxLocCapacity); + } + + process(src, j, size.width, i, + minVal, minLocPtr, minLocCount, minLocCapacity, + maxVal, maxLocPtr, maxLocCount, maxLocCapacity); + } + + minLocCount >>= 1; + maxLocCount >>= 1; +#else + (void)size; + (void)srcBase; + (void)srcStride; + (void)minVal; + (void)minLocPtr; + (void)minLocCount; + (void)minLocCapacity; + (void)maxVal; + (void)maxLocPtr; + (void)maxLocCount; + (void)maxLocCapacity; +#endif +} + +void fillMinMaxLocs(const Size2D & size, + const s16 * srcBase, ptrdiff_t srcStride, + s16 minVal, size_t * minLocPtr, s32 & minLocCount, s32 minLocCapacity, + s16 maxVal, size_t * maxLocPtr, s32 & maxLocCount, s32 maxLocCapacity) +{ + internal::assertSupportedConfiguration(); +#ifdef CAROTENE_NEON + size_t roiw16 = size.width >= 15 ? size.width - 15 : 0; + size_t roiw8 = size.width >= 7 ? size.width - 7 : 0; + + int16x8_t v_maxval8 = vdupq_n_s16(maxVal), + v_minval8 = vdupq_n_s16(minVal); + u64 mask[2] = { 0ul }; + + minLocCapacity <<= 1; + maxLocCapacity <<= 1; + + for (size_t i = 0; i < size.height; ++i) + { + const s16 * src = internal::getRowPtr(srcBase, srcStride, i); + size_t j = 0; + + for ( ; j < roiw16; j += 16) + { + internal::prefetch(src + j); + int16x8_t v_src0 = vld1q_s16(src + j), v_src1 = vld1q_s16(src + j + 8); + + uint16x8_t v_mask0 = vorrq_u16(vceqq_s16(v_src0, v_maxval8), vceqq_s16(v_src0, v_minval8)); + uint16x8_t v_mask1 = vorrq_u16(vceqq_s16(v_src1, v_maxval8), vceqq_s16(v_src1, v_minval8)); + + vst1q_u8((u8 *)&mask[0], vcombine_u8(vmovn_u16(v_mask0), vmovn_u16(v_mask1))); + + if (mask[0]) + process(src, j, j + 8, i, + minVal, minLocPtr, minLocCount, minLocCapacity, + maxVal, maxLocPtr, maxLocCount, maxLocCapacity); + if (mask[1]) + process(src, j + 8, j + 16, i, + minVal, minLocPtr, minLocCount, minLocCapacity, + maxVal, maxLocPtr, maxLocCount, maxLocCapacity); + } + for ( ; j < roiw8; j += 8) + { + internal::prefetch(src + j); + int16x8_t v_src = vld1q_s16(src + j); + + uint16x8_t v_maxmask = vceqq_s16(v_src, v_maxval8); + uint16x8_t v_minmask = vceqq_s16(v_src, v_minval8); + uint16x8_t v_mask = vorrq_u16(v_maxmask, v_minmask); + + vst1_u8((u8 *)&mask[0], vmovn_u16(v_mask)); + + if (mask[0]) + process(src, j, j + 8, i, + minVal, minLocPtr, minLocCount, minLocCapacity, + maxVal, maxLocPtr, maxLocCount, maxLocCapacity); + } + + process(src, j, size.width, i, + minVal, minLocPtr, minLocCount, minLocCapacity, + maxVal, maxLocPtr, maxLocCount, maxLocCapacity); + } + + minLocCount >>= 1; + maxLocCount >>= 1; +#else + (void)size; + (void)srcBase; + (void)srcStride; + (void)minVal; + (void)minLocPtr; + (void)minLocCount; + (void)minLocCapacity; + (void)maxVal; + (void)maxLocPtr; + (void)maxLocCount; + (void)maxLocCapacity; +#endif +} + +void fillMinMaxLocs(const Size2D & size, + const s32 * srcBase, ptrdiff_t srcStride, + s32 minVal, size_t * minLocPtr, s32 & minLocCount, s32 minLocCapacity, + s32 maxVal, size_t * maxLocPtr, s32 & maxLocCount, s32 maxLocCapacity) +{ + internal::assertSupportedConfiguration(); +#ifdef CAROTENE_NEON + size_t roiw8 = size.width >= 7 ? size.width - 7 : 0; + + int32x4_t v_maxval4 = vdupq_n_s32(maxVal), + v_minval4 = vdupq_n_s32(minVal); + u64 mask = 0ul; + + minLocCapacity <<= 1; + maxLocCapacity <<= 1; + + for (size_t i = 0; i < size.height; ++i) + { + const s32 * src = internal::getRowPtr(srcBase, srcStride, i); + size_t j = 0; + + for ( ; j < roiw8; j += 8) + { + internal::prefetch(src + j); + int32x4_t v_src0 = vld1q_s32(src + j), v_src1 = vld1q_s32(src + j + 4); + + uint32x4_t v_mask0 = vorrq_u32(vceqq_s32(v_src0, v_maxval4), vceqq_s32(v_src0, v_minval4)); + uint32x4_t v_mask1 = vorrq_u32(vceqq_s32(v_src1, v_maxval4), vceqq_s32(v_src1, v_minval4)); + + vst1_u8((u8 *)&mask, vmovn_u16(vcombine_u16(vmovn_u32(v_mask0), vmovn_u32(v_mask1)))); + + if (mask) + process(src, j, j + 8, i, + minVal, minLocPtr, minLocCount, minLocCapacity, + maxVal, maxLocPtr, maxLocCount, maxLocCapacity); + } + + process(src, j, size.width, i, + minVal, minLocPtr, minLocCount, minLocCapacity, + maxVal, maxLocPtr, maxLocCount, maxLocCapacity); + } + + minLocCount >>= 1; + maxLocCount >>= 1; +#else + (void)size; + (void)srcBase; + (void)srcStride; + (void)minVal; + (void)minLocPtr; + (void)minLocCount; + (void)minLocCapacity; + (void)maxVal; + (void)maxLocPtr; + (void)maxLocCount; + (void)maxLocCapacity; +#endif +} + +void fillMinMaxLocs(const Size2D & size, + const u32 * srcBase, ptrdiff_t srcStride, + u32 minVal, size_t * minLocPtr, s32 & minLocCount, s32 minLocCapacity, + u32 maxVal, size_t * maxLocPtr, s32 & maxLocCount, s32 maxLocCapacity) +{ + internal::assertSupportedConfiguration(); +#ifdef CAROTENE_NEON + size_t roiw8 = size.width >= 7 ? size.width - 7 : 0; + + uint32x4_t v_maxval4 = vdupq_n_u32(maxVal), + v_minval4 = vdupq_n_u32(minVal); + u64 mask = 0ul; + + minLocCapacity <<= 1; + maxLocCapacity <<= 1; + + for (size_t i = 0; i < size.height; ++i) + { + const u32 * src = internal::getRowPtr(srcBase, srcStride, i); + size_t j = 0; + + for ( ; j < roiw8; j += 8) + { + internal::prefetch(src + j); + uint32x4_t v_src0 = vld1q_u32(src + j), v_src1 = vld1q_u32(src + j + 4); + + uint32x4_t v_mask0 = vorrq_u32(vceqq_u32(v_src0, v_maxval4), vceqq_u32(v_src0, v_minval4)); + uint32x4_t v_mask1 = vorrq_u32(vceqq_u32(v_src1, v_maxval4), vceqq_u32(v_src1, v_minval4)); + + vst1_u8((u8 *)&mask, vmovn_u16(vcombine_u16(vmovn_u32(v_mask0), vmovn_u32(v_mask1)))); + + if (mask) + process(src, j, j + 8, i, + minVal, minLocPtr, minLocCount, minLocCapacity, + maxVal, maxLocPtr, maxLocCount, maxLocCapacity); + } + + process(src, j, size.width, i, + minVal, minLocPtr, minLocCount, minLocCapacity, + maxVal, maxLocPtr, maxLocCount, maxLocCapacity); + } + + minLocCount >>= 1; + maxLocCount >>= 1; +#else + (void)size; + (void)srcBase; + (void)srcStride; + (void)minVal; + (void)minLocPtr; + (void)minLocCount; + (void)minLocCapacity; + (void)maxVal; + (void)maxLocPtr; + (void)maxLocCount; + (void)maxLocCapacity; +#endif +} + +} // namespace CAROTENE_NS diff --git a/3rdparty/carotene/src/flip.cpp b/3rdparty/carotene/src/flip.cpp new file mode 100644 index 0000000000..339398dd92 --- /dev/null +++ b/3rdparty/carotene/src/flip.cpp @@ -0,0 +1,222 @@ +/* + * By downloading, copying, installing or using the software you agree to this license. + * If you do not agree to this license, do not download, install, + * copy or use the software. + * + * + * License Agreement + * For Open Source Computer Vision Library + * (3-clause BSD License) + * + * Copyright (C) 2014, NVIDIA Corporation, all rights reserved. + * Third party copyrights are property of their respective owners. + * + * Redistribution and use in source and binary forms, with or without modification, + * are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * * Neither the names of the copyright holders nor the names of the contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * This software is provided by the copyright holders and contributors "as is" and + * any express or implied warranties, including, but not limited to, the implied + * warranties of merchantability and fitness for a particular purpose are disclaimed. + * In no event shall copyright holders or contributors be liable for any direct, + * indirect, incidental, special, exemplary, or consequential damages + * (including, but not limited to, procurement of substitute goods or services; + * loss of use, data, or profits; or business interruption) however caused + * and on any theory of liability, whether in contract, strict liability, + * or tort (including negligence or otherwise) arising in any way out of + * the use of this software, even if advised of the possibility of such damage. + */ + +#include "common.hpp" +#include "vtransform.hpp" + +#include + +namespace CAROTENE_NS { + +bool isFlipSupported(FLIP_MODE flipMode, u32 elemSize) +{ + bool supportedElemSize = (elemSize == 1) || (elemSize == 2) || (elemSize == 3) || (elemSize == 4); + return isSupportedConfiguration() && + ((supportedElemSize && ((flipMode == FLIP_BOTH_MODE) || (flipMode == FLIP_HORIZONTAL_MODE))) || + (flipMode == FLIP_VERTICAL_MODE)); +} + +#ifdef CAROTENE_NEON + +namespace { + +template +void flip(const Size2D & size, + const void * srcBase, ptrdiff_t srcStride, + void * dstBase, ptrdiff_t dstStride, + FLIP_MODE flipMode) +{ + using namespace internal; + + typedef typename VecTraits::vec128 vec128; + typedef typename VecTraits::vec64 vec64; + + u32 step_base = 16 / sizeof(T), step_tail = 8 / sizeof(T); + size_t roiw_base = size.width >= (step_base - 1) ? size.width - step_base + 1 : 0; + size_t roiw_tail = size.width >= (step_tail - 1) ? size.width - step_tail + 1 : 0; + + for (size_t i = 0; i < size.height; ++i) + { + const T * src = getRowPtr((const T *)srcBase, srcStride, i); + T * dst = getRowPtr((T *)dstBase, dstStride, (flipMode & FLIP_VERTICAL_MODE) != 0 ? size.height - i - 1 : i); + size_t js = 0, jd = size.width; + + for (; js < roiw_base; js += step_base, jd -= step_base) + { + prefetch(src + js); + + vec128 v_src = vld1q(src + js); + vec128 v_dst = vrev64q(v_src); + v_dst = vcombine(vget_high(v_dst), vget_low(v_dst)); + vst1q(dst + jd - step_base, v_dst); + } + for (; js < roiw_tail; js += step_tail, jd -= step_tail) + { + vec64 v_src = vld1(src + js); + vst1(dst + jd - step_tail, vrev64(v_src)); + } + + for (--jd; js < size.width; ++js, --jd) + dst[jd] = src[js]; + } +} + +template +void flip3(const Size2D & size, + const void * srcBase, ptrdiff_t srcStride, + void * dstBase, ptrdiff_t dstStride, + FLIP_MODE flipMode) +{ + using namespace internal; + +#ifndef ANDROID + typedef typename VecTraits::vec128 vec128; +#endif + typedef typename VecTraits::vec64 vec64; + +#ifndef ANDROID + u32 step_base = 16 / sizeof(T), step_base3 = step_base * 3; + size_t roiw_base = size.width >= (step_base - 1) ? size.width - step_base + 1 : 0; +#endif + u32 step_tail = 8 / sizeof(T), step_tail3 = step_tail * 3; + size_t roiw_tail = size.width >= (step_tail - 1) ? size.width - step_tail + 1 : 0; + + for (size_t i = 0; i < size.height; ++i) + { + const T * src = getRowPtr((const T *)srcBase, srcStride, i); + T * dst = getRowPtr((T *)dstBase, dstStride, (flipMode & FLIP_VERTICAL_MODE) != 0 ? size.height - i - 1 : i); + size_t j = 0, js = 0, jd = size.width * 3; + +#ifndef ANDROID + for (; j < roiw_base; j += step_base, js += step_base3, jd -= step_base3) + { + prefetch(src + js); + + vec128 v_src = vld3q(src + js), v_dst; + v_src.val[0] = vrev64q(v_src.val[0]); + v_src.val[1] = vrev64q(v_src.val[1]); + v_src.val[2] = vrev64q(v_src.val[2]); + + v_dst.val[0] = vcombine(vget_high(v_src.val[0]), vget_low(v_src.val[0])); + v_dst.val[1] = vcombine(vget_high(v_src.val[1]), vget_low(v_src.val[1])); + v_dst.val[2] = vcombine(vget_high(v_src.val[2]), vget_low(v_src.val[2])); + + vst3q(dst + jd - step_base3, v_dst); + } +#endif // ANDROID + + for (; j < roiw_tail; j += step_tail, js += step_tail3, jd -= step_tail3) + { + vec64 v_src = vld3(src + js), v_dst; + v_dst.val[0] = vrev64(v_src.val[0]); + v_dst.val[1] = vrev64(v_src.val[1]); + v_dst.val[2] = vrev64(v_src.val[2]); + + vst3(dst + jd - step_tail3, v_dst); + } + + for (jd -= 3; j < size.width; ++j, js += 3, jd -= 3) + { + dst[jd] = src[js]; + dst[jd + 1] = src[js + 1]; + dst[jd + 2] = src[js + 2]; + } + } +} + +typedef void (* flipFunc)(const Size2D &size, + const void * srcBase, ptrdiff_t srcStride, + void * dstBase, ptrdiff_t dstStride, + FLIP_MODE flipMode); + +} // namespace + +#endif + +void flip(const Size2D &size, + const u8 * srcBase, ptrdiff_t srcStride, + u8 * dstBase, ptrdiff_t dstStride, + FLIP_MODE flipMode, u32 elemSize) +{ + internal::assertSupportedConfiguration(isFlipSupported(flipMode, elemSize)); +#ifdef CAROTENE_NEON + + if (flipMode == FLIP_VERTICAL_MODE) + { + for (size_t y = 0; y < size.height; ++y) + { + const u8 * src_row = internal::getRowPtr(srcBase, srcStride, y); + u8 * dst_row = internal::getRowPtr(dstBase, dstStride, size.height - y - 1); + + std::memcpy(dst_row, src_row, elemSize * size.width); + } + return; + } + + flipFunc func = NULL; + + if (elemSize == (u32)sizeof(u8)) + func = &flip; + if (elemSize == (u32)sizeof(u16)) + func = &flip; + if (elemSize == (u32)sizeof(u32)) + func = &flip; + if (elemSize == (u32)sizeof(u8) * 3) + func = &flip3; + + if (func == NULL) + return; + + func(size, + srcBase, srcStride, + dstBase, dstStride, + flipMode); + +#else + (void)size; + (void)srcBase; + (void)srcStride; + (void)dstBase; + (void)dstStride; + (void)flipMode; + (void)elemSize; +#endif +} + +} // namespace CAROTENE_NS diff --git a/3rdparty/carotene/src/gaussian_blur.cpp b/3rdparty/carotene/src/gaussian_blur.cpp new file mode 100644 index 0000000000..069373e419 --- /dev/null +++ b/3rdparty/carotene/src/gaussian_blur.cpp @@ -0,0 +1,1059 @@ +/* + * By downloading, copying, installing or using the software you agree to this license. + * If you do not agree to this license, do not download, install, + * copy or use the software. + * + * + * License Agreement + * For Open Source Computer Vision Library + * (3-clause BSD License) + * + * Copyright (C) 2012-2015, NVIDIA Corporation, all rights reserved. + * Third party copyrights are property of their respective owners. + * + * Redistribution and use in source and binary forms, with or without modification, + * are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * * Neither the names of the copyright holders nor the names of the contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * This software is provided by the copyright holders and contributors "as is" and + * any express or implied warranties, including, but not limited to, the implied + * warranties of merchantability and fitness for a particular purpose are disclaimed. + * In no event shall copyright holders or contributors be liable for any direct, + * indirect, incidental, special, exemplary, or consequential damages + * (including, but not limited to, procurement of substitute goods or services; + * loss of use, data, or profits; or business interruption) however caused + * and on any theory of liability, whether in contract, strict liability, + * or tort (including negligence or otherwise) arising in any way out of + * the use of this software, even if advised of the possibility of such damage. + */ + +#include "common.hpp" +#include "saturate_cast.hpp" +#include "separable_filter.hpp" + +namespace CAROTENE_NS { + +bool isGaussianBlur3x3Supported(const Size2D &size, BORDER_MODE border) +{ + return isSupportedConfiguration() && size.width >= 8 && + (border == BORDER_MODE_CONSTANT || + border == BORDER_MODE_REPLICATE); +} + +void gaussianBlur3x3(const Size2D &size, + const u8 * srcBase, ptrdiff_t srcStride, + u8 * dstBase, ptrdiff_t dstStride, + BORDER_MODE border, u8 borderValue) +{ + internal::assertSupportedConfiguration(isGaussianBlur3x3Supported(size, border)); +#ifdef CAROTENE_NEON + const uint16x8_t v_border_x4 = vdupq_n_u16(borderValue << 2); + const uint16x8_t v_zero = vdupq_n_u16(0); + const uint8x8_t v_border = vdup_n_u8(borderValue); + + uint16x8_t tprev = v_zero, tcurr = v_zero, tnext = v_zero; + uint16x8_t t0 = v_zero, t1 = v_zero, t2 = v_zero; + + ptrdiff_t width = (ptrdiff_t)size.width, height = (ptrdiff_t)size.height; + + for (ptrdiff_t y = 0; y < height; ++y) + { + const u8 * srow0 = y == 0 && border == BORDER_MODE_CONSTANT ? NULL : internal::getRowPtr(srcBase, srcStride, std::max(y - 1, 0)); + const u8 * srow1 = internal::getRowPtr(srcBase, srcStride, y); + const u8 * srow2 = y + 1 == height && border == BORDER_MODE_CONSTANT ? NULL : internal::getRowPtr(srcBase, srcStride, std::min(y + 1, height - 1)); + u8 * drow = internal::getRowPtr(dstBase, dstStride, y); + + s16 prevx = 0, currx = 0, nextx = 0; + ptrdiff_t x = 0; + const ptrdiff_t bwidth = y + 2 < height ? width : (width - 8); + + // perform vertical convolution + for ( ; x <= bwidth; x += 8) + { + internal::prefetch(srow0 + x); + internal::prefetch(srow1 + x); + internal::prefetch(srow2 + x); + + uint8x8_t x0 = !srow0 ? v_border : vld1_u8(srow0 + x); + uint8x8_t x1 = vld1_u8(srow1 + x); + uint8x8_t x2 = !srow2 ? v_border : vld1_u8(srow2 + x); + + // calculate values for plain CPU part below if needed + if (x + 8 >= bwidth) + { + ptrdiff_t x3 = x == width ? width - 1 : x; + ptrdiff_t x4 = border == BORDER_MODE_CONSTANT ? x3 - 1 : std::max(x3 - 1, 0); + + if (border == BORDER_MODE_CONSTANT && x4 < 0) + prevx = borderValue; + else + prevx = (srow2 ? srow2[x4] : borderValue) + (srow1[x4] << 1) + (srow0 ? srow0[x4] : borderValue); + + currx = (srow2 ? srow2[x3] : borderValue) + (srow1[x3] << 1) + (srow0 ? srow0[x3] : borderValue); + } + + // make shift + if (x) + { + tprev = tcurr; + tcurr = tnext; + } + + // and calculate next value + tnext = vaddq_u16(vaddl_u8(x0, x2), vshll_n_u8(x1, 1)); + + // make extrapolation for the first elements + if (!x) + { + // make border + if (border == BORDER_MODE_CONSTANT) + tcurr = v_border_x4; + else if (border == BORDER_MODE_REPLICATE) + tcurr = vdupq_n_u16(vgetq_lane_u16(tnext, 0)); + + continue; + } + + // combine 3 "shifted" vectors + t0 = vextq_u16(tprev, tcurr, 7); + t1 = tcurr; + t2 = vextq_u16(tcurr, tnext, 1); + + // and add them + t0 = vqaddq_u16(vshlq_n_u16(t1, 1), vqaddq_u16(t0, t2)); + vst1_u8(drow + x - 8, vshrn_n_u16(t0, 4)); + } + + x -= 8; + if (x == width) + --x; + + for ( ; x < width; ++x) + { + // make extrapolation for the last elements + if (x + 1 >= width) + { + if (border == BORDER_MODE_CONSTANT) + nextx = borderValue << 2; + else if (border == BORDER_MODE_REPLICATE) + nextx = srow2[x] + (srow1[x] << 1) + srow0[x]; + } + else + nextx = (srow2 ? srow2[x + 1] : borderValue) + + (srow1[x + 1] << 1) + + (srow0 ? srow0[x + 1] : borderValue); + + f32 val = (prevx + (currx << 1) + nextx) >> 4; + drow[x] = internal::saturate_cast((s32)val); + + // make shift + prevx = currx; + currx = nextx; + } + } +#else + (void)srcBase; + (void)srcStride; + (void)dstBase; + (void)dstStride; + (void)borderValue; +#endif +} + +bool isGaussianBlur3x3MarginSupported(const Size2D &size, BORDER_MODE border, Margin borderMargin) +{ + return isSeparableFilter3x3Supported(size, border, 0, 0, borderMargin); +} + +void gaussianBlur3x3Margin(const Size2D &size, + const u8 * srcBase, ptrdiff_t srcStride, + u8 * dstBase, ptrdiff_t dstStride, + BORDER_MODE border, u8 borderValue, Margin borderMargin) +{ + internal::assertSupportedConfiguration(isGaussianBlur3x3MarginSupported(size, border, borderMargin)); +#ifdef CAROTENE_NEON + internal::sepFilter3x3::process( + size, srcBase, srcStride, dstBase, dstStride, + 0, 0, border, borderValue, borderMargin); +#else + (void)srcBase; + (void)srcStride; + (void)dstBase; + (void)dstStride; + (void)borderValue; +#endif +} + +bool isGaussianBlur5x5Supported(const Size2D &size, s32 cn, BORDER_MODE border) +{ + return isSupportedConfiguration() && + cn > 0 && cn <= 4 && + size.width >= 8 && size.height >= 2 && + (border == BORDER_MODE_CONSTANT || + border == BORDER_MODE_REFLECT101 || + border == BORDER_MODE_REFLECT || + border == BORDER_MODE_REPLICATE || + border == BORDER_MODE_WRAP); +} + +void gaussianBlur5x5(const Size2D &size, s32 cn, + const u8 * srcBase, ptrdiff_t srcStride, + u8 * dstBase, ptrdiff_t dstStride, + BORDER_MODE borderType, u8 borderValue, Margin borderMargin) +{ + internal::assertSupportedConfiguration(isGaussianBlur5x5Supported(size, cn, borderType)); +#ifdef CAROTENE_NEON + size_t colsn = size.width * cn; + + std::vector _tmp; + u8 *tmp = 0; + if (borderType == BORDER_MODE_CONSTANT) + { + _tmp.assign(colsn + 4*cn, borderValue); + tmp = &_tmp[cn << 1]; + } + + ptrdiff_t idx_l1 = internal::borderInterpolate(-1, size.width, borderType, borderMargin.left, borderMargin.right) * cn; + ptrdiff_t idx_l2 = internal::borderInterpolate(-2, size.width, borderType, borderMargin.left, borderMargin.right) * cn; + ptrdiff_t idx_r1 = internal::borderInterpolate(size.width + 0, size.width, borderType, borderMargin.left, borderMargin.right) * cn; + ptrdiff_t idx_r2 = internal::borderInterpolate(size.width + 1, size.width, borderType, borderMargin.left, borderMargin.right) * cn; + + //1-line buffer + std::vector _buf(cn * (size.width + 4) + 32 / sizeof(u16)); + u16* lane = internal::alignPtr(&_buf[cn << 1], 32); + + if (borderType == BORDER_MODE_CONSTANT) + for (s32 k = 0; k < cn; ++k) + { + lane[-cn+k] = borderValue; + lane[-cn-cn+k] = borderValue; + lane[colsn+k] = borderValue; + lane[colsn+cn+k] = borderValue; + } + + uint8x8_t vc6u8 = vmov_n_u8(6); + uint16x8_t vc6u16 = vmovq_n_u16(6); + uint16x8_t vc4u16 = vmovq_n_u16(4); + + for (size_t i = 0; i < size.height; ++i) + { + u8* dst = internal::getRowPtr(dstBase, dstStride, i); + //vertical convolution + ptrdiff_t idx_rm2 = internal::borderInterpolate(i - 2, size.height, borderType, borderMargin.top, borderMargin.bottom); + ptrdiff_t idx_rm1 = internal::borderInterpolate(i - 1, size.height, borderType, borderMargin.top, borderMargin.bottom); + ptrdiff_t idx_rp1 = internal::borderInterpolate(i + 1, size.height, borderType, borderMargin.top, borderMargin.bottom); + ptrdiff_t idx_rp2 = internal::borderInterpolate(i + 2, size.height, borderType, borderMargin.top, borderMargin.bottom); + + const u8* ln0 = idx_rm2 >= -(ptrdiff_t)borderMargin.top ? internal::getRowPtr(srcBase, srcStride, idx_rm2) : tmp; + const u8* ln1 = idx_rm1 >= -(ptrdiff_t)borderMargin.top ? internal::getRowPtr(srcBase, srcStride, idx_rm1) : tmp; + const u8* ln2 = internal::getRowPtr(srcBase, srcStride, i); + const u8* ln3 = idx_rp1 >= -(ptrdiff_t)borderMargin.top ? internal::getRowPtr(srcBase, srcStride, idx_rp1) : tmp; + const u8* ln4 = idx_rp2 >= -(ptrdiff_t)borderMargin.top ? internal::getRowPtr(srcBase, srcStride, idx_rp2) : tmp; + + size_t x = 0; + for (; x <= colsn - 8; x += 8) + { + internal::prefetch(internal::getRowPtr(ln2 + x, srcStride, x % 5 - 2)); + uint8x8_t v0 = vld1_u8(ln0+x); + uint8x8_t v1 = vld1_u8(ln1+x); + uint8x8_t v2 = vld1_u8(ln2+x); + uint8x8_t v3 = vld1_u8(ln3+x); + uint8x8_t v4 = vld1_u8(ln4+x); + + uint16x8_t v = vaddl_u8(v0, v4); + uint16x8_t v13 = vaddl_u8(v1, v3); + + v = vmlal_u8(v, v2, vc6u8); + v = vmlaq_u16(v, v13, vc4u16); + + vst1q_u16(lane + x, v); + } + for (; x < colsn; ++x) + lane[x] = ln0[x] + ln4[x] + u16(4) * (ln1[x] + ln3[x]) + u16(6) * ln2[x]; + + //left&right borders + if (borderType != BORDER_MODE_CONSTANT) + for (s32 k = 0; k < cn; ++k) + { + lane[-cn+k] = lane[idx_l1 + k]; + lane[-cn-cn+k] = lane[idx_l2 + k]; + + lane[colsn+k] = lane[idx_r1 + k]; + lane[colsn+cn+k] = lane[idx_r2 + k]; + } + + //horizontal convolution + x = 0; + switch(cn) + { + case 1: + for (; x <= colsn - 8; x += 8) + { + internal::prefetch(lane + x); + + uint16x8_t lane0 = vld1q_u16(lane + x - 2); + uint16x8_t lane4 = vld1q_u16(lane + x + 2); + uint16x8_t lane1 = vld1q_u16(lane + x - 1); + uint16x8_t lane3 = vld1q_u16(lane + x + 1); + uint16x8_t lane2 = vld1q_u16(lane + x + 0); + + uint16x8_t ln04 = vaddq_u16(lane0, lane4); + uint16x8_t ln13 = vaddq_u16(lane1, lane3); + + uint16x8_t ln042 = vmlaq_u16(ln04, lane2, vc6u16); + uint16x8_t lsw = vmlaq_u16(ln042, ln13, vc4u16); + + uint8x8_t ls = vrshrn_n_u16(lsw, 8); + + vst1_u8(dst + x, ls); + } + break; + case 2: + for (; x <= colsn - 8*2; x += 8*2) + { + internal::prefetch(lane + x); + + u16* lidx0 = lane + x - 2*2; + u16* lidx1 = lane + x - 1*2; + u16* lidx3 = lane + x + 1*2; + u16* lidx4 = lane + x + 2*2; +#if __GNUC_MINOR__ < 7 + __asm__ __volatile__ ( + "vld2.16 {d0, d2}, [%[in0]]! \n\t" + "vld2.16 {d1, d3}, [%[in0]] \n\t" + "vld2.16 {d8, d10}, [%[in4]]! \n\t" + "vld2.16 {d9, d11}, [%[in4]] \n\t" + "vadd.i16 q0, q4 \n\t" + "vadd.i16 q1, q5 \n\t" + "vld2.16 {d16, d18}, [%[in1]]! \n\t" + "vld2.16 {d17, d19}, [%[in1]] \n\t" + "vld2.16 {d8, d10}, [%[in3]]! \n\t" + "vld2.16 {d9, d11}, [%[in3]] \n\t" + "vadd.i16 q4, q8 \n\t" + "vadd.i16 q5, q9 \n\t" + "vld2.16 {d16, d18}, [%[in2]] \n\t" + "vld2.16 {d17, d19}, [%[in22]] \n\t" + "vmla.i16 q0, q4, %q[c4] \n\t" + "vmla.i16 q1, q5, %q[c4] \n\t" + "vmla.i16 q0, q8, %q[c6] \n\t" + "vmla.i16 q1, q9, %q[c6] \n\t" + "vrshrn.u16 d8, q0, #8 \n\t" + "vrshrn.u16 d9, q1, #8 \n\t" + "vst2.8 {d8-d9}, [%[out]] \n\t" + : [in0] "=r" (lidx0), + [in1] "=r" (lidx1), + [in3] "=r" (lidx3), + [in4] "=r" (lidx4) + : [out] "r" (dst + x), + "0" (lidx0), + "1" (lidx1), + "2" (lidx3), + "3" (lidx4), + [in2] "r" (lane + x), + [in22] "r" (lane + x + 4*2), + [c4] "w" (vc4u16), [c6] "w" (vc6u16) + : "d0","d1","d2","d3","d4","d5","d6","d7","d8","d9","d10","d11","d12","d13","d14","d15","d16","d17","d18","d19","d20","d21","d22","d23" + ); +#else + uint16x8x2_t vLane0 = vld2q_u16(lidx0); + uint16x8x2_t vLane1 = vld2q_u16(lidx1); + uint16x8x2_t vLane2 = vld2q_u16(lane + x); + uint16x8x2_t vLane3 = vld2q_u16(lidx3); + uint16x8x2_t vLane4 = vld2q_u16(lidx4); + + uint16x8_t vSum_0_4 = vaddq_u16(vLane0.val[0], vLane4.val[0]); + uint16x8_t vSum_1_5 = vaddq_u16(vLane0.val[1], vLane4.val[1]); + + uint16x8_t vSum_4_8 = vaddq_u16(vLane1.val[0], vLane3.val[0]); + uint16x8_t vSum_5_9 = vaddq_u16(vLane1.val[1], vLane3.val[1]); + + vSum_0_4 = vmlaq_u16(vSum_0_4, vSum_4_8, vc4u16); + vSum_1_5 = vmlaq_u16(vSum_1_5, vSum_5_9, vc4u16); + vSum_0_4 = vmlaq_u16(vSum_0_4, vLane2.val[0], vc6u16); + vSum_1_5 = vmlaq_u16(vSum_1_5, vLane2.val[1], vc6u16); + + uint8x8x2_t vRes; + vRes.val[0] = vrshrn_n_u16(vSum_0_4, 8); + vRes.val[1] = vrshrn_n_u16(vSum_1_5, 8); + vst2_u8(dst + x, vRes); +#endif + } + break; + case 3: + for (; x <= colsn - 8*3; x += 8*3) + { + internal::prefetch(lane + x); + + u16* lidx0 = lane + x - 2*3; + u16* lidx1 = lane + x - 1*3; + u16* lidx3 = lane + x + 1*3; + u16* lidx4 = lane + x + 2*3; +#if defined(__GNUC__) && defined(__arm__) + __asm__ __volatile__ ( + "vld3.16 {d0, d2, d4}, [%[in0]]! \n\t" + "vld3.16 {d1, d3, d5}, [%[in0]] \n\t" + "vld3.16 {d8, d10, d12}, [%[in4]]! \n\t" + "vld3.16 {d9, d11, d13}, [%[in4]] \n\t" + "vadd.i16 q0, q4 \n\t" + "vadd.i16 q1, q5 \n\t" + "vadd.i16 q2, q6 \n\t" + "vld3.16 {d16, d18, d20}, [%[in1]]! \n\t" + "vld3.16 {d17, d19, d21}, [%[in1]] \n\t" + "vld3.16 {d8, d10, d12}, [%[in3]]! \n\t" + "vld3.16 {d9, d11, d13}, [%[in3]] \n\t" + "vadd.i16 q4, q8 \n\t" + "vadd.i16 q5, q9 \n\t" + "vadd.i16 q6, q10 \n\t" + "vld3.16 {d16, d18, d20}, [%[in2]] \n\t" + "vld3.16 {d17, d19, d21}, [%[in22]] \n\t" + "vmla.i16 q0, q4, %q[c4] \n\t" + "vmla.i16 q1, q5, %q[c4] \n\t" + "vmla.i16 q2, q6, %q[c4] \n\t" + "vmla.i16 q0, q8, %q[c6] \n\t" + "vmla.i16 q1, q9, %q[c6] \n\t" + "vmla.i16 q2, q10, %q[c6] \n\t" + "vrshrn.u16 d8, q0, #8 \n\t" + "vrshrn.u16 d9, q1, #8 \n\t" + "vrshrn.u16 d10, q2, #8 \n\t" + "vst3.8 {d8-d10}, [%[out]] \n\t" + : [in0] "=r" (lidx0), + [in1] "=r" (lidx1), + [in3] "=r" (lidx3), + [in4] "=r" (lidx4) + : [out] "r" (dst + x), + "0" (lidx0), + "1" (lidx1), + "2" (lidx3), + "3" (lidx4), + [in2] "r" (lane + x), + [in22] "r" (lane + x + 4*3), + [c4] "w" (vc4u16), [c6] "w" (vc6u16) + : "d0","d1","d2","d3","d4","d5","d6","d7","d8","d9","d10","d11","d12","d13","d14","d15","d16","d17","d18","d19","d20","d21","d22","d23" + ); +#else + uint16x8x3_t vLane0 = vld3q_u16(lidx0); + uint16x8x3_t vLane1 = vld3q_u16(lidx1); + uint16x8x3_t vLane2 = vld3q_u16(lane + x); + uint16x8x3_t vLane3 = vld3q_u16(lidx3); + uint16x8x3_t vLane4 = vld3q_u16(lidx4); + + uint16x8_t vSum_0_4 = vaddq_u16(vLane0.val[0], vLane4.val[0]); + uint16x8_t vSum_1_5 = vaddq_u16(vLane0.val[1], vLane4.val[1]); + uint16x8_t vSum_2_6 = vaddq_u16(vLane0.val[2], vLane4.val[2]); + + uint16x8_t vSum_3_1 = vaddq_u16(vLane3.val[0], vLane1.val[0]); + uint16x8_t vSum_4_2 = vaddq_u16(vLane3.val[1], vLane1.val[1]); + uint16x8_t vSum_5_6 = vaddq_u16(vLane3.val[2], vLane1.val[2]); + + vSum_0_4 = vmlaq_u16(vSum_0_4, vSum_3_1, vc4u16); + vSum_1_5 = vmlaq_u16(vSum_1_5, vSum_4_2, vc4u16); + vSum_2_6 = vmlaq_u16(vSum_2_6, vSum_5_6, vc4u16); + + vSum_0_4 = vmlaq_u16(vSum_0_4, vLane2.val[0], vc6u16); + vSum_1_5 = vmlaq_u16(vSum_1_5, vLane2.val[1], vc6u16); + vSum_2_6 = vmlaq_u16(vSum_2_6, vLane2.val[2], vc6u16); + + uint8x8x3_t vRes; + vRes.val[0] = vrshrn_n_u16(vSum_0_4, 8); + vRes.val[1] = vrshrn_n_u16(vSum_1_5, 8); + vRes.val[2] = vrshrn_n_u16(vSum_2_6, 8); + + vst3_u8(dst + x, vRes); +#endif + } + break; + case 4: + for (; x <= colsn - 8*4; x += 8*4) + { + internal::prefetch(lane + x); + internal::prefetch(lane + x + 16); + + u16* lidx0 = lane + x - 2*4; + u16* lidx1 = lane + x - 1*4; + u16* lidx3 = lane + x + 1*4; + u16* lidx4 = lane + x + 2*4; +#if defined(__GNUC__) && defined(__arm__) + __asm__ __volatile__ ( + "vld4.16 {d0, d2, d4, d6}, [%[in0]]! \n\t" + "vld4.16 {d1, d3, d5, d7}, [%[in0]] \n\t" + "vld4.16 {d8, d10, d12, d14}, [%[in4]]! \n\t" + "vld4.16 {d9, d11, d13, d15}, [%[in4]] \n\t" + "vadd.i16 q0, q4 \n\t" + "vadd.i16 q1, q5 \n\t" + "vadd.i16 q2, q6 \n\t" + "vadd.i16 q3, q7 \n\t" + "vld4.16 {d16, d18, d20, d22}, [%[in1]]! \n\t" + "vld4.16 {d17, d19, d21, d23}, [%[in1]] \n\t" + "vld4.16 {d8, d10, d12, d14}, [%[in3]]! \n\t" + "vld4.16 {d9, d11, d13, d15}, [%[in3]] \n\t" + "vadd.i16 q4, q8 \n\t" + "vadd.i16 q5, q9 \n\t" + "vadd.i16 q6, q10 \n\t" + "vadd.i16 q7, q11 \n\t" + "vld4.16 {d16, d18, d20, d22}, [%[in2],:256] \n\t" + "vld4.16 {d17, d19, d21, d23}, [%[in22],:256] \n\t" + "vmla.i16 q0, q4, %q[c4] \n\t" + "vmla.i16 q1, q5, %q[c4] \n\t" + "vmla.i16 q2, q6, %q[c4] \n\t" + "vmla.i16 q3, q7, %q[c4] \n\t" + "vmla.i16 q0, q8, %q[c6] \n\t" + "vmla.i16 q1, q9, %q[c6] \n\t" + "vmla.i16 q2, q10, %q[c6] \n\t" + "vmla.i16 q3, q11, %q[c6] \n\t" + "vrshrn.u16 d8, q0, #8 \n\t" + "vrshrn.u16 d9, q1, #8 \n\t" + "vrshrn.u16 d10, q2, #8 \n\t" + "vrshrn.u16 d11, q3, #8 \n\t" + "vst4.8 {d8-d11}, [%[out]] \n\t" + : [in0] "=r" (lidx0), + [in1] "=r" (lidx1), + [in3] "=r" (lidx3), + [in4] "=r" (lidx4) + : [out] "r" (dst + x), + "0" (lidx0), + "1" (lidx1), + "2" (lidx3), + "3" (lidx4), + [in2] "r" (lane + x), + [in22] "r" (lane + x + 4*4), + [c4] "w" (vc4u16), [c6] "w" (vc6u16) + : "d0","d1","d2","d3","d4","d5","d6","d7","d8","d9","d10","d11","d12","d13","d14","d15","d16","d17","d18","d19","d20","d21","d22","d23" + ); +#else + uint16x8x4_t vLane0 = vld4q_u16(lidx0); + uint16x8x4_t vLane2 = vld4q_u16(lidx4); + uint16x8x4_t vLane4 = vld4q_u16(lidx1); + uint16x8x4_t vLane6 = vld4q_u16(lidx3); + uint16x8x4_t vLane8 = vld4q_u16(lane + x); + + uint16x8_t vSum_0_4 = vaddq_u16(vLane0.val[0], vLane2.val[0]); + uint16x8_t vSum_1_5 = vaddq_u16(vLane0.val[1], vLane2.val[1]); + uint16x8_t vSum_2_6 = vaddq_u16(vLane0.val[2], vLane2.val[2]); + uint16x8_t vSum_3_7 = vaddq_u16(vLane0.val[3], vLane2.val[3]); + + uint16x8_t vSum_4_8 = vaddq_u16(vLane4.val[0], vLane6.val[0]); + uint16x8_t vSum_5_9 = vaddq_u16(vLane4.val[1], vLane6.val[1]); + uint16x8_t vSum_6_10 = vaddq_u16(vLane4.val[2], vLane6.val[2]); + uint16x8_t vSum_7_11 = vaddq_u16(vLane4.val[3], vLane6.val[3]); + + vSum_0_4 = vmlaq_u16(vSum_0_4, vSum_4_8, vc4u16); + vSum_1_5 = vmlaq_u16(vSum_1_5, vSum_5_9, vc4u16); + vSum_2_6 = vmlaq_u16(vSum_2_6, vSum_6_10, vc4u16); + vSum_3_7 = vmlaq_u16(vSum_3_7, vSum_7_11, vc4u16); + + vSum_0_4 = vmlaq_u16(vSum_0_4, vLane8.val[0], vc6u16); + vSum_1_5 = vmlaq_u16(vSum_1_5, vLane8.val[1], vc6u16); + vSum_2_6 = vmlaq_u16(vSum_2_6, vLane8.val[2], vc6u16); + vSum_3_7 = vmlaq_u16(vSum_3_7, vLane8.val[3], vc6u16); + + uint8x8x4_t vRes; + vRes.val[0] = vrshrn_n_u16(vSum_0_4, 8); + vRes.val[1] = vrshrn_n_u16(vSum_1_5, 8); + vRes.val[2] = vrshrn_n_u16(vSum_2_6, 8); + vRes.val[3] = vrshrn_n_u16(vSum_3_7, 8); + + vst4_u8(dst + x, vRes); +#endif + } + break; + } + for (s32 h = 0; h < cn; ++h) + { + u16* ln = lane + h; + u8* dt = dst + h; + for (size_t k = x; k < colsn; k += cn) + { + dt[k] = (u8)((ln[k-2*cn] + ln[k+2*cn] + + u16(4) * (ln[k-cn] + ln[k+cn]) + + u16(6) * ln[k] + (1 << 7)) >> 8); + } + } + } +#else + (void)srcBase; + (void)srcStride; + (void)dstBase; + (void)dstStride; + (void)borderValue; + (void)borderMargin; +#endif +} + +void gaussianBlur5x5(const Size2D &size, s32 cn, + const u16 * srcBase, ptrdiff_t srcStride, + u16 * dstBase, ptrdiff_t dstStride, + BORDER_MODE borderType, u16 borderValue, Margin borderMargin) +{ + internal::assertSupportedConfiguration(isGaussianBlur5x5Supported(size, cn, borderType)); +#ifdef CAROTENE_NEON + size_t colsn = size.width * cn; + + std::vector _tmp; + u16 *tmp = 0; + if (borderType == BORDER_MODE_CONSTANT) + { + _tmp.assign(colsn + 4*cn, borderValue); + tmp = &_tmp[cn << 1]; + } + + ptrdiff_t idx_l1 = internal::borderInterpolate(-1, size.width, borderType, borderMargin.left, borderMargin.right) * cn; + ptrdiff_t idx_l2 = internal::borderInterpolate(-2, size.width, borderType, borderMargin.left, borderMargin.right) * cn; + ptrdiff_t idx_r1 = internal::borderInterpolate(size.width + 0, size.width, borderType, borderMargin.left, borderMargin.right) * cn; + ptrdiff_t idx_r2 = internal::borderInterpolate(size.width + 1, size.width, borderType, borderMargin.left, borderMargin.right) * cn; + + //1-line buffer + std::vector _buf(cn * (size.width + 4) + 32 / sizeof(u32)); + u32* lane = internal::alignPtr(&_buf[cn << 1], 32); + + if (borderType == BORDER_MODE_CONSTANT) + for (s32 k = 0; k < cn; ++k) + { + lane[-cn+k] = borderValue; + lane[-cn-cn+k] = borderValue; + lane[colsn+k] = borderValue; + lane[colsn+cn+k] = borderValue; + } + + uint16x4_t vc6u16 = vmov_n_u16(6); + uint32x4_t vc6u32 = vmovq_n_u32(6); + uint32x4_t vc4u32 = vmovq_n_u32(4); + + for (size_t i = 0; i < size.height; ++i) + { + u16* dst = internal::getRowPtr(dstBase, dstStride, i); + //vertical convolution + ptrdiff_t idx_rm2 = internal::borderInterpolate(i - 2, size.height, borderType, borderMargin.top, borderMargin.bottom); + ptrdiff_t idx_rm1 = internal::borderInterpolate(i - 1, size.height, borderType, borderMargin.top, borderMargin.bottom); + ptrdiff_t idx_rp1 = internal::borderInterpolate(i + 1, size.height, borderType, borderMargin.top, borderMargin.bottom); + ptrdiff_t idx_rp2 = internal::borderInterpolate(i + 2, size.height, borderType, borderMargin.top, borderMargin.bottom); + + const u16* ln0 = idx_rm2 >= -(ptrdiff_t)borderMargin.top ? internal::getRowPtr(srcBase, srcStride, idx_rm2) : tmp; + const u16* ln1 = idx_rm1 >= -(ptrdiff_t)borderMargin.top ? internal::getRowPtr(srcBase, srcStride, idx_rm1) : tmp; + const u16* ln2 = internal::getRowPtr(srcBase, srcStride, i); + const u16* ln3 = idx_rp1 >= -(ptrdiff_t)borderMargin.top ? internal::getRowPtr(srcBase, srcStride, idx_rp1) : tmp; + const u16* ln4 = idx_rp2 >= -(ptrdiff_t)borderMargin.top ? internal::getRowPtr(srcBase, srcStride, idx_rp2) : tmp; + + size_t x = 0; + for (; x <= colsn - 4; x += 4) + { + internal::prefetch(internal::getRowPtr(ln2 + x, srcStride, x % 5 - 2)); + uint16x4_t v0 = vld1_u16(ln0+x); + uint16x4_t v1 = vld1_u16(ln1+x); + uint16x4_t v2 = vld1_u16(ln2+x); + uint16x4_t v3 = vld1_u16(ln3+x); + uint16x4_t v4 = vld1_u16(ln4+x); + + uint32x4_t v = vaddl_u16(v0, v4); + uint32x4_t v13 = vaddl_u16(v1, v3); + + v = vmlal_u16(v, v2, vc6u16); + v = vmlaq_u32(v, v13, vc4u32); + + vst1q_u32(lane + x, v); + } + for (; x < colsn; ++x) + lane[x] = ln0[x] + ln4[x] + 4*(ln1[x] + ln3[x]) + 6*ln2[x]; + + //left&right borders + if (borderType != BORDER_MODE_CONSTANT) + for (s32 k = 0; k < cn; ++k) + { + lane[-cn+k] = lane[idx_l1 + k]; + lane[-cn-cn+k] = lane[idx_l2 + k]; + + lane[colsn+k] = lane[idx_r1 + k]; + lane[colsn+cn+k] = lane[idx_r2 + k]; + } + + //horizontal convolution + x = 0; + for (; x <= colsn - 4; x += 4) + { + internal::prefetch(lane + x); + + uint32x4_t lane0 = vld1q_u32(lane + x - 2); + uint32x4_t lane4 = vld1q_u32(lane + x + 2); + uint32x4_t lane1 = vld1q_u32(lane + x - 1); + uint32x4_t lane3 = vld1q_u32(lane + x + 1); + uint32x4_t lane2 = vld1q_u32(lane + x + 0); + + uint32x4_t ln04 = vaddq_u32(lane0, lane4); + uint32x4_t ln13 = vaddq_u32(lane1, lane3); + + uint32x4_t ln042 = vmlaq_u32(ln04, lane2, vc6u32); + uint32x4_t lsw = vmlaq_u32(ln042, ln13, vc4u32); + + uint16x4_t ls = vrshrn_n_u32(lsw, 8); + + vst1_u16(dst + x, ls); + } + for (s32 h = 0; h < cn; ++h) + { + u32* ln = lane + h; + u16* dt = dst + h; + for (size_t k = x; k < colsn; k += cn) + { + dt[k] = (u16)((ln[k-2*cn] + ln[k+2*cn] + 4*(ln[k-cn] + ln[k+cn]) + 6*ln[k] + (1<<7))>>8); + } + } + } +#else + (void)srcBase; + (void)srcStride; + (void)dstBase; + (void)dstStride; + (void)borderValue; + (void)borderMargin; +#endif +} + +void gaussianBlur5x5(const Size2D &size, s32 cn, + const s16 * srcBase, ptrdiff_t srcStride, + s16 * dstBase, ptrdiff_t dstStride, + BORDER_MODE borderType, s16 borderValue, Margin borderMargin) +{ + internal::assertSupportedConfiguration(isGaussianBlur5x5Supported(size, cn, borderType)); +#ifdef CAROTENE_NEON + size_t colsn = size.width * cn; + + std::vector _tmp; + s16 *tmp = 0; + if (borderType == BORDER_MODE_CONSTANT) + { + _tmp.assign(colsn + 4*cn, borderValue); + tmp = &_tmp[cn << 1]; + } + + ptrdiff_t idx_l1 = internal::borderInterpolate(-1, size.width, borderType, borderMargin.left, borderMargin.right) * cn; + ptrdiff_t idx_l2 = internal::borderInterpolate(-2, size.width, borderType, borderMargin.left, borderMargin.right) * cn; + ptrdiff_t idx_r1 = internal::borderInterpolate(size.width + 0, size.width, borderType, borderMargin.left, borderMargin.right) * cn; + ptrdiff_t idx_r2 = internal::borderInterpolate(size.width + 1, size.width, borderType, borderMargin.left, borderMargin.right) * cn; + + //1-line buffer + std::vector _buf(cn * (size.width + 4) + 32 / sizeof(s32)); + s32* lane = internal::alignPtr(&_buf[cn << 1], 32); + + if (borderType == BORDER_MODE_CONSTANT) + for (s32 k = 0; k < cn; ++k) + { + lane[-cn+k] = borderValue; + lane[-cn-cn+k] = borderValue; + lane[colsn+k] = borderValue; + lane[colsn+cn+k] = borderValue; + } + + int16x4_t vc6s16 = vmov_n_s16(6); + int32x4_t vc6s32 = vmovq_n_s32(6); + int32x4_t vc4s32 = vmovq_n_s32(4); + + for (size_t i = 0; i < size.height; ++i) + { + s16* dst = internal::getRowPtr(dstBase, dstStride, i); + //vertical convolution + ptrdiff_t idx_rm2 = internal::borderInterpolate(i - 2, size.height, borderType, borderMargin.top, borderMargin.bottom); + ptrdiff_t idx_rm1 = internal::borderInterpolate(i - 1, size.height, borderType, borderMargin.top, borderMargin.bottom); + ptrdiff_t idx_rp1 = internal::borderInterpolate(i + 1, size.height, borderType, borderMargin.top, borderMargin.bottom); + ptrdiff_t idx_rp2 = internal::borderInterpolate(i + 2, size.height, borderType, borderMargin.top, borderMargin.bottom); + + const s16* ln0 = idx_rm2 >= -(ptrdiff_t)borderMargin.top ? internal::getRowPtr(srcBase, srcStride, idx_rm2) : tmp; + const s16* ln1 = idx_rm1 >= -(ptrdiff_t)borderMargin.top ? internal::getRowPtr(srcBase, srcStride, idx_rm1) : tmp; + const s16* ln2 = internal::getRowPtr(srcBase, srcStride, i); + const s16* ln3 = idx_rp1 >= -(ptrdiff_t)borderMargin.top ? internal::getRowPtr(srcBase, srcStride, idx_rp1) : tmp; + const s16* ln4 = idx_rp2 >= -(ptrdiff_t)borderMargin.top ? internal::getRowPtr(srcBase, srcStride, idx_rp2) : tmp; + + size_t x = 0; + for (; x <= colsn - 4; x += 4) + { + internal::prefetch(internal::getRowPtr(ln2 + x, srcStride, x % 5 - 2)); + int16x4_t v0 = vld1_s16(ln0+x); + int16x4_t v1 = vld1_s16(ln1+x); + int16x4_t v2 = vld1_s16(ln2+x); + int16x4_t v3 = vld1_s16(ln3+x); + int16x4_t v4 = vld1_s16(ln4+x); + + int32x4_t v = vaddl_s16(v0, v4); + int32x4_t v13 = vaddl_s16(v1, v3); + + v = vmlal_s16(v, v2, vc6s16); + v = vmlaq_s32(v, v13, vc4s32); + + vst1q_s32(lane + x, v); + } + for (; x < colsn; ++x) + lane[x] = ln0[x] + ln4[x] + 4*(ln1[x] + ln3[x]) + 6*ln2[x]; + + //left&right borders + if (borderType != BORDER_MODE_CONSTANT) + for (s32 k = 0; k < cn; ++k) + { + lane[-cn+k] = lane[idx_l1 + k]; + lane[-cn-cn+k] = lane[idx_l2 + k]; + + lane[colsn+k] = lane[idx_r1 + k]; + lane[colsn+cn+k] = lane[idx_r2 + k]; + } + + //horizontal convolution + x = 0; + switch(cn) + { + case 1: + case 2: + case 3: + for (; x <= colsn - 4; x += 4) + { + internal::prefetch(lane + x); + + int32x4_t lane0 = vld1q_s32(lane + x - 2); + int32x4_t lane4 = vld1q_s32(lane + x + 2); + int32x4_t lane1 = vld1q_s32(lane + x - 1); + int32x4_t lane3 = vld1q_s32(lane + x + 1); + int32x4_t lane2 = vld1q_s32(lane + x + 0); + + int32x4_t ln04 = vaddq_s32(lane0, lane4); + int32x4_t ln13 = vaddq_s32(lane1, lane3); + + int32x4_t ln042 = vmlaq_s32(ln04, lane2, vc6s32); + int32x4_t lsw = vmlaq_s32(ln042, ln13, vc4s32); + + int16x4_t ls = vrshrn_n_s32(lsw, 8); + + vst1_s16(dst + x, ls); + } + break; + case 4: +/* for (; x <= colsn - 4*4; x += 4*4) + { + internal::prefetch(lane + x); + internal::prefetch(lane + x + 16); + + ptrdiff_t* lidx0 = lane + x - 2*4; + ptrdiff_t* lidx1 = lane + x - 1*4; + ptrdiff_t* lidx3 = lane + x + 1*4; + ptrdiff_t* lidx4 = lane + x + 2*4; + + __asm__ __volatile__ ( + "vld4.32 {d0, d2, d4, d6}, [%[in0]]! \n\t" + "vld4.32 {d1, d3, d5, d7}, [%[in0]] \n\t" + "vld4.32 {d8, d10, d12, d14}, [%[in4]]! \n\t" + "vld4.32 {d9, d11, d13, d15}, [%[in4]] \n\t" + "vadd.i32 q0, q4 \n\t" + "vadd.i32 q1, q5 \n\t" + "vadd.i32 q2, q6 \n\t" + "vadd.i32 q3, q7 \n\t" + "vld4.32 {d16, d18, d20, d22}, [%[in1]]! \n\t" + "vld4.32 {d17, d19, d21, d23}, [%[in1]] \n\t" + "vld4.32 {d8, d10, d12, d14}, [%[in3]]! \n\t" + "vld4.32 {d9, d11, d13, d15}, [%[in3]] \n\t" + "vadd.i32 q4, q8 \n\t" + "vadd.i32 q5, q9 \n\t" + "vadd.i32 q6, q10 \n\t" + "vadd.i32 q7, q11 \n\t" + "vld4.32 {d16, d18, d20, d22}, [%[in2],:256] \n\t" + "vld4.32 {d17, d19, d21, d23}, [%[in22],:256] \n\t" + "vmla.i32 q0, q4, %q[c4] \n\t" + "vmla.i32 q1, q5, %q[c4] \n\t" + "vmla.i32 q2, q6, %q[c4] \n\t" + "vmla.i32 q3, q7, %q[c4] \n\t" + "vmla.i32 q0, q8, %q[c6] \n\t" + "vmla.i32 q1, q9, %q[c6] \n\t" + "vmla.i32 q2, q10, %q[c6] \n\t" + "vmla.i32 q3, q11, %q[c6] \n\t" + "vrshrn.i32 d8, q0, #8 \n\t" + "vrshrn.i32 d9, q1, #8 \n\t" + "vrshrn.i32 d10, q2, #8 \n\t" + "vrshrn.i32 d11, q3, #8 \n\t" + "vst4.16 {d8-d11}, [%[out]] \n\t" + : [in0] "=r" (lidx0), + [in1] "=r" (lidx1), + [in3] "=r" (lidx3), + [in4] "=r" (lidx4) + : [out] "r" (dst + x), + "0" (lidx0), + "1" (lidx1), + "2" (lidx3), + "3" (lidx4), + [in2] "r" (lane + x), + [in22] "r" (lane + x + 4*2), + [c4] "w" (vc4s32), [c6] "w" (vc6s32) + : "d0","d1","d2","d3","d4","d5","d6","d7","d8","d9","d10","d11","d12","d13","d14","d15","d16","d17","d18","d19","d20","d21","d22","d23" + ); +*/ + for (; x <= colsn - 4; x += 4) + { + internal::prefetch(lane + x); + + int32x4_t lane0 = vld1q_s32(lane + x - 2); + int32x4_t lane4 = vld1q_s32(lane + x + 2); + int32x4_t lane1 = vld1q_s32(lane + x - 1); + int32x4_t lane3 = vld1q_s32(lane + x + 1); + int32x4_t lane2 = vld1q_s32(lane + x + 0); + + int32x4_t ln04 = vaddq_s32(lane0, lane4); + int32x4_t ln13 = vaddq_s32(lane1, lane3); + + int32x4_t ln042 = vmlaq_s32(ln04, lane2, vc6s32); + int32x4_t lsw = vmlaq_s32(ln042, ln13, vc4s32); + + int16x4_t ls = vrshrn_n_s32(lsw, 8); + + vst1_s16(dst + x, ls); + } + break; + } + for (s32 h = 0; h < cn; ++h) + { + s32* ln = lane + h; + s16* dt = dst + h; + for (size_t k = x; k < colsn; k += cn) + { + dt[k] = (s16)((ln[k-2*cn] + ln[k+2*cn] + 4*(ln[k-cn] + ln[k+cn]) + 6*ln[k] + (1<<7))>>8); + } + } + } +#else + (void)srcBase; + (void)srcStride; + (void)dstBase; + (void)dstStride; + (void)borderValue; + (void)borderMargin; +#endif +} + +void gaussianBlur5x5(const Size2D &size, s32 cn, + const s32 * srcBase, ptrdiff_t srcStride, + s32 * dstBase, ptrdiff_t dstStride, + BORDER_MODE borderType, s32 borderValue, Margin borderMargin) +{ + internal::assertSupportedConfiguration(isGaussianBlur5x5Supported(size, cn, borderType)); +#ifdef CAROTENE_NEON + size_t colsn = size.width * cn; + + std::vector _tmp; + s32 *tmp = 0; + if (borderType == BORDER_MODE_CONSTANT) + { + _tmp.assign(colsn + 4*cn, borderValue); + tmp = &_tmp[cn << 1]; + } + + ptrdiff_t idx_l1 = internal::borderInterpolate(-1, size.width, borderType, borderMargin.left, borderMargin.right) * cn; + ptrdiff_t idx_l2 = internal::borderInterpolate(-2, size.width, borderType, borderMargin.left, borderMargin.right) * cn; + ptrdiff_t idx_r1 = internal::borderInterpolate(size.width + 0, size.width, borderType, borderMargin.left, borderMargin.right) * cn; + ptrdiff_t idx_r2 = internal::borderInterpolate(size.width + 1, size.width, borderType, borderMargin.left, borderMargin.right) * cn; + + //1-line buffer + std::vector _buf(cn * (size.width + 4) + 32 / sizeof(s32)); + s32* lane = internal::alignPtr(&_buf[cn << 1], 32); + + if (borderType == BORDER_MODE_CONSTANT) + for (s32 k = 0; k < cn; ++k) + { + lane[-cn+k] = borderValue; + lane[-cn-cn+k] = borderValue; + lane[colsn+k] = borderValue; + lane[colsn+cn+k] = borderValue; + } + + int32x4_t vc6s32 = vmovq_n_s32(6); + int32x4_t vc4s32 = vmovq_n_s32(4); + + for (size_t i = 0; i < size.height; ++i) + { + s32* dst = internal::getRowPtr(dstBase, dstStride, i); + //vertical convolution + ptrdiff_t idx_rm2 = internal::borderInterpolate(i - 2, size.height, borderType, borderMargin.top, borderMargin.bottom); + ptrdiff_t idx_rm1 = internal::borderInterpolate(i - 1, size.height, borderType, borderMargin.top, borderMargin.bottom); + ptrdiff_t idx_rp1 = internal::borderInterpolate(i + 1, size.height, borderType, borderMargin.top, borderMargin.bottom); + ptrdiff_t idx_rp2 = internal::borderInterpolate(i + 2, size.height, borderType, borderMargin.top, borderMargin.bottom); + + const s32* ln0 = idx_rm2 >= -(ptrdiff_t)borderMargin.top ? internal::getRowPtr(srcBase, srcStride, idx_rm2) : tmp; + const s32* ln1 = idx_rm1 >= -(ptrdiff_t)borderMargin.top ? internal::getRowPtr(srcBase, srcStride, idx_rm1) : tmp; + const s32* ln2 = internal::getRowPtr(srcBase, srcStride, i); + const s32* ln3 = idx_rp1 >= -(ptrdiff_t)borderMargin.top ? internal::getRowPtr(srcBase, srcStride, idx_rp1) : tmp; + const s32* ln4 = idx_rp2 >= -(ptrdiff_t)borderMargin.top ? internal::getRowPtr(srcBase, srcStride, idx_rp2) : tmp; + + size_t x = 0; + for (; x <= colsn - 4; x += 4) + { + internal::prefetch(internal::getRowPtr(ln2 + x, srcStride, x % 5 - 2)); + int32x4_t v0 = vld1q_s32(ln0+x); + int32x4_t v1 = vld1q_s32(ln1+x); + int32x4_t v2 = vld1q_s32(ln2+x); + int32x4_t v3 = vld1q_s32(ln3+x); + int32x4_t v4 = vld1q_s32(ln4+x); + + int32x4_t v = vaddq_s32(v0, v4); + int32x4_t v13 = vaddq_s32(v1, v3); + + v = vmlaq_s32(v, v2, vc6s32); + v = vmlaq_s32(v, v13, vc4s32); + + vst1q_s32(lane + x, v); + } + for (; x < colsn; ++x) + lane[x] = ln0[x] + ln4[x] + 4*(ln1[x] + ln3[x]) + 6*ln2[x]; + + //left&right borders + if (borderType != BORDER_MODE_CONSTANT) + for (s32 k = 0; k < cn; ++k) + { + lane[-cn+k] = lane[idx_l1 + k]; + lane[-cn-cn+k] = lane[idx_l2 + k]; + + lane[colsn+k] = lane[idx_r1 + k]; + lane[colsn+cn+k] = lane[idx_r2 + k]; + } + + //horizontal convolution + x = 0; + for (; x <= colsn - 4; x += 4) + { + internal::prefetch(lane + x); + + int32x4_t lane0 = vld1q_s32(lane + x - 2); + int32x4_t lane4 = vld1q_s32(lane + x + 2); + int32x4_t lane1 = vld1q_s32(lane + x - 1); + int32x4_t lane3 = vld1q_s32(lane + x + 1); + int32x4_t lane2 = vld1q_s32(lane + x + 0); + + int32x4_t ln04 = vaddq_s32(lane0, lane4); + int32x4_t ln13 = vaddq_s32(lane1, lane3); + + int32x4_t ln042 = vmlaq_s32(ln04, lane2, vc6s32); + int32x4_t lsw = vmlaq_s32(ln042, ln13, vc4s32); + + vst1q_s32(dst + x, lsw); + } + for (s32 h = 0; h < cn; ++h) + { + s32* ln = lane + h; + s32* dt = dst + h; + for (size_t k = x; k < colsn; k += cn) + { + dt[k] = ln[k-2*cn] + ln[k+2*cn] + 4*(ln[k-cn] + ln[k+cn]) + 6*ln[k]; + } + } + } +#else + (void)srcBase; + (void)srcStride; + (void)dstBase; + (void)dstStride; + (void)borderValue; + (void)borderMargin; +#endif +} + +} // namespace CAROTENE_NS diff --git a/3rdparty/carotene/src/in_range.cpp b/3rdparty/carotene/src/in_range.cpp new file mode 100644 index 0000000000..b79a237e39 --- /dev/null +++ b/3rdparty/carotene/src/in_range.cpp @@ -0,0 +1,195 @@ +/* + * By downloading, copying, installing or using the software you agree to this license. + * If you do not agree to this license, do not download, install, + * copy or use the software. + * + * + * License Agreement + * For Open Source Computer Vision Library + * (3-clause BSD License) + * + * Copyright (C) 2012-2015, NVIDIA Corporation, all rights reserved. + * Third party copyrights are property of their respective owners. + * + * Redistribution and use in source and binary forms, with or without modification, + * are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * * Neither the names of the copyright holders nor the names of the contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * This software is provided by the copyright holders and contributors "as is" and + * any express or implied warranties, including, but not limited to, the implied + * warranties of merchantability and fitness for a particular purpose are disclaimed. + * In no event shall copyright holders or contributors be liable for any direct, + * indirect, incidental, special, exemplary, or consequential damages + * (including, but not limited to, procurement of substitute goods or services; + * loss of use, data, or profits; or business interruption) however caused + * and on any theory of liability, whether in contract, strict liability, + * or tort (including negligence or otherwise) arising in any way out of + * the use of this software, even if advised of the possibility of such damage. + */ + +#include "common.hpp" + +#include "vtransform.hpp" + +namespace CAROTENE_NS { + +#ifdef CAROTENE_NEON + +namespace { + +inline void vnst(u8* dst, uint8x16_t v1, uint8x16_t v2) { vst1q_u8(dst, v1); vst1q_u8(dst+16, v2); } +inline void vnst(u8* dst, uint16x8_t v1, uint16x8_t v2) { vst1q_u8(dst, vcombine_u8(vmovn_u16(v1), vmovn_u16(v2))); } +inline void vnst(u8* dst, uint32x4_t v1, uint32x4_t v2) { vst1_u8(dst, vmovn_u16(vcombine_u16(vmovn_u32(v1), vmovn_u32(v2)))); } + +template struct vtail +{ + static inline void inRange(const T *, const T *, const T *, + u8 *, size_t &, size_t) + { + //do nothing since there couldn't be enough data + } +}; +template struct vtail +{ + static inline void inRange(const T * src, const T * rng1, const T * rng2, + u8 * dst, size_t &x, size_t width) + { + typedef typename internal::VecTraits::vec128 vec128; + typedef typename internal::VecTraits::unsign::vec128 uvec128; + //There no more than 15 elements in the tail, so we could handle 8 element vector only once + if( x + 8 < width) + { + vec128 vs = internal::vld1q( src + x); + vec128 vr1 = internal::vld1q(rng1 + x); + vec128 vr2 = internal::vld1q(rng2 + x); + uvec128 vd = internal::vandq(internal::vcgeq(vs, vr1), internal::vcgeq(vr2, vs)); + internal::vst1(dst + x, internal::vmovn(vd)); + x+=8; + } + } +}; +template struct vtail +{ + static inline void inRange(const T * src, const T * rng1, const T * rng2, + u8 * dst, size_t &x, size_t width) + { + typedef typename internal::VecTraits::vec128 vec128; + typedef typename internal::VecTraits::unsign::vec128 uvec128; + typedef typename internal::VecTraits::vec64 vec64; + typedef typename internal::VecTraits::unsign::vec64 uvec64; + //There no more than 31 elements in the tail, so we could handle once 16+8 or 16 or 8 elements + if( x + 16 < width) + { + vec128 vs = internal::vld1q( src + x); + vec128 vr1 = internal::vld1q(rng1 + x); + vec128 vr2 = internal::vld1q(rng2 + x); + uvec128 vd = internal::vandq(internal::vcgeq(vs, vr1), internal::vcgeq(vr2, vs)); + internal::vst1q(dst + x, vd); + x+=16; + } + if( x + 8 < width) + { + vec64 vs = internal::vld1( src + x); + vec64 vr1 = internal::vld1(rng1 + x); + vec64 vr2 = internal::vld1(rng2 + x); + uvec64 vd = internal::vand(internal::vcge(vs, vr1), internal::vcge(vr2, vs)); + internal::vst1(dst + x, vd); + x+=8; + } + } +}; + +template +inline void inRangeCheck(const Size2D &_size, + const T * srcBase, ptrdiff_t srcStride, + const T * rng1Base, ptrdiff_t rng1Stride, + const T * rng2Base, ptrdiff_t rng2Stride, + u8 * dstBase, ptrdiff_t dstStride) +{ + typedef typename internal::VecTraits::vec128 vec128; + typedef typename internal::VecTraits::unsign::vec128 uvec128; + + Size2D size(_size); + if (srcStride == dstStride && + srcStride == rng1Stride && + srcStride == rng2Stride && + srcStride == (ptrdiff_t)(size.width)) + { + size.width *= size.height; + size.height = 1; + } + const size_t width = size.width & ~( 32/sizeof(T) - 1 ); + + for(size_t j = 0; j < size.height; ++j) + { + const T * src = internal::getRowPtr( srcBase, srcStride, j); + const T * rng1 = internal::getRowPtr(rng1Base, rng1Stride, j); + const T * rng2 = internal::getRowPtr(rng2Base, rng2Stride, j); + u8 * dst = internal::getRowPtr( dstBase, dstStride, j); + size_t i = 0; + for( ; i < width; i += 32/sizeof(T) ) + { + internal::prefetch(src + i); + internal::prefetch(rng1 + i); + internal::prefetch(rng2 + i); + + vec128 vs = internal::vld1q( src + i); + vec128 vr1 = internal::vld1q(rng1 + i); + vec128 vr2 = internal::vld1q(rng2 + i); + uvec128 vd1 = internal::vandq(internal::vcgeq(vs, vr1), internal::vcgeq(vr2, vs)); + vs = internal::vld1q( src + i + 16/sizeof(T)); + vr1 = internal::vld1q(rng1 + i + 16/sizeof(T)); + vr2 = internal::vld1q(rng2 + i + 16/sizeof(T)); + uvec128 vd2 = internal::vandq(internal::vcgeq(vs, vr1), internal::vcgeq(vr2, vs)); + vnst(dst + i, vd1, vd2); + } + vtail::inRange(src, rng1, rng2, dst, i, size.width); + for( ; i < size.width; i++ ) + dst[i] = (u8)(-(rng1[i] <= src[i] && src[i] <= rng2[i])); + } +} + +} + +#define INRANGEFUNC(T) \ +void inRange(const Size2D &_size, \ + const T * srcBase, ptrdiff_t srcStride, \ + const T * rng1Base, ptrdiff_t rng1Stride, \ + const T * rng2Base, ptrdiff_t rng2Stride, \ + u8 * dstBase, ptrdiff_t dstStride) \ +{ \ + internal::assertSupportedConfiguration(); \ + inRangeCheck(_size, srcBase, srcStride, \ + rng1Base, rng1Stride, rng2Base, rng2Stride, \ + dstBase, dstStride); \ +} +#else +#define INRANGEFUNC(T) \ +void inRange(const Size2D &, \ + const T *, ptrdiff_t, \ + const T *, ptrdiff_t, \ + const T *, ptrdiff_t, \ + u8 *, ptrdiff_t) \ +{ \ + internal::assertSupportedConfiguration(); \ +} +#endif + +INRANGEFUNC(u8) +INRANGEFUNC(s8) +INRANGEFUNC(u16) +INRANGEFUNC(s16) +INRANGEFUNC(s32) +INRANGEFUNC(f32) + +} // namespace CAROTENE_NS diff --git a/3rdparty/carotene/src/integral.cpp b/3rdparty/carotene/src/integral.cpp new file mode 100644 index 0000000000..56c919500e --- /dev/null +++ b/3rdparty/carotene/src/integral.cpp @@ -0,0 +1,238 @@ +/* + * By downloading, copying, installing or using the software you agree to this license. + * If you do not agree to this license, do not download, install, + * copy or use the software. + * + * + * License Agreement + * For Open Source Computer Vision Library + * (3-clause BSD License) + * + * Copyright (C) 2012-2014, NVIDIA Corporation, all rights reserved. + * Third party copyrights are property of their respective owners. + * + * Redistribution and use in source and binary forms, with or without modification, + * are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * * Neither the names of the copyright holders nor the names of the contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * This software is provided by the copyright holders and contributors "as is" and + * any express or implied warranties, including, but not limited to, the implied + * warranties of merchantability and fitness for a particular purpose are disclaimed. + * In no event shall copyright holders or contributors be liable for any direct, + * indirect, incidental, special, exemplary, or consequential damages + * (including, but not limited to, procurement of substitute goods or services; + * loss of use, data, or profits; or business interruption) however caused + * and on any theory of liability, whether in contract, strict liability, + * or tort (including negligence or otherwise) arising in any way out of + * the use of this software, even if advised of the possibility of such damage. + */ + +#include "common.hpp" + +namespace CAROTENE_NS { + +void integral(const Size2D &size, + const u8 * srcBase, ptrdiff_t srcStride, + u32 * sumBase, ptrdiff_t sumStride) +{ + internal::assertSupportedConfiguration(); +#ifdef CAROTENE_NEON + uint32x4_t v_zero = vmovq_n_u32(0u); + + // the first iteration + const u8 * src = internal::getRowPtr(srcBase, srcStride, 0); + u32 * sum = internal::getRowPtr(sumBase, sumStride, 0); + + uint32x4_t prev = v_zero; + size_t j = 0u; + + for ( ; j + 7 < size.width; j += 8) + { + internal::prefetch(sum + j); + internal::prefetch(src + j); + + uint8x8_t el8shr0 = vld1_u8(src + j); + uint8x8_t el8shr1 = vreinterpret_u8_u64(vshl_n_u64(vreinterpret_u64_u8(el8shr0), 8)); + uint8x8_t el8shr2 = vreinterpret_u8_u64(vshl_n_u64(vreinterpret_u64_u8(el8shr0), 16)); + uint8x8_t el8shr3 = vreinterpret_u8_u64(vshl_n_u64(vreinterpret_u64_u8(el8shr0), 24)); + + uint16x8_t el8shr12 = vaddl_u8(el8shr1, el8shr2); + uint16x8_t el8shr03 = vaddl_u8(el8shr0, el8shr3); + + uint16x8_t el8 = vaddq_u16(el8shr12, el8shr03); + uint16x4_t el4h = vadd_u16(vget_low_u16(el8), vget_high_u16(el8)); + + uint32x4_t vsuml = vaddw_u16(prev, vget_low_u16(el8)); + uint32x4_t vsumh = vaddw_u16(prev, el4h); + + vst1q_u32(sum + j, vsuml); + vst1q_u32(sum + j + 4, vsumh); + + prev = vaddw_u16(prev, vdup_lane_u16(el4h, 3)); + } + + for (u32 v = vgetq_lane_u32(prev, 3); j < size.width; ++j) + sum[j] = (v += src[j]); + + // the others + for (size_t i = 1; i < size.height ; ++i) + { + src = internal::getRowPtr(srcBase, srcStride, i); + u32 * prevSum = internal::getRowPtr(sumBase, sumStride, i - 1); + sum = internal::getRowPtr(sumBase, sumStride, i); + + prev = v_zero; + j = 0u; + + for ( ; j + 7 < size.width; j += 8) + { + internal::prefetch(sum + j); + internal::prefetch(src + j); + + uint32x4_t vsuml = vld1q_u32(prevSum + j); + uint32x4_t vsumh = vld1q_u32(prevSum + j + 4); + + uint8x8_t el8shr0 = vld1_u8(src + j); + uint8x8_t el8shr1 = vreinterpret_u8_u64(vshl_n_u64(vreinterpret_u64_u8(el8shr0), 8)); + uint8x8_t el8shr2 = vreinterpret_u8_u64(vshl_n_u64(vreinterpret_u64_u8(el8shr0), 16)); + uint8x8_t el8shr3 = vreinterpret_u8_u64(vshl_n_u64(vreinterpret_u64_u8(el8shr0), 24)); + + vsuml = vaddq_u32(vsuml, prev); + vsumh = vaddq_u32(vsumh, prev); + + uint16x8_t el8shr12 = vaddl_u8(el8shr1, el8shr2); + uint16x8_t el8shr03 = vaddl_u8(el8shr0, el8shr3); + + uint16x8_t el8 = vaddq_u16(el8shr12, el8shr03); + uint16x4_t el4h = vadd_u16(vget_low_u16(el8), vget_high_u16(el8)); + + vsuml = vaddw_u16(vsuml, vget_low_u16(el8)); + vsumh = vaddw_u16(vsumh, el4h); + + vst1q_u32(sum + j, vsuml); + vst1q_u32(sum + j + 4, vsumh); + + prev = vaddw_u16(prev, vdup_lane_u16(el4h, 3)); + } + + for (u32 v = vgetq_lane_u32(prev, 3); j < size.width; ++j) + sum[j] = (v += src[j]) + prevSum[j]; + } +#else + (void)size; + (void)srcBase; + (void)srcStride; + (void)sumBase; + (void)sumStride; +#endif +} + +void sqrIntegral(const Size2D &size, + const u8 * srcBase, ptrdiff_t srcStride, + f64 * sqsumBase, ptrdiff_t sqsumStride) +{ + internal::assertSupportedConfiguration(); +#ifdef CAROTENE_NEON + uint16x8_t v_zero8 = vmovq_n_u16(0u); + + // the first iteration + const u8 * src = internal::getRowPtr(srcBase, srcStride, 0); + f64 * sqsum = internal::getRowPtr(sqsumBase, sqsumStride, 0); + + double prev = 0.; + size_t j = 0u; + + for ( ; j + 7 < size.width; j += 8) + { + internal::prefetch(sqsum + j); + internal::prefetch(src + j); + + uint8x8_t vsrc = vld1_u8(src + j); + + uint16x8_t el8shr0 = vmull_u8(vsrc, vsrc); + uint16x8_t el8shr1 = vextq_u16(v_zero8, el8shr0, 7); + + uint32x4_t el8shr01l = vaddl_u16(vget_low_u16(el8shr0), vget_low_u16(el8shr1)); + uint32x4_t el8shr01h = vaddl_u16(vget_high_u16(el8shr0), vget_high_u16(el8shr1)); + + uint32x4_t el4h = vaddq_u32(el8shr01l, el8shr01h); + + uint32x2_t el2l = vadd_u32(vget_low_u32(el8shr01l), vget_high_u32(el8shr01l)); + uint32x2_t el2hl = vadd_u32(vget_low_u32(el4h), vget_high_u32(el8shr01l)); + uint32x2_t el2hh = vadd_u32(vget_low_u32(el4h), vget_high_u32(el4h)); + + u32 buf[8]; + vst1_u32(buf, vget_low_u32(el8shr01l)); + vst1_u32(buf+2, el2l); + vst1_u32(buf+4, el2hl); + vst1_u32(buf+6, el2hh); + for(u32 k=0; k < 8; k++) + sqsum[j+k] = prev + buf[k]; + prev += buf[7]; + } + + for (; j < size.width; ++j) + sqsum[j] = (prev += src[j]*src[j]); + + // the others + for (size_t i = 1; i < size.height ; ++i) + { + src = internal::getRowPtr(srcBase, srcStride, i); + f64 * prevSqSum = internal::getRowPtr(sqsumBase, sqsumStride, i - 1); + sqsum = internal::getRowPtr(sqsumBase, sqsumStride, i); + + prev = 0.; + j = 0u; + + for ( ; j + 7 < size.width; j += 8) + { + internal::prefetch(sqsum + j); + internal::prefetch(src + j); + + uint8x8_t vsrc = vld1_u8(src + j); + + uint16x8_t el8shr0 = vmull_u8(vsrc, vsrc); + uint16x8_t el8shr1 = vextq_u16(v_zero8, el8shr0, 7); + + uint32x4_t el8shr01l = vaddl_u16(vget_low_u16(el8shr0), vget_low_u16(el8shr1)); + uint32x4_t el8shr01h = vaddl_u16(vget_high_u16(el8shr0), vget_high_u16(el8shr1)); + + uint32x4_t el4h = vaddq_u32(el8shr01l, el8shr01h); + + uint32x2_t el2l = vadd_u32(vget_low_u32(el8shr01l), vget_high_u32(el8shr01l)); + uint32x2_t el2hl = vadd_u32(vget_low_u32(el4h), vget_high_u32(el8shr01l)); + uint32x2_t el2hh = vadd_u32(vget_low_u32(el4h), vget_high_u32(el4h)); + + u32 buf[8]; + vst1_u32(buf, vget_low_u32(el8shr01l)); + vst1_u32(buf+2, el2l); + vst1_u32(buf+4, el2hl); + vst1_u32(buf+6, el2hh); + for(u32 k=0; k < 8; k++) + sqsum[j+k] = prev + prevSqSum[j+k] + buf[k]; + prev += buf[7]; + } + + for (; j < size.width; ++j) + sqsum[j] = (prev += src[j]*src[j]) + prevSqSum[j]; + } +#else + (void)size; + (void)srcBase; + (void)srcStride; + (void)sqsumBase; + (void)sqsumStride; +#endif +} + +} // namespace CAROTENE_NS diff --git a/3rdparty/carotene/src/intrinsics.hpp b/3rdparty/carotene/src/intrinsics.hpp new file mode 100644 index 0000000000..062a3f897b --- /dev/null +++ b/3rdparty/carotene/src/intrinsics.hpp @@ -0,0 +1,112 @@ +/* + * By downloading, copying, installing or using the software you agree to this license. + * If you do not agree to this license, do not download, install, + * copy or use the software. + * + * + * License Agreement + * For Open Source Computer Vision Library + * (3-clause BSD License) + * + * Copyright (C) 2014, NVIDIA Corporation, all rights reserved. + * Third party copyrights are property of their respective owners. + * + * Redistribution and use in source and binary forms, with or without modification, + * are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * * Neither the names of the copyright holders nor the names of the contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * This software is provided by the copyright holders and contributors "as is" and + * any express or implied warranties, including, but not limited to, the implied + * warranties of merchantability and fitness for a particular purpose are disclaimed. + * In no event shall copyright holders or contributors be liable for any direct, + * indirect, incidental, special, exemplary, or consequential damages + * (including, but not limited to, procurement of substitute goods or services; + * loss of use, data, or profits; or business interruption) however caused + * and on any theory of liability, whether in contract, strict liability, + * or tort (including negligence or otherwise) arising in any way out of + * the use of this software, even if advised of the possibility of such damage. + */ + +#ifndef CAROTENE_INTRINSICS_HPP +#define CAROTENE_INTRINSICS_HPP + +#include + +#include + +namespace CAROTENE_NS { namespace internal { + +/////////////// Custom NEON intrinsics /////////////////// + +// calculate reciprocal value + +inline float32x4_t vrecpq_f32(float32x4_t val) +{ + float32x4_t reciprocal = vrecpeq_f32(val); + reciprocal = vmulq_f32(vrecpsq_f32(val, reciprocal), reciprocal); + reciprocal = vmulq_f32(vrecpsq_f32(val, reciprocal), reciprocal); + return reciprocal; +} + +inline float32x2_t vrecp_f32(float32x2_t val) +{ + float32x2_t reciprocal = vrecpe_f32(val); + reciprocal = vmul_f32(vrecps_f32(val, reciprocal), reciprocal); + reciprocal = vmul_f32(vrecps_f32(val, reciprocal), reciprocal); + return reciprocal; +} + +// caclulate sqrt value + +inline float32x4_t vrsqrtq_f32(float32x4_t val) +{ + float32x4_t e = vrsqrteq_f32(val); + e = vmulq_f32(vrsqrtsq_f32(vmulq_f32(e, e), val), e); + e = vmulq_f32(vrsqrtsq_f32(vmulq_f32(e, e), val), e); + return e; +} + +inline float32x2_t vrsqrt_f32(float32x2_t val) +{ + float32x2_t e = vrsqrte_f32(val); + e = vmul_f32(vrsqrts_f32(vmul_f32(e, e), val), e); + e = vmul_f32(vrsqrts_f32(vmul_f32(e, e), val), e); + return e; +} + +inline float32x4_t vsqrtq_f32(float32x4_t val) +{ + return vrecpq_f32(vrsqrtq_f32(val)); +} + +inline float32x2_t vsqrt_f32(float32x2_t val) +{ + return vrecp_f32(vrsqrt_f32(val)); +} + +// table lookup with the table in a 128-bit register + +inline uint8x8_t vqtbl1_u8 (uint8x16_t a, uint8x8_t b) +{ +#ifdef __aarch64__ + // AArch64 supports this natively + return ::vqtbl1_u8(a, b); +#else + union { uint8x16_t v; uint8x8x2_t w; } u = { a }; + return vtbl2_u8(u.w, b); +#endif +} + +} } + +#endif diff --git a/3rdparty/carotene/src/laplacian.cpp b/3rdparty/carotene/src/laplacian.cpp new file mode 100644 index 0000000000..b9148de1b4 --- /dev/null +++ b/3rdparty/carotene/src/laplacian.cpp @@ -0,0 +1,713 @@ +/* + * By downloading, copying, installing or using the software you agree to this license. + * If you do not agree to this license, do not download, install, + * copy or use the software. + * + * + * License Agreement + * For Open Source Computer Vision Library + * (3-clause BSD License) + * + * Copyright (C) 2015, NVIDIA Corporation, all rights reserved. + * Third party copyrights are property of their respective owners. + * + * Redistribution and use in source and binary forms, with or without modification, + * are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * * Neither the names of the copyright holders nor the names of the contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * This software is provided by the copyright holders and contributors "as is" and + * any express or implied warranties, including, but not limited to, the implied + * warranties of merchantability and fitness for a particular purpose are disclaimed. + * In no event shall copyright holders or contributors be liable for any direct, + * indirect, incidental, special, exemplary, or consequential damages + * (including, but not limited to, procurement of substitute goods or services; + * loss of use, data, or profits; or business interruption) however caused + * and on any theory of liability, whether in contract, strict liability, + * or tort (including negligence or otherwise) arising in any way out of + * the use of this software, even if advised of the possibility of such damage. + */ + +#include "common.hpp" +#include "saturate_cast.hpp" + +#include + +namespace CAROTENE_NS { + +bool isLaplacian3x3Supported(const Size2D &size, BORDER_MODE border) +{ + return isSupportedConfiguration() && size.width >= 8 && + (border == BORDER_MODE_CONSTANT || + border == BORDER_MODE_REPLICATE); +} + +void Laplacian3x3(const Size2D &size, + const u8 * srcBase, ptrdiff_t srcStride, + u8 * dstBase, ptrdiff_t dstStride, + BORDER_MODE border, u8 borderValue) +{ + internal::assertSupportedConfiguration(isLaplacian3x3Supported(size, border)); +#ifdef CAROTENE_NEON + const uint16x8_t v_border_x3 = vdupq_n_u16(borderValue * 3); + const uint16x8_t v_zero = vdupq_n_u16(0); + const uint8x8_t v_border = vdup_n_u8(borderValue); + + uint8x8_t vsub; + uint16x8_t tprev = v_zero, tcurr = v_zero, tnext = v_zero; + uint16x8_t t0 = v_zero, t1 = v_zero, t2 = v_zero; + + ptrdiff_t width = (ptrdiff_t)size.width, height = (ptrdiff_t)size.height; + + for (ptrdiff_t y = 0; y < height; ++y) + { + const u8 * srow0 = y == 0 && border == BORDER_MODE_CONSTANT ? NULL : internal::getRowPtr(srcBase, srcStride, std::max(y - 1, 0)); + const u8 * srow1 = internal::getRowPtr(srcBase, srcStride, y); + const u8 * srow2 = y + 1 == height && border == BORDER_MODE_CONSTANT ? NULL : internal::getRowPtr(srcBase, srcStride, std::min(y + 1, height - 1)); + u8 * drow = internal::getRowPtr(dstBase, dstStride, y); + + s16 prevx = 0, currx = 0, nextx = 0; + ptrdiff_t x = 0; + const ptrdiff_t bwidth = y + 2 < height ? width : (width - 8); + + // perform vertical convolution + for ( ; x <= bwidth; x += 8) + { + internal::prefetch(srow0 + x); + internal::prefetch(srow1 + x); + internal::prefetch(srow2 + x); + + uint8x8_t x0 = !srow0 ? v_border : vld1_u8(srow0 + x); + uint8x8_t x1 = vld1_u8(srow1 + x); + uint8x8_t x2 = !srow2 ? v_border : vld1_u8(srow2 + x); + + // calculate values for plain CPU part below if needed + if (x + 8 >= bwidth) + { + ptrdiff_t x3 = x == width ? width - 1 : x; + ptrdiff_t x4 = border == BORDER_MODE_CONSTANT ? x3 - 1 : std::max(x3 - 1, 0); + + if (border == BORDER_MODE_CONSTANT && x4 < 0) + prevx = borderValue; + else + prevx = (srow2 ? srow2[x4] : borderValue) + srow1[x4] + (srow0 ? srow0[x4] : borderValue); + + currx = (srow2 ? srow2[x3] : borderValue) + srow1[x3] + (srow0 ? srow0[x3] : borderValue); + } + + // make shift + if (x) + { + tprev = tcurr; + tcurr = tnext; + } + + // and calculate next value + tnext = vaddw_u8(vaddl_u8(x0, x1), x2); + + // make extrapolation for the first elements + if (!x) + { + // make border + if (border == BORDER_MODE_CONSTANT) + tcurr = v_border_x3; + else if (border == BORDER_MODE_REPLICATE) + tcurr = vdupq_n_u16(vgetq_lane_u16(tnext, 0)); + + vsub = x1; + + continue; + } + + // combine 3 "shifted" vectors + t0 = vextq_u16(tprev, tcurr, 7); + t1 = tcurr; + t2 = vextq_u16(tcurr, tnext, 1); + + // and add them + t0 = vqaddq_u16(t0, vqaddq_u16(t1, t2)); + + int16x8_t tt0 = vsubq_s16(vreinterpretq_s16_u16(t0), + vreinterpretq_s16_u16(vaddw_u8(vshll_n_u8(vsub, 3), vsub))); + uint8x8_t it0 = vqmovun_s16(tt0); + vst1_u8(drow + x - 8, it0); + + vsub = x1; + } + + x -= 8; + if (x == width) + --x; + + for ( ; x < width; ++x) + { + // make extrapolation for the last elements + if (x + 1 >= width) + { + if (border == BORDER_MODE_CONSTANT) + nextx = borderValue * 3; + else if (border == BORDER_MODE_REPLICATE) + nextx = srow2[x] + srow1[x] + srow0[x]; + } + else + { + nextx = (srow2 ? srow2[x + 1] : borderValue) + + srow1[x + 1] + + (srow0 ? srow0[x + 1] : borderValue); + } + + s32 val = (prevx + currx + nextx) - 9 * srow1[x]; + drow[x] = internal::saturate_cast((s32)val); + + // make shift + prevx = currx; + currx = nextx; + } + } +#else + (void)size; + (void)srcBase; + (void)srcStride; + (void)dstBase; + (void)dstStride; + (void)border; + (void)borderValue; +#endif +} + +bool isLaplacianOpenCVSupported(const Size2D &size, BORDER_MODE border) +{ + return isSupportedConfiguration() && + size.width >= 8 && size.height >= 1 && + (border == BORDER_MODE_CONSTANT || + border == BORDER_MODE_REFLECT || + border == BORDER_MODE_REFLECT101 || + border == BORDER_MODE_REPLICATE); +} + +void Laplacian1OpenCV(const Size2D &size, + const u8 * srcBase, ptrdiff_t srcStride, + s16 * dstBase, ptrdiff_t dstStride, + BORDER_MODE border, u8 borderValue) +{ + internal::assertSupportedConfiguration(isLaplacianOpenCVSupported(size, border)); +#ifdef CAROTENE_NEON + ptrdiff_t rows = size.height, cols = size.width; + + std::vector _tmp; + u8 *tmp = 0; + if (border == BORDER_MODE_CONSTANT) + { + _tmp.assign(cols + 4,borderValue); + tmp = &_tmp[2]; + } + + for( ptrdiff_t y = 0; y < rows; y++ ) + { + const u8* v0 = 0; + const u8* v1 = internal::getRowPtr(srcBase, srcStride, y); + const u8* v2 = 0; + // make border + if (border == BORDER_MODE_REFLECT101) { + v0 = internal::getRowPtr(srcBase, srcStride, y > 0 ? y-1 : y+1); + v2 = internal::getRowPtr(srcBase, srcStride, y < rows-1 ? y+1 : rows > 1 ? rows-2 : 0); + } else if (border == BORDER_MODE_CONSTANT) { + v0 = y > 0 ? internal::getRowPtr(srcBase, srcStride, y-1) : tmp; + v2 = y < rows-1 ? internal::getRowPtr(srcBase, srcStride, y+1) : tmp; + } else { + v0 = internal::getRowPtr(srcBase, srcStride, y > 0 ? y-1 : 0); + v2 = internal::getRowPtr(srcBase, srcStride, y < rows-1 ? y+1 : rows > 0 ? rows-1 : 0); + } + s16* drow = internal::getRowPtr(dstBase, dstStride, y); + + int16x8_t tcurr = vmovq_n_s16(0x0); + int16x8_t tnext = vmovq_n_s16(0x0); + int16x8_t t0, t2; + uint8x8_t xx0 = vmov_n_u8(0x0); + uint8x8_t xx1 = vmov_n_u8(0x0); + uint8x8_t xx2 = vmov_n_u8(0x0); + ptrdiff_t x = 0; + const ptrdiff_t bcols = y + 2 < rows ? cols : (cols - 8); + for( ; x <= bcols; x += 8 ) + { + internal::prefetch(v0 + x); + internal::prefetch(v1 + x); + internal::prefetch(v2 + x); + + uint8x8_t x0 = vld1_u8(v0 + x); + uint8x8_t x1 = vld1_u8(v1 + x); + uint8x8_t x2 = vld1_u8(v2 + x); + + if(x) { + xx0 = xx1; + xx1 = xx2; + } else { + xx1 = x1; + // make border + if (border == BORDER_MODE_REPLICATE || border == BORDER_MODE_REFLECT) + { + xx1 = vset_lane_u8(vget_lane_u8(x1, 0),x1, 7); + } + else if (border == BORDER_MODE_CONSTANT) + { + xx1 = vset_lane_u8(borderValue, x1, 7); + } + else if (border == BORDER_MODE_REFLECT101) + { + xx1 = vset_lane_u8(vget_lane_u8(x1, 1),x1, 7); + } + } + xx2 = x1; + + if(x) { + tcurr = tnext; + } + tnext = vsubq_s16(vreinterpretq_s16_u16(vaddl_u8(x0, x2)), + vreinterpretq_s16_u16(vshll_n_u8(x1, 2))); + + if(!x) { + tcurr = tnext; + continue; + } + t0 = vreinterpretq_s16_u16(vmovl_u8(vext_u8(xx0, xx1, 7))); + t2 = vreinterpretq_s16_u16(vmovl_u8(vext_u8(xx1, xx2, 1))); + t0 = vaddq_s16(vqaddq_s16(t0, t2), tcurr); + + vst1q_s16(drow + x - 8, t0); + } + + x -= 8; + if(x == cols){ + x--; + } + + for( ; x < cols; x++ ) + { + s16 nextx; + s16 prevx; + // make border + if (border == BORDER_MODE_REPLICATE || border == BORDER_MODE_REFLECT) + { + prevx = x == 0 ? v1[0] : v1[x-1]; + nextx = x == cols-1 ? v1[x] : v1[x+1]; + } + else if (border == BORDER_MODE_REFLECT101) + { + prevx = x == 0 ? v1[1] : v1[x-1]; + nextx = x == cols-1 ? v1[x-1] : v1[x+1]; + } + else //if (border == BORDER_MODE_CONSTANT) + { + prevx = x == 0 ? borderValue : v1[x-1]; + nextx = x == cols-1 ? borderValue : v1[x+1]; + } + *(drow+x) = prevx + nextx - 4*v1[x] + v0[x] + v2[x]; + } + } +#else + (void)size; + (void)srcBase; + (void)srcStride; + (void)dstBase; + (void)dstStride; + (void)border; + (void)borderValue; +#endif +} + +void Laplacian3OpenCV(const Size2D &size, + const u8 * srcBase, ptrdiff_t srcStride, + s16 * dstBase, ptrdiff_t dstStride, + BORDER_MODE border, u8 borderValue) +{ + internal::assertSupportedConfiguration(isLaplacianOpenCVSupported(size, border)); +#ifdef CAROTENE_NEON + ptrdiff_t rows = size.height, cols = size.width; + + std::vector _tmp; + u8 *tmp = 0; + if (border == BORDER_MODE_CONSTANT) + { + _tmp.assign(cols + 4,borderValue); + tmp = &_tmp[2]; + } + + for( ptrdiff_t y = 0; y < rows; y++ ) + { + const u8* v0 = 0; + const u8* v1 = internal::getRowPtr(srcBase, srcStride, y); + const u8* v2 = 0; + // make border + if (border == BORDER_MODE_REFLECT101) { + v0 = internal::getRowPtr(srcBase, srcStride, y > 0 ? y-1 : y+1); + v2 = internal::getRowPtr(srcBase, srcStride, y < rows-1 ? y+1 : rows > 1 ? rows-2 : 0); + } else if (border == BORDER_MODE_CONSTANT) { + v0 = y > 0 ? internal::getRowPtr(srcBase, srcStride, y-1) : tmp; + v2 = y < rows-1 ? internal::getRowPtr(srcBase, srcStride, y+1) : tmp; + } else { + v0 = internal::getRowPtr(srcBase, srcStride, y > 0 ? y-1 : 0); + v2 = internal::getRowPtr(srcBase, srcStride, y < rows-1 ? y+1 : rows > 0 ? rows-1 : 0); + } + s16* drow = internal::getRowPtr(dstBase, dstStride, y); + + int16x8_t tprev = vmovq_n_s16(0x0); + int16x8_t tcurr = vmovq_n_s16(0x0); + int16x8_t tnext = vmovq_n_s16(0x0); + int16x8_t tc = vmovq_n_s16(0x0); + int16x8_t t0, t2, tcnext; + ptrdiff_t x = 0; + const ptrdiff_t bcols = y + 2 < rows ? cols : (cols - 8); + for( ; x <= bcols; x += 8 ) + { + internal::prefetch(v0 + x); + internal::prefetch(v1 + x); + internal::prefetch(v2 + x); + + uint8x8_t x0 = vld1_u8(v0 + x); + uint8x8_t x1 = vld1_u8(v1 + x); + uint8x8_t x2 = vld1_u8(v2 + x); + tcnext = vreinterpretq_s16_u16(vshll_n_u8(x1, 2)); + + if(x) { + tprev = tcurr; + tcurr = tnext; + } + tnext = vreinterpretq_s16_u16(vaddl_u8(x0, x2)); + + if(!x) { + tcurr = tnext; + tc = tcnext; + + // make border + if (border == BORDER_MODE_REPLICATE || border == BORDER_MODE_REFLECT) + { + tcurr = vsetq_lane_s16(vgetq_lane_s16(tcurr, 0),tcurr, 7); + } + else if (border == BORDER_MODE_CONSTANT) + { + tcurr = vsetq_lane_s16(borderValue, tcurr, 7); + } + else if (border == BORDER_MODE_REFLECT101) + { + tcurr = vsetq_lane_s16(vgetq_lane_s16(tcurr, 1),tcurr, 7); + } + continue; + } + + t0 = vextq_s16(tprev, tcurr, 7); + t2 = vextq_s16(tcurr, tnext, 1); + + t0 = vsubq_s16(vqaddq_s16(t0, t2), tc); + tc = tcnext; + + t0 = vshlq_n_s16(t0, 1); + vst1q_s16(drow + x - 8, t0); + } + x -= 8; + if(x == cols){ + x--; + } + + for( ; x < cols; x++ ) + { + s16 nextx, nextx2; + s16 prevx, prevx2; + // make border + if (border == BORDER_MODE_REPLICATE || border == BORDER_MODE_REFLECT) + { + prevx = x == 0 ? v0[0] : v0[x-1]; + prevx2 = x == 0 ? v2[0] : v2[x-1]; + nextx = x == cols-1 ? v0[x] : v0[x+1]; + nextx2 = x == cols-1 ? v2[x] : v2[x+1]; + } + else if (border == BORDER_MODE_REFLECT101) + { + prevx = x == 0 ? v0[1] : v0[x-1]; + prevx2 = x == 0 ? v2[1] : v2[x-1]; + nextx = x == cols-1 ? v0[x-1] : v0[x+1]; + nextx2 = x == cols-1 ? v2[x-1] : v2[x+1]; + } + else //if (border == BORDER_MODE_CONSTANT) + { + prevx = x == 0 ? borderValue : v0[x-1]; + prevx2 = x == 0 ? borderValue : v2[x-1]; + nextx = x == cols-1 ? borderValue : v0[x+1]; + nextx2 = x == cols-1 ? borderValue : v2[x+1]; + } + s16 res = prevx + nextx - 4*v1[x] + prevx2 + nextx2; + *(drow+x) = 2*res; + } + } +#else + (void)size; + (void)srcBase; + (void)srcStride; + (void)dstBase; + (void)dstStride; + (void)border; + (void)borderValue; +#endif +} + +void Laplacian5OpenCV(const Size2D &size, + const u8 * srcBase, ptrdiff_t srcStride, + s16 * dstBase, ptrdiff_t dstStride, + BORDER_MODE border, u8 borderValue) +{ + internal::assertSupportedConfiguration(isLaplacianOpenCVSupported(size, border)); +#ifdef CAROTENE_NEON + ptrdiff_t rows = size.height, cols = size.width; + + std::vector _tmp; + u8 *tmp = 0; + if (border == BORDER_MODE_CONSTANT) + { + _tmp.assign(cols + 4,borderValue); + tmp = &_tmp[2]; + } + + for( ptrdiff_t y = 0; y < rows; y++ ) + { + const u8* v0 = 0; + const u8* v1 = 0; + const u8* v2 = internal::getRowPtr(srcBase, srcStride, y); + const u8* v3 = 0; + const u8* v4 = 0; + // make border + if (border == BORDER_MODE_REPLICATE) { + v0 = internal::getRowPtr(srcBase, srcStride, y > 1 ? y-2 : 0); + v1 = internal::getRowPtr(srcBase, srcStride, y > 0 ? y-1 : 0); + v3 = internal::getRowPtr(srcBase, srcStride, y < rows-1 ? y+1 : rows > 0 ? rows-1 : 0); + v4 = internal::getRowPtr(srcBase, srcStride, y < rows-2 ? y+2 : rows > 0 ? rows-1 : 0); + } else if (border == BORDER_MODE_REFLECT) { + v0 = internal::getRowPtr(srcBase, srcStride, y > 1 ? y-2 : rows > 1 ? 1-y : 0); + v1 = internal::getRowPtr(srcBase, srcStride, y > 0 ? y-1 : 0); + v3 = internal::getRowPtr(srcBase, srcStride, y < rows-1 ? y+1 : rows > 0 ? rows-1 : 0); + v4 = internal::getRowPtr(srcBase, srcStride, y < rows-2 ? y+2 : rows > 1 ? 2*rows-(y+3) : 0); + } else if (border == BORDER_MODE_REFLECT101) { + v0 = internal::getRowPtr(srcBase, srcStride, y > 1 ? y-2 : rows > 2-y ? 2-y : 0); ///check + v1 = internal::getRowPtr(srcBase, srcStride, y > 0 ? y-1 : rows > 1 ? 1 : 0); + v3 = internal::getRowPtr(srcBase, srcStride, y < rows-1 ? y+1 : rows > 1 ? rows-2 : 0); + v4 = internal::getRowPtr(srcBase, srcStride, y < rows-2 ? y+2 : rows > 2 ? 2*rows-(y+4) : 0);///bad if rows=2 y=1 rows - 4 + (2,1) + } else if (border == BORDER_MODE_CONSTANT) { + v0 = y > 1 ? internal::getRowPtr(srcBase, srcStride, y-2) : tmp; + v1 = y > 0 ? internal::getRowPtr(srcBase, srcStride, y-1) : tmp; + v3 = y < rows-1 ? internal::getRowPtr(srcBase, srcStride, y+1) : tmp; + v4 = y < rows-2 ? internal::getRowPtr(srcBase, srcStride, y+2) : tmp; + } + s16* drow = internal::getRowPtr(dstBase, dstStride, y); + + int16x8_t tnext, tc, t0; + int16x8_t tnext2, tnext3; + int16x8_t tnext1Old, tnext2Old, tnext3Old; + int16x8_t tnext4OldOldOld, tnext5OldOldOld; + + int16x8_t tcurr1 = vmovq_n_s16(0x0); + int16x8_t tnext1 = vmovq_n_s16(0x0); + int16x8_t tprev1 = vmovq_n_s16(0x0); + int16x8_t tpprev1 = vmovq_n_s16(0x0); + int16x8_t tppprev1 = vmovq_n_s16(0x0); + + int16x8_t tnext4Old = vmovq_n_s16(0x0); + int16x8_t tnext5Old = vmovq_n_s16(0x0); + int16x8_t tnext1OldOld = vmovq_n_s16(0x0); + int16x8_t tnext2OldOld = vmovq_n_s16(0x0); + int16x8_t tnext3OldOld = vmovq_n_s16(0x0); + int16x8_t tnext4OldOld = vmovq_n_s16(0x0); + int16x8_t tnext5OldOld = vmovq_n_s16(0x0); + + // do vertical convolution + ptrdiff_t x = 0; + const ptrdiff_t bcols = y + 3 < rows ? cols : (cols - 8); + for( ; x <= bcols; x += 8 ) + { + internal::prefetch(v0 + x); + internal::prefetch(v1 + x); + internal::prefetch(v2 + x); + internal::prefetch(v3 + x); + internal::prefetch(v4 + x); + + uint8x8_t x0 = vld1_u8(v0 + x); + uint8x8_t x1 = vld1_u8(v1 + x); + uint8x8_t x2 = vld1_u8(v2 + x); + uint8x8_t x3 = vld1_u8(v3 + x); + uint8x8_t x4 = vld1_u8(v4 + x); + if(x) { + tcurr1 = tnext1; + } + + tnext4OldOldOld = tnext4Old; + tnext5OldOldOld = tnext5Old; + tnext1Old = tnext1OldOld; + tnext2Old = tnext2OldOld; + tnext3Old = tnext3OldOld; + tnext4Old = tnext4OldOld; + tnext5Old = tnext5OldOld; + + tnext3 = vreinterpretq_s16_u16(vaddq_u16(vaddl_u8(x3, x2),vaddl_u8(x2, x1))); + tnext3 = vshlq_n_s16(tnext3, 1); + + tc = vreinterpretq_s16_u16(vsubl_u8(x4, x2)); + tnext = vreinterpretq_s16_u16(vsubl_u8(x2, x0)); + tnext2 = vsubq_s16(tc, tnext); + + tnext1 = vaddq_s16(tnext3, tnext2); + // tnext1 = x0 + 2*x1 + 2*x2 + 2*x3 + x4 + + tnext2 = vshlq_n_s16(tnext2, 1); + // tnext2 = 2*x4 - 4*x2 + 2*x0 + + tnext3 = vsubq_s16(tnext2, vshlq_n_s16(tnext3, 1)); + // tnext3 = 2*x0 - 4*x1 - 12*x2 - 4*x3 + 2*x4 + + tnext1OldOld = tnext1; + tnext2OldOld = tnext2; + tnext3OldOld = tnext3; + tnext4OldOld = tnext2; + tnext5OldOld = tnext1; + + if(x) { + tnext1 = vextq_s16(tnext1Old, tnext1, 2); + tcurr1 = vextq_s16(tnext2Old, tnext2, 1); + tprev1 = tnext3Old; + + if(x!=8) { + tpprev1 = vextq_s16(tnext4OldOldOld, tnext4Old, 7); + tppprev1 = vextq_s16(tnext5OldOldOld, tnext5Old, 6); + } + } + + if(!x) { + // make border + if (border == BORDER_MODE_REPLICATE) { + tpprev1 = vextq_s16(tnext2, tnext2, 7); + tpprev1 = vsetq_lane_s16(vgetq_lane_s16(tpprev1, 1),tpprev1, 0); + + tprev1 = vextq_s16(tnext1, tnext1, 6); + tprev1 = vsetq_lane_s16(vgetq_lane_s16(tprev1, 2),tprev1, 0); + tprev1 = vsetq_lane_s16(vgetq_lane_s16(tprev1, 2),tprev1, 1); + } else if (border == BORDER_MODE_REFLECT) { + tpprev1 = vextq_s16(tnext2, tnext2, 7); + tpprev1 = vsetq_lane_s16(vgetq_lane_s16(tpprev1, 1),tpprev1, 0); + + tprev1 = vextq_s16(tnext1, tnext1, 6); + tprev1 = vsetq_lane_s16(vgetq_lane_s16(tprev1, 3),tprev1, 0); + tprev1 = vsetq_lane_s16(vgetq_lane_s16(tprev1, 2),tprev1, 1); + } else if (border == BORDER_MODE_REFLECT101) { + tpprev1 = vextq_s16(tnext2, tnext2, 7); + tpprev1 = vsetq_lane_s16(vgetq_lane_s16(tpprev1, 2),tpprev1, 0); + + tprev1 = vextq_s16(tnext1, tnext1, 6); + tprev1 = vsetq_lane_s16(vgetq_lane_s16(tprev1, 3),tprev1, 1); + tprev1 = vsetq_lane_s16(vgetq_lane_s16(tprev1, 4),tprev1, 0); + } else if (border == BORDER_MODE_CONSTANT) { + tpprev1 = vextq_s16(tnext2, tnext2, 7); + tpprev1 = vsetq_lane_s16(borderValue, tpprev1, 0); + + tprev1 = vextq_s16(tnext1, tnext1, 6); + tprev1 = vsetq_lane_s16(borderValue, tprev1, 0); + tprev1 = vsetq_lane_s16(borderValue, tprev1, 1); + } + tppprev1 = tprev1; + continue; + } + + t0 = vaddq_s16(vaddq_s16(vqaddq_s16(tcurr1, tprev1), vqaddq_s16(tpprev1, tppprev1)), tnext1); + t0 = vaddq_s16(t0, t0); + vst1q_s16(drow + x - 8, t0); + } + x -= 8; + if(x >= cols - 1) + x = cols-2; + + s16 pprevx = 0; + s16 prevx = 0; + s16 nextx = 0; + s16 nnextx = 0; + + for( ; x < cols; x++ ) + { + if (x == 0) { + // make border + if (border == BORDER_MODE_REPLICATE) { + pprevx = v0[0] + 2*v1[0] + 2*v2[0] + 2*v3[0] + v4[0]; + prevx = 2*v0[0] - 4*v2[0] + 2*v4[0]; + } else if (border == BORDER_MODE_REFLECT) { + pprevx = v0[1] + 2*v1[1] + 2*v2[1] + 2*v3[1] + v4[1]; + prevx = 2*v0[0] - 4*v2[0] + 2*v4[0]; + } else if (border == BORDER_MODE_REFLECT101) { + pprevx = v0[2] + 2*v1[2] + 2*v2[2] + 2*v3[2] + v4[2]; + prevx = 2*v0[1] - 4*v2[1] + 2*v4[1]; + } else if (border == BORDER_MODE_CONSTANT) { + pprevx = 8 * borderValue; + prevx = 0; + } + } else if (x == 1) { + // make border + if (border == BORDER_MODE_REPLICATE || border == BORDER_MODE_REFLECT) { + pprevx = v0[0] + 2*v1[0] + 2*v2[0] + 2*v3[0] + v4[0]; + } else if (border == BORDER_MODE_REFLECT101) { + pprevx = v0[1] + 2*v1[1] + 2*v2[1] + 2*v3[1] + v4[1]; + } else if (border == BORDER_MODE_CONSTANT) { + pprevx = 8 * borderValue; + } + prevx = 2*v0[0] - 4*v2[0] + 2*v4[0]; + } else { + pprevx = v0[x-2] + 2*v1[x-2] + 2*v2[x-2] + 2*v3[x-2] + v4[x-2]; + prevx = 2*v0[x-1] - 4*v2[x-1] + 2*v4[x-1]; + } + s16 currx = 2*v0[x] - 4*v1[x] - 12*v2[x] - 4*v3[x] + 2*v4[x]; + if (x == cols-1) { + // make border + if (border == BORDER_MODE_REPLICATE) { + nextx = 2*v0[x] - 4*v2[x] + 2*v4[x]; + nnextx = v0[x] + 2*v1[x] + 2*v2[x] + 2*v3[x] + v4[x]; + } else if (border == BORDER_MODE_REFLECT) { + nextx = 2*v0[x] - 4*v2[x] + 2*v4[x]; + nnextx = v0[x-1] + 2*v1[x-1] + 2*v2[x-1] + 2*v3[x-1] + v4[x-1]; + } else if (border == BORDER_MODE_REFLECT101) { + nextx = 2*v0[x-1] - 4*v2[x-1] + 2*v4[x-1]; + nnextx = v0[x-2] + 2*v1[x-2] + 2*v2[x-2] + 2*v3[x-2] + v4[x-2]; + } else if (border == BORDER_MODE_CONSTANT) { + nextx = 0; + nnextx = 8 * borderValue; + } + } else if (x == cols-2) { + // make border + if (border == BORDER_MODE_REPLICATE || border == BORDER_MODE_REFLECT) { + nnextx = v0[x+1] + 2*v1[x+1] + 2*v2[x+1] + 2*v3[x+1] + v4[x+1]; + } else if (border == BORDER_MODE_REFLECT101) { + nnextx = v0[x] + 2*v1[x] + 2*v2[x] + 2*v3[x] + v4[x]; + } else if (border == BORDER_MODE_CONSTANT) { + nnextx = 8 * borderValue; + } + nextx = 2*v0[x+1] - 4*v2[x+1] + 2*v4[x+1]; + } else { + nextx = 2*v0[x+1] - 4*v2[x+1] + 2*v4[x+1]; + nnextx = v0[x+2] + 2*v1[x+2] + 2*v2[x+2] + 2*v3[x+2] + v4[x+2]; + } + s16 res = pprevx + prevx + currx + nextx + nnextx; + *(drow+x) = 2*res; + } + } +#else + (void)size; + (void)srcBase; + (void)srcStride; + (void)dstBase; + (void)dstStride; + (void)border; + (void)borderValue; +#endif +} + +} // namespace CAROTENE_NS diff --git a/3rdparty/carotene/src/magnitude.cpp b/3rdparty/carotene/src/magnitude.cpp new file mode 100644 index 0000000000..cd9d82bf6c --- /dev/null +++ b/3rdparty/carotene/src/magnitude.cpp @@ -0,0 +1,160 @@ +/* + * By downloading, copying, installing or using the software you agree to this license. + * If you do not agree to this license, do not download, install, + * copy or use the software. + * + * + * License Agreement + * For Open Source Computer Vision Library + * (3-clause BSD License) + * + * Copyright (C) 2014, NVIDIA Corporation, all rights reserved. + * Third party copyrights are property of their respective owners. + * + * Redistribution and use in source and binary forms, with or without modification, + * are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * * Neither the names of the copyright holders nor the names of the contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * This software is provided by the copyright holders and contributors "as is" and + * any express or implied warranties, including, but not limited to, the implied + * warranties of merchantability and fitness for a particular purpose are disclaimed. + * In no event shall copyright holders or contributors be liable for any direct, + * indirect, incidental, special, exemplary, or consequential damages + * (including, but not limited to, procurement of substitute goods or services; + * loss of use, data, or profits; or business interruption) however caused + * and on any theory of liability, whether in contract, strict liability, + * or tort (including negligence or otherwise) arising in any way out of + * the use of this software, even if advised of the possibility of such damage. + */ + +#include "common.hpp" +#include "vtransform.hpp" + +#include + +namespace CAROTENE_NS { + +#ifdef CAROTENE_NEON + +namespace { + +struct Magnitude +{ + typedef s16 type; + + void operator() (const int16x8_t & v_src0, const int16x8_t & v_src1, + int16x8_t & v_dst) const + { + int16x4_t v_src0_p = vget_low_s16(v_src0), v_src1_p = vget_low_s16(v_src1); + float32x4_t v_sqr0 = vaddq_f32(vcvtq_f32_s32(vmull_s16(v_src0_p, v_src0_p)), + vcvtq_f32_s32(vmull_s16(v_src1_p, v_src1_p))); + v_src0_p = vget_high_s16(v_src0); + v_src1_p = vget_high_s16(v_src1); + float32x4_t v_sqr1 = vaddq_f32(vcvtq_f32_s32(vmull_s16(v_src0_p, v_src0_p)), + vcvtq_f32_s32(vmull_s16(v_src1_p, v_src1_p))); + + int32x4_t v_sqrt0 = vcvtq_s32_f32(internal::vsqrtq_f32(v_sqr0)); + int32x4_t v_sqrt1 = vcvtq_s32_f32(internal::vsqrtq_f32(v_sqr1)); + + v_dst = vcombine_s16(vqmovn_s32(v_sqrt0), vqmovn_s32(v_sqrt1)); + } + + void operator() (const int16x4_t & v_src0, const int16x4_t & v_src1, + int16x4_t & v_dst) const + { + float32x4_t v_tmp = vaddq_f32(vcvtq_f32_s32(vmull_s16(v_src0, v_src0)), + vcvtq_f32_s32(vmull_s16(v_src1, v_src1))); + int32x4_t v_sqrt = vcvtq_s32_f32(internal::vsqrtq_f32(v_tmp)); + v_dst = vqmovn_s32(v_sqrt); + } + + void operator() (const short * src0, const short * src1, short * dst) const + { + f32 src0val = (f32)src0[0], src1val = (f32)src1[0]; + dst[0] = internal::saturate_cast((s32)sqrtf(src0val * src0val + src1val * src1val)); + } +}; + +struct MagnitudeF32 +{ + typedef f32 type; + + void operator() (const float32x4_t & v_src0, const float32x4_t & v_src1, + float32x4_t & v_dst) const + { + v_dst = internal::vsqrtq_f32(vaddq_f32(vmulq_f32(v_src0, v_src0), vmulq_f32(v_src1, v_src1))); + } + + void operator() (const float32x2_t & v_src0, const float32x2_t & v_src1, + float32x2_t & v_dst) const + { + v_dst = internal::vsqrt_f32(vadd_f32(vmul_f32(v_src0, v_src0), vmul_f32(v_src1, v_src1))); + } + + void operator() (const f32 * src0, const f32 * src1, f32 * dst) const + { + dst[0] = sqrtf(src0[0] * src0[0] + src1[0] * src1[0]); + } +}; + +} // namespace + +#endif + +void magnitude(const Size2D &size, + const s16 * src0Base, ptrdiff_t src0Stride, + const s16 * src1Base, ptrdiff_t src1Stride, + s16 * dstBase, ptrdiff_t dstStride) +{ + internal::assertSupportedConfiguration(); +#ifdef CAROTENE_NEON + internal::vtransform(size, + src0Base, src0Stride, + src1Base, src1Stride, + dstBase, dstStride, + Magnitude()); +#else + (void)size; + (void)src0Base; + (void)src0Stride; + (void)src1Base; + (void)src1Stride; + (void)dstBase; + (void)dstStride; +#endif +} + +void magnitude(const Size2D &size, + const f32 * src0Base, ptrdiff_t src0Stride, + const f32 * src1Base, ptrdiff_t src1Stride, + f32 * dstBase, ptrdiff_t dstStride) +{ + internal::assertSupportedConfiguration(); +#ifdef CAROTENE_NEON + internal::vtransform(size, + src0Base, src0Stride, + src1Base, src1Stride, + dstBase, dstStride, + MagnitudeF32()); +#else + (void)size; + (void)src0Base; + (void)src0Stride; + (void)src1Base; + (void)src1Stride; + (void)dstBase; + (void)dstStride; +#endif +} + +} // namespace CAROTENE_NS diff --git a/3rdparty/carotene/src/meanstddev.cpp b/3rdparty/carotene/src/meanstddev.cpp new file mode 100644 index 0000000000..a847493429 --- /dev/null +++ b/3rdparty/carotene/src/meanstddev.cpp @@ -0,0 +1,163 @@ +/* + * By downloading, copying, installing or using the software you agree to this license. + * If you do not agree to this license, do not download, install, + * copy or use the software. + * + * + * License Agreement + * For Open Source Computer Vision Library + * (3-clause BSD License) + * + * Copyright (C) 2014, NVIDIA Corporation, all rights reserved. + * Third party copyrights are property of their respective owners. + * + * Redistribution and use in source and binary forms, with or without modification, + * are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * * Neither the names of the copyright holders nor the names of the contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * This software is provided by the copyright holders and contributors "as is" and + * any express or implied warranties, including, but not limited to, the implied + * warranties of merchantability and fitness for a particular purpose are disclaimed. + * In no event shall copyright holders or contributors be liable for any direct, + * indirect, incidental, special, exemplary, or consequential damages + * (including, but not limited to, procurement of substitute goods or services; + * loss of use, data, or profits; or business interruption) however caused + * and on any theory of liability, whether in contract, strict liability, + * or tort (including negligence or otherwise) arising in any way out of + * the use of this software, even if advised of the possibility of such damage. + */ + +#include "common.hpp" + +#include + +namespace CAROTENE_NS { + +void meanStdDev(const Size2D &size, + const u8 * srcBase, ptrdiff_t srcStride, + f32 * pMean, f32 * pStdDev) +{ + internal::assertSupportedConfiguration(); +#ifdef CAROTENE_NEON + f64 fsum = 0.0f, fsqsum = 0.0f; + sqsum(size, srcBase, srcStride, &fsum, &fsqsum, 1); + + // calc mean and stddev + f64 itotal = 1.0 / size.total(); + f64 mean = fsum * itotal; + f64 stddev = sqrt(std::max(fsqsum * itotal - mean * mean, 0.0)); + + if (pMean) + *pMean = mean; + if (pStdDev) + *pStdDev = stddev; +#else + (void)size; + (void)srcBase; + (void)srcStride; + (void)pMean; + (void)pStdDev; +#endif +} + +void meanStdDev(const Size2D &size, + const u16 * srcBase, ptrdiff_t srcStride, + f32 * pMean, f32 * pStdDev) +{ + internal::assertSupportedConfiguration(); +#ifdef CAROTENE_NEON + size_t blockSize0 = 1 << 10, roiw4 = size.width & ~3; + f64 fsum = 0.0f, fsqsum = 0.0f; + + f32 arsum[8]; + uint32x4_t v_zero = vdupq_n_u32(0u), v_sum; + float32x4_t v_zero_f = vdupq_n_f32(0.0f), v_sqsum; + + for (size_t i = 0; i < size.height; ++i) + { + const u16 * src = internal::getRowPtr(srcBase, srcStride, i); + size_t j = 0u; + + while (j < roiw4) + { + size_t blockSize = std::min(roiw4 - j, blockSize0) + j; + v_sum = v_zero; + v_sqsum = v_zero_f; + + for ( ; j + 16 < blockSize ; j += 16) + { + internal::prefetch(src + j); + uint16x8_t v_src0 = vld1q_u16(src + j), v_src1 = vld1q_u16(src + j + 8); + + // 0 + uint32x4_t v_srclo = vmovl_u16(vget_low_u16(v_src0)); + uint32x4_t v_srchi = vmovl_u16(vget_high_u16(v_src0)); + v_sum = vaddq_u32(v_sum, vaddq_u32(v_srclo, v_srchi)); + float32x4_t v_srclo_f = vcvtq_f32_u32(v_srclo); + float32x4_t v_srchi_f = vcvtq_f32_u32(v_srchi); + v_sqsum = vmlaq_f32(v_sqsum, v_srclo_f, v_srclo_f); + v_sqsum = vmlaq_f32(v_sqsum, v_srchi_f, v_srchi_f); + + // 1 + v_srclo = vmovl_u16(vget_low_u16(v_src1)); + v_srchi = vmovl_u16(vget_high_u16(v_src1)); + v_sum = vaddq_u32(v_sum, vaddq_u32(v_srclo, v_srchi)); + v_srclo_f = vcvtq_f32_u32(v_srclo); + v_srchi_f = vcvtq_f32_u32(v_srchi); + v_sqsum = vmlaq_f32(v_sqsum, v_srclo_f, v_srclo_f); + v_sqsum = vmlaq_f32(v_sqsum, v_srchi_f, v_srchi_f); + } + + for ( ; j < blockSize; j += 4) + { + uint32x4_t v_src = vmovl_u16(vld1_u16(src + j)); + float32x4_t v_src_f = vcvtq_f32_u32(v_src); + v_sum = vaddq_u32(v_sum, v_src); + v_sqsum = vmlaq_f32(v_sqsum, v_src_f, v_src_f); + } + + vst1q_f32(arsum, vcvtq_f32_u32(v_sum)); + vst1q_f32(arsum + 4, v_sqsum); + + fsum += (f64)arsum[0] + arsum[1] + arsum[2] + arsum[3]; + fsqsum += (f64)arsum[4] + arsum[5] + arsum[6] + arsum[7]; + } + + // collect a few last elements in the current row + for ( ; j < size.width; ++j) + { + f32 srcval = src[j]; + fsum += srcval; + fsqsum += srcval * srcval; + } + } + + // calc mean and stddev + f64 itotal = 1.0 / size.total(); + f64 mean = fsum * itotal; + f64 stddev = sqrt(std::max(fsqsum * itotal - mean * mean, 0.0)); + + if (pMean) + *pMean = mean; + if (pStdDev) + *pStdDev = stddev; +#else + (void)size; + (void)srcBase; + (void)srcStride; + (void)pMean; + (void)pStdDev; +#endif +} + +} // namespace CAROTENE_NS diff --git a/3rdparty/carotene/src/median_filter.cpp b/3rdparty/carotene/src/median_filter.cpp new file mode 100644 index 0000000000..8c5d08b7ee --- /dev/null +++ b/3rdparty/carotene/src/median_filter.cpp @@ -0,0 +1,227 @@ +/* + * By downloading, copying, installing or using the software you agree to this license. + * If you do not agree to this license, do not download, install, + * copy or use the software. + * + * + * License Agreement + * For Open Source Computer Vision Library + * (3-clause BSD License) + * + * Copyright (C) 2012-2014, NVIDIA Corporation, all rights reserved. + * Third party copyrights are property of their respective owners. + * + * Redistribution and use in source and binary forms, with or without modification, + * are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * * Neither the names of the copyright holders nor the names of the contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * This software is provided by the copyright holders and contributors "as is" and + * any express or implied warranties, including, but not limited to, the implied + * warranties of merchantability and fitness for a particular purpose are disclaimed. + * In no event shall copyright holders or contributors be liable for any direct, + * indirect, incidental, special, exemplary, or consequential damages + * (including, but not limited to, procurement of substitute goods or services; + * loss of use, data, or profits; or business interruption) however caused + * and on any theory of liability, whether in contract, strict liability, + * or tort (including negligence or otherwise) arising in any way out of + * the use of this software, even if advised of the possibility of such damage. + */ + +#include "common.hpp" + +/* + * The code here is based on the code in + * , which is in public domain. + * See also . + */ + +namespace CAROTENE_NS { + +#ifdef CAROTENE_NEON +namespace { + + uint8x16_t getLeftReplicate(uint8x16_t r, u32 cn) + { + u8 buf[16+8]; + vst1q_u8(buf+cn, r); + for (u32 i = 0; i < cn; ++i) buf[i] = buf[cn+i]; + return vld1q_u8(buf); + } + + uint8x8_t getRightReplicate(uint8x8_t r, u32 cn) + { + u8 buf[8+8]; + vst1_u8(buf, r); + for (u32 i = 0; i < cn; ++i) buf[8+i] = buf[8-cn+i]; + return vld1_u8(buf+cn); + } + +} // namespace + +//o------^-------^-----------------------------o 0 +// | | +//o--^---v---^---|-------^---------------------o 1 +// | | | | +//o--v-------v---|-------|-^-------^-------^---o 2 +// | | | | | +//o------^-------v-----^-|-|-------|-------|---o 3 +// | | | | | | +//o--^---v---^-----^---|-v-|---^---v---^---v---o 4 +// | | | | | | | +//o--v-------v---^-|---|---v---|-------|-------o 5 +// | | | | | +//o------^-------|-|---v-------|-------v-------o 6 +// | | | | +//o--^---v---^---|-v-----------v---------------o 7 +// | | | +//o--v-------v---v-----------------------------o 8 + +#define ELT(num, level) v ## num ## _lv ## level +#define PIX_SORT(a, alvl, b, blvl, newlvl) \ + PIX_MIN(a, alvl, b, blvl, newlvl); \ + PIX_MAX(a, alvl, b, blvl, newlvl); + +#define SORT9 \ + PIX_SORT(1, 00, 2, 00, 01); \ + PIX_SORT(4, 00, 5, 00, 02); \ + PIX_SORT(7, 00, 8, 00, 03); \ + PIX_SORT(0, 00, 1, 01, 04); \ + PIX_SORT(3, 00, 4, 02, 05); \ + PIX_SORT(6, 00, 7, 03, 06); \ + PIX_SORT(1, 04, 2, 01, 07); \ + PIX_SORT(4, 05, 5, 02, 08); \ + PIX_SORT(7, 06, 8, 03, 09); \ + PIX_MAX (0, 04, 3, 05, 10); \ + PIX_MIN (5, 08, 8, 09, 11); \ + PIX_SORT(4, 08, 7, 09, 12); \ + PIX_MAX (3, 10, 6, 06, 13); \ + PIX_MAX (1, 07, 4, 12, 14); \ + PIX_MIN (2, 07, 5, 11, 15); \ + PIX_MIN (4, 14, 7, 12, 16); \ + PIX_SORT(4, 16, 2, 15, 17); \ + PIX_MAX (6, 13, 4, 17, 18); \ + PIX_MIN (4, 18, 2, 17, 19); + +#endif + +bool isMedianFilter3x3Supported(const Size2D &size, u32 numChannels) +{ + return isSupportedConfiguration() && size.width >= 16 + numChannels && numChannels <= 8; +} + +void medianFilter3x3(const Size2D &size, u32 numChannels, + const u8 *srcBase, ptrdiff_t srcStride, + const Margin &srcMargin, + u8 *dstBase, ptrdiff_t dstStride) +{ + internal::assertSupportedConfiguration(isMedianFilter3x3Supported(size, numChannels)); +#ifdef CAROTENE_NEON + u32 cn = numChannels; + size_t colsn = size.width * cn; + + for (size_t i = 0; i < size.height; ++i) { + const u8* psrc1 = internal::getRowPtr(srcBase, srcStride, i); + const u8* psrc0 = i == 0 && srcMargin.top == 0 ? psrc1 : psrc1 - srcStride; + const u8* psrc2 = i + 1 == size.height && srcMargin.bottom == 0 ? psrc1 : psrc1 + srcStride; + u8* pdst = internal::getRowPtr(dstBase, dstStride, i); + size_t j = 0; + + { + uint8x16_t v3_lv00 = vld1q_u8(psrc0); + uint8x16_t v4_lv00 = vld1q_u8(psrc1); + uint8x16_t v5_lv00 = vld1q_u8(psrc2); + uint8x16_t v6_lv00 = vld1q_u8(psrc0 + cn); + uint8x16_t v7_lv00 = vld1q_u8(psrc1 + cn); + uint8x16_t v8_lv00 = vld1q_u8(psrc2 + cn); + uint8x16_t v0_lv00 = srcMargin.left > 0 ? vld1q_u8(psrc0 - cn) : getLeftReplicate(v3_lv00, cn); + uint8x16_t v1_lv00 = srcMargin.left > 0 ? vld1q_u8(psrc1 - cn) : getLeftReplicate(v4_lv00, cn); + uint8x16_t v2_lv00 = srcMargin.left > 0 ? vld1q_u8(psrc2 - cn) : getLeftReplicate(v5_lv00, cn); + + goto medianBlur3x3_mainBody; + + for (; j < colsn - 16; j += 16) { + internal::prefetch(psrc0 + j); + internal::prefetch(psrc1 + j); + internal::prefetch(psrc2 + j); + + v0_lv00 = vld1q_u8(psrc0 + j - cn); + v1_lv00 = vld1q_u8(psrc1 + j - cn); + v2_lv00 = vld1q_u8(psrc2 + j - cn); + v3_lv00 = vld1q_u8(psrc0 + j); + v4_lv00 = vld1q_u8(psrc1 + j); + v5_lv00 = vld1q_u8(psrc2 + j); + v6_lv00 = vld1q_u8(psrc0 + j + cn); + v7_lv00 = vld1q_u8(psrc1 + j + cn); + v8_lv00 = vld1q_u8(psrc2 + j + cn); + +medianBlur3x3_mainBody: + +#define PIX_MIN(a, alvl, b, blvl, newlvl) uint8x16_t ELT(a, newlvl) = vminq_u8(ELT(a, alvl), ELT(b, blvl)) +#define PIX_MAX(a, alvl, b, blvl, newlvl) uint8x16_t ELT(b, newlvl) = vmaxq_u8(ELT(a, alvl), ELT(b, blvl)) + SORT9; +#undef PIX_MAX +#undef PIX_MIN + + vst1q_u8(pdst + j, v4_lv19); + } + } + + { + size_t k = colsn - 8; + uint8x8_t v0_lv00 = vld1_u8(psrc0 + k - cn); + uint8x8_t v1_lv00 = vld1_u8(psrc1 + k - cn); + uint8x8_t v2_lv00 = vld1_u8(psrc2 + k - cn); + uint8x8_t v3_lv00 = vld1_u8(psrc0 + k); + uint8x8_t v4_lv00 = vld1_u8(psrc1 + k); + uint8x8_t v5_lv00 = vld1_u8(psrc2 + k); + uint8x8_t v6_lv00 = srcMargin.right > 0 ? vld1_u8(psrc0 + k + cn) : getRightReplicate(v3_lv00, cn); + uint8x8_t v7_lv00 = srcMargin.right > 0 ? vld1_u8(psrc1 + k + cn) : getRightReplicate(v4_lv00, cn); + uint8x8_t v8_lv00 = srcMargin.right > 0 ? vld1_u8(psrc2 + k + cn) : getRightReplicate(v5_lv00, cn); + + goto medianBlur3x3_tailBody; + + for (; k >= j - 8; k -= 8) { + v0_lv00 = vld1_u8(psrc0 + k - cn); + v1_lv00 = vld1_u8(psrc1 + k - cn); + v2_lv00 = vld1_u8(psrc2 + k - cn); + v3_lv00 = vld1_u8(psrc0 + k); + v4_lv00 = vld1_u8(psrc1 + k); + v5_lv00 = vld1_u8(psrc2 + k); + v6_lv00 = vld1_u8(psrc0 + k + cn); + v7_lv00 = vld1_u8(psrc1 + k + cn); + v8_lv00 = vld1_u8(psrc2 + k + cn); + +medianBlur3x3_tailBody: + +#define PIX_MIN(a, alvl, b, blvl, newlvl) uint8x8_t ELT(a, newlvl) = vmin_u8(ELT(a, alvl), ELT(b, blvl)) +#define PIX_MAX(a, alvl, b, blvl, newlvl) uint8x8_t ELT(b, newlvl) = vmax_u8(ELT(a, alvl), ELT(b, blvl)) + SORT9; +#undef PIX_MAX +#undef PIX_MIN + + vst1_u8(pdst + k, v4_lv19); + } + } + } +#else + (void)size; + (void)numChannels; + (void)srcBase; + (void)srcStride; + (void)srcMargin; + (void)dstBase; + (void)dstStride; +#endif +} + +} // namespace CAROTENE_NS diff --git a/3rdparty/carotene/src/min_max.cpp b/3rdparty/carotene/src/min_max.cpp new file mode 100644 index 0000000000..d6f4017841 --- /dev/null +++ b/3rdparty/carotene/src/min_max.cpp @@ -0,0 +1,139 @@ +/* + * By downloading, copying, installing or using the software you agree to this license. + * If you do not agree to this license, do not download, install, + * copy or use the software. + * + * + * License Agreement + * For Open Source Computer Vision Library + * (3-clause BSD License) + * + * Copyright (C) 2014, NVIDIA Corporation, all rights reserved. + * Third party copyrights are property of their respective owners. + * + * Redistribution and use in source and binary forms, with or without modification, + * are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * * Neither the names of the copyright holders nor the names of the contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * This software is provided by the copyright holders and contributors "as is" and + * any express or implied warranties, including, but not limited to, the implied + * warranties of merchantability and fitness for a particular purpose are disclaimed. + * In no event shall copyright holders or contributors be liable for any direct, + * indirect, incidental, special, exemplary, or consequential damages + * (including, but not limited to, procurement of substitute goods or services; + * loss of use, data, or profits; or business interruption) however caused + * and on any theory of liability, whether in contract, strict liability, + * or tort (including negligence or otherwise) arising in any way out of + * the use of this software, even if advised of the possibility of such damage. + */ + +#include + +#include "common.hpp" +#include "vtransform.hpp" + +namespace CAROTENE_NS { + +#ifdef CAROTENE_NEON + +namespace { + +template +struct Min +{ + typedef T type; + + void operator() (const typename internal::VecTraits::vec128 & v_src0, + const typename internal::VecTraits::vec128 & v_src1, + typename internal::VecTraits::vec128 & v_dst) const + { + v_dst = internal::vminq(v_src0, v_src1); + } + + void operator() (const typename internal::VecTraits::vec64 & v_src0, + const typename internal::VecTraits::vec64 & v_src1, + typename internal::VecTraits::vec64 & v_dst) const + { + v_dst = internal::vmin(v_src0, v_src1); + } + + void operator() (const T * src0, const T * src1, T * dst) const + { + dst[0] = std::min(src0[0], src1[0]); + } +}; + +template +struct Max +{ + typedef T type; + + void operator() (const typename internal::VecTraits::vec128 & v_src0, + const typename internal::VecTraits::vec128 & v_src1, + typename internal::VecTraits::vec128 & v_dst) const + { + v_dst = internal::vmaxq(v_src0, v_src1); + } + + void operator() (const typename internal::VecTraits::vec64 & v_src0, + const typename internal::VecTraits::vec64 & v_src1, + typename internal::VecTraits::vec64 & v_dst) const + { + v_dst = internal::vmax(v_src0, v_src1); + } + + void operator() (const T * src0, const T * src1, T * dst) const + { + dst[0] = std::max(src0[0], src1[0]); + } +}; + +} // namespace + +#define IMPL_OP(fun, op, type) \ +void fun(const Size2D &size, \ + const type * src0Base, ptrdiff_t src0Stride, \ + const type * src1Base, ptrdiff_t src1Stride, \ + type * dstBase, ptrdiff_t dstStride) \ +{ \ + internal::assertSupportedConfiguration(); \ + internal::vtransform(size, \ + src0Base, src0Stride, \ + src1Base, src1Stride, \ + dstBase, dstStride, op()); \ +} + +#else + +#define IMPL_OP(fun, op, type) \ +void fun(const Size2D &, \ + const type *, ptrdiff_t, \ + const type *, ptrdiff_t, \ + type *, ptrdiff_t) \ +{ \ + internal::assertSupportedConfiguration(); \ +} + +#endif + +#define IMPL_MINMAX(type) IMPL_OP(min, Min, type) IMPL_OP(max, Max, type) + +IMPL_MINMAX(u8) +IMPL_MINMAX(s8) +IMPL_MINMAX(u16) +IMPL_MINMAX(s16) +IMPL_MINMAX(u32) +IMPL_MINMAX(s32) +IMPL_MINMAX(f32) + +} // namespace CAROTENE_NS diff --git a/3rdparty/carotene/src/minmaxloc.cpp b/3rdparty/carotene/src/minmaxloc.cpp new file mode 100644 index 0000000000..a7f30bc4f8 --- /dev/null +++ b/3rdparty/carotene/src/minmaxloc.cpp @@ -0,0 +1,1340 @@ +/* + * By downloading, copying, installing or using the software you agree to this license. + * If you do not agree to this license, do not download, install, + * copy or use the software. + * + * + * License Agreement + * For Open Source Computer Vision Library + * (3-clause BSD License) + * + * Copyright (C) 2012-2015, NVIDIA Corporation, all rights reserved. + * Third party copyrights are property of their respective owners. + * + * Redistribution and use in source and binary forms, with or without modification, + * are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * * Neither the names of the copyright holders nor the names of the contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * This software is provided by the copyright holders and contributors "as is" and + * any express or implied warranties, including, but not limited to, the implied + * warranties of merchantability and fitness for a particular purpose are disclaimed. + * In no event shall copyright holders or contributors be liable for any direct, + * indirect, incidental, special, exemplary, or consequential damages + * (including, but not limited to, procurement of substitute goods or services; + * loss of use, data, or profits; or business interruption) however caused + * and on any theory of liability, whether in contract, strict liability, + * or tort (including negligence or otherwise) arising in any way out of + * the use of this software, even if advised of the possibility of such damage. + */ + +#include "common.hpp" +#include "vtransform.hpp" + +#include + +namespace CAROTENE_NS { + +#ifdef CAROTENE_NEON + +namespace { + +template +void minMaxVals(const Size2D &size, + const T * srcBase, ptrdiff_t srcStride, + T * pMinVal, T * pMaxVal) +{ + using namespace internal; + + typedef typename VecTraits::vec128 vec128; + typedef typename VecTraits::vec64 vec64; + + u32 step_base = 32 / sizeof(T), step_tail = 8 / sizeof(T); + size_t roiw_base = size.width >= (step_base - 1) ? size.width - step_base + 1 : 0; + size_t roiw_tail = size.width >= (step_tail - 1) ? size.width - step_tail + 1 : 0; + + T maxVal = std::numeric_limits::min(); + T minVal = std::numeric_limits::max(); + vec128 v_min_base = vdupq_n(minVal), v_max_base = vdupq_n(maxVal); + vec64 v_min_tail = vdup_n(minVal), v_max_tail = vdup_n(maxVal); + + for (size_t i = 0; i < size.height; ++i) + { + const T * src = getRowPtr(srcBase, srcStride, i); + size_t j = 0; + + for (; j < roiw_base; j += step_base) + { + prefetch(src + j); + vec128 v_src0 = vld1q(src + j), v_src1 = vld1q(src + j + 16 / sizeof(T)); + v_min_base = vminq(v_min_base, v_src0); + v_max_base = vmaxq(v_max_base, v_src0); + v_min_base = vminq(v_min_base, v_src1); + v_max_base = vmaxq(v_max_base, v_src1); + } + for (; j < roiw_tail; j += step_tail) + { + vec64 v_src0 = vld1(src + j); + v_min_tail = vmin(v_min_tail, v_src0); + v_max_tail = vmax(v_max_tail, v_src0); + } + + for (; j < size.width; j++) + { + T srcval = src[j]; + minVal = std::min(srcval, minVal); + maxVal = std::max(srcval, maxVal); + } + } + + // collect min & max values + T ar[16 / sizeof(T)]; + vst1q(ar, vcombine(vmin(v_min_tail, vmin(vget_low(v_min_base), vget_high(v_min_base))), + vmax(v_max_tail, vmax(vget_low(v_max_base), vget_high(v_max_base))))); + + for (size_t x = 0; x < 8u / sizeof(T); ++x) + { + minVal = std::min(minVal, ar[x]); + maxVal = std::max(maxVal, ar[x + 8 / sizeof(T)]); + } + + if (pMaxVal) + *pMaxVal = maxVal; + if (pMinVal) + *pMinVal = minVal; +} + +} // namespace + +#endif + +void minMaxVals(const Size2D &size, + const u8 * srcBase, ptrdiff_t srcStride, + u8 * pMinVal, u8 * pMaxVal) +{ + internal::assertSupportedConfiguration(); +#ifdef CAROTENE_NEON + minMaxVals(size, + srcBase, srcStride, + pMinVal, pMaxVal); +#else + (void)size; + (void)srcBase; + (void)srcStride; + (void)pMinVal; + (void)pMaxVal; +#endif +} + +void minMaxVals(const Size2D &size, + const s16 * srcBase, ptrdiff_t srcStride, + s16 * pMinVal, s16 * pMaxVal) +{ + internal::assertSupportedConfiguration(); +#ifdef CAROTENE_NEON + minMaxVals(size, + srcBase, srcStride, + pMinVal, pMaxVal); +#else + (void)size; + (void)srcBase; + (void)srcStride; + (void)pMinVal; + (void)pMaxVal; +#endif +} + +void minMaxVals(const Size2D &size, + const u16 * srcBase, ptrdiff_t srcStride, + u16 * pMinVal, u16 * pMaxVal) +{ + internal::assertSupportedConfiguration(); +#ifdef CAROTENE_NEON + minMaxVals(size, + srcBase, srcStride, + pMinVal, pMaxVal); +#else + (void)size; + (void)srcBase; + (void)srcStride; + (void)pMinVal; + (void)pMaxVal; +#endif +} + +void minMaxVals(const Size2D &size, + const s32 * srcBase, ptrdiff_t srcStride, + s32 * pMinVal, s32 * pMaxVal) +{ + internal::assertSupportedConfiguration(); +#ifdef CAROTENE_NEON + minMaxVals(size, + srcBase, srcStride, + pMinVal, pMaxVal); +#else + (void)size; + (void)srcBase; + (void)srcStride; + (void)pMinVal; + (void)pMaxVal; +#endif +} + +void minMaxVals(const Size2D &size, + const u32 * srcBase, ptrdiff_t srcStride, + u32 * pMinVal, u32 * pMaxVal) +{ + internal::assertSupportedConfiguration(); +#ifdef CAROTENE_NEON + minMaxVals(size, + srcBase, srcStride, + pMinVal, pMaxVal); +#else + (void)size; + (void)srcBase; + (void)srcStride; + (void)pMinVal; + (void)pMaxVal; +#endif +} + +void minMaxLoc(const Size2D &size, + const f32 * srcBase, ptrdiff_t srcStride, + f32 &minVal, size_t &minCol, size_t &minRow, + f32 &maxVal, size_t &maxCol, size_t &maxRow) +{ + internal::assertSupportedConfiguration(); +#ifdef CAROTENE_NEON + minVal = srcBase[0]; + minCol = 0; + minRow = 0; + maxVal = srcBase[0]; + maxCol = 0; + maxRow = 0; + for(size_t l = 0, i = 0; l < size.height; ++l, i = 0) + { + const f32 * src = internal::getRowPtr( srcBase, srcStride, l); + if (size.width >= 16) + { + u32 tmp0123[4] = { 0, 1, 2, 3 }; + uint32x4_t c4 = vdupq_n_u32(4); + +#if SIZE_MAX > UINT32_MAX + size_t boundAll = size.width - (4 - 1); + for(size_t b = 0; i < boundAll; b = i) + { + size_t bound = std::min(boundAll, b + 0xffffFFFC); +#else + { + size_t bound = size.width - (4 - 1); +#endif + uint32x4_t lineIdxOffset = vld1q_u32(tmp0123); + float32x4_t n_min = vdupq_n_f32(minVal); + uint32x4_t n_minIdx = vdupq_n_u32(0xffffFFFC); + float32x4_t n_max = vdupq_n_f32(maxVal); + uint32x4_t n_maxIdx = vdupq_n_u32(0xffffFFFC); + + for(; i < bound; i+=4) + { + internal::prefetch(src + i); + float32x4_t line = vld1q_f32(src + i); + + uint32x4_t minmask = vcltq_f32(line, n_min); + uint32x4_t maxmask = vcgtq_f32(line, n_max); + + n_min = vbslq_f32(minmask, line, n_min); + n_minIdx = vbslq_u32(minmask, lineIdxOffset, n_minIdx); + n_max = vbslq_f32(maxmask, line, n_max); + n_maxIdx = vbslq_u32(maxmask, lineIdxOffset, n_maxIdx); + + // idx[] +=4 + lineIdxOffset = vaddq_u32(lineIdxOffset, c4); + } + + f32 fmin[4], fmax[4]; + u32 fminIdx[4], fmaxIdx[4]; + + vst1q_f32(fmin, n_min); + vst1q_f32(fmax, n_max); + + vst1q_u32(fminIdx, n_minIdx); + vst1q_u32(fmaxIdx, n_maxIdx); + + size_t minIdx = fminIdx[0]; + size_t maxIdx = fmaxIdx[0]; + minVal = fmin[0]; + maxVal = fmax[0]; + + for (s32 j = 1; j < 4; ++j) + { + f32 minval = fmin[j]; + f32 maxval = fmax[j]; + if (minval < minVal || (minval == minVal && fminIdx[j] < minIdx)) + { + minIdx = fminIdx[j]; + minVal = minval; + } + if (maxval > maxVal || (maxval == maxVal && fmaxIdx[j] < maxIdx)) + { + maxIdx = fmaxIdx[j]; + maxVal = maxval; + } + } + if(minIdx < 0xffffFFFC) + { +#if SIZE_MAX > UINT32_MAX + minCol = b + minIdx; +#else + minCol = minIdx; +#endif + minRow = l; + } + if(maxIdx < 0xffffFFFC) + { +#if SIZE_MAX > UINT32_MAX + maxCol = b + maxIdx; +#else + maxCol = maxIdx; +#endif + maxRow = l; + } + } + } + for(; i < size.width; ++i ) + { + float val = src[i]; + if( val < minVal ) + { + minVal = val; + minCol = i; + minRow = l; + } + else if( val > maxVal ) + { + maxVal = val; + maxCol = i; + maxRow = l; + } + } + } +#else + (void)size; + (void)srcBase; + (void)srcStride; + (void)minVal; + (void)minCol; + (void)minRow; + (void)maxVal; + (void)maxCol; + (void)maxRow; +#endif +} + +void minMaxLoc(const Size2D &size, + const f32 * srcBase, ptrdiff_t srcStride, + const u8 * maskBase, ptrdiff_t maskStride, + f32 &minVal, size_t &minCol, size_t &minRow, + f32 &maxVal, size_t &maxCol, size_t &maxRow) +{ + internal::assertSupportedConfiguration(); +#ifdef CAROTENE_NEON + minVal = std::numeric_limits::max(); + minCol = size.width; + minRow = size.height; + maxVal = -std::numeric_limits::max(); + maxCol = size.width; + maxRow = size.height; + for(size_t l = 0, i = 0; l < size.height; ++l, i = 0) + { + const f32 * src = internal::getRowPtr( srcBase, srcStride, l); + const u8 * mask = internal::getRowPtr( maskBase, maskStride, l); + if (size.width >= 16) + { + u32 tmp0123[4] = { 0, 1, 2, 3 }; + uint32x4_t uOne = vdupq_n_u32(1); + uint32x4_t c4 = vdupq_n_u32(4); + +#if SIZE_MAX > UINT32_MAX + size_t boundAll = size.width - (4 - 1); + for(size_t b = 0; i < boundAll; b = i) + { + size_t bound = std::min(boundAll, b + 0xffffFFFC); +#else + { + size_t bound = size.width - (4 - 1); +#endif + uint32x4_t lineIdxOffset = vld1q_u32(tmp0123); + float32x4_t n_min = vdupq_n_f32(minVal); + uint32x4_t n_minIdx = vdupq_n_u32(0xffffFFFC); + float32x4_t n_max = vdupq_n_f32(maxVal); + uint32x4_t n_maxIdx = vdupq_n_u32(0xffffFFFC); + + for(; i < bound; i+=4) + { + internal::prefetch(src + i); + internal::prefetch(mask + i); + float32x4_t line = vld1q_f32(src + i); + uint8x8_t maskLine = vld1_u8(mask + i); + + uint32x4_t maskLine4 = vmovl_u16(vget_low_u16(vmovl_u8(maskLine))); + maskLine4 = vcgeq_u32(maskLine4, uOne); + + uint32x4_t minmask = vcltq_f32(line, n_min); + uint32x4_t maxmask = vcgtq_f32(line, n_max); + + minmask = vandq_u32(minmask, maskLine4); + maxmask = vandq_u32(maxmask, maskLine4); + + n_min = vbslq_f32(minmask, line, n_min); + n_minIdx = vbslq_u32(minmask, lineIdxOffset, n_minIdx); + n_max = vbslq_f32(maxmask, line, n_max); + n_maxIdx = vbslq_u32(maxmask, lineIdxOffset, n_maxIdx); + + // idx[] +=4 + lineIdxOffset = vaddq_u32(lineIdxOffset, c4); + } + + f32 fmin[4], fmax[4]; + u32 fminIdx[4], fmaxIdx[4]; + + vst1q_f32(fmin, n_min); + vst1q_f32(fmax, n_max); + + vst1q_u32(fminIdx, n_minIdx); + vst1q_u32(fmaxIdx, n_maxIdx); + + size_t minIdx = fminIdx[0]; + size_t maxIdx = fmaxIdx[0]; + minVal = fmin[0]; + maxVal = fmax[0]; + + for (s32 j = 1; j < 4; ++j) + { + f32 minval = fmin[j]; + f32 maxval = fmax[j]; + if (minval < minVal || (minval == minVal && fminIdx[j] < minIdx)) + { + minIdx = fminIdx[j]; + minVal = minval; + } + if (maxval > maxVal || (maxval == maxVal && fmaxIdx[j] < maxIdx)) + { + maxIdx = fmaxIdx[j]; + maxVal = maxval; + } + } + if(minIdx < 0xffffFFFC) + { +#if SIZE_MAX > UINT32_MAX + minCol = b + minIdx; +#else + minCol = minIdx; +#endif + minRow = l; + } + if(maxIdx < 0xffffFFFC) + { +#if SIZE_MAX > UINT32_MAX + maxCol = b + maxIdx; +#else + maxCol = maxIdx; +#endif + maxRow = l; + } + } + } + for(; i < size.width; i++ ) + { + if (!mask[i]) + continue; + f32 val = src[i]; + if( val < minVal ) + { + minVal = val; + minCol = i; + minRow = l; + } + if( val > maxVal ) + { + maxVal = val; + maxCol = i; + maxRow = l; + } + } + } +#else + (void)size; + (void)srcBase; + (void)srcStride; + (void)maskBase; + (void)maskStride; + (void)minVal; + (void)minCol; + (void)minRow; + (void)maxVal; + (void)maxCol; + (void)maxRow; +#endif +} + +void minMaxLoc(const Size2D &size, + const s32 * srcBase, ptrdiff_t srcStride, + s32 &minVal, size_t &minCol, size_t &minRow, + s32 &maxVal, size_t &maxCol, size_t &maxRow) +{ + internal::assertSupportedConfiguration(); +#ifdef CAROTENE_NEON + minVal = srcBase[0]; + minCol = 0; + minRow = 0; + maxVal = srcBase[0]; + maxCol = 0; + maxRow = 0; + for(size_t l = 0, i = 0; l < size.height; ++l, i = 0) + { + const s32 * src = internal::getRowPtr( srcBase, srcStride, l); + if (size.width >= 16) + { + u32 tmp0123[4] = { 0, 1, 2, 3 }; + uint32x4_t c4 = vdupq_n_u32(4); + +#if SIZE_MAX > UINT32_MAX + size_t boundAll = size.width - (4 - 1); + for(size_t b = 0; i < boundAll; b = i) + { + size_t bound = std::min(boundAll, b + 0xffffFFFC); +#else + { + size_t bound = size.width - (4 - 1); +#endif + uint32x4_t lineIdxOffset = vld1q_u32(tmp0123); + int32x4_t n_min = vdupq_n_s32(minVal); + uint32x4_t n_minIdx = vdupq_n_u32(0xffffFFFC); + int32x4_t n_max = vdupq_n_s32(maxVal); + uint32x4_t n_maxIdx = vdupq_n_u32(0xffffFFFC); + + for(; i < bound; i+=4 ) + { + internal::prefetch(src + i); + int32x4_t line = vld1q_s32(src + i); + + uint32x4_t minmask = vcltq_s32(line, n_min); + uint32x4_t maxmask = vcgtq_s32(line, n_max); + + n_min = vbslq_s32(minmask, line, n_min); + n_minIdx = vbslq_u32(minmask, lineIdxOffset, n_minIdx); + n_max = vbslq_s32(maxmask, line, n_max); + n_maxIdx = vbslq_u32(maxmask, lineIdxOffset, n_maxIdx); + + // idx[] +=4 + lineIdxOffset = vaddq_u32(lineIdxOffset, c4); + } + + s32 fmin[4], fmax[4]; + u32 fminIdx[4], fmaxIdx[4]; + + vst1q_s32(fmin, n_min); + vst1q_s32(fmax, n_max); + + vst1q_u32(fminIdx, n_minIdx); + vst1q_u32(fmaxIdx, n_maxIdx); + + size_t minIdx = fminIdx[0]; + size_t maxIdx = fmaxIdx[0]; + minVal = fmin[0]; + maxVal = fmax[0]; + + for (s32 j = 1; j < 4; ++j) + { + s32 minval = fmin[j]; + s32 maxval = fmax[j]; + if (minval < minVal || (minval == minVal && fminIdx[j] < minIdx)) + { + minIdx = fminIdx[j]; + minVal = minval; + } + if (maxval > maxVal || (maxval == maxVal && fmaxIdx[j] < maxIdx)) + { + maxIdx = fmaxIdx[j]; + maxVal = maxval; + } + } + if(minIdx < 0xffffFFFC) + { +#if SIZE_MAX > UINT32_MAX + minCol = b + minIdx; +#else + minCol = minIdx; +#endif + minRow = l; + } + if(maxIdx < 0xffffFFFC) + { +#if SIZE_MAX > UINT32_MAX + maxCol = b + maxIdx; +#else + maxCol = maxIdx; +#endif + maxRow = l; + } + } + } + for(; i < size.width; ++i ) + { + s32 val = src[i]; + if( val < minVal ) + { + minVal = val; + minCol = i; + minRow = l; + } + else if( val > maxVal ) + { + maxVal = val; + maxCol = i; + maxRow = l; + } + } + } +#else + (void)size; + (void)srcBase; + (void)srcStride; + (void)minVal; + (void)minCol; + (void)minRow; + (void)maxVal; + (void)maxCol; + (void)maxRow; +#endif +} + +void minMaxLoc(const Size2D &size, + const s16 * srcBase, ptrdiff_t srcStride, + s16 &minVal, size_t &minCol, size_t &minRow, + s16 &maxVal, size_t &maxCol, size_t &maxRow) +{ + internal::assertSupportedConfiguration(); +#ifdef CAROTENE_NEON + minVal = srcBase[0]; + minCol = 0; + minRow = 0; + maxVal = srcBase[0]; + maxCol = 0; + maxRow = 0; + for(size_t l = 0, i = 0; l < size.height; ++l, i = 0) + { + const s16 * src = internal::getRowPtr( srcBase, srcStride, l); + if (size.width >= 32) + { + u32 tmp0123[4] = { 0, 1, 2, 3 }; + uint32x4_t c8 = vdupq_n_u32(8); + +#if SIZE_MAX > UINT32_MAX + size_t boundAll = size.width - (8 - 1); + for(size_t b = 0; i < boundAll; b = i) + { + size_t bound = std::min(boundAll, b + 0xffffFFF8); +#else + { + size_t bound = size.width - (8 - 1); +#endif + uint32x4_t lineIdxOffset = vld1q_u32(tmp0123); + int16x8_t n_min = vdupq_n_s16(minVal); + uint32x4_t n_minIdxl = vdupq_n_u32(0xffffFFF8); + uint32x4_t n_minIdxh = vdupq_n_u32(0xffffFFF8); + int16x8_t n_max = vdupq_n_s16(maxVal); + uint32x4_t n_maxIdxl = vdupq_n_u32(0xffffFFF8); + uint32x4_t n_maxIdxh = vdupq_n_u32(0xffffFFF8); + + for(; i < bound; i+=8 ) + { + internal::prefetch(src + i); + int16x8_t line = vld1q_s16(src + i); + + uint16x8_t minmask = vcltq_s16(line, n_min); + uint16x8_t maxmask = vcgtq_s16(line, n_max); + + n_min = vbslq_s16(minmask, line, n_min); + uint16x4_t minml = vget_low_u16(minmask); + uint16x4_t minmh = vget_high_u16(minmask); + uint32x4_t minml2 = vmovl_u16(minml); + uint32x4_t minmh2 = vmovl_u16(minmh); + minml2 = vqshlq_n_u32(minml2, 31); + minmh2 = vqshlq_n_u32(minmh2, 31); + n_minIdxl = vbslq_u32(minml2, lineIdxOffset, n_minIdxl); + n_minIdxh = vbslq_u32(minmh2, lineIdxOffset, n_minIdxh); + + n_max = vbslq_s16(maxmask, line, n_max); + uint16x4_t maxml = vget_low_u16(maxmask); + uint16x4_t maxmh = vget_high_u16(maxmask); + uint32x4_t maxml2 = vmovl_u16(maxml); + uint32x4_t maxmh2 = vmovl_u16(maxmh); + maxml2 = vqshlq_n_u32(maxml2, 31); + maxmh2 = vqshlq_n_u32(maxmh2, 31); + n_maxIdxl = vbslq_u32(maxml2, lineIdxOffset, n_maxIdxl); + n_maxIdxh = vbslq_u32(maxmh2, lineIdxOffset, n_maxIdxh); + + // idx[] +=8 + lineIdxOffset = vaddq_u32(lineIdxOffset, c8); + } + + // fix high part of indexes + uint32x4_t c4 = vdupq_n_u32((int32_t) 4); + n_minIdxh = vaddq_u32(n_minIdxh, c4); + n_maxIdxh = vaddq_u32(n_maxIdxh, c4); + + s16 fmin[8], fmax[8]; + u32 fminIdx[8], fmaxIdx[8]; + + vst1q_s16(fmin, n_min); + vst1q_s16(fmax, n_max); + vst1q_u32(fminIdx+0, n_minIdxl); + vst1q_u32(fmaxIdx+0, n_maxIdxl); + vst1q_u32(fminIdx+4, n_minIdxh); + vst1q_u32(fmaxIdx+4, n_maxIdxh); + + size_t minIdx = fminIdx[0]; + size_t maxIdx = fmaxIdx[0]; + minVal = fmin[0]; + maxVal = fmax[0]; + + for (s32 j = 1; j < 8; ++j) + { + s16 minval = fmin[j]; + s16 maxval = fmax[j]; + if (minval < minVal || (minval == minVal && fminIdx[j] < minIdx)) + { + minIdx = fminIdx[j]; + minVal = minval; + } + if (maxval > maxVal || (maxval == maxVal && fmaxIdx[j] < maxIdx)) + { + maxIdx = fmaxIdx[j]; + maxVal = maxval; + } + } + if(minIdx < 0xffffFFF8) + { +#if SIZE_MAX > UINT32_MAX + minCol = b + minIdx; +#else + minCol = minIdx; +#endif + minRow = l; + } + if(maxIdx < 0xffffFFF8) + { +#if SIZE_MAX > UINT32_MAX + maxCol = b + maxIdx; +#else + maxCol = maxIdx; +#endif + maxRow = l; + } + } + } + for(; i < size.width; ++i ) + { + short val = src[i]; + if( val < minVal ) + { + minVal = val; + minCol = i; + minRow = l; + } + else if( val > maxVal ) + { + maxVal = val; + maxCol = i; + maxRow = l; + } + } + } +#else + (void)size; + (void)srcBase; + (void)srcStride; + (void)minVal; + (void)minCol; + (void)minRow; + (void)maxVal; + (void)maxCol; + (void)maxRow; +#endif +} + +void minMaxLoc(const Size2D &size, + const u16 * srcBase, ptrdiff_t srcStride, + u16 &minVal, size_t &minCol, size_t &minRow, + u16 &maxVal, size_t &maxCol, size_t &maxRow) +{ + internal::assertSupportedConfiguration(); +#ifdef CAROTENE_NEON + minVal = srcBase[0]; + minCol = 0; + minRow = 0; + maxVal = srcBase[0]; + maxCol = 0; + maxRow = 0; + for(size_t l = 0, i = 0; l < size.height; ++l, i = 0) + { + const u16 * src = internal::getRowPtr( srcBase, srcStride, l); + if (size.width >= 32) + { + u32 tmp0123[4] = { 0, 1, 2, 3 }; + uint32x4_t c8 = vdupq_n_u32(8); + +#if SIZE_MAX > UINT32_MAX + size_t boundAll = size.width - (8 - 1); + for(size_t b = 0; i < boundAll; b = i) + { + size_t bound = std::min(boundAll, b + 0xffffFFF8); +#else + { + size_t bound = size.width - (8 - 1); +#endif + uint32x4_t lineIdxOffset = vld1q_u32(tmp0123); + uint16x8_t n_min = vdupq_n_u16(minVal); + uint32x4_t n_minIdxl = vdupq_n_u32(0xffffFFF8); + uint32x4_t n_minIdxh = vdupq_n_u32(0xffffFFF8); + uint16x8_t n_max = vdupq_n_u16(maxVal); + uint32x4_t n_maxIdxl = vdupq_n_u32(0xffffFFF8); + uint32x4_t n_maxIdxh = vdupq_n_u32(0xffffFFF8); + + for(; i < bound; i+=8 ) + { + internal::prefetch(src + i); + uint16x8_t line = vld1q_u16(src + i); + + uint16x8_t minmask = vcltq_u16(line, n_min); + uint16x8_t maxmask = vcgtq_u16(line, n_max); + + n_min = vbslq_u16(minmask, line, n_min); + uint16x4_t minml = vget_low_u16(minmask); + uint16x4_t minmh = vget_high_u16(minmask); + uint32x4_t minml2 = vmovl_u16(minml); + uint32x4_t minmh2 = vmovl_u16(minmh); + minml2 = vqshlq_n_u32(minml2, 31); + minmh2 = vqshlq_n_u32(minmh2, 31); + n_minIdxl = vbslq_u32(minml2, lineIdxOffset, n_minIdxl); + n_minIdxh = vbslq_u32(minmh2, lineIdxOffset, n_minIdxh); + + n_max = vbslq_u16(maxmask, line, n_max); + uint16x4_t maxml = vget_low_u16(maxmask); + uint16x4_t maxmh = vget_high_u16(maxmask); + uint32x4_t maxml2 = vmovl_u16(maxml); + uint32x4_t maxmh2 = vmovl_u16(maxmh); + maxml2 = vqshlq_n_u32(maxml2, 31); + maxmh2 = vqshlq_n_u32(maxmh2, 31); + n_maxIdxl = vbslq_u32(maxml2, lineIdxOffset, n_maxIdxl); + n_maxIdxh = vbslq_u32(maxmh2, lineIdxOffset, n_maxIdxh); + + // idx[] +=8 + lineIdxOffset = vaddq_u32(lineIdxOffset, c8); + } + + // fix high part of indexes + uint32x4_t c4 = vdupq_n_u32(4); + n_minIdxh = vaddq_u32(n_minIdxh, c4); + n_maxIdxh = vaddq_u32(n_maxIdxh, c4); + + u16 fmin[8], fmax[8]; + u32 fminIdx[8], fmaxIdx[8]; + + vst1q_u16(fmin, n_min); + vst1q_u16(fmax, n_max); + vst1q_u32(fminIdx+0, n_minIdxl); + vst1q_u32(fmaxIdx+0, n_maxIdxl); + vst1q_u32(fminIdx+4, n_minIdxh); + vst1q_u32(fmaxIdx+4, n_maxIdxh); + + size_t minIdx = fminIdx[0]; + size_t maxIdx = fmaxIdx[0]; + minVal = fmin[0]; + maxVal = fmax[0]; + + for (s32 j = 1; j < 8; ++j) + { + u16 minval = fmin[j]; + u16 maxval = fmax[j]; + if (minval < minVal || (minval == minVal && fminIdx[j] < minIdx)) + { + minIdx = fminIdx[j]; + minVal = minval; + } + if (maxval > maxVal || (maxval == maxVal && fmaxIdx[j] < maxIdx)) + { + maxIdx = fmaxIdx[j]; + maxVal = maxval; + } + } + if(minIdx < 0xffffFFF8) + { +#if SIZE_MAX > UINT32_MAX + minCol = b + minIdx; +#else + minCol = minIdx; +#endif + minRow = l; + } + if(maxIdx < 0xffffFFF8) + { +#if SIZE_MAX > UINT32_MAX + maxCol = b + maxIdx; +#else + maxCol = maxIdx; +#endif + maxRow = l; + } + } + } + for(; i < size.width; ++i ) + { + u16 val = src[i]; + if( val < minVal ) + { + minVal = val; + minCol = i; + minRow = l; + } + else if( val > maxVal ) + { + maxVal = val; + maxCol = i; + maxRow = l; + } + } + } +#else + (void)size; + (void)srcBase; + (void)srcStride; + (void)minVal; + (void)minCol; + (void)minRow; + (void)maxVal; + (void)maxCol; + (void)maxRow; +#endif +} + +#ifdef CAROTENE_NEON +namespace { + +void minMaxLocBlock(const u8 * src, u32 len, + u8 &minVal, u16 &minIdx, + u8 &maxVal, u16 &maxIdx) +{ + u16 tmp0123[8] = { 0, 1, 2, 3, 4, 5, 6, 7 }; + + uint8x16_t n_min = vdupq_n_u8(src[0]); + uint16x8_t n_minIdxl = vdupq_n_u16(0); + uint16x8_t n_minIdxh = vdupq_n_u16(0); + uint8x16_t n_max = vdupq_n_u8(src[0]); + uint16x8_t n_maxIdxl = vdupq_n_u16(0); + uint16x8_t n_maxIdxh = vdupq_n_u16(0); + uint16x8_t c16 = vdupq_n_u16(16); + uint16x8_t lineIdxOffset = vld1q_u16(tmp0123); + + s32 i = 0; + s32 bound = len - (16 - 1); + for(; i < bound; i+=16 ) + { + internal::prefetch(src + i); + uint8x16_t line = vld1q_u8(src + i); + + uint8x16_t minmask = vcltq_u8(line, n_min); + uint8x16_t maxmask = vcgtq_u8(line, n_max); + + n_min = vbslq_u8(minmask, line, n_min); + uint8x8_t minml = vget_low_u8(minmask); + uint8x8_t minmh = vget_high_u8(minmask); + uint16x8_t minml2 = vmovl_u8(minml); + uint16x8_t minmh2 = vmovl_u8(minmh); + minml2 = vqshlq_n_u16(minml2, 15); + minmh2 = vqshlq_n_u16(minmh2, 15); + n_minIdxl = vbslq_u16(minml2, lineIdxOffset, n_minIdxl); + n_minIdxh = vbslq_u16(minmh2, lineIdxOffset, n_minIdxh); + + n_max = vbslq_u8(maxmask, line, n_max); + uint8x8_t maxml = vget_low_u8(maxmask); + uint8x8_t maxmh = vget_high_u8(maxmask); + uint16x8_t maxml2 = vmovl_u8(maxml); + uint16x8_t maxmh2 = vmovl_u8(maxmh); + maxml2 = vqshlq_n_u16(maxml2, 15); + maxmh2 = vqshlq_n_u16(maxmh2, 15); + n_maxIdxl = vbslq_u16(maxml2, lineIdxOffset, n_maxIdxl); + n_maxIdxh = vbslq_u16(maxmh2, lineIdxOffset, n_maxIdxh); + + // idx[] +=16 + lineIdxOffset = vaddq_u16(lineIdxOffset, c16); + } + + // fix high part of indexes + uint16x8_t c8 = vdupq_n_u16(8); + n_minIdxh = vaddq_u16(n_minIdxh, c8); + n_maxIdxh = vaddq_u16(n_maxIdxh, c8); + + u8 fmin[16], fmax[16]; + u16 fminIdx[16], fmaxIdx[16]; + /*{ + uint8x8_t min_low = vget_low_u8(n_min); + uint8x8_t min_high = vget_high_u8(n_min); + uint8x8_t max_low = vget_low_u8(n_max); + uint8x8_t max_high = vget_high_u8(n_max); + + uint8x8_t minmask = vclt_u8(min_low, min_high); + uint8x8_t maxmask = vcgt_u8(max_low, max_high); + + uint8x8_t min2 = vbsl_u8(minmask, min_low, min_high); + uint8x8_t max2 = vbsl_u8(maxmask, max_low, max_high); + + uint16x8_t minidxmask = vmovl_u8(minmask); + uint16x8_t maxidxmask = vmovl_u8(maxmask); + minidxmask = vqshlq_n_u16(minidxmask, 15); + maxidxmask = vqshlq_n_u16(maxidxmask, 15); + + uint16x8_t n_minIdx = vbslq_u16(minidxmask, n_minIdxl, n_minIdxh); + uint16x8_t n_maxIdx = vbslq_u16(maxidxmask, n_maxIdxl, n_maxIdxh); + + vst1_u8((uint8_t*)fmin, min2); + vst1_u8((uint8_t*)fmax, max2); + + vst1q_u16((uint16_t*)(fminIdx), n_minIdx); + vst1q_u16((uint16_t*)(fmaxIdx), n_maxIdx); + }*/ + + vst1q_u8(fmin, n_min); + vst1q_u8(fmax, n_max); + vst1q_u16(fminIdx+0, n_minIdxl); + vst1q_u16(fmaxIdx+0, n_maxIdxl); + vst1q_u16(fminIdx+8, n_minIdxh); + vst1q_u16(fmaxIdx+8, n_maxIdxh); + + minIdx = fminIdx[0]; + maxIdx = fmaxIdx[0]; + minVal = fmin[0]; + maxVal = fmax[0]; + + for (s32 j = 1; j < 16; ++j) + { + u8 minval = fmin[j]; + u8 maxval = fmax[j]; + if (minval < minVal || (minval == minVal && fminIdx[j] < minIdx)) + { + minIdx = fminIdx[j]; + minVal = minval; + } + if (maxval > maxVal || (maxval == maxVal && fmaxIdx[j] < maxIdx)) + { + maxIdx = fmaxIdx[j]; + maxVal = maxval; + } + } + + for(; i < (s32)len; ++i ) + { + u8 val = src[i]; + if( val < minVal ) + { + minVal = val; + minIdx = (u16)i; + } + else if( val > maxVal ) + { + maxVal = val; + maxIdx = (u16)i; + } + } +} + +void minMaxLocBlock(const s8 * src, u32 len, + s8 &minVal, u16 &minIdx, + s8 &maxVal, u16 &maxIdx) +{ + u16 tmp0123[16] = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 }; + + int8x16_t n_min = vdupq_n_s8(src[0]); + uint16x8_t n_minIdxl = vdupq_n_u16(0); + uint16x8_t n_minIdxh = vdupq_n_u16(0); + int8x16_t n_max = vdupq_n_s8(src[0]); + uint16x8_t n_maxIdxl = vdupq_n_u16(0); + uint16x8_t n_maxIdxh = vdupq_n_u16(0); + uint16x8_t c16 = vdupq_n_u16(16); + uint16x8_t lineIdxOffset = vld1q_u16(tmp0123); + + s32 i = 0; + s32 bound = len - (16 - 1); + for(; i < bound; i+=16 ) + { + internal::prefetch(src + i); + int8x16_t line = vld1q_s8(src + i); + + uint8x16_t minmask = vcltq_s8(line, n_min); + uint8x16_t maxmask = vcgtq_s8(line, n_max); + + n_min = vbslq_s8(minmask, line, n_min); + uint8x8_t minml = vget_low_u8(minmask); + uint8x8_t minmh = vget_high_u8(minmask); + uint16x8_t minml2 = vmovl_u8(minml); + uint16x8_t minmh2 = vmovl_u8(minmh); + minml2 = vqshlq_n_u16(minml2, 15); + minmh2 = vqshlq_n_u16(minmh2, 15); + n_minIdxl = vbslq_u16(minml2, lineIdxOffset, n_minIdxl); + n_minIdxh = vbslq_u16(minmh2, lineIdxOffset, n_minIdxh); + + n_max = vbslq_s8(maxmask, line, n_max); + uint8x8_t maxml = vget_low_u8(maxmask); + uint8x8_t maxmh = vget_high_u8(maxmask); + uint16x8_t maxml2 = vmovl_u8(maxml); + uint16x8_t maxmh2 = vmovl_u8(maxmh); + maxml2 = vqshlq_n_u16(maxml2, 15); + maxmh2 = vqshlq_n_u16(maxmh2, 15); + n_maxIdxl = vbslq_u16(maxml2, lineIdxOffset, n_maxIdxl); + n_maxIdxh = vbslq_u16(maxmh2, lineIdxOffset, n_maxIdxh); + + // idx[] +=16 + lineIdxOffset = vaddq_u16(lineIdxOffset, c16); + } + + // fix high part of indexes + uint16x8_t c8 = vdupq_n_u16(8); + n_minIdxh = vaddq_u16(n_minIdxh, c8); + n_maxIdxh = vaddq_u16(n_maxIdxh, c8); + + s8 fmin[16], fmax[16]; + u16 fminIdx[16], fmaxIdx[16]; + + vst1q_s8(fmin, n_min); + vst1q_s8(fmax, n_max); + vst1q_u16(fminIdx+0, n_minIdxl); + vst1q_u16(fmaxIdx+0, n_maxIdxl); + vst1q_u16(fminIdx+8, n_minIdxh); + vst1q_u16(fmaxIdx+8, n_maxIdxh); + + minIdx = fminIdx[0]; + maxIdx = fmaxIdx[0]; + minVal = fmin[0]; + maxVal = fmax[0]; + + for (s32 j = 1; j < 16; ++j) + { + s8 minval = fmin[j]; + s8 maxval = fmax[j]; + if (minval < minVal || (minval == minVal && fminIdx[j] < minIdx)) + { + minIdx = fminIdx[j]; + minVal = minval; + } + if (maxval > maxVal || (maxval == maxVal && fmaxIdx[j] < maxIdx)) + { + maxIdx = fmaxIdx[j]; + maxVal = maxval; + } + } + + for(; i < (s32)len; ++i ) + { + s8 val = src[i]; + if( val < minVal ) + { + minVal = val; + minIdx = (u16)i; + } + else if( val > maxVal ) + { + maxVal = val; + maxIdx = (u16)i; + } + } +} + +} // namespace +#endif // CAROTENE_NEON + +#define USHORT_BLOCK_MAX_SIZE (1 << 16) + +void minMaxLoc(const Size2D &size, + const u8 * srcBase, ptrdiff_t srcStride, + u8 &minVal, size_t &minCol, size_t &minRow, + u8 &maxVal, size_t &maxCol, size_t &maxRow) +{ + internal::assertSupportedConfiguration(); +#ifdef CAROTENE_NEON + minVal = srcBase[0]; + minCol = 0; + minRow = 0; + maxVal = srcBase[0]; + maxCol = 0; + maxRow = 0; + for(size_t l = 0; l < size.height; ++l) + { + const u8 * src = internal::getRowPtr( srcBase, srcStride, l); + if (size.width > 128) + { + for(size_t blockStart = 0; blockStart < size.width; blockStart += USHORT_BLOCK_MAX_SIZE) + { + u8 locMinVal, locMaxVal; + u16 locMinIdx, locMaxIdx; + size_t tail = size.width - blockStart; + minMaxLocBlock(src + blockStart, tail < USHORT_BLOCK_MAX_SIZE ? tail : USHORT_BLOCK_MAX_SIZE, + locMinVal, locMinIdx, locMaxVal, locMaxIdx); + + if (locMinVal == 0 && locMaxVal == 255) + { + minCol = blockStart + locMinIdx; + maxCol = blockStart + locMaxIdx; + minRow = l; + maxRow = l; + minVal = 0; + maxVal = 255; + return; + } + else + { + if (locMinVal < minVal) + { + minCol = blockStart + locMinIdx; + minRow = l; + minVal = locMinVal; + } + if (locMaxVal > maxVal) + { + maxCol = blockStart + locMaxIdx; + maxRow = l; + maxVal = locMaxVal; + } + } + } + } + else + { + for(size_t i = 0; i < size.width; ++i ) + { + u8 val = src[i]; + if( val < minVal ) + { + minVal = val; + minCol = i; + minRow = l; + } + else if( val > maxVal ) + { + maxVal = val; + maxCol = i; + maxRow = l; + } + } + } + + } +#else + (void)size; + (void)srcBase; + (void)srcStride; + (void)minVal; + (void)minCol; + (void)minRow; + (void)maxVal; + (void)maxCol; + (void)maxRow; +#endif +} + +void minMaxLoc(const Size2D &size, + const s8 * srcBase, ptrdiff_t srcStride, + s8 &minVal, size_t &minCol, size_t &minRow, + s8 &maxVal, size_t &maxCol, size_t &maxRow) +{ + internal::assertSupportedConfiguration(); +#ifdef CAROTENE_NEON + minVal = srcBase[0]; + minCol = 0; + minRow = 0; + maxVal = srcBase[0]; + maxCol = 0; + maxRow = 0; + for(size_t l = 0; l < size.height; ++l) + { + const s8 * src = internal::getRowPtr( srcBase, srcStride, l); + if (size.width > 128) + { + for(size_t blockStart = 0; blockStart < size.width; blockStart += USHORT_BLOCK_MAX_SIZE) + { + s8 locMinVal, locMaxVal; + u16 locMinIdx, locMaxIdx; + size_t tail = size.width - blockStart; + minMaxLocBlock(src + blockStart, tail < USHORT_BLOCK_MAX_SIZE ? tail : USHORT_BLOCK_MAX_SIZE, + locMinVal, locMinIdx, locMaxVal, locMaxIdx); + + if (locMinVal == -128 && locMaxVal == 127) + { + minCol = blockStart + locMinIdx; + maxCol = blockStart + locMaxIdx; + minRow = l; + maxRow = l; + minVal = -128; + maxVal = 127; + return; + } + else + { + if (locMinVal < minVal) + { + minCol = blockStart + locMinIdx; + minRow = l; + minVal = locMinVal; + } + if (locMaxVal > maxVal) + { + maxCol = blockStart + locMaxIdx; + maxRow = l; + maxVal = locMaxVal; + } + } + } + } + else + { + for(size_t i = 0; i < size.width; ++i ) + { + s8 val = src[i]; + if( val < minVal ) + { + minVal = val; + minRow = l; + minCol = i; + } + else if( val > maxVal ) + { + maxVal = val; + maxRow = l; + maxCol = i; + } + } + } + } +#else + (void)size; + (void)srcBase; + (void)srcStride; + (void)minVal; + (void)minCol; + (void)minRow; + (void)maxVal; + (void)maxCol; + (void)maxRow; +#endif +} + +} // namespace CAROTENE_NS diff --git a/3rdparty/carotene/src/morph.cpp b/3rdparty/carotene/src/morph.cpp new file mode 100644 index 0000000000..bcc6aa7e06 --- /dev/null +++ b/3rdparty/carotene/src/morph.cpp @@ -0,0 +1,728 @@ +/* + * By downloading, copying, installing or using the software you agree to this license. + * If you do not agree to this license, do not download, install, + * copy or use the software. + * + * + * License Agreement + * For Open Source Computer Vision Library + * (3-clause BSD License) + * + * Copyright (C) 2014, NVIDIA Corporation, all rights reserved. + * Third party copyrights are property of their respective owners. + * + * Redistribution and use in source and binary forms, with or without modification, + * are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * * Neither the names of the copyright holders nor the names of the contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * This software is provided by the copyright holders and contributors "as is" and + * any express or implied warranties, including, but not limited to, the implied + * warranties of merchantability and fitness for a particular purpose are disclaimed. + * In no event shall copyright holders or contributors be liable for any direct, + * indirect, incidental, special, exemplary, or consequential damages + * (including, but not limited to, procurement of substitute goods or services; + * loss of use, data, or profits; or business interruption) however caused + * and on any theory of liability, whether in contract, strict liability, + * or tort (including negligence or otherwise) arising in any way out of + * the use of this software, even if advised of the possibility of such damage. + */ + +#include "common.hpp" + +#include +#include +#include +#include + +namespace CAROTENE_NS { + +bool isMorph3x3Supported(const Size2D &size, BORDER_MODE border) +{ + return isSupportedConfiguration() && size.width >= 16 && + (border == BORDER_MODE_CONSTANT || + border == BORDER_MODE_REPLICATE); +} + +#ifdef CAROTENE_NEON + +namespace { + +struct ErodeVecOp +{ + ErodeVecOp():borderValue(0){} + + ErodeVecOp(BORDER_MODE border, u8 borderValue_) : + borderValue(borderValue_) + { + if (border == BORDER_MODE_REPLICATE) + borderValue = std::numeric_limits::max(); + } + + inline uint8x16_t operator()(uint8x16_t a, uint8x16_t b) const + { + return vminq_u8(a, b); + } + + inline uint8x8_t operator()(uint8x8_t a, uint8x8_t b) const + { + return vmin_u8(a, b); + } + + inline u8 operator()(u8 a, u8 b) const + { + return std::min(a, b); + } + + u8 borderValue; +}; + +struct DilateVecOp +{ + DilateVecOp():borderValue(0){} + + DilateVecOp(BORDER_MODE border, u8 borderValue_) : + borderValue(borderValue_) + { + if (border == BORDER_MODE_REPLICATE) + borderValue = std::numeric_limits::min(); + } + + inline uint8x16_t operator()(uint8x16_t a, uint8x16_t b) const + { + return vmaxq_u8(a, b); + } + + inline uint8x8_t operator()(uint8x8_t a, uint8x8_t b) const + { + return vmax_u8(a, b); + } + + inline u8 operator()(u8 a, u8 b) const + { + return std::max(a, b); + } + + u8 borderValue; +}; + +template +void morph3x3(const Size2D &size, + const u8 * srcBase, ptrdiff_t srcStride, + u8 * dstBase, ptrdiff_t dstStride, + BORDER_MODE border, const VecOp & vop) +{ + u8 borderValue = vop.borderValue; + ptrdiff_t width = (ptrdiff_t)size.width, height = (ptrdiff_t)size.height; + + const uint8x16_t v_zero = vdupq_n_u8(0); + const uint8x16_t v_border = vdupq_n_u8(borderValue); + + uint8x16_t tprev = v_zero, tcurr = v_zero, tnext = v_zero; + uint8x16_t t0 = v_zero, t1 = v_zero, t2 = v_zero; + + for (ptrdiff_t y = 0; y < height; ++y) + { + const u8 * srow0 = y == 0 && border == BORDER_MODE_CONSTANT ? NULL : internal::getRowPtr(srcBase, srcStride, std::max(y - 1, 0)); + const u8 * srow1 = internal::getRowPtr(srcBase, srcStride, y); + const u8 * srow2 = y + 1 == height && border == BORDER_MODE_CONSTANT ? NULL : internal::getRowPtr(srcBase, srcStride, std::min(y + 1, height - 1)); + u8 * drow = internal::getRowPtr(dstBase, dstStride, y); + + u8 prevx = 0, currx = 0, nextx = 0; + ptrdiff_t x = 0; + const ptrdiff_t bwidth = y + 2 < height ? width : (width - 16); + + // perform vertical convolution + for ( ; x <= bwidth; x += 16) + { + internal::prefetch(srow0 + x); + internal::prefetch(srow1 + x); + internal::prefetch(srow2 + x); + + uint8x16_t x0 = !srow0 ? v_border : vld1q_u8(srow0 + x); + uint8x16_t x1 = vld1q_u8(srow1 + x); + uint8x16_t x2 = !srow2 ? v_border : vld1q_u8(srow2 + x); + + // calculate values for plain CPU part below if needed + if (x + 16 >= bwidth) + { + ptrdiff_t x3 = x == width ? width - 1 : x; + ptrdiff_t x4 = border == BORDER_MODE_CONSTANT ? x3 - 1 : std::max(x3 - 1, 0); + + if (border == BORDER_MODE_CONSTANT && x4 < 0) + prevx = borderValue; + else + prevx = vop(srow1[x4], + vop(srow2 ? srow2[x4] : borderValue, + srow0 ? srow0[x4] : borderValue)); + + currx = vop(srow2 ? srow2[x3] : borderValue, vop(srow1[x3], srow0 ? srow0[x3] : borderValue)); + } + + // make shift + if (x) + { + tprev = tcurr; + tcurr = tnext; + } + + // and calculate next value + tnext = vop(vop(x0, x1), x2); + + // make extrapolation for the first elements + if (!x) + { + // make border + if (border == BORDER_MODE_CONSTANT) + tcurr = v_border; + else if (border == BORDER_MODE_REPLICATE) + tcurr = vdupq_n_u8(vgetq_lane_u8(tnext, 0)); + + continue; + } + + // combine 3 "shifted" vectors + t0 = vextq_u8(tprev, tcurr, 15); + t1 = tcurr; + t2 = vextq_u8(tcurr, tnext, 1); + + // and add them + t0 = vop(t0, vop(t1, t2)); + + vst1q_u8(drow + x - 16, t0); + } + + x -= 16; + if (x == width) + --x; + + for ( ; x < width; ++x) + { + // make extrapolation for the last elements + if (x + 1 >= width) + { + if (border == BORDER_MODE_CONSTANT) + nextx = borderValue; + else if (border == BORDER_MODE_REPLICATE) + nextx = vop(srow2[x], vop(srow1[x], srow0[x])); + } + else + nextx = vop(vop(srow2 ? srow2[x + 1] : borderValue, + srow0 ? srow0[x + 1] : borderValue), + srow1[x + 1]); + + drow[x] = vop(prevx, vop(currx, nextx)); + + // make shift + prevx = currx; + currx = nextx; + } + } +} + +} // namespace + +#endif + +void erode3x3(const Size2D &size, + const u8 * srcBase, ptrdiff_t srcStride, + u8 * dstBase, ptrdiff_t dstStride, + BORDER_MODE border, u8 borderValue) +{ + internal::assertSupportedConfiguration(isMorph3x3Supported(size, border)); +#ifdef CAROTENE_NEON + morph3x3(size, + srcBase, srcStride, + dstBase, dstStride, + border, ErodeVecOp(border, borderValue)); +#else + (void)size; + (void)srcBase; + (void)srcStride; + (void)dstBase; + (void)dstStride; + (void)border; + (void)borderValue; +#endif +} + +void dilate3x3(const Size2D &size, + const u8 * srcBase, ptrdiff_t srcStride, + u8 * dstBase, ptrdiff_t dstStride, + BORDER_MODE border, u8 borderValue) +{ + internal::assertSupportedConfiguration(isMorph3x3Supported(size, border)); +#ifdef CAROTENE_NEON + morph3x3(size, + srcBase, srcStride, + dstBase, dstStride, + border, DilateVecOp(border, borderValue)); +#else + (void)size; + (void)srcBase; + (void)srcStride; + (void)dstBase; + (void)dstStride; + (void)border; + (void)borderValue; +#endif +} + +#ifdef CAROTENE_NEON +namespace { + +template +void MorphRow(const u8* src, u8* dst, size_t width, s32 cn, size_t ksize) +{ + size_t i, j, k; + size_t width16 = (width & -16) * cn; + size_t width8 = (width & -8) * cn; + width *= cn; + + if (ksize == 1) + { + for (i = 0; i < width; i++) + dst[i] = src[i]; + return; + } + + ksize = ksize*cn; + VecUpdate updateOp; + switch(cn) + { + case 1: + for (i = 0; i < width16; i += 16) + { + const u8* sptr = src + i; + uint8x16_t s = vld1q_u8(sptr); + internal::prefetch(sptr); + + for( k = 1; k < ksize; ++k) + s = updateOp(s, vld1q_u8(sptr + k)); + + vst1q_u8(dst + i, s); + } + + for (; i < width8; i += 8) + { + const u8* sptr = src + i; + uint8x8_t s = vld1_u8(sptr); + internal::prefetch(sptr); + + for( k = 1; k < ksize; ++k) + s = updateOp(s, vld1_u8(sptr + k)); + + vst1_u8(dst + i, s); + } + break; + default: + for (i = 0; i < width16; i += 16) + { + uint8x16_t s = vld1q_u8(src + i); + internal::prefetch(src + i); + + for (k = cn; k < ksize; k += cn) + s = updateOp(s, vld1q_u8(src + i + k)); + + vst1q_u8(dst + i, s); + } + + for (; i < width8; i += 8) + { + uint8x8_t s = vld1_u8(src + i); + internal::prefetch(src + i); + + for (k = cn; k < ksize; k += cn) + s = updateOp(s, vld1_u8(src + i + k)); + + vst1_u8(dst + i, s); + } + break; + } + + ptrdiff_t i0 = i; + for( k = 0; k < (size_t)cn; k++, src++, dst++ ) + { + for( i = i0; i <= width - cn*2; i += cn*2 ) + { + const u8* s = src + i; + u8 m = s[cn]; + for( j = cn*2; j < ksize; j += cn ) + m = updateOp(m, s[j]); + dst[i] = updateOp(m, s[0]); + dst[i+cn] = updateOp(m, s[j]); + } + + for( ; i < width; i += cn ) + { + const u8* s = src + i; + u8 m = s[0]; + for( j = cn; j < ksize; j += cn ) + m = updateOp(m, s[j]); + dst[i] = m; + } + } +} + +template +void MorphColumn(const u8** src, u8* dst, ptrdiff_t dststep, size_t count, size_t width, size_t ksize) +{ + size_t i, k; + size_t width32 = width & -32; + VecUpdate updateOp; + + uint8x16_t x0,x1,s0,s1; + if (ksize == 3) + { + for (; count > 1; count -= 2, dst += dststep * 2, src += 2) + { + for (i = 0; i < width32; i += 32) + { + const u8* sptr = src[1] + i; + s0 = vld1q_u8(sptr); + s1 = vld1q_u8(sptr + 16); + internal::prefetch(sptr); + + sptr = src[2] + i; + x0 = vld1q_u8(sptr); + x1 = vld1q_u8(sptr + 16); + internal::prefetch(sptr); + + s0 = updateOp(s0, x0); + s1 = updateOp(s1, x1); + + sptr = src[0] + i; + x0 = vld1q_u8(sptr); + x1 = vld1q_u8(sptr + 16); + internal::prefetch(sptr); + + vst1q_u8(dst+i, updateOp(s0, x0)); + vst1q_u8(dst+i+16, updateOp(s1, x1)); + + sptr = src[3] + i; + x0 = vld1q_u8(sptr); + x1 = vld1q_u8(sptr + 16); + internal::prefetch(sptr); + vst1q_u8(dst + dststep + i, updateOp(s0, x0)); + vst1q_u8(dst + dststep + i + 16, updateOp(s1, x1)); + + } + for(; i < width; i++ ) + { + u8 s = src[1][i]; + + for( k = 2; k < ksize; k++ ) + s = updateOp(s, src[k][i]); + + dst[i] = updateOp(s, src[0][i]); + dst[i+dststep] = updateOp(s, src[k][i]); + } + } + } + else if (ksize > 1) + for (; count > 1; count -= 2, dst += dststep*2, src += 2) + { + for (i = 0; i < width32; i += 32) + { + const u8* sptr = src[1] + i; + s0 = vld1q_u8(sptr); + s1 = vld1q_u8(sptr + 16); + internal::prefetch(sptr); + for (k = 2; k < ksize; k++) + { + sptr = src[k] + i; + x0 = vld1q_u8(sptr); + x1 = vld1q_u8(sptr + 16); + internal::prefetch(sptr); + + s0 = updateOp(s0, x0); + s1 = updateOp(s1, x1); + } + + sptr = src[0] + i; + x0 = vld1q_u8(sptr); + x1 = vld1q_u8(sptr + 16); + internal::prefetch(sptr); + + vst1q_u8(dst+i, updateOp(s0, x0)); + vst1q_u8(dst+i+16, updateOp(s1, x1)); + + sptr = src[k] + i; + x0 = vld1q_u8(sptr); + x1 = vld1q_u8(sptr + 16); + internal::prefetch(sptr); + vst1q_u8(dst + dststep + i, updateOp(s0, x0)); + vst1q_u8(dst + dststep + i + 16, updateOp(s1, x1)); + } + for(; i < width; i++ ) + { + u8 s = src[1][i]; + + for( k = 2; k < ksize; k++ ) + s = updateOp(s, src[k][i]); + + dst[i] = updateOp(s, src[0][i]); + dst[i+dststep] = updateOp(s, src[k][i]); + } + } + + for (; count > 0; count--, dst += dststep, src++) + { + for (i = 0; i < width32; i += 32) + { + const u8* sptr = src[0] + i; + s0 = vld1q_u8(sptr); + s1 = vld1q_u8(sptr + 16); + internal::prefetch(sptr); + + for (k = 1; k < ksize; k++) + { + sptr = src[k] + i; + x0 = vld1q_u8(sptr); + x1 = vld1q_u8(sptr + 16); + internal::prefetch(sptr); + s0 = updateOp(s0, x0); + s1 = updateOp(s1, x1); + } + + vst1q_u8(dst + i, s0); + vst1q_u8(dst + i + 16, s1); + } + for(; i < width; i++ ) + { + u8 s = src[0][i]; + for( k = 1; k < ksize; k++ ) + s = updateOp(s, src[k][i]); + dst[i] = s; + } + } +} + +template +inline void morphology(const Size2D &ssize, u32 cn, + const u8 * srcBase, ptrdiff_t srcStride, + u8 * dstBase, ptrdiff_t dstStride, + const Size2D &ksize, + size_t anchorX, size_t anchorY, + BORDER_MODE rowBorderType, BORDER_MODE columnBorderType, + const u8 * borderValues, Margin borderMargin) +{ + //Temporary buffers common for all iterations + std::vector _srcRow(cn*(ssize.width + ksize.width - 1)); + u8* srcRow = &_srcRow[0]; + + size_t bufRows = std::max(ksize.height + 3, std::max(anchorY, ksize.height-anchorY-1)*2+1); + std::vector _rows(bufRows); + u8** rows = &_rows[0]; + + // adjust swidthcn so that the used part of buffers stays compact in memory + ptrdiff_t swidthcn = cn*((ssize.width + 15) & -16);// cn * (aligned ssize.width size) + std::vector _ringBuf(swidthcn*bufRows+16); + u8 * ringBuf = internal::alignPtr(&_ringBuf[0], 16); + + size_t borderLength = std::max(ksize.width - 1, 1) * cn; + std::vector _borderTab(borderLength); + ptrdiff_t * borderTab = &_borderTab[0]; + + std::vector _constBorderValue; + std::vector _constBorderRow; + u8 * constBorderValue = NULL; + u8 * constBorderRow = NULL; + if( rowBorderType == BORDER_MODE_CONSTANT || columnBorderType == BORDER_MODE_CONSTANT ) + { + _constBorderValue.resize(borderLength); + constBorderValue = &_constBorderValue[0]; + size_t i; + for(i = 0; i < cn; i++) + constBorderValue[i] = borderValues[i]; + for(; i < borderLength; i++) + constBorderValue[i] = constBorderValue[i-cn]; + + if( columnBorderType == BORDER_MODE_CONSTANT ) + { + _constBorderRow.resize(cn*(ssize.width + ksize.width - 1 + 16)); + constBorderRow = internal::alignPtr(&_constBorderRow[0], 16); + size_t N = (ssize.width + ksize.width - 1)*cn; + for( i = 0; i < N; i += borderLength ) + { + size_t n = std::min( borderLength, N - i ); + for(size_t j = 0; j < n; j++) + srcRow[i+j] = constBorderValue[j]; + } + MorphRow(srcRow, constBorderRow, ssize.width, cn, ksize.width); + } + } + + Size2D wholeSize(ssize.width + borderMargin.left + borderMargin.right, + ssize.height + borderMargin.top + borderMargin.bottom); + + ptrdiff_t dx1 = std::max(anchorX - (ptrdiff_t)borderMargin.left, 0); + ptrdiff_t dx2 = std::max((ptrdiff_t)ksize.width - anchorX - 1 - (ptrdiff_t)borderMargin.right, 0); + // recompute border tables + if( dx1 > 0 || dx2 > 0 ) + { + if( rowBorderType == BORDER_MODE_CONSTANT ) + { + memcpy( srcRow, &constBorderValue[0], dx1*cn ); + memcpy( srcRow + (ssize.width + ksize.width - 1 - dx2)*cn, &constBorderValue[0], dx2*cn ); + } + else + { + ptrdiff_t xofs1 = std::min(borderMargin.left, anchorX) - borderMargin.left; + + ptrdiff_t wholeWidth = wholeSize.width; + + ptrdiff_t i, j; + for( i = 0; i < dx1; i++ ) + { + ptrdiff_t p0 = (internal::borderInterpolate(i-dx1, wholeWidth, rowBorderType) + xofs1)*cn; + for( j = 0; j < (ptrdiff_t)cn; j++ ) + borderTab[i*cn + j] = p0 + j; + } + + for( i = 0; i < dx2; i++ ) + { + ptrdiff_t p0 = (internal::borderInterpolate(wholeWidth + i, wholeWidth, rowBorderType) + xofs1)*cn; + for( j = 0; j < (ptrdiff_t)cn; j++ ) + borderTab[(i + dx1)*cn + j] = p0 + j; + } + } + } + + ptrdiff_t startY, startY0, endY, rowCount; + startY = startY0 = std::max(borderMargin.top - anchorY, 0); + endY = std::min(borderMargin.top + ssize.height + ksize.height - anchorY - 1, wholeSize.height); + + const u8* src = srcBase + (startY - borderMargin.top)*srcStride; + u8* dst = dstBase; + + ptrdiff_t width = ssize.width, kwidth = ksize.width; + ptrdiff_t kheight = ksize.height, ay = anchorY; + ptrdiff_t width1 = ssize.width + kwidth - 1; + ptrdiff_t xofs1 = std::min(borderMargin.left, anchorX); + bool makeBorder = (dx1 > 0 || dx2 > 0) && rowBorderType != BORDER_MODE_CONSTANT; + ptrdiff_t dy = 0, i = 0; + + src -= xofs1*cn; + ptrdiff_t count = endY - startY; + + rowCount = 0; + for(;; dst += dstStride*i, dy += i) + { + ptrdiff_t dcount = bufRows - ay - startY - rowCount + borderMargin.top; + dcount = dcount > 0 ? dcount : bufRows - kheight + 1; + dcount = std::min(dcount, count); + count -= dcount; + for( ; dcount-- > 0; src += srcStride ) + { + ptrdiff_t bi = (startY - startY0 + rowCount) % bufRows; + u8* brow = ringBuf + bi*swidthcn; + + if( (size_t)(++rowCount) > bufRows ) + { + --rowCount; + ++startY; + } + + memcpy( srcRow + dx1*cn, src, (width1 - dx2 - dx1)*cn ); + + if( makeBorder ) + { + for( i = 0; i < (ptrdiff_t)(dx1*cn); i++ ) + srcRow[i] = src[borderTab[i]]; + for( i = 0; i < (ptrdiff_t)(dx2*cn); i++ ) + srcRow[i + (width1 - dx2)*cn] = src[borderTab[i+dx1*cn]]; + } + + MorphRow(srcRow, brow, width, cn, ksize.width); + } + + ptrdiff_t max_i = std::min(bufRows, ssize.height - dy + (kheight - 1)); + for( i = 0; i < max_i; i++ ) + { + ptrdiff_t srcY = internal::borderInterpolate(dy + i + borderMargin.top - ay, + wholeSize.height, columnBorderType); + if( srcY < 0 ) // can happen only with constant border type + rows[i] = constBorderRow; + else + { + if( srcY >= startY + rowCount ) + break; + ptrdiff_t bi = (srcY - startY0) % bufRows; + rows[i] = ringBuf + bi*swidthcn; + } + } + if( i < kheight ) + break; + i -= kheight - 1; + MorphColumn((const u8**)rows, dst, dstStride, i, ssize.width*cn, ksize.height); + } +} + +} // namespace +#endif // CAROTENE_NEON + +void erode(const Size2D &ssize, u32 cn, + const u8 * srcBase, ptrdiff_t srcStride, + u8 * dstBase, ptrdiff_t dstStride, + const Size2D &ksize, + size_t anchorX, size_t anchorY, + BORDER_MODE rowBorderType, BORDER_MODE columnBorderType, + const u8 * borderValues, Margin borderMargin) +{ + internal::assertSupportedConfiguration(ssize.width > 0 && ssize.height > 0 && + anchorX < ksize.width && anchorY < ksize.height); +#ifdef CAROTENE_NEON + morphology(ssize, cn, srcBase, srcStride, dstBase, dstStride, + ksize, anchorX, anchorY, rowBorderType, columnBorderType, + borderValues, borderMargin); +#else + (void)cn; + (void)srcBase; + (void)srcStride; + (void)dstBase; + (void)dstStride; + (void)rowBorderType; + (void)columnBorderType; + (void)borderValues; + (void)borderMargin; +#endif +} + +void dilate(const Size2D &ssize, u32 cn, + const u8 * srcBase, ptrdiff_t srcStride, + u8 * dstBase, ptrdiff_t dstStride, + const Size2D &ksize, + size_t anchorX, size_t anchorY, + BORDER_MODE rowBorderType, BORDER_MODE columnBorderType, + const u8 * borderValues, Margin borderMargin) +{ + internal::assertSupportedConfiguration(ssize.width > 0 && ssize.height > 0 && + anchorX < ksize.width && anchorY < ksize.height); +#ifdef CAROTENE_NEON + morphology(ssize, cn, srcBase, srcStride, dstBase, dstStride, + ksize, anchorX, anchorY, rowBorderType, columnBorderType, + borderValues, borderMargin); +#else + (void)cn; + (void)srcBase; + (void)srcStride; + (void)dstBase; + (void)dstStride; + (void)rowBorderType; + (void)columnBorderType; + (void)borderValues; + (void)borderMargin; +#endif +} + +} // namespace CAROTENE_NS diff --git a/3rdparty/carotene/src/mul.cpp b/3rdparty/carotene/src/mul.cpp new file mode 100644 index 0000000000..3bbbfc50aa --- /dev/null +++ b/3rdparty/carotene/src/mul.cpp @@ -0,0 +1,1572 @@ +/* + * By downloading, copying, installing or using the software you agree to this license. + * If you do not agree to this license, do not download, install, + * copy or use the software. + * + * + * License Agreement + * For Open Source Computer Vision Library + * (3-clause BSD License) + * + * Copyright (C) 2014-2016, NVIDIA Corporation, all rights reserved. + * Third party copyrights are property of their respective owners. + * + * Redistribution and use in source and binary forms, with or without modification, + * are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * * Neither the names of the copyright holders nor the names of the contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * This software is provided by the copyright holders and contributors "as is" and + * any express or implied warranties, including, but not limited to, the implied + * warranties of merchantability and fitness for a particular purpose are disclaimed. + * In no event shall copyright holders or contributors be liable for any direct, + * indirect, incidental, special, exemplary, or consequential damages + * (including, but not limited to, procurement of substitute goods or services; + * loss of use, data, or profits; or business interruption) however caused + * and on any theory of liability, whether in contract, strict liability, + * or tort (including negligence or otherwise) arising in any way out of + * the use of this software, even if advised of the possibility of such damage. + */ + +#include "common.hpp" +#include "vtransform.hpp" + +#include +#include +#include +#include + +namespace CAROTENE_NS { + +#ifdef CAROTENE_NEON + +namespace { + +bool isIntegerScale(f32 scale) +{ + return std::fabs(scale - static_cast(scale)) < FLT_EPSILON; +} + +template +void mulu8(const Size2D &size, + const u8 * src0Base, ptrdiff_t src0Stride, + const u8 * src1Base, ptrdiff_t src1Stride, + u8 * dstBase, ptrdiff_t dstStride, + CONVERT_POLICY cpolicy) +{ + size_t roiw16 = size.width >= 15 ? size.width - 15 : 0; + size_t roiw8 = size.width >= 7 ? size.width - 7 : 0; + + for (size_t i = 0; i < size.height; ++i) + { + const u8 * src0 = internal::getRowPtr(src0Base, src0Stride, i); + const u8 * src1 = internal::getRowPtr(src1Base, src1Stride, i); + u8 * dst = internal::getRowPtr(dstBase, dstStride, i); + size_t j = 0; + + if (cpolicy == CONVERT_POLICY_SATURATE) + { + for (; j < roiw16; j += 16) + { + internal::prefetch(src0 + j); + internal::prefetch(src1 + j); + uint8x16_t v_src0 = vld1q_u8(src0 + j), v_src1 = vld1q_u8(src1 + j); + + uint16x8_t v_dst0 = vmull_u8(vget_low_u8(v_src0), vget_low_u8(v_src1)); + uint16x8_t v_dst1 = vmull_u8(vget_high_u8(v_src0), vget_high_u8(v_src1)); + + v_dst0 = vshrq_n_u16(v_dst0, shift); + v_dst1 = vshrq_n_u16(v_dst1, shift); + + vst1q_u8(dst + j, vcombine_u8(vqmovn_u16(v_dst0), vqmovn_u16(v_dst1))); + } + for (; j < roiw8; j += 8) + { + uint16x8_t v_dst = vmull_u8(vld1_u8(src0 + j), vld1_u8(src1 + j)); + vst1_u8(dst + j, vqmovn_u16(vshrq_n_u16(v_dst, shift))); + } + + for (; j < size.width; j++) + { + u16 val = (u16)src0[j] * (u16)src1[j]; + dst[j] = internal::saturate_cast(val >> shift); + } + } + else // CONVERT_POLICY_WRAP + { + for (; j < roiw16; j += 16) + { + internal::prefetch(src0 + j); + internal::prefetch(src1 + j); + uint8x16_t v_src0 = vld1q_u8(src0 + j), v_src1 = vld1q_u8(src1 + j); + + uint16x8_t v_dst0 = vmull_u8(vget_low_u8(v_src0), vget_low_u8(v_src1)); + uint16x8_t v_dst1 = vmull_u8(vget_high_u8(v_src0), vget_high_u8(v_src1)); + + v_dst0 = vshrq_n_u16(v_dst0, shift); + v_dst1 = vshrq_n_u16(v_dst1, shift); + + vst1q_u8(dst + j, vcombine_u8(vmovn_u16(v_dst0), vmovn_u16(v_dst1))); + } + for (; j < roiw8; j += 8) + { + uint16x8_t v_dst = vmull_u8(vld1_u8(src0 + j), vld1_u8(src1 + j)); + vst1_u8(dst + j, vmovn_u16(vshrq_n_u16(v_dst, shift))); + } + + for (; j < size.width; j++) + { + u16 val = (u16)src0[j] * (u16)src1[j]; + dst[j] = (u8)(val >> shift); + } + } + } +} + +template +void muls16(const Size2D &size, + const u8 * src0Base, ptrdiff_t src0Stride, + const u8 * src1Base, ptrdiff_t src1Stride, + s16 * dstBase, ptrdiff_t dstStride, + CONVERT_POLICY cpolicy) +{ + size_t roiw16 = size.width >= 15 ? size.width - 15 : 0; + size_t roiw8 = size.width >= 7 ? size.width - 7 : 0; + + uint16x8_t v_32767 = vdupq_n_u16(0x7FFF); + + for (size_t i = 0; i < size.height; ++i) + { + const u8 * src0 = internal::getRowPtr(src0Base, src0Stride, i); + const u8 * src1 = internal::getRowPtr(src1Base, src1Stride, i); + s16 * dst = internal::getRowPtr(dstBase, dstStride, i); + size_t j = 0; + + if (cpolicy == CONVERT_POLICY_SATURATE) + { + for (; j < roiw16; j += 16) + { + internal::prefetch(src0 + j); + internal::prefetch(src1 + j); + uint8x16_t v_src0 = vld1q_u8(src0 + j), v_src1 = vld1q_u8(src1 + j); + + uint16x8_t v_dst0 = vmull_u8(vget_low_u8(v_src0), vget_low_u8(v_src1)); + uint16x8_t v_dst1 = vmull_u8(vget_high_u8(v_src0), vget_high_u8(v_src1)); + + v_dst0 = vshrq_n_u16(v_dst0, shift); + v_dst1 = vshrq_n_u16(v_dst1, shift); + + vst1q_s16(dst + j, vreinterpretq_s16_u16(vminq_u16(v_32767, v_dst0))); + vst1q_s16(dst + j + 8, vreinterpretq_s16_u16(vminq_u16(v_32767, v_dst1))); + } + for (; j < roiw8; j += 8) + { + uint16x8_t v_dst = vmull_u8(vld1_u8(src0 + j), vld1_u8(src1 + j)); + v_dst = vshrq_n_u16(v_dst, shift); + vst1q_s16(dst + j, vreinterpretq_s16_u16(vminq_u16(v_32767, v_dst))); + } + + for (; j < size.width; j++) + { + u16 val = (u16)src0[j] * (u16)src1[j]; + dst[j] = internal::saturate_cast(val >> shift); + } + } + else // CONVERT_POLICY_WRAP + { + for (; j < roiw16; j += 16) + { + internal::prefetch(src0 + j); + internal::prefetch(src1 + j); + uint8x16_t v_src0 = vld1q_u8(src0 + j), v_src1 = vld1q_u8(src1 + j); + + uint16x8_t v_dst0 = vmull_u8(vget_low_u8(v_src0), vget_low_u8(v_src1)); + uint16x8_t v_dst1 = vmull_u8(vget_high_u8(v_src0), vget_high_u8(v_src1)); + + v_dst0 = vshrq_n_u16(v_dst0, shift); + v_dst1 = vshrq_n_u16(v_dst1, shift); + + vst1q_s16(dst + j, vreinterpretq_s16_u16(v_dst0)); + vst1q_s16(dst + j + 8, vreinterpretq_s16_u16(v_dst1)); + } + for (; j < roiw8; j += 8) + { + uint16x8_t v_dst = vmull_u8(vld1_u8(src0 + j), vld1_u8(src1 + j)); + v_dst = vshrq_n_u16(v_dst, shift); + vst1q_s16(dst + j, vreinterpretq_s16_u16(v_dst)); + } + + for (; j < size.width; j++) + { + u16 val = (u16)src0[j] * (u16)src1[j]; + dst[j] = (s16)(val >> shift); + } + } + } +} + +typedef void (* mulFuncu8)(const Size2D &size, + const u8 * src0Base, ptrdiff_t src0Stride, + const u8 * src1Base, ptrdiff_t src1Stride, + u8 * dstBase, ptrdiff_t dstStride, + CONVERT_POLICY cpolicy); + +typedef void (* mulFuncs16)(const Size2D &size, + const u8 * src0Base, ptrdiff_t src0Stride, + const u8 * src1Base, ptrdiff_t src1Stride, + s16 * dstBase, ptrdiff_t dstStride, + CONVERT_POLICY cpolicy); + +} // namespace + +#endif + +void mul(const Size2D &size, + const u8 * src0Base, ptrdiff_t src0Stride, + const u8 * src1Base, ptrdiff_t src1Stride, + u8 * dstBase, ptrdiff_t dstStride, + f32 scale, + CONVERT_POLICY cpolicy) +{ + internal::assertSupportedConfiguration(); + +#ifdef CAROTENE_NEON + if ((scale * UCHAR_MAX * UCHAR_MAX) < 1.0f) + { + for (size_t y = 0; y < size.height; ++y) + { + u8 * dst = internal::getRowPtr(dstBase, dstStride, y); + std::memset(dst, 0, sizeof(u8) * size.width); + } + return; + } + + s32 iscale = static_cast(scale), exp = 0; + f32 significand = frexp(scale, &exp); + bool is_integer_scale = isIntegerScale(scale), + is_power_of_2 = (significand == 0.5f) && (exp <= 0); + exp = -exp + 1; + + if (is_power_of_2) + { + static const mulFuncu8 funcs[16] = + { + NULL, + mulu8<1>, + mulu8<2>, + mulu8<3>, + mulu8<4>, + mulu8<5>, + mulu8<6>, + mulu8<7>, + mulu8<8>, + mulu8<9>, + mulu8<10>, + mulu8<11>, + mulu8<12>, + mulu8<13>, + mulu8<14>, + mulu8<15> + }; + + mulFuncu8 func = funcs[exp]; + + func(size, + src0Base, src0Stride, + src1Base, src1Stride, + dstBase, dstStride, + cpolicy); + + return; + } + + size_t roiw16 = size.width >= 15 ? size.width - 15 : 0; + size_t roiw8 = size.width >= 7 ? size.width - 7 : 0; + + for (size_t i = 0; i < size.height; ++i) + { + const u8 * src0 = internal::getRowPtr(src0Base, src0Stride, i); + const u8 * src1 = internal::getRowPtr(src1Base, src1Stride, i); + u8 * dst = internal::getRowPtr(dstBase, dstStride, i); + size_t j = 0; + + if (cpolicy == CONVERT_POLICY_SATURATE) + { + if (is_integer_scale && iscale == 1) + { + for (; j < roiw16; j += 16) + { + internal::prefetch(src0 + j); + internal::prefetch(src1 + j); + uint8x16_t v_src0 = vld1q_u8(src0 + j), v_src1 = vld1q_u8(src1 + j); + + uint16x8_t v_dst0 = vmull_u8(vget_low_u8(v_src0), vget_low_u8(v_src1)); + uint16x8_t v_dst1 = vmull_u8(vget_high_u8(v_src0), vget_high_u8(v_src1)); + + vst1q_u8(dst + j, vcombine_u8(vqmovn_u16(v_dst0), vqmovn_u16(v_dst1))); + } + for (; j < roiw8; j += 8) + { + vst1_u8(dst + j, vqmovn_u16(vmull_u8(vld1_u8(src0 + j), vld1_u8(src1 + j)))); + } + + for (; j < size.width; j++) + { + u16 val = (u16)src0[j] * (u16)src1[j]; + dst[j] = internal::saturate_cast(val); + } + } + else // generic case using floats + { + for (; j < roiw16; j += 16) + { + internal::prefetch(src0 + j); + internal::prefetch(src1 + j); + + uint8x16_t v_src0 = vld1q_u8(src0 + j); + uint8x16_t v_src1 = vld1q_u8(src1 + j); + + uint16x8_t v_src0_p = vmovl_u8(vget_low_u8(v_src0)); + uint16x8_t v_src1_p = vmovl_u8(vget_low_u8(v_src1)); + float32x4_t v_dst0f = vmulq_n_f32(vmulq_f32(vcvtq_f32_u32(vmovl_u16(vget_low_u16(v_src0_p))), + vcvtq_f32_u32(vmovl_u16(vget_low_u16(v_src1_p)))), scale); + float32x4_t v_dst1f = vmulq_n_f32(vmulq_f32(vcvtq_f32_u32(vmovl_u16(vget_high_u16(v_src0_p))), + vcvtq_f32_u32(vmovl_u16(vget_high_u16(v_src1_p)))), scale); + v_src0_p = vmovl_u8(vget_high_u8(v_src0)); + v_src1_p = vmovl_u8(vget_high_u8(v_src1)); + float32x4_t v_dst2f = vmulq_n_f32(vmulq_f32(vcvtq_f32_u32(vmovl_u16(vget_low_u16(v_src0_p))), + vcvtq_f32_u32(vmovl_u16(vget_low_u16(v_src1_p)))), scale); + float32x4_t v_dst3f = vmulq_n_f32(vmulq_f32(vcvtq_f32_u32(vmovl_u16(vget_high_u16(v_src0_p))), + vcvtq_f32_u32(vmovl_u16(vget_high_u16(v_src1_p)))), scale); + uint16x8_t v_dst0u = vcombine_u16(vqmovn_u32(vcvtq_u32_f32(v_dst0f)), + vqmovn_u32(vcvtq_u32_f32(v_dst1f))); + uint16x8_t v_dst1u = vcombine_u16(vqmovn_u32(vcvtq_u32_f32(v_dst2f)), + vqmovn_u32(vcvtq_u32_f32(v_dst3f))); + vst1q_u8(dst + j, vcombine_u8(vqmovn_u16(v_dst0u), vqmovn_u16(v_dst1u))); + } + for (; j < roiw8; j += 8) + { + uint16x8_t v_src0 = vmovl_u8(vld1_u8(src0 + j)); + uint16x8_t v_src1 = vmovl_u8(vld1_u8(src1 + j)); + float32x4_t v_dst0f = vmulq_n_f32(vmulq_f32(vcvtq_f32_u32(vmovl_u16(vget_low_u16(v_src0))), + vcvtq_f32_u32(vmovl_u16(vget_low_u16(v_src1)))), scale); + float32x4_t v_dst1f = vmulq_n_f32(vmulq_f32(vcvtq_f32_u32(vmovl_u16(vget_high_u16(v_src0))), + vcvtq_f32_u32(vmovl_u16(vget_high_u16(v_src1)))), scale); + uint16x8_t v_dstu = vcombine_u16(vqmovn_u32(vcvtq_u32_f32(v_dst0f)), + vqmovn_u32(vcvtq_u32_f32(v_dst1f))); + vst1_u8(dst + j, vqmovn_u16(v_dstu)); + } + + for (; j < size.width; j++) + { + f32 fval = (f32)src0[j] * (f32)src1[j] * scale; + dst[j] = internal::saturate_cast((s32)trunc(fval)); + } + } + } + else // CONVERT_POLICY_WRAP + { + if (is_integer_scale && iscale == 1) + { + for (; j < roiw16; j += 16) + { + internal::prefetch(src0 + j); + internal::prefetch(src1 + j); + uint8x16_t v_src0 = vld1q_u8(src0 + j), v_src1 = vld1q_u8(src1 + j); + + uint16x8_t v_dst0 = vmull_u8(vget_low_u8(v_src0), vget_low_u8(v_src1)); + uint16x8_t v_dst1 = vmull_u8(vget_high_u8(v_src0), vget_high_u8(v_src1)); + + vst1q_u8(dst + j, vcombine_u8(vmovn_u16(v_dst0), vmovn_u16(v_dst1))); + } + for (; j < roiw8; j += 8) + { + vst1_u8(dst + j, vmovn_u16(vmull_u8(vld1_u8(src0 + j), vld1_u8(src1 + j)))); + } + + for (; j < size.width; j++) + { + u16 val = (u16)src0[j] * (u16)src1[j]; + dst[j] = (u8)(val); + } + } + else // generic case using floats + { + for (; j < roiw16; j += 16) + { + internal::prefetch(src0 + j); + internal::prefetch(src1 + j); + uint8x16_t v_src0 = vld1q_u8(src0 + j); + uint8x16_t v_src1 = vld1q_u8(src1 + j); + + uint16x8_t v_src0_p = vmovl_u8(vget_low_u8(v_src0)); + uint16x8_t v_src1_p = vmovl_u8(vget_low_u8(v_src1)); + float32x4_t v_dst0f = vmulq_n_f32(vmulq_f32(vcvtq_f32_u32(vmovl_u16(vget_low_u16(v_src0_p))), + vcvtq_f32_u32(vmovl_u16(vget_low_u16(v_src1_p)))), scale); + float32x4_t v_dst1f = vmulq_n_f32(vmulq_f32(vcvtq_f32_u32(vmovl_u16(vget_high_u16(v_src0_p))), + vcvtq_f32_u32(vmovl_u16(vget_high_u16(v_src1_p)))), scale); + v_src0_p = vmovl_u8(vget_high_u8(v_src0)); + v_src1_p = vmovl_u8(vget_high_u8(v_src1)); + float32x4_t v_dst2f = vmulq_n_f32(vmulq_f32(vcvtq_f32_u32(vmovl_u16(vget_low_u16(v_src0_p))), + vcvtq_f32_u32(vmovl_u16(vget_low_u16(v_src1_p)))), scale); + float32x4_t v_dst3f = vmulq_n_f32(vmulq_f32(vcvtq_f32_u32(vmovl_u16(vget_high_u16(v_src0_p))), + vcvtq_f32_u32(vmovl_u16(vget_high_u16(v_src1_p)))), scale); + uint16x8_t v_dst0u = vcombine_u16(vmovn_u32(vcvtq_u32_f32(v_dst0f)), + vmovn_u32(vcvtq_u32_f32(v_dst1f))); + uint16x8_t v_dst1u = vcombine_u16(vmovn_u32(vcvtq_u32_f32(v_dst2f)), + vmovn_u32(vcvtq_u32_f32(v_dst3f))); + vst1q_u8(dst + j, vcombine_u8(vmovn_u16(v_dst0u), vmovn_u16(v_dst1u))); + } + for (; j < roiw8; j += 8) + { + uint16x8_t v_src0 = vmovl_u8(vld1_u8(src0 + j)); + uint16x8_t v_src1 = vmovl_u8(vld1_u8(src1 + j)); + float32x4_t v_dst0f = vmulq_n_f32(vmulq_f32(vcvtq_f32_u32(vmovl_u16(vget_low_u16(v_src0))), + vcvtq_f32_u32(vmovl_u16(vget_low_u16(v_src1)))), scale); + float32x4_t v_dst1f = vmulq_n_f32(vmulq_f32(vcvtq_f32_u32(vmovl_u16(vget_high_u16(v_src0))), + vcvtq_f32_u32(vmovl_u16(vget_high_u16(v_src1)))), scale); + uint16x8_t v_dstu = vcombine_u16(vmovn_u32(vcvtq_u32_f32(v_dst0f)), + vmovn_u32(vcvtq_u32_f32(v_dst1f))); + vst1_u8(dst + j, vmovn_u16(v_dstu)); + } + + for (; j < size.width; j++) + { + f32 fval = (f32)src0[j] * (f32)src1[j] * scale; + dst[j] = (u8)(s32)trunc(fval); + } + } + } + } +#else + (void)size; + (void)src0Base; + (void)src0Stride; + (void)src1Base; + (void)src1Stride; + (void)dstBase; + (void)dstStride; + (void)cpolicy; + (void)scale; +#endif +} + +void mul(const Size2D &size, + const u8 * src0Base, ptrdiff_t src0Stride, + const u8 * src1Base, ptrdiff_t src1Stride, + s16 * dstBase, ptrdiff_t dstStride, + f32 scale, + CONVERT_POLICY cpolicy) +{ + internal::assertSupportedConfiguration(); +#ifdef CAROTENE_NEON + if (((scale * UCHAR_MAX * UCHAR_MAX) < 1.0f) && (scale >= 0)) + { + for (size_t y = 0; y < size.height; ++y) + { + s16 * dst = internal::getRowPtr(dstBase, dstStride, y); + std::memset(dst, 0, sizeof(s16) * size.width); + } + return; + } + + s32 iscale = static_cast(scale), exp = 0; + f32 significand = frexp(scale, &exp); + bool is_integer_scale = isIntegerScale(scale), + is_power_of_2 = (significand == 0.5f) && (exp <= 0); + exp = -exp + 1; + + if (is_power_of_2) + { + static const mulFuncs16 funcs[16] = + { + NULL, + muls16<1>, + muls16<2>, + muls16<3>, + muls16<4>, + muls16<5>, + muls16<6>, + muls16<7>, + muls16<8>, + muls16<9>, + muls16<10>, + muls16<11>, + muls16<12>, + muls16<13>, + muls16<14>, + muls16<15> + }; + + mulFuncs16 func = funcs[exp]; + + func(size, + src0Base, src0Stride, + src1Base, src1Stride, + dstBase, dstStride, + cpolicy); + + return; + } + + size_t roiw16 = size.width >= 15 ? size.width - 15 : 0; + size_t roiw8 = size.width >= 7 ? size.width - 7 : 0; + + uint16x8_t v_32767 = vdupq_n_u16(0x7FFF); + + for (size_t i = 0; i < size.height; ++i) + { + const u8 * src0 = internal::getRowPtr(src0Base, src0Stride, i); + const u8 * src1 = internal::getRowPtr(src1Base, src1Stride, i); + s16 * dst = internal::getRowPtr(dstBase, dstStride, i); + size_t j = 0; + + if (cpolicy == CONVERT_POLICY_SATURATE) + { + if (is_integer_scale && iscale == 1) + { + for (; j < roiw16; j += 16) + { + internal::prefetch(src0 + j); + internal::prefetch(src1 + j); + uint8x16_t v_src0 = vld1q_u8(src0 + j), v_src1 = vld1q_u8(src1 + j); + + uint16x8_t v_dst0 = vmull_u8(vget_low_u8(v_src0), vget_low_u8(v_src1)); + uint16x8_t v_dst1 = vmull_u8(vget_high_u8(v_src0), vget_high_u8(v_src1)); + + vst1q_s16(dst + j, vreinterpretq_s16_u16(vminq_u16(v_32767, v_dst0))); + vst1q_s16(dst + j +8, vreinterpretq_s16_u16(vminq_u16(v_32767, v_dst1))); + } + for (; j < roiw8; j += 8) + { + uint16x8_t v_dst = vmull_u8(vld1_u8(src0 + j), vld1_u8(src1 + j)); + vst1q_s16(dst + j, vreinterpretq_s16_u16(vminq_u16(v_32767, v_dst))); + } + + for (; j < size.width; j++) + { + u16 val = (u16)src0[j] * (u16)src1[j]; + dst[j] = internal::saturate_cast(val); + } + } + else // generic case using floats + { + for (; j < roiw16; j += 16) + { + internal::prefetch(src0 + j); + internal::prefetch(src1 + j); + uint8x16_t v_src0 = vld1q_u8(src0 + j); + uint8x16_t v_src1 = vld1q_u8(src1 + j); + + uint16x8_t v_src0_p = vmovl_u8(vget_low_u8(v_src0)); + uint16x8_t v_src1_p = vmovl_u8(vget_low_u8(v_src1)); + float32x4_t v_dst0f = vmulq_n_f32(vmulq_f32(vcvtq_f32_u32(vmovl_u16(vget_low_u16(v_src0_p))), + vcvtq_f32_u32(vmovl_u16(vget_low_u16(v_src1_p)))), scale); + float32x4_t v_dst1f = vmulq_n_f32(vmulq_f32(vcvtq_f32_u32(vmovl_u16(vget_high_u16(v_src0_p))), + vcvtq_f32_u32(vmovl_u16(vget_high_u16(v_src1_p)))), scale); + vst1q_s16(dst + j, vcombine_s16(vqmovn_s32(vcvtq_s32_f32(v_dst0f)), + vqmovn_s32(vcvtq_s32_f32(v_dst1f)))); + + v_src0_p = vmovl_u8(vget_high_u8(v_src0)); + v_src1_p = vmovl_u8(vget_high_u8(v_src1)); + v_dst0f = vmulq_n_f32(vmulq_f32(vcvtq_f32_u32(vmovl_u16(vget_low_u16(v_src0_p))), + vcvtq_f32_u32(vmovl_u16(vget_low_u16(v_src1_p)))), scale); + v_dst1f = vmulq_n_f32(vmulq_f32(vcvtq_f32_u32(vmovl_u16(vget_high_u16(v_src0_p))), + vcvtq_f32_u32(vmovl_u16(vget_high_u16(v_src1_p)))), scale); + vst1q_s16(dst + j + 8, vcombine_s16(vqmovn_s32(vcvtq_s32_f32(v_dst0f)), + vqmovn_s32(vcvtq_s32_f32(v_dst1f)))); + } + for (; j < roiw8; j += 8) + { + uint16x8_t v_src0 = vmovl_u8(vld1_u8(src0 + j)); + uint16x8_t v_src1 = vmovl_u8(vld1_u8(src1 + j)); + float32x4_t v_dst0f = vmulq_n_f32(vmulq_f32(vcvtq_f32_u32(vmovl_u16(vget_low_u16(v_src0))), + vcvtq_f32_u32(vmovl_u16(vget_low_u16(v_src1)))), scale); + float32x4_t v_dst1f = vmulq_n_f32(vmulq_f32(vcvtq_f32_u32(vmovl_u16(vget_high_u16(v_src0))), + vcvtq_f32_u32(vmovl_u16(vget_high_u16(v_src1)))), scale); + vst1q_s16(dst + j, vcombine_s16(vqmovn_s32(vcvtq_s32_f32(v_dst0f)), + vqmovn_s32(vcvtq_s32_f32(v_dst1f)))); + } + + for (; j < size.width; j++) + { + f32 fval = (f32)src0[j] * (f32)src1[j] * scale; + dst[j] = internal::saturate_cast((s32)trunc(fval)); + } + } + } + else // CONVERT_POLICY_WRAP + { + if (is_integer_scale && iscale == 1) + { + for (; j < roiw16; j += 16) + { + internal::prefetch(src0 + j); + internal::prefetch(src1 + j); + uint8x16_t v_src0 = vld1q_u8(src0 + j), v_src1 = vld1q_u8(src1 + j); + + uint16x8_t v_dst0 = vmull_u8(vget_low_u8(v_src0), vget_low_u8(v_src1)); + uint16x8_t v_dst1 = vmull_u8(vget_high_u8(v_src0), vget_high_u8(v_src1)); + + vst1q_s16(dst + j, vreinterpretq_s16_u16(v_dst0)); + vst1q_s16(dst + j + 8, vreinterpretq_s16_u16(v_dst1)); + } + for (; j < roiw8; j += 8) + { + uint16x8_t v_dst = vmull_u8(vld1_u8(src0 + j), vld1_u8(src1 + j)); + vst1q_s16(dst + j, vreinterpretq_s16_u16(v_dst)); + } + + for (; j < size.width; j++) + { + u16 val = (u16)src0[j] * (u16)src1[j]; + dst[j] = (s16)(val); + } + } + else // generic case using floats + { + for (; j < roiw16; j += 16) + { + internal::prefetch(src0 + j); + internal::prefetch(src1 + j); + uint8x16_t v_src0 = vld1q_u8(src0 + j); + uint8x16_t v_src1 = vld1q_u8(src1 + j); + + uint16x8_t v_src0_p = vmovl_u8(vget_low_u8(v_src0)); + uint16x8_t v_src1_p = vmovl_u8(vget_low_u8(v_src1)); + float32x4_t v_dst0f = vmulq_n_f32(vmulq_f32(vcvtq_f32_u32(vmovl_u16(vget_low_u16(v_src0_p))), + vcvtq_f32_u32(vmovl_u16(vget_low_u16(v_src1_p)))), scale); + float32x4_t v_dst1f = vmulq_n_f32(vmulq_f32(vcvtq_f32_u32(vmovl_u16(vget_high_u16(v_src0_p))), + vcvtq_f32_u32(vmovl_u16(vget_high_u16(v_src1_p)))), scale); + vst1q_s16(dst + j, vcombine_s16(vmovn_s32(vcvtq_s32_f32(v_dst0f)), + vmovn_s32(vcvtq_s32_f32(v_dst1f)))); + + v_src0_p = vmovl_u8(vget_high_u8(v_src0)); + v_src1_p = vmovl_u8(vget_high_u8(v_src1)); + v_dst0f = vmulq_n_f32(vmulq_f32(vcvtq_f32_u32(vmovl_u16(vget_low_u16(v_src0_p))), + vcvtq_f32_u32(vmovl_u16(vget_low_u16(v_src1_p)))), scale); + v_dst1f = vmulq_n_f32(vmulq_f32(vcvtq_f32_u32(vmovl_u16(vget_high_u16(v_src0_p))), + vcvtq_f32_u32(vmovl_u16(vget_high_u16(v_src1_p)))), scale); + vst1q_s16(dst + j + 8, vcombine_s16(vmovn_s32(vcvtq_s32_f32(v_dst0f)), + vmovn_s32(vcvtq_s32_f32(v_dst1f)))); + } + for (; j < roiw8; j += 8) + { + uint16x8_t v_src0 = vmovl_u8(vld1_u8(src0 + j)); + uint16x8_t v_src1 = vmovl_u8(vld1_u8(src1 + j)); + float32x4_t v_dst0f = vmulq_n_f32(vmulq_f32(vcvtq_f32_u32(vmovl_u16(vget_low_u16(v_src0))), + vcvtq_f32_u32(vmovl_u16(vget_low_u16(v_src1)))), scale); + float32x4_t v_dst1f = vmulq_n_f32(vmulq_f32(vcvtq_f32_u32(vmovl_u16(vget_high_u16(v_src0))), + vcvtq_f32_u32(vmovl_u16(vget_high_u16(v_src1)))), scale); + vst1q_s16(dst + j, vcombine_s16(vmovn_s32(vcvtq_s32_f32(v_dst0f)), + vmovn_s32(vcvtq_s32_f32(v_dst1f)))); + } + + for (; j < size.width; j++) + { + f32 fval = (f32)src0[j] * (f32)src1[j] * scale; + dst[j] = (s16)(s32)trunc(fval); + } + } + } + } +#else + (void)size; + (void)src0Base; + (void)src0Stride; + (void)src1Base; + (void)src1Stride; + (void)dstBase; + (void)dstStride; + (void)cpolicy; + (void)scale; +#endif +} + +void mul(const Size2D &size, + const u8 * src0Base, ptrdiff_t src0Stride, + const s16 * src1Base, ptrdiff_t src1Stride, + s16 * dstBase, ptrdiff_t dstStride, + f32 scale, + CONVERT_POLICY cpolicy) +{ + internal::assertSupportedConfiguration(); + +#ifdef CAROTENE_NEON + if (scale == 0.0f) + { + for (size_t y = 0; y < size.height; ++y) + { + s16 * dst = internal::getRowPtr(dstBase, dstStride, y); + std::memset(dst, 0, sizeof(s16) * size.width); + } + return; + } + + size_t roiw16 = size.width >= 15 ? size.width - 15 : 0; + size_t roiw8 = size.width >= 7 ? size.width - 7 : 0; + + bool is_integer_scale = isIntegerScale(scale); + s32 iscale = static_cast(scale); + + for (size_t i = 0; i < size.height; ++i) + { + const u8 * src0 = internal::getRowPtr(src0Base, src0Stride, i); + const s16 * src1 = internal::getRowPtr(src1Base, src1Stride, i); + s16 * dst = internal::getRowPtr(dstBase, dstStride, i); + size_t j = 0; + + if (cpolicy == CONVERT_POLICY_SATURATE) + { + if (is_integer_scale && iscale == 1) + { + for (; j < roiw16; j += 16) + { + internal::prefetch(src0 + j); + internal::prefetch(src1 + j); + uint8x16_t v_src0 = vld1q_u8(src0 + j); + + int16x8_t v_src0_p = vreinterpretq_s16_u16(vmovl_u8(vget_low_u8(v_src0))); + int16x8_t v_src1_p = vld1q_s16(src1 + j); + int16x8_t v_dst = vcombine_s16(vqmovn_s32(vmull_s16(vget_low_s16(v_src0_p), vget_low_s16(v_src1_p))), + vqmovn_s32(vmull_s16(vget_high_s16(v_src0_p), vget_high_s16(v_src1_p)))); + vst1q_s16(dst + j, v_dst); + + v_src0_p = vreinterpretq_s16_u16(vmovl_u8(vget_high_u8(v_src0))); + v_src1_p = vld1q_s16(src1 + j + 8); + v_dst = vcombine_s16(vqmovn_s32(vmull_s16(vget_low_s16(v_src0_p), vget_low_s16(v_src1_p))), + vqmovn_s32(vmull_s16(vget_high_s16(v_src0_p), vget_high_s16(v_src1_p)))); + vst1q_s16(dst + j + 8, v_dst); + } + for (; j < roiw8; j += 8) + { + int16x8_t v_src0 = vreinterpretq_s16_u16(vmovl_u8(vget_low_u8(vld1q_u8(src0 + j)))); + int16x8_t v_src1 = vld1q_s16(src1 + j); + int16x8_t v_dst = vcombine_s16(vqmovn_s32(vmull_s16(vget_low_s16(v_src0), vget_low_s16(v_src1))), + vqmovn_s32(vmull_s16(vget_high_s16(v_src0), vget_high_s16(v_src1)))); + vst1q_s16(dst + j, v_dst); + } + + for (; j < size.width; j++) + { + s32 val = (s32)src0[j] * (s32)src1[j]; + dst[j] = internal::saturate_cast(val); + } + } + else // generic case using floats + { + for (; j < roiw16; j += 16) + { + internal::prefetch(src0 + j); + internal::prefetch(src1 + j); + uint8x16_t v_src0 = vld1q_u8(src0 + j); + + uint16x8_t v_src0_p = vmovl_u8(vget_low_u8(v_src0)); + int16x8_t v_src1_p = vld1q_s16(src1 + j); + float32x4_t v_dst0f = vmulq_n_f32(vmulq_f32(vcvtq_f32_u32(vmovl_u16(vget_low_u16(v_src0_p))), + vcvtq_f32_s32(vmovl_s16(vget_low_s16(v_src1_p)))), scale); + float32x4_t v_dst1f = vmulq_n_f32(vmulq_f32(vcvtq_f32_u32(vmovl_u16(vget_high_u16(v_src0_p))), + vcvtq_f32_s32(vmovl_s16(vget_high_s16(v_src1_p)))), scale); + vst1q_s16(dst + j, vcombine_s16(vqmovn_s32(vcvtq_s32_f32(v_dst0f)), + vqmovn_s32(vcvtq_s32_f32(v_dst1f)))); + + v_src0_p = vmovl_u8(vget_high_u8(v_src0)); + v_src1_p = vld1q_s16(src1 + j + 8); + v_dst0f = vmulq_n_f32(vmulq_f32(vcvtq_f32_u32(vmovl_u16(vget_low_u16(v_src0_p))), + vcvtq_f32_s32(vmovl_s16(vget_low_s16(v_src1_p)))), scale); + v_dst1f = vmulq_n_f32(vmulq_f32(vcvtq_f32_u32(vmovl_u16(vget_high_u16(v_src0_p))), + vcvtq_f32_s32(vmovl_s16(vget_high_s16(v_src1_p)))), scale); + vst1q_s16(dst + j + 8, vcombine_s16(vqmovn_s32(vcvtq_s32_f32(v_dst0f)), + vqmovn_s32(vcvtq_s32_f32(v_dst1f)))); + } + for (; j < roiw8; j += 8) + { + uint16x8_t v_src0 = vmovl_u8(vld1_u8(src0 + j)); + int16x8_t v_src1 = vld1q_s16(src1 + j); + float32x4_t v_dst0f = vmulq_n_f32(vmulq_f32(vcvtq_f32_u32(vmovl_u16(vget_low_u16(v_src0))), + vcvtq_f32_s32(vmovl_s16(vget_low_s16(v_src1)))), scale); + float32x4_t v_dst1f = vmulq_n_f32(vmulq_f32(vcvtq_f32_u32(vmovl_u16(vget_high_u16(v_src0))), + vcvtq_f32_s32(vmovl_s16(vget_high_s16(v_src1)))), scale); + vst1q_s16(dst + j, vcombine_s16(vqmovn_s32(vcvtq_s32_f32(v_dst0f)), + vqmovn_s32(vcvtq_s32_f32(v_dst1f)))); + } + + for (; j < size.width; j++) + { + f32 fval = (f32)src0[j] * (f32)src1[j] * scale; + dst[j] = internal::saturate_cast((s32)trunc(fval)); + } + } + } + else // CONVERT_POLICY_WRAP + { + if (is_integer_scale && iscale == 1) + { + for (; j < roiw16; j += 16) + { + internal::prefetch(src0 + j); + internal::prefetch(src1 + j); + uint8x16_t v_src0 = vld1q_u8(src0 + j); + + int16x8_t v_src0_p = vreinterpretq_s16_u16(vmovl_u8(vget_low_u8(v_src0))); + int16x8_t v_src1_p = vld1q_s16(src1 + j); + int16x8_t v_dst = vcombine_s16(vmovn_s32(vmull_s16(vget_low_s16(v_src0_p), vget_low_s16(v_src1_p))), + vmovn_s32(vmull_s16(vget_high_s16(v_src0_p), vget_high_s16(v_src1_p)))); + vst1q_s16(dst + j, v_dst); + + v_src0_p = vreinterpretq_s16_u16(vmovl_u8(vget_high_u8(v_src0))); + v_src1_p = vld1q_s16(src1 + j + 8); + v_dst = vcombine_s16(vmovn_s32(vmull_s16(vget_low_s16(v_src0_p), vget_low_s16(v_src1_p))), + vmovn_s32(vmull_s16(vget_high_s16(v_src0_p), vget_high_s16(v_src1_p)))); + vst1q_s16(dst + j + 8, v_dst); + } + for (; j < roiw8; j += 8) + { + int16x8_t v_src0 = vreinterpretq_s16_u16(vmovl_u8(vget_low_u8(vld1q_u8(src0 + j)))); + int16x8_t v_src1 = vld1q_s16(src1 + j); + int16x8_t v_dst = vcombine_s16(vmovn_s32(vmull_s16(vget_low_s16(v_src0), vget_low_s16(v_src1))), + vmovn_s32(vmull_s16(vget_high_s16(v_src0), vget_high_s16(v_src1)))); + vst1q_s16(dst + j, v_dst); + } + + for (; j < size.width; j++) + { + s32 val = (s32)src0[j] * (s32)src1[j]; + dst[j] = (s16)(val); + } + } + else // generic case using floats + { + for (; j < roiw16; j += 16) + { + internal::prefetch(src0 + j); + internal::prefetch(src1 + j); + uint8x16_t v_src0 = vld1q_u8(src0 + j); + + uint16x8_t v_src0_p = vmovl_u8(vget_low_u8(v_src0)); + int16x8_t v_src1_p = vld1q_s16(src1 + j); + float32x4_t v_dst0f = vmulq_n_f32(vmulq_f32(vcvtq_f32_u32(vmovl_u16(vget_low_u16(v_src0_p))), + vcvtq_f32_s32(vmovl_s16(vget_low_s16(v_src1_p)))), scale); + float32x4_t v_dst1f = vmulq_n_f32(vmulq_f32(vcvtq_f32_u32(vmovl_u16(vget_high_u16(v_src0_p))), + vcvtq_f32_s32(vmovl_s16(vget_high_s16(v_src1_p)))), scale); + vst1q_s16(dst + j, vcombine_s16(vmovn_s32(vcvtq_s32_f32(v_dst0f)), + vmovn_s32(vcvtq_s32_f32(v_dst1f)))); + + v_src0_p = vmovl_u8(vget_high_u8(v_src0)); + v_src1_p = vld1q_s16(src1 + j + 8); + v_dst0f = vmulq_n_f32(vmulq_f32(vcvtq_f32_u32(vmovl_u16(vget_low_u16(v_src0_p))), + vcvtq_f32_s32(vmovl_s16(vget_low_s16(v_src1_p)))), scale); + v_dst1f = vmulq_n_f32(vmulq_f32(vcvtq_f32_u32(vmovl_u16(vget_high_u16(v_src0_p))), + vcvtq_f32_s32(vmovl_s16(vget_high_s16(v_src1_p)))), scale); + vst1q_s16(dst + j + 8, vcombine_s16(vmovn_s32(vcvtq_s32_f32(v_dst0f)), + vmovn_s32(vcvtq_s32_f32(v_dst1f)))); + } + for (; j < roiw8; j += 8) + { + uint16x8_t v_src0 = vmovl_u8(vld1_u8(src0 + j)); + int16x8_t v_src1 = vld1q_s16(src1 + j); + float32x4_t v_dst0f = vmulq_n_f32(vmulq_f32(vcvtq_f32_u32(vmovl_u16(vget_low_u16(v_src0))), + vcvtq_f32_s32(vmovl_s16(vget_low_s16(v_src1)))), scale); + float32x4_t v_dst1f = vmulq_n_f32(vmulq_f32(vcvtq_f32_u32(vmovl_u16(vget_high_u16(v_src0))), + vcvtq_f32_s32(vmovl_s16(vget_high_s16(v_src1)))), scale); + vst1q_s16(dst + j, vcombine_s16(vmovn_s32(vcvtq_s32_f32(v_dst0f)), + vmovn_s32(vcvtq_s32_f32(v_dst1f)))); + } + + for (; j < size.width; j++) + { + f32 fval = (f32)src0[j] * (f32)src1[j] * scale; + dst[j] = (s16)(s32)trunc(fval); + } + } + } + } +#else + (void)size; + (void)src0Base; + (void)src0Stride; + (void)src1Base; + (void)src1Stride; + (void)dstBase; + (void)dstStride; + (void)cpolicy; + (void)scale; +#endif +} + +namespace { + +#ifdef CAROTENE_NEON + +template +inline T mulSaturateQ(const T &v1, const T &v2, const float scale) +{ + return internal::vcombine(internal::vqmovn(mulSaturateQ(internal::vmovl(internal::vget_low(v1)), + internal::vmovl(internal::vget_low(v2)), scale)), + internal::vqmovn(mulSaturateQ(internal::vmovl(internal::vget_high(v1)), + internal::vmovl(internal::vget_high(v2)), scale)) + ); +} +template <> +inline int32x4_t mulSaturateQ(const int32x4_t &v1, const int32x4_t &v2, const float scale) +{ return vcvtq_s32_f32(vmulq_n_f32(vmulq_f32(vcvtq_f32_s32(v1), vcvtq_f32_s32(v2)), scale)); } +template <> +inline uint32x4_t mulSaturateQ(const uint32x4_t &v1, const uint32x4_t &v2, const float scale) +{ return vcvtq_u32_f32(vmulq_n_f32(vmulq_f32(vcvtq_f32_u32(v1), vcvtq_f32_u32(v2)), scale)); } + +template +inline T mulSaturate(const T &v1, const T &v2, const float scale) +{ + return internal::vqmovn(mulSaturateQ(internal::vmovl(v1), internal::vmovl(v2), scale)); +} +template <> +inline int32x2_t mulSaturate(const int32x2_t &v1, const int32x2_t &v2, const float scale) +{ return vcvt_s32_f32(vmul_n_f32(vmul_f32(vcvt_f32_s32(v1), vcvt_f32_s32(v2)), scale)); } +template <> +inline uint32x2_t mulSaturate(const uint32x2_t &v1, const uint32x2_t &v2, const float scale) +{ return vcvt_u32_f32(vmul_n_f32(vmul_f32(vcvt_f32_u32(v1), vcvt_f32_u32(v2)), scale)); } + + +template +inline T mulWrapQ(const T &v1, const T &v2, const float scale) +{ + return internal::vcombine(internal::vmovn(mulWrapQ(internal::vmovl(internal::vget_low(v1)), + internal::vmovl(internal::vget_low(v2)), scale)), + internal::vmovn(mulWrapQ(internal::vmovl(internal::vget_high(v1)), + internal::vmovl(internal::vget_high(v2)), scale)) + ); +} +template <> +inline int32x4_t mulWrapQ(const int32x4_t &v1, const int32x4_t &v2, const float scale) +{ return vcvtq_s32_f32(vmulq_n_f32(vmulq_f32(vcvtq_f32_s32(v1), vcvtq_f32_s32(v2)), scale)); } +template <> +inline uint32x4_t mulWrapQ(const uint32x4_t &v1, const uint32x4_t &v2, const float scale) +{ return vcvtq_u32_f32(vmulq_n_f32(vmulq_f32(vcvtq_f32_u32(v1), vcvtq_f32_u32(v2)), scale)); } + +template +inline T mulWrap(const T &v1, const T &v2, const float scale) +{ + return internal::vmovn(mulWrapQ(internal::vmovl(v1), internal::vmovl(v2), scale)); +} +template <> +inline int32x2_t mulWrap(const int32x2_t &v1, const int32x2_t &v2, const float scale) +{ return vcvt_s32_f32(vmul_n_f32(vmul_f32(vcvt_f32_s32(v1), vcvt_f32_s32(v2)), scale)); } +template <> +inline uint32x2_t mulWrap(const uint32x2_t &v1, const uint32x2_t &v2, const float scale) +{ return vcvt_u32_f32(vmul_n_f32(vmul_f32(vcvt_f32_u32(v1), vcvt_f32_u32(v2)), scale)); } + + +template inline uint8x16_t vshrq_n(const uint8x16_t & v0) { return vshrq_n_u8 (v0, n); } +template inline int8x16_t vshrq_n(const int8x16_t & v0) { return vshrq_n_s8 (v0, n); } +template inline uint16x8_t vshrq_n(const uint16x8_t & v0) { return vshrq_n_u16(v0, n); } +template inline int16x8_t vshrq_n(const int16x8_t & v0) { return vshrq_n_s16(v0, n); } +template inline uint32x4_t vshrq_n(const uint32x4_t & v0) { return vshrq_n_u32(v0, n); } +template inline int32x4_t vshrq_n(const int32x4_t & v0) { return vshrq_n_s32(v0, n); } +template inline uint64x2_t vshrq_n(const uint64x2_t & v0) { return vshrq_n_u64(v0, n); } +template inline int64x2_t vshrq_n(const int64x2_t & v0) { return vshrq_n_s64(v0, n); } + +template inline uint8x8_t vshr_n(const uint8x8_t & v0) { return vshr_n_u8 (v0, n); } +template inline int8x8_t vshr_n(const int8x8_t & v0) { return vshr_n_s8 (v0, n); } +template inline uint16x4_t vshr_n(const uint16x4_t & v0) { return vshr_n_u16(v0, n); } +template inline int16x4_t vshr_n(const int16x4_t & v0) { return vshr_n_s16(v0, n); } +template inline uint32x2_t vshr_n(const uint32x2_t & v0) { return vshr_n_u32(v0, n); } +template inline int32x2_t vshr_n(const int32x2_t & v0) { return vshr_n_s32(v0, n); } +template inline uint64x1_t vshr_n(const uint64x1_t & v0) { return vshr_n_u64(v0, n); } +template inline int64x1_t vshr_n(const int64x1_t & v0) { return vshr_n_s64(v0, n); } + +template inline uint8x16_t vrshrq_n(const uint8x16_t & v0) { return vrshrq_n_u8 (v0, n); } +template inline int8x16_t vrshrq_n(const int8x16_t & v0) { return vrshrq_n_s8 (v0, n); } +template inline uint16x8_t vrshrq_n(const uint16x8_t & v0) { return vrshrq_n_u16(v0, n); } +template inline int16x8_t vrshrq_n(const int16x8_t & v0) { return vrshrq_n_s16(v0, n); } +template inline uint32x4_t vrshrq_n(const uint32x4_t & v0) { return vrshrq_n_u32(v0, n); } +template inline int32x4_t vrshrq_n(const int32x4_t & v0) { return vrshrq_n_s32(v0, n); } +template inline uint64x2_t vrshrq_n(const uint64x2_t & v0) { return vrshrq_n_u64(v0, n); } +template inline int64x2_t vrshrq_n(const int64x2_t & v0) { return vrshrq_n_s64(v0, n); } + +template inline uint8x8_t vrshr_n(const uint8x8_t & v0) { return vrshr_n_u8 (v0, n); } +template inline int8x8_t vrshr_n(const int8x8_t & v0) { return vrshr_n_s8 (v0, n); } +template inline uint16x4_t vrshr_n(const uint16x4_t & v0) { return vrshr_n_u16(v0, n); } +template inline int16x4_t vrshr_n(const int16x4_t & v0) { return vrshr_n_s16(v0, n); } +template inline uint32x2_t vrshr_n(const uint32x2_t & v0) { return vrshr_n_u32(v0, n); } +template inline int32x2_t vrshr_n(const int32x2_t & v0) { return vrshr_n_s32(v0, n); } +template inline uint64x1_t vrshr_n(const uint64x1_t & v0) { return vrshr_n_u64(v0, n); } +template inline int64x1_t vrshr_n(const int64x1_t & v0) { return vrshr_n_s64(v0, n); } + +template +void mulShift(const Size2D &size, + const T * src0Base, ptrdiff_t src0Stride, + const T * src1Base, ptrdiff_t src1Stride, + T * dstBase, ptrdiff_t dstStride, + CONVERT_POLICY cpolicy) +{ + typedef typename internal::VecTraits::vec128 vec128; + typedef typename internal::VecTraits::vec128 wvec128; + typedef typename internal::VecTraits::vec64 vec64; + const size_t step128 = 16 / sizeof(T); + size_t roiw128 = size.width >= (step128 - 1) ? size.width - step128 + 1 : 0; + const size_t step64 = 8 / sizeof(T); + size_t roiw64 = size.width >= (step64 - 1) ? size.width - step64 + 1 : 0; + + wvec128 v_mask = internal::vdupq_n((WT)(1<(internal::vqsubq(v_mul0, vshrq_n(internal::vbicq(v_mask, v_mul0)) ))); + vec64 v_res1 = internal::vqmovn(vrshrq_n(internal::vqsubq(v_mul1, vshrq_n(internal::vbicq(v_mask, v_mul1)) ))); + + internal::vst1q(dst + j, internal::vcombine(v_res0, v_res1)); + } + for (; j < roiw64; j += step64) + { + wvec128 v_mul = internal::vmull(internal::vld1(src0 + j), internal::vld1(src1 + j)); + vec64 v_res = internal::vqmovn(vrshrq_n(internal::vqsubq(v_mul, vshrq_n(internal::vbicq(v_mask, v_mul)) ))); + internal::vst1(dst + j, v_res); + } + + for (; j < size.width; j++) + { + WT val = (WT)src0[j] * (WT)src1[j]; + dst[j] = internal::saturate_cast((val - (((1<> shift) + (1<<(shift-1))) >> shift); + } + } + else // CONVERT_POLICY_WRAP + { + for (; j < roiw128; j += step128) + { + internal::prefetch(src0 + j); + internal::prefetch(src1 + j); + vec128 v_src0 = internal::vld1q(src0 + j), v_src1 = internal::vld1q(src1 + j); + wvec128 v_mul0 = internal::vmull( internal::vget_low(v_src0), internal::vget_low(v_src1)); + wvec128 v_mul1 = internal::vmull(internal::vget_high(v_src0), internal::vget_high(v_src1)); + + vec64 v_res0 = internal::vmovn(vrshrq_n(internal::vqsubq(v_mul0, vshrq_n(internal::vbicq(v_mask, v_mul0)) ))); + vec64 v_res1 = internal::vmovn(vrshrq_n(internal::vqsubq(v_mul1, vshrq_n(internal::vbicq(v_mask, v_mul1)) ))); + + internal::vst1q(dst + j, internal::vcombine(v_res0, v_res1)); + } + for (; j < roiw64; j += step64) + { + wvec128 v_mul = internal::vmull(internal::vld1(src0 + j), internal::vld1(src1 + j)); + vec64 v_res = internal::vmovn(vrshrq_n(internal::vqsubq(v_mul, vshrq_n(internal::vbicq(v_mask, v_mul)) ))); + internal::vst1(dst + j, v_res); + } + + for (; j < size.width; j++) + { + WT val = (WT)src0[j] * (WT)src1[j]; + dst[j] = (T)((val - (((1<> shift) + (1<<(shift-1))) >> shift); + } + } + } +} +#endif + +template +void mul(const Size2D &size, + const T * src0Base, ptrdiff_t src0Stride, + const T * src1Base, ptrdiff_t src1Stride, + T * dstBase, ptrdiff_t dstStride, + f32 scale, + CONVERT_POLICY cpolicy) +{ + internal::assertSupportedConfiguration(); + +#ifdef CAROTENE_NEON + typedef typename internal::VecTraits::vec128 vec128; + + typedef void (* mulFunc)(const Size2D &size, + const T * src0Base, ptrdiff_t src0Stride, + const T * src1Base, ptrdiff_t src1Stride, + T * dstBase, ptrdiff_t dstStride, + CONVERT_POLICY cpolicy); + + if (scale == 0.0f || + (std::numeric_limits::is_integer && + (scale * std::numeric_limits::max() * std::numeric_limits::max()) < 1.0f && + (scale * std::numeric_limits::max() * std::numeric_limits::max()) > -1.0f)) + { + for (size_t y = 0; y < size.height; ++y) + { + T * dst = internal::getRowPtr(dstBase, dstStride, y); + std::memset(dst, 0, sizeof(T) * size.width); + } + return; + } + + s32 iscale = static_cast(scale), exp = 0; + f32 significand = frexp(scale, &exp); + bool is_integer_scale = isIntegerScale(scale), + is_power_of_2 = (significand == 0.5f) && (exp <= 0); + exp = -exp + 1; + + if (is_power_of_2) + { + static const mulFunc funcs[16] = + { + NULL, + mulShift, + mulShift, + mulShift, + mulShift, + mulShift, + mulShift, + mulShift, + mulShift, + mulShift, + mulShift, + mulShift, + mulShift, + mulShift, + mulShift, + mulShift + }; + + mulFunc func = funcs[exp]; + + func(size, + src0Base, src0Stride, + src1Base, src1Stride, + dstBase, dstStride, + cpolicy); + + return; + } + + const size_t step128 = 16 / sizeof(T); + size_t roiw128 = size.width >= (step128 - 1) ? size.width - step128 + 1 : 0; + const size_t step64 = 8 / sizeof(T); + size_t roiw64 = size.width >= (step64 - 1) ? size.width - step64 + 1 : 0; + + for (size_t i = 0; i < size.height; ++i) + { + const T * src0 = internal::getRowPtr(src0Base, src0Stride, i); + const T * src1 = internal::getRowPtr(src1Base, src1Stride, i); + T * dst = internal::getRowPtr(dstBase, dstStride, i); + size_t j = 0; + + if (cpolicy == CONVERT_POLICY_SATURATE) + { + if (is_integer_scale && iscale == 1) + { + for (; j < roiw128; j += step128) + { + internal::prefetch(src0 + j); + internal::prefetch(src1 + j); + vec128 v_src0 = internal::vld1q(src0 + j), v_src1 = internal::vld1q(src1 + j); + internal::vst1q(dst + j, internal::vcombine( + internal::vqmovn(internal::vmull(internal::vget_low(v_src0), + internal::vget_low(v_src1))), + internal::vqmovn(internal::vmull(internal::vget_high(v_src0), + internal::vget_high(v_src1))) + ) + ); + } + for (; j < roiw64; j += step64) + { + internal::vst1(dst + j, internal::vqmovn(internal::vmull(internal::vld1(src0 + j), + internal::vld1(src1 + j)))); + } + + for (; j < size.width; j++) + { + WT val = (WT)src0[j] * (WT)src1[j]; + dst[j] = internal::saturate_cast(val); + } + } + else // generic case using floats + { + for (; j < roiw128; j += step128) + { + internal::prefetch(src0 + j); + internal::prefetch(src1 + j); + internal::vst1q(dst + j, mulSaturateQ(internal::vld1q(src0 + j), + internal::vld1q(src1 + j), scale)); + } + for (; j < roiw64; j += step64) + { + internal::vst1(dst + j, mulSaturate(internal::vld1(src0 + j), + internal::vld1(src1 + j), scale)); + } + + for (; j < size.width; j++) + { + f32 fval = (f32)src0[j] * (f32)src1[j] * scale; + dst[j] = internal::saturate_cast(fval); + } + } + } + else // CONVERT_POLICY_WRAP + { + if (is_integer_scale && iscale == 1) + { + for (; j < roiw128; j += step128) + { + internal::prefetch(src0 + j); + internal::prefetch(src1 + j); + vec128 v_src0 = internal::vld1q(src0 + j), v_src1 = internal::vld1q(src1 + j); + internal::vst1q(dst + j, internal::vcombine( + internal::vmovn(internal::vmull(internal::vget_low(v_src0), + internal::vget_low(v_src1))), + internal::vmovn(internal::vmull(internal::vget_high(v_src0), + internal::vget_high(v_src1))) + ) + ); + } + for (; j < roiw64; j += step64) + { + internal::vst1(dst + j, internal::vmovn(internal::vmull(internal::vld1(src0 + j), + internal::vld1(src1 + j)))); + } + + for (; j < size.width; j++) + { + WT val = (WT)src0[j] * (WT)src1[j]; + dst[j] = (T)(val); + } + } + else // generic case using floats + { + for (; j < roiw128; j += step128) + { + internal::prefetch(src0 + j); + internal::prefetch(src1 + j); + internal::vst1q(dst + j, mulWrapQ(internal::vld1q(src0 + j), + internal::vld1q(src1 + j), scale)); + } + for (; j < roiw64; j += step64) + { + internal::vst1(dst + j, mulWrap(internal::vld1(src0 + j), + internal::vld1(src1 + j), scale)); + } + + for (; j < size.width; j++) + { + f32 fval = (f32)src0[j] * (f32)src1[j] * scale; + dst[j] = (T)((s32)trunc(fval)); + } + } + } + } +#else + (void)size; + (void)src0Base; + (void)src0Stride; + (void)src1Base; + (void)src1Stride; + (void)dstBase; + (void)dstStride; + (void)cpolicy; + (void)scale; +#endif +} + +} + +void mul(const Size2D &size, + const s8 * src0Base, ptrdiff_t src0Stride, + const s8 * src1Base, ptrdiff_t src1Stride, + s8 * dstBase, ptrdiff_t dstStride, + f32 scale, + CONVERT_POLICY cpolicy) +{ + mul(size, src0Base, src0Stride, src1Base, src1Stride, dstBase, dstStride, scale, cpolicy); +} + +void mul(const Size2D &size, + const u16 * src0Base, ptrdiff_t src0Stride, + const u16 * src1Base, ptrdiff_t src1Stride, + u16 * dstBase, ptrdiff_t dstStride, + f32 scale, + CONVERT_POLICY cpolicy) +{ + mul(size, src0Base, src0Stride, src1Base, src1Stride, dstBase, dstStride, scale, cpolicy); +} + +void mul(const Size2D &size, + const s16 * src0Base, ptrdiff_t src0Stride, + const s16 * src1Base, ptrdiff_t src1Stride, + s16 * dstBase, ptrdiff_t dstStride, + f32 scale, + CONVERT_POLICY cpolicy) +{ + mul(size, src0Base, src0Stride, src1Base, src1Stride, dstBase, dstStride, scale, cpolicy); +} + +void mul(const Size2D &size, + const s32 * src0Base, ptrdiff_t src0Stride, + const s32 * src1Base, ptrdiff_t src1Stride, + s32 * dstBase, ptrdiff_t dstStride, + f64 scale, + CONVERT_POLICY cpolicy) +{ + internal::assertSupportedConfiguration(); +#ifdef CAROTENE_NEON + typedef void (* mulFunc)(const Size2D &size, + const s32 * src0Base, ptrdiff_t src0Stride, + const s32 * src1Base, ptrdiff_t src1Stride, + s32 * dstBase, ptrdiff_t dstStride, + CONVERT_POLICY cpolicy); + + if (!std::isnormal(scale) || + ((scale * std::numeric_limits::max() * std::numeric_limits::max()) < 1.0f && + (scale * std::numeric_limits::max() * std::numeric_limits::max()) > -1.0f)) + { + for (size_t y = 0; y < size.height; ++y) + { + s32 * dst = internal::getRowPtr(dstBase, dstStride, y); + std::memset(dst, 0, sizeof(s32) * size.width); + } + return; + } + + s32 iscale = static_cast(scale), exp = 0; + f64 significand = frexp(scale, &exp); + bool is_integer_scale = isIntegerScale(scale), + is_power_of_2 = (significand == 0.5) && (exp <= 0); + exp = -exp + 1; + + if (is_power_of_2) + { + static const mulFunc funcs[16] = + { + NULL, + mulShift, + mulShift, + mulShift, + mulShift, + mulShift, + mulShift, + mulShift, + mulShift, + mulShift, + mulShift, + mulShift, + mulShift, + mulShift, + mulShift, + mulShift + }; + + mulFunc func = funcs[exp]; + + func(size, + src0Base, src0Stride, + src1Base, src1Stride, + dstBase, dstStride, + cpolicy); + + return; + } + + size_t roiw128 = size.width >= 3 ? size.width - 3 : 0; + size_t roiw64 = size.width >= 1 ? size.width - 1 : 0; + + for (size_t i = 0; i < size.height; ++i) + { + const s32 * src0 = internal::getRowPtr(src0Base, src0Stride, i); + const s32 * src1 = internal::getRowPtr(src1Base, src1Stride, i); + s32 * dst = internal::getRowPtr(dstBase, dstStride, i); + size_t j = 0; + + if (cpolicy == CONVERT_POLICY_SATURATE) + { + if (is_integer_scale && iscale == 1) + { + for (; j < roiw128; j += 4) + { + internal::prefetch(src0 + j); + internal::prefetch(src1 + j); + int32x4_t v_src0 = internal::vld1q(src0 + j), v_src1 = internal::vld1q(src1 + j); + internal::vst1q(dst + j, internal::vcombine( + internal::vqmovn(internal::vmull(internal::vget_low(v_src0), + internal::vget_low(v_src1))), + internal::vqmovn(internal::vmull(internal::vget_high(v_src0), + internal::vget_high(v_src1))) + ) + ); + } + for (; j < roiw64; j += 2) + { + internal::vst1(dst + j, internal::vqmovn(internal::vmull(internal::vld1(src0 + j), + internal::vld1(src1 + j)))); + } + + for (; j < size.width; j++) + { + s64 val = (s64)src0[j] * (s64)src1[j]; + dst[j] = internal::saturate_cast(val); + } + } + else // generic case using floats + { + for (; j < size.width; j++) + { + f64 fval = src0[j] * src1[j] * scale; + dst[j] = internal::saturate_cast(fval); + } + } + } + else // CONVERT_POLICY_WRAP + { + if (is_integer_scale && iscale == 1) + { + for (; j < roiw128; j += 4) + { + internal::prefetch(src0 + j); + internal::prefetch(src1 + j); + int32x4_t v_src0 = internal::vld1q(src0 + j), v_src1 = internal::vld1q(src1 + j); + internal::vst1q(dst + j, internal::vcombine( + internal::vmovn(internal::vmull(internal::vget_low(v_src0), + internal::vget_low(v_src1))), + internal::vmovn(internal::vmull(internal::vget_high(v_src0), + internal::vget_high(v_src1))) + ) + ); + } + for (; j < roiw64; j += 2) + { + internal::vst1(dst + j, internal::vmovn(internal::vmull(internal::vld1(src0 + j), + internal::vld1(src1 + j)))); + } + + for (; j < size.width; j++) + { + s64 val = (s64)src0[j] * (s64)src1[j]; + dst[j] = (s32)(val); + } + } + else // generic case using floats + { + for (; j < size.width; j++) + { + f64 fval = src0[j] * src1[j] * scale; + dst[j] = (s32)trunc(fval); + } + } + } + } +#else + (void)size; + (void)src0Base; + (void)src0Stride; + (void)src1Base; + (void)src1Stride; + (void)dstBase; + (void)dstStride; + (void)cpolicy; + (void)scale; +#endif +} + +void mul(const Size2D &size, + const f32 * src0Base, ptrdiff_t src0Stride, + const f32 * src1Base, ptrdiff_t src1Stride, + f32 * dstBase, ptrdiff_t dstStride, + f32 scale) +{ + internal::assertSupportedConfiguration(); +#ifdef CAROTENE_NEON + if (scale == 0.0f) + { + for (size_t y = 0; y < size.height; ++y) + { + f32 * dst = internal::getRowPtr(dstBase, dstStride, y); + std::memset(dst, 0, sizeof(f32) * size.width); + } + return; + } + + size_t roiw128 = size.width >= 3 ? size.width - 3 : 0; + size_t roiw64 = size.width >= 1 ? size.width - 1 : 0; + + if (std::fabs(scale - 1.0f) < FLT_EPSILON) + { + for (size_t i = 0; i < size.height; ++i) + { + const f32 * src0 = internal::getRowPtr(src0Base, src0Stride, i); + const f32 * src1 = internal::getRowPtr(src1Base, src1Stride, i); + f32 * dst = internal::getRowPtr(dstBase, dstStride, i); + size_t j = 0; + + for (; j < roiw128; j += 4) + { + internal::prefetch(src0 + j); + internal::prefetch(src1 + j); + vst1q_f32(dst + j, vmulq_f32(vld1q_f32(src0 + j), vld1q_f32(src1 + j))); + } + + for (; j < roiw64; j += 2) + { + vst1_f32(dst + j, vmul_f32(vld1_f32(src0 + j), vld1_f32(src1 + j))); + } + + for (; j < size.width; j++) + { + dst[j] = src0[j] * src1[j]; + } + } + } + else + { + for (size_t i = 0; i < size.height; ++i) + { + const f32 * src0 = internal::getRowPtr(src0Base, src0Stride, i); + const f32 * src1 = internal::getRowPtr(src1Base, src1Stride, i); + f32 * dst = internal::getRowPtr(dstBase, dstStride, i); + size_t j = 0; + + for (; j < roiw128; j += 4) + { + internal::prefetch(src0 + j); + internal::prefetch(src1 + j); + vst1q_f32(dst + j, vmulq_n_f32(vmulq_f32(vld1q_f32(src0 + j), vld1q_f32(src1 + j)), scale)); + } + + for (; j < roiw64; j += 2) + { + vst1_f32(dst + j, vmul_n_f32(vmul_f32(vld1_f32(src0 + j), vld1_f32(src1 + j)), scale)); + } + + for (; j < size.width; j++) + { + dst[j] = src0[j] * src1[j] * scale; + } + } + } +#else + (void)size; + (void)src0Base; + (void)src0Stride; + (void)src1Base; + (void)src1Stride; + (void)dstBase; + (void)dstStride; + (void)scale; +#endif +} + +} // namespace CAROTENE_NS diff --git a/3rdparty/carotene/src/norm.cpp b/3rdparty/carotene/src/norm.cpp new file mode 100644 index 0000000000..6ff2456597 --- /dev/null +++ b/3rdparty/carotene/src/norm.cpp @@ -0,0 +1,1310 @@ +/* + * By downloading, copying, installing or using the software you agree to this license. + * If you do not agree to this license, do not download, install, + * copy or use the software. + * + * + * License Agreement + * For Open Source Computer Vision Library + * (3-clause BSD License) + * + * Copyright (C) 2012-2015, NVIDIA Corporation, all rights reserved. + * Third party copyrights are property of their respective owners. + * + * Redistribution and use in source and binary forms, with or without modification, + * are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * * Neither the names of the copyright holders nor the names of the contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * This software is provided by the copyright holders and contributors "as is" and + * any express or implied warranties, including, but not limited to, the implied + * warranties of merchantability and fitness for a particular purpose are disclaimed. + * In no event shall copyright holders or contributors be liable for any direct, + * indirect, incidental, special, exemplary, or consequential damages + * (including, but not limited to, procurement of substitute goods or services; + * loss of use, data, or profits; or business interruption) however caused + * and on any theory of liability, whether in contract, strict liability, + * or tort (including negligence or otherwise) arising in any way out of + * the use of this software, even if advised of the possibility of such damage. + */ + +#include "common.hpp" + +namespace CAROTENE_NS { + +//magic number; must be multiple of 4 +#define NORM32F_BLOCK_SIZE 2048 + +s32 normInf(const Size2D &_size, + const u8 * srcBase, ptrdiff_t srcStride) +{ + internal::assertSupportedConfiguration(); +#ifdef CAROTENE_NEON + Size2D size(_size); + if (srcStride == (ptrdiff_t)(size.width)) + { + size.width *= size.height; + size.height = 1; + } + s32 result = 0; + for(size_t k = 0; k < size.height; ++k) + { + const u8* src = internal::getRowPtr( srcBase, srcStride, k); + size_t i = 0; + if (size.width >= 16) + { + uint8x16_t s = vld1q_u8(src); + for (i = 16; i <= size.width - 16; i += 16) + { + internal::prefetch(src + i); + uint8x16_t s1 = vld1q_u8(src + i); + s = vmaxq_u8(s1, s); + } + u8 s2[8]; + uint8x8_t s3 = vmax_u8(vget_low_u8(s), vget_high_u8(s)); + vst1_u8(s2, s3); + for (u32 j = 0; j < 8; j++) + result = std::max((s32)(s2[j]), result); + } + for ( ; i < size.width; i++) + result = std::max((s32)(src[i]), result); + } + return result; +#else + (void)_size; + (void)srcBase; + (void)srcStride; + + return 0; +#endif +} + +s32 normInf(const Size2D &_size, + const s8 * srcBase, ptrdiff_t srcStride) +{ + internal::assertSupportedConfiguration(); +#ifdef CAROTENE_NEON + Size2D size(_size); + if (srcStride == (ptrdiff_t)(size.width)) + { + size.width *= size.height; + size.height = 1; + } + s32 result = 0; + for(size_t k = 0; k < size.height; ++k) + { + const s8* src = internal::getRowPtr( srcBase, srcStride, k); + size_t i = 0; + if (size.width >= 16) + { + uint8x16_t s = vreinterpretq_u8_s8(vabsq_s8(vld1q_s8(src))); + for (i = 16; i <= size.width - 16; i += 16) + { + internal::prefetch(src + i); + uint8x16_t s1 = vreinterpretq_u8_s8(vabsq_s8(vld1q_s8(src + i))); + s = vmaxq_u8(s1, s); + } + u8 s2[8]; + uint8x8_t s3 = vmax_u8(vget_low_u8(s), vget_high_u8(s)); + vst1_u8(s2, s3); + for (u32 j = 0; j < 8; j++) + result = std::max((s32)(s2[j]), result); + } + for ( ; i < size.width; i++) + result = std::max((s32)(std::abs(src[i])), result); + } + return result; +#else + (void)_size; + (void)srcBase; + (void)srcStride; + + return 0; +#endif +} + +s32 normInf(const Size2D &_size, + const u16 * srcBase, ptrdiff_t srcStride) +{ + internal::assertSupportedConfiguration(); +#ifdef CAROTENE_NEON + Size2D size(_size); + if (srcStride == (ptrdiff_t)(size.width)) + { + size.width *= size.height; + size.height = 1; + } + s32 result = 0; + for(size_t k = 0; k < size.height; ++k) + { + const u16* src = internal::getRowPtr( srcBase, srcStride, k); + size_t i = 0; + if (size.width >= 8) + { + uint16x8_t s = vld1q_u16(src); + for (i = 8; i <= size.width - 8; i += 8) + { + internal::prefetch(src + i); + uint16x8_t s1 = vld1q_u16(src + i); + s = vmaxq_u16(s1, s); + } + u16 s2[4]; + uint16x4_t s3 = vmax_u16(vget_low_u16(s), vget_high_u16(s)); + vst1_u16(s2, s3); + for (u32 j = 0; j < 4; j++) + result = std::max((s32)(s2[j]), result); + } + for ( ; i < size.width; i++) + result = std::max((s32)(src[i]), result); + } + return result; +#else + (void)_size; + (void)srcBase; + (void)srcStride; + + return 0; +#endif +} + +s32 normInf(const Size2D &_size, + const s16 * srcBase, ptrdiff_t srcStride) +{ + internal::assertSupportedConfiguration(); +#ifdef CAROTENE_NEON + Size2D size(_size); + if (srcStride == (ptrdiff_t)(size.width)) + { + size.width *= size.height; + size.height = 1; + } + s32 result = 0; + for(size_t k = 0; k < size.height; ++k) + { + const s16* src = internal::getRowPtr( srcBase, srcStride, k); + size_t i = 0; + if (size.width >= 8) + { + uint16x8_t s = vreinterpretq_u16_s16(vabsq_s16(vld1q_s16(src))); + for (i = 8; i <= size.width - 8; i += 8) + { + internal::prefetch(src + i); + uint16x8_t s1 = vreinterpretq_u16_s16(vabsq_s16(vld1q_s16(src + i))); + s = vmaxq_u16(s1, s); + } + u16 s2[4]; + uint16x4_t s3 = vmax_u16(vget_low_u16(s), vget_high_u16(s)); + vst1_u16(s2, s3); + for (u32 j = 0; j < 4; j++) + result = std::max((s32)(s2[j]), result); + } + for ( ; i < size.width; i++) + result = std::max(std::abs((s32)(src[i])), result); + } + return result; +#else + (void)_size; + (void)srcBase; + (void)srcStride; + + return 0; +#endif +} + +s32 normInf(const Size2D &_size, + const s32 * srcBase, ptrdiff_t srcStride) +{ + internal::assertSupportedConfiguration(); +#ifdef CAROTENE_NEON + Size2D size(_size); + if (srcStride == (ptrdiff_t)(size.width)) + { + size.width *= size.height; + size.height = 1; + } + s32 result = 0; + for(size_t k = 0; k < size.height; ++k) + { + const s32* src = internal::getRowPtr( srcBase, srcStride, k); + size_t i = 0; + if (size.width >= 4) + { + uint32x4_t s = vreinterpretq_u32_s32(vabsq_s32(vld1q_s32(src))); + for (i = 4; i <= size.width - 4; i += 4) + { + internal::prefetch(src + i); + uint32x4_t s1 = vreinterpretq_u32_s32(vabsq_s32(vld1q_s32(src + i))); + s = vmaxq_u32(s1, s); + } + u32 s2[2]; + uint32x2_t s3 = vmax_u32(vget_low_u32(s), vget_high_u32(s)); + vst1_u32(s2, s3); + for (u32 j = 0; j < 2; j++) + result = std::max((s32)(s2[j]), result); + } + for ( ; i < size.width; i++) + result = std::max((s32)(std::abs(src[i])), result); + } + return result; +#else + (void)_size; + (void)srcBase; + (void)srcStride; + + return 0; +#endif +} + +f32 normInf(const Size2D &_size, + const f32 * srcBase, ptrdiff_t srcStride) +{ + internal::assertSupportedConfiguration(); +#ifdef CAROTENE_NEON + Size2D size(_size); + if (srcStride == (ptrdiff_t)(size.width)) + { + size.width *= size.height; + size.height = 1; + } + f32 result = 0; + for(size_t k = 0; k < size.height; ++k) + { + const f32* src = internal::getRowPtr( srcBase, srcStride, k); + size_t i = 0; + if (size.width >= 4) + { + float32x4_t s = vabsq_f32(vld1q_f32(src)); + for (i = 4; i <= size.width - 4; i += 4 ) + { + internal::prefetch(src + i); + float32x4_t s1 = vld1q_f32(src + i); + float32x4_t sa = vabsq_f32(s1); + s = vmaxq_f32(sa, s); + } + f32 s2[2]; + float32x2_t s3 = vmax_f32(vget_low_f32(s), vget_high_f32(s)); + vst1_f32(s2, s3); + for (u32 j = 0; j < 2; j++) + result = std::max(s2[j], result); + } + for (; i < size.width; i++) + result = std::max(std::abs(src[i]), result); + } + return result; +#else + (void)_size; + (void)srcBase; + (void)srcStride; + + return 0.; +#endif +} + +s32 normL1(const Size2D &_size, + const u8 * srcBase, ptrdiff_t srcStride) +{ + internal::assertSupportedConfiguration(); +#ifdef CAROTENE_NEON + Size2D size(_size); + if (srcStride == (ptrdiff_t)(size.width)) + { + size.width *= size.height; + size.height = 1; + } + size_t roiw8 = size.width >= 7 ? size.width - 7 : 0; + s32 result = 0; + for(size_t k = 0; k < size.height; ++k) + { + const u8* src = internal::getRowPtr( srcBase, srcStride, k); + size_t i = 0; + uint32x4_t vs = vmovq_n_u32(0); + for (; i < roiw8;) + { + size_t limit = std::min(size.width, i + 256) - 8; + uint8x8_t s0 = vld1_u8(src + i); + uint16x8_t s = vmovl_u8(s0); + + for (i += 8; i <= limit; i += 8) + { + internal::prefetch(src + i); + uint8x8_t s1 = vld1_u8(src + i); + s = vaddw_u8(s, s1); + } + + uint16x4_t s4 = vadd_u16(vget_low_u16(s), vget_high_u16(s)); + vs = vaddw_u16(vs, s4); + } + + u32 s2[2]; + uint32x2_t vs2 = vadd_u32(vget_low_u32(vs), vget_high_u32(vs)); + vst1_u32(s2, vs2); + + result += (s32)(s2[0] + s2[1]); + + for ( ; i < size.width; i++) + result += (s32)(src[i]); + } + return result; +#else + (void)_size; + (void)srcBase; + (void)srcStride; + + return 0; +#endif +} + +s32 normL1(const Size2D &_size, + const s8 * srcBase, ptrdiff_t srcStride) +{ + internal::assertSupportedConfiguration(); +#ifdef CAROTENE_NEON + Size2D size(_size); + if (srcStride == (ptrdiff_t)(size.width)) + { + size.width *= size.height; + size.height = 1; + } + size_t roiw8 = size.width >= 7 ? size.width - 7 : 0; + s32 result = 0; + for(size_t k = 0; k < size.height; ++k) + { + const s8* src = internal::getRowPtr( srcBase, srcStride, k); + size_t i = 0; + uint32x4_t vs = vmovq_n_u32(0); + + for (; i < roiw8;) + { + size_t limit = std::min(size.width, i + 256) - 8; + uint8x8_t s0 = vreinterpret_u8_s8(vabs_s8(vld1_s8(src + i))); + uint16x8_t s = vmovl_u8(s0); + + for (i += 8; i <= limit; i += 8) + { + internal::prefetch(src + i); + uint8x8_t s1 = vreinterpret_u8_s8(vabs_s8(vld1_s8(src + i))); + s = vaddw_u8(s, s1); + } + + uint16x4_t s4 = vadd_u16(vget_low_u16(s), vget_high_u16(s)); + vs = vaddw_u16(vs, s4); + } + + u32 s2[2]; + uint32x2_t vs2 = vadd_u32(vget_low_u32(vs), vget_high_u32(vs)); + vst1_u32(s2, vs2); + + result += (s32)(s2[0] + s2[1]); + + for ( ; i < size.width; i++) + result += (s32)(std::abs(src[i])); + } + return result; +#else + (void)_size; + (void)srcBase; + (void)srcStride; + + return 0; +#endif +} + +s32 normL1(const Size2D &_size, + const u16 * srcBase, ptrdiff_t srcStride) +{ + internal::assertSupportedConfiguration(); +#ifdef CAROTENE_NEON + Size2D size(_size); + if (srcStride == (ptrdiff_t)(size.width)) + { + size.width *= size.height; + size.height = 1; + } + size_t roiw4 = size.width >= 3 ? size.width - 3 : 0; + s32 result = 0; + for(size_t k = 0; k < size.height; ++k) + { + const u16* src = internal::getRowPtr( srcBase, srcStride, k); + size_t i = 0; + uint32x4_t vs = vmovq_n_u32(0); + for (; i < roiw4; i += 4) + { + internal::prefetch(src + i); + uint16x4_t s = vld1_u16(src + i); + vs = vaddw_u16(vs, s); + } + u32 s2[4]; + vst1q_u32(s2, vs); + for (u32 j = 0; j < 4; j++) + result += s2[j]; + for ( ; i < size.width; i++) + result += (s32)(src[i]); + } + return result; +#else + (void)_size; + (void)srcBase; + (void)srcStride; + + return 0; +#endif +} + +s32 normL1(const Size2D &_size, + const s16 * srcBase, ptrdiff_t srcStride) +{ + internal::assertSupportedConfiguration(); +#ifdef CAROTENE_NEON + Size2D size(_size); + if (srcStride == (ptrdiff_t)(size.width)) + { + size.width *= size.height; + size.height = 1; + } + size_t roiw4 = size.width >= 3 ? size.width - 3 : 0; + s32 result = 0; + for(size_t k = 0; k < size.height; ++k) + { + const s16* src = internal::getRowPtr( srcBase, srcStride, k); + size_t i = 0; + uint32x4_t vs = vmovq_n_u32(0); + for (; i < roiw4; i += 4) + { + internal::prefetch(src + i); + uint16x4_t s = vreinterpret_u16_s16(vabs_s16(vld1_s16(src + i))); + vs = vaddw_u16(vs, s); + } + u32 s2[4]; + vst1q_u32(s2, vs); + for (u32 j = 0; j < 4; j++) + result += s2[j]; + for ( ; i < size.width; i++) + result += (s32)(std::abs(src[i])); + } + return result; +#else + (void)_size; + (void)srcBase; + (void)srcStride; + + return 0; +#endif +} + +f64 normL1(const Size2D &_size, + const s32 * srcBase, ptrdiff_t srcStride) +{ + internal::assertSupportedConfiguration(); +#ifdef CAROTENE_NEON + Size2D size(_size); + if (srcStride == (ptrdiff_t)(size.width)) + { + size.width *= size.height; + size.height = 1; + } + size_t roiw4 = size.width >= 3 ? size.width - 3 : 0; + f64 result = 0; + for(size_t k = 0; k < size.height; ++k) + { + const s32* src = internal::getRowPtr( srcBase, srcStride, k); + size_t i = 0; + for (; i < roiw4;) + { + size_t limit = std::min(size.width, i + NORM32F_BLOCK_SIZE) - 4; + float32x4_t s = vcvtq_f32_s32(vabsq_s32(vld1q_s32(src + i))); + for (i += 4; i <= limit; i += 4 ) + { + internal::prefetch(src + i); + float32x4_t s1 = vcvtq_f32_s32(vabsq_s32(vld1q_s32(src + i))); + s = vaddq_f32(s, s1); + } + + f32 s2[4]; + vst1q_f32(s2, s); + + for (u32 j = 0; j < 4; j++) + result += (f64)(s2[j]); + } + for ( ; i < size.width; i++) + result += (f64)(std::abs(src[i])); + } + return result; +#else + (void)_size; + (void)srcBase; + (void)srcStride; + + return 0.; +#endif +} + +f64 normL1(const Size2D &_size, + const f32 * srcBase, ptrdiff_t srcStride) +{ + internal::assertSupportedConfiguration(); +#ifdef CAROTENE_NEON + Size2D size(_size); + if (srcStride == (ptrdiff_t)(size.width)) + { + size.width *= size.height; + size.height = 1; + } + size_t roiw4 = size.width >= 3 ? size.width - 3 : 0; + f64 result = 0; + for(size_t k = 0; k < size.height; ++k) + { + const f32* src = internal::getRowPtr( srcBase, srcStride, k); + size_t i = 0; + + for (; i < roiw4;) + { + size_t limit = std::min(size.width, i + NORM32F_BLOCK_SIZE) - 4; + float32x4_t s = vabsq_f32(vld1q_f32(src + i)); + for (i += 4; i <= limit; i += 4) + { + internal::prefetch(src + i); + float32x4_t s1 = vld1q_f32(src + i); + float32x4_t sa = vabsq_f32(s1); + s = vaddq_f32(sa, s); + } + + f32 s2[4]; + vst1q_f32(s2, s); + + for (u32 j = 0; j < 4; j++) + result += (f64)(s2[j]); + } + for (; i < size.width; i++) + result += std::abs((f64)(src[i])); + } + return result; +#else + (void)_size; + (void)srcBase; + (void)srcStride; + + return 0.; +#endif +} + +s32 normL2(const Size2D &_size, + const u8 * srcBase, ptrdiff_t srcStride) +{ + internal::assertSupportedConfiguration(); +#ifdef CAROTENE_NEON + Size2D size(_size); + if (srcStride == (ptrdiff_t)(size.width)) + { + size.width *= size.height; + size.height = 1; + } + size_t roiw8 = size.width >= 7 ? size.width - 7 : 0; + s32 result = 0; + for(size_t k = 0; k < size.height; ++k) + { + const u8* src = internal::getRowPtr( srcBase, srcStride, k); + size_t i = 0; + + uint32x4_t sl = vmovq_n_u32(0); + uint32x4_t sh = vmovq_n_u32(0); + + for (; i < roiw8; i += 8) + { + internal::prefetch(src + i); + uint8x8_t s1 = vld1_u8(src + i); + uint16x8_t sq = vmull_u8(s1, s1); + + sl = vaddw_u16(sl, vget_low_u16(sq)); + sh = vaddw_u16(sh, vget_high_u16(sq)); + } + + uint32x4_t s = vaddq_u32(sl, sh); + uint32x2_t ss = vadd_u32(vget_low_u32(s), vget_high_u32(s)); + + u32 s2[2]; + vst1_u32(s2, ss); + + result += (s32)(s2[0] + s2[1]); + + for (; i < size.width; i++) + result += (s32)(src[i]) * (s32)(src[i]); + } + return result; +#else + (void)_size; + (void)srcBase; + (void)srcStride; + + return 0; +#endif +} + +s32 normL2(const Size2D &_size, + const s8 * srcBase, ptrdiff_t srcStride) +{ + internal::assertSupportedConfiguration(); +#ifdef CAROTENE_NEON + Size2D size(_size); + if (srcStride == (ptrdiff_t)(size.width)) + { + size.width *= size.height; + size.height = 1; + } + size_t roiw8 = size.width >= 7 ? size.width - 7 : 0; + s32 result = 0; + for(size_t k = 0; k < size.height; ++k) + { + const s8* src = internal::getRowPtr( srcBase, srcStride, k); + size_t i = 0; + + int32x4_t sl = vmovq_n_s32(0); + int32x4_t sh = vmovq_n_s32(0); + + for (; i < roiw8; i += 8) + { + internal::prefetch(src + i); + int8x8_t s1 = vld1_s8(src + i); + int16x8_t sq = vmull_s8(s1, s1); + + sl = vaddw_s16(sl, vget_low_s16(sq)); + sh = vaddw_s16(sh, vget_high_s16(sq)); + } + + int32x4_t s = vaddq_s32(sl, sh); + int32x2_t ss = vadd_s32(vget_low_s32(s), vget_high_s32(s)); + + s32 s2[2]; + vst1_s32(s2, ss); + + result += s2[0] + s2[1]; + + for (; i < size.width; i++) + result += (s32)(src[i]) * (s32)(src[i]); + } + return result; +#else + (void)_size; + (void)srcBase; + (void)srcStride; + + return 0; +#endif +} + +f64 normL2(const Size2D &_size, + const u16 * srcBase, ptrdiff_t srcStride) +{ + internal::assertSupportedConfiguration(); +#ifdef CAROTENE_NEON + Size2D size(_size); + if (srcStride == (ptrdiff_t)(size.width)) + { + size.width *= size.height; + size.height = 1; + } + size_t roiw4 = size.width >= 3 ? size.width - 3 : 0; + f64 result = 0; + for(size_t k = 0; k < size.height; ++k) + { + const u16* src = internal::getRowPtr( srcBase, srcStride, k); + size_t i = 0; + for (; i < roiw4;) + { + size_t limit = std::min(size.width, i + NORM32F_BLOCK_SIZE) - 4; + uint16x4_t s0 = vld1_u16(src+i); + float32x4_t s = vcvtq_f32_u32(vmull_u16(s0,s0)); + for (i += 4; i <= limit; i += 4 ) + { + internal::prefetch(src + i); + uint16x4_t s1 = vld1_u16(src+i); + float32x4_t sq = vcvtq_f32_u32(vmull_u16(s1, s1)); + s = vaddq_f32(s, sq); + } + f32 s2[4]; + vst1q_f32(s2, s); + for (u32 j = 0; j < 4; j++) + result += (f64)(s2[j]); + } + + for ( ; i < size.width; i++) + result += (f64)(src[i]) * (f64)(src[i]); + } + return result; +#else + (void)_size; + (void)srcBase; + (void)srcStride; + + return 0.; +#endif +} + +f64 normL2(const Size2D &_size, + const s16 * srcBase, ptrdiff_t srcStride) +{ + internal::assertSupportedConfiguration(); +#ifdef CAROTENE_NEON + Size2D size(_size); + if (srcStride == (ptrdiff_t)(size.width)) + { + size.width *= size.height; + size.height = 1; + } + size_t roiw4 = size.width >= 3 ? size.width - 3 : 0; + f64 result = 0; + for(size_t k = 0; k < size.height; ++k) + { + const s16* src = internal::getRowPtr( srcBase, srcStride, k); + size_t i = 0; + for (; i < roiw4;) + { + size_t limit = std::min(size.width, i + NORM32F_BLOCK_SIZE) - 4; + int16x4_t s0 = vld1_s16(src+i); + float32x4_t s = vcvtq_f32_s32(vmull_s16(s0,s0)); + for (i += 4; i <= limit; i += 4 ) + { + internal::prefetch(src + i); + int16x4_t s1 = vld1_s16(src+i); + float32x4_t sq = vcvtq_f32_s32(vmull_s16(s1, s1)); + s = vaddq_f32(s, sq); + } + f32 s2[4]; + vst1q_f32(s2, s); + for (u32 j = 0; j < 4; j++) + result += (f64)(s2[j]); + } + + for ( ; i < size.width; i++) + result += (f64)(src[i]) * (f64)(src[i]); + } + return result; +#else + (void)_size; + (void)srcBase; + (void)srcStride; + + return 0.; +#endif +} + +f64 normL2(const Size2D &_size, + const s32 * srcBase, ptrdiff_t srcStride) +{ + internal::assertSupportedConfiguration(); +#ifdef CAROTENE_NEON + Size2D size(_size); + if (srcStride == (ptrdiff_t)(size.width)) + { + size.width *= size.height; + size.height = 1; + } + size_t roiw4 = size.width >= 3 ? size.width - 3 : 0; + f64 result = 0; + for(size_t k = 0; k < size.height; ++k) + { + const s32* src = internal::getRowPtr( srcBase, srcStride, k); + size_t i = 0; + for (; i < roiw4;) + { + size_t limit = std::min(size.width, i + NORM32F_BLOCK_SIZE) - 4; + float32x4_t s = vcvtq_f32_s32(vld1q_s32(src + i)); + s = vmulq_f32(s, s); + for (i += 4; i <= limit; i += 4 ) + { + internal::prefetch(src + i); + float32x4_t s1 = vcvtq_f32_s32(vld1q_s32(src + i)); + s = vmlaq_f32(s, s1, s1); + } + + f32 s2[4]; + vst1q_f32(s2, s); + + for (u32 j = 0; j < 4; j++) + result += (f64)(s2[j]); + } + for ( ; i < size.width; i++) + result += (f64)(src[i]) * (f64)(src[i]); + } + return result; +#else + (void)_size; + (void)srcBase; + (void)srcStride; + + return 0.; +#endif +} + +f64 normL2(const Size2D &_size, + const f32 * srcBase, ptrdiff_t srcStride) +{ + internal::assertSupportedConfiguration(); +#ifdef CAROTENE_NEON + Size2D size(_size); + if (srcStride == (ptrdiff_t)(size.width)) + { + size.width *= size.height; + size.height = 1; + } + size_t roiw4 = size.width >= 3 ? size.width - 3 : 0; + f64 result = 0; + for(size_t k = 0; k < size.height; ++k) + { + const f32* src = internal::getRowPtr( srcBase, srcStride, k); + size_t i = 0; + for (; i < roiw4;) + { + size_t limit = std::min(size.width, i + NORM32F_BLOCK_SIZE) - 4; + float32x4_t s = vld1q_f32(src + i); + s = vmulq_f32(s, s); + for (i += 4; i <= limit; i += 4 ) + { + internal::prefetch(src + i); + float32x4_t s1 = vld1q_f32(src + i); + s = vmlaq_f32(s, s1, s1); + } + + f32 s2[4]; + vst1q_f32(s2, s); + + for (u32 j = 0; j < 4; j++) + result += (f64)(s2[j]); + } + for ( ; i < size.width; i++) + result += (f64)(src[i]) * (f64)(src[i]); + } + return result; +#else + (void)_size; + (void)srcBase; + (void)srcStride; + + return 0.; +#endif +} + +s32 diffNormInf(const Size2D &_size, + const u8 * src0Base, ptrdiff_t src0Stride, + const u8 * src1Base, ptrdiff_t src1Stride) +{ + internal::assertSupportedConfiguration(); +#ifdef CAROTENE_NEON + Size2D size(_size); + if (src0Stride == src1Stride && + src0Stride == (ptrdiff_t)(size.width)) + { + size.width *= size.height; + size.height = 1; + } + s32 result = 0; + for(size_t k = 0; k < size.height; ++k) + { + const u8* src1 = internal::getRowPtr( src0Base, src0Stride, k); + const u8* src2 = internal::getRowPtr( src1Base, src1Stride, k); + size_t i = 0; + + if (size.width >= 16) + { + uint8x16_t vs3 = vdupq_n_u8(0); + for (; i < size.width - 16; i += 16) + { + internal::prefetch(src1 + i); + internal::prefetch(src2 + i); + + uint8x16_t vs1 = vld1q_u8(src1 + i); + uint8x16_t vs2 = vld1q_u8(src2 + i); + + vs3 = vmaxq_u8(vs3, vabdq_u8(vs1, vs2)); + } + + u8 s2[8]; + vst1_u8(s2, vpmax_u8(vget_low_u8(vs3), vget_high_u8(vs3))); + + for (u32 j = 0; j < 8; j++) + result = std::max((s32)(s2[j]), result); + } + + for (; i < size.width; i++) + { + result = std::max(std::abs((s32)(src1[i]) - (s32)(src2[i])), result); + } + } + return result; +#else + (void)_size; + (void)src0Base; + (void)src0Stride; + (void)src1Base; + (void)src1Stride; + + return 0; +#endif +} + +f32 diffNormInf(const Size2D &_size, + const f32 * src0Base, ptrdiff_t src0Stride, + const f32 * src1Base, ptrdiff_t src1Stride) +{ + internal::assertSupportedConfiguration(); +#ifdef CAROTENE_NEON + Size2D size(_size); + if (src0Stride == src1Stride && + src0Stride == (ptrdiff_t)(size.width)) + { + size.width *= size.height; + size.height = 1; + } + f32 result = 0; + for(size_t k = 0; k < size.height; ++k) + { + const f32* src1 = internal::getRowPtr( src0Base, src0Stride, k); + const f32* src2 = internal::getRowPtr( src1Base, src1Stride, k); + size_t i = 0; + + if (size.width >= 4) + { + float32x4_t s = vabdq_f32(vld1q_f32(src1), vld1q_f32(src2)); + + for (i += 4; i <= size.width - 4; i += 4 ) + { + internal::prefetch(src1 + i); + internal::prefetch(src2 + i); + + float32x4_t vs1 = vld1q_f32(src1 + i); + float32x4_t vs2 = vld1q_f32(src2 + i); + + float32x4_t vd = vabdq_f32(vs2, vs1); + s = vmaxq_f32(s, vd); + } + + f32 s2[4]; + vst1q_f32(s2, s); + + for (u32 j = 0; j < 4; j++) + if (s2[j] > result) + result = s2[j]; + } + + for (; i < size.width; i++) + { + f32 v = std::abs(src1[i] - src2[i]); + if (v > result) + result = v; + } + } + return result; +#else + (void)_size; + (void)src0Base; + (void)src0Stride; + (void)src1Base; + (void)src1Stride; + + return 0.; +#endif +} + +s32 diffNormL1(const Size2D &_size, + const u8 * src0Base, ptrdiff_t src0Stride, + const u8 * src1Base, ptrdiff_t src1Stride) +{ + internal::assertSupportedConfiguration(); +#ifdef CAROTENE_NEON + Size2D size(_size); + if (src0Stride == src1Stride && + src0Stride == (ptrdiff_t)(size.width)) + { + size.width *= size.height; + size.height = 1; + } + s32 result = 0; + for(size_t k = 0; k < size.height; ++k) + { + const u8* src1 = internal::getRowPtr( src0Base, src0Stride, k); + const u8* src2 = internal::getRowPtr( src1Base, src1Stride, k); + size_t i = 0; + + if (size.width >= 16) + { + for(; i <= size.width - 16;) + { + size_t limit = std::min(size.width, i + 2*256) - 16; + uint16x8_t si1 = vmovq_n_u16(0); + uint16x8_t si2 = vmovq_n_u16(0); + + for (; i <= limit; i += 16) + { + internal::prefetch(src1 + i); + internal::prefetch(src2 + i); + + uint8x16_t vs1 = vld1q_u8(src1 + i); + uint8x16_t vs2 = vld1q_u8(src2 + i); + + si1 = vabal_u8(si1, vget_low_u8(vs1), vget_low_u8(vs2)); + si2 = vabal_u8(si2, vget_high_u8(vs1), vget_high_u8(vs2)); + } + + u32 s2[4]; + vst1q_u32(s2, vaddq_u32(vpaddlq_u16(si1), vpaddlq_u16(si2))); + + for (u32 j = 0; j < 4; j++) + { + if ((s32)(0x7fFFffFFu - s2[j]) <= result) + { + return 0x7fFFffFF; //result already saturated + } + result = (s32)((u32)(result) + s2[j]); + } + } + + } + + for (; i < size.width; i++) + { + u32 v = std::abs((s32)(src1[i]) - (s32)(src2[i])); + + if ((s32)(0x7fFFffFFu - v) <= result) + { + return 0x7fFFffFF; //result already saturated + } + result = (s32)((u32)(result) + v); + } + } + return result; +#else + (void)_size; + (void)src0Base; + (void)src0Stride; + (void)src1Base; + (void)src1Stride; + + return 0; +#endif +} + +f64 diffNormL1(const Size2D &_size, + const f32 * src0Base, ptrdiff_t src0Stride, + const f32 * src1Base, ptrdiff_t src1Stride) +{ + internal::assertSupportedConfiguration(); +#ifdef CAROTENE_NEON + Size2D size(_size); + if (src0Stride == src1Stride && + src0Stride == (ptrdiff_t)(size.width)) + { + size.width *= size.height; + size.height = 1; + } + f64 result = 0; + for(size_t k = 0; k < size.height; ++k) + { + const f32* src1 = internal::getRowPtr( src0Base, src0Stride, k); + const f32* src2 = internal::getRowPtr( src1Base, src1Stride, k); + size_t i = 0; + + if (size.width >= 4) + { + for(; i <= size.width - 4;) + { + size_t limit = std::min(size.width, i + NORM32F_BLOCK_SIZE) - 4; + float32x4_t s = vmovq_n_f32(0.0f); + + for (; i <= limit; i += 4 ) + { + internal::prefetch(src1 + i); + internal::prefetch(src2 + i); + + float32x4_t vs1 = vld1q_f32(src1 + i); + float32x4_t vs2 = vld1q_f32(src2 + i); + + float32x4_t vd = vabdq_f32(vs2, vs1); + s = vaddq_f32(s, vd); + } + + f32 s2[4]; + vst1q_f32(s2, s); + + for (u32 j = 0; j < 4; j++) + result += (f64)(s2[j]); + } + } + + for (; i < size.width; i++) + { + f32 v = std::abs(src1[i] - src2[i]); + result += (f64)(v); + } + } + return result; +#else + (void)_size; + (void)src0Base; + (void)src0Stride; + (void)src1Base; + (void)src1Stride; + + return 0.; +#endif +} + +s32 diffNormL2(const Size2D &_size, + const u8 * src0Base, ptrdiff_t src0Stride, + const u8 * src1Base, ptrdiff_t src1Stride) +{ + internal::assertSupportedConfiguration(); +#ifdef CAROTENE_NEON + Size2D size(_size); + if (src0Stride == src1Stride && + src0Stride == (ptrdiff_t)(size.width)) + { + size.width *= size.height; + size.height = 1; + } + s32 result = 0; + for(size_t k = 0; k < size.height; ++k) + { + const u8* src1 = internal::getRowPtr( src0Base, src0Stride, k); + const u8* src2 = internal::getRowPtr( src1Base, src1Stride, k); + size_t i = 0; + +#define NORML28U_BLOCK_SIZE (33024*2) //bigger block size can result in integer overflow + if (size.width >= 16) + { + for(; i <= size.width - 16;) + { + size_t limit = std::min(size.width, i + NORML28U_BLOCK_SIZE) - 16; + uint32x4_t si1 = vmovq_n_u32(0); + uint32x4_t si2 = vmovq_n_u32(0); + + for (; i <= limit; i += 16) + { + internal::prefetch(src1 + i); + internal::prefetch(src2 + i); + + uint8x16_t vs1 = vld1q_u8(src1 + i); + uint8x16_t vs2 = vld1q_u8(src2 + i); + + uint16x8_t vdlo = vabdl_u8(vget_low_u8(vs1), vget_low_u8(vs2)); + uint16x8_t vdhi = vabdl_u8(vget_high_u8(vs1), vget_high_u8(vs2)); + + si1 = vmlal_u16(si1, vget_low_u16(vdlo), vget_low_u16(vdlo)); + si2 = vmlal_u16(si2, vget_high_u16(vdlo), vget_high_u16(vdlo)); + + si1 = vmlal_u16(si1, vget_low_u16(vdhi), vget_low_u16(vdhi)); + si2 = vmlal_u16(si2, vget_high_u16(vdhi), vget_high_u16(vdhi)); + } + + u32 s2[4]; + vst1q_u32(s2, vqaddq_u32(si1, si2)); + + for (u32 j = 0; j < 4; j++) + { + if ((s32)(0x7fFFffFFu - s2[j]) <= result) + { + return 0x7fFFffFF; //result already saturated + } + result += (s32)s2[j]; + } + } + + } + + for (; i < size.width; i++) + { + s32 v = (s32)(src1[i]) - (s32)(src2[i]); + v *= v; + + if ((s32)(0x7fFFffFFu - (u32)(v)) <= result) + { + return 0x7fFFffFF; //result already saturated + } + result += v; + } + } + return result; +#else + (void)_size; + (void)src0Base; + (void)src0Stride; + (void)src1Base; + (void)src1Stride; + + return 0; +#endif +} + +f64 diffNormL2(const Size2D &_size, + const f32 * src0Base, ptrdiff_t src0Stride, + const f32 * src1Base, ptrdiff_t src1Stride) +{ + internal::assertSupportedConfiguration(); +#ifdef CAROTENE_NEON + Size2D size(_size); + if (src0Stride == src1Stride && + src0Stride == (ptrdiff_t)(size.width)) + { + size.width *= size.height; + size.height = 1; + } + f64 result = 0; + for(size_t k = 0; k < size.height; ++k) + { + const f32* src1 = internal::getRowPtr( src0Base, src0Stride, k); + const f32* src2 = internal::getRowPtr( src1Base, src1Stride, k); + size_t i = 0; + + if (size.width >= 4) + { + for(; i <= size.width - 4;) + { + size_t limit = std::min(size.width, i + NORM32F_BLOCK_SIZE) - 4; + float32x4_t s = vmovq_n_f32(0.0f); + + for (; i <= limit; i += 4 ) + { + internal::prefetch(src1 + i); + internal::prefetch(src2 + i); + + float32x4_t vs1 = vld1q_f32(src1 + i); + float32x4_t vs2 = vld1q_f32(src2 + i); + + float32x4_t vd = vsubq_f32(vs2,vs1); + s = vmlaq_f32(s, vd, vd); + } + + f32 s2[4]; + vst1q_f32(s2, s); + + for (u32 j = 0; j < 4; j++) + result += (f64)(s2[j]); + } + } + + for (; i < size.width; i++) + { + f32 v = src1[i] - src2[i]; + result += v * v; + } + } + return result; +#else + (void)_size; + (void)src0Base; + (void)src0Stride; + (void)src1Base; + (void)src1Stride; + + return 0.; +#endif +} + +} // namespace CAROTENE_NS diff --git a/3rdparty/carotene/src/opticalflow.cpp b/3rdparty/carotene/src/opticalflow.cpp new file mode 100644 index 0000000000..fa9402a05c --- /dev/null +++ b/3rdparty/carotene/src/opticalflow.cpp @@ -0,0 +1,539 @@ +/* + * By downloading, copying, installing or using the software you agree to this license. + * If you do not agree to this license, do not download, install, + * copy or use the software. + * + * + * License Agreement + * For Open Source Computer Vision Library + * (3-clause BSD License) + * + * Copyright (C) 2012-2015, NVIDIA Corporation, all rights reserved. + * Third party copyrights are property of their respective owners. + * + * Redistribution and use in source and binary forms, with or without modification, + * are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * * Neither the names of the copyright holders nor the names of the contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * This software is provided by the copyright holders and contributors "as is" and + * any express or implied warranties, including, but not limited to, the implied + * warranties of merchantability and fitness for a particular purpose are disclaimed. + * In no event shall copyright holders or contributors be liable for any direct, + * indirect, incidental, special, exemplary, or consequential damages + * (including, but not limited to, procurement of substitute goods or services; + * loss of use, data, or profits; or business interruption) however caused + * and on any theory of liability, whether in contract, strict liability, + * or tort (including negligence or otherwise) arising in any way out of + * the use of this software, even if advised of the possibility of such damage. + */ + +#include "common.hpp" +#include "saturate_cast.hpp" +#include +#include // For FLT_EPSILON + +namespace CAROTENE_NS { + +#define CV_DESCALE(x,n) (((x) + (1 << ((n)-1))) >> (n)) + +/* + * Pyramidal Lucas-Kanade Optical Flow level processing + */ +void pyrLKOptFlowLevel(const Size2D &size, s32 cn, + const u8 *prevData, ptrdiff_t prevStride, + const s16 *prevDerivData, ptrdiff_t prevDerivStride, + const u8 *nextData, ptrdiff_t nextStride, + u32 ptCount, + const f32 *prevPts, f32 *nextPts, + u8 *status, f32 *err, + const Size2D &winSize, + u32 terminationCount, f64 terminationEpsilon, + u32 level, u32 maxLevel, bool useInitialFlow, bool getMinEigenVals, + f32 minEigThreshold) +{ + internal::assertSupportedConfiguration(); +#ifdef CAROTENE_NEON + f32 halfWinX = (winSize.width-1)*0.5f, halfWinY = (winSize.height-1)*0.5f; + s32 cn2 = cn*2; + + std::vector _buf(winSize.total()*(cn + cn2)); + s16* IWinBuf = &_buf[0]; + s32 IWinBufStride = winSize.width*cn; + s16* derivIWinBuf = &_buf[winSize.total()*cn]; + s32 derivIWinBufStride = winSize.width*cn2; + + for( u32 ptidx = 0; ptidx < ptCount; ptidx++ ) + { + f32 levscale = (1./(1 << level)); + u32 ptref = ptidx << 1; + f32 prevPtX = prevPts[ptref+0]*levscale; + f32 prevPtY = prevPts[ptref+1]*levscale; + f32 nextPtX; + f32 nextPtY; + if( level == maxLevel ) + { + if( useInitialFlow ) + { + nextPtX = nextPts[ptref+0]*levscale; + nextPtY = nextPts[ptref+1]*levscale; + } + else + { + nextPtX = prevPtX; + nextPtY = prevPtY; + } + } + else + { + nextPtX = nextPts[ptref+0]*2.f; + nextPtY = nextPts[ptref+1]*2.f; + } + nextPts[ptref+0] = nextPtX; + nextPts[ptref+1] = nextPtY; + + s32 iprevPtX, iprevPtY; + s32 inextPtX, inextPtY; + prevPtX -= halfWinX; + prevPtY -= halfWinY; + iprevPtX = floor(prevPtX); + iprevPtY = floor(prevPtY); + + if( iprevPtX < -(s32)winSize.width || iprevPtX >= (s32)size.width || + iprevPtY < -(s32)winSize.height || iprevPtY >= (s32)size.height ) + { + if( level == 0 ) + { + if( status ) + status[ptidx] = false; + if( err ) + err[ptidx] = 0; + } + continue; + } + + f32 a = prevPtX - iprevPtX; + f32 b = prevPtY - iprevPtY; + const s32 W_BITS = 14, W_BITS1 = 14; + const f32 FLT_SCALE = 1.f/(1 << 20); + s32 iw00 = round((1.f - a)*(1.f - b)*(1 << W_BITS)); + s32 iw01 = round(a*(1.f - b)*(1 << W_BITS)); + s32 iw10 = round((1.f - a)*b*(1 << W_BITS)); + s32 iw11 = (1 << W_BITS) - iw00 - iw01 - iw10; + + s32 dstep = prevDerivStride/sizeof(s16); + f32 A11 = 0, A12 = 0, A22 = 0; + + int16x4_t viw00 = vmov_n_s16((s16)iw00); + int16x4_t viw01 = vmov_n_s16((s16)iw01); + int16x4_t viw10 = vmov_n_s16((s16)iw10); + int16x4_t viw11 = vmov_n_s16((s16)iw11); + + float32x4_t vA11 = vmovq_n_f32(0); + float32x4_t vA12 = vmovq_n_f32(0); + float32x4_t vA22 = vmovq_n_f32(0); + + s32 wwcn = winSize.width*cn; + + // extract the patch from the first image, compute covariation matrix of derivatives + s32 x = 0; + for(s32 y = 0; y < (s32)winSize.height; y++ ) + { + const u8* src = prevData + prevStride*(y + iprevPtY) + iprevPtX*cn; + const s16* dsrc = prevDerivData + dstep*(y + iprevPtY) + iprevPtX*cn2; + + s16* Iptr = IWinBuf + y*IWinBufStride; + s16* dIptr = derivIWinBuf + y*derivIWinBufStride; + + internal::prefetch(src + x + prevStride * 2, 0); + for(x = 0; x <= wwcn - 8; x += 8) + { + uint8x8_t vsrc00 = vld1_u8(src + x); + uint8x8_t vsrc10 = vld1_u8(src + x + prevStride); + uint8x8_t vsrc01 = vld1_u8(src + x + cn); + uint8x8_t vsrc11 = vld1_u8(src + x + prevStride + cn); + + int16x8_t vs00 = vreinterpretq_s16_u16(vmovl_u8(vsrc00)); + int16x8_t vs10 = vreinterpretq_s16_u16(vmovl_u8(vsrc10)); + int16x8_t vs01 = vreinterpretq_s16_u16(vmovl_u8(vsrc01)); + int16x8_t vs11 = vreinterpretq_s16_u16(vmovl_u8(vsrc11)); + + int32x4_t vsuml = vmull_s16(vget_low_s16(vs00), viw00); + int32x4_t vsumh = vmull_s16(vget_high_s16(vs10), viw10); + + vsuml = vmlal_s16(vsuml, vget_low_s16(vs01), viw01); + vsumh = vmlal_s16(vsumh, vget_high_s16(vs11), viw11); + + vsuml = vmlal_s16(vsuml, vget_low_s16(vs10), viw10); + vsumh = vmlal_s16(vsumh, vget_high_s16(vs00), viw00); + + vsuml = vmlal_s16(vsuml, vget_low_s16(vs11), viw11); + vsumh = vmlal_s16(vsumh, vget_high_s16(vs01), viw01); + + int16x4_t vsumnl = vrshrn_n_s32(vsuml, W_BITS1-5); + int16x4_t vsumnh = vrshrn_n_s32(vsumh, W_BITS1-5); + + vst1q_s16(Iptr + x, vcombine_s16(vsumnl, vsumnh)); + } + for(; x <= wwcn - 4; x += 4) + { + uint8x8_t vsrc00 = vld1_u8(src + x); + uint8x8_t vsrc10 = vld1_u8(src + x + prevStride); + uint8x8_t vsrc01 = vld1_u8(src + x + cn); + uint8x8_t vsrc11 = vld1_u8(src + x + prevStride + cn); + + int16x4_t vs00 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(vsrc00))); + int16x4_t vs10 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(vsrc10))); + int16x4_t vs01 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(vsrc01))); + int16x4_t vs11 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(vsrc11))); + + int32x4_t vsuml1 = vmull_s16(vs00, viw00); + int32x4_t vsuml2 = vmull_s16(vs01, viw01); + vsuml1 = vmlal_s16(vsuml1, vs10, viw10); + vsuml2 = vmlal_s16(vsuml2, vs11, viw11); + int32x4_t vsuml = vaddq_s32(vsuml1, vsuml2); + + int16x4_t vsumnl = vrshrn_n_s32(vsuml, W_BITS1-5); + + vst1_s16(Iptr + x, vsumnl); + } + + internal::prefetch(dsrc + dstep * 2, 0); + for(x = 0; x <= wwcn - 4; x += 4, dsrc += 4*2, dIptr += 4*2 ) + { +#if __GNUC_MINOR__ < 0 + __asm__ ( + "vld2.16 {d0-d1}, [%[dsrc00]] \n\t" + "vld2.16 {d2-d3}, [%[dsrc10]] \n\t" + "vld2.16 {d4-d5}, [%[dsrc01]] \n\t" + "vld2.16 {d6-d7}, [%[dsrc11]] \n\t" + "vmull.s16 q4, d3, %P[viw10] \n\t" + "vmull.s16 q5, d0, %P[viw00] \n\t" + "vmlal.s16 q4, d7, %P[viw11] \n\t" + "vmlal.s16 q5, d4, %P[viw01] \n\t" + "vmlal.s16 q4, d1, %P[viw00] \n\t" + "vmlal.s16 q5, d2, %P[viw10] \n\t" + "vmlal.s16 q4, d5, %P[viw01] \n\t" + "vmlal.s16 q5, d6, %P[viw11] \n\t" + "vrshrn.s32 d13, q4, %[W_BITS1] \n\t" + "vrshrn.s32 d12, q5, %[W_BITS1] \n\t" + "vmull.s16 q3, d13, d13 \n\t" + "vmull.s16 q4, d12, d12 \n\t" + "vmull.s16 q5, d13, d12 \n\t" + "vcvt.f32.s32 q3, q3 \n\t" + "vcvt.f32.s32 q4, q4 \n\t" + "vcvt.f32.s32 q5, q5 \n\t" + "vadd.f32 %q[vA22], q3 \n\t" + "vadd.f32 %q[vA11], q4 \n\t" + "vadd.f32 %q[vA12], q5 \n\t" + "vst2.16 {d12-d13}, [%[out]] \n\t" + : [vA22] "=w" (vA22), + [vA11] "=w" (vA11), + [vA12] "=w" (vA12) + : "0" (vA22), + "1" (vA11), + "2" (vA12), + [out] "r" (dIptr), + [dsrc00] "r" (dsrc), + [dsrc10] "r" (dsrc + dstep), + [dsrc01] "r" (dsrc + cn2), + [dsrc11] "r" (dsrc + dstep + cn2), + [viw00] "w" (viw00), + [viw10] "w" (viw10), + [viw01] "w" (viw01), + [viw11] "w" (viw11), + [W_BITS1] "I" (W_BITS1) + : "d0","d1","d2","d3","d4","d5","d6","d7","d8","d9","d10","d11","d12","d13" + ); +#else + int16x4x2_t vdsrc00 = vld2_s16(dsrc); + int16x4x2_t vdsrc10 = vld2_s16(dsrc + dstep); + int16x4x2_t vdsrc01 = vld2_s16(dsrc + cn2); + int16x4x2_t vdsrc11 = vld2_s16(dsrc + dstep + cn2); + + int32x4_t vsumy = vmull_s16(vdsrc10.val[1], viw10); + int32x4_t vsumx = vmull_s16(vdsrc00.val[0], viw00); + + vsumy = vmlal_s16(vsumy, vdsrc11.val[1], viw11); + vsumx = vmlal_s16(vsumx, vdsrc01.val[0], viw01); + + vsumy = vmlal_s16(vsumy, vdsrc00.val[1], viw00); + vsumx = vmlal_s16(vsumx, vdsrc10.val[0], viw10); + + vsumy = vmlal_s16(vsumy, vdsrc01.val[1], viw01); + vsumx = vmlal_s16(vsumx, vdsrc11.val[0], viw11); + + int16x4_t vsumny = vrshrn_n_s32(vsumy, W_BITS1); + int16x4_t vsumnx = vrshrn_n_s32(vsumx, W_BITS1); + + int32x4_t va22i = vmull_s16(vsumny, vsumny); + int32x4_t va11i = vmull_s16(vsumnx, vsumnx); + int32x4_t va12i = vmull_s16(vsumnx, vsumny); + + float32x4_t va22f = vcvtq_f32_s32(va22i); + float32x4_t va11f = vcvtq_f32_s32(va11i); + float32x4_t va12f = vcvtq_f32_s32(va12i); + + vA22 = vaddq_f32(vA22, va22f); + vA11 = vaddq_f32(vA11, va11f); + vA12 = vaddq_f32(vA12, va12f); + + int16x4x2_t vsum; + vsum.val[0] = vsumnx; + vsum.val[1] = vsumny; + vst2_s16(dIptr, vsum); +#endif + } + + for( ; x < wwcn; x++, dsrc += 2, dIptr += 2 ) + { + s32 ival = CV_DESCALE(src[x]*iw00 + src[x+cn]*iw01 + + src[x+prevStride]*iw10 + src[x+prevStride+cn]*iw11, W_BITS1-5); + s32 ixval = CV_DESCALE(dsrc[0]*iw00 + dsrc[cn2]*iw01 + + dsrc[dstep]*iw10 + dsrc[dstep+cn2]*iw11, W_BITS1); + s32 iyval = CV_DESCALE(dsrc[1]*iw00 + dsrc[cn2+1]*iw01 + dsrc[dstep+1]*iw10 + + dsrc[dstep+cn2+1]*iw11, W_BITS1); + Iptr[x] = (s16)ival; + dIptr[0] = (s16)ixval; + dIptr[1] = (s16)iyval; + + A11 += (f32)(ixval*ixval); + A12 += (f32)(ixval*iyval); + A22 += (f32)(iyval*iyval); + } + } + + f32 A11buf[2], A12buf[2], A22buf[2]; + vst1_f32(A11buf, vadd_f32(vget_low_f32(vA11), vget_high_f32(vA11))); + vst1_f32(A12buf, vadd_f32(vget_low_f32(vA12), vget_high_f32(vA12))); + vst1_f32(A22buf, vadd_f32(vget_low_f32(vA22), vget_high_f32(vA22))); + A11 += A11buf[0] + A11buf[1]; + A12 += A12buf[0] + A12buf[1]; + A22 += A22buf[0] + A22buf[1]; + + A11 *= FLT_SCALE; + A12 *= FLT_SCALE; + A22 *= FLT_SCALE; + + f32 D = A11*A22 - A12*A12; + f32 minEig = (A22 + A11 - std::sqrt((A11-A22)*(A11-A22) + + 4.f*A12*A12))/(2*winSize.width*winSize.height); + + if( err && getMinEigenVals ) + err[ptidx] = (f32)minEig; + + if( minEig < minEigThreshold || D < FLT_EPSILON ) + { + if( level == 0 && status ) + status[ptidx] = false; + continue; + } + + D = 1.f/D; + + nextPtX -= halfWinX; + nextPtY -= halfWinY; + f32 prevDeltaX = 0; + f32 prevDeltaY = 0; + + for(u32 j = 0; j < terminationCount; j++ ) + { + inextPtX = floor(nextPtX); + inextPtY = floor(nextPtY); + + if( inextPtX < -(s32)winSize.width || inextPtX >= (s32)size.width || + inextPtY < -(s32)winSize.height || inextPtY >= (s32)size.height ) + { + if( level == 0 && status ) + status[ptidx] = false; + break; + } + + a = nextPtX - inextPtX; + b = nextPtY - inextPtY; + iw00 = round((1.f - a)*(1.f - b)*(1 << W_BITS)); + iw01 = round(a*(1.f - b)*(1 << W_BITS)); + iw10 = round((1.f - a)*b*(1 << W_BITS)); + iw11 = (1 << W_BITS) - iw00 - iw01 - iw10; + f32 b1 = 0, b2 = 0; + + viw00 = vmov_n_s16((s16)iw00); + viw01 = vmov_n_s16((s16)iw01); + viw10 = vmov_n_s16((s16)iw10); + viw11 = vmov_n_s16((s16)iw11); + + float32x4_t vb1 = vmovq_n_f32(0); + float32x4_t vb2 = vmovq_n_f32(0); + + for(s32 y = 0; y < (s32)winSize.height; y++ ) + { + const u8* Jptr = nextData + nextStride*(y + inextPtY) + inextPtX*cn; + const s16* Iptr = IWinBuf + y*IWinBufStride; + const s16* dIptr = derivIWinBuf + y*derivIWinBufStride; + + x = 0; + + internal::prefetch(Jptr, nextStride * 2); + internal::prefetch(Iptr, IWinBufStride/2); + internal::prefetch(dIptr, derivIWinBufStride/2); + + for( ; x <= wwcn - 8; x += 8, dIptr += 8*2 ) + { + uint8x8_t vj00 = vld1_u8(Jptr + x); + uint8x8_t vj10 = vld1_u8(Jptr + x + nextStride); + uint8x8_t vj01 = vld1_u8(Jptr + x + cn); + uint8x8_t vj11 = vld1_u8(Jptr + x + nextStride + cn); + int16x8_t vI = vld1q_s16(Iptr + x); + int16x8x2_t vDerivI = vld2q_s16(dIptr); + + int16x8_t vs00 = vreinterpretq_s16_u16(vmovl_u8(vj00)); + int16x8_t vs10 = vreinterpretq_s16_u16(vmovl_u8(vj10)); + int16x8_t vs01 = vreinterpretq_s16_u16(vmovl_u8(vj01)); + int16x8_t vs11 = vreinterpretq_s16_u16(vmovl_u8(vj11)); + + int32x4_t vsuml = vmull_s16(vget_low_s16(vs00), viw00); + int32x4_t vsumh = vmull_s16(vget_high_s16(vs10), viw10); + + vsuml = vmlal_s16(vsuml, vget_low_s16(vs01), viw01); + vsumh = vmlal_s16(vsumh, vget_high_s16(vs11), viw11); + + vsuml = vmlal_s16(vsuml, vget_low_s16(vs10), viw10); + vsumh = vmlal_s16(vsumh, vget_high_s16(vs00), viw00); + + vsuml = vmlal_s16(vsuml, vget_low_s16(vs11), viw11); + vsumh = vmlal_s16(vsumh, vget_high_s16(vs01), viw01); + + int16x4_t vsumnl = vrshrn_n_s32(vsuml, W_BITS1-5); + int16x4_t vsumnh = vrshrn_n_s32(vsumh, W_BITS1-5); + + int16x8_t diff = vqsubq_s16(vcombine_s16(vsumnl, vsumnh), vI); + + int32x4_t vb1l = vmull_s16(vget_low_s16(diff), vget_low_s16(vDerivI.val[0])); + int32x4_t vb2h = vmull_s16(vget_high_s16(diff), vget_high_s16(vDerivI.val[1])); + int32x4_t vb1i = vmlal_s16(vb1l, vget_high_s16(diff), vget_high_s16(vDerivI.val[0])); + int32x4_t vb2i = vmlal_s16(vb2h, vget_low_s16(diff), vget_low_s16(vDerivI.val[1])); + + float32x4_t vb1f = vcvtq_f32_s32(vb1i); + float32x4_t vb2f = vcvtq_f32_s32(vb2i); + + vb1 = vaddq_f32(vb1, vb1f); + vb2 = vaddq_f32(vb2, vb2f); + } + + for( ; x < wwcn; x++, dIptr += 2 ) + { + s32 diff = CV_DESCALE(Jptr[x]*iw00 + Jptr[x+cn]*iw01 + + Jptr[x+nextStride]*iw10 + Jptr[x+nextStride+cn]*iw11, + W_BITS1-5) - Iptr[x]; + b1 += (f32)(diff*dIptr[0]); + b2 += (f32)(diff*dIptr[1]); + } + } + + f32 bbuf[2]; + float32x2_t vb = vpadd_f32(vadd_f32(vget_low_f32(vb1), vget_high_f32(vb1)), vadd_f32(vget_low_f32(vb2), vget_high_f32(vb2))); + vst1_f32(bbuf, vb); + b1 += bbuf[0]; + b2 += bbuf[1]; + + b1 *= FLT_SCALE; + b2 *= FLT_SCALE; + + f32 deltaX = (f32)((A12*b2 - A22*b1) * D); + f32 deltaY = (f32)((A12*b1 - A11*b2) * D); + + nextPtX += deltaX; + nextPtY += deltaY; + nextPts[ptref+0] = nextPtX + halfWinX; + nextPts[ptref+1] = nextPtY + halfWinY; + + if( ((double)deltaX*deltaX + (double)deltaY*deltaY) <= terminationEpsilon ) + break; + + if( j > 0 && std::abs(deltaX + prevDeltaX) < 0.01 && + std::abs(deltaY + prevDeltaY) < 0.01 ) + { + nextPts[ptref+0] -= deltaX*0.5f; + nextPts[ptref+1] -= deltaY*0.5f; + break; + } + prevDeltaX = deltaX; + prevDeltaY = deltaY; + } + + if( status && status[ptidx] && err && level == 0 && !getMinEigenVals ) + { + f32 nextPointX = nextPts[ptref+0] - halfWinX; + f32 nextPointY = nextPts[ptref+1] - halfWinY; + + s32 inextPointX = floor(nextPointX); + s32 inextPointY = floor(nextPointY); + + if( inextPointX < -(s32)winSize.width || inextPointX >= (s32)size.width || + inextPointY < -(s32)winSize.height || inextPointY >= (s32)size.height ) + { + if( status ) + status[ptidx] = false; + continue; + } + + f32 aa = nextPointX - inextPointX; + f32 bb = nextPointY - inextPointY; + iw00 = round((1.f - aa)*(1.f - bb)*(1 << W_BITS)); + iw01 = round(aa*(1.f - bb)*(1 << W_BITS)); + iw10 = round((1.f - aa)*bb*(1 << W_BITS)); + iw11 = (1 << W_BITS) - iw00 - iw01 - iw10; + f32 errval = 0.f; + + for(s32 y = 0; y < (s32)winSize.height; y++ ) + { + const u8* Jptr = nextData + nextStride*(y + inextPointY) + inextPointX*cn; + const s16* Iptr = IWinBuf + y*IWinBufStride; + + for( x = 0; x < wwcn; x++ ) + { + s32 diff = CV_DESCALE(Jptr[x]*iw00 + Jptr[x+cn]*iw01 + + Jptr[x+nextStride]*iw10 + Jptr[x+nextStride+cn]*iw11, + W_BITS1-5) - Iptr[x]; + errval += std::abs((f32)diff); + } + } + err[ptidx] = errval / (32*wwcn*winSize.height); + } + } +#else + (void)size; + (void)cn; + (void)prevData; + (void)prevStride; + (void)prevDerivData; + (void)prevDerivStride; + (void)nextData; + (void)nextStride; + (void)prevPts; + (void)nextPts; + (void)status; + (void)err; + (void)winSize; + (void)terminationCount; + (void)terminationEpsilon; + (void)level; + (void)maxLevel; + (void)useInitialFlow; + (void)getMinEigenVals; + (void)minEigThreshold; + (void)ptCount; +#endif +} + +}//CAROTENE_NS + diff --git a/3rdparty/carotene/src/phase.cpp b/3rdparty/carotene/src/phase.cpp new file mode 100644 index 0000000000..141b1e864a --- /dev/null +++ b/3rdparty/carotene/src/phase.cpp @@ -0,0 +1,274 @@ +/* + * By downloading, copying, installing or using the software you agree to this license. + * If you do not agree to this license, do not download, install, + * copy or use the software. + * + * + * License Agreement + * For Open Source Computer Vision Library + * (3-clause BSD License) + * + * Copyright (C) 2012-2015, NVIDIA Corporation, all rights reserved. + * Third party copyrights are property of their respective owners. + * + * Redistribution and use in source and binary forms, with or without modification, + * are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * * Neither the names of the copyright holders nor the names of the contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * This software is provided by the copyright holders and contributors "as is" and + * any express or implied warranties, including, but not limited to, the implied + * warranties of merchantability and fitness for a particular purpose are disclaimed. + * In no event shall copyright holders or contributors be liable for any direct, + * indirect, incidental, special, exemplary, or consequential damages + * (including, but not limited to, procurement of substitute goods or services; + * loss of use, data, or profits; or business interruption) however caused + * and on any theory of liability, whether in contract, strict liability, + * or tort (including negligence or otherwise) arising in any way out of + * the use of this software, even if advised of the possibility of such damage. + */ + +#include +#include + +#include "common.hpp" + +namespace CAROTENE_NS { + +#ifdef CAROTENE_NEON + +namespace { + +#define FASTATAN2CONST(scale) \ + f32 P1((f32)( 0.9997878412794807 * (180.0 / M_PI) * scale)), \ + P3((f32)(-0.3258083974640975 * (180.0 / M_PI) * scale)), \ + P5((f32)( 0.1555786518463281 * (180.0 / M_PI) * scale)), \ + P7((f32)(-0.04432655554792128 * (180.0 / M_PI) * scale)), \ + A_90((f32)(90.f * scale)), \ + A_180((f32)(180.f * scale)), \ + A_360((f32)(360.f * scale)); \ + float32x4_t eps(vdupq_n_f32((float)DBL_EPSILON)), \ + _90(vdupq_n_f32(A_90)), \ + _180(vdupq_n_f32(A_180)), \ + _360(vdupq_n_f32(A_360)), \ + z(vdupq_n_f32(0.0f)), \ + p1(vdupq_n_f32(P1)), \ + p3(vdupq_n_f32(P3)), \ + p5(vdupq_n_f32(P5)), \ + p7(vdupq_n_f32(P7)); + +#define FASTATAN2SCALAR(y, x, a) \ + { \ + f32 ax = std::abs(x), ay = std::abs(y); \ + f32 c, c2; \ + if (ax >= ay) \ + { \ + c = ay / (ax + (float)DBL_EPSILON); \ + c2 = c * c; \ + a = (((P7 * c2 + P5) * c2 + P3) * c2 + P1) * c; \ + } \ + else \ + { \ + c = ax / (ay + (float)DBL_EPSILON); \ + c2 = c * c; \ + a = A_90 - (((P7 * c2 + P5) * c2 + P3) * c2 + P1) * c; \ + } \ + if (x < 0) \ + a = A_180 - a; \ + if (y < 0) \ + a = A_360 - a; \ + } + +#define FASTATAN2VECTOR(v_y, v_x, a) \ + { \ + float32x4_t ax = vabsq_f32(v_x), ay = vabsq_f32(v_y); \ + float32x4_t tmin = vminq_f32(ax, ay), tmax = vmaxq_f32(ax, ay); \ + float32x4_t c = vmulq_f32(tmin, internal::vrecpq_f32(vaddq_f32(tmax, eps))); \ + float32x4_t c2 = vmulq_f32(c, c); \ + a = vmulq_f32(c2, p7); \ + \ + a = vmulq_f32(vaddq_f32(a, p5), c2); \ + a = vmulq_f32(vaddq_f32(a, p3), c2); \ + a = vmulq_f32(vaddq_f32(a, p1), c); \ + \ + a = vbslq_f32(vcgeq_f32(ax, ay), a, vsubq_f32(_90, a)); \ + a = vbslq_f32(vcltq_f32(v_x, z), vsubq_f32(_180, a), a); \ + a = vbslq_f32(vcltq_f32(v_y, z), vsubq_f32(_360, a), a); \ + \ + } + +} // namespace + +#endif + +void phase(const Size2D &size, + const s16 * src0Base, ptrdiff_t src0Stride, + const s16 * src1Base, ptrdiff_t src1Stride, + u8 * dstBase, ptrdiff_t dstStride) +{ + internal::assertSupportedConfiguration(); +#ifdef CAROTENE_NEON + FASTATAN2CONST(256.0f / 360.0f) + size_t roiw16 = size.width >= 15 ? size.width - 15 : 0; + size_t roiw8 = size.width >= 7 ? size.width - 7 : 0; + + float32x4_t v_05 = vdupq_n_f32(0.5f); + + for (size_t i = 0; i < size.height; ++i) + { + const s16 * src0 = internal::getRowPtr(src0Base, src0Stride, i); + const s16 * src1 = internal::getRowPtr(src1Base, src1Stride, i); + u8 * dst = internal::getRowPtr(dstBase, dstStride, i); + size_t j = 0; + + for (; j < roiw16; j += 16) + { + internal::prefetch(src0 + j); + internal::prefetch(src1 + j); + + int16x8_t v_src00 = vld1q_s16(src0 + j), v_src01 = vld1q_s16(src0 + j + 8); + int16x8_t v_src10 = vld1q_s16(src1 + j), v_src11 = vld1q_s16(src1 + j + 8); + + // 0 + float32x4_t v_src0_p = vcvtq_f32_s32(vmovl_s16(vget_low_s16(v_src00))); + float32x4_t v_src1_p = vcvtq_f32_s32(vmovl_s16(vget_low_s16(v_src10))); + float32x4_t v_dst32f0; + FASTATAN2VECTOR(v_src1_p, v_src0_p, v_dst32f0) + + v_src0_p = vcvtq_f32_s32(vmovl_s16(vget_high_s16(v_src00))); + v_src1_p = vcvtq_f32_s32(vmovl_s16(vget_high_s16(v_src10))); + float32x4_t v_dst32f1; + FASTATAN2VECTOR(v_src1_p, v_src0_p, v_dst32f1) + + uint16x8_t v_dst16s0 = vcombine_u16(vmovn_u32(vcvtq_u32_f32(vaddq_f32(v_dst32f0, v_05))), + vmovn_u32(vcvtq_u32_f32(vaddq_f32(v_dst32f1, v_05)))); + + // 1 + v_src0_p = vcvtq_f32_s32(vmovl_s16(vget_low_s16(v_src01))); + v_src1_p = vcvtq_f32_s32(vmovl_s16(vget_low_s16(v_src11))); + FASTATAN2VECTOR(v_src1_p, v_src0_p, v_dst32f0) + + v_src0_p = vcvtq_f32_s32(vmovl_s16(vget_high_s16(v_src01))); + v_src1_p = vcvtq_f32_s32(vmovl_s16(vget_high_s16(v_src11))); + FASTATAN2VECTOR(v_src1_p, v_src0_p, v_dst32f1) + + uint16x8_t v_dst16s1 = vcombine_u16(vmovn_u32(vcvtq_u32_f32(vaddq_f32(v_dst32f0, v_05))), + vmovn_u32(vcvtq_u32_f32(vaddq_f32(v_dst32f1, v_05)))); + + vst1q_u8(dst + j, vcombine_u8(vmovn_u16(v_dst16s0), + vmovn_u16(v_dst16s1))); + } + for (; j < roiw8; j += 8) + { + int16x8_t v_src0 = vld1q_s16(src0 + j); + int16x8_t v_src1 = vld1q_s16(src1 + j); + + float32x4_t v_src0_p = vcvtq_f32_s32(vmovl_s16(vget_low_s16(v_src0))); + float32x4_t v_src1_p = vcvtq_f32_s32(vmovl_s16(vget_low_s16(v_src1))); + float32x4_t v_dst32f0; + FASTATAN2VECTOR(v_src1_p, v_src0_p, v_dst32f0) + + v_src0_p = vcvtq_f32_s32(vmovl_s16(vget_high_s16(v_src0))); + v_src1_p = vcvtq_f32_s32(vmovl_s16(vget_high_s16(v_src1))); + float32x4_t v_dst32f1; + FASTATAN2VECTOR(v_src1_p, v_src0_p, v_dst32f1) + + uint16x8_t v_dst = vcombine_u16(vmovn_u32(vcvtq_u32_f32(vaddq_f32(v_dst32f0, v_05))), + vmovn_u32(vcvtq_u32_f32(vaddq_f32(v_dst32f1, v_05)))); + + vst1_u8(dst + j, vmovn_u16(v_dst)); + } + + for (; j < size.width; j++) + { + f32 x = src0[j], y = src1[j]; + f32 a; + FASTATAN2SCALAR(y, x, a) + dst[j] = (u8)(s32)floor(a + 0.5f); + } + } +#else + (void)size; + (void)src0Base; + (void)src0Stride; + (void)src1Base; + (void)src1Stride; + (void)dstBase; + (void)dstStride; +#endif +} + +void phase(const Size2D &size, + const f32 * src0Base, ptrdiff_t src0Stride, + const f32 * src1Base, ptrdiff_t src1Stride, + f32 * dstBase, ptrdiff_t dstStride, + f32 scale) +{ + internal::assertSupportedConfiguration(); +#ifdef CAROTENE_NEON + FASTATAN2CONST(scale) + size_t roiw8 = size.width >= 7 ? size.width - 7 : 0; + + for (size_t i = 0; i < size.height; ++i) + { + const f32 * src0 = internal::getRowPtr(src0Base, src0Stride, i); + const f32 * src1 = internal::getRowPtr(src1Base, src1Stride, i); + f32 * dst = internal::getRowPtr(dstBase, dstStride, i); + size_t j = 0; + + for (; j < roiw8; j += 8) + { + internal::prefetch(src0 + j); + internal::prefetch(src1 + j); + + float32x4_t v_src00 = vld1q_f32(src0 + j), v_src01 = vld1q_f32(src0 + j + 4); + float32x4_t v_src10 = vld1q_f32(src1 + j), v_src11 = vld1q_f32(src1 + j + 4); + + float32x4_t v_dst32f; + // 0 + FASTATAN2VECTOR(v_src10, v_src00, v_dst32f) + vst1q_f32(dst + j, v_dst32f); + // 1 + FASTATAN2VECTOR(v_src11, v_src01, v_dst32f) + vst1q_f32(dst + j + 4, v_dst32f); + } + if(j + 4 <= size.width) + { + float32x4_t v_src0 = vld1q_f32(src0 + j); + float32x4_t v_src1 = vld1q_f32(src1 + j); + + float32x4_t v_dst32f; + FASTATAN2VECTOR(v_src1, v_src0, v_dst32f) + vst1q_f32(dst + j, v_dst32f); + j += 4; + } + + for (; j < size.width; j++) + { + f32 a; + FASTATAN2SCALAR(src1[j], src0[j], a) + dst[j] = a; + } + } +#else + (void)size; + (void)src0Base; + (void)src0Stride; + (void)src1Base; + (void)src1Stride; + (void)dstBase; + (void)dstStride; + (void)scale; +#endif +} + +} // namespace CAROTENE_NS diff --git a/3rdparty/carotene/src/pyramid.cpp b/3rdparty/carotene/src/pyramid.cpp new file mode 100644 index 0000000000..546ccecd97 --- /dev/null +++ b/3rdparty/carotene/src/pyramid.cpp @@ -0,0 +1,1414 @@ +/* + * By downloading, copying, installing or using the software you agree to this license. + * If you do not agree to this license, do not download, install, + * copy or use the software. + * + * + * License Agreement + * For Open Source Computer Vision Library + * (3-clause BSD License) + * + * Copyright (C) 2012-2015, NVIDIA Corporation, all rights reserved. + * Third party copyrights are property of their respective owners. + * + * Redistribution and use in source and binary forms, with or without modification, + * are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * * Neither the names of the copyright holders nor the names of the contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * This software is provided by the copyright holders and contributors "as is" and + * any express or implied warranties, including, but not limited to, the implied + * warranties of merchantability and fitness for a particular purpose are disclaimed. + * In no event shall copyright holders or contributors be liable for any direct, + * indirect, incidental, special, exemplary, or consequential damages + * (including, but not limited to, procurement of substitute goods or services; + * loss of use, data, or profits; or business interruption) however caused + * and on any theory of liability, whether in contract, strict liability, + * or tort (including negligence or otherwise) arising in any way out of + * the use of this software, even if advised of the possibility of such damage. + */ + +#include "common.hpp" + +#include + +namespace CAROTENE_NS { + +bool isGaussianPyramidDownRTZSupported(const Size2D &srcSize, const Size2D &dstSize, BORDER_MODE border_mode) +{ + if (!isSupportedConfiguration()) + return false; + // Need at least 8 pixels for vectorization. + // Need to make sure dst width is half the src width. + // Don't care about dst height. + if ( dstSize.width < 8 || std::abs((ptrdiff_t)dstSize.width*2 - (ptrdiff_t)srcSize.width) > 2 ) + return false; + + // Current implementation only supports Reflect101 (ie: UNDEFINED mode) + if (border_mode != BORDER_MODE_UNDEFINED) + return false; + + return true; +} + +bool isGaussianPyramidDownU8Supported(const Size2D &srcSize, const Size2D &dstSize, u8 cn) +{ + if (!isSupportedConfiguration()) + return false; + if ( (dstSize.width * cn) < 8 || + (cn != 1 && cn !=3 && cn!=4) || + std::abs((ptrdiff_t)dstSize.width*2 - (ptrdiff_t)srcSize.width) > 2 || + std::abs((ptrdiff_t)dstSize.height*2 - (ptrdiff_t)srcSize.height) > 2 ) + return false; + + return true; +} + +bool isGaussianPyramidDownS16Supported(const Size2D &srcSize, const Size2D &dstSize, u8 cn) +{ + if (!isSupportedConfiguration()) + return false; + if ( (dstSize.width * cn) < 4 || + (cn != 1 && cn !=3 && cn!=4) || + std::abs((ptrdiff_t)dstSize.width*2 - (ptrdiff_t)srcSize.width) > 2 || + std::abs((ptrdiff_t)dstSize.height*2 - (ptrdiff_t)srcSize.height) > 2 ) + return false; + + return true; +} + +bool isGaussianPyramidDownF32Supported(const Size2D &srcSize, const Size2D &dstSize, u8 cn) +{ + if (!isSupportedConfiguration()) + return false; + if ( (dstSize.width * cn) < 4 || + (cn != 1 && cn !=3 && cn!=4) || + std::abs((ptrdiff_t)dstSize.width*2 - (ptrdiff_t)srcSize.width) > 2 || + std::abs((ptrdiff_t)dstSize.height*2 - (ptrdiff_t)srcSize.height) > 2 ) + return false; + + return true; +} + +bool isGaussianPyramidUpU8Supported(const Size2D &srcSize, const Size2D &dstSize, u8 cn) +{ + if (!isSupportedConfiguration()) + return false; + if ( (srcSize.width * cn) < 8 || + (cn != 1 && cn !=3 && cn!=4) || + std::abs((ptrdiff_t)dstSize.width - (ptrdiff_t)srcSize.width*2) != (ptrdiff_t)dstSize.width % 2 || + std::abs((ptrdiff_t)dstSize.height - (ptrdiff_t)srcSize.height*2) != (ptrdiff_t)dstSize.height % 2 ) + return false; + + return true; +} + +bool isGaussianPyramidUpS16Supported(const Size2D &srcSize, const Size2D &dstSize, u8 cn) +{ + if (!isSupportedConfiguration()) + return false; + if ( (srcSize.width * cn) < 12 || + (cn != 1 && cn !=3 && cn!=4) || + std::abs((ptrdiff_t)dstSize.width - (ptrdiff_t)srcSize.width*2) != (ptrdiff_t)dstSize.width % 2 || + std::abs((ptrdiff_t)dstSize.height - (ptrdiff_t)srcSize.height*2) != (ptrdiff_t)dstSize.height % 2 ) + return false; + + return true; +} + +#ifdef CAROTENE_NEON + +namespace { + +ptrdiff_t borderInterpolate101(ptrdiff_t p, ptrdiff_t len) +{ + if (len == 1) + return 0; + else + { + while ((unsigned)p >= (unsigned)len) + { + if (p < 0) + p = -p; + else + p = (len - 1)*2 - p; + } + } + return p; +} + +} // namespace + +#endif + +void gaussianPyramidDownRTZ(const Size2D &srcSize, + const u8 *srcBase, ptrdiff_t srcStride, + const Size2D &dstSize, + u8 *dstBase, ptrdiff_t dstStride, + BORDER_MODE border, u8 borderValue) +{ + internal::assertSupportedConfiguration(isGaussianPyramidDownRTZSupported(srcSize, dstSize, border)); +#ifdef CAROTENE_NEON + // Single-core NEON code + const size_t dwidth = dstSize.width; + const size_t dheight = dstSize.height; + const size_t swidth = srcSize.width; + const size_t sheight = srcSize.height; + + ptrdiff_t idx_l1 = borderInterpolate101(-1, swidth); + ptrdiff_t idx_l2 = borderInterpolate101(-2, swidth); + ptrdiff_t idx_r1 = borderInterpolate101(swidth + 0, swidth); + ptrdiff_t idx_r2 = borderInterpolate101(swidth + 1, swidth); + + //1-line buffer + std::vector _buf((swidth + 4) + 32/sizeof(u16)); + u16* lane = internal::alignPtr(&_buf[2], 32); + + uint8x8_t vc6u8 = vmov_n_u8(6); + uint16x8_t vc6u16 = vmovq_n_u16(6); + uint16x8_t vc4u16 = vmovq_n_u16(4); + + u8* dst = dstBase; + + for (size_t i = 0; i < dheight; ++i, dst += dstStride) + { + //vertical convolution + const u8* ln0 = internal::getRowPtr(srcBase, srcStride, borderInterpolate101(i*2-2, sheight)); + const u8* ln1 = internal::getRowPtr(srcBase, srcStride, borderInterpolate101(i*2-1, sheight)); + const u8* ln2 = internal::getRowPtr(srcBase, srcStride, borderInterpolate101(i*2+0, sheight)); + const u8* ln3 = internal::getRowPtr(srcBase, srcStride, borderInterpolate101(i*2+1, sheight)); + const u8* ln4 = internal::getRowPtr(srcBase, srcStride, borderInterpolate101(i*2+2, sheight)); + + size_t x = 0; + for (; x <= swidth - 8; x += 8) + { + internal::prefetch(internal::getRowPtr(ln2 + x, srcStride, x % 5 - 2)); + uint8x8_t v0 = vld1_u8(ln0+x); + uint8x8_t v1 = vld1_u8(ln1+x); + uint8x8_t v2 = vld1_u8(ln2+x); + uint8x8_t v3 = vld1_u8(ln3+x); + uint8x8_t v4 = vld1_u8(ln4+x); + + uint16x8_t v = vaddl_u8(v0, v4); + uint16x8_t v13 = vaddl_u8(v1, v3); + + v = vmlal_u8(v, v2, vc6u8); + v = vmlaq_u16(v, v13, vc4u16); + + vst1q_u16(lane + x, v); + } + for (; x < swidth; ++x) + { + lane[x] = ln0[x] + ln4[x] + 4u * (ln1[x] + ln3[x]) + 6u * ln2[x]; + } + + //left&right borders + lane[-1] = lane[idx_l1]; + lane[-2] = lane[idx_l2]; + + lane[swidth] = lane[idx_r1]; + lane[swidth+1] = lane[idx_r2]; + + //horizontal convolution + x = 0; + size_t vw = (swidth/2) - 7; // Using 7 instead of 8 allows swidth of 14 or 15. + for (; x < vw; x += 8) + { + internal::prefetch(lane + 2 * x); + uint16x8x2_t vLane0 = vld2q_u16(lane + 2*x-2); // L0[0] = x0 x2 x4 x6 x8 x10 x12 x14 L0[1] = x1 x3 x5 x7 x9 x11 x13 x15 + uint16x8x2_t vLane1 = vld2q_u16(lane + 2*x-1); // L1[0] = x1 x3 x5 x7 x9 x11 x13 x15 L1[1] = x2 x4 x6 x8 x10 x12 x14 x16 + uint16x8x2_t vLane2 = vld2q_u16(lane + 2*x+0); // L2[0] = x2 x4 x6 x8 x10 x12 x14 x16 L2[1] = x3 x5 x7 x9 x11 x13 x15 x17 + uint16x8x2_t vLane3 = vld2q_u16(lane + 2*x+1); // L3[0] = x3 x5 x7 x9 x11 x13 x15 x17 L3[1] = x4 x6 x8 x10 x12 x14 x16 x18 + uint16x8x2_t vLane4 = vld2q_u16(lane + 2*x+2); // L4[0] = x4 x6 x8 x10 x12 x14 x16 x18 L4[1] = x5 x7 x9 x11 x13 x15 x17 x19 + uint16x8_t vSum_0_4 = vaddq_u16(vLane0.val[0], vLane4.val[0]); + uint16x8_t vSum_1_3 = vaddq_u16(vLane1.val[0], vLane3.val[0]); + vSum_0_4 = vmlaq_u16(vSum_0_4, vLane2.val[0], vc6u16); + vSum_0_4 = vmlaq_u16(vSum_0_4, vSum_1_3, vc4u16); + uint8x8_t vRes = vshrn_n_u16(vSum_0_4, 8); + + vst1_u8(dst + x, vRes); + } + + for (; x < dwidth; x++) + { + dst[x] = u8((lane[2*x-2] + lane[2*x+2] + 4u * (lane[2*x-1] + lane[2*x+1]) + 6u * lane[2*x]) >> 8); + } + } +#else + // Remove 'unused parameter' warnings. + (void)srcSize; + (void)srcBase; + (void)srcStride; + (void)dstSize; + (void)dstBase; + (void)dstStride; + (void)border; +#endif + (void)borderValue; +} + +void gaussianPyramidDown(const Size2D &srcSize, + const u8 *srcBase, ptrdiff_t srcStride, + const Size2D &dstSize, + u8 *dstBase, ptrdiff_t dstStride, u8 cn) +{ + internal::assertSupportedConfiguration(isGaussianPyramidDownU8Supported(srcSize, dstSize, cn)); +#ifdef CAROTENE_NEON + size_t dcolcn = dstSize.width*cn; + size_t scolcn = srcSize.width*cn; + size_t roiw8 = dcolcn - 7; + + size_t idx_l1 = borderInterpolate101(-1, srcSize.width) * cn; + size_t idx_l2 = borderInterpolate101(-2, srcSize.width) * cn; + size_t idx_r1 = borderInterpolate101(srcSize.width + 0, srcSize.width) * cn; + size_t idx_r2 = borderInterpolate101(srcSize.width + 1, srcSize.width) * cn; + + //1-line buffer + std::vector _buf(cn*(srcSize.width + 4) + 32/sizeof(u16)); + u16* lane = internal::alignPtr(&_buf[2*cn], 32); + + uint8x8_t vc6u8 = vmov_n_u8(6); + uint16x8_t vc6u16 = vmovq_n_u16(6); + uint16x8_t vc4u16 = vmovq_n_u16(4); + + for (size_t i = 0; i < dstSize.height; ++i) + { + u8* dst = internal::getRowPtr(dstBase, dstStride, i); + //vertical convolution + const u8* ln0 = internal::getRowPtr(srcBase, srcStride, borderInterpolate101(i*2-2, srcSize.height)); + const u8* ln1 = internal::getRowPtr(srcBase, srcStride, borderInterpolate101(i*2-1, srcSize.height)); + const u8* ln2 = internal::getRowPtr(srcBase, srcStride, borderInterpolate101(i*2+0, srcSize.height)); + const u8* ln3 = internal::getRowPtr(srcBase, srcStride, borderInterpolate101(i*2+1, srcSize.height)); + const u8* ln4 = internal::getRowPtr(srcBase, srcStride, borderInterpolate101(i*2+2, srcSize.height)); + + size_t x = 0; + for (; x <= scolcn - 8; x += 8) + { + internal::prefetch(internal::getRowPtr(ln2 + x, srcStride, (ptrdiff_t)x % 5 - 2)); + uint8x8_t v0 = vld1_u8(ln0+x); + uint8x8_t v1 = vld1_u8(ln1+x); + uint8x8_t v2 = vld1_u8(ln2+x); + uint8x8_t v3 = vld1_u8(ln3+x); + uint8x8_t v4 = vld1_u8(ln4+x); + + uint16x8_t v = vaddl_u8(v0, v4); + uint16x8_t v13 = vaddl_u8(v1, v3); + + v = vmlal_u8(v, v2, vc6u8); + v = vmlaq_u16(v, v13, vc4u16); + + vst1q_u16(lane + x, v); + } + for (; x < scolcn; ++x) + { + lane[x] = ln0[x] + ln4[x] + 4u * (ln1[x] + ln3[x]) + 6u * ln2[x]; + } + + //left&right borders + for (u32 k = 0; k < cn; ++k) + { + lane[(s32)(-cn+k)] = lane[idx_l1 + k]; + lane[(s32)(-cn-cn+k)] = lane[idx_l2 + k]; + + lane[scolcn+k] = lane[idx_r1 + k]; + lane[scolcn+cn+k] = lane[idx_r2 + k]; + } + + //horizontal convolution + x = 0; + switch(cn) + { + case 1: + for (; x < roiw8; x += 8) + { + internal::prefetch(lane + 2 * x); +#if __GNUC_MINOR__ < 7 + __asm__ ( + "vld2.16 {d0-d3}, [%[in0]] \n\t" + "vld2.16 {d4-d7}, [%[in4]] \n\t" + "vld2.16 {d12-d15}, [%[in1]] \n\t" + "vld2.16 {d16-d19}, [%[in3]] \n\t" + "vld2.16 {d8-d11}, [%[in2],:256] \n\t" + "vadd.i16 q0, q2 /*q0 = v0 + v4*/ \n\t" + "vadd.i16 q6, q8 /*q6 = v1 + v3*/ \n\t" + "vmla.i16 q0, q4, %q[c6] /*q0 += v2 * 6*/ \n\t" + "vmla.i16 q0, q6, %q[c4] /*q1 += (v1+v3) * 4*/ \n\t" + "vrshrn.u16 d8, q0, #8 \n\t" + "vst1.8 {d8}, [%[out]] \n\t" + : /*no output*/ + : [out] "r" (dst + x), + [in0] "r" (lane + 2*x-2), + [in1] "r" (lane + 2*x-1), + [in2] "r" (lane + 2*x+0), + [in3] "r" (lane + 2*x+1), + [in4] "r" (lane + 2*x+2), + [c4] "w" (vc4u16), [c6] "w" (vc6u16) + : "d0","d1","d2","d3","d4","d5","d6","d7","d8","d9","d10","d11","d12","d13","d14","d15","d16","d17","d18","d19" + ); +#else + uint16x8x2_t vLane0 = vld2q_u16(lane + 2*x-2); + uint16x8x2_t vLane1 = vld2q_u16(lane + 2*x-1); + uint16x8x2_t vLane2 = vld2q_u16(lane + 2*x+0); + uint16x8x2_t vLane3 = vld2q_u16(lane + 2*x+1); + uint16x8x2_t vLane4 = vld2q_u16(lane + 2*x+2); + + uint16x8_t vSum_0_4 = vaddq_u16(vLane0.val[0], vLane4.val[0]); + uint16x8_t vSum_1_3 = vaddq_u16(vLane1.val[0], vLane3.val[0]); + vSum_0_4 = vmlaq_u16(vSum_0_4, vLane2.val[0], vc6u16); + vSum_0_4 = vmlaq_u16(vSum_0_4, vSum_1_3, vc4u16); + uint8x8_t vRes = vrshrn_n_u16(vSum_0_4, 8); + + vst1_u8(dst + x, vRes); +#endif + } + break; + case 3: + { + uint16x4_t vx1 = vld1_u16(lane - 2*3); + uint16x4_t vx2 = vld1_u16(lane - 1*3); + uint16x4_t vx3 = vld1_u16(lane + 0*3); + uint16x8_t v0 = vcombine_u16(vx1, vx3); + + uint8x8_t map = vreinterpret_u8_u64(vmov_n_u64(0xFFFF060504020100ULL)); + for (; x < roiw8; x += 6) + { + internal::prefetch(lane + 2 * x + 12); + + uint16x4_t vx_ = vld1_u16(lane + 2*x-1*3 + 6); + uint16x4_t vx4 = vld1_u16(lane + 2*x+0*3 + 6); + uint16x4_t vx5 = vld1_u16(lane + 2*x+1*3 + 6); + uint16x4_t vx6 = vld1_u16(lane + 2*x+2*3 + 6); + + uint16x8_t v1 = vcombine_u16(vx2, vx_); + uint16x8_t v2 = vcombine_u16(vget_high_u16(v0), vx4); + uint16x8_t v3 = vcombine_u16(vx_, vx5); + uint16x8_t v4 = vcombine_u16(vx4, vx6); + vx2 = vx5; + + uint16x8_t v = vaddq_u16(v0, v4); + uint16x8_t v13 = vaddq_u16(v1, v3); + + v = vmlaq_u16(v, v2, vc6u16); + v = vmlaq_u16(v, v13, vc4u16); + + uint8x8_t v8 = vrshrn_n_u16(v, 8); + + v0 = v4; + + vst1_u8(dst + x, vtbl1_u8(v8, map)); + } + } + break; + case 4: + { + uint16x4_t vx1 = vld1_u16(lane - 2*4); + uint16x4_t vx2 = vld1_u16(lane - 1*4); + uint16x4_t vx3 = vld1_u16(lane + 0*4); + uint16x8_t v0 = vcombine_u16(vx1, vx3); + + for (; x < roiw8; x += 8) + { + internal::prefetch(lane + 2 * x + 16); + + uint16x4_t vx_ = vld1_u16(lane + 2 * x - 1*4 + 8); + uint16x4_t vx4 = vld1_u16(lane + 2 * x + 0*4 + 8); + uint16x4_t vx5 = vld1_u16(lane + 2 * x + 1*4 + 8); + uint16x4_t vx6 = vld1_u16(lane + 2 * x + 2*4 + 8); + + uint16x8_t v1 = vcombine_u16(vx2, vx_); + uint16x8_t v2 = vcombine_u16(vget_high_u16(v0), vx4); + uint16x8_t v3 = vcombine_u16(vx_, vx5); + uint16x8_t v4 = vcombine_u16(vx4, vx6); + vx2 = vx5; + + uint16x8_t v = vaddq_u16(v0, v4); + uint16x8_t v13 = vaddq_u16(v1, v3); + + v = vmlaq_u16(v, v2, vc6u16); + v = vmlaq_u16(v, v13, vc4u16); + + uint8x8_t v8 = vrshrn_n_u16(v, 8); + + v0 = v4; + + vst1_u8(dst + x, v8); + } + } + break; + } + + for (u32 h = 0; h < cn; ++h) + { + u16* ln = lane + h; + u8* dt = dst + h; + for (size_t k = x; k < dcolcn; k += cn) + dt[k] = u8((ln[2*k-2*cn] + ln[2*k+2*cn] + 4u * (ln[2*k-cn] + ln[2*k+cn]) + 6u * ln[2*k] + (1 << 7)) >> 8); + } + } +#else + // Remove 'unused parameter' warnings. + (void)srcBase; + (void)srcStride; + (void)dstBase; + (void)dstStride; +#endif +} + +void gaussianPyramidDown(const Size2D &srcSize, + const s16 *srcBase, ptrdiff_t srcStride, + const Size2D &dstSize, + s16 *dstBase, ptrdiff_t dstStride, u8 cn) +{ + internal::assertSupportedConfiguration(isGaussianPyramidDownS16Supported(srcSize, dstSize, cn)); +#ifdef CAROTENE_NEON + size_t dcolcn = dstSize.width*cn; + size_t scolcn = srcSize.width*cn; + size_t roiw4 = dcolcn - 3; + + size_t idx_l1 = borderInterpolate101(-1, srcSize.width) * cn; + size_t idx_l2 = borderInterpolate101(-2, srcSize.width) * cn; + size_t idx_r1 = borderInterpolate101(srcSize.width + 0, srcSize.width) * cn; + size_t idx_r2 = borderInterpolate101(srcSize.width + 1, srcSize.width) * cn; + + //1-line buffer + std::vector _buf(cn*(srcSize.width + 4) + 32/sizeof(s32)); + s32* lane = internal::alignPtr(&_buf[2*cn], 32); + + int16x4_t vc6s16 = vmov_n_s16(6); + int32x4_t vc6s32 = vmovq_n_s32(6); + int32x4_t vc4s32 = vmovq_n_s32(4); + + for (size_t i = 0; i < dstSize.height; ++i) + { + s16* dst = internal::getRowPtr(dstBase, dstStride, i); + //vertical convolution + const s16* ln0 = internal::getRowPtr(srcBase, srcStride, borderInterpolate101(i*2-2, srcSize.height)); + const s16* ln1 = internal::getRowPtr(srcBase, srcStride, borderInterpolate101(i*2-1, srcSize.height)); + const s16* ln2 = internal::getRowPtr(srcBase, srcStride, borderInterpolate101(i*2+0, srcSize.height)); + const s16* ln3 = internal::getRowPtr(srcBase, srcStride, borderInterpolate101(i*2+1, srcSize.height)); + const s16* ln4 = internal::getRowPtr(srcBase, srcStride, borderInterpolate101(i*2+2, srcSize.height)); + + size_t x = 0; + for (; x <= scolcn - 4; x += 4) + { + internal::prefetch(internal::getRowPtr(ln2 + x, srcStride, (ptrdiff_t)x % 5 - 2)); + int16x4_t v0 = vld1_s16(ln0 + x); + int16x4_t v1 = vld1_s16(ln1 + x); + int16x4_t v2 = vld1_s16(ln2 + x); + int16x4_t v3 = vld1_s16(ln3 + x); + int16x4_t v4 = vld1_s16(ln4 + x); + + int32x4_t v = vaddl_s16(v0, v4); + int32x4_t v13 = vaddl_s16(v1, v3); + + v = vmlal_s16(v, v2, vc6s16); + v = vmlaq_s32(v, v13, vc4s32); + + vst1q_s32(lane + x, v); + } + for (; x < scolcn; ++x) + { + lane[x] = ln0[x] + ln4[x] + 4 * (ln1[x] + ln3[x]) + 6 * ln2[x]; + } + + //left&right borders + for (u32 k = 0; k < cn; ++k) + { + lane[(s32)(-cn+k)] = lane[idx_l1 + k]; + lane[(s32)(-cn-cn+k)] = lane[idx_l2 + k]; + + lane[scolcn+k] = lane[idx_r1 + k]; + lane[scolcn+cn+k] = lane[idx_r2 + k]; + } + + //horizontal convolution + x = 0; + switch(cn) + { + case 1: + for (; x < roiw4; x += 4) + { + internal::prefetch(lane + 2 * x); +#if __GNUC_MINOR__ < 7 + __asm__ ( + "vld2.32 {d0-d3}, [%[in0]] \n\t" + "vld2.32 {d4-d7}, [%[in4]] \n\t" + "vld2.32 {d12-d15}, [%[in1]] \n\t" + "vld2.32 {d16-d19}, [%[in3]] \n\t" + "vld2.32 {d8-d11}, [%[in2],:256] \n\t" + "vadd.i32 q0, q2 \n\t" + "vadd.i32 q6, q8 \n\t" + "vmla.i32 q0, q4, %q[c6] \n\t" + "vmla.i32 q0, q6, %q[c4] \n\t" + "vrshrn.s32 d8, q0, #8 \n\t" + "vst1.16 {d8}, [%[out]] \n\t" + : /*no output*/ + : [out] "r" (dst + x), + [in0] "r" (lane + 2*x-2), + [in1] "r" (lane + 2*x-1), + [in2] "r" (lane + 2*x+0), + [in3] "r" (lane + 2*x+1), + [in4] "r" (lane + 2*x+2), + [c4] "w" (vc4s32), [c6] "w" (vc6s32) + : "d0","d1","d2","d3","d4","d5","d6","d7","d8","d9","d10","d11","d12","d13","d14","d15","d16","d17","d18","d19" + ); +#else + int32x4x2_t vLane0 = vld2q_s32(lane + 2*x-2); + int32x4x2_t vLane1 = vld2q_s32(lane + 2*x-1); + int32x4x2_t vLane2 = vld2q_s32(lane + 2*x+0); + int32x4x2_t vLane3 = vld2q_s32(lane + 2*x+1); + int32x4x2_t vLane4 = vld2q_s32(lane + 2*x+2); + + int32x4_t vSum_0_4 = vaddq_s32(vLane0.val[0], vLane4.val[0]); + int32x4_t vSum_1_3 = vaddq_s32(vLane1.val[0], vLane3.val[0]); + vSum_0_4 = vmlaq_s32(vSum_0_4, vLane2.val[0], vc6s32); + vSum_0_4 = vmlaq_s32(vSum_0_4, vSum_1_3, vc4s32); + int16x4_t vRes = vrshrn_n_s32(vSum_0_4, 8); + + vst1_s16(dst + x, vRes); +#endif + } + break; + case 3: + { + int32x4_t v0 = vld1q_s32(lane - 2*3); + int32x4_t v1 = vld1q_s32(lane - 1*3); + int32x4_t v2 = vld1q_s32(lane + 0*3); + for (; x < roiw4; x += 3) + { + internal::prefetch(lane + 2 * x); + + int32x4_t v3 = vld1q_s32(lane + 2 * x + 1*3); + int32x4_t v4 = vld1q_s32(lane + 2 * x + 2*3); + + int32x4_t v = vaddq_s32(v0, v4); + int32x4_t v13 = vaddq_s32(v1, v3); + + v = vmlaq_s32(v, v2, vc6s32); + v = vmlaq_s32(v, v13, vc4s32); + + int16x4_t vv = vrshrn_n_s32(v, 8); + + v0 = v2; + v1 = v3; + v2 = v4; + + vst1_s16(dst + x, vv); + } + } + break; + case 4: + { + int32x4_t v0 = vld1q_s32(lane - 2*4); + int32x4_t v1 = vld1q_s32(lane - 1*4); + int32x4_t v2 = vld1q_s32(lane + 0*4); + for (; x < roiw4; x += 4) + { + internal::prefetch(lane + 2 * x + 8); + int32x4_t v3 = vld1q_s32(lane + 2 * x + 1*4); + int32x4_t v4 = vld1q_s32(lane + 2 * x + 2*4); + + int32x4_t v = vaddq_s32(v0, v4); + int32x4_t v13 = vaddq_s32(v1, v3); + + v = vmlaq_s32(v, v2, vc6s32); + v = vmlaq_s32(v, v13, vc4s32); + + int16x4_t vv = vrshrn_n_s32(v, 8); + + v0 = v2; + v1 = v3; + v2 = v4; + + vst1_s16(dst + x, vv); + } + } + break; + } + + for (u32 h = 0; h < cn; ++h) + { + s32* ln = lane + h; + s16* dt = dst + h; + for (size_t k = x; k < dcolcn; k += cn) + dt[k] = s16((ln[2*k-2*cn] + ln[2*k+2*cn] + 4 * (ln[2*k-cn] + ln[2*k+cn]) + 6 * ln[2*k] + (1 << 7)) >> 8); + } + } +#else + // Remove 'unused parameter' warnings. + (void)srcBase; + (void)srcStride; + (void)dstBase; + (void)dstStride; +#endif +} + +void gaussianPyramidDown(const Size2D &srcSize, + const f32 *srcBase, ptrdiff_t srcStride, + const Size2D &dstSize, + f32 *dstBase, ptrdiff_t dstStride, u8 cn) +{ + internal::assertSupportedConfiguration(isGaussianPyramidDownF32Supported(srcSize, dstSize, cn)); +#ifdef CAROTENE_NEON + size_t dcolcn = dstSize.width*cn; + size_t scolcn = srcSize.width*cn; + size_t roiw4 = dcolcn - 3; + + size_t idx_l1 = borderInterpolate101(-1, srcSize.width) * cn; + size_t idx_l2 = borderInterpolate101(-2, srcSize.width) * cn; + size_t idx_r1 = borderInterpolate101(srcSize.width + 0, srcSize.width) * cn; + size_t idx_r2 = borderInterpolate101(srcSize.width + 1, srcSize.width) * cn; + + //1-line buffer + std::vector _buf(cn*(srcSize.width + 4) + 32/sizeof(f32)); + f32* lane = internal::alignPtr(&_buf[2*cn], 32); + +#if __GNUC_MINOR__ < 7 + register float32x4_t vc6d4f32 asm ("q11") = vmovq_n_f32(1.5f); // 6/4 + register float32x4_t vc1d4f32 asm ("q12") = vmovq_n_f32(0.25f); // 1/4 + + register float32x4_t vc1d64f32 asm ("q13") = vmovq_n_f32(0.015625f); //1/4/16 + register float32x4_t vc4d64f32 asm ("q14") = vmovq_n_f32(0.0625f); //4/4/16 + register float32x4_t vc6d64f32 asm ("q15") = vmovq_n_f32(0.09375f); //6/4/16 +#else + register float32x4_t vc6d4f32 = vmovq_n_f32(1.5f); // 6/4 + register float32x4_t vc1d4f32 = vmovq_n_f32(0.25f); // 1/4 + + register float32x4_t vc1d64f32 = vmovq_n_f32(0.015625f); //1/4/16 + register float32x4_t vc4d64f32 = vmovq_n_f32(0.0625f); //4/4/16 + register float32x4_t vc6d64f32 = vmovq_n_f32(0.09375f); //6/4/16 +#endif + + for (size_t i = 0; i < dstSize.height; ++i) + { + f32* dst = internal::getRowPtr(dstBase, dstStride, i); + //vertical convolution + const f32* ln0 = internal::getRowPtr(srcBase, srcStride, borderInterpolate101(i*2-2, srcSize.height)); + const f32* ln1 = internal::getRowPtr(srcBase, srcStride, borderInterpolate101(i*2-1, srcSize.height)); + const f32* ln2 = internal::getRowPtr(srcBase, srcStride, borderInterpolate101(i*2+0, srcSize.height)); + const f32* ln3 = internal::getRowPtr(srcBase, srcStride, borderInterpolate101(i*2+1, srcSize.height)); + const f32* ln4 = internal::getRowPtr(srcBase, srcStride, borderInterpolate101(i*2+2, srcSize.height)); + + size_t x = 0; + for (; x <= scolcn - 4; x += 4) + { + internal::prefetch(internal::getRowPtr(ln2 + x, srcStride, (ptrdiff_t)x % 5 - 2)); + float32x4_t v0 = vld1q_f32((const float32_t*)ln0 + x); + float32x4_t v1 = vld1q_f32((const float32_t*)ln1 + x); + float32x4_t v2 = vld1q_f32((const float32_t*)ln2 + x); + float32x4_t v3 = vld1q_f32((const float32_t*)ln3 + x); + float32x4_t v4 = vld1q_f32((const float32_t*)ln4 + x); + + float32x4_t v = vaddq_f32(v1, v3); + float32x4_t v04 = vaddq_f32(v0, v4); + + v = vmlaq_f32(v, v2, vc6d4f32); + v = vmlaq_f32(v, v04, vc1d4f32); + + vst1q_f32(lane + x, v); + } + for (; x < scolcn; ++x) + { + lane[x] = 0.25f*(ln0[x] + ln4[x]) + (ln1[x] + ln3[x]) + 1.5f * ln2[x]; + } + + //left&right borders + for (u32 k = 0; k < cn; ++k) + { + lane[(s32)(-cn+k)] = lane[idx_l1 + k]; + lane[(s32)(-cn-cn+k)] = lane[idx_l2 + k]; + + lane[scolcn+k] = lane[idx_r1 + k]; + lane[scolcn+cn+k] = lane[idx_r2 + k]; + } + + //horizontal convolution + x = 0; + switch(cn) + { + case 1: + for (; x < roiw4; x += 4) + { + internal::prefetch(lane + 2 * x); +#if __GNUC_MINOR__ < 7 + __asm__ __volatile__ ( + "vld2.32 {d0-d3}, [%[in0]] \n\t" + "vld2.32 {d8-d11}, [%[in4]] \n\t" + "vld2.32 {d14-d17}, [%[in2],:256] \n\t" + "vld2.32 {d10-d13}, [%[in1]] \n\t" + "vld2.32 {d16-d19}, [%[in3]] \n\t" + "vmul.f32 q7, %q[c6d64] \n\t" + "vadd.f32 q0, q4 @v04 \n\t" + "vadd.f32 q5, q8 @v13 \n\t" + "vmla.f32 q7, q0, %q[c1d64] \n\t" + "vmla.f32 q7, q5, %q[c4d64] \n\t" + "vst1.32 {d14-d15}, [%[out]] \n\t" + : + : [out] "r" (dst + x), + [in0] "r" (lane + 2*x-2), + [in1] "r" (lane + 2*x-1), + [in2] "r" (lane + 2*x+0), + [in3] "r" (lane + 2*x+1), + [in4] "r" (lane + 2*x+2), + [c4d64] "w" (vc4d64f32), [c6d64] "w" (vc6d64f32), [c1d64] "w" (vc1d64f32) + : "d0","d1","d2","d3","d4",/*"d5","d6","d7",*/"d8","d9","d10","d11","d12","d13","d14","d15","d16","d17","d18","d19" //ugly compiler "bug" - can't touch d5-d7 + ); +#else + float32x4x2_t vLane0 = vld2q_f32(lane + 2*x-2); + float32x4x2_t vLane1 = vld2q_f32(lane + 2*x-1); + float32x4x2_t vLane2 = vld2q_f32(lane + 2*x+0); + float32x4x2_t vLane3 = vld2q_f32(lane + 2*x+1); + float32x4x2_t vLane4 = vld2q_f32(lane + 2*x+2); + + float32x4_t vSum_0_4 = vaddq_f32(vLane0.val[0], vLane4.val[0]); + float32x4_t vSum_1_3 = vaddq_f32(vLane1.val[0], vLane3.val[0]); + float32x4_t vRes = vmulq_f32(vLane2.val[0], vc6d64f32); + vRes = vmlaq_f32(vRes, vSum_0_4, vc1d64f32); + vRes = vmlaq_f32(vRes, vSum_1_3, vc4d64f32); + + vst1q_f32(dst + x, vRes); +#endif + } + break; + case 3: + { + float32x4_t v0 = vld1q_f32((const float32_t*)lane - 2*3); + float32x4_t v1 = vld1q_f32((const float32_t*)lane - 1*3); + float32x4_t v2 = vld1q_f32((const float32_t*)lane + 0*3); + + for (; x < roiw4; x += 3) + { + internal::prefetch(lane + 2 * x); + + float32x4_t v3 = vld1q_f32((const float32_t*)lane + 2 * x + 1*3); + float32x4_t v4 = vld1q_f32((const float32_t*)lane + 2 * x + 2*3); + + float32x4_t v04 = vaddq_f32(v0, v4); + float32x4_t v13 = vaddq_f32(v1, v3); + + float32x4_t v = vmulq_f32(v2, vc6d64f32); + v = vmlaq_f32(v, v04, vc1d64f32); + v = vmlaq_f32(v, v13, vc4d64f32); + + v0 = v2; + v1 = v3; + v2 = v4; + + vst1q_f32(dst + x, v); + } + } + break; + case 4: + { + float32x4_t v0 = vld1q_f32((const float32_t*)lane - 2*4); + float32x4_t v1 = vld1q_f32((const float32_t*)lane - 1*4); + float32x4_t v2 = vld1q_f32((const float32_t*)lane + 0*4); + + for (; x < roiw4; x += 4) + { + internal::prefetch(lane + 2 * x + 8); + + float32x4_t v3 = vld1q_f32((const float32_t*)lane + 2 * x + 1*4); + float32x4_t v4 = vld1q_f32((const float32_t*)lane + 2 * x + 2*4); + + float32x4_t v04 = vaddq_f32(v0, v4); + float32x4_t v13 = vaddq_f32(v1, v3); + + float32x4_t v = vmulq_f32(v2, vc6d64f32); + v = vmlaq_f32(v, v04, vc1d64f32); + v = vmlaq_f32(v, v13, vc4d64f32); + + v0 = v2; + v1 = v3; + v2 = v4; + + vst1q_f32(dst + x, v); + } + } + break; + } + + for (u32 h = 0; h < cn; ++h) + { + f32* ln = lane + h; + f32* dt = dst + h; + for (size_t k = x; k < dcolcn; k += cn) + dt[k] = 0.015625f * (ln[2*k-2*cn] + ln[2*k+2*cn]) + 0.0625f * (ln[2*k-cn] + ln[2*k+cn]) + 0.09375f * ln[2*k]; + } + } +#else + // Remove 'unused parameter' warnings. + (void)srcBase; + (void)srcStride; + (void)dstBase; + (void)dstStride; +#endif +} + +void gaussianPyramidUp(const Size2D &srcSize, + const u8 *srcBase, ptrdiff_t srcStride, + const Size2D &dstSize, + u8 *dstBase, ptrdiff_t dstStride, u8 cn) +{ + internal::assertSupportedConfiguration(isGaussianPyramidUpU8Supported(srcSize, dstSize, cn)); +#ifdef CAROTENE_NEON + size_t dcolshn = (dstSize.width/2) * cn; + size_t dcolshw = ((dstSize.width+1)/2) * cn; + size_t scolsn = srcSize.width*cn; + + size_t idx_l = (borderInterpolate101(-2, 2 * srcSize.width)/2) * cn; + size_t idx_r1 = (borderInterpolate101(2 * srcSize.width + 0, 2 * srcSize.width)/2) * cn; + size_t idx_r2 = (borderInterpolate101(2 * srcSize.width + 2, 2 * srcSize.width + 2)/2) * cn; + + //2-lines buffer + std::vector _buf(2*(cn*(srcSize.width + 3) + 32/sizeof(u16))); + u16* lane0 = internal::alignPtr(&_buf[cn], 32); + u16* lane1 = internal::alignPtr(lane0 + (3 + srcSize.width)*cn, 32); + + uint8x8_t vc6u8 = vmov_n_u8(6); + uint16x8_t vc6u16 = vmovq_n_u16(6); + + for (size_t i = 0; i < (dstSize.height + 1)/2; ++i) + { + u8* dst = internal::getRowPtr(dstBase, dstStride, 2*i); + //vertical convolution + const u8* ln0 = internal::getRowPtr(srcBase, srcStride, borderInterpolate101(i * 2 - 2, srcSize.height * 2)/2); + const u8* ln1 = internal::getRowPtr(srcBase, srcStride, borderInterpolate101(i * 2 + 0, srcSize.height * 2)/2); + const u8* ln2 = internal::getRowPtr(srcBase, srcStride, borderInterpolate101(i * 2 + 2, srcSize.height * 2)/2); + + size_t x = 0; + for (; x <= scolsn - 8; x += 8) + { + internal::prefetch(internal::getRowPtr(ln1 + x, srcStride, (ptrdiff_t)x % 3 - 1)); + uint8x8_t v0 = vld1_u8(ln0+x); + uint8x8_t v2 = vld1_u8(ln2+x); + uint8x8_t v1 = vld1_u8(ln1+x); + + uint16x8_t vl0 = vaddl_u8(v0, v2); + uint16x8_t vl1 = vaddl_u8(v1, v2); + + vl0 = vmlal_u8(vl0, v1, vc6u8); + vl1 = vshlq_n_u16(vl1, 2); + + vst1q_u16(lane0 + x, vl0); + vst1q_u16(lane1 + x, vl1); + } + for (; x < scolsn; ++x) + { + lane0[x] = ln0[x] + ln2[x] + 6u * ln1[x]; + lane1[x] = 4u * (ln1[x] + ln2[x]); + } + + //left&right borders + for (u32 k = 0; k < cn; ++k) + { + lane0[(s32)(-cn+k)] = lane0[idx_l + k]; + lane1[(s32)(-cn+k)] = lane1[idx_l + k]; + + lane0[scolsn+k] = lane0[idx_r1 + k]; + lane0[scolsn+cn+k] = lane0[idx_r2 + k]; + lane1[scolsn+k] = lane1[idx_r1 + k]; + lane1[scolsn+cn+k] = lane1[idx_r2 + k]; + } + + //horizontal convolution + const u16* lane = lane0; +pyrUp8uHorizontalConvolution: + x = 0; + size_t lim; + switch(cn) + { + case 1: + lim = dcolshn > 7 ? dcolshn - 7 : 0; + for (; x < lim; x += 8) + { + internal::prefetch(lane + x); +#if defined(__GNUC__) && defined(__arm__) + __asm__ ( + "vld1.16 {d0-d1}, [%[in0]] /*q0 = v0*/ \n\t" + "vld1.16 {d2-d3}, [%[in2]] /*q1 = v2*/ \n\t" + "vld1.16 {d4-d5}, [%[in1],:128] /*q2 = v1*/ \n\t" + "vadd.i16 q0, q1 /*q0 = v0 + v2*/ \n\t" + "vadd.i16 q3, q1, q2 /*q3 = v1 + v2*/ \n\t" + "vmla.i16 q0, q2, %q[c6] /*q0 += v1*6*/ \n\t" + "vrshrn.u16 d9, q3, #4 \n\t" + "vrshrn.u16 d8, q0, #6 \n\t" + "vst2.8 {d8-d9}, [%[out]] \n\t" + : /*no output*/ + : [out] "r" (dst + x*2), + [in0] "r" (lane + x - 1), + [in1] "r" (lane + x + 0), + [in2] "r" (lane + x + 1), + [c6] "w" (vc6u16) + : "d0","d1","d2","d3","d4","d5","d6","d7","d8","d9" + ); +#else + uint16x8_t vLane0 = vld1q_u16(lane + x - 1); + uint16x8_t vLane1 = vld1q_u16(lane + x + 0); + uint16x8_t vLane2 = vld1q_u16(lane + x + 1); + + vLane0 = vaddq_u16(vLane0, vLane2); + vLane2 = vaddq_u16(vLane2, vLane1); + vLane0 = vmlaq_u16(vLane0, vLane1, vc6u16); + uint8x8x2_t vRes; + vRes.val[0] = vrshrn_n_u16(vLane0, 6); + vRes.val[1] = vrshrn_n_u16(vLane2, 4); + + vst2_u8(dst + x*2, vRes); +#endif + } + break; + case 3: + { + lim = dcolshn > 23 ? dcolshn - 23 : 0; + for (; x < lim; x += 24) + { + internal::prefetch(lane + x); +#if defined(__GNUC__) && defined(__arm__) + __asm__ ( + "vmov.u16 q9, #6 \n\t" + "vld3.16 {d0, d2, d4}, [%[in0]] /*v0*/ \n\t" + "vld3.16 {d1, d3, d5}, [%[in02]] \n\t" + "vld3.16 {d6, d8, d10}, [%[in2]] /*v2*/ \n\t" + "vld3.16 {d7, d9, d11}, [%[in22]] \n\t" + "vld3.16 {d12, d14, d16}, [%[in1]] /*v1*/ \n\t" + "vld3.16 {d13, d15, d17}, [%[in12]] \n\t" + "vadd.i16 q0, q3 /*v0 + v2*/ \n\t" + "vadd.i16 q1, q4 /*v0 + v2*/ \n\t" + "vadd.i16 q2, q5 /*v0 + v2*/ \n\t" + "vadd.i16 q3, q6 /*v1 + v2*/ \n\t" + "vadd.i16 q4, q7 /*v1 + v2*/ \n\t" + "vadd.i16 q5, q8 /*v1 + v2*/ \n\t" + "vmla.i16 q0, q6, q9 /*v0 + v2 + v1*6 */ \n\t" + "vmla.i16 q1, q7, q9 /*v0 + v2 + v1*6 */ \n\t" + "vmla.i16 q2, q8, q9 /*v0 + v2 + v1*6 */ \n\t" + "vrshrn.u16 d19, q3, #4 \n\t" + "vrshrn.u16 d21, q4, #4 \n\t" + "vrshrn.u16 d23, q5, #4 \n\t" + "vrshrn.u16 d18, q0, #6 \n\t" + "vrshrn.u16 d20, q1, #6 \n\t" + "vrshrn.u16 d22, q2, #6 \n\t" + "vzip.8 d18, d19 \n\t" + "vzip.8 d20, d21 \n\t" + "vzip.8 d22, d23 \n\t" + "vst3.8 {d18, d20, d22}, [%[out1]] \n\t" + "vst3.8 {d19, d21, d23}, [%[out2]] \n\t" + : /*no output*/ + : [out1] "r" (dst + 2 * x), + [out2] "r" (dst + 2 * x + 24), + [in0] "r" (lane + x - 3), + [in02] "r" (lane + x + 9), + [in1] "r" (lane + x), + [in12] "r" (lane + x + 12), + [in2] "r" (lane + x + 3), + [in22] "r" (lane + x + 15) + : "d0","d1","d2","d3","d4","d5","d6","d7","d8","d9","d10","d11","d12","d13","d14","d15","d16","d17","d18","d19","d20","d21","d22","d23" + ); +#else + uint16x8_t vc6 = vmovq_n_u16(6); + uint16x8x3_t vLane0 = vld3q_u16(lane + x - 3); + uint16x8x3_t vLane1 = vld3q_u16(lane + x + 0); + uint16x8x3_t vLane2 = vld3q_u16(lane + x + 3); + + uint16x8_t vSum_0_3 = vaddq_u16(vLane0.val[0], vLane2.val[0]); + uint16x8_t vSum_1_4 = vaddq_u16(vLane0.val[1], vLane2.val[1]); + uint16x8_t vSum_2_5 = vaddq_u16(vLane0.val[2], vLane2.val[2]); + uint16x8_t vSum_3_6 = vaddq_u16(vLane2.val[0], vLane1.val[0]); + uint16x8_t vSum_4_7 = vaddq_u16(vLane2.val[1], vLane1.val[1]); + uint16x8_t vSum_5_8 = vaddq_u16(vLane2.val[2], vLane1.val[2]); + + vSum_0_3 = vmlaq_u16(vSum_0_3, vLane1.val[0], vc6); + vSum_1_4 = vmlaq_u16(vSum_1_4, vLane1.val[1], vc6); + vSum_2_5 = vmlaq_u16(vSum_2_5, vLane1.val[2], vc6); + + uint8x8x2_t vSumShr3; + vSumShr3.val[0] = vrshrn_n_u16(vSum_3_6, 4); + vSumShr3.val[1] = vrshrn_n_u16(vSum_0_3, 6);; + uint8x8x2_t vSumShr4; + vSumShr4.val[0] = vrshrn_n_u16(vSum_4_7, 4); + vSumShr4.val[1] = vrshrn_n_u16(vSum_1_4, 6); + uint8x8x2_t vSumShr5; + vSumShr5.val[0] = vrshrn_n_u16(vSum_5_8, 4); + vSumShr5.val[1] = vrshrn_n_u16(vSum_2_5, 6); + + vSumShr3 = vzip_u8(vSumShr3.val[1], vSumShr3.val[0]); + vSumShr4 = vzip_u8(vSumShr4.val[1], vSumShr4.val[0]); + vSumShr5 = vzip_u8(vSumShr5.val[1], vSumShr5.val[0]); + + uint8x8x3_t vRes1; + vRes1.val[0] = vSumShr3.val[0]; + vRes1.val[1] = vSumShr4.val[0]; + vRes1.val[2] = vSumShr5.val[0]; + vst3_u8(dst + 2 * x, vRes1); + + uint8x8x3_t vRes2; + vRes2.val[0] = vSumShr3.val[1]; + vRes2.val[1] = vSumShr4.val[1]; + vRes2.val[2] = vSumShr5.val[1]; + vst3_u8(dst + 2 * x + 24, vRes2); +#endif + } + } + break; + case 4: + lim = dcolshn > 7 ? dcolshn - 7 : 0; + for (; x < lim; x += 8) + { + internal::prefetch(lane + x); +#if defined(__GNUC__) && defined(__arm__) + __asm__ ( + "vld1.16 {d0-d1}, [%[in0]] /*q0 = v0*/ \n\t" + "vld1.16 {d2-d3}, [%[in2]] /*q1 = v2*/ \n\t" + "vld1.16 {d4-d5}, [%[in1],:128] /*q2 = v1*/ \n\t" + "vadd.i16 q0, q1 /*q0 = v0 + v2*/ \n\t" + "vadd.i16 q3, q1, q2 /*q3 = v1 + v2*/ \n\t" + "vmla.i16 q0, q2, %q[c6] /*q0 += v1*6*/ \n\t" + "vrshrn.u16 d9, q3, #4 \n\t" + "vrshrn.u16 d8, q0, #6 \n\t" + "vst2.32 {d8-d9}, [%[out]] \n\t" + : /*no output*/ + : [out] "r" (dst + x*2), + [in0] "r" (lane + x-4), + [in1] "r" (lane + x), + [in2] "r" (lane + x+4), + [c6] "w" (vc6u16) + : "d0","d1","d2","d3","d4","d5","d6","d7","d8","d9" + ); +#else + uint16x8_t vLane0 = vld1q_u16(lane + x-4); + uint16x8_t vLane1 = vld1q_u16(lane + x+0); + uint16x8_t vLane2 = vld1q_u16(lane + x+4); + + vLane0 = vaddq_u16(vLane0, vLane2); + vLane2 = vaddq_u16(vLane2, vLane1); + vLane0 = vmlaq_u16(vLane0, vLane1, vc6u16); + uint32x2x2_t vRes; + vRes.val[1] = vreinterpret_u32_u8(vrshrn_n_u16(vLane2, 4)); + vRes.val[0] = vreinterpret_u32_u8(vrshrn_n_u16(vLane0, 6)); + + vst2_u32((uint32_t*)(dst + x*2), vRes); +#endif + } + break; + }; + + for (u32 h = 0; h < cn; ++h) + { + const u16* ln = lane + h; + u8* dt = dst + h; + size_t k = x; + for (; k < dcolshn; k += cn) + { + dt[2*k+0] = u8((ln[(ptrdiff_t)(k-cn)] + ln[k+cn] + 6u * ln[k] + (1 << 5)) >> 6); + dt[2*k+cn] = u8(((ln[k] + ln[k+cn]) * 4u + (1 << 5)) >> 6); + } + for (; k < dcolshw; k += cn) + dt[2*k] = u8((ln[(ptrdiff_t)(k-cn)] + ln[k+cn] + 6u * ln[k] + (1 << 5)) >> 6); + } + dst = internal::getRowPtr(dstBase, dstStride, 2*i+1); + + //second row + if (lane == lane0 && 2*i+1 < dstSize.height) + { + lane = lane1; + goto pyrUp8uHorizontalConvolution; + } + } +#else + // Remove 'unused parameter' warnings. + (void)srcBase; + (void)srcStride; + (void)dstBase; + (void)dstStride; +#endif +} + +void gaussianPyramidUp(const Size2D &srcSize, + const s16 *srcBase, ptrdiff_t srcStride, + const Size2D &dstSize, + s16 *dstBase, ptrdiff_t dstStride, u8 cn) +{ + internal::assertSupportedConfiguration(isGaussianPyramidUpS16Supported(srcSize, dstSize, cn)); +#ifdef CAROTENE_NEON + size_t dcolshn = (dstSize.width/2) * cn; + size_t dcolshw = ((dstSize.width+1)/2) * cn; + size_t scolsn = srcSize.width*cn; + + size_t idx_l = (borderInterpolate101(-2, 2 * srcSize.width)/2) * cn; + size_t idx_r1 = (borderInterpolate101(2 * srcSize.width + 0, 2 * srcSize.width)/2) * cn; + size_t idx_r2 = (borderInterpolate101(2 * srcSize.width + 2, 2 * srcSize.width + 2)/2) * cn; + + //2-lines buffer + std::vector _buf(2*(cn*(srcSize.width + 3) + 32/sizeof(s32))); + s32* lane0 = internal::alignPtr(&_buf[cn], 32); + s32* lane1 = internal::alignPtr(lane0 + (3 + srcSize.width)*cn, 32); + + int16x4_t vc6s16 = vmov_n_s16(6); + int32x4_t vc6s32 = vmovq_n_s32(6); + + for (size_t i = 0; i < (dstSize.height + 1)/2; ++i) + { + s16* dst = internal::getRowPtr(dstBase, dstStride, 2*i); + //vertical convolution + const s16* ln0 = internal::getRowPtr(srcBase, srcStride, borderInterpolate101(i * 2 - 2, srcSize.height * 2)/2); + const s16* ln1 = internal::getRowPtr(srcBase, srcStride, borderInterpolate101(i * 2 + 0, srcSize.height * 2)/2); + const s16* ln2 = internal::getRowPtr(srcBase, srcStride, borderInterpolate101(i * 2 + 2, srcSize.height * 2)/2); + + size_t x = 0; + for (; x <= scolsn - 4; x += 4) + { + internal::prefetch(internal::getRowPtr(ln1 + x, srcStride, (ptrdiff_t)x % 3 - 1)); + int16x4_t v0 = vld1_s16(ln0 + x); + int16x4_t v2 = vld1_s16(ln2 + x); + int16x4_t v1 = vld1_s16(ln1 + x); + + int32x4_t vl0 = vaddl_s16(v0, v2); + int32x4_t vl1 = vaddl_s16(v1, v2); + + vl0 = vmlal_s16(vl0, v1, vc6s16); + vl1 = vshlq_n_s32(vl1, 2); + + vst1q_s32(lane0 + x, vl0); + vst1q_s32(lane1 + x, vl1); + } + for (; x < scolsn; ++x) + { + lane0[x] = ln0[x] + ln2[x] + 6 * ln1[x]; + lane1[x] = 4 * (ln1[x] + ln2[x]); + } + + //left&right borders + for (u32 k = 0; k < cn; ++k) + { + lane0[(s32)(-cn+k)] = lane0[idx_l + k]; + lane1[(s32)(-cn+k)] = lane1[idx_l + k]; + + lane0[scolsn+k] = lane0[idx_r1 + k]; + lane0[scolsn+cn+k] = lane0[idx_r2 + k]; + lane1[scolsn+k] = lane1[idx_r1 + k]; + lane1[scolsn+cn+k] = lane1[idx_r2 + k]; + } + + //horizontal convolution + const s32* lane = lane0; +pyrUp16sHorizontalConvolution: + x = 0; + size_t lim; + switch(cn) + { + case 1: + lim = dcolshn > 3 ? dcolshn - 3 : 0; + for (; x < lim; x += 4) + { + internal::prefetch(lane + x); +#if defined(__GNUC__) && defined(__arm__) + __asm__ ( + "vld1.32 {d0-d1}, [%[in0]] /*q0 = v0*/ \n\t" + "vld1.32 {d2-d3}, [%[in2]] /*q1 = v2*/ \n\t" + "vld1.32 {d4-d5}, [%[in1],:128] /*q2 = v1*/ \n\t" + "vadd.i32 q0, q0, q1 /*q0 = v0 + v2*/ \n\t" + "vadd.i32 q3, q1, q2 /*q3 = v1 + v2*/ \n\t" + "vmla.i32 q0, q2, %q[c6] /*q0 += v1*6*/ \n\t" + "vrshrn.s32 d9, q3, #4 \n\t" + "vrshrn.s32 d8, q0, #6 \n\t" + "vst2.16 {d8-d9}, [%[out]] \n\t" + : /*no output*/ + : [out] "r" (dst + x * 2), + [in0] "r" (lane + x - 1), + [in1] "r" (lane + x), + [in2] "r" (lane + x + 1), + [c6] "w" (vc6s32) + : "d0","d1","d2","d3","d4","d5","d6","d7","d8","d9" + ); +#else + int32x4_t vLane0 = vld1q_s32(lane + x - 1); + int32x4_t vLane1 = vld1q_s32(lane + x); + int32x4_t vLane2 = vld1q_s32(lane + x + 1); + + vLane0 = vaddq_s32(vLane0, vLane2); + vLane2 = vaddq_s32(vLane2, vLane1); + vLane0 = vmlaq_s32(vLane0, vLane1, vc6s32); + int16x4x2_t vRes; + vRes.val[0] = vrshrn_n_s32(vLane0, 6); + vRes.val[1] = vrshrn_n_s32(vLane2, 4); + + vst2_s16(dst + x * 2, vRes); +#endif + } + break; + case 3: + { + lim = dcolshn > 11 ? dcolshn - 11 : 0; + for (; x < lim; x += 12) + { + internal::prefetch(lane + x + 3); +#if defined(__GNUC__) && defined(__arm__) + __asm__ ( + "vmov.s32 q9, #6 \n\t" + "vld3.32 {d0, d2, d4}, [%[in0]] /*v0*/ \n\t" + "vld3.32 {d1, d3, d5}, [%[in2]] \n\t" + "vld3.32 {d6, d8, d10}, [%[in2]] /*v2*/ \n\t" + "vld3.32 {d7, d9, d11}, [%[in22]] \n\t" + "vld3.32 {d12, d14, d16}, [%[in1]] /*v1*/ \n\t" + "vld3.32 {d13, d15, d17}, [%[in12]] \n\t" + "vadd.i32 q0, q3 /*v0 + v2*/ \n\t" + "vadd.i32 q1, q4 /*v0 + v2*/ \n\t" + "vadd.i32 q2, q5 /*v0 + v2*/ \n\t" + "vadd.i32 q3, q6 /*v1 + v2*/ \n\t" + "vadd.i32 q4, q7 /*v1 + v2*/ \n\t" + "vadd.i32 q5, q8 /*v1 + v2*/ \n\t" + "vmla.i32 q0, q6, q9 /*v0 + v2 + v1*6 */ \n\t" + "vmla.i32 q1, q7, q9 /*v0 + v2 + v1*6 */ \n\t" + "vmla.i32 q2, q8, q9 /*v0 + v2 + v1*6 */ \n\t" + "vrshrn.s32 d19, q3, #4 \n\t" + "vrshrn.s32 d21, q4, #4 \n\t" + "vrshrn.s32 d23, q5, #4 \n\t" + "vrshrn.s32 d18, q0, #6 \n\t" + "vrshrn.s32 d20, q1, #6 \n\t" + "vrshrn.s32 d22, q2, #6 \n\t" + "vzip.16 d18, d19 \n\t" + "vzip.16 d20, d21 \n\t" + "vzip.16 d22, d23 \n\t" + "vst3.16 {d18, d20, d22}, [%[out1]] \n\t" + "vst3.16 {d19, d21, d23}, [%[out2]] \n\t" + : /*no output*/ + : [out1] "r" (dst + 2*x), + [out2] "r" (dst + 2*x + 12), + [in0] "r" (lane + x - 3), + [in1] "r" (lane + x), + [in12] "r" (lane + x + 6), + [in2] "r" (lane + x + 3), + [in22] "r" (lane + x + 9) + : "d0","d1","d2","d3","d4","d5","d6","d7","d8","d9","d10","d11","d12","d13","d14","d15","d16","d17","d18","d19","d20","d21","d22","d23" + ); +#else + int32x4_t vc6 = vmovq_n_s32(6); + int32x4x3_t vLane0 = vld3q_s32(lane + x - 3); + int32x4x3_t vLane1 = vld3q_s32(lane + x); + int32x4x3_t vLane2 = vld3q_s32(lane + x + 3); + + int32x4_t vSum_0_3 = vaddq_s32(vLane0.val[0], vLane2.val[0]); + int32x4_t vSum_1_4 = vaddq_s32(vLane0.val[1], vLane2.val[1]); + int32x4_t vSum_2_5 = vaddq_s32(vLane0.val[2], vLane2.val[2]); + int32x4_t vSum_3_6 = vaddq_s32(vLane2.val[0], vLane1.val[0]); + int32x4_t vSum_4_7 = vaddq_s32(vLane2.val[1], vLane1.val[1]); + int32x4_t vSum_5_8 = vaddq_s32(vLane2.val[2], vLane1.val[2]); + + vSum_0_3 = vmlaq_s32(vSum_0_3, vLane1.val[0], vc6); + vSum_1_4 = vmlaq_s32(vSum_1_4, vLane1.val[1], vc6); + vSum_2_5 = vmlaq_s32(vSum_2_5, vLane1.val[2], vc6); + + int16x4x2_t vSumShr1; + vSumShr1.val[1] = vrshrn_n_s32(vSum_3_6, 4); + vSumShr1.val[0] = vrshrn_n_s32(vSum_0_3, 6); + + int16x4x2_t vSumShr2; + vSumShr2.val[1] = vrshrn_n_s32(vSum_4_7, 4); + vSumShr2.val[0] = vrshrn_n_s32(vSum_1_4, 6); + + int16x4x2_t vSumShr3; + vSumShr3.val[1] = vrshrn_n_s32(vSum_5_8, 4); + vSumShr3.val[0] = vrshrn_n_s32(vSum_2_5, 6); + + vSumShr1 = vzip_s16(vSumShr1.val[0], vSumShr1.val[1]); + vSumShr2 = vzip_s16(vSumShr2.val[0], vSumShr2.val[1]); + vSumShr3 = vzip_s16(vSumShr3.val[0], vSumShr3.val[1]); + + int16x4x3_t vRes1; + vRes1.val[0] = vSumShr1.val[0]; + vRes1.val[1] = vSumShr2.val[0]; + vRes1.val[2] = vSumShr3.val[0]; + vst3_s16((int16_t*)(dst + 2 * x), vRes1); + + int16x4x3_t vRes2; + vRes2.val[0] = vSumShr1.val[1]; + vRes2.val[1] = vSumShr2.val[1]; + vRes2.val[2] = vSumShr3.val[1]; + vst3_s16(dst + 2 * x + 12, vRes2); +#endif + } + } + break; + case 4: + lim = dcolshn > 3 ? dcolshn - 3 : 0; + for (; x < lim; x += 4) + { + internal::prefetch(lane + x); +#if defined(__GNUC__) && defined(__arm__) + __asm__ ( + "vld1.32 {d0-d1}, [%[in0]] /*q0 = v0*/ \n\t" + "vld1.32 {d2-d3}, [%[in2]] /*q1 = v2*/ \n\t" + "vld1.32 {d4-d5}, [%[in1],:128] /*q2 = v1*/ \n\t" + "vadd.i32 q0, q1 /*q0 = v0 + v2*/ \n\t" + "vadd.i32 q3, q1, q2 /*q3 = v1 + v2*/ \n\t" + "vmla.i32 q0, q2, %q[c6] /*q0 += v1*6*/ \n\t" + "vrshrn.s32 d9, q3, #4 \n\t" + "vrshrn.s32 d8, q0, #6 \n\t" + "vst1.16 {d8-d9}, [%[out]] \n\t" + : /*no output*/ + : [out] "r" (dst + x * 2), + [in0] "r" (lane + x - 4), + [in1] "r" (lane + x), + [in2] "r" (lane + x + 4), + [c6] "w" (vc6s32) + : "d0","d1","d2","d3","d4","d5","d6","d7","d8","d9" + ); +#else + int32x4_t vLane0 = vld1q_s32(lane + x - 4); + int32x4_t vLane1 = vld1q_s32(lane + x); + int32x4_t vLane2 = vld1q_s32(lane + x + 4); + + vLane0 = vaddq_s32(vLane0, vLane2); + vLane2 = vaddq_s32(vLane2, vLane1); + vLane0 = vmlaq_s32(vLane0, vLane1, vc6s32); + int16x4x2_t vRes; + vRes.val[0] = vrshrn_n_s32(vLane0, 6); + vRes.val[1] = vrshrn_n_s32(vLane2, 4); + + vst1q_s16(dst + x * 2, vcombine_s16(vRes.val[0], vRes.val[1])); +#endif + } + break; + }; + + for (u32 h = 0; h < cn; ++h) + { + const s32* ln = lane + h; + s16* dt = dst + h; + size_t k = x; + for (; k < dcolshn; k += cn) + { + dt[2*k+0] = s16((ln[(ptrdiff_t)(k-cn)] + ln[k+cn] + 6 * ln[k] + (1 << 5)) >> 6); + dt[2*k+cn] = s16(((ln[k] + ln[k+cn]) * 4 + (1 << 5)) >> 6); + } + for (; k < dcolshw; k += cn) + dt[2*k] = s16((ln[(ptrdiff_t)(k-cn)] + ln[k+cn] + 6 * ln[k] + (1 << 5)) >> 6); + } + dst = internal::getRowPtr(dstBase, dstStride, 2*i+1); + + //second row + if (lane == lane0 && 2*i+1 < dstSize.height) + { + lane = lane1; + goto pyrUp16sHorizontalConvolution; + } + } +#else + // Remove 'unused parameter' warnings. + (void)srcBase; + (void)srcStride; + (void)dstBase; + (void)dstStride; +#endif +} + +} // namespace CAROTENE_NS diff --git a/3rdparty/carotene/src/reduce.cpp b/3rdparty/carotene/src/reduce.cpp new file mode 100644 index 0000000000..8c11c39e80 --- /dev/null +++ b/3rdparty/carotene/src/reduce.cpp @@ -0,0 +1,460 @@ +/* + * By downloading, copying, installing or using the software you agree to this license. + * If you do not agree to this license, do not download, install, + * copy or use the software. + * + * + * License Agreement + * For Open Source Computer Vision Library + * (3-clause BSD License) + * + * Copyright (C) 2012-2015, NVIDIA Corporation, all rights reserved. + * Third party copyrights are property of their respective owners. + * + * Redistribution and use in source and binary forms, with or without modification, + * are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * * Neither the names of the copyright holders nor the names of the contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * This software is provided by the copyright holders and contributors "as is" and + * any express or implied warranties, including, but not limited to, the implied + * warranties of merchantability and fitness for a particular purpose are disclaimed. + * In no event shall copyright holders or contributors be liable for any direct, + * indirect, incidental, special, exemplary, or consequential damages + * (including, but not limited to, procurement of substitute goods or services; + * loss of use, data, or profits; or business interruption) however caused + * and on any theory of liability, whether in contract, strict liability, + * or tort (including negligence or otherwise) arising in any way out of + * the use of this software, even if advised of the possibility of such damage. + */ + +#include "common.hpp" + +#include + +namespace CAROTENE_NS { + +void reduceColSum(const Size2D &size, + const u8 * srcBase, ptrdiff_t srcStride, + s32 * dstBase) +{ + internal::assertSupportedConfiguration(); +#ifdef CAROTENE_NEON + memset(dstBase, 0, size.width*sizeof(s32)); + size_t i = 0; + for (; i + 16 <= size.width; i += 16) + { + const u8* src_address = srcBase + i; + + int32x4_t sll = vmovq_n_s32(0); + int32x4_t slh = vmovq_n_s32(0); + int32x4_t shl = vmovq_n_s32(0); + int32x4_t shh = vmovq_n_s32(0); + + for (size_t h = 0; h < size.height; h += 256) + { + size_t lim = std::min(h + 256, size.height); + + uint16x8_t sl = vmovq_n_u16(0); + uint16x8_t sh = vmovq_n_u16(0); + + for (size_t k = h; k < lim; ++k, src_address += srcStride) + { + internal::prefetch(src_address + srcStride, 0); + + uint8x16_t v = vld1q_u8(src_address); + + sl = vaddw_u8(sl, vget_low_u8(v)); + sh = vaddw_u8(sh, vget_high_u8(v)); + } + + int32x4_t vsll = vreinterpretq_s32_u32(vmovl_u16(vget_low_u16(sl))); + int32x4_t vslh = vreinterpretq_s32_u32(vmovl_u16(vget_high_u16(sl))); + int32x4_t vshl = vreinterpretq_s32_u32(vmovl_u16(vget_low_u16(sh))); + int32x4_t vshh = vreinterpretq_s32_u32(vmovl_u16(vget_high_u16(sh))); + + sll = vqaddq_s32(sll, vsll); + slh = vqaddq_s32(slh, vslh); + shl = vqaddq_s32(shl, vshl); + shh = vqaddq_s32(shh, vshh); + } + + vst1q_s32(dstBase + i + 0, sll); + vst1q_s32(dstBase + i + 4, slh); + vst1q_s32(dstBase + i + 8, shl); + vst1q_s32(dstBase + i + 12, shh); + } + + for(size_t h = 0; h < size.height; ++h) + { + for(size_t j = i ; j < size.width; j++ ) + { + if (((u32)(dstBase[j] += srcBase[j + srcStride * h])) > 0x7fFFffFFu) + dstBase[j] = 0x7fFFffFF; + } + } +#else + (void)size; + (void)srcBase; + (void)srcStride; + (void)dstBase; +#endif +} + +void reduceColMax(const Size2D &size, + const u8 * srcBase, ptrdiff_t srcStride, + u8 * dstBase) +{ + internal::assertSupportedConfiguration(); +#ifdef CAROTENE_NEON + memcpy(dstBase, srcBase, size.width); + size_t i = 0; + for (; i + 16*4 <= size.width; i += 16*4) + { + const u8* src_address = srcBase + i; + + uint8x16_t s1 = vld1q_u8(src_address + 0); + uint8x16_t s2 = vld1q_u8(src_address + 16); + uint8x16_t s3 = vld1q_u8(src_address + 32); + uint8x16_t s4 = vld1q_u8(src_address + 48); + + src_address += srcStride; + + for(size_t h = 1; h < size.height; ++h, src_address += srcStride) + { + internal::prefetch(src_address + srcStride, 0); + internal::prefetch(src_address + srcStride, 32); + + uint8x16_t v1 = vld1q_u8(src_address + 0); + uint8x16_t v2 = vld1q_u8(src_address + 16); + uint8x16_t v3 = vld1q_u8(src_address + 32); + uint8x16_t v4 = vld1q_u8(src_address + 48); + + s1 = vmaxq_u8(s1, v1); + s2 = vmaxq_u8(s2, v2); + s3 = vmaxq_u8(s3, v3); + s4 = vmaxq_u8(s4, v4); + } + + vst1q_u8(dstBase + i + 0, s1); + vst1q_u8(dstBase + i + 16, s2); + vst1q_u8(dstBase + i + 32, s3); + vst1q_u8(dstBase + i + 48, s4); + } + + for (; i + 16 <= size.width; i += 16) + { + const u8* src_address = srcBase + i; + uint8x16_t s1 = vld1q_u8(src_address); + src_address += srcStride; + for(size_t h = 1; h < size.height; ++h, src_address += srcStride) + { + internal::prefetch(src_address + srcStride, 0); + + uint8x16_t v1 = vld1q_u8(src_address); + s1 = vmaxq_u8(s1, v1); + } + vst1q_u8(dstBase + i, s1); + } + + if (i < size.width) + for(size_t h = 1; h < size.height; ++h) + for(size_t j = i ; j < size.width; j++ ) + dstBase[j] = std::max(dstBase[j], srcBase[j + srcStride * h]); +#else + (void)size; + (void)srcBase; + (void)srcStride; + (void)dstBase; +#endif +} + +void reduceColMin(const Size2D &size, + const u8 * srcBase, ptrdiff_t srcStride, + u8 * dstBase) +{ + internal::assertSupportedConfiguration(); +#ifdef CAROTENE_NEON + memcpy(dstBase, srcBase, size.width); + size_t i = 0; + for (; i + 16*4 <= size.width; i += 16*4) + { + const u8* src_address = srcBase + i; + + uint8x16_t s1 = vld1q_u8(src_address + 0); + uint8x16_t s2 = vld1q_u8(src_address + 16); + uint8x16_t s3 = vld1q_u8(src_address + 32); + uint8x16_t s4 = vld1q_u8(src_address + 48); + + src_address += srcStride; + + for(size_t h = 1; h < size.height; ++h, src_address += srcStride) + { + internal::prefetch(src_address + srcStride, 0); + internal::prefetch(src_address + srcStride, 32); + + uint8x16_t v1 = vld1q_u8(src_address + 0); + uint8x16_t v2 = vld1q_u8(src_address + 16); + uint8x16_t v3 = vld1q_u8(src_address + 32); + uint8x16_t v4 = vld1q_u8(src_address + 48); + + s1 = vminq_u8(s1, v1); + s2 = vminq_u8(s2, v2); + s3 = vminq_u8(s3, v3); + s4 = vminq_u8(s4, v4); + } + + vst1q_u8(dstBase + i + 0, s1); + vst1q_u8(dstBase + i + 16, s2); + vst1q_u8(dstBase + i + 32, s3); + vst1q_u8(dstBase + i + 48, s4); + } + + for (; i + 16 <= size.width; i += 16) + { + const u8* src_address = srcBase + i; + uint8x16_t s1 = vld1q_u8(src_address); + src_address += srcStride; + for(size_t h = 1; h < size.height; ++h, src_address += srcStride) + { + internal::prefetch(src_address + srcStride, 0); + + uint8x16_t v1 = vld1q_u8(src_address); + s1 = vminq_u8(s1, v1); + } + vst1q_u8(dstBase + i, s1); + } + + if (i < size.width) + for(size_t h = 1; h < size.height; ++h) + for(size_t j = i ; j < size.width; j++ ) + dstBase[j] = std::min(dstBase[j], srcBase[j + srcStride * h]); +#else + (void)size; + (void)srcBase; + (void)srcStride; + (void)dstBase; +#endif +} + +void reduceColSum(const Size2D &size, + const f32 * srcBase, ptrdiff_t srcStride, + f32 * dstBase) +{ + internal::assertSupportedConfiguration(); +#ifdef CAROTENE_NEON + memcpy(dstBase, srcBase, size.width*sizeof(f32)); + size_t srcstep = srcStride/sizeof(f32); + size_t i = 0; + for (; i + 16 <= size.width; i += 16) + { + const f32* src_address = srcBase + i; + + float32x4_t s1 = vld1q_f32(src_address + 0); + float32x4_t s2 = vld1q_f32(src_address + 4); + float32x4_t s3 = vld1q_f32(src_address + 8); + float32x4_t s4 = vld1q_f32(src_address + 12); + + src_address += srcstep; + + for(size_t h = 1; h < size.height; ++h, src_address += srcstep) + { + internal::prefetch(src_address + srcstep, 0); + internal::prefetch(src_address + srcstep, 32); + + float32x4_t v1 = vld1q_f32(src_address + 0); + float32x4_t v2 = vld1q_f32(src_address + 4); + float32x4_t v3 = vld1q_f32(src_address + 8); + float32x4_t v4 = vld1q_f32(src_address + 12); + + s1 = vaddq_f32(s1, v1); + s2 = vaddq_f32(s2, v2); + s3 = vaddq_f32(s3, v3); + s4 = vaddq_f32(s4, v4); + } + + vst1q_f32(dstBase + i + 0, s1); + vst1q_f32(dstBase + i + 4, s2); + vst1q_f32(dstBase + i + 8, s3); + vst1q_f32(dstBase + i + 12, s4); + } + + for (; i + 4 <= size.width; i += 4) + { + const f32* src_address = srcBase + i; + float32x4_t s1 = vld1q_f32(src_address); + src_address += srcstep; + for(size_t h = 1; h < size.height; ++h, src_address += srcstep) + { + internal::prefetch(src_address + srcstep, 0); + + float32x4_t v1 = vld1q_f32(src_address); + s1 = vaddq_f32(s1, v1); + } + vst1q_f32(dstBase + i, s1); + } + + if (i < size.width) + for(size_t h = 1; h < size.height; ++h) + { + for(size_t j = i ; j < size.width; j++ ) + { + dstBase[j] += srcBase[j + srcstep * h]; + } + } +#else + (void)size; + (void)srcBase; + (void)srcStride; + (void)dstBase; +#endif +} + +void reduceColMax(const Size2D &size, + const f32 * srcBase, ptrdiff_t srcStride, + f32 * dstBase) +{ + internal::assertSupportedConfiguration(); +#ifdef CAROTENE_NEON + memcpy(dstBase, srcBase, size.width*sizeof(f32)); + size_t srcstep = srcStride/sizeof(f32); + size_t i = 0; + for (; i + 16 <= size.width; i += 16) + { + const f32* src_address = srcBase + i; + + float32x4_t s1 = vld1q_f32(src_address + 0); + float32x4_t s2 = vld1q_f32(src_address + 4); + float32x4_t s3 = vld1q_f32(src_address + 8); + float32x4_t s4 = vld1q_f32(src_address + 12); + + src_address += srcstep; + + for(size_t h = 1; h < size.height; ++h, src_address += srcstep) + { + internal::prefetch(src_address + srcstep, 0); + internal::prefetch(src_address + srcstep, 32); + + float32x4_t v1 = vld1q_f32(src_address + 0); + float32x4_t v2 = vld1q_f32(src_address + 4); + float32x4_t v3 = vld1q_f32(src_address + 8); + float32x4_t v4 = vld1q_f32(src_address + 12); + + s1 = vmaxq_f32(s1, v1); + s2 = vmaxq_f32(s2, v2); + s3 = vmaxq_f32(s3, v3); + s4 = vmaxq_f32(s4, v4); + } + + vst1q_f32(dstBase + i + 0, s1); + vst1q_f32(dstBase + i + 4, s2); + vst1q_f32(dstBase + i + 8, s3); + vst1q_f32(dstBase + i + 12, s4); + } + + for (; i + 4 <= size.width; i += 4) + { + const f32* src_address = srcBase + i; + float32x4_t s1 = vld1q_f32(src_address); + src_address += srcstep; + for(size_t h = 1; h < size.height; ++h, src_address += srcstep) + { + internal::prefetch(src_address + srcstep, 0); + + float32x4_t v1 = vld1q_f32(src_address); + s1 = vmaxq_f32(s1, v1); + } + vst1q_f32(dstBase + i, s1); + } + + if (i < size.width) + for(size_t h = 1; h < size.height; ++h) + for(size_t j = i ; j < size.width; j++ ) + dstBase[j] = std::max(dstBase[j], srcBase[j + srcstep * h]); +#else + (void)size; + (void)srcBase; + (void)srcStride; + (void)dstBase; +#endif +} + +void reduceColMin(const Size2D &size, + const f32 * srcBase, ptrdiff_t srcStride, + f32 * dstBase) +{ + internal::assertSupportedConfiguration(); +#ifdef CAROTENE_NEON + memcpy(dstBase, srcBase, size.width*sizeof(f32)); + size_t srcstep = srcStride/sizeof(f32); + size_t i = 0; + for (; i + 16 <= size.width; i += 16) + { + const f32* src_address = srcBase + i; + + float32x4_t s1 = vld1q_f32(src_address + 0); + float32x4_t s2 = vld1q_f32(src_address + 4); + float32x4_t s3 = vld1q_f32(src_address + 8); + float32x4_t s4 = vld1q_f32(src_address + 12); + + src_address += srcstep; + + for(size_t h = 1; h < size.height; ++h, src_address += srcstep) + { + internal::prefetch(src_address + srcstep, 0); + internal::prefetch(src_address + srcstep, 32); + + float32x4_t v1 = vld1q_f32(src_address + 0); + float32x4_t v2 = vld1q_f32(src_address + 4); + float32x4_t v3 = vld1q_f32(src_address + 8); + float32x4_t v4 = vld1q_f32(src_address + 12); + + s1 = vminq_f32(s1, v1); + s2 = vminq_f32(s2, v2); + s3 = vminq_f32(s3, v3); + s4 = vminq_f32(s4, v4); + } + + vst1q_f32(dstBase + i + 0, s1); + vst1q_f32(dstBase + i + 4, s2); + vst1q_f32(dstBase + i + 8, s3); + vst1q_f32(dstBase + i + 12, s4); + } + + for (; i + 4 <= size.width; i += 4) + { + const f32* src_address = srcBase + i; + float32x4_t s1 = vld1q_f32(src_address); + src_address += srcstep; + for(size_t h = 1; h < size.height; ++h, src_address += srcstep) + { + internal::prefetch(src_address + srcstep, 0); + + float32x4_t v1 = vld1q_f32(src_address); + s1 = vminq_f32(s1, v1); + } + vst1q_f32(dstBase + i, s1); + } + + if (i < size.width) + for(size_t h = 1; h < size.height; ++h) + for(size_t j = i ; j < size.width; j++ ) + dstBase[j] = std::min(dstBase[j], srcBase[j + srcstep * h]); +#else + (void)size; + (void)srcBase; + (void)srcStride; + (void)dstBase; +#endif +} + +} // namespace CAROTENE_NS diff --git a/3rdparty/carotene/src/remap.cpp b/3rdparty/carotene/src/remap.cpp new file mode 100644 index 0000000000..a4b99c3db0 --- /dev/null +++ b/3rdparty/carotene/src/remap.cpp @@ -0,0 +1,694 @@ +/* + * By downloading, copying, installing or using the software you agree to this license. + * If you do not agree to this license, do not download, install, + * copy or use the software. + * + * + * License Agreement + * For Open Source Computer Vision Library + * (3-clause BSD License) + * + * Copyright (C) 2015, NVIDIA Corporation, all rights reserved. + * Third party copyrights are property of their respective owners. + * + * Redistribution and use in source and binary forms, with or without modification, + * are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * * Neither the names of the copyright holders nor the names of the contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * This software is provided by the copyright holders and contributors "as is" and + * any express or implied warranties, including, but not limited to, the implied + * warranties of merchantability and fitness for a particular purpose are disclaimed. + * In no event shall copyright holders or contributors be liable for any direct, + * indirect, incidental, special, exemplary, or consequential damages + * (including, but not limited to, procurement of substitute goods or services; + * loss of use, data, or profits; or business interruption) however caused + * and on any theory of liability, whether in contract, strict liability, + * or tort (including negligence or otherwise) arising in any way out of + * the use of this software, even if advised of the possibility of such damage. + */ + +#include "remap.hpp" + +namespace CAROTENE_NS { + +#ifdef CAROTENE_NEON + +namespace internal { + +void remapNearestNeighborReplicate(const Size2D size, + const u8 * srcBase, + const s32 * map, + u8 * dstBase, ptrdiff_t dstStride) +{ + for (size_t y = 0; y < size.height; ++y) + { + const s32 * map_row = internal::getRowPtr(map, size.width * sizeof(s32), y); + u8 * dst_row = internal::getRowPtr(dstBase, dstStride, y); + + for (size_t x = 0; x < size.width; ++x) + { + dst_row[x] = srcBase[map_row[x]]; + } + } +} + +void remapNearestNeighborConst(const Size2D size, + const u8 * srcBase, + const s32 * map, + u8 * dstBase, ptrdiff_t dstStride, + u8 borderValue) +{ + for (size_t y = 0; y < size.height; ++y) + { + const s32 * map_row = internal::getRowPtr(map, size.width * sizeof(s32), y); + u8 * dst_row = internal::getRowPtr(dstBase, dstStride, y); + + for (size_t x = 0; x < size.width; ++x) + { + s32 src_idx = map_row[x]; + dst_row[x] = src_idx >= 0 ? srcBase[map_row[x]] : borderValue; + } + } +} + +void remapLinearReplicate(const Size2D size, + const u8 * srcBase, + const s32 * map, + const f32 * coeffs, + u8 * dstBase, ptrdiff_t dstStride) +{ + int16x8_t v_zero16 = vdupq_n_s16(0); + + for (size_t y = 0; y < size.height; ++y) + { + const s32 * map_row = internal::getRowPtr(map, size.width * sizeof(s32) * 4, y); + const f32 * coeff_row = internal::getRowPtr(coeffs, size.width * sizeof(f32) * 2, y); + + u8 * dst_row = internal::getRowPtr(dstBase, dstStride, y); + + size_t x = 0; + for ( ; x + 8 < size.width; x += 8) + { + int16x8_t v_src00 = vsetq_lane_s16(srcBase[map_row[(x << 2)]], v_zero16, 0); + v_src00 = vsetq_lane_s16(srcBase[map_row[(x << 2) + 4]], v_src00, 1); + v_src00 = vsetq_lane_s16(srcBase[map_row[(x << 2) + 8]], v_src00, 2); + v_src00 = vsetq_lane_s16(srcBase[map_row[(x << 2) + 12]], v_src00, 3); + v_src00 = vsetq_lane_s16(srcBase[map_row[(x << 2) + 16]], v_src00, 4); + v_src00 = vsetq_lane_s16(srcBase[map_row[(x << 2) + 20]], v_src00, 5); + v_src00 = vsetq_lane_s16(srcBase[map_row[(x << 2) + 24]], v_src00, 6); + v_src00 = vsetq_lane_s16(srcBase[map_row[(x << 2) + 28]], v_src00, 7); + + int16x8_t v_src01 = vsetq_lane_s16(srcBase[map_row[(x << 2) + 1]], v_zero16, 0); + v_src01 = vsetq_lane_s16(srcBase[map_row[(x << 2) + 5]], v_src01, 1); + v_src01 = vsetq_lane_s16(srcBase[map_row[(x << 2) + 9]], v_src01, 2); + v_src01 = vsetq_lane_s16(srcBase[map_row[(x << 2) + 13]], v_src01, 3); + v_src01 = vsetq_lane_s16(srcBase[map_row[(x << 2) + 17]], v_src01, 4); + v_src01 = vsetq_lane_s16(srcBase[map_row[(x << 2) + 21]], v_src01, 5); + v_src01 = vsetq_lane_s16(srcBase[map_row[(x << 2) + 25]], v_src01, 6); + v_src01 = vsetq_lane_s16(srcBase[map_row[(x << 2) + 29]], v_src01, 7); + + int16x8_t v_src10 = vsetq_lane_s16(srcBase[map_row[(x << 2) + 2]], v_zero16, 0); + v_src10 = vsetq_lane_s16(srcBase[map_row[(x << 2) + 6]], v_src10, 1); + v_src10 = vsetq_lane_s16(srcBase[map_row[(x << 2) + 10]], v_src10, 2); + v_src10 = vsetq_lane_s16(srcBase[map_row[(x << 2) + 14]], v_src10, 3); + v_src10 = vsetq_lane_s16(srcBase[map_row[(x << 2) + 18]], v_src10, 4); + v_src10 = vsetq_lane_s16(srcBase[map_row[(x << 2) + 22]], v_src10, 5); + v_src10 = vsetq_lane_s16(srcBase[map_row[(x << 2) + 26]], v_src10, 6); + v_src10 = vsetq_lane_s16(srcBase[map_row[(x << 2) + 30]], v_src10, 7); + + int16x8_t v_src11 = vsetq_lane_s16(srcBase[map_row[(x << 2) + 3]], v_zero16, 0); + v_src11 = vsetq_lane_s16(srcBase[map_row[(x << 2) + 7]], v_src11, 1); + v_src11 = vsetq_lane_s16(srcBase[map_row[(x << 2) + 11]], v_src11, 2); + v_src11 = vsetq_lane_s16(srcBase[map_row[(x << 2) + 15]], v_src11, 3); + v_src11 = vsetq_lane_s16(srcBase[map_row[(x << 2) + 19]], v_src11, 4); + v_src11 = vsetq_lane_s16(srcBase[map_row[(x << 2) + 23]], v_src11, 5); + v_src11 = vsetq_lane_s16(srcBase[map_row[(x << 2) + 27]], v_src11, 6); + v_src11 = vsetq_lane_s16(srcBase[map_row[(x << 2) + 31]], v_src11, 7); + + // first part + float32x4_t v_src00_f = vcvtq_f32_s32(vmovl_s16(vget_low_s16(v_src00))); + float32x4_t v_src10_f = vcvtq_f32_s32(vmovl_s16(vget_low_s16(v_src10))); + + float32x4x2_t v_coeff = vld2q_f32(coeff_row + (x << 1)); + float32x4_t v_dst_0 = vmlaq_f32(v_src00_f, vcvtq_f32_s32(vsubl_s16(vget_low_s16(v_src01), + vget_low_s16(v_src00))), v_coeff.val[0]); + float32x4_t v_dst_1 = vmlaq_f32(v_src10_f, vcvtq_f32_s32(vsubl_s16(vget_low_s16(v_src11), + vget_low_s16(v_src10))), v_coeff.val[0]); + + float32x4_t v_dst = vmlaq_f32(v_dst_0, vsubq_f32(v_dst_1, v_dst_0), v_coeff.val[1]); + uint16x4_t v_dst0 = vmovn_u32(vcvtq_u32_f32(v_dst)); + + // second part + v_src00_f = vcvtq_f32_s32(vmovl_s16(vget_high_s16(v_src00))); + v_src10_f = vcvtq_f32_s32(vmovl_s16(vget_high_s16(v_src10))); + + v_coeff = vld2q_f32(coeff_row + (x << 1) + 8); + v_dst_0 = vmlaq_f32(v_src00_f, vcvtq_f32_s32(vsubl_s16(vget_high_s16(v_src01), + vget_high_s16(v_src00))), v_coeff.val[0]); + v_dst_1 = vmlaq_f32(v_src10_f, vcvtq_f32_s32(vsubl_s16(vget_high_s16(v_src11), + vget_high_s16(v_src10))), v_coeff.val[0]); + + v_dst = vmlaq_f32(v_dst_0, vsubq_f32(v_dst_1, v_dst_0), v_coeff.val[1]); + uint16x4_t v_dst1 = vmovn_u32(vcvtq_u32_f32(v_dst)); + + // store + vst1_u8(dst_row + x, vmovn_u16(vcombine_u16(v_dst0, v_dst1))); + } + + for ( ; x < size.width; ++x) + { + s32 src00_index = map_row[(x << 2)]; + s32 src10_index = map_row[(x << 2) + 2]; + f32 dst_val_0 = (srcBase[map_row[(x << 2) + 1]] - srcBase[src00_index]) * coeff_row[x << 1] + + srcBase[src00_index]; + f32 dst_val_1 = (srcBase[map_row[(x << 2) + 3]] - srcBase[src10_index]) * coeff_row[x << 1] + + srcBase[src10_index]; + dst_row[x] = floorf((dst_val_1 - dst_val_0) * coeff_row[(x << 1) + 1] + dst_val_0); + } + } +} + +void remapLinearConst(const Size2D size, + const u8 * srcBase, + const s32 * map, + const f32 * coeffs, + u8 * dstBase, ptrdiff_t dstStride, + u8 borderValue) +{ + int16x8_t v_zero16 = vdupq_n_s16(0); + + for (size_t y = 0; y < size.height; ++y) + { + const s32 * map_row = internal::getRowPtr(map, size.width * sizeof(s32) * 4, y); + const f32 * coeff_row = internal::getRowPtr(coeffs, size.width * sizeof(f32) * 2, y); + + u8 * dst_row = internal::getRowPtr(dstBase, dstStride, y); + + size_t x = 0; + for ( ; x + 8 < size.width; x += 8) + { + int16x8_t v_src00 = vsetq_lane_s16(map_row[(x << 2)] >= 0 ? srcBase[map_row[(x << 2)]] : borderValue, v_zero16, 0); + v_src00 = vsetq_lane_s16(map_row[(x << 2) + 4] >= 0 ? srcBase[map_row[(x << 2) + 4]] : borderValue, v_src00, 1); + v_src00 = vsetq_lane_s16(map_row[(x << 2) + 8] >= 0 ? srcBase[map_row[(x << 2) + 8]] : borderValue, v_src00, 2); + v_src00 = vsetq_lane_s16(map_row[(x << 2) + 12] >= 0 ? srcBase[map_row[(x << 2) + 12]] : borderValue, v_src00, 3); + v_src00 = vsetq_lane_s16(map_row[(x << 2) + 16] >= 0 ? srcBase[map_row[(x << 2) + 16]] : borderValue, v_src00, 4); + v_src00 = vsetq_lane_s16(map_row[(x << 2) + 20] >= 0 ? srcBase[map_row[(x << 2) + 20]] : borderValue, v_src00, 5); + v_src00 = vsetq_lane_s16(map_row[(x << 2) + 24] >= 0 ? srcBase[map_row[(x << 2) + 24]] : borderValue, v_src00, 6); + v_src00 = vsetq_lane_s16(map_row[(x << 2) + 28] >= 0 ? srcBase[map_row[(x << 2) + 28]] : borderValue, v_src00, 7); + + int16x8_t v_src01 = vsetq_lane_s16(map_row[(x << 2) + 1] >= 0 ? srcBase[map_row[(x << 2) + 1]] : borderValue, v_zero16, 0); + v_src01 = vsetq_lane_s16(map_row[(x << 2) + 5] >= 0 ? srcBase[map_row[(x << 2) + 5]] : borderValue, v_src01, 1); + v_src01 = vsetq_lane_s16(map_row[(x << 2) + 9] >= 0 ? srcBase[map_row[(x << 2) + 9]] : borderValue, v_src01, 2); + v_src01 = vsetq_lane_s16(map_row[(x << 2) + 13] >= 0 ? srcBase[map_row[(x << 2) + 13]] : borderValue, v_src01, 3); + v_src01 = vsetq_lane_s16(map_row[(x << 2) + 17] >= 0 ? srcBase[map_row[(x << 2) + 17]] : borderValue, v_src01, 4); + v_src01 = vsetq_lane_s16(map_row[(x << 2) + 21] >= 0 ? srcBase[map_row[(x << 2) + 21]] : borderValue, v_src01, 5); + v_src01 = vsetq_lane_s16(map_row[(x << 2) + 25] >= 0 ? srcBase[map_row[(x << 2) + 25]] : borderValue, v_src01, 6); + v_src01 = vsetq_lane_s16(map_row[(x << 2) + 29] >= 0 ? srcBase[map_row[(x << 2) + 29]] : borderValue, v_src01, 7); + + int16x8_t v_src10 = vsetq_lane_s16(map_row[(x << 2) + 2] >= 0 ? srcBase[map_row[(x << 2) + 2]] : borderValue, v_zero16, 0); + v_src10 = vsetq_lane_s16(map_row[(x << 2) + 6] >= 0 ? srcBase[map_row[(x << 2) + 6]] : borderValue, v_src10, 1); + v_src10 = vsetq_lane_s16(map_row[(x << 2) + 10] >= 0 ? srcBase[map_row[(x << 2) + 10]] : borderValue, v_src10, 2); + v_src10 = vsetq_lane_s16(map_row[(x << 2) + 14] >= 0 ? srcBase[map_row[(x << 2) + 14]] : borderValue, v_src10, 3); + v_src10 = vsetq_lane_s16(map_row[(x << 2) + 18] >= 0 ? srcBase[map_row[(x << 2) + 18]] : borderValue, v_src10, 4); + v_src10 = vsetq_lane_s16(map_row[(x << 2) + 22] >= 0 ? srcBase[map_row[(x << 2) + 22]] : borderValue, v_src10, 5); + v_src10 = vsetq_lane_s16(map_row[(x << 2) + 26] >= 0 ? srcBase[map_row[(x << 2) + 26]] : borderValue, v_src10, 6); + v_src10 = vsetq_lane_s16(map_row[(x << 2) + 30] >= 0 ? srcBase[map_row[(x << 2) + 30]] : borderValue, v_src10, 7); + + int16x8_t v_src11 = vsetq_lane_s16(map_row[(x << 2) + 3] >= 0 ? srcBase[map_row[(x << 2) + 3]] : borderValue, v_zero16, 0); + v_src11 = vsetq_lane_s16(map_row[(x << 2) + 7] >= 0 ? srcBase[map_row[(x << 2) + 7]] : borderValue, v_src11, 1); + v_src11 = vsetq_lane_s16(map_row[(x << 2) + 11] >= 0 ? srcBase[map_row[(x << 2) + 11]] : borderValue, v_src11, 2); + v_src11 = vsetq_lane_s16(map_row[(x << 2) + 15] >= 0 ? srcBase[map_row[(x << 2) + 15]] : borderValue, v_src11, 3); + v_src11 = vsetq_lane_s16(map_row[(x << 2) + 19] >= 0 ? srcBase[map_row[(x << 2) + 19]] : borderValue, v_src11, 4); + v_src11 = vsetq_lane_s16(map_row[(x << 2) + 23] >= 0 ? srcBase[map_row[(x << 2) + 23]] : borderValue, v_src11, 5); + v_src11 = vsetq_lane_s16(map_row[(x << 2) + 27] >= 0 ? srcBase[map_row[(x << 2) + 27]] : borderValue, v_src11, 6); + v_src11 = vsetq_lane_s16(map_row[(x << 2) + 31] >= 0 ? srcBase[map_row[(x << 2) + 31]] : borderValue, v_src11, 7); + + // first part + float32x4_t v_src00_f = vcvtq_f32_s32(vmovl_s16(vget_low_s16(v_src00))); + float32x4_t v_src10_f = vcvtq_f32_s32(vmovl_s16(vget_low_s16(v_src10))); + + float32x4x2_t v_coeff = vld2q_f32(coeff_row + (x << 1)); + float32x4_t v_dst_0 = vmlaq_f32(v_src00_f, vcvtq_f32_s32(vsubl_s16(vget_low_s16(v_src01), + vget_low_s16(v_src00))), v_coeff.val[0]); + float32x4_t v_dst_1 = vmlaq_f32(v_src10_f, vcvtq_f32_s32(vsubl_s16(vget_low_s16(v_src11), + vget_low_s16(v_src10))), v_coeff.val[0]); + + float32x4_t v_dst = vmlaq_f32(v_dst_0, vsubq_f32(v_dst_1, v_dst_0), v_coeff.val[1]); + uint16x4_t v_dst0 = vmovn_u32(vcvtq_u32_f32(v_dst)); + + // second part + v_src00_f = vcvtq_f32_s32(vmovl_s16(vget_high_s16(v_src00))); + v_src10_f = vcvtq_f32_s32(vmovl_s16(vget_high_s16(v_src10))); + + v_coeff = vld2q_f32(coeff_row + (x << 1) + 8); + v_dst_0 = vmlaq_f32(v_src00_f, vcvtq_f32_s32(vsubl_s16(vget_high_s16(v_src01), + vget_high_s16(v_src00))), v_coeff.val[0]); + v_dst_1 = vmlaq_f32(v_src10_f, vcvtq_f32_s32(vsubl_s16(vget_high_s16(v_src11), + vget_high_s16(v_src10))), v_coeff.val[0]); + + v_dst = vmlaq_f32(v_dst_0, vsubq_f32(v_dst_1, v_dst_0), v_coeff.val[1]); + uint16x4_t v_dst1 = vmovn_u32(vcvtq_u32_f32(v_dst)); + + // store + vst1_u8(dst_row + x, vmovn_u16(vcombine_u16(v_dst0, v_dst1))); + } + + for ( ; x < size.width; ++x) + { + s16 src00 = map_row[(x << 2) + 0] >= 0 ? srcBase[map_row[(x << 2) + 0]] : borderValue; + s16 src01 = map_row[(x << 2) + 1] >= 0 ? srcBase[map_row[(x << 2) + 1]] : borderValue; + s16 src10 = map_row[(x << 2) + 2] >= 0 ? srcBase[map_row[(x << 2) + 2]] : borderValue; + s16 src11 = map_row[(x << 2) + 3] >= 0 ? srcBase[map_row[(x << 2) + 3]] : borderValue; + + f32 dst_val_0 = (src01 - src00) * coeff_row[(x << 1)] + src00; + f32 dst_val_1 = (src11 - src10) * coeff_row[(x << 1)] + src10; + dst_row[x] = floorf((dst_val_1 - dst_val_0) * coeff_row[(x << 1) + 1] + dst_val_0); + } + } +} + +} // namespace internal + +#endif // CAROTENE_NEON + +bool isRemapNearestNeighborSupported(const Size2D &ssize) +{ +#if SIZE_MAX > UINT32_MAX + return !(ssize.width > 0xffffFFFF || ssize.height > 0xffffFFFF) && // Restrict image size since internal index evaluation + // is performed with u32 + isSupportedConfiguration(); +#else + (void)ssize; + return isSupportedConfiguration(); +#endif +} + +bool isRemapLinearSupported(const Size2D &ssize) +{ +#if SIZE_MAX > UINT32_MAX + return !(ssize.width > 0xffffFFFF || ssize.height > 0xffffFFFF) && // Restrict image size since internal index evaluation + // is performed with u32 + isSupportedConfiguration(); +#else + (void)ssize; + return isSupportedConfiguration(); +#endif +} + +void remapNearestNeighbor(const Size2D &ssize, const Size2D &dsize, + const u8 * srcBase, ptrdiff_t srcStride, + const f32 * tableBase, ptrdiff_t tableStride, + u8 * dstBase, ptrdiff_t dstStride, + BORDER_MODE borderMode, u8 borderValue) +{ + internal::assertSupportedConfiguration(isRemapNearestNeighborSupported(ssize)); +#ifdef CAROTENE_NEON + using namespace internal; + + s32 _map[BLOCK_SIZE * BLOCK_SIZE + 16]; + s32 * map = alignPtr(_map, 16); + + int32x4_t v_width4 = vdupq_n_s32(ssize.width - 1), v_height4 = vdupq_n_s32(ssize.height - 1); + int32x2_t v_width2 = vdup_n_s32(ssize.width - 1), v_height2 = vdup_n_s32(ssize.height - 1); + int32x4_t v_step4 = vdupq_n_s32(srcStride); + int32x2_t v_step2 = vdup_n_s32(srcStride); + + if (borderMode == BORDER_MODE_REPLICATE) + { + int32x4_t v_zero4 = vdupq_n_s32(0); + int32x2_t v_zero2 = vdup_n_s32(0); + + for (size_t i = 0; i < dsize.height; i += BLOCK_SIZE) + { + size_t blockHeight = std::min(BLOCK_SIZE, dsize.height - i); + for (size_t j = 0; j < dsize.width; j += BLOCK_SIZE) + { + size_t blockWidth = std::min(BLOCK_SIZE, dsize.width - j); + + // compute table + for (size_t y = 0; y < blockHeight; ++y) + { + const f32 * table_row = getRowPtr(tableBase, tableStride, i + y) + (j << 1); + s32 * map_row = getRowPtr(&map[0], blockWidth * sizeof(s32), y); + + size_t x = 0; + for ( ; x + 8 <= blockWidth; x += 8) + { + float32x4x2_t v_table0 = vld2q_f32(table_row + (x << 1)), + v_table1 = vld2q_f32(table_row + (x << 1) + 8); + + int32x4_t v_dst_x = vmaxq_s32(v_zero4, vminq_s32(v_width4, vcvtq_s32_f32(v_table0.val[0]))); + int32x4_t v_dst_y = vmaxq_s32(v_zero4, vminq_s32(v_height4, vcvtq_s32_f32(v_table0.val[1]))); + int32x4_t v_dst_index = vmlaq_s32(v_dst_x, v_dst_y, v_step4); + vst1q_s32(map_row + x, v_dst_index); + + v_dst_x = vmaxq_s32(v_zero4, vminq_s32(v_width4, vcvtq_s32_f32(v_table1.val[0]))); + v_dst_y = vmaxq_s32(v_zero4, vminq_s32(v_height4, vcvtq_s32_f32(v_table1.val[1]))); + v_dst_index = vmlaq_s32(v_dst_x, v_dst_y, v_step4); + vst1q_s32(map_row + x + 4, v_dst_index); + } + + for ( ; x + 4 <= blockWidth; x += 4) + { + float32x4x2_t v_table0 = vld2q_f32(table_row + (x << 1)); + + int32x4_t v_dst_x = vmaxq_s32(v_zero4, vminq_s32(v_width4, vcvtq_s32_f32(v_table0.val[0]))); + int32x4_t v_dst_y = vmaxq_s32(v_zero4, vminq_s32(v_height4, vcvtq_s32_f32(v_table0.val[1]))); + int32x4_t v_dst_index = vmlaq_s32(v_dst_x, v_dst_y, v_step4); + vst1q_s32(map_row + x, v_dst_index); + } + + for ( ; x + 2 <= blockWidth; x += 2) + { + float32x2x2_t v_table0 = vld2_f32(table_row + (x << 1)); + + int32x2_t v_dst_x = vmax_s32(v_zero2, vmin_s32(v_width2, vcvt_s32_f32(v_table0.val[0]))); + int32x2_t v_dst_y = vmax_s32(v_zero2, vmin_s32(v_height2, vcvt_s32_f32(v_table0.val[1]))); + int32x2_t v_dst_index = vmla_s32(v_dst_x, v_dst_y, v_step2); + vst1_s32(map_row + x, v_dst_index); + } + + for ( ; x < blockWidth; ++x) + { + s32 src_x = std::max(0, std::min(ssize.width - 1, (s32)floorf(table_row[(x << 1) + 0]))); + s32 src_y = std::max(0, std::min(ssize.height - 1, (s32)floorf(table_row[(x << 1) + 1]))); + map_row[x] = src_y * srcStride + src_x; + } + } + + // make remap + remapNearestNeighborReplicate(Size2D(blockWidth, blockHeight), srcBase, &map[0], + getRowPtr(dstBase, dstStride, i) + j, dstStride); + } + } + } + else if (borderMode == BORDER_MODE_CONSTANT) + { + int32x4_t v_m1_4 = vdupq_n_s32(-1); + int32x2_t v_m1_2 = vdup_n_s32(-1); + float32x4_t v_zero4 = vdupq_n_f32(0.0f); + float32x2_t v_zero2 = vdup_n_f32(0.0f); + + for (size_t i = 0; i < dsize.height; i += BLOCK_SIZE) + { + size_t blockHeight = std::min(BLOCK_SIZE, dsize.height - i); + for (size_t j = 0; j < dsize.width; j += BLOCK_SIZE) + { + size_t blockWidth = std::min(BLOCK_SIZE, dsize.width - j); + + // compute table + for (size_t y = 0; y < blockHeight; ++y) + { + const f32 * table_row = getRowPtr(tableBase, tableStride, i + y) + (j << 1); + s32 * map_row = getRowPtr(&map[0], blockWidth * sizeof(s32), y); + + size_t x = 0; + for ( ; x + 8 <= blockWidth; x += 8) + { + float32x4x2_t v_table0 = vld2q_f32(table_row + (x << 1)), + v_table1 = vld2q_f32(table_row + (x << 1) + 8); + + int32x4_t v_dst_x = vcvtq_s32_f32(v_table0.val[0]); + int32x4_t v_dst_y = vcvtq_s32_f32(v_table0.val[1]); + uint32x4_t v_mask = vandq_u32(vandq_u32(vcgeq_f32(v_table0.val[0], v_zero4), vcleq_s32(v_dst_x, v_width4)), + vandq_u32(vcgeq_f32(v_table0.val[1], v_zero4), vcleq_s32(v_dst_y, v_height4))); + int32x4_t v_dst_index = vbslq_s32(v_mask, vmlaq_s32(v_dst_x, v_dst_y, v_step4), v_m1_4); + vst1q_s32(map_row + x, v_dst_index); + + v_dst_x = vcvtq_s32_f32(v_table1.val[0]); + v_dst_y = vcvtq_s32_f32(v_table1.val[1]); + v_mask = vandq_u32(vandq_u32(vcgeq_f32(v_table1.val[0], v_zero4), vcleq_s32(v_dst_x, v_width4)), + vandq_u32(vcgeq_f32(v_table1.val[1], v_zero4), vcleq_s32(v_dst_y, v_height4))); + v_dst_index = vbslq_s32(v_mask, vmlaq_s32(v_dst_x, v_dst_y, v_step4), v_m1_4); + vst1q_s32(map_row + x + 4, v_dst_index); + } + + for ( ; x + 4 <= blockWidth; x += 4) + { + float32x4x2_t v_table0 = vld2q_f32(table_row + (x << 1)); + + int32x4_t v_dst_x = vcvtq_s32_f32(v_table0.val[0]); + int32x4_t v_dst_y = vcvtq_s32_f32(v_table0.val[1]); + uint32x4_t v_mask = vandq_u32(vandq_u32(vcgeq_f32(v_table0.val[0], v_zero4), vcleq_s32(v_dst_x, v_width4)), + vandq_u32(vcgeq_f32(v_table0.val[1], v_zero4), vcleq_s32(v_dst_y, v_height4))); + int32x4_t v_dst_index = vbslq_s32(v_mask, vmlaq_s32(v_dst_x, v_dst_y, v_step4), v_m1_4); + vst1q_s32(map_row + x, v_dst_index); + } + + for ( ; x + 2 <= blockWidth; x += 2) + { + float32x2x2_t v_table0 = vld2_f32(table_row + (x << 1)); + + int32x2_t v_dst_x = vcvt_s32_f32(v_table0.val[0]); + int32x2_t v_dst_y = vcvt_s32_f32(v_table0.val[1]); + uint32x2_t v_mask = vand_u32(vand_u32(vcge_f32(v_table0.val[0], v_zero2), vcle_s32(v_dst_x, v_width2)), + vand_u32(vcge_f32(v_table0.val[1], v_zero2), vcle_s32(v_dst_y, v_height2))); + int32x2_t v_dst_index = vbsl_s32(v_mask, vmla_s32(v_dst_x, v_dst_y, v_step2), v_m1_2); + vst1_s32(map_row + x, v_dst_index); + } + + for ( ; x < blockWidth; ++x) + { + s32 src_x = (s32)floorf(table_row[(x << 1) + 0]); + s32 src_y = (s32)floorf(table_row[(x << 1) + 1]); + map_row[x] = (src_x >= 0) && (src_x < (s32)ssize.width) && + (src_y >= 0) && (src_y < (s32)ssize.height) ? src_y * srcStride + src_x : -1; + } + } + + // make remap + remapNearestNeighborConst(Size2D(blockWidth, blockHeight), srcBase, &map[0], + getRowPtr(dstBase, dstStride, i) + j, dstStride, borderValue); + } + } + } + +#else + (void)ssize; + (void)dsize; + (void)srcBase; + (void)srcStride; + (void)tableBase; + (void)tableStride; + (void)dstBase; + (void)dstStride; + (void)borderMode; + (void)borderValue; +#endif +} + +void remapLinear(const Size2D &ssize, const Size2D &dsize, + const u8 * srcBase, ptrdiff_t srcStride, + const f32 * tableBase, ptrdiff_t tableStride, + u8 * dstBase, ptrdiff_t dstStride, + BORDER_MODE borderMode, u8 borderValue) +{ + internal::assertSupportedConfiguration(isRemapLinearSupported(ssize)); +#ifdef CAROTENE_NEON + using namespace internal; + + s32 _map[((BLOCK_SIZE * BLOCK_SIZE) << 2) + 16]; + f32 _coeffs[((BLOCK_SIZE * BLOCK_SIZE) << 1) + 16]; + + s32 * map = alignPtr(_map, 16); + f32 * coeffs = alignPtr(_coeffs, 16); + + int32x4_t v_width4 = vdupq_n_s32(ssize.width - 1), v_height4 = vdupq_n_s32(ssize.height - 1); + int32x4_t v_step4 = vdupq_n_s32(srcStride), v_1 = vdupq_n_s32(1); + float32x4_t v_zero4f = vdupq_n_f32(0.0f), v_one4f = vdupq_n_f32(1.0f); + + if (borderMode == BORDER_MODE_REPLICATE) + { + int32x4_t v_zero4 = vdupq_n_s32(0); + + for (size_t i = 0; i < dsize.height; i += BLOCK_SIZE) + { + size_t blockHeight = std::min(BLOCK_SIZE, dsize.height - i); + for (size_t j = 0; j < dsize.width; j += BLOCK_SIZE) + { + size_t blockWidth = std::min(BLOCK_SIZE, dsize.width - j); + + // compute table + for (size_t y = 0; y < blockHeight; ++y) + { + const f32 * table_row = getRowPtr(tableBase, tableStride, i + y) + (j << 1); + + s32 * map_row = getRowPtr(map, blockWidth * sizeof(s32) * 4, y); + f32 * coeff_row = getRowPtr(coeffs, blockWidth * sizeof(f32) * 2, y); + + size_t x = 0; + for ( ; x + 4 <= blockWidth; x += 4) + { + float32x4x2_t v_table = vld2q_f32(table_row + (x << 1)); + + int32x4_t v_src_x = vcvtq_s32_f32(v_table.val[0]); + int32x4_t v_src_y = vcvtq_s32_f32(v_table.val[1]); + + float32x4x2_t v_coeff; + v_coeff.val[0] = vsubq_f32(v_table.val[0], vcvtq_f32_s32(v_src_x)); + v_coeff.val[1] = vsubq_f32(v_table.val[1], vcvtq_f32_s32(v_src_y)); + uint32x4_t v_maskx = vcltq_f32(v_coeff.val[0], v_zero4f); + uint32x4_t v_masky = vcltq_f32(v_coeff.val[1], v_zero4f); + v_coeff.val[0] = vbslq_f32(v_maskx, vaddq_f32(v_one4f, v_coeff.val[0]), v_coeff.val[0]); + v_coeff.val[1] = vbslq_f32(v_masky, vaddq_f32(v_one4f, v_coeff.val[1]), v_coeff.val[1]); + v_src_x = vbslq_s32(v_maskx, vsubq_s32(v_src_x, v_1), v_src_x); + v_src_y = vbslq_s32(v_masky, vsubq_s32(v_src_y, v_1), v_src_y); + + int32x4_t v_dst0_x = vmaxq_s32(v_zero4, vminq_s32(v_width4, v_src_x)); + int32x4_t v_dst0_y = vmaxq_s32(v_zero4, vminq_s32(v_height4, v_src_y)); + int32x4_t v_dst1_x = vmaxq_s32(v_zero4, vminq_s32(v_width4, vaddq_s32(v_1, v_src_x))); + int32x4_t v_dst1_y = vmaxq_s32(v_zero4, vminq_s32(v_height4, vaddq_s32(v_1, v_src_y))); + + int32x4x4_t v_dst_index; + v_dst_index.val[0] = vmlaq_s32(v_dst0_x, v_dst0_y, v_step4); + v_dst_index.val[1] = vmlaq_s32(v_dst1_x, v_dst0_y, v_step4); + v_dst_index.val[2] = vmlaq_s32(v_dst0_x, v_dst1_y, v_step4); + v_dst_index.val[3] = vmlaq_s32(v_dst1_x, v_dst1_y, v_step4); + + vst2q_f32(coeff_row + (x << 1), v_coeff); + vst4q_s32(map_row + (x << 2), v_dst_index); + } + + for ( ; x < blockWidth; ++x) + { + f32 src_x_f = table_row[(x << 1) + 0]; + f32 src_y_f = table_row[(x << 1) + 1]; + + s32 src0_x = (s32)floorf(src_x_f); + s32 src0_y = (s32)floorf(src_y_f); + + coeff_row[x << 1] = src_x_f - src0_x; + coeff_row[(x << 1) + 1] = src_y_f - src0_y; + + s32 src1_y = std::max(0, std::min(ssize.height - 1, src0_y + 1)); + src0_y = std::max(0, std::min(ssize.height - 1, src0_y)); + s32 src1_x = std::max(0, std::min(ssize.width - 1, src0_x + 1)); + src0_x = std::max(0, std::min(ssize.width - 1, src0_x)); + + map_row[(x << 2) + 0] = src0_y * srcStride + src0_x; + map_row[(x << 2) + 1] = src0_y * srcStride + src1_x; + map_row[(x << 2) + 2] = src1_y * srcStride + src0_x; + map_row[(x << 2) + 3] = src1_y * srcStride + src1_x; + } + } + + remapLinearReplicate(Size2D(blockWidth, blockHeight), + srcBase, &map[0], &coeffs[0], + getRowPtr(dstBase, dstStride, i) + j, dstStride); + } + } + } + else if (borderMode == BORDER_MODE_CONSTANT) + { + float32x4_t v_zero4 = vdupq_n_f32(0.0f); + int32x4_t v_m1_4 = vdupq_n_s32(-1); + + for (size_t i = 0; i < dsize.height; i += BLOCK_SIZE) + { + size_t blockHeight = std::min(BLOCK_SIZE, dsize.height - i); + for (size_t j = 0; j < dsize.width; j += BLOCK_SIZE) + { + size_t blockWidth = std::min(BLOCK_SIZE, dsize.width - j); + + // compute table + for (size_t y = 0; y < blockHeight; ++y) + { + const f32 * table_row = getRowPtr(tableBase, tableStride, i + y) + (j << 1); + + s32 * map_row = getRowPtr(map, blockWidth * sizeof(s32) * 4, y); + f32 * coeff_row = getRowPtr(coeffs, blockWidth * sizeof(f32) * 2, y); + + size_t x = 0; + for ( ; x + 4 <= blockWidth; x += 4) + { + float32x4x2_t v_table = vld2q_f32(table_row + (x << 1)); + + int32x4_t v_src_x0 = vcvtq_s32_f32(v_table.val[0]); + int32x4_t v_src_y0 = vcvtq_s32_f32(v_table.val[1]); + + float32x4x2_t v_coeff; + v_coeff.val[0] = vsubq_f32(v_table.val[0], vcvtq_f32_s32(v_src_x0)); + v_coeff.val[1] = vsubq_f32(v_table.val[1], vcvtq_f32_s32(v_src_y0)); + uint32x4_t v_maskx = vcltq_f32(v_coeff.val[0], v_zero4f); + uint32x4_t v_masky = vcltq_f32(v_coeff.val[1], v_zero4f); + v_coeff.val[0] = vbslq_f32(v_maskx, vaddq_f32(v_one4f, v_coeff.val[0]), v_coeff.val[0]); + v_coeff.val[1] = vbslq_f32(v_masky, vaddq_f32(v_one4f, v_coeff.val[1]), v_coeff.val[1]); + v_src_x0 = vbslq_s32(v_maskx, vsubq_s32(v_src_x0, v_1), v_src_x0); + v_src_y0 = vbslq_s32(v_masky, vsubq_s32(v_src_y0, v_1), v_src_y0); + + int32x4_t v_src_x1 = vaddq_s32(v_src_x0, v_1); + int32x4_t v_src_y1 = vaddq_s32(v_src_y0, v_1); + + int32x4x4_t v_dst_index; + v_dst_index.val[0] = vmlaq_s32(v_src_x0, v_src_y0, v_step4); + v_dst_index.val[1] = vmlaq_s32(v_src_x1, v_src_y0, v_step4); + v_dst_index.val[2] = vmlaq_s32(v_src_x0, v_src_y1, v_step4); + v_dst_index.val[3] = vmlaq_s32(v_src_x1, v_src_y1, v_step4); + + uint32x4_t v_mask_x0 = vandq_u32(vcgeq_f32(v_table.val[0], v_zero4), vcleq_s32(v_src_x0, v_width4)); + uint32x4_t v_mask_x1 = vandq_u32(vcgeq_f32(vaddq_f32(v_table.val[0], v_one4f), v_zero4), vcleq_s32(v_src_x1, v_width4)); + uint32x4_t v_mask_y0 = vandq_u32(vcgeq_f32(v_table.val[1], v_zero4), vcleq_s32(v_src_y0, v_height4)); + uint32x4_t v_mask_y1 = vandq_u32(vcgeq_f32(vaddq_f32(v_table.val[1], v_one4f), v_zero4), vcleq_s32(v_src_y1, v_height4)); + + v_dst_index.val[0] = vbslq_s32(vandq_u32(v_mask_x0, v_mask_y0), v_dst_index.val[0], v_m1_4); + v_dst_index.val[1] = vbslq_s32(vandq_u32(v_mask_x1, v_mask_y0), v_dst_index.val[1], v_m1_4); + v_dst_index.val[2] = vbslq_s32(vandq_u32(v_mask_x0, v_mask_y1), v_dst_index.val[2], v_m1_4); + v_dst_index.val[3] = vbslq_s32(vandq_u32(v_mask_x1, v_mask_y1), v_dst_index.val[3], v_m1_4); + + vst2q_f32(coeff_row + (x << 1), v_coeff); + vst4q_s32(map_row + (x << 2), v_dst_index); + } + + for ( ; x < blockWidth; ++x) + { + f32 src_x_f = table_row[(x << 1) + 0]; + f32 src_y_f = table_row[(x << 1) + 1]; + + s32 src0_x = (s32)floorf(src_x_f), src1_x = src0_x + 1; + s32 src0_y = (s32)floorf(src_y_f), src1_y = src0_y + 1; + + coeff_row[(x << 1)] = src_x_f - src0_x; + coeff_row[(x << 1) + 1] = src_y_f - src0_y; + + map_row[(x << 2) + 0] = (src0_x >= 0) && (src0_x < (s32)ssize.width) && + (src0_y >= 0) && (src0_y < (s32)ssize.height) ? src0_y * srcStride + src0_x : -1; + map_row[(x << 2) + 1] = (src1_x >= 0) && (src1_x < (s32)ssize.width) && + (src0_y >= 0) && (src0_y < (s32)ssize.height) ? src0_y * srcStride + src1_x : -1; + map_row[(x << 2) + 2] = (src0_x >= 0) && (src0_x < (s32)ssize.width) && + (src1_y >= 0) && (src1_y < (s32)ssize.height) ? src1_y * srcStride + src0_x : -1; + map_row[(x << 2) + 3] = (src1_x >= 0) && (src1_x < (s32)ssize.width) && + (src1_y >= 0) && (src1_y < (s32)ssize.height) ? src1_y * srcStride + src1_x : -1; + } + } + + remapLinearConst(Size2D(blockWidth, blockHeight), + srcBase, &map[0], &coeffs[0], + getRowPtr(dstBase, dstStride, i) + j, dstStride, borderValue); + } + } + } +#else + (void)ssize; + (void)dsize; + (void)srcBase; + (void)srcStride; + (void)tableBase; + (void)tableStride; + (void)dstBase; + (void)dstStride; + (void)borderMode; + (void)borderValue; +#endif +} + +} // namespace CAROTENE_NS diff --git a/3rdparty/carotene/src/remap.hpp b/3rdparty/carotene/src/remap.hpp new file mode 100644 index 0000000000..0f9765965f --- /dev/null +++ b/3rdparty/carotene/src/remap.hpp @@ -0,0 +1,85 @@ +/* + * By downloading, copying, installing or using the software you agree to this license. + * If you do not agree to this license, do not download, install, + * copy or use the software. + * + * + * License Agreement + * For Open Source Computer Vision Library + * (3-clause BSD License) + * + * Copyright (C) 2015, NVIDIA Corporation, all rights reserved. + * Third party copyrights are property of their respective owners. + * + * Redistribution and use in source and binary forms, with or without modification, + * are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * * Neither the names of the copyright holders nor the names of the contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * This software is provided by the copyright holders and contributors "as is" and + * any express or implied warranties, including, but not limited to, the implied + * warranties of merchantability and fitness for a particular purpose are disclaimed. + * In no event shall copyright holders or contributors be liable for any direct, + * indirect, incidental, special, exemplary, or consequential damages + * (including, but not limited to, procurement of substitute goods or services; + * loss of use, data, or profits; or business interruption) however caused + * and on any theory of liability, whether in contract, strict liability, + * or tort (including negligence or otherwise) arising in any way out of + * the use of this software, even if advised of the possibility of such damage. + */ + +#ifndef CAROTENE_SRC_REMAP_HPP +#define CAROTENE_SRC_REMAP_HPP + +#include "common.hpp" + +#include + +#ifdef CAROTENE_NEON + +namespace CAROTENE_NS { namespace internal { + +enum +{ + BLOCK_SIZE = 32 +}; + + +void remapNearestNeighborReplicate(const Size2D size, + const u8 * srcBase, + const s32 * map, + u8 * dstBase, ptrdiff_t dstStride); + +void remapNearestNeighborConst(const Size2D size, + const u8 * srcBase, + const s32 * map, + u8 * dstBase, ptrdiff_t dstStride, + u8 borderValue); + +void remapLinearReplicate(const Size2D size, + const u8 * srcBase, + const s32 * map, + const f32 * coeffs, + u8 * dstBase, ptrdiff_t dstStride); + +void remapLinearConst(const Size2D size, + const u8 * srcBase, + const s32 * map, + const f32 * coeffs, + u8 * dstBase, ptrdiff_t dstStride, + u8 borderValue); + +} } + +#endif // CAROTENE_NEON + +#endif // CAROTENE_SRC_REMAP_HPP diff --git a/3rdparty/carotene/src/resize.cpp b/3rdparty/carotene/src/resize.cpp new file mode 100644 index 0000000000..122a5f2201 --- /dev/null +++ b/3rdparty/carotene/src/resize.cpp @@ -0,0 +1,2191 @@ +/* + * By downloading, copying, installing or using the software you agree to this license. + * If you do not agree to this license, do not download, install, + * copy or use the software. + * + * + * License Agreement + * For Open Source Computer Vision Library + * (3-clause BSD License) + * + * Copyright (C) 2015, NVIDIA Corporation, all rights reserved. + * Third party copyrights are property of their respective owners. + * + * Redistribution and use in source and binary forms, with or without modification, + * are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * * Neither the names of the copyright holders nor the names of the contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * This software is provided by the copyright holders and contributors "as is" and + * any express or implied warranties, including, but not limited to, the implied + * warranties of merchantability and fitness for a particular purpose are disclaimed. + * In no event shall copyright holders or contributors be liable for any direct, + * indirect, incidental, special, exemplary, or consequential damages + * (including, but not limited to, procurement of substitute goods or services; + * loss of use, data, or profits; or business interruption) however caused + * and on any theory of liability, whether in contract, strict liability, + * or tort (including negligence or otherwise) arising in any way out of + * the use of this software, even if advised of the possibility of such damage. + */ + +#include "common.hpp" +#include "vtransform.hpp" + +#include +#include +#include + +namespace CAROTENE_NS { + +bool isResizeNearestNeighborSupported(const Size2D &ssize, u32 elemSize) +{ +#if SIZE_MAX <= UINT32_MAX + (void)ssize; +#endif + bool supportedElemSize = (elemSize == 1) || (elemSize == 3) || (elemSize == 4); + return isSupportedConfiguration() +#if SIZE_MAX > UINT32_MAX + && !(ssize.width > 0xffffFFFF || ssize.height > 0xffffFFFF)// Restrict image size since internally used resizeGeneric performs + // index evaluation with u32 +#endif + && supportedElemSize; +} + +bool isResizeAreaSupported(f32 wr, f32 hr, u32 channels) +{ + bool supportedRatio = false; + + if (channels == 1) + supportedRatio = (hr == wr) && ((wr == 2.0f) || (wr == 4.0f) || (wr == 0.5)); + else if (channels == 3) + supportedRatio = (hr == wr) && ((wr == 2.0f) || (wr == 4.0f) || (wr == 0.5f)); + else if (channels == 4) + supportedRatio = (hr == wr) && ((wr == 2.0f) || (wr == 4.0f) || (wr == 0.5f)); + + return isSupportedConfiguration() && supportedRatio; +} + +bool isResizeLinearSupported(const Size2D &ssize, const Size2D &dsize, + f32 wr, f32 hr, u32 channels) +{ + if ((wr <= 2.0f) && (hr <= 2.0f)) + { + bool channelsSupport = (channels == 1) || (channels == 3) || (channels == 4); + return (ssize.width >= 16) && (dsize.height >= 8) && + (dsize.width >= 8) && channelsSupport; + } + + return false; +} + +bool isResizeLinearOpenCVSupported(const Size2D &ssize, const Size2D &dsize, u32 channels) +{ + switch(channels) + { + case 1: + if (ssize.width >= 8 +#if SIZE_MAX > UINT32_MAX + && !(ssize.width > 0xffffFFFF || ssize.height > 0xffffFFFF)// Restrict image size since internal index evaluation + // is performed with u32 +#endif + && dsize.width >= 8 && dsize.height >= 8) + return isSupportedConfiguration(); + return false; + case 4: + if (ssize.width >= 2 +#if SIZE_MAX > UINT32_MAX + && !(ssize.width > 0xffffFFFF || ssize.height > 0xffffFFFF)// Restrict image size since internal index evaluation + // is performed with u32 +#endif + && dsize.width >= 2 && dsize.height >= 8) + return isSupportedConfiguration(); + default: + return false; + }; +} + +#ifdef CAROTENE_NEON + +namespace { + +u32 * calcLUT(size_t size, f32 ratio, + std::vector & _ofs) +{ + _ofs.resize(size); + u32 * ofs = &_ofs[0]; + + size_t roiw8 = size >= 7 ? size - 7 : 0; + size_t roiw4 = size >= 3 ? size - 3 : 0; + size_t x = 0; + + f32 indeces[4] = { 0, 1, 2, 3 }; + float32x4_t v_index = vld1q_f32(indeces), v_inc = vdupq_n_f32(4); + float32x4_t v_05 = vdupq_n_f32(0.5f), v_ratio = vdupq_n_f32(ratio); + + for ( ; x < roiw8; x += 8) + { + float32x4_t v_dstf = vmulq_f32(vaddq_f32(v_index, v_05), v_ratio); + vst1q_u32(ofs + x, vcvtq_u32_f32(v_dstf)); + v_index = vaddq_f32(v_index, v_inc); + + v_dstf = vmulq_f32(vaddq_f32(v_index, v_05), v_ratio); + vst1q_u32(ofs + x + 4, vcvtq_u32_f32(v_dstf)); + v_index = vaddq_f32(v_index, v_inc); + } + + for ( ; x < roiw4; x += 4) + { + float32x4_t v_dstf = vmulq_f32(vaddq_f32(v_index, v_05), v_ratio); + vst1q_u32(ofs + x, vcvtq_u32_f32(v_dstf)); + v_index = vaddq_f32(v_index, v_inc); + } + + for ( ; x < size; ++x) + { + ofs[x] = static_cast(floorf((x + 0.5f) * ratio)); + } + + return ofs; +} + +template +void resizeGeneric(const Size2D &dsize, + const void * srcBase, ptrdiff_t srcStride, + void * dstBase, ptrdiff_t dstStride, + f32 wr, f32 hr) +{ + std::vector _x_ofs; + u32 * x_ofs = calcLUT(dsize.width, wr, _x_ofs);//32bit LUT is used so we could get issues on src image dimensions greater than (2^32-1) + + for (size_t dst_y = 0; dst_y < dsize.height; ++dst_y) + { + size_t src_y = static_cast(floorf((dst_y + 0.5f) * hr)); + const T * src = internal::getRowPtr(static_cast(srcBase), srcStride, src_y); + T * dst = internal::getRowPtr(static_cast(dstBase), dstStride, dst_y); + + for (size_t dst_x = 0; dst_x < dsize.width; ++dst_x) + { + internal::prefetch(src + dst_x); + dst[dst_x] = src[x_ofs[dst_x]]; + } + } +} + +typedef struct _24bit_ +{ + u8 a[3]; +} _24bit; + +} // namespace + + +#endif + +void resizeNearestNeighbor(const Size2D &ssize, const Size2D &dsize, + const void * srcBase, ptrdiff_t srcStride, + void * dstBase, ptrdiff_t dstStride, + f32 wr, f32 hr, u32 elemSize) +{ + internal::assertSupportedConfiguration(wr > 0 && hr > 0 && + (dsize.width - 0.5) * wr < ssize.width && + (dsize.height - 0.5) * hr < ssize.height && // Ensure we have enough source data + (dsize.width + 0.5) * wr >= ssize.width && + (dsize.height + 0.5) * hr >= ssize.height && // Ensure source isn't too big + isResizeNearestNeighborSupported(ssize, elemSize)); +#ifdef CAROTENE_NEON + + if (elemSize == 1) + { + resizeGeneric(dsize, + srcBase, srcStride, + dstBase, dstStride, + wr, hr); + } + else if (elemSize == 3) + { + resizeGeneric<_24bit>(dsize, + srcBase, srcStride, + dstBase, dstStride, + wr, hr); + } + else if (elemSize == 4) + { + resizeGeneric(dsize, + srcBase, srcStride, + dstBase, dstStride, + wr, hr); + } + +#else + (void)dsize; + (void)srcBase; + (void)srcStride; + (void)dstBase; + (void)dstStride; + (void)wr; + (void)hr; +#endif +} + +#ifdef CAROTENE_NEON +template +inline uint8x8_t areaDownsamplingDivision(uint16x8_t data) +{ + return vshrn_n_u16(data, shiftsize); +} +template <> +inline uint8x8_t areaDownsamplingDivision(uint16x8_t data) +{ + // rounding + return vrshrn_n_u16(data,2); +} +template <> +inline uint8x8_t areaDownsamplingDivision(uint16x8_t data) +{ + // bankers rounding + return vrshrn_n_u16(vqsubq_u16(data, vshrq_n_u16(vbicq_u16(vdupq_n_u16(1<<4), data), 4)),4); +} + +template +inline u8 areaDownsamplingDivision(u16 data) +{ + return data >> shiftsize; +} +template <> +inline u8 areaDownsamplingDivision(u16 data) +{ + // rounding + return (data + 2) >> 2; +} +template <> +inline u8 areaDownsamplingDivision(u16 data) +{ + // bankers rounding + return (data - (((1<<4) & ~data) >> 4) + 8) >> 4; +} +#endif + +template +inline void resizeAreaRounding(const Size2D &ssize, const Size2D &dsize, + const u8 * srcBase, ptrdiff_t srcStride, + u8 * dstBase, ptrdiff_t dstStride, + f32 wr, f32 hr, u32 channels) +{ + internal::assertSupportedConfiguration(isResizeAreaSupported(wr, hr, channels) && + std::abs(dsize.width * wr - ssize.width) < 0.1 && + std::abs(dsize.height * hr - ssize.height) < 0.1); +#ifdef CAROTENE_NEON + if (channels == 1) + { + if ((wr == 2.0f) && (hr == 2.0f)) + { + size_t roiw8 = dsize.width >= 7 ? dsize.width - 7 : 0; + + for (size_t i = 0; i < dsize.height; ++i) + { + const u8 * src0_row = internal::getRowPtr(srcBase, srcStride, i << 1); + const u8 * src1_row = internal::getRowPtr(srcBase, srcStride, (i << 1) + 1); + u8 * dst_row = internal::getRowPtr(dstBase, dstStride, i); + size_t sj = 0, dj = 0; + + for ( ; dj < roiw8; dj += 8, sj += 16) + { + internal::prefetch(src0_row + sj); + internal::prefetch(src1_row + sj); + + uint16x8_t vSum1 = vpaddlq_u8(vld1q_u8(src0_row + sj)); + uint16x8_t vSum2 = vpaddlq_u8(vld1q_u8(src1_row + sj)); + uint8x8_t vRes1 = areaDownsamplingDivision(vaddq_u16(vSum1, vSum2)); + + vst1_u8(dst_row + dj, vRes1); + } + + for ( ; dj < dsize.width; ++dj, sj += 2) + { + dst_row[dj] = areaDownsamplingDivision( + (u16)src0_row[sj] + src0_row[sj + 1] + + src1_row[sj] + src1_row[sj + 1]); + } + } + } + else if ((wr == 0.5f) && (hr == 0.5f)) + { + size_t roiw32 = dsize.width >= 31 ? dsize.width - 31 : 0; + size_t roiw16 = dsize.width >= 15 ? dsize.width - 15 : 0; + + for (size_t i = 0; i < dsize.height; i += 2) + { + const u8 * src_row = internal::getRowPtr(srcBase, srcStride, i >> 1); + u8 * dst0_row = internal::getRowPtr(dstBase, dstStride, i); + u8 * dst1_row = internal::getRowPtr(dstBase, dstStride, std::min(i + 1, dsize.height - 1)); + size_t sj = 0, dj = 0; + + for ( ; dj < roiw32; dj += 32, sj += 16) + { + internal::prefetch(src_row + sj); + + uint8x16x2_t v_dst; + v_dst.val[0] = v_dst.val[1] = vld1q_u8(src_row + sj); + + vst2q_u8(dst0_row + dj, v_dst); + vst2q_u8(dst1_row + dj, v_dst); + } + + for ( ; dj < roiw16; dj += 16, sj += 8) + { + uint8x8x2_t v_dst; + v_dst.val[0] = v_dst.val[1] = vld1_u8(src_row + sj); + + vst2_u8(dst0_row + dj, v_dst); + vst2_u8(dst1_row + dj, v_dst); + } + + for ( ; dj < dsize.width; dj += 2, ++sj) + { + u8 src_val = src_row[sj]; + dst0_row[dj] = dst0_row[dj + 1] = src_val; + dst1_row[dj] = dst1_row[dj + 1] = src_val; + } + } + } + else //if ((wr == 4.0f) && (hr == 4.0f)) //the only scale that lasts after isSupported check + { +#ifndef ANDROID + size_t roiw16 = dsize.width >= 15 ? dsize.width - 15 : 0; +#endif + size_t roiw8 = dsize.width >= 7 ? dsize.width - 7 : 0; + + for (size_t i = 0; i < dsize.height; ++i) + { + const u8 * src0_row = internal::getRowPtr(srcBase, srcStride, i << 2); + const u8 * src1_row = internal::getRowPtr(srcBase, srcStride, (i << 2) + 1); + const u8 * src2_row = internal::getRowPtr(srcBase, srcStride, (i << 2) + 2); + const u8 * src3_row = internal::getRowPtr(srcBase, srcStride, (i << 2) + 3); + u8 * dst_row = internal::getRowPtr(dstBase, dstStride, i); + size_t sj = 0, dj = 0; + +#ifndef ANDROID + for ( ; dj < roiw16; dj += 16, sj += 64) + { + internal::prefetch(src0_row + sj); + internal::prefetch(src1_row + sj); + internal::prefetch(src2_row + sj); + internal::prefetch(src3_row + sj); + + uint8x16x4_t vLane1 = vld4q_u8(src0_row + sj); + uint8x16x4_t vLane2 = vld4q_u8(src1_row + sj); + uint8x16x4_t vLane3 = vld4q_u8(src2_row + sj); + uint8x16x4_t vLane4 = vld4q_u8(src3_row + sj); + + uint16x8_t vSum_0 = vaddl_u8(vget_low_u8(vLane1.val[0]), vget_low_u8(vLane1.val[1])); + vSum_0 = vaddq_u16(vSum_0, vaddl_u8(vget_low_u8(vLane1.val[2]), vget_low_u8(vLane1.val[3]))); + vSum_0 = vaddq_u16(vSum_0, vaddl_u8(vget_low_u8(vLane2.val[0]), vget_low_u8(vLane2.val[1]))); + vSum_0 = vaddq_u16(vSum_0, vaddl_u8(vget_low_u8(vLane2.val[2]), vget_low_u8(vLane2.val[3]))); + vSum_0 = vaddq_u16(vSum_0, vaddl_u8(vget_low_u8(vLane3.val[0]), vget_low_u8(vLane3.val[1]))); + vSum_0 = vaddq_u16(vSum_0, vaddl_u8(vget_low_u8(vLane3.val[2]), vget_low_u8(vLane3.val[3]))); + vSum_0 = vaddq_u16(vSum_0, vaddl_u8(vget_low_u8(vLane4.val[0]), vget_low_u8(vLane4.val[1]))); + vSum_0 = vaddq_u16(vSum_0, vaddl_u8(vget_low_u8(vLane4.val[2]), vget_low_u8(vLane4.val[3]))); + + uint16x8_t vSum_1 = vaddl_u8(vget_high_u8(vLane1.val[0]), vget_high_u8(vLane1.val[1])); + vSum_1 = vaddq_u16(vSum_1, vaddl_u8(vget_high_u8(vLane1.val[2]), vget_high_u8(vLane1.val[3]))); + vSum_1 = vaddq_u16(vSum_1, vaddl_u8(vget_high_u8(vLane2.val[0]), vget_high_u8(vLane2.val[1]))); + vSum_1 = vaddq_u16(vSum_1, vaddl_u8(vget_high_u8(vLane2.val[2]), vget_high_u8(vLane2.val[3]))); + vSum_1 = vaddq_u16(vSum_1, vaddl_u8(vget_high_u8(vLane3.val[0]), vget_high_u8(vLane3.val[1]))); + vSum_1 = vaddq_u16(vSum_1, vaddl_u8(vget_high_u8(vLane3.val[2]), vget_high_u8(vLane3.val[3]))); + vSum_1 = vaddq_u16(vSum_1, vaddl_u8(vget_high_u8(vLane4.val[0]), vget_high_u8(vLane4.val[1]))); + vSum_1 = vaddq_u16(vSum_1, vaddl_u8(vget_high_u8(vLane4.val[2]), vget_high_u8(vLane4.val[3]))); + + uint8x8_t vRes_0 = areaDownsamplingDivision(vSum_0); + uint8x8_t vRes_1 = areaDownsamplingDivision(vSum_1); + + vst1q_u8(dst_row + dj, vcombine_u8(vRes_0, vRes_1)); + } +#endif + + for ( ; dj < roiw8; dj += 8, sj += 32) + { + internal::prefetch(src0_row + sj); + internal::prefetch(src1_row + sj); + internal::prefetch(src2_row + sj); + internal::prefetch(src3_row + sj); + + uint8x8x4_t vLane1 = vld4_u8(src0_row + sj); + uint8x8x4_t vLane2 = vld4_u8(src1_row + sj); + uint8x8x4_t vLane3 = vld4_u8(src2_row + sj); + uint8x8x4_t vLane4 = vld4_u8(src3_row + sj); + + uint16x8_t vSum = vaddl_u8(vLane1.val[0], vLane1.val[1]); + vSum = vaddq_u16(vSum, vaddl_u8(vLane1.val[2], vLane1.val[3])); + vSum = vaddq_u16(vSum, vaddl_u8(vLane2.val[0], vLane2.val[1])); + vSum = vaddq_u16(vSum, vaddl_u8(vLane2.val[2], vLane2.val[3])); + vSum = vaddq_u16(vSum, vaddl_u8(vLane3.val[0], vLane3.val[1])); + vSum = vaddq_u16(vSum, vaddl_u8(vLane3.val[2], vLane3.val[3])); + vSum = vaddq_u16(vSum, vaddl_u8(vLane4.val[0], vLane4.val[1])); + vSum = vaddq_u16(vSum, vaddl_u8(vLane4.val[2], vLane4.val[3])); + + vst1_u8(dst_row + dj, areaDownsamplingDivision(vSum)); + } + + for ( ; dj < dsize.width; ++dj, sj += 4) + { + dst_row[dj] = areaDownsamplingDivision( + (u16)src0_row[sj] + src0_row[sj + 1] + src0_row[sj + 2] + src0_row[sj + 3] + + src1_row[sj] + src1_row[sj + 1] + src1_row[sj + 2] + src1_row[sj + 3] + + src2_row[sj] + src2_row[sj + 1] + src2_row[sj + 2] + src2_row[sj + 3] + + src3_row[sj] + src3_row[sj + 1] + src3_row[sj + 2] + src3_row[sj + 3]); + } + } + } + } + else if (channels == 4) + { + if ((wr == 2.0f) && (hr == 2.0f)) + { +#ifndef ANDROID + size_t roiw4 = dsize.width >= 3 ? (dsize.width - 3) << 2 : 0; +#endif + size_t roiw2 = dsize.width >= 1 ? (dsize.width - 1) << 2 : 0; + + for (size_t i = 0; i < dsize.height; ++i) + { + const u8 * src0_row = internal::getRowPtr(srcBase, srcStride, i << 1); + const u8 * src1_row = internal::getRowPtr(srcBase, srcStride, (i << 1) + 1); + u8 * dst_row = internal::getRowPtr(dstBase, dstStride, i); + size_t sj = 0, dj = 0; + +#ifndef ANDROID + for ( ; dj < roiw4; dj += 16, sj += 32) + { + internal::prefetch(src0_row + sj); + internal::prefetch(src1_row + sj); + + uint8x8_t vRes_0, vRes_1; + + { + uint8x16_t vLane1 = vld1q_u8(src0_row + sj); + uint8x16_t vLane2 = vld1q_u8(src1_row + sj); + + uint16x8_t vLane_l = vaddl_u8(vget_low_u8(vLane1), vget_low_u8(vLane2)); + uint16x8_t vLane_h = vaddl_u8(vget_high_u8(vLane1), vget_high_u8(vLane2)); + + uint16x4_t vSum_l = vadd_u16(vget_low_u16(vLane_l), vget_high_u16(vLane_l)); + uint16x4_t vSum_h = vadd_u16(vget_low_u16(vLane_h), vget_high_u16(vLane_h)); + + vRes_0 = areaDownsamplingDivision(vcombine_u16(vSum_l, vSum_h)); + } + + { + uint8x16_t vLane1 = vld1q_u8(src0_row + sj + 16); + uint8x16_t vLane2 = vld1q_u8(src1_row + sj + 16); + + uint16x8_t vLane_l = vaddl_u8(vget_low_u8(vLane1), vget_low_u8(vLane2)); + uint16x8_t vLane_h = vaddl_u8(vget_high_u8(vLane1), vget_high_u8(vLane2)); + + uint16x4_t vSum_l = vadd_u16(vget_low_u16(vLane_l), vget_high_u16(vLane_l)); + uint16x4_t vSum_h = vadd_u16(vget_low_u16(vLane_h), vget_high_u16(vLane_h)); + + vRes_1 = areaDownsamplingDivision(vcombine_u16(vSum_l, vSum_h)); + } + + vst1q_u8(dst_row + dj, vcombine_u8(vRes_0, vRes_1)); + } +#endif + + for ( ; dj < roiw2; dj += 8, sj += 16) + { + internal::prefetch(src0_row + sj); + internal::prefetch(src1_row + sj); + + uint8x16_t vLane1 = vld1q_u8(src0_row + sj); + uint8x16_t vLane2 = vld1q_u8(src1_row + sj); + + uint16x8_t vLane_l = vaddl_u8(vget_low_u8(vLane1), vget_low_u8(vLane2)); + uint16x8_t vLane_h = vaddl_u8(vget_high_u8(vLane1), vget_high_u8(vLane2)); + + uint16x4_t vSum_l = vadd_u16(vget_low_u16(vLane_l), vget_high_u16(vLane_l)); + uint16x4_t vSum_h = vadd_u16(vget_low_u16(vLane_h), vget_high_u16(vLane_h)); + + uint8x8_t vRes = areaDownsamplingDivision(vcombine_u16(vSum_l, vSum_h)); + vst1_u8(dst_row + dj, vRes); + } + + for (size_t dwidth = dsize.width << 2; dj < dwidth; dj += 4, sj += 8) + { + dst_row[dj ] = areaDownsamplingDivision( + (u16)src0_row[sj ] + src0_row[sj + 4] + + src1_row[sj ] + src1_row[sj + 4]); + dst_row[dj + 1] = areaDownsamplingDivision( + (u16)src0_row[sj + 1] + src0_row[sj + 5] + + src1_row[sj + 1] + src1_row[sj + 5]); + dst_row[dj + 2] = areaDownsamplingDivision( + (u16)src0_row[sj + 2] + src0_row[sj + 6] + + src1_row[sj + 2] + src1_row[sj + 6]); + dst_row[dj + 3] = areaDownsamplingDivision( + (u16)src0_row[sj + 3] + src0_row[sj + 7] + + src1_row[sj + 3] + src1_row[sj + 7]); + } + } + } + else if ((wr == 0.5f) && (hr == 0.5f)) + { +#ifndef ANDROID + size_t roiw32 = dsize.width >= 31 ? (dsize.width - 31) << 2 : 0; +#endif + size_t roiw16 = dsize.width >= 15 ? (dsize.width - 15) << 2 : 0; + + for (size_t i = 0; i < dsize.height; i += 2) + { + const u8 * src_row = internal::getRowPtr(srcBase, srcStride, i >> 1); + u8 * dst0_row = internal::getRowPtr(dstBase, dstStride, i); + u8 * dst1_row = internal::getRowPtr(dstBase, dstStride, std::min(i + 1, dsize.height - 1)); + size_t sj = 0, dj = 0; + +#ifndef ANDROID + for ( ; dj < roiw32; dj += 128, sj += 64) + { + internal::prefetch(src_row + sj); + + uint8x16x4_t v_src = vld4q_u8(src_row + sj); + uint8x16x2_t v_c0 = vzipq_u8(v_src.val[0], v_src.val[0]); + uint8x16x2_t v_c1 = vzipq_u8(v_src.val[1], v_src.val[1]); + uint8x16x2_t v_c2 = vzipq_u8(v_src.val[2], v_src.val[2]); + uint8x16x2_t v_c3 = vzipq_u8(v_src.val[3], v_src.val[3]); + + uint8x16x4_t v_dst; + v_dst.val[0] = v_c0.val[0]; + v_dst.val[1] = v_c1.val[0]; + v_dst.val[2] = v_c2.val[0]; + v_dst.val[3] = v_c3.val[0]; + vst4q_u8(dst0_row + dj, v_dst); + vst4q_u8(dst1_row + dj, v_dst); + + v_dst.val[0] = v_c0.val[1]; + v_dst.val[1] = v_c1.val[1]; + v_dst.val[2] = v_c2.val[1]; + v_dst.val[3] = v_c3.val[1]; + vst4q_u8(dst0_row + dj + 64, v_dst); + vst4q_u8(dst1_row + dj + 64, v_dst); + } +#endif + + for ( ; dj < roiw16; dj += 64, sj += 32) + { + internal::prefetch(src_row + sj); + + uint8x8x4_t v_src = vld4_u8(src_row + sj); + uint8x8x2_t v_c0 = vzip_u8(v_src.val[0], v_src.val[0]); + uint8x8x2_t v_c1 = vzip_u8(v_src.val[1], v_src.val[1]); + uint8x8x2_t v_c2 = vzip_u8(v_src.val[2], v_src.val[2]); + uint8x8x2_t v_c3 = vzip_u8(v_src.val[3], v_src.val[3]); + + uint8x16x4_t v_dst; + v_dst.val[0] = vcombine_u8(v_c0.val[0], v_c0.val[1]); + v_dst.val[1] = vcombine_u8(v_c1.val[0], v_c1.val[1]); + v_dst.val[2] = vcombine_u8(v_c2.val[0], v_c2.val[1]); + v_dst.val[3] = vcombine_u8(v_c3.val[0], v_c3.val[1]); + vst4q_u8(dst0_row + dj, v_dst); + vst4q_u8(dst1_row + dj, v_dst); + } + + for (size_t dwidth = dsize.width << 2; dj < dwidth; dj += 8, sj += 4) + { + u8 src_val = src_row[sj]; + dst0_row[dj] = dst0_row[dj + 4] = src_val; + dst1_row[dj] = dst1_row[dj + 4] = src_val; + + src_val = src_row[sj + 1]; + dst0_row[dj + 1] = dst0_row[dj + 5] = src_val; + dst1_row[dj + 1] = dst1_row[dj + 5] = src_val; + + src_val = src_row[sj + 2]; + dst0_row[dj + 2] = dst0_row[dj + 6] = src_val; + dst1_row[dj + 2] = dst1_row[dj + 6] = src_val; + + src_val = src_row[sj + 3]; + dst0_row[dj + 3] = dst0_row[dj + 7] = src_val; + dst1_row[dj + 3] = dst1_row[dj + 7] = src_val; + } + } + } + else //if ((hr == 4.0f) && (wr == 4.0f)) //the only scale that lasts after isSupported check + { + size_t roiw4 = dsize.width >= 3 ? (dsize.width - 3) << 2 : 0; + size_t roiw2 = dsize.width >= 1 ? (dsize.width - 1) << 2 : 0; + + for (size_t i = 0; i < dsize.height; ++i) + { + const u8 * src0_row = internal::getRowPtr(srcBase, srcStride, i << 2); + const u8 * src1_row = internal::getRowPtr(srcBase, srcStride, (i << 2) + 1); + const u8 * src2_row = internal::getRowPtr(srcBase, srcStride, (i << 2) + 2); + const u8 * src3_row = internal::getRowPtr(srcBase, srcStride, (i << 2) + 3); + u8 * dst_row = internal::getRowPtr(dstBase, dstStride, i); + size_t sj = 0, dj = 0; + + for ( ; dj < roiw4; dj += 16, sj += 64) + { + internal::prefetch(src0_row + sj); + internal::prefetch(src1_row + sj); + internal::prefetch(src2_row + sj); + internal::prefetch(src3_row + sj); + + uint8x16_t vLane10 = vld1q_u8(src0_row + sj), vLane11 = vld1q_u8(src0_row + sj + 16); + uint8x16_t vLane20 = vld1q_u8(src1_row + sj), vLane21 = vld1q_u8(src1_row + sj + 16); + uint8x16_t vLane30 = vld1q_u8(src2_row + sj), vLane31 = vld1q_u8(src2_row + sj + 16); + uint8x16_t vLane40 = vld1q_u8(src3_row + sj), vLane41 = vld1q_u8(src3_row + sj + 16); + + uint16x8_t v_part_0, v_part_1; + { + uint16x8_t v_sum0 = vaddl_u8(vget_low_u8(vLane10), vget_high_u8(vLane10)); + v_sum0 = vaddq_u16(v_sum0, vaddl_u8(vget_low_u8(vLane20), vget_high_u8(vLane20))); + v_sum0 = vaddq_u16(v_sum0, vaddl_u8(vget_low_u8(vLane30), vget_high_u8(vLane30))); + v_sum0 = vaddq_u16(v_sum0, vaddl_u8(vget_low_u8(vLane40), vget_high_u8(vLane40))); + + uint16x8_t v_sum1 = vaddl_u8(vget_low_u8(vLane11), vget_high_u8(vLane11)); + v_sum1 = vaddq_u16(v_sum1, vaddl_u8(vget_low_u8(vLane21), vget_high_u8(vLane21))); + v_sum1 = vaddq_u16(v_sum1, vaddl_u8(vget_low_u8(vLane31), vget_high_u8(vLane31))); + v_sum1 = vaddq_u16(v_sum1, vaddl_u8(vget_low_u8(vLane41), vget_high_u8(vLane41))); + + v_part_0 = vcombine_u16(vadd_u16(vget_low_u16(v_sum0), vget_high_u16(v_sum0)), + vadd_u16(vget_low_u16(v_sum1), vget_high_u16(v_sum1))); + } + + vLane10 = vld1q_u8(src0_row + sj + 32); + vLane11 = vld1q_u8(src0_row + sj + 48); + vLane20 = vld1q_u8(src1_row + sj + 32); + vLane21 = vld1q_u8(src1_row + sj + 48); + vLane30 = vld1q_u8(src2_row + sj + 32); + vLane31 = vld1q_u8(src2_row + sj + 48); + vLane40 = vld1q_u8(src3_row + sj + 32); + vLane41 = vld1q_u8(src3_row + sj + 48); + + { + uint16x8_t v_sum0 = vaddl_u8(vget_low_u8(vLane10), vget_high_u8(vLane10)); + v_sum0 = vaddq_u16(v_sum0, vaddl_u8(vget_low_u8(vLane20), vget_high_u8(vLane20))); + v_sum0 = vaddq_u16(v_sum0, vaddl_u8(vget_low_u8(vLane30), vget_high_u8(vLane30))); + v_sum0 = vaddq_u16(v_sum0, vaddl_u8(vget_low_u8(vLane40), vget_high_u8(vLane40))); + + uint16x8_t v_sum1 = vaddl_u8(vget_low_u8(vLane11), vget_high_u8(vLane11)); + v_sum1 = vaddq_u16(v_sum1, vaddl_u8(vget_low_u8(vLane21), vget_high_u8(vLane21))); + v_sum1 = vaddq_u16(v_sum1, vaddl_u8(vget_low_u8(vLane31), vget_high_u8(vLane31))); + v_sum1 = vaddq_u16(v_sum1, vaddl_u8(vget_low_u8(vLane41), vget_high_u8(vLane41))); + + v_part_1 = vcombine_u16(vadd_u16(vget_low_u16(v_sum0), vget_high_u16(v_sum0)), + vadd_u16(vget_low_u16(v_sum1), vget_high_u16(v_sum1))); + } + + vst1q_u8(dst_row + dj, vcombine_u8(areaDownsamplingDivision(v_part_0), + areaDownsamplingDivision(v_part_1))); + } + + for ( ; dj < roiw2; dj += 8, sj += 32) + { + uint8x16_t vLane10 = vld1q_u8(src0_row + sj), vLane11 = vld1q_u8(src0_row + sj + 16); + uint8x16_t vLane20 = vld1q_u8(src1_row + sj), vLane21 = vld1q_u8(src1_row + sj + 16); + uint8x16_t vLane30 = vld1q_u8(src2_row + sj), vLane31 = vld1q_u8(src2_row + sj + 16); + uint8x16_t vLane40 = vld1q_u8(src3_row + sj), vLane41 = vld1q_u8(src3_row + sj + 16); + + uint16x8_t v_sum0 = vaddl_u8(vget_low_u8(vLane10), vget_high_u8(vLane10)); + v_sum0 = vaddq_u16(v_sum0, vaddl_u8(vget_low_u8(vLane20), vget_high_u8(vLane20))); + v_sum0 = vaddq_u16(v_sum0, vaddl_u8(vget_low_u8(vLane30), vget_high_u8(vLane30))); + v_sum0 = vaddq_u16(v_sum0, vaddl_u8(vget_low_u8(vLane40), vget_high_u8(vLane40))); + + uint16x8_t v_sum1 = vaddl_u8(vget_low_u8(vLane11), vget_high_u8(vLane11)); + v_sum1 = vaddq_u16(v_sum1, vaddl_u8(vget_low_u8(vLane21), vget_high_u8(vLane21))); + v_sum1 = vaddq_u16(v_sum1, vaddl_u8(vget_low_u8(vLane31), vget_high_u8(vLane31))); + v_sum1 = vaddq_u16(v_sum1, vaddl_u8(vget_low_u8(vLane41), vget_high_u8(vLane41))); + + uint16x8_t v_sum = vcombine_u16(vadd_u16(vget_low_u16(v_sum0), vget_high_u16(v_sum0)), + vadd_u16(vget_low_u16(v_sum1), vget_high_u16(v_sum1))); + + vst1_u8(dst_row + dj, areaDownsamplingDivision(v_sum)); + } + + for (size_t dwidth = dsize.width << 2; dj < dwidth; dj += 4, sj += 16) + { + dst_row[dj ] = areaDownsamplingDivision( + (u16)src0_row[sj ] + src0_row[sj + 4] + + src0_row[sj + 8] + src0_row[sj + 12] + + src1_row[sj ] + src1_row[sj + 4] + + src1_row[sj + 8] + src1_row[sj + 12] + + src2_row[sj ] + src2_row[sj + 4] + + src2_row[sj + 8] + src2_row[sj + 12] + + src3_row[sj ] + src3_row[sj + 4] + + src3_row[sj + 8] + src3_row[sj + 12]); + + dst_row[dj + 1] = areaDownsamplingDivision( + (u16)src0_row[sj + 1] + src0_row[sj + 5] + + src0_row[sj + 9] + src0_row[sj + 13] + + src1_row[sj + 1] + src1_row[sj + 5] + + src1_row[sj + 9] + src1_row[sj + 13] + + src2_row[sj + 1] + src2_row[sj + 5] + + src2_row[sj + 9] + src2_row[sj + 13] + + src3_row[sj + 1] + src3_row[sj + 5] + + src3_row[sj + 9] + src3_row[sj + 13]); + + dst_row[dj + 2] = areaDownsamplingDivision( + (u16)src0_row[sj + 2] + src0_row[sj + 6] + + src0_row[sj + 10] + src0_row[sj + 14] + + src1_row[sj + 2] + src1_row[sj + 6] + + src1_row[sj + 10] + src1_row[sj + 14] + + src2_row[sj + 2] + src2_row[sj + 6] + + src2_row[sj + 10] + src2_row[sj + 14] + + src3_row[sj + 2] + src3_row[sj + 6] + + src3_row[sj + 10] + src3_row[sj + 14]); + + dst_row[dj + 3] = areaDownsamplingDivision( + (u16)src0_row[sj + 3] + src0_row[sj + 7] + + src0_row[sj + 11] + src0_row[sj + 15] + + src1_row[sj + 3] + src1_row[sj + 7] + + src1_row[sj + 11] + src1_row[sj + 15] + + src2_row[sj + 3] + src2_row[sj + 7] + + src2_row[sj + 11] + src2_row[sj + 15] + + src3_row[sj + 3] + src3_row[sj + 7] + + src3_row[sj + 11] + src3_row[sj + 15]); + } + } + } + } + else if (channels == 3) + { + if ((wr == 2.0f) && (wr == 2.0f)) + { +#ifndef ANDROID + size_t roiw16 = dsize.width >= 15 ? (dsize.width - 15) * 3 : 0; +#endif + size_t roiw8 = dsize.width >= 7 ? (dsize.width - 7) * 3 : 0; + + for (size_t i = 0; i < dsize.height; ++i) + { + const u8 * src0_row = internal::getRowPtr(srcBase, srcStride, i << 1); + const u8 * src1_row = internal::getRowPtr(srcBase, srcStride, (i << 1) + 1); + u8 * dst_row = internal::getRowPtr(dstBase, dstStride, i); + size_t sj = 0, dj = 0; + +#ifndef ANDROID + for ( ; dj < roiw16; dj += 48, sj += 96) + { + internal::prefetch(src0_row + sj); + internal::prefetch(src1_row + sj); + + uint8x16x3_t vLane1 = vld3q_u8(src0_row + sj); + uint8x16x3_t vLane2 = vld3q_u8(src1_row + sj); + + uint8x8x3_t v_dst0, v_dst1; + { + uint16x8_t v_el0 = vpaddlq_u8(vLane1.val[0]); + uint16x8_t v_el1 = vpaddlq_u8(vLane1.val[1]); + uint16x8_t v_el2 = vpaddlq_u8(vLane1.val[2]); + v_el0 = vpadalq_u8(v_el0, vLane2.val[0]); + v_el1 = vpadalq_u8(v_el1, vLane2.val[1]); + v_el2 = vpadalq_u8(v_el2, vLane2.val[2]); + + v_dst0.val[0] = areaDownsamplingDivision(v_el0); + v_dst0.val[1] = areaDownsamplingDivision(v_el1); + v_dst0.val[2] = areaDownsamplingDivision(v_el2); + } + + vLane1 = vld3q_u8(src0_row + sj + 48); + vLane2 = vld3q_u8(src1_row + sj + 48); + { + uint16x8_t v_el0 = vpaddlq_u8(vLane1.val[0]); + uint16x8_t v_el1 = vpaddlq_u8(vLane1.val[1]); + uint16x8_t v_el2 = vpaddlq_u8(vLane1.val[2]); + v_el0 = vpadalq_u8(v_el0, vLane2.val[0]); + v_el1 = vpadalq_u8(v_el1, vLane2.val[1]); + v_el2 = vpadalq_u8(v_el2, vLane2.val[2]); + + v_dst1.val[0] = areaDownsamplingDivision(v_el0); + v_dst1.val[1] = areaDownsamplingDivision(v_el1); + v_dst1.val[2] = areaDownsamplingDivision(v_el2); + } + + uint8x16x3_t v_dst; + v_dst.val[0] = vcombine_u8(v_dst0.val[0], v_dst1.val[0]); + v_dst.val[1] = vcombine_u8(v_dst0.val[1], v_dst1.val[1]); + v_dst.val[2] = vcombine_u8(v_dst0.val[2], v_dst1.val[2]); + + vst3q_u8(dst_row + dj, v_dst); + } +#endif + + for ( ; dj < roiw8; dj += 24, sj += 48) + { + internal::prefetch(src0_row + sj); + internal::prefetch(src1_row + sj); + + uint8x16x3_t vLane1 = vld3q_u8(src0_row + sj); + uint8x16x3_t vLane2 = vld3q_u8(src1_row + sj); + + uint16x8_t v_el0 = vpaddlq_u8(vLane1.val[0]); + uint16x8_t v_el1 = vpaddlq_u8(vLane1.val[1]); + uint16x8_t v_el2 = vpaddlq_u8(vLane1.val[2]); + v_el0 = vpadalq_u8(v_el0, vLane2.val[0]); + v_el1 = vpadalq_u8(v_el1, vLane2.val[1]); + v_el2 = vpadalq_u8(v_el2, vLane2.val[2]); + + uint8x8x3_t v_dst; + v_dst.val[0] = areaDownsamplingDivision(v_el0); + v_dst.val[1] = areaDownsamplingDivision(v_el1); + v_dst.val[2] = areaDownsamplingDivision(v_el2); + + vst3_u8(dst_row + dj, v_dst); + } + + for (size_t dwidth = dsize.width * 3; dj < dwidth; dj += 3, sj += 6) + { + dst_row[dj ] = areaDownsamplingDivision( + (u16)src0_row[sj ] + src0_row[sj + 3] + + src1_row[sj ] + src1_row[sj + 3]); + dst_row[dj + 1] = areaDownsamplingDivision( + (u16)src0_row[sj + 1] + src0_row[sj + 4] + + src1_row[sj + 1] + src1_row[sj + 4]); + dst_row[dj + 2] = areaDownsamplingDivision( + (u16)src0_row[sj + 2] + src0_row[sj + 5] + + src1_row[sj + 2] + src1_row[sj + 5]); + } + } + } + else if ((wr == 0.5f) && (hr == 0.5f)) + { +#ifndef ANDROID + size_t roiw32 = dsize.width >= 31 ? (dsize.width - 31) * 3 : 0; +#endif + size_t roiw16 = dsize.width >= 15 ? (dsize.width - 15) * 3 : 0; + + for (size_t i = 0; i < dsize.height; i += 2) + { + const u8 * src_row = internal::getRowPtr(srcBase, srcStride, i >> 1); + u8 * dst0_row = internal::getRowPtr(dstBase, dstStride, i); + u8 * dst1_row = internal::getRowPtr(dstBase, dstStride, std::min(i + 1, dsize.height - 1)); + size_t sj = 0, dj = 0; + +#ifndef ANDROID + for ( ; dj < roiw32; dj += 96, sj += 48) + { + internal::prefetch(src_row + sj); + + uint8x16x3_t v_src = vld3q_u8(src_row + sj); + uint8x16x2_t v_c0 = vzipq_u8(v_src.val[0], v_src.val[0]); + uint8x16x2_t v_c1 = vzipq_u8(v_src.val[1], v_src.val[1]); + uint8x16x2_t v_c2 = vzipq_u8(v_src.val[2], v_src.val[2]); + + uint8x16x3_t v_dst; + v_dst.val[0] = v_c0.val[0]; + v_dst.val[1] = v_c1.val[0]; + v_dst.val[2] = v_c2.val[0]; + vst3q_u8(dst0_row + dj, v_dst); + vst3q_u8(dst1_row + dj, v_dst); + + v_dst.val[0] = v_c0.val[1]; + v_dst.val[1] = v_c1.val[1]; + v_dst.val[2] = v_c2.val[1]; + vst3q_u8(dst0_row + dj + 48, v_dst); + vst3q_u8(dst1_row + dj + 48, v_dst); + } +#endif + + for ( ; dj < roiw16; dj += 48, sj += 24) + { + internal::prefetch(src_row + sj); + + uint8x8x3_t v_src = vld3_u8(src_row + sj); + uint8x8x2_t v_c0 = vzip_u8(v_src.val[0], v_src.val[0]); + uint8x8x2_t v_c1 = vzip_u8(v_src.val[1], v_src.val[1]); + uint8x8x2_t v_c2 = vzip_u8(v_src.val[2], v_src.val[2]); + + uint8x16x3_t v_dst; + v_dst.val[0] = vcombine_u8(v_c0.val[0], v_c0.val[1]); + v_dst.val[1] = vcombine_u8(v_c1.val[0], v_c1.val[1]); + v_dst.val[2] = vcombine_u8(v_c2.val[0], v_c2.val[1]); + vst3q_u8(dst0_row + dj, v_dst); + vst3q_u8(dst1_row + dj, v_dst); + } + + for (size_t dwidth = dsize.width * 3; dj < dwidth; dj += 6, sj += 3) + { + u8 src_val = src_row[sj]; + dst0_row[dj] = dst0_row[dj + 3] = src_val; + dst1_row[dj] = dst1_row[dj + 3] = src_val; + + src_val = src_row[sj + 1]; + dst0_row[dj + 1] = dst0_row[dj + 4] = src_val; + dst1_row[dj + 1] = dst1_row[dj + 4] = src_val; + + src_val = src_row[sj + 2]; + dst0_row[dj + 2] = dst0_row[dj + 5] = src_val; + dst1_row[dj + 2] = dst1_row[dj + 5] = src_val; + } + } + } + else //if ((hr == 4.0f) && (wr == 4.0f)) //the only scale that lasts after isSupported check + { +#ifndef ANDROID + size_t roiw8 = dsize.width >= 7 ? (dsize.width - 7) * 3 : 0; +#endif + + for (size_t i = 0; i < dsize.height; ++i) + { + const u8 * src0_row = internal::getRowPtr(srcBase, srcStride, i << 2); + const u8 * src1_row = internal::getRowPtr(srcBase, srcStride, (i << 2) + 1); + const u8 * src2_row = internal::getRowPtr(srcBase, srcStride, (i << 2) + 2); + const u8 * src3_row = internal::getRowPtr(srcBase, srcStride, (i << 2) + 3); + u8 * dst_row = internal::getRowPtr(dstBase, dstStride, i); + size_t sj = 0, dj = 0; + +#ifndef ANDROID + for ( ; dj < roiw8; dj += 24, sj += 96) + { + internal::prefetch(src0_row + sj); + internal::prefetch(src1_row + sj); + internal::prefetch(src2_row + sj); + internal::prefetch(src3_row + sj); + + uint8x16x3_t vLane10 = vld3q_u8(src0_row + sj), vLane11 = vld3q_u8(src0_row + sj + 48); + uint8x16x3_t vLane20 = vld3q_u8(src1_row + sj), vLane21 = vld3q_u8(src1_row + sj + 48); + uint8x16x3_t vLane30 = vld3q_u8(src2_row + sj), vLane31 = vld3q_u8(src2_row + sj + 48); + uint8x16x3_t vLane40 = vld3q_u8(src3_row + sj), vLane41 = vld3q_u8(src3_row + sj + 48); + + uint8x8x3_t v_dst; + + // channel 0 + { + uint16x8_t v_lane0 = vpaddlq_u8(vLane10.val[0]); + uint16x8_t v_lane1 = vpaddlq_u8(vLane20.val[0]); + uint16x8_t v_lane2 = vpaddlq_u8(vLane30.val[0]); + uint16x8_t v_lane3 = vpaddlq_u8(vLane40.val[0]); + v_lane0 = vaddq_u16(v_lane0, v_lane1); + v_lane0 = vaddq_u16(v_lane0, v_lane2); + v_lane0 = vaddq_u16(v_lane0, v_lane3); + + uint16x8_t v_lane0_ = vpaddlq_u8(vLane11.val[0]); + uint16x8_t v_lane1_ = vpaddlq_u8(vLane21.val[0]); + uint16x8_t v_lane2_ = vpaddlq_u8(vLane31.val[0]); + uint16x8_t v_lane3_ = vpaddlq_u8(vLane41.val[0]); + v_lane0_ = vaddq_u16(v_lane0_, v_lane1_); + v_lane0_ = vaddq_u16(v_lane0_, v_lane2_); + v_lane0_ = vaddq_u16(v_lane0_, v_lane3_); + + v_dst.val[0] = areaDownsamplingDivision( + vcombine_u16(vmovn_u32(vpaddlq_u16(v_lane0)), + vmovn_u32(vpaddlq_u16(v_lane0_)))); + } + + // channel 1 + { + uint16x8_t v_lane0 = vpaddlq_u8(vLane10.val[1]); + uint16x8_t v_lane1 = vpaddlq_u8(vLane20.val[1]); + uint16x8_t v_lane2 = vpaddlq_u8(vLane30.val[1]); + uint16x8_t v_lane3 = vpaddlq_u8(vLane40.val[1]); + v_lane0 = vaddq_u16(v_lane0, v_lane1); + v_lane0 = vaddq_u16(v_lane0, v_lane2); + v_lane0 = vaddq_u16(v_lane0, v_lane3); + + uint16x8_t v_lane0_ = vpaddlq_u8(vLane11.val[1]); + uint16x8_t v_lane1_ = vpaddlq_u8(vLane21.val[1]); + uint16x8_t v_lane2_ = vpaddlq_u8(vLane31.val[1]); + uint16x8_t v_lane3_ = vpaddlq_u8(vLane41.val[1]); + v_lane0_ = vaddq_u16(v_lane0_, v_lane1_); + v_lane0_ = vaddq_u16(v_lane0_, v_lane2_); + v_lane0_ = vaddq_u16(v_lane0_, v_lane3_); + + v_dst.val[1] = areaDownsamplingDivision( + vcombine_u16(vmovn_u32(vpaddlq_u16(v_lane0)), + vmovn_u32(vpaddlq_u16(v_lane0_)))); + } + + // channel 2 + { + uint16x8_t v_lane0 = vpaddlq_u8(vLane10.val[2]); + uint16x8_t v_lane1 = vpaddlq_u8(vLane20.val[2]); + uint16x8_t v_lane2 = vpaddlq_u8(vLane30.val[2]); + uint16x8_t v_lane3 = vpaddlq_u8(vLane40.val[2]); + v_lane0 = vaddq_u16(v_lane0, v_lane1); + v_lane0 = vaddq_u16(v_lane0, v_lane2); + v_lane0 = vaddq_u16(v_lane0, v_lane3); + + uint16x8_t v_lane0_ = vpaddlq_u8(vLane11.val[2]); + uint16x8_t v_lane1_ = vpaddlq_u8(vLane21.val[2]); + uint16x8_t v_lane2_ = vpaddlq_u8(vLane31.val[2]); + uint16x8_t v_lane3_ = vpaddlq_u8(vLane41.val[2]); + v_lane0_ = vaddq_u16(v_lane0_, v_lane1_); + v_lane0_ = vaddq_u16(v_lane0_, v_lane2_); + v_lane0_ = vaddq_u16(v_lane0_, v_lane3_); + + v_dst.val[2] = areaDownsamplingDivision( + vcombine_u16(vmovn_u32(vpaddlq_u16(v_lane0)), + vmovn_u32(vpaddlq_u16(v_lane0_)))); + } + + vst3_u8(dst_row + dj, v_dst); + } +#endif + + for (size_t dwidth = dsize.width * 3; dj < dwidth; dj += 3, sj += 12) + { + dst_row[dj ] = areaDownsamplingDivision( + (u16)src0_row[sj ] + src0_row[sj + 3] + + src0_row[sj + 6] + src0_row[sj + 9] + + src1_row[sj ] + src1_row[sj + 3] + + src1_row[sj + 6] + src1_row[sj + 9] + + src2_row[sj ] + src2_row[sj + 3] + + src2_row[sj + 6] + src2_row[sj + 9] + + src3_row[sj ] + src3_row[sj + 3] + + src3_row[sj + 6] + src3_row[sj + 9]); + + dst_row[dj + 1] = areaDownsamplingDivision( + (u16)src0_row[sj + 1] + src0_row[sj + 4] + + src0_row[sj + 7] + src0_row[sj + 10] + + src1_row[sj + 1] + src1_row[sj + 4] + + src1_row[sj + 7] + src1_row[sj + 10] + + src2_row[sj + 1] + src2_row[sj + 4] + + src2_row[sj + 7] + src2_row[sj + 10] + + src3_row[sj + 1] + src3_row[sj + 4] + + src3_row[sj + 7] + src3_row[sj + 10]); + + dst_row[dj + 2] = areaDownsamplingDivision( + (u16)src0_row[sj + 2] + src0_row[sj + 5] + + src0_row[sj + 8] + src0_row[sj + 11] + + src1_row[sj + 2] + src1_row[sj + 5] + + src1_row[sj + 8] + src1_row[sj + 11] + + src2_row[sj + 2] + src2_row[sj + 5] + + src2_row[sj + 8] + src2_row[sj + 11] + + src3_row[sj + 2] + src3_row[sj + 5] + + src3_row[sj + 8] + src3_row[sj + 11]); + } + } + } + } +#else + (void)dsize; + (void)srcBase; + (void)srcStride; + (void)dstBase; + (void)dstStride; + (void)wr; + (void)hr; +#endif + (void)ssize; +} + +void resizeAreaOpenCV(const Size2D &ssize, const Size2D &dsize, + const u8 * srcBase, ptrdiff_t srcStride, + u8 * dstBase, ptrdiff_t dstStride, + f32 wr, f32 hr, u32 channels) +{ + resizeAreaRounding(ssize, dsize, srcBase, srcStride, dstBase, dstStride, wr, hr, channels); +} + +void resizeArea(const Size2D &ssize, const Size2D &dsize, + const u8 * srcBase, ptrdiff_t srcStride, + u8 * dstBase, ptrdiff_t dstStride, + f32 wr, f32 hr, u32 channels) +{ + resizeAreaRounding(ssize, dsize, srcBase, srcStride, dstBase, dstStride, wr, hr, channels); +} + +#ifdef CAROTENE_NEON + +namespace { + +uint8x8_t resizeLinearStep(uint8x16_t vr1, uint8x16_t vr2, + uint8x8_t vlutl, uint8x8_t vluth, + float32x4_t vrw, float32x4_t vcw0, float32x4_t vcw1) +{ + uint8x8_t vr1l = internal::vqtbl1_u8(vr1, vlutl); + uint8x8_t vr1h = internal::vqtbl1_u8(vr1, vluth); + uint8x8_t vr2l = internal::vqtbl1_u8(vr2, vlutl); + uint8x8_t vr2h = internal::vqtbl1_u8(vr2, vluth); + + uint16x8_t v1hw = vmovl_u8(vr1h); + uint16x8_t v2hw = vmovl_u8(vr2h); + + int16x8_t v1df = vreinterpretq_s16_u16(vsubl_u8(vr1l, vr1h)); + int16x8_t v2df = vreinterpretq_s16_u16(vsubl_u8(vr2l, vr2h)); + + float32x4_t v1L = vcvtq_f32_u32(vmovl_u16(vget_low_u16(v1hw))); + float32x4_t v1H = vcvtq_f32_u32(vmovl_u16(vget_high_u16(v1hw))); + float32x4_t v2L = vcvtq_f32_u32(vmovl_u16(vget_low_u16(v2hw))); + float32x4_t v2H = vcvtq_f32_u32(vmovl_u16(vget_high_u16(v2hw))); + + v1L = vmlaq_f32(v1L, vcvtq_f32_s32(vmovl_s16(vget_low_s16(v1df))), vcw0); + v1H = vmlaq_f32(v1H, vcvtq_f32_s32(vmovl_s16(vget_high_s16(v1df))), vcw1); + v2L = vmlaq_f32(v2L, vcvtq_f32_s32(vmovl_s16(vget_low_s16(v2df))), vcw0); + v2H = vmlaq_f32(v2H, vcvtq_f32_s32(vmovl_s16(vget_high_s16(v2df))), vcw1); + + float32x4_t vdiffL = vsubq_f32(v1L, v2L); + float32x4_t vdiffH = vsubq_f32(v1H, v2H); + + float32x4_t vL = vmlaq_f32(v2L, vdiffL, vrw); + float32x4_t vH = vmlaq_f32(v2H, vdiffH, vrw); + uint16x4_t vL_ = vmovn_u32(vcvtq_u32_f32(vL)); + uint16x4_t vH_ = vmovn_u32(vcvtq_u32_f32(vH)); + return vmovn_u16(vcombine_u16(vL_, vH_)); +} + +} // namespace + +namespace { + +void resize_bilinear_rows(const Size2D &ssize, const Size2D &dsize, + const u8 * srcBase, ptrdiff_t srcStride, + u8 * dstBase, ptrdiff_t dstStride, + f32 hr, const u8** gcols, u8* gcweight, u8* buf) +{ + f32 scale_y_offset = 0.5f * hr - 0.5f; + + size_t dst_h8 = dsize.height & ~7; + size_t dst_w8 = dsize.width & ~7; + size_t src_w8 = ssize.width & ~7; + + size_t r = 0; + for (; r < dst_h8; r += 8) + { +resize8u_xystretch: + const u8* rows[16]; + u8 rweight[8]; + + for (u32 i = 0; i < 8; ++i) + { + f32 w = (i + r) * hr + scale_y_offset; + ptrdiff_t src_row = floorf(w); + ptrdiff_t src_row2 = src_row + 1; + + rweight[i] = (u8)((src_row2-w) * 128); + + if (src_row < 0) + src_row = 0; + if (src_row2 >= (ptrdiff_t)ssize.height) + src_row2 = ssize.height-1; + + rows[2 * i] = srcBase + src_row * srcStride; + rows[2 * i + 1] = srcBase + src_row2 * srcStride; + } + + uint8x8_t vr0w = vdup_n_u8(rweight[0]); + uint8x8_t vr1w = vdup_n_u8(rweight[1]); + uint8x8_t vr2w = vdup_n_u8(rweight[2]); + uint8x8_t vr3w = vdup_n_u8(rweight[3]); + uint8x8_t vr4w = vdup_n_u8(rweight[4]); + uint8x8_t vr5w = vdup_n_u8(rweight[5]); + uint8x8_t vr6w = vdup_n_u8(rweight[6]); + uint8x8_t vr7w = vdup_n_u8(rweight[7]); + + uint8x8_t vr0w2 = vdup_n_u8(128 - rweight[0]); + uint8x8_t vr1w2 = vdup_n_u8(128 - rweight[1]); + uint8x8_t vr2w2 = vdup_n_u8(128 - rweight[2]); + uint8x8_t vr3w2 = vdup_n_u8(128 - rweight[3]); + uint8x8_t vr4w2 = vdup_n_u8(128 - rweight[4]); + uint8x8_t vr5w2 = vdup_n_u8(128 - rweight[5]); + uint8x8_t vr6w2 = vdup_n_u8(128 - rweight[6]); + uint8x8_t vr7w2 = vdup_n_u8(128 - rweight[7]); + + size_t col = 0; + for(; col < src_w8; col += 8) + { + internal::prefetch(rows[3] + col); + internal::prefetch(rows[7] + col); + internal::prefetch(rows[11] + col); + internal::prefetch(rows[15] + col); +resize8u_ystretch: + uint8x8_t vsrc0l1 = vld1_u8(rows[0] + col); + uint8x8_t vsrc0l2 = vld1_u8(rows[1] + col); + uint8x8_t vsrc1l1 = vld1_u8(rows[2] + col); + uint8x8_t vsrc1l2 = vld1_u8(rows[3] + col); + + // (l1 * w + l2 * (128 - w) + 64) / 128 + uint16x8_t vdst0l = vmull_u8(vsrc0l1, vr0w); + uint16x8_t vdst1l = vmull_u8(vsrc1l1, vr1w); + + uint8x8_t vsrc2l1 = vld1_u8(rows[4] + col); + uint8x8_t vsrc2l2 = vld1_u8(rows[5] + col); + uint8x8_t vsrc3l1 = vld1_u8(rows[6] + col); + uint8x8_t vsrc3l2 = vld1_u8(rows[7] + col); + + vdst0l = vmlal_u8(vdst0l, vsrc0l2, vr0w2); + vdst1l = vmlal_u8(vdst1l, vsrc1l2, vr1w2); + uint16x8_t vdst2l = vmull_u8(vsrc2l1, vr2w); + uint16x8_t vdst3l = vmull_u8(vsrc3l1, vr3w); + + uint8x8_t vsrc4l1 = vld1_u8(rows[8] + col); + uint8x8_t vsrc4l2 = vld1_u8(rows[9] + col); + uint8x8_t vsrc5l1 = vld1_u8(rows[10] + col); + uint8x8_t vsrc5l2 = vld1_u8(rows[11] + col); + + vdst2l = vmlal_u8(vdst2l, vsrc2l2, vr2w2); + vdst3l = vmlal_u8(vdst3l, vsrc3l2, vr3w2); + uint16x8_t vdst4l = vmull_u8(vsrc4l1, vr4w); + uint16x8_t vdst5l = vmull_u8(vsrc5l1, vr5w); + + uint8x8_t vsrc6l1 = vld1_u8(rows[12] + col); + uint8x8_t vsrc6l2 = vld1_u8(rows[13] + col); + uint8x8_t vsrc7l1 = vld1_u8(rows[14] + col); + uint8x8_t vsrc7l2 = vld1_u8(rows[15] + col); + + uint8x8_t vdst0 = vrshrn_n_u16(vdst0l, 7); + uint8x8_t vdst1 = vrshrn_n_u16(vdst1l, 7); + vdst4l = vmlal_u8(vdst4l, vsrc4l2, vr4w2); + vdst5l = vmlal_u8(vdst5l, vsrc5l2, vr5w2); + uint16x8_t vdst6l = vmull_u8(vsrc6l1, vr6w); + uint16x8_t vdst7l = vmull_u8(vsrc7l1, vr7w); + + uint8x8_t vdst2 = vrshrn_n_u16(vdst2l, 7); + uint8x8_t vdst3 = vrshrn_n_u16(vdst3l, 7); + vdst6l = vmlal_u8(vdst6l, vsrc6l2, vr6w2); + vdst7l = vmlal_u8(vdst7l, vsrc7l2, vr7w2); + + uint8x8_t vdst4 = vrshrn_n_u16(vdst4l, 7); + uint8x8_t vdst5 = vrshrn_n_u16(vdst5l, 7); + uint8x8_t vdst6 = vrshrn_n_u16(vdst6l, 7); + uint8x8_t vdst7 = vrshrn_n_u16(vdst7l, 7); + + // == 8x8 matrix transpose == + + //00 01 02 03 04 05 06 07 d0 + //10 11 12 13 14 15 16 17 d1 + //20 21 22 23 24 25 26 27 d2 + //30 31 32 33 34 35 36 37 d3 + //40 41 42 43 44 45 46 47 d4 + //50 51 52 53 54 55 56 57 d5 + //60 61 62 63 64 65 66 67 d6 + //70 71 72 73 74 75 76 77 d7 + + uint8x8x2_t vdst10t = vtrn_u8(vdst0, vdst1); + uint8x8x2_t vdst32t = vtrn_u8(vdst2, vdst3); + uint8x8x2_t vdst54t = vtrn_u8(vdst4, vdst5); + uint8x8x2_t vdst76t = vtrn_u8(vdst6, vdst7); + + uint8x16_t vd1d0 = vcombine_u8(vdst10t.val[0], vdst10t.val[1]); + uint8x16_t vd3d2 = vcombine_u8(vdst32t.val[0], vdst32t.val[1]); + uint8x16_t vd5d4 = vcombine_u8(vdst54t.val[0], vdst54t.val[1]); + uint8x16_t vd7d6 = vcombine_u8(vdst76t.val[0], vdst76t.val[1]); + + //00 10 02 12 04 14 06 16 d0 + //01 11 03 13 05 15 07 17 d1 + //20 30 22 32 24 34 26 36 d2 + //21 31 23 33 25 35 27 37 d3 + //40 50 42 52 44 54 46 56 d4 + //41 51 43 53 45 55 47 57 d5 + //60 70 62 72 64 74 66 76 d6 + //61 71 63 73 65 75 67 77 d7 + + uint16x8x2_t vq1q0t = vtrnq_u16((uint16x8_t)vd1d0, (uint16x8_t)vd3d2); + uint16x8x2_t vq3q2t = vtrnq_u16((uint16x8_t)vd5d4, (uint16x8_t)vd7d6); + + //00 10 20 30 04 14 24 34 d0 + //01 11 21 31 05 15 25 35 d1 + //02 12 22 32 06 16 26 36 d2 + //03 13 23 33 07 17 27 37 d3 + //40 50 60 70 44 54 64 74 d4 + //41 51 61 71 45 55 65 75 d5 + //42 52 62 72 46 56 66 76 d6 + //43 53 63 73 47 57 67 77 d7 + + uint32x4x2_t vq2q0t = vtrnq_u32((uint32x4_t)vq1q0t.val[0], (uint32x4_t)vq3q2t.val[0]); + uint32x4x2_t vq3q1t = vtrnq_u32((uint32x4_t)vq1q0t.val[1], (uint32x4_t)vq3q2t.val[1]); + + //00 10 20 30 40 50 60 70 d0 + //01 11 21 31 41 51 61 71 d1 + //02 12 22 32 42 52 62 72 d2 + //03 13 23 33 43 53 63 73 d3 + //04 14 24 34 44 54 64 74 d4 + //05 15 25 35 45 55 65 75 d5 + //06 16 26 36 46 56 66 76 d6 + //07 17 27 37 47 57 67 77 d7 + + vst1q_u8(buf + col * 8 + 0, (uint8x16_t)vq2q0t.val[0]); + vst1q_u8(buf + col * 8 + 16, (uint8x16_t)vq3q1t.val[0]); + vst1q_u8(buf + col * 8 + 32, (uint8x16_t)vq2q0t.val[1]); + vst1q_u8(buf + col * 8 + 48, (uint8x16_t)vq3q1t.val[1]); + } + + if (col < ssize.width) + { + col = ssize.width - 8; + goto resize8u_ystretch; + } + + u8* dst_data = dstBase + r * dstStride; + const u8** cols = gcols; + u8* cweight = gcweight; + + size_t dcol = 0; + for (; dcol < dst_w8; dcol += 8, cols += 16, cweight += 8) + { + internal::prefetch(cols[0], 64*4); +resize8u_xstretch: + uint8x8_t vc0w = vdup_n_u8(cweight[0]); + uint8x8_t vc1w = vdup_n_u8(cweight[1]); + uint8x8_t vc2w = vdup_n_u8(cweight[2]); + uint8x8_t vc3w = vdup_n_u8(cweight[3]); + uint8x8_t vc4w = vdup_n_u8(cweight[4]); + uint8x8_t vc5w = vdup_n_u8(cweight[5]); + uint8x8_t vc6w = vdup_n_u8(cweight[6]); + uint8x8_t vc7w = vdup_n_u8(cweight[7]); + + uint8x8_t vc0w2 = vdup_n_u8(128 - cweight[0]); + uint8x8_t vc1w2 = vdup_n_u8(128 - cweight[1]); + uint8x8_t vc2w2 = vdup_n_u8(128 - cweight[2]); + uint8x8_t vc3w2 = vdup_n_u8(128 - cweight[3]); + uint8x8_t vc4w2 = vdup_n_u8(128 - cweight[4]); + uint8x8_t vc5w2 = vdup_n_u8(128 - cweight[5]); + uint8x8_t vc6w2 = vdup_n_u8(128 - cweight[6]); + uint8x8_t vc7w2 = vdup_n_u8(128 - cweight[7]); + + uint8x8_t vsrc0l1 = vld1_u8(cols[0]); + uint8x8_t vsrc0l2 = vld1_u8(cols[1]); + uint8x8_t vsrc1l1 = vld1_u8(cols[2]); + uint8x8_t vsrc1l2 = vld1_u8(cols[3]); + uint8x8_t vsrc2l1 = vld1_u8(cols[4]); + uint8x8_t vsrc2l2 = vld1_u8(cols[5]); + uint8x8_t vsrc3l1 = vld1_u8(cols[6]); + uint8x8_t vsrc3l2 = vld1_u8(cols[7]); + uint8x8_t vsrc4l1 = vld1_u8(cols[8]); + uint8x8_t vsrc4l2 = vld1_u8(cols[9]); + uint8x8_t vsrc5l1 = vld1_u8(cols[10]); + uint8x8_t vsrc5l2 = vld1_u8(cols[11]); + uint8x8_t vsrc6l1 = vld1_u8(cols[12]); + uint8x8_t vsrc6l2 = vld1_u8(cols[13]); + uint8x8_t vsrc7l1 = vld1_u8(cols[14]); + uint8x8_t vsrc7l2 = vld1_u8(cols[15]); + + // (l1 * w + l2 * (128 - w) + 64) / 128 + uint16x8_t vdst0l = vmull_u8(vsrc0l1, vc0w); + uint16x8_t vdst1l = vmull_u8(vsrc1l1, vc1w); + uint16x8_t vdst2l = vmull_u8(vsrc2l1, vc2w); + uint16x8_t vdst3l = vmull_u8(vsrc3l1, vc3w); + uint16x8_t vdst4l = vmull_u8(vsrc4l1, vc4w); + uint16x8_t vdst5l = vmull_u8(vsrc5l1, vc5w); + uint16x8_t vdst6l = vmull_u8(vsrc6l1, vc6w); + uint16x8_t vdst7l = vmull_u8(vsrc7l1, vc7w); + + vdst0l = vmlal_u8(vdst0l, vsrc0l2, vc0w2); + vdst1l = vmlal_u8(vdst1l, vsrc1l2, vc1w2); + vdst2l = vmlal_u8(vdst2l, vsrc2l2, vc2w2); + vdst3l = vmlal_u8(vdst3l, vsrc3l2, vc3w2); + vdst4l = vmlal_u8(vdst4l, vsrc4l2, vc4w2); + vdst5l = vmlal_u8(vdst5l, vsrc5l2, vc5w2); + vdst6l = vmlal_u8(vdst6l, vsrc6l2, vc6w2); + vdst7l = vmlal_u8(vdst7l, vsrc7l2, vc7w2); + + uint8x8_t vdst0 = vrshrn_n_u16(vdst0l, 7); + uint8x8_t vdst1 = vrshrn_n_u16(vdst1l, 7); + uint8x8_t vdst2 = vrshrn_n_u16(vdst2l, 7); + uint8x8_t vdst3 = vrshrn_n_u16(vdst3l, 7); + uint8x8_t vdst4 = vrshrn_n_u16(vdst4l, 7); + uint8x8_t vdst5 = vrshrn_n_u16(vdst5l, 7); + uint8x8_t vdst6 = vrshrn_n_u16(vdst6l, 7); + uint8x8_t vdst7 = vrshrn_n_u16(vdst7l, 7); + + // == 8x8 matrix transpose == + uint8x8x2_t vdst10t = vtrn_u8(vdst0, vdst1); + uint8x8x2_t vdst32t = vtrn_u8(vdst2, vdst3); + uint8x8x2_t vdst54t = vtrn_u8(vdst4, vdst5); + uint8x8x2_t vdst76t = vtrn_u8(vdst6, vdst7); + uint8x16_t vd1d0 = vcombine_u8(vdst10t.val[0], vdst10t.val[1]); + uint8x16_t vd3d2 = vcombine_u8(vdst32t.val[0], vdst32t.val[1]); + uint8x16_t vd5d4 = vcombine_u8(vdst54t.val[0], vdst54t.val[1]); + uint8x16_t vd7d6 = vcombine_u8(vdst76t.val[0], vdst76t.val[1]); + uint16x8x2_t vq1q0t = vtrnq_u16((uint16x8_t)vd1d0, (uint16x8_t)vd3d2); + uint16x8x2_t vq3q2t = vtrnq_u16((uint16x8_t)vd5d4, (uint16x8_t)vd7d6); + uint32x4x2_t vq2q0t = vtrnq_u32((uint32x4_t)vq1q0t.val[0], (uint32x4_t)vq3q2t.val[0]); + uint32x4x2_t vq3q1t = vtrnq_u32((uint32x4_t)vq1q0t.val[1], (uint32x4_t)vq3q2t.val[1]); + + //save results + vst1_u8(dst_data + 0 * dstStride + dcol, (uint8x8_t)vget_low_u32(vq2q0t.val[0])); + vst1_u8(dst_data + 1 * dstStride + dcol, (uint8x8_t)vget_high_u32(vq2q0t.val[0])); + vst1_u8(dst_data + 2 * dstStride + dcol, (uint8x8_t)vget_low_u32(vq3q1t.val[0])); + vst1_u8(dst_data + 3 * dstStride + dcol, (uint8x8_t)vget_high_u32(vq3q1t.val[0])); + vst1_u8(dst_data + 4 * dstStride + dcol, (uint8x8_t)vget_low_u32(vq2q0t.val[1])); + vst1_u8(dst_data + 5 * dstStride + dcol, (uint8x8_t)vget_high_u32(vq2q0t.val[1])); + vst1_u8(dst_data + 6 * dstStride + dcol, (uint8x8_t)vget_low_u32(vq3q1t.val[1])); + vst1_u8(dst_data + 7 * dstStride + dcol, (uint8x8_t)vget_high_u32(vq3q1t.val[1])); + } + + if (dcol < dsize.width) + { + dcol = dsize.width - 8; + cols = gcols + dcol * 2; + cweight = gcweight + dcol; + goto resize8u_xstretch; + } + } + + if (r < dsize.height) + { + r = dsize.height - 8; + goto resize8u_xystretch; + } +} + +template struct resizeLinearInternals; +template <> struct resizeLinearInternals<1> +{ + int32x4_t vc_upd; + int32x4_t vc0; + int32x4_t vcmax; + + inline resizeLinearInternals(int32x4_t & vi, u32 srccols) + { + vc_upd = vdupq_n_s32(4); + vc0 = vdupq_n_s32(0); + vcmax = vdupq_n_s32(srccols-1); + + s32 tmp0123[] = {0, 1, 2, 3 }; + vi = vld1q_s32(tmp0123); + } + inline void updateIndexes(int32x4_t & vi, int32x4_t & vsrch, int32x4_t & vsrcl) + { + vsrch = vminq_s32(vsrch, vcmax); + vsrcl = vmaxq_s32(vsrcl, vc0); + vsrcl = vminq_s32(vsrcl, vcmax);//for safe tail + vsrch = vshlq_n_s32(vsrch, 3); + vsrcl = vshlq_n_s32(vsrcl, 3); + vi = vaddq_s32(vi, vc_upd); + } +}; +template <> struct resizeLinearInternals<4> +{ + int32x4_t vc_upd; + int32x4_t vc0; + int32x4_t vcmax; + int32x4_t v0123x8; + + inline resizeLinearInternals(int32x4_t & vi, u32 srccols) + { + vc_upd = vdupq_n_s32(1); + vc0 = vdupq_n_s32(0); + vcmax = vdupq_n_s32(srccols-1); + s32 tmp0123x8[] = {0, 8, 16, 24}; + v0123x8 = vld1q_s32(tmp0123x8); + + vi = vc0; + } + inline void updateIndexes(int32x4_t & vi, int32x4_t & vsrch, int32x4_t & vsrcl) + { + vsrch = vminq_s32(vsrch, vcmax); + vsrcl = vmaxq_s32(vsrcl, vc0); + vsrch = vshlq_n_s32(vsrch, 5); + vsrcl = vshlq_n_s32(vsrcl, 5); + vsrch = vaddq_s32(vsrch, v0123x8); + vsrcl = vaddq_s32(vsrcl, v0123x8); + vi = vaddq_s32(vi, vc_upd); + } +}; + +template +void resizeLinearOpenCVchan(const Size2D &_ssize, const Size2D &_dsize, + const u8 * srcBase, ptrdiff_t srcStride, + u8 * dstBase, ptrdiff_t dstStride, + f32 wr, f32 hr) +{ + float scale_x_offset = 0.5f * wr - 0.5f; + + Size2D ssize(_ssize.width*channels, _ssize.height); + Size2D dsize(_dsize.width*channels, _dsize.height); + + std::vector gcweight((dsize.width + 7) & ~7); + std::vector gcols(((dsize.width + 7) & ~7) * 2); + std::vector buf(((ssize.width + 7) & ~7) * 8); // (8 rows) x (width of src) + + float32x4_t vscale_x = vdupq_n_f32(wr); + float32x4_t vscale_x_offset = vdupq_n_f32(scale_x_offset); + int32x4_t vc1 = vdupq_n_s32(1); + float32x4_t vc128f = vdupq_n_f32(128.0f); + + int32x4_t vi; + resizeLinearInternals indexes(vi, _ssize.width);//u32 is used to store indexes + //so we could get issues on src image dimensions greater than (2^32-1) + + for (size_t dcol = 0; dcol < dsize.width; dcol += 8) + { + s32 idx[16]; + + float32x4_t vif = vcvtq_f32_s32(vi); + float32x4_t vw = vmlaq_f32(vscale_x_offset, vscale_x, vif); + int32x4_t vwi = vcvtq_s32_f32(vw); + float32x4_t vwif = vcvtq_f32_s32(vwi); + int32x4_t vmask = (int32x4_t)vcltq_f32(vwif, vw); + int32x4_t vsrch = vsubq_s32(vwi, vmask); + int32x4_t vsrcl = vsubq_s32(vsrch, vc1); + float32x4_t vsrchf = vcvtq_f32_s32(vsrch); + float32x4_t vw2 = vsubq_f32(vsrchf, vw); + + vw2 = vmulq_f32(vw2, vc128f); + uint32x4_t vw32u = vcvtq_u32_f32(vw2); + uint16x4_t vw16ul = vmovn_u32(vw32u); + indexes.updateIndexes(vi, vsrch, vsrcl); + + vst1q_s32(idx + 0, vsrcl); + vst1q_s32(idx + 8, vsrch); + + vif = vcvtq_f32_s32(vi); + vw = vmlaq_f32(vscale_x_offset, vscale_x, vif); + vwi = vcvtq_s32_f32(vw); + vwif = vcvtq_f32_s32(vwi); + vmask = (int32x4_t)vcltq_f32(vwif, vw); + vsrch = vsubq_s32(vwi, vmask); + vsrcl = vsubq_s32(vsrch, vc1); + vsrchf = vcvtq_f32_s32(vsrch); + vw2 = vsubq_f32(vsrchf, vw); + + vw2 = vmulq_f32(vw2, vc128f); + vw32u = vcvtq_u32_f32(vw2); + indexes.updateIndexes(vi, vsrch, vsrcl); + + uint16x4_t vw16uh = vmovn_u32(vw32u); + + vst1q_s32(idx + 4, vsrcl); + vst1q_s32(idx + 12, vsrch); + + uint8x8_t vw8u = vmovn_u16(vcombine_u16(vw16ul, vw16uh)); + + for (u32 i = 0; i < 8; ++i) + { + gcols[dcol * 2 + i*2] = &buf[idx[i]]; + gcols[dcol * 2 + i*2 + 1] = &buf[idx[i + 8]]; + } + + vst1_u8(&gcweight[dcol], vw8u); + } + + resize_bilinear_rows(ssize, dsize, srcBase, srcStride, dstBase, dstStride, hr, &gcols[0], &gcweight[0], &buf[0]); +} + +void downsample_bilinear_8uc1(const Size2D &ssize, const Size2D &dsize, + const u8 * srcBase, ptrdiff_t srcStride, + u8 * dstBase, ptrdiff_t dstStride, + f32 wr, f32 hr) +{ + internal::assertSupportedConfiguration(wr <= 2.f && hr <= 2.f); + + enum { SHIFT_BITS = 11 }; + + f32 scale_x_offset = 0.5f * wr - 0.5f; + f32 scale_y_offset = 0.5f * hr - 0.5f; + + std::vector _buf(dsize.height*(2*(sizeof(ptrdiff_t)/sizeof(s32))+1)+1); + ptrdiff_t* buf = (ptrdiff_t*)&_buf[0]; + s32* buf2 = (s32*)buf+2*(sizeof(ptrdiff_t)/sizeof(s32))*dsize.height; + for(size_t row = 0; row < (size_t)dsize.height; ++row) + { + f32 r = row * hr + scale_y_offset; + ptrdiff_t src_row = floorf(r); + ptrdiff_t src_row2 = src_row + 1; + + f32 rweight = src_row2 - r; + buf2[row] = floorf(rweight * (1 << SHIFT_BITS) + 0.5f); + buf[0 * dsize.height + row] = std::max(0, src_row); + buf[1 * dsize.height + row] = std::min((ptrdiff_t)ssize.height-1, src_row2); + } + +#define USE_CORRECT_VERSION 0 + + ptrdiff_t col = 0; +/***********************************************/ + for(; col <= (ptrdiff_t)dsize.width-16; col+=16) + { + ptrdiff_t col1[16]; + ptrdiff_t col2[16]; + s16 cwi[16]; + + for(s32 k = 0; k < 16; ++k) + { + f32 c = (col + k) * wr + scale_x_offset; + col1[k] = (ptrdiff_t)c; + col2[k] = col1[k] + 1; + + cwi[k] = (short)floorf((col2[k] - c) * (1 << SHIFT_BITS) + 0.5f); + + if(col1[k] < 0) col1[k] = 0; + if(col2[k] >= (ptrdiff_t)ssize.width) col2[k] = ssize.width-1; + } + + ptrdiff_t x = std::min(col1[0], (ptrdiff_t)ssize.width-16); + ptrdiff_t y = std::min(col1[8], (ptrdiff_t)ssize.width-16); + u8 lutl[16]; + u8 luth[16]; + for(s32 k = 0; k < 8; ++k) + { + lutl[k] = (u8)(col1[k] - x); + luth[k] = (u8)(col2[k] - x); + lutl[k+8] = (u8)(col1[k+8] - y); + luth[k+8] = (u8)(col2[k+8] - y); + } + + uint8x8_t vlutl = vld1_u8(lutl); + uint8x8_t vluth = vld1_u8(luth); + int16x8_t vcw = vld1q_s16(cwi); + + uint8x8_t vlutl_ = vld1_u8(lutl+8); + uint8x8_t vluth_ = vld1_u8(luth+8); + int16x8_t vcw_ = vld1q_s16(cwi+8); + + for(ptrdiff_t row = 0; row < (ptrdiff_t)dsize.height; ++row) + { +#if USE_CORRECT_VERSION + int32x4_t vrw = vdupq_n_s32(buf2[row]); +#else + int16x8_t vrw = vdupq_n_s16((int16_t)buf2[row]); + int16x8_t vrW = vdupq_n_s16((int16_t)((1 << SHIFT_BITS) - buf2[row])); +#endif + + internal::prefetch(internal::getRowPtr(srcBase, srcStride, buf[1*dsize.height + row]) + x, 2*srcStride); + internal::prefetch(internal::getRowPtr(srcBase, srcStride, buf[1*dsize.height + row]) + x, 3*srcStride); + + { + union { uint8x16_t v; uint8x8x2_t w; } vr1 = { vld1q_u8(internal::getRowPtr(srcBase, srcStride, buf[0*dsize.height + row]) + x) }; + union { uint8x16_t v; uint8x8x2_t w; } vr2 = { vld1q_u8(internal::getRowPtr(srcBase, srcStride, buf[1*dsize.height + row]) + x) }; + + uint8x8_t vr1l = vtbl2_u8(vr1.w, vlutl); + uint8x8_t vr1h = vtbl2_u8(vr1.w, vluth); + uint8x8_t vr2l = vtbl2_u8(vr2.w, vlutl); + uint8x8_t vr2h = vtbl2_u8(vr2.w, vluth); + + uint16x8_t v1hw = vmovl_u8(vr1h); + uint16x8_t v2hw = vmovl_u8(vr2h); + + int16x8_t v1df = vreinterpretq_s16_u16(vsubl_u8(vr1l, vr1h)); + int16x8_t v2df = vreinterpretq_s16_u16(vsubl_u8(vr2l, vr2h)); + + int32x4_t v1L = vreinterpretq_s32_u32(vshll_n_u16(vget_low_u16(v1hw), SHIFT_BITS)); + int32x4_t v1H = vreinterpretq_s32_u32(vshll_n_u16(vget_high_u16(v1hw), SHIFT_BITS)); + int32x4_t v2L = vreinterpretq_s32_u32(vshll_n_u16(vget_low_u16(v2hw), SHIFT_BITS)); + int32x4_t v2H = vreinterpretq_s32_u32(vshll_n_u16(vget_high_u16(v2hw), SHIFT_BITS)); + + v1L = vmlal_s16(v1L, vget_low_s16(v1df), vget_low_s16(vcw)); + v1H = vmlal_s16(v1H, vget_high_s16(v1df), vget_high_s16(vcw)); + v2L = vmlal_s16(v2L, vget_low_s16(v2df), vget_low_s16(vcw)); + v2H = vmlal_s16(v2H, vget_high_s16(v2df), vget_high_s16(vcw)); + +#if USE_CORRECT_VERSION + /* correct version */ + int32x4_t vL = vshlq_n_s32(v2L, SHIFT_BITS); + int32x4_t vH = vshlq_n_s32(v2H, SHIFT_BITS); + int32x4_t vdiffL = vsubq_s32(v1L, v2L); + int32x4_t vdiffH = vsubq_s32(v1H, v2H); + + vL = vmlaq_s32(vL, vdiffL, vrw); + vH = vmlaq_s32(vH, vdiffH, vrw); + uint16x4_t vL_ = vqrshrun_n_s32(vL, 2*SHIFT_BITS - 8); + uint16x4_t vH_ = vqrshrun_n_s32(vH, 2*SHIFT_BITS - 8); + uint8x8_t vres = vrshrn_n_u16(vcombine_u16(vL_, vH_), 8); + vst1_u8(internal::getRowPtr(dstBase, dstStride, row) + col, vres); +#else + /* ugly version matching to OpenCV's SSE optimization */ + int16x4_t v1Ls = vshrn_n_s32(v1L, 5); + int16x4_t v1Hs = vshrn_n_s32(v1H, 5); + int16x4_t v2Ls = vshrn_n_s32(v2L, 5); + int16x4_t v2Hs = vshrn_n_s32(v2H, 5); + + int16x8_t v1s = vqdmulhq_s16(vcombine_s16(v1Ls, v1Hs), vrw); + int16x8_t v2s = vqdmulhq_s16(vcombine_s16(v2Ls, v2Hs), vrW); + + int16x8_t vsum = vaddq_s16(v1s, v2s); + uint8x8_t vres = vqrshrun_n_s16(vsum, 2); + + vst1_u8(internal::getRowPtr(dstBase, dstStride, row) + col, vres); +#endif + } + + { + union { uint8x16_t v; uint8x8x2_t w; } vr1 = { vld1q_u8(internal::getRowPtr(srcBase, srcStride, buf[0*dsize.height + row]) + y) }; + union { uint8x16_t v; uint8x8x2_t w; } vr2 = { vld1q_u8(internal::getRowPtr(srcBase, srcStride, buf[1*dsize.height + row]) + y) }; + + uint8x8_t vr1l = vtbl2_u8(vr1.w, vlutl_); + uint8x8_t vr1h = vtbl2_u8(vr1.w, vluth_); + uint8x8_t vr2l = vtbl2_u8(vr2.w, vlutl_); + uint8x8_t vr2h = vtbl2_u8(vr2.w, vluth_); + + uint16x8_t v1hw = vmovl_u8(vr1h); + uint16x8_t v2hw = vmovl_u8(vr2h); + + int16x8_t v1df = vreinterpretq_s16_u16(vsubl_u8(vr1l, vr1h)); + int16x8_t v2df = vreinterpretq_s16_u16(vsubl_u8(vr2l, vr2h)); + + int32x4_t v1L = vreinterpretq_s32_u32(vshll_n_u16(vget_low_u16(v1hw), SHIFT_BITS)); + int32x4_t v1H = vreinterpretq_s32_u32(vshll_n_u16(vget_high_u16(v1hw), SHIFT_BITS)); + int32x4_t v2L = vreinterpretq_s32_u32(vshll_n_u16(vget_low_u16(v2hw), SHIFT_BITS)); + int32x4_t v2H = vreinterpretq_s32_u32(vshll_n_u16(vget_high_u16(v2hw), SHIFT_BITS)); + + v1L = vmlal_s16(v1L, vget_low_s16(v1df), vget_low_s16(vcw_)); + v1H = vmlal_s16(v1H, vget_high_s16(v1df), vget_high_s16(vcw_)); + v2L = vmlal_s16(v2L, vget_low_s16(v2df), vget_low_s16(vcw_)); + v2H = vmlal_s16(v2H, vget_high_s16(v2df), vget_high_s16(vcw_)); + +#if USE_CORRECT_VERSION + /* correct version */ + int32x4_t vL = vshlq_n_s32(v2L, SHIFT_BITS); + int32x4_t vH = vshlq_n_s32(v2H, SHIFT_BITS); + int32x4_t vdiffL = vsubq_s32(v1L, v2L); + int32x4_t vdiffH = vsubq_s32(v1H, v2H); + + vL = vmlaq_s32(vL, vdiffL, vrw); + vH = vmlaq_s32(vH, vdiffH, vrw); + uint16x4_t vL_ = vqrshrun_n_s32(vL, 2*SHIFT_BITS - 8); + uint16x4_t vH_ = vqrshrun_n_s32(vH, 2*SHIFT_BITS - 8); + uint8x8_t vres = vrshrn_n_u16(vcombine_u16(vL_, vH_), 8); + vst1_u8(internal::getRowPtr(dstBase, dstStride, row) + col + 8, vres); +#else + /* ugly version matching to OpenCV's SSE optimization */ + int16x4_t v1Ls = vshrn_n_s32(v1L, 5); + int16x4_t v1Hs = vshrn_n_s32(v1H, 5); + int16x4_t v2Ls = vshrn_n_s32(v2L, 5); + int16x4_t v2Hs = vshrn_n_s32(v2H, 5); + + int16x8_t v1s = vqdmulhq_s16(vcombine_s16(v1Ls, v1Hs), vrw); + int16x8_t v2s = vqdmulhq_s16(vcombine_s16(v2Ls, v2Hs), vrW); + + int16x8_t vsum = vaddq_s16(v1s, v2s); + uint8x8_t vres = vqrshrun_n_s16(vsum, 2); + + vst1_u8(internal::getRowPtr(dstBase, dstStride, row) + col + 8, vres); +#endif + } + } + } +/***********************************************/ + for(; col <= (ptrdiff_t)dsize.width-8; col+=8) + { +downsample_bilinear_8uc1_col_loop8: + ptrdiff_t col1[8]; + ptrdiff_t col2[8]; + s16 cwi[8]; + + for(s32 k = 0; k < 8; ++k) + { + f32 c = (col + k) * wr + scale_x_offset; + col1[k] = (ptrdiff_t)c; + col2[k] = col1[k] + 1; + + cwi[k] = (s16)floorf((col2[k] - c) * (1 << SHIFT_BITS) + 0.5f); + + if(col1[k] < 0) col1[k] = 0; + if(col2[k] >= (ptrdiff_t)ssize.width) col2[k] = (ptrdiff_t)ssize.width-1; + } + + ptrdiff_t x = std::min(col1[0], (ptrdiff_t)ssize.width-16); + u8 lutl[8]; + u8 luth[8]; + for(s32 k = 0; k < 8; ++k) + { + lutl[k] = (u8)(col1[k] - x); + luth[k] = (u8)(col2[k] - x); + } + + uint8x8_t vlutl = vld1_u8(lutl); + uint8x8_t vluth = vld1_u8(luth); + int16x8_t vcw = vld1q_s16(cwi); + + for(ptrdiff_t row = 0; row < (ptrdiff_t)dsize.height; ++row) + { +#if USE_CORRECT_VERSION + int32x4_t vrw = vdupq_n_s32(buf2[row]); +#else + int16x8_t vrw = vdupq_n_s16((int16_t)buf2[row]); + int16x8_t vrW = vdupq_n_s16((int16_t)((1 << SHIFT_BITS) - buf2[row])); +#endif + + internal::prefetch(internal::getRowPtr(srcBase, srcStride, buf[1*dsize.height + row]) + x, 2*srcStride); + internal::prefetch(internal::getRowPtr(srcBase, srcStride, buf[1*dsize.height + row]) + x, 3*srcStride); + + union { uint8x16_t v; uint8x8x2_t w; } vr1 = { vld1q_u8(internal::getRowPtr(srcBase, srcStride, buf[0*dsize.height + row]) + x) }; + union { uint8x16_t v; uint8x8x2_t w; } vr2 = { vld1q_u8(internal::getRowPtr(srcBase, srcStride, buf[1*dsize.height + row]) + x) }; + + uint8x8_t vr1l = vtbl2_u8(vr1.w, vlutl); + uint8x8_t vr1h = vtbl2_u8(vr1.w, vluth); + uint8x8_t vr2l = vtbl2_u8(vr2.w, vlutl); + uint8x8_t vr2h = vtbl2_u8(vr2.w, vluth); + + uint16x8_t v1hw = vmovl_u8(vr1h); + uint16x8_t v2hw = vmovl_u8(vr2h); + + int16x8_t v1df = vreinterpretq_s16_u16(vsubl_u8(vr1l, vr1h)); + int16x8_t v2df = vreinterpretq_s16_u16(vsubl_u8(vr2l, vr2h)); + + int32x4_t v1L = vreinterpretq_s32_u32(vshll_n_u16(vget_low_u16(v1hw), SHIFT_BITS)); + int32x4_t v1H = vreinterpretq_s32_u32(vshll_n_u16(vget_high_u16(v1hw), SHIFT_BITS)); + int32x4_t v2L = vreinterpretq_s32_u32(vshll_n_u16(vget_low_u16(v2hw), SHIFT_BITS)); + int32x4_t v2H = vreinterpretq_s32_u32(vshll_n_u16(vget_high_u16(v2hw), SHIFT_BITS)); + + v1L = vmlal_s16(v1L, vget_low_s16(v1df), vget_low_s16(vcw)); + v1H = vmlal_s16(v1H, vget_high_s16(v1df), vget_high_s16(vcw)); + v2L = vmlal_s16(v2L, vget_low_s16(v2df), vget_low_s16(vcw)); + v2H = vmlal_s16(v2H, vget_high_s16(v2df), vget_high_s16(vcw)); + +#if USE_CORRECT_VERSION + /* correct version */ + int32x4_t vL = vshlq_n_s32(v2L, SHIFT_BITS); + int32x4_t vH = vshlq_n_s32(v2H, SHIFT_BITS); + int32x4_t vdiffL = vsubq_s32(v1L, v2L); + int32x4_t vdiffH = vsubq_s32(v1H, v2H); + + vL = vmlaq_s32(vL, vdiffL, vrw); + vH = vmlaq_s32(vH, vdiffH, vrw); + uint16x4_t vL_ = vqrshrun_n_s32(vL, 2*SHIFT_BITS - 8); + uint16x4_t vH_ = vqrshrun_n_s32(vH, 2*SHIFT_BITS - 8); + uint8x8_t vres = vrshrn_n_u16(vcombine_u16(vL_, vH_), 8); + vst1_u8(internal::getRowPtr(dstBase, dstStride, row) + col, vres); +#else + /* ugly version matching to OpenCV's SSE optimization */ + int16x4_t v1Ls = vshrn_n_s32(v1L, 5); + int16x4_t v1Hs = vshrn_n_s32(v1H, 5); + int16x4_t v2Ls = vshrn_n_s32(v2L, 5); + int16x4_t v2Hs = vshrn_n_s32(v2H, 5); + + int16x8_t v1s = vqdmulhq_s16(vcombine_s16(v1Ls, v1Hs), vrw); + int16x8_t v2s = vqdmulhq_s16(vcombine_s16(v2Ls, v2Hs), vrW); + + int16x8_t vsum = vaddq_s16(v1s, v2s); + uint8x8_t vres = vqrshrun_n_s16(vsum, 2); + + vst1_u8(internal::getRowPtr(dstBase, dstStride, row) + col, vres); +#endif + } + } + if (col < (ptrdiff_t)dsize.width) + { + col = dsize.width - 8; + goto downsample_bilinear_8uc1_col_loop8; + } +} + +} // namespace + +#endif + +void resizeLinearOpenCV(const Size2D &ssize, const Size2D &dsize, + const u8 * srcBase, ptrdiff_t srcStride, + u8 * dstBase, ptrdiff_t dstStride, + f32 wr, f32 hr, u32 channels) +{ + internal::assertSupportedConfiguration(wr > 0 && hr > 0 && + (dsize.width - 0.5) * wr - 0.5 < ssize.width && + (dsize.height - 0.5) * hr - 0.5 < ssize.height && // Ensure we have enough source data + (dsize.width + 0.5) * wr + 0.5 >= ssize.width && + (dsize.height + 0.5) * hr + 0.5 >= ssize.height && // Ensure source isn't too big + isResizeLinearOpenCVSupported(ssize, dsize, channels)); +#ifdef CAROTENE_NEON + if(1 == channels) + { + if (wr <= 1.f && hr <= 1.f) + resizeLinearOpenCVchan<1>(ssize, dsize, srcBase, srcStride, dstBase, dstStride, wr, hr); + else if (wr <= 2.0f && hr <= 2.0f && ssize.width >= 16) + downsample_bilinear_8uc1(ssize, dsize, srcBase, srcStride, dstBase, dstStride, wr, hr); + else + resizeLinearOpenCVchan<1>(ssize, dsize, srcBase, srcStride, dstBase, dstStride, wr, hr); + } + else if(4 == channels) + resizeLinearOpenCVchan<4>(ssize, dsize, srcBase, srcStride, dstBase, dstStride, wr, hr); +#else + (void)ssize; + (void)dsize; + (void)srcBase; + (void)srcStride; + (void)dstBase; + (void)dstStride; + (void)wr; + (void)hr; + (void)channels; +#endif +} + +void resizeLinear(const Size2D &ssize, const Size2D &dsize, + const u8 * srcBase, ptrdiff_t srcStride, + u8 * dstBase, ptrdiff_t dstStride, + f32 wr, f32 hr, u32 channels) +{ + internal::assertSupportedConfiguration(wr > 0 && hr > 0 && + (dsize.width - 0.5) * wr - 0.5 < ssize.width && + (dsize.height - 0.5) * hr - 0.5 < ssize.height && // Ensure we have enough source data + (dsize.width + 0.5) * wr + 0.5 >= ssize.width && + (dsize.height + 0.5) * hr + 0.5 >= ssize.height && // Ensure source isn't too big + isResizeLinearSupported(ssize, dsize, + wr, hr, channels)); +#ifdef CAROTENE_NEON + f32 scale_x = wr; + f32 scale_x_offset = 0.5f * scale_x - 0.5f; + f32 scale_y = hr; + f32 scale_y_offset = 0.5f * scale_y - 0.5f; + + std::vector _buf(dsize.height * 3 + 1); + std::vector coeff(dsize.height); + ptrdiff_t * buf = &_buf[0]; + + for (size_t row = 0; row < dsize.height; ++row) + { + f32 r = row * scale_y + scale_y_offset; + ptrdiff_t src_row = floorf(r); + ptrdiff_t src_row2 = src_row + 1; + + f32 rweight = src_row2 - r; + buf[0 * dsize.height + row] = std::max(0, src_row); + buf[1 * dsize.height + row] = std::min(ssize.height - 1, src_row2); + coeff[row] = rweight; + } + + size_t col = 0; + for ( ; col + 16 <= dsize.width; col += 16) + { + ptrdiff_t col1[16], col2[16]; + f32 cwi[16]; + + for(s32 k = 0; k < 16; ++k) + { + f32 c = (col + k) * scale_x + scale_x_offset; + col1[k] = floorf(c); + col2[k] = col1[k] + 1; + + cwi[k] = col2[k] - c; + + if (col1[k] < 0) + col1[k] = 0; + if (col2[k] >= (ptrdiff_t)ssize.width) + col2[k] = ssize.width - 1; + } + + ptrdiff_t x = std::min(col1[0], ssize.width - 16); + ptrdiff_t y = std::min(col1[8], ssize.width - 16); + u8 lutl[16], luth[16]; + + for (s32 k = 0; k < 8; ++k) + { + lutl[k] = (u8)(col1[k] - x); + luth[k] = (u8)(col2[k] - x); + lutl[k + 8] = (u8)(col1[k + 8] - y); + luth[k + 8] = (u8)(col2[k + 8] - y); + } + + uint8x8_t vlutl = vld1_u8(lutl); + uint8x8_t vluth = vld1_u8(luth); + float32x4_t vcw0 = vld1q_f32(cwi); + float32x4_t vcw1 = vld1q_f32(cwi + 4); + + uint8x8_t vlutl_ = vld1_u8(lutl + 8); + uint8x8_t vluth_ = vld1_u8(luth + 8); + float32x4_t vcw0_ = vld1q_f32(cwi + 8); + float32x4_t vcw1_ = vld1q_f32(cwi + 12); + + if (channels == 1) + { + for (size_t row = 0; row < dsize.height; ++row) + { + float32x4_t vrw = vdupq_n_f32(coeff[row]); + + const u8 * srow0 = internal::getRowPtr(srcBase, srcStride, buf[0 * dsize.height + row]); + const u8 * srow1 = internal::getRowPtr(srcBase, srcStride, buf[1 * dsize.height + row]); + u8 * drow = internal::getRowPtr(dstBase, dstStride, row); + + internal::prefetch(srow0 + x + 2 * srcStride); + internal::prefetch(srow1 + x + 2 * srcStride); + + uint8x8_t vres0 = resizeLinearStep(vld1q_u8(srow0 + x), vld1q_u8(srow1 + x), + vlutl, vluth, + vrw, vcw0, vcw1); + + uint8x8_t vres1 = resizeLinearStep(vld1q_u8(srow0 + y), vld1q_u8(srow1 + y), + vlutl_, vluth_, + vrw, vcw0_, vcw1_); + + vst1q_u8(drow + col, vcombine_u8(vres0, vres1)); + } + } + else if (channels == 3) + { + for (size_t row = 0; row < dsize.height; ++row) + { + float32x4_t vrw = vdupq_n_f32(coeff[row]); + + const u8 * srow0 = internal::getRowPtr(srcBase, srcStride, buf[0 * dsize.height + row]); + const u8 * srow1 = internal::getRowPtr(srcBase, srcStride, buf[1 * dsize.height + row]); + u8 * drow = internal::getRowPtr(dstBase, dstStride, row); + + internal::prefetch(srow0 + x + 2 * srcStride); + internal::prefetch(srow1 + x + 2 * srcStride); + + uint8x16x3_t v_src10 = vld3q_u8(srow0 + (x * 3)); + uint8x16x3_t v_src20 = vld3q_u8(srow1 + (x * 3)); + + uint8x16x3_t v_src11 = vld3q_u8(srow0 + (y * 3)); + uint8x16x3_t v_src21 = vld3q_u8(srow1 + (y * 3)); + + uint8x16x3_t v_dst; + + v_dst.val[0] = vcombine_u8(resizeLinearStep(v_src10.val[0], v_src20.val[0], vlutl, vluth, vrw, vcw0, vcw1), + resizeLinearStep(v_src11.val[0], v_src21.val[0], vlutl_, vluth_, vrw, vcw0_, vcw1_)); + v_dst.val[1] = vcombine_u8(resizeLinearStep(v_src10.val[1], v_src20.val[1], vlutl, vluth, vrw, vcw0, vcw1), + resizeLinearStep(v_src11.val[1], v_src21.val[1], vlutl_, vluth_, vrw, vcw0_, vcw1_)); + v_dst.val[2] = vcombine_u8(resizeLinearStep(v_src10.val[2], v_src20.val[2], vlutl, vluth, vrw, vcw0, vcw1), + resizeLinearStep(v_src11.val[2], v_src21.val[2], vlutl_, vluth_, vrw, vcw0_, vcw1_)); + + vst3q_u8(drow + (col * 3), v_dst); + } + } + else if (channels == 4) + { + for (size_t row = 0; row < dsize.height; ++row) + { + float32x4_t vrw = vdupq_n_f32(coeff[row]); + + const u8 * srow0 = internal::getRowPtr(srcBase, srcStride, buf[0 * dsize.height + row]); + const u8 * srow1 = internal::getRowPtr(srcBase, srcStride, buf[1 * dsize.height + row]); + u8 * drow = internal::getRowPtr(dstBase, dstStride, row); + + internal::prefetch(srow0 + x + 2 * srcStride); + internal::prefetch(srow1 + x + 2 * srcStride); + + uint8x16x4_t v_src10 = vld4q_u8(srow0 + (x << 2)); + uint8x16x4_t v_src20 = vld4q_u8(srow1 + (x << 2)); + + uint8x16x4_t v_src11 = vld4q_u8(srow0 + (y << 2)); + uint8x16x4_t v_src21 = vld4q_u8(srow1 + (y << 2)); + + uint8x16x4_t v_dst; + + v_dst.val[0] = vcombine_u8(resizeLinearStep(v_src10.val[0], v_src20.val[0], vlutl, vluth, vrw, vcw0, vcw1), + resizeLinearStep(v_src11.val[0], v_src21.val[0], vlutl_, vluth_, vrw, vcw0_, vcw1_)); + v_dst.val[1] = vcombine_u8(resizeLinearStep(v_src10.val[1], v_src20.val[1], vlutl, vluth, vrw, vcw0, vcw1), + resizeLinearStep(v_src11.val[1], v_src21.val[1], vlutl_, vluth_, vrw, vcw0_, vcw1_)); + v_dst.val[2] = vcombine_u8(resizeLinearStep(v_src10.val[2], v_src20.val[2], vlutl, vluth, vrw, vcw0, vcw1), + resizeLinearStep(v_src11.val[2], v_src21.val[2], vlutl_, vluth_, vrw, vcw0_, vcw1_)); + v_dst.val[3] = vcombine_u8(resizeLinearStep(v_src10.val[3], v_src20.val[3], vlutl, vluth, vrw, vcw0, vcw1), + resizeLinearStep(v_src11.val[3], v_src21.val[3], vlutl_, vluth_, vrw, vcw0_, vcw1_)); + + vst4q_u8(drow + (col << 2), v_dst); + } + } + } + + for ( ; col + 8 <= dsize.width; col += 8) + { +downsample_bilinear_8uc1_col_loop8: + ptrdiff_t col1[8], col2[8]; + f32 cwi[8]; + + for (s32 k = 0; k < 8; ++k) + { + f32 c = (col + k) * scale_x + scale_x_offset; + col1[k] = floorf(c); + col2[k] = col1[k] + 1; + + cwi[k] = col2[k] - c; + + if (col1[k] < 0) + col1[k] = 0; + if (col2[k] >= (ptrdiff_t)ssize.width) + col2[k] = ssize.width - 1; + } + + ptrdiff_t x = std::min(col1[0], ssize.width - 16); + u8 lutl[8], luth[8]; + for (s32 k = 0; k < 8; ++k) + { + lutl[k] = (u8)(col1[k] - x); + luth[k] = (u8)(col2[k] - x); + } + + uint8x8_t vlutl = vld1_u8(lutl); + uint8x8_t vluth = vld1_u8(luth); + float32x4_t vcw0 = vld1q_f32(cwi); + float32x4_t vcw1 = vld1q_f32(cwi + 4); + + if (channels == 1) + { + for (size_t row = 0; row < dsize.height; ++row) + { + float32x4_t vrw = vdupq_n_f32(coeff[row]); + + const u8 * srow0 = internal::getRowPtr(srcBase, srcStride, buf[0 * dsize.height + row]); + const u8 * srow1 = internal::getRowPtr(srcBase, srcStride, buf[1 * dsize.height + row]); + u8 * drow = internal::getRowPtr(dstBase, dstStride, row); + + internal::prefetch(srow0 + x + 2 * srcStride); + internal::prefetch(srow1 + x + 2 * srcStride); + + uint8x8_t vres = resizeLinearStep(vld1q_u8(srow0 + x), vld1q_u8(srow1 + x), + vlutl, vluth, + vrw, vcw0, vcw1); + vst1_u8(drow + col, vres); + } + } + else if (channels == 3) + { + for (size_t row = 0; row < dsize.height; ++row) + { + float32x4_t vrw = vdupq_n_f32(coeff[row]); + + const u8 * srow0 = internal::getRowPtr(srcBase, srcStride, buf[0 * dsize.height + row]); + const u8 * srow1 = internal::getRowPtr(srcBase, srcStride, buf[1 * dsize.height + row]); + u8 * drow = internal::getRowPtr(dstBase, dstStride, row); + + internal::prefetch(srow0 + x + 2 * srcStride); + internal::prefetch(srow1 + x + 2 * srcStride); + + uint8x16x3_t v_src1 = vld3q_u8(srow0 + (x * 3)); + uint8x16x3_t v_src2 = vld3q_u8(srow1 + (x * 3)); + + uint8x8x3_t v_dst; + + v_dst.val[0] = resizeLinearStep(v_src1.val[0], v_src2.val[0], vlutl, vluth, vrw, vcw0, vcw1); + v_dst.val[1] = resizeLinearStep(v_src1.val[1], v_src2.val[1], vlutl, vluth, vrw, vcw0, vcw1); + v_dst.val[2] = resizeLinearStep(v_src1.val[2], v_src2.val[2], vlutl, vluth, vrw, vcw0, vcw1); + + vst3_u8(drow + (col * 3), v_dst); + } + } + else if (channels == 4) + { + for (size_t row = 0; row < dsize.height; ++row) + { + float32x4_t vrw = vdupq_n_f32(coeff[row]); + + const u8 * srow0 = internal::getRowPtr(srcBase, srcStride, buf[0 * dsize.height + row]); + const u8 * srow1 = internal::getRowPtr(srcBase, srcStride, buf[1 * dsize.height + row]); + u8 * drow = internal::getRowPtr(dstBase, dstStride, row); + + internal::prefetch(srow0 + x + 2 * srcStride); + internal::prefetch(srow1 + x + 2 * srcStride); + + uint8x16x4_t v_src1 = vld4q_u8(srow0 + (x << 2)); + uint8x16x4_t v_src2 = vld4q_u8(srow1 + (x << 2)); + + uint8x8x4_t v_dst; + + v_dst.val[0] = resizeLinearStep(v_src1.val[0], v_src2.val[0], vlutl, vluth, vrw, vcw0, vcw1); + v_dst.val[1] = resizeLinearStep(v_src1.val[1], v_src2.val[1], vlutl, vluth, vrw, vcw0, vcw1); + v_dst.val[2] = resizeLinearStep(v_src1.val[2], v_src2.val[2], vlutl, vluth, vrw, vcw0, vcw1); + v_dst.val[3] = resizeLinearStep(v_src1.val[3], v_src2.val[3], vlutl, vluth, vrw, vcw0, vcw1); + + vst4_u8(drow + (col << 2), v_dst); + } + } + } + + if (col < dsize.width) + { + col = dsize.width - 8; + goto downsample_bilinear_8uc1_col_loop8; + } + +#else + (void)ssize; + (void)dsize; + (void)srcBase; + (void)srcStride; + (void)dstBase; + (void)dstStride; + (void)wr; + (void)hr; + (void)channels; +#endif +} + +} // namespace CAROTENE_NS diff --git a/3rdparty/carotene/src/saturate_cast.hpp b/3rdparty/carotene/src/saturate_cast.hpp new file mode 100644 index 0000000000..98f8545009 --- /dev/null +++ b/3rdparty/carotene/src/saturate_cast.hpp @@ -0,0 +1,199 @@ +/* + * By downloading, copying, installing or using the software you agree to this license. + * If you do not agree to this license, do not download, install, + * copy or use the software. + * + * + * License Agreement + * For Open Source Computer Vision Library + * (3-clause BSD License) + * + * Copyright (C) 2012-2015, NVIDIA Corporation, all rights reserved. + * Third party copyrights are property of their respective owners. + * + * Redistribution and use in source and binary forms, with or without modification, + * are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * * Neither the names of the copyright holders nor the names of the contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * This software is provided by the copyright holders and contributors "as is" and + * any express or implied warranties, including, but not limited to, the implied + * warranties of merchantability and fitness for a particular purpose are disclaimed. + * In no event shall copyright holders or contributors be liable for any direct, + * indirect, incidental, special, exemplary, or consequential damages + * (including, but not limited to, procurement of substitute goods or services; + * loss of use, data, or profits; or business interruption) however caused + * and on any theory of liability, whether in contract, strict liability, + * or tort (including negligence or otherwise) arising in any way out of + * the use of this software, even if advised of the possibility of such damage. + */ + +#ifndef CAROTENE_SATURATE_CAST_HPP +#define CAROTENE_SATURATE_CAST_HPP + +#include +#include +#include + +#if defined _MSC_VER && defined _M_ARM +# include +#endif + +#include +#include + +namespace CAROTENE_NS { namespace internal { + +#if defined _MSC_VER && defined _M_ARM + +__declspec(naked) static void vcvtr_s32_f64_imp(f64 d) +{ + (void)d; + __emit(0xEEBD); // vcvtr.s32.f64 s0, d0 + __emit(0x0B40); + __emit(0xEE10); // vmov r0, s0 + __emit(0x0A10); + __emit(0x4770); // bx lr +} + +# define CAROTENE_ROUND_FLT(x) return ((s32 (*)(f64))vcvtr_s32_f64_imp)((f64)x); +# define CAROTENE_ROUND_DBL(x) return ((s32 (*)(f64))vcvtr_s32_f64_imp)(x); + +#elif defined CV_ICC || defined __GNUC__ + +# if defined(__VFP_FP__) && !defined(__SOFTFP__) && !(defined _DEBUG || defined DEBUG) && !defined(__CUDACC__) +# define CAROTENE_ROUND_FLT(value) { \ + register union { f32 f; s32 i; } result; \ + asm ("ftosis %0, %1 \n" : "=w" (result.f) : "w" (value) ); \ + return result.i; } +# define CAROTENE_ROUND_DBL(value) { \ + register union {f32 f; s32 i;} __tegra_result; \ + asm ( \ + "ftosid %0, %P1\n" \ + : "=w" (__tegra_result.f) \ + : "w" (value) \ + ); \ + return __tegra_result.i; \ + } +# else +# define CAROTENE_ROUND_FLT(x) return (s32)lrintf(value); +# define CAROTENE_ROUND_DBL(value) return (s32)lrint(value); +# endif + +#endif + +inline s32 round(f32 value) +{ +#ifdef CAROTENE_ROUND_FLT + CAROTENE_ROUND_FLT(value) +#else + s32 intpart = (s32)(value); + f32 fractpart = value - intpart; + if ((fractpart != 0.5 && fractpart != -0.5) || ((intpart % 2) != 0)) + return (s32)(value + (value >= 0 ? 0.5 : -0.5)); + else + return intpart; +#endif +} + +inline s32 round(f64 value) +{ +#ifdef CAROTENE_ROUND_DBL + CAROTENE_ROUND_DBL(value) +#else + s32 intpart = (s32)(value); + f64 fractpart = value - intpart; + if ((fractpart != 0.5 && fractpart != -0.5) || ((intpart % 2) != 0)) + return (s32)(value + (value >= 0 ? 0.5 : -0.5)); + else + return intpart; +#endif +} +/////////////// saturate_cast (used in image & signal processing) /////////////////// + +template inline _Tp saturate_cast(u8 v) { return _Tp(v); } +template inline _Tp saturate_cast(s8 v) { return _Tp(v); } +template inline _Tp saturate_cast(u16 v) { return _Tp(v); } +template inline _Tp saturate_cast(s16 v) { return _Tp(v); } +template inline _Tp saturate_cast(u32 v) { return _Tp(v); } +template inline _Tp saturate_cast(s32 v) { return _Tp(v); } +template inline _Tp saturate_cast(s64 v) { return _Tp(v); } +template inline _Tp saturate_cast(u64 v) { return _Tp(v); } +template inline _Tp saturate_cast(f32 v) { return _Tp(v); } +template inline _Tp saturate_cast(f64 v) { return _Tp(v); } + +template<> inline u8 saturate_cast(s8 v) { return (u8)std::max((s32)v, 0); } +template<> inline u8 saturate_cast(u16 v) { return (u8)std::min((u32)v, (u32)UCHAR_MAX); } +template<> inline u8 saturate_cast(s32 v) { return (u8)((u32)v <= UCHAR_MAX ? v : v > 0 ? UCHAR_MAX : 0); } +template<> inline u8 saturate_cast(s16 v) { return saturate_cast((s32)v); } +template<> inline u8 saturate_cast(u32 v) { return (u8)std::min(v, (u32)UCHAR_MAX); } +template<> inline u8 saturate_cast(s64 v) { return (u8)((u64)v <= UCHAR_MAX ? v : v > 0 ? UCHAR_MAX : 0); } +template<> inline u8 saturate_cast(u64 v) { return (u8)std::min(v, (u64)UCHAR_MAX); } +template<> inline u8 saturate_cast(f32 v) { return saturate_cast(round(v)); } +template<> inline u8 saturate_cast(f64 v) { return saturate_cast(round(v)); } + +template<> inline s8 saturate_cast(u8 v) { return (s8)std::min((s32)v, SCHAR_MAX); } +template<> inline s8 saturate_cast(u16 v) { return (s8)std::min((u32)v, (u32)SCHAR_MAX); } +template<> inline s8 saturate_cast(s32 v) { return (s8)((u32)(v-SCHAR_MIN) <= (u32)UCHAR_MAX ? v : v > 0 ? SCHAR_MAX : SCHAR_MIN); } +template<> inline s8 saturate_cast(s16 v) { return saturate_cast((s32)v); } +template<> inline s8 saturate_cast(u32 v) { return (s8)std::min(v, (u32)SCHAR_MAX); } +template<> inline s8 saturate_cast(s64 v) { return (s8)((u64)(v-SCHAR_MIN) <= (u64)UCHAR_MAX ? v : v > 0 ? SCHAR_MAX : SCHAR_MIN); } +template<> inline s8 saturate_cast(u64 v) { return (s8)std::min(v, (u64)SCHAR_MAX); } +template<> inline s8 saturate_cast(f32 v) { return saturate_cast(round(v)); } +template<> inline s8 saturate_cast(f64 v) { return saturate_cast(round(v)); } + +template<> inline u16 saturate_cast(s8 v) { return (u16)std::max((s32)v, 0); } +template<> inline u16 saturate_cast(s16 v) { return (u16)std::max((s32)v, 0); } +template<> inline u16 saturate_cast(s32 v) { return (u16)((u32)v <= (u32)USHRT_MAX ? v : v > 0 ? USHRT_MAX : 0); } +template<> inline u16 saturate_cast(u32 v) { return (u16)std::min(v, (u32)USHRT_MAX); } +template<> inline u16 saturate_cast(s64 v) { return (u16)((u64)v <= (u64)USHRT_MAX ? v : v > 0 ? USHRT_MAX : 0); } +template<> inline u16 saturate_cast(u64 v) { return (u16)std::min(v, (u64)USHRT_MAX); } +template<> inline u16 saturate_cast(f32 v) { return saturate_cast(round(v)); } +template<> inline u16 saturate_cast(f64 v) { return saturate_cast(round(v)); } + +template<> inline s16 saturate_cast(u16 v) { return (s16)std::min((s32)v, SHRT_MAX); } +template<> inline s16 saturate_cast(s32 v) { return (s16)((u32)(v - SHRT_MIN) <= (u32)USHRT_MAX ? v : v > 0 ? SHRT_MAX : SHRT_MIN); } +template<> inline s16 saturate_cast(u32 v) { return (s16)std::min(v, (u32)SHRT_MAX); } +template<> inline s16 saturate_cast(s64 v) { return (s16)((u64)(v - SHRT_MIN) <= (u64)USHRT_MAX ? v : v > 0 ? SHRT_MAX : SHRT_MIN); } +template<> inline s16 saturate_cast(u64 v) { return (s16)std::min(v, (u64)SHRT_MAX); } +template<> inline s16 saturate_cast(f32 v) { return saturate_cast(round(v)); } +template<> inline s16 saturate_cast(f64 v) { return saturate_cast(round(v)); } + +template<> inline u32 saturate_cast(s8 v) { return (u32)std::max(v, (s8)0); } +template<> inline u32 saturate_cast(s16 v) { return (u32)std::max(v, (s16)0); } +template<> inline u32 saturate_cast(s32 v) { return (u32)std::max(v, (s32)0); } +template<> inline u32 saturate_cast(s64 v) { return (u32)((u64)v <= (u64)UINT_MAX ? v : v > 0 ? UINT_MAX : 0); } +template<> inline u32 saturate_cast(u64 v) { return (u32)std::min(v, (u64)UINT_MAX); } +//OpenCV like f32/f64 -> u32 conversion +//we intentionally do not clip negative numbers, to make -1 become 0xffffffff etc. +template<> inline u32 saturate_cast(f32 v) { return round(v); } +template<> inline u32 saturate_cast(f64 v) { return round(v); } +//Negative clipping implementation +//template<> inline u32 saturate_cast(f32 v) { return saturate_cast(round(v)); } +//template<> inline u32 saturate_cast(f64 v) { return saturate_cast(round(v)); } + +template<> inline s32 saturate_cast(u32 v) { return (s32)std::min(v, (u32)INT_MAX); } +template<> inline s32 saturate_cast(s64 v) { return (s32)((u64)(v - INT_MIN) <= (u64)UINT_MAX ? v : v > 0 ? INT_MAX : INT_MIN); } +template<> inline s32 saturate_cast(u64 v) { return (s32)std::min(v, (u64)INT_MAX); } +template<> inline s32 saturate_cast(f32 v) { return round(v); } +template<> inline s32 saturate_cast(f64 v) { return round(v); } + +template<> inline u64 saturate_cast(s8 v) { return (u64)std::max(v, (s8)0); } +template<> inline u64 saturate_cast(s16 v) { return (u64)std::max(v, (s16)0); } +template<> inline u64 saturate_cast(s32 v) { return (u64)std::max(v, (s32)0); } +template<> inline u64 saturate_cast(s64 v) { return (u64)std::max(v, (s64)0); } + +template<> inline s64 saturate_cast(u64 v) { return (s64)std::min(v, (u64)LLONG_MAX); } + +} } + +#endif diff --git a/3rdparty/carotene/src/scharr.cpp b/3rdparty/carotene/src/scharr.cpp new file mode 100644 index 0000000000..2c4ba29742 --- /dev/null +++ b/3rdparty/carotene/src/scharr.cpp @@ -0,0 +1,219 @@ +/* + * By downloading, copying, installing or using the software you agree to this license. + * If you do not agree to this license, do not download, install, + * copy or use the software. + * + * + * License Agreement + * For Open Source Computer Vision Library + * (3-clause BSD License) + * + * Copyright (C) 2012-2015, NVIDIA Corporation, all rights reserved. + * Third party copyrights are property of their respective owners. + * + * Redistribution and use in source and binary forms, with or without modification, + * are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * * Neither the names of the copyright holders nor the names of the contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * This software is provided by the copyright holders and contributors "as is" and + * any express or implied warranties, including, but not limited to, the implied + * warranties of merchantability and fitness for a particular purpose are disclaimed. + * In no event shall copyright holders or contributors be liable for any direct, + * indirect, incidental, special, exemplary, or consequential damages + * (including, but not limited to, procurement of substitute goods or services; + * loss of use, data, or profits; or business interruption) however caused + * and on any theory of liability, whether in contract, strict liability, + * or tort (including negligence or otherwise) arising in any way out of + * the use of this software, even if advised of the possibility of such damage. + */ + +#include + +#include "common.hpp" + +namespace CAROTENE_NS { + +bool isScharr3x3Supported(const Size2D &size, BORDER_MODE border, s32 dx, s32 dy, Margin borderMargin) +{ + return (dx == 0 && dy == 1 && + isSeparableFilter3x3Supported(size, border, 3, 1, borderMargin)) || + (dx == 1 && dy == 0 && + isSeparableFilter3x3Supported(size, border, 1, 3, borderMargin)); +} + +void Scharr3x3(const Size2D &size, + const u8 * srcBase, ptrdiff_t srcStride, + s16 * dstBase, ptrdiff_t dstStride, + s32 dx, s32 dy, + BORDER_MODE border, u8 borderValue, Margin borderMargin) +{ + internal::assertSupportedConfiguration(isScharr3x3Supported(size, border, dx, dy, borderMargin)); +#ifdef CAROTENE_NEON + static s16 dw[] = {3, 10, 3}; + + if (dy == 1) + SeparableFilter3x3(size, srcBase, srcStride, dstBase, dstStride, + 3, 1, dw, 0, + border, borderValue, borderMargin); + else + SeparableFilter3x3(size, srcBase, srcStride, dstBase, dstStride, + 1, 3, 0, dw, + border, borderValue, borderMargin); +#else + (void)srcBase; + (void)srcStride; + (void)dstBase; + (void)dstStride; + (void)borderValue; +#endif +} + +void ScharrDeriv(const Size2D &size, s32 cn, + const u8 * srcBase, ptrdiff_t srcStride, + s16 * dstBase, ptrdiff_t dstStride) +{ + internal::assertSupportedConfiguration(); +#ifdef CAROTENE_NEON + size_t colsn = size.width*cn; + size_t roiw8 = colsn > 7 ? colsn - 7 : 0; + + ptrdiff_t delta = (ptrdiff_t)(((size.width + 2)*cn + 15) & -16);//align size + std::vector _tempBuf((delta << 1) + 64); + s16 *trow0 = internal::alignPtr(&_tempBuf[cn], 16), *trow1 = internal::alignPtr(trow0 + delta, 16); + + int16x8_t vc3 = vmovq_n_s16(3); + int16x8_t vc10 = vmovq_n_s16(10); + uint8x8_t v8c10 = vmov_n_u8(10); + + for(size_t y = 0; y < size.height; y++ ) + { + const u8* srow0 = internal::getRowPtr(srcBase, srcStride, y > 0 ? y-1 : size.height > 1 ? 1 : 0); + const u8* srow1 = internal::getRowPtr(srcBase, srcStride, y); + const u8* srow2 = internal::getRowPtr(srcBase, srcStride, y < size.height-1 ? y+1 : size.height > 1 ? size.height-2 : 0); + s16* drow = internal::getRowPtr(dstBase, dstStride, y); + + // do vertical convolution + size_t x = 0; + for( ; x < roiw8; x += 8 ) + { + internal::prefetch(srow0 + x); + internal::prefetch(srow1 + x); + internal::prefetch(srow2 + x); +#if __GNUC_MINOR__ < 7 + __asm__ ( + "vld1.8 {d0}, [%[src0]] \n\t" + "vld1.8 {d2}, [%[src2]] \n\t" + "vld1.8 {d1}, [%[src1]] \n\t" + "vaddl.u8 q2, d2, d0 \n\t" + "vmull.u8 q3, d1, %[vc10] \n\t" + "vsubl.u8 q4, d2, d0 \n\t" + "vmla.s16 q3, q2, %q[vc3] \n\t" + "vst1.16 {d8-d9}, [%[out1],:128] \n\t" + "vst1.16 {d6-d7}, [%[out0],:128] \n\t" + : + : [out0] "r" (trow0 + x), + [out1] "r" (trow1 + x), + [src0] "r" (srow0 + x), + [src1] "r" (srow1 + x), + [src2] "r" (srow2 + x), + [vc10] "w" (v8c10), [vc3] "w" (vc3) + : "d0","d1","d2","d3","d4","d5","d6","d7","d8","d9","d10","d11","d12","d13","d14","d15" + ); +#else + uint8x8_t s0 = vld1_u8(srow0 + x); + uint8x8_t s1 = vld1_u8(srow1 + x); + uint8x8_t s2 = vld1_u8(srow2 + x); + + int16x8_t s1x10 = vreinterpretq_s16_u16(vmull_u8(s1, v8c10)); + int16x8_t s02 = vreinterpretq_s16_u16(vaddl_u8(s2, s0)); + int16x8_t t1 = vreinterpretq_s16_u16(vsubl_u8(s2, s0)); + int16x8_t t0 = vmlaq_s16(s1x10, s02, vc3); + + vst1q_s16(trow1 + x, t1); + vst1q_s16(trow0 + x, t0); +#endif + } + for( ; x < colsn; x++ ) + { + trow0[x] = (s16)((srow0[x] + srow2[x])*3 + srow1[x]*10); + trow1[x] = (s16)(srow2[x] - srow0[x]); + } + + // make border + size_t x0 = (size.width > 1 ? cn : 0), x1 = (size.width > 1 ? (size.width-2)*cn : 0); + for( s32 k = 0; k < cn; k++ ) + { + trow0[-cn + k] = trow0[x0 + k]; trow0[colsn + k] = trow0[x1 + k]; + trow1[-cn + k] = trow1[x0 + k]; trow1[colsn + k] = trow1[x1 + k]; + } + + // do horizontal convolution, interleave the results and store them to dst + x = 0; + for( ; x < roiw8; x += 8 ) + { +#if __GNUC_MINOR__ < 6 + __asm__ ( + "vld1.16 {d4-d5}, [%[s2ptr]] \n\t" + "vld1.16 {d8-d9}, [%[s4ptr]] \n\t" + "vld1.16 {d6-d7}, [%[s3ptr],:128] \n\t" + "vld1.16 {d0-d1}, [%[s0ptr]] \n\t" + "vld1.16 {d2-d3}, [%[s1ptr]] \n\t" + "vadd.i16 q7, q2, q4 \n\t" + "vmul.s16 q6, q3, %q[vc10] \n\t" + "vsub.s16 q5, q1, q0 \n\t" + "vmla.s16 q6, q7, %q[vc3] \n\t" + "vst2.16 {d10-d13}, [%[out]] \n\t" + : + : [out] "r" (drow + x * 2), + [s0ptr] "r" (trow0 + x - cn), + [s1ptr] "r" (trow0 + x + cn), + [s2ptr] "r" (trow1 + x - cn), + [s3ptr] "r" (trow1 + x), + [s4ptr] "r" (trow1 + x + cn), + [vc10] "w" (vc10), [vc3] "w" (vc3) + : "d0","d1","d2","d3","d4","d5","d6","d7","d8","d9","d10","d11","d12","d13","d14","d15" + ); +#else + int16x8_t s0 = vld1q_s16(trow0 + x - cn); + int16x8_t s1 = vld1q_s16(trow0 + x + cn); + int16x8_t s2 = vld1q_s16(trow1 + x - cn); + int16x8_t s3 = vld1q_s16(trow1 + x); + int16x8_t s4 = vld1q_s16(trow1 + x + cn); + + int16x8_t s3x10 = vmulq_s16(s3, vc10); + int16x8_t s24 = vaddq_s16(s2, s4); + + int16x8x2_t vr; + vr.val[0] = vsubq_s16(s1, s0); + vr.val[1] = vmlaq_s16(s3x10, s24, vc3); + + vst2q_s16(drow + x*2, vr); +#endif //__GNUC_MINOR__ < 6 + } + for( ; x < colsn; x++ ) + { + drow[x*2] = (s16)(trow0[x+cn] - trow0[x-cn]); + drow[x*2+1] = (s16)((trow1[x+cn] + trow1[x-cn])*3 + trow1[x]*10); + } + } +#else + (void)size; + (void)cn; + (void)srcBase; + (void)srcStride; + (void)dstBase; + (void)dstStride; +#endif +} + +} // namespace CAROTENE_NS diff --git a/3rdparty/carotene/src/separable_filter.cpp b/3rdparty/carotene/src/separable_filter.cpp new file mode 100644 index 0000000000..a06172c4e6 --- /dev/null +++ b/3rdparty/carotene/src/separable_filter.cpp @@ -0,0 +1,109 @@ +/* + * By downloading, copying, installing or using the software you agree to this license. + * If you do not agree to this license, do not download, install, + * copy or use the software. + * + * + * License Agreement + * For Open Source Computer Vision Library + * (3-clause BSD License) + * + * Copyright (C) 2014-2015, NVIDIA Corporation, all rights reserved. + * Third party copyrights are property of their respective owners. + * + * Redistribution and use in source and binary forms, with or without modification, + * are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * * Neither the names of the copyright holders nor the names of the contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * This software is provided by the copyright holders and contributors "as is" and + * any express or implied warranties, including, but not limited to, the implied + * warranties of merchantability and fitness for a particular purpose are disclaimed. + * In no event shall copyright holders or contributors be liable for any direct, + * indirect, incidental, special, exemplary, or consequential damages + * (including, but not limited to, procurement of substitute goods or services; + * loss of use, data, or profits; or business interruption) however caused + * and on any theory of liability, whether in contract, strict liability, + * or tort (including negligence or otherwise) arising in any way out of + * the use of this software, even if advised of the possibility of such damage. + */ + +#include "common.hpp" + +#include "separable_filter.hpp" + +namespace CAROTENE_NS { + +bool isSeparableFilter3x3Supported(const Size2D &size, BORDER_MODE border, s32 dx, s32 dy, Margin borderMargin) +{ + return isSupportedConfiguration() && + size.width >= 9 && size.height >= 1 && + (size.height + borderMargin.top + borderMargin.bottom) >= 2 && + (dx >= 0) && (dx < 4) && (dy >= 0) && (dy < 4) && + (border == BORDER_MODE_CONSTANT || + border == BORDER_MODE_REFLECT || + border == BORDER_MODE_REFLECT101 || + border == BORDER_MODE_REPLICATE ); +} + +void SeparableFilter3x3(const Size2D &size, + const u8 * srcBase, ptrdiff_t srcStride, + s16 * dstBase, ptrdiff_t dstStride, + const u8 rowFilter, const u8 colFilter, const s16 *xw, const s16 *yw, + BORDER_MODE border, u8 borderValue, Margin borderMargin) +{ + internal::assertSupportedConfiguration(isSeparableFilter3x3Supported(size, border, rowFilter, colFilter, borderMargin)); +#ifdef CAROTENE_NEON + if(!((xw || rowFilter < 3) && (yw || colFilter < 3))) + std::abort();//Couldn't call generic filter without provided weights + + typedef void (*sepFilter3x3_8u16s_func)(const Size2D&, const u8*, ptrdiff_t, s16*, ptrdiff_t, + const s16*, const s16*, BORDER_MODE, u8, Margin); + + static sepFilter3x3_8u16s_func quickFilters[4][4]= + { + /*d0y*/{ /*d0x*/ internal::sepFilter3x3::process, + /*dx*/ internal::sepFilter3x3::process, + /*d2x*/ internal::sepFilter3x3::process, + /*dNx*/ internal::sepFilter3x3::process}, + + /*dy */{ /*d0x*/ internal::sepFilter3x3::process, + /*dx*/ internal::sepFilter3x3::process, + /*d2x*/ internal::sepFilter3x3::process, + /*dNx*/ internal::sepFilter3x3::process}, + + /*d2y*/{ /*d0x*/ internal::sepFilter3x3::process, + /*dx*/ internal::sepFilter3x3::process, + /*d2x*/ internal::sepFilter3x3::process, + /*dNx*/ internal::sepFilter3x3::process}, + + /*dNy*/{ /*d0x*/ internal::sepFilter3x3::process, + /*dx*/ internal::sepFilter3x3::process, + /*d2x*/ internal::sepFilter3x3::process, + /*dNx*/ internal::sepFilter3x3::process} + }; + + quickFilters[colFilter][rowFilter](size, srcBase, srcStride, dstBase, dstStride, + xw, yw, border, borderValue, borderMargin); +#else + (void)srcBase; + (void)srcStride; + (void)dstBase; + (void)dstStride; + (void)xw; + (void)yw; + (void)borderValue; +#endif +} + + +} // namespace CAROTENE_NS diff --git a/3rdparty/carotene/src/separable_filter.hpp b/3rdparty/carotene/src/separable_filter.hpp new file mode 100644 index 0000000000..b0f7307fa0 --- /dev/null +++ b/3rdparty/carotene/src/separable_filter.hpp @@ -0,0 +1,1161 @@ +/* + * By downloading, copying, installing or using the software you agree to this license. + * If you do not agree to this license, do not download, install, + * copy or use the software. + * + * + * License Agreement + * For Open Source Computer Vision Library + * (3-clause BSD License) + * + * Copyright (C) 2014-2015, NVIDIA Corporation, all rights reserved. + * Third party copyrights are property of their respective owners. + * + * Redistribution and use in source and binary forms, with or without modification, + * are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * * Neither the names of the copyright holders nor the names of the contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * This software is provided by the copyright holders and contributors "as is" and + * any express or implied warranties, including, but not limited to, the implied + * warranties of merchantability and fitness for a particular purpose are disclaimed. + * In no event shall copyright holders or contributors be liable for any direct, + * indirect, incidental, special, exemplary, or consequential damages + * (including, but not limited to, procurement of substitute goods or services; + * loss of use, data, or profits; or business interruption) however caused + * and on any theory of liability, whether in contract, strict liability, + * or tort (including negligence or otherwise) arising in any way out of + * the use of this software, even if advised of the possibility of such damage. + */ + +#ifndef CAROTENE_SRC_SEPARABLE_FILTER_HPP +#define CAROTENE_SRC_SEPARABLE_FILTER_HPP + +#include "common.hpp" + +#include + +#include + +#ifdef CAROTENE_NEON + +namespace CAROTENE_NS { + +namespace internal { + +struct RowFilter3x3S16Base +{ + typedef u8 srcType; + /* + Various border types, image boundaries are denoted with '|' + + * BORDER_REPLICATE: aaaaaa|abcdefgh|hhhhhhh + * BORDER_REFLECT: fedcba|abcdefgh|hgfedcb + * BORDER_REFLECT_101: gfedcb|abcdefgh|gfedcba + * BORDER_WRAP: cdefgh|abcdefgh|abcdefg + * BORDER_CONSTANT: iiiiii|abcdefgh|iiiiiii with some specified 'i' + */ + inline RowFilter3x3S16Base(const BORDER_MODE _borderType, const srcType _borderValue, const ptrdiff_t borderxl, const ptrdiff_t borderxr): + borderType(_borderType),borderValue(_borderValue) + { + if (borderType == BORDER_MODE_CONSTANT) + { + vfmask = vreinterpret_u8_u64(vmov_n_u64(borderxl ? 0x00ffFFffFFffFFffULL : 0x0100FFffFFffFFffULL)); + vtmask = vreinterpret_u8_u64(vmov_n_u64(borderxr ? 0xFF07060504030201ULL : 0x0706050403020100ULL)); + } + else if (borderType == BORDER_MODE_REFLECT101) + { + vfmask = vreinterpret_u8_u64(vmov_n_u64(borderxl ? 0x0001FFffFFffFFffULL : 0x0100FFffFFffFFffULL)); + vtmask = vreinterpret_u8_u64(vmov_n_u64(borderxr ? 0x0607060504030201ULL : 0x0706050403020100ULL)); + } + else //if (borderType == BORDER_MODE_REFLECT || borderType == BORDER_MODE_REPLICATE) + { + vfmask = vreinterpret_u8_u64(vmov_n_u64(borderxl ? 0x0000FFffFFffFFffULL : 0x0100FFffFFffFFffULL)); + vtmask = vreinterpret_u8_u64(vmov_n_u64(borderxr ? 0x0707060504030201ULL : 0x0706050403020100ULL)); + } + lookLeft = offsetk - borderxl; + lookRight = offsetk - borderxr; + } + + uint8x8_t vfmask; + uint8x8_t vtmask; + enum { offsetk = 1}; + ptrdiff_t lookLeft; + ptrdiff_t lookRight; + const BORDER_MODE borderType; + const srcType borderValue; +}; + +struct ColFilter3x3S16Base +{ + typedef s16 srcType; + + inline ColFilter3x3S16Base(const BORDER_MODE _borderType, const srcType _borderValue): + borderType(_borderType),borderValue(_borderValue) {} + + enum { offsetk = 1}; + const BORDER_MODE borderType; + const srcType borderValue; +}; + +struct RowFilter3x3S16Generic : public RowFilter3x3S16Base +{ + typedef s16 dstType; + + inline RowFilter3x3S16Generic(BORDER_MODE _borderType, const srcType _borderValue, ptrdiff_t borderxl, ptrdiff_t borderxr, const s16 *w): + RowFilter3x3S16Base(_borderType, _borderValue, borderxl, borderxr), borderFilter( (w[0]+w[1]+w[2]) * borderValue ) + { + vw0 = vdupq_n_s16(w[0]); + vw1 = vdupq_n_s16(w[1]); + vw2 = vdupq_n_s16(w[2]); + } + + int16x8_t vw0; + int16x8_t vw1; + int16x8_t vw2; + const dstType borderFilter; + + inline void operator()(const u8* src, s16* dst, ptrdiff_t width) + { + uint8x8_t l = vtbl1_u8(vld1_u8(src - lookLeft), vfmask); + if (lookLeft == 0 && borderType == BORDER_MODE_CONSTANT) + l = vset_lane_u8(borderValue, l, 6); + + ptrdiff_t i = 0; + for (; i < width - 16 + lookRight; i += 16) + { + internal::prefetch(src + i); + uint8x8_t l18u = vld1_u8(src + i + 1); + vst1q_s16(dst + i, vaddq_s16(vmlaq_s16(vmulq_s16(vreinterpretq_s16_u16(vmovl_u8(vext_u8(l, l18u, 6))), vw0), + vreinterpretq_s16_u16(vmovl_u8(vext_u8(l, l18u, 7))), vw1), + vmulq_s16(vreinterpretq_s16_u16(vmovl_u8(l18u)), vw2))); + l = vld1_u8(src + i + 9); + vst1q_s16(dst + i + 8, vaddq_s16(vmlaq_s16(vmulq_s16(vreinterpretq_s16_u16(vmovl_u8(vext_u8(l18u, l, 6))), vw0), + vreinterpretq_s16_u16(vmovl_u8(vext_u8(l18u, l, 7))), vw1), + vmulq_s16(vreinterpretq_s16_u16(vmovl_u8(l)), vw2))); + } + if (i < width - 8 + lookRight) + { + uint8x8_t l18u = vld1_u8(src + i + 1); + vst1q_s16(dst + i, vaddq_s16(vmlaq_s16(vmulq_s16(vreinterpretq_s16_u16(vmovl_u8(vext_u8(l, l18u, 6))), vw0), + vreinterpretq_s16_u16(vmovl_u8(vext_u8(l, l18u, 7))), vw1), + vmulq_s16(vreinterpretq_s16_u16(vmovl_u8(l18u)), vw2))); + i += 8; + } + + //tail + if (lookRight == 0 || i != width) + { + uint8x8_t tail0 = vld1_u8(src + (width - 9));//can't get left 1 pixel another way if width==8*k+1 + uint8x8_t tail2 = vtbl1_u8(vld1_u8(src + (width - 8 + lookRight)), vtmask); + if (lookLeft == 0 && borderType == BORDER_MODE_CONSTANT) + tail2 = vset_lane_u8(borderValue, tail2, 7); + uint8x8_t tail1 = vext_u8(vreinterpret_u8_u64(vshl_n_u64(vreinterpret_u64_u8(tail0), 8*6)), tail2, 7); + + int16x8_t l0 = vreinterpretq_s16_u16(vmovl_u8(tail0)); + int16x8_t l1 = vreinterpretq_s16_u16(vmovl_u8(tail1)); + int16x8_t l2 = vreinterpretq_s16_u16(vmovl_u8(tail2)); + + int16x8_t l0w = vmulq_s16(l0, vw0); + int16x8_t l2w = vmulq_s16(l2, vw2); + int16x8_t ls = vaddq_s16(vmlaq_s16(l0w, l1, vw1), l2w); + + vst1q_s16(dst + (width - 8), ls); + } + } +}; + +struct RowFilter3x3S16_m101 : public RowFilter3x3S16Base +{ + typedef s16 dstType; + + inline RowFilter3x3S16_m101(const BORDER_MODE _borderType, const srcType _borderValue, ptrdiff_t borderxl, ptrdiff_t borderxr, const s16*): + RowFilter3x3S16Base(_borderType, _borderValue, borderxl, borderxr), borderFilter(0) {} + + const dstType borderFilter; + + inline void operator()(const u8* src, s16* dst, ptrdiff_t width) + { + uint8x8_t l = vtbl1_u8(vld1_u8(src - lookLeft), vfmask); + if (lookLeft == 0 && borderType == BORDER_MODE_CONSTANT) + l = vset_lane_u8(borderValue, l, 6); + + ptrdiff_t i = 0; + for (; i < width - 16 + lookRight; i += 16) + { + internal::prefetch(src + i); + + uint8x8_t l2 = vld1_u8(src + i + 1); + vst1q_s16(dst + i, vreinterpretq_s16_u16(vsubl_u8(l2, vext_u8(l, l2, 6)))); + + l = vld1_u8(src + i + 9); + vst1q_s16(dst + i + 8, vreinterpretq_s16_u16(vsubl_u8(l, vext_u8(l2, l, 6)))); + } + + if (i < width - 8 + lookRight) + { + uint8x8_t l2 = vld1_u8(src + i + 1); + vst1q_s16(dst + i, vreinterpretq_s16_u16(vsubl_u8(l2, vext_u8(l, l2, 6)))); + i += 8; + } + + //tail + if (lookRight == 0 || i != width) + { + uint8x8_t tail0 = vld1_u8(src + (width - 9));//can't get left 1 pixel another way if width==8*k+1 + uint8x8_t tail2 = vtbl1_u8(vld1_u8(src + (width - 8 + lookRight)), vtmask); + if (lookLeft == 0 && borderType == BORDER_MODE_CONSTANT) + tail2 = vset_lane_u8(borderValue, tail2, 7); + + int16x8_t ls = vreinterpretq_s16_u16(vsubl_u8(tail2, tail0)); + + vst1q_s16(dst + (width - 8), ls); + } + } +}; + +struct RowFilter3x3S16_121 : public RowFilter3x3S16Base +{ + typedef s16 dstType; + + inline RowFilter3x3S16_121(const BORDER_MODE _borderType, const srcType _borderValue, ptrdiff_t borderxl, ptrdiff_t borderxr, const s16*): + RowFilter3x3S16Base(_borderType, _borderValue, borderxl, borderxr), borderFilter(borderValue << 2) {} + + const dstType borderFilter; + + inline void operator()(const u8* src, s16* dst, ptrdiff_t width) + { + uint8x8_t l = vtbl1_u8(vld1_u8(src - lookLeft), vfmask); + if (lookLeft == 0 && borderType == BORDER_MODE_CONSTANT) + l = vset_lane_u8(borderValue, l, 6); + + ptrdiff_t i = 0; + for (; i < width - 16 + lookRight; i += 16) + { + internal::prefetch(src + i); + + uint8x8_t l2 = vld1_u8(src + i + 1); + vst1q_s16(dst + i, vqaddq_s16(vreinterpretq_s16_u16(vaddl_u8(vext_u8(l, l2, 6), l2)), + vreinterpretq_s16_u16(vshll_n_u8(vext_u8(l, l2, 7), 1)))); + + l = vld1_u8(src + i + 9); + vst1q_s16(dst + i + 8, vqaddq_s16(vreinterpretq_s16_u16(vaddl_u8(vext_u8(l2, l, 6), l)), + vreinterpretq_s16_u16(vshll_n_u8(vext_u8(l2, l, 7), 1)))); + } + + if (i < width - 8 + lookRight) + { + uint8x8_t l2 = vld1_u8(src + i + 1); + vst1q_s16(dst + i, vqaddq_s16(vreinterpretq_s16_u16(vaddl_u8(vext_u8(l, l2, 6), l2)), + vreinterpretq_s16_u16(vshll_n_u8(vext_u8(l, l2, 7), 1)))); + i += 8; + } + + //tail + if (lookRight == 0 || i != width) + { + uint8x8_t tail0 = vld1_u8(src + (width - 9));//can't get left 1 pixel another way if width==8*k+1 + uint8x8_t tail2 = vtbl1_u8(vld1_u8(src + (width - 8 + lookRight)), vtmask); + if (lookLeft == 0 && borderType == BORDER_MODE_CONSTANT) + tail2 = vset_lane_u8(borderValue, tail2, 7); + uint8x8_t tail1 = vext_u8(vreinterpret_u8_u64(vshl_n_u64(vreinterpret_u64_u8(tail0), 8*6)), tail2, 7); + + int16x8_t tail02 = vreinterpretq_s16_u16(vaddl_u8(tail0, tail2)); + int16x8_t tail1x2 = vreinterpretq_s16_u16(vshll_n_u8(tail1, 1)); + + int16x8_t ls = vqaddq_s16(tail02, tail1x2); + + vst1q_s16(dst + (width - 8), ls); + } + } +}; + +struct RowFilter3x3S16_1m21 : public RowFilter3x3S16Base +{ + typedef s16 dstType; + + inline RowFilter3x3S16_1m21(const BORDER_MODE _borderType, const srcType _borderValue, ptrdiff_t borderxl, ptrdiff_t borderxr, const s16*): + RowFilter3x3S16Base(_borderType, _borderValue, borderxl, borderxr), borderFilter(0) {} + + const dstType borderFilter; + + inline void operator()(const u8* src, s16* dst, ptrdiff_t width) + { + uint8x8_t l = vtbl1_u8(vld1_u8(src - lookLeft), vfmask); + if (lookLeft == 0 && borderType == BORDER_MODE_CONSTANT) + l = vset_lane_u8(borderValue, l, 6); + + ptrdiff_t i = 0; + for (; i < width - 16 + lookRight; i += 16) + { + internal::prefetch(src + i); + + uint8x8_t l2 = vld1_u8(src + i + 1); + vst1q_s16(dst + i, vqsubq_s16(vreinterpretq_s16_u16(vaddl_u8(vext_u8(l, l2, 6), l2)), + vreinterpretq_s16_u16(vshll_n_u8(vext_u8(l, l2, 7), 1)))); + + l = vld1_u8(src + i + 9); + vst1q_s16(dst + i + 8, vqsubq_s16(vreinterpretq_s16_u16(vaddl_u8(vext_u8(l2, l, 6), l)), + vreinterpretq_s16_u16(vshll_n_u8(vext_u8(l2, l, 7), 1)))); + } + + if (i < width - 8 + lookRight) + { + uint8x8_t l2 = vld1_u8(src + i + 1); + vst1q_s16(dst + i, vqsubq_s16(vreinterpretq_s16_u16(vaddl_u8(vext_u8(l, l2, 6), l2)), + vreinterpretq_s16_u16(vshll_n_u8(vext_u8(l, l2, 7), 1)))); + i += 8; + } + + //tail + if (lookRight == 0 || i != width) + { + uint8x8_t tail0 = vld1_u8(src + (width - 9));//can't get left 1 pixel another way if width==8*k+1 + uint8x8_t tail2 = vtbl1_u8(vld1_u8(src + (width - 8 + lookRight)), vtmask); + if (lookLeft == 0 && borderType == BORDER_MODE_CONSTANT) + tail2 = vset_lane_u8(borderValue, tail2, 7); + uint8x8_t tail1 = vext_u8(vreinterpret_u8_u64(vshl_n_u64(vreinterpret_u64_u8(tail0), 8*6)), tail2, 7); + + int16x8_t tail02 = vreinterpretq_s16_u16(vaddl_u8(tail0, tail2)); + int16x8_t tail1x2 = vreinterpretq_s16_u16(vshll_n_u8(tail1, 1)); + + int16x8_t ls = vqsubq_s16(tail02, tail1x2); + + vst1q_s16(dst + (width - 8), ls); + } + } +}; + +struct ColFilter3x3S16Generic : public ColFilter3x3S16Base +{ + typedef s16 dstType; + + inline ColFilter3x3S16Generic(const BORDER_MODE _borderType, const srcType _borderValue, const s16 *w): + ColFilter3x3S16Base(_borderType, _borderValue) + { + vw0 = vdupq_n_s16(w[0]); + vw1 = vdupq_n_s16(w[1]); + vw2 = vdupq_n_s16(w[2]); + } + + int16x8_t vw0; + int16x8_t vw1; + int16x8_t vw2; + + inline void operator()(const s16* src0, const s16* src1, const s16* src2, const s16* src3, s16* dst0, s16* dst1, ptrdiff_t width) + { + ptrdiff_t j = 0; + for (; j <= width - 16; j += 16) + { + int16x8_t line1 = vld1q_s16(src1 + j); + int16x8_t line2 = vld1q_s16(src2 + j); + vst1q_s16(dst0 + j, vmlaq_s16(vmlaq_s16(vmulq_s16(vld1q_s16(src0 + j), vw0), line1, vw1), line2, vw2)); + vst1q_s16(dst1 + j, vmlaq_s16(vmlaq_s16(vmulq_s16(vld1q_s16(src3 + j), vw2), line1, vw0), line2, vw1)); + + line1 = vld1q_s16(src1 + j + 8); + line2 = vld1q_s16(src2 + j + 8); + vst1q_s16(dst0 + j + 8, vmlaq_s16(vmlaq_s16(vmulq_s16(vld1q_s16(src0 + j + 8), vw0), line1, vw1), line2, vw2)); + vst1q_s16(dst1 + j + 8, vmlaq_s16(vmlaq_s16(vmulq_s16(vld1q_s16(src3 + j + 8), vw2), line1, vw0), line2, vw1)); + } + if (j <= width - 8) + { + int16x8_t line1 = vld1q_s16(src1 + j); + int16x8_t line2 = vld1q_s16(src2 + j); + vst1q_s16(dst0 + j, vmlaq_s16(vmlaq_s16(vmulq_s16(vld1q_s16(src0 + j), vw0), line1, vw1), line2, vw2)); + vst1q_s16(dst1 + j, vmlaq_s16(vmlaq_s16(vmulq_s16(vld1q_s16(src3 + j), vw2), line1, vw0), line2, vw1)); + j += 8; + } + if (j != width) + { + j = width - 8; + int16x8_t line1 = vld1q_s16(src1 + j); + int16x8_t line2 = vld1q_s16(src2 + j); + vst1q_s16(dst0 + j, vmlaq_s16(vmlaq_s16(vmulq_s16(vld1q_s16(src0 + j), vw0), line1, vw1), line2, vw2)); + vst1q_s16(dst1 + j, vmlaq_s16(vmlaq_s16(vmulq_s16(vld1q_s16(src3 + j), vw2), line1, vw0), line2, vw1)); + } + } + + inline void operator()(const s16* src0, const s16* src1, const s16* src2, s16* dst, ptrdiff_t width) + { + if (src0 == 0 || src2 == 0) + { + int16x8_t vwl1 = vw0; + int16x8_t vwl2 = vw2; + if (src2 == 0) + { + src2 = src0; + vwl1 = vw2; + vwl2 = vw0; + } + + int16x8_t v_border = vdupq_n_s16(0); + if (borderType == BORDER_MODE_CONSTANT) + { + v_border = vmulq_s16(vdupq_n_s16(borderValue), vwl1); + vwl1 = vw1; + } + else if (borderType == BORDER_MODE_REFLECT101) + { + vwl1 = vw1; + vwl2 = vaddq_s16(vw0, vw2); + } + else //replicate\reflect + vwl1 = vaddq_s16(vwl1, vw1); + + ptrdiff_t j = 0; + for (; j <= width - 16; j += 16) + { + vst1q_s16(dst + j, vaddq_s16(vmlaq_s16(v_border, vld1q_s16(src1 + j), vwl1), + vmulq_s16(vld1q_s16(src2 + j), vwl2))); + vst1q_s16(dst + j + 8, vaddq_s16(vmlaq_s16(v_border, vld1q_s16(src1 + j + 8), vwl1), + vmulq_s16(vld1q_s16(src2 + j + 8), vwl2))); + } + if (j <= width - 8) + { + vst1q_s16(dst + j, vaddq_s16(vmlaq_s16(v_border, vld1q_s16(src1 + j), vwl1), + vmulq_s16(vld1q_s16(src2 + j), vwl2))); + j += 8; + } + if (j != width) + { + j = width - 8; + vst1q_s16(dst + j, vaddq_s16(vmlaq_s16(v_border, vld1q_s16(src1 + j), vwl1), + vmulq_s16(vld1q_s16(src2 + j), vwl2))); + } + } + else + { + ptrdiff_t j = 0; + for (; j <= width - 16; j += 16) + { + vst1q_s16(dst + j, vmlaq_s16(vmlaq_s16(vmulq_s16(vld1q_s16(src0 + j), vw0), + vld1q_s16(src1 + j), vw1), + vld1q_s16(src2 + j), vw2)); + vst1q_s16(dst + j + 8, vmlaq_s16(vmlaq_s16(vmulq_s16(vld1q_s16(src0 + j + 8), vw0), + vld1q_s16(src1 + j + 8), vw1), + vld1q_s16(src2 + j + 8), vw2)); + } + if (j <= width - 8) + { + vst1q_s16(dst + j, vmlaq_s16(vmlaq_s16(vmulq_s16(vld1q_s16(src0 + j), vw0), + vld1q_s16(src1 + j), vw1), + vld1q_s16(src2 + j), vw2)); + j += 8; + } + if (j != width) + { + j = width - 8; + vst1q_s16(dst + j, vmlaq_s16(vmlaq_s16(vmulq_s16(vld1q_s16(src0 + j), vw0), + vld1q_s16(src1 + j), vw1), + vld1q_s16(src2 + j), vw2)); + } + } + } +}; + +struct ColFilter3x3S16_m101 : public ColFilter3x3S16Base +{ + typedef s16 dstType; + + inline ColFilter3x3S16_m101(const BORDER_MODE _borderType, const srcType _borderValue, const s16 *): + ColFilter3x3S16Base(_borderType, _borderValue) {} + + inline void operator()(const s16* src0, const s16* src1, const s16* src2, const s16* src3, s16* dst0, s16* dst1, ptrdiff_t width) + { + ptrdiff_t j = 0; + for (; j <= width - 16; j += 16) + { + vst1q_s16(dst0 + j, vqsubq_s16(vld1q_s16(src2 + j), vld1q_s16(src0 + j))); + vst1q_s16(dst1 + j, vqsubq_s16(vld1q_s16(src3 + j), vld1q_s16(src1 + j))); + vst1q_s16(dst0 + j + 8, vqsubq_s16(vld1q_s16(src2 + j + 8), vld1q_s16(src0 + j + 8))); + vst1q_s16(dst1 + j + 8, vqsubq_s16(vld1q_s16(src3 + j + 8), vld1q_s16(src1 + j + 8))); + } + if (j <= width - 8) + { + vst1q_s16(dst0 + j, vqsubq_s16(vld1q_s16(src2 + j), vld1q_s16(src0 + j))); + vst1q_s16(dst1 + j, vqsubq_s16(vld1q_s16(src3 + j), vld1q_s16(src1 + j))); + j += 8; + } + if (j != width) + { + j = width - 8; + vst1q_s16(dst0 + j, vqsubq_s16(vld1q_s16(src2 + j), vld1q_s16(src0 + j))); + vst1q_s16(dst1 + j, vqsubq_s16(vld1q_s16(src3 + j), vld1q_s16(src1 + j))); + } + } + + inline void operator()(const s16* src0, const s16* src1, const s16* src2, s16* dst, ptrdiff_t width) + { + if (src0 == 0 || src2 == 0) + { + if (borderType == BORDER_MODE_CONSTANT) + { + int16x8_t v_border = vdupq_n_s16(borderValue); + if (src0 == 0) + { + ptrdiff_t j = 0; + for (; j <= width - 16; j += 16) + { + vst1q_s16(dst + j, vqsubq_s16(vld1q_s16(src2 + j), v_border)); + vst1q_s16(dst + j + 8, vqsubq_s16(vld1q_s16(src2 + j + 8), v_border)); + } + if (j <= width - 8) + { + vst1q_s16(dst + j, vqsubq_s16(vld1q_s16(src2 + j), v_border)); + j += 8; + } + if (j != width) + { + j = width - 8; + vst1q_s16(dst + j, vqsubq_s16(vld1q_s16(src2 + j), v_border)); + } + } + else + { + ptrdiff_t j = 0; + for (; j <= width - 16; j += 16) + { + vst1q_s16(dst + j, vqsubq_s16(v_border, vld1q_s16(src0 + j))); + vst1q_s16(dst + j + 8, vqsubq_s16(v_border, vld1q_s16(src0 + j + 8))); + } + if (j <= width - 8) + { + vst1q_s16(dst + j, vqsubq_s16(v_border, vld1q_s16(src0 + j))); + j += 8; + } + if (j != width) + { + j = width - 8; + vst1q_s16(dst + j, vqsubq_s16(v_border, vld1q_s16(src0 + j))); + } + } + } + else if (borderType == BORDER_MODE_REFLECT101) + { + int16x8_t vzero = vmovq_n_s16(0); + ptrdiff_t j = 0; + for (; j <= width - 16; j += 16) + { + vst1q_s16(dst + j, vzero); + vst1q_s16(dst + j + 8, vzero); + } + if (j <= width - 8) + { + vst1q_s16(dst + j, vzero); + j += 8; + } + if (j != width) + { + j = width - 8; + vst1q_s16(dst + j, vzero); + } + } + else //replicate\reflect + { + if (src0 == 0) src0 = src1; else src2 = src1; + ptrdiff_t j = 0; + for (; j <= width - 16; j += 16) + { + vst1q_s16(dst + j, vqsubq_s16(vld1q_s16(src2 + j), vld1q_s16(src0 + j))); + vst1q_s16(dst + j + 8, vqsubq_s16(vld1q_s16(src2 + j + 8), vld1q_s16(src0 + j + 8))); + } + if (j <= width - 8) + { + vst1q_s16(dst + j, vqsubq_s16(vld1q_s16(src2 + j), vld1q_s16(src0 + j))); + j += 8; + } + if (j != width) + { + j = width - 8; + vst1q_s16(dst + j, vqsubq_s16(vld1q_s16(src2 + j), vld1q_s16(src0 + j))); + } + } + } + else + { + ptrdiff_t j = 0; + for (; j <= width - 16; j += 16) + { + vst1q_s16(dst + j, vqsubq_s16(vld1q_s16(src2 + j), vld1q_s16(src0 + j))); + vst1q_s16(dst + j + 8, vqsubq_s16(vld1q_s16(src2 + j + 8), vld1q_s16(src0 + j + 8))); + } + if (j <= width - 8) + { + vst1q_s16(dst + j, vqsubq_s16(vld1q_s16(src2 + j), vld1q_s16(src0 + j))); + j += 8; + } + if (j != width) + { + j = width - 8; + vst1q_s16(dst + j, vqsubq_s16(vld1q_s16(src2 + j), vld1q_s16(src0 + j))); + } + } + } +}; + +struct ColFilter3x3S16_121 : public ColFilter3x3S16Base +{ + typedef s16 dstType; + + inline ColFilter3x3S16_121(const BORDER_MODE _borderType, const srcType _borderValue, const s16*): + ColFilter3x3S16Base(_borderType, _borderValue) {} + + inline void operator()(const s16* src0, const s16* src1, const s16* src2, const s16* src3, s16* dst0, s16* dst1, ptrdiff_t width) + { + ptrdiff_t j = 0; + //int16x8_t line0 = vld1q_s16(src0 + j);//1 + //int16x8_t line1 = vld1q_s16(src1 + j);//11 + //int16x8_t line2 = vld1q_s16(src2 + j);// 11 + //int16x8_t line3 = vld1q_s16(src3 + j);// 1 + for (; j <= width - 16; j += 16) + { + int16x8_t line1 = vld1q_s16(src1 + j); + int16x8_t line2 = vld1q_s16(src2 + j); + + int16x8_t l12 = vqaddq_s16(line1, line2); + + vst1q_s16(dst0 + j, vqaddq_s16(vqaddq_s16(vld1q_s16(src0 + j), line1), l12)); + vst1q_s16(dst1 + j, vqaddq_s16(l12, vqaddq_s16(line2, vld1q_s16(src3 + j)))); + + line1 = vld1q_s16(src1 + j + 8); + line2 = vld1q_s16(src2 + j + 8); + + l12 = vqaddq_s16(line1, line2); + + vst1q_s16(dst0 + j + 8, vqaddq_s16(vqaddq_s16(vld1q_s16(src0 + j + 8), line1), l12)); + vst1q_s16(dst1 + j + 8, vqaddq_s16(l12, vqaddq_s16(line2, vld1q_s16(src3 + j + 8)))); + } + if (j <= width - 8) + { + int16x8_t line1 = vld1q_s16(src1 + j); + int16x8_t line2 = vld1q_s16(src2 + j); + + int16x8_t l12 = vqaddq_s16(line1, line2); + + vst1q_s16(dst0 + j, vqaddq_s16(vqaddq_s16(vld1q_s16(src0 + j), line1), l12)); + vst1q_s16(dst1 + j, vqaddq_s16(l12, vqaddq_s16(line2, vld1q_s16(src3 + j)))); + j += 8; + } + if (j != width) + { + j = width - 8; + int16x8_t line1 = vld1q_s16(src1 + j); + int16x8_t line2 = vld1q_s16(src2 + j); + + int16x8_t l12 = vqaddq_s16(line1, line2); + + vst1q_s16(dst0 + j, vqaddq_s16(vqaddq_s16(vld1q_s16(src0 + j), line1), l12)); + vst1q_s16(dst1 + j, vqaddq_s16(l12, vqaddq_s16(line2, vld1q_s16(src3 + j)))); + } + } + + inline void operator()(const s16* src0, const s16* src1, const s16* src2, s16* dst, ptrdiff_t width) + { + if (src0 == 0 || src2 == 0) + { + if (src2 == 0) + src2 = src0; + + if (borderType == BORDER_MODE_CONSTANT) + { + int16x8_t v_border = vdupq_n_s16(borderValue); + ptrdiff_t j = 0; + for (; j <= width - 16; j += 16) + { + vst1q_s16(dst + j, vqaddq_s16(vqshlq_n_s16(vld1q_s16(src1 + j), 1), + vqaddq_s16(v_border, vld1q_s16(src2 + j)))); + vst1q_s16(dst + j + 8, vqaddq_s16(vqshlq_n_s16(vld1q_s16(src1 + j + 8), 1), + vqaddq_s16(v_border, vld1q_s16(src2 + j + 8)))); + } + if (j <= width - 8) + { + vst1q_s16(dst + j, vqaddq_s16(vqshlq_n_s16(vld1q_s16(src1 + j), 1), + vqaddq_s16(v_border, vld1q_s16(src2 + j)))); + j += 8; + } + if (j != width) + { + j = width - 8; + vst1q_s16(dst + j, vqaddq_s16(vqshlq_n_s16(vld1q_s16(src1 + j), 1), + vqaddq_s16(v_border, vld1q_s16(src2 + j)))); + } + } + else if (borderType == BORDER_MODE_REFLECT101) + { + ptrdiff_t j = 0; + for (; j <= width - 16; j += 16) + { + vst1q_s16(dst + j, vqshlq_n_s16(vqaddq_s16(vld1q_s16(src1 + j), + vld1q_s16(src2 + j)), 1)); + vst1q_s16(dst + j + 8, vqshlq_n_s16(vqaddq_s16(vld1q_s16(src1 + j + 8), + vld1q_s16(src2 + j + 8)), 1)); + } + if (j <= width - 8) + { + vst1q_s16(dst + j, vqshlq_n_s16(vqaddq_s16(vld1q_s16(src1 + j), + vld1q_s16(src2 + j)), 1)); + j += 8; + } + if (j != width) + { + j = width - 8; + vst1q_s16(dst + j, vqshlq_n_s16(vqaddq_s16(vld1q_s16(src1 + j), + vld1q_s16(src2 + j)), 1)); + } + } + else //replicate\reflect + { + ptrdiff_t j = 0; + for (; j <= width - 16; j += 16) + { + int16x8_t line1 = vld1q_s16(src1 + j); + vst1q_s16(dst + j, vqaddq_s16(vqshlq_n_s16(line1, 1), + vqaddq_s16(line1, vld1q_s16(src2 + j)))); + + line1 = vld1q_s16(src1 + j + 8); + vst1q_s16(dst + j + 8, vqaddq_s16(vqshlq_n_s16(line1, 1), + vqaddq_s16(line1, vld1q_s16(src2 + j + 8)))); + } + if (j <= width - 8) + { + int16x8_t line1 = vld1q_s16(src1 + j); + vst1q_s16(dst + j, vqaddq_s16(vqshlq_n_s16(line1, 1), + vqaddq_s16(line1, vld1q_s16(src2 + j)))); + j += 8; + } + if (j != width) + { + j = width - 8; + int16x8_t line1 = vld1q_s16(src1 + j); + vst1q_s16(dst + j, vqaddq_s16(vqshlq_n_s16(line1, 1), + vqaddq_s16(line1, vld1q_s16(src2 + j)))); + } + } + } + else + { + ptrdiff_t j = 0; + for (; j <= width - 16; j += 16) + { + vst1q_s16(dst + j, vqaddq_s16(vqshlq_n_s16(vld1q_s16(src1 + j), 1), + vqaddq_s16(vld1q_s16(src0 + j), vld1q_s16(src2 + j)))); + + vst1q_s16(dst + j + 8, vqaddq_s16(vqshlq_n_s16(vld1q_s16(src1 + j + 8), 1), + vqaddq_s16(vld1q_s16(src0 + j + 8), vld1q_s16(src2 + j + 8)))); + } + if (j <= width - 8) + { + vst1q_s16(dst + j, vqaddq_s16(vqshlq_n_s16(vld1q_s16(src1 + j), 1), + vqaddq_s16(vld1q_s16(src0 + j), vld1q_s16(src2 + j)))); + j += 8; + } + if (j != width) + { + j = width - 8; + vst1q_s16(dst + j, vqaddq_s16(vqshlq_n_s16(vld1q_s16(src1 + j), 1), + vqaddq_s16(vld1q_s16(src0 + j), vld1q_s16(src2 + j)))); + } + } + } +}; + +struct ColFilter3x3U8_121 : public ColFilter3x3S16Base +{ + typedef u8 dstType; + + inline ColFilter3x3U8_121(const BORDER_MODE _borderType, const srcType _borderValue, const s16*): + ColFilter3x3S16Base(_borderType, _borderValue) {} + + inline void operator()(const srcType* src0, const srcType* src1, const srcType* src2, const srcType* src3, dstType* dst0, dstType* dst1, ptrdiff_t width) + { + ptrdiff_t j = 0; + //int16x8_t line0 = vld1q_s16(src0 + j);//1 + //int16x8_t line1 = vld1q_s16(src1 + j);//11 + //int16x8_t line2 = vld1q_s16(src2 + j);// 11 + //int16x8_t line3 = vld1q_s16(src3 + j);// 1 + for (; j <= width - 16; j += 16) + { + int16x8_t line1 = vld1q_s16(src1 + j); + int16x8_t line2 = vld1q_s16(src2 + j); + + int16x8_t l12 = vaddq_s16(line1, line2); + + vst1_u8(dst0 + j, vqrshrun_n_s16(vaddq_s16(vaddq_s16(vld1q_s16(src0 + j), line1), l12), 4)); + vst1_u8(dst1 + j, vqrshrun_n_s16(vaddq_s16(l12, vaddq_s16(line2, vld1q_s16(src3 + j))), 4)); + + line1 = vld1q_s16(src1 + j + 8); + line2 = vld1q_s16(src2 + j + 8); + + l12 = vaddq_s16(line1, line2); + + vst1_u8(dst0 + j + 8, vqrshrun_n_s16(vaddq_s16(vaddq_s16(vld1q_s16(src0 + j + 8), line1), l12), 4)); + vst1_u8(dst1 + j + 8, vqrshrun_n_s16(vaddq_s16(l12, vaddq_s16(line2, vld1q_s16(src3 + j + 8))), 4)); + } + if (j <= width - 8) + { + int16x8_t line1 = vld1q_s16(src1 + j); + int16x8_t line2 = vld1q_s16(src2 + j); + + int16x8_t l12 = vaddq_s16(line1, line2); + + vst1_u8(dst0 + j, vqrshrun_n_s16(vaddq_s16(vaddq_s16(vld1q_s16(src0 + j), line1), l12), 4)); + vst1_u8(dst1 + j, vqrshrun_n_s16(vaddq_s16(l12, vaddq_s16(line2, vld1q_s16(src3 + j))), 4)); + j += 8; + } + if (j != width) + { + j = width - 8; + int16x8_t line1 = vld1q_s16(src1 + j); + int16x8_t line2 = vld1q_s16(src2 + j); + + int16x8_t l12 = vaddq_s16(line1, line2); + + vst1_u8(dst0 + j, vqrshrun_n_s16(vaddq_s16(vaddq_s16(vld1q_s16(src0 + j), line1), l12), 4)); + vst1_u8(dst1 + j, vqrshrun_n_s16(vaddq_s16(l12, vaddq_s16(line2, vld1q_s16(src3 + j))), 4)); + } + } + + inline void operator()(const srcType* src0, const srcType* src1, const srcType* src2, dstType* dst, ptrdiff_t width) + { + if (src0 == 0 || src2 == 0) + { + if (src2 == 0) + src2 = src0; + + if (borderType == BORDER_MODE_CONSTANT) + { + ptrdiff_t j = 0; + int16x8_t v_border = vdupq_n_s16(borderValue); + for (; j <= width - 16; j += 16) + { + //Store normalized result, essential for gaussianBlur + vst1_u8(dst + j, vqrshrun_n_s16(vaddq_s16(vshlq_n_s16(vld1q_s16(src1 + j), 1), + vaddq_s16(v_border, vld1q_s16(src2 + j))), 4)); + + vst1_u8(dst + j + 8, vqrshrun_n_s16(vaddq_s16(vshlq_n_s16(vld1q_s16(src1 + j + 8), 1), + vaddq_s16(v_border, vld1q_s16(src2 + j + 8))), 4)); + } + if (j <= width - 8) + { + vst1_u8(dst + j, vqrshrun_n_s16(vaddq_s16(vshlq_n_s16(vld1q_s16(src1 + j), 1), + vaddq_s16(v_border, vld1q_s16(src2 + j))), 4)); + j += 8; + } + if (j != width) + { + j = width - 8; + vst1_u8(dst + j, vqrshrun_n_s16(vaddq_s16(vshlq_n_s16(vld1q_s16(src1 + j), 1), + vaddq_s16(v_border, vld1q_s16(src2 + j))), 4)); + } + } + else if (borderType == BORDER_MODE_REFLECT101) + { + ptrdiff_t j = 0; + for (; j <= width - 16; j += 16) + { + vst1_u8(dst + j, vqrshrun_n_s16(vshlq_n_s16(vaddq_s16(vld1q_s16(src1 + j), + vld1q_s16(src2 + j)), 1), 4)); + vst1_u8(dst + j + 8, vqrshrun_n_s16(vshlq_n_s16(vaddq_s16(vld1q_s16(src1 + j + 8), + vld1q_s16(src2 + j + 8)), 1), 4)); + } + if (j <= width - 8) + { + vst1_u8(dst + j, vqrshrun_n_s16(vshlq_n_s16(vaddq_s16(vld1q_s16(src1 + j), + vld1q_s16(src2 + j)), 1), 4)); + j += 8; + } + if (j != width) + { + j = width - 8; + vst1_u8(dst + j, vqrshrun_n_s16(vshlq_n_s16(vaddq_s16(vld1q_s16(src1 + j), + vld1q_s16(src2 + j)), 1), 4)); + } + } + else //replicate\reflect + { + ptrdiff_t j = 0; + for (; j <= width - 16; j += 16) + { + int16x8_t line1 = vld1q_s16(src1 + j); + vst1_u8(dst + j, vqrshrun_n_s16(vaddq_s16(vshlq_n_s16(line1, 1), + vaddq_s16(line1, vld1q_s16(src2 + j))), 4)); + + line1 = vld1q_s16(src1 + j + 8); + vst1_u8(dst + j + 8, vqrshrun_n_s16(vaddq_s16(vshlq_n_s16(line1, 1), + vaddq_s16(line1, vld1q_s16(src2 + j + 8))), 4)); + } + if (j <= width - 8) + { + int16x8_t line1 = vld1q_s16(src1 + j); + vst1_u8(dst + j, vqrshrun_n_s16(vaddq_s16(vshlq_n_s16(line1, 1), + vaddq_s16(line1, vld1q_s16(src2 + j))), 4)); + j += 8; + } + if (j != width) + { + j = width - 8; + int16x8_t line1 = vld1q_s16(src1 + j); + vst1_u8(dst + j, vqrshrun_n_s16(vaddq_s16(vshlq_n_s16(line1, 1), + vaddq_s16(line1, vld1q_s16(src2 + j))), 4)); + } + } + } + else + { + ptrdiff_t j = 0; + for (; j <= width - 16; j += 16) + { + vst1_u8(dst + j, vqrshrun_n_s16(vaddq_s16(vshlq_n_s16(vld1q_s16(src1 + j), 1), + vaddq_s16(vld1q_s16(src0 + j), vld1q_s16(src2 + j))), 4)); + vst1_u8(dst + j + 8, vqrshrun_n_s16(vaddq_s16(vshlq_n_s16(vld1q_s16(src1 + j + 8), 1), + vaddq_s16(vld1q_s16(src0 + j + 8), vld1q_s16(src2 + j + 8))), 4)); + } + if (j <= width - 8) + { + vst1_u8(dst + j, vqrshrun_n_s16(vaddq_s16(vshlq_n_s16(vld1q_s16(src1 + j), 1), + vaddq_s16(vld1q_s16(src0 + j), vld1q_s16(src2 + j))), 4)); + j += 8; + } + if (j != width) + { + j = width - 8; + vst1_u8(dst + j, vqrshrun_n_s16(vaddq_s16(vshlq_n_s16(vld1q_s16(src1 + j), 1), + vaddq_s16(vld1q_s16(src0 + j), vld1q_s16(src2 + j))), 4)); + } + } + } +}; + +struct ColFilter3x3S16_1m21 : public ColFilter3x3S16Base +{ + typedef s16 dstType; + + inline ColFilter3x3S16_1m21(const BORDER_MODE _borderType, const srcType _borderValue, const s16*): + ColFilter3x3S16Base(_borderType, _borderValue) {} + + inline void operator()(const s16* src0, const s16* src1, const s16* src2, const s16* src3, s16* dst0, s16* dst1, ptrdiff_t width) + { + ptrdiff_t j = 0; + //int16x8_t line0 = vld1q_s16(src0 + j);// 1 + //int16x8_t line1 = vld1q_s16(src1 + j);//-1 1 + //int16x8_t line2 = vld1q_s16(src2 + j);// -1 -1 + //int16x8_t line3 = vld1q_s16(src3 + j);// 1 + for (; j <= width - 16; j += 16) + { + int16x8_t line1 = vld1q_s16(src1 + j); + int16x8_t line2 = vld1q_s16(src2 + j); + + int16x8_t l12 = vqsubq_s16(line1, line2); + + vst1q_s16(dst0 + j, vqsubq_s16(vqsubq_s16(vld1q_s16(src0 + j), line1), l12)); + vst1q_s16(dst1 + j, vqaddq_s16(vqsubq_s16(vld1q_s16(src3 + j), line2), l12)); + + line1 = vld1q_s16(src1 + j + 8); + line2 = vld1q_s16(src2 + j + 8); + + l12 = vqsubq_s16(line1, line2); + + vst1q_s16(dst0 + j + 8, vqsubq_s16(vqsubq_s16(vld1q_s16(src0 + j + 8), line1), l12)); + vst1q_s16(dst1 + j + 8, vqaddq_s16(vqsubq_s16(vld1q_s16(src3 + j + 8), line2), l12)); + } + if (j <= width - 8) + { + int16x8_t line1 = vld1q_s16(src1 + j); + int16x8_t line2 = vld1q_s16(src2 + j); + + int16x8_t l12 = vqsubq_s16(line1, line2); + + vst1q_s16(dst0 + j, vqsubq_s16(vqsubq_s16(vld1q_s16(src0 + j), line1), l12)); + vst1q_s16(dst1 + j, vqaddq_s16(vqsubq_s16(vld1q_s16(src3 + j), line2), l12)); + j += 8; + } + if (j != width) + { + j = width - 8; + int16x8_t line1 = vld1q_s16(src1 + j); + int16x8_t line2 = vld1q_s16(src2 + j); + + int16x8_t l12 = vqsubq_s16(line1, line2); + + vst1q_s16(dst0 + j, vqsubq_s16(vqsubq_s16(vld1q_s16(src0 + j), line1), l12)); + vst1q_s16(dst1 + j, vqaddq_s16(vqsubq_s16(vld1q_s16(src3 + j), line2), l12)); + } + } + + inline void operator()(const s16* src0, const s16* src1, const s16* src2, s16* dst, ptrdiff_t width) + { + if (src0 == 0 || src2 == 0) + { + if (src2 == 0) + src2 = src0; + + if (borderType == BORDER_MODE_CONSTANT) + { + ptrdiff_t j = 0; + int16x8_t v_border = vdupq_n_s16(borderValue); + for (; j <= width - 16; j += 16) + { + vst1q_s16(dst + j, vqsubq_s16(vqaddq_s16(v_border, vld1q_s16(src2 + j)), vshlq_n_s16(vld1q_s16(src1 + j), 1))); + vst1q_s16(dst + j + 8, vqsubq_s16(vqaddq_s16(v_border, vld1q_s16(src2 + j + 8)), vshlq_n_s16(vld1q_s16(src1 + j + 8), 1))); + } + if (j <= width - 8) + { + vst1q_s16(dst + j, vqsubq_s16(vqaddq_s16(v_border, vld1q_s16(src2 + j)), vshlq_n_s16(vld1q_s16(src1 + j), 1))); + j += 8; + } + if (j != width) + { + j = width - 8; + vst1q_s16(dst + j, vqsubq_s16(vqaddq_s16(v_border, vld1q_s16(src2 + j)), vshlq_n_s16(vld1q_s16(src1 + j), 1))); + } + } + else if (borderType == BORDER_MODE_REFLECT101) + { + ptrdiff_t j = 0; + for (; j <= width - 16; j += 16) + { + vst1q_s16(dst + j, vqshlq_n_s16(vqsubq_s16(vld1q_s16(src2 + j), vld1q_s16(src1 + j)), 1)); + vst1q_s16(dst + j + 8, vqshlq_n_s16(vqsubq_s16(vld1q_s16(src2 + j + 8), vld1q_s16(src1 + j + 8)), 1)); + } + if (j <= width - 8) + { + vst1q_s16(dst + j, vqshlq_n_s16(vqsubq_s16(vld1q_s16(src2 + j), vld1q_s16(src1 + j)), 1)); + j += 8; + } + if (j != width) + { + j = width - 8; + vst1q_s16(dst + j, vqshlq_n_s16(vqsubq_s16(vld1q_s16(src2 + j), vld1q_s16(src1 + j)), 1)); + } + } + else //replicate\reflect + { + ptrdiff_t j = 0; + for (; j <= width - 16; j += 16) + { + vst1q_s16(dst + j, vqsubq_s16(vld1q_s16(src2 + j), vld1q_s16(src1 + j))); + vst1q_s16(dst + j + 8, vqsubq_s16(vld1q_s16(src2 + j + 8), vld1q_s16(src1 + j + 8))); + } + if (j <= width - 8) + { + vst1q_s16(dst + j, vqsubq_s16(vld1q_s16(src2 + j), vld1q_s16(src1 + j))); + j += 8; + } + if (j != width) + { + j = width - 8; + vst1q_s16(dst + j, vqsubq_s16(vld1q_s16(src2 + j), vld1q_s16(src1 + j))); + } + } + } + else + { + ptrdiff_t j = 0; + for (; j <= width - 16; j += 16) + { + vst1q_s16(dst + j, vqsubq_s16(vqaddq_s16(vld1q_s16(src0 + j), vld1q_s16(src2 + j)), + vqshlq_n_s16(vld1q_s16(src1 + j), 1))); + vst1q_s16(dst + j + 8, vqsubq_s16(vqaddq_s16(vld1q_s16(src0 + j + 8), vld1q_s16(src2 + j + 8)), + vqshlq_n_s16(vld1q_s16(src1 + j + 8), 1))); + } + if (j <= width - 8) + { + vst1q_s16(dst + j, vqsubq_s16(vqaddq_s16(vld1q_s16(src0 + j), vld1q_s16(src2 + j)), + vqshlq_n_s16(vld1q_s16(src1 + j), 1))); + j += 8; + } + if (j != width) + { + j = width - 8; + vst1q_s16(dst + j, vqsubq_s16(vqaddq_s16(vld1q_s16(src0 + j), vld1q_s16(src2 + j)), + vqshlq_n_s16(vld1q_s16(src1 + j), 1))); + } + } + } +}; + +template struct sepFilter3x3 +{ + typedef typename RowFilter::srcType srcType; + typedef typename RowFilter::dstType tmpType; + typedef typename ColFilter::dstType dstType; + + static void process(const Size2D &ssize, + const srcType * srcBase, ptrdiff_t srcStride, + dstType * dstBase, ptrdiff_t dstStride, + const s16 *xw, const s16 *yw, + BORDER_MODE borderType, srcType borderValue, Margin borderMargin) + { + const ptrdiff_t offsetk = 1; + ptrdiff_t borderxl, borderxr, borderyt, borderyb; + borderxl = std::max(0, offsetk - (ptrdiff_t)borderMargin.left); + borderyt = std::max(0, offsetk - (ptrdiff_t)borderMargin.top); + borderxr = std::max(0, offsetk - (ptrdiff_t)borderMargin.right); + borderyb = std::max(0, offsetk - (ptrdiff_t)borderMargin.bottom); + + std::vector _buf(ssize.width << 2); + tmpType * buf = &_buf[0]; + + RowFilter filterX(borderType, borderValue, borderxl, borderxr, xw); + ColFilter filterY(borderType, filterX.borderFilter, yw); + const ptrdiff_t lookTop = offsetk - borderyt; + const ptrdiff_t lookBottom = offsetk - borderyb; + + const srcType* src = srcBase - lookTop * srcStride / sizeof(srcType); + dstType* dst = dstBase; + + ptrdiff_t ridx = -lookTop; + for (; ridx <= (ptrdiff_t)ssize.height + lookBottom - 2; ridx += 2) + { + for (ptrdiff_t bidx = 0; bidx < 2; ++bidx, src += srcStride / sizeof(srcType)) + filterX(src, buf + ssize.width * ((4 + ridx + bidx) % 4), ssize.width); + + if (ridx <= 0) + { + if (ridx == 0) //first row + { + filterY(0, buf + ssize.width * ((ridx + 4) % 4), buf + ssize.width * ((ridx + 1) % 4), dst, ssize.width); + dst += dstStride / sizeof(dstType); + } + continue; + } + + filterY(buf + ssize.width * ((ridx + 2) % 4), + buf + ssize.width * ((ridx + 3) % 4), + buf + ssize.width * ((ridx + 4) % 4), + buf + ssize.width * ((ridx + 1) % 4), + dst, dst + dstStride / sizeof(dstType), ssize.width); + + dst += dstStride * 2 / sizeof(dstType); + } + + if (ridx < (ptrdiff_t)ssize.height + lookBottom) + { + filterX(src, buf + ssize.width * ((4 + ridx) % 4), ssize.width); + filterY(buf + ssize.width * ((2 + ridx) % 4), + buf + ssize.width * ((3 + ridx) % 4), + buf + ssize.width * ((4 + ridx) % 4), dst, ssize.width); + dst += dstStride / sizeof(dstType); + ridx++; + } + if (lookBottom == 0) + filterY(buf + ssize.width * ((ridx + 2) % 4), buf + ssize.width * ((ridx + 3) % 4), 0, dst, ssize.width); + } +}; + +} //namespace internal + +} //namespace CAROTENE_NS + +#endif // CAROTENE_NEON + +#endif // CAROTENE_SRC_REMAP_HPP diff --git a/3rdparty/carotene/src/sobel.cpp b/3rdparty/carotene/src/sobel.cpp new file mode 100644 index 0000000000..5d46045d9f --- /dev/null +++ b/3rdparty/carotene/src/sobel.cpp @@ -0,0 +1,317 @@ +/* + * By downloading, copying, installing or using the software you agree to this license. + * If you do not agree to this license, do not download, install, + * copy or use the software. + * + * + * License Agreement + * For Open Source Computer Vision Library + * (3-clause BSD License) + * + * Copyright (C) 2012-2015, NVIDIA Corporation, all rights reserved. + * Third party copyrights are property of their respective owners. + * + * Redistribution and use in source and binary forms, with or without modification, + * are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * * Neither the names of the copyright holders nor the names of the contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * This software is provided by the copyright holders and contributors "as is" and + * any express or implied warranties, including, but not limited to, the implied + * warranties of merchantability and fitness for a particular purpose are disclaimed. + * In no event shall copyright holders or contributors be liable for any direct, + * indirect, incidental, special, exemplary, or consequential damages + * (including, but not limited to, procurement of substitute goods or services; + * loss of use, data, or profits; or business interruption) however caused + * and on any theory of liability, whether in contract, strict liability, + * or tort (including negligence or otherwise) arising in any way out of + * the use of this software, even if advised of the possibility of such damage. + */ + +#include + +#include "common.hpp" + +namespace CAROTENE_NS { + +bool isSobel3x3Supported(const Size2D &size, BORDER_MODE border, + s32 dx, s32 dy, Margin borderMargin) +{ + return dx < 3 && dx >= 0 && + dy < 3 && dy >= 0 && + (dx + dy) > 0 && + isSeparableFilter3x3Supported(size, border, dx, dy, borderMargin); +} + +void Sobel3x3(const Size2D &size, + const u8 * srcBase, ptrdiff_t srcStride, + s16 * dstBase, ptrdiff_t dstStride, + s32 dx, s32 dy, + BORDER_MODE borderType, u8 borderValue, Margin borderMargin) +{ + internal::assertSupportedConfiguration(isSobel3x3Supported(size, borderType, dx, dy, borderMargin)); +#ifdef CAROTENE_NEON + SeparableFilter3x3(size, srcBase, srcStride, dstBase, dstStride, + dx, dy, 0, 0, + borderType, borderValue, borderMargin); +#else + (void)srcBase; + (void)srcStride; + (void)dstBase; + (void)dstStride; + (void)borderValue; +#endif +} + +bool isSobel3x3f32Supported(const Size2D &size, BORDER_MODE border, + s32 dx, s32 dy) +{ + return isSupportedConfiguration() && + dx < 3 && dx >= 0 && + dy < 3 && dy >= 0 && + (dx + dy) > 0 && + size.width >= 4 && size.height >= 2 && + (border == BORDER_MODE_CONSTANT || + border == BORDER_MODE_REFLECT || + border == BORDER_MODE_REFLECT101 || + border == BORDER_MODE_REPLICATE ); +} + +void Sobel3x3(const Size2D &size, + const f32 * srcBase, ptrdiff_t srcStride, + f32 * dstBase, ptrdiff_t dstStride, + s32 dx, s32 dy, + BORDER_MODE borderType, f32 borderValue) +{ + internal::assertSupportedConfiguration(isSobel3x3f32Supported(size, borderType, dx, dy)); +#ifdef CAROTENE_NEON + std::vector _tmp; + f32 *tmp = 0; + if (borderType == BORDER_MODE_CONSTANT) + { + _tmp.assign(size.width + 2, borderValue); + tmp = &_tmp[1]; + } + + ptrdiff_t delta = (ptrdiff_t)((size.width + 2 + 31) & -32);//align size + std::vector _tempBuf((delta << 1) + 64); + f32 *trow0 = internal::alignPtr(&_tempBuf[1], 32), *trow1 = internal::alignPtr(trow0 + delta, 32); + + for( size_t y = 0; y < size.height; y++ ) + { + const f32* srow0; + const f32* srow1 = internal::getRowPtr(srcBase, srcStride, y); + const f32* srow2; + f32* drow = internal::getRowPtr(dstBase, dstStride, y > 0 ? y-1 : 0); + f32* drow1 = internal::getRowPtr(dstBase, dstStride, y); + if (borderType == BORDER_MODE_REFLECT101) { + srow0 = internal::getRowPtr(srcBase, srcStride, y > 0 ? y-1 : 1); + srow2 = internal::getRowPtr(srcBase, srcStride, y < size.height-1 ? y+1 : size.height-2); + } else if (borderType == BORDER_MODE_CONSTANT) { + srow0 = y > 0 ? internal::getRowPtr(srcBase, srcStride, y-1) : tmp; + srow2 = y < size.height-1 ? internal::getRowPtr(srcBase, srcStride, y+1) : tmp; + } else { // BORDER_MODE_REFLECT || BORDER_MODE_REPLICATE + srow0 = internal::getRowPtr(srcBase, srcStride, y > 0 ? y-1 : 0); + srow2 = internal::getRowPtr(srcBase, srcStride, y < size.height-1 ? y+1 : size.height-1); + } + + float32x4_t tprev = vmovq_n_f32(0.f); + float32x4_t tcurr = vmovq_n_f32(0.f); + float32x4_t tnext = vmovq_n_f32(0.f); + float32x4_t t0, t1, t2; + // do vertical convolution + size_t x = 0, bcolsn = y + 2 < size.height ? size.width : (size.width - 4); + for( ; x <= bcolsn; x += 4 ) + { + internal::prefetch(srow0 + x); + internal::prefetch(srow1 + x); + internal::prefetch(srow2 + x); + + float32x4_t x0 = vld1q_f32(srow0 + x); + float32x4_t x1 = vld1q_f32(srow1 + x); + float32x4_t x2 = vld1q_f32(srow2 + x); + + tprev = tcurr; + tcurr = tnext; + if(!dy) + { + tnext = vaddq_f32(vaddq_f32(vaddq_f32(x1, x1), x2), x0); + } + else if(dy == 2) + { + tnext = vsubq_f32(vsubq_f32(x2, x1), vsubq_f32(x1, x0)); + } + else + { + tnext = vsubq_f32(x2, x0); + } + + if(!x) { + tcurr = tnext; + // make border + if (borderType == BORDER_MODE_CONSTANT) + { + tcurr = vsetq_lane_f32(borderValue,tcurr, 3); + } + else if (borderType == BORDER_MODE_REFLECT101) + { + tcurr = vsetq_lane_f32(vgetq_lane_f32(tcurr, 1),tcurr, 3); + } + else // BORDER_MODE_REFLECT || BORDER_MODE_REPLICATE + { + tcurr = vsetq_lane_f32(vgetq_lane_f32(tcurr, 0),tcurr, 3); + } + continue; + } + + internal::prefetch(trow0 + x); + internal::prefetch(trow1 + x); + + t0 = vextq_f32(tprev, tcurr, 3); + t1 = tcurr; + t2 = vextq_f32(tcurr, tnext, 1); + if(!dx) + { + t0 = vaddq_f32(t0, vaddq_f32(vaddq_f32(t1, t1), t2)); + } + else if(dx == 2) + { + t0 = vsubq_f32(vsubq_f32(t2, t1), vsubq_f32(t1, t0)); + } + else + { + t0 = vsubq_f32(t2, t0); + } + + if(!(y%2)) + { + vst1q_f32(trow0 + x - 4, t0); + } + else + { + vst1q_f32(trow1 + x - 4, t0); + } + } + x -= 4; + if(x == size.width){ + x--; + } + f32 prevx = 0, rowx = 0, nextx = 0; + if(!dy) + { + prevx = x > 0 ? srow2[x-1] + 2*srow1[x-1] + srow0[x-1] : + (borderType == BORDER_MODE_REFLECT101 ? srow2[1] + 2*srow1[1] + srow0[1] : + (borderType == BORDER_MODE_CONSTANT ? 4*borderValue : + srow2[0] + 2*srow1[0] + srow0[0]) ); + rowx = srow2[x] + 2*srow1[x] + srow0[x]; + } + else if(dy == 2) + { + prevx = x > 0 ? srow2[x-1] - 2*srow1[x-1] + srow0[x-1] : + (borderType == BORDER_MODE_REFLECT101 ? srow2[1] - 2*srow1[1] + srow0[1] : + (borderType == BORDER_MODE_CONSTANT ? 0.f : + srow2[0] - 2*srow1[0] + srow0[0]) ); + rowx = srow2[x] - 2*srow1[x] + srow0[x]; + } + else + { + prevx = x > 0 ? srow2[x-1] - srow0[x-1] : + (borderType == BORDER_MODE_REFLECT101 ? srow2[1] - srow0[1] : + (borderType == BORDER_MODE_CONSTANT ? 0.f : + srow2[0] - srow0[0]) ); + rowx = srow2[x] - srow0[x]; + } + + for( ; x < size.width; x++ ) + { + if(x+1 == size.width) { + // make border + if (borderType == BORDER_MODE_CONSTANT) + { + if(!dy) { + nextx = 4*borderValue; + } else { + nextx = 0.f; + } + } else if (borderType == BORDER_MODE_REFLECT101) + { + if(!dy) { + nextx = srow2[x-1] + 2*srow1[x-1] + srow0[x-1]; + } else if(dy == 2) { + nextx = srow2[x-1] - 2*srow1[x-1] + srow0[x-1]; + } else { + nextx = srow2[x-1] - srow0[x-1]; + } + } else { + if(!dy) { + nextx = srow2[x] + 2*srow1[x] + srow0[x]; + } else if(dy == 2) { + nextx = srow2[x] - 2*srow1[x] + srow0[x]; + } else { + nextx = srow2[x] - srow0[x]; + } + } + } else { + if(!dy) { + nextx = srow2[x+1] + 2*srow1[x+1] + srow0[x+1]; + } else if(dy == 2) { + nextx = srow2[x+1] - 2*srow1[x+1] + srow0[x+1]; + } else { + nextx = srow2[x+1] - srow0[x+1]; + } + } + f32 res; + if(dx==1) { + res = nextx - prevx; + } else if(!dx) { + res = prevx + 2*rowx + nextx; + } else { + res = prevx - 2*rowx + nextx; + } + if(!(y%2)) { + *(trow0+x) = res; + } else { + *(trow1+x) = res; + } + prevx = rowx; + rowx = nextx; + } + + if(y>0) { + for(size_t x1 = 0; x1 < size.width; x1++ ) + { + if(y%2) + *(drow + x1) = trow0[x1]; + else + *(drow + x1) = trow1[x1]; + } + } + if(y == size.height-1) { + for(size_t x1 = 0; x1 < size.width; x1++ ) + { + if(!(y%2)) + *(drow1 + x1) = trow0[x1]; + else + *(drow1 + x1) = trow1[x1]; + } + } + } +#else + (void)srcBase; + (void)srcStride; + (void)dstBase; + (void)dstStride; + (void)borderValue; +#endif +} + +} // namespace CAROTENE_NS diff --git a/3rdparty/carotene/src/sub.cpp b/3rdparty/carotene/src/sub.cpp new file mode 100644 index 0000000000..38853895e7 --- /dev/null +++ b/3rdparty/carotene/src/sub.cpp @@ -0,0 +1,621 @@ +/* + * By downloading, copying, installing or using the software you agree to this license. + * If you do not agree to this license, do not download, install, + * copy or use the software. + * + * + * License Agreement + * For Open Source Computer Vision Library + * (3-clause BSD License) + * + * Copyright (C) 2014, NVIDIA Corporation, all rights reserved. + * Third party copyrights are property of their respective owners. + * + * Redistribution and use in source and binary forms, with or without modification, + * are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * * Neither the names of the copyright holders nor the names of the contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * This software is provided by the copyright holders and contributors "as is" and + * any express or implied warranties, including, but not limited to, the implied + * warranties of merchantability and fitness for a particular purpose are disclaimed. + * In no event shall copyright holders or contributors be liable for any direct, + * indirect, incidental, special, exemplary, or consequential damages + * (including, but not limited to, procurement of substitute goods or services; + * loss of use, data, or profits; or business interruption) however caused + * and on any theory of liability, whether in contract, strict liability, + * or tort (including negligence or otherwise) arising in any way out of + * the use of this software, even if advised of the possibility of such damage. + */ + +#include "common.hpp" +#include "vtransform.hpp" + +namespace CAROTENE_NS { + +#ifdef CAROTENE_NEON + +namespace { + +template +struct SubWrap +{ + typedef T type; + + void operator() (const typename internal::VecTraits::vec128 & v_src0, + const typename internal::VecTraits::vec128 & v_src1, + typename internal::VecTraits::vec128 & v_dst) const + { + v_dst = internal::vsubq(v_src0, v_src1); + } + + void operator() (const typename internal::VecTraits::vec64 & v_src0, + const typename internal::VecTraits::vec64 & v_src1, + typename internal::VecTraits::vec64 & v_dst) const + { + v_dst = internal::vsub(v_src0, v_src1); + } + + void operator() (const T * src0, const T * src1, T * dst) const + { + dst[0] = (T)((WT)src0[0] - (WT)src1[0]); + } +}; + +template +struct SubSaturate +{ + typedef T type; + + void operator() (const typename internal::VecTraits::vec128 & v_src0, + const typename internal::VecTraits::vec128 & v_src1, + typename internal::VecTraits::vec128 & v_dst) const + { + v_dst = internal::vqsubq(v_src0, v_src1); + } + + void operator() (const typename internal::VecTraits::vec64 & v_src0, + const typename internal::VecTraits::vec64 & v_src1, + typename internal::VecTraits::vec64 & v_dst) const + { + v_dst = internal::vqsub(v_src0, v_src1); + } + + void operator() (const T * src0, const T * src1, T * dst) const + { + dst[0] = internal::saturate_cast((WT)src0[0] - (WT)src1[0]); + } +}; + +} // namespace + +#endif + +void sub(const Size2D &size, + const u8 * src0Base, ptrdiff_t src0Stride, + const u8 * src1Base, ptrdiff_t src1Stride, + u8 *dstBase, ptrdiff_t dstStride, + CONVERT_POLICY policy) +{ + internal::assertSupportedConfiguration(); +#ifdef CAROTENE_NEON + if (policy == CONVERT_POLICY_SATURATE) + { + internal::vtransform(size, + src0Base, src0Stride, + src1Base, src1Stride, + dstBase, dstStride, + SubSaturate()); + } + else + { + internal::vtransform(size, + src0Base, src0Stride, + src1Base, src1Stride, + dstBase, dstStride, + SubWrap()); + } +#else + (void)size; + (void)src0Base; + (void)src0Stride; + (void)src1Base; + (void)src1Stride; + (void)dstBase; + (void)dstStride; + (void)policy; +#endif +} + +void sub(const Size2D &size, + const u8 * src0Base, ptrdiff_t src0Stride, + const u8 * src1Base, ptrdiff_t src1Stride, + s16 *dstBase, ptrdiff_t dstStride, + CONVERT_POLICY) +{ + internal::assertSupportedConfiguration(); +#ifdef CAROTENE_NEON + size_t roiw32 = size.width >= 31 ? size.width - 31 : 0; + size_t roiw8 = size.width >= 7 ? size.width - 7 : 0; + + for (size_t i = 0; i < size.height; ++i) + { + const u8 * src0 = internal::getRowPtr(src0Base, src0Stride, i); + const u8 * src1 = internal::getRowPtr(src1Base, src1Stride, i); + u16 * dstu16 = internal::getRowPtr((u16 *)dstBase, dstStride, i); + s16 * dst = internal::getRowPtr(dstBase, dstStride, i); + size_t j = 0; + + for (; j < roiw32; j += 32) + { + internal::prefetch(src0 + j); + internal::prefetch(src1 + j); + uint8x16_t v_src00 = vld1q_u8(src0 + j), v_src01 = vld1q_u8(src0 + j + 16); + uint8x16_t v_src10 = vld1q_u8(src1 + j), v_src11 = vld1q_u8(src1 + j + 16); + vst1q_u16(dstu16 + j, vsubl_u8(vget_low_u8(v_src00), vget_low_u8(v_src10))); + vst1q_u16(dstu16 + j + 8, vsubl_u8(vget_high_u8(v_src00), vget_high_u8(v_src10))); + vst1q_u16(dstu16 + j + 16, vsubl_u8(vget_low_u8(v_src01), vget_low_u8(v_src11))); + vst1q_u16(dstu16 + j + 24, vsubl_u8(vget_high_u8(v_src01), vget_high_u8(v_src11))); + } + for (; j < roiw8; j += 8) + { + uint8x8_t v_src0 = vld1_u8(src0 + j); + uint8x8_t v_src1 = vld1_u8(src1 + j); + vst1q_u16(dstu16 + j, vsubl_u8(v_src0, v_src1)); + } + + for (; j < size.width; j++) + dst[j] = (s16)src0[j] - (s16)src1[j]; + } +#else + (void)size; + (void)src0Base; + (void)src0Stride; + (void)src1Base; + (void)src1Stride; + (void)dstBase; + (void)dstStride; +#endif +} + +void sub(const Size2D &size, + const u8 * src0Base, ptrdiff_t src0Stride, + const u8 * src1Base, ptrdiff_t src1Stride, + f32 *dstBase, ptrdiff_t dstStride) +{ + internal::assertSupportedConfiguration(); +#ifdef CAROTENE_NEON + size_t roiw32 = size.width >= 31 ? size.width - 31 : 0; + size_t roiw8 = size.width >= 7 ? size.width - 7 : 0; + + for (size_t i = 0; i < size.height; ++i) + { + const u8 * src0 = internal::getRowPtr(src0Base, src0Stride, i); + const u8 * src1 = internal::getRowPtr(src1Base, src1Stride, i); + f32 * dst = internal::getRowPtr(dstBase, dstStride, i); + size_t j = 0; + + for (; j < roiw32; j += 32) + { + internal::prefetch(src0 + j); + internal::prefetch(src1 + j); + uint8x16_t v_src00 = vld1q_u8(src0 + j), v_src01 = vld1q_u8(src0 + j + 16); + uint8x16_t v_src10 = vld1q_u8(src1 + j), v_src11 = vld1q_u8(src1 + j + 16); + int16x8_t vsl = vreinterpretq_s16_u16(vsubl_u8( vget_low_u8(v_src00), vget_low_u8(v_src10))); + int16x8_t vsh = vreinterpretq_s16_u16(vsubl_u8(vget_high_u8(v_src00), vget_high_u8(v_src10))); + + vst1q_f32(dst + j + 0, vcvtq_f32_s32(vmovl_s16( vget_low_s16(vsl) ))); + vst1q_f32(dst + j + 4, vcvtq_f32_s32(vmovl_s16( vget_high_s16(vsl) ))); + vst1q_f32(dst + j + 8, vcvtq_f32_s32(vmovl_s16( vget_low_s16(vsh) ))); + vst1q_f32(dst + j + 12, vcvtq_f32_s32(vmovl_s16( vget_high_s16(vsh) ))); + + vsl = vreinterpretq_s16_u16(vsubl_u8( vget_low_u8(v_src01), vget_low_u8(v_src11))); + vsh = vreinterpretq_s16_u16(vsubl_u8(vget_high_u8(v_src01), vget_high_u8(v_src11))); + + vst1q_f32(dst + j + 16, vcvtq_f32_s32(vmovl_s16( vget_low_s16(vsl) ))); + vst1q_f32(dst + j + 20, vcvtq_f32_s32(vmovl_s16( vget_high_s16(vsl) ))); + vst1q_f32(dst + j + 24, vcvtq_f32_s32(vmovl_s16( vget_low_s16(vsh) ))); + vst1q_f32(dst + j + 28, vcvtq_f32_s32(vmovl_s16( vget_high_s16(vsh) ))); + } + for (; j < roiw8; j += 8) + { + uint8x8_t v_src0 = vld1_u8(src0 + j); + uint8x8_t v_src1 = vld1_u8(src1 + j); + + int16x8_t vs = vreinterpretq_s16_u16(vsubl_u8(v_src0, v_src1)); + vst1q_f32(dst + j + 0, vcvtq_f32_s32(vmovl_s16( vget_low_s16(vs) ))); + vst1q_f32(dst + j + 4, vcvtq_f32_s32(vmovl_s16( vget_high_s16(vs) ))); + } + for(; j < size.width; j++) + dst[j] = (f32)src0[j] - (f32)src1[j]; + } +#else + (void)size; + (void)src0Base; + (void)src0Stride; + (void)src1Base; + (void)src1Stride; + (void)dstBase; + (void)dstStride; +#endif +} + +void sub(const Size2D &size, + const u8 * src0Base, ptrdiff_t src0Stride, + const s16 * src1Base, ptrdiff_t src1Stride, + s16 *dstBase, ptrdiff_t dstStride, + CONVERT_POLICY policy) +{ + internal::assertSupportedConfiguration(); +#ifdef CAROTENE_NEON + size_t roiw16 = size.width >= 15 ? size.width - 15 : 0; + size_t roiw8 = size.width >= 7 ? size.width - 7 : 0; + + for (size_t i = 0; i < size.height; ++i) + { + const u8 * src0 = internal::getRowPtr(src0Base, src0Stride, i); + const s16 * src1 = internal::getRowPtr(src1Base, src1Stride, i); + s16 * dst = internal::getRowPtr(dstBase, dstStride, i); + size_t j = 0; + + if (policy == CONVERT_POLICY_SATURATE) + { + for (; j < roiw16; j += 16) + { + internal::prefetch(src0 + j); + internal::prefetch(src1 + j); + uint8x16_t v_src0 = vld1q_u8(src0 + j); + int16x8_t v_src00 = vreinterpretq_s16_u16(vmovl_u8(vget_low_u8(v_src0))); + int16x8_t v_src01 = vreinterpretq_s16_u16(vmovl_u8(vget_high_u8(v_src0))); + int16x8_t v_src10 = vld1q_s16(src1 + j), v_src11 = vld1q_s16(src1 + j + 8); + int16x8_t v_dst0 = vqsubq_s16(v_src00, v_src10); + int16x8_t v_dst1 = vqsubq_s16(v_src01, v_src11); + vst1q_s16(dst + j, v_dst0); + vst1q_s16(dst + j + 8, v_dst1); + } + for (; j < roiw8; j += 8) + { + int16x8_t v_src0 = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(src0 + j))); + int16x8_t v_src1 = vld1q_s16(src1 + j); + int16x8_t v_dst = vqsubq_s16(v_src0, v_src1); + vst1q_s16(dst + j, v_dst); + } + + for (; j < size.width; j++) + dst[j] = internal::saturate_cast((s32)src0[j] - (s32)src1[j]); + } + else + { + for (; j < roiw16; j += 16) + { + internal::prefetch(src0 + j); + internal::prefetch(src1 + j); + uint8x16_t v_src0 = vld1q_u8(src0 + j); + int16x8_t v_src00 = vreinterpretq_s16_u16(vmovl_u8(vget_low_u8(v_src0))); + int16x8_t v_src01 = vreinterpretq_s16_u16(vmovl_u8(vget_high_u8(v_src0))); + int16x8_t v_src10 = vld1q_s16(src1 + j), v_src11 = vld1q_s16(src1 + j + 8); + int16x8_t v_dst0 = vsubq_s16(v_src00, v_src10); + int16x8_t v_dst1 = vsubq_s16(v_src01, v_src11); + vst1q_s16(dst + j, v_dst0); + vst1q_s16(dst + j + 8, v_dst1); + } + for (; j < roiw8; j += 8) + { + int16x8_t v_src0 = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(src0 + j))); + int16x8_t v_src1 = vld1q_s16(src1 + j); + int16x8_t v_dst = vsubq_s16(v_src0, v_src1); + vst1q_s16(dst + j, v_dst); + } + + for (; j < size.width; j++) + dst[j] = (s16)((s32)src0[j] - (s32)src1[j]); + } + } +#else + (void)size; + (void)src0Base; + (void)src0Stride; + (void)src1Base; + (void)src1Stride; + (void)dstBase; + (void)dstStride; + (void)policy; +#endif +} + +void sub(const Size2D &size, + const s16 * src0Base, ptrdiff_t src0Stride, + const u8 * src1Base, ptrdiff_t src1Stride, + s16 *dstBase, ptrdiff_t dstStride, + CONVERT_POLICY policy) +{ + internal::assertSupportedConfiguration(); +#ifdef CAROTENE_NEON + size_t roiw16 = size.width >= 15 ? size.width - 15 : 0; + size_t roiw8 = size.width >= 7 ? size.width - 7 : 0; + + for (size_t i = 0; i < size.height; ++i) + { + const s16 * src0 = internal::getRowPtr(src0Base, src0Stride, i); + const u8 * src1 = internal::getRowPtr(src1Base, src1Stride, i); + s16 * dst = internal::getRowPtr(dstBase, dstStride, i); + size_t j = 0; + + if (policy == CONVERT_POLICY_SATURATE) + { + for (; j < roiw16; j += 16) + { + internal::prefetch(src0 + j); + internal::prefetch(src1 + j); + int16x8_t v_src00 = vld1q_s16(src0 + j), v_src01 = vld1q_s16(src0 + j + 8); + uint8x16_t v_src1 = vld1q_u8(src1 + j); + int16x8_t v_src10 = vreinterpretq_s16_u16(vmovl_u8(vget_low_u8(v_src1))); + int16x8_t v_src11 = vreinterpretq_s16_u16(vmovl_u8(vget_high_u8(v_src1))); + int16x8_t v_dst0 = vqsubq_s16(v_src00, v_src10); + int16x8_t v_dst1 = vqsubq_s16(v_src01, v_src11); + vst1q_s16(dst + j, v_dst0); + vst1q_s16(dst + j + 8, v_dst1); + } + for (; j < roiw8; j += 8) + { + int16x8_t v_src0 = vld1q_s16(src0 + j); + int16x8_t v_src1 = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(src1 + j))); + int16x8_t v_dst = vqsubq_s16(v_src0, v_src1); + vst1q_s16(dst + j, v_dst); + } + + for (; j < size.width; j++) + dst[j] = internal::saturate_cast((s32)src0[j] - (s32)src1[j]); + } + else + { + for (; j < roiw16; j += 16) + { + internal::prefetch(src0 + j); + internal::prefetch(src1 + j); + int16x8_t v_src00 = vld1q_s16(src0 + j), v_src01 = vld1q_s16(src0 + j + 8); + uint8x16_t v_src1 = vld1q_u8(src1 + j); + int16x8_t v_src10 = vreinterpretq_s16_u16(vmovl_u8(vget_low_u8(v_src1))); + int16x8_t v_src11 = vreinterpretq_s16_u16(vmovl_u8(vget_high_u8(v_src1))); + int16x8_t v_dst0 = vsubq_s16(v_src00, v_src10); + int16x8_t v_dst1 = vsubq_s16(v_src01, v_src11); + vst1q_s16(dst + j, v_dst0); + vst1q_s16(dst + j + 8, v_dst1); + } + for (; j < roiw8; j += 8) + { + int16x8_t v_src0 = vld1q_s16(src0 + j); + int16x8_t v_src1 = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(src1 + j))); + int16x8_t v_dst = vsubq_s16(v_src0, v_src1); + vst1q_s16(dst + j, v_dst); + } + + for (; j < size.width; j++) + dst[j] = (s16)((s32)src0[j] - (s32)src1[j]); + } + } +#else + (void)size; + (void)src0Base; + (void)src0Stride; + (void)src1Base; + (void)src1Stride; + (void)dstBase; + (void)dstStride; + (void)policy; +#endif +} + +void sub(const Size2D &size, + const s8 * src0Base, ptrdiff_t src0Stride, + const s8 * src1Base, ptrdiff_t src1Stride, + s8 *dstBase, ptrdiff_t dstStride, + CONVERT_POLICY policy) +{ + internal::assertSupportedConfiguration(); +#ifdef CAROTENE_NEON + if (policy == CONVERT_POLICY_SATURATE) + { + internal::vtransform(size, + src0Base, src0Stride, + src1Base, src1Stride, + dstBase, dstStride, + SubSaturate()); + } + else + { + internal::vtransform(size, + src0Base, src0Stride, + src1Base, src1Stride, + dstBase, dstStride, + SubWrap()); + } +#else + (void)size; + (void)src0Base; + (void)src0Stride; + (void)src1Base; + (void)src1Stride; + (void)dstBase; + (void)dstStride; + (void)policy; +#endif +} + +void sub(const Size2D &size, + const s16 * src0Base, ptrdiff_t src0Stride, + const s16 * src1Base, ptrdiff_t src1Stride, + s16 *dstBase, ptrdiff_t dstStride, + CONVERT_POLICY policy) +{ + internal::assertSupportedConfiguration(); +#ifdef CAROTENE_NEON + if (policy == CONVERT_POLICY_SATURATE) + { + internal::vtransform(size, + src0Base, src0Stride, + src1Base, src1Stride, + dstBase, dstStride, + SubSaturate()); + } + else + { + internal::vtransform(size, + src0Base, src0Stride, + src1Base, src1Stride, + dstBase, dstStride, + SubWrap()); + } +#else + (void)size; + (void)src0Base; + (void)src0Stride; + (void)src1Base; + (void)src1Stride; + (void)dstBase; + (void)dstStride; + (void)policy; +#endif +} + +void sub(const Size2D &size, + const u16 * src0Base, ptrdiff_t src0Stride, + const u16 * src1Base, ptrdiff_t src1Stride, + u16 *dstBase, ptrdiff_t dstStride, + CONVERT_POLICY policy) +{ + internal::assertSupportedConfiguration(); +#ifdef CAROTENE_NEON + if (policy == CONVERT_POLICY_SATURATE) + { + internal::vtransform(size, + src0Base, src0Stride, + src1Base, src1Stride, + dstBase, dstStride, + SubSaturate()); + } + else + { + internal::vtransform(size, + src0Base, src0Stride, + src1Base, src1Stride, + dstBase, dstStride, + SubWrap()); + } +#else + (void)size; + (void)src0Base; + (void)src0Stride; + (void)src1Base; + (void)src1Stride; + (void)dstBase; + (void)dstStride; + (void)policy; +#endif +} + +void sub(const Size2D &size, + const s32 * src0Base, ptrdiff_t src0Stride, + const s32 * src1Base, ptrdiff_t src1Stride, + s32 *dstBase, ptrdiff_t dstStride, + CONVERT_POLICY policy) +{ + internal::assertSupportedConfiguration(); +#ifdef CAROTENE_NEON + if (policy == CONVERT_POLICY_SATURATE) + { + internal::vtransform(size, + src0Base, src0Stride, + src1Base, src1Stride, + dstBase, dstStride, + SubSaturate()); + } + else + { + internal::vtransform(size, + src0Base, src0Stride, + src1Base, src1Stride, + dstBase, dstStride, + SubWrap()); + } +#else + (void)size; + (void)src0Base; + (void)src0Stride; + (void)src1Base; + (void)src1Stride; + (void)dstBase; + (void)dstStride; + (void)policy; +#endif +} + +void sub(const Size2D &size, + const u32 * src0Base, ptrdiff_t src0Stride, + const u32 * src1Base, ptrdiff_t src1Stride, + u32 *dstBase, ptrdiff_t dstStride, + CONVERT_POLICY policy) +{ + internal::assertSupportedConfiguration(); +#ifdef CAROTENE_NEON + if (policy == CONVERT_POLICY_SATURATE) + { + internal::vtransform(size, + src0Base, src0Stride, + src1Base, src1Stride, + dstBase, dstStride, + SubSaturate()); + } + else + { + internal::vtransform(size, + src0Base, src0Stride, + src1Base, src1Stride, + dstBase, dstStride, + SubWrap()); + } +#else + (void)size; + (void)src0Base; + (void)src0Stride; + (void)src1Base; + (void)src1Stride; + (void)dstBase; + (void)dstStride; + (void)policy; +#endif +} + +void sub(const Size2D &size, + const f32 * src0Base, ptrdiff_t src0Stride, + const f32 * src1Base, ptrdiff_t src1Stride, + f32 *dstBase, ptrdiff_t dstStride) +{ + internal::assertSupportedConfiguration(); +#ifdef CAROTENE_NEON + internal::vtransform(size, + src0Base, src0Stride, + src1Base, src1Stride, + dstBase, dstStride, + SubWrap()); +#else + (void)size; + (void)src0Base; + (void)src0Stride; + (void)src1Base; + (void)src1Stride; + (void)dstBase; + (void)dstStride; +#endif +} + +} // namespace CAROTENE_NS diff --git a/3rdparty/carotene/src/sum.cpp b/3rdparty/carotene/src/sum.cpp new file mode 100644 index 0000000000..812e7fca67 --- /dev/null +++ b/3rdparty/carotene/src/sum.cpp @@ -0,0 +1,385 @@ +/* + * By downloading, copying, installing or using the software you agree to this license. + * If you do not agree to this license, do not download, install, + * copy or use the software. + * + * + * License Agreement + * For Open Source Computer Vision Library + * (3-clause BSD License) + * + * Copyright (C) 2012-2015, NVIDIA Corporation, all rights reserved. + * Third party copyrights are property of their respective owners. + * + * Redistribution and use in source and binary forms, with or without modification, + * are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * * Neither the names of the copyright holders nor the names of the contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * This software is provided by the copyright holders and contributors "as is" and + * any express or implied warranties, including, but not limited to, the implied + * warranties of merchantability and fitness for a particular purpose are disclaimed. + * In no event shall copyright holders or contributors be liable for any direct, + * indirect, incidental, special, exemplary, or consequential damages + * (including, but not limited to, procurement of substitute goods or services; + * loss of use, data, or profits; or business interruption) however caused + * and on any theory of liability, whether in contract, strict liability, + * or tort (including negligence or otherwise) arising in any way out of + * the use of this software, even if advised of the possibility of such damage. + */ + +#include "common.hpp" + +#include "vtransform.hpp" + +namespace CAROTENE_NS { + +bool isSumSupported(u32 channels) +{ + return (channels && channels < 5); +} + +void sum(const Size2D &_size, + const u8 * srcBase, ptrdiff_t srcStride, + u32 * sumdst, u32 channels) +{ + internal::assertSupportedConfiguration(isSumSupported(channels)); +#ifdef CAROTENE_NEON + Size2D size(_size); + if (srcStride == (ptrdiff_t)(size.width)) + { + size.width *= size.height; + size.height = 1; + } + const ptrdiff_t width = size.width * channels; + + for(size_t k = 0; k < size.height; ++k) + { + const u8* src = internal::getRowPtr( srcBase, srcStride, k); + ptrdiff_t i = 0; + + if (channels == 3) + { + uint32x4_t vs1231 = vdupq_n_u32(0); + uint32x4_t vs3123 = vdupq_n_u32(0); + uint32x4_t vs2312 = vdupq_n_u32(0); + for (; i <= width - 257*8*3; i += 257*8*3, src += 257*8*3) + { + uint16x8_t s1 = vmovl_u8(vld1_u8(src + 0)); + uint16x8_t s2 = vmovl_u8(vld1_u8(src + 8)); + uint16x8_t s3 = vmovl_u8(vld1_u8(src + 16)); + + for (ptrdiff_t j = 8*3; j < 257*8*3; j+= 8*3) + { + internal::prefetch(src + j + 24); + s1 = vaddw_u8(s1, vld1_u8(src + j + 0)); + s2 = vaddw_u8(s2, vld1_u8(src + j + 8)); + s3 = vaddw_u8(s3, vld1_u8(src + j + 16)); + } + + vs1231 = vqaddq_u32(vs1231, vaddl_u16(vget_low_u16(s1), vget_high_u16(s2))); + vs3123 = vqaddq_u32(vs3123, vaddl_u16(vget_low_u16(s2), vget_high_u16(s3))); + vs2312 = vqaddq_u32(vs2312, vaddl_u16(vget_low_u16(s3), vget_high_u16(s1))); + } + if (i <= width - 8*3) + { + uint16x8_t s1 = vmovl_u8(vld1_u8(src + 0)); + uint16x8_t s2 = vmovl_u8(vld1_u8(src + 8)); + uint16x8_t s3 = vmovl_u8(vld1_u8(src + 16)); + + for (i += 8*3, src += 8*3; i <= width - 8*3; i += 8*3, src += 8*3) + { + internal::prefetch(src + 24); + s1 = vaddw_u8(s1, vld1_u8(src + 0)); + s2 = vaddw_u8(s2, vld1_u8(src + 8)); + s3 = vaddw_u8(s3, vld1_u8(src + 16)); + } + + vs1231 = vqaddq_u32(vs1231, vaddl_u16(vget_low_u16(s1), vget_high_u16(s2))); + vs3123 = vqaddq_u32(vs3123, vaddl_u16(vget_low_u16(s2), vget_high_u16(s3))); + vs2312 = vqaddq_u32(vs2312, vaddl_u16(vget_low_u16(s3), vget_high_u16(s1))); + } + + u32 sum[12]; + vst1q_u32(sum+0, vs1231); + vst1q_u32(sum+4, vs2312); + vst1q_u32(sum+8, vs3123); + + for (; i < width; i += 3, src += 3) + { + sumdst[0] += src[0]; + sumdst[1] += src[1]; + sumdst[2] += src[2]; + } + + sumdst[0] += sum[0] + sum[3] + sum[6] + sum[9]; + sumdst[1] += sum[1] + sum[4] + sum[7] + sum[10]; + sumdst[2] += sum[2] + sum[5] + sum[8] + sum[11]; + } + else + { + uint32x4_t vs = vdupq_n_u32(0); + for (; i <= width - 257*8; i += 257*8, src += 257 * 8) + { + uint16x8_t s1 = vmovl_u8(vld1_u8(src)); + + for (int j = 8; j < 257 * 8; j += 8) + { + internal::prefetch(src + j); + s1 = vaddw_u8(s1, vld1_u8(src + j)); + } + + vs = vqaddq_u32(vs, vaddl_u16(vget_low_u16(s1), vget_high_u16(s1))); + } + if (i < width - 7) + { + uint16x8_t s1 = vmovl_u8(vld1_u8(src)); + + for(i+=8,src+=8; i < width-7; i+=8,src+=8) + { + internal::prefetch(src); + s1 = vaddw_u8(s1, vld1_u8(src)); + } + vs = vqaddq_u32(vs, vaddl_u16(vget_low_u16(s1), vget_high_u16(s1))); + } + + if (channels == 1) + { + uint32x2_t vs2 = vqadd_u32(vget_low_u32(vs), vget_high_u32(vs)); + uint32x2_t vs1 = vreinterpret_u32_u64(vpaddl_u32(vs2)); + + u32 s0 = vget_lane_u32(vs1, 0); + for(; i < width; ++i,++src) + s0 += src[0]; + sumdst[0] += s0; + } + else if (channels == 4) + { + vst1q_u32(sumdst, vqaddq_u32(vs, vld1q_u32(sumdst))); + + for(; i < width; i+=4,src+=4) + { + sumdst[0] += src[0]; + sumdst[1] += src[1]; + sumdst[2] += src[2]; + sumdst[3] += src[3]; + } + } + else//if (channels == 2) + { + uint32x2_t vs2 = vqadd_u32(vget_low_u32(vs), vget_high_u32(vs)); + vst1_u32(sumdst, vqadd_u32(vs2, vld1_u32(sumdst))); + + for(; i < width; i+=2,src+=2) + { + sumdst[0] += src[0]; + sumdst[1] += src[1]; + } + } + }//channels != 3 + } +#else + (void)_size; + (void)srcBase; + (void)srcStride; + (void)sumdst; + (void)channels; +#endif +} + +void sum(const Size2D &_size, + const f32 * srcBase, ptrdiff_t srcStride, + f64 * sumdst, u32 channels) +{ + internal::assertSupportedConfiguration(isSumSupported(channels)); +#ifdef CAROTENE_NEON + Size2D size(_size); + if (srcStride == (ptrdiff_t)(size.width)) + { + size.width *= size.height; + size.height = 1; + } + const ptrdiff_t width = size.width * channels; + + for(size_t k = 0; k < size.height; ++k) + { + const f32* src = internal::getRowPtr( srcBase, srcStride, k); + ptrdiff_t i = 0; + + if (channels == 3) + { + float32x4_t vs1231 = vdupq_n_f32(0); + float32x4_t vs2312 = vdupq_n_f32(0); + float32x4_t vs3123 = vdupq_n_f32(0); + for(; i <= width-12; i += 12) + { + internal::prefetch(src + i + 12); + vs1231 = vaddq_f32(vs1231, vld1q_f32(src + i + 0)); + vs2312 = vaddq_f32(vs2312, vld1q_f32(src + i + 4)); + vs3123 = vaddq_f32(vs3123, vld1q_f32(src + i + 8)); + } + + f32 s[12]; + vst1q_f32(s + 0, vs1231); + vst1q_f32(s + 4, vs2312); + vst1q_f32(s + 8, vs3123); + + sumdst[0] += s[0] + s[3] + s[6] + s[9]; + sumdst[1] += s[1] + s[4] + s[7] + s[10]; + sumdst[2] += s[2] + s[5] + s[8] + s[11]; + for( ; i < width; i+=3) + { + sumdst[0] += src[i]; + sumdst[1] += src[i+1]; + sumdst[2] += src[i+2]; + } + } + else + { + float32x4_t vs = vdupq_n_f32(0); + for(; i <= width-4; i += 4) + { + internal::prefetch(src + i); + vs = vaddq_f32(vs, vld1q_f32(src+i)); + } + + if (channels == 1) + { + float32x2_t vs2 = vpadd_f32(vget_low_f32(vs), vget_high_f32(vs)); + f32 s[2]; + vst1_f32(s, vs2); + + sumdst[0] += s[0] + s[1]; + for( ; i < width; i++) + sumdst[0] += src[i]; + } + else if (channels == 4) + { + f32 s[4]; + vst1q_f32(s, vs); + + sumdst[0] += s[0]; + sumdst[1] += s[1]; + sumdst[2] += s[2]; + sumdst[3] += s[3]; + } + else//if (channels == 2) + { + float32x2_t vs2 = vadd_f32(vget_low_f32(vs), vget_high_f32(vs)); + f32 s[2]; + vst1_f32(s, vs2); + + sumdst[0] += s[0]; + sumdst[1] += s[1]; + + if(i < width) + { + sumdst[0] += src[i]; + sumdst[1] += src[i+1]; + } + } + }//channels != 3 + } +#else + (void)_size; + (void)srcBase; + (void)srcStride; + (void)sumdst; + (void)channels; +#endif +} + +bool isSqsumSupported(u32 channels) +{ + return (channels && ((4/channels)*channels == 4)); +} + +void sqsum(const Size2D &_size, + const u8 * srcBase, ptrdiff_t srcStride, + f64 * sumdst, f64 * sqsumdst, u32 channels) +{ + internal::assertSupportedConfiguration(isSqsumSupported(channels)); +#ifdef CAROTENE_NEON + Size2D size(_size); + if (srcStride == (ptrdiff_t)(size.width*channels)) + { + size.width *= size.height; + size.height = 1; + } + const size_t width = size.width * channels; + + size_t blockSize0 = 1 << 23; + size_t roiw8 = width & ~7; + + uint32x4_t v_zero = vdupq_n_u32(0u); + + for (size_t i = 0; i < size.height; ++i) + { + const u8 * src = internal::getRowPtr(srcBase, srcStride, i); + size_t j = 0u; + + while (j < roiw8) + { + size_t blockSize = std::min(roiw8 - j, blockSize0) + j; + uint32x4_t v_sum = v_zero; + uint32x4_t v_sqsum = v_zero; + + for ( ; j < blockSize ; j += 8, src += 8) + { + internal::prefetch(src); + uint8x8_t v_src0 = vld1_u8(src); + + uint16x8_t v_src = vmovl_u8(v_src0); + uint16x4_t v_srclo = vget_low_u16(v_src), v_srchi = vget_high_u16(v_src); + v_sum = vaddq_u32(v_sum, vaddl_u16(v_srclo, v_srchi)); + v_sqsum = vmlal_u16(v_sqsum, v_srclo, v_srclo); + v_sqsum = vmlal_u16(v_sqsum, v_srchi, v_srchi); + } + + u32 arsum[8]; + vst1q_u32(arsum, v_sum); + vst1q_u32(arsum + 4, v_sqsum); + + sumdst[0] += (f64)arsum[0]; + sumdst[1 % channels] += (f64)arsum[1]; + sumdst[2 % channels] += (f64)arsum[2]; + sumdst[3 % channels] += (f64)arsum[3]; + sqsumdst[0] += (f64)arsum[4]; + sqsumdst[1 % channels] += (f64)arsum[5]; + sqsumdst[2 % channels] += (f64)arsum[6]; + sqsumdst[3 % channels] += (f64)arsum[7]; + } + // collect a few last elements in the current row + // it's ok to process channels elements per step + // since we could handle 1,2 or 4 channels + // we always have channels-fold amount of elements remaining + for ( ; j < width; j+=channels, src+=channels) + { + for (u32 kk = 0; kk < channels; kk++) + { + u32 srcval = src[kk]; + sumdst[kk] += srcval; + sqsumdst[kk] += srcval * srcval; + } + } + } +#else + (void)_size; + (void)srcBase; + (void)srcStride; + (void)sumdst; + (void)sqsumdst; + (void)channels; +#endif +} + +} // namespace CAROTENE_NS diff --git a/3rdparty/carotene/src/template_matching.cpp b/3rdparty/carotene/src/template_matching.cpp new file mode 100644 index 0000000000..ad87085188 --- /dev/null +++ b/3rdparty/carotene/src/template_matching.cpp @@ -0,0 +1,241 @@ +/* + * By downloading, copying, installing or using the software you agree to this license. + * If you do not agree to this license, do not download, install, + * copy or use the software. + * + * + * License Agreement + * For Open Source Computer Vision Library + * (3-clause BSD License) + * + * Copyright (C) 2013-2015, NVIDIA Corporation, all rights reserved. + * Third party copyrights are property of their respective owners. + * + * Redistribution and use in source and binary forms, with or without modification, + * are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * * Neither the names of the copyright holders nor the names of the contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * This software is provided by the copyright holders and contributors "as is" and + * any express or implied warranties, including, but not limited to, the implied + * warranties of merchantability and fitness for a particular purpose are disclaimed. + * In no event shall copyright holders or contributors be liable for any direct, + * indirect, incidental, special, exemplary, or consequential damages + * (including, but not limited to, procurement of substitute goods or services; + * loss of use, data, or profits; or business interruption) however caused + * and on any theory of liability, whether in contract, strict liability, + * or tort (including negligence or otherwise) arising in any way out of + * the use of this software, even if advised of the possibility of such damage. + */ + +#include "common.hpp" + +#include +#include + +namespace CAROTENE_NS { + +#define ENABLE4LINESMATCHING false //Disabled since overall time for simultaneous 4 lines matching is greater than + //time for simultaneous 2 lines matching for the same amount of data + +bool isMatchTemplateSupported(const Size2D &tmplSize) +{ + return isSupportedConfiguration() && + tmplSize.width >= 8 && // Actually the function could process even shorter templates + // but there will be no NEON optimization in this case + (tmplSize.width * tmplSize.height) <= 256; +} + +void matchTemplate(const Size2D &srcSize, + const u8 * srcBase, ptrdiff_t srcStride, + const Size2D &tmplSize, + const u8 * tmplBase, ptrdiff_t tmplStride, + f32 * dstBase, ptrdiff_t dstStride, + bool normalize) +{ + internal::assertSupportedConfiguration(isMatchTemplateSupported(tmplSize)); +#ifdef CAROTENE_NEON + const size_t tmplW = tmplSize.width; + const size_t tmplH = tmplSize.height; + const size_t dstW = srcSize.width - tmplSize.width + 1; + const size_t dstH = srcSize.height - tmplSize.height + 1; + + //template correlation part + { +#if ENABLE4LINESMATCHING + const size_t dstroiw4 = dstW & ~3u; +#endif + const size_t dstroiw2 = dstW & ~1u; + const size_t tmplroiw = tmplW & ~7u; + const size_t dstride = dstStride >> 2; + + f32 *corr = dstBase; + const u8 *imgrrow = srcBase; + for(size_t r = 0; r < dstH; ++r, corr+=dstride, imgrrow+=srcStride) + { + size_t c = 0; +#if ENABLE4LINESMATCHING + for(; c < dstroiw4; c+=4) + { + u32 dot[4] = {0, 0, 0, 0}; + uint32x4_t vdot0 = vmovq_n_u32(0); + uint32x4_t vdot1 = vmovq_n_u32(0); + uint32x4_t vdot2 = vmovq_n_u32(0); + uint32x4_t vdot3 = vmovq_n_u32(0); + + const u8 *img = imgrrow; + const u8 *tmpl = tmplBase; + for(size_t i = 0; i < tmplH; ++i, tmpl+=tmplStride, img+=srcStride) + { + size_t j = 0; + for(; j < tmplroiw; j+=8) + { + uint8x8_t vtmpl = vld1_u8(tmpl + j); + + uint8x8_t vimg0 = vld1_u8(img + j + c + 0); + uint8x8_t vimg1 = vld1_u8(img + j + c + 1); + uint8x8_t vimg2 = vld1_u8(img + j + c + 2); + uint8x8_t vimg3 = vld1_u8(img + j + c + 3); + + uint16x8_t vd0 = vmull_u8(vtmpl, vimg0); + uint16x8_t vd1 = vmull_u8(vtmpl, vimg1); + uint16x8_t vd2 = vmull_u8(vtmpl, vimg2); + uint16x8_t vd3 = vmull_u8(vtmpl, vimg3); + + vdot0 = vpadalq_u16(vdot0, vd0); + vdot1 = vpadalq_u16(vdot1, vd1); + vdot2 = vpadalq_u16(vdot2, vd2); + vdot3 = vpadalq_u16(vdot3, vd3); + } + for(; j < tmplW; ++j) + { + dot[0] += tmpl[j] * img[j + c + 0]; + dot[1] += tmpl[j] * img[j + c + 1]; + dot[2] += tmpl[j] * img[j + c + 2]; + dot[3] += tmpl[j] * img[j + c + 3]; + } + } + uint32x4_t vdotx = vld1q_u32(dot); + uint32x2_t vdot_0 = vpadd_u32(vget_low_u32(vdot0), vget_high_u32(vdot0)); + uint32x2_t vdot_1 = vpadd_u32(vget_low_u32(vdot1), vget_high_u32(vdot1)); + uint32x2_t vdot_2 = vpadd_u32(vget_low_u32(vdot2), vget_high_u32(vdot2)); + uint32x2_t vdot_3 = vpadd_u32(vget_low_u32(vdot3), vget_high_u32(vdot3)); + uint32x2_t vdot_01 = vpadd_u32(vdot_0, vdot_1); + uint32x2_t vdot_23 = vpadd_u32(vdot_2, vdot_3); + + vst1q_f32(corr + c, vcvtq_f32_u32(vaddq_u32(vdotx, vcombine_u32(vdot_01, vdot_23)))); + } +#endif + + for(; c < dstroiw2; c+=2) + { + u32 dot[2] = {0, 0}; + uint32x4_t vdot0 = vmovq_n_u32(0); + uint32x4_t vdot1 = vmovq_n_u32(0); + const u8 *img = imgrrow; + const u8 *tmpl = tmplBase; + for(size_t i = 0; i < tmplH; ++i, tmpl+=tmplStride, img+=srcStride) + { + size_t j = 0; + for(; j < tmplroiw; j+=8) + { + uint8x8_t vtmpl = vld1_u8(tmpl + j); + + uint8x8_t vimg0 = vld1_u8(img + j + c + 0); + uint8x8_t vimg1 = vld1_u8(img + j + c + 1); + + uint16x8_t vd0 = vmull_u8(vtmpl, vimg0); + uint16x8_t vd1 = vmull_u8(vtmpl, vimg1); + + vdot0 = vpadalq_u16(vdot0, vd0); + vdot1 = vpadalq_u16(vdot1, vd1); + } + for(; j < tmplW; ++j) + { + dot[0] += tmpl[j] * img[j + c + 0]; + dot[1] += tmpl[j] * img[j + c + 1]; + } + } + uint32x2_t vdotx = vld1_u32(dot); + uint32x2_t vdot_0 = vpadd_u32(vget_low_u32(vdot0), vget_high_u32(vdot0)); + uint32x2_t vdot_1 = vpadd_u32(vget_low_u32(vdot1), vget_high_u32(vdot1)); + uint32x2_t vdot_ = vpadd_u32(vdot_0, vdot_1); + vst1_f32(corr + c, vcvt_f32_u32(vadd_u32(vdotx, vdot_))); + } + + for(; c < dstW; ++c) + { + u32 dot = 0; + uint32x4_t vdot = vmovq_n_u32(0); + const u8 *img = imgrrow; + const u8 *tmpl = tmplBase; + for(size_t i = 0; i < tmplH; ++i, tmpl+=tmplStride, img+=srcStride) + { + size_t j = 0; + for(; j < tmplroiw; j+=8) + { + uint8x8_t vtmpl = vld1_u8(tmpl + j); + uint8x8_t vimg = vld1_u8(img + j + c); + uint16x8_t vd = vmull_u8(vtmpl, vimg); + vdot = vpadalq_u16(vdot, vd); + } + for(; j < tmplW; ++j) + dot += tmpl[j] * img[j + c]; + } + u32 wdot[2]; + vst1_u32(wdot, vpadd_u32(vget_low_u32(vdot), vget_high_u32(vdot))); + dot += wdot[0] + wdot[1]; + corr[c] = (f32)dot; + } + } + } + + if(normalize) + { + f32 tn = std::sqrt((f32)normL2(tmplSize, tmplBase, tmplStride)); + + size_t iw = srcSize.width+1; + size_t ih = srcSize.height+1; + std::vector _sqsum(iw*ih); + f64 *sqsum = &_sqsum[0]; + memset(sqsum, 0, iw*sizeof(f64)); + for(size_t i = 1; i < ih; ++i) + sqsum[iw*i] = 0.; + sqrIntegral(srcSize, srcBase, srcStride, sqsum + iw + 1, iw*sizeof(f64)); + + for(size_t i = 0; i < dstH; ++i) + { + f32 *result = internal::getRowPtr(dstBase, dstStride, i); + for(size_t j = 0; j < dstW; ++j) + { + double s2 = sqsum[iw*i + j] + + sqsum[iw*(i + tmplSize.height) + j + tmplSize.width] - + sqsum[iw*(i + tmplSize.height) + j] - + sqsum[iw*i + j + tmplSize.width]; + + result[j] /= tn * std::sqrt(s2); + } + } + } +#else + (void)srcSize; + (void)srcBase; + (void)srcStride; + (void)tmplBase; + (void)tmplStride; + (void)dstBase; + (void)dstStride; + (void)normalize; +#endif +} + +} // namespace CAROTENE_NS diff --git a/3rdparty/carotene/src/threshold.cpp b/3rdparty/carotene/src/threshold.cpp new file mode 100644 index 0000000000..8e03798b02 --- /dev/null +++ b/3rdparty/carotene/src/threshold.cpp @@ -0,0 +1,1627 @@ +/* + * By downloading, copying, installing or using the software you agree to this license. + * If you do not agree to this license, do not download, install, + * copy or use the software. + * + * + * License Agreement + * For Open Source Computer Vision Library + * (3-clause BSD License) + * + * Copyright (C) 2012-2015, NVIDIA Corporation, all rights reserved. + * Third party copyrights are property of their respective owners. + * + * Redistribution and use in source and binary forms, with or without modification, + * are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * * Neither the names of the copyright holders nor the names of the contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * This software is provided by the copyright holders and contributors "as is" and + * any express or implied warranties, including, but not limited to, the implied + * warranties of merchantability and fitness for a particular purpose are disclaimed. + * In no event shall copyright holders or contributors be liable for any direct, + * indirect, incidental, special, exemplary, or consequential damages + * (including, but not limited to, procurement of substitute goods or services; + * loss of use, data, or profits; or business interruption) however caused + * and on any theory of liability, whether in contract, strict liability, + * or tort (including negligence or otherwise) arising in any way out of + * the use of this software, even if advised of the possibility of such damage. + */ + +#include "common.hpp" + +namespace CAROTENE_NS { + +void thresholdBinary(const Size2D &size, + const u8 *srcBase, ptrdiff_t srcStride, + u8 *dstBase, ptrdiff_t dstStride, + u8 threshold, u8 trueValue, u8 falseValue) +{ + internal::assertSupportedConfiguration(); +#ifdef CAROTENE_NEON + uint8x16_t vthreshold = vdupq_n_u8(threshold); + uint8x8_t vthreshold8 = vdup_n_u8(threshold); + size_t roiw32 = size.width >= 31 ? size.width - 31 : 0; + size_t roiw8 = size.width >= 7 ? size.width - 7 : 0; + + if(trueValue == 255 && falseValue == 0) + { + for (size_t i = 0; i < size.height; ++i) { + const u8* src = internal::getRowPtr(srcBase, srcStride, i); + u8* dst = internal::getRowPtr(dstBase, dstStride, i); + size_t j = 0; + + for (; j < roiw32; j += 32) { + internal::prefetch(src + j); + uint8x16_t v0 = vld1q_u8(src + j); + uint8x16_t v1 = vld1q_u8(src + j + 16); + uint8x16_t r0 = vcgtq_u8(v0, vthreshold); + uint8x16_t r1 = vcgtq_u8(v1, vthreshold); + vst1q_u8(dst + j, r0); + vst1q_u8(dst + j + 16, r1); + } + for (; j < roiw8; j += 8) { + uint8x8_t v0 = vld1_u8(src + j); + uint8x8_t r0 = vcgt_u8(v0, vthreshold8); + vst1_u8(dst + j, r0); + } + + for (; j < size.width; j++) { + *(dst + j) = *(src + j) > threshold ? 255 : 0; + } + } + } + else + { + uint8x16_t vtrue_value = vdupq_n_u8(trueValue); + uint8x8_t vtrue_value8 = vdup_n_u8(trueValue); + uint8x16_t vfalse_value = vdupq_n_u8(falseValue); + uint8x8_t vfalse_value8 = vdup_n_u8(falseValue); + + for (size_t i = 0; i < size.height; ++i) { + const u8* src = internal::getRowPtr(srcBase, srcStride, i); + u8* dst = internal::getRowPtr(dstBase, dstStride, i); + size_t j = 0; + + for (; j < roiw32; j += 32) { + internal::prefetch(src + j); + uint8x16_t v0 = vld1q_u8(src + j); + uint8x16_t v1 = vld1q_u8(src + j + 16); + uint8x16_t r0 = vcgtq_u8(v0, vthreshold); + uint8x16_t r1 = vcgtq_u8(v1, vthreshold); + uint8x16_t r0a = vbslq_u8(r0, vtrue_value, vfalse_value); + uint8x16_t r1a = vbslq_u8(r1, vtrue_value, vfalse_value); + vst1q_u8(dst + j, r0a); + vst1q_u8(dst + j + 16, r1a); + } + for (; j < roiw8; j += 8) { + uint8x8_t v0 = vld1_u8(src + j); + uint8x8_t r0 = vcgt_u8(v0, vthreshold8); + uint8x8_t r0a = vbsl_u8(r0, vtrue_value8, vfalse_value8); + vst1_u8(dst + j, r0a); + } + + for (; j < size.width; j++) { + *(dst + j) = *(src + j) > threshold ? trueValue : falseValue; + } + } + } +#else + (void)size; + (void)srcBase; + (void)srcStride; + (void)dstBase; + (void)dstStride; + (void)threshold; + (void)trueValue; + (void)falseValue; +#endif +} + +void thresholdRange(const Size2D &size, + const u8 *srcBase, ptrdiff_t srcStride, + u8 *dstBase, ptrdiff_t dstStride, + u8 lowerThreshold, u8 upperThreshold, + u8 trueValue, u8 falseValue) +{ + internal::assertSupportedConfiguration(); + +#ifdef CAROTENE_NEON + uint8x16_t v_lower = vdupq_n_u8(lowerThreshold), v_upper = vdupq_n_u8(upperThreshold); + uint8x8_t v_lower8 = vdup_n_u8(lowerThreshold), v_upper8 = vdup_n_u8(upperThreshold); + size_t roiw32 = size.width >= 31 ? size.width - 31 : 0; + size_t roiw8 = size.width >= 7 ? size.width - 7 : 0; + + if(trueValue == 255 && falseValue == 0) + { + for (size_t i = 0; i < size.height; ++i) + { + const u8 * src = internal::getRowPtr(srcBase, srcStride, i); + u8 * dst = internal::getRowPtr(dstBase, dstStride, i); + size_t j = 0; + + for (; j < roiw32; j += 32) + { + internal::prefetch(src + j); + uint8x16_t v_src0 = vld1q_u8(src + j), v_src1 = vld1q_u8(src + j + 16); + uint8x16_t v_dst0 = vandq_u8(vcgeq_u8(v_src0, v_lower), vcleq_u8(v_src0, v_upper)); + uint8x16_t v_dst1 = vandq_u8(vcgeq_u8(v_src1, v_lower), vcleq_u8(v_src1, v_upper)); + vst1q_u8(dst + j, v_dst0); + vst1q_u8(dst + j + 16, v_dst1); + } + for (; j < roiw8; j += 8) + { + uint8x8_t v_src = vld1_u8(src + j); + uint8x8_t v_dst = vand_u8(vcge_u8(v_src, v_lower8), vcle_u8(v_src, v_upper8)); + vst1_u8(dst + j, v_dst); + } + + for (; j < size.width; j++) + { + u8 srcVal = src[j]; + dst[j] = lowerThreshold <= srcVal && srcVal <= upperThreshold ? 255 : 0; + } + } + } + else + { + uint8x16_t vtrue_value = vdupq_n_u8(trueValue); + uint8x8_t vtrue_value8 = vdup_n_u8(trueValue); + uint8x16_t vfalse_value = vdupq_n_u8(falseValue); + uint8x8_t vfalse_value8 = vdup_n_u8(falseValue); + + for (size_t i = 0; i < size.height; ++i) + { + const u8 * src = internal::getRowPtr(srcBase, srcStride, i); + u8 * dst = internal::getRowPtr(dstBase, dstStride, i); + size_t j = 0; + + for (; j < roiw32; j += 32) + { + internal::prefetch(src + j); + uint8x16_t v_src0 = vld1q_u8(src + j), v_src1 = vld1q_u8(src + j + 16); + uint8x16_t v_dst0 = vandq_u8(vcgeq_u8(v_src0, v_lower), vcleq_u8(v_src0, v_upper)); + uint8x16_t v_dst1 = vandq_u8(vcgeq_u8(v_src1, v_lower), vcleq_u8(v_src1, v_upper)); + v_dst0 = vbslq_u8(v_dst0, vtrue_value, vfalse_value); + v_dst1 = vbslq_u8(v_dst1, vtrue_value, vfalse_value); + vst1q_u8(dst + j, v_dst0); + vst1q_u8(dst + j + 16, v_dst1); + } + for (; j < roiw8; j += 8) + { + uint8x8_t v_src = vld1_u8(src + j); + uint8x8_t v_dst = vand_u8(vcge_u8(v_src, v_lower8), vcle_u8(v_src, v_upper8)); + v_dst = vbsl_u8(v_dst, vtrue_value8, vfalse_value8); + vst1_u8(dst + j, v_dst); + } + + for (; j < size.width; j++) + { + u8 srcVal = src[j]; + dst[j] = lowerThreshold <= srcVal && srcVal <= upperThreshold ? trueValue : falseValue; + } + } + } +#else + (void)size; + (void)srcBase; + (void)srcStride; + (void)dstBase; + (void)dstStride; + (void)lowerThreshold; + (void)upperThreshold; + (void)trueValue; + (void)falseValue; +#endif +} + +void thresholdBinary(const Size2D &size, + const u8 *srcBase, ptrdiff_t srcStride, + u8 *dstBase, ptrdiff_t dstStride, + u8 threshold, u8 value) +{ + internal::assertSupportedConfiguration(); +#ifdef CAROTENE_NEON + uint8x16_t vthreshold = vdupq_n_u8(threshold); + uint8x16_t vvalue = vdupq_n_u8(value); + uint8x8_t vthreshold8 = vdup_n_u8(threshold); + uint8x8_t vvalue8 = vdup_n_u8(value); + size_t roiw32 = size.width >= 31 ? size.width - 31 : 0; + size_t roiw8 = size.width >= 7 ? size.width - 7 : 0; + + for (size_t i = 0; i < size.height; ++i) + { + const u8* src = internal::getRowPtr(srcBase, srcStride, i); + u8* dst = internal::getRowPtr(dstBase, dstStride, i); + size_t j = 0; + + for (; j < roiw32; j += 32) + { + internal::prefetch(src + j); + uint8x16_t v0 = vld1q_u8(src + j); + uint8x16_t v1 = vld1q_u8(src + j + 16); + uint8x16_t r0 = vcgtq_u8(v0, vthreshold); + uint8x16_t r1 = vcgtq_u8(v1, vthreshold); + uint8x16_t r0a = vandq_u8(r0, vvalue); + uint8x16_t r1a = vandq_u8(r1, vvalue); + vst1q_u8(dst + j, r0a); + vst1q_u8(dst + j + 16, r1a); + } + for (; j < roiw8; j += 8) + { + uint8x8_t v0 = vld1_u8(src + j); + uint8x8_t r0 = vcgt_u8(v0, vthreshold8); + uint8x8_t r0a = vand_u8(r0, vvalue8); + vst1_u8(dst + j, r0a); + } + for (; j < size.width; j++) + { + *(dst + j) = *(src + j) > threshold ? value : 0; + } + } +#else + (void)size; + (void)srcBase; + (void)srcStride; + (void)dstBase; + (void)dstStride; + (void)threshold; + (void)value; +#endif +} + +void thresholdBinaryInv(const Size2D &size, + const u8 *srcBase, ptrdiff_t srcStride, + u8 *dstBase, ptrdiff_t dstStride, + u8 threshold, u8 value) +{ + internal::assertSupportedConfiguration(); +#ifdef CAROTENE_NEON + uint8x16_t vthreshold = vdupq_n_u8(threshold); + uint8x16_t vvalue = vdupq_n_u8(value); + uint8x8_t vthreshold8 = vdup_n_u8(threshold); + uint8x8_t vvalue8 = vdup_n_u8(value); + size_t roiw32 = size.width >= 31 ? size.width - 31 : 0; + size_t roiw8 = size.width >= 7 ? size.width - 7 : 0; + + for (size_t i = 0; i < size.height; ++i) + { + const u8* src = internal::getRowPtr(srcBase, srcStride, i); + u8* dst = internal::getRowPtr(dstBase, dstStride, i); + size_t j = 0; + + for (; j < roiw32; j += 32) + { + internal::prefetch(src + j); + uint8x16_t v0 = vld1q_u8(src + j); + uint8x16_t v1 = vld1q_u8(src + j + 16); + uint8x16_t r0 = vcleq_u8(v0, vthreshold); + uint8x16_t r1 = vcleq_u8(v1, vthreshold); + uint8x16_t r0a = vandq_u8(r0, vvalue); + uint8x16_t r1a = vandq_u8(r1, vvalue); + vst1q_u8(dst + j, r0a); + vst1q_u8(dst + j + 16, r1a); + } + for (; j < roiw8; j += 8) + { + uint8x8_t v0 = vld1_u8(src + j); + uint8x8_t r0 = vcle_u8(v0, vthreshold8); + uint8x8_t r0a = vand_u8(r0, vvalue8); + vst1_u8(dst + j, r0a); + } + for (; j < size.width; j++) + { + *(dst + j) = *(src + j) > threshold ? 0 : value; + } + } +#else + (void)size; + (void)srcBase; + (void)srcStride; + (void)dstBase; + (void)dstStride; + (void)threshold; + (void)value; +#endif +} + +void thresholdTruncate(const Size2D &size, + const u8 *srcBase, ptrdiff_t srcStride, + u8 *dstBase, ptrdiff_t dstStride, + u8 threshold) +{ + internal::assertSupportedConfiguration(); +#ifdef CAROTENE_NEON + uint8x16_t vthreshold = vdupq_n_u8(threshold); + uint8x8_t vthreshold8 = vdup_n_u8(threshold); + size_t roiw32 = size.width >= 31 ? size.width - 31 : 0; + size_t roiw8 = size.width >= 7 ? size.width - 7 : 0; + + for (size_t i = 0; i < size.height; ++i) + { + const u8* src = internal::getRowPtr(srcBase, srcStride, i); + u8* dst = internal::getRowPtr(dstBase, dstStride, i); + size_t j = 0; + + for (; j < roiw32; j += 32) + { + internal::prefetch(src + j); + uint8x16_t v0 = vld1q_u8(src + j); + uint8x16_t v1 = vld1q_u8(src + j + 16); + uint8x16_t r0 = vqsubq_u8(v0, vthreshold); + uint8x16_t r1 = vqsubq_u8(v1, vthreshold); + uint8x16_t r0a = vqsubq_u8(v0, r0); + uint8x16_t r1a = vqsubq_u8(v1, r1); + vst1q_u8(dst + j, r0a); + vst1q_u8(dst + j + 16, r1a); + } + for (; j < roiw8; j += 8) + { + uint8x8_t v0 = vld1_u8(src + j); + uint8x8_t r0 = vqsub_u8(v0, vthreshold8); + uint8x8_t r0a = vqsub_u8(v0, r0); + vst1_u8(dst + j, r0a); + } + for (; j < size.width; j++) + { + *(dst + j) = *(src + j) > threshold ? threshold : *(src + j); + } + } +#else + (void)size; + (void)srcBase; + (void)srcStride; + (void)dstBase; + (void)dstStride; + (void)threshold; +#endif +} + +void thresholdToZero(const Size2D &size, + const u8 *srcBase, ptrdiff_t srcStride, + u8 *dstBase, ptrdiff_t dstStride, + u8 threshold) +{ + internal::assertSupportedConfiguration(); +#ifdef CAROTENE_NEON + uint8x16_t vthreshold = vdupq_n_u8(threshold); + uint8x8_t vthreshold8 = vdup_n_u8(threshold); + size_t roiw32 = size.width >= 31 ? size.width - 31 : 0; + size_t roiw8 = size.width >= 7 ? size.width - 7 : 0; + + for (size_t i = 0; i < size.height; ++i) + { + const u8* src = internal::getRowPtr(srcBase, srcStride, i); + u8* dst = internal::getRowPtr(dstBase, dstStride, i); + size_t j = 0; + + for (; j < roiw32; j += 32) + { + internal::prefetch(src + j); + uint8x16_t v0 = vld1q_u8(src + j); + uint8x16_t v1 = vld1q_u8(src + j + 16); + uint8x16_t r0 = vcgtq_u8(v0, vthreshold); + uint8x16_t r1 = vcgtq_u8(v1, vthreshold); + uint8x16_t r0a = vandq_u8(v0, r0); + uint8x16_t r1a = vandq_u8(v1, r1); + vst1q_u8(dst + j, r0a); + vst1q_u8(dst + j + 16, r1a); + } + for (; j < roiw8; j += 8) + { + uint8x8_t v0 = vld1_u8(src + j); + uint8x8_t r0 = vcgt_u8(v0, vthreshold8); + uint8x8_t r0a = vand_u8(v0, r0); + vst1_u8(dst + j, r0a); + } + for (; j < size.width; j++) + { + *(dst + j) = *(src + j) > threshold ? *(src + j) : 0; + } + } +#else + (void)size; + (void)srcBase; + (void)srcStride; + (void)dstBase; + (void)dstStride; + (void)threshold; +#endif +} + +void thresholdToZeroInv(const Size2D &size, + const u8 *srcBase, ptrdiff_t srcStride, + u8 *dstBase, ptrdiff_t dstStride, + u8 threshold) +{ + internal::assertSupportedConfiguration(); +#ifdef CAROTENE_NEON + uint8x16_t vthreshold = vdupq_n_u8(threshold); + uint8x8_t vthreshold8 = vdup_n_u8(threshold); + size_t roiw32 = size.width >= 31 ? size.width - 31 : 0; + size_t roiw8 = size.width >= 7 ? size.width - 7 : 0; + + for (size_t i = 0; i < size.height; ++i) + { + const u8* src = internal::getRowPtr(srcBase, srcStride, i); + u8* dst = internal::getRowPtr(dstBase, dstStride, i); + size_t j = 0; + + for (; j < roiw32; j += 32) + { + internal::prefetch(src + j); + uint8x16_t v0 = vld1q_u8(src + j); + uint8x16_t v1 = vld1q_u8(src + j + 16); + uint8x16_t r0 = vcgtq_u8(v0, vthreshold); + uint8x16_t r1 = vcgtq_u8(v1, vthreshold); + uint8x16_t r0a = vbicq_u8(v0, r0); + uint8x16_t r1a = vbicq_u8(v1, r1); + vst1q_u8(dst + j, r0a); + vst1q_u8(dst + j + 16, r1a); + } + for (; j < roiw8; j += 8) + { + uint8x8_t v0 = vld1_u8(src + j); + uint8x8_t r0 = vcgt_u8(v0, vthreshold8); + uint8x8_t r0a = vbic_u8(v0, r0); + vst1_u8(dst + j, r0a); + } + for (; j < size.width; j++) + { + *(dst + j) = *(src + j) > threshold ? 0 : *(src + j); + } + } +#else + (void)size; + (void)srcBase; + (void)srcStride; + (void)dstBase; + (void)dstStride; + (void)threshold; +#endif +} + +void thresholdBinary(const Size2D &size, + const s8 *srcBase, ptrdiff_t srcStride, + s8 *dstBase, ptrdiff_t dstStride, + s8 threshold, s8 value) +{ + internal::assertSupportedConfiguration(); +#ifdef CAROTENE_NEON + int8x16_t vthreshold = vdupq_n_s8(threshold); + int8x16_t vvalue = vdupq_n_s8(value); + int8x8_t vthreshold8 = vdup_n_s8(threshold); + int8x8_t vvalue8 = vdup_n_s8(value); + size_t roiw32 = size.width >= 31 ? size.width - 31 : 0; + size_t roiw8 = size.width >= 7 ? size.width - 7 : 0; + + for (size_t i = 0; i < size.height; ++i) + { + const s8* src = internal::getRowPtr(srcBase, srcStride, i); + s8* dst = internal::getRowPtr(dstBase, dstStride, i); + size_t j = 0; + + for (; j < roiw32; j += 32) + { + internal::prefetch(src + j); + int8x16_t v0 = vld1q_s8(src + j); + int8x16_t v1 = vld1q_s8(src + j + 16); + int8x16_t r0 = vreinterpretq_s8_u8(vcgtq_s8(v0, vthreshold)); + int8x16_t r1 = vreinterpretq_s8_u8(vcgtq_s8(v1, vthreshold)); + int8x16_t r0a = vandq_s8(r0, vvalue); + int8x16_t r1a = vandq_s8(r1, vvalue); + vst1q_s8(dst + j, r0a); + vst1q_s8(dst + j + 16, r1a); + } + for (; j < roiw8; j += 8) + { + int8x8_t v0 = vld1_s8(src + j); + int8x8_t r0 = vreinterpret_s8_u8(vcgt_s8(v0, vthreshold8)); + int8x8_t r0a = vand_s8(r0, vvalue8); + vst1_s8(dst + j, r0a); + } + for (; j < size.width; j++) + { + *(dst + j) = *(src + j) > threshold ? value : 0; + } + } +#else + (void)size; + (void)srcBase; + (void)srcStride; + (void)dstBase; + (void)dstStride; + (void)threshold; + (void)value; +#endif +} + +void thresholdBinaryInv(const Size2D &size, + const s8 *srcBase, ptrdiff_t srcStride, + s8 *dstBase, ptrdiff_t dstStride, + s8 threshold, s8 value) +{ + internal::assertSupportedConfiguration(); +#ifdef CAROTENE_NEON + int8x16_t vthreshold = vdupq_n_s8(threshold); + int8x16_t vvalue = vdupq_n_s8(value); + int8x8_t vthreshold8 = vdup_n_s8(threshold); + int8x8_t vvalue8 = vdup_n_s8(value); + size_t roiw32 = size.width >= 31 ? size.width - 31 : 0; + size_t roiw8 = size.width >= 7 ? size.width - 7 : 0; + + for (size_t i = 0; i < size.height; ++i) + { + const s8* src = internal::getRowPtr(srcBase, srcStride, i); + s8* dst = internal::getRowPtr(dstBase, dstStride, i); + size_t j = 0; + + for (; j < roiw32; j += 32) + { + internal::prefetch(src + j); + int8x16_t v0 = vld1q_s8(src + j); + int8x16_t v1 = vld1q_s8(src + j + 16); + int8x16_t r0 = vreinterpretq_s8_u8(vcleq_s8(v0, vthreshold)); + int8x16_t r1 = vreinterpretq_s8_u8(vcleq_s8(v1, vthreshold)); + int8x16_t r0a = vandq_s8(r0, vvalue); + int8x16_t r1a = vandq_s8(r1, vvalue); + vst1q_s8(dst + j, r0a); + vst1q_s8(dst + j + 16, r1a); + } + for (; j < roiw8; j += 8) + { + int8x8_t v0 = vld1_s8(src + j); + int8x8_t r0 = vreinterpret_s8_u8(vcle_s8(v0, vthreshold8)); + int8x8_t r0a = vand_s8(r0, vvalue8); + vst1_s8(dst + j, r0a); + } + for (; j < size.width; j++) + { + *(dst + j) = *(src + j) > threshold ? 0 : value; + } + } +#else + (void)size; + (void)srcBase; + (void)srcStride; + (void)dstBase; + (void)dstStride; + (void)threshold; + (void)value; +#endif +} + +void thresholdTruncate(const Size2D &size, + const s8 *srcBase, ptrdiff_t srcStride, + s8 *dstBase, ptrdiff_t dstStride, + s8 threshold) +{ + internal::assertSupportedConfiguration(); +#ifdef CAROTENE_NEON + int8x16_t vthreshold = vdupq_n_s8(threshold); + int8x8_t vthreshold8 = vdup_n_s8(threshold); + size_t roiw32 = size.width >= 31 ? size.width - 31 : 0; + size_t roiw8 = size.width >= 7 ? size.width - 7 : 0; + + for (size_t i = 0; i < size.height; ++i) + { + const s8* src = internal::getRowPtr(srcBase, srcStride, i); + s8* dst = internal::getRowPtr(dstBase, dstStride, i); + size_t j = 0; + + for (; j < roiw32; j += 32) + { + internal::prefetch(src + j); + int8x16_t v0 = vld1q_s8(src + j); + int8x16_t v1 = vld1q_s8(src + j + 16); + int8x16_t r0 = vqsubq_s8(v0, vthreshold); + int8x16_t r1 = vqsubq_s8(v1, vthreshold); + int8x16_t r0a = vqsubq_s8(v0, r0); + int8x16_t r1a = vqsubq_s8(v1, r1); + vst1q_s8(dst + j, r0a); + vst1q_s8(dst + j + 16, r1a); + } + for (; j < roiw8; j += 8) + { + int8x8_t v0 = vld1_s8(src + j); + int8x8_t r0 = vqsub_s8(v0, vthreshold8); + int8x8_t r0a = vqsub_s8(v0, r0); + vst1_s8(dst + j, r0a); + } + for (; j < size.width; j++) + { + *(dst + j) = *(src + j) > threshold ? threshold : *(src + j); + } + } +#else + (void)size; + (void)srcBase; + (void)srcStride; + (void)dstBase; + (void)dstStride; + (void)threshold; +#endif +} + +void thresholdToZero(const Size2D &size, + const s8 *srcBase, ptrdiff_t srcStride, + s8 *dstBase, ptrdiff_t dstStride, + s8 threshold) +{ + internal::assertSupportedConfiguration(); +#ifdef CAROTENE_NEON + int8x16_t vthreshold = vdupq_n_s8(threshold); + int8x8_t vthreshold8 = vdup_n_s8(threshold); + size_t roiw32 = size.width >= 31 ? size.width - 31 : 0; + size_t roiw8 = size.width >= 7 ? size.width - 7 : 0; + + for (size_t i = 0; i < size.height; ++i) + { + const s8* src = internal::getRowPtr(srcBase, srcStride, i); + s8* dst = internal::getRowPtr(dstBase, dstStride, i); + size_t j = 0; + + for (; j < roiw32; j += 32) + { + internal::prefetch(src + j); + int8x16_t v0 = vld1q_s8(src + j); + int8x16_t v1 = vld1q_s8(src + j + 16); + int8x16_t r0 = vreinterpretq_s8_u8(vcgtq_s8(v0, vthreshold)); + int8x16_t r1 = vreinterpretq_s8_u8(vcgtq_s8(v1, vthreshold)); + int8x16_t r0a = vandq_s8(v0, r0); + int8x16_t r1a = vandq_s8(v1, r1); + vst1q_s8(dst + j, r0a); + vst1q_s8(dst + j + 16, r1a); + } + for (; j < roiw8; j += 8) + { + int8x8_t v0 = vld1_s8(src + j); + int8x8_t r0 = vreinterpret_s8_u8(vcgt_s8(v0, vthreshold8)); + int8x8_t r0a = vand_s8(v0, r0); + vst1_s8(dst + j, r0a); + } + for (; j < size.width; j++) + { + *(dst + j) = *(src + j) > threshold ? *(src + j) : 0; + } + } +#else + (void)size; + (void)srcBase; + (void)srcStride; + (void)dstBase; + (void)dstStride; + (void)threshold; +#endif +} + +void thresholdToZeroInv(const Size2D &size, + const s8 *srcBase, ptrdiff_t srcStride, + s8 *dstBase, ptrdiff_t dstStride, + s8 threshold) +{ + internal::assertSupportedConfiguration(); +#ifdef CAROTENE_NEON + int8x16_t vthreshold = vdupq_n_s8(threshold); + int8x8_t vthreshold8 = vdup_n_s8(threshold); + size_t roiw32 = size.width >= 31 ? size.width - 31 : 0; + size_t roiw8 = size.width >= 7 ? size.width - 7 : 0; + + for (size_t i = 0; i < size.height; ++i) + { + const s8* src = internal::getRowPtr(srcBase, srcStride, i); + s8* dst = internal::getRowPtr(dstBase, dstStride, i); + size_t j = 0; + + for (; j < roiw32; j += 32) + { + internal::prefetch(src + j); + int8x16_t v0 = vld1q_s8(src + j); + int8x16_t v1 = vld1q_s8(src + j + 16); + int8x16_t r0 = vreinterpretq_s8_u8(vcgtq_s8(v0, vthreshold)); + int8x16_t r1 = vreinterpretq_s8_u8(vcgtq_s8(v1, vthreshold)); + int8x16_t r0a = vbicq_s8(v0, r0); + int8x16_t r1a = vbicq_s8(v1, r1); + vst1q_s8(dst + j, r0a); + vst1q_s8(dst + j + 16, r1a); + } + for (; j < roiw8; j += 8) + { + int8x8_t v0 = vld1_s8(src + j); + int8x8_t r0 = vreinterpret_s8_u8(vcgt_s8(v0, vthreshold8)); + int8x8_t r0a = vbic_s8(v0, r0); + vst1_s8(dst + j, r0a); + } + for (; j < size.width; j++) + { + *(dst + j) = *(src + j) > threshold ? 0 : *(src + j); + } + } +#else + (void)size; + (void)srcBase; + (void)srcStride; + (void)dstBase; + (void)dstStride; + (void)threshold; +#endif +} + +void thresholdBinary(const Size2D &size, + const s16 *srcBase, ptrdiff_t srcStride, + s16 *dstBase, ptrdiff_t dstStride, + s16 threshold, s16 value) +{ + internal::assertSupportedConfiguration(); +#ifdef CAROTENE_NEON + int16x8_t vthreshold16 = vdupq_n_s16(threshold); + int16x8_t vvalue16 = vdupq_n_s16(value); + size_t roiw16 = size.width >= 15 ? size.width - 15 : 0; + + for (size_t i = 0; i < size.height; ++i) + { + const s16* src = internal::getRowPtr(srcBase, srcStride, i); + s16* dst = internal::getRowPtr(dstBase, dstStride, i); + size_t j = 0; + + for (; j < roiw16; j += 16) + { + internal::prefetch(src + j); + int16x8_t v0 = vld1q_s16(src + j); + int16x8_t v1 = vld1q_s16(src + j + 8); + uint16x8_t r0 = vcgtq_s16(v0, vthreshold16); + uint16x8_t r1 = vcgtq_s16(v1, vthreshold16); + uint16x8_t r0a = vandq_u16(r0, vreinterpretq_u16_s16(vvalue16)); + uint16x8_t r1a = vandq_u16(r1, vreinterpretq_u16_s16(vvalue16)); + vst1q_u16((u16*)dst + j, r0a); + vst1q_u16((u16*)dst + j + 8, r1a); + } + for (; j < size.width; j++) + { + *(dst + j) = *(src + j) > threshold ? value : 0; + } + } +#else + (void)size; + (void)srcBase; + (void)srcStride; + (void)dstBase; + (void)dstStride; + (void)threshold; + (void)value; +#endif +} + +void thresholdBinaryInv(const Size2D &size, + const s16 *srcBase, ptrdiff_t srcStride, + s16 *dstBase, ptrdiff_t dstStride, + s16 threshold, s16 value) +{ + internal::assertSupportedConfiguration(); +#ifdef CAROTENE_NEON + int16x8_t vthreshold16 = vdupq_n_s16(threshold); + int16x8_t vvalue16 = vdupq_n_s16(value); + size_t roiw16 = size.width >= 15 ? size.width - 15 : 0; + + for (size_t i = 0; i < size.height; ++i) + { + const s16* src = internal::getRowPtr(srcBase, srcStride, i); + s16* dst = internal::getRowPtr(dstBase, dstStride, i); + size_t j = 0; + + for (; j < roiw16; j += 16) + { + internal::prefetch(src + j); + int16x8_t v0 = vld1q_s16(src + j); + int16x8_t v1 = vld1q_s16(src + j + 8); + uint16x8_t r0 = vcleq_s16(v0, vthreshold16); + uint16x8_t r1 = vcleq_s16(v1, vthreshold16); + uint16x8_t r0a = vandq_u16(r0, vreinterpretq_u16_s16(vvalue16)); + uint16x8_t r1a = vandq_u16(r1, vreinterpretq_u16_s16(vvalue16)); + vst1q_s16(dst + j, vreinterpretq_s16_u16(r0a)); + vst1q_s16(dst + j + 8, vreinterpretq_s16_u16(r1a)); + } + for (; j < size.width; j++) + { + *(dst + j) = *(src + j) > threshold ? 0 : value; + } + } +#else + (void)size; + (void)srcBase; + (void)srcStride; + (void)dstBase; + (void)dstStride; + (void)threshold; + (void)value; +#endif +} + +void thresholdTruncate(const Size2D &size, + const s16 *srcBase, ptrdiff_t srcStride, + s16 *dstBase, ptrdiff_t dstStride, + s16 threshold) +{ + internal::assertSupportedConfiguration(); +#ifdef CAROTENE_NEON + int16x8_t vthreshold16 = vdupq_n_s16(threshold); + size_t roiw16 = size.width >= 15 ? size.width - 15 : 0; + + for (size_t i = 0; i < size.height; ++i) + { + const s16* src = internal::getRowPtr(srcBase, srcStride, i); + s16* dst = internal::getRowPtr(dstBase, dstStride, i); + size_t j = 0; + + for (; j < roiw16; j += 16) + { + internal::prefetch(src + j); + int16x8_t v0 = vld1q_s16(src + j); + int16x8_t v1 = vld1q_s16(src + j + 8); + int16x8_t r0 = vminq_s16(v0, vthreshold16); + int16x8_t r1 = vminq_s16(v1, vthreshold16); + vst1q_s16(dst + j, r0); + vst1q_s16(dst + j + 8, r1); + } + for (; j < size.width; j++) + { + *(dst + j) = *(src + j) > threshold ? threshold : *(src + j); + } + } +#else + (void)size; + (void)srcBase; + (void)srcStride; + (void)dstBase; + (void)dstStride; + (void)threshold; +#endif +} + +void thresholdToZero(const Size2D &size, + const s16 *srcBase, ptrdiff_t srcStride, + s16 *dstBase, ptrdiff_t dstStride, + s16 threshold) +{ + internal::assertSupportedConfiguration(); +#ifdef CAROTENE_NEON + int16x8_t vthreshold16 = vdupq_n_s16(threshold); + size_t roiw16 = size.width >= 15 ? size.width - 15 : 0; + + for (size_t i = 0; i < size.height; ++i) + { + const s16* src = internal::getRowPtr(srcBase, srcStride, i); + s16* dst = internal::getRowPtr(dstBase, dstStride, i); + size_t j = 0; + + for (; j < roiw16; j += 16) + { + internal::prefetch(src + j); + int16x8_t v0 = vld1q_s16(src + j); + int16x8_t v1 = vld1q_s16(src + j + 8); + uint16x8_t r0 = vcgtq_s16(v0, vthreshold16); + uint16x8_t r1 = vcgtq_s16(v1, vthreshold16); + uint16x8_t r0a = vandq_u16(vreinterpretq_u16_s16(v0), r0); + uint16x8_t r1a = vandq_u16(vreinterpretq_u16_s16(v1), r1); + vst1q_u16((u16*)dst + j, r0a); + vst1q_u16((u16*)dst + j + 8, r1a); + } + for (; j < size.width; j++) + { + *(dst + j) = *(src + j) > threshold ? *(src + j) : 0; + } + } +#else + (void)size; + (void)srcBase; + (void)srcStride; + (void)dstBase; + (void)dstStride; + (void)threshold; +#endif +} + +void thresholdToZeroInv(const Size2D &size, + const s16 *srcBase, ptrdiff_t srcStride, + s16 *dstBase, ptrdiff_t dstStride, + s16 threshold) +{ + internal::assertSupportedConfiguration(); +#ifdef CAROTENE_NEON + int16x8_t vthreshold16 = vdupq_n_s16(threshold); + size_t roiw16 = size.width >= 15 ? size.width - 15 : 0; + + for (size_t i = 0; i < size.height; ++i) + { + const s16* src = internal::getRowPtr(srcBase, srcStride, i); + s16* dst = internal::getRowPtr(dstBase, dstStride, i); + size_t j = 0; + + for (; j < roiw16; j += 16) + { + internal::prefetch(src + j); + int16x8_t v0 = vld1q_s16(src + j); + int16x8_t v1 = vld1q_s16(src + j + 8); + uint16x8_t r0 = vcgtq_s16(v0, vthreshold16); + uint16x8_t r1 = vcgtq_s16(v1, vthreshold16); + uint16x8_t r0a = vbicq_u16(vreinterpretq_u16_s16(v0), r0); + uint16x8_t r1a = vbicq_u16(vreinterpretq_u16_s16(v1), r1); + vst1q_u16((u16*)dst + j, r0a); + vst1q_u16((u16*)dst + j + 8, r1a); + } + for (; j < size.width; j++) + { + *(dst + j) = *(src + j) > threshold ? 0 : *(src + j); + } + } +#else + (void)size; + (void)srcBase; + (void)srcStride; + (void)dstBase; + (void)dstStride; + (void)threshold; +#endif +} + +void thresholdBinary(const Size2D &size, + const u16 *srcBase, ptrdiff_t srcStride, + u16 *dstBase, ptrdiff_t dstStride, + u16 threshold, u16 value) +{ + internal::assertSupportedConfiguration(); +#ifdef CAROTENE_NEON + uint16x8_t vthreshold16 = vdupq_n_u16(threshold); + uint16x8_t vvalue16 = vdupq_n_u16(value); + size_t roiw16 = size.width >= 15 ? size.width - 15 : 0; + + for (size_t i = 0; i < size.height; ++i) + { + const u16* src = internal::getRowPtr(srcBase, srcStride, i); + u16* dst = internal::getRowPtr(dstBase, dstStride, i); + size_t j = 0; + + for (; j < roiw16; j += 16) + { + internal::prefetch(src + j); + uint16x8_t v0 = vld1q_u16(src + j); + uint16x8_t v1 = vld1q_u16(src + j + 8); + uint16x8_t r0 = vcgtq_u16(v0, vthreshold16); + uint16x8_t r1 = vcgtq_u16(v1, vthreshold16); + uint16x8_t r0a = vandq_u16(r0, vvalue16); + uint16x8_t r1a = vandq_u16(r1, vvalue16); + vst1q_u16(dst + j, r0a); + vst1q_u16(dst + j + 8, r1a); + } + for (; j < size.width; j++) + { + *(dst + j) = *(src + j) > threshold ? value : 0; + } + } +#else + (void)size; + (void)srcBase; + (void)srcStride; + (void)dstBase; + (void)dstStride; + (void)threshold; + (void)value; +#endif +} + +void thresholdBinaryInv(const Size2D &size, + const u16 *srcBase, ptrdiff_t srcStride, + u16 *dstBase, ptrdiff_t dstStride, + u16 threshold, u16 value) +{ + internal::assertSupportedConfiguration(); +#ifdef CAROTENE_NEON + uint16x8_t vthreshold16 = vdupq_n_u16(threshold); + uint16x8_t vvalue16 = vdupq_n_u16(value); + size_t roiw16 = size.width >= 15 ? size.width - 15 : 0; + + for (size_t i = 0; i < size.height; ++i) + { + const u16* src = internal::getRowPtr(srcBase, srcStride, i); + u16* dst = internal::getRowPtr(dstBase, dstStride, i); + size_t j = 0; + + for (; j < roiw16; j += 16) + { + internal::prefetch(src + j); + uint16x8_t v0 = vld1q_u16(src + j); + uint16x8_t v1 = vld1q_u16(src + j + 8); + uint16x8_t r0 = vcleq_u16(v0, vthreshold16); + uint16x8_t r1 = vcleq_u16(v1, vthreshold16); + uint16x8_t r0a = vandq_u16(r0, vvalue16); + uint16x8_t r1a = vandq_u16(r1, vvalue16); + vst1q_u16(dst + j, r0a); + vst1q_u16(dst + j + 8, r1a); + } + for (; j < size.width; j++) + { + *(dst + j) = *(src + j) > threshold ? 0 : value; + } + } +#else + (void)size; + (void)srcBase; + (void)srcStride; + (void)dstBase; + (void)dstStride; + (void)threshold; + (void)value; +#endif +} + +void thresholdTruncate(const Size2D &size, + const u16 *srcBase, ptrdiff_t srcStride, + u16 *dstBase, ptrdiff_t dstStride, + u16 threshold) +{ + internal::assertSupportedConfiguration(); +#ifdef CAROTENE_NEON + uint16x8_t vthreshold16 = vdupq_n_u16(threshold); + size_t roiw16 = size.width >= 15 ? size.width - 15 : 0; + + for (size_t i = 0; i < size.height; ++i) + { + const u16* src = internal::getRowPtr(srcBase, srcStride, i); + u16* dst = internal::getRowPtr(dstBase, dstStride, i); + size_t j = 0; + + for (; j < roiw16; j += 16) + { + internal::prefetch(src + j); + uint16x8_t v0 = vld1q_u16(src + j); + uint16x8_t v1 = vld1q_u16(src + j + 8); + uint16x8_t r0 = vminq_u16(v0, vthreshold16); + uint16x8_t r1 = vminq_u16(v1, vthreshold16); + vst1q_u16(dst + j, r0); + vst1q_u16(dst + j + 8, r1); + } + for (; j < size.width; j++) + { + *(dst + j) = *(src + j) > threshold ? threshold : *(src + j); + } + } +#else + (void)size; + (void)srcBase; + (void)srcStride; + (void)dstBase; + (void)dstStride; + (void)threshold; +#endif +} + +void thresholdToZero(const Size2D &size, + const u16 *srcBase, ptrdiff_t srcStride, + u16 *dstBase, ptrdiff_t dstStride, + u16 threshold) +{ + internal::assertSupportedConfiguration(); +#ifdef CAROTENE_NEON + uint16x8_t vthreshold16 = vdupq_n_u16(threshold); + size_t roiw16 = size.width >= 15 ? size.width - 15 : 0; + + for (size_t i = 0; i < size.height; ++i) + { + const u16* src = internal::getRowPtr(srcBase, srcStride, i); + u16* dst = internal::getRowPtr(dstBase, dstStride, i); + size_t j = 0; + + for (; j < roiw16; j += 16) + { + internal::prefetch(src + j); + uint16x8_t v0 = vld1q_u16(src + j); + uint16x8_t v1 = vld1q_u16(src + j + 8); + uint16x8_t r0 = vcgtq_u16(v0, vthreshold16); + uint16x8_t r1 = vcgtq_u16(v1, vthreshold16); + uint16x8_t r0a = vandq_u16(v0, r0); + uint16x8_t r1a = vandq_u16(v1, r1); + vst1q_u16(dst + j, r0a); + vst1q_u16(dst + j + 8, r1a); + } + for (; j < size.width; j++) + { + *(dst + j) = *(src + j) > threshold ? *(src + j) : 0; + } + } +#else + (void)size; + (void)srcBase; + (void)srcStride; + (void)dstBase; + (void)dstStride; + (void)threshold; +#endif +} + +void thresholdToZeroInv(const Size2D &size, + const u16 *srcBase, ptrdiff_t srcStride, + u16 *dstBase, ptrdiff_t dstStride, + u16 threshold) +{ + internal::assertSupportedConfiguration(); +#ifdef CAROTENE_NEON + uint16x8_t vthreshold16 = vdupq_n_u16(threshold); + size_t roiw16 = size.width >= 15 ? size.width - 15 : 0; + + for (size_t i = 0; i < size.height; ++i) + { + const u16* src = internal::getRowPtr(srcBase, srcStride, i); + u16* dst = internal::getRowPtr(dstBase, dstStride, i); + size_t j = 0; + + for (; j < roiw16; j += 16) + { + internal::prefetch(src + j); + uint16x8_t v0 = vld1q_u16(src + j); + uint16x8_t v1 = vld1q_u16(src + j + 8); + uint16x8_t r0 = vcgtq_u16(v0, vthreshold16); + uint16x8_t r1 = vcgtq_u16(v1, vthreshold16); + uint16x8_t r0a = vbicq_u16(v0, r0); + uint16x8_t r1a = vbicq_u16(v1, r1); + vst1q_u16(dst + j, r0a); + vst1q_u16(dst + j + 8, r1a); + } + for (; j < size.width; j++) + { + *(dst + j) = *(src + j) > threshold ? 0 : *(src + j); + } + } +#else + (void)size; + (void)srcBase; + (void)srcStride; + (void)dstBase; + (void)dstStride; + (void)threshold; +#endif +} + +void thresholdBinary(const Size2D &size, + const s32 *srcBase, ptrdiff_t srcStride, + s32 *dstBase, ptrdiff_t dstStride, + s32 threshold, s32 value) +{ + internal::assertSupportedConfiguration(); +#ifdef CAROTENE_NEON + int32x4_t vthreshold8 = vdupq_n_s32(threshold); + int32x4_t vvalue8 = vdupq_n_s32(value); + size_t roiw8 = size.width >= 7 ? size.width - 7 : 0; + + for (size_t i = 0; i < size.height; ++i) + { + const s32* src = internal::getRowPtr(srcBase, srcStride, i); + s32* dst = internal::getRowPtr(dstBase, dstStride, i); + size_t j = 0; + + for (; j < roiw8; j += 8) + { + internal::prefetch(src + j); + int32x4_t v0 = vld1q_s32(src + j); + int32x4_t v1 = vld1q_s32(src + j + 4); + uint32x4_t r0 = vcgtq_s32(v0, vthreshold8); + uint32x4_t r1 = vcgtq_s32(v1, vthreshold8); + uint32x4_t r0a = vandq_u32(r0, vreinterpretq_u32_s32(vvalue8)); + uint32x4_t r1a = vandq_u32(r1, vreinterpretq_u32_s32(vvalue8)); + vst1q_u32((u32*)dst + j, r0a); + vst1q_u32((u32*)dst + j + 4, r1a); + } + for (; j < size.width; j++) + { + *(dst + j) = *(src + j) > threshold ? value : 0; + } + } +#else + (void)size; + (void)srcBase; + (void)srcStride; + (void)dstBase; + (void)dstStride; + (void)threshold; + (void)value; +#endif +} + +void thresholdBinaryInv(const Size2D &size, + const s32 *srcBase, ptrdiff_t srcStride, + s32 *dstBase, ptrdiff_t dstStride, + s32 threshold, s32 value) +{ + internal::assertSupportedConfiguration(); +#ifdef CAROTENE_NEON + int32x4_t vthreshold8 = vdupq_n_s32(threshold); + int32x4_t vvalue8 = vdupq_n_s32(value); + size_t roiw8 = size.width >= 7 ? size.width - 7 : 0; + + for (size_t i = 0; i < size.height; ++i) + { + const s32* src = internal::getRowPtr(srcBase, srcStride, i); + s32* dst = internal::getRowPtr(dstBase, dstStride, i); + size_t j = 0; + + for (; j < roiw8; j += 8) + { + internal::prefetch(src + j); + int32x4_t v0 = vld1q_s32(src + j); + int32x4_t v1 = vld1q_s32(src + j + 4); + uint32x4_t r0 = vcleq_s32(v0, vthreshold8); + uint32x4_t r1 = vcleq_s32(v1, vthreshold8); + uint32x4_t r0a = vandq_u32(r0, vreinterpretq_u32_s32(vvalue8)); + uint32x4_t r1a = vandq_u32(r1, vreinterpretq_u32_s32(vvalue8)); + vst1q_s32(dst + j, vreinterpretq_s32_u32(r0a)); + vst1q_s32(dst + j + 4, vreinterpretq_s32_u32(r1a)); + } + for (; j < size.width; j++) + { + *(dst + j) = *(src + j) > threshold ? 0 : value; + } + } +#else + (void)size; + (void)srcBase; + (void)srcStride; + (void)dstBase; + (void)dstStride; + (void)threshold; + (void)value; +#endif +} + +void thresholdTruncate(const Size2D &size, + const s32 *srcBase, ptrdiff_t srcStride, + s32 *dstBase, ptrdiff_t dstStride, + s32 threshold) +{ + internal::assertSupportedConfiguration(); +#ifdef CAROTENE_NEON + int32x4_t vthreshold8 = vdupq_n_s32(threshold); + size_t roiw8 = size.width >= 7 ? size.width - 7 : 0; + + for (size_t i = 0; i < size.height; ++i) + { + const s32* src = internal::getRowPtr(srcBase, srcStride, i); + s32* dst = internal::getRowPtr(dstBase, dstStride, i); + size_t j = 0; + + for (; j < roiw8; j += 8) + { + internal::prefetch(src + j); + int32x4_t v0 = vld1q_s32(src + j); + int32x4_t v1 = vld1q_s32(src + j + 4); + int32x4_t r0 = vminq_s32(v0, vthreshold8); + int32x4_t r1 = vminq_s32(v1, vthreshold8); + vst1q_s32(dst + j, r0); + vst1q_s32(dst + j + 4, r1); + } + for (; j < size.width; j++) + { + *(dst + j) = *(src + j) > threshold ? threshold : *(src + j); + } + } +#else + (void)size; + (void)srcBase; + (void)srcStride; + (void)dstBase; + (void)dstStride; + (void)threshold; +#endif +} + +void thresholdToZero(const Size2D &size, + const s32 *srcBase, ptrdiff_t srcStride, + s32 *dstBase, ptrdiff_t dstStride, + s32 threshold) +{ + internal::assertSupportedConfiguration(); +#ifdef CAROTENE_NEON + int32x4_t vthreshold8 = vdupq_n_s32(threshold); + size_t roiw8 = size.width >= 7 ? size.width - 7 : 0; + + for (size_t i = 0; i < size.height; ++i) + { + const s32* src = internal::getRowPtr(srcBase, srcStride, i); + s32* dst = internal::getRowPtr(dstBase, dstStride, i); + size_t j = 0; + + for (; j < roiw8; j += 8) + { + internal::prefetch(src + j); + int32x4_t v0 = vld1q_s32(src + j); + int32x4_t v1 = vld1q_s32(src + j + 4); + uint32x4_t r0 = vcgtq_s32(v0, vthreshold8); + uint32x4_t r1 = vcgtq_s32(v1, vthreshold8); + uint32x4_t r0a = vandq_u32(vreinterpretq_u32_s32(v0), r0); + uint32x4_t r1a = vandq_u32(vreinterpretq_u32_s32(v1), r1); + vst1q_u32((u32*)dst + j, r0a); + vst1q_u32((u32*)dst + j + 4, r1a); + } + for (; j < size.width; j++) + { + *(dst + j) = *(src + j) > threshold ? *(src + j) : 0; + } + } +#else + (void)size; + (void)srcBase; + (void)srcStride; + (void)dstBase; + (void)dstStride; + (void)threshold; +#endif +} + +void thresholdToZeroInv(const Size2D &size, + const s32 *srcBase, ptrdiff_t srcStride, + s32 *dstBase, ptrdiff_t dstStride, + s32 threshold) +{ + internal::assertSupportedConfiguration(); +#ifdef CAROTENE_NEON + int32x4_t vthreshold8 = vdupq_n_s32(threshold); + size_t roiw8 = size.width >= 7 ? size.width - 7 : 0; + + for (size_t i = 0; i < size.height; ++i) + { + const s32* src = internal::getRowPtr(srcBase, srcStride, i); + s32* dst = internal::getRowPtr(dstBase, dstStride, i); + size_t j = 0; + + for (; j < roiw8; j += 8) + { + internal::prefetch(src + j); + int32x4_t v0 = vld1q_s32(src + j); + int32x4_t v1 = vld1q_s32(src + j + 4); + uint32x4_t r0 = vcgtq_s32(v0, vthreshold8); + uint32x4_t r1 = vcgtq_s32(v1, vthreshold8); + uint32x4_t r0a = vbicq_u32(vreinterpretq_u32_s32(v0), r0); + uint32x4_t r1a = vbicq_u32(vreinterpretq_u32_s32(v1), r1); + vst1q_u32((u32*)dst + j, r0a); + vst1q_u32((u32*)dst + j + 4, r1a); + } + for (; j < size.width; j++) + { + *(dst + j) = *(src + j) > threshold ? 0 : *(src + j); + } + } +#else + (void)size; + (void)srcBase; + (void)srcStride; + (void)dstBase; + (void)dstStride; + (void)threshold; +#endif +} + +void thresholdBinary(const Size2D &size, + const f32 *srcBase, ptrdiff_t srcStride, + f32 *dstBase, ptrdiff_t dstStride, + f32 threshold, f32 value) +{ + internal::assertSupportedConfiguration(); +#ifdef CAROTENE_NEON + float32x4_t vthreshold8 = vdupq_n_f32(threshold); + float32x4_t vvalue8 = vdupq_n_f32(value); + size_t roiw8 = size.width >= 7 ? size.width - 7 : 0; + + for (size_t i = 0; i < size.height; ++i) + { + const f32* src = internal::getRowPtr(srcBase, srcStride, i); + f32* dst = internal::getRowPtr(dstBase, dstStride, i); + size_t j = 0; + + for (; j < roiw8; j += 8) + { + internal::prefetch(src + j); + float32x4_t v0 = vld1q_f32(src + j); + float32x4_t v1 = vld1q_f32(src + j + 4); + uint32x4_t r0 = vcgtq_f32(v0, vthreshold8); + uint32x4_t r1 = vcgtq_f32(v1, vthreshold8); + uint32x4_t r0a = vandq_u32(r0, vreinterpretq_u32_f32(vvalue8)); + uint32x4_t r1a = vandq_u32(r1, vreinterpretq_u32_f32(vvalue8)); + vst1q_u32((u32*)dst + j, r0a); + vst1q_u32((u32*)dst + j + 4, r1a); + } + for (; j < size.width; j++) + { + *(dst + j) = *(src + j) > threshold ? value : 0; + } + } +#else + (void)size; + (void)srcBase; + (void)srcStride; + (void)dstBase; + (void)dstStride; + (void)threshold; + (void)value; +#endif +} + +void thresholdBinaryInv(const Size2D &size, + const f32 *srcBase, ptrdiff_t srcStride, + f32 *dstBase, ptrdiff_t dstStride, + f32 threshold, f32 value) +{ + internal::assertSupportedConfiguration(); +#ifdef CAROTENE_NEON + float32x4_t vthreshold8 = vdupq_n_f32(threshold); + float32x4_t vvalue8 = vdupq_n_f32(value); + size_t roiw8 = size.width >= 7 ? size.width - 7 : 0; + + for (size_t i = 0; i < size.height; ++i) + { + const f32* src = internal::getRowPtr(srcBase, srcStride, i); + f32* dst = internal::getRowPtr(dstBase, dstStride, i); + size_t j = 0; + + for (; j < roiw8; j += 8) + { + internal::prefetch(src + j); + float32x4_t v0 = vld1q_f32(src + j); + float32x4_t v1 = vld1q_f32(src + j + 4); + uint32x4_t r0 = vcleq_f32(v0, vthreshold8); + uint32x4_t r1 = vcleq_f32(v1, vthreshold8); + uint32x4_t r0a = vandq_u32(r0, vreinterpretq_u32_f32(vvalue8)); + uint32x4_t r1a = vandq_u32(r1, vreinterpretq_u32_f32(vvalue8)); + vst1q_f32(dst + j, vreinterpretq_f32_u32(r0a)); + vst1q_f32(dst + j + 4, vreinterpretq_f32_u32(r1a)); + } + for (; j < size.width; j++) + { + *(dst + j) = *(src + j) > threshold ? 0 : value; + } + } +#else + (void)size; + (void)srcBase; + (void)srcStride; + (void)dstBase; + (void)dstStride; + (void)threshold; + (void)value; +#endif +} + +void thresholdTruncate(const Size2D &size, + const f32 *srcBase, ptrdiff_t srcStride, + f32 *dstBase, ptrdiff_t dstStride, + f32 threshold) +{ + internal::assertSupportedConfiguration(); +#ifdef CAROTENE_NEON + float32x4_t vthreshold8 = vdupq_n_f32(threshold); + size_t roiw8 = size.width >= 7 ? size.width - 7 : 0; + + for (size_t i = 0; i < size.height; ++i) + { + const f32* src = internal::getRowPtr(srcBase, srcStride, i); + f32* dst = internal::getRowPtr(dstBase, dstStride, i); + size_t j = 0; + + for (; j < roiw8; j += 8) + { + internal::prefetch(src + j); + float32x4_t v0 = vld1q_f32(src + j); + float32x4_t v1 = vld1q_f32(src + j + 4); + float32x4_t r0 = vminq_f32(v0, vthreshold8); + float32x4_t r1 = vminq_f32(v1, vthreshold8); + vst1q_f32(dst + j, r0); + vst1q_f32(dst + j + 4, r1); + } + for (; j < size.width; j++) + { + *(dst + j) = *(src + j) > threshold ? threshold : *(src + j); + } + } +#else + (void)size; + (void)srcBase; + (void)srcStride; + (void)dstBase; + (void)dstStride; + (void)threshold; +#endif +} + +void thresholdToZero(const Size2D &size, + const f32 *srcBase, ptrdiff_t srcStride, + f32 *dstBase, ptrdiff_t dstStride, + f32 threshold) +{ + internal::assertSupportedConfiguration(); +#ifdef CAROTENE_NEON + float32x4_t vthreshold8 = vdupq_n_f32(threshold); + size_t roiw8 = size.width >= 7 ? size.width - 7 : 0; + + for (size_t i = 0; i < size.height; ++i) + { + const f32* src = internal::getRowPtr(srcBase, srcStride, i); + f32* dst = internal::getRowPtr(dstBase, dstStride, i); + size_t j = 0; + + for (; j < roiw8; j += 8) + { + internal::prefetch(src + j); + float32x4_t v0 = vld1q_f32(src + j); + float32x4_t v1 = vld1q_f32(src + j + 4); + uint32x4_t r0 = vcgtq_f32(v0, vthreshold8); + uint32x4_t r1 = vcgtq_f32(v1, vthreshold8); + uint32x4_t r0a = vandq_u32(vreinterpretq_u32_f32(v0), r0); + uint32x4_t r1a = vandq_u32(vreinterpretq_u32_f32(v1), r1); + vst1q_u32((u32*)dst + j, r0a); + vst1q_u32((u32*)dst + j + 4, r1a); + } + for (; j < size.width; j++) + { + *(dst + j) = *(src + j) > threshold ? *(src + j) : 0; + } + } +#else + (void)size; + (void)srcBase; + (void)srcStride; + (void)dstBase; + (void)dstStride; + (void)threshold; +#endif +} + +void thresholdToZeroInv(const Size2D &size, + const f32 *srcBase, ptrdiff_t srcStride, + f32 *dstBase, ptrdiff_t dstStride, + f32 threshold) +{ + internal::assertSupportedConfiguration(); +#ifdef CAROTENE_NEON + float32x4_t vthreshold8 = vdupq_n_f32(threshold); + size_t roiw8 = size.width >= 7 ? size.width - 7 : 0; + + for (size_t i = 0; i < size.height; ++i) + { + const f32* src = internal::getRowPtr(srcBase, srcStride, i); + f32* dst = internal::getRowPtr(dstBase, dstStride, i); + size_t j = 0; + + for (; j < roiw8; j += 8) + { + internal::prefetch(src + j); + float32x4_t v0 = vld1q_f32(src + j); + float32x4_t v1 = vld1q_f32(src + j + 4); + uint32x4_t r0 = vcgtq_f32(v0, vthreshold8); + uint32x4_t r1 = vcgtq_f32(v1, vthreshold8); + uint32x4_t r0a = vbicq_u32(vreinterpretq_u32_f32(v0), r0); + uint32x4_t r1a = vbicq_u32(vreinterpretq_u32_f32(v1), r1); + vst1q_u32((u32*)dst + j, r0a); + vst1q_u32((u32*)dst + j + 4, r1a); + } + for (; j < size.width; j++) + { + *(dst + j) = *(src + j) > threshold ? 0 : *(src + j); + } + } +#else + (void)size; + (void)srcBase; + (void)srcStride; + (void)dstBase; + (void)dstStride; + (void)threshold; +#endif +} + +} // namespace CAROTENE_NS diff --git a/3rdparty/carotene/src/vtransform.hpp b/3rdparty/carotene/src/vtransform.hpp new file mode 100644 index 0000000000..08841a2263 --- /dev/null +++ b/3rdparty/carotene/src/vtransform.hpp @@ -0,0 +1,689 @@ +/* + * By downloading, copying, installing or using the software you agree to this license. + * If you do not agree to this license, do not download, install, + * copy or use the software. + * + * + * License Agreement + * For Open Source Computer Vision Library + * (3-clause BSD License) + * + * Copyright (C) 2014-2015, NVIDIA Corporation, all rights reserved. + * Third party copyrights are property of their respective owners. + * + * Redistribution and use in source and binary forms, with or without modification, + * are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * * Neither the names of the copyright holders nor the names of the contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * This software is provided by the copyright holders and contributors "as is" and + * any express or implied warranties, including, but not limited to, the implied + * warranties of merchantability and fitness for a particular purpose are disclaimed. + * In no event shall copyright holders or contributors be liable for any direct, + * indirect, incidental, special, exemplary, or consequential damages + * (including, but not limited to, procurement of substitute goods or services; + * loss of use, data, or profits; or business interruption) however caused + * and on any theory of liability, whether in contract, strict liability, + * or tort (including negligence or otherwise) arising in any way out of + * the use of this software, even if advised of the possibility of such damage. + */ + +#ifndef CAROTENE_SRC_VTRANSFORM_HPP +#define CAROTENE_SRC_VTRANSFORM_HPP + +#include "common.hpp" + +#include + +#ifdef CAROTENE_NEON + +namespace CAROTENE_NS { namespace internal { + +////////////////////////////// Type Traits /////////////////////// + +template +struct VecTraits; + +template <> struct VecTraits< u8, 1> { typedef uint8x16_t vec128; typedef uint8x8_t vec64; typedef VecTraits< u8, 1> unsign; }; +template <> struct VecTraits< s8, 1> { typedef int8x16_t vec128; typedef int8x8_t vec64; typedef VecTraits< u8, 1> unsign; }; +template <> struct VecTraits { typedef uint16x8_t vec128; typedef uint16x4_t vec64; typedef VecTraits< u16, 1> unsign; }; +template <> struct VecTraits { typedef int16x8_t vec128; typedef int16x4_t vec64; typedef VecTraits< u16, 1> unsign; }; +template <> struct VecTraits { typedef int32x4_t vec128; typedef int32x2_t vec64; typedef VecTraits< u32, 1> unsign; }; +template <> struct VecTraits { typedef uint32x4_t vec128; typedef uint32x2_t vec64; typedef VecTraits< u32, 1> unsign; }; +template <> struct VecTraits { typedef int64x2_t vec128; typedef int64x1_t vec64; typedef VecTraits< u64, 1> unsign; }; +template <> struct VecTraits { typedef uint64x2_t vec128; typedef uint64x1_t vec64; typedef VecTraits< u64, 1> unsign; }; +template <> struct VecTraits { typedef float32x4_t vec128; typedef float32x2_t vec64; typedef VecTraits< u32, 1> unsign; }; + +template <> struct VecTraits< u8, 2> { typedef uint8x16x2_t vec128; typedef uint8x8x2_t vec64; typedef VecTraits< u8, 2> unsign; }; +template <> struct VecTraits< s8, 2> { typedef int8x16x2_t vec128; typedef int8x8x2_t vec64; typedef VecTraits< u8, 2> unsign; }; +template <> struct VecTraits { typedef uint16x8x2_t vec128; typedef uint16x4x2_t vec64; typedef VecTraits< u16, 2> unsign; }; +template <> struct VecTraits { typedef int16x8x2_t vec128; typedef int16x4x2_t vec64; typedef VecTraits< u16, 2> unsign; }; +template <> struct VecTraits { typedef int32x4x2_t vec128; typedef int32x2x2_t vec64; typedef VecTraits< u32, 2> unsign; }; +template <> struct VecTraits { typedef uint32x4x2_t vec128; typedef uint32x2x2_t vec64; typedef VecTraits< u32, 2> unsign; }; +template <> struct VecTraits { typedef int64x2x2_t vec128; typedef int64x1x2_t vec64; typedef VecTraits< u64, 2> unsign; }; +template <> struct VecTraits { typedef uint64x2x2_t vec128; typedef uint64x1x2_t vec64; typedef VecTraits< u64, 2> unsign; }; +template <> struct VecTraits { typedef float32x4x2_t vec128; typedef float32x2x2_t vec64; typedef VecTraits< u32, 2> unsign; }; + +template <> struct VecTraits< u8, 3> { typedef uint8x16x3_t vec128; typedef uint8x8x3_t vec64; typedef VecTraits< u8, 3> unsign; }; +template <> struct VecTraits< s8, 3> { typedef int8x16x3_t vec128; typedef int8x8x3_t vec64; typedef VecTraits< u8, 3> unsign; }; +template <> struct VecTraits { typedef uint16x8x3_t vec128; typedef uint16x4x3_t vec64; typedef VecTraits< u16, 3> unsign; }; +template <> struct VecTraits { typedef int16x8x3_t vec128; typedef int16x4x3_t vec64; typedef VecTraits< u16, 3> unsign; }; +template <> struct VecTraits { typedef int32x4x3_t vec128; typedef int32x2x3_t vec64; typedef VecTraits< u32, 3> unsign; }; +template <> struct VecTraits { typedef uint32x4x3_t vec128; typedef uint32x2x3_t vec64; typedef VecTraits< u32, 3> unsign; }; +template <> struct VecTraits { typedef int64x2x3_t vec128; typedef int64x1x3_t vec64; typedef VecTraits< u64, 2> unsign; }; +template <> struct VecTraits { typedef uint64x2x3_t vec128; typedef uint64x1x3_t vec64; typedef VecTraits< u64, 2> unsign; }; +template <> struct VecTraits { typedef float32x4x3_t vec128; typedef float32x2x3_t vec64; typedef VecTraits< u32, 3> unsign; }; + +template <> struct VecTraits< u8, 4> { typedef uint8x16x4_t vec128; typedef uint8x8x4_t vec64; typedef VecTraits< u8, 3> unsign; }; +template <> struct VecTraits< s8, 4> { typedef int8x16x4_t vec128; typedef int8x8x4_t vec64; typedef VecTraits< u8, 3> unsign; }; +template <> struct VecTraits { typedef uint16x8x4_t vec128; typedef uint16x4x4_t vec64; typedef VecTraits< u16, 3> unsign; }; +template <> struct VecTraits { typedef int16x8x4_t vec128; typedef int16x4x4_t vec64; typedef VecTraits< u16, 3> unsign; }; +template <> struct VecTraits { typedef int32x4x4_t vec128; typedef int32x2x4_t vec64; typedef VecTraits< u32, 3> unsign; }; +template <> struct VecTraits { typedef uint32x4x4_t vec128; typedef uint32x2x4_t vec64; typedef VecTraits< u32, 3> unsign; }; +template <> struct VecTraits { typedef int64x2x4_t vec128; typedef int64x1x4_t vec64; typedef VecTraits< u64, 2> unsign; }; +template <> struct VecTraits { typedef uint64x2x4_t vec128; typedef uint64x1x4_t vec64; typedef VecTraits< u64, 2> unsign; }; +template <> struct VecTraits { typedef float32x4x4_t vec128; typedef float32x2x4_t vec64; typedef VecTraits< u32, 3> unsign; }; + +////////////////////////////// vld1q /////////////////////// + +inline uint8x16_t vld1q(const u8 * ptr) { return vld1q_u8(ptr); } +inline int8x16_t vld1q(const s8 * ptr) { return vld1q_s8(ptr); } +inline uint16x8_t vld1q(const u16 * ptr) { return vld1q_u16(ptr); } +inline int16x8_t vld1q(const s16 * ptr) { return vld1q_s16(ptr); } +inline uint32x4_t vld1q(const u32 * ptr) { return vld1q_u32(ptr); } +inline int32x4_t vld1q(const s32 * ptr) { return vld1q_s32(ptr); } +inline float32x4_t vld1q(const f32 * ptr) { return vld1q_f32(ptr); } + +////////////////////////////// vld1 /////////////////////// + +inline uint8x8_t vld1(const u8 * ptr) { return vld1_u8(ptr); } +inline int8x8_t vld1(const s8 * ptr) { return vld1_s8(ptr); } +inline uint16x4_t vld1(const u16 * ptr) { return vld1_u16(ptr); } +inline int16x4_t vld1(const s16 * ptr) { return vld1_s16(ptr); } +inline uint32x2_t vld1(const u32 * ptr) { return vld1_u32(ptr); } +inline int32x2_t vld1(const s32 * ptr) { return vld1_s32(ptr); } +inline float32x2_t vld1(const f32 * ptr) { return vld1_f32(ptr); } + +////////////////////////////// vld2q /////////////////////// + +inline uint8x16x2_t vld2q(const u8 * ptr) { return vld2q_u8(ptr); } +inline int8x16x2_t vld2q(const s8 * ptr) { return vld2q_s8(ptr); } +inline uint16x8x2_t vld2q(const u16 * ptr) { return vld2q_u16(ptr); } +inline int16x8x2_t vld2q(const s16 * ptr) { return vld2q_s16(ptr); } +inline uint32x4x2_t vld2q(const u32 * ptr) { return vld2q_u32(ptr); } +inline int32x4x2_t vld2q(const s32 * ptr) { return vld2q_s32(ptr); } +inline float32x4x2_t vld2q(const f32 * ptr) { return vld2q_f32(ptr); } + +////////////////////////////// vld2 /////////////////////// + +inline uint8x8x2_t vld2(const u8 * ptr) { return vld2_u8(ptr); } +inline int8x8x2_t vld2(const s8 * ptr) { return vld2_s8(ptr); } +inline uint16x4x2_t vld2(const u16 * ptr) { return vld2_u16(ptr); } +inline int16x4x2_t vld2(const s16 * ptr) { return vld2_s16(ptr); } +inline uint32x2x2_t vld2(const u32 * ptr) { return vld2_u32(ptr); } +inline int32x2x2_t vld2(const s32 * ptr) { return vld2_s32(ptr); } +inline float32x2x2_t vld2(const f32 * ptr) { return vld2_f32(ptr); } + +////////////////////////////// vld3q /////////////////////// + +inline uint8x16x3_t vld3q(const u8 * ptr) { return vld3q_u8(ptr); } +inline int8x16x3_t vld3q(const s8 * ptr) { return vld3q_s8(ptr); } +inline uint16x8x3_t vld3q(const u16 * ptr) { return vld3q_u16(ptr); } +inline int16x8x3_t vld3q(const s16 * ptr) { return vld3q_s16(ptr); } +inline uint32x4x3_t vld3q(const u32 * ptr) { return vld3q_u32(ptr); } +inline int32x4x3_t vld3q(const s32 * ptr) { return vld3q_s32(ptr); } +inline float32x4x3_t vld3q(const f32 * ptr) { return vld3q_f32(ptr); } + +////////////////////////////// vld3 /////////////////////// + +inline uint8x8x3_t vld3(const u8 * ptr) { return vld3_u8(ptr); } +inline int8x8x3_t vld3(const s8 * ptr) { return vld3_s8(ptr); } +inline uint16x4x3_t vld3(const u16 * ptr) { return vld3_u16(ptr); } +inline int16x4x3_t vld3(const s16 * ptr) { return vld3_s16(ptr); } +inline uint32x2x3_t vld3(const u32 * ptr) { return vld3_u32(ptr); } +inline int32x2x3_t vld3(const s32 * ptr) { return vld3_s32(ptr); } +inline float32x2x3_t vld3(const f32 * ptr) { return vld3_f32(ptr); } + +////////////////////////////// vld4q /////////////////////// + +inline uint8x16x4_t vld4q(const u8 * ptr) { return vld4q_u8(ptr); } +inline int8x16x4_t vld4q(const s8 * ptr) { return vld4q_s8(ptr); } +inline uint16x8x4_t vld4q(const u16 * ptr) { return vld4q_u16(ptr); } +inline int16x8x4_t vld4q(const s16 * ptr) { return vld4q_s16(ptr); } +inline uint32x4x4_t vld4q(const u32 * ptr) { return vld4q_u32(ptr); } +inline int32x4x4_t vld4q(const s32 * ptr) { return vld4q_s32(ptr); } +inline float32x4x4_t vld4q(const f32 * ptr) { return vld4q_f32(ptr); } + +////////////////////////////// vld4 /////////////////////// + +inline uint8x8x4_t vld4(const u8 * ptr) { return vld4_u8(ptr); } +inline int8x8x4_t vld4(const s8 * ptr) { return vld4_s8(ptr); } +inline uint16x4x4_t vld4(const u16 * ptr) { return vld4_u16(ptr); } +inline int16x4x4_t vld4(const s16 * ptr) { return vld4_s16(ptr); } +inline uint32x2x4_t vld4(const u32 * ptr) { return vld4_u32(ptr); } +inline int32x2x4_t vld4(const s32 * ptr) { return vld4_s32(ptr); } +inline float32x2x4_t vld4(const f32 * ptr) { return vld4_f32(ptr); } + +////////////////////////////// vst1q /////////////////////// + +inline void vst1q(u8 * ptr, const uint8x16_t & v) { return vst1q_u8(ptr, v); } +inline void vst1q(s8 * ptr, const int8x16_t & v) { return vst1q_s8(ptr, v); } +inline void vst1q(u16 * ptr, const uint16x8_t & v) { return vst1q_u16(ptr, v); } +inline void vst1q(s16 * ptr, const int16x8_t & v) { return vst1q_s16(ptr, v); } +inline void vst1q(u32 * ptr, const uint32x4_t & v) { return vst1q_u32(ptr, v); } +inline void vst1q(s32 * ptr, const int32x4_t & v) { return vst1q_s32(ptr, v); } +inline void vst1q(f32 * ptr, const float32x4_t & v) { return vst1q_f32(ptr, v); } + +////////////////////////////// vst1 /////////////////////// + +inline void vst1(u8 * ptr, const uint8x8_t & v) { return vst1_u8(ptr, v); } +inline void vst1(s8 * ptr, const int8x8_t & v) { return vst1_s8(ptr, v); } +inline void vst1(u16 * ptr, const uint16x4_t & v) { return vst1_u16(ptr, v); } +inline void vst1(s16 * ptr, const int16x4_t & v) { return vst1_s16(ptr, v); } +inline void vst1(u32 * ptr, const uint32x2_t & v) { return vst1_u32(ptr, v); } +inline void vst1(s32 * ptr, const int32x2_t & v) { return vst1_s32(ptr, v); } +inline void vst1(f32 * ptr, const float32x2_t & v) { return vst1_f32(ptr, v); } + +////////////////////////////// vst2q /////////////////////// + +inline void vst2q(u8 * ptr, const uint8x16x2_t & v) { return vst2q_u8(ptr, v); } +inline void vst2q(s8 * ptr, const int8x16x2_t & v) { return vst2q_s8(ptr, v); } +inline void vst2q(u16 * ptr, const uint16x8x2_t & v) { return vst2q_u16(ptr, v); } +inline void vst2q(s16 * ptr, const int16x8x2_t & v) { return vst2q_s16(ptr, v); } +inline void vst2q(u32 * ptr, const uint32x4x2_t & v) { return vst2q_u32(ptr, v); } +inline void vst2q(s32 * ptr, const int32x4x2_t & v) { return vst2q_s32(ptr, v); } +inline void vst2q(f32 * ptr, const float32x4x2_t & v) { return vst2q_f32(ptr, v); } + +////////////////////////////// vst2 /////////////////////// + +inline void vst2(u8 * ptr, const uint8x8x2_t & v) { return vst2_u8(ptr, v); } +inline void vst2(s8 * ptr, const int8x8x2_t & v) { return vst2_s8(ptr, v); } +inline void vst2(u16 * ptr, const uint16x4x2_t & v) { return vst2_u16(ptr, v); } +inline void vst2(s16 * ptr, const int16x4x2_t & v) { return vst2_s16(ptr, v); } +inline void vst2(u32 * ptr, const uint32x2x2_t & v) { return vst2_u32(ptr, v); } +inline void vst2(s32 * ptr, const int32x2x2_t & v) { return vst2_s32(ptr, v); } +inline void vst2(f32 * ptr, const float32x2x2_t & v) { return vst2_f32(ptr, v); } + +////////////////////////////// vst3q /////////////////////// + +inline void vst3q(u8 * ptr, const uint8x16x3_t & v) { return vst3q_u8(ptr, v); } +inline void vst3q(s8 * ptr, const int8x16x3_t & v) { return vst3q_s8(ptr, v); } +inline void vst3q(u16 * ptr, const uint16x8x3_t & v) { return vst3q_u16(ptr, v); } +inline void vst3q(s16 * ptr, const int16x8x3_t & v) { return vst3q_s16(ptr, v); } +inline void vst3q(u32 * ptr, const uint32x4x3_t & v) { return vst3q_u32(ptr, v); } +inline void vst3q(s32 * ptr, const int32x4x3_t & v) { return vst3q_s32(ptr, v); } +inline void vst3q(f32 * ptr, const float32x4x3_t & v) { return vst3q_f32(ptr, v); } + +////////////////////////////// vst3 /////////////////////// + +inline void vst3(u8 * ptr, const uint8x8x3_t & v) { return vst3_u8(ptr, v); } +inline void vst3(s8 * ptr, const int8x8x3_t & v) { return vst3_s8(ptr, v); } +inline void vst3(u16 * ptr, const uint16x4x3_t & v) { return vst3_u16(ptr, v); } +inline void vst3(s16 * ptr, const int16x4x3_t & v) { return vst3_s16(ptr, v); } +inline void vst3(u32 * ptr, const uint32x2x3_t & v) { return vst3_u32(ptr, v); } +inline void vst3(s32 * ptr, const int32x2x3_t & v) { return vst3_s32(ptr, v); } +inline void vst3(f32 * ptr, const float32x2x3_t & v) { return vst3_f32(ptr, v); } + +////////////////////////////// vst4q /////////////////////// + +inline void vst4q(u8 * ptr, const uint8x16x4_t & v) { return vst4q_u8(ptr, v); } +inline void vst4q(s8 * ptr, const int8x16x4_t & v) { return vst4q_s8(ptr, v); } +inline void vst4q(u16 * ptr, const uint16x8x4_t & v) { return vst4q_u16(ptr, v); } +inline void vst4q(s16 * ptr, const int16x8x4_t & v) { return vst4q_s16(ptr, v); } +inline void vst4q(u32 * ptr, const uint32x4x4_t & v) { return vst4q_u32(ptr, v); } +inline void vst4q(s32 * ptr, const int32x4x4_t & v) { return vst4q_s32(ptr, v); } +inline void vst4q(f32 * ptr, const float32x4x4_t & v) { return vst4q_f32(ptr, v); } + +////////////////////////////// vst4 /////////////////////// + +inline void vst4(u8 * ptr, const uint8x8x4_t & v) { return vst4_u8(ptr, v); } +inline void vst4(s8 * ptr, const int8x8x4_t & v) { return vst4_s8(ptr, v); } +inline void vst4(u16 * ptr, const uint16x4x4_t & v) { return vst4_u16(ptr, v); } +inline void vst4(s16 * ptr, const int16x4x4_t & v) { return vst4_s16(ptr, v); } +inline void vst4(u32 * ptr, const uint32x2x4_t & v) { return vst4_u32(ptr, v); } +inline void vst4(s32 * ptr, const int32x2x4_t & v) { return vst4_s32(ptr, v); } +inline void vst4(f32 * ptr, const float32x2x4_t & v) { return vst4_f32(ptr, v); } + +////////////////////////////// vabdq /////////////////////// + +inline uint8x16_t vabdq(const uint8x16_t & v0, const uint8x16_t & v1) { return vabdq_u8 (v0, v1); } +inline int8x16_t vabdq(const int8x16_t & v0, const int8x16_t & v1) { return vabdq_s8 (v0, v1); } +inline uint16x8_t vabdq(const uint16x8_t & v0, const uint16x8_t & v1) { return vabdq_u16(v0, v1); } +inline int16x8_t vabdq(const int16x8_t & v0, const int16x8_t & v1) { return vabdq_s16(v0, v1); } +inline uint32x4_t vabdq(const uint32x4_t & v0, const uint32x4_t & v1) { return vabdq_u32(v0, v1); } +inline int32x4_t vabdq(const int32x4_t & v0, const int32x4_t & v1) { return vabdq_s32(v0, v1); } +inline float32x4_t vabdq(const float32x4_t & v0, const float32x4_t & v1) { return vabdq_f32(v0, v1); } + +////////////////////////////// vabd /////////////////////// + +inline uint8x8_t vabd(const uint8x8_t & v0, const uint8x8_t & v1) { return vabd_u8 (v0, v1); } +inline int8x8_t vabd(const int8x8_t & v0, const int8x8_t & v1) { return vabd_s8 (v0, v1); } +inline uint16x4_t vabd(const uint16x4_t & v0, const uint16x4_t & v1) { return vabd_u16(v0, v1); } +inline int16x4_t vabd(const int16x4_t & v0, const int16x4_t & v1) { return vabd_s16(v0, v1); } +inline uint32x2_t vabd(const uint32x2_t & v0, const uint32x2_t & v1) { return vabd_u32(v0, v1); } +inline int32x2_t vabd(const int32x2_t & v0, const int32x2_t & v1) { return vabd_s32(v0, v1); } +inline float32x2_t vabd(const float32x2_t & v0, const float32x2_t & v1) { return vabd_f32(v0, v1); } + +////////////////////////////// vminq /////////////////////// + +inline uint8x16_t vminq(const uint8x16_t & v0, const uint8x16_t & v1) { return vminq_u8 (v0, v1); } +inline int8x16_t vminq(const int8x16_t & v0, const int8x16_t & v1) { return vminq_s8 (v0, v1); } +inline uint16x8_t vminq(const uint16x8_t & v0, const uint16x8_t & v1) { return vminq_u16(v0, v1); } +inline int16x8_t vminq(const int16x8_t & v0, const int16x8_t & v1) { return vminq_s16(v0, v1); } +inline uint32x4_t vminq(const uint32x4_t & v0, const uint32x4_t & v1) { return vminq_u32(v0, v1); } +inline int32x4_t vminq(const int32x4_t & v0, const int32x4_t & v1) { return vminq_s32(v0, v1); } +inline float32x4_t vminq(const float32x4_t & v0, const float32x4_t & v1) { return vminq_f32(v0, v1); } + +////////////////////////////// vmin /////////////////////// + +inline uint8x8_t vmin(const uint8x8_t & v0, const uint8x8_t & v1) { return vmin_u8 (v0, v1); } +inline int8x8_t vmin(const int8x8_t & v0, const int8x8_t & v1) { return vmin_s8 (v0, v1); } +inline uint16x4_t vmin(const uint16x4_t & v0, const uint16x4_t & v1) { return vmin_u16(v0, v1); } +inline int16x4_t vmin(const int16x4_t & v0, const int16x4_t & v1) { return vmin_s16(v0, v1); } +inline uint32x2_t vmin(const uint32x2_t & v0, const uint32x2_t & v1) { return vmin_u32(v0, v1); } +inline int32x2_t vmin(const int32x2_t & v0, const int32x2_t & v1) { return vmin_s32(v0, v1); } +inline float32x2_t vmin(const float32x2_t & v0, const float32x2_t & v1) { return vmin_f32(v0, v1); } + +////////////////////////////// vmaxq /////////////////////// + +inline uint8x16_t vmaxq(const uint8x16_t & v0, const uint8x16_t & v1) { return vmaxq_u8 (v0, v1); } +inline int8x16_t vmaxq(const int8x16_t & v0, const int8x16_t & v1) { return vmaxq_s8 (v0, v1); } +inline uint16x8_t vmaxq(const uint16x8_t & v0, const uint16x8_t & v1) { return vmaxq_u16(v0, v1); } +inline int16x8_t vmaxq(const int16x8_t & v0, const int16x8_t & v1) { return vmaxq_s16(v0, v1); } +inline uint32x4_t vmaxq(const uint32x4_t & v0, const uint32x4_t & v1) { return vmaxq_u32(v0, v1); } +inline int32x4_t vmaxq(const int32x4_t & v0, const int32x4_t & v1) { return vmaxq_s32(v0, v1); } +inline float32x4_t vmaxq(const float32x4_t & v0, const float32x4_t & v1) { return vmaxq_f32(v0, v1); } + +////////////////////////////// vmax /////////////////////// + +inline uint8x8_t vmax(const uint8x8_t & v0, const uint8x8_t & v1) { return vmax_u8 (v0, v1); } +inline int8x8_t vmax(const int8x8_t & v0, const int8x8_t & v1) { return vmax_s8 (v0, v1); } +inline uint16x4_t vmax(const uint16x4_t & v0, const uint16x4_t & v1) { return vmax_u16(v0, v1); } +inline int16x4_t vmax(const int16x4_t & v0, const int16x4_t & v1) { return vmax_s16(v0, v1); } +inline uint32x2_t vmax(const uint32x2_t & v0, const uint32x2_t & v1) { return vmax_u32(v0, v1); } +inline int32x2_t vmax(const int32x2_t & v0, const int32x2_t & v1) { return vmax_s32(v0, v1); } +inline float32x2_t vmax(const float32x2_t & v0, const float32x2_t & v1) { return vmax_f32(v0, v1); } + +////////////////////////////// vdupq_n /////////////////////// + +inline uint8x16_t vdupq_n(const u8 & val) { return vdupq_n_u8(val); } +inline int8x16_t vdupq_n(const s8 & val) { return vdupq_n_s8(val); } +inline uint16x8_t vdupq_n(const u16 & val) { return vdupq_n_u16(val); } +inline int16x8_t vdupq_n(const s16 & val) { return vdupq_n_s16(val); } +inline uint32x4_t vdupq_n(const u32 & val) { return vdupq_n_u32(val); } +inline int32x4_t vdupq_n(const s32 & val) { return vdupq_n_s32(val); } +inline uint64x2_t vdupq_n(const u64 & val) { return vdupq_n_u64(val); } +inline int64x2_t vdupq_n(const s64 & val) { return vdupq_n_s64(val); } +inline float32x4_t vdupq_n(const f32 & val) { return vdupq_n_f32(val); } + +////////////////////////////// vdup_n /////////////////////// + +inline uint8x8_t vdup_n(const u8 & val) { return vdup_n_u8(val); } +inline int8x8_t vdup_n(const s8 & val) { return vdup_n_s8(val); } +inline uint16x4_t vdup_n(const u16 & val) { return vdup_n_u16(val); } +inline int16x4_t vdup_n(const s16 & val) { return vdup_n_s16(val); } +inline uint32x2_t vdup_n(const u32 & val) { return vdup_n_u32(val); } +inline int32x2_t vdup_n(const s32 & val) { return vdup_n_s32(val); } +inline uint64x1_t vdup_n(const u64 & val) { return vdup_n_u64(val); } +inline int64x1_t vdup_n(const s64 & val) { return vdup_n_s64(val); } +inline float32x2_t vdup_n(const f32 & val) { return vdup_n_f32(val); } + +////////////////////////////// vget_low /////////////////////// + +inline uint8x8_t vget_low(const uint8x16_t & v) { return vget_low_u8 (v); } +inline int8x8_t vget_low(const int8x16_t & v) { return vget_low_s8 (v); } +inline uint16x4_t vget_low(const uint16x8_t & v) { return vget_low_u16(v); } +inline int16x4_t vget_low(const int16x8_t & v) { return vget_low_s16(v); } +inline uint32x2_t vget_low(const uint32x4_t & v) { return vget_low_u32(v); } +inline int32x2_t vget_low(const int32x4_t & v) { return vget_low_s32(v); } +inline float32x2_t vget_low(const float32x4_t & v) { return vget_low_f32(v); } + +////////////////////////////// vget_high /////////////////////// + +inline uint8x8_t vget_high(const uint8x16_t & v) { return vget_high_u8 (v); } +inline int8x8_t vget_high(const int8x16_t & v) { return vget_high_s8 (v); } +inline uint16x4_t vget_high(const uint16x8_t & v) { return vget_high_u16(v); } +inline int16x4_t vget_high(const int16x8_t & v) { return vget_high_s16(v); } +inline uint32x2_t vget_high(const uint32x4_t & v) { return vget_high_u32(v); } +inline int32x2_t vget_high(const int32x4_t & v) { return vget_high_s32(v); } +inline float32x2_t vget_high(const float32x4_t & v) { return vget_high_f32(v); } + +////////////////////////////// vcombine /////////////////////// + +inline uint8x16_t vcombine(const uint8x8_t & v0, const uint8x8_t & v1) { return vcombine_u8 (v0, v1); } +inline int8x16_t vcombine(const int8x8_t & v0, const int8x8_t & v1) { return vcombine_s8 (v0, v1); } +inline uint16x8_t vcombine(const uint16x4_t & v0, const uint16x4_t & v1) { return vcombine_u16(v0, v1); } +inline int16x8_t vcombine(const int16x4_t & v0, const int16x4_t & v1) { return vcombine_s16(v0, v1); } +inline uint32x4_t vcombine(const uint32x2_t & v0, const uint32x2_t & v1) { return vcombine_u32(v0, v1); } +inline int32x4_t vcombine(const int32x2_t & v0, const int32x2_t & v1) { return vcombine_s32(v0, v1); } +inline float32x4_t vcombine(const float32x2_t & v0, const float32x2_t & v1) { return vcombine_f32(v0, v1); } + +////////////////////////////// vaddq /////////////////////// + +inline uint8x16_t vaddq(const uint8x16_t & v0, const uint8x16_t & v1) { return vaddq_u8 (v0, v1); } +inline int8x16_t vaddq(const int8x16_t & v0, const int8x16_t & v1) { return vaddq_s8 (v0, v1); } +inline uint16x8_t vaddq(const uint16x8_t & v0, const uint16x8_t & v1) { return vaddq_u16(v0, v1); } +inline int16x8_t vaddq(const int16x8_t & v0, const int16x8_t & v1) { return vaddq_s16(v0, v1); } +inline uint32x4_t vaddq(const uint32x4_t & v0, const uint32x4_t & v1) { return vaddq_u32(v0, v1); } +inline int32x4_t vaddq(const int32x4_t & v0, const int32x4_t & v1) { return vaddq_s32(v0, v1); } +inline float32x4_t vaddq(const float32x4_t & v0, const float32x4_t & v1) { return vaddq_f32(v0, v1); } + +////////////////////////////// vadd /////////////////////// + +inline uint8x8_t vadd(const uint8x8_t & v0, const uint8x8_t & v1) { return vadd_u8 (v0, v1); } +inline int8x8_t vadd(const int8x8_t & v0, const int8x8_t & v1) { return vadd_s8 (v0, v1); } +inline uint16x4_t vadd(const uint16x4_t & v0, const uint16x4_t & v1) { return vadd_u16(v0, v1); } +inline int16x4_t vadd(const int16x4_t & v0, const int16x4_t & v1) { return vadd_s16(v0, v1); } +inline uint32x2_t vadd(const uint32x2_t & v0, const uint32x2_t & v1) { return vadd_u32(v0, v1); } +inline int32x2_t vadd(const int32x2_t & v0, const int32x2_t & v1) { return vadd_s32(v0, v1); } +inline float32x2_t vadd(const float32x2_t & v0, const float32x2_t & v1) { return vadd_f32(v0, v1); } + +////////////////////////////// vqaddq /////////////////////// + +inline uint8x16_t vqaddq(const uint8x16_t & v0, const uint8x16_t & v1) { return vqaddq_u8 (v0, v1); } +inline int8x16_t vqaddq(const int8x16_t & v0, const int8x16_t & v1) { return vqaddq_s8 (v0, v1); } +inline uint16x8_t vqaddq(const uint16x8_t & v0, const uint16x8_t & v1) { return vqaddq_u16(v0, v1); } +inline int16x8_t vqaddq(const int16x8_t & v0, const int16x8_t & v1) { return vqaddq_s16(v0, v1); } +inline uint32x4_t vqaddq(const uint32x4_t & v0, const uint32x4_t & v1) { return vqaddq_u32(v0, v1); } +inline int32x4_t vqaddq(const int32x4_t & v0, const int32x4_t & v1) { return vqaddq_s32(v0, v1); } + +////////////////////////////// vqadd /////////////////////// + +inline uint8x8_t vqadd(const uint8x8_t & v0, const uint8x8_t & v1) { return vqadd_u8 (v0, v1); } +inline int8x8_t vqadd(const int8x8_t & v0, const int8x8_t & v1) { return vqadd_s8 (v0, v1); } +inline uint16x4_t vqadd(const uint16x4_t & v0, const uint16x4_t & v1) { return vqadd_u16(v0, v1); } +inline int16x4_t vqadd(const int16x4_t & v0, const int16x4_t & v1) { return vqadd_s16(v0, v1); } +inline uint32x2_t vqadd(const uint32x2_t & v0, const uint32x2_t & v1) { return vqadd_u32(v0, v1); } +inline int32x2_t vqadd(const int32x2_t & v0, const int32x2_t & v1) { return vqadd_s32(v0, v1); } + +////////////////////////////// vsubq /////////////////////// + +inline uint8x16_t vsubq(const uint8x16_t & v0, const uint8x16_t & v1) { return vsubq_u8 (v0, v1); } +inline int8x16_t vsubq(const int8x16_t & v0, const int8x16_t & v1) { return vsubq_s8 (v0, v1); } +inline uint16x8_t vsubq(const uint16x8_t & v0, const uint16x8_t & v1) { return vsubq_u16(v0, v1); } +inline int16x8_t vsubq(const int16x8_t & v0, const int16x8_t & v1) { return vsubq_s16(v0, v1); } +inline uint32x4_t vsubq(const uint32x4_t & v0, const uint32x4_t & v1) { return vsubq_u32(v0, v1); } +inline int32x4_t vsubq(const int32x4_t & v0, const int32x4_t & v1) { return vsubq_s32(v0, v1); } +inline float32x4_t vsubq(const float32x4_t & v0, const float32x4_t & v1) { return vsubq_f32(v0, v1); } + +////////////////////////////// vsub /////////////////////// + +inline uint8x8_t vsub(const uint8x8_t & v0, const uint8x8_t & v1) { return vsub_u8 (v0, v1); } +inline int8x8_t vsub(const int8x8_t & v0, const int8x8_t & v1) { return vsub_s8 (v0, v1); } +inline uint16x4_t vsub(const uint16x4_t & v0, const uint16x4_t & v1) { return vsub_u16(v0, v1); } +inline int16x4_t vsub(const int16x4_t & v0, const int16x4_t & v1) { return vsub_s16(v0, v1); } +inline uint32x2_t vsub(const uint32x2_t & v0, const uint32x2_t & v1) { return vsub_u32(v0, v1); } +inline int32x2_t vsub(const int32x2_t & v0, const int32x2_t & v1) { return vsub_s32(v0, v1); } +inline float32x2_t vsub(const float32x2_t & v0, const float32x2_t & v1) { return vsub_f32(v0, v1); } + +////////////////////////////// vqsubq /////////////////////// + +inline uint8x16_t vqsubq(const uint8x16_t & v0, const uint8x16_t & v1) { return vqsubq_u8 (v0, v1); } +inline int8x16_t vqsubq(const int8x16_t & v0, const int8x16_t & v1) { return vqsubq_s8 (v0, v1); } +inline uint16x8_t vqsubq(const uint16x8_t & v0, const uint16x8_t & v1) { return vqsubq_u16(v0, v1); } +inline int16x8_t vqsubq(const int16x8_t & v0, const int16x8_t & v1) { return vqsubq_s16(v0, v1); } +inline uint32x4_t vqsubq(const uint32x4_t & v0, const uint32x4_t & v1) { return vqsubq_u32(v0, v1); } +inline int32x4_t vqsubq(const int32x4_t & v0, const int32x4_t & v1) { return vqsubq_s32(v0, v1); } +inline uint64x2_t vqsubq(const uint64x2_t & v0, const uint64x2_t & v1) { return vqsubq_u64(v0, v1); } +inline int64x2_t vqsubq(const int64x2_t & v0, const int64x2_t & v1) { return vqsubq_s64(v0, v1); } + +////////////////////////////// vqsub /////////////////////// + +inline uint8x8_t vqsub(const uint8x8_t & v0, const uint8x8_t & v1) { return vqsub_u8 (v0, v1); } +inline int8x8_t vqsub(const int8x8_t & v0, const int8x8_t & v1) { return vqsub_s8 (v0, v1); } +inline uint16x4_t vqsub(const uint16x4_t & v0, const uint16x4_t & v1) { return vqsub_u16(v0, v1); } +inline int16x4_t vqsub(const int16x4_t & v0, const int16x4_t & v1) { return vqsub_s16(v0, v1); } +inline uint32x2_t vqsub(const uint32x2_t & v0, const uint32x2_t & v1) { return vqsub_u32(v0, v1); } +inline int32x2_t vqsub(const int32x2_t & v0, const int32x2_t & v1) { return vqsub_s32(v0, v1); } +inline uint64x1_t vqsub(const uint64x1_t & v0, const uint64x1_t & v1) { return vqsub_u64(v0, v1); } +inline int64x1_t vqsub(const int64x1_t & v0, const int64x1_t & v1) { return vqsub_s64(v0, v1); } + +////////////////////////////// vmull /////////////////////// + +inline uint16x8_t vmull(const uint8x8_t & v0, const uint8x8_t & v1) { return vmull_u8 (v0, v1); } +inline int16x8_t vmull(const int8x8_t & v0, const int8x8_t & v1) { return vmull_s8 (v0, v1); } +inline uint32x4_t vmull(const uint16x4_t & v0, const uint16x4_t & v1) { return vmull_u16(v0, v1); } +inline int32x4_t vmull(const int16x4_t & v0, const int16x4_t & v1) { return vmull_s16(v0, v1); } +inline uint64x2_t vmull(const uint32x2_t & v0, const uint32x2_t & v1) { return vmull_u32(v0, v1); } +inline int64x2_t vmull(const int32x2_t & v0, const int32x2_t & v1) { return vmull_s32(v0, v1); } + +////////////////////////////// vrev64q /////////////////////// + +inline uint8x16_t vrev64q(const uint8x16_t & v) { return vrev64q_u8 (v); } +inline int8x16_t vrev64q(const int8x16_t & v) { return vrev64q_s8 (v); } +inline uint16x8_t vrev64q(const uint16x8_t & v) { return vrev64q_u16(v); } +inline int16x8_t vrev64q(const int16x8_t & v) { return vrev64q_s16(v); } +inline uint32x4_t vrev64q(const uint32x4_t & v) { return vrev64q_u32(v); } +inline int32x4_t vrev64q(const int32x4_t & v) { return vrev64q_s32(v); } +inline float32x4_t vrev64q(const float32x4_t & v) { return vrev64q_f32(v); } + +////////////////////////////// vrev64 /////////////////////// + +inline uint8x8_t vrev64(const uint8x8_t & v) { return vrev64_u8 (v); } +inline int8x8_t vrev64(const int8x8_t & v) { return vrev64_s8 (v); } +inline uint16x4_t vrev64(const uint16x4_t & v) { return vrev64_u16(v); } +inline int16x4_t vrev64(const int16x4_t & v) { return vrev64_s16(v); } +inline uint32x2_t vrev64(const uint32x2_t & v) { return vrev64_u32(v); } +inline int32x2_t vrev64(const int32x2_t & v) { return vrev64_s32(v); } +inline float32x2_t vrev64(const float32x2_t & v) { return vrev64_f32(v); } + +////////////////////////////// vceqq /////////////////////// + +inline uint8x16_t vceqq(const uint8x16_t & v0, const uint8x16_t & v1) { return vceqq_u8 (v0, v1); } +inline uint8x16_t vceqq(const int8x16_t & v0, const int8x16_t & v1) { return vceqq_s8 (v0, v1); } +inline uint16x8_t vceqq(const uint16x8_t & v0, const uint16x8_t & v1) { return vceqq_u16(v0, v1); } +inline uint16x8_t vceqq(const int16x8_t & v0, const int16x8_t & v1) { return vceqq_s16(v0, v1); } +inline uint32x4_t vceqq(const uint32x4_t & v0, const uint32x4_t & v1) { return vceqq_u32(v0, v1); } +inline uint32x4_t vceqq(const int32x4_t & v0, const int32x4_t & v1) { return vceqq_s32(v0, v1); } +inline uint32x4_t vceqq(const float32x4_t & v0, const float32x4_t & v1) { return vceqq_f32(v0, v1); } + +////////////////////////////// vceq /////////////////////// + +inline uint8x8_t vceq(const uint8x8_t & v0, const uint8x8_t & v1) { return vceq_u8 (v0, v1); } +inline uint8x8_t vceq(const int8x8_t & v0, const int8x8_t & v1) { return vceq_s8 (v0, v1); } +inline uint16x4_t vceq(const uint16x4_t & v0, const uint16x4_t & v1) { return vceq_u16(v0, v1); } +inline uint16x4_t vceq(const int16x4_t & v0, const int16x4_t & v1) { return vceq_s16(v0, v1); } +inline uint32x2_t vceq(const uint32x2_t & v0, const uint32x2_t & v1) { return vceq_u32(v0, v1); } +inline uint32x2_t vceq(const int32x2_t & v0, const int32x2_t & v1) { return vceq_s32(v0, v1); } +inline uint32x2_t vceq(const float32x2_t & v0, const float32x2_t & v1) { return vceq_f32(v0, v1); } + +////////////////////////////// vcgtq /////////////////////// + +inline uint8x16_t vcgtq(const uint8x16_t & v0, const uint8x16_t & v1) { return vcgtq_u8 (v0, v1); } +inline uint8x16_t vcgtq(const int8x16_t & v0, const int8x16_t & v1) { return vcgtq_s8 (v0, v1); } +inline uint16x8_t vcgtq(const uint16x8_t & v0, const uint16x8_t & v1) { return vcgtq_u16(v0, v1); } +inline uint16x8_t vcgtq(const int16x8_t & v0, const int16x8_t & v1) { return vcgtq_s16(v0, v1); } +inline uint32x4_t vcgtq(const uint32x4_t & v0, const uint32x4_t & v1) { return vcgtq_u32(v0, v1); } +inline uint32x4_t vcgtq(const int32x4_t & v0, const int32x4_t & v1) { return vcgtq_s32(v0, v1); } +inline uint32x4_t vcgtq(const float32x4_t & v0, const float32x4_t & v1) { return vcgtq_f32(v0, v1); } + +////////////////////////////// vcgt /////////////////////// + +inline uint8x8_t vcgt(const uint8x8_t & v0, const uint8x8_t & v1) { return vcgt_u8 (v0, v1); } +inline uint8x8_t vcgt(const int8x8_t & v0, const int8x8_t & v1) { return vcgt_s8 (v0, v1); } +inline uint16x4_t vcgt(const uint16x4_t & v0, const uint16x4_t & v1) { return vcgt_u16(v0, v1); } +inline uint16x4_t vcgt(const int16x4_t & v0, const int16x4_t & v1) { return vcgt_s16(v0, v1); } +inline uint32x2_t vcgt(const uint32x2_t & v0, const uint32x2_t & v1) { return vcgt_u32(v0, v1); } +inline uint32x2_t vcgt(const int32x2_t & v0, const int32x2_t & v1) { return vcgt_s32(v0, v1); } +inline uint32x2_t vcgt(const float32x2_t & v0, const float32x2_t & v1) { return vcgt_f32(v0, v1); } + +////////////////////////////// vcgeq /////////////////////// + +inline uint8x16_t vcgeq(const uint8x16_t & v0, const uint8x16_t & v1) { return vcgeq_u8 (v0, v1); } +inline uint8x16_t vcgeq(const int8x16_t & v0, const int8x16_t & v1) { return vcgeq_s8 (v0, v1); } +inline uint16x8_t vcgeq(const uint16x8_t & v0, const uint16x8_t & v1) { return vcgeq_u16(v0, v1); } +inline uint16x8_t vcgeq(const int16x8_t & v0, const int16x8_t & v1) { return vcgeq_s16(v0, v1); } +inline uint32x4_t vcgeq(const uint32x4_t & v0, const uint32x4_t & v1) { return vcgeq_u32(v0, v1); } +inline uint32x4_t vcgeq(const int32x4_t & v0, const int32x4_t & v1) { return vcgeq_s32(v0, v1); } +inline uint32x4_t vcgeq(const float32x4_t & v0, const float32x4_t & v1) { return vcgeq_f32(v0, v1); } + +////////////////////////////// vcge /////////////////////// + +inline uint8x8_t vcge(const uint8x8_t & v0, const uint8x8_t & v1) { return vcge_u8 (v0, v1); } +inline uint8x8_t vcge(const int8x8_t & v0, const int8x8_t & v1) { return vcge_s8 (v0, v1); } +inline uint16x4_t vcge(const uint16x4_t & v0, const uint16x4_t & v1) { return vcge_u16(v0, v1); } +inline uint16x4_t vcge(const int16x4_t & v0, const int16x4_t & v1) { return vcge_s16(v0, v1); } +inline uint32x2_t vcge(const uint32x2_t & v0, const uint32x2_t & v1) { return vcge_u32(v0, v1); } +inline uint32x2_t vcge(const int32x2_t & v0, const int32x2_t & v1) { return vcge_s32(v0, v1); } +inline uint32x2_t vcge(const float32x2_t & v0, const float32x2_t & v1) { return vcge_f32(v0, v1); } + +////////////////////////////// vandq /////////////////////// + +inline uint8x16_t vandq(const uint8x16_t & v0, const uint8x16_t & v1) { return vandq_u8 (v0, v1); } +inline int8x16_t vandq(const int8x16_t & v0, const int8x16_t & v1) { return vandq_s8 (v0, v1); } +inline uint16x8_t vandq(const uint16x8_t & v0, const uint16x8_t & v1) { return vandq_u16(v0, v1); } +inline int16x8_t vandq(const int16x8_t & v0, const int16x8_t & v1) { return vandq_s16(v0, v1); } +inline uint32x4_t vandq(const uint32x4_t & v0, const uint32x4_t & v1) { return vandq_u32(v0, v1); } +inline int32x4_t vandq(const int32x4_t & v0, const int32x4_t & v1) { return vandq_s32(v0, v1); } + +////////////////////////////// vand /////////////////////// + +inline uint8x8_t vand(const uint8x8_t & v0, const uint8x8_t & v1) { return vand_u8 (v0, v1); } +inline int8x8_t vand(const int8x8_t & v0, const int8x8_t & v1) { return vand_s8 (v0, v1); } +inline uint16x4_t vand(const uint16x4_t & v0, const uint16x4_t & v1) { return vand_u16(v0, v1); } +inline int16x4_t vand(const int16x4_t & v0, const int16x4_t & v1) { return vand_s16(v0, v1); } +inline uint32x2_t vand(const uint32x2_t & v0, const uint32x2_t & v1) { return vand_u32(v0, v1); } +inline int32x2_t vand(const int32x2_t & v0, const int32x2_t & v1) { return vand_s32(v0, v1); } + +////////////////////////////// vmovn /////////////////////// + +inline uint8x8_t vmovn(const uint16x8_t & v) { return vmovn_u16(v); } +inline int8x8_t vmovn(const int16x8_t & v) { return vmovn_s16(v); } +inline uint16x4_t vmovn(const uint32x4_t & v) { return vmovn_u32(v); } +inline int16x4_t vmovn(const int32x4_t & v) { return vmovn_s32(v); } +inline uint32x2_t vmovn(const uint64x2_t & v) { return vmovn_u64(v); } +inline int32x2_t vmovn(const int64x2_t & v) { return vmovn_s64(v); } + +////////////////////////////// vqmovn /////////////////////// + +inline uint8x8_t vqmovn(const uint16x8_t & v) { return vqmovn_u16(v); } +inline int8x8_t vqmovn(const int16x8_t & v) { return vqmovn_s16(v); } +inline uint16x4_t vqmovn(const uint32x4_t & v) { return vqmovn_u32(v); } +inline int16x4_t vqmovn(const int32x4_t & v) { return vqmovn_s32(v); } +inline uint32x2_t vqmovn(const uint64x2_t & v) { return vqmovn_u64(v); } +inline int32x2_t vqmovn(const int64x2_t & v) { return vqmovn_s64(v); } + +////////////////////////////// vmovl /////////////////////// + +inline uint16x8_t vmovl(const uint8x8_t & v) { return vmovl_u8(v); } +inline int16x8_t vmovl(const int8x8_t & v) { return vmovl_s8(v); } +inline uint32x4_t vmovl(const uint16x4_t & v) { return vmovl_u16(v); } +inline int32x4_t vmovl(const int16x4_t & v) { return vmovl_s16(v); } + +////////////////////////////// vmvnq /////////////////////// + +inline uint8x16_t vmvnq(const uint8x16_t & v) { return vmvnq_u8 (v); } +inline int8x16_t vmvnq(const int8x16_t & v) { return vmvnq_s8 (v); } +inline uint16x8_t vmvnq(const uint16x8_t & v) { return vmvnq_u16(v); } +inline int16x8_t vmvnq(const int16x8_t & v) { return vmvnq_s16(v); } +inline uint32x4_t vmvnq(const uint32x4_t & v) { return vmvnq_u32(v); } +inline int32x4_t vmvnq(const int32x4_t & v) { return vmvnq_s32(v); } + +////////////////////////////// vmvn /////////////////////// + +inline uint8x8_t vmvn(const uint8x8_t & v) { return vmvn_u8 (v); } +inline int8x8_t vmvn(const int8x8_t & v) { return vmvn_s8 (v); } +inline uint16x4_t vmvn(const uint16x4_t & v) { return vmvn_u16(v); } +inline int16x4_t vmvn(const int16x4_t & v) { return vmvn_s16(v); } +inline uint32x2_t vmvn(const uint32x2_t & v) { return vmvn_u32(v); } +inline int32x2_t vmvn(const int32x2_t & v) { return vmvn_s32(v); } + +////////////////////////////// vbicq /////////////////////// + +inline uint8x16_t vbicq(const uint8x16_t & v0, const uint8x16_t & v1) { return vbicq_u8 (v0, v1); } +inline int8x16_t vbicq(const int8x16_t & v0, const int8x16_t & v1) { return vbicq_s8 (v0, v1); } +inline uint16x8_t vbicq(const uint16x8_t & v0, const uint16x8_t & v1) { return vbicq_u16(v0, v1); } +inline int16x8_t vbicq(const int16x8_t & v0, const int16x8_t & v1) { return vbicq_s16(v0, v1); } +inline uint32x4_t vbicq(const uint32x4_t & v0, const uint32x4_t & v1) { return vbicq_u32(v0, v1); } +inline int32x4_t vbicq(const int32x4_t & v0, const int32x4_t & v1) { return vbicq_s32(v0, v1); } +inline uint64x2_t vbicq(const uint64x2_t & v0, const uint64x2_t & v1) { return vbicq_u64(v0, v1); } +inline int64x2_t vbicq(const int64x2_t & v0, const int64x2_t & v1) { return vbicq_s64(v0, v1); } + +////////////////////////////// vbic /////////////////////// + +inline uint8x8_t vbic(const uint8x8_t & v0, const uint8x8_t & v1) { return vbic_u8 (v0, v1); } +inline int8x8_t vbic(const int8x8_t & v0, const int8x8_t & v1) { return vbic_s8 (v0, v1); } +inline uint16x4_t vbic(const uint16x4_t & v0, const uint16x4_t & v1) { return vbic_u16(v0, v1); } +inline int16x4_t vbic(const int16x4_t & v0, const int16x4_t & v1) { return vbic_s16(v0, v1); } +inline uint32x2_t vbic(const uint32x2_t & v0, const uint32x2_t & v1) { return vbic_u32(v0, v1); } +inline int32x2_t vbic(const int32x2_t & v0, const int32x2_t & v1) { return vbic_s32(v0, v1); } +inline uint64x1_t vbic(const uint64x1_t & v0, const uint64x1_t & v1) { return vbic_u64(v0, v1); } +inline int64x1_t vbic(const int64x1_t & v0, const int64x1_t & v1) { return vbic_s64(v0, v1); } + +////////////////////////////// vtransform /////////////////////// + +template +void vtransform(Size2D size, + const typename Op::type * src0Base, ptrdiff_t src0Stride, + const typename Op::type * src1Base, ptrdiff_t src1Stride, + typename Op::type * dstBase, ptrdiff_t dstStride, const Op & op) +{ + typedef typename Op::type type; + typedef typename VecTraits::vec128 vec128; + typedef typename VecTraits::vec64 vec64; + + if (src0Stride == src1Stride && src0Stride == dstStride && + src0Stride == (ptrdiff_t)(size.width * sizeof(type))) + { + size.width *= size.height; + size.height = 1; + } + + const size_t step_base = 32 / sizeof(type); + size_t roiw_base = size.width >= (step_base - 1) ? size.width - step_base + 1 : 0; + const size_t step_tail = 8 / sizeof(type); + size_t roiw_tail = size.width >= (step_tail - 1) ? size.width - step_tail + 1 : 0; + + for (size_t y = 0; y < size.height; ++y) + { + const type * src0 = internal::getRowPtr(src0Base, src0Stride, y); + const type * src1 = internal::getRowPtr(src1Base, src1Stride, y); + typename Op::type * dst = internal::getRowPtr(dstBase, dstStride, y); + size_t x = 0; + + for( ; x < roiw_base; x += step_base ) + { + internal::prefetch(src0 + x); + internal::prefetch(src1 + x); + + vec128 v_src00 = vld1q(src0 + x), v_src01 = vld1q(src0 + x + 16 / sizeof(type)); + vec128 v_src10 = vld1q(src1 + x), v_src11 = vld1q(src1 + x + 16 / sizeof(type)); + vec128 v_dst; + + op(v_src00, v_src10, v_dst); + vst1q(dst + x, v_dst); + + op(v_src01, v_src11, v_dst); + vst1q(dst + x + 16 / sizeof(type), v_dst); + } + for( ; x < roiw_tail; x += step_tail ) + { + vec64 v_src0 = vld1(src0 + x); + vec64 v_src1 = vld1(src1 + x); + vec64 v_dst; + + op(v_src0, v_src1, v_dst); + vst1(dst + x, v_dst); + } + + for (; x < size.width; ++x) + { + op(src0 + x, src1 + x, dst + x); + } + } +} + +} } + +#endif // CAROTENE_NEON + +#endif diff --git a/3rdparty/carotene/src/warp_affine.cpp b/3rdparty/carotene/src/warp_affine.cpp new file mode 100644 index 0000000000..d546efbc10 --- /dev/null +++ b/3rdparty/carotene/src/warp_affine.cpp @@ -0,0 +1,434 @@ +/* + * By downloading, copying, installing or using the software you agree to this license. + * If you do not agree to this license, do not download, install, + * copy or use the software. + * + * + * License Agreement + * For Open Source Computer Vision Library + * (3-clause BSD License) + * + * Copyright (C) 2015, NVIDIA Corporation, all rights reserved. + * Third party copyrights are property of their respective owners. + * + * Redistribution and use in source and binary forms, with or without modification, + * are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * * Neither the names of the copyright holders nor the names of the contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * This software is provided by the copyright holders and contributors "as is" and + * any express or implied warranties, including, but not limited to, the implied + * warranties of merchantability and fitness for a particular purpose are disclaimed. + * In no event shall copyright holders or contributors be liable for any direct, + * indirect, incidental, special, exemplary, or consequential damages + * (including, but not limited to, procurement of substitute goods or services; + * loss of use, data, or profits; or business interruption) however caused + * and on any theory of liability, whether in contract, strict liability, + * or tort (including negligence or otherwise) arising in any way out of + * the use of this software, even if advised of the possibility of such damage. + */ + +#include "remap.hpp" + +namespace CAROTENE_NS { + +bool isWarpAffineNearestNeighborSupported(const Size2D &ssize) +{ +#if SIZE_MAX > UINT32_MAX + return !(ssize.width > 0xffffFFFF || ssize.height > 0xffffFFFF) && // Restrict image size since internal index evaluation + // is performed with u32 + isSupportedConfiguration(); +#else + (void)ssize; + return isSupportedConfiguration(); +#endif +} + +bool isWarpAffineLinearSupported(const Size2D &ssize) +{ +#if SIZE_MAX > UINT32_MAX + return !(ssize.width > 0xffffFFFF || ssize.height > 0xffffFFFF) && // Restrict image size since internal index evaluation + // is performed with u32 + isSupportedConfiguration(); +#else + (void)ssize; + return isSupportedConfiguration(); +#endif +} + +void warpAffineNearestNeighbor(const Size2D &ssize, const Size2D &dsize, + const u8 * srcBase, ptrdiff_t srcStride, + const f32 * m, + u8 * dstBase, ptrdiff_t dstStride, + BORDER_MODE borderMode, u8 borderValue) +{ + internal::assertSupportedConfiguration(isWarpAffineNearestNeighborSupported(ssize)); +#ifdef CAROTENE_NEON + using namespace internal; + + s32 _map[BLOCK_SIZE * BLOCK_SIZE + 16]; + s32 * map = alignPtr(_map, 16); + + int32x4_t v_width4 = vdupq_n_s32(ssize.width - 1), v_height4 = vdupq_n_s32(ssize.height - 1); + int32x4_t v_step4 = vdupq_n_s32(srcStride); + float32x4_t v_4 = vdupq_n_f32(4.0f); + + float32x4_t v_m0 = vdupq_n_f32(m[0]); + float32x4_t v_m1 = vdupq_n_f32(m[1]); + float32x4_t v_m2 = vdupq_n_f32(m[2]); + float32x4_t v_m3 = vdupq_n_f32(m[3]); + float32x4_t v_m4 = vdupq_n_f32(m[4]); + float32x4_t v_m5 = vdupq_n_f32(m[5]); + + if (borderMode == BORDER_MODE_REPLICATE) + { + int32x4_t v_zero4 = vdupq_n_s32(0); + + for (size_t i = 0; i < dsize.height; i += BLOCK_SIZE) + { + size_t blockHeight = std::min(BLOCK_SIZE, dsize.height - i); + for (size_t j = 0; j < dsize.width; j += BLOCK_SIZE) + { + size_t blockWidth = std::min(BLOCK_SIZE, dsize.width - j); + + // compute table + for (size_t y = 0; y < blockHeight; ++y) + { + s32 * map_row = getRowPtr(&map[0], blockWidth * sizeof(s32), y); + + size_t x = 0, y_ = y + i; + f32 indeces[4] = { j + 0.0f, j + 1.0f, j + 2.0f, j + 3.0f }; + float32x4_t v_x = vld1q_f32(indeces), v_y = vdupq_n_f32(y_); + float32x4_t v_yx = vmlaq_f32(v_m4, v_m2, v_y), v_yy = vmlaq_f32(v_m5, v_m3, v_y); + + for ( ; x + 4 <= blockWidth; x += 4) + { + float32x4_t v_src_xf = vmlaq_f32(v_yx, v_m0, v_x); + float32x4_t v_src_yf = vmlaq_f32(v_yy, v_m1, v_x); + + int32x4_t v_src_x = vmaxq_s32(v_zero4, vminq_s32(v_width4, vcvtq_s32_f32(v_src_xf))); + int32x4_t v_src_y = vmaxq_s32(v_zero4, vminq_s32(v_height4, vcvtq_s32_f32(v_src_yf))); + int32x4_t v_src_index = vmlaq_s32(v_src_x, v_src_y, v_step4); + vst1q_s32(map_row + x, v_src_index); + + v_x = vaddq_f32(v_x, v_4); + } + + f32 yx = m[2] * y_ + m[4], yy = m[3] * y_ + m[5]; + for (ptrdiff_t x_ = x + j; x < blockWidth; ++x, ++x_) + { + f32 src_x_f = m[0] * x_ + yx; + f32 src_y_f = m[1] * x_ + yy; + s32 src_x = floorf(src_x_f), src_y = floorf(src_y_f); + + src_x = std::max(0, std::min(ssize.width - 1, src_x)); + src_y = std::max(0, std::min(ssize.height - 1, src_y)); + map_row[x] = src_y * srcStride + src_x; + } + } + + // make remap + remapNearestNeighborReplicate(Size2D(blockWidth, blockHeight), srcBase, &map[0], + getRowPtr(dstBase, dstStride, i) + j, dstStride); + } + } + } + else if (borderMode == BORDER_MODE_CONSTANT) + { + int32x4_t v_m1_4 = vdupq_n_s32(-1); + float32x4_t v_zero4 = vdupq_n_f32(0.0f); + + for (size_t i = 0; i < dsize.height; i += BLOCK_SIZE) + { + size_t blockHeight = std::min(BLOCK_SIZE, dsize.height - i); + for (size_t j = 0; j < dsize.width; j += BLOCK_SIZE) + { + size_t blockWidth = std::min(BLOCK_SIZE, dsize.width - j); + + // compute table + for (size_t y = 0; y < blockHeight; ++y) + { + s32 * map_row = getRowPtr(&map[0], blockWidth * sizeof(s32), y); + + size_t x = 0, y_ = y + i; + f32 indeces[4] = { j + 0.0f, j + 1.0f, j + 2.0f, j + 3.0f }; + float32x4_t v_x = vld1q_f32(indeces), v_y = vdupq_n_f32(y_); + float32x4_t v_yx = vmlaq_f32(v_m4, v_m2, v_y), v_yy = vmlaq_f32(v_m5, v_m3, v_y); + + for ( ; x + 4 <= blockWidth; x += 4) + { + float32x4_t v_src_xf = vmlaq_f32(v_yx, v_m0, v_x); + float32x4_t v_src_yf = vmlaq_f32(v_yy, v_m1, v_x); + + int32x4_t v_src_x = vcvtq_s32_f32(v_src_xf); + int32x4_t v_src_y = vcvtq_s32_f32(v_src_yf); + uint32x4_t v_mask = vandq_u32(vandq_u32(vcgeq_f32(v_src_xf, v_zero4), vcleq_s32(v_src_x, v_width4)), + vandq_u32(vcgeq_f32(v_src_yf, v_zero4), vcleq_s32(v_src_y, v_height4))); + int32x4_t v_src_index = vbslq_s32(v_mask, vmlaq_s32(v_src_x, v_src_y, v_step4), v_m1_4); + vst1q_s32(map_row + x, v_src_index); + + v_x = vaddq_f32(v_x, v_4); + } + + f32 yx = m[2] * y_ + m[4], yy = m[3] * y_ + m[5]; + for (ptrdiff_t x_ = x + j; x < blockWidth; ++x, ++x_) + { + f32 src_x_f = m[0] * x_ + yx; + f32 src_y_f = m[1] * x_ + yy; + s32 src_x = floorf(src_x_f), src_y = floorf(src_y_f); + + map_row[x] = (src_x >= 0) && (src_x < (s32)ssize.width) && + (src_y >= 0) && (src_y < (s32)ssize.height) ? src_y * srcStride + src_x : -1; + } + } + + // make remap + remapNearestNeighborConst(Size2D(blockWidth, blockHeight), srcBase, &map[0], + getRowPtr(dstBase, dstStride, i) + j, dstStride, borderValue); + } + } + } +#else + (void)ssize; + (void)dsize; + (void)srcBase; + (void)srcStride; + (void)m; + (void)dstBase; + (void)dstStride; + (void)borderMode; + (void)borderValue; +#endif +} + +void warpAffineLinear(const Size2D &ssize, const Size2D &dsize, + const u8 * srcBase, ptrdiff_t srcStride, + const f32 * m, + u8 * dstBase, ptrdiff_t dstStride, + BORDER_MODE borderMode, u8 borderValue) +{ + internal::assertSupportedConfiguration(isWarpAffineLinearSupported(ssize)); +#ifdef CAROTENE_NEON + using namespace internal; + + s32 _map[((BLOCK_SIZE * BLOCK_SIZE) << 2) + 16]; + f32 _coeffs[((BLOCK_SIZE * BLOCK_SIZE) << 1) + 16]; + s32 * map = alignPtr(_map, 16); + f32 * coeffs = alignPtr(_coeffs, 16); + + int32x4_t v_width4 = vdupq_n_s32(ssize.width - 1), v_height4 = vdupq_n_s32(ssize.height - 1); + int32x4_t v_step4 = vdupq_n_s32(srcStride), v_1 = vdupq_n_s32(1); + float32x4_t v_zero4f = vdupq_n_f32(0.0f), v_one4f = vdupq_n_f32(1.0f); + + float32x4_t v_m0 = vdupq_n_f32(m[0]); + float32x4_t v_m1 = vdupq_n_f32(m[1]); + float32x4_t v_m2 = vdupq_n_f32(m[2]); + float32x4_t v_m3 = vdupq_n_f32(m[3]); + float32x4_t v_m4 = vdupq_n_f32(m[4]); + float32x4_t v_m5 = vdupq_n_f32(m[5]); + + if (borderMode == BORDER_MODE_REPLICATE) + { + int32x4_t v_zero4 = vdupq_n_s32(0); + + for (size_t i = 0; i < dsize.height; i += BLOCK_SIZE) + { + size_t blockHeight = std::min(BLOCK_SIZE, dsize.height - i); + for (size_t j = 0; j < dsize.width; j += BLOCK_SIZE) + { + size_t blockWidth = std::min(BLOCK_SIZE, dsize.width - j); + + // compute table + for (size_t y = 0; y < blockHeight; ++y) + { + s32 * map_row = getRowPtr(map, blockWidth * sizeof(s32) * 4, y); + f32 * coeff_row = getRowPtr(coeffs, blockWidth * sizeof(f32) * 2, y); + + size_t x = 0, y_ = y + i; + f32 indeces[4] = { j + 0.0f, j + 1.0f, j + 2.0f, j + 3.0f }; + float32x4_t v_x = vld1q_f32(indeces), v_y = vdupq_n_f32(y_), v_4 = vdupq_n_f32(4.0f); + float32x4_t v_yx = vmlaq_f32(v_m4, v_m2, v_y), v_yy = vmlaq_f32(v_m5, v_m3, v_y); + + for ( ; x + 4 <= blockWidth; x += 4) + { + float32x4_t v_src_xf = vmlaq_f32(v_yx, v_m0, v_x); + float32x4_t v_src_yf = vmlaq_f32(v_yy, v_m1, v_x); + + int32x4_t v_src_x = vcvtq_s32_f32(v_src_xf); + int32x4_t v_src_y = vcvtq_s32_f32(v_src_yf); + + float32x4x2_t v_coeff; + v_coeff.val[0] = vsubq_f32(v_src_xf, vcvtq_f32_s32(v_src_x)); + v_coeff.val[1] = vsubq_f32(v_src_yf, vcvtq_f32_s32(v_src_y)); + uint32x4_t v_maskx = vcltq_f32(v_coeff.val[0], v_zero4f); + uint32x4_t v_masky = vcltq_f32(v_coeff.val[1], v_zero4f); + v_coeff.val[0] = vbslq_f32(v_maskx, vaddq_f32(v_one4f, v_coeff.val[0]), v_coeff.val[0]); + v_coeff.val[1] = vbslq_f32(v_masky, vaddq_f32(v_one4f, v_coeff.val[1]), v_coeff.val[1]); + v_src_x = vbslq_s32(v_maskx, vsubq_s32(v_src_x, v_1), v_src_x); + v_src_y = vbslq_s32(v_masky, vsubq_s32(v_src_y, v_1), v_src_y); + + int32x4_t v_dst0_x = vmaxq_s32(v_zero4, vminq_s32(v_width4, v_src_x)); + int32x4_t v_dst0_y = vmaxq_s32(v_zero4, vminq_s32(v_height4, v_src_y)); + int32x4_t v_dst1_x = vmaxq_s32(v_zero4, vminq_s32(v_width4, vaddq_s32(v_1, v_src_x))); + int32x4_t v_dst1_y = vmaxq_s32(v_zero4, vminq_s32(v_height4, vaddq_s32(v_1, v_src_y))); + + int32x4x4_t v_dst_index; + v_dst_index.val[0] = vmlaq_s32(v_dst0_x, v_dst0_y, v_step4); + v_dst_index.val[1] = vmlaq_s32(v_dst1_x, v_dst0_y, v_step4); + v_dst_index.val[2] = vmlaq_s32(v_dst0_x, v_dst1_y, v_step4); + v_dst_index.val[3] = vmlaq_s32(v_dst1_x, v_dst1_y, v_step4); + + vst2q_f32(coeff_row + (x << 1), v_coeff); + vst4q_s32(map_row + (x << 2), v_dst_index); + + v_x = vaddq_f32(v_x, v_4); + } + + f32 yx = m[2] * y_ + m[4], yy = m[3] * y_ + m[5]; + for (ptrdiff_t x_ = x + j; x < blockWidth; ++x, ++x_) + { + f32 src_x_f = m[0] * x_ + yx; + f32 src_y_f = m[1] * x_ + yy; + + s32 src0_x = (s32)floorf(src_x_f); + s32 src0_y = (s32)floorf(src_y_f); + + coeff_row[(x << 1) + 0] = src_x_f - src0_x; + coeff_row[(x << 1) + 1] = src_y_f - src0_y; + + s32 src1_y = std::max(0, std::min(ssize.height - 1, src0_y + 1)); + src0_y = std::max(0, std::min(ssize.height - 1, src0_y)); + s32 src1_x = std::max(0, std::min(ssize.width - 1, src0_x + 1)); + src0_x = std::max(0, std::min(ssize.width - 1, src0_x)); + + map_row[(x << 2) + 0] = src0_y * srcStride + src0_x; + map_row[(x << 2) + 1] = src0_y * srcStride + src1_x; + map_row[(x << 2) + 2] = src1_y * srcStride + src0_x; + map_row[(x << 2) + 3] = src1_y * srcStride + src1_x; + } + } + + remapLinearReplicate(Size2D(blockWidth, blockHeight), + srcBase, &map[0], &coeffs[0], + getRowPtr(dstBase, dstStride, i) + j, dstStride); + } + } + } + else if (borderMode == BORDER_MODE_CONSTANT) + { + float32x4_t v_zero4 = vdupq_n_f32(0.0f); + int32x4_t v_m1_4 = vdupq_n_s32(-1); + + for (size_t i = 0; i < dsize.height; i += BLOCK_SIZE) + { + size_t blockHeight = std::min(BLOCK_SIZE, dsize.height - i); + for (size_t j = 0; j < dsize.width; j += BLOCK_SIZE) + { + size_t blockWidth = std::min(BLOCK_SIZE, dsize.width - j); + + // compute table + for (size_t y = 0; y < blockHeight; ++y) + { + s32 * map_row = getRowPtr(map, blockWidth * sizeof(s32) * 4, y); + f32 * coeff_row = getRowPtr(coeffs, blockWidth * sizeof(f32) * 2, y); + + size_t x = 0, y_ = y + i; + f32 indeces[4] = { j + 0.0f, j + 1.0f, j + 2.0f, j + 3.0f }; + float32x4_t v_x = vld1q_f32(indeces), v_y = vdupq_n_f32(y_), v_4 = vdupq_n_f32(4.0f); + float32x4_t v_yx = vmlaq_f32(v_m4, v_m2, v_y), v_yy = vmlaq_f32(v_m5, v_m3, v_y); + + for ( ; x + 4 <= blockWidth; x += 4) + { + float32x4_t v_src_xf = vmlaq_f32(v_yx, v_m0, v_x); + float32x4_t v_src_yf = vmlaq_f32(v_yy, v_m1, v_x); + + int32x4_t v_src_x0 = vcvtq_s32_f32(v_src_xf); + int32x4_t v_src_y0 = vcvtq_s32_f32(v_src_yf); + + float32x4x2_t v_coeff; + v_coeff.val[0] = vsubq_f32(v_src_xf, vcvtq_f32_s32(v_src_x0)); + v_coeff.val[1] = vsubq_f32(v_src_yf, vcvtq_f32_s32(v_src_y0)); + uint32x4_t v_maskx = vcltq_f32(v_coeff.val[0], v_zero4f); + uint32x4_t v_masky = vcltq_f32(v_coeff.val[1], v_zero4f); + v_coeff.val[0] = vbslq_f32(v_maskx, vaddq_f32(v_one4f, v_coeff.val[0]), v_coeff.val[0]); + v_coeff.val[1] = vbslq_f32(v_masky, vaddq_f32(v_one4f, v_coeff.val[1]), v_coeff.val[1]); + v_src_x0 = vbslq_s32(v_maskx, vsubq_s32(v_src_x0, v_1), v_src_x0); + v_src_y0 = vbslq_s32(v_masky, vsubq_s32(v_src_y0, v_1), v_src_y0); + + int32x4_t v_src_x1 = vaddq_s32(v_src_x0, v_1); + int32x4_t v_src_y1 = vaddq_s32(v_src_y0, v_1); + + int32x4x4_t v_dst_index; + v_dst_index.val[0] = vmlaq_s32(v_src_x0, v_src_y0, v_step4); + v_dst_index.val[1] = vmlaq_s32(v_src_x1, v_src_y0, v_step4); + v_dst_index.val[2] = vmlaq_s32(v_src_x0, v_src_y1, v_step4); + v_dst_index.val[3] = vmlaq_s32(v_src_x1, v_src_y1, v_step4); + + uint32x4_t v_mask_x0 = vandq_u32(vcgeq_f32(v_src_xf, v_zero4), vcleq_s32(v_src_x0, v_width4)); + uint32x4_t v_mask_x1 = vandq_u32(vcgeq_f32(vaddq_f32(v_src_xf, v_one4f), v_zero4), vcleq_s32(v_src_x1, v_width4)); + uint32x4_t v_mask_y0 = vandq_u32(vcgeq_f32(v_src_yf, v_zero4), vcleq_s32(v_src_y0, v_height4)); + uint32x4_t v_mask_y1 = vandq_u32(vcgeq_f32(vaddq_f32(v_src_yf, v_one4f), v_zero4), vcleq_s32(v_src_y1, v_height4)); + + v_dst_index.val[0] = vbslq_s32(vandq_u32(v_mask_x0, v_mask_y0), v_dst_index.val[0], v_m1_4); + v_dst_index.val[1] = vbslq_s32(vandq_u32(v_mask_x1, v_mask_y0), v_dst_index.val[1], v_m1_4); + v_dst_index.val[2] = vbslq_s32(vandq_u32(v_mask_x0, v_mask_y1), v_dst_index.val[2], v_m1_4); + v_dst_index.val[3] = vbslq_s32(vandq_u32(v_mask_x1, v_mask_y1), v_dst_index.val[3], v_m1_4); + + vst2q_f32(coeff_row + (x << 1), v_coeff); + vst4q_s32(map_row + (x << 2), v_dst_index); + + v_x = vaddq_f32(v_x, v_4); + } + + f32 yx = m[2] * y_ + m[4], yy = m[3] * y_ + m[5]; + for (ptrdiff_t x_ = x + j; x < blockWidth; ++x, ++x_) + { + f32 src_x_f = m[0] * x_ + yx; + f32 src_y_f = m[1] * x_ + yy; + + s32 src0_x = (s32)floorf(src_x_f), src1_x = src0_x + 1; + s32 src0_y = (s32)floorf(src_y_f), src1_y = src0_y + 1; + + coeff_row[(x << 1) + 0] = src_x_f - src0_x; + coeff_row[(x << 1) + 1] = src_y_f - src0_y; + + map_row[(x << 2) + 0] = (src0_x >= 0) && (src0_x < (s32)ssize.width) && + (src0_y >= 0) && (src0_y < (s32)ssize.height) ? src0_y * srcStride + src0_x : -1; + map_row[(x << 2) + 1] = (src1_x >= 0) && (src1_x < (s32)ssize.width) && + (src0_y >= 0) && (src0_y < (s32)ssize.height) ? src0_y * srcStride + src1_x : -1; + map_row[(x << 2) + 2] = (src0_x >= 0) && (src0_x < (s32)ssize.width) && + (src1_y >= 0) && (src1_y < (s32)ssize.height) ? src1_y * srcStride + src0_x : -1; + map_row[(x << 2) + 3] = (src1_x >= 0) && (src1_x < (s32)ssize.width) && + (src1_y >= 0) && (src1_y < (s32)ssize.height) ? src1_y * srcStride + src1_x : -1; + } + } + + remapLinearConst(Size2D(blockWidth, blockHeight), + srcBase, &map[0], &coeffs[0], + getRowPtr(dstBase, dstStride, i) + j, dstStride, borderValue); + } + } + } +#else + (void)ssize; + (void)dsize; + (void)srcBase; + (void)srcStride; + (void)m; + (void)dstBase; + (void)dstStride; + (void)borderMode; + (void)borderValue; +#endif +} + +} // namespace CAROTENE_NS diff --git a/3rdparty/carotene/src/warp_perspective.cpp b/3rdparty/carotene/src/warp_perspective.cpp new file mode 100644 index 0000000000..4437661413 --- /dev/null +++ b/3rdparty/carotene/src/warp_perspective.cpp @@ -0,0 +1,464 @@ +/* + * By downloading, copying, installing or using the software you agree to this license. + * If you do not agree to this license, do not download, install, + * copy or use the software. + * + * + * License Agreement + * For Open Source Computer Vision Library + * (3-clause BSD License) + * + * Copyright (C) 2015, NVIDIA Corporation, all rights reserved. + * Third party copyrights are property of their respective owners. + * + * Redistribution and use in source and binary forms, with or without modification, + * are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * * Neither the names of the copyright holders nor the names of the contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * This software is provided by the copyright holders and contributors "as is" and + * any express or implied warranties, including, but not limited to, the implied + * warranties of merchantability and fitness for a particular purpose are disclaimed. + * In no event shall copyright holders or contributors be liable for any direct, + * indirect, incidental, special, exemplary, or consequential damages + * (including, but not limited to, procurement of substitute goods or services; + * loss of use, data, or profits; or business interruption) however caused + * and on any theory of liability, whether in contract, strict liability, + * or tort (including negligence or otherwise) arising in any way out of + * the use of this software, even if advised of the possibility of such damage. + */ + + + +#include "remap.hpp" + +namespace CAROTENE_NS { + +bool isWarpPerspectiveNearestNeighborSupported(const Size2D &ssize) +{ +#if SIZE_MAX > UINT32_MAX + return !(ssize.width > 0xffffFFFF || ssize.height > 0xffffFFFF) && // Restrict image size since internal index evaluation + // is performed with u32 + isSupportedConfiguration(); +#else + (void)ssize; + return isSupportedConfiguration(); +#endif +} + +bool isWarpPerspectiveLinearSupported(const Size2D &ssize) +{ +#if SIZE_MAX > UINT32_MAX + return !(ssize.width > 0xffffFFFF || ssize.height > 0xffffFFFF) && // Restrict image size since internal index evaluation + // is performed with u32 + isSupportedConfiguration(); +#else + (void)ssize; + return isSupportedConfiguration(); +#endif +} + +void warpPerspectiveNearestNeighbor(const Size2D &ssize, const Size2D &dsize, + const u8 * srcBase, ptrdiff_t srcStride, + const f32 * m, + u8 * dstBase, ptrdiff_t dstStride, + BORDER_MODE borderMode, u8 borderValue) +{ + internal::assertSupportedConfiguration(isWarpPerspectiveNearestNeighborSupported(ssize)); +#ifdef CAROTENE_NEON + using namespace internal; + + s32 _map[BLOCK_SIZE * BLOCK_SIZE + 16]; + s32 * map = alignPtr(_map, 16); + + int32x4_t v_width4 = vdupq_n_s32(ssize.width - 1), v_height4 = vdupq_n_s32(ssize.height - 1); + int32x4_t v_step4 = vdupq_n_s32(srcStride); + float32x4_t v_4 = vdupq_n_f32(4.0f); + + float32x4_t v_m0 = vdupq_n_f32(m[0]); + float32x4_t v_m1 = vdupq_n_f32(m[1]); + float32x4_t v_m2 = vdupq_n_f32(m[2]); + float32x4_t v_m3 = vdupq_n_f32(m[3]); + float32x4_t v_m4 = vdupq_n_f32(m[4]); + float32x4_t v_m5 = vdupq_n_f32(m[5]); + float32x4_t v_m6 = vdupq_n_f32(m[6]); + float32x4_t v_m7 = vdupq_n_f32(m[7]); + float32x4_t v_m8 = vdupq_n_f32(m[8]); + + if (borderMode == BORDER_MODE_REPLICATE) + { + int32x4_t v_zero4 = vdupq_n_s32(0); + + for (size_t i = 0; i < dsize.height; i += BLOCK_SIZE) + { + size_t blockHeight = std::min(BLOCK_SIZE, dsize.height - i); + for (size_t j = 0; j < dsize.width; j += BLOCK_SIZE) + { + size_t blockWidth = std::min(BLOCK_SIZE, dsize.width - j); + + // compute table + for (size_t y = 0; y < blockHeight; ++y) + { + s32 * map_row = getRowPtr(&map[0], blockWidth * sizeof(s32), y); + + size_t x = 0, y_ = y + i; + f32 indeces[4] = { j + 0.0f, j + 1.0f, j + 2.0f, j + 3.0f }; + float32x4_t v_x = vld1q_f32(indeces), v_y = vdupq_n_f32(y_); + float32x4_t v_yx = vmlaq_f32(v_m6, v_m3, v_y), v_yy = vmlaq_f32(v_m7, v_m4, v_y), + v_yw = vmlaq_f32(v_m8, v_m5, v_y); + + for ( ; x + 4 <= blockWidth; x += 4) + { + float32x4_t v_src_xf = vmlaq_f32(v_yx, v_m0, v_x); + float32x4_t v_src_yf = vmlaq_f32(v_yy, v_m1, v_x); + float32x4_t v_wf = vrecpq_f32(vmlaq_f32(v_yw, v_m2, v_x)); + v_src_xf = vmulq_f32(v_wf, v_src_xf); + v_src_yf = vmulq_f32(v_wf, v_src_yf); + + int32x4_t v_src_x = vmaxq_s32(v_zero4, vminq_s32(v_width4, vcvtq_s32_f32(v_src_xf))); + int32x4_t v_src_y = vmaxq_s32(v_zero4, vminq_s32(v_height4, vcvtq_s32_f32(v_src_yf))); + int32x4_t v_src_index = vmlaq_s32(v_src_x, v_src_y, v_step4); + vst1q_s32(map_row + x, v_src_index); + + v_x = vaddq_f32(v_x, v_4); + } + + f32 yx = m[3] * y_ + m[6], yy = m[4] * y_ + m[7], yw = m[5] * y_ + m[8]; + for (ptrdiff_t x_ = x + j; x < blockWidth; ++x, ++x_) + { + f32 w_f = 1.0f / (m[2] * x_ + yw); + f32 src_x_f = (m[0] * x_ + yx) * w_f; + f32 src_y_f = (m[1] * x_ + yy) * w_f; + s32 src_x = floorf(src_x_f), src_y = floorf(src_y_f); + + src_x = std::max(0, std::min(ssize.width - 1, src_x)); + src_y = std::max(0, std::min(ssize.height - 1, src_y)); + map_row[x] = src_y * srcStride + src_x; + } + } + + // make remap + remapNearestNeighborReplicate(Size2D(blockWidth, blockHeight), srcBase, &map[0], + getRowPtr(dstBase, dstStride, i) + j, dstStride); + } + } + } + else if (borderMode == BORDER_MODE_CONSTANT) + { + int32x4_t v_m1_4 = vdupq_n_s32(-1); + float32x4_t v_zero4 = vdupq_n_f32(0.0f); + + for (size_t i = 0; i < dsize.height; i += BLOCK_SIZE) + { + size_t blockHeight = std::min(BLOCK_SIZE, dsize.height - i); + for (size_t j = 0; j < dsize.width; j += BLOCK_SIZE) + { + size_t blockWidth = std::min(BLOCK_SIZE, dsize.width - j); + + // compute table + for (size_t y = 0; y < blockHeight; ++y) + { + s32 * map_row = getRowPtr(&map[0], blockWidth * sizeof(s32), y); + + size_t x = 0, y_ = y + i; + f32 indeces[4] = { j + 0.0f, j + 1.0f, j + 2.0f, j + 3.0f }; + float32x4_t v_x = vld1q_f32(indeces), v_y = vdupq_n_f32(y_); + float32x4_t v_yx = vmlaq_f32(v_m6, v_m3, v_y), v_yy = vmlaq_f32(v_m7, v_m4, v_y), + v_yw = vmlaq_f32(v_m8, v_m5, v_y); + + for ( ; x + 4 <= blockWidth; x += 4) + { + float32x4_t v_src_xf = vmlaq_f32(v_yx, v_m0, v_x); + float32x4_t v_src_yf = vmlaq_f32(v_yy, v_m1, v_x); + float32x4_t v_wf = vrecpq_f32(vmlaq_f32(v_yw, v_m2, v_x)); + v_src_xf = vmulq_f32(v_wf, v_src_xf); + v_src_yf = vmulq_f32(v_wf, v_src_yf); + + int32x4_t v_src_x = vcvtq_s32_f32(v_src_xf); + int32x4_t v_src_y = vcvtq_s32_f32(v_src_yf); + uint32x4_t v_mask = vandq_u32(vandq_u32(vcgeq_f32(v_src_xf, v_zero4), vcleq_s32(v_src_x, v_width4)), + vandq_u32(vcgeq_f32(v_src_yf, v_zero4), vcleq_s32(v_src_y, v_height4))); + int32x4_t v_src_index = vbslq_s32(v_mask, vmlaq_s32(v_src_x, v_src_y, v_step4), v_m1_4); + vst1q_s32(map_row + x, v_src_index); + + v_x = vaddq_f32(v_x, v_4); + } + + f32 yx = m[3] * y_ + m[6], yy = m[4] * y_ + m[7], yw = m[5] * y_ + m[8]; + for (ptrdiff_t x_ = x + j; x < blockWidth; ++x, ++x_) + { + f32 w_f = 1.0f / (m[2] * x_ + yw); + f32 src_x_f = (m[0] * x_ + yx) * w_f; + f32 src_y_f = (m[1] * x_ + yy) * w_f; + s32 src_x = floorf(src_x_f), src_y = floorf(src_y_f); + + map_row[x] = (src_x >= 0) && (src_x < (s32)ssize.width) && + (src_y >= 0) && (src_y < (s32)ssize.height) ? src_y * srcStride + src_x : -1; + } + } + + // make remap + remapNearestNeighborConst(Size2D(blockWidth, blockHeight), srcBase, &map[0], + getRowPtr(dstBase, dstStride, i) + j, dstStride, borderValue); + } + } + } +#else + (void)ssize; + (void)dsize; + (void)srcBase; + (void)srcStride; + (void)m; + (void)dstBase; + (void)dstStride; + (void)borderMode; + (void)borderValue; +#endif +} + +void warpPerspectiveLinear(const Size2D &ssize, const Size2D &dsize, + const u8 * srcBase, ptrdiff_t srcStride, + const f32 * m, + u8 * dstBase, ptrdiff_t dstStride, + BORDER_MODE borderMode, u8 borderValue) +{ + internal::assertSupportedConfiguration(isWarpPerspectiveLinearSupported(ssize)); +#ifdef CAROTENE_NEON + using namespace internal; + + s32 _map[((BLOCK_SIZE * BLOCK_SIZE) << 2) + 16]; + f32 _coeffs[((BLOCK_SIZE * BLOCK_SIZE) << 1) + 16]; + s32 * map = alignPtr(_map, 16); + f32 * coeffs = alignPtr(_coeffs, 16); + + int32x4_t v_width4 = vdupq_n_s32(ssize.width - 1), v_height4 = vdupq_n_s32(ssize.height - 1); + int32x4_t v_step4 = vdupq_n_s32(srcStride), v_1 = vdupq_n_s32(1); + float32x4_t v_zero4f = vdupq_n_f32(0.0f), v_one4f = vdupq_n_f32(1.0f); + + float32x4_t v_4 = vdupq_n_f32(4.0f); + + float32x4_t v_m0 = vdupq_n_f32(m[0]); + float32x4_t v_m1 = vdupq_n_f32(m[1]); + float32x4_t v_m2 = vdupq_n_f32(m[2]); + float32x4_t v_m3 = vdupq_n_f32(m[3]); + float32x4_t v_m4 = vdupq_n_f32(m[4]); + float32x4_t v_m5 = vdupq_n_f32(m[5]); + float32x4_t v_m6 = vdupq_n_f32(m[6]); + float32x4_t v_m7 = vdupq_n_f32(m[7]); + float32x4_t v_m8 = vdupq_n_f32(m[8]); + + if (borderMode == BORDER_MODE_REPLICATE) + { + int32x4_t v_zero4 = vdupq_n_s32(0); + + for (size_t i = 0; i < dsize.height; i += BLOCK_SIZE) + { + size_t blockHeight = std::min(BLOCK_SIZE, dsize.height - i); + for (size_t j = 0; j < dsize.width; j += BLOCK_SIZE) + { + size_t blockWidth = std::min(BLOCK_SIZE, dsize.width - j); + + // compute table + for (size_t y = 0; y < blockHeight; ++y) + { + s32 * map_row = getRowPtr(map, blockWidth * sizeof(s32) * 4, y); + f32 * coeff_row = getRowPtr(coeffs, blockWidth * sizeof(f32) * 2, y); + + size_t x = 0, y_ = y + i; + f32 indeces[4] = { j + 0.0f, j + 1.0f, j + 2.0f, j + 3.0f }; + float32x4_t v_x = vld1q_f32(indeces), v_y = vdupq_n_f32(y_); + float32x4_t v_yx = vmlaq_f32(v_m6, v_m3, v_y), v_yy = vmlaq_f32(v_m7, v_m4, v_y), + v_yw = vmlaq_f32(v_m8, v_m5, v_y); + + for ( ; x + 4 <= blockWidth; x += 4) + { + float32x4_t v_src_xf = vmlaq_f32(v_yx, v_m0, v_x); + float32x4_t v_src_yf = vmlaq_f32(v_yy, v_m1, v_x); + float32x4_t v_wf = vrecpq_f32(vmlaq_f32(v_yw, v_m2, v_x)); + v_src_xf = vmulq_f32(v_wf, v_src_xf); + v_src_yf = vmulq_f32(v_wf, v_src_yf); + + int32x4_t v_src_x = vcvtq_s32_f32(v_src_xf); + int32x4_t v_src_y = vcvtq_s32_f32(v_src_yf); + + float32x4x2_t v_coeff; + v_coeff.val[0] = vsubq_f32(v_src_xf, vcvtq_f32_s32(v_src_x)); + v_coeff.val[1] = vsubq_f32(v_src_yf, vcvtq_f32_s32(v_src_y)); + uint32x4_t v_maskx = vcltq_f32(v_coeff.val[0], v_zero4f); + uint32x4_t v_masky = vcltq_f32(v_coeff.val[1], v_zero4f); + v_coeff.val[0] = vbslq_f32(v_maskx, vaddq_f32(v_one4f, v_coeff.val[0]), v_coeff.val[0]); + v_coeff.val[1] = vbslq_f32(v_masky, vaddq_f32(v_one4f, v_coeff.val[1]), v_coeff.val[1]); + v_src_x = vbslq_s32(v_maskx, vsubq_s32(v_src_x, v_1), v_src_x); + v_src_y = vbslq_s32(v_masky, vsubq_s32(v_src_y, v_1), v_src_y); + + int32x4_t v_dst0_x = vmaxq_s32(v_zero4, vminq_s32(v_width4, v_src_x)); + int32x4_t v_dst0_y = vmaxq_s32(v_zero4, vminq_s32(v_height4, v_src_y)); + int32x4_t v_dst1_x = vmaxq_s32(v_zero4, vminq_s32(v_width4, vaddq_s32(v_1, v_src_x))); + int32x4_t v_dst1_y = vmaxq_s32(v_zero4, vminq_s32(v_height4, vaddq_s32(v_1, v_src_y))); + + int32x4x4_t v_dst_index; + v_dst_index.val[0] = vmlaq_s32(v_dst0_x, v_dst0_y, v_step4); + v_dst_index.val[1] = vmlaq_s32(v_dst1_x, v_dst0_y, v_step4); + v_dst_index.val[2] = vmlaq_s32(v_dst0_x, v_dst1_y, v_step4); + v_dst_index.val[3] = vmlaq_s32(v_dst1_x, v_dst1_y, v_step4); + + vst2q_f32(coeff_row + (x << 1), v_coeff); + vst4q_s32(map_row + (x << 2), v_dst_index); + + v_x = vaddq_f32(v_x, v_4); + } + + f32 yx = m[3] * y_ + m[6], yy = m[4] * y_ + m[7], yw = m[5] * y_ + m[8]; + for (ptrdiff_t x_ = x + j; x < blockWidth; ++x, ++x_) + { + f32 w_f = 1.0f / (m[2] * x_ + yw); + f32 src_x_f = (m[0] * x_ + yx) * w_f; + f32 src_y_f = (m[1] * x_ + yy) * w_f; + + s32 src0_x = (s32)floorf(src_x_f); + s32 src0_y = (s32)floorf(src_y_f); + + coeff_row[(x << 1) + 0] = src_x_f - src0_x; + coeff_row[(x << 1) + 1] = src_y_f - src0_y; + + s32 src1_y = std::max(0, std::min(ssize.height - 1, src0_y + 1)); + src0_y = std::max(0, std::min(ssize.height - 1, src0_y)); + s32 src1_x = std::max(0, std::min(ssize.width - 1, src0_x + 1)); + src0_x = std::max(0, std::min(ssize.width - 1, src0_x)); + + map_row[(x << 2) + 0] = src0_y * srcStride + src0_x; + map_row[(x << 2) + 1] = src0_y * srcStride + src1_x; + map_row[(x << 2) + 2] = src1_y * srcStride + src0_x; + map_row[(x << 2) + 3] = src1_y * srcStride + src1_x; + } + } + + remapLinearReplicate(Size2D(blockWidth, blockHeight), + srcBase, &map[0], &coeffs[0], + getRowPtr(dstBase, dstStride, i) + j, dstStride); + } + } + } + else if (borderMode == BORDER_MODE_CONSTANT) + { + float32x4_t v_zero4 = vdupq_n_f32(0.0f); + int32x4_t v_m1_4 = vdupq_n_s32(-1); + + for (size_t i = 0; i < dsize.height; i += BLOCK_SIZE) + { + size_t blockHeight = std::min(BLOCK_SIZE, dsize.height - i); + for (size_t j = 0; j < dsize.width; j += BLOCK_SIZE) + { + size_t blockWidth = std::min(BLOCK_SIZE, dsize.width - j); + + // compute table + for (size_t y = 0; y < blockHeight; ++y) + { + s32 * map_row = getRowPtr(map, blockWidth * sizeof(s32) * 4, y); + f32 * coeff_row = getRowPtr(coeffs, blockWidth * sizeof(f32) * 2, y); + + size_t x = 0, y_ = y + i; + f32 indeces[4] = { j + 0.0f, j + 1.0f, j + 2.0f, j + 3.0f }; + float32x4_t v_x = vld1q_f32(indeces), v_y = vdupq_n_f32(y_); + float32x4_t v_yx = vmlaq_f32(v_m6, v_m3, v_y), v_yy = vmlaq_f32(v_m7, v_m4, v_y), + v_yw = vmlaq_f32(v_m8, v_m5, v_y); + + for ( ; x + 4 <= blockWidth; x += 4) + { + float32x4_t v_src_xf = vmlaq_f32(v_yx, v_m0, v_x); + float32x4_t v_src_yf = vmlaq_f32(v_yy, v_m1, v_x); + float32x4_t v_wf = vrecpq_f32(vmlaq_f32(v_yw, v_m2, v_x)); + v_src_xf = vmulq_f32(v_wf, v_src_xf); + v_src_yf = vmulq_f32(v_wf, v_src_yf); + + int32x4_t v_src_x0 = vcvtq_s32_f32(v_src_xf); + int32x4_t v_src_y0 = vcvtq_s32_f32(v_src_yf); + + float32x4x2_t v_coeff; + v_coeff.val[0] = vsubq_f32(v_src_xf, vcvtq_f32_s32(v_src_x0)); + v_coeff.val[1] = vsubq_f32(v_src_yf, vcvtq_f32_s32(v_src_y0)); + uint32x4_t v_maskx = vcltq_f32(v_coeff.val[0], v_zero4f); + uint32x4_t v_masky = vcltq_f32(v_coeff.val[1], v_zero4f); + v_coeff.val[0] = vbslq_f32(v_maskx, vaddq_f32(v_one4f, v_coeff.val[0]), v_coeff.val[0]); + v_coeff.val[1] = vbslq_f32(v_masky, vaddq_f32(v_one4f, v_coeff.val[1]), v_coeff.val[1]); + v_src_x0 = vbslq_s32(v_maskx, vsubq_s32(v_src_x0, v_1), v_src_x0); + v_src_y0 = vbslq_s32(v_masky, vsubq_s32(v_src_y0, v_1), v_src_y0); + + int32x4_t v_src_x1 = vaddq_s32(v_src_x0, v_1); + int32x4_t v_src_y1 = vaddq_s32(v_src_y0, v_1); + + int32x4x4_t v_dst_index; + v_dst_index.val[0] = vmlaq_s32(v_src_x0, v_src_y0, v_step4); + v_dst_index.val[1] = vmlaq_s32(v_src_x1, v_src_y0, v_step4); + v_dst_index.val[2] = vmlaq_s32(v_src_x0, v_src_y1, v_step4); + v_dst_index.val[3] = vmlaq_s32(v_src_x1, v_src_y1, v_step4); + + uint32x4_t v_mask_x0 = vandq_u32(vcgeq_f32(v_src_xf, v_zero4), vcleq_s32(v_src_x0, v_width4)); + uint32x4_t v_mask_x1 = vandq_u32(vcgeq_f32(vaddq_f32(v_src_xf, v_one4f), v_zero4), vcleq_s32(v_src_x1, v_width4)); + uint32x4_t v_mask_y0 = vandq_u32(vcgeq_f32(v_src_yf, v_zero4), vcleq_s32(v_src_y0, v_height4)); + uint32x4_t v_mask_y1 = vandq_u32(vcgeq_f32(vaddq_f32(v_src_yf, v_one4f), v_zero4), vcleq_s32(v_src_y1, v_height4)); + + v_dst_index.val[0] = vbslq_s32(vandq_u32(v_mask_x0, v_mask_y0), v_dst_index.val[0], v_m1_4); + v_dst_index.val[1] = vbslq_s32(vandq_u32(v_mask_x1, v_mask_y0), v_dst_index.val[1], v_m1_4); + v_dst_index.val[2] = vbslq_s32(vandq_u32(v_mask_x0, v_mask_y1), v_dst_index.val[2], v_m1_4); + v_dst_index.val[3] = vbslq_s32(vandq_u32(v_mask_x1, v_mask_y1), v_dst_index.val[3], v_m1_4); + + vst2q_f32(coeff_row + (x << 1), v_coeff); + vst4q_s32(map_row + (x << 2), v_dst_index); + + v_x = vaddq_f32(v_x, v_4); + } + + f32 yx = m[3] * y_ + m[6], yy = m[4] * y_ + m[7], yw = m[5] * y_ + m[8]; + for (ptrdiff_t x_ = x + j; x < blockWidth; ++x, ++x_) + { + f32 w_f = 1.0f / (m[2] * x_ + yw); + f32 src_x_f = (m[0] * x_ + yx) * w_f; + f32 src_y_f = (m[1] * x_ + yy) * w_f; + + s32 src0_x = (s32)floorf(src_x_f), src1_x = src0_x + 1; + s32 src0_y = (s32)floorf(src_y_f), src1_y = src0_y + 1; + + coeff_row[(x << 1) + 0] = src_x_f - src0_x; + coeff_row[(x << 1) + 1] = src_y_f - src0_y; + + map_row[(x << 2) + 0] = (src0_x >= 0) && (src0_x < (s32)ssize.width) && + (src0_y >= 0) && (src0_y < (s32)ssize.height) ? src0_y * srcStride + src0_x : -1; + map_row[(x << 2) + 1] = (src1_x >= 0) && (src1_x < (s32)ssize.width) && + (src0_y >= 0) && (src0_y < (s32)ssize.height) ? src0_y * srcStride + src1_x : -1; + map_row[(x << 2) + 2] = (src0_x >= 0) && (src0_x < (s32)ssize.width) && + (src1_y >= 0) && (src1_y < (s32)ssize.height) ? src1_y * srcStride + src0_x : -1; + map_row[(x << 2) + 3] = (src1_x >= 0) && (src1_x < (s32)ssize.width) && + (src1_y >= 0) && (src1_y < (s32)ssize.height) ? src1_y * srcStride + src1_x : -1; + } + } + + remapLinearConst(Size2D(blockWidth, blockHeight), + srcBase, &map[0], &coeffs[0], + getRowPtr(dstBase, dstStride, i) + j, dstStride, borderValue); + } + } + } +#else + (void)ssize; + (void)dsize; + (void)srcBase; + (void)srcStride; + (void)m; + (void)dstBase; + (void)dstStride; + (void)borderMode; + (void)borderValue; +#endif +} + +} // namespace CAROTENE_NS From a2ec23c11225f8f2f11b2f4c4fef05294f124f34 Mon Sep 17 00:00:00 2001 From: Vladislav Sovrasov Date: Tue, 5 Jul 2016 17:28:53 +0300 Subject: [PATCH 053/128] Update cv::log documentation --- modules/core/include/opencv2/core.hpp | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/modules/core/include/opencv2/core.hpp b/modules/core/include/opencv2/core.hpp index 906c4d2aa8..d875ce2d76 100644 --- a/modules/core/include/opencv2/core.hpp +++ b/modules/core/include/opencv2/core.hpp @@ -1433,14 +1433,11 @@ CV_EXPORTS_W void exp(InputArray src, OutputArray dst); /** @brief Calculates the natural logarithm of every array element. -The function log calculates the natural logarithm of the absolute value -of every element of the input array: -\f[\texttt{dst} (I) = \fork{\log |\texttt{src}(I)|}{if \(\texttt{src}(I) \ne 0\) }{\texttt{C}}{otherwise}\f] +The function log calculates the natural logarithm of every element of the input array: +\f[\texttt{dst} (I) = \log (\texttt{src}(I)) \f] + +Output on zero, negative and special (NaN, Inf) values is undefined. -where C is a large negative number (about -700 in the current -implementation). The maximum relative error is about 7e-6 for -single-precision input and less than 1e-10 for double-precision input. -Special values (NaN, Inf) are not handled. @param src input array. @param dst output array of the same size and type as src . @sa exp, cartToPolar, polarToCart, phase, pow, sqrt, magnitude From a6ade2b91418cfb2ea7c2e0b729243cb6ec3c8ac Mon Sep 17 00:00:00 2001 From: Arthur Cinader Date: Tue, 5 Jul 2016 12:58:47 -0400 Subject: [PATCH 054/128] Add the mask to the template matching demo documentation. --- .../images/Template_Matching_Mask_Example.jpg | Bin 0 -> 79095 bytes .../template_matching.markdown | 45 +++++++++++++++--- 2 files changed, 38 insertions(+), 7 deletions(-) create mode 100644 doc/tutorials/imgproc/histograms/template_matching/images/Template_Matching_Mask_Example.jpg diff --git a/doc/tutorials/imgproc/histograms/template_matching/images/Template_Matching_Mask_Example.jpg b/doc/tutorials/imgproc/histograms/template_matching/images/Template_Matching_Mask_Example.jpg new file mode 100644 index 0000000000000000000000000000000000000000..711faec3b153f378e367e1a2ea252c41e5173356 GIT binary patch literal 79095 zcmb@t1ytKzwq!NS48!NKaz%RdDd4+k3`3xJ{U-hC_phKv7Pi}~U&GyhaR1Yl!G#m2?PC&0nn zuzj$wama81c#p^h<#q5WtRH)YQwk*)5Kz@BuzuH_+z_U=`NQTNamubJqL=ce&erFL z=yT50!g?_peLLTtsZBzTw?y}7>E5L&DHk;uP8)F1!?zHvhn9{*Oo^V;&euR3URdA5#XY@+c$Zex z@PCr5Z1}&4{(sW^|3UKI5`Y*7Gks*(WB?H0EHNMQwByKr7xHw`tJnk~ySf)GQcLo37>g6wKK5U!yoTHAg6+^YVW3_6 z+3z6u9l&e88Udr7hjmw32^|vH?15$q&}rbJel^q&1S+4h%XT)4hkNMa*ZfIHRcK)* z5{}HEI1ClIP6JIP&V(bXVV?PiBJ;skLi>{g20z-?&G|_rXD!?9&OQ~U4VWj6H1D<3 zs0VIFUh7=vOZ6j9kRzgW8?wO&?l=hmI-~P5Ga#sSp0WQOCFHU5o9?#1?NSAi8_524 zpcFDOtXOTmMpTfCeD$0ALu$OJ#YK3e3!Qp-l*a>CqTn z^b;h02N2Qvi58d-p!_^jm1sq}W`|$RiAWqNJCtF+1Nf1i)!x3m0}L&qW|XfwHmdz9 zL24Gkke?qPRQzpK4d;lL>F}* zTVonQfuq%<%w~bz=6mW)>ihYilc)U#f179rld7PL|8}Mc%<0Ys=%LK2Qk(7oMIPPz zC4#edXfI?M zz`XAO&rnlRH2)A(c+Q`R2tcR}b$~tQ0r#+gMOBfScYrm!ZqdJV>!6~GfZPA@&x&kE zmP-b9TSA|77ge?0o4;L5rT?ct&_FimU-Rb*_CUKZWscFG%gc!JRyFS39)q$6QCkQt z_>;t2b>N=LzWTYqn80X*vLJGLfrry_ued#-jV0>Unv`A)G1 zi4X1o%r3i3skdGKtga%|Pt*Xf9?as^k=A9~X7{Od_z`hOOX5g9T4l=W zQu_{|4$@sm8h3yE-i(l~37)W8fs6)P?bbtNt%Md)iA8q+-Mza;B-N8W;9s&*- zVFs~gZNf~(iZu&ymkZ^GxCMma+t~Nf-bOm8smP%kWa?m^?d&0qCODvG(b8OQoQg|L zu%%M+B+DXwsTJRnF@=q+F=fl&A@zlQ57%=P&aOKMQH`+N1gLxp`PN&yvY+X<927IW zF1{#iN05c=WDEvs{9S>C=d28fNW?3wJAf4Qqb#K8_$IGiBS|uP)(e=!bs?L^@a&)I zwnp|Kzv2<(HeOuba{Ao?41~HzwogD}-xsbG{_ZZF>&%~+6;WlNj~ut;w@BRMt=Cz; zT@2fYVy4?375hI2E5!nW_D&rKrt6hJtC1C!v&XqlndTk$KW zr9y+JqEss8tBP2?&xl*1_K_-Xgz+z~08w3vE6kNCehEJ`A`SZTI=AT99PF?#o=cdj5olk{%$6i|D=PucD-`qq=TyqNo9Hyg zCgqkhPB2}ty=@(9Qkme6vyoqY#<}sUy$=sdwx?J&FuvU`O23A6PGIJYwfAB`Ey1_nQjEt~IzPn(N96 zSy&#> zhJ&Y0PT;i8r6fm`sePwuteXo7J#NpHSXRqwzffDNACkWI+Aer7@+^krH`hz~g$)OZ z=8-%e$+x=;k&HTI-)qt3ti`rMt^fwI^9NV{X zSe141zp1a7bU}sqg9}CvMCg+?XIDhXBMxp5uNQ!1i*Fy5I~uGq?{CJ3%`@q53B19KjH3nxdZIt&p=U| znLGR0N47q#%V{kkkh>=cJ#&<mcLmFZ=U0VX}c+p>JIRPlM1Tq!rf$S{lpHK z8YirnlmvQM<{C&`PLm|!qMeuCC%u=9UrE6i_1z$V#jiRtv1?I6Fid)8DCKR`XAluk zQ?9?zPGN#jbgAh4D49J8@cCrj{J?S@Xr1J6-t$zw?vwd#adnXNjBmd?w_E}ieQHTYTm%m$k? zg#sTQ4)jfki<5bf;IB&U)PByx)_8_WK{!qOMZ$NMb*Hv78ysvK@*4_w=S7!R1lcsZ zfdqQXlCsad60CF4V6+(&#+h7_@?AghttS4KRuUImUHn$H>C!tMDgERf1>4#eWhHh)NWXtgzd~l1h{f=m zUY$&!ZxLfR3I^GbIZ*J?ZK_pzFQ@?6BHcZ@=KNt@J7JNQ$exAAFtW$3uP16aC`jf> zUZH)mGnuOI?76s4`$PYs?KgHC4u$+FClcKBdg-E9Fa0luUk|}#j+K*qLRv}s`7IAz zbYC9GH~LFWO9_&{qBSPxhOgio;D6R_kOfT(7TY?6T&g&)$jAk;p3CCwEQgf5YU<0P zs$hAp?W|dJ#n_v^><)-GVJ-dTF9js2lX@?BZ~rumOBTS>dvyEJJxwL3voihmoTIMU zKMZd5m87s7VC=)@OW&3?3jWlnYh<^g#3hi-PI2U z*9;NTesc{ufp)330fc891mHv_sT+IcfN@B$1Cxil#O=z!P1g(HC?r|3D&J!%GWpm{ zH(cIvs*39gcH-smA>$n&#VJIHLojLCvEc9y;QaN^F4gU$YnA=LuQWG}9lyRyyG;&L z4(UF(->>?Lh2Qn;1tFz>P+TCZfVP-)u7i3Ao4XuuqyIWr8e>!%-p)6+N*?@~l5YXm z-UE0U8(eAz)wWo_<*u4W@)WKcJYLEus#~40^O|Y{+eLly4!z%PE^UTk`@47L#ufH5 zI?EfHyi+dHX8L{y;OP`5m#-6491gMoleJKate5CNUV5B4Dg_t$t>Ur9Vqci@2B)C4 zr74fM-GSHZJ|aHyux5oUES55Qt41YwA2Nh7>CFO%hifb)u(wIoj~^VcEOwWUzJ5=E zjl2W6Iwj;kS!t8ixC7*lPKlfhjJ^!nh4wNF2zQi0m)xiX^nH@|Pz{d~yL>y}61mv_ zTA%K`11PQd2RJ7JvV0QOWwqWNKRIWD=;Z6qK4Eq}$Z%#$KbZdZhScrMcq@VHPp|gt zT?xDNL5hP=)2EMFlTEoq@VZQxhLYa4DwB)EX{b66sw}FXQ65Aur&#mAZfh`vP~$?z zIKh_q2j>jscYrCsF;{rLP6(n&h$>-Zo$1`%;t=IHzL4{}cQGzvxPDneY2K zfq**C5!}sv|Dd_JjYyT@vW#w}7ko%SZPaJN=k?BXCi8Q7@{uWVi4GCD%)qiT`<`wW znr(+X*3B;eJFl0a;ZoYpdPrZAC;ksx#+>&o4_HFcoIEVTQ4M}=A;K=?%ItO1vz#sQ z%P~Xu%fCu78zeVf1?yi7MT-Gonjq`rIRo0*vcdztK|`R5~9v>GUH*xed@34Y&%nfeIaXY=+osW zvY1e;=-&hhKnfQxX)*c2ec{N>WA%4fF`xD(K3xF&={zI;`|!DC}bm9nLOeppt+ z&WZSc?hk}Vm*>xTm-dIQLYd(5%D=1>aRM@rPGGA#$BHhlw*RhNq^tCut?z^kPI| z`B7am-tW%ng`0VHsx)VVMIC@Ec6*T2hie!Yn7l2}fzWvETYEzS zsEzEcdaIP3W_I8Xtu2jge!{NFQqZG(I^6kupG{KafaiEM1_c>YLY@WYTj5VD*Bdp# z(NS?Gd`E*7!ekf4;HsVlU%KD7@rw^9Y^!{vhAehJUlL^eio$-n#CjyNds_2KC#^0a zCCo)Kewf~uax|PnX`uYRg(`0VKe%SI_UCluhZj$iOJI%JQ(PI8uGLT1+xnr4eMRl{ zZuD>0OXOnVIN7Jk!&%@J3>kDt9}9RUY73Zz>V#?2m^akrT9oI>iDae@;HgB^jG7D+ zT^F2Xj<-7qC6wEkpPUgiZ0@iATkcpp736M43?n=$5xEt+CqjY#-@B)ZXHTXSNEdZ{ zSU{TI#=adv(F1UDQ1HoZQi#k0W&|My8zH5Ei>MJ~xhuK;bh-~_751y^upjv$qg5dy zt^+OR)#jCN@=9euitb*2yaVK$WNKtaU#aIU`scOuq!xx$-%Ffs&UIPuSOL&3i$vHUn9K`yyezHnwH-wP#_<>R2@Ei~F4Xo_MvNBYufD$b_q9R#%~w*DbF9Wh~A z{)O#G4}UBhHhp*nc~O+{iv2p{%jLZ`zpNdnIN~u$;g~SjUsxi8TR(%-^OP&CW4N?_ z$ayIk9f{p`5OmoC#UI!MUHMXkcll!<~YKX4DfIR|Rkvuru^a+~kH)tt3$t71jNIdNc7W%lVh@Kw6LmigaAbKc! zjHJ`(oH%$r&C}d{eb%i%^*MM|?ag&KZIBMX^UYp>zRXK;&kW6nLNsOpXSY>QHe7I_ z9Q;IfKbkTf2d#^^E^;&fm67mAo1J8hN3y3Npew%ji6Q^C7(0ZiwTg*;|v0S14D06=D zAWM|4850ga$MGDimJQSz0i{`bx1%}`&ZIJ>+TCH)gBN*J>hB zfSM)BS5N1*df(-DF)VUgN6nEh3pPjFtn8r!yWz0HM_Uxh%;%H@Wn+m(1syscA8;@k zjPzzs3<{Y)&rC+C{-V$VM@qOQecAnHIE>#Hc4UcWUf`CV6nou}LMB(vV{$M-G6+(} zscB?&9E2c}t+F$jL*^eu?jXbA9`TQ2#2$dAJe0>d@6D8-5$uhf)U-=|i;NXK$XtQ1 zR#Z^rM?C2HB;B{P$dZ^_B+{@w^P7kJsC$2OQoO#Z+&ISgHb{Pua%xS|^AS#2|MjCp z8ofdB36az?bN<;$hev}>Gx{MFmW@XlXI1g!ven{~>uFYL7uQOYs6NCC?k6p*^3bL9 zEB7R~V#9p5J3wP@zstd3hqjcTKfN0ZgTRdEAW=~ZssfIh=2oL8y3f52f^cZmZyDPF z#V^9v>Nz_?zvseiq5plHekM>1~RnSu`Qo~>2yVO0$mOE)) zm12Ir=S6`UxcwfFe2Ia&oBUo87K2JhEQ~wsmp0PX&Z#3%Zra&%VfqPc5`(Xgd-O)x z5x+@`HRcm3OfRupJ;)A7t}UxGSRJ*Vu1=t&54zUD{=UkBE1N@cgLY&<9T)#sUm{ye zeq<$>eIfI&qMIRW1hY;YO& z^gyrDn_qEwcj4x-?hQO&xeCsVQcaR%n|zi+l>|N-WKYSVuxjAWVoNfuXr&{Mg`O>e zwDs&)_*GWlk&C;0tjyv_FX4-d#sPe740s%dwIJ|HT2m;~h}yZcCEZPH5H3tU zzIvGOu6BQII~&z!Y!8Mc_OBu8qqAClP+*XY*!?nr)c37Uzc@bQ{8ZxMwmchtvSd#W zi;EC=5y3F+^K#P3Bu#%NJ4t0z@UoH8Tx&1sc%&C)S=fZ8Jcx8i5m-qE+o1K4q=;(8 zmz-l6mc@sHYRp+x-yqYzm!U{yh;!h*bScJM4(kE;*?RnHjy!YYoj<=rPsU#c523A5 zUr`e?kSD0AhOQ}(mgUu$kH?kUSQ~l5jf#(O6Z#4T25<{{fm(;1ZhlYWw&qWcL+|fo zQJwX&hQ60Uqz@Y&d;55fq|v|lk>62(-3frY$P+>eB;NcReA^Dx&u6Q3mGry8|T{XWsA|=mpjR|a(33_)|-Bn zAI-qHI5kYxBFM{a(MBJS0tc4Itt2p2!nzU3-Pr^KYlc=lzjgm0G+P-6{6JKl1&iw* z@7tbu22<2S1T$l-o`a`X4y05}pKcYLGi#+bWahi6a=ErqftvipHD#`rl zS5+TLDHP`RGKE1Gkymzd1+Lr@MTU>}wQZWIR3FP=ABr96rAF<|eo({dR+;EdhGcm2 z#~|lN$bt-K?f{C7UUD;gkAM}T7G;&cZdW9lUiAcLP6qKutfqCfX$pSEUgKCQ0xtqf z9a&5BsXrm@}QWA)S`1j_(+g?}b; zIW}_w*m`@wm%T7jF`$=x%FK-ObdY*YT*c(YK`oR&T4eoXU`!#EHYwoc^MFfdPkWoj z_KplGCqUWP)MwA}^L9&Y`&!T3a^I;`w0?q&$(t-YF}qHpKE2XjKnAGCYQ3j!gmqI+ z<-_w7G}okHKzr3~ih;nCU;ggHgt^^OsaID|gN zzS4{u_YSVY2Zd)5YDWu4$x^{fZ0^q@&_v0VmD(Q_aD-bo$U(8dCE5G)eIJKA&>$cu zV}EFxeb!5Kz*#cNnMtpEhVG>p5 zh2IGyp#RXK{u(Qg&#OiBVc+bQ77RpoF{mZB@-*{q0y8rR?+UlVtC zN#*koovVGj))y*2)50~2wz|xK-RHs63}k((RgOtxk#CBU0AYF;tPHxJ#*lU|Js4kn zTGi94Dl+Gk@@C;UXFDsFOp&sm+5F?H+~dc$ttL+nV#+yc1f>UuO6`QjI6XT^Y003q zhw+T+GGzW7UR9A(@rqd5VIX=(Lvr7cWkrodzJ}`Z&oFmCYv%?Ckps!3Kn@Q|0%1+z zZgmHF8ZAB*e6BZ(zGc7JE5)se!(^m+Gd643ng1N$X3xK&LrHjNu1~|hcVuyOOn%N2 z9;7oUCv{+;-xVxfSl&-vm{bbP{WEzi5&t8FcPoM1O)z@pCjVYw8&9t*AY?F-5KO0R*2_S zoyc_GXc4e(gs$%qvv5nwwZWBLCMxf3jf{`h68(U^Xxu%iip+YL%f2wna9q>~r4%Bug@e*-+&?2W_R$Zu9;*mk)U~D5YG%(|pbaNsk=7t=<9| zna2~jA0E|6JHPC}_geH$w##83<#>;aHY_#}vs?_D0osDqn61*=us|nSkZn}fj6dX3 zFl4$J)VF(biiL<*5xx6nd zgi@l#4h~(E5j_7e%GV4X=)U90!0reJZJcy5w(mf zl4QfL5r%PZ*GtpTBuM4dx4>-2$@wJah;% zhjgESR_zmMzYrYZRsHB3*PC$gJ#NprQVFf2g~jaCWSAK6DN%8oK9oviy2iT94LiG8 zpgqNP(_MG*WX>!!`!ddi8$dqO@5bPeqC^R}dH^LQ++|fftxuGAO%3^My4R&%pTQ&* zm)SrXTCSmBs3<>LDz40F$3&{0`*mP^K=SLa?KLZbejkCU7xRbnPv|9D-U_MRZinSJ z^#WH5egm!&A;g(Uu>E>0tnMdyR*51P_djxE+0^T>|G@e|^3GgEXMy6ErpAZM*PP^7 zoKCk*6&&R-<>rPhZo@4Bq0ncfD3g<4g&|3xM6v4Oob?S@d~J&vwHm!;fFH0sDRJ!) z1Fj6EKMn!Ii_=qPGq9}l42|LCtV=irdDVzh`4-faN_1GBASW_~JYiM;Nn=H!qH|4nulRwLX6}SU zk8z%1i6`>qGjln|a!2n($1H$Z`G`FF2GbMNPC&NliVwX(LMC*0W536(s%Q6R$Wdnz z(EoyHU!(nSUhz_NAg?a6ckK#xIVk@VU?N3cpWXsBDDcHcS^tu^>bHu9#l%WAoM z%Lfxc7^~->`M&_h|MEvn0AYd)1rI{|Ag{C`LuwTOEsJXQQs9r)qEVpEWfx+hq~Xoq zh?-zLzSgCH$D7$frVFUsx0(Jai#fV^&S|iLwi~Mh!7GCEF{?+dlD9@P?@`~%CW;K8 ziz3zTb)zz;_@g9VGy{Zn;Ji+v2Kxp2H%3y{wzMWjRM5!`vJWyBq#FnfAVI@&k7sT~ z=Nq3wgxV=+IVstWK=%TK{F&nfMUBm!OO)DAOE-H@&`-sC@x4v~E#%JmceMDf!S~ud zXA{Uyl5WHm!FEXJs&lofhz1wB6<@~iQvAZtA_YTIAibW4Oij6`{iCqcr)sNA&!@u& zZ9q6wzr{dx!A4acgSED)l2vXAKj=J;mwtSgaM6BVT%6<`v#7Z<8kFzR`Vh}VHsfn5 z+3t9uaHA{flf6Mo4<$^-O(#WjKU@7%OwLaD#Wu?X*P=?3+{*ax*Frw69y&{Y9o=o6 zU7i{f>>JIj$q|JPGDT_Otw$5DqLOHd`9 zGaI&;nV+h4LecU-5Ri^E8V$aMV=Gm2u%0|gKnp%140Q^-&-dK3+p<&sWSN8HX&wWm z9!+~)x6|7{Hjzou0r6bix_t%lfP>Wb*H`SgKRCh(DqcrrJ_O`t-Qc<2@c6IS4i}nV zx0tX#QB2ltn}t2~4#RQ)(}@En=i5j1p{prU_iWo_f8%|TcQyP|)&=KxO`Q}=&PO#T z<|g)GsdTj2tRBALKDkLVR*`wngRw8k%aynqns37vuzm>%Cfx3J_6p00-Afdi=IL`b z0BDD%BZ=E$EpTZ*iVj=x@0A>HzvXQvv^^f(g5$7q8;&VwzVOkE@Y?Yd++)tlhHdD$ ze-;-xcrDOFNHwTWwbbHTdcEVJ#qab+OG|hq%)52gc0=eqkeu} zvodvW{yb*L!rOi6(z?7685#5R80)u4%R5$>n5tGW|1I5o)D;fPi z5@x@vk$u&yd`)%8W#dM{P>Urg+Fbg2lGqqnVdJ?pz&23`wr3mMl|0RFD&m?YcW$@7 zO)8(;*wATO>P4t}xMz;UXC zU|zigFywci;rnHI6#VxRO#Zh7xtnA~o$0QBk_}<;kRGjoGsN5hN=DTkG6iNrPuefU zu3uQqB(Bh&q<{9Gt=vvZeDRFsCL#S=_`bo@#};#hW_#=Wvklu-H#oUP3CH-=2lB|Z zr%NI|s1IdF6Fb>HJ|t~I0U=@4vHrWhN zUty^ycT+UM+VdKqLH4@DJxkrYWJhb4R*2OZt(yhk3I@Le?YfovY^;?(bfrgdWx|`` zqNBiYzJN9Gx8aVRq8FJ$gaf{6@&@;4lT-zU@*$=WCe0j{L~+}~(OzcO^4}CcVe6&E zN{8=@T(d}zHxt?ib>`i}ZaRF5r|rr6w_)4MGeZTgBs4G0g0)Y}LDb1n&Q%--3b!Oi zL1zZTZ(fYH2;~!3mrK2@%~oTb5%qm+uQ1m%K$JkZR8rP7rs8vU^=k0Z*@psbamLBH z=K6JWB7+t)h4-TyKq7<6P?n&)bD3-Fhb3A4UtEcsYL`E0>AyMRd*s4d{YUzJx6A80 zfRi}fu8>akPq_UT9YF^0_<~B%3)+w=P3xgEjaZZLi~Q=`7VO> zJRnih^*vL0qPO?h-oupJZ!rqpM*8InIUWV^8b;r;gB?V$IKJ=;fA|Gg0w=|cO=Ldi z<@+Ot*5{50QTt_1xcT}a@)h+ zw#!dHc;%7ne9c@D6w0qIVI8rbcrC;ZGUkD}IIbu(-T~}X+{#T3oK7@3)Si2;&Y-L8 z(-(i2K~4Kfdf9DO`b(B0$W&!45{_M0qer~#PFimmiCg)O5Yl%58)@@Z^MZKm>ju$+ z_@G-^WD1tlU{3+F5}CVQL8tP~^PObZwAA}%uSP5ts6`mF66Nq*)D8hXgQckh(&7jP zgt%heY>vUKfK9y|u_>Q!qQR6%rK6cwM~cL%ie|b@T>^!$Vt!`8@1Rc);^IPB0`vP* z@ll6}a1EWgk8pk#Sr_G1o48ue}1C0{DKO^ zN4d!+@EukcS#apv)0HcZSzPLTLp4Bx9Du;zqJMFDZ@J2EirPFL%~v=8*2^EU^q6j7 zuv)84iGQ3JX?rShQZwY3^6ZxbnB)(>9pENXb@E8$%H7wD^1iKIxbgJSvwf}cXSs_b z(OEEyqE5l0YZI<#;oYYrQeG`iWtrv2E?Sdkzt~k*ovr&KO91@TH{x*l*1ysu;m`pt!Gh3bFcHjaHs0BL^RW? zSjB+?aw$^zqwkezA?Ld=MgPeC5Es0V;Wx_POF*i)mt%+b`Y3Oel9gR@PKG@!DP`*E z-6xi_FQ9$KF*piw)lKAZlZ*j06e;W}gv}u+53E<>C9(iV1)4x&z>FxCYzc z{=)SYkXf!_e46jubywx;OGn;_c_X{&-~M$Zx~K;*oB8$L@lGnnoz?C_xF)GjV-<{ zjX779m4#g`2_o-3fDw-g09*iVJ#ku*KMxrmXeC@UySfyX53r`l5z$izFLa}-Afs!K zKz-SlfbT!h$*s^w6Vo#XnQxvRKWInV8lfvdXZ`1mUdUl&b08mu1tyC!&&*-RW6DMR ziLEEhQkNbWlFPz+7Go1^r0Xxij^GMjAGI18RqnPzSk`+D%w!2bZFSa)4^k4LDS0z9 zd4ZTF&8S1vuZ-`$tsw{1UD+{I{AN0>f_Ro;k5Y0%Y8f=5d`?~kD@ss0@?54jS>@gV za+W*=t#>WJt9DS)MBO$6Y4F$&b@#Y_viecbkYr3Xxxopq$}T5tLXGtfKzFc~22D%G zu=1<7Su*{c`=S}S{cb?qO|$)!X)8XVjeSWKWQ))3ywR ziK%q1J{q}$)k5v)4;pB13QRthQT83+SQz6A8F(w=5^#MWFi5%skRiKmA-hcH7Aoeb zcc_**zu2tm#!rgF^nO2YZ%5+UrpMU~oZ0=^vvSJVW?+R}W%MTjS1*6CU2=jUs5!XN z4)|jxvqNj(IXCTB$(K>&EA{7v_uj{0&9nmf%GvT1(7X0Uhp6mm{lI|yzLQU=={tag z8Zvgb2b3wDSX>fsEp*BYyf)Yl%MS6Bd%KOV1>ZgqZhpNGqA~3pOiB&Zzf~E}93_J=6$)Y`lm_;)#iJ_ zQcB1-sRC8zsE$u!IT}3ytx_2IIe4^5K6s#_Tg&~b>+E-kvZImQ&Z>z(Ats-YMjFcB z6d$@a>@ zT&sk5@;?ozfGo}8f7^hXCPi-1d5!_4^Q6-P6Ah&OL8q5boZ$H;mtCg|?aqP_UC?VA zOVcdd+0TI4Sfgf-c87LD8TQ{IyE%JWPjni3A+UlQl(LT=XY|Z|f2GP85qe`=ua-yo z=IDi&4^r%R7qqXA49(e1B|@%ECih_1?ZozWp@B?~`;5PtL)Wm8^IJlwXkOEM)_BW^I;Wb&ugw=JeONKOVu|m zJ8>jqTWcUIP8y36iP|BYETGd$Xmw4ukS~dp(e&4&m0OP=$etyahoX8@$%_4tg)Uwx zk;<=s^OiLJXn!ev;<#V*q=^-pAFy@N<%J;WXzp@uYwV2g`&6LLpH2!~(_m2TDOUBGJP z(9R*JHG5Z%AJ6R`9c{T3S|7SAaI0$Rcn4^axxCf+X}s3_*|p1bhHXl^A8N5@+93a; z$*4xnQ<8u&t--sz2A4i2&H%e~bi#-VoQPyVdxOTk!P zOzoL;o8qZui7WTglG>xuy;>$=8tMq4r3^i`w*ABoA2Tt&O*c)=HD}L}?hQzmj@*LgG|oCcTnM#F z@QeZNKEmusmPXLHOb`>NQp}brs(jLpQ)P-B)Y+Qad3&WRpB1@i(W_CjR&;5Rxmc)i zKLbg+xXtwZmEDWt@zX0rVEm73>KR7ux@>-xpPO2iiw*e0vg+5=S-}++OM8-@Y#A4J z#6x0*%rZdC5nlj<-fjj(V`b?JC+-RGGv_3Z zbtkZrJ5pz=v#2wZ>+MQ+OOuUTN%ucqH6~WM{DRk9GwFmA=j9 zn}NMHeFO32rBy+4$KKDj_8q@m=Tol~$hvmRIhEK@cV6BGuG!Kga1Zd<)p0%MC!aaS z{}qai=U6~K=I_ZeY|q<%gYGCd&-s#LZt`_b+Cm=I;E_j{K!UMMW~~qjG*)I9Ssps= zc^(^65-wZpRn*Na{#@gbrZK&SzZzeLB(VVOj2#7RuHBz$hx^x}@)iOvF7R zhnL~*=%H+qy{s;YzEF^kGrgB2K`P=c;@w+h?JhYU!QQGu*d1U*1Gdf>9mScm(OCzW z-NDIZ#O!B-MOsPnDsf^G!fS7jMqTH^lxJHB?1^DlITgapMfo#QHs!_@EH~J=h{=#A{M=%*v?@ojE9>;9N;f&-3Crj|`SGP?{)q*#(au;80 z$~vJa*g9&*y|Xs}1g|}h?tQ`7Bc;Tr)jJ=T++lB`_1ep*;0`eU{RD~-hmr0acl*X< zE>$NrR2-zMC$Y7pIA`g9x++V0Uo9WgJj0jDQpZLU$L8}p;B^rGQd?>PT zc`_|*=hm1kPZSbpS)_5hL{4j%qvvCBZMQWTwR#9dB!bOuSxZ-uCBB~J&flVlma5hH z#STG?KaLnX9zNXHi8E=Q?NWSaywJXqZnoTaZp;)fMS@6~)BOG5#cuoKG`y>;PuDQ^ zTD+HGuYYO@*rwpO78Pu70`Aw!1kdv~Oi}xN$z~b=$#z|x3Xz58ASqojyLA(^PJRa% zMEVp>LHvQDN8e4${_#$-QuaUn+gV(*c*xD6{9M@}Xs=e5pGyQ}2W~!PU(X#pe0&=R z`i?>QFv|LsX-MO|TiA{c=MQPu<()K#>-&v;BwbK=*ltO`j+`9jm=SN}S5Z-|XZp_M z?+b$H^?qbA0<|@xj<2SUsWL|crg*V#_G@r@{*1u|Zx`%;>ao=E>dQq@sd-@H8whgW zZMpIuGUo_EUNY#z~M5rj^uY~_^l;ZrTa{IKenJfmde9G!3x1*lSEyg zdq!qCpoI9A?>L=#U}I#o@5%7V2yi&IJ>1>)t8wQ=x;Ly;fOYF&dH(bH7mLn58<*Y} z1+quoK&;!ZAGBvRt?NVrGMF%M0fH?C-1dce4WqaOO9zLe5K+|>G4wRr}olZ zWM!bILEA*fPiwzo*9;+APv3(D%e74+nsN=|@~Sv#!YZDK5}q8_Ex}SptL-MZ`(Ow{U z+mS8t)=;+Z3+Eet4=lQ|(?2erqZeAXaN=yeD9mZl&~eNmu3egr?|z@buFgRgD0jh*Fo!wXLxjQ@-TDZ&rCsa7Y^``dvbF}ZgdtOb|$lRc-DXeS|zMiNp(cR9_=iZxug->3-=B5Qg(@-)&KR(&IPnh=W+uIom_;COIg;iCt=cdOq616k)(}`k zjXCAIDn3SLV&=kZGAE)YN_%*wVR8S=KflPNExU8}-Q+vhLHnXV*WQ25Z{?_Lz@`{O z=shYidwJ-ZCdZ9$XELOgGzI@H57K5#xn{<_qdpjVFn`HY#<0#7RA{;)ttS;*K;7d) zWio|vpjFZ8R>+L08-2v9)_J~$YCcofJg+vmwD_cJwtt2nPNfs|wO#FLgIQC{`#HJV zuN}^;D*b-O4#^nLoo4%lKM-5%Ng%UrWDvACJ?;M8AYRq_PJ8LhZ z{avQ}=|zHycmr_q(z$TBWlHf}KU(AqtOmy>fRo@%rb-EOyaoiRAzG`d)h@&%QS~l? z#~4Ky_!^mtz)`MA=XE9%7t*u-6JiB>>M*>qnbuZ{n*?-0Y61;iE46Ot(qisi-k*uv zIQQP0Y4v8J*v#w4c|4~|J&$XmBjY-k8!f3T zdlg7#swcF;V(g`huhE_(Qb_S9(gUX=zy_ z(2M0A4qslHTvGG^c3(shu>sr)CV;wighwep4t#(xNl}m`VxQW5yK^!4(~2K?i3aN- z3o3`Ul;-@Ny;=_=S0f=9bdISd{R`&!VW6+IedYCGX4hGAt)|IAe@jr2)~J_JJ@P)< zM-lBqeKTUHqfy4B`-hp=#dPGyuk-VVJlqE}xdsViAwyMPlDLhmB&i1UygG zYrde!O#YG}K%Y|9_?mI90Z}E!RIm99!}*_2D|glB_-M#zX@`>p+%Q*=wo?V>VZpG3TcPI`8iUijL zO9<)NzvnsUne(3W&Y5rK`(wi-44IvM@4eQwuC=b~E=3P}v{r0^$7<01tLOpXr|%TO zHqFN}s1LW@%+2$6m+q$BVxcjm`6foiGQ$HjT=JT;iM^02TGY1Db}R6KuiKN?4U*-z zZgke;(njw#C4a9Iq^LM@{;%3_phZ6iwCG#YdBVT? znQuTM%sL$B;Q@Hmte$De_nJAIs(m^Bk zY5l|(xQ_0`sy4h^ZJa~1vzYO$Mp`aM-_LIvXL-rzS6*%^{Tx@G`zC2G$!QuJ(laT@ zlip+;^ME9^V)yC>Pc=wK_1#K5mW z4;9vFAvNQ{t)!SmqNw-l2Due%ae6ZI#O-{$o4=TQ>E$hGU!q%Hcg|pEDjhSkdY8|8 zeyntaK_5KQJMQu{iW<3Id1A^b*nbwY5?lNg%2VZ1AlFgDQyflc(2KArmqsPER+d1p zl`|dfTfl?NyG%Mq8Ib>6`33o(O4zz#g)|a^T^HFZ$quMv&BV4l6ssayZv#d6SZASO zh6PHf!*+|aeW37*3P+MWLyf?NlU8`$Fpo8-ck&MlYpIg1mCwBy z^;5Y35A_;JuA-vN9j7GWCD01}i6%}6k7K==ybX*?hslLG@hkBzJ)Peh8U-SzjqwXH z&Gjl{+?$2Li@e0oOVhCZ@cJ80$!2&ms`}ss;*A~)c4z{-gmx=6RPwLTkp5a_q~z;(CzA9jL&XC{H4vo^ zBcbJhGM!-{tX^Ofo6R6hMJEXeK00O}UO(2Ru(P~HCD;Gl<@$KGORVt+4aF1}^2xyc zKoAg-g{<8q5rvL-@fg+XzwuL#Kd&O-h|YA(339xaoRGfwcm{$7occ1V`5g5BiM;!? z=BfV@qM4fTxHR;|(((ja1nP{kOe-7w*f zR&d9R;#0;O27od98Ay*hngy>t!FqudhN(UOg^2avjKsE=R&c}~_i~L2{FXQ=qC!ZTn38`q&z|p&71*xkq8L9CV*vpX{iBEa)>e>uV|J!*RExgqF$h zCJdv>U%X^a82V|>6q$=x7Q34M*l$`AQrH_kzugN$$K;8!Q^JBP%kmt9&o>u2CyZp! z)$OFDa5UfX2qDiOelDH6x01~7L)jSXa!Y%S4B|U_Y16$Zyu-q*V<%po;u)N;mtFiX z^NX5ZH}v@N@Ylb1fK+-L9I#gK{XycdGyWa%$+LM~S~wZxFCO?cb_P3vjM;(%Ac?1= zbIV0T5!sct=|Y2)cqqQ-^veHRF znsL^HqfLQJ3O<*Drr_2u;pZk~Za4#tZQ2`+AfOS~+IxAU-wWW`q#5)8c((&3a?{5P z-)rVLMw`Jh0UrcthHK7|W)0#*TkjwtWHcprnx{^Ds~7*-r*{$GK9f*pJH z-U{$5*=W%Ei`V1Vc^BXz%)b>4@YtADgJKk9RXJ!}tumm1wdDRh>2sL_+It z)JNhErfVj%zA(QpP@k16X=lEn<|%NH&MFdgcS;eC?=Kvda1Z7s!u=WQ&qrb%Qm-~e zgOQy#%LY2nBVTo=Zg$bKp*4`{EB&LZ=?ws~x9s&MZr*3gOcs-4kD?1sp8$nY0-l~`HPNGk3kj;C2Ik50?Em^tg(s$lb#$? zxW2_$iUIz0Vc9xE%uNPgwcQ2GQcmoVX;+!~a`$+b(g8KKV7}t^Y$BV0gT4 z^G3%Q=|BIbrTO)&FD9v*d#`V+zF^T^A@o#pnFP=&AY@TU`hg#hGdsGY3`kSN}{xNJ>PBaN7Xpqz)L0VG_W1%7lhR|F&jx_ z^ACGXs&7!q!{@aA{rhlu&Uv^pjF-6?yMZ(fN+X{_e^!c?BT(=!{Wu_6lLrHAqPXZ` zqkr*4get+nl_{K8YI#d#b!A!BkBE-3+BBgKVn1VRa*DD%9yZYWA)eO+JHb__XDE18 z-nULN_>e0orNC#<;*^u7PdS$yJXNx>GEr-pL%0&ARH68+0{9vX7L&s zN2sZ6PQr7hM~`}?g>D!;MVnP)CmD?Z(N2NG`E^hBB8J-dNaClrXgAhC_^6V>M!S7( zt&F0AvL0KJNBeCRU|Q9lA!`GD<^|1cWT$#VIDk%vdpq(J+l}t7S)mw~EkVa4Sp;{q z4Fr23j!I~Moer2(oxxa67Dg#G;79cysZ(6OLzlS?SW^BQG2D16F$#s!c%)?{%ZBm6&!dJWlWA6$9-GZoG)wzU;Et-tBg0tOJ}^UQZ{- z9^mcS-*X;lcui}A(?FZ^@N$sTIv_HWIs~M$R^l@jKF3@-(@h}l%2bR$*$5eq0x8IhK2F{w zN~87A7#B41H(}AB`=z=M8fH=kJdFw+$?j+sWgv;|~=yq~)5uAG67ygx!+?MtGEmOU#cA$nqZI^YRfM`pAZSIVh zXOOR2;j&K0@|Hko`^kdl*8kD2|1%hd1fz$*{JSgK*n(-HDz%>`ir8DHz9AL`$GwLA z*gx=$m@F`zvyffFbdp;&y{B=kmX*W}0a+lyf2so1%JD$NHZK|-x|N9fZrxOiyiEQY zTcs=6T&v}@E|+j-SrZ4}J|!W9ZL@ochQ2}y$L0o9*YuzCLWuw4mUQzD+llmpHsAua zHqBMW)Wo|-pRqS7GhFNLhIfMhzeU=~&p?qR@2FL?ZI`oVp<0tJ6l`VIdgB9`N#*a~ zW4<^VOv6_H3so~dyRE_R5YiNBN;e;!G#{CnXSf-V^##Dyz?I_-qtsPwa^*Zb%zWhD z=Dn)zvV9-le(Y9RUX{p7T1>RWf>j=7m13m<7IEAwhr=XE_})P0*Mrw18U)Ic1MQ#) z7gn_l&FNWaZ=*%W(K^{xm<(f-tflW_iNG2DtwB>^KR1}gjVs*P^U9Z$AZV_o16NGF z?Iz|Vv%I`3Z8p`|{@(d}bG$#4M~GR`mDiK=ch1KM%G(u#F#!j4>!QJwcP>E8BeStmN9ZYrc4(U3qLP~#yZLd#Mnyx!YT^#Ye-j{z$CJw%FQMbwp zjT5w=Q=6&EZ=EX0;-zz0Pk#|cUcuzqe_G^sn^S&?V$W$b`;Got;Da&!)l_q@Hl*22 zqz}$yrc!_3WR`k(&!8^#V>J7bBZ-v(eyoBH9VBcEZd8*JY@vS+iFR&WPP_hL`<xazu(SH&oNhi%^%Vlzz5|T7Ex$FShlb|D$hQn@lB31`MmFO6m*?3TN zw{G1xmaIDh1#>68J$}WV==a_^0*-zyMZ^2P54mM(Gzm_6wUTKQ8>B=R8v+t@*23#K zL4(7)q8OhiLII3cSU!CPJ2Yo`_>*E&*g(o(ymWWnMbOa0*V5@W{n)Rt-SA>R2tq%P z`ld0#aFj|#&}MnvMo;r+VLTgNFN6j*hKGX~p~L4;1|ui$T9v^s&n29o8}u`=Y~D6+ zHDw=aDso&Cfrn4Eu4}rA09Bbh}lUn zgmOAZPlu^^Or(#Hh)+LTo_%z<9sG&(-YVg55Y>8vHIp!6>FwwXz83#UQ8MmHAKq(9=V;&b&IU5W-a-ra1UykWV41_b^D>!RN|9jmVIrq>F_k&*w5E zk1LGrgN*73UC$+U^(h$6sjbx2H*XE5-a2wONQX^q!p1m3*zzuY0PH5xqVzeAF;qF| zUI@0WOYfhz!2kVQ*-%cPO^SSmdNJ}6tuc@tIN^OPV5-D5;gBp+r}pmjrXCylxw@Ma}g$4=& zvAgIOHo&iT^zxROJW60F7OJX^yiBzGF*5Et2dnJLJf+e@_XEi?{wc7~m6Pxj!e8(`0nJ_r)edda zYFf6_lwuvijcvcJFLrE_`y!`!d0EE#wWll>A3xGKJp3@c7=p})OTdvU=nC8)Ac&j; z<9^uEnz3VSOP3?C8h}^kjb^VI&=2860Xc^II30fU0QMJ8mrtLs2JJZ_tvj8gh*C`P zFW4t3mrNG9KrW6|FoY=D7wcI(SEieC4~M(Kwh6uFg*4EY28f%YU-#+amrRXFT8L{Z z_K3b;KXwSO1a5QrZZbG_{iyB^2RQ}yw2H}5{H|dY9{x@*#PvTZ|KCQ1e}4gb;n-3G z)Dns$QNPP!*5`%Jjks&m>Auh^dWn(nV+vS{s9SuCvL zuV5KN;7V-J8?e^2`IP(@hseuxc*U3tOw-MHYhRm?IEu})JF10)^&qd}NalH0e`FS} zKVz_^&;#xK1sRl;=V08Da?aT=f@vffE$cv%^9qcz&irXli|NtZ@>y3|)^nN@rr02{ z%p^v}wVuJ-J^Fy>`A^eC$2JrD0~D#L;qt&!!qL%)5A_^TdnYQ-FZa_&z9%%KHdy}@?KUX82-0GT|06~OV&>6F;l{YUnY5c3au+>4j`fr* z&4Jt5Ne1xoN8u4x>@S}L7NAWE4OfHAp_Z85#iO4aubbj$_cQ}l-dfWnz`6Idld4S_ ze{v>l-jwigtLv;XUN$U2xPR{V^sxCzc%k3E)5O5zh5q7|HK2X0F*Elch)U#K+;uk? z0cTJ`un3!^o)WqG@V%#Hy3LsLz;cEZvVaow=Rn}0c83Yh8h>{3^D3O|DswOYJ|u~E zUZ+oAh!(+l6zI09zHeF;L3)tMz^C_APH2ibE_sLD&^<8pAVfrsVlH5R+e~p#>`dx>s9#i008NK%iu*JnO{oeX|%Wc^T67I2LYeERT#$&u*z@N+ZrWc~3mH-^s# zVr~Q~+c`0LUsKW7?isijxSJU{m{^mq=58EW8Si530>7|n9rq{Yj3+W2y!TCYjZ_sD^tv$Ww3sZ90++CIbiGWAgQJDp2|(xx%Ply}JUV86_uL(9W7C8A@KGgRNi zT2V459g2&bZH1G0Z$|1) z92G`~k_D<1;Z%a&*!x8_*X5%LhAD%p>=;HJ{GAF6yCwPQ%R4-ji0W&l2!(o$syR?_ z^gxAA8#V;{qgkVgH${l~>18tzRq0^cORLF9M>r}8qm;I7S%#=mP46Qx$R0wz)Tkxv zg|tv6rxouDYdb7<7Cu`Hm?mBP-Iws{j4oRFPgx$`5ua`hPA>*gbMxAd_NLnfwwDbs zO#0wt_&NIy?DApLU%c}76>x+h4roJ8mjF{nzlG;;W$UG#yaTw`6UWcACcPf=YD|+D zY46Pr5!w(fy#!Hm;`twkcLA3(r)AHe%>cV@|E)H*UjSP0IIM-Jh5ixTCUs4rRGgqzxP|>Fncl&ev;3j?viC5-h?t>UcAX7gyj- z65jP18MDT zzJVd8P_Uc68rWVnfP*OE=)o9lTaX)&d1sDq^!`b_@&G6j;{(hhH2?kgZ_{SKCX0JYMJlC)VyWUmWUxuo#1xUn5;rA6s!~hex4Oo`{gDBp>xX}Ty zF(acu|KU~hj5A`G&aB(}Wj{!VE2W@^=#@n4#u~5~b`!v|3IO*BaB?2BlCT9H4vYrw z-3l1pE%+Dj8Gz{v0O+V$von4Q!QA_1Gk&v=iNX5hM`VFvH!}{9nDX_SBEEkth#`Nc zwjrq4>R&;AL2v2F2u3M~f@$Wa0-t|nRJf=f2opE`N9aXklMD6sjuCRGUOkE^@?g6+ zb7REyjo3YIg*saF5nmv!7h0O$u2_joJE4dfT}}N*`&^VFYDf z8EU6TdF7A=C|s|xE@3YjDa2}@c%O-&)Pa2{`}>qcfLP22ym>KbuB3@X5_sf!K_k*8 zb1tASiFsWP+4Ch%628nRI`7N6Bw`ln=4vfoJzI*xz*K4H+!#ouuC9;sqg%@4VVG@thL3P&-vkQHmRRFI!U%iHYYJk4cprgaLY`!$&FpMdOam zvG_k%X2I%e!enHK0S;GxX`79C*L8J<+jN)M>nX@MHS=-Dw<_qhLX=U|SPIO`vMl~Z zt7V{o?%iPfhYX(T=+JZ~mn*D}etf_kpboI#lKzVqF)Re*-NTt7gM!gbh>FzpX#e&o zE{9ry7p=^}gGyXP4X85u@8yT9f`*=bMn#`2FV>M#fBdsQYvEITjs7}woYKTA2=(h9 zl-7yfLR1$5LW5D`%&Ddk9Y>zwUE@#-`-u?xEO3WV&6{?*{%C!?yt8n`>WS5tVV14S z<0D$na{;8zpzk6G@ht9)`!x+^bQa=9d=`NxNZNlpLL0-LDS~92cR(B20IqSeaY)$S|m5v*6r#5$PHZ3!B4Kpb} z+&Yl^5QMRe(7VqWnBTu`s^ir2aP7*nxwLt^RkXa(r$2V7wHMS2LXX6}gr~x`nXs27 z6))>1&##9K8Kj6mK~}-5Et6@TS;Z6^?6m8`F!?=-b5lQOKB@vd)OD1!E^-1-S54=; zH|s2sbX2s}7_76!De@5|(!XYt`M?kG_a1}ij-3G^9qQO~^vd_8K#>DZb3}UL>+^O0 z?>tg&CrYp}UbN1c2x6fvj%aybq#AkefYIvN@AnC3TNN$mRM0Pe3Zqk2fBNl4k9^fS z8H?C^Ze0dd&M5Q}GAIyD8~DLml0VMe$+PjnbJM!Y%ra79(uiwB%wQ4DRi{cWMwIR0 zXUB;bAu!N=#x$+kKztWx9%en-xAvf6(chZuJdadPzN}e2P@cvzwm2`!(TRT>NYaj= zpuZ&~VXN!2IpMo?0b*v?;GAdQBZ05GWVP6Ve)pfwAevTJ1|>)4Nd6^7sPsiWQ@s(| zt6s-+#!r zbvN{Nhb@4Z<{79Vql^4E>whe3EP;?Cf_#Rp0QMYEZtRbl%7%WaX}W*&i|xj+V)@j- zvh+b9zFfmBuVlAF@uF75wB06X+LKuw$RLdNR*~2@PZ~>c#tk`w6&1lEN?T31)R|#&`D!mZT z|Fc?|-UODF#VR;3=Ec7y3Mz;!-TVEKGTz8oMydRg4VzN?~My zymmC@db>8Ha)*QM2h7odI#^}F&+9q-z>hy}*F;=;nu_jz76uJ{2?s+kZJF8nUs5{? ztelA92EpBam#!TvM=Io96{Kfmf?CEpBHNzU9pA}$%dk3r%*i!wCGzDCO0z8wzl-zT zD)<;1-`AO{`|+AP`G|=4OEtP-oC}&icCwdF)1dXX=EdjYl~`j^7Pc1hv2w7l;499V z=69T{EV4i~&xp0Ih&_mMqDi}meqajbZ#_h&8QSe8Ay+ipkr2Z4-;arSv2_X-A?De$J9 z?X%TztM&Mir#<;<-TBRfYLu(MBV3x*^O`ZnJ@cr=&|~OJW%gF47PjZg-H>Y2Kh(FQ z<1-4V0vJ)D1IAo5$_=RII#5o>823@ref^_Xl=qSZLR-S`0$O|x4-n%Dn`r4WN<{X< z@U+TW%rzsX>wL!EYk~hnp30uaOGL`;baZPM$nXz$I^!=Mr|M{Ygu@8`lwNAZ zsb%HQlH!?>4Mi&XZ*5XZ#SPc3*ficaKm!2-gEE_>1az>2D23&8;}(#PcG{-|2Cc*= zbp_r-4F(|qlhD5)?n)2*M|S__>e%XNmiAB9l(EHw`l$$yuGn}3OQ$(MJLb}9>&S{Q z?8=nJyi^X`?RQqdUJuWcbI?`j)FFIyw-J}hi{$||e^-yxeSVlEc470vAF$71y8@hP zzZhXxu)R|o{t?H*dv%NbGRV>QKNMg?z`iy!n{5JM5gPMxdkwC~z$cPu1$m&6?~T?` zuEm-YmHDTgqnxJTtgIWmt@ZJRWeTA`-8URAh~u$N+hN_Nwz-bo(m`7$nyS`zs2I^r z{Hv6Kf=YtX}6Nt3x@+=Q-*1?$uy z3vOG*iobZ`rH95X-}M!odn}`lxB7vx^esqoKrnPmN?^=*#8)J#1>)Iigp z;nPChgglG$ClHt$ZK3l|OZX&$xNJb#ZZl_MS| z$26AMqp{8D%>5F2lFgmE_1pP(@dl4$o3jWAAZk{6c2;$^aWNXC{YRtBn6JKd7^uKF zy3eF9VJIh!lF!GWs;ZYJq+}!X?S{@9sUk{I#HK0!izH~*XQXXrn|V@+VL4%M$E_p@ z4;x|tG+KLx4@yhc^Gs>&eo#MxG&y#+zyrD-0qYs9QYV3Z6@%JOd35otjokNUf!gne zLPO~Gg3@Lq#Bi==+u}j4?p?(h>E|Vf!;ixh<#$@(hKm{Rp>e0pjxRU(B8tYP**U^m zsC=!aw{#gc-sH=X^sp~=IpdX39`0wWFAeL1_e8L24Rr^o>(q|1mg@ypvDEc{3-z7j zD(f26IiiVbrSIsFy$`%h26{%TqY-BB2L)sS{y?23i#^7s zp!6>KN)ZxhQ+d?1hlaKz-tW3w#IgZ!A0cNIhls{)Z4sXEePsD z1DJh(2t)Y}@!OUIF{{upc`a-i{^M&cUR3yJ2L38Lf0a1K0;F&>GV|s``tkEJBM?`meqgpZlTv30TJ*XP{E~g#-Izx0)GIZh1+TyGIWf-WW{d469f9K-1GP6 zD3jZTwsJg*#Wq5=oHA1H)YTG4PMR{hztu83HJFE?wgB$W+NlQn;9LU}THg!t`ag5q z%o;5!w&lSy>?UN(G2s*>>J*CGdXk;y6vIqAxSW=`{}{)LH@JFs zP4!S&eZZI?H~G>A6LMsiF%v5R>9k_#ilG&qTHEHvNOjCxO|ZKSsfgI?3y-jo`^Op4 zqqmEAq9dB>-}MIt2HwzcBhpPx_Oz8adh-_Zxy#}|)tkK`r#qa!{WM-&M?`ePa#>pC z5#56&?z8qA0%Xgp(AQBA-PW0NZWR}0`pqeF>#twrOW3|LP1RQ0QX>%R<)j_;U3*tE zxof}_%%hUGh=Id`xZlRm9M#N99WHq*R=$%D(|2As=3bdpJ@kk=Upf;Z&eckh&JLUg z%g@6(B^$`2K*^1V_YR5+%AFn$%_X$|rYJ`A`Lu=?2fwj#&`4hBzrcEE2)?<$hW}0i zLn~5rP?=g#G`#-(IiAz_@ZdA_#rRX+&}O8IHFF`rI}H8FTQO#9jcDB-AtLIKkK#o zcfLmIQxzpE-iZ~z`{FMCK`EZs*mtW)4!-$E>e7!HiFSUgGF>8nw|O|=J%S~_esqJr zHH}~qWNW{bNSeLtdIF7Iy)9)DB`w5HU2f%7y>zTF;tKxfAy9Z0jwlM5!aMrkQ zl{Wt9Y||08IOfz?>-u2U$2|A-)3aq>Z64XzGA7{Uk@0HFmaDcr@3iHaRs=8EdMy4jtVm?Kv4(LkB^lEkmCqa; z`Le#Fp`{#2jdqW_Tql@JR;7!y$CDbXV5;9}-OQnC|6WeBJqXFp4upt92L&shKZZi& z@2w2#PBQu64NFNQAa9SeM67lLxLB0-QvOJmI66wc>5#&|+w32q-`YWRC+}&u`S(c} zp3lhd_%>=vKKa0f>N+PXakoO@==(0;Lhixaq9AThhMi~R3^=7^eQrxY8&=VM6&7Ft zNkoY*moGWLvaT~nLgD8gN&}FCa*(NN`?;vlULcRvftW&Q@^oELk_{KDsp8=0W2yovpR zL}Zt+m56SpYlE=iN$v)4s1@#=1kRmy$geMFi<%bM{VLRCiZyy`Xf3$sr&vCH+?ASO zm@-3t?s0hM<$p4aWz7#5`uaWr&eLi1yU$dk4D9H|J&sRUIB*)d0DOg@pS}Lo(JZ-e zuK;n#L0Cl>efg#?9VSOc7(RE8gfTYvk$P{Gh)^%Y7^e$-u?4GX7e?4fpyJ~4-kthD zGt1Q*!|9w>%bNUv1|U5G&B{#oIqokF=p~hwpy-C*fo2~8yrUUPrVfy7+hE{$K^N9x z9ypQpq3T}VbMPq_aJ5sxsI!=%WsVgs1n)~=C$3US;Mh7R)aRBu@diX4rji3;Xqc^lu06a zc5Z!kX!+G>o{cKK_|s>yARy6>XM}-KVRIHfsGw5N4Wp4|O`H3F-MH%SEUE>&Ss4Y; zW)IN?S_=OdGn5ma-q3@!kfV|G7ScHj4Qo@~n1UmXzj*9bm3?1JUg7zNwr#V_xGY&| zHGlUS^KaaIHSRfWFB3~p7WjFSM{Y1Rt9k!0Q@>b$x#~qpwl&wHU=mrbOU%%>h-ew| zP8Jj>%CbWkmTo*=oY{+Lyh!MIbkM*}s4Y$%;`hu}y;gClT~Q+EJ(z`27}%ey6#W_U zi`BEt0TWO`y-xYjS-IIUyGgp}t~j2$*Sve`ccnaD#>xR^3CY9H#`z0SpRL7rjVZeK zC1LdzCbsJa2(t!pYWAKj72bfqcvMR}l+O@|u1*lAlN$fB;Qqk6UF4u5sYf37Peb$Q zyH3V&wIiij|HCieK4m~z&87f)8=bp~&Wh|Z`09Fme9UC)zz}t0Ye4X;f*R|Ls_qvLjj|-^r?Ppu$O@KzP9nSAKOx=~^^0;ZztHQUc zGx!tVS$<5CV~^IRYvp!O2F>t9x9S%;iFoI1)HY&5$UO~^!Y5HU*v~w&j^!2>yt03>Gy?(b0Hv6o#5A9Ync zK#Qa=@v@f%k{%aVsmn8-=s?>Nt9#Ci7gBz>g30KQZ0xALl)%)ZNfZOky)GWyi9;cwOnr=Rym*}6V7NqkNe{s=xVEFNG>X;D6 zJdrzni9GMw5qt=o`t#w=m(K=)1oXJf);h|-!|*^F8yb!u@bWojNevFNvb0B|G3Cb1 ztWve9w8rK9T#f0Sz2dT>Z)dEV8mM0zNnH`yt;H6nbiq%xAbtt($^V`e`|o$St5hAK zO@t)kTxOW&Bxg2jc=pAohbnmTN*(ll4IY)|X$m&HI+82J(>>tRZNUX!A^{?syCo5P zbhiXHL#(%Y#tV;yqg^_H-pD&{30&zv;5#o63lxjPE(p-|M~Bsan&~$WjyHL6CT?u7 z-w)IQ?w+Coz5{{L!;ni*dE6gk-+y=4<*v~H0g${jM0rme;QCe zl(n+zO}nfa%I{sYirW(Ygs}av^O3+Ki1NrZ94W^8-L$*O;CvSFZR;W9P?sJ`s_(-G zAP2zA+oP9s!ha`N9!3jCJO-RA|-KmJ7DKnYg;c8Zy}k)Ls{(J1d5K+8`Nl){f2szu4ib9iljs z0VKthS#lzG7gZ#St()vv$tP5Cak7ildMQ<^P5e*NUeU~JJJt)wt(J#o#ekGkNENzBu>Ko>55S9E^#b}gD{K>PB#UsX6`L=?a`0lT(d~309d>?OGjJ#^q5V*LU zdH|2ok^e=n9U*ub&O|r;n2^8)s1RM!uNDS5cc^hP_e4KV7hUEnbhdpgE!s-bgeBHA zN%GIXnsBPKndN_HSr7B3=a8q}I=tNUP`tDyvUBohqD*Qzn);F$^46G}V(NU0$1qLi zo1u-iUrGX)y)mNKMO`%_LeZxj1N!~}BwJVQAVi%DB?-EOl}!G{dn|ZP{b0z?Hj6Y+ z{Zh+UzvemX*24ftPSZKnm*AgNX>#&cU+oszLuN^B()BlFD4HHEOW5$I?oguK`CkOe zQ5);FRQ@z)mS@BhIg)*`sj?^kIrcSPR+kFFRtk2#b9}4$tF>2*KnC^eF~w}QV~cHK zV6%zm>ypXYf?_!fHw7_wM#2PrE+x0x8%y|p$Xf-u#lwN&esb*g(1)CD}$LcX!z@WdD-7dpopws>pQ$`tQ6CM#au+G&zh#(=422>aJ_9x)s9 zg4LXV@NDch;}!zmzU><|Q>f%=(llKVJYgef{TxYMUj^=$i=97M51;+yIm^8`tY8GcxcS-i0Z z5GZTC3jmbe287i|4#~0xxXw470^d10!2^Z`)|XMzzoQ(t0D?ruKZrp4zmMtv?}-1u z|Ml-gBF+~|tAYXy*Rm|L1u@gcM}_#B9yPPpv@u;W{T0hTL5*wucu>vj90r^Bmx-iaM)7-Le`FgDvDVaTV*xMWJlytCo+EK%HL`?6_y z&9^isk%V0g*-}q0ejcLRR*)M1A?~0zL{2$W&eWNrKr8~343};!#^z0UcaqPL1x#&a zI5Bt3+Us#EtYUPV^-;HE8I4N~UIcJ0>?|jMo_wG!1|L+MiA%W_^s#6m>^6l1_BWOD zr?8_AzU51%xZoDWWKY+p4T=@!TrnqYFH1R%vlk}5-WKuh;K#SMEBT8T9MI`j<#^#& zxR`;?6xh8`v%eU2xKoemDBOf35WQ1)@u+aKHKe;etL{VACn)dac_?zWrF7pBSV$!_QTE}kQkCXS4Q>0D}$ zGK%(c9l=J)DlTe=Z4?_{6c2h`CK|M2demsU`T3#_d!wc4dwIMsXn&o-S$EqMo7~)= zZ^k^4%kRZ4HzEe(j4sOIR@mvct4ucf>syHha7bq~I^G4KfRs|bxs3y;oJdGm;5cSqwSJ8x~p z=@*Y=(^gnOo+o>&3$3PRi&l6<7_(RW^XcecuZEW1BeeQ0wW*bMe2~-bgv{wxokB0t zo+K$uSLqxxd%RaF?nn{5dcEuN{`XQN+fNlejo0CIMM`G_(F(7kj?(wqbZd9q_ldtK z+#N5F{M1BDXwl=nM>rz-hAM=5ZR}(ql)`^CZedlSInDa^?aCiZHo1F_{+WoTPxX4P z=af(0ADr1y>e?z`zLzh(#M9Nvq=}O6loWW1EMdaL#CvFdVhAIB)ZfoRT1eubY=)?K z)g0$a?WpnE8>23#xGY@%>J$F7*RqQJC*gTeJv+Qv+gg4I@!g25u-1I3GwZkeZ{x8J zSMFkkc=o4`9!+O8lj&PLBVFJp`*T6w)XrknaZbE}Bxz8oY;`a@Z z0p5JCKdL}0pB`0rzA*zGT-li{+|;atKq6*e;3oPRVn4uItv4$gsgF9zi@b??9QGAl z7_~m+DTj(*X_h@58C+lcdezCo{$hDA2Nf+@I2~{ppqbZFSKp++t0P#&gJWGA^>uOy z$M3iq{b^qB|9O#zi%##)hr4VQQ&wtlb9A|M$y4=sDJ9vaR=Ma836O~H?KD2AA%Vx; z-Vio-TPJ<{1xI12-LD)qI{BBCbIV7w-Gv8$nY zk56dCr!DfQ5!Z5BZnT;-sJu=xjF+K?h9y%DOc^gQEkS55)%OR~k}^8slPpaf@@xQg ziK|&W-4}}r$=c)Oo8~d5i_+;6)l9YwoSz5??Q&JVGxgcFk{`shh-nV;oD&5p%FgZ=~A#NS*$)jhJQ{MjmMlQ#eo2$B6Ve!u^X5yRl zi_fIz*Hd^|bUtvc&v1LYSknIDXP;JRYShti+-ERVv}AJ}u8SigEj${mTjj5d-H7}7 z008i15C?H2+QTFj(bnegBvAUQ33MA}{fOdVPe$8MCm=KUXl>;7eJPO|LD)7d%^KVG zdC)IA!-jM(f8A^5u&`qHviaRJrwVe(kkO|H%Ef0-e-r;029dtPH(S9012=7?cFz@Y zQs&;k0tp~QJ5sjc!%>VHNyd8`><$dp#&v5Cf1fT2wCJ-eP2I^Fr|RkFjFz1s-P);l zS9JyUZ9jy{ec8mbLw%*EwwpZSp_t^(+fYkLMh>Kn|O0!&;sa0@Q2EyFPD*B8G~&GvY1}f{ZHpeV*X2 zr!E$mlDQTB#UsA0E{Nt=DH(X=x4~gD)yRzt;RDe4KXGT{(h10`B%RY-$6x7(37iOxJg|HX>nJ z`kSV*q`}d11~TS0__;=jm=BMzWmy`wdy<9}s+r1Y3F*PO=Z^SwnJ?th#D+2umW?Opm zLSgDd{1tw~YY`%Umn@}*+O@Ak(5%lhJAxgezWePXjfdjA8vAn*eH6iO0`asip4zTM zo!*YlM+WXo*Dh8bLp?gU3moOLVm?%>Kdw>Un&Zc7*;B%<10!%7mkI5#{Bgi1$8+n~ z?Oa1fy5{?iv)M%(@J9y=*jAk+7ZH4u7`y}DZR{ApX2 zS|p`SXIG9R-mULedm{9m6s3X^ndd=2d#s9MpkAFt=8_fH5-S?S#vv6x5wrwt$uf+@ zdW@tT)4ExoK4uuAmb+{5r6+j3_BDoS1<-b{z($>&A@_v>&HwPHZ4#SyyM8!)%26D- z>N4u^rrr}Hvq_~DnRZH(nW`;WXFoYao3xP_#tmXy4~_f!;Xo{_GeqAfdGzGBYYej6 zksPJqZXqSM+4QX~qn`2mkHj3JI>zlZlDLN+VG>dmNSrFIBsBCjsyN2q1jgIzDikHz zqm#JPK_hF7eP~Qp zrH|SRt_W7r$y(ODhi`r)%W&nSY$Qc4_8;qd*M#E9K$Jf0 zit$>XGM#FAI+LSW`-yP&7ej3eg=KRzUlZq^7(u{~%+~x@jxoOK*FMx~G2L_na~%eO z*cH_Z!#z!zn2Ns7Lanf37 z>5Eg23oX(h7tFoLtv0BfCVoHY;brV9`JrY`~Vmkj>58&}nJi`mG=08<-=-)hm(2oT&o ze{5&Wl$S(74Y$rNeE$dEH+!0hYqn2Q?rgPmWZ-|0_LgB$wQbulhzN=>5~6gMbTd<< zd#U6u!J-64m~ju9#-WQyz*{nymO;wq^I}j89v_E^>*mLnX1BcJJxUF;EilIL`33p0 z;d|wiF?HxOS42EN*J*lsym7|VB%9_!kv8q+=t}=sN$EAH*reMc`&q>c7KRqg6s&~t z*q5Yh1yMt5HgEP?1JVojZY=9x^2B`$cTmRQ4;|!DPcINcZ0CiA2jU(^NPvALL8=20jVSfC8?1R=ceVe|*x!F|4gvjNQ-{lnSyl53BGrLH?vlCW z(DcD_v#IxWhAUdG68&+}ebfah6aH_>KPiM7)Qq-A-->m^BCE4D_k|Oh7uWk)F$;9n zpJJd;61<5Hp)Xt(CEoAPdh4iiF!Gwq0zG!DyWQw~-z)Ba@-&aGBaHD>ikBXVa45Ep>Q9Z{kaM5 zUhH8t&@mHE46V**AlP%QG?GO((0bVJ@{G{D6X?|AbNbeMd<1F&^*IDO!bQkU)3(5m zOIQhm<9?Vq2qf{Qz~rx0UwJ=r8*+KcPpZlvIYQ4YeBbG2m8D|$%$s}UrVHwJaDBp5 zog`3vayYE;j45wfmh3+9&zCw~N6wXYe3(wJr~{3btvg%hd#kRpR<(@IrGh55K#`!2f z1-9#^a++eDS(7Y*Y*P$mCat`VD3vSVdCK?uhy8tmrlJo74lZrb{;t%o8YT7~KmF#y zrgzYF@T1o+VN|HrT$#G#HYg#oEd6T=pQ_Xr4s&`oL9^O%k>&%Z?b}ujVIN4QxHg7( zJGzd>Rbq;nBUgTf4S|HY3A=gU^NGOjQMYqzKVpemZ;}b#U1cTF^NlJhGk+~=3HH~h#6a_PUnod_f}WzSyR*g%Iy%5()pEGBNox zy`G&;PS9cp7~`LI930+TOf4a>-dI6DuBJ}!Y$-H|!#+;S28C2e#pM6s9+o#2QCPmo zL#!zn>%%>@;&gnJtS1$|gX+JP^b7E2SIq7*b*C@^8j#>*%UDvW&Fm{}REv^vm!HXHB5QG#MYJy}!KsykG_-q- zH%-a89uZn?4bl5>WF-7qqrNr6`a61-Vq2cakB%Bmx0!{X*oO2j_T7^oZySb8!k3nE z84;uV$sXPoP8!Ek$`|o!>Bp-4emXSlP1(@IyOOJT? zrio)GGM17}iWk_;e!(?k5JBn)tZAE7vq_Df-?2pdzfvrVKD`)PNXUyNq;H`DQtXo+ z^1i*@5T)BIFTuXd%H5K2wHC-1qMZ1E!{Nw0hQY;HB5#NoYHd4APrP`fwEX#JpHD5f z2x02a{96aE7^0mysGyk-C&43@#T;@f#q2KsT7LQ| z#Ru+1pyL1haA_lAzJWC|P2zMiSl8^lHZDz>QW`DVoC2yqP9cZO`rVno76pSi>-qg{ z@YzXrkH|`n^hyeSSw6hX*%yT0t(#6tA5kXMN1~yHm!MypALPfd~A5+q* zC;1Fy3-Jsk>G9Qck{ok8NIiep$lecZSE3cjB{3}rTRWcwYp$dwNrK>ueNNootK&++ z$D8V{Q4y*o^CFsp_4QtkzosJS*|0;7*a~}Hu755Y^{Czd#m2oORZ+!suuFbUKdd`d z89L4Ro5|rMOGA=VA$Xtc2gACAO>-eKB3>??vsJMNk0MBS3y#dSA=A~t^9ippZNPdj zv#ix3Lo0f{q2%)+L2#S{>CRwhUFq(i_+(>si0J#^uUUCag}~7VP3lnI)o7i z6`W@Ahi1biH+Lept=ZnbIMDh72l_rdJds&kIt&h5HUy(pguuvlk0;kA<$VQQ3(Y4w zwDaWg9yvpfE%}aNf$*wcttHHp4W5M87Nw^ym|+#z4{53}Nx!Q;W>~+i#T_RXW^Q1aY1WZc@e0qT9P7>K_ejuM zI1fRsFTi)=+RI>6#Ks+ls^T+z_1aHzKG&O{Xz<5erm;wAiq+|iYZs+_k;CDx`4RxQ z3S)`~0q>_y{KAm#h9;af*7(Z~Kel3WHy*6}!;kweeRvtT{Kpm1P4v(EB>NC2ULdGM zg{!OB-Yn38=)u0#F6Mro7^;qF?VHT{17xiUuWh3bXZWsgyLxJWzx&OlGWqGAV zv<{6@WOKF5zLaU3P=HZqdt|xR54@rHVH4H2I`+tp$4WJz&v|)FxC70-uZj6y)O2q zQU$&3o{#(9zfiI!mHFkEwEI9*K~JQYn#{frmAXw_werf_t-!&!6e~QrWS+H%90Yw0 zYg4Efk>rJ4bfhmda><5QMkY1oaV{9r-RwJgn7rGFkYH5+9A&=VJc*KS^bUP zkEC!}zK6PlpH#bXu_ErQ!usU$D+okR9rA)H1$TGUoGIk#y>bQDlfYs3^@_MbI{L!I zoLkAx!`R0AZt_L*l$V+>7162e&RcBGfr_j+Jq>1X0W$oQGfiWXM=>=expbWw&+2pv zY*y3AEOe`W67*0t9VCA~Kt!bmN4e=>>q|;w97@9Hp)#dk>pIByiW``7&PR5iWsZ~R zXA0a$RK>M-Z!1Y#&YUl$yWQ7|7qI9H2*KEjY*vIiJD;h4esA21?AW#|`ivcpB5C~; z>B_FFE>PN~sa{Xu0RQB6!Fa6rVbQ%_n>CI^a3nDKk}Rln=snxu#CyJ^+~f1MvbA%s zrxAp`fx9wiMMgh^a}-=oPy}>J&r$>^jOk}MiNjZ#uk&ekmiXNkgOc?p`iBF}sqL+l zf19PUu{^S4;(j`&S(Equ=8yK{-{1+FkoJ@JU72kahOa6+}$VjC_W!*LS> zNe5APFmW*F0)>vH5{X9iHbA>BW>F)TrA9Mt-!9J&fw6x}4jgo5*ZT74OcMPTDvFE8 z`7120=x_(M?*9dl*iBIEA(Nwv><1UJ64lY8AA9H9SS-&sPW%p<&Q9(46qLbV$`9Cy z9p*12?^!{Scu0jj{>Iep0a_cXd>5u>K@X;|>jFuo5x>mz&k!9a*iMYK`!@+Gy4)?W zmKXva1Ygsr&-#gw3pP|qS!CPfdyT~q%^?&^_2UQi@KjG2?y=+{j+1RaQuZ>SC~Fun z#eh_mNQL0!stOWfG511A|7P^ExAo{E+-j;1%5H*tuHb#FkIa9=%ry?HxU$GPAV)QPF>xgkulk zmpadynWlEVjXQ;mF#x-im?54SPJG3aw}Z}{O*gOBzF*nwjlKGC*D#NMu2x3()ZTLO z?YOA_pfNx_fP>m-h*zSj+P+J)d2tqtmwwbgINA`e_eFslp)x%%q+dap` zXl|beW$f_m6n%+`-H~1N_w%*{)7>OI2RwG*pf@9l8PXd9p-g~mlWuLo!Zjpq2_509 zaHoR`+1IB*0Oh#x(i+}+i%C^ibKna@jKf3l=p6-uWF^{Ji-8oNr2{^ z3KZ>p0(X!aPj=`>T>yxg(q@GZW;ESbIk$$3ET|Hs!UZ?LtBHk_>EXV1Bw)z7aQxR} z<{@c4x4Mfo`$fRoT1Vm|u5X8qtUlJ{Fedw_7EtwQYa=hlzghFpBpvz z#kz(d3a1{C<#HRRiR2$9d}0pN+hr3280@#)Lz|_AbyXvfu)q~y<*)&Vl*x*cx98T9 z5|`zBxM|fCxX$?uJuySElYHp*TN!;Y@E$ENgBl#IHX;(B(2h6~iAoRy8v~%sk z`kUNKWig!S8+YXMW^fC>SaAc;pfB#D1V#K8`lGu`^ z@j^~B^%ku4whZ`I3dk7$!4yT5$v~qgKTobzO9us{rkTt_9mEVCCePl^wg=H|8Jk(4Zl7_JM5ZqarA_}axq zIxC&Wz4Q0B(>>mMiazg=)7biL>n_SavNhnSkop>~O7K4pGyl3N`ZwtK->LZjsq+56 z@+tpJ{L>+nQr~J+Pto0s6}aLTbkTN}!#QjQ&M8wq8Q1c~MsH17NoWn&(k5)TzVqAM zCY&>x0!SsG%@A076#~!`8B#!eK6V^k#2t613W-EzPn{p9n;;{D(02U@k-yHHK@tFb zTgmViMC%MT{-V0{K+Kz(exC+eg#)XBi1+~+AC?`U(Gzq4oi8{C1I3m@R@eCUNdI7* zC8Jls8(NPuT>6Yc+0bg!kbgZT6g7%!G}DBQK8_(QUIe7|bzI7c$=031O-84}1%Mi4 z=Z5c~3oD4x8$w+d+Bjj<7ijhWx>wr{8He=vgJC`@#_rdE2YB=RJYF>?57DZBoxfwd zQ|Si=0JM(Z!`X&70a}X3>lFPFSp|2@yHhjSQ{T5o0!bZH9b|%H?jAbxuPb=Za#yA7 zP?^PleZr0QO$JcRwk10?lBRISV))nK{JemVoI?t00&{d;bRmqEf^*-!_EG=W=~1ui zUnewz|9(GEO-$qc3|%P6f=;OTg8^&*`)={S?-ob-ZaLkg)MwwQH32^5>0fsXKv-ze#3bH&e*>R7(ic$ zm&v~$UyZg1FfPM(64=d+k#@(k%|^0?^uz!c{QYm~%@IurOiv@;O;LJPY~fr;N$>N< zY|sc$kB@-|fdvM9o;r(Oc9Gi<(A~>|2JsG@+7Arx;lF=>S6_hoAB?i&)IS(Xv5--( zOf~QhDCOVJ7f0D_ZB79Qi9J5h|Gqr;f$4>VOS;aWi4QUC(j1fktd2^ME8t1Rz?=as z>h}{&mklfshMTxNy~$K>&h3_W4MYXtcc9S?v|oT(85{{t3wXtr0I)Z1`80w9{Hk5K*eS-J*KNvt4#=vM| zNpz6u3RtT^?}SqGjsA9Sc)(vj5Ey?eB+aj`Ou~83pyTUd0ujBTZ%P4|4ykGG^VvWm zo6un*YeENa0dG?-jaX{aFH;=O{|}1m|Iki$$pC|1v=w}sTIcP{gU};GOwvqPVFmWk2b^C zWL#EQaFXDN&1m5#q0_Fh^pIG>X@)e1#vNCq-CGK-W51edtnu@O0xz%0%2$$&`}}sv zM*Zn}5-;PT`1>^gPE^O~Li5zR$d?ZJJeZ!PhcYKumSp&KfpoD;E!vv3lZgc$m1ZjD zZ5Gn`8dBfRz?L595eehnCspCs^HC)lir^}Gb-7Ucis5xex2jvWA3YBA;@F?UKbZtD z?3~1RhMpdE8NBQ^`)#+xLZ*#{_ehss`O~Rh=LK@h$v@Wl#-{#*4SD7zRW0Mvi$&;7 ztpXjRqXJD%B!i<8FE&AMU=D5c>h$0Pfu?EOI;tNGMb$Y=%#T6?3qDfjImoW<-0^7$ z{gmddc&!M<6M4#0?IP}_)sjls14@@ZPu~X?1EU@9>f3($Knx5Tr!3nGk;fh!eEks~ zN=)ThU4pk9cEgRxKvfrW;s(*9t(=|eLxr{1mD)@hB+W^*(H)BW0f(Pne{Ei_J&PA} zjS4D0IK$E)FnV=3X|A5@7X@R7@a;Mq6+B(=dL2|+y%1~*m+RYHxsvc`dhM*1%@pA4 z{RlBYlj6EhVk6*hnda*q^D6ePp7T0#-bvi~E_T#r4a|m&Q3UwxzXJ@Tj5K)6_Hl&M;{X-)$vV8SXGtR?t(Hq= zI2BLr7L&#j)aN&!oqV{si@`sVs#jO#r8ab0YYxbUsld4YlTGkv>Yp!j$PP0SGSnOs zS{4xBRCs6o#!N8hgg+<#aDN2Md?qp(~ zGTBzUU`ARy@ybJ-Jm5KL5>_40vVN0!BQaG4X^Vv^Ziyaq&YS~^4}+i3&7|XT`Lbm6 z^y%6fBYo`>RF~~|kfq!_e!-^!D2vZw_}xYeUCvbZ`s6(v%*c6!wS1Uf(uEFDSH>5$JBc) zefOp070;t3iL(uSx=n>EE<)ib;VQ;>&&ZKjCo|xk@~Jo*xRvwLZlcshtau(@#}n?e zOb~;iW!Jd#vy-e~IJ(jKilTlJ@s6`uQW3@fjJ8no<=|lFj3{e`%~H)zN!8*HQQu22 z64ok>?eQrrx{{@Aoj8$E_Qy2A=>c+9h2gd6!5bedp{l1nimn7W)_d(n9-=V~`zGlS z=xlcCs$$C&am*eobhai%BPmJP`0j?XZ_3t>EiCBnL+XTW^y8#oolN!7`(qo1_%DSU zI~(G1vskoG#fWfePv*oQQq!5o`FbfMU3OD-JE@tA`m&?=;_5{28#T2$CKQdA+fH&; zX4-y*B}GqTU1UFg@W6#Iw04*#-)Rs5hGoMl)?oPajWtAuNZW^?nZxn!{jaxtgUi#G zN0}j39u{t!;U;f)wh$e;mC5GaRC#OD=&_wj$aw1p&ACvgy34)VA7v9bhd~7^7gD0o z;>?hYqvnr$jFM|!0fR8Co#X!~0n`6j0)`+vOeDhF%cj#YlR|gkX{f+T%Uyp@0JQv4 zV73l5brmvvSzQ*7f8R#y&^5Ak<${f9YQMVVd{Y3+604sz)jk@(NlV8vHsPIdz#ojG z!I&b)(ET%~v8flYG@E)gJVjd$LzkmO5e$Jms%#O`!sUrHp$A6GFpA_1U;26B+d3Ce zK*P|>Peut*j6`L_>y;Nrotm*1j+A$MTTI`pTz^rwc~k{!nk7-Za?yIN%gQ0fM(~3bx zcG5RqjxOI>yXT&-PKLZFakc#GxX|T5i3w6_L&Vcv^TNGJ<+4{?-?{Ee1_FJyyP$(B zE$+xR1y%e4-a2ih-DsMRztxFiJL#q?Yh-P>lR*`S+C%)Z1S4^MRX%pw+eF%~Mou+p zMIk(Zh2?4k1QF(Xc}3X{4HK;{c%i;jqnGfpgR34J0%HzzdB97C)#0+o#j2MDvH*^^BsJraYr9JHhtI^)u^an0jM z*ngj8Hsy&Fna6o~qPCs3d(uc-oRDF8eQO$>W)Disn%}BT4EeENsyW_r>#44%3J4Pn zS6>A72O>b^O*5Pv_W9Ed-b72PjqhB>wtz_Y)Ra)xL*~HRlWnOc8N@gY3iHTA1{L4E z?9Eyms^%q~9rC)#C-;fhuBsND_4r^FC9(m2>t?Xx8R}(z9@BKmo7kk{tMeFBa6ijqEUX1F~b=p_bO}7Jrz%5|XidrLxEQsPX5w4r~dvnNgVa z6BT*U=qIWDd;`{=I37-$DFwCSEQ@xzx{Q7=^QpgUH=0|75y)9&bn{^v=tCW6h919t zR7=P+z}{Q|dg<|<(0FKIQu$F&`v68u(a%pQtXwN3I#FYa_udj=)HN+wmvF?73Hi1S z$W;vZXwc}UV9kE}98yzhfiHNhQYrcEVR27(HN?(G`wgGq%p|{JB-@CKuEqQdo zAGl6>Dxb<{x1ssu)UT7r_Ss8B<-3C>zIHtkqR`1~-Oi( zfi|DSW2*SevXdzei*mVD<8&KYMoT<)e7}Vs5?=YWNTiRgC9J-O$2yn{k8;&*|5g$E zgofG&wdLH};v8|I89>2|9w^_eGMOVvz8hB`%eARX9)anjV}^34k=8|7wN9hc8y%iB@D|70*e?#V!V1gx!!**km@U36|gLf@f&U7neJ7q-SW9uqBP z_G8ZGf!7ls7ynl!y9V4aP?Dp7gN&X(7n(rg7UO5`r^|hQ;_$$1dK$^-w|g$gpuy`u z7|6Aqhwkb^`vLNkm3k%|*3YIE6eKv`9IrlnM5cltKCR&~u5+mR{CTiVj8UM}!QRcg zbmMY<39KS8#Pi2j1H*jf+Fv2GM+O>R8H>t#>+*~b(wvMb$sRW7=6RGHVt@Q(G{Ma> zVS+eL0X&P07Pws>##hrB^q|0+a@$e>GAbjGZPsJ2gRzswD`3Qmc;`PB`}~>x$Kp8x z*`Z4%9$Y)|-T0pjLj@^yHOuj87hs8|MvvrS&lu09kpcy}OJ9%G zOjkJ}td8#BTTdx~vFHOSqx<`dvmBt*>V1r+E5H#(?UXBz-@NtWsi;T=dsDO<*(>NH z#z5kvYmZ1jfeMgOKV3ET`I_pMqG^8DOPP~7UKo!2^>Bwsqm`ccOyUzz%z%|BextIG zFkJw5lQL>=wnZn%o}PT}m>}CWjk{8kJq{a|OXwc3a44gCp2g7NrajE`1P zk$p;f5cFwb4S(N|CK{t2=ZM!dS8ViLurlXW1G!tzB#2Y1>GB+VS-j5apVniPCzY?Tv=wzv)y9F%^ zvie32M}jjThV_Rh)|N#Q{1>vvR{=$1QkIjb@huwuWpDnm*$}n4qX|nwC_#MRZGN1h zkh>Jai)89eC(oQC4xbEJMo|}pa$&22)%` z!JF}vt{vuc6UjI7vz@Qu7o7zpPPDnLSsOpS0;n^w;4Hf%RC?Ct3}iRkmjsS=jhk#X zAK1^sZ@Vu}c(cL(DZpCwXYL=1aW7$Uicew$?&Nwlm#e? zMbKh<1uuhW%+u6nZ_>@r4VRcqC(ggcs>)k$u>bzuv(Pl2oz=C>^HJ$Rj{ce$LGMjO z&G_A~WADqz=*F7rd*PV(lJ?YR(hoC6o8RQ)KCDf-_7mhZjuLyMb4T{7QOlu->iexi znXi{NZ>=j`E!#{33E@y?v2R8n^U|e$))5EY&2X-F*gZkX@vdr=ntWsw-v*(wnbLMM z-}0o=+ZLjgvaa6Lk6-aK{1WP&X3fK{Sve)|h5J|ex3L~15V9mhu@R06PaQFS&`I}j zMXlWnYG&zY=!IKnvYfpkOaHCGz*4-LLr~OYA47fHe2&Z38Sw%yJdf<1*27$;SE|B` zA%%F(b>_{tS*bQ$BHcSanwMT6U4MIz640v(H?Z@^;16xD&qZHb0 z&Qg_XNgwE=Ni|}x?rY`q$9xS?WvOUh^noq94mCbQ59i^d9G00M+~SGcvZ0OSUv@qMi}U>?;>$Ach(Sb+3|Zoli97A zdax>(&##Lz-Wp!=v?V&pSM*`g&-2pk1xS`587}d%aftb4+~oF!$^FhM?mdw357O3k z4`0IuFQ?yD(O!($??g#{m9gdCCxm90I^Gz$hJn;B$GbST_>6002V9m>OZv8>qu}I}q8$)SwQJiXeoaGO4?$fpw4@YyuFtTM6ap_691XFuCMaT!IgQ%d zIB4MQCSW+d>U_bg4k;e>_H~<6XQ*#_Mk{>6Xs%^FO*_}=BZy1z9KL%t!R*Nf(iGAy za&&C;sDHbbQ#D_uWf%K&q`WHDZW23rayh#+1S_AbZj z-DHos7eBBu%q7P|a;|Bv|N-!^yGpMRc= zfkBYh-26y9@`3s#}l$*5mwGP0@W9Uk9w z{K~nA88rR2w(ckmR!bZO*(pqKOE0q>SspysDDuJVT;bPvumnsC7e~^6c}Q?$4=iYiv0g z%oW%y@fJB$7b@k>8y%#r1P7Vone6kIv#OzF9mo8WWJIH4LwgbiI1spU1Mr7xMI}Cq zJrj#bLZr0~^;Y!+aYXHVXnmt53})f!LSJ3P{xU7x48w}XDl;G^kIDb=vG0_z&@#Wm z0qy2H34MoK7X}}{T0b=b6tY(F_vZG02?MYeGrI3a#fpH-dI1cLYH_zUv= zbc@Tyv|YriPZtxvV+iPf<$ACw_2qP}pY~TkoK4C~4tktdmI|cjE#tpBaemr5 zxJWDBh(@8;Vq6#7YqoFkL75|8cMuZ!{#J0~ZQ6jLPnzTeLmlFC^n4oYDB_*#g>AvN z1@SSbSDpb~;SlN)iBFSEu!7>l>^ilgBx|-c&Y1b%!_Px^l)knbJK|yc_!&uY2~yt$ z$nFmMky|c@EB<~^&Z;w!nwIlKh&HRgQMt^jgn00&gBOwFf=@z=Of{0y9$}j(zYb(b z5Q*7{nAz+!DV0tHG*Mvp8)|04Oqle<_7g$TBWf)}w36X0p(x4Mud0md(*@W52sQQpao>u?raw4acXB*jxM}lT5=E`Z z_6t)pjj%b4I9xc`JFqlq zIla-_zVig+4J4nFTOE@6O3HW|tLu~)Taq<&b>v=j2W&PE^;ps~$d9d2!@5PXm%xPRk9mr170f9`rU;b1n^-Edhs*{PNmA6FGbnvg$moh#yU9|h7t9h; zr@KZnJ?vXun!HQdC$s5L07|644m3}m7r&y6`y3a@bqr~Dx>_XuouYcf9X8` z9qBWN<3n6yC8<;;li^XO<&q`(Op50T2nY*h_%=1wp@t?LD!yYfv~F)GH74g@pQLnT z^K<;p2DkjWm4PH>BK_Ad#@tRa|6{rCKTn{R!K#qgAUY)BL$$njwI}LVs(qTpH}9(E z*UrWfKtwm=-oln%3D_zSL_Ixf19d&n44MWr9*}U7l~Csv7hsFKk8<0l)3!+Ks4R~- zk_dmq%()~hAR@qBmcT;5-;z$u5FD;pi(i%eW#MN7Y6EjUBxU!(+Z>E(R$A?v8&}u# z%;xK{A7EYRI^Cc_8pwGvHKGs86O*kXLHV#;383}O+-uZMmTolvs&VYowdIj)F6Dyg z_#rqUkZ1G$Kz`HN57$$3<%_ZnV${Uef}SvNhoP%w&d!BwoLUO`_KLv7Zorcpxr}RKaXDdujY>Oc929 z`EvCw;jt1!4awT+S&H&6uF)UlvmqNNcf`w0=J0?6`E)IPVeM4bVwm3 z0-e>=Ijg62Q+rpqVcajN8Y7>GfFcskZ=_JAlB1KPEp*9>cX;mGzFt2?W&-Az1u$*m zVd@{{FJLC7X!QVtex%d3p_c(KOPV`vm8T?z<<<+*4(pcT%jCK~DCjhVn}7?`a(~ew zgWRF!tF$ownG%bOT^h%NS4;xiJx^{R_nrX}M9oivOgi*!?qV2uoa6f4r; zL-V*>VmWs!WnnB=w)I#nVHp7Q7b;P}eHG&57C)diD+`JjOQH`((zZ}chqMj4A<5I? zC~2#*dg1ZsS%HQ{_bq0T@-#NE)7~bY`ZJXwMJyh|!k%IlbJg$E3tc;M;u;+3YUfJ} z#`iT@BT@+aAKPf;xWBhkZ3E3EjtQw&!1Gre*Xy+ftGV_?1b;l$Q2S2sHGMl8ZpIp{ zl7sZ>SUixjd#8^3pkEo|<71n%W|9Z$m=i9j#_u$H?21wLkz$d0$-y>?;VT&|)6*_V76P~I(+RU7MZk`=YU}C2U6*nwosmJjBPg|SA_oGoEX*93fvTQ7&TA5Zh4=o!* z>trwr56JGSJVf66l?O<@=I_A}dF8*envp3lGVsob`g34Zv5!69*b8R~dYRl>3cZM7 zwC$A1aDa7FG;C({eNssO{d_-uxjWsE_fiThyuHLEd$n6rThT`H`Cy7G)H{BHua!~- z!qEC#Z*D>Mx8!V}!?7P(*NT&4Yq8q8#*-^$1k+NrWcN#E>qBi!Pvvc%FwmKIElV6v z`Ge8UJA=@)mCB}_GIxYWi=&>3BxSe_kD@S$M|?~N=u-RxF&3e~5U2+(EAP_q8k;#9 z+*f$)4{*?1azi{3~!M zvxQB~8LXcg-v0oJ6W5ghC+H)#w4}VRN24B8ywA4UxiG&i-l4~3=ttg2*xsAOj?2hm z3T$_Ux&sa~!{V8%GeCvN3et?Z0#<^$f|1aCbNl&By-7TXXIW6pAq_n_FU4Ki{2)Z0u(mK(d)T%rB88`(mI~Eae)Tx z)o7sl2yliD`E)yf&1>Aza7+(zSKd1xAEMEO*|K6#eLpgxoVX0FSlXd~f0Yu>XsUsQ z*lVTtMsq?UQ>Q*GRVmjdsPK8)fF_KLGmG0y)Blcf+9iPWciXta&1dJec~@jsLm9SA{Vk}|@q96h z!B|Z4;n(Bjy(Z^+kcU@8E#0*j`Yd07QtYB|`WD5{rI}~^hIo0jy{o`EZAxo;W0Te7 zTvGf9H!Yq2Q&Y(w3_|3V<%{$FY_0X})Yv7ntHRS2)zTFTLj{iHv0=e23j&h)Zb#oH z1b8KOoe(2C-|0Xd>w4?n7Zs7BS-U8D0HUU!r)yA zQ7@5=^G$30?|2rHs1R2c_%-2WTX>{!168Jn{R4%W^O)?-wiez{1v}n&A-jmlTxLL2 z$uFP8U`{Q24sAOy3y(w9!H_h1m3fA*l0r5ba6IzpsI@RjP1UI`T-xu0zC ztKQiWj^eUrWoOH|71*!e)4tx?WSGd~%21gZL= z9u9AT%PEx`F1;wI!~PQ2B}2AXeTv*h>3KNk7Vj{QREy(&iLBZJHy)JaV7sB`a~D>PoN<->!clVOOuV!(ay!hQOh?$;-p2eM4fXTx%aTna?R}=uI87$y`nwh z0~ki%ep`vmuk5B6dKifL2b=LE=&rbv{hBOu_mZ=WrxKYeYS}Ws->R>6 z@n={SX}Bnkz;_R0zHwBAv%Z-Pd}J@KFSr!1?yHdYNmsDAq_UexZ#4u^IXB^5>0$~9 z=eFCD5E@H02su7p3?_Pe39coS@%$M;3osS zMqTRFq`vnDQX!R>tU$5}@|Tv_J;%2$Ous9|@9042dH8?CU#&qZJ^4DMR|M_x)fbQ~ zU@vniA`j^}#_ht8+?&k2L#X%4Oyc}ygven>4<~@a3$v*IvQ_ID54QuJdrJkQDoDQl z(xB;T9z8Z|u?=ytc;2A&pv_6Gi*7xoU_czMIpb(RCFPBy8(I6Q0fYMa23Dx=f}%s) z5vQIpjm`;OQ&;Phrg2%F-O!^0c< zlc(|2&r4Xb8<{Zir<`XyfT;F|J$FI6iM{HfUQ)tsDj{|!b*;9qsy9mL5G(1s$>aM; z6h0xvRWc`;j|==dIlWt+pR3;=EnQZ@?--JOUlet5z@y{Be%rhf9!}h35_;?ICHLSD z28iX!aKr$^lFnqg!|%G9tOb)8k}${j7<7w~KT1h1stgYKntMhT%wC+(@WfQNMLp`L zI;z`;*m@BEB$_7&eO38YLlN$@ydBKa(l5c{^6JI-P7H_5H(x{cIGqT z4yk*!t}m85#`N)}-YO)a#Z#y;tm6VcoL$lZ)uz0tio&%8o2$i{q{3XUVf>(*y*=@2 zj)?uwx+=fUN}h^a$qA6Xe%9nA6wS-7AC!OQExpF0i6;HD|%~bkw(Di~Jeex?YYEoB-CXj0`jGj~`h?hFu&e zsLJC7Ed@R9@@_&JjekyCi7`yx*C2d_Ex*7+9;OK4Lk-$(gE$nKTcd|Ijuhkf^&WBl zrjJ+AQs`0mnb(4}^kx(l?MN*Ybul)UU>WSRQy&Eh+o1K}YT2BJ8j6zagCqqNwbSfs zaR2~|_O3DK@Bz)^9{#C5WbJe*!oB%3D5eK zY^7Vbw-mk(iI+&u?>HRtq4wlj^Khq%`(pn6;ST4Dd^AxdLw)ql<}KUi83%uYD}$() z4u?fl81onEw#VZYfhVpTxWW_XU8qg&glDfOm=Q_mkgn8758Ckfhu8Vbo}(-d`z`Nd zddDZt(eyW1YM(0BVYj0!Ju@foQb#PZWd`dQo1;qkYfM#b3?5lL*C>n(%-oZ?sl89s zSC%jV-l2I1s4tOyW#j2T7!&3F{B8e>wzmvxtKIf~p|nV8p-5Y#KwD^Wx1xdK8r-G0 zyKA6=LvVL@iaWHW6nFO`2@WN}0wK?t&)VQo97Ax&*X)#OP8+R^NzonjV3# zH!&4J9Mx4>e2`q|8B`^SSsYU?w!0Yhz4Gsc*{N#fZw`7MXP|jbsXQ^JhFZC0LC%;ESo* zA0}0$NUML|#S44oi}6wjaWgr|l880-4WzT!f>nOW;+FKo7p=YqHKORNzQciu^Nc2g z!_$&wIXp0km32cx{s%DhNiSq$=(2_5(X?-k{GSUloRmmnH@cb~gRA{W^gyk?q4KMc zf&GCYB?TP@(u7mRhiFr8R7&A@N(uSXjk{iG(t=wke!@bX&-M;-B*1UCE$G?YAorYe zZSuLKsvGWU`)b_EqkSgqpMIWnRYQfAgI^)25kK}C(932gj)f!k^~_b>OFJT9?Oxn^H}b4}^*OG0@YZNJx-u(HLz;xX+y;)zw{}GtiHdYYt(x1<-n+1F95y z=n6^m1RY%W&J0|1VzlZ0#|MGDDCoiwvR`zwfbx;T)J5NkLJWIh>KGfyY(X7lRp1m; zv5l?)Xmp(cOeU!2|F5O~=l1`{t|b3!R~o_pd{=b;af1aHbR(vqeF2RF;j&b(05Ngw z;ef1Y>WIQv2gSyY2RLwd-&-XW-o8g)8QT;Y(4mff%gf3QPX@AH6#HSnOAPw44m1;q zpI}3F*}5wIL+z>|YXZBB)wCZ^13IvekT3=E_?@EcK)_zw?kuMQU(uV zeQfyYt?>BH6$;w@U--gaOAy~jj{*TxC6cDL!VPvh68qF?UGRKHyEkUcjho&kY$^CD zUcEWE#qc*cITjefTR7=W{svn#Y+&hT-NRvrc+7O{Sam@IS#mMLFkXG?|C2>bA8*qT^PNf{Y75{r4hjf(Hhv-oCLNY6>Y!u(0Ahn5|Jf=Yu@|Wy zAF?}l1x@pO$F|uxUUBM>swsR+6K$HKTs1IJja@$b6KO=g3stmp!Q-xjkq{_-;46%Nyttznq35B@%hz;p?gdlZ)v4)VxN68T>8@9 z*;c~1M=RCX>5JvFA}s%N94dL~C!#=0(@VFAFa_qz=r*Dw=HpXSN_uA29n$!1#CvuU zT8+CqtVfihtc^USO-z@Epo8*yozQjE0LE?l=1MsUNv1L5{HAHvDKNZx#K{Y%$-VzI zf4gq%Y?uyS0d9NofaKzkmI!2QuZ zzmdlPTYjID0_?C20y6?>P968|QUdwtO+mLvv$WAJ3N3deC*8O8N07-~8z9~g=R8li zQ^I@^Va2EeYHh=9!NuauK;Ux-jK0B|y+`+ZI;QKIH$?5ns!%f;J#*-Rz-ayy^ohYcW3n$lpA|^Nn#B{z zlM}JMYq2u#f1zF;b$Tjpi3|vge?t8jaMw{*l*q?1;$Py0W8_EpR>?sOeAYhBQ4-}0 zSL2$k^$8-~(4pl$cl_<5DIl)P10hqX=wLseuPdMnGubVWG)1-?=;@e-rG2(vF)iU< zxjQ(jQomz6>#6k@@1jNXEr(jfa}d`)@0PQ}nQNswFEDjk+pL6ki9WdpasfkYrI~(< zor5R#Cr9>5-24ouIR){1Ox(Zoo3F<_%bPu)65+2n-);iHk_*sWmF6rpr{Ld}D;6(O6H^KipRkRC1ji(Z#mL*5)ld)kq&C>1tpL$`&&iPinifRxuQK=6>^#byu;=8 zqM9`kz^~t>pUt6qUzkTb8sM9~=Kb>SH_`OKm(Kpcu1}M)HOz{2%_25(nXPsMUyHA1 z9^&9M`;>0ANBFqo4#j0QS&=@$78i54K25gTuwFdM_6zz8O9t}+rK!skWMtsr&2fd} zqM|%Zk+i6GdYm%2`W95UeaMwz7LVX8?%ieX>MKQg+I8GltC+nY%y4e!_%Y8=t->;c zYJW;QH}>4df_K#d@z*T?y zjF&6EBs!fweWW0Q^vgrTH22kmWgXO(#}3y<0?IPpB=iW^3_YqW^D4lz4@5F5F%Nq? z;$>a7aMR@R)-w!jcETqHIj^#ioPS~QV_I*1kAa(eRnstG!2i4_*yeJ3e+saUaL%g< zhL1L_$to1MGIgXfwssP7Z3n!S0akIx^l27478*h0Eb}Zq%U}06s;rDY|%j&diO>JxZYM>C1`gYSa3E;>a-RGO$+gm9Kw}(calWLS2Kmk%Z{pmkt|IGTC*_y|RDrH3;FG z=M3un$5!cjlqWE*brLFIj60$1OmT!_(mvI4@@v4txgosAc7bjtcX0achVG>)XZSJ& zY!$q!zmee;eaeEG>S|_dkTDajc~hfMQ?uhDzZK5s#dj6G)s>?Y7sh^iDa!RJ@<)zJPjOz@_yT7~)x{fBR}y4+sEW=VW_T*RxJ zdzRc8IIGu4aI@Z$g-hIn#fMMCy-qv;t@_oP&w}E1W8d}IMq!~YU(4EWnA3kVGdt(G z@c*o^yXT*@ru>ZIMDpbmaE~d8%^-IgXTZ@0gP*AFjY$&h+yFn`$o|m-ar*YUZjaQt zgXWQNh9gfhBrVIbB?EejP)1R{j|PQ;(BC$ z;O770Tm3_c=ilLmc1=;^(?tl?dq0ZKkMq!(ZKGl?Q)dCh<`AH#i+G(brv3HRE_eq(u+fj2WLkBqrF2>}Zj1v+6- z)h=qjcXiw7v*`Ku=;~cAH|%;#{z>Vs#d{xWpK<8B)#mNgA2W!O#7-uffx}TD5BI{% zJ&yT$<7zYMD-!%}Srb)y#;d5$cX3h`)H99)W8FAk>0G)SLr6sv1a&UcNc=he!lKp; zDbf;{-EK92ar3h}rm=Li_@H6|+2HRGHj|q}#%xEv@5P4^^0uh~`pgHjT7l|ct#JxQ zjYS_3tT-%Q#l^KYo?8${VjI0RQa`;Xl{S&|1c_Txm@4DLMs7ToVL_a^kkvldwjW<0Ngqva6v8={|Dyugm<-MkpL6{M0Lx(z(W&NKXLd>l!@l02w8Y{@Lq$+IKEAIu-$nqU!xQR_Qzz(}n(tKyPpGp9nfVTjbGNWqtis;L z@aEI#OHx#N>~IeD0&&r(3(oAa6dC%yIz~0J|x^&hXT%Gzh!t%41bQP^-3gz;3Wg}0Of%F$U`Vh8a z(}g5Xhu*HnRRW7}QG*hif&9vsa3NC%@mRsTM_ac$XX2|6A0Q$w z7^Mqb2l|ZPkre-hHLvDpkzPn6XmB5W#4vVc8~rnwxWrqz%}X6%Dca*9<7R%cPW0nG zKI+r2o(`}~XKRnih+A3)<&Mazan@fJoOv!Hh%7RlfG%aU60l|+e1t$qWdBOdnB+1- zAN*R2uJM;8NOCN-+|?$Mc31Mrqq+iL{-_K&7H1_LE-(rwG{}Y*!(q|0c=5Pk;ralUZEO)8QGThu{ z##3*ppW*Z@areE><*1RX`D(f`&`1*@PCM}Cw~Hi{f`clCxAkPX+Ph$J0T!PsaGmK7 zLT|4;LvKt8l5hHbJ;p=|1`bcstwRUobdMhdEB>8)9~%FH6#Qp;UqJHxU)FA@`a64t z10JGy{624{#gh@kZDjrSs}XJABJ^zLAF-Y1S2cI%96+u{45|5Q>Y{CUscxq_#MJ2Q z_SH@3e2xcRlj`>zf5LrgZQ9JbW0HmX@^OFJHSXvikX?H9uq1j-*WY9}NRP#j4I=N9 zt_;0=2hxf@L{!a~Dc(AJl+1eA_j~5@hDxwJuzh$Ua=M*v8P3AZoSNZA3h}km{Bymq zix^e@xSD=+E=Dii)jVByL0_69W#A;P(A90fUc}THmsq9nfx)|O6lEmKR*XwV zVL;tR>p)R>^{hN$WC2-kkWWD8x)n+V4i@J|a-nylHK8My+_5Ho3Sq;`L!+&tI~N7+ zpRAqz^V^dL9(9`soM+0cqrq-@9)mpya2GW1vEzH4b&5b zR7&hXNgaYasdWWL3NXE<#}*3nr$@z2pEMn!k9AZu)|EuIot^1hK4=hDS;zN_%3SKQ z3|9#wq3B)OFE#ph^TB#kUSsThS#*MiwnlZw^mmVXY$rFTH=DkuF0WInbSKw)5VMLR z>Xuy>E4v`bEL!az!_FL2IV6wiS+6a9<9yBZf>xdmqNXg(xpbj0k#qt4s3o`S5d|#| z5Adfa(KaDQjd8LKRH0l(aby?Xb~OW*NwmLqKTUX+biaiZWk2k|$wwa^cEWTd6 zn23GhZn0_g&&pFXT9vN>thbTjKd6Q`sj{?qJOSW89mpJTE@vEd?F|(9fpuaa4RQ3@ zW&h^{fb{r6oqPX}J9Txc^LKq8c^{Gr%Dgenfi(uqer=e2Vpvpc6>~ZEg_9uJn@;(}{`Yu%jH$P3x-C}2Qe_P3H&E%KIlEfRoCqy*74Q=I zp2%h;dBM34diylZayl^jq7@ofRK1#VKZ-PDl5C4tWrZ>{9V$Wn=l!R>Hg!iQ^OnzO z?)>`!F*~Deu?@dl30q<-%Yv^%s9>=Rnv~l1cr2$T$tcUuA?EZArf-VL&R+zDug&n7 z#fLR8KkzDR;5rXi%p~gej~ODxCuIY)v1(7YF}fz<&I;!loPIc4Y`dU7o~E{Nilpk} z)z;$p{nm9lSNI%q;s0XV{tdc`4mF_)K@|OFvQzOM&8ccFaa-Z`g8YMTdIzZl%8J{a zZS8U5+PKX)8!P->FMAsMCDSnPjfTWa{7b&y`@oF$FkL>&C-h`ce%w>h@_vemy6%a+ zN1EswTLdvsZp)6jcrKsGf1P1aDGY6cBkRYko*n;*{&5E8SRQ5HwshLVCO@#tIzq9+ zv)#UuMHY2%E4{;#BTb7>yjK`%qP&Jhi;BX%SYNj?T8{+IG*r+|9+$1~R48s4>eJi& z8`VThoy1{W2RAl%(x~qOy9%@sde}z7NBWy2-`-hB0Ew%oK7OVXL%X+9x|G%t4A&Ov zFHmKWO!0t5bdG#e9{=iN`cY0NH&Gp3Y)p%^Tb_I$6;Xr2SM&E5FX4h9uj9*vRTvL{W<9QHw3HPe$>CmGSaq?DO zKI;RCEf${%`6=&U(bq+^LBe?Ha|aDp^`64F-F=lo`K#-P$d)kn*WuFjEl#?J0%xwI zreWMux2sg;@;} z2Hm{q!`y}W;URu(kBjRfO*4d2qDSWce2ww4V+>}!!t$sLFprxAu@uaDiHJOwy%b~l z0k?|V)@qNl`_L?V$XEZZj+NyKzoZM7<rz%kHQp6Z zi+PlSJ1qQDq?**k=aFmK9{1p61f8)D_A6?c+i= zx;f?0DfLK^k%A3C)PvMuf~gH^-XKYtApY@3_=<3L#yjjZkAvLtME=Rz7V&|J1wlb! zmwC^OV)1I_$=#YfJ#yPbG1!^2}(-)GUA_Kv%>h+Jfi z+WrqP*Be|IgdD{|$-?@X$!BP;>;YO#CC_-MwCJJ?4ja z|4C8N9prU^1o$2Hm`LeW%YIH?jp9q$%++Rq(c3jU9*0yPnU<_fI)ms}xJ0a}WjYD& zmVyGxSegh7QcB$usXO%3!hss&>)H&F^6Su*UDLTNli{>eysY+V;ir_E34TfC_h-QsF9xiy5-z>PVwFl&)WWr;#ilp9b&RZIURfAkn;P-D?GO# z=J2PoL$Fr%98-IXG>Vru^04U{!U@!%KG2GprcIY-AXA!BwBl|oqLa9!!|l)d{dk*I zAzf>-@d?rKW>(`8BcJ4;>NbS?M$j*C>9sQmG)=4Jc`M6knEXTL$5aWDs`U0Y3MDfZeYE(pA!wJNKvc(Q(Ptv;KX+JwxD*$M z=U(6(tUZ4Yq-6Jc9XD*A5K<|%j{`PVzxT1LQr+%Ph#&*3LA<8A=Sc98%y%+@XvhyfD1t@N?(fuzp$3MBE)0!XmW>(qJ7{w+>nGTo z!oZy0F6Yx6Tq}=i2X`|=2g*p>Dr@43YB_TG&We_rK8-p6S=?GB2E_34%^KmCo?u6* z;JqNDU%}77F?pJS+MKh!oP5U?2EZZFO-e($d}GrdB(QGh{8|C-!7$zc)-CFMRz?08M}(m#V)6bYvD4OW47m7x1k-Gm4_L_fNIM{g^YmdwP6 zK}EftD1>U~Sz5TG+eqNNguTRWU4z7Q5 z4+0s7;Gd3mb7I`y112%0vrLZ*+h)>BZztl3pMMOYnr1iPN%(cWHTL~90gILXh$J3e zHn2z7X}w^3(;mUh2kOX4b`j3iwofhYlyMjpxfy^nt z_!>AI!dS}RPXqEEL1V$3o8jS>Iy;})=O*uRJA=gehVXUyNtb`jDu3ixb=M`k^J*+vlN=L{OORh4EK#DK#E-+$8{E!5*a4 za_?t+I7wY+jvABQf{R+#m*DXiA*t5PbCLNGOnK23NPdn&w znm?*8O&ab9O3h}ksRE@lv_wxf@Z#R5HzK7Oq``YzW|1Rj^>gPJgB<=f;T084d5K?wxE11(K%`ypCsHrZc&`W5B zCu*cB?m$dE`kPm3cWag}A!G9`bo*@T zFfU2i4?Qs5RO2$rD;Kca*;`|5WU2x-F?6k>h~jw3B;SIKh5fA4I!!-Qj{Eukowzfn z<9--BHVKVIu>D zghIGQ6>S2UZ_Z;q_KnQIRvZZ1@^}vAZSLFQ0-Du_LMSJU7dyK`&3`zzFvQvE{j&Z>0F1}y_f z-q6S3Z^=K5L$@N%l92)Rlp{$?5<)!i64DZG3Kk`2E!jC?6M+f`w z$6BC9YL~F+TROF7U)9K4waK(uY0exY@YtqR|3rH2Kn!uTW&U<)OiWYw*bPlmn8nMy z*`2aJD3-3!JBzQI$?48<5T=^w4aGG#?Rd3Q>(gfJSCN0 zYHoJRHM{Q>qYA7u6l&}YwB^32DnFDwIZu>PNW3Ae5FQE z+iPL)Fz>|KolHbbu7MuS_7AHBYpEJ%_1;&%)V5bZ?G$m{fUsM9fB>$>bK1c_+f!7! zoV7Q&6P~q9^Vv8qB=`i}bd-D+an8boXS9WIU+PN}1akS~vIQ#lcKx+J+I9PkgUmL2zz`st(wlkujJYkHI38zR+O4EMk7wrbwx z%pWw~la8bBc3ErnIa2IesmL)!zMnq%tx(;QWI^s9l>9l`bC$uspR0vjR*PM?d}VL! z5~wB*@IV$Z;kok`6i3=5W6-sfdGw)^K#k8~b8@S$q2|;D3FXOY2Gg?wt;w9#vy!J0 zBFgF!ctbN*^KGVjNSmPm4Rbid}_qcXcfDQ+qi=w~_#> zrLmuWZ5pI!0&qV_rogS!Z>sLx;tIFYby(CS37qH4~^1Q3__r=tB{ubG-jwcn|)&gJOjf4LUs9#idPy#M`~Z4iA% znOR9KDeJ1KLa?-7*6;`_5(d=r$*0U)k+?YmWnN|w@;yY5p58MLQ5O7ris%P=A4RO6 zz1zti*LSb18!)+2KNkJ$iHuR%c#O*0KiRo~m;%Gopz^8SdIiGGg}+Z%UKT-6g;){Dl?e@3E2z)MhC90Yd#YCRb$GyxfJ9Xngo|H6>ridQVC5rDMjiWWkXtet6jA!$-2_F`Oh zhi-cE;OwAbjH#20a;`|0kh7QFiG_aQtAu4r=S;z^ANf29CX1kyro4m2zp!QoFw9Wk zq1>51|NHrf`AjThLaU5XsvD|vY^TmyW5Ny%l5Q{RRKYqdF<{N?g{I{rO4-zR2w6Gg zyOx+x(Fr$B>SL(gzQ^Mi+*Y4qsIIp~OoMYH`EP*VMo74jZb-~+knj1kr+`Lk@pxGi zs6+hhs%QJi6m9nnrF1tGEb8|cmNBgvy1z43kl4xLWs59z7>n&s&qdEMsLk&`O)E(F zRd-1RW#jQ+(Eb8!y*!g~cOsFKD*BPToNa&Lo1GBdZ#Nl~MqgELvh0T+(oS9Qb0ti|!voYUe@So1*V{3g(u8Ix2J8YfG z;9jfFuN@#a?eF6K*d`gk*rR|8wrFqMYGy#{8e8GLE0l3~^wj=HTy=rRuZ%}D}Tj^+?V2N*9lXP7AxZ>-v6Gl~> zj1`wsXczR%4PBZ)bik1&Vq3kPPgLS?!?6<2KS(}eE*zK}E9mDxKf7isjZHR4qZe>B z26m(s?uosA^PQM?>_R;EXEK&f&dn}s{Nk&h@nwmSajCR$+4fAD676f#HgWL*0h$Ef z7y02x&5j(R?yU80UXEgW0jGj zI8`;5pTZGkJ^k2|APKzGrjbK4m-3qIOLwXpU1CD|vy-P|kBQyVg>&ArOTXj!rYbg# z1x*M%)i3C)4F8fxkxhi=eer&PUYZ6ySAgzCJS2uHOfKFI*m@XZ!fr({>h*tPYMy^# z?a4JjU}VJTD(Ha>V>~)Uf?j$O+V*jB0R~`ZE{zZv^?y=kG(f4D$+s6~1kor*5XXZK z`T#RK&0lvuaQFDqNlZ?K6O& zsht7Ju9y-@6a*FQkAg^n&K(>8)vs+}i-P|}(!Bymx^Mtz9=Kp39yuL3!I%KKfPK;h zjNSj|!VnnoKkmdNFD);v%nb&pozqn;1ET=|PNSd)kOD{!G~K7aNxQ@M2BQnjfN0n4 zPQD7n;77007I!%yHOvL10UX!o^fGjUGW?atma zUiR)^`HhNpf3ozq;uP=hKbqa=z1G{MU%PBF3nzi52{!IWB-}JG6;@U=;e99)@=anr zg~JYPwE{BJ(c0`^`B2uQp4y9#aYj%`=_azw02(To@`Y^kL^`mMr`>`{{qafS85hgSU zDEH$-a7gC0Q~T)pAYD(y3CE0*4T&FJ0vo4idefn8K|HEaU&g`ERb0eS@LprQ4>8L^ zJMJwufhQ=y_{_Q$y%SW44v!cY3nd$5-`k{ZzFfFY`c`?dyQ=x2CvLMzRB~6FGMu zp(H59D&9l)D*8V-eBIMtU=DUz%4(igW6m_lG+KChh%8l;i#q+Vo@R1^J+zRs$SB52 z+P$tj)gklHeqE!Zv$Nq77~R)A^Y)Wmb4Q9rlOIRkWU7M?of8FGEp;Vhbp`;P{FJfY z7%pA;(1YVQ6AFHURGcn7tb38rUS#xgSOK57Sx>jXdcX|F%eJnzY=TW4Cc4vSbEk#N?6mz%BZ+Z|(Dp?<)<+OdvQ)1vrG1)>IGY_5-PpQg z7kY5+0wU;8>15p;w!H9u>Pc&nRVgc_&C0@l zY1}|p;Aw^iG8~@*ztwxH3u7G{u;U;$N>QZt7CZ2WqYJv12M&0tfi2ZPyX5P+AA!HW zipO4XTHvRtAnXljJ&OD@-nqU~mTXI(z7PQ*T3a3|JY{uTmQOI3EI%l|-oA=l@1cC@ zG>|)}bSi3_NprfFqxwhE`>6?Z@HOI=K5uE`NAr+`-q**$X|%rIrCW1Fn;uShl)pF2 zCsIqL!BeC%BSo;|DTsf}fF?tP14Ez1`W12~FApBFZb%Pl|7LAyHOY9lQxoAOJYj>} zNp}AwWu~PTY>(cY1ab8gxxhLdiue-0UZ>n~Q#~GOd{DsH%!7Hz5Z@=WL^@5X99nTr1KU!73QZ~i*xTN3}26K=WR#TF=+-E$I`*UT@@p-M6g z3ZOtlvGEXwEfFS`rmWHjn!cf{W1m;DM>W}Q%T9`Wx3b#sR9rB$p2t+E#LM>*ZPzUg zpYy!1XdfMS6#p?@MEk*o$>D6S$oSs8#IZ)@F)3%?$vYeNWw{>%OsKNP?SnVDhWAoK z;$`#_LBm27Kbdf;B8|q%<-uI|ufPz<@o1a7U4oR7j3_!#tkWvr)1kY9dEUzFbG#yA z5nJUjoTB?S3m^O326mDSPOGMW9hJZye6;P+2jv#`-gMw`O&ra@dIeshv$Wb0>Bw60 zBni}ZiKko2XrrI|HS$(r$C4H9@ILS@FgndCD{f?kixyyc7A`}(7E>d5IW6ZSygseTXWKH%ipSNgLG9wxwARpf{(;FOPWrjn6gjg@dkumWI1jr9)%|c z2b(VL-6(eyUq(Z`9Dm8z*kCPp5(9*q`_op|$*b}3`Z%U|thaJiIZ;AyVovV+Uv;?* z>t5;jzgYL4ckx3-kILFSk8Lfm!3oA_1h|`h*Yc#Cd*SJj@ivn&m3{`zz1rn5R%HcB z{DAaRr3$O7S*yu;EUjiJO{<{oKQHI@-lu##@pYw`ef03en;o%l9?0D*9@7yBr&JAU zdn8wbH-q0rQ@O3vqhIIe6_~uih27oy-3)^&S=+wTX}6AoLI2f z$pe;E)&3~e;&*$Kn_z**eFw#7idCGPUqX+QB1HKN>W4pD1lRSpq17>lh?a~SL&dqS zp&**ea4!azqUKTd@hKy03Vi>U@p1ApT8Y0kxsfFJnfDWg1kFdSgG5|&M%K}_7^@yc zZ@LpY#h(p^lr6xGw}DsP7kKj1nWNU6rljak?RWCo-3)uzSPD@c8FauM*l?NB?ZDl9 z3rm({3p5WqqjBg)nC?R1#Az2kP(&|cbE=p9+;F4X=m3i=$VDT@-}*q)cr)DVi@zH3 zgtz!8mVA?#OmJR~6CU?zWBtHONYydTsNIKnKsTxBHDSg#*Quc=x-pKn1NKhaSva&T zwXst}KU89GN|(+3-TZkwrV4#Wa3qF@wP?Eh=>i7^1+Uwmq(rHD5?(bF(L?i;v)RniTD)=`Us<{`j$=*D|z?9>~zU{uz#5*&1_e z@hY-^RAR*SQp;KJM%+)^)2OhYocnasroVan495(VRqL>$aNuUP{NH+ zbP1Hw*WKSQ{_eO5?x~t>T&%@L9Igm&)R4b-d*$u>n#9Da=`n4)v(R(j<3$J;$zXw2 z(EaSNOzj_Ho4LM=S3O_Rd9iQNrHGQva)33?P2;qMxwT)`@Ma&M`6}V zweY7No`@2yc^H$dZmD`JCda25V5QizhDD4(oI{JqGslW)uV;{^j2s2P!440J* z=Rc1tzWw@Vo z8&Aew$(x^Ufh2LNvOA9$1&O}jdp;(DAfFk6XK{^47dHdcn}A~gt-El<20LbJ%RBis z7hU-JMc47V{y+D<{&mD|2ZXwz=K+p$zLh_xa3@i^v;wcReN0)<(dqTTH%vH8FwJDu zFRK4U@kU)WX3U_X>^-L^6pBSrghAzrr5xi?k!Ug9$3BF_!sw?N?&yDFN z#5kV3Q`)P#C(ZqU|CNlaM}|s*QFjiAd(>+%IuTQh&VUz#V?pZa@;oEVR1z_{8DOQ1M5*V+C!G$uL$4 z|C)E#))E)an^xZ}G@!01AD+j=Lmt!#Wd|DFbJ;gvdMb*?G%QKK#1G$(vYieuD6#do z=IDL#jTFiayy4lHvDq!2Y-_Bqa&7AaYwN$zOQftpqgO7WnJZ86B8)xz=_m8k&yLGZfp+3D zZPdfN5bvj}&H=mYJlA?v=e-w<{uTZbHj1T!HfYH>K6%72 zFD_9a_Q2;MINXu)wH2L!?+#u&B_3$wCq>=7P;37pKSc9Uestlo{3?$a=Xh(bQSP$- zRLQh+(R&7wCTHeRI{On*Ufn#&E#n}`)%^ge5W*dk&7EFhJO7f*LFd=hQ$p{a!j4og z#&DH+K39UJUQA^NF^D}^4G`&Rpx25DkzBlc$2^c7aWA7Zp=duvA^+h8&TLF*Jq;0{ zs_@XOy>Y_T&(pYC&W5HUQ$RZL{qt$nPyS?KuPP~3raFiL0SJ)`6nY93TPH=vvuL*- zmSRSCRIN^|V{-|I1Bbk9UN}Ykp=DnC)MaV@(*3L87BV|R!^P3FGUoWo@*^QCcD%!} zFn7my!uIhkN;^vOuq4wA9Q@Ru42%^;%3*mbyv8+)SjM^8S6<4^4)h*xrdf2T)AT4S z+SG(-Zf8T&7sp6;c4)S*4{i(*GRc_tC>=SLvYEE>eF;vgrngy6nQ=>B4)75~w9F&} zr*-w&Lmtk9rEm3dG|jGNt#O&CU195mR-&z|RMT0ph?DznC09ugu61`Dy509A=8vVX zxMOt$duG|Hr%UxuUPnsQqO+~O0s5DdJTVbLoi_LOZ}@B4RvF~;14G7{hC|B0yaPPP zYYwjpudP1n{D{{yDwdeU`c!3Q^O8tbl%X0y4ChQTMS|oJ{)akHxYCrzMG*fRos4c) zJ1a;CsmxP?4MCc4$_Y_R?9J!BsxoEG=bB-u8n4>}P~pA1u76JcX(#%>4!prYC=@8| z6g>izB90$~ZqU#){kbyZVfs*4z_^6^Cr>IuBqn7ygD%oWHoQp zEIw#WdL4s{cmwr@@q*S7PvVo6_5e)_fj+;Q2^)rQ!-cP2d^rAEfZBb&+@>rcb_y;Q zl^AHh2&yiikBEhm^j?fxTLe3k7q$Bh?0P4`mOTBMT}BC`v%qhbp&AF{gXObcv|G!Q zf* z4?7LE$uP@|Me{5#J4h~rO@=9{EHonBL(Q!$CP`;kDpq(U)|1e$s9{auSEy}CWGp&y zsBrnlKQb2-`_o-waJ0|=jA3Dg=j(W6E16~pn3CpH19|afZM?hKFKGz5e|FePCh0pl zdbZN%9mb`y%CSR`D!)g=s_oc<Z6`_ohcf_r$JT^Fs7BzAD6oKM3wVN)6bN~WW%d0Z&X z@sTAYUXk)N6&$^y<8$`6tm0pBtyBq zyVJyl+z)#bbv?!Wyly|clN*1_imez++ck(8h_RLS6Y>yb4w!~lM^UHvl29@(&Vm1o z*gn9s9R9UAYTD`Y)mENH$`dOOM|og~iakE1fs=5?d^KwW)4ogI4T|8;JxI+mB%!Ye4M-Du8hUn?rMQ>cLfI7?sx&i$Qo!g$I zg8nuIn=#LZRvx@iZqZDs5u^TGNd~CnUyeu97STeq-az8uW8(SR5Hd*W)f4`(8V~aP zf>y6B$<->>g+D^!)zET0m(m38o^$w(Ueq|@jl-er$Z~k_I%T9GlnZX2BD;PCi#Rbu zrz5NrEZ=wc=T**@(`Gj+G2{Jc9+UlgW^V(NIBHGG0Cz1mz38ZjM=(APszbE_4@nH-hW2WTt@nK0qykE6PaSuaV~u+GLp9g5kqp+ zkxAf}=@|Xh{mcz`X{ardwQ$s$Bq#EGHf8Y^MpLNFHzTfMPFl2c%d&Me#PtQ#8H2?e zw!OtC5i;RPbA2h(zuU2!M+AwfL*_WF?wneZEfc<_B0Jndtm>;X9;`K6Up;)R%K zc-%j=vHv=a2h7pilLq|=I5R&%;KDm&VwvfaX2UP>ljBb0 z6a^J`sqR9MB%_r}HfZEe@!E)4*t_%XkRT0`KxWiTCBW*Tzqhx}<&w{yI~Y1K7_iby zI;Ym(d&W#!e$YRn+b*=X{O%XU7Z>Z1F{^tTeBNu;U*3MgkVFrQ+cu;d*r%BqWa-SD z%rok=R=u}W(B5YFD6y@0yqQt-va_o0lQb}JN4dF<$4>Xg+^)W2s;kryH^;_Jn13am zR8_e@HB8R?*3ylxfr&@$rH6E(M6LA^s!ZGvsnU~I;NTw=qXL0}*jEgLqbB)jdB9$W zT)g&=@}J31YG*tN{rTl=#Dd=Bij2)MYB6hzxh$D~C95h&5vgMqaM2o2H5(26Rhu7V zz7cz`$^LyqY}@4JXu@zN!UF5w10twC^7Xo*pTdPHV>K@-!r;fHfUezvTxEV-!*@Z+ z+r3acw=*n%I7cb%OX`{EFj-#i*Fc)>@nu=MU(qm^AxdxA4#7T_EieY}eedDnjS0rDOZK2$~xVG>s3dkdVe&(_!nPvge=t)dHz&Hg#N*F^^Vt3D zzC7mowl&sw5-^@yH})aod2TNS$eIPheQMGt6dH&DYj`huSP8hY5qu9dFFO8zDm&}1 zsQRuCg9sxaAV^3^BOTH)G)PHGr=)auij;ul(A_hXNH?g&&^dHTcMdf$;5*Mx?^^F# z_p|QzhjadcbIxAh^V!$FcJpSF9-kviXmcOW)Um1GuaH^|K+_d?Iw@t5Z-Ua(UoY(> zx%VlW8%DEnnH5>)oCTlY$mOx(TT{+um|mwSdTQpR=K{}*+%?-fvTRSWd``b&_2!wt z&cFz+%fkjDxr5)i1Ttbgh2)@8m>SVWbn)8VGSzM9eAj=1oUf?Mu>*B;n_+|trIe%{ zUU(QA{P{dQxHWZZV+eI}-E@!Zy>s?|0Ck&ot5#DK2sh<8EGziB-%Dz!pA0Ssb)8O8 zGk8^Dd$Lz6Gpp;x*1i6<5ew~(mIMxjOZsAT5JAZ1R}E{8B)Rq|EUWR#qCZixRFmW} zd6;#JI4%g+jmf6|7+@pBX(s(KVEgWHBDl>w{PS8Tl$VKi##6;1f@#y#BtmpY8<#F6 zlFn77GC{H2C2)@{382#Q`RK5g6f>9~? zyRe&T_lg3JuzWc(I5^kO(`zMqMDuYZX{m=}bX_ahU|jB5T;4}YhOJxKNB1STA za{6*FmH+l-B0r5ZXVb2bh?hd+erXoZ891ju!9h&p1hoAXZ0s1Lu?q^)-PL`vZ1 zmFw$=lY~WoWXdDWN!S%GpBc+Pr*!`w=G$CD+?gDYp+peEbCy1Kbn(8(xia2HyD7?g znc!~&Q~ld&r#}cz9|4y@l?%QPsD-x;O)##Z?A9U|GYHyLR86FXgXM3#W&0?6lRVsQ z1NtV^`%yI8aPOt*OoFRtUkAC3s%7R8d#{`La`pNlDsk9(uM1X-cV*-*bhlE`E-L`r|)w{H4C$(9T)nzPK7faXN1ycW7?6)9pW zV@{$`zv{S$g5cjD7ezXHLiWRB=1tUkCM+4<`Y2x^7=B=;Bq}m{tmoAbYEqvrUSR1u zdLe_34}=EAj78riEz(1&Trs4n+#uN9a1Fg3H7bk4j&i`CQQ&UbW85^bcDpik-$Z>| zu#E@Fe3S4t7wdAPkC=)tiSi2IExf~MZCsU*thxPQDqPqM^CBOOY_DTf@TzAOp zP=+R}-s!lEOKvM9wn?K);CUlcf7uX+jDS3d@JzIHKR~6laRRJROeJg!m<|bP+ddfsDg+ zNFo~mmax1qfzHarcYtnC-BFeC2TCW$-H}1QM;F`XZHlL@b>3E`R7HZL#~WfhmGW%U z@2M@!3YQ$r@AvX_jGV~CeqqUi2 zIjwWB7zov;NS2~r<6vxQ#jLC5+HQrA;H_pOFY3uOAyIHbRa0X)R}PQ1UHrtn5HEMjJ;e36ez^HjQDc*btMku`@=ZB@aCL1Z@mg%ahaQV6gBe=&U0=rQ{tK6Ds-w!+OE>2Ut zGp=5o21o^;k)`)I+=)H9FJ5E^SrC8H5G86rc#rDYWltbYdlujy{SsQbCL2>q8~D{c zHS4CoyV{T%SKSi@uKFPQ=eUdQCG|iDb=mBxb}L`C6{1t;e=KXYf9P_$2dLo}iLD+l zF#Fp*5$Oh=yz1@NYJN_yqmw(hbJ`*w7@UywbiFPs3nnoHTAQ zi78D0ld;uRvK6mZv)yxvIo?ao!#u^!aqur1ZzFCuuV1Xn%&Bvy-(o*g%jKk{{-id4 zgr+y{IW@!YF!tznY9|XSTJ*thY8ye$6hv)-$@EZ43SA*WQU>MvUD zHuqNd+knLc+k2Cp zYz{gcq$6vjJqnrmkQd-iP>IcgQ^};{aW)C@E){xZp_0^7zaYtkY$nzmuUL_x@YdG0 z5an}*y{^wV&ehBbmz};=%dm_h;x_%Fg35t$ZEdR@8@ABfFS0Zg3r?jIr|&%Ln3E)3 zonW{yIY6yG16pmfvZG=l@luoR4cIL)-kgL*AB9OW*){3L2PUO~jvAJ6oZ#1kVa{Co z9~h^1*@{XqmgDYx20v7cW0@7L5UfD%u=m8r_+P)Nq!A1%8Yzy$UkYgiK`H%EF#C|D zl!FYO2e0_p^FGQzaTM-0jj-*$L`Br8`5C%DW~WEW*f|wz7K98{qF*$(Mnoa;ojeWm z@VNASFfJq;7q3N_8O?X^LyUNdNE-=eT9Y3oNw4k?vpCF`4wyY8E{W@iWb(3|Zco$Y zu;LlPnp8S*g8Z~zYqcXCp0ie$r-*(N0c*HmdBs@aDy3%}`w~dcC6`ZOiM%4p`{=4C z3Js$?2-li^=!VQ#W@2I|lD9p{6Tu(_7|Mr(RvoD=UY7vV=Lios0;{@C}rJBf?*T5 zT-OT{64M{nt2LwSb;$M7t?7_46@($9=3Y2#Zc763!5f%4;~6A{P=-ZS0y%kD>Tf_GRB>%(986Gk zxzGNIfMV_|Vi!j~-`86N3PJpE8Lf3*)GfvPgT>mgG_Z*`p-9K|`7c_PZf4^V>#Vu) zsE{&Q_zi|N;;>>~-mV&222XMitdZvvxPj#hFNo8`%nR3##;}Gzr%_a#w zaw$=WXvlAzYb(w2S{r@d2|g&7XI+xL72g)>-1#Xry0$Tz;!oU`=}`MbqTrhhKgv_> z=N60i8dF7AekRJn6>l)s^;Ia8XZ8ldkB@lzyU?RsP-9p2i}$8Yuz>9CkDbj=ARNTkG*~x*+ft!&vm0SsMJ~+peKjdBG@>d6ec6d}0|w_AJPoJkpd?$NT-%)7 z>_1>4`clyqQ$h$K#?gw(n(&r*@GztGhXenjQSbVq(fACKmr|bcJEYACxo}SIRKQy~ zPP9TrUc#or0*ir>EH9{Xxfnz~?2og~e@_KJg-~hpM-|j~Tz-0)&P>=`4@H0Vxj%%n z2T?gjv6h3WTD;=g>@hw4rM3{T4BLSk2U79$%~stHS5KK5*X64Oa8|jU%cDbrC#eDa z;YTRny;0U1`>@`#d;TT%HtP|?C2wGaA~7Y#+;^;(PeAwNI8$L3LNsbdy@&3<7X(w? zM+Ve}H#gz0u)(U3unB&#Nk)77XRj|tcz57!Rh|a9VQCuoASqz5LGFvgH}F%$KtAIl zqIr;%V&&(i*VeHXr#*H^G8X~n_=IILcspsB@0oIQty$^#(fco=p?xroo6Ep<7ese~ z@ruf)fmcL6idI+=DO!T_F5a%I`%~|hsfN{LRmpvkrR7bp(n>rPR!%ay3bZ_s z1mM0=BUFlmMWyIc*dqZ>P1O=}YniW#*GA`da*wWQ5IvkS&q8*Y0}CzMmzUpEq9(bM zv#h7}dLSb$Mk6T0cA6PMhomO2OU^%U7y9?^u&o3HVl~2Vha1s<=p@?b_dakK+{*Zw zM2<`fblhbS%hT`Je)`Gn{DA$A`BFj@b-?1Bs0nNis@hLK(O=HUVr8w>!74rC^0HE2iCfg)?cdnR4)@ugI}gO}Q+#%vTC zr?%=hDI97*Im*5AnQ<@&K>#PvxzGh;o`BiM3zpU33QTP*JoOREP!h>jYqaz8wO$)j zjwS7!_lufzvXwI((+Hofe$AZ+qW)F*uzz@7F3H#F<3$}_K6*~8=-g~85{=DuV&P9T zYA}bL&Amom>gXWZV^mti9OZX#Z&n8)>i!gE;7&f1DR0ZY6c%-$n2_bm|Ml!f;EK=S%-!52q# zIrsdn$C=Q)h|3&{{DF_sO1amW4m;v6H7}00@ye?0#8cWGF?*fJC8Y&^Nh&_|v=1L4 zc(Fs1u{l`o#55KKp*1m5Y|ZNbGZmFX^lCJaO`B>lr^J2hdUAlE9o$Dd>*%$@oPzrJ zAX55tVqQnQNFeVHlm$)8WLt7<9B4~z-7tLu_}_Wr|0hc`0bj+CD(^Wv?P? z|LgN-=T99ngYKEPx2w3f`(WfQfxm+HIahvj|EwtBi;%1vw{vl-MKzQDC zJ+GU&5^D#;PI87J#zJ>g#N`^$@BVM6`)Or>Va*~VH^Admp!cd-9Xmt*)EqeHNRhMm zaq#T>GidDc#@47cNm)spr9|207nVQZ!Wx9+JChe1TZvm^lDp5Gxm+Z7c>{|%VszhP z)&A5gdW>yDOFGS@)ixn;$v;yHlsJ2E8R&BVXJr51qg?-?P^l|CLk939j$(QPC(BA2 zNNy=8D;}#DXsE~jqAd)%9&rDs2x%iD;iTDIZHad06+Mj7VQ)zUUV9&>Y=yp9*=sl4 zrc{*_r%P4`BHV8$r&!5V5-y@-kDKhSXqE^3#AvKb4T9CJF4lSNCV0Tq43^WvM{7Da z?GIA)3L9`#B1SoInJcM0`@MTr%m^Ce@AYH{ASdR_Lp(ZHclL0ekhaEeB@nsxq~R}` zX*rsq2ib4mz`r#1VP z@>SkW_j+(U*W%Gx#fOS1JZUy+BTfe>#1VXyftHqOx^iP?UH0gHF}-)~PnK2vz-Las zR9y7p)j&3cn&}rU?6m2<)EE8eczIpUw`$#xebTei?}bi1I+N9kE@~S>#jk!krCMqE zmM_7yI{Tmb6nQ+LW+{hU&E&VpukzWH@X|9kTJ1PF-vFl`McwiS*OMWqG>cyyl`jRz zofRjRHg8Yd>b1c=oYyrz*}uK$;~=Y)`io*RN9G1>%KlLxc92<}D!%!PcVojbLp5pg zRif!NUR2FNI8Y}tK@L&vxj6KZ*T&ItIMg@hC7%JV>a9lj{3#k7W*9G{a6pLmW3l<382P zL~~i;rw8inuZQ^p@1M^d3S5k^rzp_w?0dfL`IOU^UScpvarCTI!R+~ro=2Y2d~wec zfFFVRd|)kRoE9%_wd#1!4{C-rLhFimX~T}$_$9%Vj&BQu*xo2-dEkvW$wH%#muRgt z>FPYnhU{Oxj>hm=`m0BI)9=%WO9D^)g_a zMZ-ti&k29+NuL#tgb~snDuH{t;mM2Ks6)c8ZXOe%J|Z4B$P}$8#wsa@^`W~%`9TB( zzS@KXe85LYq`o|yRDuB5QcqTUSB0U_Uj@VqF*S?-<-QGaP@KNFTUGiq!}wW=-@Uk$ zjUwh}7Iour-N82t3j&^{($6WO;xsyln@4xa;b8p>tX#WoZHG0%42C#7JUTjCgLz+H z{MH_eBau%7F4MTk56||W^O9TL`2+g9ygJ}Cxyav-w(!#FRbz=sc5O#hnF^4GczxT% zMqgPJ7G3k=*4P`P8T{zk*ir*T@UO;Woz|6>hY@1PlWYwynmJ~M00DJwsq}rD!_lmx zUYw;;5k>C}e5BH8`U{2^GhXc+@W9EvMZJPbMGHd?_R0CMIplReifYz1&_n)FT}1ur zikNL(+i?51QCC@g#75UsZZCpe6CbYeD$XNBLr~kytO5>EtvSbXIP||5fBqk&)qhaY zx{xtpgOsPWaETn@&#EDAK21NjTm0I)ehTFO36WhtMf9XUup(+4O6{Moepysv>ysgg zflV)Ab)0Q19C+=Vr%&g@lY&jB`r;ER7iITX6@rN6wQ1r^I~5T8V}^cAS^$XCwYLsr zDLG`@%dJ#eu9{fedyh4zp(VWhpizrT?k4$c95GU!vzO&nw`|M;F^b zKL!bG8Urr~D&nWmnY!YoR6O8ke+()(<$hGYY2(5Ng^iQ5Y_M=m+Fd-P5nUM zRNvjWTTc9DXQn6H22U?eo>98_-8@B@Nk+cBNYQDF@rhJo zf+Did-Pkybnfj%re^U2Bt!m(5vD8I+)|J;?LWXB{Y#h^1-7BSf*AWoc6|uO_2231Z zk2CtU5uLFy@DBMG&Ctud8rRAHR8g9rf~za9;YELGGUY`HyfHZIMlMP5P42V(42q6oDy6;GkzoX} zqGHrFDi?s{gRI4%RmX(E5d~m8GVj@gr8L&(iM1N6{MWme93FK|(%%<*0s*O^>k!9zF z{Qv~|Z1tYD31Yf$u8Rxui``qNs(R2CeR+GUoD3_cb!uW>o^*Ju9EcXq-ShvjQ4<)! z!&j)7ZRCfI#=!BJ0<8=1tjk$K2}`!@A`X3{ckTy2>aZ7VM9EdDA(r_)mX8#jjYT&s zaG>ARLkG*HOkMjAyR2$G2boG5cV5V@FtF4IVke;iZ_39Vbpx9i+coE4su#L`(Y63$ z6yC#yWMB5NAoRUorzf`*ZxQSTA_|wItV3bp#N0aIE5I5TLyexJKb8Skw)+I*ATAq` zFE4g0ZB1{7<3e&MHr4*3`(xeOzCK*Ax?l zox7;@l(~2&GCCg!WXkz5F6Mr(4)O+p*rHkJ!{gGrB;Q%Tnhqs5>)h)yxagJ8p2*kC zbZ|dBXe}(YYhFq9&C#+Qmt6PV%}1_q)r4FbnKSf_EcJHo9 z=yWGMx| Date: Wed, 6 Jul 2016 09:54:28 +0200 Subject: [PATCH 055/128] fix for buffer-overflow in IPPCalcHistInvoker::operator() --- modules/imgproc/src/histogram.cpp | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/modules/imgproc/src/histogram.cpp b/modules/imgproc/src/histogram.cpp index 7e181684cb..640f07e95f 100644 --- a/modules/imgproc/src/histogram.cpp +++ b/modules/imgproc/src/histogram.cpp @@ -1188,6 +1188,7 @@ public: virtual void operator() (const Range & range) const { + Ipp32s levelNum = histSize + 1; Mat phist(hist->size(), hist->type(), Scalar::all(0)); #if IPP_VERSION_X100 >= 900 IppiSize roi = {src->cols, range.end - range.start}; @@ -1196,7 +1197,7 @@ public: IppiHistogramSpec *pSpec = NULL; Ipp8u *pBuffer = NULL; - if(ippiHistogramGetBufferSize(ipp8u, roi, &histSize, 1, 1, &specSize, &bufferSize) < 0) + if(ippiHistogramGetBufferSize(ipp8u, roi, &levelNum, 1, 1, &specSize, &bufferSize) < 0) { *ok = false; return; @@ -1217,7 +1218,7 @@ public: return; } - if(ippiHistogramUniformInit(ipp8u, (Ipp32f*)&low, (Ipp32f*)&high, (Ipp32s*)&histSize, 1, pSpec) < 0) + if(ippiHistogramUniformInit(ipp8u, (Ipp32f*)&low, (Ipp32f*)&high, (Ipp32s*)&levelNum, 1, pSpec) < 0) { if(pSpec) ippFree(pSpec); if(pBuffer) ippFree(pBuffer); @@ -1233,7 +1234,7 @@ public: #else CV_SUPPRESS_DEPRECATED_START IppStatus status = ippiHistogramEven_8u_C1R(src->ptr(range.start), (int)src->step, ippiSize(src->cols, range.end - range.start), - phist.ptr(), (Ipp32s*)(Ipp32f*)*levels, histSize, (Ipp32s)low, (Ipp32s)high); + phist.ptr(), (Ipp32s*)(Ipp32f*)*levels, levelNum, (Ipp32s)low, (Ipp32s)high); CV_SUPPRESS_DEPRECATED_END #endif if(status < 0) @@ -1282,7 +1283,7 @@ static bool ipp_calchist(const Mat* images, int nimages, const int* channels, !accumulate && uniform) { ihist.setTo(Scalar::all(0)); - AutoBuffer levels(histSize[0] + 1); + AutoBuffer levels(histSize[0]); bool ok = true; const Mat & src = images[0]; @@ -1290,7 +1291,7 @@ static bool ipp_calchist(const Mat* images, int nimages, const int* channels, #ifdef HAVE_CONCURRENCY nstripes = 1; #endif - IPPCalcHistInvoker invoker(src, ihist, levels, histSize[0] + 1, ranges[0][0], ranges[0][1], &ok); + IPPCalcHistInvoker invoker(src, ihist, levels, histSize[0], ranges[0][0], ranges[0][1], &ok); Range range(0, src.rows); parallel_for_(range, invoker, nstripes); From 4382302a6b55e976b20378af40ce34ee4f1953dc Mon Sep 17 00:00:00 2001 From: Vladislav Vinogradov Date: Wed, 6 Jul 2016 14:27:08 +0300 Subject: [PATCH 056/128] fix matrix type for keypoints buffer in CUDA FAST use CV_32FC1 instead of CV_16SC2 since detectAsync uses CV_32FC1 to reallocate the matrix --- modules/cudafeatures2d/src/fast.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/cudafeatures2d/src/fast.cpp b/modules/cudafeatures2d/src/fast.cpp index 2095ef7cf6..ce44b3a606 100644 --- a/modules/cudafeatures2d/src/fast.cpp +++ b/modules/cudafeatures2d/src/fast.cpp @@ -104,7 +104,7 @@ namespace } BufferPool pool(Stream::Null()); - GpuMat d_keypoints = pool.getBuffer(ROWS_COUNT, max_npoints_, CV_16SC2); + GpuMat d_keypoints = pool.getBuffer(ROWS_COUNT, max_npoints_, CV_32FC1); detectAsync(_image, d_keypoints, _mask, Stream::Null()); convert(d_keypoints, keypoints); From 62a87ce38cfaa1e5bc49f1a8cc268794cb8dca4e Mon Sep 17 00:00:00 2001 From: LaurentBerger Date: Thu, 7 Jul 2016 09:29:41 +0200 Subject: [PATCH 057/128] Issue 6780 --- samples/cpp/neural_network.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/samples/cpp/neural_network.cpp b/samples/cpp/neural_network.cpp index e0ef6fa9ae..d6e681b6c6 100644 --- a/samples/cpp/neural_network.cpp +++ b/samples/cpp/neural_network.cpp @@ -16,13 +16,13 @@ int main() { if (i < data.rows/2) { - data(i, 0) = 1; - data(i, 1) = 0; + responses(i, 0) = 1; + responses(i, 1) = 0; } else { - data(i, 0) = 0; - data(i, 1) = 1; + responses(i, 0) = 0; + responses(i, 1) = 1; } } From c0fe3744908283d743f3d782f2177b9624575bcb Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Wed, 6 Jul 2016 19:53:32 +0300 Subject: [PATCH 058/128] cmake: python, allow dynamic symbols lookup from libpython.so --- cmake/OpenCVDetectPython.cmake | 8 ++-- modules/python/common.cmake | 54 +++++++++++++++++---------- modules/python/python2/CMakeLists.txt | 19 +--------- modules/python/python3/CMakeLists.txt | 19 +--------- modules/python/src2/gen2.py | 8 ++++ 5 files changed, 50 insertions(+), 58 deletions(-) diff --git a/cmake/OpenCVDetectPython.cmake b/cmake/OpenCVDetectPython.cmake index 64054d1efa..f304a50a42 100644 --- a/cmake/OpenCVDetectPython.cmake +++ b/cmake/OpenCVDetectPython.cmake @@ -75,10 +75,10 @@ if(NOT ${found}) if(NOT ANDROID AND NOT APPLE_FRAMEWORK) ocv_check_environment_variables(${library_env} ${include_dir_env}) - if(NOT ${${library_env}} EQUAL "") + if(NOT ${${library_env}} STREQUAL "") set(PYTHON_LIBRARY "${${library_env}}") endif() - if(NOT ${${include_dir_env}} EQUAL "") + if(NOT ${${include_dir_env}} STREQUAL "") set(PYTHON_INCLUDE_DIR "${${include_dir_env}}") endif() @@ -162,10 +162,10 @@ if(NOT ${found}) message(STATUS "Cannot probe for Python/Numpy support (because we are cross-compiling OpenCV)") message(STATUS "If you want to enable Python/Numpy support, set the following variables:") message(STATUS " PYTHON2_INCLUDE_PATH") - message(STATUS " PYTHON2_LIBRARIES") + message(STATUS " PYTHON2_LIBRARIES (optional on Unix-like systems)") message(STATUS " PYTHON2_NUMPY_INCLUDE_DIRS") message(STATUS " PYTHON3_INCLUDE_PATH") - message(STATUS " PYTHON3_LIBRARIES") + message(STATUS " PYTHON3_LIBRARIES (optional on Unix-like systems)") message(STATUS " PYTHON3_NUMPY_INCLUDE_DIRS") else() # Attempt to discover the NumPy include directory. If this succeeds, then build python API with NumPy diff --git a/modules/python/common.cmake b/modules/python/common.cmake index cf74f8dd95..c33f6a9101 100644 --- a/modules/python/common.cmake +++ b/modules/python/common.cmake @@ -1,11 +1,15 @@ # This file is included from a subdirectory set(PYTHON_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../") +# try to use dynamic symbols linking with libpython.so +set(OPENCV_FORCE_PYTHON_LIBS OFF CACHE BOOL "") +string(REPLACE "-Wl,--no-undefined" "" CMAKE_MODULE_LINKER_FLAGS "${CMAKE_MODULE_LINKER_FLAGS}") + ocv_add_module(${MODULE_NAME} BINDINGS) ocv_module_include_directories( - "${PYTHON_INCLUDE_PATH}" - ${PYTHON_NUMPY_INCLUDE_DIRS} + "${${PYTHON}_INCLUDE_PATH}" + ${${PYTHON}_NUMPY_INCLUDE_DIRS} "${PYTHON_SOURCE_DIR}/src2" ) @@ -41,7 +45,7 @@ set(cv2_generated_hdrs file(WRITE "${CMAKE_CURRENT_BINARY_DIR}/headers.txt" "${opencv_hdrs}") add_custom_command( OUTPUT ${cv2_generated_hdrs} - COMMAND ${PYTHON_EXECUTABLE} "${PYTHON_SOURCE_DIR}/src2/gen2.py" ${CMAKE_CURRENT_BINARY_DIR} "${CMAKE_CURRENT_BINARY_DIR}/headers.txt" + COMMAND ${PYTHON_DEFAULT_EXECUTABLE} "${PYTHON_SOURCE_DIR}/src2/gen2.py" ${CMAKE_CURRENT_BINARY_DIR} "${CMAKE_CURRENT_BINARY_DIR}/headers.txt" "${PYTHON}" DEPENDS ${PYTHON_SOURCE_DIR}/src2/gen2.py DEPENDS ${PYTHON_SOURCE_DIR}/src2/hdr_parser.py DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/headers.txt @@ -49,21 +53,28 @@ add_custom_command( ocv_add_library(${the_module} MODULE ${PYTHON_SOURCE_DIR}/src2/cv2.cpp ${cv2_generated_hdrs}) -if(PYTHON_DEBUG_LIBRARIES AND NOT PYTHON_LIBRARIES MATCHES "optimized.*debug") - ocv_target_link_libraries(${the_module} debug ${PYTHON_DEBUG_LIBRARIES} optimized ${PYTHON_LIBRARIES}) -else() - if(APPLE) - set_target_properties(${the_module} PROPERTIES LINK_FLAGS "-undefined dynamic_lookup") +if(APPLE) + set_target_properties(${the_module} PROPERTIES LINK_FLAGS "-undefined dynamic_lookup") +elseif(WIN32 OR OPENCV_FORCE_PYTHON_LIBS) + if(${PYTHON}_DEBUG_LIBRARIES AND NOT ${PYTHON}_LIBRARIES MATCHES "optimized.*debug") + ocv_target_link_libraries(${the_module} debug ${${PYTHON}_DEBUG_LIBRARIES} optimized ${${PYTHON}_LIBRARIES}) else() - ocv_target_link_libraries(${the_module} ${PYTHON_LIBRARIES}) + ocv_target_link_libraries(${the_module} ${${PYTHON}_LIBRARIES}) endif() endif() ocv_target_link_libraries(${the_module} ${OPENCV_MODULE_${the_module}_DEPS}) -execute_process(COMMAND ${PYTHON_EXECUTABLE} -c "import distutils.sysconfig; print(distutils.sysconfig.get_config_var('SO'))" - RESULT_VARIABLE PYTHON_CVPY_PROCESS - OUTPUT_VARIABLE CVPY_SUFFIX - OUTPUT_STRIP_TRAILING_WHITESPACE) +if(DEFINED ${PYTHON}_CVPY_SUFFIX) + set(CVPY_SUFFIX "${${PYTHON}_CVPY_SUFFIX}") +else() + execute_process(COMMAND ${${PYTHON}_EXECUTABLE} -c "import distutils.sysconfig; print(distutils.sysconfig.get_config_var('SO'))" + RESULT_VARIABLE PYTHON_CVPY_PROCESS + OUTPUT_VARIABLE CVPY_SUFFIX + OUTPUT_STRIP_TRAILING_WHITESPACE) + if(NOT PYTHON_CVPY_PROCESS EQUAL 0) + set(CVPY_SUFFIX ".so") + endif() +endif() set_target_properties(${the_module} PROPERTIES LIBRARY_OUTPUT_DIRECTORY "${LIBRARY_OUTPUT_PATH}/${MODULE_INSTALL_SUBDIR}" @@ -95,7 +106,7 @@ if(MSVC AND NOT BUILD_SHARED_LIBS) set_target_properties(${the_module} PROPERTIES LINK_FLAGS "/NODEFAULTLIB:atlthunk.lib /NODEFAULTLIB:atlsd.lib /DEBUG") endif() -if(MSVC AND NOT PYTHON_DEBUG_LIBRARIES) +if(MSVC AND NOT ${PYTHON}_DEBUG_LIBRARIES) set(PYTHON_INSTALL_CONFIGURATIONS CONFIGURATIONS Release) else() set(PYTHON_INSTALL_CONFIGURATIONS "") @@ -104,19 +115,22 @@ endif() if(WIN32) set(PYTHON_INSTALL_ARCHIVE "") else() - set(PYTHON_INSTALL_ARCHIVE ARCHIVE DESTINATION ${PYTHON_PACKAGES_PATH} COMPONENT python) + set(PYTHON_INSTALL_ARCHIVE ARCHIVE DESTINATION ${${PYTHON}_PACKAGES_PATH} COMPONENT python) endif() -if(NOT INSTALL_CREATE_DISTRIB) +if(NOT INSTALL_CREATE_DISTRIB AND DEFINED ${PYTHON}_PACKAGES_PATH) + set(__dst "${${PYTHON}_PACKAGES_PATH}") install(TARGETS ${the_module} OPTIONAL ${PYTHON_INSTALL_CONFIGURATIONS} - RUNTIME DESTINATION ${PYTHON_PACKAGES_PATH} COMPONENT python - LIBRARY DESTINATION ${PYTHON_PACKAGES_PATH} COMPONENT python + RUNTIME DESTINATION "${__dst}" COMPONENT python + LIBRARY DESTINATION "${__dst}" COMPONENT python ${PYTHON_INSTALL_ARCHIVE} ) else() - if(DEFINED PYTHON_VERSION_MAJOR) - set(__ver "${PYTHON_VERSION_MAJOR}.${PYTHON_VERSION_MINOR}") + if(DEFINED ${PYTHON}_VERSION_MAJOR) + set(__ver "${${PYTHON}_VERSION_MAJOR}.${${PYTHON}_VERSION_MINOR}") + elseif(DEFINED ${PYTHON}_VERSION_STRING) + set(__ver "${${PYTHON}_VERSION_STRING}") else() set(__ver "unknown") endif() diff --git a/modules/python/python2/CMakeLists.txt b/modules/python/python2/CMakeLists.txt index 158763ec50..37e20fe330 100644 --- a/modules/python/python2/CMakeLists.txt +++ b/modules/python/python2/CMakeLists.txt @@ -1,4 +1,4 @@ -if(NOT PYTHON2LIBS_FOUND OR NOT PYTHON2_NUMPY_INCLUDE_DIRS) +if(NOT PYTHON2_INCLUDE_PATH OR NOT PYTHON2_NUMPY_INCLUDE_DIRS) ocv_module_disable(python2) endif() @@ -7,24 +7,9 @@ set(MODULE_NAME python2) # Buildbot requires Python 2 to be in root lib dir set(MODULE_INSTALL_SUBDIR "") -set(PYTHON_INCLUDE_PATH ${PYTHON2_INCLUDE_PATH}) -set(PYTHON_NUMPY_INCLUDE_DIRS ${PYTHON2_NUMPY_INCLUDE_DIRS}) -set(PYTHON_EXECUTABLE ${PYTHON2_EXECUTABLE}) -set(PYTHON_DEBUG_LIBRARIES ${PYTHON2_DEBUG_LIBRARIES}) -set(PYTHON_LIBRARIES ${PYTHON2_LIBRARIES}) -set(PYTHON_PACKAGES_PATH ${PYTHON2_PACKAGES_PATH}) -set(PYTHON_VERSION_MAJOR ${PYTHON2_VERSION_MAJOR}) -set(PYTHON_VERSION_MINOR ${PYTHON2_VERSION_MINOR}) +set(PYTHON PYTHON2) include(../common.cmake) unset(MODULE_NAME) unset(MODULE_INSTALL_SUBDIR) -unset(PYTHON_INCLUDE_PATH) -unset(PYTHON_NUMPY_INCLUDE_DIRS) -unset(PYTHON_EXECUTABLE) -unset(PYTHON_DEBUG_LIBRARIES) -unset(PYTHON_LIBRARIES) -unset(PYTHON_PACKAGES_PATH) -unset(PYTHON_VERSION_MAJOR) -unset(PYTHON_VERSION_MINOR) diff --git a/modules/python/python3/CMakeLists.txt b/modules/python/python3/CMakeLists.txt index 4b6fe4f141..da86ba5c5e 100644 --- a/modules/python/python3/CMakeLists.txt +++ b/modules/python/python3/CMakeLists.txt @@ -1,4 +1,4 @@ -if(NOT PYTHON3LIBS_FOUND OR NOT PYTHON3_NUMPY_INCLUDE_DIRS) +if(NOT PYTHON3_INCLUDE_PATH OR NOT PYTHON3_NUMPY_INCLUDE_DIRS) ocv_module_disable(python3) endif() @@ -6,24 +6,9 @@ set(the_description "The python3 bindings") set(MODULE_NAME python3) set(MODULE_INSTALL_SUBDIR python3) -set(PYTHON_INCLUDE_PATH ${PYTHON3_INCLUDE_PATH}) -set(PYTHON_NUMPY_INCLUDE_DIRS ${PYTHON3_NUMPY_INCLUDE_DIRS}) -set(PYTHON_EXECUTABLE ${PYTHON3_EXECUTABLE}) -set(PYTHON_DEBUG_LIBRARIES ${PYTHON3_DEBUG_LIBRARIES}) -set(PYTHON_LIBRARIES ${PYTHON3_LIBRARIES}) -set(PYTHON_PACKAGES_PATH ${PYTHON3_PACKAGES_PATH}) -set(PYTHON_VERSION_MAJOR ${PYTHON3_VERSION_MAJOR}) -set(PYTHON_VERSION_MINOR ${PYTHON3_VERSION_MINOR}) +set(PYTHON PYTHON3) include(../common.cmake) unset(MODULE_NAME) unset(MODULE_INSTALL_SUBDIR) -unset(PYTHON_INCLUDE_PATH) -unset(PYTHON_NUMPY_INCLUDE_DIRS) -unset(PYTHON_EXECUTABLE) -unset(PYTHON_DEBUG_LIBRARIES) -unset(PYTHON_LIBRARIES) -unset(PYTHON_PACKAGES_PATH) -unset(PYTHON_VERSION_MAJOR) -unset(PYTHON_VERSION_MINOR) diff --git a/modules/python/src2/gen2.py b/modules/python/src2/gen2.py index 20c1007dee..0175f1e6db 100755 --- a/modules/python/src2/gen2.py +++ b/modules/python/src2/gen2.py @@ -44,6 +44,14 @@ gen_template_func_body = Template("""$code_decl """) py_major_version = sys.version_info[0] +if __name__ == "__main__": + if len(sys.argv) > 3: + if sys.argv[3] == 'PYTHON3': + py_major_version = 3 + elif sys.argv[3] == 'PYTHON2': + py_major_version = 2 + else: + raise Exception('Incorrect argument: expected PYTHON2 or PYTHON3, received: ' + sys.argv[3]) if py_major_version >= 3: head_init_str = "PyVarObject_HEAD_INIT(&PyType_Type, 0)" else: From 3e649895e5207a247ba9cbcc797af129eb878342 Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Thu, 7 Jul 2016 13:37:18 +0300 Subject: [PATCH 059/128] android: increase default native API level 8=>9 --- platforms/android/android.toolchain.cmake | 2 +- platforms/android/build_sdk.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/platforms/android/android.toolchain.cmake b/platforms/android/android.toolchain.cmake index 900ca8c91c..e0b4bf8d65 100644 --- a/platforms/android/android.toolchain.cmake +++ b/platforms/android/android.toolchain.cmake @@ -244,7 +244,7 @@ set( ANDROID_SUPPORTED_ABIS_mips "mips" ) set( ANDROID_SUPPORTED_ABIS_mips64 "mips64" ) # API level defaults -set( ANDROID_DEFAULT_NDK_API_LEVEL 8 ) +set( ANDROID_DEFAULT_NDK_API_LEVEL 9 ) set( ANDROID_DEFAULT_NDK_API_LEVEL_arm64 21 ) set( ANDROID_DEFAULT_NDK_API_LEVEL_x86 9 ) set( ANDROID_DEFAULT_NDK_API_LEVEL_x86_64 21 ) diff --git a/platforms/android/build_sdk.py b/platforms/android/build_sdk.py index 61c9f7d34f..7d52eab9c0 100755 --- a/platforms/android/build_sdk.py +++ b/platforms/android/build_sdk.py @@ -130,7 +130,7 @@ class Builder: "-DBUILD_ANDROID_EXAMPLES=ON", "-DINSTALL_ANDROID_EXAMPLES=ON", "-DANDROID_STL=gnustl_static", - "-DANDROID_NATIVE_API_LEVEL=8", + "-DANDROID_NATIVE_API_LEVEL=9", "-DANDROID_ABI='%s'" % abi.cmake_name, "-DWITH_TBB=ON", "-DANDROID_TOOLCHAIN_NAME=%s" % abi.toolchain From c61f7e53344ea061d67e869fa35f2f99b5af35da Mon Sep 17 00:00:00 2001 From: Vitaly Tuzov Date: Thu, 7 Jul 2016 15:58:05 +0300 Subject: [PATCH 060/128] Fix for median blur of 2-channel images --- modules/imgproc/src/smooth.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/imgproc/src/smooth.cpp b/modules/imgproc/src/smooth.cpp index dc19f7b169..952d4e3fd2 100644 --- a/modules/imgproc/src/smooth.cpp +++ b/modules/imgproc/src/smooth.cpp @@ -2848,7 +2848,7 @@ void cv::medianBlur( InputArray _src0, OutputArray _dst, int ksize ) bool useSortNet = ksize == 3 || (ksize == 5 #if !(CV_SSE2 || CV_NEON) - && src0.depth() > CV_8U + && ( src0.depth() > CV_8U || src0.channels() == 2 || src0.channels() > 4 ) #endif ); From e3f5bbd2177323bf87897471600ce5c84257a552 Mon Sep 17 00:00:00 2001 From: Vitaly Tuzov Date: Fri, 8 Jul 2016 13:31:26 +0300 Subject: [PATCH 061/128] Removed unnecessary check for Android API level and unused flags. --- 3rdparty/carotene/hal/CMakeLists.txt | 25 ------------------------- 1 file changed, 25 deletions(-) diff --git a/3rdparty/carotene/hal/CMakeLists.txt b/3rdparty/carotene/hal/CMakeLists.txt index 9eaa94a9f8..2fb92b907b 100644 --- a/3rdparty/carotene/hal/CMakeLists.txt +++ b/3rdparty/carotene/hal/CMakeLists.txt @@ -14,10 +14,6 @@ elseif (CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64.*|AARCH64.*") set(AARCH64 TRUE) endif() -if(ANDROID AND ARM) - set(WITH_TGPU ON CACHE BOOL "Enable Tegra GPGPU optimization") -endif() - set(TEGRA_COMPILER_FLAGS "") if(CMAKE_COMPILER_IS_GNUCXX) @@ -39,23 +35,10 @@ if(CMAKE_COMPILER_IS_GNUCXX) endif() endif() -if(ARM OR AARCH64) - set(CHECK_TEGRA_HARDWARE_DEFAULT ON) -else() - set(CHECK_TEGRA_HARDWARE_DEFAULT OFF) -endif() -set(CHECK_TEGRA_HARDWARE ${CHECK_TEGRA_HARDWARE_DEFAULT} CACHE BOOL - "Verify Tegra platform before running optimized code") - string(REPLACE ";" " " TEGRA_COMPILER_FLAGS "${TEGRA_COMPILER_FLAGS}") set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${TEGRA_COMPILER_FLAGS}") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${TEGRA_COMPILER_FLAGS}") -if(ANDROID_NATIVE_API_LEVEL LESS 9 AND (WITH_TGPU OR CHECK_TEGRA_HARDWARE)) - message(FATAL_ERROR "GPU support and Hardware detector is not available for API levels below 9. -Please disable Tegra GPU support and hardware detection or configure project for API level 9 or above.") -endif() - if(ARMEABI_V7A) if (CMAKE_COMPILER_IS_GNUCXX) set( CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fno-tree-vectorize" ) @@ -63,14 +46,6 @@ if(ARMEABI_V7A) endif() endif() -if (CHECK_TEGRA_HARDWARE) - add_definitions(-DCHECK_TEGRA_HARDWARE) -endif() - -if(WITH_TGPU) - add_definitions(-DHAVE_TGPU) -endif() - if(WITH_LOGS) add_definitions(-DHAVE_LOGS) endif() From 00112bbe104efc086fce141169e9bbe042d56a23 Mon Sep 17 00:00:00 2001 From: Philipp Hasper Date: Thu, 7 Jul 2016 09:55:30 +0200 Subject: [PATCH 062/128] persistence: fixing crash with space-only values --- modules/core/src/persistence.cpp | 2 +- modules/core/test/test_io.cpp | 16 ++++++++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/modules/core/src/persistence.cpp b/modules/core/src/persistence.cpp index 2fc4f9f5bb..4d99a4a275 100644 --- a/modules/core/src/persistence.cpp +++ b/modules/core/src/persistence.cpp @@ -1852,7 +1852,7 @@ icvYMLWriteString( CvFileStorage* fs, const char* key, if( quote || len == 0 || str[0] != str[len-1] || (str[0] != '\"' && str[0] != '\'') ) { - int need_quote = quote || len == 0; + int need_quote = quote || len == 0 || str[0] == ' '; data = buf; *data++ = '\"'; for( i = 0; i < len; i++ ) diff --git a/modules/core/test/test_io.cpp b/modules/core/test/test_io.cpp index 1367776f2d..f2c53dc964 100644 --- a/modules/core/test/test_io.cpp +++ b/modules/core/test/test_io.cpp @@ -578,6 +578,22 @@ TEST(Core_InputOutput, FileStorageKey) ASSERT_STREQ(f.releaseAndGetString().c_str(), expected.c_str()); } +TEST(Core_InputOutput, FileStorageSpaces) +{ + cv::FileStorage f("dummy.yml", cv::FileStorage::WRITE | cv::FileStorage::MEMORY); + const int valueCount = 5; + std::string values[5] = { "", " ", " ", " a", " some string" }; + for (size_t i = 0; i < valueCount; i++) { + EXPECT_NO_THROW(f << cv::format("key%d", i) << values[i]); + } + cv::FileStorage f2(f.releaseAndGetString(), cv::FileStorage::READ | cv::FileStorage::MEMORY); + std::string valuesRead[valueCount]; + for (size_t i = 0; i < valueCount; i++) { + EXPECT_NO_THROW(f2[cv::format("key%d", i)] >> valuesRead[i]); + ASSERT_STREQ(values[i].c_str(), valuesRead[i].c_str()); + } +} + TEST(Core_InputOutput, filestorage_yml_compatibility) { // TODO: From 04c7d03188e5d5bc5e6d85498325ed337605a909 Mon Sep 17 00:00:00 2001 From: Vitaly Tuzov Date: Fri, 8 Jul 2016 13:21:35 +0300 Subject: [PATCH 063/128] Improved Carotene library linear resize evaluation precision and enabled it as HAL implementation. --- 3rdparty/carotene/hal/tegra_hal.hpp | 4 ++-- 3rdparty/carotene/src/resize.cpp | 30 ++++++++++++++--------------- 2 files changed, 17 insertions(+), 17 deletions(-) diff --git a/3rdparty/carotene/hal/tegra_hal.hpp b/3rdparty/carotene/hal/tegra_hal.hpp index f1bf5c67a7..401f521a64 100644 --- a/3rdparty/carotene/hal/tegra_hal.hpp +++ b/3rdparty/carotene/hal/tegra_hal.hpp @@ -1433,7 +1433,7 @@ inline int TEGRA_MORPHFREE(cvhalFilter2D *context) #define TEGRA_RESIZE(src_type, src_data, src_step, src_width, src_height, dst_data, dst_step, dst_width, dst_height, inv_scale_x, inv_scale_y, interpolation) \ ( \ - /*interpolation == CV_HAL_INTER_LINEAR ? \ + interpolation == CV_HAL_INTER_LINEAR ? \ CV_MAT_DEPTH(src_type) == CV_8U && CAROTENE_NS::isResizeLinearOpenCVSupported(CAROTENE_NS::Size2D(src_width, src_height), CAROTENE_NS::Size2D(dst_width, dst_height), ((src_type >> CV_CN_SHIFT) + 1)) && \ inv_scale_x > 0 && inv_scale_y > 0 && \ (dst_width - 0.5)/inv_scale_x - 0.5 < src_width && (dst_height - 0.5)/inv_scale_y - 0.5 < src_height && \ @@ -1441,7 +1441,7 @@ inline int TEGRA_MORPHFREE(cvhalFilter2D *context) std::abs(dst_width / inv_scale_x - src_width) < 0.1 && std::abs(dst_height / inv_scale_y - src_height) < 0.1 ? \ CAROTENE_NS::resizeLinearOpenCV(CAROTENE_NS::Size2D(src_width, src_height), CAROTENE_NS::Size2D(dst_width, dst_height), \ src_data, src_step, dst_data, dst_step, 1.0/inv_scale_x, 1.0/inv_scale_y, ((src_type >> CV_CN_SHIFT) + 1)), \ - CV_HAL_ERROR_OK : CV_HAL_ERROR_NOT_IMPLEMENTED :*/ \ + CV_HAL_ERROR_OK : CV_HAL_ERROR_NOT_IMPLEMENTED : \ interpolation == CV_HAL_INTER_AREA ? \ CV_MAT_DEPTH(src_type) == CV_8U && CAROTENE_NS::isResizeAreaSupported(1.0/inv_scale_x, 1.0/inv_scale_y, ((src_type >> CV_CN_SHIFT) + 1)) && \ std::abs(dst_width / inv_scale_x - src_width) < 0.1 && std::abs(dst_height / inv_scale_y - src_height) < 0.1 ? \ diff --git a/3rdparty/carotene/src/resize.cpp b/3rdparty/carotene/src/resize.cpp index 122a5f2201..3a80d472df 100644 --- a/3rdparty/carotene/src/resize.cpp +++ b/3rdparty/carotene/src/resize.cpp @@ -1681,15 +1681,15 @@ void downsample_bilinear_8uc1(const Size2D &ssize, const Size2D &dsize, vst1_u8(internal::getRowPtr(dstBase, dstStride, row) + col, vres); #else /* ugly version matching to OpenCV's SSE optimization */ - int16x4_t v1Ls = vshrn_n_s32(v1L, 5); - int16x4_t v1Hs = vshrn_n_s32(v1H, 5); - int16x4_t v2Ls = vshrn_n_s32(v2L, 5); - int16x4_t v2Hs = vshrn_n_s32(v2H, 5); + int16x4_t v1Ls = vshrn_n_s32(v1L, 4); + int16x4_t v1Hs = vshrn_n_s32(v1H, 4); + int16x4_t v2Ls = vshrn_n_s32(v2L, 4); + int16x4_t v2Hs = vshrn_n_s32(v2H, 4); int16x8_t v1s = vqdmulhq_s16(vcombine_s16(v1Ls, v1Hs), vrw); int16x8_t v2s = vqdmulhq_s16(vcombine_s16(v2Ls, v2Hs), vrW); - int16x8_t vsum = vaddq_s16(v1s, v2s); + int16x8_t vsum = vaddq_s16(vshrq_n_s16(v1s,1), vshrq_n_s16(v2s,1)); uint8x8_t vres = vqrshrun_n_s16(vsum, 2); vst1_u8(internal::getRowPtr(dstBase, dstStride, row) + col, vres); @@ -1736,15 +1736,15 @@ void downsample_bilinear_8uc1(const Size2D &ssize, const Size2D &dsize, vst1_u8(internal::getRowPtr(dstBase, dstStride, row) + col + 8, vres); #else /* ugly version matching to OpenCV's SSE optimization */ - int16x4_t v1Ls = vshrn_n_s32(v1L, 5); - int16x4_t v1Hs = vshrn_n_s32(v1H, 5); - int16x4_t v2Ls = vshrn_n_s32(v2L, 5); - int16x4_t v2Hs = vshrn_n_s32(v2H, 5); + int16x4_t v1Ls = vshrn_n_s32(v1L, 4); + int16x4_t v1Hs = vshrn_n_s32(v1H, 4); + int16x4_t v2Ls = vshrn_n_s32(v2L, 4); + int16x4_t v2Hs = vshrn_n_s32(v2H, 4); int16x8_t v1s = vqdmulhq_s16(vcombine_s16(v1Ls, v1Hs), vrw); int16x8_t v2s = vqdmulhq_s16(vcombine_s16(v2Ls, v2Hs), vrW); - int16x8_t vsum = vaddq_s16(v1s, v2s); + int16x8_t vsum = vaddq_s16(vshrq_n_s16(v1s,1), vshrq_n_s16(v2s,1)); uint8x8_t vres = vqrshrun_n_s16(vsum, 2); vst1_u8(internal::getRowPtr(dstBase, dstStride, row) + col + 8, vres); @@ -1836,15 +1836,15 @@ downsample_bilinear_8uc1_col_loop8: vst1_u8(internal::getRowPtr(dstBase, dstStride, row) + col, vres); #else /* ugly version matching to OpenCV's SSE optimization */ - int16x4_t v1Ls = vshrn_n_s32(v1L, 5); - int16x4_t v1Hs = vshrn_n_s32(v1H, 5); - int16x4_t v2Ls = vshrn_n_s32(v2L, 5); - int16x4_t v2Hs = vshrn_n_s32(v2H, 5); + int16x4_t v1Ls = vshrn_n_s32(v1L, 4); + int16x4_t v1Hs = vshrn_n_s32(v1H, 4); + int16x4_t v2Ls = vshrn_n_s32(v2L, 4); + int16x4_t v2Hs = vshrn_n_s32(v2H, 4); int16x8_t v1s = vqdmulhq_s16(vcombine_s16(v1Ls, v1Hs), vrw); int16x8_t v2s = vqdmulhq_s16(vcombine_s16(v2Ls, v2Hs), vrW); - int16x8_t vsum = vaddq_s16(v1s, v2s); + int16x8_t vsum = vaddq_s16(vshrq_n_s16(v1s,1), vshrq_n_s16(v2s,1)); uint8x8_t vres = vqrshrun_n_s16(vsum, 2); vst1_u8(internal::getRowPtr(dstBase, dstStride, row) + col, vres); From 6f22f49c027be4802e56d17af669f0577a968986 Mon Sep 17 00:00:00 2001 From: look4pritam Date: Mon, 20 Apr 2015 17:57:57 +0530 Subject: [PATCH 064/128] Grassroots DiCoM i.e. GDCM based DICOM image reader is added. --- CMakeLists.txt | 8 + cmake/OpenCVFindLibsGrfmt.cmake | 12 ++ cmake/templates/cvconfig.h.in | 3 + modules/imgcodecs/CMakeLists.txt | 11 ++ modules/imgcodecs/src/gdcm_dicom.cpp | 216 +++++++++++++++++++++++++++ modules/imgcodecs/src/gdcm_dicom.hpp | 71 +++++++++ modules/imgcodecs/src/loadsave.cpp | 6 + 7 files changed, 327 insertions(+) create mode 100644 modules/imgcodecs/src/gdcm_dicom.cpp create mode 100644 modules/imgcodecs/src/gdcm_dicom.hpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 7ea42680b9..317edbb139 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -191,6 +191,7 @@ OCV_OPTION(WITH_OPENGL "Include OpenGL support" OFF OCV_OPTION(WITH_OPENNI "Include OpenNI support" OFF IF (NOT ANDROID AND NOT IOS AND NOT WINRT) ) OCV_OPTION(WITH_OPENNI2 "Include OpenNI2 support" OFF IF (NOT ANDROID AND NOT IOS AND NOT WINRT) ) OCV_OPTION(WITH_PNG "Include PNG support" ON) +OCV_OPTION(WITH_GDCM "Include DICOM support" OFF) OCV_OPTION(WITH_PVAPI "Include Prosilica GigE support" OFF IF (NOT ANDROID AND NOT IOS AND NOT WINRT) ) OCV_OPTION(WITH_GIGEAPI "Include Smartek GigE support" OFF IF (NOT ANDROID AND NOT IOS AND NOT WINRT) ) OCV_OPTION(WITH_QT "Build with Qt Backend support" OFF IF (NOT ANDROID AND NOT IOS AND NOT WINRT) ) @@ -1008,6 +1009,7 @@ if(WITH_PNG) else() status(" PNG:" "NO") endif() + if(WITH_TIFF) if(TIFF_VERSION_STRING AND TIFF_FOUND) status(" TIFF:" "${TIFF_LIBRARY} (ver ${TIFF_VERSION} - ${TIFF_VERSION_STRING})") @@ -1034,6 +1036,12 @@ else() status(" GDAL:" "NO") endif() +if(WITH_GDCM) + status(" GDCM:" GDCM_FOUND THEN "YES (ver ${GDCM_VERSION})" ELSE "NO") +else() + status(" GDCM:" "NO") +endif() + # ========================== VIDEO IO ========================== status("") status(" Video I/O:") diff --git a/cmake/OpenCVFindLibsGrfmt.cmake b/cmake/OpenCVFindLibsGrfmt.cmake index 614f844c7b..ced3c8ef7f 100644 --- a/cmake/OpenCVFindLibsGrfmt.cmake +++ b/cmake/OpenCVFindLibsGrfmt.cmake @@ -212,3 +212,15 @@ if(WITH_GDAL) ocv_include_directories(${GDAL_INCLUDE_DIR}) endif() endif() + +if (WITH_GDCM) + find_package(GDCM) + if(NOT GDCM_FOUND) + set(HAVE_GDCM NO) + ocv_clear_vars(GDCM_VERSION GDCM_LIBRARIES) + else() + set(HAVE_GDCM YES) + # include(${GDCM_USE_FILE}) + set(GDCM_LIBRARIES gdcmMSFF) # GDCM does not set this variable for some reason + endif() +endif() diff --git a/cmake/templates/cvconfig.h.in b/cmake/templates/cvconfig.h.in index b86d44a6c1..2312742130 100644 --- a/cmake/templates/cvconfig.h.in +++ b/cmake/templates/cvconfig.h.in @@ -111,6 +111,9 @@ /* libpng/png.h needs to be included */ #cmakedefine HAVE_LIBPNG_PNG_H +/* GDCM DICOM codec */ +#cmakedefine HAVE_GDCM + /* V4L/V4L2 capturing support via libv4l */ #cmakedefine HAVE_LIBV4L diff --git a/modules/imgcodecs/CMakeLists.txt b/modules/imgcodecs/CMakeLists.txt index 8b8c577166..1c7cc25b9e 100644 --- a/modules/imgcodecs/CMakeLists.txt +++ b/modules/imgcodecs/CMakeLists.txt @@ -35,6 +35,11 @@ if(HAVE_PNG) list(APPEND GRFMT_LIBS ${PNG_LIBRARIES}) endif() +if(HAVE_GDCM) + ocv_include_directories(${GDCM_INCLUDE_DIRS}) + list(APPEND GRFMT_LIBS ${GDCM_LIBRARIES}) +endif() + if(HAVE_TIFF) ocv_include_directories(${TIFF_INCLUDE_DIR}) list(APPEND GRFMT_LIBS ${TIFF_LIBRARIES}) @@ -57,6 +62,12 @@ endif() file(GLOB grfmt_hdrs ${CMAKE_CURRENT_LIST_DIR}/src/grfmt*.hpp) file(GLOB grfmt_srcs ${CMAKE_CURRENT_LIST_DIR}/src/grfmt*.cpp) + +if(HAVE_GDCM) + list(APPEND grfmt_hdrs ${CMAKE_CURRENT_LIST_DIR}/src/gdcm_dicom.hpp) + list(APPEND grfmt_srcs ${CMAKE_CURRENT_LIST_DIR}/src/gdcm_dicom.cpp) +endif() + list(APPEND grfmt_hdrs ${CMAKE_CURRENT_LIST_DIR}/src/bitstrm.hpp) list(APPEND grfmt_srcs ${CMAKE_CURRENT_LIST_DIR}/src/bitstrm.cpp) list(APPEND grfmt_hdrs ${CMAKE_CURRENT_LIST_DIR}/src/rgbe.hpp) diff --git a/modules/imgcodecs/src/gdcm_dicom.cpp b/modules/imgcodecs/src/gdcm_dicom.cpp new file mode 100644 index 0000000000..58e38b95d0 --- /dev/null +++ b/modules/imgcodecs/src/gdcm_dicom.cpp @@ -0,0 +1,216 @@ +/*M/////////////////////////////////////////////////////////////////////////////////////// +// +// IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. +// +// By downloading, copying, installing or using the software you agree to this license. +// If you do not agree to this license, do not download, install, +// copy or use the software. +// +// +// License Agreement +// For Open Source Computer Vision Library +// +// Copyright (C) 2000-2008, Intel Corporation, all rights reserved. +// Copyright (C) 2009, Willow Garage Inc., all rights reserved. +// Third party copyrights are property of their respective owners. +// +// Redistribution and use in source and binary forms, with or without modification, +// are permitted provided that the following conditions are met: +// +// * Redistribution's of source code must retain the above copyright notice, +// this list of conditions and the following disclaimer. +// +// * Redistribution's in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// +// * The name of the copyright holders may not be used to endorse or promote products +// derived from this software without specific prior written permission. +// +// This software is provided by the copyright holders and contributors "as is" and +// any express or implied warranties, including, but not limited to, the implied +// warranties of merchantability and fitness for a particular purpose are disclaimed. +// In no event shall the Intel Corporation or contributors be liable for any direct, +// indirect, incidental, special, exemplary, or consequential damages +// (including, but not limited to, procurement of substitute goods or services; +// loss of use, data, or profits; or business interruption) however caused +// and on any theory of liability, whether in contract, strict liability, +// or tort (including negligence or otherwise) arising in any way out of +// the use of this software, even if advised of the possibility of such damage. +// +//M*/ + +#include "precomp.hpp" +#include "gdcm_dicom.hpp" + +#include + +namespace cv +{ + +/************************ DICOM decoder *****************************/ + +DICOMDecoder::DICOMDecoder() +{ + /// DICOM preable is 128 bytes (can have any vale, defaults to x00) + 4 bytes magic number (DICM) + m_signature = ""; + for(int iSize=0; iSize<128; iSize++) + { + m_signature = m_signature + "\xFF"; + } + + m_signature = m_signature + "\x44\x49\x43\x4D"; + + m_buf_supported = false; +} + + +DICOMDecoder::~DICOMDecoder() +{ +} + +bool DICOMDecoder::checkSignature( const String& signature ) const +{ + size_t len = signatureLength(); + bool bOK = signature.size() >= len; + for(int iIndex = 128; iIndex < len; iIndex++) + { + if(signature[iIndex] == m_signature[iIndex]) + { + continue; + } + else + { + bOK = false; + break; + } + } + + return(bOK); +} + +void DICOMDecoder::close() +{ +} + +ImageDecoder DICOMDecoder::newDecoder() const +{ + return makePtr(); +} + +bool DICOMDecoder::readHeader() +{ + gdcm::ImageReader csImageReader; + csImageReader.SetFileName(m_filename.c_str()); + if(!csImageReader.Read()) + { + return(false); + } + + bool bOK = true; + + const gdcm::Image &csImage = csImageReader.GetImage(); + if( ( csImage.GetPhotometricInterpretation() == gdcm::PhotometricInterpretation::MONOCHROME1 ) + || ( csImage.GetPhotometricInterpretation() == gdcm::PhotometricInterpretation::MONOCHROME2 ) + ) + { + gdcm::PixelFormat ePixelFormat = csImage.GetPixelFormat(); + if( ePixelFormat == gdcm::PixelFormat::INT8) + { + m_type = CV_8SC1; + } + else if( ePixelFormat == gdcm::PixelFormat::UINT8) + { + m_type = CV_8UC1; + } + else if( ePixelFormat == gdcm::PixelFormat::INT16) + { + m_type = CV_16SC1; + } + else if( ePixelFormat == gdcm::PixelFormat::UINT16) + { + m_type = CV_16UC1; + } + else if( ePixelFormat == gdcm::PixelFormat::INT32) + { + m_type = CV_32SC1; + } + else if( ePixelFormat == gdcm::PixelFormat::FLOAT32) + { + m_type = CV_32FC1; + } + else if( ePixelFormat == gdcm::PixelFormat::FLOAT64) + { + m_type = CV_64FC1; + } + else if( ePixelFormat == gdcm::PixelFormat::INT12) + { + bOK = false; + } + else if( ePixelFormat == gdcm::PixelFormat::UINT12) + { + bOK = false; + } + else if( ePixelFormat == gdcm::PixelFormat::UINT32) + { + bOK = false; + } + else if( ePixelFormat == gdcm::PixelFormat::SINGLEBIT) + { + bOK = false; + } + else + { + bOK = false; + } + } + else if( csImage.GetPhotometricInterpretation() == gdcm::PhotometricInterpretation::RGB ) + { + gdcm::PixelFormat ePixelFormat = csImage.GetPixelFormat(); + if( ePixelFormat == gdcm::PixelFormat::UINT8) + { + m_type = CV_8UC3; + } + else + { + bOK = false; + } + } + else + { + bOK = false; + } + + if(bOK) + { + const unsigned int *piDimension = csImage.GetDimensions(); + m_width = piDimension[0]; + m_height = piDimension[1]; + if( ( m_width <=0 ) || ( m_height <=0 ) ) + { + bOK = false; + } + } + + return(bOK); +} + + +bool DICOMDecoder::readData( Mat& csImage ) +{ + csImage.create(m_width,m_height,m_type); + + gdcm::ImageReader csImageReader; + csImageReader.SetFileName(m_filename.c_str()); + if(!csImageReader.Read()) + { + return(false); + } + + bool bOK = true; + const gdcm::Image &csGDCMImage = csImageReader.GetImage(); + bOK = csGDCMImage.GetBuffer((char*)csImage.ptr()); + + return(bOK); +} +} diff --git a/modules/imgcodecs/src/gdcm_dicom.hpp b/modules/imgcodecs/src/gdcm_dicom.hpp new file mode 100644 index 0000000000..c9c5c23f27 --- /dev/null +++ b/modules/imgcodecs/src/gdcm_dicom.hpp @@ -0,0 +1,71 @@ +/*M/////////////////////////////////////////////////////////////////////////////////////// +// +// IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. +// +// By downloading, copying, installing or using the software you agree to this license. +// If you do not agree to this license, do not download, install, +// copy or use the software. +// +// +// License Agreement +// For Open Source Computer Vision Library +// +// Copyright (C) 2000-2008, Intel Corporation, all rights reserved. +// Copyright (C) 2009, Willow Garage Inc., all rights reserved. +// Third party copyrights are property of their respective owners. +// +// Redistribution and use in source and binary forms, with or without modification, +// are permitted provided that the following conditions are met: +// +// * Redistribution's of source code must retain the above copyright notice, +// this list of conditions and the following disclaimer. +// +// * Redistribution's in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// +// * The name of the copyright holders may not be used to endorse or promote products +// derived from this software without specific prior written permission. +// +// This software is provided by the copyright holders and contributors "as is" and +// any express or implied warranties, including, but not limited to, the implied +// warranties of merchantability and fitness for a particular purpose are disclaimed. +// In no event shall the Intel Corporation or contributors be liable for any direct, +// indirect, incidental, special, exemplary, or consequential damages +// (including, but not limited to, procurement of substitute goods or services; +// loss of use, data, or profits; or business interruption) however caused +// and on any theory of liability, whether in contract, strict liability, +// or tort (including negligence or otherwise) arising in any way out of +// the use of this software, even if advised of the possibility of such damage. +// +//M*/ + +#ifndef _GDCM_DICOM_H_ +#define _GDCM_DICOM_H_ + +#include "grfmt_base.hpp" + +namespace cv +{ + +// DICOM image reader using GDCM +class DICOMDecoder : public BaseImageDecoder +{ +public: + + DICOMDecoder(); + ~DICOMDecoder(); + + bool readData( Mat& img ); + bool readHeader(); + void close(); + + ImageDecoder newDecoder() const; + virtual bool checkSignature( const String& signature ) const; + +protected: +}; + +} + +#endif/*_GDCM_DICOM_H_*/ diff --git a/modules/imgcodecs/src/loadsave.cpp b/modules/imgcodecs/src/loadsave.cpp index 70a31c37a2..c62b0bc722 100644 --- a/modules/imgcodecs/src/loadsave.cpp +++ b/modules/imgcodecs/src/loadsave.cpp @@ -45,6 +45,9 @@ #include "precomp.hpp" #include "grfmts.hpp" +#ifdef HAVE_GDCM +#include "gdcm_dicom.hpp" +#endif #undef min #undef max #include @@ -93,6 +96,9 @@ struct ImageCodecInitializer decoders.push_back( makePtr() ); encoders.push_back( makePtr() ); #endif + #ifdef HAVE_GDCM + decoders.push_back( makePtr() ); + #endif #ifdef HAVE_JASPER decoders.push_back( makePtr() ); encoders.push_back( makePtr() ); From 0fd0acf2e3697fd98ad89f214e3c6acc22bc55e3 Mon Sep 17 00:00:00 2001 From: Maksim Shabunin Date: Thu, 7 Jul 2016 12:33:24 +0300 Subject: [PATCH 065/128] GDCM: modified compilation scheme - renamed source files - guard the contents - always include --- modules/imgcodecs/CMakeLists.txt | 5 ----- modules/imgcodecs/src/{gdcm_dicom.cpp => grfmt_gdcm.cpp} | 6 +++++- modules/imgcodecs/src/{gdcm_dicom.hpp => grfmt_gdcm.hpp} | 6 ++++++ modules/imgcodecs/src/grfmts.hpp | 1 + modules/imgcodecs/src/loadsave.cpp | 3 --- 5 files changed, 12 insertions(+), 9 deletions(-) rename modules/imgcodecs/src/{gdcm_dicom.cpp => grfmt_gdcm.cpp} (99%) rename modules/imgcodecs/src/{gdcm_dicom.hpp => grfmt_gdcm.hpp} (98%) diff --git a/modules/imgcodecs/CMakeLists.txt b/modules/imgcodecs/CMakeLists.txt index 1c7cc25b9e..c614d79cdd 100644 --- a/modules/imgcodecs/CMakeLists.txt +++ b/modules/imgcodecs/CMakeLists.txt @@ -63,11 +63,6 @@ endif() file(GLOB grfmt_hdrs ${CMAKE_CURRENT_LIST_DIR}/src/grfmt*.hpp) file(GLOB grfmt_srcs ${CMAKE_CURRENT_LIST_DIR}/src/grfmt*.cpp) -if(HAVE_GDCM) - list(APPEND grfmt_hdrs ${CMAKE_CURRENT_LIST_DIR}/src/gdcm_dicom.hpp) - list(APPEND grfmt_srcs ${CMAKE_CURRENT_LIST_DIR}/src/gdcm_dicom.cpp) -endif() - list(APPEND grfmt_hdrs ${CMAKE_CURRENT_LIST_DIR}/src/bitstrm.hpp) list(APPEND grfmt_srcs ${CMAKE_CURRENT_LIST_DIR}/src/bitstrm.cpp) list(APPEND grfmt_hdrs ${CMAKE_CURRENT_LIST_DIR}/src/rgbe.hpp) diff --git a/modules/imgcodecs/src/gdcm_dicom.cpp b/modules/imgcodecs/src/grfmt_gdcm.cpp similarity index 99% rename from modules/imgcodecs/src/gdcm_dicom.cpp rename to modules/imgcodecs/src/grfmt_gdcm.cpp index 58e38b95d0..7aecf35f09 100644 --- a/modules/imgcodecs/src/gdcm_dicom.cpp +++ b/modules/imgcodecs/src/grfmt_gdcm.cpp @@ -41,7 +41,9 @@ //M*/ #include "precomp.hpp" -#include "gdcm_dicom.hpp" +#include "grfmt_gdcm.hpp" + +#ifdef HAVE_GDCM #include @@ -214,3 +216,5 @@ bool DICOMDecoder::readData( Mat& csImage ) return(bOK); } } + +#endif \ No newline at end of file diff --git a/modules/imgcodecs/src/gdcm_dicom.hpp b/modules/imgcodecs/src/grfmt_gdcm.hpp similarity index 98% rename from modules/imgcodecs/src/gdcm_dicom.hpp rename to modules/imgcodecs/src/grfmt_gdcm.hpp index c9c5c23f27..8a8e10032c 100644 --- a/modules/imgcodecs/src/gdcm_dicom.hpp +++ b/modules/imgcodecs/src/grfmt_gdcm.hpp @@ -43,6 +43,10 @@ #ifndef _GDCM_DICOM_H_ #define _GDCM_DICOM_H_ +#include "cvconfig.h" + +#ifdef HAVE_GDCM + #include "grfmt_base.hpp" namespace cv @@ -68,4 +72,6 @@ protected: } +#endif + #endif/*_GDCM_DICOM_H_*/ diff --git a/modules/imgcodecs/src/grfmts.hpp b/modules/imgcodecs/src/grfmts.hpp index c9e31530a8..7db1ac94a2 100644 --- a/modules/imgcodecs/src/grfmts.hpp +++ b/modules/imgcodecs/src/grfmts.hpp @@ -54,5 +54,6 @@ #include "grfmt_webp.hpp" #include "grfmt_hdr.hpp" #include "grfmt_gdal.hpp" +#include "grfmt_gdcm.hpp" #endif/*_GRFMTS_H_*/ diff --git a/modules/imgcodecs/src/loadsave.cpp b/modules/imgcodecs/src/loadsave.cpp index c62b0bc722..1c0b794b8e 100644 --- a/modules/imgcodecs/src/loadsave.cpp +++ b/modules/imgcodecs/src/loadsave.cpp @@ -45,9 +45,6 @@ #include "precomp.hpp" #include "grfmts.hpp" -#ifdef HAVE_GDCM -#include "gdcm_dicom.hpp" -#endif #undef min #undef max #include From f3bd508e6d96c3f988da2bb14a3a8ec7930193df Mon Sep 17 00:00:00 2001 From: Maksim Shabunin Date: Fri, 8 Jul 2016 16:08:11 +0300 Subject: [PATCH 066/128] GDCM: several improvements - fixed width and height order - removed unused methods - simplified signature matching - rewrote pixel format matching in more compact form - added dimensions number check (only 2 is allowed) - added target buffer size check - added debug messages in all failing points --- modules/imgcodecs/src/grfmt_gdcm.cpp | 181 ++++++++++++--------------- modules/imgcodecs/src/grfmt_gdcm.hpp | 9 +- 2 files changed, 80 insertions(+), 110 deletions(-) diff --git a/modules/imgcodecs/src/grfmt_gdcm.cpp b/modules/imgcodecs/src/grfmt_gdcm.cpp index 7aecf35f09..a1d9e9d0af 100644 --- a/modules/imgcodecs/src/grfmt_gdcm.cpp +++ b/modules/imgcodecs/src/grfmt_gdcm.cpp @@ -45,8 +45,19 @@ #ifdef HAVE_GDCM +//#define DBG(...) printf(__VA_ARGS__) +#define DBG(...) + #include +static const size_t preamble_skip = 128; +static const size_t magic_len = 4; + +inline cv::String getMagic() +{ + return cv::String("\x44\x49\x43\x4D", 4); +} + namespace cv { @@ -54,45 +65,22 @@ namespace cv DICOMDecoder::DICOMDecoder() { - /// DICOM preable is 128 bytes (can have any vale, defaults to x00) + 4 bytes magic number (DICM) - m_signature = ""; - for(int iSize=0; iSize<128; iSize++) - { - m_signature = m_signature + "\xFF"; - } - - m_signature = m_signature + "\x44\x49\x43\x4D"; - + // DICOM preamble is 128 bytes (can have any value, defaults to 0) + 4 bytes magic number (DICM) + m_signature = String(preamble_skip, (char)'\x0') + getMagic(); m_buf_supported = false; } - -DICOMDecoder::~DICOMDecoder() -{ -} - bool DICOMDecoder::checkSignature( const String& signature ) const { - size_t len = signatureLength(); - bool bOK = signature.size() >= len; - for(int iIndex = 128; iIndex < len; iIndex++) + if (signature.size() >= preamble_skip + magic_len) { - if(signature[iIndex] == m_signature[iIndex]) + if (signature.substr(preamble_skip, magic_len) == getMagic()) { - continue; - } - else - { - bOK = false; - break; + return true; } } - - return(bOK); -} - -void DICOMDecoder::close() -{ + DBG("GDCM | Signature does not match\n"); + return false; } ImageDecoder DICOMDecoder::newDecoder() const @@ -106,90 +94,66 @@ bool DICOMDecoder::readHeader() csImageReader.SetFileName(m_filename.c_str()); if(!csImageReader.Read()) { + DBG("GDCM | Failed to open DICOM file\n"); return(false); } - bool bOK = true; - const gdcm::Image &csImage = csImageReader.GetImage(); - if( ( csImage.GetPhotometricInterpretation() == gdcm::PhotometricInterpretation::MONOCHROME1 ) - || ( csImage.GetPhotometricInterpretation() == gdcm::PhotometricInterpretation::MONOCHROME2 ) - ) + bool bOK = true; + switch (csImage.GetPhotometricInterpretation().GetType()) { - gdcm::PixelFormat ePixelFormat = csImage.GetPixelFormat(); - if( ePixelFormat == gdcm::PixelFormat::INT8) + case gdcm::PhotometricInterpretation::MONOCHROME1: + case gdcm::PhotometricInterpretation::MONOCHROME2: { - m_type = CV_8SC1; + switch (csImage.GetPixelFormat().GetScalarType()) + { + case gdcm::PixelFormat::INT8: m_type = CV_8SC1; break; + case gdcm::PixelFormat::UINT8: m_type = CV_8UC1; break; + case gdcm::PixelFormat::INT16: m_type = CV_16SC1; break; + case gdcm::PixelFormat::UINT16: m_type = CV_16UC1; break; + case gdcm::PixelFormat::INT32: m_type = CV_32SC1; break; + case gdcm::PixelFormat::FLOAT32: m_type = CV_32FC1; break; + case gdcm::PixelFormat::FLOAT64: m_type = CV_64FC1; break; + default: bOK = false; DBG("GDCM | Monochrome scalar type not supported\n"); break; + } + break; } - else if( ePixelFormat == gdcm::PixelFormat::UINT8) + + case gdcm::PhotometricInterpretation::RGB: { - m_type = CV_8UC1; + switch (csImage.GetPixelFormat().GetScalarType()) + { + case gdcm::PixelFormat::UINT8: m_type = CV_8UC3; break; + default: bOK = false; DBG("GDCM | RGB scalar type not supported\n"); break; + } + break; } - else if( ePixelFormat == gdcm::PixelFormat::INT16) - { - m_type = CV_16SC1; - } - else if( ePixelFormat == gdcm::PixelFormat::UINT16) - { - m_type = CV_16UC1; - } - else if( ePixelFormat == gdcm::PixelFormat::INT32) - { - m_type = CV_32SC1; - } - else if( ePixelFormat == gdcm::PixelFormat::FLOAT32) - { - m_type = CV_32FC1; - } - else if( ePixelFormat == gdcm::PixelFormat::FLOAT64) - { - m_type = CV_64FC1; - } - else if( ePixelFormat == gdcm::PixelFormat::INT12) + + default: { bOK = false; + DBG("GDCM | PI not supported: %s\n", csImage.GetPhotometricInterpretation().GetString()); + break; } - else if( ePixelFormat == gdcm::PixelFormat::UINT12) - { - bOK = false; - } - else if( ePixelFormat == gdcm::PixelFormat::UINT32) - { - bOK = false; - } - else if( ePixelFormat == gdcm::PixelFormat::SINGLEBIT) - { - bOK = false; - } - else - { - bOK = false; - } - } - else if( csImage.GetPhotometricInterpretation() == gdcm::PhotometricInterpretation::RGB ) - { - gdcm::PixelFormat ePixelFormat = csImage.GetPixelFormat(); - if( ePixelFormat == gdcm::PixelFormat::UINT8) - { - m_type = CV_8UC3; - } - else - { - bOK = false; - } - } - else - { - bOK = false; } if(bOK) + { + unsigned int ndim = csImage.GetNumberOfDimensions(); + if (ndim != 2) + { + DBG("GDCM | Invalid dimensions number: %d\n", ndim); + bOK = false; + } + } + if (bOK) { const unsigned int *piDimension = csImage.GetDimensions(); - m_width = piDimension[0]; - m_height = piDimension[1]; + m_height = piDimension[0]; + m_width = piDimension[1]; if( ( m_width <=0 ) || ( m_height <=0 ) ) { + DBG("GDCM | Invalid dimensions: %d x %d\n", piDimension[0], piDimension[1]); bOK = false; } } @@ -206,15 +170,28 @@ bool DICOMDecoder::readData( Mat& csImage ) csImageReader.SetFileName(m_filename.c_str()); if(!csImageReader.Read()) { - return(false); + DBG("GDCM | Failed to Read\n"); + return false; } - bool bOK = true; - const gdcm::Image &csGDCMImage = csImageReader.GetImage(); - bOK = csGDCMImage.GetBuffer((char*)csImage.ptr()); + const gdcm::Image &img = csImageReader.GetImage(); - return(bOK); -} + unsigned long len = img.GetBufferLength(); + if (len > csImage.elemSize() * csImage.total()) + { + DBG("GDCM | Buffer is bigger than Mat: %ld > %ld * %ld\n", len, csImage.elemSize(), csImage.total()); + return false; + } + + if (!img.GetBuffer((char*)csImage.ptr())) + { + DBG("GDCM | Failed to GetBuffer\n"); + return false; + } + DBG("GDCM | Read OK\n"); + return true; } -#endif \ No newline at end of file +} + +#endif // HAVE_GDCM \ No newline at end of file diff --git a/modules/imgcodecs/src/grfmt_gdcm.hpp b/modules/imgcodecs/src/grfmt_gdcm.hpp index 8a8e10032c..d8dc60f522 100644 --- a/modules/imgcodecs/src/grfmt_gdcm.hpp +++ b/modules/imgcodecs/src/grfmt_gdcm.hpp @@ -56,22 +56,15 @@ namespace cv class DICOMDecoder : public BaseImageDecoder { public: - DICOMDecoder(); - ~DICOMDecoder(); - bool readData( Mat& img ); bool readHeader(); - void close(); - ImageDecoder newDecoder() const; virtual bool checkSignature( const String& signature ) const; - -protected: }; } -#endif +#endif // HAVE_GDCM #endif/*_GDCM_DICOM_H_*/ From 5f269d08b462b96bde2b8d13afaa2b7b97a48a49 Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Fri, 8 Jul 2016 17:46:08 +0300 Subject: [PATCH 067/128] bigdata: add test, resolve split/merge issue --- cmake/OpenCVModule.cmake | 4 ++++ cmake/OpenCVUtils.cmake | 10 ++++++++++ modules/core/src/convert.cpp | 29 +++++++++++++++++------------ modules/core/test/test_mat.cpp | 26 ++++++++++++++++++++++++++ 4 files changed, 57 insertions(+), 12 deletions(-) diff --git a/cmake/OpenCVModule.cmake b/cmake/OpenCVModule.cmake index ee14c7922c..197aead0d2 100644 --- a/cmake/OpenCVModule.cmake +++ b/cmake/OpenCVModule.cmake @@ -1110,6 +1110,10 @@ function(ocv_add_accuracy_tests) set_target_properties(${the_target} PROPERTIES FOLDER "tests accuracy") endif() + if(OPENCV_TEST_BIGDATA) + ocv_append_target_property(${the_target} COMPILE_DEFINITIONS "OPENCV_TEST_BIGDATA=1") + endif() + if(NOT BUILD_opencv_world) _ocv_add_precompiled_headers(${the_target}) endif() diff --git a/cmake/OpenCVUtils.cmake b/cmake/OpenCVUtils.cmake index 309aad9d0b..b1a099e695 100644 --- a/cmake/OpenCVUtils.cmake +++ b/cmake/OpenCVUtils.cmake @@ -128,6 +128,16 @@ function(ocv_include_directories) include_directories(BEFORE ${__add_before}) endfunction() +function(ocv_append_target_property target prop) + get_target_property(val ${target} ${prop}) + if(val) + set(val "${val} ${ARGN}") + set_target_properties(${target} PROPERTIES ${prop} "${val}") + else() + set_target_properties(${target} PROPERTIES ${prop} "${ARGN}") + endif() +endfunction() + # adds include directories in such way that directories from the OpenCV source tree go first function(ocv_target_include_directories target) _ocv_fix_target(target) diff --git a/modules/core/src/convert.cpp b/modules/core/src/convert.cpp index f41bfa105f..1db170a926 100644 --- a/modules/core/src/convert.cpp +++ b/modules/core/src/convert.cpp @@ -50,6 +50,7 @@ #define CV_NEON 0 #endif +#define CV_SPLIT_MERGE_MAX_BLOCK_SIZE(cn) ((INT_MAX/4)/cn) // HAL implementation accepts 'int' len, so INT_MAX doesn't work here /****************************************************************************************\ * split & merge * @@ -93,8 +94,8 @@ void cv::split(const Mat& src, Mat* mv) SplitFunc func = getSplitFunc(depth); CV_Assert( func != 0 ); - int esz = (int)src.elemSize(), esz1 = (int)src.elemSize1(); - int blocksize0 = (BLOCK_SIZE + esz-1)/esz; + size_t esz = src.elemSize(), esz1 = src.elemSize1(); + size_t blocksize0 = (BLOCK_SIZE + esz-1)/esz; AutoBuffer _buf((cn+1)*(sizeof(Mat*) + sizeof(uchar*)) + 16); const Mat** arrays = (const Mat**)(uchar*)_buf; uchar** ptrs = (uchar**)alignPtr(arrays + cn + 1, 16); @@ -107,14 +108,15 @@ void cv::split(const Mat& src, Mat* mv) } NAryMatIterator it(arrays, ptrs, cn+1); - int total = (int)it.size, blocksize = cn <= 4 ? total : std::min(total, blocksize0); + size_t total = it.size; + size_t blocksize = std::min((size_t)CV_SPLIT_MERGE_MAX_BLOCK_SIZE(cn), cn <= 4 ? total : std::min(total, blocksize0)); for( size_t i = 0; i < it.nplanes; i++, ++it ) { - for( int j = 0; j < total; j += blocksize ) + for( size_t j = 0; j < total; j += blocksize ) { - int bsz = std::min(total - j, blocksize); - func( ptrs[0], &ptrs[1], bsz, cn ); + size_t bsz = std::min(total - j, blocksize); + func( ptrs[0], &ptrs[1], (int)bsz, cn ); if( j + blocksize < total ) { @@ -241,8 +243,11 @@ void cv::merge(const Mat* mv, size_t n, OutputArray _dst) return; } + MergeFunc func = getMergeFunc(depth); + CV_Assert( func != 0 ); + size_t esz = dst.elemSize(), esz1 = dst.elemSize1(); - int blocksize0 = (int)((BLOCK_SIZE + esz-1)/esz); + size_t blocksize0 = (int)((BLOCK_SIZE + esz-1)/esz); AutoBuffer _buf((cn+1)*(sizeof(Mat*) + sizeof(uchar*)) + 16); const Mat** arrays = (const Mat**)(uchar*)_buf; uchar** ptrs = (uchar**)alignPtr(arrays + cn + 1, 16); @@ -252,15 +257,15 @@ void cv::merge(const Mat* mv, size_t n, OutputArray _dst) arrays[k+1] = &mv[k]; NAryMatIterator it(arrays, ptrs, cn+1); - int total = (int)it.size, blocksize = cn <= 4 ? total : std::min(total, blocksize0); - MergeFunc func = getMergeFunc(depth); + size_t total = (int)it.size; + size_t blocksize = std::min((size_t)CV_SPLIT_MERGE_MAX_BLOCK_SIZE(cn), cn <= 4 ? total : std::min(total, blocksize0)); for( i = 0; i < it.nplanes; i++, ++it ) { - for( int j = 0; j < total; j += blocksize ) + for( size_t j = 0; j < total; j += blocksize ) { - int bsz = std::min(total - j, blocksize); - func( (const uchar**)&ptrs[1], ptrs[0], bsz, cn ); + size_t bsz = std::min(total - j, blocksize); + func( (const uchar**)&ptrs[1], ptrs[0], (int)bsz, cn ); if( j + blocksize < total ) { diff --git a/modules/core/test/test_mat.cpp b/modules/core/test/test_mat.cpp index e2de3249b6..1e64cdf886 100644 --- a/modules/core/test/test_mat.cpp +++ b/modules/core/test/test_mat.cpp @@ -1483,3 +1483,29 @@ TEST(Mat, regression_5991) EXPECT_EQ(sz[2], mat.size[2]); EXPECT_EQ(0, cvtest::norm(mat, Mat(3, sz, CV_8U, Scalar(1)), NORM_INF)); } + +#ifdef OPENCV_TEST_BIGDATA +TEST(Mat, regression_6696_BigData_8Gb) +{ + int width = 60000; + int height = 10000; + + Mat destImageBGR = Mat(height, width, CV_8UC3, Scalar(1, 2, 3, 0)); + Mat destImageA = Mat(height, width, CV_8UC1, Scalar::all(4)); + + vector planes; + split(destImageBGR, planes); + planes.push_back(destImageA); + merge(planes, destImageBGR); + + EXPECT_EQ(1, destImageBGR.at(0)[0]); + EXPECT_EQ(2, destImageBGR.at(0)[1]); + EXPECT_EQ(3, destImageBGR.at(0)[2]); + EXPECT_EQ(4, destImageBGR.at(0)[3]); + + EXPECT_EQ(1, destImageBGR.at(height-1, width-1)[0]); + EXPECT_EQ(2, destImageBGR.at(height-1, width-1)[1]); + EXPECT_EQ(3, destImageBGR.at(height-1, width-1)[2]); + EXPECT_EQ(4, destImageBGR.at(height-1, width-1)[3]); +} +#endif From 59fabb2767f3efd59fc626573d83fdd4c7cd574c Mon Sep 17 00:00:00 2001 From: Tomoaki Teshima Date: Thu, 23 Jun 2016 13:22:46 +0900 Subject: [PATCH 068/128] use NEON as same as SSE --- modules/objdetect/src/hog.cpp | 225 +++++++++++++++++++++++++++++++++- 1 file changed, 223 insertions(+), 2 deletions(-) diff --git a/modules/objdetect/src/hog.cpp b/modules/objdetect/src/hog.cpp index 77dd71200f..1055525a82 100644 --- a/modules/objdetect/src/hog.cpp +++ b/modules/objdetect/src/hog.cpp @@ -222,6 +222,17 @@ void HOGDescriptor::copyTo(HOGDescriptor& c) const c.signedGradient = signedGradient; } +#if CV_NEON +// replace of _mm_set_ps +inline float32x4_t vsetq_f32(float f0, float f1, float f2, float f3) +{ + float32x4_t a = vdupq_n_f32(f0); + a = vsetq_lane_f32(f1, a, 1); + a = vsetq_lane_f32(f2, a, 2); + a = vsetq_lane_f32(f3, a, 3); + return a; +} +#endif void HOGDescriptor::computeGradient(const Mat& img, Mat& grad, Mat& qangle, Size paddingTL, Size paddingBR) const { @@ -259,6 +270,21 @@ void HOGDescriptor::computeGradient(const Mat& img, Mat& grad, Mat& qangle, _mm_storeu_ps(_data + i, _mm_cvtepi32_ps(idx)); idx = _mm_add_epi32(idx, ifour); } +#elif CV_NEON + const int indeces[] = { 0, 1, 2, 3 }; + uint32x4_t idx = *(uint32x4_t*)indeces; + uint32x4_t ifour = vdupq_n_u32(4); + + float* const _data = &_lut(0, 0); + if( gammaCorrection ) + for( i = 0; i < 256; i++ ) + _lut(0,i) = std::sqrt((float)i); + else + for( i = 0; i < 256; i += 4 ) + { + vst1q_f32(_data + i, vcvtq_f32_u32(idx)); + idx = vaddq_u32 (idx, ifour); + } #else if( gammaCorrection ) for( i = 0; i < 256; i++ ) @@ -299,6 +325,10 @@ void HOGDescriptor::computeGradient(const Mat& img, Mat& grad, Mat& qangle, for ( ; x <= end - 4; x += 4) _mm_storeu_si128((__m128i*)(xmap + x), _mm_mullo_epi16(ithree, _mm_loadu_si128((const __m128i*)(xmap + x)))); +#elif CV_NEON + int32x4_t ithree = vdupq_n_s32(3); + for ( ; x <= end - 4; x += 4) + vst1q_s32(xmap + x, vmulq_s32(ithree, vld1q_s32(xmap + x))); #endif for ( ; x < end; ++x) xmap[x] *= 3; @@ -368,6 +398,45 @@ void HOGDescriptor::computeGradient(const Mat& img, Mat& grad, Mat& qangle, _mm_storeu_ps(dbuf + x, _dx2); _mm_storeu_ps(dbuf + x + width, _dy2); } +#elif CV_NEON + for( ; x <= width - 4; x += 4 ) + { + int x0 = xmap[x], x1 = xmap[x+1], x2 = xmap[x+2], x3 = xmap[x+3]; + typedef const uchar* const T; + T p02 = imgPtr + xmap[x+1], p00 = imgPtr + xmap[x-1]; + T p12 = imgPtr + xmap[x+2], p10 = imgPtr + xmap[x]; + T p22 = imgPtr + xmap[x+3], p20 = p02; + T p32 = imgPtr + xmap[x+4], p30 = p12; + + float32x4_t _dx0 = vsubq_f32(vsetq_f32(lut[p02[0]], lut[p12[0]], lut[p22[0]], lut[p32[0]]), + vsetq_f32(lut[p00[0]], lut[p10[0]], lut[p20[0]], lut[p30[0]])); + float32x4_t _dx1 = vsubq_f32(vsetq_f32(lut[p02[1]], lut[p12[1]], lut[p22[1]], lut[p32[1]]), + vsetq_f32(lut[p00[1]], lut[p10[1]], lut[p20[1]], lut[p30[1]])); + float32x4_t _dx2 = vsubq_f32(vsetq_f32(lut[p02[2]], lut[p12[2]], lut[p22[2]], lut[p32[2]]), + vsetq_f32(lut[p00[2]], lut[p10[2]], lut[p20[2]], lut[p30[2]])); + + float32x4_t _dy0 = vsubq_f32(vsetq_f32(lut[nextPtr[x0]], lut[nextPtr[x1]], lut[nextPtr[x2]], lut[nextPtr[x3]]), + vsetq_f32(lut[prevPtr[x0]], lut[prevPtr[x1]], lut[prevPtr[x2]], lut[prevPtr[x3]])); + float32x4_t _dy1 = vsubq_f32(vsetq_f32(lut[nextPtr[x0+1]], lut[nextPtr[x1+1]], lut[nextPtr[x2+1]], lut[nextPtr[x3+1]]), + vsetq_f32(lut[prevPtr[x0+1]], lut[prevPtr[x1+1]], lut[prevPtr[x2+1]], lut[prevPtr[x3+1]])); + float32x4_t _dy2 = vsubq_f32(vsetq_f32(lut[nextPtr[x0+2]], lut[nextPtr[x1+2]], lut[nextPtr[x2+2]], lut[nextPtr[x3+2]]), + vsetq_f32(lut[prevPtr[x0+2]], lut[prevPtr[x1+2]], lut[prevPtr[x2+2]], lut[prevPtr[x3+2]])); + + float32x4_t _mag0 = vaddq_f32(vmulq_f32(_dx0, _dx0), vmulq_f32(_dy0, _dy0)); + float32x4_t _mag1 = vaddq_f32(vmulq_f32(_dx1, _dx1), vmulq_f32(_dy1, _dy1)); + float32x4_t _mag2 = vaddq_f32(vmulq_f32(_dx2, _dx2), vmulq_f32(_dy2, _dy2)); + + uint32x4_t mask = vcgtq_f32(_mag2, _mag1); + _dx2 = vbslq_f32(mask, _dx2, _dx1); + _dy2 = vbslq_f32(mask, _dy2, _dy1); + + mask = vcgtq_f32(vmaxq_f32(_mag2, _mag1), _mag0); + _dx2 = vbslq_f32(mask, _dx2, _dx0); + _dy2 = vbslq_f32(mask, _dy2, _dy0); + + vst1q_f32(dbuf + x, _dx2); + vst1q_f32(dbuf + x + width, _dy2); + } #endif for( ; x < width; x++ ) { @@ -600,6 +669,19 @@ void HOGCache::init(const HOGDescriptor* _descriptor, idx = _mm_add_epi32(idx, ifour); _mm_storeu_ps(_di + i, t); } + #elif CV_NEON + const int a[] = { 0, 1, 2, 3 }; + int32x4_t idx = vld1q_s32(a); + float32x4_t _bw = vdupq_n_f32(bw), _bh = vdupq_n_f32(bh); + int32x4_t ifour = vdupq_n_s32(4); + + for (; i <= blockSize.height - 4; i += 4) + { + float32x4_t t = vsubq_f32(vcvtq_f32_s32(idx), _bh); + t = vmulq_f32(t, t); + idx = vaddq_s32(idx, ifour); + vst1q_f32(_di + i, t); + } #endif for ( ; i < blockSize.height; ++i) { @@ -617,6 +699,15 @@ void HOGCache::init(const HOGDescriptor* _descriptor, idx = _mm_add_epi32(idx, ifour); _mm_storeu_ps(_dj + j, t); } + #elif CV_NEON + idx = vld1q_s32(a); + for (; j <= blockSize.width - 4; j += 4) + { + float32x4_t t = vsubq_f32(vcvtq_f32_s32(idx), _bw); + t = vmulq_f32(t, t); + idx = vaddq_s32(idx, ifour); + vst1q_f32(_dj + j, t); + } #endif for ( ; j < blockSize.width; ++j) { @@ -839,6 +930,31 @@ const float* HOGCache::getBlock(Point pt, float* buf) t1 = hist[h1] + hist1[1]; hist[h0] = t0; hist[h1] = t1; } +#elif CV_NEON + float hist0[4], hist1[4]; + for( ; k < C2; k++ ) + { + const PixData& pk = _pixData[k]; + const float* const a = gradPtr + pk.gradOfs; + const uchar* const h = qanglePtr + pk.qangleOfs; + int h0 = h[0], h1 = h[1]; + + float32x4_t _a0 = vdupq_n_f32(a[0]), _a1 = vdupq_n_f32(a[1]); + float32x4_t _w = vmulq_f32(vdupq_n_f32(pk.gradWeight), vld1q_f32(pk.histWeights)); + + float32x4_t _h0 = vsetq_f32((blockHist + pk.histOfs[0])[h0], (blockHist + pk.histOfs[1])[h0], 0, 0); + float32x4_t _h1 = vsetq_f32((blockHist + pk.histOfs[0])[h1], (blockHist + pk.histOfs[1])[h1], 0, 0); + + float32x4_t _t0 = vmlaq_f32(_h0, _a0, _w), _t1 = vmlaq_f32(_h1, _a1, _w); + vst1q_f32(hist0, _t0); + vst1q_f32(hist1, _t1); + + (blockHist + pk.histOfs[0])[h0] = hist0[0]; + (blockHist + pk.histOfs[1])[h0] = hist0[1]; + + (blockHist + pk.histOfs[0])[h1] = hist1[0]; + (blockHist + pk.histOfs[1])[h1] = hist1[1]; + } #else for( ; k < C2; k++ ) { @@ -918,6 +1034,41 @@ const float* HOGCache::getBlock(Point pt, float* buf) // (pk.histOfs[2] + blockHist)[h1] = hist1[2]; // (pk.histOfs[3] + blockHist)[h1] = hist1[3]; } +#elif CV_NEON + for( ; k < C4; k++ ) + { + const PixData& pk = _pixData[k]; + const float* const a = gradPtr + pk.gradOfs; + const uchar* const h = qanglePtr + pk.qangleOfs; + int h0 = h[0], h1 = h[1]; + + float32x4_t _a0 = vdupq_n_f32(a[0]), _a1 = vdupq_n_f32(a[1]); + float32x4_t _w = vmulq_f32(vdupq_n_f32(pk.gradWeight), vld1q_f32(pk.histWeights)); + + float32x4_t _h0 = vsetq_f32((blockHist + pk.histOfs[0])[h0], + (blockHist + pk.histOfs[1])[h0], + (blockHist + pk.histOfs[2])[h0], + (blockHist + pk.histOfs[3])[h0]); + float32x4_t _h1 = vsetq_f32((blockHist + pk.histOfs[0])[h1], + (blockHist + pk.histOfs[1])[h1], + (blockHist + pk.histOfs[2])[h1], + (blockHist + pk.histOfs[3])[h1]); + + + float32x4_t _t0 = vmlaq_f32(_h0, _a0, _w), _t1 = vmlaq_f32(_h1, _a1, _w); + vst1q_f32(hist0, _t0); + vst1q_f32(hist1, _t1); + + (blockHist + pk.histOfs[0])[h0] = hist0[0]; + (blockHist + pk.histOfs[1])[h0] = hist0[1]; + (blockHist + pk.histOfs[2])[h0] = hist0[2]; + (blockHist + pk.histOfs[3])[h0] = hist0[3]; + + (blockHist + pk.histOfs[0])[h1] = hist1[0]; + (blockHist + pk.histOfs[1])[h1] = hist1[1]; + (blockHist + pk.histOfs[2])[h1] = hist1[2]; + (blockHist + pk.histOfs[3])[h1] = hist1[3]; + } #else for( ; k < C4; k++ ) { @@ -973,6 +1124,16 @@ void HOGCache::normalizeBlockHistogram(float* _hist) const s = _mm_add_ps(s, _mm_mul_ps(p0, p0)); } _mm_storeu_ps(partSum, s); +#elif CV_NEON + float32x4_t p0 = vld1q_f32(hist); + float32x4_t s = vmulq_f32(p0, p0); + + for (i = 4; i <= sz - 4; i += 4) + { + p0 = vld1q_f32(hist + i); + s = vaddq_f32(s, vmulq_f32(p0, p0)); + } + vst1q_f32(partSum, s); #else partSum[0] = 0.0f; partSum[1] = 0.0f; @@ -1014,6 +1175,25 @@ void HOGCache::normalizeBlockHistogram(float* _hist) const } _mm_storeu_ps(partSum, s); +#elif CV_NEON + float32x4_t _scale = vdupq_n_f32(scale); + static float32x4_t _threshold = vdupq_n_f32(thresh); + + float32x4_t p = vmulq_f32(_scale, vld1q_f32(hist)); + p = vminq_f32(p, _threshold); + s = vmulq_f32(p, p); + vst1q_f32(hist, p); + + for(i = 4 ; i <= sz - 4; i += 4) + { + p = vld1q_f32(hist + i); + p = vmulq_f32(p, _scale); + p = vminq_f32(p, _threshold); + s = vaddq_f32(s, vmulq_f32(p, p)); + vst1q_f32(hist + i, p); + } + + vst1q_f32(partSum, s); #else partSum[0] = 0.0f; partSum[1] = 0.0f; @@ -1048,6 +1228,13 @@ void HOGCache::normalizeBlockHistogram(float* _hist) const __m128 t = _mm_mul_ps(_scale2, _mm_loadu_ps(hist + i)); _mm_storeu_ps(hist + i, t); } +#elif CV_NEON + float32x4_t _scale2 = vdupq_n_f32(scale); + for ( ; i <= sz - 4; i += 4) + { + float32x4_t t = vmulq_f32(_scale2, vld1q_f32(hist + i)); + vst1q_f32(hist + i, t); + } #endif for ( ; i < sz; ++i) hist[i] *= scale; @@ -1489,7 +1676,7 @@ void HOGDescriptor::detect(const Mat& img, double rho = svmDetector.size() > dsize ? svmDetector[dsize] : 0; std::vector blockHist(blockHistogramSize); -#if CV_SSE2 +#if CV_SSE2 || CV_NEON float partSum[4]; #endif @@ -1535,6 +1722,23 @@ void HOGDescriptor::detect(const Mat& img, double t0 = partSum[0] + partSum[1]; double t1 = partSum[2] + partSum[3]; s += t0 + t1; +#elif CV_NEON + float32x4_t _vec = vld1q_f32(vec); + float32x4_t _svmVec = vld1q_f32(svmVec); + float32x4_t sum = vmulq_f32(_svmVec, _vec); + + for( k = 4; k <= blockHistogramSize - 4; k += 4 ) + { + _vec = vld1q_f32(vec + k); + _svmVec = vld1q_f32(svmVec + k); + + sum = vaddq_f32(sum, vmulq_f32(_vec, _svmVec)); + } + + vst1q_f32(partSum, sum); + double t0 = partSum[0] + partSum[1]; + double t1 = partSum[2] + partSum[3]; + s += t0 + t1; #else for( k = 0; k <= blockHistogramSize - 4; k += 4 ) s += vec[k]*svmVec[k] + vec[k+1]*svmVec[k+1] + @@ -3357,7 +3561,7 @@ void HOGDescriptor::detectROI(const cv::Mat& img, const std::vector & double rho = svmDetector.size() > dsize ? svmDetector[dsize] : 0; std::vector blockHist(blockHistogramSize); -#if CV_SSE2 +#if CV_SSE2 || CV_NEON float partSum[4]; #endif @@ -3401,6 +3605,23 @@ void HOGDescriptor::detectROI(const cv::Mat& img, const std::vector & double t0 = partSum[0] + partSum[1]; double t1 = partSum[2] + partSum[3]; s += t0 + t1; +#elif CV_NEON + float32x4_t _vec = vld1q_f32(vec); + float32x4_t _svmVec = vld1q_f32(svmVec); + float32x4_t sum = vmulq_f32(_svmVec, _vec); + + for( k = 4; k <= blockHistogramSize - 4; k += 4 ) + { + _vec = vld1q_f32(vec + k); + _svmVec = vld1q_f32(svmVec + k); + + sum = vaddq_f32(sum, vmulq_f32(_vec, _svmVec)); + } + + vst1q_f32(partSum, sum); + double t0 = partSum[0] + partSum[1]; + double t1 = partSum[2] + partSum[3]; + s += t0 + t1; #else for( k = 0; k <= blockHistogramSize - 4; k += 4 ) s += vec[k]*svmVec[k] + vec[k+1]*svmVec[k+1] + From 1095076d7fd5e7ff03aa6965c603a3ff36b115f0 Mon Sep 17 00:00:00 2001 From: Tomoaki Teshima Date: Tue, 21 Jun 2016 11:36:52 +0900 Subject: [PATCH 069/128] imgproc: speed up threshold of 64F version using NEON and SSE * use NEON under aarch64 only * check 64F version correctly --- modules/imgproc/src/thresh.cpp | 188 +++++++++++++++++++++++++++ modules/imgproc/test/test_thresh.cpp | 18 +-- 2 files changed, 197 insertions(+), 9 deletions(-) diff --git a/modules/imgproc/src/thresh.cpp b/modules/imgproc/src/thresh.cpp index 13f0fa284b..aa104d0cf3 100644 --- a/modules/imgproc/src/thresh.cpp +++ b/modules/imgproc/src/thresh.cpp @@ -915,6 +915,10 @@ thresh_64f(const Mat& _src, Mat& _dst, double thresh, double maxval, int type) size_t src_step = _src.step / sizeof(src[0]); size_t dst_step = _dst.step / sizeof(dst[0]); +#if CV_SSE2 + volatile bool useSIMD = checkHardwareSupport(CV_CPU_SSE); +#endif + if (_src.isContinuous() && _dst.isContinuous()) { roi.width *= roi.height; @@ -927,6 +931,45 @@ thresh_64f(const Mat& _src, Mat& _dst, double thresh, double maxval, int type) for (i = 0; i < roi.height; i++, src += src_step, dst += dst_step) { j = 0; +#if CV_SSE2 + if( useSIMD ) + { + __m128d thresh2 = _mm_set1_pd(thresh), maxval2 = _mm_set1_pd(maxval); + for( ; j <= roi.width - 8; j += 8 ) + { + __m128d v0, v1, v2, v3; + v0 = _mm_loadu_pd( src + j ); + v1 = _mm_loadu_pd( src + j + 2 ); + v2 = _mm_loadu_pd( src + j + 4 ); + v3 = _mm_loadu_pd( src + j + 6 ); + v0 = _mm_cmpgt_pd( v0, thresh2 ); + v1 = _mm_cmpgt_pd( v1, thresh2 ); + v2 = _mm_cmpgt_pd( v2, thresh2 ); + v3 = _mm_cmpgt_pd( v3, thresh2 ); + v0 = _mm_and_pd( v0, maxval2 ); + v1 = _mm_and_pd( v1, maxval2 ); + v2 = _mm_and_pd( v2, maxval2 ); + v3 = _mm_and_pd( v3, maxval2 ); + _mm_storeu_pd( dst + j, v0 ); + _mm_storeu_pd( dst + j + 2, v1 ); + _mm_storeu_pd( dst + j + 4, v2 ); + _mm_storeu_pd( dst + j + 6, v3 ); + } + } +#elif CV_NEON && defined(__aarch64__) + float64x2_t v_thresh = vdupq_n_f64(thresh); + uint64x2_t v_maxval = vreinterpretq_u64_f64(vdupq_n_f64(maxval)); + + for( ; j <= roi.width - 4; j += 4 ) + { + float64x2_t v_src0 = vld1q_f64(src + j); + float64x2_t v_src1 = vld1q_f64(src + j + 2); + uint64x2_t v_dst0 = vandq_u64(vcgtq_f64(v_src0, v_thresh), v_maxval); + uint64x2_t v_dst1 = vandq_u64(vcgtq_f64(v_src1, v_thresh), v_maxval); + vst1q_f64(dst + j, vreinterpretq_f64_u64(v_dst0)); + vst1q_f64(dst + j + 2, vreinterpretq_f64_u64(v_dst1)); + } +#endif for (; j < roi.width; j++) dst[j] = src[j] > thresh ? maxval : 0; @@ -938,6 +981,45 @@ thresh_64f(const Mat& _src, Mat& _dst, double thresh, double maxval, int type) { j = 0; +#if CV_SSE2 + if( useSIMD ) + { + __m128d thresh2 = _mm_set1_pd(thresh), maxval2 = _mm_set1_pd(maxval); + for( ; j <= roi.width - 8; j += 8 ) + { + __m128d v0, v1, v2, v3; + v0 = _mm_loadu_pd( src + j ); + v1 = _mm_loadu_pd( src + j + 2 ); + v2 = _mm_loadu_pd( src + j + 4 ); + v3 = _mm_loadu_pd( src + j + 6 ); + v0 = _mm_cmple_pd( v0, thresh2 ); + v1 = _mm_cmple_pd( v1, thresh2 ); + v2 = _mm_cmple_pd( v2, thresh2 ); + v3 = _mm_cmple_pd( v3, thresh2 ); + v0 = _mm_and_pd( v0, maxval2 ); + v1 = _mm_and_pd( v1, maxval2 ); + v2 = _mm_and_pd( v2, maxval2 ); + v3 = _mm_and_pd( v3, maxval2 ); + _mm_storeu_pd( dst + j, v0 ); + _mm_storeu_pd( dst + j + 2, v1 ); + _mm_storeu_pd( dst + j + 4, v2 ); + _mm_storeu_pd( dst + j + 6, v3 ); + } + } +#elif CV_NEON && defined(__aarch64__) + float64x2_t v_thresh = vdupq_n_f64(thresh); + uint64x2_t v_maxval = vreinterpretq_u64_f64(vdupq_n_f64(maxval)); + + for( ; j <= roi.width - 4; j += 4 ) + { + float64x2_t v_src0 = vld1q_f64(src + j); + float64x2_t v_src1 = vld1q_f64(src + j + 2); + uint64x2_t v_dst0 = vandq_u64(vcleq_f64(v_src0, v_thresh), v_maxval); + uint64x2_t v_dst1 = vandq_u64(vcleq_f64(v_src1, v_thresh), v_maxval); + vst1q_f64(dst + j, vreinterpretq_f64_u64(v_dst0)); + vst1q_f64(dst + j + 2, vreinterpretq_f64_u64(v_dst1)); + } +#endif for (; j < roi.width; j++) dst[j] = src[j] <= thresh ? maxval : 0; } @@ -948,6 +1030,40 @@ thresh_64f(const Mat& _src, Mat& _dst, double thresh, double maxval, int type) { j = 0; +#if CV_SSE2 + if( useSIMD ) + { + __m128d thresh2 = _mm_set1_pd(thresh); + for( ; j <= roi.width - 8; j += 8 ) + { + __m128d v0, v1, v2, v3; + v0 = _mm_loadu_pd( src + j ); + v1 = _mm_loadu_pd( src + j + 2 ); + v2 = _mm_loadu_pd( src + j + 4 ); + v3 = _mm_loadu_pd( src + j + 6 ); + v0 = _mm_min_pd( v0, thresh2 ); + v1 = _mm_min_pd( v1, thresh2 ); + v2 = _mm_min_pd( v2, thresh2 ); + v3 = _mm_min_pd( v3, thresh2 ); + _mm_storeu_pd( dst + j, v0 ); + _mm_storeu_pd( dst + j + 2, v1 ); + _mm_storeu_pd( dst + j + 4, v2 ); + _mm_storeu_pd( dst + j + 6, v3 ); + } + } +#elif CV_NEON && defined(__aarch64__) + float64x2_t v_thresh = vdupq_n_f64(thresh); + + for( ; j <= roi.width - 4; j += 4 ) + { + float64x2_t v_src0 = vld1q_f64(src + j); + float64x2_t v_src1 = vld1q_f64(src + j + 2); + float64x2_t v_dst0 = vminq_f64(v_src0, v_thresh); + float64x2_t v_dst1 = vminq_f64(v_src1, v_thresh); + vst1q_f64(dst + j, v_dst0); + vst1q_f64(dst + j + 2, v_dst1); + } +#endif for (; j < roi.width; j++) dst[j] = std::min(src[j], thresh); } @@ -958,6 +1074,42 @@ thresh_64f(const Mat& _src, Mat& _dst, double thresh, double maxval, int type) { j = 0; +#if CV_SSE2 + if( useSIMD ) + { + __m128d thresh2 = _mm_set1_pd(thresh); + for( ; j <= roi.width - 8; j += 8 ) + { + __m128d v0, v1, v2, v3; + v0 = _mm_loadu_pd( src + j ); + v1 = _mm_loadu_pd( src + j + 2 ); + v2 = _mm_loadu_pd( src + j + 4 ); + v3 = _mm_loadu_pd( src + j + 6 ); + v0 = _mm_and_pd( v0, _mm_cmpgt_pd(v0, thresh2)); + v1 = _mm_and_pd( v1, _mm_cmpgt_pd(v1, thresh2)); + v2 = _mm_and_pd( v2, _mm_cmpgt_pd(v2, thresh2)); + v3 = _mm_and_pd( v3, _mm_cmpgt_pd(v3, thresh2)); + _mm_storeu_pd( dst + j, v0 ); + _mm_storeu_pd( dst + j + 2, v1 ); + _mm_storeu_pd( dst + j + 4, v2 ); + _mm_storeu_pd( dst + j + 6, v3 ); + } + } +#elif CV_NEON && defined(__aarch64__) + float64x2_t v_thresh = vdupq_n_f64(thresh); + + for( ; j <= roi.width - 4; j += 4 ) + { + float64x2_t v_src0 = vld1q_f64(src + j); + float64x2_t v_src1 = vld1q_f64(src + j + 2); + uint64x2_t v_dst0 = vandq_u64(vcgtq_f64(v_src0, v_thresh), + vreinterpretq_u64_f64(v_src0)); + uint64x2_t v_dst1 = vandq_u64(vcgtq_f64(v_src1, v_thresh), + vreinterpretq_u64_f64(v_src1)); + vst1q_f64(dst + j, vreinterpretq_f64_u64(v_dst0)); + vst1q_f64(dst + j + 2, vreinterpretq_f64_u64(v_dst1)); + } +#endif for (; j < roi.width; j++) { double v = src[j]; @@ -971,6 +1123,42 @@ thresh_64f(const Mat& _src, Mat& _dst, double thresh, double maxval, int type) { j = 0; +#if CV_SSE2 + if( useSIMD ) + { + __m128d thresh2 = _mm_set1_pd(thresh); + for( ; j <= roi.width - 8; j += 8 ) + { + __m128d v0, v1, v2, v3; + v0 = _mm_loadu_pd( src + j ); + v1 = _mm_loadu_pd( src + j + 2 ); + v2 = _mm_loadu_pd( src + j + 4 ); + v3 = _mm_loadu_pd( src + j + 6 ); + v0 = _mm_and_pd( v0, _mm_cmple_pd(v0, thresh2)); + v1 = _mm_and_pd( v1, _mm_cmple_pd(v1, thresh2)); + v2 = _mm_and_pd( v2, _mm_cmple_pd(v2, thresh2)); + v3 = _mm_and_pd( v3, _mm_cmple_pd(v3, thresh2)); + _mm_storeu_pd( dst + j, v0 ); + _mm_storeu_pd( dst + j + 2, v1 ); + _mm_storeu_pd( dst + j + 4, v2 ); + _mm_storeu_pd( dst + j + 6, v3 ); + } + } +#elif CV_NEON && defined(__aarch64__) + float64x2_t v_thresh = vdupq_n_f64(thresh); + + for( ; j <= roi.width - 4; j += 4 ) + { + float64x2_t v_src0 = vld1q_f64(src + j); + float64x2_t v_src1 = vld1q_f64(src + j + 2); + uint64x2_t v_dst0 = vandq_u64(vcleq_f64(v_src0, v_thresh), + vreinterpretq_u64_f64(v_src0)); + uint64x2_t v_dst1 = vandq_u64(vcleq_f64(v_src1, v_thresh), + vreinterpretq_u64_f64(v_src1)); + vst1q_f64(dst + j, vreinterpretq_f64_u64(v_dst0)); + vst1q_f64(dst + j + 2, vreinterpretq_f64_u64(v_dst1)); + } +#endif for (; j < roi.width; j++) { double v = src[j]; diff --git a/modules/imgproc/test/test_thresh.cpp b/modules/imgproc/test/test_thresh.cpp index b7db66e989..1ba930a752 100644 --- a/modules/imgproc/test/test_thresh.cpp +++ b/modules/imgproc/test/test_thresh.cpp @@ -75,17 +75,17 @@ void CV_ThreshTest::get_test_array_types_and_sizes( int test_case_idx, vector >& sizes, vector >& types ) { RNG& rng = ts->get_rng(); - int depth = cvtest::randInt(rng) % 3, cn = cvtest::randInt(rng) % 4 + 1; + int depth = cvtest::randInt(rng) % 4, cn = cvtest::randInt(rng) % 4 + 1; cvtest::ArrayTest::get_test_array_types_and_sizes( test_case_idx, sizes, types ); - depth = depth == 0 ? CV_8U : depth == 1 ? CV_16S : CV_32F; + depth = depth == 0 ? CV_8U : depth == 1 ? CV_16S : depth == 2 ? CV_32F : CV_64F; types[INPUT][0] = types[OUTPUT][0] = types[REF_OUTPUT][0] = CV_MAKETYPE(depth,cn); thresh_type = cvtest::randInt(rng) % 5; if( depth == CV_8U ) { - thresh_val = (float)(cvtest::randReal(rng)*350. - 50.); - max_val = (float)(cvtest::randReal(rng)*350. - 50.); + thresh_val = (cvtest::randReal(rng)*350. - 50.); + max_val = (cvtest::randReal(rng)*350. - 50.); if( cvtest::randInt(rng)%4 == 0 ) max_val = 255.f; } @@ -93,15 +93,15 @@ void CV_ThreshTest::get_test_array_types_and_sizes( int test_case_idx, { float min_val = SHRT_MIN-100.f; max_val = SHRT_MAX+100.f; - thresh_val = (float)(cvtest::randReal(rng)*(max_val - min_val) + min_val); - max_val = (float)(cvtest::randReal(rng)*(max_val - min_val) + min_val); + thresh_val = (cvtest::randReal(rng)*(max_val - min_val) + min_val); + max_val = (cvtest::randReal(rng)*(max_val - min_val) + min_val); if( cvtest::randInt(rng)%4 == 0 ) - max_val = (float)SHRT_MAX; + max_val = (double)SHRT_MAX; } else { - thresh_val = (float)(cvtest::randReal(rng)*1000. - 500.); - max_val = (float)(cvtest::randReal(rng)*1000. - 500.); + thresh_val = (cvtest::randReal(rng)*1000. - 500.); + max_val = (cvtest::randReal(rng)*1000. - 500.); } } From e233f7d199e5785dfc0743cd27766adbf500d438 Mon Sep 17 00:00:00 2001 From: berak Date: Sun, 10 Jul 2016 11:17:00 +0200 Subject: [PATCH 070/128] py_tutorials: fix cv2.findContours return val --- .../py_contour_features/py_contour_features.markdown | 2 +- .../py_contours_more_functions.markdown | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/doc/py_tutorials/py_imgproc/py_contours/py_contour_features/py_contour_features.markdown b/doc/py_tutorials/py_imgproc/py_contours/py_contour_features/py_contour_features.markdown index 06ac8e4a5f..237725ea42 100644 --- a/doc/py_tutorials/py_imgproc/py_contours/py_contour_features/py_contour_features.markdown +++ b/doc/py_tutorials/py_imgproc/py_contours/py_contour_features/py_contour_features.markdown @@ -23,7 +23,7 @@ import numpy as np img = cv2.imread('star.jpg',0) ret,thresh = cv2.threshold(img,127,255,0) -contours,hierarchy = cv2.findContours(thresh, 1, 2) +im2,contours,hierarchy = cv2.findContours(thresh, 1, 2) cnt = contours[0] M = cv2.moments(cnt) diff --git a/doc/py_tutorials/py_imgproc/py_contours/py_contours_more_functions/py_contours_more_functions.markdown b/doc/py_tutorials/py_imgproc/py_contours/py_contours_more_functions/py_contours_more_functions.markdown index 66ab00613b..2a96cb0ea0 100644 --- a/doc/py_tutorials/py_imgproc/py_contours/py_contours_more_functions/py_contours_more_functions.markdown +++ b/doc/py_tutorials/py_imgproc/py_contours/py_contours_more_functions/py_contours_more_functions.markdown @@ -38,8 +38,8 @@ import numpy as np img = cv2.imread('star.jpg') img_gray = cv2.cvtColor(img,cv2.COLOR_BGR2GRAY) -ret, thresh = cv2.threshold(img_gray, 127, 255,0) -contours,hierarchy = cv2.findContours(thresh,2,1) +ret,thresh = cv2.threshold(img_gray, 127, 255,0) +im2,contours,hierarchy = cv2.findContours(thresh,2,1) cnt = contours[0] hull = cv2.convexHull(cnt,returnPoints = False) @@ -93,9 +93,9 @@ img2 = cv2.imread('star2.jpg',0) ret, thresh = cv2.threshold(img1, 127, 255,0) ret, thresh2 = cv2.threshold(img2, 127, 255,0) -contours,hierarchy = cv2.findContours(thresh,2,1) +im2,contours,hierarchy = cv2.findContours(thresh,2,1) cnt1 = contours[0] -contours,hierarchy = cv2.findContours(thresh2,2,1) +im2,contours,hierarchy = cv2.findContours(thresh2,2,1) cnt2 = contours[0] ret = cv2.matchShapes(cnt1,cnt2,1,0.0) From 479f93397004d95da141b81b56e7c75187d142df Mon Sep 17 00:00:00 2001 From: Jan Starzynski Date: Mon, 11 Jul 2016 09:27:59 +0200 Subject: [PATCH 071/128] get/put: more type-safety and code unification using templates --- modules/java/generator/src/cpp/Mat.cpp | 250 ++++++++----------------- 1 file changed, 77 insertions(+), 173 deletions(-) diff --git a/modules/java/generator/src/cpp/Mat.cpp b/modules/java/generator/src/cpp/Mat.cpp index d8483cd02f..c85a3d7400 100644 --- a/modules/java/generator/src/cpp/Mat.cpp +++ b/modules/java/generator/src/cpp/Mat.cpp @@ -1815,6 +1815,29 @@ JNIEXPORT jint JNICALL Java_org_opencv_core_Mat_nPutD } // extern "C" +namespace { + /// map java-array-types to assigned data + template struct JavaOpenCVTrait; + +/// less typing for specialisations +#define JOCvT(t,s,c1,c2) \ + template<> struct JavaOpenCVTrait { \ + typedef t value_type; /* type of array element */ \ + static const char get[]; /* name of getter */ \ + static const char put[]; /* name of putter */ \ + enum {cvtype_1 = c1, cvtype_2 = c2 }; /* allowed OpenCV-types */ \ + }; \ + const char JavaOpenCVTrait::get[] = "Mat::nGet" s "()"; \ + const char JavaOpenCVTrait::put[] = "Mat::nPut" s "()" + + JOCvT(jbyte, "B", CV_8U, CV_8S); + JOCvT(jshort, "S", CV_16U, CV_16S); + JOCvT(jint, "I", CV_32S, CV_32S); + JOCvT(jfloat, "F", CV_32F, CV_32F); + JOCvT(jdouble, "D", CV_64F, CV_64F); +#undef JOCvT +} + template static int mat_put(cv::Mat* m, int row, int col, int count, char* buff) { if(! m) return 0; @@ -1845,6 +1868,28 @@ template static int mat_put(cv::Mat* m, int row, int col, int count, return res; } +template static jint java_mat_put(JNIEnv* env, jlong self, jint row, jint col, jint count, ARRAY vals) +{ + static const char *method_name = JavaOpenCVTrait::put; + try { + LOGD("%s", method_name); + cv::Mat* me = (cv::Mat*) self; + if(! self) return 0; // no native object behind + if(me->depth() != JavaOpenCVTrait::cvtype_1 && me->depth() != JavaOpenCVTrait::cvtype_2) return 0; // incompatible type + if(me->rows<=row || me->cols<=col) return 0; // indexes out of range + + char* values = (char*)env->GetPrimitiveArrayCritical(vals, 0); + int res = mat_put::value_type>(me, row, col, count, values); + env->ReleasePrimitiveArrayCritical(vals, values, JNI_ABORT); + return res; + } catch(const std::exception &e) { + throwJavaException(env, &e, method_name); + } catch (...) { + throwJavaException(env, 0, method_name); + } + + return 0; +} extern "C" { @@ -1854,25 +1899,7 @@ JNIEXPORT jint JNICALL Java_org_opencv_core_Mat_nPutB JNIEXPORT jint JNICALL Java_org_opencv_core_Mat_nPutB (JNIEnv* env, jclass, jlong self, jint row, jint col, jint count, jbyteArray vals) { - static const char method_name[] = "Mat::nPutB()"; - try { - LOGD("%s", method_name); - cv::Mat* me = (cv::Mat*) self; - if(! self) return 0; // no native object behind - if(me->depth() != CV_8U && me->depth() != CV_8S) return 0; // incompatible type - if(me->rows<=row || me->cols<=col) return 0; // indexes out of range - - char* values = (char*)env->GetPrimitiveArrayCritical(vals, 0); - int res = mat_put(me, row, col, count, values); - env->ReleasePrimitiveArrayCritical(vals, values, 0); - return res; - } catch(const std::exception &e) { - throwJavaException(env, &e, method_name); - } catch (...) { - throwJavaException(env, 0, method_name); - } - - return 0; + return java_mat_put(env, self, row, col, count, vals); } JNIEXPORT jint JNICALL Java_org_opencv_core_Mat_nPutS @@ -1881,25 +1908,7 @@ JNIEXPORT jint JNICALL Java_org_opencv_core_Mat_nPutS JNIEXPORT jint JNICALL Java_org_opencv_core_Mat_nPutS (JNIEnv* env, jclass, jlong self, jint row, jint col, jint count, jshortArray vals) { - static const char method_name[] = "Mat::nPutS()"; - try { - LOGD("%s", method_name); - cv::Mat* me = (cv::Mat*) self; - if(! self) return 0; // no native object behind - if(me->depth() != CV_16U && me->depth() != CV_16S) return 0; // incompatible type - if(me->rows<=row || me->cols<=col) return 0; // indexes out of range - - char* values = (char*)env->GetPrimitiveArrayCritical(vals, 0); - int res = mat_put(me, row, col, count, values); - env->ReleasePrimitiveArrayCritical(vals, values, 0); - return res; - } catch(const std::exception &e) { - throwJavaException(env, &e, method_name); - } catch (...) { - throwJavaException(env, 0, method_name); - } - - return 0; + return java_mat_put(env, self, row, col, count, vals); } JNIEXPORT jint JNICALL Java_org_opencv_core_Mat_nPutI @@ -1908,25 +1917,7 @@ JNIEXPORT jint JNICALL Java_org_opencv_core_Mat_nPutI JNIEXPORT jint JNICALL Java_org_opencv_core_Mat_nPutI (JNIEnv* env, jclass, jlong self, jint row, jint col, jint count, jintArray vals) { - static const char method_name[] = "Mat::nPutI()"; - try { - LOGD("%s", method_name); - cv::Mat* me = (cv::Mat*) self; - if(! self) return 0; // no native object behind - if(me->depth() != CV_32S) return 0; // incompatible type - if(me->rows<=row || me->cols<=col) return 0; // indexes out of range - - char* values = (char*)env->GetPrimitiveArrayCritical(vals, 0); - int res = mat_put(me, row, col, count, values); - env->ReleasePrimitiveArrayCritical(vals, values, 0); - return res; - } catch(const std::exception &e) { - throwJavaException(env, &e, method_name); - } catch (...) { - throwJavaException(env, 0, method_name); - } - - return 0; + return java_mat_put(env, self, row, col, count, vals); } JNIEXPORT jint JNICALL Java_org_opencv_core_Mat_nPutF @@ -1935,31 +1926,12 @@ JNIEXPORT jint JNICALL Java_org_opencv_core_Mat_nPutF JNIEXPORT jint JNICALL Java_org_opencv_core_Mat_nPutF (JNIEnv* env, jclass, jlong self, jint row, jint col, jint count, jfloatArray vals) { - static const char method_name[] = "Mat::nPutF()"; - try { - LOGD("%s", method_name); - cv::Mat* me = (cv::Mat*) self; - if(! self) return 0; // no native object behind - if(me->depth() != CV_32F) return 0; // incompatible type - if(me->rows<=row || me->cols<=col) return 0; // indexes out of range - - char* values = (char*)env->GetPrimitiveArrayCritical(vals, 0); - int res = mat_put(me, row, col, count, values); - env->ReleasePrimitiveArrayCritical(vals, values, 0); - return res; - } catch(const std::exception &e) { - throwJavaException(env, &e, method_name); - } catch (...) { - throwJavaException(env, 0, method_name); - } - - return 0; + return java_mat_put(env, self, row, col, count, vals); } - } // extern "C" -template int mat_get(cv::Mat* m, int row, int col, int count, char* buff) +template static int mat_get(cv::Mat* m, int row, int col, int count, char* buff) { if(! m) return 0; if(! buff) return 0; @@ -1989,6 +1961,28 @@ template int mat_get(cv::Mat* m, int row, int col, int count, char* return res; } +template static jint java_mat_get(JNIEnv* env, jlong self, jint row, jint col, jint count, ARRAY vals) { + static const char *method_name = JavaOpenCVTrait::get; + try { + LOGD("%s", method_name); + cv::Mat* me = (cv::Mat*) self; + if(! self) return 0; // no native object behind + if(me->depth() != JavaOpenCVTrait::cvtype_1 && me->depth() != JavaOpenCVTrait::cvtype_2) return 0; // incompatible type + if(me->rows<=row || me->cols<=col) return 0; // indexes out of range + + char* values = (char*)env->GetPrimitiveArrayCritical(vals, 0); + int res = mat_get::value_type>(me, row, col, count, values); + env->ReleasePrimitiveArrayCritical(vals, values, 0); + return res; + } catch(const std::exception &e) { + throwJavaException(env, &e, method_name); + } catch (...) { + throwJavaException(env, 0, method_name); + } + + return 0; +} + extern "C" { JNIEXPORT jint JNICALL Java_org_opencv_core_Mat_nGetB @@ -1997,25 +1991,7 @@ JNIEXPORT jint JNICALL Java_org_opencv_core_Mat_nGetB JNIEXPORT jint JNICALL Java_org_opencv_core_Mat_nGetB (JNIEnv* env, jclass, jlong self, jint row, jint col, jint count, jbyteArray vals) { - static const char method_name[] = "Mat::nGetB()"; - try { - LOGD("%s", method_name); - cv::Mat* me = (cv::Mat*) self; - if(! self) return 0; // no native object behind - if(me->depth() != CV_8U && me->depth() != CV_8S) return 0; // incompatible type - if(me->rows<=row || me->cols<=col) return 0; // indexes out of range - - char* values = (char*)env->GetPrimitiveArrayCritical(vals, 0); - int res = mat_get(me, row, col, count, values); - env->ReleasePrimitiveArrayCritical(vals, values, 0); - return res; - } catch(const std::exception &e) { - throwJavaException(env, &e, method_name); - } catch (...) { - throwJavaException(env, 0, method_name); - } - - return 0; + return java_mat_get(env, self, row, col, count, vals); } JNIEXPORT jint JNICALL Java_org_opencv_core_Mat_nGetS @@ -2024,25 +2000,7 @@ JNIEXPORT jint JNICALL Java_org_opencv_core_Mat_nGetS JNIEXPORT jint JNICALL Java_org_opencv_core_Mat_nGetS (JNIEnv* env, jclass, jlong self, jint row, jint col, jint count, jshortArray vals) { - static const char method_name[] = "Mat::nGetS()"; - try { - LOGD("%s", method_name); - cv::Mat* me = (cv::Mat*) self; - if(! self) return 0; // no native object behind - if(me->depth() != CV_16U && me->depth() != CV_16S) return 0; // incompatible type - if(me->rows<=row || me->cols<=col) return 0; // indexes out of range - - char* values = (char*)env->GetPrimitiveArrayCritical(vals, 0); - int res = mat_get(me, row, col, count, values); - env->ReleasePrimitiveArrayCritical(vals, values, 0); - return res; - } catch(const std::exception &e) { - throwJavaException(env, &e, method_name); - } catch (...) { - throwJavaException(env, 0, method_name); - } - - return 0; + return java_mat_get(env, self, row, col, count, vals); } JNIEXPORT jint JNICALL Java_org_opencv_core_Mat_nGetI @@ -2051,25 +2009,7 @@ JNIEXPORT jint JNICALL Java_org_opencv_core_Mat_nGetI JNIEXPORT jint JNICALL Java_org_opencv_core_Mat_nGetI (JNIEnv* env, jclass, jlong self, jint row, jint col, jint count, jintArray vals) { - static const char method_name[] = "Mat::nGetI()"; - try { - LOGD("%s", method_name); - cv::Mat* me = (cv::Mat*) self; - if(! self) return 0; // no native object behind - if(me->depth() != CV_32S) return 0; // incompatible type - if(me->rows<=row || me->cols<=col) return 0; // indexes out of range - - char* values = (char*)env->GetPrimitiveArrayCritical(vals, 0); - int res = mat_get(me, row, col, count, values); - env->ReleasePrimitiveArrayCritical(vals, values, 0); - return res; - } catch(const std::exception &e) { - throwJavaException(env, &e, method_name); - } catch (...) { - throwJavaException(env, 0, method_name); - } - - return 0; + return java_mat_get(env, self, row, col, count, vals); } JNIEXPORT jint JNICALL Java_org_opencv_core_Mat_nGetF @@ -2078,25 +2018,7 @@ JNIEXPORT jint JNICALL Java_org_opencv_core_Mat_nGetF JNIEXPORT jint JNICALL Java_org_opencv_core_Mat_nGetF (JNIEnv* env, jclass, jlong self, jint row, jint col, jint count, jfloatArray vals) { - static const char method_name[] = "Mat::nGetF()"; - try { - LOGD("%s", method_name); - cv::Mat* me = (cv::Mat*) self; - if(! self) return 0; // no native object behind - if(me->depth() != CV_32F) return 0; // incompatible type - if(me->rows<=row || me->cols<=col) return 0; // indexes out of range - - char* values = (char*)env->GetPrimitiveArrayCritical(vals, 0); - int res = mat_get(me, row, col, count, values); - env->ReleasePrimitiveArrayCritical(vals, values, 0); - return res; - } catch(const std::exception &e) { - throwJavaException(env, &e, method_name); - } catch (...) { - throwJavaException(env, 0, method_name); - } - - return 0; + return java_mat_get(env, self, row, col, count, vals); } JNIEXPORT jint JNICALL Java_org_opencv_core_Mat_nGetD @@ -2105,25 +2027,7 @@ JNIEXPORT jint JNICALL Java_org_opencv_core_Mat_nGetD JNIEXPORT jint JNICALL Java_org_opencv_core_Mat_nGetD (JNIEnv* env, jclass, jlong self, jint row, jint col, jint count, jdoubleArray vals) { - static const char method_name[] = "Mat::nGetD()"; - try { - LOGD("%s", method_name); - cv::Mat* me = (cv::Mat*) self; - if(! self) return 0; // no native object behind - if(me->depth() != CV_64F) return 0; // incompatible type - if(me->rows<=row || me->cols<=col) return 0; // indexes out of range - - char* values = (char*)env->GetPrimitiveArrayCritical(vals, 0); - int res = mat_get(me, row, col, count, values); - env->ReleasePrimitiveArrayCritical(vals, values, 0); - return res; - } catch(const std::exception &e) { - throwJavaException(env, &e, method_name); - } catch (...) { - throwJavaException(env, 0, method_name); - } - - return 0; + return java_mat_get(env, self, row, col, count, vals); } JNIEXPORT jdoubleArray JNICALL Java_org_opencv_core_Mat_nGet From c1b90a1c2269283881df1c29b7abcdf007a7082b Mon Sep 17 00:00:00 2001 From: Pavel Rojtberg Date: Tue, 26 Apr 2016 12:15:53 +0200 Subject: [PATCH 072/128] fisheye: add CALIB_FIX_PRINCIPAL_POINT --- modules/calib3d/include/opencv2/calib3d.hpp | 25 ++++++++++++--------- modules/calib3d/src/fisheye.cpp | 4 ++-- 2 files changed, 16 insertions(+), 13 deletions(-) diff --git a/modules/calib3d/include/opencv2/calib3d.hpp b/modules/calib3d/include/opencv2/calib3d.hpp index e26e8c13f9..618f18e43f 100644 --- a/modules/calib3d/include/opencv2/calib3d.hpp +++ b/modules/calib3d/include/opencv2/calib3d.hpp @@ -1756,15 +1756,16 @@ namespace fisheye //! @{ enum{ - CALIB_USE_INTRINSIC_GUESS = 1, - CALIB_RECOMPUTE_EXTRINSIC = 2, - CALIB_CHECK_COND = 4, - CALIB_FIX_SKEW = 8, - CALIB_FIX_K1 = 16, - CALIB_FIX_K2 = 32, - CALIB_FIX_K3 = 64, - CALIB_FIX_K4 = 128, - CALIB_FIX_INTRINSIC = 256 + CALIB_USE_INTRINSIC_GUESS = 1 << 0, + CALIB_RECOMPUTE_EXTRINSIC = 1 << 1, + CALIB_CHECK_COND = 1 << 2, + CALIB_FIX_SKEW = 1 << 3, + CALIB_FIX_K1 = 1 << 4, + CALIB_FIX_K2 = 1 << 5, + CALIB_FIX_K3 = 1 << 6, + CALIB_FIX_K4 = 1 << 7, + CALIB_FIX_INTRINSIC = 1 << 8, + CALIB_FIX_PRINCIPAL_POINT = 1 << 9 }; /** @brief Projects points using fisheye model @@ -1914,8 +1915,10 @@ namespace fisheye of intrinsic optimization. - **fisheye::CALIB_CHECK_COND** The functions will check validity of condition number. - **fisheye::CALIB_FIX_SKEW** Skew coefficient (alpha) is set to zero and stay zero. - - **fisheye::CALIB_FIX_K1..4** Selected distortion coefficients are set to zeros and stay - zero. + - **fisheye::CALIB_FIX_K1..fisheye::CALIB_FIX_K4** Selected distortion coefficients + are set to zeros and stay zero. + - **fisheye::CALIB_FIX_PRINCIPAL_POINT** The principal point is not changed during the global +optimization. It stays at the center or at a different location specified when CALIB_USE_INTRINSIC_GUESS is set too. @param criteria Termination criteria for the iterative optimization algorithm. */ CV_EXPORTS_W double calibrate(InputArrayOfArrays objectPoints, InputArrayOfArrays imagePoints, const Size& image_size, diff --git a/modules/calib3d/src/fisheye.cpp b/modules/calib3d/src/fisheye.cpp index 650121b46d..8120b6ed46 100644 --- a/modules/calib3d/src/fisheye.cpp +++ b/modules/calib3d/src/fisheye.cpp @@ -709,8 +709,8 @@ double cv::fisheye::calibrate(InputArrayOfArrays objectPoints, InputArrayOfArray finalParam.isEstimate[0] = 1; finalParam.isEstimate[1] = 1; - finalParam.isEstimate[2] = 1; - finalParam.isEstimate[3] = 1; + finalParam.isEstimate[2] = flags & CALIB_FIX_PRINCIPAL_POINT ? 0 : 1; + finalParam.isEstimate[3] = flags & CALIB_FIX_PRINCIPAL_POINT ? 0 : 1; finalParam.isEstimate[4] = flags & CALIB_FIX_SKEW ? 0 : 1; finalParam.isEstimate[5] = flags & CALIB_FIX_K1 ? 0 : 1; finalParam.isEstimate[6] = flags & CALIB_FIX_K2 ? 0 : 1; From 0ed250cb5d8ffa68a78526d50c6fc90ba2999999 Mon Sep 17 00:00:00 2001 From: Arthur Cinader Date: Mon, 11 Jul 2016 11:35:50 -0400 Subject: [PATCH 073/128] Implement PR feedback: 1. Explain grayscale input still read as three channel 2. Fix typo 3. Add more details to image match explanation to include the use of masks --- .../template_matching.markdown | 23 ++++++++++++------- .../MatchTemplate_Demo.cpp | 4 ++-- 2 files changed, 17 insertions(+), 10 deletions(-) diff --git a/doc/tutorials/imgproc/histograms/template_matching/template_matching.markdown b/doc/tutorials/imgproc/histograms/template_matching/template_matching.markdown index 5196f424a8..bdba55a799 100644 --- a/doc/tutorials/imgproc/histograms/template_matching/template_matching.markdown +++ b/doc/tutorials/imgproc/histograms/template_matching/template_matching.markdown @@ -20,7 +20,7 @@ Template matching is a technique for finding areas of an image that match (are s template image (patch). While the patch must be a rectangle it may be that not all of the -rectangle is relevent. In such a case, a mask can be used to isolate the portion of the patch +rectangle is relevant. In such a case, a mask can be used to isolate the portion of the patch that should be used to find the match. ### How does it work? @@ -60,7 +60,7 @@ that should be used to find the match. -# **Source image (I):** The image in which we expect to find a match to the template image -# **Template image (T):** The patch image which will be compared to the template image - -# **Mask image (M):** The mask, a greyscale image that masks the template + -# **Mask image (M):** The mask, a grayscale image that masks the template - Only two matching methods currently accept a mask: CV_TM_SQDIFF and CV_TM_CCORR_NORMED (see @@ -70,10 +70,13 @@ that should be used to find the match. - The mask must have the same dimensions as the template -- The mask should be a greyscale image where each pixel contains some value from black to white. +- The mask should be a grayscale image where each pixel contains some value from black to white. Pixels that are white are fully included in calculating the best match. Pixels that are black are excluded from the match. A value between black and white will include some of - the match proportion to how dark the pixel is. + the match in proportion to how dark the pixel is. Although the image should be a grayscale whose + output from the file command should look something like: "PNG image data, 128 x 128, 8-bit gray + +alpha, non-interlaced", opencv will read the image into an rgb matrix that will be applied + during the image match. ![](images/Template_Matching_Mask_Example.jpg) @@ -140,10 +143,14 @@ Explanation int match_method; int max_Trackbar = 5; @endcode --# Load the source image and template: +-# Load the source image, template, and optionally, if supported for the matching method, a mask: @code{.cpp} - img = imread( argv[1], IMREAD_COLOR ); - templ = imread( argv[2], IMREAD_COLOR ); + bool method_accepts_mask = (CV_TM_SQDIFF == match_method || match_method == CV_TM_CCORR_NORMED); + if (use_mask && method_accepts_mask) + { matchTemplate( img, templ, result, match_method, mask); } + else + { matchTemplate( img, templ, result, match_method); } + @endcode -# Create the windows to show the results: @code{.cpp} @@ -177,7 +184,7 @@ Explanation @endcode -# Perform the template matching operation: @code{.cpp} - bool method_accepts_mask = CV_TM_SQDIFF == match_method || match_method == CV_TM_CCORR_NORMED; + bool method_accepts_mask = (CV_TM_SQDIFF == match_method || match_method == CV_TM_CCORR_NORMED); if (use_mask && method_accepts_mask) { matchTemplate( img, templ, result, match_method, mask); } else diff --git a/samples/cpp/tutorial_code/Histograms_Matching/MatchTemplate_Demo.cpp b/samples/cpp/tutorial_code/Histograms_Matching/MatchTemplate_Demo.cpp index 7cd07a5f03..1042e2cff2 100644 --- a/samples/cpp/tutorial_code/Histograms_Matching/MatchTemplate_Demo.cpp +++ b/samples/cpp/tutorial_code/Histograms_Matching/MatchTemplate_Demo.cpp @@ -42,7 +42,7 @@ int main( int argc, char** argv ) if(argc > 3) { use_mask = true; - mask = imread(argv[3], IMREAD_COLOR); + mask = imread( argv[3], IMREAD_COLOR ); } if(img.empty() || templ.empty() || (use_mask && mask.empty())) @@ -82,7 +82,7 @@ void MatchingMethod( int, void* ) result.create( result_rows, result_cols, CV_32FC1 ); /// Do the Matching and Normalize - bool method_accepts_mask = CV_TM_SQDIFF == match_method || match_method == CV_TM_CCORR_NORMED; + bool method_accepts_mask = (CV_TM_SQDIFF == match_method || match_method == CV_TM_CCORR_NORMED); if (use_mask && method_accepts_mask) { matchTemplate( img, templ, result, match_method, mask); } else From bcc90106305609bddad93bb4c61390f77d83c3d7 Mon Sep 17 00:00:00 2001 From: Ilya Lavrenov Date: Tue, 12 Jul 2016 11:53:51 +0300 Subject: [PATCH 074/128] fixed memory leak in flann tests --- modules/flann/src/miniflann.cpp | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/modules/flann/src/miniflann.cpp b/modules/flann/src/miniflann.cpp index 7d81438dbc..b7661752c0 100644 --- a/modules/flann/src/miniflann.cpp +++ b/modules/flann/src/miniflann.cpp @@ -318,7 +318,19 @@ buildIndex_(void*& index, const Mat& data, const IndexParams& params, const Dist ::cvflann::Matrix dataset((ElementType*)data.data, data.rows, data.cols); IndexType* _index = new IndexType(dataset, get_params(params), dist); - _index->buildIndex(); + + try + { + _index->buildIndex(); + } + catch (...) + { + delete _index; + _index = NULL; + + throw; + } + index = _index; } From ddc0b42bc389ae17733fed3189acedb6791953dc Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Tue, 14 Jun 2016 16:01:36 +0300 Subject: [PATCH 075/128] migration: github.com/opencv/opencv --- .github/ISSUE_TEMPLATE.md | 2 +- 3rdparty/ffmpeg/ffmpeg.cmake | 2 +- 3rdparty/ippicv/downloader.cmake | 2 +- CONTRIBUTING.md | 2 +- README.md | 4 ++-- doc/Doxyfile.in | 2 +- .../py_feature2d/py_brief/py_brief.markdown | 2 +- .../py_sift_intro/py_sift_intro.markdown | 2 +- .../py_setup/py_intro/py_intro.markdown | 2 +- .../py_setup_in_fedora/py_setup_in_fedora.markdown | 2 +- .../py_setup_in_windows.markdown | 2 +- .../camera_calibration/camera_calibration.markdown | 6 +++--- .../interactive_calibration.markdown | 2 +- .../basic_geometric_drawing.markdown | 2 +- .../discrete_fourier_transform.markdown | 4 ++-- .../file_input_output_with_xml_yml.markdown | 2 +- .../how_to_scan_images/how_to_scan_images.markdown | 2 +- .../how_to_use_ippa_conversion.markdown | 2 +- .../interoperability_with_OpenCV_1.markdown | 4 ++-- .../mat_mask_operations.markdown | 2 +- .../mat_the_basic_image_container.markdown | 2 +- .../corner_subpixeles/corner_subpixeles.markdown | 2 +- .../generic_corner_detector.markdown | 2 +- .../good_features_to_track.markdown | 2 +- .../harris_detector/harris_detector.markdown | 2 +- .../gpu_basics_similarity.markdown | 2 +- .../erosion_dilatation/erosion_dilatation.markdown | 2 +- .../gausian_median_blur_bilateral_filter.markdown | 2 +- .../back_projection/back_projection.markdown | 6 +++--- .../histogram_calculation.markdown | 2 +- .../histogram_comparison.markdown | 2 +- .../histogram_equalization.markdown | 2 +- .../template_matching/template_matching.markdown | 2 +- .../canny_detector/canny_detector.markdown | 2 +- .../copyMakeBorder/copyMakeBorder.markdown | 2 +- .../distance_transform.markdown | 2 +- .../imgproc/imgtrans/filter_2d/filter_2d.markdown | 2 +- .../imgtrans/hough_circle/hough_circle.markdown | 4 ++-- .../imgtrans/hough_lines/hough_lines.markdown | 4 ++-- .../laplace_operator/laplace_operator.markdown | 2 +- .../imgproc/imgtrans/remap/remap.markdown | 2 +- .../sobel_derivatives/sobel_derivatives.markdown | 2 +- .../imgtrans/warp_affine/warp_affine.markdown | 2 +- .../morph_lines_detection/moprh_lines_detection.md | 2 +- .../opening_closing_hats.markdown | 2 +- doc/tutorials/imgproc/pyramids/pyramids.markdown | 2 +- .../bounding_rects_circles.markdown | 2 +- .../bounding_rotated_ellipses.markdown | 2 +- .../find_contours/find_contours.markdown | 2 +- .../imgproc/shapedescriptors/hull/hull.markdown | 2 +- .../shapedescriptors/moments/moments.markdown | 2 +- .../point_polygon_test/point_polygon_test.markdown | 2 +- doc/tutorials/imgproc/threshold/threshold.markdown | 2 +- .../threshold_inRange/threshold_inRange.markdown | 2 +- .../android_ocl_intro.markdown | 4 ++-- .../clojure_dev_intro/clojure_dev_intro.markdown | 2 +- .../arm_crosscompile_with_cmake.markdown | 6 +++--- .../desktop_java/java_dev_intro.markdown | 4 ++-- .../display_image/display_image.markdown | 2 +- .../introduction/ios_install/ios_install.markdown | 4 ++-- .../linux_install/linux_install.markdown | 14 +++++++------- .../transition_guide/transition_guide.markdown | 2 +- .../windows_install/windows_install.markdown | 4 ++-- .../windows_visual_studio_Opencv.markdown | 4 ++-- .../introduction_to_pca.markdown | 4 ++-- .../ml/non_linear_svms/non_linear_svms.markdown | 2 +- .../cascade_classifier/cascade_classifier.markdown | 4 ++-- .../background_subtraction.markdown | 2 +- doc/tutorials/videoio/intelperc.markdown | 2 +- doc/tutorials/videoio/kinect_openni.markdown | 2 +- .../video_input_psnr_ssim.markdown | 4 ++-- .../videoio/video-write/video_write.markdown | 2 +- .../viz/creating_widgets/creating_widgets.markdown | 2 +- .../viz/launching_viz/launching_viz.markdown | 2 +- .../viz/transformations/transformations.markdown | 2 +- doc/tutorials/viz/widget_pose/widget_pose.markdown | 2 +- modules/core/doc/cuda.markdown | 2 +- modules/highgui/include/opencv2/highgui.hpp | 2 +- modules/objdetect/include/opencv2/objdetect.hpp | 2 +- modules/python/test/test_digits.py | 2 +- modules/python/test/test_gaussian_mix.py | 2 +- modules/python/test/tests_common.py | 2 +- modules/videoio/include/opencv2/videoio.hpp | 2 +- modules/videoio/src/cap_libv4l.cpp | 2 +- modules/videoio/src/cap_v4l.cpp | 2 +- platforms/winrt/readme.txt | 2 +- .../CameraCalibrationActivity.java | 2 +- samples/cpp/facial_features.cpp | 4 ++-- samples/cpp/stereo_calib.cpp | 2 +- samples/python/digits.py | 2 +- samples/python/digits_video.py | 2 +- samples/python/find_obj.py | 2 +- samples/python/gaussian_mix.py | 2 +- samples/python/video_v4l2.py | 2 +- 94 files changed, 120 insertions(+), 120 deletions(-) diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md index 4f1453a2ff..e72c70d8c3 100644 --- a/.github/ISSUE_TEMPLATE.md +++ b/.github/ISSUE_TEMPLATE.md @@ -1,6 +1,6 @@ diff --git a/3rdparty/ffmpeg/ffmpeg.cmake b/3rdparty/ffmpeg/ffmpeg.cmake index 448ad2bb2c..c227f10bee 100644 --- a/3rdparty/ffmpeg/ffmpeg.cmake +++ b/3rdparty/ffmpeg/ffmpeg.cmake @@ -5,7 +5,7 @@ set(FFMPEG_FILE_HASH_BIN32 "89c783eee1c47bfc733f08334ec2e31c") set(FFMPEG_FILE_HASH_BIN64 "35fe6ccdda6d7a04e9056b0d73b98e76") set(FFMPEG_FILE_HASH_CMAKE "8606f947a780071f8fcce8cbf39ceef5") -set(FFMPEG_DOWNLOAD_URL ${OPENCV_FFMPEG_URL};$ENV{OPENCV_FFMPEG_URL};https://raw.githubusercontent.com/Itseez/opencv_3rdparty/${FFMPEG_BINARIES_COMMIT}/ffmpeg/) +set(FFMPEG_DOWNLOAD_URL ${OPENCV_FFMPEG_URL};$ENV{OPENCV_FFMPEG_URL};https://raw.githubusercontent.com/opencv/opencv_3rdparty/${FFMPEG_BINARIES_COMMIT}/ffmpeg/) ocv_download(PACKAGE opencv_ffmpeg.dll HASH ${FFMPEG_FILE_HASH_BIN32} diff --git a/3rdparty/ippicv/downloader.cmake b/3rdparty/ippicv/downloader.cmake index a6016dbe10..e20804d7b4 100644 --- a/3rdparty/ippicv/downloader.cmake +++ b/3rdparty/ippicv/downloader.cmake @@ -64,7 +64,7 @@ function(_icv_downloader) if(DEFINED ENV{OPENCV_ICV_URL}) set(OPENCV_ICV_URL $ENV{OPENCV_ICV_URL}) else() - set(OPENCV_ICV_URL "https://raw.githubusercontent.com/Itseez/opencv_3rdparty/${IPPICV_BINARIES_COMMIT}/ippicv") + set(OPENCV_ICV_URL "https://raw.githubusercontent.com/opencv/opencv_3rdparty/${IPPICV_BINARIES_COMMIT}/ippicv") endif() endif() diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 1e13c89969..318e9ac8fc 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,3 +1,3 @@ ## Contributing guidelines -All guidelines for contributing to the OpenCV repository can be found at [`How to contribute guideline`](https://github.com/Itseez/opencv/wiki/How_to_contribute). +All guidelines for contributing to the OpenCV repository can be found at [`How to contribute guideline`](https://github.com/opencv/opencv/wiki/How_to_contribute). diff --git a/README.md b/README.md index 7dbc823200..37543b6e5d 100644 --- a/README.md +++ b/README.md @@ -5,11 +5,11 @@ * Homepage: * Docs: * Q&A forum: -* Issue tracking: +* Issue tracking: #### Contributing -Please read before starting work on a pull request: +Please read before starting work on a pull request: Summary of guidelines: diff --git a/doc/Doxyfile.in b/doc/Doxyfile.in index c4da726035..45c085cc34 100644 --- a/doc/Doxyfile.in +++ b/doc/Doxyfile.in @@ -150,7 +150,7 @@ BINARY_TOC = NO TOC_EXPAND = NO GENERATE_QHP = @CMAKE_DOXYGEN_GENERATE_QHP@ QCH_FILE = ../opencv-@OPENCV_VERSION@.qch -QHP_NAMESPACE = org.itseez.opencv.@OPENCV_VERSION@ +QHP_NAMESPACE = org.opencv.@OPENCV_VERSION@ QHP_VIRTUAL_FOLDER = opencv QHP_CUST_FILTER_NAME = QHP_CUST_FILTER_ATTRS = diff --git a/doc/py_tutorials/py_feature2d/py_brief/py_brief.markdown b/doc/py_tutorials/py_feature2d/py_brief/py_brief.markdown index e2c0ed428e..33cbfebf49 100644 --- a/doc/py_tutorials/py_feature2d/py_brief/py_brief.markdown +++ b/doc/py_tutorials/py_feature2d/py_brief/py_brief.markdown @@ -49,7 +49,7 @@ BRIEF in OpenCV Below code shows the computation of BRIEF descriptors with the help of CenSurE detector. (CenSurE detector is called STAR detector in OpenCV) -note, that you need [opencv contrib](https://github.com/Itseez/opencv_contrib)) to use this. +note, that you need [opencv contrib](https://github.com/opencv/opencv_contrib)) to use this. @code{.py} import numpy as np import cv2 diff --git a/doc/py_tutorials/py_feature2d/py_sift_intro/py_sift_intro.markdown b/doc/py_tutorials/py_feature2d/py_sift_intro/py_sift_intro.markdown index b12505adc9..da4b878db0 100644 --- a/doc/py_tutorials/py_feature2d/py_sift_intro/py_sift_intro.markdown +++ b/doc/py_tutorials/py_feature2d/py_sift_intro/py_sift_intro.markdown @@ -104,7 +104,7 @@ greater than 0.8, they are rejected. It eliminaters around 90% of false matches So this is a summary of SIFT algorithm. For more details and understanding, reading the original paper is highly recommended. Remember one thing, this algorithm is patented. So this algorithm is -included in [the opencv contrib repo](https://github.com/Itseez/opencv_contrib) +included in [the opencv contrib repo](https://github.com/opencv/opencv_contrib) SIFT in OpenCV -------------- diff --git a/doc/py_tutorials/py_setup/py_intro/py_intro.markdown b/doc/py_tutorials/py_setup/py_intro/py_intro.markdown index 007a71ce72..c8041bb2c5 100644 --- a/doc/py_tutorials/py_setup/py_intro/py_intro.markdown +++ b/doc/py_tutorials/py_setup/py_intro/py_intro.markdown @@ -58,7 +58,7 @@ OpenCV Needs You !!! Since OpenCV is an open source initiative, all are welcome to make contributions to the library, documentation, and tutorials. If you find any mistake in this tutorial (from a small spelling mistake to an egregious error in code or concept), feel free to correct it by cloning OpenCV in -[GitHub](https://github.com/Itseez/opencv) and submitting a pull request. OpenCV developers will +[GitHub](https://github.com/opencv/opencv) and submitting a pull request. OpenCV developers will check your pull request, give you important feedback and (once it passes the approval of the reviewer) it will be merged into OpenCV. You will then become an open source contributor :-) diff --git a/doc/py_tutorials/py_setup/py_setup_in_fedora/py_setup_in_fedora.markdown b/doc/py_tutorials/py_setup/py_setup_in_fedora/py_setup_in_fedora.markdown index b8f57e72a6..da65dd4772 100644 --- a/doc/py_tutorials/py_setup/py_setup_in_fedora/py_setup_in_fedora.markdown +++ b/doc/py_tutorials/py_setup/py_setup_in_fedora/py_setup_in_fedora.markdown @@ -119,7 +119,7 @@ Or you can download latest source from OpenCV's github repo. (If you want to con choose this. It always keeps your OpenCV up-to-date). For that, you need to install **Git** first. @code{.sh} yum install git -git clone https://github.com/Itseez/opencv.git +git clone https://github.com/opencv/opencv.git @endcode It will create a folder OpenCV in home directory (or the directory you specify). The cloning may take some time depending upon your internet connection. diff --git a/doc/py_tutorials/py_setup/py_setup_in_windows/py_setup_in_windows.markdown b/doc/py_tutorials/py_setup/py_setup_in_windows/py_setup_in_windows.markdown index 807e5467bd..3ff4e82e8c 100644 --- a/doc/py_tutorials/py_setup/py_setup_in_windows/py_setup_in_windows.markdown +++ b/doc/py_tutorials/py_setup/py_setup_in_windows/py_setup_in_windows.markdown @@ -76,7 +76,7 @@ Building OpenCV from source -# Download OpenCV source. It can be from [Sourceforge](http://sourceforge.net/projects/opencvlibrary/) (for official release version) or - from [Github](https://github.com/Itseez/opencv) (for latest source). + from [Github](https://github.com/opencv/opencv) (for latest source). -# Extract it to a folder, opencv and create a new folder build in it. -# Open CMake-gui (*Start \> All Programs \> CMake-gui*) -# Fill the fields as follows (see the image below): diff --git a/doc/tutorials/calib3d/camera_calibration/camera_calibration.markdown b/doc/tutorials/calib3d/camera_calibration/camera_calibration.markdown index 1a7b906872..2c6973141e 100644 --- a/doc/tutorials/calib3d/camera_calibration/camera_calibration.markdown +++ b/doc/tutorials/calib3d/camera_calibration/camera_calibration.markdown @@ -77,13 +77,13 @@ Source code You may also find the source code in the `samples/cpp/tutorial_code/calib3d/camera_calibration/` folder of the OpenCV source library or [download it from here -](https://github.com/Itseez/opencv/tree/master/samples/cpp/tutorial_code/calib3d/camera_calibration/camera_calibration.cpp). The program has a +](https://github.com/opencv/opencv/tree/master/samples/cpp/tutorial_code/calib3d/camera_calibration/camera_calibration.cpp). The program has a single argument: the name of its configuration file. If none is given then it will try to open the one named "default.xml". [Here's a sample configuration file -](https://github.com/Itseez/opencv/tree/master/samples/cpp/tutorial_code/calib3d/camera_calibration/in_VID5.xml) in XML format. In the +](https://github.com/opencv/opencv/tree/master/samples/cpp/tutorial_code/calib3d/camera_calibration/in_VID5.xml) in XML format. In the configuration file you may choose to use camera as an input, a video file or an image list. If you opt for the last one, you will need to create a configuration file where you enumerate the images to -use. Here's [an example of this ](https://github.com/Itseez/opencv/tree/master/samples/cpp/tutorial_code/calib3d/camera_calibration/VID5.xml). +use. Here's [an example of this ](https://github.com/opencv/opencv/tree/master/samples/cpp/tutorial_code/calib3d/camera_calibration/VID5.xml). The important part to remember is that the images need to be specified using the absolute path or the relative one from your application's working directory. You may find all this in the samples directory mentioned above. diff --git a/doc/tutorials/calib3d/interactive_calibration/interactive_calibration.markdown b/doc/tutorials/calib3d/interactive_calibration/interactive_calibration.markdown index 123b63689d..0c4d67ec85 100644 --- a/doc/tutorials/calib3d/interactive_calibration/interactive_calibration.markdown +++ b/doc/tutorials/calib3d/interactive_calibration/interactive_calibration.markdown @@ -92,7 +92,7 @@ QR faster than SVD, but potentially less precise - *camera_resolution*: resolution of camera which is used for calibration **Note:** *charuco_dict*, *charuco_square_lenght* and *charuco_marker_size* are used for chAruco pattern generation -(see Aruco module description for details: [Aruco tutorials](https://github.com/Itseez/opencv_contrib/tree/master/modules/aruco/tutorials)) +(see Aruco module description for details: [Aruco tutorials](https://github.com/opencv/opencv_contrib/tree/master/modules/aruco/tutorials)) Default chAruco pattern: diff --git a/doc/tutorials/core/basic_geometric_drawing/basic_geometric_drawing.markdown b/doc/tutorials/core/basic_geometric_drawing/basic_geometric_drawing.markdown index 61134bd487..4c68517bbc 100644 --- a/doc/tutorials/core/basic_geometric_drawing/basic_geometric_drawing.markdown +++ b/doc/tutorials/core/basic_geometric_drawing/basic_geometric_drawing.markdown @@ -47,7 +47,7 @@ Code ---- - This code is in your OpenCV sample folder. Otherwise you can grab it from - [here](https://github.com/Itseez/opencv/tree/master/samples/cpp/tutorial_code/core/Matrix/Drawing_1.cpp) + [here](https://github.com/opencv/opencv/tree/master/samples/cpp/tutorial_code/core/Matrix/Drawing_1.cpp) Explanation ----------- diff --git a/doc/tutorials/core/discrete_fourier_transform/discrete_fourier_transform.markdown b/doc/tutorials/core/discrete_fourier_transform/discrete_fourier_transform.markdown index 1e2d5203ce..22bbc877d8 100644 --- a/doc/tutorials/core/discrete_fourier_transform/discrete_fourier_transform.markdown +++ b/doc/tutorials/core/discrete_fourier_transform/discrete_fourier_transform.markdown @@ -15,7 +15,7 @@ Source code ----------- You can [download this from here -](https://github.com/Itseez/opencv/tree/master/samples/cpp/tutorial_code/core/discrete_fourier_transform/discrete_fourier_transform.cpp) or +](https://github.com/opencv/opencv/tree/master/samples/cpp/tutorial_code/core/discrete_fourier_transform/discrete_fourier_transform.cpp) or find it in the `samples/cpp/tutorial_code/core/discrete_fourier_transform/discrete_fourier_transform.cpp` of the OpenCV source code library. @@ -140,7 +140,7 @@ An application idea would be to determine the geometrical orientation present in example, let us find out if a text is horizontal or not? Looking at some text you'll notice that the text lines sort of form also horizontal lines and the letters form sort of vertical lines. These two main components of a text snippet may be also seen in case of the Fourier transform. Let us use -[this horizontal ](https://github.com/Itseez/opencv/tree/master/samples/data/imageTextN.png) and [this rotated](https://github.com/Itseez/opencv/tree/master/samples/data/imageTextR.png) +[this horizontal ](https://github.com/opencv/opencv/tree/master/samples/data/imageTextN.png) and [this rotated](https://github.com/opencv/opencv/tree/master/samples/data/imageTextR.png) image about a text. In case of the horizontal text: diff --git a/doc/tutorials/core/file_input_output_with_xml_yml/file_input_output_with_xml_yml.markdown b/doc/tutorials/core/file_input_output_with_xml_yml/file_input_output_with_xml_yml.markdown index 7576e5734d..10521338c0 100644 --- a/doc/tutorials/core/file_input_output_with_xml_yml/file_input_output_with_xml_yml.markdown +++ b/doc/tutorials/core/file_input_output_with_xml_yml/file_input_output_with_xml_yml.markdown @@ -16,7 +16,7 @@ Source code ----------- You can [download this from here -](https://github.com/Itseez/opencv/tree/master/samples/cpp/tutorial_code/core/file_input_output/file_input_output.cpp) or find it in the +](https://github.com/opencv/opencv/tree/master/samples/cpp/tutorial_code/core/file_input_output/file_input_output.cpp) or find it in the `samples/cpp/tutorial_code/core/file_input_output/file_input_output.cpp` of the OpenCV source code library. diff --git a/doc/tutorials/core/how_to_scan_images/how_to_scan_images.markdown b/doc/tutorials/core/how_to_scan_images/how_to_scan_images.markdown index 1121caf400..409ebdbac1 100644 --- a/doc/tutorials/core/how_to_scan_images/how_to_scan_images.markdown +++ b/doc/tutorials/core/how_to_scan_images/how_to_scan_images.markdown @@ -51,7 +51,7 @@ three major ways of going through an image pixel by pixel. To make things a litt will make the scanning for each image using all of these methods, and print out how long it took. You can download the full source code [here -](https://github.com/Itseez/opencv/tree/master/samples/cpp/tutorial_code/core/how_to_scan_images/how_to_scan_images.cpp) or look it up in +](https://github.com/opencv/opencv/tree/master/samples/cpp/tutorial_code/core/how_to_scan_images/how_to_scan_images.cpp) or look it up in the samples directory of OpenCV at the cpp tutorial code for the core section. Its basic usage is: @code{.bash} how_to_scan_images imageName.jpg intValueToReduce [G] diff --git a/doc/tutorials/core/how_to_use_ippa_conversion/how_to_use_ippa_conversion.markdown b/doc/tutorials/core/how_to_use_ippa_conversion/how_to_use_ippa_conversion.markdown index b851bbfe36..50f3b545ef 100644 --- a/doc/tutorials/core/how_to_use_ippa_conversion/how_to_use_ippa_conversion.markdown +++ b/doc/tutorials/core/how_to_use_ippa_conversion/how_to_use_ippa_conversion.markdown @@ -16,7 +16,7 @@ Code You may also find the source code in the `samples/cpp/tutorial_code/core/ippasync/ippasync_sample.cpp` file of the OpenCV source library or -download it from [here](https://github.com/Itseez/opencv/tree/master/samples/cpp/tutorial_code/core/ippasync/ippasync_sample.cpp). +download it from [here](https://github.com/opencv/opencv/tree/master/samples/cpp/tutorial_code/core/ippasync/ippasync_sample.cpp). @include cpp/tutorial_code/core/ippasync/ippasync_sample.cpp diff --git a/doc/tutorials/core/interoperability_with_OpenCV_1/interoperability_with_OpenCV_1.markdown b/doc/tutorials/core/interoperability_with_OpenCV_1/interoperability_with_OpenCV_1.markdown index 3837e475ca..d28786a8b1 100644 --- a/doc/tutorials/core/interoperability_with_OpenCV_1/interoperability_with_OpenCV_1.markdown +++ b/doc/tutorials/core/interoperability_with_OpenCV_1/interoperability_with_OpenCV_1.markdown @@ -85,7 +85,7 @@ L = Mat(pI); A case study ------------ -Now that you have the basics done [here's](https://github.com/Itseez/opencv/tree/master/samples/cpp/tutorial_code/core/interoperability_with_OpenCV_1/interoperability_with_OpenCV_1.cpp) +Now that you have the basics done [here's](https://github.com/opencv/opencv/tree/master/samples/cpp/tutorial_code/core/interoperability_with_OpenCV_1/interoperability_with_OpenCV_1.cpp) an example that mixes the usage of the C interface with the C++ one. You will also find it in the sample directory of the OpenCV source code library at the `samples/cpp/tutorial_code/core/interoperability_with_OpenCV_1/interoperability_with_OpenCV_1.cpp` . @@ -132,7 +132,7 @@ output: You may observe a runtime instance of this on the [YouTube here](https://www.youtube.com/watch?v=qckm-zvo31w) and you can [download the source code from here -](https://github.com/Itseez/opencv/tree/master/samples/cpp/tutorial_code/core/interoperability_with_OpenCV_1/interoperability_with_OpenCV_1.cpp) +](https://github.com/opencv/opencv/tree/master/samples/cpp/tutorial_code/core/interoperability_with_OpenCV_1/interoperability_with_OpenCV_1.cpp) or find it in the `samples/cpp/tutorial_code/core/interoperability_with_OpenCV_1/interoperability_with_OpenCV_1.cpp` of the OpenCV source code library. diff --git a/doc/tutorials/core/mat-mask-operations/mat_mask_operations.markdown b/doc/tutorials/core/mat-mask-operations/mat_mask_operations.markdown index 48b7b13a14..980ea78024 100644 --- a/doc/tutorials/core/mat-mask-operations/mat_mask_operations.markdown +++ b/doc/tutorials/core/mat-mask-operations/mat_mask_operations.markdown @@ -133,7 +133,7 @@ For example: ![](images/resultMatMaskFilter2D.png) You can download this source code from [here -](https://github.com/Itseez/opencv/tree/master/samples/cpp/tutorial_code/core/mat_mask_operations/mat_mask_operations.cpp) or look in the +](https://github.com/opencv/opencv/tree/master/samples/cpp/tutorial_code/core/mat_mask_operations/mat_mask_operations.cpp) or look in the OpenCV source code libraries sample directory at `samples/cpp/tutorial_code/core/mat_mask_operations/mat_mask_operations.cpp`. diff --git a/doc/tutorials/core/mat_the_basic_image_container/mat_the_basic_image_container.markdown b/doc/tutorials/core/mat_the_basic_image_container/mat_the_basic_image_container.markdown index 65989744c2..9e290b88cc 100644 --- a/doc/tutorials/core/mat_the_basic_image_container/mat_the_basic_image_container.markdown +++ b/doc/tutorials/core/mat_the_basic_image_container/mat_the_basic_image_container.markdown @@ -258,7 +258,7 @@ OpenCV offers support for output of other common OpenCV data structures too via ![](images/MatBasicContainerOut15.png) Most of the samples here have been included in a small console application. You can download it from -[here](https://github.com/Itseez/opencv/tree/master/samples/cpp/tutorial_code/core/mat_the_basic_image_container/mat_the_basic_image_container.cpp) +[here](https://github.com/opencv/opencv/tree/master/samples/cpp/tutorial_code/core/mat_the_basic_image_container/mat_the_basic_image_container.cpp) or in the core section of the cpp samples. You can also find a quick video demonstration of this on diff --git a/doc/tutorials/features2d/trackingmotion/corner_subpixeles/corner_subpixeles.markdown b/doc/tutorials/features2d/trackingmotion/corner_subpixeles/corner_subpixeles.markdown index be9b9762c6..92f3459d35 100644 --- a/doc/tutorials/features2d/trackingmotion/corner_subpixeles/corner_subpixeles.markdown +++ b/doc/tutorials/features2d/trackingmotion/corner_subpixeles/corner_subpixeles.markdown @@ -16,7 +16,7 @@ Code ---- This tutorial code's is shown lines below. You can also download it from -[here](https://github.com/Itseez/opencv/tree/master/samples/cpp/tutorial_code/TrackingMotion/cornerSubPix_Demo.cpp) +[here](https://github.com/opencv/opencv/tree/master/samples/cpp/tutorial_code/TrackingMotion/cornerSubPix_Demo.cpp) @code{.cpp} #include "opencv2/highgui.hpp" #include "opencv2/imgproc.hpp" diff --git a/doc/tutorials/features2d/trackingmotion/generic_corner_detector/generic_corner_detector.markdown b/doc/tutorials/features2d/trackingmotion/generic_corner_detector/generic_corner_detector.markdown index dd04afca07..7aba636746 100644 --- a/doc/tutorials/features2d/trackingmotion/generic_corner_detector/generic_corner_detector.markdown +++ b/doc/tutorials/features2d/trackingmotion/generic_corner_detector/generic_corner_detector.markdown @@ -20,7 +20,7 @@ Code ---- This tutorial code's is shown lines below. You can also download it from -[here](https://github.com/Itseez/opencv/tree/master/samples/cpp/tutorial_code/TrackingMotion/cornerDetector_Demo.cpp) +[here](https://github.com/opencv/opencv/tree/master/samples/cpp/tutorial_code/TrackingMotion/cornerDetector_Demo.cpp) @include cpp/tutorial_code/TrackingMotion/cornerDetector_Demo.cpp diff --git a/doc/tutorials/features2d/trackingmotion/good_features_to_track/good_features_to_track.markdown b/doc/tutorials/features2d/trackingmotion/good_features_to_track/good_features_to_track.markdown index e4d9769952..23c1e3f904 100644 --- a/doc/tutorials/features2d/trackingmotion/good_features_to_track/good_features_to_track.markdown +++ b/doc/tutorials/features2d/trackingmotion/good_features_to_track/good_features_to_track.markdown @@ -15,7 +15,7 @@ Code ---- This tutorial code's is shown lines below. You can also download it from -[here](https://github.com/Itseez/opencv/tree/master/samples/cpp/tutorial_code/TrackingMotion/goodFeaturesToTrack_Demo.cpp) +[here](https://github.com/opencv/opencv/tree/master/samples/cpp/tutorial_code/TrackingMotion/goodFeaturesToTrack_Demo.cpp) @code{.cpp} #include "opencv2/highgui.hpp" #include "opencv2/imgproc.hpp" diff --git a/doc/tutorials/features2d/trackingmotion/harris_detector/harris_detector.markdown b/doc/tutorials/features2d/trackingmotion/harris_detector/harris_detector.markdown index add7db8a11..dc3070f544 100644 --- a/doc/tutorials/features2d/trackingmotion/harris_detector/harris_detector.markdown +++ b/doc/tutorials/features2d/trackingmotion/harris_detector/harris_detector.markdown @@ -119,7 +119,7 @@ Code ---- This tutorial code's is shown lines below. You can also download it from -[here](https://github.com/Itseez/opencv/tree/master/samples/cpp/tutorial_code/TrackingMotion/cornerHarris_Demo.cpp) +[here](https://github.com/opencv/opencv/tree/master/samples/cpp/tutorial_code/TrackingMotion/cornerHarris_Demo.cpp) @code{.cpp} #include "opencv2/highgui.hpp" #include "opencv2/imgproc.hpp" diff --git a/doc/tutorials/gpu/gpu-basics-similarity/gpu_basics_similarity.markdown b/doc/tutorials/gpu/gpu-basics-similarity/gpu_basics_similarity.markdown index cf8aba954d..b2298968c7 100644 --- a/doc/tutorials/gpu/gpu-basics-similarity/gpu_basics_similarity.markdown +++ b/doc/tutorials/gpu/gpu-basics-similarity/gpu_basics_similarity.markdown @@ -24,7 +24,7 @@ The source code You may also find the source code and these video file in the `samples/cpp/tutorial_code/gpu/gpu-basics-similarity/gpu-basics-similarity` folder of the OpenCV -source library or download it from [here](https://github.com/Itseez/opencv/tree/master/samples/cpp/tutorial_code/gpu/gpu-basics-similarity/gpu-basics-similarity.cpp). +source library or download it from [here](https://github.com/opencv/opencv/tree/master/samples/cpp/tutorial_code/gpu/gpu-basics-similarity/gpu-basics-similarity.cpp). The full source code is quite long (due to the controlling of the application via the command line arguments and performance measurement). Therefore, to avoid cluttering up these sections with those you'll find here only the functions itself. diff --git a/doc/tutorials/imgproc/erosion_dilatation/erosion_dilatation.markdown b/doc/tutorials/imgproc/erosion_dilatation/erosion_dilatation.markdown index 831bba8a0d..e3786e607b 100644 --- a/doc/tutorials/imgproc/erosion_dilatation/erosion_dilatation.markdown +++ b/doc/tutorials/imgproc/erosion_dilatation/erosion_dilatation.markdown @@ -60,7 +60,7 @@ Code ---- This tutorial code's is shown lines below. You can also download it from -[here](https://github.com/Itseez/opencv/tree/master/samples/cpp/tutorial_code/ImgProc/Morphology_1.cpp) +[here](https://github.com/opencv/opencv/tree/master/samples/cpp/tutorial_code/ImgProc/Morphology_1.cpp) @include samples/cpp/tutorial_code/ImgProc/Morphology_1.cpp Explanation diff --git a/doc/tutorials/imgproc/gausian_median_blur_bilateral_filter/gausian_median_blur_bilateral_filter.markdown b/doc/tutorials/imgproc/gausian_median_blur_bilateral_filter/gausian_median_blur_bilateral_filter.markdown index 43753ca6e1..2d17c3aade 100644 --- a/doc/tutorials/imgproc/gausian_median_blur_bilateral_filter/gausian_median_blur_bilateral_filter.markdown +++ b/doc/tutorials/imgproc/gausian_median_blur_bilateral_filter/gausian_median_blur_bilateral_filter.markdown @@ -94,7 +94,7 @@ Code - Applies 4 different kinds of filters (explained in Theory) and show the filtered images sequentially - **Downloadable code**: Click - [here](https://github.com/Itseez/opencv/tree/master/samples/cpp/tutorial_code/ImgProc/Smoothing.cpp) + [here](https://github.com/opencv/opencv/tree/master/samples/cpp/tutorial_code/ImgProc/Smoothing.cpp) - **Code at glance:** @code{.cpp} #include "opencv2/imgproc.hpp" diff --git a/doc/tutorials/imgproc/histograms/back_projection/back_projection.markdown b/doc/tutorials/imgproc/histograms/back_projection/back_projection.markdown index fa4ff726f0..86575418be 100644 --- a/doc/tutorials/imgproc/histograms/back_projection/back_projection.markdown +++ b/doc/tutorials/imgproc/histograms/back_projection/back_projection.markdown @@ -70,13 +70,13 @@ Code - **Downloadable code**: -# Click - [here](https://github.com/Itseez/opencv/tree/master/samples/cpp/tutorial_code/Histograms_Matching/calcBackProject_Demo1.cpp) + [here](https://github.com/opencv/opencv/tree/master/samples/cpp/tutorial_code/Histograms_Matching/calcBackProject_Demo1.cpp) for the basic version (explained in this tutorial). -# For stuff slightly fancier (using H-S histograms and floodFill to define a mask for the skin area) you can check the [improved - demo](https://github.com/Itseez/opencv/tree/master/samples/cpp/tutorial_code/Histograms_Matching/calcBackProject_Demo2.cpp) + demo](https://github.com/opencv/opencv/tree/master/samples/cpp/tutorial_code/Histograms_Matching/calcBackProject_Demo2.cpp) -# ...or you can always check out the classical - [camshiftdemo](https://github.com/Itseez/opencv/tree/master/samples/cpp/camshiftdemo.cpp) + [camshiftdemo](https://github.com/opencv/opencv/tree/master/samples/cpp/camshiftdemo.cpp) in samples. - **Code at glance:** diff --git a/doc/tutorials/imgproc/histograms/histogram_calculation/histogram_calculation.markdown b/doc/tutorials/imgproc/histograms/histogram_calculation/histogram_calculation.markdown index a966d9087b..2907bf3e56 100644 --- a/doc/tutorials/imgproc/histograms/histogram_calculation/histogram_calculation.markdown +++ b/doc/tutorials/imgproc/histograms/histogram_calculation/histogram_calculation.markdown @@ -66,7 +66,7 @@ Code - Calculate the Histogram of each 1-channel plane by calling the function @ref cv::calcHist - Plot the three histograms in a window - **Downloadable code**: Click - [here](https://github.com/Itseez/opencv/tree/master/samples/cpp/tutorial_code/Histograms_Matching/calcHist_Demo.cpp) + [here](https://github.com/opencv/opencv/tree/master/samples/cpp/tutorial_code/Histograms_Matching/calcHist_Demo.cpp) - **Code at glance:** @include samples/cpp/tutorial_code/Histograms_Matching/calcHist_Demo.cpp diff --git a/doc/tutorials/imgproc/histograms/histogram_comparison/histogram_comparison.markdown b/doc/tutorials/imgproc/histograms/histogram_comparison/histogram_comparison.markdown index 8b7399ff61..37ae5eac8a 100644 --- a/doc/tutorials/imgproc/histograms/histogram_comparison/histogram_comparison.markdown +++ b/doc/tutorials/imgproc/histograms/histogram_comparison/histogram_comparison.markdown @@ -44,7 +44,7 @@ Code histogram of the lower half base image and with the same base image histogram. - Display the numerical matching parameters obtained. - **Downloadable code**: Click - [here](https://github.com/Itseez/opencv/tree/master/samples/cpp/tutorial_code/Histograms_Matching/compareHist_Demo.cpp) + [here](https://github.com/opencv/opencv/tree/master/samples/cpp/tutorial_code/Histograms_Matching/compareHist_Demo.cpp) - **Code at glance:** @include cpp/tutorial_code/Histograms_Matching/compareHist_Demo.cpp diff --git a/doc/tutorials/imgproc/histograms/histogram_equalization/histogram_equalization.markdown b/doc/tutorials/imgproc/histograms/histogram_equalization/histogram_equalization.markdown index f3a2746de5..2d803cc7ea 100644 --- a/doc/tutorials/imgproc/histograms/histogram_equalization/histogram_equalization.markdown +++ b/doc/tutorials/imgproc/histograms/histogram_equalization/histogram_equalization.markdown @@ -62,7 +62,7 @@ Code - Equalize the Histogram by using the OpenCV function @ref cv::equalizeHist - Display the source and equalized images in a window. - **Downloadable code**: Click - [here](https://github.com/Itseez/opencv/tree/master/samples/cpp/tutorial_code/Histograms_Matching/EqualizeHist_Demo.cpp) + [here](https://github.com/opencv/opencv/tree/master/samples/cpp/tutorial_code/Histograms_Matching/EqualizeHist_Demo.cpp) - **Code at glance:** @include samples/cpp/tutorial_code/Histograms_Matching/EqualizeHist_Demo.cpp diff --git a/doc/tutorials/imgproc/histograms/template_matching/template_matching.markdown b/doc/tutorials/imgproc/histograms/template_matching/template_matching.markdown index c39fd159f0..3810bada43 100644 --- a/doc/tutorials/imgproc/histograms/template_matching/template_matching.markdown +++ b/doc/tutorials/imgproc/histograms/template_matching/template_matching.markdown @@ -96,7 +96,7 @@ Code - Localize the location with higher matching probability - Draw a rectangle around the area corresponding to the highest match - **Downloadable code**: Click - [here](https://github.com/Itseez/opencv/tree/master/samples/cpp/tutorial_code/Histograms_Matching/MatchTemplate_Demo.cpp) + [here](https://github.com/opencv/opencv/tree/master/samples/cpp/tutorial_code/Histograms_Matching/MatchTemplate_Demo.cpp) - **Code at glance:** @include samples/cpp/tutorial_code/Histograms_Matching/MatchTemplate_Demo.cpp diff --git a/doc/tutorials/imgproc/imgtrans/canny_detector/canny_detector.markdown b/doc/tutorials/imgproc/imgtrans/canny_detector/canny_detector.markdown index aa61992e59..c7bd63ed4f 100644 --- a/doc/tutorials/imgproc/imgtrans/canny_detector/canny_detector.markdown +++ b/doc/tutorials/imgproc/imgtrans/canny_detector/canny_detector.markdown @@ -74,7 +74,7 @@ Code - Applies the mask obtained on the original image and display it in a window. -# The tutorial code's is shown lines below. You can also download it from - [here](https://github.com/Itseez/opencv/tree/master/samples/cpp/tutorial_code/ImgTrans/CannyDetector_Demo.cpp) + [here](https://github.com/opencv/opencv/tree/master/samples/cpp/tutorial_code/ImgTrans/CannyDetector_Demo.cpp) @include samples/cpp/tutorial_code/ImgTrans/CannyDetector_Demo.cpp Explanation diff --git a/doc/tutorials/imgproc/imgtrans/copyMakeBorder/copyMakeBorder.markdown b/doc/tutorials/imgproc/imgtrans/copyMakeBorder/copyMakeBorder.markdown index d6cf3dfad3..dd4d785515 100644 --- a/doc/tutorials/imgproc/imgtrans/copyMakeBorder/copyMakeBorder.markdown +++ b/doc/tutorials/imgproc/imgtrans/copyMakeBorder/copyMakeBorder.markdown @@ -46,7 +46,7 @@ Code - The program finishes when the user presses 'ESC' -# The tutorial code's is shown lines below. You can also download it from - [here](https://github.com/Itseez/opencv/tree/master/samples/cpp/tutorial_code/ImgTrans/copyMakeBorder_demo.cpp) + [here](https://github.com/opencv/opencv/tree/master/samples/cpp/tutorial_code/ImgTrans/copyMakeBorder_demo.cpp) @include samples/cpp/tutorial_code/ImgTrans/copyMakeBorder_demo.cpp Explanation diff --git a/doc/tutorials/imgproc/imgtrans/distance_transformation/distance_transform.markdown b/doc/tutorials/imgproc/imgtrans/distance_transformation/distance_transform.markdown index 1e49982241..0ba6b71a31 100644 --- a/doc/tutorials/imgproc/imgtrans/distance_transformation/distance_transform.markdown +++ b/doc/tutorials/imgproc/imgtrans/distance_transformation/distance_transform.markdown @@ -17,7 +17,7 @@ Code ---- This tutorial code's is shown lines below. You can also download it from - [here](https://github.com/Itseez/opencv/tree/master/samples/cpp/tutorial_code/ImgTrans/imageSegmentation.cpp). + [here](https://github.com/opencv/opencv/tree/master/samples/cpp/tutorial_code/ImgTrans/imageSegmentation.cpp). @include samples/cpp/tutorial_code/ImgTrans/imageSegmentation.cpp Explanation / Result diff --git a/doc/tutorials/imgproc/imgtrans/filter_2d/filter_2d.markdown b/doc/tutorials/imgproc/imgtrans/filter_2d/filter_2d.markdown index 079bbed394..ef25b979f3 100644 --- a/doc/tutorials/imgproc/imgtrans/filter_2d/filter_2d.markdown +++ b/doc/tutorials/imgproc/imgtrans/filter_2d/filter_2d.markdown @@ -62,7 +62,7 @@ Code - The filter output (with each kernel) will be shown during 500 milliseconds -# The tutorial code's is shown lines below. You can also download it from - [here](https://github.com/Itseez/opencv/tree/master/samples/cpp/tutorial_code/ImgTrans/filter2D_demo.cpp) + [here](https://github.com/opencv/opencv/tree/master/samples/cpp/tutorial_code/ImgTrans/filter2D_demo.cpp) @code{.cpp} #include "opencv2/imgproc.hpp" #include "opencv2/highgui.hpp" diff --git a/doc/tutorials/imgproc/imgtrans/hough_circle/hough_circle.markdown b/doc/tutorials/imgproc/imgtrans/hough_circle/hough_circle.markdown index e796c055c9..61f83aabce 100644 --- a/doc/tutorials/imgproc/imgtrans/hough_circle/hough_circle.markdown +++ b/doc/tutorials/imgproc/imgtrans/hough_circle/hough_circle.markdown @@ -39,9 +39,9 @@ Code - Applies the *Hough Circle Transform* to the blurred image . - Display the detected circle in a window. --# The sample code that we will explain can be downloaded from [here](https://github.com/Itseez/opencv/tree/master/samples/cpp/houghcircles.cpp). +-# The sample code that we will explain can be downloaded from [here](https://github.com/opencv/opencv/tree/master/samples/cpp/houghcircles.cpp). A slightly fancier version (which shows trackbars for - changing the threshold values) can be found [here](https://github.com/Itseez/opencv/tree/master/samples/cpp/tutorial_code/ImgTrans/HoughCircle_Demo.cpp). + changing the threshold values) can be found [here](https://github.com/opencv/opencv/tree/master/samples/cpp/tutorial_code/ImgTrans/HoughCircle_Demo.cpp). @include samples/cpp/houghcircles.cpp Explanation diff --git a/doc/tutorials/imgproc/imgtrans/hough_lines/hough_lines.markdown b/doc/tutorials/imgproc/imgtrans/hough_lines/hough_lines.markdown index 8de662982e..584c3f8b68 100644 --- a/doc/tutorials/imgproc/imgtrans/hough_lines/hough_lines.markdown +++ b/doc/tutorials/imgproc/imgtrans/hough_lines/hough_lines.markdown @@ -95,9 +95,9 @@ Code - Applies either a *Standard Hough Line Transform* or a *Probabilistic Line Transform*. - Display the original image and the detected line in two windows. --# The sample code that we will explain can be downloaded from [here](https://github.com/Itseez/opencv/tree/master/samples/cpp/houghlines.cpp). A slightly fancier version +-# The sample code that we will explain can be downloaded from [here](https://github.com/opencv/opencv/tree/master/samples/cpp/houghlines.cpp). A slightly fancier version (which shows both Hough standard and probabilistic with trackbars for changing the threshold - values) can be found [here](https://github.com/Itseez/opencv/tree/master/samples/cpp/tutorial_code/ImgTrans/HoughLines_Demo.cpp). + values) can be found [here](https://github.com/opencv/opencv/tree/master/samples/cpp/tutorial_code/ImgTrans/HoughLines_Demo.cpp). @include samples/cpp/houghlines.cpp Explanation diff --git a/doc/tutorials/imgproc/imgtrans/laplace_operator/laplace_operator.markdown b/doc/tutorials/imgproc/imgtrans/laplace_operator/laplace_operator.markdown index 104c2442f2..8976803c51 100644 --- a/doc/tutorials/imgproc/imgtrans/laplace_operator/laplace_operator.markdown +++ b/doc/tutorials/imgproc/imgtrans/laplace_operator/laplace_operator.markdown @@ -51,7 +51,7 @@ Code - Display the result in a window -# The tutorial code's is shown lines below. You can also download it from - [here](https://github.com/Itseez/opencv/tree/master/samples/cpp/tutorial_code/ImgTrans/Laplace_Demo.cpp) + [here](https://github.com/opencv/opencv/tree/master/samples/cpp/tutorial_code/ImgTrans/Laplace_Demo.cpp) @include samples/cpp/tutorial_code/ImgTrans/Laplace_Demo.cpp Explanation diff --git a/doc/tutorials/imgproc/imgtrans/remap/remap.markdown b/doc/tutorials/imgproc/imgtrans/remap/remap.markdown index fd70f0133a..849157f077 100644 --- a/doc/tutorials/imgproc/imgtrans/remap/remap.markdown +++ b/doc/tutorials/imgproc/imgtrans/remap/remap.markdown @@ -52,7 +52,7 @@ Code - Wait for the user to exit the program -# The tutorial code's is shown lines below. You can also download it from - [here](https://github.com/Itseez/opencv/tree/master/samples/cpp/tutorial_code/ImgTrans/Remap_Demo.cpp) + [here](https://github.com/opencv/opencv/tree/master/samples/cpp/tutorial_code/ImgTrans/Remap_Demo.cpp) @include samples/cpp/tutorial_code/ImgTrans/Remap_Demo.cpp Explanation diff --git a/doc/tutorials/imgproc/imgtrans/sobel_derivatives/sobel_derivatives.markdown b/doc/tutorials/imgproc/imgtrans/sobel_derivatives/sobel_derivatives.markdown index 185f891c54..2d92fa0226 100644 --- a/doc/tutorials/imgproc/imgtrans/sobel_derivatives/sobel_derivatives.markdown +++ b/doc/tutorials/imgproc/imgtrans/sobel_derivatives/sobel_derivatives.markdown @@ -108,7 +108,7 @@ Code bright on a darker background. -# The tutorial code's is shown lines below. You can also download it from - [here](https://github.com/Itseez/opencv/tree/master/samples/cpp/tutorial_code/ImgTrans/Sobel_Demo.cpp) + [here](https://github.com/opencv/opencv/tree/master/samples/cpp/tutorial_code/ImgTrans/Sobel_Demo.cpp) @include samples/cpp/tutorial_code/ImgTrans/Sobel_Demo.cpp Explanation diff --git a/doc/tutorials/imgproc/imgtrans/warp_affine/warp_affine.markdown b/doc/tutorials/imgproc/imgtrans/warp_affine/warp_affine.markdown index 3612c32b54..934e8dba5e 100644 --- a/doc/tutorials/imgproc/imgtrans/warp_affine/warp_affine.markdown +++ b/doc/tutorials/imgproc/imgtrans/warp_affine/warp_affine.markdown @@ -89,7 +89,7 @@ Code - Waits until the user exits the program -# The tutorial code's is shown lines below. You can also download it from - [here](https://github.com/Itseez/opencv/tree/master/samples/cpp/tutorial_code/ImgTrans/Geometric_Transforms_Demo.cpp) + [here](https://github.com/opencv/opencv/tree/master/samples/cpp/tutorial_code/ImgTrans/Geometric_Transforms_Demo.cpp) @include samples/cpp/tutorial_code/ImgTrans/Geometric_Transforms_Demo.cpp Explanation diff --git a/doc/tutorials/imgproc/morph_lines_detection/moprh_lines_detection.md b/doc/tutorials/imgproc/morph_lines_detection/moprh_lines_detection.md index 2fd260c8fd..23b748ddd1 100644 --- a/doc/tutorials/imgproc/morph_lines_detection/moprh_lines_detection.md +++ b/doc/tutorials/imgproc/morph_lines_detection/moprh_lines_detection.md @@ -48,7 +48,7 @@ A structuring element can have many common shapes, such as lines, diamonds, disk Code ---- -This tutorial code's is shown lines below. You can also download it from [here](https://github.com/Itseez/opencv/tree/master/samples/cpp/tutorial_code/ImgProc/Morphology_3.cpp). +This tutorial code's is shown lines below. You can also download it from [here](https://github.com/opencv/opencv/tree/master/samples/cpp/tutorial_code/ImgProc/Morphology_3.cpp). @include samples/cpp/tutorial_code/ImgProc/Morphology_3.cpp Explanation / Result diff --git a/doc/tutorials/imgproc/opening_closing_hats/opening_closing_hats.markdown b/doc/tutorials/imgproc/opening_closing_hats/opening_closing_hats.markdown index e1eaed72bf..09331109d8 100644 --- a/doc/tutorials/imgproc/opening_closing_hats/opening_closing_hats.markdown +++ b/doc/tutorials/imgproc/opening_closing_hats/opening_closing_hats.markdown @@ -80,7 +80,7 @@ Code ---- This tutorial code's is shown lines below. You can also download it from -[here](https://github.com/Itseez/opencv/tree/master/samples/cpp/tutorial_code/ImgProc/Morphology_2.cpp) +[here](https://github.com/opencv/opencv/tree/master/samples/cpp/tutorial_code/ImgProc/Morphology_2.cpp) @code{.cpp} #include "opencv2/imgproc.hpp" #include "opencv2/highgui.hpp" diff --git a/doc/tutorials/imgproc/pyramids/pyramids.markdown b/doc/tutorials/imgproc/pyramids/pyramids.markdown index 1a07d8c56a..0bb5160fbb 100644 --- a/doc/tutorials/imgproc/pyramids/pyramids.markdown +++ b/doc/tutorials/imgproc/pyramids/pyramids.markdown @@ -66,7 +66,7 @@ Code ---- This tutorial code's is shown lines below. You can also download it from -[here](https://github.com/Itseez/opencv/tree/master/samples/cpp/tutorial_code/ImgProc/Pyramids.cpp) +[here](https://github.com/opencv/opencv/tree/master/samples/cpp/tutorial_code/ImgProc/Pyramids.cpp) @include samples/cpp/tutorial_code/ImgProc/Pyramids.cpp diff --git a/doc/tutorials/imgproc/shapedescriptors/bounding_rects_circles/bounding_rects_circles.markdown b/doc/tutorials/imgproc/shapedescriptors/bounding_rects_circles/bounding_rects_circles.markdown index 499ad916c9..fef66fb665 100644 --- a/doc/tutorials/imgproc/shapedescriptors/bounding_rects_circles/bounding_rects_circles.markdown +++ b/doc/tutorials/imgproc/shapedescriptors/bounding_rects_circles/bounding_rects_circles.markdown @@ -16,7 +16,7 @@ Code ---- This tutorial code's is shown lines below. You can also download it from -[here](https://github.com/Itseez/opencv/tree/master/samples/cpp/tutorial_code/ShapeDescriptors/generalContours_demo1.cpp) +[here](https://github.com/opencv/opencv/tree/master/samples/cpp/tutorial_code/ShapeDescriptors/generalContours_demo1.cpp) @include samples/cpp/tutorial_code/ShapeDescriptors/generalContours_demo1.cpp Explanation diff --git a/doc/tutorials/imgproc/shapedescriptors/bounding_rotated_ellipses/bounding_rotated_ellipses.markdown b/doc/tutorials/imgproc/shapedescriptors/bounding_rotated_ellipses/bounding_rotated_ellipses.markdown index 5dcf69ca22..e7b3a94553 100644 --- a/doc/tutorials/imgproc/shapedescriptors/bounding_rotated_ellipses/bounding_rotated_ellipses.markdown +++ b/doc/tutorials/imgproc/shapedescriptors/bounding_rotated_ellipses/bounding_rotated_ellipses.markdown @@ -16,7 +16,7 @@ Code ---- This tutorial code's is shown lines below. You can also download it from -[here](https://github.com/Itseez/opencv/tree/master/samples/cpp/tutorial_code/ShapeDescriptors/generalContours_demo2.cpp) +[here](https://github.com/opencv/opencv/tree/master/samples/cpp/tutorial_code/ShapeDescriptors/generalContours_demo2.cpp) @include samples/cpp/tutorial_code/ShapeDescriptors/generalContours_demo2.cpp Explanation diff --git a/doc/tutorials/imgproc/shapedescriptors/find_contours/find_contours.markdown b/doc/tutorials/imgproc/shapedescriptors/find_contours/find_contours.markdown index 71329db1af..11d1d9f476 100644 --- a/doc/tutorials/imgproc/shapedescriptors/find_contours/find_contours.markdown +++ b/doc/tutorials/imgproc/shapedescriptors/find_contours/find_contours.markdown @@ -16,7 +16,7 @@ Code ---- This tutorial code's is shown lines below. You can also download it from -[here](https://github.com/Itseez/opencv/tree/master/samples/cpp/tutorial_code/ShapeDescriptors/findContours_demo.cpp) +[here](https://github.com/opencv/opencv/tree/master/samples/cpp/tutorial_code/ShapeDescriptors/findContours_demo.cpp) @include samples/cpp/tutorial_code/ShapeDescriptors/findContours_demo.cpp Explanation diff --git a/doc/tutorials/imgproc/shapedescriptors/hull/hull.markdown b/doc/tutorials/imgproc/shapedescriptors/hull/hull.markdown index 5af9de16a1..38df72029e 100644 --- a/doc/tutorials/imgproc/shapedescriptors/hull/hull.markdown +++ b/doc/tutorials/imgproc/shapedescriptors/hull/hull.markdown @@ -15,7 +15,7 @@ Code ---- This tutorial code's is shown lines below. You can also download it from -[here](https://github.com/Itseez/opencv/tree/master/samples/cpp/tutorial_code/ShapeDescriptors/hull_demo.cpp) +[here](https://github.com/opencv/opencv/tree/master/samples/cpp/tutorial_code/ShapeDescriptors/hull_demo.cpp) @include samples/cpp/tutorial_code/ShapeDescriptors/hull_demo.cpp diff --git a/doc/tutorials/imgproc/shapedescriptors/moments/moments.markdown b/doc/tutorials/imgproc/shapedescriptors/moments/moments.markdown index 9398701e4c..3ef4c13615 100644 --- a/doc/tutorials/imgproc/shapedescriptors/moments/moments.markdown +++ b/doc/tutorials/imgproc/shapedescriptors/moments/moments.markdown @@ -17,7 +17,7 @@ Code ---- This tutorial code's is shown lines below. You can also download it from -[here](https://github.com/Itseez/opencv/tree/master/samples/cpp/tutorial_code/ShapeDescriptors/moments_demo.cpp) +[here](https://github.com/opencv/opencv/tree/master/samples/cpp/tutorial_code/ShapeDescriptors/moments_demo.cpp) @include samples/cpp/tutorial_code/ShapeDescriptors/moments_demo.cpp Explanation diff --git a/doc/tutorials/imgproc/shapedescriptors/point_polygon_test/point_polygon_test.markdown b/doc/tutorials/imgproc/shapedescriptors/point_polygon_test/point_polygon_test.markdown index 03f612844c..4ffb98be48 100644 --- a/doc/tutorials/imgproc/shapedescriptors/point_polygon_test/point_polygon_test.markdown +++ b/doc/tutorials/imgproc/shapedescriptors/point_polygon_test/point_polygon_test.markdown @@ -15,7 +15,7 @@ Code ---- This tutorial code's is shown lines below. You can also download it from -[here](https://github.com/Itseez/opencv/tree/master/samples/cpp/tutorial_code/ShapeDescriptors/pointPolygonTest_demo.cpp) +[here](https://github.com/opencv/opencv/tree/master/samples/cpp/tutorial_code/ShapeDescriptors/pointPolygonTest_demo.cpp) @include samples/cpp/tutorial_code/ShapeDescriptors/pointPolygonTest_demo.cpp Explanation diff --git a/doc/tutorials/imgproc/threshold/threshold.markdown b/doc/tutorials/imgproc/threshold/threshold.markdown index b78ae9cd4c..f0bc735ab6 100644 --- a/doc/tutorials/imgproc/threshold/threshold.markdown +++ b/doc/tutorials/imgproc/threshold/threshold.markdown @@ -97,7 +97,7 @@ Code ---- The tutorial code's is shown lines below. You can also download it from -[here](https://github.com/Itseez/opencv/tree/master/samples/cpp/tutorial_code/ImgProc/Threshold.cpp) +[here](https://github.com/opencv/opencv/tree/master/samples/cpp/tutorial_code/ImgProc/Threshold.cpp) @include samples/cpp/tutorial_code/ImgProc/Threshold.cpp Explanation diff --git a/doc/tutorials/imgproc/threshold_inRange/threshold_inRange.markdown b/doc/tutorials/imgproc/threshold_inRange/threshold_inRange.markdown index edafc06818..101fa01d43 100644 --- a/doc/tutorials/imgproc/threshold_inRange/threshold_inRange.markdown +++ b/doc/tutorials/imgproc/threshold_inRange/threshold_inRange.markdown @@ -19,7 +19,7 @@ Code ---- The tutorial code's is shown lines below. You can also download it from -[here](https://github.com/Itseez/opencv/tree/master/samples/cpp/tutorial_code/ImgProc/Threshold_inRange.cpp) +[here](https://github.com/opencv/opencv/tree/master/samples/cpp/tutorial_code/ImgProc/Threshold_inRange.cpp) @include samples/cpp/tutorial_code/ImgProc/Threshold_inRange.cpp Explanation diff --git a/doc/tutorials/introduction/android_binary_package/android_ocl_intro.markdown b/doc/tutorials/introduction/android_binary_package/android_ocl_intro.markdown index b7751764a9..3dab6e8c98 100644 --- a/doc/tutorials/introduction/android_binary_package/android_ocl_intro.markdown +++ b/doc/tutorials/introduction/android_binary_package/android_ocl_intro.markdown @@ -17,7 +17,7 @@ If you need help with anything of the above, you may refer to our @ref tutorial_ This tutorial also assumes you have an Android operated device with OpenCL enabled. The related source code is located within OpenCV samples at -[opencv/samples/android/tutorial-4-opencl](https://github.com/Itseez/opencv/tree/master/samples/android/tutorial-4-opencl/) directory. +[opencv/samples/android/tutorial-4-opencl](https://github.com/opencv/opencv/tree/master/samples/android/tutorial-4-opencl/) directory. Preface ------- @@ -244,7 +244,7 @@ As you can see, inheritors for `Camera` and `Camera2` APIs should implement the @endcode Let's leave the details of their implementation beyond of this tutorial, please refer the -[source code](https://github.com/Itseez/opencv/tree/master/samples/android/tutorial-4-opencl/) to see them. +[source code](https://github.com/opencv/opencv/tree/master/samples/android/tutorial-4-opencl/) to see them. Preview Frames modification --------------------------- diff --git a/doc/tutorials/introduction/clojure_dev_intro/clojure_dev_intro.markdown b/doc/tutorials/introduction/clojure_dev_intro/clojure_dev_intro.markdown index 5af960843c..7765e1af95 100644 --- a/doc/tutorials/introduction/clojure_dev_intro/clojure_dev_intro.markdown +++ b/doc/tutorials/introduction/clojure_dev_intro/clojure_dev_intro.markdown @@ -40,7 +40,7 @@ I'm assuming you already installed [xcode](https://developer.apple.com/xcode/), @code{.bash} cd ~/ mkdir opt -git clone https://github.com/Itseez/opencv.git +git clone https://github.com/opencv/opencv.git cd opencv git checkout 2.4 mkdir build diff --git a/doc/tutorials/introduction/crosscompilation/arm_crosscompile_with_cmake.markdown b/doc/tutorials/introduction/crosscompilation/arm_crosscompile_with_cmake.markdown index 1b9dc30ea5..3ef08f8a39 100644 --- a/doc/tutorials/introduction/crosscompilation/arm_crosscompile_with_cmake.markdown +++ b/doc/tutorials/introduction/crosscompilation/arm_crosscompile_with_cmake.markdown @@ -33,7 +33,7 @@ Getting OpenCV Source Code -------------------------- You can use the latest stable OpenCV version available in *sourceforge* or you can grab the latest -snapshot from our [Git repository](https://github.com/Itseez/opencv.git). +snapshot from our [Git repository](https://github.com/opencv/opencv.git). ### Getting the Latest Stable OpenCV Version @@ -42,12 +42,12 @@ snapshot from our [Git repository](https://github.com/Itseez/opencv.git). ### Getting the Cutting-edge OpenCV from the Git Repository -Launch Git client and clone [OpenCV repository](http://github.com/itseez/opencv) +Launch Git client and clone [OpenCV repository](http://github.com/opencv/opencv) In Linux it can be achieved with the following command in Terminal: @code{.bash} cd ~/ -git clone https://github.com/Itseez/opencv.git +git clone https://github.com/opencv/opencv.git @endcode Building OpenCV diff --git a/doc/tutorials/introduction/desktop_java/java_dev_intro.markdown b/doc/tutorials/introduction/desktop_java/java_dev_intro.markdown index 9e8048d168..c6e8c06f7c 100644 --- a/doc/tutorials/introduction/desktop_java/java_dev_intro.markdown +++ b/doc/tutorials/introduction/desktop_java/java_dev_intro.markdown @@ -36,7 +36,7 @@ from the [OpenCV SourceForge repository](http://sourceforge.net/projects/opencvl sources. Another option to get OpenCV sources is to clone [OpenCV git -repository](https://github.com/Itseez/opencv/). In order to build OpenCV with Java bindings you need +repository](https://github.com/opencv/opencv/). In order to build OpenCV with Java bindings you need JDK (Java Development Kit) (we recommend [Oracle/Sun JDK 6 or 7](http://www.oracle.com/technetwork/java/javase/downloads/)), [Apache Ant](http://ant.apache.org/) and Python v2.6 or higher to be installed. @@ -45,7 +45,7 @@ and Python v2.6 or higher to be installed. Let's build OpenCV: @code{.bash} -git clone git://github.com/Itseez/opencv.git +git clone git://github.com/opencv/opencv.git cd opencv git checkout 2.4 mkdir build diff --git a/doc/tutorials/introduction/display_image/display_image.markdown b/doc/tutorials/introduction/display_image/display_image.markdown index c4a1f1df59..f0eea98ee7 100644 --- a/doc/tutorials/introduction/display_image/display_image.markdown +++ b/doc/tutorials/introduction/display_image/display_image.markdown @@ -14,7 +14,7 @@ Source Code ----------- Download the source code from -[here](https://github.com/Itseez/opencv/tree/master/samples/cpp/tutorial_code/introduction/display_image/display_image.cpp). +[here](https://github.com/opencv/opencv/tree/master/samples/cpp/tutorial_code/introduction/display_image/display_image.cpp). @include cpp/tutorial_code/introduction/display_image/display_image.cpp diff --git a/doc/tutorials/introduction/ios_install/ios_install.markdown b/doc/tutorials/introduction/ios_install/ios_install.markdown index 00faf685f5..9a82e15205 100644 --- a/doc/tutorials/introduction/ios_install/ios_install.markdown +++ b/doc/tutorials/introduction/ios_install/ios_install.markdown @@ -9,13 +9,13 @@ Required Packages ### Getting the Cutting-edge OpenCV from Git Repository -Launch GIT client and clone OpenCV repository from [here](http://github.com/itseez/opencv) +Launch GIT client and clone OpenCV repository from [here](http://github.com/opencv/opencv) In MacOS it can be done using the following command in Terminal: @code{.bash} cd ~/ -git clone https://github.com/Itseez/opencv.git +git clone https://github.com/opencv/opencv.git @endcode Building OpenCV from Source, using CMake and Command Line diff --git a/doc/tutorials/introduction/linux_install/linux_install.markdown b/doc/tutorials/introduction/linux_install/linux_install.markdown index b1868f82d7..387902dfec 100644 --- a/doc/tutorials/introduction/linux_install/linux_install.markdown +++ b/doc/tutorials/introduction/linux_install/linux_install.markdown @@ -28,7 +28,7 @@ Getting OpenCV Source Code -------------------------- You can use the latest stable OpenCV version or you can grab the latest snapshot from our [Git -repository](https://github.com/Itseez/opencv.git). +repository](https://github.com/opencv/opencv.git). ### Getting the Latest Stable OpenCV Version @@ -37,14 +37,14 @@ repository](https://github.com/Itseez/opencv.git). ### Getting the Cutting-edge OpenCV from the Git Repository -Launch Git client and clone [OpenCV repository](http://github.com/itseez/opencv). If you need -modules from [OpenCV contrib repository](http://github.com/itseez/opencv_contrib) then clone it too. +Launch Git client and clone [OpenCV repository](http://github.com/opencv/opencv). If you need +modules from [OpenCV contrib repository](http://github.com/opencv/opencv_contrib) then clone it too. For example @code{.bash} cd ~/ -git clone https://github.com/Itseez/opencv.git -git clone https://github.com/Itseez/opencv_contrib.git +git clone https://github.com/opencv/opencv.git +git clone https://github.com/opencv/opencv_contrib.git @endcode Building OpenCV from Source Using CMake --------------------------------------- @@ -114,11 +114,11 @@ Building OpenCV from Source Using CMake -# [optional] Running tests - Get the required test data from [OpenCV extra - repository](https://github.com/Itseez/opencv_extra). + repository](https://github.com/opencv/opencv_extra). For example @code{.bash} - git clone https://github.com/Itseez/opencv_extra.git + git clone https://github.com/opencv/opencv_extra.git @endcode - set OPENCV_TEST_DATA_PATH environment variable to \. - execute tests from build directory. diff --git a/doc/tutorials/introduction/transition_guide/transition_guide.markdown b/doc/tutorials/introduction/transition_guide/transition_guide.markdown index b0bc44344c..3b9620a086 100644 --- a/doc/tutorials/introduction/transition_guide/transition_guide.markdown +++ b/doc/tutorials/introduction/transition_guide/transition_guide.markdown @@ -12,7 +12,7 @@ OpenCV 3.0 introduced many new algorithms and features comparing to version 2.4. This section describes most notable changes in general, all details and examples of transition actions are in the next part of the document. ##### Contrib repository - + This is a place for all new, experimental and non-free algorithms. It does not receive so much attention from the support team comparing to main repository, but the community makes an effort to keep it in a good shape. diff --git a/doc/tutorials/introduction/windows_install/windows_install.markdown b/doc/tutorials/introduction/windows_install/windows_install.markdown index c8808b493a..22c9d99683 100644 --- a/doc/tutorials/introduction/windows_install/windows_install.markdown +++ b/doc/tutorials/introduction/windows_install/windows_install.markdown @@ -43,7 +43,7 @@ These videos above are long-obsolete and contain inaccurate information. Be care solutions described in those videos are no longer supported and may even break your install. If you are building your own libraries you can take the source files from our [Git -repository](https://github.com/Itseez/opencv.git). +repository](https://github.com/opencv/opencv.git). Building the OpenCV library from scratch requires a couple of tools installed beforehand: @@ -114,7 +114,7 @@ libraries). If you do not need the support for some of these you can just freely you're doing -- it's OK. -# Clone the repository to the selected directory. After clicking *Clone* button, a window will appear where you can select from what repository you want to download source files - () and to what directory (`D:/OpenCV`). + () and to what directory (`D:/OpenCV`). -# Push the OK button and be patient as the repository is quite a heavy download. It will take some time depending on your Internet connection. diff --git a/doc/tutorials/introduction/windows_visual_studio_Opencv/windows_visual_studio_Opencv.markdown b/doc/tutorials/introduction/windows_visual_studio_Opencv/windows_visual_studio_Opencv.markdown index 78776af33e..3be7f5bb87 100644 --- a/doc/tutorials/introduction/windows_visual_studio_Opencv/windows_visual_studio_Opencv.markdown +++ b/doc/tutorials/introduction/windows_visual_studio_Opencv/windows_visual_studio_Opencv.markdown @@ -189,7 +189,7 @@ Test it! -------- Now to try this out download our little test [source code -](https://github.com/Itseez/opencv/tree/master/samples/cpp/tutorial_code/introduction/windows_visual_studio_Opencv/introduction_windows_vs.cpp) +](https://github.com/opencv/opencv/tree/master/samples/cpp/tutorial_code/introduction/windows_visual_studio_Opencv/introduction_windows_vs.cpp) or get it from the sample code folder of the OpenCV sources. Add this to your project and build it. Here's its content: @@ -205,7 +205,7 @@ the *IDE* the console window will not close once finished. It will wait for a ke This is important to remember when you code inside the code open and save commands. You're resources will be saved ( and queried for at opening!!!) relatively to your working directory. This is unless you give a full, explicit path as parameter for the I/O functions. In the code above we open [this -OpenCV logo](https://github.com/Itseez/opencv/tree/master/samples/data/opencv-logo.png). Before starting up the application make sure you place +OpenCV logo](https://github.com/opencv/opencv/tree/master/samples/data/opencv-logo.png). Before starting up the application make sure you place the image file in your current working directory. Modify the image file name inside the code to try it out on other images too. Run it and voil á: diff --git a/doc/tutorials/ml/introduction_to_pca/introduction_to_pca.markdown b/doc/tutorials/ml/introduction_to_pca/introduction_to_pca.markdown index 78ed547809..82274efedd 100644 --- a/doc/tutorials/ml/introduction_to_pca/introduction_to_pca.markdown +++ b/doc/tutorials/ml/introduction_to_pca/introduction_to_pca.markdown @@ -92,10 +92,10 @@ Source Code ----------- This tutorial code's is shown lines below. You can also download it from - [here](https://github.com/Itseez/opencv/tree/master/samples/cpp/tutorial_code/ml/introduction_to_pca/introduction_to_pca.cpp). + [here](https://github.com/opencv/tree/master/samples/cpp/tutorial_code/ml/introduction_to_pca/introduction_to_pca.cpp). @include cpp/tutorial_code/ml/introduction_to_pca/introduction_to_pca.cpp -@note Another example using PCA for dimensionality reduction while maintaining an amount of variance can be found at [opencv_source_code/samples/cpp/pca.cpp](https://github.com/Itseez/opencv/tree/master/samples/cpp/pca.cpp) +@note Another example using PCA for dimensionality reduction while maintaining an amount of variance can be found at [opencv_source_code/samples/cpp/pca.cpp](https://github.com/opencv/tree/master/samples/cpp/pca.cpp) Explanation ----------- diff --git a/doc/tutorials/ml/non_linear_svms/non_linear_svms.markdown b/doc/tutorials/ml/non_linear_svms/non_linear_svms.markdown index 52b544dd06..1a18870079 100644 --- a/doc/tutorials/ml/non_linear_svms/non_linear_svms.markdown +++ b/doc/tutorials/ml/non_linear_svms/non_linear_svms.markdown @@ -87,7 +87,7 @@ Source Code ----------- You may also find the source code in `samples/cpp/tutorial_code/ml/non_linear_svms` folder of the OpenCV source library or -[download it from here](https://github.com/Itseez/opencv/tree/master/samples/cpp/tutorial_code/ml/non_linear_svms/non_linear_svms.cpp). +[download it from here](https://github.com/opencv/tree/master/samples/cpp/tutorial_code/ml/non_linear_svms/non_linear_svms.cpp). @note The following code has been implemented with OpenCV 3.0 classes and functions. An equivalent version of the code using OpenCV 2.4 can be found in [this page.](http://docs.opencv.org/2.4/doc/tutorials/ml/non_linear_svms/non_linear_svms.html#nonlinearsvms) diff --git a/doc/tutorials/objdetect/cascade_classifier/cascade_classifier.markdown b/doc/tutorials/objdetect/cascade_classifier/cascade_classifier.markdown index 382046fe34..ec27dc2731 100644 --- a/doc/tutorials/objdetect/cascade_classifier/cascade_classifier.markdown +++ b/doc/tutorials/objdetect/cascade_classifier/cascade_classifier.markdown @@ -18,9 +18,9 @@ Code ---- This tutorial code's is shown lines below. You can also download it from -[here](https://github.com/Itseez/opencv/tree/master/samples/cpp/tutorial_code/objectDetection/objectDetection.cpp) +[here](https://github.com/opencv/tree/master/samples/cpp/tutorial_code/objectDetection/objectDetection.cpp) . The second version (using LBP for face detection) can be [found -here](https://github.com/Itseez/opencv/tree/master/samples/cpp/tutorial_code/objectDetection/objectDetection2.cpp) +here](https://github.com/opencv/tree/master/samples/cpp/tutorial_code/objectDetection/objectDetection2.cpp) @code{.cpp} #include "opencv2/objdetect.hpp" #include "opencv2/highgui.hpp" diff --git a/doc/tutorials/video/background_subtraction/background_subtraction.markdown b/doc/tutorials/video/background_subtraction/background_subtraction.markdown index f9b7eebf15..8ed68f5e25 100644 --- a/doc/tutorials/video/background_subtraction/background_subtraction.markdown +++ b/doc/tutorials/video/background_subtraction/background_subtraction.markdown @@ -45,7 +45,7 @@ Two different methods are used to generate two foreground masks: -# @ref cv::BackgroundSubtractorMOG2 The results as well as the input data are shown on the screen. -The source file can be downloaded [here ](https://github.com/Itseez/opencv/tree/master/samples/cpp/tutorial_code/video/bg_sub.cpp). +The source file can be downloaded [here ](https://github.com/opencv/opencv/tree/master/samples/cpp/tutorial_code/video/bg_sub.cpp). @include samples/cpp/tutorial_code/video/bg_sub.cpp diff --git a/doc/tutorials/videoio/intelperc.markdown b/doc/tutorials/videoio/intelperc.markdown index b5f2ed64ed..188a69e659 100644 --- a/doc/tutorials/videoio/intelperc.markdown +++ b/doc/tutorials/videoio/intelperc.markdown @@ -78,5 +78,5 @@ there are two flags that should be used to set/get property of the needed genera flag value is assumed by default if neither of the two possible values of the property is set. For more information please refer to the example of usage -[intelperc_capture.cpp](https://github.com/Itseez/opencv/tree/master/samples/cpp/intelperc_capture.cpp) +[intelperc_capture.cpp](https://github.com/opencv/tree/master/samples/cpp/intelperc_capture.cpp) in opencv/samples/cpp folder. diff --git a/doc/tutorials/videoio/kinect_openni.markdown b/doc/tutorials/videoio/kinect_openni.markdown index c9c33a2a05..82ec13ef91 100644 --- a/doc/tutorials/videoio/kinect_openni.markdown +++ b/doc/tutorials/videoio/kinect_openni.markdown @@ -134,5 +134,5 @@ property. The following properties of cameras available through OpenNI interface - CAP_OPENNI_DEPTH_GENERATOR_REGISTRATION = CAP_OPENNI_DEPTH_GENERATOR + CAP_PROP_OPENNI_REGISTRATION For more information please refer to the example of usage -[openni_capture.cpp](https://github.com/Itseez/opencv/tree/master/samples/cpp/openni_capture.cpp) in +[openni_capture.cpp](https://github.com/opencv/tree/master/samples/cpp/openni_capture.cpp) in opencv/samples/cpp folder. diff --git a/doc/tutorials/videoio/video-input-psnr-ssim/video_input_psnr_ssim.markdown b/doc/tutorials/videoio/video-input-psnr-ssim/video_input_psnr_ssim.markdown index 9dbff286ca..fdaad3aa26 100644 --- a/doc/tutorials/videoio/video-input-psnr-ssim/video_input_psnr_ssim.markdown +++ b/doc/tutorials/videoio/video-input-psnr-ssim/video_input_psnr_ssim.markdown @@ -20,8 +20,8 @@ As a test case where to show off these using OpenCV I've created a small program video files and performs a similarity check between them. This is something you could use to check just how well a new video compressing algorithms works. Let there be a reference (original) video like [this small Megamind clip -](https://github.com/Itseez/opencv/tree/master/samples/data/Megamind.avi) and [a compressed -version of it ](https://github.com/Itseez/opencv/tree/master/samples/data/Megamind_bugy.avi). +](https://github.com/opencv/opencv/tree/master/samples/data/Megamind.avi) and [a compressed +version of it ](https://github.com/opencv/opencv/tree/master/samples/data/Megamind_bugy.avi). You may also find the source code and these video file in the `samples/data` folder of the OpenCV source library. diff --git a/doc/tutorials/videoio/video-write/video_write.markdown b/doc/tutorials/videoio/video-write/video_write.markdown index 399ac7a1e2..38c9151248 100644 --- a/doc/tutorials/videoio/video-write/video_write.markdown +++ b/doc/tutorials/videoio/video-write/video_write.markdown @@ -31,7 +31,7 @@ The source code You may also find the source code and these video file in the `samples/cpp/tutorial_code/videoio/video-write/` folder of the OpenCV source library or [download it -from here ](https://github.com/Itseez/opencv/tree/master/samples/cpp/tutorial_code/videoio/video-write/video-write.cpp). +from here ](https://github.com/opencv/opencv/tree/master/samples/cpp/tutorial_code/videoio/video-write/video-write.cpp). @include cpp/tutorial_code/videoio/video-write/video-write.cpp diff --git a/doc/tutorials/viz/creating_widgets/creating_widgets.markdown b/doc/tutorials/viz/creating_widgets/creating_widgets.markdown index ea84fe3aa0..4e4f6e5aed 100644 --- a/doc/tutorials/viz/creating_widgets/creating_widgets.markdown +++ b/doc/tutorials/viz/creating_widgets/creating_widgets.markdown @@ -12,7 +12,7 @@ In this tutorial you will learn how to Code ---- -You can download the code from [here ](https://github.com/Itseez/opencv/tree/master/samples/cpp/tutorial_code/viz/creating_widgets.cpp). +You can download the code from [here ](https://github.com/opencv/opencv/tree/master/samples/cpp/tutorial_code/viz/creating_widgets.cpp). @include samples/cpp/tutorial_code/viz/creating_widgets.cpp Explanation diff --git a/doc/tutorials/viz/launching_viz/launching_viz.markdown b/doc/tutorials/viz/launching_viz/launching_viz.markdown index 25914dcb30..07719c67eb 100644 --- a/doc/tutorials/viz/launching_viz/launching_viz.markdown +++ b/doc/tutorials/viz/launching_viz/launching_viz.markdown @@ -14,7 +14,7 @@ In this tutorial you will learn how to Code ---- -You can download the code from [here ](https://github.com/Itseez/opencv/tree/master/samples/cpp/tutorial_code/viz/launching_viz.cpp). +You can download the code from [here ](https://github.com/opencv/opencv/tree/master/samples/cpp/tutorial_code/viz/launching_viz.cpp). @include samples/cpp/tutorial_code/viz/launching_viz.cpp Explanation diff --git a/doc/tutorials/viz/transformations/transformations.markdown b/doc/tutorials/viz/transformations/transformations.markdown index 8b9c1f6722..512ce80bdb 100644 --- a/doc/tutorials/viz/transformations/transformations.markdown +++ b/doc/tutorials/viz/transformations/transformations.markdown @@ -13,7 +13,7 @@ In this tutorial you will learn how to Code ---- -You can download the code from [here ](https://github.com/Itseez/opencv/tree/master/samples/cpp/tutorial_code/viz/transformations.cpp). +You can download the code from [here ](https://github.com/opencv/opencv/tree/master/samples/cpp/tutorial_code/viz/transformations.cpp). @include samples/cpp/tutorial_code/viz/transformations.cpp Explanation diff --git a/doc/tutorials/viz/widget_pose/widget_pose.markdown b/doc/tutorials/viz/widget_pose/widget_pose.markdown index 7e6a1b9da2..382ae98556 100644 --- a/doc/tutorials/viz/widget_pose/widget_pose.markdown +++ b/doc/tutorials/viz/widget_pose/widget_pose.markdown @@ -13,7 +13,7 @@ In this tutorial you will learn how to Code ---- -You can download the code from [here ](https://github.com/Itseez/opencv/tree/master/samples/cpp/tutorial_code/viz/widget_pose.cpp). +You can download the code from [here ](https://github.com/opencv/opencv/tree/master/samples/cpp/tutorial_code/viz/widget_pose.cpp). @include samples/cpp/tutorial_code/viz/widget_pose.cpp Explanation diff --git a/modules/core/doc/cuda.markdown b/modules/core/doc/cuda.markdown index fb648e25dd..ea85007a34 100644 --- a/modules/core/doc/cuda.markdown +++ b/modules/core/doc/cuda.markdown @@ -82,4 +82,4 @@ Block Matching algorithm has been successfully parallelized using the following 3. Merge the results into a single disparity map. With this algorithm, a dual GPU gave a 180% performance increase comparing to the single Fermi GPU. -For a source code example, see . +For a source code example, see . diff --git a/modules/highgui/include/opencv2/highgui.hpp b/modules/highgui/include/opencv2/highgui.hpp index a515c15b33..7f963a5cdf 100644 --- a/modules/highgui/include/opencv2/highgui.hpp +++ b/modules/highgui/include/opencv2/highgui.hpp @@ -425,7 +425,7 @@ CV_EXPORTS_W double getWindowProperty(const String& winname, int prop_id); @param winname Name of the window. @param onMouse Mouse callback. See OpenCV samples, such as -, on how to specify and +, on how to specify and use the callback. @param userdata The optional parameter passed to the callback. */ diff --git a/modules/objdetect/include/opencv2/objdetect.hpp b/modules/objdetect/include/opencv2/objdetect.hpp index 6587b3d77a..87d7ec3ac5 100644 --- a/modules/objdetect/include/opencv2/objdetect.hpp +++ b/modules/objdetect/include/opencv2/objdetect.hpp @@ -91,7 +91,7 @@ compensate for the differences in the size of areas. The sums of pixel values ov regions are calculated rapidly using integral images (see below and the integral description). To see the object detector at work, have a look at the facedetect demo: - + The following reference is for the detection part only. There is a separate application called opencv_traincascade that can train a cascade of boosted classifiers from a set of samples. diff --git a/modules/python/test/test_digits.py b/modules/python/test/test_digits.py index 2d5c900438..c7ac9964b7 100644 --- a/modules/python/test/test_digits.py +++ b/modules/python/test/test_digits.py @@ -58,7 +58,7 @@ def deskew(img): class StatModel(object): def load(self, fn): - self.model.load(fn) # Known bug: https://github.com/Itseez/opencv/issues/4969 + self.model.load(fn) # Known bug: https://github.com/opencv/opencv/issues/4969 def save(self, fn): self.model.save(fn) diff --git a/modules/python/test/test_gaussian_mix.py b/modules/python/test/test_gaussian_mix.py index cfd33ece02..78a29ceb4e 100644 --- a/modules/python/test/test_gaussian_mix.py +++ b/modules/python/test/test_gaussian_mix.py @@ -43,7 +43,7 @@ class gaussian_mix_test(NewOpenCVTests): em.setCovarianceMatrixType(cv2.ml.EM_COV_MAT_GENERIC) em.trainEM(points) means = em.getMeans() - covs = em.getCovs() # Known bug: https://github.com/Itseez/opencv/pull/4232 + covs = em.getCovs() # Known bug: https://github.com/opencv/opencv/pull/4232 found_distrs = zip(means, covs) matches_count = 0 diff --git a/modules/python/test/tests_common.py b/modules/python/test/tests_common.py index 67b5bea2f7..17ef0dc923 100644 --- a/modules/python/test/tests_common.py +++ b/modules/python/test/tests_common.py @@ -21,7 +21,7 @@ class NewOpenCVTests(unittest.TestCase): repoPath = None extraTestDataPath = None # github repository url - repoUrl = 'https://raw.github.com/Itseez/opencv/master' + repoUrl = 'https://raw.github.com/opencv/opencv/master' def get_sample(self, filename, iscolor = cv2.IMREAD_COLOR): if not filename in self.image_cache: diff --git a/modules/videoio/include/opencv2/videoio.hpp b/modules/videoio/include/opencv2/videoio.hpp index 0acb3d3f1e..c462c67ce5 100644 --- a/modules/videoio/include/opencv2/videoio.hpp +++ b/modules/videoio/include/opencv2/videoio.hpp @@ -614,7 +614,7 @@ public: Also, when a connected camera is multi-head (for example, a stereo camera or a Kinect device), the correct way of retrieving data from it is to call VideoCapture::grab first and then call VideoCapture::retrieve one or more times with different values of the channel parameter. See - + */ CV_WRAP virtual bool grab(); diff --git a/modules/videoio/src/cap_libv4l.cpp b/modules/videoio/src/cap_libv4l.cpp index 214146144a..3ff564b07f 100644 --- a/modules/videoio/src/cap_libv4l.cpp +++ b/modules/videoio/src/cap_libv4l.cpp @@ -16,7 +16,7 @@ For Release: OpenCV-Linux Beta4 opencv-0.9.6 Tested On: LMLBT44 with 8 video inputs Problems? Post your questions at answers.opencv.org, Report bugs at code.opencv.org, - Submit your fixes at https://github.com/Itseez/opencv/ + Submit your fixes at https://github.com/opencv/opencv/ Patched Comments: TW: The cv cam utils that came with the initial release of OpenCV for LINUX Beta4 diff --git a/modules/videoio/src/cap_v4l.cpp b/modules/videoio/src/cap_v4l.cpp index 227b4aa335..01a250b41e 100644 --- a/modules/videoio/src/cap_v4l.cpp +++ b/modules/videoio/src/cap_v4l.cpp @@ -16,7 +16,7 @@ For Release: OpenCV-Linux Beta4 opencv-0.9.6 Tested On: LMLBT44 with 8 video inputs Problems? Post your questions at answers.opencv.org, Report bugs at code.opencv.org, - Submit your fixes at https://github.com/Itseez/opencv/ + Submit your fixes at https://github.com/opencv/opencv/ Patched Comments: TW: The cv cam utils that came with the initial release of OpenCV for LINUX Beta4 diff --git a/platforms/winrt/readme.txt b/platforms/winrt/readme.txt index c35d18d21f..2fb4ce1f54 100644 --- a/platforms/winrt/readme.txt +++ b/platforms/winrt/readme.txt @@ -124,7 +124,7 @@ Running tests for Windows Store =============================== 1. You might need to install this if you haven't already: http://www.microsoft.com/en-US/download/details.aspx?id=40784 -2. Set OPENCV_TEST_DATA_PATH environment variable to location of opencv_extra/testdata (cloning of https://github.com/Itseez/opencv_extra repo required) to get tests work correctly. Also, set OPENCV_PERF_VALIDATION_DIR environment variable in case you are planning to have place where to store performance test results and compare them with the future test runs. +2. Set OPENCV_TEST_DATA_PATH environment variable to location of opencv_extra/testdata (cloning of https://github.com/opencv/opencv_extra repo required) to get tests work correctly. Also, set OPENCV_PERF_VALIDATION_DIR environment variable in case you are planning to have place where to store performance test results and compare them with the future test runs. 3. In case you'd like to adjust some flags that are defaulted by setup_winrt script, go to "Manual build" section. Otherwise go to platforms/winrt and execute diff --git a/samples/android/camera-calibration/src/org/opencv/samples/cameracalibration/CameraCalibrationActivity.java b/samples/android/camera-calibration/src/org/opencv/samples/cameracalibration/CameraCalibrationActivity.java index aa90504054..af0853a7cc 100644 --- a/samples/android/camera-calibration/src/org/opencv/samples/cameracalibration/CameraCalibrationActivity.java +++ b/samples/android/camera-calibration/src/org/opencv/samples/cameracalibration/CameraCalibrationActivity.java @@ -2,7 +2,7 @@ // http://docs.opencv.org/doc/tutorials/calib3d/camera_calibration/camera_calibration.html // // It uses standard OpenCV asymmetric circles grid pattern 11x4: -// https://github.com/Itseez/opencv/blob/2.4/doc/acircles_pattern.png. +// https://github.com/opencv/opencv/blob/2.4/doc/acircles_pattern.png. // The results are the camera matrix and 5 distortion coefficients. // // Tap on highlighted pattern to capture pattern corners for calibration. diff --git a/samples/cpp/facial_features.cpp b/samples/cpp/facial_features.cpp index 3ed6a442f9..6dbef75c49 100644 --- a/samples/cpp/facial_features.cpp +++ b/samples/cpp/facial_features.cpp @@ -89,10 +89,10 @@ static void help() "\tThis will detect only the face in image.jpg.\n"; cout << " \n\nThe classifiers for face and eyes can be downloaded from : " - " \nhttps://github.com/Itseez/opencv/tree/master/data/haarcascades"; + " \nhttps://github.com/opencv/opencv/tree/master/data/haarcascades"; cout << "\n\nThe classifiers for nose and mouth can be downloaded from : " - " \nhttps://github.com/Itseez/opencv_contrib/tree/master/modules/face/data/cascades\n"; + " \nhttps://github.com/opencv/opencv_contrib/tree/master/modules/face/data/cascades\n"; } static void detectFaces(Mat& img, vector >& faces, string cascade_path) diff --git a/samples/cpp/stereo_calib.cpp b/samples/cpp/stereo_calib.cpp index 025e1d3b60..000928c1cf 100644 --- a/samples/cpp/stereo_calib.cpp +++ b/samples/cpp/stereo_calib.cpp @@ -19,7 +19,7 @@ Online docs: http://docs.opencv.org Q&A forum: http://answers.opencv.org Issue tracker: http://code.opencv.org - GitHub: https://github.com/Itseez/opencv/ + GitHub: https://github.com/opencv/opencv/ ************************************************** */ #include "opencv2/calib3d.hpp" diff --git a/samples/python/digits.py b/samples/python/digits.py index 04397a1431..5cc98b713c 100755 --- a/samples/python/digits.py +++ b/samples/python/digits.py @@ -71,7 +71,7 @@ def deskew(img): class StatModel(object): def load(self, fn): - self.model.load(fn) # Known bug: https://github.com/Itseez/opencv/issues/4969 + self.model.load(fn) # Known bug: https://github.com/opencv/opencv/issues/4969 def save(self, fn): self.model.save(fn) diff --git a/samples/python/digits_video.py b/samples/python/digits_video.py index c85deb6d0a..74b9d6993b 100755 --- a/samples/python/digits_video.py +++ b/samples/python/digits_video.py @@ -32,7 +32,7 @@ def main(): model = cv2.ml.SVM_load(classifier_fn) else: model = cv2.ml.SVM_create() - model.load_(classifier_fn) #Known bug: https://github.com/Itseez/opencv/issues/4969 + model.load_(classifier_fn) #Known bug: https://github.com/opencv/opencv/issues/4969 while True: ret, frame = cap.read() diff --git a/samples/python/find_obj.py b/samples/python/find_obj.py index 09457b80be..a77024c4c0 100755 --- a/samples/python/find_obj.py +++ b/samples/python/find_obj.py @@ -3,7 +3,7 @@ ''' Feature-based image matching sample. -Note, that you will need the https://github.com/Itseez/opencv_contrib repo for SIFT and SURF +Note, that you will need the https://github.com/opencv/opencv_contrib repo for SIFT and SURF USAGE find_obj.py [--feature=[-flann]] [ ] diff --git a/samples/python/gaussian_mix.py b/samples/python/gaussian_mix.py index b0c28748f2..64fbe23462 100755 --- a/samples/python/gaussian_mix.py +++ b/samples/python/gaussian_mix.py @@ -50,7 +50,7 @@ if __name__ == '__main__': em.setCovarianceMatrixType(cv2.ml.EM_COV_MAT_GENERIC) em.trainEM(points) means = em.getMeans() - covs = em.getCovs() # Known bug: https://github.com/Itseez/opencv/pull/4232 + covs = em.getCovs() # Known bug: https://github.com/opencv/opencv/pull/4232 found_distrs = zip(means, covs) print('ready!\n') diff --git a/samples/python/video_v4l2.py b/samples/python/video_v4l2.py index 9ef958e69d..eed22d11b8 100644 --- a/samples/python/video_v4l2.py +++ b/samples/python/video_v4l2.py @@ -27,7 +27,7 @@ font = cv2.FONT_HERSHEY_SIMPLEX color = (0, 255, 0) cap = cv2.VideoCapture(0) -cap.set(cv2.CAP_PROP_AUTOFOCUS, False) # Known bug: https://github.com/Itseez/opencv/pull/5474 +cap.set(cv2.CAP_PROP_AUTOFOCUS, False) # Known bug: https://github.com/opencv/opencv/pull/5474 cv2.namedWindow("Video") From 5bc10ef796c29563e8916ff31b5c1a262422a93f Mon Sep 17 00:00:00 2001 From: Ilya Lavrenov Date: Tue, 12 Jul 2016 12:39:49 +0300 Subject: [PATCH 076/128] fixed empty image condition in resize --- modules/imgproc/src/imgwarp.cpp | 11 +++---- modules/imgproc/test/test_imgwarp_strict.cpp | 31 ++++++++++++++++++++ 2 files changed, 37 insertions(+), 5 deletions(-) diff --git a/modules/imgproc/src/imgwarp.cpp b/modules/imgproc/src/imgwarp.cpp index d98ec11ad2..54054883df 100644 --- a/modules/imgproc/src/imgwarp.cpp +++ b/modules/imgproc/src/imgwarp.cpp @@ -3477,7 +3477,7 @@ void cv::resize( InputArray _src, OutputArray _dst, Size dsize, { Size ssize = _src.size(); - CV_Assert( ssize.area() > 0 ); + CV_Assert( ssize.width > 0 && ssize.height > 0 ); CV_Assert( dsize.area() > 0 || (inv_scale_x > 0 && inv_scale_y > 0) ); if( dsize.area() == 0 ) { @@ -3498,10 +3498,11 @@ void cv::resize( InputArray _src, OutputArray _dst, Size dsize, _dst.create(dsize, src.type()); Mat dst = _dst.getMat(); - if (dsize == ssize) { - // Source and destination are of same size. Use simple copy. - src.copyTo(dst); - return; + if (dsize == ssize) + { + // Source and destination are of same size. Use simple copy. + src.copyTo(dst); + return; } hal::resize(src.type(), src.data, src.step, src.cols, src.rows, dst.data, dst.step, dst.cols, dst.rows, inv_scale_x, inv_scale_y, interpolation); diff --git a/modules/imgproc/test/test_imgwarp_strict.cpp b/modules/imgproc/test/test_imgwarp_strict.cpp index 4756b7f42e..7a9c912836 100644 --- a/modules/imgproc/test/test_imgwarp_strict.cpp +++ b/modules/imgproc/test/test_imgwarp_strict.cpp @@ -1218,3 +1218,34 @@ TEST(Imgproc_Resize_Test, accuracy) { CV_Resize_Test test; test.safe_run(); } TEST(Imgproc_Remap_Test, accuracy) { CV_Remap_Test test; test.safe_run(); } TEST(Imgproc_WarpAffine_Test, accuracy) { CV_WarpAffine_Test test; test.safe_run(); } TEST(Imgproc_WarpPerspective_Test, accuracy) { CV_WarpPerspective_Test test; test.safe_run(); } + +//////////////////////////////////////////////////////////////////////////////////////////////////////// + +#ifdef OPENCV_TEST_BIGDATA + +CV_ENUM(Interpolation, INTER_NEAREST, INTER_LINEAR, INTER_CUBIC, INTER_AREA) + +class Imgproc_Resize : + public ::testing::TestWithParam +{ +public: + virtual void SetUp() + { + inter = GetParam(); + } + +protected: + int inter; +}; + +TEST_P(Imgproc_Resize, BigSize) +{ + cv::Mat src(46342, 46342, CV_8UC3, cv::Scalar::all(10)), dst; + ASSERT_FALSE(src.empty()); + + ASSERT_NO_THROW(cv::resize(src, dst, cv::Size(), 0.5, 0.5, inter)); +} + +INSTANTIATE_TEST_CASE_P(Imgproc, Imgproc_Resize, Interpolation::all()); + +#endif From 1f26e7347841fa9c65729a4432cff4bb1e7d184d Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Fri, 8 Jul 2016 15:47:50 +0300 Subject: [PATCH 077/128] ffmpeg: try to load ffmpeg wrapper dll from the current module directory --- modules/videoio/src/cap_ffmpeg.cpp | 48 ++++++++++++++++++++++++++---- 1 file changed, 43 insertions(+), 5 deletions(-) diff --git a/modules/videoio/src/cap_ffmpeg.cpp b/modules/videoio/src/cap_ffmpeg.cpp index b4348eaa05..772996e505 100644 --- a/modules/videoio/src/cap_ffmpeg.cpp +++ b/modules/videoio/src/cap_ffmpeg.cpp @@ -41,6 +41,8 @@ #include "precomp.hpp" +#include + #if defined HAVE_FFMPEG && !defined WIN32 #include "cap_ffmpeg_impl.hpp" #else @@ -59,6 +61,17 @@ static CvWriteFrame_Plugin icvWriteFrame_FFMPEG_p = 0; static cv::Mutex _icvInitFFMPEG_mutex; +#if defined WIN32 || defined _WIN32 +static const HMODULE cv_GetCurrentModule() +{ + HMODULE h = 0; + ::GetModuleHandleEx(GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS | GET_MODULE_HANDLE_EX_FLAG_UNCHANGED_REFCOUNT, + reinterpret_cast(cv_GetCurrentModule), + &h); + return h; +} +#endif + class icvInitFFMPEG { public: @@ -95,14 +108,39 @@ private: icvFFOpenCV = LoadPackagedLibrary( module_name, 0 ); # else - const char* module_name = "opencv_ffmpeg" - CVAUX_STR(CV_MAJOR_VERSION) CVAUX_STR(CV_MINOR_VERSION) CVAUX_STR(CV_SUBMINOR_VERSION) + const std::wstring module_name = L"opencv_ffmpeg" + CVAUX_STRW(CV_MAJOR_VERSION) CVAUX_STRW(CV_MINOR_VERSION) CVAUX_STRW(CV_SUBMINOR_VERSION) #if (defined _MSC_VER && defined _M_X64) || (defined __GNUC__ && defined __x86_64__) - "_64" + L"_64" #endif - ".dll"; + L".dll"; - icvFFOpenCV = LoadLibrary( module_name ); + const wchar_t* ffmpeg_env_path = _wgetenv(L"OPENCV_FFMPEG_DLL_DIR"); + std::wstring module_path = + ffmpeg_env_path + ? ((std::wstring(ffmpeg_env_path) + L"\\") + module_name) + : module_name; + + icvFFOpenCV = LoadLibraryW(module_path.c_str()); + if(!icvFFOpenCV && !ffmpeg_env_path) + { + HMODULE m = cv_GetCurrentModule(); + if (m) + { + wchar_t path[MAX_PATH]; + size_t sz = GetModuleFileNameW(m, path, sizeof(path)); + if (sz > 0 && ERROR_SUCCESS == GetLastError()) + { + wchar_t* s = wcsrchr(path, L'\\'); + if (s) + { + s[0] = 0; + module_path = (std::wstring(path) + L"\\") + module_name; + icvFFOpenCV = LoadLibraryW(module_path.c_str()); + } + } + } + } # endif if( icvFFOpenCV ) From 491b6543dad692c38456932994f0bcce061af39d Mon Sep 17 00:00:00 2001 From: Ilya Lavrenov Date: Tue, 12 Jul 2016 13:32:58 +0300 Subject: [PATCH 078/128] clarify CUDA arithm operations usage with mask --- .../cudaarithm/include/opencv2/cudaarithm.hpp | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/modules/cudaarithm/include/opencv2/cudaarithm.hpp b/modules/cudaarithm/include/opencv2/cudaarithm.hpp index d377a70e7a..46c7bfb327 100644 --- a/modules/cudaarithm/include/opencv2/cudaarithm.hpp +++ b/modules/cudaarithm/include/opencv2/cudaarithm.hpp @@ -77,7 +77,7 @@ namespace cv { namespace cuda { @param dst Destination matrix that has the same size and number of channels as the input array(s). The depth is defined by dtype or src1 depth. @param mask Optional operation mask, 8-bit single channel array, that specifies elements of the -destination array to be changed. +destination array to be changed. The mask can be used only with single channel images. @param dtype Optional depth of the output array. @param stream Stream for the asynchronous version. @@ -92,7 +92,7 @@ CV_EXPORTS void add(InputArray src1, InputArray src2, OutputArray dst, InputArra @param dst Destination matrix that has the same size and number of channels as the input array(s). The depth is defined by dtype or src1 depth. @param mask Optional operation mask, 8-bit single channel array, that specifies elements of the -destination array to be changed. +destination array to be changed. The mask can be used only with single channel images. @param dtype Optional depth of the output array. @param stream Stream for the asynchronous version. @@ -226,7 +226,8 @@ CV_EXPORTS void compare(InputArray src1, InputArray src2, OutputArray dst, int c @param src Source matrix. @param dst Destination matrix with the same size and type as src . -@param mask Optional operation mask. 8-bit single channel image. +@param mask Optional operation mask, 8-bit single channel array, that specifies elements of the +destination array to be changed. The mask can be used only with single channel images. @param stream Stream for the asynchronous version. */ CV_EXPORTS void bitwise_not(InputArray src, OutputArray dst, InputArray mask = noArray(), Stream& stream = Stream::Null()); @@ -236,7 +237,8 @@ CV_EXPORTS void bitwise_not(InputArray src, OutputArray dst, InputArray mask = n @param src1 First source matrix or scalar. @param src2 Second source matrix or scalar. @param dst Destination matrix that has the same size and type as the input array(s). -@param mask Optional operation mask. 8-bit single channel image. +@param mask Optional operation mask, 8-bit single channel array, that specifies elements of the +destination array to be changed. The mask can be used only with single channel images. @param stream Stream for the asynchronous version. */ CV_EXPORTS void bitwise_or(InputArray src1, InputArray src2, OutputArray dst, InputArray mask = noArray(), Stream& stream = Stream::Null()); @@ -246,7 +248,8 @@ CV_EXPORTS void bitwise_or(InputArray src1, InputArray src2, OutputArray dst, In @param src1 First source matrix or scalar. @param src2 Second source matrix or scalar. @param dst Destination matrix that has the same size and type as the input array(s). -@param mask Optional operation mask. 8-bit single channel image. +@param mask Optional operation mask, 8-bit single channel array, that specifies elements of the +destination array to be changed. The mask can be used only with single channel images. @param stream Stream for the asynchronous version. */ CV_EXPORTS void bitwise_and(InputArray src1, InputArray src2, OutputArray dst, InputArray mask = noArray(), Stream& stream = Stream::Null()); @@ -256,7 +259,8 @@ CV_EXPORTS void bitwise_and(InputArray src1, InputArray src2, OutputArray dst, I @param src1 First source matrix or scalar. @param src2 Second source matrix or scalar. @param dst Destination matrix that has the same size and type as the input array(s). -@param mask Optional operation mask. 8-bit single channel image. +@param mask Optional operation mask, 8-bit single channel array, that specifies elements of the +destination array to be changed. The mask can be used only with single channel images. @param stream Stream for the asynchronous version. */ CV_EXPORTS void bitwise_xor(InputArray src1, InputArray src2, OutputArray dst, InputArray mask = noArray(), Stream& stream = Stream::Null()); From ee9a42e9cdcf58991fd22e00cc9acfe055ff0c93 Mon Sep 17 00:00:00 2001 From: Vladislav Sovrasov Date: Tue, 12 Jul 2016 13:34:34 +0300 Subject: [PATCH 079/128] Fix calibration fail on python with CALIB_THIN_PRISM_MODEL flag --- modules/calib3d/src/calibration.cpp | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/modules/calib3d/src/calibration.cpp b/modules/calib3d/src/calibration.cpp index fee02f5b9d..6ef26b2eda 100644 --- a/modules/calib3d/src/calibration.cpp +++ b/modules/calib3d/src/calibration.cpp @@ -3189,9 +3189,10 @@ static Mat prepareCameraMatrix(Mat& cameraMatrix0, int rtype) return cameraMatrix; } -static Mat prepareDistCoeffs(Mat& distCoeffs0, int rtype) +static Mat prepareDistCoeffs(Mat& distCoeffs0, int rtype, int outputSize = 14) { - Mat distCoeffs = Mat::zeros(distCoeffs0.cols == 1 ? Size(1, 14) : Size(14, 1), rtype); + CV_Assert((int)distCoeffs0.total() <= outputSize); + Mat distCoeffs = Mat::zeros(distCoeffs0.cols == 1 ? Size(1, outputSize) : Size(outputSize, 1), rtype); if( distCoeffs0.size() == Size(1, 4) || distCoeffs0.size() == Size(1, 5) || distCoeffs0.size() == Size(1, 8) || @@ -3398,7 +3399,8 @@ double cv::calibrateCamera(InputArrayOfArrays _objectPoints, Mat cameraMatrix = _cameraMatrix.getMat(); cameraMatrix = prepareCameraMatrix(cameraMatrix, rtype); Mat distCoeffs = _distCoeffs.getMat(); - distCoeffs = prepareDistCoeffs(distCoeffs, rtype); + distCoeffs = (flags & CALIB_THIN_PRISM_MODEL) && !(flags & CALIB_TILTED_MODEL) ? prepareDistCoeffs(distCoeffs, rtype, 12) : + prepareDistCoeffs(distCoeffs, rtype); if( !(flags & CALIB_RATIONAL_MODEL) && (!(flags & CALIB_THIN_PRISM_MODEL)) && (!(flags & CALIB_TILTED_MODEL))) From 74b83cfce59318c90683fb5daa73fc8d4bea118e Mon Sep 17 00:00:00 2001 From: Oliver Schreer Date: Fri, 22 May 2015 10:39:01 +0200 Subject: [PATCH 080/128] Modified and improved the method for chessboard detection. It is now faster and detects chessboards under difficult lighting condition as well as when the chessboard has strong out of plane rotations --- .../include/opencv2/calib3d/calib3d.hpp | 9 + modules/calib3d/src/calibinit.cpp | 652 +++++++++++++----- modules/calib3d/src/checkchessboard.cpp | 98 ++- 3 files changed, 585 insertions(+), 174 deletions(-) diff --git a/modules/calib3d/include/opencv2/calib3d/calib3d.hpp b/modules/calib3d/include/opencv2/calib3d/calib3d.hpp index b3da45edd5..517372edc5 100644 --- a/modules/calib3d/include/opencv2/calib3d/calib3d.hpp +++ b/modules/calib3d/include/opencv2/calib3d/calib3d.hpp @@ -46,3 +46,12 @@ #endif #include "opencv2/calib3d.hpp" +// Performs a fast check if a chessboard is in the input image. This is a workaround to +// a problem of cvFindChessboardCorners being slow on images with no chessboard. +// This method works using a binary image as input +// - src: input binary image +// - size: chessboard size +// Returns 1 if a chessboard can be in this image and findChessboardCorners should be called, +// 0 if there is no chessboard, -1 in case of error +CVAPI(int) cvCheckChessboardBinary(IplImage* src, CvSize size); + diff --git a/modules/calib3d/src/calibinit.cpp b/modules/calib3d/src/calibinit.cpp index d6b4923d11..0f4676da97 100644 --- a/modules/calib3d/src/calibinit.cpp +++ b/modules/calib3d/src/calibinit.cpp @@ -59,11 +59,22 @@ \************************************************************************************/ +/************************************************************************************\ + This version adds a new and improved variant of chessboard corner detection + that works better in poor lighting condition. It is based on work from + Oliver Schreer and Stefano Masneri. This method works faster than the previous + one and reverts back to the older method in case no chessboard detection is + possible. Overall performance improves also because now the method avoids + performing the same computation multiple times when not necessary. + +\************************************************************************************/ + #include "precomp.hpp" #include "opencv2/imgproc/imgproc_c.h" #include "opencv2/calib3d/calib3d_c.h" #include "circlesgrid.hpp" #include +#include //#define ENABLE_TRIM_COL_ROW @@ -191,6 +202,202 @@ static void icvRemoveQuadFromGroup(CvCBQuad **quads, int count, CvCBQuad *q0); static int icvCheckBoardMonotony( CvPoint2D32f* corners, CvSize pattern_size ); +/***************************************************************************************************/ +//COMPUTE INTENSITY HISTOGRAM OF INPUT IMAGE +static int icvGetIntensityHistogram( unsigned char* pucImage, int iSizeCols, int iSizeRows, std::vector& piHist ); +//SMOOTH HISTOGRAM USING WINDOW OF SIZE 2*iWidth+1 +static int icvSmoothHistogram( const std::vector& piHist, std::vector& piHistSmooth, int iWidth ); +//COMPUTE FAST HISTOGRAM GRADIENT +static int icvGradientOfHistogram( const std::vector& piHist, std::vector& piHistGrad ); +//PERFORM SMART IMAGE THRESHOLDING BASED ON ANALYSIS OF INTENSTY HISTOGRAM +static bool icvBinarizationHistogramBased( unsigned char* pucImg, int iCols, int iRows ); +/***************************************************************************************************/ +int icvGetIntensityHistogram( unsigned char* pucImage, int iSizeCols, int iSizeRows, std::vector& piHist ) +{ + int iVal; + + // sum up all pixel in row direction and divide by number of columns + for ( int j=0; j& piHist, std::vector& piHistSmooth, int iWidth ) +{ + int iIdx; + for ( int i=0; i<256; i++) + { + int iSmooth = 0; + for ( int ii=-iWidth; ii<=iWidth; ii++) + { + iIdx = i+ii; + if (iIdx > 0 && iIdx < 256) + { + iSmooth += piHist[iIdx]; + } + } + piHistSmooth[i] = iSmooth/(2*iWidth+1); + } + return 0; +} +/***************************************************************************************************/ +int icvGradientOfHistogram( const std::vector& piHist, std::vector& piHistGrad ) +{ + piHistGrad[0] = 0; + for ( int i=1; i<255; i++) + { + piHistGrad[i] = piHist[i-1] - piHist[i+1]; + if ( abs(piHistGrad[i]) < 100 ) + { + if ( piHistGrad[i-1] == 0) + piHistGrad[i] = -100; + else + piHistGrad[i] = piHistGrad[i-1]; + } + } + return 0; +} +/***************************************************************************************************/ +bool icvBinarizationHistogramBased( unsigned char* pucImg, int iCols, int iRows ) +{ + int iMaxPix = iCols*iRows; + int iMaxPix1 = iMaxPix/100; + const int iNumBins = 256; + std::vector piHistIntensity(iNumBins, 0); + std::vector piHistSmooth(iNumBins, 0); + std::vector piHistGrad(iNumBins, 0); + std::vector piAccumSum(iNumBins, 0); + std::vector piMaxPos(20, 0); + int iThresh = 0; + int iIdx; + int iWidth = 1; + + icvGetIntensityHistogram( pucImg, iCols, iRows, piHistIntensity ); + + // get accumulated sum starting from bright + piAccumSum[iNumBins-1] = piHistIntensity[iNumBins-1]; + for ( int i=iNumBins-2; i>=0; i-- ) + { + piAccumSum[i] = piHistIntensity[i] + piAccumSum[i+1]; + } + + // first smooth the distribution + icvSmoothHistogram( piHistIntensity, piHistSmooth, iWidth ); + + // compute gradient + icvGradientOfHistogram( piHistSmooth, piHistGrad ); + + // check for zeros + int iCntMaxima = 0; + for ( int i=iNumBins-2; (i>2) && (iCntMaxima<20); i--) + { + if ( (piHistGrad[i-1] < 0) && (piHistGrad[i] > 0) ) + { + piMaxPos[iCntMaxima] = i; + iCntMaxima++; + } + } + + iIdx = 0; + int iSumAroundMax = 0; + for ( int i=0; i= 3 + { + // CHECKING THRESHOLD FOR WHITE + int iIdxAccSum = 0, iAccum = 0; + for (int i=iNumBins-1; i>0; i--) + { + iAccum += piHistIntensity[i]; + // iMaxPix/18 is about 5,5%, minimum required number of pixels required for white part of chessboard + if ( iAccum > (iMaxPix/18) ) + { + iIdxAccSum = i; + break; + } + } + + int iIdxBGMax = 0; + int iBrightMax = piMaxPos[0]; + // printf("iBrightMax = %d\n", iBrightMax); + for ( int n=0; n= 250 && iIdxBGMax < iCntMaxima ) + { + iIdxBGMax++; + iMaxVal = piHistIntensity[piMaxPos[iIdxBGMax]]; + } + + for ( int n=iIdxBGMax + 1; n= iMaxVal ) + { + iMaxVal = piHistIntensity[piMaxPos[n]]; + iIdxBGMax = n; + } + } + + //SETTING THRESHOLD FOR BINARIZATION + int iDist2 = (iBrightMax - piMaxPos[iIdxBGMax])/2; + iThresh = iBrightMax - iDist2; + PRINTF("THRESHOLD SELECTED = %d, BRIGHTMAX = %d, DARKMAX = %d\n", iThresh, iBrightMax, piMaxPos[iIdxBGMax]); + } + + + if ( iThresh > 0 ) + { + for ( int jj=0; jj storage; CvMat stub, *img = (CvMat*)arr; + cImgSeg = cvCreateImage(cvGetSize(img), IPL_DEPTH_8U, 1 ); + memcpy( cImgSeg->imageData, cvPtr1D( img, 0), img->rows*img->cols ); + + CvMat stub2, *thresh_img_new; + thresh_img_new = cvGetMat( cImgSeg, &stub2, 0, 0 ); int expected_corners_num = (pattern_size.width/2+1)*(pattern_size.height/2+1); @@ -255,7 +468,6 @@ int cvFindChessboardCorners( const void* arr, CvSize pattern_size, if( out_corner_count ) *out_corner_count = 0; - IplImage _img; int quad_count = 0, group_idx = 0, dilations = 0; img = cvGetMat( img, &stub ); @@ -300,209 +512,303 @@ int cvFindChessboardCorners( const void* arr, CvSize pattern_size, if( flags & CV_CALIB_CB_FAST_CHECK) { - cvGetImage(img, &_img); - int check_chessboard_result = cvCheckChessboard(&_img, pattern_size); - if(check_chessboard_result <= 0) + //perform new method for checking chessboard using a binary image. + //image is binarised using a threshold dependent on the image histogram + icvBinarizationHistogramBased( (unsigned char*) cImgSeg->imageData, cImgSeg->width, cImgSeg->height ); + check_chessboard_result = cvCheckChessboardBinary(cImgSeg, pattern_size); + if(check_chessboard_result <= 0) //fall back to the old method { - return 0; + IplImage _img; + cvGetImage(img, &_img); + check_chessboard_result = cvCheckChessboard(&_img, pattern_size); + if(check_chessboard_result <= 0) + { + return 0; + } } } + // empiric threshold level + // thresholding performed here and not inside the cycle to save processing time + int thresh_level; + if ( !(flags & CV_CALIB_CB_ADAPTIVE_THRESH) ) + { + double mean = cvAvg( img ).val[0]; + thresh_level = cvRound( mean - 10 ); + thresh_level = MAX( thresh_level, 10 ); + cvThreshold( img, thresh_img, thresh_level, 255, CV_THRESH_BINARY ); + } // Try our standard "1" dilation, but if the pattern is not found, iterate the whole procedure with higher dilations. // This is necessary because some squares simply do not separate properly with a single dilation. However, // we want to use the minimum number of dilations possible since dilations cause the squares to become smaller, // making it difficult to detect smaller squares. - for( k = 0; k < 6; k++ ) + for( dilations = min_dilations; dilations <= max_dilations; dilations++ ) { + if (found) + break; // already found it + + cvFree(&quads); + cvFree(&corners); + + //USE BINARY IMAGE COMPUTED USING icvBinarizationHistogramBased METHOD + cvDilate( thresh_img_new, thresh_img_new, 0, 1 ); + + // So we can find rectangles that go to the edge, we draw a white line around the image edge. + // Otherwise FindContours will miss those clipped rectangle contours. + // The border color will be the image mean, because otherwise we risk screwing up filters like cvSmooth()... + cvRectangle( thresh_img_new, cvPoint(0,0), cvPoint(thresh_img_new->cols-1, thresh_img_new->rows-1), CV_RGB(255,255,255), 3, 8); + quad_count = icvGenerateQuads( &quads, &corners, storage, thresh_img_new, flags, &max_quad_buf_size ); + PRINTF("Quad count: %d/%d\n", quad_count, expected_corners_num); + + if( quad_count <= 0 ) + { + continue; + } + + // Find quad's neighbors + icvFindQuadNeighbors( quads, quad_count ); + + // allocate extra for adding in icvOrderFoundQuads + cvFree(&quad_group); + cvFree(&corner_group); + quad_group = (CvCBQuad**)cvAlloc( sizeof(quad_group[0]) * max_quad_buf_size); + corner_group = (CvCBCorner**)cvAlloc( sizeof(corner_group[0]) * max_quad_buf_size * 4 ); + + for( group_idx = 0; ; group_idx++ ) + { + int count = 0; + count = icvFindConnectedQuads( quads, quad_count, quad_group, group_idx, storage ); + + int icount = count; + if( count == 0 ) + break; + + // order the quad corners globally + // maybe delete or add some + PRINTF("Starting ordering of inner quads\n"); + count = icvOrderFoundConnectedQuads(count, quad_group, &quad_count, &quads, &corners, + pattern_size, storage ); + PRINTF("Orig count: %d After ordering: %d\n", icount, count); + + if (count == 0) + continue; // haven't found inner quads + + // If count is more than it should be, this will remove those quads + // which cause maximum deviation from a nice square pattern. + count = icvCleanFoundConnectedQuads( count, quad_group, pattern_size ); + PRINTF("Connected group: %d orig count: %d cleaned: %d\n", group_idx, icount, count); + + count = icvCheckQuadGroup( quad_group, count, corner_group, pattern_size ); + PRINTF("Connected group: %d count: %d cleaned: %d\n", group_idx, icount, count); + + int n = count > 0 ? pattern_size.width * pattern_size.height : -count; + n = MIN( n, pattern_size.width * pattern_size.height ); + float sum_dist = 0; + int total = 0; + + for(int i = 0; i < n; i++ ) + { + int ni = 0; + float avgi = corner_group[i]->meanDist(&ni); + sum_dist += avgi*ni; + total += ni; + } + prev_sqr_size = cvRound(sum_dist/MAX(total, 1)); + + if( count > 0 || (out_corner_count && -count > *out_corner_count) ) + { + // copy corners to output array + for(int i = 0; i < n; i++ ) + out_corners[i] = corner_group[i]->pt; + + if( out_corner_count ) + *out_corner_count = n; + + if( count == pattern_size.width*pattern_size.height && + icvCheckBoardMonotony( out_corners, pattern_size )) + { + found = 1; + break; + } + } + } + }//dilations + + // revert to old, slower, method if detection failed + if (!found) + { + for( k = 0; k < 6; k++ ) + { int max_quad_buf_size = 0; for( dilations = min_dilations; dilations <= max_dilations; dilations++ ) { - if (found) - break; // already found it + if (found) + break; // already found it - cvFree(&quads); - cvFree(&corners); + cvFree(&quads); + cvFree(&corners); - /*if( k == 1 ) - { - //Pattern was not found using binarization - // Run multi-level quads extraction - // In case one-level binarization did not give enough number of quads - CV_CALL( quad_count = icvGenerateQuadsEx( &quads, &corners, storage, img, thresh_img, dilations, flags )); - PRINTF("EX quad count: %d/%d\n", quad_count, expected_corners_num); - } - else*/ - { - // convert the input grayscale image to binary (black-n-white) - if( flags & CV_CALIB_CB_ADAPTIVE_THRESH ) - { - int block_size = cvRound(prev_sqr_size == 0 ? - MIN(img->cols,img->rows)*(k%2 == 0 ? 0.2 : 0.1): prev_sqr_size*2)|1; + // convert the input grayscale image to binary (black-n-white) + if( flags & CV_CALIB_CB_ADAPTIVE_THRESH ) + { + int block_size = cvRound(prev_sqr_size == 0 ? + MIN(img->cols,img->rows)*(k%2 == 0 ? 0.2 : 0.1): prev_sqr_size*2)|1; - // convert to binary - cvAdaptiveThreshold( img, thresh_img, 255, - CV_ADAPTIVE_THRESH_MEAN_C, CV_THRESH_BINARY, block_size, (k/2)*5 ); - if (dilations > 0) - cvDilate( thresh_img, thresh_img, 0, dilations-1 ); - } - else - { - // Make dilation before the thresholding. - // It splits chessboard corners - //cvDilate( img, thresh_img, 0, 1 ); - - // empiric threshold level - double mean = cvAvg( img ).val[0]; - int thresh_level = cvRound( mean - 10 ); - thresh_level = MAX( thresh_level, 10 ); - - cvThreshold( img, thresh_img, thresh_level, 255, CV_THRESH_BINARY ); - cvDilate( thresh_img, thresh_img, 0, dilations ); - } + // convert to binary + cvAdaptiveThreshold( img, thresh_img, 255, + CV_ADAPTIVE_THRESH_MEAN_C, CV_THRESH_BINARY, block_size, (k/2)*5 ); + if (dilations > 0) + cvDilate( thresh_img, thresh_img, 0, dilations-1 ); + } + //if flag CV_CALIB_CB_ADAPTIVE_THRESH is not set it doesn't make sense + //to iterate over k + else + { + k = 6; + cvDilate( thresh_img, thresh_img, 0, 1 ); + } #ifdef DEBUG_CHESSBOARD - cvCvtColor(thresh_img,dbg_img,CV_GRAY2BGR); + cvCvtColor(thresh_img,dbg_img,CV_GRAY2BGR); #endif - // So we can find rectangles that go to the edge, we draw a white line around the image edge. - // Otherwise FindContours will miss those clipped rectangle contours. - // The border color will be the image mean, because otherwise we risk screwing up filters like cvSmooth()... - cvRectangle( thresh_img, cvPoint(0,0), cvPoint(thresh_img->cols-1, - thresh_img->rows-1), CV_RGB(255,255,255), 3, 8); + // So we can find rectangles that go to the edge, we draw a white line around the image edge. + // Otherwise FindContours will miss those clipped rectangle contours. + // The border color will be the image mean, because otherwise we risk screwing up filters like cvSmooth()... + cvRectangle( thresh_img, cvPoint(0,0), cvPoint(thresh_img->cols-1, + thresh_img->rows-1), CV_RGB(255,255,255), 3, 8); - quad_count = icvGenerateQuads( &quads, &corners, storage, thresh_img, flags, &max_quad_buf_size); + quad_count = icvGenerateQuads( &quads, &corners, storage, thresh_img, flags, &max_quad_buf_size); + PRINTF("Quad count: %d/%d\n", quad_count, expected_corners_num); - PRINTF("Quad count: %d/%d\n", quad_count, expected_corners_num); - } +#ifdef DEBUG_CHESSBOARD + cvCopy(dbg_img, dbg1_img); + cvNamedWindow("all_quads", 1); + // copy corners to temp array + for(int i = 0; i < quad_count; i++ ) + { + for (int z=0; z<4; z++) + { + CvPoint2D32f pt1, pt2; + CvScalar color = CV_RGB(30,255,30); + pt1 = quads[i].corners[z]->pt; + pt2 = quads[i].corners[(z+1)%4]->pt; + pt2.x = (pt1.x + pt2.x)/2; + pt2.y = (pt1.y + pt2.y)/2; + if (z>0) + color = CV_RGB(200,200,0); + cvLine( dbg1_img, cvPointFrom32f(pt1), cvPointFrom32f(pt2), color, 3, 8); + } + } + + + cvShowImage("all_quads", (IplImage*)dbg1_img); + cvWaitKey(); +#endif + + if( quad_count <= 0 ) + { + continue; + } + + // Find quad's neighbors + icvFindQuadNeighbors( quads, quad_count ); + + // allocate extra for adding in icvOrderFoundQuads + cvFree(&quad_group); + cvFree(&corner_group); + quad_group = (CvCBQuad**)cvAlloc( sizeof(quad_group[0]) * max_quad_buf_size); + corner_group = (CvCBCorner**)cvAlloc( sizeof(corner_group[0]) * max_quad_buf_size * 4 ); + + for( group_idx = 0; ; group_idx++ ) + { + int count = 0; + count = icvFindConnectedQuads( quads, quad_count, quad_group, group_idx, storage ); + + int icount = count; + if( count == 0 ) + break; + + // order the quad corners globally + // maybe delete or add some + PRINTF("Starting ordering of inner quads\n"); + count = icvOrderFoundConnectedQuads(count, quad_group, &quad_count, &quads, &corners, pattern_size, max_quad_buf_size, storage ); + + PRINTF("Orig count: %d After ordering: %d\n", icount, count); #ifdef DEBUG_CHESSBOARD - cvCopy(dbg_img, dbg1_img); - cvNamedWindow("all_quads", 1); - // copy corners to temp array - for(int i = 0; i < quad_count; i++ ) - { - for (int k=0; k<4; k++) - { - CvPoint2D32f pt1, pt2; - CvScalar color = CV_RGB(30,255,30); - pt1 = quads[i].corners[k]->pt; - pt2 = quads[i].corners[(k+1)%4]->pt; - pt2.x = (pt1.x + pt2.x)/2; - pt2.y = (pt1.y + pt2.y)/2; - if (k>0) - color = CV_RGB(200,200,0); - cvLine( dbg1_img, cvPointFrom32f(pt1), cvPointFrom32f(pt2), color, 3, 8); - } - } - - - cvShowImage("all_quads", (IplImage*)dbg1_img); - cvWaitKey(); + cvCopy(dbg_img,dbg2_img); + cvNamedWindow("connected_group", 1); + // copy corners to temp array + for(int i = 0; i < quad_count; i++ ) + { + if (quads[i].group_idx == group_idx) + for (int z=0; z<4; z++) + { + CvPoint2D32f pt1, pt2; + CvScalar color = CV_RGB(30,255,30); + if (quads[i].ordered) + color = CV_RGB(255,30,30); + pt1 = quads[i].corners[z]->pt; + pt2 = quads[i].corners[(z+1)%4]->pt; + pt2.x = (pt1.x + pt2.x)/2; + pt2.y = (pt1.y + pt2.y)/2; + if (z>0) + color = CV_RGB(200,200,0); + cvLine( dbg2_img, cvPointFrom32f(pt1), cvPointFrom32f(pt2), color, 3, 8); + } + } + cvShowImage("connected_group", (IplImage*)dbg2_img); + cvWaitKey(); #endif - if( quad_count <= 0 ) - continue; + if (count == 0) + continue; // haven't found inner quads - // Find quad's neighbors - icvFindQuadNeighbors( quads, quad_count ); - // allocate extra for adding in icvOrderFoundQuads - cvFree(&quad_group); - cvFree(&corner_group); - quad_group = (CvCBQuad**)cvAlloc( sizeof(quad_group[0]) * max_quad_buf_size); - corner_group = (CvCBCorner**)cvAlloc( sizeof(corner_group[0]) * max_quad_buf_size * 4 ); + // If count is more than it should be, this will remove those quads + // which cause maximum deviation from a nice square pattern. + count = icvCleanFoundConnectedQuads( count, quad_group, pattern_size ); + PRINTF("Connected group: %d orig count: %d cleaned: %d\n", group_idx, icount, count); - for( group_idx = 0; ; group_idx++ ) + count = icvCheckQuadGroup( quad_group, count, corner_group, pattern_size ); + PRINTF("Connected group: %d count: %d cleaned: %d\n", group_idx, icount, count); + + int n = count > 0 ? pattern_size.width * pattern_size.height : -count; + n = MIN( n, pattern_size.width * pattern_size.height ); + float sum_dist = 0; + int total = 0; + + for(int i = 0; i < n; i++ ) { - int count = 0; - count = icvFindConnectedQuads( quads, quad_count, quad_group, group_idx, storage ); - - int icount = count; - if( count == 0 ) - break; - - // order the quad corners globally - // maybe delete or add some - PRINTF("Starting ordering of inner quads\n"); - count = icvOrderFoundConnectedQuads(count, quad_group, &quad_count, &quads, &corners, - pattern_size, max_quad_buf_size, storage ); - PRINTF("Orig count: %d After ordering: %d\n", icount, count); - - -#ifdef DEBUG_CHESSBOARD - cvCopy(dbg_img,dbg2_img); - cvNamedWindow("connected_group", 1); - // copy corners to temp array - for(int i = 0; i < quad_count; i++ ) - { - if (quads[i].group_idx == group_idx) - for (int k=0; k<4; k++) - { - CvPoint2D32f pt1, pt2; - CvScalar color = CV_RGB(30,255,30); - if (quads[i].ordered) - color = CV_RGB(255,30,30); - pt1 = quads[i].corners[k]->pt; - pt2 = quads[i].corners[(k+1)%4]->pt; - pt2.x = (pt1.x + pt2.x)/2; - pt2.y = (pt1.y + pt2.y)/2; - if (k>0) - color = CV_RGB(200,200,0); - cvLine( dbg2_img, cvPointFrom32f(pt1), cvPointFrom32f(pt2), color, 3, 8); - } - } - cvShowImage("connected_group", (IplImage*)dbg2_img); - cvWaitKey(); -#endif - - if (count == 0) - continue; // haven't found inner quads - - - // If count is more than it should be, this will remove those quads - // which cause maximum deviation from a nice square pattern. - count = icvCleanFoundConnectedQuads( count, quad_group, pattern_size ); - PRINTF("Connected group: %d orig count: %d cleaned: %d\n", group_idx, icount, count); - - count = icvCheckQuadGroup( quad_group, count, corner_group, pattern_size ); - PRINTF("Connected group: %d count: %d cleaned: %d\n", group_idx, icount, count); - - { - int n = count > 0 ? pattern_size.width * pattern_size.height : -count; - n = MIN( n, pattern_size.width * pattern_size.height ); - float sum_dist = 0; - int total = 0; - - for(int i = 0; i < n; i++ ) - { - int ni = 0; - float avgi = corner_group[i]->meanDist(&ni); - sum_dist += avgi*ni; - total += ni; - } - prev_sqr_size = cvRound(sum_dist/MAX(total, 1)); - - if( count > 0 || (out_corner_count && -count > *out_corner_count) ) - { - // copy corners to output array - for(int i = 0; i < n; i++ ) - out_corners[i] = corner_group[i]->pt; - - if( out_corner_count ) - *out_corner_count = n; - - if( count == pattern_size.width*pattern_size.height && - icvCheckBoardMonotony( out_corners, pattern_size )) - { - found = 1; - break; - } - } - } + int ni = 0; + float avgi = corner_group[i]->meanDist(&ni); + sum_dist += avgi*ni; + total += ni; } + prev_sqr_size = cvRound(sum_dist/MAX(total, 1)); + + if( count > 0 || (out_corner_count && -count > *out_corner_count) ) + { + // copy corners to output array + for(int i = 0; i < n; i++ ) + out_corners[i] = corner_group[i]->pt; + + if( out_corner_count ) + *out_corner_count = n; + + if( count == pattern_size.width*pattern_size.height && icvCheckBoardMonotony( out_corners, pattern_size )) + { + found = 1; + break; + } + } + } }//dilations - }// + }// for k = 0 -> 6 + } + if( found ) found = icvCheckBoardMonotony( out_corners, pattern_size ); @@ -559,6 +865,7 @@ int cvFindChessboardCorners( const void* arr, CvSize pattern_size, cvFree(&corners); cvFree(&quad_group); cvFree(&corner_group); + cvFree(&cImgSeg); throw; } @@ -566,6 +873,7 @@ int cvFindChessboardCorners( const void* arr, CvSize pattern_size, cvFree(&corners); cvFree(&quad_group); cvFree(&corner_group); + cvFree(&cImgSeg); return found; } diff --git a/modules/calib3d/src/checkchessboard.cpp b/modules/calib3d/src/checkchessboard.cpp index 715fe73ef8..5ecd37faab 100644 --- a/modules/calib3d/src/checkchessboard.cpp +++ b/modules/calib3d/src/checkchessboard.cpp @@ -59,8 +59,8 @@ static void icvGetQuadrangleHypotheses(CvSeq* contours, std::vector >& quads, int class_id) { - const float min_aspect_ratio = 0.3f; - const float max_aspect_ratio = 3.0f; + const float min_aspect_ratio = 0.2f; + const float max_aspect_ratio = 5.0f; const float min_box_size = 10.0f; for(CvSeq* seq = contours; seq != NULL; seq = seq->h_next) @@ -205,3 +205,97 @@ int cvCheckChessboard(IplImage* src, CvSize size) return result; } + +// does a fast check if a chessboard is in the input image. This is a workaround to +// a problem of cvFindChessboardCorners being slow on images with no chessboard +// - src: input binary image +// - size: chessboard size +// Returns 1 if a chessboard can be in this image and findChessboardCorners should be called, +// 0 if there is no chessboard, -1 in case of error +int cvCheckChessboardBinary(IplImage* src, CvSize size) +{ + if(src->nChannels > 1) + { + cvError(CV_BadNumChannels, "cvCheckChessboard", "supports single-channel images only", + __FILE__, __LINE__); + } + + if(src->depth != 8) + { + cvError(CV_BadDepth, "cvCheckChessboard", "supports depth=8 images only", + __FILE__, __LINE__); + } + + CvMemStorage* storage = cvCreateMemStorage(); + + IplImage* white = cvCloneImage(src); + IplImage* black = cvCloneImage(src); + IplImage* thresh = cvCreateImage(cvGetSize(src), IPL_DEPTH_8U, 1); + + int result = 0; + + for ( int erosion_count = 0; erosion_count <= 3; erosion_count++ ) + { + if ( 1 == result ) + break; + + if ( 0 != erosion_count ) // first iteration keeps original images + { + cvErode(white, white, NULL, 1); + cvDilate(black, black, NULL, 1); + } + + cvThreshold(white, thresh, 128, 255, CV_THRESH_BINARY); + + CvSeq* first = 0; + std::vector > quads; + cvFindContours(thresh, storage, &first, sizeof(CvContour), CV_RETR_CCOMP); + icvGetQuadrangleHypotheses(first, quads, 1); + + cvThreshold(black, thresh, 128, 255, CV_THRESH_BINARY_INV); + cvFindContours(thresh, storage, &first, sizeof(CvContour), CV_RETR_CCOMP); + icvGetQuadrangleHypotheses(first, quads, 0); + + const size_t min_quads_count = size.width*size.height/2; + std::sort(quads.begin(), quads.end(), less_pred); + + // now check if there are many hypotheses with similar sizes + // do this by floodfill-style algorithm + const float size_rel_dev = 0.4f; + + for(size_t i = 0; i < quads.size(); i++) + { + size_t j = i + 1; + for(; j < quads.size(); j++) + { + if(quads[j].first/quads[i].first > 1.0f + size_rel_dev) + { + break; + } + } + + if(j + 1 > min_quads_count + i) + { + // check the number of black and white squares + std::vector counts; + countClasses(quads, i, j, counts); + const int black_count = cvRound(ceil(size.width/2.0)*ceil(size.height/2.0)); + const int white_count = cvRound(floor(size.width/2.0)*floor(size.height/2.0)); + if(counts[0] < black_count*0.75 || + counts[1] < white_count*0.75) + { + continue; + } + result = 1; + break; + } + } + } + + cvReleaseImage(&thresh); + cvReleaseImage(&white); + cvReleaseImage(&black); + cvReleaseMemStorage(&storage); + + return result; +} \ No newline at end of file From 91d8405e52c186af90a0f8df9c0a147925552bc2 Mon Sep 17 00:00:00 2001 From: Ilya Lavrenov Date: Tue, 12 Jul 2016 16:01:12 +0300 Subject: [PATCH 081/128] stop search of markers in Exif reader to prevent infinite loop --- modules/imgcodecs/src/jpeg_exif.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/modules/imgcodecs/src/jpeg_exif.cpp b/modules/imgcodecs/src/jpeg_exif.cpp index adb87e5b77..0704c2f49a 100644 --- a/modules/imgcodecs/src/jpeg_exif.cpp +++ b/modules/imgcodecs/src/jpeg_exif.cpp @@ -140,8 +140,8 @@ std::map ExifReader::getExif() return m_exif; //Until this moment the map is empty } - bool exifFound = false; - while( ( !feof( f ) ) && !exifFound ) + bool exifFound = false, stopSearch = false; + while( ( !feof( f ) ) && !exifFound && !stopSearch ) { count = fread( appMarker, sizeof(unsigned char), markerSize, f ); if( count < markerSize ) @@ -180,6 +180,7 @@ std::map ExifReader::getExif() break; default: //No other markers are expected according to standard. May be a signal of error + stopSearch = true; break; } } From dce310e03cabcc4e7ffdb490eb70e8be44f3d02d Mon Sep 17 00:00:00 2001 From: Sergei Nosov Date: Tue, 12 Jul 2016 18:03:28 +0300 Subject: [PATCH 082/128] provide better error messages --- modules/imgproc/include/opencv2/imgproc.hpp | 6 ++++-- modules/imgproc/src/emd.cpp | 10 ++++++---- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/modules/imgproc/include/opencv2/imgproc.hpp b/modules/imgproc/include/opencv2/imgproc.hpp index 346c8e44f1..92bce5f780 100644 --- a/modules/imgproc/include/opencv2/imgproc.hpp +++ b/modules/imgproc/include/opencv2/imgproc.hpp @@ -3157,10 +3157,12 @@ same object. @param signature1 First signature, a \f$\texttt{size1}\times \texttt{dims}+1\f$ floating-point matrix. Each row stores the point weight followed by the point coordinates. The matrix is allowed to have -a single column (weights only) if the user-defined cost matrix is used. +a single column (weights only) if the user-defined cost matrix is used. The weights must be +non-negative and have at least one non-zero value. @param signature2 Second signature of the same format as signature1 , though the number of rows may be different. The total weights may be different. In this case an extra "dummy" point is added -to either signature1 or signature2 . +to either signature1 or signature2. The weights must be non-negative and have at least one non-zero +value. @param distType Used metric. See cv::DistanceTypes. @param cost User-defined \f$\texttt{size1}\times \texttt{size2}\f$ cost matrix. Also, if a cost matrix is used, lower boundary lowerBound cannot be calculated because it needs a metric function. diff --git a/modules/imgproc/src/emd.cpp b/modules/imgproc/src/emd.cpp index 22468da6d1..4dddbc6f14 100644 --- a/modules/imgproc/src/emd.cpp +++ b/modules/imgproc/src/emd.cpp @@ -387,7 +387,7 @@ static int icvInitEMD( const float* signature1, int size1, } else if( weight < 0 ) - CV_Error(CV_StsOutOfRange, ""); + CV_Error(CV_StsBadArg, "signature1 must not contain negative weights"); } for( i = 0; i < size2; i++ ) @@ -401,11 +401,13 @@ static int icvInitEMD( const float* signature1, int size1, state->idx2[dsize++] = i; } else if( weight < 0 ) - CV_Error(CV_StsOutOfRange, ""); + CV_Error(CV_StsBadArg, "signature2 must not contain negative weights"); } - if( ssize == 0 || dsize == 0 ) - CV_Error(CV_StsOutOfRange, ""); + if( ssize == 0 ) + CV_Error(CV_StsBadArg, "signature1 must contain at least one non-zero value"); + if( dsize == 0 ) + CV_Error(CV_StsBadArg, "signature2 must contain at least one non-zero value"); /* if supply different than the demand, add a zero-cost dummy cluster */ diff = s_sum - d_sum; From 20b9ff4ff6efa6941a8cce29e9b92acb65485304 Mon Sep 17 00:00:00 2001 From: berak Date: Tue, 12 Jul 2016 07:38:52 +0200 Subject: [PATCH 083/128] opencv_visualization: check cmdline args --- apps/visualisation/opencv_visualisation.cpp | 42 +++++++++++++-------- 1 file changed, 26 insertions(+), 16 deletions(-) diff --git a/apps/visualisation/opencv_visualisation.cpp b/apps/visualisation/opencv_visualisation.cpp index 2c685f521a..75703bd528 100644 --- a/apps/visualisation/opencv_visualisation.cpp +++ b/apps/visualisation/opencv_visualisation.cpp @@ -47,7 +47,7 @@ Software for visualising cascade classifier models trained by OpenCV and to get understanding of the used features. USAGE: -./visualise_models -model -image -data +./opencv_visualisation --model= --image= --data=