From 3681dcef1a6d395d5640b67fff6a846b69b5108e Mon Sep 17 00:00:00 2001 From: Vitaly Tuzov Date: Mon, 3 Jul 2017 13:49:16 +0300 Subject: [PATCH 1/2] AVX optimized implementation of resize and warp functions migrated to separate file --- modules/imgproc/src/imgwarp.avx2.cpp | 289 +++++++++++++++++++ modules/imgproc/src/imgwarp.cpp | 375 +------------------------ modules/imgproc/src/imgwarp.hpp | 73 +++++ modules/imgproc/src/imgwarp.sse4_1.cpp | 192 +++++++++++++ 4 files changed, 569 insertions(+), 360 deletions(-) create mode 100644 modules/imgproc/src/imgwarp.avx2.cpp create mode 100644 modules/imgproc/src/imgwarp.hpp create mode 100644 modules/imgproc/src/imgwarp.sse4_1.cpp diff --git a/modules/imgproc/src/imgwarp.avx2.cpp b/modules/imgproc/src/imgwarp.avx2.cpp new file mode 100644 index 0000000000..321957f39e --- /dev/null +++ b/modules/imgproc/src/imgwarp.avx2.cpp @@ -0,0 +1,289 @@ +/*M/////////////////////////////////////////////////////////////////////////////////////// +// +// IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. +// +// By downloading, copying, installing or using the software you agree to this license. +// If you do not agree to this license, do not download, install, +// copy or use the software. +// +// +// License Agreement +// For Open Source Computer Vision Library +// +// Copyright (C) 2000-2008, Intel Corporation, all rights reserved. +// Copyright (C) 2009, Willow Garage Inc., all rights reserved. +// Copyright (C) 2014-2015, Itseez Inc., all rights reserved. +// Third party copyrights are property of their respective owners. +// +// Redistribution and use in source and binary forms, with or without modification, +// are permitted provided that the following conditions are met: +// +// * Redistribution's of source code must retain the above copyright notice, +// this list of conditions and the following disclaimer. +// +// * Redistribution's in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// +// * The name of the copyright holders may not be used to endorse or promote products +// derived from this software without specific prior written permission. +// +// This software is provided by the copyright holders and contributors "as is" and +// any express or implied warranties, including, but not limited to, the implied +// warranties of merchantability and fitness for a particular purpose are disclaimed. +// In no event shall the Intel Corporation or contributors be liable for any direct, +// indirect, incidental, special, exemplary, or consequential damages +// (including, but not limited to, procurement of substitute goods or services; +// loss of use, data, or profits; or business interruption) however caused +// and on any theory of liability, whether in contract, strict liability, +// or tort (including negligence or otherwise) arising in any way out of +// the use of this software, even if advised of the possibility of such damage. +// +//M*/ + +/* //////////////////////////////////////////////////////////////////// +// +// Geometrical transforms on images and matrices: rotation, zoom etc. +// +// */ + +#include "precomp.hpp" +#include "imgwarp.hpp" + +namespace cv +{ +namespace opt_AVX2 +{ + +class resizeNNInvokerAVX4 : + public ParallelLoopBody +{ +public: + resizeNNInvokerAVX4(const Mat& _src, Mat &_dst, int *_x_ofs, int _pix_size4, double _ify) : + ParallelLoopBody(), src(_src), dst(_dst), x_ofs(_x_ofs), pix_size4(_pix_size4), + ify(_ify) + { + } + +#if defined(__INTEL_COMPILER) +#pragma optimization_parameter target_arch=AVX +#endif + virtual void operator() (const Range& range) const + { + Size ssize = src.size(), dsize = dst.size(); + int y, x; + int width = dsize.width; + int avxWidth = width - (width & 0x7); + const __m256i CV_DECL_ALIGNED(64) mask = _mm256_set1_epi32(-1); + if(((int64)(dst.data + dst.step) & 0x1f) == 0) + { + for(y = range.start; y < range.end; y++) + { + uchar* D = dst.data + dst.step*y; + uchar* Dstart = D; + int sy = std::min(cvFloor(y*ify), ssize.height-1); + const uchar* S = src.data + sy*src.step; +#pragma unroll(4) + for(x = 0; x < avxWidth; x += 8) + { + const __m256i CV_DECL_ALIGNED(64) *addr = (__m256i*)(x_ofs + x); + __m256i CV_DECL_ALIGNED(64) indices = _mm256_lddqu_si256(addr); + __m256i CV_DECL_ALIGNED(64) pixels = _mm256_i32gather_epi32((const int*)S, indices, 1); + _mm256_maskstore_epi32((int*)D, mask, pixels); + D += 32; + } + for(; x < width; x++) + { + *(int*)(Dstart + x*4) = *(int*)(S + x_ofs[x]); + } + } + } + else + { + for(y = range.start; y < range.end; y++) + { + uchar* D = dst.data + dst.step*y; + uchar* Dstart = D; + int sy = std::min(cvFloor(y*ify), ssize.height-1); + const uchar* S = src.data + sy*src.step; +#pragma unroll(4) + for(x = 0; x < avxWidth; x += 8) + { + const __m256i CV_DECL_ALIGNED(64) *addr = (__m256i*)(x_ofs + x); + __m256i CV_DECL_ALIGNED(64) indices = _mm256_lddqu_si256(addr); + __m256i CV_DECL_ALIGNED(64) pixels = _mm256_i32gather_epi32((const int*)S, indices, 1); + _mm256_storeu_si256((__m256i*)D, pixels); + D += 32; + } + for(; x < width; x++) + { + *(int*)(Dstart + x*4) = *(int*)(S + x_ofs[x]); + } + } + } + } + +private: + const Mat src; + Mat dst; + int* x_ofs, pix_size4; + double ify; + + resizeNNInvokerAVX4(const resizeNNInvokerAVX4&); + resizeNNInvokerAVX4& operator=(const resizeNNInvokerAVX4&); +}; + +class resizeNNInvokerAVX2 : + public ParallelLoopBody +{ +public: + resizeNNInvokerAVX2(const Mat& _src, Mat &_dst, int *_x_ofs, int _pix_size4, double _ify) : + ParallelLoopBody(), src(_src), dst(_dst), x_ofs(_x_ofs), pix_size4(_pix_size4), + ify(_ify) + { + } + +#if defined(__INTEL_COMPILER) +#pragma optimization_parameter target_arch=AVX +#endif + virtual void operator() (const Range& range) const + { + Size ssize = src.size(), dsize = dst.size(); + int y, x; + int width = dsize.width; + //int avxWidth = (width - 1) - ((width - 1) & 0x7); + int avxWidth = width - (width & 0xf); + const __m256i CV_DECL_ALIGNED(64) mask = _mm256_set1_epi32(-1); + const __m256i CV_DECL_ALIGNED(64) shuffle_mask = _mm256_set_epi8(15,14,11,10,13,12,9,8,7,6,3,2,5,4,1,0, + 15,14,11,10,13,12,9,8,7,6,3,2,5,4,1,0); + const __m256i CV_DECL_ALIGNED(64) permute_mask = _mm256_set_epi32(7, 5, 3, 1, 6, 4, 2, 0); + const __m256i CV_DECL_ALIGNED(64) shift_shuffle_mask = _mm256_set_epi8(13,12,15,14,9,8,11,10,5,4,7,6,1,0,3,2, + 13,12,15,14,9,8,11,10,5,4,7,6,1,0,3,2); + if(((int64)(dst.data + dst.step) & 0x1f) == 0) + { + for(y = range.start; y < range.end; y++) + { + uchar* D = dst.data + dst.step*y; + uchar* Dstart = D; + int sy = std::min(cvFloor(y*ify), ssize.height-1); + const uchar* S = src.data + sy*src.step; + const uchar* S2 = S - 2; +#pragma unroll(4) + for(x = 0; x < avxWidth; x += 16) + { + const __m256i CV_DECL_ALIGNED(64) *addr = (__m256i*)(x_ofs + x); + __m256i CV_DECL_ALIGNED(64) indices = _mm256_lddqu_si256(addr); + __m256i CV_DECL_ALIGNED(64) pixels1 = _mm256_i32gather_epi32((const int*)S, indices, 1); + const __m256i CV_DECL_ALIGNED(64) *addr2 = (__m256i*)(x_ofs + x + 8); + __m256i CV_DECL_ALIGNED(64) indices2 = _mm256_lddqu_si256(addr2); + __m256i CV_DECL_ALIGNED(64) pixels2 = _mm256_i32gather_epi32((const int*)S2, indices2, 1); + __m256i CV_DECL_ALIGNED(64) unpacked = _mm256_blend_epi16(pixels1, pixels2, 0xaa); + + __m256i CV_DECL_ALIGNED(64) bytes_shuffled = _mm256_shuffle_epi8(unpacked, shuffle_mask); + __m256i CV_DECL_ALIGNED(64) ints_permuted = _mm256_permutevar8x32_epi32(bytes_shuffled, permute_mask); + _mm256_maskstore_epi32((int*)D, mask, ints_permuted); + D += 32; + } + for(; x < width; x++) + { + *(ushort*)(Dstart + x*2) = *(ushort*)(S + x_ofs[x]); + } + + } + } + else + { + for(y = range.start; y < range.end; y++) + { + uchar* D = dst.data + dst.step*y; + uchar* Dstart = D; + int sy = std::min(cvFloor(y*ify), ssize.height-1); + const uchar* S = src.data + sy*src.step; + const uchar* S2 = S - 2; +#pragma unroll(4) + for(x = 0; x < avxWidth; x += 16) + { + const __m256i CV_DECL_ALIGNED(64) *addr = (__m256i*)(x_ofs + x); + __m256i CV_DECL_ALIGNED(64) indices = _mm256_lddqu_si256(addr); + __m256i CV_DECL_ALIGNED(64) pixels1 = _mm256_i32gather_epi32((const int*)S, indices, 1); + const __m256i CV_DECL_ALIGNED(64) *addr2 = (__m256i*)(x_ofs + x + 8); + __m256i CV_DECL_ALIGNED(64) indices2 = _mm256_lddqu_si256(addr2); + __m256i CV_DECL_ALIGNED(64) pixels2 = _mm256_i32gather_epi32((const int*)S2, indices2, 1); + __m256i CV_DECL_ALIGNED(64) unpacked = _mm256_blend_epi16(pixels1, pixels2, 0xaa); + + __m256i CV_DECL_ALIGNED(64) bytes_shuffled = _mm256_shuffle_epi8(unpacked, shuffle_mask); + __m256i CV_DECL_ALIGNED(64) ints_permuted = _mm256_permutevar8x32_epi32(bytes_shuffled, permute_mask); + _mm256_storeu_si256((__m256i*)D, ints_permuted); + D += 32; + } + for(; x < width; x++) + { + *(ushort*)(Dstart + x*2) = *(ushort*)(S + x_ofs[x]); + } + } + } + } + +private: + const Mat src; + Mat dst; + int* x_ofs, pix_size4; + double ify; + + resizeNNInvokerAVX2(const resizeNNInvokerAVX2&); + resizeNNInvokerAVX2& operator=(const resizeNNInvokerAVX2&); +}; + +void resizeNN2_AVX2(const Range& range, const Mat& src, Mat &dst, int *x_ofs, int pix_size4, double ify) +{ + resizeNNInvokerAVX2 invoker(src, dst, x_ofs, pix_size4, ify); + parallel_for_(range, invoker, dst.total() / (double)(1 << 16)); +} + +void resizeNN4_AVX2(const Range& range, const Mat& src, Mat &dst, int *x_ofs, int pix_size4, double ify) +{ + resizeNNInvokerAVX4 invoker(src, dst, x_ofs, pix_size4, ify); + parallel_for_(range, invoker, dst.total() / (double)(1 << 16)); +} + +int warpAffineBlockline(int *adelta, int *bdelta, short* xy, short* alpha, int X0, int Y0, int bw) +{ + const int AB_BITS = MAX(10, (int)INTER_BITS); + int x1 = 0; + __m256i fxy_mask = _mm256_set1_epi32(INTER_TAB_SIZE - 1); + __m256i XX = _mm256_set1_epi32(X0), YY = _mm256_set1_epi32(Y0); + for (; x1 <= bw - 16; x1 += 16) + { + __m256i tx0, tx1, ty0, ty1; + tx0 = _mm256_add_epi32(_mm256_loadu_si256((const __m256i*)(adelta + x1)), XX); + ty0 = _mm256_add_epi32(_mm256_loadu_si256((const __m256i*)(bdelta + x1)), YY); + tx1 = _mm256_add_epi32(_mm256_loadu_si256((const __m256i*)(adelta + x1 + 8)), XX); + ty1 = _mm256_add_epi32(_mm256_loadu_si256((const __m256i*)(bdelta + x1 + 8)), YY); + + tx0 = _mm256_srai_epi32(tx0, AB_BITS - INTER_BITS); + ty0 = _mm256_srai_epi32(ty0, AB_BITS - INTER_BITS); + tx1 = _mm256_srai_epi32(tx1, AB_BITS - INTER_BITS); + ty1 = _mm256_srai_epi32(ty1, AB_BITS - INTER_BITS); + + __m256i fx_ = _mm256_packs_epi32(_mm256_and_si256(tx0, fxy_mask), + _mm256_and_si256(tx1, fxy_mask)); + __m256i fy_ = _mm256_packs_epi32(_mm256_and_si256(ty0, fxy_mask), + _mm256_and_si256(ty1, fxy_mask)); + tx0 = _mm256_packs_epi32(_mm256_srai_epi32(tx0, INTER_BITS), + _mm256_srai_epi32(tx1, INTER_BITS)); + ty0 = _mm256_packs_epi32(_mm256_srai_epi32(ty0, INTER_BITS), + _mm256_srai_epi32(ty1, INTER_BITS)); + fx_ = _mm256_adds_epi16(fx_, _mm256_slli_epi16(fy_, INTER_BITS)); + fx_ = _mm256_permute4x64_epi64(fx_, (3 << 6) + (1 << 4) + (2 << 2) + 0); + + _mm256_storeu_si256((__m256i*)(xy + x1 * 2), _mm256_unpacklo_epi16(tx0, ty0)); + _mm256_storeu_si256((__m256i*)(xy + x1 * 2 + 16), _mm256_unpackhi_epi16(tx0, ty0)); + _mm256_storeu_si256((__m256i*)(alpha + x1), fx_); + } + _mm256_zeroupper(); + return x1; +} + +} +} +/* End of file. */ diff --git a/modules/imgproc/src/imgwarp.cpp b/modules/imgproc/src/imgwarp.cpp index fd0387d6a4..4f040a91cc 100644 --- a/modules/imgproc/src/imgwarp.cpp +++ b/modules/imgproc/src/imgwarp.cpp @@ -52,6 +52,7 @@ #include "hal_replacement.hpp" #include "opencv2/core/openvx/ovx_defs.hpp" +#include "imgwarp.hpp" using namespace cv; @@ -417,308 +418,6 @@ private: resizeNNInvoker& operator=(const resizeNNInvoker&); }; -#if CV_AVX2 -class resizeNNInvokerAVX4 : - public ParallelLoopBody -{ -public: - resizeNNInvokerAVX4(const Mat& _src, Mat &_dst, int *_x_ofs, int _pix_size4, double _ify) : - ParallelLoopBody(), src(_src), dst(_dst), x_ofs(_x_ofs), pix_size4(_pix_size4), - ify(_ify) - { - } - -#if defined(__INTEL_COMPILER) -#pragma optimization_parameter target_arch=AVX -#endif - virtual void operator() (const Range& range) const - { - Size ssize = src.size(), dsize = dst.size(); - int y, x, pix_size = (int)src.elemSize(); - int width = dsize.width; - int avxWidth = width - (width & 0x7); - const __m256i CV_DECL_ALIGNED(64) mask = _mm256_set1_epi32(-1); - if(((int64)(dst.data + dst.step) & 0x1f) == 0) - { - for(y = range.start; y < range.end; y++) - { - uchar* D = dst.data + dst.step*y; - uchar* Dstart = D; - int sy = std::min(cvFloor(y*ify), ssize.height-1); - const uchar* S = src.data + sy*src.step; -#pragma unroll(4) - for(x = 0; x < avxWidth; x += 8) - { - const __m256i CV_DECL_ALIGNED(64) *addr = (__m256i*)(x_ofs + x); - __m256i CV_DECL_ALIGNED(64) indices = _mm256_lddqu_si256(addr); - __m256i CV_DECL_ALIGNED(64) pixels = _mm256_i32gather_epi32((const int*)S, indices, 1); - _mm256_maskstore_epi32((int*)D, mask, pixels); - D += 32; - } - for(; x < width; x++) - { - *(int*)(Dstart + x*4) = *(int*)(S + x_ofs[x]); - } - } - } - else - { - for(y = range.start; y < range.end; y++) - { - uchar* D = dst.data + dst.step*y; - uchar* Dstart = D; - int sy = std::min(cvFloor(y*ify), ssize.height-1); - const uchar* S = src.data + sy*src.step; -#pragma unroll(4) - for(x = 0; x < avxWidth; x += 8) - { - const __m256i CV_DECL_ALIGNED(64) *addr = (__m256i*)(x_ofs + x); - __m256i CV_DECL_ALIGNED(64) indices = _mm256_lddqu_si256(addr); - __m256i CV_DECL_ALIGNED(64) pixels = _mm256_i32gather_epi32((const int*)S, indices, 1); - _mm256_storeu_si256((__m256i*)D, pixels); - D += 32; - } - for(; x < width; x++) - { - *(int*)(Dstart + x*4) = *(int*)(S + x_ofs[x]); - } - } - } - } - -private: - const Mat src; - Mat dst; - int* x_ofs, pix_size4; - double ify; - - resizeNNInvokerAVX4(const resizeNNInvokerAVX4&); - resizeNNInvokerAVX4& operator=(const resizeNNInvokerAVX4&); -}; - -class resizeNNInvokerAVX2 : - public ParallelLoopBody -{ -public: - resizeNNInvokerAVX2(const Mat& _src, Mat &_dst, int *_x_ofs, int _pix_size4, double _ify) : - ParallelLoopBody(), src(_src), dst(_dst), x_ofs(_x_ofs), pix_size4(_pix_size4), - ify(_ify) - { - } - -#if defined(__INTEL_COMPILER) -#pragma optimization_parameter target_arch=AVX -#endif - virtual void operator() (const Range& range) const - { - Size ssize = src.size(), dsize = dst.size(); - int y, x, pix_size = (int)src.elemSize(); - int width = dsize.width; - //int avxWidth = (width - 1) - ((width - 1) & 0x7); - int avxWidth = width - (width & 0xf); - const __m256i CV_DECL_ALIGNED(64) mask = _mm256_set1_epi32(-1); - const __m256i CV_DECL_ALIGNED(64) shuffle_mask = _mm256_set_epi8(15,14,11,10,13,12,9,8,7,6,3,2,5,4,1,0, - 15,14,11,10,13,12,9,8,7,6,3,2,5,4,1,0); - const __m256i CV_DECL_ALIGNED(64) permute_mask = _mm256_set_epi32(7, 5, 3, 1, 6, 4, 2, 0); - const __m256i CV_DECL_ALIGNED(64) shift_shuffle_mask = _mm256_set_epi8(13,12,15,14,9,8,11,10,5,4,7,6,1,0,3,2, - 13,12,15,14,9,8,11,10,5,4,7,6,1,0,3,2); - if(((int64)(dst.data + dst.step) & 0x1f) == 0) - { - for(y = range.start; y < range.end; y++) - { - uchar* D = dst.data + dst.step*y; - uchar* Dstart = D; - int sy = std::min(cvFloor(y*ify), ssize.height-1); - const uchar* S = src.data + sy*src.step; - const uchar* S2 = S - 2; -#pragma unroll(4) - for(x = 0; x < avxWidth; x += 16) - { - const __m256i CV_DECL_ALIGNED(64) *addr = (__m256i*)(x_ofs + x); - __m256i CV_DECL_ALIGNED(64) indices = _mm256_lddqu_si256(addr); - __m256i CV_DECL_ALIGNED(64) pixels1 = _mm256_i32gather_epi32((const int*)S, indices, 1); - const __m256i CV_DECL_ALIGNED(64) *addr2 = (__m256i*)(x_ofs + x + 8); - __m256i CV_DECL_ALIGNED(64) indices2 = _mm256_lddqu_si256(addr2); - __m256i CV_DECL_ALIGNED(64) pixels2 = _mm256_i32gather_epi32((const int*)S2, indices2, 1); - __m256i CV_DECL_ALIGNED(64) unpacked = _mm256_blend_epi16(pixels1, pixels2, 0xaa); - - __m256i CV_DECL_ALIGNED(64) bytes_shuffled = _mm256_shuffle_epi8(unpacked, shuffle_mask); - __m256i CV_DECL_ALIGNED(64) ints_permuted = _mm256_permutevar8x32_epi32(bytes_shuffled, permute_mask); - _mm256_maskstore_epi32((int*)D, mask, ints_permuted); - D += 32; - } - for(; x < width; x++) - { - *(ushort*)(Dstart + x*2) = *(ushort*)(S + x_ofs[x]); - } - - } - } - else - { - for(y = range.start; y < range.end; y++) - { - uchar* D = dst.data + dst.step*y; - uchar* Dstart = D; - int sy = std::min(cvFloor(y*ify), ssize.height-1); - const uchar* S = src.data + sy*src.step; - const uchar* S2 = S - 2; -#pragma unroll(4) - for(x = 0; x < avxWidth; x += 16) - { - const __m256i CV_DECL_ALIGNED(64) *addr = (__m256i*)(x_ofs + x); - __m256i CV_DECL_ALIGNED(64) indices = _mm256_lddqu_si256(addr); - __m256i CV_DECL_ALIGNED(64) pixels1 = _mm256_i32gather_epi32((const int*)S, indices, 1); - const __m256i CV_DECL_ALIGNED(64) *addr2 = (__m256i*)(x_ofs + x + 8); - __m256i CV_DECL_ALIGNED(64) indices2 = _mm256_lddqu_si256(addr2); - __m256i CV_DECL_ALIGNED(64) pixels2 = _mm256_i32gather_epi32((const int*)S2, indices2, 1); - __m256i CV_DECL_ALIGNED(64) unpacked = _mm256_blend_epi16(pixels1, pixels2, 0xaa); - - __m256i CV_DECL_ALIGNED(64) bytes_shuffled = _mm256_shuffle_epi8(unpacked, shuffle_mask); - __m256i CV_DECL_ALIGNED(64) ints_permuted = _mm256_permutevar8x32_epi32(bytes_shuffled, permute_mask); - _mm256_storeu_si256((__m256i*)D, ints_permuted); - D += 32; - } - for(; x < width; x++) - { - *(ushort*)(Dstart + x*2) = *(ushort*)(S + x_ofs[x]); - } - } - } - } - -private: - const Mat src; - Mat dst; - int* x_ofs, pix_size4; - double ify; - - resizeNNInvokerAVX2(const resizeNNInvokerAVX2&); - resizeNNInvokerAVX2& operator=(const resizeNNInvokerAVX2&); -}; -#endif - -#if CV_SSE4_1 -class resizeNNInvokerSSE2 : - public ParallelLoopBody -{ -public: - resizeNNInvokerSSE2(const Mat& _src, Mat &_dst, int *_x_ofs, int _pix_size4, double _ify) : - ParallelLoopBody(), src(_src), dst(_dst), x_ofs(_x_ofs), pix_size4(_pix_size4), - ify(_ify) - { - } - -#if defined(__INTEL_COMPILER) -#pragma optimization_parameter target_arch=SSE4.2 -#endif - virtual void operator() (const Range& range) const - { - Size ssize = src.size(), dsize = dst.size(); - int y, x; - int width = dsize.width; - int sseWidth = width - (width & 0x7); - for(y = range.start; y < range.end; y++) - { - uchar* D = dst.data + dst.step*y; - uchar* Dstart = D; - int sy = std::min(cvFloor(y*ify), ssize.height-1); - const uchar* S = src.data + sy*src.step; - __m128i CV_DECL_ALIGNED(64) pixels = _mm_set1_epi16(0); - for(x = 0; x < sseWidth; x += 8) - { - ushort imm = *(ushort*)(S + x_ofs[x + 0]); - pixels = _mm_insert_epi16(pixels, imm, 0); - imm = *(ushort*)(S + x_ofs[x + 1]); - pixels = _mm_insert_epi16(pixels, imm, 1); - imm = *(ushort*)(S + x_ofs[x + 2]); - pixels = _mm_insert_epi16(pixels, imm, 2); - imm = *(ushort*)(S + x_ofs[x + 3]); - pixels = _mm_insert_epi16(pixels, imm, 3); - imm = *(ushort*)(S + x_ofs[x + 4]); - pixels = _mm_insert_epi16(pixels, imm, 4); - imm = *(ushort*)(S + x_ofs[x + 5]); - pixels = _mm_insert_epi16(pixels, imm, 5); - imm = *(ushort*)(S + x_ofs[x + 6]); - pixels = _mm_insert_epi16(pixels, imm, 6); - imm = *(ushort*)(S + x_ofs[x + 7]); - pixels = _mm_insert_epi16(pixels, imm, 7); - _mm_storeu_si128((__m128i*)D, pixels); - D += 16; - } - for(; x < width; x++) - { - *(ushort*)(Dstart + x*2) = *(ushort*)(S + x_ofs[x]); - } - } - } - -private: - const Mat src; - Mat dst; - int* x_ofs, pix_size4; - double ify; - - resizeNNInvokerSSE2(const resizeNNInvokerSSE2&); - resizeNNInvokerSSE2& operator=(const resizeNNInvokerSSE2&); -}; - -class resizeNNInvokerSSE4 : - public ParallelLoopBody -{ -public: - resizeNNInvokerSSE4(const Mat& _src, Mat &_dst, int *_x_ofs, int _pix_size4, double _ify) : - ParallelLoopBody(), src(_src), dst(_dst), x_ofs(_x_ofs), pix_size4(_pix_size4), - ify(_ify) - { - } -#if defined(__INTEL_COMPILER) -#pragma optimization_parameter target_arch=SSE4.2 -#endif - virtual void operator() (const Range& range) const - { - Size ssize = src.size(), dsize = dst.size(); - int y, x; - int width = dsize.width; - int sseWidth = width - (width & 0x3); - for(y = range.start; y < range.end; y++) - { - uchar* D = dst.data + dst.step*y; - uchar* Dstart = D; - int sy = std::min(cvFloor(y*ify), ssize.height-1); - const uchar* S = src.data + sy*src.step; - __m128i CV_DECL_ALIGNED(64) pixels = _mm_set1_epi16(0); - for(x = 0; x < sseWidth; x += 4) - { - int imm = *(int*)(S + x_ofs[x + 0]); - pixels = _mm_insert_epi32(pixels, imm, 0); - imm = *(int*)(S + x_ofs[x + 1]); - pixels = _mm_insert_epi32(pixels, imm, 1); - imm = *(int*)(S + x_ofs[x + 2]); - pixels = _mm_insert_epi32(pixels, imm, 2); - imm = *(int*)(S + x_ofs[x + 3]); - pixels = _mm_insert_epi32(pixels, imm, 3); - _mm_storeu_si128((__m128i*)D, pixels); - D += 16; - } - for(; x < width; x++) - { - *(int*)(Dstart + x*4) = *(int*)(S + x_ofs[x]); - } - } - } - -private: - const Mat src; - Mat dst; - int* x_ofs, pix_size4; - double ify; - - resizeNNInvokerSSE4(const resizeNNInvokerSSE4&); - resizeNNInvokerSSE4& operator=(const resizeNNInvokerSSE4&); -}; -#endif - static void resizeNN( const Mat& src, Mat& dst, double fx, double fy ) { @@ -737,35 +436,23 @@ resizeNN( const Mat& src, Mat& dst, double fx, double fy ) } Range range(0, dsize.height); -#if CV_AVX2 - if(checkHardwareSupport(CV_CPU_AVX2) && ((pix_size == 2) || (pix_size == 4))) +#if CV_TRY_AVX2 + if(CV_CPU_HAS_SUPPORT_AVX2 && ((pix_size == 2) || (pix_size == 4))) { if(pix_size == 2) - { - resizeNNInvokerAVX2 invoker(src, dst, x_ofs, pix_size4, ify); - parallel_for_(range, invoker, dst.total()/(double)(1<<16)); - } - else if (pix_size == 4) - { - resizeNNInvokerAVX4 invoker(src, dst, x_ofs, pix_size4, ify); - parallel_for_(range, invoker, dst.total()/(double)(1<<16)); - } + opt_AVX2::resizeNN2_AVX2(range, src, dst, x_ofs, pix_size4, ify); + else + opt_AVX2::resizeNN4_AVX2(range, src, dst, x_ofs, pix_size4, ify); } else #endif -#if CV_SSE4_1 - if(checkHardwareSupport(CV_CPU_SSE4_1) && ((pix_size == 2) || (pix_size == 4))) +#if CV_TRY_SSE4_1 + if(CV_CPU_HAS_SUPPORT_SSE4_1 && ((pix_size == 2) || (pix_size == 4))) { if(pix_size == 2) - { - resizeNNInvokerSSE2 invoker(src, dst, x_ofs, pix_size4, ify); - parallel_for_(range, invoker, dst.total()/(double)(1<<16)); - } - else if(pix_size == 4) - { - resizeNNInvokerSSE4 invoker(src, dst, x_ofs, pix_size4, ify); - parallel_for_(range, invoker, dst.total()/(double)(1<<16)); - } + opt_SSE41::resizeNN2_SSE4_1(range, src, dst, x_ofs, pix_size4, ify); + else + opt_SSE41::resizeNN4_SSE4_1(range, src, dst, x_ofs, pix_size4, ify); } else #endif @@ -5864,8 +5551,8 @@ public: const int AB_BITS = MAX(10, (int)INTER_BITS); const int AB_SCALE = 1 << AB_BITS; int round_delta = interpolation == INTER_NEAREST ? AB_SCALE/2 : AB_SCALE/INTER_TAB_SIZE/2, x, y, x1, y1; - #if CV_AVX2 - bool useAVX2 = checkHardwareSupport(CV_CPU_AVX2); + #if CV_TRY_AVX2 + bool useAVX2 = CV_CPU_HAS_SUPPORT_AVX2; #endif #if CV_SSE2 bool useSSE2 = checkHardwareSupport(CV_CPU_SSE2); @@ -5947,41 +5634,9 @@ public: { short* alpha = A + y1*bw; x1 = 0; - #if CV_AVX2 + #if CV_TRY_AVX2 if ( useAVX2 ) - { - __m256i fxy_mask = _mm256_set1_epi32(INTER_TAB_SIZE - 1); - __m256i XX = _mm256_set1_epi32(X0), YY = _mm256_set1_epi32(Y0); - for( ; x1 <= bw - 16; x1 += 16 ) - { - __m256i tx0, tx1, ty0, ty1; - tx0 = _mm256_add_epi32(_mm256_loadu_si256((const __m256i*)(adelta + x + x1)), XX); - ty0 = _mm256_add_epi32(_mm256_loadu_si256((const __m256i*)(bdelta + x + x1)), YY); - tx1 = _mm256_add_epi32(_mm256_loadu_si256((const __m256i*)(adelta + x + x1 + 8)), XX); - ty1 = _mm256_add_epi32(_mm256_loadu_si256((const __m256i*)(bdelta + x + x1 + 8)), YY); - - tx0 = _mm256_srai_epi32(tx0, AB_BITS - INTER_BITS); - ty0 = _mm256_srai_epi32(ty0, AB_BITS - INTER_BITS); - tx1 = _mm256_srai_epi32(tx1, AB_BITS - INTER_BITS); - ty1 = _mm256_srai_epi32(ty1, AB_BITS - INTER_BITS); - - __m256i fx_ = _mm256_packs_epi32(_mm256_and_si256(tx0, fxy_mask), - _mm256_and_si256(tx1, fxy_mask)); - __m256i fy_ = _mm256_packs_epi32(_mm256_and_si256(ty0, fxy_mask), - _mm256_and_si256(ty1, fxy_mask)); - tx0 = _mm256_packs_epi32(_mm256_srai_epi32(tx0, INTER_BITS), - _mm256_srai_epi32(tx1, INTER_BITS)); - ty0 = _mm256_packs_epi32(_mm256_srai_epi32(ty0, INTER_BITS), - _mm256_srai_epi32(ty1, INTER_BITS)); - fx_ = _mm256_adds_epi16(fx_, _mm256_slli_epi16(fy_, INTER_BITS)); - fx_ = _mm256_permute4x64_epi64(fx_, (3 << 6) + (1 << 4) + (2 << 2) + 0); - - _mm256_storeu_si256((__m256i*)(xy + x1*2), _mm256_unpacklo_epi16(tx0, ty0)); - _mm256_storeu_si256((__m256i*)(xy + x1*2 + 16), _mm256_unpackhi_epi16(tx0, ty0)); - _mm256_storeu_si256((__m256i*)(alpha + x1), fx_); - } - _mm256_zeroupper(); - } + x1 = opt_AVX2::warpAffineBlockline(adelta + x, bdelta + x, xy, alpha, X0, Y0, bw); #endif #if CV_SSE2 if( useSSE2 ) diff --git a/modules/imgproc/src/imgwarp.hpp b/modules/imgproc/src/imgwarp.hpp new file mode 100644 index 0000000000..428bcc51e7 --- /dev/null +++ b/modules/imgproc/src/imgwarp.hpp @@ -0,0 +1,73 @@ +/*M/////////////////////////////////////////////////////////////////////////////////////// +// +// IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. +// +// By downloading, copying, installing or using the software you agree to this license. +// If you do not agree to this license, do not download, install, +// copy or use the software. +// +// +// License Agreement +// For Open Source Computer Vision Library +// +// Copyright (C) 2000-2008, Intel Corporation, all rights reserved. +// Copyright (C) 2009, Willow Garage Inc., all rights reserved. +// Copyright (C) 2014-2015, Itseez Inc., all rights reserved. +// Third party copyrights are property of their respective owners. +// +// Redistribution and use in source and binary forms, with or without modification, +// are permitted provided that the following conditions are met: +// +// * Redistribution's of source code must retain the above copyright notice, +// this list of conditions and the following disclaimer. +// +// * Redistribution's in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// +// * The name of the copyright holders may not be used to endorse or promote products +// derived from this software without specific prior written permission. +// +// This software is provided by the copyright holders and contributors "as is" and +// any express or implied warranties, including, but not limited to, the implied +// warranties of merchantability and fitness for a particular purpose are disclaimed. +// In no event shall the Intel Corporation or contributors be liable for any direct, +// indirect, incidental, special, exemplary, or consequential damages +// (including, but not limited to, procurement of substitute goods or services; +// loss of use, data, or profits; or business interruption) however caused +// and on any theory of liability, whether in contract, strict liability, +// or tort (including negligence or otherwise) arising in any way out of +// the use of this software, even if advised of the possibility of such damage. +// +//M*/ + +/* //////////////////////////////////////////////////////////////////// +// +// Geometrical transforms on images and matrices: rotation, zoom etc. +// +// */ + +#ifndef OPENCV_IMGPROC_IMGWARP_HPP +#define OPENCV_IMGPROC_IMGWARP_HPP +#include "precomp.hpp" + +namespace cv +{ +namespace opt_AVX2 +{ +#if CV_TRY_AVX2 +void resizeNN2_AVX2(const Range&, const Mat&, Mat&, int*, int, double); +void resizeNN4_AVX2(const Range&, const Mat&, Mat&, int*, int, double); +int warpAffineBlockline(int *adelta, int *bdelta, short* xy, short* alpha, int X0, int Y0, int bw); +#endif +} +namespace opt_SSE41 +{ +#if CV_TRY_SSE4_1 +void resizeNN2_SSE4_1(const Range&, const Mat&, Mat&, int*, int, double); +void resizeNN4_SSE4_1(const Range&, const Mat&, Mat&, int*, int, double); +#endif +} +} +#endif +/* End of file. */ diff --git a/modules/imgproc/src/imgwarp.sse4_1.cpp b/modules/imgproc/src/imgwarp.sse4_1.cpp new file mode 100644 index 0000000000..b326ffcb1a --- /dev/null +++ b/modules/imgproc/src/imgwarp.sse4_1.cpp @@ -0,0 +1,192 @@ +/*M/////////////////////////////////////////////////////////////////////////////////////// +// +// IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. +// +// By downloading, copying, installing or using the software you agree to this license. +// If you do not agree to this license, do not download, install, +// copy or use the software. +// +// +// License Agreement +// For Open Source Computer Vision Library +// +// Copyright (C) 2000-2008, Intel Corporation, all rights reserved. +// Copyright (C) 2009, Willow Garage Inc., all rights reserved. +// Copyright (C) 2014-2015, Itseez Inc., all rights reserved. +// Third party copyrights are property of their respective owners. +// +// Redistribution and use in source and binary forms, with or without modification, +// are permitted provided that the following conditions are met: +// +// * Redistribution's of source code must retain the above copyright notice, +// this list of conditions and the following disclaimer. +// +// * Redistribution's in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// +// * The name of the copyright holders may not be used to endorse or promote products +// derived from this software without specific prior written permission. +// +// This software is provided by the copyright holders and contributors "as is" and +// any express or implied warranties, including, but not limited to, the implied +// warranties of merchantability and fitness for a particular purpose are disclaimed. +// In no event shall the Intel Corporation or contributors be liable for any direct, +// indirect, incidental, special, exemplary, or consequential damages +// (including, but not limited to, procurement of substitute goods or services; +// loss of use, data, or profits; or business interruption) however caused +// and on any theory of liability, whether in contract, strict liability, +// or tort (including negligence or otherwise) arising in any way out of +// the use of this software, even if advised of the possibility of such damage. +// +//M*/ + +/* //////////////////////////////////////////////////////////////////// +// +// Geometrical transforms on images and matrices: rotation, zoom etc. +// +// */ + +#include "precomp.hpp" +#include "imgwarp.hpp" + +namespace cv +{ +namespace opt_SSE41 +{ + +class resizeNNInvokerSSE2 : + public ParallelLoopBody +{ +public: + resizeNNInvokerSSE2(const Mat& _src, Mat &_dst, int *_x_ofs, int _pix_size4, double _ify) : + ParallelLoopBody(), src(_src), dst(_dst), x_ofs(_x_ofs), pix_size4(_pix_size4), + ify(_ify) + { + } + +#if defined(__INTEL_COMPILER) +#pragma optimization_parameter target_arch=SSE4.2 +#endif + virtual void operator() (const Range& range) const + { + Size ssize = src.size(), dsize = dst.size(); + int y, x; + int width = dsize.width; + int sseWidth = width - (width & 0x7); + for(y = range.start; y < range.end; y++) + { + uchar* D = dst.data + dst.step*y; + uchar* Dstart = D; + int sy = std::min(cvFloor(y*ify), ssize.height-1); + const uchar* S = src.data + sy*src.step; + __m128i CV_DECL_ALIGNED(64) pixels = _mm_set1_epi16(0); + for(x = 0; x < sseWidth; x += 8) + { + ushort imm = *(ushort*)(S + x_ofs[x + 0]); + pixels = _mm_insert_epi16(pixels, imm, 0); + imm = *(ushort*)(S + x_ofs[x + 1]); + pixels = _mm_insert_epi16(pixels, imm, 1); + imm = *(ushort*)(S + x_ofs[x + 2]); + pixels = _mm_insert_epi16(pixels, imm, 2); + imm = *(ushort*)(S + x_ofs[x + 3]); + pixels = _mm_insert_epi16(pixels, imm, 3); + imm = *(ushort*)(S + x_ofs[x + 4]); + pixels = _mm_insert_epi16(pixels, imm, 4); + imm = *(ushort*)(S + x_ofs[x + 5]); + pixels = _mm_insert_epi16(pixels, imm, 5); + imm = *(ushort*)(S + x_ofs[x + 6]); + pixels = _mm_insert_epi16(pixels, imm, 6); + imm = *(ushort*)(S + x_ofs[x + 7]); + pixels = _mm_insert_epi16(pixels, imm, 7); + _mm_storeu_si128((__m128i*)D, pixels); + D += 16; + } + for(; x < width; x++) + { + *(ushort*)(Dstart + x*2) = *(ushort*)(S + x_ofs[x]); + } + } + } + +private: + const Mat src; + Mat dst; + int* x_ofs, pix_size4; + double ify; + + resizeNNInvokerSSE2(const resizeNNInvokerSSE2&); + resizeNNInvokerSSE2& operator=(const resizeNNInvokerSSE2&); +}; + +class resizeNNInvokerSSE4 : + public ParallelLoopBody +{ +public: + resizeNNInvokerSSE4(const Mat& _src, Mat &_dst, int *_x_ofs, int _pix_size4, double _ify) : + ParallelLoopBody(), src(_src), dst(_dst), x_ofs(_x_ofs), pix_size4(_pix_size4), + ify(_ify) + { + } +#if defined(__INTEL_COMPILER) +#pragma optimization_parameter target_arch=SSE4.2 +#endif + virtual void operator() (const Range& range) const + { + Size ssize = src.size(), dsize = dst.size(); + int y, x; + int width = dsize.width; + int sseWidth = width - (width & 0x3); + for(y = range.start; y < range.end; y++) + { + uchar* D = dst.data + dst.step*y; + uchar* Dstart = D; + int sy = std::min(cvFloor(y*ify), ssize.height-1); + const uchar* S = src.data + sy*src.step; + __m128i CV_DECL_ALIGNED(64) pixels = _mm_set1_epi16(0); + for(x = 0; x < sseWidth; x += 4) + { + int imm = *(int*)(S + x_ofs[x + 0]); + pixels = _mm_insert_epi32(pixels, imm, 0); + imm = *(int*)(S + x_ofs[x + 1]); + pixels = _mm_insert_epi32(pixels, imm, 1); + imm = *(int*)(S + x_ofs[x + 2]); + pixels = _mm_insert_epi32(pixels, imm, 2); + imm = *(int*)(S + x_ofs[x + 3]); + pixels = _mm_insert_epi32(pixels, imm, 3); + _mm_storeu_si128((__m128i*)D, pixels); + D += 16; + } + for(; x < width; x++) + { + *(int*)(Dstart + x*4) = *(int*)(S + x_ofs[x]); + } + } + } + +private: + const Mat src; + Mat dst; + int* x_ofs, pix_size4; + double ify; + + resizeNNInvokerSSE4(const resizeNNInvokerSSE4&); + resizeNNInvokerSSE4& operator=(const resizeNNInvokerSSE4&); +}; + +void resizeNN2_SSE4_1(const Range& range, const Mat& src, Mat &dst, int *x_ofs, int pix_size4, double ify) +{ + resizeNNInvokerSSE2 invoker(src, dst, x_ofs, pix_size4, ify); + parallel_for_(range, invoker, dst.total() / (double)(1 << 16)); +} + +void resizeNN4_SSE4_1(const Range& range, const Mat& src, Mat &dst, int *x_ofs, int pix_size4, double ify) +{ + resizeNNInvokerSSE4 invoker(src, dst, x_ofs, pix_size4, ify); + parallel_for_(range, invoker, dst.total() / (double)(1 << 16)); +} + +} +} + +/* End of file. */ From fadf25acd6e255ed03303d9354b0e0bd93325e3c Mon Sep 17 00:00:00 2001 From: Vitaly Tuzov Date: Mon, 3 Jul 2017 18:21:22 +0300 Subject: [PATCH 2/2] SSE4_1 optimized implementation of resize and warp functions migrated to separate file --- modules/imgproc/src/imgwarp.avx2.cpp | 12 +- modules/imgproc/src/imgwarp.cpp | 435 ++-------------------- modules/imgproc/src/imgwarp.hpp | 18 +- modules/imgproc/src/imgwarp.sse4_1.cpp | 491 ++++++++++++++++++++++++- 4 files changed, 549 insertions(+), 407 deletions(-) diff --git a/modules/imgproc/src/imgwarp.avx2.cpp b/modules/imgproc/src/imgwarp.avx2.cpp index 321957f39e..d434807706 100644 --- a/modules/imgproc/src/imgwarp.avx2.cpp +++ b/modules/imgproc/src/imgwarp.avx2.cpp @@ -83,7 +83,9 @@ public: uchar* Dstart = D; int sy = std::min(cvFloor(y*ify), ssize.height-1); const uchar* S = src.data + sy*src.step; +#ifdef CV_ICC #pragma unroll(4) +#endif for(x = 0; x < avxWidth; x += 8) { const __m256i CV_DECL_ALIGNED(64) *addr = (__m256i*)(x_ofs + x); @@ -106,7 +108,9 @@ public: uchar* Dstart = D; int sy = std::min(cvFloor(y*ify), ssize.height-1); const uchar* S = src.data + sy*src.step; +#ifdef CV_ICC #pragma unroll(4) +#endif for(x = 0; x < avxWidth; x += 8) { const __m256i CV_DECL_ALIGNED(64) *addr = (__m256i*)(x_ofs + x); @@ -157,8 +161,8 @@ public: const __m256i CV_DECL_ALIGNED(64) shuffle_mask = _mm256_set_epi8(15,14,11,10,13,12,9,8,7,6,3,2,5,4,1,0, 15,14,11,10,13,12,9,8,7,6,3,2,5,4,1,0); const __m256i CV_DECL_ALIGNED(64) permute_mask = _mm256_set_epi32(7, 5, 3, 1, 6, 4, 2, 0); - const __m256i CV_DECL_ALIGNED(64) shift_shuffle_mask = _mm256_set_epi8(13,12,15,14,9,8,11,10,5,4,7,6,1,0,3,2, - 13,12,15,14,9,8,11,10,5,4,7,6,1,0,3,2); + //const __m256i CV_DECL_ALIGNED(64) shift_shuffle_mask = _mm256_set_epi8(13,12,15,14,9,8,11,10,5,4,7,6,1,0,3,2, + // 13,12,15,14,9,8,11,10,5,4,7,6,1,0,3,2); if(((int64)(dst.data + dst.step) & 0x1f) == 0) { for(y = range.start; y < range.end; y++) @@ -168,7 +172,9 @@ public: int sy = std::min(cvFloor(y*ify), ssize.height-1); const uchar* S = src.data + sy*src.step; const uchar* S2 = S - 2; +#ifdef CV_ICC #pragma unroll(4) +#endif for(x = 0; x < avxWidth; x += 16) { const __m256i CV_DECL_ALIGNED(64) *addr = (__m256i*)(x_ofs + x); @@ -200,7 +206,9 @@ public: int sy = std::min(cvFloor(y*ify), ssize.height-1); const uchar* S = src.data + sy*src.step; const uchar* S2 = S - 2; +#ifdef CV_ICC #pragma unroll(4) +#endif for(x = 0; x < avxWidth; x += 16) { const __m256i CV_DECL_ALIGNED(64) *addr = (__m256i*)(x_ofs + x); diff --git a/modules/imgproc/src/imgwarp.cpp b/modules/imgproc/src/imgwarp.cpp index 4f040a91cc..a960c01ffb 100644 --- a/modules/imgproc/src/imgwarp.cpp +++ b/modules/imgproc/src/imgwarp.cpp @@ -450,9 +450,9 @@ resizeNN( const Mat& src, Mat& dst, double fx, double fy ) if(CV_CPU_HAS_SUPPORT_SSE4_1 && ((pix_size == 2) || (pix_size == 4))) { if(pix_size == 2) - opt_SSE41::resizeNN2_SSE4_1(range, src, dst, x_ofs, pix_size4, ify); + opt_SSE4_1::resizeNN2_SSE4_1(range, src, dst, x_ofs, pix_size4, ify); else - opt_SSE41::resizeNN4_SSE4_1(range, src, dst, x_ofs, pix_size4, ify); + opt_SSE4_1::resizeNN4_SSE4_1(range, src, dst, x_ofs, pix_size4, ify); } else #endif @@ -916,50 +916,14 @@ struct VResizeCubicVec_32f } }; -#if CV_SSE4_1 +#if CV_TRY_SSE4_1 struct VResizeLanczos4Vec_32f16u { int operator()(const uchar** _src, uchar* _dst, const uchar* _beta, int width ) const { - const float** src = (const float**)_src; - const float* beta = (const float*)_beta; - const float *S0 = src[0], *S1 = src[1], *S2 = src[2], *S3 = src[3], - *S4 = src[4], *S5 = src[5], *S6 = src[6], *S7 = src[7]; - short * dst = (short*)_dst; - int x = 0; - __m128 v_b0 = _mm_set1_ps(beta[0]), v_b1 = _mm_set1_ps(beta[1]), - v_b2 = _mm_set1_ps(beta[2]), v_b3 = _mm_set1_ps(beta[3]), - v_b4 = _mm_set1_ps(beta[4]), v_b5 = _mm_set1_ps(beta[5]), - v_b6 = _mm_set1_ps(beta[6]), v_b7 = _mm_set1_ps(beta[7]); - - for( ; x <= width - 8; x += 8 ) - { - __m128 v_dst0 = _mm_mul_ps(v_b0, _mm_loadu_ps(S0 + x)); - v_dst0 = _mm_add_ps(v_dst0, _mm_mul_ps(v_b1, _mm_loadu_ps(S1 + x))); - v_dst0 = _mm_add_ps(v_dst0, _mm_mul_ps(v_b2, _mm_loadu_ps(S2 + x))); - v_dst0 = _mm_add_ps(v_dst0, _mm_mul_ps(v_b3, _mm_loadu_ps(S3 + x))); - v_dst0 = _mm_add_ps(v_dst0, _mm_mul_ps(v_b4, _mm_loadu_ps(S4 + x))); - v_dst0 = _mm_add_ps(v_dst0, _mm_mul_ps(v_b5, _mm_loadu_ps(S5 + x))); - v_dst0 = _mm_add_ps(v_dst0, _mm_mul_ps(v_b6, _mm_loadu_ps(S6 + x))); - v_dst0 = _mm_add_ps(v_dst0, _mm_mul_ps(v_b7, _mm_loadu_ps(S7 + x))); - - __m128 v_dst1 = _mm_mul_ps(v_b0, _mm_loadu_ps(S0 + x + 4)); - v_dst1 = _mm_add_ps(v_dst1, _mm_mul_ps(v_b1, _mm_loadu_ps(S1 + x + 4))); - v_dst1 = _mm_add_ps(v_dst1, _mm_mul_ps(v_b2, _mm_loadu_ps(S2 + x + 4))); - v_dst1 = _mm_add_ps(v_dst1, _mm_mul_ps(v_b3, _mm_loadu_ps(S3 + x + 4))); - v_dst1 = _mm_add_ps(v_dst1, _mm_mul_ps(v_b4, _mm_loadu_ps(S4 + x + 4))); - v_dst1 = _mm_add_ps(v_dst1, _mm_mul_ps(v_b5, _mm_loadu_ps(S5 + x + 4))); - v_dst1 = _mm_add_ps(v_dst1, _mm_mul_ps(v_b6, _mm_loadu_ps(S6 + x + 4))); - v_dst1 = _mm_add_ps(v_dst1, _mm_mul_ps(v_b7, _mm_loadu_ps(S7 + x + 4))); - - __m128i v_dsti0 = _mm_cvtps_epi32(v_dst0); - __m128i v_dsti1 = _mm_cvtps_epi32(v_dst1); - - _mm_storeu_si128((__m128i *)(dst + x), _mm_packus_epi32(v_dsti0, v_dsti1)); - } - - return x; + if (CV_CPU_HAS_SUPPORT_SSE4_1) return opt_SSE4_1::VResizeLanczos4Vec_32f16u_SSE41(_src, _dst, _beta, width); + else return 0; } }; @@ -5149,8 +5113,8 @@ void cv::convertMaps( InputArray _map1, InputArray _map2, #if CV_SSE2 bool useSSE2 = checkHardwareSupport(CV_CPU_SSE2); #endif -#if CV_SSE4_1 - bool useSSE4_1 = checkHardwareSupport(CV_CPU_SSE4_1); +#if CV_TRY_SSE4_1 + bool useSSE4_1 = CV_CPU_HAS_SUPPORT_SSE4_1; #endif const float scale = 1.f/INTER_TAB_SIZE; @@ -5183,29 +5147,10 @@ void cv::convertMaps( InputArray _map1, InputArray _map2, vst2q_s16(dst1 + (x << 1), v_dst); } - #elif CV_SSE4_1 + #elif CV_TRY_SSE4_1 if (useSSE4_1) - { - for( ; x <= size.width - 16; x += 16 ) - { - __m128i v_dst0 = _mm_packs_epi32(_mm_cvtps_epi32(_mm_loadu_ps(src1f + x)), - _mm_cvtps_epi32(_mm_loadu_ps(src1f + x + 4))); - __m128i v_dst1 = _mm_packs_epi32(_mm_cvtps_epi32(_mm_loadu_ps(src1f + x + 8)), - _mm_cvtps_epi32(_mm_loadu_ps(src1f + x + 12))); - - __m128i v_dst2 = _mm_packs_epi32(_mm_cvtps_epi32(_mm_loadu_ps(src2f + x)), - _mm_cvtps_epi32(_mm_loadu_ps(src2f + x + 4))); - __m128i v_dst3 = _mm_packs_epi32(_mm_cvtps_epi32(_mm_loadu_ps(src2f + x + 8)), - _mm_cvtps_epi32(_mm_loadu_ps(src2f + x + 12))); - - _mm_interleave_epi16(v_dst0, v_dst1, v_dst2, v_dst3); - - _mm_storeu_si128((__m128i *)(dst1 + x * 2), v_dst0); - _mm_storeu_si128((__m128i *)(dst1 + x * 2 + 8), v_dst1); - _mm_storeu_si128((__m128i *)(dst1 + x * 2 + 16), v_dst2); - _mm_storeu_si128((__m128i *)(dst1 + x * 2 + 24), v_dst3); - } - } + opt_SSE4_1::convertMaps_nninterpolate32f1c16s_SSE41(src1f, src2f, dst1, size.width); + else #endif for( ; x < size.width; x++ ) { @@ -5240,52 +5185,10 @@ void cv::convertMaps( InputArray _map1, InputArray _map2, vandq_s32(v_ix1, v_mask))); vst1q_u16(dst2 + x, vcombine_u16(v_dst0, v_dst1)); } - #elif CV_SSE4_1 + #elif CV_TRY_SSE4_1 if (useSSE4_1) - { - __m128 v_its = _mm_set1_ps(INTER_TAB_SIZE); - __m128i v_its1 = _mm_set1_epi32(INTER_TAB_SIZE-1); - - for( ; x <= size.width - 16; x += 16 ) - { - __m128i v_ix0 = _mm_cvtps_epi32(_mm_mul_ps(_mm_loadu_ps(src1f + x), v_its)); - __m128i v_ix1 = _mm_cvtps_epi32(_mm_mul_ps(_mm_loadu_ps(src1f + x + 4), v_its)); - __m128i v_iy0 = _mm_cvtps_epi32(_mm_mul_ps(_mm_loadu_ps(src2f + x), v_its)); - __m128i v_iy1 = _mm_cvtps_epi32(_mm_mul_ps(_mm_loadu_ps(src2f + x + 4), v_its)); - - __m128i v_dst10 = _mm_packs_epi32(_mm_srai_epi32(v_ix0, INTER_BITS), - _mm_srai_epi32(v_ix1, INTER_BITS)); - __m128i v_dst12 = _mm_packs_epi32(_mm_srai_epi32(v_iy0, INTER_BITS), - _mm_srai_epi32(v_iy1, INTER_BITS)); - __m128i v_dst20 = _mm_add_epi32(_mm_slli_epi32(_mm_and_si128(v_iy0, v_its1), INTER_BITS), - _mm_and_si128(v_ix0, v_its1)); - __m128i v_dst21 = _mm_add_epi32(_mm_slli_epi32(_mm_and_si128(v_iy1, v_its1), INTER_BITS), - _mm_and_si128(v_ix1, v_its1)); - _mm_storeu_si128((__m128i *)(dst2 + x), _mm_packus_epi32(v_dst20, v_dst21)); - - v_ix0 = _mm_cvtps_epi32(_mm_mul_ps(_mm_loadu_ps(src1f + x + 8), v_its)); - v_ix1 = _mm_cvtps_epi32(_mm_mul_ps(_mm_loadu_ps(src1f + x + 12), v_its)); - v_iy0 = _mm_cvtps_epi32(_mm_mul_ps(_mm_loadu_ps(src2f + x + 8), v_its)); - v_iy1 = _mm_cvtps_epi32(_mm_mul_ps(_mm_loadu_ps(src2f + x + 12), v_its)); - - __m128i v_dst11 = _mm_packs_epi32(_mm_srai_epi32(v_ix0, INTER_BITS), - _mm_srai_epi32(v_ix1, INTER_BITS)); - __m128i v_dst13 = _mm_packs_epi32(_mm_srai_epi32(v_iy0, INTER_BITS), - _mm_srai_epi32(v_iy1, INTER_BITS)); - v_dst20 = _mm_add_epi32(_mm_slli_epi32(_mm_and_si128(v_iy0, v_its1), INTER_BITS), - _mm_and_si128(v_ix0, v_its1)); - v_dst21 = _mm_add_epi32(_mm_slli_epi32(_mm_and_si128(v_iy1, v_its1), INTER_BITS), - _mm_and_si128(v_ix1, v_its1)); - _mm_storeu_si128((__m128i *)(dst2 + x + 8), _mm_packus_epi32(v_dst20, v_dst21)); - - _mm_interleave_epi16(v_dst10, v_dst11, v_dst12, v_dst13); - - _mm_storeu_si128((__m128i *)(dst1 + x * 2), v_dst10); - _mm_storeu_si128((__m128i *)(dst1 + x * 2 + 8), v_dst11); - _mm_storeu_si128((__m128i *)(dst1 + x * 2 + 16), v_dst12); - _mm_storeu_si128((__m128i *)(dst1 + x * 2 + 24), v_dst13); - } - } + opt_SSE4_1::convertMaps_32f1c16s_SSE41(src1f, src2f, dst1, dst2, size.width); + else #endif for( ; x < size.width; x++ ) { @@ -5346,30 +5249,10 @@ void cv::convertMaps( InputArray _map1, InputArray _map2, vandq_s32(v_ix1, v_mask))); vst1q_u16(dst2 + x, vcombine_u16(v_dst0, v_dst1)); } - #elif CV_SSE4_1 + #elif CV_TRY_SSE4_1 if (useSSE4_1) - { - __m128 v_its = _mm_set1_ps(INTER_TAB_SIZE); - __m128i v_its1 = _mm_set1_epi32(INTER_TAB_SIZE-1); - __m128i v_y_mask = _mm_set1_epi32((INTER_TAB_SIZE-1) << 16); - - for( ; x <= size.width - 4; x += 4 ) - { - __m128i v_src0 = _mm_cvtps_epi32(_mm_mul_ps(_mm_loadu_ps(src1f + x * 2), v_its)); - __m128i v_src1 = _mm_cvtps_epi32(_mm_mul_ps(_mm_loadu_ps(src1f + x * 2 + 4), v_its)); - - __m128i v_dst1 = _mm_packs_epi32(_mm_srai_epi32(v_src0, INTER_BITS), - _mm_srai_epi32(v_src1, INTER_BITS)); - _mm_storeu_si128((__m128i *)(dst1 + x * 2), v_dst1); - - // x0 y0 x1 y1 . . . - v_src0 = _mm_packs_epi32(_mm_and_si128(v_src0, v_its1), - _mm_and_si128(v_src1, v_its1)); - __m128i v_dst2 = _mm_or_si128(_mm_srli_epi32(_mm_and_si128(v_src0, v_y_mask), 16 - INTER_BITS), // y0 0 y1 0 . . . - _mm_and_si128(v_src0, v_its1)); // 0 x0 0 x1 . . . - _mm_storel_epi64((__m128i *)(dst2 + x), _mm_packus_epi32(v_dst2, v_dst2)); - } - } + opt_SSE4_1::convertMaps_32f2c16s_SSE41(src1f, dst1, dst2, size.width); + else #endif for( ; x < size.width; x++ ) { @@ -5557,8 +5440,8 @@ public: #if CV_SSE2 bool useSSE2 = checkHardwareSupport(CV_CPU_SSE2); #endif - #if CV_SSE4_1 - bool useSSE4_1 = checkHardwareSupport(CV_CPU_SSE4_1); + #if CV_TRY_SSE4_1 + bool useSSE4_1 = CV_CPU_HAS_SUPPORT_SSE4_1; #endif int bh0 = std::min(BLOCK_SZ/2, dst.rows); @@ -5596,31 +5479,10 @@ public: vst2q_s16(xy + (x1 << 1), v_dst); } - #elif CV_SSE4_1 + #elif CV_TRY_SSE4_1 if (useSSE4_1) - { - __m128i v_X0 = _mm_set1_epi32(X0); - __m128i v_Y0 = _mm_set1_epi32(Y0); - for ( ; x1 <= bw - 16; x1 += 16) - { - __m128i v_x0 = _mm_packs_epi32(_mm_srai_epi32(_mm_add_epi32(v_X0, _mm_loadu_si128((__m128i const *)(adelta + x + x1))), AB_BITS), - _mm_srai_epi32(_mm_add_epi32(v_X0, _mm_loadu_si128((__m128i const *)(adelta + x + x1 + 4))), AB_BITS)); - __m128i v_x1 = _mm_packs_epi32(_mm_srai_epi32(_mm_add_epi32(v_X0, _mm_loadu_si128((__m128i const *)(adelta + x + x1 + 8))), AB_BITS), - _mm_srai_epi32(_mm_add_epi32(v_X0, _mm_loadu_si128((__m128i const *)(adelta + x + x1 + 12))), AB_BITS)); - - __m128i v_y0 = _mm_packs_epi32(_mm_srai_epi32(_mm_add_epi32(v_Y0, _mm_loadu_si128((__m128i const *)(bdelta + x + x1))), AB_BITS), - _mm_srai_epi32(_mm_add_epi32(v_Y0, _mm_loadu_si128((__m128i const *)(bdelta + x + x1 + 4))), AB_BITS)); - __m128i v_y1 = _mm_packs_epi32(_mm_srai_epi32(_mm_add_epi32(v_Y0, _mm_loadu_si128((__m128i const *)(bdelta + x + x1 + 8))), AB_BITS), - _mm_srai_epi32(_mm_add_epi32(v_Y0, _mm_loadu_si128((__m128i const *)(bdelta + x + x1 + 12))), AB_BITS)); - - _mm_interleave_epi16(v_x0, v_x1, v_y0, v_y1); - - _mm_storeu_si128((__m128i *)(xy + x1 * 2), v_x0); - _mm_storeu_si128((__m128i *)(xy + x1 * 2 + 8), v_x1); - _mm_storeu_si128((__m128i *)(xy + x1 * 2 + 16), v_y0); - _mm_storeu_si128((__m128i *)(xy + x1 * 2 + 24), v_y1); - } - } + opt_SSE4_1::WarpAffineInvoker_Blockline_SSE41(adelta + x, bdelta + x, xy, X0, Y0, bw); + else #endif for( ; x1 < bw; x1++ ) { @@ -6132,18 +5994,10 @@ public: int bw0 = std::min(BLOCK_SZ*BLOCK_SZ/bh0, width); bh0 = std::min(BLOCK_SZ*BLOCK_SZ/bw0, height); - #if CV_SSE4_1 - bool haveSSE4_1 = checkHardwareSupport(CV_CPU_SSE4_1); - __m128d v_M0 = _mm_set1_pd(M[0]); - __m128d v_M3 = _mm_set1_pd(M[3]); - __m128d v_M6 = _mm_set1_pd(M[6]); - __m128d v_intmax = _mm_set1_pd((double)INT_MAX); - __m128d v_intmin = _mm_set1_pd((double)INT_MIN); - __m128d v_2 = _mm_set1_pd(2), - v_zero = _mm_setzero_pd(), - v_1 = _mm_set1_pd(1), - v_its = _mm_set1_pd(INTER_TAB_SIZE); - __m128i v_itsi1 = _mm_set1_epi32(INTER_TAB_SIZE - 1); + #if CV_TRY_SSE4_1 + Ptr pwarp_impl_sse4; + if(CV_CPU_HAS_SUPPORT_SSE4_1) + pwarp_impl_sse4 = opt_SSE4_1::WarpPerspectiveLine_SSE4::getImpl(M); #endif for( y = range.start; y < range.end; y += bh0 ) @@ -6167,116 +6021,11 @@ public: { x1 = 0; - #if CV_SSE4_1 - if (haveSSE4_1) - { - __m128d v_X0d = _mm_set1_pd(X0); - __m128d v_Y0d = _mm_set1_pd(Y0); - __m128d v_W0 = _mm_set1_pd(W0); - __m128d v_x1 = _mm_set_pd(1, 0); - - for( ; x1 <= bw - 16; x1 += 16 ) - { - // 0-3 - __m128i v_X0, v_Y0; - { - __m128d v_W = _mm_add_pd(_mm_mul_pd(v_M6, v_x1), v_W0); - v_W = _mm_andnot_pd(_mm_cmpeq_pd(v_W, v_zero), _mm_div_pd(v_1, v_W)); - __m128d v_fX0 = _mm_max_pd(v_intmin, _mm_min_pd(v_intmax, _mm_mul_pd(_mm_add_pd(v_X0d, _mm_mul_pd(v_M0, v_x1)), v_W))); - __m128d v_fY0 = _mm_max_pd(v_intmin, _mm_min_pd(v_intmax, _mm_mul_pd(_mm_add_pd(v_Y0d, _mm_mul_pd(v_M3, v_x1)), v_W))); - v_x1 = _mm_add_pd(v_x1, v_2); - - v_W = _mm_add_pd(_mm_mul_pd(v_M6, v_x1), v_W0); - v_W = _mm_andnot_pd(_mm_cmpeq_pd(v_W, v_zero), _mm_div_pd(v_1, v_W)); - __m128d v_fX1 = _mm_max_pd(v_intmin, _mm_min_pd(v_intmax, _mm_mul_pd(_mm_add_pd(v_X0d, _mm_mul_pd(v_M0, v_x1)), v_W))); - __m128d v_fY1 = _mm_max_pd(v_intmin, _mm_min_pd(v_intmax, _mm_mul_pd(_mm_add_pd(v_Y0d, _mm_mul_pd(v_M3, v_x1)), v_W))); - v_x1 = _mm_add_pd(v_x1, v_2); - - v_X0 = _mm_castps_si128(_mm_movelh_ps(_mm_castsi128_ps(_mm_cvtpd_epi32(v_fX0)), - _mm_castsi128_ps(_mm_cvtpd_epi32(v_fX1)))); - v_Y0 = _mm_castps_si128(_mm_movelh_ps(_mm_castsi128_ps(_mm_cvtpd_epi32(v_fY0)), - _mm_castsi128_ps(_mm_cvtpd_epi32(v_fY1)))); - } - - // 4-8 - __m128i v_X1, v_Y1; - { - __m128d v_W = _mm_add_pd(_mm_mul_pd(v_M6, v_x1), v_W0); - v_W = _mm_andnot_pd(_mm_cmpeq_pd(v_W, v_zero), _mm_div_pd(v_1, v_W)); - __m128d v_fX0 = _mm_max_pd(v_intmin, _mm_min_pd(v_intmax, _mm_mul_pd(_mm_add_pd(v_X0d, _mm_mul_pd(v_M0, v_x1)), v_W))); - __m128d v_fY0 = _mm_max_pd(v_intmin, _mm_min_pd(v_intmax, _mm_mul_pd(_mm_add_pd(v_Y0d, _mm_mul_pd(v_M3, v_x1)), v_W))); - v_x1 = _mm_add_pd(v_x1, v_2); - - v_W = _mm_add_pd(_mm_mul_pd(v_M6, v_x1), v_W0); - v_W = _mm_andnot_pd(_mm_cmpeq_pd(v_W, v_zero), _mm_div_pd(v_1, v_W)); - __m128d v_fX1 = _mm_max_pd(v_intmin, _mm_min_pd(v_intmax, _mm_mul_pd(_mm_add_pd(v_X0d, _mm_mul_pd(v_M0, v_x1)), v_W))); - __m128d v_fY1 = _mm_max_pd(v_intmin, _mm_min_pd(v_intmax, _mm_mul_pd(_mm_add_pd(v_Y0d, _mm_mul_pd(v_M3, v_x1)), v_W))); - v_x1 = _mm_add_pd(v_x1, v_2); - - v_X1 = _mm_castps_si128(_mm_movelh_ps(_mm_castsi128_ps(_mm_cvtpd_epi32(v_fX0)), - _mm_castsi128_ps(_mm_cvtpd_epi32(v_fX1)))); - v_Y1 = _mm_castps_si128(_mm_movelh_ps(_mm_castsi128_ps(_mm_cvtpd_epi32(v_fY0)), - _mm_castsi128_ps(_mm_cvtpd_epi32(v_fY1)))); - } - - // 8-11 - __m128i v_X2, v_Y2; - { - __m128d v_W = _mm_add_pd(_mm_mul_pd(v_M6, v_x1), v_W0); - v_W = _mm_andnot_pd(_mm_cmpeq_pd(v_W, v_zero), _mm_div_pd(v_1, v_W)); - __m128d v_fX0 = _mm_max_pd(v_intmin, _mm_min_pd(v_intmax, _mm_mul_pd(_mm_add_pd(v_X0d, _mm_mul_pd(v_M0, v_x1)), v_W))); - __m128d v_fY0 = _mm_max_pd(v_intmin, _mm_min_pd(v_intmax, _mm_mul_pd(_mm_add_pd(v_Y0d, _mm_mul_pd(v_M3, v_x1)), v_W))); - v_x1 = _mm_add_pd(v_x1, v_2); - - v_W = _mm_add_pd(_mm_mul_pd(v_M6, v_x1), v_W0); - v_W = _mm_andnot_pd(_mm_cmpeq_pd(v_W, v_zero), _mm_div_pd(v_1, v_W)); - __m128d v_fX1 = _mm_max_pd(v_intmin, _mm_min_pd(v_intmax, _mm_mul_pd(_mm_add_pd(v_X0d, _mm_mul_pd(v_M0, v_x1)), v_W))); - __m128d v_fY1 = _mm_max_pd(v_intmin, _mm_min_pd(v_intmax, _mm_mul_pd(_mm_add_pd(v_Y0d, _mm_mul_pd(v_M3, v_x1)), v_W))); - v_x1 = _mm_add_pd(v_x1, v_2); - - v_X2 = _mm_castps_si128(_mm_movelh_ps(_mm_castsi128_ps(_mm_cvtpd_epi32(v_fX0)), - _mm_castsi128_ps(_mm_cvtpd_epi32(v_fX1)))); - v_Y2 = _mm_castps_si128(_mm_movelh_ps(_mm_castsi128_ps(_mm_cvtpd_epi32(v_fY0)), - _mm_castsi128_ps(_mm_cvtpd_epi32(v_fY1)))); - } - - // 12-15 - __m128i v_X3, v_Y3; - { - __m128d v_W = _mm_add_pd(_mm_mul_pd(v_M6, v_x1), v_W0); - v_W = _mm_andnot_pd(_mm_cmpeq_pd(v_W, v_zero), _mm_div_pd(v_1, v_W)); - __m128d v_fX0 = _mm_max_pd(v_intmin, _mm_min_pd(v_intmax, _mm_mul_pd(_mm_add_pd(v_X0d, _mm_mul_pd(v_M0, v_x1)), v_W))); - __m128d v_fY0 = _mm_max_pd(v_intmin, _mm_min_pd(v_intmax, _mm_mul_pd(_mm_add_pd(v_Y0d, _mm_mul_pd(v_M3, v_x1)), v_W))); - v_x1 = _mm_add_pd(v_x1, v_2); - - v_W = _mm_add_pd(_mm_mul_pd(v_M6, v_x1), v_W0); - v_W = _mm_andnot_pd(_mm_cmpeq_pd(v_W, v_zero), _mm_div_pd(v_1, v_W)); - __m128d v_fX1 = _mm_max_pd(v_intmin, _mm_min_pd(v_intmax, _mm_mul_pd(_mm_add_pd(v_X0d, _mm_mul_pd(v_M0, v_x1)), v_W))); - __m128d v_fY1 = _mm_max_pd(v_intmin, _mm_min_pd(v_intmax, _mm_mul_pd(_mm_add_pd(v_Y0d, _mm_mul_pd(v_M3, v_x1)), v_W))); - v_x1 = _mm_add_pd(v_x1, v_2); - - v_X3 = _mm_castps_si128(_mm_movelh_ps(_mm_castsi128_ps(_mm_cvtpd_epi32(v_fX0)), - _mm_castsi128_ps(_mm_cvtpd_epi32(v_fX1)))); - v_Y3 = _mm_castps_si128(_mm_movelh_ps(_mm_castsi128_ps(_mm_cvtpd_epi32(v_fY0)), - _mm_castsi128_ps(_mm_cvtpd_epi32(v_fY1)))); - } - - // convert to 16s - v_X0 = _mm_packs_epi32(v_X0, v_X1); - v_X1 = _mm_packs_epi32(v_X2, v_X3); - v_Y0 = _mm_packs_epi32(v_Y0, v_Y1); - v_Y1 = _mm_packs_epi32(v_Y2, v_Y3); - - _mm_interleave_epi16(v_X0, v_X1, v_Y0, v_Y1); - - _mm_storeu_si128((__m128i *)(xy + x1 * 2), v_X0); - _mm_storeu_si128((__m128i *)(xy + x1 * 2 + 8), v_X1); - _mm_storeu_si128((__m128i *)(xy + x1 * 2 + 16), v_Y0); - _mm_storeu_si128((__m128i *)(xy + x1 * 2 + 24), v_Y1); - } - } + #if CV_TRY_SSE4_1 + if (pwarp_impl_sse4) + pwarp_impl_sse4->processNN(M, xy, X0, Y0, W0, bw); + else #endif - for( ; x1 < bw; x1++ ) { double W = W0 + M[6]*x1; @@ -6295,129 +6044,11 @@ public: short* alpha = A + y1*bw; x1 = 0; - #if CV_SSE4_1 - if (haveSSE4_1) - { - __m128d v_X0d = _mm_set1_pd(X0); - __m128d v_Y0d = _mm_set1_pd(Y0); - __m128d v_W0 = _mm_set1_pd(W0); - __m128d v_x1 = _mm_set_pd(1, 0); - - for( ; x1 <= bw - 16; x1 += 16 ) - { - // 0-3 - __m128i v_X0, v_Y0; - { - __m128d v_W = _mm_add_pd(_mm_mul_pd(v_M6, v_x1), v_W0); - v_W = _mm_andnot_pd(_mm_cmpeq_pd(v_W, v_zero), _mm_div_pd(v_its, v_W)); - __m128d v_fX0 = _mm_max_pd(v_intmin, _mm_min_pd(v_intmax, _mm_mul_pd(_mm_add_pd(v_X0d, _mm_mul_pd(v_M0, v_x1)), v_W))); - __m128d v_fY0 = _mm_max_pd(v_intmin, _mm_min_pd(v_intmax, _mm_mul_pd(_mm_add_pd(v_Y0d, _mm_mul_pd(v_M3, v_x1)), v_W))); - v_x1 = _mm_add_pd(v_x1, v_2); - - v_W = _mm_add_pd(_mm_mul_pd(v_M6, v_x1), v_W0); - v_W = _mm_andnot_pd(_mm_cmpeq_pd(v_W, v_zero), _mm_div_pd(v_its, v_W)); - __m128d v_fX1 = _mm_max_pd(v_intmin, _mm_min_pd(v_intmax, _mm_mul_pd(_mm_add_pd(v_X0d, _mm_mul_pd(v_M0, v_x1)), v_W))); - __m128d v_fY1 = _mm_max_pd(v_intmin, _mm_min_pd(v_intmax, _mm_mul_pd(_mm_add_pd(v_Y0d, _mm_mul_pd(v_M3, v_x1)), v_W))); - v_x1 = _mm_add_pd(v_x1, v_2); - - v_X0 = _mm_castps_si128(_mm_movelh_ps(_mm_castsi128_ps(_mm_cvtpd_epi32(v_fX0)), - _mm_castsi128_ps(_mm_cvtpd_epi32(v_fX1)))); - v_Y0 = _mm_castps_si128(_mm_movelh_ps(_mm_castsi128_ps(_mm_cvtpd_epi32(v_fY0)), - _mm_castsi128_ps(_mm_cvtpd_epi32(v_fY1)))); - } - - // 4-8 - __m128i v_X1, v_Y1; - { - __m128d v_W = _mm_add_pd(_mm_mul_pd(v_M6, v_x1), v_W0); - v_W = _mm_andnot_pd(_mm_cmpeq_pd(v_W, v_zero), _mm_div_pd(v_its, v_W)); - __m128d v_fX0 = _mm_max_pd(v_intmin, _mm_min_pd(v_intmax, _mm_mul_pd(_mm_add_pd(v_X0d, _mm_mul_pd(v_M0, v_x1)), v_W))); - __m128d v_fY0 = _mm_max_pd(v_intmin, _mm_min_pd(v_intmax, _mm_mul_pd(_mm_add_pd(v_Y0d, _mm_mul_pd(v_M3, v_x1)), v_W))); - v_x1 = _mm_add_pd(v_x1, v_2); - - v_W = _mm_add_pd(_mm_mul_pd(v_M6, v_x1), v_W0); - v_W = _mm_andnot_pd(_mm_cmpeq_pd(v_W, v_zero), _mm_div_pd(v_its, v_W)); - __m128d v_fX1 = _mm_max_pd(v_intmin, _mm_min_pd(v_intmax, _mm_mul_pd(_mm_add_pd(v_X0d, _mm_mul_pd(v_M0, v_x1)), v_W))); - __m128d v_fY1 = _mm_max_pd(v_intmin, _mm_min_pd(v_intmax, _mm_mul_pd(_mm_add_pd(v_Y0d, _mm_mul_pd(v_M3, v_x1)), v_W))); - v_x1 = _mm_add_pd(v_x1, v_2); - - v_X1 = _mm_castps_si128(_mm_movelh_ps(_mm_castsi128_ps(_mm_cvtpd_epi32(v_fX0)), - _mm_castsi128_ps(_mm_cvtpd_epi32(v_fX1)))); - v_Y1 = _mm_castps_si128(_mm_movelh_ps(_mm_castsi128_ps(_mm_cvtpd_epi32(v_fY0)), - _mm_castsi128_ps(_mm_cvtpd_epi32(v_fY1)))); - } - - // 8-11 - __m128i v_X2, v_Y2; - { - __m128d v_W = _mm_add_pd(_mm_mul_pd(v_M6, v_x1), v_W0); - v_W = _mm_andnot_pd(_mm_cmpeq_pd(v_W, v_zero), _mm_div_pd(v_its, v_W)); - __m128d v_fX0 = _mm_max_pd(v_intmin, _mm_min_pd(v_intmax, _mm_mul_pd(_mm_add_pd(v_X0d, _mm_mul_pd(v_M0, v_x1)), v_W))); - __m128d v_fY0 = _mm_max_pd(v_intmin, _mm_min_pd(v_intmax, _mm_mul_pd(_mm_add_pd(v_Y0d, _mm_mul_pd(v_M3, v_x1)), v_W))); - v_x1 = _mm_add_pd(v_x1, v_2); - - v_W = _mm_add_pd(_mm_mul_pd(v_M6, v_x1), v_W0); - v_W = _mm_andnot_pd(_mm_cmpeq_pd(v_W, v_zero), _mm_div_pd(v_its, v_W)); - __m128d v_fX1 = _mm_max_pd(v_intmin, _mm_min_pd(v_intmax, _mm_mul_pd(_mm_add_pd(v_X0d, _mm_mul_pd(v_M0, v_x1)), v_W))); - __m128d v_fY1 = _mm_max_pd(v_intmin, _mm_min_pd(v_intmax, _mm_mul_pd(_mm_add_pd(v_Y0d, _mm_mul_pd(v_M3, v_x1)), v_W))); - v_x1 = _mm_add_pd(v_x1, v_2); - - v_X2 = _mm_castps_si128(_mm_movelh_ps(_mm_castsi128_ps(_mm_cvtpd_epi32(v_fX0)), - _mm_castsi128_ps(_mm_cvtpd_epi32(v_fX1)))); - v_Y2 = _mm_castps_si128(_mm_movelh_ps(_mm_castsi128_ps(_mm_cvtpd_epi32(v_fY0)), - _mm_castsi128_ps(_mm_cvtpd_epi32(v_fY1)))); - } - - // 12-15 - __m128i v_X3, v_Y3; - { - __m128d v_W = _mm_add_pd(_mm_mul_pd(v_M6, v_x1), v_W0); - v_W = _mm_andnot_pd(_mm_cmpeq_pd(v_W, v_zero), _mm_div_pd(v_its, v_W)); - __m128d v_fX0 = _mm_max_pd(v_intmin, _mm_min_pd(v_intmax, _mm_mul_pd(_mm_add_pd(v_X0d, _mm_mul_pd(v_M0, v_x1)), v_W))); - __m128d v_fY0 = _mm_max_pd(v_intmin, _mm_min_pd(v_intmax, _mm_mul_pd(_mm_add_pd(v_Y0d, _mm_mul_pd(v_M3, v_x1)), v_W))); - v_x1 = _mm_add_pd(v_x1, v_2); - - v_W = _mm_add_pd(_mm_mul_pd(v_M6, v_x1), v_W0); - v_W = _mm_andnot_pd(_mm_cmpeq_pd(v_W, v_zero), _mm_div_pd(v_its, v_W)); - __m128d v_fX1 = _mm_max_pd(v_intmin, _mm_min_pd(v_intmax, _mm_mul_pd(_mm_add_pd(v_X0d, _mm_mul_pd(v_M0, v_x1)), v_W))); - __m128d v_fY1 = _mm_max_pd(v_intmin, _mm_min_pd(v_intmax, _mm_mul_pd(_mm_add_pd(v_Y0d, _mm_mul_pd(v_M3, v_x1)), v_W))); - v_x1 = _mm_add_pd(v_x1, v_2); - - v_X3 = _mm_castps_si128(_mm_movelh_ps(_mm_castsi128_ps(_mm_cvtpd_epi32(v_fX0)), - _mm_castsi128_ps(_mm_cvtpd_epi32(v_fX1)))); - v_Y3 = _mm_castps_si128(_mm_movelh_ps(_mm_castsi128_ps(_mm_cvtpd_epi32(v_fY0)), - _mm_castsi128_ps(_mm_cvtpd_epi32(v_fY1)))); - } - - // store alpha - __m128i v_alpha0 = _mm_add_epi32(_mm_slli_epi32(_mm_and_si128(v_Y0, v_itsi1), INTER_BITS), - _mm_and_si128(v_X0, v_itsi1)); - __m128i v_alpha1 = _mm_add_epi32(_mm_slli_epi32(_mm_and_si128(v_Y1, v_itsi1), INTER_BITS), - _mm_and_si128(v_X1, v_itsi1)); - _mm_storeu_si128((__m128i *)(alpha + x1), _mm_packs_epi32(v_alpha0, v_alpha1)); - - v_alpha0 = _mm_add_epi32(_mm_slli_epi32(_mm_and_si128(v_Y2, v_itsi1), INTER_BITS), - _mm_and_si128(v_X2, v_itsi1)); - v_alpha1 = _mm_add_epi32(_mm_slli_epi32(_mm_and_si128(v_Y3, v_itsi1), INTER_BITS), - _mm_and_si128(v_X3, v_itsi1)); - _mm_storeu_si128((__m128i *)(alpha + x1 + 8), _mm_packs_epi32(v_alpha0, v_alpha1)); - - // convert to 16s - v_X0 = _mm_packs_epi32(_mm_srai_epi32(v_X0, INTER_BITS), _mm_srai_epi32(v_X1, INTER_BITS)); - v_X1 = _mm_packs_epi32(_mm_srai_epi32(v_X2, INTER_BITS), _mm_srai_epi32(v_X3, INTER_BITS)); - v_Y0 = _mm_packs_epi32(_mm_srai_epi32(v_Y0, INTER_BITS), _mm_srai_epi32(v_Y1, INTER_BITS)); - v_Y1 = _mm_packs_epi32(_mm_srai_epi32(v_Y2, INTER_BITS), _mm_srai_epi32(v_Y3, INTER_BITS)); - - _mm_interleave_epi16(v_X0, v_X1, v_Y0, v_Y1); - - _mm_storeu_si128((__m128i *)(xy + x1 * 2), v_X0); - _mm_storeu_si128((__m128i *)(xy + x1 * 2 + 8), v_X1); - _mm_storeu_si128((__m128i *)(xy + x1 * 2 + 16), v_Y0); - _mm_storeu_si128((__m128i *)(xy + x1 * 2 + 24), v_Y1); - } - } + #if CV_TRY_SSE4_1 + if (pwarp_impl_sse4) + pwarp_impl_sse4->process(M, xy, alpha, X0, Y0, W0, bw); + else #endif - for( ; x1 < bw; x1++ ) { double W = W0 + M[6]*x1; diff --git a/modules/imgproc/src/imgwarp.hpp b/modules/imgproc/src/imgwarp.hpp index 428bcc51e7..ed1146dcd3 100644 --- a/modules/imgproc/src/imgwarp.hpp +++ b/modules/imgproc/src/imgwarp.hpp @@ -61,11 +61,27 @@ void resizeNN4_AVX2(const Range&, const Mat&, Mat&, int*, int, double); int warpAffineBlockline(int *adelta, int *bdelta, short* xy, short* alpha, int X0, int Y0, int bw); #endif } -namespace opt_SSE41 + +namespace opt_SSE4_1 { #if CV_TRY_SSE4_1 void resizeNN2_SSE4_1(const Range&, const Mat&, Mat&, int*, int, double); void resizeNN4_SSE4_1(const Range&, const Mat&, Mat&, int*, int, double); + +int VResizeLanczos4Vec_32f16u_SSE41(const uchar** _src, uchar* _dst, const uchar* _beta, int width); +void convertMaps_nninterpolate32f1c16s_SSE41(const float* src1f, const float* src2f, short* dst1, int width); +void convertMaps_32f1c16s_SSE41(const float* src1f, const float* src2f, short* dst1, ushort* dst2, int width); +void convertMaps_32f2c16s_SSE41(const float* src1f, short* dst1, ushort* dst2, int width); +void WarpAffineInvoker_Blockline_SSE41(int *adelta, int *bdelta, short* xy, int X0, int Y0, int bw); + +class WarpPerspectiveLine_SSE4 +{ +public: + static Ptr getImpl(const double *M); + virtual void processNN(const double *M, short* xy, double X0, double Y0, double W0, int bw) = 0; + virtual void process(const double *M, short* xy, short* alpha, double X0, double Y0, double W0, int bw) = 0; + virtual ~WarpPerspectiveLine_SSE4() {}; +}; #endif } } diff --git a/modules/imgproc/src/imgwarp.sse4_1.cpp b/modules/imgproc/src/imgwarp.sse4_1.cpp index b326ffcb1a..79137d1230 100644 --- a/modules/imgproc/src/imgwarp.sse4_1.cpp +++ b/modules/imgproc/src/imgwarp.sse4_1.cpp @@ -52,7 +52,7 @@ namespace cv { -namespace opt_SSE41 +namespace opt_SSE4_1 { class resizeNNInvokerSSE2 : @@ -186,7 +186,494 @@ void resizeNN4_SSE4_1(const Range& range, const Mat& src, Mat &dst, int *x_ofs, parallel_for_(range, invoker, dst.total() / (double)(1 << 16)); } -} +int VResizeLanczos4Vec_32f16u_SSE41(const uchar** _src, uchar* _dst, const uchar* _beta, int width) +{ + const float** src = (const float**)_src; + const float* beta = (const float*)_beta; + const float *S0 = src[0], *S1 = src[1], *S2 = src[2], *S3 = src[3], + *S4 = src[4], *S5 = src[5], *S6 = src[6], *S7 = src[7]; + short * dst = (short*)_dst; + int x = 0; + __m128 v_b0 = _mm_set1_ps(beta[0]), v_b1 = _mm_set1_ps(beta[1]), + v_b2 = _mm_set1_ps(beta[2]), v_b3 = _mm_set1_ps(beta[3]), + v_b4 = _mm_set1_ps(beta[4]), v_b5 = _mm_set1_ps(beta[5]), + v_b6 = _mm_set1_ps(beta[6]), v_b7 = _mm_set1_ps(beta[7]); + + for (; x <= width - 8; x += 8) + { + __m128 v_dst0 = _mm_mul_ps(v_b0, _mm_loadu_ps(S0 + x)); + v_dst0 = _mm_add_ps(v_dst0, _mm_mul_ps(v_b1, _mm_loadu_ps(S1 + x))); + v_dst0 = _mm_add_ps(v_dst0, _mm_mul_ps(v_b2, _mm_loadu_ps(S2 + x))); + v_dst0 = _mm_add_ps(v_dst0, _mm_mul_ps(v_b3, _mm_loadu_ps(S3 + x))); + v_dst0 = _mm_add_ps(v_dst0, _mm_mul_ps(v_b4, _mm_loadu_ps(S4 + x))); + v_dst0 = _mm_add_ps(v_dst0, _mm_mul_ps(v_b5, _mm_loadu_ps(S5 + x))); + v_dst0 = _mm_add_ps(v_dst0, _mm_mul_ps(v_b6, _mm_loadu_ps(S6 + x))); + v_dst0 = _mm_add_ps(v_dst0, _mm_mul_ps(v_b7, _mm_loadu_ps(S7 + x))); + + __m128 v_dst1 = _mm_mul_ps(v_b0, _mm_loadu_ps(S0 + x + 4)); + v_dst1 = _mm_add_ps(v_dst1, _mm_mul_ps(v_b1, _mm_loadu_ps(S1 + x + 4))); + v_dst1 = _mm_add_ps(v_dst1, _mm_mul_ps(v_b2, _mm_loadu_ps(S2 + x + 4))); + v_dst1 = _mm_add_ps(v_dst1, _mm_mul_ps(v_b3, _mm_loadu_ps(S3 + x + 4))); + v_dst1 = _mm_add_ps(v_dst1, _mm_mul_ps(v_b4, _mm_loadu_ps(S4 + x + 4))); + v_dst1 = _mm_add_ps(v_dst1, _mm_mul_ps(v_b5, _mm_loadu_ps(S5 + x + 4))); + v_dst1 = _mm_add_ps(v_dst1, _mm_mul_ps(v_b6, _mm_loadu_ps(S6 + x + 4))); + v_dst1 = _mm_add_ps(v_dst1, _mm_mul_ps(v_b7, _mm_loadu_ps(S7 + x + 4))); + + __m128i v_dsti0 = _mm_cvtps_epi32(v_dst0); + __m128i v_dsti1 = _mm_cvtps_epi32(v_dst1); + + _mm_storeu_si128((__m128i *)(dst + x), _mm_packus_epi32(v_dsti0, v_dsti1)); + } + + return x; } +void convertMaps_nninterpolate32f1c16s_SSE41(const float* src1f, const float* src2f, short* dst1, int width) +{ + int x = 0; + for (; x <= width - 16; x += 16) + { + __m128i v_dst0 = _mm_packs_epi32(_mm_cvtps_epi32(_mm_loadu_ps(src1f + x)), + _mm_cvtps_epi32(_mm_loadu_ps(src1f + x + 4))); + __m128i v_dst1 = _mm_packs_epi32(_mm_cvtps_epi32(_mm_loadu_ps(src1f + x + 8)), + _mm_cvtps_epi32(_mm_loadu_ps(src1f + x + 12))); + + __m128i v_dst2 = _mm_packs_epi32(_mm_cvtps_epi32(_mm_loadu_ps(src2f + x)), + _mm_cvtps_epi32(_mm_loadu_ps(src2f + x + 4))); + __m128i v_dst3 = _mm_packs_epi32(_mm_cvtps_epi32(_mm_loadu_ps(src2f + x + 8)), + _mm_cvtps_epi32(_mm_loadu_ps(src2f + x + 12))); + + _mm_interleave_epi16(v_dst0, v_dst1, v_dst2, v_dst3); + + _mm_storeu_si128((__m128i *)(dst1 + x * 2), v_dst0); + _mm_storeu_si128((__m128i *)(dst1 + x * 2 + 8), v_dst1); + _mm_storeu_si128((__m128i *)(dst1 + x * 2 + 16), v_dst2); + _mm_storeu_si128((__m128i *)(dst1 + x * 2 + 24), v_dst3); + } + + for (; x < width; x++) + { + dst1[x * 2] = saturate_cast(src1f[x]); + dst1[x * 2 + 1] = saturate_cast(src2f[x]); + } +} + +void convertMaps_32f1c16s_SSE41(const float* src1f, const float* src2f, short* dst1, ushort* dst2, int width) +{ + int x = 0; + __m128 v_its = _mm_set1_ps(INTER_TAB_SIZE); + __m128i v_its1 = _mm_set1_epi32(INTER_TAB_SIZE - 1); + + for (; x <= width - 16; x += 16) + { + __m128i v_ix0 = _mm_cvtps_epi32(_mm_mul_ps(_mm_loadu_ps(src1f + x), v_its)); + __m128i v_ix1 = _mm_cvtps_epi32(_mm_mul_ps(_mm_loadu_ps(src1f + x + 4), v_its)); + __m128i v_iy0 = _mm_cvtps_epi32(_mm_mul_ps(_mm_loadu_ps(src2f + x), v_its)); + __m128i v_iy1 = _mm_cvtps_epi32(_mm_mul_ps(_mm_loadu_ps(src2f + x + 4), v_its)); + + __m128i v_dst10 = _mm_packs_epi32(_mm_srai_epi32(v_ix0, INTER_BITS), + _mm_srai_epi32(v_ix1, INTER_BITS)); + __m128i v_dst12 = _mm_packs_epi32(_mm_srai_epi32(v_iy0, INTER_BITS), + _mm_srai_epi32(v_iy1, INTER_BITS)); + __m128i v_dst20 = _mm_add_epi32(_mm_slli_epi32(_mm_and_si128(v_iy0, v_its1), INTER_BITS), + _mm_and_si128(v_ix0, v_its1)); + __m128i v_dst21 = _mm_add_epi32(_mm_slli_epi32(_mm_and_si128(v_iy1, v_its1), INTER_BITS), + _mm_and_si128(v_ix1, v_its1)); + _mm_storeu_si128((__m128i *)(dst2 + x), _mm_packus_epi32(v_dst20, v_dst21)); + + v_ix0 = _mm_cvtps_epi32(_mm_mul_ps(_mm_loadu_ps(src1f + x + 8), v_its)); + v_ix1 = _mm_cvtps_epi32(_mm_mul_ps(_mm_loadu_ps(src1f + x + 12), v_its)); + v_iy0 = _mm_cvtps_epi32(_mm_mul_ps(_mm_loadu_ps(src2f + x + 8), v_its)); + v_iy1 = _mm_cvtps_epi32(_mm_mul_ps(_mm_loadu_ps(src2f + x + 12), v_its)); + + __m128i v_dst11 = _mm_packs_epi32(_mm_srai_epi32(v_ix0, INTER_BITS), + _mm_srai_epi32(v_ix1, INTER_BITS)); + __m128i v_dst13 = _mm_packs_epi32(_mm_srai_epi32(v_iy0, INTER_BITS), + _mm_srai_epi32(v_iy1, INTER_BITS)); + v_dst20 = _mm_add_epi32(_mm_slli_epi32(_mm_and_si128(v_iy0, v_its1), INTER_BITS), + _mm_and_si128(v_ix0, v_its1)); + v_dst21 = _mm_add_epi32(_mm_slli_epi32(_mm_and_si128(v_iy1, v_its1), INTER_BITS), + _mm_and_si128(v_ix1, v_its1)); + _mm_storeu_si128((__m128i *)(dst2 + x + 8), _mm_packus_epi32(v_dst20, v_dst21)); + + _mm_interleave_epi16(v_dst10, v_dst11, v_dst12, v_dst13); + + _mm_storeu_si128((__m128i *)(dst1 + x * 2), v_dst10); + _mm_storeu_si128((__m128i *)(dst1 + x * 2 + 8), v_dst11); + _mm_storeu_si128((__m128i *)(dst1 + x * 2 + 16), v_dst12); + _mm_storeu_si128((__m128i *)(dst1 + x * 2 + 24), v_dst13); + } + for (; x < width; x++) + { + int ix = saturate_cast(src1f[x] * INTER_TAB_SIZE); + int iy = saturate_cast(src2f[x] * INTER_TAB_SIZE); + dst1[x * 2] = saturate_cast(ix >> INTER_BITS); + dst1[x * 2 + 1] = saturate_cast(iy >> INTER_BITS); + dst2[x] = (ushort)((iy & (INTER_TAB_SIZE - 1))*INTER_TAB_SIZE + (ix & (INTER_TAB_SIZE - 1))); + } +} + +void convertMaps_32f2c16s_SSE41(const float* src1f, short* dst1, ushort* dst2, int width) +{ + int x = 0; + __m128 v_its = _mm_set1_ps(INTER_TAB_SIZE); + __m128i v_its1 = _mm_set1_epi32(INTER_TAB_SIZE - 1); + __m128i v_y_mask = _mm_set1_epi32((INTER_TAB_SIZE - 1) << 16); + + for (; x <= width - 4; x += 4) + { + __m128i v_src0 = _mm_cvtps_epi32(_mm_mul_ps(_mm_loadu_ps(src1f + x * 2), v_its)); + __m128i v_src1 = _mm_cvtps_epi32(_mm_mul_ps(_mm_loadu_ps(src1f + x * 2 + 4), v_its)); + + __m128i v_dst1 = _mm_packs_epi32(_mm_srai_epi32(v_src0, INTER_BITS), + _mm_srai_epi32(v_src1, INTER_BITS)); + _mm_storeu_si128((__m128i *)(dst1 + x * 2), v_dst1); + + // x0 y0 x1 y1 . . . + v_src0 = _mm_packs_epi32(_mm_and_si128(v_src0, v_its1), + _mm_and_si128(v_src1, v_its1)); + __m128i v_dst2 = _mm_or_si128(_mm_srli_epi32(_mm_and_si128(v_src0, v_y_mask), 16 - INTER_BITS), // y0 0 y1 0 . . . + _mm_and_si128(v_src0, v_its1)); // 0 x0 0 x1 . . . + _mm_storel_epi64((__m128i *)(dst2 + x), _mm_packus_epi32(v_dst2, v_dst2)); + } + for (; x < width; x++) + { + int ix = saturate_cast(src1f[x * 2] * INTER_TAB_SIZE); + int iy = saturate_cast(src1f[x * 2 + 1] * INTER_TAB_SIZE); + dst1[x * 2] = saturate_cast(ix >> INTER_BITS); + dst1[x * 2 + 1] = saturate_cast(iy >> INTER_BITS); + dst2[x] = (ushort)((iy & (INTER_TAB_SIZE - 1))*INTER_TAB_SIZE + (ix & (INTER_TAB_SIZE - 1))); + } +} + +void WarpAffineInvoker_Blockline_SSE41(int *adelta, int *bdelta, short* xy, int X0, int Y0, int bw) +{ + const int AB_BITS = MAX(10, (int)INTER_BITS); + int x1 = 0; + + __m128i v_X0 = _mm_set1_epi32(X0); + __m128i v_Y0 = _mm_set1_epi32(Y0); + for (; x1 <= bw - 16; x1 += 16) + { + __m128i v_x0 = _mm_packs_epi32(_mm_srai_epi32(_mm_add_epi32(v_X0, _mm_loadu_si128((__m128i const *)(adelta + x1))), AB_BITS), + _mm_srai_epi32(_mm_add_epi32(v_X0, _mm_loadu_si128((__m128i const *)(adelta + x1 + 4))), AB_BITS)); + __m128i v_x1 = _mm_packs_epi32(_mm_srai_epi32(_mm_add_epi32(v_X0, _mm_loadu_si128((__m128i const *)(adelta + x1 + 8))), AB_BITS), + _mm_srai_epi32(_mm_add_epi32(v_X0, _mm_loadu_si128((__m128i const *)(adelta + x1 + 12))), AB_BITS)); + + __m128i v_y0 = _mm_packs_epi32(_mm_srai_epi32(_mm_add_epi32(v_Y0, _mm_loadu_si128((__m128i const *)(bdelta + x1))), AB_BITS), + _mm_srai_epi32(_mm_add_epi32(v_Y0, _mm_loadu_si128((__m128i const *)(bdelta + x1 + 4))), AB_BITS)); + __m128i v_y1 = _mm_packs_epi32(_mm_srai_epi32(_mm_add_epi32(v_Y0, _mm_loadu_si128((__m128i const *)(bdelta + x1 + 8))), AB_BITS), + _mm_srai_epi32(_mm_add_epi32(v_Y0, _mm_loadu_si128((__m128i const *)(bdelta + x1 + 12))), AB_BITS)); + + _mm_interleave_epi16(v_x0, v_x1, v_y0, v_y1); + + _mm_storeu_si128((__m128i *)(xy + x1 * 2), v_x0); + _mm_storeu_si128((__m128i *)(xy + x1 * 2 + 8), v_x1); + _mm_storeu_si128((__m128i *)(xy + x1 * 2 + 16), v_y0); + _mm_storeu_si128((__m128i *)(xy + x1 * 2 + 24), v_y1); + } + for (; x1 < bw; x1++) + { + int X = (X0 + adelta[x1]) >> AB_BITS; + int Y = (Y0 + bdelta[x1]) >> AB_BITS; + xy[x1 * 2] = saturate_cast(X); + xy[x1 * 2 + 1] = saturate_cast(Y); + } +} + + +class WarpPerspectiveLine_SSE4_Impl: public WarpPerspectiveLine_SSE4 +{ +public: + WarpPerspectiveLine_SSE4_Impl(const double *M) + { + v_M0 = _mm_set1_pd(M[0]); + v_M3 = _mm_set1_pd(M[3]); + v_M6 = _mm_set1_pd(M[6]); + v_intmax = _mm_set1_pd((double)INT_MAX); + v_intmin = _mm_set1_pd((double)INT_MIN); + v_2 = _mm_set1_pd(2); + v_zero = _mm_setzero_pd(); + v_1 = _mm_set1_pd(1); + v_its = _mm_set1_pd(INTER_TAB_SIZE); + v_itsi1 = _mm_set1_epi32(INTER_TAB_SIZE - 1); + } + virtual void processNN(const double *M, short* xy, double X0, double Y0, double W0, int bw) + { + int x1 = 0; + __m128d v_X0d = _mm_set1_pd(X0); + __m128d v_Y0d = _mm_set1_pd(Y0); + __m128d v_W0 = _mm_set1_pd(W0); + __m128d v_x1 = _mm_set_pd(1, 0); + + for (; x1 <= bw - 16; x1 += 16) + { + // 0-3 + __m128i v_X0, v_Y0; + { + __m128d v_W = _mm_add_pd(_mm_mul_pd(v_M6, v_x1), v_W0); + v_W = _mm_andnot_pd(_mm_cmpeq_pd(v_W, v_zero), _mm_div_pd(v_1, v_W)); + __m128d v_fX0 = _mm_max_pd(v_intmin, _mm_min_pd(v_intmax, _mm_mul_pd(_mm_add_pd(v_X0d, _mm_mul_pd(v_M0, v_x1)), v_W))); + __m128d v_fY0 = _mm_max_pd(v_intmin, _mm_min_pd(v_intmax, _mm_mul_pd(_mm_add_pd(v_Y0d, _mm_mul_pd(v_M3, v_x1)), v_W))); + v_x1 = _mm_add_pd(v_x1, v_2); + + v_W = _mm_add_pd(_mm_mul_pd(v_M6, v_x1), v_W0); + v_W = _mm_andnot_pd(_mm_cmpeq_pd(v_W, v_zero), _mm_div_pd(v_1, v_W)); + __m128d v_fX1 = _mm_max_pd(v_intmin, _mm_min_pd(v_intmax, _mm_mul_pd(_mm_add_pd(v_X0d, _mm_mul_pd(v_M0, v_x1)), v_W))); + __m128d v_fY1 = _mm_max_pd(v_intmin, _mm_min_pd(v_intmax, _mm_mul_pd(_mm_add_pd(v_Y0d, _mm_mul_pd(v_M3, v_x1)), v_W))); + v_x1 = _mm_add_pd(v_x1, v_2); + + v_X0 = _mm_castps_si128(_mm_movelh_ps(_mm_castsi128_ps(_mm_cvtpd_epi32(v_fX0)), + _mm_castsi128_ps(_mm_cvtpd_epi32(v_fX1)))); + v_Y0 = _mm_castps_si128(_mm_movelh_ps(_mm_castsi128_ps(_mm_cvtpd_epi32(v_fY0)), + _mm_castsi128_ps(_mm_cvtpd_epi32(v_fY1)))); + } + + // 4-8 + __m128i v_X1, v_Y1; + { + __m128d v_W = _mm_add_pd(_mm_mul_pd(v_M6, v_x1), v_W0); + v_W = _mm_andnot_pd(_mm_cmpeq_pd(v_W, v_zero), _mm_div_pd(v_1, v_W)); + __m128d v_fX0 = _mm_max_pd(v_intmin, _mm_min_pd(v_intmax, _mm_mul_pd(_mm_add_pd(v_X0d, _mm_mul_pd(v_M0, v_x1)), v_W))); + __m128d v_fY0 = _mm_max_pd(v_intmin, _mm_min_pd(v_intmax, _mm_mul_pd(_mm_add_pd(v_Y0d, _mm_mul_pd(v_M3, v_x1)), v_W))); + v_x1 = _mm_add_pd(v_x1, v_2); + + v_W = _mm_add_pd(_mm_mul_pd(v_M6, v_x1), v_W0); + v_W = _mm_andnot_pd(_mm_cmpeq_pd(v_W, v_zero), _mm_div_pd(v_1, v_W)); + __m128d v_fX1 = _mm_max_pd(v_intmin, _mm_min_pd(v_intmax, _mm_mul_pd(_mm_add_pd(v_X0d, _mm_mul_pd(v_M0, v_x1)), v_W))); + __m128d v_fY1 = _mm_max_pd(v_intmin, _mm_min_pd(v_intmax, _mm_mul_pd(_mm_add_pd(v_Y0d, _mm_mul_pd(v_M3, v_x1)), v_W))); + v_x1 = _mm_add_pd(v_x1, v_2); + + v_X1 = _mm_castps_si128(_mm_movelh_ps(_mm_castsi128_ps(_mm_cvtpd_epi32(v_fX0)), + _mm_castsi128_ps(_mm_cvtpd_epi32(v_fX1)))); + v_Y1 = _mm_castps_si128(_mm_movelh_ps(_mm_castsi128_ps(_mm_cvtpd_epi32(v_fY0)), + _mm_castsi128_ps(_mm_cvtpd_epi32(v_fY1)))); + } + + // 8-11 + __m128i v_X2, v_Y2; + { + __m128d v_W = _mm_add_pd(_mm_mul_pd(v_M6, v_x1), v_W0); + v_W = _mm_andnot_pd(_mm_cmpeq_pd(v_W, v_zero), _mm_div_pd(v_1, v_W)); + __m128d v_fX0 = _mm_max_pd(v_intmin, _mm_min_pd(v_intmax, _mm_mul_pd(_mm_add_pd(v_X0d, _mm_mul_pd(v_M0, v_x1)), v_W))); + __m128d v_fY0 = _mm_max_pd(v_intmin, _mm_min_pd(v_intmax, _mm_mul_pd(_mm_add_pd(v_Y0d, _mm_mul_pd(v_M3, v_x1)), v_W))); + v_x1 = _mm_add_pd(v_x1, v_2); + + v_W = _mm_add_pd(_mm_mul_pd(v_M6, v_x1), v_W0); + v_W = _mm_andnot_pd(_mm_cmpeq_pd(v_W, v_zero), _mm_div_pd(v_1, v_W)); + __m128d v_fX1 = _mm_max_pd(v_intmin, _mm_min_pd(v_intmax, _mm_mul_pd(_mm_add_pd(v_X0d, _mm_mul_pd(v_M0, v_x1)), v_W))); + __m128d v_fY1 = _mm_max_pd(v_intmin, _mm_min_pd(v_intmax, _mm_mul_pd(_mm_add_pd(v_Y0d, _mm_mul_pd(v_M3, v_x1)), v_W))); + v_x1 = _mm_add_pd(v_x1, v_2); + + v_X2 = _mm_castps_si128(_mm_movelh_ps(_mm_castsi128_ps(_mm_cvtpd_epi32(v_fX0)), + _mm_castsi128_ps(_mm_cvtpd_epi32(v_fX1)))); + v_Y2 = _mm_castps_si128(_mm_movelh_ps(_mm_castsi128_ps(_mm_cvtpd_epi32(v_fY0)), + _mm_castsi128_ps(_mm_cvtpd_epi32(v_fY1)))); + } + + // 12-15 + __m128i v_X3, v_Y3; + { + __m128d v_W = _mm_add_pd(_mm_mul_pd(v_M6, v_x1), v_W0); + v_W = _mm_andnot_pd(_mm_cmpeq_pd(v_W, v_zero), _mm_div_pd(v_1, v_W)); + __m128d v_fX0 = _mm_max_pd(v_intmin, _mm_min_pd(v_intmax, _mm_mul_pd(_mm_add_pd(v_X0d, _mm_mul_pd(v_M0, v_x1)), v_W))); + __m128d v_fY0 = _mm_max_pd(v_intmin, _mm_min_pd(v_intmax, _mm_mul_pd(_mm_add_pd(v_Y0d, _mm_mul_pd(v_M3, v_x1)), v_W))); + v_x1 = _mm_add_pd(v_x1, v_2); + + v_W = _mm_add_pd(_mm_mul_pd(v_M6, v_x1), v_W0); + v_W = _mm_andnot_pd(_mm_cmpeq_pd(v_W, v_zero), _mm_div_pd(v_1, v_W)); + __m128d v_fX1 = _mm_max_pd(v_intmin, _mm_min_pd(v_intmax, _mm_mul_pd(_mm_add_pd(v_X0d, _mm_mul_pd(v_M0, v_x1)), v_W))); + __m128d v_fY1 = _mm_max_pd(v_intmin, _mm_min_pd(v_intmax, _mm_mul_pd(_mm_add_pd(v_Y0d, _mm_mul_pd(v_M3, v_x1)), v_W))); + v_x1 = _mm_add_pd(v_x1, v_2); + + v_X3 = _mm_castps_si128(_mm_movelh_ps(_mm_castsi128_ps(_mm_cvtpd_epi32(v_fX0)), + _mm_castsi128_ps(_mm_cvtpd_epi32(v_fX1)))); + v_Y3 = _mm_castps_si128(_mm_movelh_ps(_mm_castsi128_ps(_mm_cvtpd_epi32(v_fY0)), + _mm_castsi128_ps(_mm_cvtpd_epi32(v_fY1)))); + } + + // convert to 16s + v_X0 = _mm_packs_epi32(v_X0, v_X1); + v_X1 = _mm_packs_epi32(v_X2, v_X3); + v_Y0 = _mm_packs_epi32(v_Y0, v_Y1); + v_Y1 = _mm_packs_epi32(v_Y2, v_Y3); + + _mm_interleave_epi16(v_X0, v_X1, v_Y0, v_Y1); + + _mm_storeu_si128((__m128i *)(xy + x1 * 2), v_X0); + _mm_storeu_si128((__m128i *)(xy + x1 * 2 + 8), v_X1); + _mm_storeu_si128((__m128i *)(xy + x1 * 2 + 16), v_Y0); + _mm_storeu_si128((__m128i *)(xy + x1 * 2 + 24), v_Y1); + } + + for (; x1 < bw; x1++) + { + double W = W0 + M[6] * x1; + W = W ? 1. / W : 0; + double fX = std::max((double)INT_MIN, std::min((double)INT_MAX, (X0 + M[0] * x1)*W)); + double fY = std::max((double)INT_MIN, std::min((double)INT_MAX, (Y0 + M[3] * x1)*W)); + int X = saturate_cast(fX); + int Y = saturate_cast(fY); + + xy[x1 * 2] = saturate_cast(X); + xy[x1 * 2 + 1] = saturate_cast(Y); + } + } + virtual void process(const double *M, short* xy, short* alpha, double X0, double Y0, double W0, int bw) + { + int x1 = 0; + + __m128d v_X0d = _mm_set1_pd(X0); + __m128d v_Y0d = _mm_set1_pd(Y0); + __m128d v_W0 = _mm_set1_pd(W0); + __m128d v_x1 = _mm_set_pd(1, 0); + + for (; x1 <= bw - 16; x1 += 16) + { + // 0-3 + __m128i v_X0, v_Y0; + { + __m128d v_W = _mm_add_pd(_mm_mul_pd(v_M6, v_x1), v_W0); + v_W = _mm_andnot_pd(_mm_cmpeq_pd(v_W, v_zero), _mm_div_pd(v_its, v_W)); + __m128d v_fX0 = _mm_max_pd(v_intmin, _mm_min_pd(v_intmax, _mm_mul_pd(_mm_add_pd(v_X0d, _mm_mul_pd(v_M0, v_x1)), v_W))); + __m128d v_fY0 = _mm_max_pd(v_intmin, _mm_min_pd(v_intmax, _mm_mul_pd(_mm_add_pd(v_Y0d, _mm_mul_pd(v_M3, v_x1)), v_W))); + v_x1 = _mm_add_pd(v_x1, v_2); + + v_W = _mm_add_pd(_mm_mul_pd(v_M6, v_x1), v_W0); + v_W = _mm_andnot_pd(_mm_cmpeq_pd(v_W, v_zero), _mm_div_pd(v_its, v_W)); + __m128d v_fX1 = _mm_max_pd(v_intmin, _mm_min_pd(v_intmax, _mm_mul_pd(_mm_add_pd(v_X0d, _mm_mul_pd(v_M0, v_x1)), v_W))); + __m128d v_fY1 = _mm_max_pd(v_intmin, _mm_min_pd(v_intmax, _mm_mul_pd(_mm_add_pd(v_Y0d, _mm_mul_pd(v_M3, v_x1)), v_W))); + v_x1 = _mm_add_pd(v_x1, v_2); + + v_X0 = _mm_castps_si128(_mm_movelh_ps(_mm_castsi128_ps(_mm_cvtpd_epi32(v_fX0)), + _mm_castsi128_ps(_mm_cvtpd_epi32(v_fX1)))); + v_Y0 = _mm_castps_si128(_mm_movelh_ps(_mm_castsi128_ps(_mm_cvtpd_epi32(v_fY0)), + _mm_castsi128_ps(_mm_cvtpd_epi32(v_fY1)))); + } + + // 4-8 + __m128i v_X1, v_Y1; + { + __m128d v_W = _mm_add_pd(_mm_mul_pd(v_M6, v_x1), v_W0); + v_W = _mm_andnot_pd(_mm_cmpeq_pd(v_W, v_zero), _mm_div_pd(v_its, v_W)); + __m128d v_fX0 = _mm_max_pd(v_intmin, _mm_min_pd(v_intmax, _mm_mul_pd(_mm_add_pd(v_X0d, _mm_mul_pd(v_M0, v_x1)), v_W))); + __m128d v_fY0 = _mm_max_pd(v_intmin, _mm_min_pd(v_intmax, _mm_mul_pd(_mm_add_pd(v_Y0d, _mm_mul_pd(v_M3, v_x1)), v_W))); + v_x1 = _mm_add_pd(v_x1, v_2); + + v_W = _mm_add_pd(_mm_mul_pd(v_M6, v_x1), v_W0); + v_W = _mm_andnot_pd(_mm_cmpeq_pd(v_W, v_zero), _mm_div_pd(v_its, v_W)); + __m128d v_fX1 = _mm_max_pd(v_intmin, _mm_min_pd(v_intmax, _mm_mul_pd(_mm_add_pd(v_X0d, _mm_mul_pd(v_M0, v_x1)), v_W))); + __m128d v_fY1 = _mm_max_pd(v_intmin, _mm_min_pd(v_intmax, _mm_mul_pd(_mm_add_pd(v_Y0d, _mm_mul_pd(v_M3, v_x1)), v_W))); + v_x1 = _mm_add_pd(v_x1, v_2); + + v_X1 = _mm_castps_si128(_mm_movelh_ps(_mm_castsi128_ps(_mm_cvtpd_epi32(v_fX0)), + _mm_castsi128_ps(_mm_cvtpd_epi32(v_fX1)))); + v_Y1 = _mm_castps_si128(_mm_movelh_ps(_mm_castsi128_ps(_mm_cvtpd_epi32(v_fY0)), + _mm_castsi128_ps(_mm_cvtpd_epi32(v_fY1)))); + } + + // 8-11 + __m128i v_X2, v_Y2; + { + __m128d v_W = _mm_add_pd(_mm_mul_pd(v_M6, v_x1), v_W0); + v_W = _mm_andnot_pd(_mm_cmpeq_pd(v_W, v_zero), _mm_div_pd(v_its, v_W)); + __m128d v_fX0 = _mm_max_pd(v_intmin, _mm_min_pd(v_intmax, _mm_mul_pd(_mm_add_pd(v_X0d, _mm_mul_pd(v_M0, v_x1)), v_W))); + __m128d v_fY0 = _mm_max_pd(v_intmin, _mm_min_pd(v_intmax, _mm_mul_pd(_mm_add_pd(v_Y0d, _mm_mul_pd(v_M3, v_x1)), v_W))); + v_x1 = _mm_add_pd(v_x1, v_2); + + v_W = _mm_add_pd(_mm_mul_pd(v_M6, v_x1), v_W0); + v_W = _mm_andnot_pd(_mm_cmpeq_pd(v_W, v_zero), _mm_div_pd(v_its, v_W)); + __m128d v_fX1 = _mm_max_pd(v_intmin, _mm_min_pd(v_intmax, _mm_mul_pd(_mm_add_pd(v_X0d, _mm_mul_pd(v_M0, v_x1)), v_W))); + __m128d v_fY1 = _mm_max_pd(v_intmin, _mm_min_pd(v_intmax, _mm_mul_pd(_mm_add_pd(v_Y0d, _mm_mul_pd(v_M3, v_x1)), v_W))); + v_x1 = _mm_add_pd(v_x1, v_2); + + v_X2 = _mm_castps_si128(_mm_movelh_ps(_mm_castsi128_ps(_mm_cvtpd_epi32(v_fX0)), + _mm_castsi128_ps(_mm_cvtpd_epi32(v_fX1)))); + v_Y2 = _mm_castps_si128(_mm_movelh_ps(_mm_castsi128_ps(_mm_cvtpd_epi32(v_fY0)), + _mm_castsi128_ps(_mm_cvtpd_epi32(v_fY1)))); + } + + // 12-15 + __m128i v_X3, v_Y3; + { + __m128d v_W = _mm_add_pd(_mm_mul_pd(v_M6, v_x1), v_W0); + v_W = _mm_andnot_pd(_mm_cmpeq_pd(v_W, v_zero), _mm_div_pd(v_its, v_W)); + __m128d v_fX0 = _mm_max_pd(v_intmin, _mm_min_pd(v_intmax, _mm_mul_pd(_mm_add_pd(v_X0d, _mm_mul_pd(v_M0, v_x1)), v_W))); + __m128d v_fY0 = _mm_max_pd(v_intmin, _mm_min_pd(v_intmax, _mm_mul_pd(_mm_add_pd(v_Y0d, _mm_mul_pd(v_M3, v_x1)), v_W))); + v_x1 = _mm_add_pd(v_x1, v_2); + + v_W = _mm_add_pd(_mm_mul_pd(v_M6, v_x1), v_W0); + v_W = _mm_andnot_pd(_mm_cmpeq_pd(v_W, v_zero), _mm_div_pd(v_its, v_W)); + __m128d v_fX1 = _mm_max_pd(v_intmin, _mm_min_pd(v_intmax, _mm_mul_pd(_mm_add_pd(v_X0d, _mm_mul_pd(v_M0, v_x1)), v_W))); + __m128d v_fY1 = _mm_max_pd(v_intmin, _mm_min_pd(v_intmax, _mm_mul_pd(_mm_add_pd(v_Y0d, _mm_mul_pd(v_M3, v_x1)), v_W))); + v_x1 = _mm_add_pd(v_x1, v_2); + + v_X3 = _mm_castps_si128(_mm_movelh_ps(_mm_castsi128_ps(_mm_cvtpd_epi32(v_fX0)), + _mm_castsi128_ps(_mm_cvtpd_epi32(v_fX1)))); + v_Y3 = _mm_castps_si128(_mm_movelh_ps(_mm_castsi128_ps(_mm_cvtpd_epi32(v_fY0)), + _mm_castsi128_ps(_mm_cvtpd_epi32(v_fY1)))); + } + + // store alpha + __m128i v_alpha0 = _mm_add_epi32(_mm_slli_epi32(_mm_and_si128(v_Y0, v_itsi1), INTER_BITS), + _mm_and_si128(v_X0, v_itsi1)); + __m128i v_alpha1 = _mm_add_epi32(_mm_slli_epi32(_mm_and_si128(v_Y1, v_itsi1), INTER_BITS), + _mm_and_si128(v_X1, v_itsi1)); + _mm_storeu_si128((__m128i *)(alpha + x1), _mm_packs_epi32(v_alpha0, v_alpha1)); + + v_alpha0 = _mm_add_epi32(_mm_slli_epi32(_mm_and_si128(v_Y2, v_itsi1), INTER_BITS), + _mm_and_si128(v_X2, v_itsi1)); + v_alpha1 = _mm_add_epi32(_mm_slli_epi32(_mm_and_si128(v_Y3, v_itsi1), INTER_BITS), + _mm_and_si128(v_X3, v_itsi1)); + _mm_storeu_si128((__m128i *)(alpha + x1 + 8), _mm_packs_epi32(v_alpha0, v_alpha1)); + + // convert to 16s + v_X0 = _mm_packs_epi32(_mm_srai_epi32(v_X0, INTER_BITS), _mm_srai_epi32(v_X1, INTER_BITS)); + v_X1 = _mm_packs_epi32(_mm_srai_epi32(v_X2, INTER_BITS), _mm_srai_epi32(v_X3, INTER_BITS)); + v_Y0 = _mm_packs_epi32(_mm_srai_epi32(v_Y0, INTER_BITS), _mm_srai_epi32(v_Y1, INTER_BITS)); + v_Y1 = _mm_packs_epi32(_mm_srai_epi32(v_Y2, INTER_BITS), _mm_srai_epi32(v_Y3, INTER_BITS)); + + _mm_interleave_epi16(v_X0, v_X1, v_Y0, v_Y1); + + _mm_storeu_si128((__m128i *)(xy + x1 * 2), v_X0); + _mm_storeu_si128((__m128i *)(xy + x1 * 2 + 8), v_X1); + _mm_storeu_si128((__m128i *)(xy + x1 * 2 + 16), v_Y0); + _mm_storeu_si128((__m128i *)(xy + x1 * 2 + 24), v_Y1); + } + for (; x1 < bw; x1++) + { + double W = W0 + M[6] * x1; + W = W ? INTER_TAB_SIZE / W : 0; + double fX = std::max((double)INT_MIN, std::min((double)INT_MAX, (X0 + M[0] * x1)*W)); + double fY = std::max((double)INT_MIN, std::min((double)INT_MAX, (Y0 + M[3] * x1)*W)); + int X = saturate_cast(fX); + int Y = saturate_cast(fY); + + xy[x1 * 2] = saturate_cast(X >> INTER_BITS); + xy[x1 * 2 + 1] = saturate_cast(Y >> INTER_BITS); + alpha[x1] = (short)((Y & (INTER_TAB_SIZE - 1))*INTER_TAB_SIZE + + (X & (INTER_TAB_SIZE - 1))); + } + } + virtual ~WarpPerspectiveLine_SSE4_Impl() {}; +private: + __m128d v_M0; + __m128d v_M3; + __m128d v_M6; + __m128d v_intmax; + __m128d v_intmin; + __m128d v_2, + v_zero, + v_1, + v_its; + __m128i v_itsi1; +}; + +Ptr WarpPerspectiveLine_SSE4::getImpl(const double *M) +{ + return Ptr(new WarpPerspectiveLine_SSE4_Impl(M)); +} + +} +} /* End of file. */