From d1ca9341151fe63e052809bef6f6552df1c3d1c5 Mon Sep 17 00:00:00 2001 From: Ilya Lavrenov Date: Tue, 11 Dec 2012 15:14:50 +0400 Subject: [PATCH 1/8] sse2 version of resize area fast for types CV_(8, 16)UC(1, 3, 4) --- modules/imgproc/perf/perf_resize.cpp | 2 +- modules/imgproc/src/imgwarp.cpp | 172 +++++++++++++++++++++++++-- 2 files changed, 160 insertions(+), 14 deletions(-) diff --git a/modules/imgproc/perf/perf_resize.cpp b/modules/imgproc/perf/perf_resize.cpp index 82bf0d37cf..98e4bc2f7d 100644 --- a/modules/imgproc/perf/perf_resize.cpp +++ b/modules/imgproc/perf/perf_resize.cpp @@ -71,7 +71,7 @@ typedef TestBaseWithParam MatInfo_Size_Scale; PERF_TEST_P(MatInfo_Size_Scale, ResizeAreaFast, testing::Combine( - testing::Values(CV_8UC1, CV_8UC4), + testing::Values(CV_8UC1, CV_8UC3, CV_8UC4, CV_16UC1, CV_16UC3, CV_16UC4), testing::Values(szVGA, szqHD, sz720p, sz1080p), testing::Values(2) ) diff --git a/modules/imgproc/src/imgwarp.cpp b/modules/imgproc/src/imgwarp.cpp index c2506590d6..0de9f594dc 100644 --- a/modules/imgproc/src/imgwarp.cpp +++ b/modules/imgproc/src/imgwarp.cpp @@ -1241,16 +1241,163 @@ static void resizeGeneric_( const Mat& src, Mat& dst, template struct ResizeAreaFastNoVec { - ResizeAreaFastNoVec(int /*_scale_x*/, int /*_scale_y*/, - int /*_cn*/, int /*_step*//*, const int**/ /*_ofs*/) { } - int operator() (const T* /*S*/, T* /*D*/, int /*w*/) const { return 0; } + ResizeAreaFastNoVec(int, int) { } + ResizeAreaFastNoVec(int, int, int, int) { } + int operator() (const T*, T*, int) const + { return 0; } }; -template +#if CV_SSE2 +class ResizeAreaFastVec_SIMD_8u +{ +public: + ResizeAreaFastVec_SIMD_8u(int _cn, int _step) : + cn(_cn), step(_step) + { + use_simd = checkHardwareSupport(CV_CPU_SSE2); + } + + int operator() (const uchar* S, uchar* D, int w) const + { + if (!use_simd) + return 0; + + int dx = 0; + const uchar* S0 = S; + const uchar* S1 = S0 + step; + __m128i masklow = _mm_set1_epi16(0x00ff); + __m128i zero = _mm_setzero_si128(); + + if (cn == 1) + { + for ( ; dx < w - 8; dx += 8, S0 += 16, S1 += 16, D += 8) + { + __m128i s0 = _mm_loadu_si128((const __m128i*)S0); + __m128i s1 = _mm_loadu_si128((const __m128i*)S1); + + __m128i s = _mm_avg_epu8(s0, _mm_srli_si128(s0, 1)); + s = _mm_avg_epu8(s, _mm_avg_epu8(s1, _mm_srli_si128(s1, 1))); + + _mm_storel_epi64((__m128i*)D, _mm_packus_epi16(_mm_and_si128(s, masklow), zero)); + } + } + else if (cn == 3) + for ( ; dx < w - 6; dx += 6, S0 += 12, S1 += 12, D += 6) + { + __m128i s0 = _mm_loadu_si128((const __m128i*)S0); + __m128i s1 = _mm_loadu_si128((const __m128i*)S1); + + __m128i s = _mm_avg_epu8(s0, _mm_srli_si128(s0, 3)); + s = _mm_avg_epu8(s, _mm_avg_epu8(s1, _mm_srli_si128(s1, 3))); + + _mm_storel_epi64((__m128i*)D, s); + _mm_storel_epi64((__m128i*)(D+3), _mm_srli_si128(s, 6)); + } + else + { + CV_Assert(cn == 4); + for ( ; dx < w - 8; dx += 8, S0 += 16, S1 += 16, D += 8) + { + __m128i s0 = _mm_loadu_si128((const __m128i*)S0); + __m128i s1 = _mm_loadu_si128((const __m128i*)S1); + + __m128i s = _mm_avg_epu8(s0, _mm_srli_si128(s0, 4)); + s = _mm_avg_epu8(s, _mm_avg_epu8(s1, _mm_srli_si128(s1, 4))); + + _mm_storel_epi64((__m128i*)D, s); + _mm_storel_epi64((__m128i*)(D+4), _mm_srli_si128(s, 8)); + } + } + + return dx; + } + +private: + int cn; + int step; + bool use_simd; +}; + +class ResizeAreaFastVec_SIMD_16u +{ +public: + ResizeAreaFastVec_SIMD_16u(int _cn, int _step) : + cn(_cn), step(_step) + { + use_simd = checkHardwareSupport(CV_CPU_SSE2); + } + + int operator() (const ushort* S, ushort* D, int w) const + { + if (!use_simd) + return 0; + + int dx = 0; + const ushort* S0 = (const ushort*)S; + const ushort* S1 = (const ushort*)(S0 + step); + __m128i masklow = _mm_set1_epi32(0x0000ffff); + __m128i zero = _mm_setzero_si128(); + + if (cn == 1) + { + for ( ; dx < w - 4; dx += 4, S0 += 8, S1 += 8, D += 4) + { + __m128i s0 = _mm_loadu_si128((const __m128i*)S0); + __m128i s1 = _mm_loadu_si128((const __m128i*)S1); + + __m128i s = _mm_avg_epu16(s0, _mm_srli_si128(s0, 2)); + s = _mm_avg_epu16(s, _mm_avg_epu16(s1, _mm_srli_si128(s1, 2))); + + s = _mm_and_si128(s, masklow); + s = _mm_packs_epi32(s, zero); + _mm_storel_epi64((__m128i*)D, s); + } + } + else if (cn == 3) + for ( ; dx < w - 3; dx += 3, S0 += 6, S1 += 6, D += 3) + { + __m128i s0 = _mm_loadu_si128((const __m128i*)S0); + __m128i s1 = _mm_loadu_si128((const __m128i*)S1); + + __m128i s = _mm_avg_epu16(s0, _mm_srli_si128(s0, 6)); + s = _mm_avg_epu16(s, _mm_avg_epu16(s1, _mm_srli_si128(s1, 6))); + + _mm_storel_epi64((__m128i*)D, s); + } + else + { + CV_Assert(cn == 4); + for ( ; dx < w - 4; dx += 4, S0 += 8, S1 += 8, D += 4) + { + __m128i s0 = _mm_loadu_si128((const __m128i*)S0); + __m128i s1 = _mm_loadu_si128((const __m128i*)S1); + + __m128i s = _mm_avg_epu16(s0, _mm_srli_si128(s0, 8)); + s = _mm_avg_epu16(s, _mm_avg_epu16(s1, _mm_srli_si128(s1, 8))); + + _mm_storel_epi64((__m128i*)(D), s); + } + } + + return dx; + } + +private: + int cn; + int step; + bool use_simd; +}; + +#else +typedef ResizeAreaFastNoVec ResizeAreaFastVec_SIMD_8u; +typedef ResizeAreaFastNoVec ResizeAreaFastVec_SIMD_16u; +#endif + +template struct ResizeAreaFastVec { - ResizeAreaFastVec(int _scale_x, int _scale_y, int _cn, int _step/*, const int* _ofs*/) : - scale_x(_scale_x), scale_y(_scale_y), cn(_cn), step(_step)/*, ofs(_ofs)*/ + ResizeAreaFastVec(int _scale_x, int _scale_y, int _cn, int _step) : + scale_x(_scale_x), scale_y(_scale_y), cn(_cn), step(_step), vecOp(_cn, _step) { fast_mode = scale_x == 2 && scale_y == 2 && (cn == 1 || cn == 3 || cn == 4); } @@ -1261,7 +1408,7 @@ struct ResizeAreaFastVec return 0; const T* nextS = (const T*)((const uchar*)S + step); - int dx = 0; + int dx = vecOp(S, D, w); if (cn == 1) for( ; dx < w; ++dx ) @@ -1279,7 +1426,7 @@ struct ResizeAreaFastVec } else { - assert(cn == 4); + CV_Assert(cn == 4); for( ; dx < w; dx += 4 ) { int index = dx*2; @@ -1298,6 +1445,7 @@ private: int cn; bool fast_mode; int step; + SIMDVecOp vecOp; }; template @@ -1702,10 +1850,10 @@ void cv::resize( InputArray _src, OutputArray _dst, Size dsize, static ResizeAreaFastFunc areafast_tab[] = { - resizeAreaFast_ >, + resizeAreaFast_ >, 0, - resizeAreaFast_ >, - resizeAreaFast_ >, + resizeAreaFast_ >, + resizeAreaFast_ > >, 0, resizeAreaFast_ >, resizeAreaFast_ >, @@ -1764,9 +1912,7 @@ void cv::resize( InputArray _src, OutputArray _dst, Size dsize, // in case of scale_x && scale_y is equal to 2 // INTER_AREA (fast) also is equal to INTER_LINEAR if( interpolation == INTER_LINEAR && is_area_fast && iscale_x == 2 && iscale_y == 2 ) - { interpolation = INTER_AREA; - } // true "area" interpolation is only implemented for the case (scale_x <= 1 && scale_y <= 1). // In other cases it is emulated using some variant of bilinear interpolation From 16f9b6f5e419cd84e53494838a08c7cd54f8f5d1 Mon Sep 17 00:00:00 2001 From: Ilya Lavrenov Date: Fri, 14 Dec 2012 14:32:00 +0400 Subject: [PATCH 2/8] reproducing C++ version of resize area fast --- modules/imgproc/src/imgwarp.cpp | 118 +++++++++++++++++++++----------- 1 file changed, 78 insertions(+), 40 deletions(-) diff --git a/modules/imgproc/src/imgwarp.cpp b/modules/imgproc/src/imgwarp.cpp index 0de9f594dc..7c174f2800 100644 --- a/modules/imgproc/src/imgwarp.cpp +++ b/modules/imgproc/src/imgwarp.cpp @@ -1265,47 +1265,72 @@ public: int dx = 0; const uchar* S0 = S; const uchar* S1 = S0 + step; - __m128i masklow = _mm_set1_epi16(0x00ff); __m128i zero = _mm_setzero_si128(); + __m128i delta2 = _mm_set1_epi16(2); if (cn == 1) { - for ( ; dx < w - 8; dx += 8, S0 += 16, S1 += 16, D += 8) + __m128i masklow = _mm_set1_epi16(0x00ff); + for ( ; dx < w; dx += 8, S0 += 16, S1 += 16, D += 8) { - __m128i s0 = _mm_loadu_si128((const __m128i*)S0); - __m128i s1 = _mm_loadu_si128((const __m128i*)S1); + __m128i r0 = _mm_loadu_si128((const __m128i*)S0); + __m128i r1 = _mm_loadu_si128((const __m128i*)S1); - __m128i s = _mm_avg_epu8(s0, _mm_srli_si128(s0, 1)); - s = _mm_avg_epu8(s, _mm_avg_epu8(s1, _mm_srli_si128(s1, 1))); + __m128i s0 = _mm_add_epi16(_mm_srli_epi16(r0, 8), _mm_and_si128(r0, masklow)); + __m128i s1 = _mm_add_epi16(_mm_srli_epi16(r1, 8), _mm_and_si128(r1, masklow)); + s0 = _mm_add_epi16(_mm_add_epi16(s0, s1), delta2); + s0 = _mm_packus_epi16(_mm_srli_epi16(s0, 2), zero); - _mm_storel_epi64((__m128i*)D, _mm_packus_epi16(_mm_and_si128(s, masklow), zero)); + _mm_storel_epi64((__m128i*)D, s0); } } else if (cn == 3) - for ( ; dx < w - 6; dx += 6, S0 += 12, S1 += 12, D += 6) + for ( ; dx < w; dx += 6, S0 += 12, S1 += 12, D += 6) { - __m128i s0 = _mm_loadu_si128((const __m128i*)S0); - __m128i s1 = _mm_loadu_si128((const __m128i*)S1); + __m128i r0 = _mm_loadu_si128((const __m128i*)S0); + __m128i r1 = _mm_loadu_si128((const __m128i*)S1); - __m128i s = _mm_avg_epu8(s0, _mm_srli_si128(s0, 3)); - s = _mm_avg_epu8(s, _mm_avg_epu8(s1, _mm_srli_si128(s1, 3))); + __m128i r0_16l = _mm_unpacklo_epi8(r0, zero); + __m128i r0_16h = _mm_unpacklo_epi8(_mm_srli_si128(r0, 6), zero); + __m128i r1_16l = _mm_unpacklo_epi8(r1, zero); + __m128i r1_16h = _mm_unpacklo_epi8(_mm_srli_si128(r1, 6), zero); - _mm_storel_epi64((__m128i*)D, s); - _mm_storel_epi64((__m128i*)(D+3), _mm_srli_si128(s, 6)); + __m128i s0 = _mm_add_epi16(r0_16l, _mm_srli_si128(r0_16l, 6)); + __m128i s1 = _mm_add_epi16(r1_16l, _mm_srli_si128(r1_16l, 6)); + s0 = _mm_add_epi16(s1, _mm_add_epi16(s0, delta2)); + s0 = _mm_packus_epi16(_mm_srli_epi16(s0, 2), zero); + _mm_storel_epi64((__m128i*)D, s0); + + s0 = _mm_add_epi16(r0_16h, _mm_srli_si128(r0_16h, 6)); + s1 = _mm_add_epi16(r1_16h, _mm_srli_si128(r1_16h, 6)); + s0 = _mm_add_epi16(s1, _mm_add_epi16(s0, delta2)); + s0 = _mm_packus_epi16(_mm_srli_epi16(s0, 2), zero); + _mm_storel_epi64((__m128i*)(D+3), s0); } else { CV_Assert(cn == 4); - for ( ; dx < w - 8; dx += 8, S0 += 16, S1 += 16, D += 8) + for ( ; dx < w; dx += 8, S0 += 16, S1 += 16, D += 8) { - __m128i s0 = _mm_loadu_si128((const __m128i*)S0); - __m128i s1 = _mm_loadu_si128((const __m128i*)S1); + __m128i r0 = _mm_loadu_si128((const __m128i*)S0); + __m128i r1 = _mm_loadu_si128((const __m128i*)S1); - __m128i s = _mm_avg_epu8(s0, _mm_srli_si128(s0, 4)); - s = _mm_avg_epu8(s, _mm_avg_epu8(s1, _mm_srli_si128(s1, 4))); + __m128i r0_16l = _mm_unpacklo_epi8(r0, zero); + __m128i r0_16h = _mm_unpackhi_epi8(r0, zero); + __m128i r1_16l = _mm_unpacklo_epi8(r1, zero); + __m128i r1_16h = _mm_unpackhi_epi8(r1, zero); - _mm_storel_epi64((__m128i*)D, s); - _mm_storel_epi64((__m128i*)(D+4), _mm_srli_si128(s, 8)); + __m128i s0 = _mm_add_epi16(r0_16l, _mm_srli_si128(r0_16l, 8)); + __m128i s1 = _mm_add_epi16(r1_16l, _mm_srli_si128(r1_16l, 8)); + s0 = _mm_add_epi16(s1, _mm_add_epi16(s0, delta2)); + s0 = _mm_packus_epi16(_mm_srli_epi16(s0, 2), zero); + _mm_storel_epi64((__m128i*)D, s0); + + s0 = _mm_add_epi16(r0_16h, _mm_srli_si128(r0_16h, 8)); + s1 = _mm_add_epi16(r1_16h, _mm_srli_si128(r1_16h, 8)); + s0 = _mm_add_epi16(s1, _mm_add_epi16(s0, delta2)); + s0 = _mm_packus_epi16(_mm_srli_epi16(s0, 2), zero); + _mm_storel_epi64((__m128i*)(D+4), s0); } } @@ -1314,8 +1339,8 @@ public: private: int cn; - int step; bool use_simd; + int step; }; class ResizeAreaFastVec_SIMD_16u @@ -1337,45 +1362,58 @@ public: const ushort* S1 = (const ushort*)(S0 + step); __m128i masklow = _mm_set1_epi32(0x0000ffff); __m128i zero = _mm_setzero_si128(); + __m128i delta2 = _mm_set1_epi32(2); if (cn == 1) { for ( ; dx < w - 4; dx += 4, S0 += 8, S1 += 8, D += 4) { - __m128i s0 = _mm_loadu_si128((const __m128i*)S0); - __m128i s1 = _mm_loadu_si128((const __m128i*)S1); + __m128i r0 = _mm_loadu_si128((const __m128i*)S0); + __m128i r1 = _mm_loadu_si128((const __m128i*)S1); - __m128i s = _mm_avg_epu16(s0, _mm_srli_si128(s0, 2)); - s = _mm_avg_epu16(s, _mm_avg_epu16(s1, _mm_srli_si128(s1, 2))); + __m128i s0 = _mm_add_epi32(_mm_srli_epi32(r0, 16), _mm_and_si128(r0, masklow)); + __m128i s1 = _mm_add_epi32(_mm_srli_epi32(r1, 16), _mm_and_si128(r1, masklow)); + s0 = _mm_add_epi32(_mm_add_epi32(s0, s1), delta2); + s0 = _mm_packs_epi32(_mm_srli_epi32(s0, 2), zero); - s = _mm_and_si128(s, masklow); - s = _mm_packs_epi32(s, zero); - _mm_storel_epi64((__m128i*)D, s); + _mm_storel_epi64((__m128i*)D, s0); } } else if (cn == 3) for ( ; dx < w - 3; dx += 3, S0 += 6, S1 += 6, D += 3) { - __m128i s0 = _mm_loadu_si128((const __m128i*)S0); - __m128i s1 = _mm_loadu_si128((const __m128i*)S1); + __m128i r0 = _mm_loadu_si128((const __m128i*)S0); + __m128i r1 = _mm_loadu_si128((const __m128i*)S1); - __m128i s = _mm_avg_epu16(s0, _mm_srli_si128(s0, 6)); - s = _mm_avg_epu16(s, _mm_avg_epu16(s1, _mm_srli_si128(s1, 6))); + __m128i r0_16l = _mm_unpacklo_epi16(r0, zero); + __m128i r0_16h = _mm_unpacklo_epi16(_mm_srli_si128(r0, 6), zero); + __m128i r1_16l = _mm_unpacklo_epi16(r1, zero); + __m128i r1_16h = _mm_unpacklo_epi16(_mm_srli_si128(r1, 6), zero); - _mm_storel_epi64((__m128i*)D, s); + __m128i s0 = _mm_add_epi16(r0_16l, r0_16h); + __m128i s1 = _mm_add_epi16(r1_16l, r1_16h); + s0 = _mm_add_epi32(s1, _mm_add_epi32(s0, delta2)); + s0 = _mm_packs_epi32(_mm_srli_epi32(s0, 2), zero); + _mm_storel_epi64((__m128i*)D, s0); } else { CV_Assert(cn == 4); for ( ; dx < w - 4; dx += 4, S0 += 8, S1 += 8, D += 4) { - __m128i s0 = _mm_loadu_si128((const __m128i*)S0); - __m128i s1 = _mm_loadu_si128((const __m128i*)S1); + __m128i r0 = _mm_loadu_si128((const __m128i*)S0); + __m128i r1 = _mm_loadu_si128((const __m128i*)S1); - __m128i s = _mm_avg_epu16(s0, _mm_srli_si128(s0, 8)); - s = _mm_avg_epu16(s, _mm_avg_epu16(s1, _mm_srli_si128(s1, 8))); + __m128i r0_32l = _mm_unpacklo_epi16(r0, zero); + __m128i r0_32h = _mm_unpackhi_epi16(r0, zero); + __m128i r1_32l = _mm_unpacklo_epi16(r1, zero); + __m128i r1_32h = _mm_unpackhi_epi16(r1, zero); - _mm_storel_epi64((__m128i*)(D), s); + __m128i s0 = _mm_add_epi32(r0_32l, r0_32h); + __m128i s1 = _mm_add_epi32(r1_32l, r1_32h); + s0 = _mm_add_epi32(s1, _mm_add_epi32(s0, delta2)); + s0 = _mm_packs_epi32(_mm_srli_epi32(s0, 2), zero); + _mm_storel_epi64((__m128i*)D, s0); } } @@ -1404,7 +1442,7 @@ struct ResizeAreaFastVec int operator() (const T* S, T* D, int w) const { - if( !fast_mode ) + if (!fast_mode) return 0; const T* nextS = (const T*)((const uchar*)S + step); From 4ccb5a30d9eb78a37ff2390d227d53b2dbb88dd4 Mon Sep 17 00:00:00 2001 From: Ilya Lavrenov Date: Fri, 14 Dec 2012 17:41:42 +0400 Subject: [PATCH 3/8] fixed bug with s1 calculating --- modules/imgproc/src/imgwarp.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/modules/imgproc/src/imgwarp.cpp b/modules/imgproc/src/imgwarp.cpp index 7c174f2800..3fe7fbf48a 100644 --- a/modules/imgproc/src/imgwarp.cpp +++ b/modules/imgproc/src/imgwarp.cpp @@ -1271,7 +1271,7 @@ public: if (cn == 1) { __m128i masklow = _mm_set1_epi16(0x00ff); - for ( ; dx < w; dx += 8, S0 += 16, S1 += 16, D += 8) + for ( ; dx < w - 8; dx += 8, S0 += 16, S1 += 16, D += 8) { __m128i r0 = _mm_loadu_si128((const __m128i*)S0); __m128i r1 = _mm_loadu_si128((const __m128i*)S1); @@ -1285,7 +1285,7 @@ public: } } else if (cn == 3) - for ( ; dx < w; dx += 6, S0 += 12, S1 += 12, D += 6) + for ( ; dx < w - 6; dx += 6, S0 += 12, S1 += 12, D += 6) { __m128i r0 = _mm_loadu_si128((const __m128i*)S0); __m128i r1 = _mm_loadu_si128((const __m128i*)S1); @@ -1310,7 +1310,7 @@ public: else { CV_Assert(cn == 4); - for ( ; dx < w; dx += 8, S0 += 16, S1 += 16, D += 8) + for ( ; dx < w - 8; dx += 8, S0 += 16, S1 += 16, D += 8) { __m128i r0 = _mm_loadu_si128((const __m128i*)S0); __m128i r1 = _mm_loadu_si128((const __m128i*)S1); @@ -1359,7 +1359,7 @@ public: int dx = 0; const ushort* S0 = (const ushort*)S; - const ushort* S1 = (const ushort*)(S0 + step); + const ushort* S1 = (const ushort*)(S + step); __m128i masklow = _mm_set1_epi32(0x0000ffff); __m128i zero = _mm_setzero_si128(); __m128i delta2 = _mm_set1_epi32(2); From 6059a6875a32967efc2935da1c3e9a26ba7c76f1 Mon Sep 17 00:00:00 2001 From: Ilya Lavrenov Date: Fri, 14 Dec 2012 17:54:07 +0400 Subject: [PATCH 4/8] fixed bug with s1 calculating --- modules/imgproc/src/imgwarp.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/imgproc/src/imgwarp.cpp b/modules/imgproc/src/imgwarp.cpp index 3fe7fbf48a..e6b45f647c 100644 --- a/modules/imgproc/src/imgwarp.cpp +++ b/modules/imgproc/src/imgwarp.cpp @@ -1359,7 +1359,7 @@ public: int dx = 0; const ushort* S0 = (const ushort*)S; - const ushort* S1 = (const ushort*)(S + step); + const ushort* S1 = (const ushort*)(S + step/2); __m128i masklow = _mm_set1_epi32(0x0000ffff); __m128i zero = _mm_setzero_si128(); __m128i delta2 = _mm_set1_epi32(2); From aa0dafcc1f81ca2eee54ed152cdd571a7ec99b93 Mon Sep 17 00:00:00 2001 From: Ilya Lavrenov Date: Mon, 17 Dec 2012 14:09:25 +0400 Subject: [PATCH 5/8] replaced _mm_packs_epi32 to _mm_packus_epi32 --- modules/imgproc/src/imgwarp.cpp | 35 +++++++++++++++++++++++++++++---- 1 file changed, 31 insertions(+), 4 deletions(-) diff --git a/modules/imgproc/src/imgwarp.cpp b/modules/imgproc/src/imgwarp.cpp index e6b45f647c..1e86db3057 100644 --- a/modules/imgproc/src/imgwarp.cpp +++ b/modules/imgproc/src/imgwarp.cpp @@ -1301,11 +1301,35 @@ public: s0 = _mm_packus_epi16(_mm_srli_epi16(s0, 2), zero); _mm_storel_epi64((__m128i*)D, s0); +// union +// { +// __m128i m; +// unsigned char us[16]; +// } u; + +// u.m = s0; +// unsigned char v0 = (unsigned char)((S0[0] + S0[3] + S1[0] + S1[3] + 2) >> 2); +// unsigned char v1 = (unsigned char)((S0[1] + S0[4] + S1[1] + S1[4] + 2) >> 2); +// unsigned char v2 = (unsigned char)((S0[2] + S0[5] + S1[2] + S1[5] + 2) >> 2); +// unsigned char ar1[] = { v0, v1, v2 }; +// for (unsigned int i = 0; i < 3; ++i) +// std::cout << ((int)(u.us[i]) - (int)(ar1[i])) << " "; +// std::cout << "\t1" << std::endl; + s0 = _mm_add_epi16(r0_16h, _mm_srli_si128(r0_16h, 6)); s1 = _mm_add_epi16(r1_16h, _mm_srli_si128(r1_16h, 6)); s0 = _mm_add_epi16(s1, _mm_add_epi16(s0, delta2)); s0 = _mm_packus_epi16(_mm_srli_epi16(s0, 2), zero); _mm_storel_epi64((__m128i*)(D+3), s0); + +// u.m = s0; +// v0 = (unsigned char)((S0[6] + S0[9] + S1[6] + S1[9] + 2) >> 2); +// v1 = (unsigned char)((S0[7] + S0[10] + S1[7] + S1[10] + 2) >> 2); +// v2 = (unsigned char)((S0[8] + S0[11] + S1[8] + S1[11] + 2) >> 2); +// unsigned char ar2[] = { v0, v1, v2 }; +// for (unsigned int i = 0; i < 3; ++i) +// std::cout << ((int)(u.us[i]) - (int)(ar2[i])) << " "; +// std::cout << "\t2" << std::endl; } else { @@ -1359,11 +1383,13 @@ public: int dx = 0; const ushort* S0 = (const ushort*)S; - const ushort* S1 = (const ushort*)(S + step/2); + const ushort* S1 = (const ushort*)((const uchar*)(S) + step); __m128i masklow = _mm_set1_epi32(0x0000ffff); __m128i zero = _mm_setzero_si128(); __m128i delta2 = _mm_set1_epi32(2); +#define _mm_packus_epi32(a, zero) _mm_packs_epi32(_mm_srai_epi32(_mm_slli_epi32(a, 16), 16), zero); + if (cn == 1) { for ( ; dx < w - 4; dx += 4, S0 += 8, S1 += 8, D += 4) @@ -1374,7 +1400,8 @@ public: __m128i s0 = _mm_add_epi32(_mm_srli_epi32(r0, 16), _mm_and_si128(r0, masklow)); __m128i s1 = _mm_add_epi32(_mm_srli_epi32(r1, 16), _mm_and_si128(r1, masklow)); s0 = _mm_add_epi32(_mm_add_epi32(s0, s1), delta2); - s0 = _mm_packs_epi32(_mm_srli_epi32(s0, 2), zero); + s0 = _mm_srli_epi32(s0, 2); + s0 = _mm_packus_epi32(s0, zero); _mm_storel_epi64((__m128i*)D, s0); } @@ -1393,7 +1420,7 @@ public: __m128i s0 = _mm_add_epi16(r0_16l, r0_16h); __m128i s1 = _mm_add_epi16(r1_16l, r1_16h); s0 = _mm_add_epi32(s1, _mm_add_epi32(s0, delta2)); - s0 = _mm_packs_epi32(_mm_srli_epi32(s0, 2), zero); + s0 = _mm_packus_epi32(_mm_srli_epi32(s0, 2), zero); _mm_storel_epi64((__m128i*)D, s0); } else @@ -1412,7 +1439,7 @@ public: __m128i s0 = _mm_add_epi32(r0_32l, r0_32h); __m128i s1 = _mm_add_epi32(r1_32l, r1_32h); s0 = _mm_add_epi32(s1, _mm_add_epi32(s0, delta2)); - s0 = _mm_packs_epi32(_mm_srli_epi32(s0, 2), zero); + s0 = _mm_packus_epi32(_mm_srli_epi32(s0, 2), zero); _mm_storel_epi64((__m128i*)D, s0); } } From c2f2e33a5ee07e9809f39774195d14d1c5cacdb1 Mon Sep 17 00:00:00 2001 From: Ilya Lavrenov Date: Mon, 17 Dec 2012 15:45:20 +0400 Subject: [PATCH 6/8] removed debug messages --- modules/imgproc/src/imgwarp.cpp | 24 ------------------------ 1 file changed, 24 deletions(-) diff --git a/modules/imgproc/src/imgwarp.cpp b/modules/imgproc/src/imgwarp.cpp index 1e86db3057..8afddc900c 100644 --- a/modules/imgproc/src/imgwarp.cpp +++ b/modules/imgproc/src/imgwarp.cpp @@ -1301,35 +1301,11 @@ public: s0 = _mm_packus_epi16(_mm_srli_epi16(s0, 2), zero); _mm_storel_epi64((__m128i*)D, s0); -// union -// { -// __m128i m; -// unsigned char us[16]; -// } u; - -// u.m = s0; -// unsigned char v0 = (unsigned char)((S0[0] + S0[3] + S1[0] + S1[3] + 2) >> 2); -// unsigned char v1 = (unsigned char)((S0[1] + S0[4] + S1[1] + S1[4] + 2) >> 2); -// unsigned char v2 = (unsigned char)((S0[2] + S0[5] + S1[2] + S1[5] + 2) >> 2); -// unsigned char ar1[] = { v0, v1, v2 }; -// for (unsigned int i = 0; i < 3; ++i) -// std::cout << ((int)(u.us[i]) - (int)(ar1[i])) << " "; -// std::cout << "\t1" << std::endl; - s0 = _mm_add_epi16(r0_16h, _mm_srli_si128(r0_16h, 6)); s1 = _mm_add_epi16(r1_16h, _mm_srli_si128(r1_16h, 6)); s0 = _mm_add_epi16(s1, _mm_add_epi16(s0, delta2)); s0 = _mm_packus_epi16(_mm_srli_epi16(s0, 2), zero); _mm_storel_epi64((__m128i*)(D+3), s0); - -// u.m = s0; -// v0 = (unsigned char)((S0[6] + S0[9] + S1[6] + S1[9] + 2) >> 2); -// v1 = (unsigned char)((S0[7] + S0[10] + S1[7] + S1[10] + 2) >> 2); -// v2 = (unsigned char)((S0[8] + S0[11] + S1[8] + S1[11] + 2) >> 2); -// unsigned char ar2[] = { v0, v1, v2 }; -// for (unsigned int i = 0; i < 3; ++i) -// std::cout << ((int)(u.us[i]) - (int)(ar2[i])) << " "; -// std::cout << "\t2" << std::endl; } else { From d246b415738b40c84095378bd540f7ac2abe91aa Mon Sep 17 00:00:00 2001 From: Ilya Lavrenov Date: Mon, 17 Dec 2012 15:50:01 +0400 Subject: [PATCH 7/8] changed loop condition --- modules/imgproc/src/imgwarp.cpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/modules/imgproc/src/imgwarp.cpp b/modules/imgproc/src/imgwarp.cpp index 8afddc900c..ede660417f 100644 --- a/modules/imgproc/src/imgwarp.cpp +++ b/modules/imgproc/src/imgwarp.cpp @@ -1271,7 +1271,7 @@ public: if (cn == 1) { __m128i masklow = _mm_set1_epi16(0x00ff); - for ( ; dx < w - 8; dx += 8, S0 += 16, S1 += 16, D += 8) + for ( ; dx <= w - 8; dx += 8, S0 += 16, S1 += 16, D += 8) { __m128i r0 = _mm_loadu_si128((const __m128i*)S0); __m128i r1 = _mm_loadu_si128((const __m128i*)S1); @@ -1285,7 +1285,7 @@ public: } } else if (cn == 3) - for ( ; dx < w - 6; dx += 6, S0 += 12, S1 += 12, D += 6) + for ( ; dx <= w - 6; dx += 6, S0 += 12, S1 += 12, D += 6) { __m128i r0 = _mm_loadu_si128((const __m128i*)S0); __m128i r1 = _mm_loadu_si128((const __m128i*)S1); @@ -1310,7 +1310,7 @@ public: else { CV_Assert(cn == 4); - for ( ; dx < w - 8; dx += 8, S0 += 16, S1 += 16, D += 8) + for ( ; dx <= w - 8; dx += 8, S0 += 16, S1 += 16, D += 8) { __m128i r0 = _mm_loadu_si128((const __m128i*)S0); __m128i r1 = _mm_loadu_si128((const __m128i*)S1); @@ -1368,7 +1368,7 @@ public: if (cn == 1) { - for ( ; dx < w - 4; dx += 4, S0 += 8, S1 += 8, D += 4) + for ( ; dx <= w - 4; dx += 4, S0 += 8, S1 += 8, D += 4) { __m128i r0 = _mm_loadu_si128((const __m128i*)S0); __m128i r1 = _mm_loadu_si128((const __m128i*)S1); @@ -1383,7 +1383,7 @@ public: } } else if (cn == 3) - for ( ; dx < w - 3; dx += 3, S0 += 6, S1 += 6, D += 3) + for ( ; dx <= w - 3; dx += 3, S0 += 6, S1 += 6, D += 3) { __m128i r0 = _mm_loadu_si128((const __m128i*)S0); __m128i r1 = _mm_loadu_si128((const __m128i*)S1); @@ -1402,7 +1402,7 @@ public: else { CV_Assert(cn == 4); - for ( ; dx < w - 4; dx += 4, S0 += 8, S1 += 8, D += 4) + for ( ; dx <= w - 4; dx += 4, S0 += 8, S1 += 8, D += 4) { __m128i r0 = _mm_loadu_si128((const __m128i*)S0); __m128i r1 = _mm_loadu_si128((const __m128i*)S1); From a319bae6e2fdb0ced87b54bd8b536c42597c60fe Mon Sep 17 00:00:00 2001 From: Ilya Lavrenov Date: Mon, 17 Dec 2012 16:56:37 +0400 Subject: [PATCH 8/8] added #undef _mm_packus_epi32 --- modules/imgproc/src/imgwarp.cpp | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/modules/imgproc/src/imgwarp.cpp b/modules/imgproc/src/imgwarp.cpp index ede660417f..f536623b8f 100644 --- a/modules/imgproc/src/imgwarp.cpp +++ b/modules/imgproc/src/imgwarp.cpp @@ -1271,7 +1271,7 @@ public: if (cn == 1) { __m128i masklow = _mm_set1_epi16(0x00ff); - for ( ; dx <= w - 8; dx += 8, S0 += 16, S1 += 16, D += 8) + for ( ; dx < w - 8; dx += 8, S0 += 16, S1 += 16, D += 8) { __m128i r0 = _mm_loadu_si128((const __m128i*)S0); __m128i r1 = _mm_loadu_si128((const __m128i*)S1); @@ -1285,7 +1285,7 @@ public: } } else if (cn == 3) - for ( ; dx <= w - 6; dx += 6, S0 += 12, S1 += 12, D += 6) + for ( ; dx < w - 6; dx += 6, S0 += 12, S1 += 12, D += 6) { __m128i r0 = _mm_loadu_si128((const __m128i*)S0); __m128i r1 = _mm_loadu_si128((const __m128i*)S1); @@ -1310,7 +1310,7 @@ public: else { CV_Assert(cn == 4); - for ( ; dx <= w - 8; dx += 8, S0 += 16, S1 += 16, D += 8) + for ( ; dx < w - 8; dx += 8, S0 += 16, S1 += 16, D += 8) { __m128i r0 = _mm_loadu_si128((const __m128i*)S0); __m128i r1 = _mm_loadu_si128((const __m128i*)S1); @@ -1364,11 +1364,11 @@ public: __m128i zero = _mm_setzero_si128(); __m128i delta2 = _mm_set1_epi32(2); -#define _mm_packus_epi32(a, zero) _mm_packs_epi32(_mm_srai_epi32(_mm_slli_epi32(a, 16), 16), zero); +#define _mm_packus_epi32(a, zero) _mm_packs_epi32(_mm_srai_epi32(_mm_slli_epi32(a, 16), 16), zero) if (cn == 1) { - for ( ; dx <= w - 4; dx += 4, S0 += 8, S1 += 8, D += 4) + for ( ; dx < w - 4; dx += 4, S0 += 8, S1 += 8, D += 4) { __m128i r0 = _mm_loadu_si128((const __m128i*)S0); __m128i r1 = _mm_loadu_si128((const __m128i*)S1); @@ -1383,7 +1383,7 @@ public: } } else if (cn == 3) - for ( ; dx <= w - 3; dx += 3, S0 += 6, S1 += 6, D += 3) + for ( ; dx < w - 3; dx += 3, S0 += 6, S1 += 6, D += 3) { __m128i r0 = _mm_loadu_si128((const __m128i*)S0); __m128i r1 = _mm_loadu_si128((const __m128i*)S1); @@ -1402,7 +1402,7 @@ public: else { CV_Assert(cn == 4); - for ( ; dx <= w - 4; dx += 4, S0 += 8, S1 += 8, D += 4) + for ( ; dx < w - 4; dx += 4, S0 += 8, S1 += 8, D += 4) { __m128i r0 = _mm_loadu_si128((const __m128i*)S0); __m128i r1 = _mm_loadu_si128((const __m128i*)S1); @@ -1420,6 +1420,8 @@ public: } } +#undef _mm_packus_epi32 + return dx; }