potok/opencv
1
0
Fork 0
mirror of https://github.com/opencv/opencv.git synced 2026-07-29 23:33:05 +04:00
Code Packages Releases Wiki Activity

Merge remote-tracking branch 'upstream/3.4' into merge-3.4

Browse Source
This commit is contained in:
Alexander Alekhin
2018-07-31 21:35:00 +03:00
parent 57bead3a5e 67f79aabdd
commit 82c477c9f7
71 changed files with 2658 additions and 1392 deletions
Download Patch File Download Diff File Expand all files Collapse all files
modules/core/include/opencv2/core/hal/intrin.hpp
+17 -4
View File
@@ -60,6 +60,17 @@
// access from within opencv code more accessible
namespace cv {
namespace hal {
enum StoreMode
{
STORE_UNALIGNED = 0,
STORE_ALIGNED = 1,
STORE_ALIGNED_NOCACHE = 2
};
}
template<typename _Tp> struct V_TypeTraits
{
};
@@ -154,7 +165,7 @@ using namespace CV_CPU_OPTIMIZATION_HAL_NAMESPACE;
// but some of AVX2 intrinsics get v256_ prefix instead of v_, e.g. v256_load() vs v_load().
// Correspondingly, the wide intrinsics (which are mapped to the "widest"
// available instruction set) will get vx_ prefix
// (and will be mapped to v256_ counterparts) (e.g. vx_load() => v245_load())
// (and will be mapped to v256_ counterparts) (e.g. vx_load() => v256_load())
#if CV_AVX2
#include "opencv2/core/hal/intrin_avx.hpp"
@@ -214,14 +225,16 @@ CV_CPU_OPTIMIZATION_HAL_NAMESPACE_BEGIN
inline vtyp vx_setzero_##short_typ() { return prefix##_setzero_##short_typ(); } \
inline vtyp vx_##loadsfx(const typ* ptr) { return prefix##_##loadsfx(ptr); } \
inline vtyp vx_##loadsfx##_aligned(const typ* ptr) { return prefix##_##loadsfx##_aligned(ptr); } \
inline vtyp vx_##loadsfx##_low(const typ* ptr) { return prefix##_##loadsfx##_low(ptr); } \
inline vtyp vx_##loadsfx##_halves(const typ* ptr0, const typ* ptr1) { return prefix##_##loadsfx##_halves(ptr0, ptr1); } \
inline void vx_store(typ* ptr, const vtyp& v) { return v_store(ptr, v); } \
inline void vx_store_aligned(typ* ptr, const vtyp& v) { return v_store_aligned(ptr, v); }
#define CV_INTRIN_DEFINE_WIDE_LOAD_EXPAND(typ, wtyp, prefix) \
inline wtyp vx_load_expand(const typ* ptr) { return prefix##_load_expand(ptr); }
inline wtyp vx_load_expand(const typ* ptr) { return prefix##_load_expand(ptr); }
#define CV_INTRIN_DEFINE_WIDE_LOAD_EXPAND_Q(typ, qtyp, prefix) \
inline qtyp vx_load_expand_q(const typ* ptr) { return prefix##_load_expand_q(ptr); }
inline qtyp vx_load_expand_q(const typ* ptr) { return prefix##_load_expand_q(ptr); }
#define CV_INTRIN_DEFINE_WIDE_INTRIN_WITH_EXPAND(typ, vtyp, short_typ, wtyp, qtyp, prefix, loadsfx) \
CV_INTRIN_DEFINE_WIDE_INTRIN(typ, vtyp, short_typ, prefix, loadsfx) \
@@ -316,7 +329,7 @@ template<typename _Tp> struct V_RegTraits
CV_INTRIN_DEFINE_WIDE_INTRIN_ALL_TYPES(v256)
CV_INTRIN_DEFINE_WIDE_INTRIN(double, v_float64, f64, v256, load)
inline void vx_cleanup() { v256_cleanup(); }
#elif CV_SIMD128
#elif CV_SIMD128 || CV_SIMD128_CPP
typedef v_uint8x16 v_uint8;
typedef v_int8x16 v_int8;
typedef v_uint16x8 v_uint16;
modules/core/include/opencv2/core/hal/intrin_avx.hpp
+350 -141
View File
@@ -304,6 +304,17 @@ inline v_float16x16 v256_setall_f16(short val) { return v_float16x16(_mm256_set1
{ _mm256_storeu_si256((__m256i*)ptr, a.val); } \
inline void v_store_aligned(_Tp* ptr, const _Tpvec& a) \
{ _mm256_store_si256((__m256i*)ptr, a.val); } \
inline void v_store_aligned_nocache(_Tp* ptr, const _Tpvec& a) \
{ _mm256_stream_si256((__m256i*)ptr, a.val); } \
inline void v_store(_Tp* ptr, const _Tpvec& a, hal::StoreMode mode) \
{ \
if( mode == hal::STORE_UNALIGNED ) \
_mm256_storeu_si256((__m256i*)ptr, a.val); \
else if( mode == hal::STORE_ALIGNED_NOCACHE ) \
_mm256_stream_si256((__m256i*)ptr, a.val); \
else \
_mm256_store_si256((__m256i*)ptr, a.val); \
} \
inline void v_store_low(_Tp* ptr, const _Tpvec& a) \
{ _mm_storeu_si128((__m128i*)ptr, _v256_extract_low(a.val)); } \
inline void v_store_high(_Tp* ptr, const _Tpvec& a) \
@@ -338,6 +349,17 @@ OPENCV_HAL_IMPL_AVX_LOADSTORE(v_int64x4, int64)
{ _mm256_storeu_##suffix(ptr, a.val); } \
inline void v_store_aligned(_Tp* ptr, const _Tpvec& a) \
{ _mm256_store_##suffix(ptr, a.val); } \
inline void v_store_aligned_nocache(_Tp* ptr, const _Tpvec& a) \
{ _mm256_stream_##suffix(ptr, a.val); } \
inline void v_store(_Tp* ptr, const _Tpvec& a, hal::StoreMode mode) \
{ \
if( mode == hal::STORE_UNALIGNED ) \
_mm256_storeu_##suffix(ptr, a.val); \
else if( mode == hal::STORE_ALIGNED_NOCACHE ) \
_mm256_stream_##suffix(ptr, a.val); \
else \
_mm256_store_##suffix(ptr, a.val); \
} \
inline void v_store_low(_Tp* ptr, const _Tpvec& a) \
{ _mm_storeu_##suffix(ptr, _v256_extract_low(a.val)); } \
inline void v_store_high(_Tp* ptr, const _Tpvec& a) \
@@ -407,6 +429,11 @@ inline v_float16x16 v256_load_f16(const short* ptr)
inline v_float16x16 v256_load_f16_aligned(const short* ptr)
{ return v_float16x16(_mm256_load_si256((const __m256i*)ptr)); }
inline v_float16x16 v256_load_f16_low(const short* ptr)
{ return v_float16x16(v256_load_low(ptr).val); }
inline v_float16x16 v256_load_f16_halves(const short* ptr0, const short* ptr1)
{ return v_float16x16(v256_load_halves(ptr0, ptr1).val); }
inline void v_store(short* ptr, const v_float16x16& a)
{ _mm256_storeu_si256((__m256i*)ptr, a.val); }
inline void v_store_aligned(short* ptr, const v_float16x16& a)
@@ -819,94 +846,80 @@ OPENCV_HAL_IMPL_AVX_BIN_FUNC(v_max, v_float64x4, _mm256_max_pd)
template<int imm>
inline v_uint8x32 v_rotate_left(const v_uint8x32& a, const v_uint8x32& b)
{
enum {IMM_R = (16 - imm) & 0xFF};
enum {IMM_R2 = (32 - imm) & 0xFF};
if (imm == 0) return a;
if (imm == 32) return b;
if (imm > 32) return v_uint8x32();
__m256i swap = _mm256_permute2x128_si256(a.val, b.val, 0x03);
switch(imm)
{
case 0: return a;
case 32: return b;
case 16: return v_uint8x32(swap);
}
if (imm < 16) return v_uint8x32(_mm256_alignr_epi8(a.val, swap, 16 - imm));
if (imm < 32) return v_uint8x32(_mm256_alignr_epi8(swap, b.val, 32 - imm));
return v_uint8x32();
if (imm == 16) return v_uint8x32(swap);
if (imm < 16) return v_uint8x32(_mm256_alignr_epi8(a.val, swap, IMM_R));
return v_uint8x32(_mm256_alignr_epi8(swap, b.val, IMM_R2)); // imm < 32
}
template<int imm>
inline v_uint8x32 v_rotate_right(const v_uint8x32& a, const v_uint8x32& b)
{
enum {IMM_L = (imm - 16) & 0xFF};
if (imm == 0) return a;
if (imm == 32) return b;
if (imm > 32) return v_uint8x32();
__m256i swap = _mm256_permute2x128_si256(a.val, b.val, 0x21);
switch(imm)
{
case 0: return a;
case 32: return b;
case 16: return v_uint8x32(swap);
}
if (imm < 16) return v_uint8x32(_mm256_alignr_epi8(swap, a.val, imm));
if (imm < 32) return v_uint8x32(_mm256_alignr_epi8(b.val, swap, imm - 16));
return v_uint8x32();
if (imm == 16) return v_uint8x32(swap);
if (imm < 16) return v_uint8x32(_mm256_alignr_epi8(swap, a.val, imm));
return v_uint8x32(_mm256_alignr_epi8(b.val, swap, IMM_L));
}
template<int imm>
inline v_uint8x32 v_rotate_left(const v_uint8x32& a)
{
v_uint8x32 res;
enum {IMM_L = (imm - 16) & 0xFF};
enum {IMM_R = (16 - imm) & 0xFF};
if (imm == 0) return a;
if (imm > 32) return v_uint8x32();
// ESAC control[3] ? [127:0] = 0
__m256i swapz = _mm256_permute2x128_si256(a.val, a.val, _MM_SHUFFLE(0, 0, 2, 0));
if (imm == 0)
return a;
if (imm == 16)
res.val = swapz;
else if (imm < 16)
res.val = _mm256_alignr_epi8(a.val, swapz, 16 - imm);
else if (imm < 32)
res.val = _mm256_slli_si256(swapz, imm - 16);
else
return v_uint8x32();
return res;
if (imm == 16) return v_uint8x32(swapz);
if (imm < 16) return v_uint8x32(_mm256_alignr_epi8(a.val, swapz, IMM_R));
return v_uint8x32(_mm256_slli_si256(swapz, IMM_L));
}
template<int imm>
inline v_uint8x32 v_rotate_right(const v_uint8x32& a)
{
v_uint8x32 res;
enum {IMM_L = (imm - 16) & 0xFF};
if (imm == 0) return a;
if (imm > 32) return v_uint8x32();
// ESAC control[3] ? [127:0] = 0
__m256i swapz = _mm256_permute2x128_si256(a.val, a.val, _MM_SHUFFLE(2, 0, 0, 1));
if (imm == 0)
return a;
if (imm == 16)
res.val = swapz;
else if (imm < 16)
res.val = _mm256_alignr_epi8(swapz, a.val, imm);
else if (imm < 32)
res.val = _mm256_srli_si256(swapz, imm - 16);
else
return v_uint8x32();
return res;
if (imm == 16) return v_uint8x32(swapz);
if (imm < 16) return v_uint8x32(_mm256_alignr_epi8(swapz, a.val, imm));
return v_uint8x32(_mm256_srli_si256(swapz, IMM_L));
}
#define OPENCV_HAL_IMPL_AVX_ROTATE_CAST(intrin, _Tpvec, cast) \
template<int imm> \
inline _Tpvec intrin(const _Tpvec& a, const _Tpvec& b) \
{ \
const int w = sizeof(typename _Tpvec::lane_type); \
v_uint8x32 ret = intrin<imm*w>(v_reinterpret_as_u8(a), \
v_reinterpret_as_u8(b)); \
return _Tpvec(cast(ret.val)); \
} \
template<int imm> \
inline _Tpvec intrin(const _Tpvec& a) \
{ \
const int w = sizeof(typename _Tpvec::lane_type); \
v_uint8x32 ret = intrin<imm*w>(v_reinterpret_as_u8(a)); \
return _Tpvec(cast(ret.val)); \
#define OPENCV_HAL_IMPL_AVX_ROTATE_CAST(intrin, _Tpvec, cast) \
template<int imm> \
inline _Tpvec intrin(const _Tpvec& a, const _Tpvec& b) \
{ \
enum {IMMxW = imm * sizeof(typename _Tpvec::lane_type)}; \
v_uint8x32 ret = intrin<IMMxW>(v_reinterpret_as_u8(a), \
v_reinterpret_as_u8(b)); \
return _Tpvec(cast(ret.val)); \
} \
template<int imm> \
inline _Tpvec intrin(const _Tpvec& a) \
{ \
enum {IMMxW = imm * sizeof(typename _Tpvec::lane_type)}; \
v_uint8x32 ret = intrin<IMMxW>(v_reinterpret_as_u8(a)); \
return _Tpvec(cast(ret.val)); \
}
#define OPENCV_HAL_IMPL_AVX_ROTATE(_Tpvec) \
@@ -1616,7 +1629,7 @@ inline void v_load_deinterleave( const uchar* ptr, v_uint8x32& a, v_uint8x32& b
__m256i ab0 = _mm256_loadu_si256((const __m256i*)ptr);
__m256i ab1 = _mm256_loadu_si256((const __m256i*)(ptr + 32));
static const __m256i sh = _mm256_setr_epi8(0, 2, 4, 6, 8, 10, 12, 14, 1, 3, 5, 7, 9, 11, 13, 15,
const __m256i sh = _mm256_setr_epi8(0, 2, 4, 6, 8, 10, 12, 14, 1, 3, 5, 7, 9, 11, 13, 15,
0, 2, 4, 6, 8, 10, 12, 14, 1, 3, 5, 7, 9, 11, 13, 15);
__m256i p0 = _mm256_shuffle_epi8(ab0, sh);
__m256i p1 = _mm256_shuffle_epi8(ab1, sh);
@@ -1633,7 +1646,7 @@ inline void v_load_deinterleave( const ushort* ptr, v_uint16x16& a, v_uint16x16&
__m256i ab0 = _mm256_loadu_si256((const __m256i*)ptr);
__m256i ab1 = _mm256_loadu_si256((const __m256i*)(ptr + 16));
static const __m256i sh = _mm256_setr_epi8(0, 1, 4, 5, 8, 9, 12, 13, 2, 3, 6, 7, 10, 11, 14, 15,
const __m256i sh = _mm256_setr_epi8(0, 1, 4, 5, 8, 9, 12, 13, 2, 3, 6, 7, 10, 11, 14, 15,
0, 1, 4, 5, 8, 9, 12, 13, 2, 3, 6, 7, 10, 11, 14, 15);
__m256i p0 = _mm256_shuffle_epi8(ab0, sh);
__m256i p1 = _mm256_shuffle_epi8(ab1, sh);
@@ -1683,16 +1696,16 @@ inline void v_load_deinterleave( const uchar* ptr, v_uint8x32& b, v_uint8x32& g,
__m256i s02_low = _mm256_permute2x128_si256(bgr0, bgr2, 0 + 2*16);
__m256i s02_high = _mm256_permute2x128_si256(bgr0, bgr2, 1 + 3*16);
static const __m256i m0 = _mm256_setr_epi8(0, 0, -1, 0, 0, -1, 0, 0, -1, 0, 0, -1, 0, 0, -1, 0,
const __m256i m0 = _mm256_setr_epi8(0, 0, -1, 0, 0, -1, 0, 0, -1, 0, 0, -1, 0, 0, -1, 0,
0, -1, 0, 0, -1, 0, 0, -1, 0, 0, -1, 0, 0, -1, 0, 0);
static const __m256i m1 = _mm256_setr_epi8(0, -1, 0, 0, -1, 0, 0, -1, 0, 0, -1, 0, 0, -1, 0, 0,
const __m256i m1 = _mm256_setr_epi8(0, -1, 0, 0, -1, 0, 0, -1, 0, 0, -1, 0, 0, -1, 0, 0,
-1, 0, 0, -1, 0, 0, -1, 0, 0, -1, 0, 0, -1, 0, 0, -1);
__m256i b0 = _mm256_blendv_epi8(_mm256_blendv_epi8(s02_low, s02_high, m0), bgr1, m1);
__m256i g0 = _mm256_blendv_epi8(_mm256_blendv_epi8(s02_high, s02_low, m1), bgr1, m0);
__m256i r0 = _mm256_blendv_epi8(_mm256_blendv_epi8(bgr1, s02_low, m0), s02_high, m1);
static const __m256i
const __m256i
sh_b = _mm256_setr_epi8(0, 3, 6, 9, 12, 15, 2, 5, 8, 11, 14, 1, 4, 7, 10, 13,
0, 3, 6, 9, 12, 15, 2, 5, 8, 11, 14, 1, 4, 7, 10, 13),
sh_g = _mm256_setr_epi8(1, 4, 7, 10, 13, 0, 3, 6, 9, 12, 15, 2, 5, 8, 11, 14,
@@ -1717,18 +1730,18 @@ inline void v_load_deinterleave( const ushort* ptr, v_uint16x16& b, v_uint16x16&
__m256i s02_low = _mm256_permute2x128_si256(bgr0, bgr2, 0 + 2*16);
__m256i s02_high = _mm256_permute2x128_si256(bgr0, bgr2, 1 + 3*16);
static const __m256i m0 = _mm256_setr_epi8(0, 0, -1, -1, 0, 0, 0, 0, -1, -1, 0, 0, 0, 0, -1, -1,
const __m256i m0 = _mm256_setr_epi8(0, 0, -1, -1, 0, 0, 0, 0, -1, -1, 0, 0, 0, 0, -1, -1,
0, 0, 0, 0, -1, -1, 0, 0, 0, 0, -1, -1, 0, 0, 0, 0);
static const __m256i m1 = _mm256_setr_epi8(0, 0, 0, 0, -1, -1, 0, 0, 0, 0, -1, -1, 0, 0, 0, 0,
const __m256i m1 = _mm256_setr_epi8(0, 0, 0, 0, -1, -1, 0, 0, 0, 0, -1, -1, 0, 0, 0, 0,
-1, -1, 0, 0, 0, 0, -1, -1, 0, 0, 0, 0, -1, -1, 0, 0);
__m256i b0 = _mm256_blendv_epi8(_mm256_blendv_epi8(s02_low, s02_high, m0), bgr1, m1);
__m256i g0 = _mm256_blendv_epi8(_mm256_blendv_epi8(bgr1, s02_low, m0), s02_high, m1);
__m256i r0 = _mm256_blendv_epi8(_mm256_blendv_epi8(s02_high, s02_low, m1), bgr1, m0);
static const __m256i sh_b = _mm256_setr_epi8(0, 1, 6, 7, 12, 13, 2, 3, 8, 9, 14, 15, 4, 5, 10, 11,
const __m256i sh_b = _mm256_setr_epi8(0, 1, 6, 7, 12, 13, 2, 3, 8, 9, 14, 15, 4, 5, 10, 11,
0, 1, 6, 7, 12, 13, 2, 3, 8, 9, 14, 15, 4, 5, 10, 11);
static const __m256i sh_g = _mm256_setr_epi8(2, 3, 8, 9, 14, 15, 4, 5, 10, 11, 0, 1, 6, 7, 12, 13,
const __m256i sh_g = _mm256_setr_epi8(2, 3, 8, 9, 14, 15, 4, 5, 10, 11, 0, 1, 6, 7, 12, 13,
2, 3, 8, 9, 14, 15, 4, 5, 10, 11, 0, 1, 6, 7, 12, 13);
static const __m256i sh_r = _mm256_setr_epi8(4, 5, 10, 11, 0, 1, 6, 7, 12, 13, 2, 3, 8, 9, 14, 15,
const __m256i sh_r = _mm256_setr_epi8(4, 5, 10, 11, 0, 1, 6, 7, 12, 13, 2, 3, 8, 9, 14, 15,
4, 5, 10, 11, 0, 1, 6, 7, 12, 13, 2, 3, 8, 9, 14, 15);
b0 = _mm256_shuffle_epi8(b0, sh_b);
g0 = _mm256_shuffle_epi8(g0, sh_g);
@@ -1785,7 +1798,7 @@ inline void v_load_deinterleave( const uchar* ptr, v_uint8x32& b, v_uint8x32& g,
__m256i bgr1 = _mm256_loadu_si256((const __m256i*)(ptr + 32));
__m256i bgr2 = _mm256_loadu_si256((const __m256i*)(ptr + 64));
__m256i bgr3 = _mm256_loadu_si256((const __m256i*)(ptr + 96));
static const __m256i sh = _mm256_setr_epi8(0, 4, 8, 12, 1, 5, 9, 13, 2, 6, 10, 14, 3, 7, 11, 15,
const __m256i sh = _mm256_setr_epi8(0, 4, 8, 12, 1, 5, 9, 13, 2, 6, 10, 14, 3, 7, 11, 15,
0, 4, 8, 12, 1, 5, 9, 13, 2, 6, 10, 14, 3, 7, 11, 15);
__m256i p0 = _mm256_shuffle_epi8(bgr0, sh);
@@ -1820,7 +1833,7 @@ inline void v_load_deinterleave( const ushort* ptr, v_uint16x16& b, v_uint16x16&
__m256i bgr1 = _mm256_loadu_si256((const __m256i*)(ptr + 16));
__m256i bgr2 = _mm256_loadu_si256((const __m256i*)(ptr + 32));
__m256i bgr3 = _mm256_loadu_si256((const __m256i*)(ptr + 48));
static const __m256i sh = _mm256_setr_epi8(0, 1, 8, 9, 2, 3, 10, 11, 4, 5, 12, 13, 6, 7, 14, 15,
const __m256i sh = _mm256_setr_epi8(0, 1, 8, 9, 2, 3, 10, 11, 4, 5, 12, 13, 6, 7, 14, 15,
0, 1, 8, 9, 2, 3, 10, 11, 4, 5, 12, 13, 6, 7, 14, 15);
__m256i p0 = _mm256_shuffle_epi8(bgr0, sh);
__m256i p1 = _mm256_shuffle_epi8(bgr1, sh);
@@ -1901,7 +1914,8 @@ inline void v_load_deinterleave( const uint64* ptr, v_uint64x4& b, v_uint64x4& g
///////////////////////////// store interleave /////////////////////////////////////
inline void v_store_interleave( uchar* ptr, const v_uint8x32& x, const v_uint8x32& y )
inline void v_store_interleave( uchar* ptr, const v_uint8x32& x, const v_uint8x32& y,
hal::StoreMode mode=hal::STORE_UNALIGNED )
{
__m256i xy_l = _mm256_unpacklo_epi8(x.val, y.val);
__m256i xy_h = _mm256_unpackhi_epi8(x.val, y.val);
@@ -1909,11 +1923,25 @@ inline void v_store_interleave( uchar* ptr, const v_uint8x32& x, const v_uint8x3
__m256i xy0 = _mm256_permute2x128_si256(xy_l, xy_h, 0 + 2*16);
__m256i xy1 = _mm256_permute2x128_si256(xy_l, xy_h, 1 + 3*16);
_mm256_storeu_si256((__m256i*)ptr, xy0);
_mm256_storeu_si256((__m256i*)(ptr + 32), xy1);
if( mode == hal::STORE_ALIGNED_NOCACHE )
{
_mm256_stream_si256((__m256i*)ptr, xy0);
_mm256_stream_si256((__m256i*)(ptr + 32), xy1);
}
else if( mode == hal::STORE_ALIGNED )
{
_mm256_store_si256((__m256i*)ptr, xy0);
_mm256_store_si256((__m256i*)(ptr + 32), xy1);
}
else
{
_mm256_storeu_si256((__m256i*)ptr, xy0);
_mm256_storeu_si256((__m256i*)(ptr + 32), xy1);
}
}
inline void v_store_interleave( ushort* ptr, const v_uint16x16& x, const v_uint16x16& y )
inline void v_store_interleave( ushort* ptr, const v_uint16x16& x, const v_uint16x16& y,
hal::StoreMode mode=hal::STORE_UNALIGNED )
{
__m256i xy_l = _mm256_unpacklo_epi16(x.val, y.val);
__m256i xy_h = _mm256_unpackhi_epi16(x.val, y.val);
@@ -1921,11 +1949,25 @@ inline void v_store_interleave( ushort* ptr, const v_uint16x16& x, const v_uint1
__m256i xy0 = _mm256_permute2x128_si256(xy_l, xy_h, 0 + 2*16);
__m256i xy1 = _mm256_permute2x128_si256(xy_l, xy_h, 1 + 3*16);
_mm256_storeu_si256((__m256i*)ptr, xy0);
_mm256_storeu_si256((__m256i*)(ptr + 16), xy1);
if( mode == hal::STORE_ALIGNED_NOCACHE )
{
_mm256_stream_si256((__m256i*)ptr, xy0);
_mm256_stream_si256((__m256i*)(ptr + 16), xy1);
}
else if( mode == hal::STORE_ALIGNED )
{
_mm256_store_si256((__m256i*)ptr, xy0);
_mm256_store_si256((__m256i*)(ptr + 16), xy1);
}
else
{
_mm256_storeu_si256((__m256i*)ptr, xy0);
_mm256_storeu_si256((__m256i*)(ptr + 16), xy1);
}
}
inline void v_store_interleave( unsigned* ptr, const v_uint32x8& x, const v_uint32x8& y )
inline void v_store_interleave( unsigned* ptr, const v_uint32x8& x, const v_uint32x8& y,
hal::StoreMode mode=hal::STORE_UNALIGNED )
{
__m256i xy_l = _mm256_unpacklo_epi32(x.val, y.val);
__m256i xy_h = _mm256_unpackhi_epi32(x.val, y.val);
@@ -1933,11 +1975,25 @@ inline void v_store_interleave( unsigned* ptr, const v_uint32x8& x, const v_uint
__m256i xy0 = _mm256_permute2x128_si256(xy_l, xy_h, 0 + 2*16);
__m256i xy1 = _mm256_permute2x128_si256(xy_l, xy_h, 1 + 3*16);
_mm256_storeu_si256((__m256i*)ptr, xy0);
_mm256_storeu_si256((__m256i*)(ptr + 8), xy1);
if( mode == hal::STORE_ALIGNED_NOCACHE )
{
_mm256_stream_si256((__m256i*)ptr, xy0);
_mm256_stream_si256((__m256i*)(ptr + 8), xy1);
}
else if( mode == hal::STORE_ALIGNED )
{
_mm256_store_si256((__m256i*)ptr, xy0);
_mm256_store_si256((__m256i*)(ptr + 8), xy1);
}
else
{
_mm256_storeu_si256((__m256i*)ptr, xy0);
_mm256_storeu_si256((__m256i*)(ptr + 8), xy1);
}
}
inline void v_store_interleave( uint64* ptr, const v_uint64x4& x, const v_uint64x4& y )
inline void v_store_interleave( uint64* ptr, const v_uint64x4& x, const v_uint64x4& y,
hal::StoreMode mode=hal::STORE_UNALIGNED )
{
__m256i xy_l = _mm256_unpacklo_epi64(x.val, y.val);
__m256i xy_h = _mm256_unpackhi_epi64(x.val, y.val);
@@ -1945,19 +2001,33 @@ inline void v_store_interleave( uint64* ptr, const v_uint64x4& x, const v_uint64
__m256i xy0 = _mm256_permute2x128_si256(xy_l, xy_h, 0 + 2*16);
__m256i xy1 = _mm256_permute2x128_si256(xy_l, xy_h, 1 + 3*16);
_mm256_storeu_si256((__m256i*)ptr, xy0);
_mm256_storeu_si256((__m256i*)(ptr + 4), xy1);
if( mode == hal::STORE_ALIGNED_NOCACHE )
{
_mm256_stream_si256((__m256i*)ptr, xy0);
_mm256_stream_si256((__m256i*)(ptr + 4), xy1);
}
else if( mode == hal::STORE_ALIGNED )
{
_mm256_store_si256((__m256i*)ptr, xy0);
_mm256_store_si256((__m256i*)(ptr + 4), xy1);
}
else
{
_mm256_storeu_si256((__m256i*)ptr, xy0);
_mm256_storeu_si256((__m256i*)(ptr + 4), xy1);
}
}
inline void v_store_interleave( uchar* ptr, const v_uint8x32& b, const v_uint8x32& g, const v_uint8x32& r )
inline void v_store_interleave( uchar* ptr, const v_uint8x32& b, const v_uint8x32& g, const v_uint8x32& r,
hal::StoreMode mode=hal::STORE_UNALIGNED )
{
static const __m256i sh_b = _mm256_setr_epi8(
const __m256i sh_b = _mm256_setr_epi8(
0, 11, 6, 1, 12, 7, 2, 13, 8, 3, 14, 9, 4, 15, 10, 5,
0, 11, 6, 1, 12, 7, 2, 13, 8, 3, 14, 9, 4, 15, 10, 5);
static const __m256i sh_g = _mm256_setr_epi8(
const __m256i sh_g = _mm256_setr_epi8(
5, 0, 11, 6, 1, 12, 7, 2, 13, 8, 3, 14, 9, 4, 15, 10,
5, 0, 11, 6, 1, 12, 7, 2, 13, 8, 3, 14, 9, 4, 15, 10);
static const __m256i sh_r = _mm256_setr_epi8(
const __m256i sh_r = _mm256_setr_epi8(
10, 5, 0, 11, 6, 1, 12, 7, 2, 13, 8, 3, 14, 9, 4, 15,
10, 5, 0, 11, 6, 1, 12, 7, 2, 13, 8, 3, 14, 9, 4, 15);
@@ -1965,9 +2035,9 @@ inline void v_store_interleave( uchar* ptr, const v_uint8x32& b, const v_uint8x3
__m256i g0 = _mm256_shuffle_epi8(g.val, sh_g);
__m256i r0 = _mm256_shuffle_epi8(r.val, sh_r);
static const __m256i m0 = _mm256_setr_epi8(0, -1, 0, 0, -1, 0, 0, -1, 0, 0, -1, 0, 0, -1, 0, 0,
const __m256i m0 = _mm256_setr_epi8(0, -1, 0, 0, -1, 0, 0, -1, 0, 0, -1, 0, 0, -1, 0, 0,
0, -1, 0, 0, -1, 0, 0, -1, 0, 0, -1, 0, 0, -1, 0, 0);
static const __m256i m1 = _mm256_setr_epi8(0, 0, -1, 0, 0, -1, 0, 0, -1, 0, 0, -1, 0, 0, -1, 0,
const __m256i m1 = _mm256_setr_epi8(0, 0, -1, 0, 0, -1, 0, 0, -1, 0, 0, -1, 0, 0, -1, 0,
0, 0, -1, 0, 0, -1, 0, 0, -1, 0, 0, -1, 0, 0, -1, 0);
__m256i p0 = _mm256_blendv_epi8(_mm256_blendv_epi8(b0, g0, m0), r0, m1);
@@ -1978,20 +2048,36 @@ inline void v_store_interleave( uchar* ptr, const v_uint8x32& b, const v_uint8x3
__m256i bgr1 = _mm256_permute2x128_si256(p2, p0, 0 + 3*16);
__m256i bgr2 = _mm256_permute2x128_si256(p1, p2, 1 + 3*16);
_mm256_storeu_si256((__m256i*)ptr, bgr0);
_mm256_storeu_si256((__m256i*)(ptr + 32), bgr1);
_mm256_storeu_si256((__m256i*)(ptr + 64), bgr2);
if( mode == hal::STORE_ALIGNED_NOCACHE )
{
_mm256_stream_si256((__m256i*)ptr, bgr0);
_mm256_stream_si256((__m256i*)(ptr + 32), bgr1);
_mm256_stream_si256((__m256i*)(ptr + 64), bgr2);
}
else if( mode == hal::STORE_ALIGNED )
{
_mm256_store_si256((__m256i*)ptr, bgr0);
_mm256_store_si256((__m256i*)(ptr + 32), bgr1);
_mm256_store_si256((__m256i*)(ptr + 64), bgr2);
}
else
{
_mm256_storeu_si256((__m256i*)ptr, bgr0);
_mm256_storeu_si256((__m256i*)(ptr + 32), bgr1);
_mm256_storeu_si256((__m256i*)(ptr + 64), bgr2);
}
}
inline void v_store_interleave( ushort* ptr, const v_uint16x16& b, const v_uint16x16& g, const v_uint16x16& r )
inline void v_store_interleave( ushort* ptr, const v_uint16x16& b, const v_uint16x16& g, const v_uint16x16& r,
hal::StoreMode mode=hal::STORE_UNALIGNED )
{
static const __m256i sh_b = _mm256_setr_epi8(
const __m256i sh_b = _mm256_setr_epi8(
0, 1, 6, 7, 12, 13, 2, 3, 8, 9, 14, 15, 4, 5, 10, 11,
0, 1, 6, 7, 12, 13, 2, 3, 8, 9, 14, 15, 4, 5, 10, 11);
static const __m256i sh_g = _mm256_setr_epi8(
const __m256i sh_g = _mm256_setr_epi8(
10, 11, 0, 1, 6, 7, 12, 13, 2, 3, 8, 9, 14, 15, 4, 5,
10, 11, 0, 1, 6, 7, 12, 13, 2, 3, 8, 9, 14, 15, 4, 5);
static const __m256i sh_r = _mm256_setr_epi8(
const __m256i sh_r = _mm256_setr_epi8(
4, 5, 10, 11, 0, 1, 6, 7, 12, 13, 2, 3, 8, 9, 14, 15,
4, 5, 10, 11, 0, 1, 6, 7, 12, 13, 2, 3, 8, 9, 14, 15);
@@ -1999,9 +2085,9 @@ inline void v_store_interleave( ushort* ptr, const v_uint16x16& b, const v_uint1
__m256i g0 = _mm256_shuffle_epi8(g.val, sh_g);
__m256i r0 = _mm256_shuffle_epi8(r.val, sh_r);
static const __m256i m0 = _mm256_setr_epi8(0, 0, -1, -1, 0, 0, 0, 0, -1, -1, 0, 0, 0, 0, -1, -1,
const __m256i m0 = _mm256_setr_epi8(0, 0, -1, -1, 0, 0, 0, 0, -1, -1, 0, 0, 0, 0, -1, -1,
0, 0, 0, 0, -1, -1, 0, 0, 0, 0, -1, -1, 0, 0, 0, 0);
static const __m256i m1 = _mm256_setr_epi8(0, 0, 0, 0, -1, -1, 0, 0, 0, 0, -1, -1, 0, 0, 0, 0,
const __m256i m1 = _mm256_setr_epi8(0, 0, 0, 0, -1, -1, 0, 0, 0, 0, -1, -1, 0, 0, 0, 0,
-1, -1, 0, 0, 0, 0, -1, -1, 0, 0, 0, 0, -1, -1, 0, 0);
__m256i p0 = _mm256_blendv_epi8(_mm256_blendv_epi8(b0, g0, m0), r0, m1);
@@ -2012,12 +2098,28 @@ inline void v_store_interleave( ushort* ptr, const v_uint16x16& b, const v_uint1
//__m256i bgr1 = p1;
__m256i bgr2 = _mm256_permute2x128_si256(p0, p2, 1 + 3*16);
_mm256_storeu_si256((__m256i*)ptr, bgr0);
_mm256_storeu_si256((__m256i*)(ptr + 16), p1);
_mm256_storeu_si256((__m256i*)(ptr + 32), bgr2);
if( mode == hal::STORE_ALIGNED_NOCACHE )
{
_mm256_stream_si256((__m256i*)ptr, bgr0);
_mm256_stream_si256((__m256i*)(ptr + 16), p1);
_mm256_stream_si256((__m256i*)(ptr + 32), bgr2);
}
else if( mode == hal::STORE_ALIGNED )
{
_mm256_store_si256((__m256i*)ptr, bgr0);
_mm256_store_si256((__m256i*)(ptr + 16), p1);
_mm256_store_si256((__m256i*)(ptr + 32), bgr2);
}
else
{
_mm256_storeu_si256((__m256i*)ptr, bgr0);
_mm256_storeu_si256((__m256i*)(ptr + 16), p1);
_mm256_storeu_si256((__m256i*)(ptr + 32), bgr2);
}
}
inline void v_store_interleave( unsigned* ptr, const v_uint32x8& b, const v_uint32x8& g, const v_uint32x8& r )
inline void v_store_interleave( unsigned* ptr, const v_uint32x8& b, const v_uint32x8& g, const v_uint32x8& r,
hal::StoreMode mode=hal::STORE_UNALIGNED )
{
__m256i b0 = _mm256_shuffle_epi32(b.val, 0x6c);
__m256i g0 = _mm256_shuffle_epi32(g.val, 0xb1);
@@ -2031,12 +2133,28 @@ inline void v_store_interleave( unsigned* ptr, const v_uint32x8& b, const v_uint
//__m256i bgr1 = p2;
__m256i bgr2 = _mm256_permute2x128_si256(p0, p1, 1 + 3*16);
_mm256_storeu_si256((__m256i*)ptr, bgr0);
_mm256_storeu_si256((__m256i*)(ptr + 8), p2);
_mm256_storeu_si256((__m256i*)(ptr + 16), bgr2);
if( mode == hal::STORE_ALIGNED_NOCACHE )
{
_mm256_stream_si256((__m256i*)ptr, bgr0);
_mm256_stream_si256((__m256i*)(ptr + 8), p2);
_mm256_stream_si256((__m256i*)(ptr + 16), bgr2);
}
else if( mode == hal::STORE_ALIGNED )
{
_mm256_store_si256((__m256i*)ptr, bgr0);
_mm256_store_si256((__m256i*)(ptr + 8), p2);
_mm256_store_si256((__m256i*)(ptr + 16), bgr2);
}
else
{
_mm256_storeu_si256((__m256i*)ptr, bgr0);
_mm256_storeu_si256((__m256i*)(ptr + 8), p2);
_mm256_storeu_si256((__m256i*)(ptr + 16), bgr2);
}
}
inline void v_store_interleave( uint64* ptr, const v_uint64x4& b, const v_uint64x4& g, const v_uint64x4& r )
inline void v_store_interleave( uint64* ptr, const v_uint64x4& b, const v_uint64x4& g, const v_uint64x4& r,
hal::StoreMode mode=hal::STORE_UNALIGNED )
{
__m256i s01 = _mm256_unpacklo_epi64(b.val, g.val);
__m256i s12 = _mm256_unpackhi_epi64(g.val, r.val);
@@ -2046,12 +2164,29 @@ inline void v_store_interleave( uint64* ptr, const v_uint64x4& b, const v_uint64
__m256i bgr1 = _mm256_blend_epi32(s01, s12, 0x0f);
__m256i bgr2 = _mm256_permute2x128_si256(s20, s12, 1 + 3*16);
_mm256_storeu_si256((__m256i*)ptr, bgr0);
_mm256_storeu_si256((__m256i*)(ptr + 4), bgr1);
_mm256_storeu_si256((__m256i*)(ptr + 8), bgr2);
if( mode == hal::STORE_ALIGNED_NOCACHE )
{
_mm256_stream_si256((__m256i*)ptr, bgr0);
_mm256_stream_si256((__m256i*)(ptr + 4), bgr1);
_mm256_stream_si256((__m256i*)(ptr + 8), bgr2);
}
else if( mode == hal::STORE_ALIGNED )
{
_mm256_store_si256((__m256i*)ptr, bgr0);
_mm256_store_si256((__m256i*)(ptr + 4), bgr1);
_mm256_store_si256((__m256i*)(ptr + 8), bgr2);
}
else
{
_mm256_storeu_si256((__m256i*)ptr, bgr0);
_mm256_storeu_si256((__m256i*)(ptr + 4), bgr1);
_mm256_storeu_si256((__m256i*)(ptr + 8), bgr2);
}
}
inline void v_store_interleave( uchar* ptr, const v_uint8x32& b, const v_uint8x32& g, const v_uint8x32& r, const v_uint8x32& a )
inline void v_store_interleave( uchar* ptr, const v_uint8x32& b, const v_uint8x32& g,
const v_uint8x32& r, const v_uint8x32& a,
hal::StoreMode mode=hal::STORE_UNALIGNED )
{
__m256i bg0 = _mm256_unpacklo_epi8(b.val, g.val);
__m256i bg1 = _mm256_unpackhi_epi8(b.val, g.val);
@@ -2068,14 +2203,32 @@ inline void v_store_interleave( uchar* ptr, const v_uint8x32& b, const v_uint8x3
__m256i bgra1 = _mm256_permute2x128_si256(bgra2_, bgra3_, 0 + 2*16);
__m256i bgra3 = _mm256_permute2x128_si256(bgra2_, bgra3_, 1 + 3*16);
_mm256_storeu_si256((__m256i*)ptr, bgra0);
_mm256_storeu_si256((__m256i*)(ptr + 32), bgra1);
_mm256_storeu_si256((__m256i*)(ptr + 64), bgra2);
_mm256_storeu_si256((__m256i*)(ptr + 96), bgra3);
if( mode == hal::STORE_ALIGNED_NOCACHE )
{
_mm256_stream_si256((__m256i*)ptr, bgra0);
_mm256_stream_si256((__m256i*)(ptr + 32), bgra1);
_mm256_stream_si256((__m256i*)(ptr + 64), bgra2);
_mm256_stream_si256((__m256i*)(ptr + 96), bgra3);
}
else if( mode == hal::STORE_ALIGNED )
{
_mm256_store_si256((__m256i*)ptr, bgra0);
_mm256_store_si256((__m256i*)(ptr + 32), bgra1);
_mm256_store_si256((__m256i*)(ptr + 64), bgra2);
_mm256_store_si256((__m256i*)(ptr + 96), bgra3);
}
else
{
_mm256_storeu_si256((__m256i*)ptr, bgra0);
_mm256_storeu_si256((__m256i*)(ptr + 32), bgra1);
_mm256_storeu_si256((__m256i*)(ptr + 64), bgra2);
_mm256_storeu_si256((__m256i*)(ptr + 96), bgra3);
}
}
inline void v_store_interleave( ushort* ptr, const v_uint16x16& b, const v_uint16x16& g,
const v_uint16x16& r, const v_uint16x16& a )
const v_uint16x16& r, const v_uint16x16& a,
hal::StoreMode mode=hal::STORE_UNALIGNED )
{
__m256i bg0 = _mm256_unpacklo_epi16(b.val, g.val);
__m256i bg1 = _mm256_unpackhi_epi16(b.val, g.val);
@@ -2092,14 +2245,32 @@ inline void v_store_interleave( ushort* ptr, const v_uint16x16& b, const v_uint1
__m256i bgra1 = _mm256_permute2x128_si256(bgra2_, bgra3_, 0 + 2*16);
__m256i bgra3 = _mm256_permute2x128_si256(bgra2_, bgra3_, 1 + 3*16);
_mm256_storeu_si256((__m256i*)ptr, bgra0);
_mm256_storeu_si256((__m256i*)(ptr + 16), bgra1);
_mm256_storeu_si256((__m256i*)(ptr + 32), bgra2);
_mm256_storeu_si256((__m256i*)(ptr + 48), bgra3);
if( mode == hal::STORE_ALIGNED_NOCACHE )
{
_mm256_stream_si256((__m256i*)ptr, bgra0);
_mm256_stream_si256((__m256i*)(ptr + 16), bgra1);
_mm256_stream_si256((__m256i*)(ptr + 32), bgra2);
_mm256_stream_si256((__m256i*)(ptr + 48), bgra3);
}
else if( mode == hal::STORE_ALIGNED )
{
_mm256_store_si256((__m256i*)ptr, bgra0);
_mm256_store_si256((__m256i*)(ptr + 16), bgra1);
_mm256_store_si256((__m256i*)(ptr + 32), bgra2);
_mm256_store_si256((__m256i*)(ptr + 48), bgra3);
}
else
{
_mm256_storeu_si256((__m256i*)ptr, bgra0);
_mm256_storeu_si256((__m256i*)(ptr + 16), bgra1);
_mm256_storeu_si256((__m256i*)(ptr + 32), bgra2);
_mm256_storeu_si256((__m256i*)(ptr + 48), bgra3);
}
}
inline void v_store_interleave( unsigned* ptr, const v_uint32x8& b, const v_uint32x8& g,
const v_uint32x8& r, const v_uint32x8& a )
const v_uint32x8& r, const v_uint32x8& a,
hal::StoreMode mode=hal::STORE_UNALIGNED )
{
__m256i bg0 = _mm256_unpacklo_epi32(b.val, g.val);
__m256i bg1 = _mm256_unpackhi_epi32(b.val, g.val);
@@ -2116,14 +2287,32 @@ inline void v_store_interleave( unsigned* ptr, const v_uint32x8& b, const v_uint
__m256i bgra1 = _mm256_permute2x128_si256(bgra2_, bgra3_, 0 + 2*16);
__m256i bgra3 = _mm256_permute2x128_si256(bgra2_, bgra3_, 1 + 3*16);
_mm256_storeu_si256((__m256i*)ptr, bgra0);
_mm256_storeu_si256((__m256i*)(ptr + 8), bgra1);
_mm256_storeu_si256((__m256i*)(ptr + 16), bgra2);
_mm256_storeu_si256((__m256i*)(ptr + 24), bgra3);
if( mode == hal::STORE_ALIGNED_NOCACHE )
{
_mm256_stream_si256((__m256i*)ptr, bgra0);
_mm256_stream_si256((__m256i*)(ptr + 8), bgra1);
_mm256_stream_si256((__m256i*)(ptr + 16), bgra2);
_mm256_stream_si256((__m256i*)(ptr + 24), bgra3);
}
else if( mode == hal::STORE_ALIGNED )
{
_mm256_store_si256((__m256i*)ptr, bgra0);
_mm256_store_si256((__m256i*)(ptr + 8), bgra1);
_mm256_store_si256((__m256i*)(ptr + 16), bgra2);
_mm256_store_si256((__m256i*)(ptr + 24), bgra3);
}
else
{
_mm256_storeu_si256((__m256i*)ptr, bgra0);
_mm256_storeu_si256((__m256i*)(ptr + 8), bgra1);
_mm256_storeu_si256((__m256i*)(ptr + 16), bgra2);
_mm256_storeu_si256((__m256i*)(ptr + 24), bgra3);
}
}
inline void v_store_interleave( uint64* ptr, const v_uint64x4& b, const v_uint64x4& g,
const v_uint64x4& r, const v_uint64x4& a )
const v_uint64x4& r, const v_uint64x4& a,
hal::StoreMode mode=hal::STORE_UNALIGNED )
{
__m256i bg0 = _mm256_unpacklo_epi64(b.val, g.val);
__m256i bg1 = _mm256_unpackhi_epi64(b.val, g.val);
@@ -2135,10 +2324,27 @@ inline void v_store_interleave( uint64* ptr, const v_uint64x4& b, const v_uint64
__m256i bgra2 = _mm256_permute2x128_si256(bg0, ra0, 1 + 3*16);
__m256i bgra3 = _mm256_permute2x128_si256(bg1, ra1, 1 + 3*16);
_mm256_storeu_si256((__m256i*)ptr, bgra0);
_mm256_storeu_si256((__m256i*)(ptr + 4), bgra1);
_mm256_storeu_si256((__m256i*)(ptr + 8), bgra2);
_mm256_storeu_si256((__m256i*)(ptr + 12), bgra3);
if( mode == hal::STORE_ALIGNED_NOCACHE )
{
_mm256_stream_si256((__m256i*)ptr, bgra0);
_mm256_stream_si256((__m256i*)(ptr + 4), bgra1);
_mm256_stream_si256((__m256i*)(ptr + 8), bgra2);
_mm256_stream_si256((__m256i*)(ptr + 12), bgra3);
}
else if( mode == hal::STORE_ALIGNED )
{
_mm256_store_si256((__m256i*)ptr, bgra0);
_mm256_store_si256((__m256i*)(ptr + 4), bgra1);
_mm256_store_si256((__m256i*)(ptr + 8), bgra2);
_mm256_store_si256((__m256i*)(ptr + 12), bgra3);
}
else
{
_mm256_storeu_si256((__m256i*)ptr, bgra0);
_mm256_storeu_si256((__m256i*)(ptr + 4), bgra1);
_mm256_storeu_si256((__m256i*)(ptr + 8), bgra2);
_mm256_storeu_si256((__m256i*)(ptr + 12), bgra3);
}
}
#define OPENCV_HAL_IMPL_AVX_LOADSTORE_INTERLEAVE(_Tpvec0, _Tp0, suffix0, _Tpvec1, _Tp1, suffix1) \
@@ -2166,27 +2372,30 @@ inline void v_load_deinterleave( const _Tp0* ptr, _Tpvec0& a0, _Tpvec0& b0, _Tpv
c0 = v_reinterpret_as_##suffix0(c1); \
d0 = v_reinterpret_as_##suffix0(d1); \
} \
inline void v_store_interleave( _Tp0* ptr, const _Tpvec0& a0, const _Tpvec0& b0 ) \
inline void v_store_interleave( _Tp0* ptr, const _Tpvec0& a0, const _Tpvec0& b0, \
hal::StoreMode mode=hal::STORE_UNALIGNED ) \
{ \
_Tpvec1 a1 = v_reinterpret_as_##suffix1(a0); \
_Tpvec1 b1 = v_reinterpret_as_##suffix1(b0); \
v_store_interleave((_Tp1*)ptr, a1, b1); \
v_store_interleave((_Tp1*)ptr, a1, b1, mode); \
} \
inline void v_store_interleave( _Tp0* ptr, const _Tpvec0& a0, const _Tpvec0& b0, const _Tpvec0& c0 ) \
inline void v_store_interleave( _Tp0* ptr, const _Tpvec0& a0, const _Tpvec0& b0, const _Tpvec0& c0, \
hal::StoreMode mode=hal::STORE_UNALIGNED ) \
{ \
_Tpvec1 a1 = v_reinterpret_as_##suffix1(a0); \
_Tpvec1 b1 = v_reinterpret_as_##suffix1(b0); \
_Tpvec1 c1 = v_reinterpret_as_##suffix1(c0); \
v_store_interleave((_Tp1*)ptr, a1, b1, c1); \
v_store_interleave((_Tp1*)ptr, a1, b1, c1, mode); \
} \
inline void v_store_interleave( _Tp0* ptr, const _Tpvec0& a0, const _Tpvec0& b0, \
const _Tpvec0& c0, const _Tpvec0& d0 ) \
const _Tpvec0& c0, const _Tpvec0& d0, \
hal::StoreMode mode=hal::STORE_UNALIGNED ) \
{ \
_Tpvec1 a1 = v_reinterpret_as_##suffix1(a0); \
_Tpvec1 b1 = v_reinterpret_as_##suffix1(b0); \
_Tpvec1 c1 = v_reinterpret_as_##suffix1(c0); \
_Tpvec1 d1 = v_reinterpret_as_##suffix1(d0); \
v_store_interleave((_Tp1*)ptr, a1, b1, c1, d1); \
v_store_interleave((_Tp1*)ptr, a1, b1, c1, d1, mode); \
}
OPENCV_HAL_IMPL_AVX_LOADSTORE_INTERLEAVE(v_int8x32, schar, s8, v_uint8x32, uchar, u8)
modules/core/include/opencv2/core/hal/intrin_cpp.hpp
+20 -3
View File
@@ -1319,7 +1319,8 @@ Scheme:
For all types except 64-bit. */
template<typename _Tp, int n>
inline void v_store_interleave( _Tp* ptr, const v_reg<_Tp, n>& a,
const v_reg<_Tp, n>& b)
const v_reg<_Tp, n>& b,
hal::StoreMode /*mode*/=hal::STORE_UNALIGNED)
{
int i, i2;
for( i = i2 = 0; i < n; i++, i2 += 2 )
@@ -1339,7 +1340,8 @@ Scheme:
For all types except 64-bit. */
template<typename _Tp, int n>
inline void v_store_interleave( _Tp* ptr, const v_reg<_Tp, n>& a,
const v_reg<_Tp, n>& b, const v_reg<_Tp, n>& c)
const v_reg<_Tp, n>& b, const v_reg<_Tp, n>& c,
hal::StoreMode /*mode*/=hal::STORE_UNALIGNED)
{
int i, i3;
for( i = i3 = 0; i < n; i++, i3 += 3 )
@@ -1360,7 +1362,8 @@ Scheme:
For all types except 64-bit. */
template<typename _Tp, int n> inline void v_store_interleave( _Tp* ptr, const v_reg<_Tp, n>& a,
const v_reg<_Tp, n>& b, const v_reg<_Tp, n>& c,
const v_reg<_Tp, n>& d)
const v_reg<_Tp, n>& d,
hal::StoreMode /*mode*/=hal::STORE_UNALIGNED)
{
int i, i4;
for( i = i4 = 0; i < n; i++, i4 += 4 )
@@ -1430,6 +1433,20 @@ inline void v_store_aligned(_Tp* ptr, const v_reg<_Tp, n>& a)
ptr[i] = a.s[i];
}
template<typename _Tp, int n>
inline void v_store_aligned_nocache(_Tp* ptr, const v_reg<_Tp, n>& a)
{
for( int i = 0; i < n; i++ )
ptr[i] = a.s[i];
}
template<typename _Tp, int n>
inline void v_store_aligned(_Tp* ptr, const v_reg<_Tp, n>& a, hal::StoreMode /*mode*/)
{
for( int i = 0; i < n; i++ )
ptr[i] = a.s[i];
}
/** @brief Combine vector from first elements of two vectors
Scheme:
modules/core/include/opencv2/core/hal/intrin_neon.hpp
+24 -6
View File
@@ -319,6 +319,9 @@ static inline void cv_vst1_f16(void* ptr, float16x4_t a)
#endif
}
#ifndef vdup_n_f16
#define vdup_n_f16(v) (float16x4_t){v, v, v, v}
#endif
struct v_float16x8
{
@@ -864,6 +867,10 @@ inline void v_store(_Tp* ptr, const _Tpvec& a) \
{ vst1q_##suffix(ptr, a.val); } \
inline void v_store_aligned(_Tp* ptr, const _Tpvec& a) \
{ vst1q_##suffix(ptr, a.val); } \
inline void v_store_aligned_nocache(_Tp* ptr, const _Tpvec& a) \
{ vst1q_##suffix(ptr, a.val); } \
inline void v_store(_Tp* ptr, const _Tpvec& a, hal::StoreMode /*mode*/) \
{ vst1q_##suffix(ptr, a.val); } \
inline void v_store_low(_Tp* ptr, const _Tpvec& a) \
{ vst1_##suffix(ptr, vget_low_##suffix(a.val)); } \
inline void v_store_high(_Tp* ptr, const _Tpvec& a) \
@@ -889,6 +896,11 @@ inline v_float16x8 v_load_f16(const short* ptr)
inline v_float16x8 v_load_f16_aligned(const short* ptr)
{ return v_float16x8(cv_vld1q_f16(ptr)); }
inline v_float16x8 v_load_f16_low(const short* ptr)
{ return v_float16x8(vcombine_f16(cv_vld1_f16(ptr), vdup_n_f16((float16_t)0))); }
inline v_float16x8 v_load_f16_halves(const short* ptr0, const short* ptr1)
{ return v_float16x8(vcombine_f16(cv_vld1_f16(ptr0), cv_vld1_f16(ptr1))); }
inline void v_store(short* ptr, const v_float16x8& a)
{ cv_vst1q_f16(ptr, a.val); }
inline void v_store_aligned(short* ptr, const v_float16x8& a)
@@ -1292,14 +1304,16 @@ inline void v_load_deinterleave(const _Tp* ptr, v_##_Tpvec& a, v_##_Tpvec& b, \
c.val = v.val[2]; \
d.val = v.val[3]; \
} \
inline void v_store_interleave( _Tp* ptr, const v_##_Tpvec& a, const v_##_Tpvec& b) \
inline void v_store_interleave( _Tp* ptr, const v_##_Tpvec& a, const v_##_Tpvec& b, \
hal::StoreMode /*mode*/=hal::STORE_UNALIGNED) \
{ \
_Tpvec##x2_t v; \
v.val[0] = a.val; \
v.val[1] = b.val; \
vst2q_##suffix(ptr, v); \
} \
inline void v_store_interleave( _Tp* ptr, const v_##_Tpvec& a, const v_##_Tpvec& b, const v_##_Tpvec& c) \
inline void v_store_interleave( _Tp* ptr, const v_##_Tpvec& a, const v_##_Tpvec& b, \
const v_##_Tpvec& c, hal::StoreMode /*mode*/=hal::STORE_UNALIGNED) \
{ \
_Tpvec##x3_t v; \
v.val[0] = a.val; \
@@ -1308,7 +1322,8 @@ inline void v_store_interleave( _Tp* ptr, const v_##_Tpvec& a, const v_##_Tpvec&
vst3q_##suffix(ptr, v); \
} \
inline void v_store_interleave( _Tp* ptr, const v_##_Tpvec& a, const v_##_Tpvec& b, \
const v_##_Tpvec& c, const v_##_Tpvec& d) \
const v_##_Tpvec& c, const v_##_Tpvec& d, \
hal::StoreMode /*mode*/=hal::STORE_UNALIGNED ) \
{ \
_Tpvec##x4_t v; \
v.val[0] = a.val; \
@@ -1360,7 +1375,8 @@ inline void v_load_deinterleave( const tp* ptr, v_##tp##x2& a, v_##tp##x2& b, \
d = v_##tp##x2(vcombine_##suffix(d0, d1)); \
} \
\
inline void v_store_interleave( tp* ptr, const v_##tp##x2& a, const v_##tp##x2& b ) \
inline void v_store_interleave( tp* ptr, const v_##tp##x2& a, const v_##tp##x2& b, \
hal::StoreMode /*mode*/=hal::STORE_UNALIGNED) \
{ \
vst1_##suffix(ptr, vget_low_##suffix(a.val)); \
vst1_##suffix(ptr + 1, vget_low_##suffix(b.val)); \
@@ -1369,7 +1385,8 @@ inline void v_store_interleave( tp* ptr, const v_##tp##x2& a, const v_##tp##x2&
} \
\
inline void v_store_interleave( tp* ptr, const v_##tp##x2& a, \
const v_##tp##x2& b, const v_##tp##x2& c ) \
const v_##tp##x2& b, const v_##tp##x2& c, \
hal::StoreMode /*mode*/=hal::STORE_UNALIGNED) \
{ \
vst1_##suffix(ptr, vget_low_##suffix(a.val)); \
vst1_##suffix(ptr + 1, vget_low_##suffix(b.val)); \
@@ -1380,7 +1397,8 @@ inline void v_store_interleave( tp* ptr, const v_##tp##x2& a, \
} \
\
inline void v_store_interleave( tp* ptr, const v_##tp##x2& a, const v_##tp##x2& b, \
const v_##tp##x2& c, const v_##tp##x2& d ) \
const v_##tp##x2& c, const v_##tp##x2& d, \
hal::StoreMode /*mode*/=hal::STORE_UNALIGNED) \
{ \
vst1_##suffix(ptr, vget_low_##suffix(a.val)); \
vst1_##suffix(ptr + 1, vget_low_##suffix(b.val)); \
modules/core/include/opencv2/core/hal/intrin_sse.hpp
+390 -136
View File
@@ -788,7 +788,7 @@ inline v_float32x4 v_sqrt(const v_float32x4& x)
inline v_float32x4 v_invsqrt(const v_float32x4& x)
{
static const __m128 _0_5 = _mm_set1_ps(0.5f), _1_5 = _mm_set1_ps(1.5f);
const __m128 _0_5 = _mm_set1_ps(0.5f), _1_5 = _mm_set1_ps(1.5f);
__m128 t = x.val;
__m128 h = _mm_mul_ps(t, _0_5);
t = _mm_rsqrt_ps(t);
@@ -801,7 +801,7 @@ inline v_float64x2 v_sqrt(const v_float64x2& x)
inline v_float64x2 v_invsqrt(const v_float64x2& x)
{
static const __m128d v_1 = _mm_set1_pd(1.);
const __m128d v_1 = _mm_set1_pd(1.);
return v_float64x2(_mm_div_pd(v_1, _mm_sqrt_pd(x.val)));
}
@@ -1261,6 +1261,17 @@ inline void v_store(_Tp* ptr, const _Tpvec& a) \
{ _mm_storeu_si128((__m128i*)ptr, a.val); } \
inline void v_store_aligned(_Tp* ptr, const _Tpvec& a) \
{ _mm_store_si128((__m128i*)ptr, a.val); } \
inline void v_store_aligned_nocache(_Tp* ptr, const _Tpvec& a) \
{ _mm_stream_si128((__m128i*)ptr, a.val); } \
inline void v_store(_Tp* ptr, const _Tpvec& a, hal::StoreMode mode) \
{ \
if( mode == hal::STORE_UNALIGNED ) \
_mm_storeu_si128((__m128i*)ptr, a.val); \
else if( mode == hal::STORE_ALIGNED_NOCACHE ) \
_mm_stream_si128((__m128i*)ptr, a.val); \
else \
_mm_store_si128((__m128i*)ptr, a.val); \
} \
inline void v_store_low(_Tp* ptr, const _Tpvec& a) \
{ _mm_storel_epi64((__m128i*)ptr, a.val); } \
inline void v_store_high(_Tp* ptr, const _Tpvec& a) \
@@ -1292,6 +1303,17 @@ inline void v_store(_Tp* ptr, const _Tpvec& a) \
{ _mm_storeu_##suffix(ptr, a.val); } \
inline void v_store_aligned(_Tp* ptr, const _Tpvec& a) \
{ _mm_store_##suffix(ptr, a.val); } \
inline void v_store_aligned_nocache(_Tp* ptr, const _Tpvec& a) \
{ _mm_stream_##suffix(ptr, a.val); } \
inline void v_store(_Tp* ptr, const _Tpvec& a, hal::StoreMode mode) \
{ \
if( mode == hal::STORE_UNALIGNED ) \
_mm_storeu_##suffix(ptr, a.val); \
else if( mode == hal::STORE_ALIGNED_NOCACHE ) \
_mm_stream_##suffix(ptr, a.val); \
else \
_mm_store_##suffix(ptr, a.val); \
} \
inline void v_store_low(_Tp* ptr, const _Tpvec& a) \
{ _mm_storel_epi64((__m128i*)ptr, _mm_cast##suffix##_si128(a.val)); } \
inline void v_store_high(_Tp* ptr, const _Tpvec& a) \
@@ -1308,6 +1330,11 @@ inline v_float16x8 v_load_f16(const short* ptr)
inline v_float16x8 v_load_f16_aligned(const short* ptr)
{ return v_float16x8(_mm_load_si128((const __m128i*)ptr)); }
inline v_float16x8 v_load_f16_low(const short* ptr)
{ return v_float16x8(v_load_low(ptr).val); }
inline v_float16x8 v_load_f16_halves(const short* ptr0, const short* ptr1)
{ return v_float16x8(v_load_halves(ptr0, ptr1).val); }
inline void v_store(short* ptr, const v_float16x8& a)
{ _mm_storeu_si128((__m128i*)ptr, a.val); }
inline void v_store_aligned(short* ptr, const v_float16x8& a)
@@ -1671,17 +1698,17 @@ inline void v_load_deinterleave(const uchar* ptr, v_uint8x16& a, v_uint8x16& b)
inline void v_load_deinterleave(const uchar* ptr, v_uint8x16& a, v_uint8x16& b, v_uint8x16& c)
{
#if CV_SSE4_1
static const __m128i m0 = _mm_setr_epi8(0, 0, -1, 0, 0, -1, 0, 0, -1, 0, 0, -1, 0, 0, -1, 0);
static const __m128i m1 = _mm_setr_epi8(0, -1, 0, 0, -1, 0, 0, -1, 0, 0, -1, 0, 0, -1, 0, 0);
const __m128i m0 = _mm_setr_epi8(0, 0, -1, 0, 0, -1, 0, 0, -1, 0, 0, -1, 0, 0, -1, 0);
const __m128i m1 = _mm_setr_epi8(0, -1, 0, 0, -1, 0, 0, -1, 0, 0, -1, 0, 0, -1, 0, 0);
__m128i s0 = _mm_loadu_si128((const __m128i*)ptr);
__m128i s1 = _mm_loadu_si128((const __m128i*)(ptr + 16));
__m128i s2 = _mm_loadu_si128((const __m128i*)(ptr + 32));
__m128i a0 = _mm_blendv_epi8(_mm_blendv_epi8(s0, s1, m0), s2, m1);
__m128i b0 = _mm_blendv_epi8(_mm_blendv_epi8(s1, s2, m0), s0, m1);
__m128i c0 = _mm_blendv_epi8(_mm_blendv_epi8(s2, s0, m0), s1, m1);
static const __m128i sh_b = _mm_setr_epi8(0, 3, 6, 9, 12, 15, 2, 5, 8, 11, 14, 1, 4, 7, 10, 13);
static const __m128i sh_g = _mm_setr_epi8(1, 4, 7, 10, 13, 0, 3, 6, 9, 12, 15, 2, 5, 8, 11, 14);
static const __m128i sh_r = _mm_setr_epi8(2, 5, 8, 11, 14, 1, 4, 7, 10, 13, 0, 3, 6, 9, 12, 15);
const __m128i sh_b = _mm_setr_epi8(0, 3, 6, 9, 12, 15, 2, 5, 8, 11, 14, 1, 4, 7, 10, 13);
const __m128i sh_g = _mm_setr_epi8(1, 4, 7, 10, 13, 0, 3, 6, 9, 12, 15, 2, 5, 8, 11, 14);
const __m128i sh_r = _mm_setr_epi8(2, 5, 8, 11, 14, 1, 4, 7, 10, 13, 0, 3, 6, 9, 12, 15);
a0 = _mm_shuffle_epi8(a0, sh_b);
b0 = _mm_shuffle_epi8(b0, sh_g);
c0 = _mm_shuffle_epi8(c0, sh_r);
@@ -1689,9 +1716,9 @@ inline void v_load_deinterleave(const uchar* ptr, v_uint8x16& a, v_uint8x16& b,
b.val = b0;
c.val = c0;
#elif CV_SSSE3
static const __m128i m0 = _mm_setr_epi8(0, 3, 6, 9, 12, 15, 1, 4, 7, 10, 13, 2, 5, 8, 11, 14);
static const __m128i m1 = _mm_alignr_epi8(m0, m0, 11);
static const __m128i m2 = _mm_alignr_epi8(m0, m0, 6);
const __m128i m0 = _mm_setr_epi8(0, 3, 6, 9, 12, 15, 1, 4, 7, 10, 13, 2, 5, 8, 11, 14);
const __m128i m1 = _mm_alignr_epi8(m0, m0, 11);
const __m128i m2 = _mm_alignr_epi8(m0, m0, 6);
__m128i t0 = _mm_loadu_si128((const __m128i*)ptr);
__m128i t1 = _mm_loadu_si128((const __m128i*)(ptr + 16));
@@ -1784,9 +1811,9 @@ inline void v_load_deinterleave(const ushort* ptr, v_uint16x8& a, v_uint16x8& b,
__m128i b0 = _mm_blend_epi16(_mm_blend_epi16(v2, v0, 0x92), v1, 0x24);
__m128i c0 = _mm_blend_epi16(_mm_blend_epi16(v1, v2, 0x92), v0, 0x24);
static const __m128i sh_a = _mm_setr_epi8(0, 1, 6, 7, 12, 13, 2, 3, 8, 9, 14, 15, 4, 5, 10, 11);
static const __m128i sh_b = _mm_setr_epi8(2, 3, 8, 9, 14, 15, 4, 5, 10, 11, 0, 1, 6, 7, 12, 13);
static const __m128i sh_c = _mm_setr_epi8(4, 5, 10, 11, 0, 1, 6, 7, 12, 13, 2, 3, 8, 9, 14, 15);
const __m128i sh_a = _mm_setr_epi8(0, 1, 6, 7, 12, 13, 2, 3, 8, 9, 14, 15, 4, 5, 10, 11);
const __m128i sh_b = _mm_setr_epi8(2, 3, 8, 9, 14, 15, 4, 5, 10, 11, 0, 1, 6, 7, 12, 13);
const __m128i sh_c = _mm_setr_epi8(4, 5, 10, 11, 0, 1, 6, 7, 12, 13, 2, 3, 8, 9, 14, 15);
a0 = _mm_shuffle_epi8(a0, sh_a);
b0 = _mm_shuffle_epi8(b0, sh_b);
c0 = _mm_shuffle_epi8(c0, sh_c);
@@ -1955,55 +1982,61 @@ inline void v_load_deinterleave(const uint64 *ptr, v_uint64x2& a,
// store interleave
inline void v_store_interleave( uchar* ptr, const v_uint8x16& a, const v_uint8x16& b)
inline void v_store_interleave( uchar* ptr, const v_uint8x16& a, const v_uint8x16& b,
hal::StoreMode mode = hal::STORE_UNALIGNED)
{
__m128i v0 = _mm_unpacklo_epi8(a.val, b.val);
__m128i v1 = _mm_unpackhi_epi8(a.val, b.val);
_mm_storeu_si128((__m128i*)(ptr), v0);
_mm_storeu_si128((__m128i*)(ptr + 16), v1);
if( mode == hal::STORE_ALIGNED_NOCACHE )
{
_mm_stream_si128((__m128i*)(ptr), v0);
_mm_stream_si128((__m128i*)(ptr + 16), v1);
}
else if( mode == hal::STORE_ALIGNED )
{
_mm_store_si128((__m128i*)(ptr), v0);
_mm_store_si128((__m128i*)(ptr + 16), v1);
}
else
{
_mm_storeu_si128((__m128i*)(ptr), v0);
_mm_storeu_si128((__m128i*)(ptr + 16), v1);
}
}
inline void v_store_interleave( uchar* ptr, const v_uint8x16& a, const v_uint8x16& b,
const v_uint8x16& c )
const v_uint8x16& c, hal::StoreMode mode = hal::STORE_UNALIGNED)
{
#if CV_SSE4_1
static const __m128i sh_a = _mm_setr_epi8(0, 11, 6, 1, 12, 7, 2, 13, 8, 3, 14, 9, 4, 15, 10, 5);
static const __m128i sh_b = _mm_setr_epi8(5, 0, 11, 6, 1, 12, 7, 2, 13, 8, 3, 14, 9, 4, 15, 10);
static const __m128i sh_c = _mm_setr_epi8(10, 5, 0, 11, 6, 1, 12, 7, 2, 13, 8, 3, 14, 9, 4, 15);
const __m128i sh_a = _mm_setr_epi8(0, 11, 6, 1, 12, 7, 2, 13, 8, 3, 14, 9, 4, 15, 10, 5);
const __m128i sh_b = _mm_setr_epi8(5, 0, 11, 6, 1, 12, 7, 2, 13, 8, 3, 14, 9, 4, 15, 10);
const __m128i sh_c = _mm_setr_epi8(10, 5, 0, 11, 6, 1, 12, 7, 2, 13, 8, 3, 14, 9, 4, 15);
__m128i a0 = _mm_shuffle_epi8(a.val, sh_a);
__m128i b0 = _mm_shuffle_epi8(b.val, sh_b);
__m128i c0 = _mm_shuffle_epi8(c.val, sh_c);
static const __m128i m0 = _mm_setr_epi8(0, 0, -1, 0, 0, -1, 0, 0, -1, 0, 0, -1, 0, 0, -1, 0);
static const __m128i m1 = _mm_setr_epi8(0, -1, 0, 0, -1, 0, 0, -1, 0, 0, -1, 0, 0, -1, 0, 0);
const __m128i m0 = _mm_setr_epi8(0, 0, -1, 0, 0, -1, 0, 0, -1, 0, 0, -1, 0, 0, -1, 0);
const __m128i m1 = _mm_setr_epi8(0, -1, 0, 0, -1, 0, 0, -1, 0, 0, -1, 0, 0, -1, 0, 0);
__m128i v0 = _mm_blendv_epi8(_mm_blendv_epi8(a0, b0, m1), c0, m0);
__m128i v1 = _mm_blendv_epi8(_mm_blendv_epi8(b0, c0, m1), a0, m0);
__m128i v2 = _mm_blendv_epi8(_mm_blendv_epi8(c0, a0, m1), b0, m0);
_mm_storeu_si128((__m128i*)(ptr), v0);
_mm_storeu_si128((__m128i*)(ptr + 16), v1);
_mm_storeu_si128((__m128i*)(ptr + 32), v2);
#elif CV_SSSE3
static const __m128i m0 = _mm_setr_epi8(0, 6, 11, 1, 7, 12, 2, 8, 13, 3, 9, 14, 4, 10, 15, 5);
static const __m128i m1 = _mm_setr_epi8(5, 11, 0, 6, 12, 1, 7, 13, 2, 8, 14, 3, 9, 15, 4, 10);
static const __m128i m2 = _mm_setr_epi8(10, 0, 5, 11, 1, 6, 12, 2, 7, 13, 3, 8, 14, 4, 9, 15);
const __m128i m0 = _mm_setr_epi8(0, 6, 11, 1, 7, 12, 2, 8, 13, 3, 9, 14, 4, 10, 15, 5);
const __m128i m1 = _mm_setr_epi8(5, 11, 0, 6, 12, 1, 7, 13, 2, 8, 14, 3, 9, 15, 4, 10);
const __m128i m2 = _mm_setr_epi8(10, 0, 5, 11, 1, 6, 12, 2, 7, 13, 3, 8, 14, 4, 9, 15);
__m128i t0 = _mm_alignr_epi8(b.val, _mm_slli_si128(a.val, 10), 5);
t0 = _mm_alignr_epi8(c.val, t0, 5);
__m128i s0 = _mm_shuffle_epi8(t0, m0);
__m128i v0 = _mm_shuffle_epi8(t0, m0);
__m128i t1 = _mm_alignr_epi8(_mm_srli_si128(b.val, 5), _mm_slli_si128(a.val, 5), 6);
t1 = _mm_alignr_epi8(_mm_srli_si128(c.val, 5), t1, 5);
__m128i s1 = _mm_shuffle_epi8(t1, m1);
__m128i v1 = _mm_shuffle_epi8(t1, m1);
__m128i t2 = _mm_alignr_epi8(_mm_srli_si128(c.val, 10), b.val, 11);
t2 = _mm_alignr_epi8(t2, a.val, 11);
__m128i s2 = _mm_shuffle_epi8(t2, m2);
_mm_storeu_si128((__m128i*)ptr, s0);
_mm_storeu_si128((__m128i*)(ptr + 16), s1);
_mm_storeu_si128((__m128i*)(ptr + 32), s2);
__m128i v2 = _mm_shuffle_epi8(t2, m2);
#else
__m128i z = _mm_setzero_si128();
__m128i ab0 = _mm_unpacklo_epi8(a.val, b.val);
@@ -2042,15 +2075,31 @@ inline void v_store_interleave( uchar* ptr, const v_uint8x16& a, const v_uint8x1
__m128i v0 = _mm_or_si128(_mm_srli_si128(p40, 2), _mm_slli_si128(p41, 10));
__m128i v1 = _mm_or_si128(_mm_srli_si128(p41, 6), _mm_slli_si128(p42, 6));
__m128i v2 = _mm_or_si128(_mm_srli_si128(p42, 10), _mm_slli_si128(p43, 2));
_mm_storeu_si128((__m128i*)(ptr), v0);
_mm_storeu_si128((__m128i*)(ptr + 16), v1);
_mm_storeu_si128((__m128i*)(ptr + 32), v2);
#endif
if( mode == hal::STORE_ALIGNED_NOCACHE )
{
_mm_stream_si128((__m128i*)(ptr), v0);
_mm_stream_si128((__m128i*)(ptr + 16), v1);
_mm_stream_si128((__m128i*)(ptr + 32), v2);
}
else if( mode == hal::STORE_ALIGNED )
{
_mm_store_si128((__m128i*)(ptr), v0);
_mm_store_si128((__m128i*)(ptr + 16), v1);
_mm_store_si128((__m128i*)(ptr + 32), v2);
}
else
{
_mm_storeu_si128((__m128i*)(ptr), v0);
_mm_storeu_si128((__m128i*)(ptr + 16), v1);
_mm_storeu_si128((__m128i*)(ptr + 32), v2);
}
}
inline void v_store_interleave( uchar* ptr, const v_uint8x16& a, const v_uint8x16& b,
const v_uint8x16& c, const v_uint8x16& d)
const v_uint8x16& c, const v_uint8x16& d,
hal::StoreMode mode = hal::STORE_UNALIGNED)
{
// a0 a1 a2 a3 ....
// b0 b1 b2 b3 ....
@@ -2062,33 +2111,64 @@ inline void v_store_interleave( uchar* ptr, const v_uint8x16& a, const v_uint8x1
__m128i u3 = _mm_unpackhi_epi8(b.val, d.val); // b8 d8 b9 d9 ...
__m128i v0 = _mm_unpacklo_epi8(u0, u2); // a0 b0 c0 d0 ...
__m128i v1 = _mm_unpacklo_epi8(u1, u3); // a8 b8 c8 d8 ...
__m128i v2 = _mm_unpackhi_epi8(u0, u2); // a4 b4 c4 d4 ...
__m128i v1 = _mm_unpackhi_epi8(u0, u2); // a4 b4 c4 d4 ...
__m128i v2 = _mm_unpacklo_epi8(u1, u3); // a8 b8 c8 d8 ...
__m128i v3 = _mm_unpackhi_epi8(u1, u3); // a12 b12 c12 d12 ...
_mm_storeu_si128((__m128i*)ptr, v0);
_mm_storeu_si128((__m128i*)(ptr + 16), v2);
_mm_storeu_si128((__m128i*)(ptr + 32), v1);
_mm_storeu_si128((__m128i*)(ptr + 48), v3);
if( mode == hal::STORE_ALIGNED_NOCACHE )
{
_mm_stream_si128((__m128i*)(ptr), v0);
_mm_stream_si128((__m128i*)(ptr + 16), v1);
_mm_stream_si128((__m128i*)(ptr + 32), v2);
_mm_stream_si128((__m128i*)(ptr + 48), v3);
}
else if( mode == hal::STORE_ALIGNED )
{
_mm_store_si128((__m128i*)(ptr), v0);
_mm_store_si128((__m128i*)(ptr + 16), v1);
_mm_store_si128((__m128i*)(ptr + 32), v2);
_mm_store_si128((__m128i*)(ptr + 48), v3);
}
else
{
_mm_storeu_si128((__m128i*)(ptr), v0);
_mm_storeu_si128((__m128i*)(ptr + 16), v1);
_mm_storeu_si128((__m128i*)(ptr + 32), v2);
_mm_storeu_si128((__m128i*)(ptr + 48), v3);
}
}
inline void v_store_interleave( ushort* ptr, const v_uint16x8& a, const v_uint16x8& b )
inline void v_store_interleave( ushort* ptr, const v_uint16x8& a, const v_uint16x8& b,
hal::StoreMode mode = hal::STORE_UNALIGNED)
{
__m128i t0, t1;
t0 = _mm_unpacklo_epi16(a.val, b.val);
t1 = _mm_unpackhi_epi16(a.val, b.val);
_mm_storeu_si128((__m128i*)(ptr), t0);
_mm_storeu_si128((__m128i*)(ptr + 8), t1);
__m128i v0 = _mm_unpacklo_epi16(a.val, b.val);
__m128i v1 = _mm_unpackhi_epi16(a.val, b.val);
if( mode == hal::STORE_ALIGNED_NOCACHE )
{
_mm_stream_si128((__m128i*)(ptr), v0);
_mm_stream_si128((__m128i*)(ptr + 8), v1);
}
else if( mode == hal::STORE_ALIGNED )
{
_mm_store_si128((__m128i*)(ptr), v0);
_mm_store_si128((__m128i*)(ptr + 8), v1);
}
else
{
_mm_storeu_si128((__m128i*)(ptr), v0);
_mm_storeu_si128((__m128i*)(ptr + 8), v1);
}
}
inline void v_store_interleave( ushort* ptr, const v_uint16x8& a,
const v_uint16x8& b,
const v_uint16x8& c )
const v_uint16x8& b, const v_uint16x8& c,
hal::StoreMode mode = hal::STORE_UNALIGNED)
{
#if CV_SSE4_1
static const __m128i sh_a = _mm_setr_epi8(0, 1, 6, 7, 12, 13, 2, 3, 8, 9, 14, 15, 4, 5, 10, 11);
static const __m128i sh_b = _mm_setr_epi8(10, 11, 0, 1, 6, 7, 12, 13, 2, 3, 8, 9, 14, 15, 4, 5);
static const __m128i sh_c = _mm_setr_epi8(4, 5, 10, 11, 0, 1, 6, 7, 12, 13, 2, 3, 8, 9, 14, 15);
const __m128i sh_a = _mm_setr_epi8(0, 1, 6, 7, 12, 13, 2, 3, 8, 9, 14, 15, 4, 5, 10, 11);
const __m128i sh_b = _mm_setr_epi8(10, 11, 0, 1, 6, 7, 12, 13, 2, 3, 8, 9, 14, 15, 4, 5);
const __m128i sh_c = _mm_setr_epi8(4, 5, 10, 11, 0, 1, 6, 7, 12, 13, 2, 3, 8, 9, 14, 15);
__m128i a0 = _mm_shuffle_epi8(a.val, sh_a);
__m128i b0 = _mm_shuffle_epi8(b.val, sh_b);
__m128i c0 = _mm_shuffle_epi8(c.val, sh_c);
@@ -2096,10 +2176,6 @@ inline void v_store_interleave( ushort* ptr, const v_uint16x8& a,
__m128i v0 = _mm_blend_epi16(_mm_blend_epi16(a0, b0, 0x92), c0, 0x24);
__m128i v1 = _mm_blend_epi16(_mm_blend_epi16(c0, a0, 0x92), b0, 0x24);
__m128i v2 = _mm_blend_epi16(_mm_blend_epi16(b0, c0, 0x92), a0, 0x24);
_mm_storeu_si128((__m128i*)ptr, v0);
_mm_storeu_si128((__m128i*)(ptr + 8), v1);
_mm_storeu_si128((__m128i*)(ptr + 16), v2);
#else
__m128i z = _mm_setzero_si128();
__m128i ab0 = _mm_unpacklo_epi16(a.val, b.val);
@@ -2128,15 +2204,30 @@ inline void v_store_interleave( ushort* ptr, const v_uint16x8& a,
__m128i v0 = _mm_or_si128(_mm_srli_si128(p30, 2), _mm_slli_si128(p31, 10));
__m128i v1 = _mm_or_si128(_mm_srli_si128(p31, 6), _mm_slli_si128(p32, 6));
__m128i v2 = _mm_or_si128(_mm_srli_si128(p32, 10), _mm_slli_si128(p33, 2));
_mm_storeu_si128((__m128i*)(ptr), v0);
_mm_storeu_si128((__m128i*)(ptr + 8), v1);
_mm_storeu_si128((__m128i*)(ptr + 16), v2);
#endif
if( mode == hal::STORE_ALIGNED_NOCACHE )
{
_mm_stream_si128((__m128i*)(ptr), v0);
_mm_stream_si128((__m128i*)(ptr + 8), v1);
_mm_stream_si128((__m128i*)(ptr + 16), v2);
}
else if( mode == hal::STORE_ALIGNED )
{
_mm_store_si128((__m128i*)(ptr), v0);
_mm_store_si128((__m128i*)(ptr + 8), v1);
_mm_store_si128((__m128i*)(ptr + 16), v2);
}
else
{
_mm_storeu_si128((__m128i*)(ptr), v0);
_mm_storeu_si128((__m128i*)(ptr + 8), v1);
_mm_storeu_si128((__m128i*)(ptr + 16), v2);
}
}
inline void v_store_interleave( ushort* ptr, const v_uint16x8& a, const v_uint16x8& b,
const v_uint16x8& c, const v_uint16x8& d)
const v_uint16x8& c, const v_uint16x8& d,
hal::StoreMode mode = hal::STORE_UNALIGNED)
{
// a0 a1 a2 a3 ....
// b0 b1 b2 b3 ....
@@ -2148,27 +2239,58 @@ inline void v_store_interleave( ushort* ptr, const v_uint16x8& a, const v_uint16
__m128i u3 = _mm_unpackhi_epi16(b.val, d.val); // b4 d4 b5 d5 ...
__m128i v0 = _mm_unpacklo_epi16(u0, u2); // a0 b0 c0 d0 ...
__m128i v1 = _mm_unpacklo_epi16(u1, u3); // a4 b4 c4 d4 ...
__m128i v2 = _mm_unpackhi_epi16(u0, u2); // a2 b2 c2 d2 ...
__m128i v1 = _mm_unpackhi_epi16(u0, u2); // a2 b2 c2 d2 ...
__m128i v2 = _mm_unpacklo_epi16(u1, u3); // a4 b4 c4 d4 ...
__m128i v3 = _mm_unpackhi_epi16(u1, u3); // a6 b6 c6 d6 ...
_mm_storeu_si128((__m128i*)ptr, v0);
_mm_storeu_si128((__m128i*)(ptr + 8), v2);
_mm_storeu_si128((__m128i*)(ptr + 16), v1);
_mm_storeu_si128((__m128i*)(ptr + 24), v3);
}
inline void v_store_interleave( unsigned* ptr, const v_uint32x4& a, const v_uint32x4& b )
{
__m128i t0 = _mm_unpacklo_epi32(a.val, b.val);
__m128i t1 = _mm_unpackhi_epi32(a.val, b.val);
_mm_storeu_si128((__m128i*)ptr, t0);
_mm_storeu_si128((__m128i*)(ptr + 4), t1);
if( mode == hal::STORE_ALIGNED_NOCACHE )
{
_mm_stream_si128((__m128i*)(ptr), v0);
_mm_stream_si128((__m128i*)(ptr + 8), v1);
_mm_stream_si128((__m128i*)(ptr + 16), v2);
_mm_stream_si128((__m128i*)(ptr + 24), v3);
}
else if( mode == hal::STORE_ALIGNED )
{
_mm_store_si128((__m128i*)(ptr), v0);
_mm_store_si128((__m128i*)(ptr + 8), v1);
_mm_store_si128((__m128i*)(ptr + 16), v2);
_mm_store_si128((__m128i*)(ptr + 24), v3);
}
else
{
_mm_storeu_si128((__m128i*)(ptr), v0);
_mm_storeu_si128((__m128i*)(ptr + 8), v1);
_mm_storeu_si128((__m128i*)(ptr + 16), v2);
_mm_storeu_si128((__m128i*)(ptr + 24), v3);
}
}
inline void v_store_interleave( unsigned* ptr, const v_uint32x4& a, const v_uint32x4& b,
const v_uint32x4& c )
hal::StoreMode mode = hal::STORE_UNALIGNED)
{
__m128i v0 = _mm_unpacklo_epi32(a.val, b.val);
__m128i v1 = _mm_unpackhi_epi32(a.val, b.val);
if( mode == hal::STORE_ALIGNED_NOCACHE )
{
_mm_stream_si128((__m128i*)(ptr), v0);
_mm_stream_si128((__m128i*)(ptr + 4), v1);
}
else if( mode == hal::STORE_ALIGNED )
{
_mm_store_si128((__m128i*)(ptr), v0);
_mm_store_si128((__m128i*)(ptr + 4), v1);
}
else
{
_mm_storeu_si128((__m128i*)(ptr), v0);
_mm_storeu_si128((__m128i*)(ptr + 4), v1);
}
}
inline void v_store_interleave( unsigned* ptr, const v_uint32x4& a, const v_uint32x4& b,
const v_uint32x4& c, hal::StoreMode mode = hal::STORE_UNALIGNED)
{
v_uint32x4 z = v_setzero_u32(), u0, u1, u2, u3;
v_transpose4x4(a, b, c, z, u0, u1, u2, u3);
@@ -2177,35 +2299,82 @@ inline void v_store_interleave( unsigned* ptr, const v_uint32x4& a, const v_uint
__m128i v1 = _mm_or_si128(_mm_srli_si128(u1.val, 4), _mm_slli_si128(u2.val, 8));
__m128i v2 = _mm_or_si128(_mm_srli_si128(u2.val, 8), _mm_slli_si128(u3.val, 4));
_mm_storeu_si128((__m128i*)ptr, v0);
_mm_storeu_si128((__m128i*)(ptr + 4), v1);
_mm_storeu_si128((__m128i*)(ptr + 8), v2);
if( mode == hal::STORE_ALIGNED_NOCACHE )
{
_mm_stream_si128((__m128i*)(ptr), v0);
_mm_stream_si128((__m128i*)(ptr + 4), v1);
_mm_stream_si128((__m128i*)(ptr + 8), v2);
}
else if( mode == hal::STORE_ALIGNED )
{
_mm_store_si128((__m128i*)(ptr), v0);
_mm_store_si128((__m128i*)(ptr + 4), v1);
_mm_store_si128((__m128i*)(ptr + 8), v2);
}
else
{
_mm_storeu_si128((__m128i*)(ptr), v0);
_mm_storeu_si128((__m128i*)(ptr + 4), v1);
_mm_storeu_si128((__m128i*)(ptr + 8), v2);
}
}
inline void v_store_interleave(unsigned* ptr, const v_uint32x4& a, const v_uint32x4& b,
const v_uint32x4& c, const v_uint32x4& d)
const v_uint32x4& c, const v_uint32x4& d,
hal::StoreMode mode = hal::STORE_UNALIGNED)
{
v_uint32x4 t0, t1, t2, t3;
v_transpose4x4(a, b, c, d, t0, t1, t2, t3);
v_store(ptr, t0);
v_store(ptr + 4, t1);
v_store(ptr + 8, t2);
v_store(ptr + 12, t3);
v_uint32x4 v0, v1, v2, v3;
v_transpose4x4(a, b, c, d, v0, v1, v2, v3);
if( mode == hal::STORE_ALIGNED_NOCACHE )
{
_mm_stream_si128((__m128i*)(ptr), v0.val);
_mm_stream_si128((__m128i*)(ptr + 4), v1.val);
_mm_stream_si128((__m128i*)(ptr + 8), v2.val);
_mm_stream_si128((__m128i*)(ptr + 12), v3.val);
}
else if( mode == hal::STORE_ALIGNED )
{
_mm_store_si128((__m128i*)(ptr), v0.val);
_mm_store_si128((__m128i*)(ptr + 4), v1.val);
_mm_store_si128((__m128i*)(ptr + 8), v2.val);
_mm_store_si128((__m128i*)(ptr + 12), v3.val);
}
else
{
_mm_storeu_si128((__m128i*)(ptr), v0.val);
_mm_storeu_si128((__m128i*)(ptr + 4), v1.val);
_mm_storeu_si128((__m128i*)(ptr + 8), v2.val);
_mm_storeu_si128((__m128i*)(ptr + 12), v3.val);
}
}
// 2-channel, float only
inline void v_store_interleave(float* ptr, const v_float32x4& a, const v_float32x4& b)
inline void v_store_interleave(float* ptr, const v_float32x4& a, const v_float32x4& b,
hal::StoreMode mode = hal::STORE_UNALIGNED)
{
// a0 a1 a2 a3 ...
// b0 b1 b2 b3 ...
__m128 u0 = _mm_unpacklo_ps(a.val, b.val); // a0 b0 a1 b1
__m128 u1 = _mm_unpackhi_ps(a.val, b.val); // a2 b2 a3 b3
__m128 v0 = _mm_unpacklo_ps(a.val, b.val); // a0 b0 a1 b1
__m128 v1 = _mm_unpackhi_ps(a.val, b.val); // a2 b2 a3 b3
_mm_storeu_ps(ptr, u0);
_mm_storeu_ps((ptr + 4), u1);
if( mode == hal::STORE_ALIGNED_NOCACHE )
{
_mm_stream_ps(ptr, v0);
_mm_stream_ps(ptr + 4, v1);
}
else if( mode == hal::STORE_ALIGNED )
{
_mm_store_ps(ptr, v0);
_mm_store_ps(ptr + 4, v1);
}
else
{
_mm_storeu_ps(ptr, v0);
_mm_storeu_ps(ptr + 4, v1);
}
}
inline void v_store_interleave(float* ptr, const v_float32x4& a, const v_float32x4& b, const v_float32x4& c)
inline void v_store_interleave(float* ptr, const v_float32x4& a, const v_float32x4& b,
const v_float32x4& c, hal::StoreMode mode = hal::STORE_UNALIGNED)
{
__m128 u0 = _mm_shuffle_ps(a.val, b.val, _MM_SHUFFLE(0, 0, 0, 0));
__m128 u1 = _mm_shuffle_ps(c.val, a.val, _MM_SHUFFLE(1, 1, 0, 0));
@@ -2217,13 +2386,29 @@ inline void v_store_interleave(float* ptr, const v_float32x4& a, const v_float32
__m128 u5 = _mm_shuffle_ps(b.val, c.val, _MM_SHUFFLE(3, 3, 3, 3));
__m128 v2 = _mm_shuffle_ps(u4, u5, _MM_SHUFFLE(2, 0, 2, 0));
_mm_storeu_ps(ptr + 0, v0);
_mm_storeu_ps(ptr + 4, v1);
_mm_storeu_ps(ptr + 8, v2);
if( mode == hal::STORE_ALIGNED_NOCACHE )
{
_mm_stream_ps(ptr, v0);
_mm_stream_ps(ptr + 4, v1);
_mm_stream_ps(ptr + 8, v2);
}
else if( mode == hal::STORE_ALIGNED )
{
_mm_store_ps(ptr, v0);
_mm_store_ps(ptr + 4, v1);
_mm_store_ps(ptr + 8, v2);
}
else
{
_mm_storeu_ps(ptr, v0);
_mm_storeu_ps(ptr + 4, v1);
_mm_storeu_ps(ptr + 8, v2);
}
}
inline void v_store_interleave(float* ptr, const v_float32x4& a, const v_float32x4& b,
const v_float32x4& c, const v_float32x4& d)
const v_float32x4& c, const v_float32x4& d,
hal::StoreMode mode = hal::STORE_UNALIGNED)
{
__m128 u0 = _mm_unpacklo_ps(a.val, c.val);
__m128 u1 = _mm_unpacklo_ps(b.val, d.val);
@@ -2234,43 +2419,109 @@ inline void v_store_interleave(float* ptr, const v_float32x4& a, const v_float32
__m128 v1 = _mm_unpackhi_ps(u0, u1);
__m128 v3 = _mm_unpackhi_ps(u2, u3);
_mm_storeu_ps(ptr + 0, v0);
_mm_storeu_ps(ptr + 4, v1);
_mm_storeu_ps(ptr + 8, v2);
_mm_storeu_ps(ptr + 12, v3);
if( mode == hal::STORE_ALIGNED_NOCACHE )
{
_mm_stream_ps(ptr, v0);
_mm_stream_ps(ptr + 4, v1);
_mm_stream_ps(ptr + 8, v2);
_mm_stream_ps(ptr + 12, v3);
}
else if( mode == hal::STORE_ALIGNED )
{
_mm_store_ps(ptr, v0);
_mm_store_ps(ptr + 4, v1);
_mm_store_ps(ptr + 8, v2);
_mm_store_ps(ptr + 12, v3);
}
else
{
_mm_storeu_ps(ptr, v0);
_mm_storeu_ps(ptr + 4, v1);
_mm_storeu_ps(ptr + 8, v2);
_mm_storeu_ps(ptr + 12, v3);
}
}
inline void v_store_interleave(uint64 *ptr, const v_uint64x2& a, const v_uint64x2& b)
inline void v_store_interleave(uint64 *ptr, const v_uint64x2& a, const v_uint64x2& b,
hal::StoreMode mode = hal::STORE_UNALIGNED)
{
__m128i t0 = _mm_unpacklo_epi64(a.val, b.val);
__m128i t1 = _mm_unpackhi_epi64(a.val, b.val);
__m128i v0 = _mm_unpacklo_epi64(a.val, b.val);
__m128i v1 = _mm_unpackhi_epi64(a.val, b.val);
_mm_storeu_si128((__m128i*)ptr, t0);
_mm_storeu_si128((__m128i*)(ptr + 2), t1);
if( mode == hal::STORE_ALIGNED_NOCACHE )
{
_mm_stream_si128((__m128i*)(ptr), v0);
_mm_stream_si128((__m128i*)(ptr + 2), v1);
}
else if( mode == hal::STORE_ALIGNED )
{
_mm_store_si128((__m128i*)(ptr), v0);
_mm_store_si128((__m128i*)(ptr + 2), v1);
}
else
{
_mm_storeu_si128((__m128i*)(ptr), v0);
_mm_storeu_si128((__m128i*)(ptr + 2), v1);
}
}
inline void v_store_interleave(uint64 *ptr, const v_uint64x2& a, const v_uint64x2& b, const v_uint64x2& c)
inline void v_store_interleave(uint64 *ptr, const v_uint64x2& a, const v_uint64x2& b,
const v_uint64x2& c, hal::StoreMode mode = hal::STORE_UNALIGNED)
{
__m128i t0 = _mm_unpacklo_epi64(a.val, b.val);
__m128i t1 = _mm_unpacklo_epi64(c.val, _mm_unpackhi_epi64(a.val, a.val));
__m128i t2 = _mm_unpackhi_epi64(b.val, c.val);
__m128i v0 = _mm_unpacklo_epi64(a.val, b.val);
__m128i v1 = _mm_unpacklo_epi64(c.val, _mm_unpackhi_epi64(a.val, a.val));
__m128i v2 = _mm_unpackhi_epi64(b.val, c.val);
_mm_storeu_si128((__m128i*)ptr, t0);
_mm_storeu_si128((__m128i*)(ptr + 2), t1);
_mm_storeu_si128((__m128i*)(ptr + 4), t2);
if( mode == hal::STORE_ALIGNED_NOCACHE )
{
_mm_stream_si128((__m128i*)(ptr), v0);
_mm_stream_si128((__m128i*)(ptr + 2), v1);
_mm_stream_si128((__m128i*)(ptr + 4), v2);
}
else if( mode == hal::STORE_ALIGNED )
{
_mm_store_si128((__m128i*)(ptr), v0);
_mm_store_si128((__m128i*)(ptr + 2), v1);
_mm_store_si128((__m128i*)(ptr + 4), v2);
}
else
{
_mm_storeu_si128((__m128i*)(ptr), v0);
_mm_storeu_si128((__m128i*)(ptr + 2), v1);
_mm_storeu_si128((__m128i*)(ptr + 4), v2);
}
}
inline void v_store_interleave(uint64 *ptr, const v_uint64x2& a, const v_uint64x2& b, const v_uint64x2& c, const v_uint64x2& d)
inline void v_store_interleave(uint64 *ptr, const v_uint64x2& a, const v_uint64x2& b,
const v_uint64x2& c, const v_uint64x2& d,
hal::StoreMode mode = hal::STORE_UNALIGNED)
{
__m128i t0 = _mm_unpacklo_epi64(a.val, b.val);
__m128i t1 = _mm_unpacklo_epi64(c.val, d.val);
__m128i t2 = _mm_unpackhi_epi64(a.val, b.val);
__m128i t3 = _mm_unpackhi_epi64(c.val, d.val);
__m128i v0 = _mm_unpacklo_epi64(a.val, b.val);
__m128i v1 = _mm_unpacklo_epi64(c.val, d.val);
__m128i v2 = _mm_unpackhi_epi64(a.val, b.val);
__m128i v3 = _mm_unpackhi_epi64(c.val, d.val);
_mm_storeu_si128((__m128i*)ptr, t0);
_mm_storeu_si128((__m128i*)(ptr + 2), t1);
_mm_storeu_si128((__m128i*)(ptr + 4), t2);
_mm_storeu_si128((__m128i*)(ptr + 6), t3);
if( mode == hal::STORE_ALIGNED_NOCACHE )
{
_mm_stream_si128((__m128i*)(ptr), v0);
_mm_stream_si128((__m128i*)(ptr + 2), v1);
_mm_stream_si128((__m128i*)(ptr + 4), v2);
_mm_stream_si128((__m128i*)(ptr + 6), v3);
}
else if( mode == hal::STORE_ALIGNED )
{
_mm_store_si128((__m128i*)(ptr), v0);
_mm_store_si128((__m128i*)(ptr + 2), v1);
_mm_store_si128((__m128i*)(ptr + 4), v2);
_mm_store_si128((__m128i*)(ptr + 6), v3);
}
else
{
_mm_storeu_si128((__m128i*)(ptr), v0);
_mm_storeu_si128((__m128i*)(ptr + 2), v1);
_mm_storeu_si128((__m128i*)(ptr + 4), v2);
_mm_storeu_si128((__m128i*)(ptr + 6), v3);
}
}
#define OPENCV_HAL_IMPL_SSE_LOADSTORE_INTERLEAVE(_Tpvec0, _Tp0, suffix0, _Tpvec1, _Tp1, suffix1) \
@@ -2298,27 +2549,30 @@ inline void v_load_deinterleave( const _Tp0* ptr, _Tpvec0& a0, _Tpvec0& b0, _Tpv
c0 = v_reinterpret_as_##suffix0(c1); \
d0 = v_reinterpret_as_##suffix0(d1); \
} \
inline void v_store_interleave( _Tp0* ptr, const _Tpvec0& a0, const _Tpvec0& b0 ) \
inline void v_store_interleave( _Tp0* ptr, const _Tpvec0& a0, const _Tpvec0& b0, \
hal::StoreMode mode = hal::STORE_UNALIGNED ) \
{ \
_Tpvec1 a1 = v_reinterpret_as_##suffix1(a0); \
_Tpvec1 b1 = v_reinterpret_as_##suffix1(b0); \
v_store_interleave((_Tp1*)ptr, a1, b1); \
v_store_interleave((_Tp1*)ptr, a1, b1, mode); \
} \
inline void v_store_interleave( _Tp0* ptr, const _Tpvec0& a0, const _Tpvec0& b0, const _Tpvec0& c0 ) \
inline void v_store_interleave( _Tp0* ptr, const _Tpvec0& a0, const _Tpvec0& b0, \
const _Tpvec0& c0, hal::StoreMode mode = hal::STORE_UNALIGNED ) \
{ \
_Tpvec1 a1 = v_reinterpret_as_##suffix1(a0); \
_Tpvec1 b1 = v_reinterpret_as_##suffix1(b0); \
_Tpvec1 c1 = v_reinterpret_as_##suffix1(c0); \
v_store_interleave((_Tp1*)ptr, a1, b1, c1); \
v_store_interleave((_Tp1*)ptr, a1, b1, c1, mode); \
} \
inline void v_store_interleave( _Tp0* ptr, const _Tpvec0& a0, const _Tpvec0& b0, \
const _Tpvec0& c0, const _Tpvec0& d0 ) \
const _Tpvec0& c0, const _Tpvec0& d0, \
hal::StoreMode mode = hal::STORE_UNALIGNED ) \
{ \
_Tpvec1 a1 = v_reinterpret_as_##suffix1(a0); \
_Tpvec1 b1 = v_reinterpret_as_##suffix1(b0); \
_Tpvec1 c1 = v_reinterpret_as_##suffix1(c0); \
_Tpvec1 d1 = v_reinterpret_as_##suffix1(d0); \
v_store_interleave((_Tp1*)ptr, a1, b1, c1, d1); \
v_store_interleave((_Tp1*)ptr, a1, b1, c1, d1, mode); \
}
OPENCV_HAL_IMPL_SSE_LOADSTORE_INTERLEAVE(v_int8x16, schar, s8, v_uint8x16, uchar, u8)
modules/core/include/opencv2/core/hal/intrin_vsx.hpp
+10 -3
View File
@@ -249,6 +249,10 @@ inline void v_store(_Tp* ptr, const _Tpvec& a) \
{ st(a.val, 0, ptr); } \
inline void v_store_aligned(VSX_UNUSED(_Tp* ptr), const _Tpvec& a) \
{ st_a(a.val, 0, ptr); } \
inline void v_store_aligned_nocache(VSX_UNUSED(_Tp* ptr), const _Tpvec& a) \
{ st_a(a.val, 0, ptr); } \
inline void v_store(_Tp* ptr, const _Tpvec& a, hal::StoreMode mode) \
{ if(mode == hal::STORE_UNALIGNED) st(a.val, 0, ptr); else st_a(a.val, 0, ptr); } \
inline void v_store_low(_Tp* ptr, const _Tpvec& a) \
{ vec_st_l8(a.val, ptr); } \
inline void v_store_high(_Tp* ptr, const _Tpvec& a) \
@@ -281,13 +285,16 @@ inline void v_load_deinterleave(const _Tp* ptr, _Tpvec& a, \
inline void v_load_deinterleave(const _Tp* ptr, _Tpvec& a, _Tpvec& b, \
_Tpvec& c, _Tpvec& d) \
{ vec_ld_deinterleave(ptr, a.val, b.val, c.val, d.val); } \
inline void v_store_interleave(_Tp* ptr, const _Tpvec& a, const _Tpvec& b) \
inline void v_store_interleave(_Tp* ptr, const _Tpvec& a, const _Tpvec& b, \
hal::StoreMode /*mode*/=hal::STORE_UNALIGNED) \
{ vec_st_interleave(a.val, b.val, ptr); } \
inline void v_store_interleave(_Tp* ptr, const _Tpvec& a, \
const _Tpvec& b, const _Tpvec& c) \
const _Tpvec& b, const _Tpvec& c, \
hal::StoreMode /*mode*/=hal::STORE_UNALIGNED) \
{ vec_st_interleave(a.val, b.val, c.val, ptr); } \
inline void v_store_interleave(_Tp* ptr, const _Tpvec& a, const _Tpvec& b, \
const _Tpvec& c, const _Tpvec& d) \
const _Tpvec& c, const _Tpvec& d, \
hal::StoreMode /*mode*/=hal::STORE_UNALIGNED) \
{ vec_st_interleave(a.val, b.val, c.val, d.val, ptr); }
OPENCV_HAL_IMPL_VSX_INTERLEAVE(uchar, v_uint8x16)
modules/core/include/opencv2/core/utility.hpp
+12
View File
@@ -457,6 +457,18 @@ Returns empty string if feature is not defined
*/
CV_EXPORTS_W String getHardwareFeatureName(int feature);
/** @brief Returns list of CPU features enabled during compilation.
Returned value is a string containing space separated list of CPU features with following markers:
- no markers - baseline features
- prefix `*` - features enabled in dispatcher
- suffix `?` - features enabled but not available in HW
Example: `SSE SSE2 SSE3 *SSE4.1 *SSE4.2 *FP16 *AVX *AVX2 *AVX512-SKX?`
*/
CV_EXPORTS std::string getCPUFeaturesLine();
/** @brief Returns the number of logical CPUs available for the process.
*/
CV_EXPORTS_W int getNumberOfCPUs();