mirror of
https://github.com/opencv/opencv.git
synced 2026-07-31 08:13:04 +04:00
Merge branch 4.x
This commit is contained in:
Alexander Smorkalov
@@ -172,6 +172,11 @@
|
||||
# define CV_MSA 1
|
||||
#endif
|
||||
|
||||
#ifdef CV_CPU_COMPILE_LSX
|
||||
# include <lsxintrin.h>
|
||||
# define CV_LSX 1
|
||||
#endif
|
||||
|
||||
#ifdef CV_CPU_COMPILE_LASX
|
||||
# include <lasxintrin.h>
|
||||
# define CV_LASX 1
|
||||
@@ -376,6 +381,10 @@ struct VZeroUpperGuard {
|
||||
# define CV_RVV 0
|
||||
#endif
|
||||
|
||||
#ifndef CV_LSX
|
||||
# define CV_LSX 0
|
||||
#endif
|
||||
|
||||
#ifndef CV_LASX
|
||||
# define CV_LASX 0
|
||||
#endif
|
||||
|
||||
@@ -525,6 +525,27 @@
|
||||
#endif
|
||||
#define __CV_CPU_DISPATCH_CHAIN_RVV(fn, args, mode, ...) CV_CPU_CALL_RVV(fn, args); __CV_EXPAND(__CV_CPU_DISPATCH_CHAIN_ ## mode(fn, args, __VA_ARGS__))
|
||||
|
||||
#if !defined CV_DISABLE_OPTIMIZATION && defined CV_ENABLE_INTRINSICS && defined CV_CPU_COMPILE_LSX
|
||||
# define CV_TRY_LSX 1
|
||||
# define CV_CPU_FORCE_LSX 1
|
||||
# define CV_CPU_HAS_SUPPORT_LSX 1
|
||||
# define CV_CPU_CALL_LSX(fn, args) return (cpu_baseline::fn args)
|
||||
# define CV_CPU_CALL_LSX_(fn, args) return (opt_LSX::fn args)
|
||||
#elif !defined CV_DISABLE_OPTIMIZATION && defined CV_ENABLE_INTRINSICS && defined CV_CPU_DISPATCH_COMPILE_LSX
|
||||
# define CV_TRY_LSX 1
|
||||
# define CV_CPU_FORCE_LSX 0
|
||||
# define CV_CPU_HAS_SUPPORT_LSX (cv::checkHardwareSupport(CV_CPU_LSX))
|
||||
# define CV_CPU_CALL_LSX(fn, args) if (CV_CPU_HAS_SUPPORT_LSX) return (opt_LSX::fn args)
|
||||
# define CV_CPU_CALL_LSX_(fn, args) if (CV_CPU_HAS_SUPPORT_LSX) return (opt_LSX::fn args)
|
||||
#else
|
||||
# define CV_TRY_LSX 0
|
||||
# define CV_CPU_FORCE_LSX 0
|
||||
# define CV_CPU_HAS_SUPPORT_LSX 0
|
||||
# define CV_CPU_CALL_LSX(fn, args)
|
||||
# define CV_CPU_CALL_LSX_(fn, args)
|
||||
#endif
|
||||
#define __CV_CPU_DISPATCH_CHAIN_LSX(fn, args, mode, ...) CV_CPU_CALL_LSX(fn, args); __CV_EXPAND(__CV_CPU_DISPATCH_CHAIN_ ## mode(fn, args, __VA_ARGS__))
|
||||
|
||||
#if !defined CV_DISABLE_OPTIMIZATION && defined CV_ENABLE_INTRINSICS && defined CV_CPU_COMPILE_LASX
|
||||
# define CV_TRY_LASX 1
|
||||
# define CV_CPU_FORCE_LASX 1
|
||||
|
||||
@@ -269,6 +269,8 @@ namespace cv {
|
||||
|
||||
#define CV_CPU_NEON 100
|
||||
#define CV_CPU_NEON_DOTPROD 101
|
||||
#define CV_CPU_NEON_FP16 102
|
||||
#define CV_CPU_NEON_BF16 103
|
||||
|
||||
#define CV_CPU_MSA 150
|
||||
|
||||
@@ -279,7 +281,8 @@ namespace cv {
|
||||
|
||||
#define CV_CPU_RVV 210
|
||||
|
||||
#define CV_CPU_LASX 230
|
||||
#define CV_CPU_LSX 230
|
||||
#define CV_CPU_LASX 231
|
||||
|
||||
// CPU features groups
|
||||
#define CV_CPU_AVX512_SKX 256
|
||||
@@ -328,6 +331,8 @@ enum CpuFeatures {
|
||||
|
||||
CPU_NEON = 100,
|
||||
CPU_NEON_DOTPROD = 101,
|
||||
CPU_NEON_FP16 = 102,
|
||||
CPU_NEON_BF16 = 103,
|
||||
|
||||
CPU_MSA = 150,
|
||||
|
||||
@@ -338,7 +343,8 @@ enum CpuFeatures {
|
||||
|
||||
CPU_RVV = 210,
|
||||
|
||||
CPU_LASX = 230,
|
||||
CPU_LSX = 230,
|
||||
CPU_LASX = 231,
|
||||
|
||||
CPU_AVX512_SKX = 256, //!< Skylake-X with AVX-512F/CD/BW/DQ/VL
|
||||
CPU_AVX512_COMMON = 257, //!< Common instructions AVX-512F/CD for all CPUs that support AVX-512
|
||||
|
||||
@@ -206,7 +206,7 @@ using namespace CV_CPU_OPTIMIZATION_HAL_NAMESPACE;
|
||||
# undef CV_RVV
|
||||
#endif
|
||||
|
||||
#if (CV_SSE2 || CV_NEON || CV_VSX || CV_MSA || CV_WASM_SIMD || CV_RVV071) && !defined(CV_FORCE_SIMD128_CPP)
|
||||
#if (CV_SSE2 || CV_NEON || CV_VSX || CV_MSA || CV_WASM_SIMD || CV_RVV071 || CV_LSX) && !defined(CV_FORCE_SIMD128_CPP)
|
||||
#define CV__SIMD_FORWARD 128
|
||||
#include "opencv2/core/hal/intrin_forward.hpp"
|
||||
#endif
|
||||
@@ -242,6 +242,10 @@ using namespace CV_CPU_OPTIMIZATION_HAL_NAMESPACE;
|
||||
#include "opencv2/core/hal/intrin_rvv.hpp"
|
||||
#endif
|
||||
|
||||
#elif CV_LSX && !defined(CV_FORCE_SIMD128_CPP)
|
||||
|
||||
#include "opencv2/core/hal/intrin_lsx.hpp"
|
||||
|
||||
#elif CV_LASX
|
||||
#if !defined(CV_FORCE_SIMD128_CPP)
|
||||
#define CV_FORCE_SIMD128_CPP 1
|
||||
@@ -738,7 +742,7 @@ namespace CV__SIMD_NAMESPACE {
|
||||
/** @brief SIMD processing state cleanup call */
|
||||
inline void vx_cleanup() { VXPREFIX(_cleanup)(); }
|
||||
|
||||
#if !CV_SIMD_SCALABLE
|
||||
#if !CV_SIMD_SCALABLE && !(CV_NEON && !defined(CV_FORCE_SIMD128_CPP))
|
||||
// Compatibility layer
|
||||
|
||||
template<typename T> struct VTraits {
|
||||
@@ -1166,6 +1170,74 @@ namespace CV__SIMD_NAMESPACE {
|
||||
|
||||
#endif //!CV_SIMD_SCALABLE
|
||||
|
||||
#if (CV_NEON /* || CV_others */) && !defined(CV_FORCE_SIMD128_CPP)
|
||||
// Compatibility layer for the backend that cleaned up.
|
||||
#define OPENCV_HAL_WRAP_BIN_OP_ADDSUB(_Tpvec) \
|
||||
template<typename... Args> \
|
||||
inline _Tpvec v_add(const _Tpvec& f1, const _Tpvec& f2, const Args&... vf) { \
|
||||
return v_add(v_add(f1, f2), vf...); \
|
||||
}
|
||||
|
||||
OPENCV_HAL_WRAP_BIN_OP_ADDSUB(v_uint8)
|
||||
OPENCV_HAL_WRAP_BIN_OP_ADDSUB(v_uint16)
|
||||
OPENCV_HAL_WRAP_BIN_OP_ADDSUB(v_uint32)
|
||||
OPENCV_HAL_WRAP_BIN_OP_ADDSUB(v_uint64)
|
||||
OPENCV_HAL_WRAP_BIN_OP_ADDSUB(v_int8)
|
||||
OPENCV_HAL_WRAP_BIN_OP_ADDSUB(v_int16)
|
||||
OPENCV_HAL_WRAP_BIN_OP_ADDSUB(v_int32)
|
||||
OPENCV_HAL_WRAP_BIN_OP_ADDSUB(v_int64)
|
||||
OPENCV_HAL_WRAP_BIN_OP_ADDSUB(v_float32)
|
||||
#if CV_SIMD_64F
|
||||
OPENCV_HAL_WRAP_BIN_OP_ADDSUB(v_float64)
|
||||
#endif
|
||||
|
||||
#define OPENCV_HAL_WRAP_BIN_OP_MUL(_Tpvec) \
|
||||
template<typename... Args> \
|
||||
inline _Tpvec v_mul(const _Tpvec& f1, const _Tpvec& f2, const Args&... vf) { \
|
||||
return v_mul(v_mul(f1, f2), vf...); \
|
||||
}
|
||||
OPENCV_HAL_WRAP_BIN_OP_MUL(v_uint8)
|
||||
OPENCV_HAL_WRAP_BIN_OP_MUL(v_int8)
|
||||
OPENCV_HAL_WRAP_BIN_OP_MUL(v_uint16)
|
||||
OPENCV_HAL_WRAP_BIN_OP_MUL(v_uint32)
|
||||
OPENCV_HAL_WRAP_BIN_OP_MUL(v_int16)
|
||||
OPENCV_HAL_WRAP_BIN_OP_MUL(v_int32)
|
||||
OPENCV_HAL_WRAP_BIN_OP_MUL(v_float32)
|
||||
#if CV_SIMD_64F
|
||||
OPENCV_HAL_WRAP_BIN_OP_MUL(v_float64)
|
||||
#endif
|
||||
|
||||
#define OPENCV_HAL_WRAP_EXTRACT(_Tpvec) \
|
||||
inline typename VTraits<_Tpvec>::lane_type v_extract_highest(const _Tpvec& v) \
|
||||
{ \
|
||||
return v_extract_n<VTraits<_Tpvec>::nlanes-1>(v); \
|
||||
}
|
||||
|
||||
OPENCV_HAL_WRAP_EXTRACT(v_uint8)
|
||||
OPENCV_HAL_WRAP_EXTRACT(v_int8)
|
||||
OPENCV_HAL_WRAP_EXTRACT(v_uint16)
|
||||
OPENCV_HAL_WRAP_EXTRACT(v_int16)
|
||||
OPENCV_HAL_WRAP_EXTRACT(v_uint32)
|
||||
OPENCV_HAL_WRAP_EXTRACT(v_int32)
|
||||
OPENCV_HAL_WRAP_EXTRACT(v_uint64)
|
||||
OPENCV_HAL_WRAP_EXTRACT(v_int64)
|
||||
OPENCV_HAL_WRAP_EXTRACT(v_float32)
|
||||
#if CV_SIMD_64F
|
||||
OPENCV_HAL_WRAP_EXTRACT(v_float64)
|
||||
#endif
|
||||
|
||||
#define OPENCV_HAL_WRAP_BROADCAST(_Tpvec) \
|
||||
inline _Tpvec v_broadcast_highest(const _Tpvec& v) \
|
||||
{ \
|
||||
return v_broadcast_element<VTraits<_Tpvec>::nlanes-1>(v); \
|
||||
}
|
||||
|
||||
OPENCV_HAL_WRAP_BROADCAST(v_uint32)
|
||||
OPENCV_HAL_WRAP_BROADCAST(v_int32)
|
||||
OPENCV_HAL_WRAP_BROADCAST(v_float32)
|
||||
|
||||
#endif //CV_NEON
|
||||
|
||||
//! @cond IGNORED
|
||||
|
||||
// backward compatibility
|
||||
|
||||
@@ -96,54 +96,22 @@ inline __m256d _v256_setall_pd(double f64)
|
||||
|
||||
inline __m256i _lasx_packus_h(const __m256i& a, const __m256i& b)
|
||||
{
|
||||
__m256i u8min = __lasx_xvreplgr2vr_h(0);
|
||||
__m256i u8max = __lasx_xvreplgr2vr_h(255);
|
||||
__m256i sat_a = __lasx_xvmax_h(a, u8min);
|
||||
sat_a = __lasx_xvmin_h(sat_a, u8max);
|
||||
__m256i sat_b = __lasx_xvmax_h(b, u8min);
|
||||
sat_b = __lasx_xvmin_h(sat_b, u8max);
|
||||
__m256i byteIndex = _v256_setr_b(0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30,
|
||||
0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30);
|
||||
return __lasx_xvshuf_b(sat_b, sat_a, byteIndex);
|
||||
return __lasx_xvssrarni_bu_h(b, a, 0);
|
||||
}
|
||||
|
||||
inline __m256i _lasx_packs_h(const __m256i& a, const __m256i& b)
|
||||
{
|
||||
__m256i s8min = __lasx_xvreplgr2vr_h(-128);
|
||||
__m256i s8max = __lasx_xvreplgr2vr_h(127);
|
||||
__m256i sat_a = __lasx_xvmax_h(a, s8min);
|
||||
sat_a = __lasx_xvmin_h(sat_a, s8max);
|
||||
__m256i sat_b = __lasx_xvmax_h(b, s8min);
|
||||
sat_b = __lasx_xvmin_h(sat_b, s8max);
|
||||
__m256i byteIndex = _v256_setr_b(0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30,
|
||||
0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30);
|
||||
return __lasx_xvshuf_b(sat_b, sat_a, byteIndex);
|
||||
return __lasx_xvssrarni_b_h(b, a, 0);
|
||||
}
|
||||
|
||||
inline __m256i _lasx_packus_w(const __m256i& a, const __m256i& b)
|
||||
{
|
||||
__m256i u16min = __lasx_xvreplgr2vr_w(0);
|
||||
__m256i u16max = __lasx_xvreplgr2vr_w(0xffff);
|
||||
__m256i sat_a = __lasx_xvmax_w(a, u16min);
|
||||
sat_a = __lasx_xvmin_w(sat_a, u16max);
|
||||
__m256i sat_b = __lasx_xvmax_w(b, u16min);
|
||||
sat_b = __lasx_xvmin_w(sat_b, u16max);
|
||||
__m256i hwordIndex = _v256_setr_h(0, 2, 4, 6, 8, 10, 12, 14,
|
||||
0, 2, 4, 6, 8, 10, 12, 14);
|
||||
return __lasx_xvshuf_h(hwordIndex, sat_b, sat_a);
|
||||
return __lasx_xvssrarni_hu_w(b, a, 0);
|
||||
}
|
||||
|
||||
inline __m256i _lasx_packs_w(const __m256i& a, const __m256i& b)
|
||||
{
|
||||
__m256i s16min = __lasx_xvreplgr2vr_w(-0x8000);
|
||||
__m256i s16max = __lasx_xvreplgr2vr_w(0x7fff);
|
||||
__m256i sat_a = __lasx_xvmax_w(a, s16min);
|
||||
sat_a = __lasx_xvmin_w(sat_a, s16max);
|
||||
__m256i sat_b = __lasx_xvmax_w(b, s16min);
|
||||
sat_b = __lasx_xvmin_w(sat_b, s16max);
|
||||
__m256i hwordIndex = _v256_setr_h(0, 2, 4, 6, 8, 10, 12, 14,
|
||||
0, 2, 4, 6, 8, 10, 12, 14);
|
||||
return __lasx_xvshuf_h(hwordIndex, sat_b, sat_a);
|
||||
return __lasx_xvssrarni_h_w(b, a, 0);
|
||||
}
|
||||
|
||||
inline __m256i _v256_combine(const __m128i& lo, const __m128i& hi)
|
||||
@@ -191,7 +159,7 @@ inline _Tpvec v256_permute4x64(const _Tpvec& a)
|
||||
{ return _Tpvec(_v256_permute4x64<imm>(a.val)); }
|
||||
|
||||
inline __m128i _v256_extract_high(const __m256i& v)
|
||||
{ __m256i temp256i = __lasx_xvpermi_q(v, v, 0x31);
|
||||
{ __m256i temp256i = __lasx_xvpermi_d(v, 0x4E);
|
||||
return *((__m128i*)&temp256i); }
|
||||
|
||||
inline __m128 _v256_extract_high(const __m256& v)
|
||||
@@ -211,10 +179,7 @@ inline __m128d _v256_extract_low(const __m256d& v)
|
||||
|
||||
inline __m256i _v256_packs_epu32(const __m256i& a, const __m256i& b)
|
||||
{
|
||||
const __m256i maxv = __lasx_xvreplgr2vr_w(65535);
|
||||
__m256i am = __lasx_xvmin_wu(a, maxv);
|
||||
__m256i bm = __lasx_xvmin_wu(b, maxv);
|
||||
return _lasx_packus_w(am, bm);
|
||||
return __lasx_xvssrlrni_hu_w(b, a, 0);
|
||||
}
|
||||
|
||||
template<int i>
|
||||
@@ -869,14 +834,11 @@ OPENCV_HAL_IMPL_LASX_BIN_FUNC(v_mul_wrap, v_int16x16, __lasx_xvmul_h)
|
||||
|
||||
inline v_uint8x32 v_mul_wrap(const v_uint8x32& a, const v_uint8x32& b)
|
||||
{
|
||||
__m256i ad = __lasx_xvsrai_h(a.val, 8);
|
||||
__m256i bd = __lasx_xvsrai_h(b.val, 8);
|
||||
__m256i p0 = __lasx_xvmul_h(a.val, b.val);
|
||||
__m256i p1 = __lasx_xvslli_h(__lasx_xvmul_h(ad, bd), 8);
|
||||
|
||||
const __m256i b01 = __lasx_xvreplgr2vr_w(0xFF00FF00);
|
||||
return v_uint8x32(__lasx_xvbitsel_v(p0, p1, b01));
|
||||
__m256i p0 = __lasx_xvmulwev_h_bu(a.val, b.val);
|
||||
__m256i p1 = __lasx_xvmulwod_h_bu(a.val, b.val);
|
||||
return v_uint8x32(__lasx_xvpackev_b(p1, p0));
|
||||
}
|
||||
|
||||
inline v_int8x32 v_mul_wrap(const v_int8x32& a, const v_int8x32& b)
|
||||
{
|
||||
return v_reinterpret_as_s8(v_mul_wrap(v_reinterpret_as_u8(a), v_reinterpret_as_u8(b)));
|
||||
@@ -963,14 +925,7 @@ inline v_uint16x16 v_mul_hi(const v_uint16x16& a, const v_uint16x16& b) { return
|
||||
|
||||
OPENCV_HAL_IMPL_LASX_SHIFT_OP(v_uint16x16, v_int16x16, h, __lasx_xvsra_h)
|
||||
OPENCV_HAL_IMPL_LASX_SHIFT_OP(v_uint32x8, v_int32x8, w, __lasx_xvsra_w)
|
||||
|
||||
inline __m256i _v256_srai_dx(const __m256i a, const __m256i shift)
|
||||
{
|
||||
__m256i d = __lasx_xvreplgr2vr_d((int64)1 << 63);
|
||||
__m256i r = __lasx_xvsrl_d(__lasx_xvadd_d(a, d), shift);
|
||||
return __lasx_xvsub_d(r, __lasx_xvsrl_d(d, shift));
|
||||
}
|
||||
OPENCV_HAL_IMPL_LASX_SHIFT_OP(v_uint64x4, v_int64x4, d, _v256_srai_dx)
|
||||
OPENCV_HAL_IMPL_LASX_SHIFT_OP(v_uint64x4, v_int64x4, d, __lasx_xvsra_d)
|
||||
|
||||
|
||||
/** Bitwise logic **/
|
||||
@@ -979,7 +934,7 @@ OPENCV_HAL_IMPL_LASX_SHIFT_OP(v_uint64x4, v_int64x4, d, _v256_srai_dx)
|
||||
OPENCV_HAL_IMPL_LASX_BIN_OP(|, _Tpvec, __lasx_xvor_##suffix) \
|
||||
OPENCV_HAL_IMPL_LASX_BIN_OP(^, _Tpvec, __lasx_xvxor_##suffix) \
|
||||
inline _Tpvec operator ~ (const _Tpvec& a) \
|
||||
{ return _Tpvec(__lasx_xvxor_##suffix(a.val, not_const)); }
|
||||
{ return _Tpvec(__lasx_xvnori_b(a.val, 0)); }
|
||||
|
||||
OPENCV_HAL_IMPL_LASX_LOGIC_OP(v_uint8x32, v, __lasx_xvreplgr2vr_w(-1))
|
||||
OPENCV_HAL_IMPL_LASX_LOGIC_OP(v_int8x32, v, __lasx_xvreplgr2vr_w(-1))
|
||||
@@ -1224,11 +1179,9 @@ inline v_int8x32 v_reverse(const v_int8x32 &a)
|
||||
|
||||
inline v_uint16x16 v_reverse(const v_uint16x16 &a)
|
||||
{
|
||||
static const __m256i perm = _v256_setr_b(
|
||||
14, 15, 12, 13, 10, 11, 8, 9, 6, 7, 4, 5, 2, 3, 0, 1,
|
||||
14, 15, 12, 13, 10, 11, 8, 9, 6, 7, 4, 5, 2, 3, 0, 1);
|
||||
__m256i vec = __lasx_xvshuf_b(a.val, a.val, perm);
|
||||
return v_uint16x16(__lasx_xvpermi_q(vec, vec, 1));
|
||||
__m256i vec = __lasx_xvshuf4i_h(a.val, 0x1B);
|
||||
vec = __lasx_xvshuf4i_w(vec, 0x4E);
|
||||
return v_uint16x16(__lasx_xvpermi_d(vec, 0x4E));
|
||||
}
|
||||
|
||||
inline v_int16x16 v_reverse(const v_int16x16 &a)
|
||||
@@ -1236,8 +1189,8 @@ inline v_int16x16 v_reverse(const v_int16x16 &a)
|
||||
|
||||
inline v_uint32x8 v_reverse(const v_uint32x8 &a)
|
||||
{
|
||||
static const __m256i perm = _v256_setr_w(7, 6, 5, 4, 3, 2, 1, 0);
|
||||
return v_uint32x8(__lasx_xvperm_w(a.val, perm));
|
||||
__m256i vec = __lasx_xvshuf4i_w(a.val, 0x1B);
|
||||
return v_uint32x8(__lasx_xvpermi_d(vec, 0x4E));
|
||||
}
|
||||
|
||||
inline v_int32x8 v_reverse(const v_int32x8 &a)
|
||||
@@ -1266,17 +1219,19 @@ inline unsigned v_reduce_sum(const v_uint8x32& a)
|
||||
__m256i t1 = __lasx_xvhaddw_hu_bu(a.val, a.val);
|
||||
__m256i t2 = __lasx_xvhaddw_wu_hu(t1, t1);
|
||||
__m256i t3 = __lasx_xvhaddw_du_wu(t2, t2);
|
||||
return (unsigned)(((v4u64)t3)[0]+((v4u64)t3)[1]+((v4u64)t3)[2]+((v4u64)t3)[3]);
|
||||
__m256i t4 = __lasx_xvhaddw_qu_du(t3, t3);
|
||||
return (unsigned)(((v8u32)t4)[0]+((v8u32)t4)[4]);
|
||||
}
|
||||
|
||||
inline int v_reduce_sum(const v_int8x32& a)
|
||||
{
|
||||
__m256i t1 = __lasx_xvhaddw_h_b(a.val, a.val);
|
||||
__m256i t2 = __lasx_xvhaddw_w_h(t1, t1);
|
||||
__m256i t3 = __lasx_xvhaddw_d_w(t2, t2);
|
||||
return (int)(((v4i64)t3)[0]+((v4i64)t3)[1]+((v4i64)t3)[2]+((v4i64)t3)[3]);
|
||||
__m256i t4 = __lasx_xvhaddw_q_d(t3, t3);
|
||||
return (int)(((v8i32)t4)[0]+((v8i32)t4)[4]);
|
||||
}
|
||||
|
||||
|
||||
#define OPENCV_HAL_IMPL_LASX_REDUCE_32(_Tpvec, sctype, func, intrin) \
|
||||
inline sctype v_reduce_##func(const _Tpvec& a) \
|
||||
{ \
|
||||
@@ -1344,7 +1299,8 @@ OPENCV_HAL_IMPL_LASX_REDUCE_FLT(max, __lsx_vfmax_s)
|
||||
inline int v_reduce_sum(const v_int32x8& a)
|
||||
{
|
||||
__m256i t1 = __lasx_xvhaddw_d_w(a.val, a.val);
|
||||
return (int)(((v4i64)t1)[0]+((v4i64)t1)[1]+((v4i64)t1)[2]+((v4i64)t1)[3]);
|
||||
__m256i t2 = __lasx_xvhaddw_q_d(t1, t1);
|
||||
return (int)(((v8i32)t2)[0]+((v8i32)t2)[4]);
|
||||
}
|
||||
|
||||
inline unsigned v_reduce_sum(const v_uint32x8& a)
|
||||
@@ -1367,13 +1323,13 @@ inline float v_reduce_sum(const v_float32x8& a)
|
||||
|
||||
inline uint64 v_reduce_sum(const v_uint64x4& a)
|
||||
{
|
||||
uint64 *pa = (uint64*)&a;
|
||||
return pa[0] + pa[1] + pa[2] + pa[3];
|
||||
__m256i t0 = __lasx_xvhaddw_qu_du(a.val, a.val);
|
||||
return (uint64)(((v4u64)t0)[0] + ((v4u64)t0)[2]);
|
||||
}
|
||||
inline int64 v_reduce_sum(const v_int64x4& a)
|
||||
{
|
||||
int64 *pa = (int64*)&a;
|
||||
return pa[0] + pa[1] + pa[2] + pa[3];
|
||||
__m256i t0 = __lasx_xvhaddw_q_d(a.val, a.val);
|
||||
return (int64)(((v4i64)t0)[0] + ((v4i64)t0)[2]);
|
||||
}
|
||||
inline double v_reduce_sum(const v_float64x4& a)
|
||||
{
|
||||
@@ -1406,7 +1362,8 @@ inline unsigned v_reduce_sad(const v_uint8x32& a, const v_uint8x32& b)
|
||||
__m256i t1 = __lasx_xvhaddw_hu_bu(t0, t0);
|
||||
__m256i t2 = __lasx_xvhaddw_wu_hu(t1, t1);
|
||||
__m256i t3 = __lasx_xvhaddw_du_wu(t2, t2);
|
||||
return (unsigned)(((v4u64)t3)[0]+((v4u64)t3)[1]+((v4u64)t3)[2]+((v4u64)t3)[3]);
|
||||
__m256i t4 = __lasx_xvhaddw_qu_du(t3, t3);
|
||||
return (unsigned)(((v8u32)t4)[0]+((v8u32)t4)[4]);
|
||||
}
|
||||
inline unsigned v_reduce_sad(const v_int8x32& a, const v_int8x32& b)
|
||||
{
|
||||
@@ -1414,7 +1371,8 @@ inline unsigned v_reduce_sad(const v_int8x32& a, const v_int8x32& b)
|
||||
__m256i t1 = __lasx_xvhaddw_hu_bu(t0, t0);
|
||||
__m256i t2 = __lasx_xvhaddw_wu_hu(t1, t1);
|
||||
__m256i t3 = __lasx_xvhaddw_du_wu(t2, t2);
|
||||
return (unsigned)(((v4u64)t3)[0]+((v4u64)t3)[1]+((v4u64)t3)[2]+((v4u64)t3)[3]);
|
||||
__m256i t4 = __lasx_xvhaddw_qu_du(t3, t3);
|
||||
return (unsigned)(((v8u32)t4)[0]+((v8u32)t4)[4]);
|
||||
}
|
||||
inline unsigned v_reduce_sad(const v_uint16x16& a, const v_uint16x16& b)
|
||||
{
|
||||
@@ -1445,36 +1403,13 @@ inline float v_reduce_sad(const v_float32x8& a, const v_float32x8& b)
|
||||
|
||||
/** Popcount **/
|
||||
inline v_uint8x32 v_popcount(const v_uint8x32& a)
|
||||
{
|
||||
__m256i _popcnt_table = _v256_setr_b(0, 1, 1, 2, 1, 2, 2, 3, 1, 2, 2, 3, 2, 3, 3, 4,
|
||||
0, 1, 1, 2, 1, 2, 2, 3, 1, 2, 2, 3, 2, 3, 3, 4);
|
||||
__m256i _popcnt_mask = __lasx_xvreplgr2vr_b(0x0F);
|
||||
return v_uint8x32(__lasx_xvadd_b(__lasx_xvshuf_b(_popcnt_table, _popcnt_table, __lasx_xvand_v(a.val, _popcnt_mask)),
|
||||
__lasx_xvshuf_b(_popcnt_table, _popcnt_table, __lasx_xvand_v(__lasx_xvsrli_h(a.val, 4), _popcnt_mask))));
|
||||
}
|
||||
{ return v_uint8x32(__lasx_xvpcnt_b(a.val)); }
|
||||
inline v_uint16x16 v_popcount(const v_uint16x16& a)
|
||||
{
|
||||
v_uint8x32 p = v_popcount(v_reinterpret_as_u8(a));
|
||||
p += v_rotate_right<1>(p);
|
||||
return v_reinterpret_as_u16(p) & v_uint16x16(__lasx_xvreplgr2vr_h(0x00ff));
|
||||
}
|
||||
{ return v_uint16x16(__lasx_xvpcnt_h(a.val)); }
|
||||
inline v_uint32x8 v_popcount(const v_uint32x8& a)
|
||||
{
|
||||
v_uint8x32 p = v_popcount(v_reinterpret_as_u8(a));
|
||||
p += v_rotate_right<1>(p);
|
||||
p += v_rotate_right<2>(p);
|
||||
return v_reinterpret_as_u32(p) & v_uint32x8(__lasx_xvreplgr2vr_w(0x000000ff));
|
||||
}
|
||||
{ return v_uint32x8(__lasx_xvpcnt_w(a.val)); }
|
||||
inline v_uint64x4 v_popcount(const v_uint64x4& a)
|
||||
{
|
||||
v_uint8x32 atemp = v_popcount(v_reinterpret_as_u8(a));
|
||||
uint8_t *pa = (uint8_t*)&atemp;
|
||||
uint64 v[4];
|
||||
for (int i = 0; i < 4; ++i) {
|
||||
v[i] = pa[i*8] + pa[i*8+1] + pa[i*8+2] + pa[i*8+3] + pa[i*8+4] + pa[i*8+5] + pa[i*8+6] + pa[i*8+7];
|
||||
}
|
||||
return v_uint64x4(v[0], v[1], v[2], v[3]);
|
||||
}
|
||||
{ return v_uint64x4(__lasx_xvpcnt_d(a.val)); }
|
||||
inline v_uint8x32 v_popcount(const v_int8x32& a)
|
||||
{ return v_popcount(v_reinterpret_as_u8(a)); }
|
||||
inline v_uint16x16 v_popcount(const v_int16x16& a)
|
||||
@@ -1500,10 +1435,9 @@ OPENCV_HAL_IMPL_REINTERPRET_INT(double, int64)
|
||||
|
||||
inline int v_signmask(const v_int8x32& a)
|
||||
{
|
||||
int mask = 0;
|
||||
int8_t *pa = (int8_t*)&a;
|
||||
for( int i = 0; i < 32; i++ )
|
||||
mask |= (reinterpret_int(pa[i]) < 0) << i;
|
||||
__m256i result = __lasx_xvmskltz_b(a.val);
|
||||
int mask = __lasx_xvpickve2gr_w(result, 0);
|
||||
mask |= (__lasx_xvpickve2gr_w(result, 4) << 16);
|
||||
return mask;
|
||||
}
|
||||
inline int v_signmask(const v_uint8x32& a)
|
||||
@@ -1516,10 +1450,9 @@ inline int v_signmask(const v_uint16x16& a)
|
||||
|
||||
inline int v_signmask(const v_int32x8& a)
|
||||
{
|
||||
int mask = 0;
|
||||
int *pa = (int*)&a;
|
||||
for( int i = 0; i < 8; i++ )
|
||||
mask |= (pa[i] < 0) << i;
|
||||
__m256i result = __lasx_xvmskltz_w(a.val);
|
||||
int mask = __lasx_xvpickve2gr_w(result, 0);
|
||||
mask |= (__lasx_xvpickve2gr_w(result, 4) << 4);
|
||||
return mask;
|
||||
}
|
||||
inline int v_signmask(const v_uint32x8& a)
|
||||
@@ -1527,10 +1460,9 @@ inline int v_signmask(const v_uint32x8& a)
|
||||
|
||||
inline int v_signmask(const v_int64x4& a)
|
||||
{
|
||||
int mask = 0;
|
||||
int64 *pa = (int64*)&a;
|
||||
for( int i = 0; i < 4; i++ )
|
||||
mask |= (pa[i] < 0) << i;
|
||||
__m256i result = __lasx_xvmskltz_d(a.val);
|
||||
int mask = __lasx_xvpickve2gr_d(result, 0);
|
||||
mask |= (__lasx_xvpickve2gr_w(result, 4) << 2);
|
||||
return mask;
|
||||
}
|
||||
inline int v_signmask(const v_uint64x4& a)
|
||||
@@ -1592,7 +1524,7 @@ OPENCV_HAL_IMPL_LASX_MULADD(v_float64x4, d)
|
||||
|
||||
inline v_int32x8 v_fma(const v_int32x8& a, const v_int32x8& b, const v_int32x8& c)
|
||||
{
|
||||
return a * b + c;
|
||||
return v_int32x8(__lasx_xvmadd_w(c.val, a.val, b.val));
|
||||
}
|
||||
|
||||
inline v_int32x8 v_muladd(const v_int32x8& a, const v_int32x8& b, const v_int32x8& c)
|
||||
@@ -1601,17 +1533,10 @@ inline v_int32x8 v_muladd(const v_int32x8& a, const v_int32x8& b, const v_int32x
|
||||
}
|
||||
|
||||
inline v_float32x8 v_invsqrt(const v_float32x8& x)
|
||||
{
|
||||
v_float32x8 half = x * v_float32x8(0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5);
|
||||
v_float32x8 t = v_float32x8(__lasx_xvfrsqrt_s(x.val));
|
||||
t *= v_float32x8(1.5, 1.5, 1.5, 1.5, 1.5, 1.5, 1.5, 1.5) - ((t * t) * half);
|
||||
return t;
|
||||
}
|
||||
{ return v_float32x8(__lasx_xvfrsqrt_s(x.val)); }
|
||||
|
||||
inline v_float64x4 v_invsqrt(const v_float64x4& x)
|
||||
{
|
||||
return v_float64x4(1., 1., 1., 1.) / v_sqrt(x);
|
||||
}
|
||||
{ return v_float64x4(__lasx_xvfrsqrt_d(x.val)); }
|
||||
|
||||
/** Absolute values **/
|
||||
#define OPENCV_HAL_IMPL_LASX_ABS(_Tpvec, suffix) \
|
||||
@@ -1629,28 +1554,18 @@ inline v_float64x4 v_abs(const v_float64x4& x)
|
||||
|
||||
/** Absolute difference **/
|
||||
inline v_uint8x32 v_absdiff(const v_uint8x32& a, const v_uint8x32& b)
|
||||
{ return v_add_wrap(a - b, b - a); }
|
||||
{ return (v_uint8x32)__lasx_xvabsd_bu(a.val, b.val); }
|
||||
inline v_uint16x16 v_absdiff(const v_uint16x16& a, const v_uint16x16& b)
|
||||
{ return v_add_wrap(a - b, b - a); }
|
||||
{ return (v_uint16x16)__lasx_xvabsd_hu(a.val, b.val); }
|
||||
inline v_uint32x8 v_absdiff(const v_uint32x8& a, const v_uint32x8& b)
|
||||
{ return v_max(a, b) - v_min(a, b); }
|
||||
{ return (v_uint32x8)__lasx_xvabsd_wu(a.val, b.val); }
|
||||
|
||||
inline v_uint8x32 v_absdiff(const v_int8x32& a, const v_int8x32& b)
|
||||
{
|
||||
v_int8x32 d = v_sub_wrap(a, b);
|
||||
v_int8x32 m = a < b;
|
||||
return v_reinterpret_as_u8(v_sub_wrap(d ^ m, m));
|
||||
}
|
||||
|
||||
{ return (v_uint8x32)__lasx_xvabsd_b(a.val, b.val); }
|
||||
inline v_uint16x16 v_absdiff(const v_int16x16& a, const v_int16x16& b)
|
||||
{ return v_reinterpret_as_u16(v_sub_wrap(v_max(a, b), v_min(a, b))); }
|
||||
|
||||
{ return (v_uint16x16)__lasx_xvabsd_h(a.val, b.val); }
|
||||
inline v_uint32x8 v_absdiff(const v_int32x8& a, const v_int32x8& b)
|
||||
{
|
||||
v_int32x8 d = a - b;
|
||||
v_int32x8 m = a < b;
|
||||
return v_reinterpret_as_u32((d ^ m) - m);
|
||||
}
|
||||
{ return (v_uint32x8)__lasx_xvabsd_w(a.val, b.val); }
|
||||
|
||||
inline v_float32x8 v_absdiff(const v_float32x8& a, const v_float32x8& b)
|
||||
{ return v_abs(a - b); }
|
||||
@@ -1740,28 +1655,8 @@ inline v_float64x4 v_cvt_f64_high(const v_float32x8& a)
|
||||
return v_float64x4(__lasx_xvfcvtl_d_s((__m256)ahigh));
|
||||
}
|
||||
|
||||
// from (Mysticial and wim) https://stackoverflow.com/q/41144668
|
||||
inline v_float64x4 v_cvt_f64(const v_int64x4& v)
|
||||
{
|
||||
// constants encoded as floating-point
|
||||
__m256i magic_i_lo = __lasx_xvreplgr2vr_d(0x4330000000000000);
|
||||
__m256i magic_i_hi32 = __lasx_xvreplgr2vr_d(0x4530000080000000);
|
||||
__m256i magic_i_all = __lasx_xvreplgr2vr_d(0x4530000080100000);
|
||||
__m256d magic_d_all = _lasx_256_castsi256_pd(magic_i_all);
|
||||
|
||||
// Blend the 32 lowest significant bits of v with magic_int_lo
|
||||
__m256i mask = _v256_set_w(0, -1, 0, -1, 0, -1, 0, -1);
|
||||
__m256i v_lo = __lasx_xvbitsel_v(magic_i_lo, v.val, mask);
|
||||
// Extract the 32 most significant bits of v
|
||||
__m256i v_hi = __lasx_xvsrli_d(v.val, 32);
|
||||
// Flip the msb of v_hi and blend with 0x45300000
|
||||
v_hi = __lasx_xvxor_v(v_hi, magic_i_hi32);
|
||||
// Compute in double precision
|
||||
__m256d v_hi_dbl = __lasx_xvfsub_d(_lasx_256_castsi256_pd(v_hi), magic_d_all);
|
||||
// (v_hi - magic_d_all) + v_lo Do not assume associativity of floating point addition
|
||||
__m256d result = __lasx_xvfadd_d(v_hi_dbl, _lasx_256_castsi256_pd(v_lo));
|
||||
return v_float64x4(result);
|
||||
}
|
||||
{ return v_float64x4(__lasx_xvffint_d_l(v.val)); }
|
||||
|
||||
////////////// Lookup table access ////////////////////
|
||||
|
||||
@@ -1967,11 +1862,9 @@ inline v_float32x8 v_interleave_pairs(const v_float32x8& vec)
|
||||
inline v_int8x32 v_pack_triplets(const v_int8x32& vec)
|
||||
{
|
||||
__m256i vzero = __lasx_xvreplgr2vr_w(0);
|
||||
__m256i t1 = __lasx_xvshuf_b(vec.val, vec.val,
|
||||
_v256_set_d(0xffffff0f0e0d0c0a, 0x0908060504020100, 0xffffff0f0e0d0c0a, 0x0908060504020100));
|
||||
__m256i t2 = __lasx_xvshuf_b(vzero, t1,
|
||||
_v256_set_d(0x1211100c0b0a0908, 0x0706050403020100, 0x1211100c0b0a0908, 0x0706050403020100));
|
||||
return v_int8x32(__lasx_xvperm_w(t2,
|
||||
__m256i t1 = __lasx_xvshuf_b(vzero, vec.val,
|
||||
_v256_set_d(0x1211100f0e0d0c0a, 0x0908060504020100, 0x1211100f0e0d0c0a, 0x0908060504020100));
|
||||
return v_int8x32(__lasx_xvperm_w(t1,
|
||||
_v256_set_d(0x0000000700000007, 0x0000000600000005, 0x0000000400000002, 0x0000000100000000)));
|
||||
}
|
||||
inline v_uint8x32 v_pack_triplets(const v_uint8x32& vec)
|
||||
@@ -1980,11 +1873,9 @@ inline v_uint8x32 v_pack_triplets(const v_uint8x32& vec)
|
||||
inline v_int16x16 v_pack_triplets(const v_int16x16& vec)
|
||||
{
|
||||
__m256i vzero = __lasx_xvreplgr2vr_w(0);
|
||||
__m256i t1 = __lasx_xvshuf_b(vec.val, vec.val,
|
||||
_v256_set_d(0xffff0f0e0d0c0b0a, 0x0908050403020100, 0xffff0f0e0d0c0b0a, 0x0908050403020100));
|
||||
__m256i t2 = __lasx_xvshuf_b(vzero, t1,
|
||||
_v256_set_d(0x11100d0c0b0a0908, 0x0706050403020100, 0x11100d0c0b0a0908, 0x0706050403020100));
|
||||
return v_int16x16(__lasx_xvperm_w(t2,
|
||||
__m256i t1 = __lasx_xvshuf_b(vzero, vec.val,
|
||||
_v256_set_d(0x11100f0e0d0c0b0a, 0x0908050403020100, 0x11100f0e0d0c0b0a, 0x0908050403020100));
|
||||
return v_int16x16(__lasx_xvperm_w(t1,
|
||||
_v256_set_d(0x0000000700000007, 0x0000000600000005, 0x0000000400000002, 0x0000000100000000)));
|
||||
}
|
||||
inline v_uint16x16 v_pack_triplets(const v_uint16x16& vec)
|
||||
@@ -2018,24 +1909,21 @@ inline v_int32x8 v_dotprod(const v_int16x16& a, const v_int16x16& b, const v_int
|
||||
inline v_int64x4 v_dotprod(const v_int32x8& a, const v_int32x8& b)
|
||||
{
|
||||
__m256i even = __lasx_xvmulwev_d_w(a.val, b.val);
|
||||
__m256i odd = __lasx_xvmulwod_d_w(a.val, b.val);
|
||||
return v_int64x4(__lasx_xvadd_d(even, odd));
|
||||
return v_int64x4(__lasx_xvmaddwod_d_w(even, a.val, b.val));
|
||||
}
|
||||
inline v_int64x4 v_dotprod(const v_int32x8& a, const v_int32x8& b, const v_int64x4& c)
|
||||
{ return v_dotprod(a, b) + c; }
|
||||
{
|
||||
__m256i even = __lasx_xvmaddwev_d_w(c.val, a.val, b.val);
|
||||
return v_int64x4(__lasx_xvmaddwod_d_w(even, a.val, b.val));
|
||||
}
|
||||
|
||||
// 8 >> 32
|
||||
inline v_uint32x8 v_dotprod_expand(const v_uint8x32& a, const v_uint8x32& b)
|
||||
{
|
||||
__m256i even_m = __lasx_xvreplgr2vr_w(0xFF00FF00);
|
||||
__m256i even_a = __lasx_xvbitsel_v(a.val, __lasx_xvreplgr2vr_d(0), even_m);
|
||||
__m256i odd_a = __lasx_xvsrli_h(a.val, 8);
|
||||
|
||||
__m256i even_b = __lasx_xvbitsel_v(b.val, __lasx_xvreplgr2vr_d(0), even_m);
|
||||
__m256i odd_b = __lasx_xvsrli_h(b.val, 8);
|
||||
|
||||
__m256i prod0 = __lasx_xvadd_w(__lasx_xvmulwev_w_h(even_a, even_b), __lasx_xvmulwod_w_h(even_a, even_b));
|
||||
__m256i prod1 = __lasx_xvadd_w(__lasx_xvmulwev_w_h(odd_a, odd_b),__lasx_xvmulwod_w_h(odd_a, odd_b));
|
||||
__m256i even = __lasx_xvmulwev_h_bu(a.val, b.val);
|
||||
__m256i odd = __lasx_xvmulwod_h_bu(a.val, b.val);
|
||||
__m256i prod0 = __lasx_xvhaddw_wu_hu(even, even);
|
||||
__m256i prod1 = __lasx_xvhaddw_wu_hu(odd, odd);
|
||||
return v_uint32x8(__lasx_xvadd_w(prod0, prod1));
|
||||
}
|
||||
inline v_uint32x8 v_dotprod_expand(const v_uint8x32& a, const v_uint8x32& b, const v_uint32x8& c)
|
||||
@@ -2043,14 +1931,10 @@ inline v_uint32x8 v_dotprod_expand(const v_uint8x32& a, const v_uint8x32& b, con
|
||||
|
||||
inline v_int32x8 v_dotprod_expand(const v_int8x32& a, const v_int8x32& b)
|
||||
{
|
||||
__m256i even_a = __lasx_xvsrai_h(__lasx_xvbsll_v(a.val, 1), 8);
|
||||
__m256i odd_a = __lasx_xvsrai_h(a.val, 8);
|
||||
|
||||
__m256i even_b = __lasx_xvsrai_h(__lasx_xvbsll_v(b.val, 1), 8);
|
||||
__m256i odd_b = __lasx_xvsrai_h(b.val, 8);
|
||||
|
||||
__m256i prod0 = __lasx_xvadd_w(__lasx_xvmulwev_w_h(even_a, even_b), __lasx_xvmulwod_w_h(even_a, even_b));
|
||||
__m256i prod1 = __lasx_xvadd_w(__lasx_xvmulwev_w_h(odd_a, odd_b),__lasx_xvmulwod_w_h(odd_a, odd_b));
|
||||
__m256i even = __lasx_xvmulwev_h_b(a.val, b.val);
|
||||
__m256i odd = __lasx_xvmulwod_h_b(a.val, b.val);
|
||||
__m256i prod0 = __lasx_xvhaddw_w_h(even, even);
|
||||
__m256i prod1 = __lasx_xvhaddw_w_h(odd, odd);
|
||||
return v_int32x8(__lasx_xvadd_w(prod0, prod1));
|
||||
}
|
||||
inline v_int32x8 v_dotprod_expand(const v_int8x32& a, const v_int8x32& b, const v_int32x8& c)
|
||||
@@ -2059,36 +1943,24 @@ inline v_int32x8 v_dotprod_expand(const v_int8x32& a, const v_int8x32& b, const
|
||||
// 16 >> 64
|
||||
inline v_uint64x4 v_dotprod_expand(const v_uint16x16& a, const v_uint16x16& b)
|
||||
{
|
||||
__m256i mullo = __lasx_xvmul_h(a.val, b.val);
|
||||
__m256i mulhi = __lasx_xvmuh_hu(a.val, b.val);
|
||||
__m256i mul0 = __lasx_xvilvl_h(mulhi, mullo);
|
||||
__m256i mul1 = __lasx_xvilvh_h(mulhi, mullo);
|
||||
|
||||
__m256i p02 = __lasx_xvbitsel_v(mul0, __lasx_xvreplgr2vr_d(0), _v256_set_w(-1, 0, -1, 0, -1, 0, -1, 0));
|
||||
__m256i p13 = __lasx_xvsrli_d(mul0, 32);
|
||||
__m256i p46 = __lasx_xvbitsel_v(mul1, __lasx_xvreplgr2vr_d(0), _v256_set_w(-1, 0, -1, 0, -1, 0, -1, 0));
|
||||
__m256i p57 = __lasx_xvsrli_d(mul1, 32);
|
||||
|
||||
__m256i p15_ = __lasx_xvadd_d(p02, p13);
|
||||
__m256i p9d_ = __lasx_xvadd_d(p46, p57);
|
||||
|
||||
return v_uint64x4(__lasx_xvadd_d(
|
||||
__lasx_xvilvl_d(p9d_, p15_),
|
||||
__lasx_xvilvh_d(p9d_, p15_)));
|
||||
__m256i even = __lasx_xvmulwev_w_hu(a.val, b.val);
|
||||
__m256i odd = __lasx_xvmulwod_w_hu(a.val, b.val);
|
||||
__m256i prod0 = __lasx_xvhaddw_du_wu(even, even);
|
||||
__m256i prod1 = __lasx_xvhaddw_du_wu(odd, odd);
|
||||
return v_uint64x4(__lasx_xvadd_d(prod0, prod1));
|
||||
}
|
||||
inline v_uint64x4 v_dotprod_expand(const v_uint16x16& a, const v_uint16x16& b, const v_uint64x4& c)
|
||||
{ return v_dotprod_expand(a, b) + c; }
|
||||
|
||||
inline v_int64x4 v_dotprod_expand(const v_int16x16& a, const v_int16x16& b)
|
||||
{
|
||||
__m256i prod = __lasx_xvadd_w(__lasx_xvmulwev_w_h(a.val, b.val), __lasx_xvmulwod_w_h(a.val, b.val));
|
||||
__m256i sign = __lasx_xvsrai_w(prod, 31);
|
||||
|
||||
__m256i lo = __lasx_xvilvl_w(sign, prod);
|
||||
__m256i hi = __lasx_xvilvh_w(sign, prod);
|
||||
|
||||
return v_int64x4(__lasx_xvadd_d(__lasx_xvilvl_d(hi, lo), __lasx_xvilvh_d(hi, lo)));
|
||||
__m256i even = __lasx_xvmulwev_w_h(a.val, b.val);
|
||||
__m256i odd = __lasx_xvmulwod_w_h(a.val, b.val);
|
||||
__m256i prod0 = __lasx_xvhaddw_d_w(even, even);
|
||||
__m256i prod1 = __lasx_xvhaddw_d_w(odd, odd);
|
||||
return v_int64x4(__lasx_xvadd_d(prod0, prod1));
|
||||
}
|
||||
|
||||
inline v_int64x4 v_dotprod_expand(const v_int16x16& a, const v_int16x16& b, const v_int64x4& c)
|
||||
{ return v_dotprod_expand(a, b) + c; }
|
||||
|
||||
@@ -2126,20 +1998,11 @@ inline v_int32x8 v_dotprod_expand_fast(const v_int8x32& a, const v_int8x32& b, c
|
||||
// 16 >> 64
|
||||
inline v_uint64x4 v_dotprod_expand_fast(const v_uint16x16& a, const v_uint16x16& b)
|
||||
{
|
||||
__m256i mullo = __lasx_xvmul_h(a.val, b.val);
|
||||
__m256i mulhi = __lasx_xvmuh_hu(a.val, b.val);
|
||||
__m256i mul0 = __lasx_xvilvl_h(mulhi, mullo);
|
||||
__m256i mul1 = __lasx_xvilvh_h(mulhi, mullo);
|
||||
|
||||
__m256i p02 = __lasx_xvbitsel_v(mul0, __lasx_xvreplgr2vr_d(0), _v256_set_w(-1, 0, -1, 0, -1, 0, -1, 0));
|
||||
__m256i p13 = __lasx_xvsrli_d(mul0, 32);
|
||||
__m256i p46 = __lasx_xvbitsel_v(mul1, __lasx_xvreplgr2vr_d(0), _v256_set_w(-1, 0, -1, 0, -1, 0, -1, 0));
|
||||
__m256i p57 = __lasx_xvsrli_d(mul1, 32);
|
||||
|
||||
__m256i p15_ = __lasx_xvadd_d(p02, p13);
|
||||
__m256i p9d_ = __lasx_xvadd_d(p46, p57);
|
||||
|
||||
return v_uint64x4(__lasx_xvadd_d(p15_, p9d_));
|
||||
__m256i even = __lasx_xvmulwev_w_hu(a.val, b.val);
|
||||
__m256i odd = __lasx_xvmulwod_w_hu(a.val, b.val);
|
||||
__m256i prod0 = __lasx_xvhaddw_du_wu(even, even);
|
||||
__m256i prod1 = __lasx_xvhaddw_du_wu(odd, odd);
|
||||
return v_uint64x4(__lasx_xvadd_d(__lasx_xvilvl_d(prod1, prod0), __lasx_xvilvh_d(prod1, prod0)));
|
||||
}
|
||||
inline v_uint64x4 v_dotprod_expand_fast(const v_uint16x16& a, const v_uint16x16& b, const v_uint64x4& c)
|
||||
{ return v_dotprod_expand_fast(a, b) + c; }
|
||||
@@ -2261,12 +2124,7 @@ inline v_int8x32 v_pack(const v_int16x16& a, const v_int16x16& b)
|
||||
{ return v_int8x32(_v256_shuffle_odd_64(_lasx_packs_h(a.val, b.val))); }
|
||||
|
||||
inline v_uint8x32 v_pack(const v_uint16x16& a, const v_uint16x16& b)
|
||||
{
|
||||
__m256i t = __lasx_xvreplgr2vr_h(255);
|
||||
__m256i a1 = __lasx_xvmin_hu(a.val, t);
|
||||
__m256i b1 = __lasx_xvmin_hu(b.val, t);
|
||||
return v_uint8x32(_v256_shuffle_odd_64(_lasx_packus_h(a1, b1)));
|
||||
}
|
||||
{ return v_uint8x32(_v256_shuffle_odd_64(__lasx_xvssrlrni_bu_h(b.val, a.val, 0))); }
|
||||
|
||||
inline v_uint8x32 v_pack_u(const v_int16x16& a, const v_int16x16& b)
|
||||
{
|
||||
@@ -2276,13 +2134,8 @@ inline v_uint8x32 v_pack_u(const v_int16x16& a, const v_int16x16& b)
|
||||
inline void v_pack_store(schar* ptr, const v_int16x16& a)
|
||||
{ v_store_low(ptr, v_pack(a, a)); }
|
||||
|
||||
inline void v_pack_store(uchar* ptr, const v_uint16x16& a)
|
||||
{
|
||||
const __m256i m = __lasx_xvreplgr2vr_h(255);
|
||||
__m256i am = __lasx_xvmin_hu(a.val, m);
|
||||
am = _v256_shuffle_odd_64(_lasx_packus_h(am, am));
|
||||
v_store_low(ptr, v_uint8x32(am));
|
||||
}
|
||||
inline void v_pack_store(uchar *ptr, const v_uint16x16& a)
|
||||
{ v_store_low(ptr, v_pack(a, a)); }
|
||||
|
||||
inline void v_pack_u_store(uchar* ptr, const v_int16x16& a)
|
||||
{ v_store_low(ptr, v_pack_u(a, a)); }
|
||||
@@ -2290,45 +2143,43 @@ inline void v_pack_u_store(uchar* ptr, const v_int16x16& a)
|
||||
template<int n> inline
|
||||
v_uint8x32 v_rshr_pack(const v_uint16x16& a, const v_uint16x16& b)
|
||||
{
|
||||
// we assume that n > 0, and so the shifted 16-bit values can be treated as signed numbers.
|
||||
v_uint16x16 delta = v256_setall_u16((short)(1 << (n-1)));
|
||||
return v_pack_u(v_reinterpret_as_s16((a + delta) >> n),
|
||||
v_reinterpret_as_s16((b + delta) >> n));
|
||||
__m256i res = __lasx_xvssrlrni_bu_h(b.val, a.val, n);
|
||||
return v_uint8x32(_v256_shuffle_odd_64(res));
|
||||
}
|
||||
|
||||
template<int n> inline
|
||||
void v_rshr_pack_store(uchar* ptr, const v_uint16x16& a)
|
||||
{
|
||||
v_uint16x16 delta = v256_setall_u16((short)(1 << (n-1)));
|
||||
v_pack_u_store(ptr, v_reinterpret_as_s16((a + delta) >> n));
|
||||
__m256i res = __lasx_xvssrlrni_bu_h(a.val, a.val, n);
|
||||
__lsx_vst(_v256_extract_low(_v256_shuffle_odd_64(res)), ptr, 0);
|
||||
}
|
||||
|
||||
template<int n> inline
|
||||
v_uint8x32 v_rshr_pack_u(const v_int16x16& a, const v_int16x16& b)
|
||||
{
|
||||
v_int16x16 delta = v256_setall_s16((short)(1 << (n-1)));
|
||||
return v_pack_u((a + delta) >> n, (b + delta) >> n);
|
||||
__m256i res = __lasx_xvssrarni_bu_h(b.val, a.val, n);
|
||||
return v_uint8x32(_v256_shuffle_odd_64(res));
|
||||
}
|
||||
|
||||
template<int n> inline
|
||||
void v_rshr_pack_u_store(uchar* ptr, const v_int16x16& a)
|
||||
{
|
||||
v_int16x16 delta = v256_setall_s16((short)(1 << (n-1)));
|
||||
v_pack_u_store(ptr, (a + delta) >> n);
|
||||
__m256i res = __lasx_xvssrarni_bu_h(a.val, a.val, n);
|
||||
__lsx_vst(_v256_extract_low(_v256_shuffle_odd_64(res)), ptr, 0);
|
||||
}
|
||||
|
||||
template<int n> inline
|
||||
v_int8x32 v_rshr_pack(const v_int16x16& a, const v_int16x16& b)
|
||||
{
|
||||
v_int16x16 delta = v256_setall_s16((short)(1 << (n-1)));
|
||||
return v_pack((a + delta) >> n, (b + delta) >> n);
|
||||
__m256i res = __lasx_xvssrarni_b_h(b.val, a.val, n);
|
||||
return v_int8x32(_v256_shuffle_odd_64(res));
|
||||
}
|
||||
|
||||
template<int n> inline
|
||||
void v_rshr_pack_store(schar* ptr, const v_int16x16& a)
|
||||
{
|
||||
v_int16x16 delta = v256_setall_s16((short)(1 << (n-1)));
|
||||
v_pack_store(ptr, (a + delta) >> n);
|
||||
__m256i res = __lasx_xvssrarni_b_h(a.val, a.val, n);
|
||||
__lsx_vst(_v256_extract_low(_v256_shuffle_odd_64(res)), ptr, 0);
|
||||
}
|
||||
|
||||
// 32
|
||||
@@ -2346,67 +2197,51 @@ inline void v_pack_store(short* ptr, const v_int32x8& a)
|
||||
|
||||
inline void v_pack_store(ushort* ptr, const v_uint32x8& a)
|
||||
{
|
||||
const __m256i m = __lasx_xvreplgr2vr_w(65535);
|
||||
__m256i am = __lasx_xvmin_wu(a.val, m);
|
||||
am = _v256_shuffle_odd_64(_lasx_packus_w(am, am));
|
||||
v_store_low(ptr, v_uint16x16(am));
|
||||
__m256i res = __lasx_xvssrlrni_hu_w(a.val, a.val, 0);
|
||||
__lsx_vst(_v256_extract_low(_v256_shuffle_odd_64(res)), ptr, 0);
|
||||
}
|
||||
|
||||
inline void v_pack_u_store(ushort* ptr, const v_int32x8& a)
|
||||
{ v_store_low(ptr, v_pack_u(a, a)); }
|
||||
|
||||
|
||||
template<int n> inline
|
||||
v_uint16x16 v_rshr_pack(const v_uint32x8& a, const v_uint32x8& b)
|
||||
{
|
||||
// we assume that n > 0, and so the shifted 32-bit values can be treated as signed numbers.
|
||||
v_uint32x8 delta = v256_setall_u32(1 << (n-1));
|
||||
return v_pack_u(v_reinterpret_as_s32((a + delta) >> n),
|
||||
v_reinterpret_as_s32((b + delta) >> n));
|
||||
}
|
||||
{ return v_uint16x16(_v256_shuffle_odd_64(__lasx_xvssrlrni_hu_w(b.val, a.val, n))); }
|
||||
|
||||
template<int n> inline
|
||||
void v_rshr_pack_store(ushort* ptr, const v_uint32x8& a)
|
||||
{
|
||||
v_uint32x8 delta = v256_setall_u32(1 << (n-1));
|
||||
v_pack_u_store(ptr, v_reinterpret_as_s32((a + delta) >> n));
|
||||
__m256i res = __lasx_xvssrlrni_hu_w(a.val, a.val, n);
|
||||
__lsx_vst(_v256_extract_low(_v256_shuffle_odd_64(res)), ptr, 0);
|
||||
}
|
||||
|
||||
template<int n> inline
|
||||
v_uint16x16 v_rshr_pack_u(const v_int32x8& a, const v_int32x8& b)
|
||||
{
|
||||
v_int32x8 delta = v256_setall_s32(1 << (n-1));
|
||||
return v_pack_u((a + delta) >> n, (b + delta) >> n);
|
||||
}
|
||||
{ return v_uint16x16(_v256_shuffle_odd_64(__lasx_xvssrarni_hu_w(b.val, a.val, n))); }
|
||||
|
||||
template<int n> inline
|
||||
void v_rshr_pack_u_store(ushort* ptr, const v_int32x8& a)
|
||||
{
|
||||
v_int32x8 delta = v256_setall_s32(1 << (n-1));
|
||||
v_pack_u_store(ptr, (a + delta) >> n);
|
||||
__m256i res = __lasx_xvssrarni_hu_w(a.val, a.val, n);
|
||||
__lsx_vst(_v256_extract_low(_v256_shuffle_odd_64(res)), ptr, 0);
|
||||
}
|
||||
|
||||
template<int n> inline
|
||||
v_int16x16 v_rshr_pack(const v_int32x8& a, const v_int32x8& b)
|
||||
{
|
||||
v_int32x8 delta = v256_setall_s32(1 << (n-1));
|
||||
return v_pack((a + delta) >> n, (b + delta) >> n);
|
||||
}
|
||||
{ return v_int16x16(_v256_shuffle_odd_64(__lasx_xvssrarni_h_w(b.val, a.val, n))); }
|
||||
|
||||
template<int n> inline
|
||||
void v_rshr_pack_store(short* ptr, const v_int32x8& a)
|
||||
{
|
||||
v_int32x8 delta = v256_setall_s32(1 << (n-1));
|
||||
v_pack_store(ptr, (a + delta) >> n);
|
||||
__m256i res = __lasx_xvssrarni_h_w(a.val, a.val, n);
|
||||
__lsx_vst(_v256_extract_low(_v256_shuffle_odd_64(res)), ptr, 0);
|
||||
}
|
||||
|
||||
// 64
|
||||
// Non-saturating pack
|
||||
inline v_uint32x8 v_pack(const v_uint64x4& a, const v_uint64x4& b)
|
||||
{
|
||||
__m256i a0 = __lasx_xvshuf4i_w(a.val, 0x08);
|
||||
__m256i b0 = __lasx_xvshuf4i_w(b.val, 0x08);
|
||||
__m256i ab = __lasx_xvilvl_d(b0, a0);
|
||||
__m256i ab = __lasx_xvpickev_w(b.val, a.val);
|
||||
return v_uint32x8(_v256_shuffle_odd_64(ab));
|
||||
}
|
||||
|
||||
@@ -2424,31 +2259,19 @@ inline void v_pack_store(int* ptr, const v_int64x4& b)
|
||||
|
||||
template<int n> inline
|
||||
v_uint32x8 v_rshr_pack(const v_uint64x4& a, const v_uint64x4& b)
|
||||
{
|
||||
v_uint64x4 delta = v256_setall_u64((uint64)1 << (n-1));
|
||||
return v_pack((a + delta) >> n, (b + delta) >> n);
|
||||
}
|
||||
{ return v_uint32x8(_v256_shuffle_odd_64(__lasx_xvsrlrni_w_d(b.val, a.val, n))); }
|
||||
|
||||
template<int n> inline
|
||||
void v_rshr_pack_store(unsigned* ptr, const v_uint64x4& a)
|
||||
{
|
||||
v_uint64x4 delta = v256_setall_u64((uint64)1 << (n-1));
|
||||
v_pack_store(ptr, (a + delta) >> n);
|
||||
}
|
||||
{ __lsx_vst(_v256_shuffle_odd_64(__lasx_xvsrlrni_w_d(a.val, a.val, n)), ptr, 0); }
|
||||
|
||||
template<int n> inline
|
||||
v_int32x8 v_rshr_pack(const v_int64x4& a, const v_int64x4& b)
|
||||
{
|
||||
v_int64x4 delta = v256_setall_s64((int64)1 << (n-1));
|
||||
return v_pack((a + delta) >> n, (b + delta) >> n);
|
||||
}
|
||||
{ return v_int32x8(_v256_shuffle_odd_64(__lasx_xvsrarni_w_d(b.val, a.val, n))); }
|
||||
|
||||
template<int n> inline
|
||||
void v_rshr_pack_store(int* ptr, const v_int64x4& a)
|
||||
{
|
||||
v_int64x4 delta = v256_setall_s64((int64)1 << (n-1));
|
||||
v_pack_store(ptr, (a + delta) >> n);
|
||||
}
|
||||
{ __lsx_vst(_v256_shuffle_odd_64(__lasx_xvsrarni_w_d(a.val, a.val, n)), ptr, 0); }
|
||||
|
||||
// pack boolean
|
||||
inline v_uint8x32 v_pack_b(const v_uint16x16& a, const v_uint16x16& b)
|
||||
@@ -2583,63 +2406,48 @@ template<int i>
|
||||
inline v_float32x8 v_broadcast_element(const v_float32x8 &a)
|
||||
{ return v_reinterpret_as_f32(v_broadcast_element<i>(v_reinterpret_as_u32(a))); }
|
||||
|
||||
|
||||
///////////////////// load deinterleave /////////////////////////////
|
||||
|
||||
inline void v_load_deinterleave( const uchar* ptr, v_uint8x32& a, v_uint8x32& b )
|
||||
inline void v_load_deinterleave(const uchar* ptr, v_uint8x32& a, v_uint8x32& b)
|
||||
{
|
||||
__m256i ab0 = __lasx_xvld(ptr, 0);
|
||||
__m256i ab1 = __lasx_xvld(ptr + 32, 0);
|
||||
__m256i t0 = __lasx_xvld(ptr, 0);
|
||||
__m256i t1 = __lasx_xvld(ptr, 32);
|
||||
|
||||
const __m256i sh = _v256_setr_b(0, 2, 4, 6, 8, 10, 12, 14, 1, 3, 5, 7, 9, 11, 13, 15,
|
||||
0, 2, 4, 6, 8, 10, 12, 14, 1, 3, 5, 7, 9, 11, 13, 15);
|
||||
__m256i p0 = __lasx_xvshuf_b(ab0, ab0, sh);
|
||||
__m256i p1 = __lasx_xvshuf_b(ab1, ab1, sh);
|
||||
__m256i pl = __lasx_xvpermi_q(p0, p1, 0x02);
|
||||
__m256i ph = __lasx_xvpermi_q(p0, p1, 0x13);
|
||||
__m256i a0 = __lasx_xvilvl_d(ph, pl);
|
||||
__m256i b0 = __lasx_xvilvh_d(ph, pl);
|
||||
a = v_uint8x32(a0);
|
||||
b = v_uint8x32(b0);
|
||||
__m256i p0 = __lasx_xvpickev_b(t1, t0);
|
||||
__m256i p1 = __lasx_xvpickod_b(t1, t0);
|
||||
|
||||
a.val = __lasx_xvpermi_d(p0, 0xd8);
|
||||
b.val = __lasx_xvpermi_d(p1, 0xd8);
|
||||
}
|
||||
|
||||
inline void v_load_deinterleave( const ushort* ptr, v_uint16x16& a, v_uint16x16& b )
|
||||
{
|
||||
__m256i ab0 = __lasx_xvld(ptr, 0);
|
||||
__m256i ab1 = __lasx_xvld(ptr + 16, 0);
|
||||
__m256i t0 = __lasx_xvld(ptr, 0);
|
||||
__m256i t1 = __lasx_xvld(ptr, 32);
|
||||
|
||||
const __m256i sh = _v256_setr_b(0, 1, 4, 5, 8, 9, 12, 13, 2, 3, 6, 7, 10, 11, 14, 15,
|
||||
0, 1, 4, 5, 8, 9, 12, 13, 2, 3, 6, 7, 10, 11, 14, 15);
|
||||
__m256i p0 = __lasx_xvshuf_b(ab0, ab0, sh);
|
||||
__m256i p1 = __lasx_xvshuf_b(ab1, ab1, sh);
|
||||
__m256i pl = __lasx_xvpermi_q(p0, p1, 0x02);
|
||||
__m256i ph = __lasx_xvpermi_q(p0, p1, 0x13);
|
||||
__m256i a0 = __lasx_xvilvl_d(ph, pl);
|
||||
__m256i b0 = __lasx_xvilvh_d(ph, pl);
|
||||
a = v_uint16x16(a0);
|
||||
b = v_uint16x16(b0);
|
||||
__m256i p0 = __lasx_xvpickev_h(t1, t0);
|
||||
__m256i p1 = __lasx_xvpickod_h(t1, t0);
|
||||
|
||||
a.val = __lasx_xvpermi_d(p0, 0xd8);
|
||||
b.val = __lasx_xvpermi_d(p1, 0xd8);
|
||||
}
|
||||
|
||||
inline void v_load_deinterleave( const unsigned* ptr, v_uint32x8& a, v_uint32x8& b )
|
||||
{
|
||||
__m256i ab0 = __lasx_xvld(ptr, 0);
|
||||
__m256i ab1 = __lasx_xvld(ptr + 8, 0);
|
||||
__m256i t0 = __lasx_xvld(ptr, 0);
|
||||
__m256i t1 = __lasx_xvld(ptr, 32);
|
||||
|
||||
//const int sh = 0+2*4+1*16+3*64;
|
||||
__m256i p0 = __lasx_xvshuf4i_w(ab0, 0xD8);
|
||||
__m256i p1 = __lasx_xvshuf4i_w(ab1, 0xD8);
|
||||
__m256i pl = __lasx_xvpermi_q(p0, p1, 0x02);
|
||||
__m256i ph = __lasx_xvpermi_q(p0, p1, 0x13);
|
||||
__m256i a0 = __lasx_xvilvl_d(ph, pl);
|
||||
__m256i b0 = __lasx_xvilvh_d(ph, pl);
|
||||
a = v_uint32x8(a0);
|
||||
b = v_uint32x8(b0);
|
||||
__m256i p0 = __lasx_xvpickev_w(t1, t0);
|
||||
__m256i p1 = __lasx_xvpickod_w(t1, t0);
|
||||
|
||||
a.val = __lasx_xvpermi_d(p0, 0xd8);
|
||||
b.val = __lasx_xvpermi_d(p1, 0xd8);
|
||||
}
|
||||
|
||||
inline void v_load_deinterleave( const uint64* ptr, v_uint64x4& a, v_uint64x4& b )
|
||||
{
|
||||
__m256i ab0 = __lasx_xvld(ptr, 0);
|
||||
__m256i ab1 = __lasx_xvld(ptr + 4, 0);
|
||||
__m256i ab1 = __lasx_xvld(ptr, 32);
|
||||
|
||||
__m256i pl = __lasx_xvpermi_q(ab0, ab1, 0x02);
|
||||
__m256i ph = __lasx_xvpermi_q(ab0, ab1, 0x13);
|
||||
@@ -2652,8 +2460,8 @@ inline void v_load_deinterleave( const uint64* ptr, v_uint64x4& a, v_uint64x4& b
|
||||
inline void v_load_deinterleave( const uchar* ptr, v_uint8x32& a, v_uint8x32& b, v_uint8x32& c )
|
||||
{
|
||||
__m256i bgr0 = __lasx_xvld(ptr, 0);
|
||||
__m256i bgr1 = __lasx_xvld(ptr + 32, 0);
|
||||
__m256i bgr2 = __lasx_xvld(ptr + 64, 0);
|
||||
__m256i bgr1 = __lasx_xvld(ptr, 32);
|
||||
__m256i bgr2 = __lasx_xvld(ptr, 64);
|
||||
|
||||
__m256i s02_low = __lasx_xvpermi_q(bgr0, bgr2, 0x02);
|
||||
__m256i s02_high = __lasx_xvpermi_q(bgr0, bgr2, 0x13);
|
||||
@@ -2686,8 +2494,8 @@ inline void v_load_deinterleave( const uchar* ptr, v_uint8x32& a, v_uint8x32& b,
|
||||
inline void v_load_deinterleave( const ushort* ptr, v_uint16x16& a, v_uint16x16& b, v_uint16x16& c )
|
||||
{
|
||||
__m256i bgr0 = __lasx_xvld(ptr, 0);
|
||||
__m256i bgr1 = __lasx_xvld(ptr + 16, 0);
|
||||
__m256i bgr2 = __lasx_xvld(ptr + 32, 0);
|
||||
__m256i bgr1 = __lasx_xvld(ptr, 32);
|
||||
__m256i bgr2 = __lasx_xvld(ptr, 64);
|
||||
|
||||
__m256i s02_low = __lasx_xvpermi_q(bgr0, bgr2, 0x02);
|
||||
__m256i s02_high = __lasx_xvpermi_q(bgr0, bgr2, 0x13);
|
||||
@@ -2717,8 +2525,8 @@ inline void v_load_deinterleave( const ushort* ptr, v_uint16x16& a, v_uint16x16&
|
||||
inline void v_load_deinterleave( const unsigned* ptr, v_uint32x8& a, v_uint32x8& b, v_uint32x8& c )
|
||||
{
|
||||
__m256i bgr0 = __lasx_xvld(ptr, 0);
|
||||
__m256i bgr1 = __lasx_xvld(ptr + 8, 0);
|
||||
__m256i bgr2 = __lasx_xvld(ptr + 16, 0);
|
||||
__m256i bgr1 = __lasx_xvld(ptr, 32);
|
||||
__m256i bgr2 = __lasx_xvld(ptr, 64);
|
||||
|
||||
__m256i s02_low = __lasx_xvpermi_q(bgr0, bgr2, 0x02);
|
||||
__m256i s02_high = __lasx_xvpermi_q(bgr0, bgr2, 0x13);
|
||||
@@ -2741,8 +2549,8 @@ inline void v_load_deinterleave( const unsigned* ptr, v_uint32x8& a, v_uint32x8&
|
||||
inline void v_load_deinterleave( const uint64* ptr, v_uint64x4& a, v_uint64x4& b, v_uint64x4& c )
|
||||
{
|
||||
__m256i bgr0 = __lasx_xvld(ptr, 0);
|
||||
__m256i bgr1 = __lasx_xvld(ptr + 4, 0);
|
||||
__m256i bgr2 = __lasx_xvld(ptr + 8, 0);
|
||||
__m256i bgr1 = __lasx_xvld(ptr, 32);
|
||||
__m256i bgr2 = __lasx_xvld(ptr, 64);
|
||||
|
||||
__m256i s01 = __lasx_xvpermi_q(bgr0, bgr1, 0x12); // get bgr0 low 128 and bgr1 high 128
|
||||
__m256i s12 = __lasx_xvpermi_q(bgr1, bgr2, 0x12);
|
||||
@@ -2756,81 +2564,60 @@ inline void v_load_deinterleave( const uint64* ptr, v_uint64x4& a, v_uint64x4& b
|
||||
c = v_uint64x4(r0);
|
||||
}
|
||||
|
||||
inline void v_load_deinterleave( const uchar* ptr, v_uint8x32& a, v_uint8x32& b, v_uint8x32& c, v_uint8x32& d )
|
||||
inline void v_load_deinterleave(const uchar* ptr, v_uint8x32& a, v_uint8x32& b, v_uint8x32& c, v_uint8x32& d)
|
||||
{
|
||||
__m256i bgr0 = __lasx_xvld(ptr, 0);
|
||||
__m256i bgr1 = __lasx_xvld(ptr + 32, 0);
|
||||
__m256i bgr2 = __lasx_xvld(ptr + 64, 0);
|
||||
__m256i bgr3 = __lasx_xvld(ptr + 96, 0);
|
||||
const __m256i sh = _v256_setr_b(0, 4, 8, 12, 1, 5, 9, 13, 2, 6, 10, 14, 3, 7, 11, 15,
|
||||
0, 4, 8, 12, 1, 5, 9, 13, 2, 6, 10, 14, 3, 7, 11, 15);
|
||||
__m256i t0 = __lasx_xvld(ptr, 0);
|
||||
__m256i t1 = __lasx_xvld(ptr, 32);
|
||||
__m256i t2 = __lasx_xvld(ptr, 64);
|
||||
__m256i t3 = __lasx_xvld(ptr, 96);
|
||||
|
||||
__m256i p0 = __lasx_xvshuf_b(bgr0, bgr0, sh);
|
||||
__m256i p1 = __lasx_xvshuf_b(bgr1, bgr1, sh);
|
||||
__m256i p2 = __lasx_xvshuf_b(bgr2, bgr2, sh);
|
||||
__m256i p3 = __lasx_xvshuf_b(bgr3, bgr3, sh);
|
||||
const __m256i sh = _v256_setr_w(0, 4, 1, 5, 2, 6, 3, 7);
|
||||
__m256i ac_lo = __lasx_xvpickev_b(t1, t0);
|
||||
__m256i bd_lo = __lasx_xvpickod_b(t1, t0);
|
||||
__m256i ac_hi = __lasx_xvpickev_b(t3, t2);
|
||||
__m256i bd_hi = __lasx_xvpickod_b(t3, t2);
|
||||
|
||||
__m256i p01l = __lasx_xvilvl_w(p1, p0);
|
||||
__m256i p01h = __lasx_xvilvh_w(p1, p0);
|
||||
__m256i p23l = __lasx_xvilvl_w(p3, p2);
|
||||
__m256i p23h = __lasx_xvilvh_w(p3, p2);
|
||||
__m256i a_pre = __lasx_xvpickev_b(ac_hi, ac_lo);
|
||||
__m256i c_pre = __lasx_xvpickod_b(ac_hi, ac_lo);
|
||||
__m256i b_pre = __lasx_xvpickev_b(bd_hi, bd_lo);
|
||||
__m256i d_pre = __lasx_xvpickod_b(bd_hi, bd_lo);
|
||||
|
||||
__m256i pll = __lasx_xvpermi_q(p01l, p23l, 0x02);
|
||||
__m256i plh = __lasx_xvpermi_q(p01l, p23l, 0x13);
|
||||
__m256i phl = __lasx_xvpermi_q(p01h, p23h, 0x02);
|
||||
__m256i phh = __lasx_xvpermi_q(p01h, p23h, 0x13);
|
||||
|
||||
__m256i b0 = __lasx_xvilvl_w(plh, pll);
|
||||
__m256i g0 = __lasx_xvilvh_w(plh, pll);
|
||||
__m256i r0 = __lasx_xvilvl_w(phh, phl);
|
||||
__m256i a0 = __lasx_xvilvh_w(phh, phl);
|
||||
|
||||
a = v_uint8x32(b0);
|
||||
b = v_uint8x32(g0);
|
||||
c = v_uint8x32(r0);
|
||||
d = v_uint8x32(a0);
|
||||
a.val = __lasx_xvperm_w(a_pre, sh);
|
||||
b.val = __lasx_xvperm_w(b_pre, sh);
|
||||
c.val = __lasx_xvperm_w(c_pre, sh);
|
||||
d.val = __lasx_xvperm_w(d_pre, sh);
|
||||
}
|
||||
|
||||
inline void v_load_deinterleave( const ushort* ptr, v_uint16x16& a, v_uint16x16& b, v_uint16x16& c, v_uint16x16& d )
|
||||
inline void v_load_deinterleave(const ushort* ptr, v_uint16x16& a, v_uint16x16& b, v_uint16x16& c, v_uint16x16& d)
|
||||
{
|
||||
__m256i bgr0 = __lasx_xvld(ptr, 0);
|
||||
__m256i bgr1 = __lasx_xvld(ptr + 16, 0);
|
||||
__m256i bgr2 = __lasx_xvld(ptr + 32, 0);
|
||||
__m256i bgr3 = __lasx_xvld(ptr + 48, 0);
|
||||
const __m256i sh = _v256_setr_b(0, 1, 8, 9, 2, 3, 10, 11, 4, 5, 12, 13, 6, 7, 14, 15,
|
||||
0, 1, 8, 9, 2, 3, 10, 11, 4, 5, 12, 13, 6, 7, 14, 15);
|
||||
__m256i p0 = __lasx_xvshuf_b(bgr0, bgr0, sh);
|
||||
__m256i p1 = __lasx_xvshuf_b(bgr1, bgr1, sh);
|
||||
__m256i p2 = __lasx_xvshuf_b(bgr2, bgr2, sh);
|
||||
__m256i p3 = __lasx_xvshuf_b(bgr3, bgr3, sh);
|
||||
__m256i t0 = __lasx_xvld(ptr, 0);
|
||||
__m256i t1 = __lasx_xvld(ptr, 32);
|
||||
__m256i t2 = __lasx_xvld(ptr, 64);
|
||||
__m256i t3 = __lasx_xvld(ptr, 96);
|
||||
|
||||
__m256i p01l = __lasx_xvilvl_w(p1, p0);
|
||||
__m256i p01h = __lasx_xvilvh_w(p1, p0);
|
||||
__m256i p23l = __lasx_xvilvl_w(p3, p2);
|
||||
__m256i p23h = __lasx_xvilvh_w(p3, p2);
|
||||
const __m256i sh = _v256_setr_w(0, 4, 1, 5, 2, 6, 3, 7);
|
||||
__m256i ac_lo = __lasx_xvpickev_h(t1, t0);
|
||||
__m256i bd_lo = __lasx_xvpickod_h(t1, t0);
|
||||
__m256i ac_hi = __lasx_xvpickev_h(t3, t2);
|
||||
__m256i bd_hi = __lasx_xvpickod_h(t3, t2);
|
||||
|
||||
__m256i pll = __lasx_xvpermi_q(p01l, p23l, 0x02);
|
||||
__m256i plh = __lasx_xvpermi_q(p01l, p23l, 0x13);
|
||||
__m256i phl = __lasx_xvpermi_q(p01h, p23h, 0x02);
|
||||
__m256i phh = __lasx_xvpermi_q(p01h, p23h, 0x13);
|
||||
__m256i a_pre = __lasx_xvpickev_h(ac_hi, ac_lo);
|
||||
__m256i c_pre = __lasx_xvpickod_h(ac_hi, ac_lo);
|
||||
__m256i b_pre = __lasx_xvpickev_h(bd_hi, bd_lo);
|
||||
__m256i d_pre = __lasx_xvpickod_h(bd_hi, bd_lo);
|
||||
|
||||
__m256i b0 = __lasx_xvilvl_w(plh, pll);
|
||||
__m256i g0 = __lasx_xvilvh_w(plh, pll);
|
||||
__m256i r0 = __lasx_xvilvl_w(phh, phl);
|
||||
__m256i a0 = __lasx_xvilvh_w(phh, phl);
|
||||
|
||||
a = v_uint16x16(b0);
|
||||
b = v_uint16x16(g0);
|
||||
c = v_uint16x16(r0);
|
||||
d = v_uint16x16(a0);
|
||||
a.val = __lasx_xvperm_w(a_pre, sh);
|
||||
b.val = __lasx_xvperm_w(b_pre, sh);
|
||||
c.val = __lasx_xvperm_w(c_pre, sh);
|
||||
d.val = __lasx_xvperm_w(d_pre, sh);
|
||||
}
|
||||
|
||||
inline void v_load_deinterleave( const unsigned* ptr, v_uint32x8& a, v_uint32x8& b, v_uint32x8& c, v_uint32x8& d )
|
||||
{
|
||||
__m256i p0 = __lasx_xvld(ptr, 0);
|
||||
__m256i p1 = __lasx_xvld(ptr + 8, 0);
|
||||
__m256i p2 = __lasx_xvld(ptr + 16, 0);
|
||||
__m256i p3 = __lasx_xvld(ptr + 24, 0);
|
||||
__m256i p1 = __lasx_xvld(ptr, 32);
|
||||
__m256i p2 = __lasx_xvld(ptr, 64);
|
||||
__m256i p3 = __lasx_xvld(ptr, 96);
|
||||
|
||||
__m256i p01l = __lasx_xvilvl_w(p1, p0);
|
||||
__m256i p01h = __lasx_xvilvh_w(p1, p0);
|
||||
@@ -2856,9 +2643,9 @@ inline void v_load_deinterleave( const unsigned* ptr, v_uint32x8& a, v_uint32x8&
|
||||
inline void v_load_deinterleave( const uint64* ptr, v_uint64x4& a, v_uint64x4& b, v_uint64x4& c, v_uint64x4& d )
|
||||
{
|
||||
__m256i bgra0 = __lasx_xvld(ptr, 0);
|
||||
__m256i bgra1 = __lasx_xvld(ptr + 4, 0);
|
||||
__m256i bgra2 = __lasx_xvld(ptr + 8, 0);
|
||||
__m256i bgra3 = __lasx_xvld(ptr + 12, 0);
|
||||
__m256i bgra1 = __lasx_xvld(ptr, 32);
|
||||
__m256i bgra2 = __lasx_xvld(ptr, 64);
|
||||
__m256i bgra3 = __lasx_xvld(ptr, 96);
|
||||
|
||||
__m256i l02 = __lasx_xvpermi_q(bgra0, bgra2, 0x02);
|
||||
__m256i h02 = __lasx_xvpermi_q(bgra0, bgra2, 0x13);
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -131,13 +131,22 @@ OPENCV_HAL_IMPL_NEON_UTILS_SUFFIX_I64(int64x2, int64x1, s64)
|
||||
OPENCV_HAL_IMPL_NEON_UTILS_SUFFIX_F64(float64x2, float64x1,f64)
|
||||
#endif
|
||||
|
||||
//////////// Compatibility layer ////////////
|
||||
template<typename T> struct VTraits {
|
||||
static inline int vlanes() { return T::nlanes; }
|
||||
enum { max_nlanes = T::nlanes, nlanes = T::nlanes };
|
||||
using lane_type = typename T::lane_type;
|
||||
};
|
||||
|
||||
template<typename T>
|
||||
inline typename VTraits<T>::lane_type v_get0(const T& v) \
|
||||
{ \
|
||||
return v.get0(); \
|
||||
}
|
||||
//////////// Types ////////////
|
||||
|
||||
struct v_uint8x16
|
||||
{
|
||||
typedef uchar lane_type;
|
||||
enum { nlanes = 16 };
|
||||
|
||||
v_uint8x16() {}
|
||||
explicit v_uint8x16(uint8x16_t v) : val(v) {}
|
||||
v_uint8x16(uchar v0, uchar v1, uchar v2, uchar v3, uchar v4, uchar v5, uchar v6, uchar v7,
|
||||
@@ -146,19 +155,22 @@ struct v_uint8x16
|
||||
uchar v[] = {v0, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15};
|
||||
val = vld1q_u8(v);
|
||||
}
|
||||
uint8x16_t val;
|
||||
|
||||
private:
|
||||
friend struct VTraits<v_uint8x16>;
|
||||
enum { nlanes = 16 };
|
||||
typedef uchar lane_type;
|
||||
|
||||
friend typename VTraits<v_uint8x16>::lane_type v_get0<v_uint8x16>(const v_uint8x16& v);
|
||||
uchar get0() const
|
||||
{
|
||||
return vgetq_lane_u8(val, 0);
|
||||
}
|
||||
|
||||
uint8x16_t val;
|
||||
};
|
||||
|
||||
struct v_int8x16
|
||||
{
|
||||
typedef schar lane_type;
|
||||
enum { nlanes = 16 };
|
||||
|
||||
v_int8x16() {}
|
||||
explicit v_int8x16(int8x16_t v) : val(v) {}
|
||||
v_int8x16(schar v0, schar v1, schar v2, schar v3, schar v4, schar v5, schar v6, schar v7,
|
||||
@@ -167,19 +179,22 @@ struct v_int8x16
|
||||
schar v[] = {v0, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15};
|
||||
val = vld1q_s8(v);
|
||||
}
|
||||
int8x16_t val;
|
||||
|
||||
private:
|
||||
friend struct VTraits<v_int8x16>;
|
||||
enum { nlanes = 16 };
|
||||
typedef schar lane_type;
|
||||
|
||||
friend typename VTraits<v_int8x16>::lane_type v_get0<v_int8x16>(const v_int8x16& v);
|
||||
schar get0() const
|
||||
{
|
||||
return vgetq_lane_s8(val, 0);
|
||||
}
|
||||
|
||||
int8x16_t val;
|
||||
};
|
||||
|
||||
struct v_uint16x8
|
||||
{
|
||||
typedef ushort lane_type;
|
||||
enum { nlanes = 8 };
|
||||
|
||||
v_uint16x8() {}
|
||||
explicit v_uint16x8(uint16x8_t v) : val(v) {}
|
||||
v_uint16x8(ushort v0, ushort v1, ushort v2, ushort v3, ushort v4, ushort v5, ushort v6, ushort v7)
|
||||
@@ -187,19 +202,22 @@ struct v_uint16x8
|
||||
ushort v[] = {v0, v1, v2, v3, v4, v5, v6, v7};
|
||||
val = vld1q_u16(v);
|
||||
}
|
||||
uint16x8_t val;
|
||||
|
||||
private:
|
||||
friend struct VTraits<v_uint16x8>;
|
||||
enum { nlanes = 8 };
|
||||
typedef ushort lane_type;
|
||||
|
||||
friend typename VTraits<v_uint16x8>::lane_type v_get0<v_uint16x8>(const v_uint16x8& v);
|
||||
ushort get0() const
|
||||
{
|
||||
return vgetq_lane_u16(val, 0);
|
||||
}
|
||||
|
||||
uint16x8_t val;
|
||||
};
|
||||
|
||||
struct v_int16x8
|
||||
{
|
||||
typedef short lane_type;
|
||||
enum { nlanes = 8 };
|
||||
|
||||
v_int16x8() {}
|
||||
explicit v_int16x8(int16x8_t v) : val(v) {}
|
||||
v_int16x8(short v0, short v1, short v2, short v3, short v4, short v5, short v6, short v7)
|
||||
@@ -207,19 +225,22 @@ struct v_int16x8
|
||||
short v[] = {v0, v1, v2, v3, v4, v5, v6, v7};
|
||||
val = vld1q_s16(v);
|
||||
}
|
||||
int16x8_t val;
|
||||
|
||||
private:
|
||||
friend struct VTraits<v_int16x8>;
|
||||
enum { nlanes = 8 };
|
||||
typedef short lane_type;
|
||||
|
||||
friend typename VTraits<v_int16x8>::lane_type v_get0<v_int16x8>(const v_int16x8& v);
|
||||
short get0() const
|
||||
{
|
||||
return vgetq_lane_s16(val, 0);
|
||||
}
|
||||
|
||||
int16x8_t val;
|
||||
};
|
||||
|
||||
struct v_uint32x4
|
||||
{
|
||||
typedef unsigned lane_type;
|
||||
enum { nlanes = 4 };
|
||||
|
||||
v_uint32x4() {}
|
||||
explicit v_uint32x4(uint32x4_t v) : val(v) {}
|
||||
v_uint32x4(unsigned v0, unsigned v1, unsigned v2, unsigned v3)
|
||||
@@ -227,19 +248,22 @@ struct v_uint32x4
|
||||
unsigned v[] = {v0, v1, v2, v3};
|
||||
val = vld1q_u32(v);
|
||||
}
|
||||
uint32x4_t val;
|
||||
|
||||
private:
|
||||
friend struct VTraits<v_uint32x4>;
|
||||
enum { nlanes = 4 };
|
||||
typedef unsigned lane_type;
|
||||
|
||||
friend typename VTraits<v_uint32x4>::lane_type v_get0<v_uint32x4>(const v_uint32x4& v);
|
||||
unsigned get0() const
|
||||
{
|
||||
return vgetq_lane_u32(val, 0);
|
||||
}
|
||||
|
||||
uint32x4_t val;
|
||||
};
|
||||
|
||||
struct v_int32x4
|
||||
{
|
||||
typedef int lane_type;
|
||||
enum { nlanes = 4 };
|
||||
|
||||
v_int32x4() {}
|
||||
explicit v_int32x4(int32x4_t v) : val(v) {}
|
||||
v_int32x4(int v0, int v1, int v2, int v3)
|
||||
@@ -247,18 +271,22 @@ struct v_int32x4
|
||||
int v[] = {v0, v1, v2, v3};
|
||||
val = vld1q_s32(v);
|
||||
}
|
||||
int32x4_t val;
|
||||
|
||||
private:
|
||||
friend struct VTraits<v_int32x4>;
|
||||
enum { nlanes = 4 };
|
||||
typedef int lane_type;
|
||||
|
||||
friend typename VTraits<v_int32x4>::lane_type v_get0<v_int32x4>(const v_int32x4& v);
|
||||
int get0() const
|
||||
{
|
||||
return vgetq_lane_s32(val, 0);
|
||||
}
|
||||
int32x4_t val;
|
||||
};
|
||||
|
||||
struct v_float32x4
|
||||
{
|
||||
typedef float lane_type;
|
||||
enum { nlanes = 4 };
|
||||
|
||||
v_float32x4() {}
|
||||
explicit v_float32x4(float32x4_t v) : val(v) {}
|
||||
v_float32x4(float v0, float v1, float v2, float v3)
|
||||
@@ -266,18 +294,22 @@ struct v_float32x4
|
||||
float v[] = {v0, v1, v2, v3};
|
||||
val = vld1q_f32(v);
|
||||
}
|
||||
float32x4_t val;
|
||||
|
||||
private:
|
||||
friend struct VTraits<v_float32x4>;
|
||||
enum { nlanes = 4 };
|
||||
typedef float lane_type;
|
||||
|
||||
friend typename VTraits<v_float32x4>::lane_type v_get0<v_float32x4>(const v_float32x4& v);
|
||||
float get0() const
|
||||
{
|
||||
return vgetq_lane_f32(val, 0);
|
||||
}
|
||||
float32x4_t val;
|
||||
};
|
||||
|
||||
struct v_uint64x2
|
||||
{
|
||||
typedef uint64 lane_type;
|
||||
enum { nlanes = 2 };
|
||||
|
||||
v_uint64x2() {}
|
||||
explicit v_uint64x2(uint64x2_t v) : val(v) {}
|
||||
v_uint64x2(uint64 v0, uint64 v1)
|
||||
@@ -285,18 +317,21 @@ struct v_uint64x2
|
||||
uint64 v[] = {v0, v1};
|
||||
val = vld1q_u64(v);
|
||||
}
|
||||
uint64x2_t val;
|
||||
private:
|
||||
friend struct VTraits<v_uint64x2>;
|
||||
enum { nlanes = 2 };
|
||||
typedef uint64 lane_type;
|
||||
|
||||
friend typename VTraits<v_uint64x2>::lane_type v_get0<v_uint64x2>(const v_uint64x2& v);
|
||||
uint64 get0() const
|
||||
{
|
||||
return vgetq_lane_u64(val, 0);
|
||||
}
|
||||
uint64x2_t val;
|
||||
};
|
||||
|
||||
struct v_int64x2
|
||||
{
|
||||
typedef int64 lane_type;
|
||||
enum { nlanes = 2 };
|
||||
|
||||
v_int64x2() {}
|
||||
explicit v_int64x2(int64x2_t v) : val(v) {}
|
||||
v_int64x2(int64 v0, int64 v1)
|
||||
@@ -304,19 +339,23 @@ struct v_int64x2
|
||||
int64 v[] = {v0, v1};
|
||||
val = vld1q_s64(v);
|
||||
}
|
||||
int64x2_t val;
|
||||
|
||||
private:
|
||||
friend struct VTraits<v_int64x2>;
|
||||
enum { nlanes = 2 };
|
||||
typedef int64 lane_type;
|
||||
|
||||
friend typename VTraits<v_int64x2>::lane_type v_get0<v_int64x2>(const v_int64x2& v);
|
||||
int64 get0() const
|
||||
{
|
||||
return vgetq_lane_s64(val, 0);
|
||||
}
|
||||
int64x2_t val;
|
||||
};
|
||||
|
||||
#if CV_SIMD128_64F
|
||||
struct v_float64x2
|
||||
{
|
||||
typedef double lane_type;
|
||||
enum { nlanes = 2 };
|
||||
|
||||
v_float64x2() {}
|
||||
explicit v_float64x2(float64x2_t v) : val(v) {}
|
||||
v_float64x2(double v0, double v1)
|
||||
@@ -324,11 +363,18 @@ struct v_float64x2
|
||||
double v[] = {v0, v1};
|
||||
val = vld1q_f64(v);
|
||||
}
|
||||
|
||||
float64x2_t val;
|
||||
private:
|
||||
friend struct VTraits<v_float64x2>;
|
||||
enum { nlanes = 2 };
|
||||
typedef double lane_type;
|
||||
|
||||
friend typename VTraits<v_float64x2>::lane_type v_get0<v_float64x2>(const v_float64x2& v);
|
||||
double get0() const
|
||||
{
|
||||
return vgetq_lane_f64(val, 0);
|
||||
}
|
||||
float64x2_t val;
|
||||
};
|
||||
#endif
|
||||
|
||||
@@ -460,71 +506,56 @@ inline v_float32x4 v_matmuladd(const v_float32x4& v, const v_float32x4& m0,
|
||||
}
|
||||
|
||||
#define OPENCV_HAL_IMPL_NEON_BIN_OP(bin_op, _Tpvec, intrin) \
|
||||
inline _Tpvec operator bin_op (const _Tpvec& a, const _Tpvec& b) \
|
||||
inline _Tpvec bin_op (const _Tpvec& a, const _Tpvec& b) \
|
||||
{ \
|
||||
return _Tpvec(intrin(a.val, b.val)); \
|
||||
} \
|
||||
inline _Tpvec& operator bin_op##= (_Tpvec& a, const _Tpvec& b) \
|
||||
{ \
|
||||
a.val = intrin(a.val, b.val); \
|
||||
return a; \
|
||||
}
|
||||
|
||||
OPENCV_HAL_IMPL_NEON_BIN_OP(+, v_uint8x16, vqaddq_u8)
|
||||
OPENCV_HAL_IMPL_NEON_BIN_OP(-, v_uint8x16, vqsubq_u8)
|
||||
OPENCV_HAL_IMPL_NEON_BIN_OP(+, v_int8x16, vqaddq_s8)
|
||||
OPENCV_HAL_IMPL_NEON_BIN_OP(-, v_int8x16, vqsubq_s8)
|
||||
OPENCV_HAL_IMPL_NEON_BIN_OP(+, v_uint16x8, vqaddq_u16)
|
||||
OPENCV_HAL_IMPL_NEON_BIN_OP(-, v_uint16x8, vqsubq_u16)
|
||||
OPENCV_HAL_IMPL_NEON_BIN_OP(+, v_int16x8, vqaddq_s16)
|
||||
OPENCV_HAL_IMPL_NEON_BIN_OP(-, v_int16x8, vqsubq_s16)
|
||||
OPENCV_HAL_IMPL_NEON_BIN_OP(+, v_int32x4, vaddq_s32)
|
||||
OPENCV_HAL_IMPL_NEON_BIN_OP(-, v_int32x4, vsubq_s32)
|
||||
OPENCV_HAL_IMPL_NEON_BIN_OP(*, v_int32x4, vmulq_s32)
|
||||
OPENCV_HAL_IMPL_NEON_BIN_OP(+, v_uint32x4, vaddq_u32)
|
||||
OPENCV_HAL_IMPL_NEON_BIN_OP(-, v_uint32x4, vsubq_u32)
|
||||
OPENCV_HAL_IMPL_NEON_BIN_OP(*, v_uint32x4, vmulq_u32)
|
||||
OPENCV_HAL_IMPL_NEON_BIN_OP(+, v_float32x4, vaddq_f32)
|
||||
OPENCV_HAL_IMPL_NEON_BIN_OP(-, v_float32x4, vsubq_f32)
|
||||
OPENCV_HAL_IMPL_NEON_BIN_OP(*, v_float32x4, vmulq_f32)
|
||||
OPENCV_HAL_IMPL_NEON_BIN_OP(+, v_int64x2, vaddq_s64)
|
||||
OPENCV_HAL_IMPL_NEON_BIN_OP(-, v_int64x2, vsubq_s64)
|
||||
OPENCV_HAL_IMPL_NEON_BIN_OP(+, v_uint64x2, vaddq_u64)
|
||||
OPENCV_HAL_IMPL_NEON_BIN_OP(-, v_uint64x2, vsubq_u64)
|
||||
OPENCV_HAL_IMPL_NEON_BIN_OP(v_add, v_uint8x16, vqaddq_u8)
|
||||
OPENCV_HAL_IMPL_NEON_BIN_OP(v_sub, v_uint8x16, vqsubq_u8)
|
||||
OPENCV_HAL_IMPL_NEON_BIN_OP(v_add, v_int8x16, vqaddq_s8)
|
||||
OPENCV_HAL_IMPL_NEON_BIN_OP(v_sub, v_int8x16, vqsubq_s8)
|
||||
OPENCV_HAL_IMPL_NEON_BIN_OP(v_add, v_uint16x8, vqaddq_u16)
|
||||
OPENCV_HAL_IMPL_NEON_BIN_OP(v_sub, v_uint16x8, vqsubq_u16)
|
||||
OPENCV_HAL_IMPL_NEON_BIN_OP(v_add, v_int16x8, vqaddq_s16)
|
||||
OPENCV_HAL_IMPL_NEON_BIN_OP(v_sub, v_int16x8, vqsubq_s16)
|
||||
OPENCV_HAL_IMPL_NEON_BIN_OP(v_add, v_int32x4, vaddq_s32)
|
||||
OPENCV_HAL_IMPL_NEON_BIN_OP(v_sub, v_int32x4, vsubq_s32)
|
||||
OPENCV_HAL_IMPL_NEON_BIN_OP(v_mul, v_int32x4, vmulq_s32)
|
||||
OPENCV_HAL_IMPL_NEON_BIN_OP(v_add, v_uint32x4, vaddq_u32)
|
||||
OPENCV_HAL_IMPL_NEON_BIN_OP(v_sub, v_uint32x4, vsubq_u32)
|
||||
OPENCV_HAL_IMPL_NEON_BIN_OP(v_mul, v_uint32x4, vmulq_u32)
|
||||
OPENCV_HAL_IMPL_NEON_BIN_OP(v_add, v_float32x4, vaddq_f32)
|
||||
OPENCV_HAL_IMPL_NEON_BIN_OP(v_sub, v_float32x4, vsubq_f32)
|
||||
OPENCV_HAL_IMPL_NEON_BIN_OP(v_mul, v_float32x4, vmulq_f32)
|
||||
OPENCV_HAL_IMPL_NEON_BIN_OP(v_add, v_int64x2, vaddq_s64)
|
||||
OPENCV_HAL_IMPL_NEON_BIN_OP(v_sub, v_int64x2, vsubq_s64)
|
||||
OPENCV_HAL_IMPL_NEON_BIN_OP(v_add, v_uint64x2, vaddq_u64)
|
||||
OPENCV_HAL_IMPL_NEON_BIN_OP(v_sub, v_uint64x2, vsubq_u64)
|
||||
#if CV_SIMD128_64F
|
||||
OPENCV_HAL_IMPL_NEON_BIN_OP(/, v_float32x4, vdivq_f32)
|
||||
OPENCV_HAL_IMPL_NEON_BIN_OP(+, v_float64x2, vaddq_f64)
|
||||
OPENCV_HAL_IMPL_NEON_BIN_OP(-, v_float64x2, vsubq_f64)
|
||||
OPENCV_HAL_IMPL_NEON_BIN_OP(*, v_float64x2, vmulq_f64)
|
||||
OPENCV_HAL_IMPL_NEON_BIN_OP(/, v_float64x2, vdivq_f64)
|
||||
OPENCV_HAL_IMPL_NEON_BIN_OP(v_div, v_float32x4, vdivq_f32)
|
||||
OPENCV_HAL_IMPL_NEON_BIN_OP(v_add, v_float64x2, vaddq_f64)
|
||||
OPENCV_HAL_IMPL_NEON_BIN_OP(v_sub, v_float64x2, vsubq_f64)
|
||||
OPENCV_HAL_IMPL_NEON_BIN_OP(v_mul, v_float64x2, vmulq_f64)
|
||||
OPENCV_HAL_IMPL_NEON_BIN_OP(v_div, v_float64x2, vdivq_f64)
|
||||
#else
|
||||
inline v_float32x4 operator / (const v_float32x4& a, const v_float32x4& b)
|
||||
inline v_float32x4 v_div (const v_float32x4& a, const v_float32x4& b)
|
||||
{
|
||||
float32x4_t reciprocal = vrecpeq_f32(b.val);
|
||||
reciprocal = vmulq_f32(vrecpsq_f32(b.val, reciprocal), reciprocal);
|
||||
reciprocal = vmulq_f32(vrecpsq_f32(b.val, reciprocal), reciprocal);
|
||||
return v_float32x4(vmulq_f32(a.val, reciprocal));
|
||||
}
|
||||
inline v_float32x4& operator /= (v_float32x4& a, const v_float32x4& b)
|
||||
{
|
||||
float32x4_t reciprocal = vrecpeq_f32(b.val);
|
||||
reciprocal = vmulq_f32(vrecpsq_f32(b.val, reciprocal), reciprocal);
|
||||
reciprocal = vmulq_f32(vrecpsq_f32(b.val, reciprocal), reciprocal);
|
||||
a.val = vmulq_f32(a.val, reciprocal);
|
||||
return a;
|
||||
}
|
||||
#endif
|
||||
|
||||
// saturating multiply 8-bit, 16-bit
|
||||
#define OPENCV_HAL_IMPL_NEON_MUL_SAT(_Tpvec, _Tpwvec) \
|
||||
inline _Tpvec operator * (const _Tpvec& a, const _Tpvec& b) \
|
||||
inline _Tpvec v_mul (const _Tpvec& a, const _Tpvec& b) \
|
||||
{ \
|
||||
_Tpwvec c, d; \
|
||||
v_mul_expand(a, b, c, d); \
|
||||
return v_pack(c, d); \
|
||||
} \
|
||||
inline _Tpvec& operator *= (_Tpvec& a, const _Tpvec& b) \
|
||||
{ a = a * b; return a; }
|
||||
}
|
||||
|
||||
OPENCV_HAL_IMPL_NEON_MUL_SAT(v_int8x16, v_int16x8)
|
||||
OPENCV_HAL_IMPL_NEON_MUL_SAT(v_uint8x16, v_uint16x8)
|
||||
@@ -698,7 +729,7 @@ inline v_uint32x4 v_dotprod_expand(const v_uint8x16& a, const v_uint8x16& b)
|
||||
inline v_uint32x4 v_dotprod_expand(const v_uint8x16& a, const v_uint8x16& b,
|
||||
const v_uint32x4& c)
|
||||
{
|
||||
return v_dotprod_expand(a, b) + c;
|
||||
return v_add(v_dotprod_expand(a, b), c);
|
||||
}
|
||||
|
||||
inline v_int32x4 v_dotprod_expand(const v_int8x16& a, const v_int8x16& b)
|
||||
@@ -715,7 +746,7 @@ inline v_int32x4 v_dotprod_expand(const v_int8x16& a, const v_int8x16& b)
|
||||
inline v_int32x4 v_dotprod_expand(const v_int8x16& a, const v_int8x16& b,
|
||||
const v_int32x4& c)
|
||||
{
|
||||
return v_dotprod_expand(a, b) + c;
|
||||
return v_add(v_dotprod_expand(a, b), c);
|
||||
}
|
||||
#endif
|
||||
// 16 >> 64
|
||||
@@ -735,7 +766,7 @@ inline v_uint64x2 v_dotprod_expand(const v_uint16x8& a, const v_uint16x8& b)
|
||||
return v_uint64x2(vaddq_u64(s0, s1));
|
||||
}
|
||||
inline v_uint64x2 v_dotprod_expand(const v_uint16x8& a, const v_uint16x8& b, const v_uint64x2& c)
|
||||
{ return v_dotprod_expand(a, b) + c; }
|
||||
{ return v_add(v_dotprod_expand(a, b), c); }
|
||||
|
||||
inline v_int64x2 v_dotprod_expand(const v_int16x8& a, const v_int16x8& b)
|
||||
{
|
||||
@@ -752,7 +783,7 @@ inline v_int64x2 v_dotprod_expand(const v_int16x8& a, const v_int16x8& b)
|
||||
}
|
||||
inline v_int64x2 v_dotprod_expand(const v_int16x8& a, const v_int16x8& b,
|
||||
const v_int64x2& c)
|
||||
{ return v_dotprod_expand(a, b) + c; }
|
||||
{ return v_add(v_dotprod_expand(a, b), c); }
|
||||
|
||||
// 32 >> 64f
|
||||
#if CV_SIMD128_64F
|
||||
@@ -760,7 +791,7 @@ inline v_float64x2 v_dotprod_expand(const v_int32x4& a, const v_int32x4& b)
|
||||
{ return v_cvt_f64(v_dotprod(a, b)); }
|
||||
inline v_float64x2 v_dotprod_expand(const v_int32x4& a, const v_int32x4& b,
|
||||
const v_float64x2& c)
|
||||
{ return v_dotprod_expand(a, b) + c; }
|
||||
{ return v_add(v_dotprod_expand(a, b), c); }
|
||||
#endif
|
||||
|
||||
//////// Fast Dot Product ////////
|
||||
@@ -850,7 +881,7 @@ inline v_uint32x4 v_dotprod_expand_fast(const v_uint8x16& a, const v_uint8x16& b
|
||||
}
|
||||
inline v_uint32x4 v_dotprod_expand_fast(const v_uint8x16& a, const v_uint8x16& b, const v_uint32x4& c)
|
||||
{
|
||||
return v_dotprod_expand_fast(a, b) + c;
|
||||
return v_add(v_dotprod_expand_fast(a, b), c);
|
||||
}
|
||||
|
||||
inline v_int32x4 v_dotprod_expand_fast(const v_int8x16& a, const v_int8x16& b)
|
||||
@@ -861,7 +892,7 @@ inline v_int32x4 v_dotprod_expand_fast(const v_int8x16& a, const v_int8x16& b)
|
||||
}
|
||||
inline v_int32x4 v_dotprod_expand_fast(const v_int8x16& a, const v_int8x16& b, const v_int32x4& c)
|
||||
{
|
||||
return v_dotprod_expand_fast(a, b) + c;
|
||||
return v_add(v_dotprod_expand_fast(a, b), c);
|
||||
}
|
||||
#endif
|
||||
|
||||
@@ -875,7 +906,7 @@ inline v_uint64x2 v_dotprod_expand_fast(const v_uint16x8& a, const v_uint16x8& b
|
||||
return v_uint64x2(vaddq_u64(s0, s1));
|
||||
}
|
||||
inline v_uint64x2 v_dotprod_expand_fast(const v_uint16x8& a, const v_uint16x8& b, const v_uint64x2& c)
|
||||
{ return v_dotprod_expand_fast(a, b) + c; }
|
||||
{ return v_add(v_dotprod_expand_fast(a, b), c); }
|
||||
|
||||
inline v_int64x2 v_dotprod_expand_fast(const v_int16x8& a, const v_int16x8& b)
|
||||
{
|
||||
@@ -884,22 +915,22 @@ inline v_int64x2 v_dotprod_expand_fast(const v_int16x8& a, const v_int16x8& b)
|
||||
return v_int64x2(vaddl_s32(vget_low_s32(prod), vget_high_s32(prod)));
|
||||
}
|
||||
inline v_int64x2 v_dotprod_expand_fast(const v_int16x8& a, const v_int16x8& b, const v_int64x2& c)
|
||||
{ return v_dotprod_expand_fast(a, b) + c; }
|
||||
{ return v_add(v_dotprod_expand_fast(a, b), c); }
|
||||
|
||||
// 32 >> 64f
|
||||
#if CV_SIMD128_64F
|
||||
inline v_float64x2 v_dotprod_expand_fast(const v_int32x4& a, const v_int32x4& b)
|
||||
{ return v_cvt_f64(v_dotprod_fast(a, b)); }
|
||||
inline v_float64x2 v_dotprod_expand_fast(const v_int32x4& a, const v_int32x4& b, const v_float64x2& c)
|
||||
{ return v_dotprod_expand_fast(a, b) + c; }
|
||||
{ return v_add(v_dotprod_expand_fast(a, b), c); }
|
||||
#endif
|
||||
|
||||
|
||||
#define OPENCV_HAL_IMPL_NEON_LOGIC_OP(_Tpvec, suffix) \
|
||||
OPENCV_HAL_IMPL_NEON_BIN_OP(&, _Tpvec, vandq_##suffix) \
|
||||
OPENCV_HAL_IMPL_NEON_BIN_OP(|, _Tpvec, vorrq_##suffix) \
|
||||
OPENCV_HAL_IMPL_NEON_BIN_OP(^, _Tpvec, veorq_##suffix) \
|
||||
inline _Tpvec operator ~ (const _Tpvec& a) \
|
||||
OPENCV_HAL_IMPL_NEON_BIN_OP(v_and, _Tpvec, vandq_##suffix) \
|
||||
OPENCV_HAL_IMPL_NEON_BIN_OP(v_or, _Tpvec, vorrq_##suffix) \
|
||||
OPENCV_HAL_IMPL_NEON_BIN_OP(v_xor, _Tpvec, veorq_##suffix) \
|
||||
inline _Tpvec v_not (const _Tpvec& a) \
|
||||
{ \
|
||||
return _Tpvec(vreinterpretq_##suffix##_u8(vmvnq_u8(vreinterpretq_u8_##suffix(a.val)))); \
|
||||
}
|
||||
@@ -914,21 +945,16 @@ OPENCV_HAL_IMPL_NEON_LOGIC_OP(v_uint64x2, u64)
|
||||
OPENCV_HAL_IMPL_NEON_LOGIC_OP(v_int64x2, s64)
|
||||
|
||||
#define OPENCV_HAL_IMPL_NEON_FLT_BIT_OP(bin_op, intrin) \
|
||||
inline v_float32x4 operator bin_op (const v_float32x4& a, const v_float32x4& b) \
|
||||
inline v_float32x4 bin_op (const v_float32x4& a, const v_float32x4& b) \
|
||||
{ \
|
||||
return v_float32x4(vreinterpretq_f32_s32(intrin(vreinterpretq_s32_f32(a.val), vreinterpretq_s32_f32(b.val)))); \
|
||||
} \
|
||||
inline v_float32x4& operator bin_op##= (v_float32x4& a, const v_float32x4& b) \
|
||||
{ \
|
||||
a.val = vreinterpretq_f32_s32(intrin(vreinterpretq_s32_f32(a.val), vreinterpretq_s32_f32(b.val))); \
|
||||
return a; \
|
||||
}
|
||||
|
||||
OPENCV_HAL_IMPL_NEON_FLT_BIT_OP(&, vandq_s32)
|
||||
OPENCV_HAL_IMPL_NEON_FLT_BIT_OP(|, vorrq_s32)
|
||||
OPENCV_HAL_IMPL_NEON_FLT_BIT_OP(^, veorq_s32)
|
||||
OPENCV_HAL_IMPL_NEON_FLT_BIT_OP(v_and, vandq_s32)
|
||||
OPENCV_HAL_IMPL_NEON_FLT_BIT_OP(v_or, vorrq_s32)
|
||||
OPENCV_HAL_IMPL_NEON_FLT_BIT_OP(v_xor, veorq_s32)
|
||||
|
||||
inline v_float32x4 operator ~ (const v_float32x4& a)
|
||||
inline v_float32x4 v_not (const v_float32x4& a)
|
||||
{
|
||||
return v_float32x4(vreinterpretq_f32_s32(vmvnq_s32(vreinterpretq_s32_f32(a.val))));
|
||||
}
|
||||
@@ -942,7 +968,7 @@ inline v_float32x4 v_sqrt(const v_float32x4& x)
|
||||
inline v_float32x4 v_invsqrt(const v_float32x4& x)
|
||||
{
|
||||
v_float32x4 one = v_setall_f32(1.0f);
|
||||
return one / v_sqrt(x);
|
||||
return v_div(one, v_sqrt(x));
|
||||
}
|
||||
#else
|
||||
inline v_float32x4 v_sqrt(const v_float32x4& x)
|
||||
@@ -975,21 +1001,16 @@ inline v_float32x4 v_abs(v_float32x4 x)
|
||||
|
||||
#if CV_SIMD128_64F
|
||||
#define OPENCV_HAL_IMPL_NEON_DBL_BIT_OP(bin_op, intrin) \
|
||||
inline v_float64x2 operator bin_op (const v_float64x2& a, const v_float64x2& b) \
|
||||
inline v_float64x2 bin_op (const v_float64x2& a, const v_float64x2& b) \
|
||||
{ \
|
||||
return v_float64x2(vreinterpretq_f64_s64(intrin(vreinterpretq_s64_f64(a.val), vreinterpretq_s64_f64(b.val)))); \
|
||||
} \
|
||||
inline v_float64x2& operator bin_op##= (v_float64x2& a, const v_float64x2& b) \
|
||||
{ \
|
||||
a.val = vreinterpretq_f64_s64(intrin(vreinterpretq_s64_f64(a.val), vreinterpretq_s64_f64(b.val))); \
|
||||
return a; \
|
||||
}
|
||||
|
||||
OPENCV_HAL_IMPL_NEON_DBL_BIT_OP(&, vandq_s64)
|
||||
OPENCV_HAL_IMPL_NEON_DBL_BIT_OP(|, vorrq_s64)
|
||||
OPENCV_HAL_IMPL_NEON_DBL_BIT_OP(^, veorq_s64)
|
||||
OPENCV_HAL_IMPL_NEON_DBL_BIT_OP(v_and, vandq_s64)
|
||||
OPENCV_HAL_IMPL_NEON_DBL_BIT_OP(v_or, vorrq_s64)
|
||||
OPENCV_HAL_IMPL_NEON_DBL_BIT_OP(v_xor, veorq_s64)
|
||||
|
||||
inline v_float64x2 operator ~ (const v_float64x2& a)
|
||||
inline v_float64x2 v_not (const v_float64x2& a)
|
||||
{
|
||||
return v_float64x2(vreinterpretq_f64_s32(vmvnq_s32(vreinterpretq_s32_f64(a.val))));
|
||||
}
|
||||
@@ -1002,7 +1023,7 @@ inline v_float64x2 v_sqrt(const v_float64x2& x)
|
||||
inline v_float64x2 v_invsqrt(const v_float64x2& x)
|
||||
{
|
||||
v_float64x2 one = v_setall_f64(1.0f);
|
||||
return one / v_sqrt(x);
|
||||
return v_div(one, v_sqrt(x));
|
||||
}
|
||||
|
||||
inline v_float64x2 v_abs(v_float64x2 x)
|
||||
@@ -1037,17 +1058,17 @@ OPENCV_HAL_IMPL_NEON_BIN_FUNC(v_float64x2, v_max, vmaxq_f64)
|
||||
#endif
|
||||
|
||||
#define OPENCV_HAL_IMPL_NEON_INT_CMP_OP(_Tpvec, cast, suffix, not_suffix) \
|
||||
inline _Tpvec operator == (const _Tpvec& a, const _Tpvec& b) \
|
||||
inline _Tpvec v_eq (const _Tpvec& a, const _Tpvec& b) \
|
||||
{ return _Tpvec(cast(vceqq_##suffix(a.val, b.val))); } \
|
||||
inline _Tpvec operator != (const _Tpvec& a, const _Tpvec& b) \
|
||||
inline _Tpvec v_ne (const _Tpvec& a, const _Tpvec& b) \
|
||||
{ return _Tpvec(cast(vmvnq_##not_suffix(vceqq_##suffix(a.val, b.val)))); } \
|
||||
inline _Tpvec operator < (const _Tpvec& a, const _Tpvec& b) \
|
||||
inline _Tpvec v_lt (const _Tpvec& a, const _Tpvec& b) \
|
||||
{ return _Tpvec(cast(vcltq_##suffix(a.val, b.val))); } \
|
||||
inline _Tpvec operator > (const _Tpvec& a, const _Tpvec& b) \
|
||||
inline _Tpvec v_gt (const _Tpvec& a, const _Tpvec& b) \
|
||||
{ return _Tpvec(cast(vcgtq_##suffix(a.val, b.val))); } \
|
||||
inline _Tpvec operator <= (const _Tpvec& a, const _Tpvec& b) \
|
||||
inline _Tpvec v_le (const _Tpvec& a, const _Tpvec& b) \
|
||||
{ return _Tpvec(cast(vcleq_##suffix(a.val, b.val))); } \
|
||||
inline _Tpvec operator >= (const _Tpvec& a, const _Tpvec& b) \
|
||||
inline _Tpvec v_ge (const _Tpvec& a, const _Tpvec& b) \
|
||||
{ return _Tpvec(cast(vcgeq_##suffix(a.val, b.val))); }
|
||||
|
||||
OPENCV_HAL_IMPL_NEON_INT_CMP_OP(v_uint8x16, OPENCV_HAL_NOP, u8, u8)
|
||||
@@ -1067,48 +1088,48 @@ static inline uint64x2_t vmvnq_u64(uint64x2_t a)
|
||||
OPENCV_HAL_IMPL_NEON_INT_CMP_OP(v_uint64x2, OPENCV_HAL_NOP, u64, u64)
|
||||
OPENCV_HAL_IMPL_NEON_INT_CMP_OP(v_int64x2, vreinterpretq_s64_u64, s64, u64)
|
||||
#else
|
||||
static inline v_uint64x2 operator == (const v_uint64x2& a, const v_uint64x2& b)
|
||||
static inline v_uint64x2 v_eq(const v_uint64x2& a, const v_uint64x2& b)
|
||||
{
|
||||
uint32x4_t cmp = vceqq_u32(vreinterpretq_u32_u64(a.val),
|
||||
vreinterpretq_u32_u64(b.val));
|
||||
uint32x4_t v_eq = vandq_u32(cmp, vrev64q_u32(cmp));
|
||||
return v_uint64x2(vreinterpretq_u64_u32(v_eq));
|
||||
}
|
||||
static inline v_uint64x2 operator != (const v_uint64x2& a, const v_uint64x2& b)
|
||||
static inline v_uint64x2 v_ne(const v_uint64x2& a, const v_uint64x2& b)
|
||||
{
|
||||
uint64x2_t v_mask = vorrq_u64(vsubq_u64(a.val, b.val), vsubq_u64(b.val, a.val));
|
||||
int64x2_t v_smask = vshrq_n_s64(vreinterpretq_s64_u64(v_mask), 63);
|
||||
return v_uint64x2(vreinterpretq_u64_s64(v_smask));
|
||||
}
|
||||
static inline v_int64x2 operator == (const v_int64x2& a, const v_int64x2& b)
|
||||
static inline v_int64x2 v_eq(const v_int64x2& a, const v_int64x2& b)
|
||||
{
|
||||
uint32x4_t cmp = vceqq_u32(vreinterpretq_u32_s64(a.val),
|
||||
vreinterpretq_u32_s64(b.val));
|
||||
uint32x4_t v_eq = vandq_u32(cmp, vrev64q_u32(cmp));
|
||||
return v_int64x2(vreinterpretq_s64_u32(v_eq));
|
||||
}
|
||||
static inline v_int64x2 operator != (const v_int64x2& a, const v_int64x2& b)
|
||||
static inline v_int64x2 v_ne(const v_int64x2& a, const v_int64x2& b)
|
||||
{
|
||||
int64x2_t v_mask = vorrq_s64(vsubq_s64(a.val, b.val), vsubq_s64(b.val, a.val));
|
||||
int64x2_t v_smask = vshrq_n_s64(v_mask, 63);
|
||||
return v_int64x2(v_smask);
|
||||
}
|
||||
static inline v_uint64x2 operator > (const v_uint64x2& a, const v_uint64x2& b)
|
||||
static inline v_uint64x2 v_gt(const v_uint64x2& a, const v_uint64x2& b)
|
||||
{
|
||||
int64x2_t v_mask = vreinterpretq_s64_u64(vsubq_u64(b.val, a.val));
|
||||
return v_uint64x2(vreinterpretq_u64_s64(vshrq_n_s64(v_mask, 63)));
|
||||
}
|
||||
static inline v_uint64x2 operator < (const v_uint64x2& a, const v_uint64x2& b)
|
||||
static inline v_uint64x2 v_lt(const v_uint64x2& a, const v_uint64x2& b)
|
||||
{
|
||||
int64x2_t v_mask = vreinterpretq_s64_u64(vsubq_u64(a.val, b.val));
|
||||
return v_uint64x2(vreinterpretq_u64_s64(vshrq_n_s64(v_mask, 63)));
|
||||
}
|
||||
static inline v_int64x2 operator > (const v_int64x2& a, const v_int64x2& b)
|
||||
static inline v_int64x2 v_gt(const v_int64x2& a, const v_int64x2& b)
|
||||
{
|
||||
int64x2_t v_mask = vsubq_s64(b.val, a.val);
|
||||
return v_int64x2(vshrq_n_s64(v_mask, 63));
|
||||
}
|
||||
static inline v_int64x2 operator < (const v_int64x2& a, const v_int64x2& b)
|
||||
static inline v_int64x2 v_lt (const v_int64x2& a, const v_int64x2& b)
|
||||
{
|
||||
int64x2_t v_mask = vsubq_s64(a.val, b.val);
|
||||
return v_int64x2(vshrq_n_s64(v_mask, 63));
|
||||
@@ -1224,9 +1245,9 @@ inline v_float64x2 v_muladd(const v_float64x2& a, const v_float64x2& b, const v_
|
||||
|
||||
// trade efficiency for convenience
|
||||
#define OPENCV_HAL_IMPL_NEON_SHIFT_OP(_Tpvec, suffix, _Tps, ssuffix) \
|
||||
inline _Tpvec operator << (const _Tpvec& a, int n) \
|
||||
inline _Tpvec v_shl (const _Tpvec& a, int n) \
|
||||
{ return _Tpvec(vshlq_##suffix(a.val, vdupq_n_##ssuffix((_Tps)n))); } \
|
||||
inline _Tpvec operator >> (const _Tpvec& a, int n) \
|
||||
inline _Tpvec v_shr (const _Tpvec& a, int n) \
|
||||
{ return _Tpvec(vshlq_##suffix(a.val, vdupq_n_##ssuffix((_Tps)-n))); } \
|
||||
template<int n> inline _Tpvec v_shl(const _Tpvec& a) \
|
||||
{ return _Tpvec(vshlq_n_##suffix(a.val, n)); } \
|
||||
@@ -1248,13 +1269,13 @@ OPENCV_HAL_IMPL_NEON_SHIFT_OP(v_int64x2, s64, int64, s64)
|
||||
template<int n> inline _Tpvec v_rotate_right(const _Tpvec& a) \
|
||||
{ return _Tpvec(vextq_##suffix(a.val, vdupq_n_##suffix(0), n)); } \
|
||||
template<int n> inline _Tpvec v_rotate_left(const _Tpvec& a) \
|
||||
{ return _Tpvec(vextq_##suffix(vdupq_n_##suffix(0), a.val, _Tpvec::nlanes - n)); } \
|
||||
{ return _Tpvec(vextq_##suffix(vdupq_n_##suffix(0), a.val, VTraits<_Tpvec>::nlanes - n)); } \
|
||||
template<> inline _Tpvec v_rotate_left<0>(const _Tpvec& a) \
|
||||
{ return a; } \
|
||||
template<int n> inline _Tpvec v_rotate_right(const _Tpvec& a, const _Tpvec& b) \
|
||||
{ return _Tpvec(vextq_##suffix(a.val, b.val, n)); } \
|
||||
template<int n> inline _Tpvec v_rotate_left(const _Tpvec& a, const _Tpvec& b) \
|
||||
{ return _Tpvec(vextq_##suffix(b.val, a.val, _Tpvec::nlanes - n)); } \
|
||||
{ return _Tpvec(vextq_##suffix(b.val, a.val, VTraits<_Tpvec>::nlanes - n)); } \
|
||||
template<> inline _Tpvec v_rotate_left<0>(const _Tpvec& a, const _Tpvec& b) \
|
||||
{ CV_UNUSED(b); return a; }
|
||||
|
||||
|
||||
Reference in New Issue
Block a user