potok/opencv
1
0
Fork 0
mirror of https://github.com/opencv/opencv.git synced 2026-07-30 15:53:03 +04:00
Code Packages Releases Wiki Activity

Clean up the Universal Intrinsic API.

Browse Source
This commit is contained in:
Liutong HAN
2023-10-13 19:23:30 +08:00
parent 0507043a55
commit a287605c3e
40 changed files with 1611 additions and 1531 deletions
Download Patch File Download Diff File Expand all files Collapse all files
modules/imgproc/src/bilateral_filter.simd.hpp
+17 -17
View File
@@ -140,9 +140,9 @@ public:
#if CV_SIMD128
v_uint32x4 rval = v_setall_u32(sptr[j]);
v_uint32x4 val(ksptr0[j], ksptr1[j], ksptr2[j], ksptr3[j]);
v_float32x4 w = kweight4 * v_lut(color_weight, v_reinterpret_as_s32(v_absdiff(val, rval)));
v_float32x4 w = v_mul(kweight4, v_lut(this->color_weight, v_reinterpret_as_s32(v_absdiff(val, rval))));
wsum[j] += v_reduce_sum(w);
sum[j] += v_reduce_sum(v_cvt_f32(v_reinterpret_as_s32(val)) * w);
sum[j] += v_reduce_sum(v_mul(v_cvt_f32(v_reinterpret_as_s32(val)), w));
#else
int rval = sptr[j];
@@ -407,11 +407,11 @@ public:
v_uint32x4 b(ksptr0[0], ksptr1[0], ksptr2[0], ksptr3[0]);
v_uint32x4 g(ksptr0[1], ksptr1[1], ksptr2[1], ksptr3[1]);
v_uint32x4 r(ksptr0[2], ksptr1[2], ksptr2[2], ksptr3[2]);
v_float32x4 w = kweight4 * v_lut(color_weight, v_reinterpret_as_s32(v_absdiff(b, rb) + v_absdiff(g, rg) + v_absdiff(r, rr)));
v_float32x4 w = v_mul(kweight4, v_lut(this->color_weight, v_reinterpret_as_s32(v_add(v_add(v_absdiff(b, rb), v_absdiff(g, rg)), v_absdiff(r, rr)))));
wsum[j] += v_reduce_sum(w);
sum_b[j] += v_reduce_sum(v_cvt_f32(v_reinterpret_as_s32(b)) * w);
sum_g[j] += v_reduce_sum(v_cvt_f32(v_reinterpret_as_s32(g)) * w);
sum_r[j] += v_reduce_sum(v_cvt_f32(v_reinterpret_as_s32(r)) * w);
sum_b[j] += v_reduce_sum(v_mul(v_cvt_f32(v_reinterpret_as_s32(b)), w));
sum_g[j] += v_reduce_sum(v_mul(v_cvt_f32(v_reinterpret_as_s32(g)), w));
sum_r[j] += v_reduce_sum(v_mul(v_cvt_f32(v_reinterpret_as_s32(r)), w));
#else
int rb = rsptr[0], rg = rsptr[1], rr = rsptr[2];
@@ -661,12 +661,12 @@ public:
v_float32x4 rval = v_setall_f32(sptr[j]);
v_float32x4 val(ksptr0[j], ksptr1[j], ksptr2[j], ksptr3[j]);
v_float32x4 knan = v_not_nan(val);
v_float32x4 alpha = (v_absdiff(val, rval) * sindex4) & v_not_nan(rval) & knan;
v_float32x4 alpha = v_and(v_and(v_mul(v_absdiff(val, rval), sindex4), v_not_nan(rval)), knan);
v_int32x4 idx = v_trunc(alpha);
alpha -= v_cvt_f32(idx);
v_float32x4 w = (kweight4 * v_muladd(v_lut(expLUT + 1, idx), alpha, v_lut(expLUT, idx) * (v_one4 - alpha))) & knan;
alpha = v_sub(alpha, v_cvt_f32(idx));
v_float32x4 w = v_and(v_mul(kweight4, v_muladd(v_lut(this->expLUT + 1, idx), alpha, v_mul(v_lut(this->expLUT, idx), v_sub(v_one4, alpha)))), knan);
wsum[j] += v_reduce_sum(w);
sum[j] += v_reduce_sum((val & knan) * w);
sum[j] += v_reduce_sum(v_mul(v_and(val, knan), w));
#else
float rval = sptr[j];
@@ -862,15 +862,15 @@ public:
v_float32x4 kb(ksptr0[0], ksptr1[0], ksptr2[0], ksptr3[0]);
v_float32x4 kg(ksptr0[1], ksptr1[1], ksptr2[1], ksptr3[1]);
v_float32x4 kr(ksptr0[2], ksptr1[2], ksptr2[2], ksptr3[2]);
v_float32x4 knan = v_not_nan(kb) & v_not_nan(kg) & v_not_nan(kr);
v_float32x4 alpha = ((v_absdiff(kb, rb) + v_absdiff(kg, rg) + v_absdiff(kr, rr)) * sindex4) & v_not_nan(rb) & v_not_nan(rg) & v_not_nan(rr) & knan;
v_float32x4 knan = v_and(v_and(v_not_nan(kb), v_not_nan(kg)), v_not_nan(kr));
v_float32x4 alpha = v_and(v_and(v_and(v_and(v_mul(v_add(v_add(v_absdiff(kb, rb), v_absdiff(kg, rg)), v_absdiff(kr, rr)), sindex4), v_not_nan(rb)), v_not_nan(rg)), v_not_nan(rr)), knan);
v_int32x4 idx = v_trunc(alpha);
alpha -= v_cvt_f32(idx);
v_float32x4 w = (kweight4 * v_muladd(v_lut(expLUT + 1, idx), alpha, v_lut(expLUT, idx) * (v_one4 - alpha))) & knan;
alpha = v_sub(alpha, v_cvt_f32(idx));
v_float32x4 w = v_and(v_mul(kweight4, v_muladd(v_lut(this->expLUT + 1, idx), alpha, v_mul(v_lut(this->expLUT, idx), v_sub(v_one4, alpha)))), knan);
wsum[j] += v_reduce_sum(w);
sum_b[j] += v_reduce_sum((kb & knan) * w);
sum_g[j] += v_reduce_sum((kg & knan) * w);
sum_r[j] += v_reduce_sum((kr & knan) * w);
sum_b[j] += v_reduce_sum(v_mul(v_and(kb, knan), w));
sum_g[j] += v_reduce_sum(v_mul(v_and(kg, knan), w));
sum_r[j] += v_reduce_sum(v_mul(v_and(kr, knan), w));
#else
float rb = rsptr[0], rg = rsptr[1], rr = rsptr[2];
bool r_NAN = cvIsNaN(rb) || cvIsNaN(rg) || cvIsNaN(rr);
modules/imgproc/src/box_filter.simd.hpp
+32 -32
View File
@@ -315,7 +315,7 @@ struct ColumnSum<int, uchar> :
v_store(SUM + i, v_add(vx_load(SUM + i), vx_load(Sp + i)));
}
#if !CV_SIMD_SCALABLE && CV_SIMD_WIDTH > 16
for (; i <= width - v_int32x4::nlanes; i += v_int32x4::nlanes)
for (; i <= width - VTraits<v_int32x4>::vlanes(); i += VTraits<v_int32x4>::vlanes())
{
v_store(SUM + i, v_add(v_load(SUM + i), v_load(Sp + i)));
}
@@ -357,10 +357,10 @@ struct ColumnSum<int, uchar> :
}
#if !CV_SIMD_SCALABLE && CV_SIMD_WIDTH > 16
v_float32x4 v_scale = v_setall_f32((float)_scale);
for( ; i <= width-v_uint16x8::nlanes; i+=v_uint16x8::nlanes )
for( ; i <= width-VTraits<v_uint16x8>::vlanes(); i+=VTraits<v_uint16x8>::vlanes() )
{
v_int32x4 v_s0 = v_add(v_load(SUM + i), v_load(Sp + i));
v_int32x4 v_s01 = v_add(v_load(SUM + i + v_int32x4::nlanes), v_load(Sp + i + v_int32x4::nlanes));
v_int32x4 v_s01 = v_add(v_load(SUM + i + VTraits<v_int32x4>::vlanes()), v_load(Sp + i + VTraits<v_int32x4>::vlanes()));
v_uint32x4 v_s0d = v_reinterpret_as_u32(v_round(v_mul(v_cvt_f32(v_s0), v_scale)));
v_uint32x4 v_s01d = v_reinterpret_as_u32(v_round(v_mul(v_cvt_f32(v_s01), v_scale)));
@@ -369,7 +369,7 @@ struct ColumnSum<int, uchar> :
v_pack_store(D + i, v_dst);
v_store(SUM + i, v_sub(v_s0, v_load(Sm + i)));
v_store(SUM + i + v_int32x4::nlanes, v_sub(v_s01, v_load(Sm + i + v_int32x4::nlanes)));
v_store(SUM + i + VTraits<v_int32x4>::vlanes(), v_sub(v_s01, v_load(Sm + i + VTraits<v_int32x4>::vlanes())));
}
#endif
#endif
@@ -396,16 +396,16 @@ struct ColumnSum<int, uchar> :
v_store(SUM + i + VTraits<v_int32>::vlanes(), v_sub(v_s01, vx_load(Sm + i + VTraits<v_int32>::vlanes())));
}
#if !CV_SIMD_SCALABLE && CV_SIMD_WIDTH > 16
for( ; i <= width-v_uint16x8::nlanes; i+=v_uint16x8::nlanes )
for( ; i <= width-VTraits<v_uint16x8>::vlanes(); i+=VTraits<v_uint16x8>::vlanes() )
{
v_int32x4 v_s0 = v_add(v_load(SUM + i), v_load(Sp + i));
v_int32x4 v_s01 = v_add(v_load(SUM + i + v_int32x4::nlanes), v_load(Sp + i + v_int32x4::nlanes));
v_int32x4 v_s01 = v_add(v_load(SUM + i + VTraits<v_int32x4>::vlanes()), v_load(Sp + i + VTraits<v_int32x4>::vlanes()));
v_uint16x8 v_dst = v_pack(v_reinterpret_as_u32(v_s0), v_reinterpret_as_u32(v_s01));
v_pack_store(D + i, v_dst);
v_store(SUM + i, v_sub(v_s0, v_load(Sm + i)));
v_store(SUM + i + v_int32x4::nlanes, v_sub(v_s01, v_load(Sm + i + v_int32x4::nlanes)));
v_store(SUM + i + VTraits<v_int32x4>::vlanes(), v_sub(v_s01, v_load(Sm + i + VTraits<v_int32x4>::vlanes())));
}
#endif
#endif
@@ -486,7 +486,7 @@ public BaseColumnFilter
v_store(SUM + i, v_add(vx_load(SUM + i), vx_load(Sp + i)));
}
#if !CV_SIMD_SCALABLE && CV_SIMD_WIDTH > 16
for( ; i <= width - v_uint16x8::nlanes; i += v_uint16x8::nlanes )
for( ; i <= width - VTraits<v_uint16x8>::vlanes(); i += VTraits<v_uint16x8>::vlanes() )
{
v_store(SUM + i, v_add(v_load(SUM + i), v_load(Sp + i)));
}
@@ -546,13 +546,13 @@ public BaseColumnFilter
v_uint32x4 ds4 = v_setall_u32((unsigned)ds);
v_uint16x8 dd8 = v_setall_u16((ushort)dd);
for( ; i <= width-v_uint8x16::nlanes; i+=v_uint8x16::nlanes )
for( ; i <= width-VTraits<v_uint8x16>::vlanes(); i+=VTraits<v_uint8x16>::vlanes() )
{
v_uint16x8 _sm0 = v_load(Sm + i);
v_uint16x8 _sm1 = v_load(Sm + i + v_uint16x8::nlanes);
v_uint16x8 _sm1 = v_load(Sm + i + VTraits<v_uint16x8>::vlanes());
v_uint16x8 _s0 = v_add_wrap(v_load(SUM + i), v_load(Sp + i));
v_uint16x8 _s1 = v_add_wrap(v_load(SUM + i + v_uint16x8::nlanes), v_load(Sp + i + v_uint16x8::nlanes));
v_uint16x8 _s1 = v_add_wrap(v_load(SUM + i + VTraits<v_uint16x8>::vlanes()), v_load(Sp + i + VTraits<v_uint16x8>::vlanes()));
v_uint32x4 _s00, _s01, _s10, _s11;
@@ -572,7 +572,7 @@ public BaseColumnFilter
v_store(D + i, v_pack_u(r0, r1));
v_store(SUM + i, _s0);
v_store(SUM + i + v_uint16x8::nlanes, _s1);
v_store(SUM + i + VTraits<v_uint16x8>::vlanes(), _s1);
}
#endif
#endif
@@ -649,7 +649,7 @@ struct ColumnSum<int, short> :
v_store(SUM + i, v_add(vx_load(SUM + i), vx_load(Sp + i)));
}
#if !CV_SIMD_SCALABLE && CV_SIMD_WIDTH > 16
for( ; i <= width - v_int32x4::nlanes; i+=v_int32x4::nlanes )
for( ; i <= width - VTraits<v_int32x4>::vlanes(); i+=VTraits<v_int32x4>::vlanes() )
{
v_store(SUM + i, v_add(v_load(SUM + i), v_load(Sp + i)));
}
@@ -689,17 +689,17 @@ struct ColumnSum<int, short> :
}
#if !CV_SIMD_SCALABLE && CV_SIMD_WIDTH > 16
v_float32x4 v_scale = v_setall_f32((float)_scale);
for( ; i <= width-v_int16x8::nlanes; i+=v_int16x8::nlanes )
for( ; i <= width-VTraits<v_int16x8>::vlanes(); i+=VTraits<v_int16x8>::vlanes() )
{
v_int32x4 v_s0 = v_add(v_load(SUM + i), v_load(Sp + i));
v_int32x4 v_s01 = v_add(v_load(SUM + i + v_int32x4::nlanes), v_load(Sp + i + v_int32x4::nlanes));
v_int32x4 v_s01 = v_add(v_load(SUM + i + VTraits<v_int32x4>::vlanes()), v_load(Sp + i + VTraits<v_int32x4>::vlanes()));
v_int32x4 v_s0d = v_round(v_mul(v_cvt_f32(v_s0), v_scale));
v_int32x4 v_s01d = v_round(v_mul(v_cvt_f32(v_s01), v_scale));
v_store(D + i, v_pack(v_s0d, v_s01d));
v_store(SUM + i, v_sub(v_s0, v_load(Sm + i)));
v_store(SUM + i + v_int32x4::nlanes, v_sub(v_s01, v_load(Sm + i + v_int32x4::nlanes)));
v_store(SUM + i + VTraits<v_int32x4>::vlanes(), v_sub(v_s01, v_load(Sm + i + VTraits<v_int32x4>::vlanes())));
}
#endif
#endif
@@ -725,15 +725,15 @@ struct ColumnSum<int, short> :
v_store(SUM + i + VTraits<v_int32>::vlanes(), v_sub(v_s01, vx_load(Sm + i + VTraits<v_int32>::vlanes())));
}
#if !CV_SIMD_SCALABLE && CV_SIMD_WIDTH > 16
for( ; i <= width-v_int16x8::nlanes; i+=v_int16x8::nlanes )
for( ; i <= width-VTraits<v_int16x8>::vlanes(); i+=VTraits<v_int16x8>::vlanes() )
{
v_int32x4 v_s0 = v_add(v_load(SUM + i), v_load(Sp + i));
v_int32x4 v_s01 = v_add(v_load(SUM + i + v_int32x4::nlanes), v_load(Sp + i + v_int32x4::nlanes));
v_int32x4 v_s01 = v_add(v_load(SUM + i + VTraits<v_int32x4>::vlanes()), v_load(Sp + i + VTraits<v_int32x4>::vlanes()));
v_store(D + i, v_pack(v_s0, v_s01));
v_store(SUM + i, v_sub(v_s0, v_load(Sm + i)));
v_store(SUM + i + v_int32x4::nlanes, v_sub(v_s01, v_load(Sm + i + v_int32x4::nlanes)));
v_store(SUM + i + VTraits<v_int32x4>::vlanes(), v_sub(v_s01, v_load(Sm + i + VTraits<v_int32x4>::vlanes())));
}
#endif
#endif
@@ -798,7 +798,7 @@ struct ColumnSum<int, ushort> :
v_store(SUM + i, v_add(vx_load(SUM + i), vx_load(Sp + i)));
}
#if !CV_SIMD_SCALABLE && CV_SIMD_WIDTH > 16
for (; i <= width - v_int32x4::nlanes; i += v_int32x4::nlanes)
for (; i <= width - VTraits<v_int32x4>::vlanes(); i += VTraits<v_int32x4>::vlanes())
{
v_store(SUM + i, v_add(v_load(SUM + i), v_load(Sp + i)));
}
@@ -838,17 +838,17 @@ struct ColumnSum<int, ushort> :
}
#if !CV_SIMD_SCALABLE && CV_SIMD_WIDTH > 16
v_float32x4 v_scale = v_setall_f32((float)_scale);
for( ; i <= width-v_uint16x8::nlanes; i+=v_uint16x8::nlanes )
for( ; i <= width-VTraits<v_uint16x8>::vlanes(); i+=VTraits<v_uint16x8>::vlanes() )
{
v_int32x4 v_s0 = v_add(v_load(SUM + i), v_load(Sp + i));
v_int32x4 v_s01 = v_add(v_load(SUM + i + v_int32x4::nlanes), v_load(Sp + i + v_int32x4::nlanes));
v_int32x4 v_s01 = v_add(v_load(SUM + i + VTraits<v_int32x4>::vlanes()), v_load(Sp + i + VTraits<v_int32x4>::vlanes()));
v_uint32x4 v_s0d = v_reinterpret_as_u32(v_round(v_mul(v_cvt_f32(v_s0), v_scale)));
v_uint32x4 v_s01d = v_reinterpret_as_u32(v_round(v_mul(v_cvt_f32(v_s01), v_scale)));
v_store(D + i, v_pack(v_s0d, v_s01d));
v_store(SUM + i, v_sub(v_s0, v_load(Sm + i)));
v_store(SUM + i + v_int32x4::nlanes, v_sub(v_s01, v_load(Sm + i + v_int32x4::nlanes)));
v_store(SUM + i + VTraits<v_int32x4>::vlanes(), v_sub(v_s01, v_load(Sm + i + VTraits<v_int32x4>::vlanes())));
}
#endif
#endif
@@ -874,15 +874,15 @@ struct ColumnSum<int, ushort> :
v_store(SUM + i + VTraits<v_int32>::vlanes(), v_sub(v_s01, vx_load(Sm + i + VTraits<v_int32>::vlanes())));
}
#if !CV_SIMD_SCALABLE && CV_SIMD_WIDTH > 16
for( ; i <= width-v_uint16x8::nlanes; i+=v_uint16x8::nlanes )
for( ; i <= width-VTraits<v_uint16x8>::vlanes(); i+=VTraits<v_uint16x8>::vlanes() )
{
v_int32x4 v_s0 = v_add(v_load(SUM + i), v_load(Sp + i));
v_int32x4 v_s01 = v_add(v_load(SUM + i + v_int32x4::nlanes), v_load(Sp + i + v_int32x4::nlanes));
v_int32x4 v_s01 = v_add(v_load(SUM + i + VTraits<v_int32x4>::vlanes()), v_load(Sp + i + VTraits<v_int32x4>::vlanes()));
v_store(D + i, v_pack(v_reinterpret_as_u32(v_s0), v_reinterpret_as_u32(v_s01)));
v_store(SUM + i, v_sub(v_s0, v_load(Sm + i)));
v_store(SUM + i + v_int32x4::nlanes, v_sub(v_s01, v_load(Sm + i + v_int32x4::nlanes)));
v_store(SUM + i + VTraits<v_int32x4>::vlanes(), v_sub(v_s01, v_load(Sm + i + VTraits<v_int32x4>::vlanes())));
}
#endif
#endif
@@ -945,7 +945,7 @@ struct ColumnSum<int, int> :
v_store(SUM + i, v_add(vx_load(SUM + i), vx_load(Sp + i)));
}
#if !CV_SIMD_SCALABLE && CV_SIMD_WIDTH > 16
for( ; i <= width - v_int32x4::nlanes; i+=v_int32x4::nlanes )
for( ; i <= width - VTraits<v_int32x4>::vlanes(); i+=VTraits<v_int32x4>::vlanes() )
{
v_store(SUM + i, v_add(v_load(SUM + i), v_load(Sp + i)));
}
@@ -981,7 +981,7 @@ struct ColumnSum<int, int> :
}
#if !CV_SIMD_SCALABLE && CV_SIMD_WIDTH > 16
v_float32x4 v_scale = v_setall_f32((float)_scale);
for( ; i <= width-v_int32x4::nlanes; i+=v_int32x4::nlanes )
for( ; i <= width-VTraits<v_int32x4>::vlanes(); i+=VTraits<v_int32x4>::vlanes() )
{
v_int32x4 v_s0 = v_add(v_load(SUM + i), v_load(Sp + i));
v_int32x4 v_s0d = v_round(v_mul(v_cvt_f32(v_s0), v_scale));
@@ -1010,7 +1010,7 @@ struct ColumnSum<int, int> :
v_store(SUM + i, v_sub(v_s0, vx_load(Sm + i)));
}
#if !CV_SIMD_SCALABLE && CV_SIMD_WIDTH > 16
for( ; i <= width-v_int32x4::nlanes; i+=v_int32x4::nlanes )
for( ; i <= width-VTraits<v_int32x4>::vlanes(); i+=VTraits<v_int32x4>::vlanes() )
{
v_int32x4 v_s0 = v_add(v_load(SUM + i), v_load(Sp + i));
@@ -1079,7 +1079,7 @@ struct ColumnSum<int, float> :
v_store(SUM + i, v_add(vx_load(SUM + i), vx_load(Sp + i)));
}
#if !CV_SIMD_SCALABLE && CV_SIMD_WIDTH > 16
for( ; i <= width - v_int32x4::nlanes; i+=v_int32x4::nlanes )
for( ; i <= width - VTraits<v_int32x4>::vlanes(); i+=VTraits<v_int32x4>::vlanes() )
{
v_store(SUM + i, v_add(v_load(SUM + i), v_load(Sp + i)));
}
@@ -1115,7 +1115,7 @@ struct ColumnSum<int, float> :
}
#if !CV_SIMD_SCALABLE && CV_SIMD_WIDTH > 16
v_float32x4 v_scale = v_setall_f32((float)_scale);
for (; i <= width - v_int32x4::nlanes; i += v_int32x4::nlanes)
for (; i <= width - VTraits<v_int32x4>::vlanes(); i += VTraits<v_int32x4>::vlanes())
{
v_int32x4 v_s0 = v_add(v_load(SUM + i), v_load(Sp + i));
v_store(D + i, v_mul(v_cvt_f32(v_s0), v_scale));
@@ -1142,7 +1142,7 @@ struct ColumnSum<int, float> :
v_store(SUM + i, v_sub(v_s0, vx_load(Sm + i)));
}
#if !CV_SIMD_SCALABLE && CV_SIMD_WIDTH > 16
for( ; i <= width-v_int32x4::nlanes; i+=v_int32x4::nlanes )
for( ; i <= width-VTraits<v_int32x4>::vlanes(); i+=VTraits<v_int32x4>::vlanes() )
{
v_int32x4 v_s0 = v_add(v_load(SUM + i), v_load(Sp + i));
v_store(D + i, v_cvt_f32(v_s0));
modules/imgproc/src/color_lab.cpp
+151 -148
View File
@@ -66,7 +66,7 @@ template<typename _Tp> static inline cv::v_float32 splineInterpolate(const cv::v
ix = v_shl<2>(ix);
v_float32 t0, t1, t2, t3;
// assume that v_float32::nlanes == v_int32::nlanes
// assume that VTraits<v_float32>::vlanes() == VTraits<v_int32>::vlanes()
if(VTraits<v_float32>::vlanes() == 4)
{
int32_t CV_DECL_ALIGNED(CV_SIMD_WIDTH) idx[4];
@@ -1388,16 +1388,16 @@ static inline void trilinearPackedInterpolate(const v_uint16x8& inX, const v_uin
v_uint16x8& outA, v_uint16x8& outB, v_uint16x8& outC)
{
//LUT idx of origin pt of cube
v_uint16x8 idxsX = inX >> (lab_base_shift - lab_lut_shift);
v_uint16x8 idxsY = inY >> (lab_base_shift - lab_lut_shift);
v_uint16x8 idxsZ = inZ >> (lab_base_shift - lab_lut_shift);
v_uint16x8 idxsX = v_shr<lab_base_shift - lab_lut_shift>(inX);
v_uint16x8 idxsY = v_shr<lab_base_shift - lab_lut_shift>(inY);
v_uint16x8 idxsZ = v_shr<lab_base_shift - lab_lut_shift>(inZ);
//x, y, z are [0; TRILINEAR_BASE)
const uint16_t bitMask = (1 << trilinear_shift) - 1;
v_uint16x8 bitMaskReg = v_setall_u16(bitMask);
v_uint16x8 fracX = (inX >> (lab_base_shift - 8 - 1)) & bitMaskReg;
v_uint16x8 fracY = (inY >> (lab_base_shift - 8 - 1)) & bitMaskReg;
v_uint16x8 fracZ = (inZ >> (lab_base_shift - 8 - 1)) & bitMaskReg;
v_uint16x8 fracX = v_and(v_shr<lab_base_shift - 8 - 1>(inX), bitMaskReg);
v_uint16x8 fracY = v_and(v_shr<lab_base_shift - 8 - 1>(inY), bitMaskReg);
v_uint16x8 fracZ = v_and(v_shr<lab_base_shift - 8 - 1>(inZ), bitMaskReg);
//load values to interpolate for pix0, pix1, .., pix7
v_int16x8 a0, a1, a2, a3, a4, a5, a6, a7;
@@ -1407,9 +1407,9 @@ static inline void trilinearPackedInterpolate(const v_uint16x8& inX, const v_uin
v_uint32x4 addrDw0, addrDw1, addrDw10, addrDw11;
v_mul_expand(v_setall_u16(3*8), idxsX, addrDw0, addrDw1);
v_mul_expand(v_setall_u16(3*8*LAB_LUT_DIM), idxsY, addrDw10, addrDw11);
addrDw0 += addrDw10; addrDw1 += addrDw11;
addrDw0 = v_add(addrDw0, addrDw10); addrDw1 = v_add(addrDw1, addrDw11);
v_mul_expand(v_setall_u16(3*8*LAB_LUT_DIM*LAB_LUT_DIM), idxsZ, addrDw10, addrDw11);
addrDw0 += addrDw10; addrDw1 += addrDw11;
addrDw0 = v_add(addrDw0, addrDw10); addrDw1 = v_add(addrDw1, addrDw11);
uint32_t CV_DECL_ALIGNED(16) addrofs[8];
v_store_aligned(addrofs, addrDw0);
@@ -1431,9 +1431,9 @@ static inline void trilinearPackedInterpolate(const v_uint16x8& inX, const v_uin
v_int16x8 w0, w1, w2, w3, w4, w5, w6, w7;
v_mul_expand(v_setall_u16(8), fracX, addrDw0, addrDw1);
v_mul_expand(v_setall_u16(8*TRILINEAR_BASE), fracY, addrDw10, addrDw11);
addrDw0 += addrDw10; addrDw1 += addrDw11;
addrDw0 = v_add(addrDw0, addrDw10); addrDw1 = v_add(addrDw1, addrDw11);
v_mul_expand(v_setall_u16(8*TRILINEAR_BASE*TRILINEAR_BASE), fracZ, addrDw10, addrDw11);
addrDw0 += addrDw10; addrDw1 += addrDw11;
addrDw0 = v_add(addrDw0, addrDw10); addrDw1 = v_add(addrDw1, addrDw11);
v_store_aligned(addrofs, addrDw0);
v_store_aligned(addrofs + 4, addrDw1);
@@ -1476,7 +1476,8 @@ static inline void trilinearPackedInterpolate(const v_uint16& inX, const v_uint1
const int16_t* LUT,
v_uint16& outA, v_uint16& outB, v_uint16& outC)
{
const int vsize = VTraits<v_uint16>::max_nlanes;
const int vsize = VTraits<v_uint16>::vlanes();
const int vsize_max = VTraits<v_uint16>::max_nlanes;
// LUT idx of origin pt of cube
v_uint16 tx = v_shr<lab_base_shift - lab_lut_shift>(inX);
@@ -1492,7 +1493,7 @@ static inline void trilinearPackedInterpolate(const v_uint16& inX, const v_uint1
baseIdx0 = v_add(v_add(btmp00, btmp10), btmp20);
baseIdx1 = v_add(v_add(btmp01, btmp11), btmp21);
uint32_t CV_DECL_ALIGNED(CV_SIMD_WIDTH) vbaseIdx[vsize];
uint32_t CV_DECL_ALIGNED(CV_SIMD_WIDTH) vbaseIdx[vsize_max];
v_store_aligned(vbaseIdx + 0*vsize/2, baseIdx0);
v_store_aligned(vbaseIdx + 1*vsize/2, baseIdx1);
@@ -1513,13 +1514,13 @@ static inline void trilinearPackedInterpolate(const v_uint16& inX, const v_uint1
trilinearIdx0 = v_add(v_add(v_shl<3>(fracX0), v_shl<3 + trilinear_shift>(fracY0)), v_shl<3 + trilinear_shift * 2>(fracZ0));
trilinearIdx1 = v_add(v_add(v_shl<3>(fracX1), v_shl<3 + trilinear_shift>(fracY1)), v_shl<3 + trilinear_shift * 2>(fracZ1));
uint32_t CV_DECL_ALIGNED(CV_SIMD_WIDTH) vtrilinearIdx[vsize];
uint32_t CV_DECL_ALIGNED(CV_SIMD_WIDTH) vtrilinearIdx[vsize_max];
v_store_aligned(vtrilinearIdx + 0*vsize/2, trilinearIdx0);
v_store_aligned(vtrilinearIdx + 1*vsize/2, trilinearIdx1);
v_uint32 a0, a1, b0, b1, c0, c1;
uint32_t CV_DECL_ALIGNED(CV_SIMD_WIDTH) va[vsize], vb[vsize], vc[vsize];
uint32_t CV_DECL_ALIGNED(CV_SIMD_WIDTH) va[vsize_max], vb[vsize_max], vc[vsize_max];
for(int j = 0; j < vsize; j++)
{
const int16_t* baseLUT = LUT + vbaseIdx[j];
@@ -1649,11 +1650,11 @@ struct RGB2Lab_b
vL = v_shr<lab_shift2>(vL);
/* int a = CV_DESCALE( 500*(fX - fY) + 128*(1 << lab_shift2), lab_shift2 );*/
va = v_fma(vfX - vfY, v_setall_s32(500), v_setall_s32(abShift+labDescaleShift));
va = v_fma(v_sub(vfX, vfY), v_setall_s32(500), v_setall_s32(abShift+labDescaleShift));
va = v_shr<lab_shift2>(va);
/* int b = CV_DESCALE( 200*(fY - fZ) + 128*(1 << lab_shift2), lab_shift2 );*/
vb = v_fma(vfY - vfZ, v_setall_s32(200), v_setall_s32(abShift+labDescaleShift));
vb = v_fma(v_sub(vfY, vfZ), v_setall_s32(200), v_setall_s32(abShift+labDescaleShift));
vb = v_shr<lab_shift2>(vb);
}
#endif // CV_NEON
@@ -1675,8 +1676,8 @@ struct RGB2Lab_b
#if CV_NEON
// On each loop, we load nlanes of RGB/A v_uint8s and store nlanes of
// Lab v_uint8s
for(; i <= n - v_uint8::nlanes; i += v_uint8::nlanes,
src += scn*v_uint8::nlanes, dst += 3*v_uint8::nlanes )
for(; i <= n - VTraits<v_uint8>::vlanes(); i += VTraits<v_uint8>::vlanes(),
src += scn*VTraits<v_uint8>::vlanes(), dst += 3*VTraits<v_uint8>::vlanes() )
{
// Load 4 batches of 4 src
v_uint8 vRi, vGi, vBi;
@@ -1712,7 +1713,7 @@ struct RGB2Lab_b
#endif // CV_NEON
#if CV_SIMD
const int vsize = v_uint8::nlanes;
const int vsize = VTraits<v_uint8>::vlanes();
const int xyzDescaleShift = 1 << (lab_shift - 1);
v_int16 vXYZdescale = vx_setall_s16(xyzDescaleShift);
v_int16 cxrg, cxb1, cyrg, cyb1, czrg, czb1;
@@ -1752,7 +1753,7 @@ struct RGB2Lab_b
v_expand(drgb[k], qrgb[k*2+0], qrgb[k*2+1]);
}
uint32_t CV_DECL_ALIGNED(CV_SIMD_WIDTH) vdrgb[vsize*3];
uint32_t CV_DECL_ALIGNED(CV_SIMD_WIDTH) vdrgb[VTraits<v_uint8>::max_nlanes*3];
for(int k = 0; k < 12; k++)
{
v_store_aligned(vdrgb + k*vsize/4, qrgb[k]);
@@ -1784,14 +1785,14 @@ struct RGB2Lab_b
v_uint32 x[4], y[4], z[4];
for(int j = 0; j < 4; j++)
{
x[j] = v_reinterpret_as_u32(v_dotprod(rg[j], cxrg) + v_dotprod(bd[j], cxb1)) >> lab_shift;
y[j] = v_reinterpret_as_u32(v_dotprod(rg[j], cyrg) + v_dotprod(bd[j], cyb1)) >> lab_shift;
z[j] = v_reinterpret_as_u32(v_dotprod(rg[j], czrg) + v_dotprod(bd[j], czb1)) >> lab_shift;
x[j] = v_shr<xyz_shift>(v_reinterpret_as_u32(v_add(v_dotprod(rg[j], cxrg), v_dotprod(bd[j], cxb1))));
y[j] = v_shr<xyz_shift>(v_reinterpret_as_u32(v_add(v_dotprod(rg[j], cyrg), v_dotprod(bd[j], cyb1))));
z[j] = v_shr<xyz_shift>(v_reinterpret_as_u32(v_add(v_dotprod(rg[j], czrg), v_dotprod(bd[j], czb1))));
}
// [fX, fY, fZ] = LabCbrtTab_b[vx, vy, vz]
// [4 per X, 4 per Y, 4 per Z]
uint32_t CV_DECL_ALIGNED(CV_SIMD_WIDTH) vxyz[vsize*3];
uint32_t CV_DECL_ALIGNED(CV_SIMD_WIDTH) vxyz[VTraits<v_uint8>::max_nlanes*3];
for(int j = 0; j < 4; j++)
{
v_store_aligned(vxyz + (0*4+j)*vsize/4, x[j]);
@@ -1822,7 +1823,7 @@ struct RGB2Lab_b
v_uint32 vLshift = vx_setall_u32((uint32_t)(Lshift + labDescaleShift));
for(int k = 0; k < 4; k++)
{
vL[k] = (vL[k] + vLshift) >> lab_shift2;
vL[k] = v_shr<lab_shift2>(v_add(vL[k], vLshift));
}
v_uint16 L0, L1;
L0 = v_pack(vL[0], vL[1]);
@@ -1846,7 +1847,7 @@ struct RGB2Lab_b
v_int32 abShift = vx_setall_s32(128*(1 << lab_shift2) + labDescaleShift);
for(int k = 0; k < 8; k++)
{
ab[k] = (ab[k] + abShift) >> lab_shift2;
ab[k] = v_shr<lab_shift2>(v_add(ab[k], abShift));
}
v_int16 a0, a1, b0, b1;
a0 = v_pack(ab[0], ab[1]); a1 = v_pack(ab[2], ab[3]);
@@ -1941,7 +1942,7 @@ struct RGB2Lab_f
#if CV_SIMD
if(enablePackedLab)
{
const int vsize = v_float32::nlanes;
const int vsize = VTraits<v_float32>::vlanes();
static const int nPixels = vsize*2;
for(; i < n - 3*nPixels; i += 3*nPixels, src += scn*nPixels)
{
@@ -1973,8 +1974,8 @@ struct RGB2Lab_f
#undef clipv
/* int iR = R*LAB_BASE, iG = G*LAB_BASE, iB = B*LAB_BASE, iL, ia, ib; */
v_float32 basef = vx_setall_f32(LAB_BASE);
rvec0 *= basef, gvec0 *= basef, bvec0 *= basef;
rvec1 *= basef, gvec1 *= basef, bvec1 *= basef;
rvec0 = v_mul(rvec0, basef), gvec0 = v_mul(gvec0, basef), bvec0 = v_mul(bvec0, basef);
rvec1 = v_mul(rvec1, basef), gvec1 = v_mul(gvec1, basef), bvec1 = v_mul(bvec1, basef);
v_int32 irvec0, igvec0, ibvec0, irvec1, igvec1, ibvec1;
irvec0 = v_round(rvec0); irvec1 = v_round(rvec1);
@@ -2004,8 +2005,8 @@ struct RGB2Lab_f
/* dst[i] = L*100.0f */
v_float32 v100dBase = vx_setall_f32(100.0f/LAB_BASE);
l_vec0 = l_vec0*v100dBase;
l_vec1 = l_vec1*v100dBase;
l_vec0 = v_mul(l_vec0, v100dBase);
l_vec1 = v_mul(l_vec1, v100dBase);
/*
dst[i + 1] = a*256.0f - 128.0f;
dst[i + 2] = b*256.0f - 128.0f;
@@ -2043,8 +2044,8 @@ struct RGB2Lab_f
static const float _a = (softfloat(16) / softfloat(116));
int i = 0;
#if CV_SIMD
const int vsize = v_float32::nlanes;
const int nrepeats = vsize == 4 ? 2 : 1;
const int vsize = VTraits<v_float32>::vlanes();
const int nrepeats = VTraits<v_float32>::nlanes == 4 ? 2 : 1;
v_float32 vc0 = vx_setall_f32(C0), vc1 = vx_setall_f32(C1), vc2 = vx_setall_f32(C2);
v_float32 vc3 = vx_setall_f32(C3), vc4 = vx_setall_f32(C4), vc5 = vx_setall_f32(C5);
v_float32 vc6 = vx_setall_f32(C6), vc7 = vx_setall_f32(C7), vc8 = vx_setall_f32(C8);
@@ -2080,9 +2081,9 @@ struct RGB2Lab_f
v_float32 vgscale = vx_setall_f32(gscale);
for (int k = 0; k < nrepeats; k++)
{
R[k] = splineInterpolate(R[k]*vgscale, gammaTab, GAMMA_TAB_SIZE);
G[k] = splineInterpolate(G[k]*vgscale, gammaTab, GAMMA_TAB_SIZE);
B[k] = splineInterpolate(B[k]*vgscale, gammaTab, GAMMA_TAB_SIZE);
R[k] = splineInterpolate(v_mul(R[k], vgscale), gammaTab, GAMMA_TAB_SIZE);
G[k] = splineInterpolate(v_mul(G[k], vgscale), gammaTab, GAMMA_TAB_SIZE);
B[k] = splineInterpolate(v_mul(B[k], vgscale), gammaTab, GAMMA_TAB_SIZE);
}
}
@@ -2090,26 +2091,26 @@ struct RGB2Lab_f
v_float32 FX[nrepeats], FY[nrepeats], FZ[nrepeats];
for (int k = 0; k < nrepeats; k++)
{
X[k] = v_fma(R[k], vc0, v_fma(G[k], vc1, B[k]*vc2));
Y[k] = v_fma(R[k], vc3, v_fma(G[k], vc4, B[k]*vc5));
Z[k] = v_fma(R[k], vc6, v_fma(G[k], vc7, B[k]*vc8));
X[k] = v_fma(R[k], vc0, v_fma(G[k], vc1, v_mul(B[k], vc2)));
Y[k] = v_fma(R[k], vc3, v_fma(G[k], vc4, v_mul(B[k], vc5)));
Z[k] = v_fma(R[k], vc6, v_fma(G[k], vc7, v_mul(B[k], vc8)));
// use spline interpolation instead of direct calculation
v_float32 vTabScale = vx_setall_f32(LabCbrtTabScale);
FX[k] = splineInterpolate(X[k]*vTabScale, LabCbrtTab, LAB_CBRT_TAB_SIZE);
FY[k] = splineInterpolate(Y[k]*vTabScale, LabCbrtTab, LAB_CBRT_TAB_SIZE);
FZ[k] = splineInterpolate(Z[k]*vTabScale, LabCbrtTab, LAB_CBRT_TAB_SIZE);
FX[k] = splineInterpolate(v_mul(X[k], vTabScale), LabCbrtTab, LAB_CBRT_TAB_SIZE);
FY[k] = splineInterpolate(v_mul(Y[k], vTabScale), LabCbrtTab, LAB_CBRT_TAB_SIZE);
FZ[k] = splineInterpolate(v_mul(Z[k], vTabScale), LabCbrtTab, LAB_CBRT_TAB_SIZE);
}
v_float32 L[nrepeats], a[nrepeats], b[nrepeats];
for (int k = 0; k < nrepeats; k++)
{
// 7.787f = (29/3)^3/(29*4), 0.008856f = (6/29)^3, 903.3 = (29/3)^3
v_float32 mask = Y[k] > (vx_setall_f32(0.008856f));
v_float32 mask = v_gt(Y[k], (vx_setall_f32(0.008856f)));
v_float32 v116 = vx_setall_f32(116.f), vm16 = vx_setall_f32(-16.f);
L[k] = v_select(mask, v_fma(v116, FY[k], vm16), vx_setall_f32(903.3f)*Y[k]);
a[k] = vx_setall_f32(500.f) * (FX[k] - FY[k]);
b[k] = vx_setall_f32(200.f) * (FY[k] - FZ[k]);
L[k] = v_select(mask, v_fma(v116, FY[k], vm16), v_mul(vx_setall_f32(903.3f),Y[k]));
a[k] = v_mul(vx_setall_f32(500.F), v_sub(FX[k], FY[k]));
b[k] = v_mul(vx_setall_f32(200.F), v_sub(FY[k], FZ[k]));
v_store_interleave(dst + k*3*vsize, L[k], a[k], b[k]);
}
@@ -2204,7 +2205,7 @@ struct Lab2RGBfloat
float alpha = ColorChannel<float>::max();
#if CV_SIMD
const int vsize = v_float32::nlanes;
const int vsize = VTraits<v_float32>::vlanes();
const int nrepeats = 2;
v_float32 v16_116 = vx_setall_f32(16.0f / 116.0f);
for( ; i <= n-vsize*nrepeats;
@@ -2221,14 +2222,14 @@ struct Lab2RGBfloat
v_float32 vlThresh = vx_setall_f32(lThresh);
for(int k = 0; k < nrepeats; k++)
{
limask[k] = li[k] <= vlThresh;
limask[k] = v_le(li[k], vlThresh);
}
v_float32 ylo[nrepeats], yhi[nrepeats], fylo[nrepeats], fyhi[nrepeats];
// 903.3 = (29/3)^3, 7.787 = (29/3)^3/(29*4)
v_float32 vinv903 = vx_setall_f32(1.f/903.3f);
for(int k = 0; k < nrepeats; k++)
{
ylo[k] = li[k] * vinv903;
ylo[k] = v_mul(li[k], vinv903);
}
v_float32 v7787 = vx_setall_f32(7.787f);
for(int k = 0; k < nrepeats; k++)
@@ -2238,11 +2239,11 @@ struct Lab2RGBfloat
v_float32 v16 = vx_setall_f32(16.0f), vinv116 = vx_setall_f32(1.f/116.0f);
for(int k = 0; k < nrepeats; k++)
{
fyhi[k] = (li[k] + v16) * vinv116;
fyhi[k] = v_mul(v_add(li[k], v16), vinv116);
}
for(int k = 0; k < nrepeats; k++)
{
yhi[k] = fyhi[k] * fyhi[k] * fyhi[k];
yhi[k] = v_mul(fyhi[k], fyhi[k], fyhi[k]);
}
for(int k = 0; k < nrepeats; k++)
{
@@ -2265,9 +2266,9 @@ struct Lab2RGBfloat
for (int j = 0; j < 2; j++)
{
v_float32 f = fxz[k*2+j];
v_float32 fmask = f <= vfTresh;
v_float32 flo = (f - v16_116) * vinv7787;
v_float32 fhi = f*f*f;
v_float32 fmask = v_le(f, vfTresh);
v_float32 flo = v_mul(v_sub(f, v16_116), vinv7787);
v_float32 fhi = v_mul(v_mul(f, f), f);
fxz[k*2+j] = v_select(fmask, flo, fhi);
}
}
@@ -2281,9 +2282,9 @@ struct Lab2RGBfloat
v_float32 vc6 = vx_setall_f32(C6), vc7 = vx_setall_f32(C7), vc8 = vx_setall_f32(C8);
for(int k = 0; k < nrepeats; k++)
{
ro[k] = v_fma(vc0, x[k], v_fma(vc1, y[k], vc2 * z[k]));
go[k] = v_fma(vc3, x[k], v_fma(vc4, y[k], vc5 * z[k]));
bo[k] = v_fma(vc6, x[k], v_fma(vc7, y[k], vc8 * z[k]));
ro[k] = v_fma(vc0, x[k], v_fma(vc1, y[k], v_mul(vc2, z[k])));
go[k] = v_fma(vc3, x[k], v_fma(vc4, y[k], v_mul(vc5, z[k])));
bo[k] = v_fma(vc6, x[k], v_fma(vc7, y[k], v_mul(vc8, z[k])));
}
v_float32 one = vx_setall_f32(1.f), zero = vx_setzero_f32();
for(int k = 0; k < nrepeats; k++)
@@ -2298,9 +2299,9 @@ struct Lab2RGBfloat
v_float32 vgscale = vx_setall_f32(gscale);
for(int k = 0; k < nrepeats; k++)
{
ro[k] *= vgscale;
go[k] *= vgscale;
bo[k] *= vgscale;
ro[k] = v_mul(ro[k], vgscale);
go[k] = v_mul(go[k], vgscale);
bo[k] = v_mul(bo[k], vgscale);
}
for(int k = 0; k < nrepeats; k++)
@@ -2500,8 +2501,8 @@ struct Lab2RGBinteger
for(int k = 0; k < 4; k++)
{
yf[k] = v_lut((const int*)LabToYF_b, lq[k]);
y[k] = yf[k] & mask16;
ify[k] = v_reinterpret_as_s32(v_reinterpret_as_u32(yf[k]) >> 16);
y[k] = v_and(yf[k], mask16);
ify[k] = v_reinterpret_as_s32(v_shr(v_reinterpret_as_u32(yf[k]), 16));
}
v_int16 ify0, ify1;
@@ -2516,18 +2517,18 @@ struct Lab2RGBinteger
v_uint16 mulA = vx_setall_u16(53687);
v_uint32 ma[4];
v_uint32 addA = vx_setall_u32(1 << 7);
v_mul_expand((a0 + (a0 << 2)), mulA, ma[0], ma[1]);
v_mul_expand((a1 + (a1 << 2)), mulA, ma[2], ma[3]);
adiv0 = v_reinterpret_as_s16(v_pack(((ma[0] + addA) >> 13), ((ma[1] + addA) >> 13)));
adiv1 = v_reinterpret_as_s16(v_pack(((ma[2] + addA) >> 13), ((ma[3] + addA) >> 13)));
v_mul_expand((v_add(a0, v_shl<2>(a0))), mulA, ma[0], ma[1]);
v_mul_expand((v_add(a1, v_shl<2>(a1))), mulA, ma[2], ma[3]);
adiv0 = v_reinterpret_as_s16(v_pack((v_shr<13>(v_add(ma[0], addA))), (v_shr<13>(v_add(ma[1], addA)))));
adiv1 = v_reinterpret_as_s16(v_pack((v_shr<13>(v_add(ma[2], addA))), (v_shr<13>(v_add(ma[3], addA)))));
v_uint16 mulB = vx_setall_u16(41943);
v_uint32 mb[4];
v_uint32 addB = vx_setall_u32(1 << 4);
v_mul_expand(b0, mulB, mb[0], mb[1]);
v_mul_expand(b1, mulB, mb[2], mb[3]);
bdiv0 = v_reinterpret_as_s16(v_pack((mb[0] + addB) >> 9, (mb[1] + addB) >> 9));
bdiv1 = v_reinterpret_as_s16(v_pack((mb[2] + addB) >> 9, (mb[3] + addB) >> 9));
bdiv0 = v_reinterpret_as_s16(v_pack(v_shr<9>(v_add(mb[0], addB)), v_shr<9>(v_add(mb[1], addB))));
bdiv1 = v_reinterpret_as_s16(v_pack(v_shr<9>(v_add(mb[2], addB)), v_shr<9>(v_add(mb[3], addB))));
// 0 <= adiv <= 8356, 0 <= bdiv <= 20890
/* x = ifxz[0]; y = y; z = ifxz[1]; */
@@ -2570,7 +2571,7 @@ struct Lab2RGBinteger
{
bool srgb = issRGB;
ushort* tab = sRGBInvGammaTab_b;
const int vsize = v_uint8::nlanes;
const int vsize = VTraits<v_uint8>::vlanes();
v_uint8 valpha = vx_setall_u8(alpha);
v_int32 vc[9];
for(int k = 0; k < 9; k++)
@@ -2592,9 +2593,9 @@ struct Lab2RGBinteger
v_int32 rq[4], gq[4], bq[4];
for(int k = 0; k < 4; k++)
{
rq[k] = (vc[0] * xq[k] + vc[1] * yq[k] + vc[2] * zq[k] + vdescale) >> shift;
gq[k] = (vc[3] * xq[k] + vc[4] * yq[k] + vc[5] * zq[k] + vdescale) >> shift;
bq[k] = (vc[6] * xq[k] + vc[7] * yq[k] + vc[8] * zq[k] + vdescale) >> shift;
rq[k] = v_shr<shift>(v_add(v_add(v_add(v_mul(vc[0], xq[k]), v_mul(vc[1], yq[k])), v_mul(vc[2], zq[k])), vdescale));
gq[k] = v_shr<shift>(v_add(v_add(v_add(v_mul(vc[3], xq[k]), v_mul(vc[4], yq[k])), v_mul(vc[5], zq[k])), vdescale));
bq[k] = v_shr<shift>(v_add(v_add(v_add(v_mul(vc[6], xq[k]), v_mul(vc[7], yq[k])), v_mul(vc[8], zq[k])), vdescale));
}
//limit indices in table and then substitute
@@ -2611,7 +2612,7 @@ struct Lab2RGBinteger
if(srgb)
{
// [RRR... , GGG... , BBB...]
int32_t CV_DECL_ALIGNED(CV_SIMD_WIDTH) vidx[vsize*3];
int32_t CV_DECL_ALIGNED(CV_SIMD_WIDTH) vidx[VTraits<v_uint8>::max_nlanes*3];
for (int k = 0; k < 4; k++)
v_store_aligned(vidx + 0*vsize + k*vsize/4, rq[k]);
for (int k = 0; k < 4; k++)
@@ -2631,9 +2632,9 @@ struct Lab2RGBinteger
// rgb = (rgb*255) >> inv_gamma_shift
for(int k = 0; k < 4; k++)
{
rq[k] = ((rq[k] << 8) - rq[k]) >> inv_gamma_shift;
gq[k] = ((gq[k] << 8) - gq[k]) >> inv_gamma_shift;
bq[k] = ((bq[k] << 8) - bq[k]) >> inv_gamma_shift;
rq[k] = v_shr((v_sub(v_shl(rq[k], 8), rq[k])), inv_gamma_shift);
gq[k] = v_shr((v_sub(v_shl(gq[k], 8), gq[k])), inv_gamma_shift);
bq[k] = v_shr((v_sub(v_shl(bq[k], 8), bq[k])), inv_gamma_shift);
}
rgb[0] = v_reinterpret_as_u16(v_pack(rq[0], rq[1]));
rgb[1] = v_reinterpret_as_u16(v_pack(rq[2], rq[3]));
@@ -2730,13 +2731,13 @@ struct Lab2RGB_b
static const softfloat fl = softfloat(100)/f255;
#if CV_SIMD
const int fsize = v_float32::nlanes;
const int fsize = VTraits<v_float32>::vlanes();
v_float32 vl = vx_setall_f32((float)fl);
v_float32 va = vx_setall_f32(1.f);
v_float32 vb = vx_setall_f32(1.f);
v_float32 vaLow = vx_setall_f32(-128.f), vbLow = vx_setall_f32(-128.f);
//TODO: fix that when v_interleave is available
float CV_DECL_ALIGNED(CV_SIMD_WIDTH) interTmpM[fsize*3], interTmpA[fsize*3];
float CV_DECL_ALIGNED(CV_SIMD_WIDTH) interTmpM[VTraits<v_float32>::max_nlanes*3], interTmpA[VTraits<v_float32>::max_nlanes*3];
v_store_interleave(interTmpM, vl, va, vb);
v_store_interleave(interTmpA, vx_setzero_f32(), vaLow, vbLow);
v_float32 mluv[3], aluv[3];
@@ -2754,7 +2755,7 @@ struct Lab2RGB_b
j = 0;
#if CV_SIMD
const int vsize = v_uint8::nlanes;
const int vsize = VTraits<v_uint8>::vlanes();
for( ; j <= (dn - vsize)*3; j += 3*vsize )
{
v_uint8 s0, s1, s2;
@@ -2808,7 +2809,7 @@ struct Lab2RGB_b
v_int32 vi[4*3];
for(int k = 0; k < 4*3; k++)
{
vi[k] = v_round(vf[k]*v255);
vi[k] = v_round(v_mul(vf[k], v255));
}
v_uint8 rgb[3];
@@ -2830,7 +2831,7 @@ struct Lab2RGB_b
for(int k = 0; k < 4; k++)
{
vf[k] = vx_load_aligned(buf + j + k*fsize);
vi[k] = v_round(vf[k]*v255);
vi[k] = v_round(v_mul(vf[k], v255));
}
v_store(dst, v_pack_u(v_pack(vi[0], vi[1]), v_pack(vi[2], vi[3])));
}
@@ -2910,8 +2911,8 @@ struct RGB2Luvfloat
C6 = coeffs[6], C7 = coeffs[7], C8 = coeffs[8];
#if CV_SIMD
const int vsize = v_float32::nlanes;
const int nrepeats = vsize == 4 ? 2 : 1;
const int vsize = VTraits<v_float32>::vlanes();
const int nrepeats = VTraits<v_float32>::nlanes == 4 ? 2 : 1;
for( ; i <= n-vsize*nrepeats;
i+= vsize*nrepeats, src += scn*vsize*nrepeats, dst += 3*vsize*nrepeats)
{
@@ -2944,9 +2945,9 @@ struct RGB2Luvfloat
v_float32 vgscale = vx_setall_f32(gscale);
for (int k = 0; k < nrepeats; k++)
{
R[k] *= vgscale;
G[k] *= vgscale;
B[k] *= vgscale;
R[k] = v_mul(R[k], vgscale);
G[k] = v_mul(G[k], vgscale);
B[k] = v_mul(B[k], vgscale);
}
for (int k = 0; k < nrepeats; k++)
@@ -2963,27 +2964,27 @@ struct RGB2Luvfloat
v_float32 vc6 = vx_setall_f32(C6), vc7 = vx_setall_f32(C7), vc8 = vx_setall_f32(C8);
for (int k = 0; k < nrepeats; k++)
{
X[k] = v_fma(R[k], vc0, v_fma(G[k], vc1, B[k]*vc2));
Y[k] = v_fma(R[k], vc3, v_fma(G[k], vc4, B[k]*vc5));
Z[k] = v_fma(R[k], vc6, v_fma(G[k], vc7, B[k]*vc8));
X[k] = v_fma(R[k], vc0, v_fma(G[k], vc1, v_mul(B[k], vc2)));
Y[k] = v_fma(R[k], vc3, v_fma(G[k], vc4, v_mul(B[k], vc5)));
Z[k] = v_fma(R[k], vc6, v_fma(G[k], vc7, v_mul(B[k], vc8)));
}
v_float32 L[nrepeats], u[nrepeats], v[nrepeats];
v_float32 vmun = vx_setall_f32(-un), vmvn = vx_setall_f32(-vn);
for (int k = 0; k < nrepeats; k++)
{
L[k] = splineInterpolate(Y[k]*vx_setall_f32(LabCbrtTabScale), LabCbrtTab, LAB_CBRT_TAB_SIZE);
L[k] = splineInterpolate(v_mul(Y[k], vx_setall_f32(LabCbrtTabScale)), LabCbrtTab, LAB_CBRT_TAB_SIZE);
// L = 116.f*L - 16.f;
L[k] = v_fma(L[k], vx_setall_f32(116.f), vx_setall_f32(-16.f));
v_float32 d;
// d = (4*13) / max(X + 15 * Y + 3 * Z, FLT_EPSILON)
d = v_fma(Y[k], vx_setall_f32(15.f), v_fma(Z[k], vx_setall_f32(3.f), X[k]));
d = vx_setall_f32(4.f*13.f) / v_max(d, vx_setall_f32(FLT_EPSILON));
d = v_div(vx_setall_f32(4.F * 13.F), v_max(d, vx_setall_f32(FLT_EPSILON)));
// u = L*(X*d - un)
u[k] = L[k]*v_fma(X[k], d, vmun);
u[k] = v_mul(L[k], v_fma(X[k], d, vmun));
// v = L*((9*0.25f)*Y*d - vn);
v[k] = L[k]*v_fma(vx_setall_f32(9.f*0.25f)*Y[k], d, vmvn);
v[k] = v_mul(L[k], v_fma(v_mul(vx_setall_f32(9.F * 0.25F), Y[k]), d, vmvn));
}
for (int k = 0; k < nrepeats; k++)
@@ -3099,8 +3100,8 @@ struct Luv2RGBfloat
float _un = un, _vn = vn;
#if CV_SIMD
const int vsize = v_float32::nlanes;
const int nrepeats = vsize == 4 ? 2 : 1;
const int vsize = VTraits<v_float32>::vlanes();
const int nrepeats = VTraits<v_float32>::nlanes == 4 ? 2 : 1;
for( ; i <= n - vsize*nrepeats;
i += vsize*nrepeats, src += vsize*3*nrepeats, dst += dcn*vsize*nrepeats)
{
@@ -3120,13 +3121,13 @@ struct Luv2RGBfloat
v_float32 Ylo, Yhi;
// ((L + 16)/116)^3
Ylo = (L[k] + v16) * v116inv;
Ylo = Ylo*Ylo*Ylo;
Ylo = v_mul(v_add(L[k], v16), v116inv);
Ylo = v_mul(v_mul(Ylo, Ylo), Ylo);
// L*(3./29.)^3
Yhi = L[k] * v903inv;
Yhi = v_mul(L[k], v903inv);
// Y = (L <= 8) ? Y0 : Y1;
Y[k] = v_select(L[k] >= vx_setall_f32(8.f), Ylo, Yhi);
Y[k] = v_select(v_ge(L[k], vx_setall_f32(8.f)), Ylo, Yhi);
}
v_float32 v4inv = vx_setall_f32(0.25f), v3 = vx_setall_f32(3.f);
@@ -3135,18 +3136,18 @@ struct Luv2RGBfloat
v_float32 up, vp;
// up = 3*(u + L*_un);
up = v3*(v_fma(L[k], vx_setall_f32(_un), u[k]));
up = v_mul(v3, v_fma(L[k], vx_setall_f32(_un), u[k]));
// vp = 0.25/(v + L*_vn);
vp = v4inv/(v_fma(L[k], vx_setall_f32(_vn), v[k]));
vp = v_div(v4inv, v_fma(L[k], vx_setall_f32(_vn), v[k]));
// vp = max(-0.25, min(0.25, vp));
vp = v_max(vx_setall_f32(-0.25f), v_min(v4inv, vp));
//X = 3*up*vp; // (*Y) is done later
X[k] = v3*up*vp;
X[k] = v_mul(v_mul(v3, up), vp);
//Z = ((12*13*L - up)*vp - 5); // (*Y) is done later
// xor flips the sign, works like unary minus
Z[k] = v_fma(v_fma(L[k], vx_setall_f32(12.f*13.f), (vx_setall_f32(-0.f) ^ up)), vp, vx_setall_f32(-5.f));
Z[k] = v_fma(v_fma(L[k], vx_setall_f32(12.f*13.f), (v_xor(vx_setall_f32(-0.F), up))), vp, vx_setall_f32(-5.f));
}
v_float32 R[nrepeats], G[nrepeats], B[nrepeats];
@@ -3156,9 +3157,9 @@ struct Luv2RGBfloat
for(int k = 0; k < nrepeats; k++)
{
// R = (X*C0 + C1 + Z*C2)*Y; // here (*Y) is done
R[k] = v_fma(Z[k], vc2, v_fma(X[k], vc0, vc1))*Y[k];
G[k] = v_fma(Z[k], vc5, v_fma(X[k], vc3, vc4))*Y[k];
B[k] = v_fma(Z[k], vc8, v_fma(X[k], vc6, vc7))*Y[k];
R[k] = v_mul(v_fma(Z[k], vc2, v_fma(X[k], vc0, vc1)), Y[k]);
G[k] = v_mul(v_fma(Z[k], vc5, v_fma(X[k], vc3, vc4)), Y[k]);
B[k] = v_mul(v_fma(Z[k], vc8, v_fma(X[k], vc6, vc7)), Y[k]);
}
v_float32 vzero = vx_setzero_f32(), v1 = vx_setall_f32(1.f);
@@ -3174,9 +3175,9 @@ struct Luv2RGBfloat
v_float32 vgscale = vx_setall_f32(gscale);
for(int k = 0; k < nrepeats; k++)
{
R[k] *= vgscale;
G[k] *= vgscale;
B[k] *= vgscale;
R[k] = v_mul(R[k], vgscale);
G[k] = v_mul(G[k], vgscale);
B[k] = v_mul(B[k], vgscale);
}
for(int k = 0; k < nrepeats; k++)
{
@@ -3285,7 +3286,7 @@ struct RGB2Luvinterpolate
#if CV_SIMD
if(enablePackedRGB2Luv)
{
const int vsize = v_uint16::nlanes;
const int vsize = VTraits<v_uint16>::vlanes();
static const int nPixels = vsize*2;
for(; i < n - 3*nPixels; i += 3*nPixels, src += scn*nPixels)
{
@@ -3315,9 +3316,9 @@ struct RGB2Luvinterpolate
v_expand(r, r0, r1);
v_expand(g, g0, g1);
v_expand(b, b0, b1);
r0 = r0 << (lab_base_shift - 8); r1 = r1 << (lab_base_shift - 8);
g0 = g0 << (lab_base_shift - 8); g1 = g1 << (lab_base_shift - 8);
b0 = b0 << (lab_base_shift - 8); b1 = b1 << (lab_base_shift - 8);
r0 = v_shl<lab_base_shift - 8>(r0); r1 = v_shl<lab_base_shift - 8>(r1);
g0 = v_shl<lab_base_shift - 8>(g0); g1 = v_shl<lab_base_shift - 8>(g1);
b0 = v_shl<lab_base_shift - 8>(b0); b1 = v_shl<lab_base_shift - 8>(b1);
/*
int L, u, v;
@@ -3332,9 +3333,9 @@ struct RGB2Luvinterpolate
dst[i+1] = saturate_cast<uchar>(u/baseDiv);
dst[i+2] = saturate_cast<uchar>(v/baseDiv);
*/
l0 = l0 >> (lab_base_shift - 8); l1 = l1 >> (lab_base_shift - 8);
u0 = u0 >> (lab_base_shift - 8); u1 = u1 >> (lab_base_shift - 8);
v0 = v0 >> (lab_base_shift - 8); v1 = v1 >> (lab_base_shift - 8);
l0 = v_shr<lab_base_shift - 8>(l0); l1 = v_shr<lab_base_shift - 8>(l1);
u0 = v_shr<lab_base_shift - 8>(u0); u1 = v_shr<lab_base_shift - 8>(u1);
v0 = v_shr<lab_base_shift - 8>(v0); v1 = v_shr<lab_base_shift - 8>(v1);
v_uint8 l = v_pack(l0, l1);
v_uint8 u = v_pack(u0, u1);
v_uint8 v = v_pack(v0, v1);
@@ -3405,12 +3406,12 @@ struct RGB2Luv_b
static const softfloat su = -uLow*f255/uRange;
static const softfloat sv = -vLow*f255/vRange;
#if CV_SIMD
const int fsize = v_float32::nlanes;
const int fsize = VTraits<v_float32>::vlanes();
v_float32 ml = vx_setall_f32((float)fL), al = vx_setzero_f32();
v_float32 mu = vx_setall_f32((float)fu), au = vx_setall_f32((float)su);
v_float32 mv = vx_setall_f32((float)fv), av = vx_setall_f32((float)sv);
//TODO: fix that when v_interleave is available
float CV_DECL_ALIGNED(CV_SIMD_WIDTH) interTmpM[fsize*3], interTmpA[fsize*3];
float CV_DECL_ALIGNED(CV_SIMD_WIDTH) interTmpM[VTraits<v_float32>::max_nlanes*3], interTmpA[VTraits<v_float32>::max_nlanes*3];
v_store_interleave(interTmpM, ml, mu, mv);
v_store_interleave(interTmpA, al, au, av);
v_float32 mluv[3], aluv[3];
@@ -3452,7 +3453,7 @@ struct RGB2Luv_b
v_float32 f[3*4];
for(int k = 0; k < 3*4; k++)
{
f[k] = v_cvt_f32(q[k])*v255inv;
f[k] = v_mul(v_cvt_f32(q[k]), v255inv);
}
for(int k = 0; k < 4; k++)
@@ -3478,8 +3479,8 @@ struct RGB2Luv_b
v_int32 q0, q1;
v_expand(v_reinterpret_as_s16(d), q0, q1);
v_store_aligned(buf + j + 0*fsize, v_cvt_f32(q0)*v255inv);
v_store_aligned(buf + j + 1*fsize, v_cvt_f32(q1)*v255inv);
v_store_aligned(buf + j + 0*fsize, v_mul(v_cvt_f32(q0), v255inv));
v_store_aligned(buf + j + 1*fsize, v_mul(v_cvt_f32(q1), v255inv));
}
for( ; j < dn*bufChannels; j++, src++ )
{
@@ -3633,7 +3634,8 @@ struct Luv2RGBinteger
inline void processLuvToXYZ(const v_uint8& lv, const v_uint8& uv, const v_uint8& vv,
v_int32 (&x)[4], v_int32 (&y)[4], v_int32 (&z)[4]) const
{
const int vsize = v_uint8::nlanes;
const int vsize = VTraits<v_uint8>::vlanes();
const int vsize_max = VTraits<v_uint8>::max_nlanes;
v_uint16 lv0, lv1;
v_expand(lv, lv0, lv1);
@@ -3646,7 +3648,7 @@ struct Luv2RGBinteger
v_int32 mask16 = vx_setall_s32(0xFFFF);
for(int k = 0; k < 4; k++)
{
y[k] = v_lut((const int*)LabToYF_b, v_reinterpret_as_s32(lq[k])) & mask16;
y[k] = v_and(v_lut((const int *)LabToYF_b, v_reinterpret_as_s32(lq[k])), mask16);
}
v_int32 up[4], vp[4];
@@ -3657,10 +3659,10 @@ struct Luv2RGBinteger
v_expand(vv, vv0, vv1);
// LL*256
v_uint16 ll0, ll1;
ll0 = lv0 << 8; ll1 = lv1 << 8;
ll0 = v_shl<8>(lv0); ll1 = v_shl<8>(lv1);
v_uint16 upidx0, upidx1, vpidx0, vpidx1;
upidx0 = ll0 + uv0; upidx1 = ll1 + uv1;
vpidx0 = ll0 + vv0; vpidx1 = ll1 + vv1;
upidx0 = v_add(ll0, uv0); upidx1 = v_add(ll1, uv1);
vpidx0 = v_add(ll0, vv0); vpidx1 = v_add(ll1, vv1);
v_uint32 upidx[4], vpidx[4];
v_expand(upidx0, upidx[0], upidx[1]); v_expand(upidx1, upidx[2], upidx[3]);
v_expand(vpidx0, vpidx[0], vpidx[1]); v_expand(vpidx1, vpidx[2], vpidx[3]);
@@ -3672,7 +3674,7 @@ struct Luv2RGBinteger
// long long int vpl = LUVLUT.LvToVpl_b[LL*256+v];
v_int64 vpl[8];
int32_t CV_DECL_ALIGNED(CV_SIMD_WIDTH) vpidxstore[vsize];
int32_t CV_DECL_ALIGNED(CV_SIMD_WIDTH) vpidxstore[vsize_max];
for(int k = 0; k < 4; k++)
{
v_store_aligned(vpidxstore + k*vsize/4, v_reinterpret_as_s32(vpidx[k]));
@@ -3684,12 +3686,13 @@ struct Luv2RGBinteger
// not all 64-bit arithmetic is available in univ. intrinsics
// need to handle it with scalar code
int64_t CV_DECL_ALIGNED(CV_SIMD_WIDTH) vvpl[vsize];
int64_t CV_DECL_ALIGNED(CV_SIMD_WIDTH) vvpl[vsize_max];
for(int k = 0; k < 8; k++)
{
v_store_aligned(vvpl + k*vsize/8, vpl[k]);
}
int32_t CV_DECL_ALIGNED(CV_SIMD_WIDTH) vup[vsize], vvp[vsize], vx[vsize], vy[vsize], vzm[vsize];
int32_t CV_DECL_ALIGNED(CV_SIMD_WIDTH) vup[vsize_max], vvp[vsize_max],
vx[vsize_max], vy[vsize_max], vzm[vsize_max];
for(int k = 0; k < 4; k++)
{
v_store_aligned(vup + k*vsize/4, up[k]);
@@ -3724,7 +3727,7 @@ struct Luv2RGBinteger
// z = zm/256 + zm/65536;
for (int k = 0; k < 4; k++)
{
z[k] = (zm[k] >> 8) + (zm[k] >> 16);
z[k] = v_add(v_shr<8>(zm[k]), v_shr<16>(zm[k]));
}
// (x, z) = clip((x, z), min=0, max=2*BASE)
@@ -3751,7 +3754,7 @@ struct Luv2RGBinteger
{
ushort* tab = sRGBInvGammaTab_b;
bool srgb = issRGB;
static const int vsize = v_uint8::nlanes;
static const int vsize = VTraits<v_uint8>::vlanes();
const int descaleShift = 1 << (shift-1);
v_int16 vdescale = vx_setall_s16(descaleShift);
v_int16 vc[9];
@@ -3771,12 +3774,12 @@ struct Luv2RGBinteger
// fixing 16bit signed multiplication
// by subtracting 2^(base_shift-1) and then adding result back
v_int32 dummy32, fm[3];
v_expand(vc[0]+vc[1]+vc[2], fm[0], dummy32);
v_expand(vc[3]+vc[4]+vc[5], fm[1], dummy32);
v_expand(vc[6]+vc[7]+vc[8], fm[2], dummy32);
fm[0] = fm[0] << (base_shift-1);
fm[1] = fm[1] << (base_shift-1);
fm[2] = fm[2] << (base_shift-1);
v_expand(v_add(vc[0],vc[1],vc[2]), fm[0], dummy32);
v_expand(v_add(vc[3],vc[4],vc[5]), fm[1], dummy32);
v_expand(v_add(vc[6],vc[7],vc[8]), fm[2], dummy32);
fm[0] = v_shl(fm[0], (base_shift-1));
fm[1] = v_shl(fm[1], (base_shift-1));
fm[2] = v_shl(fm[2], (base_shift-1));
for (; i <= n-vsize; i += vsize, src += 3*vsize, dst += dcn*vsize)
{
@@ -3816,15 +3819,15 @@ struct Luv2RGBinteger
// a bit faster than one loop for all
for(int k = 0; k < 4; k++)
{
i_rgb[k+4*0] = (v_dotprod(xy[k], crxy) + v_dotprod(zd[k], crz1) + fm[0]) >> shift;
i_rgb[k+4*0] = v_shr<shift>(v_add(v_add(v_dotprod(xy[k], crxy), v_dotprod(zd[k], crz1)), fm[0]));
}
for(int k = 0; k < 4; k++)
{
i_rgb[k+4*1] = (v_dotprod(xy[k], cgxy) + v_dotprod(zd[k], cgz1) + fm[1]) >> shift;
i_rgb[k+4*1] = v_shr<shift>(v_add(v_add(v_dotprod(xy[k], cgxy), v_dotprod(zd[k], cgz1)), fm[1]));
}
for(int k = 0; k < 4; k++)
{
i_rgb[k+4*2] = (v_dotprod(xy[k], cbxy) + v_dotprod(zd[k], cbz1) + fm[2]) >> shift;
i_rgb[k+4*2] = v_shr<shift>(v_add(v_add(v_dotprod(xy[k], cbxy), v_dotprod(zd[k], cbz1)), fm[2]));
}
// [rrggbb]
@@ -3842,7 +3845,7 @@ struct Luv2RGBinteger
if(srgb)
{
// [rr.., gg.., bb..]
int32_t CV_DECL_ALIGNED(CV_SIMD_WIDTH) rgbshifts[3*vsize];
int32_t CV_DECL_ALIGNED(CV_SIMD_WIDTH) rgbshifts[3*VTraits<v_uint8>::max_nlanes];
for(int k = 0; k < 12; k++)
{
v_store_aligned(rgbshifts + k*vsize/4, i_rgb[k]);
@@ -3857,7 +3860,7 @@ struct Luv2RGBinteger
// rgb = (rgb*255) >> inv_gamma_shift
for(int k = 0; k < 12; k++)
{
i_rgb[k] = ((i_rgb[k] << 8) - i_rgb[k]) >> inv_gamma_shift;
i_rgb[k] = v_shr((v_sub((v_shl(i_rgb[k], 8)), i_rgb[k])), inv_gamma_shift);
}
for(int k = 0; k < 6; k++)
@@ -3940,13 +3943,13 @@ struct Luv2RGB_b
static const softfloat fv = vRange/f255;
#if CV_SIMD
const int fsize = v_float32::nlanes;
const int fsize = VTraits<v_float32>::vlanes();
v_float32 vl = vx_setall_f32((float)fl);
v_float32 vu = vx_setall_f32((float)fu);
v_float32 vv = vx_setall_f32((float)fv);
v_float32 vuLow = vx_setall_f32((float)uLow), vvLow = vx_setall_f32((float)vLow);
//TODO: fix that when v_interleave is available
float CV_DECL_ALIGNED(CV_SIMD_WIDTH) interTmpM[fsize*3], interTmpA[fsize*3];
float CV_DECL_ALIGNED(CV_SIMD_WIDTH) interTmpM[VTraits<v_float32>::max_nlanes*3], interTmpA[VTraits<v_float32>::max_nlanes*3];
v_store_interleave(interTmpM, vl, vu, vv);
v_store_interleave(interTmpA, vx_setzero_f32(), vuLow, vvLow);
v_float32 mluv[3], aluv[3];
@@ -3964,7 +3967,7 @@ struct Luv2RGB_b
j = 0;
#if CV_SIMD
const int vsize = v_uint8::nlanes;
const int vsize = VTraits<v_uint8>::vlanes();
for( ; j <= (dn - vsize)*3; j += 3*vsize )
{
v_uint8 s0, s1, s2;
@@ -4017,7 +4020,7 @@ struct Luv2RGB_b
v_int32 vi[4*3];
for(int k = 0; k < 4*3; k++)
{
vi[k] = v_round(vf[k]*v255);
vi[k] = v_round(v_mul(vf[k], v255));
}
v_uint8 rgb[3];
@@ -4039,7 +4042,7 @@ struct Luv2RGB_b
for(int k = 0; k < 4; k++)
{
vf[k] = vx_load_aligned(buf + j + k*fsize);
vi[k] = v_round(vf[k]*v255);
vi[k] = v_round(v_mul(vf[k], v255));
}
v_store(dst, v_pack_u(v_pack(vi[0], vi[1]), v_pack(vi[2], vi[3])));
}
modules/imgproc/src/color_rgb.simd.hpp
+6 -6
View File
@@ -882,7 +882,7 @@ struct RGBA2mRGBA<uchar>
int i = 0;
#if CV_SIMD
const int vsize = v_uint8::nlanes;
const int vsize = VTraits<v_uint8>::vlanes();
v_uint8 amask = v_reinterpret_as_u8(vx_setall_u32(0xFF000000));
v_uint16 vh = vx_setall_u16(half_val+1);
@@ -901,27 +901,27 @@ struct RGBA2mRGBA<uchar>
v_uint16 a16[4];
for(int j = 0; j < 4; j++)
a16[j] = v_reinterpret_as_u16(v[j] & amask);
a16[j] = v_reinterpret_as_u16(v_and(v[j], amask));
v_uint32 a32[4];
for(int j = 0; j < 4; j++)
a32[j] = v_reinterpret_as_u32(a16[j] | (a16[j] >> 8));
a32[j] = v_reinterpret_as_u32(v_or(a16[j], (v_shr(a16[j], 8))));
v_uint8 a[4];
for(int j = 0; j < 4; j++)
a[j] = v_reinterpret_as_u8(a32[j] | (a32[j] >> 16));
a[j] = v_reinterpret_as_u8(v_or(a32[j], (v_shr(a32[j], 16))));
v_uint16 m[8];
for(int j = 0; j < 4; j++)
v_mul_expand(v[j], a[j], m[j], m[j+4]);
for(int j = 0; j < 8; j++)
m[j] += vh;
m[j] = v_add(m[j], vh);
// div 255: (v+1+(v>>8))>8
// +1 is in vh, has no effect on (v>>8)
for(int j = 0; j < 8; j++)
m[j] = (m[j] + (m[j] >> 8)) >> 8;
m[j] = v_shr((v_add(m[j], (v_shr(m[j], 8)))), 8);
v_uint8 d[4];
for(int j = 0; j < 4; j++)
modules/imgproc/src/demosaicing.cpp
+148 -146
View File
@@ -188,21 +188,21 @@ public:
v_uint16x8 r1 = v_reinterpret_as_u16(v_load(bayer+bayer_step));
v_uint16x8 r2 = v_reinterpret_as_u16(v_load(bayer+bayer_step*2));
v_uint16x8 b1 = ((r0 << 8) >> 7) + ((r2 << 8) >> 7);
v_uint16x8 b0 = v_rotate_right<1>(b1) + b1;
b1 = v_rotate_right<1>(b1) << 1;
v_uint16x8 b1 = v_add(v_shr<7>(v_shl<8>(r0)), v_shr<7>(v_shl<8>(r2)));
v_uint16x8 b0 = v_add(v_rotate_right<1>(b1), b1);
b1 = v_shl<1>(v_rotate_right<1>(b1));
v_uint16x8 g0 = (r0 >> 7) + (r2 >> 7);
v_uint16x8 g1 = (r1 << 8) >> 7;
g0 += v_rotate_right<1>(g1) + g1;
g1 = v_rotate_right<1>(g1) << 2;
v_uint16x8 g0 = v_add(v_shr<7>(r0), v_shr<7>(r2));
v_uint16x8 g1 = v_shr<7>(v_shl<8>(r1));
g0 = v_add(g0, v_add(v_rotate_right<1>(g1), g1));
g1 = v_shl<2>(v_rotate_right<1>(g1));
r0 = r1 >> 8;
r1 = (v_rotate_right<1>(r0) + r0) << 2;
r0 = r0 << 3;
r0 = v_shr<8>(r1);
r1 = v_shl<2>(v_add(v_rotate_right<1>(r0), r0));
r0 = v_shl<3>(r0);
g0 = (v_mul_hi(b0, _b2y) + v_mul_hi(g0, _g2y) + v_mul_hi(r0, _r2y)) >> 2;
g1 = (v_mul_hi(b1, _b2y) + v_mul_hi(g1, _g2y) + v_mul_hi(r1, _r2y)) >> 2;
g0 = v_shr<2>(v_add(v_add(v_mul_hi(b0, _b2y), v_mul_hi(g0, _g2y)), v_mul_hi(r0, _r2y)));
g1 = v_shr<2>(v_add(v_add(v_mul_hi(b1, _b2y), v_mul_hi(g1, _g2y)), v_mul_hi(r1, _r2y)));
v_uint8x16 pack_lo, pack_hi;
v_zip(v_pack_u(v_reinterpret_as_s16(g0), v_reinterpret_as_s16(g0)),
v_pack_u(v_reinterpret_as_s16(g1), v_reinterpret_as_s16(g1)),
@@ -269,31 +269,31 @@ public:
v_uint16x8 r1 = v_reinterpret_as_u16(v_load(bayer+bayer_step));
v_uint16x8 r2 = v_reinterpret_as_u16(v_load(bayer+bayer_step*2));
v_uint16x8 b1 = (r0 & masklo) + (r2 & masklo);
v_uint16x8 b1 = v_add(v_and(r0, masklo), v_and(r2, masklo));
v_uint16x8 nextb1 = v_rotate_right<1>(b1);
v_uint16x8 b0 = b1 + nextb1;
b1 = (nextb1 + delta1) >> 1;
b0 = (b0 + delta2) >> 2;
v_uint16x8 b0 = v_add(b1, nextb1);
b1 = v_shr<1>(v_add(nextb1, delta1));
b0 = v_shr<2>(v_add(b0, delta2));
// b0 b2 ... b14 b1 b3 ... b15
b0 = v_reinterpret_as_u16(v_pack_u(v_reinterpret_as_s16(b0), v_reinterpret_as_s16(b1)));
v_uint16x8 g0 = (r0 >> 8) + (r2 >> 8);
v_uint16x8 g1 = r1 & masklo;
g0 += v_rotate_right<1>(g1) + g1;
v_uint16x8 g0 = v_add(v_shr<8>(r0), v_shr<8>(r2));
v_uint16x8 g1 = v_and(r1, masklo);
g0 = v_add(g0, v_add(v_rotate_right<1>(g1), g1));
g1 = v_rotate_right<1>(g1);
g0 = (g0 + delta2) >> 2;
g0 = v_shr<2>(v_add(g0, delta2));
// g0 g2 ... g14 g1 g3 ... g15
g0 = v_reinterpret_as_u16(v_pack_u(v_reinterpret_as_s16(g0), v_reinterpret_as_s16(g1)));
r0 = r1 >> 8;
r1 = v_rotate_right<1>(r0) + r0;
r1 = (r1 + delta1) >> 1;
r0 = v_shr<8>(r1);
r1 = v_add(v_rotate_right<1>(r0), r0);
r1 = v_shr<1>(v_add(r1, delta1));
// r0 r2 ... r14 r1 r3 ... r15
r0 = v_reinterpret_as_u16(v_pack_u(v_reinterpret_as_s16(r0), v_reinterpret_as_s16(r1)));
b1 = (b0 ^ r0) & mask;
b0 = b0 ^ b1;
r0 = r0 ^ b1;
b1 = v_and(v_xor(b0, r0), mask);
b0 = v_xor(b0, b1);
r0 = v_xor(r0, b1);
// b1 g1 b3 g3 b5 g5...
v_uint8x16 pack_lo, pack_hi;
@@ -402,31 +402,31 @@ public:
v_uint16x8 r1 = v_reinterpret_as_u16(v_load(bayer+bayer_step));
v_uint16x8 r2 = v_reinterpret_as_u16(v_load(bayer+bayer_step*2));
v_uint16x8 b1 = (r0 & masklo) + (r2 & masklo);
v_uint16x8 b1 = v_add(v_and(r0, masklo), v_and(r2, masklo));
v_uint16x8 nextb1 = v_rotate_right<1>(b1);
v_uint16x8 b0 = b1 + nextb1;
b1 = (nextb1 + delta1) >> 1;
b0 = (b0 + delta2) >> 2;
v_uint16x8 b0 = v_add(b1, nextb1);
b1 = v_shr<1>(v_add(nextb1, delta1));
b0 = v_shr<2>(v_add(b0, delta2));
// b0 b2 ... b14 b1 b3 ... b15
b0 = v_reinterpret_as_u16(v_pack_u(v_reinterpret_as_s16(b0), v_reinterpret_as_s16(b1)));
v_uint16x8 g0 = (r0 >> 8) + (r2 >> 8);
v_uint16x8 g1 = r1 & masklo;
g0 += v_rotate_right<1>(g1) + g1;
v_uint16x8 g0 = v_add(v_shr<8>(r0), v_shr<8>(r2));
v_uint16x8 g1 = v_and(r1, masklo);
g0 = v_add(g0, v_add(v_rotate_right<1>(g1), g1));
g1 = v_rotate_right<1>(g1);
g0 = (g0 + delta2) >> 2;
g0 = v_shr<2>(v_add(g0, delta2));
// g0 g2 ... g14 g1 g3 ... g15
g0 = v_reinterpret_as_u16(v_pack_u(v_reinterpret_as_s16(g0), v_reinterpret_as_s16(g1)));
r0 = r1 >> 8;
r1 = v_rotate_right<1>(r0) + r0;
r1 = (r1 + delta1) >> 1;
r0 = v_shr<8>(r1);
r1 = v_add(v_rotate_right<1>(r0), r0);
r1 = v_shr<1>(v_add(r1, delta1));
// r0 r2 ... r14 r1 r3 ... r15
r0 = v_reinterpret_as_u16(v_pack_u(v_reinterpret_as_s16(r0), v_reinterpret_as_s16(r1)));
b1 = (b0 ^ r0) & mask;
b0 = b0 ^ b1;
r0 = r0 ^ b1;
b1 = v_and(v_xor(b0, r0), mask);
b0 = v_xor(b0, b1);
r0 = v_xor(r0, b1);
// b1 g1 b3 g3 b5 g5...
v_uint8x16 pack_lo, pack_hi;
@@ -498,40 +498,40 @@ public:
v_uint16x8 r1 = v_reinterpret_as_u16(v_load(bayer+bayer_step));
v_uint16x8 r2 = v_reinterpret_as_u16(v_load(bayer+bayer_step*2));
v_uint16x8 b1 = (r0 & masklow) + (r2 & masklow);
v_uint16x8 b1 = v_add(v_and(r0, masklow), v_and(r2, masklow));
v_uint16x8 nextb1 = v_rotate_right<1>(b1);
v_uint16x8 b0 = b1 + nextb1;
b1 = (nextb1 + delta1) >> 1;
b0 = (b0 + delta2) >> 2;
v_uint16x8 b0 = v_add(b1, nextb1);
b1 = v_shr<1>(v_add(nextb1, delta1));
b0 = v_shr<2>(v_add(b0, delta2));
// b0 b2 ... b14 b1 b3 ... b15
b0 = v_reinterpret_as_u16(v_pack_u(v_reinterpret_as_s16(b0), v_reinterpret_as_s16(b1)));
// vertical sum
v_uint16x8 r0g = r0 >> 8;
v_uint16x8 r2g = r2 >> 8;
v_uint16x8 sumv = ((r0g + r2g) + delta1) >> 1;
v_uint16x8 r0g = v_shr<8>(r0);
v_uint16x8 r2g = v_shr<8>(r2);
v_uint16x8 sumv = v_shr<1>(v_add(v_add(r0g, r2g), delta1));
// horizontal sum
v_uint16x8 g1 = r1 & masklow;
v_uint16x8 g1 = v_and(r1, masklow);
v_uint16x8 nextg1 = v_rotate_right<1>(g1);
v_uint16x8 sumg = (g1 + nextg1 + delta1) >> 1;
v_uint16x8 sumg = v_shr<1>(v_add(v_add(g1, nextg1), delta1));
// gradients
v_uint16x8 gradv = (r0g - r2g) + (r2g - r0g);
v_uint16x8 gradg = (nextg1 - g1) + (g1 - nextg1);
v_uint16x8 gmask = gradg > gradv;
v_uint16x8 g0 = (gmask & sumv) + (sumg & (gmask ^ full));
v_uint16x8 gradv = v_add(v_sub(r0g, r2g), v_sub(r2g, r0g));
v_uint16x8 gradg = v_add(v_sub(nextg1, g1), v_sub(g1, nextg1));
v_uint16x8 gmask = v_gt(gradg, gradv);
v_uint16x8 g0 = v_add(v_and(gmask, sumv), v_and(sumg, v_xor(gmask, full)));
// g0 g2 ... g14 g1 g3 ...
g0 = v_reinterpret_as_u16(v_pack_u(v_reinterpret_as_s16(g0), v_reinterpret_as_s16(nextg1)));
r0 = r1 >> 8;
r1 = v_rotate_right<1>(r0) + r0;
r1 = (r1 + delta1) >> 1;
r0 = v_shr<8>(r1);
r1 = v_add(v_rotate_right<1>(r0), r0);
r1 = v_shr<1>(v_add(r1, delta1));
// r0 r2 ... r14 r1 r3 ... r15
r0 = v_reinterpret_as_u16(v_pack_u(v_reinterpret_as_s16(r0), v_reinterpret_as_s16(r1)));
b1 = (b0 ^ r0) & mask;
b0 = b0 ^ b1;
r0 = r0 ^ b1;
b1 = v_and(v_xor(b0, r0), mask);
b0 = v_xor(b0, b1);
r0 = v_xor(r0, b1);
// b1 g1 b3 g3 b5 g5...
v_uint8x16 pack_lo, pack_hi;
@@ -1060,19 +1060,19 @@ static void Bayer2RGB_VNG_8u( const Mat& srcmat, Mat& dstmat, int code )
v_uint16x8 b0, b1, b2, b3, b4, b5, b6;
b0 = (v_absdiff(s2, s8)<<1) + v_absdiff(s1, s7) + v_absdiff(s3, s9);
b1 = (v_absdiff(s4, s6)<<1) + v_absdiff(s1, s3) + v_absdiff(s7, s9);
b2 = v_absdiff(s3, s7)<<1;
b3 = v_absdiff(s1, s9)<<1;
b0 = v_add(v_add(v_shl<1>(v_absdiff(s2, s8)), v_absdiff(s1, s7)), v_absdiff(s3, s9));
b1 = v_add(v_add(v_shl<1>(v_absdiff(s4, s6)), v_absdiff(s1, s3)), v_absdiff(s7, s9));
b2 = v_shl<1>(v_absdiff(s3, s7));
b3 = v_shl<1>(v_absdiff(s1, s9));
v_store(brow, b0);
v_store(brow + N, b1);
v_store(brow + N2, b2);
v_store(brow + N3, b3);
b4 = b2 + v_absdiff(s2, s4) + v_absdiff(s6, s8);
b5 = b3 + v_absdiff(s2, s6) + v_absdiff(s4, s8);
b6 = (s2 + s4 + s6 + s8)>>1;
b4 = v_add(v_add(b2, v_absdiff(s2, s4)), v_absdiff(s6, s8));
b5 = v_add(v_add(b3, v_absdiff(s2, s6)), v_absdiff(s4, s8));
b6 = v_shr<1>(v_add(v_add(v_add(s2, s4), s6), s8));
v_store(brow + N4, b4);
v_store(brow + N5, b5);
@@ -1279,7 +1279,7 @@ static void Bayer2RGB_VNG_8u( const Mat& srcmat, Mat& dstmat, int code )
v_uint16x8 one = v_setall_u16(1), z = v_setzero_u16();
v_float32x4 _0_5 = v_setall_f32(0.5f);
#define v_merge_u16(a, b) (((a) & v_reinterpret_as_u16(emask)) | ((b) & v_reinterpret_as_u16(omask))) //(aA_aA_aA_aA) * (bB_bB_bB_bB) => (bA_bA_bA_bA)
#define v_merge_u16(a, b) (v_or((v_and((a), v_reinterpret_as_u16(emask))), (v_and((b), v_reinterpret_as_u16(omask))))) //(aA_aA_aA_aA) * (bB_bB_bB_bB) => (bA_bA_bA_bA)
#define v_cvt_s16f32_lo(a) v_cvt_f32(v_expand_low(v_reinterpret_as_s16(a))) //(1,2,3,4,5,6,7,8) => (1f,2f,3f,4f)
#define v_cvt_s16f32_hi(a) v_cvt_f32(v_expand_high(v_reinterpret_as_s16(a))) //(1,2,3,4,5,6,7,8) => (5f,6f,7f,8f)
@@ -1287,16 +1287,16 @@ static void Bayer2RGB_VNG_8u( const Mat& srcmat, Mat& dstmat, int code )
for( ; i <= N - 10; i += 8, srow += 8, brow0 += 8, brow1 += 8, brow2 += 8 )
{
//int gradN = brow0[0] + brow1[0];
v_uint16x8 gradN = v_load(brow0) + v_load(brow1);
v_uint16x8 gradN = v_add(v_load(brow0), v_load(brow1));
//int gradS = brow1[0] + brow2[0];
v_uint16x8 gradS = v_load(brow1) + v_load(brow2);
v_uint16x8 gradS = v_add(v_load(brow1), v_load(brow2));
//int gradW = brow1[N-1] + brow1[N];
v_uint16x8 gradW = v_load(brow1+N-1) + v_load(brow1+N);
v_uint16x8 gradW = v_add(v_load(brow1 + N - 1), v_load(brow1 + N));
//int gradE = brow1[N+1] + brow1[N];
v_uint16x8 gradE = v_load(brow1+N+1) + v_load(brow1+N);
v_uint16x8 gradE = v_add(v_load(brow1 + N + 1), v_load(brow1 + N));
//int minGrad = std::min(std::min(std::min(gradN, gradS), gradW), gradE);
//int maxGrad = std::max(std::max(std::max(gradN, gradS), gradW), gradE);
@@ -1307,14 +1307,14 @@ static void Bayer2RGB_VNG_8u( const Mat& srcmat, Mat& dstmat, int code )
//int gradNE = brow0[N4+1] + brow1[N4];
//int gradNE = brow0[N2] + brow0[N2+1] + brow1[N2] + brow1[N2+1];
grad0 = v_load(brow0+N4+1) + v_load(brow1+N4);
grad1 = v_load(brow0+N2) + v_load(brow0+N2+1) + v_load(brow1+N2) + v_load(brow1+N2+1);
grad0 = v_add(v_load(brow0 + N4 + 1), v_load(brow1 + N4));
grad1 = v_add(v_add(v_add(v_load(brow0 + N2), v_load(brow0 + N2 + 1)), v_load(brow1 + N2)), v_load(brow1 + N2 + 1));
v_uint16x8 gradNE = v_merge_u16(grad0, grad1);
//int gradSW = brow1[N4] + brow2[N4-1];
//int gradSW = brow1[N2] + brow1[N2-1] + brow2[N2] + brow2[N2-1];
grad0 = v_load(brow2+N4-1) + v_load(brow1+N4);
grad1 = v_load(brow2+N2) + v_load(brow2+N2-1) + v_load(brow1+N2) + v_load(brow1+N2-1);
grad0 = v_add(v_load(brow2 + N4 - 1), v_load(brow1 + N4));
grad1 = v_add(v_add(v_add(v_load(brow2 + N2), v_load(brow2 + N2 - 1)), v_load(brow1 + N2)), v_load(brow1 + N2 - 1));
v_uint16x8 gradSW = v_merge_u16(grad0, grad1);
minGrad = v_min(v_min(minGrad, gradNE), gradSW);
@@ -1322,21 +1322,21 @@ static void Bayer2RGB_VNG_8u( const Mat& srcmat, Mat& dstmat, int code )
//int gradNW = brow0[N5-1] + brow1[N5];
//int gradNW = brow0[N3] + brow0[N3-1] + brow1[N3] + brow1[N3-1];
grad0 = v_load(brow0+N5-1) + v_load(brow1+N5);
grad1 = v_load(brow0+N3) + v_load(brow0+N3-1) + v_load(brow1+N3) + v_load(brow1+N3-1);
grad0 = v_add(v_load(brow0 + N5 - 1), v_load(brow1 + N5));
grad1 = v_add(v_add(v_add(v_load(brow0 + N3), v_load(brow0 + N3 - 1)), v_load(brow1 + N3)), v_load(brow1 + N3 - 1));
v_uint16x8 gradNW = v_merge_u16(grad0, grad1);
//int gradSE = brow1[N5] + brow2[N5+1];
//int gradSE = brow1[N3] + brow1[N3+1] + brow2[N3] + brow2[N3+1];
grad0 = v_load(brow2+N5+1) + v_load(brow1+N5);
grad1 = v_load(brow2+N3) + v_load(brow2+N3+1) + v_load(brow1+N3) + v_load(brow1+N3+1);
grad0 = v_add(v_load(brow2 + N5 + 1), v_load(brow1 + N5));
grad1 = v_add(v_add(v_add(v_load(brow2 + N3), v_load(brow2 + N3 + 1)), v_load(brow1 + N3)), v_load(brow1 + N3 + 1));
v_uint16x8 gradSE = v_merge_u16(grad0, grad1);
minGrad = v_min(v_min(minGrad, gradNW), gradSE);
maxGrad = v_max(v_max(maxGrad, gradNW), gradSE);
//int T = minGrad + maxGrad/2;
v_uint16x8 T = v_max((maxGrad >> 1), one) + minGrad;
v_uint16x8 T = v_add(v_max((v_shr<1>(maxGrad)), one), minGrad);
v_uint16x8 RGs = z, GRs = z, Bs = z, ng = z;
@@ -1361,133 +1361,135 @@ static void Bayer2RGB_VNG_8u( const Mat& srcmat, Mat& dstmat, int code )
v_uint16x8 t0, t1, mask;
// gradN ***********************************************
mask = (T > gradN); // mask = T>gradN
ng = v_reinterpret_as_u16(v_reinterpret_as_s16(ng) - v_reinterpret_as_s16(mask)); // ng += (T>gradN)
mask = (v_gt(T, gradN)); // mask = T>gradN
ng = v_reinterpret_as_u16(v_sub(v_reinterpret_as_s16(ng), v_reinterpret_as_s16(mask))); // ng += (T>gradN)
t0 = (x3 << 1); // srow[-bstep]*2
t1 = v_load_expand(srow - bstep*2) + x0; // srow[-bstep*2] + srow[0]
t0 = (v_shl<1>(x3)); // srow[-bstep]*2
t1 = v_add(v_load_expand(srow - bstep * 2), x0); // srow[-bstep*2] + srow[0]
// RGs += (srow[-bstep*2] + srow[0]) * (T>gradN)
RGs += (t1 & mask);
RGs = v_add(RGs, v_and(t1, mask));
// GRs += {srow[-bstep]*2; (srow[-bstep*2-1] + srow[-bstep*2+1])} * (T>gradN)
GRs += (v_merge_u16(t0, x2 + x4) & mask);
GRs = v_add(GRs, (v_and(v_merge_u16(t0, v_add(x2, x4)), mask)));
// Bs += {(srow[-bstep-1]+srow[-bstep+1]); srow[-bstep]*2 } * (T>gradN)
Bs += (v_merge_u16(x1 + x5, t0) & mask);
Bs = v_add(Bs, v_and(v_merge_u16(v_add(x1, x5), t0), mask));
// gradNE **********************************************
mask = (T > gradNE); // mask = T>gradNE
ng = v_reinterpret_as_u16(v_reinterpret_as_s16(ng) - v_reinterpret_as_s16(mask)); // ng += (T>gradNE)
mask = (v_gt(T, gradNE)); // mask = T>gradNE
ng = v_reinterpret_as_u16(v_sub(v_reinterpret_as_s16(ng), v_reinterpret_as_s16(mask))); // ng += (T>gradNE)
t0 = (x5 << 1); // srow[-bstep+1]*2
t1 = v_load_expand(srow - bstep*2+2) + x0; // srow[-bstep*2+2] + srow[0]
t0 = (v_shl<1>(x5)); // srow[-bstep+1]*2
t1 = v_add(v_load_expand(srow - bstep * 2 + 2), x0); // srow[-bstep*2+2] + srow[0]
// RGs += {(srow[-bstep*2+2] + srow[0]); srow[-bstep+1]*2} * (T>gradNE)
RGs += (v_merge_u16(t1, t0) & mask);
RGs = v_add(RGs, v_and(v_merge_u16(t1, t0), mask));
// GRs += {brow0[N6+1]; (srow[-bstep*2+1] + srow[1])} * (T>gradNE)
GRs += (v_merge_u16(v_load(brow0+N6+1), x4 + x7) & mask);
GRs = v_add(GRs, v_and(v_merge_u16(v_load(brow0+N6+1), v_add(x4, x7)), mask));
// Bs += {srow[-bstep+1]*2; (srow[-bstep] + srow[-bstep+2])} * (T>gradNE)
Bs += (v_merge_u16(t0, x3 + x6) & mask);
Bs = v_add(Bs, v_and(v_merge_u16(t0, v_add(x3, x6)), mask));
// gradE ***********************************************
mask = (T > gradE); // mask = T>gradE
ng = v_reinterpret_as_u16(v_reinterpret_as_s16(ng) - v_reinterpret_as_s16(mask)); // ng += (T>gradE)
mask = (v_gt(T, gradE)); // mask = T>gradE
ng = v_reinterpret_as_u16(v_sub(v_reinterpret_as_s16(ng), v_reinterpret_as_s16(mask))); // ng += (T>gradE)
t0 = (x7 << 1); // srow[1]*2
t1 = v_load_expand(srow +2) + x0; // srow[2] + srow[0]
t0 = (v_shl<1>(x7)); // srow[1]*2
t1 = v_add(v_load_expand(srow + 2), x0); // srow[2] + srow[0]
// RGs += (srow[2] + srow[0]) * (T>gradE)
RGs += (t1 & mask);
RGs = v_add(RGs, v_and(t1, mask));
// GRs += (srow[1]*2) * (T>gradE)
GRs += (t0 & mask);
GRs = v_add(GRs, v_and(t0, mask));
// Bs += {(srow[-bstep+1]+srow[bstep+1]); (srow[-bstep+2]+srow[bstep+2])} * (T>gradE)
Bs += (v_merge_u16(x5 + x9, x6 + x8) & mask);
Bs = v_add(Bs, v_and(v_merge_u16(v_add(x5, x9), v_add(x6, x8)), mask));
// gradSE **********************************************
mask = (T > gradSE); // mask = T>gradSE
ng = v_reinterpret_as_u16(v_reinterpret_as_s16(ng) - v_reinterpret_as_s16(mask)); // ng += (T>gradSE)
mask = (v_gt(T, gradSE)); // mask = T>gradSE
ng = v_reinterpret_as_u16(v_sub(v_reinterpret_as_s16(ng), v_reinterpret_as_s16(mask))); // ng += (T>gradSE)
t0 = (x9 << 1); // srow[bstep+1]*2
t1 = v_load_expand(srow + bstep*2+2) + x0; // srow[bstep*2+2] + srow[0]
t0 = (v_shl<1>(x9)); // srow[bstep+1]*2
t1 = v_add(v_load_expand(srow + bstep * 2 + 2), x0); // srow[bstep*2+2] + srow[0]
// RGs += {(srow[bstep*2+2] + srow[0]); srow[bstep+1]*2} * (T>gradSE)
RGs += (v_merge_u16(t1, t0) & mask);
RGs = v_add(RGs, v_and(v_merge_u16(t1, t0), mask));
// GRs += {brow2[N6+1]; (srow[1]+srow[bstep*2+1])} * (T>gradSE)
GRs += (v_merge_u16(v_load(brow2+N6+1), x7 + x10) & mask);
GRs = v_add(GRs, v_and(v_merge_u16(v_load(brow2+N6+1), v_add(x7, x10)), mask));
// Bs += {srow[bstep+1]*2; (srow[bstep+2]+srow[bstep])} * (T>gradSE)
Bs += (v_merge_u16((x9 << 1), x8 + x11) & mask);
Bs = v_add(Bs, v_and(v_merge_u16((v_shl<1>(x9)), v_add(x8, x11)), mask));
// gradS ***********************************************
mask = (T > gradS); // mask = T>gradS
ng = v_reinterpret_as_u16(v_reinterpret_as_s16(ng) - v_reinterpret_as_s16(mask)); // ng += (T>gradS)
mask = (v_gt(T, gradS)); // mask = T>gradS
ng = v_reinterpret_as_u16(v_sub(v_reinterpret_as_s16(ng), v_reinterpret_as_s16(mask))); // ng += (T>gradS)
t0 = (x11 << 1); // srow[bstep]*2
t1 = v_load_expand(srow + bstep*2) + x0; // srow[bstep*2]+srow[0]
t0 = (v_shl<1>(x11)); // srow[bstep]*2
t1 = v_add(v_load_expand(srow + bstep * 2), x0); // srow[bstep*2]+srow[0]
// RGs += (srow[bstep*2]+srow[0]) * (T>gradS)
RGs += (t1 & mask);
RGs = v_add(RGs, v_and(t1, mask));
// GRs += {srow[bstep]*2; (srow[bstep*2+1]+srow[bstep*2-1])} * (T>gradS)
GRs += (v_merge_u16(t0, x10 + x12) & mask);
GRs = v_add(GRs, v_and(v_merge_u16(t0, v_add(x10, x12)), mask));
// Bs += {(srow[bstep+1]+srow[bstep-1]); srow[bstep]*2} * (T>gradS)
Bs += (v_merge_u16(x9 + x13, t0) & mask);
Bs = v_add(Bs, v_and(v_merge_u16(v_add(x9, x13), t0), mask));
// gradSW **********************************************
mask = (T > gradSW); // mask = T>gradSW
ng = v_reinterpret_as_u16(v_reinterpret_as_s16(ng) - v_reinterpret_as_s16(mask)); // ng += (T>gradSW)
mask = (v_gt(T, gradSW)); // mask = T>gradSW
ng = v_reinterpret_as_u16(v_sub(v_reinterpret_as_s16(ng), v_reinterpret_as_s16(mask))); // ng += (T>gradSW)
t0 = (x13 << 1); // srow[bstep-1]*2
t1 = v_load_expand(srow + bstep*2-2) + x0; // srow[bstep*2-2]+srow[0]
t0 = (v_shl<1>(x13)); // srow[bstep-1]*2
t1 = v_add(v_load_expand(srow + bstep * 2 - 2), x0); // srow[bstep*2-2]+srow[0]
// RGs += {(srow[bstep*2-2]+srow[0]); srow[bstep-1]*2} * (T>gradSW)
RGs += (v_merge_u16(t1, t0) & mask);
RGs = v_add(RGs, v_and(v_merge_u16(t1, t0), mask));
// GRs += {brow2[N6-1]; (srow[bstep*2-1]+srow[-1])} * (T>gradSW)
GRs += (v_merge_u16(v_load(brow2+N6-1), x12 + x15) & mask);
GRs = v_add(GRs, v_and(v_merge_u16(v_load(brow2+N6-1), v_add(x12, x15)), mask));
// Bs += {srow[bstep-1]*2; (srow[bstep]+srow[bstep-2])} * (T>gradSW)
Bs += (v_merge_u16(t0, x11 + x14) & mask);
Bs = v_add(Bs, v_and(v_merge_u16(t0, v_add(x11, x14)), mask));
// gradW ***********************************************
mask = (T > gradW); // mask = T>gradW
ng = v_reinterpret_as_u16(v_reinterpret_as_s16(ng) - v_reinterpret_as_s16(mask)); // ng += (T>gradW)
mask = (v_gt(T, gradW)); // mask = T>gradW
ng = v_reinterpret_as_u16(v_sub(v_reinterpret_as_s16(ng), v_reinterpret_as_s16(mask))); // ng += (T>gradW)
t0 = (x15 << 1); // srow[-1]*2
t1 = v_load_expand(srow -2) + x0; // srow[-2]+srow[0]
t0 = (v_shl<1>(x15)); // srow[-1]*2
t1 = v_add(v_load_expand(srow - 2), x0); // srow[-2]+srow[0]
// RGs += (srow[-2]+srow[0]) * (T>gradW)
RGs += (t1 & mask);
RGs = v_add(RGs, v_and(t1, mask));
// GRs += (srow[-1]*2) * (T>gradW)
GRs += (t0 & mask);
GRs = v_add(GRs, v_and(t0, mask));
// Bs += {(srow[-bstep-1]+srow[bstep-1]); (srow[bstep-2]+srow[-bstep-2])} * (T>gradW)
Bs += (v_merge_u16(x1 + x13, x14 + x16) & mask);
Bs = v_add(Bs, v_and(v_merge_u16(v_add(x1, x13), v_add(x14, x16)), mask));
// gradNW **********************************************
mask = (T > gradNW); // mask = T>gradNW
ng = v_reinterpret_as_u16(v_reinterpret_as_s16(ng) - v_reinterpret_as_s16(mask)); // ng += (T>gradNW)
mask = (v_gt(T, gradNW)); // mask = T>gradNW
ng = v_reinterpret_as_u16(v_sub(v_reinterpret_as_s16(ng), v_reinterpret_as_s16(mask))); // ng += (T>gradNW)
t0 = (x1 << 1); // srow[-bstep-1]*2
t1 = v_load_expand(srow -bstep*2-2) + x0; // srow[-bstep*2-2]+srow[0]
t0 = (v_shl<1>(x1)); // srow[-bstep-1]*2
t1 = v_add(v_load_expand(srow - bstep * 2 - 2), x0); // srow[-bstep*2-2]+srow[0]
// RGs += {(srow[-bstep*2-2]+srow[0]); srow[-bstep-1]*2} * (T>gradNW)
RGs += (v_merge_u16(t1, t0) & mask);
RGs = v_add(RGs, v_and(v_merge_u16(t1, t0), mask));
// GRs += {brow0[N6-1]; (srow[-bstep*2-1]+srow[-1])} * (T>gradNW)
GRs += (v_merge_u16(v_load(brow0+N6-1), x2 + x15) & mask);
GRs = v_add(GRs, v_and(v_merge_u16(v_load(brow0+N6-1), v_add(x2, x15)), mask));
// Bs += {srow[-bstep-1]*2; (srow[-bstep]+srow[-bstep-2])} * (T>gradNW)
Bs += (v_merge_u16((x1 << 1), x3 + x16) & mask);
Bs = v_add(Bs, v_and(v_merge_u16(v_shl<1>(x1), v_add(x3, x16)), mask));
v_float32x4 ngf0 = _0_5 / v_cvt_s16f32_lo(ng);
v_float32x4 ngf1 = _0_5 / v_cvt_s16f32_hi(ng);
v_float32x4 ngf0 = v_div(_0_5, v_cvt_s16f32_lo(ng));
v_float32x4 ngf1 = v_div(_0_5, v_cvt_s16f32_hi(ng));
// now interpolate r, g & b
t0 = v_reinterpret_as_u16(v_reinterpret_as_s16(GRs) - v_reinterpret_as_s16(RGs));
t1 = v_reinterpret_as_u16(v_reinterpret_as_s16(Bs) - v_reinterpret_as_s16(RGs));
t0 = v_reinterpret_as_u16(v_sub(v_reinterpret_as_s16(GRs), v_reinterpret_as_s16(RGs)));
t1 = v_reinterpret_as_u16(v_sub(v_reinterpret_as_s16(Bs), v_reinterpret_as_s16(RGs)));
t0 = v_reinterpret_as_u16(v_reinterpret_as_s16(x0) +
t0 = v_reinterpret_as_u16(
v_add(v_reinterpret_as_s16(x0),
v_pack(
v_round(v_cvt_s16f32_lo(t0) * ngf0),
v_round(v_cvt_s16f32_hi(t0) * ngf1)));
v_round(v_mul(v_cvt_s16f32_lo(t0), ngf0)),
v_round(v_mul(v_cvt_s16f32_hi(t0), ngf1)))));
t1 = v_reinterpret_as_u16(v_reinterpret_as_s16(x0) +
t1 = v_reinterpret_as_u16(
v_add(v_reinterpret_as_s16(x0),
v_pack(
v_round(v_cvt_s16f32_lo(t1) * ngf0),
v_round(v_cvt_s16f32_hi(t1) * ngf1)));
v_round(v_mul(v_cvt_s16f32_lo(t1), ngf0)),
v_round(v_mul(v_cvt_s16f32_hi(t1), ngf1)))));
x1 = v_merge_u16(x0, t0);
x2 = v_merge_u16(t0, x0);
modules/imgproc/src/filter.simd.hpp
+19 -19
View File
@@ -1084,9 +1084,9 @@ struct SymmColumnVec_32s8u
i += VTraits<v_uint16>::vlanes();
}
#if CV_SIMD_WIDTH > 16
while( i <= width - 4 /*v_int32x4::nlanes*/ )
while( i <= width - 4 /*VTraits<v_int32x4>::vlanes()*/ )
#else
if( i <= width - v_int32::nlanes )
if( i <= width - VTraits<v_int32>::vlanes() )
#endif
{
v_float32 s0 = v_muladd(v_cvt_f32(vx_load(src[0] + i)), vx_setall_f32(ky[0]), vx_setall_f32(delta));
@@ -1140,9 +1140,9 @@ struct SymmColumnVec_32s8u
i += VTraits<v_uint16>::vlanes();
}
#if CV_SIMD_WIDTH > 16
while( i <= width - 4 /*v_int32x4::nlanes*/ )
while( i <= width - 4 /*VTraits<v_int32x4>::vlanes()*/ )
#else
if( i <= width - v_int32::nlanes )
if( i <= width - VTraits<v_int32>::vlanes() )
#endif
{
v_float32 s0 = v_muladd(v_cvt_f32(v_sub(vx_load(src[1] + i), vx_load(src[-1] + i))), vx_setall_f32(ky[1]), vx_setall_f32(delta));
@@ -1321,23 +1321,23 @@ struct SymmColumnSmallVec_32s16s
{
v_int32 k0 = vx_setall_s32((int)ky[0]), k1 = vx_setall_s32((int)ky[1]);
v_int32 d4 = vx_setall_s32(d);
for( ; i <= width - 2*v_int16::nlanes; i += 2*v_int16::nlanes )
for( ; i <= width - 2*VTraits<v_int16>::vlanes(); i += 2*VTraits<v_int16>::vlanes() )
{
v_store(dst + i, v_pack(v_muladd(vx_load(S0 + i) + vx_load(S2 + i), k1, v_muladd(vx_load(S1 + i), k0, d4)),
v_muladd(vx_load(S0 + i + v_int32::nlanes) + vx_load(S2 + i + v_int32::nlanes), k1, v_muladd(vx_load(S1 + i + v_int32::nlanes), k0, d4))));
v_store(dst + i + v_int16::nlanes, v_pack(v_muladd(vx_load(S0 + i + 2*v_int32::nlanes) + vx_load(S2 + i + 2*v_int32::nlanes), k1, v_muladd(vx_load(S1 + i + 2*v_int32::nlanes), k0, d4)),
v_muladd(vx_load(S0 + i + 3*v_int32::nlanes) + vx_load(S2 + i + 3*v_int32::nlanes), k1, v_muladd(vx_load(S1 + i + 3*v_int32::nlanes), k0, d4))));
v_store(dst + i, v_pack(v_muladd(v_add(vx_load(S0 + i), vx_load(S2 + i)), k1, v_muladd(vx_load(S1 + i), k0, d4)),
v_muladd(v_add(vx_load(S0 + i + VTraits<v_int32>::vlanes()), vx_load(S2 + i + VTraits<v_int32>::vlanes())), k1, v_muladd(vx_load(S1 + i + VTraits<v_int32>::vlanes()), k0, d4))));
v_store(dst + i + VTraits<v_int16>::vlanes(), v_pack(v_muladd(v_add(vx_load(S0 + i + 2 * VTraits<v_int32>::vlanes()), vx_load(S2 + i + 2 * VTraits<v_int32>::vlanes())), k1, v_muladd(vx_load(S1 + i + 2*VTraits<v_int32>::vlanes()), k0, d4)),
v_muladd(v_add(vx_load(S0 + i + 3 * VTraits<v_int32>::vlanes()), vx_load(S2 + i + 3 * VTraits<v_int32>::vlanes())), k1, v_muladd(vx_load(S1 + i + 3*VTraits<v_int32>::vlanes()), k0, d4))));
}
if( i <= width - v_int16::nlanes )
if( i <= width - VTraits<v_int16>::vlanes() )
{
v_store(dst + i, v_pack(v_muladd(vx_load(S0 + i) + vx_load(S2 + i), k1, v_muladd(vx_load(S1 + i), k0, d4)),
v_muladd(vx_load(S0 + i + v_int32::nlanes) + vx_load(S2 + i + v_int32::nlanes), k1, v_muladd(vx_load(S1 + i + v_int32::nlanes), k0, d4))));
i += v_int16::nlanes;
v_store(dst + i, v_pack(v_muladd(v_add(vx_load(S0 + i), vx_load(S2 + i)), k1, v_muladd(vx_load(S1 + i), k0, d4)),
v_muladd(v_add(vx_load(S0 + i + VTraits<v_int32>::vlanes()), vx_load(S2 + i + VTraits<v_int32>::vlanes())), k1, v_muladd(vx_load(S1 + i + VTraits<v_int32>::vlanes()), k0, d4))));
i += VTraits<v_int16>::vlanes();
}
if( i <= width - v_int32::nlanes )
if( i <= width - VTraits<v_int32>::vlanes() )
{
v_pack_store(dst + i, v_muladd(vx_load(S0 + i) + vx_load(S2 + i), k1, v_muladd(vx_load(S1 + i), k0, d4)));
i += v_int32::nlanes;
v_pack_store(dst + i, v_muladd(v_add(vx_load(S0 + i), vx_load(S2 + i)), k1, v_muladd(vx_load(S1 + i), k0, d4)));
i += VTraits<v_int32>::vlanes();
}
}
#endif
@@ -2237,9 +2237,9 @@ struct FilterVec_8u
i += VTraits<v_uint16>::vlanes();
}
#if CV_SIMD_WIDTH > 16
while( i <= width - 4 /*v_int32x4::nlanes*/ )
while( i <= width - 4 /*VTraits<v_int32x4>::vlanes()*/ )
#else
if( i <= width - v_int32::nlanes )
if( i <= width - VTraits<v_int32>::vlanes() )
#endif
{
v_float32 s0 = v_muladd(v_cvt_f32(v_reinterpret_as_s32(vx_load_expand_q(src[0] + i))), vx_setall_f32(kf[0]), vx_setall_f32(delta));
@@ -2248,7 +2248,7 @@ struct FilterVec_8u
v_int32 s32 = v_round(s0);
v_int16 s16 = v_pack(s32, s32);
*(unaligned_int*)(dst + i) = v_get0(v_reinterpret_as_s32(v_pack_u(s16, s16)));
i += 4 /*v_int32x4::nlanes*/ ;
i += 4 /*VTraits<v_int32x4>::vlanes()*/ ;
}
return i;
}
modules/imgproc/src/histogram.cpp
+4 -4
View File
@@ -2093,7 +2093,7 @@ double cv::compareHist( InputArray _H1, InputArray _H2, int method )
v_float32 v_s11 = vx_setzero_f32();
v_float32 v_s12 = vx_setzero_f32();
v_float32 v_s22 = vx_setzero_f32();
for (; j <= len - v_float32::nlanes; j += v_float32::nlanes)
for (; j <= len - VTraits<v_float32>::vlanes(); j += VTraits<v_float32>::vlanes())
{
v_float32 v_a = vx_load(h1 + j);
v_float32 v_b = vx_load(h2 + j);
@@ -2134,10 +2134,10 @@ double cv::compareHist( InputArray _H1, InputArray _H2, int method )
result += v_reduce_sum(v_result);
#elif CV_SIMD
v_float32 v_result = vx_setzero_f32();
for (; j <= len - v_float32::nlanes; j += v_float32::nlanes)
for (; j <= len - VTraits<v_float32>::vlanes(); j += VTraits<v_float32>::vlanes())
{
v_float32 v_src = v_min(vx_load(h1 + j), vx_load(h2 + j));
v_result += v_src;
v_result = v_add(v_result, v_src);
}
result += v_reduce_sum(v_result);
#endif
@@ -2174,7 +2174,7 @@ double cv::compareHist( InputArray _H1, InputArray _H2, int method )
v_float32 v_s1 = vx_setzero_f32();
v_float32 v_s2 = vx_setzero_f32();
v_float32 v_result = vx_setzero_f32();
for (; j <= len - v_float32::nlanes; j += v_float32::nlanes)
for (; j <= len - VTraits<v_float32>::vlanes(); j += VTraits<v_float32>::vlanes())
{
v_float32 v_a = vx_load(h1 + j);
v_float32 v_b = vx_load(h2 + j);
modules/imgproc/src/imgwarp.cpp
+136 -136
View File
@@ -455,7 +455,7 @@ struct RemapVec_8u
v_int32x4 delta = v_setall_s32(INTER_REMAP_COEF_SCALE / 2);
v_int16x8 xy2ofs = v_reinterpret_as_s16(v_setall_s32(cn + (sstep << 16)));
int CV_DECL_ALIGNED(16) iofs0[4], iofs1[4];
const uchar* src_limit_8bytes = _src.datalimit - v_int16x8::nlanes;
const uchar* src_limit_8bytes = _src.datalimit - VTraits<v_int16x8>::vlanes();
#define CV_PICK_AND_PACK_RGB(ptr, offset, result) \
{ \
const uchar* const p = ((const uchar*)ptr) + (offset); \
@@ -483,7 +483,7 @@ struct RemapVec_8u
v_uint8x16 rrggbbaa, dummy; \
v_uint16x8 rrggbbaa8, dummy8; \
v_uint8x16 rgba0 = v_reinterpret_as_u8(v_int32x4(*(unaligned_int*)(p), 0, 0, 0)); \
v_uint8x16 rgba1 = v_reinterpret_as_u8(v_int32x4(*(unaligned_int*)(p + v_int32x4::nlanes), 0, 0, 0)); \
v_uint8x16 rgba1 = v_reinterpret_as_u8(v_int32x4(*(unaligned_int*)(p + VTraits<v_int32x4>::vlanes()), 0, 0, 0)); \
v_zip(rgba0, rgba1, rrggbbaa, dummy); \
v_expand(rrggbbaa, rrggbbaa8, dummy8); \
result = v_reinterpret_as_s16(rrggbbaa8); \
@@ -534,8 +534,8 @@ struct RemapVec_8u
v3 = v_dotprod(v_reinterpret_as_s16(v3), v_reinterpret_as_s16(d2), delta);
v2 = v_dotprod(v_reinterpret_as_s16(v2), v_reinterpret_as_s16(c2), v3);
v0 = v0 >> INTER_REMAP_COEF_BITS;
v2 = v2 >> INTER_REMAP_COEF_BITS;
v0 = v_shr<INTER_REMAP_COEF_BITS>(v0);
v2 = v_shr<INTER_REMAP_COEF_BITS>(v2);
v_pack_u_store(D + x, v_pack(v0, v2));
}
}
@@ -563,8 +563,8 @@ struct RemapVec_8u
CV_PICK_AND_PACK_RGB(S0, iofs0[1], u1);
CV_PICK_AND_PACK_RGB(S1, iofs0[1], v1);
v_int32x4 result0 = v_dotprod(u0, w00, v_dotprod(v0, w01, delta)) >> INTER_REMAP_COEF_BITS;
v_int32x4 result1 = v_dotprod(u1, w10, v_dotprod(v1, w11, delta)) >> INTER_REMAP_COEF_BITS;
v_int32x4 result0 = v_shr<INTER_REMAP_COEF_BITS>(v_dotprod(u0, w00, v_dotprod(v0, w01, delta)));
v_int32x4 result1 = v_shr<INTER_REMAP_COEF_BITS>(v_dotprod(u1, w10, v_dotprod(v1, w11, delta)));
result0 = v_rotate_left<1>(result0);
v_int16x8 result8 = v_pack(result0, result1);
@@ -581,8 +581,8 @@ struct RemapVec_8u
CV_PICK_AND_PACK_RGB(S0, iofs0[3], u1);
CV_PICK_AND_PACK_RGB(S1, iofs0[3], v1);
result0 = v_dotprod(u0, w00, v_dotprod(v0, w01, delta)) >> INTER_REMAP_COEF_BITS;
result1 = v_dotprod(u1, w10, v_dotprod(v1, w11, delta)) >> INTER_REMAP_COEF_BITS;
result0 = v_shr<INTER_REMAP_COEF_BITS>(v_dotprod(u0, w00, v_dotprod(v0, w01, delta)));
result1 = v_shr<INTER_REMAP_COEF_BITS>(v_dotprod(u1, w10, v_dotprod(v1, w11, delta)));
result0 = v_rotate_left<1>(result0);
result8 = v_pack(result0, result1);
@@ -613,8 +613,8 @@ struct RemapVec_8u
CV_PICK_AND_PACK_RGBA(S0, iofs0[1], u1);
CV_PICK_AND_PACK_RGBA(S1, iofs0[1], v1);
v_int32x4 result0 = v_dotprod(u0, w00, v_dotprod(v0, w01, delta)) >> INTER_REMAP_COEF_BITS;
v_int32x4 result1 = v_dotprod(u1, w10, v_dotprod(v1, w11, delta)) >> INTER_REMAP_COEF_BITS;
v_int32x4 result0 = v_shr<INTER_REMAP_COEF_BITS>(v_dotprod(u0, w00, v_dotprod(v0, w01, delta)));
v_int32x4 result1 = v_shr<INTER_REMAP_COEF_BITS>(v_dotprod(u1, w10, v_dotprod(v1, w11, delta)));
v_int16x8 result8 = v_pack(result0, result1);
v_pack_u_store(D, result8);
@@ -627,8 +627,8 @@ struct RemapVec_8u
CV_PICK_AND_PACK_RGBA(S0, iofs0[3], u1);
CV_PICK_AND_PACK_RGBA(S1, iofs0[3], v1);
result0 = v_dotprod(u0, w00, v_dotprod(v0, w01, delta)) >> INTER_REMAP_COEF_BITS;
result1 = v_dotprod(u1, w10, v_dotprod(v1, w11, delta)) >> INTER_REMAP_COEF_BITS;
result0 = v_shr<INTER_REMAP_COEF_BITS>(v_dotprod(u0, w00, v_dotprod(v0, w01, delta)));
result1 = v_shr<INTER_REMAP_COEF_BITS>(v_dotprod(u1, w10, v_dotprod(v1, w11, delta)));
result8 = v_pack(result0, result1);
v_pack_u_store(D + 8, result8);
}
@@ -1164,7 +1164,7 @@ public:
#if CV_SIMD128
{
int span = v_float32x4::nlanes;
int span = VTraits<v_float32x4>::vlanes();
for( ; x1 <= bcols - span * 2; x1 += span * 2 )
{
v_int32x4 ix0 = v_round(v_load(sX + x1));
@@ -1206,9 +1206,9 @@ public:
#if CV_SIMD128
{
v_uint16x8 v_scale = v_setall_u16(INTER_TAB_SIZE2 - 1);
int span = v_uint16x8::nlanes;
int span = VTraits<v_uint16x8>::vlanes();
for( ; x1 <= bcols - span; x1 += span )
v_store((unsigned short*)(A + x1), v_load(sA + x1) & v_scale);
v_store((unsigned short*)(A + x1), v_and(v_load(sA + x1), v_scale));
}
#endif
for( ; x1 < bcols; x1++ )
@@ -1224,16 +1224,16 @@ public:
{
v_float32x4 v_scale = v_setall_f32((float)INTER_TAB_SIZE);
v_int32x4 v_scale2 = v_setall_s32(INTER_TAB_SIZE - 1);
int span = v_float32x4::nlanes;
int span = VTraits<v_float32x4>::vlanes();
for( ; x1 <= bcols - span * 2; x1 += span * 2 )
{
v_int32x4 v_sx0 = v_round(v_scale * v_load(sX + x1));
v_int32x4 v_sy0 = v_round(v_scale * v_load(sY + x1));
v_int32x4 v_sx1 = v_round(v_scale * v_load(sX + x1 + span));
v_int32x4 v_sy1 = v_round(v_scale * v_load(sY + x1 + span));
v_uint16x8 v_sx8 = v_reinterpret_as_u16(v_pack(v_sx0 & v_scale2, v_sx1 & v_scale2));
v_uint16x8 v_sy8 = v_reinterpret_as_u16(v_pack(v_sy0 & v_scale2, v_sy1 & v_scale2));
v_uint16x8 v_v = v_shl<INTER_BITS>(v_sy8) | (v_sx8);
v_int32x4 v_sx0 = v_round(v_mul(v_scale, v_load(sX + x1)));
v_int32x4 v_sy0 = v_round(v_mul(v_scale, v_load(sY + x1)));
v_int32x4 v_sx1 = v_round(v_mul(v_scale, v_load(sX + x1 + span)));
v_int32x4 v_sy1 = v_round(v_mul(v_scale, v_load(sY + x1 + span)));
v_uint16x8 v_sx8 = v_reinterpret_as_u16(v_pack(v_and(v_sx0, v_scale2), v_and(v_sx1, v_scale2)));
v_uint16x8 v_sy8 = v_reinterpret_as_u16(v_pack(v_and(v_sy0, v_scale2), v_and(v_sy1, v_scale2)));
v_uint16x8 v_v = v_or(v_shl<INTER_BITS>(v_sy8), v_sx8);
v_store(A + x1, v_v);
v_int16x8 v_d0 = v_pack(v_shr<INTER_BITS>(v_sx0), v_shr<INTER_BITS>(v_sx1));
@@ -1261,18 +1261,18 @@ public:
{
v_float32x4 v_scale = v_setall_f32((float)INTER_TAB_SIZE);
v_int32x4 v_scale2 = v_setall_s32(INTER_TAB_SIZE - 1), v_scale3 = v_setall_s32(INTER_TAB_SIZE);
int span = v_float32x4::nlanes;
int span = VTraits<v_float32x4>::vlanes();
for( ; x1 <= bcols - span * 2; x1 += span * 2 )
{
v_float32x4 v_fx, v_fy;
v_load_deinterleave(sXY + (x1 << 1), v_fx, v_fy);
v_int32x4 v_sx0 = v_round(v_fx * v_scale);
v_int32x4 v_sy0 = v_round(v_fy * v_scale);
v_int32x4 v_sx0 = v_round(v_mul(v_fx, v_scale));
v_int32x4 v_sy0 = v_round(v_mul(v_fy, v_scale));
v_load_deinterleave(sXY + ((x1 + span) << 1), v_fx, v_fy);
v_int32x4 v_sx1 = v_round(v_fx * v_scale);
v_int32x4 v_sy1 = v_round(v_fy * v_scale);
v_int32x4 v_v0 = v_muladd(v_scale3, (v_sy0 & v_scale2), (v_sx0 & v_scale2));
v_int32x4 v_v1 = v_muladd(v_scale3, (v_sy1 & v_scale2), (v_sx1 & v_scale2));
v_int32x4 v_sx1 = v_round(v_mul(v_fx, v_scale));
v_int32x4 v_sy1 = v_round(v_mul(v_fy, v_scale));
v_int32x4 v_v0 = v_muladd(v_scale3, (v_and(v_sy0, v_scale2)), (v_and(v_sx0, v_scale2)));
v_int32x4 v_v1 = v_muladd(v_scale3, (v_and(v_sy1, v_scale2)), (v_and(v_sx1, v_scale2)));
v_uint16x8 v_v8 = v_reinterpret_as_u16(v_pack(v_v0, v_v1));
v_store(A + x1, v_v8);
v_int16x8 v_dx = v_pack(v_shr<INTER_BITS>(v_sx0), v_shr<INTER_BITS>(v_sx1));
@@ -1941,7 +1941,7 @@ void cv::convertMaps( InputArray _map1, InputArray _map2,
{
#if CV_SIMD128
{
int span = v_int16x8::nlanes;
int span = VTraits<v_int16x8>::vlanes();
for( ; x <= size.width - span; x += span )
{
v_int16x8 v_dst[2];
@@ -1973,21 +1973,21 @@ void cv::convertMaps( InputArray _map1, InputArray _map2,
v_float32x4 v_scale = v_setall_f32((float)INTER_TAB_SIZE);
v_int32x4 v_mask = v_setall_s32(INTER_TAB_SIZE - 1);
v_int32x4 v_scale3 = v_setall_s32(INTER_TAB_SIZE);
int span = v_float32x4::nlanes;
int span = VTraits<v_float32x4>::vlanes();
for( ; x <= size.width - span * 2; x += span * 2 )
{
v_int32x4 v_ix0 = v_round(v_scale * (v_load(src1f + x)));
v_int32x4 v_ix1 = v_round(v_scale * (v_load(src1f + x + span)));
v_int32x4 v_iy0 = v_round(v_scale * (v_load(src2f + x)));
v_int32x4 v_iy1 = v_round(v_scale * (v_load(src2f + x + span)));
v_int32x4 v_ix0 = v_round(v_mul(v_scale, v_load(src1f + x)));
v_int32x4 v_ix1 = v_round(v_mul(v_scale, v_load(src1f + x + span)));
v_int32x4 v_iy0 = v_round(v_mul(v_scale, v_load(src2f + x)));
v_int32x4 v_iy1 = v_round(v_mul(v_scale, v_load(src2f + x + span)));
v_int16x8 v_dst[2];
v_dst[0] = v_pack(v_shr<INTER_BITS>(v_ix0), v_shr<INTER_BITS>(v_ix1));
v_dst[1] = v_pack(v_shr<INTER_BITS>(v_iy0), v_shr<INTER_BITS>(v_iy1));
v_store_interleave(dst1 + (x << 1), v_dst[0], v_dst[1]);
v_int32x4 v_dst0 = v_muladd(v_scale3, (v_iy0 & v_mask), (v_ix0 & v_mask));
v_int32x4 v_dst1 = v_muladd(v_scale3, (v_iy1 & v_mask), (v_ix1 & v_mask));
v_int32x4 v_dst0 = v_muladd(v_scale3, (v_and(v_iy0, v_mask)), (v_and(v_ix0, v_mask)));
v_int32x4 v_dst1 = v_muladd(v_scale3, (v_and(v_iy1, v_mask)), (v_and(v_ix1, v_mask)));
v_store(dst2 + x, v_pack_u(v_dst0, v_dst1));
}
}
@@ -2008,7 +2008,7 @@ void cv::convertMaps( InputArray _map1, InputArray _map2,
if( nninterpolate )
{
#if CV_SIMD128
int span = v_float32x4::nlanes;
int span = VTraits<v_float32x4>::vlanes();
{
for( ; x <= (size.width << 1) - span * 2; x += span * 2 )
v_store(dst1 + x, v_pack(v_round(v_load(src1f + x)),
@@ -2034,16 +2034,16 @@ void cv::convertMaps( InputArray _map1, InputArray _map2,
v_float32x4 v_scale = v_setall_f32((float)INTER_TAB_SIZE);
v_int32x4 v_mask = v_setall_s32(INTER_TAB_SIZE - 1);
v_int32x4 v_scale3 = v_setall_s32(INTER_TAB_SIZE);
int span = v_uint16x8::nlanes;
int span = VTraits<v_uint16x8>::vlanes();
for (; x <= size.width - span; x += span )
{
v_float32x4 v_src0[2], v_src1[2];
v_load_deinterleave(src1f + (x << 1), v_src0[0], v_src0[1]);
v_load_deinterleave(src1f + (x << 1) + span, v_src1[0], v_src1[1]);
v_int32x4 v_ix0 = v_round(v_src0[0] * v_scale);
v_int32x4 v_ix1 = v_round(v_src1[0] * v_scale);
v_int32x4 v_iy0 = v_round(v_src0[1] * v_scale);
v_int32x4 v_iy1 = v_round(v_src1[1] * v_scale);
v_int32x4 v_ix0 = v_round(v_mul(v_src0[0], v_scale));
v_int32x4 v_ix1 = v_round(v_mul(v_src1[0], v_scale));
v_int32x4 v_iy0 = v_round(v_mul(v_src0[1], v_scale));
v_int32x4 v_iy1 = v_round(v_mul(v_src1[1], v_scale));
v_int16x8 v_dst[2];
v_dst[0] = v_pack(v_shr<INTER_BITS>(v_ix0), v_shr<INTER_BITS>(v_ix1));
@@ -2051,8 +2051,8 @@ void cv::convertMaps( InputArray _map1, InputArray _map2,
v_store_interleave(dst1 + (x << 1), v_dst[0], v_dst[1]);
v_store(dst2 + x, v_pack_u(
v_muladd(v_scale3, (v_iy0 & v_mask), (v_ix0 & v_mask)),
v_muladd(v_scale3, (v_iy1 & v_mask), (v_ix1 & v_mask))));
v_muladd(v_scale3, (v_and(v_iy0, v_mask)), (v_and(v_ix0, v_mask))),
v_muladd(v_scale3, (v_and(v_iy1, v_mask)), (v_and(v_ix1, v_mask)))));
}
}
#endif
@@ -2074,13 +2074,13 @@ void cv::convertMaps( InputArray _map1, InputArray _map2,
v_uint16x8 v_mask2 = v_setall_u16(INTER_TAB_SIZE2-1);
v_uint32x4 v_zero = v_setzero_u32(), v_mask = v_setall_u32(INTER_TAB_SIZE-1);
v_float32x4 v_scale = v_setall_f32(scale);
int span = v_float32x4::nlanes;
int span = VTraits<v_float32x4>::vlanes();
for( ; x <= size.width - span * 2; x += span * 2 )
{
v_uint32x4 v_fxy1, v_fxy2;
if ( src2 )
{
v_uint16x8 v_src2 = v_load(src2 + x) & v_mask2;
v_uint16x8 v_src2 = v_and(v_load(src2 + x), v_mask2);
v_expand(v_src2, v_fxy1, v_fxy2);
}
else
@@ -2091,9 +2091,9 @@ void cv::convertMaps( InputArray _map1, InputArray _map2,
v_load_deinterleave(src1 + (x << 1), v_src[0], v_src[1]);
v_expand(v_src[0], v_src0[0], v_src0[1]);
v_expand(v_src[1], v_src1[0], v_src1[1]);
#define CV_COMPUTE_MAP_X(X, FXY) v_muladd(v_scale, v_cvt_f32(v_reinterpret_as_s32((FXY) & v_mask)),\
#define CV_COMPUTE_MAP_X(X, FXY) v_muladd(v_scale, v_cvt_f32(v_reinterpret_as_s32(v_and((FXY), v_mask))),\
v_cvt_f32(v_reinterpret_as_s32(X)))
#define CV_COMPUTE_MAP_Y(Y, FXY) v_muladd(v_scale, v_cvt_f32(v_reinterpret_as_s32((FXY) >> INTER_BITS)),\
#define CV_COMPUTE_MAP_Y(Y, FXY) v_muladd(v_scale, v_cvt_f32(v_reinterpret_as_s32(v_shr<INTER_BITS>((FXY)))),\
v_cvt_f32(v_reinterpret_as_s32(Y)))
v_float32x4 v_dst1 = CV_COMPUTE_MAP_X(v_src0[0], v_fxy1);
v_float32x4 v_dst2 = CV_COMPUTE_MAP_Y(v_src1[0], v_fxy1);
@@ -2123,13 +2123,13 @@ void cv::convertMaps( InputArray _map1, InputArray _map2,
v_int16x8 v_mask2 = v_setall_s16(INTER_TAB_SIZE2-1);
v_int32x4 v_zero = v_setzero_s32(), v_mask = v_setall_s32(INTER_TAB_SIZE-1);
v_float32x4 v_scale = v_setall_f32(scale);
int span = v_int16x8::nlanes;
int span = VTraits<v_int16x8>::vlanes();
for( ; x <= size.width - span; x += span )
{
v_int32x4 v_fxy1, v_fxy2;
if (src2)
{
v_int16x8 v_src2 = v_load((short *)src2 + x) & v_mask2;
v_int16x8 v_src2 = v_and(v_load((short *)src2 + x), v_mask2);
v_expand(v_src2, v_fxy1, v_fxy2);
}
else
@@ -2142,8 +2142,8 @@ void cv::convertMaps( InputArray _map1, InputArray _map2,
v_expand(v_src[0], v_src0[0], v_src0[1]);
v_expand(v_src[1], v_src1[0], v_src1[1]);
#define CV_COMPUTE_MAP_X(X, FXY) v_muladd(v_scale, v_cvt_f32((FXY) & v_mask), v_cvt_f32(X))
#define CV_COMPUTE_MAP_Y(Y, FXY) v_muladd(v_scale, v_cvt_f32((FXY) >> INTER_BITS), v_cvt_f32(Y))
#define CV_COMPUTE_MAP_X(X, FXY) v_muladd(v_scale, v_cvt_f32(v_and((FXY), v_mask)), v_cvt_f32(X))
#define CV_COMPUTE_MAP_Y(Y, FXY) v_muladd(v_scale, v_cvt_f32(v_shr<INTER_BITS>((FXY))), v_cvt_f32(Y))
v_dst[0] = CV_COMPUTE_MAP_X(v_src0[0], v_fxy1);
v_dst[1] = CV_COMPUTE_MAP_Y(v_src1[0], v_fxy1);
v_store_interleave(dst1f + (x << 1), v_dst[0], v_dst[1]);
@@ -2234,12 +2234,12 @@ public:
#if CV_SIMD128
{
v_int32x4 v_X0 = v_setall_s32(X0), v_Y0 = v_setall_s32(Y0);
int span = v_uint16x8::nlanes;
int span = VTraits<v_uint16x8>::vlanes();
for( ; x1 <= bw - span; x1 += span )
{
v_int16x8 v_dst[2];
#define CV_CONVERT_MAP(ptr,offset,shift) v_pack(v_shr<AB_BITS>(shift+v_load(ptr + offset)),\
v_shr<AB_BITS>(shift+v_load(ptr + offset + 4)))
#define CV_CONVERT_MAP(ptr,offset,shift) v_pack(v_shr<AB_BITS>(v_add(shift,v_load(ptr + offset))),\
v_shr<AB_BITS>(v_add(shift,v_load(ptr + offset + 4))))
v_dst[0] = CV_CONVERT_MAP(adelta, x+x1, v_X0);
v_dst[1] = CV_CONVERT_MAP(bdelta, x+x1, v_Y0);
#undef CV_CONVERT_MAP
@@ -2272,21 +2272,21 @@ public:
{
v_int32x4 v__X0 = v_setall_s32(X0), v__Y0 = v_setall_s32(Y0);
v_int32x4 v_mask = v_setall_s32(INTER_TAB_SIZE - 1);
int span = v_float32x4::nlanes;
int span = VTraits<v_float32x4>::vlanes();
for( ; x1 <= bw - span * 2; x1 += span * 2 )
{
v_int32x4 v_X0 = v_shr<AB_BITS - INTER_BITS>(v__X0 + v_load(adelta + x + x1));
v_int32x4 v_Y0 = v_shr<AB_BITS - INTER_BITS>(v__Y0 + v_load(bdelta + x + x1));
v_int32x4 v_X1 = v_shr<AB_BITS - INTER_BITS>(v__X0 + v_load(adelta + x + x1 + span));
v_int32x4 v_Y1 = v_shr<AB_BITS - INTER_BITS>(v__Y0 + v_load(bdelta + x + x1 + span));
v_int32x4 v_X0 = v_shr<AB_BITS - INTER_BITS>(v_add(v__X0, v_load(this->adelta + x + x1)));
v_int32x4 v_Y0 = v_shr<AB_BITS - INTER_BITS>(v_add(v__Y0, v_load(this->bdelta + x + x1)));
v_int32x4 v_X1 = v_shr<AB_BITS - INTER_BITS>(v_add(v__X0, v_load(this->adelta + x + x1 + span)));
v_int32x4 v_Y1 = v_shr<AB_BITS - INTER_BITS>(v_add(v__Y0, v_load(this->bdelta + x + x1 + span)));
v_int16x8 v_xy[2];
v_xy[0] = v_pack(v_shr<INTER_BITS>(v_X0), v_shr<INTER_BITS>(v_X1));
v_xy[1] = v_pack(v_shr<INTER_BITS>(v_Y0), v_shr<INTER_BITS>(v_Y1));
v_store_interleave(xy + (x1 << 1), v_xy[0], v_xy[1]);
v_int32x4 v_alpha0 = v_shl<INTER_BITS>(v_Y0 & v_mask) | (v_X0 & v_mask);
v_int32x4 v_alpha1 = v_shl<INTER_BITS>(v_Y1 & v_mask) | (v_X1 & v_mask);
v_int32x4 v_alpha0 = v_or(v_shl<INTER_BITS>(v_and(v_Y0, v_mask)), v_and(v_X0, v_mask));
v_int32x4 v_alpha1 = v_or(v_shl<INTER_BITS>(v_and(v_Y1, v_mask)), v_and(v_X1, v_mask));
v_store(alpha + x1, v_pack(v_alpha0, v_alpha1));
}
}
@@ -2866,16 +2866,16 @@ void WarpPerspectiveLine_ProcessNN_CV_SIMD(const double *M, short* xy, double X0
v_int32x4 v_X0, v_Y0;
{
v_float64x2 v_W = v_muladd(v_M6, v_x1, v_W0);
v_W = v_select(v_W != v_zero, v_1 / v_W, v_zero);
v_float64x2 v_fX0 = v_max(v_intmin, v_min(v_intmax, v_muladd(v_M0, v_x1, v_X0d) * v_W));
v_float64x2 v_fY0 = v_max(v_intmin, v_min(v_intmax, v_muladd(v_M3, v_x1, v_Y0d) * v_W));
v_x1 += v_2;
v_W = v_select(v_ne(v_W, v_zero), v_div(v_1, v_W), v_zero);
v_float64x2 v_fX0 = v_max(v_intmin, v_min(v_intmax, v_mul(v_muladd(v_M0, v_x1, v_X0d), v_W)));
v_float64x2 v_fY0 = v_max(v_intmin, v_min(v_intmax, v_mul(v_muladd(v_M3, v_x1, v_Y0d), v_W)));
v_x1 = v_add(v_x1, v_2);
v_W = v_muladd(v_M6, v_x1, v_W0);
v_W = v_select(v_W != v_zero, v_1 / v_W, v_zero);
v_float64x2 v_fX1 = v_max(v_intmin, v_min(v_intmax, v_muladd(v_M0, v_x1, v_X0d) * v_W));
v_float64x2 v_fY1 = v_max(v_intmin, v_min(v_intmax, v_muladd(v_M3, v_x1, v_Y0d) * v_W));
v_x1 += v_2;
v_W = v_select(v_ne(v_W, v_zero), v_div(v_1, v_W), v_zero);
v_float64x2 v_fX1 = v_max(v_intmin, v_min(v_intmax, v_mul(v_muladd(v_M0, v_x1, v_X0d), v_W)));
v_float64x2 v_fY1 = v_max(v_intmin, v_min(v_intmax, v_mul(v_muladd(v_M3, v_x1, v_Y0d), v_W)));
v_x1 = v_add(v_x1, v_2);
v_X0 = v_round(v_fX0, v_fX1);
v_Y0 = v_round(v_fY0, v_fY1);
@@ -2885,16 +2885,16 @@ void WarpPerspectiveLine_ProcessNN_CV_SIMD(const double *M, short* xy, double X0
v_int32x4 v_X1, v_Y1;
{
v_float64x2 v_W = v_muladd(v_M6, v_x1, v_W0);
v_W = v_select(v_W != v_zero, v_1 / v_W, v_zero);
v_float64x2 v_fX0 = v_max(v_intmin, v_min(v_intmax, v_muladd(v_M0, v_x1, v_X0d) * v_W));
v_float64x2 v_fY0 = v_max(v_intmin, v_min(v_intmax, v_muladd(v_M3, v_x1, v_Y0d) * v_W));
v_x1 += v_2;
v_W = v_select(v_ne(v_W, v_zero), v_div(v_1, v_W), v_zero);
v_float64x2 v_fX0 = v_max(v_intmin, v_min(v_intmax, v_mul(v_muladd(v_M0, v_x1, v_X0d), v_W)));
v_float64x2 v_fY0 = v_max(v_intmin, v_min(v_intmax, v_mul(v_muladd(v_M3, v_x1, v_Y0d), v_W)));
v_x1 = v_add(v_x1, v_2);
v_W = v_muladd(v_M6, v_x1, v_W0);
v_W = v_select(v_W != v_zero, v_1 / v_W, v_zero);
v_float64x2 v_fX1 = v_max(v_intmin, v_min(v_intmax, v_muladd(v_M0, v_x1, v_X0d) * v_W));
v_float64x2 v_fY1 = v_max(v_intmin, v_min(v_intmax, v_muladd(v_M3, v_x1, v_Y0d) * v_W));
v_x1 += v_2;
v_W = v_select(v_ne(v_W, v_zero), v_div(v_1, v_W), v_zero);
v_float64x2 v_fX1 = v_max(v_intmin, v_min(v_intmax, v_mul(v_muladd(v_M0, v_x1, v_X0d), v_W)));
v_float64x2 v_fY1 = v_max(v_intmin, v_min(v_intmax, v_mul(v_muladd(v_M3, v_x1, v_Y0d), v_W)));
v_x1 = v_add(v_x1, v_2);
v_X1 = v_round(v_fX0, v_fX1);
v_Y1 = v_round(v_fY0, v_fY1);
@@ -2904,16 +2904,16 @@ void WarpPerspectiveLine_ProcessNN_CV_SIMD(const double *M, short* xy, double X0
v_int32x4 v_X2, v_Y2;
{
v_float64x2 v_W = v_muladd(v_M6, v_x1, v_W0);
v_W = v_select(v_W != v_zero, v_1 / v_W, v_zero);
v_float64x2 v_fX0 = v_max(v_intmin, v_min(v_intmax, v_muladd(v_M0, v_x1, v_X0d) * v_W));
v_float64x2 v_fY0 = v_max(v_intmin, v_min(v_intmax, v_muladd(v_M3, v_x1, v_Y0d) * v_W));
v_x1 += v_2;
v_W = v_select(v_ne(v_W, v_zero), v_div(v_1, v_W), v_zero);
v_float64x2 v_fX0 = v_max(v_intmin, v_min(v_intmax, v_mul(v_muladd(v_M0, v_x1, v_X0d), v_W)));
v_float64x2 v_fY0 = v_max(v_intmin, v_min(v_intmax, v_mul(v_muladd(v_M3, v_x1, v_Y0d), v_W)));
v_x1 = v_add(v_x1, v_2);
v_W = v_muladd(v_M6, v_x1, v_W0);
v_W = v_select(v_W != v_zero, v_1 / v_W, v_zero);
v_float64x2 v_fX1 = v_max(v_intmin, v_min(v_intmax, v_muladd(v_M0, v_x1, v_X0d) * v_W));
v_float64x2 v_fY1 = v_max(v_intmin, v_min(v_intmax, v_muladd(v_M3, v_x1, v_Y0d) * v_W));
v_x1 += v_2;
v_W = v_select(v_ne(v_W, v_zero), v_div(v_1, v_W), v_zero);
v_float64x2 v_fX1 = v_max(v_intmin, v_min(v_intmax, v_mul(v_muladd(v_M0, v_x1, v_X0d), v_W)));
v_float64x2 v_fY1 = v_max(v_intmin, v_min(v_intmax, v_mul(v_muladd(v_M3, v_x1, v_Y0d), v_W)));
v_x1 = v_add(v_x1, v_2);
v_X2 = v_round(v_fX0, v_fX1);
v_Y2 = v_round(v_fY0, v_fY1);
@@ -2923,16 +2923,16 @@ void WarpPerspectiveLine_ProcessNN_CV_SIMD(const double *M, short* xy, double X0
v_int32x4 v_X3, v_Y3;
{
v_float64x2 v_W = v_muladd(v_M6, v_x1, v_W0);
v_W = v_select(v_W != v_zero, v_1 / v_W, v_zero);
v_float64x2 v_fX0 = v_max(v_intmin, v_min(v_intmax, v_muladd(v_M0, v_x1, v_X0d) * v_W));
v_float64x2 v_fY0 = v_max(v_intmin, v_min(v_intmax, v_muladd(v_M3, v_x1, v_Y0d) * v_W));
v_x1 += v_2;
v_W = v_select(v_ne(v_W, v_zero), v_div(v_1, v_W), v_zero);
v_float64x2 v_fX0 = v_max(v_intmin, v_min(v_intmax, v_mul(v_muladd(v_M0, v_x1, v_X0d), v_W)));
v_float64x2 v_fY0 = v_max(v_intmin, v_min(v_intmax, v_mul(v_muladd(v_M3, v_x1, v_Y0d), v_W)));
v_x1 = v_add(v_x1, v_2);
v_W = v_muladd(v_M6, v_x1, v_W0);
v_W = v_select(v_W != v_zero, v_1 / v_W, v_zero);
v_float64x2 v_fX1 = v_max(v_intmin, v_min(v_intmax, v_muladd(v_M0, v_x1, v_X0d) * v_W));
v_float64x2 v_fY1 = v_max(v_intmin, v_min(v_intmax, v_muladd(v_M3, v_x1, v_Y0d) * v_W));
v_x1 += v_2;
v_W = v_select(v_ne(v_W, v_zero), v_div(v_1, v_W), v_zero);
v_float64x2 v_fX1 = v_max(v_intmin, v_min(v_intmax, v_mul(v_muladd(v_M0, v_x1, v_X0d), v_W)));
v_float64x2 v_fY1 = v_max(v_intmin, v_min(v_intmax, v_mul(v_muladd(v_M3, v_x1, v_Y0d), v_W)));
v_x1 = v_add(v_x1, v_2);
v_X3 = v_round(v_fX0, v_fX1);
v_Y3 = v_round(v_fY0, v_fY1);
@@ -2987,16 +2987,16 @@ void WarpPerspectiveLine_Process_CV_SIMD(const double *M, short* xy, short* alph
v_int32x4 v_X0, v_Y0;
{
v_float64x2 v_W = v_muladd(v_M6, v_x1, v_W0);
v_W = v_select(v_W != v_zero, v_its / v_W, v_zero);
v_float64x2 v_fX0 = v_max(v_intmin, v_min(v_intmax, v_muladd(v_M0, v_x1, v_X0d) * v_W));
v_float64x2 v_fY0 = v_max(v_intmin, v_min(v_intmax, v_muladd(v_M3, v_x1, v_Y0d) * v_W));
v_x1 += v_2;
v_W = v_select(v_ne(v_W, v_zero), v_div(v_its, v_W), v_zero);
v_float64x2 v_fX0 = v_max(v_intmin, v_min(v_intmax, v_mul(v_muladd(v_M0, v_x1, v_X0d), v_W)));
v_float64x2 v_fY0 = v_max(v_intmin, v_min(v_intmax, v_mul(v_muladd(v_M3, v_x1, v_Y0d), v_W)));
v_x1 = v_add(v_x1, v_2);
v_W = v_muladd(v_M6, v_x1, v_W0);
v_W = v_select(v_W != v_zero, v_its / v_W, v_zero);
v_float64x2 v_fX1 = v_max(v_intmin, v_min(v_intmax, v_muladd(v_M0, v_x1, v_X0d) * v_W));
v_float64x2 v_fY1 = v_max(v_intmin, v_min(v_intmax, v_muladd(v_M3, v_x1, v_Y0d) * v_W));
v_x1 += v_2;
v_W = v_select(v_ne(v_W, v_zero), v_div(v_its, v_W), v_zero);
v_float64x2 v_fX1 = v_max(v_intmin, v_min(v_intmax, v_mul(v_muladd(v_M0, v_x1, v_X0d), v_W)));
v_float64x2 v_fY1 = v_max(v_intmin, v_min(v_intmax, v_mul(v_muladd(v_M3, v_x1, v_Y0d), v_W)));
v_x1 = v_add(v_x1, v_2);
v_X0 = v_round(v_fX0, v_fX1);
v_Y0 = v_round(v_fY0, v_fY1);
@@ -3006,16 +3006,16 @@ void WarpPerspectiveLine_Process_CV_SIMD(const double *M, short* xy, short* alph
v_int32x4 v_X1, v_Y1;
{
v_float64x2 v_W = v_muladd(v_M6, v_x1, v_W0);
v_W = v_select(v_W != v_zero, v_its / v_W, v_zero);
v_float64x2 v_fX0 = v_max(v_intmin, v_min(v_intmax, v_muladd(v_M0, v_x1, v_X0d) * v_W));
v_float64x2 v_fY0 = v_max(v_intmin, v_min(v_intmax, v_muladd(v_M3, v_x1, v_Y0d) * v_W));
v_x1 += v_2;
v_W = v_select(v_ne(v_W, v_zero), v_div(v_its, v_W), v_zero);
v_float64x2 v_fX0 = v_max(v_intmin, v_min(v_intmax, v_mul(v_muladd(v_M0, v_x1, v_X0d), v_W)));
v_float64x2 v_fY0 = v_max(v_intmin, v_min(v_intmax, v_mul(v_muladd(v_M3, v_x1, v_Y0d), v_W)));
v_x1 = v_add(v_x1, v_2);
v_W = v_muladd(v_M6, v_x1, v_W0);
v_W = v_select(v_W != v_zero, v_its / v_W, v_zero);
v_float64x2 v_fX1 = v_max(v_intmin, v_min(v_intmax, v_muladd(v_M0, v_x1, v_X0d) * v_W));
v_float64x2 v_fY1 = v_max(v_intmin, v_min(v_intmax, v_muladd(v_M3, v_x1, v_Y0d) * v_W));
v_x1 += v_2;
v_W = v_select(v_ne(v_W, v_zero), v_div(v_its, v_W), v_zero);
v_float64x2 v_fX1 = v_max(v_intmin, v_min(v_intmax, v_mul(v_muladd(v_M0, v_x1, v_X0d), v_W)));
v_float64x2 v_fY1 = v_max(v_intmin, v_min(v_intmax, v_mul(v_muladd(v_M3, v_x1, v_Y0d), v_W)));
v_x1 = v_add(v_x1, v_2);
v_X1 = v_round(v_fX0, v_fX1);
v_Y1 = v_round(v_fY0, v_fY1);
@@ -3025,16 +3025,16 @@ void WarpPerspectiveLine_Process_CV_SIMD(const double *M, short* xy, short* alph
v_int32x4 v_X2, v_Y2;
{
v_float64x2 v_W = v_muladd(v_M6, v_x1, v_W0);
v_W = v_select(v_W != v_zero, v_its / v_W, v_zero);
v_float64x2 v_fX0 = v_max(v_intmin, v_min(v_intmax, v_muladd(v_M0, v_x1, v_X0d) * v_W));
v_float64x2 v_fY0 = v_max(v_intmin, v_min(v_intmax, v_muladd(v_M3, v_x1, v_Y0d) * v_W));
v_x1 += v_2;
v_W = v_select(v_ne(v_W, v_zero), v_div(v_its, v_W), v_zero);
v_float64x2 v_fX0 = v_max(v_intmin, v_min(v_intmax, v_mul(v_muladd(v_M0, v_x1, v_X0d), v_W)));
v_float64x2 v_fY0 = v_max(v_intmin, v_min(v_intmax, v_mul(v_muladd(v_M3, v_x1, v_Y0d), v_W)));
v_x1 = v_add(v_x1, v_2);
v_W = v_muladd(v_M6, v_x1, v_W0);
v_W = v_select(v_W != v_zero, v_its / v_W, v_zero);
v_float64x2 v_fX1 = v_max(v_intmin, v_min(v_intmax, v_muladd(v_M0, v_x1, v_X0d) * v_W));
v_float64x2 v_fY1 = v_max(v_intmin, v_min(v_intmax, v_muladd(v_M3, v_x1, v_Y0d) * v_W));
v_x1 += v_2;
v_W = v_select(v_ne(v_W, v_zero), v_div(v_its, v_W), v_zero);
v_float64x2 v_fX1 = v_max(v_intmin, v_min(v_intmax, v_mul(v_muladd(v_M0, v_x1, v_X0d), v_W)));
v_float64x2 v_fY1 = v_max(v_intmin, v_min(v_intmax, v_mul(v_muladd(v_M3, v_x1, v_Y0d), v_W)));
v_x1 = v_add(v_x1, v_2);
v_X2 = v_round(v_fX0, v_fX1);
v_Y2 = v_round(v_fY0, v_fY1);
@@ -3044,35 +3044,35 @@ void WarpPerspectiveLine_Process_CV_SIMD(const double *M, short* xy, short* alph
v_int32x4 v_X3, v_Y3;
{
v_float64x2 v_W = v_muladd(v_M6, v_x1, v_W0);
v_W = v_select(v_W != v_zero, v_its / v_W, v_zero);
v_float64x2 v_fX0 = v_max(v_intmin, v_min(v_intmax, v_muladd(v_M0, v_x1, v_X0d) * v_W));
v_float64x2 v_fY0 = v_max(v_intmin, v_min(v_intmax, v_muladd(v_M3, v_x1, v_Y0d) * v_W));
v_x1 += v_2;
v_W = v_select(v_ne(v_W, v_zero), v_div(v_its, v_W), v_zero);
v_float64x2 v_fX0 = v_max(v_intmin, v_min(v_intmax, v_mul(v_muladd(v_M0, v_x1, v_X0d), v_W)));
v_float64x2 v_fY0 = v_max(v_intmin, v_min(v_intmax, v_mul(v_muladd(v_M3, v_x1, v_Y0d), v_W)));
v_x1 = v_add(v_x1, v_2);
v_W = v_muladd(v_M6, v_x1, v_W0);
v_W = v_select(v_W != v_zero, v_its / v_W, v_zero);
v_float64x2 v_fX1 = v_max(v_intmin, v_min(v_intmax, v_muladd(v_M0, v_x1, v_X0d) * v_W));
v_float64x2 v_fY1 = v_max(v_intmin, v_min(v_intmax, v_muladd(v_M3, v_x1, v_Y0d) * v_W));
v_x1 += v_2;
v_W = v_select(v_ne(v_W, v_zero), v_div(v_its, v_W), v_zero);
v_float64x2 v_fX1 = v_max(v_intmin, v_min(v_intmax, v_mul(v_muladd(v_M0, v_x1, v_X0d), v_W)));
v_float64x2 v_fY1 = v_max(v_intmin, v_min(v_intmax, v_mul(v_muladd(v_M3, v_x1, v_Y0d), v_W)));
v_x1 = v_add(v_x1, v_2);
v_X3 = v_round(v_fX0, v_fX1);
v_Y3 = v_round(v_fY0, v_fY1);
}
// store alpha
v_int32x4 v_alpha0 = ((v_Y0 & v_itsi1) << INTER_BITS) + (v_X0 & v_itsi1);
v_int32x4 v_alpha1 = ((v_Y1 & v_itsi1) << INTER_BITS) + (v_X1 & v_itsi1);
v_int32x4 v_alpha0 = v_add(v_shl<INTER_BITS>(v_and(v_Y0, v_itsi1)), v_and(v_X0, v_itsi1));
v_int32x4 v_alpha1 = v_add(v_shl<INTER_BITS>(v_and(v_Y1, v_itsi1)), v_and(v_X1, v_itsi1));
v_store((alpha + x1), v_pack(v_alpha0, v_alpha1));
v_alpha0 = ((v_Y2 & v_itsi1) << INTER_BITS) + (v_X2 & v_itsi1);
v_alpha1 = ((v_Y3 & v_itsi1) << INTER_BITS) + (v_X3 & v_itsi1);
v_alpha0 = v_add(v_shl<INTER_BITS>(v_and(v_Y2, v_itsi1)), v_and(v_X2, v_itsi1));
v_alpha1 = v_add(v_shl<INTER_BITS>(v_and(v_Y3, v_itsi1)), v_and(v_X3, v_itsi1));
v_store((alpha + x1 + 8), v_pack(v_alpha0, v_alpha1));
// convert to 16s
v_X0 = v_reinterpret_as_s32(v_pack(v_X0 >> INTER_BITS, v_X1 >> INTER_BITS));
v_X1 = v_reinterpret_as_s32(v_pack(v_X2 >> INTER_BITS, v_X3 >> INTER_BITS));
v_Y0 = v_reinterpret_as_s32(v_pack(v_Y0 >> INTER_BITS, v_Y1 >> INTER_BITS));
v_Y1 = v_reinterpret_as_s32(v_pack(v_Y2 >> INTER_BITS, v_Y3 >> INTER_BITS));
v_X0 = v_reinterpret_as_s32(v_pack(v_shr<INTER_BITS>(v_X0), v_shr<INTER_BITS>(v_X1)));
v_X1 = v_reinterpret_as_s32(v_pack(v_shr<INTER_BITS>(v_X2), v_shr<INTER_BITS>(v_X3)));
v_Y0 = v_reinterpret_as_s32(v_pack(v_shr<INTER_BITS>(v_Y0), v_shr<INTER_BITS>(v_Y1)));
v_Y1 = v_reinterpret_as_s32(v_pack(v_shr<INTER_BITS>(v_Y2), v_shr<INTER_BITS>(v_Y3)));
v_store_interleave(xy + x1 * 2, (v_reinterpret_as_s16)(v_X0), (v_reinterpret_as_s16)(v_Y0));
v_store_interleave(xy + x1 * 2 + 16, (v_reinterpret_as_s16)(v_X1), (v_reinterpret_as_s16)(v_Y1));
modules/imgproc/src/median_blur.simd.hpp
+21 -21
View File
@@ -179,10 +179,10 @@ medianBlur_8u_O1( const Mat& _src, Mat& _dst, int ksize )
for (k = 0; k < 16; ++k)
{
#if CV_SIMD256
v_store(H.fine[k], v_mul_wrap(v256_load(h_fine + 16 * n*(16 * c + k)), v256_setall_u16(2 * r + 1)) + v256_load(H.fine[k]));
v_store(H.fine[k], v_mul_wrap(v256_load(h_fine + 16 * n*(16 * c + k)), v_add(v256_setall_u16(2 * r + 1), v256_load(H.fine[k]))));
#elif CV_SIMD128
v_store(H.fine[k], v_mul_wrap(v_load(h_fine + 16 * n*(16 * c + k)), v_setall_u16((ushort)(2 * r + 1))) + v_load(H.fine[k]));
v_store(H.fine[k] + 8, v_mul_wrap(v_load(h_fine + 16 * n*(16 * c + k) + 8), v_setall_u16((ushort)(2 * r + 1))) + v_load(H.fine[k] + 8));
v_store(H.fine[k], v_add(v_mul_wrap(v_load(h_fine + 16 * n * (16 * c + k)), v_setall_u16((ushort)(2 * r + 1))), v_load(H.fine[k])));
v_store(H.fine[k] + 8, v_add(v_mul_wrap(v_load(h_fine + 16 * n * (16 * c + k) + 8), v_setall_u16((ushort)(2 * r + 1))), v_load(H.fine[k] + 8)));
#else
for (int ind = 0; ind < 16; ++ind)
H.fine[k][ind] = (HT)(H.fine[k][ind] + (2 * r + 1) * h_fine[16 * n*(16 * c + k) + ind]);
@@ -199,10 +199,10 @@ medianBlur_8u_O1( const Mat& _src, Mat& _dst, int ksize )
for( j = 0; j < 2*r; ++j, px += 16 )
{
#if CV_SIMD256
v_coarse += v256_load(px);
v_coarse = v_add(v_coarse, v256_load(px));
#elif CV_SIMD128
v_coarsel += v_load(px);
v_coarseh += v_load(px + 8);
v_coarsel = v_add(v_coarsel, v_load(px));
v_coarseh = v_add(v_coarseh, v_load(px + 8));
#else
for (int ind = 0; ind < 16; ++ind)
H.coarse[ind] += px[ind];
@@ -216,11 +216,11 @@ medianBlur_8u_O1( const Mat& _src, Mat& _dst, int ksize )
px = h_coarse + 16 * (n*c + std::min(j + r, n - 1));
#if CV_SIMD256
v_coarse += v256_load(px);
v_coarse = v_add(v_coarse, v256_load(px));
v_store(H.coarse, v_coarse);
#elif CV_SIMD128
v_coarsel += v_load(px);
v_coarseh += v_load(px + 8);
v_coarsel = v_add(v_coarsel, v_load(px));
v_coarseh = v_add(v_coarseh, v_load(px + 8));
v_store(H.coarse, v_coarsel);
v_store(H.coarse + 8, v_coarseh);
#else
@@ -261,10 +261,10 @@ medianBlur_8u_O1( const Mat& _src, Mat& _dst, int ksize )
for (luc[k] = HT(j - r); luc[k] < MIN(j + r + 1, n); ++luc[k], px += 16)
{
#if CV_SIMD256
v_fine += v256_load(px);
v_fine = v_add(v_fine, v256_load(px));
#elif CV_SIMD128
v_finel += v_load(px);
v_fineh += v_load(px + 8);
v_finel = v_add(v_finel, v_load(px));
v_fineh = v_add(v_fineh, v_load(px + 8));
#else
for (int ind = 0; ind < 16; ++ind)
H.fine[k][ind] += px[ind];
@@ -275,10 +275,10 @@ medianBlur_8u_O1( const Mat& _src, Mat& _dst, int ksize )
{
px = h_fine + 16 * (n*(16 * c + k) + (n - 1));
#if CV_SIMD256
v_fine += v_mul_wrap(v256_load(px), v256_setall_u16(j + r + 1 - n));
v_fine = v_add(v_fine, v_mul_wrap(v256_load(px), v256_setall_u16(j + r + 1 - n)));
#elif CV_SIMD128
v_finel += v_mul_wrap(v_load(px), v_setall_u16((ushort)(j + r + 1 - n)));
v_fineh += v_mul_wrap(v_load(px + 8), v_setall_u16((ushort)(j + r + 1 - n)));
v_finel = v_add(v_finel, v_mul_wrap(v_load(px), v_setall_u16((ushort)(j + r + 1 - n))));
v_fineh = v_add(v_fineh, v_mul_wrap(v_load(px + 8), v_setall_u16((ushort)(j + r + 1 - n))));
#else
for (int ind = 0; ind < 16; ++ind)
H.fine[k][ind] = (HT)(H.fine[k][ind] + (j + r + 1 - n) * px[ind]);
@@ -298,10 +298,10 @@ medianBlur_8u_O1( const Mat& _src, Mat& _dst, int ksize )
for ( ; luc[k] < j+r+1; ++luc[k] )
{
#if CV_SIMD256
v_fine = v_fine + v256_load(px + 16 * MIN(luc[k], n - 1)) - v256_load(px + 16 * MAX(luc[k] - 2 * r - 1, 0));
v_fine = v_sub(v_add(v_fine, v256_load(px + 16 * MIN(luc[k], n - 1))), v256_load(px + 16 * MAX(luc[k] - 2 * r - 1, 0)));
#elif CV_SIMD128
v_finel = v_finel + v_load(px + 16 * MIN(luc[k], n - 1) ) - v_load(px + 16 * MAX(luc[k] - 2 * r - 1, 0));
v_fineh = v_fineh + v_load(px + 16 * MIN(luc[k], n - 1) + 8) - v_load(px + 16 * MAX(luc[k] - 2 * r - 1, 0) + 8);
v_finel = v_sub(v_add(v_finel, v_load(px + 16 * MIN(luc[k], n - 1) )), v_load(px + 16 * MAX(luc[k] - 2 * r - 1, 0)));
v_fineh = v_sub(v_add(v_fineh, v_load(px + 16 * MIN(luc[k], n - 1) + 8)), v_load(px + 16 * MAX(luc[k] - 2 * r - 1, 0) + 8));
#else
for (int ind = 0; ind < 16; ++ind)
H.fine[k][ind] += px[16 * MIN(luc[k], n - 1) + ind] - px[16 * MAX(luc[k] - 2 * r - 1, 0) + ind];
@@ -312,12 +312,12 @@ medianBlur_8u_O1( const Mat& _src, Mat& _dst, int ksize )
px = h_coarse + 16 * (n*c + MAX(j - r, 0));
#if CV_SIMD256
v_store(H.fine[k], v_fine);
v_coarse -= v256_load(px);
v_coarse = v_sub(v_coarse, v256_load(px));
#elif CV_SIMD128
v_store(H.fine[k], v_finel);
v_store(H.fine[k] + 8, v_fineh);
v_coarsel -= v_load(px);
v_coarseh -= v_load(px + 8);
v_coarsel = v_sub(v_coarsel, v_load(px));
v_coarseh = v_sub(v_coarseh, v_load(px + 8));
#else
for (int ind = 0; ind < 16; ++ind)
H.coarse[ind] -= px[ind];
modules/imgproc/src/moments.cpp
+10 -10
View File
@@ -236,12 +236,12 @@ struct MomentsInTile_SIMD<uchar, int, int>
v_int16x8 p = v_reinterpret_as_s16(v_load_expand(ptr + x));
v_int16x8 sx = v_mul_wrap(qx, qx);
qx0 += v_reinterpret_as_u32(p);
qx0 = v_add(qx0, v_reinterpret_as_u32(p));
qx1 = v_reinterpret_as_u32(v_dotprod(p, qx, v_reinterpret_as_s32(qx1)));
qx2 = v_reinterpret_as_u32(v_dotprod(p, sx, v_reinterpret_as_s32(qx2)));
qx3 = v_reinterpret_as_u32(v_dotprod(v_mul_wrap(p, qx), sx, v_reinterpret_as_s32(qx3)));
qx += dx;
qx = v_add(qx, dx);
}
x0 = v_reduce_sum(qx0);
@@ -276,19 +276,19 @@ struct MomentsInTile_SIMD<ushort, int, int64>
{
v_int32x4 v_src = v_reinterpret_as_s32(v_load_expand(ptr + x));
v_x0 += v_reinterpret_as_u32(v_src);
v_x1 += v_reinterpret_as_u32(v_src * v_ix0);
v_x0 = v_add(v_x0, v_reinterpret_as_u32(v_src));
v_x1 = v_add(v_x1, v_reinterpret_as_u32(v_mul(v_src, v_ix0)));
v_int32x4 v_ix1 = v_ix0 * v_ix0;
v_x2 += v_reinterpret_as_u32(v_src * v_ix1);
v_int32x4 v_ix1 = v_mul(v_ix0, v_ix0);
v_x2 = v_add(v_x2, v_reinterpret_as_u32(v_mul(v_src, v_ix1)));
v_ix1 = v_ix0 * v_ix1;
v_src = v_src * v_ix1;
v_ix1 = v_mul(v_ix0, v_ix1);
v_src = v_mul(v_src, v_ix1);
v_uint64x2 v_lo, v_hi;
v_expand(v_reinterpret_as_u32(v_src), v_lo, v_hi);
v_x3 += v_lo + v_hi;
v_x3 = v_add(v_x3, v_add(v_lo, v_hi));
v_ix0 += v_delta;
v_ix0 = v_add(v_ix0, v_delta);
}
x0 = v_reduce_sum(v_x0);
modules/imgproc/src/pyramids.cpp
+7 -7
View File
@@ -463,7 +463,7 @@ template<> int PyrDownVecV<int, uchar>(int** src, uchar* dst, int width)
}
#if CV_SIMD128
typedef int CV_DECL_ALIGNED(1) unaligned_int;
for ( ; x <= width - v_int32x4::nlanes; x += v_int32x4::nlanes)
for ( ; x <= width - VTraits<v_int32x4>::vlanes(); x += VTraits<v_int32x4>::vlanes())
{
v_int32x4 r0, r1, r2, r3, r4, t0;
r0 = v_load(row0 + x);
@@ -473,7 +473,7 @@ template<> int PyrDownVecV<int, uchar>(int** src, uchar* dst, int width)
r4 = v_load(row4 + x);
t0 = v_add(v_add(v_add(r0, r4), v_add(r2, r2)), v_shl<2>(v_add(v_add(r1, r3), r2)));
*((unaligned_int*) (dst + x)) = v_reinterpret_as_s32(v_rshr_pack<8>(v_pack_u(t0, t0), v_setzero_u16())).get0();
*((unaligned_int*) (dst + x)) = v_get0(v_reinterpret_as_s32(v_rshr_pack<8>(v_pack_u(t0, t0), v_setzero_u16())));
}
#else
for (; x <= width - 1; x += 1)
@@ -615,15 +615,15 @@ template <> int PyrUpVecV<int, uchar>(int** src, uchar** dst, int width)
}
#if CV_SIMD128
typedef int CV_DECL_ALIGNED(1) unaligned_int;
for (; x <= width - v_int32x4::nlanes; x += v_int32x4::nlanes)
for (; x <= width - VTraits<v_int32x4>::vlanes(); x += VTraits<v_int32x4>::vlanes())
{
v_int32 v_r00 = vx_load(row0 + x),
v_r10 = vx_load(row1 + x),
v_r20 = vx_load(row2 + x);
v_int32 v_2r10 = v_add(v_r10, v_r10);
v_int16 d = v_pack(v_add(v_add(v_r00, v_r20), v_add(v_add(v_2r10, v_2r10), v_2r10)), v_shl<2>(v_add(v_r10, v_r20)));
*(unaligned_int*)(dst0 + x) = v_reinterpret_as_s32(v_rshr_pack_u<6>(d, vx_setzero_s16())).get0();
*(unaligned_int*)(dst1 + x) = v_reinterpret_as_s32(v_rshr_pack_u<6>(v_combine_high(d, d), vx_setzero_s16())).get0();
*(unaligned_int*)(dst0 + x) = v_get0(v_reinterpret_as_s32(v_rshr_pack_u<6>(d, vx_setzero_s16())));
*(unaligned_int*)(dst1 + x) = v_get0(v_reinterpret_as_s32(v_rshr_pack_u<6>(v_combine_high(d, d), vx_setzero_s16())));
}
#else
for (; x <= width - 1; x += 1)
@@ -754,14 +754,14 @@ template <> int PyrUpVecVOneRow<int, uchar>(int** src, uchar* dst, int width)
}
#if CV_SIMD128
typedef int CV_DECL_ALIGNED(1) unaligned_int;
for (; x <= width - v_int32x4::nlanes; x += v_int32x4::nlanes)
for (; x <= width - VTraits<v_int32x4>::vlanes(); x += VTraits<v_int32x4>::vlanes())
{
v_int32 v_r00 = vx_load(row0 + x),
v_r10 = vx_load(row1 + x),
v_r20 = vx_load(row2 + x);
v_int32 v_2r10 = v_add(v_r10, v_r10);
v_int16 d = v_pack(v_add(v_add(v_r00, v_r20), v_add(v_add(v_2r10, v_2r10), v_2r10)), v_shl<2>(v_add(v_r10, v_r20)));
*(unaligned_int*)(dst + x) = v_reinterpret_as_s32(v_rshr_pack_u<6>(d, vx_setzero_s16())).get0();
*(unaligned_int*)(dst + x) = v_get0(v_reinterpret_as_s32(v_rshr_pack_u<6>(d, vx_setzero_s16())));
}
#else
for (; x <= width - 1; x += 1)
modules/imgproc/src/resize.cpp
+17 -17
View File
@@ -2473,7 +2473,7 @@ public:
v_zip(s0, s3, t0, t1); v_zip(s1, s4, t2, t3); v_zip(s2, s5, t4, t5);
v_uint16 bl, gl, rl;
#if CV_SIMD_WIDTH == 16
bl = t0 + t3; gl = t1 + t4; rl = t2 + t5;
bl = v_add(t0, t3); gl = v_add(t1, t4); rl = v_add(t2, t5);
#elif CV_SIMD_WIDTH == 32
v_zip(t0, t3, s0, s1); v_zip(t1, t4, s2, s3); v_zip(t2, t5, s4, s5);
bl = v_add(s0, s3); gl = v_add(s1, s4); rl = v_add(s2, s5);
@@ -2493,7 +2493,7 @@ public:
v_zip(s0, s3, t0, t1); v_zip(s1, s4, t2, t3); v_zip(s2, s5, t4, t5);
v_uint16 bh, gh, rh;
#if CV_SIMD_WIDTH == 16
bh = t0 + t3; gh = t1 + t4; rh = t2 + t5;
bh = v_add(t0, t3); gh = v_add(t1, t4); rh = v_add(t2, t5);
#elif CV_SIMD_WIDTH == 32
v_zip(t0, t3, s0, s1); v_zip(t1, t4, s2, s3); v_zip(t2, t5, s4, s5);
bh = v_add(s0, s3); gh = v_add(s1, s4); rh = v_add(s2, s5);
@@ -2566,7 +2566,7 @@ public:
v_rshr_pack_store<2>(D, r0 + v_rotate_left<1>(r1, r0));
}
#else
v_rshr_pack_store<2>(D, v_load_expand(S0) + v_load_expand(S0 + 3) + v_load_expand(S1) + v_load_expand(S1 + 3));
v_rshr_pack_store<2>(D, v_add(v_add(v_add(v_load_expand(S0), v_load_expand(S0 + 3)), v_load_expand(S1)), v_load_expand(S1 + 3)));
#endif
#elif CV_SIMD_WIDTH == 32 || CV_SIMD_WIDTH == 64
for ( ; dx <= w - 3*VTraits<v_uint16>::vlanes(); dx += 3*VTraits<v_uint16>::vlanes(), S0 += 6*VTraits<v_uint16>::vlanes(), S1 += 6*VTraits<v_uint16>::vlanes(), D += 3*VTraits<v_uint16>::vlanes())
@@ -2609,7 +2609,7 @@ public:
}
#elif CV_SIMD_WIDTH >= 64
v_uint32 masklow = vx_setall_u32(0x0000ffff);
for ( ; dx <= w - 3*v_uint16::nlanes; dx += 3*v_uint16::nlanes, S0 += 6*v_uint16::nlanes, S1 += 6*v_uint16::nlanes, D += 3*v_uint16::nlanes)
for ( ; dx <= w - 3*VTraits<v_uint16>::vlanes(); dx += 3*VTraits<v_uint16>::vlanes(), S0 += 6*VTraits<v_uint16>::vlanes(), S1 += 6*VTraits<v_uint16>::vlanes(), D += 3*VTraits<v_uint16>::vlanes())
{
v_uint16 b0, g0, r0, b1, g1, r1;
v_load_deinterleave(S0, b0, g0, r0);
@@ -2617,8 +2617,8 @@ public:
v_uint32 bl = (v_reinterpret_as_u32(b0) >> 16) + (v_reinterpret_as_u32(b0) & masklow) + (v_reinterpret_as_u32(b1) >> 16) + (v_reinterpret_as_u32(b1) & masklow);
v_uint32 gl = (v_reinterpret_as_u32(g0) >> 16) + (v_reinterpret_as_u32(g0) & masklow) + (v_reinterpret_as_u32(g1) >> 16) + (v_reinterpret_as_u32(g1) & masklow);
v_uint32 rl = (v_reinterpret_as_u32(r0) >> 16) + (v_reinterpret_as_u32(r0) & masklow) + (v_reinterpret_as_u32(r1) >> 16) + (v_reinterpret_as_u32(r1) & masklow);
v_load_deinterleave(S0 + 3*v_uint16::nlanes, b0, g0, r0);
v_load_deinterleave(S1 + 3*v_uint16::nlanes, b1, g1, r1);
v_load_deinterleave(S0 + 3*VTraits<v_uint16>::vlanes(), b0, g0, r0);
v_load_deinterleave(S1 + 3*VTraits<v_uint16>::vlanes(), b1, g1, r1);
v_uint32 bh = (v_reinterpret_as_u32(b0) >> 16) + (v_reinterpret_as_u32(b0) & masklow) + (v_reinterpret_as_u32(b1) >> 16) + (v_reinterpret_as_u32(b1) & masklow);
v_uint32 gh = (v_reinterpret_as_u32(g0) >> 16) + (v_reinterpret_as_u32(g0) & masklow) + (v_reinterpret_as_u32(g1) >> 16) + (v_reinterpret_as_u32(g1) & masklow);
v_uint32 rh = (v_reinterpret_as_u32(r0) >> 16) + (v_reinterpret_as_u32(r0) & masklow) + (v_reinterpret_as_u32(r1) >> 16) + (v_reinterpret_as_u32(r1) & masklow);
@@ -2630,7 +2630,7 @@ public:
{
CV_Assert(cn == 4);
#if CV_SIMD_WIDTH >= 64
for ( ; dx <= w - v_uint16::nlanes; dx += v_uint16::nlanes, S0 += 2*v_uint16::nlanes, S1 += 2*v_uint16::nlanes, D += v_uint16::nlanes)
for ( ; dx <= w - VTraits<v_uint16>::vlanes(); dx += VTraits<v_uint16>::vlanes(), S0 += 2*VTraits<v_uint16>::vlanes(), S1 += 2*VTraits<v_uint16>::vlanes(), D += VTraits<v_uint16>::vlanes())
{
v_uint64 r00, r01, r10, r11;
v_load_deinterleave((uint64_t*)S0, r00, r01);
@@ -2652,7 +2652,7 @@ public:
r0 = v_add(r0, r2); r1 = v_add(r1, r3);
v_uint32 v_d;
#if CV_SIMD_WIDTH == 16
v_d = r0 + r1;
v_d = v_add(r0, r1);
#elif CV_SIMD_WIDTH == 32
v_uint32 t0, t1;
v_recombine(r0, r1, t0, t1);
@@ -2697,7 +2697,7 @@ public:
{
#if CV_SIMD_WIDTH == 16
for ( ; dx <= w - 4; dx += 3, S0 += 6, S1 += 6, D += 3)
v_rshr_pack_store<2>(D, v_load_expand(S0) + v_load_expand(S0 + 3) + v_load_expand(S1) + v_load_expand(S1 + 3));
v_rshr_pack_store<2>(D, v_add(v_add(v_add(v_load_expand(S0), v_load_expand(S0 + 3)), v_load_expand(S1)), v_load_expand(S1 + 3)));
#elif CV_SIMD_WIDTH == 32 || CV_SIMD_WIDTH == 64
for ( ; dx <= w - 3*VTraits<v_int16>::vlanes(); dx += 3*VTraits<v_int16>::vlanes(), S0 += 6*VTraits<v_int16>::vlanes(), S1 += 6*VTraits<v_int16>::vlanes(), D += 3*VTraits<v_int16>::vlanes())
{
@@ -2738,7 +2738,7 @@ public:
v_store_interleave(D, v_rshr_pack<2>(bl, bh), v_rshr_pack<2>(gl, gh), v_rshr_pack<2>(rl, rh));
}
#elif CV_SIMD_WIDTH >= 64
for ( ; dx <= w - 3*v_int16::nlanes; dx += 3*v_int16::nlanes, S0 += 6*v_int16::nlanes, S1 += 6*v_int16::nlanes, D += 3*v_int16::nlanes)
for ( ; dx <= w - 3*VTraits<v_int16>::vlanes(); dx += 3*VTraits<v_int16>::vlanes(), S0 += 6*VTraits<v_int16>::vlanes(), S1 += 6*VTraits<v_int16>::vlanes(), D += 3*VTraits<v_int16>::vlanes())
{
v_int16 b0, g0, r0, b1, g1, r1;
v_load_deinterleave(S0, b0, g0, r0);
@@ -2746,8 +2746,8 @@ public:
v_int32 bl = (v_reinterpret_as_s32(b0) >> 16) + ((v_reinterpret_as_s32(b0) << 16) >> 16) + (v_reinterpret_as_s32(b1) >> 16) + ((v_reinterpret_as_s32(b1) << 16) >> 16);
v_int32 gl = (v_reinterpret_as_s32(g0) >> 16) + ((v_reinterpret_as_s32(g0) << 16) >> 16) + (v_reinterpret_as_s32(g1) >> 16) + ((v_reinterpret_as_s32(g1) << 16) >> 16);
v_int32 rl = (v_reinterpret_as_s32(r0) >> 16) + ((v_reinterpret_as_s32(r0) << 16) >> 16) + (v_reinterpret_as_s32(r1) >> 16) + ((v_reinterpret_as_s32(r1) << 16) >> 16);
v_load_deinterleave(S0 + 3*v_int16::nlanes, b0, g0, r0);
v_load_deinterleave(S1 + 3*v_int16::nlanes, b1, g1, r1);
v_load_deinterleave(S0 + 3*VTraits<v_int16>::vlanes(), b0, g0, r0);
v_load_deinterleave(S1 + 3*VTraits<v_int16>::vlanes(), b1, g1, r1);
v_int32 bh = (v_reinterpret_as_s32(b0) >> 16) + ((v_reinterpret_as_s32(b0) << 16) >> 16) + (v_reinterpret_as_s32(b1) >> 16) + ((v_reinterpret_as_s32(b1) << 16) >> 16);
v_int32 gh = (v_reinterpret_as_s32(g0) >> 16) + ((v_reinterpret_as_s32(g0) << 16) >> 16) + (v_reinterpret_as_s32(g1) >> 16) + ((v_reinterpret_as_s32(g1) << 16) >> 16);
v_int32 rh = (v_reinterpret_as_s32(r0) >> 16) + ((v_reinterpret_as_s32(r0) << 16) >> 16) + (v_reinterpret_as_s32(r1) >> 16) + ((v_reinterpret_as_s32(r1) << 16) >> 16);
@@ -2779,7 +2779,7 @@ public:
r3 = v_add(vx_load_expand(S0 + 3 * VTraits<v_int32>::vlanes()), vx_load_expand(S1 + 3 * VTraits<v_int32>::vlanes()));
v_int32 dl, dh;
#if CV_SIMD_WIDTH == 16
dl = r0 + r1; dh = r2 + r3;
dl = v_add(r0, r1); dh = v_add(r2, r3);
#elif CV_SIMD_WIDTH == 32
v_int32 t0, t1, t2, t3;
v_recombine(r0, r1, t0, t1); v_recombine(r2, r3, t2, t3);
@@ -2829,14 +2829,14 @@ struct ResizeAreaFastVec_SIMD_32f
{
#if CV_SIMD_WIDTH == 16
v_float32 v_025 = vx_setall_f32(0.25f);
for (; dx <= w - v_float32::nlanes; dx += v_float32::nlanes, S0 += 2*v_float32::nlanes, S1 += 2*v_float32::nlanes, D += v_float32::nlanes)
v_store(D, ((vx_load(S0) + vx_load(S0 + v_float32::nlanes)) + (vx_load(S1) + vx_load(S1 + v_float32::nlanes))) * v_025);
for (; dx <= w - VTraits<v_float32>::vlanes(); dx += VTraits<v_float32>::vlanes(), S0 += 2*VTraits<v_float32>::vlanes(), S1 += 2*VTraits<v_float32>::vlanes(), D += VTraits<v_float32>::vlanes())
v_store(D, v_mul(v_add(v_add(vx_load(S0), vx_load(S0 + VTraits<v_float32>::vlanes())), v_add(vx_load(S1), vx_load(S1 + VTraits<v_float32>::vlanes()))), v_025));
#elif CV_SIMD256
v_float32x8 v_025 = v256_setall_f32(0.25f);
for (; dx <= w - v_float32x8::nlanes; dx += v_float32x8::nlanes, S0 += 2*v_float32x8::nlanes, S1 += 2*v_float32x8::nlanes, D += v_float32x8::nlanes)
for (; dx <= w - VTraits<v_float32x8>::vlanes(); dx += VTraits<v_float32x8>::vlanes(), S0 += 2*VTraits<v_float32x8>::vlanes(), S1 += 2*VTraits<v_float32x8>::vlanes(), D += VTraits<v_float32x8>::vlanes())
{
v_float32x8 dst0, dst1;
v_recombine(v_add(v256_load(S0), v256_load(S1)), v_add(v256_load(S0 + v_float32x8::nlanes), v256_load(S1 + v_float32x8::nlanes)), dst0, dst1);
v_recombine(v_add(v256_load(S0), v256_load(S1)), v_add(v256_load(S0 + VTraits<v_float32x8>::vlanes()), v256_load(S1 + VTraits<v_float32x8>::vlanes())), dst0, dst1);
v_store(D, v_mul(v_add(dst0, dst1), v_025));
}
#endif
modules/imgproc/src/sumpixels.simd.hpp
+234 -234
View File
@@ -114,7 +114,7 @@ struct Integral_SIMD<uchar, int, double>
v_int32 prev = vx_setzero_s32();
int j = 0;
for ( ; j + v_uint16::nlanes <= width; j += v_uint16::nlanes)
for ( ; j + VTraits<v_uint16>::vlanes() <= width; j += VTraits<v_uint16>::vlanes())
{
v_int16 el8 = v_reinterpret_as_s16(vx_load_expand(src_row + j));
v_int32 el4l, el4h;
@@ -127,8 +127,8 @@ struct Integral_SIMD<uchar, int, double>
el4h.val = _mm256_add_epi32(_mm256_cvtepi16_epi32(_v256_extract_high(vsum)), _mm256_permutevar8x32_epi32(el4l.val, shmask));
prev.val = _mm256_permutevar8x32_epi32(el4h.val, shmask);
#else
el8 += v_rotate_left<1>(el8);
el8 += v_rotate_left<2>(el8);
el8 = v_add(el8, v_rotate_left<1>(el8));
el8 = v_add(el8, v_rotate_left<2>(el8));
#if CV_SIMD_WIDTH >= 32
el8 += v_rotate_left<4>(el8);
#if CV_SIMD_WIDTH == 64
@@ -136,12 +136,12 @@ struct Integral_SIMD<uchar, int, double>
#endif
#endif
v_expand(el8, el4l, el4h);
el4l += prev;
el4h += el4l;
prev = v_broadcast_element<v_int32::nlanes - 1>(el4h);
el4l = v_add(el4l, prev);
el4h = v_add(el4h, el4l);
prev = v_broadcast_highest(el4h);
#endif
v_store(sum_row + j , el4l + vx_load(prev_sum_row + j ));
v_store(sum_row + j + v_int32::nlanes, el4h + vx_load(prev_sum_row + j + v_int32::nlanes));
v_store(sum_row + j , v_add(el4l, vx_load(prev_sum_row + j)));
v_store(sum_row + j + VTraits<v_int32>::vlanes(), v_add(el4h, vx_load(prev_sum_row + j + VTraits<v_int32>::vlanes())));
}
for (int v = sum_row[j - 1] - prev_sum_row[j - 1]; j < width; ++j)
@@ -162,11 +162,11 @@ struct Integral_SIMD<uchar, int, double>
v_int32 prev_1 = vx_setzero_s32(), prev_2 = vx_setzero_s32();
int j = 0;
for ( ; j + v_uint16::nlanes * cn <= width; j += v_uint16::nlanes * cn)
for ( ; j + VTraits<v_uint16>::vlanes() * cn <= width; j += VTraits<v_uint16>::vlanes() * cn)
{
v_int16 v_src_row = v_reinterpret_as_s16(vx_load(src_row + j));
v_int16 el8_1 = v_src_row & mask;
v_int16 el8_2 = v_reinterpret_as_s16(v_reinterpret_as_u16(v_src_row) >> 8);
v_int16 el8_1 = v_and(v_src_row, mask);
v_int16 el8_2 = v_reinterpret_as_s16(v_shr<8>(v_reinterpret_as_u16(v_src_row)));
v_int32 el4l_1, el4h_1, el4l_2, el4h_2;
#if CV_AVX2 && CV_SIMD_WIDTH == 32
__m256i vsum_1 = _mm256_add_epi16(el8_1.val, _mm256_slli_si256(el8_1.val, 2));
@@ -183,10 +183,10 @@ struct Integral_SIMD<uchar, int, double>
prev_1.val = _mm256_permutevar8x32_epi32(el4h_1.val, shmask);
prev_2.val = _mm256_permutevar8x32_epi32(el4h_2.val, shmask);
#else
el8_1 += v_rotate_left<1>(el8_1);
el8_2 += v_rotate_left<1>(el8_2);
el8_1 += v_rotate_left<2>(el8_1);
el8_2 += v_rotate_left<2>(el8_2);
el8_1 = v_add(el8_1, v_rotate_left<1>(el8_1));
el8_2 = v_add(el8_2, v_rotate_left<1>(el8_2));
el8_1 = v_add(el8_1, v_rotate_left<2>(el8_1));
el8_2 = v_add(el8_2, v_rotate_left<2>(el8_2));
#if CV_SIMD_WIDTH >= 32
el8_1 += v_rotate_left<4>(el8_1);
el8_2 += v_rotate_left<4>(el8_2);
@@ -197,20 +197,20 @@ struct Integral_SIMD<uchar, int, double>
#endif
v_expand(el8_1, el4l_1, el4h_1);
v_expand(el8_2, el4l_2, el4h_2);
el4l_1 += prev_1;
el4l_2 += prev_2;
el4h_1 += el4l_1;
el4h_2 += el4l_2;
prev_1 = v_broadcast_element<v_int32::nlanes - 1>(el4h_1);
prev_2 = v_broadcast_element<v_int32::nlanes - 1>(el4h_2);
el4l_1 = v_add(el4l_1, prev_1);
el4l_2 = v_add(el4l_2, prev_2);
el4h_1 = v_add(el4h_1, el4l_1);
el4h_2 = v_add(el4h_2, el4l_2);
prev_1 = v_broadcast_highest(el4h_1);
prev_2 = v_broadcast_highest(el4h_2);
#endif
v_int32 el4_1, el4_2, el4_3, el4_4;
v_zip(el4l_1, el4l_2, el4_1, el4_2);
v_zip(el4h_1, el4h_2, el4_3, el4_4);
v_store(sum_row + j , el4_1 + vx_load(prev_sum_row + j ));
v_store(sum_row + j + v_int32::nlanes , el4_2 + vx_load(prev_sum_row + j + v_int32::nlanes ));
v_store(sum_row + j + v_int32::nlanes * 2, el4_3 + vx_load(prev_sum_row + j + v_int32::nlanes * 2));
v_store(sum_row + j + v_int32::nlanes * 3, el4_4 + vx_load(prev_sum_row + j + v_int32::nlanes * 3));
v_store(sum_row + j , v_add(el4_1, vx_load(prev_sum_row + j)));
v_store(sum_row + j + VTraits<v_int32>::vlanes() , v_add(el4_2, vx_load(prev_sum_row + j + VTraits<v_int32>::vlanes())));
v_store(sum_row + j + VTraits<v_int32>::vlanes() * 2, v_add(el4_3, vx_load(prev_sum_row + j + VTraits<v_int32>::vlanes() * 2)));
v_store(sum_row + j + VTraits<v_int32>::vlanes() * 3, v_add(el4_4, vx_load(prev_sum_row + j + VTraits<v_int32>::vlanes() * 3)));
}
for (int v2 = sum_row[j - 1] - prev_sum_row[j - 1],
@@ -230,7 +230,7 @@ struct Integral_SIMD<uchar, int, double>
const uchar * src_row = src + _srcstep * i;
int * prev_sum_row = (int *)((uchar *)sum + _sumstep * i) + cn;
int * sum_row = (int *)((uchar *)sum + _sumstep * (i + 1)) + cn;
int row_cache[v_int32::nlanes * 6];
int row_cache[VTraits<v_int32>::max_nlanes * 6];
sum_row[-1] = sum_row[-2] = sum_row[-3] = 0;
@@ -238,10 +238,10 @@ struct Integral_SIMD<uchar, int, double>
prev_3 = vx_setzero_s32();
int j = 0;
const int j_max =
((_srcstep * i + (width - v_uint16::nlanes * cn + v_uint8::nlanes * cn)) >= _srcstep * height)
? width - v_uint8::nlanes * cn // uint8 in v_load_deinterleave()
: width - v_uint16::nlanes * cn; // v_expand_low
for ( ; j <= j_max; j += v_uint16::nlanes * cn)
((_srcstep * i + (width - VTraits<v_uint16>::vlanes() * cn + VTraits<v_uint8>::vlanes() * cn)) >= _srcstep * height)
? width - VTraits<v_uint8>::vlanes() * cn // uint8 in v_load_deinterleave()
: width - VTraits<v_uint16>::vlanes() * cn; // v_expand_low
for ( ; j <= j_max; j += VTraits<v_uint16>::vlanes() * cn)
{
v_uint8 v_src_row_1, v_src_row_2, v_src_row_3;
v_load_deinterleave(src_row + j, v_src_row_1, v_src_row_2, v_src_row_3);
@@ -270,49 +270,49 @@ struct Integral_SIMD<uchar, int, double>
prev_2.val = _mm256_permutevar8x32_epi32(el4h_2.val, shmask);
prev_3.val = _mm256_permutevar8x32_epi32(el4h_3.val, shmask);
#else
el8_1 += v_rotate_left<1>(el8_1);
el8_2 += v_rotate_left<1>(el8_2);
el8_3 += v_rotate_left<1>(el8_3);
el8_1 += v_rotate_left<2>(el8_1);
el8_2 += v_rotate_left<2>(el8_2);
el8_3 += v_rotate_left<2>(el8_3);
el8_1 = v_add(el8_1,v_rotate_left<1>(el8_1));
el8_2 = v_add(el8_2,v_rotate_left<1>(el8_2));
el8_3 = v_add(el8_3,v_rotate_left<1>(el8_3));
el8_1 = v_add(el8_1,v_rotate_left<2>(el8_1));
el8_2 = v_add(el8_2,v_rotate_left<2>(el8_2));
el8_3 = v_add(el8_3,v_rotate_left<2>(el8_3));
#if CV_SIMD_WIDTH >= 32
el8_1 += v_rotate_left<4>(el8_1);
el8_2 += v_rotate_left<4>(el8_2);
el8_3 += v_rotate_left<4>(el8_3);
el8_1 = v_add(el8_1, v_rotate_left<4>(el8_1));
el8_2 = v_add(el8_2, v_rotate_left<4>(el8_2));
el8_3 = v_add(el8_3, v_rotate_left<4>(el8_3));
#if CV_SIMD_WIDTH == 64
el8_1 += v_rotate_left<8>(el8_1);
el8_2 += v_rotate_left<8>(el8_2);
el8_3 += v_rotate_left<8>(el8_3);
el8_1 = v_add(el8_1, v_rotate_left<8>(el8_1));
el8_2 = v_add(el8_2, v_rotate_left<8>(el8_2));
el8_3 = v_add(el8_3, v_rotate_left<8>(el8_3));
#endif
#endif
v_expand(el8_1, el4l_1, el4h_1);
v_expand(el8_2, el4l_2, el4h_2);
v_expand(el8_3, el4l_3, el4h_3);
el4l_1 += prev_1;
el4l_2 += prev_2;
el4l_3 += prev_3;
el4h_1 += el4l_1;
el4h_2 += el4l_2;
el4h_3 += el4l_3;
prev_1 = v_broadcast_element<v_int32::nlanes - 1>(el4h_1);
prev_2 = v_broadcast_element<v_int32::nlanes - 1>(el4h_2);
prev_3 = v_broadcast_element<v_int32::nlanes - 1>(el4h_3);
el4l_1 = v_add(el4l_1, prev_1);
el4l_2 = v_add(el4l_2, prev_2);
el4l_3 = v_add(el4l_3, prev_3);
el4h_1 = v_add(el4h_1, el4l_1);
el4h_2 = v_add(el4h_2, el4l_2);
el4h_3 = v_add(el4h_3, el4l_3);
prev_1 = v_broadcast_highest(el4h_1);
prev_2 = v_broadcast_highest(el4h_2);
prev_3 = v_broadcast_highest(el4h_3);
#endif
v_store_interleave(row_cache , el4l_1, el4l_2, el4l_3);
v_store_interleave(row_cache + v_int32::nlanes * 3, el4h_1, el4h_2, el4h_3);
v_store_interleave(row_cache + VTraits<v_int32>::vlanes() * 3, el4h_1, el4h_2, el4h_3);
el4l_1 = vx_load(row_cache );
el4l_2 = vx_load(row_cache + v_int32::nlanes );
el4l_3 = vx_load(row_cache + v_int32::nlanes * 2);
el4h_1 = vx_load(row_cache + v_int32::nlanes * 3);
el4h_2 = vx_load(row_cache + v_int32::nlanes * 4);
el4h_3 = vx_load(row_cache + v_int32::nlanes * 5);
v_store(sum_row + j , el4l_1 + vx_load(prev_sum_row + j ));
v_store(sum_row + j + v_int32::nlanes , el4l_2 + vx_load(prev_sum_row + j + v_int32::nlanes ));
v_store(sum_row + j + v_int32::nlanes * 2, el4l_3 + vx_load(prev_sum_row + j + v_int32::nlanes * 2));
v_store(sum_row + j + v_int32::nlanes * 3, el4h_1 + vx_load(prev_sum_row + j + v_int32::nlanes * 3));
v_store(sum_row + j + v_int32::nlanes * 4, el4h_2 + vx_load(prev_sum_row + j + v_int32::nlanes * 4));
v_store(sum_row + j + v_int32::nlanes * 5, el4h_3 + vx_load(prev_sum_row + j + v_int32::nlanes * 5));
el4l_2 = vx_load(row_cache + VTraits<v_int32>::vlanes() );
el4l_3 = vx_load(row_cache + VTraits<v_int32>::vlanes() * 2);
el4h_1 = vx_load(row_cache + VTraits<v_int32>::vlanes() * 3);
el4h_2 = vx_load(row_cache + VTraits<v_int32>::vlanes() * 4);
el4h_3 = vx_load(row_cache + VTraits<v_int32>::vlanes() * 5);
v_store(sum_row + j , v_add(el4l_1, vx_load(prev_sum_row + j )));
v_store(sum_row + j + VTraits<v_int32>::vlanes() , v_add(el4l_2, vx_load(prev_sum_row + j + VTraits<v_int32>::vlanes() )));
v_store(sum_row + j + VTraits<v_int32>::vlanes() * 2, v_add(el4l_3, vx_load(prev_sum_row + j + VTraits<v_int32>::vlanes() * 2)));
v_store(sum_row + j + VTraits<v_int32>::vlanes() * 3, v_add(el4h_1, vx_load(prev_sum_row + j + VTraits<v_int32>::vlanes() * 3)));
v_store(sum_row + j + VTraits<v_int32>::vlanes() * 4, v_add(el4h_2, vx_load(prev_sum_row + j + VTraits<v_int32>::vlanes() * 4)));
v_store(sum_row + j + VTraits<v_int32>::vlanes() * 5, v_add(el4h_3, vx_load(prev_sum_row + j + VTraits<v_int32>::vlanes() * 5)));
}
for (int v3 = sum_row[j - 1] - prev_sum_row[j - 1],
@@ -339,7 +339,7 @@ struct Integral_SIMD<uchar, int, double>
v_int32 prev = vx_setzero_s32();
int j = 0;
for ( ; j + v_uint16::nlanes <= width; j += v_uint16::nlanes)
for ( ; j + VTraits<v_uint16>::vlanes() <= width; j += VTraits<v_uint16>::vlanes())
{
v_int16 el8 = v_reinterpret_as_s16(vx_load_expand(src_row + j));
v_int32 el4l, el4h;
@@ -356,8 +356,8 @@ struct Integral_SIMD<uchar, int, double>
#endif
#endif
v_expand(el8, el4l, el4h);
el4l += prev;
el4h += el4l;
el4l = v_add(el4l, prev);
el4h = v_add(el4h, el4l);
#if CV_SIMD_WIDTH == 16
prev = el4h;
#elif CV_SIMD_WIDTH == 32
@@ -368,8 +368,8 @@ struct Integral_SIMD<uchar, int, double>
prev = v_combine_low(t, t);
#endif
#endif
v_store(sum_row + j , el4l + vx_load(prev_sum_row + j ));
v_store(sum_row + j + v_int32::nlanes, el4h + vx_load(prev_sum_row + j + v_int32::nlanes));
v_store(sum_row + j , v_add(el4l, vx_load(prev_sum_row + j)));
v_store(sum_row + j + VTraits<v_int32>::vlanes(), v_add(el4h, vx_load(prev_sum_row + j + VTraits<v_int32>::vlanes())));
}
for (int v4 = sum_row[j - 1] - prev_sum_row[j - 1],
@@ -426,7 +426,7 @@ struct Integral_SIMD<uchar, float, double>
v_float32 prev = vx_setzero_f32();
int j = 0;
for (; j + v_uint16::nlanes <= width; j += v_uint16::nlanes)
for (; j + VTraits<v_uint16>::vlanes() <= width; j += VTraits<v_uint16>::vlanes())
{
v_int16 el8 = v_reinterpret_as_s16(vx_load_expand(src_row + j));
v_float32 el4l, el4h;
@@ -439,8 +439,8 @@ struct Integral_SIMD<uchar, float, double>
el4h.val = _mm256_add_ps(_mm256_cvtepi32_ps(_mm256_cvtepi16_epi32(_v256_extract_high(vsum))), _mm256_permutevar8x32_ps(el4l.val, shmask));
prev.val = _mm256_permutevar8x32_ps(el4h.val, shmask);
#else
el8 += v_rotate_left<1>(el8);
el8 += v_rotate_left<2>(el8);
el8 = v_add(el8, v_rotate_left<1>(el8));
el8 = v_add(el8, v_rotate_left<2>(el8));
#if CV_SIMD_WIDTH >= 32
el8 += v_rotate_left<4>(el8);
#if CV_SIMD_WIDTH == 64
@@ -449,12 +449,12 @@ struct Integral_SIMD<uchar, float, double>
#endif
v_int32 el4li, el4hi;
v_expand(el8, el4li, el4hi);
el4l = v_cvt_f32(el4li) + prev;
el4h = v_cvt_f32(el4hi) + el4l;
prev = v_broadcast_element<v_float32::nlanes - 1>(el4h);
el4l = v_add(v_cvt_f32(el4li), prev);
el4h = v_add(v_cvt_f32(el4hi), el4l);
prev = v_broadcast_highest(el4h);
#endif
v_store(sum_row + j , el4l + vx_load(prev_sum_row + j ));
v_store(sum_row + j + v_float32::nlanes, el4h + vx_load(prev_sum_row + j + v_float32::nlanes));
v_store(sum_row + j , v_add(el4l, vx_load(prev_sum_row + j)));
v_store(sum_row + j + VTraits<v_float32>::vlanes(), v_add(el4h, vx_load(prev_sum_row + j + VTraits<v_float32>::vlanes())));
}
for (float v = sum_row[j - 1] - prev_sum_row[j - 1]; j < width; ++j)
@@ -475,11 +475,11 @@ struct Integral_SIMD<uchar, float, double>
v_float32 prev_1 = vx_setzero_f32(), prev_2 = vx_setzero_f32();
int j = 0;
for (; j + v_uint16::nlanes * cn <= width; j += v_uint16::nlanes * cn)
for (; j + VTraits<v_uint16>::vlanes() * cn <= width; j += VTraits<v_uint16>::vlanes() * cn)
{
v_int16 v_src_row = v_reinterpret_as_s16(vx_load(src_row + j));
v_int16 el8_1 = v_src_row & mask;
v_int16 el8_2 = v_reinterpret_as_s16(v_reinterpret_as_u16(v_src_row) >> 8);
v_int16 el8_1 = v_and(v_src_row, mask);
v_int16 el8_2 = v_reinterpret_as_s16(v_shr<8>(v_reinterpret_as_u16(v_src_row)));
v_float32 el4l_1, el4h_1, el4l_2, el4h_2;
#if CV_AVX2 && CV_SIMD_WIDTH == 32
__m256i vsum_1 = _mm256_add_epi16(el8_1.val, _mm256_slli_si256(el8_1.val, 2));
@@ -496,10 +496,10 @@ struct Integral_SIMD<uchar, float, double>
prev_1.val = _mm256_permutevar8x32_ps(el4h_1.val, shmask);
prev_2.val = _mm256_permutevar8x32_ps(el4h_2.val, shmask);
#else
el8_1 += v_rotate_left<1>(el8_1);
el8_2 += v_rotate_left<1>(el8_2);
el8_1 += v_rotate_left<2>(el8_1);
el8_2 += v_rotate_left<2>(el8_2);
el8_1 = v_add(el8_1, v_rotate_left<1>(el8_1));
el8_2 = v_add(el8_2, v_rotate_left<1>(el8_2));
el8_1 = v_add(el8_1, v_rotate_left<2>(el8_1));
el8_2 = v_add(el8_2, v_rotate_left<2>(el8_2));
#if CV_SIMD_WIDTH >= 32
el8_1 += v_rotate_left<4>(el8_1);
el8_2 += v_rotate_left<4>(el8_2);
@@ -511,20 +511,20 @@ struct Integral_SIMD<uchar, float, double>
v_int32 el4li_1, el4hi_1, el4li_2, el4hi_2;
v_expand(el8_1, el4li_1, el4hi_1);
v_expand(el8_2, el4li_2, el4hi_2);
el4l_1 = v_cvt_f32(el4li_1) + prev_1;
el4l_2 = v_cvt_f32(el4li_2) + prev_2;
el4h_1 = v_cvt_f32(el4hi_1) + el4l_1;
el4h_2 = v_cvt_f32(el4hi_2) + el4l_2;
prev_1 = v_broadcast_element<v_float32::nlanes - 1>(el4h_1);
prev_2 = v_broadcast_element<v_float32::nlanes - 1>(el4h_2);
el4l_1 = v_add(v_cvt_f32(el4li_1), prev_1);
el4l_2 = v_add(v_cvt_f32(el4li_2), prev_2);
el4h_1 = v_add(v_cvt_f32(el4hi_1), el4l_1);
el4h_2 = v_add(v_cvt_f32(el4hi_2), el4l_2);
prev_1 = v_broadcast_highest(el4h_1);
prev_2 = v_broadcast_highest(el4h_2);
#endif
v_float32 el4_1, el4_2, el4_3, el4_4;
v_zip(el4l_1, el4l_2, el4_1, el4_2);
v_zip(el4h_1, el4h_2, el4_3, el4_4);
v_store(sum_row + j , el4_1 + vx_load(prev_sum_row + j ));
v_store(sum_row + j + v_float32::nlanes , el4_2 + vx_load(prev_sum_row + j + v_float32::nlanes ));
v_store(sum_row + j + v_float32::nlanes * 2, el4_3 + vx_load(prev_sum_row + j + v_float32::nlanes * 2));
v_store(sum_row + j + v_float32::nlanes * 3, el4_4 + vx_load(prev_sum_row + j + v_float32::nlanes * 3));
v_store(sum_row + j , v_add(el4_1, vx_load(prev_sum_row + j)));
v_store(sum_row + j + VTraits<v_float32>::vlanes() , v_add(el4_2, vx_load(prev_sum_row + j + VTraits<v_float32>::vlanes())));
v_store(sum_row + j + VTraits<v_float32>::vlanes() * 2, v_add(el4_3, vx_load(prev_sum_row + j + VTraits<v_float32>::vlanes() * 2)));
v_store(sum_row + j + VTraits<v_float32>::vlanes() * 3, v_add(el4_4, vx_load(prev_sum_row + j + VTraits<v_float32>::vlanes() * 3)));
}
for (float v2 = sum_row[j - 1] - prev_sum_row[j - 1],
@@ -543,7 +543,7 @@ struct Integral_SIMD<uchar, float, double>
const uchar * src_row = src + _srcstep * i;
float * prev_sum_row = (float *)((uchar *)sum + _sumstep * i) + cn;
float * sum_row = (float *)((uchar *)sum + _sumstep * (i + 1)) + cn;
float row_cache[v_float32::nlanes * 6];
float row_cache[VTraits<v_float32>::max_nlanes * 6];
sum_row[-1] = sum_row[-2] = sum_row[-3] = 0;
@@ -551,10 +551,10 @@ struct Integral_SIMD<uchar, float, double>
prev_3 = vx_setzero_f32();
int j = 0;
const int j_max =
((_srcstep * i + (width - v_uint16::nlanes * cn + v_uint8::nlanes * cn)) >= _srcstep * height)
? width - v_uint8::nlanes * cn // uint8 in v_load_deinterleave()
: width - v_uint16::nlanes * cn; // v_expand_low
for ( ; j <= j_max; j += v_uint16::nlanes * cn)
((_srcstep * i + (width - VTraits<v_uint16>::vlanes() * cn + VTraits<v_uint8>::vlanes() * cn)) >= _srcstep * height)
? width - VTraits<v_uint8>::vlanes() * cn // uint8 in v_load_deinterleave()
: width - VTraits<v_uint16>::vlanes() * cn; // v_expand_low
for ( ; j <= j_max; j += VTraits<v_uint16>::vlanes() * cn)
{
v_uint8 v_src_row_1, v_src_row_2, v_src_row_3;
v_load_deinterleave(src_row + j, v_src_row_1, v_src_row_2, v_src_row_3);
@@ -583,12 +583,12 @@ struct Integral_SIMD<uchar, float, double>
prev_2.val = _mm256_permutevar8x32_ps(el4h_2.val, shmask);
prev_3.val = _mm256_permutevar8x32_ps(el4h_3.val, shmask);
#else
el8_1 += v_rotate_left<1>(el8_1);
el8_2 += v_rotate_left<1>(el8_2);
el8_3 += v_rotate_left<1>(el8_3);
el8_1 += v_rotate_left<2>(el8_1);
el8_2 += v_rotate_left<2>(el8_2);
el8_3 += v_rotate_left<2>(el8_3);
el8_1 = v_add(el8_1, v_rotate_left<1>(el8_1));
el8_2 = v_add(el8_2, v_rotate_left<1>(el8_2));
el8_3 = v_add(el8_3, v_rotate_left<1>(el8_3));
el8_1 = v_add(el8_1, v_rotate_left<2>(el8_1));
el8_2 = v_add(el8_2, v_rotate_left<2>(el8_2));
el8_3 = v_add(el8_3, v_rotate_left<2>(el8_3));
#if CV_SIMD_WIDTH >= 32
el8_1 += v_rotate_left<4>(el8_1);
el8_2 += v_rotate_left<4>(el8_2);
@@ -603,30 +603,30 @@ struct Integral_SIMD<uchar, float, double>
v_expand(el8_1, el4li_1, el4hi_1);
v_expand(el8_2, el4li_2, el4hi_2);
v_expand(el8_3, el4li_3, el4hi_3);
el4l_1 = v_cvt_f32(el4li_1) + prev_1;
el4l_2 = v_cvt_f32(el4li_2) + prev_2;
el4l_3 = v_cvt_f32(el4li_3) + prev_3;
el4h_1 = v_cvt_f32(el4hi_1) + el4l_1;
el4h_2 = v_cvt_f32(el4hi_2) + el4l_2;
el4h_3 = v_cvt_f32(el4hi_3) + el4l_3;
prev_1 = v_broadcast_element<v_float32::nlanes - 1>(el4h_1);
prev_2 = v_broadcast_element<v_float32::nlanes - 1>(el4h_2);
prev_3 = v_broadcast_element<v_float32::nlanes - 1>(el4h_3);
el4l_1 = v_add(v_cvt_f32(el4li_1), prev_1);
el4l_2 = v_add(v_cvt_f32(el4li_2), prev_2);
el4l_3 = v_add(v_cvt_f32(el4li_3), prev_3);
el4h_1 = v_add(v_cvt_f32(el4hi_1), el4l_1);
el4h_2 = v_add(v_cvt_f32(el4hi_2), el4l_2);
el4h_3 = v_add(v_cvt_f32(el4hi_3), el4l_3);
prev_1 = v_broadcast_highest(el4h_1);
prev_2 = v_broadcast_highest(el4h_2);
prev_3 = v_broadcast_highest(el4h_3);
#endif
v_store_interleave(row_cache , el4l_1, el4l_2, el4l_3);
v_store_interleave(row_cache + v_float32::nlanes * 3, el4h_1, el4h_2, el4h_3);
v_store_interleave(row_cache + VTraits<v_float32>::vlanes() * 3, el4h_1, el4h_2, el4h_3);
el4l_1 = vx_load(row_cache );
el4l_2 = vx_load(row_cache + v_float32::nlanes );
el4l_3 = vx_load(row_cache + v_float32::nlanes * 2);
el4h_1 = vx_load(row_cache + v_float32::nlanes * 3);
el4h_2 = vx_load(row_cache + v_float32::nlanes * 4);
el4h_3 = vx_load(row_cache + v_float32::nlanes * 5);
v_store(sum_row + j , el4l_1 + vx_load(prev_sum_row + j ));
v_store(sum_row + j + v_float32::nlanes , el4l_2 + vx_load(prev_sum_row + j + v_float32::nlanes ));
v_store(sum_row + j + v_float32::nlanes * 2, el4l_3 + vx_load(prev_sum_row + j + v_float32::nlanes * 2));
v_store(sum_row + j + v_float32::nlanes * 3, el4h_1 + vx_load(prev_sum_row + j + v_float32::nlanes * 3));
v_store(sum_row + j + v_float32::nlanes * 4, el4h_2 + vx_load(prev_sum_row + j + v_float32::nlanes * 4));
v_store(sum_row + j + v_float32::nlanes * 5, el4h_3 + vx_load(prev_sum_row + j + v_float32::nlanes * 5));
el4l_2 = vx_load(row_cache + VTraits<v_float32>::vlanes() );
el4l_3 = vx_load(row_cache + VTraits<v_float32>::vlanes() * 2);
el4h_1 = vx_load(row_cache + VTraits<v_float32>::vlanes() * 3);
el4h_2 = vx_load(row_cache + VTraits<v_float32>::vlanes() * 4);
el4h_3 = vx_load(row_cache + VTraits<v_float32>::vlanes() * 5);
v_store(sum_row + j , v_add(el4l_1, vx_load(prev_sum_row + j)));
v_store(sum_row + j + VTraits<v_float32>::vlanes() , v_add(el4l_2, vx_load(prev_sum_row + j + VTraits<v_float32>::vlanes())));
v_store(sum_row + j + VTraits<v_float32>::vlanes() * 2, v_add(el4l_3, vx_load(prev_sum_row + j + VTraits<v_float32>::vlanes() * 2)));
v_store(sum_row + j + VTraits<v_float32>::vlanes() * 3, v_add(el4h_1, vx_load(prev_sum_row + j + VTraits<v_float32>::vlanes() * 3)));
v_store(sum_row + j + VTraits<v_float32>::vlanes() * 4, v_add(el4h_2, vx_load(prev_sum_row + j + VTraits<v_float32>::vlanes() * 4)));
v_store(sum_row + j + VTraits<v_float32>::vlanes() * 5, v_add(el4h_3, vx_load(prev_sum_row + j + VTraits<v_float32>::vlanes() * 5)));
}
for (float v3 = sum_row[j - 1] - prev_sum_row[j - 1],
@@ -652,7 +652,7 @@ struct Integral_SIMD<uchar, float, double>
v_float32 prev = vx_setzero_f32();
int j = 0;
for ( ; j + v_uint16::nlanes <= width; j += v_uint16::nlanes)
for ( ; j + VTraits<v_uint16>::vlanes() <= width; j += VTraits<v_uint16>::vlanes())
{
v_int16 el8 = v_reinterpret_as_s16(vx_load_expand(src_row + j));
v_float32 el4l, el4h;
@@ -670,8 +670,8 @@ struct Integral_SIMD<uchar, float, double>
#endif
v_int32 el4li, el4hi;
v_expand(el8, el4li, el4hi);
el4l = v_cvt_f32(el4li) + prev;
el4h = v_cvt_f32(el4hi) + el4l;
el4l = v_add(v_cvt_f32(el4li), prev);
el4h = v_add(v_cvt_f32(el4hi), el4l);
#if CV_SIMD_WIDTH == 16
prev = el4h;
#elif CV_SIMD_WIDTH == 32
@@ -682,8 +682,8 @@ struct Integral_SIMD<uchar, float, double>
prev = v_combine_low(t, t);
#endif
#endif
v_store(sum_row + j , el4l + vx_load(prev_sum_row + j ));
v_store(sum_row + j + v_float32::nlanes, el4h + vx_load(prev_sum_row + j + v_float32::nlanes));
v_store(sum_row + j , v_add(el4l, vx_load(prev_sum_row + j)));
v_store(sum_row + j + VTraits<v_float32>::vlanes(), v_add(el4h, vx_load(prev_sum_row + j + VTraits<v_float32>::vlanes())));
}
for (float v4 = sum_row[j - 1] - prev_sum_row[j - 1],
@@ -750,7 +750,7 @@ struct Integral_SIMD<uchar, double, double>
v_float64 prev = vx_setzero_f64();
int j = 0;
for (; j + v_uint16::nlanes <= width; j += v_uint16::nlanes)
for (; j + VTraits<v_uint16>::vlanes() <= width; j += VTraits<v_uint16>::vlanes())
{
v_int16 el8 = v_reinterpret_as_s16(vx_load_expand(src_row + j));
v_float64 el4ll, el4lh, el4hl, el4hh;
@@ -767,8 +767,8 @@ struct Integral_SIMD<uchar, double, double>
el4hh.val = _mm256_add_pd(_mm256_cvtepi32_pd(_v256_extract_high(el4h_32)), el4d);
prev.val = _mm256_permute4x64_pd(el4hh.val, 0xff);
#else
el8 += v_rotate_left<1>(el8);
el8 += v_rotate_left<2>(el8);
el8 = v_add(el8, v_rotate_left<1>(el8));
el8 = v_add(el8, v_rotate_left<2>(el8));
#if CV_SIMD_WIDTH >= 32
el8 += v_rotate_left<4>(el8);
#if CV_SIMD_WIDTH == 64
@@ -777,17 +777,17 @@ struct Integral_SIMD<uchar, double, double>
#endif
v_int32 el4li, el4hi;
v_expand(el8, el4li, el4hi);
el4ll = v_cvt_f64(el4li) + prev;
el4lh = v_cvt_f64_high(el4li) + prev;
el4hl = v_cvt_f64(el4hi) + el4ll;
el4hh = v_cvt_f64_high(el4hi) + el4lh;
prev = vx_setall_f64(v_extract_n<v_float64::nlanes - 1>(el4hh));
// prev = v_broadcast_element<v_float64::nlanes - 1>(el4hh);
el4ll = v_add(v_cvt_f64(el4li), prev);
el4lh = v_add(v_cvt_f64_high(el4li), prev);
el4hl = v_add(v_cvt_f64(el4hi), el4ll);
el4hh = v_add(v_cvt_f64_high(el4hi), el4lh);
prev = vx_setall_f64(v_extract_highest(el4hh));
// prev = v_broadcast_highest(el4hh);
#endif
v_store(sum_row + j , el4ll + vx_load(prev_sum_row + j ));
v_store(sum_row + j + v_float64::nlanes , el4lh + vx_load(prev_sum_row + j + v_float64::nlanes ));
v_store(sum_row + j + v_float64::nlanes * 2, el4hl + vx_load(prev_sum_row + j + v_float64::nlanes * 2));
v_store(sum_row + j + v_float64::nlanes * 3, el4hh + vx_load(prev_sum_row + j + v_float64::nlanes * 3));
v_store(sum_row + j , v_add(el4ll, vx_load(prev_sum_row + j)));
v_store(sum_row + j + VTraits<v_float64>::vlanes() , v_add(el4lh, vx_load(prev_sum_row + j + VTraits<v_float64>::vlanes())));
v_store(sum_row + j + VTraits<v_float64>::vlanes() * 2, v_add(el4hl, vx_load(prev_sum_row + j + VTraits<v_float64>::vlanes() * 2)));
v_store(sum_row + j + VTraits<v_float64>::vlanes() * 3, v_add(el4hh, vx_load(prev_sum_row + j + VTraits<v_float64>::vlanes() * 3)));
}
for (double v = sum_row[j - 1] - prev_sum_row[j - 1]; j < width; ++j)
@@ -808,11 +808,11 @@ struct Integral_SIMD<uchar, double, double>
v_float64 prev_1 = vx_setzero_f64(), prev_2 = vx_setzero_f64();
int j = 0;
for (; j + v_uint16::nlanes * cn <= width; j += v_uint16::nlanes * cn)
for (; j + VTraits<v_uint16>::vlanes() * cn <= width; j += VTraits<v_uint16>::vlanes() * cn)
{
v_int16 v_src_row = v_reinterpret_as_s16(vx_load(src_row + j));
v_int16 el8_1 = v_src_row & mask;
v_int16 el8_2 = v_reinterpret_as_s16(v_reinterpret_as_u16(v_src_row) >> 8);
v_int16 el8_1 = v_and(v_src_row, mask);
v_int16 el8_2 = v_reinterpret_as_s16(v_shr<8>(v_reinterpret_as_u16(v_src_row)));
v_float64 el4ll_1, el4lh_1, el4hl_1, el4hh_1, el4ll_2, el4lh_2, el4hl_2, el4hh_2;
#if CV_AVX2 && CV_SIMD_WIDTH == 32
__m256i vsum_1 = _mm256_add_epi16(el8_1.val, _mm256_slli_si256(el8_1.val, 2));
@@ -838,10 +838,10 @@ struct Integral_SIMD<uchar, double, double>
prev_1.val = _mm256_permute4x64_pd(el4hh_1.val, 0xff);
prev_2.val = _mm256_permute4x64_pd(el4hh_2.val, 0xff);
#else
el8_1 += v_rotate_left<1>(el8_1);
el8_2 += v_rotate_left<1>(el8_2);
el8_1 += v_rotate_left<2>(el8_1);
el8_2 += v_rotate_left<2>(el8_2);
el8_1 = v_add(el8_1, v_rotate_left<1>(el8_1));
el8_2 = v_add(el8_2, v_rotate_left<1>(el8_2));
el8_1 = v_add(el8_1, v_rotate_left<2>(el8_1));
el8_2 = v_add(el8_2, v_rotate_left<2>(el8_2));
#if CV_SIMD_WIDTH >= 32
el8_1 += v_rotate_left<4>(el8_1);
el8_2 += v_rotate_left<4>(el8_2);
@@ -853,32 +853,32 @@ struct Integral_SIMD<uchar, double, double>
v_int32 el4li_1, el4hi_1, el4li_2, el4hi_2;
v_expand(el8_1, el4li_1, el4hi_1);
v_expand(el8_2, el4li_2, el4hi_2);
el4ll_1 = v_cvt_f64(el4li_1) + prev_1;
el4ll_2 = v_cvt_f64(el4li_2) + prev_2;
el4lh_1 = v_cvt_f64_high(el4li_1) + prev_1;
el4lh_2 = v_cvt_f64_high(el4li_2) + prev_2;
el4hl_1 = v_cvt_f64(el4hi_1) + el4ll_1;
el4hl_2 = v_cvt_f64(el4hi_2) + el4ll_2;
el4hh_1 = v_cvt_f64_high(el4hi_1) + el4lh_1;
el4hh_2 = v_cvt_f64_high(el4hi_2) + el4lh_2;
prev_1 = vx_setall_f64(v_extract_n<v_float64::nlanes - 1>(el4hh_1));
prev_2 = vx_setall_f64(v_extract_n<v_float64::nlanes - 1>(el4hh_2));
// prev_1 = v_broadcast_element<v_float64::nlanes - 1>(el4hh_1);
// prev_2 = v_broadcast_element<v_float64::nlanes - 1>(el4hh_2);
el4ll_1 = v_add(v_cvt_f64(el4li_1), prev_1);
el4ll_2 = v_add(v_cvt_f64(el4li_2), prev_2);
el4lh_1 = v_add(v_cvt_f64_high(el4li_1), prev_1);
el4lh_2 = v_add(v_cvt_f64_high(el4li_2), prev_2);
el4hl_1 = v_add(v_cvt_f64(el4hi_1), el4ll_1);
el4hl_2 = v_add(v_cvt_f64(el4hi_2), el4ll_2);
el4hh_1 = v_add(v_cvt_f64_high(el4hi_1), el4lh_1);
el4hh_2 = v_add(v_cvt_f64_high(el4hi_2), el4lh_2);
prev_1 = vx_setall_f64(v_extract_highest(el4hh_1));
prev_2 = vx_setall_f64(v_extract_highest(el4hh_2));
// prev_1 = v_broadcast_highest(el4hh_1);
// prev_2 = v_broadcast_highest(el4hh_2);
#endif
v_float64 el4_1, el4_2, el4_3, el4_4, el4_5, el4_6, el4_7, el4_8;
v_zip(el4ll_1, el4ll_2, el4_1, el4_2);
v_zip(el4lh_1, el4lh_2, el4_3, el4_4);
v_zip(el4hl_1, el4hl_2, el4_5, el4_6);
v_zip(el4hh_1, el4hh_2, el4_7, el4_8);
v_store(sum_row + j , el4_1 + vx_load(prev_sum_row + j ));
v_store(sum_row + j + v_float64::nlanes , el4_2 + vx_load(prev_sum_row + j + v_float64::nlanes ));
v_store(sum_row + j + v_float64::nlanes * 2, el4_3 + vx_load(prev_sum_row + j + v_float64::nlanes * 2));
v_store(sum_row + j + v_float64::nlanes * 3, el4_4 + vx_load(prev_sum_row + j + v_float64::nlanes * 3));
v_store(sum_row + j + v_float64::nlanes * 4, el4_5 + vx_load(prev_sum_row + j + v_float64::nlanes * 4));
v_store(sum_row + j + v_float64::nlanes * 5, el4_6 + vx_load(prev_sum_row + j + v_float64::nlanes * 5));
v_store(sum_row + j + v_float64::nlanes * 6, el4_7 + vx_load(prev_sum_row + j + v_float64::nlanes * 6));
v_store(sum_row + j + v_float64::nlanes * 7, el4_8 + vx_load(prev_sum_row + j + v_float64::nlanes * 7));
v_store(sum_row + j , v_add(el4_1, vx_load(prev_sum_row + j)));
v_store(sum_row + j + VTraits<v_float64>::vlanes() , v_add(el4_2, vx_load(prev_sum_row + j + VTraits<v_float64>::vlanes())));
v_store(sum_row + j + VTraits<v_float64>::vlanes() * 2, v_add(el4_3, vx_load(prev_sum_row + j + VTraits<v_float64>::vlanes() * 2)));
v_store(sum_row + j + VTraits<v_float64>::vlanes() * 3, v_add(el4_4, vx_load(prev_sum_row + j + VTraits<v_float64>::vlanes() * 3)));
v_store(sum_row + j + VTraits<v_float64>::vlanes() * 4, v_add(el4_5, vx_load(prev_sum_row + j + VTraits<v_float64>::vlanes() * 4)));
v_store(sum_row + j + VTraits<v_float64>::vlanes() * 5, v_add(el4_6, vx_load(prev_sum_row + j + VTraits<v_float64>::vlanes() * 5)));
v_store(sum_row + j + VTraits<v_float64>::vlanes() * 6, v_add(el4_7, vx_load(prev_sum_row + j + VTraits<v_float64>::vlanes() * 6)));
v_store(sum_row + j + VTraits<v_float64>::vlanes() * 7, v_add(el4_8, vx_load(prev_sum_row + j + VTraits<v_float64>::vlanes() * 7)));
}
for (double v2 = sum_row[j - 1] - prev_sum_row[j - 1],
@@ -897,7 +897,7 @@ struct Integral_SIMD<uchar, double, double>
const uchar * src_row = src + _srcstep * i;
double * prev_sum_row = (double *)((uchar *)sum + _sumstep * i) + cn;
double * sum_row = (double *)((uchar *)sum + _sumstep * (i + 1)) + cn;
double row_cache[v_float64::nlanes * 12];
double row_cache[VTraits<v_float64>::max_nlanes * 12];
sum_row[-1] = sum_row[-2] = sum_row[-3] = 0;
@@ -905,10 +905,10 @@ struct Integral_SIMD<uchar, double, double>
prev_3 = vx_setzero_f64();
int j = 0;
const int j_max =
((_srcstep * i + (width - v_uint16::nlanes * cn + v_uint8::nlanes * cn)) >= _srcstep * height)
? width - v_uint8::nlanes * cn // uint8 in v_load_deinterleave()
: width - v_uint16::nlanes * cn; // v_expand_low
for ( ; j <= j_max; j += v_uint16::nlanes * cn)
((_srcstep * i + (width - VTraits<v_uint16>::vlanes() * cn + VTraits<v_uint8>::vlanes() * cn)) >= _srcstep * height)
? width - VTraits<v_uint8>::vlanes() * cn // uint8 in v_load_deinterleave()
: width - VTraits<v_uint16>::vlanes() * cn; // v_expand_low
for ( ; j <= j_max; j += VTraits<v_uint16>::vlanes() * cn)
{
v_uint8 v_src_row_1, v_src_row_2, v_src_row_3;
v_load_deinterleave(src_row + j, v_src_row_1, v_src_row_2, v_src_row_3);
@@ -951,12 +951,12 @@ struct Integral_SIMD<uchar, double, double>
prev_2.val = _mm256_permute4x64_pd(el4hh_2.val, 0xff);
prev_3.val = _mm256_permute4x64_pd(el4hh_3.val, 0xff);
#else
el8_1 += v_rotate_left<1>(el8_1);
el8_2 += v_rotate_left<1>(el8_2);
el8_3 += v_rotate_left<1>(el8_3);
el8_1 += v_rotate_left<2>(el8_1);
el8_2 += v_rotate_left<2>(el8_2);
el8_3 += v_rotate_left<2>(el8_3);
el8_1 = v_add(el8_1, v_rotate_left<1>(el8_1));
el8_2 = v_add(el8_2, v_rotate_left<1>(el8_2));
el8_3 = v_add(el8_3, v_rotate_left<1>(el8_3));
el8_1 = v_add(el8_1, v_rotate_left<2>(el8_1));
el8_2 = v_add(el8_2, v_rotate_left<2>(el8_2));
el8_3 = v_add(el8_3, v_rotate_left<2>(el8_3));
#if CV_SIMD_WIDTH >= 32
el8_1 += v_rotate_left<4>(el8_1);
el8_2 += v_rotate_left<4>(el8_2);
@@ -971,53 +971,53 @@ struct Integral_SIMD<uchar, double, double>
v_expand(el8_1, el4li_1, el4hi_1);
v_expand(el8_2, el4li_2, el4hi_2);
v_expand(el8_3, el4li_3, el4hi_3);
el4ll_1 = v_cvt_f64(el4li_1) + prev_1;
el4ll_2 = v_cvt_f64(el4li_2) + prev_2;
el4ll_3 = v_cvt_f64(el4li_3) + prev_3;
el4lh_1 = v_cvt_f64_high(el4li_1) + prev_1;
el4lh_2 = v_cvt_f64_high(el4li_2) + prev_2;
el4lh_3 = v_cvt_f64_high(el4li_3) + prev_3;
el4hl_1 = v_cvt_f64(el4hi_1) + el4ll_1;
el4hl_2 = v_cvt_f64(el4hi_2) + el4ll_2;
el4hl_3 = v_cvt_f64(el4hi_3) + el4ll_3;
el4hh_1 = v_cvt_f64_high(el4hi_1) + el4lh_1;
el4hh_2 = v_cvt_f64_high(el4hi_2) + el4lh_2;
el4hh_3 = v_cvt_f64_high(el4hi_3) + el4lh_3;
prev_1 = vx_setall_f64(v_extract_n<v_float64::nlanes - 1>(el4hh_1));
prev_2 = vx_setall_f64(v_extract_n<v_float64::nlanes - 1>(el4hh_2));
prev_3 = vx_setall_f64(v_extract_n<v_float64::nlanes - 1>(el4hh_3));
// prev_1 = v_broadcast_element<v_float64::nlanes - 1>(el4hh_1);
// prev_2 = v_broadcast_element<v_float64::nlanes - 1>(el4hh_2);
// prev_3 = v_broadcast_element<v_float64::nlanes - 1>(el4hh_3);
el4ll_1 = v_add(v_cvt_f64(el4li_1), prev_1);
el4ll_2 = v_add(v_cvt_f64(el4li_2), prev_2);
el4ll_3 = v_add(v_cvt_f64(el4li_3), prev_3);
el4lh_1 = v_add(v_cvt_f64_high(el4li_1), prev_1);
el4lh_2 = v_add(v_cvt_f64_high(el4li_2), prev_2);
el4lh_3 = v_add(v_cvt_f64_high(el4li_3), prev_3);
el4hl_1 = v_add(v_cvt_f64(el4hi_1), el4ll_1);
el4hl_2 = v_add(v_cvt_f64(el4hi_2), el4ll_2);
el4hl_3 = v_add(v_cvt_f64(el4hi_3), el4ll_3);
el4hh_1 = v_add(v_cvt_f64_high(el4hi_1), el4lh_1);
el4hh_2 = v_add(v_cvt_f64_high(el4hi_2), el4lh_2);
el4hh_3 = v_add(v_cvt_f64_high(el4hi_3), el4lh_3);
prev_1 = vx_setall_f64(v_extract_highest(el4hh_1));
prev_2 = vx_setall_f64(v_extract_highest(el4hh_2));
prev_3 = vx_setall_f64(v_extract_highest(el4hh_3));
// prev_1 = v_broadcast_highest(el4hh_1);
// prev_2 = v_broadcast_highest(el4hh_2);
// prev_3 = v_broadcast_highest(el4hh_3);
#endif
v_store_interleave(row_cache , el4ll_1, el4ll_2, el4ll_3);
v_store_interleave(row_cache + v_float64::nlanes * 3, el4lh_1, el4lh_2, el4lh_3);
v_store_interleave(row_cache + v_float64::nlanes * 6, el4hl_1, el4hl_2, el4hl_3);
v_store_interleave(row_cache + v_float64::nlanes * 9, el4hh_1, el4hh_2, el4hh_3);
v_store_interleave(row_cache + VTraits<v_float64>::vlanes() * 3, el4lh_1, el4lh_2, el4lh_3);
v_store_interleave(row_cache + VTraits<v_float64>::vlanes() * 6, el4hl_1, el4hl_2, el4hl_3);
v_store_interleave(row_cache + VTraits<v_float64>::vlanes() * 9, el4hh_1, el4hh_2, el4hh_3);
el4ll_1 = vx_load(row_cache );
el4ll_2 = vx_load(row_cache + v_float64::nlanes );
el4ll_3 = vx_load(row_cache + v_float64::nlanes * 2 );
el4lh_1 = vx_load(row_cache + v_float64::nlanes * 3 );
el4lh_2 = vx_load(row_cache + v_float64::nlanes * 4 );
el4lh_3 = vx_load(row_cache + v_float64::nlanes * 5 );
el4hl_1 = vx_load(row_cache + v_float64::nlanes * 6 );
el4hl_2 = vx_load(row_cache + v_float64::nlanes * 7 );
el4hl_3 = vx_load(row_cache + v_float64::nlanes * 8 );
el4hh_1 = vx_load(row_cache + v_float64::nlanes * 9 );
el4hh_2 = vx_load(row_cache + v_float64::nlanes * 10);
el4hh_3 = vx_load(row_cache + v_float64::nlanes * 11);
v_store(sum_row + j , el4ll_1 + vx_load(prev_sum_row + j ));
v_store(sum_row + j + v_float64::nlanes , el4ll_2 + vx_load(prev_sum_row + j + v_float64::nlanes ));
v_store(sum_row + j + v_float64::nlanes * 2 , el4ll_3 + vx_load(prev_sum_row + j + v_float64::nlanes * 2 ));
v_store(sum_row + j + v_float64::nlanes * 3 , el4lh_1 + vx_load(prev_sum_row + j + v_float64::nlanes * 3 ));
v_store(sum_row + j + v_float64::nlanes * 4 , el4lh_2 + vx_load(prev_sum_row + j + v_float64::nlanes * 4 ));
v_store(sum_row + j + v_float64::nlanes * 5 , el4lh_3 + vx_load(prev_sum_row + j + v_float64::nlanes * 5 ));
v_store(sum_row + j + v_float64::nlanes * 6 , el4hl_1 + vx_load(prev_sum_row + j + v_float64::nlanes * 6 ));
v_store(sum_row + j + v_float64::nlanes * 7 , el4hl_2 + vx_load(prev_sum_row + j + v_float64::nlanes * 7 ));
v_store(sum_row + j + v_float64::nlanes * 8 , el4hl_3 + vx_load(prev_sum_row + j + v_float64::nlanes * 8 ));
v_store(sum_row + j + v_float64::nlanes * 9 , el4hh_1 + vx_load(prev_sum_row + j + v_float64::nlanes * 9 ));
v_store(sum_row + j + v_float64::nlanes * 10, el4hh_2 + vx_load(prev_sum_row + j + v_float64::nlanes * 10));
v_store(sum_row + j + v_float64::nlanes * 11, el4hh_3 + vx_load(prev_sum_row + j + v_float64::nlanes * 11));
el4ll_2 = vx_load(row_cache + VTraits<v_float64>::vlanes() );
el4ll_3 = vx_load(row_cache + VTraits<v_float64>::vlanes() * 2 );
el4lh_1 = vx_load(row_cache + VTraits<v_float64>::vlanes() * 3 );
el4lh_2 = vx_load(row_cache + VTraits<v_float64>::vlanes() * 4 );
el4lh_3 = vx_load(row_cache + VTraits<v_float64>::vlanes() * 5 );
el4hl_1 = vx_load(row_cache + VTraits<v_float64>::vlanes() * 6 );
el4hl_2 = vx_load(row_cache + VTraits<v_float64>::vlanes() * 7 );
el4hl_3 = vx_load(row_cache + VTraits<v_float64>::vlanes() * 8 );
el4hh_1 = vx_load(row_cache + VTraits<v_float64>::vlanes() * 9 );
el4hh_2 = vx_load(row_cache + VTraits<v_float64>::vlanes() * 10);
el4hh_3 = vx_load(row_cache + VTraits<v_float64>::vlanes() * 11);
v_store(sum_row + j , v_add(el4ll_1, vx_load(prev_sum_row + j)));
v_store(sum_row + j + VTraits<v_float64>::vlanes() , v_add(el4ll_2, vx_load(prev_sum_row + j + VTraits<v_float64>::vlanes())));
v_store(sum_row + j + VTraits<v_float64>::vlanes() * 2 , v_add(el4ll_3, vx_load(prev_sum_row + j + VTraits<v_float64>::vlanes() * 2)));
v_store(sum_row + j + VTraits<v_float64>::vlanes() * 3 , v_add(el4lh_1, vx_load(prev_sum_row + j + VTraits<v_float64>::vlanes() * 3)));
v_store(sum_row + j + VTraits<v_float64>::vlanes() * 4 , v_add(el4lh_2, vx_load(prev_sum_row + j + VTraits<v_float64>::vlanes() * 4)));
v_store(sum_row + j + VTraits<v_float64>::vlanes() * 5 , v_add(el4lh_3, vx_load(prev_sum_row + j + VTraits<v_float64>::vlanes() * 5)));
v_store(sum_row + j + VTraits<v_float64>::vlanes() * 6 , v_add(el4hl_1, vx_load(prev_sum_row + j + VTraits<v_float64>::vlanes() * 6)));
v_store(sum_row + j + VTraits<v_float64>::vlanes() * 7 , v_add(el4hl_2, vx_load(prev_sum_row + j + VTraits<v_float64>::vlanes() * 7)));
v_store(sum_row + j + VTraits<v_float64>::vlanes() * 8 , v_add(el4hl_3, vx_load(prev_sum_row + j + VTraits<v_float64>::vlanes() * 8)));
v_store(sum_row + j + VTraits<v_float64>::vlanes() * 9 , v_add(el4hh_1, vx_load(prev_sum_row + j + VTraits<v_float64>::vlanes() * 9)));
v_store(sum_row + j + VTraits<v_float64>::vlanes() * 10, v_add(el4hh_2, vx_load(prev_sum_row + j + VTraits<v_float64>::vlanes() * 10)));
v_store(sum_row + j + VTraits<v_float64>::vlanes() * 11, v_add(el4hh_3, vx_load(prev_sum_row + j + VTraits<v_float64>::vlanes() * 11)));
}
for (double v3 = sum_row[j - 1] - prev_sum_row[j - 1],
@@ -1043,7 +1043,7 @@ struct Integral_SIMD<uchar, double, double>
v_float64 prev_1 = vx_setzero_f64(), prev_2 = vx_setzero_f64();
int j = 0;
for ( ; j + v_uint16::nlanes <= width; j += v_uint16::nlanes)
for ( ; j + VTraits<v_uint16>::vlanes() <= width; j += VTraits<v_uint16>::vlanes())
{
v_int16 el8 = v_reinterpret_as_s16(vx_load_expand(src_row + j));
v_float64 el4ll, el4lh, el4hl, el4hh;
@@ -1065,10 +1065,10 @@ struct Integral_SIMD<uchar, double, double>
#endif
v_int32 el4li, el4hi;
v_expand(el8, el4li, el4hi);
el4ll = v_cvt_f64(el4li) + prev_1;
el4lh = v_cvt_f64_high(el4li) + prev_2;
el4hl = v_cvt_f64(el4hi) + el4ll;
el4hh = v_cvt_f64_high(el4hi) + el4lh;
el4ll = v_add(v_cvt_f64(el4li), prev_1);
el4lh = v_add(v_cvt_f64_high(el4li), prev_2);
el4hl = v_add(v_cvt_f64(el4hi), el4ll);
el4hh = v_add(v_cvt_f64_high(el4hi), el4lh);
#if CV_SIMD_WIDTH == 16
prev_1 = el4hl;
prev_2 = el4hh;
@@ -1078,10 +1078,10 @@ struct Integral_SIMD<uchar, double, double>
prev_1 = prev_2 = v_combine_high(el4hh, el4hh);
#endif
#endif
v_store(sum_row + j , el4ll + vx_load(prev_sum_row + j ));
v_store(sum_row + j + v_float64::nlanes , el4lh + vx_load(prev_sum_row + j + v_float64::nlanes ));
v_store(sum_row + j + v_float64::nlanes * 2, el4hl + vx_load(prev_sum_row + j + v_float64::nlanes * 2));
v_store(sum_row + j + v_float64::nlanes * 3, el4hh + vx_load(prev_sum_row + j + v_float64::nlanes * 3));
v_store(sum_row + j , v_add(el4ll, vx_load(prev_sum_row + j)));
v_store(sum_row + j + VTraits<v_float64>::vlanes() , v_add(el4lh, vx_load(prev_sum_row + j + VTraits<v_float64>::vlanes())));
v_store(sum_row + j + VTraits<v_float64>::vlanes() * 2, v_add(el4hl, vx_load(prev_sum_row + j + VTraits<v_float64>::vlanes() * 2)));
v_store(sum_row + j + VTraits<v_float64>::vlanes() * 3, v_add(el4hh, vx_load(prev_sum_row + j + VTraits<v_float64>::vlanes() * 3)));
}
for (double v4 = sum_row[j - 1] - prev_sum_row[j - 1],