potok/opencv
1
0
Fork 0
mirror of https://github.com/opencv/opencv.git synced 2026-07-31 08:13:04 +04:00
Code Packages Releases Wiki Activity

Merge remote-tracking branch 'upstream/3.4' into merge-3.4

Browse Source
This commit is contained in:
Alexander Alekhin
2019-02-26 14:49:37 +03:00
parent 4188e8aaf9 dd696066b5
commit c3cf35ab63
123 changed files with 7773 additions and 6294 deletions
Download Patch File Download Diff File Expand all files Collapse all files
modules/imgproc/src/resize.cpp
+101 -167
View File
@@ -340,155 +340,6 @@ static void hlineResizeCn(ET* src, int cn, int *ofst, FT* m, FT* dst, int dst_mi
hline<ET, FT, n, mulall, cncnt>::ResizeCn(src, cn, ofst, m, dst, dst_min, dst_max, dst_width);
};
#if CV_SIMD512
inline void v_load_indexed1(uint8_t* src, int *ofst, v_uint16 &v_src0, v_uint16 &v_src1)
{
v_expand(v_reinterpret_as_u8(v_uint16(
*((uint16_t*)(src + ofst[ 0])), *((uint16_t*)(src + ofst[ 1])), *((uint16_t*)(src + ofst[ 2])), *((uint16_t*)(src + ofst[ 3])),
*((uint16_t*)(src + ofst[ 4])), *((uint16_t*)(src + ofst[ 5])), *((uint16_t*)(src + ofst[ 6])), *((uint16_t*)(src + ofst[ 7])),
*((uint16_t*)(src + ofst[ 8])), *((uint16_t*)(src + ofst[ 9])), *((uint16_t*)(src + ofst[10])), *((uint16_t*)(src + ofst[11])),
*((uint16_t*)(src + ofst[12])), *((uint16_t*)(src + ofst[13])), *((uint16_t*)(src + ofst[14])), *((uint16_t*)(src + ofst[15])),
*((uint16_t*)(src + ofst[16])), *((uint16_t*)(src + ofst[17])), *((uint16_t*)(src + ofst[14])), *((uint16_t*)(src + ofst[15])),
*((uint16_t*)(src + ofst[20])), *((uint16_t*)(src + ofst[21])), *((uint16_t*)(src + ofst[14])), *((uint16_t*)(src + ofst[15])),
*((uint16_t*)(src + ofst[24])), *((uint16_t*)(src + ofst[25])), *((uint16_t*)(src + ofst[14])), *((uint16_t*)(src + ofst[15])),
*((uint16_t*)(src + ofst[28])), *((uint16_t*)(src + ofst[29])), *((uint16_t*)(src + ofst[14])), *((uint16_t*)(src + ofst[15])))),
v_src0, v_src1);
}
inline void v_load_indexed2(uint8_t* src, int *ofst, v_uint16 &v_src0, v_uint16 &v_src1)
{
v_expand(v_reinterpret_as_u8(v_uint32(
*((uint32_t*)(src + 2 * ofst[ 0])), *((uint32_t*)(src + 2 * ofst[ 1])), *((uint32_t*)(src + 2 * ofst[ 2])), *((uint32_t*)(src + 2 * ofst[ 3])),
*((uint32_t*)(src + 2 * ofst[ 4])), *((uint32_t*)(src + 2 * ofst[ 5])), *((uint32_t*)(src + 2 * ofst[ 6])), *((uint32_t*)(src + 2 * ofst[ 7])),
*((uint32_t*)(src + 2 * ofst[ 8])), *((uint32_t*)(src + 2 * ofst[ 9])), *((uint32_t*)(src + 2 * ofst[10])), *((uint32_t*)(src + 2 * ofst[11])),
*((uint32_t*)(src + 2 * ofst[12])), *((uint32_t*)(src + 2 * ofst[13])), *((uint32_t*)(src + 2 * ofst[14])), *((uint32_t*)(src + 2 * ofst[15])))),
v_src0, v_src1);
v_uint32 v_tmp0, v_tmp1, v_tmp2, v_tmp3;
v_zip(v_reinterpret_as_u32(v_src0), v_reinterpret_as_u32(v_src1), v_tmp2, v_tmp3);
v_zip(v_tmp2, v_tmp3, v_tmp0, v_tmp1);
v_zip(v_tmp0, v_tmp1, v_tmp2, v_tmp3);
v_zip(v_tmp2, v_tmp3, v_tmp0, v_tmp1);
v_zip(v_reinterpret_as_u16(v_tmp0), v_reinterpret_as_u16(v_tmp1), v_src0, v_src1);
}
inline void v_load_indexed4(uint8_t* src, int *ofst, v_uint16 &v_src0, v_uint16 &v_src1)
{
v_expand(v_reinterpret_as_u8(v_uint64(
*((uint64_t*)(src + 4 * ofst[0])), *((uint64_t*)(src + 4 * ofst[1])), *((uint64_t*)(src + 4 * ofst[2])), *((uint64_t*)(src + 4 * ofst[3])),
*((uint64_t*)(src + 4 * ofst[4])), *((uint64_t*)(src + 4 * ofst[5])), *((uint64_t*)(src + 4 * ofst[6])), *((uint64_t*)(src + 4 * ofst[7])))),
v_src0, v_src1);
v_uint64 v_tmp0, v_tmp1, v_tmp2, v_tmp3;
v_zip(v_reinterpret_as_u64(v_src0), v_reinterpret_as_u64(v_src1), v_tmp2, v_tmp3);
v_zip(v_tmp2, v_tmp3, v_tmp0, v_tmp1);
v_zip(v_tmp0, v_tmp1, v_tmp2, v_tmp3);
v_zip(v_reinterpret_as_u16(v_tmp2), v_reinterpret_as_u16(v_tmp3), v_src0, v_src1);
}
inline void v_load_indexed_deinterleave(uint16_t* src, int *ofst, v_uint32 &v_src0, v_uint32 &v_src1)
{
v_expand(v_reinterpret_as_u16(v_uint32(
*((uint32_t*)(src + ofst[ 0])), *((uint32_t*)(src + ofst[ 1])), *((uint32_t*)(src + ofst[ 2])), *((uint32_t*)(src + ofst[ 3])),
*((uint32_t*)(src + ofst[ 4])), *((uint32_t*)(src + ofst[ 5])), *((uint32_t*)(src + ofst[ 6])), *((uint32_t*)(src + ofst[ 7])),
*((uint32_t*)(src + ofst[ 8])), *((uint32_t*)(src + ofst[ 9])), *((uint32_t*)(src + ofst[10])), *((uint32_t*)(src + ofst[11])),
*((uint32_t*)(src + ofst[12])), *((uint32_t*)(src + ofst[13])), *((uint32_t*)(src + ofst[14])), *((uint32_t*)(src + ofst[15])))),
v_src0, v_src1);
v_uint32 v_tmp0, v_tmp1;
v_zip(v_src0, v_src1, v_tmp0, v_tmp1);
v_zip(v_tmp0, v_tmp1, v_src0, v_src1);
v_zip(v_src0, v_src1, v_tmp0, v_tmp1);
v_zip(v_tmp0, v_tmp1, v_src0, v_src1);
}
#elif CV_SIMD256
inline void v_load_indexed1(uint8_t* src, int *ofst, v_uint16 &v_src0, v_uint16 &v_src1)
{
v_expand(v_reinterpret_as_u8(v_uint16(
*((uint16_t*)(src + ofst[ 0])), *((uint16_t*)(src + ofst[ 1])), *((uint16_t*)(src + ofst[ 2])), *((uint16_t*)(src + ofst[ 3])),
*((uint16_t*)(src + ofst[ 4])), *((uint16_t*)(src + ofst[ 5])), *((uint16_t*)(src + ofst[ 6])), *((uint16_t*)(src + ofst[ 7])),
*((uint16_t*)(src + ofst[ 8])), *((uint16_t*)(src + ofst[ 9])), *((uint16_t*)(src + ofst[10])), *((uint16_t*)(src + ofst[11])),
*((uint16_t*)(src + ofst[12])), *((uint16_t*)(src + ofst[13])), *((uint16_t*)(src + ofst[14])), *((uint16_t*)(src + ofst[15])))),
v_src0, v_src1);
}
inline void v_load_indexed2(uint8_t* src, int *ofst, v_uint16 &v_src0, v_uint16 &v_src1)
{
v_expand(v_reinterpret_as_u8(v_uint32(
*((uint32_t*)(src + 2 * ofst[0])), *((uint32_t*)(src + 2 * ofst[1])), *((uint32_t*)(src + 2 * ofst[2])), *((uint32_t*)(src + 2 * ofst[3])),
*((uint32_t*)(src + 2 * ofst[4])), *((uint32_t*)(src + 2 * ofst[5])), *((uint32_t*)(src + 2 * ofst[6])), *((uint32_t*)(src + 2 * ofst[7])))),
v_src0, v_src1);
v_uint32 v_tmp0, v_tmp1, v_tmp2, v_tmp3;
v_zip(v_reinterpret_as_u32(v_src0), v_reinterpret_as_u32(v_src1), v_tmp2, v_tmp3);
v_zip(v_tmp2, v_tmp3, v_tmp0, v_tmp1);
v_zip(v_tmp0, v_tmp1, v_tmp2, v_tmp3);
v_zip(v_reinterpret_as_u16(v_tmp2), v_reinterpret_as_u16(v_tmp3), v_src0, v_src1);
}
inline void v_load_indexed4(uint8_t* src, int *ofst, v_uint16 &v_src0, v_uint16 &v_src1)
{
v_expand(v_reinterpret_as_u8(v_uint64(
*((uint64_t*)(src + 4 * ofst[0])), *((uint64_t*)(src + 4 * ofst[1])), *((uint64_t*)(src + 4 * ofst[2])), *((uint64_t*)(src + 4 * ofst[3])))),
v_src0, v_src1);
v_uint64 v_tmp0, v_tmp1, v_tmp2, v_tmp3;
v_zip(v_reinterpret_as_u64(v_src0), v_reinterpret_as_u64(v_src1), v_tmp2, v_tmp3);
v_zip(v_tmp2, v_tmp3, v_tmp0, v_tmp1);
v_zip(v_reinterpret_as_u16(v_tmp0), v_reinterpret_as_u16(v_tmp1), v_src0, v_src1);
}
inline void v_load_indexed_deinterleave(uint16_t* src, int *ofst, v_uint32 &v_src0, v_uint32 &v_src1)
{
v_uint32 v_tmp0, v_tmp1;
v_expand(v_reinterpret_as_u16(v_uint32(
*((uint32_t*)(src + ofst[0])), *((uint32_t*)(src + ofst[1])), *((uint32_t*)(src + ofst[2])), *((uint32_t*)(src + ofst[3])),
*((uint32_t*)(src + ofst[4])), *((uint32_t*)(src + ofst[5])), *((uint32_t*)(src + ofst[6])), *((uint32_t*)(src + ofst[7])))),
v_tmp0, v_tmp1);
v_zip(v_tmp0, v_tmp1, v_src0, v_src1);
v_zip(v_src0, v_src1, v_tmp0, v_tmp1);
v_zip(v_tmp0, v_tmp1, v_src0, v_src1);
}
#elif CV_SIMD128
inline void v_load_indexed1(uint8_t* src, int *ofst, v_uint16 &v_src0, v_uint16 &v_src1)
{
uint16_t buf[8];
buf[0] = *((uint16_t*)(src + ofst[0]));
buf[1] = *((uint16_t*)(src + ofst[1]));
buf[2] = *((uint16_t*)(src + ofst[2]));
buf[3] = *((uint16_t*)(src + ofst[3]));
buf[4] = *((uint16_t*)(src + ofst[4]));
buf[5] = *((uint16_t*)(src + ofst[5]));
buf[6] = *((uint16_t*)(src + ofst[6]));
buf[7] = *((uint16_t*)(src + ofst[7]));
v_src0 = vx_load_expand((uint8_t*)buf);
v_src1 = vx_load_expand((uint8_t*)buf + 8);
}
inline void v_load_indexed2(uint8_t* src, int *ofst, v_uint16 &v_src0, v_uint16 &v_src1)
{
uint32_t buf[4];
buf[0] = *((uint32_t*)(src + 2 * ofst[0]));
buf[1] = *((uint32_t*)(src + 2 * ofst[1]));
buf[2] = *((uint32_t*)(src + 2 * ofst[2]));
buf[3] = *((uint32_t*)(src + 2 * ofst[3]));
v_uint32 v_tmp0, v_tmp1, v_tmp2, v_tmp3;
v_tmp0 = v_reinterpret_as_u32(vx_load_expand((uint8_t*)buf));
v_tmp1 = v_reinterpret_as_u32(vx_load_expand((uint8_t*)buf + 8));
v_zip(v_tmp0, v_tmp1, v_tmp2, v_tmp3);
v_zip(v_tmp2, v_tmp3, v_tmp0, v_tmp1);
v_zip(v_reinterpret_as_u16(v_tmp0), v_reinterpret_as_u16(v_tmp1), v_src0, v_src1);
}
inline void v_load_indexed4(uint8_t* src, int *ofst, v_uint16 &v_src0, v_uint16 &v_src1)
{
v_uint16 v_tmp0, v_tmp1;
v_src0 = vx_load_expand(src + 4 * ofst[0]);
v_src1 = vx_load_expand(src + 4 * ofst[1]);
v_recombine(v_src0, v_src1, v_tmp0, v_tmp1);
v_zip(v_tmp0, v_tmp1, v_src0, v_src1);
}
inline void v_load_indexed_deinterleave(uint16_t* src, int *ofst, v_uint32 &v_src0, v_uint32 &v_src1)
{
uint32_t buf[4];
buf[0] = *((uint32_t*)(src + ofst[0]));
buf[1] = *((uint32_t*)(src + ofst[1]));
buf[2] = *((uint32_t*)(src + ofst[2]));
buf[3] = *((uint32_t*)(src + ofst[3]));
v_src0 = vx_load_expand((uint16_t*)buf);
v_src1 = vx_load_expand((uint16_t*)buf + 4);
v_uint32 v_tmp0, v_tmp1;
v_zip(v_src0, v_src1, v_tmp0, v_tmp1);
v_zip(v_tmp0, v_tmp1, v_src0, v_src1);
}
#endif
template <>
void hlineResizeCn<uint8_t, ufixedpoint16, 2, true, 1>(uint8_t* src, int, int *ofst, ufixedpoint16* m, ufixedpoint16* dst, int dst_min, int dst_max, int dst_width)
{
@@ -507,16 +358,23 @@ void hlineResizeCn<uint8_t, ufixedpoint16, 2, true, 1>(uint8_t* src, int, int *o
*(dst++) = src_0;
}
#if CV_SIMD
for (; i <= dst_max - VECSZ; i += VECSZ, m += 2*VECSZ, dst += VECSZ)
for (; i <= dst_max - 2*VECSZ; i += 2*VECSZ, m += 4*VECSZ, dst += 2*VECSZ)
{
v_uint16 v_src0, v_src1;
v_load_indexed1(src, ofst + i, v_src0, v_src1);
v_int16 v_mul0 = vx_load((int16_t*)m);
v_int16 v_mul1 = vx_load((int16_t*)m + VECSZ);
v_uint32 v_res0 = v_reinterpret_as_u32(v_dotprod(v_reinterpret_as_s16(v_src0), v_mul0));
v_uint32 v_res1 = v_reinterpret_as_u32(v_dotprod(v_reinterpret_as_s16(v_src1), v_mul1));
v_store((uint16_t*)dst, v_pack(v_res0, v_res1));
v_expand(vx_lut_pairs(src, ofst + i), v_src0, v_src1);
v_store((uint16_t*)dst , v_pack(v_reinterpret_as_u32(v_dotprod(v_reinterpret_as_s16(v_src0), vx_load((int16_t*)m))),
v_reinterpret_as_u32(v_dotprod(v_reinterpret_as_s16(v_src1), vx_load((int16_t*)m + VECSZ)))));
v_expand(vx_lut_pairs(src, ofst + i + VECSZ), v_src0, v_src1);
v_store((uint16_t*)dst+VECSZ, v_pack(v_reinterpret_as_u32(v_dotprod(v_reinterpret_as_s16(v_src0), vx_load((int16_t*)m + 2*VECSZ))),
v_reinterpret_as_u32(v_dotprod(v_reinterpret_as_s16(v_src1), vx_load((int16_t*)m + 3*VECSZ)))));
}
if (i <= dst_max - VECSZ)
{
v_uint16 v_src0, v_src1;
v_expand(vx_lut_pairs(src, ofst + i), v_src0, v_src1);
v_store((uint16_t*)dst, v_pack(v_reinterpret_as_u32(v_dotprod(v_reinterpret_as_s16(v_src0), vx_load((int16_t*)m))),
v_reinterpret_as_u32(v_dotprod(v_reinterpret_as_s16(v_src1), vx_load((int16_t*)m + VECSZ)))));
i += VECSZ; m += 2*VECSZ; dst += VECSZ;
}
#endif
for (; i < dst_max; i += 1, m += 2)
@@ -564,7 +422,7 @@ void hlineResizeCn<uint8_t, ufixedpoint16, 2, true, 2>(uint8_t* src, int, int *o
for (; i <= dst_max - VECSZ/2; i += VECSZ/2, m += VECSZ, dst += VECSZ)
{
v_uint16 v_src0, v_src1;
v_load_indexed2(src, ofst + i, v_src0, v_src1);
v_expand(v_interleave_pairs(v_reinterpret_as_u8(vx_lut_pairs((uint16_t*)src, ofst + i))), v_src0, v_src1);
v_uint32 v_mul = vx_load((uint32_t*)m);//AaBbCcDd
v_uint32 v_zip0, v_zip1;
@@ -595,6 +453,81 @@ void hlineResizeCn<uint8_t, ufixedpoint16, 2, true, 2>(uint8_t* src, int, int *o
}
}
template <>
void hlineResizeCn<uint8_t, ufixedpoint16, 2, true, 3>(uint8_t* src, int, int *ofst, ufixedpoint16* m, ufixedpoint16* dst, int dst_min, int dst_max, int dst_width)
{
int i = 0;
union {
uint64_t q;
uint16_t w[4];
} srccn;
((ufixedpoint16*)(srccn.w))[0] = src[0];
((ufixedpoint16*)(srccn.w))[1] = src[1];
((ufixedpoint16*)(srccn.w))[2] = src[2];
((ufixedpoint16*)(srccn.w))[3] = 0;
#if CV_SIMD
const int VECSZ = v_uint16::nlanes;
v_uint16 v_srccn = v_pack_triplets(v_reinterpret_as_u16(vx_setall_u64(srccn.q)));
for (; i <= dst_min - (VECSZ+2)/3; i += VECSZ/4, m += VECSZ/2, dst += 3*VECSZ/4) // Points that fall left from src image so became equal to leftmost src point
{
v_store((uint16_t*)dst, v_srccn);
}
#endif
for (; i < dst_min; i++, m += 2)
{
*(dst++) = ((ufixedpoint16*)(srccn.w))[0];
*(dst++) = ((ufixedpoint16*)(srccn.w))[1];
*(dst++) = ((ufixedpoint16*)(srccn.w))[2];
}
#if CV_SIMD
CV_DECL_ALIGNED(CV_SIMD_WIDTH) int ofst3[VECSZ/2];
for (; i <= dst_max - (3*VECSZ/4 + (VECSZ+2)/3); i += VECSZ/2, m += VECSZ, dst += 3*VECSZ/2)
{
v_store(ofst3, vx_load(ofst + i) * vx_setall_s32(3));
v_uint8 v_src01, v_src23;
v_uint16 v_src0, v_src1, v_src2, v_src3;
v_zip(vx_lut_quads(src, ofst3), vx_lut_quads(src+3, ofst3), v_src01, v_src23);
v_expand(v_src01, v_src0, v_src1);
v_expand(v_src23, v_src2, v_src3);
v_uint32 v_mul0, v_mul1, v_mul2, v_mul3, v_tmp;
v_mul0 = vx_load((uint32_t*)m);//AaBbCcDd
v_zip(v_mul0, v_mul0, v_mul3, v_tmp );//AaAaBbBb CcCcDdDd
v_zip(v_mul3, v_mul3, v_mul0, v_mul1);//AaAaAaAa BbBbBbBb
v_zip(v_tmp , v_tmp , v_mul2, v_mul3);//CcCcCcCc DdDdDdDd
v_uint32 v_res0 = v_reinterpret_as_u32(v_dotprod(v_reinterpret_as_s16(v_src0), v_reinterpret_as_s16(v_mul0)));
v_uint32 v_res1 = v_reinterpret_as_u32(v_dotprod(v_reinterpret_as_s16(v_src1), v_reinterpret_as_s16(v_mul1)));
v_uint32 v_res2 = v_reinterpret_as_u32(v_dotprod(v_reinterpret_as_s16(v_src2), v_reinterpret_as_s16(v_mul2)));
v_uint32 v_res3 = v_reinterpret_as_u32(v_dotprod(v_reinterpret_as_s16(v_src3), v_reinterpret_as_s16(v_mul3)));
v_store((uint16_t*)dst , v_pack_triplets(v_pack(v_res0, v_res1)));
v_store((uint16_t*)dst + 3*VECSZ/4, v_pack_triplets(v_pack(v_res2, v_res3)));
}
#endif
for (; i < dst_max; i += 1, m += 2)
{
uint8_t* px = src + 3 * ofst[i];
*(dst++) = m[0] * px[0] + m[1] * px[3];
*(dst++) = m[0] * px[1] + m[1] * px[4];
*(dst++) = m[0] * px[2] + m[1] * px[5];
}
((ufixedpoint16*)(srccn.w))[0] = (src + 3*ofst[dst_width - 1])[0];
((ufixedpoint16*)(srccn.w))[1] = (src + 3*ofst[dst_width - 1])[1];
((ufixedpoint16*)(srccn.w))[2] = (src + 3*ofst[dst_width - 1])[2];
#if CV_SIMD
v_srccn = v_pack_triplets(v_reinterpret_as_u16(vx_setall_u64(srccn.q)));
for (; i <= dst_width - (VECSZ+2)/3; i += VECSZ/4, dst += 3*VECSZ/4) // Points that fall right from src image so became equal to rightmost src point
{
v_store((uint16_t*)dst, v_srccn);
}
#endif
for (; i < dst_width; i++)
{
*(dst++) = ((ufixedpoint16*)(srccn.w))[0];
*(dst++) = ((ufixedpoint16*)(srccn.w))[1];
*(dst++) = ((ufixedpoint16*)(srccn.w))[2];
}
}
template <>
void hlineResizeCn<uint8_t, ufixedpoint16, 2, true, 4>(uint8_t* src, int, int *ofst, ufixedpoint16* m, ufixedpoint16* dst, int dst_min, int dst_max, int dst_width)
{
int i = 0;
@@ -614,20 +547,19 @@ void hlineResizeCn<uint8_t, ufixedpoint16, 2, true, 4>(uint8_t* src, int, int *o
v_store((uint16_t*)dst, v_srccn);
}
#endif
if (i < dst_min) // Points that fall left from src image so became equal to leftmost src point
for (; i < dst_min; i++, m += 2)
{
*(dst++) = ((ufixedpoint16*)(srccn.w))[0];
*(dst++) = ((ufixedpoint16*)(srccn.w))[1];
*(dst++) = ((ufixedpoint16*)(srccn.w))[2];
*(dst++) = ((ufixedpoint16*)(srccn.w))[3];
i++; m += 2;
}
#if CV_SIMD
for (; i <= dst_max - VECSZ/2; i += VECSZ/2, m += VECSZ, dst += 2*VECSZ)
{
v_uint16 v_src0, v_src1, v_src2, v_src3;
v_load_indexed4(src, ofst + i, v_src0, v_src1);
v_load_indexed4(src, ofst + i + VECSZ/4, v_src2, v_src3);
v_expand(v_interleave_quads(v_reinterpret_as_u8(vx_lut_pairs((uint32_t*)src, ofst + i))), v_src0, v_src1);
v_expand(v_interleave_quads(v_reinterpret_as_u8(vx_lut_pairs((uint32_t*)src, ofst + i + VECSZ/4))), v_src2, v_src3);
v_uint32 v_mul0, v_mul1, v_mul2, v_mul3, v_tmp;
v_mul0 = vx_load((uint32_t*)m);//AaBbCcDd
@@ -660,7 +592,7 @@ void hlineResizeCn<uint8_t, ufixedpoint16, 2, true, 4>(uint8_t* src, int, int *o
v_store((uint16_t*)dst, v_srccn);
}
#endif
if (i < dst_width)
for (; i < dst_width; i++)
{
*(dst++) = ((ufixedpoint16*)(srccn.w))[0];
*(dst++) = ((ufixedpoint16*)(srccn.w))[1];
@@ -689,10 +621,12 @@ void hlineResizeCn<uint16_t, ufixedpoint32, 2, true, 1>(uint16_t* src, int, int
for (; i <= dst_max - VECSZ; i += VECSZ, m += 2*VECSZ, dst += VECSZ)
{
v_uint32 v_src0, v_src1;
v_load_indexed_deinterleave(src, ofst + i, v_src0, v_src1);
v_uint32 v_mul0, v_mul1;
v_load_deinterleave((uint32_t*)m, v_mul0, v_mul1);
v_store((uint32_t*)dst, v_src0 * v_mul0 + v_src1 * v_mul1);//abcd
v_expand(vx_lut_pairs(src, ofst + i), v_src0, v_src1);
v_uint64 v_res0 = v_reinterpret_as_u64(v_src0 * vx_load((uint32_t*)m));
v_uint64 v_res1 = v_reinterpret_as_u64(v_src1 * vx_load((uint32_t*)m + VECSZ));
v_store((uint32_t*)dst, v_pack((v_res0 & vx_setall_u64(0xFFFFFFFF)) + (v_res0 >> 32),
(v_res1 & vx_setall_u64(0xFFFFFFFF)) + (v_res1 >> 32)));
}
#endif
for (; i < dst_max; i += 1, m += 2)
modules/imgproc/src/sumpixels.avx512_skx.cpp
+220 -84
View File
@@ -13,33 +13,34 @@ namespace { // Anonymous namespace to avoid exposing the implementation classes
// NOTE: Look at the bottom of the file for the entry-point function for external callers
//
// At the moment only 3 channel support untilted is supported
// More channel support coming soon.
// TODO: Add support for sqsum and 1,2, and 4 channels
class IntegralCalculator_3Channel {
// TODO: Add support for 1 channel input (WIP: currently hitting hardware glassjaw)
template<size_t num_channels> class IntegralCalculator;
template<size_t num_channels>
class IntegralCalculator {
public:
IntegralCalculator_3Channel() {};
IntegralCalculator() {};
void calculate_integral_avx512(const uchar *src, size_t _srcstep,
double *sum, size_t _sumstep,
double *sqsum, size_t _sqsumstep,
int width, int height, int cn)
int width, int height)
{
const int srcstep = (int)(_srcstep/sizeof(uchar));
const int sumstep = (int)(_sumstep/sizeof(double));
const int sqsumstep = (int)(_sqsumstep/sizeof(double));
const int ops_per_line = width * cn;
const int ops_per_line = width * num_channels;
// Clear the first line of the sum as per spec (see integral documentation)
// Also adjust the index of sum and sqsum to be at the real 0th element
// and not point to the border pixel so it stays in sync with the src pointer
memset( sum, 0, (ops_per_line+cn)*sizeof(double));
sum += cn;
memset( sum, 0, (ops_per_line+num_channels)*sizeof(double));
sum += num_channels;
if (sqsum) {
memset( sqsum, 0, (ops_per_line+cn)*sizeof(double));
sqsum += cn;
memset( sqsum, 0, (ops_per_line+num_channels)*sizeof(double));
sqsum += num_channels;
}
// Now calculate the integral over the whole image one line at a time
@@ -50,22 +51,22 @@ public:
double * sqsum_above = (sqsum) ? &sqsum[y*sqsumstep] : NULL;
double * sqsum_line = (sqsum) ? &sqsum_above[sqsumstep] : NULL;
integral_line_3channel_avx512(src_line, sum_line, sum_above, sqsum_line, sqsum_above, ops_per_line);
calculate_integral_for_line(src_line, sum_line, sum_above, sqsum_line, sqsum_above, ops_per_line);
}
}
static inline
void integral_line_3channel_avx512(const uchar *srcs,
double *sums, double *sums_above,
double *sqsums, double *sqsums_above,
int num_ops_in_line)
static CV_ALWAYS_INLINE
void calculate_integral_for_line(const uchar *srcs,
double *sums, double *sums_above,
double *sqsums, double *sqsums_above,
int num_ops_in_line)
{
__m512i sum_accumulator = _mm512_setzero_si512(); // holds rolling sums for the line
__m512i sqsum_accumulator = _mm512_setzero_si512(); // holds rolling sqsums for the line
// The first element on each line must be zeroes as per spec (see integral documentation)
set_border_pixel_value(sums, sqsums);
zero_out_border_pixel(sums, sqsums);
// Do all 64 byte chunk operations then do the last bits that don't fit in a 64 byte chunk
aligned_integral( srcs, sums, sums_above, sqsums, sqsums_above, sum_accumulator, sqsum_accumulator, num_ops_in_line);
@@ -74,20 +75,18 @@ public:
}
static inline
void set_border_pixel_value(double *sums, double *sqsums)
static CV_ALWAYS_INLINE
void zero_out_border_pixel(double *sums, double *sqsums)
{
// Sets the border pixel value to 0s.
// Note the hard coded -3 and the 0x7 mask is because we only support 3 channel right now
__m512i zeroes = _mm512_setzero_si512();
_mm512_mask_storeu_epi64(&sums[-3], 0x7, zeroes);
// Note the negative index is because the sums/sqsums pointers point to the first real pixel
// after the border pixel so we have to look backwards
_mm512_mask_storeu_epi64(&sums[-num_channels], (1<<num_channels)-1, _mm512_setzero_si512());
if (sqsums)
_mm512_mask_storeu_epi64(&sqsums[-3], 0x7, zeroes);
_mm512_mask_storeu_epi64(&sqsums[-num_channels], (1<<num_channels)-1, _mm512_setzero_si512());
}
static inline
static CV_ALWAYS_INLINE
void aligned_integral(const uchar *&srcs,
double *&sums, double *&sums_above,
double *&sqsum, double *&sqsum_above,
@@ -110,7 +109,7 @@ public:
}
static inline
static CV_ALWAYS_INLINE
void post_aligned_integral(const uchar *srcs,
const double *sums, const double *sums_above,
const double *sqsum, const double *sqsum_above,
@@ -132,50 +131,54 @@ public:
}
static inline
static CV_ALWAYS_INLINE
void integral_64_operations_avx512(const __m512i *srcs,
__m512i *sums, const __m512i *sums_above,
__m512i *sqsums, const __m512i *sqsums_above,
__mmask64 data_mask,
__m512i &sum_accumulator, __m512i &sqsum_accumulator)
{
__m512i src_64byte_chunk = read_64_bytes(srcs, data_mask);
__m512i src_64byte_chunk = read_64_bytes(srcs, data_mask);
for(int num_16byte_chunks=0; num_16byte_chunks<4; num_16byte_chunks++) {
__m128i src_16bytes = _mm512_extracti64x2_epi64(src_64byte_chunk, 0x0); // Get lower 16 bytes of data
while (data_mask) {
__m128i src_16bytes = extract_lower_16bytes(src_64byte_chunk);
for (int num_8byte_chunks = 0; num_8byte_chunks < 2; num_8byte_chunks++) {
__m512i src_longs_lo = convert_lower_8bytes_to_longs(src_16bytes);
__m512i src_longs_hi = convert_lower_8bytes_to_longs(shift_right_8_bytes(src_16bytes));
__m512i src_longs = convert_lower_8bytes_to_longs(src_16bytes);
// Calculate integral for the sum on the 8 lanes at a time
integral_8_operations(src_longs_lo, sums_above, data_mask, sums, sum_accumulator);
integral_8_operations(src_longs_hi, sums_above+1, data_mask>>8, sums+1, sum_accumulator);
// Calculate integral for the sum on the 8 entries
integral_8_operations(src_longs, sums_above, data_mask, sums, sum_accumulator);
sums++; sums_above++;
if (sqsums){ // Calculate integral for the sum on the 8 entries
__m512i squared_source = _mm512_mullo_epi64(src_longs, src_longs);
integral_8_operations(squared_source, sqsums_above, data_mask, sqsums, sqsum_accumulator);
sqsums++; sqsums_above++;
}
// Prepare for next iteration of inner loop
// shift source to align next 8 bytes to lane 0 and shift the mask
src_16bytes = shift_right_8_bytes(src_16bytes);
data_mask = data_mask >> 8;
if (sqsums) {
__m512i squared_source_lo = square_m512(src_longs_lo);
__m512i squared_source_hi = square_m512(src_longs_hi);
integral_8_operations(squared_source_lo, sqsums_above, data_mask, sqsums, sqsum_accumulator);
integral_8_operations(squared_source_hi, sqsums_above+1, data_mask>>8, sqsums+1, sqsum_accumulator);
sqsums += 2;
sqsums_above+=2;
}
// Prepare for next iteration of outer loop
// Prepare for next iteration of loop
// shift source to align next 16 bytes to lane 0, shift the mask, and advance the pointers
sums += 2;
sums_above += 2;
data_mask = data_mask >> 16;
src_64byte_chunk = shift_right_16_bytes(src_64byte_chunk);
}
}
static inline
static CV_ALWAYS_INLINE
void integral_8_operations(const __m512i src_longs, const __m512i *above_values_ptr, __mmask64 data_mask,
__m512i *results_ptr, __m512i &accumulator)
{
// NOTE that the calculate_integral function referenced here must be implemented in the templated
// derivatives because the algorithm depends heavily on the number of channels in the image
//
_mm512_mask_storeu_pd(
results_ptr, // Store the result here
data_mask, // Using the data mask to avoid overrunning the line
@@ -188,59 +191,178 @@ public:
}
static inline
__m512d calculate_integral(__m512i src_longs, const __m512d above_values, __m512i &accumulator)
{
__m512i carryover_idxs = _mm512_set_epi64(6, 5, 7, 6, 5, 7, 6, 5);
// Align data to prepare for the adds:
// shifts data left by 3 and 6 qwords(lanes) and gets rolling sum in all lanes
// Vertical LANES: 76543210
// src_longs : HGFEDCBA
// shited3lanes : + EDCBA
// shifted6lanes : + BA
// carry_over_idxs : + 65765765 (index position of result from previous iteration)
// = integral
__m512i shifted3lanes = _mm512_maskz_expand_epi64(0xF8, src_longs);
__m512i shifted6lanes = _mm512_maskz_expand_epi64(0xC0, src_longs);
__m512i carry_over = _mm512_permutex2var_epi64(accumulator, carryover_idxs, accumulator);
// Do the adds in tree form (shift3 + shift 6) + (current_source_values + accumulator)
__m512i sum_shift3and6 = _mm512_add_epi64(shifted3lanes, shifted6lanes);
__m512i sum_src_carry = _mm512_add_epi64(src_longs, carry_over);
accumulator = _mm512_add_epi64(sum_shift3and6, sum_src_carry);
// Convert to packed double and add to the line above to get the true integral value
__m512d accumulator_pd = _mm512_cvtepu64_pd(accumulator);
__m512d integral_pd = _mm512_add_pd(accumulator_pd, above_values);
return integral_pd;
}
// The calculate_integral function referenced here must be implemented in the templated derivatives
// because the algorithm depends heavily on the number of channels in the image
// This is the incomplete definition (just the prototype) here.
//
static CV_ALWAYS_INLINE
__m512d calculate_integral(__m512i src_longs, const __m512d above_values, __m512i &accumulator);
static inline
static CV_ALWAYS_INLINE
__m512i read_64_bytes(const __m512i *srcs, __mmask64 data_mask) {
return _mm512_maskz_loadu_epi8(data_mask, srcs);
}
static inline
static CV_ALWAYS_INLINE
__m128i extract_lower_16bytes(__m512i src_64byte_chunk) {
return _mm512_extracti64x2_epi64(src_64byte_chunk, 0x0);
}
static CV_ALWAYS_INLINE
__m512i convert_lower_8bytes_to_longs(__m128i src_16bytes) {
return _mm512_cvtepu8_epi64(src_16bytes);
}
static inline
static CV_ALWAYS_INLINE
__m512i square_m512(__m512i src_longs) {
return _mm512_mullo_epi64(src_longs, src_longs);
}
static CV_ALWAYS_INLINE
__m128i shift_right_8_bytes(__m128i src_16bytes) {
return _mm_maskz_compress_epi64(2, src_16bytes);
}
static inline
static CV_ALWAYS_INLINE
__m512i shift_right_16_bytes(__m512i src_64byte_chunk) {
return _mm512_maskz_compress_epi64(0xFC, src_64byte_chunk);
}
};
//============================================================================================================
// This the only section that needs to change with respect to algorithm based on the number of channels
// It is responsible for returning the calculation of 8 lanes worth of the integral and returning in the
// accumulated sums in the accumulator parameter (NOTE: accumulator is an input and output parameter)
//
// The function prototype that needs to be implemented is:
//
// __m512d calculate_integral(__m512i src_longs, const __m512d above_values, __m512i &accumulator){ ... }
//
// Description of parameters:
// INPUTS:
// src_longs : 8 lanes worth of the source bytes converted to 64 bit integers
// above_values: 8 lanes worth of the result values from the line above (See the integral spec)
// accumulator : 8 lanes worth of sums from the previous iteration
// IMPORTANT NOTE: This parameter is both an INPUT AND OUTPUT parameter to this function
//
// OUTPUTS:
// return value: The actual integral value for all 8 lanes which is defined by the spec as
// the sum of all channel values to the left of a given pixel plus the result
// written to the line directly above the current line.
// accumulator: This is an input and and output. This parameter should be left with the accumulated
// sums for the current 8 lanes keeping all entries in the proper lane (do not shuffle it)
//
// Below here is the channel specific implementation
//
//========================================
// 2 Channel Integral Implementation
//========================================
template<>
CV_ALWAYS_INLINE
__m512d IntegralCalculator < 2 > ::calculate_integral(__m512i src_longs, const __m512d above_values, __m512i &accumulator)
{
__m512i carryover_idxs = _mm512_set_epi64(7, 6, 7, 6, 7, 6, 7, 6);
// Align data to prepare for the adds:
// shifts data left by 3 and 6 qwords(lanes) and gets rolling sum in all lanes
// Vertical LANES : 76543210
// src_longs : HGFEDCBA
// shift2lanes : + FEDCBA
// shift4lanes : + DCBA
// shift6lanes : + BA
// carry_over_idxs : + 76767676 (index position of result from previous iteration)
// = integral
__m512i shift2lanes = _mm512_maskz_expand_epi64(0xFC, src_longs);
__m512i shift4lanes = _mm512_maskz_expand_epi64(0xF0, src_longs);
__m512i shift6lanes = _mm512_maskz_expand_epi64(0xC0, src_longs);
__m512i carry_over = _mm512_permutex2var_epi64(accumulator, carryover_idxs, accumulator);
// Add all values in tree form for perf ((0+2) + (4+6))
__m512i sum_shift_02 = _mm512_add_epi64(src_longs, shift2lanes);
__m512i sum_shift_46 = _mm512_add_epi64(shift4lanes, shift6lanes);
__m512i sum_all = _mm512_add_epi64(sum_shift_02, sum_shift_46);
accumulator = _mm512_add_epi64(sum_all, carry_over);
// Convert to packed double and add to the line above to get the true integral value
__m512d accumulator_pd = _mm512_cvtepu64_pd(accumulator);
__m512d integral_pd = _mm512_add_pd(accumulator_pd, above_values);
return integral_pd;
}
//========================================
// 3 Channel Integral Implementation
//========================================
template<>
CV_ALWAYS_INLINE
__m512d IntegralCalculator < 3 > ::calculate_integral(__m512i src_longs, const __m512d above_values, __m512i &accumulator)
{
__m512i carryover_idxs = _mm512_set_epi64(6, 5, 7, 6, 5, 7, 6, 5);
// Align data to prepare for the adds:
// shifts data left by 3 and 6 qwords(lanes) and gets rolling sum in all lanes
// Vertical LANES: 76543210
// src_longs : HGFEDCBA
// shit3lanes : + EDCBA
// shift6lanes : + BA
// carry_over_idxs : + 65765765 (index position of result from previous iteration)
// = integral
__m512i shift3lanes = _mm512_maskz_expand_epi64(0xF8, src_longs);
__m512i shift6lanes = _mm512_maskz_expand_epi64(0xC0, src_longs);
__m512i carry_over = _mm512_permutex2var_epi64(accumulator, carryover_idxs, accumulator);
// Do the adds in tree form
__m512i sum_shift_03 = _mm512_add_epi64(src_longs, shift3lanes);
__m512i sum_rest = _mm512_add_epi64(shift6lanes, carry_over);
accumulator = _mm512_add_epi64(sum_shift_03, sum_rest);
// Convert to packed double and add to the line above to get the true integral value
__m512d accumulator_pd = _mm512_cvtepu64_pd(accumulator);
__m512d integral_pd = _mm512_add_pd(accumulator_pd, above_values);
return integral_pd;
}
//========================================
// 4 Channel Integral Implementation
//========================================
template<>
CV_ALWAYS_INLINE
__m512d IntegralCalculator < 4 > ::calculate_integral(__m512i src_longs, const __m512d above_values, __m512i &accumulator)
{
__m512i carryover_idxs = _mm512_set_epi64(7, 6, 5, 4, 7, 6, 5, 4);
// Align data to prepare for the adds:
// shifts data left by 3 and 6 qwords(lanes) and gets rolling sum in all lanes
// Vertical LANES: 76543210
// src_longs : HGFEDCBA
// shit4lanes : + DCBA
// carry_over_idxs : + 76547654 (index position of result from previous iteration)
// = integral
__m512i shifted4lanes = _mm512_maskz_expand_epi64(0xF0, src_longs);
__m512i carry_over = _mm512_permutex2var_epi64(accumulator, carryover_idxs, accumulator);
// Add data pixels and carry over from last iteration
__m512i sum_shift_04 = _mm512_add_epi64(src_longs, shifted4lanes);
accumulator = _mm512_add_epi64(sum_shift_04, carry_over);
// Convert to packed double and add to the line above to get the true integral value
__m512d accumulator_pd = _mm512_cvtepu64_pd(accumulator);
__m512d integral_pd = _mm512_add_pd(accumulator_pd, above_values);
return integral_pd;
}
} // end of anonymous namespace
namespace opt_AVX512_SKX {
@@ -253,8 +375,22 @@ void calculate_integral_avx512(const uchar *src, size_t _srcstep,
double *sqsum, size_t _sqsumstep,
int width, int height, int cn)
{
IntegralCalculator_3Channel calculator;
calculator.calculate_integral_avx512(src, _srcstep, sum, _sumstep, sqsum, _sqsumstep, width, height, cn);
switch(cn){
case 2: {
IntegralCalculator<2> calculator;
calculator.calculate_integral_avx512(src, _srcstep, sum, _sumstep, sqsum, _sqsumstep, width, height);
break;
}
case 3: {
IntegralCalculator<3> calculator;
calculator.calculate_integral_avx512(src, _srcstep, sum, _sumstep, sqsum, _sqsumstep, width, height);
break;
}
case 4: {
IntegralCalculator<4> calculator;
calculator.calculate_integral_avx512(src, _srcstep, sum, _sumstep, sqsum, _sqsumstep, width, height);
}
}
}
modules/imgproc/src/sumpixels.cpp
+2 -3
View File
@@ -46,7 +46,6 @@
#include "opencv2/core/hal/intrin.hpp"
#include "sumpixels.hpp"
namespace cv
{
@@ -77,8 +76,8 @@ struct Integral_SIMD<uchar, double, double> {
{
#if CV_TRY_AVX512_SKX
CV_UNUSED(_tiltedstep);
// TODO: Add support for 1,2, and 4 channels
if (CV_CPU_HAS_SUPPORT_AVX512_SKX && !tilted && cn == 3){
// TODO: Add support for 1 channel input (WIP)
if (CV_CPU_HAS_SUPPORT_AVX512_SKX && !tilted && ((cn >= 2) && (cn <= 4))){
opt_AVX512_SKX::calculate_integral_avx512(src, _srcstep, sum, _sumstep,
sqsum, _sqsumstep, width, height, cn);
return true;