potok/opencv
1
0
Fork 0
mirror of https://github.com/opencv/opencv.git synced 2026-07-29 23:33:05 +04:00
Code Packages Releases Wiki Activity
Files
4c71dbf0af70e8728b8a791695b083540af72887
opencv/3rdparty/carotene/src/pyramid.cpp
T
Khem Raj 2c7a123ed8 Do not enable asm with clang 
clang pretends to be gcc 4.2.0 which means we will
use inline asm for no reason, instead of builtins
on clang when possible.

Signed-off-by: Khem Raj <raj.khem@gmail.com>
2017-09-19 18:09:30 -07:00

1415 lines
58 KiB
C++
Raw Blame History

/*
* By downloading, copying, installing or using the software you agree to this license.
* If you do not agree to this license, do not download, install,
* copy or use the software.
*
*
* License Agreement
* For Open Source Computer Vision Library
* (3-clause BSD License)
*
* Copyright (C) 2012-2015, NVIDIA Corporation, all rights reserved.
* Third party copyrights are property of their respective owners.
*
* Redistribution and use in source and binary forms, with or without modification,
* are permitted provided that the following conditions are met:
*
* * Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
*
* * Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* * Neither the names of the copyright holders nor the names of the contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
* This software is provided by the copyright holders and contributors "as is" and
* any express or implied warranties, including, but not limited to, the implied
* warranties of merchantability and fitness for a particular purpose are disclaimed.
* In no event shall copyright holders or contributors be liable for any direct,
* indirect, incidental, special, exemplary, or consequential damages
* (including, but not limited to, procurement of substitute goods or services;
* loss of use, data, or profits; or business interruption) however caused
* and on any theory of liability, whether in contract, strict liability,
* or tort (including negligence or otherwise) arising in any way out of
* the use of this software, even if advised of the possibility of such damage.
*/
#include "common.hpp"
#include <vector>
namespace CAROTENE_NS {
bool isGaussianPyramidDownRTZSupported(const Size2D &srcSize, const Size2D &dstSize, BORDER_MODE border_mode)
{
if (!isSupportedConfiguration())
return false;
// Need at least 8 pixels for vectorization.
// Need to make sure dst width is half the src width.
// Don't care about dst height.
if ( dstSize.width < 8 || std::abs((ptrdiff_t)dstSize.width*2 - (ptrdiff_t)srcSize.width) > 2 )
return false;
// Current implementation only supports Reflect101 (ie: UNDEFINED mode)
if (border_mode != BORDER_MODE_UNDEFINED)
return false;
return true;
}
bool isGaussianPyramidDownU8Supported(const Size2D &srcSize, const Size2D &dstSize, u8 cn)
{
if (!isSupportedConfiguration())
return false;
if ( (dstSize.width * cn) < 8 ||
(cn != 1 && cn !=3 && cn!=4) ||
std::abs((ptrdiff_t)dstSize.width*2 - (ptrdiff_t)srcSize.width) > 2 ||
std::abs((ptrdiff_t)dstSize.height*2 - (ptrdiff_t)srcSize.height) > 2 )
return false;
return true;
}
bool isGaussianPyramidDownS16Supported(const Size2D &srcSize, const Size2D &dstSize, u8 cn)
{
if (!isSupportedConfiguration())
return false;
if ( (dstSize.width * cn) < 4 ||
(cn != 1 && cn !=3 && cn!=4) ||
std::abs((ptrdiff_t)dstSize.width*2 - (ptrdiff_t)srcSize.width) > 2 ||
std::abs((ptrdiff_t)dstSize.height*2 - (ptrdiff_t)srcSize.height) > 2 )
return false;
return true;
}
bool isGaussianPyramidDownF32Supported(const Size2D &srcSize, const Size2D &dstSize, u8 cn)
{
if (!isSupportedConfiguration())
return false;
if ( (dstSize.width * cn) < 4 ||
(cn != 1 && cn !=3 && cn!=4) ||
std::abs((ptrdiff_t)dstSize.width*2 - (ptrdiff_t)srcSize.width) > 2 ||
std::abs((ptrdiff_t)dstSize.height*2 - (ptrdiff_t)srcSize.height) > 2 )
return false;
return true;
}
bool isGaussianPyramidUpU8Supported(const Size2D &srcSize, const Size2D &dstSize, u8 cn)
{
if (!isSupportedConfiguration())
return false;
if ( (srcSize.width * cn) < 8 ||
(cn != 1 && cn !=3 && cn!=4) ||
std::abs((ptrdiff_t)dstSize.width - (ptrdiff_t)srcSize.width*2) != (ptrdiff_t)dstSize.width % 2 ||
std::abs((ptrdiff_t)dstSize.height - (ptrdiff_t)srcSize.height*2) != (ptrdiff_t)dstSize.height % 2 )
return false;
return true;
}
bool isGaussianPyramidUpS16Supported(const Size2D &srcSize, const Size2D &dstSize, u8 cn)
{
if (!isSupportedConfiguration())
return false;
if ( (srcSize.width * cn) < 12 ||
(cn != 1 && cn !=3 && cn!=4) ||
std::abs((ptrdiff_t)dstSize.width - (ptrdiff_t)srcSize.width*2) != (ptrdiff_t)dstSize.width % 2 ||
std::abs((ptrdiff_t)dstSize.height - (ptrdiff_t)srcSize.height*2) != (ptrdiff_t)dstSize.height % 2 )
return false;
return true;
}
#ifdef CAROTENE_NEON
namespace {
ptrdiff_t borderInterpolate101(ptrdiff_t p, ptrdiff_t len)
{
if (len == 1)
return 0;
else
{
while ((unsigned)p >= (unsigned)len)
{
if (p < 0)
p = -p;
else
p = (len - 1)*2 - p;
}
}
return p;
}
} // namespace
#endif
void gaussianPyramidDownRTZ(const Size2D &srcSize,
const u8 *srcBase, ptrdiff_t srcStride,
const Size2D &dstSize,
u8 *dstBase, ptrdiff_t dstStride,
BORDER_MODE border, u8 borderValue)
{
internal::assertSupportedConfiguration(isGaussianPyramidDownRTZSupported(srcSize, dstSize, border));
#ifdef CAROTENE_NEON
// Single-core NEON code
const size_t dwidth = dstSize.width;
const size_t dheight = dstSize.height;
const size_t swidth = srcSize.width;
const size_t sheight = srcSize.height;
ptrdiff_t idx_l1 = borderInterpolate101(-1, swidth);
ptrdiff_t idx_l2 = borderInterpolate101(-2, swidth);
ptrdiff_t idx_r1 = borderInterpolate101(swidth + 0, swidth);
ptrdiff_t idx_r2 = borderInterpolate101(swidth + 1, swidth);
//1-line buffer
std::vector<u16> _buf((swidth + 4) + 32/sizeof(u16));
u16* lane = internal::alignPtr(&_buf[2], 32);
uint8x8_t vc6u8 = vmov_n_u8(6);
uint16x8_t vc6u16 = vmovq_n_u16(6);
uint16x8_t vc4u16 = vmovq_n_u16(4);
u8* dst = dstBase;
for (size_t i = 0; i < dheight; ++i, dst += dstStride)
{
//vertical convolution
const u8* ln0 = internal::getRowPtr(srcBase, srcStride, borderInterpolate101(i*2-2, sheight));
const u8* ln1 = internal::getRowPtr(srcBase, srcStride, borderInterpolate101(i*2-1, sheight));
const u8* ln2 = internal::getRowPtr(srcBase, srcStride, borderInterpolate101(i*2+0, sheight));
const u8* ln3 = internal::getRowPtr(srcBase, srcStride, borderInterpolate101(i*2+1, sheight));
const u8* ln4 = internal::getRowPtr(srcBase, srcStride, borderInterpolate101(i*2+2, sheight));
size_t x = 0;
for (; x <= swidth - 8; x += 8)
{
internal::prefetch(internal::getRowPtr(ln2 + x, srcStride, x % 5 - 2));
uint8x8_t v0 = vld1_u8(ln0+x);
uint8x8_t v1 = vld1_u8(ln1+x);
uint8x8_t v2 = vld1_u8(ln2+x);
uint8x8_t v3 = vld1_u8(ln3+x);
uint8x8_t v4 = vld1_u8(ln4+x);
uint16x8_t v = vaddl_u8(v0, v4);
uint16x8_t v13 = vaddl_u8(v1, v3);
v = vmlal_u8(v, v2, vc6u8);
v = vmlaq_u16(v, v13, vc4u16);
vst1q_u16(lane + x, v);
}
for (; x < swidth; ++x)
{
lane[x] = ln0[x] + ln4[x] + 4u * (ln1[x] + ln3[x]) + 6u * ln2[x];
}
//left&right borders
lane[-1] = lane[idx_l1];
lane[-2] = lane[idx_l2];
lane[swidth] = lane[idx_r1];
lane[swidth+1] = lane[idx_r2];
//horizontal convolution
x = 0;
size_t vw = (swidth/2) - 7; // Using 7 instead of 8 allows swidth of 14 or 15.
for (; x < vw; x += 8)
{
internal::prefetch(lane + 2 * x);
uint16x8x2_t vLane0 = vld2q_u16(lane + 2*x-2); // L0[0] = x0 x2 x4 x6 x8 x10 x12 x14 L0[1] = x1 x3 x5 x7 x9 x11 x13 x15
uint16x8x2_t vLane1 = vld2q_u16(lane + 2*x-1); // L1[0] = x1 x3 x5 x7 x9 x11 x13 x15 L1[1] = x2 x4 x6 x8 x10 x12 x14 x16
uint16x8x2_t vLane2 = vld2q_u16(lane + 2*x+0); // L2[0] = x2 x4 x6 x8 x10 x12 x14 x16 L2[1] = x3 x5 x7 x9 x11 x13 x15 x17
uint16x8x2_t vLane3 = vld2q_u16(lane + 2*x+1); // L3[0] = x3 x5 x7 x9 x11 x13 x15 x17 L3[1] = x4 x6 x8 x10 x12 x14 x16 x18
uint16x8x2_t vLane4 = vld2q_u16(lane + 2*x+2); // L4[0] = x4 x6 x8 x10 x12 x14 x16 x18 L4[1] = x5 x7 x9 x11 x13 x15 x17 x19
uint16x8_t vSum_0_4 = vaddq_u16(vLane0.val[0], vLane4.val[0]);
uint16x8_t vSum_1_3 = vaddq_u16(vLane1.val[0], vLane3.val[0]);
vSum_0_4 = vmlaq_u16(vSum_0_4, vLane2.val[0], vc6u16);
vSum_0_4 = vmlaq_u16(vSum_0_4, vSum_1_3, vc4u16);
uint8x8_t vRes = vshrn_n_u16(vSum_0_4, 8);
vst1_u8(dst + x, vRes);
}
for (; x < dwidth; x++)
{
dst[x] = u8((lane[2*x-2] + lane[2*x+2] + 4u * (lane[2*x-1] + lane[2*x+1]) + 6u * lane[2*x]) >> 8);
}
}
#else
// Remove 'unused parameter' warnings.
(void)srcSize;
(void)srcBase;
(void)srcStride;
(void)dstSize;
(void)dstBase;
(void)dstStride;
(void)border;
#endif
(void)borderValue;
}
void gaussianPyramidDown(const Size2D &srcSize,
const u8 *srcBase, ptrdiff_t srcStride,
const Size2D &dstSize,
u8 *dstBase, ptrdiff_t dstStride, u8 cn)
{
internal::assertSupportedConfiguration(isGaussianPyramidDownU8Supported(srcSize, dstSize, cn));
#ifdef CAROTENE_NEON
size_t dcolcn = dstSize.width*cn;
size_t scolcn = srcSize.width*cn;
size_t roiw8 = dcolcn - 7;
size_t idx_l1 = borderInterpolate101(-1, srcSize.width) * cn;
size_t idx_l2 = borderInterpolate101(-2, srcSize.width) * cn;
size_t idx_r1 = borderInterpolate101(srcSize.width + 0, srcSize.width) * cn;
size_t idx_r2 = borderInterpolate101(srcSize.width + 1, srcSize.width) * cn;
//1-line buffer
std::vector<u16> _buf(cn*(srcSize.width + 4) + 32/sizeof(u16));
u16* lane = internal::alignPtr(&_buf[2*cn], 32);
uint8x8_t vc6u8 = vmov_n_u8(6);
uint16x8_t vc6u16 = vmovq_n_u16(6);
uint16x8_t vc4u16 = vmovq_n_u16(4);
for (size_t i = 0; i < dstSize.height; ++i)
{
u8* dst = internal::getRowPtr(dstBase, dstStride, i);
//vertical convolution
const u8* ln0 = internal::getRowPtr(srcBase, srcStride, borderInterpolate101(i*2-2, srcSize.height));
const u8* ln1 = internal::getRowPtr(srcBase, srcStride, borderInterpolate101(i*2-1, srcSize.height));
const u8* ln2 = internal::getRowPtr(srcBase, srcStride, borderInterpolate101(i*2+0, srcSize.height));
const u8* ln3 = internal::getRowPtr(srcBase, srcStride, borderInterpolate101(i*2+1, srcSize.height));
const u8* ln4 = internal::getRowPtr(srcBase, srcStride, borderInterpolate101(i*2+2, srcSize.height));
size_t x = 0;
for (; x <= scolcn - 8; x += 8)
{
internal::prefetch(internal::getRowPtr(ln2 + x, srcStride, (ptrdiff_t)x % 5 - 2));
uint8x8_t v0 = vld1_u8(ln0+x);
uint8x8_t v1 = vld1_u8(ln1+x);
uint8x8_t v2 = vld1_u8(ln2+x);
uint8x8_t v3 = vld1_u8(ln3+x);
uint8x8_t v4 = vld1_u8(ln4+x);
uint16x8_t v = vaddl_u8(v0, v4);
uint16x8_t v13 = vaddl_u8(v1, v3);
v = vmlal_u8(v, v2, vc6u8);
v = vmlaq_u16(v, v13, vc4u16);
vst1q_u16(lane + x, v);
}
for (; x < scolcn; ++x)
{
lane[x] = ln0[x] + ln4[x] + 4u * (ln1[x] + ln3[x]) + 6u * ln2[x];
}
//left&right borders
for (u32 k = 0; k < cn; ++k)
{
lane[(s32)(-cn+k)] = lane[idx_l1 + k];
lane[(s32)(-cn-cn+k)] = lane[idx_l2 + k];
lane[scolcn+k] = lane[idx_r1 + k];
lane[scolcn+cn+k] = lane[idx_r2 + k];
}
//horizontal convolution
x = 0;
switch(cn)
{
case 1:
for (; x < roiw8; x += 8)
{
internal::prefetch(lane + 2 * x);
#if !defined(__aarch64__) && defined(__GNUC__) && __GNUC__ == 4 && __GNUC_MINOR__ < 7 && !defined(__clang__)
__asm__ (
"vld2.16 {d0-d3}, [%[in0]] \n\t"
"vld2.16 {d4-d7}, [%[in4]] \n\t"
"vld2.16 {d12-d15}, [%[in1]] \n\t"
"vld2.16 {d16-d19}, [%[in3]] \n\t"
"vld2.16 {d8-d11}, [%[in2],:256] \n\t"
"vadd.i16 q0, q2 /*q0 = v0 + v4*/ \n\t"
"vadd.i16 q6, q8 /*q6 = v1 + v3*/ \n\t"
"vmla.i16 q0, q4, %q[c6] /*q0 += v2 * 6*/ \n\t"
"vmla.i16 q0, q6, %q[c4] /*q1 += (v1+v3) * 4*/ \n\t"
"vrshrn.u16 d8, q0, #8 \n\t"
"vst1.8 {d8}, [%[out]] \n\t"
: /*no output*/
: [out] "r" (dst + x),
[in0] "r" (lane + 2*x-2),
[in1] "r" (lane + 2*x-1),
[in2] "r" (lane + 2*x+0),
[in3] "r" (lane + 2*x+1),
[in4] "r" (lane + 2*x+2),
[c4] "w" (vc4u16), [c6] "w" (vc6u16)
: "d0","d1","d2","d3","d4","d5","d6","d7","d8","d9","d10","d11","d12","d13","d14","d15","d16","d17","d18","d19"
);
#else
uint16x8x2_t vLane0 = vld2q_u16(lane + 2*x-2);
uint16x8x2_t vLane1 = vld2q_u16(lane + 2*x-1);
uint16x8x2_t vLane2 = vld2q_u16(lane + 2*x+0);
uint16x8x2_t vLane3 = vld2q_u16(lane + 2*x+1);
uint16x8x2_t vLane4 = vld2q_u16(lane + 2*x+2);
uint16x8_t vSum_0_4 = vaddq_u16(vLane0.val[0], vLane4.val[0]);
uint16x8_t vSum_1_3 = vaddq_u16(vLane1.val[0], vLane3.val[0]);
vSum_0_4 = vmlaq_u16(vSum_0_4, vLane2.val[0], vc6u16);
vSum_0_4 = vmlaq_u16(vSum_0_4, vSum_1_3, vc4u16);
uint8x8_t vRes = vrshrn_n_u16(vSum_0_4, 8);
vst1_u8(dst + x, vRes);
#endif
}
break;
case 3:
{
uint16x4_t vx1 = vld1_u16(lane - 2*3);
uint16x4_t vx2 = vld1_u16(lane - 1*3);
uint16x4_t vx3 = vld1_u16(lane + 0*3);
uint16x8_t v0 = vcombine_u16(vx1, vx3);
uint8x8_t map = vreinterpret_u8_u64(vmov_n_u64(0xFFFF060504020100ULL));
for (; x < roiw8; x += 6)
{
internal::prefetch(lane + 2 * x + 12);
uint16x4_t vx_ = vld1_u16(lane + 2*x-1*3 + 6);
uint16x4_t vx4 = vld1_u16(lane + 2*x+0*3 + 6);
uint16x4_t vx5 = vld1_u16(lane + 2*x+1*3 + 6);
uint16x4_t vx6 = vld1_u16(lane + 2*x+2*3 + 6);
uint16x8_t v1 = vcombine_u16(vx2, vx_);
uint16x8_t v2 = vcombine_u16(vget_high_u16(v0), vx4);
uint16x8_t v3 = vcombine_u16(vx_, vx5);
uint16x8_t v4 = vcombine_u16(vx4, vx6);
vx2 = vx5;
uint16x8_t v = vaddq_u16(v0, v4);
uint16x8_t v13 = vaddq_u16(v1, v3);
v = vmlaq_u16(v, v2, vc6u16);
v = vmlaq_u16(v, v13, vc4u16);
uint8x8_t v8 = vrshrn_n_u16(v, 8);
v0 = v4;
vst1_u8(dst + x, vtbl1_u8(v8, map));
}
}
break;
case 4:
{
uint16x4_t vx1 = vld1_u16(lane - 2*4);
uint16x4_t vx2 = vld1_u16(lane - 1*4);
uint16x4_t vx3 = vld1_u16(lane + 0*4);
uint16x8_t v0 = vcombine_u16(vx1, vx3);
for (; x < roiw8; x += 8)
{
internal::prefetch(lane + 2 * x + 16);
uint16x4_t vx_ = vld1_u16(lane + 2 * x - 1*4 + 8);
uint16x4_t vx4 = vld1_u16(lane + 2 * x + 0*4 + 8);
uint16x4_t vx5 = vld1_u16(lane + 2 * x + 1*4 + 8);
uint16x4_t vx6 = vld1_u16(lane + 2 * x + 2*4 + 8);
uint16x8_t v1 = vcombine_u16(vx2, vx_);
uint16x8_t v2 = vcombine_u16(vget_high_u16(v0), vx4);
uint16x8_t v3 = vcombine_u16(vx_, vx5);
uint16x8_t v4 = vcombine_u16(vx4, vx6);
vx2 = vx5;
uint16x8_t v = vaddq_u16(v0, v4);
uint16x8_t v13 = vaddq_u16(v1, v3);
v = vmlaq_u16(v, v2, vc6u16);
v = vmlaq_u16(v, v13, vc4u16);
uint8x8_t v8 = vrshrn_n_u16(v, 8);
v0 = v4;
vst1_u8(dst + x, v8);
}
}
break;
}
for (u32 h = 0; h < cn; ++h)
{
u16* ln = lane + h;
u8* dt = dst + h;
for (size_t k = x; k < dcolcn; k += cn)
dt[k] = u8((ln[2*k-2*cn] + ln[2*k+2*cn] + 4u * (ln[2*k-cn] + ln[2*k+cn]) + 6u * ln[2*k] + (1 << 7)) >> 8);
}
}
#else
// Remove 'unused parameter' warnings.
(void)srcBase;
(void)srcStride;
(void)dstBase;
(void)dstStride;
#endif
}
void gaussianPyramidDown(const Size2D &srcSize,
const s16 *srcBase, ptrdiff_t srcStride,
const Size2D &dstSize,
s16 *dstBase, ptrdiff_t dstStride, u8 cn)
{
internal::assertSupportedConfiguration(isGaussianPyramidDownS16Supported(srcSize, dstSize, cn));
#ifdef CAROTENE_NEON
size_t dcolcn = dstSize.width*cn;
size_t scolcn = srcSize.width*cn;
size_t roiw4 = dcolcn - 3;
size_t idx_l1 = borderInterpolate101(-1, srcSize.width) * cn;
size_t idx_l2 = borderInterpolate101(-2, srcSize.width) * cn;
size_t idx_r1 = borderInterpolate101(srcSize.width + 0, srcSize.width) * cn;
size_t idx_r2 = borderInterpolate101(srcSize.width + 1, srcSize.width) * cn;
//1-line buffer
std::vector<s32> _buf(cn*(srcSize.width + 4) + 32/sizeof(s32));
s32* lane = internal::alignPtr(&_buf[2*cn], 32);
int16x4_t vc6s16 = vmov_n_s16(6);
int32x4_t vc6s32 = vmovq_n_s32(6);
int32x4_t vc4s32 = vmovq_n_s32(4);
for (size_t i = 0; i < dstSize.height; ++i)
{
s16* dst = internal::getRowPtr(dstBase, dstStride, i);
//vertical convolution
const s16* ln0 = internal::getRowPtr(srcBase, srcStride, borderInterpolate101(i*2-2, srcSize.height));
const s16* ln1 = internal::getRowPtr(srcBase, srcStride, borderInterpolate101(i*2-1, srcSize.height));
const s16* ln2 = internal::getRowPtr(srcBase, srcStride, borderInterpolate101(i*2+0, srcSize.height));
const s16* ln3 = internal::getRowPtr(srcBase, srcStride, borderInterpolate101(i*2+1, srcSize.height));
const s16* ln4 = internal::getRowPtr(srcBase, srcStride, borderInterpolate101(i*2+2, srcSize.height));
size_t x = 0;
for (; x <= scolcn - 4; x += 4)
{
internal::prefetch(internal::getRowPtr(ln2 + x, srcStride, (ptrdiff_t)x % 5 - 2));
int16x4_t v0 = vld1_s16(ln0 + x);
int16x4_t v1 = vld1_s16(ln1 + x);
int16x4_t v2 = vld1_s16(ln2 + x);
int16x4_t v3 = vld1_s16(ln3 + x);
int16x4_t v4 = vld1_s16(ln4 + x);
int32x4_t v = vaddl_s16(v0, v4);
int32x4_t v13 = vaddl_s16(v1, v3);
v = vmlal_s16(v, v2, vc6s16);
v = vmlaq_s32(v, v13, vc4s32);
vst1q_s32(lane + x, v);
}
for (; x < scolcn; ++x)
{
lane[x] = ln0[x] + ln4[x] + 4 * (ln1[x] + ln3[x]) + 6 * ln2[x];
}
//left&right borders
for (u32 k = 0; k < cn; ++k)
{
lane[(s32)(-cn+k)] = lane[idx_l1 + k];
lane[(s32)(-cn-cn+k)] = lane[idx_l2 + k];
lane[scolcn+k] = lane[idx_r1 + k];
lane[scolcn+cn+k] = lane[idx_r2 + k];
}
//horizontal convolution
x = 0;
switch(cn)
{
case 1:
for (; x < roiw4; x += 4)
{
internal::prefetch(lane + 2 * x);
#if !defined(__aarch64__) && defined(__GNUC__) && __GNUC__ == 4 && __GNUC_MINOR__ < 7 && !defined(__clang__)
__asm__ (
"vld2.32 {d0-d3}, [%[in0]] \n\t"
"vld2.32 {d4-d7}, [%[in4]] \n\t"
"vld2.32 {d12-d15}, [%[in1]] \n\t"
"vld2.32 {d16-d19}, [%[in3]] \n\t"
"vld2.32 {d8-d11}, [%[in2],:256] \n\t"
"vadd.i32 q0, q2 \n\t"
"vadd.i32 q6, q8 \n\t"
"vmla.i32 q0, q4, %q[c6] \n\t"
"vmla.i32 q0, q6, %q[c4] \n\t"
"vrshrn.s32 d8, q0, #8 \n\t"
"vst1.16 {d8}, [%[out]] \n\t"
: /*no output*/
: [out] "r" (dst + x),
[in0] "r" (lane + 2*x-2),
[in1] "r" (lane + 2*x-1),
[in2] "r" (lane + 2*x+0),
[in3] "r" (lane + 2*x+1),
[in4] "r" (lane + 2*x+2),
[c4] "w" (vc4s32), [c6] "w" (vc6s32)
: "d0","d1","d2","d3","d4","d5","d6","d7","d8","d9","d10","d11","d12","d13","d14","d15","d16","d17","d18","d19"
);
#else
int32x4x2_t vLane0 = vld2q_s32(lane + 2*x-2);
int32x4x2_t vLane1 = vld2q_s32(lane + 2*x-1);
int32x4x2_t vLane2 = vld2q_s32(lane + 2*x+0);
int32x4x2_t vLane3 = vld2q_s32(lane + 2*x+1);
int32x4x2_t vLane4 = vld2q_s32(lane + 2*x+2);
int32x4_t vSum_0_4 = vaddq_s32(vLane0.val[0], vLane4.val[0]);
int32x4_t vSum_1_3 = vaddq_s32(vLane1.val[0], vLane3.val[0]);
vSum_0_4 = vmlaq_s32(vSum_0_4, vLane2.val[0], vc6s32);
vSum_0_4 = vmlaq_s32(vSum_0_4, vSum_1_3, vc4s32);
int16x4_t vRes = vrshrn_n_s32(vSum_0_4, 8);
vst1_s16(dst + x, vRes);
#endif
}
break;
case 3:
{
int32x4_t v0 = vld1q_s32(lane - 2*3);
int32x4_t v1 = vld1q_s32(lane - 1*3);
int32x4_t v2 = vld1q_s32(lane + 0*3);
for (; x < roiw4; x += 3)
{
internal::prefetch(lane + 2 * x);
int32x4_t v3 = vld1q_s32(lane + 2 * x + 1*3);
int32x4_t v4 = vld1q_s32(lane + 2 * x + 2*3);
int32x4_t v = vaddq_s32(v0, v4);
int32x4_t v13 = vaddq_s32(v1, v3);
v = vmlaq_s32(v, v2, vc6s32);
v = vmlaq_s32(v, v13, vc4s32);
int16x4_t vv = vrshrn_n_s32(v, 8);
v0 = v2;
v1 = v3;
v2 = v4;
vst1_s16(dst + x, vv);
}
}
break;
case 4:
{
int32x4_t v0 = vld1q_s32(lane - 2*4);
int32x4_t v1 = vld1q_s32(lane - 1*4);
int32x4_t v2 = vld1q_s32(lane + 0*4);
for (; x < roiw4; x += 4)
{
internal::prefetch(lane + 2 * x + 8);
int32x4_t v3 = vld1q_s32(lane + 2 * x + 1*4);
int32x4_t v4 = vld1q_s32(lane + 2 * x + 2*4);
int32x4_t v = vaddq_s32(v0, v4);
int32x4_t v13 = vaddq_s32(v1, v3);
v = vmlaq_s32(v, v2, vc6s32);
v = vmlaq_s32(v, v13, vc4s32);
int16x4_t vv = vrshrn_n_s32(v, 8);
v0 = v2;
v1 = v3;
v2 = v4;
vst1_s16(dst + x, vv);
}
}
break;
}
for (u32 h = 0; h < cn; ++h)
{
s32* ln = lane + h;
s16* dt = dst + h;
for (size_t k = x; k < dcolcn; k += cn)
dt[k] = s16((ln[2*k-2*cn] + ln[2*k+2*cn] + 4 * (ln[2*k-cn] + ln[2*k+cn]) + 6 * ln[2*k] + (1 << 7)) >> 8);
}
}
#else
// Remove 'unused parameter' warnings.
(void)srcBase;
(void)srcStride;
(void)dstBase;
(void)dstStride;
#endif
}
void gaussianPyramidDown(const Size2D &srcSize,
const f32 *srcBase, ptrdiff_t srcStride,
const Size2D &dstSize,
f32 *dstBase, ptrdiff_t dstStride, u8 cn)
{
internal::assertSupportedConfiguration(isGaussianPyramidDownF32Supported(srcSize, dstSize, cn));
#ifdef CAROTENE_NEON
size_t dcolcn = dstSize.width*cn;
size_t scolcn = srcSize.width*cn;
size_t roiw4 = dcolcn - 3;
size_t idx_l1 = borderInterpolate101(-1, srcSize.width) * cn;
size_t idx_l2 = borderInterpolate101(-2, srcSize.width) * cn;
size_t idx_r1 = borderInterpolate101(srcSize.width + 0, srcSize.width) * cn;
size_t idx_r2 = borderInterpolate101(srcSize.width + 1, srcSize.width) * cn;
//1-line buffer
std::vector<f32> _buf(cn*(srcSize.width + 4) + 32/sizeof(f32));
f32* lane = internal::alignPtr(&_buf[2*cn], 32);
#if !defined(__aarch64__) && defined(__GNUC__) && __GNUC__ == 4 && __GNUC_MINOR__ < 7 && !defined(__clang__)
register float32x4_t vc6d4f32 asm ("q11") = vmovq_n_f32(1.5f); // 6/4
register float32x4_t vc1d4f32 asm ("q12") = vmovq_n_f32(0.25f); // 1/4
register float32x4_t vc1d64f32 asm ("q13") = vmovq_n_f32(0.015625f); //1/4/16
register float32x4_t vc4d64f32 asm ("q14") = vmovq_n_f32(0.0625f); //4/4/16
register float32x4_t vc6d64f32 asm ("q15") = vmovq_n_f32(0.09375f); //6/4/16
#else
float32x4_t vc6d4f32 = vmovq_n_f32(1.5f); // 6/4
float32x4_t vc1d4f32 = vmovq_n_f32(0.25f); // 1/4
float32x4_t vc1d64f32 = vmovq_n_f32(0.015625f); //1/4/16
float32x4_t vc4d64f32 = vmovq_n_f32(0.0625f); //4/4/16
float32x4_t vc6d64f32 = vmovq_n_f32(0.09375f); //6/4/16
#endif
for (size_t i = 0; i < dstSize.height; ++i)
{
f32* dst = internal::getRowPtr(dstBase, dstStride, i);
//vertical convolution
const f32* ln0 = internal::getRowPtr(srcBase, srcStride, borderInterpolate101(i*2-2, srcSize.height));
const f32* ln1 = internal::getRowPtr(srcBase, srcStride, borderInterpolate101(i*2-1, srcSize.height));
const f32* ln2 = internal::getRowPtr(srcBase, srcStride, borderInterpolate101(i*2+0, srcSize.height));
const f32* ln3 = internal::getRowPtr(srcBase, srcStride, borderInterpolate101(i*2+1, srcSize.height));
const f32* ln4 = internal::getRowPtr(srcBase, srcStride, borderInterpolate101(i*2+2, srcSize.height));
size_t x = 0;
for (; x <= scolcn - 4; x += 4)
{
internal::prefetch(internal::getRowPtr(ln2 + x, srcStride, (ptrdiff_t)x % 5 - 2));
float32x4_t v0 = vld1q_f32((const float32_t*)ln0 + x);
float32x4_t v1 = vld1q_f32((const float32_t*)ln1 + x);
float32x4_t v2 = vld1q_f32((const float32_t*)ln2 + x);
float32x4_t v3 = vld1q_f32((const float32_t*)ln3 + x);
float32x4_t v4 = vld1q_f32((const float32_t*)ln4 + x);
float32x4_t v = vaddq_f32(v1, v3);
float32x4_t v04 = vaddq_f32(v0, v4);
v = vmlaq_f32(v, v2, vc6d4f32);
v = vmlaq_f32(v, v04, vc1d4f32);
vst1q_f32(lane + x, v);
}
for (; x < scolcn; ++x)
{
lane[x] = 0.25f*(ln0[x] + ln4[x]) + (ln1[x] + ln3[x]) + 1.5f * ln2[x];
}
//left&right borders
for (u32 k = 0; k < cn; ++k)
{
lane[(s32)(-cn+k)] = lane[idx_l1 + k];
lane[(s32)(-cn-cn+k)] = lane[idx_l2 + k];
lane[scolcn+k] = lane[idx_r1 + k];
lane[scolcn+cn+k] = lane[idx_r2 + k];
}
//horizontal convolution
x = 0;
switch(cn)
{
case 1:
for (; x < roiw4; x += 4)
{
internal::prefetch(lane + 2 * x);
#if !defined(__aarch64__) && defined(__GNUC__) && __GNUC__ == 4 && __GNUC_MINOR__ < 7 && !defined(__clang__)
__asm__ __volatile__ (
"vld2.32 {d0-d3}, [%[in0]] \n\t"
"vld2.32 {d8-d11}, [%[in4]] \n\t"
"vld2.32 {d14-d17}, [%[in2],:256] \n\t"
"vld2.32 {d10-d13}, [%[in1]] \n\t"
"vld2.32 {d16-d19}, [%[in3]] \n\t"
"vmul.f32 q7, %q[c6d64] \n\t"
"vadd.f32 q0, q4 @v04 \n\t"
"vadd.f32 q5, q8 @v13 \n\t"
"vmla.f32 q7, q0, %q[c1d64] \n\t"
"vmla.f32 q7, q5, %q[c4d64] \n\t"
"vst1.32 {d14-d15}, [%[out]] \n\t"
:
: [out] "r" (dst + x),
[in0] "r" (lane + 2*x-2),
[in1] "r" (lane + 2*x-1),
[in2] "r" (lane + 2*x+0),
[in3] "r" (lane + 2*x+1),
[in4] "r" (lane + 2*x+2),
[c4d64] "w" (vc4d64f32), [c6d64] "w" (vc6d64f32), [c1d64] "w" (vc1d64f32)
: "d0","d1","d2","d3","d4",/*"d5","d6","d7",*/"d8","d9","d10","d11","d12","d13","d14","d15","d16","d17","d18","d19" //ugly compiler "bug" - can't touch d5-d7
);
#else
float32x4x2_t vLane0 = vld2q_f32(lane + 2*x-2);
float32x4x2_t vLane1 = vld2q_f32(lane + 2*x-1);
float32x4x2_t vLane2 = vld2q_f32(lane + 2*x+0);
float32x4x2_t vLane3 = vld2q_f32(lane + 2*x+1);
float32x4x2_t vLane4 = vld2q_f32(lane + 2*x+2);
float32x4_t vSum_0_4 = vaddq_f32(vLane0.val[0], vLane4.val[0]);
float32x4_t vSum_1_3 = vaddq_f32(vLane1.val[0], vLane3.val[0]);
float32x4_t vRes = vmulq_f32(vLane2.val[0], vc6d64f32);
vRes = vmlaq_f32(vRes, vSum_0_4, vc1d64f32);
vRes = vmlaq_f32(vRes, vSum_1_3, vc4d64f32);
vst1q_f32(dst + x, vRes);
#endif
}
break;
case 3:
{
float32x4_t v0 = vld1q_f32((const float32_t*)lane - 2*3);
float32x4_t v1 = vld1q_f32((const float32_t*)lane - 1*3);
float32x4_t v2 = vld1q_f32((const float32_t*)lane + 0*3);
for (; x < roiw4; x += 3)
{
internal::prefetch(lane + 2 * x);
float32x4_t v3 = vld1q_f32((const float32_t*)lane + 2 * x + 1*3);
float32x4_t v4 = vld1q_f32((const float32_t*)lane + 2 * x + 2*3);
float32x4_t v04 = vaddq_f32(v0, v4);
float32x4_t v13 = vaddq_f32(v1, v3);
float32x4_t v = vmulq_f32(v2, vc6d64f32);
v = vmlaq_f32(v, v04, vc1d64f32);
v = vmlaq_f32(v, v13, vc4d64f32);
v0 = v2;
v1 = v3;
v2 = v4;
vst1q_f32(dst + x, v);
}
}
break;
case 4:
{
float32x4_t v0 = vld1q_f32((const float32_t*)lane - 2*4);
float32x4_t v1 = vld1q_f32((const float32_t*)lane - 1*4);
float32x4_t v2 = vld1q_f32((const float32_t*)lane + 0*4);
for (; x < roiw4; x += 4)
{
internal::prefetch(lane + 2 * x + 8);
float32x4_t v3 = vld1q_f32((const float32_t*)lane + 2 * x + 1*4);
float32x4_t v4 = vld1q_f32((const float32_t*)lane + 2 * x + 2*4);
float32x4_t v04 = vaddq_f32(v0, v4);
float32x4_t v13 = vaddq_f32(v1, v3);
float32x4_t v = vmulq_f32(v2, vc6d64f32);
v = vmlaq_f32(v, v04, vc1d64f32);
v = vmlaq_f32(v, v13, vc4d64f32);
v0 = v2;
v1 = v3;
v2 = v4;
vst1q_f32(dst + x, v);
}
}
break;
}
for (u32 h = 0; h < cn; ++h)
{
f32* ln = lane + h;
f32* dt = dst + h;
for (size_t k = x; k < dcolcn; k += cn)
dt[k] = 0.015625f * (ln[2*k-2*cn] + ln[2*k+2*cn]) + 0.0625f * (ln[2*k-cn] + ln[2*k+cn]) + 0.09375f * ln[2*k];
}
}
#else
// Remove 'unused parameter' warnings.
(void)srcBase;
(void)srcStride;
(void)dstBase;
(void)dstStride;
#endif
}
void gaussianPyramidUp(const Size2D &srcSize,
const u8 *srcBase, ptrdiff_t srcStride,
const Size2D &dstSize,
u8 *dstBase, ptrdiff_t dstStride, u8 cn)
{
internal::assertSupportedConfiguration(isGaussianPyramidUpU8Supported(srcSize, dstSize, cn));
#ifdef CAROTENE_NEON
size_t dcolshn = (dstSize.width/2) * cn;
size_t dcolshw = ((dstSize.width+1)/2) * cn;
size_t scolsn = srcSize.width*cn;
size_t idx_l = (borderInterpolate101(-2, 2 * srcSize.width)/2) * cn;
size_t idx_r1 = (borderInterpolate101(2 * srcSize.width + 0, 2 * srcSize.width)/2) * cn;
size_t idx_r2 = (borderInterpolate101(2 * srcSize.width + 2, 2 * srcSize.width + 2)/2) * cn;
//2-lines buffer
std::vector<u16> _buf(2*(cn*(srcSize.width + 3) + 32/sizeof(u16)));
u16* lane0 = internal::alignPtr(&_buf[cn], 32);
u16* lane1 = internal::alignPtr(lane0 + (3 + srcSize.width)*cn, 32);
uint8x8_t vc6u8 = vmov_n_u8(6);
uint16x8_t vc6u16 = vmovq_n_u16(6);
for (size_t i = 0; i < (dstSize.height + 1)/2; ++i)
{
u8* dst = internal::getRowPtr(dstBase, dstStride, 2*i);
//vertical convolution
const u8* ln0 = internal::getRowPtr(srcBase, srcStride, borderInterpolate101(i * 2 - 2, srcSize.height * 2)/2);
const u8* ln1 = internal::getRowPtr(srcBase, srcStride, borderInterpolate101(i * 2 + 0, srcSize.height * 2)/2);
const u8* ln2 = internal::getRowPtr(srcBase, srcStride, borderInterpolate101(i * 2 + 2, srcSize.height * 2)/2);
size_t x = 0;
for (; x <= scolsn - 8; x += 8)
{
internal::prefetch(internal::getRowPtr(ln1 + x, srcStride, (ptrdiff_t)x % 3 - 1));
uint8x8_t v0 = vld1_u8(ln0+x);
uint8x8_t v2 = vld1_u8(ln2+x);
uint8x8_t v1 = vld1_u8(ln1+x);
uint16x8_t vl0 = vaddl_u8(v0, v2);
uint16x8_t vl1 = vaddl_u8(v1, v2);
vl0 = vmlal_u8(vl0, v1, vc6u8);
vl1 = vshlq_n_u16(vl1, 2);
vst1q_u16(lane0 + x, vl0);
vst1q_u16(lane1 + x, vl1);
}
for (; x < scolsn; ++x)
{
lane0[x] = ln0[x] + ln2[x] + 6u * ln1[x];
lane1[x] = 4u * (ln1[x] + ln2[x]);
}
//left&right borders
for (u32 k = 0; k < cn; ++k)
{
lane0[(s32)(-cn+k)] = lane0[idx_l + k];
lane1[(s32)(-cn+k)] = lane1[idx_l + k];
lane0[scolsn+k] = lane0[idx_r1 + k];
lane0[scolsn+cn+k] = lane0[idx_r2 + k];
lane1[scolsn+k] = lane1[idx_r1 + k];
lane1[scolsn+cn+k] = lane1[idx_r2 + k];
}
//horizontal convolution
const u16* lane = lane0;
pyrUp8uHorizontalConvolution:
x = 0;
size_t lim;
switch(cn)
{
case 1:
lim = dcolshn > 7 ? dcolshn - 7 : 0;
for (; x < lim; x += 8)
{
internal::prefetch(lane + x);
#if !defined(__aarch64__) && defined(__GNUC__) && defined(__arm__)
__asm__ (
"vld1.16 {d0-d1}, [%[in0]] /*q0 = v0*/ \n\t"
"vld1.16 {d2-d3}, [%[in2]] /*q1 = v2*/ \n\t"
"vld1.16 {d4-d5}, [%[in1],:128] /*q2 = v1*/ \n\t"
"vadd.i16 q0, q1 /*q0 = v0 + v2*/ \n\t"
"vadd.i16 q3, q1, q2 /*q3 = v1 + v2*/ \n\t"
"vmla.i16 q0, q2, %q[c6] /*q0 += v1*6*/ \n\t"
"vrshrn.u16 d9, q3, #4 \n\t"
"vrshrn.u16 d8, q0, #6 \n\t"
"vst2.8 {d8-d9}, [%[out]] \n\t"
: /*no output*/
: [out] "r" (dst + x*2),
[in0] "r" (lane + x - 1),
[in1] "r" (lane + x + 0),
[in2] "r" (lane + x + 1),
[c6] "w" (vc6u16)
: "d0","d1","d2","d3","d4","d5","d6","d7","d8","d9"
);
#else
uint16x8_t vLane0 = vld1q_u16(lane + x - 1);
uint16x8_t vLane1 = vld1q_u16(lane + x + 0);
uint16x8_t vLane2 = vld1q_u16(lane + x + 1);
vLane0 = vaddq_u16(vLane0, vLane2);
vLane2 = vaddq_u16(vLane2, vLane1);
vLane0 = vmlaq_u16(vLane0, vLane1, vc6u16);
uint8x8x2_t vRes;
vRes.val[0] = vrshrn_n_u16(vLane0, 6);
vRes.val[1] = vrshrn_n_u16(vLane2, 4);
vst2_u8(dst + x*2, vRes);
#endif
}
break;
case 3:
{
lim = dcolshn > 23 ? dcolshn - 23 : 0;
for (; x < lim; x += 24)
{
internal::prefetch(lane + x);
#if !defined(__aarch64__) && defined(__GNUC__) && defined(__arm__)
__asm__ (
"vmov.u16 q9, #6 \n\t"
"vld3.16 {d0, d2, d4}, [%[in0]] /*v0*/ \n\t"
"vld3.16 {d1, d3, d5}, [%[in02]] \n\t"
"vld3.16 {d6, d8, d10}, [%[in2]] /*v2*/ \n\t"
"vld3.16 {d7, d9, d11}, [%[in22]] \n\t"
"vld3.16 {d12, d14, d16}, [%[in1]] /*v1*/ \n\t"
"vld3.16 {d13, d15, d17}, [%[in12]] \n\t"
"vadd.i16 q0, q3 /*v0 + v2*/ \n\t"
"vadd.i16 q1, q4 /*v0 + v2*/ \n\t"
"vadd.i16 q2, q5 /*v0 + v2*/ \n\t"
"vadd.i16 q3, q6 /*v1 + v2*/ \n\t"
"vadd.i16 q4, q7 /*v1 + v2*/ \n\t"
"vadd.i16 q5, q8 /*v1 + v2*/ \n\t"
"vmla.i16 q0, q6, q9 /*v0 + v2 + v1*6 */ \n\t"
"vmla.i16 q1, q7, q9 /*v0 + v2 + v1*6 */ \n\t"
"vmla.i16 q2, q8, q9 /*v0 + v2 + v1*6 */ \n\t"
"vrshrn.u16 d19, q3, #4 \n\t"
"vrshrn.u16 d21, q4, #4 \n\t"
"vrshrn.u16 d23, q5, #4 \n\t"
"vrshrn.u16 d18, q0, #6 \n\t"
"vrshrn.u16 d20, q1, #6 \n\t"
"vrshrn.u16 d22, q2, #6 \n\t"
"vzip.8 d18, d19 \n\t"
"vzip.8 d20, d21 \n\t"
"vzip.8 d22, d23 \n\t"
"vst3.8 {d18, d20, d22}, [%[out1]] \n\t"
"vst3.8 {d19, d21, d23}, [%[out2]] \n\t"
: /*no output*/
: [out1] "r" (dst + 2 * x),
[out2] "r" (dst + 2 * x + 24),
[in0] "r" (lane + x - 3),
[in02] "r" (lane + x + 9),
[in1] "r" (lane + x),
[in12] "r" (lane + x + 12),
[in2] "r" (lane + x + 3),
[in22] "r" (lane + x + 15)
: "d0","d1","d2","d3","d4","d5","d6","d7","d8","d9","d10","d11","d12","d13","d14","d15","d16","d17","d18","d19","d20","d21","d22","d23"
);
#else
uint16x8_t vc6 = vmovq_n_u16(6);
uint16x8x3_t vLane0 = vld3q_u16(lane + x - 3);
uint16x8x3_t vLane1 = vld3q_u16(lane + x + 0);
uint16x8x3_t vLane2 = vld3q_u16(lane + x + 3);
uint16x8_t vSum_0_3 = vaddq_u16(vLane0.val[0], vLane2.val[0]);
uint16x8_t vSum_1_4 = vaddq_u16(vLane0.val[1], vLane2.val[1]);
uint16x8_t vSum_2_5 = vaddq_u16(vLane0.val[2], vLane2.val[2]);
uint16x8_t vSum_3_6 = vaddq_u16(vLane2.val[0], vLane1.val[0]);
uint16x8_t vSum_4_7 = vaddq_u16(vLane2.val[1], vLane1.val[1]);
uint16x8_t vSum_5_8 = vaddq_u16(vLane2.val[2], vLane1.val[2]);
vSum_0_3 = vmlaq_u16(vSum_0_3, vLane1.val[0], vc6);
vSum_1_4 = vmlaq_u16(vSum_1_4, vLane1.val[1], vc6);
vSum_2_5 = vmlaq_u16(vSum_2_5, vLane1.val[2], vc6);
uint8x8x2_t vSumShr3;
vSumShr3.val[0] = vrshrn_n_u16(vSum_3_6, 4);
vSumShr3.val[1] = vrshrn_n_u16(vSum_0_3, 6);;
uint8x8x2_t vSumShr4;
vSumShr4.val[0] = vrshrn_n_u16(vSum_4_7, 4);
vSumShr4.val[1] = vrshrn_n_u16(vSum_1_4, 6);
uint8x8x2_t vSumShr5;
vSumShr5.val[0] = vrshrn_n_u16(vSum_5_8, 4);
vSumShr5.val[1] = vrshrn_n_u16(vSum_2_5, 6);
vSumShr3 = vzip_u8(vSumShr3.val[1], vSumShr3.val[0]);
vSumShr4 = vzip_u8(vSumShr4.val[1], vSumShr4.val[0]);
vSumShr5 = vzip_u8(vSumShr5.val[1], vSumShr5.val[0]);
uint8x8x3_t vRes1;
vRes1.val[0] = vSumShr3.val[0];
vRes1.val[1] = vSumShr4.val[0];
vRes1.val[2] = vSumShr5.val[0];
vst3_u8(dst + 2 * x, vRes1);
uint8x8x3_t vRes2;
vRes2.val[0] = vSumShr3.val[1];
vRes2.val[1] = vSumShr4.val[1];
vRes2.val[2] = vSumShr5.val[1];
vst3_u8(dst + 2 * x + 24, vRes2);
#endif
}
}
break;
case 4:
lim = dcolshn > 7 ? dcolshn - 7 : 0;
for (; x < lim; x += 8)
{
internal::prefetch(lane + x);
#if !defined(__aarch64__) && defined(__GNUC__) && defined(__arm__)
__asm__ (
"vld1.16 {d0-d1}, [%[in0]] /*q0 = v0*/ \n\t"
"vld1.16 {d2-d3}, [%[in2]] /*q1 = v2*/ \n\t"
"vld1.16 {d4-d5}, [%[in1],:128] /*q2 = v1*/ \n\t"
"vadd.i16 q0, q1 /*q0 = v0 + v2*/ \n\t"
"vadd.i16 q3, q1, q2 /*q3 = v1 + v2*/ \n\t"
"vmla.i16 q0, q2, %q[c6] /*q0 += v1*6*/ \n\t"
"vrshrn.u16 d9, q3, #4 \n\t"
"vrshrn.u16 d8, q0, #6 \n\t"
"vst2.32 {d8-d9}, [%[out]] \n\t"
: /*no output*/
: [out] "r" (dst + x*2),
[in0] "r" (lane + x-4),
[in1] "r" (lane + x),
[in2] "r" (lane + x+4),
[c6] "w" (vc6u16)
: "d0","d1","d2","d3","d4","d5","d6","d7","d8","d9"
);
#else
uint16x8_t vLane0 = vld1q_u16(lane + x-4);
uint16x8_t vLane1 = vld1q_u16(lane + x+0);
uint16x8_t vLane2 = vld1q_u16(lane + x+4);
vLane0 = vaddq_u16(vLane0, vLane2);
vLane2 = vaddq_u16(vLane2, vLane1);
vLane0 = vmlaq_u16(vLane0, vLane1, vc6u16);
uint32x2x2_t vRes;
vRes.val[1] = vreinterpret_u32_u8(vrshrn_n_u16(vLane2, 4));
vRes.val[0] = vreinterpret_u32_u8(vrshrn_n_u16(vLane0, 6));
vst2_u32((uint32_t*)(dst + x*2), vRes);
#endif
}
break;
};
for (u32 h = 0; h < cn; ++h)
{
const u16* ln = lane + h;
u8* dt = dst + h;
size_t k = x;
for (; k < dcolshn; k += cn)
{
dt[2*k+0] = u8((ln[(ptrdiff_t)(k-cn)] + ln[k+cn] + 6u * ln[k] + (1 << 5)) >> 6);
dt[2*k+cn] = u8(((ln[k] + ln[k+cn]) * 4u + (1 << 5)) >> 6);
}
for (; k < dcolshw; k += cn)
dt[2*k] = u8((ln[(ptrdiff_t)(k-cn)] + ln[k+cn] + 6u * ln[k] + (1 << 5)) >> 6);
}
dst = internal::getRowPtr(dstBase, dstStride, 2*i+1);
//second row
if (lane == lane0 && 2*i+1 < dstSize.height)
{
lane = lane1;
goto pyrUp8uHorizontalConvolution;
}
}
#else
// Remove 'unused parameter' warnings.
(void)srcBase;
(void)srcStride;
(void)dstBase;
(void)dstStride;
#endif
}
void gaussianPyramidUp(const Size2D &srcSize,
const s16 *srcBase, ptrdiff_t srcStride,
const Size2D &dstSize,
s16 *dstBase, ptrdiff_t dstStride, u8 cn)
{
internal::assertSupportedConfiguration(isGaussianPyramidUpS16Supported(srcSize, dstSize, cn));
#ifdef CAROTENE_NEON
size_t dcolshn = (dstSize.width/2) * cn;
size_t dcolshw = ((dstSize.width+1)/2) * cn;
size_t scolsn = srcSize.width*cn;
size_t idx_l = (borderInterpolate101(-2, 2 * srcSize.width)/2) * cn;
size_t idx_r1 = (borderInterpolate101(2 * srcSize.width + 0, 2 * srcSize.width)/2) * cn;
size_t idx_r2 = (borderInterpolate101(2 * srcSize.width + 2, 2 * srcSize.width + 2)/2) * cn;
//2-lines buffer
std::vector<s32> _buf(2*(cn*(srcSize.width + 3) + 32/sizeof(s32)));
s32* lane0 = internal::alignPtr(&_buf[cn], 32);
s32* lane1 = internal::alignPtr(lane0 + (3 + srcSize.width)*cn, 32);
int16x4_t vc6s16 = vmov_n_s16(6);
int32x4_t vc6s32 = vmovq_n_s32(6);
for (size_t i = 0; i < (dstSize.height + 1)/2; ++i)
{
s16* dst = internal::getRowPtr(dstBase, dstStride, 2*i);
//vertical convolution
const s16* ln0 = internal::getRowPtr(srcBase, srcStride, borderInterpolate101(i * 2 - 2, srcSize.height * 2)/2);
const s16* ln1 = internal::getRowPtr(srcBase, srcStride, borderInterpolate101(i * 2 + 0, srcSize.height * 2)/2);
const s16* ln2 = internal::getRowPtr(srcBase, srcStride, borderInterpolate101(i * 2 + 2, srcSize.height * 2)/2);
size_t x = 0;
for (; x <= scolsn - 4; x += 4)
{
internal::prefetch(internal::getRowPtr(ln1 + x, srcStride, (ptrdiff_t)x % 3 - 1));
int16x4_t v0 = vld1_s16(ln0 + x);
int16x4_t v2 = vld1_s16(ln2 + x);
int16x4_t v1 = vld1_s16(ln1 + x);
int32x4_t vl0 = vaddl_s16(v0, v2);
int32x4_t vl1 = vaddl_s16(v1, v2);
vl0 = vmlal_s16(vl0, v1, vc6s16);
vl1 = vshlq_n_s32(vl1, 2);
vst1q_s32(lane0 + x, vl0);
vst1q_s32(lane1 + x, vl1);
}
for (; x < scolsn; ++x)
{
lane0[x] = ln0[x] + ln2[x] + 6 * ln1[x];
lane1[x] = 4 * (ln1[x] + ln2[x]);
}
//left&right borders
for (u32 k = 0; k < cn; ++k)
{
lane0[(s32)(-cn+k)] = lane0[idx_l + k];
lane1[(s32)(-cn+k)] = lane1[idx_l + k];
lane0[scolsn+k] = lane0[idx_r1 + k];
lane0[scolsn+cn+k] = lane0[idx_r2 + k];
lane1[scolsn+k] = lane1[idx_r1 + k];
lane1[scolsn+cn+k] = lane1[idx_r2 + k];
}
//horizontal convolution
const s32* lane = lane0;
pyrUp16sHorizontalConvolution:
x = 0;
size_t lim;
switch(cn)
{
case 1:
lim = dcolshn > 3 ? dcolshn - 3 : 0;
for (; x < lim; x += 4)
{
internal::prefetch(lane + x);
#if !defined(__aarch64__) && defined(__GNUC__) && defined(__arm__)
__asm__ (
"vld1.32 {d0-d1}, [%[in0]] /*q0 = v0*/ \n\t"
"vld1.32 {d2-d3}, [%[in2]] /*q1 = v2*/ \n\t"
"vld1.32 {d4-d5}, [%[in1],:128] /*q2 = v1*/ \n\t"
"vadd.i32 q0, q0, q1 /*q0 = v0 + v2*/ \n\t"
"vadd.i32 q3, q1, q2 /*q3 = v1 + v2*/ \n\t"
"vmla.i32 q0, q2, %q[c6] /*q0 += v1*6*/ \n\t"
"vrshrn.s32 d9, q3, #4 \n\t"
"vrshrn.s32 d8, q0, #6 \n\t"
"vst2.16 {d8-d9}, [%[out]] \n\t"
: /*no output*/
: [out] "r" (dst + x * 2),
[in0] "r" (lane + x - 1),
[in1] "r" (lane + x),
[in2] "r" (lane + x + 1),
[c6] "w" (vc6s32)
: "d0","d1","d2","d3","d4","d5","d6","d7","d8","d9"
);
#else
int32x4_t vLane0 = vld1q_s32(lane + x - 1);
int32x4_t vLane1 = vld1q_s32(lane + x);
int32x4_t vLane2 = vld1q_s32(lane + x + 1);
vLane0 = vaddq_s32(vLane0, vLane2);
vLane2 = vaddq_s32(vLane2, vLane1);
vLane0 = vmlaq_s32(vLane0, vLane1, vc6s32);
int16x4x2_t vRes;
vRes.val[0] = vrshrn_n_s32(vLane0, 6);
vRes.val[1] = vrshrn_n_s32(vLane2, 4);
vst2_s16(dst + x * 2, vRes);
#endif
}
break;
case 3:
{
lim = dcolshn > 11 ? dcolshn - 11 : 0;
for (; x < lim; x += 12)
{
internal::prefetch(lane + x + 3);
#if !defined(__aarch64__) && defined(__GNUC__) && defined(__arm__)
__asm__ (
"vmov.s32 q9, #6 \n\t"
"vld3.32 {d0, d2, d4}, [%[in0]] /*v0*/ \n\t"
"vld3.32 {d1, d3, d5}, [%[in2]] \n\t"
"vld3.32 {d6, d8, d10}, [%[in2]] /*v2*/ \n\t"
"vld3.32 {d7, d9, d11}, [%[in22]] \n\t"
"vld3.32 {d12, d14, d16}, [%[in1]] /*v1*/ \n\t"
"vld3.32 {d13, d15, d17}, [%[in12]] \n\t"
"vadd.i32 q0, q3 /*v0 + v2*/ \n\t"
"vadd.i32 q1, q4 /*v0 + v2*/ \n\t"
"vadd.i32 q2, q5 /*v0 + v2*/ \n\t"
"vadd.i32 q3, q6 /*v1 + v2*/ \n\t"
"vadd.i32 q4, q7 /*v1 + v2*/ \n\t"
"vadd.i32 q5, q8 /*v1 + v2*/ \n\t"
"vmla.i32 q0, q6, q9 /*v0 + v2 + v1*6 */ \n\t"
"vmla.i32 q1, q7, q9 /*v0 + v2 + v1*6 */ \n\t"
"vmla.i32 q2, q8, q9 /*v0 + v2 + v1*6 */ \n\t"
"vrshrn.s32 d19, q3, #4 \n\t"
"vrshrn.s32 d21, q4, #4 \n\t"
"vrshrn.s32 d23, q5, #4 \n\t"
"vrshrn.s32 d18, q0, #6 \n\t"
"vrshrn.s32 d20, q1, #6 \n\t"
"vrshrn.s32 d22, q2, #6 \n\t"
"vzip.16 d18, d19 \n\t"
"vzip.16 d20, d21 \n\t"
"vzip.16 d22, d23 \n\t"
"vst3.16 {d18, d20, d22}, [%[out1]] \n\t"
"vst3.16 {d19, d21, d23}, [%[out2]] \n\t"
: /*no output*/
: [out1] "r" (dst + 2*x),
[out2] "r" (dst + 2*x + 12),
[in0] "r" (lane + x - 3),
[in1] "r" (lane + x),
[in12] "r" (lane + x + 6),
[in2] "r" (lane + x + 3),
[in22] "r" (lane + x + 9)
: "d0","d1","d2","d3","d4","d5","d6","d7","d8","d9","d10","d11","d12","d13","d14","d15","d16","d17","d18","d19","d20","d21","d22","d23"
);
#else
int32x4_t vc6 = vmovq_n_s32(6);
int32x4x3_t vLane0 = vld3q_s32(lane + x - 3);
int32x4x3_t vLane1 = vld3q_s32(lane + x);
int32x4x3_t vLane2 = vld3q_s32(lane + x + 3);
int32x4_t vSum_0_3 = vaddq_s32(vLane0.val[0], vLane2.val[0]);
int32x4_t vSum_1_4 = vaddq_s32(vLane0.val[1], vLane2.val[1]);
int32x4_t vSum_2_5 = vaddq_s32(vLane0.val[2], vLane2.val[2]);
int32x4_t vSum_3_6 = vaddq_s32(vLane2.val[0], vLane1.val[0]);
int32x4_t vSum_4_7 = vaddq_s32(vLane2.val[1], vLane1.val[1]);
int32x4_t vSum_5_8 = vaddq_s32(vLane2.val[2], vLane1.val[2]);
vSum_0_3 = vmlaq_s32(vSum_0_3, vLane1.val[0], vc6);
vSum_1_4 = vmlaq_s32(vSum_1_4, vLane1.val[1], vc6);
vSum_2_5 = vmlaq_s32(vSum_2_5, vLane1.val[2], vc6);
int16x4x2_t vSumShr1;
vSumShr1.val[1] = vrshrn_n_s32(vSum_3_6, 4);
vSumShr1.val[0] = vrshrn_n_s32(vSum_0_3, 6);
int16x4x2_t vSumShr2;
vSumShr2.val[1] = vrshrn_n_s32(vSum_4_7, 4);
vSumShr2.val[0] = vrshrn_n_s32(vSum_1_4, 6);
int16x4x2_t vSumShr3;
vSumShr3.val[1] = vrshrn_n_s32(vSum_5_8, 4);
vSumShr3.val[0] = vrshrn_n_s32(vSum_2_5, 6);
vSumShr1 = vzip_s16(vSumShr1.val[0], vSumShr1.val[1]);
vSumShr2 = vzip_s16(vSumShr2.val[0], vSumShr2.val[1]);
vSumShr3 = vzip_s16(vSumShr3.val[0], vSumShr3.val[1]);
int16x4x3_t vRes1;
vRes1.val[0] = vSumShr1.val[0];
vRes1.val[1] = vSumShr2.val[0];
vRes1.val[2] = vSumShr3.val[0];
vst3_s16((int16_t*)(dst + 2 * x), vRes1);
int16x4x3_t vRes2;
vRes2.val[0] = vSumShr1.val[1];
vRes2.val[1] = vSumShr2.val[1];
vRes2.val[2] = vSumShr3.val[1];
vst3_s16(dst + 2 * x + 12, vRes2);
#endif
}
}
break;
case 4:
lim = dcolshn > 3 ? dcolshn - 3 : 0;
for (; x < lim; x += 4)
{
internal::prefetch(lane + x);
#if !defined(__aarch64__) && defined(__GNUC__) && defined(__arm__)
__asm__ (
"vld1.32 {d0-d1}, [%[in0]] /*q0 = v0*/ \n\t"
"vld1.32 {d2-d3}, [%[in2]] /*q1 = v2*/ \n\t"
"vld1.32 {d4-d5}, [%[in1],:128] /*q2 = v1*/ \n\t"
"vadd.i32 q0, q1 /*q0 = v0 + v2*/ \n\t"
"vadd.i32 q3, q1, q2 /*q3 = v1 + v2*/ \n\t"
"vmla.i32 q0, q2, %q[c6] /*q0 += v1*6*/ \n\t"
"vrshrn.s32 d9, q3, #4 \n\t"
"vrshrn.s32 d8, q0, #6 \n\t"
"vst1.16 {d8-d9}, [%[out]] \n\t"
: /*no output*/
: [out] "r" (dst + x * 2),
[in0] "r" (lane + x - 4),
[in1] "r" (lane + x),
[in2] "r" (lane + x + 4),
[c6] "w" (vc6s32)
: "d0","d1","d2","d3","d4","d5","d6","d7","d8","d9"
);
#else
int32x4_t vLane0 = vld1q_s32(lane + x - 4);
int32x4_t vLane1 = vld1q_s32(lane + x);
int32x4_t vLane2 = vld1q_s32(lane + x + 4);
vLane0 = vaddq_s32(vLane0, vLane2);
vLane2 = vaddq_s32(vLane2, vLane1);
vLane0 = vmlaq_s32(vLane0, vLane1, vc6s32);
int16x4x2_t vRes;
vRes.val[0] = vrshrn_n_s32(vLane0, 6);
vRes.val[1] = vrshrn_n_s32(vLane2, 4);
vst1q_s16(dst + x * 2, vcombine_s16(vRes.val[0], vRes.val[1]));
#endif
}
break;
};
for (u32 h = 0; h < cn; ++h)
{
const s32* ln = lane + h;
s16* dt = dst + h;
size_t k = x;
for (; k < dcolshn; k += cn)
{
dt[2*k+0] = s16((ln[(ptrdiff_t)(k-cn)] + ln[k+cn] + 6 * ln[k] + (1 << 5)) >> 6);
dt[2*k+cn] = s16(((ln[k] + ln[k+cn]) * 4 + (1 << 5)) >> 6);
}
for (; k < dcolshw; k += cn)
dt[2*k] = s16((ln[(ptrdiff_t)(k-cn)] + ln[k+cn] + 6 * ln[k] + (1 << 5)) >> 6);
}
dst = internal::getRowPtr(dstBase, dstStride, 2*i+1);
//second row
if (lane == lane0 && 2*i+1 < dstSize.height)
{
lane = lane1;
goto pyrUp16sHorizontalConvolution;
}
}
#else
// Remove 'unused parameter' warnings.
(void)srcBase;
(void)srcStride;
(void)dstBase;
(void)dstStride;
#endif
}
} // namespace CAROTENE_NS
View Git Blame Copy Permalink